diff --git a/Documentation/core-api/floating-point.rst b/Documentation/core-api/floating-point.rst
new file mode 100644
index 000000000000..a8d0d4b05052
--- /dev/null
+++ b/Documentation/core-api/floating-point.rst
@@ -0,0 +1,78 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+Floating-point API
+==================
+
+Kernel code is normally prohibited from using floating-point (FP) registers or
+instructions, including the C float and double data types. This rule reduces
+system call overhead, because the kernel does not need to save and restore the
+userspace floating-point register state.
+
+However, occasionally drivers or library functions may need to include FP code.
+This is supported by isolating the functions containing FP code to a separate
+translation unit (a separate source file), and saving/restoring the FP register
+state around calls to those functions. This creates "critical sections" of
+floating-point usage.
+
+The reason for this isolation is to prevent the compiler from generating code
+touching the FP registers outside these critical sections. Compilers sometimes
+use FP registers to optimize inlined ``memcpy`` or variable assignment, as
+floating-point registers may be wider than general-purpose registers.
+
+Usability of floating-point code within the kernel is architecture-specific.
+Additionally, because a single kernel may be configured to support platforms
+both with and without a floating-point unit, FPU availability must be checked
+both at build time and at run time.
+
+Several architectures implement the generic kernel floating-point API from
+``linux/fpu.h``, as described below. Some other architectures implement their
+own unique APIs, which are documented separately.
+
+Build-time API
+--------------
+
+Floating-point code may be built if the option ``ARCH_HAS_KERNEL_FPU_SUPPORT``
+is enabled. For C code, such code must be placed in a separate file, and that
+file must have its compilation flags adjusted using the following pattern::
+
+    CFLAGS_foo.o += $(CC_FLAGS_FPU)
+    CFLAGS_REMOVE_foo.o += $(CC_FLAGS_NO_FPU)
+
+Architectures are expected to define one or both of these variables in their
+top-level Makefile as needed. For example::
+
+    CC_FLAGS_FPU := -mhard-float
+
+or::
+
+    CC_FLAGS_NO_FPU := -msoft-float
+
+Normal kernel code is assumed to use the equivalent of ``CC_FLAGS_NO_FPU``.
+
+Runtime API
+-----------
+
+The runtime API is provided in ``linux/fpu.h``. This header cannot be included
+from files implementing FP code (those with their compilation flags adjusted as
+above). Instead, it must be included when defining the FP critical sections.
+
+.. c:function:: bool kernel_fpu_available( void )
+
+        This function reports if floating-point code can be used on this CPU or
+        platform. The value returned by this function is not expected to change
+        at runtime, so it only needs to be called once, not before every
+        critical section.
+
+.. c:function:: void kernel_fpu_begin( void )
+                void kernel_fpu_end( void )
+
+        These functions create a floating-point critical section. It is only
+        valid to call ``kernel_fpu_begin()`` after a previous call to
+        ``kernel_fpu_available()`` returned ``true``. These functions are only
+        guaranteed to be callable from (preemptible or non-preemptible) process
+        context.
+
+        Preemption may be disabled inside critical sections, so their size
+        should be minimized. They are *not* required to be reentrant. If the
+        caller expects to nest critical sections, it must implement its own
+        reference counting.
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index 7a3a08d81f11..974beccd671f 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -48,6 +48,7 @@ Library functionality that is used throughout the kernel.
    errseq
    wrappers/atomic_t
    wrappers/atomic_bitops
+   floating-point
 
 Low level entry and exit
 ========================
diff --git a/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml b/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml
index 8a6a78e1a7ab..e7bdb334476c 100644
--- a/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml
+++ b/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml
@@ -79,6 +79,11 @@ properties:
       The reference to the reserved-memory for the L2 Loosely Integrated Memory region.
       The reserved memory node should be defined as per the bindings in reserved-memory.txt.
 
+  sifive,perfmon-counters:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 0
+    description: Number of PMU counter registers
+
 allOf:
   - $ref: /schemas/cache-controller.yaml#
 
diff --git a/Documentation/devicetree/bindings/cache/sifive,pl2cache0.yaml b/Documentation/devicetree/bindings/cache/sifive,pl2cache0.yaml
new file mode 100644
index 000000000000..d89e2e5d0a97
--- /dev/null
+++ b/Documentation/devicetree/bindings/cache/sifive,pl2cache0.yaml
@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+# Copyright (C) 2023-2024 SiFive, Inc.
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/cache/sifive,pl2cache0.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: SiFive Private L2 Cache Controller
+
+maintainers:
+  - Eric Lin <eric.lin@sifive.com>
+
+description:
+  The SiFive Private L2 Cache Controller is a per-core cache which communicates
+  with both the upstream L1 caches and downstream L3 cache or memory, enabling a
+  high-performance cache subsystem.
+
+allOf:
+  - $ref: /schemas/cache-controller.yaml#
+
+select:
+  properties:
+    compatible:
+      contains:
+        enum:
+          - sifive,pl2cache1
+
+  required:
+    - compatible
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - const: sifive,pl2cache0
+          - const: sifive,pl2cache1
+          - const: cache
+      - items:
+          - const: sifive,pl2cache1
+          - const: cache
+
+  reg:
+    maxItems: 1
+
+  cache-block-size: true
+  cache-level: true
+  cache-sets: true
+  cache-size: true
+  cache-unified: true
+
+  next-level-cache: true
+
+  sifive,perfmon-counters:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 0
+    description: Number of PMU counter registers
+
+required:
+  - compatible
+  - reg
+  - cache-block-size
+  - cache-level
+  - cache-sets
+  - cache-size
+  - cache-unified
+
+additionalProperties: false
+
+examples:
+  - |
+    cache-controller@10104000 {
+        compatible = "sifive,pl2cache1", "cache";
+        reg = <0x10104000 0x4000>;
+        cache-block-size = <64>;
+        cache-level = <2>;
+        cache-sets = <512>;
+        cache-size = <262144>;
+        cache-unified;
+        next-level-cache = <&L4>;
+        sifive,perfmon-counters = <6>;
+    };
diff --git a/Documentation/devicetree/bindings/riscv/physical-memory.yaml b/Documentation/devicetree/bindings/riscv/physical-memory.yaml
new file mode 100644
index 000000000000..deb49b34672f
--- /dev/null
+++ b/Documentation/devicetree/bindings/riscv/physical-memory.yaml
@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/riscv/physical-memory.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: RISC-V Physical Memory Regions
+
+maintainers:
+  - Samuel Holland <samuel.holland@sifive.com>
+
+description:
+  The RISC-V Privileged Architecture defines a number of Physical Memory
+  Attributes (PMAs) which apply to a given region of memory. These include the
+  types of accesses (read, write, execute, LR/SC, and/or AMO) allowed within
+  a region, the supported access widths and alignments, the cacheability and
+  coherence of the region, and whether or not accesses to the region may have
+  side effects.
+
+  Some RISC-V platforms provide multiple physical address mappings for main
+  memory or certain peripherals. Each alias of a region generally has different
+  PMAs (e.g. cacheable vs non-cacheable), which allows software to dynamically
+  select the PMAs for an access by referencing the corresponding alias.
+
+  The RISC-V Supervisor Domains specification defines a platform-specific
+  Physical Address Width (PAW), which describes the largest physical address
+  supported by a platform. Any access to an address >= 2^PAW is guaranteed to
+  raise an access fault, and therefore metadata (e.g. Memory Protection Tables)
+  need not be maintained for those addresses.
+
+  On DT-based RISC-V platforms, all of this information is provided by the
+  riscv,physical-memory-regions property of the root node.
+
+properties:
+  $nodename:
+    const: '/'
+
+  riscv,physical-memory-regions:
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    description:
+      A table of physical memory regions. The first entry in the table must
+      cover the entire range of physical addresses supported by the platform
+      (i.e. 0 to 2^PAW-1) and provides the default PMAs for all addresses not
+      covered by another table entry. Remaining table entries provide PMAs for
+      more specific physical memory regions, which must be contained within the
+      range of entry 0, but which must not overlap with each other.
+    minItems: 1
+    maxItems: 256
+    items:
+      minItems: 4
+      maxItems: 6
+      additionalItems: true
+      items:
+        - description: CPU physical address (#address-cells)
+        - description: >
+            Size (#size-cells). For entry 0, if the size is zero, the size is
+            assumed to be 2^(32 * #size-cells).
+        - description: >
+            Flags describing the most restrictive PMAs for any address within
+            the region.
+
+            The least significant byte indicates the types of accesses allowed
+            for this region. Note that a memory region may support a type of
+            access (e.g. AMOs) even if the CPU does not.
+
+            The next byte describes the cacheability, coherence, idempotency,
+            and ordering PMAs for this region. It also includes a flag to
+            indicate that accesses to a region are unsafe and must be
+            prohibited by software (for example using PMPs or Smmpt).
+
+            The third byte is reserved for future PMAs.
+
+            The most significant byte is the index of the lowest-numbered entry
+            which this entry is an alias of, if any. Aliases need not be the
+            same size, for example if a smaller memory region repeats within a
+            larger alias.
+        - description: Reserved for describing future PMAs
+
+additionalProperties: true
+
+examples:
+  - |
+    #include <dt-bindings/riscv/physical-memory.h>
+
+    / {
+      compatible = "starfive,jh7100";
+      #address-cells = <2>;
+      #size-cells = <2>;
+      riscv,physical-memory-regions =
+          <0x00 0x00000000 0x40 0x00000000 (PMA_RW | PMA_IO) 0x0>,
+          <0x00 0x18000000 0x00 0x00020000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>,
+          <0x00 0x18080000 0x00 0x00020000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>,
+          <0x00 0x41000000 0x00 0x1f000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>,
+          <0x00 0x61000000 0x00 0x1f000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY | PMR_ALIAS(3)) 0x0>,
+          <0x00 0x80000000 0x08 0x00000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY) 0x0>,
+          <0x10 0x00000000 0x08 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY | PMR_ALIAS(5)) 0x0>,
+          <0x20 0x00000000 0x10 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>,
+          <0x30 0x00000000 0x10 0x00000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY | PMR_ALIAS(7)) 0x0>;
+    };
+
+...
diff --git a/Makefile b/Makefile
index 1391d545aee9..9a4b32f1c14e 100644
--- a/Makefile
+++ b/Makefile
@@ -980,6 +980,11 @@ KBUILD_CFLAGS	+= $(CC_FLAGS_CFI)
 export CC_FLAGS_CFI
 endif
 
+# Architectures can define flags to add/remove for floating-point support
+CC_FLAGS_FPU	+= -D_LINUX_FPU_COMPILATION_UNIT
+export CC_FLAGS_FPU
+export CC_FLAGS_NO_FPU
+
 ifneq ($(CONFIG_FUNCTION_ALIGNMENT),0)
 KBUILD_CFLAGS += -falign-functions=$(CONFIG_FUNCTION_ALIGNMENT)
 endif
diff --git a/arch/Kconfig b/arch/Kconfig
index 09603e0bc2cc..ae26a98428cd 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -1488,6 +1488,12 @@ config ARCH_HAS_NONLEAF_PMD_YOUNG
 	  address translations. Page table walkers that clear the accessed bit
 	  may use this capability to reduce their search space.
 
+config ARCH_HAS_KERNEL_FPU_SUPPORT
+	bool
+	help
+	  Architectures that select this option can run floating-point code in
+	  the kernel, as described in Documentation/core-api/floating-point.rst.
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 1304992232ad..06a4b08fceb8 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -28,6 +28,7 @@ config RISCV
 	select ARCH_HAS_GIGANTIC_PAGE
 	select ARCH_HAS_KCOV
 	select ARCH_HAS_MEMBARRIER_CALLBACKS
+	select ARCH_HAS_KERNEL_FPU_SUPPORT if 64BIT && FPU
 	select ARCH_HAS_MMIOWB
 	select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 	select ARCH_HAS_PMEM_API
@@ -37,6 +38,7 @@ config RISCV
 	select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
 	select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL
 	select ARCH_HAS_SYSCALL_WRAPPER
+	select ARCH_HAS_TEARDOWN_DMA_OPS if IOMMU_SUPPORT && SOC_SIFIVE_EIC7700
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
 	select ARCH_HAS_UBSAN_SANITIZE_ALL
 	select ARCH_HAS_VDSO_DATA
@@ -142,6 +144,7 @@ config RISCV
 	select HAVE_STACKPROTECTOR
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
+	select IOMMU_DMA if IOMMU_SUPPORT && SOC_SIFIVE_EIC7700
 	select IRQ_DOMAIN
 	select IRQ_FORCED_THREADING
 	select KASAN_VMALLOC if KASAN
@@ -490,6 +493,9 @@ config RISCV_ISA_SVPBMT
 
 	   The Svpbmt extension is only available on 64-bit cpus.
 
+	   This option also controls selection of memory type based on
+	   physical memory aliases.
+
 	   If you don't know what to do here, say Y.
 
 config TOOLCHAIN_HAS_V
diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata
index e2c731cfed8c..719c5f546945 100644
--- a/arch/riscv/Kconfig.errata
+++ b/arch/riscv/Kconfig.errata
@@ -67,6 +67,7 @@ config ERRATA_THEAD_PBMT
 	bool "Apply T-Head memory type errata"
 	depends on ERRATA_THEAD && 64BIT && MMU
 	select RISCV_ALTERNATIVE_EARLY
+	select RISCV_ISA_SVPBMT
 	default y
 	help
 	  This will apply the memory type errata to handle the non-standard
diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs
index 30fd6a512828..5c54e9936a79 100644
--- a/arch/riscv/Kconfig.socs
+++ b/arch/riscv/Kconfig.socs
@@ -16,6 +16,11 @@ config ARCH_RENESAS
 config ARCH_SIFIVE
 	def_bool SOC_SIFIVE
 
+config SOC_SIFIVE_EIC7700
+	bool "SiFive eic7700 SoC"
+	help
+	  This enables support for SiFive EIC7700 platform hardware.
+
 config SOC_SIFIVE
 	bool "SiFive SoCs"
 	select ERRATA_SIFIVE if !XIP_KERNEL
diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
index 4d06f3402674..e4d368abd8e3 100644
--- a/arch/riscv/Makefile
+++ b/arch/riscv/Makefile
@@ -77,6 +77,9 @@ KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64i
 
 KBUILD_AFLAGS += -march=$(riscv-march-y)
 
+# For C code built with floating-point support, exclude V but keep F and D.
+CC_FLAGS_FPU  := -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)([^v_]*)v?/\1\2/')
+
 KBUILD_CFLAGS += -mno-save-restore
 KBUILD_CFLAGS += -DCONFIG_PAGE_OFFSET=$(CONFIG_PAGE_OFFSET)
 
diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile
index f60a280abb15..a0889f1b1375 100644
--- a/arch/riscv/boot/dts/Makefile
+++ b/arch/riscv/boot/dts/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 subdir-y += allwinner
 subdir-y += canaan
+subdir-y += eswin
 subdir-y += microchip
 subdir-y += renesas
 subdir-y += sifive
diff --git a/arch/riscv/boot/dts/eswin/Makefile b/arch/riscv/boot/dts/eswin/Makefile
new file mode 100644
index 000000000000..f550122f0335
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+dtb-$(CONFIG_SOC_SIFIVE) += eic7700-hifive-premier-p550.dtb
+obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y))
diff --git a/arch/riscv/boot/dts/eswin/eic7700-arch.dtsi b/arch/riscv/boot/dts/eswin/eic7700-arch.dtsi
new file mode 100644
index 000000000000..a89388ed58bd
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-arch.dtsi
@@ -0,0 +1,272 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ */
+
+#include <dt-bindings/clock/eic7700-clock.h>
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	compatible = "eswin,eic7700";
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		cpu0: cpu@0 {
+			compatible = "sifive,p550", "riscv";
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_0>;
+			clock-frequency = <0>;
+			device_type = "cpu";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&l2_cache_0>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			reg = <0x0>;
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "zicsr",
+							"zifencei", "zba", "zbb", "sscofpmf";
+			sifive,buserror = <&bus_error0>;
+			tlb-split;
+			cpu0_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+		cpu1: cpu@1 {
+			compatible = "sifive,p550", "riscv";
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_1>;
+			clock-frequency = <0>;
+			device_type = "cpu";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&l2_cache_1>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			reg = <0x1>;
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "zicsr",
+							"zifencei", "zba", "zbb", "sscofpmf";
+			sifive,buserror = <&bus_error1>;
+			tlb-split;
+			cpu1_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+		cpu2: cpu@2 {
+			compatible = "sifive,p550", "riscv";
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_2>;
+			clock-frequency = <0>;
+			device_type = "cpu";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&l2_cache_2>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			reg = <0x2>;
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "zicsr",
+							"zifencei", "zba", "zbb", "sscofpmf";
+			sifive,buserror = <&bus_error2>;
+			tlb-split;
+			cpu2_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+		cpu3: cpu@3 {
+			compatible = "sifive,p550", "riscv";
+			clocks = <&d0_clock EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_3>;
+			clock-frequency = <0>;
+			device_type = "cpu";
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv48";
+			next-level-cache = <&l2_cache_3>;
+			operating-points-v2 = <&d0_cpu_opp_table>;
+			reg = <0x3>;
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "zicsr",
+							"zifencei", "zba", "zbb", "sscofpmf";
+			sifive,buserror = <&bus_error3>;
+			tlb-split;
+			cpu3_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+	};
+
+	pmu {
+		compatible = "riscv,pmu";
+		riscv,event-to-mhpmcounters =
+				<0x00001 0x00001 0x00000001>,
+				<0x00002 0x00002 0x00000002>,
+				<0x00004 0x00006 0x00000078>,
+				<0x10009 0x10009 0x00000078>,
+				<0x10019 0x10019 0x00000078>,
+				<0x10021 0x10021 0x00000078>;
+		riscv,event-to-mhpmevent =
+				<0x00004 0x00000000 0x00000202>,
+				<0x00005 0x00000000 0x00004000>,
+				<0x00006 0x00000000 0x00002001>,
+				<0x10009 0x00000000 0x00000102>,
+				<0x10019 0x00000000 0x00001002>,
+				<0x10021 0x00000000 0x00000802>;
+		riscv,raw-event-to-mhpmcounters =
+				<0x00000000 0x00000000 0xffffffff 0x000000ff 0x00000078>,
+				<0x00000000 0x00000001 0xffffffff 0xfffe07ff 0x00000078>,
+				<0x00000000 0x00000002 0xffffffff 0xfffe00ff 0x00000078>,
+				<0x00000000 0x00000003 0xfffffffc 0x000000ff 0x00000078>,
+				<0x00000000 0x00000004 0xffffffc0 0x000000ff 0x00000078>,
+				<0x00000000 0x00000005 0xffffffff 0xfffffdff 0x00000078>,
+				<0x00000000 0x00000006 0xfffffe00 0x110204ff 0x00000078>,
+				<0x00000000 0x00000007 0xffffffff 0xf00000ff 0x00000078>,
+				<0x00000000 0x00000008 0xfffffe04 0x000000ff 0x00000078>,
+				<0x00000000 0x00000009 0xffffffff 0xffffc0ff 0x00000078>,
+				<0x00000000 0x0000000a 0xffffffff 0xf00000ff 0x00000078>,
+				<0x00000000 0x0000000b 0xffffffff 0xfffffcff 0x00000078>,
+				<0x00000000 0x0000000c 0xfffffff0 0x000000ff 0x00000078>,
+				<0x00000000 0x0000000d 0xffffffff 0x800000ff 0x00000078>,
+				<0x00000000 0x0000000e 0xffffffff 0xf80000ff 0x00000078>,
+				<0x00000000 0x0000000f 0xfffffffc 0x000000ff 0x00000078>;
+	};
+
+	SOC: soc {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		compatible = "simple-bus";
+		ranges;
+		plic0: interrupt-controller@c000000 {
+			compatible = "sifive,fu540-c000-plic", "sifive,plic-1.0.0";
+			reg = <0x0 0xc000000 0x0 0x4000000>;
+			#address-cells = <0>;
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			interrupts-extended =
+				<&cpu0_intc 0xffffffff>, <&cpu0_intc 9>,
+				<&cpu1_intc 0xffffffff>, <&cpu1_intc 9>,
+				<&cpu2_intc 0xffffffff>, <&cpu2_intc 9>,
+				<&cpu3_intc 0xffffffff>, <&cpu3_intc 9>;
+			riscv,ndev = <520>;
+		};
+		l2_cache_0: pl2@104000 {
+		        cache-block-size = <64>;
+		        cache-level = <2>;
+		        cache-sets = <512>;
+		        cache-size = <262144>;
+		        cache-unified;
+		        compatible = "sifive,pl2cache1", "cache";
+		        next-level-cache = <&ccache>;
+		        reg = <0x0 0x104000 0x0 0x4000>;
+		        sifive,perfmon-counters = <6>;
+		};
+		l2_cache_1: pl2@108000 {
+		        cache-block-size = <64>;
+		        cache-level = <2>;
+		        cache-sets = <512>;
+		        cache-size = <262144>;
+		        cache-unified;
+		        compatible = "sifive,pl2cache1", "cache";
+		        next-level-cache = <&ccache>;
+		        reg = <0x0 0x108000 0x0 0x4000>;
+		        sifive,perfmon-counters = <6>;
+		};
+		l2_cache_2: pl2@10c000 {
+		        cache-block-size = <64>;
+		        cache-level = <2>;
+		        cache-sets = <512>;
+		        cache-size = <262144>;
+		        cache-unified;
+		        compatible = "sifive,pl2cache1", "cache";
+		        next-level-cache = <&ccache>;
+		        reg = <0x0 0x10c000 0x0 0x4000>;
+		        sifive,perfmon-counters = <6>;
+		};
+		l2_cache_3: pl2@110000 {
+		        cache-block-size = <64>;
+		        cache-level = <2>;
+		        cache-sets = <512>;
+		        cache-size = <262144>;
+		        cache-unified;
+		        compatible = "sifive,pl2cache1", "cache";
+		        next-level-cache = <&ccache>;
+		        reg = <0x0 0x110000 0x0 0x4000>;
+		        sifive,perfmon-counters = <6>;
+		};
+		ccache: cache-controller@2010000 {
+			compatible = "sifive,eic7700", "sifive,ccache0";
+			cache-block-size = <64>;
+			cache-level = <3>;
+			cache-sets = <4096>;
+			cache-size = <4194304>;
+			cache-unified;
+			interrupt-parent = <&plic0>;
+			interrupts = <1>, <3>, <4>, <2>;
+			reg = <0x0 0x2010000 0x0 0x4000>;
+			sifive,perfmon-counters = <6>;
+		};
+		bus_error0: bus-error-unit@hart0 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <517>;
+			reg = <0x0 0x1700000 0x0 0x1000>;
+		};
+		bus_error1: bus-error-unit@hart1 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <518>;
+			reg = <0x0 0x1701000 0x0 0x1000>;
+		};
+		bus_error2: bus-error-unit@hart2 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <519>;
+			reg = <0x0 0x1702000 0x0 0x1000>;
+		};
+		bus_error3: bus-error-unit@hart3 {
+			compatible = "sifive,buserror";
+			interrupt-parent = <&plic0>;
+			interrupts = <520>;
+			reg = <0x0 0x1703000 0x0 0x1000>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts
new file mode 100644
index 000000000000..377fb9a06adf
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Device Tree file for Eswin EIC7700 SoC.
+ *
+ * Copyright (c) 2024, Beijing ESWIN Computing Technology Co., Ltd.
+ */
+
+/dts-v1/;
+
+#define RTCCLK_FREQ		1000000
+#define LSPCLK_FREQ		200000000
+#define CMA_SIZE		0x20000000
+
+#include "eic7700.dtsi"
+#include "hifive-premier-p550-pinctrl.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "SiFive HiFive Premier P550";
+	compatible = "sifive,hifive-premier-p550", "eswin,eic7700";
+
+	aliases {
+		serial0 = &d0_uart0;
+		ethernet0 = &d0_gmac0;
+		ethernet1 = &d0_gmac1;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x7f 0x80000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x0 CMA_SIZE>;
+			alignment = <0x0 0x1000>;
+			linux,cma-default;
+		};
+
+		npu0_reserved: sprammemory@59000000 {
+			no-map;
+			reg = <0x0 0x59000000 0x0 0x400000>;
+		};
+
+		lpcpu0_reserved: lpcpu@a0000000 {
+			no-map;
+			reg = <0x0 0xa0000000 0x0 0x100000>;
+		};
+
+		g2d_4GB_boundary_reserved_4k {
+			no-map;
+			reg = <0x0 0xfffff000 0x0 0x1000>;
+		};
+	};
+};
+
+&d0_clock {
+	status = "okay";
+};
+
+&d0_reset {
+	status = "okay";
+};
+
+&smmu0 {
+	status = "okay";
+};
+
+&pcie {
+	/* GPIO12 PCIE PRSNT input */
+	status = "okay";
+};
+
+&gpu0 {
+	status = "okay";
+};
+
+&video_output {
+	status = "okay";
+};
+
+&dc {
+	status = "okay";
+};
+
+&dsi_output {
+    status = "okay";
+};
+
+&dsi_controller {
+    status = "okay";
+};
+
+&dsi_panel {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio82_default &pinctrl_gpio85_default>;
+	backlight0-gpios = <&portc 18 GPIO_ACTIVE_HIGH>;
+	rst-gpios = <&portc 21 GPIO_ACTIVE_HIGH>;
+};
+
+&dw_hdmi {
+	status = "okay";
+	ports {
+		port@2 {
+			reg = <2>;
+			hdmi_in_i2s: endpoint@1 {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s0_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2s0 {
+	status = "okay";
+	d0_i2s0_port: port {
+		d0_i2s0_endpoint: endpoint {
+			remote-endpoint = <&hdmi_in_i2s>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_i2s1 {
+	status = "okay";
+	d0_i2s1_port: port {
+		d0_i2s1_endpoint: endpoint {
+			remote-endpoint = <&d0_codec0_endpoint>;
+			dai-format = "i2s";
+		};
+	};
+};
+
+&d0_graphcard0 {
+	status = "okay";
+	dais = <&d0_i2s1_port>;
+};
+
+&d0_graphcard1 {
+	status = "okay";
+	dais = <&d0_i2s0_port>;
+};
+
+&sdhci_emmc {
+	/* emmc */
+	status = "okay";
+	delay_code = <0x17>;
+	drive-impedance-ohm = <50>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_emmc_led_control_default>;
+};
+
+&sdio0 {
+	/* sd card */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	no-sdio;
+	no-mmc;
+};
+
+&sdio1 {
+	/* wifi module */
+	status = "okay";
+	delay_code = <0x29>;
+	drive-impedance-ohm = <33>;
+	enable-cmd-pullup;
+	enable-data-pullup;
+	non-removable;
+	no-sd;
+	no-mmc;
+};
+
+&d0_gmac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio106_default>;
+	rst-gpios = <&portd 10 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x290 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+};
+
+&d0_gmac1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio111_default>;
+	rst-gpios = <&portd 15 GPIO_ACTIVE_LOW>;
+	eswin,rgmiisel = <&pinctrl 0x294 0x3>;
+	eswin,led-cfgs = <0x6100 0xa40 0x420>;
+};
+
+&d0_sata {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sata_act_led_default>;
+};
+
+&d0_usbdrd3_0 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+};
+
+&d0_usbdrd3_1 {
+	status = "okay";
+};
+
+&d0_usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+	maximum-speed = "super-speed";
+	usb-hub {
+		gpio-hog;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio43_default>;
+		gpios = <&portb 11 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "usb-hub-reset";
+	};
+};
+
+&d0_dmac0 {
+	status = "okay";
+};
+
+&d0_aon_dmac {
+	status = "okay";
+};
+
+&d0_uart0 {
+	/* debug */
+	status = "okay";
+};
+
+&d0_uart1 {
+	/* M.2 KEY E */
+	status = "okay";
+};
+
+&d0_uart2 {
+	/* connect MCU */
+	status = "okay";
+};
+
+&bootspi {
+	/* spi flash */
+	status = "okay";
+	num-cs = <1>;
+	cs-gpios = <&portd 0 GPIO_ACTIVE_LOW>;
+	spi-flash@0 {
+		compatible = "winbond,w25q128jw",
+					"jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <4800000>;
+		rx-sample-delay-ns = <10>;
+	};
+};
+
+&fan_control {
+	status = "okay";
+};
+
+&d0_i2c0 {
+	/* codec es8388 */
+	status = "okay";
+	d0_es8388_0: es8388-0@11 {
+		compatible = "eswin,es8388";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		eswin-plat = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio0_default &pinctrl_gpio28_default>;
+		front-jack-gpios = <&porta 0 GPIO_ACTIVE_HIGH>;
+		back-jack-gpios = <&porta 28 GPIO_ACTIVE_HIGH>;
+		port {
+			d0_codec0_endpoint: endpoint {
+				system-clock-frequency = <12288000>;
+				remote-endpoint = <&d0_i2s1_endpoint>;
+			};
+		};
+	};
+};
+
+&d0_i2c1 {
+	status = "okay";
+};
+
+&d0_i2c3 {
+	/* FUSB303B cc logic */
+	status = "okay";
+	fusb303b@21 {
+		compatible = "fcs,fusb303b";
+		status = "okay";
+		reg = <0x21>;
+		eswin,syscfg = <&d0_sys_con 0x3C0 12>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio5_default>;
+		int-gpios = <&porta 5 GPIO_ACTIVE_HIGH>;
+		connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "host";
+		};
+	};
+};
+
+&d0_i2c5 {
+	/* TCA9548 */
+	status = "okay";
+};
+
+&d0_aon_i2c0 {
+	/* AT24C02C EEPROM */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 16>;
+	aon_eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
+&d0_aon_i2c1 {
+	/* INA226 & PAC1934 */
+	status = "okay";
+	eswin,syscfg = <&d0_sys_con 0x3C0 15>;
+	i2c-sda-hold-time-ns = <0x40>;
+	pac1934:pac1934@10 {
+		compatible = "microchip,pac1934";
+		/*update all register data*/
+		update_time_ms = <1000>;
+		eswin,chan_label = "som vdd", "soc vdd", "cpu vdd", "ddr lpvdd";
+		label = "som_info";
+		/*The update number of times the energy accumulates*/
+		energy_acc_count = <0>;
+		shunt_resistors=<1 1 1 1>;
+		reg = <0x10>;
+	};
+
+	sys_power:ina226@44 {
+		compatible = "ti,ina226";
+		#io-channel-cells = <1>;
+		label = "sys_power";
+		reg = <0x44>;
+		shunt-resistor = <1000>;
+	};
+};
+
+&pwm0 {
+	/* fan */
+	status = "okay";
+};
+
+&pvt0 {
+	status = "okay";
+};
+
+&pvt1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "okay";
+};
+
+&wdt1 {
+	status = "okay";
+};
+
+&wdt2 {
+	status = "okay";
+};
+
+&wdt3 {
+	status = "okay";
+};
+
+&die0_rtc {
+	status = "okay";
+};
+
+&pinctrl {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio6_default &pinctrl_gpio7_default
+		&pinctrl_gpio8_default &pinctrl_gpio9_default
+		&pinctrl_gpio10_default &pinctrl_gpio17_default
+		&pinctrl_gpio35_default &pinctrl_gpio36_default
+		&pinctrl_gpio37_default &pinctrl_gpio38_default
+		&pinctrl_gpio39_default &pinctrl_gpio40_default
+		&pinctrl_gpio41_default &pinctrl_gpio52_default
+		&pinctrl_gpio53_default &pinctrl_gpio64_default
+		&pinctrl_gpio65_default &pinctrl_gpio66_default
+		&pinctrl_gpio67_default &pinctrl_gpio70_default
+		&pinctrl_gpio73_default &pinctrl_gpio83_default
+		&pinctrl_gpio86_default &pinctrl_gpio87_default
+		&pinctrl_gpio92_default &pinctrl_gpio93_default>;
+
+    /* pin header default function for GPIO
+       SPI1 (CS0,SCLK,MOSI,MISO,D2,D3): GPIO 35,36,37,38,39,40
+       I2C1 (SCL,SDA): GPIO 46,47
+       UART3(TX,RX): GPIO 92,93
+    */
+};
+
+&gpio0 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700-noc.dtsi b/arch/riscv/boot/dts/eswin/eic7700-noc.dtsi
new file mode 100644
index 000000000000..a4e143548832
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700-noc.dtsi
@@ -0,0 +1,2620 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Include file for Die0 NOC monitor of Eswin EIC770x family SoC.
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+d0_cfg_noc:d0_cfg_noc{
+	compatible = "eswin,eic7700-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	reg = <0 0x52060000 0 0x4000>;
+
+	interrupts = <446>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 5>;
+
+	sideband_manager@52061000{
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52061000 0 0x10>;
+		SenseIn0 =
+			<SBM_CLMM 	OFFSET0>,
+			<SBM_CNOC_AON 	OFFSET1>,
+			<SBM_CNOC_DDRT0_CTRL 	OFFSET2>,
+			<SBM_CNOC_DDRT0_PHY 	OFFSET3>,
+			<SBM_CNOC_DDRT1_CTRL 	OFFSET4>,
+			<SBM_CNOC_DDRT1_PHY 	OFFSET5>,
+			<SBM_CNOC_DSPT 	OFFSET6>,
+			<SBM_CNOC_GPU 	OFFSET7>,
+			<SBM_CNOC_HSP 	OFFSET8>,
+			<SBM_CNOC_LSP_APB2 	OFFSET9>,
+			<SBM_CNOC_LSP_APB3	OFFSET10>,
+			<SBM_CNOC_LSP_APB4 	OFFSET11>,
+			<SBM_CNOC_LSP_APB6 	OFFSET12>,
+			<SBM_CNOC_MCPUT_D2D 	OFFSET13>,
+			<SBM_CNOC_NPU 	OFFSET14>,
+			<SBM_CNOC_PCIET_P 	OFFSET15>,
+			<SBM_CNOC_PCIET_X 	OFFSET16>,
+			<SBM_CNOC_TCU 	OFFSET17>,
+			<SBM_CNOC_VC 	OFFSET18>,
+			<SBM_CNOC_VI 	OFFSET19>,
+			<SBM_CNOC_VO 	OFFSET20>;
+		bf-name =
+			"SBM_CLMM",
+			"SBM_CNOC_AON",
+			"SBM_CNOC_DDRT0_CTRL",
+			"SBM_CNOC_DDRT0_PHY ",
+			"SBM_CNOC_DDRT1_CTRL",
+			"SBM_CNOC_DDRT1_PHY",
+			"SBM_CNOC_DSPT",
+			"SBM_CNOC_GPU",
+			"SBM_CNOC_HSP",
+			"SBM_CNOC_LSP_APB2",
+			"SBM_CNOC_LSP_APB3",
+			"SBM_CNOC_LSP_APB4",
+			"SBM_CNOC_LSP_APB6",
+			"SBM_CNOC_MCPUT_D2D",
+			"SBM_CNOC_NPU",
+			"SBM_CNOC_PCIET_P",
+			"SBM_CNOC_PCIET_X",
+			"SBM_CNOC_TCU",
+			"SBM_CNOC_VC",
+			"SBM_CNOC_VI",
+			"SBM_CNOC_VO";
+	};
+	ErrorLogger0 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 1>;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 1 4>;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,eic7700,register", "eswin,eic7700,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 15 1 >;  /*bit 15 will aloways be 0, then we will always get "snoc_cnoc/I/0"*/
+			lut =
+				"snoc_cnoc/I/0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 10 5 >;
+			lut =
+				"clmm/T/0",
+				"cnoc_aon/T/0",
+				"cnoc_ddrt0_ctrl/T/0",
+				"cnoc_ddrt0_phy/T/0",
+				"cnoc_ddrt1_ctrl/T/0",
+				"cnoc_ddrt1_phy/T/0",
+				"cnoc_dspt/T/0",
+				"cnoc_gpu/T/0",
+				"cnoc_hsp/T/0",
+				"cnoc_lsp_apb2/T/0",
+				"cnoc_lsp_apb3/T/0",
+				"cnoc_lsp_apb4/T/0",
+				"cnoc_lsp_apb6/T/0",
+				"cnoc_mcput_d2d/T/0",
+				"cnoc_npu/T/0",
+				"cnoc_pciet_p/T/0",
+				"cnoc_pciet_x/T/0",
+				"cnoc_service/T/0",
+				"cnoc_tcu/T/0",
+				"cnoc_vc/T/0",
+				"cnoc_vi/T/0",
+				"cnoc_vo/T/0",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED",
+				"RESERVED";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 2 >;
+		};
+
+		SeqId {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 8 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,eic7700,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger1 whose information are required to calculate real absolute address */
+		AbsoluteAddress {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 8 >;	/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 46 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <0x0 0x0    0x0  0x51600000>,
+				/bits/ 64  <0x0 0x0    0x1  0x71600000>,
+				/bits/ 64  <0x0 0x1    0x0  0x51800000>,
+				/bits/ 64  <0x0 0x1    0x1  0x71800000>,
+				/bits/ 64  <0x0 0x1    0x2  0x0       >,
+				/bits/ 64  <0x0 0x1    0x3  0x0       >,
+				/bits/ 64  <0x0 0x2    0x0  0x52300000>,
+				/bits/ 64  <0x0 0x2    0x1  0x72300000>,
+				/bits/ 64  <0x0 0x3    0x0  0x53000000>,
+				/bits/ 64  <0x0 0x3    0x1  0x73000000>,
+				/bits/ 64  <0x0 0x4    0x0  0x52380000>,
+				/bits/ 64  <0x0 0x4    0x1  0x72380000>,
+				/bits/ 64  <0x0 0x5    0x0  0x53800000>,
+				/bits/ 64  <0x0 0x5    0x1  0x73800000>,
+				/bits/ 64  <0x0 0x6    0x0  0x52200000>,
+				/bits/ 64  <0x0 0x6    0x1  0x72200000>,
+				/bits/ 64  <0x0 0x7    0x0  0x51400000>,
+				/bits/ 64  <0x0 0x7    0x1  0x71400000>,
+				/bits/ 64  <0x0 0x8    0x0  0x50400000>,
+				/bits/ 64  <0x0 0x8    0x1  0x70400000>,
+				/bits/ 64  <0x0 0x9    0x0  0x50800000>,
+				/bits/ 64  <0x0 0x9    0x1  0x70800000>,
+				/bits/ 64  <0x0 0xa    0x0  0x50900000>,
+				/bits/ 64  <0x0 0xa    0x1  0x70900000>,
+				/bits/ 64  <0x0 0xb    0x0  0x50a00000>,
+				/bits/ 64  <0x0 0xb    0x1  0x70a00000>,
+				/bits/ 64  <0x0 0xc    0x0  0x50b00000>,
+				/bits/ 64  <0x0 0xc    0x1  0x70b00000>,
+				/bits/ 64  <0x0 0xd    0x0  0x52100000>,
+				/bits/ 64  <0x0 0xd    0x1  0x72100000>,
+				/bits/ 64  <0x0 0xe    0x0  0x51c00000>,
+				/bits/ 64  <0x0 0xe    0x1  0x71c00000>,
+				/bits/ 64  <0x0 0xf    0x0  0x50000000>,
+				/bits/ 64  <0x0 0xf    0x1  0x70000000>,
+				/bits/ 64  <0x0 0x10   0x0  0x54000000>,
+				/bits/ 64  <0x0 0x10   0x1  0x74000000>,
+				/bits/ 64  <0x0 0x11   0x0  0x52060000>,
+				/bits/ 64  <0x0 0x11   0x1  0x72060000>,
+				/bits/ 64  <0x0 0x12   0x0  0x50c00000>,
+				/bits/ 64  <0x0 0x12   0x1  0x70c00000>,
+				/bits/ 64  <0x0 0x13   0x0  0x50100000>,
+				/bits/ 64  <0x0 0x13   0x1  0x70100000>,
+				/bits/ 64  <0x0 0x14   0x0  0x51000000>,
+				/bits/ 64  <0x0 0x14   0x1  0x71000000>,
+				/bits/ 64  <0x0 0x15   0x0  0x50200000>,
+				/bits/ 64  <0x0 0x15   0x1  0x70200000>;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x28 32>;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0x0 7 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0",
+				"Prot_1",
+				"Prot_2";
+		};
+	};
+};
+
+d0_llc_noc:d0_llc_noc@52081400 {
+	compatible = "eswin,eic7700-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	reg = <0 0x52081400 0 0x4000>;
+	interrupts = <441>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+	sideband_manager@52082000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52082000 0 0x10>;
+		SenseIn0 =
+			<SBM_LNOC_NPU_LLC0 	OFFSET0>,
+			<SBM_LNOC_NPU_LLC1 	OFFSET1>,
+			<SBM_LNOC_DDRT0_P0 	OFFSET2>,
+			<SBM_LNOC_DDRT1_P0 	OFFSET3>;
+		bf-name =
+			"SBM_LNOC_NPU_LLC0",
+			"SBM_LNOC_NPU_LLC1",
+			"SBM_LNOC_DDRT0_P0",
+			"SBM_LNOC_DDRT1_P0";
+	};
+
+	llcnoc_packet_ddr0_p0_req_probe@52080000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52080000 0 0x4000>;
+		clocks = <&d0_clock EIC7700_CLK_DDRT0_P0_ACLK>;
+		clock-names = "clk";
+		interrupts = <445>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p0_req";
+	};
+	llcnoc_packet_ddr1_p0_req_probe@52080800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52080800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P0_ACLK>;
+		clock-names = "clk";
+		interrupts = <443>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p0_req";
+	};
+	llcnoc_trans_probe@52081000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52081000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <441>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "npu_llc0", "npu_llc1";
+		llcnoc_trans_npu_llc0_filter@52081480 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52081480 0 0x80>;
+		};
+		llcnoc_trans_npu_llc1_filter@52081500 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52081500 0 0x80>;
+		};
+		llcnoc_trans_profiler@52081580 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52081580 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,eic7700,register", "eswin,eic7700,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"npu_lnoc_llc0/I/0",
+				"npu_lnoc_llc1/I/0",
+				"snoc_lnoc/I/0",
+				"RESERVED";
+		};
+
+		TargetFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"lnoc_ddrt0_p0/T/0",
+				"lnoc_ddrt1_p0/T/0",
+				"lnoc_service/T/0",
+				"RESERVED";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 4 >;
+		};
+
+		SeqId {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 8 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,eic7700,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 8 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 53 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =		/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <0x0	0x0	0x0	0x80000000      >,
+				/bits/ 64  <0x0	0x0	0x1	0x100000000     >,
+				/bits/ 64  <0x0	0x0	0x2	0x200000000     >,
+				/bits/ 64  <0x0	0x0	0x3	0x400000000     >,
+				/bits/ 64  <0x0	0x0	0x4	0x800000000     >,
+				/bits/ 64  <0x0	0x0	0x5	0x2000000000    >,
+				/bits/ 64  <0x0	0x0	0x6	0xc000000000    >,
+				/bits/ 64  <0x0	0x0	0x7	0xe000000000    >,
+				/bits/ 64  <0x0	0x0	0x8	0x4000000000    >,
+				/bits/ 64  <0x0	0x0	0x9	0x4000000100    >,
+				/bits/ 64  <0x0	0x0	0xa	0x10000000000   >,
+				/bits/ 64  <0x0	0x0	0xb	0x10000000100   >,
+				/bits/ 64  <0x0	0x0	0xc	0x0             >,
+				/bits/ 64  <0x0	0x1	0x0	0x80000080      >,
+				/bits/ 64  <0x0	0x1	0x1	0x100000080     >,
+				/bits/ 64  <0x0	0x1	0x2	0x200000080     >,
+				/bits/ 64  <0x0	0x1	0x3	0x400000080     >,
+				/bits/ 64  <0x0	0x1	0x4	0x800000080     >,
+				/bits/ 64  <0x0	0x1	0x5	0x2000000080    >,
+				/bits/ 64  <0x0	0x1	0x6	0xc000000080    >,
+				/bits/ 64  <0x0	0x1	0x7	0xe000000080    >,
+				/bits/ 64  <0x0	0x1	0x8	0x4000000080    >,
+				/bits/ 64  <0x0	0x1	0x9	0x4000000180    >,
+				/bits/ 64  <0x0	0x1	0xa	0x10000000080   >,
+				/bits/ 64  <0x0	0x1	0xb	0x10000000180   >,
+				/bits/ 64  <0x1	0x0	0x0	0x80000000      >,
+				/bits/ 64  <0x1	0x0	0x1	0x100000000     >,
+				/bits/ 64  <0x1	0x0	0x2	0x200000000     >,
+				/bits/ 64  <0x1	0x0	0x3	0x400000000     >,
+				/bits/ 64  <0x1	0x0	0x4	0x800000000     >,
+				/bits/ 64  <0x1	0x0	0x5	0x2000000000    >,
+				/bits/ 64  <0x1	0x0	0x6	0xc000000000    >,
+				/bits/ 64  <0x1	0x0	0x7	0xe000000000    >,
+				/bits/ 64  <0x1	0x0	0x8	0x4000000000    >,
+				/bits/ 64  <0x1	0x0	0x9	0x4000000100    >,
+				/bits/ 64  <0x1	0x0	0xa	0x10000000000   >,
+				/bits/ 64  <0x1	0x0	0xb	0x10000000100   >,
+				/bits/ 64  <0x1	0x0	0xc	0x0             >,
+				/bits/ 64  <0x1	0x1	0x0	0x80000080      >,
+				/bits/ 64  <0x1	0x1	0x1	0x100000080     >,
+				/bits/ 64  <0x1	0x1	0x2	0x200000080     >,
+				/bits/ 64  <0x1	0x1	0x3	0x400000080     >,
+				/bits/ 64  <0x1	0x1	0x4	0x800000080     >,
+				/bits/ 64  <0x1	0x1	0x5	0x2000000080    >,
+				/bits/ 64  <0x1	0x1	0x6	0xc000000080    >,
+				/bits/ 64  <0x1	0x1	0x7	0xe000000080    >,
+				/bits/ 64  <0x1	0x1	0x8	0x4000000080    >,
+				/bits/ 64  <0x1	0x1	0x9	0x4000000180    >,
+				/bits/ 64  <0x1	0x1	0xa	0x10000000080   >,
+				/bits/ 64  <0x1	0x1	0xb	0x10000000180   >,
+				/bits/ 64  <0x2	0x2	0x0	0x52080000      >,
+				/bits/ 64  <0x2	0x2	0x1	0x72080000      >,
+				/bits/ 64  <0x2	0x2	0x2	0x0             >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,eic7700,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0x0 16 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0",
+				"Prot_1",
+				"Prot_2",
+				"Qos_0",
+				"Qos_1",
+				"Qos_2",
+				"Qos_3",
+				"User_0",
+				"User_1",
+				"User_2",
+				"User_3",
+				"User_4";
+		};
+	};
+};
+
+d0_sys_noc:d0_sys_noc@52002C00 {
+	compatible = "eswin,eic7700-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	reg = <0 0x52002C00 0 0x4000>;
+	interrupts = <431>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+
+	eswin,qos-configs = "DSPT", "NPU", "SPISLV_TBU3";
+	eswin,DSPT-qos-base = <0x52002C80>;
+	eswin,DSPT-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode 0:fixed 1:limiter 2:bypass 3:regulator*/
+		/* a number of (1/256)th of Bytes/cycle.
+		   Ex:zebu zdfi design feature, dsp AXI Clk=1040MHz, BW=12.1875MB/s, register value = (9.375/1040)*256 = 0x03
+		*/
+		0x10 0x03    /* bandwidth. 12.1875MB/s */
+		/*
+		   Saturation(B) = ((Requried Bandwidth)*(Windows Time of Bandwidth Calculation))/16
+		   Ex:16 byte saturation for BW=12.1875MB/s means 1.313us window time.
+		   The desired value is number of saturation bytes divided by 16(ex,1 for 16byte B)
+		*/
+		0x14 0x1     /* saturation, 1.313us*/
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,NPU-qos-base = <0x52002D00>;
+	eswin,NPU-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,SPISLV_TBU3-qos-base = <0x52002D80>;
+	eswin,SPISLV_TBU3-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+	sideband_manager@52004000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52004000 0 0x10>;
+		SenseIn0 =
+			<SBM_AON_SNOC_SP0 	OFFSET0>,
+			<SBM_DSPT_SNOC 		OFFSET1>,
+			<SBM_JTAG_SNOC 		OFFSET2>,
+			<SBM_MCPUT_SNOC_D2D 	OFFSET3>,
+			<SBM_MCPUT_SNOC_MP 	OFFSET4>,
+			<SBM_MCPUT_SNOC_SP0 	OFFSET5>,
+			<SBM_MCPUT_SNOC_SP1 	OFFSET6>,
+			<SBM_NPU_SNOC_SP0 	OFFSET7>,
+			<SBM_NPU_SNOC_SP1 	OFFSET8>,
+			<SBM_PCIET_SNOC_P 	OFFSET9>,
+			<SBM_SPISLV_PCIET_SNOC	OFFSET10>,
+			<SBM_TBU4_SNOC 		OFFSET11>,
+			<SBM_TCU_SNOC 		OFFSET12>,
+			<SBM_SNOC_AON 		OFFSET13>,
+			<SBM_SNOC_DDR0_P1 	OFFSET14>,
+			<SBM_SNOC_DDR0_P2 	OFFSET15>,
+			<SBM_SNOC_DDR1_P1 	OFFSET16>,
+			<SBM_SNOC_DDR1_P2 	OFFSET17>,
+			<SBM_SNOC_DSPT 		OFFSET18>,
+			<SBM_SNOC_MCPUT_D2D 	OFFSET19>,
+			<SBM_SNOC_NPU 		OFFSET20>,
+			<SBM_SNOC_PCIET 	OFFSET21>;
+		bf-name =
+			"SBM_AON_SNOC_SP0",
+			"SBM_DSPT_SNOC",
+			"SBM_JTAG_SNOC",
+			"SBM_MCPUT_SNOC_D2D ",
+			"SBM_MCPUT_SNOC_MP",
+			"SBM_MCPUT_SNOC_SP0",
+			"SBM_MCPUT_SNOC_SP1",
+			"SBM_NPU_SNOC_SP0",
+			"SBM_NPU_SNOC_SP1",
+			"SBM_PCIET_SNOC_P",
+			"SBM_SPISLV_PCIET_SNOC",
+			"SBM_TBU4_SNOC",
+			"SBM_TCU_SNOC",
+			"SBM_SNOC_AON",
+			"SBM_SNOC_DDR0_P1",
+			"SBM_SNOC_DDR0_P2",
+			"SBM_SNOC_DDR1_P1",
+			"SBM_SNOC_DDR1_P2",
+			"SBM_SNOC_DSPT",
+			"SBM_SNOC_MCPUT_D2D",
+			"SBM_SNOC_NPU",
+			"SBM_SNOC_PCIET";
+	};
+	sysnoc_packet_ddr0_p1_req_probe@52000000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52000000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT0_P1_ACLK>;
+		clock-names = "clk";
+		interrupts = <439>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p1_req";
+	};
+	sysnoc_packet_ddr0_p2_req_probe@52000800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52000800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT0_P2_ACLK>;
+		clock-names = "clk";
+		interrupts = <437>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p2_req";
+	};
+	sysnoc_packet_ddr1_p1_req_probe@52001000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52001000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P1_ACLK>;
+		clock-names = "clk";
+		interrupts = <435>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p1_req";
+	};
+	sysnoc_packet_ddr1_p2_req_probe@52001800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52001800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P2_ACLK>;
+		clock-names = "clk";
+		interrupts = <433>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p2_req";
+	};
+	sysnoc_trans_probe_0@52002000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52002000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <430>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "dspt_snoc", "npu_sp1";
+		sysnoc_trans_dspt_filter@52002E00 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002E00 0 0x80>;
+		};
+		sysnoc_trans_npu_sp1_filter@52002F80 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002F80 0 0x80>;
+		};
+		sysnoc_trans_profiler@52003180 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52003180 0 0x80>;
+		};
+	};
+	sysnoc_trans_probe_1@52002400 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52002400 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <429>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <3>;
+		counter,nr = <12>;
+		profiler,nr = <1>;
+		portsel = "mcput_mp", "mcput_sp1", "tcu";
+		sysnoc_trans_mcput_mp_filter@52002E80 {
+			status = "okay";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002E80 0 0x80>;
+		};
+		sysnoc_trans_mcput_sp1_filter@52002F00 {
+			status = "okay";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52002F00 0 0x80>;
+		};
+		sysnoc_trans_tcu_filter@52003100 {
+			status = "okay";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52003100 0 0x80>;
+		};
+		sysnoc_trans_profiler@52003200 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52003200 0 0x80>;
+		};
+	};
+	sysnoc_trans_probe_2@52002800 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52002800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <428>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "spislv_tbu3", "tbu4_snoc";
+		sysnoc_trans_spislv_tbu3_filter@52003000 {	/*pcie subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52003000 0 0x80>;
+		};
+		sysnoc_trans_tbu4_filter@52003080 {	/*aon subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52003080 0 0x80>;
+		};
+		sysnoc_trans_profiler@52003280 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52003280 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 1>;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 1 4>;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,eic7700,register", "eswin,eic7700,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 19 4 >;
+			lut =
+				"aon_snoc_sp0/I/0",
+				"dspt_snoc/I/0",
+				"jtag_snoc/I/0",
+				"mcput_snoc_d2d/I/0",
+				"mcput_snoc_mp/I/0",
+				"mcput_snoc_sp0/I/0",
+				"mcput_snoc_sp1/I/0",
+				"mnoc_snoc/I/0",
+				"npu_snoc_sp0/I/0",
+				"npu_snoc_sp1/I/0",
+				"pciet_snoc_p/I/0",
+				"rnoc_snoc/I/0",
+				"spislv_tbu3_snoc/I/0",
+				"tbu4_snoc/I/0",
+				"tcu_snoc/I/0",
+				"RESERVED0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 15 4 >;
+			lut =
+				"snoc_aon/T/0",
+				"snoc_cnoc/T/0",
+				"snoc_ddrt0_p1/T/0",
+				"snoc_ddrt0_p2/T/0",
+				"snoc_ddrt1_p1/T/0",
+				"snoc_ddrt1_p2/T/0",
+				"snoc_dspt/T/0",
+				"snoc_lnoc/T/0",
+				"snoc_mcput_d2d/T/0",
+				"snoc_mnoc/T/0",
+				"snoc_npu/T/0",
+				"snoc_pciet/T/0",
+				"snoc_rnoc/T/0",
+				"snoc_service/T/0",
+				"RESERVED1",
+				"RESERVED2";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 9 6 >;
+		};
+
+		SeqId {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,eic7700,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 9 14 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 1181 >;		/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =		/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <aon_snoc_sp0_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <dspt_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <jtag_snoc_I_O        snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <mcput_snoc_d2d_I_O   snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x6             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt0_p1_T_O  0x7             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x6             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_ddrt1_p1_T_O  0x7             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x6             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x7             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x8             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0x9             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0xa             0x0               >,
+				/bits/ 64  <mcput_snoc_mp_I_O    snoc_npu_T_O       0xb             0x0               >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x4             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x5             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x6             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_aon_T_O       0x7             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <mcput_snoc_sp0_I_O   snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xe             0x0               >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_aon_T_O       0xf             0x0               >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mcput_snoc_sp1_I_O   snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <mnoc_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x4             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x5             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x6             0x5c000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_aon_T_O       0x7             0x7c000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x2             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x3             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x4             0x59000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x5             0x5a000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x6             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x7             0x79000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x8             0x7a000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0x9             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xa             0x50000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xb             0x70000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xc             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mcput_d2d_T_O 0xd             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <npu_snoc_sp0_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xe             0x0               >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_aon_T_O       0xf             0x0               >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <npu_snoc_sp1_I_O     snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0xf             0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_cnoc_T_O      0x10            0x0               >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <pciet_snoc_p_I_O     snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <rnoc_snoc_I_O        snoc_npu_T_O       0x13            0x0               >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x0             0x520c0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x1             0x720c0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x2             0x58400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x3             0x78400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x4             0x14000400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x5             0x14008400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x6             0x58800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x7             0x78800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x8             0x14000800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0x9             0x14008800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xa             0x5c000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xb             0x7c000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xc             0x14004000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_aon_T_O       0xd             0x1400c000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x0             0x52060000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x1             0x72060000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x2             0x52100000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x3             0x52200000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x4             0x52300000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x5             0x72100000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x6             0x72200000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x7             0x72300000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x8             0x53000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0x9             0x73000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xa             0x50000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xb             0x70000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xc             0x54000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xd             0x74000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_cnoc_T_O      0xe             0x0               >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_lnoc_T_O      0x0             0x52080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_lnoc_T_O      0x1             0x72080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mnoc_T_O      0x0             0x52020000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_mnoc_T_O      0x1             0x72020000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_rnoc_T_O      0x0             0x52040000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_rnoc_T_O      0x1             0x72040000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_service_T_O   0x0             0x52000000        >,
+				/bits/ 64  <spislv_tbu3_snoc_I_O snoc_service_T_O   0x1             0x72000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_npu_T_O       0x13            0x0               >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x0             0x40000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x1             0x60000000        >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x2             0x8000000000      >,
+				/bits/ 64  <tbu4_snoc_I_O        snoc_pciet_T_O     0x3             0xa000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x0             0x80000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x1             0x100000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x2             0x200000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x3             0x400000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x4             0x800000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x5             0x2000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x6             0xc000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x7             0xe000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x8             0x4000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0x9             0x4000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0xa             0x10000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt0_p2_T_O  0xb             0x10000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x0             0x80000080        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x1             0x100000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x2             0x200000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x3             0x400000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x4             0x800000080       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x5             0x2000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x6             0xc000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x7             0xe000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x8             0x4000000080      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0x9             0x4000000180      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0xa             0x10000000080     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_ddrt1_p2_T_O  0xb             0x10000000180     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x0             0x520e0000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x1             0x720e0000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x2             0x5b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x3             0x7b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x4             0x14003000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_dspt_T_O      0x5             0x1400b000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x0             0x58400000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1             0x78400000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2             0x14000400000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x3             0x14008400000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x4             0x58800000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x5             0x78800000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x6             0x14000800000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x7             0x14008800000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x8             0x59000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x9             0x5a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xa             0x5b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xb             0x79000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xc             0x7a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xd             0x7b000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xe             0x14001000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0xf             0x14002000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x10            0x14003000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x11            0x14009000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x12            0x1400a000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x13            0x1400b000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x14            0x50000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x15            0x70000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x16            0x40000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x17            0x60000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x18            0x80000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x19            0x100000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1a            0x200000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1b            0x400000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1c            0x800000000       >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1d            0x1800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1e            0x3800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x1f            0xd800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x20            0xf800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x21            0x2000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x22            0x7000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x23            0x7000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x24            0xc000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x25            0xe000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x26            0x13000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x27            0x13000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x28            0x4000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x29            0x4000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2a            0x8000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2b            0xa000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2c            0x10000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_mcput_d2d_T_O 0x2d            0x10000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x0             0x52080000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x1             0x72080000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x2             0x59000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x3             0x5a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x4             0x79000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x5             0x7a000000        >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x6             0x14001000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x7             0x14002000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x8             0x14009000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x9             0x1400a000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xa             0x1800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xb             0x3800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xc             0xd800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xd             0xf800000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xe             0x7000000000      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0xf             0x7000000100      >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x10            0x13000000000     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x11            0x13000000100     >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x12            0x0               >,
+				/bits/ 64  <tcu_snoc_I_O         snoc_npu_T_O       0x13            0x0               >;
+			};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,eic7700,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x28 32>;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0x0 18 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0 ",
+				"Prot_1 ",
+				"Prot_2 ",
+				"User_0 ",
+				"User_1 ",
+				"User_2 ",
+				"User_3 ",
+				"User_4 ",
+				"User_5 ",
+				"User_6 ",
+				"qos0",
+				"qos1",
+				"qos2",
+				"qos3";
+		};
+	};
+};
+
+d0_media_noc:d0_media_noc@52021400 {
+	compatible = "eswin,eic7700-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	reg = <0 0x52021400 0 0x4000>;
+	interrupts = <454>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+	eswin,qos-configs = "GPU", "TBU2", "VC";
+	eswin,GPU-qos-base = <0x52021480>;
+	eswin,GPU-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode */
+		0x10 0x1E0    /* bandwidth*/
+		0x14 0x1     /* saturation*/
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,TBU2-qos-base = <0x52021500>;
+	eswin,TBU2-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,VC-qos-base = <0x52021580>;
+	eswin,VC-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+	sideband_manager@52022000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52022000 0 0x10>;
+		SenseIn0 =
+			<SBM_MNOC_GPU 	OFFSET0>,
+			<SBM_MNOC_TBU2 	OFFSET1>,
+			<SBM_MNOC_VC 	OFFSET2>,
+			<SBM_MNOC_DDRT0_P3 	OFFSET3>,
+			<SBM_MNOC_DDRT1_P3 	OFFSET4>;
+		bf-name =
+			"SBM_MNOC_GPU",
+			"SBM_MNOC_TBU2",
+			"SBM_MNOC_VC",
+			"SBM_MNOC_DDRT0_P3",
+			"SBM_MNOC_DDRT1_P3";
+	};
+
+	mnoc_packet_ddr0_p3_req_probe@52020000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52020000 0 0x4000>;
+		clocks = <&d0_clock EIC7700_CLK_DDRT0_P3_ACLK>;
+		clock-names = "clk";
+		interrupts = <458>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p3_req";
+	};
+	mnoc_packet_ddr1_p3_req_probe@52020800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52020800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P3_ACLK>;
+		clock-names = "clk";
+		interrupts = <456>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p3_req";
+	};
+	mnoc_trans_probe@52021000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52021000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <453>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <3>;
+		counter,nr = <12>;
+		profiler,nr = <1>;
+		portsel = "gpu", "tbu2", "vc";
+		mnoc_trans_gpu_filter@52021600 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52021600 0 0x80>;
+		};
+		mnoc_trans_tbu2_filter@52021680 {	/*hsp subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52021680 0 0x80>;
+		};
+		mnoc_trans_vc_filter@52021700 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52021700 0 0x80>;
+		};
+		mnoc_trans_profiler@52021780 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52021780 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,eic7700,register", "eswin,eic7700,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"gpu_mnoc/I/0",
+				"snoc_mnoc/I/0",
+				"tbu2_mnoc/I/0",
+				"vc_mnoc/I/0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"mnoc_ddrt0_p3/T/0",
+				"mnoc_ddrt1_p3/T/0",
+				"mnoc_service/T/0",
+				"mnoc_snoc/T/0";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 9 3 >;
+		};
+
+		SeqId {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,eic7700,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 9 7 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 55 >;			/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+					/bits/ 64  <0x0  0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x0  0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x0  0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x0  0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x0  0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x0  0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x0  0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x0  0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x0  0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x0  0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x0  0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x0  0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x0  0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x0  0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x0  0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x0  0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x0  0x3 0x0           0x0              >,
+					/bits/ 64  <0x1 0x2  0x0           0x52020000       >,
+					/bits/ 64  <0x1 0x2  0x1           0x72020000       >,
+					/bits/ 64  <0x1 0x2  0x2           0x0              >,
+					/bits/ 64  <0x1 0x2  0x3           0x0              >,
+					/bits/ 64  <0x2 0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x2 0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x2 0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x2 0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x2 0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x2 0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x2 0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x2 0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x2 0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x2 0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x2 0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x2 0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x2 0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x2 0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x2 0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x2 0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x2 0x3   0x0           0x0              >,
+					/bits/ 64  <0x3   0x0 0x0           0x80000000       >,
+					/bits/ 64  <0x3   0x0 0x1           0x100000000      >,
+					/bits/ 64  <0x3   0x0 0x2           0x200000000      >,
+					/bits/ 64  <0x3   0x0 0x3           0x400000000      >,
+					/bits/ 64  <0x3   0x0 0x4           0x800000000      >,
+					/bits/ 64  <0x3   0x0 0x5           0x2000000000     >,
+					/bits/ 64  <0x3   0x0 0x6           0x4000000000     >,
+					/bits/ 64  <0x3   0x0 0x7           0x4000000100     >,
+					/bits/ 64  <0x3   0x1 0x0           0x80000080       >,
+					/bits/ 64  <0x3   0x1 0x1           0x100000080      >,
+					/bits/ 64  <0x3   0x1 0x2           0x200000080      >,
+					/bits/ 64  <0x3   0x1 0x3           0x400000080      >,
+					/bits/ 64  <0x3   0x1 0x4           0x800000080      >,
+					/bits/ 64  <0x3   0x1 0x5           0x2000000080     >,
+					/bits/ 64  <0x3   0x1 0x6           0x4000000080     >,
+					/bits/ 64  <0x3   0x1 0x7           0x4000000180     >,
+					/bits/ 64  <0x3   0x3 0x0           0x0              >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,eic7700,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0x0 18 >;
+			lut =
+				"Cache_0",
+				"Cache_1",
+				"Cache_2",
+				"Cache_3",
+				"Prot_0 ",
+				"Prot_1 ",
+				"Prot_2 ",
+				"User_0 ",
+				"User_1 ",
+				"User_2 ",
+				"User_3 ",
+				"User_4 ",
+				"User_5 ",
+				"User_6 ",
+				"qos0",
+				"qos1",
+				"qos2",
+				"qos3";
+		};
+	};
+};
+
+d0_realtime_noc:d0_realtime_noc@52041400 {
+	compatible = "eswin,eic7700-noc";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	reg = <0 0x52041400 0 0x4000>;
+	interrupts = <448>;
+	interrupt-names = "error";
+	interrupt-parent = <&plic0>;
+	errlogger,idx = <0 1 3 4 5>;
+	eswin,qos-configs = "TBU0", "VO";
+	eswin,TBU0-qos-base = <0x52041480>;
+	eswin,TBU0-qos-settings = <
+		0x8 0x4        /* prio */
+		0xC 0x2        /* mode */
+		0x10 0x1E0    /* bandwidth */
+		0x14 0x1     /* saturation */
+		0x18 0x1>;   /* QoSEn */
+
+	eswin,VO-qos-base = <0x52041500>;
+	eswin,VO-qos-settings = <
+		0x8 0x4      /* prio */
+		0xC 0x2      /* mode */
+		0x10 0x18    /* bandwidth */
+		0x14 0x10    /* saturation */
+		0x18 0x1>;   /* QoSEn */
+	sideband_manager@52042000 {
+		compatible = "eswin,win2xxx-noc-sideband-manager";
+		reg = <0 0x52042000 0 0x10>;
+		SenseIn0 =
+			<SBM_RNOC_TBU0 	OFFSET0>,
+			<SBM_RNOC_VO 	OFFSET1>,
+			<SBM_RNOC_DDRT0_P4 	OFFSET2>,
+			<SBM_RNOC_DDRT1_P4 	OFFSET3>;
+		bf-name =
+			"SBM_RNOC_TBU0",
+			"SBM_RNOC_VO",
+			"SBM_RNOC_DDRT0_P4",
+			"SBM_RNOC_DDRT1_P4";
+	};
+
+	rnoc_packet_ddr0_p4_req_probe@52040000 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52040000 0 0x4000>;
+		clocks = <&d0_clock EIC7700_CLK_DDRT0_P4_ACLK>;
+		clock-names = "clk";
+		interrupts = <452>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr0_p4_req";
+	};
+	rnoc_packet_ddr1_p4_req_probe@52040800 {
+		compatible = "eswin,win2xxx-noc-packet-probe";
+		reg = <0 0x52040800 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_DDRT1_P4_ACLK>;
+		clock-names = "clk";
+		interrupts = <450>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <1>;
+		counter,nr = <2>;
+		portsel = "ddr1_p4_req";
+	};
+	rnoc_trans_probe@52041000 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		compatible = "eswin,win2xxx-noc-trans-probe";
+		reg = <0 0x52041000 0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_NOC_NSP_CLK>;
+		clock-names = "clk";
+		interrupts = <447>;
+		interrupt-names = "stat";
+		interrupt-parent = <&plic0>;
+		filter,nr = <2>;
+		counter,nr = <8>;
+		profiler,nr = <1>;
+		portsel = "tbu0", "vo";
+		rnoc_trans_tbu0_filter@52041580 {	/*vi subsys*/
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52041580 0 0x80>;
+		};
+		rnoc_trans_vo_filter@52041600 {
+			status = "disabled";
+			compatible = "eswin,win2xxx-noc-trans-filter";
+			reg = <0 0x52041600 0 0x80>;
+		};
+		rnoc_trans_profiler@52041680 {
+			compatible = "eswin,win2xxx-noc-trans-profiler";
+			reg = <0 0x52041680 0 0x80>;
+		};
+	};
+	ErrorLogger0 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x14 32 >;
+		description = "Register 0 to log errors";
+		lock {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 1 >;
+			lut = "Not Locked", "Locked";
+		};
+		OpCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 1 4 >;
+			lut = "RD: data read with incrementing address",
+				"RDW: data read with wrapping address",
+				"RDL: allocates monitor in the Target with incrementing address",
+				"RDX: exclusive read with incrementing address",
+				"WR: data write with incrementing address",
+				"WRW: data write with wrapping address",
+				"WRC: conditional write if matching monitor in Target is found",
+				"RSV: reserved",
+				"PRE: preamble packet of linked sequence (locked sequence)",
+				"URG: urgency packet used for QoS (status must be REQ)";
+		};
+		ErrCode {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 8 3 >;
+			lut = "SLV (error source: Target): error detected by the Slave without any information or no Error",
+				"DEC (error source: Initiator NIU): decode error",
+				"UNS (error source: Target NIU): unsupported access type",
+				"DISC (error source: Power Disconnect): disconnected Target or NoC domain",
+				"SEC (error source: Initiator NIU or Firewall): security error",
+				"HIDE (error source: Firewall): hidden security error. Will be reported as OK to the initiator",
+				"TMO (error source: Target NIU): time-out",
+				"RSV: reserved";
+
+		};
+		Len1 {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 16 12 >;
+		};
+		Format {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 31 1 >;
+			lut = "NTTP v3.0 (Invalid)", "NTTP v3.5 (Ok)";
+		};
+	};
+
+	ErrorLogger1 {
+		compatible = "eswin,eic7700,register", "eswin,eic7700,noc,filter,routeid";
+		offset,length = < 0x18 32>;
+		description = "Register 1 to log errors";
+		InitFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 14 2 >;
+			lut =
+				"snoc_rnoc/I/0",
+				"tbu0_rnoc/I/0",
+				"vo_rnoc/I/0",
+				"RESERVED0";
+		};
+
+		TargetFlow {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 12 2 >;
+			lut =
+				"rnoc_ddrt0_p4/T/0",
+				"rnoc_ddrt1_p4/T/0",
+				"rnoc_service/T/0",
+				"rnoc_snoc/T/0";
+		};
+
+		TargetSubRange {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 9 3 >;
+		};
+
+		SeqId {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger3 {
+		/* This register does not really include a bitfield or its bitfield is 32 bits and it provides an offset address */
+		/* To calculate the absolute address, you must use the initflow:targetflow:subrange from ErrorLogger1 register */
+		/* And use it as an index in the aperture table. Then you must add the value of this register to the value given */
+		/* by the table to get the absolute address. */
+		compatible = "eswin,eic7700,register";
+		offset,length = <0x20 32>;
+		description = "Register 3 to log errors";
+		aperture-link = <1>; /* link to ErrorLogger whose information are required to calculate real absolute address */
+		msb-link = <4>; /*indicate which ErrorLogger contains the msb addrs, -1 means no*/
+		AbsoluteAddress {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 9 7 >;		/* bitfield initflow:targeflow:subrange from aperture-link register */
+			aperture-size= < 40 >;			/* Number of line in the aperture table below */
+			aperture-idx,aperture-base =	/* Aperture_idx is concatenation of initflow:targetflow:subrange bitfield from ErrorLogger1 */
+					/bits/ 64  <0x0 0x2 0x0          0x52040000       >,
+					/bits/ 64  <0x0 0x2 0x1          0x72040000       >,
+					/bits/ 64  <0x0 0x2 0x2          0x0              >,
+					/bits/ 64  <0x0 0x2 0x3          0x0              >,
+					/bits/ 64  <0x1 0x0 0x0          0x80000000       >,
+					/bits/ 64  <0x1 0x0 0x1          0x100000000      >,
+					/bits/ 64  <0x1 0x0 0x2          0x200000000      >,
+					/bits/ 64  <0x1 0x0 0x3          0x400000000      >,
+					/bits/ 64  <0x1 0x0 0x4          0x800000000      >,
+					/bits/ 64  <0x1 0x0 0x5          0x2000000000     >,
+					/bits/ 64  <0x1 0x1 0x0          0x80000080       >,
+					/bits/ 64  <0x1 0x1 0x1          0x100000080      >,
+					/bits/ 64  <0x1 0x1 0x2          0x200000080      >,
+					/bits/ 64  <0x1 0x1 0x3          0x400000080      >,
+					/bits/ 64  <0x1 0x1 0x4          0x800000080      >,
+					/bits/ 64  <0x1 0x1 0x5          0x2000000080     >,
+					/bits/ 64  <0x1 0x3 0x0          0x59000000       >,
+					/bits/ 64  <0x1 0x3 0x1          0x79000000       >,
+					/bits/ 64  <0x1 0x3 0x2          0x14001000000    >,
+					/bits/ 64  <0x1 0x3 0x3          0x14009000000    >,
+					/bits/ 64  <0x1 0x3 0x4          0x0              >,
+					/bits/ 64  <0x1 0x3 0x5          0x0              >,
+					/bits/ 64  <0x2 0x0 0x0          0x80000000       >,
+					/bits/ 64  <0x2 0x0 0x1          0x100000000      >,
+					/bits/ 64  <0x2 0x0 0x2          0x200000000      >,
+					/bits/ 64  <0x2 0x0 0x3          0x400000000      >,
+					/bits/ 64  <0x2 0x0 0x4          0x800000000      >,
+					/bits/ 64  <0x2 0x0 0x5          0x2000000000     >,
+					/bits/ 64  <0x2 0x1 0x0          0x80000080       >,
+					/bits/ 64  <0x2 0x1 0x1          0x100000080      >,
+					/bits/ 64  <0x2 0x1 0x2          0x200000080      >,
+					/bits/ 64  <0x2 0x1 0x3          0x400000080      >,
+					/bits/ 64  <0x2 0x1 0x4          0x800000080      >,
+					/bits/ 64  <0x2 0x1 0x5          0x2000000080     >,
+					/bits/ 64  <0x2 0x3 0x0          0x59000000       >,
+					/bits/ 64  <0x2 0x3 0x1          0x79000000       >,
+					/bits/ 64  <0x2 0x3 0x2          0x14001000000    >,
+					/bits/ 64  <0x2 0x3 0x3          0x14009000000    >,
+					/bits/ 64  <0x2 0x3 0x4          0x0              >,
+					/bits/ 64  <0x2 0x3 0x5          0x0              >;
+		};
+	};
+
+	ErrorLogger4 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x24 32>;
+		description = "Register 4 to log errors";
+		addr_msb {
+			compatible = "eswin,eic7700,bitfield";
+			description = "Stores NTTP packet header field Addr (MSBs) of the logged error";
+			offset,length = < 0 9 >;
+		};
+	};
+
+	ErrorLogger5 {
+		compatible = "eswin,eic7700,register";
+		offset,length = < 0x28 32 >;
+		description = "Register 5 to log errors";
+
+		User_flag {
+			compatible = "eswin,eic7700,bitfield";
+			offset,length = < 0x0 16 >;
+			lut =
+				 "Cache_0",
+				 "Cache_1",
+				 "Cache_2",
+				 "Cache_3",
+				 "Prot_0 ",
+				 "Prot_1 ",
+				 "Prot_2 ",
+				 "User_0 ",
+				 "User_1 ",
+				 "User_2 ",
+				 "User_3 ",
+				 "User_4 ",
+				 "qos0",
+				 "qos1",
+				 "qos2",
+				 "qos3";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/eic7700.dtsi b/arch/riscv/boot/dts/eswin/eic7700.dtsi
new file mode 100644
index 000000000000..9dd5be3d0d8d
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/eic7700.dtsi
@@ -0,0 +1,1171 @@
+﻿// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ */
+
+#include "eic7700-arch.dtsi"
+#include <dt-bindings/memory/eic7700-sid.h>
+#include <dt-bindings/reset/eic7700-syscrg.h>
+#include <dt-bindings/clock/eic7700-clock.h>
+#include <dt-bindings/riscv/physical-memory.h>
+
+/ {
+	riscv,physical-memory-regions =
+		<0x000 0x00000000 0x200 0x00000000 (PMA_RW | PMA_IO) 0x0>,
+		<0x000 0x80000000 0x00f 0x80000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY) 0x0>,
+		<0x0c0 0x00000000 0x010 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY | PMR_ALIAS(1)) 0x0>;
+
+    compatible = "eswin,eic7700";
+	d0_cpu_opp_table: opp-table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-24000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_24M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-100000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_100M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-200000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_200M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-400000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_400M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-500000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_500M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-600000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_600M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-700000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_700M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-800000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_800M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-900000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_900M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1000M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1200M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1300000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1300M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+		opp-1400000000 {
+			opp-hz = /bits/ 64 <CLK_FREQ_1400M>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <70000>;
+		};
+	};
+};
+
+&SOC {
+	dma-noncoherent;
+
+	d0_uart0: serial@0x50900000 {
+		compatible = "snps,dw-apb-uart";
+		clock-frequency = <LSPCLK_FREQ>;
+		interrupt-parent = <&plic0>;
+		interrupts = <100>;
+		reg = <0x0 0x50900000 0x0 0x10000>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+	};
+
+	d0_uart1: serial@0x50910000 {
+		compatible = "snps,dw-apb-uart";
+		clock-frequency = <LSPCLK_FREQ>;
+		interrupt-parent = <&plic0>;
+		interrupts = <101>;
+		reg = <0x0 0x50910000 0x0 0x10000>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+	};
+
+	d0_uart2: serial@0x50920000 {
+		compatible = "snps,dw-apb-uart";
+		clock-frequency = <LSPCLK_FREQ>;
+		interrupt-parent = <&plic0>;
+		interrupts = <102>;
+		reg = <0x0 0x50920000 0x0 0x10000>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		status = "disabled";
+	};
+
+	d0_uart3: serial@0x50930000 {
+		compatible = "snps,dw-apb-uart";
+		clock-frequency = <LSPCLK_FREQ>;
+		interrupt-parent = <&plic0>;
+		interrupts = <103>;
+		reg = <0x0 0x50930000 0x0 0x10000>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		status = "disabled";
+	};
+
+	d0_uart4: serial@0x50940000 {
+		compatible = "snps,dw-apb-uart";
+		clock-frequency = <LSPCLK_FREQ>;
+		interrupt-parent = <&plic0>;
+		interrupts = <104>;
+		reg = <0x0 0x50940000 0x0 0x10000>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		status = "disabled";
+	};
+
+	d0_sys_con: scu_sys_con@0x51810000 {
+		compatible = "eswin,eic7700-scu-sys-con", "syscon", "simple-mfd";
+		#syscon-cells = <2>;
+		#size-cells = <2>;
+		reg = <0x0 0x51810000 0x0 0x8000>;
+	};
+
+	d0_sys_crg: sys-crg@51828000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0x000000 0x51828000 0x000000 0x80000>;
+		d0_reset: reset-controller {
+			compatible = "eswin,eic7700-reset";
+			#reset-cells = <2>;
+		};
+		d0_clock: clock-controller {
+			compatible = "eswin,eic7700-clock";
+			#clock-cells = <1>;
+		};
+	};
+
+	d0_hsp_sp_csr: hsp_sp_top_csr@0x50440000 {
+		compatible = "syscon";
+		#size-cells = <2>;
+		reg = <0x0 0x50440000 0x0 0x2000>;
+	};
+
+	smmu0: iommu@50c00000 {
+		compatible = "arm,smmu-v3";
+		eswin,syscfg = <&d0_sys_con 0x3fc>;
+		interrupt-parent = <&plic0>;
+		interrupts = <356>,
+			<360>,
+			<357>,
+			<358>;
+		interrupt-names = "eventq", "gerror", "priq", "cmdq-sync";
+		#iommu-cells = <1>;
+		reg = <0x0 0x50c00000 0x0 0x100000>;
+		resets = <&d0_reset TCU_RST_CTRL SW_TCU_AXI_RSTN>,
+			<&d0_reset TCU_RST_CTRL SW_TCU_CFG_RSTN>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_0>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_1>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_2>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_3>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_4>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_5>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_6>,
+			<&d0_reset TCU_RST_CTRL TBU_RSTN_7>;
+		reset-names = "axi_rst", "cfg_rst", "tbu0_rst", "tbu1_rst",
+			"tbu2_rst", "tbu3_rst", "tbu4_rst", "tbu5_rst",
+			"tbu6_rst", "tbu7_rst";
+		status = "disabled";
+	};
+
+	d0_dmac0: dma-controller-hsp@0x50430000 {
+		compatible = "snps,axi-dma-1.01a";
+		reg = <0x0 0x50430000 0x0 0x10000>;
+		interrupt-parent = <&plic0>;
+		interrupts = <57>;
+		clocks = <&d0_clock EIC7700_CLK_HSP_DMA0_CLK>;
+		clock-names = "core-clk";
+		#dma-cells = <2>;
+		dma-channels = <12>;
+		snps,dma-masters = <1>;
+		snps,priority = <0 1 2 3 4 5 6 7 8 9 10 11>;
+		snps,data-width = <2>;
+		snps,block-size = <0x80000 0x80000 0x80000 0x80000
+			0x80000 0x80000 0x80000 0x80000
+			0x80000 0x80000 0x80000 0x80000>;
+		snps,axi-max-burst-len = <16>;
+		snps,max-msize = <64>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_HSP_DMA0_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_DMA_PRSTN>;
+		reset-names = "arst", "prst";
+		iommus = <&smmu0 EIC7700_SID_DMA0>;
+		tbus = <EIC7700_TBUID_DMA0>;
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x104c>;
+		eswin,syscfg = <&d0_sys_con DMA1_SID_REG_OFFSET 0x370>;
+	};
+
+	d0_aon_dmac: dma-controller-aon@0x518c0000 {
+		compatible = "snps,axi-dma-1.01a";
+		#size-cells = <2>;
+		#address-cells = <2>;
+		reg = <0x0 0x518c0000 0x0 0x10000>;
+		interrupt-parent = <&plic0>;
+		interrupts = <289>;
+		clocks = <&d0_clock EIC7700_CLK_AONDMA_ACLK>;
+		clock-names = "core-clk";
+		#dma-cells = <2>;
+		dma-channels = <16>;
+		dma-ranges = <0x0 0x80000000 0x0 0x80000000 0x100 0x0>;
+		snps,dma-masters = <2>;
+		snps,priority = <0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15>;
+		snps,data-width = <3>;
+		snps,block-size = <0x80000 0x80000 0x80000 0x80000
+			0x80000 0x80000 0x80000 0x80000
+			0x80000 0x80000 0x80000 0x80000
+			0x80000 0x80000 0x80000 0x80000>;
+		snps,axi-max-burst-len = <32>;
+		snps,max-msize = <64>;
+		resets = <&d0_reset DMA1_RST_CTRL SW_DMA1_ARSTN>,
+			<&d0_reset DMA1_RST_CTRL SW_DMA1_HRSTN>;
+		reset-names = "arst", "prst";
+		iommus = <&smmu0 EIC7700_SID_DMA1>;
+		tbus = <EIC7700_TBUID_DMA1>;
+		eswin,syscfg = <&d0_sys_con DMA1_SID_REG_OFFSET 0x370>;
+	};
+
+	d0_gmac0: ethernet@50400000 {
+		compatible = "eswin,eic7700-qos-eth";
+		clocks = <&d0_clock EIC7700_CLK_HSP_ETH_APP_CLK>,
+			<&d0_clock EIC7700_CLK_HSP_ETH_CSR_CLK>,
+			<&d0_clock EIC7700_CLK_HSP_ETH0_CORE_CLK>;
+		clock-names = "app", "stmmaceth","tx";
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1030 0x100 0x108>;
+		eswin,syscrg_csr = <&d0_sys_crg 0x148 0x14c>;
+		id = <0>;
+		interrupts = <61>;
+		interrupt-names = "macirq";
+		interrupt-parent = <&plic0>;
+		phy-mode = "rgmii";
+		reg = <0x0 0x50400000 0x0 0x10000>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_HSP_ETH0_ARSTN>;
+		reset-names = "ethrst";
+		snps,axi-config = <&d0_stmmac_axi_setup>;
+		tbus = <EIC7700_TBUID_ETH>;
+		status = "disabled";
+		d0_stmmac_axi_setup: stmmac-axi-config {
+			snps,blen = <0 0 0 0 16 8 4>;
+			snps,rd_osr_lmt = <2>;
+			snps,wr_osr_lmt = <2>;
+			snps,lpi_en = <0>;
+		};
+	};
+
+	d0_gmac1: ethernet@50410000 {
+		compatible = "eswin,eic7700-qos-eth";
+		clocks = <&d0_clock EIC7700_CLK_HSP_ETH_APP_CLK>,
+			<&d0_clock EIC7700_CLK_HSP_ETH_CSR_CLK>,
+			<&d0_clock EIC7700_CLK_HSP_ETH1_CORE_CLK>;
+		clock-names = "app", "stmmaceth","tx";
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1034 0x200 0x208>;
+		eswin,syscrg_csr = <&d0_sys_crg 0x148 0x14c>;
+		id = <1>;
+		interrupts = <70>;
+		interrupt-names = "macirq";
+		interrupt-parent = <&plic0>;
+		phy-mode = "rgmii";
+		reg = <0x0 0x50410000 0x0 0x10000>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_HSP_ETH1_ARSTN>;
+		reset-names = "ethrst";
+		tbus = <EIC7700_TBUID_ETH>;
+		snps,axi-config = <&d0_stmmac_axi_setup_gmac1>;
+		status = "disabled";
+		d0_stmmac_axi_setup_gmac1: stmmac-axi-config {
+			snps,blen = <0 0 0 0 16 8 4>;
+			snps,rd_osr_lmt = <2>;
+			snps,wr_osr_lmt = <2>;
+			snps,lpi_en = <0>;
+		};
+	};
+
+	gpu0: gpu@51400000 {
+		compatible = "img,gpu";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		reg = <0x0 0x51400000 0x0 0xFFFFF>;
+		clocks = <&d0_clock EIC7700_CLK_GPU_ACLK>,
+			<&d0_clock EIC7700_CLK_GPU_GRAY_CLK>,
+			<&d0_clock EIC7700_CLK_GPU_CFG_CLK>;
+		clock-names = "aclk", "gray_clk", "cfg_clk";
+		resets =<&d0_reset GPU_RST_CTRL SW_GPU_AXI_RSTN>,
+			<&d0_reset GPU_RST_CTRL SW_GPU_CFG_RSTN>,
+			<&d0_reset GPU_RST_CTRL SW_GPU_GRAY_RSTN>,
+			<&d0_reset GPU_RST_CTRL SW_GPU_JONES_RSTN>,
+			<&d0_reset GPU_RST_CTRL SW_GPU_SPU_RSTN>;
+		reset-names = "axi", "cfg", "gray", "jones","spu";
+		interrupt-parent = <&plic0>;
+		interrupts = <15>;
+	};
+
+	d0_sata: sata@0x50420000{
+		compatible = "snps,eswin-ahci";
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1050>;
+		interrupt-parent = <&plic0>;
+		interrupt-names = "intrq", "msi", "pme";
+		interrupts = <58>, <59>, <60>;
+		iommus = <&smmu0 EIC7700_SID_SATA>;
+		ports-implemented = <0x1>;
+		reg = <0x0 0x50420000 0x0 0x10000>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_SATA_ASIC0_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_SATA_OOB_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_SATA_PMALIVE_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_SATA_RBC_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_SATA_ARSTN>;
+		reset-names = "asic0", "oob", "pmalive", "rbc", "apb";
+		#size-cells = <2>;
+		tbus = <EIC7700_TBUID_SATA>;
+		dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+	};
+
+	pcie: pcie@0x54000000 {
+		compatible = "eswin,eic7700-pcie";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		clocks = <&d0_clock EIC7700_CLK_PCIET_ACLK>,
+			<&d0_clock EIC7700_CLK_PCIET_CFG_CLK>,
+			<&d0_clock EIC7700_CLK_PCIET_CR_CLK>,
+			<&d0_clock EIC7700_CLK_PCIET_AUX_CLK>;
+		clock-names = "pcie_aclk", "pcie_cfg_clk",
+			"pcie_cr_clk", "pcie_aux_clk";
+		device_type = "pci";
+		#interrupt-cells = <1>;
+		interrupts = <220>, <179>, <180>, <181>,
+			<182>, <183>, <184>, <185>, <186>;
+		interrupt-names = "msi", "inta", "intb", "intc", "intd";
+		interrupt-parent = <&plic0>;
+		interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+		interrupt-map = <0x0 0x0 0x0 0x1 &plic0 179>,
+			<0x0 0x0 0x0 0x2 &plic0 180>,
+			<0x0 0x0 0x0 0x3 &plic0 181>,
+			<0x0 0x0 0x0 0x4 &plic0 182>;
+		iommus = <&smmu0 0xfe0000>;
+		iommu-map = <0x0 &smmu0 0xff0000 0xffffff>;
+		num-lanes = <0x4>;
+		resets = <&d0_reset PCIE_RST_CTRL SW_PCIE_CFG_RSTN>,
+			 <&d0_reset PCIE_RST_CTRL SW_PCIE_POWERUP_RSTN>,
+			 <&d0_reset PCIE_RST_CTRL SW_PCIE_PERST_N>;
+		reset-names = "pcie_cfg", "pcie_powerup", "pcie_pwren";
+		reg = <0x0 0x54000000 0x0 0x4000000>,
+			<0x0 0x40000000 0x0 0x800000>,
+		    <0x0 0x50000000 0x0 0x100000>;
+		reg-names = "dbi", "config", "mgmt";
+		bus-range = <0x0 0xff>;
+		ranges = <0x81000000  0x0 0x40800000  0x0 0x40800000  0x0   0x800000>,
+			 <0x82000000  0x0 0x41000000  0x0 0x41000000  0x0  0xf000000>,
+			 <0xc3000000 0x80 0x00000000 0x80 0x00000000 0x0a 0x00000000>;
+		gen-x = <3>;
+		lane-x = <4>;
+		tbus = <EIC7700_TBUID_PCIE>;
+		status = "disabled";
+	};
+
+	ssi0: spi@50810000 {
+		compatible = "snps,eic7700-spi";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x0 0x50810000 0x0 0x4000>;
+		spi-max-frequency = <4800000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_SSI0_PCLK>;
+		clock-names = "clk";
+		interrupt-parent = <&plic0>;
+		interrupts = <91>;
+		resets = <&d0_reset SSI_RST_CTRL SW_SSI_RST_N_0>;
+		reset-names = "spi";
+		eswin,spi_dma = <&d0_aon_dmac>;
+		dmas = <&d0_aon_dmac 38 3>, <&d0_aon_dmac 39 3>;
+		dma-names = "rx", "tx";
+		status = "disabled";
+	};
+
+	ssi1: spi@50814000 {
+		compatible = "snps,eic7700-spi";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x0 0x50814000 0x0 0x4000>;
+		spi-max-frequency = <4800000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_SSI1_PCLK>;
+		clock-names = "clk";
+		interrupt-parent = <&plic0>;
+		interrupts = <92>;
+		resets = <&d0_reset SSI_RST_CTRL SW_SSI_RST_N_1>;
+		reset-names = "spi";
+		eswin,spi_dma = <&d0_aon_dmac>;
+		dmas = <&d0_aon_dmac 36 4>, <&d0_aon_dmac 37 4>;
+		dma-names = "rx", "tx";
+		status = "disabled";
+	};
+
+	bootspi: spi@51800000 {
+		compatible = "eswin,bootspi";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&d0_clock EIC7700_CLK_CLK_BOOTSPI_CFG>,
+				<&d0_clock EIC7700_CLK_CLK_BOOTSPI>;
+		clock-names = "cfg_clk", "clk";
+		reg = <0x0 0x51800000 0x0 0x8000>,
+			<0x0 0x51828000 0x0 0x8000>,
+			<0x0 0x5c000000 0x0 0x8000>;
+		resets = <&d0_reset BOOTSPI_RST_CTRL SW_BOOTSPI_RSTN>;
+		reset-names = "rst";
+		reg-io-width = <4>;
+		spi-max-frequency = <4800000>;
+		status = "disabled";
+	};
+
+	sdhci_emmc: mmc@50450000 {
+		compatible = "eswin,emmc-sdhci-5.1";
+		#clock-cells = <0>;
+		clocks = <&d0_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>, <&d0_clock EIC7700_CLK_HSP_CFG_CLK>;
+		clock-names = "clk_xin", "clk_ahb";
+		clock-output-names = "emmc_cardclock";
+		assigned-clocks = <&d0_clock EIC7700_CLK_HSP_MSHC0_CORE_CLK>;
+		assigned-clock-rates = <200000000>;
+		reg = <0x0 0x50450000 0x0 0x10000>;
+		interrupt-parent = <&plic0>;
+		interrupts = <79>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_MSHC0_TXRX_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_MSHC0_PHY_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_EMMC_PRSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_EMMC_ARSTN>;
+		reset-names = "txrx_rst", "phy_rst", "prstn", "arstn";
+		disable-cqe-dcmd;
+		bus-width = <8>;
+		non-removable;
+		mmc-hs400-1_8v;
+		max-frequency = <200000000>;
+		#size-cells = <2>;
+		iommus = <&smmu0 EIC7700_SID_EMMC0>;
+		tbus = <EIC7700_TBUID_EMMC>;
+		dma-ranges = <0x0 0x00000000 0x0 0xc0000000 0x1 0x0>;
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1038>;
+		status = "disabled";
+	};
+
+	sdio0: mmc@0x50460000{
+		compatible = "eswin,sdhci-sdio";
+		reg = <0x0 0x50460000 0x0 0x10000>;
+		interrupt-parent = <&plic0>;
+		interrupts = <81>;
+		clocks =<&d0_clock EIC7700_CLK_HSP_MSHC1_CORE_CLK>,
+			<&d0_clock EIC7700_CLK_HSP_CFG_CLK>,
+			<&d0_clock EIC7700_SPLL2_FOUT3>,
+			<&d0_clock EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1>;
+		clock-names ="clk_xin","clk_ahb","clk_spll2_fout3","clk_mux1_1";
+		clock-output-names = "sdio0_cardclock";
+		#clock-cells = <0>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_MSHC1_TXRX_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_MSHC1_PHY_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD0_PRSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD0_ARSTN>;
+		reset-names = "txrx_rst","phy_rst","prstn","arstn";
+		core-clk-reg = <0x51828164>;
+		clock-frequency = <208000000>;
+		max-frequency = <208000000>;
+		#size-cells = <2>;
+		dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+		iommus = <&smmu0 EIC7700_SID_SD0>;
+		tbus = <EIC7700_TBUID_SD>;
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x103c>;
+		bus-width = <4>;
+		sdio-id = <0>;
+		status = "disabled";
+	};
+
+	sdio1: mmc@0x50470000{
+		compatible = "eswin,sdhci-sdio";
+		reg = <0x0 0x50470000 0x0 0x10000>;
+		interrupt-parent = <&plic0>;
+		interrupts = <83>;
+		clocks =<&d0_clock EIC7700_CLK_HSP_MSHC2_CORE_CLK>,
+			<&d0_clock EIC7700_CLK_HSP_CFG_CLK>,
+			<&d0_clock EIC7700_SPLL2_FOUT3>,
+			<&d0_clock EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1>;
+		clock-names ="clk_xin","clk_ahb","clk_spll2_fout3","clk_mux1_1";
+		clock-output-names = "sdio1_cardclock";
+		#clock-cells = <0>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_MSHC2_TXRX_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_MSHC2_PHY_RSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD1_PRSTN>,
+			<&d0_reset HSPDMA_RST_CTRL SW_HSP_SD1_ARSTN>;
+		reset-names = "txrx_rst","phy_rst","prstn","arstn";
+		core-clk-reg = <0x51828168>;
+		clock-frequency = <208000000>;
+		max-frequency = <208000000>;
+		#size-cells = <2>;
+		dma-ranges = <0x0 0x20000000 0x0 0xc0000000 0x0 0x40000000>;
+		iommus = <&smmu0 EIC7700_SID_SD1>;
+		tbus = <EIC7700_TBUID_SD>;
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1040>;
+		bus-width = <4>;
+		sdio-id = <1>;
+		status = "disabled";
+	};
+
+	pvt0: pvt@0x50b00000 {
+		compatible = "eswin,eswin-pvt";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&d0_clock EIC7700_CLK_PVT_CLK_0>;
+		clock-names = "pvt_clk";
+		interrupts = <349>;
+		interrupt-parent = <&plic0>;
+		resets = <&d0_reset PVT_RST_CTRL SW_PVT_RST_N_0>;
+		reset-names = "pvt_rst";
+		reg = <0x0 0x50b00000 0x0 0x10000>;
+		status = "disabled";
+	};
+
+	pvt1: pvt@0x52360000 {
+		compatible = "eswin,eswin-pvt";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&d0_clock EIC7700_CLK_PVT_CLK_1>;
+		clock-names = "pvt_clk";
+		interrupts = <350>;
+		interrupt-parent = <&plic0>;
+		resets = <&d0_reset PVT_RST_CTRL SW_PVT_RST_N_1>;
+		reset-names = "pvt_rst";
+		reg = <0x0 0x52360000 0x0 0x20000>;
+		status = "disabled";
+	};
+
+	fan_control: fan_control@50b50000 {
+		compatible = "eswin-fan-control";
+		reg = <0x0 0x50b50000 0x0 0x10000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_FAN_PCLK>;
+		clock-names = "pclk";
+		interrupt-parent = <&plic0>;
+		interrupt-names = "fanirq";
+		interrupts = <354>;
+		pulses-per-revolution = <2>;
+		pwm-minimum-period = <1000>;
+		pwms = <&pwm0 0 100000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_fan_tach_default>;
+		resets = <&d0_reset FAN_RST_CTRL SW_FAN_RST_N>;
+		reset-names = "fan_rst";
+		status = "disabled";
+	};
+
+	d0_i2c0: i2c@50950000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x50950000 0x0 0x8000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C0_PCLK>;
+		clock-names = "pclk";
+		clock-frequency = <100000>;
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_0>;
+		reset-names = "rst";
+		interrupts = <105>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	d0_i2c1: i2c@50960000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x50960000 0x0 0x10000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C1_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_1>;
+		reset-names = "rst";
+		interrupts = <106>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	d0_i2c2: i2c@50970000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x50970000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C2_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_2>;
+		reset-names = "rst";
+		interrupts = <107>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	d0_i2c3: i2c@50980000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x50980000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C3_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_3>;
+		reset-names = "rst";
+		interrupts = <108>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	d0_i2c4: i2c@50990000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x50990000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C4_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_4>;
+		reset-names = "rst";
+		interrupts = <109>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	d0_i2c5: i2c@509a0000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x509a0000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C5_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_5>;
+		reset-names = "rst";
+		interrupts = <110>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	d0_i2c6: i2c@509b0000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x509b0000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C6_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_6>;
+		reset-names = "rst";
+		interrupts = <111>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	d0_i2c7: i2c@509c0000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x509c0000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C7_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_7>;
+		reset-names = "rst";
+		interrupts = <112>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	d0_i2c8: i2c@509d0000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x509d0000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C8_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_8>;
+		reset-names = "rst";
+		interrupts = <113>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	d0_i2c9: i2c@509e0000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x509e0000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_LSP_I2C9_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C_RST_CTRL SW_I2C_RST_N_9>;
+		reset-names = "rst";
+		interrupts = <114>;
+		interrupt-parent = <&plic0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	d0_aon_i2c0: i2c@51830000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x51830000 0x0 0x8000>;
+		clock-frequency = <100000>;
+		clocks = <&d0_clock EIC7700_CLK_AON_I2C0_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset I2C0_RST_CTRL SW_I2C0_PRSTN>;
+		reset-names = "rst";
+		interrupts = <290>;
+		interrupt-parent = <&plic0>;
+		eswin,i2c_dma = <&d0_aon_dmac>;
+		dma-names = "rx", "tx";
+		/*
+		* dmas : DMA specifiers
+		* &d0_aon_dmac : dma controller
+		* 41 : i2c0 aon dma handshake number
+		* 0xff : no need to select to dma controller
+		*/
+		dmas = <&d0_aon_dmac 41 0xff>, <&d0_aon_dmac 42 0xff>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	d0_aon_i2c1: i2c@51838000 {
+			compatible = "snps,designware-i2c";
+			reg = <0x0 0x51838000 0x0 0x8000>;
+			clock-frequency = <100000>;
+			clocks = <&d0_clock EIC7700_CLK_AON_I2C1_PCLK>;
+			clock-names = "pclk";
+			resets = <&d0_reset I2C1_RST_CTRL SW_I2C1_PRSTN>;
+			reset-names = "rst";
+			interrupts = <291>;
+			interrupt-parent = <&plic0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+	};
+
+	pinctrl: pinctrl@0x51600080 {
+		compatible = "eswin,eic7700-pinctrl";
+		reg = <0x0 0x51600080 0x0 0x1FFF80>;
+		status = "disabled";
+	};
+
+	gpio0: gpio@51600000 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dw-apb-gpio";
+		reg = <0x0 0x51600000 0x0 0x80>;
+
+		porta: gpio-port@0 {
+			compatible = "snps,dw-apb-gpio-port";
+			gpio-controller;
+			#gpio-cells = <2>;
+			ngpios = <32>;
+			reg = <0>;
+			interrupt-parent = <&plic0>;
+			interrupts = <303 304 305 306 307 308 309 310 311 312 313
+				314 315 316 317 318 319 320 321 322 323 324 325 326
+				327 328 329 330 331 332 333 334>;
+		};
+
+		portb: gpio-port@1 {
+			compatible = "snps,dw-apb-gpio-port";
+			gpio-controller;
+			#gpio-cells = <2>;
+			ngpios = <32>;
+			reg = <1>;
+		};
+
+		portc: gpio-port@2 {
+			compatible = "snps,dw-apb-gpio-port";
+			gpio-controller;
+			#gpio-cells = <2>;
+			ngpios = <32>;
+			reg = <2>;
+		};
+
+		portd: gpio-port@3 {
+			compatible = "snps,dw-apb-gpio-port";
+			gpio-controller;
+			#gpio-cells = <2>;
+			ngpios = <16>;
+			reg = <3>;
+		};
+	};
+
+	pwm0: pwm@0x50818000 {
+		compatible = "eswin,pwm-eswin";
+		#pwm-cells = <2>;
+		clock-names = "pwm","pclk";
+		clocks = <&d0_clock EIC7700_CLK_LSP_TIMER_PCLK>;
+		clock-frequency = <200000000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pwm0_default>;
+		reg = <0x0 0x50818000 0x0 0x4000>;
+		resets = <&d0_reset TIMER_RST_CTRL SW_TIMER_RST_N>;
+		reset-names = "pwmrst";
+		status = "disabled";
+	};
+
+	wdt0: watchdog@0x50800000 {
+		compatible = "snps,dw-wdt";
+		reg = <0x0 0x50800000 0x0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_LSP_WDT0_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_0>;
+		reset-names = "rst";
+		interrupts = <87>;
+		interrupt-parent = <&plic0>;
+		status = "disabled";
+	};
+
+	wdt1: watchdog@0x50804000 {
+		compatible = "snps,dw-wdt";
+		reg = <0x0 0x50804000 0x0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_LSP_WDT1_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_1>;
+		reset-names = "rst";
+		interrupts = <88>;
+		interrupt-parent = <&plic0>;
+		status = "disabled";
+	};
+
+	wdt2: watchdog@0x50808000 {
+		compatible = "snps,dw-wdt";
+		reg = <0x0 0x50808000 0x0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_LSP_WDT2_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_2>;
+		reset-names = "rst";
+		interrupts = <89>;
+		interrupt-parent = <&plic0>;
+		status = "disabled";
+	};
+
+	wdt3: watchdog@0x5080c000 {
+		compatible = "snps,dw-wdt";
+		reg = <0x0 0x5080c000 0x0 0x4000>;
+		clocks =<&d0_clock EIC7700_CLK_LSP_WDT3_PCLK>;
+		clock-names = "pclk";
+		resets = <&d0_reset WDT_RST_CTRL SW_WDT_RST_N_3>;
+		reset-names = "rst";
+		interrupts = <90>;
+		interrupt-parent = <&plic0>;
+		status = "disabled";
+	};
+
+	die0_rtc: rtc@51818000 {
+		compatible = "eswin,eic7700-rtc";
+		reg = <0x0 0x51818000 0x0 0x400>;
+		eswin,syscfg = <&d0_sys_con 0x3c0>;
+		interrupt-parent = <&plic0>;
+		interrupts = <292>;
+		clocks = <&d0_clock EIC7700_CLK_CLK_RTC>;
+		clock-names = "rtcclk";
+		clock-frequency = <15625>;
+		resets = <&d0_reset RTC_RST_CTRL SW_RTC_RSTN>;
+		reset-names = "rtcrst";
+		status = "disabled";
+	};
+
+	d0_i2s0: i2s0@50200000 {
+		compatible = "snps,i2s";
+		clocks = <&d0_clock EIC7700_CLK_VO_I2S_MCLK>;
+		clock-names = "mclk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#sound-dai-cells = <0x00000000>;
+		reg = <0x0 0x50200000 0x0 0x10000>;
+		dma-names = "rx", "tx";
+		dmas = <&d0_aon_dmac 4 0>, <&d0_aon_dmac 5 0>;
+		vo_mclk_sel,syscrg = <&d0_sys_crg 0x1bc>;
+		resets = <&d0_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+			<&d0_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+			<&d0_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+		reset-names = "i2srst", "i2sprst", "voprst";
+	};
+
+	d0_i2s1: i2s1@50210000 {
+		compatible = "snps,i2s";
+		clocks = <&d0_clock EIC7700_CLK_VO_I2S_MCLK>;
+		clock-names = "mclk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#sound-dai-cells = <0x00000000>;
+		reg = <0x0 0x50210000 0x0 0x10000>;
+		dma-names = "rx", "tx";
+		dmas = <&d0_aon_dmac 2 1>, <&d0_aon_dmac 3 1>;
+		vo_mclk_sel,syscrg = <&d0_sys_crg 0x1bc>;
+		resets = <&d0_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+			<&d0_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+			<&d0_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+		reset-names = "i2srst", "i2sprst", "voprst";
+	};
+
+	d0_i2s2: i2s2@50220000 {
+		compatible = "snps,i2s";
+		clocks = <&d0_clock EIC7700_CLK_VO_I2S_MCLK>;
+		clock-names = "mclk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#sound-dai-cells = <0x00000000>;
+		reg = <0x0 0x50220000 0x0 0x10000>;
+		dma-names = "rx", "tx";
+		dmas = <&d0_aon_dmac 0 2>, <&d0_aon_dmac 1 2>;
+		vo_mclk_sel,syscrg = <&d0_sys_crg 0x1bc>;
+		resets = <&d0_reset VO_I2SRST_CTRL SW_VO_I2S_RSTN>,
+			<&d0_reset VO_I2SRST_CTRL SW_VO_I2S_PRSTN>,
+			<&d0_reset VO_PHYRST_CTRL SW_VO_PRSTN>;
+		reset-names = "i2srst", "i2sprst", "voprst";
+		status = "disabled";
+	};
+
+	d0_graphcard0: graphcard0 {
+		compatible = "audio-graph-card";
+	};
+
+	d0_graphcard1: graphcard1 {
+		compatible = "audio-graph-card";
+	};
+
+	d0_graphcard2: graphcard2 {
+		compatible = "audio-graph-card";
+		status = "disabled";
+	};
+
+	video_output: display-subsystem {
+		compatible = "eswin,display-subsystem";
+		ports = <&dc_out>;
+	};
+
+	dc: display_control@502c0000 {
+		compatible = "eswin,dc";
+		reg = <0x0 0x502c0000 0x0 0x100>, <0x0 0x502c0180 0x0 0x700>, <0x0 0x502c1400 0x0 0x1400>;
+		interrupt-parent = <&plic0>;
+		interrupts = <238>;
+		clocks = <&d0_clock EIC7700_CLK_VO_CFG_CLK>,
+			<&d0_clock EIC7700_CLK_VO_PIXEL_CLK>,
+			<&d0_clock EIC7700_CLK_VO_ACLK>,
+			<&d0_clock EIC7700_SPLL0_FOUT1>,
+			<&d0_clock EIC7700_MUX_U_VO_ACLK_ROOT_2MUX1_GFREE>;
+		clock-names = "cfg_clk", "pix_clk", "axi_clk", "spll0_fout1", "vo_mux";
+		resets = <&d0_reset VO_RST_CTRL SW_VO_AXI_RSTN>,
+			<&d0_reset VO_RST_CTRL SW_VO_CFG_RSTN>,
+			<&d0_reset VO_RST_CTRL SW_VO_DC_RSTN>,
+			<&d0_reset VO_RST_CTRL SW_VO_DC_PRSTN>;
+		reset-names = "vo_arst", "vo_prst", "dc_arst", "dc_prst";
+
+		dc_out: port {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dc_out_dsi: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&dsi_input>;
+			};
+
+			dc_out_dpi1: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&vd_input>;
+			};
+
+			dc_out_hdmi: endpoint@2 {
+				reg = <2>;
+				remote-endpoint = <&hdmi_in_dc>;
+			};
+		};
+	};
+
+	virtual_display: es_wb {
+		compatible = "eswin,virtual_display";
+		bpp = /bits/ 8 <8>;
+		status = "disabled";
+
+		port {
+			vd_input: endpoint {
+				remote-endpoint = <&dc_out_dpi1>;
+			};
+		};
+	};
+
+	dsi_output: dsi-output {
+		compatible = "eswin,dsi-encoder";
+		status = "disabled";
+	};
+
+	dsi_controller: mipi_dsi@50270000 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "eswin,dsi";
+		reg = <0x0 0x50270000 0x0 0x10000>;
+		clocks = <&d0_clock EIC7700_CLK_CLK_MIPI_TXESC>;
+		clock-names = "pclk";
+		resets = <&d0_reset VO_PHYRST_CTRL SW_VO_MIPI_PRSTN>;
+		reset-names ="phyrstn";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				dsi_input: endpoint {
+					remote-endpoint = <&dc_out_dsi>;
+				};
+			};
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+
+				mipi_dsi_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
+
+		dsi_panel:dsi_panel@0 {
+			compatible = "eswin,generic-panel";
+			reg = <0>;
+
+			port {
+				panel_in: endpoint {
+					remote-endpoint = <&mipi_dsi_out>;
+				};
+			};
+		};
+	};
+
+	dw_hdmi: hdmi@502a0000 {
+		compatible = "eswin,eswin-dw-hdmi";
+		reg = <0x0 0x502a0000 0x0 0x20000>;
+		pinctrl-names = "default";
+		interrupt-parent = <&plic0>;
+		interrupts = <274>;
+		clocks = <&d0_clock EIC7700_CLK_VO_CFG_CLK>,
+			<&d0_clock EIC7700_CLK_VO_CEC_CLK>,
+			<&d0_clock EIC7700_CLK_VO_CR_CLK>;
+		clock-names = "iahb", "cec", "isfr";
+		reg-io-width = <4>;
+		ddc-i2c-scl-high-time-ns = <4708>;
+		ddc-i2c-scl-low-time-ns = <4916>;
+		#sound-dai-cells = <0>;
+		resets = <&d0_reset VO_PHYRST_CTRL SW_VO_HDMI_PRSTN>,
+			<&d0_reset VO_PHYRST_CTRL SW_HDMI_PHYCTRL_RSTN>,
+			<&d0_reset VO_PHYRST_CTRL SW_VO_HDMI_RSTN>;
+		reset-names = "prstn", "phyrstn", "rstn";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			port@0 {
+				reg = <0>;
+				hdmi_in_dc: endpoint@0 {
+					remote-endpoint = <&dc_out_hdmi>;
+				};
+			};
+		};
+	};
+
+	dw_hdmi_hdcp2: hdmi-hdcp2@50290000 {
+		compatible = "eswin,dw-hdmi-hdcp2";
+		reg = <0x0 0x50290000 0x0 0x10000>;
+		interrupt-parent = <&plic0>;
+		interrupts = <275>;
+		clocks = <&d0_clock EIC7700_CLK_VO_CFG_CLK>,
+			<&d0_clock EIC7700_CLK_VO_HDMI_IESMCLK>;
+		clock-names ="pclk_hdcp2", "hdcp2_clk_hdmi";
+		status = "disabled";
+	};
+
+	d0_usbdrd3_0: usb0@50480000 {
+		compatible = "eswin,eic7700-dwc3";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		clocks =<&d0_clock EIC7700_GATE_HSP_USB0_SUSPEND_CLK>;
+		clock-names = "suspend";
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x800 0x808 0x83c 0x840>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_USB0_VAUX_RSTN>;
+		reset-names = "vaux";
+		ranges;
+		status = "disabled";
+		d0_usbdrd_dwc3_0: dwc3@50480000 {
+			compatible = "snps,dwc3";
+			reg = <0x0 0x50480000 0x0 0x10000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			interrupt-parent = <&plic0>;
+			interrupts = <85>;
+			interrupt-names = "peripheral";
+			dr_mode = "peripheral";
+			phy_type = "utmi";
+			maximum-speed = "high-speed";
+			iommus = <&smmu0 EIC7700_SID_USB0>;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1044>;
+			dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+			snps,dis_enblslpm_quirk;
+			snps,dis-u2-freeclk-exists-quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,parkmode-disable-ss-quirk;
+			status = "disabled";
+			numa-node-id = <0>;
+			tbus = <EIC7700_TBUID_USB>;
+		};
+	};
+
+	d0_usbdrd3_1: usb1@50490000  {
+		compatible = "eswin,eic7700-dwc3";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		clocks =<&d0_clock EIC7700_GATE_HSP_USB1_SUSPEND_CLK>;
+		clock-names = "suspend";
+		eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x900 0x908 0x93c 0x940>;
+		resets = <&d0_reset HSPDMA_RST_CTRL SW_USB1_VAUX_RSTN>;
+		reset-names = "vaux";
+		ranges;
+		status = "disabled";
+		d0_usbdrd_dwc3_1: dwc3@50490000 {
+			compatible = "snps,dwc3";
+			reg = <0x0 0x50490000 0x0 0x10000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			interrupt-parent = <&plic0>;
+			interrupts = <86>;
+			interrupt-names = "host";
+			dr_mode = "host";
+			phy_type = "utmi";
+			maximum-speed = "high-speed";
+			iommus = <&smmu0 EIC7700_SID_USB1>;
+			eswin,hsp_sp_csr = <&d0_hsp_sp_csr 0x1048>;
+			dma-ranges = <0x0 0x0 0x0 0xc0000000 0x200 0x0>;
+			snps,dis_enblslpm_quirk;
+			snps,dis-u2-freeclk-exists-quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,parkmode-disable-ss-quirk;
+			status = "disabled";
+			numa-node-id = <0>;
+			tbus = <EIC7700_TBUID_USB>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/eswin/hifive-premier-p550-pinctrl.dtsi b/arch/riscv/boot/dts/eswin/hifive-premier-p550-pinctrl.dtsi
new file mode 100644
index 000000000000..9b0e18db1d2b
--- /dev/null
+++ b/arch/riscv/boot/dts/eswin/hifive-premier-p550-pinctrl.dtsi
@@ -0,0 +1,1339 @@
+#include "eic7700.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+    #address-cells = <2>;
+    #size-cells = <2>;
+    soc {
+        pinctrl: pinctrl@0x51600080 {
+			compatible = "eswin,eic7700-pinctrl", "syscon";
+			reg = <0x0 0x51600080 0x0 0x1FFF80>;
+			status = "disabled";
+            //func0
+            pinctrl_sdio0_default: sdio0-default{
+				mux {
+					groups = "sdio0_group";
+					function = "sdio0_func";
+				};
+			};
+            pinctrl_sdio1_default: sdio1-default{
+				mux {
+					groups = "sdio1_group";
+					function = "sdio1_func";
+				};
+			};
+            pinctrl_por_sel_default: por_sel-default{
+				mux {
+					groups = "por_sel_group";
+					function = "por_sel_func";
+				};
+			};
+            pinctrl_jtag0_default: jtag0-default{
+				mux {
+					groups = "jtag0_group";
+					function = "jtag0_func";
+				};
+			};
+            pinctrl_jtag1_default: jtag1-default{
+				mux {
+					groups = "jtag1_group";
+					function = "jtag1_func";
+				};
+			};
+            pinctrl_jtag2_default: jtag2-default{
+				mux {
+					groups = "jtag2_group";
+					function = "jtag2_func";
+				};
+			};
+            pinctrl_pcie_default: pcie-default{
+				mux{
+					groups = "pcie_group";
+					function = "pcie_func";
+				};
+			};
+            pinctrl_hdmi_default: hdmi-default{
+				mux{
+					groups = "hdmi_group";
+					function = "hdmi_func";
+				};
+			};
+            pinctrl_rgmii0_default: rgmii0-default{
+				mux {
+					groups = "rgmii0_group";
+					function = "rgmii0_func";
+				};
+			};
+            pinctrl_rgmii1_default: rgmii1-default{
+				mux{
+					groups = "rgmii1_group";
+					function = "rgmii1_func";
+				};
+			};
+            pinctrl_spi0_default: spi0-default{
+				mux {
+					groups = "spi0_group";
+					function = "spi0_func";
+				};
+			};
+            pinctrl_spi1_default: spi1-default{
+				mux{
+					groups = "spi1_group";
+					function = "spi1_func";
+				};
+			};
+            pinctrl_spi3_default: spi3-default{
+				mux {
+					groups = "spi3_group";
+					function = "spi3_func";
+				};
+			};
+            pinctrl_por_time_sel0_default: por_time_sel0-default{
+				mux {
+					groups = "por_time_sel0_group";
+					function = "por_time_sel0_func";
+				};
+			};
+            pinctrl_por_time_sel1_default: por_time_sel1-default{
+				mux {
+					groups = "por_time_sel1_group";
+					function = "por_time_sel1_func";
+				};
+			};
+            pinctrl_i2s0_default: i2s0-default{
+				mux {
+					groups = "i2s0_group";
+					function = "i2s0_func";
+				};
+			};
+            pinctrl_i2s1_default: i2s1-default{
+				mux {
+					groups = "i2s1_group";
+					function = "i2s1_func";
+				};
+			};
+            pinctrl_i2s2_default: i2s2-default{
+				mux {
+					groups = "i2s2_group";
+					function = "i2s2_func";
+				};
+			};
+            pinctrl_usb0_pwren_default: usb0_pwren-default{
+				mux {
+					groups = "usb0_pwren_group";
+					function = "usb0_pwren_func";
+				};
+			};
+            pinctrl_usb1_pwren_default: usb1_pwren-default{
+				mux {
+					groups = "usb1_pwren_group";
+					function = "usb1_pwren_func";
+				};
+			};
+            pinctrl_i2c0_default: i2c0-default{
+				mux {
+					groups = "i2c0_group";
+					function = "i2c0_func";
+				};
+			};
+            pinctrl_i2c1_default: i2c1-default{
+				mux {
+					groups = "i2c1_group";
+					function = "i2c1_func";
+				};
+			};
+            pinctrl_i2c2_default: i2c2-default{
+				mux {
+					groups = "i2c2_group";
+					function = "i2c2_func";
+				};
+			};
+            pinctrl_i2c3_default: i2c3-default{
+				mux {
+					groups = "i2c3_group";
+					function = "i2c3_func";
+				};
+			};
+            pinctrl_i2c4_default: i2c4-default{
+				mux {
+					groups = "i2c4_group";
+					function = "i2c4_func";
+				};
+			};
+            pinctrl_i2c5_default: i2c5-default{
+				mux {
+					groups = "i2c5_group";
+					function = "i2c5_func";
+				};
+			};
+            pinctrl_i2c8_default: i2c8-default{
+				mux {
+					groups = "i2c8_group";
+					function = "i2c8_func";
+				};
+			};
+            pinctrl_i2c10_default: i2c10-default{
+				mux {
+					groups = "i2c10_group";
+					function = "i2c10_func";
+				};
+			};
+            pinctrl_i2c11_default: i2c11-default{
+				mux {
+					groups = "i2c11_group";
+					function = "i2c11_func";
+				};
+			};
+            pinctrl_uart0_default: uart0-default{
+				mux {
+					groups = "uart0_group";
+					function = "uart0_func";
+				};
+			};
+            pinctrl_uart1_default: uart1-default{
+				mux {
+					groups = "uart1_group";
+					function = "uart1_func";
+				};
+			};
+            pinctrl_uart2_default: uart2-default{
+				mux {
+					groups = "uart2_group";
+					function = "uart2_func";
+				};
+			};
+            pinctrl_pwm0_default: pwm0-default{
+                mux {
+                    groups = "pwm0_group";
+                    function = "pwm0_func";
+                };
+            };
+            pinctrl_fan_tach_default: fan_tach-default{
+				mux {
+					groups = "fan_tach_group";
+					function = "fan_tach_func";
+				};
+			};
+            pinctrl_mipi_csi0_default: mipi_csi0-default{
+				mux {
+					groups = "mipi_csi0_group";
+					function = "mipi_csi0_func";
+				};
+			};
+            pinctrl_mipi_csi1_default: mipi_csi1-default{
+				mux {
+					groups = "mipi_csi1_group";
+					function = "mipi_csi1_func";
+				};
+			};
+            pinctrl_mipi_csi2_default: mipi_csi2-default{
+				mux {
+					groups = "mipi_csi2_group";
+					function = "mipi_csi2_func";
+				};
+			};
+            pinctrl_mipi_csi3_default: mipi_csi3-default{
+				mux {
+					groups = "mipi_csi3_group";
+					function = "mipi_csi3_func";
+				};
+			};
+            pinctrl_mipi_csi4_default: mipi_csi4-default{
+				mux {
+					groups = "mipi_csi4_group";
+					function = "mipi_csi4_func";
+				};
+			};
+            pinctrl_mipi_csi5_default: mipi_csi5-default{
+				mux {
+					groups = "mipi_csi5_group";
+					function = "mipi_csi5_func";
+				};
+			};
+            pinctrl_s_mode_default: s_mode-default{
+				mux {
+					groups = "s_mode_group";
+					function = "s_mode_func";
+				};
+			};
+            pinctrl_pinmux_ddr_refclk_sel_default: pinmux_ddr_refclk_sel-default{
+				mux {
+					groups = "pinmux_ddr_refclk_sel_group";
+					function = "pinmux_ddr_refclk_sel_func";
+				};
+			};
+            pinctrl_boot_sel_default: boot_sel-default{
+				mux {
+					groups = "boot_sel_group";
+					function = "boot_sel_func";
+				};
+			};
+            pinctrl_lpddr_ref_clk_default: lpddr_ref_clk-default{
+				mux {
+					groups = "lpddr_ref_clk_group";
+					function = "lpddr_ref_clk_func";
+				};
+			};
+
+            //func1
+            pinctrl_spi2_default: spi2-default{
+				mux1 {
+					groups = "spi2_clk_group";
+					function = "spi2_clk_func";
+				};
+				conf1 {
+					groups = "spi2_clk_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux2 {
+					groups = "spi2_d0_group";
+					function = "spi2_d0_func";
+				};
+				conf2 {
+					groups = "spi2_d0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux3 {
+					groups = "spi2_d1_d2_d3_group";
+					function = "spi2_d1_d2_d3_func";
+				};
+				conf3 {
+					groups = "spi2_d1_d2_d3_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+					bias-pull-down = <0>;
+				};
+				mux4 {
+					groups = "spi2_cs_group";
+					function = "spi2_cs_func";
+				};
+			};
+
+            pinctrl_sata_act_led_default: sata_act_led-default{
+				mux {
+					groups = "sata_act_led_group";
+					function = "sata_act_led_func";
+				};
+				conf {
+					groups = "sata_act_led_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_emmc_led_control_default: emmc_led_control-default{
+				mux {
+					groups = "emmc_led_control_group";
+					function = "emmc_led_control_func";
+				};
+				conf {
+					groups = "emmc_led_control_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_sd0_led_control_default: sd0_led_control-default{
+				mux {
+					groups = "sd0_led_control_group";
+					function = "sd0_led_control_func";
+				};
+			};
+            pinctrl_sd1_led_control_default: sd1_led_control-default{
+				mux {
+					groups = "sd1_led_control_group";
+					function = "sd1_led_control_func";
+				};
+			};
+            pinctrl_i2c6_default: i2c6-default{
+				mux {
+					groups = "i2c6_group";
+					function = "i2c6_func";
+				};
+			};
+            pinctrl_i2c7_default: i2c7-default{
+				mux {
+					groups = "i2c7_group";
+					function = "i2c7_func";
+				};
+			};
+            pinctrl_i2c9_default: i2c9-default{
+				mux {
+					groups = "i2c9_group";
+					function = "i2c9_func";
+				};
+			};
+            pinctrl_pwm1_default: pwm1-default{
+				mux {
+					groups = "pwm1_group";
+					function = "pwm1_func";
+				};
+			};
+            pinctrl_pwm2_default: pwm2-default{
+				mux {
+					groups = "pwm2_group";
+					function = "pwm2_func";
+				};
+			};
+            pinctrl_mipi_csi_xtrig_default: mipi_csi_xtrig-default{
+				mux {
+					groups = "mipi_csi_xtrig_group";
+					function = "mipi_csi_xtrig_func";
+				};
+			};
+
+            //func3
+            pinctrl_uart3_default: uart3-default{
+				mux {
+					groups = "uart3_group";
+					function = "uart3_func";
+				};
+			};
+            pinctrl_uart4_default: uart4-default{
+				mux {
+					groups = "uart4_group";
+					function = "uart4_func";
+				};
+			};
+
+            //gpio
+            pinctrl_gpio0_default: gpio0-default{
+				mux {
+					groups = "gpio0_group";
+					function = "gpio0_func";
+				};
+				conf {
+					groups = "gpio0_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio1_default: gpio1-default{
+				mux {
+					groups = "gpio1_group";
+					function = "gpio1_func";
+				};
+			};
+            pinctrl_gpio2_default: gpio2-default{
+				mux {
+					groups = "gpio2_group";
+					function = "gpio2_func";
+				};
+			};
+            pinctrl_gpio3_default: gpio3-default{
+				mux {
+					groups = "gpio3_group";
+					function = "gpio3_func";
+				};
+			};
+            pinctrl_gpio4_default: gpio4-default{
+				mux {
+					groups = "gpio4_group";
+					function = "gpio4_func";
+				};
+			};
+            pinctrl_gpio5_default: gpio5-default{
+				mux {
+					groups = "gpio5_group";
+					function = "gpio5_func";
+				};
+				conf {
+					groups = "gpio5_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio6_default: gpio6-default{
+				mux {
+					groups = "gpio6_group";
+					function = "gpio6_func";
+				};
+				conf {
+					groups = "gpio6_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio7_default: gpio7-default{
+				mux {
+					groups = "gpio7_group";
+					function = "gpio7_func";
+				};
+				conf {
+					groups = "gpio7_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio8_default: gpio8-default{
+				mux {
+					groups = "gpio8_group";
+					function = "gpio8_func";
+				};
+				conf {
+					groups = "gpio8_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio9_default: gpio9-default{
+				mux {
+					groups = "gpio9_group";
+					function = "gpio9_func";
+				};
+				conf {
+					groups = "gpio9_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio10_default: gpio10-default{
+				mux {
+					groups = "gpio10_group";
+					function = "gpio10_func";
+				};
+				conf {
+					groups = "gpio10_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio11_default: gpio11-default{
+				mux {
+					groups = "gpio11_group";
+					function = "gpio11_func";
+				};
+				conf {
+					groups = "gpio11_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio12_default: gpio12-default{
+				mux {
+					groups = "gpio12_group";
+					function = "gpio12_func";
+				};
+			};
+            pinctrl_gpio13_default: gpio13-default{
+				mux {
+					groups = "gpio13_group";
+					function = "gpio13_func";
+				};
+			};
+            pinctrl_gpio14_default: gpio14-default{
+				mux {
+					groups = "gpio14_group";
+					function = "gpio14_func";
+				};
+			};
+            pinctrl_gpio15_default: gpio15-default{
+				mux {
+					groups = "gpio15_group";
+					function = "gpio15_func";
+				};
+			};
+            pinctrl_gpio16_default: gpio16-default{
+				mux {
+					groups = "gpio16_group";
+					function = "gpio16_func";
+				};
+				conf {
+					groups = "gpio16_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio17_default: gpio17-default{
+				mux {
+					groups = "gpio17_group";
+					function = "gpio17_func";
+				};
+				conf {
+					groups = "gpio17_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio18_default: gpio18-default{
+				mux {
+					groups = "gpio18_group";
+					function = "gpio18_func";
+				};
+				conf {
+					groups = "gpio18_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio19_default: gpio19-default{
+				mux {
+					groups = "gpio19_group";
+					function = "gpio19_func";
+				};
+				conf {
+					groups = "gpio19_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio20_default: gpio20-default{
+				mux {
+					groups = "gpio20_group";
+					function = "gpio20_func";
+				};
+				conf {
+					groups = "gpio20_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio21_default: gpio21-default{
+				mux {
+					groups = "gpio21_group";
+					function = "gpio21_func";
+				};
+				conf {
+					groups = "gpio21_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio22_default: gpio22-default{
+				mux {
+					groups = "gpio22_group";
+					function = "gpio22_func";
+				};
+			};
+            pinctrl_gpio23_default: gpio23-default{
+				mux {
+					groups = "gpio23_group";
+					function = "gpio23_func";
+				};
+			};
+            pinctrl_gpio24_default: gpio24-default{
+				mux {
+					groups = "gpio24_group";
+					function = "gpio24_func";
+				};
+			};
+            pinctrl_gpio25_default: gpio25-default{
+				mux {
+					groups = "gpio25_group";
+					function = "gpio25_func";
+				};
+			};
+            pinctrl_gpio26_default: gpio26-default{
+				mux {
+					groups = "gpio26_group";
+					function = "gpio26_func";
+				};
+			};
+            pinctrl_gpio27_default: gpio27-default{
+				mux {
+					groups = "gpio27_group";
+					function = "gpio27_func";
+				};
+			};
+            pinctrl_gpio28_default: gpio28-default{
+				mux {
+					groups = "gpio28_group";
+					function = "gpio28_func";
+				};
+				conf {
+					groups = "gpio28_group";
+					input-enable = <1>;
+					bias-pull-up = <1>;
+				};
+			};
+            pinctrl_gpio29_default: gpio29-default{
+				mux {
+					groups = "gpio29_group";
+					function = "gpio29_func";
+				};
+			};
+            pinctrl_gpio30_default: gpio30-default{
+				mux {
+					groups = "gpio30_group";
+					function = "gpio30_func";
+				};
+			};
+            pinctrl_gpio31_default: gpio31-default{
+				mux {
+					groups = "gpio31_group";
+					function = "gpio31_func";
+				};
+			};
+            pinctrl_gpio32_default: gpio32-default{
+				mux {
+					groups = "gpio32_group";
+					function = "gpio32_func";
+				};
+			};
+            pinctrl_gpio33_default: gpio33-default{
+				mux {
+					groups = "gpio33_group";
+					function = "gpio33_func";
+				};
+			};
+            pinctrl_gpio34_default: gpio34-default{
+				mux {
+					groups = "gpio34_group";
+					function = "gpio34_func";
+				};
+			};
+            pinctrl_gpio35_default: gpio35-default{
+				mux {
+					groups = "gpio35_group";
+					function = "gpio35_func";
+				};
+				conf {
+					groups = "gpio35_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio36_default: gpio36-default{
+				mux {
+					groups = "gpio36_group";
+					function = "gpio36_func";
+				};
+				conf {
+					groups = "gpio36_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio37_default: gpio37-default{
+				mux {
+					groups = "gpio37_group";
+					function = "gpio37_func";
+				};
+				conf {
+					groups = "gpio37_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio38_default: gpio38-default{
+				mux {
+					groups = "gpio38_group";
+					function = "gpio38_func";
+				};
+				conf {
+					groups = "gpio38_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio39_default: gpio39-default{
+				mux {
+					groups = "gpio39_group";
+					function = "gpio39_func";
+				};
+				conf {
+					groups = "gpio39_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio40_default: gpio40-default{
+				mux {
+					groups = "gpio40_group";
+					function = "gpio40_func";
+				};
+				conf {
+					groups = "gpio40_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio41_default: gpio41-default{
+				mux {
+					groups = "gpio41_group";
+					function = "gpio41_func";
+				};
+			};
+            pinctrl_gpio42_default: gpio42-default{
+				mux {
+					groups = "gpio42_group";
+					function = "gpio42_func";
+				};
+				conf {
+					groups = "gpio42_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio43_default: gpio43-default{
+				mux {
+					groups = "gpio43_group";
+					function = "gpio43_func";
+				};
+			};
+            pinctrl_gpio44_default: gpio44-default{
+				mux {
+					groups = "gpio44_group";
+					function = "gpio44_func";
+				};
+			};
+            pinctrl_gpio45_default: gpio45-default{
+				mux {
+					groups = "gpio45_group";
+					function = "gpio45_func";
+				};
+			};
+            pinctrl_gpio46_default: gpio46-default{
+				mux {
+					groups = "gpio46_group";
+					function = "gpio46_func";
+				};
+				conf {
+					groups = "gpio46_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio47_default: gpio47-default{
+				mux {
+					groups = "gpio47_group";
+					function = "gpio47_func";
+				};
+				conf {
+					groups = "gpio47_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio48_default: gpio48-default{
+				mux {
+					groups = "gpio48_group";
+					function = "gpio48_func";
+				};
+			};
+            pinctrl_gpio49_default: gpio49-default{
+				mux {
+					groups = "gpio49_group";
+					function = "gpio49_func";
+				};
+			};
+            pinctrl_gpio50_default: gpio50-default{
+				mux {
+					groups = "gpio50_group";
+					function = "gpio50_func";
+				};
+			};
+            pinctrl_gpio51_default: gpio51-default{
+				mux {
+					groups = "gpio51_group";
+					function = "gpio51_func";
+				};
+			};
+            pinctrl_gpio52_default: gpio52-default{
+				mux {
+					groups = "gpio52_group";
+					function = "gpio52_func";
+				};
+				conf {
+					groups = "gpio52_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio53_default: gpio53-default{
+				mux {
+					groups = "gpio53_group";
+					function = "gpio53_func";
+				};
+				conf {
+					groups = "gpio53_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio54_default: gpio54-default{
+				mux {
+					groups = "gpio54_group";
+					function = "gpio54_func";
+				};
+			};
+            pinctrl_gpio55_default: gpio55-default{
+				mux {
+					groups = "gpio55_group";
+					function = "gpio55_func";
+				};
+			};
+            pinctrl_gpio56_default: gpio56-default{
+				mux {
+					groups = "gpio56_group";
+					function = "gpio56_func";
+				};
+			};
+            pinctrl_gpio57_default: gpio57-default{
+				mux {
+					groups = "gpio57_group";
+					function = "gpio57_func";
+				};
+			};
+            pinctrl_gpio58_default: gpio58-default{
+				mux {
+					groups = "gpio58_group";
+					function = "gpio58_func";
+				};
+			};
+            pinctrl_gpio59_default: gpio59-default{
+				mux {
+					groups = "gpio59_group";
+					function = "gpio59_func";
+				};
+			};
+            pinctrl_gpio60_default: gpio60-default{
+				mux {
+					groups = "gpio60_group";
+					function = "gpio60_func";
+				};
+			};
+            pinctrl_gpio61_default: gpio61-default{
+				mux {
+					groups = "gpio61_group";
+					function = "gpio61_func";
+				};
+			};
+            pinctrl_gpio62_default: gpio62-default{
+				mux {
+					groups = "gpio62_group";
+					function = "gpio62_func";
+				};
+			};
+            pinctrl_gpio63_default: gpio63-default{
+				mux {
+					groups = "gpio63_group";
+					function = "gpio63_func";
+				};
+			};
+            pinctrl_gpio64_default: gpio64-default{
+				mux {
+					groups = "gpio64_group";
+					function = "gpio64_func";
+				};
+				conf {
+					groups = "gpio64_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio65_default: gpio65-default{
+				mux {
+					groups = "gpio65_group";
+					function = "gpio65_func";
+				};
+				conf {
+					groups = "gpio65_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio66_default: gpio66-default{
+				mux {
+					groups = "gpio66_group";
+					function = "gpio66_func";
+				};
+				conf {
+					groups = "gpio66_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio67_default: gpio67-default{
+				mux {
+					groups = "gpio67_group";
+					function = "gpio67_func";
+				};
+				conf {
+					groups = "gpio67_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio68_default: gpio68-default{
+				mux {
+					groups = "gpio68_group";
+					function = "gpio68_func";
+				};
+			};
+            pinctrl_gpio69_default: gpio69-default{
+				mux {
+					groups = "gpio69_group";
+					function = "gpio69_func";
+				};
+			};
+            pinctrl_gpio70_default: gpio70-default{
+				mux {
+					groups = "gpio70_group";
+					function = "gpio70_func";
+				};
+			};
+            pinctrl_gpio71_default: gpio71-default{
+				mux {
+					groups = "gpio71_group";
+					function = "gpio71_func";
+				};
+			};
+            pinctrl_gpio72_default: gpio72-default{
+				mux {
+					groups = "gpio72_group";
+					function = "gpio72_func";
+				};
+			};
+            pinctrl_gpio73_default: gpio73-default{
+				mux {
+					groups = "gpio73_group";
+					function = "gpio73_func";
+				};
+			};
+            pinctrl_gpio74_default: gpio74-default{
+				mux {
+					groups = "gpio74_group";
+					function = "gpio74_func";
+				};
+			};
+            pinctrl_gpio75_default: gpio75-default{
+				mux {
+					groups = "gpio75_group";
+					function = "gpio75_func";
+				};
+			};
+            pinctrl_gpio76_default: gpio76-default{
+				mux {
+					groups = "gpio76_group";
+					function = "gpio76_func";
+				};
+			};
+            pinctrl_gpio77_default: gpio77-default{
+				mux {
+					groups = "gpio77_group";
+					function = "gpio77_func";
+				};
+			};
+            pinctrl_gpio78_default: gpio78-default{
+				mux {
+					groups = "gpio78_group";
+					function = "gpio78_func";
+				};
+			};
+            pinctrl_gpio79_default: gpio79-default{
+				mux {
+					groups = "gpio79_group";
+					function = "gpio79_func";
+				};
+			};
+            pinctrl_gpio80_default: gpio80-default{
+				mux {
+					groups = "gpio80_group";
+					function = "gpio80_func";
+				};
+			};
+            pinctrl_gpio81_default: gpio81-default{
+				mux {
+					groups = "gpio81_group";
+					function = "gpio81_func";
+				};
+			};
+            pinctrl_gpio82_default: gpio82-default{
+				mux {
+					groups = "gpio82_group";
+					function = "gpio82_func";
+				};
+			};
+            pinctrl_gpio83_default: gpio83-default{
+				mux {
+					groups = "gpio83_group";
+					function = "gpio83_func";
+				};
+			};
+            pinctrl_gpio84_default: gpio84-default{
+				mux {
+					groups = "gpio84_group";
+					function = "gpio84_func";
+				};
+			};
+            pinctrl_gpio85_default: gpio85-default{
+				mux {
+					groups = "gpio85_group";
+					function = "gpio85_func";
+				};
+			};
+            pinctrl_gpio86_default: gpio86-default{
+				mux {
+					groups = "gpio86_group";
+					function = "gpio86_func";
+				};
+			};
+            pinctrl_gpio87_default: gpio87-default{
+				mux {
+					groups = "gpio87_group";
+					function = "gpio87_func";
+				};
+			};
+            pinctrl_gpio88_default: gpio88-default{
+				mux {
+					groups = "gpio88_group";
+					function = "gpio88_func";
+				};
+			};
+            pinctrl_gpio89_default: gpio89-default{
+				mux {
+					groups = "gpio89_group";
+					function = "gpio89_func";
+				};
+			};
+            pinctrl_gpio90_default: gpio90-default{
+				mux {
+					groups = "gpio90_group";
+					function = "gpio90_func";
+				};
+			};
+            pinctrl_gpio91_default: gpio91-default{
+				mux {
+					groups = "gpio91_group";
+					function = "gpio91_func";
+				};
+			};
+            pinctrl_gpio92_default: gpio92-default{
+				mux {
+					groups = "gpio92_group";
+					function = "gpio92_func";
+				};
+				conf {
+					groups = "gpio92_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio93_default: gpio93-default{
+				mux {
+					groups = "gpio93_group";
+					function = "gpio93_func";
+				};
+				conf {
+					groups = "gpio93_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio94_default: gpio94-default{
+				mux {
+					groups = "gpio94_group";
+					function = "gpio94_func";
+				};
+				conf {
+					groups = "gpio94_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio95_default: gpio95-default{
+				mux {
+					groups = "gpio95_group";
+					function = "gpio95_func";
+				};
+			};
+            pinctrl_gpio96_default: gpio96-default{
+				mux {
+					groups = "gpio96_group";
+					function = "gpio96_func";
+				};
+			};
+            pinctrl_gpio97_default: gpio97-default{
+				mux {
+					groups = "gpio97_group";
+					function = "gpio97_func";
+				};
+			};
+            pinctrl_gpio98_default: gpio98-default{
+				mux {
+					groups = "gpio98_group";
+					function = "gpio98_func";
+				};
+			};
+            pinctrl_gpio99_default: gpio99-default{
+				mux {
+					groups = "gpio99_group";
+					function = "gpio99_func";
+				};
+			};
+            pinctrl_gpio100_default: gpio100-default{
+				mux {
+					groups = "gpio100_group";
+					function = "gpio100_func";
+				};
+			};
+            pinctrl_gpio101_default: gpio101-default{
+				mux {
+					groups = "gpio101_group";
+					function = "gpio101_func";
+				};
+			};
+            pinctrl_gpio102_default: gpio102-default{
+				mux {
+					groups = "gpio102_group";
+					function = "gpio102_func";
+				};
+			};
+            pinctrl_gpio103_default: gpio103-default{
+				mux {
+					groups = "gpio103_group";
+					function = "gpio103_func";
+				};
+			};
+            pinctrl_gpio104_default: gpio104-default{
+				mux {
+					groups = "gpio104_group";
+					function = "gpio104_func";
+				};
+			};
+            pinctrl_gpio105_default: gpio105-default{
+				mux {
+					groups = "gpio105_group";
+					function = "gpio105_func";
+				};
+			};
+            pinctrl_gpio106_default: gpio106-default{
+				mux {
+					groups = "gpio106_group";
+					function = "gpio106_func";
+				};
+				conf {
+					groups = "gpio106_group";
+					input-enable = <0>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio107_default: gpio107-default{
+				mux {
+					groups = "gpio107_group";
+					function = "gpio107_func";
+				};
+				conf {
+					groups = "gpio107_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio108_default: gpio108-default{
+				mux {
+					groups = "gpio108_group";
+					function = "gpio108_func";
+				};
+				conf {
+					groups = "gpio108_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio109_default: gpio109-default{
+				mux {
+					groups = "gpio109_group";
+					function = "gpio109_func";
+				};
+				conf {
+					groups = "gpio109_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio110_default: gpio110-default{
+				mux {
+					groups = "gpio110_group";
+					function = "gpio110_func";
+				};
+				conf {
+					groups = "gpio110_group";
+					input-enable = <1>;
+					bias-pull-down = <1>;
+				};
+			};
+            pinctrl_gpio111_default: gpio111-default{
+				mux {
+					groups = "gpio111_group";
+					function = "gpio111_func";
+				};
+				conf {
+					groups = "gpio111_group";
+					input-enable = <0>;
+					bias-pull-up = <1>;
+				};
+			};
+
+            //func6
+            pinctrl_csi_mon_out_default: csi_mon_out-default{
+				mux {
+					groups = "csi_mon_out_group";
+					function = "csi_mon_out_func";
+				};
+			};
+            pinctrl_csi_ocla_clk_default: csi_ocla_clk-default{
+				mux {
+					groups = "csi_ocla_clk_group";
+					function = "csi_ocla_clk_func";
+				};
+			};
+            pinctrl_csi_mon_out_valid_default: csi_mon_out_valid-default{
+				mux {
+					groups = "csi_mon_out_valid_group";
+					function = "csi_mon_out_valid_func";
+				};
+			};
+            pinctrl_csi_parity_error_default: csi_parity_error-default{
+				mux {
+					groups = "csi_parity_error_group";
+					function = "csi_parity_error_func";
+				};
+			};
+            pinctrl_csi_dtb_out_default: csi_dtb_out-default{
+				mux {
+					groups = "csi_dtb_out_group";
+					function = "csi_dtb_out_func";
+				};
+			};
+            pinctrl_csi_phy_sel_default: csi_phy_sel-default{
+				mux {
+					groups = "csi_phy_sel_group";
+					function = "csi_phy_sel_func";
+				};
+			};
+            pinctrl_vc_g2d0_debug_out_default: vc_g2d0_debug_out-default{
+				mux {
+					groups = "vc_g2d0_debug_out_group";
+					function = "vc_g2d0_debug_out_func";
+				};
+			};
+            pinctrl_vc_g2d1_debug_out_default: vc_g2d1_debug_out-default{
+				mux {
+					groups = "vc_g2d1_debug_out_group";
+					function = "vc_g2d1_debug_out_func";
+				};
+			};
+            pinctrl_sata_mpll_clk_default: sata_mpll_clk-default{
+				mux {
+					groups = "sata_mpll_clk_group";
+					function = "sata_mpll_clk_func";
+				};
+			};
+            pinctrl_sata_ref_repeat_clk_m_default: sata_ref_repeat_clk_m-default{
+				mux {
+					groups = "sata_ref_repeat_clk_m_group";
+					function = "sata_ref_repeat_clk_m_func";
+				};
+			};
+            pinctrl_sata_ref_repeat_clk_p_default: sata_ref_repeat_clk_p-default{
+				mux {
+					groups = "sata_ref_repeat_clk_p_group";
+					function = "sata_ref_repeat_clk_p_func";
+				};
+			};
+		};
+    };
+};
diff --git a/arch/riscv/boot/dts/starfive/jh7100-common.dtsi b/arch/riscv/boot/dts/starfive/jh7100-common.dtsi
index b93ce351a90f..83c9de3f6791 100644
--- a/arch/riscv/boot/dts/starfive/jh7100-common.dtsi
+++ b/arch/riscv/boot/dts/starfive/jh7100-common.dtsi
@@ -9,8 +9,14 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/leds/common.h>
 #include <dt-bindings/pinctrl/pinctrl-starfive-jh7100.h>
+#include <dt-bindings/riscv/physical-memory.h>
 
 / {
+	riscv,physical-memory-regions =
+		<0x00 0x00000000 0x40 0x00000000 (PMA_RW | PMA_IO) 0x0>,
+		<0x00 0x80000000 0x08 0x00000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY) 0x0>,
+		<0x10 0x00000000 0x08 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY | PMR_ALIAS(1)) 0x0>;
+
 	aliases {
 		serial0 = &uart3;
 	};
diff --git a/arch/riscv/configs/hifive-premier-p550_defconfig b/arch/riscv/configs/hifive-premier-p550_defconfig
new file mode 100644
index 000000000000..c1edf3f3691e
--- /dev/null
+++ b/arch/riscv/configs/hifive-premier-p550_defconfig
@@ -0,0 +1,2506 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_USELIB=y
+# CONFIG_CONTEXT_TRACKING_USER_FORCE is not set
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_JIT=y
+CONFIG_BPF_JIT_ALWAYS_ON=y
+CONFIG_BPF_LSM=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_PREEMPT_DYNAMIC=y
+CONFIG_VIRT_CPU_ACCOUNTING_GEN=y
+CONFIG_BSD_PROCESS_ACCT=y
+CONFIG_BSD_PROCESS_ACCT_V3=y
+CONFIG_TASKSTATS=y
+CONFIG_TASK_DELAY_ACCT=y
+CONFIG_TASK_XACCT=y
+CONFIG_TASK_IO_ACCOUNTING=y
+CONFIG_PSI=y
+CONFIG_IKHEADERS=m
+CONFIG_LOG_BUF_SHIFT=18
+CONFIG_UCLAMP_TASK=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_UCLAMP_TASK_GROUP=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_RDMA=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CGROUP_HUGETLB=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_SCHED_AUTOGROUP=y
+CONFIG_EXPERT=y
+CONFIG_SGETMASK_SYSCALL=y
+CONFIG_PC104=y
+CONFIG_PROFILING=y
+CONFIG_KEXEC=y
+CONFIG_KEXEC_FILE=y
+CONFIG_CRASH_DUMP=y
+CONFIG_SOC_SIFIVE_EIC7700=y
+CONFIG_SOC_SIFIVE=y
+CONFIG_SOC_VIRT=y
+CONFIG_ERRATA_THEAD=y
+CONFIG_SMP=y
+CONFIG_SCHED_MC=y
+CONFIG_NR_CPUS=32
+# CONFIG_RISCV_ISA_SVPBMT is not set
+# CONFIG_COMPAT is not set
+CONFIG_RANDOMIZE_BASE=y
+CONFIG_CMDLINE="clk_ignore_unused"
+CONFIG_CMDLINE_EXTEND=y
+CONFIG_PM_WAKELOCKS=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_ADVANCED_DEBUG=y
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_LADDER=y
+CONFIG_CPU_IDLE_GOV_MENU=y
+CONFIG_CPU_IDLE_GOV_TEO=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y
+CONFIG_CPUFREQ_DT=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=y
+CONFIG_ARCH_MMAP_RND_BITS=24
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODVERSIONS=y
+CONFIG_MODULE_SRCVERSION_ALL=y
+CONFIG_MODULE_SIG_SHA512=y
+CONFIG_MODULE_COMPRESS_ZSTD=y
+CONFIG_MODULE_DECOMPRESS=y
+CONFIG_BLK_DEV_ZONED=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BLK_WBT=y
+CONFIG_BLK_CGROUP_FC_APPID=y
+CONFIG_BLK_CGROUP_IOCOST=y
+CONFIG_BLK_CGROUP_IOPRIO=y
+CONFIG_BLK_SED_OPAL=y
+CONFIG_BLK_INLINE_ENCRYPTION=y
+CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y
+CONFIG_PARTITION_ADVANCED=y
+CONFIG_AIX_PARTITION=y
+CONFIG_OSF_PARTITION=y
+CONFIG_AMIGA_PARTITION=y
+CONFIG_ATARI_PARTITION=y
+CONFIG_MAC_PARTITION=y
+CONFIG_BSD_DISKLABEL=y
+CONFIG_MINIX_SUBPARTITION=y
+CONFIG_SOLARIS_X86_PARTITION=y
+CONFIG_UNIXWARE_DISKLABEL=y
+CONFIG_LDM_PARTITION=y
+CONFIG_SGI_PARTITION=y
+CONFIG_ULTRIX_PARTITION=y
+CONFIG_SUN_PARTITION=y
+CONFIG_KARMA_PARTITION=y
+CONFIG_SYSV68_PARTITION=y
+CONFIG_CMDLINE_PARTITION=y
+CONFIG_MQ_IOSCHED_KYBER=m
+CONFIG_IOSCHED_BFQ=m
+CONFIG_BINFMT_MISC=m
+CONFIG_ZSWAP=y
+CONFIG_Z3FOLD=m
+CONFIG_ZSMALLOC=y
+CONFIG_SLAB_FREELIST_RANDOM=y
+CONFIG_SLAB_FREELIST_HARDENED=y
+CONFIG_RANDOM_KMALLOC_CACHES=y
+CONFIG_SHUFFLE_PAGE_ALLOCATOR=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_KSM=y
+CONFIG_DEFAULT_MMAP_MIN_ADDR=65536
+CONFIG_TRANSPARENT_HUGEPAGE=y
+CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
+CONFIG_CMA=y
+CONFIG_CMA_SYSFS=y
+CONFIG_IDLE_PAGE_TRACKING=y
+CONFIG_ANON_VMA_NAME=y
+CONFIG_USERFAULTFD=y
+CONFIG_LRU_GEN=y
+CONFIG_LRU_GEN_ENABLED=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_XFRM_USER=m
+CONFIG_XFRM_INTERFACE=m
+CONFIG_XFRM_STATISTICS=y
+CONFIG_NET_KEY=m
+CONFIG_XDP_SOCKETS=y
+CONFIG_XDP_SOCKETS_DIAG=m
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_FIB_TRIE_STATS=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_IP_ROUTE_MULTIPATH=y
+CONFIG_IP_ROUTE_VERBOSE=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPGRE_DEMUX=m
+CONFIG_NET_IPGRE=m
+CONFIG_NET_IPGRE_BROADCAST=y
+CONFIG_IP_MROUTE=y
+CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IP_PIMSM_V1=y
+CONFIG_IP_PIMSM_V2=y
+CONFIG_NET_IPVTI=m
+CONFIG_NET_FOU_IP_TUNNELS=y
+CONFIG_INET_AH=m
+CONFIG_INET_ESP=m
+CONFIG_INET_ESP_OFFLOAD=m
+CONFIG_INET_ESPINTCP=y
+CONFIG_INET_IPCOMP=m
+CONFIG_INET_DIAG=m
+CONFIG_INET_UDP_DIAG=m
+CONFIG_INET_RAW_DIAG=m
+CONFIG_INET_DIAG_DESTROY=y
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=m
+CONFIG_TCP_MD5SIG=y
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_ESP_OFFLOAD=m
+CONFIG_INET6_ESPINTCP=y
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_SIT=m
+CONFIG_IPV6_SIT_6RD=y
+CONFIG_IPV6_GRE=m
+CONFIG_IPV6_SUBTREES=y
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IPV6_PIMSM_V2=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_MPTCP=y
+CONFIG_NETWORK_PHY_TIMESTAMPING=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NETFILTER_NETLINK_HOOK=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_SECMARK=y
+CONFIG_NF_CONNTRACK_ZONES=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_TIMEOUT=y
+CONFIG_NF_CONNTRACK_TIMESTAMP=y
+CONFIG_NF_CONNTRACK_AMANDA=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_H323=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_NETBIOS_NS=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SANE=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_CT_NETLINK_TIMEOUT=m
+CONFIG_NF_CT_NETLINK_HELPER=m
+CONFIG_NETFILTER_NETLINK_GLUE_CT=y
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_FLOW_OFFLOAD=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_FIB_INET=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_FIB_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NF_FLOW_TABLE_INET=m
+CONFIG_NF_FLOW_TABLE=m
+CONFIG_NETFILTER_XT_SET=m
+CONFIG_NETFILTER_XT_TARGET_AUDIT=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LED=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_SECMARK=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_SET_BITMAP_IP=m
+CONFIG_IP_SET_BITMAP_IPMAC=m
+CONFIG_IP_SET_BITMAP_PORT=m
+CONFIG_IP_SET_HASH_IP=m
+CONFIG_IP_SET_HASH_IPMARK=m
+CONFIG_IP_SET_HASH_IPPORT=m
+CONFIG_IP_SET_HASH_IPPORTIP=m
+CONFIG_IP_SET_HASH_IPPORTNET=m
+CONFIG_IP_SET_HASH_IPMAC=m
+CONFIG_IP_SET_HASH_MAC=m
+CONFIG_IP_SET_HASH_NETPORTNET=m
+CONFIG_IP_SET_HASH_NET=m
+CONFIG_IP_SET_HASH_NETNET=m
+CONFIG_IP_SET_HASH_NETPORT=m
+CONFIG_IP_SET_HASH_NETIFACE=m
+CONFIG_IP_SET_LIST_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_PROTO_AH=y
+CONFIG_IP_VS_PROTO_SCTP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_WRR=m
+CONFIG_IP_VS_LC=m
+CONFIG_IP_VS_WLC=m
+CONFIG_IP_VS_FO=m
+CONFIG_IP_VS_OVF=m
+CONFIG_IP_VS_LBLC=m
+CONFIG_IP_VS_LBLCR=m
+CONFIG_IP_VS_DH=m
+CONFIG_IP_VS_SH=m
+CONFIG_IP_VS_MH=m
+CONFIG_IP_VS_SED=m
+CONFIG_IP_VS_NQ=m
+CONFIG_IP_VS_TWOS=m
+CONFIG_IP_VS_FTP=m
+CONFIG_IP_VS_PE_SIP=m
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_NF_LOG_IPV4=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_FILTER=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_MANGLE=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_SECURITY=m
+CONFIG_IP_NF_ARPTABLES=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_FILTER=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_MANGLE=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_SECURITY=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NFT_BRIDGE_META=m
+CONFIG_NFT_BRIDGE_REJECT=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_BPFILTER=y
+CONFIG_IP_DCCP=m
+# CONFIG_IP_DCCP_CCID3 is not set
+CONFIG_SCTP_DEFAULT_COOKIE_HMAC_SHA1=y
+CONFIG_SCTP_COOKIE_HMAC_MD5=y
+CONFIG_RDS=m
+CONFIG_RDS_TCP=m
+CONFIG_TIPC=m
+CONFIG_ATM=m
+CONFIG_ATM_CLIP=m
+CONFIG_ATM_LANE=m
+CONFIG_ATM_MPOA=m
+CONFIG_ATM_BR2684=m
+CONFIG_L2TP=m
+CONFIG_L2TP_DEBUGFS=m
+CONFIG_L2TP_V3=y
+CONFIG_L2TP_IP=m
+CONFIG_L2TP_ETH=m
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_NET_DSA=m
+CONFIG_NET_DSA_TAG_RTL4_A=m
+CONFIG_NET_DSA_TAG_RTL8_4=m
+CONFIG_NET_DSA_TAG_RZN1_A5PSW=m
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_VLAN_8021Q_MVRP=y
+CONFIG_LLC2=m
+CONFIG_ATALK=m
+CONFIG_DEV_APPLETALK=m
+CONFIG_X25=m
+CONFIG_LAPB=m
+CONFIG_PHONET=m
+CONFIG_6LOWPAN=m
+CONFIG_IEEE802154=m
+CONFIG_IEEE802154_6LOWPAN=m
+CONFIG_MAC802154=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_HTB=m
+CONFIG_NET_SCH_HFSC=m
+CONFIG_NET_SCH_PRIO=m
+CONFIG_NET_SCH_MULTIQ=m
+CONFIG_NET_SCH_RED=m
+CONFIG_NET_SCH_SFB=m
+CONFIG_NET_SCH_SFQ=m
+CONFIG_NET_SCH_TEQL=m
+CONFIG_NET_SCH_TBF=m
+CONFIG_NET_SCH_CBS=m
+CONFIG_NET_SCH_ETF=m
+CONFIG_NET_SCH_TAPRIO=m
+CONFIG_NET_SCH_GRED=m
+CONFIG_NET_SCH_NETEM=m
+CONFIG_NET_SCH_DRR=m
+CONFIG_NET_SCH_MQPRIO=m
+CONFIG_NET_SCH_SKBPRIO=m
+CONFIG_NET_SCH_CHOKE=m
+CONFIG_NET_SCH_QFQ=m
+CONFIG_NET_SCH_CODEL=m
+CONFIG_NET_SCH_FQ_CODEL=m
+CONFIG_NET_SCH_CAKE=m
+CONFIG_NET_SCH_FQ=m
+CONFIG_NET_SCH_HHF=m
+CONFIG_NET_SCH_PIE=m
+CONFIG_NET_SCH_FQ_PIE=m
+CONFIG_NET_SCH_INGRESS=m
+CONFIG_NET_SCH_PLUG=m
+CONFIG_NET_SCH_ETS=m
+CONFIG_NET_CLS_BASIC=m
+CONFIG_NET_CLS_ROUTE4=m
+CONFIG_NET_CLS_FW=m
+CONFIG_NET_CLS_U32=m
+CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_FLOW=m
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_BPF=m
+CONFIG_NET_CLS_FLOWER=m
+CONFIG_NET_CLS_MATCHALL=m
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_CMP=m
+CONFIG_NET_EMATCH_NBYTE=m
+CONFIG_NET_EMATCH_U32=m
+CONFIG_NET_EMATCH_META=m
+CONFIG_NET_EMATCH_TEXT=m
+CONFIG_NET_EMATCH_IPSET=m
+CONFIG_NET_EMATCH_IPT=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_ACT_POLICE=m
+CONFIG_NET_ACT_GACT=m
+CONFIG_GACT_PROB=y
+CONFIG_NET_ACT_MIRRED=m
+CONFIG_NET_ACT_SAMPLE=m
+CONFIG_NET_ACT_IPT=m
+CONFIG_NET_ACT_NAT=m
+CONFIG_NET_ACT_PEDIT=m
+CONFIG_NET_ACT_SIMP=m
+CONFIG_NET_ACT_SKBEDIT=m
+CONFIG_NET_ACT_CSUM=m
+CONFIG_NET_ACT_MPLS=m
+CONFIG_NET_ACT_VLAN=m
+CONFIG_NET_ACT_BPF=m
+CONFIG_NET_ACT_CONNMARK=m
+CONFIG_NET_ACT_CTINFO=m
+CONFIG_NET_ACT_SKBMOD=m
+CONFIG_NET_ACT_TUNNEL_KEY=m
+CONFIG_NET_ACT_CT=m
+CONFIG_NET_ACT_GATE=m
+CONFIG_NET_TC_SKB_EXT=y
+CONFIG_DCB=y
+CONFIG_DNS_RESOLVER=y
+CONFIG_BATMAN_ADV=m
+# CONFIG_BATMAN_ADV_BATMAN_V is not set
+CONFIG_BATMAN_ADV_NC=y
+CONFIG_OPENVSWITCH=m
+CONFIG_VSOCKETS=m
+CONFIG_VIRTIO_VSOCKETS=m
+CONFIG_NETLINK_DIAG=m
+CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
+CONFIG_HSR=m
+CONFIG_QRTR_SMD=m
+CONFIG_QRTR_TUN=m
+CONFIG_NET_NCSI=y
+CONFIG_NCSI_OEM_CMD_GET_MAC=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BPF_STREAM_PARSER=y
+CONFIG_NET_PKTGEN=m
+CONFIG_NET_DROP_MONITOR=y
+CONFIG_HAMRADIO=y
+CONFIG_AX25=m
+CONFIG_NETROM=m
+CONFIG_ROSE=m
+CONFIG_MKISS=m
+CONFIG_6PACK=m
+CONFIG_BPQETHER=m
+CONFIG_BAYCOM_SER_FDX=m
+CONFIG_BAYCOM_SER_HDX=m
+CONFIG_BAYCOM_PAR=m
+CONFIG_YAM=m
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=m
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_CMTP=m
+CONFIG_BT_HIDP=m
+CONFIG_BT_HS=y
+CONFIG_BT_6LOWPAN=m
+CONFIG_BT_LEDS=y
+CONFIG_BT_MSFTEXT=y
+CONFIG_BT_AOSPEXT=y
+CONFIG_BT_HCIBTUSB=m
+CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y
+CONFIG_BT_HCIBTUSB_MTK=y
+CONFIG_BT_HCIBTSDIO=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_NOKIA=m
+CONFIG_BT_HCIUART_BCSP=y
+CONFIG_BT_HCIUART_ATH3K=y
+CONFIG_BT_HCIUART_LL=y
+CONFIG_BT_HCIUART_INTEL=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_BT_HCIUART_QCA=y
+CONFIG_BT_HCIUART_AG6XX=y
+CONFIG_BT_HCIUART_MRVL=y
+CONFIG_BT_HCIBCM203X=m
+CONFIG_BT_HCIBCM4377=m
+CONFIG_BT_HCIBPA10X=m
+CONFIG_BT_HCIBFUSB=m
+CONFIG_BT_HCIVHCI=m
+CONFIG_BT_MRVL=m
+CONFIG_BT_MRVL_SDIO=m
+CONFIG_BT_ATH3K=m
+CONFIG_BT_MTKSDIO=m
+CONFIG_BT_MTKUART=m
+CONFIG_BT_VIRTIO=m
+CONFIG_BT_NXPUART=m
+CONFIG_AF_RXRPC_IPV6=y
+CONFIG_RXKAD=y
+CONFIG_RXPERF=m
+CONFIG_AF_KCM=m
+CONFIG_MCTP=y
+CONFIG_CFG80211_DEBUGFS=y
+CONFIG_MAC80211=m
+CONFIG_MAC80211_MESH=y
+CONFIG_MAC80211_MESSAGE_TRACING=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=m
+CONFIG_NET_9P=m
+CONFIG_NET_9P_VIRTIO=m
+CONFIG_CAIF=m
+CONFIG_CAIF_USB=m
+CONFIG_CEPH_LIB_USE_DNS_RESOLVER=y
+CONFIG_NET_IFE=m
+CONFIG_PAGE_POOL_STATS=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_HOTPLUG_PCI_PCIE=y
+CONFIG_PCIEAER=y
+CONFIG_PCIE_DPC=y
+CONFIG_PCIE_PTM=y
+CONFIG_PCI_REALLOC_ENABLE_AUTO=y
+CONFIG_PCI_STUB=m
+CONFIG_PCI_PF_STUB=m
+CONFIG_PCI_IOV=y
+CONFIG_PCI_PRI=y
+CONFIG_PCI_PASID=y
+CONFIG_PCI_DYNAMIC_OF_NODES=y
+CONFIG_HOTPLUG_PCI=y
+CONFIG_HOTPLUG_PCI_CPCI=y
+CONFIG_HOTPLUG_PCI_SHPC=y
+CONFIG_PCI_FTPCI100=y
+CONFIG_PCI_HOST_GENERIC=y
+CONFIG_PCIE_MICROCHIP_HOST=y
+CONFIG_PCIE_XILINX=y
+CONFIG_PCIE_CADENCE_PLAT_HOST=y
+CONFIG_PCIE_CADENCE_PLAT_EP=y
+CONFIG_PCI_J721E_HOST=y
+CONFIG_PCI_J721E_EP=y
+CONFIG_PCIE_DW_PLAT_HOST=y
+CONFIG_PCIE_DW_PLAT_EP=y
+CONFIG_PCIE_FU740=y
+CONFIG_PCIE_ESWIN=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_PCI_ENDPOINT_CONFIGFS=y
+CONFIG_PCI_EPF_NTB=m
+CONFIG_PCI_EPF_MHI=m
+CONFIG_PCI_SW_SWITCHTEC=m
+CONFIG_CXL_BUS=m
+CONFIG_UEVENT_HELPER=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_DEVTMPFS_SAFE=y
+CONFIG_FW_LOADER_USER_HELPER=y
+CONFIG_FW_LOADER_COMPRESS=y
+CONFIG_FW_LOADER_COMPRESS_ZSTD=y
+CONFIG_MOXTET=m
+CONFIG_MHI_BUS_PCI_GENERIC=m
+CONFIG_MHI_BUS_EP=m
+CONFIG_CONNECTOR=y
+CONFIG_FIRMWARE_MEMMAP=y
+CONFIG_SYSFB_SIMPLEFB=y
+CONFIG_EFI_VARS_PSTORE=m
+CONFIG_EFI_BOOTLOADER_CONTROL=m
+CONFIG_EFI_CAPSULE_LOADER=m
+CONFIG_EFI_TEST=m
+CONFIG_RESET_ATTACK_MITIGATION=y
+CONFIG_EFI_COCO_SECRET=y
+CONFIG_GNSS=m
+CONFIG_GNSS_MTK_SERIAL=m
+CONFIG_GNSS_SIRF_SERIAL=m
+CONFIG_GNSS_UBX_SERIAL=m
+CONFIG_GNSS_USB=m
+CONFIG_MTD=m
+CONFIG_MTD_AR7_PARTS=m
+CONFIG_MTD_CMDLINE_PARTS=m
+CONFIG_MTD_REDBOOT_PARTS=m
+CONFIG_MTD_BLOCK=m
+CONFIG_MTD_BLOCK_RO=m
+CONFIG_FTL=m
+CONFIG_NFTL=m
+CONFIG_NFTL_RW=y
+CONFIG_INFTL=m
+CONFIG_RFD_FTL=m
+CONFIG_SSFDC=m
+CONFIG_SM_FTL=m
+CONFIG_MTD_OOPS=m
+CONFIG_MTD_PSTORE=m
+CONFIG_MTD_SWAP=m
+CONFIG_MTD_JEDECPROBE=m
+CONFIG_MTD_CFI_INTELEXT=m
+CONFIG_MTD_CFI_STAA=m
+CONFIG_MTD_ROM=m
+CONFIG_MTD_ABSENT=m
+CONFIG_MTD_PHYSMAP=m
+CONFIG_MTD_PHYSMAP_OF=y
+CONFIG_MTD_PHYSMAP_GPIO_ADDR=y
+CONFIG_MTD_PCI=m
+CONFIG_MTD_INTEL_VR_NOR=m
+CONFIG_MTD_PLATRAM=m
+CONFIG_MTD_PMC551=m
+CONFIG_MTD_DATAFLASH=m
+CONFIG_MTD_DATAFLASH_OTP=y
+CONFIG_MTD_MCHP23K256=m
+CONFIG_MTD_MCHP48L640=m
+CONFIG_MTD_SST25L=m
+CONFIG_MTD_SLRAM=m
+CONFIG_MTD_PHRAM=m
+CONFIG_MTD_MTDRAM=m
+CONFIG_MTD_BLOCK2MTD=m
+CONFIG_MTD_ONENAND=m
+CONFIG_MTD_ONENAND_VERIFY_WRITE=y
+CONFIG_MTD_ONENAND_GENERIC=m
+CONFIG_MTD_ONENAND_2X_PROGRAM=y
+CONFIG_MTD_RAW_NAND=m
+CONFIG_MTD_NAND_DENALI_PCI=m
+CONFIG_MTD_NAND_DENALI_DT=m
+CONFIG_MTD_NAND_CAFE=m
+CONFIG_MTD_NAND_MXIC=m
+CONFIG_MTD_NAND_GPIO=m
+CONFIG_MTD_NAND_PLATFORM=m
+CONFIG_MTD_NAND_CADENCE=m
+CONFIG_MTD_NAND_ARASAN=m
+CONFIG_MTD_NAND_INTEL_LGM=m
+CONFIG_MTD_NAND_NANDSIM=m
+CONFIG_MTD_NAND_RICOH=m
+CONFIG_MTD_NAND_DISKONCHIP=m
+CONFIG_MTD_SPI_NAND=m
+CONFIG_MTD_NAND_ECC_SW_BCH=y
+CONFIG_MTD_NAND_ECC_MXIC=y
+CONFIG_MTD_LPDDR=m
+CONFIG_MTD_SPI_NOR=m
+CONFIG_MTD_UBI=m
+CONFIG_MTD_UBI_FASTMAP=y
+CONFIG_MTD_UBI_GLUEBI=m
+CONFIG_MTD_UBI_BLOCK=y
+CONFIG_MTD_HYPERBUS=m
+CONFIG_OF_OVERLAY=y
+CONFIG_PARPORT=m
+CONFIG_PARPORT_PC=m
+CONFIG_PARPORT_PC_FIFO=y
+CONFIG_PARPORT_1284=y
+CONFIG_BLK_DEV_NULL_BLK=m
+CONFIG_BLK_DEV_PCIESSD_MTIP32XX=m
+CONFIG_ZRAM=m
+CONFIG_ZRAM_WRITEBACK=y
+CONFIG_ZRAM_MEMORY_TRACKING=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_DRBD=m
+CONFIG_BLK_DEV_NBD=m
+CONFIG_BLK_DEV_RAM=m
+CONFIG_BLK_DEV_RAM_SIZE=65536
+CONFIG_ATA_OVER_ETH=m
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_RBD=m
+CONFIG_BLK_DEV_UBLK=m
+# CONFIG_BLKDEV_UBLK_LEGACY_OPCODES is not set
+CONFIG_BLK_DEV_NVME=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_NVME_HWMON=y
+CONFIG_NVME_FC=m
+CONFIG_NVME_TCP=m
+CONFIG_NVME_AUTH=y
+CONFIG_NVME_TARGET=m
+CONFIG_NVME_TARGET_PASSTHRU=y
+CONFIG_NVME_TARGET_LOOP=m
+CONFIG_NVME_TARGET_FC=m
+CONFIG_NVME_TARGET_TCP=m
+CONFIG_NVME_TARGET_AUTH=y
+CONFIG_DUMMY_IRQ=m
+CONFIG_PHANTOM=m
+CONFIG_ICS932S401=m
+CONFIG_ENCLOSURE_SERVICES=m
+CONFIG_HI6421V600_IRQ=m
+CONFIG_APDS9802ALS=m
+CONFIG_ISL29003=m
+CONFIG_ISL29020=m
+CONFIG_SENSORS_TSL2550=m
+CONFIG_SENSORS_BH1770=m
+CONFIG_SENSORS_APDS990X=m
+CONFIG_HMC6352=m
+CONFIG_DS1682=m
+CONFIG_LATTICE_ECP3_CONFIG=m
+CONFIG_SRAM=y
+CONFIG_DW_XDATA_PCIE=m
+CONFIG_XILINX_SDFEC=m
+CONFIG_OPEN_DICE=m
+CONFIG_VCPU_STALL_DETECTOR=m
+CONFIG_C2PORT=m
+CONFIG_EEPROM_AT24=m
+CONFIG_EEPROM_AT25=m
+CONFIG_EEPROM_LEGACY=m
+CONFIG_EEPROM_MAX6875=m
+CONFIG_EEPROM_93XX46=m
+CONFIG_EEPROM_IDT_89HPESX=m
+CONFIG_EEPROM_EE1004=m
+CONFIG_TI_ST=m
+CONFIG_SENSORS_LIS3_I2C=m
+CONFIG_GENWQE=m
+CONFIG_ECHO=m
+CONFIG_BCM_VK=m
+CONFIG_BCM_VK_TTY=y
+CONFIG_MISC_ALCOR_PCI=m
+CONFIG_MISC_RTSX_PCI=m
+CONFIG_MISC_RTSX_USB=m
+CONFIG_UACCE=m
+CONFIG_PVPANIC=y
+CONFIG_PVPANIC_MMIO=m
+CONFIG_PVPANIC_PCI=m
+CONFIG_GP_PCI1XXXX=m
+CONFIG_RAID_ATTRS=m
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+CONFIG_CHR_DEV_ST=m
+CONFIG_BLK_DEV_SR=y
+CONFIG_CHR_DEV_SG=y
+# CONFIG_BLK_DEV_BSG is not set
+CONFIG_CHR_DEV_SCH=y
+CONFIG_SCSI_FC_ATTRS=m
+CONFIG_SCSI_SAS_LIBSAS=m
+CONFIG_SCSI_SAS_ATA=y
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=m
+CONFIG_SATA_MOBILE_LPM_POLICY=3
+CONFIG_SATA_AHCI_PLATFORM=m
+CONFIG_AHCI_DWC=m
+CONFIG_AHCI_CEVA=m
+CONFIG_AHCI_ESWIN=y
+CONFIG_SATA_INIC162X=m
+CONFIG_SATA_ACARD_AHCI=m
+CONFIG_SATA_SIL24=m
+CONFIG_PDC_ADMA=m
+CONFIG_SATA_QSTOR=m
+CONFIG_SATA_SX4=m
+CONFIG_ATA_PIIX=m
+CONFIG_SATA_DWC=m
+CONFIG_SATA_DWC_OLD_DMA=y
+CONFIG_SATA_MV=m
+CONFIG_SATA_NV=m
+CONFIG_SATA_PROMISE=m
+CONFIG_SATA_SIL=m
+CONFIG_SATA_SIS=m
+CONFIG_SATA_SVW=m
+CONFIG_SATA_ULI=m
+CONFIG_SATA_VIA=m
+CONFIG_SATA_VITESSE=m
+CONFIG_ATA_GENERIC=m
+CONFIG_MD=y
+CONFIG_BCACHE=m
+CONFIG_BCACHE_ASYNC_REGISTRATION=y
+CONFIG_BLK_DEV_DM=y
+CONFIG_DM_UNSTRIPED=m
+CONFIG_DM_CRYPT=m
+CONFIG_DM_SNAPSHOT=m
+CONFIG_DM_THIN_PROVISIONING=m
+CONFIG_DM_CACHE=m
+CONFIG_DM_WRITECACHE=m
+CONFIG_DM_EBS=m
+CONFIG_DM_ERA=m
+CONFIG_DM_CLONE=m
+CONFIG_DM_MIRROR=m
+CONFIG_DM_LOG_USERSPACE=m
+CONFIG_DM_ZERO=m
+CONFIG_DM_MULTIPATH=m
+CONFIG_DM_MULTIPATH_QL=m
+CONFIG_DM_MULTIPATH_ST=m
+CONFIG_DM_MULTIPATH_HST=m
+CONFIG_DM_MULTIPATH_IOA=m
+CONFIG_DM_DELAY=m
+CONFIG_DM_INIT=y
+CONFIG_DM_UEVENT=y
+CONFIG_DM_FLAKEY=m
+CONFIG_DM_VERITY=m
+CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y
+CONFIG_DM_SWITCH=m
+CONFIG_DM_LOG_WRITES=m
+CONFIG_DM_ZONED=m
+CONFIG_DM_AUDIT=y
+CONFIG_TARGET_CORE=m
+CONFIG_TCM_IBLOCK=m
+CONFIG_TCM_FILEIO=m
+CONFIG_TCM_PSCSI=m
+CONFIG_TCM_USER2=m
+CONFIG_LOOPBACK_TARGET=m
+CONFIG_ISCSI_TARGET=m
+CONFIG_REMOTE_TARGET=m
+CONFIG_FUSION=y
+CONFIG_FUSION_SPI=m
+CONFIG_FUSION_FC=m
+CONFIG_FUSION_SAS=m
+CONFIG_FUSION_CTL=m
+CONFIG_FUSION_LAN=m
+CONFIG_FUSION_LOGGING=y
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_EQUALIZER=m
+CONFIG_NET_FC=y
+CONFIG_IFB=m
+CONFIG_NET_TEAM=m
+CONFIG_NET_TEAM_MODE_BROADCAST=m
+CONFIG_NET_TEAM_MODE_ROUNDROBIN=m
+CONFIG_NET_TEAM_MODE_RANDOM=m
+CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m
+CONFIG_NET_TEAM_MODE_LOADBALANCE=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_BAREUDP=m
+CONFIG_GTP=m
+CONFIG_AMT=m
+CONFIG_MACSEC=m
+CONFIG_NETCONSOLE=m
+CONFIG_NETCONSOLE_DYNAMIC=y
+CONFIG_TUN=y
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=y
+CONFIG_NLMON=m
+CONFIG_NET_VRF=m
+CONFIG_VSOCKMON=m
+CONFIG_MHI_NET=m
+CONFIG_ARCNET=m
+CONFIG_ARCNET_1201=m
+CONFIG_ARCNET_1051=m
+CONFIG_ARCNET_RAW=m
+CONFIG_ARCNET_CAP=m
+CONFIG_ARCNET_COM90xx=m
+CONFIG_ARCNET_COM90xxIO=m
+CONFIG_ARCNET_RIM_I=m
+CONFIG_ARCNET_COM20020=m
+CONFIG_ARCNET_COM20020_PCI=m
+CONFIG_ATM_DUMMY=m
+CONFIG_ATM_TCP=m
+CONFIG_ATM_LANAI=m
+CONFIG_ATM_ENI=m
+CONFIG_ATM_NICSTAR=m
+CONFIG_ATM_IDT77252=m
+CONFIG_ATM_IA=m
+CONFIG_ATM_FORE200E=m
+CONFIG_ATM_HE=m
+CONFIG_ATM_HE_USE_SUNI=y
+CONFIG_ATM_SOLOS=m
+CONFIG_CAIF_DRIVERS=y
+CONFIG_CAIF_TTY=m
+CONFIG_CAIF_VIRTIO=m
+CONFIG_B53_SPI_DRIVER=m
+CONFIG_B53_MDIO_DRIVER=m
+CONFIG_B53_MMAP_DRIVER=m
+CONFIG_B53_SRAB_DRIVER=m
+CONFIG_B53_SERDES=m
+CONFIG_NET_DSA_BCM_SF2=m
+CONFIG_NET_DSA_HIRSCHMANN_HELLCREEK=m
+CONFIG_NET_DSA_LANTIQ_GSWIP=m
+CONFIG_NET_DSA_MT7530=m
+CONFIG_NET_DSA_MV88E6060=m
+CONFIG_NET_DSA_MICROCHIP_KSZ_COMMON=m
+CONFIG_NET_DSA_MICROCHIP_KSZ9477_I2C=m
+CONFIG_NET_DSA_MICROCHIP_KSZ_SPI=m
+CONFIG_NET_DSA_MICROCHIP_KSZ_PTP=y
+CONFIG_NET_DSA_MICROCHIP_KSZ8863_SMI=m
+CONFIG_NET_DSA_MV88E6XXX=m
+CONFIG_NET_DSA_MV88E6XXX_PTP=y
+CONFIG_NET_DSA_MSCC_OCELOT_EXT=m
+CONFIG_NET_DSA_MSCC_SEVILLE=m
+CONFIG_NET_DSA_AR9331=m
+CONFIG_NET_DSA_QCA8K=m
+CONFIG_NET_DSA_QCA8K_LEDS_SUPPORT=y
+CONFIG_NET_DSA_SJA1105=m
+CONFIG_NET_DSA_SJA1105_PTP=y
+CONFIG_NET_DSA_SJA1105_TAS=y
+CONFIG_NET_DSA_SJA1105_VL=y
+CONFIG_NET_DSA_XRS700X_I2C=m
+CONFIG_NET_DSA_XRS700X_MDIO=m
+CONFIG_NET_DSA_SMSC_LAN9303_I2C=m
+CONFIG_NET_DSA_SMSC_LAN9303_MDIO=m
+CONFIG_NET_DSA_VITESSE_VSC73XX_SPI=m
+CONFIG_NET_DSA_VITESSE_VSC73XX_PLATFORM=m
+CONFIG_VORTEX=m
+CONFIG_TYPHOON=m
+CONFIG_ADAPTEC_STARFIRE=m
+CONFIG_ET131X=m
+CONFIG_SLICOSS=m
+CONFIG_ACENIC=m
+CONFIG_ALTERA_TSE=m
+CONFIG_ENA_ETHERNET=m
+CONFIG_AMD8111_ETH=m
+CONFIG_PCNET32=m
+CONFIG_PDS_CORE=m
+CONFIG_AQTION=m
+CONFIG_SPI_AX88796C=m
+CONFIG_ATL2=m
+CONFIG_ATL1=m
+CONFIG_ATL1E=m
+CONFIG_ATL1C=m
+CONFIG_ALX=m
+CONFIG_B44=m
+CONFIG_BCMGENET=m
+CONFIG_BNX2=m
+CONFIG_TIGON3=m
+CONFIG_BNX2X=m
+CONFIG_SYSTEMPORT=m
+CONFIG_BNXT=m
+CONFIG_BNXT_DCB=y
+CONFIG_MACB=m
+CONFIG_MACB_PCI=m
+# CONFIG_NET_VENDOR_CAVIUM is not set
+# CONFIG_NET_VENDOR_CHELSIO is not set
+CONFIG_ENIC=m
+CONFIG_GEMINI_ETHERNET=m
+CONFIG_DM9051=m
+CONFIG_DNET=m
+CONFIG_NET_TULIP=y
+CONFIG_DE2104X=m
+CONFIG_TULIP=m
+CONFIG_WINBOND_840=m
+CONFIG_DM9102=m
+CONFIG_ULI526X=m
+CONFIG_DL2K=m
+CONFIG_SUNDANCE=m
+CONFIG_TSNEP=m
+CONFIG_EZCHIP_NPS_MANAGEMENT_ENET=m
+CONFIG_FUN_ETH=m
+CONFIG_E100=m
+CONFIG_E1000=m
+CONFIG_E1000E=m
+CONFIG_IGB=m
+CONFIG_IGBVF=m
+CONFIG_IXGBE=m
+CONFIG_IXGBE_DCB=y
+CONFIG_IXGBEVF=m
+CONFIG_I40E=m
+CONFIG_I40E_DCB=y
+CONFIG_I40EVF=m
+CONFIG_ICE=m
+CONFIG_FM10K=m
+CONFIG_IGC=m
+CONFIG_JME=m
+CONFIG_ADIN1110=m
+CONFIG_LITEX_LITEETH=m
+CONFIG_MVMDIO=m
+CONFIG_SKGE=m
+CONFIG_SKGE_GENESIS=y
+CONFIG_SKY2=m
+CONFIG_OCTEON_EP=m
+CONFIG_PRESTERA=m
+CONFIG_MLXSW_CORE=m
+CONFIG_KS8842=m
+CONFIG_KS8851=m
+CONFIG_KS8851_MLL=m
+CONFIG_KSZ884X_PCI=m
+CONFIG_ENC28J60=m
+CONFIG_ENCX24J600=m
+CONFIG_LAN743X=m
+CONFIG_LAN966X_SWITCH=m
+CONFIG_MSCC_OCELOT_SWITCH=m
+CONFIG_MYRI10GE=m
+CONFIG_FEALNX=m
+CONFIG_NI_XGE_MANAGEMENT_ENET=m
+CONFIG_NATSEMI=m
+CONFIG_NS83820=m
+CONFIG_S2IO=m
+CONFIG_NFP=m
+CONFIG_NE2K_PCI=m
+CONFIG_FORCEDETH=m
+CONFIG_ETHOC=m
+CONFIG_HAMACHI=m
+CONFIG_YELLOWFIN=m
+CONFIG_IONIC=m
+CONFIG_QLA3XXX=m
+CONFIG_QLCNIC=m
+CONFIG_NETXEN_NIC=m
+CONFIG_QED=m
+CONFIG_QEDE=m
+CONFIG_BNA=m
+CONFIG_QCA7000_SPI=m
+CONFIG_QCA7000_UART=m
+CONFIG_QCOM_EMAC=m
+CONFIG_RMNET=m
+CONFIG_R6040=m
+CONFIG_8139CP=m
+CONFIG_8139TOO=m
+CONFIG_8139TOO_8129=y
+CONFIG_ROCKER=m
+CONFIG_SXGBE_ETH=m
+CONFIG_SC92031=m
+CONFIG_SIS900=m
+CONFIG_SIS190=m
+CONFIG_SFC=m
+CONFIG_SFC_FALCON=m
+CONFIG_SFC_SIENA=m
+CONFIG_SFC_SIENA_SRIOV=y
+CONFIG_EPIC100=m
+CONFIG_SMSC911X=m
+CONFIG_SMSC9420=m
+CONFIG_STMMAC_ETH=m
+CONFIG_DWMAC_DWC_QOS_ETH=m
+CONFIG_DWMAC_EIC7700=m
+CONFIG_DWMAC_INTEL_PLAT=m
+CONFIG_STMMAC_PCI=m
+CONFIG_HAPPYMEAL=m
+CONFIG_SUNGEM=m
+CONFIG_CASSINI=m
+CONFIG_NIU=m
+CONFIG_DWC_XLGMAC=m
+CONFIG_DWC_XLGMAC_PCI=m
+CONFIG_TEHUTI=m
+CONFIG_TLAN=m
+CONFIG_MSE102X=m
+CONFIG_VIA_RHINE=m
+CONFIG_VIA_RHINE_MMIO=y
+CONFIG_VIA_VELOCITY=m
+CONFIG_NGBE=m
+CONFIG_TXGBE=m
+CONFIG_WIZNET_W5100=m
+CONFIG_WIZNET_W5300=m
+CONFIG_WIZNET_W5100_SPI=m
+CONFIG_XILINX_EMACLITE=m
+CONFIG_XILINX_AXI_EMAC=m
+CONFIG_XILINX_LL_TEMAC=m
+CONFIG_FDDI=y
+CONFIG_DEFXX=m
+CONFIG_SKFP=m
+CONFIG_PHYLIB=y
+CONFIG_LED_TRIGGER_PHY=y
+CONFIG_AMD_PHY=m
+CONFIG_ADIN_PHY=m
+CONFIG_ADIN1100_PHY=m
+CONFIG_AQUANTIA_PHY=m
+CONFIG_BROADCOM_PHY=m
+CONFIG_BCM54140_PHY=m
+CONFIG_BCM84881_PHY=m
+CONFIG_BCM87XX_PHY=m
+CONFIG_CICADA_PHY=m
+CONFIG_CORTINA_PHY=m
+CONFIG_DAVICOM_PHY=m
+CONFIG_ICPLUS_PHY=m
+CONFIG_LXT_PHY=m
+CONFIG_INTEL_XWAY_PHY=m
+CONFIG_LSI_ET1011C_PHY=m
+CONFIG_MARVELL_PHY=m
+CONFIG_MARVELL_88Q2XXX_PHY=m
+CONFIG_MARVELL_88X2222_PHY=m
+CONFIG_MAXLINEAR_GPHY=m
+CONFIG_MICROCHIP_T1S_PHY=m
+CONFIG_MICROCHIP_T1_PHY=m
+CONFIG_MICROSEMI_PHY=m
+CONFIG_MOTORCOMM_PHY=m
+CONFIG_NATIONAL_PHY=m
+CONFIG_NXP_CBTX_PHY=m
+CONFIG_NXP_C45_TJA11XX_PHY=m
+CONFIG_NXP_TJA11XX_PHY=m
+CONFIG_NCN26000_PHY=m
+CONFIG_AT803X_PHY=m
+CONFIG_QSEMI_PHY=m
+CONFIG_RENESAS_PHY=m
+CONFIG_ROCKCHIP_PHY=m
+CONFIG_STE10XP=m
+CONFIG_TERANETICS_PHY=m
+CONFIG_DP83822_PHY=m
+CONFIG_DP83TC811_PHY=m
+CONFIG_DP83848_PHY=m
+CONFIG_DP83867_PHY=m
+CONFIG_DP83869_PHY=m
+CONFIG_DP83TD510_PHY=m
+CONFIG_XILINX_GMII2RGMII=m
+CONFIG_MICREL_KS8995MA=m
+CONFIG_PSE_CONTROLLER=y
+CONFIG_PSE_REGULATOR=m
+CONFIG_MCTP_SERIAL=m
+CONFIG_MCTP_TRANSPORT_I2C=m
+CONFIG_MDIO_GPIO=m
+CONFIG_MDIO_HISI_FEMAC=m
+CONFIG_MDIO_MVUSB=m
+CONFIG_MDIO_OCTEON=m
+CONFIG_MDIO_IPQ4019=m
+CONFIG_MDIO_IPQ8064=m
+CONFIG_MDIO_BUS_MUX_GPIO=m
+CONFIG_MDIO_BUS_MUX_MULTIPLEXER=m
+CONFIG_MDIO_BUS_MUX_MMIOREG=m
+CONFIG_PLIP=m
+CONFIG_PPP=y
+CONFIG_PPP_BSDCOMP=m
+CONFIG_PPP_DEFLATE=m
+CONFIG_PPP_FILTER=y
+CONFIG_PPP_MPPE=m
+CONFIG_PPP_MULTILINK=y
+CONFIG_PPPOATM=m
+CONFIG_PPPOE=m
+CONFIG_PPTP=m
+CONFIG_PPPOL2TP=m
+CONFIG_PPP_ASYNC=m
+CONFIG_PPP_SYNC_TTY=m
+CONFIG_SLIP=m
+CONFIG_SLIP_COMPRESSED=y
+CONFIG_SLIP_SMART=y
+CONFIG_SLIP_MODE_SLIP6=y
+CONFIG_USB_NET_DRIVERS=m
+CONFIG_USB_CATC=m
+CONFIG_USB_KAWETH=m
+CONFIG_USB_PEGASUS=m
+CONFIG_USB_RTL8150=m
+CONFIG_USB_RTL8152=m
+CONFIG_USB_LAN78XX=m
+CONFIG_USB_NET_CDC_EEM=m
+CONFIG_USB_NET_HUAWEI_CDC_NCM=m
+CONFIG_USB_NET_CDC_MBIM=m
+CONFIG_USB_NET_DM9601=m
+CONFIG_USB_NET_SR9700=m
+CONFIG_USB_NET_SR9800=m
+CONFIG_USB_NET_SMSC75XX=m
+CONFIG_USB_NET_SMSC95XX=m
+CONFIG_USB_NET_GL620A=m
+CONFIG_USB_NET_PLUSB=m
+CONFIG_USB_NET_MCS7830=m
+CONFIG_USB_ALI_M5632=y
+CONFIG_USB_AN2720=y
+CONFIG_USB_EPSON2888=y
+CONFIG_USB_KC2190=y
+CONFIG_USB_NET_CX82310_ETH=m
+CONFIG_USB_NET_KALMIA=m
+CONFIG_USB_NET_QMI_WWAN=m
+CONFIG_USB_HSO=m
+CONFIG_USB_NET_INT51X1=m
+CONFIG_USB_CDC_PHONET=m
+CONFIG_USB_IPHETH=m
+CONFIG_USB_SIERRA_NET=m
+CONFIG_USB_VL600=m
+CONFIG_USB_NET_CH9200=m
+CONFIG_USB_NET_AQC111=m
+CONFIG_ADM8211=m
+CONFIG_ATH5K=m
+CONFIG_ATH9K=m
+CONFIG_ATH9K_AHB=y
+CONFIG_ATH9K_DEBUGFS=y
+CONFIG_ATH9K_STATION_STATISTICS=y
+CONFIG_ATH9K_WOW=y
+CONFIG_ATH9K_CHANNEL_CONTEXT=y
+CONFIG_ATH9K_PCI_NO_EEPROM=m
+CONFIG_ATH9K_HTC=m
+CONFIG_ATH9K_HTC_DEBUGFS=y
+CONFIG_ATH9K_HWRNG=y
+CONFIG_ATH9K_COMMON_SPECTRAL=y
+CONFIG_CARL9170=m
+CONFIG_CARL9170_HWRNG=y
+CONFIG_ATH6KL=m
+CONFIG_ATH6KL_SDIO=m
+CONFIG_ATH6KL_USB=m
+CONFIG_AR5523=m
+CONFIG_WIL6210=m
+CONFIG_WIL6210_TRACING=y
+CONFIG_ATH10K=m
+CONFIG_ATH10K_PCI=m
+CONFIG_ATH10K_AHB=y
+CONFIG_ATH10K_SDIO=m
+CONFIG_ATH10K_USB=m
+CONFIG_ATH10K_DEBUGFS=y
+CONFIG_ATH10K_SPECTRAL=y
+CONFIG_ATH10K_TRACING=y
+CONFIG_WCN36XX=m
+CONFIG_ATH11K=m
+CONFIG_ATH11K_PCI=m
+CONFIG_ATH11K_DEBUGFS=y
+CONFIG_ATH11K_TRACING=y
+CONFIG_ATH11K_SPECTRAL=y
+CONFIG_ATH12K=m
+CONFIG_ATH12K_TRACING=y
+CONFIG_ATMEL=m
+CONFIG_PCI_ATMEL=m
+CONFIG_AT76C50X_USB=m
+CONFIG_B43=m
+CONFIG_B43LEGACY=m
+# CONFIG_B43LEGACY_DEBUG is not set
+CONFIG_BRCMSMAC=m
+CONFIG_BRCMFMAC=m
+CONFIG_BRCMFMAC_USB=y
+CONFIG_BRCMFMAC_PCIE=y
+CONFIG_BRCM_TRACING=y
+CONFIG_AIRO=m
+CONFIG_IPW2100=m
+CONFIG_IPW2100_MONITOR=y
+CONFIG_IPW2200=m
+CONFIG_IPW2200_MONITOR=y
+CONFIG_IPW2200_PROMISCUOUS=y
+CONFIG_IPW2200_QOS=y
+CONFIG_IWL4965=m
+CONFIG_IWL3945=m
+CONFIG_IWLEGACY_DEBUGFS=y
+CONFIG_IWLWIFI=m
+CONFIG_IWLDVM=m
+CONFIG_IWLMVM=m
+CONFIG_IWLWIFI_DEBUGFS=y
+CONFIG_HOSTAP=m
+CONFIG_HOSTAP_FIRMWARE=y
+CONFIG_HOSTAP_FIRMWARE_NVRAM=y
+CONFIG_HOSTAP_PLX=m
+CONFIG_HOSTAP_PCI=m
+CONFIG_HERMES=m
+CONFIG_PLX_HERMES=m
+CONFIG_TMD_HERMES=m
+CONFIG_NORTEL_HERMES=m
+CONFIG_ORINOCO_USB=m
+CONFIG_P54_COMMON=m
+CONFIG_P54_USB=m
+CONFIG_P54_PCI=m
+CONFIG_P54_SPI=m
+CONFIG_LIBERTAS=m
+CONFIG_LIBERTAS_USB=m
+CONFIG_LIBERTAS_SDIO=m
+CONFIG_LIBERTAS_SPI=m
+CONFIG_LIBERTAS_MESH=y
+CONFIG_LIBERTAS_THINFIRM=m
+CONFIG_LIBERTAS_THINFIRM_USB=m
+CONFIG_MWIFIEX=m
+CONFIG_MWIFIEX_SDIO=m
+CONFIG_MWIFIEX_PCIE=m
+CONFIG_MWIFIEX_USB=m
+CONFIG_MWL8K=m
+CONFIG_MT7601U=m
+CONFIG_MT76x0U=m
+CONFIG_MT76x0E=m
+CONFIG_MT76x2E=m
+CONFIG_MT76x2U=m
+CONFIG_MT7603E=m
+CONFIG_MT7615E=m
+CONFIG_MT7663U=m
+CONFIG_MT7663S=m
+CONFIG_MT7915E=m
+CONFIG_MT7921E=m
+CONFIG_MT7921S=m
+CONFIG_MT7921U=m
+CONFIG_MT7996E=m
+CONFIG_WILC1000_SDIO=m
+CONFIG_WILC1000_SPI=m
+CONFIG_WILC1000_HW_OOB_INTR=y
+CONFIG_PLFXLC=m
+CONFIG_RT2X00=m
+CONFIG_RT2400PCI=m
+CONFIG_RT2500PCI=m
+CONFIG_RT61PCI=m
+CONFIG_RT2800PCI=m
+CONFIG_RT2500USB=m
+CONFIG_RT73USB=m
+CONFIG_RT2800USB=m
+CONFIG_RT2800USB_RT3573=y
+CONFIG_RT2800USB_RT53XX=y
+CONFIG_RT2800USB_RT55XX=y
+CONFIG_RT2800USB_UNKNOWN=y
+CONFIG_RTL8180=m
+CONFIG_RTL8187=m
+CONFIG_RTL8192CE=m
+CONFIG_RTL8192SE=m
+CONFIG_RTL8192DE=m
+CONFIG_RTL8723AE=m
+CONFIG_RTL8723BE=m
+CONFIG_RTL8188EE=m
+CONFIG_RTL8192EE=m
+CONFIG_RTL8821AE=m
+CONFIG_RTL8192CU=m
+# CONFIG_RTLWIFI_DEBUG is not set
+CONFIG_RTL8XXXU=m
+CONFIG_RTL8XXXU_UNTESTED=y
+CONFIG_RTW88=m
+CONFIG_RTW88_8822BE=m
+CONFIG_RTW88_8822BS=m
+CONFIG_RTW88_8822BU=m
+CONFIG_RTW88_8822CE=m
+CONFIG_RTW88_8822CS=m
+CONFIG_RTW88_8822CU=m
+CONFIG_RTW88_8723DE=m
+CONFIG_RTW88_8723DS=m
+CONFIG_RTW88_8723DU=m
+CONFIG_RTW88_8821CE=m
+CONFIG_RTW88_8821CS=m
+CONFIG_RTW88_8821CU=m
+CONFIG_RTW88_DEBUG=y
+CONFIG_RTW88_DEBUGFS=y
+CONFIG_RTW89=m
+CONFIG_RTW89_8851BE=m
+CONFIG_RTW89_8852AE=m
+CONFIG_RTW89_8852BE=m
+CONFIG_RTW89_8852CE=m
+CONFIG_RTW89_DEBUGMSG=y
+CONFIG_RTW89_DEBUGFS=y
+CONFIG_RSI_91X=m
+# CONFIG_RSI_DEBUGFS is not set
+CONFIG_WFX=m
+CONFIG_CW1200=m
+CONFIG_CW1200_WLAN_SDIO=m
+CONFIG_CW1200_WLAN_SPI=m
+CONFIG_WL1251=m
+CONFIG_WL1251_SPI=m
+CONFIG_WL1251_SDIO=m
+CONFIG_WL12XX=m
+CONFIG_WL18XX=m
+CONFIG_WLCORE_SPI=m
+CONFIG_WLCORE_SDIO=m
+CONFIG_USB_ZD1201=m
+CONFIG_ZD1211RW=m
+CONFIG_QTNFMAC_PCIE=m
+CONFIG_USB_NET_RNDIS_WLAN=m
+CONFIG_MAC80211_HWSIM=m
+CONFIG_VIRT_WIFI=m
+CONFIG_WAN=y
+CONFIG_HDLC=m
+CONFIG_HDLC_RAW=m
+CONFIG_HDLC_RAW_ETH=m
+CONFIG_HDLC_CISCO=m
+CONFIG_HDLC_FR=m
+CONFIG_HDLC_PPP=m
+CONFIG_HDLC_X25=m
+CONFIG_PCI200SYN=m
+CONFIG_WANXL=m
+CONFIG_PC300TOO=m
+CONFIG_FARSYNC=m
+CONFIG_LAPBETHER=m
+CONFIG_IEEE802154_FAKELB=m
+CONFIG_IEEE802154_AT86RF230=m
+CONFIG_IEEE802154_MRF24J40=m
+CONFIG_IEEE802154_CC2520=m
+CONFIG_IEEE802154_ATUSB=m
+CONFIG_IEEE802154_ADF7242=m
+CONFIG_IEEE802154_CA8210=m
+CONFIG_IEEE802154_CA8210_DEBUGFS=y
+CONFIG_IEEE802154_MCR20A=m
+CONFIG_IEEE802154_HWSIM=m
+CONFIG_WWAN=m
+CONFIG_WWAN_HWSIM=m
+CONFIG_MHI_WWAN_CTRL=m
+CONFIG_MHI_WWAN_MBIM=m
+CONFIG_RPMSG_WWAN_CTRL=m
+CONFIG_IOSM=m
+CONFIG_MTK_T7XX=m
+CONFIG_VMXNET3=m
+CONFIG_NETDEVSIM=m
+CONFIG_ISDN=y
+CONFIG_MISDN=m
+CONFIG_MISDN_DSP=m
+CONFIG_MISDN_L1OIP=m
+CONFIG_MISDN_HFCPCI=m
+CONFIG_MISDN_HFCMULTI=m
+CONFIG_MISDN_HFCUSB=m
+CONFIG_MISDN_AVMFRITZ=m
+CONFIG_MISDN_SPEEDFAX=m
+CONFIG_MISDN_INFINEON=m
+CONFIG_MISDN_W6692=m
+CONFIG_MISDN_NETJET=m
+CONFIG_INPUT_LEDS=m
+CONFIG_INPUT_SPARSEKMAP=m
+CONFIG_INPUT_MATRIXKMAP=m
+CONFIG_INPUT_MOUSEDEV=y
+CONFIG_INPUT_MOUSEDEV_PSAUX=y
+CONFIG_INPUT_JOYDEV=m
+CONFIG_INPUT_EVDEV=y
+CONFIG_INPUT_EVBUG=m
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_SERIO_SERPORT=m
+CONFIG_SERIO_PARKBD=m
+CONFIG_SERIO_PCIPS2=m
+CONFIG_SERIO_LIBPS2=y
+CONFIG_SERIO_RAW=m
+CONFIG_SERIO_ALTERA_PS2=m
+CONFIG_SERIO_PS2MULT=m
+CONFIG_SERIO_ARC_PS2=m
+CONFIG_SERIO_APBPS2=m
+CONFIG_SERIO_GPIO_PS2=m
+CONFIG_USERIO=m
+CONFIG_LEGACY_PTY_COUNT=0
+# CONFIG_LEGACY_TIOCSTI is not set
+CONFIG_SERIAL_8250=y
+# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_DW=y
+# CONFIG_SERIAL_8250_PERICOM is not set
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_SEMIHOST=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_TTY_PRINTK=y
+CONFIG_PRINTER=m
+CONFIG_PPDEV=m
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_IPMI_HANDLER=m
+CONFIG_IPMI_DEVICE_INTERFACE=m
+CONFIG_IPMI_SSIF=m
+CONFIG_IPMI_IPMB=m
+CONFIG_IPMI_WATCHDOG=m
+CONFIG_IPMI_POWEROFF=m
+CONFIG_SSIF_IPMI_BMC=m
+CONFIG_IPMB_DEVICE_INTERFACE=m
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_TIMERIOMEM=m
+CONFIG_HW_RANDOM_BA431=m
+CONFIG_HW_RANDOM_VIRTIO=m
+CONFIG_HW_RANDOM_CCTRNG=m
+CONFIG_HW_RANDOM_XIPHERA=m
+CONFIG_APPLICOM=m
+CONFIG_TCG_TIS=y
+CONFIG_TCG_TIS_SPI=m
+CONFIG_TCG_TIS_SPI_CR50=y
+CONFIG_TCG_TIS_I2C=m
+CONFIG_TCG_TIS_I2C_CR50=m
+CONFIG_TCG_TIS_I2C_ATMEL=m
+CONFIG_TCG_TIS_I2C_INFINEON=m
+CONFIG_TCG_TIS_I2C_NUVOTON=m
+CONFIG_TCG_ATMEL=m
+CONFIG_TCG_VTPM_PROXY=m
+CONFIG_TCG_TIS_ST33ZP24_I2C=m
+CONFIG_TCG_TIS_ST33ZP24_SPI=m
+CONFIG_XILLYBUS=m
+CONFIG_XILLYBUS_PCIE=m
+CONFIG_XILLYBUS_OF=m
+CONFIG_XILLYUSB=m
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_DESIGNWARE_PLATFORM=y
+CONFIG_I2C_DESIGNWARE_PCI=m
+CONFIG_I2C_GPIO=m
+CONFIG_I2C_SLAVE=y
+CONFIG_I2C_SLAVE_EEPROM=m
+CONFIG_SPI=y
+CONFIG_SPI_DESIGNWARE=m
+CONFIG_SPI_DW_DMA=y
+CONFIG_SPI_DW_PCI=m
+CONFIG_SPI_DW_MMIO=m
+CONFIG_SPI_ESWIN_BOOTSPI=m
+CONFIG_SPI_SIFIVE=y
+CONFIG_SPMI=m
+CONFIG_SPMI_HISI3670=m
+CONFIG_HSI=m
+CONFIG_HSI_CHAR=m
+CONFIG_PPS_CLIENT_LDISC=m
+CONFIG_PPS_CLIENT_PARPORT=m
+CONFIG_PPS_CLIENT_GPIO=m
+CONFIG_DP83640_PHY=m
+CONFIG_PTP_1588_CLOCK_INES=m
+CONFIG_PTP_1588_CLOCK_IDT82P33=m
+CONFIG_PTP_1588_CLOCK_IDTCM=m
+CONFIG_PTP_1588_CLOCK_MOCK=m
+CONFIG_PTP_1588_CLOCK_OCP=m
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_EIC7700=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_DWAPB=m
+CONFIG_POWER_RESET_GPIO=y
+CONFIG_POWER_RESET_GPIO_RESTART=y
+CONFIG_POWER_RESET_LTC2952=y
+CONFIG_POWER_RESET_REGULATOR=y
+CONFIG_POWER_RESET_RESTART=y
+CONFIG_SYSCON_REBOOT_MODE=m
+CONFIG_NVMEM_REBOOT_MODE=m
+CONFIG_SENSORS_ESWIN_FAN_CONTROL=y
+CONFIG_SENSORS_ESWIN_PVT=y
+CONFIG_SENSORS_GPIO_FAN=m
+CONFIG_SENSORS_PAC1934=m
+CONFIG_PMBUS=m
+CONFIG_SENSORS_PWM_FAN=m
+CONFIG_SENSORS_INA2XX=m
+CONFIG_SENSORS_INA238=m
+CONFIG_SENSORS_INA3221=m
+CONFIG_THERMAL=y
+CONFIG_THERMAL_NETLINK=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_GOV_FAIR_SHARE=y
+CONFIG_THERMAL_GOV_BANG_BANG=y
+CONFIG_THERMAL_GOV_USER_SPACE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_CPU_IDLE_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_THERMAL_MMIO=m
+CONFIG_WATCHDOG=y
+CONFIG_WATCHDOG_CORE=y
+CONFIG_WATCHDOG_SYSFS=y
+CONFIG_WATCHDOG_PRETIMEOUT_GOV=y
+CONFIG_WATCHDOG_PRETIMEOUT_GOV_PANIC=m
+CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_NOOP=y
+CONFIG_DW_WATCHDOG=m
+CONFIG_SSB_SDIOHOST=y
+CONFIG_SSB_DRIVER_GPIO=y
+CONFIG_BCMA_HOST_SOC=y
+CONFIG_BCMA_DRIVER_GMAC_CMN=y
+CONFIG_BCMA_DRIVER_GPIO=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_REGULATOR_USERSPACE_CONSUMER=m
+CONFIG_MEDIA_CEC_SUPPORT=y
+CONFIG_CEC_CH7322=m
+CONFIG_CEC_GPIO=m
+CONFIG_USB_PULSE8_CEC=m
+CONFIG_USB_RAINSHADOW_CEC=m
+CONFIG_MEDIA_SUPPORT=m
+CONFIG_MEDIA_SUPPORT_FILTER=y
+CONFIG_MEDIA_PLATFORM_SUPPORT=y
+CONFIG_MEDIA_TEST_SUPPORT=y
+CONFIG_V4L2_FLASH_LED_CLASS=m
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_MEM2MEM_DEINTERLACE=m
+CONFIG_VIDEO_MUX=m
+CONFIG_V4L_TEST_DRIVERS=y
+CONFIG_VIDEO_VIM2M=m
+CONFIG_VIDEO_VICODEC=m
+CONFIG_VIDEO_VIMC=m
+CONFIG_VIDEO_VIVID=m
+CONFIG_VIDEO_VIVID_CEC=y
+CONFIG_VIDEO_VISL=m
+CONFIG_VIDEO_I2C=m
+CONFIG_IMG_ASCII_LCD=m
+CONFIG_LCD2S=m
+CONFIG_PANEL=m
+CONFIG_DRM_DEBUG_MM=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_RADEON_USERPTR=y
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AST=y
+CONFIG_DRM_DISPLAY_CONNECTOR=y
+CONFIG_DRM_SIMPLE_BRIDGE=y
+CONFIG_DRM_ESWIN=y
+CONFIG_ESWIN_VIRTUAL_DISPLAY=y
+CONFIG_ESWIN_MMU=y
+CONFIG_ESWIN_DW_HDMI=y
+CONFIG_DW_HDMI_I2S_AUDIO=y
+CONFIG_DW_HDMI_CEC=y
+CONFIG_DRM_IMG_VOLCANIC=m
+CONFIG_DRM_LEGACY=y
+CONFIG_FB=y
+CONFIG_FB_UVESA=m
+CONFIG_FB_EFI=y
+CONFIG_FB_OPENCORES=m
+CONFIG_FB_NVIDIA=m
+CONFIG_FB_NVIDIA_I2C=y
+CONFIG_FB_RIVA=m
+CONFIG_FB_RIVA_I2C=y
+CONFIG_FB_I740=m
+CONFIG_FB_MATROX=m
+CONFIG_FB_MATROX_MILLENIUM=y
+CONFIG_FB_MATROX_MYSTIQUE=y
+CONFIG_FB_MATROX_G=y
+CONFIG_FB_MATROX_I2C=m
+CONFIG_FB_MATROX_MAVEN=m
+CONFIG_FB_RADEON=m
+CONFIG_FB_ATY128=m
+CONFIG_FB_ATY=m
+CONFIG_FB_ATY_CT=y
+CONFIG_FB_ATY_GX=y
+CONFIG_FB_S3=m
+CONFIG_FB_SAVAGE=m
+CONFIG_FB_SAVAGE_I2C=y
+CONFIG_FB_SIS=m
+CONFIG_FB_SIS_300=y
+CONFIG_FB_SIS_315=y
+CONFIG_FB_NEOMAGIC=m
+CONFIG_FB_KYRO=m
+CONFIG_FB_3DFX=m
+# CONFIG_FB_3DFX_I2C is not set
+CONFIG_FB_VOODOO1=m
+CONFIG_FB_VT8623=m
+CONFIG_FB_TRIDENT=m
+CONFIG_FB_ARK=m
+CONFIG_FB_PM3=m
+CONFIG_FB_CARMINE=m
+CONFIG_FB_SMSCUFX=m
+CONFIG_FB_UDL=m
+CONFIG_FB_GOLDFISH=m
+CONFIG_FB_METRONOME=m
+CONFIG_FB_MB862XX=m
+CONFIG_FB_SIMPLE=m
+CONFIG_FB_SSD1307=m
+CONFIG_FB_SM712=m
+CONFIG_FIRMWARE_EDID=y
+CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y
+CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER=y
+CONFIG_DRM_ACCEL=y
+CONFIG_DRM_ACCEL_QAIC=m
+CONFIG_SOUND=m
+# CONFIG_SOUND_OSS_CORE_PRECLAIM is not set
+CONFIG_SND=m
+CONFIG_SND_OSSEMUL=y
+CONFIG_SND_MIXER_OSS=m
+CONFIG_SND_HRTIMER=m
+# CONFIG_SND_CTL_FAST_LOOKUP is not set
+CONFIG_SND_SEQUENCER=m
+CONFIG_SND_SEQ_DUMMY=m
+CONFIG_SND_DUMMY=m
+CONFIG_SND_ALOOP=m
+CONFIG_SND_PCMTEST=m
+CONFIG_SND_VIRMIDI=m
+CONFIG_SND_MTPAV=m
+CONFIG_SND_MTS64=m
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_HWDEP=y
+CONFIG_SND_HDA_INPUT_BEEP=y
+CONFIG_SND_HDA_INPUT_BEEP_MODE=0
+CONFIG_SND_HDA_PATCH_LOADER=y
+CONFIG_SND_HDA_CODEC_REALTEK=m
+CONFIG_SND_HDA_CODEC_ANALOG=m
+CONFIG_SND_HDA_CODEC_SIGMATEL=m
+CONFIG_SND_HDA_CODEC_VIA=m
+CONFIG_SND_HDA_CODEC_HDMI=m
+CONFIG_SND_HDA_CODEC_CIRRUS=m
+CONFIG_SND_HDA_CODEC_CS8409=m
+CONFIG_SND_HDA_CODEC_CONEXANT=m
+CONFIG_SND_HDA_CODEC_CA0110=m
+CONFIG_SND_HDA_CODEC_CA0132=m
+CONFIG_SND_HDA_CODEC_CMEDIA=m
+CONFIG_SND_HDA_CODEC_SI3054=m
+CONFIG_SND_HDA_POWER_SAVE_DEFAULT=1
+CONFIG_SND_USB_AUDIO=m
+CONFIG_SND_USB_AUDIO_MIDI_V2=y
+CONFIG_SND_USB_UA101=m
+CONFIG_SND_USB_CAIAQ=m
+CONFIG_SND_USB_CAIAQ_INPUT=y
+CONFIG_SND_USB_6FIRE=m
+CONFIG_SND_USB_HIFACE=m
+CONFIG_SND_BCD2000=m
+CONFIG_SND_USB_POD=m
+CONFIG_SND_USB_PODHD=m
+CONFIG_SND_USB_TONEPORT=m
+CONFIG_SND_USB_VARIAX=m
+CONFIG_SND_SOC=m
+CONFIG_SND_SOC_AMD_ACP=m
+CONFIG_SND_SOC_AMD_CZ_RT5645_MACH=m
+CONFIG_SND_AMD_ACP_CONFIG=m
+CONFIG_SND_DESIGNWARE_I2S=m
+CONFIG_SND_DESIGNWARE_PCM=y
+CONFIG_SND_ESWIN_DW_I2S=m
+CONFIG_SND_SOC_IMG=y
+CONFIG_SND_SOC_IMG_I2S_IN=m
+CONFIG_SND_SOC_IMG_I2S_OUT=m
+CONFIG_SND_SOC_IMG_PARALLEL_OUT=m
+CONFIG_SND_SOC_IMG_SPDIF_IN=m
+CONFIG_SND_SOC_IMG_SPDIF_OUT=m
+CONFIG_SND_SOC_IMG_PISTACHIO_INTERNAL_DAC=m
+CONFIG_SND_SOC_MTK_BTCVSD=m
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_PCI=m
+CONFIG_SND_SOC_SOF_OF=m
+CONFIG_SND_SOC_BT_SCO=m
+CONFIG_SND_SOC_DMIC=m
+CONFIG_SND_SOC_HDA=m
+CONFIG_SND_SOC_ICS43432=m
+CONFIG_SND_SOC_SIMPLE_AMPLIFIER=m
+CONFIG_SND_SOC_SIMPLE_MUX=m
+CONFIG_SND_SOC_SPDIF=m
+CONFIG_ESWIN_SND_SOC_CODECS=m
+CONFIG_ESWIN_SND_ES8388_CODEC=m
+CONFIG_SND_SIMPLE_CARD=m
+CONFIG_SND_AUDIO_GRAPH_CARD=m
+CONFIG_SND_AUDIO_GRAPH_CARD2=m
+CONFIG_SND_AUDIO_GRAPH_CARD2_CUSTOM_SAMPLE=m
+CONFIG_SND_TEST_COMPONENT=m
+CONFIG_SND_VIRTIO=m
+CONFIG_HID=m
+CONFIG_HID_BATTERY_STRENGTH=y
+CONFIG_HIDRAW=y
+CONFIG_UHID=m
+CONFIG_HID_LOGITECH=m
+CONFIG_HID_LOGITECH_DJ=m
+CONFIG_LOGITECH_FF=y
+CONFIG_LOGIRUMBLEPAD2_FF=y
+CONFIG_LOGIG940_FF=y
+CONFIG_HID_SAMSUNG=m
+CONFIG_HID_SEMITEK=m
+CONFIG_HID_SIGMAMICRO=m
+CONFIG_HID_SPEEDLINK=m
+CONFIG_USB_HID=m
+CONFIG_HID_PID=y
+CONFIG_USB_HIDDEV=y
+CONFIG_USB_KBD=m
+CONFIG_USB_MOUSE=m
+CONFIG_I2C_HID_OF=m
+CONFIG_I2C_HID_OF_ELAN=m
+CONFIG_I2C_HID_OF_GOODIX=m
+CONFIG_USB_LED_TRIG=y
+CONFIG_USB_CONN_GPIO=m
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+CONFIG_USB_DYNAMIC_MINORS=y
+CONFIG_USB_OTG=y
+CONFIG_USB_OTG_FSM=m
+CONFIG_USB_LEDS_TRIGGER_USBPORT=m
+CONFIG_USB_MON=m
+CONFIG_USB_C67X00_HCD=m
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_XHCI_DBGCAP=y
+CONFIG_USB_XHCI_PCI_RENESAS=m
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_FSL=m
+CONFIG_USB_OXU210HP_HCD=m
+CONFIG_USB_ISP116X_HCD=m
+CONFIG_USB_MAX3421_HCD=m
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_UHCI_HCD=y
+CONFIG_USB_SL811_HCD=m
+CONFIG_USB_SL811_HCD_ISO=y
+CONFIG_USB_R8A66597_HCD=m
+CONFIG_USB_HCD_BCMA=m
+CONFIG_USB_HCD_SSB=m
+CONFIG_USB_PRINTER=m
+CONFIG_USB_TMC=m
+CONFIG_USB_STORAGE=y
+CONFIG_USB_STORAGE_REALTEK=y
+CONFIG_USB_STORAGE_DATAFAB=y
+CONFIG_USB_STORAGE_FREECOM=y
+CONFIG_USB_STORAGE_ISD200=y
+CONFIG_USB_STORAGE_USBAT=y
+CONFIG_USB_STORAGE_SDDR09=y
+CONFIG_USB_STORAGE_SDDR55=y
+CONFIG_USB_STORAGE_JUMPSHOT=y
+CONFIG_USB_STORAGE_ALAUDA=y
+CONFIG_USB_STORAGE_ONETOUCH=y
+CONFIG_USB_STORAGE_KARMA=y
+CONFIG_USB_STORAGE_CYPRESS_ATACB=y
+CONFIG_USB_STORAGE_ENE_UB6250=y
+CONFIG_USB_UAS=y
+CONFIG_USB_MDC800=m
+CONFIG_USB_MICROTEK=m
+CONFIG_USBIP_CORE=m
+CONFIG_USBIP_VHCI_HCD=m
+CONFIG_USBIP_HOST=m
+CONFIG_USBIP_VUDC=m
+CONFIG_USB_CDNS_SUPPORT=m
+CONFIG_USB_CDNS3=m
+CONFIG_USB_CDNS3_GADGET=y
+CONFIG_USB_CDNS3_HOST=y
+CONFIG_USB_MUSB_HDRC=m
+CONFIG_MUSB_PIO_ONLY=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_DWC3_HAPS=m
+# CONFIG_USB_DWC3_OF_SIMPLE is not set
+CONFIG_USB_DWC2=y
+CONFIG_USB_DWC2_PCI=m
+CONFIG_USB_CHIPIDEA=m
+CONFIG_USB_CHIPIDEA_UDC=y
+CONFIG_USB_CHIPIDEA_HOST=y
+CONFIG_USB_ISP1760=m
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_SIMPLE=m
+CONFIG_USB_SERIAL_CP210X=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_SERIAL_PL2303=m
+CONFIG_USB_SERIAL_OTI6858=m
+CONFIG_USB_SERIAL_QUALCOMM=m
+CONFIG_USB_SERIAL_OPTION=m
+CONFIG_USB_SERIAL_DEBUG=m
+CONFIG_USB_USS720=m
+CONFIG_USB_EMI62=m
+CONFIG_USB_EMI26=m
+CONFIG_USB_ADUTUX=m
+CONFIG_USB_SEVSEG=m
+CONFIG_USB_LEGOTOWER=m
+CONFIG_USB_LCD=m
+CONFIG_USB_CYPRESS_CY7C63=m
+CONFIG_USB_CYTHERM=m
+CONFIG_USB_IDMOUSE=m
+CONFIG_USB_APPLEDISPLAY=m
+CONFIG_APPLE_MFI_FASTCHARGE=m
+CONFIG_USB_SISUSBVGA=m
+CONFIG_USB_LD=m
+CONFIG_USB_TRANCEVIBRATOR=m
+CONFIG_USB_IOWARRIOR=m
+CONFIG_USB_TEST=m
+CONFIG_USB_EHSET_TEST_FIXTURE=m
+CONFIG_USB_ISIGHTFW=m
+CONFIG_USB_YUREX=m
+CONFIG_USB_HUB_USB251XB=m
+CONFIG_USB_HSIC_USB3503=m
+CONFIG_USB_HSIC_USB4604=m
+CONFIG_USB_LINK_LAYER_TEST=m
+CONFIG_USB_CHAOSKEY=m
+CONFIG_USB_ATM=m
+CONFIG_USB_SPEEDTOUCH=m
+CONFIG_USB_CXACRU=m
+CONFIG_USB_UEAGLEATM=m
+CONFIG_USB_XUSBATM=m
+CONFIG_USB_GPIO_VBUS=m
+CONFIG_USB_ISP1301=m
+CONFIG_USB_GADGET=m
+CONFIG_U_SERIAL_CONSOLE=y
+CONFIG_USB_GR_UDC=m
+CONFIG_USB_R8A66597=m
+CONFIG_USB_PXA27X=m
+CONFIG_USB_MV_UDC=m
+CONFIG_USB_MV_U3D=m
+CONFIG_USB_SNP_UDC_PLAT=m
+CONFIG_USB_BDC_UDC=m
+CONFIG_USB_AMD5536UDC=m
+CONFIG_USB_NET2272=m
+CONFIG_USB_NET2272_DMA=y
+CONFIG_USB_NET2280=m
+CONFIG_USB_GOKU=m
+CONFIG_USB_EG20T=m
+CONFIG_USB_GADGET_XILINX=m
+CONFIG_USB_MAX3420_UDC=m
+CONFIG_USB_CONFIGFS=m
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_ACM=y
+CONFIG_USB_CONFIGFS_OBEX=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_ECM=y
+CONFIG_USB_CONFIGFS_ECM_SUBSET=y
+CONFIG_USB_CONFIGFS_RNDIS=y
+CONFIG_USB_CONFIGFS_EEM=y
+CONFIG_USB_CONFIGFS_PHONET=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_UAC1=y
+CONFIG_USB_CONFIGFS_F_UAC1_LEGACY=y
+CONFIG_USB_CONFIGFS_F_UAC2=y
+CONFIG_USB_CONFIGFS_F_MIDI=y
+CONFIG_USB_CONFIGFS_F_MIDI2=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_USB_CONFIGFS_F_PRINTER=y
+CONFIG_USB_CONFIGFS_F_TCM=y
+CONFIG_USB_ZERO=m
+CONFIG_USB_AUDIO=m
+CONFIG_GADGET_UAC1=y
+CONFIG_USB_ETH=m
+CONFIG_USB_ETH_EEM=y
+CONFIG_USB_G_NCM=m
+CONFIG_USB_GADGETFS=m
+CONFIG_USB_FUNCTIONFS=m
+CONFIG_USB_FUNCTIONFS_ETH=y
+CONFIG_USB_FUNCTIONFS_RNDIS=y
+CONFIG_USB_FUNCTIONFS_GENERIC=y
+CONFIG_USB_MASS_STORAGE=m
+CONFIG_USB_GADGET_TARGET=m
+CONFIG_USB_G_SERIAL=m
+CONFIG_USB_MIDI_GADGET=m
+CONFIG_USB_G_PRINTER=m
+CONFIG_USB_CDC_COMPOSITE=m
+CONFIG_USB_G_NOKIA=m
+CONFIG_USB_G_ACM_MS=m
+CONFIG_USB_G_HID=m
+CONFIG_USB_G_DBGP=m
+CONFIG_USB_G_WEBCAM=m
+CONFIG_USB_RAW_GADGET=m
+CONFIG_TYPEC=m
+CONFIG_TYPEC_TCPM=m
+CONFIG_TYPEC_TCPCI=m
+CONFIG_TYPEC_FUSB302=m
+CONFIG_TYPEC_FUSB303B=m
+CONFIG_MMC=y
+CONFIG_PWRSEQ_EMMC=m
+CONFIG_PWRSEQ_SD8787=m
+CONFIG_PWRSEQ_SIMPLE=m
+CONFIG_SDIO_UART=m
+CONFIG_MMC_CRYPTO=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PCI=m
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ESWIN=y
+CONFIG_MMC_SDHCI_OF_SDIO_ESWIN=y
+CONFIG_MMC_SDHCI_OF_DWCMSHC=m
+CONFIG_MMC_DW=m
+CONFIG_MMC_DW_PCI=m
+CONFIG_MMC_USHC=m
+CONFIG_MMC_HSQ=m
+CONFIG_MEMSTICK=m
+CONFIG_MSPRO_BLOCK=m
+CONFIG_MS_BLOCK=m
+CONFIG_MEMSTICK_TIFM_MS=m
+CONFIG_MEMSTICK_JMICRON_38X=m
+CONFIG_MEMSTICK_R592=m
+CONFIG_MEMSTICK_REALTEK_PCI=m
+CONFIG_MEMSTICK_REALTEK_USB=m
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_CLASS_FLASH=m
+CONFIG_LEDS_CLASS_MULTICOLOR=m
+CONFIG_LEDS_BRIGHTNESS_HW_CHANGED=y
+CONFIG_LEDS_GPIO=m
+CONFIG_LEDS_PWM=m
+CONFIG_LEDS_REGULATOR=m
+CONFIG_LEDS_PWM_MULTICOLOR=m
+CONFIG_LEDS_TRIGGER_TIMER=m
+CONFIG_LEDS_TRIGGER_ONESHOT=m
+CONFIG_LEDS_TRIGGER_DISK=y
+CONFIG_LEDS_TRIGGER_MTD=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=m
+CONFIG_LEDS_TRIGGER_BACKLIGHT=m
+CONFIG_LEDS_TRIGGER_CPU=y
+CONFIG_LEDS_TRIGGER_ACTIVITY=m
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=m
+CONFIG_LEDS_TRIGGER_TRANSIENT=m
+CONFIG_LEDS_TRIGGER_CAMERA=m
+CONFIG_LEDS_TRIGGER_PANIC=y
+CONFIG_LEDS_TRIGGER_NETDEV=m
+CONFIG_LEDS_TRIGGER_PATTERN=m
+CONFIG_LEDS_TRIGGER_TTY=m
+CONFIG_EDAC=y
+# CONFIG_EDAC_LEGACY_SYSFS is not set
+CONFIG_EDAC_SIFIVE=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_ESWIN=y
+CONFIG_DMADEVICES=y
+CONFIG_DW_AXI_DMAC=m
+CONFIG_SW_SYNC=y
+CONFIG_UDMABUF=y
+CONFIG_DMABUF_MOVE_NOTIFY=y
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_UIO=m
+CONFIG_UIO_CIF=m
+CONFIG_UIO_PDRV_GENIRQ=m
+CONFIG_UIO_DMEM_GENIRQ=m
+CONFIG_UIO_AEC=m
+CONFIG_UIO_SERCOS3=m
+CONFIG_UIO_PCI_GENERIC=m
+CONFIG_UIO_NETX=m
+CONFIG_UIO_PRUSS=m
+CONFIG_UIO_MF624=m
+CONFIG_VIRT_DRIVERS=y
+CONFIG_VIRTIO_PCI=y
+CONFIG_VIRTIO_VDPA=m
+CONFIG_VIRTIO_PMEM=m
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=m
+CONFIG_VIRTIO_MMIO=y
+CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
+CONFIG_VDPA=m
+CONFIG_VDPA_SIM=m
+CONFIG_VDPA_SIM_NET=m
+CONFIG_VDPA_SIM_BLOCK=m
+CONFIG_VDPA_USER=m
+CONFIG_IFCVF=m
+CONFIG_MLX5_VDPA_STEERING_DEBUG=y
+CONFIG_VP_VDPA=m
+CONFIG_SNET_VDPA=m
+CONFIG_PDS_VDPA=m
+CONFIG_VHOST_NET=m
+CONFIG_VHOST_SCSI=m
+CONFIG_VHOST_VSOCK=m
+CONFIG_VHOST_VDPA=m
+CONFIG_STAGING=y
+CONFIG_PRISM2_USB=m
+CONFIG_RTL8192U=m
+CONFIG_RTLLIB=m
+CONFIG_RTL8192E=m
+CONFIG_RTL8723BS=m
+CONFIG_R8712U=m
+CONFIG_RTS5208=m
+CONFIG_VT6655=m
+CONFIG_VT6656=m
+CONFIG_FB_SM750=m
+CONFIG_STAGING_MEDIA=y
+CONFIG_VIDEO_MAX96712=m
+CONFIG_LTE_GDM724X=m
+CONFIG_KS7010=m
+CONFIG_QLGE=m
+CONFIG_GOLDFISH_PIPE=m
+CONFIG_LMK04832=m
+CONFIG_COMMON_CLK_MAX9485=m
+CONFIG_COMMON_CLK_SI5341=m
+CONFIG_COMMON_CLK_SI5351=m
+CONFIG_COMMON_CLK_SI514=m
+CONFIG_COMMON_CLK_SI544=m
+CONFIG_COMMON_CLK_SI570=m
+CONFIG_COMMON_CLK_CDCE706=m
+CONFIG_COMMON_CLK_CDCE925=m
+CONFIG_COMMON_CLK_CS2000_CP=m
+CONFIG_COMMON_CLK_AXI_CLKGEN=m
+CONFIG_COMMON_CLK_PWM=m
+CONFIG_COMMON_CLK_RS9_PCIE=m
+CONFIG_COMMON_CLK_SI521XX=m
+CONFIG_COMMON_CLK_VC3=m
+CONFIG_COMMON_CLK_VC5=m
+CONFIG_COMMON_CLK_VC7=m
+CONFIG_COMMON_CLK_FIXED_MMIO=y
+CONFIG_COMMON_CLK_EIC7700=y
+CONFIG_XILINX_VCU=m
+CONFIG_HWSPINLOCK=y
+CONFIG_MAILBOX=y
+CONFIG_PLATFORM_MHU=m
+CONFIG_MAILBOX_TEST=m
+CONFIG_IOMMU_ESWIN=y
+CONFIG_IOMMUFD=m
+CONFIG_ARM_SMMU_V3=y
+CONFIG_RPMSG_CHAR=m
+CONFIG_RPMSG_CTRL=m
+CONFIG_RPMSG_VIRTIO=m
+CONFIG_SOUNDWIRE=m
+CONFIG_SOUNDWIRE_QCOM=m
+CONFIG_LITEX_SOC_CONTROLLER=m
+CONFIG_WPCM450_SOC=m
+CONFIG_QCOM_PMIC_GLINK=m
+CONFIG_SIFIVE_CCACHE=y
+CONFIG_SOC_TI=y
+CONFIG_PM_DEVFREQ=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_POWERSAVE=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PASSIVE=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_EXTCON_ADC_JACK=m
+CONFIG_EXTCON_FSA9480=m
+CONFIG_EXTCON_GPIO=m
+CONFIG_EXTCON_MAX3355=m
+CONFIG_EXTCON_PTN5150=m
+CONFIG_EXTCON_RT8973A=m
+CONFIG_EXTCON_SM5502=m
+CONFIG_EXTCON_USB_GPIO=m
+CONFIG_EXTCON_USBC_TUSB320=m
+CONFIG_MEMORY=y
+CONFIG_IIO=y
+CONFIG_PWM=y
+CONFIG_PWM_CLK=m
+CONFIG_PWM_ESWIN=y
+CONFIG_RESET_ESWIN_EIC7700=y
+CONFIG_POWERCAP=y
+CONFIG_IDLE_INJECT=y
+CONFIG_DTPM=y
+CONFIG_DTPM_CPU=y
+CONFIG_LIBNVDIMM=y
+CONFIG_BLK_DEV_PMEM=m
+CONFIG_OF_PMEM=m
+CONFIG_DAX=y
+CONFIG_DEV_DAX=m
+CONFIG_NVMEM_LAYOUT_SL28_VPD=m
+CONFIG_NVMEM_LAYOUT_ONIE_TLV=m
+CONFIG_NVMEM_RMEM=m
+CONFIG_NVMEM_SPMI_SDAM=m
+CONFIG_NVMEM_U_BOOT_ENV=m
+# CONFIG_SLIMBUS is not set
+CONFIG_INTERCONNECT=y
+CONFIG_VALIDATE_FS_PARSER=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_REISERFS_FS=m
+CONFIG_REISERFS_FS_XATTR=y
+CONFIG_REISERFS_FS_POSIX_ACL=y
+CONFIG_REISERFS_FS_SECURITY=y
+CONFIG_JFS_FS=m
+CONFIG_JFS_POSIX_ACL=y
+CONFIG_JFS_SECURITY=y
+CONFIG_JFS_STATISTICS=y
+CONFIG_XFS_FS=m
+CONFIG_XFS_QUOTA=y
+CONFIG_XFS_POSIX_ACL=y
+CONFIG_XFS_RT=y
+CONFIG_GFS2_FS=m
+CONFIG_GFS2_FS_LOCKING_DLM=y
+CONFIG_OCFS2_FS=m
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_NILFS2_FS=m
+CONFIG_F2FS_FS=m
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_F2FS_FS_COMPRESSION=y
+# CONFIG_F2FS_IOSTAT is not set
+CONFIG_F2FS_UNFAIR_RWSEM=y
+CONFIG_ZONEFS_FS=m
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y
+CONFIG_FS_VERITY=y
+CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y
+CONFIG_FANOTIFY=y
+CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y
+CONFIG_QUOTA_NETLINK_INTERFACE=y
+CONFIG_QFMT_V1=m
+CONFIG_QFMT_V2=m
+CONFIG_AUTOFS_FS=y
+CONFIG_FUSE_FS=y
+CONFIG_CUSE=m
+CONFIG_VIRTIO_FS=m
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_FSCACHE=y
+CONFIG_FSCACHE_STATS=y
+CONFIG_CACHEFILES=m
+CONFIG_CACHEFILES_ERROR_INJECTION=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_UDF_FS=m
+CONFIG_MSDOS_FS=m
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS_FS=m
+CONFIG_NTFS3_FS=m
+CONFIG_NTFS3_LZX_XPRESS=y
+CONFIG_NTFS3_FS_POSIX_ACL=y
+CONFIG_PROC_KCORE=y
+CONFIG_PROC_VMCORE_DEVICE_DUMP=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_TMPFS_INODE64=y
+CONFIG_TMPFS_QUOTA=y
+CONFIG_HUGETLBFS=y
+CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON=y
+CONFIG_EFIVAR_FS=y
+CONFIG_ORANGEFS_FS=m
+CONFIG_ADFS_FS=m
+CONFIG_AFFS_FS=m
+CONFIG_ECRYPT_FS=y
+CONFIG_ECRYPT_FS_MESSAGING=y
+CONFIG_HFS_FS=m
+CONFIG_HFSPLUS_FS=m
+CONFIG_BEFS_FS=m
+CONFIG_BFS_FS=m
+CONFIG_EFS_FS=m
+CONFIG_JFFS2_FS=m
+CONFIG_JFFS2_FS_XATTR=y
+CONFIG_JFFS2_COMPRESSION_OPTIONS=y
+CONFIG_JFFS2_LZO=y
+CONFIG_JFFS2_CMODE_FAVOURLZO=y
+CONFIG_UBIFS_FS=m
+CONFIG_UBIFS_FS_AUTHENTICATION=y
+CONFIG_CRAMFS=m
+CONFIG_CRAMFS_MTD=y
+CONFIG_SQUASHFS=y
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_VXFS_FS=m
+CONFIG_MINIX_FS=m
+CONFIG_OMFS_FS=m
+CONFIG_HPFS_FS=m
+CONFIG_QNX4FS_FS=m
+CONFIG_QNX6FS_FS=m
+CONFIG_ROMFS_FS=m
+CONFIG_PSTORE=y
+CONFIG_PSTORE_RAM=m
+CONFIG_PSTORE_BLK=m
+CONFIG_SYSV_FS=m
+CONFIG_UFS_FS=m
+CONFIG_EROFS_FS=m
+CONFIG_EROFS_FS_ZIP_DEFLATE=y
+CONFIG_EROFS_FS_PCPU_KTHREAD=y
+# CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI is not set
+CONFIG_NFS_FS=m
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=m
+CONFIG_NFS_SWAP=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_NFS_V4_1_MIGRATION=y
+CONFIG_NFS_FSCACHE=y
+# CONFIG_NFS_V4_2_READ_PLUS is not set
+CONFIG_NFSD=m
+CONFIG_NFSD_V3_ACL=y
+CONFIG_NFSD_V4=y
+CONFIG_NFSD_BLOCKLAYOUT=y
+CONFIG_NFSD_SCSILAYOUT=y
+CONFIG_NFSD_FLEXFILELAYOUT=y
+CONFIG_NFSD_V4_2_INTER_SSC=y
+CONFIG_NFSD_V4_SECURITY_LABEL=y
+CONFIG_RPCSEC_GSS_KRB5_ENCTYPES_CAMELLIA=y
+CONFIG_RPCSEC_GSS_KRB5_ENCTYPES_AES_SHA2=y
+CONFIG_SUNRPC_DEBUG=y
+CONFIG_CEPH_FS=m
+CONFIG_CEPH_FSCACHE=y
+CONFIG_CEPH_FS_POSIX_ACL=y
+CONFIG_CEPH_FS_SECURITY_LABEL=y
+CONFIG_CIFS=m
+# CONFIG_CIFS_STATS2 is not set
+CONFIG_CIFS_UPCALL=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_CIFS_DFS_UPCALL=y
+CONFIG_CIFS_SWN_UPCALL=y
+CONFIG_CIFS_FSCACHE=y
+CONFIG_SMB_SERVER=m
+CONFIG_SMB_SERVER_KERBEROS5=y
+CONFIG_CODA_FS=m
+CONFIG_AFS_FS=m
+CONFIG_AFS_FSCACHE=y
+CONFIG_9P_FS=m
+CONFIG_9P_FSCACHE=y
+CONFIG_9P_FS_POSIX_ACL=y
+CONFIG_9P_FS_SECURITY=y
+CONFIG_NLS_DEFAULT="utf8"
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_737=m
+CONFIG_NLS_CODEPAGE_775=m
+CONFIG_NLS_CODEPAGE_850=m
+CONFIG_NLS_CODEPAGE_852=m
+CONFIG_NLS_CODEPAGE_855=m
+CONFIG_NLS_CODEPAGE_857=m
+CONFIG_NLS_CODEPAGE_860=m
+CONFIG_NLS_CODEPAGE_861=m
+CONFIG_NLS_CODEPAGE_862=m
+CONFIG_NLS_CODEPAGE_863=m
+CONFIG_NLS_CODEPAGE_864=m
+CONFIG_NLS_CODEPAGE_865=m
+CONFIG_NLS_CODEPAGE_866=m
+CONFIG_NLS_CODEPAGE_869=m
+CONFIG_NLS_CODEPAGE_936=m
+CONFIG_NLS_CODEPAGE_950=m
+CONFIG_NLS_CODEPAGE_932=m
+CONFIG_NLS_CODEPAGE_949=m
+CONFIG_NLS_CODEPAGE_874=m
+CONFIG_NLS_ISO8859_8=m
+CONFIG_NLS_CODEPAGE_1250=m
+CONFIG_NLS_CODEPAGE_1251=m
+CONFIG_NLS_ASCII=m
+CONFIG_NLS_ISO8859_1=m
+CONFIG_NLS_ISO8859_2=m
+CONFIG_NLS_ISO8859_3=m
+CONFIG_NLS_ISO8859_4=m
+CONFIG_NLS_ISO8859_5=m
+CONFIG_NLS_ISO8859_6=m
+CONFIG_NLS_ISO8859_7=m
+CONFIG_NLS_ISO8859_9=m
+CONFIG_NLS_ISO8859_13=m
+CONFIG_NLS_ISO8859_14=m
+CONFIG_NLS_ISO8859_15=m
+CONFIG_NLS_KOI8_R=m
+CONFIG_NLS_KOI8_U=m
+CONFIG_NLS_MAC_ROMAN=m
+CONFIG_NLS_MAC_CELTIC=m
+CONFIG_NLS_MAC_CENTEURO=m
+CONFIG_NLS_MAC_CROATIAN=m
+CONFIG_NLS_MAC_CYRILLIC=m
+CONFIG_NLS_MAC_GAELIC=m
+CONFIG_NLS_MAC_GREEK=m
+CONFIG_NLS_MAC_ICELAND=m
+CONFIG_NLS_MAC_INUIT=m
+CONFIG_NLS_MAC_ROMANIAN=m
+CONFIG_NLS_MAC_TURKISH=m
+CONFIG_DLM=m
+CONFIG_UNICODE=y
+CONFIG_KEYS_REQUEST_CACHE=y
+CONFIG_PERSISTENT_KEYRINGS=y
+CONFIG_TRUSTED_KEYS=y
+CONFIG_USER_DECRYPTED_DATA=y
+CONFIG_KEY_DH_OPERATIONS=y
+CONFIG_KEY_NOTIFICATIONS=y
+CONFIG_SECURITY_DMESG_RESTRICT=y
+CONFIG_SECURITY=y
+CONFIG_SECURITY_NETWORK_XFRM=y
+CONFIG_LSM_MMAP_MIN_ADDR=0
+CONFIG_HARDENED_USERCOPY=y
+CONFIG_SECURITY_SELINUX=y
+CONFIG_SECURITY_SELINUX_BOOTPARAM=y
+CONFIG_SECURITY_SMACK=y
+CONFIG_SECURITY_SMACK_NETFILTER=y
+CONFIG_SECURITY_SMACK_APPEND_SIGNALS=y
+CONFIG_SECURITY_TOMOYO=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_SECURITY_YAMA=y
+CONFIG_SECURITY_SAFESETID=y
+CONFIG_SECURITY_LOCKDOWN_LSM=y
+CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y
+CONFIG_SECURITY_LANDLOCK=y
+CONFIG_INTEGRITY_SIGNATURE=y
+CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y
+CONFIG_INTEGRITY_PLATFORM_KEYRING=y
+CONFIG_INTEGRITY_MACHINE_KEYRING=y
+CONFIG_IMA=y
+CONFIG_IMA_KEXEC=y
+CONFIG_IMA_DEFAULT_HASH_SHA256=y
+CONFIG_IMA_APPRAISE=y
+CONFIG_IMA_ARCH_POLICY=y
+CONFIG_IMA_APPRAISE_MODSIG=y
+CONFIG_EVM=y
+CONFIG_EVM_EXTRA_SMACK_XATTRS=y
+CONFIG_EVM_ADD_XATTRS=y
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor"
+CONFIG_ZERO_CALL_USED_REGS=y
+CONFIG_CRYPTO_USER=m
+CONFIG_CRYPTO_PCRYPT=m
+CONFIG_CRYPTO_CRYPTD=m
+CONFIG_CRYPTO_TEST=m
+CONFIG_CRYPTO_ECDSA=m
+CONFIG_CRYPTO_ECRDSA=m
+CONFIG_CRYPTO_SM2=m
+CONFIG_CRYPTO_CURVE25519=m
+CONFIG_CRYPTO_AES_TI=m
+CONFIG_CRYPTO_ARIA=m
+CONFIG_CRYPTO_BLOWFISH=m
+CONFIG_CRYPTO_CAMELLIA=m
+CONFIG_CRYPTO_CAST5=m
+CONFIG_CRYPTO_CAST6=m
+CONFIG_CRYPTO_DES=m
+CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4_GENERIC=m
+CONFIG_CRYPTO_TWOFISH=m
+CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_CFB=m
+CONFIG_CRYPTO_HCTR2=m
+CONFIG_CRYPTO_KEYWRAP=m
+CONFIG_CRYPTO_LRW=m
+CONFIG_CRYPTO_OFB=m
+CONFIG_CRYPTO_AEGIS128=m
+CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_GCM=y
+CONFIG_CRYPTO_SEQIV=y
+CONFIG_CRYPTO_MD4=m
+CONFIG_CRYPTO_RMD160=m
+CONFIG_CRYPTO_SM3_GENERIC=m
+CONFIG_CRYPTO_VMAC=m
+CONFIG_CRYPTO_WP512=m
+CONFIG_CRYPTO_XCBC=m
+CONFIG_CRYPTO_DEFLATE=y
+CONFIG_CRYPTO_842=m
+CONFIG_CRYPTO_LZ4=m
+CONFIG_CRYPTO_LZ4HC=m
+CONFIG_CRYPTO_ANSI_CPRNG=m
+CONFIG_CRYPTO_DRBG_HASH=y
+CONFIG_CRYPTO_DRBG_CTR=y
+CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE=y
+CONFIG_CRYPTO_USER_API_HASH=m
+CONFIG_CRYPTO_USER_API_SKCIPHER=m
+CONFIG_CRYPTO_USER_API_RNG=m
+CONFIG_CRYPTO_USER_API_AEAD=m
+# CONFIG_CRYPTO_USER_API_ENABLE_OBSOLETE is not set
+CONFIG_CRYPTO_STATS=y
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_PKCS8_PRIVATE_KEY_PARSER=m
+CONFIG_PKCS7_TEST_KEY=m
+CONFIG_SIGNED_PE_FILE_VERIFICATION=y
+CONFIG_SYSTEM_EXTRA_CERTIFICATE=y
+CONFIG_SECONDARY_TRUSTED_KEYRING=y
+CONFIG_SYSTEM_BLACKLIST_KEYRING=y
+CONFIG_SYSTEM_REVOCATION_LIST=y
+CONFIG_CRC_ITU_T=y
+CONFIG_CRC7=y
+CONFIG_XZ_DEC_MICROLZMA=y
+CONFIG_XZ_DEC_TEST=m
+CONFIG_DMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=32
+CONFIG_IRQ_POLL=y
+CONFIG_FONTS=y
+CONFIG_FONT_8x8=y
+CONFIG_FONT_ACORN_8x8=y
+CONFIG_FONT_6x10=y
+CONFIG_FONT_TER16x32=y
+CONFIG_PRINTK_TIME=y
+CONFIG_CONSOLE_LOGLEVEL_QUIET=3
+CONFIG_BOOT_PRINTK_DELAY=y
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_VMLINUX_MAP=y
+CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x01b6
+CONFIG_KGDB=y
+CONFIG_KGDB_KDB=y
+CONFIG_KDB_KEYBOARD=y
+CONFIG_PAGE_POISONING=y
+CONFIG_DEBUG_WX=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_KFENCE=y
+CONFIG_KFENCE_SAMPLE_INTERVAL=0
+CONFIG_HARDLOCKUP_DETECTOR=y
+CONFIG_WQ_CPU_INTENSIVE_REPORT=y
+CONFIG_RCU_CPU_STALL_TIMEOUT=60
+CONFIG_RCU_CPU_STALL_CPUTIME=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_LATENCYTOP=y
+CONFIG_BOOTTIME_TRACING=y
+CONFIG_SCHED_TRACER=y
+CONFIG_HWLAT_TRACER=y
+CONFIG_TIMERLAT_TRACER=y
+CONFIG_FTRACE_SYSCALLS=y
+CONFIG_BLK_DEV_IO_TRACE=y
+CONFIG_BPF_KPROBE_OVERRIDE=y
+CONFIG_SYNTH_EVENTS=y
+CONFIG_USER_EVENTS=y
+CONFIG_TRACE_EVENT_INJECT=y
+CONFIG_RV=y
+CONFIG_RV_MON_WWNR=y
+CONFIG_SAMPLES=y
+CONFIG_SAMPLE_TRACE_PRINTK=m
+CONFIG_SAMPLE_TRACE_ARRAY=m
+CONFIG_NOTIFIER_ERROR_INJECTION=m
+CONFIG_FUNCTION_ERROR_INJECTION=y
+CONFIG_TEST_BPF=m
+CONFIG_TEST_BLACKHOLE_DEV=m
+CONFIG_MEMTEST=y
+CONFIG_SIFIVE_PL2_PMU=y
+CONFIG_SIFIVE_CCACHE_PMU=y
diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h
index 721ec275ce57..b6027a8b6b50 100644
--- a/arch/riscv/include/asm/alternative-macros.h
+++ b/arch/riscv/include/asm/alternative-macros.h
@@ -50,8 +50,17 @@
 	ALT_NEW_CONTENT \vendor_id_2, \patch_id_2, \enable_2, "\new_c_2"
 .endm
 
+.macro ALTERNATIVE_CFG_3 old_c, new_c_1, vendor_id_1, patch_id_1, enable_1,	\
+				new_c_2, vendor_id_2, patch_id_2, enable_2,	\
+				new_c_3, vendor_id_3, patch_id_3, enable_3
+	ALTERNATIVE_CFG_2 "\old_c", "\new_c_1", \vendor_id_1, \patch_id_1, \enable_1 \
+				    "\new_c_2", \vendor_id_2, \patch_id_2, \enable_2 \
+	ALT_NEW_CONTENT \vendor_id_3, \patch_id_3, \enable_3, "\new_c_3"
+.endm
+
 #define __ALTERNATIVE_CFG(...)		ALTERNATIVE_CFG __VA_ARGS__
 #define __ALTERNATIVE_CFG_2(...)	ALTERNATIVE_CFG_2 __VA_ARGS__
+#define __ALTERNATIVE_CFG_3(...)	ALTERNATIVE_CFG_3 __VA_ARGS__
 
 #else /* !__ASSEMBLY__ */
 
@@ -98,6 +107,13 @@
 	__ALTERNATIVE_CFG(old_c, new_c_1, vendor_id_1, patch_id_1, enable_1)	\
 	ALT_NEW_CONTENT(vendor_id_2, patch_id_2, enable_2, new_c_2)
 
+#define __ALTERNATIVE_CFG_3(old_c, new_c_1, vendor_id_1, patch_id_1, enable_1,	\
+				   new_c_2, vendor_id_2, patch_id_2, enable_2,	\
+				   new_c_3, vendor_id_3, patch_id_3, enable_3)	\
+	__ALTERNATIVE_CFG_2(old_c, new_c_1, vendor_id_1, patch_id_1, enable_1,	\
+				   new_c_2, vendor_id_2, patch_id_2, enable_2)	\
+	ALT_NEW_CONTENT(vendor_id_3, patch_id_3, enable_3, new_c_3)
+
 #endif /* __ASSEMBLY__ */
 
 #define _ALTERNATIVE_CFG(old_c, new_c, vendor_id, patch_id, CONFIG_k)	\
@@ -108,6 +124,13 @@
 	__ALTERNATIVE_CFG_2(old_c, new_c_1, vendor_id_1, patch_id_1, IS_ENABLED(CONFIG_k_1),	\
 				   new_c_2, vendor_id_2, patch_id_2, IS_ENABLED(CONFIG_k_2))
 
+#define _ALTERNATIVE_CFG_3(old_c, new_c_1, vendor_id_1, patch_id_1, CONFIG_k_1,		\
+				  new_c_2, vendor_id_2, patch_id_2, CONFIG_k_2,		\
+				  new_c_3, vendor_id_3, patch_id_3, CONFIG_k_3)		\
+	__ALTERNATIVE_CFG_3(old_c, new_c_1, vendor_id_1, patch_id_1, IS_ENABLED(CONFIG_k_1),	\
+				   new_c_2, vendor_id_2, patch_id_2, IS_ENABLED(CONFIG_k_2),	\
+				   new_c_3, vendor_id_3, patch_id_3, IS_ENABLED(CONFIG_k_3))
+
 #else /* CONFIG_RISCV_ALTERNATIVE */
 #ifdef __ASSEMBLY__
 
@@ -121,6 +144,9 @@
 #define _ALTERNATIVE_CFG_2(old_c, ...)	\
 	ALTERNATIVE_CFG old_c
 
+#define _ALTERNATIVE_CFG_3(old_c, ...)	\
+	ALTERNATIVE_CFG old_c
+
 #else /* !__ASSEMBLY__ */
 
 #define __ALTERNATIVE_CFG(old_c)	\
@@ -132,6 +158,9 @@
 #define _ALTERNATIVE_CFG_2(old_c, ...)	\
 	__ALTERNATIVE_CFG(old_c)
 
+#define _ALTERNATIVE_CFG_3(old_c, ...)	\
+	__ALTERNATIVE_CFG(old_c)
+
 #endif /* __ASSEMBLY__ */
 #endif /* CONFIG_RISCV_ALTERNATIVE */
 
@@ -152,15 +181,21 @@
 	_ALTERNATIVE_CFG(old_content, new_content, vendor_id, patch_id, CONFIG_k)
 
 /*
- * A vendor wants to replace an old_content, but another vendor has used
- * ALTERNATIVE() to patch its customized content at the same location. In
- * this case, this vendor can create a new macro ALTERNATIVE_2() based
- * on the following sample code and then replace ALTERNATIVE() with
- * ALTERNATIVE_2() to append its customized content.
+ * Variant of ALTERNATIVE() that supports two sets of replacement content.
  */
 #define ALTERNATIVE_2(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1,		\
 				   new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2)		\
 	_ALTERNATIVE_CFG_2(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1,	\
 					new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2)
 
+/*
+ * Variant of ALTERNATIVE() that supports three sets of replacement content.
+ */
+#define ALTERNATIVE_3(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1,		\
+				   new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2,		\
+				   new_content_3, vendor_id_3, patch_id_3, CONFIG_k_3)		\
+	_ALTERNATIVE_CFG_3(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1,	\
+					new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2,	\
+					new_content_3, vendor_id_3, patch_id_3, CONFIG_k_3)
+
 #endif
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index 110752594228..ca07d68c70a2 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -71,6 +71,8 @@ do {									\
  */
 #define smp_mb__after_spinlock()	RISCV_FENCE(iorw,iorw)
 
+#define wfe()   do { } while (0)
+
 #include <asm-generic/barrier.h>
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h
index d3f3c237adad..015f44e13978 100644
--- a/arch/riscv/include/asm/errata_list.h
+++ b/arch/riscv/include/asm/errata_list.h
@@ -59,51 +59,6 @@ asm(ALTERNATIVE("sfence.vma %0, %1", "sfence.vma", SIFIVE_VENDOR_ID,	\
 		ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200)	\
 		: : "r" (addr), "r" (asid) : "memory")
 
-/*
- * _val is marked as "will be overwritten", so need to set it to 0
- * in the default case.
- */
-#define ALT_SVPBMT_SHIFT 61
-#define ALT_THEAD_PBMT_SHIFT 59
-#define ALT_SVPBMT(_val, prot)						\
-asm(ALTERNATIVE_2("li %0, 0\t\nnop",					\
-		  "li %0, %1\t\nslli %0,%0,%3", 0,			\
-			RISCV_ISA_EXT_SVPBMT, CONFIG_RISCV_ISA_SVPBMT,	\
-		  "li %0, %2\t\nslli %0,%0,%4", THEAD_VENDOR_ID,	\
-			ERRATA_THEAD_PBMT, CONFIG_ERRATA_THEAD_PBMT)	\
-		: "=r"(_val)						\
-		: "I"(prot##_SVPBMT >> ALT_SVPBMT_SHIFT),		\
-		  "I"(prot##_THEAD >> ALT_THEAD_PBMT_SHIFT),		\
-		  "I"(ALT_SVPBMT_SHIFT),				\
-		  "I"(ALT_THEAD_PBMT_SHIFT))
-
-#ifdef CONFIG_ERRATA_THEAD_PBMT
-/*
- * IO/NOCACHE memory types are handled together with svpbmt,
- * so on T-Head chips, check if no other memory type is set,
- * and set the non-0 PMA type if applicable.
- */
-#define ALT_THEAD_PMA(_val)						\
-asm volatile(ALTERNATIVE(						\
-	__nops(7),							\
-	"li      t3, %1\n\t"						\
-	"slli    t3, t3, %3\n\t"					\
-	"and     t3, %0, t3\n\t"					\
-	"bne     t3, zero, 2f\n\t"					\
-	"li      t3, %2\n\t"						\
-	"slli    t3, t3, %3\n\t"					\
-	"or      %0, %0, t3\n\t"					\
-	"2:",  THEAD_VENDOR_ID,						\
-		ERRATA_THEAD_PBMT, CONFIG_ERRATA_THEAD_PBMT)		\
-	: "+r"(_val)							\
-	: "I"(_PAGE_MTMASK_THEAD >> ALT_THEAD_PBMT_SHIFT),		\
-	  "I"(_PAGE_PMA_THEAD >> ALT_THEAD_PBMT_SHIFT),			\
-	  "I"(ALT_THEAD_PBMT_SHIFT)					\
-	: "t3")
-#else
-#define ALT_THEAD_PMA(_val)
-#endif
-
 /*
  * dcache.ipa rs1 (invalidate, physical address)
  * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
diff --git a/arch/riscv/include/asm/fpu.h b/arch/riscv/include/asm/fpu.h
new file mode 100644
index 000000000000..91c04c244e12
--- /dev/null
+++ b/arch/riscv/include/asm/fpu.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2023 SiFive
+ */
+
+#ifndef _ASM_RISCV_FPU_H
+#define _ASM_RISCV_FPU_H
+
+#include <asm/switch_to.h>
+
+#define kernel_fpu_available()	has_fpu()
+
+void kernel_fpu_begin(void);
+void kernel_fpu_end(void);
+
+#endif /* ! _ASM_RISCV_FPU_H */
diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index f4157034efa9..64d7511fb966 100644
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -59,6 +59,8 @@
 #define RISCV_ISA_EXT_ZIFENCEI		41
 #define RISCV_ISA_EXT_ZIHPM		42
 
+#define RISCV_ISA_EXT_XLINUXMEMALIAS	63
+
 #define RISCV_ISA_EXT_MAX		64
 
 #ifdef CONFIG_RISCV_M_MODE
diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
index 42497d487a17..d1f0453c6580 100644
--- a/arch/riscv/include/asm/io.h
+++ b/arch/riscv/include/asm/io.h
@@ -51,6 +51,7 @@
 #define __io_par(v)	__asm__ __volatile__ ("fence i,ior" : : : "memory");
 #define __io_pbw()	__asm__ __volatile__ ("fence iow,o" : : : "memory");
 #define __io_paw()	__asm__ __volatile__ ("fence o,io"  : : : "memory");
+#define __iomb()    mb()
 
 /*
  * Accesses from a single hart to a single I/O address must be ordered.  This
diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h
index 59ba1fbaf784..aeaab299e1a2 100644
--- a/arch/riscv/include/asm/pgtable-32.h
+++ b/arch/riscv/include/asm/pgtable-32.h
@@ -17,20 +17,11 @@
 
 #define MAX_POSSIBLE_PHYSMEM_BITS 34
 
-/*
- * rv32 PTE format:
- * | XLEN-1  10 | 9             8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
- *       PFN      reserved for SW   D   A   G   U   X   W   R   V
- */
-#define _PAGE_PFN_MASK  GENMASK(31, 10)
-
-#define _PAGE_NOCACHE		0
-#define _PAGE_IO		0
-#define _PAGE_MTMASK		0
+#define ALT_FIXUP_MT(_val)
+#define ALT_UNFIX_MT(_val)
 
-/* Set of bits to preserve across pte_modify() */
-#define _PAGE_CHG_MASK  (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ |	\
-					  _PAGE_WRITE | _PAGE_EXEC |	\
-					  _PAGE_USER | _PAGE_GLOBAL))
+#define pud_pfn(pud)				(pmd_pfn((pmd_t){ pud }))
+#define p4d_pfn(p4d)				(pud_pfn((pud_t){ p4d }))
+#define pgd_pfn(pgd)				(p4d_pfn((p4d_t){ pgd }))
 
 #endif /* _ASM_RISCV_PGTABLE_32_H */
diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h
index 3272ca7a5270..e9b63086ac46 100644
--- a/arch/riscv/include/asm/pgtable-64.h
+++ b/arch/riscv/include/asm/pgtable-64.h
@@ -8,7 +8,7 @@
 
 #include <linux/bits.h>
 #include <linux/const.h>
-#include <asm/errata_list.h>
+#include <asm/alternative-macros.h>
 
 extern bool pgtable_l4_enabled;
 extern bool pgtable_l5_enabled;
@@ -68,23 +68,8 @@ typedef struct {
 
 #define pmd_val(x)      ((x).pmd)
 #define __pmd(x)        ((pmd_t) { (x) })
-
 #define PTRS_PER_PMD    (PAGE_SIZE / sizeof(pmd_t))
 
-/*
- * rv64 PTE format:
- * | 63 | 62 61 | 60 54 | 53  10 | 9             8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
- *   N      MT     RSV    PFN      reserved for SW   D   A   G   U   X   W   R   V
- */
-#define _PAGE_PFN_MASK  GENMASK(53, 10)
-
-/*
- * [63] Svnapot definitions:
- * 0 Svnapot disabled
- * 1 Svnapot enabled
- */
-#define _PAGE_NAPOT_SHIFT	63
-#define _PAGE_NAPOT		BIT(_PAGE_NAPOT_SHIFT)
 /*
  * Only 64KB (order 4) napot ptes supported.
  */
@@ -112,63 +97,90 @@ enum napot_cont_order {
 #define HUGE_MAX_HSTATE		2
 #endif
 
-/*
- * [62:61] Svpbmt Memory Type definitions:
- *
- *  00 - PMA    Normal Cacheable, No change to implied PMA memory type
- *  01 - NC     Non-cacheable, idempotent, weakly-ordered Main Memory
- *  10 - IO     Non-cacheable, non-idempotent, strongly-ordered I/O memory
- *  11 - Rsvd   Reserved for future standard use
- */
-#define _PAGE_NOCACHE_SVPBMT	(1UL << 61)
-#define _PAGE_IO_SVPBMT		(1UL << 62)
-#define _PAGE_MTMASK_SVPBMT	(_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT)
+#ifdef CONFIG_RISCV_ISA_SVPBMT
 
 /*
  * [63:59] T-Head Memory Type definitions:
- *
- * 00000 - NC   Weakly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable
+ * bit[63] SO - Strong Order
+ * bit[62] C - Cacheable
+ * bit[61] B - Bufferable
+ * bit[60] SH - Shareable
+ * bit[59] Sec - Trustable
  * 01110 - PMA  Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable
- * 10000 - IO   Strongly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable
+ * 00110 - NC   Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable
+ * 10010 - IO   Strongly-ordered, Non-cacheable, Non-bufferable, Shareable, Non-trustable
+ *
+ * ALT_FIXUP_MT translates Svpbmt memory types to XTheadMae memory types.
+ * Pseudocode operating on bits [63:60]:
+ *   t0 = mt << 1
+ *   if (t0 == 0)
+ *     t0 |= 2
+ *   t0 ^= 0x5
+ *   mt ^= t0
+ *
+ * ALT_UNFIX_MT translates XTheadMae memory types to Svpbmt memory types.
+ * Pseudocode operating on bits [63:60]:
+ *   t0 = mt & 0xd
+ *   t0 ^= t0 >> 1
+ *   mt ^= t0
  */
-#define _PAGE_PMA_THEAD		((1UL << 62) | (1UL << 61) | (1UL << 60))
-#define _PAGE_NOCACHE_THEAD	0UL
-#define _PAGE_IO_THEAD		(1UL << 63)
-#define _PAGE_MTMASK_THEAD	(_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59))
-
-static inline u64 riscv_page_mtmask(void)
-{
-	u64 val;
-
-	ALT_SVPBMT(val, _PAGE_MTMASK);
-	return val;
-}
 
-static inline u64 riscv_page_nocache(void)
-{
-	u64 val;
+#define ALT_FIXUP_MT(_val)								\
+	asm(ALTERNATIVE_3("addi	t0, zero, 0x3\n\t"					\
+			  "slli	t0, t0, 61\n\t"						\
+			  "not	t0, t0\n\t"						\
+			  "and	%0, %0, t0\n\t"						\
+			  "nop\n\t"							\
+			  "nop\n\t"							\
+			  "nop\n\t"							\
+			  "nop",							\
+			  __nops(8),							\
+			  0, RISCV_ISA_EXT_SVPBMT, CONFIG_RISCV_ISA_SVPBMT,		\
+			  "addi t0, zero, 0x3\n\t"					\
+			  "slli	t0, t0, 61\n\t"						\
+			  "and	t0, %0, t0\n\t"						\
+			  "beqz t0, 2f\n\t"						\
+			  "xor	t1, %0, t0\n\t"						\
+			  "1: auipc t0, %%pcrel_hi(riscv_fixup_memory_alias)\n\t"	\
+			  "jalr	t0, t0, %%pcrel_lo(1b)\n\t"				\
+			  "mv	%0, t1\n"						\
+			  "2:",								\
+			  0, RISCV_ISA_EXT_XLINUXMEMALIAS, CONFIG_RISCV_ISA_SVPBMT,	\
+			  "srli	t0, %0, 59\n\t"						\
+			  "seqz t1, t0\n\t"						\
+			  "slli	t1, t1, 1\n\t"						\
+			  "or	t0, t0, t1\n\t"						\
+			  "xori	t0, t0, 0x5\n\t"					\
+			  "slli	t0, t0, 60\n\t"						\
+			  "xor	%0, %0, t0\n\t"						\
+			  "nop",							\
+			  THEAD_VENDOR_ID, ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE)	\
+			  : "+r" (_val) :: "t0", "t1")
+
+#define ALT_UNFIX_MT(_val)								\
+	asm(ALTERNATIVE_2(__nops(6),							\
+			  "mv	t1, %0\n\t"						\
+			  "1: auipc t0, %%pcrel_hi(riscv_unfix_memory_alias)\n\t"	\
+			  "jalr	t0, t0, %%pcrel_lo(1b)\n\t"				\
+			  "mv	%0, t1\n\t"						\
+			  "nop\n\t"							\
+			  "nop",							\
+			  0, RISCV_ISA_EXT_XLINUXMEMALIAS, CONFIG_RISCV_ISA_SVPBMT,	\
+			  "srli	t0, %0, 60\n\t"						\
+			  "andi	t0, t0, 0xd\n\t"					\
+			  "srli	t1, t0, 1\n\t"						\
+			  "xor	t0, t0, t1\n\t"						\
+			  "slli	t0, t0, 60\n\t"						\
+			  "xor	%0, %0, t0",						\
+			  THEAD_VENDOR_ID, ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE)	\
+			  : "+r" (_val) :: "t0", "t1")
 
-	ALT_SVPBMT(val, _PAGE_NOCACHE);
-	return val;
-}
-
-static inline u64 riscv_page_io(void)
-{
-	u64 val;
-
-	ALT_SVPBMT(val, _PAGE_IO);
-	return val;
-}
+#else
 
-#define _PAGE_NOCACHE		riscv_page_nocache()
-#define _PAGE_IO		riscv_page_io()
-#define _PAGE_MTMASK		riscv_page_mtmask()
+#define ALT_FIXUP_MT(_val)
+#define ALT_UNFIX_MT(_val)
 
-/* Set of bits to preserve across pte_modify() */
-#define _PAGE_CHG_MASK  (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ |	\
-					  _PAGE_WRITE | _PAGE_EXEC |	\
-					  _PAGE_USER | _PAGE_GLOBAL |	\
-					  _PAGE_MTMASK))
+#endif /* CONFIG_RISCV_ISA_SVPBMT */
 
 static inline int pud_present(pud_t pud)
 {
@@ -208,22 +220,27 @@ static inline void pud_clear(pud_t *pudp)
 
 static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot)
 {
-	return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+	pud_t pud = __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+
+	ALT_FIXUP_MT(pud);
+
+	return pud;
 }
 
-static inline unsigned long _pud_pfn(pud_t pud)
+#define pud_pfn pud_pfn
+static inline unsigned long pud_pfn(pud_t pud)
 {
 	return __page_val_to_pfn(pud_val(pud));
 }
 
 static inline pmd_t *pud_pgtable(pud_t pud)
 {
-	return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud)));
+	return (pmd_t *)pfn_to_virt(pud_pfn(pud));
 }
 
 static inline struct page *pud_page(pud_t pud)
 {
-	return pfn_to_page(__page_val_to_pfn(pud_val(pud)));
+	return pfn_to_page(pud_pfn(pud));
 }
 
 #define mm_p4d_folded  mm_p4d_folded
@@ -248,16 +265,11 @@ static inline bool mm_pud_folded(struct mm_struct *mm)
 
 static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot)
 {
-	unsigned long prot_val = pgprot_val(prot);
+	pmd_t pmd = __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
 
-	ALT_THEAD_PMA(prot_val);
+	ALT_FIXUP_MT(pmd);
 
-	return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val);
-}
-
-static inline unsigned long _pmd_pfn(pmd_t pmd)
-{
-	return __page_val_to_pfn(pmd_val(pmd));
+	return pmd;
 }
 
 #define mk_pmd(page, prot)    pfn_pmd(page_to_pfn(page), prot)
@@ -311,10 +323,14 @@ static inline void p4d_clear(p4d_t *p4d)
 
 static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot)
 {
-	return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+	p4d_t p4d = __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+
+	ALT_FIXUP_MT(p4d);
+
+	return p4d;
 }
 
-static inline unsigned long _p4d_pfn(p4d_t p4d)
+static inline unsigned long p4d_pfn(p4d_t p4d)
 {
 	return __page_val_to_pfn(p4d_val(p4d));
 }
@@ -322,7 +338,7 @@ static inline unsigned long _p4d_pfn(p4d_t p4d)
 static inline pud_t *p4d_pgtable(p4d_t p4d)
 {
 	if (pgtable_l4_enabled)
-		return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d)));
+		return (pud_t *)pfn_to_virt(p4d_pfn(p4d));
 
 	return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) });
 }
@@ -330,7 +346,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d)
 
 static inline struct page *p4d_page(p4d_t p4d)
 {
-	return pfn_to_page(__page_val_to_pfn(p4d_val(p4d)));
+	return pfn_to_page(p4d_pfn(p4d));
 }
 
 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
@@ -376,10 +392,15 @@ static inline void pgd_clear(pgd_t *pgd)
 		set_pgd(pgd, __pgd(0));
 }
 
+static inline unsigned long pgd_pfn(pgd_t pgd)
+{
+	return __page_val_to_pfn(pgd_val(pgd));
+}
+
 static inline p4d_t *pgd_pgtable(pgd_t pgd)
 {
 	if (pgtable_l5_enabled)
-		return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd)));
+		return (p4d_t *)pfn_to_virt(pgd_pfn(pgd));
 
 	return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) });
 }
@@ -387,9 +408,8 @@ static inline p4d_t *pgd_pgtable(pgd_t pgd)
 
 static inline struct page *pgd_page(pgd_t pgd)
 {
-	return pfn_to_page(__page_val_to_pfn(pgd_val(pgd)));
+	return pfn_to_page(pgd_pfn(pgd));
 }
-#define pgd_page(pgd)	pgd_page(pgd)
 
 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
 
diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h
index f896708e8331..3c014c76ae36 100644
--- a/arch/riscv/include/asm/pgtable-bits.h
+++ b/arch/riscv/include/asm/pgtable-bits.h
@@ -6,6 +6,16 @@
 #ifndef _ASM_RISCV_PGTABLE_BITS_H
 #define _ASM_RISCV_PGTABLE_BITS_H
 
+/*
+ * rv32 PTE format:
+ * | XLEN-1  10 | 9             8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
+ *       PFN      reserved for SW   D   A   G   U   X   W   R   V
+ *
+ * rv64 PTE format:
+ * | 63 | 62 61 | 60 54 | 53 10 | 9             8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
+ *   N      MT     RSV     PFN    reserved for SW   D   A   G   U   X   W   R   V
+ */
+
 #define _PAGE_ACCESSED_OFFSET 6
 
 #define _PAGE_PRESENT   (1 << 0)
@@ -16,11 +26,41 @@
 #define _PAGE_GLOBAL    (1 << 5)    /* Global */
 #define _PAGE_ACCESSED  (1 << 6)    /* Set by hardware on any access */
 #define _PAGE_DIRTY     (1 << 7)    /* Set by hardware on any write */
-#define _PAGE_SOFT      (1 << 8)    /* Reserved for software */
+#define _PAGE_SOFT      (3 << 8)    /* Reserved for software */
 
-#define _PAGE_SPECIAL   _PAGE_SOFT
+#define _PAGE_SPECIAL   (1 << 8)    /* RSW: 0x1 */
 #define _PAGE_TABLE     _PAGE_PRESENT
 
+#define _PAGE_PFN_SHIFT		10
+#ifdef CONFIG_64BIT
+#define _PAGE_PFN_MASK		GENMASK(53, 10)
+#else
+#define _PAGE_PFN_MASK		GENMASK(31, 10)
+#endif /* CONFIG_64BIT */
+
+#ifdef CONFIG_RISCV_ISA_SVPBMT
+/*
+ * [62:61] Svpbmt Memory Type definitions:
+ *
+ *  00 - PMA    Normal Cacheable, No change to implied PMA memory type
+ *  01 - NC     Non-cacheable, idempotent, weakly-ordered Main Memory
+ *  10 - IO     Non-cacheable, non-idempotent, strongly-ordered I/O memory
+ *  11 - Rsvd   Reserved for future standard use
+ */
+#define _PAGE_NOCACHE		(UL(1) << 61)
+#define _PAGE_IO		(UL(2) << 61)
+#define _PAGE_MTMASK		(UL(3) << 61)
+#else
+#define _PAGE_NOCACHE		0
+#define _PAGE_IO		0
+#define _PAGE_MTMASK		0
+#endif /* CONFIG_RISCV_ISA_SVPBMT */
+
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+#define _PAGE_NAPOT_SHIFT	63
+#define _PAGE_NAPOT		BIT(_PAGE_NAPOT_SHIFT)
+#endif /* CONFIG_RISCV_ISA_SVNAPOT */
+
 /*
  * _PAGE_PROT_NONE is set on not-present pages (and ignored by the hardware) to
  * distinguish them from swapped out pages
@@ -30,8 +70,6 @@
 /* Used for swap PTEs only. */
 #define _PAGE_SWP_EXCLUSIVE _PAGE_ACCESSED
 
-#define _PAGE_PFN_SHIFT 10
-
 /*
  * when all of R/W/X are zero, the PTE is a pointer to the next level
  * of the page table; otherwise, it is a leaf PTE.
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index f540b2625714..71e7fc6c8c14 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -208,6 +208,12 @@ extern struct pt_alloc_ops pt_ops __initdata;
 #define _PAGE_IOREMAP	((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_IO)
 #define PAGE_KERNEL_IO		__pgprot(_PAGE_IOREMAP)
 
+/* Set of bits to preserve across pte_modify() */
+#define _PAGE_CHG_MASK  (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ |	\
+					  _PAGE_WRITE | _PAGE_EXEC |	\
+					  _PAGE_USER | _PAGE_GLOBAL |	\
+					  _PAGE_MTMASK))
+
 extern pgd_t swapper_pg_dir[];
 extern pgd_t trampoline_pg_dir[];
 extern pgd_t early_pg_dir[];
@@ -258,26 +264,26 @@ static inline void pmd_clear(pmd_t *pmdp)
 
 static inline pgd_t pfn_pgd(unsigned long pfn, pgprot_t prot)
 {
-	unsigned long prot_val = pgprot_val(prot);
+	pgd_t pgd = __pgd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
 
-	ALT_THEAD_PMA(prot_val);
+	ALT_FIXUP_MT(pgd);
 
-	return __pgd((pfn << _PAGE_PFN_SHIFT) | prot_val);
+	return pgd;
 }
 
-static inline unsigned long _pgd_pfn(pgd_t pgd)
+static inline unsigned long pmd_pfn(pmd_t pmd)
 {
-	return __page_val_to_pfn(pgd_val(pgd));
+	return __page_val_to_pfn(pmd_val(pmd));
 }
 
 static inline struct page *pmd_page(pmd_t pmd)
 {
-	return pfn_to_page(__page_val_to_pfn(pmd_val(pmd)));
+	return pfn_to_page(pmd_pfn(pmd));
 }
 
 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 {
-	return (unsigned long)pfn_to_virt(__page_val_to_pfn(pmd_val(pmd)));
+	return (unsigned long)pfn_to_virt(pmd_pfn(pmd));
 }
 
 static inline pte_t pmd_pte(pmd_t pmd)
@@ -325,7 +331,10 @@ static inline unsigned long pte_napot(pte_t pte)
 /* Yields the page frame number (PFN) of a page table entry */
 static inline unsigned long pte_pfn(pte_t pte)
 {
-	unsigned long res  = __page_val_to_pfn(pte_val(pte));
+	unsigned long res;
+
+	ALT_UNFIX_MT(pte);
+	res = __page_val_to_pfn(pte_val(pte));
 
 	if (has_svnapot() && pte_napot(pte))
 		res = res & (res - 1UL);
@@ -338,11 +347,11 @@ static inline unsigned long pte_pfn(pte_t pte)
 /* Constructs a page table entry */
 static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
 {
-	unsigned long prot_val = pgprot_val(prot);
+	pte_t pte = __pte((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
 
-	ALT_THEAD_PMA(prot_val);
+	ALT_FIXUP_MT(pte);
 
-	return __pte((pfn << _PAGE_PFN_SHIFT) | prot_val);
+	return pte;
 }
 
 #define mk_pte(page, prot)       pfn_pte(page_to_pfn(page), prot)
@@ -464,9 +473,11 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 {
 	unsigned long newprot_val = pgprot_val(newprot);
 
-	ALT_THEAD_PMA(newprot_val);
+	ALT_UNFIX_MT(pte);
+	pte = __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val);
+	ALT_FIXUP_MT(pte);
 
-	return __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val);
+	return pte;
 }
 
 #define pgd_ERROR(e) \
@@ -636,20 +647,6 @@ static inline pmd_t pmd_mkinvalid(pmd_t pmd)
 	return __pmd(pmd_val(pmd) & ~(_PAGE_PRESENT|_PAGE_PROT_NONE));
 }
 
-#define __pmd_to_phys(pmd)  (__page_val_to_pfn(pmd_val(pmd)) << PAGE_SHIFT)
-
-static inline unsigned long pmd_pfn(pmd_t pmd)
-{
-	return ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT);
-}
-
-#define __pud_to_phys(pud)  (__page_val_to_pfn(pud_val(pud)) << PAGE_SHIFT)
-
-static inline unsigned long pud_pfn(pud_t pud)
-{
-	return ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT);
-}
-
 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 {
 	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
@@ -869,7 +866,7 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte)
 #define TASK_SIZE_MIN	(PGDIR_SIZE_L3 * PTRS_PER_PGD / 2)
 
 #ifdef CONFIG_COMPAT
-#define TASK_SIZE_32	(_AC(0x80000000, UL))
+#define TASK_SIZE_32	(_AC(0x80000000, UL) - PAGE_SIZE)
 #define TASK_SIZE	(test_thread_flag(TIF_32BIT) ? \
 			 TASK_SIZE_32 : TASK_SIZE_64)
 #else
@@ -908,6 +905,14 @@ extern bool pgtable_l4_enabled;
 void paging_init(void);
 void misc_mem_init(void);
 
+#ifdef CONFIG_RISCV_ISA_SVPBMT
+bool __init riscv_have_memory_alias(void);
+void __init riscv_init_memory_alias(void);
+#else
+static inline bool riscv_have_memory_alias(void) { return false; }
+static inline void riscv_init_memory_alias(void) {}
+#endif /* CONFIG_RISCV_ISA_SVPBMT */
+
 /*
  * ZERO_PAGE is a global shared page that is always zero,
  * used for zero-mapped memory areas, etc.
diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h
index 3ed853b8a8c8..2e87571bd837 100644
--- a/arch/riscv/include/asm/sbi.h
+++ b/arch/riscv/include/asm/sbi.h
@@ -30,6 +30,7 @@ enum sbi_ext_id {
 	SBI_EXT_HSM = 0x48534D,
 	SBI_EXT_SRST = 0x53525354,
 	SBI_EXT_PMU = 0x504D55,
+	SBI_EXT_DBCN = 0x4442434E,
 
 	/* Experimentals extensions must lie within this range */
 	SBI_EXT_EXPERIMENTAL_START = 0x08000000,
@@ -236,6 +237,12 @@ enum sbi_pmu_ctr_type {
 /* Flags defined for counter stop function */
 #define SBI_PMU_STOP_FLAG_RESET (1 << 0)
 
+enum sbi_ext_dbcn_fid {
+	SBI_EXT_DBCN_CONSOLE_WRITE = 0,
+	SBI_EXT_DBCN_CONSOLE_READ = 1,
+	SBI_EXT_DBCN_CONSOLE_WRITE_BYTE = 2,
+};
+
 #define SBI_SPEC_VERSION_DEFAULT	0x1
 #define SBI_SPEC_VERSION_MAJOR_SHIFT	24
 #define SBI_SPEC_VERSION_MAJOR_MASK	0x7f
@@ -322,6 +329,11 @@ static inline unsigned long sbi_mk_version(unsigned long major,
 }
 
 int sbi_err_map_linux_errno(int err);
+
+extern bool sbi_debug_console_available;
+int sbi_debug_console_write(const char *bytes, unsigned int num_bytes);
+int sbi_debug_console_read(char *bytes, unsigned int num_bytes);
+
 #else /* CONFIG_RISCV_SBI */
 static inline int sbi_remote_fence_i(const struct cpumask *cpu_mask) { return -1; }
 static inline void sbi_init(void) {}
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index 03968c06258c..b5fbf6b8fe72 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -59,10 +59,12 @@ obj-y	+= stacktrace.o
 obj-y	+= cacheinfo.o
 obj-y	+= patch.o
 obj-y	+= probes/
+obj-y	+= bus_error.o
 obj-$(CONFIG_MMU) += vdso.o vdso/
 
 obj-$(CONFIG_RISCV_M_MODE)	+= traps_misaligned.o
 obj-$(CONFIG_FPU)		+= fpu.o
+obj-$(CONFIG_FPU)		+= kernel_mode_fpu.o
 obj-$(CONFIG_RISCV_ISA_V)	+= vector.o
 obj-$(CONFIG_SMP)		+= smpboot.o
 obj-$(CONFIG_SMP)		+= smp.o
diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c
index 319a1da0358b..e19cf782b1f5 100644
--- a/arch/riscv/kernel/alternative.c
+++ b/arch/riscv/kernel/alternative.c
@@ -121,8 +121,8 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
 			if (!riscv_insn_is_jalr(insn2))
 				continue;
 
-			/* if instruction pair is a call, it will use the ra register */
-			if (RV_EXTRACT_RD_REG(insn) != 1)
+			/* if instruction pair is a call, it will save a link register */
+			if (RV_EXTRACT_RD_REG(insn) == 0)
 				continue;
 
 			riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32),
diff --git a/arch/riscv/kernel/bus_error.c b/arch/riscv/kernel/bus_error.c
new file mode 100644
index 000000000000..67ea83094fed
--- /dev/null
+++ b/arch/riscv/kernel/bus_error.c
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * BUS error monitor of core driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yu Ning <ningyu@eswincomputing.com>
+ */
+
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/interrupt.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/**
+ * @brief Bus error unit register map
+ * 0x00 : cause of error event
+ * 0x08 : physical address of error event
+ * 0x10 : event enable mask
+ * 0x18 : platform-level interrupt enable mask
+ * 0x20 : accrued event mask
+ * 0x28 : hart-local interrupt enable mask
+ */
+
+struct bus_error_device {
+	struct device *dev;
+	void __iomem *control;
+	int plic_irq;
+};
+
+static irqreturn_t bus_error_handle(int irq, void *dev_id)
+{
+	struct bus_error_device *bus_err = dev_id;
+	void __iomem *base = bus_err->control;
+
+	printk(KERN_ERR "bus error of cause event: %d, accrued: 0x%x, physical address: 0x%llx\n",
+		readl(base),readl(base+0x20),readq(base+0x8));
+
+	/* clean interrupt */
+	writel(0,base);
+	writel(0,base+0x20);
+
+	return IRQ_HANDLED;
+}
+
+static const struct of_device_id eswin_bus_error_of_match[] = {
+	{.compatible = "sifive,buserror", },
+	{ /* sentinel */ }
+};
+
+static int  bus_error_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = pdev->dev.of_node;
+	struct bus_error_device *bus_err_dev;
+	int ret;
+	struct resource *res;
+
+	bus_err_dev = devm_kcalloc(dev, 1,
+		sizeof(struct bus_error_device), GFP_KERNEL);
+	if (!bus_err_dev)
+		return -ENOMEM;
+
+	bus_err_dev->dev = dev;
+	dev_set_drvdata(dev, bus_err_dev);
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(dev, "Error while get mem resource\n");
+		return -ENODEV;
+	}
+	bus_err_dev->control = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR_OR_NULL(bus_err_dev->control)) {
+		dev_err(dev, "Fail to get resource %s from 0x%llx!\n",
+			node->name, res->start);
+		ret = -EINVAL;
+		goto free_bus_err_dev;
+	}
+	bus_err_dev->plic_irq = platform_get_irq(pdev, 0);
+	ret = devm_request_irq(dev, bus_err_dev->plic_irq, bus_error_handle,
+				IRQF_SHARED, dev_name(dev), bus_err_dev);
+	if (ret) {
+		dev_err(dev, "Fail to request irq %d \n",
+				(int)bus_err_dev->plic_irq);
+		return ret;
+	}
+
+	/* clean any interrupt before */
+	writel(0,bus_err_dev->control);
+	writel(0,bus_err_dev->control+0x20);
+
+	/* enable interrupt */
+	writel(0xee2,bus_err_dev->control+0x18);
+	writel(0xee2,bus_err_dev->control+0x10);
+	dev_dbg(dev, "Bus-err unit init OK\n");
+	return 0;
+
+free_bus_err_dev:
+	return ret;
+
+}
+
+static struct platform_driver bus_error_driver = {
+	.probe = bus_error_probe,
+	.driver = {
+		.name = "buserror",
+		.owner = THIS_MODULE,
+		.of_match_table = of_match_ptr(eswin_bus_error_of_match),},
+};
+
+static int __init init_bus_error_unit(void)
+{
+	return platform_driver_register(&bus_error_driver);
+}
+
+subsys_initcall(init_bus_error_unit);
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index bb5fb2b820a2..294117ea5b07 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -550,6 +550,12 @@ void __init riscv_fill_hwcap(void)
 			elf_hwcap &= ~COMPAT_HWCAP_ISA_V;
 	}
 
+	/* Vendor-independent alternatives require a bit in the ISA bitmap. */
+	if (riscv_have_memory_alias()) {
+		set_bit(RISCV_ISA_EXT_XLINUXMEMALIAS, riscv_isa);
+		pr_info("Using physical memory alias for noncached mappings\n");
+	}
+
 	memset(print_str, 0, sizeof(print_str));
 	for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
 		if (riscv_isa[0] & BIT_MASK(i))
diff --git a/arch/riscv/kernel/kernel_mode_fpu.c b/arch/riscv/kernel/kernel_mode_fpu.c
new file mode 100644
index 000000000000..0ac8348876c4
--- /dev/null
+++ b/arch/riscv/kernel/kernel_mode_fpu.c
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023 SiFive
+ */
+
+#include <linux/export.h>
+#include <linux/preempt.h>
+
+#include <asm/csr.h>
+#include <asm/fpu.h>
+#include <asm/processor.h>
+#include <asm/switch_to.h>
+
+void kernel_fpu_begin(void)
+{
+	preempt_disable();
+	fstate_save(current, task_pt_regs(current));
+	csr_set(CSR_SSTATUS, SR_FS);
+}
+EXPORT_SYMBOL_GPL(kernel_fpu_begin);
+
+void kernel_fpu_end(void)
+{
+	csr_clear(CSR_SSTATUS, SR_FS);
+	fstate_restore(current, task_pt_regs(current));
+	preempt_enable();
+}
+EXPORT_SYMBOL_GPL(kernel_fpu_end);
diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c
index e264e59e596e..a191ba7f8892 100644
--- a/arch/riscv/kernel/module-sections.c
+++ b/arch/riscv/kernel/module-sections.c
@@ -9,6 +9,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleloader.h>
+#include <linux/sort.h>
 
 unsigned long module_emit_got_entry(struct module *mod, unsigned long val)
 {
@@ -55,19 +56,27 @@ unsigned long module_emit_plt_entry(struct module *mod, unsigned long val)
 	return (unsigned long)&plt[i];
 }
 
-static int is_rela_equal(const Elf_Rela *x, const Elf_Rela *y)
+#define cmp_3way(a, b)	((a) < (b) ? -1 : (a) > (b))
+
+static int cmp_rela(const void *a, const void *b)
 {
-	return x->r_info == y->r_info && x->r_addend == y->r_addend;
+	const Elf_Rela *x = a, *y = b;
+	int i;
+
+	/* sort by type, symbol index and addend */
+	i = cmp_3way(x->r_info, y->r_info);
+	if (i == 0)
+		i = cmp_3way(x->r_addend, y->r_addend);
+	return i;
 }
 
 static bool duplicate_rela(const Elf_Rela *rela, int idx)
 {
-	int i;
-	for (i = 0; i < idx; i++) {
-		if (is_rela_equal(&rela[i], &rela[idx]))
-			return true;
-	}
-	return false;
+	/*
+	 * Entries are sorted by type, symbol index and addend. That means
+	 * that, if a duplicate entry exists, it must be in the preceding slot.
+	 */
+	return idx > 0 && cmp_rela(rela + idx, rela + idx - 1) == 0;
 }
 
 static void count_max_entries(Elf_Rela *relas, int num,
@@ -87,11 +96,33 @@ static void count_max_entries(Elf_Rela *relas, int num,
 	}
 }
 
+static bool rela_needs_plt_got(const Elf_Rela *rela)
+{
+	unsigned int type = ELF_R_TYPE(rela->r_info);
+
+	return type == R_RISCV_CALL_PLT || type == R_RISCV_GOT_HI20;
+}
+
+/* Copy PLT and GOT relas to the scratch array. */
+static unsigned int partition_plt_got_relas(const Elf_Rela *relas, Elf_Rela *scratch,
+					    unsigned int num_rela)
+{
+	int j = 0;
+
+	for (int i = 0; i < num_rela; i++)
+		if (rela_needs_plt_got(&relas[i]))
+			scratch[j++] = relas[i];
+
+	return j;
+}
+
 int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
 			      char *secstrings, struct module *mod)
 {
 	unsigned int num_plts = 0;
 	unsigned int num_gots = 0;
+	Elf_Rela *scratch = NULL;
+	size_t scratch_size = 0;
 	int i;
 
 	/*
@@ -132,9 +163,26 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
 		if (!(dst_sec->sh_flags & SHF_EXECINSTR))
 			continue;
 
-		count_max_entries(relas, num_rela, &num_plts, &num_gots);
+		/*
+		 * apply_relocate_add() relies on HI20 and LO12 relocation pairs being
+		 * close together, so sort a copy of the section to avoid interfering.
+		 */
+		if (sechdrs[i].sh_size > scratch_size) {
+			scratch = kvrealloc(scratch, scratch_size, sechdrs[i].sh_size, GFP_KERNEL);
+			if (!scratch)
+				return -ENOMEM;
+			scratch_size = sechdrs[i].sh_size;
+		}
+
+		/* sort relocations requiring a PLT or GOT entry so duplicates are adjacent */
+		num_rela = partition_plt_got_relas(relas, scratch, num_rela);
+		sort(scratch, num_rela, sizeof(Elf_Rela), cmp_rela, NULL);
+		count_max_entries(scratch, num_rela, &num_plts, &num_gots);
 	}
 
+	if (scratch)
+		kvfree(scratch);
+
 	mod->arch.plt.shdr->sh_type = SHT_NOBITS;
 	mod->arch.plt.shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
 	mod->arch.plt.shdr->sh_addralign = L1_CACHE_BYTES;
diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c
index df4f6fec5d17..31000dacbba0 100644
--- a/arch/riscv/kernel/module.c
+++ b/arch/riscv/kernel/module.c
@@ -346,6 +346,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 	Elf_Sym *sym;
 	u32 *location;
 	unsigned int i, type;
+	unsigned int j_idx = 0;
 	Elf_Addr v;
 	int res;
 
@@ -384,9 +385,10 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 		v = sym->st_value + rel[i].r_addend;
 
 		if (type == R_RISCV_PCREL_LO12_I || type == R_RISCV_PCREL_LO12_S) {
-			unsigned int j;
+			unsigned int j = j_idx;
+			bool found = false;
 
-			for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) {
+			do {
 				unsigned long hi20_loc =
 					sechdrs[sechdrs[relsec].sh_info].sh_addr
 					+ rel[j].r_offset;
@@ -415,16 +417,26 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 					hi20 = (offset + 0x800) & 0xfffff000;
 					lo12 = offset - hi20;
 					v = lo12;
+					found = true;
 
 					break;
 				}
-			}
-			if (j == sechdrs[relsec].sh_size / sizeof(*rel)) {
+
+				j++;
+				if (j > sechdrs[relsec].sh_size / sizeof(*rel))
+					j = 0;
+
+			} while (j_idx != j);
+
+			if (!found) {
 				pr_err(
 				  "%s: Can not find HI20 relocation information\n",
 				  me->name);
 				return -EINVAL;
 			}
+
+			/* Record the previous j-loop end index */
+			j_idx = j;
 		}
 
 		res = handler(me, location, v);
diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c
index 5a62ed1da453..e66e0999a800 100644
--- a/arch/riscv/kernel/sbi.c
+++ b/arch/riscv/kernel/sbi.c
@@ -7,6 +7,7 @@
 
 #include <linux/bits.h>
 #include <linux/init.h>
+#include <linux/mm.h>
 #include <linux/pm.h>
 #include <linux/reboot.h>
 #include <asm/sbi.h>
@@ -571,6 +572,66 @@ long sbi_get_mimpid(void)
 }
 EXPORT_SYMBOL_GPL(sbi_get_mimpid);
 
+bool sbi_debug_console_available;
+
+int sbi_debug_console_write(const char *bytes, unsigned int num_bytes)
+{
+	phys_addr_t base_addr;
+	struct sbiret ret;
+
+	if (!sbi_debug_console_available)
+		return -EOPNOTSUPP;
+
+	if (is_vmalloc_addr(bytes))
+		base_addr = page_to_phys(vmalloc_to_page(bytes)) +
+			    offset_in_page(bytes);
+	else
+		base_addr = __pa(bytes);
+	if (PAGE_SIZE < (offset_in_page(bytes) + num_bytes))
+		num_bytes = PAGE_SIZE - offset_in_page(bytes);
+
+	if (IS_ENABLED(CONFIG_32BIT))
+		ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_WRITE,
+				num_bytes, lower_32_bits(base_addr),
+				upper_32_bits(base_addr), 0, 0, 0);
+	else
+		ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_WRITE,
+				num_bytes, base_addr, 0, 0, 0, 0);
+
+	if (ret.error == SBI_ERR_FAILURE)
+		return -EIO;
+	return ret.error ? sbi_err_map_linux_errno(ret.error) : ret.value;
+}
+
+int sbi_debug_console_read(char *bytes, unsigned int num_bytes)
+{
+	phys_addr_t base_addr;
+	struct sbiret ret;
+
+	if (!sbi_debug_console_available)
+		return -EOPNOTSUPP;
+
+	if (is_vmalloc_addr(bytes))
+		base_addr = page_to_phys(vmalloc_to_page(bytes)) +
+			    offset_in_page(bytes);
+	else
+		base_addr = __pa(bytes);
+	if (PAGE_SIZE < (offset_in_page(bytes) + num_bytes))
+		num_bytes = PAGE_SIZE - offset_in_page(bytes);
+
+	if (IS_ENABLED(CONFIG_32BIT))
+		ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_READ,
+				num_bytes, lower_32_bits(base_addr),
+				upper_32_bits(base_addr), 0, 0, 0);
+	else
+		ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_READ,
+				num_bytes, base_addr, 0, 0, 0, 0);
+
+	if (ret.error == SBI_ERR_FAILURE)
+		return -EIO;
+	return ret.error ? sbi_err_map_linux_errno(ret.error) : ret.value;
+}
+
 void __init sbi_init(void)
 {
 	int ret;
@@ -612,6 +673,11 @@ void __init sbi_init(void)
 			sbi_srst_reboot_nb.priority = 192;
 			register_restart_handler(&sbi_srst_reboot_nb);
 		}
+		if ((sbi_spec_version >= sbi_mk_version(2, 0)) &&
+		    (sbi_probe_extension(SBI_EXT_DBCN) > 0)) {
+			pr_info("SBI DBCN extension detected\n");
+			sbi_debug_console_available = true;
+		}
 	} else {
 		__sbi_set_timer = __sbi_set_timer_v01;
 		__sbi_send_ipi	= __sbi_send_ipi_v01;
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index ff802d100a57..e156b996f073 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -304,6 +304,7 @@ void __init setup_arch(char **cmdline_p)
 		acpi_init_rintc_map();
 
 	riscv_init_cbo_blocksizes();
+	riscv_init_memory_alias();
 	riscv_fill_hwcap();
 	init_rt_signal_env();
 	apply_boot_alternatives();
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 82229db1ce73..9f1c1f887da2 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -487,7 +487,10 @@ static void kvm_riscv_vcpu_update_config(const unsigned long *isa)
 	if (riscv_isa_extension_available(isa, ZICBOZ))
 		henvcfg |= ENVCFG_CBZE;
 
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 	csr_write(CSR_HENVCFG, henvcfg);
+#endif
+
 #ifdef CONFIG_32BIT
 	csr_write(CSR_HENVCFGH, henvcfg >> 32);
 #endif
diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile
index 2c869f8026a8..48f2049d2fd2 100644
--- a/arch/riscv/mm/Makefile
+++ b/arch/riscv/mm/Makefile
@@ -36,3 +36,4 @@ endif
 obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
 obj-$(CONFIG_RISCV_DMA_NONCOHERENT) += dma-noncoherent.o
 obj-$(CONFIG_RISCV_NONSTANDARD_CACHE_OPS) += cache-ops.o
+obj-$(CONFIG_RISCV_ISA_SVPBMT) += memory-alias.o
diff --git a/arch/riscv/mm/dma-noncoherent.c b/arch/riscv/mm/dma-noncoherent.c
index 341bd6706b4c..2b2d3c05a819 100644
--- a/arch/riscv/mm/dma-noncoherent.c
+++ b/arch/riscv/mm/dma-noncoherent.c
@@ -10,6 +10,10 @@
 #include <linux/mm.h>
 #include <asm/cacheflush.h>
 #include <asm/dma-noncoherent.h>
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+#include <linux/io.h>
+#include <linux/iommu.h>
+#endif
 
 static bool noncoherent_supported __ro_after_init;
 int dma_cache_alignment __ro_after_init = ARCH_DMA_MINALIGN;
@@ -94,6 +98,7 @@ void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
 		break;
 	}
 }
+EXPORT_SYMBOL (arch_sync_dma_for_device);
 
 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
 			   enum dma_data_direction dir)
@@ -113,6 +118,7 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
 		break;
 	}
 }
+EXPORT_SYMBOL (arch_sync_dma_for_cpu);
 
 void arch_dma_prep_coherent(struct page *page, size_t size)
 {
@@ -128,6 +134,13 @@ void arch_dma_prep_coherent(struct page *page, size_t size)
 	ALT_CMO_OP(flush, flush_addr, size, riscv_cbom_block_size);
 }
 
+#ifdef CONFIG_IOMMU_DMA
+void arch_teardown_dma_ops(struct device *dev)
+{
+    dev->dma_ops = NULL;
+}
+#endif
+
 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 		const struct iommu_ops *iommu, bool coherent)
 {
@@ -142,6 +155,11 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 		   dev_driver_string(dev), dev_name(dev));
 
 	dev->dma_coherent = coherent;
+
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	if (iommu)
+		iommu_setup_dma_ops(dev, dma_base, size);
+#endif
 }
 
 void riscv_noncoherent_supported(void)
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index bdf8ac6c7e30..81d4c1852e6f 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -491,7 +491,7 @@ static void __init create_pmd_mapping(pmd_t *pmdp,
 		ptep = pt_ops.get_pte_virt(pte_phys);
 		memset(ptep, 0, PAGE_SIZE);
 	} else {
-		pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx]));
+		pte_phys = PFN_PHYS(pmd_pfn(pmdp[pmd_idx]));
 		ptep = pt_ops.get_pte_virt(pte_phys);
 	}
 
@@ -594,7 +594,7 @@ static void __init create_pud_mapping(pud_t *pudp,
 		nextp = pt_ops.get_pmd_virt(next_phys);
 		memset(nextp, 0, PAGE_SIZE);
 	} else {
-		next_phys = PFN_PHYS(_pud_pfn(pudp[pud_index]));
+		next_phys = PFN_PHYS(pud_pfn(pudp[pud_index]));
 		nextp = pt_ops.get_pmd_virt(next_phys);
 	}
 
@@ -621,7 +621,7 @@ static void __init create_p4d_mapping(p4d_t *p4dp,
 		nextp = pt_ops.get_pud_virt(next_phys);
 		memset(nextp, 0, PAGE_SIZE);
 	} else {
-		next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index]));
+		next_phys = PFN_PHYS(p4d_pfn(p4dp[p4d_index]));
 		nextp = pt_ops.get_pud_virt(next_phys);
 	}
 
@@ -679,7 +679,7 @@ void __init create_pgd_mapping(pgd_t *pgdp,
 		nextp = get_pgd_next_virt(next_phys);
 		memset(nextp, 0, PAGE_SIZE);
 	} else {
-		next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx]));
+		next_phys = PFN_PHYS(pgd_pfn(pgdp[pgd_idx]));
 		nextp = get_pgd_next_virt(next_phys);
 	}
 
diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
index e96251853037..3d6eaaa3b87c 100644
--- a/arch/riscv/mm/kasan_init.c
+++ b/arch/riscv/mm/kasan_init.c
@@ -171,7 +171,7 @@ static void __init kasan_early_clear_pud(p4d_t *p4dp,
 	if (!pgtable_l4_enabled) {
 		pudp = (pud_t *)p4dp;
 	} else {
-		base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp))));
+		base_pud = pt_ops.get_pud_virt(pfn_to_phys(p4d_pfn(p4dp_get(p4dp))));
 		pudp = base_pud + pud_index(vaddr);
 	}
 
@@ -196,7 +196,7 @@ static void __init kasan_early_clear_p4d(pgd_t *pgdp,
 	if (!pgtable_l5_enabled) {
 		p4dp = (p4d_t *)pgdp;
 	} else {
-		base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp))));
+		base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(pgd_pfn(pgdp_get(pgdp))));
 		p4dp = base_p4d + p4d_index(vaddr);
 	}
 
@@ -242,7 +242,7 @@ static void __init kasan_early_populate_pud(p4d_t *p4dp,
 	if (!pgtable_l4_enabled) {
 		pudp = (pud_t *)p4dp;
 	} else {
-		base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp))));
+		base_pud = pt_ops.get_pud_virt(pfn_to_phys(p4d_pfn(p4dp_get(p4dp))));
 		pudp = base_pud + pud_index(vaddr);
 	}
 
@@ -280,7 +280,7 @@ static void __init kasan_early_populate_p4d(pgd_t *pgdp,
 	if (!pgtable_l5_enabled) {
 		p4dp = (p4d_t *)pgdp;
 	} else {
-		base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp))));
+		base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(pgd_pfn(pgdp_get(pgdp))));
 		p4dp = base_p4d + p4d_index(vaddr);
 	}
 
diff --git a/arch/riscv/mm/memory-alias.S b/arch/riscv/mm/memory-alias.S
new file mode 100644
index 000000000000..df2e8cc3f69c
--- /dev/null
+++ b/arch/riscv/mm/memory-alias.S
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 SiFive
+ */
+
+#include <linux/bits.h>
+#include <linux/linkage.h>
+#include <asm/asm.h>
+#include <asm/pgtable.h>
+
+#define CACHED_BASE_OFFSET	(0 * RISCV_SZPTR)
+#define NONCACHED_BASE_OFFSET	(1 * RISCV_SZPTR)
+#define SIZE_OFFSET		(2 * RISCV_SZPTR)
+
+#define SIZEOF_PAIR		(4 * RISCV_SZPTR)
+
+SYM_CODE_START(riscv_fixup_memory_alias)
+	addi	sp, sp, -4 * SZREG
+	REG_S	t2, (0 * SZREG)(sp)
+	REG_S	t3, (1 * SZREG)(sp)
+	REG_S	t4, (2 * SZREG)(sp)
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+	REG_S	t5, (3 * SZREG)(sp)
+
+	/* Save and mask off _PAGE_NAPOT if present. */
+	li	t5, _PAGE_NAPOT
+	and	t5, t1, t5
+	xor	t1, t1, t5
+#endif
+
+	lla	t2, memory_alias_pairs
+.Lfixup_loop:
+	REG_L	t3, SIZE_OFFSET(t2)
+	beqz	t3, .Lfixup_end
+	REG_L	t4, CACHED_BASE_OFFSET(t2)
+	sub	t4, t1, t4
+	bltu	t4, t3, .Lfixup_found
+	addi	t2, t2, SIZEOF_PAIR
+	j	.Lfixup_loop
+
+.Lfixup_found:
+	REG_L	t3, NONCACHED_BASE_OFFSET(t2)
+	add	t1, t3, t4
+
+.Lfixup_end:
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+	xor	t1, t1, t5
+
+	REG_L	t5, (3 * SZREG)(sp)
+#endif
+	REG_L	t4, (2 * SZREG)(sp)
+	REG_L	t3, (1 * SZREG)(sp)
+	REG_L	t2, (0 * SZREG)(sp)
+	addi	sp, sp, 4 * SZREG
+	jr	t0
+SYM_CODE_END(riscv_fixup_memory_alias)
+
+SYM_CODE_START(riscv_unfix_memory_alias)
+	addi	sp, sp, -4 * SZREG
+	REG_S	t2, (0 * SZREG)(sp)
+	REG_S	t3, (1 * SZREG)(sp)
+	REG_S	t4, (2 * SZREG)(sp)
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+	REG_S	t5, (3 * SZREG)(sp)
+
+	/* Save and mask off _PAGE_NAPOT if present. */
+	li	t5, _PAGE_NAPOT
+	and	t5, t1, t5
+	xor	t1, t1, t5
+#endif
+
+	lla	t2, memory_alias_pairs
+.Lunfix_loop:
+	REG_L	t3, SIZE_OFFSET(t2)
+	beqz	t3, .Lunfix_end
+	REG_L	t4, NONCACHED_BASE_OFFSET(t2)
+	sub	t4, t1, t4
+	bltu	t4, t3, .Lunfix_found
+	addi	t2, t2, SIZEOF_PAIR
+	j	.Lunfix_loop
+
+.Lunfix_found:
+	REG_L	t3, CACHED_BASE_OFFSET(t2)
+	add	t1, t3, t4
+
+	/* PFN was in the noncached alias, so mark it as such. */
+	li	t2, _PAGE_NOCACHE
+	or	t1, t1, t2
+
+.Lunfix_end:
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+	xor	t1, t1, t5
+
+	REG_L	t5, (3 * SZREG)(sp)
+#endif
+	REG_L	t4, (2 * SZREG)(sp)
+	REG_L	t3, (1 * SZREG)(sp)
+	REG_L	t2, (0 * SZREG)(sp)
+	addi	sp, sp, 4 * SZREG
+	jr	t0
+SYM_CODE_END(riscv_unfix_memory_alias)
diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c
index 271d01a5ba4d..5634dde881f8 100644
--- a/arch/riscv/mm/pageattr.c
+++ b/arch/riscv/mm/pageattr.c
@@ -109,9 +109,8 @@ static int __split_linear_mapping_pmd(pud_t *pudp,
 			continue;
 
 		if (pmd_leaf(pmdp_get(pmdp))) {
+			pte_t pte = pmd_pte(pmdp_get(pmdp));
 			struct page *pte_page;
-			unsigned long pfn = _pmd_pfn(pmdp_get(pmdp));
-			pgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) & ~_PAGE_PFN_MASK);
 			pte_t *ptep_new;
 			int i;
 
@@ -121,7 +120,8 @@ static int __split_linear_mapping_pmd(pud_t *pudp,
 
 			ptep_new = (pte_t *)page_address(pte_page);
 			for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep_new)
-				set_pte(ptep_new, pfn_pte(pfn + i, prot));
+				set_pte(ptep_new,
+					__pte(pte_val(pte) + (i << _PAGE_PFN_SHIFT)));
 
 			smp_wmb();
 
@@ -149,9 +149,8 @@ static int __split_linear_mapping_pud(p4d_t *p4dp,
 			continue;
 
 		if (pud_leaf(pudp_get(pudp))) {
+			pmd_t pmd = __pmd(pud_val(pudp_get(pudp)));
 			struct page *pmd_page;
-			unsigned long pfn = _pud_pfn(pudp_get(pudp));
-			pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK);
 			pmd_t *pmdp_new;
 			int i;
 
@@ -162,7 +161,8 @@ static int __split_linear_mapping_pud(p4d_t *p4dp,
 			pmdp_new = (pmd_t *)page_address(pmd_page);
 			for (i = 0; i < PTRS_PER_PMD; ++i, ++pmdp_new)
 				set_pmd(pmdp_new,
-					pfn_pmd(pfn + ((i * PMD_SIZE) >> PAGE_SHIFT), prot));
+					__pmd(pmd_val(pmd) +
+					      (i << (PMD_SHIFT - PAGE_SHIFT + _PAGE_PFN_SHIFT))));
 
 			smp_wmb();
 
@@ -198,9 +198,8 @@ static int __split_linear_mapping_p4d(pgd_t *pgdp,
 			continue;
 
 		if (p4d_leaf(p4dp_get(p4dp))) {
+			pud_t pud = __pud(p4d_val(p4dp_get(p4dp)));
 			struct page *pud_page;
-			unsigned long pfn = _p4d_pfn(p4dp_get(p4dp));
-			pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK);
 			pud_t *pudp_new;
 			int i;
 
@@ -215,7 +214,8 @@ static int __split_linear_mapping_p4d(pgd_t *pgdp,
 			pudp_new = (pud_t *)page_address(pud_page);
 			for (i = 0; i < PTRS_PER_PUD; ++i, ++pudp_new)
 				set_pud(pudp_new,
-					pfn_pud(pfn + ((i * PUD_SIZE) >> PAGE_SHIFT), prot));
+					__pud(pud_val(pud) +
+					      (i << (PUD_SHIFT - PAGE_SHIFT + _PAGE_PFN_SHIFT))));
 
 			/*
 			 * Make sure the pud filling is not reordered with the
diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c
index ef887efcb679..5fb9ff6aa31d 100644
--- a/arch/riscv/mm/pgtable.c
+++ b/arch/riscv/mm/pgtable.c
@@ -1,8 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0
 
 #include <asm/pgalloc.h>
+#include <dt-bindings/riscv/physical-memory.h>
+#include <linux/bitfield.h>
 #include <linux/gfp.h>
 #include <linux/kernel.h>
+#include <linux/memblock.h>
+#include <linux/of.h>
 #include <linux/pgtable.h>
 
 int ptep_set_access_flags(struct vm_area_struct *vma,
@@ -142,3 +146,90 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
 	return pmd;
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
+#ifdef CONFIG_RISCV_ISA_SVPBMT
+struct memory_alias_pair {
+	unsigned long cached_base;
+	unsigned long noncached_base;
+	unsigned long size;
+	int index;
+} memory_alias_pairs[5];
+
+bool __init riscv_have_memory_alias(void)
+{
+	return memory_alias_pairs[0].size;
+}
+
+void __init riscv_init_memory_alias(void)
+{
+	int na = of_n_addr_cells(of_root);
+	int ns = of_n_size_cells(of_root);
+	int nc = na + ns + 2;
+	const __be32 *prop;
+	int pairs = 0;
+	int len;
+
+	prop = of_get_property(of_root, "riscv,physical-memory-regions", &len);
+	if (!prop)
+		return;
+
+	len /= sizeof(__be32);
+	for (int i = 0; len >= nc; i++, prop += nc, len -= nc) {
+		unsigned long base = of_read_ulong(prop, na);
+		unsigned long size = of_read_ulong(prop + na, ns);
+		unsigned long flags = be32_to_cpup(prop + na + ns);
+		struct memory_alias_pair *pair;
+		int alias;
+
+		/* We only care about non-coherent memory. */
+		if ((flags & PMA_ORDER_MASK) != PMA_ORDER_MEMORY || (flags & PMA_COHERENT))
+			continue;
+
+		/* The cacheable alias must be usable memory. */
+		if ((flags & PMA_CACHEABLE) &&
+		    !memblock_overlaps_region(&memblock.memory, base, size))
+			continue;
+
+		alias = FIELD_GET(PMR_ALIAS_MASK, flags);
+		if (alias) {
+			pair = NULL;
+			for (int j = 0; j < pairs; j++) {
+				if (alias == memory_alias_pairs[j].index) {
+					pair = &memory_alias_pairs[j];
+					break;
+				}
+			}
+			if (!pair)
+				continue;
+		} else {
+			/* Leave room for the null sentinel. */
+			if (pairs == ARRAY_SIZE(memory_alias_pairs) - 1)
+				continue;
+			pair = &memory_alias_pairs[pairs++];
+			pair->index = i;
+		}
+
+		/* Align the address and size with the page table PFN field. */
+		base >>= PAGE_SHIFT - _PAGE_PFN_SHIFT;
+		size >>= PAGE_SHIFT - _PAGE_PFN_SHIFT;
+
+		if (flags & PMA_CACHEABLE)
+			pair->cached_base = base;
+		else
+			pair->noncached_base = base;
+		pair->size = min_not_zero(pair->size, size);
+	}
+
+	/* Remove any unmatched pairs. */
+	for (int i = 0; i < pairs; i++) {
+		struct memory_alias_pair *pair = &memory_alias_pairs[i];
+
+		if (pair->cached_base && pair->noncached_base && pair->size)
+			continue;
+
+		for (int j = i + 1; j < pairs; j++)
+			memory_alias_pairs[j - 1] = memory_alias_pairs[j];
+		memory_alias_pairs[--pairs].size = 0;
+	}
+}
+#endif /* CONFIG_RISCV_ISA_SVPBMT */
diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c
index e9090b38f811..a58de0a6cbdc 100644
--- a/arch/riscv/mm/ptdump.c
+++ b/arch/riscv/mm/ptdump.c
@@ -129,55 +129,57 @@ static struct ptd_mm_info efi_ptd_info = {
 /* Page Table Entry */
 struct prot_bits {
 	u64 mask;
-	u64 val;
 	const char *set;
 	const char *clear;
 };
 
 static const struct prot_bits pte_bits[] = {
 	{
+#ifdef CONFIG_RISCV_ISA_SVNAPOT
+		.mask = _PAGE_NAPOT,
+		.set = "N",
+		.clear = ".",
+	}, {
+#endif
+#ifdef CONFIG_RISCV_ISA_SVPBMT
+		.mask = _PAGE_MTMASK,
+		.set = "MT(%s)",
+		.clear = "  ..  ",
+	}, {
+#endif
 		.mask = _PAGE_SOFT,
-		.val = _PAGE_SOFT,
-		.set = "RSW",
-		.clear = "   ",
+		.set = "RSW(%d)",
+		.clear = "  ..  ",
 	}, {
 		.mask = _PAGE_DIRTY,
-		.val = _PAGE_DIRTY,
 		.set = "D",
 		.clear = ".",
 	}, {
 		.mask = _PAGE_ACCESSED,
-		.val = _PAGE_ACCESSED,
 		.set = "A",
 		.clear = ".",
 	}, {
 		.mask = _PAGE_GLOBAL,
-		.val = _PAGE_GLOBAL,
 		.set = "G",
 		.clear = ".",
 	}, {
 		.mask = _PAGE_USER,
-		.val = _PAGE_USER,
 		.set = "U",
 		.clear = ".",
 	}, {
 		.mask = _PAGE_EXEC,
-		.val = _PAGE_EXEC,
 		.set = "X",
 		.clear = ".",
 	}, {
 		.mask = _PAGE_WRITE,
-		.val = _PAGE_WRITE,
 		.set = "W",
 		.clear = ".",
 	}, {
 		.mask = _PAGE_READ,
-		.val = _PAGE_READ,
 		.set = "R",
 		.clear = ".",
 	}, {
 		.mask = _PAGE_PRESENT,
-		.val = _PAGE_PRESENT,
 		.set = "V",
 		.clear = ".",
 	}
@@ -208,15 +210,30 @@ static void dump_prot(struct pg_state *st)
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(pte_bits); i++) {
-		const char *s;
-
-		if ((st->current_prot & pte_bits[i].mask) == pte_bits[i].val)
-			s = pte_bits[i].set;
-		else
-			s = pte_bits[i].clear;
+		char s[7];
+		unsigned long val;
+
+		val = st->current_prot & pte_bits[i].mask;
+		if (val) {
+			if (pte_bits[i].mask == _PAGE_SOFT)
+				sprintf(s, pte_bits[i].set, val >> 8);
+#ifdef CONFIG_RISCV_ISA_SVPBMT
+			else if (pte_bits[i].mask == _PAGE_MTMASK) {
+				if (val == _PAGE_NOCACHE)
+					sprintf(s, pte_bits[i].set, "NC");
+				else if (val == _PAGE_IO)
+					sprintf(s, pte_bits[i].set, "IO");
+				else
+					sprintf(s, pte_bits[i].set, "??");
+			}
+#endif
+			else
+				sprintf(s, "%s", pte_bits[i].set);
+		} else {
+			sprintf(s, "%s", pte_bits[i].clear);
+		}
 
-		if (s)
-			pt_dump_seq_printf(st->seq, " %s", s);
+		pt_dump_seq_printf(st->seq, " %s", s);
 	}
 }
 
@@ -265,9 +282,12 @@ static void note_page(struct ptdump_state *pt_st, unsigned long addr,
 		      int level, u64 val)
 {
 	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
-	u64 pa = PFN_PHYS(pte_pfn(__pte(val)));
 	u64 prot = 0;
+	u64 pa;
+
+	ALT_UNFIX_MT(val);
 
+	pa = PFN_PHYS(pte_pfn(__pte(val)));
 	if (level >= 0)
 		prot = val & pg_level[level].mask;
 
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index 42b51c9812a0..4761bf0d39d1 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -216,6 +216,15 @@ config AHCI_CEVA
 
 	  If unsure, say N.
 
+config AHCI_ESWIN
+	tristate "Eswin AHCI SATA support"
+	select SATA_HOST
+	help
+	  This option enables support for Eswin AHCI Serial ATA
+	  controllers.
+
+	  If unsure, say N.
+
 config AHCI_MTK
 	tristate "MediaTek AHCI SATA support"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile
index 20e6645ab737..515e62a23ae7 100644
--- a/drivers/ata/Makefile
+++ b/drivers/ata/Makefile
@@ -18,6 +18,7 @@ obj-$(CONFIG_AHCI_CEVA)		+= ahci_ceva.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_DA850)	+= ahci_da850.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_DM816)	+= ahci_dm816.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_DWC)		+= ahci_dwc.o libahci.o libahci_platform.o
+obj-$(CONFIG_AHCI_ESWIN)	+= ahci_eswin.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_IMX)		+= ahci_imx.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_MTK)		+= ahci_mtk.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_MVEBU)	+= ahci_mvebu.o libahci.o libahci_platform.o
diff --git a/drivers/ata/ahci_eswin.c b/drivers/ata/ahci_eswin.c
new file mode 100644
index 000000000000..ceec74a5f975
--- /dev/null
+++ b/drivers/ata/ahci_eswin.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN AHCI SATA Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yulin Lu <luyulin@eswincomputing.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pm.h>
+#include <linux/device.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/libata.h>
+#include <linux/ahci_platform.h>
+#include <linux/acpi.h>
+#include <linux/pci_ids.h>
+#include <linux/iommu.h>
+#include <linux/eic7700-sid-cfg.h>
+#include  <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include "ahci.h"
+
+#define DRV_NAME "ahci"
+
+#define AWSMMUSID                     GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID                    GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID                     GENMASK(15, 8)  // The sid of read operation
+#define ARSMMUSSID                    GENMASK(7, 0)   // The ssid of read operation
+#define SATA_REF_CTRL1                0x338
+#define SATA_PHY_CTRL0                0x328
+#define SATA_PHY_CTRL1                0x32c
+#define SATA_LOS_IDEN                 0x33c
+#define SATA_AXI_LP_CTRL              0x308
+#define SATA_REG_CTRL                 0x334
+#define SATA_MPLL_CTRL                0x320
+#define SATA_RESET_CTRL               0x340
+#define SATA_RESET_CTRL_ASSERT        0x3
+#define SATA_RESET_CTRL_DEASSERT      0x0
+#define SATA_PHY_RESET                BIT(0)
+#define SATA_P0_RESET                 BIT(1)
+#define SATA_LOS_LEVEL                0x9
+#define SATA_LOS_BIAS                 (0x02 << 16)
+#define SATA_REF_REPEATCLK_EN         BIT(0)
+#define SATA_REF_USE_PAD              BIT(20)
+#define SATA_P0_AMPLITUDE_GEN1        0x42
+#define SATA_P0_AMPLITUDE_GEN2        (0x46 << 8)
+#define SATA_P0_AMPLITUDE_GEN3        (0x73 << 16)
+#define SATA_P0_PHY_TX_PREEMPH_GEN1   0x05
+#define SATA_P0_PHY_TX_PREEMPH_GEN2   (0x05 << 8)
+#define SATA_P0_PHY_TX_PREEMPH_GEN3   (0x23 << 16)
+#define SATA_MPLL_MULTIPLIER          (0x3c << 16)
+#define SATA_M_CSYSREQ                BIT(0)
+#define SATA_S_CSYSREQ                BIT(16)
+#define HSPDME_RST_CTRL               0x41C
+#define SW_HSP_SATA_ARSTN             BIT(27)
+#define SW_SATA_RSTN                  (0xf << 9)
+
+static const struct ata_port_info ahci_port_info = {
+    .flags		= AHCI_FLAG_COMMON,
+    .pio_mask	= ATA_PIO4,
+    .udma_mask	= ATA_UDMA6,
+    .port_ops	= &ahci_platform_ops,
+};
+
+static const struct ata_port_info ahci_port_info_nolpm = {
+    .flags		= AHCI_FLAG_COMMON | ATA_FLAG_NO_LPM,
+    .pio_mask	= ATA_PIO4,
+    .udma_mask	= ATA_UDMA6,
+    .port_ops	= &ahci_platform_ops,
+};
+
+static struct scsi_host_template ahci_platform_sht = {
+    AHCI_SHT(DRV_NAME),
+};
+
+static int eswin_sata_sid_cfg(struct device *dev)
+{
+    int ret;
+    struct regmap *regmap;
+    int hsp_mmu_sata_reg;
+    u32 rdwr_sid_ssid;
+    u32 sid;
+    struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+    /* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+    if (fwspec == NULL) {
+        dev_dbg(dev, "dev is not behind smmu, skip configuration of sid\n");
+        return 0;
+    }
+    sid = fwspec->ids[0];
+    regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+    if (IS_ERR(regmap)) {
+        dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+        return 0;
+    }
+    ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+            &hsp_mmu_sata_reg);
+    if (ret) {
+        dev_err(dev, "can't get sata sid cfg reg offset (%d)\n", ret);
+        return ret;
+    }
+
+    /* make the reading sid the same as writing sid, ssid is fixed to zero */
+    rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+    rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+    rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+    rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+    regmap_write(regmap, hsp_mmu_sata_reg, rdwr_sid_ssid);
+
+    ret = eic7700_dynm_sid_enable(dev_to_node(dev));
+    if (ret < 0)
+        dev_err(dev, "failed to config sata streamID(%d)!\n", sid);
+     else
+        dev_dbg(dev, "success to config sata streamID(%d)!\n", sid);
+    pr_err("eswin_sata_sid_cfg success\n");
+   
+	return ret;
+}
+
+static int eswin_sata_init(struct device *dev)
+{
+    struct regmap *regmap;
+    regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+    if (IS_ERR(regmap)) {
+        dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+        return -1;
+    }
+    regmap_write(regmap, SATA_REF_CTRL1, 0x1);
+    regmap_write(regmap, SATA_PHY_CTRL0, (SATA_P0_AMPLITUDE_GEN1|SATA_P0_AMPLITUDE_GEN2|SATA_P0_AMPLITUDE_GEN3));
+    regmap_write(regmap, SATA_PHY_CTRL1, (SATA_P0_PHY_TX_PREEMPH_GEN1|SATA_P0_PHY_TX_PREEMPH_GEN2|SATA_P0_PHY_TX_PREEMPH_GEN3));
+    regmap_write(regmap, SATA_LOS_IDEN, SATA_LOS_LEVEL|SATA_LOS_BIAS);
+    regmap_write(regmap, SATA_AXI_LP_CTRL, (SATA_M_CSYSREQ|SATA_S_CSYSREQ));
+    regmap_write(regmap, SATA_REG_CTRL, (SATA_REF_REPEATCLK_EN|SATA_REF_USE_PAD));
+    regmap_write(regmap, SATA_MPLL_CTRL, SATA_MPLL_MULTIPLIER);
+    regmap_write(regmap, SATA_RESET_CTRL, 0x0);
+    
+	return 0;
+}
+
+static int eswin_reset(struct device *dev)
+{
+    struct reset_control *asic0_rst;
+    struct reset_control *oob_rst;
+    struct reset_control *pmalive_rst;
+    struct reset_control *rbc_rst;
+    struct reset_control *apb_rst;
+    int rc;
+    
+	asic0_rst = devm_reset_control_get_shared(dev, "asic0");
+    if (IS_ERR_OR_NULL(asic0_rst)) {
+        dev_err(dev, "Failed to asic0_rst handle\n");
+        return -EFAULT;
+    }
+    oob_rst = devm_reset_control_get_shared(dev, "oob");
+    if (IS_ERR_OR_NULL(oob_rst)) {
+        dev_err(dev, "Failed to oob_rst handle\n");
+        return -EFAULT;
+    }
+    pmalive_rst = devm_reset_control_get_shared(dev, "pmalive");
+    if (IS_ERR_OR_NULL(pmalive_rst)) {
+        dev_err(dev, "Failed to pmalive_rst handle\n");
+        return -EFAULT;
+    }
+    rbc_rst = devm_reset_control_get_shared(dev, "rbc");
+    if (IS_ERR_OR_NULL(rbc_rst)) {
+        dev_err(dev, "Failed to rbc_rst handle\n");
+        return -EFAULT;
+    }
+    apb_rst = devm_reset_control_get_shared(dev, "apb");
+    if (IS_ERR_OR_NULL(apb_rst)) {
+        dev_err(dev, "Failed to apb_rst handle\n");
+        return -EFAULT;
+    }
+
+    if (asic0_rst) {
+        rc = reset_control_deassert(asic0_rst);
+        WARN_ON(0 != rc);
+    }
+    if (oob_rst) {
+        rc = reset_control_deassert(oob_rst);
+        WARN_ON(0 != rc);
+    }
+    if (pmalive_rst) {
+        rc = reset_control_deassert(pmalive_rst);
+        WARN_ON(0 != rc);
+    }
+    if (rbc_rst) {
+        rc = reset_control_deassert(rbc_rst);
+        WARN_ON(0 != rc);
+    }
+    if (apb_rst) {
+        rc = reset_control_deassert(apb_rst);
+        WARN_ON(0 != rc);
+    }
+
+    return 0;
+}
+
+
+static int eswin_unreset(struct device *dev)
+{
+    struct reset_control *asic0_rst;
+    struct reset_control *oob_rst;
+    struct reset_control *pmalive_rst;
+    struct reset_control *rbc_rst;
+    int rc;
+
+    asic0_rst = devm_reset_control_get_shared(dev, "asic0");
+    if (IS_ERR_OR_NULL(asic0_rst)) {
+        dev_err(dev, "Failed to asic0_rst handle\n");
+        return -EFAULT;
+    }
+    oob_rst = devm_reset_control_get_shared(dev, "oob");
+    if (IS_ERR_OR_NULL(oob_rst)) {
+        dev_err(dev, "Failed to oob_rst handle\n");
+        return -EFAULT;
+    }
+    pmalive_rst = devm_reset_control_get_shared(dev, "pmalive");
+    if (IS_ERR_OR_NULL(pmalive_rst)) {
+        dev_err(dev, "Failed to pmalive_rst handle\n");
+        return -EFAULT;
+    }
+    rbc_rst = devm_reset_control_get_shared(dev, "rbc");
+    if (IS_ERR_OR_NULL(rbc_rst)) {
+        dev_err(dev, "Failed to rbc_rst handle\n");
+        return -EFAULT;
+    }
+    if (asic0_rst) {
+        rc = reset_control_assert(asic0_rst);
+        WARN_ON(0 != rc);
+    }
+    if (oob_rst) {
+        rc = reset_control_assert(oob_rst);
+        WARN_ON(0 != rc);
+    }
+    if (pmalive_rst) {
+        rc = reset_control_assert(pmalive_rst);
+        WARN_ON(0 != rc);
+    }
+    if (rbc_rst) {
+        rc = reset_control_assert(rbc_rst);
+        WARN_ON(0 != rc);
+    }
+
+return 0;
+}
+
+static int ahci_probe(struct platform_device *pdev)
+{
+    struct device *dev = &pdev->dev;
+    struct ahci_host_priv *hpriv;
+    const struct ata_port_info *port;
+    int rc;
+
+    hpriv = ahci_platform_get_resources(pdev,
+                        0);
+    if (IS_ERR(hpriv))
+        return PTR_ERR(hpriv);
+
+    rc = eswin_reset(dev);
+    if (rc)
+        return rc;
+    eswin_sata_init(dev);
+    eswin_sata_sid_cfg(dev);
+    eic7700_tbu_power(&pdev->dev, true);
+    rc = dma_set_mask_and_coherent(dev,DMA_BIT_MASK(64));
+
+    if (of_device_is_compatible(dev->of_node, "hisilicon,hisi-ahci"))
+        hpriv->flags |= AHCI_HFLAG_NO_FBS | AHCI_HFLAG_NO_NCQ;
+
+    port = acpi_device_get_match_data(dev);
+    if (!port){
+        port = &ahci_port_info;
+    }
+    rc = ahci_platform_init_host(pdev, hpriv, port,
+                &ahci_platform_sht);
+    if (rc)
+        goto disable_resources;
+
+    return 0;
+
+disable_resources:
+    ahci_platform_disable_resources(hpriv);
+    return rc;
+}
+
+static void ahci_remove(struct platform_device *pdev)
+{
+    eic7700_tbu_power(&pdev->dev, false);
+    eswin_unreset(&pdev->dev);
+
+    ata_platform_remove_one(pdev);
+}
+
+static SIMPLE_DEV_PM_OPS(ahci_pm_ops, ahci_platform_suspend,
+            ahci_platform_resume);
+
+static const struct of_device_id ahci_of_match[] = {
+    { .compatible = "snps,eswin-ahci", },
+    {},
+};
+MODULE_DEVICE_TABLE(of, ahci_of_match);
+
+static const struct acpi_device_id ahci_acpi_match[] = {
+    { "APMC0D33", (unsigned long)&ahci_port_info_nolpm },
+    { ACPI_DEVICE_CLASS(PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff) },
+    {},
+};
+MODULE_DEVICE_TABLE(acpi, ahci_acpi_match);
+
+static struct platform_driver ahci_driver = {
+    .probe = ahci_probe,
+    .remove_new = ahci_remove,
+    .shutdown = ahci_platform_shutdown,
+    .driver = {
+        .name = DRV_NAME,
+        .of_match_table = ahci_of_match,
+        .acpi_match_table = ahci_acpi_match,
+        .pm = &ahci_pm_ops,
+    },
+};
+module_platform_driver(ahci_driver);
+
+MODULE_DESCRIPTION("ESWIN AHCI SATA driver");
+MODULE_AUTHOR("Lu Yulin <luyulin@eswincomputing.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index c30099866174..151de7e16525 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -471,6 +471,7 @@ source "drivers/clk/actions/Kconfig"
 source "drivers/clk/analogbits/Kconfig"
 source "drivers/clk/baikal-t1/Kconfig"
 source "drivers/clk/bcm/Kconfig"
+source "drivers/clk/eswin/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
 source "drivers/clk/imgtec/Kconfig"
 source "drivers/clk/imx/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index 18969cbd4bb1..1d36e1ef74df 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -91,6 +91,7 @@ obj-$(CONFIG_CLK_BAIKAL_T1)		+= baikal-t1/
 obj-y					+= bcm/
 obj-$(CONFIG_ARCH_BERLIN)		+= berlin/
 obj-$(CONFIG_ARCH_DAVINCI)		+= davinci/
+obj-$(CONFIG_SOC_SIFIVE_EIC7700)	+= eswin/
 obj-$(CONFIG_ARCH_HISI)			+= hisilicon/
 obj-y					+= imgtec/
 obj-y					+= imx/
diff --git a/drivers/clk/eswin/Kconfig b/drivers/clk/eswin/Kconfig
new file mode 100755
index 000000000000..cc906e4b2343
--- /dev/null
+++ b/drivers/clk/eswin/Kconfig
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config COMMON_CLK_EIC7700
+	bool "EIC7700 Clock Driver"
+	help
+	  Build the Eswin EIC7700 clock driver based on the common clock framework.
diff --git a/drivers/clk/eswin/Makefile b/drivers/clk/eswin/Makefile
new file mode 100755
index 000000000000..a3139e34ee22
--- /dev/null
+++ b/drivers/clk/eswin/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Eswin Clock specific Makefile
+#
+
+obj-y	+= clk.o
+
+obj-$(CONFIG_COMMON_CLK_EIC7700)	+= clk-eic7700.o
diff --git a/drivers/clk/eswin/clk-eic7700.c b/drivers/clk/eswin/clk-eic7700.c
new file mode 100755
index 000000000000..1b6c0b9e0450
--- /dev/null
+++ b/drivers/clk/eswin/clk-eic7700.c
@@ -0,0 +1,1187 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Clk Provider Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+
+#include <dt-bindings/reset/eic7700-syscrg.h>
+#include <dt-bindings/clock/eic7700-clock.h>
+
+#include "clk.h"
+
+/* clock parent list */
+static const char *const mux_u_cpu_root_3mux1_gfree_p[] = {"clk_pll_cpu", "clk_clk_u84_core_lp", "fixed_rate_clk_xtal_24m"};
+static u32  mux_u_cpu_root_3mux1_gfree_p_table[] = {0x000000, 0x000001, 0x000002};
+
+static const char *const mux_u_cpu_aclk_2mux1_gfree_p[] = {"fixed_factor_u_cpu_div2", "mux_u_cpu_root_3mux1_gfree"};
+
+static const char *const dsp_aclk_root_2mux1_gfree_mux_p[] = {"fixed_rate_clk_spll2_fout1", "fixed_rate_clk_spll0_fout1"};
+
+static const char *const d2d_aclk_root_2mux1_gfree_mux_p[] = { "fixed_rate_clk_spll2_fout1", "fixed_rate_clk_spll0_fout1", };
+
+static const char *const ddr_aclk_root_2mux1_gfree_mux_p[] = { "fixed_rate_clk_spll2_fout1", "fixed_rate_clk_spll0_fout1", };
+
+static const char *const mshcore_root_3mux1_0_mux_p[] = {"fixed_rate_clk_spll0_fout3", "fixed_rate_clk_spll2_fout3", "fixed_rate_clk_xtal_24m"};
+static u32  mshcore_root_3mux1_0_mux_p_table[] = {0x000000, 0x000001, 0x100000};
+
+static const char *const mshcore_root_3mux1_1_mux_p[] = {"fixed_rate_clk_spll0_fout3", "fixed_rate_clk_spll2_fout3", "fixed_rate_clk_xtal_24m"};
+static u32  mshcore_root_3mux1_1_mux_p_table[] = {0x000000, 0x000001, 0x100000};
+
+static const char *const mshcore_root_3mux1_2_mux_p[] = {"fixed_rate_clk_spll0_fout3", "fixed_rate_clk_spll2_fout3", "fixed_rate_clk_xtal_24m"};
+static u32  mshcore_root_3mux1_2_mux_p_table[] = {0x000000, 0x000001, 0x100000};
+
+static const char *const npu_llclk_3mux1_gfree_mux_p[] = { "clk_clk_npu_llc_src0", "clk_clk_npu_llc_src1", "fixed_rate_clk_vpll_fout1"};
+static u32  npu_llclk_3mux1_gfree_mux_p_table[] = {0x000000, 0x000001, 0x000002};
+
+static const char *const npu_core_3mux1_gfree_mux_p[] = { "fixed_rate_clk_spll1_fout1", "fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll2_fout2"};
+static u32  npu_core_3mux1_gfree_mux_p_table[] = {0x000000, 0x000001, 0x000002};
+
+static const char *const npu_e31_3mux1_gfree_mux_p[] = { "fixed_rate_clk_spll1_fout1", "fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll2_fout2"};
+static u32  npu_e31_3mux1_gfree_mux_p_table[] = {0x000000, 0x000100, 0x000200};
+
+static const char *const vi_aclk_root_2mux1_gfree_mux_p[] = { "fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"};
+
+static const char *const mux_u_vi_dw_root_2mux1_p[] = { "fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll0_fout1"};
+
+static const char *const mux_u_vi_dvp_root_2mux1_gfree_p[] = { "fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll0_fout1"};
+
+static const char *const mux_u_vi_dig_isp_root_2mux1_gfree_p[] = { "fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll0_fout1"};
+
+static const char *const mux_u_vo_aclk_root_2mux1_gfree_p[] = { "fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"};
+
+static const char *const mux_u_vo_pixel_root_2mux1_p[] = { "fixed_rate_clk_vpll_fout1", "fixed_rate_clk_spll2_fout2"};
+
+static const char *const mux_u_vcdec_root_2mux1_gfree_p[] = { "fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"};
+
+static const char *const mux_u_vcaclk_root_2mux1_gfree_p[] = { "fixed_rate_clk_spll0_fout1", "fixed_rate_clk_spll2_fout1"};
+
+static const char *const mux_u_syscfg_clk_root_2mux1_gfree_p[] = { "divder_u_sys_cfg_div_dynm", "fixed_rate_clk_xtal_24m"};
+
+static const char *const mux_u_bootspi_clk_2mux1_gfree_p[] = {  "divder_u_bootspi_div_dynm", "fixed_rate_clk_xtal_24m"};
+
+static const char *const mux_u_scpu_core_clk_2mux1_gfree_p[] = { "divder_u_scpu_core_div_dynm", "fixed_rate_clk_xtal_24m"};
+
+static const char *const mux_u_lpcpu_core_clk_2mux1_gfree_p[] = {"divder_u_lpcpu_core_div_dynm", "fixed_rate_clk_xtal_24m"};
+
+static const char *const mux_u_vo_mclk_2mux_ext_mclk_p[] = {"divder_u_vo_mclk_div_dynm", "fixed_rate_ext_mclk"};
+
+static const char *const mux_u_aondma_axi2mux1_gfree_p[] = { "divder_u_aondma_axi_div_dynm", "fixed_rate_clk_xtal_24m"};
+
+static const char *const mux_u_rmii_ref_2mux1_p[] = { "fixed_factor_u_hsp_rmii_ref_div6", "fixed_rate_lpddr_ref_bak"};
+
+static const char *const mux_u_eth_core_2mux1_p[] = { "fixed_rate_clk_spll1_fout3", "fixed_rate_lpddr_ref_bak"};
+
+static const char *const mux_u_sata_phy_2mux1_p[] = { "divder_u_sata_phy_ref_div_dynm", "fixed_rate_lpddr_ref_bak"};
+
+/* fixed rate clocks */
+static struct eswin_fixed_rate_clock eic7700_fixed_rate_clks[] = {
+	{ EIC7700_XTAL_24M,		"fixed_rate_clk_xtal_24m",	NULL, 0,	24000000, },
+	{ EIC7700_XTAL_32K,		"fixed_rate_clk_xtal_32k",	NULL, 0,	32768, },
+	{ EIC7700_SPLL0_FOUT1,		"fixed_rate_clk_spll0_fout1",	NULL, 0,	1600000000, },
+	{ EIC7700_SPLL0_FOUT2,		"fixed_rate_clk_spll0_fout2",	NULL, 0,	800000000, },
+	{ EIC7700_SPLL0_FOUT3,		"fixed_rate_clk_spll0_fout3",	NULL, 0,	400000000, },
+	{ EIC7700_SPLL1_FOUT1,		"fixed_rate_clk_spll1_fout1",	NULL, 0,	1500000000, },
+	{ EIC7700_SPLL1_FOUT2,  	"fixed_rate_clk_spll1_fout2",	NULL, 0,	300000000, },
+	{ EIC7700_SPLL1_FOUT3,		"fixed_rate_clk_spll1_fout3",	NULL, 0,	250000000, },
+	{ EIC7700_SPLL2_FOUT1,		"fixed_rate_clk_spll2_fout1",	NULL, 0,	2080000000, },
+	{ EIC7700_SPLL2_FOUT2,		"fixed_rate_clk_spll2_fout2",	NULL, 0,	1040000000, },
+	{ EIC7700_SPLL2_FOUT3,		"fixed_rate_clk_spll2_fout3",	NULL, 0,	416000000, },
+	{ EIC7700_VPLL_FOUT1,		"fixed_rate_clk_vpll_fout1",	NULL, 0,	1188000000, },
+	{ EIC7700_VPLL_FOUT2,		"fixed_rate_clk_vpll_fout2",	NULL, 0,	594000000, },
+	{ EIC7700_VPLL_FOUT3,		"fixed_rate_clk_vpll_fout3",	NULL, 0,	49500000, },
+	{ EIC7700_APLL_FOUT2,		"fixed_rate_clk_apll_fout2",	NULL, 0,	0, },
+	{ EIC7700_APLL_FOUT3,		"fixed_rate_clk_apll_fout3",	NULL, 0,	0, },
+	{ EIC7700_EXT_MCLK,		"fixed_rate_ext_mclk",		NULL, 0,	0, },
+	{ EIC7700_LPDDR_REF_BAK,	"fixed_rate_lpddr_ref_bak",	NULL, 0,	50000000, },
+};
+
+static struct eswin_pll_clock eic7700_pll_clks[] = {
+	{
+		EIC7700_APLL_FOUT1, "clk_apll_fout1", NULL,
+		EIC7700_REG_OFFSET_APLL_CFG_0, 0, 1, 12,6, 20,12,
+		EIC7700_REG_OFFSET_APLL_CFG_1, 4,24,
+		EIC7700_REG_OFFSET_APLL_CFG_2, 1,3, 16, 3,
+		EIC7700_REG_OFFSET_PLL_STATUS, 4, 1,
+	},
+	{
+		EIC7700_PLL_CPU, "clk_pll_cpu", NULL,
+		EIC7700_REG_OFFSET_MCPUT_PLL_CFG_0, 0,1, 12,6, 20,12,
+		EIC7700_REG_OFFSET_MCPUT_PLL_CFG_1, 4, 24,
+		EIC7700_REG_OFFSET_MCPUT_PLL_CFG_2, 1,3, 16,3,
+		EIC7700_REG_OFFSET_PLL_STATUS, 5, 1,
+	},
+};
+
+/* fixed factor clocks */
+static struct eswin_fixed_factor_clock eic7700_fixed_factor_clks[] = {
+	{ EIC7700_FIXED_FACTOR_U_CPU_DIV2,	 "fixed_factor_u_cpu_div2",   "mux_u_cpu_root_3mux1_gfree", 1, 2, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_CLK_1M_DIV24,   "fixed_factor_u_clk_1m_div24",   "fixed_rate_clk_xtal_24m", 1, 24, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_MIPI_TXESC_DIV10, "fixed_factor_u_mipi_txesc_div10", "clk_clk_sys_cfg",  1, 10, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_U84_CORE_LP_DIV2,   "fixed_factor_u_u84_core_lp_div2",   "gate_clk_spll0_fout2", 1, 2, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_SCPU_BUS_DIV2,   "fixed_factor_u_scpu_bus_div2",   "mux_u_scpu_core_clk_2mux1_gfree", 1, 2, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_LPCPU_BUS_DIV2, "fixed_factor_u_lpcpu_bus_div2", "mux_u_lpcpu_core_clk_2mux1_gfree",  1, 2, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_PCIE_CR_DIV2,   "fixed_factor_u_pcie_cr_div2",   "clk_clk_sys_cfg", 1, 2, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_PCIE_AUX_DIV4,   "fixed_factor_u_pcie_aux_div4",   "clk_clk_sys_cfg", 1, 4, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_PVT_DIV20, "fixed_factor_u_pvt_div20", "fixed_rate_clk_xtal_24m",  1, 20, 0, },
+
+	{ EIC7700_FIXED_FACTOR_U_HSP_RMII_REF_DIV6, "fixed_factor_u_hsp_rmii_ref_div6", "fixed_rate_clk_spll1_fout2",  1, 6, 0, },
+};
+
+static struct eswin_mux_clock eic7700_mux_clks[] = {
+	{ EIC7700_MUX_U_CPU_ROOT_3MUX1_GFREE, "mux_u_cpu_root_3mux1_gfree", mux_u_cpu_root_3mux1_gfree_p,
+		ARRAY_SIZE(mux_u_cpu_root_3mux1_gfree_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		0, BIT_MASK(0) | BIT_MASK(1), 0, mux_u_cpu_root_3mux1_gfree_p_table, },
+
+	{ EIC7700_MUX_U_CPU_ACLK_2MUX1_GFREE, "mux_u_cpu_aclk_2mux1_gfree", mux_u_cpu_aclk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_cpu_aclk_2mux1_gfree_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_U84_CLK_CTRL,
+		20, 1, 0,},
+
+	{ EIC7700_MUX_U_DSP_ACLK_ROOT_2MUX1_GFREE, "mux_u_dsp_aclk_root_2mux1_gfree", dsp_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(dsp_aclk_root_2mux1_gfree_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_DSP_ACLK_CTRL,
+		0, 1, 0,},
+
+	{ EIC7700_MUX_U_D2D_ACLK_ROOT_2MUX1_GFREE, "mux_u_d2d_aclk_root_2mux1_gfree", d2d_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(d2d_aclk_root_2mux1_gfree_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_D2D_ACLK_CTRL,
+		0, 1, 0,},
+
+	{ EIC7700_MUX_U_DDR_ACLK_ROOT_2MUX1_GFREE, "mux_u_ddr_aclk_root_2mux1_gfree", ddr_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(ddr_aclk_root_2mux1_gfree_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_DDR_CLK_CTRL,
+		16, 1, 0,},
+
+	{ EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_0, "mux_u_mshcore_root_3mux1_0", mshcore_root_3mux1_0_mux_p,
+		ARRAY_SIZE(mshcore_root_3mux1_0_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL,
+		0, BIT_MASK(0) | BIT_MASK(20), 0, mshcore_root_3mux1_0_mux_p_table},
+
+	{ EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1, "mux_u_mshcore_root_3mux1_1", mshcore_root_3mux1_1_mux_p,
+		ARRAY_SIZE(mshcore_root_3mux1_1_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL,
+		0, BIT_MASK(0) | BIT_MASK(20), 0, mshcore_root_3mux1_1_mux_p_table, },
+
+	{ EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_2, "mux_u_mshcore_root_3mux1_2", mshcore_root_3mux1_2_mux_p,
+		ARRAY_SIZE(mshcore_root_3mux1_2_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL,
+		0, BIT_MASK(0) | BIT_MASK(20), 0, mshcore_root_3mux1_2_mux_p_table },
+
+	{ EIC7700_MUX_U_NPU_LLCLK_3MUX1_GFREE, "mux_u_npu_llclk_3mux1_gfree", npu_llclk_3mux1_gfree_mux_p,
+		ARRAY_SIZE(npu_llclk_3mux1_gfree_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_NPU_LLC_CTRL,
+		0, BIT_MASK(0) | BIT_MASK(1), 0, npu_llclk_3mux1_gfree_mux_p_table},
+
+	{ EIC7700_MUX_U_NPU_CORE_3MUX1_GFREE, "mux_u_npu_core_3mux1_gfree", npu_core_3mux1_gfree_mux_p,
+		ARRAY_SIZE(npu_core_3mux1_gfree_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_NPU_CORE_CTRL,
+		0, BIT_MASK(0) | BIT_MASK(1), 0, npu_core_3mux1_gfree_mux_p_table},
+
+	{ EIC7700_MUX_U_NPU_E31_3MUX1_GFREE, "mux_u_npu_e31_3mux1_gfree", npu_e31_3mux1_gfree_mux_p,
+		ARRAY_SIZE(npu_e31_3mux1_gfree_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_NPU_CORE_CTRL,
+		0, BIT_MASK(8) | BIT_MASK(9), 0, npu_e31_3mux1_gfree_mux_p_table},
+
+	{ EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE, "mux_u_vi_aclk_root_2mux1_gfree", vi_aclk_root_2mux1_gfree_mux_p,
+		ARRAY_SIZE(vi_aclk_root_2mux1_gfree_mux_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VI_ACLK_CTRL,
+		0, 1, 0,},
+
+	{ EIC7700_MUX_U_VI_DW_ROOT_2MUX1, "mux_u_vi_dw_root_2mux1", mux_u_vi_dw_root_2mux1_p,
+		ARRAY_SIZE(mux_u_vi_dw_root_2mux1_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VI_DWCLK_CTRL,
+		0, 1, 0,},
+
+	{ EIC7700_MUX_U_VI_DVP_ROOT_2MUX1_GFREE, "mux_u_vi_dvp_root_2mux1_gfree", mux_u_vi_dvp_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vi_dvp_root_2mux1_gfree_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_VI_DIG_ISP_ROOT_2MUX1_GFREE, "mux_u_vi_dig_isp_root_2mux1_gfree", mux_u_vi_dig_isp_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vi_dig_isp_root_2mux1_gfree_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_VO_ACLK_ROOT_2MUX1_GFREE, "mux_u_vo_aclk_root_2mux1_gfree", mux_u_vo_aclk_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vo_aclk_root_2mux1_gfree_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VO_ACLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_VO_PIXEL_ROOT_2MUX1, "mux_u_vo_pixel_root_2mux1", mux_u_vo_pixel_root_2mux1_p,
+		ARRAY_SIZE(mux_u_vo_pixel_root_2mux1_p), CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VO_PIXEL_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_VCDEC_ROOT_2MUX1_GFREE,  "mux_u_vcdec_root_2mux1_gfree",  mux_u_vcdec_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vcdec_root_2mux1_gfree_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VCDEC_ROOTCLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_VCACLK_ROOT_2MUX1_GFREE,  "mux_u_vcaclk_root_2mux1_gfree",  mux_u_vcaclk_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_vcaclk_root_2mux1_gfree_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VC_ACLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_SYSCFG_CLK_ROOT_2MUX1_GFREE,  "mux_u_syscfg_clk_root_2mux1_gfree",  mux_u_syscfg_clk_root_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_syscfg_clk_root_2mux1_gfree_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_SYSCFG_CLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_BOOTSPI_CLK_2MUX1_GFREE,  "mux_u_bootspi_clk_2mux1_gfree",  mux_u_bootspi_clk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_bootspi_clk_2mux1_gfree_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_SCPU_CORE_CLK_2MUX1_GFREE,  "mux_u_scpu_core_clk_2mux1_gfree",  mux_u_scpu_core_clk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_scpu_core_clk_2mux1_gfree_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_LPCPU_CORE_CLK_2MUX1_GFREE,  "mux_u_lpcpu_core_clk_2mux1_gfree",  mux_u_lpcpu_core_clk_2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_lpcpu_core_clk_2mux1_gfree_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_VO_MCLK_2MUX_EXT_MCLK,  "mux_u_vo_mclk_2mux_ext_mclk",  mux_u_vo_mclk_2mux_ext_mclk_p,
+		ARRAY_SIZE(mux_u_vo_mclk_2mux_ext_mclk_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_VO_MCLK_CTRL,
+		0, 1, 0, },
+
+	{ EIC7700_MUX_U_AONDMA_AXI2MUX1_GFREE,  "mux_u_aondma_axi2mux1_gfree",  mux_u_aondma_axi2mux1_gfree_p,
+		ARRAY_SIZE(mux_u_aondma_axi2mux1_gfree_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL,
+		20, 1, 0, },
+
+	{ EIC7700_MUX_U_RMII_REF_2MUX,  "mux_u_rmii_ref_2mux1",  mux_u_rmii_ref_2mux1_p,
+		ARRAY_SIZE(mux_u_rmii_ref_2mux1_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_ETH0_CTRL,
+		2, 1, 0, },
+
+	{ EIC7700_MUX_U_ETH_CORE_2MUX1,  "mux_u_eth_core_2mux1",  mux_u_eth_core_2mux1_p,
+		ARRAY_SIZE(mux_u_eth_core_2mux1_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_ETH0_CTRL,
+		1, 1, 0, },
+
+	{ EIC7700_MUX_U_SATA_PHY_2MUX1,  "mux_u_sata_phy_2mux1",  mux_u_sata_phy_2mux1_p,
+		ARRAY_SIZE(mux_u_sata_phy_2mux1_p),  CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_SATA_OOB_CTRL,
+		9, 1, 0, },
+
+};
+
+/*The hardware decides vaule 0, 1 and 2 both means 2 divsor, so we have to add these ugly tables*/
+static struct clk_div_table u_3_bit_special_div_table[8];
+static struct clk_div_table u_4_bit_special_div_table[16];
+static struct clk_div_table u_6_bit_special_div_table[64];
+static struct clk_div_table u_7_bit_special_div_table[128];
+static struct clk_div_table u_8_bit_special_div_table[256];
+static struct clk_div_table u_11_bit_special_div_table[2048];
+static struct clk_div_table u_16_bit_special_div_table[65536];
+
+static struct eswin_divider_clock eic7700_div_clks[] = {
+	{ EIC7700_DIVDER_U_SYS_CFG_DIV_DYNM, "divder_u_sys_cfg_div_dynm",   "fixed_rate_clk_spll0_fout3", 0,
+		EIC7700_REG_OFFSET_SYSCFG_CLK_CTRL, 4, 3, CLK_DIVIDER_ROUND_CLOSEST, u_3_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_NOC_NSP_DIV_DYNM,   "divder_u_noc_nsp_div_dynm", "fixed_rate_clk_spll2_fout1", 0,
+		EIC7700_REG_OFFSET_NOC_CLK_CTRL, 0, 3, CLK_DIVIDER_ROUND_CLOSEST, u_3_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_BOOTSPI_DIV_DYNM,       "divder_u_bootspi_div_dynm",     "gate_clk_spll0_fout2", 0,
+		EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL, 4, 6, CLK_DIVIDER_ROUND_CLOSEST, u_6_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_SCPU_CORE_DIV_DYNM,     "divder_u_scpu_core_div_dynm",   "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_LPCPU_CORE_DIV_DYNM,     "divder_u_lpcpu_core_div_dynm",   "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_GPU_ACLK_DIV_DYNM,     "divder_u_gpu_aclk_div_dynm",   "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_GPU_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_DSP_ACLK_DIV_DYNM,     "divder_u_dsp_aclk_div_dynm",   "clk_clk_dsp_root", 0,
+		EIC7700_REG_OFFSET_DSP_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_D2D_ACLK_DIV_DYNM, "divder_u_d2d_aclk_div_dynm",   "mux_u_d2d_aclk_root_2mux1_gfree", 0,
+		EIC7700_REG_OFFSET_D2D_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_DDR_ACLK_DIV_DYNM, "divder_u_ddr_aclk_div_dynm",   "mux_u_ddr_aclk_root_2mux1_gfree", 0,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL, 20, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_HSP_ACLK_DIV_DYNM,   "divder_u_hsp_aclk_div_dynm", "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_HSP_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_ETH_TXCLK_DIV_DYNM_0,     "divder_u_eth_txclk_div_dynm_0",   "mux_u_eth_core_2mux1", 0,
+		EIC7700_REG_OFFSET_ETH0_CTRL, 4, 7, CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_ETH_TXCLK_DIV_DYNM_1,     "divder_u_eth_txclk_div_dynm_1",   "mux_u_eth_core_2mux1", 0,
+		EIC7700_REG_OFFSET_ETH1_CTRL, 4, 7, CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_0,     "divder_u_mshc_core_div_dynm_0",   "mux_u_mshcore_root_3mux1_0", 0,
+		EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL, 4, 12, CLK_DIVIDER_ONE_BASED},
+
+	{ EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_1,     "divder_u_mshc_core_div_dynm_1",   "mux_u_mshcore_root_3mux1_1", 0,
+		EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL, 4, 12, CLK_DIVIDER_ONE_BASED},
+
+	{ EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_2,     "divder_u_mshc_core_div_dynm_2",   "mux_u_mshcore_root_3mux1_2",  0,
+		EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL, 4, 12, CLK_DIVIDER_ONE_BASED},
+
+	{ EIC7700_DIVDER_U_PCIE_ACLK_DIV_DYNM,   "divder_u_pcie_aclk_div_dynm", "fixed_rate_clk_spll2_fout2", 0,
+		EIC7700_REG_OFFSET_PCIE_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_NPU_ACLK_DIV_DYNM,        "divder_u_npu_aclk_div_dynm", "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_NPU_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_NPU_LLC_SRC0_DIV_DYNM,    "divder_u_npu_llc_src0_div_dynm",  "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_NPU_LLC_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_NPU_LLC_SRC1_DIV_DYNM,    "divder_u_npu_llc_src1_div_dynm",   "fixed_rate_clk_spll2_fout1", 0,
+		EIC7700_REG_OFFSET_NPU_LLC_CTRL, 8, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_NPU_CORECLK_DIV_DYNM,     "divder_u_npu_coreclk_div_dynm",   "mux_u_npu_core_3mux1_gfree", 0,
+		EIC7700_REG_OFFSET_NPU_CORE_CTRL, 4, 4, CLK_DIVIDER_ONE_BASED},
+
+	{ EIC7700_DIVDER_U_NPU_E31_DIV_DYNM,         "divder_u_npu_e31_div_dynm",   "mux_u_npu_e31_3mux1_gfree", 0,
+		EIC7700_REG_OFFSET_NPU_CORE_CTRL, 12, 4, CLK_DIVIDER_ONE_BASED},
+
+	{ EIC7700_DIVDER_U_VI_ACLK_DIV_DYNM,          "divder_u_vi_aclk_div_dynm",   "mux_u_vi_aclk_root_2mux1_gfree", 0,
+		EIC7700_REG_OFFSET_VI_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_DW_DIV_DYNM,            "divder_u_vi_dw_div_dynm",   "mux_u_vi_dw_root_2mux1", 0,
+		EIC7700_REG_OFFSET_VI_DWCLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_DVP_DIV_DYNM,        "divder_u_vi_dvp_div_dynm",   "mux_u_vi_dig_root_2mux1_gfree", 0,
+		EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_DIG_ISP_DIV_DYNM,       "divder_u_vi_dig_isp_div_dynm", "mux_u_vi_dig_isp_root_2mux1_gfree", 0,
+		EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_0,     "divder_u_vi_shutter_div_dynm_0",   "fixed_rate_clk_vpll_fout2",0,
+		EIC7700_REG_OFFSET_VI_SHUTTER0, 4, 7, CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_1,     "divder_u_vi_shutter_div_dynm_1",   "fixed_rate_clk_vpll_fout2", 0,
+		EIC7700_REG_OFFSET_VI_SHUTTER1, 4, 7,  CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_2,     "divder_u_vi_shutter_div_dynm_2",   "fixed_rate_clk_vpll_fout2", 0,
+		EIC7700_REG_OFFSET_VI_SHUTTER2, 4, 7, CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_3,     "divder_u_vi_shutter_div_dynm_3",   "fixed_rate_clk_vpll_fout2", 0,
+		EIC7700_REG_OFFSET_VI_SHUTTER3, 4, 7, CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_4,     "divder_u_vi_shutter_div_dynm_4",   "fixed_rate_clk_vpll_fout2", 0,
+		EIC7700_REG_OFFSET_VI_SHUTTER4, 4, 7, CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_5,     "divder_u_vi_shutter_div_dynm_5",   "fixed_rate_clk_vpll_fout2", 0,
+		EIC7700_REG_OFFSET_VI_SHUTTER5, 4, 7, CLK_DIVIDER_ROUND_CLOSEST, u_7_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VO_ACLK_DIV_DYNM,         "divder_u_vo_aclk_div_dynm", "mux_u_vo_aclk_root_2mux1_gfree", 0,
+		EIC7700_REG_OFFSET_VO_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_IESMCLK_DIV_DYNM,       "divder_u_iesmclk_div_dynm",     "fixed_rate_clk_spll0_fout3", 0,
+		EIC7700_REG_OFFSET_VO_IESMCLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VO_PIXEL_DIV_DYNM,	 "divder_u_vo_pixel_div_dynm",	"mux_u_vo_pixel_root_2mux1", 0,
+		EIC7700_REG_OFFSET_VO_PIXEL_CTRL, 4, 6, CLK_DIVIDER_ROUND_CLOSEST, u_6_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VO_MCLK_DIV_DYNM,     "divder_u_vo_mclk_div_dynm",   "clk_apll_fout1", 0,
+		EIC7700_REG_OFFSET_VO_MCLK_CTRL, 4, 8, CLK_DIVIDER_ROUND_CLOSEST, u_8_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VO_CEC_DIV_DYNM,     "divder_u_vo_cec_div_dynm",   "fixed_rate_clk_vpll_fout2", 0,
+		EIC7700_REG_OFFSET_VO_PHY_CLKCTRL, 16, 16, CLK_DIVIDER_ROUND_CLOSEST, u_16_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VC_ACLK_DIV_DYNM,     "divder_u_vc_aclk_div_dynm",   "mux_u_vcaclk_root_2mux1_gfree", 0,
+		EIC7700_REG_OFFSET_VC_ACLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_JD_DIV_DYNM,     "divder_u_jd_div_dynm",   "clk_clk_vc_root", 0,
+		EIC7700_REG_OFFSET_JD_CLK_CTRL, 4, 4,  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_JE_DIV_DYNM,     "divder_u_je_div_dynm",   "clk_clk_vc_root", 0,
+		EIC7700_REG_OFFSET_JE_CLK_CTRL, 4, 4,  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VE_DIV_DYNM,     "divder_u_ve_div_dynm",   "clk_clk_vc_root", 0,
+		EIC7700_REG_OFFSET_VE_CLK_CTRL, 4, 4,  CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_VD_DIV_DYNM,     "divder_u_vd_div_dynm",    "clk_clk_vc_root", 0,
+		EIC7700_REG_OFFSET_VD_CLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_G2D_DIV_DYNM,       "divder_u_g2d_div_dynm",     "clk_clk_dsp_root", 0,
+		EIC7700_REG_OFFSET_G2D_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_AONDMA_AXI_DIV_DYNM,     "divder_u_aondma_axi_div_dynm",   "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_CRYPTO_DIV_DYNM,     "divder_u_crypto_div_dynm",   "fixed_rate_clk_spll0_fout1", 0,
+		EIC7700_REG_OFFSET_SPACC_CLK_CTRL, 4, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_SATA_PHY_REF_DIV_DYNM,     "divder_u_sata_phy_ref_div_dynm",   "fixed_rate_clk_spll1_fout2", 0,
+		EIC7700_REG_OFFSET_SATA_OOB_CTRL, 0, 4, CLK_DIVIDER_ROUND_CLOSEST, u_4_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_DSP_0_ACLK_DIV_DYNM,     "divder_u_dsp_0_aclk_div_dynm",   "gate_dspt_aclk", 0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL, 19, 1, },
+
+	{ EIC7700_DIVDER_U_DSP_1_ACLK_DIV_DYNM,     "divder_u_dsp_1_aclk_div_dynm",   "gate_dspt_aclk", 0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL, 20, 1, },
+
+	{ EIC7700_DIVDER_U_DSP_2_ACLK_DIV_DYNM,     "divder_u_dsp_2_aclk_div_dynm",   "gate_dspt_aclk", 0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL, 21, 1, },
+
+	{ EIC7700_DIVDER_U_DSP_3_ACLK_DIV_DYNM,     "divder_u_dsp_3_aclk_div_dynm",   "gate_dspt_aclk", 0,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL, 22, 1, },
+
+	{ EIC7700_DIVDER_U_AON_RTC_DIV_DYNM,     "divder_u_aon_rtc_div_dynm",   "clk_clk_1m", 0,
+		EIC7700_REG_OFFSET_RTC_CLK_CTRL, 21, 11, CLK_DIVIDER_ROUND_CLOSEST, u_11_bit_special_div_table},
+
+	{ EIC7700_DIVDER_U_U84_RTC_TOGGLE_DIV_DYNM,     "divder_u_u84_rtc_toggle_dynm",   "fixed_rate_clk_xtal_24m", 0,
+		EIC7700_REG_OFFSET_RTC_CLK_CTRL, 16, 5, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ROUND_CLOSEST},
+
+};
+
+/*
+	these clks should init early to cost down the whole clk module init time
+*/
+static struct eswin_clock eic7700_clks_early_0[] = {
+	{ EIC7700_CLK_CLK_DSP_ROOT , "clk_clk_dsp_root", "mux_u_dsp_aclk_root_2mux1_gfree", CLK_SET_RATE_PARENT,},
+	{ EIC7700_CLK_CLK_VC_ROOT, "clk_clk_vc_root", "mux_u_vcdec_root_2mux1_gfree", CLK_SET_RATE_PARENT,},
+};
+
+static struct eswin_clock eic7700_clks_early_1[] = {
+	{ EIC7700_CLK_CLK_SYS_CFG	,"clk_clk_sys_cfg", "mux_u_syscfg_clk_root_2mux1_gfree", CLK_SET_RATE_PARENT,},
+	{ EIC7700_CLK_CLK_D2DDR_ACLK	,"clk_clk_d2ddr_aclk", "divder_u_ddr_aclk_div_dynm", CLK_SET_RATE_PARENT,},
+	{ EIC7700_CLK_CLK_AONDMA_AXI_ST3,"clk_clk_aondma_axi_st3","mux_u_aondma_axi2mux1_gfree", CLK_SET_RATE_PARENT,},
+	{ EIC7700_CLK_CLK_G2D_ST2	,"clk_clk_g2d_st2",	"divder_u_g2d_div_dynm", CLK_SET_RATE_PARENT,},
+	{ EIC7700_CLK_CLK_MIPI_TXESC	,"clk_clk_mipi_txesc",	"fixed_factor_u_mipi_txesc_div10", CLK_SET_RATE_PARENT,},
+	{ EIC7700_CLK_CLK_VI_ACLK_ST1	,"clk_clk_vi_aclk_st1",	"divder_u_vi_aclk_div_dynm", CLK_SET_RATE_PARENT,},
+};
+
+static struct eswin_gate_clock eic7700_gate_clks[] = {
+
+	{ EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_0 ,	"gate_clk_cpu_ext_src_core_clk_0", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 28, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_1 ,	"gate_clk_cpu_ext_src_core_clk_1", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 29, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_2 ,	"gate_clk_cpu_ext_src_core_clk_2", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_3 ,	"gate_clk_cpu_ext_src_core_clk_3", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_TRACE_CLK_0 ,		"gate_clk_cpu_trace_clk_0", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 24, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_TRACE_CLK_1 ,		"gate_clk_cpu_trace_clk_1", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 25, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_TRACE_CLK_2 ,		"gate_clk_cpu_trace_clk_2", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 26, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_TRACE_CLK_3 ,		"gate_clk_cpu_trace_clk_3", "mux_u_cpu_root_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 27, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_TRACE_COM_CLK ,		"gate_clk_cpu_trace_com_clk", "mux_u_cpu_aclk_2mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 23, 0, },
+
+	{ EIC7700_GATE_CLK_CPU_CLK ,			"gate_clk_cpu_clk", "mux_u_cpu_aclk_2mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_U84_CLK_CTRL, 28, 0, }, /*same as EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_0 */
+
+	{ EIC7700_GATE_CLK_SPLL0_FOUT2 ,		"gate_clk_spll0_fout2", "fixed_rate_clk_spll0_fout2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SPLL0_CFG_2, 31, 0, },
+
+	{ EIC7700_GATE_NOC_NSP_CLK ,			"gate_noc_nsp_clk", "divder_u_noc_nsp_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NOC_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_BOOTSPI ,			"gate_clk_bootspi", "mux_u_bootspi_clk_2mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_BOOTSPI_CFG	,		"gate_clk_bootspi_cfg",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_BOOTSPI_CFGCLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_SCPU_CORE	,		"gate_clk_scpu_core",   "mux_u_scpu_core_clk_2mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_SCPU_BUS 			,"gate_clk_scpu_bus",   "fixed_factor_u_scpu_bus_div2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SCPU_BUSCLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_LPCPU_CORE			,"gate_clk_lpcpu_core",   "mux_u_lpcpu_core_clk_2mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_LPCPU_BUS			,"gate_clk_lpcpu_bus",   "fixed_factor_u_lpcpu_bus_div2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LPCPU_BUSCLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_GPU_ACLK 			,"gate_gpu_aclk",   "divder_u_gpu_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_GPU_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_GPU_GRAY_CLK 			,"gate_gpu_gray_clk",   "fixed_rate_clk_xtal_24m", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_GPU_GRAY_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_GPU_CFG_CLK			,"gate_gpu_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_GPU_CFG_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_DSPT_ACLK			,"gate_dspt_aclk",   "divder_u_dsp_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DSP_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_DSPT_CFG_CLK 			,"gate_dspt_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DSP_CFG_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_D2D_ACLK 			,"gate_d2d_aclk",   "divder_u_d2d_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_D2D_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_D2D_CFG_CLK			,"gate_d2d_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_D2D_CFG_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_TCU_ACLK 			,"gate_tcu_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TCU_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_TCU_CFG_CLK			,"gate_tcu_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TCU_CFG_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_DDRT_CFG_CLK			,"gate_ddrt_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL, 9, 0, },
+
+	{ EIC7700_GATE_DDRT0_P0_ACLK			,"gate_ddrt0_p0_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL, 4, 0, },
+
+	{ EIC7700_GATE_DDRT0_P1_ACLK			,"gate_ddrt0_p1_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL, 5, 0, },
+
+	{ EIC7700_GATE_DDRT0_P2_ACLK			,"gate_ddrt0_p2_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL, 6, 0, },
+
+	{ EIC7700_GATE_DDRT0_P3_ACLK			,"gate_ddrt0_p3_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL, 7, 0, },
+
+	{ EIC7700_GATE_DDRT0_P4_ACLK			,"gate_ddrt0_p4_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR_CLK_CTRL, 8, 0, },
+
+	{ EIC7700_GATE_DDRT1_P0_ACLK			,"gate_ddrt1_p0_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL, 4, 0, },
+
+	{ EIC7700_GATE_DDRT1_P1_ACLK			,"gate_ddrt1_p1_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL, 5, 0, },
+
+	{ EIC7700_GATE_DDRT1_P2_ACLK			,"gate_ddrt1_p2_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL, 6, 0, },
+
+	{ EIC7700_GATE_DDRT1_P3_ACLK			,"gate_ddrt1_p3_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL, 7, 0, },
+
+	{ EIC7700_GATE_DDRT1_P4_ACLK			,"gate_ddrt1_p4_aclk",   "clk_clk_d2ddr_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_DDR1_CLK_CTRL, 8, 0, },
+
+	{ EIC7700_GATE_CLK_HSP_ACLK			,"gate_clk_hsp_aclk",   "divder_u_hsp_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_HSP_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_HSP_CFGCLK			,"gate_clk_hsp_cfgclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_HSP_CFG_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_PCIET_ACLK			,"gate_pciet_aclk",   "divder_u_pcie_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_PCIET_CFG_CLK			,"gate_pciet_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_CFG_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_PCIET_CR_CLK 			,"gate_pciet_cr_clk",   "fixed_factor_u_pcie_cr_div2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_CFG_CTRL, 0, 0, },
+
+	{ EIC7700_GATE_PCIET_AUX_CLK			,"gate_pciet_aux_clk",   "fixed_factor_u_pcie_aux_div4", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PCIE_CFG_CTRL, 1, 0, },
+
+	{ EIC7700_GATE_NPU_ACLK 			,"gate_npu_aclk",   "divder_u_npu_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_NPU_CFG_CLK			,"gate_npu_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_ACLK_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_NPU_LLC_ACLK 			,"gate_npu_llc_aclk",   "mux_u_npu_llclk_3mux1_gfree", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_LLC_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_NPU_CLK				,"gate_npu_clk",   "divder_u_npu_coreclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_CORE_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_NPU_E31_CLK			,"gate_npu_e31_clk",   "divder_u_npu_e31_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_NPU_CORE_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_VI_ACLK				,"gate_vi_aclk",   "clk_clk_vi_aclk_st1", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VI_CFG_CLK			,"gate_vi_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_ACLK_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_VI_DIG_DW_CLK			,"gate_vi_dig_dw_clk",   "divder_u_vi_dw_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DWCLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VI_DVP_CLK			,"gate_vi_dvp_clk",   "divder_u_vi_dvp_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VI_DIG_ISP_CLK			,"gate_vi_dig_isp_clk",   "divder_u_vi_dig_isp_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VI_SHUTTER_0			,"gate_vi_shutter_0",   "divder_u_vi_shutter_div_dynm_0", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER0, 31, 0, },
+
+	{ EIC7700_GATE_VI_SHUTTER_1			,"gate_vi_shutter_1",   "divder_u_vi_shutter_div_dynm_1", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER1, 31, 0, },
+
+	{ EIC7700_GATE_VI_SHUTTER_2			,"gate_vi_shutter_2",   "divder_u_vi_shutter_div_dynm_2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER2, 31, 0, },
+
+	{ EIC7700_GATE_VI_SHUTTER_3			,"gate_vi_shutter_3",   "divder_u_vi_shutter_div_dynm_3", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER3, 31, 0, },
+
+	{ EIC7700_GATE_VI_SHUTTER_4			,"gate_vi_shutter_4",   "divder_u_vi_shutter_div_dynm_4", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER4, 31, 0, },
+
+	{ EIC7700_GATE_VI_SHUTTER_5			,"gate_vi_shutter_5",   "divder_u_vi_shutter_div_dynm_5", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_SHUTTER5, 31, 0, },
+
+	{ EIC7700_GATE_VI_PHY_TXCLKESC			,"gate_vi_phy_txclkesc",   "clk_clk_mipi_txesc", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_PHY_CLKCTRL, 0, 0, },
+
+	{ EIC7700_GATE_VI_PHY_CFG			,"gate_vi_phy_cfg",   "fixed_rate_clk_xtal_24m", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VI_PHY_CLKCTRL, 1, 0, },
+
+	{ EIC7700_GATE_VO_ACLK				,"gate_vo_aclk",   "divder_u_vo_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VO_CFG_CLK			,"gate_vo_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_ACLK_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_VO_HDMI_IESMCLK			,"gate_vo_hdmi_iesmclk",   "divder_u_iesmclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_IESMCLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VO_PIXEL_CLK 			,"gate_vo_pixel_clk",   "divder_u_vo_pixel_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_PIXEL_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VO_I2S_MCLK			,"gate_vo_i2s_mclk",   "mux_u_vo_mclk_2mux_ext_mclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_MCLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VO_CR_CLK			,"gate_vo_cr_clk",   "clk_clk_mipi_txesc", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VO_PHY_CLKCTRL, 1, 0, },
+
+	{ EIC7700_GATE_VC_ACLK				,"gate_vc_aclk",   "divder_u_vc_aclk_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_ACLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VC_CFG_CLK			,"gate_vc_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL, 0, 0, },
+
+	{ EIC7700_GATE_VC_JE_CLK			,"gate_vc_je_clk",   "divder_u_je_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_JE_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VC_JD_CLK			,"gate_vc_jd_clk",   "divder_u_jd_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_JD_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VC_VE_CLK			,"gate_vc_ve_clk",   "divder_u_ve_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VE_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_VC_VD_CLK			,"gate_vc_vd_clk",   "divder_u_vd_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VD_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_G2D_CFG_CLK			,"gate_g2d_cfg_clk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_G2D_CTRL, 28, 0, },
+
+	{ EIC7700_GATE_G2D_CLK				,"gate_g2d_clk",   "clk_clk_g2d_st2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_G2D_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_G2D_ACLK 			,"gate_g2d_aclk",   "clk_clk_g2d_st2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_G2D_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_AONDMA_CFG			,"gate_clk_aondma_cfg",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_AONDMA_ACLK			,"gate_aondma_aclk",   "clk_clk_aondma_axi_st3", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_AON_ACLK 			,"gate_aon_aclk",   "clk_clk_aondma_axi_st3", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL, 29, 0, },
+
+	{ EIC7700_GATE_TIMER_CLK_0 			,"gate_time_clk_0",   "fixed_rate_clk_xtal_24m", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 0, 0, },
+
+	{ EIC7700_GATE_TIMER_CLK_1 			,"gate_time_clk_1",   "fixed_rate_clk_xtal_24m", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 1, 0, },
+
+	{ EIC7700_GATE_TIMER_CLK_2 			,"gate_time_clk_2",   "fixed_rate_clk_xtal_24m", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 2, 0, },
+
+	{ EIC7700_GATE_TIMER_CLK_3 			,"gate_time_clk_3",   "fixed_rate_clk_xtal_24m", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 3, 0, },
+
+	{ EIC7700_GATE_TIMER_PCLK_0			,"gate_timer_pclk_0",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 4, 0, },
+
+	{ EIC7700_GATE_TIMER_PCLK_1			,"gate_timer_pclk_1",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 5, 0, },
+
+	{ EIC7700_GATE_TIMER_PCLK_2			,"gate_timer_pclk_2",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 6, 0, },
+
+	{ EIC7700_GATE_TIMER_PCLK_3			,"gate_timer_pclk_3",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 7, 0, },
+
+	{ EIC7700_GATE_TIMER3_CLK8			,"gate_timer3_clk8",   "fixed_rate_clk_vpll_fout3", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TIMER_CLK_CTRL, 8, 0, },
+
+	{ EIC7700_GATE_CLK_RTC_CFG			,"gate_clk_rtc_cfg",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_RTC_CLK_CTRL, 2, 0, },
+
+	{ EIC7700_GATE_CLK_RTC				,"gate_clk_rtc",   "divder_u_aon_rtc_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_RTC_CLK_CTRL, 1, 0, },
+
+	{ EIC7700_GATE_CLK_PKA_CFG			,"gate_clk_pka_cfg",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_PKA_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_SPACC_CFG			,"gate_clk_spacc_cfg",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SPACC_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_CRYPTO			,"gate_clk_crypto",   "divder_u_crypto_div_dynm", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SPACC_CLK_CTRL, 30, 0, },
+
+	{ EIC7700_GATE_CLK_TRNG_CFG 			,"gate_clk_trng_cfg",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_TRNG_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_OTP_CFG			,"gate_clk_otp_cfg",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_OTP_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_0			,"gate_clk_mailbox_0",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 0, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_1			,"gate_clk_mailbox_1",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 1, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_2			,"gate_clk_mailbox_2",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 2, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_3			,"gate_clk_mailbox_3",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 3, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_4			,"gate_clk_mailbox_4",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 4, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_5			,"gate_clk_mailbox_5",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 5, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_6			,"gate_clk_mailbox_6",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 6, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_7			,"gate_clk_mailbox_7",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 7, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_8			,"gate_clk_mailbox_8",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 8, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_9			,"gate_clk_mailbox_9",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 9, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_10			,"gate_clk_mailbox_10",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 10, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_11			,"gate_clk_mailbox_11",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 11, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_12			,"gate_clk_mailbox_12",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 12, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_13			,"gate_clk_mailbox_13",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 13, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_14			,"gate_clk_mailbox_14",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 14, 0, },
+
+	{ EIC7700_GATE_CLK_MAILBOX_15			,"gate_clk_mailbox_15",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 15, 0, },
+
+	{ EIC7700_GATE_LSP_I2C0_PCLK			,"gate_i2c0_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 7, 0, },
+
+	{ EIC7700_GATE_LSP_I2C1_PCLK			,"gate_i2c1_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 8, 0, },
+
+	{ EIC7700_GATE_LSP_I2C2_PCLK			,"gate_i2c2_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 9, 0, },
+
+	{ EIC7700_GATE_LSP_I2C3_PCLK			,"gate_i2c3_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 10, 0, },
+
+	{ EIC7700_GATE_LSP_I2C4_PCLK			,"gate_i2c4_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 11, 0, },
+
+	{ EIC7700_GATE_LSP_I2C5_PCLK			,"gate_i2c5_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 12, 0, },
+
+	{ EIC7700_GATE_LSP_I2C6_PCLK			,"gate_i2c6_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 13, 0, },
+
+	{ EIC7700_GATE_LSP_I2C7_PCLK			,"gate_i2c7_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 14, 0, },
+
+	{ EIC7700_GATE_LSP_I2C8_PCLK			,"gate_i2c8_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 15, 0, },
+
+	{ EIC7700_GATE_LSP_I2C9_PCLK			,"gate_i2c9_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 16, 0, },
+
+	{ EIC7700_GATE_LSP_WDT0_PCLK			,"gate_lsp_wdt0_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 28, 0, },
+
+	{ EIC7700_GATE_LSP_WDT1_PCLK			,"gate_lsp_wdt1_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 29, 0, },
+
+	{ EIC7700_GATE_LSP_WDT2_PCLK			,"gate_lsp_wdt2_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 30, 0, },
+
+	{ EIC7700_GATE_LSP_WDT3_PCLK			,"gate_lsp_wdt3_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 31, 0, },
+
+	{ EIC7700_GATE_LSP_SSI0_PCLK			,"gate_lsp_ssi0_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 26, 0, },
+
+	{ EIC7700_GATE_LSP_SSI1_PCLK			,"gate_lsp_ssi1_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 27, 0, },
+
+	{ EIC7700_GATE_LSP_UART0_PCLK			,"gate_lsp_uart0_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 17, 0, },
+
+	{ EIC7700_GATE_LSP_UART1_PCLK			,"gate_lsp_uart1_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 18, 0, },
+
+	{ EIC7700_GATE_LSP_UART2_PCLK			,"gate_lsp_uart2_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 19, 0, },
+
+	{ EIC7700_GATE_LSP_UART3_PCLK			,"gate_lsp_uart3_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 20, 0, },
+
+	{ EIC7700_GATE_LSP_UART4_PCLK			,"gate_lsp_uart4_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 21, 0, },
+
+	{ EIC7700_GATE_LSP_TIMER_PCLK			,"gate_lsp_timer_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 25, 0, },
+
+	{ EIC7700_GATE_LSP_FAN_PCLK			,"gate_lsp_fan_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 0, 0, },
+
+	{ EIC7700_GATE_LSP_PVT_PCLK			,"gate_lsp_pvt_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN0, 1, 0, },
+
+	{ EIC7700_GATE_LSP_PVT0_CLK			,"gate_pvt0_clk",   "fixed_factor_u_pvt_div20", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 16, 0, },
+
+	{ EIC7700_GATE_LSP_PVT1_CLK			,"gate_pvt1_clk",   "fixed_factor_u_pvt_div20", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_LSP_CLK_EN1, 17, 0, },
+
+	{ EIC7700_GATE_VC_JE_PCLK			,"gate_vc_je_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL, 2, 0, },
+
+	{ EIC7700_GATE_VC_JD_PCLK			,"gate_vc_jd_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL, 1, 0, },
+
+	{ EIC7700_GATE_VC_VE_PCLK			,"gate_vc_ve_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL, 5, 0, },
+
+	{ EIC7700_GATE_VC_VD_PCLK			,"gate_vc_vd_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL, 4, 0, },
+
+	{ EIC7700_GATE_VC_MON_PCLK			,"gate_vc_mon_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_VC_CLKEN_CTRL, 3, 0, },
+
+	{ EIC7700_GATE_HSP_MSHC0_CORE_CLK		,"gate_hsp_mshc0_core_clk",   "divder_u_mshc_core_div_dynm_0", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL, 16, 0, },
+
+	{ EIC7700_GATE_HSP_MSHC1_CORE_CLK		,"gate_hsp_mshc1_core_clk",   "divder_u_mshc_core_div_dynm_1", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL, 16, 0, },
+
+	{ EIC7700_GATE_HSP_MSHC2_CORE_CLK		,"gate_hsp_mshc2_core_clk",   "divder_u_mshc_core_div_dynm_2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL, 16, 0, },
+
+	{ EIC7700_GATE_HSP_SATA_RBC_CLK			,"gate_hsp_sata_rbc_clk",   "fixed_rate_clk_spll1_fout2", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SATA_RBC_CTRL, 0, 0, },
+
+	{ EIC7700_GATE_HSP_SATA_OOB_CLK			,"gate_hsp_sata_oob_clk",   "mux_u_sata_phy_2mux1", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_SATA_OOB_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_HSP_DMA0_CLK			,"gate_hsp_dma0_clk",   "gate_clk_hsp_aclk", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_HSP_ACLK_CTRL, 0, 0, },
+
+	{ EIC7700_GATE_HSP_ETH0_CORE_CLK		,"gate_hsp_eth0_core_clk",   "divder_u_eth_txclk_div_dynm_0", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH0_CTRL, 0, 0, },
+
+	{ EIC7700_GATE_HSP_ETH1_CORE_CLK		,"gate_hsp_eth1_core_clk",   "divder_u_eth_txclk_div_dynm_1", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH1_CTRL, 0, 0, },
+
+	{ EIC7700_GATE_HSP_RMII_REF_0			,"gate_hsp_rmii_ref_0",   "mux_u_rmii_ref_2mux1", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH0_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_HSP_RMII_REF_1			,"gate_hsp_rmii_ref_1",   "mux_u_rmii_ref_2mux1", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_ETH1_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_AON_I2C0_PCLK			,"gate_aon_i2c0_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_I2C0_CLK_CTRL, 31, 0, },
+
+	{ EIC7700_GATE_AON_I2C1_PCLK			,"gate_aon_i2c1_pclk",   "clk_clk_sys_cfg", CLK_SET_RATE_PARENT,
+		EIC7700_REG_OFFSET_I2C1_CLK_CTRL, 31, 0, },
+};
+
+/* eic7700 clocks */
+static struct eswin_clock eic7700_clks[] = {
+	{  EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_0	,"clk_cpu_ext_src_core_clk_0",	"gate_clk_cpu_ext_src_core_clk_0", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_1	,"clk_cpu_ext_src_core_clk_1",	"gate_clk_cpu_ext_src_core_clk_1", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_2	,"clk_cpu_ext_src_core_clk_2",	"gate_clk_cpu_ext_src_core_clk_2", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_3	,"clk_cpu_ext_src_core_clk_3",	"gate_clk_cpu_ext_src_core_clk_3", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_TRACE_CLK_0		,"clk_cpu_trace_clk_0",	 	"gate_clk_cpu_trace_clk_0", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_TRACE_CLK_1		,"clk_cpu_trace_clk_1",	 	"gate_clk_cpu_trace_clk_1", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_TRACE_CLK_2		,"clk_cpu_trace_clk_2",	 	"gate_clk_cpu_trace_clk_2", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_TRACE_CLK_3		,"clk_cpu_trace_clk_3",	 	"gate_clk_cpu_trace_clk_3", CLK_SET_RATE_PARENT,},
+
+	{  EIC7700_CLK_CPU_TRACE_COM_CLK 	,"clk_cpu_trace_com_clk",	"gate_clk_cpu_trace_com_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CPU_CLK			,"clk_cpu_clk",			"gate_clk_cpu_clk", CLK_SET_RATE_PARENT,},
+
+	{  EIC7700_CLK_CLK_1M			,"clk_clk_1m",			"fixed_factor_u_clk_1m_div24", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_NOC_CFG_CLK		,"clk_noc_cfg_clk",		"clk_clk_sys_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_NOC_NSP_CLK		,"clk_noc_nsp_clk",		"gate_noc_nsp_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_BOOTSPI		,"clk_clk_bootspi",		"gate_clk_bootspi", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_BOOTSPI_CFG		,"clk_clk_bootspi_cfg",		"gate_clk_bootspi_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_U84_CORE_LP		,"clk_clk_u84_core_lp",		"fixed_factor_u_u84_core_lp_div2", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_SCPU_CORE		,"clk_clk_scpu_core",		"gate_clk_scpu_core", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_SCPU_BUS 		,"clk_clk_scpu_bus",		"gate_clk_scpu_bus", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_LPCPU_CORE		,"clk_clk_lpcpu_core",		"gate_clk_lpcpu_core", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_LPCPU_BUS		,"clk_clk_lpcpu_bus",		"gate_clk_lpcpu_bus", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_GPU_ACLK 		,"clk_gpu_aclk",		"gate_gpu_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_GPU_GRAY_CLK 		,"clk_gpu_gray_clk",		"gate_gpu_gray_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_GPU_CFG_CLK		,"clk_gpu_cfg_clk",		"gate_gpu_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DSPT_ACLK		,"clk_dspt_aclk",		"gate_dspt_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DSPT_CFG_CLK 		,"clk_dspt_cfg_clk",		"gate_dspt_cfg_clk", CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE,},
+	{  EIC7700_CLK_D2D_ACLK 		,"clk_d2d_aclk",		"gate_d2d_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_D2D_CFG_CLK		,"clk_d2d_cfg_clk",	 	"gate_d2d_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TCU_ACLK 		,"clk_tcu_aclk",		"gate_tcu_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TCU_CFG_CLK		,"clk_tcu_cfg_clk",		"gate_tcu_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT_CFG_CLK 		,"clk_ddrt_cfg_clk",		"gate_ddrt_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT0_P0_ACLK		,"clk_ddrt0_p0_aclk",		"gate_ddrt0_p0_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT0_P1_ACLK		,"clk_ddrt0_p1_aclk",		"gate_ddrt0_p1_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT0_P2_ACLK		,"clk_ddrt0_p2_aclk",		"gate_ddrt0_p2_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT0_P3_ACLK		,"clk_ddrt0_p3_aclk",		"gate_ddrt0_p3_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT0_P4_ACLK		,"clk_ddrt0_p4_aclk",		"gate_ddrt0_p4_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT1_P0_ACLK		,"clk_ddrt1_p0_aclk",		"gate_ddrt1_p0_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT1_P1_ACLK		,"clk_ddrt1_p1_aclk",		"gate_ddrt1_p1_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT1_P2_ACLK		,"clk_ddrt1_p2_aclk",		"gate_ddrt1_p2_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT1_P3_ACLK		,"clk_ddrt1_p3_aclk",		"gate_ddrt1_p3_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DDRT1_P4_ACLK		,"clk_ddrt1_p4_aclk",		"gate_ddrt1_p4_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_ACLK 		,"clk_hsp_aclk",		"gate_clk_hsp_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_CFG_CLK		,"clk_hsp_cfg_clk",		"gate_clk_hsp_cfgclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_SATA_RBC_CLK 	,"clk_hsp_sata_rbc_clk",	"gate_hsp_sata_rbc_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_SATA_OOB_CLK 	,"clk_hsp_sata_oob_clk",	"gate_hsp_sata_oob_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_SATA_PHY_REF		,"clk_hsp_sata_phy_ref",	"gate_hsp_sata_oob_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_SATA_PMALIVE_CLK 	,"clk_hsp_sata_pmalive_clk",  	"gate_hsp_sata_oob_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_ETH_APP_CLK		,"clk_hsp_eth_app_clk",		"clk_clk_sys_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_ETH_CSR_CLK		,"clk_hsp_eth_csr_clk",		"clk_clk_sys_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_ETH0_CORE_CLK	,"clk_hsp_eth0_core_clk",	"gate_hsp_eth0_core_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_ETH1_CORE_CLK	,"clk_hsp_eth1_core_clk",	"gate_hsp_eth1_core_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_MSHC0_CORE_CLK	,"clk_hsp_mshc0_core_clk",	"gate_hsp_mshc0_core_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_MSHC1_CORE_CLK	,"clk_hsp_mshc1_core_clk",	"gate_hsp_mshc1_core_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_MSHC2_CORE_CLK	,"clk_hsp_mshc2_core_clk",	"gate_hsp_mshc2_core_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_MSHC0_TMR_CLK	,"clk_hsp_mshc0_tmr_clk",	"clk_clk_1m", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_MSHC1_TMR_CLK	,"clk_hsp_mshc1_tmr_clk",	"clk_clk_1m", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_MSHC2_TMR_CLK	,"clk_hsp_mshc2_tmr_clk",	"clk_clk_1m", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_USB0_SUSPEND_CLK 	,"clk_hsp_usb0_suspend_clk",	"clk_clk_1m", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_USB1_SUSPEND_CLK 	,"clk_hsp_usb1_suspend_clk",	"clk_clk_1m", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_PCIET_ACLK		,"clk_pciet_aclk",		"gate_pciet_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_PCIET_CFG_CLK		,"clk_pciet_cfg_clk",		"gate_pciet_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_PCIET_CR_CLK 		,"clk_pciet_cr_clk",		"gate_pciet_cr_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_PCIET_AUX_CLK		,"clk_pciet_aux_clk",		"gate_pciet_aux_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_NPU_ACLK 		,"clk_npu_aclk",		"gate_npu_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_NPU_CFG_CLK		,"clk_npu_cfg_clk",		"gate_npu_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_NPU_LLC_SRC0 	,"clk_clk_npu_llc_src0",	"divder_u_npu_llc_src0_div_dynm", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_NPU_LLC_SRC1 	,"clk_clk_npu_llc_src1",	"divder_u_npu_llc_src1_div_dynm", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_NPU_LLC_ACLK 		,"clk_npu_llc_aclk",		"gate_npu_llc_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_NPU_CLK			,"clk_npu_clk",			"gate_npu_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_NPU_E31_CLK		,"clk_npu_e31_clk",		"gate_npu_e31_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_ACLK			,"clk_vi_aclk",			"gate_vi_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_DIG_DW_CLK		,"clk_vi_dig_dw_clk",		"gate_vi_dig_dw_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_CFG_CLK		,"clk_vi_cfg_clk",		"gate_vi_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_DVP_CLK		,"clk_vi_dvp_clk",		"gate_vi_dvp_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_DIG_ISP_CLK		,"clk_vi_dig_isp_clk",		"gate_vi_dig_isp_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_SHUTTER_0 		,"clk_vi_shutter_0",		"gate_vi_shutter_0", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_SHUTTER_1 		,"clk_vi_shutter_1",		"gate_vi_shutter_1", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_SHUTTER_2 		,"clk_vi_shutter_2",		"gate_vi_shutter_2", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_SHUTTER_3 		,"clk_vi_shutter_3",		"gate_vi_shutter_3", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_SHUTTER_4 		,"clk_vi_shutter_4",		"gate_vi_shutter_4", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_SHUTTER_5 		,"clk_vi_shutter_5",		"gate_vi_shutter_5", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_PHY_TXCLKESC		,"clk_vi_phy_txclkesc",		"gate_vi_phy_txclkesc", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VI_PHY_CFG		,"clk_vi_phy_cfg",		"gate_vi_phy_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VO_ACLK			,"clk_vo_aclk",			"gate_vo_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VO_CFG_CLK		,"clk_vo_cfg_clk",		"gate_vo_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VO_HDMI_IESMCLK		,"clk_vo_hdmi_iesmclk",		"gate_vo_hdmi_iesmclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VO_PIXEL_CLK 		,"clk_vo_pixel_clk",		"gate_vo_pixel_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VO_I2S_MCLK		,"clk_vo_i2s_mclk",		"gate_vo_i2s_mclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VO_CR_CLK		,"clk_vo_cr_clk",		"gate_vo_cr_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VO_CEC_CLK		,"clk_vo_cec_clk",		"divder_u_vo_cec_div_dynm", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_ACLK			,"clk_vc_aclk",			"gate_vc_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_CFG_CLK		,"clk_vc_cfg_clk",		"gate_vc_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_JE_CLK		,"clk_vc_je_clk",		"gate_vc_je_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_JD_CLK		,"clk_vc_jd_clk",		"gate_vc_jd_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_VE_CLK		,"clk_vc_ve_clk",		"gate_vc_ve_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_VD_CLK		,"clk_vc_vd_clk",		"gate_vc_vd_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_G2D_CFG_CLK		,"clk_g2d_cfg_clk",		"gate_g2d_cfg_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_G2D_CLK			,"clk_g2d_clk",			"gate_g2d_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_G2D_ACLK 		,"clk_g2d_aclk",		"gate_g2d_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_AONDMA_CFG		,"clk_aondma_cfg",		"gate_clk_aondma_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_AONDMA_ACLK		,"clk_aondma_aclk",		"gate_aondma_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_AON_ACLK 		,"clk_aon_aclk",		"gate_aon_aclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_CLK_0 		,"clk_timer_clk_0",		"gate_time_clk_0", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_CLK_1 		,"clk_timer_clk_1",		"gate_time_clk_1", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_CLK_2 		,"clk_timer_clk_2",		"gate_time_clk_2", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_CLK_3 		,"clk_timer_clk_3",		"gate_time_clk_3", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_PCLK_0		,"clk_timer_pclk_0",		"gate_timer_pclk_0", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_PCLK_1		,"clk_timer_pclk_1",		"gate_timer_pclk_1", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_PCLK_2		,"clk_timer_pclk_2",		"gate_timer_pclk_2", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER_PCLK_3		,"clk_timer_pclk_3",		"gate_timer_pclk_3", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_TIMER3_CLK8		,"clk_timer3_clk8",		"gate_timer3_clk8", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_RTC_CFG		,"clk_clk_rtc_cfg",		"gate_clk_rtc_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_RTC			,"clk_clk_rtc",			"gate_clk_rtc", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_PKA_CFG		,"clk_clk_pka_cfg",	 	"gate_clk_pka_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_SPACC_CFG		,"clk_clk_spacc_cfg",		"gate_clk_spacc_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_CRYPTO		,"clk_clk_crypto",		"gate_clk_crypto", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_TRNG_CFG 		,"clk_clk_trng_cfg",		"gate_clk_trng_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_OTP_CFG		,"clk_clk_otp_cfg",	 	"gate_clk_otp_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLMM_CFG_CLK 		,"clk_clmm_cfg_clk",		"clk_clk_sys_cfg", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLMM_DEB_CLK 		,"clk_clmm_deb_clk",		"clk_clk_1m", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_0 		,"clk_mailbox0",		"gate_clk_mailbox_0", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_1 		,"clk_mailbox1",		"gate_clk_mailbox_1", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_2 		,"clk_mailbox2",		"gate_clk_mailbox_2", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_3 		,"clk_mailbox3",		"gate_clk_mailbox_3", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_4 		,"clk_mailbox4",		"gate_clk_mailbox_4", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_5 		,"clk_mailbox5",		"gate_clk_mailbox_5", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_6 		,"clk_mailbox6",		"gate_clk_mailbox_6", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_7 		,"clk_mailbox7",		"gate_clk_mailbox_7", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_8 		,"clk_mailbox8",		"gate_clk_mailbox_8", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_9 		,"clk_mailbox9",		"gate_clk_mailbox_9", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_10 		,"clk_mailbox10",		"gate_clk_mailbox_10", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_11 		,"clk_mailbox11",		"gate_clk_mailbox_11", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_12 		,"clk_mailbox12",		"gate_clk_mailbox_12", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_13 		,"clk_mailbox13",		"gate_clk_mailbox_13", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_14 		,"clk_mailbox14",		"gate_clk_mailbox_14", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_MAILBOX_15 		,"clk_mailbox15",		"gate_clk_mailbox_15", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C0_PCLK		,"clk_i2c0_pclk",		"gate_i2c0_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C1_PCLK		,"clk_i2c1_pclk",		"gate_i2c1_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C2_PCLK		,"clk_i2c2_pclk",		"gate_i2c2_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C3_PCLK		,"clk_i2c3_pclk",		"gate_i2c3_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C4_PCLK		,"clk_i2c4_pclk",		"gate_i2c4_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C5_PCLK		,"clk_i2c5_pclk",		"gate_i2c5_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C6_PCLK		,"clk_i2c6_pclk",		"gate_i2c6_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C7_PCLK		,"clk_i2c7_pclk",		"gate_i2c7_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C8_PCLK		,"clk_i2c8_pclk",		"gate_i2c8_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_I2C9_PCLK		,"clk_i2c9_pclk",		"gate_i2c9_pclk", CLK_SET_RATE_PARENT,},
+
+	{  EIC7700_CLK_LSP_WDT0_PCLK		,"clk_lsp_wdt0_pclk",		"gate_lsp_wdt0_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_WDT1_PCLK		,"clk_lsp_wdt1_pclk",		"gate_lsp_wdt1_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_WDT2_PCLK		,"clk_lsp_wdt2_pclk",		"gate_lsp_wdt2_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_WDT3_PCLK		,"clk_lsp_wdt3_pclk",		"gate_lsp_wdt3_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_SSI0_PCLK		,"clk_lsp_ssi0_pclk",		"gate_lsp_ssi0_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_SSI1_PCLK		,"clk_lsp_ssi1_pclk",		"gate_lsp_ssi1_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_UART0_PCLK		,"clk_lsp_uart0_pclk",		"gate_lsp_uart0_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_UART1_PCLK		,"clk_lsp_uart1_pclk",		"gate_lsp_uart1_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_UART2_PCLK		,"clk_lsp_uart2_pclk",		"gate_lsp_uart2_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_UART3_PCLK		,"clk_lsp_uart3_pclk",		"gate_lsp_uart3_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_UART4_PCLK		,"clk_lsp_uart4_pclk",		"gate_lsp_uart4_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_TIMER_PCLK		,"clk_lsp_timer_pclk",		"gate_lsp_timer_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_FAN_PCLK		,"clk_lsp_fan_pclk",		"gate_lsp_fan_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_LSP_PVT_PCLK		,"clk_lsp_pvt_pclk",		"gate_lsp_pvt_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_PVT_CLK_0		,"clk_pvt0_clk",		"gate_pvt0_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_PVT_CLK_1		,"clk_pvt1_clk",		"gate_pvt1_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_JE_PCLK		,"clk_vc_je_pclk",		"gate_vc_je_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_JD_PCLK		,"clk_vc_jd_pclk",		"gate_vc_jd_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_VE_PCLK		,"clk_vc_ve_pclk",		"gate_vc_ve_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_VD_PCLK		,"clk_vc_vd_pclk",		"gate_vc_vd_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_VC_MON_PCLK		,"clk_vc_mon_pclk",		"gate_vc_mon_pclk", CLK_SET_RATE_PARENT,},
+
+	{  EIC7700_CLK_HSP_DMA0_CLK		,"clk_hsp_dma0_clk",		"gate_hsp_dma0_clk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_RMII_REF_0		,"clk_hsp_rmii_ref_0",		"gate_hsp_rmii_ref_0", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_HSP_RMII_REF_1		,"clk_hsp_rmii_ref_1",		"gate_hsp_rmii_ref_1", CLK_SET_RATE_PARENT,},
+
+	{  EIC7700_CLK_DSP_ACLK_0		,"clk_dsp_aclk_0",	"divder_u_dsp_0_aclk_div_dynm", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DSP_ACLK_1		,"clk_dsp_aclk_1",	"divder_u_dsp_1_aclk_div_dynm", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DSP_ACLK_2		,"clk_dsp_aclk_2",	"divder_u_dsp_2_aclk_div_dynm", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_DSP_ACLK_3		,"clk_dsp_aclk_3",	"divder_u_dsp_3_aclk_div_dynm", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_CLK_U84_RTC_TOGGLE		,"clk_clk_u84_rtc_toggle",	"divder_u_u84_rtc_toggle_dynm", CLK_SET_RATE_PARENT,},
+
+	{  EIC7700_CLK_AON_I2C0_PCLK		,"clk_aon_i2c0_pclk",	"gate_aon_i2c0_pclk", CLK_SET_RATE_PARENT,},
+	{  EIC7700_CLK_AON_I2C1_PCLK		,"clk_aon_i2c1_pclk",	"gate_aon_i2c1_pclk", CLK_SET_RATE_PARENT,},
+};
+
+static void special_div_table_init(struct clk_div_table *table, int table_size)
+{
+	int i;
+
+	if (table_size < 3) {
+		return;
+	}
+	if (!table) {
+		return;
+	}
+	/*The hardware decides vaule 0, 1 and 2 both means 2 divsor*/
+	for (i = 0; i < 3; i++) {
+		table[i].val = i;
+		table[i].div = 2;
+	}
+	for (i = 3; i < table_size; i++) {
+		table[i].val = i;
+		table[i].div = i;
+	}
+	return;
+}
+
+static int eswin_clk_probe(struct platform_device *pdev)
+{
+	struct eswin_clock_data *clk_data;
+
+	clk_data = eswin_clk_init(pdev, EIC7700_NR_CLKS);
+	if (!clk_data)
+		return -EAGAIN;
+
+	special_div_table_init(u_3_bit_special_div_table, ARRAY_SIZE(u_3_bit_special_div_table));
+	special_div_table_init(u_4_bit_special_div_table, ARRAY_SIZE(u_4_bit_special_div_table));
+	special_div_table_init(u_6_bit_special_div_table, ARRAY_SIZE(u_6_bit_special_div_table));
+	special_div_table_init(u_7_bit_special_div_table, ARRAY_SIZE(u_7_bit_special_div_table));
+	special_div_table_init(u_8_bit_special_div_table, ARRAY_SIZE(u_8_bit_special_div_table));
+	special_div_table_init(u_11_bit_special_div_table, ARRAY_SIZE(u_11_bit_special_div_table));
+	special_div_table_init(u_16_bit_special_div_table, ARRAY_SIZE(u_16_bit_special_div_table));
+
+	eswin_clk_register_fixed_rate(eic7700_fixed_rate_clks,
+				ARRAY_SIZE(eic7700_fixed_rate_clks),
+				clk_data);
+	eswin_clk_register_pll(eic7700_pll_clks,
+			ARRAY_SIZE(eic7700_pll_clks), clk_data, &pdev->dev);
+
+	eswin_clk_register_fixed_factor(eic7700_fixed_factor_clks,
+				ARRAY_SIZE(eic7700_fixed_factor_clks),
+				clk_data);
+	eswin_clk_register_mux(eic7700_mux_clks, ARRAY_SIZE(eic7700_mux_clks),
+				clk_data);
+	eswin_clk_register_clk(eic7700_clks_early_0, ARRAY_SIZE(eic7700_clks_early_0),
+				clk_data);
+	eswin_clk_register_divider(eic7700_div_clks, ARRAY_SIZE(eic7700_div_clks),
+				clk_data);
+	eswin_clk_register_clk(eic7700_clks_early_1, ARRAY_SIZE(eic7700_clks_early_1),
+				clk_data);
+	eswin_clk_register_gate(eic7700_gate_clks, ARRAY_SIZE(eic7700_gate_clks),
+				clk_data);
+	eswin_clk_register_clk(eic7700_clks, ARRAY_SIZE(eic7700_clks), clk_data);
+	return 0;
+}
+
+static const struct of_device_id eswin_clock_dt_ids[] = {
+	 { .compatible = "eswin,eic7700-clock", },
+	 { /* sentinel */ },
+};
+
+static struct platform_driver eswin_clock_driver = {
+	.probe	= eswin_clk_probe,
+	.driver = {
+		.name	= "eswin-clock",
+		.of_match_table	= eswin_clock_dt_ids,
+	},
+};
+static int __init eic7700_clk_init(void)
+{
+	return platform_driver_register(&eswin_clock_driver);
+}
+arch_initcall(eic7700_clk_init);
diff --git a/drivers/clk/eswin/clk.c b/drivers/clk/eswin/clk.c
new file mode 100755
index 000000000000..62bd09700a00
--- /dev/null
+++ b/drivers/clk/eswin/clk.c
@@ -0,0 +1,915 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Clk Provider Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/slab.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/util_macros.h>
+#include <dt-bindings/clock/eic7700-clock.h>
+
+#include "clk.h"
+
+struct clk_hw *eswin_clk_find_parent(struct eswin_clock_data *data, char *parent_name)
+{
+	int i;
+	struct clk *clks;
+
+	for (i = 0; i < data->clk_data.clk_num; i++) {
+		clks = data->clk_data.clks[i];
+		if (NULL == clks) {
+			continue;
+		}
+		if (!strcmp(__clk_get_name(clks), parent_name)) {
+			return __clk_get_hw(clks);
+		}
+	}
+	return NULL;
+}
+
+struct eswin_clock_data *eswin_clk_init(struct platform_device *pdev,
+					     int nr_clks)
+{
+	struct eswin_clock_data *clk_data;
+	struct clk **clk_table;
+	void __iomem *base;
+	struct device *parent;
+
+	parent = pdev->dev.parent;
+	if (!parent) {
+		dev_err(&pdev->dev, "no parent\n");
+		goto err;
+	}
+
+	base = of_iomap(parent->of_node, 0);
+	if (!base) {
+		dev_err(&pdev->dev,"failed to map clock registers\n");
+		goto err;
+	}
+	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
+	if (!clk_data)
+		goto err;
+
+	clk_data->base = base;
+	clk_table = kcalloc(nr_clks, sizeof(*clk_table), GFP_KERNEL);
+	if (!clk_table)
+		goto err_data;
+
+	clk_data->clk_data.clks = clk_table;
+	clk_data->clk_data.clk_num = nr_clks;
+	clk_data->numa_id = dev_to_node(parent);
+	spin_lock_init(&clk_data->lock);
+
+	of_clk_add_provider(pdev->dev.of_node, of_clk_src_onecell_get, &clk_data->clk_data);
+	return clk_data;
+
+err_data:
+	kfree(clk_data);
+err:
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(eswin_clk_init);
+
+int eswin_clk_register_fixed_rate(const struct eswin_fixed_rate_clock *clks,
+					 int nums, struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name) + 2 * sizeof(char) + sizeof(int),
+			GFP_KERNEL );
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+		}
+		clk = clk_register_fixed_rate(NULL, name, clks[i].parent_name,
+				clks[i].flags, clks[i].fixed_rate);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__, name);
+			kfree(name);
+			goto err;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+	}
+
+	return 0;
+err:
+	while (i--)
+		clk_unregister_fixed_rate(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_fixed_rate);
+
+static int eswin_calc_pll(u32 *frac_val, u32 *postdiv1_val,
+				 u32 *fbdiv_val, u32 *refdiv_val, u64 rate,
+				 const struct eswin_clk_pll *clk)
+{
+	int ret = 0;
+
+	switch (clk->id) {
+		case EIC7700_APLL_FOUT1:
+			switch (rate) {
+				case APLL_LOW_FREQ:
+					*frac_val = 10603200;
+					*postdiv1_val = 0;
+					*fbdiv_val = 37;
+					*refdiv_val = 1;
+					break;
+				case APLL_HIGH_FREQ:
+				default:
+					*frac_val = 14092861;
+					*postdiv1_val = 0;
+					*fbdiv_val = 163;
+					*refdiv_val = 1;
+					break;
+			}
+			break;
+		case EIC7700_PLL_CPU:
+			switch (rate) {
+				case CLK_FREQ_1800M:
+					*frac_val = 0;
+					*postdiv1_val = 0;
+					*fbdiv_val = 300;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_1700M:
+					*frac_val = 5592405;
+					*postdiv1_val = 0;
+					*fbdiv_val = 283;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_1600M:
+					*frac_val = 11184810;
+					*postdiv1_val = 0;
+					*fbdiv_val = 266;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_1500M:
+					*frac_val = 0;
+					*postdiv1_val = 0;
+					*fbdiv_val = 250;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_1300M:
+					*frac_val = 11184810;
+					*postdiv1_val = 0;
+					*fbdiv_val = 216;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_1200M:
+					*frac_val = 0;
+					*postdiv1_val = 0;
+					*fbdiv_val = 200;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_1000M:
+					*frac_val = 11184810;
+					*postdiv1_val = 0;
+					*fbdiv_val = 166;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_900M:
+					*frac_val = 0;
+					*postdiv1_val = 0;
+					*fbdiv_val = 150;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_800M:
+					*frac_val = 5592405;
+					*postdiv1_val = 0;
+					*fbdiv_val = 133;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_700M:
+					*frac_val = 11184810;
+					*postdiv1_val = 0;
+					*fbdiv_val = 116;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_600M:
+					*frac_val = 0;
+					*postdiv1_val = 0;
+					*fbdiv_val = 100;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_500M:
+					*frac_val = 5592405;
+					*postdiv1_val = 0;
+					*fbdiv_val = 83;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_400M:
+					*frac_val = 11184810;
+					*postdiv1_val = 0;
+					*fbdiv_val = 66;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_200M:
+					*frac_val = 5592405;
+					*postdiv1_val = 0;
+					*fbdiv_val = 33;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_100M:
+					*frac_val = 11184810;
+					*postdiv1_val = 0;
+					*fbdiv_val = 16;
+					*refdiv_val = 1;
+					break;
+				case CLK_FREQ_1400M:
+				default:
+					*frac_val = 5592405;
+					*postdiv1_val = 0;
+					*fbdiv_val = 233;
+					*refdiv_val = 1;
+					break;
+			}
+			break;
+		default:
+			ret = -EINVAL;
+			pr_err("%s %d, Invalid pll set req, rate %lld, clk id %d\n", __func__, __LINE__, rate, clk->id);
+			break;
+	}
+	return ret;
+}
+
+#define to_pll_clk(_hw) container_of(_hw, struct eswin_clk_pll, hw)
+static int clk_pll_set_rate(struct clk_hw *hw,
+			    unsigned long rate,
+			    unsigned long parent_rate)
+{
+	struct eswin_clk_pll *clk = to_pll_clk(hw);
+	u32 frac_val = 0, postdiv1_val, fbdiv_val, refdiv_val;
+	u32 val;
+	int ret;
+	struct clk *clk_cpu_mux = NULL;
+	struct clk *clk_cpu_lp_pll = NULL;
+	struct clk *clk_cpu_pll = NULL;
+	int try_count = 0;
+	bool lock_flag = false;
+	char clk_cpu_mux_name[50] = {0};
+	char clk_cpu_lp_pll_name[50] = {0};
+	char clk_cpu_pll_name[50] = {0};
+
+	ret = eswin_calc_pll(&frac_val, &postdiv1_val, &fbdiv_val, &refdiv_val, (u64)rate, clk);
+	if (ret) {
+		return ret;
+	}
+
+	/*
+	  we must switch the cpu to other clk before we change the cpu pll
+	*/
+	if (EIC7700_PLL_CPU == clk->id) {
+		if (clk->numa_id < 0) {
+			sprintf(clk_cpu_mux_name, "%s", "mux_u_cpu_root_3mux1_gfree");
+			sprintf(clk_cpu_lp_pll_name, "%s", "clk_clk_u84_core_lp");
+			sprintf(clk_cpu_pll_name, "%s", "clk_pll_cpu");
+		} else {
+			sprintf(clk_cpu_mux_name, "d%d_%s", clk->numa_id, "mux_u_cpu_root_3mux1_gfree");
+			sprintf(clk_cpu_lp_pll_name, "d%d_%s", clk->numa_id, "clk_clk_u84_core_lp");
+			sprintf(clk_cpu_pll_name, "d%d_%s", clk->numa_id, "clk_pll_cpu");
+		}
+
+		clk_cpu_mux = __clk_lookup(clk_cpu_mux_name);
+		if (!clk_cpu_mux) {
+			pr_err("%s %d, failed to get %s\n",__func__,__LINE__, clk_cpu_mux_name);
+			return -EINVAL;
+		}
+		clk_cpu_lp_pll = __clk_lookup(clk_cpu_lp_pll_name);
+		if (!clk_cpu_lp_pll) {
+			pr_err("%s %d, failed to get %s\n",__func__,__LINE__, clk_cpu_lp_pll_name);
+			return -EINVAL;
+		}
+		clk_cpu_pll = __clk_lookup(clk_cpu_pll_name);
+		if (!clk_cpu_pll) {
+			pr_err("%s %d, failed to get %s\n",__func__,__LINE__, clk_cpu_pll_name);
+			return -EINVAL;
+		}
+
+		ret = clk_set_parent(clk_cpu_mux, clk_cpu_lp_pll);
+		if (ret) {
+			pr_err("%s %d, faild to switch %s to %s, ret %d\n",__func__,__LINE__, clk_cpu_mux_name,
+				clk_cpu_lp_pll_name, ret);
+			return -EPERM;
+		}
+	}
+
+	/*first disable pll */
+	val = readl_relaxed(clk->ctrl_reg0);
+	val &= ~(((1 << clk->pllen_width) - 1) << clk->pllen_shift);
+	val |= 0 << clk->pllen_shift;
+	writel_relaxed(val, clk->ctrl_reg0);
+
+	val = readl_relaxed(clk->ctrl_reg0);
+	val &= ~(((1 << clk->fbdiv_width) - 1) << clk->fbdiv_shift);
+	val &= ~(((1 << clk->refdiv_width) - 1) << clk->refdiv_shift);
+	val |= refdiv_val << clk->refdiv_shift;
+	val |= fbdiv_val << clk->fbdiv_shift;
+	writel_relaxed(val, clk->ctrl_reg0);
+
+	val = readl_relaxed(clk->ctrl_reg1);
+	val &= ~(((1 << clk->frac_width) - 1) << clk->frac_shift);
+	val |= frac_val << clk->frac_shift;
+	writel_relaxed(val, clk->ctrl_reg1);
+
+	val = readl_relaxed(clk->ctrl_reg2);
+	val &= ~(((1 << clk->postdiv1_width) - 1) << clk->postdiv1_shift);
+	val |= postdiv1_val << clk->postdiv1_shift;
+	writel_relaxed(val, clk->ctrl_reg2);
+
+	/*at last, enable pll */
+	val = readl_relaxed(clk->ctrl_reg0);
+	val &= ~(((1 << clk->pllen_width) - 1) << clk->pllen_shift);
+	val |= 1 << clk->pllen_shift;
+	writel_relaxed(val, clk->ctrl_reg0);
+
+	/*
+	  usually the pll wil lock in 50us
+	*/
+	do {
+		usleep_range(refdiv_val * 80, refdiv_val * 80 * 2);
+		val = readl_relaxed(clk->status_reg);
+		if (val & 1 << clk->lock_shift) {
+			lock_flag = true;
+			break;
+		}
+	} while (try_count++ < 10);
+
+	if (false == lock_flag) {
+		pr_err("%s %d, faild to lock the cpu pll, cpu will work on low power pll\n",__func__,__LINE__);
+		return -EBUSY;
+	}
+	if (EIC7700_PLL_CPU == clk->id) {
+		ret = clk_set_parent(clk_cpu_mux, clk_cpu_pll);
+		if (ret) {
+			pr_err("%s %d, faild to switch %s to %s, ret %d\n",__func__,__LINE__,
+				clk_cpu_mux_name, clk_cpu_pll_name, ret);
+			return -EPERM;
+		}
+	}
+	return  0;
+}
+
+static unsigned long clk_pll_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct eswin_clk_pll *clk = to_pll_clk(hw);
+	u64 frac_val, fbdiv_val, refdiv_val;
+	u32 postdiv1_val;
+	u32 val;
+	u64 rate;
+
+	val = readl_relaxed(clk->ctrl_reg0);
+	val = val >> clk->fbdiv_shift;
+	val &= ((1 << clk->fbdiv_width) - 1);
+	fbdiv_val = val;
+
+	val = readl_relaxed(clk->ctrl_reg0);
+	val = val >> clk->refdiv_shift;
+	val &= ((1 << clk->refdiv_width) - 1);
+	refdiv_val = val;
+
+	val = readl_relaxed(clk->ctrl_reg1);
+	val = val >> clk->frac_shift;
+	val &= ((1 << clk->frac_width) - 1);
+	frac_val = val;
+
+	val = readl_relaxed(clk->ctrl_reg2);
+	val = val >> clk->postdiv1_shift;
+	val &= ((1 << clk->postdiv1_width) - 1);
+	postdiv1_val = val;
+
+	switch (clk->id) {
+		case EIC7700_APLL_FOUT1:
+			switch (frac_val) {
+				case 14092861:
+					rate = APLL_HIGH_FREQ;
+					break;
+				case 10603200:
+					rate = APLL_LOW_FREQ;
+					break;
+				default:
+					pr_err("%s %d, clk id %d, unknow frac_val %llu\n", __func__, __LINE__, clk->id, frac_val);
+					rate = 0;
+					break;
+			}
+			break;
+		case EIC7700_PLL_CPU:
+			switch (fbdiv_val) {
+				case 300:
+					rate = CLK_FREQ_1800M;
+					break;
+				case 283:
+					rate = CLK_FREQ_1700M;
+					break;
+				case 266:
+					rate = CLK_FREQ_1600M;
+					break;
+				case 250:
+					rate = CLK_FREQ_1500M;
+					break;
+				case 216:
+					rate = CLK_FREQ_1300M;
+					break;
+				case 200:
+					rate = CLK_FREQ_1200M;
+					break;
+				case 166:
+					rate = CLK_FREQ_1000M;
+					break;
+				case 150:
+					rate = CLK_FREQ_900M;
+					break;
+				case 133:
+					rate = CLK_FREQ_800M;
+					break;
+				case 116:
+					rate = CLK_FREQ_700M;
+					break;
+				case 100:
+					rate = CLK_FREQ_600M;
+					break;
+				case 83:
+					rate = CLK_FREQ_500M;
+					break;
+				case 66:
+					rate = CLK_FREQ_400M;
+					break;
+				case 33:
+					rate = CLK_FREQ_200M;
+					break;
+				case 16:
+					rate = CLK_FREQ_100M;
+					break;
+				case 233:
+					rate = CLK_FREQ_1400M;
+					break;
+				default:
+					pr_err("%s %d, clk id %d, unknow fbdiv_val %llu\n", __func__, __LINE__, clk->id, fbdiv_val);
+					rate = 0;
+					break;
+			}
+			break;
+		default:
+			pr_err("%s %d, unknow clk id %d\n", __func__, __LINE__, clk->id);
+			rate = 0;
+			break;
+	}
+	return rate;
+}
+
+static long clk_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+					unsigned long *parent_rate)
+{
+	struct eswin_clk_pll *clk = to_pll_clk(hw);
+	int index;
+	u64 round_rate = 0;
+
+	/*Must be sorted in ascending order*/
+	u64 apll_clk[] = {APLL_LOW_FREQ, APLL_HIGH_FREQ};
+	u64 cpu_pll_clk[] = {CLK_FREQ_100M, CLK_FREQ_200M, CLK_FREQ_400M, CLK_FREQ_500M, CLK_FREQ_600M, CLK_FREQ_700M,
+				CLK_FREQ_800M, CLK_FREQ_900M, CLK_FREQ_1000M, CLK_FREQ_1200M, CLK_FREQ_1300M,
+				CLK_FREQ_1400M, CLK_FREQ_1500M, CLK_FREQ_1600M, CLK_FREQ_1700M, CLK_FREQ_1800M};
+
+	switch (clk->id) {
+		case EIC7700_APLL_FOUT1:
+			index = find_closest(rate, apll_clk, ARRAY_SIZE(apll_clk));
+			round_rate = apll_clk[index];
+			break;
+		case EIC7700_PLL_CPU:
+			index = find_closest(rate, cpu_pll_clk, ARRAY_SIZE(cpu_pll_clk));
+			round_rate = cpu_pll_clk[index];
+			break;
+		default:
+			pr_err("%s %d, unknow clk id %d\n", __func__, __LINE__, clk->id);
+			round_rate = 0;
+			break;
+	}
+	return round_rate;
+}
+
+static const struct clk_ops eswin_clk_pll_ops = {
+	.set_rate = clk_pll_set_rate,
+	.recalc_rate = clk_pll_recalc_rate,
+	.round_rate = clk_pll_round_rate,
+};
+
+void eswin_clk_register_pll(struct eswin_pll_clock *clks,
+		int nums, struct eswin_clock_data *data, struct device *dev)
+{
+	void __iomem *base = data->base;
+	struct eswin_clk_pll *p_clk = NULL;
+	struct clk *clk = NULL;
+	struct clk_init_data init;
+	int i;
+
+	p_clk = devm_kzalloc(dev, sizeof(*p_clk) * nums, GFP_KERNEL);
+
+	if (!p_clk)
+		return;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL);
+		const char *parent_name = clks[i].parent_name ? kzalloc(strlen(clks[i].parent_name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL) : NULL;
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			if (parent_name) {
+				sprintf((char *)parent_name, "%s", clks[i].parent_name);
+			}
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			if (parent_name) {
+				sprintf((char *)parent_name, "d%d_%s", data->numa_id, clks[i].parent_name);
+			}
+		}
+
+		init.name = name;
+		init.flags = 0;
+		init.parent_names = parent_name ? &parent_name : NULL;
+		init.num_parents = parent_name ? 1 : 0;
+		init.ops = &eswin_clk_pll_ops;
+
+		p_clk->id = clks[i].id;
+		p_clk->numa_id = data->numa_id;
+		p_clk->ctrl_reg0 = base + clks[i].ctrl_reg0;
+		p_clk->pllen_shift = clks[i].pllen_shift;
+		p_clk->pllen_width = clks[i].pllen_width;
+		p_clk->refdiv_shift = clks[i].refdiv_shift;
+		p_clk->refdiv_width = clks[i].refdiv_width;
+		p_clk->fbdiv_shift = clks[i].fbdiv_shift;
+		p_clk->fbdiv_width = clks[i].fbdiv_width;
+
+		p_clk->ctrl_reg1 = base + clks[i].ctrl_reg1;
+		p_clk->frac_shift = clks[i].frac_shift;
+		p_clk->frac_width = clks[i].frac_width;
+
+		p_clk->ctrl_reg2 = base + clks[i].ctrl_reg2;
+		p_clk->postdiv1_shift = clks[i].postdiv1_shift;
+		p_clk->postdiv1_width = clks[i].postdiv1_width;
+		p_clk->postdiv2_shift = clks[i].postdiv2_shift;
+		p_clk->postdiv2_width = clks[i].postdiv2_width;
+
+		p_clk->status_reg = base + clks[i].status_reg;
+		p_clk->lock_shift = clks[i].lock_shift;
+		p_clk->lock_width = clks[i].lock_width;
+
+		p_clk->hw.init = &init;
+
+		clk = clk_register(dev, &p_clk->hw);
+		if (IS_ERR(clk)) {
+			devm_kfree(dev, p_clk);
+			dev_err(dev, "%s: failed to register clock %s\n", __func__, clks[i].name);
+			continue;
+		}
+
+		data->clk_data.clks[clks[i].id] = clk;
+		p_clk++;
+		kfree(name);
+		if (parent_name) {
+			kfree(parent_name);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_pll);
+
+int eswin_clk_register_fixed_factor(const struct eswin_fixed_factor_clock *clks,
+					   int nums,
+					   struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+		char *parent_name = kzalloc(strlen(clks[i].parent_name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id, clks[i].parent_name);
+		}
+
+		clk = clk_register_fixed_factor(NULL, name,
+						parent_name,
+						clks[i].flags, clks[i].mult,
+						clks[i].div);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__, name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+
+		kfree(name);
+		kfree(parent_name);
+	}
+
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_fixed_factor(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_fixed_factor);
+
+int eswin_clk_register_mux(const struct eswin_mux_clock *clks,
+				  int nums, struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+	int j;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+
+		char **parent_names = kzalloc(sizeof(char *) * clks[i].num_parents,
+			GFP_KERNEL );
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+		}
+		for (j = 0; j < clks[i].num_parents; j++) {
+			parent_names[j] = kzalloc(strlen(clks[i].parent_names[j])
+				+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+			if (data->numa_id < 0) {
+				sprintf(parent_names[j], "%s", clks[i].parent_names[j]);
+			} else {
+				sprintf(parent_names[j], "d%d_%s",
+					data->numa_id, clks[i].parent_names[j]);
+			}
+		}
+		clk = clk_register_mux_table(NULL, name,
+				(const char * const*)parent_names,
+				clks[i].num_parents, clks[i].flags,
+				base + clks[i].offset, clks[i].shift,
+				clks[i].mask, clks[i].mux_flags,
+				clks[i].table, &data->lock);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__, clks[i].name);
+			kfree(name);
+			for (j = 0; j < clks[i].num_parents; j++) {
+				kfree(parent_names[j]);
+			}
+			kfree(parent_names);
+			goto err;
+		}
+
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+
+		kfree(name);
+		for (j = 0; j < clks[i].num_parents; j++) {
+			kfree(parent_names[j]);
+		}
+		kfree(parent_names);
+	}
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_mux(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_mux);
+
+int eswin_clk_register_divider(const struct eswin_divider_clock *clks,
+				      int nums, struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+	struct clk_hw *clk_hw;
+	struct clk_hw *parent_hw;
+	struct clk_parent_data	parent_data;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+		char *parent_name = kzalloc(strlen(clks[i].parent_name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id, clks[i].parent_name);
+		}
+		parent_hw = eswin_clk_find_parent(data, parent_name);
+		parent_data.name = parent_name;
+		parent_data.hw = parent_hw;
+		parent_data.fw_name = NULL;
+		clk_hw = clk_hw_register_divider_table_parent_data(NULL, name,
+						&parent_data,
+						clks[i].flags,
+						base + clks[i].offset,
+						clks[i].shift, clks[i].width,
+						clks[i].div_flags,
+						clks[i].table,
+						&data->lock);
+		if (IS_ERR(clk_hw)) {
+			pr_err("%s: failed to register clock %s\n", __func__, clks[i].name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+		clk = clk_hw->clk;
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+		kfree(parent_name);
+	}
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_divider(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_divider);
+
+int eswin_clk_register_gate(const struct eswin_gate_clock *clks,
+				       int nums, struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+	struct clk_hw *clk_hw;
+	struct clk_hw *parent_hw;
+	struct clk_parent_data	parent_data;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL);
+		char *parent_name = kzalloc(strlen(clks[i].parent_name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL);
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id, clks[i].parent_name);
+		}
+		parent_hw = eswin_clk_find_parent(data, parent_name);
+		parent_data.name = parent_name;
+		parent_data.hw = parent_hw;
+		parent_data.fw_name = NULL;
+		clk_hw = clk_hw_register_gate_parent_data(NULL, name,
+				&parent_data,
+				clks[i].flags,
+				base + clks[i].offset,
+				clks[i].bit_idx,
+				clks[i].gate_flags,
+				&data->lock);
+		if (IS_ERR(clk_hw)) {
+			pr_err("%s: failed to register clock %s\n",__func__, clks[i].name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+		clk = clk_hw->clk;
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+		kfree(parent_name);
+	}
+	return 0;
+
+err:
+	while (i--)
+		clk_unregister_gate(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_gate);
+
+static const struct clk_ops clk_dummpy_ops = {
+
+};
+
+struct clk *eswin_register_clk(struct eswin_clock_data *data,
+				     struct device *dev, const char *name,
+				      const char *parent_name,
+				      unsigned long flags,
+				      spinlock_t *lock)
+{
+	struct eswin_clock *eclk;
+	struct clk *clk;
+	struct clk_init_data init;
+	struct clk_parent_data	parent_data;
+	struct clk_hw 		*parent_hw;
+
+	eclk = kzalloc(sizeof(*eclk), GFP_KERNEL );
+	if (!eclk)
+		return ERR_PTR(-ENOMEM);
+
+	init.ops = &clk_dummpy_ops;
+
+	init.name = name;
+	init.flags = flags;
+	init.parent_names = NULL;
+	init.num_parents = (parent_name ? 1 : 0);
+	init.parent_data = &parent_data;
+
+	parent_hw = eswin_clk_find_parent(data, (char *)parent_name);
+	parent_data.name = parent_name;
+	parent_data.hw = parent_hw;
+	parent_data.fw_name = NULL;
+
+	eclk->hw.init = &init;
+
+	clk = clk_register(dev, &eclk->hw);
+	if (IS_ERR(clk))
+		kfree(eclk);
+
+	return clk;
+}
+
+int eswin_clk_register_clk(const struct eswin_clock *clks,
+				       int nums, struct eswin_clock_data *data)
+{
+	struct clk *clk;
+	int 	i;
+
+	for (i = 0; i < nums; i++) {
+		char *name = kzalloc(strlen(clks[i].name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+		char *parent_name = kzalloc(strlen(clks[i].parent_name)
+			+ 2 * sizeof(char) + sizeof(int), GFP_KERNEL );
+		if (data->numa_id < 0) {
+			sprintf(name, "%s", clks[i].name);
+			sprintf(parent_name, "%s", clks[i].parent_name);
+		} else {
+			sprintf(name, "d%d_%s", data->numa_id, clks[i].name);
+			sprintf(parent_name, "d%d_%s", data->numa_id, clks[i].parent_name);
+		}
+		clk = eswin_register_clk(data, NULL, name, parent_name,
+					clks[i].flags, &data->lock);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n", __func__, clks[i].name);
+			kfree(name);
+			kfree(parent_name);
+			goto err;
+		}
+
+		if (clks[i].alias)
+			clk_register_clkdev(clk, clks[i].alias, NULL);
+
+		data->clk_data.clks[clks[i].id] = clk;
+		kfree(name);
+		kfree(parent_name);
+	}
+	return 0;
+err:
+	while (i--)
+		clk_unregister_gate(data->clk_data.clks[clks[i].id]);
+
+	return PTR_ERR(clk);
+}
+EXPORT_SYMBOL_GPL(eswin_clk_register_clk);
diff --git a/drivers/clk/eswin/clk.h b/drivers/clk/eswin/clk.h
new file mode 100755
index 000000000000..c5906c420769
--- /dev/null
+++ b/drivers/clk/eswin/clk.h
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Clk Provider Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef	__ESWIN_CLK_H
+#define	__ESWIN_CLK_H
+
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+
+struct platform_device;
+
+struct eswin_clock_data {
+	struct clk_onecell_data	clk_data;
+	void __iomem		*base;
+	int 			numa_id;
+	spinlock_t 		lock;
+};
+
+struct eswin_clock {
+	unsigned int		id;
+	char			*name;
+	const char		*parent_name;
+	unsigned long		flags;
+	struct clk_hw		hw;
+	const char		*alias;
+};
+
+struct eswin_fixed_rate_clock {
+	unsigned int		id;
+	char			*name;
+	const char		*parent_name;
+	unsigned long		flags;
+	unsigned long		fixed_rate;
+};
+
+struct eswin_fixed_factor_clock {
+	unsigned int		id;
+	char				*name;
+	const char			*parent_name;
+	unsigned long		mult;
+	unsigned long		div;
+	unsigned long		flags;
+};
+
+struct eswin_mux_clock {
+	unsigned int		id;
+	const char			*name;
+	const char	*const 	*parent_names;
+	u8					num_parents;
+	unsigned long		flags;
+	unsigned long		offset;
+	u8					shift;
+	u32 				mask;
+	u8					mux_flags;
+	u32					*table;
+	const char			*alias;
+};
+
+
+struct eswin_divider_clock {
+	unsigned int		id;
+	const char		*name;
+	const char		*parent_name;
+	unsigned long		flags;
+	unsigned long		offset;
+	u8			shift;
+	u8			width;
+	u8			div_flags;
+	struct clk_div_table	*table;
+	const char		*alias;
+};
+
+struct eswin_gate_clock {
+	unsigned int		id;
+	const char		*name;
+	const char		*parent_name;
+	unsigned long		flags;
+	unsigned long		offset;
+	u8			bit_idx;
+	u8			gate_flags;
+	const char		*alias;
+};
+
+struct eswin_pll_clock {
+	u32	id;
+	const char	*name;
+	const char	*parent_name;
+	const u32	ctrl_reg0;
+	const u8	pllen_shift;
+	const u8	pllen_width;
+	const u8	refdiv_shift;
+	const u8	refdiv_width;
+	const u8	fbdiv_shift;
+	const u8	fbdiv_width;
+
+	const u32	ctrl_reg1;
+	const u8	frac_shift;
+	const u8	frac_width;
+
+	const u32	ctrl_reg2;
+	const u8	postdiv1_shift;
+	const u8	postdiv1_width;
+	const u8	postdiv2_shift;
+	const u8	postdiv2_width;
+
+	const u32	status_reg;
+	const u8	lock_shift;
+	const u8	lock_width;
+};
+
+struct eswin_clk_pll {
+	struct clk_hw	hw;
+	u32	id;
+	int 	numa_id;
+	void __iomem	*ctrl_reg0;
+	u8	pllen_shift;
+	u8	pllen_width;
+	u8	refdiv_shift;
+	u8	refdiv_width;
+	u8	fbdiv_shift;
+	u8	fbdiv_width;
+
+	void __iomem	*ctrl_reg1;
+	u8	frac_shift;
+	u8	frac_width;
+
+	void __iomem	*ctrl_reg2;
+	u8	postdiv1_shift;
+	u8	postdiv1_width;
+	u8	postdiv2_shift;
+	u8	postdiv2_width;
+
+	void __iomem	*status_reg;
+	u8	lock_shift;
+	u8	lock_width;
+};
+
+struct eswin_clock_data *eswin_clk_init(struct platform_device *, int);
+int eswin_clk_register_fixed_rate(const struct eswin_fixed_rate_clock *,
+				int, struct eswin_clock_data *);
+
+void eswin_clk_register_pll(struct eswin_pll_clock *clks,
+		int nums, struct eswin_clock_data *data, struct device *dev);
+
+int eswin_clk_register_fixed_factor(const struct eswin_fixed_factor_clock *,
+				int, struct eswin_clock_data *);
+int eswin_clk_register_mux(const struct eswin_mux_clock *, int,
+				struct eswin_clock_data *);
+
+int eswin_clk_register_divider(const struct eswin_divider_clock *,
+				int, struct eswin_clock_data *);
+int eswin_clk_register_gate(const struct eswin_gate_clock *,
+				int, struct eswin_clock_data *);
+
+int eswin_clk_register_clk(const struct eswin_clock *,
+				       int , struct eswin_clock_data *);
+
+#define eswin_clk_unregister(type) \
+static inline \
+void eswin_clk_unregister_##type(const struct eswin_##type##_clock *clks, \
+				int nums, struct eswin_clock_data *data) \
+{ \
+	struct clk **clocks = data->clk_data.clks; \
+	int i; \
+	for (i = 0; i < nums; i++) { \
+		int id = clks[i].id; \
+		if (clocks[id])  \
+			clk_unregister_##type(clocks[id]); \
+	} \
+}
+
+eswin_clk_unregister(fixed_rate)
+eswin_clk_unregister(fixed_factor)
+eswin_clk_unregister(mux)
+eswin_clk_unregister(divider)
+eswin_clk_unregister(gate)
+
+#endif	/* __ESWIN_CLK_H */
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
index 72fb40de58b3..63520e00accb 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -32,6 +32,13 @@
 #include "dw-axi-dmac.h"
 #include "../dmaengine.h"
 #include "../virt-dma.h"
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+#include <linux/iommu.h>
+#include <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/eic7700-sid-cfg.h>
+#endif
 
 /*
  * The set of bus widths supported by the DMA controller. DW AXI DMAC supports
@@ -51,6 +58,15 @@
 #define AXI_DMA_FLAG_HAS_RESETS		BIT(1)
 #define AXI_DMA_FLAG_USE_CFG2		BIT(2)
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+#define AWSMMUSID	GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID	GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID	GENMASK(15, 8)	// The sid of read operation
+#define ARSMMUSSID	GENMASK(7, 0)	// The ssid of read operation
+
+static int eswin_dma_sid_cfg(struct device *dev);
+#endif
+
 static inline void
 axi_dma_iowrite32(struct axi_dma_chip *chip, u32 reg, u32 val)
 {
@@ -98,14 +114,26 @@ static inline void axi_chan_config_write(struct axi_dma_chan *chan,
 			 config->hs_sel_dst << CH_CFG_H_HS_SEL_DST_POS |
 			 config->src_per << CH_CFG_H_SRC_PER_POS |
 			 config->dst_per << CH_CFG_H_DST_PER_POS |
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+			 config->prior << CH_CFG_H_PRIORITY_POS |
+			 0xF <<CH_CFG_H_SRC_OSR_LMT_POS |
+			 0xF <<CH_CFG_H_DST_OSR_LMT_POS;
+#else
 			 config->prior << CH_CFG_H_PRIORITY_POS;
+#endif
 	} else {
 		cfg_lo |= config->src_per << CH_CFG2_L_SRC_PER_POS |
 			  config->dst_per << CH_CFG2_L_DST_PER_POS;
 		cfg_hi = config->tt_fc << CH_CFG2_H_TT_FC_POS |
 			 config->hs_sel_src << CH_CFG2_H_HS_SEL_SRC_POS |
 			 config->hs_sel_dst << CH_CFG2_H_HS_SEL_DST_POS |
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+			 config->prior << CH_CFG2_H_PRIORITY_POS |
+			 0xF <<CH_CFG2_H_SRC_OSR_LMT_POS |
+			 0xF <<CH_CFG2_H_DST_OSR_LMT_POS;
+#else
 			 config->prior << CH_CFG2_H_PRIORITY_POS;
+#endif
 	}
 	axi_chan_iowrite32(chan, CH_CFG_L, cfg_lo);
 	axi_chan_iowrite32(chan, CH_CFG_H, cfg_hi);
@@ -114,37 +142,49 @@ static inline void axi_chan_config_write(struct axi_dma_chan *chan,
 static inline void axi_dma_disable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val &= ~DMAC_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_dma_enable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val |= DMAC_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_dma_irq_disable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val &= ~INT_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_dma_irq_enable(struct axi_dma_chip *chip)
 {
 	u32 val;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chip->lock, flags);
 	val = axi_dma_ioread32(chip, DMAC_CFG);
 	val |= INT_EN_MASK;
 	axi_dma_iowrite32(chip, DMAC_CFG, val);
+	spin_unlock_irqrestore(&chip->lock, flags);
 }
 
 static inline void axi_chan_irq_disable(struct axi_dma_chan *chan, u32 irq_mask)
@@ -228,6 +268,20 @@ static void axi_dma_hw_init(struct axi_dma_chip *chip)
 	ret = dma_set_mask_and_coherent(chip->dev, DMA_BIT_MASK(64));
 	if (ret)
 		dev_warn(chip->dev, "Unable to set coherent mask\n");
+
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	if (of_node_name_prefix(chip->dev->of_node, "dma-controller-hsp"))
+	{
+		eswin_dma_sid_cfg(chip->dev);
+	}
+	else
+	{
+		eic7700_aon_sid_cfg(chip->dev);
+	}
+
+	/* TBU power up */
+	eic7700_tbu_power(chip->dev, true);
+#endif
 }
 
 static u32 axi_chan_get_xfer_width(struct axi_dma_chan *chan, dma_addr_t src,
@@ -455,6 +509,9 @@ static void dma_chan_issue_pending(struct dma_chan *dchan)
 	unsigned long flags;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	chan->trigger = true;
+#endif
 	if (vchan_issue_pending(&chan->vc))
 		axi_chan_start_first_queued(chan);
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
@@ -575,25 +632,60 @@ static void write_desc_dar(struct axi_dma_hw_desc *desc, dma_addr_t adr)
 	desc->lli->dar = cpu_to_le64(adr);
 }
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+static void set_desc_src_master(struct axi_dma_hw_desc *desc, struct axi_dma_chan *chan)
+#else
 static void set_desc_src_master(struct axi_dma_hw_desc *desc)
+#endif
 {
 	u32 val;
 
 	/* Select AXI0 for source master */
 	val = le32_to_cpu(desc->lli->ctl_lo);
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	if (chan->chip->dw->hdata->nr_masters > 1)
+	{
+		if(DMA_DEV_TO_MEM == chan->direction || DMA_DEV_TO_DEV == chan->direction) {
+			val |= CH_CTL_L_SRC_MAST;
+		}
+		else
+		{
+			val &= ~CH_CTL_L_SRC_MAST;
+		}
+	}
+	else
+		val &= ~CH_CTL_L_SRC_MAST;
+#else
 	val &= ~CH_CTL_L_SRC_MAST;
+#endif
 	desc->lli->ctl_lo = cpu_to_le32(val);
 }
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+static void set_desc_dest_master(struct axi_dma_hw_desc *hw_desc,
+				 struct axi_dma_chan *chan)
+#else
 static void set_desc_dest_master(struct axi_dma_hw_desc *hw_desc,
 				 struct axi_dma_desc *desc)
+#endif
 {
 	u32 val;
 
 	/* Select AXI1 for source master if available */
 	val = le32_to_cpu(hw_desc->lli->ctl_lo);
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	if (chan->chip->dw->hdata->nr_masters > 1)
+	{
+		if(DMA_MEM_TO_DEV == chan->direction || DMA_DEV_TO_DEV == chan->direction) {
+			val |= CH_CTL_L_DST_MAST;
+		}
+		else
+			val &= ~CH_CTL_L_DST_MAST;
+	}
+#else
 	if (desc->chan->chip->dw->hdata->nr_masters > 1)
 		val |= CH_CTL_L_DST_MAST;
+#endif
 	else
 		val &= ~CH_CTL_L_DST_MAST;
 
@@ -612,6 +704,10 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 	size_t block_ts;
 	u32 ctllo, ctlhi;
 	u32 burst_len;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	u32 src_maxburst;
+	u32 dst_maxburst;
+#endif
 
 	axi_block_ts = chan->chip->dw->hdata->block_size[chan->id];
 
@@ -624,6 +720,11 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 		return -EINVAL;
 	}
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	src_maxburst = chan->chip->dw->hdata->max_msize;
+	dst_maxburst  = chan->chip->dw->hdata->max_msize;
+#endif
+
 	switch (chan->direction) {
 	case DMA_MEM_TO_DEV:
 		reg_width = __ffs(chan->config.dst_addr_width);
@@ -632,6 +733,9 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 			mem_width << CH_CTL_L_SRC_WIDTH_POS |
 			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS |
 			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+		dst_maxburst = chan->config.dst_maxburst;
+#endif
 		block_ts = len >> mem_width;
 		break;
 	case DMA_DEV_TO_MEM:
@@ -641,6 +745,9 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 			mem_width << CH_CTL_L_DST_WIDTH_POS |
 			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |
 			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+		src_maxburst = chan->config.src_maxburst;
+#endif
 		block_ts = len >> reg_width;
 		break;
 	default:
@@ -674,12 +781,27 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 	}
 
 	hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1);
-
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	if(is_power_of_2(dst_maxburst) && is_power_of_2(src_maxburst))
+	{
+		dst_maxburst = order_base_2(dst_maxburst)? order_base_2(dst_maxburst) - 1 : 0;
+		src_maxburst = order_base_2(src_maxburst)? order_base_2(src_maxburst) - 1 : 0;
+	}else
+		dev_err(chan->chip->dev, "dst_burst or src_burst error!\n");
+	ctllo |= dst_maxburst << CH_CTL_L_DST_MSIZE_POS |
+		 src_maxburst << CH_CTL_L_SRC_MSIZE_POS;
+#else
 	ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
-		 DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS;
+	         DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS;
+#endif
 	hw_desc->lli->ctl_lo = cpu_to_le32(ctllo);
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	set_desc_src_master(hw_desc, chan);
+	set_desc_dest_master(hw_desc, chan);
+#else
 	set_desc_src_master(hw_desc);
+#endif
 
 	hw_desc->len = len;
 	return 0;
@@ -880,6 +1002,9 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 	struct axi_dma_hw_desc *hw_desc = NULL;
 	struct axi_dma_desc *desc = NULL;
 	u32 xfer_width, reg, num;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	u32 max_burst_len;
+#endif
 	u64 llp = 0;
 	u8 lms = 0; /* Select AXI0 master for LLI fetching */
 
@@ -887,6 +1012,12 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 		axi_chan_name(chan), &src_adr, &dst_adr, len, flags);
 
 	max_block_ts = chan->chip->dw->hdata->block_size[chan->id];
+
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	max_burst_len = chan->chip->dw->hdata->max_msize;
+	max_burst_len = order_base_2(max_burst_len)? order_base_2(max_burst_len) - 1 : 0;
+#endif
+
 	xfer_width = axi_chan_get_xfer_width(chan, src_adr, dst_adr, len);
 	num = DIV_ROUND_UP(len, max_block_ts << xfer_width);
 	desc = axi_desc_alloc(num);
@@ -937,17 +1068,26 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 		}
 		hw_desc->lli->ctl_hi = cpu_to_le32(reg);
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+		reg = (max_burst_len << CH_CTL_L_DST_MSIZE_POS |
+		       max_burst_len << CH_CTL_L_SRC_MSIZE_POS |
+#else
 		reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
 		       DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS |
+#endif
 		       xfer_width << CH_CTL_L_DST_WIDTH_POS |
 		       xfer_width << CH_CTL_L_SRC_WIDTH_POS |
 		       DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |
 		       DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS);
 		hw_desc->lli->ctl_lo = cpu_to_le32(reg);
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+		set_desc_src_master(hw_desc, chan);
+		set_desc_dest_master(hw_desc, chan);
+#else
 		set_desc_src_master(hw_desc);
 		set_desc_dest_master(hw_desc, desc);
-
+#endif
 		hw_desc->len = xfer_len;
 		desc->length += hw_desc->len;
 		/* update the length and addresses for the next loop cycle */
@@ -1157,7 +1297,9 @@ static int dma_chan_terminate_all(struct dma_chan *dchan)
 	spin_lock_irqsave(&chan->vc.lock, flags);
 
 	vchan_get_all_descriptors(&chan->vc, &head);
-
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	chan->trigger = false;
+#endif
 	chan->cyclic = false;
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 
@@ -1246,8 +1388,9 @@ static int axi_dma_suspend(struct axi_dma_chip *chip)
 	axi_dma_disable(chip);
 
 	clk_disable_unprepare(chip->core_clk);
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 	clk_disable_unprepare(chip->cfgr_clk);
-
+#endif
 	return 0;
 }
 
@@ -1255,9 +1398,11 @@ static int axi_dma_resume(struct axi_dma_chip *chip)
 {
 	int ret;
 
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 	ret = clk_prepare_enable(chip->cfgr_clk);
 	if (ret < 0)
 		return ret;
+#endif
 
 	ret = clk_prepare_enable(chip->core_clk);
 	if (ret < 0)
@@ -1283,6 +1428,43 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev)
 	return axi_dma_resume(chip);
 }
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+int eic7700_dma_sel_cfg(struct axi_dma_chan *chan, u32 val)
+{
+	struct axi_dma_chip *chip = chan->chip;
+	struct device *dev = chan->chip->dev;
+	int ret = 0;
+	struct regmap *regmap;
+	int dma_sel_reg;
+	u32 dma_sel = 0;
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "No eswin,syscfg phandle specified\n");
+		return -1;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg", 2,
+					&dma_sel_reg);
+	if (ret) {
+		dev_err(dev, "can't get sid cfg reg offset in sys_con(errno:%d)\n", ret);
+		return ret;
+	}
+	regmap_read(regmap, dma_sel_reg, &dma_sel);
+
+	if (of_node_name_prefix(chip->dev->of_node, "dma-controller-hsp")) {
+		if (val < 32)
+			dma_sel &= ~(1 << val);
+	}
+	else {
+		if (val < 32)
+			dma_sel |= (1 << val);
+	}
+	regmap_write(regmap, dma_sel_reg, dma_sel);
+	return 0;
+}
+#endif
+
 static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec,
 					    struct of_dma *ofdma)
 {
@@ -1296,6 +1478,10 @@ static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec,
 
 	chan = dchan_to_axi_dma_chan(dchan);
 	chan->hw_handshake_num = dma_spec->args[0];
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	if (dma_spec->args_count > 1)
+		eic7700_dma_sel_cfg(chan, dma_spec->args[1]);
+#endif
 	return dchan;
 }
 
@@ -1366,6 +1552,20 @@ static int parse_device_properties(struct axi_dma_chip *chip)
 		chip->dw->hdata->axi_rw_burst_len = tmp;
 	}
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	/* max-size is optional property */
+	ret = device_property_read_u32(dev, "snps,max-msize", &tmp);
+	if (!ret) {
+		if (tmp > 1024)
+			return -EINVAL;
+		if (tmp < 1)
+			return -EINVAL;
+
+		chip->dw->hdata->max_msize = tmp;
+	}else
+		chip->dw->hdata->max_msize = 4;
+#endif
+
 	return 0;
 }
 
@@ -1403,6 +1603,8 @@ static int dw_probe(struct platform_device *pdev)
 	if (IS_ERR(chip->regs))
 		return PTR_ERR(chip->regs);
 
+	spin_lock_init(&chip->lock);
+
 	flags = (uintptr_t)of_device_get_match_data(&pdev->dev);
 	if (flags & AXI_DMA_FLAG_HAS_APB_REGS) {
 		chip->apb_regs = devm_platform_ioremap_resource(pdev, 1);
@@ -1426,10 +1628,31 @@ static int dw_probe(struct platform_device *pdev)
 	if (IS_ERR(chip->core_clk))
 		return PTR_ERR(chip->core_clk);
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	chip->arst = devm_reset_control_get_optional(&pdev->dev, "arst");
+	if (IS_ERR_OR_NULL(chip->arst)) {
+		dev_err(chip->dev, "Failed to get axi reset handle\n");
+		return -EFAULT;
+	}
+	reset_control_deassert(chip->arst);
+
+	chip->prst = devm_reset_control_get_optional(&pdev->dev, "prst");
+	if (IS_ERR_OR_NULL(chip->prst)) {
+		dev_err(chip->dev, "Failed to get ahb reset handle\n");
+		return -EFAULT;
+	}
+	reset_control_deassert(chip->prst);
+#endif
+
+	/*
+	  For EIC7700 SoC, aon dma has separate cfg clk register bit while dma0 havn't.
+	  Since dma cfg clk is default on, so we don't control it
+	*/
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 	chip->cfgr_clk = devm_clk_get(chip->dev, "cfgr-clk");
 	if (IS_ERR(chip->cfgr_clk))
 		return PTR_ERR(chip->cfgr_clk);
-
+#endif
 	ret = parse_device_properties(chip);
 	if (ret)
 		return ret;
@@ -1462,8 +1685,17 @@ static int dw_probe(struct platform_device *pdev)
 	dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
 	dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask);
 
+	if (of_node_name_prefix(chip->dev->of_node, "dma-controller-aon")) {
+		dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
+	}
+
 	/* DMA capabilities */
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	dw->dma.chancnt = hdata->nr_channels;
+	dw->dma.max_burst = hdata->max_msize;
+#else
 	dw->dma.max_burst = hdata->axi_rw_burst_len;
+#endif
 	dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS;
 	dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS;
 	dw->dma.directions = BIT(DMA_MEM_TO_MEM);
@@ -1533,6 +1765,53 @@ static int dw_probe(struct platform_device *pdev)
 	return ret;
 }
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+static int eswin_dma_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_dma_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev, "dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+					&hsp_mmu_dma_reg);
+	if (ret) {
+		dev_err(dev, "can't get dma sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_dma_reg, rdwr_sid_ssid);
+
+	ret = eic7700_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config dma streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config dma streamID(%d)!\n", sid);
+
+	return ret;
+}
+#endif
+
 static int dw_remove(struct platform_device *pdev)
 {
 	struct axi_dma_chip *chip = platform_get_drvdata(pdev);
@@ -1541,7 +1820,9 @@ static int dw_remove(struct platform_device *pdev)
 	u32 i;
 
 	/* Enable clk before accessing to registers */
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 	clk_prepare_enable(chip->cfgr_clk);
+#endif
 	clk_prepare_enable(chip->core_clk);
 	axi_dma_irq_disable(chip);
 	for (i = 0; i < dw->hdata->nr_channels; i++) {
@@ -1559,10 +1840,20 @@ static int dw_remove(struct platform_device *pdev)
 
 	list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
 			vc.chan.device_node) {
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 		list_del(&chan->vc.chan.device_node);
+#endif
 		tasklet_kill(&chan->vc.task);
 	}
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	/* TBU power down before reset */
+	eic7700_tbu_power(chip->dev, false);
+
+	reset_control_assert(chip->arst);
+	reset_control_assert(chip->prst);
+#endif
+
 	return 0;
 }
 
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
index 8521530a34ec..7af0587384a0 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
@@ -15,7 +15,9 @@
 #include <linux/device.h>
 #include <linux/dmaengine.h>
 #include <linux/types.h>
-
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+#include <linux/reset.h>
+#endif
 #include "../virt-dma.h"
 
 #define DMAC_MAX_CHANNELS	16
@@ -30,6 +32,9 @@ struct dw_axi_dma_hcfg {
 	u32	priority[DMAC_MAX_CHANNELS];
 	/* maximum supported axi burst length */
 	u32	axi_rw_burst_len;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	u32	max_msize;
+#endif
 	/* Register map for DMAX_NUM_CHANNELS <= 8 */
 	bool	reg_map_8_channels;
 	bool	restrict_axi_burst_len;
@@ -52,6 +57,9 @@ struct axi_dma_chan {
 	bool				cyclic;
 	/* these other elements are all protected by vc.lock */
 	bool				is_paused;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	bool				trigger;
+#endif
 };
 
 struct dw_axi_dma {
@@ -71,6 +79,11 @@ struct axi_dma_chip {
 	struct clk		*core_clk;
 	struct clk		*cfgr_clk;
 	struct dw_axi_dma	*dw;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	struct reset_control	*arst;
+	struct reset_control	*prst;
+#endif
+	spinlock_t lock;
 };
 
 /* LLI == Linked List Item */
@@ -283,6 +296,10 @@ enum {
 #define CH_CTL_L_SRC_MAST		BIT(0)
 
 /* CH_CFG_H */
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+#define CH_CFG_H_DST_OSR_LMT_POS	27
+#define CH_CFG_H_SRC_OSR_LMT_POS	23
+#endif
 #define CH_CFG_H_PRIORITY_POS		17
 #define CH_CFG_H_DST_PER_POS		12
 #define CH_CFG_H_SRC_PER_POS		7
@@ -322,7 +339,13 @@ enum {
 #define CH_CFG2_H_TT_FC_POS		0
 #define CH_CFG2_H_HS_SEL_SRC_POS	3
 #define CH_CFG2_H_HS_SEL_DST_POS	4
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+#define CH_CFG2_H_PRIORITY_POS		15
+#define CH_CFG2_H_SRC_OSR_LMT_POS	23
+#define CH_CFG2_H_DST_OSR_LMT_POS	27
+#else
 #define CH_CFG2_H_PRIORITY_POS		20
+#endif
 
 /**
  * DW AXI DMA channel interrupts
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index ec4abf9ff47b..e24445697aa7 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -388,6 +388,10 @@ source "drivers/gpu/drm/solomon/Kconfig"
 
 source "drivers/gpu/drm/sprd/Kconfig"
 
+source "drivers/gpu/drm/eswin/Kconfig"
+
+source "drivers/gpu/drm/img/Kconfig"
+
 config DRM_HYPERV
 	tristate "DRM Support for Hyper-V synthetic video device"
 	depends on DRM && PCI && MMU && HYPERV
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 215e78e79125..6067543def36 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -198,3 +198,5 @@ obj-$(CONFIG_DRM_HYPERV) += hyperv/
 obj-y			+= solomon/
 obj-$(CONFIG_DRM_SPRD) += sprd/
 obj-$(CONFIG_DRM_LOONGSON) += loongson/
+obj-$(CONFIG_DRM_ESWIN) += eswin/
+obj-$(CONFIG_DRM_IMG) += img/
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 901d1961b739..5fcd4f778dc3 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -8,7 +8,7 @@ config DRM_AMD_DC
 	depends on BROKEN || !CC_IS_CLANG || ARM64 || RISCV || SPARC64 || X86_64
 	select SND_HDA_COMPONENT if SND_HDA_CORE
 	# !CC_IS_CLANG: https://github.com/ClangBuiltLinux/linux/issues/1752
-	select DRM_AMD_DC_FP if (X86 || LOONGARCH || (PPC64 && ALTIVEC) || (ARM64 && KERNEL_MODE_NEON && !CC_IS_CLANG))
+	select DRM_AMD_DC_FP if ARCH_HAS_KERNEL_FPU_SUPPORT && (!ARM64 || !CC_IS_CLANG)
 	help
 	  Choose this option if you want to use the new display engine
 	  support for AMDGPU. This adds required support for Vega and
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
index 172aa10a8800..e46f8ce41d87 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
@@ -26,16 +26,7 @@
 
 #include "dc_trace.h"
 
-#if defined(CONFIG_X86)
-#include <asm/fpu/api.h>
-#elif defined(CONFIG_PPC64)
-#include <asm/switch_to.h>
-#include <asm/cputable.h>
-#elif defined(CONFIG_ARM64)
-#include <asm/neon.h>
-#elif defined(CONFIG_LOONGARCH)
-#include <asm/fpu.h>
-#endif
+#include <linux/fpu.h>
 
 /**
  * DOC: DC FPU manipulation overview
@@ -60,11 +51,9 @@ static DEFINE_PER_CPU(int, fpu_recursion_depth);
  */
 inline void dc_assert_fp_enabled(void)
 {
-	int *pcpu, depth = 0;
+	int depth;
 
-	pcpu = get_cpu_ptr(&fpu_recursion_depth);
-	depth = *pcpu;
-	put_cpu_ptr(&fpu_recursion_depth);
+	depth = __this_cpu_read(fpu_recursion_depth);
 
 	ASSERT(depth >= 1);
 }
@@ -84,33 +73,17 @@ inline void dc_assert_fp_enabled(void)
  */
 void dc_fpu_begin(const char *function_name, const int line)
 {
-	int *pcpu;
+	int depth;
 
-	pcpu = get_cpu_ptr(&fpu_recursion_depth);
-	*pcpu += 1;
-
-	if (*pcpu == 1) {
-#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH)
-		migrate_disable();
+	WARN_ON_ONCE(!in_task());
+	preempt_disable();
+	depth = __this_cpu_inc_return(fpu_recursion_depth);
+	if (depth == 1) {
+		BUG_ON(!kernel_fpu_available());
 		kernel_fpu_begin();
-#elif defined(CONFIG_PPC64)
-		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
-			preempt_disable();
-			enable_kernel_vsx();
-		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
-			preempt_disable();
-			enable_kernel_altivec();
-		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
-			preempt_disable();
-			enable_kernel_fp();
-		}
-#elif defined(CONFIG_ARM64)
-		kernel_neon_begin();
-#endif
 	}
 
-	TRACE_DCN_FPU(true, function_name, line, *pcpu);
-	put_cpu_ptr(&fpu_recursion_depth);
+	TRACE_DCN_FPU(true, function_name, line, depth);
 }
 
 /**
@@ -125,30 +98,15 @@ void dc_fpu_begin(const char *function_name, const int line)
  */
 void dc_fpu_end(const char *function_name, const int line)
 {
-	int *pcpu;
+	int depth;
 
-	pcpu = get_cpu_ptr(&fpu_recursion_depth);
-	*pcpu -= 1;
-	if (*pcpu <= 0) {
-#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH)
+	depth = __this_cpu_dec_return(fpu_recursion_depth);
+	if (depth == 0) {
 		kernel_fpu_end();
-		migrate_enable();
-#elif defined(CONFIG_PPC64)
-		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
-			disable_kernel_vsx();
-			preempt_enable();
-		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
-			disable_kernel_altivec();
-			preempt_enable();
-		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
-			disable_kernel_fp();
-			preempt_enable();
-		}
-#elif defined(CONFIG_ARM64)
-		kernel_neon_end();
-#endif
+	} else {
+		WARN_ON_ONCE(depth < 0);
 	}
 
-	TRACE_DCN_FPU(false, function_name, line, *pcpu);
-	put_cpu_ptr(&fpu_recursion_depth);
+	TRACE_DCN_FPU(false, function_name, line, depth);
+	preempt_enable();
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
index 0ba9a7997d56..2d04de698dd6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
@@ -25,40 +25,8 @@
 # It provides the general basic services required by other DAL
 # subcomponents.
 
-ifdef CONFIG_X86
-dml_ccflags-$(CONFIG_CC_IS_GCC) := -mhard-float
-dml_ccflags := $(dml_ccflags-y) -msse
-endif
-
-ifdef CONFIG_PPC64
-dml_ccflags := -mhard-float -maltivec
-endif
-
-ifdef CONFIG_ARM64
-dml_rcflags := -mgeneral-regs-only
-endif
-
-ifdef CONFIG_LOONGARCH
-dml_ccflags := -mfpu=64
-dml_rcflags := -msoft-float
-endif
-
-ifdef CONFIG_CC_IS_GCC
-ifneq ($(call gcc-min-version, 70100),y)
-IS_OLD_GCC = 1
-endif
-endif
-
-ifdef CONFIG_X86
-ifdef IS_OLD_GCC
-# Stack alignment mismatch, proceed with caution.
-# GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3
-# (8B stack alignment).
-dml_ccflags += -mpreferred-stack-boundary=4
-else
-dml_ccflags += -msse2
-endif
-endif
+dml_ccflags := $(CC_FLAGS_FPU)
+dml_rcflags := $(CC_FLAGS_NO_FPU)
 
 ifneq ($(CONFIG_FRAME_WARN),0)
 ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y)
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
index 04d4a1a10698..3f3bf54f5da3 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
@@ -29,6 +29,7 @@
 #include <drm/drm_print.h>
 
 #define HWVER_131			0x31333100	/* IP version 1.31 */
+#define HWVER_30313500			0x30313500
 
 #define DSI_VERSION			0x00
 #define VERSION				GENMASK(31, 8)
@@ -781,7 +782,7 @@ static void dw_mipi_dsi_dphy_timing_config(struct dw_mipi_dsi *dsi)
 
 	hw_version = dsi_read(dsi, DSI_VERSION) & VERSION;
 
-	if (hw_version >= HWVER_131) {
+	if (hw_version >= HWVER_131 || hw_version == HWVER_30313500)  {
 		dsi_write(dsi, DSI_PHY_TMR_CFG,
 			  PHY_HS2LP_TIME_V131(timing.data_hs2lp) |
 			  PHY_LP2HS_TIME_V131(timing.data_lp2hs));
diff --git a/drivers/gpu/drm/drm_gem_vram_helper.c b/drivers/gpu/drm/drm_gem_vram_helper.c
index b67eafa55715..aaf97dde9b73 100644
--- a/drivers/gpu/drm/drm_gem_vram_helper.c
+++ b/drivers/gpu/drm/drm_gem_vram_helper.c
@@ -870,7 +870,11 @@ static struct ttm_tt *bo_driver_ttm_tt_create(struct ttm_buffer_object *bo,
 	if (!tt)
 		return NULL;
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	ret = ttm_tt_init(tt, bo, page_flags, ttm_write_combined, 0);
+#else
 	ret = ttm_tt_init(tt, bo, page_flags, ttm_cached, 0);
+#endif
 	if (ret < 0)
 		goto err_ttm_tt_init;
 
diff --git a/drivers/gpu/drm/eswin/Kconfig b/drivers/gpu/drm/eswin/Kconfig
new file mode 100644
index 000000000000..77733cd77352
--- /dev/null
+++ b/drivers/gpu/drm/eswin/Kconfig
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config DRM_ESWIN
+	tristate "DRM Support for Eswin"
+	depends on DRM
+	select DRM_KMS_HELPER
+	help
+	  Choose this option if you have a Eswin soc chipset.
+	  This driver provides Eswin kernel mode
+	  setting and buffer management. It does not
+	  provide 2D or 3D acceleration.
+
+config ESWIN_VIRTUAL_DISPLAY
+	bool "display content output to debugfs file"
+	depends on DRM_ESWIN
+	select DEBUG_FS
+	help
+	  This is a debug feature which capture video content output
+	  from display controller. Output path is debugfs/dri/connector/.
+	  The content format is ARGB which Alpha is 0 for 8bits.
+	  Disabled in default.
+
+config ESWIN_MMU
+	bool "MMU support for Eswin display controller"
+	depends on DRM_ESWIN
+	help
+	  This is a memory management function which translate virtual address
+	  to physical address. DPU MMU only do address translate, doesn't
+	  support security and shareable.
+
+config ESWIN_DW_HDMI
+    bool "ESWIN specific extensions for Synopsys DW HDMI"
+    depends on DRM_ESWIN
+    select CEC_CORE if CEC_NOTIFIER
+    select DRM_DISPLAY_HDMI_HELPER
+    select DRM_DISPLAY_HELPER
+    select DRM_KMS_HELPER
+    help
+      This selects support for ESWIN SoC specific extensions
+      for the Synopsys DesignWare HDMI driver. If you want to
+      enable HDMI on win2030 based SoC, you should select
+      this option.
+
+config DW_HDMI_I2S_AUDIO
+    bool "Synopsys Designware I2S Audio interface"
+    depends on SND_SOC
+    depends on ESWIN_DW_HDMI
+    select SND_SOC_HDMI_CODEC
+    help
+      Support the I2S Audio interface which is part of the Synopsys
+      Designware HDMI block.
+
+config DW_HDMI_CEC
+    bool "Synopsis Designware CEC interface"
+    depends on ESWIN_DW_HDMI
+    select CEC_CORE
+    select CEC_NOTIFIER
+    help
+      Support the CEC interface which is part of the Synopsys
+      Designware HDMI block.
+
+config DW_HDMI_HDCP
+    bool "Synopsis Designware HDCP interface"
+    depends on ESWIN_DW_HDMI
+    help
+      Support the HDCP interface which is part of the Synopsys
+      Designware HDMI block.
+
+config DW_HDMI_HDCP2
+    tristate "Synopsis Designware HDCP2 interface"
+    depends on DW_HDMI_HDCP
+    help
+      Support the HDCP2 interface which is part of the Synopsys
+      Designware HDMI block.
+
+config DW_HDMI_HDCP1X_ENABLED
+    bool "Synopsis Designware HDCP1X enabled"
+    depends on DW_HDMI_HDCP
+    help
+      Enable/Disable HDCP1X of the Synopsys Designware HDMI block.
+
+
+config ESWIN_MIPI_DSI
+    tristate "Eswin mipi dsi and 1080p panel driver"
+    select DRM_ESWIN
+    select DRM_DW_MIPI_DSI
+    help
+      Support mipi dsi 4-lanes panel output
diff --git a/drivers/gpu/drm/eswin/Makefile b/drivers/gpu/drm/eswin/Makefile
new file mode 100644
index 000000000000..6d4043ca833c
--- /dev/null
+++ b/drivers/gpu/drm/eswin/Makefile
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: GPL-2.0
+
+es_drm-objs := es_dc_hw.o \
+		es_dc.o \
+		es_crtc.o \
+		es_drv.o \
+		es_fb.o \
+		es_gem.o \
+		es_plane.o \
+		es_simple_enc.o
+
+es_drm-$(CONFIG_ESWIN_VIRTUAL_DISPLAY) += es_virtual.o
+es_drm-$(CONFIG_ESWIN_MMU) += es_dc_mmu.o
+es_drm-$(CONFIG_ESWIN_DW_HDMI) += eswin_dw_hdmi.o dw-hdmi.o
+
+es_drm-$(CONFIG_DW_HDMI_I2S_AUDIO) += dw_hdmi_i2s_audio.o
+es_drm-$(CONFIG_DW_HDMI_CEC) += dw_hdmi_cec.o
+es_drm-$(CONFIG_DW_HDMI_HDCP) += dw_hdmi_hdcp.o
+
+dw_hdcp2-$(CONFIG_DW_HDMI_HDCP2) += dw_hdmi_hdcp2.o
+
+obj-$(CONFIG_DRM_ESWIN) += es_drm.o
+obj-$(CONFIG_DW_HDMI_HDCP2) += dw_hdcp2.o
+
+obj-$(CONFIG_ESWIN_MIPI_DSI)+= es_mipi_dsi.o es_panel.o
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/dw-hdmi.c b/drivers/gpu/drm/eswin/dw-hdmi.c
new file mode 100644
index 000000000000..d1d66508f965
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw-hdmi.c
@@ -0,0 +1,4598 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * DesignWare High-Definition Multimedia Interface (HDMI) driver
+ *
+ * Copyright (C) 2013-2015 Mentor Graphics Inc.
+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc.
+ * Copyright (C) 2010, Guennadi Liakhovetski <g.liakhovetski@gmx.de>
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/hdmi.h>
+#include <linux/i2c.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/regmap.h>
+#include <linux/dma-mapping.h>
+#include <linux/spinlock.h>
+
+#include <media/cec-notifier.h>
+
+#include <linux/media-bus-format.h>
+#include <linux/videodev2.h>
+#include <linux/reset.h>
+
+#include <drm/bridge/dw_hdmi.h>
+#include <drm/display/drm_hdmi_helper.h>
+#include <drm/display/drm_scdc_helper.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_of.h>
+#include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
+
+#include "dw_hdmi_audio.h"
+#include "dw_hdmi_cec.h"
+#include "dw-hdmi.h"
+#include "dw_hdmi_hdcp.h"
+
+#define DDC_CI_ADDR 0x37
+#define DDC_SEGMENT_ADDR 0x30
+
+#define HDMI_EDID_LEN 512
+
+/* DW-HDMI Controller >= 0x200a are at least compliant with SCDC version 1 */
+#define SCDC_MIN_SOURCE_VERSION 0x1
+
+#define HDMI14_MAX_TMDSCLK 340000000
+
+#define HDMI_CEC_CLK 32768
+
+static bool hpd_flag = false;
+
+static const u16 csc_coeff_default[3][4] = {
+	{ 0x2000, 0x0000, 0x0000, 0x0000 },
+	{ 0x0000, 0x2000, 0x0000, 0x0000 },
+	{ 0x0000, 0x0000, 0x2000, 0x0000 }
+};
+
+static const u16 csc_coeff_rgb_out_eitu601[3][4] = {
+	{ 0x2000, 0x6926, 0x74fd, 0x010e },
+	{ 0x2000, 0x2cdd, 0x0000, 0x7e9a },
+	{ 0x2000, 0x0000, 0x38b4, 0x7e3b }
+};
+
+static const u16 csc_coeff_rgb_out_eitu709[3][4] = {
+	{ 0x2000, 0x7106, 0x7a02, 0x00a7 },
+	{ 0x2000, 0x3264, 0x0000, 0x7e6d },
+	{ 0x2000, 0x0000, 0x3b61, 0x7e25 }
+};
+
+static const u16 csc_coeff_rgb_in_eitu601[3][4] = {
+	{ 0x2591, 0x1322, 0x074b, 0x0000 },
+	{ 0x6535, 0x2000, 0x7acc, 0x0200 },
+	{ 0x6acd, 0x7534, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_in_eitu709[3][4] = {
+	{ 0x2dc5, 0x0d9b, 0x049e, 0x0000 },
+	{ 0x62f0, 0x2000, 0x7d11, 0x0200 },
+	{ 0x6756, 0x78ab, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_full_to_rgb_limited[3][4] = {
+	{ 0x1b7c, 0x0000, 0x0000, 0x0020 },
+	{ 0x0000, 0x1b7c, 0x0000, 0x0020 },
+	{ 0x0000, 0x0000, 0x1b7c, 0x0020 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_8bit_eitu601[3][4] = {
+	{ 0x2591, 0x1323, 0x074c, 0x0000 },
+	{ 0xe534, 0x2000, 0xfacc, 0x0200 },
+	{ 0xeacd, 0xf533, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_8bit_eitu709[3][4] = {
+	{ 0x2dc6, 0x0d9b, 0x049f, 0x0000 },
+	{ 0xe2ef, 0x2000, 0xfd11, 0x0200 },
+	{ 0xe755, 0xf8ab, 0x2000, 0x0200 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_10bit_eitu601[3][4] = {
+	{ 0x2591, 0x1323, 0x074c, 0x0000 },
+	{ 0xe534, 0x2000, 0xfacc, 0x0800 },
+	{ 0xeacd, 0xf533, 0x2000, 0x0800 }
+};
+
+static const u16 csc_coeff_rgb_to_yuv_10bit_eitu709[3][4] = {
+	{ 0x2dc6, 0x0d9b, 0x049f, 0x0000 },
+	{ 0xe2ef, 0x2000, 0xfd11, 0x0800 },
+	{ 0xe755, 0xf8ab, 0x2000, 0x0800 }
+};
+
+struct hdmi_vmode {
+	bool mdataenablepolarity;
+
+	unsigned int mpixelclock;
+	unsigned int mpixelrepetitioninput;
+	unsigned int mpixelrepetitionoutput;
+	unsigned int mtmdsclock;
+};
+
+struct hdmi_data_info {
+	unsigned int enc_in_bus_format;
+	unsigned int enc_out_bus_format;
+	unsigned int enc_in_encoding;
+	unsigned int enc_out_encoding;
+	unsigned int pix_repet_factor;
+	unsigned int hdcp_enable;
+	struct hdmi_vmode video_mode;
+	bool rgb_limited_range;
+};
+
+struct dw_hdmi_i2c {
+	struct i2c_adapter	adap;
+
+	struct mutex		lock;	/* used to serialize data transfers */
+	struct completion	cmp;
+	u8			stat;
+
+	u8			slave_reg;
+	bool			is_regaddr;
+	bool			is_segment;
+};
+
+struct dw_hdmi_phy_data {
+	enum dw_hdmi_phy_type type;
+	const char *name;
+	unsigned int gen;
+	bool has_svsret;
+	int (*configure)(struct dw_hdmi *hdmi,
+			 const struct dw_hdmi_plat_data *pdata,
+			 unsigned long mpixelclock);
+};
+
+struct dw_hdmi {
+	struct drm_connector connector;
+	struct drm_bridge bridge;
+	struct drm_bridge *next_bridge;
+	unsigned int version;
+
+	struct platform_device *hdcp_dev;
+	struct platform_device *audio;
+	struct platform_device *cec;
+	struct device *dev;
+	struct clk *isfr_clk;
+	struct clk *iahb_clk;
+	struct clk *cec_clk;
+	struct dw_hdmi_i2c *i2c;
+
+	struct hdmi_data_info hdmi_data;
+	const struct dw_hdmi_plat_data *plat_data;
+	struct dw_hdcp *hdcp;
+	int vic;
+
+	u8 edid[HDMI_EDID_LEN];
+
+	struct {
+		const struct dw_hdmi_phy_ops *ops;
+		const char *name;
+		void *data;
+		bool enabled;
+	} phy;
+
+	struct drm_display_mode previous_mode;
+
+	struct i2c_adapter *ddc;
+	void __iomem *regs;
+	bool sink_is_hdmi;
+	bool sink_has_audio;
+	bool hpd_state;
+
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *default_state;
+	struct pinctrl_state *unwedge_state;
+
+	struct mutex mutex;		/* for state below and previous_mode */
+	enum drm_connector_force force;	/* mutex-protected force state */
+	struct drm_connector *curr_conn;/* current connector (only valid when !disabled) */
+	bool disabled;			/* DRM has disabled our bridge */
+	bool bridge_is_on;		/* indicates the bridge is on */
+	bool rxsense;			/* rxsense state */
+	u8 phy_mask;			/* desired phy int mask settings */
+	u8 mc_clkdis;			/* clock disable register */
+
+	spinlock_t audio_lock;
+	struct mutex audio_mutex;
+	unsigned int sample_non_pcm;
+	unsigned int sample_width;
+	unsigned int sample_rate;
+	unsigned int channels;
+	unsigned int audio_cts;
+	unsigned int audio_n;
+	bool audio_enable;
+	struct dentry *debugfs_dir;
+
+	unsigned int reg_shift;
+	struct regmap *regm;
+	void (*enable_audio)(struct dw_hdmi *hdmi);
+	void (*disable_audio)(struct dw_hdmi *hdmi);
+
+	struct mutex cec_notifier_mutex;
+	struct cec_notifier *cec_notifier;
+
+	hdmi_codec_plugged_cb plugged_cb;
+	struct device *codec_dev;
+	enum drm_connector_status last_connector_result;
+
+	int irq;
+	struct delayed_work work;
+	struct workqueue_struct *workqueue;
+	struct reset_control *rst_hdmi_prstn;
+	struct reset_control *rst_hdmi_phyrstn;
+	struct reset_control *rst_hdmi_rstn;
+};
+
+#define HDMI_IH_PHY_STAT0_RX_SENSE                                             \
+	(HDMI_IH_PHY_STAT0_RX_SENSE0 | HDMI_IH_PHY_STAT0_RX_SENSE1 |           \
+	 HDMI_IH_PHY_STAT0_RX_SENSE2 | HDMI_IH_PHY_STAT0_RX_SENSE3)
+
+#define HDMI_PHY_RX_SENSE \
+	(HDMI_PHY_RX_SENSE0 | HDMI_PHY_RX_SENSE1 | \
+	 HDMI_PHY_RX_SENSE2 | HDMI_PHY_RX_SENSE3)
+
+static struct edid *dw_hdmi_get_edid(struct dw_hdmi *hdmi,
+				     struct drm_connector *connector);
+static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi,
+						const struct drm_display_mode *mode);
+
+static inline void hdmi_writeb(struct dw_hdmi *hdmi, u8 val, int offset)
+{
+	regmap_write(hdmi->regm, offset << hdmi->reg_shift, val);
+}
+
+static inline u8 hdmi_readb(struct dw_hdmi *hdmi, int offset)
+{
+	unsigned int val = 0;
+
+	regmap_read(hdmi->regm, offset << hdmi->reg_shift, &val);
+
+	return val;
+}
+
+static void handle_plugged_change(struct dw_hdmi *hdmi, bool plugged)
+{
+	if (hdmi->plugged_cb && hdmi->codec_dev)
+		hdmi->plugged_cb(hdmi->codec_dev, plugged);
+}
+
+int dw_hdmi_set_plugged_cb(struct dw_hdmi *hdmi, hdmi_codec_plugged_cb fn,
+			   struct device *codec_dev)
+{
+	bool plugged;
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->plugged_cb = fn;
+	hdmi->codec_dev = codec_dev;
+	plugged = hdmi->last_connector_result == connector_status_connected;
+	handle_plugged_change(hdmi, plugged);
+	mutex_unlock(&hdmi->mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_plugged_cb);
+
+static void hdmi_modb(struct dw_hdmi *hdmi, u8 data, u8 mask, unsigned reg)
+{
+	regmap_update_bits(hdmi->regm, reg << hdmi->reg_shift, mask, data);
+}
+
+static void hdmi_mask_writeb(struct dw_hdmi *hdmi, u8 data, unsigned int reg,
+			     u8 shift, u8 mask)
+{
+	hdmi_modb(hdmi, data << shift, mask, reg);
+}
+
+static void repo_hpd_event(struct work_struct *p_work)
+{
+	struct dw_hdmi *hdmi = container_of(p_work, struct dw_hdmi, work.work);
+	u8 phy_stat = hdmi_readb(hdmi, HDMI_PHY_STAT0);
+
+	mutex_lock(&hdmi->mutex);
+	if (!(phy_stat & HDMI_PHY_RX_SENSE))
+		hdmi->rxsense = false;
+	if (phy_stat & HDMI_PHY_HPD)
+		hdmi->rxsense = true;
+	mutex_unlock(&hdmi->mutex);
+
+	if (hdmi->bridge.dev) {
+		drm_helper_hpd_irq_event(hdmi->bridge.dev);
+	}
+
+	msleep(150);
+	if (hdmi->hpd_state) {
+		if (hdmi->hdcp && hdmi->hdcp->hdcp2 &&
+		    hdmi->hdcp->hdcp2->enable &&
+		    (tv_hdmi_hdcp2_support(hdmi)) == 1) {
+			hdmi->hdcp->hdcp2->start();
+		} else {
+			hdmi_tx_hdcp_config(hdmi, &hdmi->previous_mode);
+		}
+	}
+}
+
+static bool check_hdmi_irq(struct dw_hdmi *hdmi, int intr_stat, int phy_int_pol)
+{
+	int msecs;
+
+	/* To determine whether interrupt type is HPD */
+	if (!(intr_stat & HDMI_IH_PHY_STAT0_HPD))
+		return false;
+
+	if (phy_int_pol & HDMI_PHY_HPD) {
+		dev_info(hdmi->dev, "dw hdmi plug in\n");
+		msecs = 100;
+		hdmi->hpd_state = true;
+	} else {
+		dev_info(hdmi->dev, "dw hdmi plug out\n");
+		msecs = 1000;
+		hdmi->hpd_state = false;
+		if (hdmi->hdcp && hdmi->hdcp->hdcp2 &&
+		    hdmi->hdcp->hdcp2->enable) {
+			hdmi->hdcp->hdcp2->stop();
+		}
+		if (hdmi->hdcp && hdmi->hdcp->hdcp_stop) {
+			hdmi->hdcp->hdcp_stop(hdmi->hdcp);
+		}
+	}
+	mod_delayed_work(hdmi->workqueue, &hdmi->work, msecs_to_jiffies(msecs));
+
+	return true;
+}
+
+static void init_hpd_work(struct dw_hdmi *hdmi)
+{
+	hdmi->workqueue = create_workqueue("hpd_queue");
+	INIT_DELAYED_WORK(&hdmi->work, repo_hpd_event);
+}
+
+static void dw_hdmi_i2c_set_divs(struct dw_hdmi *hdmi)
+{
+	unsigned long clk_rate_khz;
+	unsigned long low_ns, high_ns;
+	unsigned long div_low, div_high;
+
+	/* Standard-mode */
+	high_ns = 4708;
+	low_ns = 4916;
+
+	/* Adjust to avoid overflow */
+	clk_rate_khz = DIV_ROUND_UP(clk_get_rate(hdmi->isfr_clk), 1000);
+
+	div_low = (clk_rate_khz * low_ns) / 1000000;
+	if ((clk_rate_khz * low_ns) % 1000000)
+		div_low++;
+
+	div_high = (clk_rate_khz * high_ns) / 1000000;
+	if ((clk_rate_khz * high_ns) % 1000000)
+		div_high++;
+
+	/* Maximum divider supported by hw is 0xffff */
+	if (div_low > 0xffff)
+		div_low = 0xffff;
+
+	if (div_high > 0xffff)
+		div_high = 0xffff;
+
+	hdmi_writeb(hdmi, div_high & 0xff, HDMI_I2CM_SS_SCL_HCNT_0_ADDR);
+	hdmi_writeb(hdmi, (div_high >> 8) & 0xff, HDMI_I2CM_SS_SCL_HCNT_1_ADDR);
+	hdmi_writeb(hdmi, div_low & 0xff, HDMI_I2CM_SS_SCL_LCNT_0_ADDR);
+	hdmi_writeb(hdmi, (div_low >> 8) & 0xff, HDMI_I2CM_SS_SCL_LCNT_1_ADDR);
+}
+
+static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi)
+{
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL,
+		    HDMI_PHY_I2CM_INT_ADDR);
+
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_CTLINT_ADDR_NAC_POL |
+		    HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL,
+		    HDMI_PHY_I2CM_CTLINT_ADDR);
+
+	/* Software reset */
+	hdmi_writeb(hdmi, 0x00, HDMI_I2CM_SOFTRSTZ);
+
+	/* Set Standard Mode speed (determined to be 100KHz on iMX6) */
+	hdmi_writeb(hdmi, 0x00, HDMI_I2CM_DIV);
+
+	/* Set done, not acknowledged and arbitration interrupt polarities */
+	hdmi_writeb(hdmi, HDMI_I2CM_INT_DONE_POL, HDMI_I2CM_INT);
+	hdmi_writeb(hdmi, HDMI_I2CM_CTLINT_NAC_POL | HDMI_I2CM_CTLINT_ARB_POL,
+		    HDMI_I2CM_CTLINT);
+
+	/* Clear DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_I2CM_STAT0);
+
+	/* Mute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_MUTE_I2CM_STAT0);
+
+	/* set SDA high level holding time */
+	hdmi_writeb(hdmi, 0x48, HDMI_I2CM_SDA_HOLD);
+	dw_hdmi_i2c_set_divs(hdmi);
+}
+
+static bool dw_hdmi_i2c_unwedge(struct dw_hdmi *hdmi)
+{
+	/* If no unwedge state then give up */
+	if (!hdmi->unwedge_state)
+		return false;
+
+	dev_info(hdmi->dev, "Attempting to unwedge stuck i2c bus\n");
+
+	/*
+	 * This is a huge hack to workaround a problem where the dw_hdmi i2c
+	 * bus could sometimes get wedged.  Once wedged there doesn't appear
+	 * to be any way to unwedge it (including the HDMI_I2CM_SOFTRSTZ)
+	 * other than pulsing the SDA line.
+	 *
+	 * We appear to be able to pulse the SDA line (in the eyes of dw_hdmi)
+	 * by:
+	 * 1. Remux the pin as a GPIO output, driven low.
+	 * 2. Wait a little while.  1 ms seems to work, but we'll do 10.
+	 * 3. Immediately jump to remux the pin as dw_hdmi i2c again.
+	 *
+	 * At the moment of remuxing, the line will still be low due to its
+	 * recent stint as an output, but then it will be pulled high by the
+	 * (presumed) external pullup.  dw_hdmi seems to see this as a rising
+	 * edge and that seems to get it out of its jam.
+	 *
+	 * This wedging was only ever seen on one TV, and only on one of
+	 * its HDMI ports.  It happened when the TV was powered on while the
+	 * device was plugged in.  A scope trace shows the TV bringing both SDA
+	 * and SCL low, then bringing them both back up at roughly the same
+	 * time.  Presumably this confuses dw_hdmi because it saw activity but
+	 * no real STOP (maybe it thinks there's another master on the bus?).
+	 * Giving it a clean rising edge of SDA while SCL is already high
+	 * presumably makes dw_hdmi see a STOP which seems to bring dw_hdmi out
+	 * of its stupor.
+	 *
+	 * Note that after coming back alive, transfers seem to immediately
+	 * resume, so if we unwedge due to a timeout we should wait a little
+	 * longer for our transfer to finish, since it might have just started
+	 * now.
+	 */
+	pinctrl_select_state(hdmi->pinctrl, hdmi->unwedge_state);
+	msleep(10);
+	pinctrl_select_state(hdmi->pinctrl, hdmi->default_state);
+
+	return true;
+}
+
+static int dw_hdmi_i2c_wait(struct dw_hdmi *hdmi)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int stat;
+
+	stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10);
+	if (!stat) {
+		/* If we can't unwedge, return timeout */
+		if (!dw_hdmi_i2c_unwedge(hdmi))
+			return -EAGAIN;
+
+		/* We tried to unwedge; give it another chance */
+		stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10);
+		if (!stat)
+			return -EAGAIN;
+	}
+
+	/* Check for error condition on the bus */
+	if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+		return -EIO;
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_read(struct dw_hdmi *hdmi,
+			    unsigned char *buf, unsigned int length)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int ret;
+
+	if (!i2c->is_regaddr) {
+		dev_dbg(hdmi->dev, "set read register address to 0\n");
+		i2c->slave_reg = 0x00;
+		i2c->is_regaddr = true;
+	}
+
+	while (length--) {
+		reinit_completion(&i2c->cmp);
+
+		hdmi_writeb(hdmi, i2c->slave_reg++, HDMI_I2CM_ADDRESS);
+		if (i2c->is_segment)
+			hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_READ_EXT,
+				    HDMI_I2CM_OPERATION);
+		else
+			hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_READ,
+				    HDMI_I2CM_OPERATION);
+
+		ret = dw_hdmi_i2c_wait(hdmi);
+		if (ret)
+			return ret;
+
+		/* Check for error condition on the bus */
+		if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+			return -EIO;
+
+		*buf++ = hdmi_readb(hdmi, HDMI_I2CM_DATAI);
+	}
+	i2c->is_segment = false;
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_write(struct dw_hdmi *hdmi,
+			     unsigned char *buf, unsigned int length)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int ret;
+
+	if (!i2c->is_regaddr) {
+		/* Use the first write byte as register address */
+		i2c->slave_reg = buf[0];
+		length--;
+		buf++;
+		i2c->is_regaddr = true;
+	}
+
+	while (length--) {
+		reinit_completion(&i2c->cmp);
+
+		hdmi_writeb(hdmi, *buf++, HDMI_I2CM_DATAO);
+		hdmi_writeb(hdmi, i2c->slave_reg++, HDMI_I2CM_ADDRESS);
+		hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_WRITE,
+			    HDMI_I2CM_OPERATION);
+
+		ret = dw_hdmi_i2c_wait(hdmi);
+		if (ret)
+			return ret;
+
+		/* Check for error condition on the bus */
+		if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+			return -EIO;
+	}
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_xfer(struct i2c_adapter *adap,
+			    struct i2c_msg *msgs, int num)
+{
+	struct dw_hdmi *hdmi = i2c_get_adapdata(adap);
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	u8 addr = msgs[0].addr;
+	int i, ret = 0;
+
+	if (addr == DDC_CI_ADDR)
+		/*
+		 * The internal I2C controller does not support the multi-byte
+		 * read and write operations needed for DDC/CI.
+		 * TOFIX: Blacklist the DDC/CI address until we filter out
+		 * unsupported I2C operations.
+		 */
+		return -EOPNOTSUPP;
+
+	dev_dbg(hdmi->dev, "xfer: num: %d, addr: %#x\n", num, addr);
+
+	for (i = 0; i < num; i++) {
+		if (msgs[i].len == 0) {
+			dev_dbg(hdmi->dev,
+				"unsupported transfer %d/%d, no data\n",
+				i + 1, num);
+			return -EOPNOTSUPP;
+		}
+	}
+
+	mutex_lock(&i2c->lock);
+
+	/* Unmute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, 0x00, HDMI_IH_MUTE_I2CM_STAT0);
+
+	/* Set slave device address taken from the first I2C message */
+	if (addr == DDC_SEGMENT_ADDR && msgs[0].len == 1)
+		addr = DDC_ADDR;
+	hdmi_writeb(hdmi, addr, HDMI_I2CM_SLAVE);
+
+	/* Set slave device register address on transfer */
+	i2c->is_regaddr = false;
+
+	/* Set segment pointer for I2C extended read mode operation */
+	i2c->is_segment = false;
+
+	for (i = 0; i < num; i++) {
+		dev_dbg(hdmi->dev, "xfer: num: %d/%d, len: %d, flags: %#x\n",
+			i + 1, num, msgs[i].len, msgs[i].flags);
+		if (msgs[i].addr == DDC_SEGMENT_ADDR && msgs[i].len == 1) {
+			i2c->is_segment = true;
+			hdmi_writeb(hdmi, DDC_SEGMENT_ADDR, HDMI_I2CM_SEGADDR);
+			hdmi_writeb(hdmi, *msgs[i].buf, HDMI_I2CM_SEGPTR);
+		} else {
+			if (msgs[i].flags & I2C_M_RD)
+				ret = dw_hdmi_i2c_read(hdmi, msgs[i].buf,
+						       msgs[i].len);
+			else
+				ret = dw_hdmi_i2c_write(hdmi, msgs[i].buf,
+							msgs[i].len);
+		}
+		if (ret < 0) {
+			dev_info(hdmi->dev, "i2c transfer fail\n");
+			break;
+		}
+	}
+
+	if (!ret)
+		ret = num;
+
+	/* Mute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_MUTE_I2CM_STAT0);
+
+	mutex_unlock(&i2c->lock);
+
+	return ret;
+}
+
+static u32 dw_hdmi_i2c_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm dw_hdmi_algorithm = {
+	.master_xfer = dw_hdmi_i2c_xfer,
+	.functionality = dw_hdmi_i2c_func,
+};
+
+u8 tv_hdmi_hdcp2_support(struct dw_hdmi *hdmi)
+{
+	int ret = 0;
+	int try_times = 10;
+	unsigned char start = 0x50;
+	u8 buf;
+	int i;
+
+	struct i2c_msg msgs[] = { {
+					  .addr = 0x3a,
+					  .flags = 0,
+					  .len = 1,
+					  .buf = &start,
+				  },
+				  {
+					  .addr = 0x3a,
+					  .flags = I2C_M_RD,
+					  .len = 1,
+					  .buf = &buf,
+				  } };
+
+	for (i = 0; i < try_times; i++) {
+		//if (hdmi->bridge_is_on)
+		ret = dw_hdmi_i2c_xfer(hdmi->ddc, msgs, 2);
+
+		printk("ret = %d; buf = %d\n", ret, buf);
+		if (ret == 2 && (buf & 0x4))
+			return ((buf & 0x4) >> 2);
+		udelay(100);
+	}
+
+	return -1;
+}
+EXPORT_SYMBOL_GPL(tv_hdmi_hdcp2_support);
+
+static struct i2c_adapter *dw_hdmi_i2c_adapter(struct dw_hdmi *hdmi)
+{
+	struct i2c_adapter *adap;
+	struct dw_hdmi_i2c *i2c;
+	int ret;
+
+	i2c = devm_kzalloc(hdmi->dev, sizeof(*i2c), GFP_KERNEL);
+	if (!i2c)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&i2c->lock);
+	init_completion(&i2c->cmp);
+
+	adap = &i2c->adap;
+	adap->class = I2C_CLASS_DDC;
+	adap->owner = THIS_MODULE;
+	adap->dev.parent = hdmi->dev;
+	adap->algo = &dw_hdmi_algorithm;
+	strscpy(adap->name, "DesignWare HDMI", sizeof(adap->name));
+	i2c_set_adapdata(adap, hdmi);
+
+	ret = i2c_add_adapter(adap);
+	if (ret) {
+		dev_warn(hdmi->dev, "cannot add %s I2C adapter\n", adap->name);
+		devm_kfree(hdmi->dev, i2c);
+		return ERR_PTR(ret);
+	}
+
+	hdmi->i2c = i2c;
+
+	dev_info(hdmi->dev, "registered %s I2C bus driver\n", adap->name);
+
+	return adap;
+}
+
+static void hdmi_set_cts_n(struct dw_hdmi *hdmi, unsigned int cts,
+			   unsigned int n)
+{
+	/* Must be set/cleared first */
+	hdmi_modb(hdmi, 0, HDMI_AUD_CTS3_CTS_MANUAL, HDMI_AUD_CTS3);
+
+	/* nshift factor = 0 */
+	hdmi_modb(hdmi, 0, HDMI_AUD_CTS3_N_SHIFT_MASK, HDMI_AUD_CTS3);
+
+	/* Use automatic CTS generation mode when CTS is not set */
+	if (cts)
+		hdmi_writeb(hdmi, ((cts >> 16) &
+				   HDMI_AUD_CTS3_AUDCTS19_16_MASK) |
+				  HDMI_AUD_CTS3_CTS_MANUAL,
+			    HDMI_AUD_CTS3);
+	else
+		hdmi_writeb(hdmi, 0, HDMI_AUD_CTS3);
+	hdmi_writeb(hdmi, (cts >> 8) & 0xff, HDMI_AUD_CTS2);
+	hdmi_writeb(hdmi, cts & 0xff, HDMI_AUD_CTS1);
+
+	hdmi_writeb(hdmi, (n >> 16) & 0x0f, HDMI_AUD_N3);
+	hdmi_writeb(hdmi, (n >> 8) & 0xff, HDMI_AUD_N2);
+	hdmi_writeb(hdmi, n & 0xff, HDMI_AUD_N1);
+}
+
+static unsigned int hdmi_compute_n(unsigned int freq, unsigned long pixel_clk)
+{
+	unsigned int n = (128 * freq) / 1000;
+	unsigned int mult = 1;
+
+	while (freq > 48000) {
+		mult *= 2;
+		freq /= 2;
+	}
+
+	switch (freq) {
+	case 32000:
+		if (pixel_clk == 25175000)
+			n = 4576;
+		else if (pixel_clk == 27027000)
+			n = 4096;
+		else if (pixel_clk == 74176000 || pixel_clk == 148352000)
+			n = 11648;
+		else if (pixel_clk == 297000000)
+			n = 3072;
+		else
+			n = 4096;
+		n *= mult;
+		break;
+
+	case 44100:
+		if (pixel_clk == 25175000)
+			n = 7007;
+		else if (pixel_clk == 74176000)
+			n = 17836;
+		else if (pixel_clk == 148352000)
+			n = 8918;
+		else if (pixel_clk == 297000000)
+			n = 4704;
+		else
+			n = 6272;
+		n *= mult;
+		break;
+
+	case 48000:
+		if (pixel_clk == 25175000)
+			n = 6864;
+		else if (pixel_clk == 27027000)
+			n = 6144;
+		else if (pixel_clk == 74176000)
+			n = 11648;
+		else if (pixel_clk == 148352000)
+			n = 5824;
+		else if (pixel_clk == 297000000)
+			n = 5120;
+		else
+			n = 6144;
+		n *= mult;
+		break;
+
+	default:
+		break;
+	}
+
+	return n;
+}
+
+/*
+ * When transmitting IEC60958 linear PCM audio, these registers allow to
+ * configure the channel status information of all the channel status
+ * bits in the IEC60958 frame. For the moment this configuration is only
+ * used when the I2S audio interface, General Purpose Audio (GPA),
+ * or AHB audio DMA (AHBAUDDMA) interface is active
+ * (for S/PDIF interface this information comes from the stream).
+ */
+void dw_hdmi_set_channel_status(struct dw_hdmi *hdmi,
+				u8 *channel_status)
+{
+	/*
+	 * Set channel status register for frequency and word length.
+	 * Use default values for other registers.
+	 */
+	hdmi_writeb(hdmi, channel_status[3], HDMI_FC_AUDSCHNLS7);
+	hdmi_writeb(hdmi, channel_status[4], HDMI_FC_AUDSCHNLS8);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_channel_status);
+
+static void hdmi_set_clk_regenerator(struct dw_hdmi *hdmi,
+	unsigned long pixel_clk, unsigned int sample_rate)
+{
+	unsigned long ftdms = pixel_clk;
+	unsigned int n, cts;
+	u8 config3;
+	u64 tmp;
+
+	n = hdmi_compute_n(sample_rate, pixel_clk);
+
+	config3 = hdmi_readb(hdmi, HDMI_CONFIG3_ID);
+
+	/* Compute CTS when using internal AHB audio or General Parallel audio*/
+	if ((config3 & HDMI_CONFIG3_AHBAUDDMA) || (config3 & HDMI_CONFIG3_GPAUD)) {
+		/*
+		 * Compute the CTS value from the N value.  Note that CTS and N
+		 * can be up to 20 bits in total, so we need 64-bit math.  Also
+		 * note that our TDMS clock is not fully accurate; it is
+		 * accurate to kHz.  This can introduce an unnecessary remainder
+		 * in the calculation below, so we don't try to warn about that.
+		 */
+		tmp = (u64)ftdms * n;
+		do_div(tmp, 128 * sample_rate);
+		cts = tmp;
+
+		dev_dbg(hdmi->dev, "%s: fs=%uHz ftdms=%lu.%03luMHz N=%d cts=%d\n",
+			__func__, sample_rate,
+			ftdms / 1000000, (ftdms / 1000) % 1000,
+			n, cts);
+	} else {
+		cts = 0;
+	}
+
+	spin_lock_irq(&hdmi->audio_lock);
+	hdmi->audio_n = n;
+	hdmi->audio_cts = cts;
+	hdmi_set_cts_n(hdmi, cts, hdmi->audio_enable ? n : 0);
+	spin_unlock_irq(&hdmi->audio_lock);
+}
+
+static void hdmi_init_clk_regenerator(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi_set_clk_regenerator(hdmi, 74250000, hdmi->sample_rate);
+	mutex_unlock(&hdmi->audio_mutex);
+}
+
+static void hdmi_clk_regenerator_update_pixel_clock(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock,
+				 hdmi->sample_rate);
+	mutex_unlock(&hdmi->audio_mutex);
+}
+
+void dw_hdmi_set_sample_width(struct dw_hdmi *hdmi, unsigned int width)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->sample_width = width;
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_width);
+
+void dw_hdmi_set_sample_non_pcm(struct dw_hdmi *hdmi, unsigned int non_pcm)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->sample_non_pcm = non_pcm;
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_non_pcm);
+
+void dw_hdmi_set_sample_rate(struct dw_hdmi *hdmi, unsigned int rate)
+{
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->sample_rate = rate;
+	hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock,
+				 hdmi->sample_rate);
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_rate);
+
+void dw_hdmi_set_channel_count(struct dw_hdmi *hdmi, unsigned int cnt)
+{
+	u8 layout;
+
+	mutex_lock(&hdmi->audio_mutex);
+	hdmi->channels = cnt;
+
+	/*
+	 * For >2 channel PCM audio, we need to select layout 1
+	 * and set an appropriate channel map.
+	 */
+	if (cnt > 2)
+		layout = HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT1;
+	else
+		layout = HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT0;
+
+	hdmi_modb(hdmi, layout, HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_MASK,
+		  HDMI_FC_AUDSCONF);
+
+	/* Set the audio infoframes channel count */
+	hdmi_modb(hdmi, (cnt - 1) << HDMI_FC_AUDICONF0_CC_OFFSET,
+		  HDMI_FC_AUDICONF0_CC_MASK, HDMI_FC_AUDICONF0);
+
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_channel_count);
+
+void dw_hdmi_set_channel_allocation(struct dw_hdmi *hdmi, unsigned int ca)
+{
+	mutex_lock(&hdmi->audio_mutex);
+
+	hdmi_writeb(hdmi, ca, HDMI_FC_AUDICONF2);
+
+	mutex_unlock(&hdmi->audio_mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_channel_allocation);
+
+static void hdmi_enable_audio_clk(struct dw_hdmi *hdmi, bool enable)
+{
+	if (enable)
+		hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_AUDCLK_DISABLE;
+	else
+		hdmi->mc_clkdis |= HDMI_MC_CLKDIS_AUDCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+}
+
+static u8 *hdmi_audio_get_eld(struct dw_hdmi *hdmi)
+{
+	if (!hdmi->curr_conn)
+		return NULL;
+
+	return hdmi->curr_conn->eld;
+}
+
+static void dw_hdmi_gp_audio_enable(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+	int sample_freq = 0x2, org_sample_freq = 0xD;
+	int ch_mask = BIT(hdmi->channels) - 1;
+
+	switch (hdmi->sample_rate) {
+	case 32000:
+		sample_freq = 0x03;
+		org_sample_freq = 0x0C;
+		break;
+	case 44100:
+		sample_freq = 0x00;
+		org_sample_freq = 0x0F;
+		break;
+	case 48000:
+		sample_freq = 0x02;
+		org_sample_freq = 0x0D;
+		break;
+	case 88200:
+		sample_freq = 0x08;
+		org_sample_freq = 0x07;
+		break;
+	case 96000:
+		sample_freq = 0x0A;
+		org_sample_freq = 0x05;
+		break;
+	case 176400:
+		sample_freq = 0x0C;
+		org_sample_freq = 0x03;
+		break;
+	case 192000:
+		sample_freq = 0x0E;
+		org_sample_freq = 0x01;
+		break;
+	default:
+		break;
+	}
+
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n);
+	hdmi_enable_audio_clk(hdmi, true);
+
+	hdmi_writeb(hdmi, 0x1, HDMI_FC_AUDSCHNLS0);
+	hdmi_writeb(hdmi, hdmi->channels, HDMI_FC_AUDSCHNLS2);
+	hdmi_writeb(hdmi, 0x22, HDMI_FC_AUDSCHNLS3);
+	hdmi_writeb(hdmi, 0x22, HDMI_FC_AUDSCHNLS4);
+	hdmi_writeb(hdmi, 0x11, HDMI_FC_AUDSCHNLS5);
+	hdmi_writeb(hdmi, 0x11, HDMI_FC_AUDSCHNLS6);
+	hdmi_writeb(hdmi, (0x3 << 4) | sample_freq, HDMI_FC_AUDSCHNLS7);
+	hdmi_writeb(hdmi, (org_sample_freq << 4) | 0xb, HDMI_FC_AUDSCHNLS8);
+
+	hdmi_writeb(hdmi, ch_mask, HDMI_GP_CONF1);
+	hdmi_writeb(hdmi, 0x02, HDMI_GP_CONF2);
+	hdmi_writeb(hdmi, 0x01, HDMI_GP_CONF0);
+
+	hdmi_modb(hdmi,  0x3, 0x3, HDMI_FC_DATAUTO3);
+
+	/* hbr */
+	if (hdmi->sample_rate == 192000 && hdmi->channels == 8 &&
+	    hdmi->sample_width == 32 && hdmi->sample_non_pcm)
+		hdmi_modb(hdmi, 0x01, 0x01, HDMI_GP_CONF2);
+
+	if (pdata->enable_audio)
+		pdata->enable_audio(hdmi,
+				    hdmi->channels,
+				    hdmi->sample_width,
+				    hdmi->sample_rate,
+				    hdmi->sample_non_pcm);
+}
+
+static void dw_hdmi_gp_audio_disable(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, 0);
+
+	hdmi_modb(hdmi,  0, 0x3, HDMI_FC_DATAUTO3);
+	if (pdata->disable_audio)
+		pdata->disable_audio(hdmi);
+
+	hdmi_enable_audio_clk(hdmi, false);
+}
+
+static void dw_hdmi_ahb_audio_enable(struct dw_hdmi *hdmi)
+{
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n);
+}
+
+static void dw_hdmi_ahb_audio_disable(struct dw_hdmi *hdmi)
+{
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, 0);
+}
+
+static void dw_hdmi_i2s_audio_enable(struct dw_hdmi *hdmi)
+{
+	hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n);
+	hdmi_enable_audio_clk(hdmi, true);
+}
+
+static void dw_hdmi_i2s_audio_disable(struct dw_hdmi *hdmi)
+{
+	hdmi_enable_audio_clk(hdmi, false);
+}
+
+void dw_hdmi_audio_enable(struct dw_hdmi *hdmi)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&hdmi->audio_lock, flags);
+	hdmi->audio_enable = true;
+	if (hdmi->enable_audio)
+		hdmi->enable_audio(hdmi);
+	spin_unlock_irqrestore(&hdmi->audio_lock, flags);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_audio_enable);
+
+void dw_hdmi_audio_disable(struct dw_hdmi *hdmi)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&hdmi->audio_lock, flags);
+	hdmi->audio_enable = false;
+	if (hdmi->disable_audio)
+		hdmi->disable_audio(hdmi);
+	spin_unlock_irqrestore(&hdmi->audio_lock, flags);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_audio_disable);
+
+static bool hdmi_bus_fmt_is_rgb(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static bool hdmi_bus_fmt_is_yuv444(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_YUV12_1X36:
+	case MEDIA_BUS_FMT_YUV16_1X48:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static bool hdmi_bus_fmt_is_yuv422(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static bool hdmi_bus_fmt_is_yuv420(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static int hdmi_bus_fmt_color_depth(unsigned int bus_format)
+{
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+		return 8;
+
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		return 10;
+
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+	case MEDIA_BUS_FMT_YUV12_1X36:
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+		return 12;
+
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+	case MEDIA_BUS_FMT_YUV16_1X48:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		return 16;
+
+	default:
+		return 0;
+	}
+}
+
+/*
+ * this submodule is responsible for the video data synchronization.
+ * for example, for RGB 4:4:4 input, the data map is defined as
+ *			pin{47~40} <==> R[7:0]
+ *			pin{31~24} <==> G[7:0]
+ *			pin{15~8}  <==> B[7:0]
+ */
+static void hdmi_video_sample(struct dw_hdmi *hdmi)
+{
+	int color_format = 0;
+	u8 val;
+
+	switch (hdmi->hdmi_data.enc_in_bus_format) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+		color_format = 0x01;
+		break;
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+		color_format = 0x03;
+		break;
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+		color_format = 0x05;
+		break;
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+		color_format = 0x07;
+		break;
+
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+		color_format = 0x09;
+		break;
+	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		color_format = 0x0B;
+		break;
+	case MEDIA_BUS_FMT_YUV12_1X36:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+		color_format = 0x0D;
+		break;
+	case MEDIA_BUS_FMT_YUV16_1X48:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		color_format = 0x0F;
+		break;
+
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		color_format = 0x16;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		color_format = 0x14;
+		break;
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+		color_format = 0x12;
+		break;
+
+	default:
+		return;
+	}
+
+	val = HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_DISABLE |
+		((color_format << HDMI_TX_INVID0_VIDEO_MAPPING_OFFSET) &
+		HDMI_TX_INVID0_VIDEO_MAPPING_MASK);
+	hdmi_writeb(hdmi, val, HDMI_TX_INVID0);
+
+	/* Enable TX stuffing: When DE is inactive, fix the output data to 0 */
+	val = HDMI_TX_INSTUFFING_BDBDATA_STUFFING_ENABLE |
+		HDMI_TX_INSTUFFING_RCRDATA_STUFFING_ENABLE |
+		HDMI_TX_INSTUFFING_GYDATA_STUFFING_ENABLE;
+	hdmi_writeb(hdmi, val, HDMI_TX_INSTUFFING);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_GYDATA0);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_GYDATA1);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_RCRDATA0);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_RCRDATA1);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_BCBDATA0);
+	hdmi_writeb(hdmi, 0x0, HDMI_TX_BCBDATA1);
+}
+
+static int is_color_space_conversion(struct dw_hdmi *hdmi)
+{
+	struct hdmi_data_info *hdmi_data = &hdmi->hdmi_data;
+	bool is_input_rgb, is_output_rgb;
+
+	is_input_rgb = hdmi_bus_fmt_is_rgb(hdmi_data->enc_in_bus_format);
+	is_output_rgb = hdmi_bus_fmt_is_rgb(hdmi_data->enc_out_bus_format);
+
+	return (is_input_rgb != is_output_rgb) ||
+	       (is_input_rgb && is_output_rgb && hdmi_data->rgb_limited_range);
+}
+
+static int is_color_space_decimation(struct dw_hdmi *hdmi)
+{
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		return 0;
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_in_bus_format) ||
+	    hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_in_bus_format))
+		return 1;
+
+	return 0;
+}
+
+static int is_color_space_interpolation(struct dw_hdmi *hdmi)
+{
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_in_bus_format))
+		return 0;
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format) ||
+	    hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format))
+		return 1;
+
+	return 0;
+}
+
+static bool is_csc_needed(struct dw_hdmi *hdmi)
+{
+	return is_color_space_conversion(hdmi) ||
+	       is_color_space_decimation(hdmi) ||
+	       is_color_space_interpolation(hdmi);
+}
+
+static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi)
+{
+	const u16(*csc_coeff)[3][4] = &csc_coeff_default;
+	bool is_input_rgb, is_output_rgb, is_output_yuv;
+	unsigned i;
+	u32 csc_scale = 1;
+	unsigned int depth =
+		hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format);
+
+	is_input_rgb = hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_in_bus_format);
+	is_output_rgb = hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format);
+	is_output_yuv =
+		hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format) ||
+		hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format);
+
+	if (!is_input_rgb && is_output_rgb) {
+		if (hdmi->hdmi_data.enc_out_encoding == V4L2_YCBCR_ENC_601)
+			csc_coeff = &csc_coeff_rgb_out_eitu601;
+		else
+			csc_coeff = &csc_coeff_rgb_out_eitu709;
+	} else if (is_input_rgb && !is_output_rgb) {
+		if (hdmi->hdmi_data.enc_out_encoding == V4L2_YCBCR_ENC_601) {
+			if (is_output_yuv && depth == 8)
+				csc_coeff = &csc_coeff_rgb_to_yuv_8bit_eitu601;
+			else if (is_output_yuv && depth == 10)
+				csc_coeff = &csc_coeff_rgb_to_yuv_10bit_eitu601;
+			else
+				csc_coeff = &csc_coeff_rgb_in_eitu601;
+		} else {
+			if (is_output_yuv && depth == 8)
+				csc_coeff = &csc_coeff_rgb_to_yuv_8bit_eitu709;
+			else if (is_output_yuv && depth == 10)
+				csc_coeff = &csc_coeff_rgb_to_yuv_10bit_eitu709;
+			else
+				csc_coeff = &csc_coeff_rgb_in_eitu709;
+		}
+		csc_scale = 0;
+	} else if (is_input_rgb && is_output_rgb &&
+		   hdmi->hdmi_data.rgb_limited_range) {
+		csc_coeff = &csc_coeff_rgb_full_to_rgb_limited;
+	}
+
+	/* The CSC registers are sequential, alternating MSB then LSB */
+	for (i = 0; i < ARRAY_SIZE(csc_coeff_default[0]); i++) {
+		u16 coeff_a = (*csc_coeff)[0][i];
+		u16 coeff_b = (*csc_coeff)[1][i];
+		u16 coeff_c = (*csc_coeff)[2][i];
+
+		hdmi_writeb(hdmi, coeff_a & 0xff, HDMI_CSC_COEF_A1_LSB + i * 2);
+		hdmi_writeb(hdmi, coeff_a >> 8, HDMI_CSC_COEF_A1_MSB + i * 2);
+		hdmi_writeb(hdmi, coeff_b & 0xff, HDMI_CSC_COEF_B1_LSB + i * 2);
+		hdmi_writeb(hdmi, coeff_b >> 8, HDMI_CSC_COEF_B1_MSB + i * 2);
+		hdmi_writeb(hdmi, coeff_c & 0xff, HDMI_CSC_COEF_C1_LSB + i * 2);
+		hdmi_writeb(hdmi, coeff_c >> 8, HDMI_CSC_COEF_C1_MSB + i * 2);
+	}
+
+	hdmi_modb(hdmi, csc_scale, HDMI_CSC_SCALE_CSCSCALE_MASK,
+		  HDMI_CSC_SCALE);
+}
+
+static void hdmi_video_csc(struct dw_hdmi *hdmi)
+{
+	int color_depth = 0;
+	int interpolation = HDMI_CSC_CFG_INTMODE_DISABLE;
+	int decimation = 0;
+
+	/* YCC422 interpolation to 444 mode */
+	if (is_color_space_interpolation(hdmi))
+		interpolation = HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA1;
+	else if (is_color_space_decimation(hdmi))
+		decimation = HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA3;
+
+	switch (hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format)) {
+	case 8:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_24BPP;
+		break;
+	case 10:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_30BPP;
+		break;
+	case 12:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_36BPP;
+		break;
+	case 16:
+		color_depth = HDMI_CSC_SCALE_CSC_COLORDE_PTH_48BPP;
+		break;
+
+	default:
+		return;
+	}
+
+	/* Configure the CSC registers */
+	hdmi_writeb(hdmi, interpolation | decimation, HDMI_CSC_CFG);
+	hdmi_modb(hdmi, color_depth, HDMI_CSC_SCALE_CSC_COLORDE_PTH_MASK,
+		  HDMI_CSC_SCALE);
+
+	dw_hdmi_update_csc_coeffs(hdmi);
+}
+
+/*
+ * HDMI video packetizer is used to packetize the data.
+ * for example, if input is YCC422 mode or repeater is used,
+ * data should be repacked this module can be bypassed.
+ */
+static void hdmi_video_packetize(struct dw_hdmi *hdmi)
+{
+	unsigned int color_depth = 0;
+	unsigned int remap_size = HDMI_VP_REMAP_YCC422_16bit;
+	unsigned int output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_PP;
+	struct hdmi_data_info *hdmi_data = &hdmi->hdmi_data;
+	u8 val, vp_conf;
+	u8 clear_gcp_auto = 0;
+
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format) ||
+	    hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format) ||
+	    hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi_bus_fmt_color_depth(
+					hdmi->hdmi_data.enc_out_bus_format)) {
+		case 8:
+			color_depth = 4;
+			output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS;
+			clear_gcp_auto = 1;
+			break;
+		case 10:
+			color_depth = 5;
+			break;
+		case 12:
+			color_depth = 6;
+			break;
+		case 16:
+			color_depth = 7;
+			break;
+		default:
+			output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS;
+		}
+	} else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi_bus_fmt_color_depth(
+					hdmi->hdmi_data.enc_out_bus_format)) {
+		case 0:
+		case 8:
+			remap_size = HDMI_VP_REMAP_YCC422_16bit;
+			clear_gcp_auto = 1;
+			break;
+		case 10:
+			remap_size = HDMI_VP_REMAP_YCC422_20bit;
+			break;
+		case 12:
+			remap_size = HDMI_VP_REMAP_YCC422_24bit;
+			break;
+
+		default:
+			return;
+		}
+		output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_YCC422;
+	} else {
+		return;
+	}
+
+	/* set the packetizer registers */
+	val = ((color_depth << HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET) &
+		HDMI_VP_PR_CD_COLOR_DEPTH_MASK) |
+		((hdmi_data->pix_repet_factor <<
+		HDMI_VP_PR_CD_DESIRED_PR_FACTOR_OFFSET) &
+		HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK);
+	hdmi_writeb(hdmi, val, HDMI_VP_PR_CD);
+
+	/* HDMI1.4b specification section 6.5.3:
+	 * Source shall only send GCPs with non-zero CD to sinks
+	 * that indicate support for Deep Color.
+	 * GCP only transmit CD and do not handle AVMUTE, PP norDefault_Phase (yet).
+	 * Disable Auto GCP when 24-bit color for sinks that not support Deep Color.
+	 */
+	val = hdmi_readb(hdmi, HDMI_FC_DATAUTO3);
+	if (clear_gcp_auto == 1)
+		val &= ~HDMI_FC_DATAUTO3_GCP_AUTO;
+	else
+		val |= HDMI_FC_DATAUTO3_GCP_AUTO;
+	hdmi_writeb(hdmi, val, HDMI_FC_DATAUTO3);
+
+	hdmi_modb(hdmi, HDMI_VP_STUFF_PR_STUFFING_STUFFING_MODE,
+		  HDMI_VP_STUFF_PR_STUFFING_MASK, HDMI_VP_STUFF);
+
+	/* Data from pixel repeater block */
+	if (hdmi_data->pix_repet_factor > 1) {
+		vp_conf = HDMI_VP_CONF_PR_EN_ENABLE |
+			  HDMI_VP_CONF_BYPASS_SELECT_PIX_REPEATER;
+	} else { /* data from packetizer block */
+		vp_conf = HDMI_VP_CONF_PR_EN_DISABLE |
+			  HDMI_VP_CONF_BYPASS_SELECT_VID_PACKETIZER;
+	}
+
+	hdmi_modb(hdmi, vp_conf,
+		  HDMI_VP_CONF_PR_EN_MASK |
+		  HDMI_VP_CONF_BYPASS_SELECT_MASK, HDMI_VP_CONF);
+
+	hdmi_modb(hdmi, 1 << HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET,
+		  HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, HDMI_VP_STUFF);
+
+	hdmi_writeb(hdmi, remap_size, HDMI_VP_REMAP);
+
+	if (output_select == HDMI_VP_CONF_OUTPUT_SELECTOR_PP) {
+		vp_conf = HDMI_VP_CONF_BYPASS_EN_DISABLE |
+			  HDMI_VP_CONF_PP_EN_ENABLE |
+			  HDMI_VP_CONF_YCC422_EN_DISABLE;
+	} else if (output_select == HDMI_VP_CONF_OUTPUT_SELECTOR_YCC422) {
+		vp_conf = HDMI_VP_CONF_BYPASS_EN_DISABLE |
+			  HDMI_VP_CONF_PP_EN_DISABLE |
+			  HDMI_VP_CONF_YCC422_EN_ENABLE;
+	} else if (output_select == HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS) {
+		vp_conf = HDMI_VP_CONF_BYPASS_EN_ENABLE |
+			  HDMI_VP_CONF_PP_EN_DISABLE |
+			  HDMI_VP_CONF_YCC422_EN_DISABLE;
+	} else {
+		return;
+	}
+
+	hdmi_modb(hdmi, vp_conf,
+		  HDMI_VP_CONF_BYPASS_EN_MASK | HDMI_VP_CONF_PP_EN_ENMASK |
+		  HDMI_VP_CONF_YCC422_EN_MASK, HDMI_VP_CONF);
+
+	hdmi_modb(hdmi, HDMI_VP_STUFF_PP_STUFFING_STUFFING_MODE |
+			HDMI_VP_STUFF_YCC422_STUFFING_STUFFING_MODE,
+		  HDMI_VP_STUFF_PP_STUFFING_MASK |
+		  HDMI_VP_STUFF_YCC422_STUFFING_MASK, HDMI_VP_STUFF);
+
+	hdmi_modb(hdmi, output_select, HDMI_VP_CONF_OUTPUT_SELECTOR_MASK,
+		  HDMI_VP_CONF);
+}
+
+/* -----------------------------------------------------------------------------
+ * Synopsys PHY Handling
+ */
+
+static inline void hdmi_phy_test_clear(struct dw_hdmi *hdmi,
+				       unsigned char bit)
+{
+	hdmi_modb(hdmi, bit << HDMI_PHY_TST0_TSTCLR_OFFSET,
+		  HDMI_PHY_TST0_TSTCLR_MASK, HDMI_PHY_TST0);
+}
+
+static bool hdmi_phy_wait_i2c_done(struct dw_hdmi *hdmi, int msec)
+{
+	u32 val;
+
+	while ((val = hdmi_readb(hdmi, HDMI_IH_I2CMPHY_STAT0) & 0x3) == 0) {
+		if (msec-- == 0)
+			return false;
+		udelay(1000);
+	}
+	hdmi_writeb(hdmi, val, HDMI_IH_I2CMPHY_STAT0);
+
+	return true;
+}
+
+void dw_hdmi_phy_i2c_write(struct dw_hdmi *hdmi, unsigned short data,
+			   unsigned char addr)
+{
+	hdmi_writeb(hdmi, 0xFF, HDMI_IH_I2CMPHY_STAT0);
+	hdmi_writeb(hdmi, addr, HDMI_PHY_I2CM_ADDRESS_ADDR);
+	hdmi_writeb(hdmi, (unsigned char)(data >> 8),
+		    HDMI_PHY_I2CM_DATAO_1_ADDR);
+	hdmi_writeb(hdmi, (unsigned char)(data >> 0),
+		    HDMI_PHY_I2CM_DATAO_0_ADDR);
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_OPERATION_ADDR_WRITE,
+		    HDMI_PHY_I2CM_OPERATION_ADDR);
+	hdmi_phy_wait_i2c_done(hdmi, 1000);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_i2c_write);
+
+static int hdmi_phy_i2c_read(struct dw_hdmi *hdmi, unsigned char addr)
+{
+	int val;
+
+	hdmi_writeb(hdmi, 0xFF, HDMI_IH_I2CMPHY_STAT0);
+	hdmi_writeb(hdmi, addr, HDMI_PHY_I2CM_ADDRESS_ADDR);
+	hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_1_ADDR);
+	hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_0_ADDR);
+	hdmi_writeb(hdmi, HDMI_PHY_I2CM_OPERATION_ADDR_READ,
+		    HDMI_PHY_I2CM_OPERATION_ADDR);
+	hdmi_phy_wait_i2c_done(hdmi, 1000);
+	val = hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_1_ADDR);
+	val = (val & 0xff) << 8;
+	val += hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_0_ADDR) & 0xff;
+	return val;
+}
+
+/* Filter out invalid setups to avoid configuring SCDC and scrambling */
+static bool dw_hdmi_support_scdc(struct dw_hdmi *hdmi,
+				 const struct drm_display_info *display)
+{
+	/* Completely disable SCDC support for older controllers */
+	if (hdmi->version < 0x200a)
+		return false;
+
+	/* Disable if no DDC bus */
+	if (!hdmi->ddc)
+		return false;
+
+	/* Disable if SCDC is not supported, or if an HF-VSDB block is absent */
+	if (!display->hdmi.scdc.supported ||
+	    !display->hdmi.scdc.scrambling.supported)
+		return false;
+
+	/*
+	 * Disable if display only support low TMDS rates and scrambling
+	 * for low rates is not supported either
+	 */
+	if (!display->hdmi.scdc.scrambling.low_rates &&
+	    display->max_tmds_clock <= 340000)
+		return false;
+
+	return true;
+}
+
+/*
+ * HDMI2.0 Specifies the following procedure for High TMDS Bit Rates:
+ * - The Source shall suspend transmission of the TMDS clock and data
+ * - The Source shall write to the TMDS_Bit_Clock_Ratio bit to change it
+ * from a 0 to a 1 or from a 1 to a 0
+ * - The Source shall allow a minimum of 1 ms and a maximum of 100 ms from
+ * the time the TMDS_Bit_Clock_Ratio bit is written until resuming
+ * transmission of TMDS clock and data
+ *
+ * To respect the 100ms maximum delay, the dw_hdmi_set_high_tmds_clock_ratio()
+ * helper should called right before enabling the TMDS Clock and Data in
+ * the PHY configuration callback.
+ */
+void dw_hdmi_set_high_tmds_clock_ratio(struct dw_hdmi *hdmi,
+				       const struct drm_display_info *display)
+{
+	unsigned long mtmdsclock = hdmi->hdmi_data.video_mode.mtmdsclock;
+
+	/* Control for TMDS Bit Period/TMDS Clock-Period Ratio */
+	if (dw_hdmi_support_scdc(hdmi, display)) {
+		if (mtmdsclock > HDMI14_MAX_TMDSCLK)
+			drm_scdc_set_high_tmds_clock_ratio(hdmi->curr_conn, 1);
+		else
+			drm_scdc_set_high_tmds_clock_ratio(hdmi->curr_conn, 0);
+	}
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_set_high_tmds_clock_ratio);
+
+static void dw_hdmi_phy_enable_powerdown(struct dw_hdmi *hdmi, bool enable)
+{
+	hdmi_mask_writeb(hdmi, !enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_PDZ_OFFSET,
+			 HDMI_PHY_CONF0_PDZ_MASK);
+}
+
+static void dw_hdmi_phy_enable_tmds(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_ENTMDS_OFFSET,
+			 HDMI_PHY_CONF0_ENTMDS_MASK);
+}
+
+static void dw_hdmi_phy_enable_svsret(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_SVSRET_OFFSET,
+			 HDMI_PHY_CONF0_SVSRET_MASK);
+}
+
+void dw_hdmi_phy_gen2_pddq(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_GEN2_PDDQ_OFFSET,
+			 HDMI_PHY_CONF0_GEN2_PDDQ_MASK);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen2_pddq);
+
+void dw_hdmi_phy_gen2_txpwron(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_GEN2_TXPWRON_OFFSET,
+			 HDMI_PHY_CONF0_GEN2_TXPWRON_MASK);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen2_txpwron);
+
+static void dw_hdmi_phy_sel_data_en_pol(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_SELDATAENPOL_OFFSET,
+			 HDMI_PHY_CONF0_SELDATAENPOL_MASK);
+}
+
+static void dw_hdmi_phy_sel_interface_control(struct dw_hdmi *hdmi, u8 enable)
+{
+	hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0,
+			 HDMI_PHY_CONF0_SELDIPIF_OFFSET,
+			 HDMI_PHY_CONF0_SELDIPIF_MASK);
+}
+
+void dw_hdmi_phy_gen1_reset(struct dw_hdmi *hdmi)
+{
+	/* PHY reset. The reset signal is active low on Gen1 PHYs. */
+	hdmi_writeb(hdmi, 0, HDMI_MC_PHYRSTZ);
+	hdmi_writeb(hdmi, HDMI_MC_PHYRSTZ_PHYRSTZ, HDMI_MC_PHYRSTZ);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen1_reset);
+
+void dw_hdmi_phy_gen2_reset(struct dw_hdmi *hdmi)
+{
+	/* PHY reset. The reset signal is active high on Gen2 PHYs. */
+	hdmi_writeb(hdmi, HDMI_MC_PHYRSTZ_PHYRSTZ, HDMI_MC_PHYRSTZ);
+	hdmi_writeb(hdmi, 0, HDMI_MC_PHYRSTZ);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_gen2_reset);
+
+void dw_hdmi_phy_i2c_set_addr(struct dw_hdmi *hdmi, u8 address)
+{
+	hdmi_phy_test_clear(hdmi, 1);
+	hdmi_writeb(hdmi, address, HDMI_PHY_I2CM_SLAVE_ADDR);
+	hdmi_phy_test_clear(hdmi, 0);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_i2c_set_addr);
+
+static void dw_hdmi_phy_power_off(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_phy_data *phy = hdmi->phy.data;
+	unsigned int i;
+	u16 val;
+
+	if (phy->gen == 1) {
+		dw_hdmi_phy_enable_tmds(hdmi, 0);
+		dw_hdmi_phy_enable_powerdown(hdmi, true);
+		return;
+	}
+
+	dw_hdmi_phy_gen2_txpwron(hdmi, 0);
+
+	/*
+	 * Wait for TX_PHY_LOCK to be deasserted to indicate that the PHY went
+	 * to low power mode.
+	 */
+	for (i = 0; i < 5; ++i) {
+		val = hdmi_readb(hdmi, HDMI_PHY_STAT0);
+		if (!(val & HDMI_PHY_TX_PHY_LOCK))
+			break;
+
+		usleep_range(1000, 2000);
+	}
+
+	if (val & HDMI_PHY_TX_PHY_LOCK)
+		dev_warn(hdmi->dev, "PHY failed to power down\n");
+	else
+		dev_dbg(hdmi->dev, "PHY powered down in %u iterations\n", i);
+
+	dw_hdmi_phy_gen2_pddq(hdmi, 1);
+}
+
+static int dw_hdmi_phy_power_on(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_phy_data *phy = hdmi->phy.data;
+	unsigned int i;
+	u8 val;
+
+	if (phy->gen == 1) {
+		dw_hdmi_phy_enable_powerdown(hdmi, false);
+
+		/* Toggle TMDS enable. */
+		dw_hdmi_phy_enable_tmds(hdmi, 0);
+		dw_hdmi_phy_enable_tmds(hdmi, 1);
+		return 0;
+	}
+
+	dw_hdmi_phy_gen2_txpwron(hdmi, 1);
+	dw_hdmi_phy_gen2_pddq(hdmi, 0);
+
+	/* Wait for PHY PLL lock */
+	for (i = 0; i < 5; ++i) {
+		val = hdmi_readb(hdmi, HDMI_PHY_STAT0) & HDMI_PHY_TX_PHY_LOCK;
+		if (val)
+			break;
+
+		usleep_range(1000, 2000);
+	}
+
+	if (!val) {
+		dev_err(hdmi->dev, "PHY PLL failed to lock\n");
+		return -ETIMEDOUT;
+	}
+
+	dev_dbg(hdmi->dev, "PHY PLL locked %u iterations\n", i);
+	return 0;
+}
+
+/*
+ * PHY configuration function for the DWC HDMI 3D TX PHY. Based on the available
+ * information the DWC MHL PHY has the same register layout and is thus also
+ * supported by this function.
+ */
+static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi,
+		const struct dw_hdmi_plat_data *pdata,
+		unsigned long mpixelclock)
+{
+	const struct dw_hdmi_mpll_config *mpll_config = pdata->mpll_cfg;
+	const struct dw_hdmi_curr_ctrl *curr_ctrl = pdata->cur_ctr;
+	const struct dw_hdmi_phy_config *phy_config = pdata->phy_config;
+
+	/* TOFIX Will need 420 specific PHY configuration tables */
+	unsigned int depth =
+		hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format);
+
+	/* PLL/MPLL Cfg - always match on final entry */
+	for (; mpll_config->mpixelclock != ~0UL; mpll_config++)
+		if (mpixelclock <= mpll_config->mpixelclock)
+			break;
+
+	for (; curr_ctrl->mpixelclock != ~0UL; curr_ctrl++)
+		if (mpixelclock <= curr_ctrl->mpixelclock)
+			break;
+
+	for (; phy_config->mpixelclock != ~0UL; phy_config++)
+		if (mpixelclock <= phy_config->mpixelclock)
+			break;
+
+	if (mpll_config->mpixelclock == ~0UL ||
+	    curr_ctrl->mpixelclock == ~0UL || phy_config->mpixelclock == ~0UL)
+		return -EINVAL;
+
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		depth = fls(depth - 8);
+	else
+		depth = 0;
+	if (depth)
+		depth--;
+
+	dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].cpce,
+			      HDMI_3D_TX_PHY_CPCE_CTRL);
+	dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].gmp,
+			      HDMI_3D_TX_PHY_DRVANACTRL);
+	dw_hdmi_phy_i2c_write(hdmi, curr_ctrl->curr[depth],
+			      HDMI_3D_TX_PHY_CURRCTRL);
+
+	dw_hdmi_phy_i2c_write(hdmi, 0, HDMI_3D_TX_PHY_PLLPHBYCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_FB_CLK,
+			      HDMI_3D_TX_PHY_MSM_CTRL);
+
+	dw_hdmi_phy_i2c_write(hdmi, phy_config->term, HDMI_3D_TX_PHY_TXTERM);
+	dw_hdmi_phy_i2c_write(hdmi, phy_config->sym_ctr,
+			      HDMI_3D_TX_PHY_CKSYMTXCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, phy_config->vlev_ctr,
+			      HDMI_3D_TX_PHY_VLEVCTRL);
+
+	/* Override and disable clock termination. */
+	dw_hdmi_phy_i2c_write(hdmi, HDMI_3D_TX_PHY_CKCALCTRL_OVERRIDE,
+			      HDMI_3D_TX_PHY_CKCALCTRL);
+
+	return 0;
+}
+
+static int hdmi_phy_configure(struct dw_hdmi *hdmi,
+			      const struct drm_display_info *display)
+{
+	const struct dw_hdmi_phy_data *phy = hdmi->phy.data;
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+	unsigned long mpixelclock = hdmi->hdmi_data.video_mode.mpixelclock;
+	unsigned long mtmdsclock = hdmi->hdmi_data.video_mode.mtmdsclock;
+	int ret;
+
+	dw_hdmi_phy_power_off(hdmi);
+
+	dw_hdmi_set_high_tmds_clock_ratio(hdmi, display);
+
+	/* Leave low power consumption mode by asserting SVSRET. */
+	if (phy->has_svsret)
+		dw_hdmi_phy_enable_svsret(hdmi, 1);
+
+	dw_hdmi_phy_gen2_reset(hdmi);
+
+	hdmi_writeb(hdmi, HDMI_MC_HEACPHY_RST_ASSERT, HDMI_MC_HEACPHY_RST);
+
+	dw_hdmi_phy_i2c_set_addr(hdmi, HDMI_PHY_I2CM_SLAVE_ADDR_PHY_GEN2);
+
+	/* Write to the PHY as configured by the platform */
+	if (pdata->configure_phy)
+		ret = pdata->configure_phy(hdmi, pdata->priv_data, mpixelclock);
+	else
+		ret = phy->configure(hdmi, pdata, mpixelclock);
+	if (ret) {
+		dev_err(hdmi->dev, "PHY configuration failed (clock %lu)\n",
+			mpixelclock);
+		return ret;
+	}
+
+	/* Wait for resuming transmission of TMDS clock and data */
+	if (mtmdsclock > HDMI14_MAX_TMDSCLK)
+		msleep(100);
+
+	return dw_hdmi_phy_power_on(hdmi);
+}
+
+static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
+			    const struct drm_display_info *display,
+			    const struct drm_display_mode *mode)
+{
+	int i, ret;
+
+	/* HDMI Phy spec says to do the phy initialization sequence twice */
+	for (i = 0; i < 2; i++) {
+		dw_hdmi_phy_sel_data_en_pol(hdmi, 1);
+		dw_hdmi_phy_sel_interface_control(hdmi, 0);
+
+		ret = hdmi_phy_configure(hdmi, display);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void dw_hdmi_phy_disable(struct dw_hdmi *hdmi, void *data)
+{
+	dw_hdmi_phy_power_off(hdmi);
+}
+
+enum drm_connector_status dw_hdmi_phy_read_hpd(struct dw_hdmi *hdmi,
+					       void *data)
+{
+	return hdmi_readb(hdmi, HDMI_PHY_STAT0) & HDMI_PHY_HPD ?
+		connector_status_connected : connector_status_disconnected;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_read_hpd);
+
+void dw_hdmi_phy_update_hpd(struct dw_hdmi *hdmi, void *data,
+			    bool force, bool disabled, bool rxsense)
+{
+	u8 old_mask = hdmi->phy_mask;
+
+	if (force || disabled || !rxsense)
+		hdmi->phy_mask |= HDMI_PHY_RX_SENSE;
+	else
+		hdmi->phy_mask &= ~HDMI_PHY_RX_SENSE;
+
+	if (old_mask != hdmi->phy_mask)
+		hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_update_hpd);
+
+void dw_hdmi_phy_setup_hpd(struct dw_hdmi *hdmi, void *data)
+{
+	/*
+	 * Configure the PHY RX SENSE and HPD interrupts polarities and clear
+	 * any pending interrupt.
+	 */
+	hdmi_writeb(hdmi, HDMI_PHY_HPD | HDMI_PHY_RX_SENSE, HDMI_PHY_POL0);
+	hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE,
+		    HDMI_IH_PHY_STAT0);
+
+	/* Enable cable hot plug irq. */
+	hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0);
+
+	/* Clear and unmute interrupts. */
+	hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE,
+		    HDMI_IH_PHY_STAT0);
+	hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE),
+		    HDMI_IH_MUTE_PHY_STAT0);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_phy_setup_hpd);
+
+static const struct dw_hdmi_phy_ops dw_hdmi_synopsys_phy_ops = {
+	.init = dw_hdmi_phy_init,
+	.disable = dw_hdmi_phy_disable,
+	.read_hpd = dw_hdmi_phy_read_hpd,
+	.update_hpd = dw_hdmi_phy_update_hpd,
+	.setup_hpd = dw_hdmi_phy_setup_hpd,
+};
+
+/* -----------------------------------------------------------------------------
+ * HDMI TX Setup
+ */
+
+static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi,
+				const struct drm_display_mode *mode)
+{
+	u8 de, vsync_pol, hsync_pol, hdmi_dvi;
+
+	/* Configure the video polarity */
+	if (hdmi->hdmi_data.video_mode.mdataenablepolarity)
+		de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH;
+	else
+		de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW;
+
+	vsync_pol = mode->flags & DRM_MODE_FLAG_PVSYNC ?
+			    HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_HIGH :
+			    HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_LOW;
+
+	hsync_pol = mode->flags & DRM_MODE_FLAG_PHSYNC ?
+			    HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_HIGH :
+			    HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW;
+
+	/* Config the display mode */
+	hdmi_dvi = hdmi->sink_is_hdmi ? HDMI_A_HDCPCFG0_HDMIDVI_HDMI :
+					HDMI_A_HDCPCFG0_HDMIDVI_DVI;
+
+	hdmi_modb(hdmi, hdmi_dvi, HDMI_A_HDCPCFG0_HDMIDVI_MASK,
+		  HDMI_A_HDCPCFG0);
+
+	/* disable rx detect */
+	hdmi_modb(hdmi, HDMI_A_HDCPCFG0_RXDETECT_DISABLE,
+		  HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0);
+
+	hdmi_modb(hdmi, vsync_pol | hsync_pol | de,
+		  HDMI_A_VIDPOLCFG_VSYNCPOL_MASK |
+			  HDMI_A_VIDPOLCFG_HSYNCPOL_MASK |
+			  HDMI_A_VIDPOLCFG_DATAENPOL_MASK,
+		  HDMI_A_VIDPOLCFG);
+
+	hdmi_modb(hdmi, HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE,
+		  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK, HDMI_A_HDCPCFG1);
+
+	if (hdmi->hdcp && hdmi->hdcp->hdcp_start) {
+		hdmi->hdcp->hdcp_start(hdmi->hdcp);
+	}
+}
+
+static void hdmi_config_AVI(struct dw_hdmi *hdmi,
+			    const struct drm_connector *connector,
+			    const struct drm_display_mode *mode)
+{
+	struct hdmi_avi_infoframe frame;
+	u8 val;
+
+	/* Initialise info frame from DRM mode */
+	drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode);
+
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) {
+		drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode,
+						   hdmi->hdmi_data.rgb_limited_range ?
+						   HDMI_QUANTIZATION_RANGE_LIMITED :
+						   HDMI_QUANTIZATION_RANGE_FULL);
+	} else {
+		frame.quantization_range = HDMI_QUANTIZATION_RANGE_DEFAULT;
+		frame.ycc_quantization_range =
+			HDMI_YCC_QUANTIZATION_RANGE_LIMITED;
+	}
+
+	if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format))
+		frame.colorspace = HDMI_COLORSPACE_YUV444;
+	else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		frame.colorspace = HDMI_COLORSPACE_YUV422;
+	else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format))
+		frame.colorspace = HDMI_COLORSPACE_YUV420;
+	else
+		frame.colorspace = HDMI_COLORSPACE_RGB;
+
+	/* Set up colorimetry */
+	if (!hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi->hdmi_data.enc_out_encoding) {
+		case V4L2_YCBCR_ENC_601:
+			if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV601)
+				frame.colorimetry = HDMI_COLORIMETRY_EXTENDED;
+			else
+				frame.colorimetry = HDMI_COLORIMETRY_ITU_601;
+			frame.extended_colorimetry =
+					HDMI_EXTENDED_COLORIMETRY_XV_YCC_601;
+			break;
+		case V4L2_YCBCR_ENC_709:
+			if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV709)
+				frame.colorimetry = HDMI_COLORIMETRY_EXTENDED;
+			else
+				frame.colorimetry = HDMI_COLORIMETRY_ITU_709;
+			frame.extended_colorimetry =
+					HDMI_EXTENDED_COLORIMETRY_XV_YCC_709;
+			break;
+		case V4L2_YCBCR_ENC_BT2020:
+			if (hdmi->hdmi_data.enc_in_encoding ==
+			    V4L2_YCBCR_ENC_BT2020)
+				frame.colorimetry = HDMI_COLORIMETRY_EXTENDED;
+			else
+				frame.colorimetry = HDMI_COLORIMETRY_ITU_709;
+			frame.extended_colorimetry =
+				HDMI_EXTENDED_COLORIMETRY_BT2020;
+			break;
+		default: /* Carries no data */
+			frame.colorimetry = HDMI_COLORIMETRY_ITU_601;
+			frame.extended_colorimetry =
+					HDMI_EXTENDED_COLORIMETRY_XV_YCC_601;
+			break;
+		}
+	} else {
+		frame.colorimetry = HDMI_COLORIMETRY_NONE;
+		frame.extended_colorimetry =
+			HDMI_EXTENDED_COLORIMETRY_XV_YCC_601;
+	}
+
+	frame.scan_mode = HDMI_SCAN_MODE_NONE;
+
+	/*
+	 * The Designware IP uses a different byte format from standard
+	 * AVI info frames, though generally the bits are in the correct
+	 * bytes.
+	 */
+
+	/*
+	 * AVI data byte 1 differences: Colorspace in bits 0,1 rather than 5,6,
+	 * scan info in bits 4,5 rather than 0,1 and active aspect present in
+	 * bit 6 rather than 4.
+	 */
+	val = (frame.scan_mode & 3) << 4 | (frame.colorspace & 3);
+	if (frame.active_aspect & 15)
+		val |= HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT;
+	if (frame.top_bar || frame.bottom_bar)
+		val |= HDMI_FC_AVICONF0_BAR_DATA_HORIZ_BAR;
+	if (frame.left_bar || frame.right_bar)
+		val |= HDMI_FC_AVICONF0_BAR_DATA_VERT_BAR;
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF0);
+
+	/* AVI data byte 2 differences: none */
+	val = ((frame.colorimetry & 0x3) << 6) |
+	      ((frame.picture_aspect & 0x3) << 4) |
+	      (frame.active_aspect & 0xf);
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF1);
+
+	/* AVI data byte 3 differences: none */
+	val = ((frame.extended_colorimetry & 0x7) << 4) |
+	      ((frame.quantization_range & 0x3) << 2) |
+	      (frame.nups & 0x3);
+	if (frame.itc)
+		val |= HDMI_FC_AVICONF2_IT_CONTENT_VALID;
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF2);
+
+	/* AVI data byte 4 differences: none */
+	val = frame.video_code & 0x7f;
+	hdmi_writeb(hdmi, val, HDMI_FC_AVIVID);
+
+	/* AVI Data Byte 5- set up input and output pixel repetition */
+	val = (((hdmi->hdmi_data.video_mode.mpixelrepetitioninput + 1) <<
+		HDMI_FC_PRCONF_INCOMING_PR_FACTOR_OFFSET) &
+		HDMI_FC_PRCONF_INCOMING_PR_FACTOR_MASK) |
+		((hdmi->hdmi_data.video_mode.mpixelrepetitionoutput <<
+		HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_OFFSET) &
+		HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK);
+	hdmi_writeb(hdmi, val, HDMI_FC_PRCONF);
+
+	/*
+	 * AVI data byte 5 differences: content type in 0,1 rather than 4,5,
+	 * ycc range in bits 2,3 rather than 6,7
+	 */
+	val = ((frame.ycc_quantization_range & 0x3) << 2) |
+	      (frame.content_type & 0x3);
+	hdmi_writeb(hdmi, val, HDMI_FC_AVICONF3);
+
+	/* AVI Data Bytes 6-13 */
+	hdmi_writeb(hdmi, frame.top_bar & 0xff, HDMI_FC_AVIETB0);
+	hdmi_writeb(hdmi, (frame.top_bar >> 8) & 0xff, HDMI_FC_AVIETB1);
+	hdmi_writeb(hdmi, frame.bottom_bar & 0xff, HDMI_FC_AVISBB0);
+	hdmi_writeb(hdmi, (frame.bottom_bar >> 8) & 0xff, HDMI_FC_AVISBB1);
+	hdmi_writeb(hdmi, frame.left_bar & 0xff, HDMI_FC_AVIELB0);
+	hdmi_writeb(hdmi, (frame.left_bar >> 8) & 0xff, HDMI_FC_AVIELB1);
+	hdmi_writeb(hdmi, frame.right_bar & 0xff, HDMI_FC_AVISRB0);
+	hdmi_writeb(hdmi, (frame.right_bar >> 8) & 0xff, HDMI_FC_AVISRB1);
+}
+
+static void hdmi_config_vendor_specific_infoframe(struct dw_hdmi *hdmi,
+						  const struct drm_connector *connector,
+						  const struct drm_display_mode *mode)
+{
+	struct hdmi_vendor_infoframe frame;
+	u8 buffer[10];
+	ssize_t err;
+
+	err = drm_hdmi_vendor_infoframe_from_display_mode(&frame, connector,
+							  mode);
+	if (err < 0)
+		/*
+		 * Going into that statement does not means vendor infoframe
+		 * fails. It just informed us that vendor infoframe is not
+		 * needed for the selected mode. Only 4k or stereoscopic 3D
+		 * mode requires vendor infoframe. So just simply return.
+		 */
+		return;
+
+	err = hdmi_vendor_infoframe_pack(&frame, buffer, sizeof(buffer));
+	if (err < 0) {
+		dev_err(hdmi->dev, "Failed to pack vendor infoframe: %zd\n",
+			err);
+		return;
+	}
+	hdmi_mask_writeb(hdmi, 0, HDMI_FC_DATAUTO0, HDMI_FC_DATAUTO0_VSD_OFFSET,
+			HDMI_FC_DATAUTO0_VSD_MASK);
+
+	/* Set the length of HDMI vendor specific InfoFrame payload */
+	hdmi_writeb(hdmi, buffer[2], HDMI_FC_VSDSIZE);
+
+	/* Set 24bit IEEE Registration Identifier */
+	hdmi_writeb(hdmi, buffer[4], HDMI_FC_VSDIEEEID0);
+	hdmi_writeb(hdmi, buffer[5], HDMI_FC_VSDIEEEID1);
+	hdmi_writeb(hdmi, buffer[6], HDMI_FC_VSDIEEEID2);
+
+	/* Set HDMI_Video_Format and HDMI_VIC/3D_Structure */
+	hdmi_writeb(hdmi, buffer[7], HDMI_FC_VSDPAYLOAD0);
+	hdmi_writeb(hdmi, buffer[8], HDMI_FC_VSDPAYLOAD1);
+
+	if (frame.s3d_struct >= HDMI_3D_STRUCTURE_SIDE_BY_SIDE_HALF)
+		hdmi_writeb(hdmi, buffer[9], HDMI_FC_VSDPAYLOAD2);
+
+	/* Packet frame interpolation */
+	hdmi_writeb(hdmi, 1, HDMI_FC_DATAUTO1);
+
+	/* Auto packets per frame and line spacing */
+	hdmi_writeb(hdmi, 0x11, HDMI_FC_DATAUTO2);
+
+	/* Configures the Frame Composer On RDRB mode */
+	hdmi_mask_writeb(hdmi, 1, HDMI_FC_DATAUTO0, HDMI_FC_DATAUTO0_VSD_OFFSET,
+			HDMI_FC_DATAUTO0_VSD_MASK);
+}
+
+static void hdmi_config_drm_infoframe(struct dw_hdmi *hdmi,
+				      const struct drm_connector *connector)
+{
+	const struct drm_connector_state *conn_state = connector->state;
+	struct hdmi_drm_infoframe frame;
+	u8 buffer[30];
+	ssize_t err;
+	int i;
+
+	if (!hdmi->plat_data->use_drm_infoframe)
+		return;
+
+	hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_DISABLE,
+		  HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN);
+
+	err = drm_hdmi_infoframe_set_hdr_metadata(&frame, conn_state);
+	if (err < 0)
+		return;
+
+	err = hdmi_drm_infoframe_pack(&frame, buffer, sizeof(buffer));
+	if (err < 0) {
+		dev_err(hdmi->dev, "Failed to pack drm infoframe: %zd\n", err);
+		return;
+	}
+
+	hdmi_writeb(hdmi, frame.version, HDMI_FC_DRM_HB0);
+	hdmi_writeb(hdmi, frame.length, HDMI_FC_DRM_HB1);
+
+	for (i = 0; i < frame.length; i++)
+		hdmi_writeb(hdmi, buffer[4 + i], HDMI_FC_DRM_PB0 + i);
+
+	hdmi_writeb(hdmi, 1, HDMI_FC_DRM_UP);
+	hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_ENABLE,
+		  HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN);
+}
+
+static void hdmi_av_composer(struct dw_hdmi *hdmi,
+			     const struct drm_display_info *display,
+			     const struct drm_display_mode *mode)
+{
+	u8 inv_val, bytes;
+	const struct drm_hdmi_info *hdmi_info = &display->hdmi;
+	struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode;
+	int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len;
+	unsigned int vdisplay, hdisplay;
+
+	vmode->mpixelclock = mode->clock * 1000;
+
+	dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock);
+
+	vmode->mtmdsclock = vmode->mpixelclock;
+
+	if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) {
+		switch (hdmi_bus_fmt_color_depth(
+				hdmi->hdmi_data.enc_out_bus_format)) {
+		case 16:
+			vmode->mtmdsclock = vmode->mpixelclock * 2;
+			break;
+		case 12:
+			vmode->mtmdsclock = vmode->mpixelclock * 3 / 2;
+			break;
+		case 10:
+			vmode->mtmdsclock = vmode->mpixelclock * 5 / 4;
+			break;
+		}
+	}
+
+	if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format))
+		vmode->mtmdsclock /= 2;
+
+	dev_dbg(hdmi->dev, "final tmdsclock = %d\n", vmode->mtmdsclock);
+
+	/* Set up HDMI_FC_INVIDCONF */
+	inv_val = (hdmi->hdmi_data.hdcp_enable ||
+		   (dw_hdmi_support_scdc(hdmi, display) &&
+		    (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK ||
+		     hdmi_info->scdc.scrambling.low_rates)) ?
+		HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE :
+		HDMI_FC_INVIDCONF_HDCP_KEEPOUT_INACTIVE);
+
+	inv_val |= mode->flags & DRM_MODE_FLAG_PVSYNC ?
+		HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_HIGH :
+		HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_LOW;
+
+	inv_val |= mode->flags & DRM_MODE_FLAG_PHSYNC ?
+		HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_HIGH :
+		HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_LOW;
+
+	inv_val |= (vmode->mdataenablepolarity ?
+		HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_HIGH :
+		HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_LOW);
+
+	if (hdmi->vic == 39)
+		inv_val |= HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH;
+	else
+		inv_val |= mode->flags & DRM_MODE_FLAG_INTERLACE ?
+			HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH :
+			HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_LOW;
+
+	inv_val |= mode->flags & DRM_MODE_FLAG_INTERLACE ?
+		HDMI_FC_INVIDCONF_IN_I_P_INTERLACED :
+		HDMI_FC_INVIDCONF_IN_I_P_PROGRESSIVE;
+
+	inv_val |= hdmi->sink_is_hdmi ?
+		HDMI_FC_INVIDCONF_DVI_MODEZ_HDMI_MODE :
+		HDMI_FC_INVIDCONF_DVI_MODEZ_DVI_MODE;
+
+	hdmi_writeb(hdmi, inv_val, HDMI_FC_INVIDCONF);
+
+	hdisplay = mode->hdisplay;
+	hblank = mode->htotal - mode->hdisplay;
+	h_de_hs = mode->hsync_start - mode->hdisplay;
+	hsync_len = mode->hsync_end - mode->hsync_start;
+
+	/*
+	 * When we're setting a YCbCr420 mode, we need
+	 * to adjust the horizontal timing to suit.
+	 */
+	if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) {
+		hdisplay /= 2;
+		hblank /= 2;
+		h_de_hs /= 2;
+		hsync_len /= 2;
+	}
+
+	vdisplay = mode->vdisplay;
+	vblank = mode->vtotal - mode->vdisplay;
+	v_de_vs = mode->vsync_start - mode->vdisplay;
+	vsync_len = mode->vsync_end - mode->vsync_start;
+
+	/*
+	 * When we're setting an interlaced mode, we need
+	 * to adjust the vertical timing to suit.
+	 */
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		vdisplay /= 2;
+		vblank /= 2;
+		v_de_vs /= 2;
+		vsync_len /= 2;
+	}
+
+	/* Scrambling Control */
+	if (dw_hdmi_support_scdc(hdmi, display)) {
+		if (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK ||
+		    hdmi_info->scdc.scrambling.low_rates) {
+			/*
+			 * HDMI2.0 Specifies the following procedure:
+			 * After the Source Device has determined that
+			 * SCDC_Present is set (=1), the Source Device should
+			 * write the accurate Version of the Source Device
+			 * to the Source Version field in the SCDCS.
+			 * Source Devices compliant shall set the
+			 * Source Version = 1.
+			 */
+			drm_scdc_readb(hdmi->ddc, SCDC_SINK_VERSION,
+				       &bytes);
+			drm_scdc_writeb(hdmi->ddc, SCDC_SOURCE_VERSION,
+				min_t(u8, bytes, SCDC_MIN_SOURCE_VERSION));
+
+			/* Enabled Scrambling in the Sink */
+			drm_scdc_set_scrambling(hdmi->curr_conn, 1);
+
+			/*
+			 * To activate the scrambler feature, you must ensure
+			 * that the quasi-static configuration bit
+			 * fc_invidconf.HDCP_keepout is set at configuration
+			 * time, before the required mc_swrstzreq.tmdsswrst_req
+			 * reset request is issued.
+			 */
+			hdmi_writeb(hdmi, (u8)~HDMI_MC_SWRSTZ_TMDSSWRST_REQ,
+				    HDMI_MC_SWRSTZ);
+			hdmi_writeb(hdmi, 1, HDMI_FC_SCRAMBLER_CTRL);
+		} else {
+			hdmi_writeb(hdmi, 0, HDMI_FC_SCRAMBLER_CTRL);
+			hdmi_writeb(hdmi, (u8)~HDMI_MC_SWRSTZ_TMDSSWRST_REQ,
+				    HDMI_MC_SWRSTZ);
+			drm_scdc_set_scrambling(hdmi->curr_conn, 0);
+		}
+	} else {
+		hdmi_writeb(hdmi, 0, HDMI_FC_SCRAMBLER_CTRL);
+	}
+
+	/* Set up horizontal active pixel width */
+	hdmi_writeb(hdmi, hdisplay >> 8, HDMI_FC_INHACTV1);
+	hdmi_writeb(hdmi, hdisplay, HDMI_FC_INHACTV0);
+
+	/* Set up vertical active lines */
+	hdmi_writeb(hdmi, vdisplay >> 8, HDMI_FC_INVACTV1);
+	hdmi_writeb(hdmi, vdisplay, HDMI_FC_INVACTV0);
+
+	/* Set up horizontal blanking pixel region width */
+	hdmi_writeb(hdmi, hblank >> 8, HDMI_FC_INHBLANK1);
+	hdmi_writeb(hdmi, hblank, HDMI_FC_INHBLANK0);
+
+	/* Set up vertical blanking pixel region width */
+	hdmi_writeb(hdmi, vblank, HDMI_FC_INVBLANK);
+
+	/* Set up HSYNC active edge delay width (in pixel clks) */
+	hdmi_writeb(hdmi, h_de_hs >> 8, HDMI_FC_HSYNCINDELAY1);
+	hdmi_writeb(hdmi, h_de_hs, HDMI_FC_HSYNCINDELAY0);
+
+	/* Set up VSYNC active edge delay (in lines) */
+	hdmi_writeb(hdmi, v_de_vs, HDMI_FC_VSYNCINDELAY);
+
+	/* Set up HSYNC active pulse width (in pixel clks) */
+	hdmi_writeb(hdmi, hsync_len >> 8, HDMI_FC_HSYNCINWIDTH1);
+	hdmi_writeb(hdmi, hsync_len, HDMI_FC_HSYNCINWIDTH0);
+
+	/* Set up VSYNC active edge delay (in lines) */
+	hdmi_writeb(hdmi, vsync_len, HDMI_FC_VSYNCINWIDTH);
+}
+
+/* HDMI Initialization Step B.4 */
+static void dw_hdmi_enable_video_path(struct dw_hdmi *hdmi)
+{
+	/* control period minimum duration */
+	hdmi_writeb(hdmi, 12, HDMI_FC_CTRLDUR);
+	hdmi_writeb(hdmi, 32, HDMI_FC_EXCTRLDUR);
+	hdmi_writeb(hdmi, 1, HDMI_FC_EXCTRLSPAC);
+
+	/* Set to fill TMDS data channels */
+	hdmi_writeb(hdmi, 0x0B, HDMI_FC_CH0PREAM);
+	hdmi_writeb(hdmi, 0x16, HDMI_FC_CH1PREAM);
+	hdmi_writeb(hdmi, 0x21, HDMI_FC_CH2PREAM);
+
+	/* Enable pixel clock and tmds data path */
+	hdmi->mc_clkdis |= HDMI_MC_CLKDIS_HDCPCLK_DISABLE |
+			   HDMI_MC_CLKDIS_CSCCLK_DISABLE |
+			   HDMI_MC_CLKDIS_AUDCLK_DISABLE |
+			   HDMI_MC_CLKDIS_PREPCLK_DISABLE |
+			   HDMI_MC_CLKDIS_TMDSCLK_DISABLE;
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_PIXELCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_TMDSCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+	/* Enable csc path */
+	if (is_csc_needed(hdmi)) {
+		hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CSCCLK_DISABLE;
+		hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+		hdmi_writeb(hdmi, HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_IN_PATH,
+			    HDMI_MC_FLOWCTRL);
+	} else {
+		hdmi->mc_clkdis |= HDMI_MC_CLKDIS_CSCCLK_DISABLE;
+		hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+
+		hdmi_writeb(hdmi, HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_BYPASS,
+			    HDMI_MC_FLOWCTRL);
+	}
+}
+
+/* Workaround to clear the overflow condition */
+static void dw_hdmi_clear_overflow(struct dw_hdmi *hdmi)
+{
+	unsigned int count;
+	unsigned int i;
+	u8 val;
+
+	/*
+	 * Under some circumstances the Frame Composer arithmetic unit can miss
+	 * an FC register write due to being busy processing the previous one.
+	 * The issue can be worked around by issuing a TMDS software reset and
+	 * then write one of the FC registers several times.
+	 *
+	 * The number of iterations matters and depends on the HDMI TX revision
+	 * (and possibly on the platform).
+	 * 4 iterations for i.MX6Q(v1.30a) and 1 iteration for others.
+	 * i.MX6DL (v1.31a), Allwinner SoCs (v1.32a), Rockchip RK3288 SoC (v2.00a),
+	 * Amlogic Meson GX SoCs (v2.01a), RK3328/RK3399 SoCs (v2.11a)
+	 * and i.MX8MPlus (v2.13a) have been identified as needing the workaround
+	 * with a single iteration.
+	 */
+
+	switch (hdmi->version) {
+	case 0x130a:
+		count = 4;
+		break;
+	default:
+		count = 1;
+		break;
+	}
+
+	/* TMDS software reset */
+	hdmi_writeb(hdmi, (u8)~HDMI_MC_SWRSTZ_TMDSSWRST_REQ, HDMI_MC_SWRSTZ);
+
+	val = hdmi_readb(hdmi, HDMI_FC_INVIDCONF);
+	for (i = 0; i < count; i++)
+		hdmi_writeb(hdmi, val, HDMI_FC_INVIDCONF);
+}
+
+static void hdmi_disable_overflow_interrupts(struct dw_hdmi *hdmi)
+{
+	hdmi_writeb(hdmi, HDMI_IH_MUTE_FC_STAT2_OVERFLOW_MASK,
+		    HDMI_IH_MUTE_FC_STAT2);
+}
+
+static int dw_hdmi_setup(struct dw_hdmi *hdmi,
+			 const struct drm_connector *connector,
+			 const struct drm_display_mode *mode)
+{
+	void *data = hdmi->plat_data->phy_data;
+	int ret;
+
+	hdmi_disable_overflow_interrupts(hdmi);
+
+	hdmi->vic = drm_match_cea_mode(mode);
+
+	if (!hdmi->vic) {
+		dev_dbg(hdmi->dev, "Non-CEA mode used in HDMI\n");
+	} else {
+		dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic);
+	}
+
+	if (hdmi->plat_data->get_enc_out_encoding)
+		hdmi->hdmi_data.enc_out_encoding =
+			hdmi->plat_data->get_enc_out_encoding(data);
+	else if ((hdmi->vic == 6) || (hdmi->vic == 7) || (hdmi->vic == 21) ||
+		 (hdmi->vic == 22) || (hdmi->vic == 2) || (hdmi->vic == 3) ||
+		 (hdmi->vic == 17) || (hdmi->vic == 18))
+		hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_601;
+	else
+		hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_709;
+
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK) {
+		hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1;
+		hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 1;
+	} else {
+		hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0;
+		hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0;
+	}
+
+	/* TOFIX: Get input format from plat data or fallback to RGB888 */
+	if (hdmi->plat_data->get_input_bus_format)
+		hdmi->hdmi_data.enc_in_bus_format =
+			hdmi->plat_data->get_input_bus_format(data);
+	else if (hdmi->hdmi_data.enc_in_bus_format == MEDIA_BUS_FMT_FIXED)
+		hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	else
+		hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+
+	/* TOFIX: Get input encoding from plat data or fallback to none */
+	if (hdmi->plat_data->get_enc_in_encoding)
+		hdmi->hdmi_data.enc_in_encoding =
+			hdmi->plat_data->get_enc_in_encoding(data);
+	else
+		hdmi->hdmi_data.enc_in_encoding = V4L2_YCBCR_ENC_DEFAULT;
+
+	/* TOFIX: Default to RGB888 output format */
+	if (hdmi->plat_data->get_output_bus_format)
+		hdmi->hdmi_data.enc_out_bus_format =
+			hdmi->plat_data->get_output_bus_format(data);
+	else if (hdmi->hdmi_data.enc_out_bus_format == MEDIA_BUS_FMT_FIXED)
+		hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	else
+		hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+
+	hdmi->hdmi_data.rgb_limited_range = hdmi->sink_is_hdmi &&
+		drm_default_rgb_quant_range(mode) ==
+		HDMI_QUANTIZATION_RANGE_LIMITED;
+
+	hdmi->hdmi_data.pix_repet_factor = 0;
+	hdmi->hdmi_data.hdcp_enable = 0;
+	hdmi->hdmi_data.video_mode.mdataenablepolarity = true;
+
+	/* HDMI Initialization Step B.1 */
+	hdmi_av_composer(hdmi, &connector->display_info, mode);
+
+	/* HDMI Initializateion Step B.2 */
+	ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data,
+				  &connector->display_info,
+				  &hdmi->previous_mode);
+	if (ret)
+		return ret;
+	hdmi->phy.enabled = true;
+
+	/* HDMI Initialization Step B.3 */
+	dw_hdmi_enable_video_path(hdmi);
+
+	if (hdmi->sink_has_audio) {
+		dev_dbg(hdmi->dev, "sink has audio support\n");
+
+		/* HDMI Initialization Step E - Configure audio */
+		hdmi_clk_regenerator_update_pixel_clock(hdmi);
+		hdmi_enable_audio_clk(hdmi, hdmi->audio_enable);
+	}
+
+	/* not for DVI mode */
+	if (hdmi->sink_is_hdmi) {
+		dev_dbg(hdmi->dev, "%s HDMI mode\n", __func__);
+
+		/* HDMI Initialization Step F - Configure AVI InfoFrame */
+		hdmi_config_AVI(hdmi, connector, mode);
+		hdmi_config_vendor_specific_infoframe(hdmi, connector, mode);
+		hdmi_config_drm_infoframe(hdmi, connector);
+	} else {
+		dev_dbg(hdmi->dev, "%s DVI mode\n", __func__);
+	}
+
+	hdmi_video_packetize(hdmi);
+	hdmi_video_csc(hdmi);
+	hdmi_video_sample(hdmi);
+	if (!hpd_flag) {
+		hdmi_tx_hdcp_config(hdmi, mode);
+	}
+	dw_hdmi_clear_overflow(hdmi);
+
+	return 0;
+}
+
+static void initialize_hdmi_ih_mutes(struct dw_hdmi *hdmi)
+{
+	u8 ih_mute;
+
+	/*
+	 * Boot up defaults are:
+	 * HDMI_IH_MUTE   = 0x03 (disabled)
+	 * HDMI_IH_MUTE_* = 0x00 (enabled)
+	 *
+	 * Disable top level interrupt bits in HDMI block
+	 */
+	ih_mute = hdmi_readb(hdmi, HDMI_IH_MUTE) |
+		  HDMI_IH_MUTE_MUTE_WAKEUP_INTERRUPT |
+		  HDMI_IH_MUTE_MUTE_ALL_INTERRUPT;
+
+	hdmi_writeb(hdmi, ih_mute, HDMI_IH_MUTE);
+
+	/* by default mask all interrupts */
+	hdmi_writeb(hdmi, 0xff, HDMI_VP_MASK);
+	hdmi_writeb(hdmi, 0xff, HDMI_FC_MASK0);
+	hdmi_writeb(hdmi, 0xff, HDMI_FC_MASK1);
+	hdmi_writeb(hdmi, 0xff, HDMI_FC_MASK2);
+	hdmi_writeb(hdmi, 0xff, HDMI_PHY_MASK0);
+	hdmi_writeb(hdmi, 0xff, HDMI_PHY_I2CM_INT_ADDR);
+	hdmi_writeb(hdmi, 0xff, HDMI_PHY_I2CM_CTLINT_ADDR);
+	hdmi_writeb(hdmi, 0xff, HDMI_AUD_INT);
+	hdmi_writeb(hdmi, 0xff, HDMI_AUD_SPDIFINT);
+	hdmi_writeb(hdmi, 0xff, HDMI_AUD_HBR_MASK);
+	hdmi_writeb(hdmi, 0xff, HDMI_GP_MASK);
+	hdmi_writeb(hdmi, 0xff, HDMI_A_APIINTMSK);
+	hdmi_writeb(hdmi, 0xff, HDMI_I2CM_INT);
+	hdmi_writeb(hdmi, 0xff, HDMI_I2CM_CTLINT);
+
+	/* Disable interrupts in the IH_MUTE_* registers */
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_FC_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_FC_STAT1);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_FC_STAT2);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_AS_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_PHY_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_I2CM_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_CEC_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_VP_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_I2CMPHY_STAT0);
+	hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_AHBDMAAUD_STAT0);
+
+	/* Enable top level interrupt bits in HDMI block */
+	ih_mute &= ~(HDMI_IH_MUTE_MUTE_WAKEUP_INTERRUPT |
+		     HDMI_IH_MUTE_MUTE_ALL_INTERRUPT);
+	hdmi_writeb(hdmi, ih_mute, HDMI_IH_MUTE);
+}
+
+static void dw_hdmi_poweron(struct dw_hdmi *hdmi)
+{
+	hdmi->bridge_is_on = true;
+
+	/*
+	 * The curr_conn field is guaranteed to be valid here, as this function
+	 * is only be called when !hdmi->disabled.
+	 */
+	dev_dbg(hdmi->dev, "%s", __func__);
+	dw_hdmi_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode);
+}
+
+static void dw_hdmi_poweroff(struct dw_hdmi *hdmi)
+{
+	dev_dbg(hdmi->dev, "%s", __func__);
+	if (hdmi->phy.enabled) {
+		hdmi->phy.ops->disable(hdmi, hdmi->phy.data);
+		hdmi->phy.enabled = false;
+	}
+
+	hdmi->bridge_is_on = false;
+}
+
+static void dw_hdmi_update_power(struct dw_hdmi *hdmi)
+{
+	int force = hdmi->force;
+
+	if (hdmi->disabled) {
+		force = DRM_FORCE_OFF;
+	} else if (force == DRM_FORCE_UNSPECIFIED) {
+		if (hdmi->rxsense)
+			force = DRM_FORCE_ON;
+		else
+			force = DRM_FORCE_OFF;
+	}
+
+	if (force == DRM_FORCE_OFF) {
+		if (hdmi->bridge_is_on)
+			dw_hdmi_poweroff(hdmi);
+	} else {
+		if (!hdmi->bridge_is_on) {
+			struct edid *edid;
+			edid = dw_hdmi_get_edid(hdmi, &hdmi->connector);
+			dw_hdmi_poweron(hdmi);
+		}
+	}
+}
+
+/*
+ * Adjust the detection of RXSENSE according to whether we have a forced
+ * connection mode enabled, or whether we have been disabled.  There is
+ * no point processing RXSENSE interrupts if we have a forced connection
+ * state, or DRM has us disabled.
+ *
+ * We also disable rxsense interrupts when we think we're disconnected
+ * to avoid floating TDMS signals giving false rxsense interrupts.
+ *
+ * Note: we still need to listen for HPD interrupts even when DRM has us
+ * disabled so that we can detect a connect event.
+ */
+static void dw_hdmi_update_phy_mask(struct dw_hdmi *hdmi)
+{
+	if (hdmi->phy.ops->update_hpd)
+		hdmi->phy.ops->update_hpd(hdmi, hdmi->phy.data,
+					  hdmi->force, hdmi->disabled,
+					  hdmi->rxsense);
+}
+
+static enum drm_connector_status dw_hdmi_detect(struct dw_hdmi *hdmi)
+{
+	enum drm_connector_status result;
+
+	result = hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data);
+	hdmi->last_connector_result = result;
+
+	return result;
+}
+
+static struct edid *dw_hdmi_get_edid(struct dw_hdmi *hdmi,
+				     struct drm_connector *connector)
+{
+	struct edid *edid;
+
+	if (!hdmi->ddc)
+		return NULL;
+
+	edid = drm_get_edid(connector, hdmi->ddc);
+	if (!edid) {
+		dev_err(hdmi->dev, "failed to get edid\n");
+		return NULL;
+	}
+
+	dev_info(hdmi->dev, "got edid: width[%d] x height[%d]\n",
+		 edid->width_cm, edid->height_cm);
+
+	hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid);
+	hdmi->sink_has_audio = drm_detect_monitor_audio(edid);
+
+	return edid;
+}
+
+/* -----------------------------------------------------------------------------
+ * DRM Connector Operations
+ */
+
+static enum drm_connector_status
+dw_hdmi_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
+					     connector);
+	return dw_hdmi_detect(hdmi);
+}
+
+static int dw_hdmi_connector_get_modes(struct drm_connector *connector)
+{
+	struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
+					     connector);
+	struct edid *edid;
+	int ret;
+
+	edid = dw_hdmi_get_edid(hdmi, connector);
+	if (!edid)
+		return 0;
+
+	drm_connector_update_edid_property(connector, edid);
+	cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid);
+	ret = drm_add_edid_modes(connector, edid);
+	kfree(edid);
+
+	return ret;
+}
+
+static int dw_hdmi_connector_atomic_check(struct drm_connector *connector,
+					  struct drm_atomic_state *state)
+{
+	struct drm_connector_state *old_state =
+		drm_atomic_get_old_connector_state(state, connector);
+	struct drm_connector_state *new_state =
+		drm_atomic_get_new_connector_state(state, connector);
+	struct drm_crtc *crtc = new_state->crtc;
+	struct drm_crtc_state *crtc_state;
+
+	if (!crtc)
+		return 0;
+
+	if (!drm_connector_atomic_hdr_metadata_equal(old_state, new_state)) {
+		crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+
+		crtc_state->mode_changed = true;
+	}
+
+	return 0;
+}
+
+static void dw_hdmi_connector_force(struct drm_connector *connector)
+{
+	struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
+					     connector);
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->force = connector->force;
+	dw_hdmi_update_power(hdmi);
+	dw_hdmi_update_phy_mask(hdmi);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static int dw_hdmi_atomic_connector_set_property(
+	struct drm_connector *connector, struct drm_connector_state *state,
+	struct drm_property *property, uint64_t val)
+{
+	struct dw_hdmi *hdmi =
+		container_of(connector, struct dw_hdmi, connector);
+	const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (ops && ops->set_property)
+		return ops->set_property(connector, state, property, val,
+					 hdmi->plat_data->phy_data);
+	else
+		return -EINVAL;
+}
+
+static int
+dw_hdmi_atomic_connector_get_property(struct drm_connector *connector,
+				      const struct drm_connector_state *state,
+				      struct drm_property *property,
+				      uint64_t *val)
+{
+	struct dw_hdmi *hdmi =
+		container_of(connector, struct dw_hdmi, connector);
+	const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (ops && ops->get_property)
+		return ops->get_property(connector, state, property, val,
+					 hdmi->plat_data->phy_data);
+	else
+		return -EINVAL;
+}
+
+static int dw_hdmi_connector_set_property(struct drm_connector *connector,
+					  struct drm_property *property,
+					  uint64_t val)
+{
+	return dw_hdmi_atomic_connector_set_property(connector, NULL, property,
+						     val);
+}
+
+static const struct drm_connector_funcs dw_hdmi_connector_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.detect = dw_hdmi_connector_detect,
+	.destroy = drm_connector_cleanup,
+	.force = dw_hdmi_connector_force,
+	.reset = drm_atomic_helper_connector_reset,
+	.set_property = dw_hdmi_connector_set_property,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.atomic_set_property = dw_hdmi_atomic_connector_set_property,
+	.atomic_get_property = dw_hdmi_atomic_connector_get_property,
+};
+
+static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = {
+	.get_modes = dw_hdmi_connector_get_modes,
+	.atomic_check = dw_hdmi_connector_atomic_check,
+};
+
+static void dw_hdmi_attatch_properties(struct dw_hdmi *hdmi)
+{
+	unsigned int color = MEDIA_BUS_FMT_RGB888_1X24;
+	int video_mapping, colorspace;
+
+	enum drm_connector_status connect_status =
+		hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data);
+		const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (connect_status == connector_status_connected) {
+		video_mapping = (hdmi_readb(hdmi, HDMI_TX_INVID0) &
+				 HDMI_TX_INVID0_VIDEO_MAPPING_MASK);
+		colorspace = (hdmi_readb(hdmi, HDMI_FC_AVICONF0) &
+			      HDMI_FC_AVICONF0_PIX_FMT_MASK);
+		switch (video_mapping) {
+		case 0x01:
+			color = MEDIA_BUS_FMT_RGB888_1X24;
+			break;
+		case 0x03:
+			color = MEDIA_BUS_FMT_RGB101010_1X30;
+			break;
+		case 0x09:
+			if (colorspace == HDMI_COLORSPACE_YUV420)
+				color = MEDIA_BUS_FMT_UYYVYY8_0_5X24;
+			else if (colorspace == HDMI_COLORSPACE_YUV422)
+				color = MEDIA_BUS_FMT_UYVY8_1X16;
+			else
+				color = MEDIA_BUS_FMT_YUV8_1X24;
+			break;
+		case 0x0b:
+			if (colorspace == HDMI_COLORSPACE_YUV420)
+				color = MEDIA_BUS_FMT_UYYVYY10_0_5X30;
+			else if (colorspace == HDMI_COLORSPACE_YUV422)
+				color = MEDIA_BUS_FMT_UYVY10_1X20;
+			else
+				color = MEDIA_BUS_FMT_YUV10_1X30;
+			break;
+		case 0x14:
+			color = MEDIA_BUS_FMT_UYVY10_1X20;
+			break;
+		case 0x16:
+			color = MEDIA_BUS_FMT_UYVY8_1X16;
+			break;
+		default:
+			color = MEDIA_BUS_FMT_RGB888_1X24;
+			dev_err(hdmi->dev, "unexpected mapping: 0x%x\n",
+				video_mapping);
+		}
+
+		hdmi->hdmi_data.enc_in_bus_format = color;
+		hdmi->hdmi_data.enc_out_bus_format = color;
+		/*
+         * input format will be set as yuv444 when output
+         * format is yuv420
+         */
+		if (color == MEDIA_BUS_FMT_UYVY10_1X20)
+			hdmi->hdmi_data.enc_in_bus_format =
+				MEDIA_BUS_FMT_YUV10_1X30;
+		else if (color == MEDIA_BUS_FMT_UYVY8_1X16)
+			hdmi->hdmi_data.enc_in_bus_format =
+				MEDIA_BUS_FMT_YUV8_1X24;
+	}
+
+	if (ops && ops->attatch_properties)
+		return ops->attatch_properties(&hdmi->connector, color,
+					       hdmi->version,
+					       hdmi->plat_data->phy_data);
+}
+
+static void dw_hdmi_destroy_properties(struct dw_hdmi *hdmi)
+{
+	const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops;
+
+	if (ops && ops->destroy_properties)
+		return ops->destroy_properties(&hdmi->connector,
+					       hdmi->plat_data->phy_data);
+}
+
+static int dw_hdmi_connector_create(struct dw_hdmi *hdmi)
+{
+	struct drm_connector *connector = &hdmi->connector;
+	struct cec_connector_info conn_info;
+	struct cec_notifier *notifier;
+
+	if (hdmi->version >= 0x200a)
+		connector->ycbcr_420_allowed =
+			hdmi->plat_data->ycbcr_420_allowed;
+	else
+		connector->ycbcr_420_allowed = false;
+
+	connector->interlace_allowed = 1;
+	connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+	drm_connector_helper_add(connector, &dw_hdmi_connector_helper_funcs);
+
+	drm_connector_init_with_ddc(hdmi->bridge.dev, connector,
+				    &dw_hdmi_connector_funcs,
+				    DRM_MODE_CONNECTOR_HDMIA,
+				    hdmi->ddc);
+
+	/*
+	 * drm_connector_attach_max_bpc_property() requires the
+	 * connector to have a state.
+	 */
+	drm_atomic_helper_connector_reset(connector);
+
+	drm_connector_attach_max_bpc_property(connector, 8, 16);
+
+	if (hdmi->version >= 0x200a && hdmi->plat_data->use_drm_infoframe)
+		drm_connector_attach_hdr_output_metadata_property(connector);
+
+	drm_connector_attach_encoder(connector, hdmi->bridge.encoder);
+
+	dw_hdmi_attatch_properties(hdmi);
+
+	cec_fill_conn_info_from_drm(&conn_info, connector);
+
+	notifier = cec_notifier_conn_register(hdmi->dev, NULL, &conn_info);
+	if (!notifier)
+		return -ENOMEM;
+
+	mutex_lock(&hdmi->cec_notifier_mutex);
+	hdmi->cec_notifier = notifier;
+	mutex_unlock(&hdmi->cec_notifier_mutex);
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * DRM Bridge Operations
+ */
+
+/*
+ * Possible output formats :
+ * - MEDIA_BUS_FMT_UYYVYY16_0_5X48,
+ * - MEDIA_BUS_FMT_UYYVYY12_0_5X36,
+ * - MEDIA_BUS_FMT_UYYVYY10_0_5X30,
+ * - MEDIA_BUS_FMT_UYYVYY8_0_5X24,
+ * - MEDIA_BUS_FMT_YUV16_1X48,
+ * - MEDIA_BUS_FMT_RGB161616_1X48,
+ * - MEDIA_BUS_FMT_UYVY12_1X24,
+ * - MEDIA_BUS_FMT_YUV12_1X36,
+ * - MEDIA_BUS_FMT_RGB121212_1X36,
+ * - MEDIA_BUS_FMT_UYVY10_1X20,
+ * - MEDIA_BUS_FMT_YUV10_1X30,
+ * - MEDIA_BUS_FMT_RGB101010_1X30,
+ * - MEDIA_BUS_FMT_UYVY8_1X16,
+ * - MEDIA_BUS_FMT_YUV8_1X24,
+ * - MEDIA_BUS_FMT_RGB888_1X24,
+ */
+
+/* Can return a maximum of 11 possible output formats for a mode/connector */
+#define MAX_OUTPUT_SEL_FORMATS	11
+
+static u32 *dw_hdmi_bridge_atomic_get_output_bus_fmts(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					unsigned int *num_output_fmts)
+{
+	struct drm_connector *conn = conn_state->connector;
+	struct drm_display_info *info = &conn->display_info;
+	struct drm_display_mode *mode = &crtc_state->mode;
+	u8 max_bpc = conn_state->max_requested_bpc;
+	bool is_hdmi2_sink = info->hdmi.scdc.supported ||
+			     (info->color_formats & DRM_COLOR_FORMAT_YCBCR420);
+	u32 *output_fmts;
+	unsigned int i = 0;
+
+	*num_output_fmts = 0;
+
+	output_fmts = kcalloc(MAX_OUTPUT_SEL_FORMATS, sizeof(*output_fmts),
+			      GFP_KERNEL);
+	if (!output_fmts)
+		return NULL;
+
+	/* If dw-hdmi is the first or only bridge, avoid negociating with ourselves */
+	if (list_is_singular(&bridge->encoder->bridge_chain) ||
+	    list_is_first(&bridge->chain_node, &bridge->encoder->bridge_chain)) {
+		*num_output_fmts = 1;
+		output_fmts[0] = MEDIA_BUS_FMT_FIXED;
+
+		return output_fmts;
+	}
+
+	/*
+	 * If the current mode enforces 4:2:0, force the output but format
+	 * to 4:2:0 and do not add the YUV422/444/RGB formats
+	 */
+	if (conn->ycbcr_420_allowed &&
+	    (drm_mode_is_420_only(info, mode) ||
+	     (is_hdmi2_sink && drm_mode_is_420_also(info, mode)))) {
+
+		/* Order bus formats from 16bit to 8bit if supported */
+		if (max_bpc >= 16 && info->bpc == 16 &&
+		    (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48))
+			output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY16_0_5X48;
+
+		if (max_bpc >= 12 && info->bpc >= 12 &&
+		    (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36))
+			output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY12_0_5X36;
+
+		if (max_bpc >= 10 && info->bpc >= 10 &&
+		    (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30))
+			output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY10_0_5X30;
+
+		/* Default 8bit fallback */
+		output_fmts[i++] = MEDIA_BUS_FMT_UYYVYY8_0_5X24;
+
+		if (drm_mode_is_420_only(info, mode)) {
+			*num_output_fmts = i;
+			return output_fmts;
+		}
+	}
+
+	/*
+	 * Order bus formats from 16bit to 8bit and from YUV422 to RGB
+	 * if supported. In any case the default RGB888 format is added
+	 */
+
+	/* Default 8bit RGB fallback */
+	output_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+
+	if (max_bpc >= 16 && info->bpc == 16) {
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			output_fmts[i++] = MEDIA_BUS_FMT_YUV16_1X48;
+
+		output_fmts[i++] = MEDIA_BUS_FMT_RGB161616_1X48;
+	}
+
+	if (max_bpc >= 12 && info->bpc >= 12) {
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			output_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			output_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+
+		output_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+	}
+
+	if (max_bpc >= 10 && info->bpc >= 10) {
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			output_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			output_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+
+		output_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+	}
+
+	if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+		output_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+
+	if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+		output_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+
+	*num_output_fmts = i;
+
+	return output_fmts;
+}
+
+/*
+ * Possible input formats :
+ * - MEDIA_BUS_FMT_RGB888_1X24
+ * - MEDIA_BUS_FMT_YUV8_1X24
+ * - MEDIA_BUS_FMT_UYVY8_1X16
+ * - MEDIA_BUS_FMT_UYYVYY8_0_5X24
+ * - MEDIA_BUS_FMT_RGB101010_1X30
+ * - MEDIA_BUS_FMT_YUV10_1X30
+ * - MEDIA_BUS_FMT_UYVY10_1X20
+ * - MEDIA_BUS_FMT_UYYVYY10_0_5X30
+ * - MEDIA_BUS_FMT_RGB121212_1X36
+ * - MEDIA_BUS_FMT_YUV12_1X36
+ * - MEDIA_BUS_FMT_UYVY12_1X24
+ * - MEDIA_BUS_FMT_UYYVYY12_0_5X36
+ * - MEDIA_BUS_FMT_RGB161616_1X48
+ * - MEDIA_BUS_FMT_YUV16_1X48
+ * - MEDIA_BUS_FMT_UYYVYY16_0_5X48
+ */
+
+/* Can return a maximum of 3 possible input formats for an output format */
+#define MAX_INPUT_SEL_FORMATS	3
+
+static u32 *dw_hdmi_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					u32 output_fmt,
+					unsigned int *num_input_fmts)
+{
+	u32 *input_fmts;
+	unsigned int i = 0;
+
+	*num_input_fmts = 0;
+
+	input_fmts = kcalloc(MAX_INPUT_SEL_FORMATS, sizeof(*input_fmts),
+			     GFP_KERNEL);
+	if (!input_fmts)
+		return NULL;
+
+	switch (output_fmt) {
+	/* If MEDIA_BUS_FMT_FIXED is tested, return default bus format */
+	case MEDIA_BUS_FMT_FIXED:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	/* 8bit */
+	case MEDIA_BUS_FMT_RGB888_1X24:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+		break;
+	case MEDIA_BUS_FMT_YUV8_1X24:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV8_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+
+	/* 10bit */
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+		break;
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV10_1X30;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB101010_1X30;
+		break;
+
+	/* 12bit */
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+		break;
+	case MEDIA_BUS_FMT_YUV12_1X36:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+		break;
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+		input_fmts[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV12_1X36;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
+		break;
+
+	/* 16bit */
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB161616_1X48;
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV16_1X48;
+		break;
+	case MEDIA_BUS_FMT_YUV16_1X48:
+		input_fmts[i++] = MEDIA_BUS_FMT_YUV16_1X48;
+		input_fmts[i++] = MEDIA_BUS_FMT_RGB161616_1X48;
+		break;
+
+	/*YUV 4:2:0 */
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		input_fmts[i++] = output_fmt;
+		break;
+	}
+
+	*num_input_fmts = i;
+
+	if (*num_input_fmts == 0) {
+		kfree(input_fmts);
+		input_fmts = NULL;
+	}
+
+	return input_fmts;
+}
+
+static int dw_hdmi_bridge_atomic_check(struct drm_bridge *bridge,
+				       struct drm_bridge_state *bridge_state,
+				       struct drm_crtc_state *crtc_state,
+				       struct drm_connector_state *conn_state)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	hdmi->hdmi_data.enc_out_bus_format =
+			bridge_state->output_bus_cfg.format;
+
+	hdmi->hdmi_data.enc_in_bus_format =
+			bridge_state->input_bus_cfg.format;
+
+	dev_dbg(hdmi->dev, "input format 0x%04x, output format 0x%04x\n",
+		bridge_state->input_bus_cfg.format,
+		bridge_state->output_bus_cfg.format);
+
+	return 0;
+}
+
+static int dw_hdmi_bridge_attach(struct drm_bridge *bridge,
+				 enum drm_bridge_attach_flags flags)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
+		return drm_bridge_attach(bridge->encoder, hdmi->next_bridge,
+					 bridge, flags);
+
+	return dw_hdmi_connector_create(hdmi);
+}
+
+static void dw_hdmi_bridge_detach(struct drm_bridge *bridge)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	mutex_lock(&hdmi->cec_notifier_mutex);
+	cec_notifier_conn_unregister(hdmi->cec_notifier);
+	hdmi->cec_notifier = NULL;
+	mutex_unlock(&hdmi->cec_notifier_mutex);
+}
+
+static enum drm_mode_status
+dw_hdmi_bridge_mode_valid(struct drm_bridge *bridge,
+			  const struct drm_display_info *info,
+			  const struct drm_display_mode *mode)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+	const struct dw_hdmi_plat_data *pdata = hdmi->plat_data;
+	enum drm_mode_status mode_status = MODE_OK;
+
+	/* We don't support double-clocked modes */
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+		return MODE_BAD;
+
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return MODE_NO_INTERLACE;
+
+	if (mode->clock != 594000 && mode->clock != 297000 &&
+	    mode->clock != 148500 && mode->clock != 108000 &&
+	    mode->clock != 74250 && mode->clock != 54000 &&
+	    mode->clock != 27000) {
+		return MODE_NOCLOCK;
+	}
+
+	if (pdata->mode_valid)
+		mode_status = pdata->mode_valid(hdmi, pdata->priv_data, info,
+						mode);
+
+	return mode_status;
+}
+
+static void dw_hdmi_bridge_mode_set(struct drm_bridge *bridge,
+				    const struct drm_display_mode *orig_mode,
+				    const struct drm_display_mode *mode)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	mutex_lock(&hdmi->mutex);
+
+	/* Store the display mode for plugin/DKMS poweron events */
+	drm_mode_copy(&hdmi->previous_mode, mode);
+
+	mutex_unlock(&hdmi->mutex);
+}
+
+static void dw_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
+					  struct drm_bridge_state *old_state)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->disabled = true;
+	hdmi->curr_conn = NULL;
+	dw_hdmi_update_power(hdmi);
+	dw_hdmi_update_phy_mask(hdmi);
+	handle_plugged_change(hdmi, false);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static void dw_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
+					 struct drm_bridge_state *old_state)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+	struct drm_atomic_state *state = old_state->base.state;
+	struct drm_connector *connector;
+
+	connector = drm_atomic_get_new_connector_for_encoder(state,
+							     bridge->encoder);
+
+	mutex_lock(&hdmi->mutex);
+	hdmi->disabled = false;
+	hdmi->curr_conn = connector;
+	dw_hdmi_update_power(hdmi);
+	dw_hdmi_update_phy_mask(hdmi);
+	handle_plugged_change(hdmi, true);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static enum drm_connector_status dw_hdmi_bridge_detect(struct drm_bridge *bridge)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	return dw_hdmi_detect(hdmi);
+}
+
+static struct edid *dw_hdmi_bridge_get_edid(struct drm_bridge *bridge,
+					    struct drm_connector *connector)
+{
+	struct dw_hdmi *hdmi = bridge->driver_private;
+
+	return dw_hdmi_get_edid(hdmi, connector);
+}
+
+static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = {
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.attach = dw_hdmi_bridge_attach,
+	.detach = dw_hdmi_bridge_detach,
+	.atomic_check = dw_hdmi_bridge_atomic_check,
+	.atomic_get_output_bus_fmts = dw_hdmi_bridge_atomic_get_output_bus_fmts,
+	.atomic_get_input_bus_fmts = dw_hdmi_bridge_atomic_get_input_bus_fmts,
+	.atomic_enable = dw_hdmi_bridge_atomic_enable,
+	.atomic_disable = dw_hdmi_bridge_atomic_disable,
+	.mode_set = dw_hdmi_bridge_mode_set,
+	.mode_valid = dw_hdmi_bridge_mode_valid,
+	.detect = dw_hdmi_bridge_detect,
+	.get_edid = dw_hdmi_bridge_get_edid,
+};
+
+/* -----------------------------------------------------------------------------
+ * IRQ Handling
+ */
+
+static irqreturn_t dw_hdmi_i2c_irq(struct dw_hdmi *hdmi)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	unsigned int stat;
+
+	stat = hdmi_readb(hdmi, HDMI_IH_I2CM_STAT0);
+	if (!stat)
+		return IRQ_NONE;
+
+	hdmi_writeb(hdmi, stat, HDMI_IH_I2CM_STAT0);
+
+	i2c->stat = stat;
+
+	complete(&i2c->cmp);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id)
+{
+	struct dw_hdmi *hdmi = dev_id;
+	u8 intr_stat, hdcp_stat;
+	irqreturn_t ret = IRQ_NONE;
+
+	if (hdmi->i2c)
+		ret = dw_hdmi_i2c_irq(hdmi);
+	if (ret == IRQ_HANDLED)
+		return ret;
+
+	intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0);
+	hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT);
+	if (intr_stat | hdcp_stat) {
+		if (intr_stat) {
+			dev_info(hdmi->dev, "intr status %#x\n", intr_stat);
+			hdmi_writeb(hdmi, 0xff, HDMI_IH_MUTE_PHY_STAT0);
+		}
+		if (hdcp_stat) {
+			dev_info(hdmi->dev, "HDCP status %#x\n", hdcp_stat);
+			hdmi_writeb(hdmi, 0xff, HDMI_A_APIINTMSK);
+		}
+		return IRQ_WAKE_THREAD;
+	}
+
+	return ret;
+}
+
+void dw_hdmi_setup_rx_sense(struct dw_hdmi *hdmi, bool hpd, bool rx_sense)
+{
+	mutex_lock(&hdmi->mutex);
+
+	if (!hdmi->force) {
+		/*
+		 * If the RX sense status indicates we're disconnected,
+		 * clear the software rxsense status.
+		 */
+		if (!rx_sense)
+			hdmi->rxsense = false;
+
+		/*
+		 * Only set the software rxsense status when both
+		 * rxsense and hpd indicates we're connected.
+		 * This avoids what seems to be bad behaviour in
+		 * at least iMX6S versions of the phy.
+		 */
+		if (hpd)
+			hdmi->rxsense = true;
+
+		dw_hdmi_update_power(hdmi);
+		dw_hdmi_update_phy_mask(hdmi);
+	}
+	mutex_unlock(&hdmi->mutex);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_setup_rx_sense);
+
+static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
+{
+	struct dw_hdmi *hdmi = dev_id;
+	u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat, hdcp_stat;
+
+	intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0);
+	if (intr_stat) {
+		phy_int_pol = hdmi_readb(hdmi, HDMI_PHY_POL0);
+		phy_stat = hdmi_readb(hdmi, HDMI_PHY_STAT0);
+		dev_dbg(hdmi->dev, "phy_int_pol:0x%x, phy_stat:0x%x\n",
+			phy_int_pol, phy_stat);
+
+		phy_pol_mask = 0;
+		if (intr_stat & HDMI_IH_PHY_STAT0_HPD)
+			phy_pol_mask |= HDMI_PHY_HPD;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE0)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE0;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE1)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE1;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE2)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE2;
+		if (intr_stat & HDMI_IH_PHY_STAT0_RX_SENSE3)
+			phy_pol_mask |= HDMI_PHY_RX_SENSE3;
+
+		if (phy_pol_mask)
+			hdmi_modb(hdmi, ~phy_int_pol, phy_pol_mask, HDMI_PHY_POL0);
+		if (phy_int_pol & HDMI_PHY_HPD)
+			hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_BUSCLEAR,
+				    HDMI_I2CM_OPERATION);
+
+		/*
+        * RX sense tells us whether the TDMS transmitters are detecting
+        * load - in other words, there's something listening on the
+        * other end of the link.  Use this to decide whether we should
+        * power on the phy as HPD may be toggled by the sink to merely
+        * ask the source to re-read the EDID.
+        */
+		if (intr_stat &
+		    (HDMI_IH_PHY_STAT0_RX_SENSE | HDMI_IH_PHY_STAT0_HPD)) {
+			hpd_flag = true;
+			dw_hdmi_setup_rx_sense(hdmi,
+							phy_stat & HDMI_PHY_HPD,
+					        phy_stat & HDMI_PHY_RX_SENSE);
+
+			if ((phy_stat & (HDMI_PHY_RX_SENSE | HDMI_PHY_HPD)) == 0) {
+				mutex_lock(&hdmi->cec_notifier_mutex);
+				cec_notifier_phys_addr_invalidate(hdmi->cec_notifier);
+				mutex_unlock(&hdmi->cec_notifier_mutex);
+			}
+		}
+
+		check_hdmi_irq(hdmi, intr_stat, phy_int_pol);
+		hdmi_writeb(hdmi, intr_stat, HDMI_IH_PHY_STAT0);
+		hdmi_writeb(hdmi,
+			    ~(HDMI_IH_PHY_STAT0_HPD |
+			      HDMI_IH_PHY_STAT0_RX_SENSE),
+			    HDMI_IH_MUTE_PHY_STAT0);
+	}
+
+	hpd_flag = false;
+
+	hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT);
+	if (hdcp_stat) {
+		if (hdmi->hdcp && hdmi->hdcp->hdcp_isr)
+			hdmi->hdcp->hdcp_isr(hdmi->hdcp, hdcp_stat);
+		hdmi_writeb(hdmi, hdcp_stat, HDMI_A_APIINTCLR);
+		hdmi_writeb(hdmi, 0x00, HDMI_A_APIINTMSK);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static const struct dw_hdmi_phy_data dw_hdmi_phys[] = {
+	{
+		.type = DW_HDMI_PHY_DWC_HDMI_TX_PHY,
+		.name = "DWC HDMI TX PHY",
+		.gen = 1,
+	}, {
+		.type = DW_HDMI_PHY_DWC_MHL_PHY_HEAC,
+		.name = "DWC MHL PHY + HEAC PHY",
+		.gen = 2,
+		.has_svsret = true,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_MHL_PHY,
+		.name = "DWC MHL PHY",
+		.gen = 2,
+		.has_svsret = true,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_HDMI_3D_TX_PHY_HEAC,
+		.name = "DWC HDMI 3D TX PHY + HEAC PHY",
+		.gen = 2,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_HDMI_3D_TX_PHY,
+		.name = "DWC HDMI 3D TX PHY",
+		.gen = 2,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_DWC_HDMI20_TX_PHY,
+		.name = "DWC HDMI 2.0 TX PHY",
+		.gen = 2,
+		.has_svsret = true,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
+	}, {
+		.type = DW_HDMI_PHY_VENDOR_PHY,
+		.name = "Vendor PHY",
+	}
+};
+
+static int dw_hdmi_detect_phy(struct dw_hdmi *hdmi)
+{
+	unsigned int i;
+	u8 phy_type;
+
+	phy_type = hdmi->plat_data->phy_force_vendor ?
+				DW_HDMI_PHY_VENDOR_PHY :
+				hdmi_readb(hdmi, HDMI_CONFIG2_ID);
+
+	if (phy_type == DW_HDMI_PHY_VENDOR_PHY) {
+		/* Vendor PHYs require support from the glue layer. */
+		if (!hdmi->plat_data->phy_ops || !hdmi->plat_data->phy_name) {
+			dev_err(hdmi->dev,
+				"Vendor HDMI PHY not supported by glue layer\n");
+			return -ENODEV;
+		}
+
+		hdmi->phy.ops = hdmi->plat_data->phy_ops;
+		hdmi->phy.data = hdmi->plat_data->phy_data;
+		hdmi->phy.name = hdmi->plat_data->phy_name;
+		return 0;
+	}
+
+	/* Synopsys PHYs are handled internally. */
+	for (i = 0; i < ARRAY_SIZE(dw_hdmi_phys); ++i) {
+		if (dw_hdmi_phys[i].type == phy_type) {
+			hdmi->phy.ops = &dw_hdmi_synopsys_phy_ops;
+			hdmi->phy.name = dw_hdmi_phys[i].name;
+			hdmi->phy.data = (void *)&dw_hdmi_phys[i];
+
+			if (!dw_hdmi_phys[i].configure &&
+			    !hdmi->plat_data->configure_phy) {
+				dev_err(hdmi->dev, "%s requires platform support\n",
+					hdmi->phy.name);
+				return -ENODEV;
+			}
+
+			return 0;
+		}
+	}
+
+	dev_err(hdmi->dev, "Unsupported HDMI PHY type (%02x)\n", phy_type);
+	return -ENODEV;
+}
+
+static void dw_hdmi_cec_enable(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->mutex);
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CECCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static void dw_hdmi_cec_disable(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->mutex);
+	hdmi->mc_clkdis |= HDMI_MC_CLKDIS_CECCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static const struct dw_hdmi_cec_ops dw_hdmi_cec_ops = {
+	.write = hdmi_writeb,
+	.read = hdmi_readb,
+	.enable = dw_hdmi_cec_enable,
+	.disable = dw_hdmi_cec_disable,
+};
+
+static int dw_hdmi_status_show(struct seq_file *s, void *v)
+{
+	struct dw_hdmi *hdmi = s->private;
+	u32 val;
+
+	seq_puts(s, "PHY: ");
+	if (!hdmi->phy.enabled) {
+		seq_puts(s, "disabled\n");
+		return 0;
+	}
+	seq_puts(s, "enabled\t\t\tMode: ");
+	if (hdmi->sink_is_hdmi)
+		seq_puts(s, "HDMI\n");
+	else
+		seq_puts(s, "DVI\n");
+	if (hdmi->hdmi_data.video_mode.mtmdsclock > 340000000)
+		val = hdmi->hdmi_data.video_mode.mtmdsclock / 4;
+	else
+		val = hdmi->hdmi_data.video_mode.mtmdsclock;
+	seq_printf(s, "Pixel Clk: %uHz\t\tTMDS Clk: %uHz\n",
+		   hdmi->hdmi_data.video_mode.mpixelclock, val);
+	seq_puts(s, "Color Format: ");
+	if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "RGB");
+	else if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "YUV444");
+	else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "YUV422");
+	else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format))
+		seq_puts(s, "YUV420");
+	else
+		seq_puts(s, "UNKNOWN");
+	val = hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format);
+	seq_printf(s, "\t\tColor Depth: %d bit\n", val);
+	seq_puts(s, "Colorimetry: ");
+	switch (hdmi->hdmi_data.enc_out_encoding) {
+	case V4L2_YCBCR_ENC_601:
+		seq_puts(s, "ITU.BT601");
+		break;
+	case V4L2_YCBCR_ENC_709:
+		seq_puts(s, "ITU.BT709");
+		break;
+	case V4L2_YCBCR_ENC_BT2020:
+		seq_puts(s, "ITU.BT2020");
+		break;
+	default: /* Carries no data */
+		seq_puts(s, "ITU.BT601");
+		break;
+	}
+
+	seq_puts(s, "\t\tEOTF: ");
+
+	if (hdmi->version < 0x211a) {
+		seq_puts(s, "Unsupported\n");
+		return 0;
+	}
+
+	val = hdmi_readb(hdmi, HDMI_FC_PACKET_TX_EN);
+	if (!(val & HDMI_FC_PACKET_TX_EN_DRM_MASK)) {
+		seq_puts(s, "Off\n");
+		return 0;
+	}
+
+	switch (hdmi_readb(hdmi, HDMI_FC_DRM_PB0)) {
+	case TRADITIONAL_GAMMA_SDR:
+		seq_puts(s, "SDR");
+		break;
+	case TRADITIONAL_GAMMA_HDR:
+		seq_puts(s, "HDR");
+		break;
+	case SMPTE_ST2084:
+		seq_puts(s, "ST2084");
+		break;
+	case HLG:
+		seq_puts(s, "HLG");
+		break;
+	default:
+		seq_puts(s, "Not Defined\n");
+		return 0;
+	}
+
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB3) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB2);
+	seq_printf(s, "\nx0: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB5) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB4);
+	seq_printf(s, "\t\t\t\ty0: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB7) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB6);
+	seq_printf(s, "x1: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB9) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB8);
+	seq_printf(s, "\t\t\t\ty1: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB11) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB10);
+	seq_printf(s, "x2: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB13) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB12);
+	seq_printf(s, "\t\t\t\ty2: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB15) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB14);
+	seq_printf(s, "white x: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB17) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB16);
+	seq_printf(s, "\t\t\twhite y: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB19) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB18);
+	seq_printf(s, "max lum: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB21) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB20);
+	seq_printf(s, "\t\t\tmin lum: %d\n", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB23) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB22);
+	seq_printf(s, "max cll: %d", val);
+	val = hdmi_readb(hdmi, HDMI_FC_DRM_PB25) << 8;
+	val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB24);
+	seq_printf(s, "\t\t\tmax fall: %d\n", val);
+	return 0;
+}
+
+static int dw_hdmi_status_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dw_hdmi_status_show, inode->i_private);
+}
+
+static const struct file_operations dw_hdmi_status_fops = {
+	.owner = THIS_MODULE,
+	.open = dw_hdmi_status_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+struct dw_hdmi_reg_table {
+	int reg_base;
+	int reg_end;
+};
+
+static const struct dw_hdmi_reg_table hdmi_reg_table[] = {
+	{ HDMI_DESIGN_ID, HDMI_CONFIG3_ID },
+	{ HDMI_IH_FC_STAT0, HDMI_IH_MUTE },
+	{ HDMI_TX_INVID0, HDMI_TX_BCBDATA1 },
+	{ HDMI_VP_STATUS, HDMI_VP_POL },
+	{ HDMI_FC_INVIDCONF, HDMI_FC_DBGTMDS2 },
+	{ HDMI_PHY_CONF0, HDMI_PHY_POL0 },
+	{ HDMI_PHY_I2CM_SLAVE_ADDR, HDMI_PHY_I2CM_FS_SCL_LCNT_0_ADDR },
+	{ HDMI_AUD_CONF0, 0x3624 },
+	{ HDMI_MC_SFRDIV, HDMI_MC_HEACPHY_RST },
+	{ HDMI_CSC_CFG, HDMI_CSC_COEF_C4_LSB },
+	{ HDMI_A_HDCPCFG0, 0x52bb },
+	{ 0x7800, 0x7818 },
+	{ 0x7900, 0x790e },
+	{ HDMI_CEC_CTRL, HDMI_CEC_WKUPCTRL },
+	{ HDMI_I2CM_SLAVE, HDMI_I2CM_SDA_HOLD },
+};
+
+static int dw_hdmi_ctrl_show(struct seq_file *s, void *v)
+{
+	struct dw_hdmi *hdmi = s->private;
+	u32 i = 0, j = 0, val = 0;
+
+	seq_puts(s, "\n>>>hdmi_ctl reg ");
+	for (i = 0; i < 16; i++)
+		seq_printf(s, " %2x", i);
+	seq_puts(s, "\n---------------------------------------------------");
+
+	for (i = 0; i < ARRAY_SIZE(hdmi_reg_table); i++) {
+		for (j = hdmi_reg_table[i].reg_base;
+		     j <= hdmi_reg_table[i].reg_end; j++) {
+			val = hdmi_readb(hdmi, j);
+			if ((j - hdmi_reg_table[i].reg_base) % 16 == 0)
+				seq_printf(s, "\n>>>hdmi_ctl %04x:", j);
+			seq_printf(s, " %02x", val);
+		}
+	}
+	seq_puts(s, "\n---------------------------------------------------\n");
+
+	return 0;
+}
+
+static int dw_hdmi_ctrl_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dw_hdmi_ctrl_show, inode->i_private);
+}
+
+static ssize_t dw_hdmi_ctrl_write(struct file *file, const char __user *buf,
+				  size_t count, loff_t *ppos)
+{
+	struct dw_hdmi *hdmi = ((struct seq_file *)file->private_data)->private;
+	u32 reg, val;
+	char kbuf[25];
+
+	if (copy_from_user(kbuf, buf, count))
+		return -EFAULT;
+	if (sscanf(kbuf, "%x%x", &reg, &val) == -1)
+		return -EFAULT;
+	if ((reg < 0) || (reg > HDMI_I2CM_FS_SCL_LCNT_0_ADDR)) {
+		dev_err(hdmi->dev, "it is no a hdmi register\n");
+		return count;
+	}
+	dev_info(hdmi->dev, "/**********hdmi register config******/");
+	dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val);
+	hdmi_writeb(hdmi, val, reg);
+	return count;
+}
+
+static const struct file_operations dw_hdmi_ctrl_fops = {
+	.owner = THIS_MODULE,
+	.open = dw_hdmi_ctrl_open,
+	.read = seq_read,
+	.write = dw_hdmi_ctrl_write,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static int dw_hdmi_phy_show(struct seq_file *s, void *v)
+{
+	struct dw_hdmi *hdmi = s->private;
+	u32 i, val;
+
+	seq_puts(s, "\n>>>hdmi_phy reg\n");
+	for (i = 0; i < 0x28; i++) {
+		val = hdmi_phy_i2c_read(hdmi, i);
+		seq_printf(s, "regs %02x val %04x\n", i, val);
+	}
+	return 0;
+}
+
+static int dw_hdmi_phy_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dw_hdmi_phy_show, inode->i_private);
+}
+
+static ssize_t dw_hdmi_phy_write(struct file *file, const char __user *buf,
+				 size_t count, loff_t *ppos)
+{
+	struct dw_hdmi *hdmi = ((struct seq_file *)file->private_data)->private;
+	u32 reg, val;
+	char kbuf[25];
+
+	if (copy_from_user(kbuf, buf, count))
+		return -EFAULT;
+	if (sscanf(kbuf, "%x%x", &reg, &val) == -1)
+		return -EFAULT;
+	if ((reg < 0) || (reg > 0x100)) {
+		dev_err(hdmi->dev, "it is not a hdmi phy register\n");
+		return count;
+	}
+	dev_info(hdmi->dev, "/*******hdmi phy register config******/");
+	dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val);
+	dw_hdmi_phy_i2c_write(hdmi, val, reg);
+	return count;
+}
+
+static const struct file_operations dw_hdmi_phy_fops = {
+	.owner = THIS_MODULE,
+	.open = dw_hdmi_phy_open,
+	.read = seq_read,
+	.write = dw_hdmi_phy_write,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static void dw_hdmi_register_debugfs(struct device *dev, struct dw_hdmi *hdmi)
+{
+	hdmi->debugfs_dir = debugfs_create_dir("dw-hdmi", NULL);
+	if (IS_ERR(hdmi->debugfs_dir)) {
+		dev_err(dev, "failed to create debugfs dir!\n");
+		return;
+	}
+
+	debugfs_create_file("status", 0400, hdmi->debugfs_dir, hdmi,
+			    &dw_hdmi_status_fops);
+
+	debugfs_create_file("ctrl", 0400, hdmi->debugfs_dir, hdmi,
+			    &dw_hdmi_ctrl_fops);
+
+	debugfs_create_file("phy", 0400, hdmi->debugfs_dir, hdmi,
+			    &dw_hdmi_phy_fops);
+}
+
+static void dw_hdmi_register_hdcp(struct device *dev, struct dw_hdmi *hdmi,
+				  u32 val, bool hdcp1x_enable)
+{
+	struct dw_hdcp hdmi_hdcp = {
+		.hdmi = hdmi,
+		.write = hdmi_writeb,
+		.read = hdmi_readb,
+		.regs = hdmi->regs,
+		.reg_io_width = val,
+		.enable = hdcp1x_enable,
+	};
+	struct platform_device_info hdcp_device_info = {
+		.parent = dev,
+		.id = PLATFORM_DEVID_AUTO,
+		.res = NULL,
+		.num_res = 0,
+		.name = DW_HDCP_DRIVER_NAME,
+		.data = &hdmi_hdcp,
+		.size_data = sizeof(hdmi_hdcp),
+		.dma_mask = DMA_BIT_MASK(32),
+	};
+
+	hdmi->hdcp_dev = platform_device_register_full(&hdcp_device_info);
+	if (IS_ERR(hdmi->hdcp_dev))
+		dev_err(dev, "failed to register hdcp!\n");
+	else
+		hdmi->hdcp = hdmi->hdcp_dev->dev.platform_data;
+}
+
+static const struct regmap_config hdmi_regmap_8bit_config = {
+	.reg_bits	= 32,
+	.val_bits	= 8,
+	.reg_stride	= 1,
+	.max_register	= HDMI_I2CM_SDA_HOLD,
+};
+
+static const struct regmap_config hdmi_regmap_32bit_config = {
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+	.max_register	= HDMI_I2CM_SDA_HOLD << 2,
+};
+
+static void dw_hdmi_init_hw(struct dw_hdmi *hdmi)
+{
+	initialize_hdmi_ih_mutes(hdmi);
+
+	/*
+	 * Reset HDMI DDC I2C master controller and mute I2CM interrupts.
+	 * Even if we are using a separate i2c adapter doing this doesn't
+	 * hurt.
+	 */
+	dw_hdmi_i2c_init(hdmi);
+
+	if (hdmi->phy.ops->setup_hpd)
+		hdmi->phy.ops->setup_hpd(hdmi, hdmi->phy.data);
+
+	/*
+     * HDMI status maybe incorrect in the following condition:
+     * HDMI plug in -> system sleep ->  HDMI plug out -> system wake up.
+     * At this time, cat /sys/class/drm/card 0-HDMI-A-1/status is connected.
+     * There is no hpd interrupt, because HDMI is powerdown during suspend.
+     * So we need check the current HDMI status in this case.
+     */
+	if (hdmi->connector.status == connector_status_connected)
+		if (hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data) ==
+		    connector_status_disconnected) {
+			hdmi->hpd_state = false;
+			mod_delayed_work(hdmi->workqueue, &hdmi->work,
+					 msecs_to_jiffies(20));
+		}
+}
+
+/* -----------------------------------------------------------------------------
+ * Probe/remove API, used from platforms based on the DRM bridge API.
+ */
+
+static int dw_hdmi_parse_dt(struct dw_hdmi *hdmi)
+{
+	struct device_node *endpoint;
+	struct device_node *remote;
+
+	if (!hdmi->plat_data->output_port)
+		return 0;
+
+	endpoint = of_graph_get_endpoint_by_regs(hdmi->dev->of_node,
+						 hdmi->plat_data->output_port,
+						 -1);
+	if (!endpoint) {
+		/*
+		 * On platforms whose bindings don't make the output port
+		 * mandatory (such as Rockchip) the plat_data->output_port
+		 * field isn't set, so it's safe to make this a fatal error.
+		 */
+		dev_err(hdmi->dev, "Missing endpoint in port@%u\n",
+			hdmi->plat_data->output_port);
+		return -ENODEV;
+	}
+
+	remote = of_graph_get_remote_port_parent(endpoint);
+	of_node_put(endpoint);
+	if (!remote) {
+		dev_err(hdmi->dev, "Endpoint in port@%u unconnected\n",
+			hdmi->plat_data->output_port);
+		return -ENODEV;
+	}
+
+	if (!of_device_is_available(remote)) {
+		dev_err(hdmi->dev, "port@%u remote device is disabled\n",
+			hdmi->plat_data->output_port);
+		of_node_put(remote);
+		return -ENODEV;
+	}
+
+	hdmi->next_bridge = of_drm_find_bridge(remote);
+	of_node_put(remote);
+	if (!hdmi->next_bridge)
+		return -EPROBE_DEFER;
+
+	return 0;
+}
+
+bool dw_hdmi_bus_fmt_is_420(struct dw_hdmi *hdmi)
+{
+	return hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_bus_fmt_is_420);
+
+void dw_hdmi_enable_video(struct dw_hdmi *hdmi)
+{
+	dev_info(hdmi->dev, "%s", __func__);
+	hdmi_writeb(hdmi, HDMI_FC_GCP_CLEAR_AVMUTE, HDMI_FC_GCP);
+}
+void dw_hdmi_disable_video(struct dw_hdmi *hdmi)
+{
+	dev_info(hdmi->dev, "%s", __func__);
+	hdmi_writeb(hdmi, HDMI_FC_GCP_SET_AVMUTE, HDMI_FC_GCP);
+}
+
+struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev,
+			      const struct dw_hdmi_plat_data *plat_data)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct platform_device_info pdevinfo;
+	struct device_node *ddc_node;
+	struct dw_hdmi_cec_data cec;
+	struct dw_hdmi *hdmi;
+	struct resource *iores = NULL;
+	int irq;
+	int ret;
+	u32 val = 1;
+	u8 prod_id0;
+	u8 prod_id1;
+	u8 config0;
+	u8 config3;
+	bool hdcp1x_enable = false;
+
+	hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return ERR_PTR(-ENOMEM);
+
+	hdmi->plat_data = plat_data;
+	hdmi->dev = dev;
+	hdmi->sample_rate = 48000;
+	hdmi->disabled = true;
+	hdmi->rxsense = true;
+	hdmi->phy_mask = (u8) ~(HDMI_PHY_HPD | HDMI_PHY_RX_SENSE);
+	hdmi->mc_clkdis = 0x7f;
+	hdmi->last_connector_result = connector_status_disconnected;
+
+	mutex_init(&hdmi->mutex);
+	mutex_init(&hdmi->audio_mutex);
+	mutex_init(&hdmi->cec_notifier_mutex);
+	spin_lock_init(&hdmi->audio_lock);
+
+	ret = dw_hdmi_parse_dt(hdmi);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	ddc_node = of_parse_phandle(np, "ddc-i2c-bus", 0);
+	if (ddc_node) {
+		hdmi->ddc = of_get_i2c_adapter_by_node(ddc_node);
+		of_node_put(ddc_node);
+		if (!hdmi->ddc) {
+			dev_dbg(hdmi->dev, "failed to read ddc node\n");
+			return ERR_PTR(-EPROBE_DEFER);
+		}
+
+	} else {
+		dev_dbg(hdmi->dev, "no ddc property found\n");
+	}
+
+	if (!plat_data->regm) {
+		const struct regmap_config *reg_config;
+
+		of_property_read_u32(np, "reg-io-width", &val);
+		switch (val) {
+		case 4:
+			reg_config = &hdmi_regmap_32bit_config;
+			hdmi->reg_shift = 2;
+			break;
+		case 1:
+			reg_config = &hdmi_regmap_8bit_config;
+			break;
+		default:
+			dev_err(dev, "reg-io-width must be 1 or 4\n");
+			return ERR_PTR(-EINVAL);
+		}
+
+		iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		hdmi->regs = devm_ioremap_resource(dev, iores);
+		if (IS_ERR(hdmi->regs)) {
+			ret = PTR_ERR(hdmi->regs);
+			goto err_res;
+		}
+
+		hdmi->regm = devm_regmap_init_mmio(dev, hdmi->regs, reg_config);
+		if (IS_ERR(hdmi->regm)) {
+			dev_err(dev, "Failed to configure regmap\n");
+			ret = PTR_ERR(hdmi->regm);
+			goto err_res;
+		}
+	} else {
+		hdmi->regm = plat_data->regm;
+	}
+
+	hdmi->isfr_clk = devm_clk_get(hdmi->dev, "isfr");
+	if (IS_ERR(hdmi->isfr_clk)) {
+		ret = PTR_ERR(hdmi->isfr_clk);
+		dev_err(hdmi->dev, "Unable to get HDMI isfr clk: %d\n", ret);
+		goto err_res;
+	}
+
+	ret = clk_prepare_enable(hdmi->isfr_clk);
+	if (ret) {
+		dev_err(hdmi->dev, "Cannot enable HDMI isfr clock: %d\n", ret);
+		goto err_res;
+	}
+
+	hdmi->iahb_clk = devm_clk_get(hdmi->dev, "iahb");
+	if (IS_ERR(hdmi->iahb_clk)) {
+		ret = PTR_ERR(hdmi->iahb_clk);
+		dev_err(hdmi->dev, "Unable to get HDMI iahb clk: %d\n", ret);
+		goto err_isfr;
+	}
+
+	ret = clk_prepare_enable(hdmi->iahb_clk);
+	if (ret) {
+		dev_err(hdmi->dev, "Cannot enable HDMI iahb clock: %d\n", ret);
+		goto err_isfr;
+	}
+
+	hdmi->cec_clk = devm_clk_get(hdmi->dev, "cec");
+	if (PTR_ERR(hdmi->cec_clk) == -ENOENT) {
+		hdmi->cec_clk = NULL;
+	} else if (IS_ERR(hdmi->cec_clk)) {
+		ret = PTR_ERR(hdmi->cec_clk);
+		if (ret != -EPROBE_DEFER)
+			dev_err(hdmi->dev, "Cannot get HDMI cec clock: %d\n",
+				ret);
+
+		hdmi->cec_clk = NULL;
+		goto err_iahb;
+	} else {
+		ret = clk_prepare_enable(hdmi->cec_clk);
+		if (ret) {
+			dev_err(hdmi->dev, "Cannot enable HDMI cec clock: %d\n",
+				ret);
+			goto err_iahb;
+		}
+
+		ret = clk_set_rate(hdmi->cec_clk, HDMI_CEC_CLK);
+		if (ret) {
+			dev_err(hdmi->dev, "Cannot set CEC clock rate, error:%d\n", ret);
+			goto err_iahb;
+		}
+	}
+
+	/* hdmi prstn reset */
+	hdmi->rst_hdmi_prstn =
+		devm_reset_control_get_optional(hdmi->dev, "prstn");
+	if (IS_ERR_OR_NULL(hdmi->rst_hdmi_prstn)) {
+		dev_err(hdmi->dev, "Failed to get hdmi prstn reset handle\n");
+		ret = -EFAULT;
+		goto err_cec;
+	}
+
+	/* hdmi phyctl reset */
+	hdmi->rst_hdmi_phyrstn =
+		devm_reset_control_get_optional(hdmi->dev, "phyrstn");
+	if (IS_ERR_OR_NULL(hdmi->rst_hdmi_phyrstn)) {
+		dev_err(hdmi->dev, "Failed to get hdmi phyrstn reset handle\n");
+		ret = -EFAULT;
+		goto err_cec;
+	}
+
+	/* hdmi rstn reset */
+	hdmi->rst_hdmi_rstn =
+		devm_reset_control_get_optional(hdmi->dev, "rstn");
+	if (IS_ERR_OR_NULL(hdmi->rst_hdmi_rstn)) {
+		dev_err(hdmi->dev, "Failed to get hdmi rstn reset handle\n");
+		ret = -EFAULT;
+		goto err_cec;
+	}
+
+	if (hdmi->rst_hdmi_prstn) {
+		ret = reset_control_reset(hdmi->rst_hdmi_prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_phyrstn) {
+		ret = reset_control_reset(hdmi->rst_hdmi_phyrstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_rstn) {
+		ret = reset_control_reset(hdmi->rst_hdmi_rstn);
+		WARN_ON(0 != ret);
+	}
+
+	/* Product and revision IDs */
+	hdmi->version = (hdmi_readb(hdmi, HDMI_DESIGN_ID) << 8)
+		      | (hdmi_readb(hdmi, HDMI_REVISION_ID) << 0);
+	prod_id0 = hdmi_readb(hdmi, HDMI_PRODUCT_ID0);
+	prod_id1 = hdmi_readb(hdmi, HDMI_PRODUCT_ID1);
+
+	if (prod_id0 != HDMI_PRODUCT_ID0_HDMI_TX ||
+	    (prod_id1 & ~HDMI_PRODUCT_ID1_HDCP) != HDMI_PRODUCT_ID1_HDMI_TX) {
+		dev_err(dev, "Unsupported HDMI controller (%04x:%02x:%02x)\n",
+			hdmi->version, prod_id0, prod_id1);
+		ret = -ENODEV;
+		goto err_cec;
+	}
+
+	ret = dw_hdmi_detect_phy(hdmi);
+	if (ret < 0)
+		goto err_cec;
+
+	dev_info(dev, "Detected HDMI TX controller v%x.%03x %s HDCP (%s)\n",
+		 hdmi->version >> 12, hdmi->version & 0xfff,
+		 prod_id1 & HDMI_PRODUCT_ID1_HDCP ? "with" : "without",
+		 hdmi->phy.name);
+
+	dw_hdmi_init_hw(hdmi);
+	init_hpd_work(hdmi);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
+		goto err_cec;
+	}
+
+	ret = devm_request_threaded_irq(dev, irq, dw_hdmi_hardirq,
+					dw_hdmi_irq, IRQF_SHARED,
+					dev_name(dev), hdmi);
+	if (ret)
+		goto err_cec;
+
+	/*
+	 * To prevent overflows in HDMI_IH_FC_STAT2, set the clk regenerator
+	 * N and cts values before enabling phy
+	 */
+	hdmi_init_clk_regenerator(hdmi);
+
+	/* If DDC bus is not specified, try to register HDMI I2C bus */
+	if (!hdmi->ddc) {
+		/* Look for (optional) stuff related to unwedging */
+		hdmi->pinctrl = devm_pinctrl_get(dev);
+		if (!IS_ERR(hdmi->pinctrl)) {
+			hdmi->unwedge_state =
+				pinctrl_lookup_state(hdmi->pinctrl, "unwedge");
+			hdmi->default_state =
+				pinctrl_lookup_state(hdmi->pinctrl, "default");
+
+			if (IS_ERR(hdmi->default_state) ||
+			    IS_ERR(hdmi->unwedge_state)) {
+				if (!IS_ERR(hdmi->unwedge_state))
+					dev_warn(dev,
+						 "Unwedge requires default pinctrl\n");
+				hdmi->default_state = NULL;
+				hdmi->unwedge_state = NULL;
+			}
+		}
+
+		hdmi->ddc = dw_hdmi_i2c_adapter(hdmi);
+		if (IS_ERR(hdmi->ddc))
+			hdmi->ddc = NULL;
+	}
+
+	hdmi->bridge.driver_private = hdmi;
+	hdmi->bridge.funcs = &dw_hdmi_bridge_funcs;
+	hdmi->bridge.ops = DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID
+			 | DRM_BRIDGE_OP_HPD;
+	hdmi->bridge.interlace_allowed = true;
+	hdmi->bridge.ddc = hdmi->ddc;
+#ifdef CONFIG_OF
+	hdmi->bridge.of_node = pdev->dev.of_node;
+#endif
+
+	memset(&pdevinfo, 0, sizeof(pdevinfo));
+	pdevinfo.parent = dev;
+	pdevinfo.id = PLATFORM_DEVID_AUTO;
+
+	config0 = hdmi_readb(hdmi, HDMI_CONFIG0_ID);
+	config3 = hdmi_readb(hdmi, HDMI_CONFIG3_ID);
+
+	if (iores && config3 & HDMI_CONFIG3_AHBAUDDMA) {
+		struct dw_hdmi_audio_data audio;
+
+		audio.phys = iores->start;
+		audio.base = hdmi->regs;
+		audio.irq = irq;
+		audio.hdmi = hdmi;
+		audio.get_eld = hdmi_audio_get_eld;
+		hdmi->enable_audio = dw_hdmi_ahb_audio_enable;
+		hdmi->disable_audio = dw_hdmi_ahb_audio_disable;
+
+		pdevinfo.name = "dw-hdmi-ahb-audio";
+		pdevinfo.data = &audio;
+		pdevinfo.size_data = sizeof(audio);
+		pdevinfo.dma_mask = DMA_BIT_MASK(32);
+		hdmi->audio = platform_device_register_full(&pdevinfo);
+	} else if (config0 & HDMI_CONFIG0_I2S) {
+		struct dw_hdmi_i2s_audio_data audio;
+
+		audio.hdmi	= hdmi;
+		audio.get_eld	= hdmi_audio_get_eld;
+		audio.write	= hdmi_writeb;
+		audio.read	= hdmi_readb;
+		hdmi->enable_audio = dw_hdmi_i2s_audio_enable;
+		hdmi->disable_audio = dw_hdmi_i2s_audio_disable;
+
+		pdevinfo.name = "dw-hdmi-i2s-audio";
+		pdevinfo.data = &audio;
+		pdevinfo.size_data = sizeof(audio);
+		pdevinfo.dma_mask = DMA_BIT_MASK(32);
+		hdmi->audio = platform_device_register_full(&pdevinfo);
+	} else if (iores && config3 & HDMI_CONFIG3_GPAUD) {
+		struct dw_hdmi_audio_data audio;
+
+		audio.phys = iores->start;
+		audio.base = hdmi->regs;
+		audio.irq = irq;
+		audio.hdmi = hdmi;
+		audio.get_eld = hdmi_audio_get_eld;
+
+		hdmi->enable_audio = dw_hdmi_gp_audio_enable;
+		hdmi->disable_audio = dw_hdmi_gp_audio_disable;
+
+		pdevinfo.name = "dw-hdmi-gp-audio";
+		pdevinfo.id = PLATFORM_DEVID_NONE;
+		pdevinfo.data = &audio;
+		pdevinfo.size_data = sizeof(audio);
+		pdevinfo.dma_mask = DMA_BIT_MASK(32);
+		hdmi->audio = platform_device_register_full(&pdevinfo);
+	}
+
+	if (!plat_data->disable_cec && (config0 & HDMI_CONFIG0_CEC)) {
+		cec.hdmi = hdmi;
+		cec.ops = &dw_hdmi_cec_ops;
+		cec.irq = irq;
+
+		pdevinfo.name = "dw-hdmi-cec";
+		pdevinfo.data = &cec;
+		pdevinfo.size_data = sizeof(cec);
+		pdevinfo.dma_mask = 0;
+
+		hdmi->cec = platform_device_register_full(&pdevinfo);
+	}
+
+	drm_bridge_add(&hdmi->bridge);
+
+	dw_hdmi_register_debugfs(dev, hdmi);
+
+	if (of_property_read_bool(np, "hdcp1x-enable"))
+		hdcp1x_enable = true;
+	dw_hdmi_register_hdcp(dev, hdmi, val, hdcp1x_enable);
+
+	return hdmi;
+
+err_cec:
+	clk_disable_unprepare(hdmi->cec_clk);
+err_iahb:
+	clk_disable_unprepare(hdmi->iahb_clk);
+err_isfr:
+	clk_disable_unprepare(hdmi->isfr_clk);
+err_res:
+	i2c_put_adapter(hdmi->ddc);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_probe);
+
+void dw_hdmi_remove(struct dw_hdmi *hdmi)
+{
+	int ret;
+
+	drm_bridge_remove(&hdmi->bridge);
+
+	cancel_delayed_work(&hdmi->work);
+	flush_workqueue(hdmi->workqueue);
+	destroy_workqueue(hdmi->workqueue);
+
+	if (hdmi->audio && !IS_ERR(hdmi->audio))
+		platform_device_unregister(hdmi->audio);
+	if (!IS_ERR(hdmi->cec))
+		platform_device_unregister(hdmi->cec);
+	if (hdmi->hdcp_dev && !IS_ERR(hdmi->hdcp_dev))
+		platform_device_unregister(hdmi->hdcp_dev);
+
+	/* Disable all interrupts */
+	hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0);
+
+	dw_hdmi_destroy_properties(hdmi);
+	hdmi->connector.funcs->destroy(&hdmi->connector);
+
+	clk_disable_unprepare(hdmi->iahb_clk);
+	clk_disable_unprepare(hdmi->isfr_clk);
+	clk_disable_unprepare(hdmi->cec_clk);
+
+	if (hdmi->i2c)
+		i2c_del_adapter(&hdmi->i2c->adap);
+	else
+		i2c_put_adapter(hdmi->ddc);
+
+	debugfs_remove_recursive(hdmi->debugfs_dir);
+
+	if (hdmi->rst_hdmi_prstn) {
+		ret = reset_control_assert(hdmi->rst_hdmi_prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_phyrstn) {
+		ret = reset_control_assert(hdmi->rst_hdmi_phyrstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (hdmi->rst_hdmi_rstn) {
+		ret = reset_control_assert(hdmi->rst_hdmi_rstn);
+		WARN_ON(0 != ret);
+	}
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_remove);
+
+/* -----------------------------------------------------------------------------
+ * Bind/unbind API, used from platforms based on the component framework.
+ */
+struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev,
+			     struct drm_encoder *encoder,
+			     const struct dw_hdmi_plat_data *plat_data)
+{
+	struct dw_hdmi *hdmi;
+	int ret;
+
+	hdmi = dw_hdmi_probe(pdev, plat_data);
+	if (IS_ERR(hdmi))
+		return hdmi;
+
+	ret = drm_bridge_attach(encoder, &hdmi->bridge, NULL, 0);
+	if (ret) {
+		dw_hdmi_remove(hdmi);
+		DRM_ERROR("Failed to initialize bridge with drm\n");
+		return ERR_PTR(ret);
+	}
+
+	return hdmi;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_bind);
+
+void dw_hdmi_unbind(struct dw_hdmi *hdmi)
+{
+	dw_hdmi_remove(hdmi);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_unbind);
+
+void dw_hdmi_suspend(struct dw_hdmi *hdmi)
+{
+	if (!hdmi) {
+		dev_warn(hdmi->dev, "HDMI has not been initialized\n");
+		return;
+	}
+
+	mutex_lock(&hdmi->mutex);
+
+	/*
+     * When system shutdown, hdmi should be disabled.
+     * When system suspend, dw_hdmi_bridge_disable will disable hdmi first.
+     * To prevent duplicate operation, we should determine whether hdmi
+     * has been disabled.
+     */
+	if (!hdmi->disabled) {
+		hdmi->disabled = true;
+		dw_hdmi_update_power(hdmi);
+		dw_hdmi_update_phy_mask(hdmi);
+	}
+	mutex_unlock(&hdmi->mutex);
+
+	if (hdmi->irq)
+		disable_irq(hdmi->irq);
+	cancel_delayed_work(&hdmi->work);
+	flush_workqueue(hdmi->workqueue);
+	pinctrl_pm_select_sleep_state(hdmi->dev);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_suspend);
+
+void dw_hdmi_resume(struct dw_hdmi *hdmi)
+{
+	dw_hdmi_init_hw(hdmi);
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_resume);
+
+MODULE_AUTHOR("Sascha Hauer <s.hauer@pengutronix.de>");
+MODULE_AUTHOR("Andy Yan <andy.yan@rock-chips.com>");
+MODULE_AUTHOR("Yakir Yang <ykk@rock-chips.com>");
+MODULE_AUTHOR("Vladimir Zapolskiy <vladimir_zapolskiy@mentor.com>");
+MODULE_DESCRIPTION("DW HDMI transmitter driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:dw-hdmi");
diff --git a/drivers/gpu/drm/eswin/dw-hdmi.h b/drivers/gpu/drm/eswin/dw-hdmi.h
new file mode 100644
index 000000000000..147774f90094
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw-hdmi.h
@@ -0,0 +1,1272 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2011 Freescale Semiconductor, Inc.
+ */
+
+#ifndef __DW_HDMI_H__
+#define __DW_HDMI_H__
+
+#include <sound/hdmi-codec.h>
+#include <drm/drm_property.h>
+
+extern struct platform_driver dw_hdmi_eswin_pltfm_driver;
+extern struct platform_driver snd_dw_hdmi_driver;
+extern struct platform_driver dw_hdmi_cec_driver;
+extern struct platform_driver dw_hdmi_hdcp_driver;
+extern struct platform_driver dw_hdmi_hdcp2_driver;
+
+/* Identification Registers */
+#define HDMI_DESIGN_ID                          0x0000
+#define HDMI_REVISION_ID                        0x0001
+#define HDMI_PRODUCT_ID0                        0x0002
+#define HDMI_PRODUCT_ID1                        0x0003
+#define HDMI_CONFIG0_ID                         0x0004
+#define HDMI_CONFIG1_ID                         0x0005
+#define HDMI_CONFIG2_ID                         0x0006
+#define HDMI_CONFIG3_ID                         0x0007
+
+/* Interrupt Registers */
+#define HDMI_IH_FC_STAT0                        0x0100
+#define HDMI_IH_FC_STAT1                        0x0101
+#define HDMI_IH_FC_STAT2                        0x0102
+#define HDMI_IH_AS_STAT0                        0x0103
+#define HDMI_IH_PHY_STAT0                       0x0104
+#define HDMI_IH_I2CM_STAT0                      0x0105
+#define HDMI_IH_CEC_STAT0                       0x0106
+#define HDMI_IH_VP_STAT0                        0x0107
+#define HDMI_IH_I2CMPHY_STAT0                   0x0108
+#define HDMI_IH_AHBDMAAUD_STAT0                 0x0109
+
+#define HDMI_IH_MUTE_FC_STAT0                   0x0180
+#define HDMI_IH_MUTE_FC_STAT1                   0x0181
+#define HDMI_IH_MUTE_FC_STAT2                   0x0182
+#define HDMI_IH_MUTE_AS_STAT0                   0x0183
+#define HDMI_IH_MUTE_PHY_STAT0                  0x0184
+#define HDMI_IH_MUTE_I2CM_STAT0                 0x0185
+#define HDMI_IH_MUTE_CEC_STAT0                  0x0186
+#define HDMI_IH_MUTE_VP_STAT0                   0x0187
+#define HDMI_IH_MUTE_I2CMPHY_STAT0              0x0188
+#define HDMI_IH_MUTE_AHBDMAAUD_STAT0            0x0189
+#define HDMI_IH_MUTE                            0x01FF
+
+/* Video Sample Registers */
+#define HDMI_TX_INVID0                          0x0200
+#define HDMI_TX_INSTUFFING                      0x0201
+#define HDMI_TX_GYDATA0                         0x0202
+#define HDMI_TX_GYDATA1                         0x0203
+#define HDMI_TX_RCRDATA0                        0x0204
+#define HDMI_TX_RCRDATA1                        0x0205
+#define HDMI_TX_BCBDATA0                        0x0206
+#define HDMI_TX_BCBDATA1                        0x0207
+
+/* Video Packetizer Registers */
+#define HDMI_VP_STATUS                          0x0800
+#define HDMI_VP_PR_CD                           0x0801
+#define HDMI_VP_STUFF                           0x0802
+#define HDMI_VP_REMAP                           0x0803
+#define HDMI_VP_CONF                            0x0804
+#define HDMI_VP_STAT                            0x0805
+#define HDMI_VP_INT                             0x0806
+#define HDMI_VP_MASK                            0x0807
+#define HDMI_VP_POL                             0x0808
+
+/* Frame Composer Registers */
+#define HDMI_FC_INVIDCONF                       0x1000
+#define HDMI_FC_INHACTV0                        0x1001
+#define HDMI_FC_INHACTV1                        0x1002
+#define HDMI_FC_INHBLANK0                       0x1003
+#define HDMI_FC_INHBLANK1                       0x1004
+#define HDMI_FC_INVACTV0                        0x1005
+#define HDMI_FC_INVACTV1                        0x1006
+#define HDMI_FC_INVBLANK                        0x1007
+#define HDMI_FC_HSYNCINDELAY0                   0x1008
+#define HDMI_FC_HSYNCINDELAY1                   0x1009
+#define HDMI_FC_HSYNCINWIDTH0                   0x100A
+#define HDMI_FC_HSYNCINWIDTH1                   0x100B
+#define HDMI_FC_VSYNCINDELAY                    0x100C
+#define HDMI_FC_VSYNCINWIDTH                    0x100D
+#define HDMI_FC_INFREQ0                         0x100E
+#define HDMI_FC_INFREQ1                         0x100F
+#define HDMI_FC_INFREQ2                         0x1010
+#define HDMI_FC_CTRLDUR                         0x1011
+#define HDMI_FC_EXCTRLDUR                       0x1012
+#define HDMI_FC_EXCTRLSPAC                      0x1013
+#define HDMI_FC_CH0PREAM                        0x1014
+#define HDMI_FC_CH1PREAM                        0x1015
+#define HDMI_FC_CH2PREAM                        0x1016
+#define HDMI_FC_AVICONF3                        0x1017
+#define HDMI_FC_GCP                             0x1018
+#define HDMI_FC_AVICONF0                        0x1019
+#define HDMI_FC_AVICONF1                        0x101A
+#define HDMI_FC_AVICONF2                        0x101B
+#define HDMI_FC_AVIVID                          0x101C
+#define HDMI_FC_AVIETB0                         0x101D
+#define HDMI_FC_AVIETB1                         0x101E
+#define HDMI_FC_AVISBB0                         0x101F
+#define HDMI_FC_AVISBB1                         0x1020
+#define HDMI_FC_AVIELB0                         0x1021
+#define HDMI_FC_AVIELB1                         0x1022
+#define HDMI_FC_AVISRB0                         0x1023
+#define HDMI_FC_AVISRB1                         0x1024
+#define HDMI_FC_AUDICONF0                       0x1025
+#define HDMI_FC_AUDICONF1                       0x1026
+#define HDMI_FC_AUDICONF2                       0x1027
+#define HDMI_FC_AUDICONF3                       0x1028
+#define HDMI_FC_VSDIEEEID0                      0x1029
+#define HDMI_FC_VSDSIZE                         0x102A
+#define HDMI_FC_VSDIEEEID1                      0x1030
+#define HDMI_FC_VSDIEEEID2                      0x1031
+#define HDMI_FC_VSDPAYLOAD0                     0x1032
+#define HDMI_FC_VSDPAYLOAD1                     0x1033
+#define HDMI_FC_VSDPAYLOAD2                     0x1034
+#define HDMI_FC_VSDPAYLOAD3                     0x1035
+#define HDMI_FC_VSDPAYLOAD4                     0x1036
+#define HDMI_FC_VSDPAYLOAD5                     0x1037
+#define HDMI_FC_VSDPAYLOAD6                     0x1038
+#define HDMI_FC_VSDPAYLOAD7                     0x1039
+#define HDMI_FC_VSDPAYLOAD8                     0x103A
+#define HDMI_FC_VSDPAYLOAD9                     0x103B
+#define HDMI_FC_VSDPAYLOAD10                    0x103C
+#define HDMI_FC_VSDPAYLOAD11                    0x103D
+#define HDMI_FC_VSDPAYLOAD12                    0x103E
+#define HDMI_FC_VSDPAYLOAD13                    0x103F
+#define HDMI_FC_VSDPAYLOAD14                    0x1040
+#define HDMI_FC_VSDPAYLOAD15                    0x1041
+#define HDMI_FC_VSDPAYLOAD16                    0x1042
+#define HDMI_FC_VSDPAYLOAD17                    0x1043
+#define HDMI_FC_VSDPAYLOAD18                    0x1044
+#define HDMI_FC_VSDPAYLOAD19                    0x1045
+#define HDMI_FC_VSDPAYLOAD20                    0x1046
+#define HDMI_FC_VSDPAYLOAD21                    0x1047
+#define HDMI_FC_VSDPAYLOAD22                    0x1048
+#define HDMI_FC_VSDPAYLOAD23                    0x1049
+#define HDMI_FC_SPDVENDORNAME0                  0x104A
+#define HDMI_FC_SPDVENDORNAME1                  0x104B
+#define HDMI_FC_SPDVENDORNAME2                  0x104C
+#define HDMI_FC_SPDVENDORNAME3                  0x104D
+#define HDMI_FC_SPDVENDORNAME4                  0x104E
+#define HDMI_FC_SPDVENDORNAME5                  0x104F
+#define HDMI_FC_SPDVENDORNAME6                  0x1050
+#define HDMI_FC_SPDVENDORNAME7                  0x1051
+#define HDMI_FC_SDPPRODUCTNAME0                 0x1052
+#define HDMI_FC_SDPPRODUCTNAME1                 0x1053
+#define HDMI_FC_SDPPRODUCTNAME2                 0x1054
+#define HDMI_FC_SDPPRODUCTNAME3                 0x1055
+#define HDMI_FC_SDPPRODUCTNAME4                 0x1056
+#define HDMI_FC_SDPPRODUCTNAME5                 0x1057
+#define HDMI_FC_SDPPRODUCTNAME6                 0x1058
+#define HDMI_FC_SDPPRODUCTNAME7                 0x1059
+#define HDMI_FC_SDPPRODUCTNAME8                 0x105A
+#define HDMI_FC_SDPPRODUCTNAME9                 0x105B
+#define HDMI_FC_SDPPRODUCTNAME10                0x105C
+#define HDMI_FC_SDPPRODUCTNAME11                0x105D
+#define HDMI_FC_SDPPRODUCTNAME12                0x105E
+#define HDMI_FC_SDPPRODUCTNAME13                0x105F
+#define HDMI_FC_SDPPRODUCTNAME14                0x1060
+#define HDMI_FC_SPDPRODUCTNAME15                0x1061
+#define HDMI_FC_SPDDEVICEINF                    0x1062
+#define HDMI_FC_AUDSCONF                        0x1063
+#define HDMI_FC_AUDSSTAT                        0x1064
+#define HDMI_FC_AUDSV                           0x1065
+#define HDMI_FC_AUDSU                           0x1066
+#define HDMI_FC_AUDSCHNLS0                       0x1067
+#define HDMI_FC_AUDSCHNLS1                       0x1068
+#define HDMI_FC_AUDSCHNLS2                       0x1069
+#define HDMI_FC_AUDSCHNLS3                       0x106A
+#define HDMI_FC_AUDSCHNLS4                       0x106B
+#define HDMI_FC_AUDSCHNLS5                       0x106C
+#define HDMI_FC_AUDSCHNLS6                       0x106D
+#define HDMI_FC_AUDSCHNLS7                       0x106E
+#define HDMI_FC_AUDSCHNLS8                       0x106F
+#define HDMI_FC_DATACH0FILL                     0x1070
+#define HDMI_FC_DATACH1FILL                     0x1071
+#define HDMI_FC_DATACH2FILL                     0x1072
+#define HDMI_FC_CTRLQHIGH                       0x1073
+#define HDMI_FC_CTRLQLOW                        0x1074
+#define HDMI_FC_ACP0                            0x1075
+#define HDMI_FC_ACP28                           0x1076
+#define HDMI_FC_ACP27                           0x1077
+#define HDMI_FC_ACP26                           0x1078
+#define HDMI_FC_ACP25                           0x1079
+#define HDMI_FC_ACP24                           0x107A
+#define HDMI_FC_ACP23                           0x107B
+#define HDMI_FC_ACP22                           0x107C
+#define HDMI_FC_ACP21                           0x107D
+#define HDMI_FC_ACP20                           0x107E
+#define HDMI_FC_ACP19                           0x107F
+#define HDMI_FC_ACP18                           0x1080
+#define HDMI_FC_ACP17                           0x1081
+#define HDMI_FC_ACP16                           0x1082
+#define HDMI_FC_ACP15                           0x1083
+#define HDMI_FC_ACP14                           0x1084
+#define HDMI_FC_ACP13                           0x1085
+#define HDMI_FC_ACP12                           0x1086
+#define HDMI_FC_ACP11                           0x1087
+#define HDMI_FC_ACP10                           0x1088
+#define HDMI_FC_ACP9                            0x1089
+#define HDMI_FC_ACP8                            0x108A
+#define HDMI_FC_ACP7                            0x108B
+#define HDMI_FC_ACP6                            0x108C
+#define HDMI_FC_ACP5                            0x108D
+#define HDMI_FC_ACP4                            0x108E
+#define HDMI_FC_ACP3                            0x108F
+#define HDMI_FC_ACP2                            0x1090
+#define HDMI_FC_ACP1                            0x1091
+#define HDMI_FC_ISCR1_0                         0x1092
+#define HDMI_FC_ISCR1_16                        0x1093
+#define HDMI_FC_ISCR1_15                        0x1094
+#define HDMI_FC_ISCR1_14                        0x1095
+#define HDMI_FC_ISCR1_13                        0x1096
+#define HDMI_FC_ISCR1_12                        0x1097
+#define HDMI_FC_ISCR1_11                        0x1098
+#define HDMI_FC_ISCR1_10                        0x1099
+#define HDMI_FC_ISCR1_9                         0x109A
+#define HDMI_FC_ISCR1_8                         0x109B
+#define HDMI_FC_ISCR1_7                         0x109C
+#define HDMI_FC_ISCR1_6                         0x109D
+#define HDMI_FC_ISCR1_5                         0x109E
+#define HDMI_FC_ISCR1_4                         0x109F
+#define HDMI_FC_ISCR1_3                         0x10A0
+#define HDMI_FC_ISCR1_2                         0x10A1
+#define HDMI_FC_ISCR1_1                         0x10A2
+#define HDMI_FC_ISCR2_15                        0x10A3
+#define HDMI_FC_ISCR2_14                        0x10A4
+#define HDMI_FC_ISCR2_13                        0x10A5
+#define HDMI_FC_ISCR2_12                        0x10A6
+#define HDMI_FC_ISCR2_11                        0x10A7
+#define HDMI_FC_ISCR2_10                        0x10A8
+#define HDMI_FC_ISCR2_9                         0x10A9
+#define HDMI_FC_ISCR2_8                         0x10AA
+#define HDMI_FC_ISCR2_7                         0x10AB
+#define HDMI_FC_ISCR2_6                         0x10AC
+#define HDMI_FC_ISCR2_5                         0x10AD
+#define HDMI_FC_ISCR2_4                         0x10AE
+#define HDMI_FC_ISCR2_3                         0x10AF
+#define HDMI_FC_ISCR2_2                         0x10B0
+#define HDMI_FC_ISCR2_1                         0x10B1
+#define HDMI_FC_ISCR2_0                         0x10B2
+#define HDMI_FC_DATAUTO0                        0x10B3
+#define HDMI_FC_DATAUTO1                        0x10B4
+#define HDMI_FC_DATAUTO2                        0x10B5
+#define HDMI_FC_DATMAN                          0x10B6
+#define HDMI_FC_DATAUTO3                        0x10B7
+#define HDMI_FC_RDRB0                           0x10B8
+#define HDMI_FC_RDRB1                           0x10B9
+#define HDMI_FC_RDRB2                           0x10BA
+#define HDMI_FC_RDRB3                           0x10BB
+#define HDMI_FC_RDRB4                           0x10BC
+#define HDMI_FC_RDRB5                           0x10BD
+#define HDMI_FC_RDRB6                           0x10BE
+#define HDMI_FC_RDRB7                           0x10BF
+#define HDMI_FC_STAT0                           0x10D0
+#define HDMI_FC_INT0                            0x10D1
+#define HDMI_FC_MASK0                           0x10D2
+#define HDMI_FC_POL0                            0x10D3
+#define HDMI_FC_STAT1                           0x10D4
+#define HDMI_FC_INT1                            0x10D5
+#define HDMI_FC_MASK1                           0x10D6
+#define HDMI_FC_POL1                            0x10D7
+#define HDMI_FC_STAT2                           0x10D8
+#define HDMI_FC_INT2                            0x10D9
+#define HDMI_FC_MASK2                           0x10DA
+#define HDMI_FC_POL2                            0x10DB
+#define HDMI_FC_PRCONF                          0x10E0
+#define HDMI_FC_SCRAMBLER_CTRL                  0x10E1
+#define HDMI_FC_PACKET_TX_EN                    0x10E3
+
+#define HDMI_FC_GMD_STAT                        0x1100
+#define HDMI_FC_GMD_EN                          0x1101
+#define HDMI_FC_GMD_UP                          0x1102
+#define HDMI_FC_GMD_CONF                        0x1103
+#define HDMI_FC_GMD_HB                          0x1104
+#define HDMI_FC_GMD_PB0                         0x1105
+#define HDMI_FC_GMD_PB1                         0x1106
+#define HDMI_FC_GMD_PB2                         0x1107
+#define HDMI_FC_GMD_PB3                         0x1108
+#define HDMI_FC_GMD_PB4                         0x1109
+#define HDMI_FC_GMD_PB5                         0x110A
+#define HDMI_FC_GMD_PB6                         0x110B
+#define HDMI_FC_GMD_PB7                         0x110C
+#define HDMI_FC_GMD_PB8                         0x110D
+#define HDMI_FC_GMD_PB9                         0x110E
+#define HDMI_FC_GMD_PB10                        0x110F
+#define HDMI_FC_GMD_PB11                        0x1110
+#define HDMI_FC_GMD_PB12                        0x1111
+#define HDMI_FC_GMD_PB13                        0x1112
+#define HDMI_FC_GMD_PB14                        0x1113
+#define HDMI_FC_GMD_PB15                        0x1114
+#define HDMI_FC_GMD_PB16                        0x1115
+#define HDMI_FC_GMD_PB17                        0x1116
+#define HDMI_FC_GMD_PB18                        0x1117
+#define HDMI_FC_GMD_PB19                        0x1118
+#define HDMI_FC_GMD_PB20                        0x1119
+#define HDMI_FC_GMD_PB21                        0x111A
+#define HDMI_FC_GMD_PB22                        0x111B
+#define HDMI_FC_GMD_PB23                        0x111C
+#define HDMI_FC_GMD_PB24                        0x111D
+#define HDMI_FC_GMD_PB25                        0x111E
+#define HDMI_FC_GMD_PB26                        0x111F
+#define HDMI_FC_GMD_PB27                        0x1120
+
+#define HDMI_FC_DRM_UP                          0x1167
+#define HDMI_FC_DRM_HB0                         0x1168
+#define HDMI_FC_DRM_HB1                         0x1169
+#define HDMI_FC_DRM_PB0                         0x116A
+#define HDMI_FC_DRM_PB1                         0x116B
+#define HDMI_FC_DRM_PB2                         0x116C
+#define HDMI_FC_DRM_PB3                         0x116D
+#define HDMI_FC_DRM_PB4                         0x116E
+#define HDMI_FC_DRM_PB5                         0x116F
+#define HDMI_FC_DRM_PB6                         0x1170
+#define HDMI_FC_DRM_PB7                         0x1171
+#define HDMI_FC_DRM_PB8                         0x1172
+#define HDMI_FC_DRM_PB9                         0x1173
+#define HDMI_FC_DRM_PB10                        0x1174
+#define HDMI_FC_DRM_PB11                        0x1175
+#define HDMI_FC_DRM_PB12                        0x1176
+#define HDMI_FC_DRM_PB13                        0x1177
+#define HDMI_FC_DRM_PB14                        0x1178
+#define HDMI_FC_DRM_PB15                        0x1179
+#define HDMI_FC_DRM_PB16                        0x117A
+#define HDMI_FC_DRM_PB17                        0x117B
+#define HDMI_FC_DRM_PB18                        0x117C
+#define HDMI_FC_DRM_PB19                        0x117D
+#define HDMI_FC_DRM_PB20                        0x117E
+#define HDMI_FC_DRM_PB21                        0x117F
+#define HDMI_FC_DRM_PB22                        0x1180
+#define HDMI_FC_DRM_PB23                        0x1181
+#define HDMI_FC_DRM_PB24                        0x1182
+#define HDMI_FC_DRM_PB25                        0x1183
+#define HDMI_FC_DRM_PB26                        0x1184
+
+#define HDMI_FC_DBGFORCE                        0x1200
+#define HDMI_FC_DBGAUD0CH0                      0x1201
+#define HDMI_FC_DBGAUD1CH0                      0x1202
+#define HDMI_FC_DBGAUD2CH0                      0x1203
+#define HDMI_FC_DBGAUD0CH1                      0x1204
+#define HDMI_FC_DBGAUD1CH1                      0x1205
+#define HDMI_FC_DBGAUD2CH1                      0x1206
+#define HDMI_FC_DBGAUD0CH2                      0x1207
+#define HDMI_FC_DBGAUD1CH2                      0x1208
+#define HDMI_FC_DBGAUD2CH2                      0x1209
+#define HDMI_FC_DBGAUD0CH3                      0x120A
+#define HDMI_FC_DBGAUD1CH3                      0x120B
+#define HDMI_FC_DBGAUD2CH3                      0x120C
+#define HDMI_FC_DBGAUD0CH4                      0x120D
+#define HDMI_FC_DBGAUD1CH4                      0x120E
+#define HDMI_FC_DBGAUD2CH4                      0x120F
+#define HDMI_FC_DBGAUD0CH5                      0x1210
+#define HDMI_FC_DBGAUD1CH5                      0x1211
+#define HDMI_FC_DBGAUD2CH5                      0x1212
+#define HDMI_FC_DBGAUD0CH6                      0x1213
+#define HDMI_FC_DBGAUD1CH6                      0x1214
+#define HDMI_FC_DBGAUD2CH6                      0x1215
+#define HDMI_FC_DBGAUD0CH7                      0x1216
+#define HDMI_FC_DBGAUD1CH7                      0x1217
+#define HDMI_FC_DBGAUD2CH7                      0x1218
+#define HDMI_FC_DBGTMDS0                        0x1219
+#define HDMI_FC_DBGTMDS1                        0x121A
+#define HDMI_FC_DBGTMDS2                        0x121B
+
+/* HDMI Source PHY Registers */
+#define HDMI_PHY_CONF0                          0x3000
+#define HDMI_PHY_TST0                           0x3001
+#define HDMI_PHY_TST1                           0x3002
+#define HDMI_PHY_TST2                           0x3003
+#define HDMI_PHY_STAT0                          0x3004
+#define HDMI_PHY_INT0                           0x3005
+#define HDMI_PHY_MASK0                          0x3006
+#define HDMI_PHY_POL0                           0x3007
+
+/* HDMI Master PHY Registers */
+#define HDMI_PHY_I2CM_SLAVE_ADDR                0x3020
+#define HDMI_PHY_I2CM_ADDRESS_ADDR              0x3021
+#define HDMI_PHY_I2CM_DATAO_1_ADDR              0x3022
+#define HDMI_PHY_I2CM_DATAO_0_ADDR              0x3023
+#define HDMI_PHY_I2CM_DATAI_1_ADDR              0x3024
+#define HDMI_PHY_I2CM_DATAI_0_ADDR              0x3025
+#define HDMI_PHY_I2CM_OPERATION_ADDR            0x3026
+#define HDMI_PHY_I2CM_INT_ADDR                  0x3027
+#define HDMI_PHY_I2CM_CTLINT_ADDR               0x3028
+#define HDMI_PHY_I2CM_DIV_ADDR                  0x3029
+#define HDMI_PHY_I2CM_SOFTRSTZ_ADDR             0x302a
+#define HDMI_PHY_I2CM_SS_SCL_HCNT_1_ADDR        0x302b
+#define HDMI_PHY_I2CM_SS_SCL_HCNT_0_ADDR        0x302c
+#define HDMI_PHY_I2CM_SS_SCL_LCNT_1_ADDR        0x302d
+#define HDMI_PHY_I2CM_SS_SCL_LCNT_0_ADDR        0x302e
+#define HDMI_PHY_I2CM_FS_SCL_HCNT_1_ADDR        0x302f
+#define HDMI_PHY_I2CM_FS_SCL_HCNT_0_ADDR        0x3030
+#define HDMI_PHY_I2CM_FS_SCL_LCNT_1_ADDR        0x3031
+#define HDMI_PHY_I2CM_FS_SCL_LCNT_0_ADDR        0x3032
+
+/* Audio Sampler Registers */
+#define HDMI_AUD_CONF0                          0x3100
+#define HDMI_AUD_CONF1                          0x3101
+#define HDMI_AUD_INT                            0x3102
+#define HDMI_AUD_CONF2                          0x3103
+#define HDMI_AUD_N1                             0x3200
+#define HDMI_AUD_N2                             0x3201
+#define HDMI_AUD_N3                             0x3202
+#define HDMI_AUD_CTS1                           0x3203
+#define HDMI_AUD_CTS2                           0x3204
+#define HDMI_AUD_CTS3                           0x3205
+#define HDMI_AUD_INPUTCLKFS                     0x3206
+#define HDMI_AUD_SPDIFINT			0x3302
+#define HDMI_AUD_CONF0_HBR                      0x3400
+#define HDMI_AUD_HBR_STATUS                     0x3401
+#define HDMI_AUD_HBR_INT                        0x3402
+#define HDMI_AUD_HBR_POL                        0x3403
+#define HDMI_AUD_HBR_MASK                       0x3404
+
+/*
+ * Generic Parallel Audio Interface Registers
+ * Not used as GPAUD interface is not enabled in hw
+ */
+#define HDMI_GP_CONF0                           0x3500
+#define HDMI_GP_CONF1                           0x3501
+#define HDMI_GP_CONF2                           0x3502
+#define HDMI_GP_STAT                            0x3503
+#define HDMI_GP_INT                             0x3504
+#define HDMI_GP_MASK                            0x3505
+#define HDMI_GP_POL                             0x3506
+
+/* Audio DMA Registers */
+#define HDMI_AHB_DMA_CONF0                      0x3600
+#define HDMI_AHB_DMA_START                      0x3601
+#define HDMI_AHB_DMA_STOP                       0x3602
+#define HDMI_AHB_DMA_THRSLD                     0x3603
+#define HDMI_AHB_DMA_STRADDR0                   0x3604
+#define HDMI_AHB_DMA_STRADDR1                   0x3605
+#define HDMI_AHB_DMA_STRADDR2                   0x3606
+#define HDMI_AHB_DMA_STRADDR3                   0x3607
+#define HDMI_AHB_DMA_STPADDR0                   0x3608
+#define HDMI_AHB_DMA_STPADDR1                   0x3609
+#define HDMI_AHB_DMA_STPADDR2                   0x360a
+#define HDMI_AHB_DMA_STPADDR3                   0x360b
+#define HDMI_AHB_DMA_BSTADDR0                   0x360c
+#define HDMI_AHB_DMA_BSTADDR1                   0x360d
+#define HDMI_AHB_DMA_BSTADDR2                   0x360e
+#define HDMI_AHB_DMA_BSTADDR3                   0x360f
+#define HDMI_AHB_DMA_MBLENGTH0                  0x3610
+#define HDMI_AHB_DMA_MBLENGTH1                  0x3611
+#define HDMI_AHB_DMA_STAT                       0x3612
+#define HDMI_AHB_DMA_INT                        0x3613
+#define HDMI_AHB_DMA_MASK                       0x3614
+#define HDMI_AHB_DMA_POL                        0x3615
+#define HDMI_AHB_DMA_CONF1                      0x3616
+#define HDMI_AHB_DMA_BUFFSTAT                   0x3617
+#define HDMI_AHB_DMA_BUFFINT                    0x3618
+#define HDMI_AHB_DMA_BUFFMASK                   0x3619
+#define HDMI_AHB_DMA_BUFFPOL                    0x361a
+
+/* Main Controller Registers */
+#define HDMI_MC_SFRDIV                          0x4000
+#define HDMI_MC_CLKDIS                          0x4001
+#define HDMI_MC_SWRSTZ                          0x4002
+#define HDMI_MC_OPCTRL                          0x4003
+#define HDMI_MC_FLOWCTRL                        0x4004
+#define HDMI_MC_PHYRSTZ                         0x4005
+#define HDMI_MC_LOCKONCLOCK                     0x4006
+#define HDMI_MC_HEACPHY_RST                     0x4007
+
+/* Color Space  Converter Registers */
+#define HDMI_CSC_CFG                            0x4100
+#define HDMI_CSC_SCALE                          0x4101
+#define HDMI_CSC_COEF_A1_MSB                    0x4102
+#define HDMI_CSC_COEF_A1_LSB                    0x4103
+#define HDMI_CSC_COEF_A2_MSB                    0x4104
+#define HDMI_CSC_COEF_A2_LSB                    0x4105
+#define HDMI_CSC_COEF_A3_MSB                    0x4106
+#define HDMI_CSC_COEF_A3_LSB                    0x4107
+#define HDMI_CSC_COEF_A4_MSB                    0x4108
+#define HDMI_CSC_COEF_A4_LSB                    0x4109
+#define HDMI_CSC_COEF_B1_MSB                    0x410A
+#define HDMI_CSC_COEF_B1_LSB                    0x410B
+#define HDMI_CSC_COEF_B2_MSB                    0x410C
+#define HDMI_CSC_COEF_B2_LSB                    0x410D
+#define HDMI_CSC_COEF_B3_MSB                    0x410E
+#define HDMI_CSC_COEF_B3_LSB                    0x410F
+#define HDMI_CSC_COEF_B4_MSB                    0x4110
+#define HDMI_CSC_COEF_B4_LSB                    0x4111
+#define HDMI_CSC_COEF_C1_MSB                    0x4112
+#define HDMI_CSC_COEF_C1_LSB                    0x4113
+#define HDMI_CSC_COEF_C2_MSB                    0x4114
+#define HDMI_CSC_COEF_C2_LSB                    0x4115
+#define HDMI_CSC_COEF_C3_MSB                    0x4116
+#define HDMI_CSC_COEF_C3_LSB                    0x4117
+#define HDMI_CSC_COEF_C4_MSB                    0x4118
+#define HDMI_CSC_COEF_C4_LSB                    0x4119
+
+/* HDCP Encryption Engine Registers */
+#define HDMI_A_HDCPCFG0                         0x5000
+#define HDMI_A_HDCPCFG1                         0x5001
+#define HDMI_A_HDCPOBS0                         0x5002
+#define HDMI_A_HDCPOBS1                         0x5003
+#define HDMI_A_HDCPOBS2                         0x5004
+#define HDMI_A_HDCPOBS3                         0x5005
+#define HDMI_A_APIINTCLR                        0x5006
+#define HDMI_A_APIINTSTAT                       0x5007
+#define HDMI_A_APIINTMSK                        0x5008
+#define HDMI_A_VIDPOLCFG                        0x5009
+#define HDMI_A_OESSWCFG                         0x500A
+#define HDMI_A_TIMER1SETUP0                     0x500B
+#define HDMI_A_TIMER1SETUP1                     0x500C
+#define HDMI_A_TIMER2SETUP0                     0x500D
+#define HDMI_A_TIMER2SETUP1                     0x500E
+#define HDMI_A_100MSCFG                         0x500F
+#define HDMI_A_2SCFG0                           0x5010
+#define HDMI_A_2SCFG1                           0x5011
+#define HDMI_A_5SCFG0                           0x5012
+#define HDMI_A_5SCFG1                           0x5013
+#define HDMI_A_SRMVERLSB                        0x5014
+#define HDMI_A_SRMVERMSB                        0x5015
+#define HDMI_A_SRMCTRL                          0x5016
+#define HDMI_A_SFRSETUP                         0x5017
+#define HDMI_A_I2CHSETUP                        0x5018
+#define HDMI_A_INTSETUP                         0x5019
+#define HDMI_A_PRESETUP                         0x501A
+#define HDMI_A_SRM_BASE                         0x5020
+
+/* HDCP Registers */
+#define HDMI_HDCPREG_RMCTL                      0x780e
+#define HDMI_HDCPREG_RMSTS                      0x780f
+#define HDMI_HDCPREG_SEED0                      0x7810
+#define HDMI_HDCPREG_SEED1                      0x7811
+#define HDMI_HDCPREG_DPK0                       0x7812
+#define HDMI_HDCPREG_DPK1                       0x7813
+#define HDMI_HDCPREG_DPK2                       0x7814
+#define HDMI_HDCPREG_DPK3                       0x7815
+#define HDMI_HDCPREG_DPK4                       0x7816
+#define HDMI_HDCPREG_DPK5                       0x7817
+#define HDMI_HDCPREG_DPK6                       0x7818
+#define HDMI_HDCP2REG_CTRL                      0x7904
+#define HDMI_HDCP2REG_MASK                      0x790c
+#define HDMI_HDCP2REG_MUTE                      0x7912
+
+/* CEC Engine Registers */
+#define HDMI_CEC_CTRL                           0x7D00
+#define HDMI_CEC_STAT                           0x7D01
+#define HDMI_CEC_MASK                           0x7D02
+#define HDMI_CEC_POLARITY                       0x7D03
+#define HDMI_CEC_INT                            0x7D04
+#define HDMI_CEC_ADDR_L                         0x7D05
+#define HDMI_CEC_ADDR_H                         0x7D06
+#define HDMI_CEC_TX_CNT                         0x7D07
+#define HDMI_CEC_RX_CNT                         0x7D08
+#define HDMI_CEC_TX_DATA0                       0x7D10
+#define HDMI_CEC_TX_DATA1                       0x7D11
+#define HDMI_CEC_TX_DATA2                       0x7D12
+#define HDMI_CEC_TX_DATA3                       0x7D13
+#define HDMI_CEC_TX_DATA4                       0x7D14
+#define HDMI_CEC_TX_DATA5                       0x7D15
+#define HDMI_CEC_TX_DATA6                       0x7D16
+#define HDMI_CEC_TX_DATA7                       0x7D17
+#define HDMI_CEC_TX_DATA8                       0x7D18
+#define HDMI_CEC_TX_DATA9                       0x7D19
+#define HDMI_CEC_TX_DATA10                      0x7D1a
+#define HDMI_CEC_TX_DATA11                      0x7D1b
+#define HDMI_CEC_TX_DATA12                      0x7D1c
+#define HDMI_CEC_TX_DATA13                      0x7D1d
+#define HDMI_CEC_TX_DATA14                      0x7D1e
+#define HDMI_CEC_TX_DATA15                      0x7D1f
+#define HDMI_CEC_RX_DATA0                       0x7D20
+#define HDMI_CEC_RX_DATA1                       0x7D21
+#define HDMI_CEC_RX_DATA2                       0x7D22
+#define HDMI_CEC_RX_DATA3                       0x7D23
+#define HDMI_CEC_RX_DATA4                       0x7D24
+#define HDMI_CEC_RX_DATA5                       0x7D25
+#define HDMI_CEC_RX_DATA6                       0x7D26
+#define HDMI_CEC_RX_DATA7                       0x7D27
+#define HDMI_CEC_RX_DATA8                       0x7D28
+#define HDMI_CEC_RX_DATA9                       0x7D29
+#define HDMI_CEC_RX_DATA10                      0x7D2a
+#define HDMI_CEC_RX_DATA11                      0x7D2b
+#define HDMI_CEC_RX_DATA12                      0x7D2c
+#define HDMI_CEC_RX_DATA13                      0x7D2d
+#define HDMI_CEC_RX_DATA14                      0x7D2e
+#define HDMI_CEC_RX_DATA15                      0x7D2f
+#define HDMI_CEC_LOCK                           0x7D30
+#define HDMI_CEC_WKUPCTRL                       0x7D31
+
+/* I2C Master Registers (E-DDC) */
+#define HDMI_I2CM_SLAVE                         0x7E00
+#define HDMI_I2CM_ADDRESS                       0x7E01
+#define HDMI_I2CM_DATAO                         0x7E02
+#define HDMI_I2CM_DATAI                         0x7E03
+#define HDMI_I2CM_OPERATION                     0x7E04
+#define HDMI_I2CM_INT                           0x7E05
+#define HDMI_I2CM_CTLINT                        0x7E06
+#define HDMI_I2CM_DIV                           0x7E07
+#define HDMI_I2CM_SEGADDR                       0x7E08
+#define HDMI_I2CM_SOFTRSTZ                      0x7E09
+#define HDMI_I2CM_SEGPTR                        0x7E0A
+#define HDMI_I2CM_SS_SCL_HCNT_1_ADDR            0x7E0B
+#define HDMI_I2CM_SS_SCL_HCNT_0_ADDR            0x7E0C
+#define HDMI_I2CM_SS_SCL_LCNT_1_ADDR            0x7E0D
+#define HDMI_I2CM_SS_SCL_LCNT_0_ADDR            0x7E0E
+#define HDMI_I2CM_FS_SCL_HCNT_1_ADDR            0x7E0F
+#define HDMI_I2CM_FS_SCL_HCNT_0_ADDR            0x7E10
+#define HDMI_I2CM_FS_SCL_LCNT_1_ADDR            0x7E11
+#define HDMI_I2CM_FS_SCL_LCNT_0_ADDR            0x7E12
+#define HDMI_I2CM_SDA_HOLD                      0x7E13
+
+enum {
+/* PRODUCT_ID0 field values */
+	HDMI_PRODUCT_ID0_HDMI_TX = 0xa0,
+
+/* PRODUCT_ID1 field values */
+	HDMI_PRODUCT_ID1_HDCP = 0xc0,
+	HDMI_PRODUCT_ID1_HDMI_RX = 0x02,
+	HDMI_PRODUCT_ID1_HDMI_TX = 0x01,
+
+/* CONFIG0_ID field values */
+	HDMI_CONFIG0_I2S = 0x10,
+	HDMI_CONFIG0_CEC = 0x02,
+
+/* CONFIG1_ID field values */
+	HDMI_CONFIG1_AHB = 0x01,
+
+/* CONFIG3_ID field values */
+	HDMI_CONFIG3_AHBAUDDMA = 0x02,
+	HDMI_CONFIG3_GPAUD = 0x01,
+
+/* IH_FC_INT2 field values */
+	HDMI_IH_FC_INT2_OVERFLOW_MASK = 0x03,
+	HDMI_IH_FC_INT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_IH_FC_INT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* IH_FC_STAT2 field values */
+	HDMI_IH_FC_STAT2_OVERFLOW_MASK = 0x03,
+	HDMI_IH_FC_STAT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_IH_FC_STAT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* IH_PHY_STAT0 field values */
+	HDMI_IH_PHY_STAT0_RX_SENSE3 = 0x20,
+	HDMI_IH_PHY_STAT0_RX_SENSE2 = 0x10,
+	HDMI_IH_PHY_STAT0_RX_SENSE1 = 0x8,
+	HDMI_IH_PHY_STAT0_RX_SENSE0 = 0x4,
+	HDMI_IH_PHY_STAT0_TX_PHY_LOCK = 0x2,
+	HDMI_IH_PHY_STAT0_HPD = 0x1,
+
+/* IH_I2CM_STAT0 and IH_MUTE_I2CM_STAT0 field values */
+	HDMI_IH_I2CM_STAT0_DONE = 0x2,
+	HDMI_IH_I2CM_STAT0_ERROR = 0x1,
+
+/* IH_MUTE_I2CMPHY_STAT0 field values */
+	HDMI_IH_MUTE_I2CMPHY_STAT0_I2CMPHYDONE = 0x2,
+	HDMI_IH_MUTE_I2CMPHY_STAT0_I2CMPHYERROR = 0x1,
+
+/* IH_AHBDMAAUD_STAT0 field values */
+	HDMI_IH_AHBDMAAUD_STAT0_ERROR = 0x20,
+	HDMI_IH_AHBDMAAUD_STAT0_LOST = 0x10,
+	HDMI_IH_AHBDMAAUD_STAT0_RETRY = 0x08,
+	HDMI_IH_AHBDMAAUD_STAT0_DONE = 0x04,
+	HDMI_IH_AHBDMAAUD_STAT0_BUFFFULL = 0x02,
+	HDMI_IH_AHBDMAAUD_STAT0_BUFFEMPTY = 0x01,
+
+/* IH_MUTE_FC_STAT2 field values */
+	HDMI_IH_MUTE_FC_STAT2_OVERFLOW_MASK = 0x03,
+	HDMI_IH_MUTE_FC_STAT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_IH_MUTE_FC_STAT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* IH_MUTE_AHBDMAAUD_STAT0 field values */
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_ERROR = 0x20,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_LOST = 0x10,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_RETRY = 0x08,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_DONE = 0x04,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFFULL = 0x02,
+	HDMI_IH_MUTE_AHBDMAAUD_STAT0_BUFFEMPTY = 0x01,
+
+/* IH_MUTE field values */
+	HDMI_IH_MUTE_MUTE_WAKEUP_INTERRUPT = 0x2,
+	HDMI_IH_MUTE_MUTE_ALL_INTERRUPT = 0x1,
+
+/* TX_INVID0 field values */
+	HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_MASK = 0x80,
+	HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_ENABLE = 0x80,
+	HDMI_TX_INVID0_INTERNAL_DE_GENERATOR_DISABLE = 0x00,
+	HDMI_TX_INVID0_VIDEO_MAPPING_MASK = 0x1F,
+	HDMI_TX_INVID0_VIDEO_MAPPING_OFFSET = 0,
+
+/* TX_INSTUFFING field values */
+	HDMI_TX_INSTUFFING_BDBDATA_STUFFING_MASK = 0x4,
+	HDMI_TX_INSTUFFING_BDBDATA_STUFFING_ENABLE = 0x4,
+	HDMI_TX_INSTUFFING_BDBDATA_STUFFING_DISABLE = 0x0,
+	HDMI_TX_INSTUFFING_RCRDATA_STUFFING_MASK = 0x2,
+	HDMI_TX_INSTUFFING_RCRDATA_STUFFING_ENABLE = 0x2,
+	HDMI_TX_INSTUFFING_RCRDATA_STUFFING_DISABLE = 0x0,
+	HDMI_TX_INSTUFFING_GYDATA_STUFFING_MASK = 0x1,
+	HDMI_TX_INSTUFFING_GYDATA_STUFFING_ENABLE = 0x1,
+	HDMI_TX_INSTUFFING_GYDATA_STUFFING_DISABLE = 0x0,
+
+/* VP_PR_CD field values */
+	HDMI_VP_PR_CD_COLOR_DEPTH_MASK = 0xF0,
+	HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET = 4,
+	HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK = 0x0F,
+	HDMI_VP_PR_CD_DESIRED_PR_FACTOR_OFFSET = 0,
+
+/* VP_STUFF field values */
+	HDMI_VP_STUFF_IDEFAULT_PHASE_MASK = 0x20,
+	HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET = 5,
+	HDMI_VP_STUFF_IFIX_PP_TO_LAST_MASK = 0x10,
+	HDMI_VP_STUFF_IFIX_PP_TO_LAST_OFFSET = 4,
+	HDMI_VP_STUFF_ICX_GOTO_P0_ST_MASK = 0x8,
+	HDMI_VP_STUFF_ICX_GOTO_P0_ST_OFFSET = 3,
+	HDMI_VP_STUFF_YCC422_STUFFING_MASK = 0x4,
+	HDMI_VP_STUFF_YCC422_STUFFING_STUFFING_MODE = 0x4,
+	HDMI_VP_STUFF_YCC422_STUFFING_DIRECT_MODE = 0x0,
+	HDMI_VP_STUFF_PP_STUFFING_MASK = 0x2,
+	HDMI_VP_STUFF_PP_STUFFING_STUFFING_MODE = 0x2,
+	HDMI_VP_STUFF_PP_STUFFING_DIRECT_MODE = 0x0,
+	HDMI_VP_STUFF_PR_STUFFING_MASK = 0x1,
+	HDMI_VP_STUFF_PR_STUFFING_STUFFING_MODE = 0x1,
+	HDMI_VP_STUFF_PR_STUFFING_DIRECT_MODE = 0x0,
+
+/* VP_CONF field values */
+	HDMI_VP_CONF_BYPASS_EN_MASK = 0x40,
+	HDMI_VP_CONF_BYPASS_EN_ENABLE = 0x40,
+	HDMI_VP_CONF_BYPASS_EN_DISABLE = 0x00,
+	HDMI_VP_CONF_PP_EN_ENMASK = 0x20,
+	HDMI_VP_CONF_PP_EN_ENABLE = 0x20,
+	HDMI_VP_CONF_PP_EN_DISABLE = 0x00,
+	HDMI_VP_CONF_PR_EN_MASK = 0x10,
+	HDMI_VP_CONF_PR_EN_ENABLE = 0x10,
+	HDMI_VP_CONF_PR_EN_DISABLE = 0x00,
+	HDMI_VP_CONF_YCC422_EN_MASK = 0x8,
+	HDMI_VP_CONF_YCC422_EN_ENABLE = 0x8,
+	HDMI_VP_CONF_YCC422_EN_DISABLE = 0x0,
+	HDMI_VP_CONF_BYPASS_SELECT_MASK = 0x4,
+	HDMI_VP_CONF_BYPASS_SELECT_VID_PACKETIZER = 0x4,
+	HDMI_VP_CONF_BYPASS_SELECT_PIX_REPEATER = 0x0,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_MASK = 0x3,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS = 0x3,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_YCC422 = 0x1,
+	HDMI_VP_CONF_OUTPUT_SELECTOR_PP = 0x0,
+
+/* VP_REMAP field values */
+	HDMI_VP_REMAP_MASK = 0x3,
+	HDMI_VP_REMAP_YCC422_24bit = 0x2,
+	HDMI_VP_REMAP_YCC422_20bit = 0x1,
+	HDMI_VP_REMAP_YCC422_16bit = 0x0,
+
+/* FC_INVIDCONF field values */
+	HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK = 0x80,
+	HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE = 0x80,
+	HDMI_FC_INVIDCONF_HDCP_KEEPOUT_INACTIVE = 0x00,
+	HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_MASK = 0x40,
+	HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_HIGH = 0x40,
+	HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_LOW = 0x00,
+	HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_MASK = 0x20,
+	HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_HIGH = 0x20,
+	HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_LOW = 0x00,
+	HDMI_FC_INVIDCONF_DE_IN_POLARITY_MASK = 0x10,
+	HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_HIGH = 0x10,
+	HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_LOW = 0x00,
+	HDMI_FC_INVIDCONF_DVI_MODEZ_MASK = 0x8,
+	HDMI_FC_INVIDCONF_DVI_MODEZ_HDMI_MODE = 0x8,
+	HDMI_FC_INVIDCONF_DVI_MODEZ_DVI_MODE = 0x0,
+	HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_MASK = 0x2,
+	HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH = 0x2,
+	HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_LOW = 0x0,
+	HDMI_FC_INVIDCONF_IN_I_P_MASK = 0x1,
+	HDMI_FC_INVIDCONF_IN_I_P_INTERLACED = 0x1,
+	HDMI_FC_INVIDCONF_IN_I_P_PROGRESSIVE = 0x0,
+
+/* FC_AUDICONF0 field values */
+	HDMI_FC_AUDICONF0_CC_OFFSET = 4,
+	HDMI_FC_AUDICONF0_CC_MASK = 0x70,
+	HDMI_FC_AUDICONF0_CT_OFFSET = 0,
+	HDMI_FC_AUDICONF0_CT_MASK = 0xF,
+
+/* FC_AUDICONF1 field values */
+	HDMI_FC_AUDICONF1_SS_OFFSET = 3,
+	HDMI_FC_AUDICONF1_SS_MASK = 0x18,
+	HDMI_FC_AUDICONF1_SF_OFFSET = 0,
+	HDMI_FC_AUDICONF1_SF_MASK = 0x7,
+
+/* FC_AUDICONF3 field values */
+	HDMI_FC_AUDICONF3_LFEPBL_OFFSET = 5,
+	HDMI_FC_AUDICONF3_LFEPBL_MASK = 0x60,
+	HDMI_FC_AUDICONF3_DM_INH_OFFSET = 4,
+	HDMI_FC_AUDICONF3_DM_INH_MASK = 0x10,
+	HDMI_FC_AUDICONF3_LSV_OFFSET = 0,
+	HDMI_FC_AUDICONF3_LSV_MASK = 0xF,
+
+/* FC_AUDSCHNLS0 field values */
+	HDMI_FC_AUDSCHNLS0_CGMSA_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS0_CGMSA_MASK = 0x30,
+	HDMI_FC_AUDSCHNLS0_COPYRIGHT_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS0_COPYRIGHT_MASK = 0x01,
+
+/* FC_AUDSCHNLS3-6 field values */
+	HDMI_FC_AUDSCHNLS3_OIEC_CH0_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS3_OIEC_CH0_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS3_OIEC_CH1_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS3_OIEC_CH1_MASK = 0xf0,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH2_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH2_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH3_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS4_OIEC_CH3_MASK = 0xf0,
+
+	HDMI_FC_AUDSCHNLS5_OIEC_CH0_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS5_OIEC_CH0_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS5_OIEC_CH1_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS5_OIEC_CH1_MASK = 0xf0,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH2_OFFSET = 0,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH2_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH3_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS6_OIEC_CH3_MASK = 0xf0,
+
+/* HDMI_FC_AUDSCHNLS7 field values */
+	HDMI_FC_AUDSCHNLS7_ACCURACY_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS7_ACCURACY_MASK = 0x30,
+
+/* HDMI_FC_AUDSCHNLS8 field values */
+	HDMI_FC_AUDSCHNLS8_ORIGSAMPFREQ_MASK = 0xf0,
+	HDMI_FC_AUDSCHNLS8_ORIGSAMPFREQ_OFFSET = 4,
+	HDMI_FC_AUDSCHNLS8_WORDLEGNTH_MASK = 0x0f,
+	HDMI_FC_AUDSCHNLS8_WORDLEGNTH_OFFSET = 0,
+
+/* FC_AUDSCONF field values */
+	HDMI_FC_AUDSCONF_AUD_PACKET_SAMPFIT_MASK = 0xF0,
+	HDMI_FC_AUDSCONF_AUD_PACKET_SAMPFIT_OFFSET = 4,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_MASK = 0x1,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_OFFSET = 0,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT1 = 0x1,
+	HDMI_FC_AUDSCONF_AUD_PACKET_LAYOUT_LAYOUT0 = 0x0,
+
+/* FC_STAT2 field values */
+	HDMI_FC_STAT2_OVERFLOW_MASK = 0x03,
+	HDMI_FC_STAT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_FC_STAT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* FC_INT2 field values */
+	HDMI_FC_INT2_OVERFLOW_MASK = 0x03,
+	HDMI_FC_INT2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_FC_INT2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* FC_MASK2 field values */
+	HDMI_FC_MASK2_OVERFLOW_MASK = 0x03,
+	HDMI_FC_MASK2_LOW_PRIORITY_OVERFLOW = 0x02,
+	HDMI_FC_MASK2_HIGH_PRIORITY_OVERFLOW = 0x01,
+
+/* FC_PRCONF field values */
+	HDMI_FC_PRCONF_INCOMING_PR_FACTOR_MASK = 0xF0,
+	HDMI_FC_PRCONF_INCOMING_PR_FACTOR_OFFSET = 4,
+	HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK = 0x0F,
+	HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_OFFSET = 0,
+
+/* FC_PACKET_TX_EN field values */
+	HDMI_FC_PACKET_TX_EN_DRM_MASK = 0x80,
+	HDMI_FC_PACKET_TX_EN_DRM_ENABLE = 0x80,
+	HDMI_FC_PACKET_TX_EN_DRM_DISABLE = 0x00,
+
+/* FC_AVICONF0-FC_AVICONF3 field values */
+	HDMI_FC_AVICONF0_PIX_FMT_MASK = 0x03,
+	HDMI_FC_AVICONF0_PIX_FMT_RGB = 0x00,
+	HDMI_FC_AVICONF0_PIX_FMT_YCBCR422 = 0x01,
+	HDMI_FC_AVICONF0_PIX_FMT_YCBCR444 = 0x02,
+	HDMI_FC_AVICONF0_ACTIVE_FMT_MASK = 0x40,
+	HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT = 0x40,
+	HDMI_FC_AVICONF0_ACTIVE_FMT_NO_INFO = 0x00,
+	HDMI_FC_AVICONF0_BAR_DATA_MASK = 0x0C,
+	HDMI_FC_AVICONF0_BAR_DATA_NO_DATA = 0x00,
+	HDMI_FC_AVICONF0_BAR_DATA_VERT_BAR = 0x04,
+	HDMI_FC_AVICONF0_BAR_DATA_HORIZ_BAR = 0x08,
+	HDMI_FC_AVICONF0_BAR_DATA_VERT_HORIZ_BAR = 0x0C,
+	HDMI_FC_AVICONF0_SCAN_INFO_MASK = 0x30,
+	HDMI_FC_AVICONF0_SCAN_INFO_OVERSCAN = 0x10,
+	HDMI_FC_AVICONF0_SCAN_INFO_UNDERSCAN = 0x20,
+	HDMI_FC_AVICONF0_SCAN_INFO_NODATA = 0x00,
+
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_MASK = 0x0F,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_USE_CODED = 0x08,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_4_3 = 0x09,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_16_9 = 0x0A,
+	HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_14_9 = 0x0B,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_MASK = 0x30,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_NO_DATA = 0x00,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_4_3 = 0x10,
+	HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_16_9 = 0x20,
+	HDMI_FC_AVICONF1_COLORIMETRY_MASK = 0xC0,
+	HDMI_FC_AVICONF1_COLORIMETRY_NO_DATA = 0x00,
+	HDMI_FC_AVICONF1_COLORIMETRY_SMPTE = 0x40,
+	HDMI_FC_AVICONF1_COLORIMETRY_ITUR = 0x80,
+	HDMI_FC_AVICONF1_COLORIMETRY_EXTENDED_INFO = 0xC0,
+
+	HDMI_FC_AVICONF2_SCALING_MASK = 0x03,
+	HDMI_FC_AVICONF2_SCALING_NONE = 0x00,
+	HDMI_FC_AVICONF2_SCALING_HORIZ = 0x01,
+	HDMI_FC_AVICONF2_SCALING_VERT = 0x02,
+	HDMI_FC_AVICONF2_SCALING_HORIZ_VERT = 0x03,
+	HDMI_FC_AVICONF2_RGB_QUANT_MASK = 0x0C,
+	HDMI_FC_AVICONF2_RGB_QUANT_DEFAULT = 0x00,
+	HDMI_FC_AVICONF2_RGB_QUANT_LIMITED_RANGE = 0x04,
+	HDMI_FC_AVICONF2_RGB_QUANT_FULL_RANGE = 0x08,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_MASK = 0x70,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC601 = 0x00,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC709 = 0x10,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_SYCC601 = 0x20,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_ADOBE_YCC601 = 0x30,
+	HDMI_FC_AVICONF2_EXT_COLORIMETRY_ADOBE_RGB = 0x40,
+	HDMI_FC_AVICONF2_IT_CONTENT_MASK = 0x80,
+	HDMI_FC_AVICONF2_IT_CONTENT_NO_DATA = 0x00,
+	HDMI_FC_AVICONF2_IT_CONTENT_VALID = 0x80,
+
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_MASK = 0x03,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_GRAPHICS = 0x00,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_PHOTO = 0x01,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_CINEMA = 0x02,
+	HDMI_FC_AVICONF3_IT_CONTENT_TYPE_GAME = 0x03,
+	HDMI_FC_AVICONF3_QUANT_RANGE_MASK = 0x0C,
+	HDMI_FC_AVICONF3_QUANT_RANGE_LIMITED = 0x00,
+	HDMI_FC_AVICONF3_QUANT_RANGE_FULL = 0x04,
+
+/* HDMI_FC_GCP */
+	HDMI_FC_GCP_SET_AVMUTE = 0x2,
+	HDMI_FC_GCP_CLEAR_AVMUTE = 0x1,
+
+/* FC_DBGFORCE field values */
+	HDMI_FC_DBGFORCE_FORCEAUDIO = 0x10,
+	HDMI_FC_DBGFORCE_FORCEVIDEO = 0x1,
+
+/* FC_DATAUTO0 field values */
+	HDMI_FC_DATAUTO0_VSD_MASK = 0x08,
+	HDMI_FC_DATAUTO0_VSD_OFFSET = 3,
+
+/* FC_DATAUTO3 field values */
+	HDMI_FC_DATAUTO3_GCP_AUTO = 0x04,
+
+/* PHY_CONF0 field values */
+	HDMI_PHY_CONF0_PDZ_MASK = 0x80,
+	HDMI_PHY_CONF0_PDZ_OFFSET = 7,
+	HDMI_PHY_CONF0_ENTMDS_MASK = 0x40,
+	HDMI_PHY_CONF0_ENTMDS_OFFSET = 6,
+	HDMI_PHY_CONF0_SVSRET_MASK = 0x20,
+	HDMI_PHY_CONF0_SVSRET_OFFSET = 5,
+	HDMI_PHY_CONF0_GEN2_PDDQ_MASK = 0x10,
+	HDMI_PHY_CONF0_GEN2_PDDQ_OFFSET = 4,
+	HDMI_PHY_CONF0_GEN2_TXPWRON_MASK = 0x8,
+	HDMI_PHY_CONF0_GEN2_TXPWRON_OFFSET = 3,
+	HDMI_PHY_CONF0_GEN2_ENHPDRXSENSE_MASK = 0x4,
+	HDMI_PHY_CONF0_GEN2_ENHPDRXSENSE_OFFSET = 2,
+	HDMI_PHY_CONF0_SELDATAENPOL_MASK = 0x2,
+	HDMI_PHY_CONF0_SELDATAENPOL_OFFSET = 1,
+	HDMI_PHY_CONF0_SELDIPIF_MASK = 0x1,
+	HDMI_PHY_CONF0_SELDIPIF_OFFSET = 0,
+
+/* PHY_TST0 field values */
+	HDMI_PHY_TST0_TSTCLR_MASK = 0x20,
+	HDMI_PHY_TST0_TSTCLR_OFFSET = 5,
+	HDMI_PHY_TST0_TSTEN_MASK = 0x10,
+	HDMI_PHY_TST0_TSTEN_OFFSET = 4,
+	HDMI_PHY_TST0_TSTCLK_MASK = 0x1,
+	HDMI_PHY_TST0_TSTCLK_OFFSET = 0,
+
+/* PHY_STAT0 field values */
+	HDMI_PHY_RX_SENSE3 = 0x80,
+	HDMI_PHY_RX_SENSE2 = 0x40,
+	HDMI_PHY_RX_SENSE1 = 0x20,
+	HDMI_PHY_RX_SENSE0 = 0x10,
+	HDMI_PHY_HPD = 0x02,
+	HDMI_PHY_TX_PHY_LOCK = 0x01,
+
+/* PHY_I2CM_SLAVE_ADDR field values */
+	HDMI_PHY_I2CM_SLAVE_ADDR_PHY_GEN2 = 0x69,
+	HDMI_PHY_I2CM_SLAVE_ADDR_HEAC_PHY = 0x49,
+
+/* PHY_I2CM_OPERATION_ADDR field values */
+	HDMI_PHY_I2CM_OPERATION_ADDR_WRITE = 0x10,
+	HDMI_PHY_I2CM_OPERATION_ADDR_READ = 0x1,
+
+/* HDMI_PHY_I2CM_INT_ADDR */
+	HDMI_PHY_I2CM_INT_ADDR_DONE_POL = 0x08,
+	HDMI_PHY_I2CM_INT_ADDR_DONE_MASK = 0x04,
+
+/* HDMI_PHY_I2CM_CTLINT_ADDR */
+	HDMI_PHY_I2CM_CTLINT_ADDR_NAC_POL = 0x80,
+	HDMI_PHY_I2CM_CTLINT_ADDR_NAC_MASK = 0x40,
+	HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL = 0x08,
+	HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_MASK = 0x04,
+
+/* AUD_CONF0 field values */
+	HDMI_AUD_CONF0_SW_RESET = 0x80,
+	HDMI_AUD_CONF0_I2S_SELECT = 0x20,
+	HDMI_AUD_CONF0_I2S_EN3 = 0x08,
+	HDMI_AUD_CONF0_I2S_EN2 = 0x04,
+	HDMI_AUD_CONF0_I2S_EN1 = 0x02,
+	HDMI_AUD_CONF0_I2S_EN0 = 0x01,
+
+/* AUD_CONF1 field values */
+	HDMI_AUD_CONF1_MODE_I2S = 0x00,
+	HDMI_AUD_CONF1_MODE_RIGHT_J = 0x20,
+	HDMI_AUD_CONF1_MODE_LEFT_J = 0x40,
+	HDMI_AUD_CONF1_MODE_BURST_1 = 0x60,
+	HDMI_AUD_CONF1_MODE_BURST_2 = 0x80,
+	HDMI_AUD_CONF1_WIDTH_16 = 0x10,
+	HDMI_AUD_CONF1_WIDTH_24 = 0x18,
+
+/* AUD_CTS3 field values */
+	HDMI_AUD_CTS3_N_SHIFT_OFFSET = 5,
+	HDMI_AUD_CTS3_N_SHIFT_MASK = 0xe0,
+	HDMI_AUD_CTS3_N_SHIFT_1 = 0,
+	HDMI_AUD_CTS3_N_SHIFT_16 = 0x20,
+	HDMI_AUD_CTS3_N_SHIFT_32 = 0x40,
+	HDMI_AUD_CTS3_N_SHIFT_64 = 0x60,
+	HDMI_AUD_CTS3_N_SHIFT_128 = 0x80,
+	HDMI_AUD_CTS3_N_SHIFT_256 = 0xa0,
+	/* note that the CTS3 MANUAL bit has been removed
+	   from our part. Can't set it, will read as 0. */
+	HDMI_AUD_CTS3_CTS_MANUAL = 0x10,
+	HDMI_AUD_CTS3_AUDCTS19_16_MASK = 0x0f,
+
+/* HDMI_AUD_INPUTCLKFS field values */
+	HDMI_AUD_INPUTCLKFS_128FS = 0,
+	HDMI_AUD_INPUTCLKFS_256FS = 1,
+	HDMI_AUD_INPUTCLKFS_512FS = 2,
+	HDMI_AUD_INPUTCLKFS_64FS = 4,
+
+/* AHB_DMA_CONF0 field values */
+	HDMI_AHB_DMA_CONF0_SW_FIFO_RST_OFFSET = 7,
+	HDMI_AHB_DMA_CONF0_SW_FIFO_RST_MASK = 0x80,
+	HDMI_AHB_DMA_CONF0_HBR = 0x10,
+	HDMI_AHB_DMA_CONF0_EN_HLOCK_OFFSET = 3,
+	HDMI_AHB_DMA_CONF0_EN_HLOCK_MASK = 0x08,
+	HDMI_AHB_DMA_CONF0_INCR_TYPE_OFFSET = 1,
+	HDMI_AHB_DMA_CONF0_INCR_TYPE_MASK = 0x06,
+	HDMI_AHB_DMA_CONF0_INCR4 = 0x0,
+	HDMI_AHB_DMA_CONF0_INCR8 = 0x2,
+	HDMI_AHB_DMA_CONF0_INCR16 = 0x4,
+	HDMI_AHB_DMA_CONF0_BURST_MODE = 0x1,
+
+/* HDMI_AHB_DMA_START field values */
+	HDMI_AHB_DMA_START_START_OFFSET = 0,
+	HDMI_AHB_DMA_START_START_MASK = 0x01,
+
+/* HDMI_AHB_DMA_STOP field values */
+	HDMI_AHB_DMA_STOP_STOP_OFFSET = 0,
+	HDMI_AHB_DMA_STOP_STOP_MASK = 0x01,
+
+/* AHB_DMA_STAT, AHB_DMA_INT, AHB_DMA_MASK, AHB_DMA_POL field values */
+	HDMI_AHB_DMA_DONE = 0x80,
+	HDMI_AHB_DMA_RETRY_SPLIT = 0x40,
+	HDMI_AHB_DMA_LOSTOWNERSHIP = 0x20,
+	HDMI_AHB_DMA_ERROR = 0x10,
+	HDMI_AHB_DMA_FIFO_THREMPTY = 0x04,
+	HDMI_AHB_DMA_FIFO_FULL = 0x02,
+	HDMI_AHB_DMA_FIFO_EMPTY = 0x01,
+
+/* AHB_DMA_BUFFSTAT, AHB_DMA_BUFFINT,AHB_DMA_BUFFMASK,AHB_DMA_BUFFPOL values */
+	HDMI_AHB_DMA_BUFFSTAT_FULL = 0x02,
+	HDMI_AHB_DMA_BUFFSTAT_EMPTY = 0x01,
+
+/* MC_CLKDIS field values */
+	HDMI_MC_CLKDIS_HDCPCLK_DISABLE = 0x40,
+	HDMI_MC_CLKDIS_CECCLK_DISABLE = 0x20,
+	HDMI_MC_CLKDIS_CSCCLK_DISABLE = 0x10,
+	HDMI_MC_CLKDIS_AUDCLK_DISABLE = 0x8,
+	HDMI_MC_CLKDIS_PREPCLK_DISABLE = 0x4,
+	HDMI_MC_CLKDIS_TMDSCLK_DISABLE = 0x2,
+	HDMI_MC_CLKDIS_PIXELCLK_DISABLE = 0x1,
+	HDMI_MC_CLKDIS_PIXELCLK_MASK = 0x1,
+	HDMI_MC_CLKDIS_PIXELCLK_ENABLE = 0,
+
+/* MC_SWRSTZ field values */
+	HDMI_MC_SWRSTZ_I2SSWRST_REQ = 0x08,
+	HDMI_MC_SWRSTZ_TMDSSWRST_REQ = 0x02,
+
+/* MC_FLOWCTRL field values */
+	HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_MASK = 0x1,
+	HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_IN_PATH = 0x1,
+	HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_BYPASS = 0x0,
+
+/* MC_PHYRSTZ field values */
+	HDMI_MC_PHYRSTZ_PHYRSTZ = 0x01,
+
+/* MC_HEACPHY_RST field values */
+	HDMI_MC_HEACPHY_RST_ASSERT = 0x1,
+	HDMI_MC_HEACPHY_RST_DEASSERT = 0x0,
+
+/* CSC_CFG field values */
+	HDMI_CSC_CFG_INTMODE_MASK = 0x30,
+	HDMI_CSC_CFG_INTMODE_OFFSET = 4,
+	HDMI_CSC_CFG_INTMODE_DISABLE = 0x00,
+	HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA1 = 0x10,
+	HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA2 = 0x20,
+	HDMI_CSC_CFG_DECMODE_MASK = 0x3,
+	HDMI_CSC_CFG_DECMODE_OFFSET = 0,
+	HDMI_CSC_CFG_DECMODE_DISABLE = 0x0,
+	HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA1 = 0x1,
+	HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA2 = 0x2,
+	HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA3 = 0x3,
+
+/* CSC_SCALE field values */
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_MASK = 0xF0,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_24BPP = 0x00,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_30BPP = 0x50,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_36BPP = 0x60,
+	HDMI_CSC_SCALE_CSC_COLORDE_PTH_48BPP = 0x70,
+	HDMI_CSC_SCALE_CSCSCALE_MASK = 0x03,
+
+/* A_HDCPCFG0 field values */
+	HDMI_A_HDCPCFG0_ELVENA_MASK = 0x80,
+	HDMI_A_HDCPCFG0_ELVENA_ENABLE = 0x80,
+	HDMI_A_HDCPCFG0_ELVENA_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_I2CFASTMODE_MASK = 0x40,
+	HDMI_A_HDCPCFG0_I2CFASTMODE_ENABLE = 0x40,
+	HDMI_A_HDCPCFG0_I2CFASTMODE_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_BYPENCRYPTION_MASK = 0x20,
+	HDMI_A_HDCPCFG0_BYPENCRYPTION_ENABLE = 0x20,
+	HDMI_A_HDCPCFG0_BYPENCRYPTION_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_SYNCRICHECK_MASK = 0x10,
+	HDMI_A_HDCPCFG0_SYNCRICHECK_ENABLE = 0x10,
+	HDMI_A_HDCPCFG0_SYNCRICHECK_DISABLE = 0x00,
+	HDMI_A_HDCPCFG0_AVMUTE_MASK = 0x8,
+	HDMI_A_HDCPCFG0_AVMUTE_ENABLE = 0x8,
+	HDMI_A_HDCPCFG0_AVMUTE_DISABLE = 0x0,
+	HDMI_A_HDCPCFG0_RXDETECT_MASK = 0x4,
+	HDMI_A_HDCPCFG0_RXDETECT_ENABLE = 0x4,
+	HDMI_A_HDCPCFG0_RXDETECT_DISABLE = 0x0,
+	HDMI_A_HDCPCFG0_EN11FEATURE_MASK = 0x2,
+	HDMI_A_HDCPCFG0_EN11FEATURE_ENABLE = 0x2,
+	HDMI_A_HDCPCFG0_EN11FEATURE_DISABLE = 0x0,
+	HDMI_A_HDCPCFG0_HDMIDVI_MASK = 0x1,
+	HDMI_A_HDCPCFG0_HDMIDVI_HDMI = 0x1,
+	HDMI_A_HDCPCFG0_HDMIDVI_DVI = 0x0,
+
+/* A_HDCPCFG1 field values */
+	HDMI_A_HDCPCFG1_DISSHA1CHECK_MASK = 0x8,
+	HDMI_A_HDCPCFG1_DISSHA1CHECK_DISABLE = 0x8,
+	HDMI_A_HDCPCFG1_DISSHA1CHECK_ENABLE = 0x0,
+	HDMI_A_HDCPCFG1_PH2UPSHFTENC_MASK = 0x4,
+	HDMI_A_HDCPCFG1_PH2UPSHFTENC_ENABLE = 0x4,
+	HDMI_A_HDCPCFG1_PH2UPSHFTENC_DISABLE = 0x0,
+	HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK = 0x2,
+	HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE = 0x2,
+	HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_ENABLE = 0x0,
+	HDMI_A_HDCPCFG1_SWRESET_MASK = 0x1,
+	HDMI_A_HDCPCFG1_SWRESET_ASSERT = 0x0,
+
+/* A_VIDPOLCFG field values */
+	HDMI_A_VIDPOLCFG_UNENCRYPTCONF_MASK = 0x60,
+	HDMI_A_VIDPOLCFG_UNENCRYPTCONF_OFFSET = 5,
+	HDMI_A_VIDPOLCFG_DATAENPOL_MASK = 0x10,
+	HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH = 0x10,
+	HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW = 0x0,
+	HDMI_A_VIDPOLCFG_VSYNCPOL_MASK = 0x8,
+	HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_HIGH = 0x8,
+	HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_LOW = 0x0,
+	HDMI_A_VIDPOLCFG_HSYNCPOL_MASK = 0x2,
+	HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_HIGH = 0x2,
+	HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW = 0x0,
+
+/* I2CM_OPERATION field values */
+	HDMI_I2CM_OPERATION_BUSCLEAR = 0x20,
+	HDMI_I2CM_OPERATION_WRITE = 0x10,
+	HDMI_I2CM_OPERATION_READ_EXT = 0x2,
+	HDMI_I2CM_OPERATION_READ = 0x1,
+
+/* I2CM_INT field values */
+	HDMI_I2CM_INT_DONE_POL = 0x8,
+	HDMI_I2CM_INT_DONE_MASK = 0x4,
+
+/* I2CM_CTLINT field values */
+	HDMI_I2CM_CTLINT_NAC_POL = 0x80,
+	HDMI_I2CM_CTLINT_NAC_MASK = 0x40,
+	HDMI_I2CM_CTLINT_ARB_POL = 0x8,
+	HDMI_I2CM_CTLINT_ARB_MASK = 0x4,
+};
+
+/*
+ * HDMI 3D TX PHY registers
+ */
+#define HDMI_3D_TX_PHY_PWRCTRL			0x00
+#define HDMI_3D_TX_PHY_SERDIVCTRL		0x01
+#define HDMI_3D_TX_PHY_SERCKCTRL		0x02
+#define HDMI_3D_TX_PHY_SERCKKILLCTRL		0x03
+#define HDMI_3D_TX_PHY_TXRESCTRL		0x04
+#define HDMI_3D_TX_PHY_CKCALCTRL		0x05
+#define HDMI_3D_TX_PHY_CPCE_CTRL		0x06
+#define HDMI_3D_TX_PHY_TXCLKMEASCTRL		0x07
+#define HDMI_3D_TX_PHY_TXMEASCTRL		0x08
+#define HDMI_3D_TX_PHY_CKSYMTXCTRL		0x09
+#define HDMI_3D_TX_PHY_CMPSEQCTRL		0x0a
+#define HDMI_3D_TX_PHY_CMPPWRCTRL		0x0b
+#define HDMI_3D_TX_PHY_CMPMODECTRL		0x0c
+#define HDMI_3D_TX_PHY_MEASCTRL			0x0d
+#define HDMI_3D_TX_PHY_VLEVCTRL			0x0e
+#define HDMI_3D_TX_PHY_D2ACTRL			0x0f
+#define HDMI_3D_TX_PHY_CURRCTRL			0x10
+#define HDMI_3D_TX_PHY_DRVANACTRL		0x11
+#define HDMI_3D_TX_PHY_PLLMEASCTRL		0x12
+#define HDMI_3D_TX_PHY_PLLPHBYCTRL		0x13
+#define HDMI_3D_TX_PHY_GRP_CTRL			0x14
+#define HDMI_3D_TX_PHY_GMPCTRL			0x15
+#define HDMI_3D_TX_PHY_MPLLMEASCTRL		0x16
+#define HDMI_3D_TX_PHY_MSM_CTRL			0x17
+#define HDMI_3D_TX_PHY_SCRPB_STATUS		0x18
+#define HDMI_3D_TX_PHY_TXTERM			0x19
+#define HDMI_3D_TX_PHY_PTRPT_ENBL		0x1a
+#define HDMI_3D_TX_PHY_PATTERNGEN		0x1b
+#define HDMI_3D_TX_PHY_SDCAP_MODE		0x1c
+#define HDMI_3D_TX_PHY_SCOPEMODE		0x1d
+#define HDMI_3D_TX_PHY_DIGTXMODE		0x1e
+#define HDMI_3D_TX_PHY_STR_STATUS		0x1f
+#define HDMI_3D_TX_PHY_SCOPECNT0		0x20
+#define HDMI_3D_TX_PHY_SCOPECNT1		0x21
+#define HDMI_3D_TX_PHY_SCOPECNT2		0x22
+#define HDMI_3D_TX_PHY_SCOPECNTCLK		0x23
+#define HDMI_3D_TX_PHY_SCOPESAMPLE		0x24
+#define HDMI_3D_TX_PHY_SCOPECNTMSB01		0x25
+#define HDMI_3D_TX_PHY_SCOPECNTMSB2CK		0x26
+
+/* HDMI_3D_TX_PHY_CKCALCTRL values */
+#define HDMI_3D_TX_PHY_CKCALCTRL_OVERRIDE		BIT(15)
+
+/* HDMI_3D_TX_PHY_MSM_CTRL values */
+#define HDMI_3D_TX_PHY_MSM_CTRL_MPLL_PH_SEL_CK		BIT(13)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_CLK_REF_MPLL	(0 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_OFF		(1 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_PCLK		(2 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_CKO_SEL_FB_CLK		(3 << 1)
+#define HDMI_3D_TX_PHY_MSM_CTRL_SCOPE_CK_SEL		BIT(0)
+
+/* HDMI_3D_TX_PHY_PTRPT_ENBL values */
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_OVERRIDE		BIT(15)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_PG_SKIP_BIT2		BIT(8)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_PG_SKIP_BIT1		BIT(7)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_PG_SKIP_BIT0		BIT(6)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_CK_REF_ENB		BIT(5)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_RCAL_ENB		BIT(4)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_TX_CLK_ALIGN_ENB	BIT(3)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_TX_READY		BIT(2)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_CKO_WORD_ENB		BIT(1)
+#define HDMI_3D_TX_PHY_PTRPT_ENBL_REFCLK_ENB		BIT(0)
+
+struct dw_hdmi;
+
+struct dw_hdmi_property_ops {
+	void (*attatch_properties)(struct drm_connector *connector,
+				   unsigned int color, int version, void *data);
+	void (*destroy_properties)(struct drm_connector *connector, void *data);
+	int (*set_property)(struct drm_connector *connector,
+			    struct drm_connector_state *state,
+			    struct drm_property *property, u64 val, void *data);
+	int (*get_property)(struct drm_connector *connector,
+			    const struct drm_connector_state *state,
+			    struct drm_property *property, u64 *val,
+			    void *data);
+};
+
+enum supported_eotf_type {
+	TRADITIONAL_GAMMA_SDR = 0,
+	TRADITIONAL_GAMMA_HDR,
+	SMPTE_ST2084,
+	HLG,
+	FUTURE_EOTF
+};
+
+void dw_hdmi_suspend(struct dw_hdmi *hdmi);
+void dw_hdmi_enable_video(struct dw_hdmi *hdmi);
+void dw_hdmi_disable_video(struct dw_hdmi *hdmi);
+#endif /* __DW_HDMI_H__ */
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_audio.h b/drivers/gpu/drm/eswin/dw_hdmi_audio.h
new file mode 100644
index 000000000000..f72d27208ebe
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_audio.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef DW_HDMI_AUDIO_H
+#define DW_HDMI_AUDIO_H
+
+struct dw_hdmi;
+
+struct dw_hdmi_audio_data {
+	phys_addr_t phys;
+	void __iomem *base;
+	int irq;
+	struct dw_hdmi *hdmi;
+	u8 *(*get_eld)(struct dw_hdmi *hdmi);
+};
+
+struct dw_hdmi_i2s_audio_data {
+	struct dw_hdmi *hdmi;
+
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+	u8 *(*get_eld)(struct dw_hdmi *hdmi);
+};
+
+#endif
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_cec.c b/drivers/gpu/drm/eswin/dw_hdmi_cec.c
new file mode 100644
index 000000000000..42c57cf1dbc0
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_cec.c
@@ -0,0 +1,334 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Designware HDMI CEC driver
+ *
+ * Copyright (C) 2015-2017 Russell King.
+ */
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include <drm/drm_edid.h>
+
+#include <media/cec.h>
+#include <media/cec-notifier.h>
+
+#include "dw_hdmi_cec.h"
+
+#define CEC_IO_ADDR 0x516000fc
+
+enum {
+	HDMI_IH_CEC_STAT0 = 0x0106,
+	HDMI_IH_MUTE_CEC_STAT0 = 0x0186,
+
+	HDMI_CEC_CTRL = 0x7d00,
+	CEC_CTRL_START = BIT(0),
+	CEC_CTRL_FRAME_TYP = 3 << 1,
+	CEC_CTRL_RETRY = 0 << 1,
+	CEC_CTRL_NORMAL = 1 << 1,
+	CEC_CTRL_IMMED = 2 << 1,
+
+	HDMI_CEC_STAT = 0x7d01,
+	CEC_STAT_DONE = BIT(0),
+	CEC_STAT_EOM = BIT(1),
+	CEC_STAT_NACK = BIT(2),
+	CEC_STAT_ARBLOST = BIT(3),
+	CEC_STAT_ERROR_INIT = BIT(4),
+	CEC_STAT_ERROR_FOLL = BIT(5),
+	CEC_STAT_WAKEUP = BIT(6),
+
+	HDMI_CEC_MASK = 0x7d02,
+	HDMI_CEC_POLARITY = 0x7d03,
+	HDMI_CEC_INT = 0x7d04,
+	HDMI_CEC_ADDR_L = 0x7d05,
+	HDMI_CEC_ADDR_H = 0x7d06,
+	HDMI_CEC_TX_CNT = 0x7d07,
+	HDMI_CEC_RX_CNT = 0x7d08,
+	HDMI_CEC_TX_DATA0 = 0x7d10,
+	HDMI_CEC_RX_DATA0 = 0x7d20,
+	HDMI_CEC_LOCK = 0x7d30,
+	HDMI_CEC_WKUPCTRL = 0x7d31,
+};
+
+struct dw_hdmi_cec {
+	struct dw_hdmi *hdmi;
+	const struct dw_hdmi_cec_ops *ops;
+	u32 addresses;
+	struct cec_adapter *adap;
+	struct cec_msg rx_msg;
+	unsigned int tx_status;
+	bool tx_done;
+	bool rx_done;
+	struct cec_notifier *notify;
+	int irq;
+};
+
+static void dw_hdmi_write(struct dw_hdmi_cec *cec, u8 val, int offset)
+{
+	cec->ops->write(cec->hdmi, val, offset);
+}
+
+static u8 dw_hdmi_read(struct dw_hdmi_cec *cec, int offset)
+{
+	return cec->ops->read(cec->hdmi, offset);
+}
+
+static int dw_hdmi_cec_log_addr(struct cec_adapter *adap, u8 logical_addr)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (logical_addr == CEC_LOG_ADDR_INVALID)
+		cec->addresses = 0;
+	else
+		cec->addresses |= BIT(logical_addr) | BIT(15);
+
+	dw_hdmi_write(cec, cec->addresses & 255, HDMI_CEC_ADDR_L);
+	dw_hdmi_write(cec, cec->addresses >> 8, HDMI_CEC_ADDR_H);
+
+	return 0;
+}
+
+static int dw_hdmi_cec_transmit(struct cec_adapter *adap, u8 attempts,
+				u32 signal_free_time, struct cec_msg *msg)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+	unsigned int i, ctrl;
+
+	switch (signal_free_time) {
+	case CEC_SIGNAL_FREE_TIME_RETRY:
+		ctrl = CEC_CTRL_RETRY;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEW_INITIATOR:
+	default:
+		ctrl = CEC_CTRL_NORMAL;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEXT_XFER:
+		ctrl = CEC_CTRL_IMMED;
+		break;
+	}
+
+	for (i = 0; i < msg->len; i++)
+		dw_hdmi_write(cec, msg->msg[i], HDMI_CEC_TX_DATA0 + i);
+
+	dw_hdmi_write(cec, msg->len, HDMI_CEC_TX_CNT);
+	dw_hdmi_write(cec, ctrl | CEC_CTRL_START, HDMI_CEC_CTRL);
+
+	return 0;
+}
+
+static irqreturn_t dw_hdmi_cec_hardirq(int irq, void *data)
+{
+	struct cec_adapter *adap = data;
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+	unsigned int stat = dw_hdmi_read(cec, HDMI_IH_CEC_STAT0);
+	irqreturn_t ret = IRQ_HANDLED;
+
+	if (stat == 0)
+		return IRQ_NONE;
+
+	dw_hdmi_write(cec, stat, HDMI_IH_CEC_STAT0);
+
+	if (stat & CEC_STAT_ERROR_INIT) {
+		cec->tx_status = CEC_TX_STATUS_ERROR;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	} else if (stat & CEC_STAT_DONE) {
+		cec->tx_status = CEC_TX_STATUS_OK;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	} else if (stat & CEC_STAT_NACK) {
+		cec->tx_status = CEC_TX_STATUS_NACK;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	if (stat & CEC_STAT_EOM) {
+		unsigned int len, i;
+
+		len = dw_hdmi_read(cec, HDMI_CEC_RX_CNT);
+		if (len > sizeof(cec->rx_msg.msg))
+			len = sizeof(cec->rx_msg.msg);
+
+		for (i = 0; i < len; i++)
+			cec->rx_msg.msg[i] =
+				dw_hdmi_read(cec, HDMI_CEC_RX_DATA0 + i);
+
+		dw_hdmi_write(cec, 0, HDMI_CEC_LOCK);
+
+		cec->rx_msg.len = len;
+		smp_wmb();
+		cec->rx_done = true;
+
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	return ret;
+}
+
+static irqreturn_t dw_hdmi_cec_thread(int irq, void *data)
+{
+	struct cec_adapter *adap = data;
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (cec->tx_done) {
+		cec->tx_done = false;
+		cec_transmit_attempt_done(adap, cec->tx_status);
+	}
+	if (cec->rx_done) {
+		cec->rx_done = false;
+		smp_rmb();
+		cec_received_msg(adap, &cec->rx_msg);
+	}
+	return IRQ_HANDLED;
+}
+
+static int dw_hdmi_cec_enable(struct cec_adapter *adap, bool enable)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (!enable) {
+		dw_hdmi_write(cec, ~0, HDMI_CEC_MASK);
+		dw_hdmi_write(cec, ~0, HDMI_IH_MUTE_CEC_STAT0);
+		dw_hdmi_write(cec, 0, HDMI_CEC_POLARITY);
+
+		cec->ops->disable(cec->hdmi);
+	} else {
+		unsigned int irqs;
+
+		dw_hdmi_write(cec, 0, HDMI_CEC_CTRL);
+		dw_hdmi_write(cec, ~0, HDMI_IH_CEC_STAT0);
+		dw_hdmi_write(cec, 0, HDMI_CEC_LOCK);
+
+		dw_hdmi_cec_log_addr(cec->adap, CEC_LOG_ADDR_INVALID);
+
+		cec->ops->enable(cec->hdmi);
+
+		irqs = CEC_STAT_ERROR_INIT | CEC_STAT_NACK | CEC_STAT_EOM |
+		       CEC_STAT_DONE;
+		dw_hdmi_write(cec, irqs, HDMI_CEC_POLARITY);
+		dw_hdmi_write(cec, ~irqs, HDMI_CEC_MASK);
+		dw_hdmi_write(cec, ~irqs, HDMI_IH_MUTE_CEC_STAT0);
+	}
+	return 0;
+}
+
+static const struct cec_adap_ops dw_hdmi_cec_ops = {
+	.adap_enable = dw_hdmi_cec_enable,
+	.adap_log_addr = dw_hdmi_cec_log_addr,
+	.adap_transmit = dw_hdmi_cec_transmit,
+};
+
+static void dw_hdmi_cec_del(void *data)
+{
+	struct dw_hdmi_cec *cec = data;
+
+	cec_delete_adapter(cec->adap);
+}
+
+static int dw_hdmi_cec_probe(struct platform_device *pdev)
+{
+	struct dw_hdmi_cec_data *data = dev_get_platdata(&pdev->dev);
+	struct dw_hdmi_cec *cec;
+	int ret;
+	void __iomem *cec_io_base;
+	int reg_val;
+
+	if (!data)
+		return -ENXIO;
+
+	/*
+     * Our device is just a convenience - we want to link to the real
+     * hardware device here, so that userspace can see the association
+     * between the HDMI hardware and its associated CEC chardev.
+     */
+	cec = devm_kzalloc(&pdev->dev, sizeof(*cec), GFP_KERNEL);
+	if (!cec)
+		return -ENOMEM;
+
+	cec->irq = data->irq;
+	cec->ops = data->ops;
+	cec->hdmi = data->hdmi;
+
+	platform_set_drvdata(pdev, cec);
+
+	dw_hdmi_write(cec, 0, HDMI_CEC_TX_CNT);
+	dw_hdmi_write(cec, ~0, HDMI_CEC_MASK);
+	dw_hdmi_write(cec, ~0, HDMI_IH_MUTE_CEC_STAT0);
+	dw_hdmi_write(cec, 0, HDMI_CEC_POLARITY);
+
+	cec->adap =
+		cec_allocate_adapter(&dw_hdmi_cec_ops, cec, "dw_hdmi",
+				     CEC_CAP_DEFAULTS | CEC_CAP_CONNECTOR_INFO,
+				     CEC_MAX_LOG_ADDRS);
+	if (IS_ERR(cec->adap))
+		return PTR_ERR(cec->adap);
+
+	/* override the module pointer */
+	cec->adap->owner = THIS_MODULE;
+
+	ret = devm_add_action_or_reset(&pdev->dev, dw_hdmi_cec_del, cec);
+	if (ret)
+		return ret;
+
+	ret = devm_request_threaded_irq(&pdev->dev, cec->irq,
+					dw_hdmi_cec_hardirq, dw_hdmi_cec_thread,
+					IRQF_SHARED, "dw-hdmi-cec", cec->adap);
+	if (ret < 0)
+		return ret;
+
+	cec->notify = cec_notifier_cec_adap_register(pdev->dev.parent, NULL,
+						     cec->adap);
+	if (!cec->notify)
+		return -ENOMEM;
+
+	ret = cec_register_adapter(cec->adap, pdev->dev.parent);
+	if (ret < 0) {
+		cec_notifier_cec_adap_unregister(cec->notify, cec->adap);
+		return ret;
+	}
+
+	/*
+     * CEC documentation says we must not call cec_delete_adapter
+     * after a successful call to cec_register_adapter().
+     */
+	devm_remove_action(&pdev->dev, dw_hdmi_cec_del, cec);
+
+	cec_io_base = devm_ioremap(&pdev->dev, CEC_IO_ADDR, 4);
+	if (!cec_io_base) {
+		dev_err(&pdev->dev, "failed to remap cec io ctl\n");
+		return -ENOMEM;
+	}
+
+	/* set the cec io to output */
+	reg_val = readl((char *)cec_io_base);
+	reg_val |= 0x1;
+	writel(reg_val, (char *)cec_io_base);
+
+	return 0;
+}
+
+static int dw_hdmi_cec_remove(struct platform_device *pdev)
+{
+	struct dw_hdmi_cec *cec = platform_get_drvdata(pdev);
+
+	cec_notifier_cec_adap_unregister(cec->notify, cec->adap);
+	cec_unregister_adapter(cec->adap);
+
+	return 0;
+}
+
+struct platform_driver dw_hdmi_cec_driver = {
+    .probe  = dw_hdmi_cec_probe,
+    .remove = dw_hdmi_cec_remove,
+    .driver = {
+        .name = "dw-hdmi-cec",
+    },
+};
+//module_platform_driver(dw_hdmi_cec_driver);
+
+MODULE_DESCRIPTION("Synopsys Designware HDMI CEC driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS(PLATFORM_MODULE_PREFIX "dw-hdmi-cec");
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_cec.h b/drivers/gpu/drm/eswin/dw_hdmi_cec.h
new file mode 100644
index 000000000000..9d2547f1ea16
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_cec.h
@@ -0,0 +1,18 @@
+#ifndef DW_HDMI_CEC_H
+#define DW_HDMI_CEC_H
+
+struct dw_hdmi;
+
+struct dw_hdmi_cec_ops {
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+	void (*enable)(struct dw_hdmi *hdmi);
+	void (*disable)(struct dw_hdmi *hdmi);
+};
+
+struct dw_hdmi_cec_data {
+	struct dw_hdmi *hdmi;
+	const struct dw_hdmi_cec_ops *ops;
+	int irq;
+};
+#endif
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_hdcp.c b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.c
new file mode 100644
index 000000000000..a6bb3a8b939d
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.c
@@ -0,0 +1,724 @@
+/*
+ * Copyright (C) ESWIN Electronics Co.Ltd
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/hdmi.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/kthread.h>
+#include <linux/mutex.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include "dw-hdmi.h"
+#include "dw_hdmi_hdcp.h"
+
+#define HDCP_KEY_PATH "/usr/hdcp1.4_key/Tx_A2_TestDPK_encrypted"
+
+#define HDCP_KEY_SIZE 308
+#define HDCP_KEY_SEED_SIZE 2
+
+#define KSV_LEN 5
+#define HEADER 10
+#define SHAMAX 20
+
+#define MAX_DOWNSTREAM_DEVICE_NUM 5
+#define DPK_WR_OK_TIMEOUT_US 30000
+#define HDMI_HDCP1X_ID 5
+
+/* HDCP Registers */
+#define HDMI_A_KSVMEMCTRL 0x5016
+#define HDMI_HDCPREG_ANCONF 0x7805
+#define HDMI_HDCPREG_AN0 0x7806
+#define HDMI_HDCPREG_AN1 0x7807
+#define HDMI_HDCPREG_AN2 0x7808
+#define HDMI_HDCPREG_AN3 0x7809
+#define HDMI_HDCPREG_AN4 0x780a
+#define HDMI_HDCPREG_AN5 0x780b
+#define HDMI_HDCPREG_AN6 0x780c
+#define HDMI_HDCPREG_AN7 0x780d
+#define HDMI_HDCPREG_RMCTL 0x780e
+#define HDMI_HDCPREG_RMSTS 0x780f
+#define HDMI_HDCPREG_SEED0 0x7810
+#define HDMI_HDCPREG_SEED1 0x7811
+#define HDMI_HDCPREG_DPK0 0x7812
+#define HDMI_HDCPREG_DPK1 0x7813
+#define HDMI_HDCPREG_DPK2 0x7814
+#define HDMI_HDCPREG_DPK3 0x7815
+#define HDMI_HDCPREG_DPK4 0x7816
+#define HDMI_HDCPREG_DPK5 0x7817
+#define HDMI_HDCPREG_DPK6 0x7818
+#define HDMI_HDCP2REG_CTRL 0x7904
+#define HDMI_HDCP2REG_MASK 0x790c
+#define HDMI_HDCP2REG_MUTE 0x7912
+
+enum dw_hdmi_hdcp_state {
+	DW_HDCP_DISABLED,
+	DW_HDCP_AUTH_START,
+	DW_HDCP_AUTH_SUCCESS,
+	DW_HDCP_AUTH_FAIL,
+};
+
+enum {
+	DW_HDMI_HDCP_KSV_LEN = 8,
+	DW_HDMI_HDCP_SHA_LEN = 20,
+	DW_HDMI_HDCP_DPK_LEN = 280,
+	DW_HDMI_HDCP_KEY_LEN = 308,
+	DW_HDMI_HDCP_SEED_LEN = 2,
+};
+
+enum {
+	HDMI_MC_CLKDIS_HDCPCLK_MASK = 0x40,
+	HDMI_MC_CLKDIS_HDCPCLK_ENABLE = 0x00,
+
+	HDMI_A_SRMCTRL_SHA1_FAIL_MASK = 0X08,
+	HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_SHA1_FAIL_ENABLE = 0X08,
+
+	HDMI_A_SRMCTRL_KSV_UPDATE_MASK = 0X04,
+	HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE = 0X04,
+
+	HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK = 0X01,
+	HDMI_A_SRMCTRL_KSV_MEM_REQ_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_KSV_MEM_REQ_ENABLE = 0X01,
+
+	HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK = 0X02,
+	HDMI_A_SRMCTRL_KSV_MEM_ACCESS_DISABLE = 0X00,
+	HDMI_A_SRMCTRL_KSV_MEM_ACCESS_ENABLE = 0X02,
+
+	HDMI_A_SRM_BASE_MAX_DEVS_EXCEEDED = 0x80,
+	HDMI_A_SRM_BASE_DEVICE_COUNT = 0x7f,
+
+	HDMI_A_SRM_BASE_MAX_CASCADE_EXCEEDED = 0x08,
+
+	HDMI_A_APIINTSTAT_KSVSHA1_CALC_INT = 0x02,
+	HDMI_A_APIINTSTAT_KSVSHA1_CALC_DONE_INT = 0x20,
+	/* HDCPREG_RMSTS field values */
+	DPK_WR_OK_STS = 0x40,
+
+	HDMI_A_HDCP22_MASK = 0x40,
+
+	HDMI_HDCP2_OVR_EN_MASK = 0x02,
+	HDMI_HDCP2_OVR_ENABLE = 0x02,
+	HDMI_HDCP2_OVR_DISABLE = 0x00,
+
+	HDMI_HDCP2_FORCE_MASK = 0x04,
+	HDMI_HDCP2_FORCE_ENABLE = 0x04,
+	HDMI_HDCP2_FORCE_DISABLE = 0x00,
+	HDMI_A_KSVMEMCTRL_KSV_SHA1_STATUS = 0x08,
+};
+
+static struct dw_hdcp *g_hdcp;
+static int trytimes = 0;
+
+static void hdcp_modb(struct dw_hdcp *hdcp, u8 data, u8 mask, unsigned int reg)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	u8 val = hdcp->read(hdmi, reg) & ~mask;
+
+	val |= data & mask;
+	hdcp->write(hdmi, val, reg);
+}
+
+static int hdcp_load_keys_cb(struct dw_hdcp *hdcp)
+{
+	u8 hdcp_vendor_data[320];
+	struct file *fp;
+	loff_t pos = 0;
+	ssize_t nread;
+
+	hdcp->keys = kmalloc(HDCP_KEY_SIZE, GFP_KERNEL);
+	if (!hdcp->keys)
+		return -ENOMEM;
+
+	hdcp->seeds = kmalloc(HDCP_KEY_SEED_SIZE, GFP_KERNEL);
+	if (!hdcp->seeds) {
+		kfree(hdcp->keys);
+		return -ENOMEM;
+	}
+
+	fp = filp_open(HDCP_KEY_PATH, O_RDONLY, 0644);
+	if (IS_ERR(fp)) {
+		printk("Error, Tx_A2_TestDPK_encrypted doesn't exist.\n");
+		goto err;
+	}
+
+	nread = kernel_read(fp, hdcp_vendor_data, sizeof(hdcp_vendor_data),
+			    &pos);
+
+	if (nread != sizeof(hdcp_vendor_data)) {
+		printk("Error, failed to read %ld bytes to non volatile memory area,ret %ld\n",
+		       sizeof(hdcp_vendor_data), nread);
+		filp_close(fp, NULL);
+		goto err;
+	}
+
+	memcpy(hdcp->keys, hdcp_vendor_data, HDCP_KEY_SIZE);
+	memcpy(hdcp->seeds, hdcp_vendor_data + HDCP_KEY_SIZE,
+	       HDCP_KEY_SEED_SIZE);
+
+	filp_close(fp, NULL);
+	return 0;
+err:
+	kfree(hdcp->keys);
+	kfree(hdcp->seeds);
+	hdcp->keys = NULL;
+	hdcp->seeds = NULL;
+	return -EIO;
+}
+
+static int dw_hdmi_hdcp_load_key(struct dw_hdcp *hdcp)
+{
+	int i, j;
+	int ret, val;
+	void __iomem *reg_rmsts_addr;
+	struct hdcp_keys *hdcp_keys;
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+
+	if (!hdcp->keys) {
+		ret = hdcp_load_keys_cb(hdcp);
+		if (ret)
+			return ret;
+	}
+	hdcp_keys = hdcp->keys;
+
+	if (hdcp->reg_io_width == 4)
+		reg_rmsts_addr = hdcp->regs + (HDMI_HDCPREG_RMSTS << 2);
+	else if (hdcp->reg_io_width == 1)
+		reg_rmsts_addr = hdcp->regs + HDMI_HDCPREG_RMSTS;
+	else
+		return -EPERM;
+
+	/* Disable decryption logic */
+	hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL);
+	ret = readx_poll_timeout(readl, reg_rmsts_addr, val,
+				 val & DPK_WR_OK_STS, 1000,
+				 DPK_WR_OK_TIMEOUT_US);
+	if (ret)
+		return ret;
+
+	hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK6);
+	hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK5);
+
+	/* The useful data in ksv should be 5 byte */
+	for (i = 4; i >= 0; i--)
+		hdcp->write(hdmi, hdcp_keys->KSV[i], HDMI_HDCPREG_DPK0 + i);
+	ret = readx_poll_timeout(readl, reg_rmsts_addr, val,
+				 val & DPK_WR_OK_STS, 1000,
+				 DPK_WR_OK_TIMEOUT_US);
+
+	if (ret)
+		return ret;
+
+	/* Enable decryption logic */
+	if (hdcp->seeds) {
+		hdcp->write(hdmi, 1, HDMI_HDCPREG_RMCTL);
+		hdcp->write(hdmi, hdcp->seeds[0], HDMI_HDCPREG_SEED1);
+		hdcp->write(hdmi, hdcp->seeds[1], HDMI_HDCPREG_SEED0);
+	} else {
+		hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL);
+	}
+
+	/* Write encrypt device private key */
+	for (i = 0; i < DW_HDMI_HDCP_DPK_LEN - 6; i += 7) {
+		for (j = 6; j >= 0; j--)
+			hdcp->write(hdmi, hdcp_keys->devicekey[i + j],
+				    HDMI_HDCPREG_DPK0 + j);
+		ret = readx_poll_timeout(readl, reg_rmsts_addr, val,
+					 val & DPK_WR_OK_STS, 1000,
+					 DPK_WR_OK_TIMEOUT_US);
+
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+static int dw_hdmi_hdcp1x_start(struct dw_hdcp *hdcp)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	int i;
+	int val;
+	u8 An[8];
+	int ret;
+
+	if (!hdcp->enable)
+		return -EPERM;
+
+	if (hdcp->status == DW_HDCP_AUTH_START ||
+	    hdcp->status == DW_HDCP_AUTH_SUCCESS)
+		return 0;
+
+	/* Update An */
+	get_random_bytes(&An, sizeof(An));
+	for (i = 0; i < 8; i++)
+		hdcp->write(hdmi, An[i], HDMI_HDCPREG_AN0 + i);
+
+	hdcp->write(hdmi, 0x01, HDMI_HDCPREG_ANCONF);
+
+	if (!(hdcp->read(hdmi, HDMI_HDCPREG_RMSTS) & 0x3f)) {
+		ret = dw_hdmi_hdcp_load_key(hdcp);
+		if (ret) {
+			dev_err(hdcp->dev, "load hdcp key failed, ret=%d\n", ret);
+			return ret;
+		}
+	}
+
+	/* disable the pixel clock*/
+	dev_dbg(hdcp->dev, "start hdcp with disable hdmi pixel clock\n");
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_DISABLE,
+		  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+
+	if (hdcp->hdcp2) {
+		for (i = 0; i < 100; i++) {
+			if (hdcp->hdcp2->wait_hdcp2_reset) {
+				msleep(80);
+			} else {
+				break;
+			}
+		}
+		printk("wait_hdcp2_reset i = %d\n", i);
+	}
+	val = hdcp->read(hdmi, HDMI_HDCP2REG_CTRL);
+	dev_dbg(hdcp->dev, "before set HDMI_HDCP2REG_CTRL val = %d\n", val);
+
+	hdcp_modb(hdcp, HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE,
+		  HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK, HDMI_FC_INVIDCONF);
+
+	hdcp->remaining_times = hdcp->retry_times;
+	if (hdcp->read(hdmi, HDMI_CONFIG1_ID) & HDMI_A_HDCP22_MASK) {
+		hdcp_modb(hdcp,
+			  HDMI_HDCP2_OVR_ENABLE | HDMI_HDCP2_FORCE_DISABLE,
+			  HDMI_HDCP2_OVR_EN_MASK | HDMI_HDCP2_FORCE_MASK,
+			  HDMI_HDCP2REG_CTRL);
+		hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MASK);
+		hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MUTE);
+	}
+
+	hdcp->write(hdmi, 0x40, HDMI_A_OESSWCFG);
+	hdcp_modb(hdcp,
+		  HDMI_A_HDCPCFG0_BYPENCRYPTION_DISABLE |
+			  HDMI_A_HDCPCFG0_EN11FEATURE_DISABLE |
+			  HDMI_A_HDCPCFG0_SYNCRICHECK_ENABLE,
+		  HDMI_A_HDCPCFG0_BYPENCRYPTION_MASK |
+			  HDMI_A_HDCPCFG0_EN11FEATURE_MASK |
+			  HDMI_A_HDCPCFG0_SYNCRICHECK_MASK,
+		  HDMI_A_HDCPCFG0);
+
+	hdcp_modb(hdcp,
+		  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_ENABLE |
+			  HDMI_A_HDCPCFG1_PH2UPSHFTENC_ENABLE,
+		  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK |
+			  HDMI_A_HDCPCFG1_PH2UPSHFTENC_MASK,
+		  HDMI_A_HDCPCFG1);
+
+	/* Reset HDCP Engine */
+	if (hdcp->read(hdmi, HDMI_MC_CLKDIS) & HDMI_MC_CLKDIS_HDCPCLK_MASK) {
+		hdcp_modb(hdcp, HDMI_A_HDCPCFG1_SWRESET_ASSERT,
+			  HDMI_A_HDCPCFG1_SWRESET_MASK, HDMI_A_HDCPCFG1);
+	}
+
+	hdcp->write(hdmi, 0x00, HDMI_A_APIINTMSK);
+	hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_ENABLE,
+		  HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0);
+
+	/*
+     * XXX: to sleep 500ms here between output hdmi and enable hdcpclk,
+     * otherwise hdcp auth fail when Connect to repeater
+     */
+	msleep(500);
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_ENABLE,
+		  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+
+	hdcp->status = DW_HDCP_AUTH_START;
+	dev_info(hdcp->dev, "%s success\n", __func__);
+
+	/* enable the pixel clock*/
+	dev_dbg(hdcp->dev, "start hdcp with enable hdmi pixel clock\n");
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_ENABLE,
+		  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+
+	return 0;
+}
+
+static int dw_hdmi_hdcp1x_stop(struct dw_hdcp *hdcp)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	u8 val;
+	bool phy_enable = false;
+
+	if (!hdcp->enable)
+		return -EPERM;
+
+	val = hdcp->read(hdmi, HDMI_PHY_CONF0);
+	if (val & HDMI_PHY_CONF0_GEN2_TXPWRON_MASK) {
+		phy_enable = true;
+	}
+
+	dev_dbg(hdcp->dev, "dw_hdmi_hdcp1x_stop\n");
+	if (phy_enable) {
+		dev_dbg(hdcp->dev, "stop hdcp with disable hdmi pixel clock\n");
+		hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_DISABLE,
+			  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+	}
+
+	hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_DISABLE,
+		  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+	hdcp->write(hdmi, 0xff, HDMI_A_APIINTMSK);
+
+	hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_DISABLE,
+		  HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0);
+
+	hdcp_modb(hdcp,
+		  HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE |
+			  HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE,
+		  HDMI_A_SRMCTRL_SHA1_FAIL_MASK |
+			  HDMI_A_SRMCTRL_KSV_UPDATE_MASK,
+		  HDMI_A_SRMCTRL);
+
+	hdcp->status = DW_HDCP_DISABLED;
+
+	if (phy_enable) {
+		dev_dbg(hdcp->dev, "stop hdcp with enable hdmi pixel clock\n");
+		hdcp_modb(hdcp, HDMI_MC_CLKDIS_PIXELCLK_ENABLE,
+			  HDMI_MC_CLKDIS_PIXELCLK_MASK, HDMI_MC_CLKDIS);
+	}
+
+	return 0;
+}
+
+void dw_hdmi_hdcp2_init(struct dw_hdcp2 *hdcp2)
+{
+	if (g_hdcp)
+		g_hdcp->hdcp2 = hdcp2;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_hdcp2_init);
+
+void dw_hdmi_hdcp2_remove(void)
+{
+	printk("func: %s; line: %d\n", __func__, __LINE__);
+	if (g_hdcp->hdcp2)
+		g_hdcp->hdcp2->stop();
+	g_hdcp->hdcp2 = NULL;
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_hdcp2_remove);
+
+void dw_hdmi_hdcp2_start(int enable)
+{
+	int val;
+
+	if (!(g_hdcp->hdcp2))
+		return;
+
+	dev_dbg(g_hdcp->dev, "%s enable = %d\n", __func__, enable);
+	if (enable == 0) {
+		hdcp_modb(g_hdcp,
+			  HDMI_HDCP2_OVR_ENABLE | HDMI_HDCP2_FORCE_DISABLE,
+			  HDMI_HDCP2_OVR_EN_MASK | HDMI_HDCP2_FORCE_MASK,
+			  HDMI_HDCP2REG_CTRL);
+		hdcp_modb(g_hdcp, HDMI_MC_CLKDIS_HDCPCLK_DISABLE,
+			  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+	} else if (enable == 1) {
+		hdcp_modb(g_hdcp, HDMI_MC_CLKDIS_HDCPCLK_ENABLE,
+			  HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS);
+		hdcp_modb(g_hdcp,
+			  HDMI_HDCP2_OVR_ENABLE | HDMI_HDCP2_FORCE_ENABLE,
+			  HDMI_HDCP2_OVR_EN_MASK | HDMI_HDCP2_FORCE_MASK,
+			  HDMI_HDCP2REG_CTRL);
+		hdcp_modb(g_hdcp, HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE,
+			  HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK,
+			  HDMI_FC_INVIDCONF);
+	} else if (enable == 2) {
+		val = g_hdcp->read(g_hdcp->hdmi, HDMI_PHY_STAT0);
+		if (val & HDMI_PHY_HPD)
+			dw_hdmi_hdcp1x_start(g_hdcp);
+	} else if (enable == 3) {
+		if (g_hdcp->hdcp2 && g_hdcp->hdcp2->enable &&
+		    (tv_hdmi_hdcp2_support(g_hdcp->hdmi) == 1)) {
+			if (g_hdcp->status != DW_HDCP_DISABLED)
+				dw_hdmi_hdcp1x_stop(g_hdcp);
+			g_hdcp->hdcp2->start();
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(dw_hdmi_hdcp2_start);
+
+static int dw_hdmi_hdcp_start(struct dw_hdcp *hdcp)
+{
+	if (hdcp->hdcp2 && hdcp->hdcp2->enable &&
+	    (tv_hdmi_hdcp2_support(hdcp->hdmi) == 1)) {
+		// hdcp->hdcp2->start();
+		return 0;
+	}
+	return dw_hdmi_hdcp1x_start(hdcp);
+}
+
+static int dw_hdmi_hdcp_stop(struct dw_hdcp *hdcp)
+{
+	if (hdcp->hdcp2 && hdcp->hdcp2->hot_plug) {
+		// g_hdcp->hdcp2->stop();
+		printk("func: %s; line: %d\n", __func__, __LINE__);
+	}
+
+	return dw_hdmi_hdcp1x_stop(hdcp);
+}
+
+static void dw_hdmi_hdcp_isr(struct dw_hdcp *hdcp, int hdcp_int)
+{
+	struct dw_hdmi *hdmi = hdcp->hdmi;
+	int val;
+
+	dev_info(hdcp->dev, "hdcp_int is 0x%02x\n", hdcp_int);
+
+	if (hdcp_int & HDMI_A_APIINTSTAT_KSVSHA1_CALC_DONE_INT) {
+		dev_dbg(hdcp->dev, "hdcp sink is a repeater\n");
+		val = hdcp->read(hdmi, HDMI_A_KSVMEMCTRL);
+		if (val | HDMI_A_KSVMEMCTRL_KSV_SHA1_STATUS) {
+			dev_dbg(hdcp->dev,
+				"hdcp verifivation failed, waiting hdmi controller re-authentication!\n");
+		} else {
+			dev_dbg(hdcp->dev, "hdcp verifivation succeeded!\n");
+			/* reset HDCP */
+			hdcp_modb(hdcp, HDMI_A_HDCPCFG1_SWRESET_ASSERT,
+				  HDMI_A_HDCPCFG1_SWRESET_MASK,
+				  HDMI_A_HDCPCFG1);
+		}
+	}
+
+	if (hdcp_int & 0x40) {
+		hdcp->status = DW_HDCP_AUTH_FAIL;
+		dev_info(hdcp->dev, "hdcp auth fail\n");
+		if (hdcp->remaining_times > 1)
+			hdcp->remaining_times--;
+		else if (hdcp->remaining_times == 1)
+			hdcp_modb(hdcp,
+				  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE,
+				  HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK,
+				  HDMI_A_HDCPCFG1);
+	} else if (hdcp_int & 0x80) {
+		dev_info(hdcp->dev, "hdcp auth success\n");
+		hdcp->status = DW_HDCP_AUTH_SUCCESS;
+	} else if (hdcp_int & 0x10) {
+		dev_info(hdcp->dev, "i2c nack\n");
+		trytimes++;
+		if (trytimes == 20) {
+			trytimes = 0;
+			dw_hdmi_hdcp1x_stop(hdcp);
+		}
+	}
+}
+
+static ssize_t hdcp_enable_read(struct device *device,
+				struct device_attribute *attr, char *buf)
+{
+	bool enable = 0;
+	struct dw_hdcp *hdcp = g_hdcp;
+
+	if (hdcp)
+		enable = hdcp->enable;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", enable);
+}
+
+static ssize_t hdcp_enable_write(struct device *device,
+				 struct device_attribute *attr, const char *buf,
+				 size_t count)
+{
+	bool enable;
+	struct dw_hdcp *hdcp = g_hdcp;
+	int ret;
+
+	if (!hdcp)
+		return -EINVAL;
+
+	if (kstrtobool(buf, &enable))
+		return -EINVAL;
+
+	if (hdcp->enable != enable) {
+		if (enable) {
+			hdcp->enable = enable;
+			if (hdcp->hdcp2 && hdcp->hdcp2->hot_plug) {
+				return count;
+			}
+
+			if (hdcp->read(hdcp->hdmi, HDMI_PHY_STAT0) &
+			    HDMI_PHY_HPD) {
+				ret = dw_hdmi_hdcp1x_start(hdcp);
+				if (ret) {
+					hdcp->enable = 0;
+				}
+			}
+		} else {
+			if (hdcp->status != DW_HDCP_DISABLED) {
+				dw_hdmi_hdcp1x_stop(hdcp);
+			}
+			hdcp->enable = enable;
+		}
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR(enable, 0644, hdcp_enable_read, hdcp_enable_write);
+
+static ssize_t hdcp_trytimes_read(struct device *device,
+				  struct device_attribute *attr, char *buf)
+{
+	int trytimes = 0;
+	struct dw_hdcp *hdcp = g_hdcp;
+
+	if (hdcp)
+		trytimes = hdcp->retry_times;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", trytimes);
+}
+
+static ssize_t hdcp_trytimes_write(struct device *device,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	int trytimes;
+	struct dw_hdcp *hdcp = g_hdcp;
+
+	if (!hdcp)
+		return -EINVAL;
+
+	if (kstrtoint(buf, 0, &trytimes))
+		return -EINVAL;
+
+	if (hdcp->retry_times != trytimes) {
+		hdcp->retry_times = trytimes;
+		hdcp->remaining_times = hdcp->retry_times;
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR(trytimes, 0644, hdcp_trytimes_read, hdcp_trytimes_write);
+
+static ssize_t hdcp_status_read(struct device *device,
+				struct device_attribute *attr, char *buf)
+{
+	int status = DW_HDCP_DISABLED;
+	struct dw_hdcp *hdcp = g_hdcp;
+
+	if (hdcp)
+		status = hdcp->status;
+
+	if (status == DW_HDCP_DISABLED)
+		return snprintf(buf, PAGE_SIZE, "hdcp disable\n");
+	else if (status == DW_HDCP_AUTH_START)
+		return snprintf(buf, PAGE_SIZE, "hdcp_auth_start\n");
+	else if (status == DW_HDCP_AUTH_SUCCESS)
+		return snprintf(buf, PAGE_SIZE, "hdcp_auth_success\n");
+	else if (status == DW_HDCP_AUTH_FAIL)
+		return snprintf(buf, PAGE_SIZE, "hdcp_auth_fail\n");
+	else
+		return snprintf(buf, PAGE_SIZE, "unknown status\n");
+}
+
+static DEVICE_ATTR(status, 0444, hdcp_status_read, NULL);
+
+static int dw_hdmi_hdcp_probe(struct platform_device *pdev)
+{
+	int ret = 0;
+	struct dw_hdcp *hdcp = pdev->dev.platform_data;
+
+	dev_info(&pdev->dev, "%s...\n", __func__);
+	g_hdcp = hdcp;
+	hdcp->mdev.minor = MISC_DYNAMIC_MINOR;
+	hdcp->mdev.name = "hdmi_hdcp1x";
+	hdcp->mdev.mode = 0666;
+
+	if (misc_register(&hdcp->mdev)) {
+		dev_err(&pdev->dev, "HDCP: Could not add character driver\n");
+		return -EINVAL;
+	}
+
+	ret = device_create_file(hdcp->mdev.this_device, &dev_attr_enable);
+	if (ret) {
+		dev_err(&pdev->dev, "HDCP: Could not add sys file enable\n");
+		ret = -EINVAL;
+		goto error0;
+	}
+
+	ret = device_create_file(hdcp->mdev.this_device, &dev_attr_trytimes);
+	if (ret) {
+		dev_err(&pdev->dev, "HDCP: Could not add sys file trytimes\n");
+		ret = -EINVAL;
+		goto error1;
+	}
+
+	ret = device_create_file(hdcp->mdev.this_device, &dev_attr_status);
+	if (ret) {
+		dev_err(&pdev->dev, "HDCP: Could not add sys file status\n");
+		ret = -EINVAL;
+		goto error2;
+	}
+
+	/* retry time if hdcp auth fail. unlimited time if set 0 */
+	hdcp->retry_times = 0;
+	hdcp->dev = &pdev->dev;
+	hdcp->hdcp_start = dw_hdmi_hdcp_start;
+	hdcp->hdcp_stop = dw_hdmi_hdcp_stop;
+	hdcp->hdcp_isr = dw_hdmi_hdcp_isr;
+
+#ifdef CONFIG_DW_HDMI_HDCP1X_ENABLED
+	hdcp_enable_write(NULL, NULL, "1", 1);
+#endif
+
+	dev_dbg(hdcp->dev, "%s success\n", __func__);
+	return 0;
+
+error2:
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes);
+error1:
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_enable);
+error0:
+	misc_deregister(&hdcp->mdev);
+	return ret;
+}
+
+static int dw_hdmi_hdcp_remove(struct platform_device *pdev)
+{
+	struct dw_hdcp *hdcp = pdev->dev.platform_data;
+
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes);
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_enable);
+	device_remove_file(hdcp->mdev.this_device, &dev_attr_status);
+	misc_deregister(&hdcp->mdev);
+
+	kfree(hdcp->keys);
+	kfree(hdcp->seeds);
+
+	return 0;
+}
+
+struct platform_driver dw_hdmi_hdcp_driver = {
+    .probe  = dw_hdmi_hdcp_probe,
+    .remove = dw_hdmi_hdcp_remove,
+    .driver = {
+        .name = DW_HDCP_DRIVER_NAME,
+    },
+};
+
+//module_platform_driver(dw_hdmi_hdcp_driver);
+MODULE_DESCRIPTION("DW HDMI transmitter HDCP driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_hdcp.h b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.h
new file mode 100644
index 000000000000..a4992e8a8cbc
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_hdcp.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) ESWIN Electronics Co.Ltd
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef DW_HDMI_HDCP_H
+#define DW_HDMI_HDCP_H
+
+#include <linux/miscdevice.h>
+
+#define DW_HDCP_DRIVER_NAME "dw-hdmi-hdcp"
+#define HDCP_PRIVATE_KEY_SIZE 280
+#define HDCP_KEY_SHA_SIZE 20
+
+struct hdcp_keys {
+	u8 KSV[8];
+	u8 devicekey[HDCP_PRIVATE_KEY_SIZE];
+	u8 sha1[HDCP_KEY_SHA_SIZE];
+};
+
+struct dw_hdcp2 {
+	int enable;
+	void (*start)(void);
+	void (*stop)(void);
+
+	struct device *dev;
+	int wait_hdcp2_reset;
+	int hot_plug;
+	struct miscdevice mdev;
+	int auth_sucess;
+};
+
+struct dw_hdcp {
+	bool enable;
+	int retry_times;
+	int remaining_times;
+	char *seeds;
+	int invalidkey;
+	char *invalidkeys;
+	int hdcp2_enable;
+	int status;
+	u32 reg_io_width;
+
+	struct dw_hdcp2 *hdcp2;
+	struct miscdevice mdev;
+	struct hdcp_keys *keys;
+	struct device *dev;
+	struct dw_hdmi *hdmi;
+	void __iomem *regs;
+
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+	int (*hdcp_start)(struct dw_hdcp *hdcp);
+	int (*hdcp_stop)(struct dw_hdcp *hdcp);
+	void (*hdcp_isr)(struct dw_hdcp *hdcp, int hdcp_int);
+};
+
+extern u8 tv_hdmi_hdcp2_support(struct dw_hdmi *hdmi);
+extern void dw_hdmi_hdcp2_init(struct dw_hdcp2 *hdcp2);
+extern void dw_hdmi_hdcp2_remove(void);
+extern void dw_hdmi_hdcp2_start(int enable);
+#endif
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_hdcp2.c b/drivers/gpu/drm/eswin/dw_hdmi_hdcp2.c
new file mode 100644
index 000000000000..e387f079eb89
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_hdcp2.c
@@ -0,0 +1,774 @@
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/miscdevice.h>
+#include <linux/dma-mapping.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/platform_device.h>
+
+#include <linux/version.h>
+#include <linux/netlink.h>
+#include <linux/proc_fs.h>
+#include <linux/moduleparam.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+
+#include "host_lib_driver_linux_if.h"
+#include "dw_hdmi_hdcp.h"
+
+/**
+ * \file
+ * \ingroup HL_Driver_Kernel
+ * \brief Sample Linux Host Library Driver
+ * \copydoc HL_Driver_Kernel
+ */
+
+/**
+ * \defgroup HL_Driver_Linux Sample Linux Host Library Driver
+ * \ingroup HL_Driver
+ * \brief Sample code for the Linux Host Library Driver.
+ * The Linux Host Library Driver is composed of 2 parts:
+ * 1. A kernel driver.
+ * 2. A file access instance.
+ *
+ * The kernel driver is the kernel executable code enabling the firmware to execute.
+ * It provides the access to the hardware register to interact with the firmware.
+ *
+ * The file access instance initializes the #hl_driver_t structure for the
+ * host library access.  The Host Library references the file access to request the
+ * kernel operations.
+ */
+
+/**
+ * \defgroup HL_Driver_Kernel Sample Linux Kernel Host Library Driver
+ * \ingroup HL_Driver_Linux
+ * \brief Example code for the Linux Kernel Host Library Driver.
+ *
+ * The Sample Linux Kernel Driver operates on the linux kernel.
+ * To install (requires root access):
+ * \code
+ insmod bin/linux_hld_module.ko verbose=0
+ * \endcode
+ *
+ * To remove (requires root access):
+ * \code
+ rmmod linux_hld_module
+ * \endcode
+ *
+ * Example Linux Host Library Code:
+ * \code
+ */
+
+#define MAX_HL_DEVICES 16
+#define TROOT_GRIFFIN
+
+static bool randomize_mem = false;
+module_param(randomize_mem, bool, 0);
+MODULE_PARM_DESC(noverify, "Wipe memory allocations on startup (for debug)");
+
+static struct dw_hdcp2 *g_dw_hdcp2;
+
+static void dw_hdcp2_stop(void)
+{
+	printk("func: %s; line: %d\n", __func__, __LINE__);
+	g_dw_hdcp2->hot_plug = 0;
+	dw_hdmi_hdcp2_start(0);
+}
+
+static void dw_hdcp2_start(void)
+{
+	printk("func: %s; line: %d\n", __func__, __LINE__);
+	dw_hdmi_hdcp2_start(1);
+	g_dw_hdcp2->hot_plug = 1;
+}
+
+//
+// HL Device
+//
+typedef struct {
+	int allocated, initialized;
+	int code_loaded;
+
+	int code_is_phys_mem;
+	dma_addr_t code_base;
+	uint32_t code_size;
+	uint8_t *code;
+	int data_is_phys_mem;
+	dma_addr_t data_base;
+	uint32_t data_size;
+	uint8_t *data;
+
+	struct resource *hpi_resource;
+	uint8_t __iomem *hpi;
+} hl_device;
+
+static hl_device hl_devices[MAX_HL_DEVICES];
+
+/* HL_DRV_IOC_MEMINFO implementation */
+static long get_meminfo(hl_device *hl_dev, void __user *arg)
+{
+	struct hl_drv_ioc_meminfo info;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->hpi_resource == 0)) {
+		return -EFAULT;
+	}
+
+	info.hpi_base = hl_dev->hpi_resource->start;
+	info.code_base = hl_dev->code_base;
+	info.code_size = hl_dev->code_size;
+	info.data_base = hl_dev->data_base;
+	info.data_size = hl_dev->data_size;
+
+	if (copy_to_user(arg, &info, sizeof info) != 0) {
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/* HL_DRV_IOC_LOAD_CODE implementation */
+static long load_code(hl_device *hl_dev, struct hl_drv_ioc_code __user *arg)
+{
+	struct hl_drv_ioc_code head;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->code == 0) ||
+	    (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&head, arg, sizeof head) != 0)
+		return -EFAULT;
+
+	if (head.len > hl_dev->code_size)
+		return -ENOSPC;
+
+	if (hl_dev->code_loaded)
+		return -EBUSY;
+
+	if (randomize_mem) {
+		get_random_bytes(hl_dev->code, hl_dev->code_size);
+		get_random_bytes(hl_dev->data, hl_dev->data_size);
+	}
+
+	if (copy_from_user(hl_dev->code, &arg->data, head.len) != 0)
+		return -EFAULT;
+
+	hl_dev->code_loaded = 1;
+	return 0;
+}
+
+/* HL_DRV_IOC_WRITE_DATA implementation */
+static long write_data(hl_device *hl_dev, struct hl_drv_ioc_data __user *arg)
+{
+	struct hl_drv_ioc_data head;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&head, arg, sizeof head) != 0)
+		return -EFAULT;
+
+	if (hl_dev->data_size < head.len)
+		return -ENOSPC;
+	if (hl_dev->data_size - head.len < head.offset)
+		return -ENOSPC;
+
+	if (copy_from_user(hl_dev->data + head.offset, &arg->data, head.len) !=
+	    0)
+		return -EFAULT;
+
+	return 0;
+}
+
+/* HL_DRV_IOC_READ_DATA implementation */
+static long read_data(hl_device *hl_dev, struct hl_drv_ioc_data __user *arg)
+{
+	struct hl_drv_ioc_data head;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&head, arg, sizeof head) != 0)
+		return -EFAULT;
+
+	if (hl_dev->data_size < head.len)
+		return -ENOSPC;
+	if (hl_dev->data_size - head.len < head.offset)
+		return -ENOSPC;
+
+	if (copy_to_user(&arg->data, hl_dev->data + head.offset, head.len) != 0)
+		return -EFAULT;
+
+	return 0;
+}
+
+/* HL_DRV_IOC_MEMSET_DATA implementation */
+static long set_data(hl_device *hl_dev, void __user *arg)
+{
+	union {
+		struct hl_drv_ioc_data data;
+		unsigned char buf[sizeof(struct hl_drv_ioc_data) + 1];
+	} u;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->data == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&u.data, arg, sizeof u.buf) != 0)
+		return -EFAULT;
+
+	if (hl_dev->data_size < u.data.len)
+		return -ENOSPC;
+	if (hl_dev->data_size - u.data.len < u.data.offset)
+		return -ENOSPC;
+
+	memset(hl_dev->data + u.data.offset, u.data.data[0], u.data.len);
+	return 0;
+}
+
+/* HL_DRV_IOC_READ_HPI implementation */
+static long hpi_read(hl_device *hl_dev, void __user *arg)
+{
+	struct hl_drv_ioc_hpi_reg reg;
+
+	if ((hl_dev == 0) || (arg == 0) || (hl_dev->hpi_resource == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&reg, arg, sizeof reg) != 0)
+		return -EFAULT;
+
+	if ((reg.offset & 3) ||
+	    reg.offset >= resource_size(hl_dev->hpi_resource))
+		return -EINVAL;
+
+	reg.value = ioread32(hl_dev->hpi + reg.offset);
+
+	if (copy_to_user(arg, &reg, sizeof reg) != 0)
+		return -EFAULT;
+
+	return 0;
+}
+
+/* HL_DRV_IOC_WRITE_HPI implementation */
+static long hpi_write(hl_device *hl_dev, void __user *arg)
+{
+	struct hl_drv_ioc_hpi_reg reg;
+
+	if ((hl_dev == 0) || (arg == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&reg, arg, sizeof reg) != 0)
+		return -EFAULT;
+
+	if ((reg.offset & 3) ||
+	    reg.offset >= resource_size(hl_dev->hpi_resource))
+		return -EINVAL;
+
+	iowrite32(reg.value, hl_dev->hpi + reg.offset);
+
+#ifdef TROOT_GRIFFIN
+	//  If Kill command
+	//  (HL_GET_CMD_EVENT(krequest.data) == TROOT_CMD_SYSTEM_ON_EXIT_REQ))
+	//
+	if ((reg.offset == 0x38) && ((reg.value & 0x000000ff) == 0x08)) {
+		hl_dev->code_loaded = 0;
+	}
+#endif
+	return 0;
+}
+
+static hl_device *alloc_hl_dev_slot(const struct hl_drv_ioc_meminfo *info)
+{
+	int i;
+
+	if (info == 0) {
+		return 0;
+	}
+
+	/* Check if we have a matching device (same HPI base) */
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		hl_device *slot = &hl_devices[i];
+		if (slot->allocated &&
+		    (info->hpi_base == slot->hpi_resource->start))
+			return slot;
+	}
+
+	/* Find unused slot */
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		hl_device *slot = &hl_devices[i];
+		if (!slot->allocated) {
+			slot->allocated = 1;
+			return slot;
+		}
+	}
+
+	return 0;
+}
+
+static void free_dma_areas(hl_device *hl_dev)
+{
+	if (hl_dev == 0) {
+		return;
+	}
+
+	if (!hl_dev->code_is_phys_mem && hl_dev->code) {
+		dma_free_coherent(0, hl_dev->code_size, hl_dev->code,
+				  hl_dev->code_base);
+		hl_dev->code = 0;
+	}
+
+	if (!hl_dev->data_is_phys_mem && hl_dev->data) {
+		dma_free_coherent(0, hl_dev->data_size, hl_dev->data,
+				  hl_dev->data_base);
+		hl_dev->data = 0;
+	}
+}
+
+static int alloc_dma_areas(hl_device *hl_dev,
+			   const struct hl_drv_ioc_meminfo *info)
+{
+	if ((hl_dev == 0) || (info == 0)) {
+		return -EFAULT;
+	}
+
+	hl_dev->code_size = info->code_size;
+	hl_dev->code_is_phys_mem =
+		(info->code_base != HL_DRIVER_ALLOCATE_DYNAMIC_MEM);
+
+	if (hl_dev->code_is_phys_mem && (hl_dev->code == 0)) {
+		/* TODO: support highmem */
+		hl_dev->code_base = info->code_base;
+		hl_dev->code = phys_to_virt(hl_dev->code_base);
+	} else {
+		hl_dev->code =
+			dma_alloc_coherent(g_dw_hdcp2->dev, hl_dev->code_size,
+					   &hl_dev->code_base, GFP_KERNEL);
+		if (!hl_dev->code) {
+			return -ENOMEM;
+		}
+	}
+
+	hl_dev->data_size = info->data_size;
+	hl_dev->data_is_phys_mem =
+		(info->data_base != HL_DRIVER_ALLOCATE_DYNAMIC_MEM);
+
+	if (hl_dev->data_is_phys_mem && (hl_dev->data == 0)) {
+		hl_dev->data_base = info->data_base;
+		hl_dev->data = phys_to_virt(hl_dev->data_base);
+	} else {
+		hl_dev->data =
+			dma_alloc_coherent(g_dw_hdcp2->dev, hl_dev->data_size,
+					   &hl_dev->data_base, GFP_KERNEL);
+		if (!hl_dev->data) {
+			free_dma_areas(hl_dev);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+/* HL_DRV_IOC_INIT implementation */
+static long init(struct file *f, void __user *arg)
+{
+	struct resource *hpi_mem;
+	struct hl_drv_ioc_meminfo info;
+	hl_device *hl_dev;
+	int rc;
+
+	if ((f == 0) || (arg == 0)) {
+		return -EFAULT;
+	}
+
+	if (copy_from_user(&info, arg, sizeof info) != 0)
+		return -EFAULT;
+	hl_dev = alloc_hl_dev_slot(&info);
+	if (!hl_dev)
+		return -EMFILE;
+
+	if (!hl_dev->initialized) {
+		rc = alloc_dma_areas(hl_dev, &info);
+		if (rc < 0)
+			goto err_free;
+
+		hpi_mem = request_mem_region(info.hpi_base, 128, "hl_dev-hpi");
+		if (!hpi_mem) {
+			rc = -EADDRNOTAVAIL;
+			goto err_free;
+		}
+
+		hl_dev->hpi = ioremap(hpi_mem->start, resource_size(hpi_mem));
+		if (!hl_dev->hpi) {
+			rc = -ENOMEM;
+			goto err_release_region;
+		}
+		hl_dev->hpi_resource = hpi_mem;
+		hl_dev->initialized = 1;
+	}
+
+	f->private_data = hl_dev;
+	return 0;
+
+err_release_region:
+	release_resource(hpi_mem);
+err_free:
+	free_dma_areas(hl_dev);
+	hl_dev->initialized = 0;
+	hl_dev->allocated = 0;
+	hl_dev->hpi_resource = 0;
+	hl_dev->hpi = 0;
+
+	return rc;
+}
+
+static void free_hl_dev_slot(hl_device *slot)
+{
+	if (slot == 0) {
+		return;
+	}
+
+	if (!slot->allocated)
+		return;
+
+	if (slot->initialized) {
+		if (slot->hpi) {
+			iounmap(slot->hpi);
+			slot->hpi = 0;
+		}
+
+		if (slot->hpi_resource) {
+			release_mem_region(slot->hpi_resource->start, 128);
+			slot->hpi_resource = 0;
+		}
+
+		free_dma_areas(slot);
+	}
+
+	slot->initialized = 0;
+	slot->allocated = 0;
+}
+
+static long hld_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
+{
+	hl_device *hl_dev;
+	void __user *data;
+
+	if (f == 0) {
+		return -EFAULT;
+	}
+
+	hl_dev = f->private_data;
+	data = (void __user *)arg;
+
+	if (cmd == HL_DRV_IOC_INIT) {
+		return init(f, data);
+	} else if (!hl_dev) {
+		return -EAGAIN;
+	}
+
+	switch (cmd) {
+	case HL_DRV_IOC_INIT:
+		return init(f, data);
+	case HL_DRV_IOC_MEMINFO:
+		return get_meminfo(hl_dev, data);
+	case HL_DRV_IOC_READ_HPI:
+		return hpi_read(hl_dev, data);
+	case HL_DRV_IOC_WRITE_HPI:
+		return hpi_write(hl_dev, data);
+	case HL_DRV_IOC_LOAD_CODE:
+		return load_code(hl_dev, data);
+	case HL_DRV_IOC_WRITE_DATA:
+		return write_data(hl_dev, data);
+	case HL_DRV_IOC_READ_DATA:
+		return read_data(hl_dev, data);
+	case HL_DRV_IOC_MEMSET_DATA:
+		return set_data(hl_dev, data);
+
+	case DW_DRV_IOC_CONNECT_STATUS:
+		return g_dw_hdcp2->hot_plug;
+	case DW_DRV_IOC_CONNECT_SET:
+		printk("set hdcp2 reset one\n");
+		g_dw_hdcp2->wait_hdcp2_reset = 1;
+		dw_hdmi_hdcp2_start(1);
+		return 0;
+	case DW_DRV_IOC_DISCONNECT_SET:
+		if (g_dw_hdcp2->wait_hdcp2_reset == 1) {
+			printk("set hdcp2 reset zero\n");
+			g_dw_hdcp2->wait_hdcp2_reset = 0;
+			dw_hdmi_hdcp2_start(0);
+		}
+		if (g_dw_hdcp2->auth_sucess == 1) {
+			g_dw_hdcp2->auth_sucess = 0;
+		}
+		return 0;
+	case DW_DRV_IOC_AUTH_SUCCESS:
+		g_dw_hdcp2->auth_sucess = 1;
+		return 0;
+	case DW_DRV_IOC_AUTH_FAIL:
+		g_dw_hdcp2->auth_sucess = 0;
+		return 0;
+	case DW_DRV_IOC_NO_CAPACITY:
+		printk("set hdcp2 reset zero 3005\n");
+		g_dw_hdcp2->hot_plug = 0;
+		g_dw_hdcp2->wait_hdcp2_reset = 0;
+		dw_hdmi_hdcp2_start(0);
+		dw_hdmi_hdcp2_start(2);
+		return 0;
+	}
+
+	return -ENOTTY;
+}
+
+static const struct file_operations hld_file_operations = {
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = hld_ioctl,
+#endif
+	.unlocked_ioctl = hld_ioctl,
+	.owner = THIS_MODULE,
+};
+
+static struct miscdevice hld_device = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "hl_dev",
+	.fops = &hld_file_operations,
+};
+
+static int hld_init(void)
+{
+	int i;
+
+	printk("%s...\n", __func__);
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		hl_devices[i].allocated = 0;
+		hl_devices[i].initialized = 0;
+		hl_devices[i].code_loaded = 0;
+		hl_devices[i].code = 0;
+		hl_devices[i].data = 0;
+		hl_devices[i].hpi_resource = 0;
+		hl_devices[i].hpi = 0;
+	}
+	return misc_register(&hld_device);
+}
+
+static void hld_exit(void)
+{
+	int i;
+
+	for (i = 0; i < MAX_HL_DEVICES; i++) {
+		free_hl_dev_slot(&hl_devices[i]);
+	}
+
+	misc_deregister(&hld_device);
+}
+
+static int dw_hdmi2_hdcp2_clk_enable(struct device *dev)
+{
+	struct clk *pclk;
+	//struct clk *aclk;
+	struct clk *hdcp2_clk_hdmi;
+
+	pclk = devm_clk_get(dev, "pclk_hdcp2");
+	if (IS_ERR(pclk)) {
+		pr_err("Unable to get hdcp2 pclk\n");
+		return -1;
+	}
+	clk_prepare_enable(pclk);
+#if 0
+    aclk = devm_clk_get(dev, "aclk_hdcp2");
+    if (IS_ERR(aclk)) {
+        pr_err("Unable to get hdcp2 aclk\n");
+        return -1;
+    }
+    clk_prepare_enable(aclk);
+#endif
+	hdcp2_clk_hdmi = devm_clk_get(dev, "hdcp2_clk_hdmi");
+	if (IS_ERR(hdcp2_clk_hdmi)) {
+		pr_err("Unable to get hdcp2_clk_hdmi\n");
+		return -1;
+	}
+	clk_prepare_enable(hdcp2_clk_hdmi);
+
+	return 0;
+}
+
+static ssize_t hdcp2_show_enable(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", g_dw_hdcp2->enable);
+}
+
+static ssize_t hdcp2_store_enable(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t size)
+{
+	int enable;
+	if (kstrtoint(buf, 0, &enable))
+		return size;
+
+	if (g_dw_hdcp2->enable != enable) {
+		g_dw_hdcp2->enable = enable;
+		if (enable) {
+			dw_hdmi_hdcp2_start(3);
+		} else {
+			if (g_dw_hdcp2->hot_plug) {
+				g_dw_hdcp2->stop();
+				dw_hdmi_hdcp2_start(2);
+			}
+		}
+	}
+	return size;
+}
+
+static ssize_t hdcp2_show_status(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	if (g_dw_hdcp2->enable != 1) {
+		return snprintf(buf, PAGE_SIZE, "%s\n", "no enable hdcp2");
+	} else if (!g_dw_hdcp2->hot_plug) {
+		return snprintf(buf, PAGE_SIZE, "%s\n", "hdcp2 no auth");
+	} else {
+		if (g_dw_hdcp2->auth_sucess)
+			return snprintf(buf, PAGE_SIZE, "%s\n",
+					"hdcp2 auth sucess");
+		else
+			return snprintf(buf, PAGE_SIZE, "%s\n",
+					"no already auth sucess");
+	}
+}
+
+static DEVICE_ATTR(enable, 0644, hdcp2_show_enable, hdcp2_store_enable);
+static DEVICE_ATTR(status, 0444, hdcp2_show_status, NULL);
+
+static int create_device_node(void)
+{
+	int ret;
+
+	if (!g_dw_hdcp2)
+		return -1;
+	g_dw_hdcp2->mdev.minor = MISC_DYNAMIC_MINOR;
+	g_dw_hdcp2->mdev.name = "hdcp2_node";
+	g_dw_hdcp2->mdev.mode = 0666;
+	if (misc_register(&(g_dw_hdcp2->mdev))) {
+		pr_err("HDCP2: Could not add character driver\n");
+		return -1;
+	}
+
+	ret = device_create_file(g_dw_hdcp2->mdev.this_device,
+				 &dev_attr_enable);
+	if (ret) {
+		pr_err("HDCP: Could not add sys file enable\n");
+		ret = -EINVAL;
+		goto error0;
+	}
+
+	ret = device_create_file(g_dw_hdcp2->mdev.this_device,
+				 &dev_attr_status);
+	if (ret) {
+		pr_err("HDCP: Could not add sys file status\n");
+		ret = -EINVAL;
+		goto error1;
+	}
+
+	return 0;
+
+error1:
+	device_remove_file(g_dw_hdcp2->mdev.this_device, &dev_attr_enable);
+error0:
+	misc_deregister(&g_dw_hdcp2->mdev);
+	return ret;
+}
+
+static void end_device_node(void)
+{
+	if (g_dw_hdcp2)
+		misc_deregister(&(g_dw_hdcp2->mdev));
+}
+
+static int eswin_hdmi_hdcp2_probe(struct platform_device *pdev)
+{
+	struct device *hdcp2_dev = &pdev->dev;
+
+	printk("%s...\n", __func__);
+	g_dw_hdcp2 = kmalloc(sizeof(*g_dw_hdcp2), GFP_KERNEL);
+	if (!g_dw_hdcp2) {
+		printk("malloc g_dw_hdcp2 error\n");
+		return -ENOMEM;
+	}
+	memset(g_dw_hdcp2, 0, sizeof(*g_dw_hdcp2));
+
+	g_dw_hdcp2->dev = hdcp2_dev;
+	g_dw_hdcp2->stop = dw_hdcp2_stop;
+	g_dw_hdcp2->start = dw_hdcp2_start;
+	hld_init();
+	dw_hdmi2_hdcp2_clk_enable(hdcp2_dev);
+	dma_set_mask_and_coherent(hdcp2_dev, DMA_BIT_MASK(32));
+	dw_hdmi_hdcp2_init(g_dw_hdcp2);
+	dw_hdmi_hdcp2_start(3);
+
+	create_device_node();
+	return 0;
+}
+
+static int eswin_hdmi_hdcp2_remove(struct platform_device *pdev)
+{
+	printk("%s...\n", __func__);
+	dw_hdmi_hdcp2_remove();
+	end_device_node();
+	hld_exit();
+	kfree(g_dw_hdcp2);
+	g_dw_hdcp2 = NULL;
+
+	return 0;
+}
+
+static void eswin_hdmi_hdcp2_shutdown(struct platform_device *pdev)
+{
+	printk("%s...\n", __func__);
+}
+
+#if defined(CONFIG_OF)
+static const struct of_device_id dw_hdmi_hdcp2_dt_ids[] = {
+	{
+		.compatible = "eswin,dw-hdmi-hdcp2",
+	},
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, dw_hdmi_hdcp2_dt_ids);
+#endif
+
+struct platform_driver dw_hdmi_hdcp2_driver = {
+        .probe = eswin_hdmi_hdcp2_probe,
+        .remove = eswin_hdmi_hdcp2_remove,
+        .shutdown = eswin_hdmi_hdcp2_shutdown,
+        .driver = {
+        .name = "dw-hdmi-hdcp2",
+        .owner = THIS_MODULE,
+#if defined(CONFIG_OF)
+        .of_match_table = of_match_ptr(dw_hdmi_hdcp2_dt_ids),
+#endif
+    },
+};
+
+static int __init hdmi_hdcp2_init(void)
+{
+	printk("%s...\n", __func__);
+	return platform_driver_register(&dw_hdmi_hdcp2_driver);
+}
+
+static void hdmi_hdcp2_exit(void)
+{
+	platform_driver_unregister(&dw_hdmi_hdcp2_driver);
+}
+
+late_initcall_sync(hdmi_hdcp2_init);
+module_exit(hdmi_hdcp2_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Synopsys, Inc.");
+MODULE_DESCRIPTION("Linux Host Library Driver");
diff --git a/drivers/gpu/drm/eswin/dw_hdmi_i2s_audio.c b/drivers/gpu/drm/eswin/dw_hdmi_i2s_audio.c
new file mode 100644
index 000000000000..e211f8b9ab73
--- /dev/null
+++ b/drivers/gpu/drm/eswin/dw_hdmi_i2s_audio.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * dw-hdmi-i2s-audio.c
+ *
+ * Copyright (c) 2017 Renesas Solutions Corp.
+ * Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+
+#include <drm/bridge/dw_hdmi.h>
+#include <drm/drm_crtc.h>
+
+#include <sound/hdmi-codec.h>
+
+#include "dw-hdmi.h"
+#include "dw_hdmi_audio.h"
+
+#define DRIVER_NAME "dw-hdmi-i2s-audio"
+
+static inline void hdmi_write(struct dw_hdmi_i2s_audio_data *audio,
+			      u8 val, int offset)
+{
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	audio->write(hdmi, val, offset);
+}
+
+static inline u8 hdmi_read(struct dw_hdmi_i2s_audio_data *audio, int offset)
+{
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	return audio->read(hdmi, offset);
+}
+
+static int dw_hdmi_i2s_hw_params(struct device *dev, void *data,
+				 struct hdmi_codec_daifmt *fmt,
+				 struct hdmi_codec_params *hparms)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+	u8 conf0 = 0;
+	u8 conf1 = 0;
+	u8 inputclkfs = 0;
+
+	/* it cares I2S only */
+	if (fmt->bit_clk_provider | fmt->frame_clk_provider) {
+		dev_err(dev, "unsupported clock settings\n");
+		return -EINVAL;
+	}
+
+	/* Reset the FIFOs before applying new params */
+	hdmi_write(audio, HDMI_AUD_CONF0_SW_RESET, HDMI_AUD_CONF0);
+	hdmi_write(audio, (u8)~HDMI_MC_SWRSTZ_I2SSWRST_REQ, HDMI_MC_SWRSTZ);
+
+	inputclkfs	= HDMI_AUD_INPUTCLKFS_64FS;
+	conf0		= (HDMI_AUD_CONF0_I2S_SELECT | HDMI_AUD_CONF0_I2S_EN0);
+
+	/* Enable the required i2s lanes */
+	switch (hparms->channels) {
+	case 7 ... 8:
+		conf0 |= HDMI_AUD_CONF0_I2S_EN3;
+		fallthrough;
+	case 5 ... 6:
+		conf0 |= HDMI_AUD_CONF0_I2S_EN2;
+		fallthrough;
+	case 3 ... 4:
+		conf0 |= HDMI_AUD_CONF0_I2S_EN1;
+		/* Fall-thru */
+	}
+
+	switch (hparms->sample_width) {
+	case 16:
+		conf1 = HDMI_AUD_CONF1_WIDTH_16;
+		break;
+	case 24:
+	case 32:
+		conf1 = HDMI_AUD_CONF1_WIDTH_24;
+		break;
+	}
+
+	switch (fmt->fmt) {
+	case HDMI_I2S:
+		conf1 |= HDMI_AUD_CONF1_MODE_I2S;
+		break;
+	case HDMI_RIGHT_J:
+		conf1 |= HDMI_AUD_CONF1_MODE_RIGHT_J;
+		break;
+	case HDMI_LEFT_J:
+		conf1 |= HDMI_AUD_CONF1_MODE_LEFT_J;
+		break;
+	case HDMI_DSP_A:
+		conf1 |= HDMI_AUD_CONF1_MODE_BURST_1;
+		break;
+	case HDMI_DSP_B:
+		conf1 |= HDMI_AUD_CONF1_MODE_BURST_2;
+		break;
+	default:
+		dev_err(dev, "unsupported format\n");
+		return -EINVAL;
+	}
+
+	dw_hdmi_set_sample_rate(hdmi, hparms->sample_rate);
+	dw_hdmi_set_channel_status(hdmi, hparms->iec.status);
+	dw_hdmi_set_channel_count(hdmi, hparms->channels);
+	dw_hdmi_set_channel_allocation(hdmi, hparms->cea.channel_allocation);
+
+	hdmi_write(audio, inputclkfs, HDMI_AUD_INPUTCLKFS);
+	hdmi_write(audio, conf0, HDMI_AUD_CONF0);
+	hdmi_write(audio, conf1, HDMI_AUD_CONF1);
+
+	return 0;
+}
+
+static int dw_hdmi_i2s_audio_startup(struct device *dev, void *data)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	dw_hdmi_audio_enable(hdmi);
+
+	return 0;
+}
+
+static void dw_hdmi_i2s_audio_shutdown(struct device *dev, void *data)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	dw_hdmi_audio_disable(hdmi);
+}
+
+static int dw_hdmi_i2s_get_eld(struct device *dev, void *data, uint8_t *buf,
+			       size_t len)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	u8 *eld;
+
+	eld = audio->get_eld(audio->hdmi);
+	if (eld)
+		memcpy(buf, eld, min_t(size_t, MAX_ELD_BYTES, len));
+	else
+		/* Pass en empty ELD if connector not available */
+		memset(buf, 0, len);
+
+	return 0;
+}
+
+static int dw_hdmi_i2s_get_dai_id(struct snd_soc_component *component,
+				  struct device_node *endpoint)
+{
+	struct of_endpoint of_ep;
+	int ret;
+
+	ret = of_graph_parse_endpoint(endpoint, &of_ep);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * HDMI sound should be located as reg = <2>
+	 * Then, it is sound port 0
+	 */
+	if (of_ep.port == 2)
+		return 0;
+
+	return -EINVAL;
+}
+
+static int dw_hdmi_i2s_hook_plugged_cb(struct device *dev, void *data,
+				       hdmi_codec_plugged_cb fn,
+				       struct device *codec_dev)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	return dw_hdmi_set_plugged_cb(hdmi, fn, codec_dev);
+}
+
+static const struct hdmi_codec_ops dw_hdmi_i2s_ops = {
+	.hw_params	= dw_hdmi_i2s_hw_params,
+	.audio_startup  = dw_hdmi_i2s_audio_startup,
+	.audio_shutdown	= dw_hdmi_i2s_audio_shutdown,
+	.get_eld	= dw_hdmi_i2s_get_eld,
+	.get_dai_id	= dw_hdmi_i2s_get_dai_id,
+	.hook_plugged_cb = dw_hdmi_i2s_hook_plugged_cb,
+};
+
+static int snd_dw_hdmi_probe(struct platform_device *pdev)
+{
+	struct dw_hdmi_i2s_audio_data *audio = pdev->dev.platform_data;
+	struct platform_device_info pdevinfo;
+	struct hdmi_codec_pdata pdata;
+	struct platform_device *platform;
+
+	memset(&pdata, 0, sizeof(pdata));
+	pdata.ops		= &dw_hdmi_i2s_ops;
+	pdata.i2s		= 1;
+	pdata.max_i2s_channels	= 8;
+	pdata.data		= audio;
+
+	memset(&pdevinfo, 0, sizeof(pdevinfo));
+	pdevinfo.parent		= pdev->dev.parent;
+	pdevinfo.id		= PLATFORM_DEVID_AUTO;
+	pdevinfo.name		= HDMI_CODEC_DRV_NAME;
+	pdevinfo.data		= &pdata;
+	pdevinfo.size_data	= sizeof(pdata);
+	pdevinfo.dma_mask	= DMA_BIT_MASK(32);
+
+	platform = platform_device_register_full(&pdevinfo);
+	if (IS_ERR(platform))
+		return PTR_ERR(platform);
+
+	dev_set_drvdata(&pdev->dev, platform);
+
+	return 0;
+}
+
+static void snd_dw_hdmi_remove(struct platform_device *pdev)
+{
+	struct platform_device *platform = dev_get_drvdata(&pdev->dev);
+
+	platform_device_unregister(platform);
+}
+
+struct platform_driver snd_dw_hdmi_driver = {
+	.probe	= snd_dw_hdmi_probe,
+	.remove_new = snd_dw_hdmi_remove,
+	.driver	= {
+		.name = DRIVER_NAME,
+	},
+};
+//module_platform_driver(snd_dw_hdmi_driver);
+
+MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>");
+MODULE_DESCRIPTION("Synopsis Designware HDMI I2S ALSA SoC interface");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:" DRIVER_NAME);
diff --git a/drivers/gpu/drm/eswin/es_crtc.c b/drivers/gpu/drm/eswin/es_crtc.c
new file mode 100644
index 000000000000..96cbe4badf91
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_crtc.c
@@ -0,0 +1,436 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/clk.h>
+#include <linux/debugfs.h>
+#include <linux/media-bus-format.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_vblank.h>
+
+#include "es_drm.h"
+#include "es_crtc.h"
+
+void es_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	drm_crtc_cleanup(crtc);
+	kfree(es_crtc);
+}
+
+static void es_crtc_reset(struct drm_crtc *crtc)
+{
+	struct es_crtc_state *state;
+
+	if (crtc->state) {
+		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+
+		state = to_es_crtc_state(crtc->state);
+		kfree(state);
+		crtc->state = NULL;
+	}
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state == NULL)
+		return;
+
+	__drm_atomic_helper_crtc_reset(crtc, &state->base);
+
+	state->sync_mode = ES_SINGLE_DC;
+	state->output_fmt = MEDIA_BUS_FMT_RBG888_1X24;
+	state->encoder_type = DRM_MODE_ENCODER_NONE;
+#ifdef CONFIG_ESWIN_MMU
+	state->mmu_prefetch = ES_MMU_PREFETCH_DISABLE;
+#endif
+}
+
+static struct drm_crtc_state *
+es_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
+{
+	struct es_crtc_state *ori_state;
+	struct es_crtc_state *state;
+
+	if (WARN_ON(!crtc->state))
+		return NULL;
+
+	ori_state = to_es_crtc_state(crtc->state);
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &state->base);
+
+	state->sync_mode = ori_state->sync_mode;
+	state->output_fmt = ori_state->output_fmt;
+	state->encoder_type = ori_state->encoder_type;
+	state->bpp = ori_state->bpp;
+	state->dither_enable = ori_state->dither_enable;
+	state->underflow = ori_state->underflow;
+	state->bg_color = ori_state->bg_color;
+#ifdef CONFIG_ESWIN_MMU
+	state->mmu_prefetch = ori_state->mmu_prefetch;
+#endif
+
+	return &state->base;
+}
+
+static void es_crtc_atomic_destroy_state(struct drm_crtc *crtc,
+					 struct drm_crtc_state *state)
+{
+	__drm_atomic_helper_crtc_destroy_state(state);
+	kfree(to_es_crtc_state(state));
+}
+
+static int es_crtc_atomic_set_property(struct drm_crtc *crtc,
+				       struct drm_crtc_state *state,
+				       struct drm_property *property,
+				       uint64_t val)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(state);
+
+	if (property == es_crtc->sync_mode)
+		es_crtc_state->sync_mode = val;
+	else if (property == es_crtc->mmu_prefetch)
+		es_crtc_state->mmu_prefetch = val;
+	else if (property == es_crtc->dither)
+		es_crtc_state->dither_enable = val;
+	else if (property == es_crtc->bg_color)
+		es_crtc_state->bg_color = val;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int es_crtc_atomic_get_property(struct drm_crtc *crtc,
+				       const struct drm_crtc_state *state,
+				       struct drm_property *property,
+				       uint64_t *val)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	const struct es_crtc_state *es_crtc_state =
+		container_of(state, const struct es_crtc_state, base);
+
+	if (property == es_crtc->sync_mode)
+		*val = es_crtc_state->sync_mode;
+	else if (property == es_crtc->mmu_prefetch)
+		*val = es_crtc_state->mmu_prefetch;
+	else if (property == es_crtc->dither)
+		*val = es_crtc_state->dither_enable;
+	else if (property == es_crtc->bg_color)
+		*val = es_crtc_state->bg_color;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int es_crtc_debugfs_show(struct seq_file *s, void *data)
+{
+	struct drm_crtc *crtc = s->private;
+	struct es_crtc_state *crtc_state = to_es_crtc_state(crtc->state);
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+
+	seq_printf(s, "crtc[%u]: %s\n", crtc->base.id, crtc->name);
+	seq_printf(s, "\tactive = %d\n", crtc->state->active);
+	seq_printf(s, "\tsize = %dx%d\n", mode->hdisplay, mode->vdisplay);
+	seq_printf(s, "\tbpp = %u\n", crtc_state->bpp);
+	seq_printf(s, "\tunderflow = %d\n", crtc_state->underflow);
+
+	return 0;
+}
+
+static int es_crtc_debugfs_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, es_crtc_debugfs_show, inode->i_private);
+}
+
+static const struct file_operations es_crtc_debugfs_fops = {
+	.open = es_crtc_debugfs_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static int es_crtc_debugfs_init(struct drm_crtc *crtc)
+{
+	debugfs_create_file("status", 0444, crtc->debugfs_entry, crtc,
+			    &es_crtc_debugfs_fops);
+
+	return 0;
+}
+#else
+static int es_crtc_debugfs_init(struct drm_crtc *crtc)
+{
+	return 0;
+}
+#endif /* CONFIG_DEBUG_FS */
+
+static int es_crtc_late_register(struct drm_crtc *crtc)
+{
+	return es_crtc_debugfs_init(crtc);
+}
+
+static int es_crtc_enable_vblank(struct drm_crtc *crtc)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	es_crtc->funcs->enable_vblank(es_crtc->dev, true);
+
+	return 0;
+}
+
+static void es_crtc_disable_vblank(struct drm_crtc *crtc)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	es_crtc->funcs->enable_vblank(es_crtc->dev, false);
+}
+
+static const struct drm_crtc_funcs es_crtc_funcs = {
+	.set_config = drm_atomic_helper_set_config,
+	.destroy = es_crtc_destroy,
+	.page_flip = drm_atomic_helper_page_flip,
+	.reset = es_crtc_reset,
+	.atomic_duplicate_state = es_crtc_atomic_duplicate_state,
+	.atomic_destroy_state = es_crtc_atomic_destroy_state,
+	.atomic_set_property = es_crtc_atomic_set_property,
+	.atomic_get_property = es_crtc_atomic_get_property,
+	// .gamma_set      = drm_atomic_helper_legacy_gamma_set, /* TODO: */
+	.late_register = es_crtc_late_register,
+	.enable_vblank = es_crtc_enable_vblank,
+	.disable_vblank = es_crtc_disable_vblank,
+};
+
+static u8 cal_pixel_bits(u32 bus_format)
+{
+	u8 bpp;
+
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_RGB565_1X16:
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		bpp = 16;
+		break;
+	case MEDIA_BUS_FMT_RGB666_1X18:
+	case MEDIA_BUS_FMT_RGB666_1X24_CPADHI:
+		bpp = 18;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		bpp = 20;
+		break;
+	case MEDIA_BUS_FMT_BGR888_1X24:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+		bpp = 24;
+		break;
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		bpp = 30;
+		break;
+	default:
+		bpp = 24;
+		break;
+	}
+
+	return bpp;
+}
+
+static bool es_crtc_mode_fixup(struct drm_crtc *crtc,
+			       const struct drm_display_mode *mode,
+			       struct drm_display_mode *adjusted_mode)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	return es_crtc->funcs->mode_fixup(es_crtc->dev, mode, adjusted_mode);
+}
+
+static void es_crtc_atomic_enable(struct drm_crtc *crtc,
+				  struct drm_atomic_state *old_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc->state);
+
+	es_crtc_state->bpp = cal_pixel_bits(es_crtc_state->output_fmt);
+
+	es_crtc->funcs->enable(es_crtc->dev, crtc);
+
+	drm_crtc_vblank_on(crtc);
+}
+
+static void es_crtc_atomic_disable(struct drm_crtc *crtc,
+				   struct drm_atomic_state *old_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	drm_crtc_vblank_off(crtc);
+
+	es_crtc->funcs->disable(es_crtc->dev);
+
+	if (crtc->state->event && !crtc->state->active) {
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		spin_unlock_irq(&crtc->dev->event_lock);
+
+		crtc->state->event = NULL;
+	}
+}
+
+static void es_crtc_atomic_begin(struct drm_crtc *crtc,
+				 struct drm_atomic_state *old_crtc_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct device *dev = es_crtc->dev;
+	struct drm_property_blob *blob = crtc->state->gamma_lut;
+	struct drm_color_lut *lut;
+
+	if (crtc->state->color_mgmt_changed) {
+		if ((blob) && (blob->length)) {
+			lut = blob->data;
+			es_crtc->funcs->set_gamma(dev, lut,
+						  blob->length / sizeof(*lut));
+			es_crtc->funcs->enable_gamma(dev, true);
+		} else {
+			es_crtc->funcs->enable_gamma(dev, false);
+		}
+	}
+}
+
+static void es_crtc_atomic_flush(struct drm_crtc *crtc,
+				 struct drm_atomic_state *old_crtc_state)
+{
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+	struct drm_pending_vblank_event *event = crtc->state->event;
+
+	es_crtc->funcs->commit(es_crtc->dev);
+
+	if (event) {
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_arm_vblank_event(crtc, event);
+		spin_unlock_irq(&crtc->dev->event_lock);
+		crtc->state->event = NULL;
+	}
+}
+
+static const struct drm_crtc_helper_funcs es_crtc_helper_funcs = {
+	.mode_fixup = es_crtc_mode_fixup,
+	.atomic_enable = es_crtc_atomic_enable,
+	.atomic_disable = es_crtc_atomic_disable,
+	.atomic_begin = es_crtc_atomic_begin,
+	.atomic_flush = es_crtc_atomic_flush,
+};
+
+static const struct drm_prop_enum_list es_sync_mode_enum_list[] = {
+	{ ES_SINGLE_DC, "single dc mode" },
+	{ ES_MULTI_DC_PRIMARY, "primary dc for multi dc mode" },
+	{ ES_MULTI_DC_SECONDARY, "secondary dc for multi dc mode" },
+};
+
+#ifdef CONFIG_ESWIN_MMU
+static const struct drm_prop_enum_list es_mmu_prefetch_enum_list[] = {
+	{ ES_MMU_PREFETCH_DISABLE, "disable mmu prefetch" },
+	{ ES_MMU_PREFETCH_ENABLE, "enable mmu prefetch" },
+};
+#endif
+
+struct es_crtc *es_crtc_create(struct drm_device *drm_dev,
+			       struct es_dc_info *info)
+{
+	struct es_crtc *crtc;
+	int ret;
+
+	if (!info)
+		return NULL;
+
+	crtc = kzalloc(sizeof(struct es_crtc), GFP_KERNEL);
+	if (!crtc)
+		return NULL;
+
+	ret = drm_crtc_init_with_planes(drm_dev, &crtc->base, NULL, NULL,
+					&es_crtc_funcs,
+					info->name ? info->name : NULL);
+	if (ret)
+		goto err_free_crtc;
+
+	drm_crtc_helper_add(&crtc->base, &es_crtc_helper_funcs);
+
+	/* Set up the crtc properties */
+	if (info->pipe_sync) {
+		crtc->sync_mode = drm_property_create_enum(
+			drm_dev, 0, "SYNC_MODE", es_sync_mode_enum_list,
+			ARRAY_SIZE(es_sync_mode_enum_list));
+
+		if (!crtc->sync_mode)
+			goto err_cleanup_crts;
+
+		drm_object_attach_property(&crtc->base.base, crtc->sync_mode,
+					   ES_SINGLE_DC);
+	}
+
+	if (info->gamma_size) {
+		ret = drm_mode_crtc_set_gamma_size(&crtc->base,
+						   info->gamma_size);
+		if (ret)
+			goto err_cleanup_crts;
+
+		drm_crtc_enable_color_mgmt(&crtc->base, 0, false,
+					   info->gamma_size);
+	}
+
+	if (info->background) {
+		crtc->bg_color = drm_property_create_range(
+			drm_dev, 0, "BG_COLOR", 0, 0xffffffff);
+
+		if (!crtc->bg_color)
+			goto err_cleanup_crts;
+
+		drm_object_attach_property(&crtc->base.base, crtc->bg_color, 0);
+	}
+
+	crtc->dither = drm_property_create_bool(drm_dev, 0, "DITHER_ENABLED");
+	if (!crtc->dither)
+		goto err_cleanup_crts;
+
+	drm_object_attach_property(&crtc->base.base, crtc->dither, 0);
+
+#ifdef CONFIG_ESWIN_MMU
+	if (info->mmu_prefetch) {
+		crtc->mmu_prefetch = drm_property_create_enum(
+			drm_dev, 0, "MMU_PREFETCH", es_mmu_prefetch_enum_list,
+			ARRAY_SIZE(es_mmu_prefetch_enum_list));
+		if (!crtc->mmu_prefetch)
+			goto err_cleanup_crts;
+
+		drm_object_attach_property(&crtc->base.base, crtc->mmu_prefetch,
+					   ES_MMU_PREFETCH_DISABLE);
+	}
+#endif
+
+	crtc->max_bpc = info->max_bpc;
+	crtc->color_formats = info->color_formats;
+	return crtc;
+
+err_cleanup_crts:
+	drm_crtc_cleanup(&crtc->base);
+
+err_free_crtc:
+	kfree(crtc);
+	return NULL;
+}
+
+void es_crtc_handle_vblank(struct drm_crtc *crtc, bool underflow)
+{
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc->state);
+
+	drm_crtc_handle_vblank(crtc);
+
+	es_crtc_state->underflow = underflow;
+}
diff --git a/drivers/gpu/drm/eswin/es_crtc.h b/drivers/gpu/drm/eswin/es_crtc.h
new file mode 100644
index 000000000000..5f7cf9009d4f
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_crtc.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_CRTC_H__
+#define __ES_CRTC_H__
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "es_type.h"
+
+struct es_crtc_funcs {
+	void (*enable)(struct device *dev, struct drm_crtc *crtc);
+	void (*disable)(struct device *dev);
+	bool (*mode_fixup)(struct device *dev,
+			   const struct drm_display_mode *mode,
+			   struct drm_display_mode *adjusted_mode);
+	void (*set_gamma)(struct device *dev, struct drm_color_lut *lut,
+			  unsigned int size);
+	void (*enable_gamma)(struct device *dev, bool enable);
+	void (*enable_vblank)(struct device *dev, bool enable);
+	void (*commit)(struct device *dev);
+};
+
+struct es_crtc_state {
+	struct drm_crtc_state base;
+
+	u32 sync_mode;
+	u32 output_fmt;
+	u32 bg_color;
+	u8 encoder_type;
+	u8 mmu_prefetch;
+	u8 bpp;
+	bool dither_enable;
+	bool underflow;
+};
+
+struct es_crtc {
+	struct drm_crtc base;
+	struct device *dev;
+	struct drm_pending_vblank_event *event;
+	unsigned int max_bpc;
+	unsigned int color_formats; /* supported color format */
+
+	struct drm_property *sync_mode;
+	struct drm_property *mmu_prefetch;
+	struct drm_property *dither;
+	struct drm_property *bg_color;
+
+	const struct es_crtc_funcs *funcs;
+};
+
+void es_crtc_destroy(struct drm_crtc *crtc);
+
+struct es_crtc *es_crtc_create(struct drm_device *drm_dev,
+			       struct es_dc_info *info);
+void es_crtc_handle_vblank(struct drm_crtc *crtc, bool underflow);
+
+static inline struct es_crtc *to_es_crtc(struct drm_crtc *crtc)
+{
+	return container_of(crtc, struct es_crtc, base);
+}
+
+static inline struct es_crtc_state *
+to_es_crtc_state(struct drm_crtc_state *state)
+{
+	return container_of(state, struct es_crtc_state, base);
+}
+#endif /* __ES_CRTC_H__ */
diff --git a/drivers/gpu/drm/eswin/es_dc.c b/drivers/gpu/drm/eswin/es_dc.c
new file mode 100644
index 000000000000..ac43c604a2dc
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc.c
@@ -0,0 +1,1160 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/component.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/reset.h>
+#include <linux/delay.h>
+#include <linux/media-bus-format.h>
+
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_blend.h>
+
+#include "es_drm.h"
+#include "es_type.h"
+#include "es_dc_hw.h"
+#include "es_dc.h"
+#include "es_crtc.h"
+#include "es_drv.h"
+
+#define VO_ACLK_HIGHEST 800000000
+
+static inline void update_format(u32 format, struct dc_hw_fb *fb)
+{
+	u8 f;
+
+	switch (format) {
+	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_XBGR4444:
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_BGRX4444:
+		f = FORMAT_X4R4G4B4;
+		break;
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_BGRA4444:
+		f = FORMAT_A4R4G4B4;
+		break;
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_XBGR1555:
+	case DRM_FORMAT_RGBX5551:
+	case DRM_FORMAT_BGRX5551:
+		f = FORMAT_X1R5G5B5;
+		break;
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_ABGR1555:
+	case DRM_FORMAT_BGRA5551:
+		f = FORMAT_A1R5G5B5;
+		break;
+	case DRM_FORMAT_BGR565:
+	case DRM_FORMAT_RGB565:
+		f = FORMAT_R5G6B5;
+		break;
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_BGRX8888:
+		f = FORMAT_X8R8G8B8;
+		break;
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_BGRA8888:
+		f = FORMAT_A8R8G8B8;
+		break;
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+		f = FORMAT_YUY2;
+		break;
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+		f = FORMAT_UYVY;
+		break;
+	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_YUV420:
+		f = FORMAT_YV12;
+		break;
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+		f = FORMAT_NV12;
+		break;
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV61:
+		f = FORMAT_NV16;
+		break;
+	case DRM_FORMAT_P010:
+		f = FORMAT_P010;
+		break;
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_BGRA1010102:
+		f = FORMAT_A2R10G10B10;
+		break;
+	default:
+		f = FORMAT_A8R8G8B8;
+		break;
+	}
+	fb->format = f;
+}
+
+static inline void update_swizzle(u32 format, struct dc_hw_fb *fb)
+{
+	fb->swizzle = SWIZZLE_ARGB;
+	fb->uv_swizzle = 0;
+
+	switch (format) {
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_RGBX5551:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_RGBA1010102:
+		fb->swizzle = SWIZZLE_RGBA;
+		break;
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_XBGR4444:
+	case DRM_FORMAT_XBGR1555:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR1555:
+	case DRM_FORMAT_BGR565:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_ABGR2101010:
+		fb->swizzle = SWIZZLE_ABGR;
+		break;
+	case DRM_FORMAT_BGRA4444:
+	case DRM_FORMAT_BGRX4444:
+	case DRM_FORMAT_BGRX5551:
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_BGRA5551:
+	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_BGRA1010102:
+		fb->swizzle = SWIZZLE_BGRA;
+		break;
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV61:
+		fb->uv_swizzle = 1;
+		break;
+	default:
+		break;
+	}
+}
+
+static inline u8 to_es_rotation(unsigned int rotation)
+{
+	u8 rot;
+
+	switch (rotation & DRM_MODE_REFLECT_MASK) {
+	case DRM_MODE_REFLECT_X:
+		rot = FLIP_X;
+		return rot;
+	case DRM_MODE_REFLECT_Y:
+		rot = FLIP_Y;
+		return rot;
+	case DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y:
+		rot = FLIP_XY;
+		return rot;
+	default:
+		break;
+	}
+
+	switch (rotation & DRM_MODE_ROTATE_MASK) {
+	case DRM_MODE_ROTATE_0:
+		rot = ROT_0;
+		break;
+	case DRM_MODE_ROTATE_90:
+		rot = ROT_90;
+		break;
+	case DRM_MODE_ROTATE_180:
+		rot = ROT_180;
+		break;
+	case DRM_MODE_ROTATE_270:
+		rot = ROT_270;
+		break;
+	default:
+		rot = ROT_0;
+		break;
+	}
+
+	return rot;
+}
+
+static inline u8 to_es_yuv_color_space(u32 color_space)
+{
+	u8 cs;
+
+	switch (color_space) {
+	case DRM_COLOR_YCBCR_BT601:
+		cs = COLOR_SPACE_601;
+		break;
+	case DRM_COLOR_YCBCR_BT709:
+		cs = COLOR_SPACE_709;
+		break;
+	case DRM_COLOR_YCBCR_BT2020:
+		cs = COLOR_SPACE_2020;
+		break;
+	default:
+		cs = COLOR_SPACE_601;
+		break;
+	}
+
+	return cs;
+}
+
+static inline u8 to_es_tile_mode(u64 modifier)
+{
+	// DRM_FORMAT_MOD_ES_NORM_MODE_MASK 0x1F
+	return (u8)(modifier & 0x1F);
+}
+
+static int es_dc_clk_configs(struct device *dev, bool enable)
+{
+	int ret = 0;
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	if (enable) {
+		if (dc->dc_clkon) {
+			return 0;
+		}
+
+		ret = clk_prepare_enable(dc->cfg_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to prepare/enable cfg_clk\n");
+			return ret;
+		}
+
+		ret = clk_prepare_enable(dc->pix_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to prepare/enable pix_clk\n");
+			return ret;
+		}
+
+		ret = clk_prepare_enable(dc->axi_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to prepare/enable axi_clk\n");
+			return ret;
+		}
+
+		dc->pix_clk_rate = clk_get_rate(dc->pix_clk);
+		dc->dc_clkon = true;
+	} else {
+		if (!dc->dc_clkon) {
+			return 0;
+		}
+		clk_disable_unprepare(dc->pix_clk);
+		clk_disable_unprepare(dc->axi_clk);
+		clk_disable_unprepare(dc->cfg_clk);
+		dc->dc_clkon = false;
+	}
+
+	return 0;
+}
+
+static void dc_deinit(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_interrupt(&dc->hw, 0);
+	dc_hw_deinit(&dc->hw);
+	es_dc_clk_configs(dev, false);
+}
+
+static int dc_init(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	int ret;
+	long rate;
+
+	dc->first_frame = true;
+
+	ret = clk_set_parent(dc->vo_mux, dc->spll0_fout1);
+	if (ret < 0) {
+		pr_err("DC: failed to set core clk parent: %d\n", ret);
+		return ret;
+	}
+
+	rate = clk_round_rate(dc->axi_clk, VO_ACLK_HIGHEST);
+	if (rate > 0) {
+		ret = clk_set_rate(dc->axi_clk, rate);
+		if (ret) {
+			pr_err("DC: failed to set axi clk: %d\n", ret);
+			return ret;
+		}
+	}
+
+	ret = es_dc_clk_configs(dev, true);
+	if (ret < 0) {
+		dev_err(dev, "failed to prepare/enable clks\n");
+		return ret;
+	}
+
+	ret = dc_hw_init(&dc->hw);
+	if (ret) {
+		dev_err(dev, "failed to init DC HW\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void es_dc_dump_enable(struct device *dev, dma_addr_t addr,
+			      unsigned int pitch)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_dump(&dc->hw, addr, pitch);
+}
+
+static void es_dc_dump_disable(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_disable_dump(&dc->hw);
+}
+
+static void es_dc_enable(struct device *dev, struct drm_crtc *crtc)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct es_crtc_state *crtc_state = to_es_crtc_state(crtc->state);
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	struct dc_hw_display display;
+
+	display.bus_format = crtc_state->output_fmt;
+	display.h_active = mode->hdisplay;
+	display.h_total = mode->htotal;
+	display.h_sync_start = mode->hsync_start;
+	display.h_sync_end = mode->hsync_end;
+	if (mode->flags & DRM_MODE_FLAG_PHSYNC)
+		display.h_sync_polarity = true;
+	else
+		display.h_sync_polarity = false;
+
+	display.v_active = mode->vdisplay;
+	display.v_total = mode->vtotal;
+	display.v_sync_start = mode->vsync_start;
+	display.v_sync_end = mode->vsync_end;
+	if (mode->flags & DRM_MODE_FLAG_PVSYNC)
+		display.v_sync_polarity = true;
+	else
+		display.v_sync_polarity = false;
+
+	display.sync_mode = crtc_state->sync_mode;
+	display.bg_color = crtc_state->bg_color;
+	display.dither_enable = crtc_state->dither_enable;
+
+	display.enable = true;
+	es_dc_clk_configs(dev, true);
+
+	if (dc->pix_clk_rate != mode->clock) {
+		clk_set_rate(dc->pix_clk, mode->clock * 1000);
+		dc->pix_clk_rate = mode->clock;
+	}
+
+	if (crtc_state->encoder_type == DRM_MODE_ENCODER_DSI ||
+	    crtc_state->encoder_type == DRM_MODE_ENCODER_VIRTUAL ||
+	    crtc_state->encoder_type == DRM_MODE_ENCODER_NONE)
+		dc_hw_set_out(&dc->hw, OUT_DPI);
+	else
+		dc_hw_set_out(&dc->hw, OUT_DP);
+
+#ifdef CONFIG_ESWIN_MMU
+	if (crtc_state->mmu_prefetch == ES_MMU_PREFETCH_ENABLE)
+		dc_hw_enable_mmu_prefetch(&dc->hw, true);
+	else
+		dc_hw_enable_mmu_prefetch(&dc->hw, false);
+#endif
+
+	dc_hw_setup_display(&dc->hw, &display);
+}
+
+static void es_dc_disable(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct dc_hw_display display;
+
+	display.enable = false;
+
+	dc_hw_setup_display(&dc->hw, &display);
+	es_dc_clk_configs(dev, false);
+}
+
+static bool es_dc_mode_fixup(struct device *dev,
+			     const struct drm_display_mode *mode,
+			     struct drm_display_mode *adjusted_mode)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	long clk_rate;
+
+	if (dc->pix_clk) {
+		clk_rate = clk_round_rate(dc->pix_clk,
+					  adjusted_mode->clock * 1000);
+		adjusted_mode->clock = DIV_ROUND_UP(clk_rate, 1000);
+	}
+
+	return true;
+}
+
+static void es_dc_set_gamma(struct device *dev, struct drm_color_lut *lut,
+			    unsigned int size)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	u16 i, r, g, b;
+	u8 bits;
+
+	if (size != dc->hw.info->gamma_size) {
+		dev_err(dev, "gamma size does not match!\n");
+		return;
+	}
+
+	bits = dc->hw.info->gamma_bits;
+	for (i = 0; i < size; i++) {
+		r = drm_color_lut_extract(lut[i].red, bits);
+		g = drm_color_lut_extract(lut[i].green, bits);
+		b = drm_color_lut_extract(lut[i].blue, bits);
+		dc_hw_update_gamma(&dc->hw, i, r, g, b);
+	}
+}
+
+static void es_dc_enable_gamma(struct device *dev, bool enable)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_gamma(&dc->hw, enable);
+}
+
+static void es_dc_enable_vblank(struct device *dev, bool enable)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	dc_hw_enable_interrupt(&dc->hw, enable);
+}
+
+static u32 calc_factor(u32 src, u32 dest)
+{
+	u32 factor = 1 << 16;
+
+	if ((src > 1) && (dest > 1))
+		factor = ((src - 1) << 16) / (dest - 1);
+
+	return factor;
+}
+
+static void update_scale(struct dc_hw_scale *scale, int width, int height,
+			 int dst_w, int dst_h, unsigned int rotation,
+			 struct dc_hw_roi *roi)
+{
+	int src_w, src_h, temp;
+
+	scale->enable = false;
+	if (roi->enable) {
+		src_w = roi->width;
+		src_h = roi->height;
+	} else {
+		src_w = width;
+		src_h = height;
+	}
+
+	if (drm_rotation_90_or_270(rotation)) {
+		temp = src_w;
+		src_w = src_h;
+		src_h = temp;
+	}
+
+	if (src_w != dst_w) {
+		scale->scale_factor_x = calc_factor(src_w, dst_w);
+		scale->enable = true;
+	} else {
+		scale->scale_factor_x = 1 << 16;
+	}
+	if (src_h != dst_h) {
+		scale->scale_factor_y = calc_factor(src_h, dst_h);
+		scale->enable = true;
+	} else {
+		scale->scale_factor_y = 1 << 16;
+	}
+}
+
+static void update_fb(struct es_plane *plane, struct dc_hw_fb *fb)
+{
+	struct drm_plane_state *state = plane->base.state;
+	struct drm_framebuffer *drm_fb = state->fb;
+	struct drm_rect *src = &state->src;
+
+	fb->y_address = plane->dma_addr[0];
+	fb->y_stride = drm_fb->pitches[0];
+	if (drm_fb->format->format == DRM_FORMAT_YVU420) {
+		fb->u_address = plane->dma_addr[2];
+		fb->v_address = plane->dma_addr[1];
+		fb->u_stride = drm_fb->pitches[2];
+		fb->v_stride = drm_fb->pitches[1];
+	} else {
+		fb->u_address = plane->dma_addr[1];
+		fb->v_address = plane->dma_addr[2];
+		fb->u_stride = drm_fb->pitches[1];
+		fb->v_stride = drm_fb->pitches[2];
+	}
+	fb->width = drm_rect_width(src) >> 16;
+	fb->height = drm_rect_height(src) >> 16;
+	fb->tile_mode = to_es_tile_mode(drm_fb->modifier);
+	fb->rotation = to_es_rotation(state->rotation);
+	fb->yuv_color_space = to_es_yuv_color_space(state->color_encoding);
+	fb->enable = state->visible;
+	update_format(drm_fb->format->format, fb);
+	update_swizzle(drm_fb->format->format, fb);
+}
+
+static void update_degamma(struct es_dc *dc, struct es_plane *plane,
+			   struct es_plane_state *plane_state)
+{
+	dc_hw_update_degamma(&dc->hw, plane->id, plane_state->degamma);
+	plane_state->degamma_changed = false;
+}
+
+void update_roi(struct es_dc *dc, enum dc_hw_plane_id id,
+		struct es_plane_state *plane_state, struct dc_hw_roi *roi,
+		struct dc_hw_fb *fb)
+{
+	struct drm_es_roi *data;
+	u16 src_w = fb->width;
+	u16 src_h = fb->height;
+
+	if (plane_state->roi) {
+		data = (struct drm_es_roi *)plane_state->roi->data;
+		if (data->enable) {
+			roi->x = data->roi_x;
+			roi->y = data->roi_y;
+			roi->width = (data->roi_w + data->roi_x > src_w) ?
+					     (src_w - data->roi_x) :
+					     data->roi_w;
+			roi->height = (data->roi_h + data->roi_y > src_h) ?
+					      (src_h - data->roi_y) :
+					      data->roi_h;
+			roi->enable = true;
+		} else {
+			roi->enable = false;
+			roi->width = src_w;
+			roi->height = src_h;
+		}
+
+		dc_hw_update_roi(&dc->hw, id, roi);
+	} else {
+		roi->enable = false;
+	}
+}
+
+static void update_color_mgmt(struct es_dc *dc, u8 id, struct dc_hw_fb *fb,
+			      struct es_plane_state *plane_state)
+{
+	struct drm_es_color_mgmt *data;
+	struct dc_hw_colorkey colorkey;
+
+	if (plane_state->color_mgmt) {
+		data = plane_state->color_mgmt->data;
+
+		fb->clear_enable = data->clear_enable;
+		fb->clear_value = data->clear_value;
+
+		if (data->colorkey > data->colorkey_high)
+			data->colorkey = data->colorkey_high;
+
+		colorkey.colorkey = data->colorkey;
+		colorkey.colorkey_high = data->colorkey_high;
+		colorkey.transparency = (data->transparency) ?
+						DC_TRANSPARENCY_KEY :
+						DC_TRANSPARENCY_OPAQUE;
+		dc_hw_update_colorkey(&dc->hw, id, &colorkey);
+	}
+}
+
+static void update_primary_plane(struct es_dc *dc, struct es_plane *plane)
+{
+	struct dc_hw_fb fb = { 0 };
+	struct dc_hw_scale scale;
+	struct drm_plane_state *state = plane->base.state;
+	struct es_plane_state *plane_state = to_es_plane_state(state);
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	struct dc_hw_roi roi;
+
+	update_fb(plane, &fb);
+
+	update_roi(dc, plane->id, plane_state, &roi, &fb);
+
+	update_scale(&scale, fb.width, fb.height, mode->hdisplay,
+		     mode->vdisplay, state->rotation, &roi);
+
+	if (plane_state->degamma_changed)
+		update_degamma(dc, plane, plane_state);
+
+	update_color_mgmt(dc, plane->id, &fb, plane_state);
+
+	dc_hw_update_plane(&dc->hw, plane->id, &fb, &scale);
+}
+
+static void update_overlay_plane(struct es_dc *dc, struct es_plane *plane)
+{
+	struct dc_hw_fb fb = { 0 };
+	struct dc_hw_scale scale;
+	struct dc_hw_position pos;
+	struct dc_hw_blend blend;
+	struct drm_plane_state *state = plane->base.state;
+	struct es_plane_state *plane_state = to_es_plane_state(state);
+	struct drm_rect *dest = &state->dst;
+	struct dc_hw_roi roi;
+
+	update_fb(plane, &fb);
+	update_roi(dc, plane->id, plane_state, &roi, &fb);
+	update_scale(&scale, fb.width, fb.height, drm_rect_width(dest),
+		     drm_rect_height(dest), state->rotation, &roi);
+
+	if (plane_state->degamma_changed)
+		update_degamma(dc, plane, plane_state);
+
+	update_color_mgmt(dc, plane->id, &fb, plane_state);
+
+	dc_hw_update_plane(&dc->hw, plane->id, &fb, &scale);
+
+	pos.start_x = dest->x1;
+	pos.start_y = dest->y1;
+	pos.end_x = dest->x2;
+	pos.end_y = dest->y2;
+	dc_hw_set_position(&dc->hw, &pos);
+
+	blend.alpha = (u8)(state->alpha >> 8);
+	blend.blend_mode = (u8)(state->pixel_blend_mode);
+	dc_hw_set_blend(&dc->hw, &blend);
+}
+
+static void update_cursor_size(struct drm_plane_state *state, struct dc_hw_cursor *cursor)
+{
+	u8 size_type;
+
+	switch (state->crtc_w) {
+		case 32:
+			size_type = CURSOR_SIZE_32X32;
+			break;
+		case 64:
+			size_type = CURSOR_SIZE_64X64;
+			break;
+		case 128:
+			size_type = CURSOR_SIZE_128X128;
+			break;
+		case 256:
+			size_type = CURSOR_SIZE_256X256;
+			break;
+		default:
+			size_type = CURSOR_SIZE_32X32;
+			break;
+	}
+
+	cursor->size = size_type;
+}
+
+static void update_cursor_plane(struct es_dc *dc, struct es_plane *plane)
+{
+	struct drm_plane_state *state = plane->base.state;
+	struct dc_hw_cursor cursor;
+
+	cursor.address = plane->dma_addr[0];
+
+	if (state->crtc_x > 0) {
+		cursor.x = state->crtc_x;
+		cursor.hot_x = 0;
+	} else {
+		cursor.hot_x = -state->crtc_x;
+		cursor.x = 0;
+	}
+	if (state->crtc_y > 0) {
+		cursor.y = state->crtc_y;
+		cursor.hot_y = 0;
+	} else {
+		cursor.hot_y = -state->crtc_y;
+		cursor.y = 0;
+	}
+
+	update_cursor_size(state, &cursor);
+	cursor.enable = true;
+
+	dc_hw_update_cursor(&dc->hw, &cursor);
+}
+
+static void es_dc_update_plane(struct device *dev, struct es_plane *plane)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	enum drm_plane_type type = plane->base.type;
+
+	switch (type) {
+	case DRM_PLANE_TYPE_PRIMARY:
+		update_primary_plane(dc, plane);
+		break;
+	case DRM_PLANE_TYPE_OVERLAY:
+		update_overlay_plane(dc, plane);
+		break;
+	case DRM_PLANE_TYPE_CURSOR:
+		update_cursor_plane(dc, plane);
+		break;
+	default:
+		break;
+	}
+}
+
+static void es_dc_disable_plane(struct device *dev, struct es_plane *plane)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	enum drm_plane_type type = plane->base.type;
+	struct dc_hw_fb fb = { 0 };
+	struct dc_hw_cursor cursor = { 0 };
+
+	switch (type) {
+	case DRM_PLANE_TYPE_PRIMARY:
+	case DRM_PLANE_TYPE_OVERLAY:
+		fb.enable = false;
+		dc_hw_update_plane(&dc->hw, plane->id, &fb, NULL);
+		break;
+	case DRM_PLANE_TYPE_CURSOR:
+		cursor.enable = false;
+		dc_hw_update_cursor(&dc->hw, &cursor);
+		break;
+	default:
+		break;
+	}
+}
+
+static bool es_dc_mod_supported(const struct es_plane_info *plane_info,
+				u64 modifier)
+{
+	const u64 *mods;
+
+	if (plane_info->modifiers == NULL)
+		return false;
+
+	for (mods = plane_info->modifiers; *mods != DRM_FORMAT_MOD_INVALID;
+	     mods++) {
+		if (*mods == modifier)
+			return true;
+	}
+
+	return false;
+}
+
+static int es_dc_check_plane(struct device *dev, struct es_plane *plane,
+			     struct drm_plane_state *state)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct drm_framebuffer *fb = state->fb;
+	const struct es_plane_info *plane_info;
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_crtc_state *crtc_state;
+
+	plane_info = &dc->hw.info->planes[plane->id];
+	if (plane_info == NULL)
+		return -EINVAL;
+
+	if (fb->width < plane_info->min_width ||
+	    fb->width > plane_info->max_width ||
+	    fb->height < plane_info->min_height ||
+	    fb->height > plane_info->max_height)
+		dev_err_once(dev, "buffer size may not support on plane%d.\n",
+			     plane->id);
+
+	if ((plane->base.type != DRM_PLANE_TYPE_CURSOR) &&
+	    (!es_dc_mod_supported(plane_info, fb->modifier))) {
+		dev_err(dev, "unsupported modifier on plane%d.\n", plane->id);
+		return -EINVAL;
+	}
+
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc);
+	if (IS_ERR(crtc_state))
+		return -EINVAL;
+
+	return drm_atomic_helper_check_plane_state(state, crtc_state,
+						   plane_info->min_scale,
+						   plane_info->max_scale, true,
+						   true);
+}
+
+static irqreturn_t dc_isr(int irq, void *data)
+{
+	struct es_dc *dc = data;
+
+	dc_hw_get_interrupt(&dc->hw);
+
+	if (!dc->dc_initialized) {
+		return IRQ_HANDLED;
+	}
+
+	es_crtc_handle_vblank(&dc->crtc->base, dc_hw_check_underflow(&dc->hw));
+
+	return IRQ_HANDLED;
+}
+
+static void es_dc_commit(struct device *dev)
+{
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+#ifdef CONFIG_ESWIN_MMU
+	dc_mmu_flush(&dc->hw);
+#endif
+
+	if (!dc->first_frame) {
+		if (dc_hw_flip_in_progress(&dc->hw))
+			udelay(100);
+
+		dc_hw_enable_shadow_register(&dc->hw, false);
+	}
+
+	dc_hw_commit(&dc->hw);
+
+	if (dc->first_frame)
+		dc->first_frame = false;
+
+	if (!dc->dc_initialized)
+		dc->dc_initialized = true;
+
+	dc_hw_enable_shadow_register(&dc->hw, true);
+}
+
+static const struct es_crtc_funcs dc_crtc_funcs = {
+	.enable = es_dc_enable,
+	.disable = es_dc_disable,
+	.mode_fixup = es_dc_mode_fixup,
+	.set_gamma = es_dc_set_gamma,
+	.enable_gamma = es_dc_enable_gamma,
+	.enable_vblank = es_dc_enable_vblank,
+	.commit = es_dc_commit,
+};
+
+static const struct es_plane_funcs dc_plane_funcs = {
+	.update = es_dc_update_plane,
+	.disable = es_dc_disable_plane,
+	.check = es_dc_check_plane,
+};
+
+static const struct es_dc_funcs dc_funcs = {
+	.dump_enable = es_dc_dump_enable,
+	.dump_disable = es_dc_dump_disable,
+};
+
+static int dc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+#ifdef CONFIG_ESWIN_MMU
+	struct es_drm_private *priv = drm_dev->dev_private;
+#endif
+	struct es_dc *dc = dev_get_drvdata(dev);
+	struct device_node *port;
+	struct es_crtc *crtc;
+	struct es_dc_info *dc_info;
+	struct es_plane *plane;
+	struct drm_plane *drm_plane, *tmp;
+	struct es_plane_info *plane_info;
+	int i, ret;
+
+	if (!drm_dev || !dc) {
+		dev_err(dev, "devices are not created.\n");
+		return -ENODEV;
+	}
+
+	ret = dc_init(dev);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize DC hardware.\n");
+		return ret;
+	}
+
+#ifdef CONFIG_ESWIN_MMU
+	ret = dc_mmu_construct(priv->dma_dev, &priv->mmu);
+	if (ret) {
+		dev_err(dev, "failed to construct DC MMU\n");
+		goto err_clean_dc;
+	}
+
+	ret = dc_hw_mmu_init(&dc->hw, priv->mmu);
+	if (ret) {
+		dev_err(dev, "failed to init DC MMU\n");
+		goto err_clean_dc;
+	}
+#endif
+
+	ret = es_drm_iommu_attach_device(drm_dev, dev);
+	if (ret < 0) {
+		dev_err(dev, "Failed to attached iommu device.\n");
+		goto err_clean_dc;
+	}
+
+	port = of_get_child_by_name(dev->of_node, "port");
+	if (!port) {
+		dev_err(dev, "no port node found\n");
+		goto err_detach_dev;
+	}
+	of_node_put(port);
+
+	dc_info = dc->hw.info;
+	crtc = es_crtc_create(drm_dev, dc_info);
+	if (!crtc) {
+		dev_err(dev, "Failed to create CRTC.\n");
+		ret = -ENOMEM;
+		goto err_detach_dev;
+	}
+
+	crtc->base.port = port;
+	crtc->dev = dev;
+	crtc->funcs = &dc_crtc_funcs;
+
+	for (i = 0; i < dc_info->plane_num; i++) {
+		plane_info = (struct es_plane_info *)&dc_info->planes[i];
+
+		plane = es_plane_create(drm_dev, plane_info,
+					drm_crtc_mask(&crtc->base));
+		if (!plane)
+			goto err_cleanup_planes;
+
+		plane->funcs = &dc_plane_funcs;
+
+		if (plane_info->type == DRM_PLANE_TYPE_PRIMARY) {
+			crtc->base.primary = &plane->base;
+			drm_dev->mode_config.min_width = plane_info->min_width;
+			drm_dev->mode_config.min_height =
+				plane_info->min_height;
+			drm_dev->mode_config.max_width = plane_info->max_width;
+			drm_dev->mode_config.max_height =
+				plane_info->max_height;
+		}
+
+		if (plane_info->type == DRM_PLANE_TYPE_CURSOR) {
+			crtc->base.cursor = &plane->base;
+			drm_dev->mode_config.cursor_width =
+				plane_info->max_width;
+			drm_dev->mode_config.cursor_height =
+				plane_info->max_height;
+		}
+	}
+
+	dc->crtc = crtc;
+	dc->funcs = &dc_funcs;
+
+	es_drm_update_pitch_alignment(drm_dev, dc_info->pitch_alignment);
+
+	es_dc_clk_configs(dev, false);
+
+	return 0;
+
+err_cleanup_planes:
+	list_for_each_entry_safe (drm_plane, tmp,
+				  &drm_dev->mode_config.plane_list, head)
+		if (drm_plane->possible_crtcs == drm_crtc_mask(&crtc->base))
+			es_plane_destory(drm_plane);
+
+	es_crtc_destroy(&crtc->base);
+err_detach_dev:
+	es_drm_iommu_detach_device(drm_dev, dev);
+err_clean_dc:
+	dc_deinit(dev);
+	return ret;
+}
+
+static void dc_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+
+	dc_deinit(dev);
+
+	es_drm_iommu_detach_device(drm_dev, dev);
+}
+
+const struct component_ops dc_component_ops = {
+	.bind = dc_bind,
+	.unbind = dc_unbind,
+};
+
+static void vo_qos_cfg(void)
+{
+	void __iomem *qos;
+
+	#define VO_QOS_CSR	0x50281050UL
+	qos = ioremap(VO_QOS_CSR, 8);
+	if (!qos) {
+		printk("qos ioremap fail---------------\n");
+		return;
+	}
+	writel(0x9, qos);
+	writel(0x9, (char *)qos + 4);
+
+	iounmap(qos);
+	return;
+}
+
+static const struct of_device_id dc_driver_dt_match[] = {
+	{
+		.compatible = "eswin,dc",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, dc_driver_dt_match);
+
+static int dc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct es_dc *dc;
+	int irq, ret;
+
+	dc = devm_kzalloc(dev, sizeof(*dc), GFP_KERNEL);
+	if (!dc)
+		return -ENOMEM;
+
+	dc->hw.hi_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(dc->hw.hi_base))
+		return PTR_ERR(dc->hw.hi_base);
+
+#ifdef CONFIG_ESWIN_MMU
+	dc->hw.mmu_base = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(dc->hw.mmu_base))
+		return PTR_ERR(dc->hw.mmu_base);
+#endif
+
+	dc->hw.reg_base = devm_platform_ioremap_resource(pdev, 2);
+	if (IS_ERR(dc->hw.reg_base))
+		return PTR_ERR(dc->hw.reg_base);
+
+	irq = platform_get_irq(pdev, 0);
+	ret = devm_request_irq(dev, irq, dc_isr, 0, dev_name(dev), dc);
+	if (ret < 0) {
+		dev_err(dev, "Failed to install irq:%u.\n", irq);
+		return ret;
+	}
+
+	dc->vo_mux = devm_clk_get(dev, "vo_mux");
+	if (IS_ERR(dc->vo_mux)) {
+		ret = PTR_ERR(dc->vo_mux);
+		dev_err(dev, "failed to get vo_mux: %d\n", ret);
+		return ret;
+	}
+
+	dc->spll0_fout1 = devm_clk_get_optional(dev, "spll0_fout1");
+	if (IS_ERR(dc->spll0_fout1)) {
+		dev_err(dev, "failed to get spll0_fout1 source\n");
+		return PTR_ERR(dc->spll0_fout1);
+	}
+
+	dc->cfg_clk = devm_clk_get_optional(dev, "cfg_clk");
+	if (IS_ERR(dc->cfg_clk)) {
+		dev_err(dev, "failed to get cfg_clk source\n");
+		return PTR_ERR(dc->cfg_clk);
+	}
+
+	dc->pix_clk = devm_clk_get_optional(dev, "pix_clk");
+	if (IS_ERR(dc->pix_clk)) {
+		dev_err(dev, "failed to get pix_clk source\n");
+		return PTR_ERR(dc->pix_clk);
+	}
+
+	dc->axi_clk = devm_clk_get_optional(dev, "axi_clk");
+	if (IS_ERR(dc->axi_clk)) {
+		dev_err(dev, "failed to get axi_clk source\n");
+		return PTR_ERR(dc->axi_clk);
+	}
+
+	dc->vo_arst = devm_reset_control_get_optional(dev, "vo_arst");
+	if (IS_ERR_OR_NULL(dc->vo_arst)) {
+		dev_err(dev, "Failed to vo_arst handle\n");
+		return PTR_ERR(dc->vo_arst);
+	}
+
+	dc->vo_prst = devm_reset_control_get_optional(dev, "vo_prst");
+	if (IS_ERR_OR_NULL(dc->vo_prst)) {
+		dev_err(dev, "Failed to vo_prst handle\n");
+		return PTR_ERR(dc->vo_prst);
+	}
+
+	dc->dc_arst = devm_reset_control_get_optional(dev, "dc_arst");
+	if (IS_ERR_OR_NULL(dc->dc_arst)) {
+		dev_err(dev, "Failed to dc_arst handle\n");
+		return PTR_ERR(dc->dc_arst);
+	}
+
+	dc->dc_prst = devm_reset_control_get_optional(dev, "dc_prst");
+	if (IS_ERR_OR_NULL(dc->dc_prst)) {
+		dev_err(dev, "Failed to dc_prst handle\n");
+		return PTR_ERR(dc->dc_prst);
+	}
+
+	/* reset dc first to ensure no data on axi bus */
+	if (dc->dc_arst) {
+		ret = reset_control_reset(dc->dc_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->dc_prst) {
+		ret = reset_control_reset(dc->dc_prst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_arst) {
+		ret = reset_control_reset(dc->vo_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_prst) {
+		ret = reset_control_reset(dc->vo_prst);
+		WARN_ON(0 != ret);
+	}
+
+	dev_set_drvdata(dev, dc);
+	vo_qos_cfg();
+	return component_add(dev, &dc_component_ops);
+}
+
+static int dc_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+	struct es_dc *dc = dev_get_drvdata(dev);
+
+	component_del(dev, &dc_component_ops);
+
+	if (dc->dc_arst) {
+		ret = reset_control_assert(dc->dc_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->dc_prst) {
+		ret = reset_control_assert(dc->dc_prst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_arst) {
+		ret = reset_control_assert(dc->vo_arst);
+		WARN_ON(0 != ret);
+	}
+
+	if (dc->vo_prst) {
+		ret = reset_control_assert(dc->vo_prst);
+		WARN_ON(0 != ret);
+	}
+
+	dev_set_drvdata(dev, NULL);
+
+	return 0;
+}
+
+struct platform_driver dc_platform_driver = {
+    .probe = dc_probe,
+    .remove = dc_remove,
+    .driver = {
+        .name = "es-dc",
+        .of_match_table = of_match_ptr(dc_driver_dt_match),
+    },
+};
+
+MODULE_DESCRIPTION("Eswin DC Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_dc.h b/drivers/gpu/drm/eswin/es_dc.h
new file mode 100644
index 000000000000..8dbc3b4d7eb2
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_DC_H__
+#define __ES_DC_H__
+
+#include <linux/mm_types.h>
+#include <drm/drm_modes.h>
+
+#include "es_plane.h"
+#include "es_crtc.h"
+#include "es_dc_hw.h"
+#ifdef CONFIG_ESWIN_MMU
+#include "es_dc_mmu.h"
+#endif
+
+struct es_dc_funcs {
+	void (*dump_enable)(struct device *dev, dma_addr_t addr,
+			    unsigned int pitch);
+	void (*dump_disable)(struct device *dev);
+};
+
+struct es_dc {
+	struct es_crtc *crtc;
+	struct dc_hw hw;
+
+	struct clk *vo_mux;
+	struct clk *spll0_fout1;
+	struct clk *cfg_clk;
+	struct clk *pix_clk;
+	struct clk *axi_clk;
+	unsigned int pix_clk_rate; /* in KHz */
+
+	struct reset_control *vo_arst;
+	struct reset_control *vo_prst;
+	struct reset_control *dc_arst;
+	struct reset_control *dc_prst;
+
+	bool first_frame;
+	bool dc_initialized;
+	bool dc_clkon;
+
+	const struct es_dc_funcs *funcs;
+};
+
+extern struct platform_driver dc_platform_driver;
+#endif /* __ES_DC_H__ */
diff --git a/drivers/gpu/drm/eswin/es_dc_hw.c b/drivers/gpu/drm/eswin/es_dc_hw.c
new file mode 100644
index 000000000000..440fd1645694
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_hw.c
@@ -0,0 +1,1615 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/io.h>
+#include <linux/bits.h>
+#include <linux/media-bus-format.h>
+#include <linux/delay.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_blend.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_atomic_helper.h>
+
+#include "es_drm.h"
+#include "es_type.h"
+#include "es_dc_hw.h"
+
+static const u32 horKernel[] = {
+	0x00000000, 0x20000000, 0x00002000, 0x00000000, 0x00000000, 0x00000000,
+	0x23fd1c03, 0x00000000, 0x00000000, 0x00000000, 0x181f0000, 0x000027e1,
+	0x00000000, 0x00000000, 0x00000000, 0x2b981468, 0x00000000, 0x00000000,
+	0x00000000, 0x10f00000, 0x00002f10, 0x00000000, 0x00000000, 0x00000000,
+	0x32390dc7, 0x00000000, 0x00000000, 0x00000000, 0x0af50000, 0x0000350b,
+	0x00000000, 0x00000000, 0x00000000, 0x3781087f, 0x00000000, 0x00000000,
+	0x00000000, 0x06660000, 0x0000399a, 0x00000000, 0x00000000, 0x00000000,
+	0x3b5904a7, 0x00000000, 0x00000000, 0x00000000, 0x033c0000, 0x00003cc4,
+	0x00000000, 0x00000000, 0x00000000, 0x3de1021f, 0x00000000, 0x00000000,
+	0x00000000, 0x01470000, 0x00003eb9, 0x00000000, 0x00000000, 0x00000000,
+	0x3f5300ad, 0x00000000, 0x00000000, 0x00000000, 0x00480000, 0x00003fb8,
+	0x00000000, 0x00000000, 0x00000000, 0x3fef0011, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00004000, 0x00000000, 0x00000000, 0x00000000,
+	0x20002000, 0x00000000, 0x00000000, 0x00000000, 0x1c030000, 0x000023fd,
+	0x00000000, 0x00000000, 0x00000000, 0x27e1181f, 0x00000000, 0x00000000,
+	0x00000000, 0x14680000, 0x00002b98, 0x00000000, 0x00000000, 0x00000000,
+	0x2f1010f0, 0x00000000, 0x00000000, 0x00000000, 0x0dc70000, 0x00003239,
+	0x00000000, 0x00000000, 0x00000000, 0x350b0af5, 0x00000000, 0x00000000,
+	0x00000000, 0x087f0000, 0x00003781, 0x00000000, 0x00000000, 0x00000000,
+	0x399a0666, 0x00000000, 0x00000000, 0x00000000, 0x04a70000, 0x00003b59,
+	0x00000000, 0x00000000, 0x00000000, 0x3cc4033c, 0x00000000, 0x00000000,
+	0x00000000, 0x021f0000,
+};
+#define H_COEF_SIZE (sizeof(horKernel) / sizeof(u32))
+
+static const u32 verKernel[] = {
+	0x00000000, 0x20000000, 0x00002000, 0x00000000, 0x00000000, 0x00000000,
+	0x23fd1c03, 0x00000000, 0x00000000, 0x00000000, 0x181f0000, 0x000027e1,
+	0x00000000, 0x00000000, 0x00000000, 0x2b981468, 0x00000000, 0x00000000,
+	0x00000000, 0x10f00000, 0x00002f10, 0x00000000, 0x00000000, 0x00000000,
+	0x32390dc7, 0x00000000, 0x00000000, 0x00000000, 0x0af50000, 0x0000350b,
+	0x00000000, 0x00000000, 0x00000000, 0x3781087f, 0x00000000, 0x00000000,
+	0x00000000, 0x06660000, 0x0000399a, 0x00000000, 0x00000000, 0x00000000,
+	0x3b5904a7, 0x00000000, 0x00000000, 0x00000000, 0x033c0000, 0x00003cc4,
+	0x00000000, 0x00000000, 0x00000000, 0x3de1021f, 0x00000000, 0x00000000,
+	0x00000000, 0x01470000, 0x00003eb9, 0x00000000, 0x00000000, 0x00000000,
+	0x3f5300ad, 0x00000000, 0x00000000, 0x00000000, 0x00480000, 0x00003fb8,
+	0x00000000, 0x00000000, 0x00000000, 0x3fef0011, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00004000, 0x00000000, 0xcdcd0000, 0xfdfdfdfd,
+	0xabababab, 0xabababab, 0x00000000, 0x00000000, 0x5ff5f456, 0x000f5f58,
+	0x02cc6c78, 0x02cc0c28, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee, 0xfeeefeee,
+	0xfeeefeee, 0xfeeefeee,
+};
+#define V_COEF_SIZE (sizeof(verKernel) / sizeof(u32))
+
+/*
+ * RGB 709->2020 conversion parameters
+ */
+static u16 RGB2RGB[RGB_TO_RGB_TABLE_SIZE] = { 10279, 5395, 709,	 1132, 15065,
+					      187,   269,  1442, 14674 };
+
+/*
+ * YUV601 to RGB conversion parameters
+ * YUV2RGB[0]  - [8] : C0 - C8;
+ * YUV2RGB[9]  - [11]: D0 - D2;
+ * YUV2RGB[12] - [13]: Y clamp min & max calue;
+ * YUV2RGB[14] - [15]: UV clamp min & max calue;
+ */
+static s32 YUV601_2RGB[YUV_TO_RGB_TABLE_SIZE] = {
+	1196, 0,       1640,   1196,	 -404, -836, 1196, 2076,
+	0,    -916224, 558336, -1202944, 64,   940,  64,   960
+};
+
+/*
+ * YUV709 to RGB conversion parameters
+ * YUV2RGB[0]  - [8] : C0 - C8;
+ * YUV2RGB[9]  - [11]: D0 - D2;
+ * YUV2RGB[12] - [13]: Y clamp min & max calue;
+ * YUV2RGB[14] - [15]: UV clamp min & max calue;
+ */
+static s32 YUV709_2RGB[YUV_TO_RGB_TABLE_SIZE] = {
+	1196, 0,	1844,	1196,	  -220, -548, 1196, 2172,
+	0,    -1020672, 316672, -1188608, 64,	940,  64,   960
+};
+
+/*
+ * YUV2020 to RGB conversion parameters
+ * YUV2RGB[0]  - [8] : C0 - C8;
+ * YUV2RGB[9]  - [11]: D0 - D2;
+ * YUV2RGB[12] - [13]: Y clamp min & max calue;
+ * YUV2RGB[14] - [15]: UV clamp min & max calue;
+ */
+static s32 YUV2020_2RGB[YUV_TO_RGB_TABLE_SIZE] = {
+	1196, 0,       1724,   1196,	 -192, -668, 1196, 2200,
+	0,    -959232, 363776, -1202944, 64,   940,  64,   960
+};
+
+/*
+ * RGB to YUV2020 conversion parameters
+ * RGB2YUV[0] - [8] : C0 - C8;
+ * RGB2YUV[9] - [11]: D0 - D2;
+ */
+static s16 RGB2YUV[RGB_TO_YUV_TABLE_SIZE] = { 230, 594,	 52,  -125, -323, 448,
+					      448, -412, -36, 64,   512,  512 };
+
+/*
+ * Degamma table for 709 color space data.
+ */
+static u16 DEGAMMA_709[DEGAMMA_SIZE] = {
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0005, 0x0007,
+	0x000a, 0x000d, 0x0011, 0x0015, 0x0019, 0x001e, 0x0024, 0x002a, 0x0030,
+	0x0038, 0x003f, 0x0048, 0x0051, 0x005a, 0x0064, 0x006f, 0x007b, 0x0087,
+	0x0094, 0x00a1, 0x00af, 0x00be, 0x00ce, 0x00de, 0x00ef, 0x0101, 0x0114,
+	0x0127, 0x013b, 0x0150, 0x0166, 0x017c, 0x0193, 0x01ac, 0x01c4, 0x01de,
+	0x01f9, 0x0214, 0x0230, 0x024d, 0x026b, 0x028a, 0x02aa, 0x02ca, 0x02ec,
+	0x030e, 0x0331, 0x0355, 0x037a, 0x03a0, 0x03c7, 0x03ef, 0x0418, 0x0441,
+	0x046c, 0x0498, 0x04c4, 0x04f2, 0x0520, 0x0550, 0x0581, 0x05b2, 0x05e5,
+	0x0618, 0x064d, 0x0682, 0x06b9, 0x06f0, 0x0729, 0x0763, 0x079d, 0x07d9,
+	0x0816, 0x0854, 0x0893, 0x08d3, 0x0914, 0x0956, 0x0999, 0x09dd, 0x0a23,
+	0x0a69, 0x0ab1, 0x0afa, 0x0b44, 0x0b8f, 0x0bdb, 0x0c28, 0x0c76, 0x0cc6,
+	0x0d17, 0x0d69, 0x0dbb, 0x0e10, 0x0e65, 0x0ebb, 0x0f13, 0x0f6c, 0x0fc6,
+	0x1021, 0x107d, 0x10db, 0x113a, 0x119a, 0x11fb, 0x125d, 0x12c1, 0x1325,
+	0x138c, 0x13f3, 0x145b, 0x14c5, 0x1530, 0x159c, 0x160a, 0x1678, 0x16e8,
+	0x175a, 0x17cc, 0x1840, 0x18b5, 0x192b, 0x19a3, 0x1a1c, 0x1a96, 0x1b11,
+	0x1b8e, 0x1c0c, 0x1c8c, 0x1d0c, 0x1d8e, 0x1e12, 0x1e96, 0x1f1c, 0x1fa3,
+	0x202c, 0x20b6, 0x2141, 0x21ce, 0x225c, 0x22eb, 0x237c, 0x240e, 0x24a1,
+	0x2536, 0x25cc, 0x2664, 0x26fc, 0x2797, 0x2832, 0x28cf, 0x296e, 0x2a0e,
+	0x2aaf, 0x2b51, 0x2bf5, 0x2c9b, 0x2d41, 0x2dea, 0x2e93, 0x2f3e, 0x2feb,
+	0x3099, 0x3148, 0x31f9, 0x32ab, 0x335f, 0x3414, 0x34ca, 0x3582, 0x363c,
+	0x36f7, 0x37b3, 0x3871, 0x3930, 0x39f1, 0x3ab3, 0x3b77, 0x3c3c, 0x3d02,
+	0x3dcb, 0x3e94, 0x3f5f, 0x402c, 0x40fa, 0x41ca, 0x429b, 0x436d, 0x4442,
+	0x4517, 0x45ee, 0x46c7, 0x47a1, 0x487d, 0x495a, 0x4a39, 0x4b19, 0x4bfb,
+	0x4cde, 0x4dc3, 0x4eaa, 0x4f92, 0x507c, 0x5167, 0x5253, 0x5342, 0x5431,
+	0x5523, 0x5616, 0x570a, 0x5800, 0x58f8, 0x59f1, 0x5aec, 0x5be9, 0x5ce7,
+	0x5de6, 0x5ee7, 0x5fea, 0x60ef, 0x61f5, 0x62fc, 0x6406, 0x6510, 0x661d,
+	0x672b, 0x683b, 0x694c, 0x6a5f, 0x6b73, 0x6c8a, 0x6da2, 0x6ebb, 0x6fd6,
+	0x70f3, 0x7211, 0x7331, 0x7453, 0x7576, 0x769b, 0x77c2, 0x78ea, 0x7a14,
+	0x7b40, 0x7c6d, 0x7d9c, 0x7ecd, 0x3f65, 0x3f8c, 0x3fb2, 0x3fd8
+};
+
+/*
+ * Degamma table for 2020 color space data.
+ */
+static u16 DEGAMMA_2020[DEGAMMA_SIZE] = {
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0001,
+	0x0001, 0x0001, 0x0001, 0x0001, 0x0001, 0x0001, 0x0002, 0x0002, 0x0002,
+	0x0002, 0x0002, 0x0003, 0x0003, 0x0003, 0x0003, 0x0004, 0x0004, 0x0004,
+	0x0005, 0x0005, 0x0006, 0x0006, 0x0006, 0x0007, 0x0007, 0x0008, 0x0008,
+	0x0009, 0x000a, 0x000a, 0x000b, 0x000c, 0x000c, 0x000d, 0x000e, 0x000f,
+	0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0016, 0x0017, 0x0018,
+	0x0019, 0x001b, 0x001c, 0x001e, 0x001f, 0x0021, 0x0022, 0x0024, 0x0026,
+	0x0028, 0x002a, 0x002c, 0x002e, 0x0030, 0x0033, 0x0035, 0x0038, 0x003a,
+	0x003d, 0x0040, 0x0043, 0x0046, 0x0049, 0x004d, 0x0050, 0x0054, 0x0057,
+	0x005b, 0x005f, 0x0064, 0x0068, 0x006d, 0x0071, 0x0076, 0x007c, 0x0081,
+	0x0086, 0x008c, 0x0092, 0x0098, 0x009f, 0x00a5, 0x00ac, 0x00b4, 0x00bb,
+	0x00c3, 0x00cb, 0x00d3, 0x00dc, 0x00e5, 0x00ee, 0x00f8, 0x0102, 0x010c,
+	0x0117, 0x0123, 0x012e, 0x013a, 0x0147, 0x0154, 0x0161, 0x016f, 0x017e,
+	0x018d, 0x019c, 0x01ac, 0x01bd, 0x01ce, 0x01e0, 0x01f3, 0x0206, 0x021a,
+	0x022f, 0x0244, 0x025a, 0x0272, 0x0289, 0x02a2, 0x02bc, 0x02d6, 0x02f2,
+	0x030f, 0x032c, 0x034b, 0x036b, 0x038b, 0x03ae, 0x03d1, 0x03f5, 0x041b,
+	0x0443, 0x046b, 0x0495, 0x04c1, 0x04ee, 0x051d, 0x054e, 0x0580, 0x05b4,
+	0x05ea, 0x0622, 0x065c, 0x0698, 0x06d6, 0x0717, 0x075a, 0x079f, 0x07e7,
+	0x0831, 0x087e, 0x08cd, 0x0920, 0x0976, 0x09ce, 0x0a2a, 0x0a89, 0x0aec,
+	0x0b52, 0x0bbc, 0x0c2a, 0x0c9b, 0x0d11, 0x0d8b, 0x0e0a, 0x0e8d, 0x0f15,
+	0x0fa1, 0x1033, 0x10ca, 0x1167, 0x120a, 0x12b2, 0x1360, 0x1415, 0x14d1,
+	0x1593, 0x165d, 0x172e, 0x1806, 0x18e7, 0x19d0, 0x1ac1, 0x1bbb, 0x1cbf,
+	0x1dcc, 0x1ee3, 0x2005, 0x2131, 0x2268, 0x23ab, 0x24fa, 0x2656, 0x27be,
+	0x2934, 0x2ab8, 0x2c4a, 0x2dec, 0x2f9d, 0x315f, 0x3332, 0x3516, 0x370d,
+	0x3916, 0x3b34, 0x3d66, 0x3fad, 0x420b, 0x4480, 0x470d, 0x49b3, 0x4c73,
+	0x4f4e, 0x5246, 0x555a, 0x588e, 0x5be1, 0x5f55, 0x62eb, 0x66a6, 0x6a86,
+	0x6e8c, 0x72bb, 0x7714, 0x7b99, 0x3dcb, 0x3e60, 0x3ef5, 0x3f8c
+};
+
+/* one is for primary plane and the other is for all overlay planes */
+static const struct dc_hw_plane_reg dc_plane_reg[] = {
+	{
+		.y_address = DC_FRAMEBUFFER_ADDRESS,
+		.u_address = DC_FRAMEBUFFER_U_ADDRESS,
+		.v_address = DC_FRAMEBUFFER_V_ADDRESS,
+		.y_stride = DC_FRAMEBUFFER_STRIDE,
+		.u_stride = DC_FRAMEBUFFER_U_STRIDE,
+		.v_stride = DC_FRAMEBUFFER_V_STRIDE,
+		.size = DC_FRAMEBUFFER_SIZE,
+		.scale_factor_x = DC_FRAMEBUFFER_SCALE_FACTOR_X,
+		.scale_factor_y = DC_FRAMEBUFFER_SCALE_FACTOR_Y,
+		.h_filter_coef_index = DC_FRAMEBUFFER_H_FILTER_COEF_INDEX,
+		.h_filter_coef_data = DC_FRAMEBUFFER_H_FILTER_COEF_DATA,
+		.v_filter_coef_index = DC_FRAMEBUFFER_V_FILTER_COEF_INDEX,
+		.v_filter_coef_data = DC_FRAMEBUFFER_V_FILTER_COEF_DATA,
+		.init_offset = DC_FRAMEBUFFER_INIT_OFFSET,
+		.color_key = DC_FRAMEBUFFER_COLOR_KEY,
+		.color_key_high = DC_FRAMEBUFFER_COLOR_KEY_HIGH,
+		.clear_value = DC_FRAMEBUFFER_CLEAR_VALUE,
+		.color_table_index = DC_FRAMEBUFFER_COLOR_TABLE_INDEX,
+		.color_table_data = DC_FRAMEBUFFER_COLOR_TABLE_DATA,
+		.scale_config = DC_FRAMEBUFFER_SCALE_CONFIG,
+		.degamma_index = DC_FRAMEBUFFER_DEGAMMA_INDEX,
+		.degamma_data = DC_FRAMEBUFFER_DEGAMMA_DATA,
+		.degamma_ex_data = DC_FRAMEBUFFER_DEGAMMA_EX_DATA,
+		.roi_origin = DC_FRAMEBUFFER_ROI_ORIGIN,
+		.roi_size = DC_FRAMEBUFFER_ROI_SIZE,
+		.YUVToRGBCoef0 = DC_FRAMEBUFFER_YUVTORGB_COEF0,
+		.YUVToRGBCoef1 = DC_FRAMEBUFFER_YUVTORGB_COEF1,
+		.YUVToRGBCoef2 = DC_FRAMEBUFFER_YUVTORGB_COEF2,
+		.YUVToRGBCoef3 = DC_FRAMEBUFFER_YUVTORGB_COEF3,
+		.YUVToRGBCoef4 = DC_FRAMEBUFFER_YUVTORGB_COEF4,
+		.YUVToRGBCoefD0 = DC_FRAMEBUFFER_YUVTORGB_COEFD0,
+		.YUVToRGBCoefD1 = DC_FRAMEBUFFER_YUVTORGB_COEFD1,
+		.YUVToRGBCoefD2 = DC_FRAMEBUFFER_YUVTORGB_COEFD2,
+		.YClampBound = DC_FRAMEBUFFER_Y_CLAMP_BOUND,
+		.UVClampBound = DC_FRAMEBUFFER_UV_CLAMP_BOUND,
+		.RGBToRGBCoef0 = DC_FRAMEBUFFER_RGBTORGB_COEF0,
+		.RGBToRGBCoef1 = DC_FRAMEBUFFER_RGBTORGB_COEF1,
+		.RGBToRGBCoef2 = DC_FRAMEBUFFER_RGBTORGB_COEF2,
+		.RGBToRGBCoef3 = DC_FRAMEBUFFER_RGBTORGB_COEF3,
+		.RGBToRGBCoef4 = DC_FRAMEBUFFER_RGBTORGB_COEF4,
+	},
+	{
+		.y_address = DC_OVERLAY_ADDRESS,
+		.u_address = DC_OVERLAY_U_ADDRESS,
+		.v_address = DC_OVERLAY_V_ADDRESS,
+		.y_stride = DC_OVERLAY_STRIDE,
+		.u_stride = DC_OVERLAY_U_STRIDE,
+		.v_stride = DC_OVERLAY_V_STRIDE,
+		.size = DC_OVERLAY_SIZE,
+		.scale_factor_x = DC_OVERLAY_SCALE_FACTOR_X,
+		.scale_factor_y = DC_OVERLAY_SCALE_FACTOR_Y,
+		.h_filter_coef_index = DC_OVERLAY_H_FILTER_COEF_INDEX,
+		.h_filter_coef_data = DC_OVERLAY_H_FILTER_COEF_DATA,
+		.v_filter_coef_index = DC_OVERLAY_V_FILTER_COEF_INDEX,
+		.v_filter_coef_data = DC_OVERLAY_V_FILTER_COEF_DATA,
+		.init_offset = DC_OVERLAY_INIT_OFFSET,
+		.color_key = DC_OVERLAY_COLOR_KEY,
+		.color_key_high = DC_OVERLAY_COLOR_KEY_HIGH,
+		.clear_value = DC_OVERLAY_CLEAR_VALUE,
+		.color_table_index = DC_OVERLAY_COLOR_TABLE_INDEX,
+		.color_table_data = DC_OVERLAY_COLOR_TABLE_DATA,
+		.scale_config = DC_OVERLAY_SCALE_CONFIG,
+		.degamma_index = DC_OVERLAY_DEGAMMA_INDEX,
+		.degamma_data = DC_OVERLAY_DEGAMMA_DATA,
+		.degamma_ex_data = DC_OVERLAY_DEGAMMA_EX_DATA,
+		.roi_origin = DC_OVERLAY_ROI_ORIGIN,
+		.roi_size = DC_OVERLAY_ROI_SIZE,
+		.YUVToRGBCoef0 = DC_OVERLAY_YUVTORGB_COEF0,
+		.YUVToRGBCoef1 = DC_OVERLAY_YUVTORGB_COEF1,
+		.YUVToRGBCoef2 = DC_OVERLAY_YUVTORGB_COEF2,
+		.YUVToRGBCoef3 = DC_OVERLAY_YUVTORGB_COEF3,
+		.YUVToRGBCoef4 = DC_OVERLAY_YUVTORGB_COEF4,
+		.YUVToRGBCoefD0 = DC_OVERLAY_YUVTORGB_COEFD0,
+		.YUVToRGBCoefD1 = DC_OVERLAY_YUVTORGB_COEFD1,
+		.YUVToRGBCoefD2 = DC_OVERLAY_YUVTORGB_COEFD2,
+		.YClampBound = DC_OVERLAY_Y_CLAMP_BOUND,
+		.UVClampBound = DC_OVERLAY_UV_CLAMP_BOUND,
+		.RGBToRGBCoef0 = DC_OVERLAY_RGBTORGB_COEF0,
+		.RGBToRGBCoef1 = DC_OVERLAY_RGBTORGB_COEF1,
+		.RGBToRGBCoef2 = DC_OVERLAY_RGBTORGB_COEF2,
+		.RGBToRGBCoef3 = DC_OVERLAY_RGBTORGB_COEF3,
+		.RGBToRGBCoef4 = DC_OVERLAY_RGBTORGB_COEF4,
+	},
+};
+
+static const u32 read_reg[] = {
+	DC_FRAMEBUFFER_CONFIG,
+	DC_OVERLAY_CONFIG,
+	DC_OVERLAY_SCALE_CONFIG,
+};
+#define READ_REG_COUNT ARRAY_SIZE(read_reg)
+
+static const u32 primary_overlay_formats[] = {
+	DRM_FORMAT_XRGB4444, DRM_FORMAT_ARGB4444,    DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555, DRM_FORMAT_RGB565,	     DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_ARGB2101010, DRM_FORMAT_RGBA4444,
+	DRM_FORMAT_RGBA5551, DRM_FORMAT_RGBA8888,    DRM_FORMAT_RGBA1010102,
+	DRM_FORMAT_ABGR4444, DRM_FORMAT_ABGR1555,    DRM_FORMAT_BGR565,
+	DRM_FORMAT_ABGR8888, DRM_FORMAT_ABGR2101010, DRM_FORMAT_BGRA4444,
+	DRM_FORMAT_BGRA5551, DRM_FORMAT_BGRA8888,    DRM_FORMAT_BGRA1010102,
+	DRM_FORMAT_XBGR4444, DRM_FORMAT_RGBX4444,    DRM_FORMAT_BGRX4444,
+	DRM_FORMAT_XBGR1555, DRM_FORMAT_RGBX5551,    DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_XBGR8888, DRM_FORMAT_RGBX8888,    DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_YUYV,     DRM_FORMAT_YVYU,	     DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,     DRM_FORMAT_NV12,	     DRM_FORMAT_NV21,
+	DRM_FORMAT_NV16,     DRM_FORMAT_NV61,	     DRM_FORMAT_P010,
+	/* TODO. */
+};
+
+static const u32 cursor_formats[] = { DRM_FORMAT_ARGB8888 };
+
+static const u64 cursor_dumy_modifiers[] = { DRM_FORMAT_MOD_LINEAR,
+					     DRM_FORMAT_MOD_INVALID };
+
+static const u64 format_modifiers[] = { DRM_FORMAT_MOD_LINEAR,
+					DRM_FORMAT_MOD_INVALID };
+
+#define FRAC_16_16(mult, div) (((mult) << 16) / (div))
+
+static const struct es_plane_info dc_hw_planes[][PLANE_NUM] = {
+	{
+		/* DC_REV_5551 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = NULL,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 32,
+			.max_height = 32,
+			.rotation = DRM_MODE_ROTATE_0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+	},
+	{
+		/* DC_REV_5551_306 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
+				    DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 |
+				    DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = NULL,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 32,
+			.max_height = 32,
+			.rotation = 0,
+			.color_encoding = 0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
+				    DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 |
+				    DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = 0,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+	},
+	{
+		/* DC_REV_5701_303 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = NULL,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 256,
+			.max_height = 256,
+			.rotation = DRM_MODE_ROTATE_0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 3),
+			.max_scale = FRAC_16_16(10, 1),
+		},
+	},
+	{
+		/* DC_REV_5701_309 */
+		{
+			.name = "Primary",
+			.id = PRIMARY_PLANE,
+			.type = DRM_PLANE_TYPE_PRIMARY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180,
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 32),
+			.max_scale = FRAC_16_16(2, 1),
+			.roi = true,
+			.color_mgmt = true,
+			.background = true,
+		},
+		{
+			.name = "Cursor",
+			.id = CURSOR_PLANE,
+			.type = DRM_PLANE_TYPE_CURSOR,
+			.num_formats = ARRAY_SIZE(cursor_formats),
+			.formats = cursor_formats,
+			.modifiers = cursor_dumy_modifiers,
+			.min_width = 32,
+			.min_height = 32,
+			.max_width = 256,
+			.max_height = 256,
+			.rotation = DRM_MODE_ROTATE_0,
+			.degamma_size = 0,
+			.min_scale = DRM_PLANE_NO_SCALING,
+			.max_scale = DRM_PLANE_NO_SCALING,
+		},
+		{
+			.name = "Overlay",
+			.id = OVERLAY_PLANE,
+			.type = DRM_PLANE_TYPE_OVERLAY,
+			.num_formats = ARRAY_SIZE(primary_overlay_formats),
+			.formats = primary_overlay_formats,
+			.modifiers = format_modifiers,
+			.min_width = 0,
+			.min_height = 0,
+			.max_width = 4096,
+			.max_height = 4096,
+			.rotation = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180,
+			.blend_mode = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+				      BIT(DRM_MODE_BLEND_PREMULTI) |
+				      BIT(DRM_MODE_BLEND_COVERAGE),
+			.color_encoding = BIT(DRM_COLOR_YCBCR_BT601) |
+					  BIT(DRM_COLOR_YCBCR_BT709) |
+					  BIT(DRM_COLOR_YCBCR_BT2020),
+			.degamma_size = DEGAMMA_SIZE,
+			.min_scale = FRAC_16_16(1, 32),
+			.max_scale = FRAC_16_16(2, 1),
+			.roi = true,
+			.color_mgmt = true,
+		},
+	},
+};
+
+static const struct es_dc_info dc_info[] = {
+	{
+		/* DC_REV_5551 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5551],
+		.max_bpc = 10,
+		.color_formats = DRM_COLOR_FORMAT_RGB444,
+		.gamma_size = GAMMA_SIZE,
+		.gamma_bits = 10,
+		.pitch_alignment = 128,
+		.pipe_sync = false,
+		.mmu_prefetch = false,
+	},
+	{
+		/* DC_REV_5551_306 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5551_306],
+		.max_bpc = 10,
+		.color_formats = DRM_COLOR_FORMAT_RGB444,
+		.gamma_size = GAMMA_SIZE,
+		.gamma_bits = 10,
+		.pitch_alignment = 128,
+		.pipe_sync = false,
+		.mmu_prefetch = false,
+	},
+	{
+		/* DC_REV_5701_303 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5701_303],
+		.max_bpc = 10,
+		.color_formats =
+			DRM_COLOR_FORMAT_RGB444 | DRM_COLOR_FORMAT_YCBCR444 |
+			DRM_COLOR_FORMAT_YCBCR422 | DRM_COLOR_FORMAT_YCBCR420,
+		.gamma_size = GAMMA_EX_SIZE,
+		.gamma_bits = 12,
+		.pitch_alignment = 128,
+		.pipe_sync = false,
+		.mmu_prefetch = false,
+	},
+	{
+		/* DC_REV_5701_309 */
+		.name = "DisplayControl",
+		.plane_num = PLANE_NUM,
+		.planes = dc_hw_planes[DC_REV_5701_309],
+		.max_bpc = 10,
+		.color_formats =
+			DRM_COLOR_FORMAT_RGB444 | DRM_COLOR_FORMAT_YCBCR444 |
+			DRM_COLOR_FORMAT_YCBCR422 | DRM_COLOR_FORMAT_YCBCR420,
+		.gamma_size = GAMMA_EX_SIZE,
+		.gamma_bits = 12,
+		.pitch_alignment = 128,
+		.pipe_sync = true,
+		.mmu_prefetch = true,
+		.background = true,
+	},
+};
+
+static const struct dc_hw_funcs hw_func[];
+
+static inline u32 hi_write(struct dc_hw *hw, u32 reg, u32 val)
+{
+	writel(val, hw->hi_base + reg);
+
+	return 0;
+}
+
+static inline u32 hi_read(struct dc_hw *hw, u32 reg)
+{
+	u32 value;
+
+	value = readl(hw->hi_base + reg);
+	return value;
+}
+
+static inline int dc_get_read_index(struct dc_hw *hw, u32 reg)
+{
+	int i;
+
+	for (i = 0; i < READ_REG_COUNT; i++) {
+		if (hw->read_block[i].reg == reg)
+			return i;
+	}
+	return i;
+}
+
+static inline void dc_write(struct dc_hw *hw, u32 reg, u32 value)
+{
+	writel(value, hw->reg_base + reg - DC_REG_BASE);
+
+	if (hw->read_block) {
+		int i = dc_get_read_index(hw, reg);
+
+		if (i < READ_REG_COUNT)
+			hw->read_block[i].value = value;
+	}
+}
+
+static inline u32 dc_read(struct dc_hw *hw, u32 reg)
+{
+	u32 value = readl(hw->reg_base + reg - DC_REG_BASE);
+
+	if (hw->read_block) {
+		int i = dc_get_read_index(hw, reg);
+
+		if (i < READ_REG_COUNT) {
+			value &= ~BIT(0);
+			value |= hw->read_block[i].value & BIT(0);
+		}
+	}
+
+	return value;
+}
+
+static inline void dc_set_clear(struct dc_hw *hw, u32 reg, u32 set, u32 clear)
+{
+	u32 value = dc_read(hw, reg);
+
+	value &= ~clear;
+	value |= set;
+	dc_write(hw, reg, value);
+}
+
+static void load_default_filter(struct dc_hw *hw,
+				const struct dc_hw_plane_reg *reg)
+{
+	u8 i;
+
+	dc_write(hw, reg->scale_config, 0x33);
+	dc_write(hw, reg->init_offset, 0x80008000);
+	dc_write(hw, reg->h_filter_coef_index, 0x00);
+	for (i = 0; i < H_COEF_SIZE; i++)
+		dc_write(hw, reg->h_filter_coef_data, horKernel[i]);
+
+	dc_write(hw, reg->v_filter_coef_index, 0x00);
+	for (i = 0; i < V_COEF_SIZE; i++)
+		dc_write(hw, reg->v_filter_coef_data, verKernel[i]);
+}
+
+static void load_rgb_to_rgb(struct dc_hw *hw, const struct dc_hw_plane_reg *reg,
+			    u16 *table)
+{
+	dc_write(hw, reg->RGBToRGBCoef0, table[0] | (table[1] << 16));
+	dc_write(hw, reg->RGBToRGBCoef1, table[2] | (table[3] << 16));
+	dc_write(hw, reg->RGBToRGBCoef2, table[4] | (table[5] << 16));
+	dc_write(hw, reg->RGBToRGBCoef3, table[6] | (table[7] << 16));
+	dc_write(hw, reg->RGBToRGBCoef4, table[8]);
+}
+
+static void load_yuv_to_rgb(struct dc_hw *hw, const struct dc_hw_plane_reg *reg,
+			    s32 *table)
+{
+	dc_write(hw, reg->YUVToRGBCoef0,
+		 (0xFFFF & table[0]) | (table[1] << 16));
+	dc_write(hw, reg->YUVToRGBCoef1,
+		 (0xFFFF & table[2]) | (table[3] << 16));
+	dc_write(hw, reg->YUVToRGBCoef2,
+		 (0xFFFF & table[4]) | (table[5] << 16));
+	dc_write(hw, reg->YUVToRGBCoef3,
+		 (0xFFFF & table[6]) | (table[7] << 16));
+	dc_write(hw, reg->YUVToRGBCoef4, table[8]);
+	dc_write(hw, reg->YUVToRGBCoefD0, table[9]);
+	dc_write(hw, reg->YUVToRGBCoefD1, table[10]);
+	dc_write(hw, reg->YUVToRGBCoefD2, table[11]);
+	dc_write(hw, reg->YClampBound, table[12] | (table[13] << 16));
+	dc_write(hw, reg->UVClampBound, table[14] | (table[15] << 16));
+}
+
+static void load_rgb_to_yuv(struct dc_hw *hw, s16 *table)
+{
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF0, table[0] | (table[1] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF1, table[2] | (table[3] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF2, table[4] | (table[5] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF3, table[6] | (table[7] << 16));
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEF4, table[8]);
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEFD0, table[9]);
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEFD1, table[10]);
+	dc_write(hw, DC_DISPLAY_RGBTOYUV_COEFD2, table[11]);
+}
+
+static bool is_rgb(enum dc_hw_color_format format)
+{
+	switch (format) {
+	case FORMAT_X4R4G4B4:
+	case FORMAT_A4R4G4B4:
+	case FORMAT_X1R5G5B5:
+	case FORMAT_A1R5G5B5:
+	case FORMAT_R5G6B5:
+	case FORMAT_X8R8G8B8:
+	case FORMAT_A8R8G8B8:
+	case FORMAT_A2R10G10B10:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static void load_degamma_table(struct dc_hw *hw,
+			       const struct dc_hw_plane_reg *reg, u16 *table)
+{
+	u16 i;
+	u32 value;
+
+	for (i = 0; i < DEGAMMA_SIZE; i++) {
+		dc_write(hw, reg->degamma_index, i);
+		value = table[i] | (table[i] << 16);
+		dc_write(hw, reg->degamma_data, value);
+		dc_write(hw, reg->degamma_ex_data, table[i]);
+	}
+}
+
+int dc_hw_init(struct dc_hw *hw)
+{
+	u8 i;
+	u32 revision = hi_read(hw, DC_HW_REVISION);
+	u32 cid = hi_read(hw, DC_HW_CHIP_CID);
+
+	switch (revision) {
+	case 0x5551:
+		if (cid < 0x306)
+			hw->rev = DC_REV_5551;
+		else
+			hw->rev = DC_REV_5551_306;
+		break;
+	case 0x5701:
+		if (cid < 0x309)
+			hw->rev = DC_REV_5701_303;
+		else
+			hw->rev = DC_REV_5701_309;
+		break;
+	default:
+		return -ENXIO;
+	}
+
+	if (hw->rev == DC_REV_5551) {
+		hw->read_block = kzalloc(
+			sizeof(struct dc_hw_read) * READ_REG_COUNT, GFP_KERNEL);
+		if (!hw->read_block)
+			return -ENOMEM;
+		for (i = 0; i < READ_REG_COUNT; i++)
+			hw->read_block[i].reg = read_reg[i];
+	}
+
+	/*
+     * Do soft reset before configuring DC
+     * Sleep 50ms to ensure no data transfer on AXI bus,
+     * because dc may be initialized in uboot
+     */
+	{
+		u32 val;
+
+		val = dc_read(hw, DC_FRAMEBUFFER_CONFIG);
+		val &= ~BIT(0);
+		val |= BIT(3);
+		dc_write(hw, DC_FRAMEBUFFER_CONFIG, val);
+		mdelay(50);
+		hi_write(hw, 0x0, 0x1000);
+		mdelay(10);
+	}
+
+	hw->info = (struct es_dc_info *)&dc_info[hw->rev];
+	hw->func = (struct dc_hw_funcs *)&hw_func[hw->rev];
+
+	for (i = 0; i < ARRAY_SIZE(dc_plane_reg); i++) {
+		load_default_filter(hw, &dc_plane_reg[i]);
+		if ((hw->rev == DC_REV_5701_303) ||
+		    (hw->rev == DC_REV_5701_309))
+			load_rgb_to_rgb(hw, &dc_plane_reg[i], RGB2RGB);
+	}
+
+	if (hw->rev == DC_REV_5701_303)
+		load_rgb_to_yuv(hw, RGB2YUV);
+
+	dc_write(hw, DC_CURSOR_BACKGROUND, 0x00FFFFFF);
+	dc_write(hw, DC_CURSOR_FOREGROUND, 0x00AAAAAA);
+	dc_write(hw, DC_DISPLAY_PANEL_CONFIG, 0x111);
+
+	return 0;
+}
+
+void dc_hw_deinit(struct dc_hw *hw)
+{
+	if (hw->read_block)
+		kfree(hw->read_block);
+}
+
+void dc_hw_update_plane(struct dc_hw *hw, enum dc_hw_plane_id id,
+			struct dc_hw_fb *fb, struct dc_hw_scale *scale)
+{
+	struct dc_hw_plane *plane = NULL;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		if (fb) {
+			if (fb->enable == false)
+				plane->fb.enable = false;
+			else
+				memcpy(&plane->fb, fb,
+				       sizeof(*fb) - sizeof(fb->dirty));
+			plane->fb.dirty = true;
+		}
+		if (scale) {
+			memcpy(&plane->scale, scale,
+			       sizeof(*scale) - sizeof(scale->dirty));
+			plane->scale.dirty = true;
+		}
+	}
+}
+
+void dc_hw_update_degamma(struct dc_hw *hw, enum dc_hw_plane_id id, u32 mode)
+{
+	struct dc_hw_plane *plane = NULL;
+	int i = (int)id;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		if (hw->info->planes[i].degamma_size) {
+			plane->degamma.mode = mode;
+			plane->degamma.dirty = true;
+		} else {
+			plane->degamma.dirty = false;
+		}
+	}
+}
+
+void dc_hw_set_position(struct dc_hw *hw, struct dc_hw_position *pos)
+{
+	memcpy(&hw->overlay.pos, pos, sizeof(*pos) - sizeof(pos->dirty));
+	hw->overlay.pos.dirty = true;
+}
+
+void dc_hw_set_blend(struct dc_hw *hw, struct dc_hw_blend *blend)
+{
+	memcpy(&hw->overlay.blend, blend,
+	       sizeof(*blend) - sizeof(blend->dirty));
+	hw->overlay.blend.dirty = true;
+}
+
+void dc_hw_update_cursor(struct dc_hw *hw, struct dc_hw_cursor *cursor)
+{
+	memcpy(&hw->cursor, cursor, sizeof(*cursor) - sizeof(cursor->dirty));
+	hw->cursor.dirty = true;
+}
+
+void dc_hw_update_gamma(struct dc_hw *hw, u16 index, u16 r, u16 g, u16 b)
+{
+	if (index >= hw->info->gamma_size)
+		return;
+
+	hw->gamma.gamma[index][0] = r;
+	hw->gamma.gamma[index][1] = g;
+	hw->gamma.gamma[index][2] = b;
+	hw->gamma.dirty = true;
+}
+
+void dc_hw_enable_gamma(struct dc_hw *hw, bool enable)
+{
+	hw->gamma.enable = enable;
+	hw->gamma.dirty = true;
+}
+
+void dc_hw_enable_dump(struct dc_hw *hw, u32 addr, u32 pitch)
+{
+	dc_write(hw, 0x14F0, addr);
+	dc_write(hw, 0x14E8, addr);
+	dc_write(hw, 0x1500, pitch);
+	dc_write(hw, 0x14F8, 0x30000);
+}
+
+void dc_hw_disable_dump(struct dc_hw *hw)
+{
+	dc_write(hw, 0x14F8, 0x00);
+}
+
+void dc_hw_setup_display(struct dc_hw *hw, struct dc_hw_display *display)
+{
+	memcpy(&hw->display, display, sizeof(*display));
+
+	hw->func->display(hw, display);
+}
+
+void dc_hw_enable_interrupt(struct dc_hw *hw, bool enable)
+{
+	dc_write(hw, DC_DISPLAY_INT_ENABLE, enable);
+}
+
+u32 dc_hw_get_interrupt(struct dc_hw *hw)
+{
+	return dc_read(hw, DC_DISPLAY_INT);
+}
+
+bool dc_hw_flip_in_progress(struct dc_hw *hw)
+{
+	return dc_read(hw, DC_FRAMEBUFFER_CONFIG) & BIT(6);
+}
+
+bool dc_hw_check_underflow(struct dc_hw *hw)
+{
+	return dc_read(hw, DC_FRAMEBUFFER_CONFIG) & BIT(5);
+}
+
+void dc_hw_enable_shadow_register(struct dc_hw *hw, bool enable)
+{
+	if (enable)
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(3));
+	else
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(3), 0);
+}
+
+void dc_hw_set_out(struct dc_hw *hw, enum dc_hw_out out)
+{
+	if (out <= OUT_DP)
+		hw->out = out;
+}
+
+static void gamma_commit(struct dc_hw *hw)
+{
+	u16 i;
+	u32 value;
+
+	if (hw->gamma.dirty) {
+		if (hw->gamma.enable) {
+			dc_write(hw, DC_DISPLAY_GAMMA_INDEX, 0x00);
+			for (i = 0; i < GAMMA_SIZE; i++) {
+				value = hw->gamma.gamma[i][2] |
+					(hw->gamma.gamma[i][1] << 10) |
+					(hw->gamma.gamma[i][0] << 20);
+				dc_write(hw, DC_DISPLAY_GAMMA_DATA, value);
+			}
+
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(2), 0);
+		} else {
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(2));
+		}
+		hw->gamma.dirty = false;
+	}
+}
+
+static void gamma_ex_commit(struct dc_hw *hw)
+{
+	u16 i;
+	u32 value;
+
+	if (hw->gamma.dirty) {
+		if (hw->gamma.enable) {
+			dc_write(hw, DC_DISPLAY_GAMMA_EX_INDEX, 0x00);
+			for (i = 0; i < GAMMA_EX_SIZE; i++) {
+				value = hw->gamma.gamma[i][2] |
+					(hw->gamma.gamma[i][1] << 16);
+				dc_write(hw, DC_DISPLAY_GAMMA_EX_DATA, value);
+				dc_write(hw, DC_DISPLAY_GAMMA_EX_ONE_DATA,
+					 hw->gamma.gamma[i][0]);
+			}
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(2), 0);
+		} else {
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(2));
+		}
+		hw->gamma.dirty = false;
+	}
+}
+
+static void plane_commit(struct dc_hw *hw)
+{
+	struct dc_hw_plane *plane;
+	const struct dc_hw_plane_reg *reg;
+	u16 i;
+
+	for (i = 0; i < PLANE_NUM; i++) {
+		if (i == PRIMARY_PLANE) {
+			plane = &hw->primary;
+			reg = &dc_plane_reg[0];
+		} else if (i == OVERLAY_PLANE) {
+			plane = &hw->overlay.plane;
+			reg = &dc_plane_reg[1];
+		} else {
+			continue;
+		}
+
+		if (plane->fb.dirty) {
+			if (plane->fb.enable) {
+				// sifive_l2_flush64_range(plane->fb.y_address, plane->fb.width * plane->fb.height * 4);
+
+				dc_write(hw, reg->y_address,
+					 plane->fb.y_address);
+				dc_write(hw, reg->u_address,
+					 plane->fb.u_address);
+				dc_write(hw, reg->v_address,
+					 plane->fb.v_address);
+				dc_write(hw, reg->y_stride, plane->fb.y_stride);
+				dc_write(hw, reg->u_stride, plane->fb.u_stride);
+				dc_write(hw, reg->v_stride, plane->fb.v_stride);
+				dc_write(hw, reg->size,
+					 plane->fb.width |
+						 (plane->fb.height << 15));
+				if (plane->fb.clear_enable)
+					dc_write(hw, reg->clear_value,
+						 plane->fb.clear_value);
+			} else {
+				dc_hw_enable_shadow_register(hw, true);
+				if (i == PRIMARY_PLANE) {
+					u32 reg_clr = BIT(24), reg_set = 0;
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     reg_set, reg_clr);
+
+					reg_clr = (0x1F << 26),
+					reg_set = BIT(8) | (05 << 26);
+					dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+						     reg_set, reg_clr);
+				}
+				mdelay(50);
+				dc_hw_enable_shadow_register(hw, false);
+			}
+
+			if (i == PRIMARY_PLANE) {
+				u32 reg_clr = 0, reg_set = 0;
+
+				reg_clr = (0x1F << 26) | BIT(25) |
+					  (0x03 << 23) | (0x1F << 17) |
+					  (0x07 << 14) | (0x07 << 11) | BIT(0);
+				reg_set = (plane->fb.format << 26) |
+					  (plane->fb.uv_swizzle << 25) |
+					  (plane->fb.swizzle << 23) |
+					  (plane->fb.tile_mode << 17) |
+					  (plane->fb.yuv_color_space << 14) |
+					  (plane->fb.rotation << 11) |
+					  (plane->fb.clear_enable << 8) |
+					  plane->fb.enable;
+				if (plane->fb.enable) {
+					reg_clr |= BIT(8);
+					reg_set |= BIT(4);
+				}
+
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, reg_set,
+					     reg_clr);
+			} else
+				dc_set_clear(
+					hw, DC_OVERLAY_CONFIG,
+					(plane->fb.format << 16) |
+						(plane->fb.uv_swizzle << 15) |
+						(plane->fb.swizzle << 13) |
+						(plane->fb.tile_mode << 8) |
+						(plane->fb.yuv_color_space
+						 << 5) |
+						(plane->fb.rotation << 2) |
+						(plane->fb.enable << 24) |
+						(plane->fb.clear_enable << 25),
+					(0x1F << 16) | BIT(15) | (0x03 << 13) |
+						(0x1F << 8) | (0x07 << 5) |
+						(0x07 << 2) | BIT(24));
+			plane->fb.dirty = false;
+		}
+
+		if (plane->scale.dirty) {
+			if (plane->scale.enable) {
+				dc_write(hw, reg->scale_factor_x,
+					 plane->scale.scale_factor_x);
+				dc_write(hw, reg->scale_factor_y,
+					 plane->scale.scale_factor_y);
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+						     BIT(22), 0);
+				else
+					dc_set_clear(hw,
+						     DC_OVERLAY_SCALE_CONFIG,
+						     BIT(8), 0);
+
+			} else {
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+						     0, BIT(22));
+				else
+					dc_set_clear(hw,
+						     DC_OVERLAY_SCALE_CONFIG, 0,
+						     BIT(8));
+			}
+			plane->scale.dirty = false;
+		}
+
+		if (plane->colorkey.dirty) {
+			dc_write(hw, reg->color_key, plane->colorkey.colorkey);
+			dc_write(hw, reg->color_key_high,
+				 plane->colorkey.colorkey_high);
+
+			if (i == PRIMARY_PLANE)
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG,
+					     plane->colorkey.transparency << 9,
+					     0x03 << 9);
+			else
+				dc_set_clear(hw, DC_OVERLAY_CONFIG,
+					     plane->colorkey.transparency,
+					     0x03);
+
+			plane->colorkey.dirty = false;
+		}
+
+		if (plane->roi.dirty) {
+			if (plane->roi.enable) {
+				dc_write(hw, reg->roi_origin,
+					 plane->roi.x | (plane->roi.y << 16));
+				dc_write(hw, reg->roi_size,
+					 plane->roi.width |
+						 (plane->roi.height << 16));
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(0), 0);
+			} else {
+				// if (i == PRIMARY_PLANE)
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, 0,
+					     BIT(0));
+			}
+			plane->roi.dirty = false;
+		}
+	}
+}
+
+static void plane_ex_commit(struct dc_hw *hw)
+{
+	struct dc_hw_plane *plane;
+	const struct dc_hw_plane_reg *reg;
+	u16 i;
+
+	for (i = 0; i < PLANE_NUM; i++) {
+		if (i == PRIMARY_PLANE) {
+			plane = &hw->primary;
+			reg = &dc_plane_reg[0];
+		} else if (i == OVERLAY_PLANE) {
+			plane = &hw->overlay.plane;
+			reg = &dc_plane_reg[1];
+		} else {
+			continue;
+		}
+
+		if (plane->fb.dirty) {
+			if (is_rgb(plane->fb.format)) {
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     0, BIT(8));
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     0, BIT(30));
+			} else {
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(8), BIT(6));
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     BIT(30), BIT(29));
+				switch (plane->fb.yuv_color_space) {
+				case COLOR_SPACE_601:
+					load_yuv_to_rgb(hw, reg, YUV601_2RGB);
+					break;
+				case COLOR_SPACE_709:
+					load_yuv_to_rgb(hw, reg, YUV709_2RGB);
+					break;
+				case COLOR_SPACE_2020:
+					load_yuv_to_rgb(hw, reg, YUV2020_2RGB);
+					break;
+				default:
+					break;
+				}
+			}
+		}
+		if (plane->degamma.dirty) {
+			switch (plane->degamma.mode) {
+			case ES_DEGAMMA_DISABLE:
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     0, BIT(5));
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG, 0,
+						     BIT(28));
+				break;
+			case ES_DEGAMMA_BT709:
+				load_degamma_table(hw, reg, DEGAMMA_709);
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(5), 0);
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     BIT(28), 0);
+				break;
+			case ES_DEGAMMA_BT2020:
+				load_degamma_table(hw, reg, DEGAMMA_2020);
+				if (i == PRIMARY_PLANE)
+					dc_set_clear(hw,
+						     DC_FRAMEBUFFER_CONFIG_EX,
+						     BIT(5), 0);
+				else
+					dc_set_clear(hw, DC_OVERLAY_CONFIG,
+						     BIT(28), 0);
+				break;
+			default:
+				break;
+			}
+			plane->degamma.dirty = false;
+		}
+	}
+	plane_commit(hw);
+}
+
+static void setup_display(struct dc_hw *hw, struct dc_hw_display *display)
+{
+	u32 dpi_cfg;
+
+	if (hw->display.enable) {
+		switch (display->bus_format) {
+		case MEDIA_BUS_FMT_RGB565_1X16:
+			dpi_cfg = 0;
+			break;
+		case MEDIA_BUS_FMT_RGB666_1X18:
+			dpi_cfg = 3;
+			break;
+		case MEDIA_BUS_FMT_RGB666_1X24_CPADHI:
+			dpi_cfg = 4;
+			break;
+		case MEDIA_BUS_FMT_RGB888_1X24:
+			dpi_cfg = 5;
+			break;
+		case MEDIA_BUS_FMT_RGB101010_1X30:
+			dpi_cfg = 6;
+			break;
+		default:
+			dpi_cfg = 5;
+			break;
+		}
+		dc_write(hw, DC_DISPLAY_DPI_CONFIG, dpi_cfg);
+
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(4));
+
+		dc_write(hw, DC_DISPLAY_H,
+			 hw->display.h_active | (hw->display.h_total << 16));
+		dc_write(hw, DC_DISPLAY_H_SYNC,
+			 hw->display.h_sync_start |
+				 (hw->display.h_sync_end << 15) |
+				 (hw->display.h_sync_polarity ? 0 : BIT(31)) |
+				 BIT(30));
+		dc_write(hw, DC_DISPLAY_V,
+			 hw->display.v_active | (hw->display.v_total << 16));
+		dc_write(hw, DC_DISPLAY_V_SYNC,
+			 hw->display.v_sync_start |
+				 (hw->display.v_sync_end << 15) |
+				 (hw->display.v_sync_polarity ? 0 : BIT(31)) |
+				 BIT(30));
+
+		if (hw->info->pipe_sync) {
+			switch (display->sync_mode) {
+			case ES_SINGLE_DC:
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, 0,
+					     BIT(3) | BIT(4));
+				break;
+			case ES_MULTI_DC_PRIMARY:
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX,
+					     BIT(3) | BIT(4), 0);
+				break;
+			case ES_MULTI_DC_SECONDARY:
+				dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX,
+					     BIT(3), BIT(4));
+				break;
+			default:
+				break;
+			}
+		}
+
+		if (hw->display.dither_enable) {
+			dc_write(hw, DC_DISPLAY_DITHER_TABLE_LOW,
+				 DC_DISPLAY_DITHERTABLE_LOW);
+			dc_write(hw, DC_DISPLAY_DITHER_TABLE_HIGH,
+				 DC_DISPLAY_DITHERTABLE_HIGH);
+			dc_write(hw, DC_DISPLAY_DITHER_CONFIG, BIT(31));
+		} else {
+			dc_write(hw, DC_DISPLAY_DITHER_CONFIG, 0);
+		}
+
+		if (hw->info->background)
+			dc_write(hw, DC_FRAMEBUFFER_BG_COLOR,
+				 hw->display.bg_color);
+
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, BIT(4), 0);
+	} else {
+		dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG, 0, BIT(4) | BIT(0));
+	}
+}
+
+static void setup_display_ex(struct dc_hw *hw, struct dc_hw_display *display)
+{
+	u32 dp_cfg;
+	bool is_yuv = false;
+
+	if (hw->display.enable && hw->out == OUT_DP) {
+		switch (display->bus_format) {
+		case MEDIA_BUS_FMT_RGB565_1X16:
+			dp_cfg = 0;
+			break;
+		case MEDIA_BUS_FMT_RGB666_1X18:
+			dp_cfg = 1;
+			break;
+		case MEDIA_BUS_FMT_RGB888_1X24:
+			dp_cfg = 2;
+			break;
+		case MEDIA_BUS_FMT_RGB101010_1X30:
+			dp_cfg = 3;
+			break;
+		case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+			dp_cfg = 0;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_UYVY8_1X16:
+			dp_cfg = 2 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_YUV8_1X24:
+			dp_cfg = 4 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+			dp_cfg = 6 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_UYVY10_1X20:
+			dp_cfg = 8 << 4;
+			is_yuv = true;
+			break;
+		case MEDIA_BUS_FMT_YUV10_1X30:
+			dp_cfg = 10 << 4;
+			is_yuv = true;
+			break;
+		default:
+			dp_cfg = 2;
+			break;
+		}
+		if (is_yuv)
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, BIT(7), 0);
+		else
+			dc_set_clear(hw, DC_FRAMEBUFFER_CONFIG_EX, 0, BIT(7));
+		dc_write(hw, DC_DISPLAY_DP_CONFIG, dp_cfg | BIT(3));
+	}
+
+	if (hw->out == OUT_DPI)
+		dc_set_clear(hw, DC_DISPLAY_DP_CONFIG, 0, BIT(3));
+
+	setup_display(hw, display);
+}
+
+static const struct dc_hw_funcs hw_func[] = {
+	{
+		.gamma = &gamma_commit,
+		.plane = &plane_commit,
+		.display = setup_display,
+	},
+	{
+		.gamma = &gamma_commit,
+		.plane = &plane_commit,
+		.display = setup_display,
+	},
+	{
+		.gamma = &gamma_ex_commit,
+		.plane = &plane_ex_commit,
+		.display = setup_display_ex,
+	},
+	{
+		.gamma = &gamma_ex_commit,
+		.plane = &plane_ex_commit,
+		.display = setup_display_ex,
+	},
+};
+
+void dc_hw_commit(struct dc_hw *hw)
+{
+	hw->func->gamma(hw);
+	hw->func->plane(hw);
+
+	if (hw->overlay.pos.dirty) {
+		dc_write(hw, DC_OVERLAY_TOP_LEFT,
+			 hw->overlay.pos.start_x |
+				 (hw->overlay.pos.start_y << 15));
+		dc_write(hw, DC_OVERLAY_BOTTOM_RIGHT,
+			 hw->overlay.pos.end_x | (hw->overlay.pos.end_y << 15));
+		hw->overlay.pos.dirty = false;
+	}
+
+	if (hw->overlay.blend.dirty) {
+		dc_write(hw, DC_OVERLAY_SRC_GLOBAL_COLOR,
+			 hw->overlay.blend.alpha << 24);
+		dc_write(hw, DC_OVERLAY_DST_GLOBAL_COLOR,
+			 hw->overlay.blend.alpha << 24);
+		switch (hw->overlay.blend.blend_mode) {
+		case BLEND_PREMULTI:
+			dc_write(hw, DC_OVERLAY_BLEND_CONFIG, 0x3450);
+			break;
+		case BLEND_COVERAGE:
+			dc_write(hw, DC_OVERLAY_BLEND_CONFIG, 0x3950);
+			break;
+		case BLEND_PIXEL_NONE:
+			dc_write(hw, DC_OVERLAY_BLEND_CONFIG, 0x3548);
+			break;
+		default:
+			break;
+		}
+		hw->overlay.blend.dirty = false;
+	}
+
+	if (hw->cursor.dirty) {
+		if (hw->cursor.enable) {
+			dc_write(hw, DC_CURSOR_ADDRESS, hw->cursor.address);
+			dc_write(hw, DC_CURSOR_LOCATION,
+				 hw->cursor.x | (hw->cursor.y << 16));
+			dc_write(hw, DC_CURSOR_CONFIG,
+				  (hw->cursor.hot_x << 16) |
+				  (hw->cursor.hot_y << 8) |
+				  (hw->cursor.size << 5) | 0x06);
+		} else {
+			dc_write(hw, DC_CURSOR_CONFIG, 0x00);
+		}
+		hw->cursor.dirty = false;
+	}
+}
+
+#ifdef CONFIG_ESWIN_MMU
+static u32 mmu_read(struct dc_hw *hw, u32 reg)
+{
+	return readl(hw->mmu_base + reg - MMU_REG_BASE);
+}
+
+static void mmu_write(struct dc_hw *hw, u32 reg, u32 value)
+{
+	writel(value, hw->mmu_base + reg - MMU_REG_BASE);
+}
+
+static void mmu_set_clear(struct dc_hw *hw, u32 reg, u32 set, u32 clear)
+{
+	u32 value = mmu_read(hw, reg);
+
+	value &= ~clear;
+	value |= set;
+	mmu_write(hw, reg, value);
+}
+
+int dc_hw_mmu_init(struct dc_hw *hw, dc_mmu_pt mmu)
+{
+	u32 mtlb = 0, ext_mtlb = 0;
+	u32 safe_addr = 0, ext_safe_addr = 0;
+	u32 config = 0;
+
+	mtlb = (u32)(mmu->mtlb_physical & 0xFFFFFFFF);
+	ext_mtlb = (u32)(mmu->mtlb_physical >> 32);
+
+	/* more than 40bit physical address */
+	if (ext_mtlb & 0xFFFFFF00) {
+		pr_err("Mtlb address out of range.\n");
+		return -EFAULT;
+	}
+
+	config = (ext_mtlb << 20) | (mtlb >> 12);
+	if (mmu->mode == MMU_MODE_1K)
+		mmu_set_clear(hw, MMU_REG_CONTEXT_PD, (config << 4) | BIT(0),
+			      (0xFFFFFFF << 4) | (0x03));
+	else
+		mmu_set_clear(hw, MMU_REG_CONTEXT_PD, (config << 4),
+			      (0xFFFFFFF << 4) | (0x03));
+
+	safe_addr = (u32)(mmu->safe_page_physical & 0xFFFFFFFF);
+	ext_safe_addr = (u32)(mmu->safe_page_physical >> 32);
+
+	if ((safe_addr & 0x3F) || (ext_safe_addr & 0xFFFFFF00)) {
+		pr_err("Invalid safe_address.\n");
+		return -EFAULT;
+	}
+
+	mmu_write(hw, MMU_REG_TABLE_ARRAY_SIZE, 1);
+	mmu_write(hw, MMU_REG_SAFE_SECURE, safe_addr);
+	mmu_write(hw, MMU_REG_SAFE_NON_SECURE, safe_addr);
+
+	mmu_set_clear(hw, MMU_REG_SAFE_EXT_ADDRESS,
+		      (ext_safe_addr << 16) | ext_safe_addr,
+		      BIT(31) | (0xFF << 16) | BIT(15) | 0xFF);
+
+	mmu_write(hw, MMU_REG_CONTROL, BIT(5) | BIT(0));
+
+	mmu_write(hw, DEC_REG_CONTROL, DEC_REG_CONTROL_VALUE);
+
+	return 0;
+}
+
+void dc_hw_enable_mmu_prefetch(struct dc_hw *hw, bool enable)
+{
+	if (enable)
+		dc_write(hw, DC_MMU_PREFETCH, BIT(0));
+	else
+		dc_write(hw, DC_MMU_PREFETCH, 0);
+}
+
+void dc_mmu_flush(struct dc_hw *hw)
+{
+	u32 config, read;
+
+	read = mmu_read(hw, MMU_REG_CONFIG);
+	config = read | BIT(4);
+
+	mmu_write(hw, MMU_REG_CONFIG, config);
+	mmu_write(hw, MMU_REG_CONFIG, read);
+}
+#endif
+
+void dc_hw_update_roi(struct dc_hw *hw, enum dc_hw_plane_id id,
+		      struct dc_hw_roi *roi)
+{
+	struct dc_hw_plane *plane = NULL;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		memcpy(&plane->roi, roi, sizeof(*roi) - sizeof(roi->dirty));
+		plane->roi.dirty = true;
+	}
+}
+
+void dc_hw_update_colorkey(struct dc_hw *hw, enum dc_hw_plane_id id,
+			   struct dc_hw_colorkey *colorkey)
+{
+	struct dc_hw_plane *plane = NULL;
+
+	if (id == PRIMARY_PLANE)
+		plane = &hw->primary;
+	else if (id == OVERLAY_PLANE)
+		plane = &hw->overlay.plane;
+
+	if (plane) {
+		memcpy(&plane->colorkey, colorkey,
+		       sizeof(*colorkey) - sizeof(colorkey->dirty));
+		plane->colorkey.dirty = true;
+	}
+}
diff --git a/drivers/gpu/drm/eswin/es_dc_hw.h b/drivers/gpu/drm/eswin/es_dc_hw.h
new file mode 100644
index 000000000000..1578aad0d79c
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_hw.h
@@ -0,0 +1,481 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_DC_HW_H__
+#define __ES_DC_HW_H__
+
+#ifdef CONFIG_ESWIN_MMU
+#include "es_dc_mmu.h"
+#endif
+
+#define DC_HW_REVISION 0x24
+#define DC_HW_CHIP_CID 0x30
+
+#define DC_REG_BASE 0x1400
+
+#define DC_FRAMEBUFFER_CONFIG 0x1518
+#define DC_FRAMEBUFFER_CONFIG_EX 0x1CC0
+#define DC_FRAMEBUFFER_SCALE_CONFIG 0x1520
+#define DC_FRAMEBUFFER_ADDRESS 0x1400
+#define DC_FRAMEBUFFER_U_ADDRESS 0x1530
+#define DC_FRAMEBUFFER_V_ADDRESS 0x1538
+#define DC_FRAMEBUFFER_STRIDE 0x1408
+#define DC_FRAMEBUFFER_U_STRIDE 0x1800
+#define DC_FRAMEBUFFER_V_STRIDE 0x1808
+#define DC_FRAMEBUFFER_SIZE 0x1810
+#define DC_FRAMEBUFFER_SCALE_FACTOR_X 0x1828
+#define DC_FRAMEBUFFER_SCALE_FACTOR_Y 0x1830
+#define DC_FRAMEBUFFER_H_FILTER_COEF_INDEX 0x1838
+#define DC_FRAMEBUFFER_H_FILTER_COEF_DATA 0x1A00
+#define DC_FRAMEBUFFER_V_FILTER_COEF_INDEX 0x1A08
+#define DC_FRAMEBUFFER_V_FILTER_COEF_DATA 0x1A10
+#define DC_FRAMEBUFFER_INIT_OFFSET 0x1A20
+#define DC_FRAMEBUFFER_COLOR_KEY 0x1508
+#define DC_FRAMEBUFFER_COLOR_KEY_HIGH 0x1510
+#define DC_FRAMEBUFFER_CLEAR_VALUE 0x1A18
+#define DC_FRAMEBUFFER_COLOR_TABLE_INDEX 0x1818
+#define DC_FRAMEBUFFER_COLOR_TABLE_DATA 0x1820
+#define DC_FRAMEBUFFER_BG_COLOR 0x1528
+#define DC_FRAMEBUFFER_ROI_ORIGIN 0x1CB0
+#define DC_FRAMEBUFFER_ROI_SIZE 0x1CB8
+#define DC_FRAMEBUFFER_DEGAMMA_INDEX 0x1D88
+#define DC_FRAMEBUFFER_DEGAMMA_DATA 0x1D90
+#define DC_FRAMEBUFFER_DEGAMMA_EX_DATA 0x1D98
+#define DC_FRAMEBUFFER_YUVTORGB_COEF0 0x1DA0
+#define DC_FRAMEBUFFER_YUVTORGB_COEF1 0x1DA8
+#define DC_FRAMEBUFFER_YUVTORGB_COEF2 0x1DB0
+#define DC_FRAMEBUFFER_YUVTORGB_COEF3 0x1DB8
+#define DC_FRAMEBUFFER_YUVTORGB_COEF4 0x1E00
+#define DC_FRAMEBUFFER_YUVTORGB_COEFD0 0x1E08
+#define DC_FRAMEBUFFER_YUVTORGB_COEFD1 0x1E10
+#define DC_FRAMEBUFFER_YUVTORGB_COEFD2 0x1E18
+#define DC_FRAMEBUFFER_Y_CLAMP_BOUND 0x1E88
+#define DC_FRAMEBUFFER_UV_CLAMP_BOUND 0x1E90
+#define DC_FRAMEBUFFER_RGBTORGB_COEF0 0x1E20
+#define DC_FRAMEBUFFER_RGBTORGB_COEF1 0x1E28
+#define DC_FRAMEBUFFER_RGBTORGB_COEF2 0x1E30
+#define DC_FRAMEBUFFER_RGBTORGB_COEF3 0x1E38
+#define DC_FRAMEBUFFER_RGBTORGB_COEF4 0x1E40
+
+#define DC_OVERLAY_CONFIG 0x1540
+#define DC_OVERLAY_SCALE_CONFIG 0x1C00
+#define DC_OVERLAY_BLEND_CONFIG 0x1580
+#define DC_OVERLAY_TOP_LEFT 0x1640
+#define DC_OVERLAY_BOTTOM_RIGHT 0x1680
+#define DC_OVERLAY_ADDRESS 0x15C0
+#define DC_OVERLAY_U_ADDRESS 0x1840
+#define DC_OVERLAY_V_ADDRESS 0x1880
+#define DC_OVERLAY_STRIDE 0x1600
+#define DC_OVERLAY_U_STRIDE 0x18C0
+#define DC_OVERLAY_V_STRIDE 0x1900
+#define DC_OVERLAY_SIZE 0x17C0
+#define DC_OVERLAY_SCALE_FACTOR_X 0x1A40
+#define DC_OVERLAY_SCALE_FACTOR_Y 0x1A80
+#define DC_OVERLAY_H_FILTER_COEF_INDEX 0x1AC0
+#define DC_OVERLAY_H_FILTER_COEF_DATA 0x1B00
+#define DC_OVERLAY_V_FILTER_COEF_INDEX 0x1B40
+#define DC_OVERLAY_V_FILTER_COEF_DATA 0x1B80
+#define DC_OVERLAY_INIT_OFFSET 0x1BC0
+#define DC_OVERLAY_COLOR_KEY 0x1740
+#define DC_OVERLAY_COLOR_KEY_HIGH 0x1780
+#define DC_OVERLAY_CLEAR_VALUE 0x1940
+#define DC_OVERLAY_COLOR_TABLE_INDEX 0x1980
+#define DC_OVERLAY_COLOR_TABLE_DATA 0x19C0
+#define DC_OVERLAY_SRC_GLOBAL_COLOR 0x16C0
+#define DC_OVERLAY_DST_GLOBAL_COLOR 0x1700
+#define DC_OVERLAY_ROI_ORIGIN 0x1D00
+#define DC_OVERLAY_ROI_SIZE 0x1D40
+#define DC_OVERLAY_DEGAMMA_INDEX 0x2200
+#define DC_OVERLAY_DEGAMMA_DATA 0x2240
+#define DC_OVERLAY_DEGAMMA_EX_DATA 0x2280
+#define DC_OVERLAY_YUVTORGB_COEF0 0x1EC0
+#define DC_OVERLAY_YUVTORGB_COEF1 0x1F00
+#define DC_OVERLAY_YUVTORGB_COEF2 0x1F40
+#define DC_OVERLAY_YUVTORGB_COEF3 0x1F80
+#define DC_OVERLAY_YUVTORGB_COEF4 0x1FC0
+#define DC_OVERLAY_YUVTORGB_COEFD0 0x2000
+#define DC_OVERLAY_YUVTORGB_COEFD1 0x2040
+#define DC_OVERLAY_YUVTORGB_COEFD2 0x2080
+#define DC_OVERLAY_Y_CLAMP_BOUND 0x22C0
+#define DC_OVERLAY_UV_CLAMP_BOUND 0x2300
+#define DC_OVERLAY_RGBTORGB_COEF0 0x20C0
+#define DC_OVERLAY_RGBTORGB_COEF1 0x2100
+#define DC_OVERLAY_RGBTORGB_COEF2 0x2140
+#define DC_OVERLAY_RGBTORGB_COEF3 0x2180
+#define DC_OVERLAY_RGBTORGB_COEF4 0x21C0
+
+#define DC_CURSOR_CONFIG 0x1468
+#define DC_CURSOR_ADDRESS 0x146C
+#define DC_CURSOR_LOCATION 0x1470
+#define DC_CURSOR_BACKGROUND 0x1474
+#define DC_CURSOR_FOREGROUND 0x1478
+#define DC_CURSOR_CLK_GATING 0x1484
+
+#define DC_DISPLAY_DITHER_CONFIG 0x1410
+#define DC_DISPLAY_PANEL_CONFIG 0x1418
+#define DC_DISPLAY_DITHER_TABLE_LOW 0x1420
+#define DC_DISPLAY_DITHER_TABLE_HIGH 0x1428
+#define DC_DISPLAY_H 0x1430
+#define DC_DISPLAY_H_SYNC 0x1438
+#define DC_DISPLAY_V 0x1440
+#define DC_DISPLAY_V_SYNC 0x1448
+#define DC_DISPLAY_CURRENT_LOCATION 0x1450
+#define DC_DISPLAY_GAMMA_INDEX 0x1458
+#define DC_DISPLAY_GAMMA_DATA 0x1460
+#define DC_DISPLAY_INT 0x147C
+#define DC_DISPLAY_INT_ENABLE 0x1480
+#define DC_DISPLAY_DBI_CONFIG 0x1488
+#define DC_DISPLAY_GENERAL_CONFIG 0x14B0
+#define DC_DISPLAY_DPI_CONFIG 0x14B8
+#define DC_DISPLAY_DEBUG_COUNTER_SELECT 0x14D0
+#define DC_DISPLAY_DEBUG_COUNTER_VALUE 0x14D8
+#define DC_DISPLAY_DP_CONFIG 0x1CD0
+#define DC_DISPLAY_GAMMA_EX_INDEX 0x1CF0
+#define DC_DISPLAY_GAMMA_EX_DATA 0x1CF8
+#define DC_DISPLAY_GAMMA_EX_ONE_DATA 0x1D80
+#define DC_DISPLAY_RGBTOYUV_COEF0 0x1E48
+#define DC_DISPLAY_RGBTOYUV_COEF1 0x1E50
+#define DC_DISPLAY_RGBTOYUV_COEF2 0x1E58
+#define DC_DISPLAY_RGBTOYUV_COEF3 0x1E60
+#define DC_DISPLAY_RGBTOYUV_COEF4 0x1E68
+#define DC_DISPLAY_RGBTOYUV_COEFD0 0x1E70
+#define DC_DISPLAY_RGBTOYUV_COEFD1 0x1E78
+#define DC_DISPLAY_RGBTOYUV_COEFD2 0x1E80
+
+#define DC_CLK_GATTING 0x1A28
+
+#define DC_TRANSPARENCY_OPAQUE 0x00
+#define DC_TRANSPARENCY_KEY 0x02
+#define DC_DISPLAY_DITHERTABLE_LOW 0x7B48F3C0
+#define DC_DISPLAY_DITHERTABLE_HIGH 0x596AD1E2
+
+#define GAMMA_SIZE 256
+#define GAMMA_EX_SIZE 300
+#define DEGAMMA_SIZE 260
+
+#define RGB_TO_RGB_TABLE_SIZE 9
+#define YUV_TO_RGB_TABLE_SIZE 16
+#define RGB_TO_YUV_TABLE_SIZE 12
+
+#ifdef CONFIG_ESWIN_MMU
+#define DC_MMU_PREFETCH 0x1E98
+
+#define MMU_REG_BASE 0x0180
+
+#define MMU_REG_CONFIG 0x0184
+#define MMU_REG_CONTROL 0x0388
+#define MMU_REG_TABLE_ARRAY_SIZE 0x0394
+#define MMU_REG_SAFE_NON_SECURE 0x0398
+#define MMU_REG_SAFE_SECURE 0x039C
+#define MMU_REG_SAFE_EXT_ADDRESS 0x03A0
+#define MMU_REG_CONTEXT_PD 0x03B4
+
+#define DEC_REG_CONTROL 0x0800
+#define DEC_REG_CONTROL_VALUE 0x02010188
+#endif
+
+enum dc_hw_chip_rev {
+	DC_REV_5551,
+	DC_REV_5551_306,
+	DC_REV_5701_303,
+	DC_REV_5701_309,
+};
+
+enum dc_hw_plane_id { PRIMARY_PLANE, CURSOR_PLANE, OVERLAY_PLANE, PLANE_NUM };
+
+enum dc_hw_color_format {
+	FORMAT_X4R4G4B4,
+	FORMAT_A4R4G4B4,
+	FORMAT_X1R5G5B5,
+	FORMAT_A1R5G5B5,
+	FORMAT_R5G6B5,
+	FORMAT_X8R8G8B8,
+	FORMAT_A8R8G8B8,
+	FORMAT_YUY2,
+	FORMAT_UYVY,
+	FORMAT_INDEX8,
+	FORMAT_MONOCHROME,
+	FORMAT_YV12 = 0xf,
+	FORMAT_A8,
+	FORMAT_NV12,
+	FORMAT_NV16,
+	FORMAT_RG16,
+	FORMAT_R8,
+	FORMAT_NV12_10BIT,
+	FORMAT_A2R10G10B10,
+	FORMAT_NV16_10BIT,
+	FORMAT_INDEX1,
+	FORMAT_INDEX2,
+	FORMAT_INDEX4,
+	FORMAT_P010,
+};
+
+enum dc_hw_yuv_color_space {
+	COLOR_SPACE_601 = 0,
+	COLOR_SPACE_709 = 1,
+	COLOR_SPACE_2020 = 3,
+};
+
+enum dc_hw_rotation {
+	ROT_0 = 0,
+	ROT_90 = 4,
+	ROT_180 = 5,
+	ROT_270 = 6,
+	FLIP_X = 1,
+	FLIP_Y = 2,
+	FLIP_XY = 3,
+};
+
+enum dc_hw_swizzle {
+	SWIZZLE_ARGB = 0,
+	SWIZZLE_RGBA,
+	SWIZZLE_ABGR,
+	SWIZZLE_BGRA,
+};
+
+enum dc_hw_cursor_size {
+	CURSOR_SIZE_32X32 = 0,
+	CURSOR_SIZE_64X64,
+	CURSOR_SIZE_128X128,
+	CURSOR_SIZE_256X256,
+};
+
+enum dc_hw_out {
+	OUT_DPI,
+	OUT_DP,
+};
+
+enum dc_hw_blend_mode {
+	/* out.rgb = plane_alpha * fg.rgb +
+     *      (1 - (plane_alpha * fg.alpha)) * bg.rgb
+     */
+	BLEND_PREMULTI,
+	/* out.rgb = plane_alpha * fg.alpha * fg.rgb +
+     *      (1 - (plane_alpha * fg.alpha)) * bg.rgb
+     */
+	BLEND_COVERAGE,
+	/* out.rgb = plane_alpha * fg.rgb +
+     *      (1 - plane_alpha) * bg.rgb
+     */
+	BLEND_PIXEL_NONE,
+};
+
+struct dc_hw_plane_reg {
+	u32 y_address;
+	u32 u_address;
+	u32 v_address;
+	u32 y_stride;
+	u32 u_stride;
+	u32 v_stride;
+	u32 size;
+	u32 scale_factor_x;
+	u32 scale_factor_y;
+	u32 h_filter_coef_index;
+	u32 h_filter_coef_data;
+	u32 v_filter_coef_index;
+	u32 v_filter_coef_data;
+	u32 init_offset;
+	u32 color_key;
+	u32 color_key_high;
+	u32 clear_value;
+	u32 color_table_index;
+	u32 color_table_data;
+	u32 scale_config;
+	u32 degamma_index;
+	u32 degamma_data;
+	u32 degamma_ex_data;
+	u32 roi_origin;
+	u32 roi_size;
+	u32 YUVToRGBCoef0;
+	u32 YUVToRGBCoef1;
+	u32 YUVToRGBCoef2;
+	u32 YUVToRGBCoef3;
+	u32 YUVToRGBCoef4;
+	u32 YUVToRGBCoefD0;
+	u32 YUVToRGBCoefD1;
+	u32 YUVToRGBCoefD2;
+	u32 YClampBound;
+	u32 UVClampBound;
+	u32 RGBToRGBCoef0;
+	u32 RGBToRGBCoef1;
+	u32 RGBToRGBCoef2;
+	u32 RGBToRGBCoef3;
+	u32 RGBToRGBCoef4;
+};
+
+struct dc_hw_fb {
+	u32 y_address;
+	u32 u_address;
+	u32 v_address;
+	u32 clear_value;
+	u16 y_stride;
+	u16 u_stride;
+	u16 v_stride;
+	u16 width;
+	u16 height;
+	u8 format;
+	u8 tile_mode;
+	u8 rotation;
+	u8 yuv_color_space;
+	u8 swizzle;
+	u8 uv_swizzle;
+	bool clear_enable;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_scale {
+	u32 scale_factor_x;
+	u32 scale_factor_y;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_position {
+	u16 start_x;
+	u16 start_y;
+	u16 end_x;
+	u16 end_y;
+	bool dirty;
+};
+
+struct dc_hw_blend {
+	u8 alpha;
+	u8 blend_mode;
+	bool dirty;
+};
+
+struct dc_hw_colorkey {
+	u32 colorkey;
+	u32 colorkey_high;
+	u8 transparency;
+	bool dirty;
+};
+
+struct dc_hw_roi {
+	u16 x;
+	u16 y;
+	u16 width;
+	u16 height;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_cursor {
+	u32 address;
+	u16 x;
+	u16 y;
+	u16 hot_x;
+	u16 hot_y;
+	u8 size;
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_display {
+	u32 bus_format;
+	u16 h_active;
+	u16 h_total;
+	u16 h_sync_start;
+	u16 h_sync_end;
+	u16 v_active;
+	u16 v_total;
+	u16 v_sync_start;
+	u16 v_sync_end;
+	u16 sync_mode;
+	u32 bg_color;
+	bool h_sync_polarity;
+	bool v_sync_polarity;
+	bool enable;
+	bool dither_enable;
+};
+
+struct dc_hw_gamma {
+	u16 gamma[GAMMA_EX_SIZE][3];
+	bool enable;
+	bool dirty;
+};
+
+struct dc_hw_degamma {
+	u16 degamma[DEGAMMA_SIZE][3];
+	u32 mode;
+	bool dirty;
+};
+
+struct dc_hw_plane {
+	struct dc_hw_fb fb;
+	struct dc_hw_scale scale;
+	struct dc_hw_roi roi;
+	struct dc_hw_colorkey colorkey;
+	struct dc_hw_degamma degamma;
+};
+
+struct dc_hw_overlay {
+	struct dc_hw_plane plane;
+	struct dc_hw_position pos;
+	struct dc_hw_blend blend;
+};
+
+struct dc_hw_read {
+	u32 reg;
+	u32 value;
+};
+
+struct dc_hw;
+struct dc_hw_funcs {
+	void (*gamma)(struct dc_hw *hw);
+	void (*plane)(struct dc_hw *hw);
+	void (*display)(struct dc_hw *hw, struct dc_hw_display *display);
+};
+
+struct dc_hw {
+	enum dc_hw_chip_rev rev;
+	enum dc_hw_out out;
+	void *hi_base;
+	void *reg_base;
+#ifdef CONFIG_ESWIN_MMU
+	void *mmu_base;
+#endif
+	struct dc_hw_read *read_block;
+	struct dc_hw_display display;
+	struct dc_hw_gamma gamma;
+	struct dc_hw_plane primary;
+	struct dc_hw_overlay overlay;
+	struct dc_hw_cursor cursor;
+	struct dc_hw_funcs *func;
+	struct es_dc_info *info;
+};
+
+int dc_hw_init(struct dc_hw *hw);
+void dc_hw_deinit(struct dc_hw *hw);
+void dc_hw_update_plane(struct dc_hw *hw, enum dc_hw_plane_id id,
+			struct dc_hw_fb *fb, struct dc_hw_scale *scale);
+void dc_hw_update_degamma(struct dc_hw *hw, enum dc_hw_plane_id id, u32 mode);
+void dc_hw_set_position(struct dc_hw *hw, struct dc_hw_position *pos);
+void dc_hw_set_blend(struct dc_hw *hw, struct dc_hw_blend *blend);
+void dc_hw_update_cursor(struct dc_hw *hw, struct dc_hw_cursor *cursor);
+void dc_hw_update_gamma(struct dc_hw *hw, u16 index, u16 r, u16 g, u16 b);
+void dc_hw_enable_gamma(struct dc_hw *hw, bool enable);
+void dc_hw_enable_dump(struct dc_hw *hw, u32 addr, u32 pitch);
+void dc_hw_disable_dump(struct dc_hw *hw);
+void dc_hw_setup_display(struct dc_hw *hw, struct dc_hw_display *display);
+void dc_hw_enable_interrupt(struct dc_hw *hw, bool enable);
+u32 dc_hw_get_interrupt(struct dc_hw *hw);
+bool dc_hw_flip_in_progress(struct dc_hw *hw);
+bool dc_hw_check_underflow(struct dc_hw *hw);
+void dc_hw_enable_shadow_register(struct dc_hw *hw, bool enable);
+void dc_hw_set_out(struct dc_hw *hw, enum dc_hw_out out);
+void dc_hw_commit(struct dc_hw *hw);
+#ifdef CONFIG_ESWIN_MMU
+int dc_hw_mmu_init(struct dc_hw *hw, dc_mmu_pt mmu);
+void dc_hw_enable_mmu_prefetch(struct dc_hw *hw, bool enable);
+void dc_mmu_flush(struct dc_hw *hw);
+#endif
+void dc_hw_update_roi(struct dc_hw *hw, enum dc_hw_plane_id id,
+		      struct dc_hw_roi *roi);
+void dc_hw_update_colorkey(struct dc_hw *hw, enum dc_hw_plane_id id,
+			   struct dc_hw_colorkey *colorkey);
+
+#endif /* __ES_DC_HW_H__ */
diff --git a/drivers/gpu/drm/eswin/es_dc_mmu.c b/drivers/gpu/drm/eswin/es_dc_mmu.c
new file mode 100644
index 000000000000..80cbeb421740
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_mmu.c
@@ -0,0 +1,778 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/vmalloc.h>
+
+#include <asm/io.h>
+#include <asm/delay.h>
+
+#include "es_dc_mmu.h"
+
+static bool mmu_construct = false;
+
+int _allocate_memory(u32 bytes, void **memory)
+{
+	void *mem = NULL;
+
+	if (bytes == 0 || memory == NULL) {
+		pr_err("%s has invalid arguments.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	if (bytes > PAGE_SIZE) {
+		mem = vmalloc(bytes);
+	} else {
+		mem = kmalloc(bytes, GFP_KERNEL);
+	}
+
+	if (!mem) {
+		pr_err("%s out of memory.\n", __FUNCTION__);
+		return -ENOMEM;
+	}
+
+	memset((u8 *)mem, 0, bytes);
+	*memory = mem;
+
+	return 0;
+}
+
+static int _create_mutex(void **mutex)
+{
+	int ret = 0;
+
+	if (mutex == NULL) {
+		return -EINVAL;
+	}
+
+	ret = _allocate_memory(sizeof(struct mutex), mutex);
+	if (ret)
+		return ret;
+
+	mutex_init(*(struct mutex **)mutex);
+
+	return 0;
+}
+
+static int _acquire_mutex(void *mutex, u32 timeout)
+{
+	if (mutex == NULL) {
+		pr_err("%s has invalid argument.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	if (timeout == DC_INFINITE) {
+		mutex_lock(mutex);
+		return 0;
+	}
+
+	for (;;) {
+		/* Try to acquire the mutex. */
+		if (mutex_trylock(mutex)) {
+			/* Success. */
+			return 0;
+		}
+
+		if (timeout-- == 0) {
+			break;
+		}
+
+		/* Wait for 1 millisecond. */
+		udelay(1000);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int _release_mutex(void *mutex)
+{
+	if (mutex == NULL) {
+		pr_err("%s has invalid argument.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	mutex_unlock(mutex);
+
+	return 0;
+}
+
+static u32 _mtlb_offset(u32 address)
+{
+	return (address & MMU_MTLB_MASK) >> MMU_MTLB_SHIFT;
+}
+
+static u32 _stlb_offset(u32 address)
+{
+	return (address & MMU_STLB_4K_MASK) >> MMU_STLB_4K_SHIFT;
+}
+
+static u32 _address_to_index(dc_mmu_pt mmu, u32 address)
+{
+	return _mtlb_offset(address) * MMU_STLB_4K_ENTRY_NUM +
+	       _stlb_offset(address);
+}
+
+static u32 _set_page(u32 page_address, u32 page_address_ext, bool writable)
+{
+	u32 entry = page_address
+		    /* AddressExt */
+		    | (page_address_ext << 4)
+		    /* Ignore exception */
+		    | (0 << 1)
+		    /* Present */
+		    | (1 << 0);
+
+	if (writable) {
+		/* writable */
+		entry |= (1 << 2);
+	}
+
+	return entry;
+}
+
+static void _write_page_entry(u32 *page_entry, u32 entry_value)
+{
+	*page_entry = entry_value;
+}
+
+static u32 _read_page_entry(u32 *page_entry)
+{
+	return *page_entry;
+}
+
+int _allocate_stlb(dc_mmu_stlb_pt *stlb)
+{
+	dc_mmu_stlb_pt stlb_t = NULL;
+	void *mem = NULL;
+
+	mem = kzalloc(sizeof(dc_mmu_stlb), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	stlb_t = (dc_mmu_stlb_pt)mem;
+
+	stlb_t->size = MMU_STLB_4K_SIZE;
+
+	*stlb = stlb_t;
+
+	return 0;
+}
+
+int _allocate_all_stlb(struct device *dev, dc_mmu_stlb_pt *stlb)
+{
+	dc_mmu_stlb_pt stlb_t = NULL;
+	void *mem = NULL;
+	void *cookie = NULL;
+	dma_addr_t dma_addr;
+	size_t size;
+
+	mem = kzalloc(sizeof(dc_mmu_stlb), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	stlb_t = (dc_mmu_stlb_pt)mem;
+
+	stlb_t->size = MMU_STLB_4K_SIZE * MMU_MTLB_ENTRY_NUM;
+	size = PAGE_ALIGN(stlb_t->size);
+
+	cookie = dma_alloc_wc(dev, size, &dma_addr, GFP_KERNEL);
+	if (!cookie) {
+		dev_err(dev, "Failed to alloc stlb buffer.\n");
+		return -ENOMEM;
+	}
+
+	stlb_t->logical = cookie;
+	stlb_t->physBase = (u64)dma_addr;
+	memset(stlb_t->logical, 0, size);
+
+	*stlb = stlb_t;
+
+	return 0;
+}
+
+int _setup_process_address_space(struct device *dev, dc_mmu_pt mmu)
+{
+	u32 *map = NULL;
+	u32 free, i;
+	u32 dynamic_mapping_entries, address;
+	dc_mmu_stlb_pt all_stlb;
+	int ret = 0;
+
+	dynamic_mapping_entries = MMU_MTLB_ENTRY_NUM;
+	mmu->dynamic_mapping_start = 0;
+	mmu->page_table_size = dynamic_mapping_entries * MMU_STLB_4K_SIZE;
+
+	mmu->page_table_entries = mmu->page_table_size / sizeof(u32);
+
+	ret = _allocate_memory(mmu->page_table_size,
+			       (void **)&mmu->map_logical);
+	if (ret) {
+		pr_err("Failed to alloc mmu map buffer.\n");
+		return ret;
+		;
+	}
+
+	ret = _allocate_memory(mmu->page_table_size,
+			       (void **)&mmu->page_entry_info);
+	if (ret) {
+		pr_err("Failed to alloc entry info buffer.\n");
+		return ret;
+		;
+	}
+
+	map = mmu->map_logical;
+
+	/* Initialize free area */
+	free = mmu->page_table_entries;
+	_write_page_entry(map, (free << 8) | DC_MMU_FREE);
+	_write_page_entry(map + 1, ~0U);
+
+	mmu->heap_list = 0;
+	mmu->free_nodes = false;
+
+	ret = _allocate_all_stlb(dev, &all_stlb);
+	if (ret)
+		return ret;
+
+	BUG_ON((all_stlb->physBase & (0xffUL << 32)) !=
+	       (mmu->mtlb_physical & (0xffUL << 32)));
+
+	mmu->stlb_physical = all_stlb->physBase;
+	mmu->stlb_bytes = all_stlb->size;
+
+	for (i = 0; i < dynamic_mapping_entries; i++) {
+		dc_mmu_stlb_pt stlb;
+		dc_mmu_stlb_pt *stlbs = (dc_mmu_stlb_pt *)mmu->stlbs;
+
+		ret = _allocate_stlb(&stlb);
+		if (ret)
+			return ret;
+
+		stlb->physBase = all_stlb->physBase + i * MMU_STLB_4K_SIZE;
+		stlb->logical =
+			all_stlb->logical + i * MMU_STLB_4K_SIZE / sizeof(u32);
+
+		stlbs[i] = stlb;
+	}
+
+	address = (u32)all_stlb->physBase;
+
+	ret = _acquire_mutex(mmu->page_table_mutex, DC_INFINITE);
+	if (ret)
+		return ret;
+
+	for (i = mmu->dynamic_mapping_start;
+	     i < mmu->dynamic_mapping_start + dynamic_mapping_entries; i++) {
+		u32 mtlb_entry;
+
+		mtlb_entry = address | MMU_MTLB_4K_PAGE | MMU_MTLB_PRESENT;
+
+		address += MMU_STLB_4K_SIZE;
+
+		/* Insert Slave TLB address to Master TLB entry.*/
+		_write_page_entry(mmu->mtlb_logical + i, mtlb_entry);
+	}
+
+	kfree(all_stlb);
+	_release_mutex(mmu->page_table_mutex);
+
+	return 0;
+}
+
+/* MMU Construct */
+int dc_mmu_construct(struct device *dev, dc_mmu_pt *mmu)
+{
+	dc_mmu_pt mmu_t = NULL;
+	void *mem = NULL;
+	void *cookie = NULL, *cookie_safe = NULL;
+	dma_addr_t dma_addr, dma_addr_safe;
+	u32 size = 0;
+	int ret = 0;
+
+	if (mmu_construct)
+		return 0;
+
+	mem = kzalloc(sizeof(dc_mmu), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	mmu_t = (dc_mmu_pt)mem;
+	mmu_t->mtlb_bytes = MMU_MTLB_SIZE;
+	size = PAGE_ALIGN(mmu_t->mtlb_bytes);
+	if (size == PAGE_SIZE) {
+		size = PAGE_SIZE * 2;
+	}
+
+	/* Allocate MTLB */
+	cookie = dma_alloc_wc(dev, size, &dma_addr, GFP_KERNEL);
+	if (!cookie) {
+		dev_err(dev, "Failed to alloc mtlb buffer.\n");
+		return -ENOMEM;
+	}
+
+	mmu_t->mtlb_logical = cookie;
+	mmu_t->mtlb_physical = (u64)dma_addr;
+	memset(mmu_t->mtlb_logical, 0, size);
+
+	size = MMU_MTLB_ENTRY_NUM * sizeof(dc_mmu_stlb_pt);
+
+	ret = _allocate_memory(size, &mmu_t->stlbs);
+	if (ret)
+		return ret;
+
+	ret = _create_mutex(&mmu_t->page_table_mutex);
+	if (ret)
+		return ret;
+
+	mmu_t->mode = MMU_MODE_1K;
+
+	ret = _setup_process_address_space(dev, mmu_t);
+	if (ret)
+		return ret;
+
+	/* Allocate safe page */
+	cookie_safe = dma_alloc_wc(dev, 4096, &dma_addr_safe, GFP_KERNEL);
+	if (!cookie_safe) {
+		dev_err(dev, "Failed to alloc safe page.\n");
+		return -ENOMEM;
+	}
+
+	mmu_t->safe_page_logical = cookie_safe;
+	mmu_t->safe_page_physical = (u64)dma_addr_safe;
+	memset(mmu_t->safe_page_logical, 0, size);
+
+	*mmu = mmu_t;
+	mmu_construct = true;
+
+	return 0;
+}
+
+int dc_mmu_get_page_entry(dc_mmu_pt mmu, u32 address, u32 **page_table)
+{
+	dc_mmu_stlb_pt stlb;
+	dc_mmu_stlb_pt *stlbs = (dc_mmu_stlb_pt *)mmu->stlbs;
+	u32 mtlb_offset = _mtlb_offset(address);
+	u32 stlb_offset = _stlb_offset(address);
+
+	stlb = stlbs[mtlb_offset - mmu->dynamic_mapping_start];
+	if (stlb == NULL) {
+		pr_err("BUG: invalid stlb,  mmu=%p stlbs=%p  mtlb_offset=0x%x %s(%d)\n",
+		       mmu, stlbs, mtlb_offset, __FUNCTION__, __LINE__);
+		return -ENXIO;
+	}
+
+	*page_table = &stlb->logical[stlb_offset];
+
+	return 0;
+}
+
+int _link(dc_mmu_pt mmu, u32 index, u32 node)
+{
+	if (index >= mmu->page_table_entries) {
+		mmu->heap_list = node;
+	} else {
+		u32 *map = mmu->map_logical;
+
+		switch (DC_ENTRY_TYPE(_read_page_entry(&map[index]))) {
+		case DC_MMU_SINGLE:
+			/* Previous is a single node, link to it*/
+			_write_page_entry(&map[index],
+					  (node << 8) | DC_MMU_SINGLE);
+			break;
+		case DC_MMU_FREE:
+			/* Link to FREE TYPE node */
+			_write_page_entry(&map[index + 1], node);
+			break;
+		default:
+			pr_err("MMU table corrupted at index %u!", index);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+int _add_free(dc_mmu_pt mmu, u32 index, u32 node, u32 count)
+{
+	u32 *map = mmu->map_logical;
+
+	if (count == 1) {
+		/* Initialize a single page node */
+		_write_page_entry(map + node, DC_SINGLE_PAGE_NODE_INITIALIZE |
+						      DC_MMU_SINGLE);
+	} else {
+		/* Initialize the FREE node*/
+		_write_page_entry(map + node, (count << 8) | DC_MMU_FREE);
+		_write_page_entry(map + node + 1, ~0U);
+	}
+
+	return _link(mmu, index, node);
+}
+
+/* Collect free nodes */
+int _collect(dc_mmu_pt mmu)
+{
+	u32 *map = mmu->map_logical;
+	u32 count = 0, start = 0, i = 0;
+	u32 previous = ~0U;
+	int ret = 0;
+
+	mmu->heap_list = ~0U;
+	mmu->free_nodes = false;
+
+	/* Walk the entire page table */
+	for (i = 0; i < mmu->page_table_entries; i++) {
+		switch (DC_ENTRY_TYPE(_read_page_entry(&map[i]))) {
+		case DC_MMU_SINGLE:
+			if (count++ == 0) {
+				/* Set new start node */
+				start = i;
+			}
+			break;
+		case DC_MMU_FREE:
+			if (count == 0) {
+				/* Set new start node */
+				start = i;
+			}
+
+			count += _read_page_entry(&map[i]) >> 8;
+			/* Advance the index of the page table */
+			i += (_read_page_entry(&map[i]) >> 8) - 1;
+			break;
+		case DC_MMU_USED:
+			/* Meet used node, start to collect */
+			if (count > 0) {
+				/* Add free node to list*/
+				ret = _add_free(mmu, previous, start, count);
+				if (ret)
+					return ret;
+				/* Reset previous unused node index */
+				previous = start;
+				count = 0;
+			}
+			break;
+		default:
+			pr_err("MMU page table corrupted at index %u!", i);
+			return -EINVAL;
+		}
+	}
+
+	/* If left node is an open node. */
+	if (count > 0) {
+		ret = _add_free(mmu, previous, start, count);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int _fill_page_table(u32 *page_table, u32 page_count, u32 entry_value)
+{
+	u32 i;
+
+	for (i = 0; i < page_count; i++) {
+		_write_page_entry(page_table + i, entry_value);
+	}
+
+	return 0;
+}
+
+int dc_mmu_allocate_pages(dc_mmu_pt mmu, u32 page_count, u32 *address)
+{
+	bool got = false, acquired = false;
+	u32 *map;
+	u32 index = 0, vaddr, left;
+	u32 previous = ~0U;
+	u32 mtlb_offset, stlb_offset;
+	int ret = 0;
+
+	if (page_count == 0 || page_count > mmu->page_table_entries) {
+		pr_err("%s has invalid arguments.\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	_acquire_mutex(mmu->page_table_mutex, DC_INFINITE);
+	acquired = true;
+
+	for (map = mmu->map_logical; !got;) {
+		for (index = mmu->heap_list;
+		     !got && (index < mmu->page_table_entries);) {
+			switch (DC_ENTRY_TYPE(_read_page_entry(&map[index]))) {
+			case DC_MMU_SINGLE:
+				if ((index != 0) && (page_count == 1)) {
+					got = true;
+				} else {
+					/* Move to next node */
+					previous = index;
+					index = _read_page_entry(&map[index]) >>
+						8;
+				}
+				break;
+			case DC_MMU_FREE:
+				if ((index == 0) &&
+				    (page_count <
+				     ((_read_page_entry(&map[index]) >> 8)))) {
+					got = true;
+				} else if ((index != 0) &&
+					   (page_count <=
+					    ((_read_page_entry(&map[index]) >>
+					      8)))) {
+					got = true;
+				} else {
+					/* Move to next node */
+					previous = index;
+					index = _read_page_entry(
+						&map[index + 1]);
+				}
+				break;
+			default:
+				/* Only link SINGLE and FREE node */
+				pr_err("MMU table corrupted at index %u!",
+				       index);
+				ret = -EINVAL;
+				goto OnError;
+			}
+		}
+
+		/* If out of index */
+		if (index >= mmu->page_table_entries) {
+			if (mmu->free_nodes) {
+				/* Collect the free node */
+				ret = _collect(mmu);
+				if (ret)
+					goto OnError;
+				previous = ~0U;
+			} else {
+				ret = -ENODATA;
+				goto OnError;
+			}
+		}
+	}
+
+	switch (DC_ENTRY_TYPE(_read_page_entry(&map[index]))) {
+	case DC_MMU_SINGLE:
+		/* Unlink single node from node list */
+		ret = _link(mmu, previous, _read_page_entry(&map[index]) >> 8);
+		if (ret)
+			goto OnError;
+		break;
+
+	case DC_MMU_FREE:
+		left = (_read_page_entry(&map[index]) >> 8) - page_count;
+		switch (left) {
+		case 0:
+			/* Unlink the entire FREE type node */
+			ret = _link(mmu, previous,
+				    _read_page_entry(&map[index + 1]));
+			if (ret)
+				goto OnError;
+			break;
+		case 1:
+			/* Keep the map[index] as a single node,
+             * mark the left as used
+             */
+			_write_page_entry(&map[index],
+					  (_read_page_entry(&map[index + 1])
+					   << 8) | DC_MMU_SINGLE);
+			index++;
+			break;
+		default:
+			/* FREE type node left */
+			_write_page_entry(&map[index],
+					  (left << 8) | DC_MMU_FREE);
+			index += left;
+			if (index + page_count > mmu->page_table_entries) {
+				pr_err("Error: index %d , left %d\n", index,
+				       left);
+				ret = -EINVAL;
+				goto OnError;
+			}
+
+			break;
+		}
+		break;
+	default:
+		/* Only link SINGLE and FREE node */
+		pr_err("MMU table corrupted at index %u!", index);
+		ret = -EINVAL;
+		goto OnError;
+	}
+
+	if (index + page_count > mmu->page_table_entries) {
+		pr_err("Fatal ERROR: This may caused by free pages more than allocated pages\n");
+		_write_page_entry(mmu->map_logical,
+				  (mmu->page_table_entries << 8) | DC_MMU_FREE);
+		_write_page_entry(mmu->map_logical + 1, ~0U);
+		previous = ~0U;
+		mmu->heap_list = 0;
+		memset(mmu->page_entry_info, 0,
+		       mmu->page_table_entries * sizeof(u32));
+
+		ret = -EINVAL;
+		goto OnError;
+	}
+
+	/* Mark node as used */
+	ret = _fill_page_table(&map[index], page_count, DC_MMU_USED);
+	if (ret)
+		goto OnError;
+
+	_release_mutex(mmu->page_table_mutex);
+
+	mtlb_offset =
+		index / MMU_STLB_4K_ENTRY_NUM + mmu->dynamic_mapping_start;
+	stlb_offset = index % MMU_STLB_4K_ENTRY_NUM;
+
+	vaddr = (mtlb_offset << MMU_MTLB_SHIFT) |
+		(stlb_offset << MMU_STLB_4K_SHIFT);
+
+	if (vaddr == 0) {
+		pr_err("Error occured on alloc vaddr\n");
+		ret = -ENODATA;
+		goto OnError;
+	}
+
+	mmu->page_entry_info[index] = (index << 12) | (page_count & 0xfff);
+
+	if (address != NULL) {
+		*address = vaddr;
+	}
+
+	return 0;
+
+OnError:
+	if (acquired) {
+		_release_mutex(mmu->page_table_mutex);
+	}
+
+	return ret;
+}
+
+int dc_mmu_free_pages(dc_mmu_pt mmu, u32 address, u32 page_count)
+{
+	u32 *node, index;
+
+	if (page_count == 0)
+		return -EINVAL;
+
+	index = _address_to_index(mmu, address);
+	node = mmu->map_logical + _address_to_index(mmu, address);
+
+	_acquire_mutex(mmu->page_table_mutex, DC_INFINITE);
+
+	if (((mmu->page_entry_info[index] & 0xfff) != (page_count & 0xfff)) ||
+	    ((mmu->page_entry_info[index] >> 12) != index)) {
+		pr_err("index %d, page_count %d: free pages not equal to alloc\n",
+		       index, page_count);
+		_release_mutex(mmu->page_table_mutex);
+		return -EINVAL;
+	} else {
+		mmu->page_entry_info[index] = 0;
+	}
+
+	if (page_count == 1) {
+		/* Mark the Single page node free */
+		_write_page_entry(node, DC_SINGLE_PAGE_NODE_INITIALIZE |
+						DC_MMU_SINGLE);
+	} else {
+		/* Mark the FREE type node free */
+		_write_page_entry(node, (page_count << 8) | DC_MMU_FREE);
+		_write_page_entry(node + 1, ~0U);
+	}
+
+	mmu->free_nodes = true;
+
+	_release_mutex(mmu->page_table_mutex);
+
+	return 0;
+}
+
+int dc_mmu_set_page(dc_mmu_pt mmu, u64 page_address, u32 *page_entry)
+{
+	u32 address_ext;
+	u32 address;
+
+	if (page_entry == NULL || (page_address & 0xFFF)) {
+		return -EINVAL;
+	}
+
+	/* [31:0]. */
+	address = (u32)(page_address & 0xFFFFFFFF);
+	/* [39:32]. */
+	address_ext = (u32)((page_address >> 32) & 0xFF);
+
+	_write_page_entry(page_entry, _set_page(address, address_ext, true));
+
+	return 0;
+}
+
+int dc_mmu_map_memory(dc_mmu_pt mmu, u64 physical, u32 page_count, u32 *address,
+		      bool continuous)
+{
+	u32 virutal_address, i = 0;
+	bool allocated = false;
+	int ret = 0;
+
+	ret = dc_mmu_allocate_pages(mmu, page_count, &virutal_address);
+	if (ret)
+		goto OnError;
+
+	if (continuous)
+		*address = virutal_address | (physical & MMU_PAGE_4K_MASK);
+	else
+		*address = virutal_address;
+
+	allocated = true;
+
+	/* Fill in page table */
+	for (i = 0; i < page_count; i++) {
+		u64 page_phy;
+		u32 *page_entry;
+		struct page **pages;
+
+		if (continuous == true) {
+			page_phy = physical + i * MMU_PAGE_4K_SIZE;
+		} else {
+			pages = (struct page **)physical;
+			page_phy = page_to_phys(pages[i]);
+		}
+
+		ret = dc_mmu_get_page_entry(mmu, virutal_address, &page_entry);
+		if (ret)
+			goto OnError;
+
+		/* Write the page address to the page entry */
+		ret = dc_mmu_set_page(mmu, page_phy, page_entry);
+		if (ret)
+			goto OnError;
+
+		/* Get next page */
+		virutal_address += MMU_PAGE_4K_SIZE;
+	}
+
+	wmb();
+
+	return 0;
+
+OnError:
+	if (allocated)
+		dc_mmu_free_pages(mmu, virutal_address, page_count);
+	pr_info("%s fail!\n", __FUNCTION__);
+
+	return ret;
+}
+
+int dc_mmu_unmap_memory(dc_mmu_pt mmu, u32 gpu_address, u32 page_count)
+{
+	return dc_mmu_free_pages(mmu, gpu_address, page_count);
+}
diff --git a/drivers/gpu/drm/eswin/es_dc_mmu.h b/drivers/gpu/drm/eswin/es_dc_mmu.h
new file mode 100644
index 000000000000..ee049e3f40ec
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_dc_mmu.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef _ES_DC_MMU_H_
+#define _ES_DC_MMU_H_
+
+#include "es_type.h"
+
+#define DC_INFINITE ((u32)(~0U))
+
+#define DC_ENTRY_TYPE(x) (x & 0xF0)
+#define DC_SINGLE_PAGE_NODE_INITIALIZE (~((1U << 8) - 1))
+
+#define DC_INVALID_PHYSICAL_ADDRESS ~0ULL
+#define DC_INVALID_ADDRESS ~0U
+
+/* 1k mode */
+#define MMU_MTLB_SHIFT 24
+#define MMU_STLB_4K_SHIFT 12
+
+#define MMU_MTLB_BITS (32 - MMU_MTLB_SHIFT)
+#define MMU_PAGE_4K_BITS MMU_STLB_4K_SHIFT
+#define MMU_STLB_4K_BITS (32 - MMU_MTLB_BITS - MMU_PAGE_4K_BITS)
+
+#define MMU_MTLB_ENTRY_NUM (1 << MMU_MTLB_BITS)
+#define MMU_MTLB_SIZE (MMU_MTLB_ENTRY_NUM << 2)
+#define MMU_STLB_4K_ENTRY_NUM (1 << MMU_STLB_4K_BITS)
+#define MMU_STLB_4K_SIZE (MMU_STLB_4K_ENTRY_NUM << 2)
+#define MMU_PAGE_4K_SIZE (1 << MMU_STLB_4K_SHIFT)
+
+#define MMU_MTLB_MASK (~((1U << MMU_MTLB_SHIFT) - 1))
+#define MMU_STLB_4K_MASK ((~0U << MMU_STLB_4K_SHIFT) ^ MMU_MTLB_MASK)
+#define MMU_PAGE_4K_MASK (MMU_PAGE_4K_SIZE - 1)
+
+/* page offset definitions. */
+#define MMU_OFFSET_4K_BITS (32 - MMU_MTLB_BITS - MMU_STLB_4K_BITS)
+#define MMU_OFFSET_4K_MASK ((1U << MMU_OFFSET_4K_BITS) - 1)
+
+#define MMU_MTLB_PRESENT 0x00000001
+#define MMU_MTLB_EXCEPTION 0x00000002
+#define MMU_MTLB_4K_PAGE 0x00000000
+
+typedef enum _dc_mmu_type {
+	DC_MMU_USED = (0 << 4),
+	DC_MMU_SINGLE = (1 << 4),
+	DC_MMU_FREE = (2 << 4),
+} dc_mmu_type;
+
+typedef enum _dc_mmu_mode {
+	MMU_MODE_1K,
+	MMU_MODE_4K,
+} dc_mmu_mode;
+
+typedef struct _dc_mmu_stlb {
+	u32 *logical;
+	void *physical;
+	u32 size;
+	u64 physBase;
+	u32 pageCount;
+} dc_mmu_stlb, *dc_mmu_stlb_pt;
+
+typedef struct _dc_mmu {
+	u32 mtlb_bytes;
+	u64 mtlb_physical;
+	u32 *mtlb_logical;
+
+	void *safe_page_logical;
+	u64 safe_page_physical;
+
+	u32 dynamic_mapping_start;
+
+	void *stlbs;
+
+	// u64    stlb_physicals[MMU_MTLB_ENTRY_NUM];
+	u32 stlb_bytes;
+	u64 stlb_physical;
+
+	u32 page_table_entries;
+	u32 page_table_size;
+	u32 heap_list;
+
+	u32 *map_logical;
+	u32 *page_entry_info;
+	bool free_nodes;
+
+	void *page_table_mutex;
+
+	dc_mmu_mode mode;
+
+	void *static_stlb;
+} dc_mmu, *dc_mmu_pt;
+
+int dc_mmu_construct(struct device *dev, dc_mmu_pt *mmu);
+int dc_mmu_map_memory(dc_mmu_pt mmu, u64 physical, u32 page_count, u32 *address,
+		      bool continuous);
+int dc_mmu_unmap_memory(dc_mmu_pt mmu, u32 gpu_address, u32 page_count);
+
+#endif /* _ES_DC_MMU_H_ */
diff --git a/drivers/gpu/drm/eswin/es_drm.h b/drivers/gpu/drm/eswin/es_drm.h
new file mode 100644
index 000000000000..73817dcf024b
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_drm.h
@@ -0,0 +1,23 @@
+#ifndef __ES_DRM_H__
+#define __ES_DRM_H__
+
+#include <drm/drm.h>
+
+enum drm_es_degamma_mode {
+	ES_DEGAMMA_DISABLE = 0,
+	ES_DEGAMMA_BT709 = 1,
+	ES_DEGAMMA_BT2020 = 2,
+};
+
+enum drm_es_sync_dc_mode {
+	ES_SINGLE_DC = 0,
+	ES_MULTI_DC_PRIMARY = 1,
+	ES_MULTI_DC_SECONDARY = 2,
+};
+
+enum drm_es_mmu_prefetch_mode {
+	ES_MMU_PREFETCH_DISABLE = 0,
+	ES_MMU_PREFETCH_ENABLE = 1,
+};
+
+#endif /* __ES_DRM_H__ */
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/es_drv.c b/drivers/gpu/drm/eswin/es_drv.c
new file mode 100644
index 000000000000..2596e8e492a3
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_drv.c
@@ -0,0 +1,527 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/of_graph.h>
+#include <linux/component.h>
+#include <linux/iommu.h>
+#include <linux/version.h>
+#include <linux/of_address.h>
+#include <linux/dma-map-ops.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_of.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_debugfs.h>
+#include <drm/drm_vblank.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_fbdev_generic.h>
+
+#include "es_drv.h"
+#include "es_fb.h"
+#include "es_gem.h"
+#include "es_plane.h"
+#include "es_crtc.h"
+#include "es_simple_enc.h"
+#include "es_dc.h"
+#include "es_virtual.h"
+#ifdef CONFIG_ESWIN_DW_HDMI
+#include "dw-hdmi.h"
+#endif
+
+#include "es_mipi_dsi.h"
+
+#define DRV_NAME "es_drm"
+#define DRV_DESC "Eswin DRM driver"
+#define DRV_DATE "20191101"
+#define DRV_MAJOR 1
+#define DRV_MINOR 0
+
+static bool has_iommu = true;
+static struct platform_driver es_drm_platform_driver;
+
+static const struct file_operations fops = {
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+	.compat_ioctl = drm_compat_ioctl,
+	.poll = drm_poll,
+	.read = drm_read,
+	.mmap = es_gem_mmap,
+};
+
+#ifdef CONFIG_DEBUG_FS
+static int es_debugfs_planes_show(struct seq_file *s, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *)s->private;
+	struct drm_device *dev = node->minor->dev;
+	struct drm_plane *plane = NULL;
+
+	list_for_each_entry (plane, &dev->mode_config.plane_list, head) {
+		struct drm_plane_state *state = plane->state;
+		struct es_plane_state *plane_state = to_es_plane_state(state);
+
+		seq_printf(s, "plane[%u]: %s\n", plane->base.id, plane->name);
+		seq_printf(s, "\tcrtc = %s\n",
+			   state->crtc ? state->crtc->name : "(null)");
+		seq_printf(s, "\tcrtc id = %u\n",
+			   state->crtc ? state->crtc->base.id : 0);
+		seq_printf(s, "\tcrtc-pos = " DRM_RECT_FMT "\n",
+			   DRM_RECT_ARG(&plane_state->status.dest));
+		seq_printf(s, "\tsrc-pos = " DRM_RECT_FP_FMT "\n",
+			   DRM_RECT_FP_ARG(&plane_state->status.src));
+		seq_printf(s, "\tformat = %p4cc\n",
+			   state->fb ? &state->fb->format->format : NULL);
+		seq_printf(s, "\trotation = 0x%x\n", state->rotation);
+		seq_printf(s, "\ttiling = %u\n", plane_state->status.tile_mode);
+
+		seq_puts(s, "\n");
+	}
+
+	return 0;
+}
+
+static struct drm_info_list es_debugfs_list[] = {
+	{ "planes", es_debugfs_planes_show, 0, NULL },
+};
+
+static void es_debugfs_init(struct drm_minor *minor)
+{
+	drm_debugfs_create_files(es_debugfs_list, ARRAY_SIZE(es_debugfs_list),
+				 minor->debugfs_root, minor);
+}
+#endif
+
+static struct drm_driver es_drm_driver = {
+	.driver_features =
+		DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_GEM | DRIVER_SYNCOBJ,
+	.lastclose = drm_fb_helper_lastclose,
+	.gem_prime_import = es_gem_prime_import,
+	.gem_prime_import_sg_table = es_gem_prime_import_sg_table,
+	.dumb_create = es_gem_dumb_create,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init = es_debugfs_init,
+#endif
+	.fops = &fops,
+	.name = DRV_NAME,
+	.desc = DRV_DESC,
+	.date = DRV_DATE,
+	.major = DRV_MAJOR,
+	.minor = DRV_MINOR,
+};
+
+int es_drm_iommu_attach_device(struct drm_device *drm_dev, struct device *dev)
+{
+	struct es_drm_private *priv = drm_dev->dev_private;
+	int ret;
+
+	if (!has_iommu)
+		return 0;
+
+	if (!priv->domain) {
+		priv->domain = iommu_get_domain_for_dev(dev);
+		if (IS_ERR(priv->domain))
+			return PTR_ERR(priv->domain);
+		priv->dma_dev = dev;
+	}
+
+	ret = iommu_attach_device(priv->domain, dev);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "Failed to attach iommu device\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+void es_drm_iommu_detach_device(struct drm_device *drm_dev, struct device *dev)
+{
+	struct es_drm_private *priv = drm_dev->dev_private;
+
+	if (!has_iommu)
+		return;
+
+	iommu_detach_device(priv->domain, dev);
+
+	if (priv->dma_dev == dev)
+		priv->dma_dev = drm_dev->dev;
+}
+
+void es_drm_update_pitch_alignment(struct drm_device *drm_dev,
+				   unsigned int alignment)
+{
+	struct es_drm_private *priv = drm_dev->dev_private;
+
+	if (alignment > priv->pitch_alignment)
+		priv->pitch_alignment = alignment;
+}
+
+#ifdef CONFIG_ESWIN_DW_HDMI
+static int es_drm_create_properties(struct drm_device *dev)
+{
+	struct drm_property *prop;
+	struct es_drm_private *private = dev->dev_private;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC, "EOTF", 0,
+					 5);
+	if (!prop)
+		return -ENOMEM;
+	private->eotf_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "COLOR_SPACE", 0, 12);
+	if (!prop)
+		return -ENOMEM;
+	private->color_space_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "GLOBAL_ALPHA", 0, 255);
+	if (!prop)
+		return -ENOMEM;
+	private->global_alpha_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "BLEND_MODE", 0, 1);
+	if (!prop)
+		return -ENOMEM;
+	private->blend_mode_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "ALPHA_SCALE", 0, 1);
+	if (!prop)
+		return -ENOMEM;
+	private->alpha_scale_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "ASYNC_COMMIT", 0, 1);
+	if (!prop)
+		return -ENOMEM;
+	private->async_commit_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC, "SHARE_ID",
+					 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	private->share_id_prop = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+					 "CONNECTOR_ID", 0, 0xf);
+	if (!prop)
+		return -ENOMEM;
+	private->connector_id_prop = prop;
+
+	return drm_mode_create_tv_properties(dev, 0);
+}
+#endif
+
+static int es_drm_bind(struct device *dev)
+{
+	struct drm_device *drm_dev;
+	struct es_drm_private *priv;
+	int ret;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
+	static u64 dma_mask = DMA_BIT_MASK(40);
+#else
+	static u64 dma_mask = DMA_40BIT_MASK;
+#endif
+
+	drm_dev = drm_dev_alloc(&es_drm_driver, dev);
+	if (IS_ERR(drm_dev))
+		return PTR_ERR(drm_dev);
+
+	dev_set_drvdata(dev, drm_dev);
+
+	priv = devm_kzalloc(drm_dev->dev, sizeof(struct es_drm_private),
+			    GFP_KERNEL);
+	if (!priv) {
+		ret = -ENOMEM;
+		goto err_put_dev;
+	}
+
+	priv->pitch_alignment = 64;
+	priv->dma_dev = drm_dev->dev;
+	priv->dma_dev->coherent_dma_mask = dma_mask;
+	dma_set_mask_and_coherent(priv->dma_dev, DMA_BIT_MASK(40));
+
+	drm_dev->dev_private = priv;
+
+	drm_mode_config_init(drm_dev);
+
+#ifdef CONFIG_ESWIN_DW_HDMI
+	es_drm_create_properties(drm_dev);
+#endif
+
+	/* Now try and bind all our sub-components */
+	ret = component_bind_all(dev, drm_dev);
+	if (ret)
+		goto err_mode;
+
+	es_mode_config_init(drm_dev);
+
+	ret = drm_vblank_init(drm_dev, drm_dev->mode_config.num_crtc);
+	if (ret)
+		goto err_bind;
+
+	drm_mode_config_reset(drm_dev);
+
+	drm_kms_helper_poll_init(drm_dev);
+
+	ret = drm_dev_register(drm_dev, 0);
+	if (ret)
+		goto err_helper;
+
+	drm_fbdev_generic_setup(drm_dev, 32);
+
+	return 0;
+
+err_helper:
+	drm_kms_helper_poll_fini(drm_dev);
+err_bind:
+	component_unbind_all(drm_dev->dev, drm_dev);
+err_mode:
+	drm_mode_config_cleanup(drm_dev);
+	if (priv->domain)
+		iommu_domain_free(priv->domain);
+err_put_dev:
+	drm_dev->dev_private = NULL;
+	dev_set_drvdata(dev, NULL);
+	drm_dev_put(drm_dev);
+	return ret;
+}
+
+static void es_drm_unbind(struct device *dev)
+{
+	struct drm_device *drm_dev = dev_get_drvdata(dev);
+	struct es_drm_private *priv = drm_dev->dev_private;
+
+	drm_dev_unregister(drm_dev);
+	drm_kms_helper_poll_fini(drm_dev);
+	drm_atomic_helper_shutdown(drm_dev);
+	component_unbind_all(drm_dev->dev, drm_dev);
+	drm_mode_config_cleanup(drm_dev);
+
+	if (priv->domain) {
+		iommu_domain_free(priv->domain);
+		priv->domain = NULL;
+	}
+
+	drm_dev->dev_private = NULL;
+	dev_set_drvdata(dev, NULL);
+	drm_dev_put(drm_dev);
+}
+
+static const struct component_master_ops es_drm_ops = {
+	.bind = es_drm_bind,
+	.unbind = es_drm_unbind,
+};
+
+static struct platform_driver *drm_sub_drivers[] = {
+	/* put display control driver at start */
+	&dc_platform_driver,
+
+/* connector */
+
+/* bridge */
+#if 1
+#ifdef CONFIG_ESWIN_DW_HDMI
+	&dw_hdmi_eswin_pltfm_driver,
+#endif
+#ifdef CONFIG_DW_HDMI_I2S_AUDIO
+	&snd_dw_hdmi_driver,
+#endif
+
+#ifdef CONFIG_DW_HDMI_CEC
+	&dw_hdmi_cec_driver,
+#endif
+
+#ifdef CONFIG_DW_HDMI_HDCP
+	&dw_hdmi_hdcp_driver,
+#endif
+#endif
+
+#ifdef CONFIG_ESWIN_VIRTUAL_DISPLAY
+	&virtual_display_platform_driver,
+#endif
+
+#ifdef CONFIG_ESWIN_MIPI_DSI
+	&es_mipi_dsi_driver,
+#endif
+
+};
+#define NUM_DRM_DRIVERS                                                        \
+	(sizeof(drm_sub_drivers) / sizeof(struct platform_driver *))
+
+static int compare_dev(struct device *dev, void *data)
+{
+	return dev == (struct device *)data;
+}
+
+static struct component_match *es_drm_match_add(struct device *dev)
+{
+	struct component_match *match = NULL;
+	int i;
+
+	for (i = 0; i < NUM_DRM_DRIVERS; ++i) {
+		struct platform_driver *drv = drm_sub_drivers[i];
+		struct device *p = NULL, *d;
+
+		while ((d = platform_find_device_by_driver(p, &drv->driver))) {
+			put_device(p);
+
+			component_match_add(dev, &match, compare_dev, d);
+			p = d;
+		}
+		put_device(p);
+	}
+
+	return match ?: ERR_PTR(-ENODEV);
+}
+
+static int es_drm_platform_of_probe(struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	struct device_node *port;
+	bool found = false;
+	int i;
+
+	if (!np)
+		return -ENODEV;
+
+	for (i = 0;; i++) {
+		struct device_node *iommu;
+
+		port = of_parse_phandle(np, "ports", i);
+		if (!port)
+			break;
+
+		if (!of_device_is_available(port->parent)) {
+			of_node_put(port);
+			continue;
+		}
+
+		iommu = of_parse_phandle(port->parent, "iommus", 0);
+
+		/*
+         * if there is a crtc not support iommu, force set all
+         * crtc use non-iommu buffer.
+         */
+		if (!iommu || !of_device_is_available(iommu->parent))
+			has_iommu = false;
+
+		found = true;
+
+		of_node_put(iommu);
+		of_node_put(port);
+	}
+
+	if (i == 0) {
+		DRM_DEV_ERROR(dev, "missing 'ports' property\n");
+		return -ENODEV;
+	}
+
+	if (!found) {
+		DRM_DEV_ERROR(dev, "No available DC found.\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int es_drm_platform_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct component_match *match;
+	int ret;
+
+	ret = es_drm_platform_of_probe(dev);
+	if (ret)
+		return ret;
+
+	match = es_drm_match_add(dev);
+	if (IS_ERR(match))
+		return PTR_ERR(match);
+
+	return component_master_add_with_match(dev, &es_drm_ops, match);
+}
+
+static int es_drm_platform_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &es_drm_ops);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int es_drm_suspend(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int es_drm_resume(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_resume(drm);
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(es_drm_pm_ops, es_drm_suspend, es_drm_resume);
+
+static const struct of_device_id es_drm_dt_ids[] = {
+
+	{
+		.compatible = "eswin,display-subsystem",
+	},
+
+	{ /* sentinel */ },
+
+};
+
+MODULE_DEVICE_TABLE(of, es_drm_dt_ids);
+
+static struct platform_driver es_drm_platform_driver = {
+    .probe = es_drm_platform_probe,
+    .remove = es_drm_platform_remove,
+
+    .driver = {
+        .name = DRV_NAME,
+        .of_match_table = es_drm_dt_ids,
+        .pm = &es_drm_pm_ops,
+    },
+};
+
+static int __init es_drm_init(void)
+{
+	int ret;
+
+	ret = platform_register_drivers(drm_sub_drivers, NUM_DRM_DRIVERS);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&es_drm_platform_driver);
+	if (ret)
+		platform_unregister_drivers(drm_sub_drivers, NUM_DRM_DRIVERS);
+
+	return ret;
+}
+
+static void __exit es_drm_fini(void)
+{
+	platform_driver_unregister(&es_drm_platform_driver);
+	platform_unregister_drivers(drm_sub_drivers, NUM_DRM_DRIVERS);
+}
+
+module_init(es_drm_init);
+module_exit(es_drm_fini);
+
+MODULE_DESCRIPTION("Eswin DRM Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_drv.h b/drivers/gpu/drm/eswin/es_drv.h
new file mode 100644
index 000000000000..85465f559de0
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_drv.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_DRV_H__
+#define __ES_DRV_H__
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include <drm/drm_gem.h>
+#include <drm/drm_device.h>
+
+#include "es_plane.h"
+#ifdef CONFIG_ESWIN_MMU
+#include "es_dc_mmu.h"
+#endif
+
+/*
+ *
+ * @dma_dev: device for DMA API.
+ *  - use the first attached device if support iommu
+    else use drm device (only contiguous buffer support)
+ * @domain: iommu domain for DRM.
+ *  - all DC IOMMU share same domain to reduce mapping
+ * @pitch_alignment: buffer pitch alignment required by sub-devices.
+ *
+ */
+struct es_drm_private {
+	struct device *dma_dev;
+	struct iommu_domain *domain;
+#ifdef CONFIG_ESWIN_DW_HDMI
+	struct drm_property *connector_id_prop;
+	struct drm_property *eotf_prop;
+	struct drm_property *color_space_prop;
+	struct drm_property *global_alpha_prop;
+	struct drm_property *blend_mode_prop;
+	struct drm_property *alpha_scale_prop;
+	struct drm_property *async_commit_prop;
+	struct drm_property *share_id_prop;
+#endif
+
+#ifdef CONFIG_ESWIN_MMU
+	dc_mmu *mmu;
+#endif
+
+	unsigned int pitch_alignment;
+};
+
+int es_drm_iommu_attach_device(struct drm_device *drm_dev, struct device *dev);
+
+void es_drm_iommu_detach_device(struct drm_device *drm_dev, struct device *dev);
+
+void es_drm_update_pitch_alignment(struct drm_device *drm_dev,
+				   unsigned int alignment);
+
+static inline struct device *to_dma_dev(struct drm_device *dev)
+{
+	struct es_drm_private *priv = dev->dev_private;
+
+	return priv->dma_dev;
+}
+
+static inline bool is_iommu_enabled(struct drm_device *dev)
+{
+	struct es_drm_private *priv = dev->dev_private;
+
+	return priv->domain != NULL ? true : false;
+}
+#endif /* __ES_DRV_H__ */
diff --git a/drivers/gpu/drm/eswin/es_fb.c b/drivers/gpu/drm/eswin/es_fb.c
new file mode 100644
index 000000000000..9254b784003d
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_fb.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/module.h>
+
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+
+#include "es_fb.h"
+#include "es_gem.h"
+
+static struct drm_framebuffer_funcs es_fb_funcs = {
+	.create_handle = drm_gem_fb_create_handle,
+	.destroy = drm_gem_fb_destroy,
+	.dirty = drm_atomic_helper_dirtyfb,
+};
+
+static struct drm_framebuffer *
+es_fb_alloc(struct drm_device *dev, const struct drm_mode_fb_cmd2 *mode_cmd,
+	    struct es_gem_object **obj, unsigned int num_planes)
+{
+	struct drm_framebuffer *fb;
+	int ret, i;
+
+	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
+	if (!fb)
+		return ERR_PTR(-ENOMEM);
+
+	drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd);
+
+	for (i = 0; i < num_planes; i++)
+		fb->obj[i] = &obj[i]->base;
+
+	ret = drm_framebuffer_init(dev, fb, &es_fb_funcs);
+	if (ret) {
+		dev_err(dev->dev, "Failed to initialize framebuffer: %d\n",
+			ret);
+		kfree(fb);
+		return ERR_PTR(ret);
+	}
+
+	return fb;
+}
+
+static struct drm_framebuffer *
+es_fb_create(struct drm_device *dev, struct drm_file *file_priv,
+	     const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_framebuffer *fb;
+	const struct drm_format_info *info;
+	struct es_gem_object *objs[MAX_NUM_PLANES];
+	struct drm_gem_object *obj;
+	unsigned int height, size;
+	unsigned char i, num_planes;
+	int ret = 0;
+
+	info = drm_format_info(mode_cmd->pixel_format);
+	if (!info)
+		return ERR_PTR(-EINVAL);
+
+	num_planes = info->num_planes;
+	if (num_planes > MAX_NUM_PLANES)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < num_planes; i++) {
+		obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[i]);
+		if (!obj) {
+			dev_err(dev->dev, "Failed to lookup GEM object.\n");
+			ret = -ENXIO;
+			goto err;
+		}
+
+		height =
+			drm_format_info_plane_height(info, mode_cmd->height, i);
+
+		size = height * mode_cmd->pitches[i] + mode_cmd->offsets[i];
+
+		if (obj->size < size) {
+			drm_gem_object_put(obj);
+			ret = -EINVAL;
+			goto err;
+		}
+
+		objs[i] = to_es_gem_object(obj);
+	}
+
+	fb = es_fb_alloc(dev, mode_cmd, objs, i);
+	if (IS_ERR(fb)) {
+		ret = PTR_ERR(fb);
+		goto err;
+	}
+
+	return fb;
+
+err:
+	for (; i > 0; i--) {
+		drm_gem_object_put(&objs[i - 1]->base);
+	}
+
+	return ERR_PTR(ret);
+}
+
+struct es_gem_object *es_fb_get_gem_obj(struct drm_framebuffer *fb,
+					unsigned char plane)
+{
+	if (plane > MAX_NUM_PLANES)
+		return NULL;
+
+	return to_es_gem_object(fb->obj[plane]);
+}
+
+static const struct drm_mode_config_funcs es_mode_config_funcs = {
+	.fb_create = es_fb_create,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static struct drm_mode_config_helper_funcs es_mode_config_helpers = {
+	.atomic_commit_tail = drm_atomic_helper_commit_tail_rpm,
+};
+
+void es_mode_config_init(struct drm_device *dev)
+{
+	if (dev->mode_config.max_width == 0 ||
+	    dev->mode_config.max_height == 0) {
+		dev->mode_config.min_width = 0;
+		dev->mode_config.min_height = 0;
+		dev->mode_config.max_width = 4096;
+		dev->mode_config.max_height = 4096;
+	}
+	dev->mode_config.funcs = &es_mode_config_funcs;
+	dev->mode_config.helper_private = &es_mode_config_helpers;
+}
diff --git a/drivers/gpu/drm/eswin/es_fb.h b/drivers/gpu/drm/eswin/es_fb.h
new file mode 100644
index 000000000000..6cd8b31684aa
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_fb.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_FB_H__
+#define __ES_FB_H__
+
+struct es_gem_object *es_fb_get_gem_obj(struct drm_framebuffer *fb,
+					unsigned char plane);
+
+void es_mode_config_init(struct drm_device *dev);
+#endif /* __ES_FB_H__ */
diff --git a/drivers/gpu/drm/eswin/es_gem.c b/drivers/gpu/drm/eswin/es_gem.c
new file mode 100644
index 000000000000..5406448ee2fb
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_gem.c
@@ -0,0 +1,605 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/dma-buf.h>
+#include <drm/drm_prime.h>
+
+#include "es_drv.h"
+#include "es_gem.h"
+
+MODULE_IMPORT_NS(DMA_BUF);
+
+static const struct drm_gem_object_funcs es_gem_default_funcs;
+
+static void nonseq_free(struct page **pages, unsigned int nr_page)
+{
+	u32 i;
+
+	if (!pages)
+		return;
+
+	for (i = 0; i < nr_page; i++)
+		__free_page(pages[i]);
+}
+
+#ifdef CONFIG_ESWIN_MMU
+static int get_pages(unsigned int nr_page, struct es_gem_object *es_obj)
+{
+	struct page *pages;
+	u32 i, num_page, page_count = 0;
+	int order = 0;
+	gfp_t gfp = GFP_KERNEL;
+
+	if (!es_obj->pages)
+		return -EINVAL;
+
+	gfp &= ~__GFP_HIGHMEM;
+	gfp |= __GFP_DMA32;
+
+	num_page = nr_page;
+
+	do {
+		pages = NULL;
+		order = get_order(num_page * PAGE_SIZE);
+		num_page = 1 << order;
+
+		if ((num_page + page_count > nr_page) || (order >= MAX_ORDER)) {
+			num_page = num_page >> 1;
+			continue;
+		}
+
+		pages = alloc_pages(gfp, order);
+		if (!pages) {
+			if (num_page == 1) {
+				nonseq_free(es_obj->pages, page_count);
+				return -ENOMEM;
+			}
+
+			num_page = num_page >> 1;
+		} else {
+			for (i = 0; i < num_page; i++) {
+				es_obj->pages[page_count + i] = &pages[i];
+				SetPageReserved(es_obj->pages[page_count + i]);
+			}
+
+			page_count += num_page;
+			num_page = nr_page - page_count;
+		}
+
+	} while (page_count < nr_page);
+
+	es_obj->get_pages = true;
+
+	return 0;
+}
+#endif
+
+static void put_pages(unsigned int nr_page, struct es_gem_object *es_obj)
+{
+	u32 i;
+
+	for (i = 0; i < nr_page; i++)
+		ClearPageReserved(es_obj->pages[i]);
+
+	nonseq_free(es_obj->pages, nr_page);
+
+	return;
+}
+
+static int es_gem_alloc_buf(struct es_gem_object *es_obj)
+{
+	struct drm_device *dev = es_obj->base.dev;
+	unsigned int nr_pages;
+	struct sg_table sgt;
+	int ret = -ENOMEM;
+#ifdef CONFIG_ESWIN_MMU
+	struct es_drm_private *priv = dev->dev_private;
+#endif
+
+	if (es_obj->dma_addr) {
+		DRM_DEV_DEBUG_KMS(dev->dev, "already allocated.\n");
+		return 0;
+	}
+
+	es_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;
+
+	if (!is_iommu_enabled(dev))
+		es_obj->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS;
+
+	nr_pages = es_obj->size >> PAGE_SHIFT;
+
+	es_obj->pages = kvmalloc_array(nr_pages, sizeof(struct page *),
+				       GFP_KERNEL | __GFP_ZERO);
+	if (!es_obj->pages) {
+		DRM_DEV_ERROR(dev->dev, "failed to allocate pages.\n");
+		return -ENOMEM;
+	}
+
+	es_obj->cookie = dma_alloc_attrs(to_dma_dev(dev), es_obj->size,
+					 &es_obj->dma_addr, GFP_KERNEL,
+					 es_obj->dma_attrs);
+	if (!es_obj->cookie) {
+#ifdef CONFIG_ESWIN_MMU
+		ret = get_pages(nr_pages, es_obj);
+		if (ret) {
+			DRM_DEV_ERROR(dev->dev, "fail to allocate buffer.\n");
+			goto err_free;
+		}
+#else
+		DRM_DEV_ERROR(dev->dev, "failed to allocate buffer.\n");
+		goto err_free;
+#endif
+	}
+
+#ifdef CONFIG_ESWIN_MMU
+	/* MMU map*/
+	if (!priv->mmu) {
+		DRM_DEV_ERROR(dev->dev, "invalid mmu.\n");
+		ret = -EINVAL;
+		goto err_mem_free;
+	}
+
+	if (!es_obj->get_pages)
+		ret = dc_mmu_map_memory(priv->mmu, (u64)es_obj->dma_addr,
+					nr_pages, &es_obj->iova, true);
+	else
+		ret = dc_mmu_map_memory(priv->mmu, (u64)es_obj->pages, nr_pages,
+					&es_obj->iova, false);
+
+	if (ret) {
+		DRM_DEV_ERROR(dev->dev, "failed to do mmu map.\n");
+		goto err_mem_free;
+	}
+#else
+	es_obj->iova = es_obj->dma_addr;
+#endif
+
+	if (!es_obj->get_pages) {
+		ret = dma_get_sgtable_attrs(to_dma_dev(dev), &sgt,
+					    es_obj->cookie, es_obj->dma_addr,
+					    es_obj->size, es_obj->dma_attrs);
+		if (ret < 0) {
+			DRM_DEV_ERROR(dev->dev, "failed to get sgtable.\n");
+			goto err_mem_free;
+		}
+
+		if (drm_prime_sg_to_page_array(&sgt, es_obj->pages, nr_pages)) {
+			DRM_DEV_ERROR(dev->dev, "invalid sgtable.\n");
+			ret = -EINVAL;
+			goto err_sgt_free;
+		}
+
+		sg_free_table(&sgt);
+	}
+
+	return 0;
+
+err_sgt_free:
+	sg_free_table(&sgt);
+err_mem_free:
+	if (!es_obj->get_pages)
+		dma_free_attrs(to_dma_dev(dev), es_obj->size, es_obj->cookie,
+			       es_obj->dma_addr, es_obj->dma_attrs);
+	else
+		put_pages(nr_pages, es_obj);
+err_free:
+	kvfree(es_obj->pages);
+
+	return ret;
+}
+
+static void es_gem_free_buf(struct es_gem_object *es_obj)
+{
+	struct drm_device *dev = es_obj->base.dev;
+#ifdef CONFIG_ESWIN_MMU
+	struct es_drm_private *priv = dev->dev_private;
+	unsigned int nr_pages;
+#endif
+
+	if ((!es_obj->get_pages) && (!es_obj->dma_addr)) {
+		DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr is invalid.\n");
+		return;
+	}
+
+#ifdef CONFIG_ESWIN_MMU
+	if (!priv->mmu) {
+		DRM_DEV_ERROR(dev->dev, "invalid mmu.\n");
+		return;
+	}
+
+	if (!es_obj->sgt) { // dumb buffer release
+		nr_pages = es_obj->size >> PAGE_SHIFT;
+		if (es_obj->iova) {
+			dc_mmu_unmap_memory(priv->mmu, es_obj->iova, nr_pages);
+		}
+	} else { // prime buffer release
+		if (es_obj->iova_list) {
+			if (es_obj->iova_list->iova) {
+				dc_mmu_unmap_memory(
+					priv->mmu, es_obj->iova_list->iova,
+					es_obj->iova_list->nr_pages);
+				kfree(es_obj->iova_list);
+			}
+		}
+	}
+#endif
+
+	if (!es_obj->get_pages) {
+		dma_free_attrs(to_dma_dev(dev), es_obj->size, es_obj->cookie,
+			       (dma_addr_t)es_obj->dma_addr, es_obj->dma_attrs);
+	} else {
+		if (!es_obj->dma_addr) {
+			DRM_DEV_ERROR(dev->dev, "No dma addr allocated, no need to free\n");
+			return;
+		}
+		put_pages(es_obj->size >> PAGE_SHIFT, es_obj);
+	}
+
+	kvfree(es_obj->pages);
+}
+
+static void es_gem_free_object(struct drm_gem_object *obj)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+
+#ifdef CONFIG_ESWIN_MMU
+	if (es_obj)
+		es_gem_free_buf(es_obj);
+#endif
+	if (obj->import_attach) {
+		drm_prime_gem_destroy(obj, es_obj->sgt);
+	}
+
+	drm_gem_object_release(obj);
+
+	kfree(es_obj);
+}
+
+static struct es_gem_object *es_gem_alloc_object(struct drm_device *dev,
+						 size_t size)
+{
+	struct es_gem_object *es_obj;
+	struct drm_gem_object *obj;
+	int ret;
+
+	es_obj = kzalloc(sizeof(*es_obj), GFP_KERNEL);
+	if (!es_obj)
+		return ERR_PTR(-ENOMEM);
+
+	es_obj->size = size;
+	obj = &es_obj->base;
+
+	ret = drm_gem_object_init(dev, obj, size);
+	if (ret)
+		goto err_free;
+
+	es_obj->base.funcs = &es_gem_default_funcs;
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret) {
+		drm_gem_object_release(obj);
+		goto err_free;
+	}
+
+	return es_obj;
+
+err_free:
+	kfree(es_obj);
+	return ERR_PTR(ret);
+}
+
+struct es_gem_object *es_gem_create_object(struct drm_device *dev, size_t size)
+{
+	struct es_gem_object *es_obj;
+	int ret;
+
+	size = PAGE_ALIGN(size);
+
+	es_obj = es_gem_alloc_object(dev, size);
+	if (IS_ERR(es_obj))
+		return es_obj;
+
+	ret = es_gem_alloc_buf(es_obj);
+	if (ret) {
+		drm_gem_object_release(&es_obj->base);
+		kfree(es_obj);
+		return ERR_PTR(ret);
+	}
+
+	return es_obj;
+}
+
+static struct es_gem_object *es_gem_create_with_handle(struct drm_device *dev,
+						       struct drm_file *file,
+						       size_t size,
+						       unsigned int *handle)
+{
+	struct es_gem_object *es_obj;
+	struct drm_gem_object *obj;
+	int ret;
+
+	es_obj = es_gem_create_object(dev, size);
+	if (IS_ERR(es_obj))
+		return es_obj;
+
+	obj = &es_obj->base;
+
+	ret = drm_gem_handle_create(file, obj, handle);
+	drm_gem_object_put(obj);
+	if (ret) {
+		pr_err("Drm GEM handle create failed\n");
+		return ERR_PTR(ret);
+	}
+
+	return es_obj;
+}
+
+static int es_gem_mmap_obj(struct drm_gem_object *obj,
+			   struct vm_area_struct *vma)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+	struct drm_device *drm_dev = es_obj->base.dev;
+	unsigned long vm_size;
+	int ret = 0;
+
+	vm_size = vma->vm_end - vma->vm_start;
+	if (vm_size > es_obj->size)
+		return -EINVAL;
+
+	vma->vm_pgoff = 0;
+
+	if (!es_obj->get_pages) {
+		vm_flags_clear(vma, VM_PFNMAP);
+
+		ret = dma_mmap_attrs(to_dma_dev(drm_dev), vma, es_obj->cookie,
+				     es_obj->dma_addr, es_obj->size,
+				     es_obj->dma_attrs);
+	} else {
+		u32 i, nr_pages, pfn = 0U;
+		unsigned long start;
+
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+		vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP);
+		start = vma->vm_start;
+		vm_size = PAGE_ALIGN(vm_size);
+		nr_pages = vm_size >> PAGE_SHIFT;
+
+		for (i = 0; i < nr_pages; i++) {
+			pfn = page_to_pfn(es_obj->pages[i]);
+
+			ret = remap_pfn_range(vma, start, pfn, PAGE_SIZE,
+					      vma->vm_page_prot);
+			if (ret < 0)
+				break;
+
+			start += PAGE_SIZE;
+		}
+	}
+
+	if (ret)
+		drm_gem_vm_close(vma);
+
+	return ret;
+}
+
+struct sg_table *es_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+
+	return drm_prime_pages_to_sg(obj->dev, es_obj->pages,
+				     es_obj->size >> PAGE_SHIFT);
+}
+
+static int es_gem_prime_vmap(struct drm_gem_object *obj,
+			     struct iosys_map *map)
+{
+	struct es_gem_object *es_obj = to_es_gem_object(obj);
+
+	void * vaddr = es_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING ?
+		       page_address(es_obj->cookie) : es_obj->cookie;
+
+	iosys_map_set_vaddr(map, vaddr);
+
+	return 0;
+}
+
+static void es_gem_prime_vunmap(struct drm_gem_object *obj,
+				struct iosys_map *map)
+{
+	/* Nothing to do */
+}
+
+static const struct vm_operations_struct es_vm_ops = {
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+static const struct drm_gem_object_funcs es_gem_default_funcs = {
+	.free = es_gem_free_object,
+	.get_sg_table = es_gem_prime_get_sg_table,
+	.vmap = es_gem_prime_vmap,
+	.vunmap = es_gem_prime_vunmap,
+	.vm_ops = &es_vm_ops,
+};
+
+int es_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
+		       struct drm_mode_create_dumb *args)
+{
+	struct es_drm_private *priv = dev->dev_private;
+	struct es_gem_object *es_obj;
+	unsigned int pitch = args->width * DIV_ROUND_UP(args->bpp, 8);
+
+	args->pitch = ALIGN(pitch, priv->pitch_alignment);
+	args->size = PAGE_ALIGN(args->pitch * args->height);
+
+	es_obj =
+		es_gem_create_with_handle(dev, file, args->size, &args->handle);
+	return PTR_ERR_OR_ZERO(es_obj);
+}
+
+struct drm_gem_object *es_gem_prime_import(struct drm_device *dev,
+					   struct dma_buf *dma_buf)
+{
+	return drm_gem_prime_import_dev(dev, dma_buf, to_dma_dev(dev));
+}
+
+struct drm_gem_object *
+es_gem_prime_import_sg_table(struct drm_device *dev,
+			     struct dma_buf_attachment *attach,
+			     struct sg_table *sgt)
+{
+	struct es_gem_object *es_obj;
+	int npages;
+	int ret;
+	struct scatterlist *s = NULL;
+	u32 i = 0;
+	dma_addr_t expected;
+	size_t size = attach->dmabuf->size;
+#ifdef CONFIG_ESWIN_MMU
+	u32 iova, j;
+	struct scatterlist **splist;
+	struct es_drm_private *priv = dev->dev_private;
+
+	if (!priv->mmu) {
+		DRM_ERROR("invalid mmu.\n");
+		ret = -EINVAL;
+		return ERR_PTR(ret);
+	}
+#endif
+
+	size = PAGE_ALIGN(size);
+
+	es_obj = es_gem_alloc_object(dev, size);
+	if (IS_ERR(es_obj))
+		return ERR_CAST(es_obj);
+
+	npages = es_obj->size >> PAGE_SHIFT;
+	es_obj->pages =
+		kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
+	if (!es_obj->pages) {
+		ret = -ENOMEM;
+		goto err_gemalloc;
+	}
+
+	ret = drm_prime_sg_to_page_array(sgt, es_obj->pages, npages);
+	if (ret)
+		goto err_free_page;
+
+	expected = sg_dma_address(sgt->sgl);
+#ifdef CONFIG_ESWIN_MMU
+	splist = (struct scatterlist **)kzalloc(sizeof(s) * sgt->nents,
+						GFP_KERNEL);
+	if (!splist) {
+		DRM_ERROR("Allocate splist failed");
+		ret = -ENOMEM;
+		goto err_free_page;
+	}
+
+	es_obj->iova_list =
+		(iova_info_t *)kzalloc(sizeof(iova_info_t), GFP_KERNEL);
+	if (!es_obj->iova_list) {
+		DRM_ERROR("Allocate splist failed");
+		ret = -ENOMEM;
+		goto err_sp;
+	}
+
+	for_each_sg (sgt->sgl, s, sgt->nents, i) {
+		splist[i] = s;
+	}
+	i = 0;
+	es_obj->nr_iova = sgt->nents;
+
+	for (j = sgt->nents; j > 0; j--) {
+		s = splist[j - 1];
+#else
+	for_each_sg (sgt->sgl, s, sgt->nents, i) {
+#endif
+		if (sg_dma_address(s) != expected) {
+#ifndef CONFIG_ESWIN_MMU
+			DRM_ERROR("sg_table is not contiguous");
+			ret = -EINVAL;
+			goto err;
+#endif
+		}
+
+		if (sg_dma_len(s) & (PAGE_SIZE - 1)) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+#ifdef CONFIG_ESWIN_MMU
+		iova = 0;
+
+		if (j == 1) {
+			ret = dc_mmu_map_memory(priv->mmu, (u64)es_obj->pages,
+						npages, &iova, false);
+			if (ret) {
+				DRM_ERROR("failed to do mmu map.\n");
+				goto err;
+			}
+			es_obj->iova_list->iova = iova;
+			es_obj->iova_list->nr_pages = npages;
+		}
+
+		if (i == 0)
+			es_obj->iova = iova;
+#else
+		if (i == 0)
+			es_obj->iova = sg_dma_address(s);
+#endif
+
+		expected = sg_dma_address(s) + sg_dma_len(s);
+	}
+
+	es_obj->dma_addr = sg_dma_address(sgt->sgl);
+
+	es_obj->sgt = sgt;
+#ifdef CONFIG_ESWIN_MMU
+	kfree(splist);
+#endif
+
+	return &es_obj->base;
+
+#ifdef CONFIG_ESWIN_MMU
+err:
+	kfree(es_obj->iova_list);
+err_sp:
+	kfree(splist);
+#endif
+err_free_page:
+	kvfree(es_obj->pages);
+err_gemalloc:
+	es_gem_free_object(&es_obj->base);
+
+	return ERR_PTR(ret);
+}
+
+int es_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	int ret = 0;
+
+	ret = drm_gem_mmap_obj(obj, obj->size, vma);
+	if (ret < 0)
+		return ret;
+
+	return es_gem_mmap_obj(obj, vma);
+}
+
+int es_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	obj = vma->vm_private_data;
+
+	if (obj->import_attach)
+		return dma_buf_mmap(obj->dma_buf, vma, 0);
+
+	return es_gem_mmap_obj(obj, vma);
+}
diff --git a/drivers/gpu/drm/eswin/es_gem.h b/drivers/gpu/drm/eswin/es_gem.h
new file mode 100644
index 000000000000..b14b32ea5cb4
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_gem.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_GEM_H__
+#define __ES_GEM_H__
+
+#include <linux/dma-buf.h>
+
+#include <drm/drm_gem.h>
+
+#include "es_drv.h"
+
+#ifdef CONFIG_ESWIN_MMU
+typedef struct _iova_info {
+	u32 iova;
+	u32 nr_pages;
+} iova_info_t;
+#endif
+
+/*
+ *
+ * @base: drm gem object.
+ * @size: size requested from user
+ * @cookie: cookie returned by dma_alloc_attrs
+ *  - not kernel virtual address with DMA_ATTR_NO_KERNEL_MAPPING
+ * @dma_addr: bus address(accessed by dma) to allocated memory region.
+ *  - this address could be physical address without IOMMU and
+ *  device address with IOMMU.
+ * @dma_attrs: attribute for DMA API
+ * @get_pages: flag for manually applying for non-contiguous memory.
+ * @pages: Array of backing pages.
+ * @sgt: Imported sg_table.
+ *
+ */
+struct es_gem_object {
+	struct drm_gem_object base;
+	size_t size;
+	void *cookie;
+	dma_addr_t dma_addr;
+	u32 iova;
+	unsigned long dma_attrs;
+	bool get_pages;
+	struct page **pages;
+	struct sg_table *sgt;
+#ifdef CONFIG_ESWIN_MMU
+	iova_info_t *iova_list;
+	u32 nr_iova;
+#endif
+};
+
+static inline struct es_gem_object *to_es_gem_object(struct drm_gem_object *obj)
+{
+	return container_of(obj, struct es_gem_object, base);
+}
+
+struct es_gem_object *es_gem_create_object(struct drm_device *dev, size_t size);
+
+int es_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+
+int es_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm,
+		       struct drm_mode_create_dumb *args);
+
+int es_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+
+struct sg_table *es_gem_prime_get_sg_table(struct drm_gem_object *obj);
+
+struct drm_gem_object *es_gem_prime_import(struct drm_device *dev,
+					   struct dma_buf *dma_buf);
+struct drm_gem_object *
+es_gem_prime_import_sg_table(struct drm_device *dev,
+			     struct dma_buf_attachment *attach,
+			     struct sg_table *sgt);
+
+#endif /* __ES_GEM_H__ */
diff --git a/drivers/gpu/drm/eswin/es_mipi_dsi.c b/drivers/gpu/drm/eswin/es_mipi_dsi.c
new file mode 100644
index 000000000000..8aa3c69378f2
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_mipi_dsi.c
@@ -0,0 +1,661 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/iopoll.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/mfd/syscon.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+#include <video/mipi_display.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/bridge/dw_mipi_dsi.h>
+
+#include "es_crtc.h"
+#include "es_mipi_dsi.h"
+#include "es_panel.h"
+
+#define MAX_LANE_COUNT 4
+#define PLL_INPUT_REF_CLK 20000000
+
+/* Defines the offset of the dphy register */
+#define DSI_PHY_TST_CTRL0 0xb4
+#define PHY_TESTCLK BIT(1)
+#define PHY_UNTESTCLK 0
+#define PHY_TESTCLR BIT(0)
+#define PHY_UNTESTCLR 0
+
+#define DSI_PHY_TST_CTRL1 0xb8
+#define PHY_TESTEN BIT(16)
+#define PHY_UNTESTEN 0
+#define PHY_TESTDOUT(n) (((n) & 0xff) << 8)
+#define PHY_TESTDIN(n) (((n) & 0xff) << 0)
+#define INPUT_DIVIDER(val) (val & 0x7f)
+
+#define LOW_PROGRAM_EN 0
+#define HIGH_PROGRAM_EN BIT(7)
+#define LOOP_DIV_LOW_SEL(val) (val & 0x1f)
+#define LOOP_DIV_HIGH_SEL(val) ((val >> 5) & 0xf)
+#define PLL_LOOP_DIV_EN BIT(5)
+#define PLL_INPUT_DIV_EN BIT(4)
+
+#define LPF_RESISTORS_15_5KOHM 0x1
+#define LPF_RESISTORS_13KOHM 0x2
+#define LPF_RESISTORS_11_5KOHM 0x4
+#define LPF_RESISTORS_10_5KOHM 0x8
+#define LPF_RESISTORS_8KOHM 0x10
+#define LPF_PROGRAM_EN BIT(6)
+#define LPF_RESISTORS_SEL(val) ((val) & 0x3f)
+
+#define LOW_PROGRAM_EN 0
+#define HIGH_PROGRAM_EN BIT(7)
+#define LOOP_DIV_LOW_SEL(val) (val & 0x1f)
+#define LOOP_DIV_HIGH_SEL(val) ((val >> 5) & 0xf)
+#define PLL_LOOP_DIV_EN BIT(5)
+#define PLL_INPUT_DIV_EN BIT(4)
+
+#define PLL_BIAS_CUR_SEL_CAP_VCO_CONTROL 0x10
+#define PLL_CP_CONTROL_PLL_LOCK_BYPASS 0x11
+#define PLL_LPF_AND_CP_CONTROL 0x12
+#define PLL_INPUT_DIVIDER_RATIO 0x17
+#define PLL_LOOP_DIVIDER_RATIO 0x18
+#define PLL_INPUT_AND_LOOP_DIVIDER_RATIOS_CONTROL 0x19
+
+#define DSI_TO_CNT_CFG 0x78
+#define HSTX_TO_CNT(p) (((p) & 0xffff) << 16)
+#define LPRX_TO_CNT(p) ((p) & 0xffff)
+
+struct hstt {
+	unsigned int maxfreq;
+	struct dw_mipi_dsi_dphy_timing timing;
+};
+
+struct pll_parameter {
+	u32 max_freq;
+	u32 actual_freq;
+	u16 hs_freq_range;
+	u16 pll_n;
+	u16 pll_m;
+	u8 vco_cntrl;
+	u8 cpbias_cntrl;
+	u8 gmp_cntrl;
+	u8 int_cntrl;
+	u8 prop_cntrl;
+};
+
+#define HSTT(_maxfreq, _c_lp2hs, _c_hs2lp, _d_lp2hs, _d_hs2lp)                 \
+	{                                                                      \
+		.maxfreq = _maxfreq, .timing = {                               \
+			.clk_lp2hs = _c_lp2hs,                                 \
+			.clk_hs2lp = _c_hs2lp,                                 \
+			.data_lp2hs = _d_lp2hs,                                \
+			.data_hs2lp = _d_hs2lp,                                \
+		}                                                              \
+	}
+
+#define PLL_PARAM(_maxfreq, _vco_ctl, _cpbias_ctl, _gmp_ctl, _int_ctl,         \
+		  _prop_ctl)                                                   \
+	{                                                                      \
+		.max_freq = _maxfreq, .vco_cntrl = _vco_ctl,                   \
+		.cpbias_cntrl = _cpbias_ctl, .gmp_cntrl = _gmp_ctl,            \
+		.int_cntrl = _int_ctl, .prop_cntrl = _prop_ctl,                \
+	}
+
+struct mipi_dsi_priv {
+	u32 lanes;
+	u32 format;
+	unsigned long mode_flags;
+
+	struct pll_parameter pll_param;
+
+	void __iomem *dphy_base;
+
+	struct clk *ivideo_clk;
+	u64 pll_ref_clk;
+
+	struct dw_mipi_dsi_plat_data plat_data;
+	struct reset_control *rst_dsi_phyrstn;
+	void *data;
+};
+
+struct es_mipi_dsi {
+	struct mipi_dsi_priv dsi_priv;
+	struct drm_encoder encoder;
+	struct drm_crtc *crtc;
+};
+
+/* Table A-4 High-Speed Transition Times (High-Speed Entry LP->HS DATA LANE(Considers HS_CLK_LANE_ENTRY)d)*/
+struct hstt hstt_table[] = {
+	HSTT(80, 21, 17, 35, 10),      HSTT(90, 23, 17, 39, 10),
+	HSTT(100, 22, 17, 37, 10),     HSTT(110, 25, 18, 43, 11),
+	HSTT(120, 26, 20, 46, 11),     HSTT(130, 27, 19, 46, 11),
+	HSTT(140, 27, 19, 46, 11),     HSTT(150, 28, 20, 47, 12),
+	HSTT(160, 30, 21, 53, 13),     HSTT(170, 30, 21, 55, 13),
+	HSTT(180, 31, 21, 53, 13),     HSTT(190, 32, 22, 58, 13),
+	HSTT(205, 35, 22, 58, 13),     HSTT(220, 37, 26, 63, 15),
+	HSTT(235, 38, 28, 65, 16),     HSTT(250, 41, 29, 71, 17),
+	HSTT(275, 43, 29, 74, 18),     HSTT(300, 45, 32, 80, 19),
+	HSTT(325, 48, 33, 86, 18),     HSTT(350, 51, 35, 91, 20),
+	HSTT(400, 59, 37, 102, 21),    HSTT(450, 65, 40, 115, 23),
+	HSTT(500, 71, 41, 126, 24),    HSTT(550, 77, 44, 135, 26),
+	HSTT(600, 82, 46, 147, 27),    HSTT(650, 87, 48, 156, 28),
+	HSTT(700, 94, 52, 166, 29),    HSTT(750, 99, 52, 175, 31),
+	HSTT(800, 105, 55, 187, 32),   HSTT(850, 110, 58, 196, 32),
+	HSTT(900, 115, 58, 206, 35),   HSTT(950, 120, 62, 213, 36),
+	HSTT(1000, 128, 63, 225, 38),  HSTT(1050, 132, 65, 234, 38),
+	HSTT(1100, 138, 67, 243, 39),  HSTT(1150, 146, 69, 259, 42),
+	HSTT(1200, 151, 71, 269, 43),  HSTT(1250, 153, 74, 273, 45),
+	HSTT(1300, 160, 73, 282, 46),  HSTT(1350, 165, 76, 294, 47),
+	HSTT(1400, 172, 78, 304, 49),  HSTT(1450, 177, 80, 314, 49),
+	HSTT(1500, 183, 81, 326, 52),  HSTT(1550, 191, 84, 339, 52),
+	HSTT(1600, 194, 85, 345, 52),  HSTT(1650, 201, 86, 355, 53),
+	HSTT(1700, 208, 88, 368, 53),  HSTT(1750, 212, 89, 378, 53),
+	HSTT(1800, 220, 90, 389, 54),  HSTT(1850, 223, 92, 401, 54),
+	HSTT(1900, 231, 91, 413, 55),  HSTT(1950, 236, 95, 422, 56),
+	HSTT(2000, 243, 97, 432, 56),  HSTT(2050, 248, 99, 442, 58),
+	HSTT(2100, 252, 100, 454, 59), HSTT(2150, 259, 102, 460, 61),
+	HSTT(2200, 266, 105, 476, 62), HSTT(2250, 269, 109, 481, 63),
+	HSTT(2300, 272, 109, 490, 65), HSTT(2350, 281, 112, 502, 66),
+	HSTT(2400, 283, 115, 509, 66), HSTT(2450, 282, 115, 510, 67),
+	HSTT(2500, 281, 118, 508, 67),
+};
+
+static struct pll_parameter pll_tbl[] = {
+	PLL_PARAM(55, 0x3f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(82, 0x37, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(110, 0x2f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(165, 0x27, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(220, 0x1f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(330, 0x17, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(440, 0x0f, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(660, 0x07, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(1149, 0x03, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(1152, 0x01, 0x10, 0x01, 0x00, 0x0D),
+	PLL_PARAM(1250, 0x01, 0x10, 0x01, 0x00, 0x0E),
+};
+
+static const struct of_device_id es_mipi_dsi_dt_ids[] = {
+	{
+		.compatible = "eswin,dsi",
+	},
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(of, es_mipi_dsi_dt_ids);
+
+static void dsi_write(struct mipi_dsi_priv *priv, u32 reg, u32 val)
+{
+	writel(val, priv->dphy_base + reg);
+}
+
+static u32 dsi_read(struct mipi_dsi_priv *priv, u32 reg)
+{
+	return readl(priv->dphy_base + reg);
+}
+
+static void es_dsi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct es_mipi_dsi *es_dsi =
+		container_of(encoder, struct es_mipi_dsi, encoder);
+	struct mipi_dsi_priv *dsi_priv = &es_dsi->dsi_priv;
+
+	DRM_INFO("enter, encoder = 0x%px\n", encoder);
+
+	clk_disable_unprepare(dsi_priv->ivideo_clk);
+}
+
+static bool es_dsi_encoder_mode_fixup(struct drm_encoder *encoder,
+				      const struct drm_display_mode *mode,
+				      struct drm_display_mode *adj_mode)
+{
+	struct es_mipi_dsi *es_dsi =
+		container_of(encoder, struct es_mipi_dsi, encoder);
+	struct drm_crtc_state *crtc_state =
+		container_of(mode, struct drm_crtc_state, mode);
+
+	DRM_INFO("enter, bind crtc%d\n", drm_crtc_index(crtc_state->crtc));
+	DRM_INFO("mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
+	DRM_INFO("adj_mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(adj_mode));
+
+	es_dsi->crtc = crtc_state->crtc;
+	return true;
+}
+
+static void es_dsi_encoder_mode_set(struct drm_encoder *encoder,
+				    struct drm_display_mode *mode,
+				    struct drm_display_mode *adj_mode)
+{
+	DRM_INFO("mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
+	DRM_INFO("adj_mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(adj_mode));
+}
+
+static void es_dsi_encoder_mode_enable(struct drm_encoder *encoder)
+{
+	int ret;
+	struct es_mipi_dsi *es_dsi =
+		container_of(encoder, struct es_mipi_dsi, encoder);
+	struct mipi_dsi_priv *dsi_priv = &es_dsi->dsi_priv;
+
+	DRM_INFO("enter, encoder = 0x%px\n", encoder);
+
+	ret = clk_prepare_enable(dsi_priv->ivideo_clk);
+	if (ret)
+		DRM_ERROR("enable ivideo clk failed, ret = %d\n", ret);
+}
+
+static const struct drm_encoder_helper_funcs es_dsi_encoder_helper_funcs = {
+	.mode_fixup = es_dsi_encoder_mode_fixup,
+	.mode_set = es_dsi_encoder_mode_set,
+	.enable = es_dsi_encoder_mode_enable,
+	.disable = es_dsi_encoder_disable,
+};
+
+static void es_mipi_dsi_phy_write(struct mipi_dsi_priv *dsi, u8 test_code,
+				  u8 test_data)
+{
+	/*
+     * With the falling edge on TESTCLK, the TESTDIN[7:0] signal content
+     * is latched internally as the current test code. Test data is
+     * programmed internally by rising edge on TESTCLK.
+     */
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  PHY_TESTEN | PHY_TESTDOUT(0) | PHY_TESTDIN(test_code));
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  PHY_UNTESTEN | PHY_TESTDOUT(0) | PHY_TESTDIN(test_data));
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+}
+
+static void es_mipi_dsi_phy_write_m(struct mipi_dsi_priv *dsi, u16 m)
+{
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  PHY_TESTEN | PHY_TESTDOUT(0) | PLL_LOOP_DIVIDER_RATIO);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL1,
+		  LOOP_DIV_HIGH_SEL(m) | HIGH_PROGRAM_EN);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL1, LOOP_DIV_LOW_SEL(m) | LOW_PROGRAM_EN);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+}
+
+static int es_dsi_phy_init(void *priv_data)
+{
+	struct mipi_dsi_priv *dsi_priv = (struct mipi_dsi_priv *)priv_data;
+	struct pll_parameter *pll_param = &dsi_priv->pll_param;
+
+	dsi_write(dsi_priv, DSI_TO_CNT_CFG, HSTX_TO_CNT(0) | LPRX_TO_CNT(0));
+
+	es_mipi_dsi_phy_write(dsi_priv,
+			      PLL_INPUT_AND_LOOP_DIVIDER_RATIOS_CONTROL,
+			      PLL_LOOP_DIV_EN | PLL_INPUT_DIV_EN);
+
+	DRM_INFO(
+		"set pll m:0x%x, n:0x%x, vco_cntrl:0x%x ,cpbias:0x%x, gmp:0x%x, int:0x%x, prop:0x%x\n",
+		pll_param->pll_m, pll_param->pll_n, pll_param->vco_cntrl,
+		pll_param->cpbias_cntrl, pll_param->gmp_cntrl,
+		pll_param->int_cntrl, pll_param->prop_cntrl);
+	//  0x17 cfg -> n
+	es_mipi_dsi_phy_write(dsi_priv, PLL_INPUT_DIVIDER_RATIO,
+			      INPUT_DIVIDER(pll_param->pll_n));
+
+	es_mipi_dsi_phy_write_m(dsi_priv, pll_param->pll_m);
+
+	// 0x12 cfg
+	es_mipi_dsi_phy_write(dsi_priv, PLL_LPF_AND_CP_CONTROL,
+			      LPF_PROGRAM_EN | pll_param->vco_cntrl);
+
+	// 0x1c cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x1c, pll_param->cpbias_cntrl);
+
+	// 0x13 cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x13, pll_param->gmp_cntrl << 4);
+
+	// 0x0f cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x0f, pll_param->int_cntrl);
+	// 0x0f cfg
+	es_mipi_dsi_phy_write(dsi_priv, 0x0e, pll_param->prop_cntrl);
+
+	return 0;
+}
+
+static int dsi_phy_pll_get_params(struct mipi_dsi_priv *dsi_priv, u64 lane_bps)
+{
+	int i;
+	unsigned int min_prediv, max_prediv;
+	unsigned int _prediv0 = 1, _prediv, best_prediv = 0;
+	unsigned long best_freq = 0;
+	unsigned long fvco_min, fvco_max, fin, fout, fvco;
+	unsigned long f_multi, best_multi;
+	unsigned long min_delta = ULONG_MAX;
+	struct pll_parameter *pll_param = &dsi_priv->pll_param;
+
+	fin = dsi_priv->pll_ref_clk;
+	fout = lane_bps / 2;
+
+	/* constraint: 2Mhz <= Fref / N <= 8MHz */
+	min_prediv = DIV_ROUND_UP(fin, 8 * USEC_PER_SEC);
+	max_prediv = fin / (2 * USEC_PER_SEC);
+
+	/* constraint: 320MHz <= Fvco <= 1250Mhz */
+	fvco_min = 320 * USEC_PER_SEC;
+	fvco_max = 1250 * USEC_PER_SEC;
+
+	if (fout * 4 < fvco_min)
+		_prediv0 = 8;
+	else if (fout * 2 < fvco_min)
+		_prediv0 = 4;
+	else if (fout < fvco_min)
+		_prediv0 = 2;
+
+	fvco = fout * _prediv0;
+
+	for (_prediv = min_prediv; _prediv <= max_prediv; _prediv++) {
+		u64 tmp;
+		u32 delta;
+
+		/* Fvco = Fref * M / N */
+		tmp = (u64)fvco * _prediv;
+		do_div(tmp, fin);
+		f_multi = tmp;
+		/* M range */
+		if (f_multi < 64 || f_multi > 625)
+			continue;
+
+		tmp = (u64)f_multi * fin;
+		do_div(tmp, _prediv); // fvco
+		if (tmp < fvco_min || tmp > fvco_max)
+			continue;
+
+		delta = abs(fvco - tmp);
+		if (delta < min_delta) {
+			best_prediv = _prediv;
+			best_multi = f_multi; // M
+			min_delta = delta;
+			best_freq = tmp / _prediv0; // fout
+		}
+	}
+
+	if (best_freq) {
+		pll_param->pll_n = best_prediv - 1;
+		pll_param->pll_m = best_multi - 2;
+		pll_param->actual_freq = best_freq;
+	} else {
+		DRM_ERROR("Can not find best_freq for DPHY\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(pll_tbl); i++) {
+		if (pll_param->actual_freq <=
+		    pll_tbl[i].max_freq * USEC_PER_SEC) {
+			pll_param->max_freq =
+				pll_tbl[i].max_freq * USEC_PER_SEC;
+			/* cfgclkfreqrange[5:0] = round[(Fcfg_clk(MHz)-17)*4] */
+			pll_param->vco_cntrl = pll_tbl[i].vco_cntrl;
+			pll_param->cpbias_cntrl = pll_tbl[i].cpbias_cntrl;
+			pll_param->gmp_cntrl = pll_tbl[i].gmp_cntrl;
+			pll_param->int_cntrl = pll_tbl[i].int_cntrl;
+			pll_param->prop_cntrl = pll_tbl[i].prop_cntrl;
+			break;
+		}
+	}
+
+	DRM_INFO(
+		"max_freq = %d, actual_freq = %d, pll_n = %d, pll_m = %d, i=%d\n",
+		pll_param->max_freq, pll_param->actual_freq, pll_param->pll_n,
+		pll_param->pll_m, i);
+	return 0;
+}
+
+static int es_dsi_get_lane_mbps(void *priv_data,
+				const struct drm_display_mode *mode,
+				unsigned long mode_flags, u32 lanes, u32 format,
+				unsigned int *lane_mbps)
+{
+	int ret, bpp;
+	u64 tmp;
+	struct mipi_dsi_priv *dsi_priv = (struct mipi_dsi_priv *)priv_data;
+
+	DRM_INFO("mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
+	DRM_INFO("lanes: %d, mode_flags: 0x%lx, format: %d\n", lanes,
+		 mode_flags, format);
+
+	dsi_priv->format = format;
+	bpp = mipi_dsi_pixel_format_to_bpp(dsi_priv->format);
+	if (bpp < 0) {
+		DRM_DEV_ERROR(NULL, "failed to get bpp for pixel format %d\n",
+			      dsi_priv->format);
+		return bpp;
+	}
+
+	if (!lanes)
+		lanes = dsi_priv->plat_data.max_data_lanes;
+
+	dsi_priv->mode_flags = mode_flags;
+	dsi_priv->lanes = lanes;
+
+	tmp = (u64)mode->clock * 1000;
+
+	tmp = tmp * bpp / lanes;
+
+	ret = dsi_phy_pll_get_params(dsi_priv, tmp);
+	if (ret)
+		return ret;
+
+	*lane_mbps =
+		DIV_ROUND_UP(dsi_priv->pll_param.actual_freq * 2, USEC_PER_SEC);
+
+	DRM_INFO("bpp = %d, lane_mbps = %d\n", bpp, *lane_mbps);
+
+	return 0;
+}
+
+static int es_dsi_get_timing(void *priv_data, unsigned int lane_mbps,
+			     struct dw_mipi_dsi_dphy_timing *timing)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(hstt_table); i++)
+		if (lane_mbps < hstt_table[i].maxfreq)
+			break;
+
+	if (i == ARRAY_SIZE(hstt_table))
+		i--;
+
+	*timing = hstt_table[i].timing;
+
+	DRM_INFO(
+		"lane_mbps: %d, c_lp2hs: %d, c_hs2lp: %d, d_lp2hs: %d, d_hs2lp: %d\n",
+		lane_mbps, timing->clk_lp2hs, timing->clk_hs2lp,
+		timing->data_lp2hs, timing->data_lp2hs);
+
+	return 0;
+}
+
+struct dw_mipi_dsi_phy_ops dphy_ops = {
+	.init = es_dsi_phy_init,
+	.get_lane_mbps = es_dsi_get_lane_mbps,
+	.get_timing = es_dsi_get_timing,
+};
+
+static int es_mipi_dsi_bind(struct device *dev, struct device *master,
+			    void *data)
+{
+	int ret = 0;
+	unsigned int max_data_lanes;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm_dev = (struct drm_device *)data;
+	struct es_mipi_dsi *es_dsi;
+	struct mipi_dsi_priv *dsi_priv;
+	struct drm_encoder *encoder;
+	struct resource *res;
+
+	es_dsi = devm_kzalloc(&pdev->dev, sizeof(*es_dsi), GFP_KERNEL);
+	if (!es_dsi) {
+		ret = -ENODEV;
+		goto exit0;
+	}
+
+	if (!pdev->dev.of_node) {
+		DRM_ERROR("of_node is null, dev: %px\n", dev);
+		ret = -ENODEV;
+		goto exit0;
+	}
+
+	dsi_priv = &es_dsi->dsi_priv;
+
+	ret = of_property_read_u32(pdev->dev.of_node, "max-lanes",
+				   &max_data_lanes);
+	if (ret) {
+		DRM_WARN("max-lanes not defined, default max-lanes is 4\n");
+		max_data_lanes = MAX_LANE_COUNT;
+	}
+	dsi_priv->plat_data.max_data_lanes = max_data_lanes;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	dsi_priv->dphy_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(dsi_priv->dphy_base)) {
+		DRM_ERROR("get mm_sys register base failed, of_node: %pOF\n",
+			  dev->of_node);
+		ret = PTR_ERR(dsi_priv->dphy_base);
+		goto exit0;
+	}
+
+	dsi_priv->plat_data.base = dsi_priv->dphy_base;
+
+	dsi_priv->plat_data.phy_ops = &dphy_ops;
+	dsi_priv->plat_data.priv_data = dsi_priv;
+	dsi_priv->pll_ref_clk = PLL_INPUT_REF_CLK;
+
+	dsi_priv->ivideo_clk = devm_clk_get(dev, "pclk");
+	if (IS_ERR(dsi_priv->ivideo_clk)) {
+		DRM_ERROR("get ivideo clk failed, of_node: %pOF\n",
+			  dev->of_node);
+		ret = PTR_ERR(dsi_priv->ivideo_clk);
+		goto exit0;
+	}
+
+	/* dsi phyctl reset */
+	dsi_priv->rst_dsi_phyrstn =
+		devm_reset_control_get_optional(dev, "phyrstn");
+	if (IS_ERR_OR_NULL(dsi_priv->rst_dsi_phyrstn)) {
+		DRM_ERROR("Failed to get dsi phyrstn reset handle\n");
+		goto exit0;
+	}
+	if (dsi_priv->rst_dsi_phyrstn) {
+		ret = reset_control_reset(dsi_priv->rst_dsi_phyrstn);
+		WARN_ON(0 != ret);
+	}
+
+	encoder = &es_dsi->encoder;
+	encoder->possible_crtcs =
+		drm_of_find_possible_crtcs(drm_dev, dev->of_node);
+	if (encoder->possible_crtcs == 0) {
+		DRM_ERROR("Failed to find possible_crtcs:0\n");
+	}
+
+	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_DSI);
+	if (ret) {
+		DRM_ERROR("Failed to initialize encoder with drm\n");
+		return ret;
+	}
+
+	drm_encoder_helper_add(encoder, &es_dsi_encoder_helper_funcs);
+
+	platform_set_drvdata(pdev, es_dsi);
+
+	dsi_priv->data = dw_mipi_dsi_probe(pdev, &dsi_priv->plat_data);
+	if (IS_ERR(dsi_priv->data)) {
+		DRM_ERROR("probe dw-mipi dsi failed\n");
+		ret = -ENODEV;
+		goto exit2;
+	}
+
+	ret = dw_mipi_dsi_bind((struct dw_mipi_dsi *)dsi_priv->data, encoder);
+	if (ret)
+		goto exit3;
+
+	DRM_INFO("mipi dsi bind done\n");
+
+	return 0;
+
+exit3:
+	dw_mipi_dsi_remove((struct dw_mipi_dsi *)dsi_priv->data);
+exit2:
+	drm_encoder_cleanup(encoder);
+exit0:
+	DRM_INFO("mipi dsi bind failed, ret = %d\n", ret);
+	return ret;
+}
+
+static void es_mipi_dsi_unbind(struct device *dev, struct device *master,
+			       void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct es_mipi_dsi *es_dsi =
+		(struct es_mipi_dsi *)platform_get_drvdata(pdev);
+	struct mipi_dsi_priv *dsi_priv = &es_dsi->dsi_priv;
+
+	DRM_INFO("mipi dsi unbind\n");
+
+	dw_mipi_dsi_unbind((struct dw_mipi_dsi *)dsi_priv->data);
+	dw_mipi_dsi_remove((struct dw_mipi_dsi *)dsi_priv->data);
+	drm_encoder_cleanup(&es_dsi->encoder);
+}
+
+static const struct component_ops es_mipi_dsi_ops = {
+	.bind = es_mipi_dsi_bind,
+	.unbind = es_mipi_dsi_unbind,
+};
+
+static int es_mipi_dsi_probe(struct platform_device *pdev)
+{
+	mipi_dsi_driver_register(&es_panel_driver);
+
+	return component_add(&pdev->dev, &es_mipi_dsi_ops);
+}
+
+static int es_mipi_dsi_remove(struct platform_device *pdev)
+{
+	DRM_INFO("mipi dsi remove\n");
+	mipi_dsi_driver_unregister(&es_panel_driver);
+	component_del(&pdev->dev, &es_mipi_dsi_ops);
+
+	return 0;
+}
+
+struct platform_driver es_mipi_dsi_driver = {
+	.probe = es_mipi_dsi_probe,
+	.remove = es_mipi_dsi_remove,
+	.driver = {
+		   .of_match_table = es_mipi_dsi_dt_ids,
+		   .name = "mipi-dsi-drv",
+	},
+};
+
+MODULE_AUTHOR("lilijun@eswin.com");
+MODULE_DESCRIPTION("Eswin dw dsi driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_mipi_dsi.h b/drivers/gpu/drm/eswin/es_mipi_dsi.h
new file mode 100644
index 000000000000..4a757e527736
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_mipi_dsi.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_MIPI_DSI_H__
+#define __ES_MIPI_DSI_H__
+
+extern struct platform_driver es_mipi_dsi_driver;
+
+#endif // __ES_MIPI_DSI_H__
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/es_panel.c b/drivers/gpu/drm/eswin/es_panel.c
new file mode 100644
index 000000000000..7c2bd90393e2
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_panel.c
@@ -0,0 +1,943 @@
+#include "es_panel.h"
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_modes.h>
+#include <linux/backlight.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+#include <linux/media-bus-format.h>
+
+#include <video/mipi_display.h>
+#include <video/of_videomode.h>
+#include <video/videomode.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/slab.h>
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/list.h>
+#include <linux/gpio/consumer.h>
+#include <drm/drm_print.h>
+#include <linux/version.h>
+
+#define ES_PANEL_MAJOR 0
+#define ES_PANEL_NR_DEVS 1
+#define ES_PANEL_NAME "es_panel"
+
+static int panel_major = ES_PANEL_MAJOR;
+static int panel_minor = 0;
+static int panel_nr_devs = ES_PANEL_NR_DEVS;
+static struct device *pr_dev = NULL;
+
+typedef struct user_cmd_buffer_s {
+	struct list_head head;
+	u32 sleepUs;
+	int buf_size;
+	u8 *buf;
+} user_cmd_buffer_t;
+
+struct es_panel {
+	struct device *dev;
+	struct drm_panel panel;
+	struct backlight_device *backlight;
+	struct gpio_desc *reset_gpio;
+	struct gpio_desc *bias_pos, *bias_neg;
+	bool prepared;
+	bool enabled;
+	int error;
+
+	// chr dev:
+	struct cdev cdev;
+	dev_t devt;
+	struct class *es_panel_class;
+
+	// user data:
+	MIPI_MODE_INFO_S mode_info;
+	struct drm_display_mode user_mode;
+	bool user_mode_inited;
+
+	struct list_head init_cmd_list;
+	int init_cmd_writted;
+	user_cmd_buffer_t enable_cmd_buf;
+	user_cmd_buffer_t disable_cmd_buf;
+
+	struct gpio_desc *gpio_backlight0;
+	struct gpio_desc *gpio_reset;
+};
+
+static void es_panel_dcs_write(struct es_panel *ctx, const void *data,
+			       size_t len);
+
+#define es_panel_dcs_write_seq_static(ctx, seq...)         \
+	({                                                 \
+		static const u8 d[] = { seq };             \
+		es_panel_dcs_write(ctx, d, ARRAY_SIZE(d)); \
+		udelay(10);                                \
+	})
+
+#define es_panel_check_retval(x)     \
+	do {                         \
+		if ((x))             \
+			return -EIO; \
+	} while (0)
+
+// #ifdef CONFIG_1080P_4LANES_PANEL
+#define HFP (60)
+#define HSA (60)
+#define HBP (60)
+#define VFP (8)
+#define VSA (8)
+#define VBP (8)
+#define VAC (1920)
+#define HAC (1080)
+#define LANES 4
+#define PIX_CLK 148500
+// #elif CONFIG_720P_3LANES_PANEL
+// #define HFP (30)
+// #define HSA (30)
+// #define HBP (30)
+// #define VFP (30)
+// #define VSA (30)
+// #define VBP (30)
+// #define VAC (1280)
+// #define HAC (720)
+// #define LANES 3
+// #define PIX_CLK 59400
+// #else
+// #define HFP (60)
+// #define HSA (60)
+// #define HBP (60)
+// #define VFP (8)
+// #define VSA (8)
+// #define VBP (8)
+// #define VAC (1920)
+// #define HAC (1080)
+// #define LANES 4
+// #define PIX_CLK 148500
+// #endif
+
+static struct drm_display_mode default_mode = {
+	.clock = PIX_CLK,
+	.hdisplay = HAC,
+	.hsync_start = HAC + HFP,
+	.hsync_end = HAC + HFP + HSA,
+	.htotal = HAC + HFP + HSA + HBP,
+	.vdisplay = VAC,
+	.vsync_start = VAC + VFP,
+	.vsync_end = VAC + VFP + VSA,
+	.vtotal = VAC + VFP + VSA + VBP,
+	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+};
+
+void GP_COMMAD_PA(struct es_panel *ctx, u8 a)
+{
+	return;
+	// u8 b[3] = {0};
+	// b[0] = 0xBC;
+	// b[1] = a;
+	// b[2] = a >> 8;
+	// es_panel_dcs_write(ctx, b, 3);
+	// es_panel_dcs_write_seq_static(ctx, 0xBf);
+}
+
+static void free_cmd_buf(user_cmd_buffer_t *pcmd_buf)
+{
+	if (pcmd_buf->buf) {
+		pcmd_buf->buf_size = 0;
+		vfree(pcmd_buf->buf);
+		pcmd_buf->buf = NULL;
+	}
+	return;
+}
+
+void free_cmd_list(struct list_head *list)
+{
+	if (!list_empty(list)) {
+		user_cmd_buffer_t *pos = NULL, *n = NULL;
+		list_for_each_entry_safe(pos, n, list, head) {
+			if (pos) {
+				list_del_init(&pos->head);
+				if (pos->buf) {
+					vfree(pos->buf);
+					pos->buf = NULL;
+				}
+				vfree(pos);
+				pos = NULL;
+			}
+		}
+	}
+}
+
+static int parser_and_creat_cmd_buf(MIPI_CMD_S *cmd, user_cmd_buffer_t *cmd_buf)
+{
+	if (!cmd) {
+		dev_err(pr_dev, "cmd invalid\n");
+		return -EPERM;
+	}
+	if (!cmd_buf) {
+		dev_err(pr_dev, "cmd_buf invalid\n");
+		return -EPERM;
+	}
+
+	if (cmd_buf->buf) {
+		vfree(cmd_buf->buf);
+		cmd_buf->buf = NULL;
+	}
+
+	if (cmd->dataType == 0x13) {
+		cmd_buf->buf_size = 1;
+		cmd_buf->buf = vmalloc(cmd_buf->buf_size);
+		cmd_buf->buf[0] = cmd->cmdSize;
+	} else if (cmd->dataType == 0x23) {
+		cmd_buf->buf_size = 2;
+		cmd_buf->buf = vmalloc(cmd_buf->buf_size);
+		cmd_buf->buf[0] = cmd->cmdSize >> 8;
+		cmd_buf->buf[1] = cmd->cmdSize & 0x00ff;
+	} else if (cmd->dataType == 0x29) {
+		cmd_buf->buf_size = cmd->cmdSize;
+		cmd_buf->buf = vmalloc(cmd_buf->buf_size);
+		if (copy_from_user(cmd_buf->buf, (u8 __user *)cmd->pCmd,
+				   cmd_buf->buf_size)) {
+			dev_err(pr_dev, "cmd buffer copy_from_user failed\n");
+			return -ENOMEM;
+		}
+	}
+	cmd_buf->sleepUs = cmd->sleepUs;
+	dev_dbg(pr_dev, "user set cmd size:%d, sleep:%dus,cmd:0x%x \n",
+		cmd_buf->buf_size, cmd_buf->sleepUs, cmd_buf->buf[0]);
+	return 0;
+}
+
+int es_panel_fops_open(struct inode *inode, struct file *filp)
+{
+	struct es_panel *ctx;
+	ctx = container_of(inode->i_cdev, struct es_panel, cdev);
+	filp->private_data = ctx;
+	return 0;
+}
+
+int es_panel_fops_release(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static long es_panel_fops_ioctl(struct file *file, unsigned int cmd,
+				unsigned long arg)
+{
+	struct es_panel *ctx;
+	struct mipi_dsi_device *dsi;
+	MIPI_CMD_S ucmd;
+	int ret = 0;
+
+	ctx = (struct es_panel *)file->private_data;
+	dsi = to_mipi_dsi_device(ctx->dev);
+
+	switch (cmd) {
+	case IOC_ES_MIPI_TX_SET_MODE:
+		es_panel_check_retval(copy_from_user(
+			&ctx->mode_info, (MIPI_MODE_INFO_S __user *)arg,
+			sizeof(MIPI_MODE_INFO_S)));
+		dev_dbg(pr_dev,
+			"cmd: IOC_ES_MIPI_TX_SET_MODE, devId:%d, hdisplay:%u, hsyncStart:%u, hsyncEnd:%u, htotal:%u, hskew:%u\n"
+			"vdisplay:%u vsyncStart:%u vsyncEnd:%u vtotal:%u vscan:%u vrefresh:%d flags:%d type:%d "
+			"outputFormat:%d outputMode:%d lanes:%d\n",
+			ctx->mode_info.devId, ctx->mode_info.hdisplay,
+			ctx->mode_info.hsyncStart, ctx->mode_info.hsyncEnd,
+			ctx->mode_info.htotal, ctx->mode_info.hskew,
+			ctx->mode_info.vdisplay, ctx->mode_info.vsyncStart,
+			ctx->mode_info.vsyncEnd, ctx->mode_info.vtotal,
+			ctx->mode_info.vscan, ctx->mode_info.vrefresh,
+			ctx->mode_info.flags, ctx->mode_info.type,
+			ctx->mode_info.outputFormat, ctx->mode_info.outputMode,
+			ctx->mode_info.lanes);
+
+		memset(&ctx->user_mode, 0, sizeof(struct drm_display_mode));
+		ctx->user_mode.clock = 12500;
+		ctx->user_mode.hdisplay = ctx->mode_info.hdisplay;
+		ctx->user_mode.hsync_start = ctx->mode_info.hsyncStart;
+		ctx->user_mode.hsync_end = ctx->mode_info.hsyncEnd;
+		ctx->user_mode.htotal = ctx->mode_info.htotal;
+		ctx->user_mode.vdisplay = ctx->mode_info.vdisplay;
+		ctx->user_mode.vsync_start = ctx->mode_info.vsyncStart;
+		ctx->user_mode.vsync_end = ctx->mode_info.vsyncEnd;
+		ctx->user_mode.vtotal = ctx->mode_info.vtotal;
+		ctx->user_mode.width_mm = 36;
+		ctx->user_mode.height_mm = 49;
+		ctx->user_mode.flags = DRM_MODE_FLAG_NVSYNC |
+				       DRM_MODE_FLAG_NHSYNC;
+		ctx->user_mode_inited = true;
+
+		dsi->lanes = ctx->mode_info.lanes;
+		if (ctx->mode_info.outputFormat == MIPI_OUT_FORMAT_RGB_24_BIT) {
+			dsi->format = MIPI_DSI_FMT_RGB888;
+		} else {
+			dev_err(pr_dev, "panel not support fmt:%d now",
+				ctx->mode_info.outputFormat);
+		}
+		dsi->mode_flags = MIPI_DSI_MODE_VIDEO;
+
+		break;
+	case IOC_ES_MIPI_TX_SET_INIT_CMD:
+		user_cmd_buffer_t *cmd_buf = NULL;
+
+		if (ctx->init_cmd_writted) {
+			dev_info(pr_dev,
+				 "init_cmd_writted : clean init cmd list\n");
+			free_cmd_list(&ctx->init_cmd_list);
+			INIT_LIST_HEAD(&ctx->init_cmd_list);
+			ctx->init_cmd_writted = 0;
+		}
+		cmd_buf = vmalloc(sizeof(user_cmd_buffer_t));
+		if (!cmd_buf) {
+			dev_err(pr_dev, "cmd buffer alloc failed\n");
+			return -ENOMEM;
+		}
+		memset(cmd_buf, 0, sizeof(user_cmd_buffer_t));
+
+		es_panel_check_retval(copy_from_user(
+			&ucmd, (MIPI_CMD_S __user *)arg, sizeof(MIPI_CMD_S)));
+		ret = parser_and_creat_cmd_buf(&ucmd, cmd_buf);
+		if (ret) {
+			break;
+		}
+		if (!cmd_buf) {
+			dev_err(pr_dev, "cmd_buf creat failed!!!!");
+			break;
+		}
+		list_add_tail(&cmd_buf->head, &ctx->init_cmd_list);
+
+		break;
+	case IOC_ES_MIPI_TX_ENABLE:
+		user_cmd_buffer_t *enable_cmd_buf = &ctx->enable_cmd_buf;
+		dev_dbg(pr_dev, "IOC_ES_MIPI_TX_ENABLE In");
+		es_panel_check_retval(copy_from_user(
+			&ucmd, (MIPI_CMD_S __user *)arg, sizeof(MIPI_CMD_S)));
+		ret = parser_and_creat_cmd_buf(&ucmd, enable_cmd_buf);
+		break;
+	case IOC_ES_MIPI_TX_DISABLE:
+		user_cmd_buffer_t *disable_cmd_buf = &ctx->disable_cmd_buf;
+		dev_dbg(pr_dev, "IOC_ES_MIPI_TX_DISABLE In");
+		es_panel_check_retval(copy_from_user(
+			&ucmd, (MIPI_CMD_S __user *)arg, sizeof(MIPI_CMD_S)));
+		ret = parser_and_creat_cmd_buf(&ucmd, disable_cmd_buf);
+		break;
+	default:
+		dev_err(pr_dev, "unsupported command %d", cmd);
+		break;
+	}
+	return ret;
+}
+
+struct file_operations es_panel_fops = {
+	.owner = THIS_MODULE,
+	.open = es_panel_fops_open,
+	.release = es_panel_fops_release,
+	.unlocked_ioctl = es_panel_fops_ioctl,
+};
+
+int es_panel_chrdev_create(struct es_panel *ctx)
+{
+	int ret = 0;
+	dev_dbg(pr_dev, "---BEGIN es_panel_chrdev_create---\n");
+
+	ret = alloc_chrdev_region(&ctx->devt, panel_minor, panel_nr_devs,
+				  ES_PANEL_NAME);
+	if (ret < 0) {
+		dev_err(pr_dev,
+			"es-panel-chrdev: can't allocate major number\n");
+		goto fail;
+	}
+	panel_major = MAJOR(ctx->devt);
+
+	cdev_init(&ctx->cdev, &es_panel_fops);
+	ctx->cdev.owner = THIS_MODULE;
+
+	ret = cdev_add(&ctx->cdev, ctx->devt, 1);
+	if (ret) {
+		dev_err(pr_dev, "failed to add cdev\n");
+		goto unregister_chrdev;
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+	ctx->es_panel_class = class_create(ES_PANEL_NAME);
+#else
+	ctx->es_panel_class = class_create(THIS_MODULE, ES_PANEL_NAME);
+#endif
+	if (IS_ERR(ctx->es_panel_class)) {
+		dev_err(pr_dev, "class_create error\n");
+		ret = PTR_ERR(ctx->es_panel_class);
+		goto del_cdev;
+	}
+
+	if (device_create(ctx->es_panel_class, NULL, ctx->devt, ctx,
+			  ES_PANEL_NAME) == NULL) {
+		dev_err(pr_dev, "device_create error\n");
+		ret = -EINVAL;
+		goto destroy_class;
+	}
+
+	dev_dbg(pr_dev, "---END es_panel_chrdev_create---\n");
+	return 0;
+
+destroy_class:
+	class_destroy(ctx->es_panel_class);
+del_cdev:
+	cdev_del(&ctx->cdev);
+unregister_chrdev:
+	unregister_chrdev_region(ctx->devt, panel_nr_devs);
+fail:
+	return ret;
+}
+
+void es_panel_chrdev_destroy(struct es_panel *ctx)
+{
+	device_destroy(ctx->es_panel_class, MKDEV(panel_major, panel_minor));
+	class_destroy(ctx->es_panel_class);
+
+	cdev_del(&ctx->cdev);
+
+	unregister_chrdev_region(ctx->devt, panel_nr_devs);
+	dev_dbg(pr_dev, "es_panel_chrdev_destroy done\n");
+}
+
+static inline struct es_panel *panel_to_es_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct es_panel, panel);
+}
+
+static void es_panel_dcs_write(struct es_panel *ctx, const void *data,
+			       size_t len)
+{
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+	ssize_t ret;
+	char *addr;
+
+	if (ctx->error < 0)
+		return;
+
+	addr = (char *)data;
+	if ((int)*addr < 0xB0)
+		ret = mipi_dsi_dcs_write_buffer(dsi, data, len);
+	else
+		ret = mipi_dsi_generic_write(dsi, data, len);
+	if (ret < 0) {
+		dev_err(ctx->dev, "error %zd writing seq: %ph\n", ret, data);
+		ctx->error = ret;
+	}
+	dev_dbg(ctx->dev, "write cmd size:%ld, data0[0x%x]  ret:%d\n", len,
+		addr[0], ctx->error);
+}
+
+#ifdef PANEL_SUPPORT_READBACK
+static int es_panel_dcs_read(struct es_panel *ctx, u8 cmd, void *data,
+			     size_t len)
+{
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+	ssize_t ret;
+
+	if (ctx->error < 0)
+		return 0;
+
+	ret = mipi_dsi_dcs_read(dsi, cmd, data, len);
+	if (ret < 0) {
+		dev_err(ctx->dev, "error %d reading dcs seq:(%#x)\n", ret, cmd);
+		ctx->error = ret;
+	}
+
+	return ret;
+}
+
+static void es_panel_panel_get_data(struct es_panel *ctx)
+{
+	u8 buffer[3] = { 0 };
+	static int ret;
+
+	if (ret == 0) {
+		ret = es_panel_dcs_read(ctx, 0x0A, buffer, 1);
+		dev_info(ctx->dev, "return %d data(0x%08x) to dsi engine\n",
+			 ret, buffer[0] | (buffer[1] << 8));
+	}
+}
+#endif
+
+void es_panel_panel_init(struct es_panel *ctx)
+{
+	user_cmd_buffer_t *pos = NULL;
+	list_for_each_entry(pos, &ctx->init_cmd_list, head) {
+		if (pos) {
+			es_panel_dcs_write(ctx, pos->buf, pos->buf_size);
+			if (pos->sleepUs) {
+				msleep(pos->sleepUs / 1000);
+			}
+		}
+	}
+	ctx->init_cmd_writted = 1;
+}
+
+void es_panel_3_lanes_panel_init(struct es_panel *ctx)
+{
+	dev_info(ctx->dev, "init 3lanes panel\n");
+	GP_COMMAD_PA(ctx, 0x04);
+	es_panel_dcs_write_seq_static(ctx, 0xB9, 0xF1, 0x12, 0x83);
+	GP_COMMAD_PA(ctx, 0x1C);
+	es_panel_dcs_write_seq_static(ctx, 0xBA, 0x32, 0x81, 0x05, 0xF9, 0x0E,
+				      0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x44, 0x25, 0x00, 0x91, 0x0A,
+				      0x00, 0x00, 0x02, 0x4F, 0xD1, 0x00, 0x00,
+				      0x37);
+	GP_COMMAD_PA(ctx, 0x05);
+	es_panel_dcs_write_seq_static(ctx, 0xB8, 0x26, 0x22, 0x20, 0x00);
+	GP_COMMAD_PA(ctx, 0x04);
+	es_panel_dcs_write_seq_static(ctx, 0xBF, 0x02, 0x11, 0x00);
+	GP_COMMAD_PA(ctx, 0x0B);
+	es_panel_dcs_write_seq_static(ctx, 0xB3, 0x0C, 0x10, 0x0A, 0x50, 0x03,
+				      0xFF, 0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x0A);
+	es_panel_dcs_write_seq_static(ctx, 0xB3, 0x0C, 0x10, 0x0A, 0x50, 0x03,
+				      0xFF, 0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x02);
+	es_panel_dcs_write_seq_static(ctx, 0xBC, 0x46);
+	GP_COMMAD_PA(ctx, 0x02);
+	es_panel_dcs_write_seq_static(ctx, 0xCC, 0x0B);
+	GP_COMMAD_PA(ctx, 0x02);
+	es_panel_dcs_write_seq_static(ctx, 0xB4, 0x80);
+	GP_COMMAD_PA(ctx, 0x04);
+	es_panel_dcs_write_seq_static(ctx, 0xB2, 0xC8, 0x02, 0x30);
+	GP_COMMAD_PA(ctx, 0x0F);
+	es_panel_dcs_write_seq_static(ctx, 0xE3, 0x07, 0x07, 0x0B, 0x0B, 0x03,
+				      0x0B, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x80,
+				      0xC0, 0x10);
+	GP_COMMAD_PA(ctx, 0x0D);
+	es_panel_dcs_write_seq_static(ctx, 0xC1, 0x25, 0x00, 0x1E, 0x1E, 0x77,
+				      0xF1, 0xFF, 0xFF, 0xCC, 0xCC, 0x77, 0x77);
+	GP_COMMAD_PA(ctx, 0x03);
+	es_panel_dcs_write_seq_static(ctx, 0xB5, 0x0A, 0x0A);
+	GP_COMMAD_PA(ctx, 0x03);
+	es_panel_dcs_write_seq_static(ctx, 0xB6, 0x50, 0x50);
+	GP_COMMAD_PA(ctx, 0x40);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xE9, 0xC4, 0x10, 0x0F, 0x00, 0x00, 0xB2, 0xB8, 0x12, 0x31,
+		0x23, 0x48, 0x8B, 0xB2, 0xB8, 0x47, 0x20, 0x00, 0x00, 0x30,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x02,
+		0x46, 0x02, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0xF8,
+		0x13, 0x57, 0x13, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88,
+		0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x3E);
+	es_panel_dcs_write_seq_static(ctx, 0xEA, 0x00, 0x1A, 0x00, 0x00, 0x00,
+				      0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x75, 0x31, 0x31, 0x88, 0x88, 0x88,
+				      0x88, 0x88, 0x88, 0x88, 0x8F, 0x64, 0x20,
+				      0x20, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88,
+				      0x88, 0x8F, 0x23, 0x10, 0x00, 0x00, 0x02,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x30, 0x02, 0xA0, 0x00, 0x00, 0x00, 0x00);
+	GP_COMMAD_PA(ctx, 0x23);
+	es_panel_dcs_write_seq_static(ctx, 0xE0, 0x01, 0x11, 0x16, 0x28, 0x30,
+				      0x38, 0x45, 0x39, 0x07, 0x0B, 0x0D, 0x12,
+				      0x14, 0x12, 0x14, 0x11, 0x19, 0x01, 0x11,
+				      0x16, 0x28, 0x30, 0x38, 0x45, 0x39, 0x07,
+				      0x0B, 0x0D, 0x12, 0x14, 0x12, 0x14, 0x11,
+				      0x19);
+
+	es_panel_dcs_write_seq_static(ctx, 0xC7, 0xE7, 0x00);
+	// es_panel_dcs_write_seq_static(ctx,0xC8, 0x10, 0x40);
+	es_panel_dcs_write_seq_static(ctx, 0x51,
+				      0xFF); // brightness [0x00-> 0xFF]
+	es_panel_dcs_write_seq_static(ctx, 0x53, (1 << 2) | (1 << 5));
+	msleep(20);
+}
+
+void es_panel_4_lanes_panel_init(struct es_panel *ctx)
+{
+	dev_info(ctx->dev, "init 4lanes panel\n");
+	es_panel_dcs_write_seq_static(ctx, 0xB0, 0x98, 0x85, 0x0A);
+	es_panel_dcs_write_seq_static(ctx, 0xC1, 0x00, 0x00,
+				      0x00); // 01反扫  00正扫
+	es_panel_dcs_write_seq_static(ctx, 0xC4, 0x70, 0x19, 0x23, 0x00, 0x0F,
+				      0x0F, 0x00);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xD0, 0x33, 0x05, 0x21, 0xE7, 0x62, 0x00, 0x88, 0xA0, 0xA0,
+		0x03, 0x02, 0x80, 0xD0, 0x1B,
+		0x10); // VGH dual mode,      VGL single mode,      VGH=12V,      VGL=-12V
+	es_panel_dcs_write_seq_static(ctx, 0xD2, 0x13, 0x13, 0xEA, 0x22);
+	es_panel_dcs_write_seq_static(ctx, 0xD1, 0x09, 0x09,
+				      0xc2); ////4003 & 4003B EN
+	es_panel_dcs_write_seq_static(ctx, 0xD3, 0x44, 0x33, 0x05, 0x03, 0x4A,
+				      0x4A, 0x33, 0x17, 0x22, 0x43,
+				      0x6E); // set GVDDP=4.1V, GVDDN=-4.1V,
+		// VGHO=12V,      VGLO=-11V
+	es_panel_dcs_write_seq_static(ctx, 0xD5, 0x8B, 0x00, 0x00, 0x00, 0x01,
+				      0x7D, 0x01, 0x7D, 0x01, 0x7D, 0x00, 0x00,
+				      0x00, 0x00); // set VCOM
+	es_panel_dcs_write_seq_static(ctx, 0xD6, 0x00, 0x00, 0x08, 0x17, 0x23,
+				      0x65, 0x77, 0x44, 0x87, 0x00, 0x00,
+				      0x09); // P12_D[3] for sleep in
+		// current reduce setting
+	es_panel_dcs_write_seq_static(ctx, 0xEC, 0x76, 0x1E, 0x32, 0x00, 0x46,
+				      0x00, 0x00);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xC7, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x2D, 0x00, 0x43, 0x00,
+		0x58, 0x00, 0x6C, 0x00, 0x75, 0x00, 0x8C, 0x00, 0x9A, 0x00,
+		0xCA, 0x00, 0xF1, 0x01, 0x2F, 0x01, 0x5F, 0x01, 0xAE, 0x01,
+		0xEC, 0x01, 0xEE, 0x02, 0x25, 0x02, 0x62, 0x02, 0x8A, 0x02,
+		0xC4, 0x02, 0xEA, 0x03, 0x1F, 0x03, 0x33, 0x03, 0x3E, 0x03,
+		0x59, 0x03, 0x70, 0x03, 0x88, 0x03, 0xB4, 0x03, 0xC8, 0x03,
+		0xE8, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x2D, 0x00, 0x43, 0x00,
+		0x58, 0x00, 0x6C, 0x00, 0x75, 0x00, 0x8C, 0x00, 0x9A, 0x00,
+		0xCA, 0x00, 0xF1, 0x01, 0x2F, 0x01, 0x5F, 0x01, 0xAE, 0x01,
+		0xEC, 0x01, 0xEE, 0x02, 0x25, 0x02, 0x62, 0x02, 0x8A, 0x02,
+		0xC4, 0x02, 0xEA, 0x03, 0x1F, 0x03, 0x33, 0x03, 0x3E, 0x03,
+		0x59, 0x03, 0x70, 0x03, 0x88, 0x03, 0xB4, 0x03, 0xC8, 0x03,
+		0xE8, 0x01, 0x01);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xE5, 0x6D, 0x92, 0x80, 0x34, 0x76, 0x12, 0x12, 0x00, 0x00,
+		0x36, 0x36, 0x24, 0x24, 0x26, 0xF6, 0xF6, 0xF6, 0xF6, 0xF6,
+		0xF6, 0x6D, 0x9B, 0x89, 0x34, 0x76, 0x1B, 0x1B, 0x09, 0x09,
+		0x3F, 0x3F, 0x2D, 0x2D, 0x26, 0xF6, 0xF6, 0xF6, 0xF6, 0xF6,
+		0xF6, 0x04, 0x40, 0x90, 0x00, 0xD6, 0x00, 0x00, 0x00, 0x00,
+		0x01, 0x01, 0x44, 0x00, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x03, 0x03,
+		0x03, 0x03, 0x00, 0x07);
+	es_panel_dcs_write_seq_static(
+		ctx, 0xEA, 0x70, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x40, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x89,
+		0x8a, 0x05, 0x05, 0x22, 0x0a, 0x0a, 0x0b, 0x00, 0x08, 0x00,
+		0x30, 0x01, 0x09, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x00, 0x01,
+		0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xDD, 0x22, 0x22,
+		0xCC, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0xCC, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33);
+	es_panel_dcs_write_seq_static(ctx, 0xEE, 0x22, 0x10, 0x02, 0x02, 0x0F,
+				      0x40, 0x00, 0x07, 0x00, 0x04, 0x00, 0x00,
+				      0x00, 0xB9, 0x77, 0x00, 0x55, 0x05,
+				      0x00); // 4003
+	es_panel_dcs_write_seq_static(ctx, 0xEB, 0xa3, 0xcf, 0x73, 0x18, 0x54,
+				      0x55, 0x55, 0x55, 0x55, 0x00, 0x55, 0x55,
+				      0x55, 0x55, 0x55, 0x25, 0x0D, 0x0F, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x55, 0x55, 0x55,
+				      0x55, 0x32, 0x3E, 0x55, 0x43, 0x55);
+
+	es_panel_dcs_write_seq_static(ctx, 0x51, 0x0F,
+				      0xFF); // brightness [0x00-> 0xFFF]
+	es_panel_dcs_write_seq_static(ctx, 0x53, (1 << 2) | (1 << 5));
+}
+
+static int es_panel_disable(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	int ret = 0;
+	user_cmd_buffer_t *cmd_buf = &ctx->disable_cmd_buf;
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(panel->dev);
+
+	dev_dbg(pr_dev, "%s:%d size:%x ctx->enabled:%d\n", __func__, __LINE__,
+		ctx->disable_cmd_buf.buf_size, ctx->enabled);
+
+	if (!ctx->enabled)
+		return 0;
+
+	if (ctx->backlight) {
+		ctx->backlight->props.power = FB_BLANK_POWERDOWN;
+		backlight_update_status(ctx->backlight);
+	}
+#if 1
+	gpiod_set_value(ctx->gpio_backlight0, 0);
+#endif
+
+	ctx->enabled = false;
+	if (cmd_buf->buf_size > 0) {
+		if (cmd_buf->sleepUs) {
+			msleep(cmd_buf->sleepUs / 1000);
+		}
+		es_panel_dcs_write(ctx, cmd_buf->buf, cmd_buf->buf_size);
+	} else {
+		dev_warn(pr_dev, "%s[%d]:Disable cmd invalid,Please set it\n",
+			 __func__, __LINE__);
+	}
+
+	dev_info(pr_dev, "[%s]display off... sleep in..\n", __func__);
+	ret = mipi_dsi_dcs_set_display_off(dsi);
+	if (ret < 0)
+		dev_err(ctx->dev, "Failed to turn off the display: %d\n", ret);
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
+	if (ret < 0)
+		dev_err(ctx->dev, "Failed to enter sleep mode: %d\n", ret);
+
+	return ret;
+}
+
+static int es_panel_unprepare(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+
+	if (!ctx->prepared)
+		return 0;
+
+	ctx->error = 0;
+	dev_dbg(pr_dev, "%s:%d\n", __func__, __LINE__);
+	ctx->prepared = false;
+	return 0;
+}
+
+static int es_panel_prepare(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	int ret;
+
+	if (ctx->prepared)
+		return 0;
+	dev_dbg(pr_dev, "%s:%d\n", __func__, __LINE__);
+
+	// #ifdef CONFIG_1080P_4LANES_PANEL
+	es_panel_4_lanes_panel_init(ctx);
+	// #elif CONFIG_720P_3LANES_PANEL
+	//     es_panel_3_lanes_panel_init(ctx);
+	// #else
+	//     es_panel_panel_init(ctx);
+	// #endif
+
+	ret = ctx->error;
+	if (ret < 0)
+		es_panel_unprepare(panel);
+	ctx->prepared = true;
+#ifdef PANEL_SUPPORT_READBACK
+	es_panel_panel_get_data(ctx);
+#endif
+
+	return ret;
+}
+
+static int es_panel_enable(struct drm_panel *panel)
+{
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	int ret = 0;
+	user_cmd_buffer_t *cmd_buf = &ctx->enable_cmd_buf;
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(panel->dev);
+
+	if (ctx->enabled)
+		return 0;
+
+	if (ctx->backlight) {
+		ctx->backlight->props.power = FB_BLANK_UNBLANK;
+		backlight_update_status(ctx->backlight);
+	}
+#if 1
+	gpiod_set_value(ctx->gpio_backlight0, 1);
+#endif
+
+	ctx->enabled = true;
+	if (cmd_buf->buf_size > 0) {
+		if (cmd_buf->sleepUs) {
+			msleep(cmd_buf->sleepUs / 1000);
+		}
+		es_panel_dcs_write(ctx, cmd_buf->buf, cmd_buf->buf_size);
+	} else {
+		dev_warn(pr_dev, "%s[%d]:Enable cmd invalid,Please set it\n",
+			 __func__, __LINE__);
+	}
+
+	dev_info(pr_dev, "[%s] sleep out.., display on...\n", __func__);
+	msleep(20);
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(ctx->dev, "Failed to exit sleep mode: %d\n", ret);
+		return ret;
+	}
+	/* Panel is operational 120 msec after reset */
+	msleep(200);
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret)
+		return ret;
+	msleep(50);
+
+	return ret;
+}
+
+static int es_panel_get_modes(struct drm_panel *panel,
+			      struct drm_connector *connector)
+{
+	u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	struct es_panel *ctx = panel_to_es_panel(panel);
+	struct drm_display_mode *mode;
+
+	dev_dbg(pr_dev, "%s:%d\n", __func__, __LINE__);
+	if (ctx->user_mode_inited == false) {
+		mode = drm_mode_duplicate(connector->dev, &default_mode);
+		if (!mode) {
+			dev_err(panel->dev, "failed to add mode %ux%ux\n",
+				default_mode.hdisplay, default_mode.vdisplay);
+			return -ENOMEM;
+		}
+	} else {
+		mode = drm_mode_duplicate(connector->dev, &ctx->user_mode);
+		if (!mode) {
+			dev_err(panel->dev, "failed to add mode %ux%ux\n",
+				ctx->user_mode.hdisplay,
+				ctx->user_mode.vdisplay);
+			return -ENOMEM;
+		}
+	}
+	dev_info(
+		pr_dev,
+		"get modes user_mode_inited:%d, clock:%d, hdisplay:%u, hsyncStart:%u, hsyncEnd:%u, htotal:%u\n"
+		"vdisplay:%u vsyncStart:%u vsyncEnd:%u vtotal:%u width_mm:%d, height_mm:%d flags:0x%08x \n",
+		ctx->user_mode_inited, mode->clock, mode->hdisplay,
+		mode->hsync_start, mode->hsync_end, mode->htotal,
+		mode->vdisplay, mode->vsync_start, mode->vsync_end,
+		mode->vtotal, mode->width_mm, mode->height_mm, mode->flags);
+
+	drm_mode_set_name(mode);
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	drm_mode_probed_add(connector, mode);
+
+	connector->display_info.width_mm = ctx->user_mode.width_mm;
+	connector->display_info.height_mm = ctx->user_mode.height_mm;
+	drm_display_info_set_bus_formats(&connector->display_info, &bus_format,
+					 1);
+	connector->display_info.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
+
+	return 1;
+}
+
+static const struct drm_panel_funcs es_panel_drm_funcs = {
+	.disable = es_panel_disable,
+	.unprepare = es_panel_unprepare,
+	.prepare = es_panel_prepare,
+	.enable = es_panel_enable,
+	.get_modes = es_panel_get_modes,
+};
+
+int es_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct es_panel *ctx;
+	int ret;
+	struct device_node *dsi_node, *remote_node = NULL, *endpoint = NULL;
+
+	// for print
+	pr_dev = dev;
+
+	dev_info(pr_dev, "[%s] Enter\n", __func__);
+
+	dsi_node = of_get_parent(dev->of_node);
+	if (dsi_node) {
+		endpoint = of_graph_get_next_endpoint(dsi_node, NULL);
+		if (endpoint) {
+			remote_node = of_graph_get_remote_port_parent(endpoint);
+			if (!remote_node) {
+				dev_err(pr_dev,
+					"No panel connected,skip probe es_panel\n");
+				return -ENODEV;
+			}
+			dev_info(pr_dev, "device node name:%s\n",
+				 remote_node->name);
+		}
+	}
+
+	/*
+        if (remote_node != dev->of_node) {
+            dev_info(pr_dev,"%s+ skip probe due to not current es_panel\n", __func__);
+            return -ENODEV;
+        }
+    */
+	ctx = devm_kzalloc(dev, sizeof(struct es_panel), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	mipi_dsi_set_drvdata(dsi, ctx);
+	ctx->dev = dev;
+
+	// default val, if other value, set it on:IOC_ES_MIPI_TX_SET_MODE.
+	dsi->lanes = LANES;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	// dsi->mode_flags = MIPI_DSI_MODE_VIDEO;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_LPM |
+			  MIPI_DSI_MODE_VIDEO_SYNC_PULSE;
+
+	ctx->gpio_backlight0 =
+		devm_gpiod_get(ctx->dev, "backlight0", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpio_backlight0)) {
+		dev_err(ctx->dev, "failed to get backlight0 gpio, err: %ld\n",
+			PTR_ERR(ctx->gpio_backlight0));
+		return PTR_ERR(ctx->gpio_backlight0);
+	}
+
+	ctx->gpio_reset = devm_gpiod_get(ctx->dev, "rst", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpio_reset)) {
+		dev_err(ctx->dev, "failed to get rst gpio, err: %ld\n",
+			PTR_ERR(ctx->gpio_reset));
+		return PTR_ERR(ctx->gpio_reset);
+	}
+	msleep(50);
+
+	gpiod_set_value(ctx->gpio_reset, 1);
+	gpiod_set_value(ctx->gpio_backlight0, 1);
+
+	ctx->prepared = false;
+	ctx->enabled = false;
+	ctx->user_mode_inited = false;
+
+	drm_panel_init(&ctx->panel, dev, &es_panel_drm_funcs,
+		       DRM_MODE_CONNECTOR_DPI);
+	//	ctx->panel.dev = dev;
+	//	ctx->panel.funcs = &es_panel_drm_funcs;
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0)
+		drm_panel_remove(&ctx->panel);
+
+	es_panel_chrdev_create(ctx);
+	INIT_LIST_HEAD(&ctx->init_cmd_list);
+	ctx->init_cmd_writted = 0;
+	memset(&ctx->enable_cmd_buf, 0, sizeof(user_cmd_buffer_t));
+	memset(&ctx->disable_cmd_buf, 0, sizeof(user_cmd_buffer_t));
+	return ret;
+}
+
+void es_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct es_panel *ctx = mipi_dsi_get_drvdata(dsi);
+
+	free_cmd_list(&ctx->init_cmd_list);
+	free_cmd_buf(&ctx->enable_cmd_buf);
+	free_cmd_buf(&ctx->disable_cmd_buf);
+
+	devm_gpiod_put(ctx->dev, ctx->gpio_backlight0);
+	devm_gpiod_put(ctx->dev, ctx->gpio_reset);
+
+	es_panel_chrdev_destroy(ctx);
+	mipi_dsi_detach(dsi);
+	drm_panel_remove(&ctx->panel);
+
+	return;
+}
+
+static const struct of_device_id es_panel_of_match[] = {
+	{
+		.compatible = "eswin,generic-panel",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, es_panel_of_match);
+
+struct mipi_dsi_driver es_panel_driver = {
+    .probe = es_panel_probe,
+    .remove = es_panel_remove,
+    .driver =
+        {
+            .name = "es-panel",
+            .of_match_table = es_panel_of_match,
+        },
+};
+
+MODULE_AUTHOR("Eswin");
+MODULE_DESCRIPTION("Eswin Generic Panel Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_panel.h b/drivers/gpu/drm/eswin/es_panel.h
new file mode 100644
index 000000000000..96ae0171f379
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_panel.h
@@ -0,0 +1,67 @@
+#ifndef _ES_PANEL_H__
+#define _ES_PANEL_H__
+
+#include <drm/drm_mipi_dsi.h>
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+
+enum {
+	IOC_ES_MIPI_TX_SET_MODE = 0x100,
+	IOC_ES_MIPI_TX_SET_INIT_CMD,
+	IOC_ES_MIPI_TX_SET_CMD,
+	IOC_ES_MIPI_TX_ENABLE,
+	IOC_ES_MIPI_TX_DISABLE,
+};
+
+typedef enum MIPI_OUT_FORMAT_E {
+	MIPI_OUT_FORMAT_RGB_16_BIT = 0x0,
+	MIPI_OUT_FORMAT_RGB_18_BIT,
+	MIPI_OUT_FORMAT_RGB_24_BIT,
+	MIPI_OUT_FORMAT_BUTT
+} MIPI_OUT_FORMAT_E;
+
+typedef enum MIPI_OUT_MODE_E {
+	MIPI_OUTPUT_MODE_CSI = 0x0,
+	MIPI_OUTPUT_MODE_DSI_VIDEO,
+	MIPI_OUTPUT_MODE_DSI_CMD,
+	MIPI_OUTPUT_MODE_DSI_BUTT
+} MIPI_OUT_MODE_E;
+
+typedef struct MIPI_MODE_INFO {
+	u32 devId;
+	u16 hdisplay;
+	u16 hsyncStart;
+	u16 hsyncEnd;
+	u16 htotal;
+	u16 hskew;
+	u16 vdisplay;
+	u16 vsyncStart;
+	u16 vsyncEnd;
+	u16 vtotal;
+	u16 vscan;
+
+	u32 vrefresh;
+	u32 flags;
+	u32 type;
+
+	MIPI_OUT_FORMAT_E outputFormat;
+	MIPI_OUT_MODE_E outputMode;
+	u32 lanes;
+} MIPI_MODE_INFO_S;
+
+typedef struct MIPI_CMD_S {
+	int devId;
+	u16 dataType;
+	u16 cmdSize;
+	u32 sleepUs;
+	u8 *pCmd;
+} MIPI_CMD_S;
+
+extern struct mipi_dsi_driver es_panel_driver;
+void es_panel_remove(struct mipi_dsi_device *dsi);
+int es_panel_probe(struct mipi_dsi_device *dsi);
+
+#endif //_ES_PANEL_H__
\ No newline at end of file
diff --git a/drivers/gpu/drm/eswin/es_plane.c b/drivers/gpu/drm/eswin/es_plane.c
new file mode 100644
index 000000000000..68e14d921de5
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_plane.c
@@ -0,0 +1,399 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_blend.h>
+
+#include "es_drm.h"
+#include "es_type.h"
+#include "es_crtc.h"
+#include "es_plane.h"
+#include "es_gem.h"
+#include "es_fb.h"
+
+void es_plane_destory(struct drm_plane *plane)
+{
+	struct es_plane *es_plane = to_es_plane(plane);
+
+	drm_plane_cleanup(plane);
+	kfree(es_plane);
+}
+
+static void es_plane_reset(struct drm_plane *plane)
+{
+	struct es_plane_state *state;
+
+	if (plane->state) {
+		__drm_atomic_helper_plane_destroy_state(plane->state);
+
+		state = to_es_plane_state(plane->state);
+		kfree(state);
+		plane->state = NULL;
+	}
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state == NULL)
+		return;
+
+	__drm_atomic_helper_plane_reset(plane, &state->base);
+
+	state->degamma = ES_DEGAMMA_DISABLE;
+	state->degamma_changed = false;
+	memset(&state->status, 0, sizeof(state->status));
+}
+
+static void _es_plane_duplicate_blob(struct es_plane_state *state,
+				     struct es_plane_state *ori_state)
+{
+	state->roi = ori_state->roi;
+	state->color_mgmt = ori_state->color_mgmt;
+	if (state->roi)
+		drm_property_blob_get(state->roi);
+	if (state->color_mgmt)
+		drm_property_blob_get(state->color_mgmt);
+}
+
+static struct drm_plane_state *
+es_plane_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct es_plane_state *ori_state;
+	struct es_plane_state *state;
+
+	if (WARN_ON(!plane->state))
+		return NULL;
+
+	ori_state = to_es_plane_state(plane->state);
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &state->base);
+
+	state->degamma = ori_state->degamma;
+	state->degamma_changed = ori_state->degamma_changed;
+
+	_es_plane_duplicate_blob(state, ori_state);
+	memcpy(&state->status, &ori_state->status, sizeof(ori_state->status));
+
+	return &state->base;
+}
+
+static void es_plane_atomic_destroy_state(struct drm_plane *plane,
+					  struct drm_plane_state *state)
+{
+	struct es_plane_state *es_plane_state = to_es_plane_state(state);
+
+	__drm_atomic_helper_plane_destroy_state(state);
+	drm_property_blob_put(es_plane_state->roi);
+	drm_property_blob_put(es_plane_state->color_mgmt);
+	kfree(es_plane_state);
+}
+
+static int _es_plane_set_property_blob_from_id(struct drm_device *dev,
+					       struct drm_property_blob **blob,
+					       u64 blob_id,
+					       size_t expected_size)
+{
+	struct drm_property_blob *new_blob = NULL;
+
+	if (blob_id) {
+		new_blob = drm_property_lookup_blob(dev, blob_id);
+		if (!new_blob) {
+			return -EINVAL;
+		}
+
+		if (new_blob->length != expected_size) {
+			drm_property_blob_put(new_blob);
+			return -EINVAL;
+		}
+	}
+	drm_property_replace_blob(blob, new_blob);
+	drm_property_blob_put(new_blob);
+	return 0;
+}
+
+static int es_plane_atomic_set_property(struct drm_plane *plane,
+					struct drm_plane_state *state,
+					struct drm_property *property,
+					uint64_t val)
+{
+	struct drm_device *dev = plane->dev;
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct es_plane_state *es_plane_state = to_es_plane_state(state);
+	int ret = 0;
+
+	if (property == es_plane->degamma_mode) {
+		if (es_plane_state->degamma != val) {
+			es_plane_state->degamma = val;
+			es_plane_state->degamma_changed = true;
+		} else {
+			es_plane_state->degamma_changed = false;
+		}
+	} else if (property == es_plane->roi_prop) {
+		ret = _es_plane_set_property_blob_from_id(
+			dev, &es_plane_state->roi, val,
+			sizeof(struct drm_es_roi));
+		return ret;
+	} else if (property == es_plane->color_mgmt_prop) {
+		ret = _es_plane_set_property_blob_from_id(
+			dev, &es_plane_state->color_mgmt, val,
+			sizeof(struct drm_es_color_mgmt));
+		return ret;
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int es_plane_atomic_get_property(struct drm_plane *plane,
+					const struct drm_plane_state *state,
+					struct drm_property *property,
+					uint64_t *val)
+{
+	struct es_plane *es_plane = to_es_plane(plane);
+	const struct es_plane_state *es_plane_state =
+		container_of(state, const struct es_plane_state, base);
+
+	if (property == es_plane->degamma_mode)
+		*val = es_plane_state->degamma;
+	else if (property == es_plane->roi_prop)
+		*val = (es_plane_state->roi) ? es_plane_state->roi->base.id : 0;
+	else if (property == es_plane->color_mgmt_prop)
+		*val = (es_plane_state->color_mgmt) ?
+			       es_plane_state->color_mgmt->base.id :
+			       0;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+static bool es_format_mod_supported(struct drm_plane *plane, u32 format,
+				    u64 modifier)
+{
+	int i;
+
+	/* We always have to allow these modifiers:
+     * 1. Core DRM checks for LINEAR support if userspace does not provide
+ modifiers.
+     * 2. Not passing any modifiers is the same as explicitly passing
+ INVALID.
+     */
+	if (modifier == DRM_FORMAT_MOD_LINEAR)
+		return true;
+
+	/* Check that the modifier is on the list of the plane's supported
+ modifiers. */
+	for (i = 0; i < plane->modifier_count; i++) {
+		if (modifier == plane->modifiers[i])
+			break;
+	}
+
+	if (i == plane->modifier_count)
+		return false;
+
+	return true;
+}
+
+const struct drm_plane_funcs es_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = es_plane_destory,
+	.reset = es_plane_reset,
+	.atomic_duplicate_state = es_plane_atomic_duplicate_state,
+	.atomic_destroy_state = es_plane_atomic_destroy_state,
+	.atomic_set_property = es_plane_atomic_set_property,
+	.atomic_get_property = es_plane_atomic_get_property,
+	.format_mod_supported = es_format_mod_supported,
+};
+
+static unsigned char es_get_plane_number(struct drm_framebuffer *fb)
+{
+	const struct drm_format_info *info;
+
+	if (!fb)
+		return 0;
+
+	info = drm_format_info(fb->format->format);
+	if (!info || info->num_planes > MAX_NUM_PLANES)
+		return 0;
+
+	return info->num_planes;
+}
+
+static int es_plane_atomic_check(struct drm_plane *plane,
+				 struct drm_atomic_state *state)
+{
+	struct drm_plane_state *new_plane_state =
+		drm_atomic_get_new_plane_state(state, plane);
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct drm_framebuffer *fb = new_plane_state->fb;
+	struct drm_crtc *crtc = new_plane_state->crtc;
+	struct es_crtc *es_crtc = to_es_crtc(crtc);
+
+	if (!crtc || !fb)
+		return 0;
+
+	return es_plane->funcs->check(es_crtc->dev, es_plane, new_plane_state);
+}
+
+static void es_plane_atomic_update(struct drm_plane *plane,
+				   struct drm_atomic_state *old_state)
+{
+	unsigned char i, num_planes;
+	struct drm_framebuffer *fb;
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct drm_plane_state *state = plane->state;
+	struct es_crtc *es_crtc = to_es_crtc(state->crtc);
+	struct es_plane_state *plane_state = to_es_plane_state(state);
+	// struct drm_format_name_buf *name = &plane_state->status.format_name;
+
+	if (!state->fb || !state->crtc)
+		return;
+
+	fb = state->fb;
+
+	num_planes = es_get_plane_number(fb);
+
+	for (i = 0; i < num_planes; i++) {
+		struct es_gem_object *es_obj;
+
+		es_obj = es_fb_get_gem_obj(fb, i);
+		es_plane->dma_addr[i] = es_obj->iova + fb->offsets[i];
+	}
+
+	plane_state->status.tile_mode = 0; /* to be updated */
+	plane_state->status.src = drm_plane_state_src(state);
+	plane_state->status.dest = drm_plane_state_dest(state);
+	// drm_get_format_name(fb->format->format, name);
+
+	es_plane->funcs->update(es_crtc->dev, es_plane);
+}
+
+static void es_plane_atomic_disable(struct drm_plane *plane,
+				    struct drm_atomic_state *state)
+{
+	struct drm_plane_state *old_plane_state =
+		drm_atomic_get_old_plane_state(state, plane);
+	struct es_plane *es_plane = to_es_plane(plane);
+	struct es_crtc *es_crtc = to_es_crtc(old_plane_state->crtc);
+
+	if (!old_plane_state->crtc)
+		return;
+
+	if (es_plane->funcs && es_plane->funcs->disable)
+		es_plane->funcs->disable(es_crtc->dev, es_plane);
+}
+
+const struct drm_plane_helper_funcs es_plane_helper_funcs = {
+	.atomic_check = es_plane_atomic_check,
+	.atomic_update = es_plane_atomic_update,
+	.atomic_disable = es_plane_atomic_disable,
+};
+
+static const struct drm_prop_enum_list es_degamma_mode_enum_list[] = {
+	{ ES_DEGAMMA_DISABLE, "disabled" },
+	{ ES_DEGAMMA_BT709, "preset degamma for BT709" },
+	{ ES_DEGAMMA_BT2020, "preset degamma for BT2020" },
+};
+
+struct es_plane *es_plane_create(struct drm_device *drm_dev,
+				 struct es_plane_info *info,
+				 unsigned int possible_crtcs)
+{
+	struct es_plane *plane;
+	int ret;
+
+	if (!info)
+		return NULL;
+
+	plane = kzalloc(sizeof(struct es_plane), GFP_KERNEL);
+	if (!plane)
+		return NULL;
+
+	plane->id = info->id;
+
+	ret = drm_universal_plane_init(drm_dev, &plane->base, possible_crtcs,
+				       &es_plane_funcs, info->formats,
+				       info->num_formats, info->modifiers,
+				       info->type,
+				       info->name ? info->name : NULL);
+	if (ret)
+		goto err_free_plane;
+
+	drm_plane_helper_add(&plane->base, &es_plane_helper_funcs);
+
+	/* Set up the plane properties */
+	if (info->degamma_size) {
+		plane->degamma_mode = drm_property_create_enum(
+			drm_dev, 0, "DEGAMMA_MODE", es_degamma_mode_enum_list,
+			ARRAY_SIZE(es_degamma_mode_enum_list));
+
+		if (!plane->degamma_mode)
+			goto error_cleanup_plane;
+
+		drm_object_attach_property(&plane->base.base,
+					   plane->degamma_mode,
+					   ES_DEGAMMA_DISABLE);
+	}
+
+	if (info->rotation) {
+		ret = drm_plane_create_rotation_property(
+			&plane->base, DRM_MODE_ROTATE_0, info->rotation);
+		if (ret)
+			goto error_cleanup_plane;
+	}
+
+	if (info->blend_mode) {
+		ret = drm_plane_create_blend_mode_property(&plane->base,
+							   info->blend_mode);
+		if (ret)
+			goto error_cleanup_plane;
+		ret = drm_plane_create_alpha_property(&plane->base);
+		if (ret)
+			goto error_cleanup_plane;
+	}
+
+	if (info->color_encoding) {
+		ret = drm_plane_create_color_properties(
+			&plane->base, info->color_encoding,
+			BIT(DRM_COLOR_YCBCR_LIMITED_RANGE),
+			DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE);
+		if (ret)
+			goto error_cleanup_plane;
+	}
+
+	if (info->roi) {
+		plane->roi_prop = drm_property_create(
+			drm_dev, DRM_MODE_PROP_BLOB, "ROI", 0);
+		if (!plane->roi_prop)
+			goto error_cleanup_plane;
+
+		drm_object_attach_property(&plane->base.base, plane->roi_prop,
+					   0);
+	}
+
+	if (info->color_mgmt) {
+		plane->color_mgmt_prop = drm_property_create(
+			drm_dev, DRM_MODE_PROP_BLOB, "COLOR_CONFIG", 0);
+		if (!plane->color_mgmt_prop)
+			return NULL;
+
+		drm_object_attach_property(&plane->base.base,
+					   plane->color_mgmt_prop, 0);
+	}
+
+	return plane;
+
+error_cleanup_plane:
+	drm_plane_cleanup(&plane->base);
+err_free_plane:
+	kfree(plane);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/eswin/es_plane.h b/drivers/gpu/drm/eswin/es_plane.h
new file mode 100644
index 000000000000..cc41d0741361
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_plane.h
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_PLANE_H__
+#define __ES_PLANE_H__
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_plane_helper.h>
+
+#include "es_type.h"
+#include "es_fb.h"
+
+#define MAX_NUM_PLANES 3 /* colour format plane */
+
+struct es_plane;
+
+struct es_plane_funcs {
+	void (*update)(struct device *dev, struct es_plane *plane);
+	void (*disable)(struct device *dev, struct es_plane *plane);
+	int (*check)(struct device *dev, struct es_plane *plane,
+		     struct drm_plane_state *state);
+};
+
+struct drm_es_roi {
+	__u16 enable;
+	__u16 roi_x;
+	__u16 roi_y;
+	__u16 roi_w;
+	__u16 roi_h;
+};
+
+struct drm_es_color_mgmt {
+	__u32 colorkey;
+	__u32 colorkey_high;
+	__u32 clear_value;
+	__u16 clear_enable;
+	__u16 transparency;
+};
+
+struct es_plane_status {
+	u32 tile_mode;
+	struct drm_rect src;
+	struct drm_rect dest;
+	// struct drm_format_name_buf format_name;
+};
+
+struct es_plane_state {
+	struct drm_plane_state base;
+	struct es_plane_status status; /* for debugfs */
+	struct drm_property_blob *roi;
+	struct drm_property_blob *color_mgmt;
+
+	u32 degamma;
+	bool degamma_changed;
+};
+
+struct es_plane {
+	struct drm_plane base;
+	u8 id;
+	dma_addr_t dma_addr[MAX_NUM_PLANES];
+
+	struct drm_property *degamma_mode;
+	struct drm_property *roi_prop;
+	struct drm_property *color_mgmt_prop;
+
+	const struct es_plane_funcs *funcs;
+};
+
+void es_plane_destory(struct drm_plane *plane);
+
+struct es_plane *es_plane_create(struct drm_device *drm_dev,
+				 struct es_plane_info *info,
+				 unsigned int possible_crtcs);
+
+static inline struct es_plane *to_es_plane(struct drm_plane *plane)
+{
+	return container_of(plane, struct es_plane, base);
+}
+
+static inline struct es_plane_state *
+to_es_plane_state(struct drm_plane_state *state)
+{
+	return container_of(state, struct es_plane_state, base);
+}
+#endif /* __ES_PLANE_H__ */
diff --git a/drivers/gpu/drm/eswin/es_simple_enc.c b/drivers/gpu/drm/eswin/es_simple_enc.c
new file mode 100644
index 000000000000..fbc725fc64ea
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_simple_enc.c
@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/component.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <linux/media-bus-format.h>
+
+#include <drm/drm_bridge.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_blend.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_modeset_helper_vtables.h>
+
+#include "es_crtc.h"
+#include "es_simple_enc.h"
+
+static const struct simple_encoder_priv hdmi_priv = {
+	.encoder_type = DRM_MODE_ENCODER_TMDS
+};
+
+static const struct simple_encoder_priv dsi_priv = {
+	.encoder_type = DRM_MODE_ENCODER_DSI
+};
+
+static const struct drm_encoder_funcs encoder_funcs = {
+	.destroy = drm_encoder_cleanup
+};
+
+static inline struct simple_encoder *to_simple_encoder(struct drm_encoder *enc)
+{
+	return container_of(enc, struct simple_encoder, encoder);
+}
+
+static int encoder_parse_dt(struct device *dev)
+{
+	struct simple_encoder *simple = dev_get_drvdata(dev);
+	int ret = 0;
+	int cnt, i;
+	u32 *vals;
+	u32 *masks;
+
+	simple->dss_regmap = syscon_regmap_lookup_by_phandle(
+		dev->of_node, "eswin,dss-syscon");
+
+	if (IS_ERR(simple->dss_regmap)) {
+		if (PTR_ERR(simple->dss_regmap) != -ENODEV) {
+			dev_err(dev, "failed to get dss-syscon\n");
+			ret = PTR_ERR(simple->dss_regmap);
+			goto err;
+		}
+		simple->dss_regmap = NULL;
+		goto err;
+	}
+
+	cnt = of_property_count_elems_of_size(dev->of_node, "eswin,mux-mask",
+					      4);
+	if (!cnt) {
+		ret = cnt;
+		goto err;
+	}
+
+	simple->dss_regdatas = devm_kzalloc(
+		dev, sizeof(*simple->dss_regdatas) * cnt, GFP_KERNEL);
+
+	masks = kcalloc(cnt, sizeof(*masks), GFP_KERNEL);
+	if (!masks) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	vals = kcalloc(cnt, sizeof(*vals), GFP_KERNEL);
+	if (!vals) {
+		ret = -ENOMEM;
+		goto err_free_masks;
+	}
+
+	ret = of_property_read_u32_array(dev->of_node, "eswin,mux-mask", masks,
+					 cnt);
+	if (ret)
+		goto err_free_vals;
+
+	ret = of_property_read_u32_array(dev->of_node, "eswin,mux-val", vals,
+					 cnt);
+	if (ret)
+		goto err_free_vals;
+
+	for (i = 0; i < cnt; i++) {
+		simple->dss_regdatas[i].mask = masks[i];
+		simple->dss_regdatas[i].value = vals[i];
+	}
+
+err_free_vals:
+	kfree(vals);
+err_free_masks:
+	kfree(masks);
+err:
+	return ret;
+}
+
+void encoder_atomic_enable(struct drm_encoder *encoder,
+			   struct drm_atomic_state *state)
+{
+	struct simple_encoder *simple = to_simple_encoder(encoder);
+	struct dss_data *data = simple->dss_regdatas;
+	int crtc_id;
+
+	if (!simple->dss_regmap)
+		return;
+
+	crtc_id = drm_of_encoder_active_endpoint_id(simple->dev->of_node,
+						    encoder);
+
+	regmap_update_bits(simple->dss_regmap, 0, data[crtc_id].mask,
+			   data[crtc_id].value);
+}
+
+int encoder_atomic_check(struct drm_encoder *encoder,
+			 struct drm_crtc_state *crtc_state,
+			 struct drm_connector_state *conn_state)
+{
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc_state);
+	struct drm_connector *connector = conn_state->connector;
+
+	es_crtc_state->encoder_type = encoder->encoder_type;
+
+	if (connector->display_info.num_bus_formats)
+		es_crtc_state->output_fmt =
+			connector->display_info.bus_formats[0];
+	else
+		es_crtc_state->output_fmt = MEDIA_BUS_FMT_RGB888_1X24;
+
+	switch (es_crtc_state->output_fmt) {
+	case MEDIA_BUS_FMT_RGB565_1X16:
+	case MEDIA_BUS_FMT_RGB666_1X18:
+	case MEDIA_BUS_FMT_RGB666_1X24_CPADHI:
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		break;
+	default:
+		es_crtc_state->output_fmt = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	}
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
+	.atomic_enable = encoder_atomic_enable,
+	.atomic_check = encoder_atomic_check,
+};
+
+static int encoder_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+	struct simple_encoder *simple = dev_get_drvdata(dev);
+	struct drm_encoder *encoder;
+	struct drm_bridge *bridge;
+	int ret;
+
+	encoder = &simple->encoder;
+
+	/* Encoder. */
+	ret = drm_encoder_init(drm_dev, encoder, &encoder_funcs,
+			       simple->priv->encoder_type, NULL);
+	if (ret)
+		return ret;
+
+	drm_encoder_helper_add(encoder, &encoder_helper_funcs);
+
+	encoder->possible_crtcs =
+		drm_of_find_possible_crtcs(drm_dev, dev->of_node);
+
+	/* output port is port1*/
+	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, -1, NULL, &bridge);
+	if (ret)
+		goto err;
+
+	ret = drm_bridge_attach(encoder, bridge, NULL, 0);
+	if (ret)
+		goto err;
+
+	return 0;
+err:
+	drm_encoder_cleanup(encoder);
+
+	return ret;
+}
+
+static void encoder_unbind(struct device *dev, struct device *master,
+			   void *data)
+{
+	struct simple_encoder *simple = dev_get_drvdata(dev);
+
+	drm_encoder_cleanup(&simple->encoder);
+}
+
+static const struct component_ops encoder_component_ops = {
+	.bind = encoder_bind,
+	.unbind = encoder_unbind,
+};
+
+static const struct of_device_id simple_encoder_dt_match[] = {
+	{ .compatible = "eswin,hdmi-encoder", .data = &hdmi_priv },
+	{ .compatible = "eswin,dp-encoder", .data = &hdmi_priv },
+	{ .compatible = "eswin,dsi-encoder", .data = &dsi_priv },
+	{},
+};
+MODULE_DEVICE_TABLE(of, simple_encoder_dt_match);
+
+static int encoder_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct simple_encoder *simple;
+	int ret;
+
+	simple = devm_kzalloc(dev, sizeof(*simple), GFP_KERNEL);
+	if (!simple)
+		return -ENOMEM;
+
+	simple->priv = of_device_get_match_data(dev);
+
+	simple->dev = dev;
+
+	dev_set_drvdata(dev, simple);
+
+	ret = encoder_parse_dt(dev);
+	if (ret)
+		return ret;
+
+	return component_add(dev, &encoder_component_ops);
+}
+
+static int encoder_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	component_del(dev, &encoder_component_ops);
+
+	dev_set_drvdata(dev, NULL);
+
+	return 0;
+}
+
+struct platform_driver simple_encoder_driver = {
+    .probe = encoder_probe,
+    .remove = encoder_remove,
+    .driver = {
+        .name = "es-simple-encoder",
+        .of_match_table = of_match_ptr(simple_encoder_dt_match),
+    },
+};
+
+MODULE_DESCRIPTION("Simple Encoder Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_simple_enc.h b/drivers/gpu/drm/eswin/es_simple_enc.h
new file mode 100644
index 000000000000..bc461e1231e3
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_simple_enc.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_SIMPLE_ENC_H_
+#define __ES_SIMPLE_ENC_H_
+
+struct simple_encoder_priv {
+	unsigned char encoder_type;
+};
+
+struct dss_data {
+	u32 mask;
+	u32 value;
+};
+
+struct simple_encoder {
+	struct drm_encoder encoder;
+	struct device *dev;
+	const struct simple_encoder_priv *priv;
+	struct regmap *dss_regmap;
+	struct dss_data *dss_regdatas;
+};
+
+extern struct platform_driver simple_encoder_driver;
+#endif /* __ES_SIMPLE_ENC_H_ */
diff --git a/drivers/gpu/drm/eswin/es_type.h b/drivers/gpu/drm/eswin/es_type.h
new file mode 100644
index 000000000000..aca0a83b0017
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_type.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_TYPE_H__
+#define __ES_TYPE_H__
+
+#include <drm/drm_plane.h>
+#include <drm/drm_plane_helper.h>
+
+struct es_plane_info {
+	const char *name;
+	u8 id;
+	enum drm_plane_type type;
+	unsigned int num_formats;
+	const u32 *formats;
+	const u64 *modifiers;
+	unsigned int min_width;
+	unsigned int min_height;
+	unsigned int max_width;
+	unsigned int max_height;
+	unsigned int rotation;
+	unsigned int blend_mode;
+	unsigned int color_encoding;
+
+	/* 0 means no de-gamma LUT */
+	unsigned int degamma_size;
+
+	int min_scale; /* 16.16 fixed point */
+	int max_scale; /* 16.16 fixed point */
+	bool roi;
+	bool color_mgmt;
+	bool background;
+};
+
+struct es_dc_info {
+	const char *name;
+
+	/* planes */
+	unsigned char plane_num;
+	const struct es_plane_info *planes;
+
+	unsigned int max_bpc;
+	unsigned int color_formats;
+
+	/* 0 means no gamma LUT */
+	u16 gamma_size;
+	u8 gamma_bits;
+
+	u16 pitch_alignment;
+
+	bool pipe_sync;
+	bool mmu_prefetch;
+	bool background;
+};
+
+#endif /* __ES_TYPE_H__ */
diff --git a/drivers/gpu/drm/eswin/es_virtual.c b/drivers/gpu/drm/eswin/es_virtual.c
new file mode 100644
index 000000000000..1d35ada1c0c0
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_virtual.c
@@ -0,0 +1,1310 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#include <linux/component.h>
+#include <linux/of_platform.h>
+#include <linux/media-bus-format.h>
+#include <linux/debugfs.h>
+
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_atomic_helper.h>
+
+#include "es_virtual.h"
+#include "es_dc.h"
+#include "es_gem.h"
+
+static unsigned char __get_bpp(struct es_virtual_display *vd)
+{
+	if (vd->bus_format == MEDIA_BUS_FMT_RGB101010_1X30)
+		return 10;
+	return 8;
+}
+
+static void vd_dump_destroy(struct es_virtual_display *vd)
+{
+	struct drm_device *drm_dev = vd->encoder.dev;
+
+	if (vd->dump_blob.data) {
+		vunmap(vd->dump_blob.data);
+		vd->dump_blob.data = NULL;
+	}
+	vd->dump_blob.size = 0;
+
+	debugfs_remove(vd->dump_debugfs);
+	vd->dump_debugfs = NULL;
+
+	if (vd->dump_obj) {
+		mutex_lock(&drm_dev->struct_mutex);
+		drm_gem_object_put(&vd->dump_obj->base);
+		mutex_unlock(&drm_dev->struct_mutex);
+		vd->dump_obj = NULL;
+	}
+}
+
+static void vd_dump_create(struct es_virtual_display *vd,
+			   struct drm_display_mode *mode)
+{
+	struct drm_device *drm_dev = vd->encoder.dev;
+	struct es_dc *dc = dev_get_drvdata(vd->dc);
+	struct es_gem_object *obj;
+	unsigned int pitch, size;
+	void *kvaddr;
+	char *name;
+
+	if (!dc->funcs)
+		return;
+
+	vd_dump_destroy(vd);
+
+	/* dump in 4bytes XRGB format */
+	pitch = mode->hdisplay * 4;
+	pitch = ALIGN(pitch, dc->hw.info->pitch_alignment);
+	size = PAGE_ALIGN(pitch * mode->vdisplay);
+
+	obj = es_gem_create_object(drm_dev, size);
+	if (IS_ERR(obj)) {
+		return;
+	}
+
+	vd->dump_obj = obj;
+	vd->pitch = pitch;
+
+	kvaddr = vmap(obj->pages, obj->size >> PAGE_SHIFT, VM_MAP,
+		      pgprot_writecombine(PAGE_KERNEL));
+	if (!kvaddr)
+		goto err;
+
+	vd->dump_blob.data = kvaddr;
+	vd->dump_blob.size = obj->size;
+
+	name = kasprintf(GFP_KERNEL, "%dx%d-XRGB-%d.raw", mode->hdisplay,
+			 mode->vdisplay, __get_bpp(vd));
+	if (!name)
+		goto err;
+
+	vd->dump_debugfs = debugfs_create_blob(
+		name, 0444, vd->connector.debugfs_entry, &vd->dump_blob);
+	kfree(name);
+
+	return;
+
+err:
+	vd_dump_destroy(vd);
+}
+
+static void vd_encoder_destroy(struct drm_encoder *encoder)
+{
+	struct es_virtual_display *vd;
+
+	drm_encoder_cleanup(encoder);
+	vd = to_virtual_display_with_encoder(encoder);
+	vd_dump_destroy(vd);
+}
+
+static const struct drm_encoder_funcs vd_encoder_funcs = {
+	.destroy = vd_encoder_destroy
+};
+
+static void vd_mode_set(struct drm_encoder *encoder,
+			struct drm_display_mode *mode,
+			struct drm_display_mode *adjusted_mode)
+{
+	struct es_virtual_display *vd;
+
+	vd = to_virtual_display_with_encoder(encoder);
+	vd_dump_create(vd, adjusted_mode);
+}
+
+static void vd_encoder_disable(struct drm_encoder *encoder)
+{
+	struct es_virtual_display *vd;
+	struct es_dc *dc;
+
+	vd = to_virtual_display_with_encoder(encoder);
+	dc = dev_get_drvdata(vd->dc);
+	if (dc->funcs && dc->funcs->dump_disable)
+		dc->funcs->dump_disable(vd->dc);
+}
+
+static void vd_encoder_enable(struct drm_encoder *encoder)
+{
+	struct es_virtual_display *vd;
+	struct es_dc *dc;
+
+	vd = to_virtual_display_with_encoder(encoder);
+	dc = dev_get_drvdata(vd->dc);
+	if (dc->funcs && dc->funcs->dump_enable && vd->dump_obj)
+		dc->funcs->dump_enable(vd->dc, vd->dump_obj->iova, vd->pitch);
+}
+
+int vd_encoder_atomic_check(struct drm_encoder *encoder,
+			    struct drm_crtc_state *crtc_state,
+			    struct drm_connector_state *conn_state)
+{
+	struct es_crtc_state *es_crtc_state = to_es_crtc_state(crtc_state);
+
+	es_crtc_state->encoder_type = encoder->encoder_type;
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs vd_encoder_helper_funcs = {
+	.mode_set = vd_mode_set,
+	.enable = vd_encoder_enable,
+	.disable = vd_encoder_disable,
+	.atomic_check = vd_encoder_atomic_check,
+};
+
+static const struct drm_display_mode edid_cea_modes_1[] = {
+	/* 1 - 640x480@60Hz 4:3 */
+	{
+		DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656, 752,
+			 800, 0, 480, 490, 492, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 2 - 720x480@60Hz 4:3 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 3 - 720x480@60Hz 16:9 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 4 - 1280x720@60Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 5 - 1920x1080i@60Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 6 - 720(1440)x480i@60Hz 4:3 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 7 - 720(1440)x480i@60Hz 16:9 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 8 - 720(1440)x240@60Hz 4:3 */
+	{
+		DRM_MODE("720x240", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 9 - 720(1440)x240@60Hz 16:9 */
+	{
+		DRM_MODE("720x240", DRM_MODE_TYPE_DRIVER, 13500, 720, 739, 801,
+			 858, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 10 - 2880x480i@60Hz 4:3 */
+	{
+		DRM_MODE("2880x480i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 11 - 2880x480i@60Hz 16:9 */
+	{
+		DRM_MODE("2880x480i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 12 - 2880x240@60Hz 4:3 */
+	{
+		DRM_MODE("2880x240", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 13 - 2880x240@60Hz 16:9 */
+	{
+		DRM_MODE("2880x240", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+			 3204, 3432, 0, 240, 244, 247, 262, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 14 - 1440x480@60Hz 4:3 */
+	{
+		DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1472,
+			 1596, 1716, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 15 - 1440x480@60Hz 16:9 */
+	{
+		DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1472,
+			 1596, 1716, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 16 - 1920x1080@60Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 17 - 720x576@50Hz 4:3 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 18 - 720x576@50Hz 16:9 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 19 - 1280x720@50Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 20 - 1920x1080i@50Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 21 - 720(1440)x576i@50Hz 4:3 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 22 - 720(1440)x576i@50Hz 16:9 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 23 - 720(1440)x288@50Hz 4:3 */
+	{
+		DRM_MODE("720x288", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 24 - 720(1440)x288@50Hz 16:9 */
+	{
+		DRM_MODE("720x288", DRM_MODE_TYPE_DRIVER, 13500, 720, 732, 795,
+			 864, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 25 - 2880x576i@50Hz 4:3 */
+	{
+		DRM_MODE("2880x576i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 26 - 2880x576i@50Hz 16:9 */
+	{
+		DRM_MODE("2880x576i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 27 - 2880x288@50Hz 4:3 */
+	{
+		DRM_MODE("2880x288", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 28 - 2880x288@50Hz 16:9 */
+	{
+		DRM_MODE("2880x288", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+			 3180, 3456, 0, 288, 290, 293, 312, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 29 - 1440x576@50Hz 4:3 */
+	{
+		DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+			 1592, 1728, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 30 - 1440x576@50Hz 16:9 */
+	{
+		DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+			 1592, 1728, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 31 - 1920x1080@50Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 32 - 1920x1080@24Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 33 - 1920x1080@25Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 34 - 1920x1080@30Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 35 - 2880x480@60Hz 4:3 */
+	{
+		DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2944,
+			 3192, 3432, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 36 - 2880x480@60Hz 16:9 */
+	{
+		DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2944,
+			 3192, 3432, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 37 - 2880x576@50Hz 4:3 */
+	{
+		DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2928,
+			 3184, 3456, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 38 - 2880x576@50Hz 16:9 */
+	{
+		DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2928,
+			 3184, 3456, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 39 - 1920x1080i@50Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 72000, 1920, 1952,
+			 2120, 2304, 0, 1080, 1126, 1136, 1250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 40 - 1920x1080i@100Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 41 - 1280x720@100Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 42 - 720x576@100Hz 4:3 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 43 - 720x576@100Hz 16:9 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 44 - 720(1440)x576i@100Hz 4:3 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 45 - 720(1440)x576i@100Hz 16:9 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 46 - 1920x1080i@120Hz 16:9 */
+	{
+		DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+				 DRM_MODE_FLAG_INTERLACE),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 47 - 1280x720@120Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 48 - 720x480@120Hz 4:3 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 54000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 49 - 720x480@120Hz 16:9 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 54000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 50 - 720(1440)x480i@120Hz 4:3 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 27000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 51 - 720(1440)x480i@120Hz 16:9 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 27000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 52 - 720x576@200Hz 4:3 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 108000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 53 - 720x576@200Hz 16:9 */
+	{
+		DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 108000, 720, 732, 796,
+			 864, 0, 576, 581, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 54 - 720(1440)x576i@200Hz 4:3 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 55 - 720(1440)x576i@200Hz 16:9 */
+	{
+		DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 54000, 720, 732, 795,
+			 864, 0, 576, 580, 586, 625, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 56 - 720x480@240Hz 4:3 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 108000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 57 - 720x480@240Hz 16:9 */
+	{
+		DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 108000, 720, 736, 798,
+			 858, 0, 480, 489, 495, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 58 - 720(1440)x480i@240Hz 4:3 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 54000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+	},
+	/* 59 - 720(1440)x480i@240Hz 16:9 */
+	{
+		DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 54000, 720, 739, 801,
+			 858, 0, 480, 488, 494, 525, 0,
+			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+				 DRM_MODE_FLAG_INTERLACE |
+				 DRM_MODE_FLAG_DBLCLK),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 60 - 1280x720@24Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 59400, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 61 - 1280x720@25Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3700,
+			 3740, 3960, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 62 - 1280x720@30Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 63 - 1920x1080@120Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 64 - 1920x1080@100Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 65 - 1280x720@24Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 59400, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 66 - 1280x720@25Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3700,
+			 3740, 3960, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 67 - 1280x720@30Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 68 - 1280x720@50Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 69 - 1280x720@60Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 70 - 1280x720@100Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1720,
+			 1760, 1980, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 71 - 1280x720@120Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1390,
+			 1430, 1650, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 72 - 1920x1080@24Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 73 - 1920x1080@25Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 74 - 1920x1080@30Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 75 - 1920x1080@50Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 76 - 1920x1080@60Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 77 - 1920x1080@100Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448,
+			 2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 78 - 1920x1080@120Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2008,
+			 2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 79 - 1680x720@24Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 3040,
+			 3080, 3300, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 80 - 1680x720@25Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2908,
+			 2948, 3168, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 81 - 1680x720@30Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2380,
+			 2420, 2640, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 82 - 1680x720@50Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 82500, 1680, 1940,
+			 1980, 2200, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 83 - 1680x720@60Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 99000, 1680, 1940,
+			 1980, 2200, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 84 - 1680x720@100Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 165000, 1680, 1740,
+			 1780, 2000, 0, 720, 725, 730, 825, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 85 - 1680x720@120Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 198000, 1680, 1740,
+			 1780, 2000, 0, 720, 725, 730, 825, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 86 - 2560x1080@24Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 99000, 2560, 3558,
+			 3602, 3750, 0, 1080, 1084, 1089, 1100, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 87 - 2560x1080@25Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 90000, 2560, 3008,
+			 3052, 3200, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 88 - 2560x1080@30Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 118800, 2560, 3328,
+			 3372, 3520, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 89 - 2560x1080@50Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 185625, 2560, 3108,
+			 3152, 3300, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 90 - 2560x1080@60Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 198000, 2560, 2808,
+			 2852, 3000, 0, 1080, 1084, 1089, 1100, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 91 - 2560x1080@100Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 371250, 2560, 2778,
+			 2822, 2970, 0, 1080, 1084, 1089, 1250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 92 - 2560x1080@120Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 495000, 2560, 3108,
+			 3152, 3300, 0, 1080, 1084, 1089, 1250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 93 - 3840x2160@24Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 94 - 3840x2160@25Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 95 - 3840x2160@30Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 96 - 3840x2160@50Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 97 - 3840x2160@60Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 98 - 4096x2160@24Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 99 - 4096x2160@25Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5064,
+			 5152, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 100 - 4096x2160@30Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 4184,
+			 4272, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 101 - 4096x2160@50Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 5064,
+			 5152, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 102 - 4096x2160@60Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 4184,
+			 4272, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 103 - 3840x2160@24Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 104 - 3840x2160@25Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 105 - 3840x2160@30Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 106 - 3840x2160@50Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 107 - 3840x2160@60Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 108 - 1280x720@48Hz 16:9 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 90000, 1280, 2240,
+			 2280, 2500, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 109 - 1280x720@48Hz 64:27 */
+	{
+		DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 90000, 1280, 2240,
+			 2280, 2500, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 110 - 1680x720@48Hz 64:27 */
+	{
+		DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 99000, 1680, 2490,
+			 2530, 2750, 0, 720, 725, 730, 750, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 111 - 1920x1080@48Hz 16:9 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 112 - 1920x1080@48Hz 64:27 */
+	{
+		DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2558,
+			 2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 113 - 2560x1080@48Hz 64:27 */
+	{
+		DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 198000, 2560, 3558,
+			 3602, 3750, 0, 1080, 1084, 1089, 1100, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 114 - 3840x2160@48Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 115 - 4096x2160@48Hz 256:135 */
+	{
+		DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135,
+	},
+	/* 116 - 3840x2160@48Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 5116,
+			 5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 117 - 3840x2160@100Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 118 - 3840x2160@120Hz 16:9 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9,
+	},
+	/* 119 - 3840x2160@100Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4896,
+			 4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 120 - 3840x2160@120Hz 64:27 */
+	{
+		DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4016,
+			 4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 121 - 5120x2160@24Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 396000, 5120, 7116,
+			 7204, 7500, 0, 2160, 2168, 2178, 2200, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 122 - 5120x2160@25Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 396000, 5120, 6816,
+			 6904, 7200, 0, 2160, 2168, 2178, 2200, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 123 - 5120x2160@30Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 396000, 5120, 5784,
+			 5872, 6000, 0, 2160, 2168, 2178, 2200, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 124 - 5120x2160@48Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 742500, 5120, 5866,
+			 5954, 6250, 0, 2160, 2168, 2178, 2475, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 125 - 5120x2160@50Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 742500, 5120, 6216,
+			 6304, 6600, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 126 - 5120x2160@60Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 742500, 5120, 5284,
+			 5372, 5500, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+	/* 127 - 5120x2160@100Hz 64:27 */
+	{
+		DRM_MODE("5120x2160", DRM_MODE_TYPE_DRIVER, 1485000, 5120, 6216,
+			 6304, 6600, 0, 2160, 2168, 2178, 2250, 0,
+			 DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+		.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27,
+	},
+};
+
+static int vd_get_modes(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode = NULL;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(edid_cea_modes_1); i++) {
+		if (edid_cea_modes_1[i].clock == 594000 ||
+		    edid_cea_modes_1[i].clock == 297000 ||
+		    edid_cea_modes_1[i].clock == 148500 ||
+		    edid_cea_modes_1[i].clock == 108000 ||
+		    edid_cea_modes_1[i].clock == 74250 ||
+		    edid_cea_modes_1[i].clock == 54000 ||
+		    edid_cea_modes_1[i].clock == 27000) {
+			mode = drm_mode_duplicate(dev, &edid_cea_modes_1[i]);
+			drm_mode_probed_add(connector, mode);
+		}
+	}
+
+	return 0;
+}
+
+static struct drm_encoder *vd_best_encoder(struct drm_connector *connector)
+{
+	struct es_virtual_display *vd;
+
+	vd = to_virtual_display_with_connector(connector);
+	return &vd->encoder;
+}
+
+static enum drm_mode_status vd_mode_valid(struct drm_connector *connector,
+					  struct drm_display_mode *mode)
+{
+	if (mode->clock != 594000 && mode->clock != 297000 &&
+	    mode->clock != 148500 && mode->clock != 108000 &&
+	    mode->clock != 74250 && mode->clock != 54000 &&
+	    mode->clock != 27000) {
+		return MODE_NOCLOCK;
+	}
+
+	return MODE_OK;
+}
+
+static const struct drm_connector_helper_funcs vd_connector_helper_funcs = {
+	.get_modes = vd_get_modes,
+	.mode_valid = vd_mode_valid,
+	.best_encoder = vd_best_encoder,
+};
+
+static void vd_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static enum drm_connector_status
+vd_connector_detect(struct drm_connector *connector, bool force)
+{
+	return connector_status_connected;
+}
+
+static const struct drm_connector_funcs vd_connector_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = vd_connector_destroy,
+	.detect = vd_connector_detect,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.reset = drm_atomic_helper_connector_reset,
+};
+
+static int vd_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm_dev = data;
+	struct es_virtual_display *vd = dev_get_drvdata(dev);
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	struct device_node *ep, *np;
+	struct platform_device *pdev;
+	int ret;
+
+	/* Encoder */
+	encoder = &vd->encoder;
+	ret = drm_encoder_init(drm_dev, encoder, &vd_encoder_funcs,
+			       DRM_MODE_ENCODER_VIRTUAL, NULL);
+	if (ret) {
+		return ret;
+	}
+
+	encoder->encoder_type = DRM_MODE_ENCODER_VIRTUAL;
+	drm_encoder_helper_add(encoder, &vd_encoder_helper_funcs);
+
+	encoder->possible_crtcs =
+		drm_of_find_possible_crtcs(drm_dev, dev->of_node);
+
+	/* Connector */
+	connector = &vd->connector;
+	ret = drm_connector_init(drm_dev, connector, &vd_connector_funcs,
+				 DRM_MODE_CONNECTOR_VIRTUAL);
+	if (ret)
+		goto connector_init_err;
+	drm_connector_helper_add(connector, &vd_connector_helper_funcs);
+	connector->interlace_allowed = false;
+	connector->doublescan_allowed = false;
+	connector->dpms = DRM_MODE_DPMS_OFF;
+	connector->polled =
+		DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
+	ret = drm_connector_register(connector);
+	if (ret)
+		goto connector_reg_err;
+
+	drm_display_info_set_bus_formats(&connector->display_info,
+					 &vd->bus_format, 1);
+
+	/* attach */
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret)
+		goto attach_err;
+
+	ep = of_graph_get_endpoint_by_regs(dev->of_node, 0, -1);
+	if (!ep) {
+		ret = -EINVAL;
+		goto attach_err;
+	}
+
+	np = of_graph_get_remote_port_parent(ep);
+	of_node_put(ep);
+	if (!np) {
+		ret = -EINVAL;
+		goto attach_err;
+	}
+
+	pdev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (!pdev) {
+		ret = -EPROBE_DEFER;
+		goto attach_err;
+	}
+	get_device(&pdev->dev);
+	vd->dc = &pdev->dev;
+
+	return 0;
+
+attach_err:
+	drm_connector_unregister(connector);
+connector_reg_err:
+	drm_connector_cleanup(connector);
+connector_init_err:
+	drm_encoder_cleanup(encoder);
+	return ret;
+}
+
+static void vd_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct es_virtual_display *vd = dev_get_drvdata(dev);
+
+	drm_connector_unregister(&vd->connector);
+	drm_connector_cleanup(&vd->connector);
+	drm_encoder_cleanup(&vd->encoder);
+	if (vd->dump_obj) {
+		drm_gem_object_put(&vd->dump_obj->base);
+		vd->dump_obj = NULL;
+	}
+}
+
+const struct component_ops vd_component_ops = {
+	.bind = vd_bind,
+	.unbind = vd_unbind,
+};
+
+static const struct of_device_id vd_driver_dt_match[] = {
+	{
+		.compatible = "eswin,virtual_display",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, vd_driver_dt_match);
+
+static int vd_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct es_virtual_display *vd;
+	unsigned char bpp;
+
+	vd = devm_kzalloc(dev, sizeof(*vd), GFP_KERNEL);
+	if (!vd)
+		return -ENOMEM;
+
+	vd->bus_format = MEDIA_BUS_FMT_RGB101010_1X30;
+	of_property_read_u8(dev->of_node, "bpp", &bpp);
+	if (bpp == 8)
+		vd->bus_format = MEDIA_BUS_FMT_RBG888_1X24;
+
+	dev_set_drvdata(dev, vd);
+
+	return component_add(dev, &vd_component_ops);
+}
+
+static int vd_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	component_del(dev, &vd_component_ops);
+
+	dev_set_drvdata(dev, NULL);
+
+	return 0;
+}
+
+struct platform_driver virtual_display_platform_driver = {
+    .probe = vd_probe,
+    .remove = vd_remove,
+    .driver = {
+        .name = "es-virtual-display",
+        .of_match_table = of_match_ptr(vd_driver_dt_match),
+    },
+};
+
+MODULE_DESCRIPTION("Eswin Virtual Display Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/eswin/es_virtual.h b/drivers/gpu/drm/eswin/es_virtual.h
new file mode 100644
index 000000000000..16371232d605
--- /dev/null
+++ b/drivers/gpu/drm/eswin/es_virtual.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Eswin Holdings Co., Ltd.
+ */
+
+#ifndef __ES_VIRTUAL_H_
+#define __ES_VIRTUAL_H_
+
+struct es_virtual_display {
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+	struct device *dc;
+	u32 bus_format;
+
+	struct dentry *dump_debugfs;
+	struct debugfs_blob_wrapper dump_blob;
+	struct es_gem_object *dump_obj;
+	unsigned int pitch;
+};
+
+static inline struct es_virtual_display *
+to_virtual_display_with_connector(struct drm_connector *connector)
+{
+	return container_of(connector, struct es_virtual_display, connector);
+}
+
+static inline struct es_virtual_display *
+to_virtual_display_with_encoder(struct drm_encoder *encoder)
+{
+	return container_of(encoder, struct es_virtual_display, encoder);
+}
+
+extern struct platform_driver virtual_display_platform_driver;
+#endif /* __ES_VIRTUAL_H_ */
diff --git a/drivers/gpu/drm/eswin/eswin_dw_hdmi.c b/drivers/gpu/drm/eswin/eswin_dw_hdmi.c
new file mode 100644
index 000000000000..1412036bdc31
--- /dev/null
+++ b/drivers/gpu/drm/eswin/eswin_dw_hdmi.c
@@ -0,0 +1,957 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2021, ESWIN Electronics Co., Ltd
+ */
+
+#include <linux/component.h>
+#include <linux/clk.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regmap.h>
+
+#include <drm/drm_mode_config.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_of.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/bridge/dw_hdmi.h>
+
+#include <drm/drm_mode.h>
+#include <linux/videodev2.h>
+#include <linux/media-bus-format.h>
+#include <linux/pm_runtime.h>
+
+#include "dw-hdmi.h"
+#include "es_drv.h"
+#include "es_crtc.h"
+
+#define HIWORD_UPDATE(val, mask) (val | (mask) << 16)
+#define ESWIN_HDMI_COLORIMETRY_BT2020                                          \
+	(HDMI_COLORIMETRY_EXTENDED + HDMI_EXTENDED_COLORIMETRY_BT2020)
+
+/* HDMI output pixel format */
+enum drm_hdmi_output_type {
+	DRM_HDMI_OUTPUT_DEFAULT_RGB, /* default RGB */
+	DRM_HDMI_OUTPUT_YCBCR444, /* YCBCR 444 */
+	DRM_HDMI_OUTPUT_YCBCR422, /* YCBCR 422 */
+	DRM_HDMI_OUTPUT_YCBCR420, /* YCBCR 420 */
+	DRM_HDMI_OUTPUT_YCBCR_HQ, /* Highest subsampled YUV */
+	DRM_HDMI_OUTPUT_YCBCR_LQ, /* Lowest subsampled YUV */
+	DRM_HDMI_OUTPUT_INVALID, /* Guess what ? */
+};
+
+enum dw_hdmi_eswin_color_depth {
+	ESWIN_HDMI_DEPTH_8,
+	ESWIN_HDMI_DEPTH_10,
+	ESWIN_HDMI_DEPTH_12,
+	ESWIN_HDMI_DEPTH_16,
+	ESWIN_HDMI_DEPTH_420_10,
+	ESWIN_HDMI_DEPTH_420_12,
+	ESWIN_HDMI_DEPTH_420_16
+};
+
+struct eswin_hdmi {
+	struct device *dev;
+	struct regmap *regmap;
+	struct dw_hdmi *hdmi;
+	struct drm_encoder encoder;
+	u8 id;
+	unsigned long bus_format;
+	unsigned long output_bus_format;
+	unsigned long enc_out_encoding;
+
+	struct drm_property *color_depth_property;
+	struct drm_property *output_format_property;
+	struct drm_property *colorimetry_property;
+	struct drm_property *video_enable_property;
+
+	struct drm_property *color_depth_capacity;
+	struct drm_property *output_format_capacity;
+	struct drm_property *is_hdmi_capacity;
+	struct drm_property *width_heigth_capacity;
+	struct drm_property *quant_range_select_capacity;
+	struct drm_property *max_tmds_clock_capacity;
+
+	unsigned int colordepth;
+	unsigned int colorimetry;
+	unsigned int phy_bus_width;
+	unsigned int hdmi_quant_range;
+	enum drm_hdmi_output_type hdmi_output;
+	bool video_enable;
+};
+
+#define to_eswin_hdmi(x) container_of(x, struct eswin_hdmi, x)
+
+static const struct dw_hdmi_mpll_config eswin_mpll_cfg[] = {
+	{
+		27000000,
+		{
+			{ 0x0003, 0x0628 },
+			{ 0x1003, 0x0632 },
+			{ 0x2003, 0x023c },
+		},
+	},
+	{
+		54000000,
+		{
+			{ 0x0002, 0x0614 },
+			{ 0x1002, 0x0619 },
+			{ 0x2002, 0x021e },
+		},
+	},
+	{
+		74250000,
+		{
+			{ 0x0002, 0x0214 },
+			{ 0x1009, 0x0619 },
+			{ 0x2001, 0x060f },
+		},
+	},
+	{
+		108000000,
+		{
+			{ 0x0001, 0x060a },
+			{ 0x1009, 0x0619 },
+			{ 0x2001, 0x020f },
+		},
+	},
+	{
+		148500000,
+		{
+			{ 0x0001, 0x020a },
+			{ 0x1018, 0x0619 },
+			{ 0x2008, 0x060f },
+		},
+	},
+	{
+		297000000,
+		{
+			{ 0x0000, 0x0205 },
+			{ 0x1658, 0x0219 },
+			{ 0x2648, 0x020f },
+		},
+	},
+	{
+		594000000,
+		{
+			{ 0x0640, 0x0005 },
+			{ 0x1658, 0x0019 },
+			{ 0x2648, 0x000f },
+		},
+	},
+	{
+		~0UL,
+		{
+			{ 0x0000, 0x0000 },
+			{ 0x0000, 0x0000 },
+			{ 0x0000, 0x0000 },
+		},
+	}
+};
+
+static const struct dw_hdmi_curr_ctrl eswin_cur_ctr[] = {
+	/*      pixelclk    bpp8    bpp10   bpp12 */
+	{
+		27000000,
+		{ 0x0283, 0x0281, 0x02c2 },
+	},
+	{
+		54000000,
+		{ 0x1183, 0x1203, 0x1202 },
+	},
+	{
+		74250000,
+		{ 0x1142, 0x2203, 0x2141 },
+	},
+	{
+		108000000,
+		{ 0x20c0, 0x2203, 0x2100 },
+	},
+	{
+		148500000,
+		{ 0x2080, 0x3203, 0x3141 },
+	},
+	{
+		297000000,
+		{ 0x3041, 0x3182, 0x3100 },
+	},
+	{
+		594000000,
+		{ 0x3080, 0x31c0, 0x3100 },
+	},
+	{
+		~0UL,
+		{ 0x0000, 0x0000, 0x0000 },
+	}
+};
+
+static struct dw_hdmi_phy_config eswin_phy_config[] = {
+	/*pixelclk   symbol   term   vlev*/
+	{ 165000000, 0x8088, 0x0007, 0x0180 },
+	{ 297000000, 0x80c8, 0x0004, 0x0180 },
+	{ 594000000, 0x80f8, 0x0000, 0x0180 },
+	{ ~0UL, 0x0000, 0x0000, 0x0000 }
+};
+
+static enum drm_mode_status
+dw_hdmi_eswin_mode_valid(struct dw_hdmi *hdmi, void *data,
+			 const struct drm_display_info *info,
+			 const struct drm_display_mode *mode)
+{
+	const struct dw_hdmi_mpll_config *mpll_cfg = eswin_mpll_cfg;
+	int pclk = mode->clock * 1000;
+	bool valid = false;
+	int i;
+
+	for (i = 0; mpll_cfg[i].mpixelclock != (~0UL); i++) {
+		if (pclk == mpll_cfg[i].mpixelclock) {
+			valid = true;
+			break;
+		}
+	}
+
+	return (valid) ? MODE_OK : MODE_BAD;
+}
+
+static bool
+dw_hdmi_eswin_encoder_mode_fixup(struct drm_encoder *encoder,
+				 const struct drm_display_mode *mode,
+				 struct drm_display_mode *adj_mode)
+{
+	return true;
+}
+
+static void dw_hdmi_eswin_select_output(struct drm_connector_state *conn_state,
+					struct drm_crtc_state *crtc_state,
+					struct eswin_hdmi *hdmi,
+					unsigned int *color_format,
+					unsigned int *color_depth,
+					unsigned long *enc_out_encoding,
+					unsigned int *eotf)
+{
+	struct drm_display_info *info = &conn_state->connector->display_info;
+	struct drm_display_mode *mode = &crtc_state->mode;
+	struct hdr_output_metadata *hdr_metadata;
+	u32 vic = drm_match_cea_mode(mode);
+	unsigned long tmdsclock, pixclock = mode->crtc_clock;
+	bool support_dc = false;
+	int max_tmds_clock = info->max_tmds_clock;
+	int output_eotf;
+
+	*color_format = DRM_HDMI_OUTPUT_DEFAULT_RGB;
+
+	switch (hdmi->hdmi_output) {
+	case DRM_HDMI_OUTPUT_YCBCR_HQ:
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR444;
+		else if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+		else if (conn_state->connector->ycbcr_420_allowed &&
+			 drm_mode_is_420(info, mode))
+			*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR_LQ:
+		if (conn_state->connector->ycbcr_420_allowed &&
+		    drm_mode_is_420(info, mode))
+			*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		else if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+		else if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR444;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR420:
+		if (conn_state->connector->ycbcr_420_allowed &&
+		    drm_mode_is_420(info, mode))
+			*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR422:
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+		break;
+	case DRM_HDMI_OUTPUT_YCBCR444:
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR444;
+		break;
+	case DRM_HDMI_OUTPUT_DEFAULT_RGB:
+	default:
+		break;
+	}
+
+	if (*color_format == DRM_HDMI_OUTPUT_DEFAULT_RGB &&
+	    info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30)
+		support_dc = true;
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR444 &&
+	    info->edid_hdmi_rgb444_dc_modes &
+		    (DRM_EDID_HDMI_DC_Y444 | DRM_EDID_HDMI_DC_30))
+		support_dc = true;
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR422)
+		support_dc = true;
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR420 &&
+	    info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30)
+		support_dc = true;
+
+	if (hdmi->colordepth > 8 && support_dc)
+		*color_depth = 10;
+	else
+		*color_depth = 8;
+
+	*eotf = TRADITIONAL_GAMMA_SDR;
+	if (conn_state->hdr_output_metadata) {
+		hdr_metadata = (struct hdr_output_metadata *)
+				       conn_state->hdr_output_metadata->data;
+		output_eotf = hdr_metadata->hdmi_metadata_type1.eotf;
+		if (output_eotf > TRADITIONAL_GAMMA_HDR &&
+		    output_eotf < FUTURE_EOTF)
+			*eotf = output_eotf;
+	}
+
+	if ((*eotf > TRADITIONAL_GAMMA_HDR &&
+	     conn_state->connector->hdr_sink_metadata.hdmi_type1.eotf &
+		     BIT(*eotf)) ||
+	    (hdmi->colorimetry == ESWIN_HDMI_COLORIMETRY_BT2020))
+		*enc_out_encoding = V4L2_YCBCR_ENC_BT2020;
+	else if ((vic == 6) || (vic == 7) || (vic == 21) || (vic == 22) ||
+		 (vic == 2) || (vic == 3) || (vic == 17) || (vic == 18))
+		*enc_out_encoding = V4L2_YCBCR_ENC_601;
+	else
+		*enc_out_encoding = V4L2_YCBCR_ENC_709;
+
+	if (*enc_out_encoding == V4L2_YCBCR_ENC_BT2020) {
+		/* BT2020 require color depth at lest 10bit */
+		*color_depth = 10;
+		/* We prefer use YCbCr422 to send 10bit */
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*color_format = DRM_HDMI_OUTPUT_YCBCR422;
+	}
+
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+		pixclock *= 2;
+	if ((mode->flags & DRM_MODE_FLAG_3D_MASK) ==
+	    DRM_MODE_FLAG_3D_FRAME_PACKING)
+		pixclock *= 2;
+
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR422 || *color_depth == 8)
+		tmdsclock = pixclock;
+	else
+		tmdsclock = pixclock * (*color_depth) / 8;
+
+	if (*color_format == DRM_HDMI_OUTPUT_YCBCR420)
+		tmdsclock /= 2;
+
+	/* XXX: max_tmds_clock of some sink is 0, we think it is 340MHz. */
+	if (!max_tmds_clock)
+		max_tmds_clock = 340000;
+
+	max_tmds_clock = min(max_tmds_clock, 594000);
+
+	if (tmdsclock > max_tmds_clock) {
+		if (max_tmds_clock >= 594000) {
+			*color_depth = 8;
+		} else if (max_tmds_clock > 340000) {
+			if (drm_mode_is_420(info, mode) || tmdsclock >= 594000)
+				*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		} else {
+			*color_depth = 8;
+			if (drm_mode_is_420(info, mode) || tmdsclock >= 594000)
+				*color_format = DRM_HDMI_OUTPUT_YCBCR420;
+		}
+	}
+}
+
+static int
+dw_hdmi_eswin_encoder_atomic_check(struct drm_encoder *encoder,
+				   struct drm_crtc_state *crtc_state,
+				   struct drm_connector_state *conn_state)
+{
+	struct es_crtc_state *s = to_es_crtc_state(crtc_state);
+	struct eswin_hdmi *hdmi = to_eswin_hdmi(encoder);
+	unsigned int colordepth, colorformat, bus_width, eotf;
+
+	dw_hdmi_eswin_select_output(conn_state, crtc_state, hdmi, &colorformat,
+				    &colordepth, &hdmi->enc_out_encoding,
+				    &eotf);
+	if (colordepth > 8)
+		hdmi->bus_format = MEDIA_BUS_FMT_RGB101010_1X30;
+	else
+		hdmi->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+
+	/* DC does not support YUV output */
+	s->output_fmt = hdmi->bus_format;
+
+	if (colorformat == DRM_HDMI_OUTPUT_YCBCR420) {
+		if (colordepth > 8)
+			hdmi->output_bus_format = MEDIA_BUS_FMT_UYYVYY10_0_5X30;
+		else
+			hdmi->output_bus_format = MEDIA_BUS_FMT_UYYVYY8_0_5X24;
+
+		bus_width = colordepth / 2;
+	} else {
+		if ((colordepth > 8) &&
+		    (colorformat != DRM_HDMI_OUTPUT_YCBCR422)) {
+			if (colorformat != DRM_HDMI_OUTPUT_DEFAULT_RGB)
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_YUV10_1X30;
+			else
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_RGB101010_1X30;
+		} else {
+			if (colorformat != DRM_HDMI_OUTPUT_DEFAULT_RGB)
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_YUV8_1X24;
+			else
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_RGB888_1X24;
+		}
+		if (colorformat == DRM_HDMI_OUTPUT_YCBCR422) {
+			bus_width = 8;
+			if (colordepth > 8)
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_UYVY10_1X20;
+			else
+				hdmi->output_bus_format =
+					MEDIA_BUS_FMT_UYVY8_1X16;
+		} else {
+			bus_width = colordepth;
+		}
+	}
+
+	hdmi->phy_bus_width = bus_width;
+
+	s->encoder_type = DRM_MODE_ENCODER_TMDS;
+
+	return 0;
+}
+
+static unsigned long dw_hdmi_eswin_get_input_bus_format(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->bus_format;
+}
+
+static unsigned long dw_hdmi_eswin_get_output_bus_format(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->output_bus_format;
+}
+
+static unsigned long dw_hdmi_eswin_get_enc_in_encoding(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->enc_out_encoding;
+}
+
+static unsigned long dw_hdmi_eswin_get_enc_out_encoding(void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	return hdmi->enc_out_encoding;
+}
+
+static const struct drm_prop_enum_list color_depth_enum_list[] = {
+	{ 0, "Automatic" }, /* Same as 24bit */
+	{ 8, "24bit" },
+	{ 10, "30bit" },
+};
+
+static const struct drm_prop_enum_list drm_hdmi_output_enum_list[] = {
+	{ DRM_HDMI_OUTPUT_DEFAULT_RGB, "output_rgb" },
+	{ DRM_HDMI_OUTPUT_YCBCR444, "output_ycbcr444" },
+	{ DRM_HDMI_OUTPUT_YCBCR422, "output_ycbcr422" },
+	{ DRM_HDMI_OUTPUT_YCBCR420, "output_ycbcr420" },
+	{ DRM_HDMI_OUTPUT_YCBCR_HQ, "output_ycbcr_high_subsampling" },
+	{ DRM_HDMI_OUTPUT_YCBCR_LQ, "output_ycbcr_low_subsampling" },
+	{ DRM_HDMI_OUTPUT_INVALID, "invalid_output" },
+};
+
+static const struct drm_prop_enum_list colorimetry_enum_list[] = {
+	{ HDMI_COLORIMETRY_NONE, "None" },
+	{ ESWIN_HDMI_COLORIMETRY_BT2020, "ITU_2020" },
+};
+
+static const struct drm_prop_enum_list quant_range_enum_list[] = {
+	{ HDMI_QUANTIZATION_RANGE_DEFAULT, "default" },
+	{ HDMI_QUANTIZATION_RANGE_LIMITED, "limit" },
+	{ HDMI_QUANTIZATION_RANGE_FULL, "full" },
+};
+
+static const struct drm_prop_enum_list color_depth_capacity_list[] = {
+	{ BIT(ESWIN_HDMI_DEPTH_8), "8bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_10), "10bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_12), "12bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_16), "16bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_420_10), "yuv420_10bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_420_12), "yuv420_12bit" },
+	{ BIT(ESWIN_HDMI_DEPTH_420_16), "yuv420_16bit" },
+};
+
+static const struct drm_prop_enum_list output_format_capacity_list[] = {
+	{ BIT(DRM_HDMI_OUTPUT_DEFAULT_RGB), "rgb" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR444), "yuv444" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR422), "yuv422" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR420), "yuv420" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR_HQ), "yuv_hq" },
+	{ BIT(DRM_HDMI_OUTPUT_YCBCR_LQ), "yuv_lq" },
+};
+
+static void dw_hdmi_eswin_attatch_properties(struct drm_connector *connector,
+					     unsigned int color, int version,
+					     void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+	struct drm_property *prop;
+#ifdef CONFIG_ESWIN_DW_HDMI
+	struct es_drm_private *private = connector->dev->dev_private;
+#endif
+	switch (color) {
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_DEFAULT_RGB;
+		hdmi->colordepth = 10;
+		break;
+	case MEDIA_BUS_FMT_YUV8_1X24:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR444;
+		hdmi->colordepth = 8;
+		break;
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR444;
+		hdmi->colordepth = 10;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR422;
+		hdmi->colordepth = 10;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR422;
+		hdmi->colordepth = 8;
+		break;
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR420;
+		hdmi->colordepth = 8;
+		break;
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_YCBCR420;
+		hdmi->colordepth = 10;
+		break;
+	default:
+		hdmi->hdmi_output = DRM_HDMI_OUTPUT_DEFAULT_RGB;
+		hdmi->colordepth = 8;
+	}
+
+	if (!hdmi->color_depth_property) {
+		prop = drm_property_create_enum(
+			connector->dev, 0, "hdmi_output_color_depth",
+			color_depth_enum_list,
+			ARRAY_SIZE(color_depth_enum_list));
+		if (prop) {
+			hdmi->color_depth_property = prop;
+			drm_object_attach_property(&connector->base, prop, 0);
+		}
+	}
+
+	prop = drm_property_create_enum(connector->dev, 0, "hdmi_output_format",
+					drm_hdmi_output_enum_list,
+					ARRAY_SIZE(drm_hdmi_output_enum_list));
+	if (prop) {
+		hdmi->output_format_property = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_enum(connector->dev, 0,
+					"hdmi_output_colorimetry",
+					colorimetry_enum_list,
+					ARRAY_SIZE(colorimetry_enum_list));
+	if (prop) {
+		hdmi->colorimetry_property = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_bool(connector->dev, 0, "video_enable");
+	if (prop) {
+		hdmi->video_enable_property = prop;
+		drm_object_attach_property(&connector->base, prop, 1);
+		hdmi->video_enable = true;
+	}
+
+	prop = drm_property_create_enum(connector->dev, 0,
+					"hdmi_color_depth_capacity",
+					color_depth_capacity_list,
+					ARRAY_SIZE(color_depth_capacity_list));
+	if (prop) {
+		hdmi->color_depth_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_enum(
+		connector->dev, 0, "hdmi_output_format_capacity",
+		output_format_capacity_list,
+		ARRAY_SIZE(output_format_capacity_list));
+	if (prop) {
+		hdmi->output_format_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_bool(connector->dev, 0, "is_hdmi_capacity");
+	if (prop) {
+		hdmi->is_hdmi_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_range(
+		connector->dev, 0, "hdmi_width_height_mm_capacity", 0, 0xff);
+	if (prop) {
+		hdmi->width_heigth_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_bool(connector->dev, 0,
+					"hdmi_quant_range_sel_capacity");
+	if (prop) {
+		hdmi->quant_range_select_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = drm_property_create_range(
+		connector->dev, 0, "hdmi_max_tmds_clock_capacity", 0, 340000);
+	if (prop) {
+		hdmi->max_tmds_clock_capacity = prop;
+		drm_object_attach_property(&connector->base, prop, 0);
+	}
+
+	prop = connector->dev->mode_config.hdr_output_metadata_property;
+	if (version >= 0x211a)
+		drm_object_attach_property(&connector->base, prop, 0);
+
+#ifdef CONFIG_ESWIN_DW_HDMI
+	drm_object_attach_property(&connector->base, private->connector_id_prop,
+				   0);
+#endif
+}
+
+static void dw_hdmi_eswin_destroy_properties(struct drm_connector *connector,
+					     void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	if (hdmi->color_depth_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->color_depth_property);
+		hdmi->color_depth_property = NULL;
+	}
+
+	if (hdmi->output_format_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->output_format_property);
+		hdmi->output_format_property = NULL;
+	}
+
+	if (hdmi->colorimetry_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->colorimetry_property);
+		hdmi->colorimetry_property = NULL;
+	}
+
+	if (hdmi->video_enable_property) {
+		drm_property_destroy(connector->dev,
+				     hdmi->video_enable_property);
+		hdmi->video_enable_property = NULL;
+	}
+
+	if (hdmi->color_depth_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->color_depth_capacity);
+		hdmi->color_depth_capacity = NULL;
+	}
+
+	if (hdmi->output_format_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->output_format_capacity);
+		hdmi->output_format_capacity = NULL;
+	}
+
+	if (hdmi->is_hdmi_capacity) {
+		drm_property_destroy(connector->dev, hdmi->is_hdmi_capacity);
+		hdmi->is_hdmi_capacity = NULL;
+	}
+
+	if (hdmi->width_heigth_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->width_heigth_capacity);
+		hdmi->width_heigth_capacity = NULL;
+	}
+
+	if (hdmi->quant_range_select_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->quant_range_select_capacity);
+		hdmi->quant_range_select_capacity = NULL;
+	}
+
+	if (hdmi->max_tmds_clock_capacity) {
+		drm_property_destroy(connector->dev,
+				     hdmi->max_tmds_clock_capacity);
+		hdmi->max_tmds_clock_capacity = NULL;
+	}
+}
+
+static int dw_hdmi_eswin_set_property(struct drm_connector *connector,
+				      struct drm_connector_state *state,
+				      struct drm_property *property,
+				      uint64_t val, void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+
+	if (property == hdmi->color_depth_property) {
+		hdmi->colordepth = val;
+	} else if (property == hdmi->output_format_property) {
+		hdmi->hdmi_output = val;
+	} else if (property == hdmi->colorimetry_property) {
+		hdmi->colorimetry = val;
+	} else if (property == hdmi->video_enable_property) {
+		if (hdmi->video_enable != val) {
+			if (val == true) {
+				dw_hdmi_enable_video(hdmi->hdmi);
+			} else {
+				dw_hdmi_disable_video(hdmi->hdmi);
+			}
+			hdmi->video_enable = val;
+		}
+	} else {
+		DRM_DEBUG("don't support set %s property\n", property->name);
+		return 0;
+	}
+	return 0;
+}
+
+static int dw_hdmi_eswin_get_property(struct drm_connector *connector,
+				      const struct drm_connector_state *state,
+				      struct drm_property *property,
+				      uint64_t *val, void *data)
+{
+	struct eswin_hdmi *hdmi = (struct eswin_hdmi *)data;
+	struct drm_display_info *info = &connector->display_info;
+	struct drm_mode_config *config = &connector->dev->mode_config;
+#ifdef CONFIG_ESWIN_DW_HDMI
+	struct es_drm_private *private = connector->dev->dev_private;
+#endif
+	if (property == hdmi->color_depth_property) {
+		*val = hdmi->colordepth;
+	} else if (property == hdmi->output_format_property) {
+		*val = hdmi->hdmi_output;
+	} else if (property == hdmi->color_depth_capacity) {
+		*val = BIT(ESWIN_HDMI_DEPTH_8);
+		if (info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30)
+			*val |= BIT(ESWIN_HDMI_DEPTH_10);
+		if (info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_36)
+			*val |= BIT(ESWIN_HDMI_DEPTH_12);
+		if (info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_48)
+			*val |= BIT(ESWIN_HDMI_DEPTH_16);
+		if (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30)
+			*val |= BIT(ESWIN_HDMI_DEPTH_420_10);
+		if (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36)
+			*val |= BIT(ESWIN_HDMI_DEPTH_420_12);
+		if (info->hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48)
+			*val |= BIT(ESWIN_HDMI_DEPTH_420_16);
+	} else if (property == hdmi->output_format_capacity) {
+		*val = BIT(DRM_HDMI_OUTPUT_DEFAULT_RGB);
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+			*val |= BIT(DRM_HDMI_OUTPUT_YCBCR444);
+		if (info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+			*val |= BIT(DRM_HDMI_OUTPUT_YCBCR422);
+		if (connector->ycbcr_420_allowed &&
+		    info->color_formats & DRM_COLOR_FORMAT_YCBCR420)
+			*val |= BIT(DRM_HDMI_OUTPUT_YCBCR420);
+	} else if (property == config->hdr_output_metadata_property) {
+		*val = state->hdr_output_metadata ?
+			       state->hdr_output_metadata->base.id :
+			       0;
+	} else if (property == hdmi->colorimetry_property) {
+		*val = hdmi->colorimetry;
+	}
+#ifdef CONFIG_ESWIN_DW_HDMI
+	else if (property == private->connector_id_prop) {
+		*val = hdmi->id;
+	}
+#endif
+	else if (property == hdmi->is_hdmi_capacity) {
+		*val = info->is_hdmi;
+	} else if (property == hdmi->quant_range_select_capacity) {
+		*val = info->rgb_quant_range_selectable;
+	} else if (property == hdmi->width_heigth_capacity) {
+		property->values[0] = info->width_mm;
+		property->values[1] = info->height_mm;
+		*val = 0;
+	} else if (property == hdmi->max_tmds_clock_capacity) {
+		*val = info->max_tmds_clock;
+	} else if (property == hdmi->video_enable_property) {
+		*val = hdmi->video_enable;
+	} else {
+		DRM_ERROR("failed to get eswin hdmi connector %s property\n",
+			  property->name);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static const struct dw_hdmi_property_ops dw_hdmi_eswin_property_ops = {
+	.attatch_properties = dw_hdmi_eswin_attatch_properties,
+	.destroy_properties = dw_hdmi_eswin_destroy_properties,
+	.set_property = dw_hdmi_eswin_set_property,
+	.get_property = dw_hdmi_eswin_get_property,
+};
+
+static const struct drm_encoder_helper_funcs
+	dw_hdmi_eswin_encoder_helper_funcs = {
+		.mode_fixup = dw_hdmi_eswin_encoder_mode_fixup,
+		.atomic_check = dw_hdmi_eswin_encoder_atomic_check,
+	};
+
+static const struct dw_hdmi_plat_data win2030_hdmi_drv_data = {
+	.mode_valid = dw_hdmi_eswin_mode_valid,
+	.mpll_cfg = eswin_mpll_cfg,
+	.cur_ctr = eswin_cur_ctr,
+	.phy_config = eswin_phy_config,
+	.use_drm_infoframe = true,
+	.ycbcr_420_allowed = false,
+};
+
+static const struct of_device_id dw_hdmi_eswin_dt_ids[] = {
+	{ .compatible = "eswin,eswin-dw-hdmi", .data = &win2030_hdmi_drv_data },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dw_hdmi_eswin_dt_ids);
+
+static int dw_hdmi_eswin_bind(struct device *dev, struct device *master,
+			      void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dw_hdmi_plat_data *plat_data;
+	const struct of_device_id *match;
+	struct drm_device *drm = data;
+	struct drm_encoder *encoder;
+	struct eswin_hdmi *hdmi;
+	int ret = 0;
+
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
+	hdmi = devm_kzalloc(&pdev->dev, sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return -ENOMEM;
+
+	match = of_match_node(dw_hdmi_eswin_dt_ids, pdev->dev.of_node);
+	plat_data = devm_kmemdup(&pdev->dev, match->data, sizeof(*plat_data),
+				 GFP_KERNEL);
+	if (!plat_data)
+		return -ENOMEM;
+
+	hdmi->dev = &pdev->dev;
+
+	plat_data->phy_data = hdmi;
+	encoder = &hdmi->encoder;
+
+	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
+	/*
+     * If we failed to find the CRTC(s) which this encoder is
+     * supposed to be connected to, it's because the CRTC has
+     * not been registered yet.  Defer probing, and hope that
+     * the required CRTC is added later.
+     */
+	if (encoder->possible_crtcs == 0)
+		return -EPROBE_DEFER;
+
+	plat_data->phy_data = hdmi;
+	plat_data->get_input_bus_format = dw_hdmi_eswin_get_input_bus_format;
+	plat_data->get_output_bus_format = dw_hdmi_eswin_get_output_bus_format;
+	plat_data->get_enc_in_encoding = dw_hdmi_eswin_get_enc_in_encoding;
+	plat_data->get_enc_out_encoding = dw_hdmi_eswin_get_enc_out_encoding;
+	plat_data->property_ops = &dw_hdmi_eswin_property_ops;
+
+	drm_encoder_helper_add(encoder, &dw_hdmi_eswin_encoder_helper_funcs);
+	drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+
+	platform_set_drvdata(pdev, hdmi);
+
+	hdmi->hdmi = dw_hdmi_bind(pdev, encoder, plat_data);
+
+	/*
+     * If dw_hdmi_bind() fails we'll never call dw_hdmi_unbind(),
+     * which would have called the encoder cleanup.  Do it manually.
+     */
+	if (IS_ERR(hdmi->hdmi)) {
+		ret = PTR_ERR(hdmi->hdmi);
+		drm_encoder_cleanup(encoder);
+	}
+
+	return ret;
+}
+
+static void dw_hdmi_eswin_unbind(struct device *dev, struct device *master,
+				 void *data)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(dev);
+
+	dw_hdmi_unbind(hdmi->hdmi);
+}
+
+static const struct component_ops dw_hdmi_eswin_ops = {
+	.bind = dw_hdmi_eswin_bind,
+	.unbind = dw_hdmi_eswin_unbind,
+};
+
+static int dw_hdmi_eswin_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &dw_hdmi_eswin_ops);
+}
+
+static void dw_hdmi_eswin_shutdown(struct platform_device *pdev)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(&pdev->dev);
+
+	dw_hdmi_suspend(hdmi->hdmi);
+}
+
+static int dw_hdmi_eswin_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dw_hdmi_eswin_ops);
+	return 0;
+}
+
+static int __maybe_unused dw_hdmi_eswin_suspend(struct device *dev)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(dev);
+
+	dw_hdmi_suspend(hdmi->hdmi);
+
+	return 0;
+}
+
+static int __maybe_unused dw_hdmi_eswin_resume(struct device *dev)
+{
+	struct eswin_hdmi *hdmi = dev_get_drvdata(dev);
+
+	dw_hdmi_resume(hdmi->hdmi);
+
+	return 0;
+}
+
+static const struct dev_pm_ops dw_hdmi_eswin_pm = { SET_SYSTEM_SLEEP_PM_OPS(
+	dw_hdmi_eswin_suspend, dw_hdmi_eswin_resume) };
+
+struct platform_driver dw_hdmi_eswin_pltfm_driver = {
+    .probe  = dw_hdmi_eswin_probe,
+    .remove = dw_hdmi_eswin_remove,
+    .shutdown = dw_hdmi_eswin_shutdown,
+    .driver = {
+        .name = "dw-hdmi-eswin",
+        .pm = &dw_hdmi_eswin_pm,
+        .of_match_table = dw_hdmi_eswin_dt_ids,
+    },
+};
+
+//module_platform_driver(dw_hdmi_eswin_pltfm_driver);
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/eswin/host_lib_driver_linux_if.h b/drivers/gpu/drm/eswin/host_lib_driver_linux_if.h
new file mode 100644
index 000000000000..0cde34421994
--- /dev/null
+++ b/drivers/gpu/drm/eswin/host_lib_driver_linux_if.h
@@ -0,0 +1,146 @@
+#ifndef _HOST_LIB_DRIVER_LINUX_IF_H_
+#define _HOST_LIB_DRIVER_LINUX_IF_H_
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define HL_DRIVER_ALLOCATE_DYNAMIC_MEM 0xffffffff
+// hl_drv_ioctl numbers
+enum {
+	HL_DRV_NR_MIN = 0x10,
+	HL_DRV_NR_INIT,
+	HL_DRV_NR_MEMINFO,
+	HL_DRV_NR_LOAD_CODE,
+	HL_DRV_NR_READ_DATA,
+	HL_DRV_NR_WRITE_DATA,
+	HL_DRV_NR_MEMSET_DATA,
+	HL_DRV_NR_READ_HPI,
+	HL_DRV_NR_WRITE_HPI,
+
+	DW_DRV_NR_CONNECT_STATUS,
+	DW_DRV_NR_CONNECT_SET,
+	DW_DRV_NR_DISCONNECT_SET,
+	DW_DRV_NR_AUTH_SUCCESS,
+	DW_DRV_NR_AUTH_FAIL,
+	DW_DRV_NR_NO_CAPACITY,
+
+	HL_DRV_NR_MAX
+};
+
+/*
+ * HL_DRV_IOC_INIT: associate file descriptor with the indicated memory.  This
+ * must be called before any other hl_drv_ioctl on the file descriptor.
+ *
+ *   - hpi_base = base address of HPI registers.
+ *   - code_base = base address of firmware memory (0 to allocate internally)
+ *   - data_base = base address of data memory (0 to allocate internally)
+ *   - code_len, data_len = length of firmware and data memory, respectively.
+ */
+#define HL_DRV_IOC_INIT _IOW('H', HL_DRV_NR_INIT, struct hl_drv_ioc_meminfo)
+
+/*
+ * HL_DRV_IOC_MEMINFO: retrieve memory information from file descriptor.
+ *
+ * Fills out the meminfo struct, returning the values passed to HL_DRV_IOC_INIT
+ * except that the actual base addresses of internal allocations (if any) are
+ * reported.
+ */
+#define HL_DRV_IOC_MEMINFO                                                     \
+	_IOR('H', HL_DRV_NR_MEMINFO, struct hl_drv_ioc_meminfo)
+
+struct hl_drv_ioc_meminfo {
+	__u32 hpi_base;
+	__u32 code_base;
+	__u32 code_size;
+	__u32 data_base;
+	__u32 data_size;
+};
+
+/*
+ * HL_DRV_IOC_LOAD_CODE: write the provided buffer to the firmware memory.
+ *
+ *   - len = number of bytes in data buffer
+ *   - data = data to write to firmware memory.
+ *
+ * This can only be done once (successfully).  Subsequent attempts will
+ * return -EBUSY.
+ */
+#define HL_DRV_IOC_LOAD_CODE                                                   \
+	_IOW('H', HL_DRV_NR_LOAD_CODE, struct hl_drv_ioc_code)
+
+struct hl_drv_ioc_code {
+	__u32 len;
+	__u8 data[];
+};
+
+/*
+ * HL_DRV_IOC_READ_DATA: copy from data memory.
+ * HL_DRV_IOC_WRITE_DATA: copy to data memory.
+ *
+ *   - offset = start copying at this byte offset into the data memory.
+ *   - len    = number of bytes to copy.
+ *   - data   = for write, buffer containing data to copy.
+ *              for read, buffer to which read data will be written.
+ *
+ */
+#define HL_DRV_IOC_READ_DATA                                                   \
+	_IOWR('H', HL_DRV_NR_READ_DATA, struct hl_drv_ioc_data)
+#define HL_DRV_IOC_WRITE_DATA                                                  \
+	_IOW('H', HL_DRV_NR_WRITE_DATA, struct hl_drv_ioc_data)
+
+/*
+ * HL_DRV_IOC_MEMSET_DATA: initialize data memory.
+ *
+ *   - offset  = start initializatoin at this byte offset into the data memory.
+ *   - len     = number of bytes to set.
+ *   - data[0] = byte value to write to all indicated memory locations.
+ */
+#define HL_DRV_IOC_MEMSET_DATA                                                 \
+	_IOW('H', HL_DRV_NR_MEMSET_DATA, struct hl_drv_ioc_data)
+
+struct hl_drv_ioc_data {
+	__u32 offset;
+	__u32 len;
+	__u8 data[];
+};
+
+/*
+ * HL_DRV_IOC_READ_HPI: read HPI register.
+ * HL_DRV_IOC_WRITE_HPI: write HPI register.
+ *
+ *   - offset = byte offset of HPI register to access.
+ *   - value  = for write, value to write.
+ *              for read, location to which result is stored.
+ */
+#define HL_DRV_IOC_READ_HPI                                                    \
+	_IOWR('H', HL_DRV_NR_READ_HPI, struct hl_drv_ioc_hpi_reg)
+#define HL_DRV_IOC_WRITE_HPI                                                   \
+	_IOW('H', HL_DRV_NR_WRITE_HPI, struct hl_drv_ioc_hpi_reg)
+
+struct hl_drv_ioc_hpi_reg {
+	__u32 offset;
+	__u32 value;
+};
+
+#define DW_DRV_IOC_CONNECT_STATUS                                              \
+	_IOR('H', DW_DRV_NR_CONNECT_STATUS, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_CONNECT_SET                                                 \
+	_IOW('H', DW_DRV_NR_CONNECT_SET, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_DISCONNECT_SET                                              \
+	_IOW('H', DW_DRV_NR_DISCONNECT_SET, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_AUTH_SUCCESS                                                \
+	_IOW('H', DW_DRV_NR_AUTH_SUCCESS, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_AUTH_FAIL                                                   \
+	_IOW('H', DW_DRV_NR_AUTH_FAIL, struct hl_drv_ioc_info)
+#define DW_DRV_IOC_NO_CAPACITY                                                 \
+	_IOW('H', DW_DRV_NR_NO_CAPACITY, struct hl_drv_ioc_info)
+
+struct hl_drv_ioc_info {
+	union {
+		__u32 connect;
+		__u32 auth;
+		__u32 capacity;
+	};
+};
+
+#endif // _HOST_LIB_DRIVER_LINUX_IF_H_
diff --git a/drivers/gpu/drm/img/Kconfig b/drivers/gpu/drm/img/Kconfig
new file mode 100644
index 000000000000..a83d094127e7
--- /dev/null
+++ b/drivers/gpu/drm/img/Kconfig
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config DRM_IMG
+	bool "Imagination PowerVR GPU"
+        select DRM
+        default n
+
+source "drivers/gpu/drm/img/img-volcanic/Kconfig"
diff --git a/drivers/gpu/drm/img/Makefile b/drivers/gpu/drm/img/Makefile
new file mode 100644
index 000000000000..fa5c41d9908d
--- /dev/null
+++ b/drivers/gpu/drm/img/Makefile
@@ -0,0 +1,4 @@
+# volcanic
+obj-$(CONFIG_DRM_IMG_VOLCANIC) += img-volcanic/
+
+ccflags-y+= -I$(srctree)/mm
diff --git a/drivers/gpu/drm/img/img-volcanic/Kconfig b/drivers/gpu/drm/img/img-volcanic/Kconfig
new file mode 100644
index 000000000000..587a1c60ad8f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/Kconfig
@@ -0,0 +1,10 @@
+config DRM_IMG_VOLCANIC
+	tristate "DRM support for PowerVR GPU"
+        select DRM_IMG
+	select SYNC_FILE
+	default n
+	help
+	  Driver for PowerVR Rogue graphics hardware.
+
+	  Say Y here if your SoC contains a PowerVR Volcanic GPU. For more
+	  information, see <http://www.imgtec.com/powervr/>.
diff --git a/drivers/gpu/drm/img/img-volcanic/Makefile b/drivers/gpu/drm/img/img-volcanic/Makefile
new file mode 100644
index 000000000000..087e156a263f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/Makefile
@@ -0,0 +1,138 @@
+########################################################################### ###
+#@Title         Root kernel makefile
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+# This top-level kbuild makefile builds all the Linux kernel modules in the
+# DDK. To run kbuild, this makefile is copied to $(TARGET_PRIMARY_OUT)/kbuild/Makefile
+# and make is invoked in $(TARGET_PRIMARY_OUT)/kbuild.
+
+# This makefile doesn't define any kbuild special variables apart from
+# ccflags-y and obj-m. The variables for objects are picked up by including
+# the kbuild makefile fragments named in $(INTERNAL_KBUILD_MAKEFILES). The
+# list of objects that these fragments make is collected in
+# $(INTERNAL_KBUILD_OBJECTS) and $(INTERNAL_EXTRA_KBUILD_OBJECTS). These
+# variables are set according to the build's $(KERNEL_COMPONENTS) and
+# $(EXTRA_PVRSRVKM_COMPONENTS). To add a new kernel module to the build, edit
+# these variables in the per-build Makefile.
+
+INTERNAL_KBUILD_OBJECTS=pvrsrvkm.o
+INTERNAL_EXTRA_KBUILD_OBJECTS=""
+TOP:=$(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+TOP:=$(TOP:/=)
+BRIDGE_SOURCE_ROOT=$(TOP)/generated/volcanic
+TARGET_PRIMARY_ARCH=target_riscv64
+PVR_ARCH=volcanic
+PVR_ARCH_DEFS=volcanic
+PVR_SYSTEM := eswin_cpu
+#PDUMP ?= 1
+PDUMP ?= 0
+BUILD ?= release
+RGX_BVNC ?= 30.3.408.101
+RGX_BNC ?= 30.3.408.101
+PVRSRV_NEED_PVR_DPF=1
+PVRSRV_NEED_PVR_ASSERT=1
+PVR_SERVICES_DEBUG=1
+WINDOW_SYSTEM=nulldrmws
+
+#include $(OUT)/config_kernel.mk
+include $(srctree)/$(src)/config_kernel.mk
+
+# Use the logic from kernel's Kbuild.include to select .NOTINTERMEDIATE or .SECONDARY based on make version
+ifneq ($(and $(filter notintermediate, $(.FEATURES)),$(filter-out 4.4,$(MAKE_VERSION))),)
+.NOTINTERMEDIATE:
+else
+.SECONDARY:
+endif
+
+
+define symlink-source-file
+@if [ ! -e $(dir $@) ]; then mkdir -p $(dir $@); fi
+@if [ ! -h $@ ]; then ln -sf $< $@; fi
+endef
+
+bridge_base := $(BRIDGE_SOURCE_ROOT)
+
+#$(OUT)/$(TARGET_PRIMARY_ARCH)/kbuild/%.c: $(TOP)/%.c
+#	$(symlink-source-file)
+
+#$(OUT)/$(TARGET_PRIMARY_ARCH)/kbuild/generated/$(PVR_ARCH)/%.c: $(bridge_base)/%.c
+#	$(symlink-source-file)
+
+#$(OUT)/$(TARGET_PRIMARY_ARCH)/kbuild/external/%.c: $(abspath $(srctree))/%.c
+#	$(symlink-source-file)
+
+#ccflags-y += -D__linux__ -include $(OUT)/config_kernel.h \
+# -include kernel_config_compatibility.h \
+# -I$(OUT)/include -I$(TOP)/kernel/drivers/staging/imgtec
+ccflags-y += -D__linux__ -include $(srctree)/$(src)/config_kernel.h \
+ -include $(srctree)/drivers/gpu/drm/img/kernel_config_compatibility.h \
+ -I$(OUT)/include \
+ -I$(srctree)/drivers/gpu/drm/img \
+ -I$(srctree)/$(src)/hwdefs/volcanic \
+ -I$(srctree)/$(src)/hwdefs/volcanic/km \
+ -I$(srctree)/$(src)/include/system/rgx_tc \
+ -I$(srctree)/$(src)/include/volcanic/system/rgx_tc
+
+include $(srctree)/$(src)/services/server/env/linux/Kbuild.mk
+include $(srctree)/$(src)/services/system/eswin_cpu/Kbuild.mk
+
+#include $(INTERNAL_KBUILD_MAKEFILES)
+
+define add-file-cflags
+CFLAGS_$(notdir $(1)) := $(CFLAGS_$(1))
+endef
+
+# Define old style CFLAG_ variables for kernels older than 5.4
+$(foreach _m,$(INTERNAL_KBUILD_OBJECTS:.o=-y), \
+ $(foreach _f,$($(_m)), \
+  $(if $(CFLAGS_$(_f)), \
+   $(eval $(call add-file-cflags,$(_f))))))
+
+#ifneq ($(KERNEL_DRIVER_DIR),)
+# ccflags-y += \
+#   -I$(abspath $(srctree))/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM) \
+#   -I$(abspath $(srctree))/$(KERNEL_DRIVER_DIR)
+#endif
+
+$(if $($(PVRSRVKM_NAME)-y),,$(warning $(PVRSRVKM_NAME)-y was empty, which could mean that srvkm is missing from $$(KERNEL_COMPONENTS)))
+$(PVRSRVKM_NAME)-y += $(foreach _m,$(INTERNAL_EXTRA_KBUILD_OBJECTS:.o=),$($(_m)-y))
+
+#obj-m += $(INTERNAL_KBUILD_OBJECTS)
+obj-$(CONFIG_DRM_IMG_VOLCANIC) += $(INTERNAL_KBUILD_OBJECTS)
diff --git a/drivers/gpu/drm/img/img-volcanic/config_kernel.h b/drivers/gpu/drm/img/img-volcanic/config_kernel.h
new file mode 100644
index 000000000000..1e73467fd7bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/config_kernel.h
@@ -0,0 +1,324 @@
+#define SUPPORT_RGXRAY_BRIDGE
+#define RGX_FW_FILENAME "rgx.fw"
+#define RGX_SH_FILENAME "rgx.sh"
+#define PVR_BUILD_DIR "eswin_linux"
+#define PVR_BUILD_TYPE "release"
+#define PVRSRV_MODNAME "pvrsrvkm"
+#define PVRHMMU_MODNAME ""
+#define PVRSYNC_MODNAME "pvr_sync"
+#define SUPPORT_RGX 1
+#define DISPLAY_CONTROLLER drm_nulldisp
+#define PVRSRV_MAX_DEVICES 8
+#define PVRSRV_HWPERF_COUNTERS_PERBLK 12
+#define RELEASE
+#define RGX_BVNC_CORE_KM_HEADER "cores/rgxcore_km_30.3.408.101.h"
+#define RGX_BNC_CONFIG_KM_HEADER "configs/rgxconfig_km_30.V.408.101.h"
+#define PVRSRV_ENABLE_XD_MEM
+#define PVRSRV_NEED_PVR_DPF
+#define PVRSRV_NEED_PVR_ASSERT
+#define SUPPORT_LINUX_FDINFO
+#define PVRSRV_TRACE_ROGUE_EVENTS
+#define SUPPORT_PHYSMEM_TEST
+#define PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY 5
+#define PVRSRV_POISON_ON_ALLOC_VALUE 0xd9
+#define PVRSRV_POISON_ON_FREE_VALUE 0x63
+#define RGX_NUM_OS_SUPPORTED 1
+#define RGX_NUM_DRIVERS_SUPPORTED 1
+#define RGX_HCS_DEFAULT_DEADLINE_MS 0xFFFFFFFFU
+#define RGX_FW_HEAP_USES_FIRMWARE_OSID 0
+#define RGX_FW_HEAP_USES_HOST_OSID 1
+#define RGX_FW_HEAP_USES_DEDICATED_OSID 2
+#define RGX_FW_HEAP_OSID_ASSIGNMENT RGX_FW_HEAP_USES_FIRMWARE_OSID
+#define PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION 0
+#define RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION  512
+#define PVRSRV_APPHINT_DRIVERMODE "default"
+#define RGX_FW_HEAP_SHIFT 25
+#define PVRSRV_APPHINT_GUESTFWHEAPSTRIDE (1 << RGX_FW_HEAP_SHIFT)
+#define PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN IMG_FALSE
+#define RGX_VZ_CONNECTION_TIMEOUT_US 60000000
+#define PVRSRV_VZ_BYPASS_HMMU
+#define GPUVIRT_VALIDATION_NUM_OS 8
+#define PVRSRV_ENABLE_CCCB_GROW
+#define SUPPORT_POWMON_COMPONENT
+#define PVR_POWER_ACTOR_MEASUREMENT_PERIOD_MS 10U
+#define PVR_POWER_MONITOR_HWPERF 1
+#define SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+#define PVR_LDM_PLATFORM_PRE_REGISTERED
+#define PVR_LDM_DRIVER_REGISTRATION_NAME "pvrsrvkm"
+#define PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN 256
+#define ION_DEFAULT_HEAP_NAME "ion_system_heap"
+#define ION_DEFAULT_HEAP_ID_MASK (1 << ION_HEAP_TYPE_SYSTEM)
+#define SUPPORT_FASTRENDER_DM
+#define ASSERTONHWRTRIGGER_DEFAULT_ENABLED
+#define PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT APPHNT_BLDVAR_DBGDUMPLIMIT
+#define PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG IMG_FALSE
+#define PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE 0x4000
+#define PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE 786432
+#define PVRSRV_APPHINT_ENABLESIGNATURECHECKS APPHNT_BLDVAR_ENABLESIGNATURECHECKS
+#define PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE RGXFW_SIG_BUFFER_SIZE_MIN
+#define PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING IMG_FALSE
+#define PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG
+#define PVRSRV_APPHINT_VALIDATEIRQ 0
+#define PVRSRV_APPHINT_DISABLECLOCKGATING 0
+#define PVRSRV_APPHINT_DISABLEDMOVERLAP 0
+#define PVRSRV_APPHINT_FABRICCOHERENCYOVERRIDE 1
+#define PVRSRV_APPHINT_ENABLEDMKILLINGRANDMODE 0
+#define PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL
+#define PVRSRV_APPHINT_ENABLERDPOWERISLAND RGX_RD_POWER_ISLAND_DEFAULT
+#define PVRSRV_APPHINT_ENABLESPUCLOCKGATING IMG_FALSE
+#define PVRSRV_APPHINT_HWVALENABLESPUPOWERMASKCHANGE 0
+#define PVRSRV_APPHINT_HWVALAVAILABLESPUMASK 0xFFFFFFFF
+#define PVRSRV_APPHINT_HWVALAVAILABLERACMASK 0xFFFFFFFF
+#define PVRSRV_APPHINT_HWVALTPUF20BILPRECISION 1
+#define PVRSRV_APPHINT_KILLINGCTL 0
+#define PVRSRV_APPHINT_CDMTDM_KILLINGCTL 0
+#define PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH 0
+#define PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH 0
+#define PVRSRV_APPHINT_FIRMWAREPERF FW_PERF_CONF_NONE
+#define PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN
+#define PVRSRV_APPHINT_HWPERFDISABLECOUNTERFILTER 0
+#define PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB 2048
+#define PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB 2048
+#define PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS 50
+#define PVRSRV_APPHINT_NEWFILTERINGMODE IMG_FALSE
+#define PVRSRV_APPHINT_TRUNCATEMODE 0
+#define PVRSRV_APPHINT_EMUMAXFREQ 0
+#define PVRSRV_APPHINT_GPIOVALIDATIONMODE 0
+#define PVRSRV_APPHINT_RGXBVNC ""
+#define PVRSRV_APPHINT_RISCVDMITEST 0
+#define PVRSRV_APPHINT_RCEDISABLEMASK 0x00000000
+#define PVRSRV_APPHINT_PCGPKTDROPTHRESH  0x0000000b
+#define PVRSRV_APPHINT_RAYSLCMMUAUTOCACHEOPS  0x0000000f
+#define RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US 2000000
+#define PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE 1
+#define PVRSRV_APPHINT_VALIDATESOCUSCTIMERS 0
+#define PVRSRV_APPHINT_SHGEOMPIPEMASK_OVERRIDE 0
+#define PVRSRV_APPHINT_CLEANUPTHREADPRIORITY 5
+#define PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY 0
+#define PVRSRV_APPHINT_CACHEOPTHREADPRIORITY 1
+#define PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 11
+#define PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES 10000
+#define PVRSRV_APPHINT_ASSERTONHWRTRIGGER IMG_FALSE
+#define PVRSRV_APPHINT_ASSERTOUTOFMEMORY IMG_FALSE
+#define PVRSRV_APPHINT_CHECKMLIST APPHNT_BLDVAR_DEBUG
+#define PVRSRV_APPHINT_DISABLEFEDLOGGING IMG_FALSE
+#define PVRSRV_APPHINT_KCCB_SIZE_LOG2 10
+#define PVRSRV_APPHINT_ENABLEAPM RGX_ACTIVEPM_DEFAULT
+#define PVRSRV_APPHINT_ENABLEHTBLOGGROUP 0
+#define PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING RGXFWIF_INICFG_OS_ICS_DM_APPHINT
+#define PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC 40000
+#define PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD 90
+#define PVRSRV_APPHINT_ENABLELOGGROUP RGXFWIF_LOG_TYPE_NONE
+#define PVRSRV_APPHINT_FIRMWARELOGTYPE 0
+#define PVRSRV_APPHINT_FWTRACEBUFSIZEINDWORDS RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS
+#define PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE 1
+#define PVRSRV_APPHINT_FBCDCVERSIONOVERRIDE 0
+#define PVRSRV_APPHINT_HTBOPERATIONMODE HTB_OPMODE_DROPOLDEST
+#define PVRSRV_APPHINT_HTBUFFERSIZE 64
+#define PVRSRV_APPHINT_ENABLEFTRACEGPU IMG_FALSE
+#define PVRSRV_APPHINT_HWPERFFWFILTER 0
+#define PVRSRV_APPHINT_HWPERFHOSTFILTER 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_SERVICES 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_EGL 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGLES 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENCL 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_VULKAN 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGL 0
+#define PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE 2
+#define PVRSRV_APPHINT_ENABLEFWPOISONONFREE IMG_FALSE
+#define PVRSRV_APPHINT_FWPOISONONFREEVALUE 0xBD
+#define PVRSRV_APPHINT_ZEROFREELIST IMG_FALSE
+#define PVRSRV_APPHINT_GPUUNITSPOWERCHANGE IMG_FALSE
+#define PVRSRV_APPHINT_DISABLEPDUMPPANIC IMG_FALSE
+#define PVRSRV_APPHINT_CACHEOPCONFIG 0
+#define PVRSRV_APPHINT_CACHEOPUMKMHRESHOLDSIZE 0
+#define PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC IMG_FALSE
+#define PVRSRV_APPHINT_PHYSMEMTESTPASSES APPHNT_PHYSMEMTEST_ENABLE
+#define PVRSRV_APPHINT_MMUVAL_PARITYFLIPPMR "unset"
+#define PVRSRV_APPHINT_TESTSLRINTERVAL 0
+#define PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2 8
+#define PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2 7
+#define PVRSRV_APPHINT_CLKCTRL0 0xAAAAAAAAAAAAAAAAULL
+#define PVRSRV_APPHINT_CLKCTRL1 0xAAAAAAAAAAAAAAAAULL
+#define PVRSRV_APPHINT_CLKCTRL2 0xAAAAAAAAULL
+#define SUPPORT_SOC_TIMER
+#define PDVFS_COM_HOST 1
+#define PDVFS_COM_AP 2
+#define PDVFS_COM_PMC 3
+#define PDVFS_COM_IMG_CLKDIV 4
+#define PDVFS_COM_SYSREG 5
+#define PDVFS_COM PDVFS_COM_HOST
+#define SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY
+#define PVR_GPIO_MODE_GENERAL 1
+#define PVR_GPIO_MODE PVR_GPIO_MODE_GENERAL
+#define PVRSRV_ENABLE_PROCESS_STATS
+#define PVR_ANNOTATION_MAX_LEN 63
+#define PVRSRV_DEVICE_INIT_MODE PVRSRV_LINUX_DEV_INIT_ON_CONNECT
+#define SUPPORT_DI_BRG_IMPL
+#define PVR_LINUX_PHYSMEM_MAX_POOL_PAGES 10240
+#define PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES 20480
+#define PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES
+#define PVR_DIRTY_BYTES_FLUSH_THRESHOLD 524288
+#define PVR_LINUX_HIGHORDER_ALLOCATION_THRESHOLD 256
+#define PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM 2
+#define PVR_LINUX_KMALLOC_ALLOCATION_THRESHOLD 16384
+#define SUPPORT_PMR_DEFERRED_FREE
+#define SUPPORT_PMR_PAGES_DEFERRED_FREE
+#define SUPPORT_MMU_DEFERRED_FREE
+#define SUPPORT_WRAP_EXTMEM
+#define PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC 1
+#define SUPPORT_NATIVE_FENCE_SYNC
+#define PVRSRV_STALLED_CCB_ACTION
+#define PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP
+#define UPDATE_FENCE_CHECKPOINT_COUNT 1
+#define RGX_VZ_CONNECTION_COOLDOWN_PERIOD 0
+#define PVR_DRM_NAME "pvr"
+#define DEVICE_MEMSETCPY_ALIGN_IN_BYTES 16
+#define RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS 0
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA 15
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D 16
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ3D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ2D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_CDM 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TA 16
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_3D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_KICKSYNC 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TDM 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_RDM 15
+#define RGX_DRIVERID_0_DEFAULT_PRIORITY (1 - 0)
+#define RGX_DRIVERID_1_DEFAULT_PRIORITY (1 - 1)
+#define RGX_DRIVERID_2_DEFAULT_PRIORITY (1 - 2)
+#define RGX_DRIVERID_3_DEFAULT_PRIORITY (1 - 3)
+#define RGX_DRIVERID_4_DEFAULT_PRIORITY (1 - 4)
+#define RGX_DRIVERID_5_DEFAULT_PRIORITY (1 - 5)
+#define RGX_DRIVERID_6_DEFAULT_PRIORITY (1 - 6)
+#define RGX_DRIVERID_7_DEFAULT_PRIORITY (1 - 7)
+#define RGX_DRIVERID_8_DEFAULT_PRIORITY (1 - 8)
+#define RGX_DRIVERID_9_DEFAULT_PRIORITY (1 - 9)
+#define RGX_DRIVERID_10_DEFAULT_PRIORITY (1 - 10)
+#define RGX_DRIVERID_11_DEFAULT_PRIORITY (1 - 11)
+#define RGX_DRIVERID_12_DEFAULT_PRIORITY (1 - 12)
+#define RGX_DRIVERID_13_DEFAULT_PRIORITY (1 - 13)
+#define RGX_DRIVERID_14_DEFAULT_PRIORITY (1 - 14)
+#define RGX_DRIVERID_15_DEFAULT_PRIORITY (1 - 15)
+#define RGX_DRIVERID_16_DEFAULT_PRIORITY (1 - 16)
+#define RGX_DRIVERID_17_DEFAULT_PRIORITY (1 - 17)
+#define RGX_DRIVERID_18_DEFAULT_PRIORITY (1 - 18)
+#define RGX_DRIVERID_19_DEFAULT_PRIORITY (1 - 19)
+#define RGX_DRIVERID_20_DEFAULT_PRIORITY (1 - 20)
+#define RGX_DRIVERID_21_DEFAULT_PRIORITY (1 - 21)
+#define RGX_DRIVERID_22_DEFAULT_PRIORITY (1 - 22)
+#define RGX_DRIVERID_23_DEFAULT_PRIORITY (1 - 23)
+#define RGX_DRIVERID_24_DEFAULT_PRIORITY (1 - 24)
+#define RGX_DRIVERID_25_DEFAULT_PRIORITY (1 - 25)
+#define RGX_DRIVERID_26_DEFAULT_PRIORITY (1 - 26)
+#define RGX_DRIVERID_27_DEFAULT_PRIORITY (1 - 27)
+#define RGX_DRIVERID_28_DEFAULT_PRIORITY (1 - 28)
+#define RGX_DRIVERID_29_DEFAULT_PRIORITY (1 - 29)
+#define RGX_DRIVERID_30_DEFAULT_PRIORITY (1 - 30)
+#define RGX_DRIVERID_31_DEFAULT_PRIORITY (1 - 31)
+#define RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_1_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_2_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_3_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_4_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_5_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_6_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_7_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_8_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_9_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_10_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_11_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_12_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_13_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_14_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_15_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_16_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_17_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_18_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_19_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_20_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_21_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_22_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_23_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_24_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_25_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_26_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_27_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_28_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_29_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_30_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_31_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_0_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_1_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_2_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_3_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_4_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_5_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_6_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_7_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_8_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_9_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_10_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_11_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_12_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_13_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_14_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_15_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_16_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_17_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_18_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_19_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_20_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_21_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_22_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_23_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_24_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_25_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_26_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_27_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_28_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_29_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_30_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_31_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL 0
+#define RGX_DRIVER_DEFAULT_TIME_SLICES_SUM (((((((((((((((((((((((((((((((( + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0) + 0)
+#define DRIVER0_SECURITY_SUPPORT 0
+#define DRIVER1_SECURITY_SUPPORT 0
+#define DRIVER2_SECURITY_SUPPORT 0
+#define DRIVER3_SECURITY_SUPPORT 0
+#define DRIVER4_SECURITY_SUPPORT 0
+#define DRIVER5_SECURITY_SUPPORT 0
+#define DRIVER6_SECURITY_SUPPORT 0
+#define DRIVER7_SECURITY_SUPPORT 0
+#define DRIVER8_SECURITY_SUPPORT 0
+#define DRIVER9_SECURITY_SUPPORT 0
+#define DRIVER10_SECURITY_SUPPORT 0
+#define DRIVER11_SECURITY_SUPPORT 0
+#define DRIVER12_SECURITY_SUPPORT 0
+#define DRIVER13_SECURITY_SUPPORT 0
+#define DRIVER14_SECURITY_SUPPORT 0
+#define DRIVER15_SECURITY_SUPPORT 0
+#define DRIVER16_SECURITY_SUPPORT 0
+#define DRIVER17_SECURITY_SUPPORT 0
+#define DRIVER18_SECURITY_SUPPORT 0
+#define DRIVER19_SECURITY_SUPPORT 0
+#define DRIVER20_SECURITY_SUPPORT 0
+#define DRIVER21_SECURITY_SUPPORT 0
+#define DRIVER22_SECURITY_SUPPORT 0
+#define DRIVER23_SECURITY_SUPPORT 0
+#define DRIVER24_SECURITY_SUPPORT 0
+#define DRIVER25_SECURITY_SUPPORT 0
+#define DRIVER26_SECURITY_SUPPORT 0
+#define DRIVER27_SECURITY_SUPPORT 0
+#define DRIVER28_SECURITY_SUPPORT 0
+#define DRIVER29_SECURITY_SUPPORT 0
+#define DRIVER30_SECURITY_SUPPORT 0
+#define DRIVER31_SECURITY_SUPPORT 0
+#define SUPPORT_BUFFER_SYNC 1
diff --git a/drivers/gpu/drm/img/img-volcanic/config_kernel.mk b/drivers/gpu/drm/img/img-volcanic/config_kernel.mk
new file mode 100644
index 000000000000..a1ca43790015
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/config_kernel.mk
@@ -0,0 +1,55 @@
+override HOST_PRIMARY_ARCH := host_x86_64
+override HOST_32BIT_ARCH := host_i386
+override HOST_FORCE_32BIT := -m32
+override HOST_ALL_ARCH := host_x86_64 host_i386
+override TARGET_PRIMARY_ARCH := target_riscv64
+override TARGET_SECONDARY_ARCH :=
+override TARGET_ALL_ARCH := target_riscv64
+override TARGET_FORCE_32BIT :=
+override PVR_ARCH := volcanic
+override PVRSRV_DIR := services
+override METAG_VERSION_NEEDED := 2.8.1.0.3
+override RISCV_VERSION_NEEDED := 1.7.1
+override KERNELDIR := /lib/modules/6.6.21-eic7x/build/
+override KERNEL_ID := 6.6.21-eic7x
+override PVRSRV_MODULE_BASEDIR := /lib/modules/6.6.21-eic7x/extra/
+override KERNEL_COMPONENTS := srvkm drm_nulldisp drm_nulldisp
+override KERNEL_CROSS_COMPILE := riscv64-unknown-linux-gnu-
+override WINDOW_SYSTEM := xorg
+override PVRSRV_MODNAME := pvrsrvkm
+override PVRHMMU_MODNAME :=
+override PVR_BUILD_DIR := eswin_linux
+override PVR_BUILD_TYPE := release
+override SUPPORT_RGX := 1
+override DISPLAY_CONTROLLER := drm_nulldisp
+override PVR_SYSTEM := eswin_cpu
+override PVR_LOADER :=
+override BUILD := release
+override SORT_BRIDGE_STRUCTS := 1
+override DEBUGLINK := 1
+override RGX_BNC := 30.V.408.101
+override PVRSRV_ENABLE_XD_MEM := 1
+override SUPPORT_LINUX_FDINFO := 1
+override PVRSRV_TRACE_ROGUE_EVENTS := 1
+override SUPPORT_PHYSMEM_TEST := 1
+override RGX_NUM_DRIVERS_SUPPORTED := 1
+override RGX_FW_HEAP_OSID_ASSIGNMENT := RGX_FW_HEAP_USES_FIRMWARE_OSID
+override VMM_TYPE := stub
+override SUPPORT_POWMON_COMPONENT := 1
+override RGX_SECONDARY_OS_CLOCK_SOURCE := mono 
+override PDVFS_COM_HOST := 1
+override PDVFS_COM_AP := 2
+override PDVFS_COM_PMC := 3
+override PDVFS_COM_IMG_CLKDIV := 4
+override PDVFS_COM_SYSREG := 5
+override PDVFS_COM := PDVFS_COM_HOST
+override SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY := 1
+override PVR_GPIO_MODE_GENERAL := 1
+override PVR_GPIO_MODE := PVR_GPIO_MODE_GENERAL
+override PVR_HANDLE_BACKEND := idr
+override SUPPORT_DMABUF_BRIDGE := 1
+override SUPPORT_DI_BRG_IMPL := 1
+override SUPPORT_WRAP_EXTMEM := 1
+override SUPPORT_NATIVE_FENCE_SYNC := 1
+override SUPPORT_DMA_FENCE := 1
+override SUPPORT_BUFFER_SYNC := 1
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_bridge.h
new file mode 100644
index 000000000000..1dec13fccce7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_CACHE_BRIDGE_H
+#define CLIENT_CACHE_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_cache_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp);
+
+#endif /* CLIENT_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_direct_bridge.c
new file mode 100644
index 000000000000..9691bae9352f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/client_cache_direct_bridge.c
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for cache
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_cache_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "cache_ops.h"
+
+#include "cache_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PMR **psPMRInt;
+
+	psPMRInt = (PMR **) phPMR;
+
+	eError =
+	    CacheOpQueue(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			 ui32NumCacheOps,
+			 psPMRInt, pui64Address, puiOffset, puiSize, piuCacheOp, ui32OpTimeline);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = CacheOpValExec(psPMRInt, ui64Address, uiOffset, uiSize, iuCacheOp);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    CacheOpLog(psPMRInt,
+		       ui64Address, uiOffset, uiSize, i64StartTime, i64EndTime, iuCacheOp);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/common_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/common_cache_bridge.h
new file mode 100644
index 000000000000..cc848753f7fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/common_cache_bridge.h
@@ -0,0 +1,126 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CACHE_BRIDGE_H
+#define COMMON_CACHE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_CACHE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE			PVRSRV_BRIDGE_CACHE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPEXEC			PVRSRV_BRIDGE_CACHE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CACHE_CACHEOPLOG			PVRSRV_BRIDGE_CACHE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CACHE_CMD_LAST			(PVRSRV_BRIDGE_CACHE_CMD_FIRST+2)
+
+/*******************************************
+            CacheOpQueue
+ *******************************************/
+
+/* Bridge in structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPQUEUE_TAG
+{
+	PVRSRV_CACHE_OP *piuCacheOp;
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	IMG_UINT32 ui32NumCacheOps;
+	IMG_UINT32 ui32OpTimeline;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPQUEUE;
+
+/* Bridge out structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPQUEUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPQUEUE;
+
+/*******************************************
+            CacheOpExec
+ *******************************************/
+
+/* Bridge in structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPEXEC_TAG
+{
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPEXEC;
+
+/* Bridge out structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPEXEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPEXEC;
+
+/*******************************************
+            CacheOpLog
+ *******************************************/
+
+/* Bridge in structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPLOG_TAG
+{
+	IMG_INT64 i64EndTime;
+	IMG_INT64 i64StartTime;
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPLOG;
+
+/* Bridge out structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPLOG;
+
+#endif /* COMMON_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/server_cache_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/server_cache_bridge.c
new file mode 100644
index 000000000000..b2f5267cc597
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cache_bridge/server_cache_bridge.c
@@ -0,0 +1,452 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "cache_km.h"
+
+#include "common_cache_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(CACHE_BATCH_MAX <= IMG_UINT32_MAX,
+	      "CACHE_BATCH_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeCacheOpQueue(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psCacheOpQueueIN_UI8,
+			 IMG_UINT8 * psCacheOpQueueOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPQUEUE *psCacheOpQueueIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *psCacheOpQueueOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+	PVRSRV_CACHE_OP *iuCacheOpInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) + 0;
+
+	if (unlikely(psCacheOpQueueIN->ui32NumCacheOps > CACHE_BATCH_MAX))
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto CacheOpQueue_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto CacheOpQueue_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psCacheOpQueueIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psCacheOpQueueIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psCacheOpQueueOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *));
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psCacheOpQueueIN->phPMR,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psCacheOpQueueIN->pui64Address,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psCacheOpQueueIN->puiOffset,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psCacheOpQueueIN->puiSize,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		iuCacheOpInt =
+		    (PVRSRV_CACHE_OP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, iuCacheOpInt, (const void __user *)psCacheOpQueueIN->piuCacheOp,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+			/* Look up the address from the handle */
+			psCacheOpQueueOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psCacheOpQueueOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpQueueOUT->eError =
+	    CacheOpQueue(psConnection, OSGetDevNode(psConnection),
+			 psCacheOpQueueIN->ui32NumCacheOps,
+			 psPMRInt,
+			 ui64AddressInt,
+			 uiOffsetInt, uiSizeInt, iuCacheOpInt, psCacheOpQueueIN->ui32OpTimeline);
+
+CacheOpQueue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psCacheOpQueueOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpExec(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psCacheOpExecIN_UI8,
+			IMG_UINT8 * psCacheOpExecOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPEXEC *psCacheOpExecIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPEXEC *psCacheOpExecOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpExecIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpExecOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpExecOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpExec_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpExecOUT->eError =
+	    CacheOpValExec(psPMRInt,
+			   psCacheOpExecIN->ui64Address,
+			   psCacheOpExecIN->uiOffset,
+			   psCacheOpExecIN->uiSize, psCacheOpExecIN->iuCacheOp);
+
+CacheOpExec_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpLog(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psCacheOpLogIN_UI8,
+		       IMG_UINT8 * psCacheOpLogOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPLOG *psCacheOpLogIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPLOG *psCacheOpLogOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpLogIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpLogOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpLogOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpLog_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpLogOUT->eError =
+	    CacheOpLog(psPMRInt,
+		       psCacheOpLogIN->ui64Address,
+		       psCacheOpLogIN->uiOffset,
+		       psCacheOpLogIN->uiSize,
+		       psCacheOpLogIN->i64StartTime,
+		       psCacheOpLogIN->i64EndTime, psCacheOpLogIN->iuCacheOp);
+
+CacheOpLog_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+
+/*
+ * Register all CACHE functions with services
+ */
+PVRSRV_ERROR InitCACHEBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE,
+			      PVRSRVBridgeCacheOpQueue, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPQUEUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPQUEUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC,
+			      PVRSRVBridgeCacheOpExec, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPEXEC),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPEXEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG,
+			      PVRSRVBridgeCacheOpLog, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPLOG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cache functions with services
+ */
+void DeinitCACHEBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/common_cmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/common_cmm_bridge.h
new file mode 100644
index 000000000000..da48de3598be
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/common_cmm_bridge.h
@@ -0,0 +1,114 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CMM_BRIDGE_H
+#define COMMON_CMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_CMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CMM_CMD_LAST			(PVRSRV_BRIDGE_CMM_CMD_FIRST+2)
+
+/*******************************************
+            DevmemIntExportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX;
+
+/* Bridge out structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX;
+
+/*******************************************
+            DevmemIntUnexportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX;
+
+/* Bridge out structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX;
+
+/*******************************************
+            DevmemIntAcquireRemoteCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX;
+
+/* Bridge out structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX;
+
+#endif /* COMMON_CMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/server_cmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/server_cmm_bridge.c
new file mode 100644
index 000000000000..cc658c793d3b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/cmm_bridge/server_cmm_bridge.c
@@ -0,0 +1,414 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_server.h"
+
+#include "common_cmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DevmemIntExportCtxpsContextExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnexportCtx((DEVMEMINT_CTX_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntExportCtx(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntExportCtxIN_UI8,
+			       IMG_UINT8 * psDevmemIntExportCtxOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxOUT_UI8,
+								     0);
+
+	IMG_HANDLE hContext = psDevmemIntExportCtxIN->hContext;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntExportCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX_EXPORT *psContextExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError =
+	    DevmemIntExportCtx(psContextInt, psPMRInt, &psContextExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntExportCtxOUT->
+								    hContextExport,
+								    (void *)psContextExportInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+								    PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntExportCtxpsContextExportIntRelease);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntExportCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntExportCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psContextExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DevmemIntUnexportCtx(psContextExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnexportCtx(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntUnexportCtxIN_UI8,
+				 IMG_UINT8 * psDevmemIntUnexportCtxOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntUnexportCtxIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnexportCtxOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnexportCtxOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnexportCtxIN->hContextExport,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT);
+	if (unlikely((psDevmemIntUnexportCtxOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnexportCtxOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnexportCtx_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnexportCtx_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntAcquireRemoteCtxpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntAcquireRemoteCtx(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxIN_UI8,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevmemIntAcquireRemoteCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntAcquireRemoteCtxOUT->hContext = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    DevmemIntAcquireRemoteCtx(psPMRInt, &psContextInt, &hPrivDataInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntAcquireRemoteCtxOUT->hContext,
+				      (void *)psContextInt, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntAcquireRemoteCtxpsContextIntRelease);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psDevmemIntAcquireRemoteCtxOUT->hPrivData,
+					 (void *)hPrivDataInt, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psDevmemIntAcquireRemoteCtxOUT->hContext);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntAcquireRemoteCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntAcquireRemoteCtxOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psDevmemIntAcquireRemoteCtxOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DevmemIntCtxDestroy(psContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_CMM_BRIDGE */
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+
+/*
+ * Register all CMM functions with services
+ */
+PVRSRV_ERROR InitCMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX,
+			      PVRSRVBridgeDevmemIntExportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX,
+			      PVRSRVBridgeDevmemIntUnexportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX,
+			      PVRSRVBridgeDevmemIntAcquireRemoteCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cmm functions with services
+ */
+void DeinitCMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX);
+
+}
+#else /* EXCLUDE_CMM_BRIDGE */
+/* This bridge is conditional on EXCLUDE_CMM_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitCMMBridge() \
+	PVRSRV_OK
+
+#define DeinitCMMBridge()
+
+#endif /* EXCLUDE_CMM_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/common_dc_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/common_dc_bridge.h
new file mode 100644
index 000000000000..d811fbd4cef1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/common_dc_bridge.h
@@ -0,0 +1,548 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DC_BRIDGE_H
+#define COMMON_DC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_surface.h"
+#include "dc_external.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE			PVRSRV_BRIDGE_DC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DC_DCDEVICERELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DC_DCGETINFO			PVRSRV_BRIDGE_DC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_DC_DCPANELQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_DC_DCFORMATQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_DC_DCDIMQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_DC_DCSETBLANK			PVRSRV_BRIDGE_DC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING			PVRSRV_BRIDGE_DC_CMD_FIRST+10
+#define PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+11
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+12
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+13
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE			PVRSRV_BRIDGE_DC_CMD_FIRST+14
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK			PVRSRV_BRIDGE_DC_CMD_FIRST+15
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY			PVRSRV_BRIDGE_DC_CMD_FIRST+16
+#define PVRSRV_BRIDGE_DC_DCBUFFERALLOC			PVRSRV_BRIDGE_DC_CMD_FIRST+17
+#define PVRSRV_BRIDGE_DC_DCBUFFERIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+18
+#define PVRSRV_BRIDGE_DC_DCBUFFERFREE			PVRSRV_BRIDGE_DC_CMD_FIRST+19
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+20
+#define PVRSRV_BRIDGE_DC_DCBUFFERPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+21
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+22
+#define PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+23
+#define PVRSRV_BRIDGE_DC_DCBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+24
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2			PVRSRV_BRIDGE_DC_CMD_FIRST+25
+#define PVRSRV_BRIDGE_DC_CMD_LAST			(PVRSRV_BRIDGE_DC_CMD_FIRST+25)
+
+/*******************************************
+            DCDevicesQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT;
+
+/* Bridge out structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT;
+
+/*******************************************
+            DCDevicesEnumerate
+ *******************************************/
+
+/* Bridge in structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	IMG_UINT32 ui32DeviceArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE;
+
+/* Bridge out structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE;
+
+/*******************************************
+            DCDeviceAcquire
+ *******************************************/
+
+/* Bridge in structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE_TAG
+{
+	IMG_UINT32 ui32DeviceIndex;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE;
+
+/* Bridge out structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE;
+
+/*******************************************
+            DCDeviceRelease
+ *******************************************/
+
+/* Bridge in structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICERELEASE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICERELEASE;
+
+/* Bridge out structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICERELEASE;
+
+/*******************************************
+            DCGetInfo
+ *******************************************/
+
+/* Bridge in structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_IN_DCGETINFO_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCGETINFO;
+
+/* Bridge out structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DCGETINFO_TAG
+{
+	DC_DISPLAY_INFO sDisplayInfo;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCGETINFO;
+
+/*******************************************
+            DCPanelQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT;
+
+/* Bridge out structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT;
+
+/*******************************************
+            DCPanelQuery
+ *******************************************/
+
+/* Bridge in structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	IMG_UINT32 ui32PanelsArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERY;
+
+/* Bridge out structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERY_TAG
+{
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERY;
+
+/*******************************************
+            DCFormatQuery
+ *******************************************/
+
+/* Bridge in structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCFORMATQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_FORMAT *psFormat;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumFormats;
+} __packed PVRSRV_BRIDGE_IN_DCFORMATQUERY;
+
+/* Bridge out structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCFORMATQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCFORMATQUERY;
+
+/*******************************************
+            DCDimQuery
+ *******************************************/
+
+/* Bridge in structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCDIMQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_DIMS *psDim;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumDims;
+} __packed PVRSRV_BRIDGE_IN_DCDIMQUERY;
+
+/* Bridge out structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCDIMQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDIMQUERY;
+
+/*******************************************
+            DCSetBlank
+ *******************************************/
+
+/* Bridge in structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_IN_DCSETBLANK_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETBLANK;
+
+/* Bridge out structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETBLANK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETBLANK;
+
+/*******************************************
+            DCSetVSyncReporting
+ *******************************************/
+
+/* Bridge in structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING;
+
+/* Bridge out structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING;
+
+/*******************************************
+            DCLastVSyncQuery
+ *******************************************/
+
+/* Bridge in structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY;
+
+/* Bridge out structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY_TAG
+{
+	IMG_INT64 i64Timestamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY;
+
+/*******************************************
+            DCSystemBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE;
+
+/* Bridge out structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE;
+
+/*******************************************
+            DCSystemBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE;
+
+/* Bridge out structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextCreate
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE;
+
+/* Bridge out structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE;
+
+/*******************************************
+            DCDisplayContextConfigureCheck
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/* Bridge out structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/*******************************************
+            DCDisplayContextDestroy
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	IMG_HANDLE hDisplayContext;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY;
+
+/* Bridge out structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY;
+
+/*******************************************
+            DCBufferAlloc
+ *******************************************/
+
+/* Bridge in structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	DC_BUFFER_CREATE_INFO sSurfInfo;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERALLOC;
+
+/* Bridge out structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERALLOC;
+
+/*******************************************
+            DCBufferImport
+ *******************************************/
+
+/* Bridge in structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	IMG_HANDLE *phImport;
+	DC_BUFFER_IMPORT_INFO sSurfAttrib;
+	IMG_UINT32 ui32NumPlanes;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERIMPORT;
+
+/* Bridge out structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT;
+
+/*******************************************
+            DCBufferFree
+ *******************************************/
+
+/* Bridge in structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERFREE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERFREE;
+
+/* Bridge out structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERFREE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERFREE;
+
+/*******************************************
+            DCBufferUnimport
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT;
+
+/* Bridge out structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT;
+
+/*******************************************
+            DCBufferPin
+ *******************************************/
+
+/* Bridge in structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERPIN;
+
+/* Bridge out structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERPIN;
+
+/*******************************************
+            DCBufferUnpin
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNPIN;
+
+/* Bridge out structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN;
+
+/*******************************************
+            DCBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE;
+
+/* Bridge out structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE;
+
+/*******************************************
+            DCBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERRELEASE;
+
+/* Bridge out structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextConfigure2
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	PVRSRV_FENCE hAcquireFence;
+	PVRSRV_TIMELINE hReleaseFenceTimeline;
+	IMG_UINT32 ui32DisplayPeriod;
+	IMG_UINT32 ui32MaxDepth;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2;
+
+/* Bridge out structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hReleaseFence;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2;
+
+#endif /* COMMON_DC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/server_dc_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/server_dc_bridge.c
new file mode 100644
index 000000000000..3b5b9150c2ec
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dc_bridge/server_dc_bridge.c
@@ -0,0 +1,2554 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dc_server.h"
+
+#include "common_dc_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeDCDevicesQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCDevicesQueryCountIN_UI8,
+				IMG_UINT8 * psDCDevicesQueryCountOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDCDevicesQueryCountIN);
+
+	psDCDevicesQueryCountOUT->eError =
+	    DCDevicesQueryCount(&psDCDevicesQueryCountOUT->ui32DeviceCount);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DEVICE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_DEVICE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDevicesEnumerate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDCDevicesEnumerateIN_UI8,
+			       IMG_UINT8 * psDCDevicesEnumerateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *psDCDevicesEnumerateIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *psDCDevicesEnumerateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateOUT_UI8,
+								     0);
+
+	IMG_UINT32 *pui32DeviceIndexInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32)) + 0;
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize > DC_MAX_DEVICE_COUNT)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	psDCDevicesEnumerateOUT->pui32DeviceIndex = psDCDevicesEnumerateIN->pui32DeviceIndex;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDevicesEnumerateIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDevicesEnumerateIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDevicesEnumerate_exit;
+			}
+		}
+	}
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize != 0)
+	{
+		pui32DeviceIndexInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32);
+	}
+
+	psDCDevicesEnumerateOUT->eError =
+	    DCDevicesEnumerate(psConnection, OSGetDevNode(psConnection),
+			       psDCDevicesEnumerateIN->ui32DeviceArraySize,
+			       &psDCDevicesEnumerateOUT->ui32DeviceCount, pui32DeviceIndexInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDevicesEnumerateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDevicesEnumerate_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32DeviceIndexInt) &&
+	    ((psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDevicesEnumerateOUT->pui32DeviceIndex,
+		      pui32DeviceIndexInt,
+		      (psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDevicesEnumerate_exit;
+		}
+	}
+
+DCDevicesEnumerate_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDevicesEnumerateOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDeviceAcquirepsDeviceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDeviceRelease((DC_DEVICE *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceAcquireIN_UI8,
+			    IMG_UINT8 * psDCDeviceAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *psDCDeviceAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *psDCDeviceAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireOUT_UI8, 0);
+
+	DC_DEVICE *psDeviceInt = NULL;
+
+	psDCDeviceAcquireOUT->eError =
+	    DCDeviceAcquire(psConnection, OSGetDevNode(psConnection),
+			    psDCDeviceAcquireIN->ui32DeviceIndex, &psDeviceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDCDeviceAcquireOUT->hDevice,
+								 (void *)psDeviceInt,
+								 PVRSRV_HANDLE_TYPE_DC_DEVICE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _DCDeviceAcquirepsDeviceIntRelease);
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceAcquire_exit:
+
+	if (psDCDeviceAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psDeviceInt)
+		{
+			DCDeviceRelease(psDeviceInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceReleaseIN_UI8,
+			    IMG_UINT8 * psDCDeviceReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICERELEASE *psDCDeviceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *psDCDeviceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDeviceReleaseIN->hDevice,
+					      PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	if (unlikely((psDCDeviceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCDeviceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCGetInfo(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psDCGetInfoIN_UI8,
+		      IMG_UINT8 * psDCGetInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCGETINFO *psDCGetInfoIN =
+	    (PVRSRV_BRIDGE_IN_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCGETINFO *psDCGetInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCGetInfoIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCGetInfoOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCGetInfoOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCGetInfo_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCGetInfoOUT->eError = DCGetInfo(psDeviceInt, &psDCGetInfoOUT->sDisplayInfo);
+
+DCGetInfo_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCPanelQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDCPanelQueryCountIN_UI8,
+			      IMG_UINT8 * psDCPanelQueryCountOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *psDCPanelQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *psDCPanelQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryCountIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryCountOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryCountOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQueryCount_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryCountOUT->eError =
+	    DCPanelQueryCount(psDeviceInt, &psDCPanelQueryCountOUT->ui32NumPanels);
+
+DCPanelQueryCount_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(DC_MAX_PANEL_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PANEL_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCPanelQuery(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCPanelQueryIN_UI8,
+			 IMG_UINT8 * psDCPanelQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERY *psDCPanelQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERY *psDCPanelQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_PANEL_INFO *psPanelInfoInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO)) + 0;
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize > DC_MAX_PANEL_COUNT)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCPanelQuery_exit;
+	}
+
+	psDCPanelQueryOUT->psPanelInfo = psDCPanelQueryIN->psPanelInfo;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCPanelQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCPanelQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCPanelQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCPanelQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCPanelQuery_exit;
+			}
+		}
+	}
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize != 0)
+	{
+		psPanelInfoInt =
+		    (PVRSRV_PANEL_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryOUT->eError =
+	    DCPanelQuery(psDeviceInt,
+			 psDCPanelQueryIN->ui32PanelsArraySize,
+			 &psDCPanelQueryOUT->ui32NumPanels, psPanelInfoInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCPanelQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psPanelInfoInt) &&
+	    ((psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCPanelQueryOUT->psPanelInfo, psPanelInfoInt,
+		      (psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO))) != PVRSRV_OK))
+		{
+			psDCPanelQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCPanelQuery_exit;
+		}
+	}
+
+DCPanelQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCPanelQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_FORMATS <= IMG_UINT32_MAX,
+	      "DC_MAX_FORMATS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCFormatQuery(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCFormatQueryIN_UI8,
+			  IMG_UINT8 * psDCFormatQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCFORMATQUERY *psDCFormatQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCFORMATQUERY *psDCFormatQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCFormatQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_FORMAT *psFormatInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) +
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCFormatQueryIN->ui32NumFormats > DC_MAX_FORMATS))
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCFormatQuery_exit;
+	}
+
+	psDCFormatQueryOUT->pui32Supported = psDCFormatQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCFormatQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCFormatQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCFormatQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCFormatQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCFormatQuery_exit;
+			}
+		}
+	}
+
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		psFormatInt =
+		    (PVRSRV_SURFACE_FORMAT *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT);
+	}
+
+	/* Copy the data over */
+	if (psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psFormatInt, (const void __user *)psDCFormatQueryIN->psFormat,
+		     psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) !=
+		    PVRSRV_OK)
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCFormatQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCFormatQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCFormatQueryOUT->eError =
+	    DCFormatQuery(psDeviceInt,
+			  psDCFormatQueryIN->ui32NumFormats, psFormatInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCFormatQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCFormatQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+
+DCFormatQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCFormatQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DIMENSIONS <= IMG_UINT32_MAX,
+	      "DC_MAX_DIMENSIONS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDimQuery(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCDimQueryIN_UI8,
+		       IMG_UINT8 * psDCDimQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDIMQUERY *psDCDimQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDIMQUERY *psDCDimQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDimQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_DIMS *psDimInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) +
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCDimQueryIN->ui32NumDims > DC_MAX_DIMENSIONS))
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDimQuery_exit;
+	}
+
+	psDCDimQueryOUT->pui32Supported = psDCDimQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDimQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDimQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDimQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDimQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDimQuery_exit;
+			}
+		}
+	}
+
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		psDimInt =
+		    (PVRSRV_SURFACE_DIMS *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS);
+	}
+
+	/* Copy the data over */
+	if (psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psDimInt, (const void __user *)psDCDimQueryIN->psDim,
+		     psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) != PVRSRV_OK)
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDimQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDimQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDimQueryOUT->eError =
+	    DCDimQuery(psDeviceInt, psDCDimQueryIN->ui32NumDims, psDimInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCDimQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDimQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+
+DCDimQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDimQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetBlank(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCSetBlankIN_UI8,
+		       IMG_UINT8 * psDCSetBlankOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETBLANK *psDCSetBlankIN =
+	    (PVRSRV_BRIDGE_IN_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSETBLANK *psDCSetBlankOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSetBlankIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetBlankOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetBlankOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetBlank_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetBlankOUT->eError = DCSetBlank(psDeviceInt, psDCSetBlankIN->bEnabled);
+
+DCSetBlank_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetVSyncReporting(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCSetVSyncReportingIN_UI8,
+				IMG_UINT8 * psDCSetVSyncReportingOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *psDCSetVSyncReportingIN =
+	    (PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *psDCSetVSyncReportingOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevice = psDCSetVSyncReportingIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetVSyncReportingOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetVSyncReportingOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetVSyncReporting_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetVSyncReportingOUT->eError =
+	    DCSetVSyncReporting(psDeviceInt, psDCSetVSyncReportingIN->bEnabled);
+
+DCSetVSyncReporting_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCLastVSyncQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCLastVSyncQueryIN_UI8,
+			     IMG_UINT8 * psDCLastVSyncQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *psDCLastVSyncQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *psDCLastVSyncQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCLastVSyncQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCLastVSyncQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCLastVSyncQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCLastVSyncQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCLastVSyncQueryOUT->eError =
+	    DCLastVSyncQuery(psDeviceInt, &psDCLastVSyncQueryOUT->i64Timestamp);
+
+DCLastVSyncQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCSystemBufferAcquirepsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCSystemBufferRelease((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferAcquireIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferAcquireOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSystemBufferAcquireIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSystemBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError =
+	    DCSystemBufferAcquire(psDeviceInt,
+				  &psDCSystemBufferAcquireOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDCSystemBufferAcquireOUT->
+								       hBuffer, (void *)psBufferInt,
+								       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DCSystemBufferAcquirepsBufferIntRelease);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCSystemBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCSystemBufferRelease(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferReleaseIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCSystemBufferReleaseIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCSystemBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCSystemBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDisplayContextCreatepsDisplayContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDisplayContextDestroy((DC_DISPLAY_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextCreate(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDCDisplayContextCreateIN_UI8,
+				   IMG_UINT8 * psDCDisplayContextCreateOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDisplayContextCreateIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError =
+	    DCDisplayContextCreate(psDeviceInt, &psDisplayContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDCDisplayContextCreateOUT->
+									hDisplayContext,
+									(void *)psDisplayContextInt,
+									PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DCDisplayContextCreatepsDisplayContextIntRelease);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCDisplayContextCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDisplayContextInt)
+		{
+			DCDisplayContextDestroy(psDisplayContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigureCheck(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckIN_UI8,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigureCheckIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) +
+	    0;
+
+	if (unlikely(psDCDisplayContextConfigureCheckIN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigureCheckOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigureCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigureCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigureCheckOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) >
+	    0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->psSurfInfo,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+			       sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->phBuffers,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigureCheckOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    DCDisplayContextConfigureCheck(psDisplayContextInt,
+					   psDCDisplayContextConfigureCheckIN->ui32PipeCount,
+					   psSurfInfoInt, psBuffersInt);
+
+DCDisplayContextConfigureCheck_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigureCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDCDisplayContextDestroyIN_UI8,
+				    IMG_UINT8 * psDCDisplayContextDestroyOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDisplayContextDestroyIN->
+					      hDisplayContext,
+					      PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	if (unlikely
+	    ((psDCDisplayContextDestroyOUT->eError != PVRSRV_OK)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCDisplayContextDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAllocpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAlloc(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferAllocIN_UI8,
+			  IMG_UINT8 * psDCBufferAllocOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERALLOC *psDCBufferAllocIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *psDCBufferAllocOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferAllocIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAllocOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError =
+	    DCBufferAlloc(psDisplayContextInt,
+			  &psDCBufferAllocIN->sSurfInfo,
+			  &psDCBufferAllocOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psDCBufferAllocOUT->hBuffer,
+							       (void *)psBufferInt,
+							       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+							       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							       (PFN_HANDLE_RELEASE) &
+							       _DCBufferAllocpsBufferIntRelease);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferAlloc_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAllocOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferImportpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static_assert(DC_MAX_PLANES <= IMG_UINT32_MAX,
+	      "DC_MAX_PLANES must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCBufferImport(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psDCBufferImportIN_UI8,
+			   IMG_UINT8 * psDCBufferImportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *psDCBufferImportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *psDCBufferImportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferImportIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PMR **psImportInt = NULL;
+	IMG_HANDLE *hImportInt2 = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCBufferImportIN->ui32NumPlanes > DC_MAX_PLANES))
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCBufferImport_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCBufferImport_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCBufferImportIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCBufferImportIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCBufferImportOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+
+	if (psDCBufferImportIN->ui32NumPlanes != 0)
+	{
+		psImportInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psImportInt, 0, psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *));
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *);
+		hImportInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hImportInt2, (const void __user *)psDCBufferImportIN->phImport,
+		     psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDCBufferImportOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCBufferImport_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferImportOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+			/* Look up the address from the handle */
+			psDCBufferImportOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psImportInt[i],
+						       hImportInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError =
+	    DCBufferImport(psDisplayContextInt,
+			   psDCBufferImportIN->ui32NumPlanes,
+			   psImportInt, &psDCBufferImportIN->sSurfAttrib, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferImport_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								&psDCBufferImportOUT->hBuffer,
+								(void *)psBufferInt,
+								PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								(PFN_HANDLE_RELEASE) &
+								_DCBufferImportpsBufferIntRelease);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferImport_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hImportInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psImportInt && psImportInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hImportInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferImportOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCBufferImportOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferFree(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCBufferFreeIN_UI8,
+			 IMG_UINT8 * psDCBufferFreeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERFREE *psDCBufferFreeIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERFREE *psDCBufferFreeOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferFreeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferFreeIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferFreeOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferFreeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferFree_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferFree_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnimport(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCBufferUnimportIN_UI8,
+			     IMG_UINT8 * psDCBufferUnimportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *psDCBufferUnimportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *psDCBufferUnimportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnimportOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnimportIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferUnimportOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnimportOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnimport_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnimport_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferPinhPinHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferUnpin((DC_PIN_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferPin(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDCBufferPinIN_UI8,
+			IMG_UINT8 * psDCBufferPinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERPIN *psDCBufferPinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERPIN *psDCBufferPinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferPinIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	DC_PIN_HANDLE hPinHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferPinOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = DCBufferPin(psBufferInt, &hPinHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferPin_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							     &psDCBufferPinOUT->hPinHandle,
+							     (void *)hPinHandleInt,
+							     PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE,
+							     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							     (PFN_HANDLE_RELEASE) &
+							     _DCBufferPinhPinHandleIntRelease);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferPin_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferPinOUT->eError != PVRSRV_OK)
+	{
+		if (hPinHandleInt)
+		{
+			DCBufferUnpin(hPinHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnpin(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferUnpinIN_UI8,
+			  IMG_UINT8 * psDCBufferUnpinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *psDCBufferUnpinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *psDCBufferUnpinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnpinOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnpinIN->hPinHandle,
+					      PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE);
+	if (unlikely((psDCBufferUnpinOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnpinOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnpin_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnpin_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAcquirepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferRelease((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferAcquireIN_UI8,
+			    IMG_UINT8 * psDCBufferAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *psDCBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *psDCBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferAcquireIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAcquireOUT->eError = DCBufferAcquire(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psDCBufferAcquireOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _DCBufferAcquirepsExtMemIntRelease);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferRelease(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferReleaseIN_UI8,
+			    IMG_UINT8 * psDCBufferReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *psDCBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *psDCBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psDCBufferReleaseIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psDCBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferRelease_exit:
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigure2(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psDCDisplayContextConfigure2IN_UI8,
+				       IMG_UINT8 * psDCDisplayContextConfigure2OUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2IN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2OUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2OUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigure2IN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCDisplayContextConfigure2IN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigure2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigure2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigure2OUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigure2IN->psSurfInfo,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigure2IN->phBuffers,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigure2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigure2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigure2OUT->eError =
+	    DCDisplayContextConfigure(psDisplayContextInt,
+				      psDCDisplayContextConfigure2IN->ui32PipeCount,
+				      psSurfInfoInt,
+				      psBuffersInt,
+				      psDCDisplayContextConfigure2IN->ui32DisplayPeriod,
+				      psDCDisplayContextConfigure2IN->ui32MaxDepth,
+				      psDCDisplayContextConfigure2IN->hAcquireFence,
+				      psDCDisplayContextConfigure2IN->hReleaseFenceTimeline,
+				      &psDCDisplayContextConfigure2OUT->hReleaseFence);
+
+DCDisplayContextConfigure2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigure2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDCBridge(void);
+void DeinitDCBridge(void);
+
+/*
+ * Register all DC functions with services
+ */
+PVRSRV_ERROR InitDCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT,
+			      PVRSRVBridgeDCDevicesQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE,
+			      PVRSRVBridgeDCDevicesEnumerate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE,
+			      PVRSRVBridgeDCDeviceAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE,
+			      PVRSRVBridgeDCDeviceRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO, PVRSRVBridgeDCGetInfo,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT,
+			      PVRSRVBridgeDCPanelQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY,
+			      PVRSRVBridgeDCPanelQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY,
+			      PVRSRVBridgeDCFormatQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY, PVRSRVBridgeDCDimQuery,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK, PVRSRVBridgeDCSetBlank,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING,
+			      PVRSRVBridgeDCSetVSyncReporting, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY,
+			      PVRSRVBridgeDCLastVSyncQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE,
+			      PVRSRVBridgeDCSystemBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE,
+			      PVRSRVBridgeDCSystemBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE,
+			      PVRSRVBridgeDCDisplayContextCreate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK,
+			      PVRSRVBridgeDCDisplayContextConfigureCheck, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY,
+			      PVRSRVBridgeDCDisplayContextDestroy, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC,
+			      PVRSRVBridgeDCBufferAlloc, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT,
+			      PVRSRVBridgeDCBufferImport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE,
+			      PVRSRVBridgeDCBufferFree, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT,
+			      PVRSRVBridgeDCBufferUnimport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN,
+			      PVRSRVBridgeDCBufferPin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN,
+			      PVRSRVBridgeDCBufferUnpin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE,
+			      PVRSRVBridgeDCBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE,
+			      PVRSRVBridgeDCBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2,
+			      PVRSRVBridgeDCDisplayContextConfigure2, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dc functions with services
+ */
+void DeinitDCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_bridge.h
new file mode 100644
index 000000000000..bfa6bfb9c037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_bridge.h
@@ -0,0 +1,111 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+#define CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_devicememhistory_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut);
+
+#endif /* CLIENT_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
new file mode 100644
index 000000000000..a593a7e2cc7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
@@ -0,0 +1,195 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for devicememhistory
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_devicememhistory_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  uiOffset,
+				  sDevVAddr,
+				  uiSize,
+				  puiText,
+				  ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    uiOffset,
+				    sDevVAddr,
+				    uiSize,
+				    puiText,
+				    ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryMapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					sBaseDevVAddr,
+					ui32ui32StartPage,
+					ui32NumPages,
+					uiAllocSize,
+					puiText,
+					ui32Log2PageSize,
+					ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryUnmapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  sBaseDevVAddr,
+					  ui32ui32StartPage,
+					  ui32NumPages,
+					  uiAllocSize,
+					  puiText,
+					  ui32Log2PageSize,
+					  ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   uiOffset,
+					   sDevVAddr,
+					   uiSize,
+					   puiText,
+					   ui32Log2PageSize,
+					   ui32AllocPageCount,
+					   pui32AllocPageIndices,
+					   ui32FreePageCount,
+					   pui32FreePageIndices,
+					   ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/common_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/common_devicememhistory_bridge.h
new file mode 100644
index 000000000000..800f98dd2836
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/common_devicememhistory_bridge.h
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DEVICEMEMHISTORY_BRIDGE_H
+#define COMMON_DEVICEMEMHISTORY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST			(PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4)
+
+/*******************************************
+            DevicememHistoryMap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP;
+
+/* Bridge out structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP;
+
+/*******************************************
+            DevicememHistoryUnmap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP;
+
+/* Bridge out structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP;
+
+/*******************************************
+            DevicememHistoryMapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE;
+
+/*******************************************
+            DevicememHistoryUnmapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/*******************************************
+            DevicememHistorySparseChange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE;
+
+/* Bridge out structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE;
+
+#endif /* COMMON_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/server_devicememhistory_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/server_devicememhistory_bridge.c
new file mode 100644
index 000000000000..72c6a86a0ec8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/devicememhistory_bridge/server_devicememhistory_bridge.c
@@ -0,0 +1,835 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+#include "common_devicememhistory_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMap(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevicememHistoryMapIN_UI8,
+				IMG_UINT8 * psDevicememHistoryMapOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *psDevicememHistoryMapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *psDevicememHistoryMapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryMapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryMapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryMapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryMap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryMapOUT->eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  psDevicememHistoryMapIN->uiOffset,
+				  psDevicememHistoryMapIN->sDevVAddr,
+				  psDevicememHistoryMapIN->uiSize,
+				  uiTextInt,
+				  psDevicememHistoryMapIN->ui32Log2PageSize,
+				  psDevicememHistoryMapIN->ui32AllocationIndex,
+				  &psDevicememHistoryMapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmap(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevicememHistoryUnmapIN_UI8,
+				  IMG_UINT8 * psDevicememHistoryUnmapOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryUnmapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryUnmapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryUnmapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryUnmapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryUnmap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryUnmapOUT->eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    psDevicememHistoryUnmapIN->uiOffset,
+				    psDevicememHistoryUnmapIN->sDevVAddr,
+				    psDevicememHistoryUnmapIN->uiSize,
+				    uiTextInt,
+				    psDevicememHistoryUnmapIN->ui32Log2PageSize,
+				    psDevicememHistoryUnmapIN->ui32AllocationIndex,
+				    &psDevicememHistoryUnmapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryUnmap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMapVRange(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeIN_UI8,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryMapVRangeOUT->eError =
+	    DevicememHistoryMapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					psDevicememHistoryMapVRangeIN->sBaseDevVAddr,
+					psDevicememHistoryMapVRangeIN->ui32ui32StartPage,
+					psDevicememHistoryMapVRangeIN->ui32NumPages,
+					psDevicememHistoryMapVRangeIN->uiAllocSize,
+					uiTextInt,
+					psDevicememHistoryMapVRangeIN->ui32Log2PageSize,
+					psDevicememHistoryMapVRangeIN->ui32AllocationIndex,
+					&psDevicememHistoryMapVRangeOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmapVRange(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeIN_UI8,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistoryUnmapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapVRangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryUnmapVRangeOUT->eError =
+	    DevicememHistoryUnmapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					  psDevicememHistoryUnmapVRangeIN->sBaseDevVAddr,
+					  psDevicememHistoryUnmapVRangeIN->ui32ui32StartPage,
+					  psDevicememHistoryUnmapVRangeIN->ui32NumPages,
+					  psDevicememHistoryUnmapVRangeIN->uiAllocSize,
+					  uiTextInt,
+					  psDevicememHistoryUnmapVRangeIN->ui32Log2PageSize,
+					  psDevicememHistoryUnmapVRangeIN->ui32AllocationIndex,
+					  &psDevicememHistoryUnmapVRangeOUT->
+					  ui32AllocationIndexOut);
+
+DevicememHistoryUnmapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistorySparseChange(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psDevicememHistorySparseChangeIN_UI8,
+					 IMG_UINT8 * psDevicememHistorySparseChangeOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistorySparseChangeIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32AllocPageCount *
+	     sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32FreePageCount *
+	     sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32AllocPageCount >
+	     PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistorySparseChangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistorySparseChangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistorySparseChangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistorySparseChange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32AllocPageIndices,
+		     psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32FreePageIndices,
+		     psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistorySparseChangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistorySparseChangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistorySparseChange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistorySparseChangeOUT->eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   psDevicememHistorySparseChangeIN->uiOffset,
+					   psDevicememHistorySparseChangeIN->sDevVAddr,
+					   psDevicememHistorySparseChangeIN->uiSize,
+					   uiTextInt,
+					   psDevicememHistorySparseChangeIN->ui32Log2PageSize,
+					   psDevicememHistorySparseChangeIN->ui32AllocPageCount,
+					   ui32AllocPageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32FreePageCount,
+					   ui32FreePageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32AllocationIndex,
+					   &psDevicememHistorySparseChangeOUT->
+					   ui32AllocationIndexOut);
+
+DevicememHistorySparseChange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistorySparseChangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pDEVICEMEMHISTORYBridgeLock;
+
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+
+/*
+ * Register all DEVICEMEMHISTORY functions with services
+ */
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pDEVICEMEMHISTORYBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP,
+			      PVRSRVBridgeDevicememHistoryMap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP,
+			      PVRSRVBridgeDevicememHistoryUnmap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryMapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryUnmapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE,
+			      PVRSRVBridgeDevicememHistorySparseChange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all devicememhistory functions with services
+ */
+void DeinitDEVICEMEMHISTORYBridge(void)
+{
+	OSLockDestroy(pDEVICEMEMHISTORYBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/common_di_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/common_di_bridge.h
new file mode 100644
index 000000000000..8591006140b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/common_di_bridge.h
@@ -0,0 +1,153 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DI_BRIDGE_H
+#define COMMON_DI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "pvr_dicommon.h"
+
+#define PVRSRV_BRIDGE_DI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DI_DICREATECONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DI_DIREADENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DI_DIWRITEENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DI_DILISTALLENTRIES			PVRSRV_BRIDGE_DI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DI_CMD_LAST			(PVRSRV_BRIDGE_DI_CMD_FIRST+4)
+
+/*******************************************
+            DICreateContext
+ *******************************************/
+
+/* Bridge in structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_IN_DICREATECONTEXT_TAG
+{
+	IMG_CHAR *puiStreamName;
+} __packed PVRSRV_BRIDGE_IN_DICREATECONTEXT;
+
+/* Bridge out structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_OUT_DICREATECONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_CHAR *puiStreamName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DICREATECONTEXT;
+
+/*******************************************
+            DIDestroyContext
+ *******************************************/
+
+/* Bridge in structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT;
+
+/* Bridge out structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT;
+
+/*******************************************
+            DIReadEntry
+ *******************************************/
+
+/* Bridge in structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIREADENTRY_TAG
+{
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+} __packed PVRSRV_BRIDGE_IN_DIREADENTRY;
+
+/* Bridge out structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIREADENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIREADENTRY;
+
+/*******************************************
+            DIWriteEntry
+ *******************************************/
+
+/* Bridge in structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIWRITEENTRY_TAG
+{
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+	const IMG_CHAR *puiValue;
+	IMG_UINT32 ui32ValueSize;
+} __packed PVRSRV_BRIDGE_IN_DIWRITEENTRY;
+
+/* Bridge out structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIWRITEENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIWRITEENTRY;
+
+/*******************************************
+            DIListAllEntries
+ *******************************************/
+
+/* Bridge in structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_IN_DILISTALLENTRIES_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DILISTALLENTRIES;
+
+/* Bridge out structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_OUT_DILISTALLENTRIES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DILISTALLENTRIES;
+
+#endif /* COMMON_DI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/server_di_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/server_di_bridge.c
new file mode 100644
index 000000000000..154bccdaae5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/di_bridge/server_di_bridge.c
@@ -0,0 +1,652 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "di_impl_brg.h"
+
+#include "common_di_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DICreateContextpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DIDestroyContextKM((DI_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDICreateContext(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDICreateContextIN_UI8,
+			    IMG_UINT8 * psDICreateContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DICREATECONTEXT *psDICreateContextIN =
+	    (PVRSRV_BRIDGE_IN_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DICREATECONTEXT *psDICreateContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextOUT_UI8, 0);
+
+	IMG_CHAR *puiStreamNameInt = NULL;
+	DI_CONTEXT *psContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	PVR_UNREFERENCED_PARAMETER(psDICreateContextIN);
+
+	psDICreateContextOUT->puiStreamName = psDICreateContextIN->puiStreamName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDICreateContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DICreateContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDICreateContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDICreateContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDICreateContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DICreateContext_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		puiStreamNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	psDICreateContextOUT->eError = DICreateContextKM(puiStreamNameInt, &psContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		goto DICreateContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDICreateContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDICreateContextOUT->hContext,
+								 (void *)psContextInt,
+								 PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								 (PFN_HANDLE_RELEASE) &
+								 _DICreateContextpsContextIntRelease);
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DICreateContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamNameInt) && ((PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDICreateContextOUT->puiStreamName, puiStreamNameInt,
+		      (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psDICreateContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DICreateContext_exit;
+		}
+	}
+
+DICreateContext_exit:
+
+	if (psDICreateContextOUT->eError != PVRSRV_OK)
+	{
+		if (psDICreateContextOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDICreateContextOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DIDestroyContextKM(psContextInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDICreateContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIDestroyContext(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIDestroyContextIN_UI8,
+			     IMG_UINT8 * psDIDestroyContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *psDIDestroyContextIN =
+	    (PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *psDIDestroyContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDIDestroyContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDIDestroyContextIN->hContext,
+					      PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	if (unlikely((psDIDestroyContextOUT->eError != PVRSRV_OK) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDIDestroyContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIDestroyContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DIDestroyContext_exit:
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIReadEntry(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDIReadEntryIN_UI8,
+			IMG_UINT8 * psDIReadEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIREADENTRY *psDIReadEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIREADENTRY *psDIReadEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIReadEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIReadEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIReadEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIReadEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIReadEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIReadEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIReadEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIReadEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIReadEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIReadEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIReadEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIReadEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIReadEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIReadEntryOUT->eError =
+	    DIReadEntryKM(psContextInt,
+			  uiEntryPathInt, psDIReadEntryIN->ui64Offset, psDIReadEntryIN->ui64Size);
+
+DIReadEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIReadEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIWriteEntry(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDIWriteEntryIN_UI8,
+			 IMG_UINT8 * psDIWriteEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIWRITEENTRY *psDIWriteEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIWRITEENTRY *psDIWriteEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIWriteEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+	IMG_CHAR *uiValueInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psDIWriteEntryIN->ui32ValueSize > DI_IMPL_BRG_PATH_LEN))
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DIWriteEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIWriteEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIWriteEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIWriteEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIWriteEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIWriteEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIWriteEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDIWriteEntryIN->ui32ValueSize != 0)
+	{
+		uiValueInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiValueInt, (const void __user *)psDIWriteEntryIN->puiValue,
+		     psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiValueInt)[(psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) -
+					  1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIWriteEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIWriteEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIWriteEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIWriteEntryOUT->eError =
+	    DIWriteEntryKM(psContextInt,
+			   uiEntryPathInt, psDIWriteEntryIN->ui32ValueSize, uiValueInt);
+
+DIWriteEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIWriteEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIListAllEntries(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIListAllEntriesIN_UI8,
+			     IMG_UINT8 * psDIListAllEntriesOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DILISTALLENTRIES *psDIListAllEntriesIN =
+	    (PVRSRV_BRIDGE_IN_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *psDIListAllEntriesOUT =
+	    (PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIListAllEntriesIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIListAllEntriesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIListAllEntriesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIListAllEntries_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIListAllEntriesOUT->eError = DIListAllEntriesKM(psContextInt);
+
+DIListAllEntries_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+
+/*
+ * Register all DI functions with services
+ */
+PVRSRV_ERROR InitDIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT,
+			      PVRSRVBridgeDICreateContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DICREATECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DICREATECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT,
+			      PVRSRVBridgeDIDestroyContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY,
+			      PVRSRVBridgeDIReadEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIREADENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIREADENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY,
+			      PVRSRVBridgeDIWriteEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIWRITEENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIWRITEENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES,
+			      PVRSRVBridgeDIListAllEntries, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DILISTALLENTRIES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DILISTALLENTRIES));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all di functions with services
+ */
+void DeinitDIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/common_dma_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/common_dma_bridge.h
new file mode 100644
index 000000000000..dafa475b9c9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/common_dma_bridge.h
@@ -0,0 +1,123 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMA_BRIDGE_H
+#define COMMON_DMA_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DMA_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMA_DMATRANSFER			PVRSRV_BRIDGE_DMA_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE			PVRSRV_BRIDGE_DMA_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS			PVRSRV_BRIDGE_DMA_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMA_CMD_LAST			(PVRSRV_BRIDGE_DMA_CMD_FIRST+2)
+
+/*******************************************
+            DmaTransfer
+ *******************************************/
+
+/* Bridge in structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_IN_DMATRANSFER_TAG
+{
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32NumDMAs;
+	IMG_UINT32 ui32uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DMATRANSFER;
+
+/* Bridge out structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_OUT_DMATRANSFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMATRANSFER;
+
+/*******************************************
+            DmaSparseMappingTable
+ *******************************************/
+
+/* Bridge in structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_BOOL *pbTable;
+	IMG_UINT32 ui32SizeInPages;
+} __packed PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE;
+
+/* Bridge out structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_BOOL *pbTable;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE;
+
+/*******************************************
+            DmaDeviceParams
+ *******************************************/
+
+/* Bridge in structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_IN_DMADEVICEPARAMS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DMADEVICEPARAMS;
+
+/* Bridge out structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DmaBuffAlign;
+	IMG_UINT32 ui32DmaTransferMult;
+} __packed PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS;
+
+#endif /* COMMON_DMA_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/server_dma_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/server_dma_bridge.c
new file mode 100644
index 000000000000..7f2c5dd495e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dma_bridge/server_dma_bridge.c
@@ -0,0 +1,490 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dma_km.h"
+
+#include "common_dma_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(MAX_DMA_OPS <= IMG_UINT32_MAX, "MAX_DMA_OPS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaTransfer(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDmaTransferIN_UI8,
+			IMG_UINT8 * psDmaTransferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMATRANSFER *psDmaTransferIN =
+	    (PVRSRV_BRIDGE_IN_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMATRANSFER *psDmaTransferOUT =
+	    (PVRSRV_BRIDGE_OUT_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) + 0;
+
+	if (unlikely(psDmaTransferIN->ui32NumDMAs > MAX_DMA_OPS))
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaTransfer_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaTransfer_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaTransferIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaTransferIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaTransferOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psDmaTransferIN->ui32NumDMAs * sizeof(PMR *));
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psDmaTransferIN->phPMR,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psDmaTransferIN->pui64Address,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psDmaTransferIN->puiOffset,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psDmaTransferIN->puiSize,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+			/* Look up the address from the handle */
+			psDmaTransferOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDmaTransferOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaTransferOUT->eError =
+	    DmaTransfer(psConnection, OSGetDevNode(psConnection),
+			psDmaTransferIN->ui32NumDMAs,
+			psPMRInt,
+			ui64AddressInt,
+			uiOffsetInt,
+			uiSizeInt, psDmaTransferIN->ui32uiFlags, psDmaTransferIN->hUpdateTimeline);
+
+DmaTransfer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaTransferOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(32 <= IMG_UINT32_MAX, "32 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaSparseMappingTable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDmaSparseMappingTableIN_UI8,
+				  IMG_UINT8 * psDmaSparseMappingTableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableIN =
+	    (PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDmaSparseMappingTableIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_BOOL *pbTableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) + 0;
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages > 32)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	psDmaSparseMappingTableOUT->pbTable = psDmaSparseMappingTableIN->pbTable;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaSparseMappingTableIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaSparseMappingTableIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaSparseMappingTable_exit;
+			}
+		}
+	}
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages != 0)
+	{
+		pbTableInt = (IMG_BOOL *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDmaSparseMappingTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DmaSparseMappingTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaSparseMappingTableOUT->eError =
+	    DmaSparseMappingTable(psPMRInt,
+				  psDmaSparseMappingTableIN->uiOffset,
+				  psDmaSparseMappingTableIN->ui32SizeInPages, pbTableInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		goto DmaSparseMappingTable_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pbTableInt) && ((psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDmaSparseMappingTableOUT->pbTable, pbTableInt,
+		      (psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL))) !=
+		     PVRSRV_OK))
+		{
+			psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaSparseMappingTable_exit;
+		}
+	}
+
+DmaSparseMappingTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaSparseMappingTableOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDmaDeviceParams(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDmaDeviceParamsIN_UI8,
+			    IMG_UINT8 * psDmaDeviceParamsOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *psDmaDeviceParamsIN =
+	    (PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *psDmaDeviceParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psDmaDeviceParamsIN);
+
+	psDmaDeviceParamsOUT->eError =
+	    DmaDeviceParams(psConnection, OSGetDevNode(psConnection),
+			    &psDmaDeviceParamsOUT->ui32DmaBuffAlign,
+			    &psDmaDeviceParamsOUT->ui32DmaTransferMult);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABridge(void);
+void DeinitDMABridge(void);
+
+/*
+ * Register all DMA functions with services
+ */
+PVRSRV_ERROR InitDMABridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER,
+			      PVRSRVBridgeDmaTransfer, NULL, sizeof(PVRSRV_BRIDGE_IN_DMATRANSFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMATRANSFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE,
+			      PVRSRVBridgeDmaSparseMappingTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS,
+			      PVRSRVBridgeDmaDeviceParams, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dma functions with services
+ */
+void DeinitDMABridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/common_dmabuf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/common_dmabuf_bridge.h
new file mode 100644
index 000000000000..cc631e3e1116
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/common_dmabuf_bridge.h
@@ -0,0 +1,145 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMABUF_BRIDGE_H
+#define COMMON_DMABUF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_DMABUF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DMABUF_CMD_LAST			(PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3)
+
+/*******************************************
+            PhysmemImportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF;
+
+/* Bridge out structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF;
+
+/*******************************************
+            PhysmemExportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF;
+
+/* Bridge out structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF;
+
+/*******************************************
+            PhysmemExportGemHandle
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE;
+
+/* Bridge out structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Handle;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE;
+
+/*******************************************
+            PhysmemImportSparseDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_SIZE_T uiChunkSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF;
+
+/* Bridge out structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF;
+
+#endif /* COMMON_DMABUF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/server_dmabuf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/server_dmabuf_bridge.c
new file mode 100644
index 000000000000..89aa1477f55a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/dmabuf_bridge/server_dmabuf_bridge.c
@@ -0,0 +1,574 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "physmem_dmabuf.h"
+#include "pmr.h"
+
+#include "common_dmabuf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemImportDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemImportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemImportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufOUT_UI8,
+								      0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPhysmemImportDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportDmaBufIN->puiName,
+		     psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportDmaBuf_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR))
+					 - 1] = '\0';
+	}
+
+	psPhysmemImportDmaBufOUT->eError =
+	    PhysmemImportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPhysmemImportDmaBufIN->ifd,
+				psPhysmemImportDmaBufIN->uiFlags,
+				psPhysmemImportDmaBufIN->ui32NameSize,
+				uiNameInt,
+				&psPMRPtrInt,
+				&psPhysmemImportDmaBufOUT->uiSize,
+				&psPhysmemImportDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportDmaBufOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psPhysmemImportDmaBufOUT->
+								     hPMRPtr, (void *)psPMRPtrInt,
+								     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _PhysmemImportDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportDmaBuf_exit:
+
+	if (psPhysmemImportDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemExportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemExportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psPhysmemExportDmaBufIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportDmaBufOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportDmaBuf_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportDmaBufOUT->eError =
+	    PhysmemExportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPMRInt, &psPhysmemExportDmaBufOUT->iFd);
+
+PhysmemExportDmaBuf_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportGemHandle(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemExportGemHandleIN_UI8,
+				   IMG_UINT8 * psPhysmemExportGemHandleOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPhysmemExportGemHandleIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportGemHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportGemHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportGemHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportGemHandleOUT->eError =
+	    PhysmemExportGemHandle(psConnection, OSGetDevNode(psConnection),
+				   psPMRInt, &psPhysmemExportGemHandleOUT->ui32Handle);
+
+PhysmemExportGemHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemImportSparseDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportSparseDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufIN_UI8,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufOUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (unlikely(psPhysmemImportSparseDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportSparseDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportSparseDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportSparseDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemImportSparseDmaBufIN->pui32MappingTable,
+		     psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+	}
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportSparseDmaBufIN->puiName,
+		     psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+		((IMG_CHAR *)
+		 uiNameInt)[(psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PhysmemImportSparseDmaBuf(psConnection, OSGetDevNode(psConnection),
+				      psPhysmemImportSparseDmaBufIN->ifd,
+				      psPhysmemImportSparseDmaBufIN->uiFlags,
+				      psPhysmemImportSparseDmaBufIN->uiChunkSize,
+				      psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks,
+				      psPhysmemImportSparseDmaBufIN->ui32NumVirtChunks,
+				      ui32MappingTableInt,
+				      psPhysmemImportSparseDmaBufIN->ui32NameSize,
+				      uiNameInt,
+				      &psPMRPtrInt,
+				      &psPhysmemImportSparseDmaBufOUT->uiSize,
+				      &psPhysmemImportSparseDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psPhysmemImportSparseDmaBufOUT->hPMRPtr, (void *)psPMRPtrInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PhysmemImportSparseDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportSparseDmaBuf_exit:
+
+	if (psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportSparseDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+/*
+ * Register all DMABUF functions with services
+ */
+PVRSRV_ERROR InitDMABUFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF,
+			      PVRSRVBridgePhysmemImportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF,
+			      PVRSRVBridgePhysmemExportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE,
+			      PVRSRVBridgePhysmemExportGemHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF,
+			      PVRSRVBridgePhysmemImportSparseDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dmabuf functions with services
+ */
+void DeinitDMABUFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF,
+				PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_bridge.h
new file mode 100644
index 000000000000..2c5d18843120
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_bridge.h
@@ -0,0 +1,64 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_HTBUFFER_BRIDGE_H
+#define CLIENT_HTBUFFER_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_htbuffer_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode);
+
+#endif /* CLIENT_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_direct_bridge.c
new file mode 100644
index 000000000000..3e6295cdfdf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/client_htbuffer_direct_bridge.c
@@ -0,0 +1,70 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for htbuffer
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_htbuffer_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#include "htbserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError =
+	    HTBControlKM(ui32NumGroups,
+			 pui32GroupEnable, ui32LogLevel, ui32EnablePID, ui32LogMode, ui32OpMode);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/common_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/common_htbuffer_bridge.h
new file mode 100644
index 000000000000..32be202186dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/common_htbuffer_bridge.h
@@ -0,0 +1,82 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_HTBUFFER_BRIDGE_H
+#define COMMON_HTBUFFER_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST			0
+#define PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL			PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_LAST			(PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0)
+
+/*******************************************
+            HTBControl
+ *******************************************/
+
+/* Bridge in structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_IN_HTBCONTROL_TAG
+{
+	IMG_UINT32 *pui32GroupEnable;
+	IMG_UINT32 ui32EnablePID;
+	IMG_UINT32 ui32LogLevel;
+	IMG_UINT32 ui32LogMode;
+	IMG_UINT32 ui32NumGroups;
+	IMG_UINT32 ui32OpMode;
+} __packed PVRSRV_BRIDGE_IN_HTBCONTROL;
+
+/* Bridge out structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_OUT_HTBCONTROL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HTBCONTROL;
+
+#endif /* COMMON_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/server_htbuffer_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/server_htbuffer_bridge.c
new file mode 100644
index 000000000000..9b730d28e5b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/htbuffer_bridge/server_htbuffer_bridge.c
@@ -0,0 +1,227 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "htbserver.h"
+
+#include "common_htbuffer_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(HTB_FLAG_NUM_EL <= IMG_UINT32_MAX,
+	      "HTB_FLAG_NUM_EL must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHTBControl(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psHTBControlIN_UI8,
+		       IMG_UINT8 * psHTBControlOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HTBCONTROL *psHTBControlIN =
+	    (PVRSRV_BRIDGE_IN_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HTBCONTROL *psHTBControlOUT =
+	    (PVRSRV_BRIDGE_OUT_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlOUT_UI8, 0);
+
+	IMG_UINT32 *ui32GroupEnableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psHTBControlIN->ui32NumGroups > HTB_FLAG_NUM_EL))
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HTBControl_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HTBControl_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHTBControlIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHTBControlIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHTBControlOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HTBControl_exit;
+			}
+		}
+	}
+
+	if (psHTBControlIN->ui32NumGroups != 0)
+	{
+		ui32GroupEnableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32GroupEnableInt,
+		     (const void __user *)psHTBControlIN->pui32GroupEnable,
+		     psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psHTBControlOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HTBControl_exit;
+		}
+	}
+
+	psHTBControlOUT->eError =
+	    HTBControlKM(psHTBControlIN->ui32NumGroups,
+			 ui32GroupEnableInt,
+			 psHTBControlIN->ui32LogLevel,
+			 psHTBControlIN->ui32EnablePID,
+			 psHTBControlIN->ui32LogMode, psHTBControlIN->ui32OpMode);
+
+HTBControl_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHTBControlOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pHTBUFFERBridgeLock;
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+
+/*
+ * Register all HTBUFFER functions with services
+ */
+PVRSRV_ERROR InitHTBUFFERBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pHTBUFFERBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL,
+			      PVRSRVBridgeHTBControl, pHTBUFFERBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_HTBCONTROL),
+			      sizeof(PVRSRV_BRIDGE_OUT_HTBCONTROL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all htbuffer functions with services
+ */
+void DeinitHTBUFFERBridge(void)
+{
+	OSLockDestroy(pHTBUFFERBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL);
+
+}
+#else /* EXCLUDE_HTBUFFER_BRIDGE */
+/* This bridge is conditional on EXCLUDE_HTBUFFER_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitHTBUFFERBridge() \
+	PVRSRV_OK
+
+#define DeinitHTBUFFERBridge()
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_bridge.h
new file mode 100644
index 000000000000..d5ba6dd74da3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_bridge.h
@@ -0,0 +1,234 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_MM_BRIDGE_H
+#define CLIENT_MM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_mm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge,
+							IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags,
+						IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats);
+
+#endif /* CLIENT_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_direct_bridge.c
new file mode 100644
index 000000000000..c1810e959ca8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/client_mm_direct_bridge.c
@@ -0,0 +1,719 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for mm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_mm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRExportPMR(psPMRInt, &psPMRExportInt, pui64Size, pui32Log2Contig, pui64Password);
+
+	*phPMRExport = psPMRExportInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMR);
+	PVR_UNREFERENCED_PARAMETER(phPMRExport);
+	PVR_UNREFERENCED_PARAMETER(pui64Size);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2Contig);
+	PVR_UNREFERENCED_PARAMETER(pui64Password);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError = PMRUnexportPMR(psPMRExportInt);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRGetUID(psPMRInt, pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psBufferInt;
+	PMR *psExtMemInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psBufferInt = (PMR *) hBuffer;
+
+	eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+
+	*phExtMem = psExtMemInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtMemInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtMemInt = (PMR *) hExtMem;
+
+	eError = PMRUnmakeLocalImportHandle(psExtMemInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PMR *psPMRInt = NULL;
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError =
+	    PhysmemImportPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     psPMRExportInt,
+			     ui64uiPassword, ui64uiSize, ui32uiLog2Contig, &psPMRInt);
+
+	*phPMR = psPMRInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+	PVR_UNREFERENCED_PARAMETER(ui64uiPassword);
+	PVR_UNREFERENCED_PARAMETER(ui64uiSize);
+	PVR_UNREFERENCED_PARAMETER(ui32uiLog2Contig);
+	PVR_UNREFERENCED_PARAMETER(phPMR);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtHandleInt;
+	PMR *psPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtHandleInt = (PMR *) hExtHandle;
+
+	eError = PMRLocalImportPMR(psExtHandleInt, &psPMRInt, puiSize, puiAlign);
+
+	*phPMR = psPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRUnrefPMR(psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRPtrInt = NULL;
+
+	eError =
+	    PhysmemNewRamBackedPMR_direct(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  uiSize,
+					  ui32NumPhysChunks,
+					  ui32NumVirtChunks,
+					  pui32MappingTable,
+					  ui32Log2PageSize,
+					  uiFlags,
+					  ui32AnnotationLength,
+					  puiAnnotation,
+					  ui32PID, &psPMRPtrInt, ui32PDumpFlags, puiOutFlags);
+
+	*phPMRPtr = psPMRPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	eError =
+	    DevmemIntCtxCreate(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       bbKernelMemoryCtx,
+			       &psDevMemServerContextInt, &hPrivDataInt, pui32CPUCacheLineSize);
+
+	*phDevMemServerContext = psDevMemServerContextInt;
+	*phPrivData = hPrivDataInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError = DevmemIntCtxDestroy(psDevmemServerContextInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				ui32HeapConfigIndex, ui32HeapIndex, &psDevmemHeapPtrInt);
+
+	*phDevmemHeapPtr = psDevmemHeapPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeapInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemHeapInt = (DEVMEMINT_HEAP *) hDevmemHeap;
+
+	eError = DevmemIntHeapDestroy(psDevmemHeapInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnmapPMR(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemIntReserveRange(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psDevmemServerHeapInt,
+				  sAddress, uiLength, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	PMR *psPMRInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemIntReserveRangeAndMapPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   psDevmemServerHeapInt,
+					   sAddress,
+					   uiLength, psPMRInt, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemIntChangeSparse(ui32AllocPageCount,
+				  pui32AllocPageIndices,
+				  ui32FreePageCount,
+				  pui32FreePageIndices, ui32SparseFlags, psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntIsVDevAddrValid(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, sAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries)
+{
+#if defined(RGX_FEATURE_FBCDC)
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntInvalidateFBSCTable(psDevmemCtxInt, ui64FBSCEntries);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(ui64FBSCEntries);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   pui32NumHeapConfigs);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     ui32HeapConfigIndex, pui32NumHeaps);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigName(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  ui32HeapConfigIndex, ui32HeapConfigNameBufSz, puiHeapConfigName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapDetails(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       ui32HeapConfigIndex,
+			       ui32HeapIndex,
+			       ui32HeapNameBufSz,
+			       puiHeapNameOut,
+			       psDevVAddrBase,
+			       puiHeapLength,
+			       puiReservedRegionLength,
+			       pui32Log2DataPageSizeOut, pui32Log2ImportAlignmentOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, bRegister);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfoKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				       ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), peHeap);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntGetFaultAddress(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, psFaultAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVStatsUpdateOOMStat(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32ui32StatType, ui32pid);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32ui32StatType);
+	PVR_UNREFERENCED_PARAMETER(ui32pid);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt, sAddress, uiLength, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       ui32PageCount, ui32PhysPageOffset, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnmapPages(psReservationInt, ui32VirtPageOffset, ui32PageCount);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     ui32PageCount, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/common_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/common_mm_bridge.h
new file mode 100644
index 000000000000..3b3c8c2dc215
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/common_mm_bridge.h
@@ -0,0 +1,757 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MM_BRIDGE_H
+#define COMMON_MM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MM_PMREXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_MM_PMRGETUID			PVRSRV_BRIDGE_MM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_MM_PMRIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_MM_PMRUNREFPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+9
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+10
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+11
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+12
+#define PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+13
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+14
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+15
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+16
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+17
+#define PVRSRV_BRIDGE_MM_CHANGESPARSEMEM			PVRSRV_BRIDGE_MM_CMD_FIRST+18
+#define PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID			PVRSRV_BRIDGE_MM_CMD_FIRST+19
+#define PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE			PVRSRV_BRIDGE_MM_CMD_FIRST+20
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+21
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+22
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME			PVRSRV_BRIDGE_MM_CMD_FIRST+23
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS			PVRSRV_BRIDGE_MM_CMD_FIRST+24
+#define PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM			PVRSRV_BRIDGE_MM_CMD_FIRST+25
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO			PVRSRV_BRIDGE_MM_CMD_FIRST+26
+#define PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP			PVRSRV_BRIDGE_MM_CMD_FIRST+27
+#define PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS			PVRSRV_BRIDGE_MM_CMD_FIRST+28
+#define PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT			PVRSRV_BRIDGE_MM_CMD_FIRST+29
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+30
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+31
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+32
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+33
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE			PVRSRV_BRIDGE_MM_CMD_FIRST+34
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2			PVRSRV_BRIDGE_MM_CMD_FIRST+35
+#define PVRSRV_BRIDGE_MM_CMD_LAST			(PVRSRV_BRIDGE_MM_CMD_FIRST+35)
+
+/*******************************************
+            PMRExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMREXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMREXPORTPMR;
+
+/* Bridge out structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMREXPORTPMR_TAG
+{
+	IMG_UINT64 ui64Password;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRExport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2Contig;
+} __packed PVRSRV_BRIDGE_OUT_PMREXPORTPMR;
+
+/*******************************************
+            PMRUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMRExport;
+} __packed PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR;
+
+/* Bridge out structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR;
+
+/*******************************************
+            PMRGetUID
+ *******************************************/
+
+/* Bridge in structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_IN_PMRGETUID_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRGETUID;
+
+/* Bridge out structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_OUT_PMRGETUID_TAG
+{
+	IMG_UINT64 ui64UID;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRGETUID;
+
+/*******************************************
+            PMRMakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRUnmakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRIMPORTPMR_TAG
+{
+	IMG_UINT64 ui64uiPassword;
+	IMG_UINT64 ui64uiSize;
+	IMG_HANDLE hPMRExport;
+	IMG_UINT32 ui32uiLog2Contig;
+} __packed PVRSRV_BRIDGE_IN_PMRIMPORTPMR;
+
+/* Bridge out structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRIMPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRIMPORTPMR;
+
+/*******************************************
+            PMRLocalImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_HANDLE hExtHandle;
+} __packed PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR;
+
+/* Bridge out structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR;
+
+/*******************************************
+            PMRUnrefPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNREFPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRUNREFPMR;
+
+/* Bridge out structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNREFPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNREFPMR;
+
+/*******************************************
+            PhysmemNewRamBackedPMR
+ *******************************************/
+
+/* Bridge in structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiAnnotation;
+	IMG_UINT32 ui32AnnotationLength;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_PID ui32PID;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR;
+
+/* Bridge out structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR;
+
+/*******************************************
+            DevmemIntCtxCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_BOOL bbKernelMemoryCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE;
+
+/* Bridge out structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_HANDLE hDevMemServerContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CPUCacheLineSize;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE;
+
+/*******************************************
+            DevmemIntCtxDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY_TAG
+{
+	IMG_HANDLE hDevmemServerContext;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY;
+
+/* Bridge out structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY;
+
+/*******************************************
+            DevmemIntHeapCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE;
+
+/* Bridge out structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemHeapPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE;
+
+/*******************************************
+            DevmemIntHeapDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY_TAG
+{
+	IMG_HANDLE hDevmemHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY;
+
+/* Bridge out structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY;
+
+/*******************************************
+            DevmemIntMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR;
+
+/* Bridge out structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR;
+
+/*******************************************
+            DevmemIntUnmapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR;
+
+/* Bridge out structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR;
+
+/*******************************************
+            DevmemIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE;
+
+/* Bridge out structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE;
+
+/*******************************************
+            DevmemIntReserveRangeAndMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	IMG_HANDLE hPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/* Bridge out structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/*******************************************
+            DevmemIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE;
+
+/*******************************************
+            ChangeSparseMem
+ *******************************************/
+
+/* Bridge in structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_IN_CHANGESPARSEMEM_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32SparseFlags;
+} __packed PVRSRV_BRIDGE_IN_CHANGESPARSEMEM;
+
+/* Bridge out structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM;
+
+/*******************************************
+            DevmemIsVDevAddrValid
+ *******************************************/
+
+/* Bridge in structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID;
+
+/* Bridge out structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID;
+
+/*******************************************
+            DevmemInvalidateFBSCTable
+ *******************************************/
+
+/* Bridge in structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	IMG_UINT64 ui64FBSCEntries;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE;
+
+/* Bridge out structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE;
+
+/*******************************************
+            HeapCfgHeapConfigCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT;
+
+/* Bridge out structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeapConfigs;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT;
+
+/*******************************************
+            HeapCfgHeapCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT_TAG
+{
+	IMG_UINT32 ui32HeapConfigIndex;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT;
+
+/* Bridge out structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT;
+
+/*******************************************
+            HeapCfgHeapConfigName
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapConfigNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME;
+
+/* Bridge out structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME;
+
+/*******************************************
+            HeapCfgHeapDetails
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_CHAR *puiHeapNameOut;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+	IMG_UINT32 ui32HeapNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS;
+
+/* Bridge out structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_CHAR *puiHeapNameOut;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2DataPageSizeOut;
+	IMG_UINT32 ui32Log2ImportAlignmentOut;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS;
+
+/*******************************************
+            DevmemIntRegisterPFNotifyKM
+ *******************************************/
+
+/* Bridge in structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_BOOL bRegister;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/* Bridge out structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/*******************************************
+            PhysHeapGetMemInfo
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO;
+
+/* Bridge out structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO;
+
+/*******************************************
+            GetDefaultPhysicalHeap
+ *******************************************/
+
+/* Bridge in structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP;
+
+/* Bridge out structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP eHeap;
+} __packed PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP;
+
+/*******************************************
+            DevmemGetFaultAddress
+ *******************************************/
+
+/* Bridge in structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS;
+
+/* Bridge out structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_DEV_VIRTADDR sFaultAddress;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS;
+
+/*******************************************
+            PVRSRVStatsUpdateOOMStat
+ *******************************************/
+
+/* Bridge in structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	IMG_PID ui32pid;
+	IMG_UINT32 ui32ui32StatType;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT;
+
+/* Bridge out structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT;
+
+/*******************************************
+            DevmemXIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE;
+
+/*******************************************
+            DevmemXIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE;
+
+/*******************************************
+            DevmemXIntMapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32PhysPageOffset;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES;
+
+/* Bridge out structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES;
+
+/*******************************************
+            DevmemXIntUnmapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES;
+
+/* Bridge out structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES;
+
+/*******************************************
+            DevmemXIntMapVRangeToBackingPage
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/* Bridge out structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/*******************************************
+            PhysHeapGetMemInfo2
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2;
+
+/* Bridge out structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2;
+
+#endif /* COMMON_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/server_mm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/server_mm_bridge.c
new file mode 100644
index 000000000000..a79a92f5e9d1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mm_bridge/server_mm_bridge.c
@@ -0,0 +1,3237 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+#include "common_mm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleasePMRExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRExportPMRpsPMRExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnexportPMR((PMR_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRExportPMRIN_UI8,
+			 IMG_UINT8 * psPMRExportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMREXPORTPMR *psPMRExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMREXPORTPMR *psPMRExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PMRExportPMR(psPMRInt,
+			 &psPMRExportInt,
+			 &psPMRExportPMROUT->ui64Size,
+			 &psPMRExportPMROUT->ui32Log2Contig, &psPMRExportPMROUT->ui64Password);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRExportPMR_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &hPMRExportInt, (void *)psPMRExportInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _PMRExportPMRpsPMRExportIntRelease);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRExportPMROUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+							      &psPMRExportPMROUT->hPMRExport,
+							      (void *)psPMRExportInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      ReleasePMRExport);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRExportPMROUT->hPMRExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE) psPMRExportPMROUT->
+							     hPMRExport,
+							     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hPMRExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hPMRExportInt,
+							PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psPMRExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnexportPMR(psPMRExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRExportPMR NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgePMRUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psPMRUnexportPMRIN_UI8,
+			   IMG_UINT8 * psPMRUnexportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *psPMRUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *psPMRUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMROUT_UI8, 0);
+
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+				       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_FALSE);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hPMRExportInt,
+				     psPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT((psPMRUnexportPMROUT->eError == PVRSRV_OK) ||
+		   (psPMRUnexportPMROUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRUnexportPMR_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRUnexportPMR NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgePMRGetUID(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psPMRGetUIDIN_UI8,
+		      IMG_UINT8 * psPMRGetUIDOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRGETUID *psPMRGetUIDIN =
+	    (PVRSRV_BRIDGE_IN_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRGETUID *psPMRGetUIDOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRGetUIDIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRGetUIDOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRGetUIDOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRGetUID_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRGetUIDOUT->eError = PMRGetUID(psPMRInt, &psPMRGetUIDOUT->ui64UID);
+
+PMRGetUID_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRMakeLocalImportHandlepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnmakeLocalImportHandle((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRMakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleIN_UI8,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psPMRMakeLocalImportHandleIN->hBuffer;
+	PMR *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer,
+				       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE, IMG_TRUE);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psPMRMakeLocalImportHandleOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PMRMakeLocalImportHandlepsExtMemIntRelease);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRMakeLocalImportHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnmakeLocalImportHandle(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnmakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleIN_UI8,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRUnmakeLocalImportHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psPMRUnmakeLocalImportHandleIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_OK) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRUnmakeLocalImportHandleOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRUnmakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRUnmakeLocalImportHandle_exit:
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRImportPMRIN_UI8,
+			 IMG_UINT8 * psPMRImportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRIMPORTPMR *psPMRImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *psPMRImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMRExport = psPMRImportPMRIN->hPMRExport;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psPMRImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_TRUE);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRImportPMROUT->eError =
+	    PhysmemImportPMR(psConnection, OSGetDevNode(psConnection),
+			     psPMRExportInt,
+			     psPMRImportPMRIN->ui64uiPassword,
+			     psPMRImportPMRIN->ui64uiSize,
+			     psPMRImportPMRIN->ui32uiLog2Contig, &psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psPMRImportPMROUT->hPMR,
+							      (void *)psPMRInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _PMRImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRExportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psPMRImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRImportPMR NULL
+#endif
+
+static PVRSRV_ERROR _PMRLocalImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRLocalImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPMRLocalImportPMRIN_UI8,
+			      IMG_UINT8 * psPMRLocalImportPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *psPMRLocalImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *psPMRLocalImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hExtHandle = psPMRLocalImportPMRIN->hExtHandle;
+	PMR *psExtHandleInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRLocalImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psExtHandleInt,
+				       hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT, IMG_TRUE);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError =
+	    PMRLocalImportPMR(psExtHandleInt,
+			      &psPMRInt,
+			      &psPMRLocalImportPMROUT->uiSize, &psPMRLocalImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPMRLocalImportPMROUT->hPMR,
+								   (void *)psPMRInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PMRLocalImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRLocalImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psExtHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psPMRLocalImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnrefPMR(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRUnrefPMRIN_UI8,
+			IMG_UINT8 * psPMRUnrefPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNREFPMR *psPMRUnrefPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNREFPMR *psPMRUnrefPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRUnrefPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRUnrefPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (unlikely((psPMRUnrefPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnrefPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRUnrefPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRUnrefPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemNewRamBackedPMRpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemNewRamBackedPMR(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMRIN_UI8,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMROUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMRIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMROUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiAnnotationInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (unlikely(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemNewRamBackedPMRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemNewRamBackedPMRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemNewRamBackedPMR_exit;
+			}
+		}
+	}
+
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->pui32MappingTable,
+		     psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+	}
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength != 0)
+	{
+		uiAnnotationInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiAnnotationInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->puiAnnotation,
+		     psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+		((IMG_CHAR *)
+		 uiAnnotationInt)[(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength *
+				   sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psPhysmemNewRamBackedPMROUT->eError =
+	    PhysmemNewRamBackedPMR(psConnection, OSGetDevNode(psConnection),
+				   psPhysmemNewRamBackedPMRIN->uiSize,
+				   psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks,
+				   psPhysmemNewRamBackedPMRIN->ui32NumVirtChunks,
+				   ui32MappingTableInt,
+				   psPhysmemNewRamBackedPMRIN->ui32Log2PageSize,
+				   psPhysmemNewRamBackedPMRIN->uiFlags,
+				   psPhysmemNewRamBackedPMRIN->ui32AnnotationLength,
+				   uiAnnotationInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PID,
+				   &psPMRPtrInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PDumpFlags,
+				   &psPhysmemNewRamBackedPMROUT->uiOutFlags);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemNewRamBackedPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psPhysmemNewRamBackedPMROUT->
+									hPMRPtr,
+									(void *)psPMRPtrInt,
+									PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_PhysmemNewRamBackedPMRpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemNewRamBackedPMR_exit:
+
+	if (psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemNewRamBackedPMROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntCtxCreatepsDevMemServerContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxCreate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntCtxCreateIN_UI8,
+			       IMG_UINT8 * psDevmemIntCtxCreateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateOUT_UI8,
+								     0);
+
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntCtxCreateOUT->hDevMemServerContext = NULL;
+
+	psDevmemIntCtxCreateOUT->eError =
+	    DevmemIntCtxCreate(psConnection, OSGetDevNode(psConnection),
+			       psDevmemIntCtxCreateIN->bbKernelMemoryCtx,
+			       &psDevMemServerContextInt,
+			       &hPrivDataInt, &psDevmemIntCtxCreateOUT->ui32CPUCacheLineSize);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntCtxCreateOUT->
+								    hDevMemServerContext,
+								    (void *)
+								    psDevMemServerContextInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntCtxCreatepsDevMemServerContextIntRelease);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntCtxCreateOUT->
+								       hPrivData,
+								       (void *)hPrivDataInt,
+								       PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       psDevmemIntCtxCreateOUT->
+								       hDevMemServerContext);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxCreate_exit:
+
+	if (psDevmemIntCtxCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntCtxCreateOUT->hDevMemServerContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDevmemIntCtxCreateOUT->
+							     hDevMemServerContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psDevMemServerContextInt)
+		{
+			DevmemIntCtxDestroy(psDevMemServerContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntCtxDestroyIN_UI8,
+				IMG_UINT8 * psDevmemIntCtxDestroyOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyOUT_UI8,
+								      0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntCtxDestroyIN->
+					      hDevmemServerContext,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	if (unlikely
+	    ((psDevmemIntCtxDestroyOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntCtxDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntHeapDestroy((DEVMEMINT_HEAP *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapCreate(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntHeapCreateIN_UI8,
+				IMG_UINT8 * psDevmemIntHeapCreateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntHeapCreateIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntHeapCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				psDevmemIntHeapCreateIN->ui32HeapConfigIndex,
+				psDevmemIntHeapCreateIN->ui32HeapIndex, &psDevmemHeapPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psDevmemIntHeapCreateOUT->
+								     hDevmemHeapPtr,
+								     (void *)psDevmemHeapPtrInt,
+								     PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntHeapCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemHeapPtrInt)
+		{
+			DevmemIntHeapDestroy(psDevmemHeapPtrInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntHeapDestroyIN_UI8,
+				 IMG_UINT8 * psDevmemIntHeapDestroyOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *) IMG_OFFSET_ADDR(psDevmemIntHeapDestroyIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *)
+	    IMG_OFFSET_ADDR(psDevmemIntHeapDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntHeapDestroyIN->hDevmemHeap,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	if (unlikely((psDevmemIntHeapDestroyOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntHeapDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapDestroy_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDevmemIntMapPMRIN_UI8,
+			    IMG_UINT8 * psDevmemIntMapPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *psDevmemIntMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *psDevmemIntMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntMapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntMapPMROUT->eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+DevmemIntMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnmapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDevmemIntUnmapPMRIN_UI8,
+			      IMG_UINT8 * psDevmemIntUnmapPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntUnmapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntUnmapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntUnmapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnmapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnmapPMROUT->eError = DevmemIntUnmapPMR(psReservationInt);
+
+DevmemIntUnmapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIntReserveRangeIN_UI8,
+				  IMG_UINT8 * psDevmemIntReserveRangeOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError =
+	    DevmemIntReserveRange(psConnection, OSGetDevNode(psConnection),
+				  psDevmemServerHeapInt,
+				  psDevmemIntReserveRangeIN->sAddress,
+				  psDevmemIntReserveRangeIN->uiLength,
+				  psDevmemIntReserveRangeIN->uiFlags, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntReserveRangeOUT->
+								       hReservation,
+								       (void *)psReservationInt,
+								       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DevmemIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRangeAndMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMRIN_UI8,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMROUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeAndMapPMRIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntReserveRangeAndMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    DevmemIntReserveRangeAndMapPMR(psConnection, OSGetDevNode(psConnection),
+					   psDevmemServerHeapInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->sAddress,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiLength,
+					   psPMRInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiFlags,
+					   &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntReserveRangeAndMapPMROUT->hReservation,
+				      (void *)psReservationInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRangeAndMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeIN_UI8,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeChangeSparseMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psChangeSparseMemIN_UI8,
+			    IMG_UINT8 * psChangeSparseMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *psChangeSparseMemIN =
+	    (PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *psChangeSparseMemOUT =
+	    (PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+	IMG_HANDLE hReservation = psChangeSparseMemIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psChangeSparseMemIN->ui32AllocPageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (unlikely(psChangeSparseMemIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto ChangeSparseMem_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psChangeSparseMemIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psChangeSparseMemIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psChangeSparseMemOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto ChangeSparseMem_exit;
+			}
+		}
+	}
+
+	if (psChangeSparseMemIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32AllocPageIndices,
+		     psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+	if (psChangeSparseMemIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32FreePageIndices,
+		     psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psChangeSparseMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psChangeSparseMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto ChangeSparseMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psChangeSparseMemOUT->eError =
+	    DevmemIntChangeSparse(psChangeSparseMemIN->ui32AllocPageCount,
+				  ui32AllocPageIndicesInt,
+				  psChangeSparseMemIN->ui32FreePageCount,
+				  ui32FreePageIndicesInt,
+				  psChangeSparseMemIN->ui32SparseFlags, psReservationInt);
+
+ChangeSparseMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psChangeSparseMemOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIsVDevAddrValid(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidIN_UI8,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIsVDevAddrValidIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIsVDevAddrValidOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIsVDevAddrValidOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIsVDevAddrValid_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIsVDevAddrValidOUT->eError =
+	    DevmemIntIsVDevAddrValid(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, psDevmemIsVDevAddrValidIN->sAddress);
+
+DevmemIsVDevAddrValid_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+
+static IMG_INT
+PVRSRVBridgeDevmemInvalidateFBSCTable(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableIN_UI8,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemInvalidateFBSCTableIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemInvalidateFBSCTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemInvalidateFBSCTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    DevmemIntInvalidateFBSCTable(psDevmemCtxInt,
+					 psDevmemInvalidateFBSCTableIN->ui64FBSCEntries);
+
+DevmemInvalidateFBSCTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeDevmemInvalidateFBSCTable NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigCount(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountIN_UI8,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHeapCfgHeapConfigCountIN);
+
+	psHeapCfgHeapConfigCountOUT->eError =
+	    HeapCfgHeapConfigCount(psConnection, OSGetDevNode(psConnection),
+				   &psHeapCfgHeapConfigCountOUT->ui32NumHeapConfigs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapCount(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psHeapCfgHeapCountIN_UI8,
+			     IMG_UINT8 * psHeapCfgHeapCountOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountOUT_UI8, 0);
+
+	psHeapCfgHeapCountOUT->eError =
+	    HeapCfgHeapCount(psConnection, OSGetDevNode(psConnection),
+			     psHeapCfgHeapCountIN->ui32HeapConfigIndex,
+			     &psHeapCfgHeapCountOUT->ui32NumHeaps);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigName(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameIN_UI8,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameOUT_UI8, 0);
+
+	IMG_CHAR *puiHeapConfigNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	psHeapCfgHeapConfigNameOUT->puiHeapConfigName =
+	    psHeapCfgHeapConfigNameIN->puiHeapConfigName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapConfigNameIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapConfigNameIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapConfigName_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz != 0)
+	{
+		puiHeapConfigNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapConfigNameOUT->eError =
+	    HeapCfgHeapConfigName(psConnection, OSGetDevNode(psConnection),
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigIndex,
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz,
+				  puiHeapConfigNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapConfigNameOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapConfigNameInt) &&
+	    ((psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapConfigNameOUT->puiHeapConfigName,
+		      puiHeapConfigNameInt,
+		      (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapConfigName_exit;
+		}
+	}
+
+HeapCfgHeapConfigName_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapConfigNameOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapDetails(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psHeapCfgHeapDetailsIN_UI8,
+			       IMG_UINT8 * psHeapCfgHeapDetailsOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsOUT_UI8,
+								     0);
+
+	IMG_CHAR *puiHeapNameOutInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) + 0;
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	psHeapCfgHeapDetailsOUT->puiHeapNameOut = psHeapCfgHeapDetailsIN->puiHeapNameOut;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapDetailsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapDetailsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapDetails_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz != 0)
+	{
+		puiHeapNameOutInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapDetailsOUT->eError =
+	    HeapCfgHeapDetails(psConnection, OSGetDevNode(psConnection),
+			       psHeapCfgHeapDetailsIN->ui32HeapConfigIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapNameBufSz,
+			       puiHeapNameOutInt,
+			       &psHeapCfgHeapDetailsOUT->sDevVAddrBase,
+			       &psHeapCfgHeapDetailsOUT->uiHeapLength,
+			       &psHeapCfgHeapDetailsOUT->uiReservedRegionLength,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2DataPageSizeOut,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2ImportAlignmentOut);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapDetailsOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapNameOutInt) &&
+	    ((psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapDetailsOUT->puiHeapNameOut,
+		      puiHeapNameOutInt,
+		      (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapDetails_exit;
+		}
+	}
+
+HeapCfgHeapDetails_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapDetailsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntRegisterPFNotifyKM(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMIN_UI8,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntRegisterPFNotifyKMIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntRegisterPFNotifyKMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntRegisterPFNotifyKM_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, psDevmemIntRegisterPFNotifyKMIN->bRegister);
+
+DevmemIntRegisterPFNotifyKM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPhysHeapGetMemInfoIN_UI8,
+			       IMG_UINT8 * psPhysHeapGetMemInfoOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoIN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoOUT_UI8,
+								     0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V1)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfoIN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfoIN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfoIN->peaPhysHeapID,
+		     psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V1 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1);
+	}
+
+	psPhysHeapGetMemInfoOUT->eError =
+	    PVRSRVPhysHeapGetMemInfoKM(psConnection, OSGetDevNode(psConnection),
+				       psPhysHeapGetMemInfoIN->ui32PhysHeapCount,
+				       eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfoOUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V1))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+
+PhysHeapGetMemInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDefaultPhysicalHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapIN_UI8,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapIN =
+	    (PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDefaultPhysicalHeapIN);
+
+	psGetDefaultPhysicalHeapOUT->eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(psConnection, OSGetDevNode(psConnection),
+					   &psGetDefaultPhysicalHeapOUT->eHeap);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemGetFaultAddress(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemGetFaultAddressIN_UI8,
+				  IMG_UINT8 * psDevmemGetFaultAddressOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemGetFaultAddressIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemGetFaultAddressOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemGetFaultAddressOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemGetFaultAddress_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemGetFaultAddressOUT->eError =
+	    DevmemIntGetFaultAddress(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, &psDevmemGetFaultAddressOUT->sFaultAddress);
+
+DevmemGetFaultAddress_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+
+static IMG_INT
+PVRSRVBridgePVRSRVStatsUpdateOOMStat(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatIN_UI8,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatOUT_UI8, 0);
+
+	psPVRSRVStatsUpdateOOMStatOUT->eError =
+	    PVRSRVStatsUpdateOOMStat(psConnection, OSGetDevNode(psConnection),
+				     psPVRSRVStatsUpdateOOMStatIN->ui32ui32StatType,
+				     psPVRSRVStatsUpdateOOMStatIN->ui32pid);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePVRSRVStatsUpdateOOMStat NULL
+#endif
+
+static PVRSRV_ERROR _DevmemXIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemXIntUnreserveRange((DEVMEMXINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDevmemXIntReserveRangeIN_UI8,
+				   IMG_UINT8 * psDevmemXIntReserveRangeOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemXIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt,
+				   psDevmemXIntReserveRangeIN->sAddress,
+				   psDevmemXIntReserveRangeIN->uiLength, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDevmemXIntReserveRangeOUT->
+									hReservation,
+									(void *)psReservationInt,
+									PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DevmemXIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemXIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeIN_UI8,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemXIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemXIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapPages(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemXIntMapPagesIN_UI8,
+			       IMG_UINT8 * psDevmemXIntMapPagesOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesOUT_UI8,
+								     0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemXIntMapPagesIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapPagesOUT->eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       psDevmemXIntMapPagesIN->ui32PageCount,
+			       psDevmemXIntMapPagesIN->ui32PhysPageOffset,
+			       psDevmemXIntMapPagesIN->uiFlags,
+			       psDevmemXIntMapPagesIN->ui32VirtPageOffset);
+
+DevmemXIntMapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnmapPages(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesIN_UI8,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntUnmapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntUnmapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntUnmapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnmapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnmapPagesOUT->eError =
+	    DevmemXIntUnmapPages(psReservationInt,
+				 psDevmemXIntUnmapPagesIN->ui32VirtPageOffset,
+				 psDevmemXIntUnmapPagesIN->ui32PageCount);
+
+DevmemXIntUnmapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapVRangeToBackingPage(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageIN_UI8,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapVRangeToBackingPageIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapVRangeToBackingPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapVRangeToBackingPage_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     psDevmemXIntMapVRangeToBackingPageIN->ui32PageCount,
+					     psDevmemXIntMapVRangeToBackingPageIN->uiFlags,
+					     psDevmemXIntMapVRangeToBackingPageIN->
+					     ui32VirtPageOffset);
+
+DevmemXIntMapVRangeToBackingPage_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo2(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysHeapGetMemInfo2IN_UI8,
+				IMG_UINT8 * psPhysHeapGetMemInfo2OUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2IN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2IN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2OUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2OUT_UI8,
+								      0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V2)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfo2IN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfo2IN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfo2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfo2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo2_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfo2IN->peaPhysHeapID,
+		     psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V2 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2);
+	}
+
+	psPhysHeapGetMemInfo2OUT->eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(psConnection, OSGetDevNode(psConnection),
+					psPhysHeapGetMemInfo2IN->ui32PhysHeapCount,
+					eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfo2OUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V2))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+
+PhysHeapGetMemInfo2_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfo2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+
+/*
+ * Register all MM functions with services
+ */
+PVRSRV_ERROR InitMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR,
+			      PVRSRVBridgePMRExportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMREXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMREXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR,
+			      PVRSRVBridgePMRUnexportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID, PVRSRVBridgePMRGetUID,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRGETUID),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRGETUID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRMakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRUnmakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR,
+			      PVRSRVBridgePMRImportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR,
+			      PVRSRVBridgePMRLocalImportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR,
+			      PVRSRVBridgePMRUnrefPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRUNREFPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNREFPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR,
+			      PVRSRVBridgePhysmemNewRamBackedPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE,
+			      PVRSRVBridgeDevmemIntCtxCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY,
+			      PVRSRVBridgeDevmemIntCtxDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE,
+			      PVRSRVBridgeDevmemIntHeapCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY,
+			      PVRSRVBridgeDevmemIntHeapDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR,
+			      PVRSRVBridgeDevmemIntMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR,
+			      PVRSRVBridgeDevmemIntUnmapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE,
+			      PVRSRVBridgeDevmemIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR,
+			      PVRSRVBridgeDevmemIntReserveRangeAndMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM,
+			      PVRSRVBridgeChangeSparseMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_CHANGESPARSEMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID,
+			      PVRSRVBridgeDevmemIsVDevAddrValid, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE,
+			      PVRSRVBridgeDevmemInvalidateFBSCTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT,
+			      PVRSRVBridgeHeapCfgHeapConfigCount, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT,
+			      PVRSRVBridgeHeapCfgHeapCount, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME,
+			      PVRSRVBridgeHeapCfgHeapConfigName, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS,
+			      PVRSRVBridgeHeapCfgHeapDetails, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM,
+			      PVRSRVBridgeDevmemIntRegisterPFNotifyKM, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO,
+			      PVRSRVBridgePhysHeapGetMemInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP,
+			      PVRSRVBridgeGetDefaultPhysicalHeap, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS,
+			      PVRSRVBridgeDevmemGetFaultAddress, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT,
+			      PVRSRVBridgePVRSRVStatsUpdateOOMStat, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES,
+			      PVRSRVBridgeDevmemXIntMapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES,
+			      PVRSRVBridgeDevmemXIntUnmapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE,
+			      PVRSRVBridgeDevmemXIntMapVRangeToBackingPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2,
+			      PVRSRVBridgePhysHeapGetMemInfo2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mm functions with services
+ */
+void DeinitMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM,
+				PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/common_mmextmem_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/common_mmextmem_bridge.h
new file mode 100644
index 000000000000..f52637853325
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/common_mmextmem_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MMEXTMEM_BRIDGE_H
+#define COMMON_MMEXTMEM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM			PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST			(PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0)
+
+/*******************************************
+            PhysmemWrapExtMem
+ *******************************************/
+
+/* Bridge in structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_UINT64 ui64CpuVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM;
+
+/* Bridge out structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM;
+
+#endif /* COMMON_MMEXTMEM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/server_mmextmem_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/server_mmextmem_bridge.c
new file mode 100644
index 000000000000..538d04cdf0e0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/mmextmem_bridge/server_mmextmem_bridge.c
@@ -0,0 +1,166 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "physmem_extmem.h"
+
+#include "common_mmextmem_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemWrapExtMempsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemWrapExtMem(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPhysmemWrapExtMemIN_UI8,
+			      IMG_UINT8 * psPhysmemWrapExtMemOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemOUT_UI8, 0);
+
+	PMR *psPMRPtrInt = NULL;
+
+	psPhysmemWrapExtMemOUT->eError =
+	    PhysmemWrapExtMem(psConnection, OSGetDevNode(psConnection),
+			      psPhysmemWrapExtMemIN->uiSize,
+			      psPhysmemWrapExtMemIN->ui64CpuVAddr,
+			      psPhysmemWrapExtMemIN->uiFlags, &psPMRPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemWrapExtMemOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPhysmemWrapExtMemOUT->hPMRPtr,
+								   (void *)psPMRPtrInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PhysmemWrapExtMempsPMRPtrIntRelease);
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemWrapExtMem_exit:
+
+	if (psPhysmemWrapExtMemOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+
+/*
+ * Register all MMEXTMEM functions with services
+ */
+PVRSRV_ERROR InitMMEXTMEMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM,
+			      PVRSRVBridgePhysmemWrapExtMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mmextmem functions with services
+ */
+void DeinitMMEXTMEMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_bridge.h
new file mode 100644
index 000000000000..2bf967bfcea9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMP_BRIDGE_H
+#define CLIENT_PDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_direct_bridge.c
new file mode 100644
index 000000000000..7c012dbcf21e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/client_pdump_direct_bridge.c
@@ -0,0 +1,151 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpImageDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  psDevmemCtxInt,
+					  ui32StringSize,
+					  puiFileName,
+					  sDataDevAddr,
+					  ui32DataSize,
+					  ui32LogicalWidth,
+					  ui32LogicalHeight,
+					  ui32PhysicalWidth,
+					  ui32PhysicalHeight,
+					  ePixelFormat,
+					  eMemLayout,
+					  eFBCompression,
+					  pui32FBCClearColour,
+					  eeFBCSwizzle,
+					  sHeaderDevAddr, ui32HeaderSize, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpCommentKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			   ui32CommentSize, puiComment, ui32Flags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpSetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpDataDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 psDevmemCtxInt,
+					 ui32StringSize,
+					 puiFileName,
+					 sDataDevAddr,
+					 ui32DataSize,
+					 ui32HeaderType,
+					 ui32ElementType, ui32ElementCount, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/common_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/common_pdump_bridge.h
new file mode 100644
index 000000000000..eb6ab9488a20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/common_pdump_bridge.h
@@ -0,0 +1,155 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMP_BRIDGE_H
+#define COMMON_PDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_BRIDGE_PDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMP_CMD_LAST			(PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3)
+
+/*******************************************
+            PDumpImageDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_DEV_VIRTADDR sHeaderDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_UINT32 *pui32FBCClearColour;
+	const IMG_CHAR *puiFileName;
+	IMG_FB_COMPRESSION eFBCompression;
+	IMG_MEMLAYOUT eMemLayout;
+	PDUMP_PIXEL_FORMAT ePixelFormat;
+	PDUMP_FBC_SWIZZLE eeFBCSwizzle;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32HeaderSize;
+	IMG_UINT32 ui32LogicalHeight;
+	IMG_UINT32 ui32LogicalWidth;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PhysicalHeight;
+	IMG_UINT32 ui32PhysicalWidth;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR;
+
+/* Bridge out structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR;
+
+/*******************************************
+            PVRSRVPDumpComment
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT_TAG
+{
+	IMG_CHAR *puiComment;
+	IMG_UINT32 ui32CommentSize;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT;
+
+/* Bridge out structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT;
+
+/*******************************************
+            PVRSRVPDumpSetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME_TAG
+{
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME;
+
+/*******************************************
+            PDumpDataDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32ElementCount;
+	IMG_UINT32 ui32ElementType;
+	IMG_UINT32 ui32HeaderType;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR;
+
+/* Bridge out structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR;
+
+#endif /* COMMON_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/server_pdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/server_pdump_bridge.c
new file mode 100644
index 000000000000..6b3baf668710
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdump_bridge/server_pdump_bridge.c
@@ -0,0 +1,571 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+#include "common_pdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(4 <= IMG_UINT32_MAX, "4 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpImageDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpImageDescriptorIN_UI8,
+				 IMG_UINT8 * psPDumpImageDescriptorOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpImageDescriptorIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *)
+	    IMG_OFFSET_ADDR(psPDumpImageDescriptorOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpImageDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+	IMG_UINT32 *ui32FBCClearColourInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) 4 * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psPDumpImageDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpImageDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpImageDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpImageDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpImageDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psPDumpImageDescriptorIN->puiFileName,
+		     psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	{
+		ui32FBCClearColourInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 4 * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (4 * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FBCClearColourInt,
+		     (const void __user *)psPDumpImageDescriptorIN->pui32FBCClearColour,
+		     4 * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpImageDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpImageDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpImageDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpImageDescriptorOUT->eError =
+	    DevmemIntPDumpImageDescriptor(psConnection, OSGetDevNode(psConnection),
+					  psDevmemCtxInt,
+					  psPDumpImageDescriptorIN->ui32StringSize,
+					  uiFileNameInt,
+					  psPDumpImageDescriptorIN->sDataDevAddr,
+					  psPDumpImageDescriptorIN->ui32DataSize,
+					  psPDumpImageDescriptorIN->ui32LogicalWidth,
+					  psPDumpImageDescriptorIN->ui32LogicalHeight,
+					  psPDumpImageDescriptorIN->ui32PhysicalWidth,
+					  psPDumpImageDescriptorIN->ui32PhysicalHeight,
+					  psPDumpImageDescriptorIN->ePixelFormat,
+					  psPDumpImageDescriptorIN->eMemLayout,
+					  psPDumpImageDescriptorIN->eFBCompression,
+					  ui32FBCClearColourInt,
+					  psPDumpImageDescriptorIN->eeFBCSwizzle,
+					  psPDumpImageDescriptorIN->sHeaderDevAddr,
+					  psPDumpImageDescriptorIN->ui32HeaderSize,
+					  psPDumpImageDescriptorIN->ui32PDumpFlags);
+
+PDumpImageDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpImageDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_COMMENT_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_COMMENT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpComment(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPVRSRVPDumpCommentIN_UI8,
+			       IMG_UINT8 * psPVRSRVPDumpCommentOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentOUT_UI8,
+								     0);
+
+	IMG_CHAR *uiCommentInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPVRSRVPDumpCommentIN->ui32CommentSize > PVRSRV_PDUMP_MAX_COMMENT_SIZE))
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPVRSRVPDumpCommentIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPVRSRVPDumpCommentIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PVRSRVPDumpComment_exit;
+			}
+		}
+	}
+
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize != 0)
+	{
+		uiCommentInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiCommentInt, (const void __user *)psPVRSRVPDumpCommentIN->puiComment,
+		     psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PVRSRVPDumpComment_exit;
+		}
+		((IMG_CHAR *)
+		 uiCommentInt)[(psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPVRSRVPDumpCommentOUT->eError =
+	    PDumpCommentKM(psConnection, OSGetDevNode(psConnection),
+			   psPVRSRVPDumpCommentIN->ui32CommentSize,
+			   uiCommentInt, psPVRSRVPDumpCommentIN->ui32Flags);
+
+PVRSRVPDumpComment_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPVRSRVPDumpCommentOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameOUT_UI8,
+								      0);
+
+	psPVRSRVPDumpSetFrameOUT->eError =
+	    PDumpSetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    psPVRSRVPDumpSetFrameIN->ui32Frame);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpDataDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPDumpDataDescriptorIN_UI8,
+				IMG_UINT8 * psPDumpDataDescriptorOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpDataDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPDumpDataDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpDataDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpDataDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpDataDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpDataDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPDumpDataDescriptorIN->puiFileName,
+		     psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpDataDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpDataDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpDataDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpDataDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpDataDescriptorOUT->eError =
+	    DevmemIntPDumpDataDescriptor(psConnection, OSGetDevNode(psConnection),
+					 psDevmemCtxInt,
+					 psPDumpDataDescriptorIN->ui32StringSize,
+					 uiFileNameInt,
+					 psPDumpDataDescriptorIN->sDataDevAddr,
+					 psPDumpDataDescriptorIN->ui32DataSize,
+					 psPDumpDataDescriptorIN->ui32HeaderType,
+					 psPDumpDataDescriptorIN->ui32ElementType,
+					 psPDumpDataDescriptorIN->ui32ElementCount,
+					 psPDumpDataDescriptorIN->ui32PDumpFlags);
+
+PDumpDataDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpDataDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPBridge(void);
+void DeinitPDUMPBridge(void);
+
+/*
+ * Register all PDUMP functions with services
+ */
+PVRSRV_ERROR InitPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR,
+			      PVRSRVBridgePDumpImageDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT,
+			      PVRSRVBridgePVRSRVPDumpComment, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME,
+			      PVRSRVBridgePVRSRVPDumpSetFrame, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR,
+			      PVRSRVBridgePDumpDataDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdump functions with services
+ */
+void DeinitPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..a2ba27b88b30
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_bridge.h
@@ -0,0 +1,74 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPCTRL_BRIDGE_H
+#define CLIENT_PDUMPCTRL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpctrl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge);
+
+#endif /* CLIENT_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
new file mode 100644
index 000000000000..a2bc30360aea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
@@ -0,0 +1,109 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpctrl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpctrl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpGetStateKM(pui64State);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpGetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), pui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpSetDefaultCaptureParamsKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32Mode,
+					   ui32Start,
+					   ui32End,
+					   ui32Interval, ui32MaxParamFileSize, ui32AutoTermTimeout);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpIsLastCaptureFrameKM(pbpbIsLastCaptureFrame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpForceCaptureStopKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge));
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/common_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/common_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..f710e4c06131
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/common_pdumpctrl_bridge.h
@@ -0,0 +1,150 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPCTRL_BRIDGE_H
+#define COMMON_PDUMPCTRL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_LAST			(PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4)
+
+/*******************************************
+            PVRSRVPDumpGetState
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE;
+
+/* Bridge out structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT64 ui64State;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE;
+
+/*******************************************
+            PVRSRVPDumpGetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME;
+
+/*******************************************
+            PVRSRVPDumpSetDefaultCaptureParams
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	IMG_UINT32 ui32AutoTermTimeout;
+	IMG_UINT32 ui32End;
+	IMG_UINT32 ui32Interval;
+	IMG_UINT32 ui32MaxParamFileSize;
+	IMG_UINT32 ui32Mode;
+	IMG_UINT32 ui32Start;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/* Bridge out structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/*******************************************
+            PVRSRVPDumpIsLastCaptureFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/* Bridge out structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bpbIsLastCaptureFrame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/*******************************************
+            PVRSRVPDumpForceCaptureStop
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP;
+
+/* Bridge out structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP;
+
+#endif /* COMMON_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/server_pdumpctrl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/server_pdumpctrl_bridge.c
new file mode 100644
index 000000000000..2d5578cf83f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpctrl_bridge/server_pdumpctrl_bridge.c
@@ -0,0 +1,252 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pdump_km.h"
+
+#include "common_pdumpctrl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetState(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetStateIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetStateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetStateIN);
+
+	psPVRSRVPDumpGetStateOUT->eError = PDumpGetStateKM(&psPVRSRVPDumpGetStateOUT->ui64State);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetFrameIN);
+
+	psPVRSRVPDumpGetFrameOUT->eError =
+	    PDumpGetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    &psPVRSRVPDumpGetFrameOUT->ui32Frame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams(IMG_UINT32 ui32DispatchTableEntry,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8,
+					       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *psPVRSRVPDumpSetDefaultCaptureParamsIN
+	    =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS
+	    *psPVRSRVPDumpSetDefaultCaptureParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8, 0);
+
+	psPVRSRVPDumpSetDefaultCaptureParamsOUT->eError =
+	    PDumpSetDefaultCaptureParamsKM(psConnection, OSGetDevNode(psConnection),
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Mode,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Start,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32End,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Interval,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32MaxParamFileSize,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32AutoTermTimeout);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameIN_UI8,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpIsLastCaptureFrameIN);
+
+	psPVRSRVPDumpIsLastCaptureFrameOUT->eError =
+	    PDumpIsLastCaptureFrameKM(&psPVRSRVPDumpIsLastCaptureFrameOUT->bpbIsLastCaptureFrame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpForceCaptureStop(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopIN_UI8,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpForceCaptureStopIN);
+
+	psPVRSRVPDumpForceCaptureStopOUT->eError =
+	    PDumpForceCaptureStopKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pPDUMPCTRLBridgeLock;
+
+PVRSRV_ERROR InitPDUMPCTRLBridge(void);
+void DeinitPDUMPCTRLBridge(void);
+
+/*
+ * Register all PDUMPCTRL functions with services
+ */
+PVRSRV_ERROR InitPDUMPCTRLBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pPDUMPCTRLBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE,
+			      PVRSRVBridgePVRSRVPDumpGetState, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME,
+			      PVRSRVBridgePVRSRVPDumpGetFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS,
+			      PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams, pPDUMPCTRLBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME,
+			      PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP,
+			      PVRSRVBridgePVRSRVPDumpForceCaptureStop, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpctrl functions with services
+ */
+void DeinitPDUMPCTRLBridge(void)
+{
+	OSLockDestroy(pPDUMPCTRLBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_bridge.h
new file mode 100644
index 000000000000..a4368477e225
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_bridge.h
@@ -0,0 +1,125 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPMM_BRIDGE_H
+#define CLIENT_PDUMPMM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpmm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
new file mode 100644
index 000000000000..1358f75a7537
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpmm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpmm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMem(psPMRInt, uiOffset, uiSize, ui32PDumpFlags, bbZero);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue32(psPMRInt, uiOffset, ui32Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue64(psPMRInt, uiOffset, ui64Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       uiOffset, uiSize, ui32ArraySize, puiFileName, ui32uiFileOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  uiOffset,
+				  ui32MemspaceNameLen,
+				  puiMemspaceName,
+				  ui32SymbolicAddrLen,
+				  puiSymbolicAddr, puiNewOffset, puiNextSymName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpPol32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpCheck32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpCBP(psPMRInt, uiReadOffset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError =
+	    DevmemIntPDumpSaveToFileVirtual(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    psDevmemServerContextInt,
+					    sAddress,
+					    uiSize,
+					    ui32ArraySize,
+					    puiFileName, ui32FileOffset, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/common_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/common_pdumpmm_bridge.h
new file mode 100644
index 000000000000..2020cbf5b30b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/common_pdumpmm_bridge.h
@@ -0,0 +1,259 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPMM_BRIDGE_H
+#define COMMON_PDUMPMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_LAST			(PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8)
+
+/*******************************************
+            PMRPDumpLoadMem
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_BOOL bbZero;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM;
+
+/* Bridge out structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM;
+
+/*******************************************
+            PMRPDumpLoadMemValue32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32;
+
+/* Bridge out structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32;
+
+/*******************************************
+            PMRPDumpLoadMemValue64
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	IMG_UINT64 ui64Value;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64;
+
+/* Bridge out structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64;
+
+/*******************************************
+            PMRPDumpSaveToFile
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32uiFileOffset;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE;
+
+/* Bridge out structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE;
+
+/*******************************************
+            PMRPDumpSymbolicAddr
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	IMG_UINT32 ui32MemspaceNameLen;
+	IMG_UINT32 ui32SymbolicAddrLen;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR;
+
+/* Bridge out structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiNewOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR;
+
+/*******************************************
+            PMRPDumpPol32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPPOL32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPPOL32;
+
+/* Bridge out structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32;
+
+/*******************************************
+            PMRPDumpCheck32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32;
+
+/* Bridge out structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32;
+
+/*******************************************
+            PMRPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiReadOffset;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCBP;
+
+/* Bridge out structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCBP;
+
+/*******************************************
+            DevmemIntPDumpSaveToFileVirtual
+ *******************************************/
+
+/* Bridge in structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hDevmemServerContext;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32FileOffset;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+/* Bridge out structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+#endif /* COMMON_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/server_pdumpmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/server_pdumpmm_bridge.c
new file mode 100644
index 000000000000..38e1a6610fd5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pdumpmm_bridge/server_pdumpmm_bridge.c
@@ -0,0 +1,958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+#include "common_pdumpmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpLoadMemIN_UI8,
+			    IMG_UINT8 * psPMRPDumpLoadMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *psPMRPDumpLoadMemIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *psPMRPDumpLoadMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemOUT->eError =
+	    PMRPDumpLoadMem(psPMRInt,
+			    psPMRPDumpLoadMemIN->uiOffset,
+			    psPMRPDumpLoadMemIN->uiSize,
+			    psPMRPDumpLoadMemIN->ui32PDumpFlags, psPMRPDumpLoadMemIN->bbZero);
+
+PMRPDumpLoadMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue32(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PMRPDumpLoadMemValue32(psPMRInt,
+				   psPMRPDumpLoadMemValue32IN->uiOffset,
+				   psPMRPDumpLoadMemValue32IN->ui32Value,
+				   psPMRPDumpLoadMemValue32IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue64(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue64IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue64OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue64_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PMRPDumpLoadMemValue64(psPMRInt,
+				   psPMRPDumpLoadMemValue64IN->uiOffset,
+				   psPMRPDumpLoadMemValue64IN->ui64Value,
+				   psPMRPDumpLoadMemValue64IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue64_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSaveToFile(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRPDumpSaveToFileIN_UI8,
+			       IMG_UINT8 * psPMRPDumpSaveToFileOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileOUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSaveToFileIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPMRPDumpSaveToFileIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSaveToFileIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSaveToFileIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSaveToFile_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPMRPDumpSaveToFileIN->puiFileName,
+		     psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSaveToFile_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSaveToFileOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSaveToFileOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSaveToFile_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSaveToFileOUT->eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       psPMRPDumpSaveToFileIN->uiOffset,
+			       psPMRPDumpSaveToFileIN->uiSize,
+			       psPMRPDumpSaveToFileIN->ui32ArraySize,
+			       uiFileNameInt, psPMRPDumpSaveToFileIN->ui32uiFileOffset);
+
+PMRPDumpSaveToFile_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSaveToFileOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSymbolicAddr(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrIN_UI8,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *) IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *)
+	    IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSymbolicAddrIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *puiMemspaceNameInt = NULL;
+	IMG_CHAR *puiSymbolicAddrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) + 0;
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen > PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen > PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	psPMRPDumpSymbolicAddrOUT->puiMemspaceName = psPMRPDumpSymbolicAddrIN->puiMemspaceName;
+	psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr = psPMRPDumpSymbolicAddrIN->puiSymbolicAddr;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSymbolicAddrIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSymbolicAddrIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSymbolicAddr_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen != 0)
+	{
+		puiMemspaceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR);
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen != 0)
+	{
+		puiSymbolicAddrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  psPMRPDumpSymbolicAddrIN->uiOffset,
+				  psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen,
+				  puiMemspaceNameInt,
+				  psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen,
+				  puiSymbolicAddrInt,
+				  &psPMRPDumpSymbolicAddrOUT->uiNewOffset,
+				  &psPMRPDumpSymbolicAddrOUT->uiNextSymName);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiMemspaceNameInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiMemspaceName,
+		      puiMemspaceNameInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiSymbolicAddrInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr,
+		      puiSymbolicAddrInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+PMRPDumpSymbolicAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSymbolicAddrOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpPol32(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psPMRPDumpPol32IN_UI8,
+			  IMG_UINT8 * psPMRPDumpPol32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *psPMRPDumpPol32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *psPMRPDumpPol32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpPol32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpPol32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpPol32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpPol32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpPol32OUT->eError =
+	    PMRPDumpPol32(psPMRInt,
+			  psPMRPDumpPol32IN->uiOffset,
+			  psPMRPDumpPol32IN->ui32Value,
+			  psPMRPDumpPol32IN->ui32Mask,
+			  psPMRPDumpPol32IN->eOperator, psPMRPDumpPol32IN->ui32PDumpFlags);
+
+PMRPDumpPol32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCheck32(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpCheck32IN_UI8,
+			    IMG_UINT8 * psPMRPDumpCheck32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *psPMRPDumpCheck32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *psPMRPDumpCheck32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCheck32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCheck32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCheck32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCheck32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCheck32OUT->eError =
+	    PMRPDumpCheck32(psPMRInt,
+			    psPMRPDumpCheck32IN->uiOffset,
+			    psPMRPDumpCheck32IN->ui32Value,
+			    psPMRPDumpCheck32IN->ui32Mask,
+			    psPMRPDumpCheck32IN->eOperator, psPMRPDumpCheck32IN->ui32PDumpFlags);
+
+PMRPDumpCheck32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRPDumpCBPIN_UI8,
+			IMG_UINT8 * psPMRPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCBP *psPMRPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *psPMRPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCBPIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCBPOUT->eError =
+	    PMRPDumpCBP(psPMRInt,
+			psPMRPDumpCBPIN->uiReadOffset,
+			psPMRPDumpCBPIN->uiWriteOffset,
+			psPMRPDumpCBPIN->uiPacketSize, psPMRPDumpCBPIN->uiBufferSize);
+
+PMRPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualIN_UI8,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerContext = psDevmemIntPDumpSaveToFileVirtualIN->hDevmemServerContext;
+	DEVMEMINT_CTX *psDevmemServerContextInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (unlikely
+	    (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevmemIntPDumpSaveToFileVirtualIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevmemIntPDumpSaveToFileVirtualIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevmemIntPDumpSaveToFileVirtual_exit;
+			}
+		}
+	}
+
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psDevmemIntPDumpSaveToFileVirtualIN->puiFileName,
+		     psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevmemIntPDumpSaveToFileVirtual_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize *
+				 sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerContextInt,
+				       hDevmemServerContext,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntPDumpSaveToFileVirtualOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    DevmemIntPDumpSaveToFileVirtual(psConnection, OSGetDevNode(psConnection),
+					    psDevmemServerContextInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->sAddress,
+					    psDevmemIntPDumpSaveToFileVirtualIN->uiSize,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize,
+					    uiFileNameInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32FileOffset,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32PDumpFlags);
+
+DevmemIntPDumpSaveToFileVirtual_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevmemIntPDumpSaveToFileVirtualOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPMMBridge(void);
+void DeinitPDUMPMMBridge(void);
+
+/*
+ * Register all PDUMPMM functions with services
+ */
+PVRSRV_ERROR InitPDUMPMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM,
+			      PVRSRVBridgePMRPDumpLoadMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32,
+			      PVRSRVBridgePMRPDumpLoadMemValue32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64,
+			      PVRSRVBridgePMRPDumpLoadMemValue64, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE,
+			      PVRSRVBridgePMRPDumpSaveToFile, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR,
+			      PVRSRVBridgePMRPDumpSymbolicAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32,
+			      PVRSRVBridgePMRPDumpPol32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPPOL32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32,
+			      PVRSRVBridgePMRPDumpCheck32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP,
+			      PVRSRVBridgePMRPDumpCBP, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+			      PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL,
+			      PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpmm functions with services
+ */
+void DeinitPDUMPMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_bridge.h
new file mode 100644
index 000000000000..2cfafd5a83e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PVRTL_BRIDGE_H
+#define CLIENT_PVRTL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pvrtl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams,
+						  IMG_UINT32 * pui32NumFound);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin,
+						IMG_UINT32 * pui32Available);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data);
+
+#endif /* CLIENT_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_direct_bridge.c
new file mode 100644
index 000000000000..fa2fbed9595b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/client_pvrtl_direct_bridge.c
@@ -0,0 +1,175 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pvrtl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pvrtl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#include "tlserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerOpenStreamKM(puiName, ui32Mode, &psSDInt, &psTLPMRInt);
+
+	*phSD = psSDInt;
+	*phTLPMR = psTLPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCloseStreamKM(psSDInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerAcquireDataKM(psSDInt, pui32ReadOffset, pui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerReleaseDataKM(psSDInt, ui32ReadOffset, ui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams, IMG_UINT32 * pui32NumFound)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerDiscoverStreamsKM(puiNamePattern, ui32Size, puiStreams, pui32NumFound);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin, IMG_UINT32 * pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    pui32BufferOffset, ui32Size, ui32SizeMin, pui32Available);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCommitStreamKM(psSDInt, ui32ReqSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerWriteDataKM(psSDInt, ui32Size, pui8Data);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/common_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/common_pvrtl_bridge.h
new file mode 100644
index 000000000000..edc822375222
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/common_pvrtl_bridge.h
@@ -0,0 +1,214 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PVRTL_BRIDGE_H
+#define COMMON_PVRTL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#define PVRSRV_BRIDGE_PVRTL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PVRTL_TLWRITEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PVRTL_CMD_LAST			(PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7)
+
+/*******************************************
+            TLOpenStream
+ *******************************************/
+
+/* Bridge in structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_IN_TLOPENSTREAM_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_UINT32 ui32Mode;
+} __packed PVRSRV_BRIDGE_IN_TLOPENSTREAM;
+
+/* Bridge out structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLOPENSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_HANDLE hTLPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLOPENSTREAM;
+
+/*******************************************
+            TLCloseStream
+ *******************************************/
+
+/* Bridge in structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCLOSESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLCLOSESTREAM;
+
+/* Bridge out structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCLOSESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCLOSESTREAM;
+
+/*******************************************
+            TLAcquireData
+ *******************************************/
+
+/* Bridge in structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_IN_TLACQUIREDATA_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLACQUIREDATA;
+
+/* Bridge out structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_OUT_TLACQUIREDATA_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLACQUIREDATA;
+
+/*******************************************
+            TLReleaseData
+ *******************************************/
+
+/* Bridge in structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_IN_TLRELEASEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_IN_TLRELEASEDATA;
+
+/* Bridge out structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_OUT_TLRELEASEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLRELEASEDATA;
+
+/*******************************************
+            TLDiscoverStreams
+ *******************************************/
+
+/* Bridge in structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS_TAG
+{
+	const IMG_CHAR *puiNamePattern;
+	IMG_CHAR *puiStreams;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS;
+
+/* Bridge out structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS_TAG
+{
+	IMG_CHAR *puiStreams;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumFound;
+} __packed PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS;
+
+/*******************************************
+            TLReserveStream
+ *******************************************/
+
+/* Bridge in structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_IN_TLRESERVESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32SizeMin;
+} __packed PVRSRV_BRIDGE_IN_TLRESERVESTREAM;
+
+/* Bridge out structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLRESERVESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Available;
+	IMG_UINT32 ui32BufferOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLRESERVESTREAM;
+
+/*******************************************
+            TLCommitStream
+ *******************************************/
+
+/* Bridge in structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCOMMITSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReqSize;
+} __packed PVRSRV_BRIDGE_IN_TLCOMMITSTREAM;
+
+/* Bridge out structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM;
+
+/*******************************************
+            TLWriteData
+ *******************************************/
+
+/* Bridge in structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_IN_TLWRITEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLWRITEDATA;
+
+/* Bridge out structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_OUT_TLWRITEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLWRITEDATA;
+
+#endif /* COMMON_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/server_pvrtl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/server_pvrtl_bridge.c
new file mode 100644
index 000000000000..9facf794011e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/pvrtl_bridge/server_pvrtl_bridge.c
@@ -0,0 +1,827 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "tlserver.h"
+
+#include "common_pvrtl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _TLOpenStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = TLServerCloseStreamKM((TL_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLOpenStream(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psTLOpenStreamIN_UI8,
+			 IMG_UINT8 * psTLOpenStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLOPENSTREAM *psTLOpenStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLOPENSTREAM *psTLOpenStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamOUT_UI8, 0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	psTLOpenStreamOUT->hSD = NULL;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLOpenStreamOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLOpenStream_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLOpenStreamIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLOpenStreamIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLOpenStreamOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLOpenStream_exit;
+			}
+		}
+	}
+
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psTLOpenStreamIN->puiName,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLOpenStreamOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLOpenStream_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) - 1] =
+		    '\0';
+	}
+
+	psTLOpenStreamOUT->eError =
+	    TLServerOpenStreamKM(uiNameInt, psTLOpenStreamIN->ui32Mode, &psSDInt, &psTLPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		goto TLOpenStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psTLOpenStreamOUT->hSD,
+							      (void *)psSDInt,
+							      PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _TLOpenStreampsSDIntRelease);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								 &psTLOpenStreamOUT->hTLPMR,
+								 (void *)psTLPMRInt,
+								 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 psTLOpenStreamOUT->hSD);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLOpenStream_exit:
+
+	if (psTLOpenStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psTLOpenStreamOUT->hSD)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psTLOpenStreamOUT->hSD,
+							     PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSDInt)
+		{
+			TLServerCloseStreamKM(psSDInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLOpenStreamOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCloseStream(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLCloseStreamIN_UI8,
+			  IMG_UINT8 * psTLCloseStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCLOSESTREAM *psTLCloseStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *psTLCloseStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLCloseStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psTLCloseStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	if (unlikely((psTLCloseStreamOUT->eError != PVRSRV_OK) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psTLCloseStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCloseStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLCloseStream_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLAcquireData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLAcquireDataIN_UI8,
+			  IMG_UINT8 * psTLAcquireDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLACQUIREDATA *psTLAcquireDataIN =
+	    (PVRSRV_BRIDGE_IN_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLACQUIREDATA *psTLAcquireDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLAcquireDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLAcquireDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLAcquireDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLAcquireData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLAcquireDataOUT->eError =
+	    TLServerAcquireDataKM(psSDInt,
+				  &psTLAcquireDataOUT->ui32ReadOffset,
+				  &psTLAcquireDataOUT->ui32ReadLen);
+
+TLAcquireData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReleaseData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLReleaseDataIN_UI8,
+			  IMG_UINT8 * psTLReleaseDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRELEASEDATA *psTLReleaseDataIN =
+	    (PVRSRV_BRIDGE_IN_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRELEASEDATA *psTLReleaseDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReleaseDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReleaseDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReleaseDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReleaseData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReleaseDataOUT->eError =
+	    TLServerReleaseDataKM(psSDInt,
+				  psTLReleaseDataIN->ui32ReadOffset,
+				  psTLReleaseDataIN->ui32ReadLen);
+
+TLReleaseData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLDiscoverStreams(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psTLDiscoverStreamsIN_UI8,
+			      IMG_UINT8 * psTLDiscoverStreamsOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *psTLDiscoverStreamsIN =
+	    (PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *psTLDiscoverStreamsOUT =
+	    (PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsOUT_UI8, 0);
+
+	IMG_CHAR *uiNamePatternInt = NULL;
+	IMG_CHAR *puiStreamsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) + 0;
+
+	if (psTLDiscoverStreamsIN->ui32Size > PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLDiscoverStreams_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psTLDiscoverStreamsOUT->puiStreams = psTLDiscoverStreamsIN->puiStreams;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLDiscoverStreams_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLDiscoverStreamsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLDiscoverStreamsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLDiscoverStreams_exit;
+			}
+		}
+	}
+
+	{
+		uiNamePatternInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNamePatternInt,
+		     (const void __user *)psTLDiscoverStreamsIN->puiNamePattern,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+		((IMG_CHAR *) uiNamePatternInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) -
+						1] = '\0';
+	}
+	if (psTLDiscoverStreamsIN->ui32Size != 0)
+	{
+		puiStreamsInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR);
+	}
+
+	psTLDiscoverStreamsOUT->eError =
+	    TLServerDiscoverStreamsKM(uiNamePatternInt,
+				      psTLDiscoverStreamsIN->ui32Size,
+				      puiStreamsInt, &psTLDiscoverStreamsOUT->ui32NumFound);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLDiscoverStreamsOUT->eError != PVRSRV_OK))
+	{
+		goto TLDiscoverStreams_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamsInt) && ((psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psTLDiscoverStreamsOUT->puiStreams, puiStreamsInt,
+		      (psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+	}
+
+TLDiscoverStreams_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLDiscoverStreamsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReserveStream(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psTLReserveStreamIN_UI8,
+			    IMG_UINT8 * psTLReserveStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRESERVESTREAM *psTLReserveStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *psTLReserveStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReserveStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReserveStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReserveStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReserveStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReserveStreamOUT->eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    &psTLReserveStreamOUT->ui32BufferOffset,
+				    psTLReserveStreamIN->ui32Size,
+				    psTLReserveStreamIN->ui32SizeMin,
+				    &psTLReserveStreamOUT->ui32Available);
+
+TLReserveStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCommitStream(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psTLCommitStreamIN_UI8,
+			   IMG_UINT8 * psTLCommitStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *psTLCommitStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *psTLCommitStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLCommitStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLCommitStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLCommitStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCommitStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLCommitStreamOUT->eError =
+	    TLServerCommitStreamKM(psSDInt, psTLCommitStreamIN->ui32ReqSize);
+
+TLCommitStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLWriteData(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psTLWriteDataIN_UI8,
+			IMG_UINT8 * psTLWriteDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLWRITEDATA *psTLWriteDataIN =
+	    (PVRSRV_BRIDGE_IN_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLWRITEDATA *psTLWriteDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLWriteDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psTLWriteDataIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLWriteData_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLWriteData_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLWriteDataIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLWriteDataIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLWriteDataOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLWriteData_exit;
+			}
+		}
+	}
+
+	if (psTLWriteDataIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psTLWriteDataIN->pui8Data,
+		     psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psTLWriteDataOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLWriteData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLWriteDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLWriteDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLWriteData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLWriteDataOUT->eError =
+	    TLServerWriteDataKM(psSDInt, psTLWriteDataIN->ui32Size, ui8DataInt);
+
+TLWriteData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLWriteDataOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+
+/*
+ * Register all PVRTL functions with services
+ */
+PVRSRV_ERROR InitPVRTLBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM,
+			      PVRSRVBridgeTLOpenStream, NULL, sizeof(PVRSRV_BRIDGE_IN_TLOPENSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLOPENSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM,
+			      PVRSRVBridgeTLCloseStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCLOSESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCLOSESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA,
+			      PVRSRVBridgeTLAcquireData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLACQUIREDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLACQUIREDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA,
+			      PVRSRVBridgeTLReleaseData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRELEASEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRELEASEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS,
+			      PVRSRVBridgeTLDiscoverStreams, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM,
+			      PVRSRVBridgeTLReserveStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRESERVESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRESERVESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM,
+			      PVRSRVBridgeTLCommitStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCOMMITSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA,
+			      PVRSRVBridgeTLWriteData, NULL, sizeof(PVRSRV_BRIDGE_IN_TLWRITEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLWRITEDATA));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pvrtl functions with services
+ */
+void DeinitPVRTLBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/common_rgxbreakpoint_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/common_rgxbreakpoint_bridge.h
new file mode 100644
index 000000000000..ca5290a88266
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/common_rgxbreakpoint_bridge.h
@@ -0,0 +1,150 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxbreakpoint
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxbreakpoint
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXBREAKPOINT_BRIDGE_H
+#define COMMON_RGXBREAKPOINT_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_LAST			(PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXSetBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT_TAG
+{
+	IMG_UINT64 ui64TempSpillingAddr;
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 eFWDataMaster;
+	IMG_UINT32 ui32BreakpointAddr;
+	IMG_UINT32 ui32DM;
+	IMG_UINT32 ui32HandlerAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT;
+
+/* Bridge out structure for RGXSetBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT;
+
+/*******************************************
+            RGXClearBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXClearBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT;
+
+/* Bridge out structure for RGXClearBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT;
+
+/*******************************************
+            RGXEnableBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXEnableBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT;
+
+/* Bridge out structure for RGXEnableBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT;
+
+/*******************************************
+            RGXDisableBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXDisableBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT;
+
+/* Bridge out structure for RGXDisableBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT;
+
+/*******************************************
+            RGXOverallocateBPRegisters
+ *******************************************/
+
+/* Bridge in structure for RGXOverallocateBPRegisters */
+typedef struct PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS_TAG
+{
+	IMG_UINT32 ui32SharedRegs;
+	IMG_UINT32 ui32TempRegs;
+} __packed PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS;
+
+/* Bridge out structure for RGXOverallocateBPRegisters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS;
+
+#endif /* COMMON_RGXBREAKPOINT_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.c
new file mode 100644
index 000000000000..78fd2b405e92
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.c
@@ -0,0 +1,381 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxbreakpoint
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxbreakpoint
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxbreakpoint.h"
+
+#include "common_rgxbreakpoint_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXBREAKPOINT_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSetBreakpointIN_UI8,
+			     IMG_UINT8 * psRGXSetBreakpointOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT *psRGXSetBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT *) IMG_OFFSET_ADDR(psRGXSetBreakpointIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT *psRGXSetBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT *) IMG_OFFSET_ADDR(psRGXSetBreakpointOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXSetBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXSetBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetBreakpointOUT->eError =
+	    PVRSRVRGXSetBreakpointKM(psConnection, OSGetDevNode(psConnection),
+				     hPrivDataInt,
+				     psRGXSetBreakpointIN->eFWDataMaster,
+				     psRGXSetBreakpointIN->ui64TempSpillingAddr,
+				     psRGXSetBreakpointIN->ui32BreakpointAddr,
+				     psRGXSetBreakpointIN->ui32HandlerAddr,
+				     psRGXSetBreakpointIN->ui32DM);
+
+RGXSetBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXClearBreakpointIN_UI8,
+			       IMG_UINT8 * psRGXClearBreakpointOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT *psRGXClearBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT *) IMG_OFFSET_ADDR(psRGXClearBreakpointIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT *psRGXClearBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT *) IMG_OFFSET_ADDR(psRGXClearBreakpointOUT_UI8,
+								     0);
+
+	IMG_HANDLE hPrivData = psRGXClearBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXClearBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXClearBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXClearBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXClearBreakpointOUT->eError =
+	    PVRSRVRGXClearBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXClearBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXEnableBreakpointIN_UI8,
+				IMG_UINT8 * psRGXEnableBreakpointOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT *psRGXEnableBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXEnableBreakpointIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT *psRGXEnableBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXEnableBreakpointOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPrivData = psRGXEnableBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXEnableBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXEnableBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXEnableBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXEnableBreakpointOUT->eError =
+	    PVRSRVRGXEnableBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXEnableBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXDisableBreakpointIN_UI8,
+				 IMG_UINT8 * psRGXDisableBreakpointOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT *psRGXDisableBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXDisableBreakpointIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT *psRGXDisableBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT *)
+	    IMG_OFFSET_ADDR(psRGXDisableBreakpointOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXDisableBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXDisableBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXDisableBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDisableBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXDisableBreakpointOUT->eError =
+	    PVRSRVRGXDisableBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXDisableBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXOverallocateBPRegisters(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXOverallocateBPRegistersIN_UI8,
+				       IMG_UINT8 * psRGXOverallocateBPRegistersOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS *psRGXOverallocateBPRegistersIN =
+	    (PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS *)
+	    IMG_OFFSET_ADDR(psRGXOverallocateBPRegistersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS *psRGXOverallocateBPRegistersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS *)
+	    IMG_OFFSET_ADDR(psRGXOverallocateBPRegistersOUT_UI8, 0);
+
+	psRGXOverallocateBPRegistersOUT->eError =
+	    PVRSRVRGXOverallocateBPRegistersKM(psConnection, OSGetDevNode(psConnection),
+					       psRGXOverallocateBPRegistersIN->ui32TempRegs,
+					       psRGXOverallocateBPRegistersIN->ui32SharedRegs);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
+
+#if !defined(EXCLUDE_RGXBREAKPOINT_BRIDGE)
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void);
+void DeinitRGXBREAKPOINTBridge(void);
+
+/*
+ * Register all RGXBREAKPOINT functions with services
+ */
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT,
+			      PVRSRVBridgeRGXSetBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT,
+			      PVRSRVBridgeRGXClearBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT,
+			      PVRSRVBridgeRGXEnableBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT,
+			      PVRSRVBridgeRGXDisableBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS,
+			      PVRSRVBridgeRGXOverallocateBPRegisters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxbreakpoint functions with services
+ */
+void DeinitRGXBREAKPOINTBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS);
+
+}
+#else /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXBREAKPOINT_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXBREAKPOINTBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXBREAKPOINTBridge()
+
+#endif /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/common_rgxcmp_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/common_rgxcmp_bridge.h
new file mode 100644
index 000000000000..aa1d2b73985a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/common_rgxcmp_bridge.h
@@ -0,0 +1,309 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXCMP_BRIDGE_H
+#define COMMON_RGXCMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXCMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXCMP_CMD_LAST			(PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11)
+
+/*******************************************
+            RGXCreateComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticComputeContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+	IMG_UINT32 ui32StaticComputeContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT;
+
+/* Bridge out structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT;
+
+/*******************************************
+            RGXDestroyComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT;
+
+/* Bridge out structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT;
+
+/*******************************************
+            RGXFlushComputeData
+ *******************************************/
+
+/* Bridge in structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA;
+
+/* Bridge out structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA;
+
+/*******************************************
+            RGXSendCancelCmd
+ *******************************************/
+
+/* Bridge in structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32FirstIntJobRefToCancel;
+	IMG_INT32 i32LastIntJobRefToCancel;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD;
+
+/* Bridge out structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD;
+
+/*******************************************
+            RGXSetComputeContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/*******************************************
+            RGXNotifyComputeWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXSetComputeContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/*******************************************
+            RGXGetLastDeviceError
+ *******************************************/
+
+/* Bridge in structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR;
+
+/* Bridge out structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Error;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR;
+
+/*******************************************
+            RGXKickTimestampQuery
+ *******************************************/
+
+/* Bridge in structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY;
+
+/* Bridge out structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY;
+
+/*******************************************
+            RGXKickCDM
+ *******************************************/
+
+/* Bridge in structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKCDM_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 *pui32ClientUpdateOffset;
+	IMG_UINT32 *pui32ClientUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phClientUpdateUFOSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumOfWorkgroups;
+	IMG_UINT32 ui32NumOfWorkitems;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKCDM;
+
+/* Bridge out structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKCDM_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	IMG_UINT32 ui32IntJobRef;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKCDM;
+
+/*******************************************
+            RGXCDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXCDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY;
+
+#endif /* COMMON_RGXCMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/server_rgxcmp_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/server_rgxcmp_bridge.c
new file mode 100644
index 000000000000..8910cfb5ba6a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxcmp_bridge/server_rgxcmp_bridge.c
@@ -0,0 +1,1580 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxcompute.h"
+
+#include "common_rgxcmp_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateComputeContextpsComputeContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyComputeContextKM((RGX_SERVER_COMPUTE_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_COMPUTECONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_COMPUTECONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXCreateComputeContextIN_UI8,
+				    IMG_UINT8 * psRGXCreateComputeContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateComputeContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticComputeContextStateInt = NULL;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateComputeContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize >
+	     RGXFWIF_STATIC_COMPUTECONTEXT_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateComputeContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateComputeContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateComputeContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateComputeContextIN->pui8FrameworkCmd,
+		     psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize != 0)
+	{
+		ui8StaticComputeContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		    sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticComputeContextStateInt,
+		     (const void __user *)psRGXCreateComputeContextIN->
+		     pui8StaticComputeContextState,
+		     psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVRGXCreateComputeContextKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXCreateComputeContextIN->i32Priority,
+					    psRGXCreateComputeContextIN->ui32FrameworkCmdSize,
+					    ui8FrameworkCmdInt,
+					    hPrivDataInt,
+					    psRGXCreateComputeContextIN->
+					    ui32StaticComputeContextStateSize,
+					    ui8StaticComputeContextStateInt,
+					    psRGXCreateComputeContextIN->ui32PackedCCBSizeU88,
+					    psRGXCreateComputeContextIN->ui32ContextFlags,
+					    psRGXCreateComputeContextIN->ui64RobustnessAddress,
+					    psRGXCreateComputeContextIN->ui32MaxDeadlineMS,
+					    &psComputeContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psRGXCreateComputeContextOUT->
+									 hComputeContext,
+									 (void *)
+									 psComputeContextInt,
+									 PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _RGXCreateComputeContextpsComputeContextIntRelease);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateComputeContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateComputeContextOUT->eError != PVRSRV_OK)
+	{
+		if (psComputeContextInt)
+		{
+			PVRSRVRGXDestroyComputeContextKM(psComputeContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateComputeContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXDestroyComputeContextIN_UI8,
+				     IMG_UINT8 * psRGXDestroyComputeContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXDestroyComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXDestroyComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyComputeContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyComputeContextIN->
+					      hComputeContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyComputeContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyComputeContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyComputeContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyComputeContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFlushComputeData(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXFlushComputeDataIN_UI8,
+				IMG_UINT8 * psRGXFlushComputeDataOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataIN =
+	    (PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataOUT_UI8,
+								      0);
+
+	IMG_HANDLE hComputeContext = psRGXFlushComputeDataIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXFlushComputeDataOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXFlushComputeData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXFlushComputeDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXFlushComputeDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXFlushComputeData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXFlushComputeDataOUT->eError = PVRSRVRGXFlushComputeDataKM(psComputeContextInt);
+
+RGXFlushComputeData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendCancelCmd(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSendCancelCmdIN_UI8,
+			     IMG_UINT8 * psRGXSendCancelCmdOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *psRGXSendCancelCmdIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *psRGXSendCancelCmdOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSendCancelCmdIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSendCancelCmdOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSendCancelCmd_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendCancelCmdOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendCancelCmd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVRGXSendCancelCmdKM(psComputeContextInt,
+				     psRGXSendCancelCmdIN->i32FirstIntJobRefToCancel,
+				     psRGXSendCancelCmdIN->i32LastIntJobRefToCancel);
+
+RGXSendCancelCmd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPriorityIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVRGXSetComputeContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						 psComputeContextInt,
+						 psRGXSetComputeContextPriorityIN->i32Priority);
+
+RGXSetComputeContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXNotifyComputeWriteOffsetUpdateIN_UI8,
+					      IMG_UINT8 *
+					      psRGXNotifyComputeWriteOffsetUpdateOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXNotifyComputeWriteOffsetUpdateIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXNotifyComputeWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXNotifyComputeWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXNotifyComputeWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXNotifyComputeWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(psComputeContextInt);
+
+RGXNotifyComputeWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPropertyIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVRGXSetComputeContextPropertyKM(psComputeContextInt,
+						 psRGXSetComputeContextPropertyIN->ui32Property,
+						 psRGXSetComputeContextPropertyIN->ui64Input,
+						 &psRGXSetComputeContextPropertyOUT->ui64Output);
+
+RGXSetComputeContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetLastDeviceError(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorIN_UI8,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXGetLastDeviceErrorOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXGetLastDeviceError_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetLastDeviceErrorIN);
+
+	psRGXGetLastDeviceErrorOUT->eError =
+	    PVRSRVRGXGetLastDeviceErrorKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetLastDeviceErrorOUT->ui32Error);
+
+RGXGetLastDeviceError_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTimestampQuery(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXKickTimestampQueryIN_UI8,
+				  IMG_UINT8 * psRGXKickTimestampQueryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickTimestampQueryIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXKickTimestampQueryIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTimestampQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTimestampQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTimestampQuery_exit;
+			}
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTimestampQueryIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickTimestampQueryIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickTimestampQueryIN->pui8DMCmd,
+		     psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTimestampQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTimestampQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVRGXKickTimestampQueryKM(psComputeContextInt,
+					  psRGXKickTimestampQueryIN->hCheckFenceFd,
+					  psRGXKickTimestampQueryIN->hUpdateTimeline,
+					  &psRGXKickTimestampQueryOUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXKickTimestampQueryIN->ui32CmdSize,
+					  ui8DMCmdInt, psRGXKickTimestampQueryIN->ui32ExtJobRef);
+
+RGXKickTimestampQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTimestampQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickCDM(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRGXKickCDMIN_UI8,
+		       IMG_UINT8 * psRGXKickCDMOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKCDM *psRGXKickCDMIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKCDM *psRGXKickCDMOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickCDMIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientUpdateOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickCDMIN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickCDM_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickCDMIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickCDMIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickCDMOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		psClientUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXKickCDMIN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickCDMIN->phClientUpdateUFOSyncPrimBlock,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateOffsetInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateOffset,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateValueInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateValue,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickCDMIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickCDMIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickCDMIN->pui8DMCmd,
+		     psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickCDMIN->pui32SyncPMRFlags,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0, psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickCDMIN->phSyncPMRs,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickCDMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientUpdateUFOSyncPrimBlockInt[i],
+						       hClientUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickCDMOUT->eError =
+	    PVRSRVRGXKickCDMKM(psComputeContextInt,
+			       psRGXKickCDMIN->ui32ClientUpdateCount,
+			       psClientUpdateUFOSyncPrimBlockInt,
+			       ui32ClientUpdateOffsetInt,
+			       ui32ClientUpdateValueInt,
+			       psRGXKickCDMIN->hCheckFenceFd,
+			       psRGXKickCDMIN->hUpdateTimeline,
+			       &psRGXKickCDMOUT->hUpdateFence,
+			       uiUpdateFenceNameInt,
+			       psRGXKickCDMIN->hExportFenceToSignal,
+			       psRGXKickCDMIN->ui32CmdSize,
+			       ui8DMCmdInt,
+			       psRGXKickCDMIN->ui32PDumpFlags,
+			       psRGXKickCDMIN->ui32ExtJobRef,
+			       psRGXKickCDMIN->ui32SyncPMRCount,
+			       ui32SyncPMRFlagsInt,
+			       psSyncPMRsInt,
+			       psRGXKickCDMIN->ui32NumOfWorkgroups,
+			       psRGXKickCDMIN->ui32NumOfWorkitems,
+			       psRGXKickCDMIN->ui64DeadlineInus, &psRGXKickCDMOUT->ui32IntJobRef);
+
+RGXKickCDM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+
+	if (hClientUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientUpdateUFOSyncPrimBlockInt
+			    && psClientUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickCDMOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXCDMGetSharedMemoryIN);
+
+	psRGXCDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXCDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXCDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMGetSharedMemory_exit:
+
+	if (psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXCDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+
+/*
+ * Register all RGXCMP functions with services
+ */
+PVRSRV_ERROR InitRGXCMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT,
+			      PVRSRVBridgeRGXCreateComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT,
+			      PVRSRVBridgeRGXDestroyComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA,
+			      PVRSRVBridgeRGXFlushComputeData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD,
+			      PVRSRVBridgeRGXSendCancelCmd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetComputeContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetComputeContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR,
+			      PVRSRVBridgeRGXGetLastDeviceError, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY,
+			      PVRSRVBridgeRGXKickTimestampQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM,
+			      PVRSRVBridgeRGXKickCDM, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKCDM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKCDM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxcmp functions with services
+ */
+void DeinitRGXCMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
new file mode 100644
index 000000000000..534b8e9536ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
@@ -0,0 +1,339 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXFWDBG_BRIDGE_H
+#define COMMON_RGXFWDBG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "rgx_bridge.h"
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST			(PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15)
+
+/*******************************************
+            RGXFWDebugSetFWLog
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG_TAG
+{
+	IMG_UINT32 ui32RGXFWLogType;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG;
+
+/* Bridge out structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG;
+
+/*******************************************
+            RGXFWDebugDumpFreelistPageList
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/* Bridge out structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/*******************************************
+            RGXFWDebugPowerOff
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF;
+
+/* Bridge out structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF;
+
+/*******************************************
+            RGXFWDebugPowerOn
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON;
+
+/* Bridge out structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON;
+
+/*******************************************
+            RGXFWDebugSetVzConnectionCooldownPeriodInSec
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	IMG_UINT32 ui32VzConne;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/* Bridge out structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/*******************************************
+            RGXFWDebugSetHCSDeadline
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	IMG_UINT32 ui32RGXHCSDeadline;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE;
+
+/* Bridge out structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE;
+
+/*******************************************
+            RGXFWDebugSetDriverPriority
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/* Bridge out structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSlice
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32TSPercentage;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSliceInterval
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	IMG_UINT32 ui32TSIntervalMs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/*******************************************
+            RGXFWDebugSetDriverIsolationGroup
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32IsolationGroup;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/* Bridge out structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/*******************************************
+            RGXFWDebugSetOSNewOnlineState
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32OSNewState;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/* Bridge out structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/*******************************************
+            RGXFWDebugMapGuestHeap
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	IMG_UINT64 ui64ui64GuestHeapBase;
+	IMG_UINT32 ui32DriverID;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP;
+
+/* Bridge out structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP;
+
+/*******************************************
+            RGXFWDebugPHRConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32PHRMode;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE;
+
+/*******************************************
+            RGXFWDebugWdgConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32WdgPeriodUs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE;
+
+/*******************************************
+            RGXCurrentTime
+ *******************************************/
+
+/* Bridge in structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_IN_RGXCURRENTTIME_TAG
+{
+	IMG_UINT8 ui8TimerType;
+} __packed PVRSRV_BRIDGE_IN_RGXCURRENTTIME;
+
+/* Bridge out structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCURRENTTIME_TAG
+{
+	IMG_UINT64 ui64Time;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCURRENTTIME;
+
+/*******************************************
+            RGXFWDebugInjectFault
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT;
+
+/* Bridge out structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT;
+
+#endif /* COMMON_RGXFWDBG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
new file mode 100644
index 000000000000..e5646490aa2d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
@@ -0,0 +1,542 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "rgxfwdbg.h"
+#include "pmr.h"
+#include "rgxtimecorr.h"
+
+#include "common_rgxfwdbg_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetFWLog(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogOUT_UI8,
+								     0);
+
+	psRGXFWDebugSetFWLogOUT->eError =
+	    PVRSRVRGXFWDebugSetFWLogKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXFWDebugSetFWLogIN->ui32RGXFWLogType);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugDumpFreelistPageList(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListIN_UI8,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugDumpFreelistPageListIN);
+
+	psRGXFWDebugDumpFreelistPageListOUT->eError =
+	    PVRSRVRGXFWDebugDumpFreelistPageListKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOff(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugPowerOffIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugPowerOffOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOffIN);
+
+	psRGXFWDebugPowerOffOUT->eError =
+	    PVRSRVRGXFWDebugPowerOffKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOn(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXFWDebugPowerOnIN_UI8,
+			      IMG_UINT8 * psRGXFWDebugPowerOnOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOnIN);
+
+	psRGXFWDebugPowerOnOUT->eError =
+	    PVRSRVRGXFWDebugPowerOnKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec(IMG_UINT32 ui32DispatchTableEntry,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8,
+							 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8, 0);
+
+	psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT->eError =
+	    PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(psConnection,
+								 OSGetDevNode(psConnection),
+								 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN->
+								 ui32VzConne);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetHCSDeadline(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineIN_UI8,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineOUT_UI8, 0);
+
+	psRGXFWDebugSetHCSDeadlineOUT->eError =
+	    PVRSRVRGXFWDebugSetHCSDeadlineKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXFWDebugSetHCSDeadlineIN->ui32RGXHCSDeadline);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityIN_UI8,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverPriorityOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRGXFWDebugSetDriverPriorityIN->ui32DriverID,
+						psRGXFWDebugSetDriverPriorityIN->ui32Priority);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSlice(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceIN_UI8,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceKM(psConnection, OSGetDevNode(psConnection),
+						 psRGXFWDebugSetDriverTimeSliceIN->ui32DriverID,
+						 psRGXFWDebugSetDriverTimeSliceIN->
+						 ui32TSPercentage);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval(IMG_UINT32 ui32DispatchTableEntry,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8,
+						 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceIntervalOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(psConnection, OSGetDevNode(psConnection),
+							 psRGXFWDebugSetDriverTimeSliceIntervalIN->
+							 ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXFWDebugSetDriverIsolationGroupIN_UI8,
+					      IMG_UINT8 *
+					      psRGXFWDebugSetDriverIsolationGroupOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverIsolationGroupOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverIsolationGroupKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32DriverID,
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32IsolationGroup);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetOSNewOnlineState(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateIN_UI8,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateOUT_UI8, 0);
+
+	psRGXFWDebugSetOSNewOnlineStateOUT->eError =
+	    PVRSRVRGXFWDebugSetOSNewOnlineStateKM(psConnection, OSGetDevNode(psConnection),
+						  psRGXFWDebugSetOSNewOnlineStateIN->ui32DriverID,
+						  psRGXFWDebugSetOSNewOnlineStateIN->
+						  ui32OSNewState);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugMapGuestHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapOUT_UI8, 0);
+
+	psRGXFWDebugMapGuestHeapOUT->eError =
+	    PVRSRVRGXFWDebugMapGuestHeapKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugMapGuestHeapIN->ui32DriverID,
+					   psRGXFWDebugMapGuestHeapIN->ui64ui64GuestHeapBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPHRConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureOUT_UI8, 0);
+
+	psRGXFWDebugPHRConfigureOUT->eError =
+	    PVRSRVRGXFWDebugPHRConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugPHRConfigureIN->ui32ui32PHRMode);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugWdgConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureOUT_UI8, 0);
+
+	psRGXFWDebugWdgConfigureOUT->eError =
+	    PVRSRVRGXFWDebugWdgConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugWdgConfigureIN->ui32ui32WdgPeriodUs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCurrentTime(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psRGXCurrentTimeIN_UI8,
+			   IMG_UINT8 * psRGXCurrentTimeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCURRENTTIME *psRGXCurrentTimeIN =
+	    (PVRSRV_BRIDGE_IN_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *psRGXCurrentTimeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeOUT_UI8, 0);
+
+	psRGXCurrentTimeOUT->eError =
+	    PVRSRVRGXCurrentTime(psConnection, OSGetDevNode(psConnection),
+				 psRGXCurrentTimeIN->ui8TimerType, &psRGXCurrentTimeOUT->ui64Time);
+
+	return 0;
+}
+
+#define PVRSRVBridgeRGXFWDebugInjectFault NULL
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+
+/*
+ * Register all RGXFWDBG functions with services
+ */
+PVRSRV_ERROR InitRGXFWDBGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG,
+			      PVRSRVBridgeRGXFWDebugSetFWLog, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST,
+			      PVRSRVBridgeRGXFWDebugDumpFreelistPageList, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF,
+			      PVRSRVBridgeRGXFWDebugPowerOff, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON,
+			      PVRSRVBridgeRGXFWDebugPowerOn, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC,
+			      PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC),
+			      sizeof
+			      (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE,
+			      PVRSRVBridgeRGXFWDebugSetHCSDeadline, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY,
+			      PVRSRVBridgeRGXFWDebugSetDriverPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSlice, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP,
+			      PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE,
+			      PVRSRVBridgeRGXFWDebugSetOSNewOnlineState, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP,
+			      PVRSRVBridgeRGXFWDebugMapGuestHeap, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugPHRConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugWdgConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME,
+			      PVRSRVBridgeRGXCurrentTime, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCURRENTTIME),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCURRENTTIME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT,
+			      PVRSRVBridgeRGXFWDebugInjectFault, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxfwdbg functions with services
+ */
+void DeinitRGXFWDBGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/common_rgxhwperf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/common_rgxhwperf_bridge.h
new file mode 100644
index 000000000000..d1d664f2d05f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/common_rgxhwperf_bridge.h
@@ -0,0 +1,302 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXHWPERF_BRIDGE_H
+#define COMMON_RGXHWPERF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_hwperf.h"
+
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST			(PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+12)
+
+/*******************************************
+            RGXGetConfiguredHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/*******************************************
+            RGXGetEnabledHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS;
+
+/* Bridge out structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32BlockCount;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetHWPerfTimeStamp
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP;
+
+/* Bridge out structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT64 ui64TimeStamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP;
+
+/*******************************************
+            RGXCtrlHWPerf
+ *******************************************/
+
+/* Bridge in structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_IN_RGXCTRLHWPERF_TAG
+{
+	IMG_UINT64 ui64Mask;
+	IMG_UINT32 ui32StreamId;
+	IMG_BOOL bToggle;
+} __packed PVRSRV_BRIDGE_IN_RGXCTRLHWPERF;
+
+/* Bridge out structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF;
+
+/*******************************************
+            RGXGetHWPerfBvncFeatureFlags
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/* Bridge out structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	RGX_HWPERF_BVNC sBVNC;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/*******************************************
+            RGXControlHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	IMG_UINT16 *pui16BlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_BOOL bEnable;
+} __packed PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS;
+
+/* Bridge out structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS;
+
+/*******************************************
+            RGXConfigMuxHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXConfigMuxHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXConfigMuxHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS;
+
+/*******************************************
+            RGXConfigCustomCounters
+ *******************************************/
+
+/* Bridge in structure for RGXConfigCustomCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS_TAG
+{
+	IMG_UINT32 *pui32CustomCounterIDs;
+	IMG_UINT16 ui16CustomBlockID;
+	IMG_UINT16 ui16NumCustomCounters;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS;
+
+/* Bridge out structure for RGXConfigCustomCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS;
+
+/*******************************************
+            RGXConfigureHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_UINT32 ui32CtrlWord;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS;
+
+/* Bridge out structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetConfiguredHWPerfMuxCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfMuxCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfMuxCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS;
+
+/*******************************************
+            RGXOpenHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXCloseHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXWriteHWPerfClientEvent
+ *******************************************/
+
+/* Bridge in structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT;
+
+/* Bridge out structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT;
+
+#endif /* COMMON_RGXHWPERF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/server_rgxhwperf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/server_rgxhwperf_bridge.c
new file mode 100644
index 000000000000..fc8d05d6036a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxhwperf_bridge/server_rgxhwperf_bridge.c
@@ -0,0 +1,1292 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxhwperf.h"
+#include "rgx_fwif_km.h"
+
+#include "common_rgxhwperf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersIN_UI8,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters =
+	    psRGXGetConfiguredHWPerfCountersIN->psConfiguredCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredCountersInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+						   psRGXGetConfiguredHWPerfCountersIN->ui32BlockID,
+						   psConfiguredCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters,
+		      psConfiguredCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetEnabledHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksIN_UI8,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT32 *pui32EnabledBlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) + 0;
+
+	psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs =
+	    psRGXGetEnabledHWPerfBlocksIN->pui32EnabledBlockIDs;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetEnabledHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXGetEnabledHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetEnabledHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		pui32EnabledBlockIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32);
+	}
+
+	psRGXGetEnabledHWPerfBlocksOUT->eError =
+	    PVRSRVRGXGetEnabledHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					      psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen,
+					      &psRGXGetEnabledHWPerfBlocksOUT->ui32BlockCount,
+					      pui32EnabledBlockIDsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetEnabledHWPerfBlocksOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32EnabledBlockIDsInt) &&
+	    ((psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs,
+		      pui32EnabledBlockIDsInt,
+		      (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetEnabledHWPerfBlocks_exit;
+		}
+	}
+
+RGXGetEnabledHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetEnabledHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfTimeStamp(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampIN_UI8,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfTimeStampIN);
+
+	psRGXGetHWPerfTimeStampOUT->eError =
+	    PVRSRVRGXGetHWPerfTimeStampKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetHWPerfTimeStampOUT->ui64TimeStamp);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCtrlHWPerf(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXCtrlHWPerfIN_UI8,
+			  IMG_UINT8 * psRGXCtrlHWPerfOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *psRGXCtrlHWPerfIN =
+	    (PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *psRGXCtrlHWPerfOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfOUT_UI8, 0);
+
+	psRGXCtrlHWPerfOUT->eError =
+	    PVRSRVRGXCtrlHWPerfKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXCtrlHWPerfIN->ui32StreamId,
+				  psRGXCtrlHWPerfIN->bToggle, psRGXCtrlHWPerfIN->ui64Mask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsIN_UI8,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfBvncFeatureFlagsIN);
+
+	psRGXGetHWPerfBvncFeatureFlagsOUT->eError =
+	    PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(psConnection, OSGetDevNode(psConnection),
+						 &psRGXGetHWPerfBvncFeatureFlagsOUT->sBVNC);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXControlHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksIN_UI8,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT16 *ui16BlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) + 0;
+
+	if (unlikely(psRGXControlHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXControlHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXControlHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXControlHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		ui16BlockIDsInt = (IMG_UINT16 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16);
+	}
+
+	/* Copy the data over */
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui16BlockIDsInt,
+		     (const void __user *)psRGXControlHWPerfBlocksIN->pui16BlockIDs,
+		     psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) != PVRSRV_OK)
+		{
+			psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXControlHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXControlHWPerfBlocksOUT->eError =
+	    PVRSRVRGXControlHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXControlHWPerfBlocksIN->bEnable,
+					   psRGXControlHWPerfBlocksIN->ui32ArrayLen,
+					   ui16BlockIDsInt);
+
+RGXControlHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXControlHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigMuxHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXConfigMuxHWPerfCountersIN_UI8,
+				       IMG_UINT8 * psRGXConfigMuxHWPerfCountersOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS *psRGXConfigMuxHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigMuxHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS *psRGXConfigMuxHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigMuxHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigMuxHWPerfCounters_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigMuxHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigMuxHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXConfigMuxHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigMuxHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigMuxHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_MUX_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								     ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+		    sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigMuxHWPerfCountersIN->psBlockConfigs,
+		     psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigMuxHWPerfCounters_exit;
+		}
+	}
+
+	psRGXConfigMuxHWPerfCountersOUT->eError =
+	    PVRSRVRGXConfigMuxHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+					       psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen,
+					       psBlockConfigsInt);
+
+RGXConfigMuxHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigMuxHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGX_HWPERF_MAX_CUSTOM_CNTRS <= IMG_UINT32_MAX,
+	      "RGX_HWPERF_MAX_CUSTOM_CNTRS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigCustomCounters(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXConfigCustomCountersIN_UI8,
+				    IMG_UINT8 * psRGXConfigCustomCountersOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS *psRGXConfigCustomCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigCustomCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS *psRGXConfigCustomCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigCustomCountersOUT_UI8, 0);
+
+	IMG_UINT32 *ui32CustomCounterIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32)) +
+	    0;
+
+	if (unlikely
+	    (psRGXConfigCustomCountersIN->ui16NumCustomCounters > RGX_HWPERF_MAX_CUSTOM_CNTRS))
+	{
+		psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigCustomCounters_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigCustomCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigCustomCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigCustomCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigCustomCounters_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigCustomCountersIN->ui16NumCustomCounters != 0)
+	{
+		ui32CustomCounterIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CustomCounterIDsInt,
+		     (const void __user *)psRGXConfigCustomCountersIN->pui32CustomCounterIDs,
+		     psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigCustomCounters_exit;
+		}
+	}
+
+	psRGXConfigCustomCountersOUT->eError =
+	    PVRSRVRGXConfigCustomCountersKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXConfigCustomCountersIN->ui16CustomBlockID,
+					    psRGXConfigCustomCountersIN->ui16NumCustomCounters,
+					    ui32CustomCounterIDsInt);
+
+RGXConfigCustomCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigCustomCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigureHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksIN_UI8,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigureHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigureHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigureHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigureHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigureHWPerfBlocksIN->psBlockConfigs,
+		     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigureHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXConfigureHWPerfBlocksOUT->eError =
+	    PVRSRVRGXConfigureHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXConfigureHWPerfBlocksIN->ui32CtrlWord,
+					     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen,
+					     psBlockConfigsInt);
+
+RGXConfigureHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigureHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfMuxCounters(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXGetConfiguredHWPerfMuxCountersIN_UI8,
+					      IMG_UINT8 *
+					      psRGXGetConfiguredHWPerfMuxCountersOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *psRGXGetConfiguredHWPerfMuxCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfMuxCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *psRGXGetConfiguredHWPerfMuxCountersOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfMuxCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfMuxCountersOUT->psConfiguredMuxCounters =
+	    psRGXGetConfiguredHWPerfMuxCountersIN->psConfiguredMuxCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfMuxCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfMuxCountersIN),
+			      sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfMuxCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfMuxCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredMuxCountersInt =
+		    (RGX_HWPERF_CONFIG_MUX_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								     ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXGetConfiguredHWPerfMuxCountersIN->
+						      ui32BlockID, psConfiguredMuxCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfMuxCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfMuxCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredMuxCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfMuxCountersOUT->
+		      psConfiguredMuxCounters, psConfiguredMuxCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+			    PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfMuxCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfMuxCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfMuxCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXOpenHWPerfClientStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCloseHWPerfClientStreamKM((HWPERF_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXOpenHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamIN_UI8,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamOUT_UI8, 0);
+
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psRGXOpenHWPerfClientStreamIN);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVRGXOpenHWPerfClientStreamKM(psConnection, OSGetDevNode(psConnection), &psSDInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXOpenHWPerfClientStreamOUT->hSD, (void *)psSDInt,
+				      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXOpenHWPerfClientStreampsSDIntRelease);
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXOpenHWPerfClientStream_exit:
+
+	if (psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psSDInt)
+		{
+			PVRSRVRGXCloseHWPerfClientStreamKM(psSDInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCloseHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamIN_UI8,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCloseHWPerfClientStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCloseHWPerfClientStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	if (unlikely((psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_OK) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCloseHWPerfClientStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCloseHWPerfClientStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCloseHWPerfClientStream_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXWriteHWPerfClientEvent(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventIN_UI8,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventIN =
+	    (PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psRGXWriteHWPerfClientEventIN->hSD;
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXWriteHWPerfClientEventIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXWriteHWPerfClientEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXWriteHWPerfClientEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXWriteHWPerfClientEvent_exit;
+			}
+		}
+	}
+
+	if (psRGXWriteHWPerfClientEventIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psRGXWriteHWPerfClientEventIN->pui8Data,
+		     psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXWriteHWPerfClientEvent_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD, IMG_TRUE);
+	if (unlikely(psRGXWriteHWPerfClientEventOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVRGXWriteHWPerfClientEventKM(psSDInt,
+					      psRGXWriteHWPerfClientEventIN->ui32Size, ui8DataInt);
+
+RGXWriteHWPerfClientEvent_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXWriteHWPerfClientEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+
+/*
+ * Register all RGXHWPERF functions with services
+ */
+PVRSRV_ERROR InitRGXHWPERFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS,
+			      PVRSRVBridgeRGXGetEnabledHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP,
+			      PVRSRVBridgeRGXGetHWPerfTimeStamp, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF,
+			      PVRSRVBridgeRGXCtrlHWPerf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCTRLHWPERF),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS,
+			      PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS,
+			      PVRSRVBridgeRGXControlHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXConfigMuxHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS,
+			      PVRSRVBridgeRGXConfigCustomCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS,
+			      PVRSRVBridgeRGXConfigureHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfMuxCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXOpenHWPerfClientStream, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXCloseHWPerfClientStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT,
+			      PVRSRVBridgeRGXWriteHWPerfClientEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxhwperf functions with services
+ */
+void DeinitRGXHWPERFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/common_rgxkicksync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/common_rgxkicksync_bridge.h
new file mode 100644
index 000000000000..afd882ca7529
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/common_rgxkicksync_bridge.h
@@ -0,0 +1,143 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXKICKSYNC_BRIDGE_H
+#define COMMON_RGXKICKSYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST			(PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3)
+
+/*******************************************
+            RGXCreateKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXDestroyKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXKickSync2
+ *******************************************/
+
+/* Bridge in structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKSYNC2_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 *pui32UpdateDevVarOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phUpdateUFODevVarBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hTimelineFenceFD;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKSYNC2;
+
+/* Bridge out structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKSYNC2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFenceFD;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKSYNC2;
+
+/*******************************************
+            RGXSetKickSyncContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+#endif /* COMMON_RGXKICKSYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/server_rgxkicksync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/server_rgxkicksync_bridge.c
new file mode 100644
index 000000000000..8d943cd536e2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxkicksync_bridge/server_rgxkicksync_bridge.c
@@ -0,0 +1,593 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxkicksync.h"
+
+#include "common_rgxkicksync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateKickSyncContextpsKickSyncContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyKickSyncContextKM((RGX_SERVER_KICKSYNC_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateKickSyncContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateKickSyncContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXCreateKickSyncContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVRGXCreateKickSyncContextKM(psConnection, OSGetDevNode(psConnection),
+					     hPrivDataInt,
+					     psRGXCreateKickSyncContextIN->ui32PackedCCBSizeU88,
+					     psRGXCreateKickSyncContextIN->ui32ContextFlags,
+					     &psKickSyncContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateKickSyncContextOUT->hKickSyncContext,
+				      (void *)psKickSyncContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateKickSyncContextpsKickSyncContextIntRelease);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateKickSyncContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK)
+	{
+		if (psKickSyncContextInt)
+		{
+			PVRSRVRGXDestroyKickSyncContextKM(psKickSyncContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyKickSyncContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyKickSyncContextIN->
+					      hKickSyncContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyKickSyncContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyKickSyncContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyKickSyncContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyKickSyncContext_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_DEV_VARS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_DEV_VARS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickSync2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickSync2IN_UI8,
+			 IMG_UINT8 * psRGXKickSync2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *psRGXKickSync2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *psRGXKickSync2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2OUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXKickSync2IN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFODevVarBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFODevVarBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateDevVarOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRGXKickSync2IN->ui32ClientUpdateCount > PVRSRV_MAX_DEV_VARS))
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickSync2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickSync2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickSync2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickSync2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickSync2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFODevVarBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFODevVarBlockInt, 0,
+			       psRGXKickSync2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFODevVarBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFODevVarBlockInt2,
+		     (const void __user *)psRGXKickSync2IN->phUpdateUFODevVarBlock,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateDevVarOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateDevVarOffsetInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateDevVarOffset,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateValue,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickSync2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickSync2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickSync2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickSync2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFODevVarBlockInt[i],
+						       hUpdateUFODevVarBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickSync2OUT->eError =
+	    PVRSRVRGXKickSyncKM(psKickSyncContextInt,
+				psRGXKickSync2IN->ui32ClientUpdateCount,
+				psUpdateUFODevVarBlockInt,
+				ui32UpdateDevVarOffsetInt,
+				ui32UpdateValueInt,
+				psRGXKickSync2IN->hCheckFenceFD,
+				psRGXKickSync2IN->hTimelineFenceFD,
+				&psRGXKickSync2OUT->hUpdateFenceFD,
+				uiUpdateFenceNameInt, psRGXKickSync2IN->ui32ExtJobRef);
+
+RGXKickSync2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+
+	if (hUpdateUFODevVarBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFODevVarBlockInt && psUpdateUFODevVarBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFODevVarBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickSync2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetKickSyncContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXSetKickSyncContextPropertyIN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetKickSyncContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetKickSyncContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVRGXSetKickSyncContextPropertyKM(psKickSyncContextInt,
+						  psRGXSetKickSyncContextPropertyIN->ui32Property,
+						  psRGXSetKickSyncContextPropertyIN->ui64Input,
+						  &psRGXSetKickSyncContextPropertyOUT->ui64Output);
+
+RGXSetKickSyncContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+
+/*
+ * Register all RGXKICKSYNC functions with services
+ */
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXCreateKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXDestroyKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2,
+			      PVRSRVBridgeRGXKickSync2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKSYNC2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKSYNC2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetKickSyncContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxkicksync functions with services
+ */
+void DeinitRGXKICKSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY);
+
+}
+#else /* SUPPORT_RGXKICKSYNC_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXKICKSYNC_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXKICKSYNCBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXKICKSYNCBridge()
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_bridge.h
new file mode 100644
index 000000000000..ed5a664ef17e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_bridge.h
@@ -0,0 +1,73 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RGXPDUMP_BRIDGE_H
+#define CLIENT_RGXPDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_rgxpdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
new file mode 100644
index 000000000000..7ba5126053dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
@@ -0,0 +1,118 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for rgxpdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_rgxpdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "rgx_bridge.h"
+
+#include "rgxpdump.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpTraceBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpSignatureBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPreCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPostCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					     ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/common_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/common_rgxpdump_bridge.h
new file mode 100644
index 000000000000..37317acb4707
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/common_rgxpdump_bridge.h
@@ -0,0 +1,161 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXPDUMP_BRIDGE_H
+#define COMMON_RGXPDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_LAST			(PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5)
+
+/*******************************************
+            PDumpTraceBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER;
+
+/* Bridge out structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER;
+
+/*******************************************
+            PDumpSignatureBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER;
+
+/* Bridge out structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER;
+
+/*******************************************
+            PDumpComputeCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpValCheckPreCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND;
+
+/* Bridge out structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND;
+
+/*******************************************
+            PDumpValCheckPostCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND;
+
+/* Bridge out structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND;
+
+#endif /* COMMON_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/server_rgxpdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/server_rgxpdump_bridge.c
new file mode 100644
index 000000000000..58ed4ebb037f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxpdump_bridge/server_rgxpdump_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxpdump.h"
+
+#include "common_rgxpdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePDumpTraceBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psPDumpTraceBufferIN_UI8,
+			     IMG_UINT8 * psPDumpTraceBufferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *psPDumpTraceBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *psPDumpTraceBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferOUT_UI8, 0);
+
+	psPDumpTraceBufferOUT->eError =
+	    PVRSRVPDumpTraceBufferKM(psConnection, OSGetDevNode(psConnection),
+				     psPDumpTraceBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpSignatureBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpSignatureBufferIN_UI8,
+				 IMG_UINT8 * psPDumpSignatureBufferOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *) IMG_OFFSET_ADDR(psPDumpSignatureBufferIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *)
+	    IMG_OFFSET_ADDR(psPDumpSignatureBufferOUT_UI8, 0);
+
+	psPDumpSignatureBufferOUT->eError =
+	    PVRSRVPDumpSignatureBufferKM(psConnection, OSGetDevNode(psConnection),
+					 psPDumpSignatureBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+#define PVRSRVBridgePDumpComputeCRCSignatureCheck NULL
+
+static IMG_INT
+PVRSRVBridgePDumpCRCSignatureCheck(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckIN_UI8,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckOUT_UI8, 0);
+
+	psPDumpCRCSignatureCheckOUT->eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(psConnection, OSGetDevNode(psConnection),
+					   psPDumpCRCSignatureCheckIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPreCommand(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psPDumpValCheckPreCommandIN_UI8,
+				    IMG_UINT8 * psPDumpValCheckPreCommandOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandOUT_UI8, 0);
+
+	psPDumpValCheckPreCommandOUT->eError =
+	    PVRSRVPDumpValCheckPreCommandKM(psConnection, OSGetDevNode(psConnection),
+					    psPDumpValCheckPreCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPostCommand(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPDumpValCheckPostCommandIN_UI8,
+				     IMG_UINT8 * psPDumpValCheckPostCommandOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandOUT_UI8, 0);
+
+	psPDumpValCheckPostCommandOUT->eError =
+	    PVRSRVPDumpValCheckPostCommandKM(psConnection, OSGetDevNode(psConnection),
+					     psPDumpValCheckPostCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXPDUMPBridge(void);
+void DeinitRGXPDUMPBridge(void);
+
+/*
+ * Register all RGXPDUMP functions with services
+ */
+PVRSRV_ERROR InitRGXPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER,
+			      PVRSRVBridgePDumpTraceBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER,
+			      PVRSRVBridgePDumpSignatureBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpComputeCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND,
+			      PVRSRVBridgePDumpValCheckPreCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND,
+			      PVRSRVBridgePDumpValCheckPostCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxpdump functions with services
+ */
+void DeinitRGXPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/common_rgxregconfig_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/common_rgxregconfig_bridge.h
new file mode 100644
index 000000000000..942b7e4b06d0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/common_rgxregconfig_bridge.h
@@ -0,0 +1,146 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXREGCONFIG_BRIDGE_H
+#define COMMON_RGXREGCONFIG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST			(PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetRegConfigType
+ *******************************************/
+
+/* Bridge in structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE_TAG
+{
+	IMG_UINT8 ui8RegPowerIsland;
+} __packed PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE;
+
+/* Bridge out structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE;
+
+/*******************************************
+            RGXAddRegconfig
+ *******************************************/
+
+/* Bridge in structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXADDREGCONFIG_TAG
+{
+	IMG_UINT64 ui64RegMask;
+	IMG_UINT64 ui64RegValue;
+	IMG_UINT32 ui32RegAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXADDREGCONFIG;
+
+/* Bridge out structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG;
+
+/*******************************************
+            RGXClearRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG;
+
+/* Bridge out structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG;
+
+/*******************************************
+            RGXEnableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG;
+
+/* Bridge out structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG;
+
+/*******************************************
+            RGXDisableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG;
+
+/* Bridge out structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG;
+
+#endif /* COMMON_RGXREGCONFIG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/server_rgxregconfig_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/server_rgxregconfig_bridge.c
new file mode 100644
index 000000000000..548401470d09
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxregconfig_bridge/server_rgxregconfig_bridge.c
@@ -0,0 +1,246 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxregconfig.h"
+
+#include "common_rgxregconfig_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetRegConfigType(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXSetRegConfigTypeIN_UI8,
+				IMG_UINT8 * psRGXSetRegConfigTypeOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeOUT_UI8,
+								      0);
+
+	psRGXSetRegConfigTypeOUT->eError =
+	    PVRSRVRGXSetRegConfigTypeKM(psConnection, OSGetDevNode(psConnection),
+					psRGXSetRegConfigTypeIN->ui8RegPowerIsland);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXAddRegconfig(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRGXAddRegconfigIN_UI8,
+			    IMG_UINT8 * psRGXAddRegconfigOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *psRGXAddRegconfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *psRGXAddRegconfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigOUT_UI8, 0);
+
+	psRGXAddRegconfigOUT->eError =
+	    PVRSRVRGXAddRegConfigKM(psConnection, OSGetDevNode(psConnection),
+				    psRGXAddRegconfigIN->ui32RegAddr,
+				    psRGXAddRegconfigIN->ui64RegValue,
+				    psRGXAddRegconfigIN->ui64RegMask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXClearRegConfigIN_UI8,
+			      IMG_UINT8 * psRGXClearRegConfigOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *psRGXClearRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *psRGXClearRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXClearRegConfigIN);
+
+	psRGXClearRegConfigOUT->eError =
+	    PVRSRVRGXClearRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXEnableRegConfigIN_UI8,
+			       IMG_UINT8 * psRGXEnableRegConfigOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *psRGXEnableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *psRGXEnableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEnableRegConfigIN);
+
+	psRGXEnableRegConfigOUT->eError =
+	    PVRSRVRGXEnableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXDisableRegConfigIN_UI8,
+				IMG_UINT8 * psRGXDisableRegConfigOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *psRGXDisableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *psRGXDisableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXDisableRegConfigIN);
+
+	psRGXDisableRegConfigOUT->eError =
+	    PVRSRVRGXDisableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+
+/*
+ * Register all RGXREGCONFIG functions with services
+ */
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE,
+			      PVRSRVBridgeRGXSetRegConfigType, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG,
+			      PVRSRVBridgeRGXAddRegconfig, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXADDREGCONFIG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG,
+			      PVRSRVBridgeRGXClearRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG,
+			      PVRSRVBridgeRGXEnableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG,
+			      PVRSRVBridgeRGXDisableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxregconfig functions with services
+ */
+void DeinitRGXREGCONFIGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG);
+
+}
+#else /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXREGCONFIG_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXREGCONFIGBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXREGCONFIGBridge()
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/common_rgxta3d_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/common_rgxta3d_bridge.h
new file mode 100644
index 000000000000..cf215caeddff
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/common_rgxta3d_bridge.h
@@ -0,0 +1,466 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTA3D_BRIDGE_H
+#define COMMON_RGXTA3D_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_fwif_shared.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_LAST			(PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15)
+
+/*******************************************
+            RGXDestroyHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET_TAG
+{
+	IMG_HANDLE hKmHwRTDataSet;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET;
+
+/* Bridge out structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET;
+
+/*******************************************
+            RGXCreateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER;
+
+/* Bridge out structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferMemDesc;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER;
+
+/* Bridge out structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER;
+
+/*******************************************
+            RGXPopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+} __packed PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXUnpopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+} __packed PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST;
+
+/* Bridge out structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST;
+
+/*******************************************
+            RGXDestroyRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT;
+
+/* Bridge out structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT;
+
+/*******************************************
+            RGXSendZSStoreDisable
+ *******************************************/
+
+/* Bridge in structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32ExtJobRefToDisableZSStore;
+	IMG_BOOL bDisableDepthStore;
+	IMG_BOOL bDisableStencilStore;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE;
+
+/* Bridge out structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE;
+
+/*******************************************
+            RGXSetRenderContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXRenderContextStalled
+ *******************************************/
+
+/* Bridge in structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED_TAG
+{
+	IMG_HANDLE hRenderContext;
+} __packed PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED;
+
+/* Bridge out structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED;
+
+/*******************************************
+            RGXKickTA3D2
+ *******************************************/
+
+/* Bridge in structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTA3D2_TAG
+{
+	IMG_UINT64 ui64Deadline;
+	IMG_HANDLE hKMHWRTDataSet;
+	IMG_HANDLE hMSAAScratchBuffer;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock;
+	IMG_HANDLE hRenderContext;
+	IMG_HANDLE hZSBuffer;
+	IMG_UINT32 *pui32Client3DUpdateSyncOffset;
+	IMG_UINT32 *pui32Client3DUpdateValue;
+	IMG_UINT32 *pui32ClientTAFenceSyncOffset;
+	IMG_UINT32 *pui32ClientTAFenceValue;
+	IMG_UINT32 *pui32ClientTAUpdateSyncOffset;
+	IMG_UINT32 *pui32ClientTAUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui83DCmd;
+	IMG_BYTE *pui83DPRCmd;
+	IMG_BYTE *pui8TACmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_CHAR *puiUpdateFenceName3D;
+	IMG_HANDLE *phClient3DUpdateSyncPrimBlock;
+	IMG_HANDLE *phClientTAFenceSyncPrimBlock;
+	IMG_HANDLE *phClientTAUpdateSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFence;
+	PVRSRV_FENCE hCheckFence3D;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	PVRSRV_TIMELINE hUpdateTimeline3D;
+	IMG_UINT32 ui323DCmdSize;
+	IMG_UINT32 ui323DPRCmdSize;
+	IMG_UINT32 ui32Client3DUpdateCount;
+	IMG_UINT32 ui32ClientTAFenceCount;
+	IMG_UINT32 ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumberOfDrawCalls;
+	IMG_UINT32 ui32NumberOfIndices;
+	IMG_UINT32 ui32NumberOfMRTs;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PRFenceUFOSyncOffset;
+	IMG_UINT32 ui32PRFenceValue;
+	IMG_UINT32 ui32RenderTargetSize;
+	IMG_UINT32 ui32SyncPMRCount;
+	IMG_UINT32 ui32TACmdSize;
+	IMG_BOOL bbAbort;
+	IMG_BOOL bbKick3D;
+	IMG_BOOL bbKickPR;
+	IMG_BOOL bbKickTA;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTA3D2;
+
+/* Bridge out structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTA3D2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	PVRSRV_FENCE hUpdateFence3D;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTA3D2;
+
+/*******************************************
+            RGXSetRenderContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hRenderContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXCreateHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_UINT64 ui64FlippedMultiSampleCtl;
+	IMG_UINT64 ui64MultiSampleCtl;
+	IMG_DEV_VIRTADDR *psMacrotileArrayDevVAddr;
+	IMG_DEV_VIRTADDR *psPMMlistDevVAddr;
+	IMG_DEV_VIRTADDR *psRTCDevVAddr;
+	IMG_DEV_VIRTADDR *psRgnHeaderDevVAddr;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_DEV_VIRTADDR *psVHeapTableDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT32 ui32ISPMergeLowerX;
+	IMG_UINT32 ui32ISPMergeLowerY;
+	IMG_UINT32 ui32ISPMergeScaleX;
+	IMG_UINT32 ui32ISPMergeScaleY;
+	IMG_UINT32 ui32ISPMergeUpperX;
+	IMG_UINT32 ui32ISPMergeUpperY;
+	IMG_UINT32 ui32ISPMtileSize;
+	IMG_UINT32 ui32MTileStride;
+	IMG_UINT32 ui32PPPScreen;
+	IMG_UINT32 ui32RgnHeaderSize;
+	IMG_UINT32 ui32TEAA;
+	IMG_UINT32 ui32TEMTILE1;
+	IMG_UINT32 ui32TEMTILE2;
+	IMG_UINT32 ui32TEScreen;
+	IMG_UINT32 ui32TPCSize;
+	IMG_UINT32 ui32TPCStride;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET;
+
+/* Bridge out structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET;
+
+/*******************************************
+            RGXCreateFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hFreeListReservation;
+	IMG_HANDLE hMemCtxPrivData;
+	IMG_HANDLE hsGlobalFreeList;
+	IMG_UINT32 ui32GrowFLPages;
+	IMG_UINT32 ui32GrowParamThreshold;
+	IMG_UINT32 ui32InitFLPages;
+	IMG_UINT32 ui32MaxFLPages;
+	IMG_BOOL bbFreeListCheck;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEFREELIST;
+
+/* Bridge out structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST;
+
+/*******************************************
+            RGXCreateRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_DEV_VIRTADDR sVDMCallStackAddr;
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticRenderContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32Max3DDeadlineMS;
+	IMG_UINT32 ui32MaxTADeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+	IMG_UINT32 ui32StaticRenderContextStateSize;
+	IMG_UINT32 ui32ui32CallStackDepth;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT;
+
+/* Bridge out structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_HANDLE hRenderContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT;
+
+/*******************************************
+            RGXCreateHWRTDataSet2
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_UINT64 ui64FlippedMultiSampleCtl;
+	IMG_UINT64 ui64MultiSampleCtl;
+	IMG_HANDLE hPMMlistsReservation;
+	IMG_DEV_VIRTADDR *psMacrotileArrayDevVAddr;
+	IMG_DEV_VIRTADDR *psRTCDevVAddr;
+	IMG_DEV_VIRTADDR *psRgnHeaderDevVAddr;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_DEV_VIRTADDR *psVHeapTableDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT32 ui32ISPMergeLowerX;
+	IMG_UINT32 ui32ISPMergeLowerY;
+	IMG_UINT32 ui32ISPMergeScaleX;
+	IMG_UINT32 ui32ISPMergeScaleY;
+	IMG_UINT32 ui32ISPMergeUpperX;
+	IMG_UINT32 ui32ISPMergeUpperY;
+	IMG_UINT32 ui32ISPMtileSize;
+	IMG_UINT32 ui32MTileStride;
+	IMG_UINT32 ui32PPPScreen;
+	IMG_UINT32 ui32RgnHeaderSize;
+	IMG_UINT32 ui32TEAA;
+	IMG_UINT32 ui32TEMTILE1;
+	IMG_UINT32 ui32TEMTILE2;
+	IMG_UINT32 ui32TEScreen;
+	IMG_UINT32 ui32TPCSize;
+	IMG_UINT32 ui32TPCStride;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2;
+
+/* Bridge out structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2;
+
+#endif /* COMMON_RGXTA3D_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/server_rgxta3d_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/server_rgxta3d_bridge.c
new file mode 100644
index 000000000000..121ec9714cd3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxta3d_bridge/server_rgxta3d_bridge.c
@@ -0,0 +1,3005 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxta3d.h"
+
+#include "common_rgxta3d_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetIN_UI8,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyHWRTDataSetOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyHWRTDataSetIN->
+					      hKmHwRTDataSet,
+					      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+	if (unlikely
+	    ((psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyHWRTDataSetOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyHWRTDataSet_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyHWRTDataSet_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateZSBufferpssZSBufferKMIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyZSBufferKM((RGX_ZSBUFFER_DATA *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateZSBufferIN_UI8,
+			      IMG_UINT8 * psRGXCreateZSBufferOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *psRGXCreateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *psRGXCreateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psRGXCreateZSBufferIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psRGXCreateZSBufferIN->hPMR;
+	PMR *psPMRInt = NULL;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError =
+	    RGXCreateZSBufferKM(psConnection, OSGetDevNode(psConnection),
+				psReservationInt,
+				psPMRInt, psRGXCreateZSBufferIN->uiMapFlags, &pssZSBufferKMInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateZSBufferOUT->
+								   hsZSBufferKM,
+								   (void *)pssZSBufferKMInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateZSBufferpssZSBufferKMIntRelease);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssZSBufferKMInt)
+		{
+			RGXDestroyZSBufferKM(pssZSBufferKMInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyZSBufferIN_UI8,
+			       IMG_UINT8 * psRGXDestroyZSBufferOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyZSBufferIN->
+					      hsZSBufferMemDesc,
+					      PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	if (unlikely
+	    ((psRGXDestroyZSBufferOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyZSBuffer_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXPopulateZSBufferpssPopulationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXUnpopulateZSBufferKM((RGX_POPULATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXPopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXPopulateZSBufferIN_UI8,
+				IMG_UINT8 * psRGXPopulateZSBufferOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferOUT_UI8,
+								      0);
+
+	IMG_HANDLE hsZSBufferKM = psRGXPopulateZSBufferIN->hsZSBufferKM;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+	RGX_POPULATION *pssPopulationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXPopulateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pssZSBufferKMInt,
+				       hsZSBufferKM,
+				       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError =
+	    RGXPopulateZSBufferKM(pssZSBufferKMInt, &pssPopulationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRGXPopulateZSBufferOUT->
+								     hsPopulation,
+								     (void *)pssPopulationInt,
+								     PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RGXPopulateZSBufferpssPopulationIntRelease);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXPopulateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pssZSBufferKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hsZSBufferKM, PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXPopulateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssPopulationInt)
+		{
+			RGXUnpopulateZSBufferKM(pssPopulationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXUnpopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferIN_UI8,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXUnpopulateZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXUnpopulateZSBufferIN->hsPopulation,
+					      PVRSRV_HANDLE_TYPE_RGX_POPULATION);
+	if (unlikely((psRGXUnpopulateZSBufferOUT->eError != PVRSRV_OK) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXUnpopulateZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXUnpopulateZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXUnpopulateZSBuffer_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyFreeListIN_UI8,
+			       IMG_UINT8 * psRGXDestroyFreeListOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *psRGXDestroyFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *psRGXDestroyFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyFreeListOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyFreeListIN->hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+	if (unlikely((psRGXDestroyFreeListOUT->eError != PVRSRV_OK) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyFreeListOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyFreeList_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyFreeList_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXDestroyRenderContextIN_UI8,
+				    IMG_UINT8 * psRGXDestroyRenderContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyRenderContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyRenderContextIN->
+					      hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyRenderContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyRenderContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyRenderContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyRenderContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendZSStoreDisable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXSendZSStoreDisableIN_UI8,
+				  IMG_UINT8 * psRGXSendZSStoreDisableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSendZSStoreDisableIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendZSStoreDisableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendZSStoreDisable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVRGXSendZSStoreDisableKM(psConnection, OSGetDevNode(psConnection),
+					  psRenderContextInt,
+					  psRGXSendZSStoreDisableIN->bDisableDepthStore,
+					  psRGXSendZSStoreDisableIN->bDisableStencilStore,
+					  psRGXSendZSStoreDisableIN->i32ExtJobRefToDisableZSStore);
+
+RGXSendZSStoreDisable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPriorityIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPriorityIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVRGXSetRenderContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRenderContextInt,
+						psRGXSetRenderContextPriorityIN->i32Priority);
+
+RGXSetRenderContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXRenderContextStalled(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXRenderContextStalledIN_UI8,
+				    IMG_UINT8 * psRGXRenderContextStalledOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledIN =
+	    (PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXRenderContextStalledIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXRenderContextStalledOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXRenderContextStalledOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXRenderContextStalled_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXRenderContextStalledOUT->eError = RGXRenderContextStalledKM(psRenderContextInt);
+
+RGXRenderContextStalled_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTA3D2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickTA3D2IN_UI8,
+			 IMG_UINT8 * psRGXKickTA3D2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *psRGXKickTA3D2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *psRGXKickTA3D2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2OUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXKickTA3D2IN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAFenceSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAFenceSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAFenceSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAFenceValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClient3DUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClient3DUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32Client3DUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32Client3DUpdateValueInt = NULL;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock = psRGXKickTA3D2IN->hPRFenceUFOSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *psPRFenceUFOSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_CHAR *uiUpdateFenceName3DInt = NULL;
+	IMG_BYTE *ui8TACmdInt = NULL;
+	IMG_BYTE *ui83DPRCmdInt = NULL;
+	IMG_BYTE *ui83DCmdInt = NULL;
+	IMG_HANDLE hKMHWRTDataSet = psRGXKickTA3D2IN->hKMHWRTDataSet;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSetInt = NULL;
+	IMG_HANDLE hZSBuffer = psRGXKickTA3D2IN->hZSBuffer;
+	RGX_ZSBUFFER_DATA *psZSBufferInt = NULL;
+	IMG_HANDLE hMSAAScratchBuffer = psRGXKickTA3D2IN->hMSAAScratchBuffer;
+	RGX_ZSBUFFER_DATA *psMSAAScratchBufferInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAFenceCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32Client3DUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32TACmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DPRCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTA3D2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTA3D2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTA3D2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		psClientTAFenceSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAFenceSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAFenceSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAFenceSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAFenceSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceValue,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		psClientTAUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateValue,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		psClient3DUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClient3DUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClient3DUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClient3DUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClient3DUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateValue,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	{
+		uiUpdateFenceName3DInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceName3DInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName3D,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceName3DInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						      1] = '\0';
+	}
+	if (psRGXKickTA3D2IN->ui32TACmdSize != 0)
+	{
+		ui8TACmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8TACmdInt, (const void __user *)psRGXKickTA3D2IN->pui8TACmd,
+		     psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize != 0)
+	{
+		ui83DPRCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DPRCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DPRCmd,
+		     psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DCmdSize != 0)
+	{
+		ui83DCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DCmd,
+		     psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32SyncPMRFlags,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickTA3D2IN->phSyncPMRs,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psClientTAFenceSyncPrimBlockInt[i],
+						       hClientTAFenceSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientTAUpdateSyncPrimBlockInt[i],
+						       hClientTAUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClient3DUpdateSyncPrimBlockInt[i],
+						       hClient3DUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPRFenceUFOSyncPrimBlockInt,
+				       hPRFenceUFOSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psKMHWRTDataSetInt,
+					       hKMHWRTDataSet,
+					       PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psZSBufferInt,
+					       hZSBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psMSAAScratchBufferInt,
+					       hMSAAScratchBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVRGXKickTA3DKM(psRenderContextInt,
+				psRGXKickTA3D2IN->ui32ClientTAFenceCount,
+				psClientTAFenceSyncPrimBlockInt,
+				ui32ClientTAFenceSyncOffsetInt,
+				ui32ClientTAFenceValueInt,
+				psRGXKickTA3D2IN->ui32ClientTAUpdateCount,
+				psClientTAUpdateSyncPrimBlockInt,
+				ui32ClientTAUpdateSyncOffsetInt,
+				ui32ClientTAUpdateValueInt,
+				psRGXKickTA3D2IN->ui32Client3DUpdateCount,
+				psClient3DUpdateSyncPrimBlockInt,
+				ui32Client3DUpdateSyncOffsetInt,
+				ui32Client3DUpdateValueInt,
+				psPRFenceUFOSyncPrimBlockInt,
+				psRGXKickTA3D2IN->ui32PRFenceUFOSyncOffset,
+				psRGXKickTA3D2IN->ui32PRFenceValue,
+				psRGXKickTA3D2IN->hCheckFence,
+				psRGXKickTA3D2IN->hUpdateTimeline,
+				&psRGXKickTA3D2OUT->hUpdateFence,
+				uiUpdateFenceNameInt,
+				psRGXKickTA3D2IN->hCheckFence3D,
+				psRGXKickTA3D2IN->hUpdateTimeline3D,
+				&psRGXKickTA3D2OUT->hUpdateFence3D,
+				uiUpdateFenceName3DInt,
+				psRGXKickTA3D2IN->ui32TACmdSize,
+				ui8TACmdInt,
+				psRGXKickTA3D2IN->ui323DPRCmdSize,
+				ui83DPRCmdInt,
+				psRGXKickTA3D2IN->ui323DCmdSize,
+				ui83DCmdInt,
+				psRGXKickTA3D2IN->ui32ExtJobRef,
+				psRGXKickTA3D2IN->bbKickTA,
+				psRGXKickTA3D2IN->bbKickPR,
+				psRGXKickTA3D2IN->bbKick3D,
+				psRGXKickTA3D2IN->bbAbort,
+				psRGXKickTA3D2IN->ui32PDumpFlags,
+				psKMHWRTDataSetInt,
+				psZSBufferInt,
+				psMSAAScratchBufferInt,
+				psRGXKickTA3D2IN->ui32SyncPMRCount,
+				ui32SyncPMRFlagsInt,
+				psSyncPMRsInt,
+				psRGXKickTA3D2IN->ui32RenderTargetSize,
+				psRGXKickTA3D2IN->ui32NumberOfDrawCalls,
+				psRGXKickTA3D2IN->ui32NumberOfIndices,
+				psRGXKickTA3D2IN->ui32NumberOfMRTs, psRGXKickTA3D2IN->ui64Deadline);
+
+RGXKickTA3D2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+
+	if (hClientTAFenceSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAFenceSyncPrimBlockInt && psClientTAFenceSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAFenceSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClientTAUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAUpdateSyncPrimBlockInt && psClientTAUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClient3DUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClient3DUpdateSyncPrimBlockInt && psClient3DUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClient3DUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPRFenceUFOSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPRFenceUFOSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psKMHWRTDataSetInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hKMHWRTDataSet,
+						    PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psZSBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hZSBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psMSAAScratchBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hMSAAScratchBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTA3D2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPropertyIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPropertyOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPropertyIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVRGXSetRenderContextPropertyKM(psRenderContextInt,
+						psRGXSetRenderContextPropertyIN->ui32Property,
+						psRGXSetRenderContextPropertyIN->ui64Input,
+						&psRGXSetRenderContextPropertyOUT->ui64Output);
+
+RGXSetRenderContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetIN_UI8,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *) IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetOUT_UI8, 0);
+
+	IMG_DEV_VIRTADDR *sVHeapTableDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sPMMlistDevVAddrInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sMacrotileArrayDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRgnHeaderDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRTCDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet = psRGXCreateHWRTDataSetIN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSetIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSetIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+
+	{
+		sVHeapTableDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sVHeapTableDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psVHeapTableDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sPMMlistDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sPMMlistDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psPMMlistDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sMacrotileArrayDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sMacrotileArrayDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psMacrotileArrayDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sRgnHeaderDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRgnHeaderDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psRgnHeaderDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sRTCDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRTCDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psRTCDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSetOUT->eError =
+	    RGXCreateHWRTDataSet(psConnection, OSGetDevNode(psConnection),
+				 sVHeapTableDevVAddrInt,
+				 sPMMlistDevVAddrInt,
+				 psapsFreeListsInt,
+				 psRGXCreateHWRTDataSetIN->ui32PPPScreen,
+				 psRGXCreateHWRTDataSetIN->ui64MultiSampleCtl,
+				 psRGXCreateHWRTDataSetIN->ui64FlippedMultiSampleCtl,
+				 psRGXCreateHWRTDataSetIN->ui32TPCStride,
+				 sTailPtrsDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui32TPCSize,
+				 psRGXCreateHWRTDataSetIN->ui32TEScreen,
+				 psRGXCreateHWRTDataSetIN->ui32TEAA,
+				 psRGXCreateHWRTDataSetIN->ui32TEMTILE1,
+				 psRGXCreateHWRTDataSetIN->ui32TEMTILE2,
+				 psRGXCreateHWRTDataSetIN->ui32MTileStride,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeLowerX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeLowerY,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeUpperX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeUpperY,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeScaleX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeScaleY,
+				 sMacrotileArrayDevVAddrInt,
+				 sRgnHeaderDevVAddrInt,
+				 sRTCDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui32RgnHeaderSize,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMtileSize,
+				 psRGXCreateHWRTDataSetIN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSetOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateFreeListpsCleanupCookieIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyFreeList((RGX_FREELIST *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateFreeListIN_UI8,
+			      IMG_UINT8 * psRGXCreateFreeListOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *psRGXCreateFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *psRGXCreateFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListOUT_UI8, 0);
+
+	IMG_HANDLE hMemCtxPrivData = psRGXCreateFreeListIN->hMemCtxPrivData;
+	IMG_HANDLE hMemCtxPrivDataInt = NULL;
+	IMG_HANDLE hsGlobalFreeList = psRGXCreateFreeListIN->hsGlobalFreeList;
+	RGX_FREELIST *pssGlobalFreeListInt = NULL;
+	IMG_HANDLE hFreeListReservation = psRGXCreateFreeListIN->hFreeListReservation;
+	DEVMEMINT_RESERVATION *psFreeListReservationInt = NULL;
+	RGX_FREELIST *psCleanupCookieInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hMemCtxPrivDataInt,
+				       hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+		/* Look up the address from the handle */
+		psRGXCreateFreeListOUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&pssGlobalFreeListInt,
+					       hsGlobalFreeList,
+					       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+		if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXCreateFreeList_exit;
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psFreeListReservationInt,
+				       hFreeListReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError =
+	    RGXCreateFreeList(psConnection, OSGetDevNode(psConnection),
+			      hMemCtxPrivDataInt,
+			      psRGXCreateFreeListIN->ui32MaxFLPages,
+			      psRGXCreateFreeListIN->ui32InitFLPages,
+			      psRGXCreateFreeListIN->ui32GrowFLPages,
+			      psRGXCreateFreeListIN->ui32GrowParamThreshold,
+			      pssGlobalFreeListInt,
+			      psRGXCreateFreeListIN->bbFreeListCheck,
+			      psFreeListReservationInt, &psCleanupCookieInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateFreeListOUT->
+								   hCleanupCookie,
+								   (void *)psCleanupCookieInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateFreeListpsCleanupCookieIntRelease);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateFreeList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hMemCtxPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (pssGlobalFreeListInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hsGlobalFreeList,
+						    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psFreeListReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hFreeListReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateFreeListOUT->eError != PVRSRV_OK)
+	{
+		if (psCleanupCookieInt)
+		{
+			RGXDestroyFreeList(psCleanupCookieInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateRenderContextpsRenderContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyRenderContextKM((RGX_SERVER_RENDER_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_RENDERCONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_RENDERCONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXCreateRenderContextIN_UI8,
+				   IMG_UINT8 * psRGXCreateRenderContextOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateRenderContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticRenderContextStateInt = NULL;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateRenderContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize >
+	     RGXFWIF_STATIC_RENDERCONTEXT_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateRenderContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateRenderContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateRenderContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8FrameworkCmd,
+		     psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize != 0)
+	{
+		ui8StaticRenderContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticRenderContextStateInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8StaticRenderContextState,
+		     psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVRGXCreateRenderContextKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXCreateRenderContextIN->i32Priority,
+					   psRGXCreateRenderContextIN->sVDMCallStackAddr,
+					   psRGXCreateRenderContextIN->ui32ui32CallStackDepth,
+					   psRGXCreateRenderContextIN->ui32FrameworkCmdSize,
+					   ui8FrameworkCmdInt,
+					   hPrivDataInt,
+					   psRGXCreateRenderContextIN->
+					   ui32StaticRenderContextStateSize,
+					   ui8StaticRenderContextStateInt,
+					   psRGXCreateRenderContextIN->ui32PackedCCBSizeU8888,
+					   psRGXCreateRenderContextIN->ui32ContextFlags,
+					   psRGXCreateRenderContextIN->ui64RobustnessAddress,
+					   psRGXCreateRenderContextIN->ui32MaxTADeadlineMS,
+					   psRGXCreateRenderContextIN->ui32Max3DDeadlineMS,
+					   &psRenderContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psRGXCreateRenderContextOUT->
+									hRenderContext,
+									(void *)psRenderContextInt,
+									PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_RGXCreateRenderContextpsRenderContextIntRelease);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateRenderContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateRenderContextOUT->eError != PVRSRV_OK)
+	{
+		if (psRenderContextInt)
+		{
+			PVRSRVRGXDestroyRenderContextKM(psRenderContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateRenderContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2IN_UI8,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2IN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2OUT_UI8, 0);
+
+	IMG_DEV_VIRTADDR *sVHeapTableDevVAddrInt = NULL;
+	IMG_HANDLE hPMMlistsReservation = psRGXCreateHWRTDataSet2IN->hPMMlistsReservation;
+	DEVMEMINT_RESERVATION *psPMMlistsReservationInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sMacrotileArrayDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRgnHeaderDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRTCDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet = psRGXCreateHWRTDataSet2IN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSet2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSet2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+
+	{
+		sVHeapTableDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sVHeapTableDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psVHeapTableDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sMacrotileArrayDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sMacrotileArrayDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psMacrotileArrayDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sRgnHeaderDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRgnHeaderDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psRgnHeaderDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sRTCDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRTCDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psRTCDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMMlistsReservationInt,
+				       hPMMlistsReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    RGXCreateHWRTDataSet2(psConnection, OSGetDevNode(psConnection),
+				  sVHeapTableDevVAddrInt,
+				  psPMMlistsReservationInt,
+				  psapsFreeListsInt,
+				  psRGXCreateHWRTDataSet2IN->ui32PPPScreen,
+				  psRGXCreateHWRTDataSet2IN->ui64MultiSampleCtl,
+				  psRGXCreateHWRTDataSet2IN->ui64FlippedMultiSampleCtl,
+				  psRGXCreateHWRTDataSet2IN->ui32TPCStride,
+				  sTailPtrsDevVAddrInt,
+				  psRGXCreateHWRTDataSet2IN->ui32TPCSize,
+				  psRGXCreateHWRTDataSet2IN->ui32TEScreen,
+				  psRGXCreateHWRTDataSet2IN->ui32TEAA,
+				  psRGXCreateHWRTDataSet2IN->ui32TEMTILE1,
+				  psRGXCreateHWRTDataSet2IN->ui32TEMTILE2,
+				  psRGXCreateHWRTDataSet2IN->ui32MTileStride,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeLowerX,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeLowerY,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeUpperX,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeUpperY,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeScaleX,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMergeScaleY,
+				  sMacrotileArrayDevVAddrInt,
+				  sRgnHeaderDevVAddrInt,
+				  sRTCDevVAddrInt,
+				  psRGXCreateHWRTDataSet2IN->ui32RgnHeaderSize,
+				  psRGXCreateHWRTDataSet2IN->ui32ISPMtileSize,
+				  psRGXCreateHWRTDataSet2IN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMMlistsReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMMlistsReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSet2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+
+/*
+ * Register all RGXTA3D functions with services
+ */
+PVRSRV_ERROR InitRGXTA3DBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET,
+			      PVRSRVBridgeRGXDestroyHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER,
+			      PVRSRVBridgeRGXCreateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER,
+			      PVRSRVBridgeRGXDestroyZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXPopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXUnpopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST,
+			      PVRSRVBridgeRGXDestroyFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT,
+			      PVRSRVBridgeRGXDestroyRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE,
+			      PVRSRVBridgeRGXSendZSStoreDisable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetRenderContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED,
+			      PVRSRVBridgeRGXRenderContextStalled, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2,
+			      PVRSRVBridgeRGXKickTA3D2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKTA3D2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTA3D2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetRenderContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET,
+			      PVRSRVBridgeRGXCreateHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST,
+			      PVRSRVBridgeRGXCreateFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT,
+			      PVRSRVBridgeRGXCreateRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2,
+			      PVRSRVBridgeRGXCreateHWRTDataSet2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxta3d functions with services
+ */
+void DeinitRGXTA3DBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
new file mode 100644
index 000000000000..34d7c2c56a2c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTIMERQUERY_BRIDGE_H
+#define COMMON_RGXTIMERQUERY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST			(PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2)
+
+/*******************************************
+            RGXBeginTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY;
+
+/* Bridge out structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY;
+
+/*******************************************
+            RGXEndTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY;
+
+/* Bridge out structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY;
+
+/*******************************************
+            RGXQueryTimer
+ *******************************************/
+
+/* Bridge in structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_IN_RGXQUERYTIMER_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXQUERYTIMER;
+
+/* Bridge out structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXQUERYTIMER_TAG
+{
+	IMG_UINT64 ui64EndTime;
+	IMG_UINT64 ui64StartTime;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXQUERYTIMER;
+
+#endif /* COMMON_RGXTIMERQUERY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
new file mode 100644
index 000000000000..4af3cfc35cba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
@@ -0,0 +1,171 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtimerquery.h"
+
+#include "common_rgxtimerquery_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXBeginTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXBeginTimerQueryIN_UI8,
+			       IMG_UINT8 * psRGXBeginTimerQueryOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryOUT_UI8,
+								     0);
+
+	psRGXBeginTimerQueryOUT->eError =
+	    PVRSRVRGXBeginTimerQueryKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXBeginTimerQueryIN->ui32QueryId);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEndTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXEndTimerQueryIN_UI8,
+			     IMG_UINT8 * psRGXEndTimerQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *psRGXEndTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *psRGXEndTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEndTimerQueryIN);
+
+	psRGXEndTimerQueryOUT->eError =
+	    PVRSRVRGXEndTimerQueryKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXQueryTimer(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXQueryTimerIN_UI8,
+			  IMG_UINT8 * psRGXQueryTimerOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXQUERYTIMER *psRGXQueryTimerIN =
+	    (PVRSRV_BRIDGE_IN_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *psRGXQueryTimerOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerOUT_UI8, 0);
+
+	psRGXQueryTimerOUT->eError =
+	    PVRSRVRGXQueryTimerKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXQueryTimerIN->ui32QueryId,
+				  &psRGXQueryTimerOUT->ui64StartTime,
+				  &psRGXQueryTimerOUT->ui64EndTime);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+
+/*
+ * Register all RGXTIMERQUERY functions with services
+ */
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY,
+			      PVRSRVBridgeRGXBeginTimerQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY,
+			      PVRSRVBridgeRGXEndTimerQuery, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER, PVRSRVBridgeRGXQueryTimer,
+			      NULL, sizeof(PVRSRV_BRIDGE_IN_RGXQUERYTIMER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXQUERYTIMER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtimerquery functions with services
+ */
+void DeinitRGXTIMERQUERYBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/common_rgxtq2_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/common_rgxtq2_bridge.h
new file mode 100644
index 000000000000..f20fca015dc0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/common_rgxtq2_bridge.h
@@ -0,0 +1,264 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ2_BRIDGE_H
+#define COMMON_RGXTQ2_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8)
+
+/*******************************************
+            RGXTDMCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXTDMNotifyWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXTDMSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXTDMSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXTDMSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ2_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/server_rgxtq2_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/server_rgxtq2_bridge.c
new file mode 100644
index 000000000000..1129b824d850
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq2_bridge/server_rgxtq2_bridge.c
@@ -0,0 +1,1610 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtdmtransfer.h"
+
+#include "common_rgxtq2_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXTDMCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMDestroyTransferContextKM((RGX_SERVER_TQ_TDM_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXTDMCreateTransferContextIN_UI8,
+					IMG_UINT8 * psRGXTDMCreateTransferContextOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXTDMCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXTDMCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMCreateTransferContextOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXTDMCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVRGXTDMCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+						psRGXTDMCreateTransferContextIN->i32Priority,
+						psRGXTDMCreateTransferContextIN->
+						ui32FrameworkCmdSize, ui8FrameworkCmdInt,
+						hPrivDataInt,
+						psRGXTDMCreateTransferContextIN->
+						ui32PackedCCBSizeU88,
+						psRGXTDMCreateTransferContextIN->ui32ContextFlags,
+						psRGXTDMCreateTransferContextIN->
+						ui64RobustnessAddress,
+						psRGXTDMCreateTransferContextIN->ui32MaxDeadlineMS,
+						&psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXTDMCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXTDMCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXTDMDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextIN_UI8,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMDestroyTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMDestroyTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	if (unlikely
+	    ((psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psRGXTDMDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						     psTransferContextInt,
+						     psRGXTDMSetTransferContextPriorityIN->
+						     i32Priority);
+
+RGXTDMSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateIN_UI8,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMNotifyWriteOffsetUpdateIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMNotifyWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMNotifyWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMNotifyWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMNotifyWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(psTransferContextInt,
+						  psRGXTDMNotifyWriteOffsetUpdateIN->
+						  ui32PDumpFlags);
+
+RGXTDMNotifyWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer2IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVRGXTDMSubmitTransferKM(psTransferContextInt,
+					 psRGXTDMSubmitTransfer2IN->ui32PDumpFlags,
+					 psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount,
+					 psUpdateUFOSyncPrimBlockInt,
+					 ui32UpdateSyncOffsetInt,
+					 ui32UpdateValueInt,
+					 psRGXTDMSubmitTransfer2IN->hCheckFenceFD,
+					 psRGXTDMSubmitTransfer2IN->hUpdateTimeline,
+					 &psRGXTDMSubmitTransfer2OUT->hUpdateFence,
+					 uiUpdateFenceNameInt,
+					 psRGXTDMSubmitTransfer2IN->ui32CommandSize,
+					 ui8FWCommandInt,
+					 psRGXTDMSubmitTransfer2IN->ui32ExternalJobReference,
+					 psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount,
+					 ui32SyncPMRFlagsInt,
+					 psSyncPMRsInt,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic1,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic2,
+					 psRGXTDMSubmitTransfer2IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTDMGetSharedMemoryIN);
+
+	psRGXTDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXTDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMGetSharedMemory_exit:
+
+	if (psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPropertyKM(psTransferContextInt,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui32Property,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui64Input,
+						     &psRGXTDMSetTransferContextPropertyOUT->
+						     ui64Output);
+
+RGXTDMSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer3IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVRGXTDMSubmitTransfer3KM(psTransferContextInt,
+					  psRGXTDMSubmitTransfer3IN->ui32PDumpFlags,
+					  psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount,
+					  psUpdateUFOSyncPrimBlockInt,
+					  ui32UpdateSyncOffsetInt,
+					  ui32UpdateValueInt,
+					  psRGXTDMSubmitTransfer3IN->hCheckFenceFD,
+					  psRGXTDMSubmitTransfer3IN->hUpdateTimeline,
+					  &psRGXTDMSubmitTransfer3OUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXTDMSubmitTransfer3IN->hExportFenceToSignal,
+					  psRGXTDMSubmitTransfer3IN->ui32CommandSize,
+					  ui8FWCommandInt,
+					  psRGXTDMSubmitTransfer3IN->ui32ExternalJobReference,
+					  psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount,
+					  ui32SyncPMRFlagsInt,
+					  psSyncPMRsInt,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic1,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic2,
+					  psRGXTDMSubmitTransfer3IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+
+/*
+ * Register all RGXTQ2 functions with services
+ */
+PVRSRV_ERROR InitRGXTQ2Bridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXTDMSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXTDMSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXTDMSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXTDMSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq2 functions with services
+ */
+void DeinitRGXTQ2Bridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/common_rgxtq_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/common_rgxtq_bridge.h
new file mode 100644
index 000000000000..31d90dc0f993
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/common_rgxtq_bridge.h
@@ -0,0 +1,245 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ_BRIDGE_H
+#define COMMON_RGXTQ_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+7)
+
+/*******************************************
+            RGXCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdize;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32ClientUpdateCount;
+	IMG_UINT32 *pui32CommandSize;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32TQPrepareFlags;
+	IMG_UINT32 **pui32UpdateSyncOffset;
+	IMG_UINT32 **pui32UpdateValue;
+	IMG_UINT8 **pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE **phUpdateUFOSyncPrimBlock;
+	PVRSRV_TIMELINE h2DUpdateTimeline;
+	PVRSRV_TIMELINE h3DUpdateTimeline;
+	PVRSRV_FENCE hCheckFenceFD;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PrepareCount;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE h2DUpdateFence;
+	PVRSRV_FENCE h3DUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTQGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTQGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTQGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTQReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTQReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTQReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32ClientUpdateCount;
+	IMG_UINT32 *pui32CommandSize;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32TQPrepareFlags;
+	IMG_UINT32 **pui32UpdateSyncOffset;
+	IMG_UINT32 **pui32UpdateValue;
+	IMG_UINT8 **pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE **phUpdateUFOSyncPrimBlock;
+	PVRSRV_TIMELINE h2DUpdateTimeline;
+	PVRSRV_TIMELINE h3DUpdateTimeline;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PrepareCount;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE h2DUpdateFence;
+	PVRSRV_FENCE h3DUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/server_rgxtq_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/server_rgxtq_bridge.c
new file mode 100644
index 000000000000..f50c1184676a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/rgxtq_bridge/server_rgxtq_bridge.c
@@ -0,0 +1,1954 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtransfer.h"
+#include "rgx_tq_shared.h"
+
+#include "common_rgxtq_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyTransferContextKM((RGX_SERVER_TQ_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateTransferContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateTransferContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT *psRGXCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT *psRGXCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateTransferContextIN->ui32FrameworkCmdize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateTransferContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateTransferContextIN->ui32FrameworkCmdize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVRGXCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXCreateTransferContextIN->i32Priority,
+					     psRGXCreateTransferContextIN->ui32FrameworkCmdize,
+					     ui8FrameworkCmdInt,
+					     hPrivDataInt,
+					     psRGXCreateTransferContextIN->ui32PackedCCBSizeU8888,
+					     psRGXCreateTransferContextIN->ui32ContextFlags,
+					     psRGXCreateTransferContextIN->ui64RobustnessAddress,
+					     &psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyTransferContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyTransferContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT *psRGXDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT *psRGXDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyTransferContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetTransferContextPriorityIN_UI8,
+					  IMG_UINT8 * psRGXSetTransferContextPriorityOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY *psRGXSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY *psRGXSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						  psTransferContextInt,
+						  psRGXSetTransferContextPriorityIN->i32Priority);
+
+RGXSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXSubmitTransfer2IN_UI8,
+			       IMG_UINT8 * psRGXSubmitTransfer2OUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2 *psRGXSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2 *psRGXSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer2OUT_UI8,
+								     0);
+
+	IMG_HANDLE hTransferContext = psRGXSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateCountInt = NULL;
+	SYNC_PRIMITIVE_BLOCK ***psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE **hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 **ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 **ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT32 *ui32CommandSizeInt = NULL;
+	IMG_UINT8 **ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32TQPrepareFlagsInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BYTE *pArrayArgsBuffer2 = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+	IMG_UINT32 ui32BufferSize2 = 0;
+	IMG_UINT32 ui32NextOffset2 = 0;
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount *
+		     sizeof(SYNC_PRIMITIVE_BLOCK **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_HANDLE **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT8 *));
+	}
+
+	if (unlikely(psRGXSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32ClientUpdateCountInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateCountInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32ClientUpdateCount,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer for first dimension */
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK ***) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(SYNC_PRIMITIVE_BLOCK **);
+		/* Assigning hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer for first dimension */
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_HANDLE);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateSyncOffsetInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateValueInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateValueInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32CommandSizeInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CommandSizeInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32CommandSize,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui8FWCommandInt to the right offset in the pool buffer for first dimension */
+		ui8FWCommandInt = (IMG_UINT8 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT8 *);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32TQPrepareFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32TQPrepareFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32TQPrepareFlags,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXSubmitTransfer2IN->phSyncPMRs,
+		     psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		ui64BufferSize = 0;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] *
+			     sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize += ((IMG_UINT64) ui32CommandSizeInt[i] * sizeof(IMG_UINT8));
+		}
+		if (ui64BufferSize > IMG_UINT32_MAX)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			goto RGXSubmitTransfer2_exit;
+		}
+		ui32BufferSize2 = (IMG_UINT32) ui64BufferSize;
+	}
+
+	if (ui32BufferSize2 != 0)
+	{
+		pArrayArgsBuffer2 = OSAllocMemNoStats(ui32BufferSize2);
+
+		if (!pArrayArgsBuffer2)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			if (ui32ClientUpdateCountInt[i] > PVRSRV_MAX_SYNCS)
+			{
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Assigning each psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer (this is the second dimension) */
+			psUpdateUFOSyncPrimBlockInt[i] =
+			    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer2,
+								      ui32NextOffset2);
+			OSCachedMemSet(psUpdateUFOSyncPrimBlockInt[i], 0,
+				       ui32ClientUpdateCountInt[i] *
+				       sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui32NextOffset2 +=
+			    ui32ClientUpdateCountInt[i] * sizeof(SYNC_PRIMITIVE_BLOCK *);
+			/* Assigning each hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer (this is the second dimension) */
+			hUpdateUFOSyncPrimBlockInt2[i] =
+			    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateSyncOffsetInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateSyncOffsetInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateValueInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateValueInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			if (ui32CommandSizeInt[i] > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE)
+			{
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Assigning each ui8FWCommandInt to the right offset in the pool buffer (this is the second dimension) */
+			ui8FWCommandInt[i] =
+			    (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32CommandSizeInt[i] * sizeof(IMG_UINT8);
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_HANDLE **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->
+			     phUpdateUFOSyncPrimBlock[i], sizeof(IMG_HANDLE **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (hUpdateUFOSyncPrimBlockInt2[i]),
+				     (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui32UpdateSyncOffset[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateSyncOffsetInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui32UpdateValue[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateValueInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT8 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui8FWCommand[i],
+			     sizeof(IMG_UINT8 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32CommandSizeInt[i] * sizeof(IMG_UINT8)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui8FWCommandInt[i]), (const void __user *)psPtr,
+				     (ui32CommandSizeInt[i] * sizeof(IMG_UINT8))) != PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+				/* Look up the address from the handle */
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+							       (void **)
+							       &psUpdateUFOSyncPrimBlockInt[i][j],
+							       hUpdateUFOSyncPrimBlockInt2[i][j],
+							       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+							       IMG_TRUE);
+				if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+				{
+					UnlockHandle(psConnection->psHandleBase);
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSubmitTransfer2OUT->eError =
+	    PVRSRVRGXSubmitTransferKM(psTransferContextInt,
+				      psRGXSubmitTransfer2IN->ui32PrepareCount,
+				      ui32ClientUpdateCountInt,
+				      psUpdateUFOSyncPrimBlockInt,
+				      ui32UpdateSyncOffsetInt,
+				      ui32UpdateValueInt,
+				      psRGXSubmitTransfer2IN->hCheckFenceFD,
+				      psRGXSubmitTransfer2IN->h2DUpdateTimeline,
+				      &psRGXSubmitTransfer2OUT->h2DUpdateFence,
+				      psRGXSubmitTransfer2IN->h3DUpdateTimeline,
+				      &psRGXSubmitTransfer2OUT->h3DUpdateFence,
+				      uiUpdateFenceNameInt,
+				      ui32CommandSizeInt,
+				      ui8FWCommandInt,
+				      ui32TQPrepareFlagsInt,
+				      psRGXSubmitTransfer2IN->ui32ExtJobRef,
+				      psRGXSubmitTransfer2IN->ui32SyncPMRCount,
+				      ui32SyncPMRFlagsInt, psSyncPMRsInt);
+
+RGXSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+
+				/* Unreference the previously looked up handle */
+				if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i]
+				    && psUpdateUFOSyncPrimBlockInt[i][j])
+				{
+					PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+								    hUpdateUFOSyncPrimBlockInt2[i]
+								    [j],
+								    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+				}
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize2 == ui32NextOffset2);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (pArrayArgsBuffer2)
+		OSFreeMemNoStats(pArrayArgsBuffer2);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTQGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTQReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTQGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXTQGetSharedMemoryIN_UI8,
+				 IMG_UINT8 * psRGXTQGetSharedMemoryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY *psRGXTQGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY *) IMG_OFFSET_ADDR(psRGXTQGetSharedMemoryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY *psRGXTQGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTQGetSharedMemoryIN);
+
+	psRGXTQGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTQGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection), &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTQGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTQGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRGXTQGetSharedMemoryOUT->
+								      hCLIPMRMem,
+								      (void *)psCLIPMRMemInt,
+								      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RGXTQGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTQGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTQGetSharedMemory_exit:
+
+	if (psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTQReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTQReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXTQReleaseSharedMemoryIN_UI8,
+				     IMG_UINT8 * psRGXTQReleaseSharedMemoryOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY *psRGXTQReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY *psRGXTQReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQReleaseSharedMemoryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTQReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTQReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTQReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTQReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTQReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetTransferContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetTransferContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY *psRGXSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY *psRGXSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXSetTransferContextPropertyKM(psTransferContextInt,
+						  psRGXSetTransferContextPropertyIN->ui32Property,
+						  psRGXSetTransferContextPropertyIN->ui64Input,
+						  &psRGXSetTransferContextPropertyOUT->ui64Output);
+
+RGXSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXSubmitTransfer3IN_UI8,
+			       IMG_UINT8 * psRGXSubmitTransfer3OUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3 *psRGXSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3 *psRGXSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer3OUT_UI8,
+								     0);
+
+	IMG_HANDLE hTransferContext = psRGXSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateCountInt = NULL;
+	SYNC_PRIMITIVE_BLOCK ***psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE **hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 **ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 **ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT32 *ui32CommandSizeInt = NULL;
+	IMG_UINT8 **ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32TQPrepareFlagsInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BYTE *pArrayArgsBuffer2 = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+	IMG_UINT32 ui32BufferSize2 = 0;
+	IMG_UINT32 ui32NextOffset2 = 0;
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount *
+		     sizeof(SYNC_PRIMITIVE_BLOCK **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_HANDLE **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT8 *));
+	}
+
+	if (unlikely(psRGXSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32ClientUpdateCountInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateCountInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32ClientUpdateCount,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer for first dimension */
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK ***) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(SYNC_PRIMITIVE_BLOCK **);
+		/* Assigning hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer for first dimension */
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_HANDLE);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateSyncOffsetInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateValueInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateValueInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32CommandSizeInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CommandSizeInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32CommandSize,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui8FWCommandInt to the right offset in the pool buffer for first dimension */
+		ui8FWCommandInt = (IMG_UINT8 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT8 *);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32TQPrepareFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32TQPrepareFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32TQPrepareFlags,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXSubmitTransfer3IN->phSyncPMRs,
+		     psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		ui64BufferSize = 0;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] *
+			     sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize += ((IMG_UINT64) ui32CommandSizeInt[i] * sizeof(IMG_UINT8));
+		}
+		if (ui64BufferSize > IMG_UINT32_MAX)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			goto RGXSubmitTransfer3_exit;
+		}
+		ui32BufferSize2 = (IMG_UINT32) ui64BufferSize;
+	}
+
+	if (ui32BufferSize2 != 0)
+	{
+		pArrayArgsBuffer2 = OSAllocMemNoStats(ui32BufferSize2);
+
+		if (!pArrayArgsBuffer2)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			if (ui32ClientUpdateCountInt[i] > PVRSRV_MAX_SYNCS)
+			{
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Assigning each psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer (this is the second dimension) */
+			psUpdateUFOSyncPrimBlockInt[i] =
+			    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer2,
+								      ui32NextOffset2);
+			OSCachedMemSet(psUpdateUFOSyncPrimBlockInt[i], 0,
+				       ui32ClientUpdateCountInt[i] *
+				       sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui32NextOffset2 +=
+			    ui32ClientUpdateCountInt[i] * sizeof(SYNC_PRIMITIVE_BLOCK *);
+			/* Assigning each hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer (this is the second dimension) */
+			hUpdateUFOSyncPrimBlockInt2[i] =
+			    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateSyncOffsetInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateSyncOffsetInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateValueInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateValueInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			if (ui32CommandSizeInt[i] > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE)
+			{
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Assigning each ui8FWCommandInt to the right offset in the pool buffer (this is the second dimension) */
+			ui8FWCommandInt[i] =
+			    (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32CommandSizeInt[i] * sizeof(IMG_UINT8);
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_HANDLE **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->
+			     phUpdateUFOSyncPrimBlock[i], sizeof(IMG_HANDLE **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (hUpdateUFOSyncPrimBlockInt2[i]),
+				     (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui32UpdateSyncOffset[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateSyncOffsetInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui32UpdateValue[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateValueInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT8 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui8FWCommand[i],
+			     sizeof(IMG_UINT8 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32CommandSizeInt[i] * sizeof(IMG_UINT8)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui8FWCommandInt[i]), (const void __user *)psPtr,
+				     (ui32CommandSizeInt[i] * sizeof(IMG_UINT8))) != PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+				/* Look up the address from the handle */
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+							       (void **)
+							       &psUpdateUFOSyncPrimBlockInt[i][j],
+							       hUpdateUFOSyncPrimBlockInt2[i][j],
+							       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+							       IMG_TRUE);
+				if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+				{
+					UnlockHandle(psConnection->psHandleBase);
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSubmitTransfer3OUT->eError =
+	    PVRSRVRGXSubmitTransfer3KM(psTransferContextInt,
+				       psRGXSubmitTransfer3IN->ui32PrepareCount,
+				       ui32ClientUpdateCountInt,
+				       psUpdateUFOSyncPrimBlockInt,
+				       ui32UpdateSyncOffsetInt,
+				       ui32UpdateValueInt,
+				       psRGXSubmitTransfer3IN->hCheckFenceFD,
+				       psRGXSubmitTransfer3IN->h2DUpdateTimeline,
+				       &psRGXSubmitTransfer3OUT->h2DUpdateFence,
+				       psRGXSubmitTransfer3IN->h3DUpdateTimeline,
+				       &psRGXSubmitTransfer3OUT->h3DUpdateFence,
+				       uiUpdateFenceNameInt,
+				       psRGXSubmitTransfer3IN->hExportFenceToSignal,
+				       ui32CommandSizeInt,
+				       ui8FWCommandInt,
+				       ui32TQPrepareFlagsInt,
+				       psRGXSubmitTransfer3IN->ui32ExtJobRef,
+				       psRGXSubmitTransfer3IN->ui32SyncPMRCount,
+				       ui32SyncPMRFlagsInt, psSyncPMRsInt);
+
+RGXSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+
+				/* Unreference the previously looked up handle */
+				if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i]
+				    && psUpdateUFOSyncPrimBlockInt[i][j])
+				{
+					PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+								    hUpdateUFOSyncPrimBlockInt2[i]
+								    [j],
+								    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+				}
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize2 == ui32NextOffset2);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (pArrayArgsBuffer2)
+		OSFreeMemNoStats(pArrayArgsBuffer2);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXTQ_BRIDGE */
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+PVRSRV_ERROR InitRGXTQBridge(void);
+void DeinitRGXTQBridge(void);
+
+/*
+ * Register all RGXTQ functions with services
+ */
+PVRSRV_ERROR InitRGXTQBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+			      PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTQGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTQReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+			      PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq functions with services
+ */
+void DeinitRGXTQBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+				PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+				PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3);
+
+}
+#else /* SUPPORT_RGXTQ_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXTQ_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXTQBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXTQBridge()
+
+#endif /* SUPPORT_RGXTQ_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_bridge.h
new file mode 100644
index 000000000000..c419e21e9d07
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_bridge.h
@@ -0,0 +1,88 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RI_BRIDGE_H
+#define CLIENT_RI_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_ri_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner);
+
+#endif /* CLIENT_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_direct_bridge.c
new file mode 100644
index 000000000000..de037ee78be5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/client_ri_direct_bridge.c
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for ri
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_ri_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "ri_typedefs.h"
+
+#include "ri_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError =
+	    RIWriteMEMDESCEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psPMRHandleInt,
+				  ui32TextBSize,
+				  puiTextB, ui64Offset, ui64Size, ui64Flags, &psRIHandleInt);
+
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	eError =
+	    RIWriteProcListEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   ui32TextBSize, puiTextB, ui64Size, ui64DevVAddr, &psRIHandleInt);
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIUpdateMEMDESCAddrKM(psRIHandleInt, sAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIDeleteMEMDESCEntryKM(psRIHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIDumpListKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpAllKM();
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpProcessKM(ui32Pid);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRHandleInt, ui32Owner);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/common_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/common_ri_bridge.h
new file mode 100644
index 000000000000..baa41adccdfe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/common_ri_bridge.h
@@ -0,0 +1,224 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RI_BRIDGE_H
+#define COMMON_RI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "ri_typedefs.h"
+
+#define PVRSRV_BRIDGE_RI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR			PVRSRV_BRIDGE_RI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RI_RIDUMPLIST			PVRSRV_BRIDGE_RI_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RI_RIDUMPALL			PVRSRV_BRIDGE_RI_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RI_RIDUMPPROCESS			PVRSRV_BRIDGE_RI_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER			PVRSRV_BRIDGE_RI_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RI_CMD_LAST			(PVRSRV_BRIDGE_RI_CMD_FIRST+8)
+
+/*******************************************
+            RIWritePMREntry
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY;
+
+/* Bridge out structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY;
+
+/*******************************************
+            RIWriteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_UINT64 ui64Flags;
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRHandle;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY;
+
+/* Bridge out structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY;
+
+/*******************************************
+            RIWriteProcListEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_UINT64 ui64DevVAddr;
+	IMG_UINT64 ui64Size;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY;
+
+/* Bridge out structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY;
+
+/*******************************************
+            RIUpdateMEMDESCAddr
+ *******************************************/
+
+/* Bridge in structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR_TAG
+{
+	IMG_DEV_VIRTADDR sAddr;
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR;
+
+/* Bridge out structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR;
+
+/*******************************************
+            RIDeleteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY;
+
+/* Bridge out structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY;
+
+/*******************************************
+            RIDumpList
+ *******************************************/
+
+/* Bridge in structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPLIST_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPLIST;
+
+/* Bridge out structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPLIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPLIST;
+
+/*******************************************
+            RIDumpAll
+ *******************************************/
+
+/* Bridge in structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPALL_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPALL;
+
+/* Bridge out structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPALL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPALL;
+
+/*******************************************
+            RIDumpProcess
+ *******************************************/
+
+/* Bridge in structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPPROCESS_TAG
+{
+	IMG_PID ui32Pid;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPPROCESS;
+
+/* Bridge out structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPPROCESS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPPROCESS;
+
+/*******************************************
+            RIWritePMREntryWithOwner
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	IMG_HANDLE hPMRHandle;
+	IMG_PID ui32Owner;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER;
+
+/* Bridge out structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER;
+
+#endif /* COMMON_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/server_ri_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/server_ri_bridge.c
new file mode 100644
index 000000000000..f8a54d70ef0c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/ri_bridge/server_ri_bridge.c
@@ -0,0 +1,761 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "ri_server.h"
+
+#include "common_ri_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntry(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRIWritePMREntryIN_UI8,
+			    IMG_UINT8 * psRIWritePMREntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *psRIWritePMREntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *psRIWritePMREntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryOUT->eError = RIWritePMREntryKM(psPMRHandleInt);
+
+RIWritePMREntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteMEMDESCEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIWriteMEMDESCEntryIN_UI8,
+				IMG_UINT8 * psRIWriteMEMDESCEntryOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMRHandle = psRIWriteMEMDESCEntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteMEMDESCEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteMEMDESCEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteMEMDESCEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteMEMDESCEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteMEMDESCEntryIN->puiTextB,
+		     psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteMEMDESCEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWriteMEMDESCEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError =
+	    RIWriteMEMDESCEntryKM(psConnection, OSGetDevNode(psConnection),
+				  psPMRHandleInt,
+				  psRIWriteMEMDESCEntryIN->ui32TextBSize,
+				  uiTextBInt,
+				  psRIWriteMEMDESCEntryIN->ui64Offset,
+				  psRIWriteMEMDESCEntryIN->ui64Size,
+				  psRIWriteMEMDESCEntryIN->ui64Flags, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRIWriteMEMDESCEntryOUT->
+								     hRIHandle,
+								     (void *)psRIHandleInt,
+								     PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RIWriteMEMDESCEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteMEMDESCEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteMEMDESCEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteProcListEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteProcListEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIWriteProcListEntryIN_UI8,
+				 IMG_UINT8 * psRIWriteProcListEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *) IMG_OFFSET_ADDR(psRIWriteProcListEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *)
+	    IMG_OFFSET_ADDR(psRIWriteProcListEntryOUT_UI8, 0);
+
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteProcListEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteProcListEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteProcListEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteProcListEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteProcListEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteProcListEntryIN->puiTextB,
+		     psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteProcListEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psRIWriteProcListEntryOUT->eError =
+	    RIWriteProcListEntryKM(psConnection, OSGetDevNode(psConnection),
+				   psRIWriteProcListEntryIN->ui32TextBSize,
+				   uiTextBInt,
+				   psRIWriteProcListEntryIN->ui64Size,
+				   psRIWriteProcListEntryIN->ui64DevVAddr, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteProcListEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRIWriteProcListEntryOUT->
+								      hRIHandle,
+								      (void *)psRIHandleInt,
+								      PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RIWriteProcListEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteProcListEntry_exit:
+
+	if (psRIWriteProcListEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteProcListEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIUpdateMEMDESCAddr(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrIN_UI8,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrIN =
+	    (PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrOUT_UI8,
+								      0);
+
+	IMG_HANDLE hRIHandle = psRIUpdateMEMDESCAddrIN->hRIHandle;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRIHandleInt,
+				       hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE, IMG_TRUE);
+	if (unlikely(psRIUpdateMEMDESCAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIUpdateMEMDESCAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    RIUpdateMEMDESCAddrKM(psRIHandleInt, psRIUpdateMEMDESCAddrIN->sAddr);
+
+RIUpdateMEMDESCAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRIHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDeleteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryIN_UI8,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *)
+	    IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIDeleteMEMDESCEntryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRIDeleteMEMDESCEntryIN->hRIHandle,
+					      PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	if (unlikely((psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_OK) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRIDeleteMEMDESCEntryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDeleteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIDeleteMEMDESCEntry_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpList(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRIDumpListIN_UI8,
+		       IMG_UINT8 * psRIDumpListOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPLIST *psRIDumpListIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPLIST *psRIDumpListOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIDumpListIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIDumpListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIDumpListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDumpList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIDumpListOUT->eError = RIDumpListKM(psPMRHandleInt);
+
+RIDumpList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpAll(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psRIDumpAllIN_UI8,
+		      IMG_UINT8 * psRIDumpAllOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPALL *psRIDumpAllIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPALL *psRIDumpAllOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psRIDumpAllIN);
+
+	psRIDumpAllOUT->eError = RIDumpAllKM();
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpProcess(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRIDumpProcessIN_UI8,
+			  IMG_UINT8 * psRIDumpProcessOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPPROCESS *psRIDumpProcessIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *psRIDumpProcessOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psRIDumpProcessOUT->eError = RIDumpProcessKM(psRIDumpProcessIN->ui32Pid);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntryWithOwner(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerIN_UI8,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryWithOwnerIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryWithOwnerOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntryWithOwner_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    RIWritePMREntryWithOwnerKM(psPMRHandleInt, psRIWritePMREntryWithOwnerIN->ui32Owner);
+
+RIWritePMREntryWithOwner_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+
+/*
+ * Register all RI functions with services
+ */
+PVRSRV_ERROR InitRIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY,
+			      PVRSRVBridgeRIWritePMREntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY,
+			      PVRSRVBridgeRIWriteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY,
+			      PVRSRVBridgeRIWriteProcListEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR,
+			      PVRSRVBridgeRIUpdateMEMDESCAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY,
+			      PVRSRVBridgeRIDeleteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST, PVRSRVBridgeRIDumpList,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPLIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPLIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL, PVRSRVBridgeRIDumpAll,
+			      NULL, 0, sizeof(PVRSRV_BRIDGE_OUT_RIDUMPALL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS,
+			      PVRSRVBridgeRIDumpProcess, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPPROCESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPPROCESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER,
+			      PVRSRVBridgeRIWritePMREntryWithOwner, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all ri functions with services
+ */
+void DeinitRIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/common_smm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/common_smm_bridge.h
new file mode 100644
index 000000000000..7d6603dfbe8b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/common_smm_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SMM_BRIDGE_H
+#define COMMON_SMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SMM_CMD_LAST			(PVRSRV_BRIDGE_SMM_CMD_FIRST+2)
+
+/*******************************************
+            PMRSecureExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR;
+
+/* Bridge out structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR;
+
+/*******************************************
+            PMRSecureUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR;
+
+/* Bridge out structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR;
+
+/*******************************************
+            PMRSecureImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR;
+
+/* Bridge out structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR;
+
+#endif /* COMMON_SMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/server_smm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/server_smm_bridge.c
new file mode 100644
index 000000000000..5641b1ea12f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/smm_bridge/server_smm_bridge.c
@@ -0,0 +1,281 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "secure_export.h"
+
+#include "common_smm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRSecureExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureExportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureExportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *psPMRSecureExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *psPMRSecureExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMROUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRSecureExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR *psPMROutInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRSecureExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRSecureExportPMROUT->eError =
+	    PMRSecureExportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRInt,
+			       &psPMRSecureExportPMROUT->Export, &psPMROutInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureExportPMR_exit;
+	}
+
+PMRSecureExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRSecureExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMROutInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRSecureUnexportPMR(psPMROutInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRSecureUnexportPMRIN_UI8,
+				 IMG_UINT8 * psPMRSecureUnexportPMROUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureUnexportPMRIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *)
+	    IMG_OFFSET_ADDR(psPMRSecureUnexportPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRSecureUnexportPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_SECURE_EXPORT);
+	if (unlikely((psPMRSecureUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRSecureUnexportPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureUnexportPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRSecureImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureImportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureImportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *psPMRSecureImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *psPMRSecureImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMROUT_UI8,
+								     0);
+
+	PMR *psPMRInt = NULL;
+
+	psPMRSecureImportPMROUT->eError =
+	    PMRSecureImportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRSecureImportPMRIN->Export,
+			       &psPMRInt,
+			       &psPMRSecureImportPMROUT->uiSize, &psPMRSecureImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psPMRSecureImportPMROUT->hPMR,
+								    (void *)psPMRInt,
+								    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _PMRSecureImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureImportPMR_exit:
+
+	if (psPMRSecureImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSMMBridge(void);
+void DeinitSMMBridge(void);
+
+/*
+ * Register all SMM functions with services
+ */
+PVRSRV_ERROR InitSMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR,
+			      PVRSRVBridgePMRSecureExportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR,
+			      PVRSRVBridgePMRSecureUnexportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR,
+			      PVRSRVBridgePMRSecureImportPMR, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all smm functions with services
+ */
+void DeinitSMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/common_srvcore_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/common_srvcore_bridge.h
new file mode 100644
index 000000000000..603ba5b1d89e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/common_srvcore_bridge.h
@@ -0,0 +1,369 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SRVCORE_BRIDGE_H
+#define COMMON_SRVCORE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_device_types.h"
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_SRVCORE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SRVCORE_CONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SRVCORE_DISCONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16
+#define PVRSRV_BRIDGE_SRVCORE_CMD_LAST			(PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16)
+
+/*******************************************
+            Connect
+ *******************************************/
+
+/* Bridge in structure for Connect */
+typedef struct PVRSRV_BRIDGE_IN_CONNECT_TAG
+{
+	IMG_UINT32 ui32ClientBuildOptions;
+	IMG_UINT32 ui32ClientDDKBuild;
+	IMG_UINT32 ui32ClientDDKVersion;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_CONNECT;
+
+/* Bridge out structure for Connect */
+typedef struct PVRSRV_BRIDGE_OUT_CONNECT_TAG
+{
+	IMG_UINT64 ui64PackedBvnc;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CapabilityFlags;
+	IMG_UINT8 ui8KernelArch;
+} __packed PVRSRV_BRIDGE_OUT_CONNECT;
+
+/*******************************************
+            Disconnect
+ *******************************************/
+
+/* Bridge in structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_IN_DISCONNECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DISCONNECT;
+
+/* Bridge out structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_OUT_DISCONNECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DISCONNECT;
+
+/*******************************************
+            AcquireGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT;
+
+/* Bridge out structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT;
+
+/*******************************************
+            ReleaseGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+} __packed PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT;
+
+/* Bridge out structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT;
+
+/*******************************************
+            EventObjectOpen
+ *******************************************/
+
+/* Bridge in structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hEventObject;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN;
+
+/* Bridge out structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN;
+
+/*******************************************
+            EventObjectWait
+ *******************************************/
+
+/* Bridge in structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT;
+
+/* Bridge out structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT;
+
+/*******************************************
+            EventObjectClose
+ *******************************************/
+
+/* Bridge in structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE;
+
+/* Bridge out structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE;
+
+/*******************************************
+            DumpDebugInfo
+ *******************************************/
+
+/* Bridge in structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_IN_DUMPDEBUGINFO_TAG
+{
+	IMG_UINT32 ui32VerbLevel;
+} __packed PVRSRV_BRIDGE_IN_DUMPDEBUGINFO;
+
+/* Bridge out structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO;
+
+/*******************************************
+            GetDevClockSpeed
+ *******************************************/
+
+/* Bridge in structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED;
+
+/* Bridge out structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ClockSpeed;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED;
+
+/*******************************************
+            HWOpTimeout
+ *******************************************/
+
+/* Bridge in structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_IN_HWOPTIMEOUT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HWOPTIMEOUT;
+
+/* Bridge out structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_HWOPTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HWOPTIMEOUT;
+
+/*******************************************
+            AlignmentCheck
+ *******************************************/
+
+/* Bridge in structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_IN_ALIGNMENTCHECK_TAG
+{
+	IMG_UINT32 *pui32AlignChecks;
+	IMG_UINT32 ui32AlignChecksSize;
+} __packed PVRSRV_BRIDGE_IN_ALIGNMENTCHECK;
+
+/* Bridge out structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK;
+
+/*******************************************
+            GetDeviceStatus
+ *******************************************/
+
+/* Bridge in structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVICESTATUS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVICESTATUS;
+
+/* Bridge out structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVICESTATUS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceSatus;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVICESTATUS;
+
+/*******************************************
+            GetMultiCoreInfo
+ *******************************************/
+
+/* Bridge in structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_IN_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	IMG_UINT32 ui32CapsSize;
+} __packed PVRSRV_BRIDGE_IN_GETMULTICOREINFO;
+
+/* Bridge out structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_OUT_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumCores;
+} __packed PVRSRV_BRIDGE_OUT_GETMULTICOREINFO;
+
+/*******************************************
+            EventObjectWaitTimeout
+ *******************************************/
+
+/* Bridge in structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	IMG_UINT64 ui64uiTimeoutus;
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT;
+
+/* Bridge out structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT;
+
+/*******************************************
+            FindProcessMemStats
+ *******************************************/
+
+/* Bridge in structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	IMG_UINT32 ui32ArrSize;
+	IMG_UINT32 ui32PID;
+	IMG_BOOL bbAllProcessStats;
+} __packed PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS;
+
+/* Bridge out structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS;
+
+/*******************************************
+            AcquireInfoPage
+ *******************************************/
+
+/* Bridge in structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE;
+
+/* Bridge out structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE;
+
+/*******************************************
+            ReleaseInfoPage
+ *******************************************/
+
+/* Bridge in structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_RELEASEINFOPAGE;
+
+/* Bridge out structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE;
+
+#endif /* COMMON_SRVCORE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/server_srvcore_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/server_srvcore_bridge.c
new file mode 100644
index 000000000000..d40e53b244f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/srvcore_bridge/server_srvcore_bridge.c
@@ -0,0 +1,1080 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "srvcore.h"
+#include "info_page.h"
+#include "proc_stats.h"
+#include "rgx_fwif_alignchecks.h"
+
+#include "common_srvcore_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeConnect(IMG_UINT32 ui32DispatchTableEntry,
+		    IMG_UINT8 * psConnectIN_UI8,
+		    IMG_UINT8 * psConnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CONNECT *psConnectIN =
+	    (PVRSRV_BRIDGE_IN_CONNECT *) IMG_OFFSET_ADDR(psConnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CONNECT *psConnectOUT =
+	    (PVRSRV_BRIDGE_OUT_CONNECT *) IMG_OFFSET_ADDR(psConnectOUT_UI8, 0);
+
+	psConnectOUT->eError =
+	    PVRSRVConnectKM(psConnection, OSGetDevNode(psConnection),
+			    psConnectIN->ui32Flags,
+			    psConnectIN->ui32ClientBuildOptions,
+			    psConnectIN->ui32ClientDDKVersion,
+			    psConnectIN->ui32ClientDDKBuild,
+			    &psConnectOUT->ui8KernelArch,
+			    &psConnectOUT->ui32CapabilityFlags, &psConnectOUT->ui64PackedBvnc);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDisconnect(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDisconnectIN_UI8,
+		       IMG_UINT8 * psDisconnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DISCONNECT *psDisconnectIN =
+	    (PVRSRV_BRIDGE_IN_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DISCONNECT *psDisconnectOUT =
+	    (PVRSRV_BRIDGE_OUT_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDisconnectIN);
+
+	psDisconnectOUT->eError = PVRSRVDisconnectKM();
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireGlobalEventObjecthGlobalEventObjectIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseGlobalEventObjectKM((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psAcquireGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psAcquireGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectOUT_UI8, 0);
+
+	IMG_HANDLE hGlobalEventObjectInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireGlobalEventObjectIN);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAcquireGlobalEventObjectKM(&hGlobalEventObjectInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psAcquireGlobalEventObjectOUT->hGlobalEventObject,
+				      (void *)hGlobalEventObjectInt,
+				      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _AcquireGlobalEventObjecthGlobalEventObjectIntRelease);
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AcquireGlobalEventObject_exit:
+
+	if (psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK)
+	{
+		if (hGlobalEventObjectInt)
+		{
+			PVRSRVReleaseGlobalEventObjectKM(hGlobalEventObjectInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psReleaseGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psReleaseGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psReleaseGlobalEventObjectOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psReleaseGlobalEventObjectIN->
+					      hGlobalEventObject,
+					      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	if (unlikely
+	    ((psReleaseGlobalEventObjectOUT->eError != PVRSRV_OK)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psReleaseGlobalEventObjectOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto ReleaseGlobalEventObject_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+ReleaseGlobalEventObject_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _EventObjectOpenhOSEventIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = OSEventObjectClose((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectOpen(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectOpenIN_UI8,
+			    IMG_UINT8 * psEventObjectOpenOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *psEventObjectOpenIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *psEventObjectOpenOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenOUT_UI8, 0);
+
+	IMG_HANDLE hEventObject = psEventObjectOpenIN->hEventObject;
+	IMG_HANDLE hEventObjectInt = NULL;
+	IMG_HANDLE hOSEventInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectOpenOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hEventObjectInt,
+				       hEventObject,
+				       PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT, IMG_TRUE);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = OSEventObjectOpen(hEventObjectInt, &hOSEventInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		goto EventObjectOpen_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psEventObjectOpenOUT->hOSEvent,
+								 (void *)hOSEventInt,
+								 PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _EventObjectOpenhOSEventIntRelease);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectOpen_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hEventObjectInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hEventObject, PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psEventObjectOpenOUT->eError != PVRSRV_OK)
+	{
+		if (hOSEventInt)
+		{
+			OSEventObjectClose(hOSEventInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectWaitIN_UI8,
+			    IMG_UINT8 * psEventObjectWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *psEventObjectWaitIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *psEventObjectWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitOUT->eError = OSEventObjectWait(hOSEventKMInt);
+
+EventObjectWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectClose(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psEventObjectCloseIN_UI8,
+			     IMG_UINT8 * psEventObjectCloseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *psEventObjectCloseIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *psEventObjectCloseOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectCloseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psEventObjectCloseIN->hOSEventKM,
+					      PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	if (unlikely((psEventObjectCloseOUT->eError != PVRSRV_OK) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psEventObjectCloseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectClose_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectClose_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDumpDebugInfo(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDumpDebugInfoIN_UI8,
+			  IMG_UINT8 * psDumpDebugInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *psDumpDebugInfoIN =
+	    (PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *psDumpDebugInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoOUT_UI8, 0);
+
+	psDumpDebugInfoOUT->eError =
+	    PVRSRVDumpDebugInfoKM(psConnection, OSGetDevNode(psConnection),
+				  psDumpDebugInfoIN->ui32VerbLevel);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDevClockSpeed(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetDevClockSpeedIN_UI8,
+			     IMG_UINT8 * psGetDevClockSpeedOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *psGetDevClockSpeedIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *psGetDevClockSpeedOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDevClockSpeedIN);
+
+	psGetDevClockSpeedOUT->eError =
+	    PVRSRVGetDevClockSpeedKM(psConnection, OSGetDevNode(psConnection),
+				     &psGetDevClockSpeedOUT->ui32ClockSpeed);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHWOpTimeout(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psHWOpTimeoutIN_UI8,
+			IMG_UINT8 * psHWOpTimeoutOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HWOPTIMEOUT *psHWOpTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *psHWOpTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHWOpTimeoutIN);
+
+	psHWOpTimeoutOUT->eError = PVRSRVHWOpTimeoutKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static_assert(RGXFW_ALIGN_CHECKS_UM_MAX <= IMG_UINT32_MAX,
+	      "RGXFW_ALIGN_CHECKS_UM_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeAlignmentCheck(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psAlignmentCheckIN_UI8,
+			   IMG_UINT8 * psAlignmentCheckOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *psAlignmentCheckIN =
+	    (PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *psAlignmentCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AlignChecksInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psAlignmentCheckIN->ui32AlignChecksSize > RGXFW_ALIGN_CHECKS_UM_MAX))
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto AlignmentCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto AlignmentCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psAlignmentCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psAlignmentCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psAlignmentCheckOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto AlignmentCheck_exit;
+			}
+		}
+	}
+
+	if (psAlignmentCheckIN->ui32AlignChecksSize != 0)
+	{
+		ui32AlignChecksInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AlignChecksInt,
+		     (const void __user *)psAlignmentCheckIN->pui32AlignChecks,
+		     psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psAlignmentCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto AlignmentCheck_exit;
+		}
+	}
+
+	psAlignmentCheckOUT->eError =
+	    PVRSRVAlignmentCheckKM(psConnection, OSGetDevNode(psConnection),
+				   psAlignmentCheckIN->ui32AlignChecksSize, ui32AlignChecksInt);
+
+AlignmentCheck_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psAlignmentCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDeviceStatus(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psGetDeviceStatusIN_UI8,
+			    IMG_UINT8 * psGetDeviceStatusOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVICESTATUS *psGetDeviceStatusIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *psGetDeviceStatusOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDeviceStatusIN);
+
+	psGetDeviceStatusOUT->eError =
+	    PVRSRVGetDeviceStatusKM(psConnection, OSGetDevNode(psConnection),
+				    &psGetDeviceStatusOUT->ui32DeviceSatus);
+
+	return 0;
+}
+
+static_assert(8 <= IMG_UINT32_MAX, "8 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeGetMultiCoreInfo(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetMultiCoreInfoIN_UI8,
+			     IMG_UINT8 * psGetMultiCoreInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETMULTICOREINFO *psGetMultiCoreInfoIN =
+	    (PVRSRV_BRIDGE_IN_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *psGetMultiCoreInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoOUT_UI8, 0);
+
+	IMG_UINT64 *pui64CapsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize > 8)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	psGetMultiCoreInfoOUT->pui64Caps = psGetMultiCoreInfoIN->pui64Caps;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psGetMultiCoreInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psGetMultiCoreInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto GetMultiCoreInfo_exit;
+			}
+		}
+	}
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize != 0)
+	{
+		pui64CapsInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64);
+	}
+
+	psGetMultiCoreInfoOUT->eError =
+	    PVRSRVGetMultiCoreInfoKM(psConnection, OSGetDevNode(psConnection),
+				     psGetMultiCoreInfoIN->ui32CapsSize,
+				     &psGetMultiCoreInfoOUT->ui32NumCores, pui64CapsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psGetMultiCoreInfoOUT->eError != PVRSRV_OK))
+	{
+		goto GetMultiCoreInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64CapsInt) && ((psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psGetMultiCoreInfoOUT->pui64Caps, pui64CapsInt,
+		      (psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto GetMultiCoreInfo_exit;
+		}
+	}
+
+GetMultiCoreInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psGetMultiCoreInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWaitTimeout(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psEventObjectWaitTimeoutIN_UI8,
+				   IMG_UINT8 * psEventObjectWaitTimeoutOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitTimeoutIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitTimeoutOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitTimeoutOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWaitTimeout_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitTimeoutOUT->eError =
+	    OSEventObjectWaitTimeout(hOSEventKMInt, psEventObjectWaitTimeoutIN->ui64uiTimeoutus);
+
+EventObjectWaitTimeout_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PROCESS_STAT_TYPE_COUNT <= IMG_UINT32_MAX,
+	      "PVRSRV_PROCESS_STAT_TYPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeFindProcessMemStats(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psFindProcessMemStatsIN_UI8,
+				IMG_UINT8 * psFindProcessMemStatsOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *psFindProcessMemStatsIN =
+	    (PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *psFindProcessMemStatsOUT =
+	    (PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsOUT_UI8,
+								      0);
+
+	IMG_UINT64 *pui64MemStatsArrayInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psFindProcessMemStatsIN->ui32ArrSize > PVRSRV_PROCESS_STAT_TYPE_COUNT)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto FindProcessMemStats_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psFindProcessMemStatsOUT->pui64MemStatsArray = psFindProcessMemStatsIN->pui64MemStatsArray;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto FindProcessMemStats_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psFindProcessMemStatsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psFindProcessMemStatsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto FindProcessMemStats_exit;
+			}
+		}
+	}
+
+	if (psFindProcessMemStatsIN->ui32ArrSize != 0)
+	{
+		pui64MemStatsArrayInt =
+		    (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64);
+	}
+
+	psFindProcessMemStatsOUT->eError =
+	    PVRSRVFindProcessMemStatsKM(psFindProcessMemStatsIN->ui32PID,
+					psFindProcessMemStatsIN->ui32ArrSize,
+					psFindProcessMemStatsIN->bbAllProcessStats,
+					pui64MemStatsArrayInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psFindProcessMemStatsOUT->eError != PVRSRV_OK))
+	{
+		goto FindProcessMemStats_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64MemStatsArrayInt) &&
+	    ((psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psFindProcessMemStatsOUT->pui64MemStatsArray,
+		      pui64MemStatsArrayInt,
+		      (psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto FindProcessMemStats_exit;
+		}
+	}
+
+FindProcessMemStats_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psFindProcessMemStatsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireInfoPagepsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseInfoPageKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psAcquireInfoPageIN_UI8,
+			    IMG_UINT8 * psAcquireInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *psAcquireInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *psAcquireInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageOUT_UI8, 0);
+
+	PMR *psPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireInfoPageIN);
+
+	psAcquireInfoPageOUT->eError = PVRSRVAcquireInfoPageKM(&psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psAcquireInfoPageOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psAcquireInfoPageOUT->hPMR, (void *)psPMRInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _AcquireInfoPagepsPMRIntRelease);
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+AcquireInfoPage_exit:
+
+	if (psAcquireInfoPageOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			PVRSRVReleaseInfoPageKM(psPMRInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psReleaseInfoPageIN_UI8,
+			    IMG_UINT8 * psReleaseInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *psReleaseInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *psReleaseInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psReleaseInfoPageOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psReleaseInfoPageIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psReleaseInfoPageOUT->eError != PVRSRV_OK) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psReleaseInfoPageOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto ReleaseInfoPage_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+ReleaseInfoPage_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+
+/*
+ * Register all SRVCORE functions with services
+ */
+PVRSRV_ERROR InitSRVCOREBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT,
+			      PVRSRVBridgeConnect, NULL, sizeof(PVRSRV_BRIDGE_IN_CONNECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_CONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT,
+			      PVRSRVBridgeDisconnect, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DISCONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT,
+			      PVRSRVBridgeAcquireGlobalEventObject, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT,
+			      PVRSRVBridgeReleaseGlobalEventObject, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN,
+			      PVRSRVBridgeEventObjectOpen, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT,
+			      PVRSRVBridgeEventObjectWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE,
+			      PVRSRVBridgeEventObjectClose, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO,
+			      PVRSRVBridgeDumpDebugInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DUMPDEBUGINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED,
+			      PVRSRVBridgeGetDevClockSpeed, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT,
+			      PVRSRVBridgeHWOpTimeout, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HWOPTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK,
+			      PVRSRVBridgeAlignmentCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_ALIGNMENTCHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS,
+			      PVRSRVBridgeGetDeviceStatus, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVICESTATUS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO,
+			      PVRSRVBridgeGetMultiCoreInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_GETMULTICOREINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_GETMULTICOREINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT,
+			      PVRSRVBridgeEventObjectWaitTimeout, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS,
+			      PVRSRVBridgeFindProcessMemStats, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS),
+			      sizeof(PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE,
+			      PVRSRVBridgeAcquireInfoPage, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE,
+			      PVRSRVBridgeReleaseInfoPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEINFOPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all srvcore functions with services
+ */
+void DeinitSRVCOREBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_bridge.h
new file mode 100644
index 000000000000..19f1b0ece5b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_bridge.h
@@ -0,0 +1,102 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNC_BRIDGE_H
+#define CLIENT_SYNC_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_sync_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence);
+
+#endif /* CLIENT_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_direct_bridge.c
new file mode 100644
index 000000000000..d631aea421ad
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/client_sync_direct_bridge.c
@@ -0,0 +1,262 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for sync
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_sync_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    &psSyncHandleInt,
+					    pui32SyncPrimVAddr,
+					    pui32SyncPrimBlockSize, &pshSyncPMRInt);
+
+	*phSyncHandle = psSyncHandleInt;
+	*phhSyncPMR = pshSyncPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimSetKM(psSyncHandleInt, ui32Index, ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpKM(psSyncHandleInt, ui32Offset);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt, ui32Offset, ui32Value);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     ui32Offset, ui32Value, ui32Mask, eOperator, uiPDumpFlags);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     ui32Offset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVSyncAllocEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   bServerSync, ui32FWAddr, ui32ClassNameSize, puiClassName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVSyncFreeEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32FWAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PVRSRVSyncCheckpointSignalledPDumpPolKM(hFence);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hFence);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/common_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/common_sync_bridge.h
new file mode 100644
index 000000000000..753d40b64f12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/common_sync_bridge.h
@@ -0,0 +1,254 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNC_BRIDGE_H
+#define COMMON_SYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#define PVRSRV_BRIDGE_SYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMSET			PVRSRV_BRIDGE_SYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE			PVRSRV_BRIDGE_SYNC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNC_CMD_LAST			(PVRSRV_BRIDGE_SYNC_CMD_FIRST+9)
+
+/*******************************************
+            AllocSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_HANDLE hhSyncPMR;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32SyncPrimBlockSize;
+	IMG_UINT32 ui32SyncPrimVAddr;
+} __packed PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            FreeSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+} __packed PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            SyncPrimSet
+ *******************************************/
+
+/* Bridge in structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMSET_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMSET;
+
+/* Bridge out structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMSET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMSET;
+
+/*******************************************
+            SyncPrimPDump
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP;
+
+/* Bridge out structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP;
+
+/*******************************************
+            SyncPrimPDumpValue
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE;
+
+/* Bridge out structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE;
+
+/*******************************************
+            SyncPrimPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+	PDUMP_FLAGS_T uiPDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL;
+
+/* Bridge out structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL;
+
+/*******************************************
+            SyncPrimPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP;
+
+/* Bridge out structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP;
+
+/*******************************************
+            SyncAllocEvent
+ *******************************************/
+
+/* Bridge in structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCALLOCEVENT_TAG
+{
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32FWAddr;
+	IMG_BOOL bServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCALLOCEVENT;
+
+/* Bridge out structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT;
+
+/*******************************************
+            SyncFreeEvent
+ *******************************************/
+
+/* Bridge in structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFREEEVENT_TAG
+{
+	IMG_UINT32 ui32FWAddr;
+} __packed PVRSRV_BRIDGE_IN_SYNCFREEEVENT;
+
+/* Bridge out structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFREEEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFREEEVENT;
+
+/*******************************************
+            SyncCheckpointSignalledPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_FENCE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+/* Bridge out structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+#endif /* COMMON_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/server_sync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/server_sync_bridge.c
new file mode 100644
index 000000000000..a0d9210086c9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/sync_bridge/server_sync_bridge.c
@@ -0,0 +1,755 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+#include "common_sync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _AllocSyncPrimitiveBlockpsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVFreeSyncPrimitiveBlockKM((SYNC_PRIMITIVE_BLOCK *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAllocSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockIN_UI8,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockOUT_UI8, 0);
+
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAllocSyncPrimitiveBlockIN);
+
+	psAllocSyncPrimitiveBlockOUT->hSyncHandle = NULL;
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(psConnection, OSGetDevNode(psConnection),
+					    &psSyncHandleInt,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimVAddr,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimBlockSize,
+					    &pshSyncPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAllocSyncPrimitiveBlockOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psAllocSyncPrimitiveBlockOUT->
+									 hSyncHandle,
+									 (void *)psSyncHandleInt,
+									 PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _AllocSyncPrimitiveBlockpsSyncHandleIntRelease);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psAllocSyncPrimitiveBlockOUT->hhSyncPMR,
+					 (void *)pshSyncPMRInt,
+					 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psAllocSyncPrimitiveBlockOUT->hSyncHandle);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AllocSyncPrimitiveBlock_exit:
+
+	if (psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK)
+	{
+		if (psAllocSyncPrimitiveBlockOUT->hSyncHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psAllocSyncPrimitiveBlockOUT->
+							     hSyncHandle,
+							     PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSyncHandleInt)
+		{
+			PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeFreeSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockIN_UI8,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psFreeSyncPrimitiveBlockOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psFreeSyncPrimitiveBlockIN->hSyncHandle,
+					      PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	if (unlikely((psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_OK) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psFreeSyncPrimitiveBlockOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto FreeSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+FreeSyncPrimitiveBlock_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncPrimSet(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psSyncPrimSetIN_UI8,
+			IMG_UINT8 * psSyncPrimSetOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMSET *psSyncPrimSetIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMSET *psSyncPrimSetOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimSetIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimSetOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimSetOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimSet_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimSetOUT->eError =
+	    PVRSRVSyncPrimSetKM(psSyncHandleInt,
+				psSyncPrimSetIN->ui32Index, psSyncPrimSetIN->ui32Value);
+
+SyncPrimSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDump(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncPrimPDumpIN_UI8,
+			  IMG_UINT8 * psSyncPrimPDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *psSyncPrimPDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *psSyncPrimPDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVSyncPrimPDumpKM(psSyncHandleInt, psSyncPrimPDumpIN->ui32Offset);
+
+SyncPrimPDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDump NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpValue(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncPrimPDumpValueIN_UI8,
+			       IMG_UINT8 * psSyncPrimPDumpValueOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueOUT_UI8,
+								     0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpValueIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpValueOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpValue_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt,
+				       psSyncPrimPDumpValueIN->ui32Offset,
+				       psSyncPrimPDumpValueIN->ui32Value);
+
+SyncPrimPDumpValue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpValue NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpPolIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpPolOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpPolIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpPolOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpPol_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     psSyncPrimPDumpPolIN->ui32Offset,
+				     psSyncPrimPDumpPolIN->ui32Value,
+				     psSyncPrimPDumpPolIN->ui32Mask,
+				     psSyncPrimPDumpPolIN->eOperator,
+				     psSyncPrimPDumpPolIN->uiPDumpFlags);
+
+SyncPrimPDumpPol_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpPol NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpCBPIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpCBPIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     psSyncPrimPDumpCBPIN->ui32Offset,
+				     psSyncPrimPDumpCBPIN->uiWriteOffset,
+				     psSyncPrimPDumpCBPIN->uiPacketSize,
+				     psSyncPrimPDumpCBPIN->uiBufferSize);
+
+SyncPrimPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpCBP NULL
+#endif
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncAllocEvent(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncAllocEventIN_UI8,
+			   IMG_UINT8 * psSyncAllocEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *psSyncAllocEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *psSyncAllocEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventOUT_UI8, 0);
+
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncAllocEventIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncAllocEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncAllocEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncAllocEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncAllocEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncAllocEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncAllocEvent_exit;
+			}
+		}
+	}
+
+	if (psSyncAllocEventIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncAllocEventIN->puiClassName,
+		     psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncAllocEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncAllocEvent_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psSyncAllocEventOUT->eError =
+	    PVRSRVSyncAllocEventKM(psConnection, OSGetDevNode(psConnection),
+				   psSyncAllocEventIN->bServerSync,
+				   psSyncAllocEventIN->ui32FWAddr,
+				   psSyncAllocEventIN->ui32ClassNameSize, uiClassNameInt);
+
+SyncAllocEvent_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncAllocEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFreeEvent(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncFreeEventIN_UI8,
+			  IMG_UINT8 * psSyncFreeEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFREEEVENT *psSyncFreeEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *psSyncFreeEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventOUT_UI8, 0);
+
+	psSyncFreeEventOUT->eError =
+	    PVRSRVSyncFreeEventKM(psConnection, OSGetDevNode(psConnection),
+				  psSyncFreeEventIN->ui32FWAddr);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncCheckpointSignalledPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolIN_UI8,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psSyncCheckpointSignalledPDumpPolOUT->eError =
+	    PVRSRVSyncCheckpointSignalledPDumpPolKM(psSyncCheckpointSignalledPDumpPolIN->hFence);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncCheckpointSignalledPDumpPol NULL
+#endif
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+
+/*
+ * Register all SYNC functions with services
+ */
+PVRSRV_ERROR InitSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeAllocSyncPrimitiveBlock, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeFreeSyncPrimitiveBlock, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK),
+			      sizeof(PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET,
+			      PVRSRVBridgeSyncPrimSet, NULL, sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMSET),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMSET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP,
+			      PVRSRVBridgeSyncPrimPDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE,
+			      PVRSRVBridgeSyncPrimPDumpValue, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL,
+			      PVRSRVBridgeSyncPrimPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP,
+			      PVRSRVBridgeSyncPrimPDumpCBP, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT,
+			      PVRSRVBridgeSyncAllocEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCALLOCEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT,
+			      PVRSRVBridgeSyncFreeEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFREEEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFREEEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+			      PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL,
+			      PVRSRVBridgeSyncCheckpointSignalledPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all sync functions with services
+ */
+void DeinitSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+				PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/common_syncfallback_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/common_syncfallback_bridge.h
new file mode 100644
index 000000000000..87016c4706c7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/common_syncfallback_bridge.h
@@ -0,0 +1,357 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCFALLBACK_BRIDGE_H
+#define COMMON_SYNCFALLBACK_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_LAST			(PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15)
+
+/*******************************************
+            SyncFbTimelineCreatePVR
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR;
+
+/* Bridge out structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR;
+
+/*******************************************
+            SyncFbTimelineRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE;
+
+/* Bridge out structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE;
+
+/*******************************************
+            SyncFbFenceDup
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hInFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP;
+
+/* Bridge out structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP;
+
+/*******************************************
+            SyncFbFenceMerge
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hInFence1;
+	IMG_HANDLE hInFence2;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE;
+
+/* Bridge out structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE;
+
+/*******************************************
+            SyncFbFenceRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE;
+
+/* Bridge out structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE;
+
+/*******************************************
+            SyncFbFenceWait
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT_TAG
+{
+	IMG_HANDLE hFence;
+	IMG_UINT32 ui32Timeout;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT;
+
+/* Bridge out structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT;
+
+/*******************************************
+            SyncFbFenceDump
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP_TAG
+{
+	IMG_HANDLE hFence;
+	const IMG_CHAR *puiDescStr;
+	const IMG_CHAR *puiFileStr;
+	const IMG_CHAR *puiModuleStr;
+	IMG_UINT32 ui32DescStrLength;
+	IMG_UINT32 ui32FileStrLength;
+	IMG_UINT32 ui32Line;
+	IMG_UINT32 ui32ModuleStrLength;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP;
+
+/* Bridge out structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP;
+
+/*******************************************
+            SyncFbTimelineCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW;
+
+/* Bridge out structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW;
+
+/*******************************************
+            SyncFbFenceCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW;
+
+/* Bridge out structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW;
+
+/*******************************************
+            SyncFbTimelineAdvanceSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW;
+
+/* Bridge out structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW;
+
+/*******************************************
+            SyncFbFenceExportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroyInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/*******************************************
+            SyncFbFenceImportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hImport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroySecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/*******************************************
+            SyncFbFenceImportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Import;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE;
+
+#endif /* COMMON_SYNCFALLBACK_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/server_syncfallback_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/server_syncfallback_bridge.c
new file mode 100644
index 000000000000..68e650d89602
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/syncfallback_bridge/server_syncfallback_bridge.c
@@ -0,0 +1,1958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync_fallback_server.h"
+#include "pvrsrv_sync_server.h"
+
+#include "common_syncfallback_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleaseExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _SyncFbTimelineCreatePVRpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_TIMELINE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_TIMELINE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreatePVR(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVRIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVROUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVRIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVROUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVROUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize > SYNC_FB_TIMELINE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreatePVRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreatePVRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreatePVR_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreatePVRIN->puiTimelineName,
+		     psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreatePVR_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    SyncFbTimelineCreatePVR(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize,
+				    uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreatePVROUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreatePVRpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreatePVR_exit:
+
+	if (psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreatePVROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psSyncFbTimelineReleaseIN_UI8,
+				  IMG_UINT8 * psSyncFbTimelineReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbTimelineReleaseIN->hTimeline,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	if (unlikely((psSyncFbTimelineReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbTimelineReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceDuppsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDup(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncFbFenceDupIN_UI8,
+			   IMG_UINT8 * psSyncFbFenceDupOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *psSyncFbFenceDupIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *psSyncFbFenceDupOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupOUT_UI8, 0);
+
+	IMG_HANDLE hInFence = psSyncFbFenceDupIN->hInFence;
+	PVRSRV_FENCE_SERVER *psInFenceInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFenceInt,
+				       hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError = SyncFbFenceDup(psInFenceInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceDupOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceDuppsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceDup_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceDupOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceMergepsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceMerge(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncFbFenceMergeIN_UI8,
+			     IMG_UINT8 * psSyncFbFenceMergeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *psSyncFbFenceMergeIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *psSyncFbFenceMergeOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeOUT_UI8, 0);
+
+	IMG_HANDLE hInFence1 = psSyncFbFenceMergeIN->hInFence1;
+	PVRSRV_FENCE_SERVER *psInFence1Int = NULL;
+	IMG_HANDLE hInFence2 = psSyncFbFenceMergeIN->hInFence2;
+	PVRSRV_FENCE_SERVER *psInFence2Int = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceMergeIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceMergeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceMergeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceMerge_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt, (const void __user *)psSyncFbFenceMergeIN->puiFenceName,
+		     psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceMerge_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence1Int,
+				       hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence2Int,
+				       hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    SyncFbFenceMerge(psInFence1Int,
+			     psInFence2Int,
+			     psSyncFbFenceMergeIN->ui32FenceNameSize,
+			     uiFenceNameInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceMergeOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceMergepsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceMerge_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFence1Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psInFence2Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceMergeOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceMergeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncFbFenceReleaseIN_UI8,
+			       IMG_UINT8 * psSyncFbFenceReleaseOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *psSyncFbFenceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *psSyncFbFenceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceReleaseIN->hFence,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	if (unlikely((psSyncFbFenceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbFenceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceWaitIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *psSyncFbFenceWaitIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *psSyncFbFenceWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceWaitIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceWaitOUT->eError =
+	    SyncFbFenceWait(psFenceInt, psSyncFbFenceWaitIN->ui32Timeout);
+
+SyncFbFenceWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+static_assert((SYNC_FB_FILE_STRING_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_FILE_STRING_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_MODULE_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_MODULE_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_DESC_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_DESC_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDump(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceDumpIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *psSyncFbFenceDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *psSyncFbFenceDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceDumpIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	IMG_CHAR *uiFileStrInt = NULL;
+	IMG_CHAR *uiModuleStrInt = NULL;
+	IMG_CHAR *uiDescStrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32FileStrLength > (SYNC_FB_FILE_STRING_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely
+	    (psSyncFbFenceDumpIN->ui32ModuleStrLength > (SYNC_FB_MODULE_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32DescStrLength > (SYNC_FB_DESC_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceDump_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceDumpIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceDumpIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceDump_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceDumpIN->ui32FileStrLength != 0)
+	{
+		uiFileStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileStrInt, (const void __user *)psSyncFbFenceDumpIN->puiFileStr,
+		     psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileStrInt)[(psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength != 0)
+	{
+		uiModuleStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiModuleStrInt, (const void __user *)psSyncFbFenceDumpIN->puiModuleStr,
+		     psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiModuleStrInt)[(psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32DescStrLength != 0)
+	{
+		uiDescStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiDescStrInt, (const void __user *)psSyncFbFenceDumpIN->puiDescStr,
+		     psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiDescStrInt)[(psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDumpOUT->eError =
+	    SyncFbFenceDump(psFenceInt,
+			    psSyncFbFenceDumpIN->ui32Line,
+			    psSyncFbFenceDumpIN->ui32FileStrLength,
+			    uiFileStrInt,
+			    psSyncFbFenceDumpIN->ui32ModuleStrLength,
+			    uiModuleStrInt, psSyncFbFenceDumpIN->ui32DescStrLength, uiDescStrInt);
+
+SyncFbFenceDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceDumpOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbTimelineCreateSWpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWIN_UI8,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWOUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreateSWIN->puiTimelineName,
+		     psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    SyncFbTimelineCreateSW(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize,
+				   uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreateSWOUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreateSWpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreateSW_exit:
+
+	if (psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceCreateSWpsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psSyncFbFenceCreateSWIN_UI8,
+				IMG_UINT8 * psSyncFbFenceCreateSWOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWOUT_UI8,
+								      0);
+
+	IMG_HANDLE hTimeline = psSyncFbFenceCreateSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceCreateSWIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt,
+		     (const void __user *)psSyncFbFenceCreateSWIN->puiFenceName,
+		     psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    SyncFbFenceCreateSW(psConnection, OSGetDevNode(psConnection),
+				psTimelineInt,
+				psSyncFbFenceCreateSWIN->ui32FenceNameSize,
+				uiFenceNameInt,
+				&psOutFenceInt, &psSyncFbFenceCreateSWOUT->ui64SyncPtIdx);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceCreateSWOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceCreateSWpsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceCreateSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineAdvanceSW(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWOUT_UI8, 0);
+
+	IMG_HANDLE hTimeline = psSyncFbTimelineAdvanceSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbTimelineAdvanceSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineAdvanceSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    SyncFbTimelineAdvanceSW(psTimelineInt, &psSyncFbTimelineAdvanceSWOUT->ui64SyncPtIdx);
+
+SyncFbTimelineAdvanceSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceExportInsecurepsExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceExportDestroyInsecure((PVRSRV_FENCE_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportInsecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    SyncFbFenceExportInsecure(psFenceInt, &psExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase, &hExportInt,
+				      (void *)psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceExportInsecurepsExportIntRelease);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportInsecureOUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+									   &psSyncFbFenceExportInsecureOUT->
+									   hExport,
+									   (void *)psExportInt,
+									   PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+									   PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+									   (PFN_HANDLE_RELEASE) &
+									   ReleaseExport);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncFbFenceExportInsecureOUT->hExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE)
+							     psSyncFbFenceExportInsecureOUT->
+							     hExport,
+							     PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hExportInt,
+							PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psExportInt)
+		{
+			SyncFbFenceExportDestroyInsecure(psExportInt);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroyInsecure(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureIN_UI8,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psExportInt,
+				       (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->hExport,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_FALSE);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hExportInt,
+				     psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+		     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT((psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK) ||
+		   (psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportDestroyInsecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportDestroyInsecure_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportDestroyInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceImportInsecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hImport = psSyncFbFenceImportInsecureIN->hImport;
+	PVRSRV_FENCE_EXPORT *psImportInt = NULL;
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psImportInt,
+				       hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_TRUE);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    SyncFbFenceImportInsecure(psConnection, OSGetDevNode(psConnection),
+				      psImportInt, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportInsecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportInsecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psImportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceImportInsecure NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceExportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceExportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportSecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportSecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportSecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportSecureOUT->eError =
+	    SyncFbFenceExportSecure(psConnection, OSGetDevNode(psConnection),
+				    psFenceInt,
+				    &psSyncFbFenceExportSecureOUT->Export,
+				    &psExportInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportSecure_exit;
+	}
+
+SyncFbFenceExportSecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psExportInt)
+		{
+			SyncFbFenceExportDestroySecure(psExportInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroySecure(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureIN_UI8,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncFbFenceExportDestroySecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroySecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroySecureOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncFbFenceExportDestroySecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncFbFenceExportDestroySecure_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceImportSecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceImportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceImportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    SyncFbFenceImportSecure(psConnection, OSGetDevNode(psConnection),
+				    psSyncFbFenceImportSecureIN->Import, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportSecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportSecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportSecure_exit:
+
+	if (psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void);
+void DeinitSYNCFALLBACKBridge(void);
+
+/*
+ * Register all SYNCFALLBACK functions with services
+ */
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR,
+			      PVRSRVBridgeSyncFbTimelineCreatePVR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE,
+			      PVRSRVBridgeSyncFbTimelineRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK, PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP,
+			      PVRSRVBridgeSyncFbFenceDup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE,
+			      PVRSRVBridgeSyncFbFenceMerge, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE,
+			      PVRSRVBridgeSyncFbFenceRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT,
+			      PVRSRVBridgeSyncFbFenceWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP,
+			      PVRSRVBridgeSyncFbFenceDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW,
+			      PVRSRVBridgeSyncFbTimelineCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW,
+			      PVRSRVBridgeSyncFbFenceCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW,
+			      PVRSRVBridgeSyncFbTimelineAdvanceSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroyInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceImportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceExportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroySecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceImportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all syncfallback functions with services
+ */
+void DeinitSYNCFALLBACKBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_bridge.h
new file mode 100644
index 000000000000..544efd962834
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_bridge.h
@@ -0,0 +1,68 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNCTRACKING_BRIDGE_H
+#define CLIENT_SYNCTRACKING_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_synctracking_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName);
+
+#endif /* CLIENT_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_direct_bridge.c
new file mode 100644
index 000000000000..baeb89a6a0ae
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/client_synctracking_direct_bridge.c
@@ -0,0 +1,92 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for synctracking
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_synctracking_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "sync.h"
+#include "sync_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	pshRecordInt = (SYNC_RECORD_HANDLE) hhRecord;
+
+	eError = PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt;
+
+	pshServerSyncPrimBlockInt = (SYNC_PRIMITIVE_BLOCK *) hhServerSyncPrimBlock;
+
+	eError =
+	    PVRSRVSyncRecordAddKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  ui32ui32FwBlockAddr,
+				  ui32ui32SyncOffset,
+				  bbServerSync, ui32ClassNameSize, puiClassName);
+
+	*phhRecord = pshRecordInt;
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/common_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/common_synctracking_bridge.h
new file mode 100644
index 000000000000..9fd9fdb796a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/common_synctracking_bridge.h
@@ -0,0 +1,97 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCTRACKING_BRIDGE_H
+#define COMMON_SYNCTRACKING_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST			(PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1)
+
+/*******************************************
+            SyncRecordRemoveByHandle
+ *******************************************/
+
+/* Bridge in structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	IMG_HANDLE hhRecord;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE;
+
+/* Bridge out structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE;
+
+/*******************************************
+            SyncRecordAdd
+ *******************************************/
+
+/* Bridge in structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhServerSyncPrimBlock;
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32ui32FwBlockAddr;
+	IMG_UINT32 ui32ui32SyncOffset;
+	IMG_BOOL bbServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDADD;
+
+/* Bridge out structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhRecord;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDADD;
+
+#endif /* COMMON_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/server_synctracking_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/server_synctracking_bridge.c
new file mode 100644
index 000000000000..06757e5ef8d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/rogue/synctracking_bridge/server_synctracking_bridge.c
@@ -0,0 +1,329 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+
+#include "common_synctracking_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeSyncRecordRemoveByHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleIN_UI8,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordRemoveByHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncRecordRemoveByHandleIN->hhRecord,
+					      PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE);
+	if (unlikely((psSyncRecordRemoveByHandleOUT->eError != PVRSRV_OK) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncRecordRemoveByHandleOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordRemoveByHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordRemoveByHandle_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncRecordAddpshRecordIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVSyncRecordRemoveByHandleKM((SYNC_RECORD_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncRecordAdd(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncRecordAddIN_UI8,
+			  IMG_UINT8 * psSyncRecordAddOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDADD *psSyncRecordAddIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDADD *psSyncRecordAddOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddOUT_UI8, 0);
+
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	IMG_HANDLE hhServerSyncPrimBlock = psSyncRecordAddIN->hhServerSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncRecordAddIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncRecordAdd_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncRecordAdd_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncRecordAddIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncRecordAddIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncRecordAddOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncRecordAdd_exit;
+			}
+		}
+	}
+
+	if (psSyncRecordAddIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncRecordAddIN->puiClassName,
+		     psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncRecordAddOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncRecordAdd_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncRecordAddOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pshServerSyncPrimBlockInt,
+				       hhServerSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError =
+	    PVRSRVSyncRecordAddKM(psConnection, OSGetDevNode(psConnection),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  psSyncRecordAddIN->ui32ui32FwBlockAddr,
+				  psSyncRecordAddIN->ui32ui32SyncOffset,
+				  psSyncRecordAddIN->bbServerSync,
+				  psSyncRecordAddIN->ui32ClassNameSize, uiClassNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psSyncRecordAddOUT->hhRecord,
+							       (void *)pshRecordInt,
+							       PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+							       PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+							       (PFN_HANDLE_RELEASE) &
+							       _SyncRecordAddpshRecordIntRelease);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordAdd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pshServerSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hhServerSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psSyncRecordAddOUT->eError != PVRSRV_OK)
+	{
+		if (pshRecordInt)
+		{
+			PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncRecordAddOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+
+/*
+ * Register all SYNCTRACKING functions with services
+ */
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+			      PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE,
+			      PVRSRVBridgeSyncRecordRemoveByHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING, PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD,
+			      PVRSRVBridgeSyncRecordAdd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDADD),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDADD));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all synctracking functions with services
+ */
+void DeinitSYNCTRACKINGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_bridge.h
new file mode 100644
index 000000000000..1dec13fccce7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_CACHE_BRIDGE_H
+#define CLIENT_CACHE_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_cache_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp);
+
+#endif /* CLIENT_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_direct_bridge.c
new file mode 100644
index 000000000000..9691bae9352f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/client_cache_direct_bridge.c
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for cache
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_cache_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "cache_ops.h"
+
+#include "cache_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PMR **psPMRInt;
+
+	psPMRInt = (PMR **) phPMR;
+
+	eError =
+	    CacheOpQueue(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			 ui32NumCacheOps,
+			 psPMRInt, pui64Address, puiOffset, puiSize, piuCacheOp, ui32OpTimeline);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = CacheOpValExec(psPMRInt, ui64Address, uiOffset, uiSize, iuCacheOp);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    CacheOpLog(psPMRInt,
+		       ui64Address, uiOffset, uiSize, i64StartTime, i64EndTime, iuCacheOp);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/common_cache_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/common_cache_bridge.h
new file mode 100644
index 000000000000..cc848753f7fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/common_cache_bridge.h
@@ -0,0 +1,126 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CACHE_BRIDGE_H
+#define COMMON_CACHE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_CACHE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE			PVRSRV_BRIDGE_CACHE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPEXEC			PVRSRV_BRIDGE_CACHE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CACHE_CACHEOPLOG			PVRSRV_BRIDGE_CACHE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CACHE_CMD_LAST			(PVRSRV_BRIDGE_CACHE_CMD_FIRST+2)
+
+/*******************************************
+            CacheOpQueue
+ *******************************************/
+
+/* Bridge in structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPQUEUE_TAG
+{
+	PVRSRV_CACHE_OP *piuCacheOp;
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	IMG_UINT32 ui32NumCacheOps;
+	IMG_UINT32 ui32OpTimeline;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPQUEUE;
+
+/* Bridge out structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPQUEUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPQUEUE;
+
+/*******************************************
+            CacheOpExec
+ *******************************************/
+
+/* Bridge in structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPEXEC_TAG
+{
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPEXEC;
+
+/* Bridge out structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPEXEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPEXEC;
+
+/*******************************************
+            CacheOpLog
+ *******************************************/
+
+/* Bridge in structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPLOG_TAG
+{
+	IMG_INT64 i64EndTime;
+	IMG_INT64 i64StartTime;
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPLOG;
+
+/* Bridge out structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPLOG;
+
+#endif /* COMMON_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/server_cache_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/server_cache_bridge.c
new file mode 100644
index 000000000000..b2f5267cc597
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cache_bridge/server_cache_bridge.c
@@ -0,0 +1,452 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "cache_km.h"
+
+#include "common_cache_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(CACHE_BATCH_MAX <= IMG_UINT32_MAX,
+	      "CACHE_BATCH_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeCacheOpQueue(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psCacheOpQueueIN_UI8,
+			 IMG_UINT8 * psCacheOpQueueOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPQUEUE *psCacheOpQueueIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *psCacheOpQueueOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+	PVRSRV_CACHE_OP *iuCacheOpInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) + 0;
+
+	if (unlikely(psCacheOpQueueIN->ui32NumCacheOps > CACHE_BATCH_MAX))
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto CacheOpQueue_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto CacheOpQueue_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psCacheOpQueueIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psCacheOpQueueIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psCacheOpQueueOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *));
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psCacheOpQueueIN->phPMR,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psCacheOpQueueIN->pui64Address,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psCacheOpQueueIN->puiOffset,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psCacheOpQueueIN->puiSize,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		iuCacheOpInt =
+		    (PVRSRV_CACHE_OP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, iuCacheOpInt, (const void __user *)psCacheOpQueueIN->piuCacheOp,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+			/* Look up the address from the handle */
+			psCacheOpQueueOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psCacheOpQueueOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpQueueOUT->eError =
+	    CacheOpQueue(psConnection, OSGetDevNode(psConnection),
+			 psCacheOpQueueIN->ui32NumCacheOps,
+			 psPMRInt,
+			 ui64AddressInt,
+			 uiOffsetInt, uiSizeInt, iuCacheOpInt, psCacheOpQueueIN->ui32OpTimeline);
+
+CacheOpQueue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psCacheOpQueueOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpExec(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psCacheOpExecIN_UI8,
+			IMG_UINT8 * psCacheOpExecOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPEXEC *psCacheOpExecIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPEXEC *psCacheOpExecOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpExecIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpExecOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpExecOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpExec_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpExecOUT->eError =
+	    CacheOpValExec(psPMRInt,
+			   psCacheOpExecIN->ui64Address,
+			   psCacheOpExecIN->uiOffset,
+			   psCacheOpExecIN->uiSize, psCacheOpExecIN->iuCacheOp);
+
+CacheOpExec_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpLog(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psCacheOpLogIN_UI8,
+		       IMG_UINT8 * psCacheOpLogOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPLOG *psCacheOpLogIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPLOG *psCacheOpLogOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpLogIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpLogOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpLogOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpLog_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpLogOUT->eError =
+	    CacheOpLog(psPMRInt,
+		       psCacheOpLogIN->ui64Address,
+		       psCacheOpLogIN->uiOffset,
+		       psCacheOpLogIN->uiSize,
+		       psCacheOpLogIN->i64StartTime,
+		       psCacheOpLogIN->i64EndTime, psCacheOpLogIN->iuCacheOp);
+
+CacheOpLog_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+
+/*
+ * Register all CACHE functions with services
+ */
+PVRSRV_ERROR InitCACHEBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE,
+			      PVRSRVBridgeCacheOpQueue, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPQUEUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPQUEUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC,
+			      PVRSRVBridgeCacheOpExec, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPEXEC),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPEXEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG,
+			      PVRSRVBridgeCacheOpLog, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPLOG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cache functions with services
+ */
+void DeinitCACHEBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/common_cmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/common_cmm_bridge.h
new file mode 100644
index 000000000000..da48de3598be
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/common_cmm_bridge.h
@@ -0,0 +1,114 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CMM_BRIDGE_H
+#define COMMON_CMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_CMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CMM_CMD_LAST			(PVRSRV_BRIDGE_CMM_CMD_FIRST+2)
+
+/*******************************************
+            DevmemIntExportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX;
+
+/* Bridge out structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX;
+
+/*******************************************
+            DevmemIntUnexportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX;
+
+/* Bridge out structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX;
+
+/*******************************************
+            DevmemIntAcquireRemoteCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX;
+
+/* Bridge out structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX;
+
+#endif /* COMMON_CMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/server_cmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/server_cmm_bridge.c
new file mode 100644
index 000000000000..cc658c793d3b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/cmm_bridge/server_cmm_bridge.c
@@ -0,0 +1,414 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_server.h"
+
+#include "common_cmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DevmemIntExportCtxpsContextExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnexportCtx((DEVMEMINT_CTX_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntExportCtx(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntExportCtxIN_UI8,
+			       IMG_UINT8 * psDevmemIntExportCtxOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxOUT_UI8,
+								     0);
+
+	IMG_HANDLE hContext = psDevmemIntExportCtxIN->hContext;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntExportCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX_EXPORT *psContextExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError =
+	    DevmemIntExportCtx(psContextInt, psPMRInt, &psContextExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntExportCtxOUT->
+								    hContextExport,
+								    (void *)psContextExportInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+								    PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntExportCtxpsContextExportIntRelease);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntExportCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntExportCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psContextExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DevmemIntUnexportCtx(psContextExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnexportCtx(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntUnexportCtxIN_UI8,
+				 IMG_UINT8 * psDevmemIntUnexportCtxOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntUnexportCtxIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnexportCtxOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnexportCtxOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnexportCtxIN->hContextExport,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT);
+	if (unlikely((psDevmemIntUnexportCtxOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnexportCtxOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnexportCtx_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnexportCtx_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntAcquireRemoteCtxpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntAcquireRemoteCtx(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxIN_UI8,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevmemIntAcquireRemoteCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntAcquireRemoteCtxOUT->hContext = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    DevmemIntAcquireRemoteCtx(psPMRInt, &psContextInt, &hPrivDataInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntAcquireRemoteCtxOUT->hContext,
+				      (void *)psContextInt, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntAcquireRemoteCtxpsContextIntRelease);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psDevmemIntAcquireRemoteCtxOUT->hPrivData,
+					 (void *)hPrivDataInt, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psDevmemIntAcquireRemoteCtxOUT->hContext);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntAcquireRemoteCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntAcquireRemoteCtxOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psDevmemIntAcquireRemoteCtxOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DevmemIntCtxDestroy(psContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_CMM_BRIDGE */
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+
+/*
+ * Register all CMM functions with services
+ */
+PVRSRV_ERROR InitCMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX,
+			      PVRSRVBridgeDevmemIntExportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX,
+			      PVRSRVBridgeDevmemIntUnexportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX,
+			      PVRSRVBridgeDevmemIntAcquireRemoteCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cmm functions with services
+ */
+void DeinitCMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX);
+
+}
+#else /* EXCLUDE_CMM_BRIDGE */
+/* This bridge is conditional on EXCLUDE_CMM_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitCMMBridge() \
+	PVRSRV_OK
+
+#define DeinitCMMBridge()
+
+#endif /* EXCLUDE_CMM_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/common_dc_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/common_dc_bridge.h
new file mode 100644
index 000000000000..d811fbd4cef1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/common_dc_bridge.h
@@ -0,0 +1,548 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DC_BRIDGE_H
+#define COMMON_DC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_surface.h"
+#include "dc_external.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE			PVRSRV_BRIDGE_DC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DC_DCDEVICERELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DC_DCGETINFO			PVRSRV_BRIDGE_DC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT			PVRSRV_BRIDGE_DC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_DC_DCPANELQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_DC_DCFORMATQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_DC_DCDIMQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_DC_DCSETBLANK			PVRSRV_BRIDGE_DC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING			PVRSRV_BRIDGE_DC_CMD_FIRST+10
+#define PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY			PVRSRV_BRIDGE_DC_CMD_FIRST+11
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+12
+#define PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+13
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE			PVRSRV_BRIDGE_DC_CMD_FIRST+14
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK			PVRSRV_BRIDGE_DC_CMD_FIRST+15
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY			PVRSRV_BRIDGE_DC_CMD_FIRST+16
+#define PVRSRV_BRIDGE_DC_DCBUFFERALLOC			PVRSRV_BRIDGE_DC_CMD_FIRST+17
+#define PVRSRV_BRIDGE_DC_DCBUFFERIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+18
+#define PVRSRV_BRIDGE_DC_DCBUFFERFREE			PVRSRV_BRIDGE_DC_CMD_FIRST+19
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT			PVRSRV_BRIDGE_DC_CMD_FIRST+20
+#define PVRSRV_BRIDGE_DC_DCBUFFERPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+21
+#define PVRSRV_BRIDGE_DC_DCBUFFERUNPIN			PVRSRV_BRIDGE_DC_CMD_FIRST+22
+#define PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE			PVRSRV_BRIDGE_DC_CMD_FIRST+23
+#define PVRSRV_BRIDGE_DC_DCBUFFERRELEASE			PVRSRV_BRIDGE_DC_CMD_FIRST+24
+#define PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2			PVRSRV_BRIDGE_DC_CMD_FIRST+25
+#define PVRSRV_BRIDGE_DC_CMD_LAST			(PVRSRV_BRIDGE_DC_CMD_FIRST+25)
+
+/*******************************************
+            DCDevicesQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT;
+
+/* Bridge out structure for DCDevicesQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT;
+
+/*******************************************
+            DCDevicesEnumerate
+ *******************************************/
+
+/* Bridge in structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	IMG_UINT32 ui32DeviceArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE;
+
+/* Bridge out structure for DCDevicesEnumerate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE_TAG
+{
+	IMG_UINT32 *pui32DeviceIndex;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceCount;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE;
+
+/*******************************************
+            DCDeviceAcquire
+ *******************************************/
+
+/* Bridge in structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE_TAG
+{
+	IMG_UINT32 ui32DeviceIndex;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE;
+
+/* Bridge out structure for DCDeviceAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE;
+
+/*******************************************
+            DCDeviceRelease
+ *******************************************/
+
+/* Bridge in structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCDEVICERELEASE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDEVICERELEASE;
+
+/* Bridge out structure for DCDeviceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCDEVICERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDEVICERELEASE;
+
+/*******************************************
+            DCGetInfo
+ *******************************************/
+
+/* Bridge in structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_IN_DCGETINFO_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCGETINFO;
+
+/* Bridge out structure for DCGetInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DCGETINFO_TAG
+{
+	DC_DISPLAY_INFO sDisplayInfo;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCGETINFO;
+
+/*******************************************
+            DCPanelQueryCount
+ *******************************************/
+
+/* Bridge in structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT;
+
+/* Bridge out structure for DCPanelQueryCount */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT;
+
+/*******************************************
+            DCPanelQuery
+ *******************************************/
+
+/* Bridge in structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCPANELQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	IMG_UINT32 ui32PanelsArraySize;
+} __packed PVRSRV_BRIDGE_IN_DCPANELQUERY;
+
+/* Bridge out structure for DCPanelQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCPANELQUERY_TAG
+{
+	PVRSRV_PANEL_INFO *psPanelInfo;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumPanels;
+} __packed PVRSRV_BRIDGE_OUT_DCPANELQUERY;
+
+/*******************************************
+            DCFormatQuery
+ *******************************************/
+
+/* Bridge in structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCFORMATQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_FORMAT *psFormat;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumFormats;
+} __packed PVRSRV_BRIDGE_IN_DCFORMATQUERY;
+
+/* Bridge out structure for DCFormatQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCFORMATQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCFORMATQUERY;
+
+/*******************************************
+            DCDimQuery
+ *******************************************/
+
+/* Bridge in structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCDIMQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+	PVRSRV_SURFACE_DIMS *psDim;
+	IMG_UINT32 *pui32Supported;
+	IMG_UINT32 ui32NumDims;
+} __packed PVRSRV_BRIDGE_IN_DCDIMQUERY;
+
+/* Bridge out structure for DCDimQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCDIMQUERY_TAG
+{
+	IMG_UINT32 *pui32Supported;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDIMQUERY;
+
+/*******************************************
+            DCSetBlank
+ *******************************************/
+
+/* Bridge in structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_IN_DCSETBLANK_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETBLANK;
+
+/* Bridge out structure for DCSetBlank */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETBLANK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETBLANK;
+
+/*******************************************
+            DCSetVSyncReporting
+ *******************************************/
+
+/* Bridge in structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING_TAG
+{
+	IMG_HANDLE hDevice;
+	IMG_BOOL bEnabled;
+} __packed PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING;
+
+/* Bridge out structure for DCSetVSyncReporting */
+typedef struct PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING;
+
+/*******************************************
+            DCLastVSyncQuery
+ *******************************************/
+
+/* Bridge in structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY;
+
+/* Bridge out structure for DCLastVSyncQuery */
+typedef struct PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY_TAG
+{
+	IMG_INT64 i64Timestamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY;
+
+/*******************************************
+            DCSystemBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE;
+
+/* Bridge out structure for DCSystemBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE;
+
+/*******************************************
+            DCSystemBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE;
+
+/* Bridge out structure for DCSystemBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextCreate
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDevice;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE;
+
+/* Bridge out structure for DCDisplayContextCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE;
+
+/*******************************************
+            DCDisplayContextConfigureCheck
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/* Bridge out structure for DCDisplayContextConfigureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK;
+
+/*******************************************
+            DCDisplayContextDestroy
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	IMG_HANDLE hDisplayContext;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY;
+
+/* Bridge out structure for DCDisplayContextDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY;
+
+/*******************************************
+            DCBufferAlloc
+ *******************************************/
+
+/* Bridge in structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	DC_BUFFER_CREATE_INFO sSurfInfo;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERALLOC;
+
+/* Bridge out structure for DCBufferAlloc */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERALLOC_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Stride;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERALLOC;
+
+/*******************************************
+            DCBufferImport
+ *******************************************/
+
+/* Bridge in structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	IMG_HANDLE *phImport;
+	DC_BUFFER_IMPORT_INFO sSurfAttrib;
+	IMG_UINT32 ui32NumPlanes;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERIMPORT;
+
+/* Bridge out structure for DCBufferImport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT;
+
+/*******************************************
+            DCBufferFree
+ *******************************************/
+
+/* Bridge in structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERFREE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERFREE;
+
+/* Bridge out structure for DCBufferFree */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERFREE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERFREE;
+
+/*******************************************
+            DCBufferUnimport
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT;
+
+/* Bridge out structure for DCBufferUnimport */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT;
+
+/*******************************************
+            DCBufferPin
+ *******************************************/
+
+/* Bridge in structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERPIN;
+
+/* Bridge out structure for DCBufferPin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERPIN;
+
+/*******************************************
+            DCBufferUnpin
+ *******************************************/
+
+/* Bridge in structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERUNPIN_TAG
+{
+	IMG_HANDLE hPinHandle;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERUNPIN;
+
+/* Bridge out structure for DCBufferUnpin */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN;
+
+/*******************************************
+            DCBufferAcquire
+ *******************************************/
+
+/* Bridge in structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE;
+
+/* Bridge out structure for DCBufferAcquire */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE;
+
+/*******************************************
+            DCBufferRelease
+ *******************************************/
+
+/* Bridge in structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_IN_DCBUFFERRELEASE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_DCBUFFERRELEASE;
+
+/* Bridge out structure for DCBufferRelease */
+typedef struct PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE;
+
+/*******************************************
+            DCDisplayContextConfigure2
+ *******************************************/
+
+/* Bridge in structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	IMG_HANDLE hDisplayContext;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfo;
+	IMG_HANDLE *phBuffers;
+	PVRSRV_FENCE hAcquireFence;
+	PVRSRV_TIMELINE hReleaseFenceTimeline;
+	IMG_UINT32 ui32DisplayPeriod;
+	IMG_UINT32 ui32MaxDepth;
+	IMG_UINT32 ui32PipeCount;
+} __packed PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2;
+
+/* Bridge out structure for DCDisplayContextConfigure2 */
+typedef struct PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hReleaseFence;
+} __packed PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2;
+
+#endif /* COMMON_DC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/server_dc_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/server_dc_bridge.c
new file mode 100644
index 000000000000..3b5b9150c2ec
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dc_bridge/server_dc_bridge.c
@@ -0,0 +1,2554 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dc
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dc
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dc_server.h"
+
+#include "common_dc_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeDCDevicesQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCDevicesQueryCountIN_UI8,
+				IMG_UINT8 * psDCDevicesQueryCountOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *psDCDevicesQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESQUERYCOUNT *) IMG_OFFSET_ADDR(psDCDevicesQueryCountOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDCDevicesQueryCountIN);
+
+	psDCDevicesQueryCountOUT->eError =
+	    DCDevicesQueryCount(&psDCDevicesQueryCountOUT->ui32DeviceCount);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DEVICE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_DEVICE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDevicesEnumerate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDCDevicesEnumerateIN_UI8,
+			       IMG_UINT8 * psDCDevicesEnumerateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *psDCDevicesEnumerateIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *psDCDevicesEnumerateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICESENUMERATE *) IMG_OFFSET_ADDR(psDCDevicesEnumerateOUT_UI8,
+								     0);
+
+	IMG_UINT32 *pui32DeviceIndexInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32)) + 0;
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize > DC_MAX_DEVICE_COUNT)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	psDCDevicesEnumerateOUT->pui32DeviceIndex = psDCDevicesEnumerateIN->pui32DeviceIndex;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDevicesEnumerate_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDevicesEnumerateIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDevicesEnumerateIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDevicesEnumerate_exit;
+			}
+		}
+	}
+
+	if (psDCDevicesEnumerateIN->ui32DeviceArraySize != 0)
+	{
+		pui32DeviceIndexInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDevicesEnumerateIN->ui32DeviceArraySize * sizeof(IMG_UINT32);
+	}
+
+	psDCDevicesEnumerateOUT->eError =
+	    DCDevicesEnumerate(psConnection, OSGetDevNode(psConnection),
+			       psDCDevicesEnumerateIN->ui32DeviceArraySize,
+			       &psDCDevicesEnumerateOUT->ui32DeviceCount, pui32DeviceIndexInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDevicesEnumerateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDevicesEnumerate_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32DeviceIndexInt) &&
+	    ((psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDevicesEnumerateOUT->pui32DeviceIndex,
+		      pui32DeviceIndexInt,
+		      (psDCDevicesEnumerateOUT->ui32DeviceCount * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psDCDevicesEnumerateOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDevicesEnumerate_exit;
+		}
+	}
+
+DCDevicesEnumerate_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDevicesEnumerateOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDeviceAcquirepsDeviceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDeviceRelease((DC_DEVICE *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceAcquireIN_UI8,
+			    IMG_UINT8 * psDCDeviceAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *psDCDeviceAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *psDCDeviceAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICEACQUIRE *) IMG_OFFSET_ADDR(psDCDeviceAcquireOUT_UI8, 0);
+
+	DC_DEVICE *psDeviceInt = NULL;
+
+	psDCDeviceAcquireOUT->eError =
+	    DCDeviceAcquire(psConnection, OSGetDevNode(psConnection),
+			    psDCDeviceAcquireIN->ui32DeviceIndex, &psDeviceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDCDeviceAcquireOUT->hDevice,
+								 (void *)psDeviceInt,
+								 PVRSRV_HANDLE_TYPE_DC_DEVICE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _DCDeviceAcquirepsDeviceIntRelease);
+	if (unlikely(psDCDeviceAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceAcquire_exit:
+
+	if (psDCDeviceAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psDeviceInt)
+		{
+			DCDeviceRelease(psDeviceInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDeviceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCDeviceReleaseIN_UI8,
+			    IMG_UINT8 * psDCDeviceReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDEVICERELEASE *psDCDeviceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *psDCDeviceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDEVICERELEASE *) IMG_OFFSET_ADDR(psDCDeviceReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDeviceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDeviceReleaseIN->hDevice,
+					      PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	if (unlikely((psDCDeviceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCDeviceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCDeviceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDeviceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDeviceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCGetInfo(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psDCGetInfoIN_UI8,
+		      IMG_UINT8 * psDCGetInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCGETINFO *psDCGetInfoIN =
+	    (PVRSRV_BRIDGE_IN_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCGETINFO *psDCGetInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DCGETINFO *) IMG_OFFSET_ADDR(psDCGetInfoOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCGetInfoIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCGetInfoOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCGetInfoOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCGetInfo_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCGetInfoOUT->eError = DCGetInfo(psDeviceInt, &psDCGetInfoOUT->sDisplayInfo);
+
+DCGetInfo_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCPanelQueryCount(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDCPanelQueryCountIN_UI8,
+			      IMG_UINT8 * psDCPanelQueryCountOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *psDCPanelQueryCountIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *psDCPanelQueryCountOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERYCOUNT *) IMG_OFFSET_ADDR(psDCPanelQueryCountOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryCountIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryCountOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryCountOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQueryCount_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryCountOUT->eError =
+	    DCPanelQueryCount(psDeviceInt, &psDCPanelQueryCountOUT->ui32NumPanels);
+
+DCPanelQueryCount_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(DC_MAX_PANEL_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PANEL_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCPanelQuery(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCPanelQueryIN_UI8,
+			 IMG_UINT8 * psDCPanelQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCPANELQUERY *psDCPanelQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCPANELQUERY *psDCPanelQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCPANELQUERY *) IMG_OFFSET_ADDR(psDCPanelQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCPanelQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_PANEL_INFO *psPanelInfoInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO)) + 0;
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize > DC_MAX_PANEL_COUNT)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCPanelQuery_exit;
+	}
+
+	psDCPanelQueryOUT->psPanelInfo = psDCPanelQueryIN->psPanelInfo;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCPanelQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCPanelQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCPanelQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCPanelQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCPanelQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCPanelQuery_exit;
+			}
+		}
+	}
+
+	if (psDCPanelQueryIN->ui32PanelsArraySize != 0)
+	{
+		psPanelInfoInt =
+		    (PVRSRV_PANEL_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCPanelQueryIN->ui32PanelsArraySize * sizeof(PVRSRV_PANEL_INFO);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCPanelQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCPanelQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCPanelQueryOUT->eError =
+	    DCPanelQuery(psDeviceInt,
+			 psDCPanelQueryIN->ui32PanelsArraySize,
+			 &psDCPanelQueryOUT->ui32NumPanels, psPanelInfoInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCPanelQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCPanelQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psPanelInfoInt) &&
+	    ((psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCPanelQueryOUT->psPanelInfo, psPanelInfoInt,
+		      (psDCPanelQueryOUT->ui32NumPanels * sizeof(PVRSRV_PANEL_INFO))) != PVRSRV_OK))
+		{
+			psDCPanelQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCPanelQuery_exit;
+		}
+	}
+
+DCPanelQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCPanelQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_FORMATS <= IMG_UINT32_MAX,
+	      "DC_MAX_FORMATS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCFormatQuery(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCFormatQueryIN_UI8,
+			  IMG_UINT8 * psDCFormatQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCFORMATQUERY *psDCFormatQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCFORMATQUERY *psDCFormatQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCFORMATQUERY *) IMG_OFFSET_ADDR(psDCFormatQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCFormatQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_FORMAT *psFormatInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) +
+	    ((IMG_UINT64) psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCFormatQueryIN->ui32NumFormats > DC_MAX_FORMATS))
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCFormatQuery_exit;
+	}
+
+	psDCFormatQueryOUT->pui32Supported = psDCFormatQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCFormatQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCFormatQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCFormatQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCFormatQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCFormatQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCFormatQuery_exit;
+			}
+		}
+	}
+
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		psFormatInt =
+		    (PVRSRV_SURFACE_FORMAT *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT);
+	}
+
+	/* Copy the data over */
+	if (psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psFormatInt, (const void __user *)psDCFormatQueryIN->psFormat,
+		     psDCFormatQueryIN->ui32NumFormats * sizeof(PVRSRV_SURFACE_FORMAT)) !=
+		    PVRSRV_OK)
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+	if (psDCFormatQueryIN->ui32NumFormats != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCFormatQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCFormatQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCFormatQueryOUT->eError =
+	    DCFormatQuery(psDeviceInt,
+			  psDCFormatQueryIN->ui32NumFormats, psFormatInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCFormatQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCFormatQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCFormatQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCFormatQueryIN->ui32NumFormats * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCFormatQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCFormatQuery_exit;
+		}
+	}
+
+DCFormatQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCFormatQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DC_MAX_DIMENSIONS <= IMG_UINT32_MAX,
+	      "DC_MAX_DIMENSIONS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDimQuery(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCDimQueryIN_UI8,
+		       IMG_UINT8 * psDCDimQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDIMQUERY *psDCDimQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDIMQUERY *psDCDimQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDIMQUERY *) IMG_OFFSET_ADDR(psDCDimQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDimQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	PVRSRV_SURFACE_DIMS *psDimInt = NULL;
+	IMG_UINT32 *pui32SupportedInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) +
+	    ((IMG_UINT64) psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psDCDimQueryIN->ui32NumDims > DC_MAX_DIMENSIONS))
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDimQuery_exit;
+	}
+
+	psDCDimQueryOUT->pui32Supported = psDCDimQueryIN->pui32Supported;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDimQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDimQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDimQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCDimQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDimQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDimQuery_exit;
+			}
+		}
+	}
+
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		psDimInt =
+		    (PVRSRV_SURFACE_DIMS *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS);
+	}
+
+	/* Copy the data over */
+	if (psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psDimInt, (const void __user *)psDCDimQueryIN->psDim,
+		     psDCDimQueryIN->ui32NumDims * sizeof(PVRSRV_SURFACE_DIMS)) != PVRSRV_OK)
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+	if (psDCDimQueryIN->ui32NumDims != 0)
+	{
+		pui32SupportedInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDimQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDimQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDimQueryOUT->eError =
+	    DCDimQuery(psDeviceInt, psDCDimQueryIN->ui32NumDims, psDimInt, pui32SupportedInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDimQueryOUT->eError != PVRSRV_OK))
+	{
+		goto DCDimQuery_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32SupportedInt) && ((psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDCDimQueryOUT->pui32Supported, pui32SupportedInt,
+		      (psDCDimQueryIN->ui32NumDims * sizeof(IMG_UINT32))) != PVRSRV_OK))
+		{
+			psDCDimQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDimQuery_exit;
+		}
+	}
+
+DCDimQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDimQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetBlank(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDCSetBlankIN_UI8,
+		       IMG_UINT8 * psDCSetBlankOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETBLANK *psDCSetBlankIN =
+	    (PVRSRV_BRIDGE_IN_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSETBLANK *psDCSetBlankOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETBLANK *) IMG_OFFSET_ADDR(psDCSetBlankOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSetBlankIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetBlankOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetBlankOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetBlank_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetBlankOUT->eError = DCSetBlank(psDeviceInt, psDCSetBlankIN->bEnabled);
+
+DCSetBlank_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSetVSyncReporting(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDCSetVSyncReportingIN_UI8,
+				IMG_UINT8 * psDCSetVSyncReportingOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *psDCSetVSyncReportingIN =
+	    (PVRSRV_BRIDGE_IN_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *psDCSetVSyncReportingOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSETVSYNCREPORTING *) IMG_OFFSET_ADDR(psDCSetVSyncReportingOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevice = psDCSetVSyncReportingIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSetVSyncReportingOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSetVSyncReportingOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSetVSyncReporting_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSetVSyncReportingOUT->eError =
+	    DCSetVSyncReporting(psDeviceInt, psDCSetVSyncReportingIN->bEnabled);
+
+DCSetVSyncReporting_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCLastVSyncQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCLastVSyncQueryIN_UI8,
+			     IMG_UINT8 * psDCLastVSyncQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *psDCLastVSyncQueryIN =
+	    (PVRSRV_BRIDGE_IN_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *psDCLastVSyncQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_DCLASTVSYNCQUERY *) IMG_OFFSET_ADDR(psDCLastVSyncQueryOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCLastVSyncQueryIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCLastVSyncQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCLastVSyncQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCLastVSyncQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCLastVSyncQueryOUT->eError =
+	    DCLastVSyncQuery(psDeviceInt, &psDCLastVSyncQueryOUT->i64Timestamp);
+
+DCLastVSyncQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCSystemBufferAcquirepsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCSystemBufferRelease((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferAcquireIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferAcquireOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *psDCSystemBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERACQUIRE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCSystemBufferAcquireIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCSystemBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError =
+	    DCSystemBufferAcquire(psDeviceInt,
+				  &psDCSystemBufferAcquireOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferAcquireOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDCSystemBufferAcquireOUT->
+								       hBuffer, (void *)psBufferInt,
+								       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DCSystemBufferAcquirepsBufferIntRelease);
+	if (unlikely(psDCSystemBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCSystemBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCSystemBufferRelease(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCSystemBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDCSystemBufferReleaseIN_UI8,
+				  IMG_UINT8 * psDCSystemBufferReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *psDCSystemBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCSYSTEMBUFFERRELEASE *)
+	    IMG_OFFSET_ADDR(psDCSystemBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCSystemBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCSystemBufferReleaseIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCSystemBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCSystemBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCSystemBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCSystemBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCSystemBufferRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCDisplayContextCreatepsDisplayContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCDisplayContextDestroy((DC_DISPLAY_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextCreate(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDCDisplayContextCreateIN_UI8,
+				   IMG_UINT8 * psDCDisplayContextCreateOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *psDCDisplayContextCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCREATE *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextCreateOUT_UI8, 0);
+
+	IMG_HANDLE hDevice = psDCDisplayContextCreateIN->hDevice;
+	DC_DEVICE *psDeviceInt = NULL;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDeviceInt,
+				       hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE, IMG_TRUE);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError =
+	    DCDisplayContextCreate(psDeviceInt, &psDisplayContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDCDisplayContextCreateOUT->
+									hDisplayContext,
+									(void *)psDisplayContextInt,
+									PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DCDisplayContextCreatepsDisplayContextIntRelease);
+	if (unlikely(psDCDisplayContextCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDeviceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevice, PVRSRV_HANDLE_TYPE_DC_DEVICE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCDisplayContextCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDisplayContextInt)
+		{
+			DCDisplayContextDestroy(psDisplayContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigureCheck(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckIN_UI8,
+					   IMG_UINT8 * psDCDisplayContextConfigureCheckOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *psDCDisplayContextConfigureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURECHECK *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigureCheckOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigureCheckIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) +
+	    0;
+
+	if (unlikely(psDCDisplayContextConfigureCheckIN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigureCheckOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigureCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigureCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigureCheckOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) >
+	    0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->psSurfInfo,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigureCheckIN->ui32PipeCount *
+			       sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigureCheckIN->phBuffers,
+		     psDCDisplayContextConfigureCheckIN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigureCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigureCheck_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigureCheck_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigureCheckOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigureCheckOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigureCheck_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigureCheckOUT->eError =
+	    DCDisplayContextConfigureCheck(psDisplayContextInt,
+					   psDCDisplayContextConfigureCheckIN->ui32PipeCount,
+					   psSurfInfoInt, psBuffersInt);
+
+DCDisplayContextConfigureCheck_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigureCheckIN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigureCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDCDisplayContextDestroyIN_UI8,
+				    IMG_UINT8 * psDCDisplayContextDestroyOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *psDCDisplayContextDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTDESTROY *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCDisplayContextDestroyIN->
+					      hDisplayContext,
+					      PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	if (unlikely
+	    ((psDCDisplayContextDestroyOUT->eError != PVRSRV_OK)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDCDisplayContextDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDCDisplayContextDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCDisplayContextDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAllocpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAlloc(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferAllocIN_UI8,
+			  IMG_UINT8 * psDCBufferAllocOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERALLOC *psDCBufferAllocIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *psDCBufferAllocOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERALLOC *) IMG_OFFSET_ADDR(psDCBufferAllocOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferAllocIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAllocOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError =
+	    DCBufferAlloc(psDisplayContextInt,
+			  &psDCBufferAllocIN->sSurfInfo,
+			  &psDCBufferAllocOUT->ui32Stride, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferAllocOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psDCBufferAllocOUT->hBuffer,
+							       (void *)psBufferInt,
+							       PVRSRV_HANDLE_TYPE_DC_BUFFER,
+							       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							       (PFN_HANDLE_RELEASE) &
+							       _DCBufferAllocpsBufferIntRelease);
+	if (unlikely(psDCBufferAllocOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAlloc_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferAlloc_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAllocOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferImportpsBufferIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferFree((DC_BUFFER *) pvData);
+	return eError;
+}
+
+static_assert(DC_MAX_PLANES <= IMG_UINT32_MAX,
+	      "DC_MAX_PLANES must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCBufferImport(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psDCBufferImportIN_UI8,
+			   IMG_UINT8 * psDCBufferImportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *psDCBufferImportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *psDCBufferImportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERIMPORT *) IMG_OFFSET_ADDR(psDCBufferImportOUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCBufferImportIN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PMR **psImportInt = NULL;
+	IMG_HANDLE *hImportInt2 = NULL;
+	DC_BUFFER *psBufferInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCBufferImportIN->ui32NumPlanes > DC_MAX_PLANES))
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCBufferImport_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCBufferImportOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCBufferImport_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCBufferImportIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDCBufferImportIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCBufferImportOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+
+	if (psDCBufferImportIN->ui32NumPlanes != 0)
+	{
+		psImportInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psImportInt, 0, psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *));
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(PMR *);
+		hImportInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hImportInt2, (const void __user *)psDCBufferImportIN->phImport,
+		     psDCBufferImportIN->ui32NumPlanes * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDCBufferImportOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCBufferImport_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferImportOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+			/* Look up the address from the handle */
+			psDCBufferImportOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psImportInt[i],
+						       hImportInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCBufferImport_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError =
+	    DCBufferImport(psDisplayContextInt,
+			   psDCBufferImportIN->ui32NumPlanes,
+			   psImportInt, &psDCBufferImportIN->sSurfAttrib, &psBufferInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferImport_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferImportOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								&psDCBufferImportOUT->hBuffer,
+								(void *)psBufferInt,
+								PVRSRV_HANDLE_TYPE_DC_BUFFER,
+								PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								(PFN_HANDLE_RELEASE) &
+								_DCBufferImportpsBufferIntRelease);
+	if (unlikely(psDCBufferImportOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferImport_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferImport_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hImportInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCBufferImportIN->ui32NumPlanes; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psImportInt && psImportInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hImportInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferImportOUT->eError != PVRSRV_OK)
+	{
+		if (psBufferInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferFree(psBufferInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCBufferImportOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferFree(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDCBufferFreeIN_UI8,
+			 IMG_UINT8 * psDCBufferFreeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERFREE *psDCBufferFreeIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERFREE *psDCBufferFreeOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERFREE *) IMG_OFFSET_ADDR(psDCBufferFreeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferFreeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferFreeIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferFreeOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferFreeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferFreeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferFree_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferFree_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnimport(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDCBufferUnimportIN_UI8,
+			     IMG_UINT8 * psDCBufferUnimportOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *psDCBufferUnimportIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *psDCBufferUnimportOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNIMPORT *) IMG_OFFSET_ADDR(psDCBufferUnimportOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnimportOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnimportIN->hBuffer,
+					      PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	if (unlikely((psDCBufferUnimportOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnimportOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnimportOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnimport_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnimport_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferPinhPinHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferUnpin((DC_PIN_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferPin(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDCBufferPinIN_UI8,
+			IMG_UINT8 * psDCBufferPinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERPIN *psDCBufferPinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERPIN *psDCBufferPinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERPIN *) IMG_OFFSET_ADDR(psDCBufferPinOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferPinIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	DC_PIN_HANDLE hPinHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferPinOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = DCBufferPin(psBufferInt, &hPinHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferPin_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferPinOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							     &psDCBufferPinOUT->hPinHandle,
+							     (void *)hPinHandleInt,
+							     PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE,
+							     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							     (PFN_HANDLE_RELEASE) &
+							     _DCBufferPinhPinHandleIntRelease);
+	if (unlikely(psDCBufferPinOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferPin_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferPin_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferPinOUT->eError != PVRSRV_OK)
+	{
+		if (hPinHandleInt)
+		{
+			DCBufferUnpin(hPinHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferUnpin(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDCBufferUnpinIN_UI8,
+			  IMG_UINT8 * psDCBufferUnpinOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *psDCBufferUnpinIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *psDCBufferUnpinOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERUNPIN *) IMG_OFFSET_ADDR(psDCBufferUnpinOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDCBufferUnpinOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDCBufferUnpinIN->hPinHandle,
+					      PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE);
+	if (unlikely((psDCBufferUnpinOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferUnpinOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferUnpinOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferUnpin_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DCBufferUnpin_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DCBufferAcquirepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DCBufferRelease((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferAcquire(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferAcquireIN_UI8,
+			    IMG_UINT8 * psDCBufferAcquireOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *psDCBufferAcquireIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *psDCBufferAcquireOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERACQUIRE *) IMG_OFFSET_ADDR(psDCBufferAcquireOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psDCBufferAcquireIN->hBuffer;
+	DC_BUFFER *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCBufferAcquireOUT->eError = DCBufferAcquire(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferAcquireOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psDCBufferAcquireOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _DCBufferAcquirepsExtMemIntRelease);
+	if (unlikely(psDCBufferAcquireOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferAcquire_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferAcquire_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_DC_BUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDCBufferAcquireOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DCBufferRelease(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDCBufferRelease(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDCBufferReleaseIN_UI8,
+			    IMG_UINT8 * psDCBufferReleaseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *psDCBufferReleaseIN =
+	    (PVRSRV_BRIDGE_IN_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *psDCBufferReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_DCBUFFERRELEASE *) IMG_OFFSET_ADDR(psDCBufferReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psDCBufferReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psDCBufferReleaseIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psDCBufferReleaseOUT->eError != PVRSRV_OK) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDCBufferReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDCBufferReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto DCBufferRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+DCBufferRelease_exit:
+
+	return 0;
+}
+
+static_assert(DC_MAX_PIPE_COUNT <= IMG_UINT32_MAX,
+	      "DC_MAX_PIPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDCDisplayContextConfigure2(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psDCDisplayContextConfigure2IN_UI8,
+				       IMG_UINT8 * psDCDisplayContextConfigure2OUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2IN =
+	    (PVRSRV_BRIDGE_IN_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *psDCDisplayContextConfigure2OUT =
+	    (PVRSRV_BRIDGE_OUT_DCDISPLAYCONTEXTCONFIGURE2 *)
+	    IMG_OFFSET_ADDR(psDCDisplayContextConfigure2OUT_UI8, 0);
+
+	IMG_HANDLE hDisplayContext = psDCDisplayContextConfigure2IN->hDisplayContext;
+	DC_DISPLAY_CONTEXT *psDisplayContextInt = NULL;
+	PVRSRV_SURFACE_CONFIG_INFO *psSurfInfoInt = NULL;
+	DC_BUFFER **psBuffersInt = NULL;
+	IMG_HANDLE *hBuffersInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+#if !defined(INTEGRITY_OS)
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+#endif
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount *
+	     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *)) +
+	    ((IMG_UINT64) psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psDCDisplayContextConfigure2IN->ui32PipeCount > DC_MAX_PIPE_COUNT))
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+#if !defined(INTEGRITY_OS)
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDCDisplayContextConfigure2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDCDisplayContextConfigure2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+#endif
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDCDisplayContextConfigure2OUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psSurfInfoInt =
+		    (PVRSRV_SURFACE_CONFIG_INFO *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								   ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount *
+		    sizeof(PVRSRV_SURFACE_CONFIG_INFO);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(PVRSRV_SURFACE_CONFIG_INFO) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psSurfInfoInt,
+		     (const void __user *)psDCDisplayContextConfigure2IN->psSurfInfo,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount *
+		     sizeof(PVRSRV_SURFACE_CONFIG_INFO)) != PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount != 0)
+	{
+		psBuffersInt = (DC_BUFFER **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psBuffersInt, 0,
+			       psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *));
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(DC_BUFFER *);
+		hBuffersInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hBuffersInt2,
+		     (const void __user *)psDCDisplayContextConfigure2IN->phBuffers,
+		     psDCDisplayContextConfigure2IN->ui32PipeCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psDCDisplayContextConfigure2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DCDisplayContextConfigure2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDCDisplayContextConfigure2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDisplayContextInt,
+				       hDisplayContext,
+				       PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DCDisplayContextConfigure2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+			/* Look up the address from the handle */
+			psDCDisplayContextConfigure2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psBuffersInt[i],
+						       hBuffersInt2[i],
+						       PVRSRV_HANDLE_TYPE_DC_BUFFER, IMG_TRUE);
+			if (unlikely(psDCDisplayContextConfigure2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DCDisplayContextConfigure2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDCDisplayContextConfigure2OUT->eError =
+	    DCDisplayContextConfigure(psDisplayContextInt,
+				      psDCDisplayContextConfigure2IN->ui32PipeCount,
+				      psSurfInfoInt,
+				      psBuffersInt,
+				      psDCDisplayContextConfigure2IN->ui32DisplayPeriod,
+				      psDCDisplayContextConfigure2IN->ui32MaxDepth,
+				      psDCDisplayContextConfigure2IN->hAcquireFence,
+				      psDCDisplayContextConfigure2IN->hReleaseFenceTimeline,
+				      &psDCDisplayContextConfigure2OUT->hReleaseFence);
+
+DCDisplayContextConfigure2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDisplayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDisplayContext, PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT);
+	}
+
+	if (hBuffersInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDCDisplayContextConfigure2IN->ui32PipeCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psBuffersInt && psBuffersInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hBuffersInt2[i],
+							    PVRSRV_HANDLE_TYPE_DC_BUFFER);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDCDisplayContextConfigure2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+#if defined(INTEGRITY_OS)
+	if (pArrayArgsBuffer)
+#else
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+#endif
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDCBridge(void);
+void DeinitDCBridge(void);
+
+/*
+ * Register all DC functions with services
+ */
+PVRSRV_ERROR InitDCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT,
+			      PVRSRVBridgeDCDevicesQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE,
+			      PVRSRVBridgeDCDevicesEnumerate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE,
+			      PVRSRVBridgeDCDeviceAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE,
+			      PVRSRVBridgeDCDeviceRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO, PVRSRVBridgeDCGetInfo,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT,
+			      PVRSRVBridgeDCPanelQueryCount, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY,
+			      PVRSRVBridgeDCPanelQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY,
+			      PVRSRVBridgeDCFormatQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY, PVRSRVBridgeDCDimQuery,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK, PVRSRVBridgeDCSetBlank,
+			      NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING,
+			      PVRSRVBridgeDCSetVSyncReporting, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY,
+			      PVRSRVBridgeDCLastVSyncQuery, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE,
+			      PVRSRVBridgeDCSystemBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE,
+			      PVRSRVBridgeDCSystemBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE,
+			      PVRSRVBridgeDCDisplayContextCreate, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK,
+			      PVRSRVBridgeDCDisplayContextConfigureCheck, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY,
+			      PVRSRVBridgeDCDisplayContextDestroy, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC,
+			      PVRSRVBridgeDCBufferAlloc, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT,
+			      PVRSRVBridgeDCBufferImport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE,
+			      PVRSRVBridgeDCBufferFree, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT,
+			      PVRSRVBridgeDCBufferUnimport, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN,
+			      PVRSRVBridgeDCBufferPin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN,
+			      PVRSRVBridgeDCBufferUnpin, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE,
+			      PVRSRVBridgeDCBufferAcquire, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE,
+			      PVRSRVBridgeDCBufferRelease, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2,
+			      PVRSRVBridgeDCDisplayContextConfigure2, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dc functions with services
+ */
+void DeinitDCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICESENUMERATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICEACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDEVICERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCGETINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERYCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCPANELQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCFORMATQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDIMQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETBLANK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSETVSYNCREPORTING);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCLASTVSYNCQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCSYSTEMBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERALLOC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERFREE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNIMPORT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERUNPIN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERACQUIRE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCBUFFERRELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DC, PVRSRV_BRIDGE_DC_DCDISPLAYCONTEXTCONFIGURE2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_bridge.h
new file mode 100644
index 000000000000..bfa6bfb9c037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_bridge.h
@@ -0,0 +1,111 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+#define CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_devicememhistory_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut);
+
+#endif /* CLIENT_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
new file mode 100644
index 000000000000..a593a7e2cc7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/client_devicememhistory_direct_bridge.c
@@ -0,0 +1,195 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for devicememhistory
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_devicememhistory_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  uiOffset,
+				  sDevVAddr,
+				  uiSize,
+				  puiText,
+				  ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    uiOffset,
+				    sDevVAddr,
+				    uiSize,
+				    puiText,
+				    ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryMapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					sBaseDevVAddr,
+					ui32ui32StartPage,
+					ui32NumPages,
+					uiAllocSize,
+					puiText,
+					ui32Log2PageSize,
+					ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryUnmapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  sBaseDevVAddr,
+					  ui32ui32StartPage,
+					  ui32NumPages,
+					  uiAllocSize,
+					  puiText,
+					  ui32Log2PageSize,
+					  ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   uiOffset,
+					   sDevVAddr,
+					   uiSize,
+					   puiText,
+					   ui32Log2PageSize,
+					   ui32AllocPageCount,
+					   pui32AllocPageIndices,
+					   ui32FreePageCount,
+					   pui32FreePageIndices,
+					   ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/common_devicememhistory_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/common_devicememhistory_bridge.h
new file mode 100644
index 000000000000..800f98dd2836
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/common_devicememhistory_bridge.h
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DEVICEMEMHISTORY_BRIDGE_H
+#define COMMON_DEVICEMEMHISTORY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST			(PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4)
+
+/*******************************************
+            DevicememHistoryMap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP;
+
+/* Bridge out structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP;
+
+/*******************************************
+            DevicememHistoryUnmap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP;
+
+/* Bridge out structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP;
+
+/*******************************************
+            DevicememHistoryMapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE;
+
+/*******************************************
+            DevicememHistoryUnmapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/*******************************************
+            DevicememHistorySparseChange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE;
+
+/* Bridge out structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE;
+
+#endif /* COMMON_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/server_devicememhistory_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/server_devicememhistory_bridge.c
new file mode 100644
index 000000000000..72c6a86a0ec8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/devicememhistory_bridge/server_devicememhistory_bridge.c
@@ -0,0 +1,835 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+#include "common_devicememhistory_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMap(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevicememHistoryMapIN_UI8,
+				IMG_UINT8 * psDevicememHistoryMapOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *psDevicememHistoryMapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *psDevicememHistoryMapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryMapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryMapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryMapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryMap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryMapOUT->eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  psDevicememHistoryMapIN->uiOffset,
+				  psDevicememHistoryMapIN->sDevVAddr,
+				  psDevicememHistoryMapIN->uiSize,
+				  uiTextInt,
+				  psDevicememHistoryMapIN->ui32Log2PageSize,
+				  psDevicememHistoryMapIN->ui32AllocationIndex,
+				  &psDevicememHistoryMapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmap(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevicememHistoryUnmapIN_UI8,
+				  IMG_UINT8 * psDevicememHistoryUnmapOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryUnmapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryUnmapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryUnmapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryUnmapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryUnmap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryUnmapOUT->eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    psDevicememHistoryUnmapIN->uiOffset,
+				    psDevicememHistoryUnmapIN->sDevVAddr,
+				    psDevicememHistoryUnmapIN->uiSize,
+				    uiTextInt,
+				    psDevicememHistoryUnmapIN->ui32Log2PageSize,
+				    psDevicememHistoryUnmapIN->ui32AllocationIndex,
+				    &psDevicememHistoryUnmapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryUnmap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMapVRange(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeIN_UI8,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryMapVRangeOUT->eError =
+	    DevicememHistoryMapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					psDevicememHistoryMapVRangeIN->sBaseDevVAddr,
+					psDevicememHistoryMapVRangeIN->ui32ui32StartPage,
+					psDevicememHistoryMapVRangeIN->ui32NumPages,
+					psDevicememHistoryMapVRangeIN->uiAllocSize,
+					uiTextInt,
+					psDevicememHistoryMapVRangeIN->ui32Log2PageSize,
+					psDevicememHistoryMapVRangeIN->ui32AllocationIndex,
+					&psDevicememHistoryMapVRangeOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmapVRange(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeIN_UI8,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistoryUnmapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapVRangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryUnmapVRangeOUT->eError =
+	    DevicememHistoryUnmapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					  psDevicememHistoryUnmapVRangeIN->sBaseDevVAddr,
+					  psDevicememHistoryUnmapVRangeIN->ui32ui32StartPage,
+					  psDevicememHistoryUnmapVRangeIN->ui32NumPages,
+					  psDevicememHistoryUnmapVRangeIN->uiAllocSize,
+					  uiTextInt,
+					  psDevicememHistoryUnmapVRangeIN->ui32Log2PageSize,
+					  psDevicememHistoryUnmapVRangeIN->ui32AllocationIndex,
+					  &psDevicememHistoryUnmapVRangeOUT->
+					  ui32AllocationIndexOut);
+
+DevicememHistoryUnmapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistorySparseChange(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psDevicememHistorySparseChangeIN_UI8,
+					 IMG_UINT8 * psDevicememHistorySparseChangeOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistorySparseChangeIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32AllocPageCount *
+	     sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32FreePageCount *
+	     sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32AllocPageCount >
+	     PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistorySparseChangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistorySparseChangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistorySparseChangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistorySparseChange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32AllocPageIndices,
+		     psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32FreePageIndices,
+		     psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistorySparseChangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistorySparseChangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistorySparseChange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistorySparseChangeOUT->eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   psDevicememHistorySparseChangeIN->uiOffset,
+					   psDevicememHistorySparseChangeIN->sDevVAddr,
+					   psDevicememHistorySparseChangeIN->uiSize,
+					   uiTextInt,
+					   psDevicememHistorySparseChangeIN->ui32Log2PageSize,
+					   psDevicememHistorySparseChangeIN->ui32AllocPageCount,
+					   ui32AllocPageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32FreePageCount,
+					   ui32FreePageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32AllocationIndex,
+					   &psDevicememHistorySparseChangeOUT->
+					   ui32AllocationIndexOut);
+
+DevicememHistorySparseChange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistorySparseChangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pDEVICEMEMHISTORYBridgeLock;
+
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+
+/*
+ * Register all DEVICEMEMHISTORY functions with services
+ */
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pDEVICEMEMHISTORYBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP,
+			      PVRSRVBridgeDevicememHistoryMap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP,
+			      PVRSRVBridgeDevicememHistoryUnmap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryMapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryUnmapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE,
+			      PVRSRVBridgeDevicememHistorySparseChange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all devicememhistory functions with services
+ */
+void DeinitDEVICEMEMHISTORYBridge(void)
+{
+	OSLockDestroy(pDEVICEMEMHISTORYBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/common_di_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/common_di_bridge.h
new file mode 100644
index 000000000000..8591006140b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/common_di_bridge.h
@@ -0,0 +1,153 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DI_BRIDGE_H
+#define COMMON_DI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "pvr_dicommon.h"
+
+#define PVRSRV_BRIDGE_DI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DI_DICREATECONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DI_DIREADENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DI_DIWRITEENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DI_DILISTALLENTRIES			PVRSRV_BRIDGE_DI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DI_CMD_LAST			(PVRSRV_BRIDGE_DI_CMD_FIRST+4)
+
+/*******************************************
+            DICreateContext
+ *******************************************/
+
+/* Bridge in structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_IN_DICREATECONTEXT_TAG
+{
+	IMG_CHAR *puiStreamName;
+} __packed PVRSRV_BRIDGE_IN_DICREATECONTEXT;
+
+/* Bridge out structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_OUT_DICREATECONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_CHAR *puiStreamName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DICREATECONTEXT;
+
+/*******************************************
+            DIDestroyContext
+ *******************************************/
+
+/* Bridge in structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT;
+
+/* Bridge out structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT;
+
+/*******************************************
+            DIReadEntry
+ *******************************************/
+
+/* Bridge in structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIREADENTRY_TAG
+{
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+} __packed PVRSRV_BRIDGE_IN_DIREADENTRY;
+
+/* Bridge out structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIREADENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIREADENTRY;
+
+/*******************************************
+            DIWriteEntry
+ *******************************************/
+
+/* Bridge in structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIWRITEENTRY_TAG
+{
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+	const IMG_CHAR *puiValue;
+	IMG_UINT32 ui32ValueSize;
+} __packed PVRSRV_BRIDGE_IN_DIWRITEENTRY;
+
+/* Bridge out structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIWRITEENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIWRITEENTRY;
+
+/*******************************************
+            DIListAllEntries
+ *******************************************/
+
+/* Bridge in structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_IN_DILISTALLENTRIES_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DILISTALLENTRIES;
+
+/* Bridge out structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_OUT_DILISTALLENTRIES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DILISTALLENTRIES;
+
+#endif /* COMMON_DI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/server_di_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/server_di_bridge.c
new file mode 100644
index 000000000000..154bccdaae5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/di_bridge/server_di_bridge.c
@@ -0,0 +1,652 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "di_impl_brg.h"
+
+#include "common_di_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DICreateContextpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DIDestroyContextKM((DI_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDICreateContext(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDICreateContextIN_UI8,
+			    IMG_UINT8 * psDICreateContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DICREATECONTEXT *psDICreateContextIN =
+	    (PVRSRV_BRIDGE_IN_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DICREATECONTEXT *psDICreateContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextOUT_UI8, 0);
+
+	IMG_CHAR *puiStreamNameInt = NULL;
+	DI_CONTEXT *psContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	PVR_UNREFERENCED_PARAMETER(psDICreateContextIN);
+
+	psDICreateContextOUT->puiStreamName = psDICreateContextIN->puiStreamName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDICreateContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DICreateContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDICreateContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDICreateContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDICreateContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DICreateContext_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		puiStreamNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	psDICreateContextOUT->eError = DICreateContextKM(puiStreamNameInt, &psContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		goto DICreateContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDICreateContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDICreateContextOUT->hContext,
+								 (void *)psContextInt,
+								 PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								 (PFN_HANDLE_RELEASE) &
+								 _DICreateContextpsContextIntRelease);
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DICreateContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamNameInt) && ((PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDICreateContextOUT->puiStreamName, puiStreamNameInt,
+		      (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psDICreateContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DICreateContext_exit;
+		}
+	}
+
+DICreateContext_exit:
+
+	if (psDICreateContextOUT->eError != PVRSRV_OK)
+	{
+		if (psDICreateContextOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDICreateContextOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DIDestroyContextKM(psContextInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDICreateContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIDestroyContext(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIDestroyContextIN_UI8,
+			     IMG_UINT8 * psDIDestroyContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *psDIDestroyContextIN =
+	    (PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *psDIDestroyContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDIDestroyContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDIDestroyContextIN->hContext,
+					      PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	if (unlikely((psDIDestroyContextOUT->eError != PVRSRV_OK) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDIDestroyContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIDestroyContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DIDestroyContext_exit:
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIReadEntry(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDIReadEntryIN_UI8,
+			IMG_UINT8 * psDIReadEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIREADENTRY *psDIReadEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIREADENTRY *psDIReadEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIReadEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIReadEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIReadEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIReadEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIReadEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIReadEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIReadEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIReadEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIReadEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIReadEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIReadEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIReadEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIReadEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIReadEntryOUT->eError =
+	    DIReadEntryKM(psContextInt,
+			  uiEntryPathInt, psDIReadEntryIN->ui64Offset, psDIReadEntryIN->ui64Size);
+
+DIReadEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIReadEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIWriteEntry(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDIWriteEntryIN_UI8,
+			 IMG_UINT8 * psDIWriteEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIWRITEENTRY *psDIWriteEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIWRITEENTRY *psDIWriteEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIWriteEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+	IMG_CHAR *uiValueInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psDIWriteEntryIN->ui32ValueSize > DI_IMPL_BRG_PATH_LEN))
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DIWriteEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIWriteEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIWriteEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIWriteEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIWriteEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIWriteEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIWriteEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDIWriteEntryIN->ui32ValueSize != 0)
+	{
+		uiValueInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiValueInt, (const void __user *)psDIWriteEntryIN->puiValue,
+		     psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiValueInt)[(psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) -
+					  1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIWriteEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIWriteEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIWriteEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIWriteEntryOUT->eError =
+	    DIWriteEntryKM(psContextInt,
+			   uiEntryPathInt, psDIWriteEntryIN->ui32ValueSize, uiValueInt);
+
+DIWriteEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIWriteEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIListAllEntries(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIListAllEntriesIN_UI8,
+			     IMG_UINT8 * psDIListAllEntriesOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DILISTALLENTRIES *psDIListAllEntriesIN =
+	    (PVRSRV_BRIDGE_IN_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *psDIListAllEntriesOUT =
+	    (PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIListAllEntriesIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIListAllEntriesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIListAllEntriesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIListAllEntries_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIListAllEntriesOUT->eError = DIListAllEntriesKM(psContextInt);
+
+DIListAllEntries_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+
+/*
+ * Register all DI functions with services
+ */
+PVRSRV_ERROR InitDIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT,
+			      PVRSRVBridgeDICreateContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DICREATECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DICREATECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT,
+			      PVRSRVBridgeDIDestroyContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY,
+			      PVRSRVBridgeDIReadEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIREADENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIREADENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY,
+			      PVRSRVBridgeDIWriteEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIWRITEENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIWRITEENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES,
+			      PVRSRVBridgeDIListAllEntries, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DILISTALLENTRIES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DILISTALLENTRIES));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all di functions with services
+ */
+void DeinitDIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/common_dma_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/common_dma_bridge.h
new file mode 100644
index 000000000000..dafa475b9c9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/common_dma_bridge.h
@@ -0,0 +1,123 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMA_BRIDGE_H
+#define COMMON_DMA_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_DMA_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMA_DMATRANSFER			PVRSRV_BRIDGE_DMA_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE			PVRSRV_BRIDGE_DMA_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS			PVRSRV_BRIDGE_DMA_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMA_CMD_LAST			(PVRSRV_BRIDGE_DMA_CMD_FIRST+2)
+
+/*******************************************
+            DmaTransfer
+ *******************************************/
+
+/* Bridge in structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_IN_DMATRANSFER_TAG
+{
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32NumDMAs;
+	IMG_UINT32 ui32uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DMATRANSFER;
+
+/* Bridge out structure for DmaTransfer */
+typedef struct PVRSRV_BRIDGE_OUT_DMATRANSFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMATRANSFER;
+
+/*******************************************
+            DmaSparseMappingTable
+ *******************************************/
+
+/* Bridge in structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_BOOL *pbTable;
+	IMG_UINT32 ui32SizeInPages;
+} __packed PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE;
+
+/* Bridge out structure for DmaSparseMappingTable */
+typedef struct PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE_TAG
+{
+	IMG_BOOL *pbTable;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE;
+
+/*******************************************
+            DmaDeviceParams
+ *******************************************/
+
+/* Bridge in structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_IN_DMADEVICEPARAMS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DMADEVICEPARAMS;
+
+/* Bridge out structure for DmaDeviceParams */
+typedef struct PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DmaBuffAlign;
+	IMG_UINT32 ui32DmaTransferMult;
+} __packed PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS;
+
+#endif /* COMMON_DMA_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/server_dma_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/server_dma_bridge.c
new file mode 100644
index 000000000000..7f2c5dd495e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dma_bridge/server_dma_bridge.c
@@ -0,0 +1,490 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dma
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dma
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "dma_km.h"
+
+#include "common_dma_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(MAX_DMA_OPS <= IMG_UINT32_MAX, "MAX_DMA_OPS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaTransfer(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDmaTransferIN_UI8,
+			IMG_UINT8 * psDmaTransferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMATRANSFER *psDmaTransferIN =
+	    (PVRSRV_BRIDGE_IN_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMATRANSFER *psDmaTransferOUT =
+	    (PVRSRV_BRIDGE_OUT_DMATRANSFER *) IMG_OFFSET_ADDR(psDmaTransferOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(PMR *)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) + 0;
+
+	if (unlikely(psDmaTransferIN->ui32NumDMAs > MAX_DMA_OPS))
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaTransfer_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaTransferOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaTransfer_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaTransferIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaTransferIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaTransferOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psDmaTransferIN->ui32NumDMAs * sizeof(PMR *));
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psDmaTransferIN->phPMR,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psDmaTransferIN->pui64Address,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psDmaTransferIN->puiOffset,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+	if (psDmaTransferIN->ui32NumDMAs != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psDmaTransferIN->puiSize,
+		     psDmaTransferIN->ui32NumDMAs * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psDmaTransferOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaTransfer_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+			/* Look up the address from the handle */
+			psDmaTransferOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psDmaTransferOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto DmaTransfer_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaTransferOUT->eError =
+	    DmaTransfer(psConnection, OSGetDevNode(psConnection),
+			psDmaTransferIN->ui32NumDMAs,
+			psPMRInt,
+			ui64AddressInt,
+			uiOffsetInt,
+			uiSizeInt, psDmaTransferIN->ui32uiFlags, psDmaTransferIN->hUpdateTimeline);
+
+DmaTransfer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psDmaTransferIN->ui32NumDMAs; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaTransferOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(32 <= IMG_UINT32_MAX, "32 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDmaSparseMappingTable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDmaSparseMappingTableIN_UI8,
+				  IMG_UINT8 * psDmaSparseMappingTableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableIN =
+	    (PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *psDmaSparseMappingTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE *)
+	    IMG_OFFSET_ADDR(psDmaSparseMappingTableOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDmaSparseMappingTableIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_BOOL *pbTableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) + 0;
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages > 32)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	psDmaSparseMappingTableOUT->pbTable = psDmaSparseMappingTableIN->pbTable;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DmaSparseMappingTable_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDmaSparseMappingTableIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDmaSparseMappingTableIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DmaSparseMappingTable_exit;
+			}
+		}
+	}
+
+	if (psDmaSparseMappingTableIN->ui32SizeInPages != 0)
+	{
+		pbTableInt = (IMG_BOOL *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDmaSparseMappingTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DmaSparseMappingTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDmaSparseMappingTableOUT->eError =
+	    DmaSparseMappingTable(psPMRInt,
+				  psDmaSparseMappingTableIN->uiOffset,
+				  psDmaSparseMappingTableIN->ui32SizeInPages, pbTableInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDmaSparseMappingTableOUT->eError != PVRSRV_OK))
+	{
+		goto DmaSparseMappingTable_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pbTableInt) && ((psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDmaSparseMappingTableOUT->pbTable, pbTableInt,
+		      (psDmaSparseMappingTableIN->ui32SizeInPages * sizeof(IMG_BOOL))) !=
+		     PVRSRV_OK))
+		{
+			psDmaSparseMappingTableOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DmaSparseMappingTable_exit;
+		}
+	}
+
+DmaSparseMappingTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDmaSparseMappingTableOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDmaDeviceParams(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDmaDeviceParamsIN_UI8,
+			    IMG_UINT8 * psDmaDeviceParamsOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *psDmaDeviceParamsIN =
+	    (PVRSRV_BRIDGE_IN_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *psDmaDeviceParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS *) IMG_OFFSET_ADDR(psDmaDeviceParamsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psDmaDeviceParamsIN);
+
+	psDmaDeviceParamsOUT->eError =
+	    DmaDeviceParams(psConnection, OSGetDevNode(psConnection),
+			    &psDmaDeviceParamsOUT->ui32DmaBuffAlign,
+			    &psDmaDeviceParamsOUT->ui32DmaTransferMult);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABridge(void);
+void DeinitDMABridge(void);
+
+/*
+ * Register all DMA functions with services
+ */
+PVRSRV_ERROR InitDMABridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER,
+			      PVRSRVBridgeDmaTransfer, NULL, sizeof(PVRSRV_BRIDGE_IN_DMATRANSFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMATRANSFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE,
+			      PVRSRVBridgeDmaSparseMappingTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DMASPARSEMAPPINGTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DMASPARSEMAPPINGTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS,
+			      PVRSRVBridgeDmaDeviceParams, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DMADEVICEPARAMS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dma functions with services
+ */
+void DeinitDMABridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMATRANSFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMASPARSEMAPPINGTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMA, PVRSRV_BRIDGE_DMA_DMADEVICEPARAMS);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/common_dmabuf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/common_dmabuf_bridge.h
new file mode 100644
index 000000000000..cc631e3e1116
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/common_dmabuf_bridge.h
@@ -0,0 +1,145 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMABUF_BRIDGE_H
+#define COMMON_DMABUF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_DMABUF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DMABUF_CMD_LAST			(PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3)
+
+/*******************************************
+            PhysmemImportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF;
+
+/* Bridge out structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF;
+
+/*******************************************
+            PhysmemExportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF;
+
+/* Bridge out structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF;
+
+/*******************************************
+            PhysmemExportGemHandle
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE;
+
+/* Bridge out structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Handle;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE;
+
+/*******************************************
+            PhysmemImportSparseDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_SIZE_T uiChunkSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF;
+
+/* Bridge out structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF;
+
+#endif /* COMMON_DMABUF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/server_dmabuf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/server_dmabuf_bridge.c
new file mode 100644
index 000000000000..89aa1477f55a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/dmabuf_bridge/server_dmabuf_bridge.c
@@ -0,0 +1,574 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "physmem_dmabuf.h"
+#include "pmr.h"
+
+#include "common_dmabuf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemImportDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemImportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemImportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufOUT_UI8,
+								      0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPhysmemImportDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportDmaBufIN->puiName,
+		     psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportDmaBuf_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR))
+					 - 1] = '\0';
+	}
+
+	psPhysmemImportDmaBufOUT->eError =
+	    PhysmemImportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPhysmemImportDmaBufIN->ifd,
+				psPhysmemImportDmaBufIN->uiFlags,
+				psPhysmemImportDmaBufIN->ui32NameSize,
+				uiNameInt,
+				&psPMRPtrInt,
+				&psPhysmemImportDmaBufOUT->uiSize,
+				&psPhysmemImportDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportDmaBufOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psPhysmemImportDmaBufOUT->
+								     hPMRPtr, (void *)psPMRPtrInt,
+								     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _PhysmemImportDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportDmaBuf_exit:
+
+	if (psPhysmemImportDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemExportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemExportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psPhysmemExportDmaBufIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportDmaBufOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportDmaBuf_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportDmaBufOUT->eError =
+	    PhysmemExportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPMRInt, &psPhysmemExportDmaBufOUT->iFd);
+
+PhysmemExportDmaBuf_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportGemHandle(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemExportGemHandleIN_UI8,
+				   IMG_UINT8 * psPhysmemExportGemHandleOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPhysmemExportGemHandleIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportGemHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportGemHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportGemHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportGemHandleOUT->eError =
+	    PhysmemExportGemHandle(psConnection, OSGetDevNode(psConnection),
+				   psPMRInt, &psPhysmemExportGemHandleOUT->ui32Handle);
+
+PhysmemExportGemHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemImportSparseDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportSparseDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufIN_UI8,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufOUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (unlikely(psPhysmemImportSparseDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportSparseDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportSparseDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportSparseDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemImportSparseDmaBufIN->pui32MappingTable,
+		     psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+	}
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportSparseDmaBufIN->puiName,
+		     psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+		((IMG_CHAR *)
+		 uiNameInt)[(psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PhysmemImportSparseDmaBuf(psConnection, OSGetDevNode(psConnection),
+				      psPhysmemImportSparseDmaBufIN->ifd,
+				      psPhysmemImportSparseDmaBufIN->uiFlags,
+				      psPhysmemImportSparseDmaBufIN->uiChunkSize,
+				      psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks,
+				      psPhysmemImportSparseDmaBufIN->ui32NumVirtChunks,
+				      ui32MappingTableInt,
+				      psPhysmemImportSparseDmaBufIN->ui32NameSize,
+				      uiNameInt,
+				      &psPMRPtrInt,
+				      &psPhysmemImportSparseDmaBufOUT->uiSize,
+				      &psPhysmemImportSparseDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psPhysmemImportSparseDmaBufOUT->hPMRPtr, (void *)psPMRPtrInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PhysmemImportSparseDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportSparseDmaBuf_exit:
+
+	if (psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportSparseDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+/*
+ * Register all DMABUF functions with services
+ */
+PVRSRV_ERROR InitDMABUFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF,
+			      PVRSRVBridgePhysmemImportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF,
+			      PVRSRVBridgePhysmemExportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE,
+			      PVRSRVBridgePhysmemExportGemHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF,
+			      PVRSRVBridgePhysmemImportSparseDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dmabuf functions with services
+ */
+void DeinitDMABUFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF,
+				PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_bridge.h
new file mode 100644
index 000000000000..2c5d18843120
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_bridge.h
@@ -0,0 +1,64 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_HTBUFFER_BRIDGE_H
+#define CLIENT_HTBUFFER_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_htbuffer_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode);
+
+#endif /* CLIENT_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_direct_bridge.c
new file mode 100644
index 000000000000..3e6295cdfdf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/client_htbuffer_direct_bridge.c
@@ -0,0 +1,70 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for htbuffer
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_htbuffer_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#include "htbserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError =
+	    HTBControlKM(ui32NumGroups,
+			 pui32GroupEnable, ui32LogLevel, ui32EnablePID, ui32LogMode, ui32OpMode);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/common_htbuffer_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/common_htbuffer_bridge.h
new file mode 100644
index 000000000000..32be202186dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/common_htbuffer_bridge.h
@@ -0,0 +1,82 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_HTBUFFER_BRIDGE_H
+#define COMMON_HTBUFFER_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST			0
+#define PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL			PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_LAST			(PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0)
+
+/*******************************************
+            HTBControl
+ *******************************************/
+
+/* Bridge in structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_IN_HTBCONTROL_TAG
+{
+	IMG_UINT32 *pui32GroupEnable;
+	IMG_UINT32 ui32EnablePID;
+	IMG_UINT32 ui32LogLevel;
+	IMG_UINT32 ui32LogMode;
+	IMG_UINT32 ui32NumGroups;
+	IMG_UINT32 ui32OpMode;
+} __packed PVRSRV_BRIDGE_IN_HTBCONTROL;
+
+/* Bridge out structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_OUT_HTBCONTROL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HTBCONTROL;
+
+#endif /* COMMON_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/server_htbuffer_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/server_htbuffer_bridge.c
new file mode 100644
index 000000000000..9b730d28e5b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/htbuffer_bridge/server_htbuffer_bridge.c
@@ -0,0 +1,227 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "htbserver.h"
+
+#include "common_htbuffer_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(HTB_FLAG_NUM_EL <= IMG_UINT32_MAX,
+	      "HTB_FLAG_NUM_EL must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHTBControl(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psHTBControlIN_UI8,
+		       IMG_UINT8 * psHTBControlOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HTBCONTROL *psHTBControlIN =
+	    (PVRSRV_BRIDGE_IN_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HTBCONTROL *psHTBControlOUT =
+	    (PVRSRV_BRIDGE_OUT_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlOUT_UI8, 0);
+
+	IMG_UINT32 *ui32GroupEnableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psHTBControlIN->ui32NumGroups > HTB_FLAG_NUM_EL))
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HTBControl_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HTBControl_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHTBControlIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHTBControlIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHTBControlOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HTBControl_exit;
+			}
+		}
+	}
+
+	if (psHTBControlIN->ui32NumGroups != 0)
+	{
+		ui32GroupEnableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32GroupEnableInt,
+		     (const void __user *)psHTBControlIN->pui32GroupEnable,
+		     psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psHTBControlOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HTBControl_exit;
+		}
+	}
+
+	psHTBControlOUT->eError =
+	    HTBControlKM(psHTBControlIN->ui32NumGroups,
+			 ui32GroupEnableInt,
+			 psHTBControlIN->ui32LogLevel,
+			 psHTBControlIN->ui32EnablePID,
+			 psHTBControlIN->ui32LogMode, psHTBControlIN->ui32OpMode);
+
+HTBControl_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHTBControlOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pHTBUFFERBridgeLock;
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+
+/*
+ * Register all HTBUFFER functions with services
+ */
+PVRSRV_ERROR InitHTBUFFERBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pHTBUFFERBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL,
+			      PVRSRVBridgeHTBControl, pHTBUFFERBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_HTBCONTROL),
+			      sizeof(PVRSRV_BRIDGE_OUT_HTBCONTROL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all htbuffer functions with services
+ */
+void DeinitHTBUFFERBridge(void)
+{
+	OSLockDestroy(pHTBUFFERBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL);
+
+}
+#else /* EXCLUDE_HTBUFFER_BRIDGE */
+/* This bridge is conditional on EXCLUDE_HTBUFFER_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitHTBUFFERBridge() \
+	PVRSRV_OK
+
+#define DeinitHTBUFFERBridge()
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_bridge.h
new file mode 100644
index 000000000000..d5ba6dd74da3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_bridge.h
@@ -0,0 +1,234 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_MM_BRIDGE_H
+#define CLIENT_MM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_mm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge,
+							IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags,
+						IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats);
+
+#endif /* CLIENT_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_direct_bridge.c
new file mode 100644
index 000000000000..c1810e959ca8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/client_mm_direct_bridge.c
@@ -0,0 +1,719 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for mm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_mm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRExportPMR(psPMRInt, &psPMRExportInt, pui64Size, pui32Log2Contig, pui64Password);
+
+	*phPMRExport = psPMRExportInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMR);
+	PVR_UNREFERENCED_PARAMETER(phPMRExport);
+	PVR_UNREFERENCED_PARAMETER(pui64Size);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2Contig);
+	PVR_UNREFERENCED_PARAMETER(pui64Password);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError = PMRUnexportPMR(psPMRExportInt);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRGetUID(psPMRInt, pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psBufferInt;
+	PMR *psExtMemInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psBufferInt = (PMR *) hBuffer;
+
+	eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+
+	*phExtMem = psExtMemInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtMemInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtMemInt = (PMR *) hExtMem;
+
+	eError = PMRUnmakeLocalImportHandle(psExtMemInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PMR *psPMRInt = NULL;
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError =
+	    PhysmemImportPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     psPMRExportInt,
+			     ui64uiPassword, ui64uiSize, ui32uiLog2Contig, &psPMRInt);
+
+	*phPMR = psPMRInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+	PVR_UNREFERENCED_PARAMETER(ui64uiPassword);
+	PVR_UNREFERENCED_PARAMETER(ui64uiSize);
+	PVR_UNREFERENCED_PARAMETER(ui32uiLog2Contig);
+	PVR_UNREFERENCED_PARAMETER(phPMR);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtHandleInt;
+	PMR *psPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtHandleInt = (PMR *) hExtHandle;
+
+	eError = PMRLocalImportPMR(psExtHandleInt, &psPMRInt, puiSize, puiAlign);
+
+	*phPMR = psPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRUnrefPMR(psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRPtrInt = NULL;
+
+	eError =
+	    PhysmemNewRamBackedPMR_direct(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  uiSize,
+					  ui32NumPhysChunks,
+					  ui32NumVirtChunks,
+					  pui32MappingTable,
+					  ui32Log2PageSize,
+					  uiFlags,
+					  ui32AnnotationLength,
+					  puiAnnotation,
+					  ui32PID, &psPMRPtrInt, ui32PDumpFlags, puiOutFlags);
+
+	*phPMRPtr = psPMRPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	eError =
+	    DevmemIntCtxCreate(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       bbKernelMemoryCtx,
+			       &psDevMemServerContextInt, &hPrivDataInt, pui32CPUCacheLineSize);
+
+	*phDevMemServerContext = psDevMemServerContextInt;
+	*phPrivData = hPrivDataInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError = DevmemIntCtxDestroy(psDevmemServerContextInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				ui32HeapConfigIndex, ui32HeapIndex, &psDevmemHeapPtrInt);
+
+	*phDevmemHeapPtr = psDevmemHeapPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeapInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemHeapInt = (DEVMEMINT_HEAP *) hDevmemHeap;
+
+	eError = DevmemIntHeapDestroy(psDevmemHeapInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnmapPMR(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemIntReserveRange(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psDevmemServerHeapInt,
+				  sAddress, uiLength, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	PMR *psPMRInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemIntReserveRangeAndMapPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   psDevmemServerHeapInt,
+					   sAddress,
+					   uiLength, psPMRInt, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemIntChangeSparse(ui32AllocPageCount,
+				  pui32AllocPageIndices,
+				  ui32FreePageCount,
+				  pui32FreePageIndices, ui32SparseFlags, psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntIsVDevAddrValid(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, sAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries)
+{
+#if defined(RGX_FEATURE_FBCDC)
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntInvalidateFBSCTable(psDevmemCtxInt, ui64FBSCEntries);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(ui64FBSCEntries);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   pui32NumHeapConfigs);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     ui32HeapConfigIndex, pui32NumHeaps);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigName(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  ui32HeapConfigIndex, ui32HeapConfigNameBufSz, puiHeapConfigName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapDetails(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       ui32HeapConfigIndex,
+			       ui32HeapIndex,
+			       ui32HeapNameBufSz,
+			       puiHeapNameOut,
+			       psDevVAddrBase,
+			       puiHeapLength,
+			       puiReservedRegionLength,
+			       pui32Log2DataPageSizeOut, pui32Log2ImportAlignmentOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, bRegister);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS_V1 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfoKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				       ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), peHeap);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntGetFaultAddress(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, psFaultAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVStatsUpdateOOMStat(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32ui32StatType, ui32pid);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32ui32StatType);
+	PVR_UNREFERENCED_PARAMETER(ui32pid);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt, sAddress, uiLength, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       ui32PageCount, ui32PhysPageOffset, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnmapPages(psReservationInt, ui32VirtPageOffset, ui32PageCount);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     ui32PageCount, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo2(IMG_HANDLE hBridge,
+						    IMG_UINT32 ui32PhysHeapCount,
+						    PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						    PHYS_HEAP_MEM_STATS_V2 * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/common_mm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/common_mm_bridge.h
new file mode 100644
index 000000000000..3b3c8c2dc215
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/common_mm_bridge.h
@@ -0,0 +1,757 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MM_BRIDGE_H
+#define COMMON_MM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MM_PMREXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_MM_PMRGETUID			PVRSRV_BRIDGE_MM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_MM_PMRIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_MM_PMRUNREFPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+9
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+10
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+11
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+12
+#define PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+13
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+14
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+15
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+16
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+17
+#define PVRSRV_BRIDGE_MM_CHANGESPARSEMEM			PVRSRV_BRIDGE_MM_CMD_FIRST+18
+#define PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID			PVRSRV_BRIDGE_MM_CMD_FIRST+19
+#define PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE			PVRSRV_BRIDGE_MM_CMD_FIRST+20
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+21
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+22
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME			PVRSRV_BRIDGE_MM_CMD_FIRST+23
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS			PVRSRV_BRIDGE_MM_CMD_FIRST+24
+#define PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM			PVRSRV_BRIDGE_MM_CMD_FIRST+25
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO			PVRSRV_BRIDGE_MM_CMD_FIRST+26
+#define PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP			PVRSRV_BRIDGE_MM_CMD_FIRST+27
+#define PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS			PVRSRV_BRIDGE_MM_CMD_FIRST+28
+#define PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT			PVRSRV_BRIDGE_MM_CMD_FIRST+29
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+30
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+31
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+32
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+33
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE			PVRSRV_BRIDGE_MM_CMD_FIRST+34
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2			PVRSRV_BRIDGE_MM_CMD_FIRST+35
+#define PVRSRV_BRIDGE_MM_CMD_LAST			(PVRSRV_BRIDGE_MM_CMD_FIRST+35)
+
+/*******************************************
+            PMRExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMREXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMREXPORTPMR;
+
+/* Bridge out structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMREXPORTPMR_TAG
+{
+	IMG_UINT64 ui64Password;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRExport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2Contig;
+} __packed PVRSRV_BRIDGE_OUT_PMREXPORTPMR;
+
+/*******************************************
+            PMRUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMRExport;
+} __packed PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR;
+
+/* Bridge out structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR;
+
+/*******************************************
+            PMRGetUID
+ *******************************************/
+
+/* Bridge in structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_IN_PMRGETUID_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRGETUID;
+
+/* Bridge out structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_OUT_PMRGETUID_TAG
+{
+	IMG_UINT64 ui64UID;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRGETUID;
+
+/*******************************************
+            PMRMakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRUnmakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRIMPORTPMR_TAG
+{
+	IMG_UINT64 ui64uiPassword;
+	IMG_UINT64 ui64uiSize;
+	IMG_HANDLE hPMRExport;
+	IMG_UINT32 ui32uiLog2Contig;
+} __packed PVRSRV_BRIDGE_IN_PMRIMPORTPMR;
+
+/* Bridge out structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRIMPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRIMPORTPMR;
+
+/*******************************************
+            PMRLocalImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_HANDLE hExtHandle;
+} __packed PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR;
+
+/* Bridge out structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR;
+
+/*******************************************
+            PMRUnrefPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNREFPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRUNREFPMR;
+
+/* Bridge out structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNREFPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNREFPMR;
+
+/*******************************************
+            PhysmemNewRamBackedPMR
+ *******************************************/
+
+/* Bridge in structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiAnnotation;
+	IMG_UINT32 ui32AnnotationLength;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_PID ui32PID;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR;
+
+/* Bridge out structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR;
+
+/*******************************************
+            DevmemIntCtxCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_BOOL bbKernelMemoryCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE;
+
+/* Bridge out structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_HANDLE hDevMemServerContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CPUCacheLineSize;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE;
+
+/*******************************************
+            DevmemIntCtxDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY_TAG
+{
+	IMG_HANDLE hDevmemServerContext;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY;
+
+/* Bridge out structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY;
+
+/*******************************************
+            DevmemIntHeapCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE;
+
+/* Bridge out structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemHeapPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE;
+
+/*******************************************
+            DevmemIntHeapDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY_TAG
+{
+	IMG_HANDLE hDevmemHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY;
+
+/* Bridge out structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY;
+
+/*******************************************
+            DevmemIntMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR;
+
+/* Bridge out structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR;
+
+/*******************************************
+            DevmemIntUnmapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR;
+
+/* Bridge out structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR;
+
+/*******************************************
+            DevmemIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE;
+
+/* Bridge out structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE;
+
+/*******************************************
+            DevmemIntReserveRangeAndMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	IMG_HANDLE hPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/* Bridge out structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/*******************************************
+            DevmemIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE;
+
+/*******************************************
+            ChangeSparseMem
+ *******************************************/
+
+/* Bridge in structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_IN_CHANGESPARSEMEM_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32SparseFlags;
+} __packed PVRSRV_BRIDGE_IN_CHANGESPARSEMEM;
+
+/* Bridge out structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM;
+
+/*******************************************
+            DevmemIsVDevAddrValid
+ *******************************************/
+
+/* Bridge in structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID;
+
+/* Bridge out structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID;
+
+/*******************************************
+            DevmemInvalidateFBSCTable
+ *******************************************/
+
+/* Bridge in structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	IMG_UINT64 ui64FBSCEntries;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE;
+
+/* Bridge out structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE;
+
+/*******************************************
+            HeapCfgHeapConfigCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT;
+
+/* Bridge out structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeapConfigs;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT;
+
+/*******************************************
+            HeapCfgHeapCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT_TAG
+{
+	IMG_UINT32 ui32HeapConfigIndex;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT;
+
+/* Bridge out structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT;
+
+/*******************************************
+            HeapCfgHeapConfigName
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapConfigNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME;
+
+/* Bridge out structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME;
+
+/*******************************************
+            HeapCfgHeapDetails
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_CHAR *puiHeapNameOut;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+	IMG_UINT32 ui32HeapNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS;
+
+/* Bridge out structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_CHAR *puiHeapNameOut;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2DataPageSizeOut;
+	IMG_UINT32 ui32Log2ImportAlignmentOut;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS;
+
+/*******************************************
+            DevmemIntRegisterPFNotifyKM
+ *******************************************/
+
+/* Bridge in structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_BOOL bRegister;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/* Bridge out structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/*******************************************
+            PhysHeapGetMemInfo
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO;
+
+/* Bridge out structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO;
+
+/*******************************************
+            GetDefaultPhysicalHeap
+ *******************************************/
+
+/* Bridge in structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP;
+
+/* Bridge out structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP eHeap;
+} __packed PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP;
+
+/*******************************************
+            DevmemGetFaultAddress
+ *******************************************/
+
+/* Bridge in structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS;
+
+/* Bridge out structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_DEV_VIRTADDR sFaultAddress;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS;
+
+/*******************************************
+            PVRSRVStatsUpdateOOMStat
+ *******************************************/
+
+/* Bridge in structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	IMG_PID ui32pid;
+	IMG_UINT32 ui32ui32StatType;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT;
+
+/* Bridge out structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT;
+
+/*******************************************
+            DevmemXIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE;
+
+/*******************************************
+            DevmemXIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE;
+
+/*******************************************
+            DevmemXIntMapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32PhysPageOffset;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES;
+
+/* Bridge out structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES;
+
+/*******************************************
+            DevmemXIntUnmapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES;
+
+/* Bridge out structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES;
+
+/*******************************************
+            DevmemXIntMapVRangeToBackingPage
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/* Bridge out structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/*******************************************
+            PhysHeapGetMemInfo2
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2;
+
+/* Bridge out structure for PhysHeapGetMemInfo2 */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2_TAG
+{
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2;
+
+#endif /* COMMON_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/server_mm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/server_mm_bridge.c
new file mode 100644
index 000000000000..a79a92f5e9d1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mm_bridge/server_mm_bridge.c
@@ -0,0 +1,3237 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+#include "common_mm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleasePMRExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRExportPMRpsPMRExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnexportPMR((PMR_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRExportPMRIN_UI8,
+			 IMG_UINT8 * psPMRExportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMREXPORTPMR *psPMRExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMREXPORTPMR *psPMRExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PMRExportPMR(psPMRInt,
+			 &psPMRExportInt,
+			 &psPMRExportPMROUT->ui64Size,
+			 &psPMRExportPMROUT->ui32Log2Contig, &psPMRExportPMROUT->ui64Password);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRExportPMR_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &hPMRExportInt, (void *)psPMRExportInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _PMRExportPMRpsPMRExportIntRelease);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRExportPMROUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+							      &psPMRExportPMROUT->hPMRExport,
+							      (void *)psPMRExportInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      ReleasePMRExport);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRExportPMROUT->hPMRExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE) psPMRExportPMROUT->
+							     hPMRExport,
+							     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hPMRExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hPMRExportInt,
+							PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psPMRExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnexportPMR(psPMRExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRExportPMR NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgePMRUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psPMRUnexportPMRIN_UI8,
+			   IMG_UINT8 * psPMRUnexportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *psPMRUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *psPMRUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMROUT_UI8, 0);
+
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+				       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_FALSE);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hPMRExportInt,
+				     psPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT((psPMRUnexportPMROUT->eError == PVRSRV_OK) ||
+		   (psPMRUnexportPMROUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRUnexportPMR_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRUnexportPMR NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgePMRGetUID(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psPMRGetUIDIN_UI8,
+		      IMG_UINT8 * psPMRGetUIDOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRGETUID *psPMRGetUIDIN =
+	    (PVRSRV_BRIDGE_IN_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRGETUID *psPMRGetUIDOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRGetUIDIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRGetUIDOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRGetUIDOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRGetUID_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRGetUIDOUT->eError = PMRGetUID(psPMRInt, &psPMRGetUIDOUT->ui64UID);
+
+PMRGetUID_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRMakeLocalImportHandlepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnmakeLocalImportHandle((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRMakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleIN_UI8,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psPMRMakeLocalImportHandleIN->hBuffer;
+	PMR *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer,
+				       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE, IMG_TRUE);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psPMRMakeLocalImportHandleOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PMRMakeLocalImportHandlepsExtMemIntRelease);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRMakeLocalImportHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnmakeLocalImportHandle(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnmakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleIN_UI8,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRUnmakeLocalImportHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psPMRUnmakeLocalImportHandleIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_OK) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRUnmakeLocalImportHandleOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRUnmakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRUnmakeLocalImportHandle_exit:
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRImportPMRIN_UI8,
+			 IMG_UINT8 * psPMRImportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRIMPORTPMR *psPMRImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *psPMRImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMRExport = psPMRImportPMRIN->hPMRExport;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psPMRImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_TRUE);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRImportPMROUT->eError =
+	    PhysmemImportPMR(psConnection, OSGetDevNode(psConnection),
+			     psPMRExportInt,
+			     psPMRImportPMRIN->ui64uiPassword,
+			     psPMRImportPMRIN->ui64uiSize,
+			     psPMRImportPMRIN->ui32uiLog2Contig, &psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psPMRImportPMROUT->hPMR,
+							      (void *)psPMRInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _PMRImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRExportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psPMRImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRImportPMR NULL
+#endif
+
+static PVRSRV_ERROR _PMRLocalImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRLocalImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPMRLocalImportPMRIN_UI8,
+			      IMG_UINT8 * psPMRLocalImportPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *psPMRLocalImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *psPMRLocalImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hExtHandle = psPMRLocalImportPMRIN->hExtHandle;
+	PMR *psExtHandleInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRLocalImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psExtHandleInt,
+				       hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT, IMG_TRUE);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError =
+	    PMRLocalImportPMR(psExtHandleInt,
+			      &psPMRInt,
+			      &psPMRLocalImportPMROUT->uiSize, &psPMRLocalImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPMRLocalImportPMROUT->hPMR,
+								   (void *)psPMRInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PMRLocalImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRLocalImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psExtHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psPMRLocalImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnrefPMR(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRUnrefPMRIN_UI8,
+			IMG_UINT8 * psPMRUnrefPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNREFPMR *psPMRUnrefPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNREFPMR *psPMRUnrefPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRUnrefPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRUnrefPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (unlikely((psPMRUnrefPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnrefPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRUnrefPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRUnrefPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemNewRamBackedPMRpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemNewRamBackedPMR(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMRIN_UI8,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMROUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMRIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMROUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiAnnotationInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (unlikely(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemNewRamBackedPMRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemNewRamBackedPMRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemNewRamBackedPMR_exit;
+			}
+		}
+	}
+
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->pui32MappingTable,
+		     psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+	}
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength != 0)
+	{
+		uiAnnotationInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiAnnotationInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->puiAnnotation,
+		     psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+		((IMG_CHAR *)
+		 uiAnnotationInt)[(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength *
+				   sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psPhysmemNewRamBackedPMROUT->eError =
+	    PhysmemNewRamBackedPMR(psConnection, OSGetDevNode(psConnection),
+				   psPhysmemNewRamBackedPMRIN->uiSize,
+				   psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks,
+				   psPhysmemNewRamBackedPMRIN->ui32NumVirtChunks,
+				   ui32MappingTableInt,
+				   psPhysmemNewRamBackedPMRIN->ui32Log2PageSize,
+				   psPhysmemNewRamBackedPMRIN->uiFlags,
+				   psPhysmemNewRamBackedPMRIN->ui32AnnotationLength,
+				   uiAnnotationInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PID,
+				   &psPMRPtrInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PDumpFlags,
+				   &psPhysmemNewRamBackedPMROUT->uiOutFlags);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemNewRamBackedPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psPhysmemNewRamBackedPMROUT->
+									hPMRPtr,
+									(void *)psPMRPtrInt,
+									PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_PhysmemNewRamBackedPMRpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemNewRamBackedPMR_exit:
+
+	if (psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemNewRamBackedPMROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntCtxCreatepsDevMemServerContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxCreate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntCtxCreateIN_UI8,
+			       IMG_UINT8 * psDevmemIntCtxCreateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateOUT_UI8,
+								     0);
+
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntCtxCreateOUT->hDevMemServerContext = NULL;
+
+	psDevmemIntCtxCreateOUT->eError =
+	    DevmemIntCtxCreate(psConnection, OSGetDevNode(psConnection),
+			       psDevmemIntCtxCreateIN->bbKernelMemoryCtx,
+			       &psDevMemServerContextInt,
+			       &hPrivDataInt, &psDevmemIntCtxCreateOUT->ui32CPUCacheLineSize);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntCtxCreateOUT->
+								    hDevMemServerContext,
+								    (void *)
+								    psDevMemServerContextInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntCtxCreatepsDevMemServerContextIntRelease);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntCtxCreateOUT->
+								       hPrivData,
+								       (void *)hPrivDataInt,
+								       PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       psDevmemIntCtxCreateOUT->
+								       hDevMemServerContext);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxCreate_exit:
+
+	if (psDevmemIntCtxCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntCtxCreateOUT->hDevMemServerContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDevmemIntCtxCreateOUT->
+							     hDevMemServerContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psDevMemServerContextInt)
+		{
+			DevmemIntCtxDestroy(psDevMemServerContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntCtxDestroyIN_UI8,
+				IMG_UINT8 * psDevmemIntCtxDestroyOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyOUT_UI8,
+								      0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntCtxDestroyIN->
+					      hDevmemServerContext,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	if (unlikely
+	    ((psDevmemIntCtxDestroyOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntCtxDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntHeapDestroy((DEVMEMINT_HEAP *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapCreate(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntHeapCreateIN_UI8,
+				IMG_UINT8 * psDevmemIntHeapCreateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntHeapCreateIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntHeapCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				psDevmemIntHeapCreateIN->ui32HeapConfigIndex,
+				psDevmemIntHeapCreateIN->ui32HeapIndex, &psDevmemHeapPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psDevmemIntHeapCreateOUT->
+								     hDevmemHeapPtr,
+								     (void *)psDevmemHeapPtrInt,
+								     PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntHeapCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemHeapPtrInt)
+		{
+			DevmemIntHeapDestroy(psDevmemHeapPtrInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntHeapDestroyIN_UI8,
+				 IMG_UINT8 * psDevmemIntHeapDestroyOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *) IMG_OFFSET_ADDR(psDevmemIntHeapDestroyIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *)
+	    IMG_OFFSET_ADDR(psDevmemIntHeapDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntHeapDestroyIN->hDevmemHeap,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	if (unlikely((psDevmemIntHeapDestroyOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntHeapDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapDestroy_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDevmemIntMapPMRIN_UI8,
+			    IMG_UINT8 * psDevmemIntMapPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *psDevmemIntMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *psDevmemIntMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntMapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntMapPMROUT->eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+DevmemIntMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnmapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDevmemIntUnmapPMRIN_UI8,
+			      IMG_UINT8 * psDevmemIntUnmapPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntUnmapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntUnmapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntUnmapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnmapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnmapPMROUT->eError = DevmemIntUnmapPMR(psReservationInt);
+
+DevmemIntUnmapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIntReserveRangeIN_UI8,
+				  IMG_UINT8 * psDevmemIntReserveRangeOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError =
+	    DevmemIntReserveRange(psConnection, OSGetDevNode(psConnection),
+				  psDevmemServerHeapInt,
+				  psDevmemIntReserveRangeIN->sAddress,
+				  psDevmemIntReserveRangeIN->uiLength,
+				  psDevmemIntReserveRangeIN->uiFlags, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntReserveRangeOUT->
+								       hReservation,
+								       (void *)psReservationInt,
+								       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DevmemIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRangeAndMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMRIN_UI8,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMROUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeAndMapPMRIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntReserveRangeAndMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    DevmemIntReserveRangeAndMapPMR(psConnection, OSGetDevNode(psConnection),
+					   psDevmemServerHeapInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->sAddress,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiLength,
+					   psPMRInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiFlags,
+					   &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntReserveRangeAndMapPMROUT->hReservation,
+				      (void *)psReservationInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRangeAndMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeIN_UI8,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeChangeSparseMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psChangeSparseMemIN_UI8,
+			    IMG_UINT8 * psChangeSparseMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *psChangeSparseMemIN =
+	    (PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *psChangeSparseMemOUT =
+	    (PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+	IMG_HANDLE hReservation = psChangeSparseMemIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psChangeSparseMemIN->ui32AllocPageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (unlikely(psChangeSparseMemIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto ChangeSparseMem_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psChangeSparseMemIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psChangeSparseMemIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psChangeSparseMemOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto ChangeSparseMem_exit;
+			}
+		}
+	}
+
+	if (psChangeSparseMemIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32AllocPageIndices,
+		     psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+	if (psChangeSparseMemIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32FreePageIndices,
+		     psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psChangeSparseMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psChangeSparseMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto ChangeSparseMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psChangeSparseMemOUT->eError =
+	    DevmemIntChangeSparse(psChangeSparseMemIN->ui32AllocPageCount,
+				  ui32AllocPageIndicesInt,
+				  psChangeSparseMemIN->ui32FreePageCount,
+				  ui32FreePageIndicesInt,
+				  psChangeSparseMemIN->ui32SparseFlags, psReservationInt);
+
+ChangeSparseMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psChangeSparseMemOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIsVDevAddrValid(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidIN_UI8,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIsVDevAddrValidIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIsVDevAddrValidOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIsVDevAddrValidOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIsVDevAddrValid_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIsVDevAddrValidOUT->eError =
+	    DevmemIntIsVDevAddrValid(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, psDevmemIsVDevAddrValidIN->sAddress);
+
+DevmemIsVDevAddrValid_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+
+static IMG_INT
+PVRSRVBridgeDevmemInvalidateFBSCTable(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableIN_UI8,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemInvalidateFBSCTableIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemInvalidateFBSCTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemInvalidateFBSCTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    DevmemIntInvalidateFBSCTable(psDevmemCtxInt,
+					 psDevmemInvalidateFBSCTableIN->ui64FBSCEntries);
+
+DevmemInvalidateFBSCTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeDevmemInvalidateFBSCTable NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigCount(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountIN_UI8,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHeapCfgHeapConfigCountIN);
+
+	psHeapCfgHeapConfigCountOUT->eError =
+	    HeapCfgHeapConfigCount(psConnection, OSGetDevNode(psConnection),
+				   &psHeapCfgHeapConfigCountOUT->ui32NumHeapConfigs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapCount(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psHeapCfgHeapCountIN_UI8,
+			     IMG_UINT8 * psHeapCfgHeapCountOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountOUT_UI8, 0);
+
+	psHeapCfgHeapCountOUT->eError =
+	    HeapCfgHeapCount(psConnection, OSGetDevNode(psConnection),
+			     psHeapCfgHeapCountIN->ui32HeapConfigIndex,
+			     &psHeapCfgHeapCountOUT->ui32NumHeaps);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigName(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameIN_UI8,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameOUT_UI8, 0);
+
+	IMG_CHAR *puiHeapConfigNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	psHeapCfgHeapConfigNameOUT->puiHeapConfigName =
+	    psHeapCfgHeapConfigNameIN->puiHeapConfigName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapConfigNameIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapConfigNameIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapConfigName_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz != 0)
+	{
+		puiHeapConfigNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapConfigNameOUT->eError =
+	    HeapCfgHeapConfigName(psConnection, OSGetDevNode(psConnection),
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigIndex,
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz,
+				  puiHeapConfigNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapConfigNameOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapConfigNameInt) &&
+	    ((psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapConfigNameOUT->puiHeapConfigName,
+		      puiHeapConfigNameInt,
+		      (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapConfigName_exit;
+		}
+	}
+
+HeapCfgHeapConfigName_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapConfigNameOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapDetails(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psHeapCfgHeapDetailsIN_UI8,
+			       IMG_UINT8 * psHeapCfgHeapDetailsOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsOUT_UI8,
+								     0);
+
+	IMG_CHAR *puiHeapNameOutInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) + 0;
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	psHeapCfgHeapDetailsOUT->puiHeapNameOut = psHeapCfgHeapDetailsIN->puiHeapNameOut;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapDetailsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapDetailsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapDetails_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz != 0)
+	{
+		puiHeapNameOutInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapDetailsOUT->eError =
+	    HeapCfgHeapDetails(psConnection, OSGetDevNode(psConnection),
+			       psHeapCfgHeapDetailsIN->ui32HeapConfigIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapNameBufSz,
+			       puiHeapNameOutInt,
+			       &psHeapCfgHeapDetailsOUT->sDevVAddrBase,
+			       &psHeapCfgHeapDetailsOUT->uiHeapLength,
+			       &psHeapCfgHeapDetailsOUT->uiReservedRegionLength,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2DataPageSizeOut,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2ImportAlignmentOut);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapDetailsOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapNameOutInt) &&
+	    ((psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapDetailsOUT->puiHeapNameOut,
+		      puiHeapNameOutInt,
+		      (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapDetails_exit;
+		}
+	}
+
+HeapCfgHeapDetails_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapDetailsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntRegisterPFNotifyKM(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMIN_UI8,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntRegisterPFNotifyKMIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntRegisterPFNotifyKMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntRegisterPFNotifyKM_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, psDevmemIntRegisterPFNotifyKMIN->bRegister);
+
+DevmemIntRegisterPFNotifyKM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPhysHeapGetMemInfoIN_UI8,
+			       IMG_UINT8 * psPhysHeapGetMemInfoOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoIN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoOUT_UI8,
+								     0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V1 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V1)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfoIN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfoIN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfoIN->peaPhysHeapID,
+		     psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V1 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1);
+	}
+
+	psPhysHeapGetMemInfoOUT->eError =
+	    PVRSRVPhysHeapGetMemInfoKM(psConnection, OSGetDevNode(psConnection),
+				       psPhysHeapGetMemInfoIN->ui32PhysHeapCount,
+				       eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfoOUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V1)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfoIN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V1))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+
+PhysHeapGetMemInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDefaultPhysicalHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapIN_UI8,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapIN =
+	    (PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDefaultPhysicalHeapIN);
+
+	psGetDefaultPhysicalHeapOUT->eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(psConnection, OSGetDevNode(psConnection),
+					   &psGetDefaultPhysicalHeapOUT->eHeap);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemGetFaultAddress(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemGetFaultAddressIN_UI8,
+				  IMG_UINT8 * psDevmemGetFaultAddressOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemGetFaultAddressIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemGetFaultAddressOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemGetFaultAddressOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemGetFaultAddress_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemGetFaultAddressOUT->eError =
+	    DevmemIntGetFaultAddress(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, &psDevmemGetFaultAddressOUT->sFaultAddress);
+
+DevmemGetFaultAddress_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+
+static IMG_INT
+PVRSRVBridgePVRSRVStatsUpdateOOMStat(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatIN_UI8,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatOUT_UI8, 0);
+
+	psPVRSRVStatsUpdateOOMStatOUT->eError =
+	    PVRSRVStatsUpdateOOMStat(psConnection, OSGetDevNode(psConnection),
+				     psPVRSRVStatsUpdateOOMStatIN->ui32ui32StatType,
+				     psPVRSRVStatsUpdateOOMStatIN->ui32pid);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePVRSRVStatsUpdateOOMStat NULL
+#endif
+
+static PVRSRV_ERROR _DevmemXIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemXIntUnreserveRange((DEVMEMXINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDevmemXIntReserveRangeIN_UI8,
+				   IMG_UINT8 * psDevmemXIntReserveRangeOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemXIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt,
+				   psDevmemXIntReserveRangeIN->sAddress,
+				   psDevmemXIntReserveRangeIN->uiLength, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDevmemXIntReserveRangeOUT->
+									hReservation,
+									(void *)psReservationInt,
+									PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DevmemXIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemXIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeIN_UI8,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemXIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemXIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapPages(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemXIntMapPagesIN_UI8,
+			       IMG_UINT8 * psDevmemXIntMapPagesOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesOUT_UI8,
+								     0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemXIntMapPagesIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapPagesOUT->eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       psDevmemXIntMapPagesIN->ui32PageCount,
+			       psDevmemXIntMapPagesIN->ui32PhysPageOffset,
+			       psDevmemXIntMapPagesIN->uiFlags,
+			       psDevmemXIntMapPagesIN->ui32VirtPageOffset);
+
+DevmemXIntMapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnmapPages(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesIN_UI8,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntUnmapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntUnmapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntUnmapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnmapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnmapPagesOUT->eError =
+	    DevmemXIntUnmapPages(psReservationInt,
+				 psDevmemXIntUnmapPagesIN->ui32VirtPageOffset,
+				 psDevmemXIntUnmapPagesIN->ui32PageCount);
+
+DevmemXIntUnmapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapVRangeToBackingPage(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageIN_UI8,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapVRangeToBackingPageIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapVRangeToBackingPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapVRangeToBackingPage_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     psDevmemXIntMapVRangeToBackingPageIN->ui32PageCount,
+					     psDevmemXIntMapVRangeToBackingPageIN->uiFlags,
+					     psDevmemXIntMapVRangeToBackingPageIN->
+					     ui32VirtPageOffset);
+
+DevmemXIntMapVRangeToBackingPage_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo2(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysHeapGetMemInfo2IN_UI8,
+				IMG_UINT8 * psPhysHeapGetMemInfo2OUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2IN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2IN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *psPhysHeapGetMemInfo2OUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2 *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfo2OUT_UI8,
+								      0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS_V2 *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+	     sizeof(PHYS_HEAP_MEM_STATS_V2)) + 0;
+
+	if (unlikely(psPhysHeapGetMemInfo2IN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfo2IN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfo2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfo2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo2_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfo2IN->peaPhysHeapID,
+		     psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS_V2 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2);
+	}
+
+	psPhysHeapGetMemInfo2OUT->eError =
+	    PVRSRVPhysHeapGetMemInfo2KM(psConnection, OSGetDevNode(psConnection),
+					psPhysHeapGetMemInfo2IN->ui32PhysHeapCount,
+					eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfo2OUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo2_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfo2IN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS_V2)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfo2OUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfo2IN->ui32PhysHeapCount *
+		       sizeof(PHYS_HEAP_MEM_STATS_V2))) != PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfo2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo2_exit;
+		}
+	}
+
+PhysHeapGetMemInfo2_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfo2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+
+/*
+ * Register all MM functions with services
+ */
+PVRSRV_ERROR InitMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR,
+			      PVRSRVBridgePMRExportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMREXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMREXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR,
+			      PVRSRVBridgePMRUnexportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID, PVRSRVBridgePMRGetUID,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRGETUID),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRGETUID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRMakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRUnmakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR,
+			      PVRSRVBridgePMRImportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR,
+			      PVRSRVBridgePMRLocalImportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR,
+			      PVRSRVBridgePMRUnrefPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRUNREFPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNREFPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR,
+			      PVRSRVBridgePhysmemNewRamBackedPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE,
+			      PVRSRVBridgeDevmemIntCtxCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY,
+			      PVRSRVBridgeDevmemIntCtxDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE,
+			      PVRSRVBridgeDevmemIntHeapCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY,
+			      PVRSRVBridgeDevmemIntHeapDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR,
+			      PVRSRVBridgeDevmemIntMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR,
+			      PVRSRVBridgeDevmemIntUnmapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE,
+			      PVRSRVBridgeDevmemIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR,
+			      PVRSRVBridgeDevmemIntReserveRangeAndMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM,
+			      PVRSRVBridgeChangeSparseMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_CHANGESPARSEMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID,
+			      PVRSRVBridgeDevmemIsVDevAddrValid, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE,
+			      PVRSRVBridgeDevmemInvalidateFBSCTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT,
+			      PVRSRVBridgeHeapCfgHeapConfigCount, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT,
+			      PVRSRVBridgeHeapCfgHeapCount, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME,
+			      PVRSRVBridgeHeapCfgHeapConfigName, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS,
+			      PVRSRVBridgeHeapCfgHeapDetails, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM,
+			      PVRSRVBridgeDevmemIntRegisterPFNotifyKM, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO,
+			      PVRSRVBridgePhysHeapGetMemInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP,
+			      PVRSRVBridgeGetDefaultPhysicalHeap, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS,
+			      PVRSRVBridgeDevmemGetFaultAddress, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT,
+			      PVRSRVBridgePVRSRVStatsUpdateOOMStat, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES,
+			      PVRSRVBridgeDevmemXIntMapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES,
+			      PVRSRVBridgeDevmemXIntUnmapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE,
+			      PVRSRVBridgeDevmemXIntMapVRangeToBackingPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2,
+			      PVRSRVBridgePhysHeapGetMemInfo2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO2),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mm functions with services
+ */
+void DeinitMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM,
+				PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/common_mmextmem_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/common_mmextmem_bridge.h
new file mode 100644
index 000000000000..f52637853325
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/common_mmextmem_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MMEXTMEM_BRIDGE_H
+#define COMMON_MMEXTMEM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM			PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST			(PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0)
+
+/*******************************************
+            PhysmemWrapExtMem
+ *******************************************/
+
+/* Bridge in structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_UINT64 ui64CpuVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM;
+
+/* Bridge out structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM;
+
+#endif /* COMMON_MMEXTMEM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/server_mmextmem_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/server_mmextmem_bridge.c
new file mode 100644
index 000000000000..538d04cdf0e0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/mmextmem_bridge/server_mmextmem_bridge.c
@@ -0,0 +1,166 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "physmem_extmem.h"
+
+#include "common_mmextmem_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemWrapExtMempsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemWrapExtMem(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPhysmemWrapExtMemIN_UI8,
+			      IMG_UINT8 * psPhysmemWrapExtMemOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemOUT_UI8, 0);
+
+	PMR *psPMRPtrInt = NULL;
+
+	psPhysmemWrapExtMemOUT->eError =
+	    PhysmemWrapExtMem(psConnection, OSGetDevNode(psConnection),
+			      psPhysmemWrapExtMemIN->uiSize,
+			      psPhysmemWrapExtMemIN->ui64CpuVAddr,
+			      psPhysmemWrapExtMemIN->uiFlags, &psPMRPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemWrapExtMemOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPhysmemWrapExtMemOUT->hPMRPtr,
+								   (void *)psPMRPtrInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PhysmemWrapExtMempsPMRPtrIntRelease);
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemWrapExtMem_exit:
+
+	if (psPhysmemWrapExtMemOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+
+/*
+ * Register all MMEXTMEM functions with services
+ */
+PVRSRV_ERROR InitMMEXTMEMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM,
+			      PVRSRVBridgePhysmemWrapExtMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mmextmem functions with services
+ */
+void DeinitMMEXTMEMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_bridge.h
new file mode 100644
index 000000000000..2bf967bfcea9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMP_BRIDGE_H
+#define CLIENT_PDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_direct_bridge.c
new file mode 100644
index 000000000000..7c012dbcf21e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/client_pdump_direct_bridge.c
@@ -0,0 +1,151 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpImageDescriptor(IMG_HANDLE hBridge,
+						     IMG_HANDLE hDevmemCtx,
+						     IMG_UINT32 ui32StringSize,
+						     const IMG_CHAR * puiFileName,
+						     IMG_DEV_VIRTADDR sDataDevAddr,
+						     IMG_UINT32 ui32DataSize,
+						     IMG_UINT32 ui32LogicalWidth,
+						     IMG_UINT32 ui32LogicalHeight,
+						     IMG_UINT32 ui32PhysicalWidth,
+						     IMG_UINT32 ui32PhysicalHeight,
+						     PDUMP_PIXEL_FORMAT ePixelFormat,
+						     IMG_MEMLAYOUT eMemLayout,
+						     IMG_FB_COMPRESSION eFBCompression,
+						     const IMG_UINT32 * pui32FBCClearColour,
+						     PDUMP_FBC_SWIZZLE eeFBCSwizzle,
+						     IMG_DEV_VIRTADDR sHeaderDevAddr,
+						     IMG_UINT32 ui32HeaderSize,
+						     IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpImageDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  psDevmemCtxInt,
+					  ui32StringSize,
+					  puiFileName,
+					  sDataDevAddr,
+					  ui32DataSize,
+					  ui32LogicalWidth,
+					  ui32LogicalHeight,
+					  ui32PhysicalWidth,
+					  ui32PhysicalHeight,
+					  ePixelFormat,
+					  eMemLayout,
+					  eFBCompression,
+					  pui32FBCClearColour,
+					  eeFBCSwizzle,
+					  sHeaderDevAddr, ui32HeaderSize, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpComment(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32CommentSize,
+						   IMG_CHAR * puiComment, IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpCommentKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			   ui32CommentSize, puiComment, ui32Flags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetFrame(IMG_HANDLE hBridge, IMG_UINT32 ui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpSetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpDataDescriptor(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32StringSize,
+						    const IMG_CHAR * puiFileName,
+						    IMG_DEV_VIRTADDR sDataDevAddr,
+						    IMG_UINT32 ui32DataSize,
+						    IMG_UINT32 ui32HeaderType,
+						    IMG_UINT32 ui32ElementType,
+						    IMG_UINT32 ui32ElementCount,
+						    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntPDumpDataDescriptor(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 psDevmemCtxInt,
+					 ui32StringSize,
+					 puiFileName,
+					 sDataDevAddr,
+					 ui32DataSize,
+					 ui32HeaderType,
+					 ui32ElementType, ui32ElementCount, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/common_pdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/common_pdump_bridge.h
new file mode 100644
index 000000000000..eb6ab9488a20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/common_pdump_bridge.h
@@ -0,0 +1,155 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMP_BRIDGE_H
+#define COMMON_PDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_BRIDGE_PDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR			PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMP_CMD_LAST			(PVRSRV_BRIDGE_PDUMP_CMD_FIRST+3)
+
+/*******************************************
+            PDumpImageDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_DEV_VIRTADDR sHeaderDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_UINT32 *pui32FBCClearColour;
+	const IMG_CHAR *puiFileName;
+	IMG_FB_COMPRESSION eFBCompression;
+	IMG_MEMLAYOUT eMemLayout;
+	PDUMP_PIXEL_FORMAT ePixelFormat;
+	PDUMP_FBC_SWIZZLE eeFBCSwizzle;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32HeaderSize;
+	IMG_UINT32 ui32LogicalHeight;
+	IMG_UINT32 ui32LogicalWidth;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PhysicalHeight;
+	IMG_UINT32 ui32PhysicalWidth;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR;
+
+/* Bridge out structure for PDumpImageDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR;
+
+/*******************************************
+            PVRSRVPDumpComment
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT_TAG
+{
+	IMG_CHAR *puiComment;
+	IMG_UINT32 ui32CommentSize;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT;
+
+/* Bridge out structure for PVRSRVPDumpComment */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT;
+
+/*******************************************
+            PVRSRVPDumpSetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME_TAG
+{
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpSetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME;
+
+/*******************************************
+            PDumpDataDescriptor
+ *******************************************/
+
+/* Bridge in structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR_TAG
+{
+	IMG_DEV_VIRTADDR sDataDevAddr;
+	IMG_HANDLE hDevmemCtx;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32DataSize;
+	IMG_UINT32 ui32ElementCount;
+	IMG_UINT32 ui32ElementType;
+	IMG_UINT32 ui32HeaderType;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32StringSize;
+} __packed PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR;
+
+/* Bridge out structure for PDumpDataDescriptor */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR;
+
+#endif /* COMMON_PDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/server_pdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/server_pdump_bridge.c
new file mode 100644
index 000000000000..6b3baf668710
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdump_bridge/server_pdump_bridge.c
@@ -0,0 +1,571 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pdump_km.h"
+
+#include "common_pdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(4 <= IMG_UINT32_MAX, "4 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpImageDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpImageDescriptorIN_UI8,
+				 IMG_UINT8 * psPDumpImageDescriptorOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpImageDescriptorIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *psPDumpImageDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR *)
+	    IMG_OFFSET_ADDR(psPDumpImageDescriptorOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpImageDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+	IMG_UINT32 *ui32FBCClearColourInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) 4 * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psPDumpImageDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpImageDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpImageDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpImageDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpImageDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpImageDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psPDumpImageDescriptorIN->puiFileName,
+		     psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpImageDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	{
+		ui32FBCClearColourInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 4 * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (4 * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FBCClearColourInt,
+		     (const void __user *)psPDumpImageDescriptorIN->pui32FBCClearColour,
+		     4 * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psPDumpImageDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpImageDescriptor_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpImageDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpImageDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpImageDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpImageDescriptorOUT->eError =
+	    DevmemIntPDumpImageDescriptor(psConnection, OSGetDevNode(psConnection),
+					  psDevmemCtxInt,
+					  psPDumpImageDescriptorIN->ui32StringSize,
+					  uiFileNameInt,
+					  psPDumpImageDescriptorIN->sDataDevAddr,
+					  psPDumpImageDescriptorIN->ui32DataSize,
+					  psPDumpImageDescriptorIN->ui32LogicalWidth,
+					  psPDumpImageDescriptorIN->ui32LogicalHeight,
+					  psPDumpImageDescriptorIN->ui32PhysicalWidth,
+					  psPDumpImageDescriptorIN->ui32PhysicalHeight,
+					  psPDumpImageDescriptorIN->ePixelFormat,
+					  psPDumpImageDescriptorIN->eMemLayout,
+					  psPDumpImageDescriptorIN->eFBCompression,
+					  ui32FBCClearColourInt,
+					  psPDumpImageDescriptorIN->eeFBCSwizzle,
+					  psPDumpImageDescriptorIN->sHeaderDevAddr,
+					  psPDumpImageDescriptorIN->ui32HeaderSize,
+					  psPDumpImageDescriptorIN->ui32PDumpFlags);
+
+PDumpImageDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpImageDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_COMMENT_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_COMMENT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpComment(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPVRSRVPDumpCommentIN_UI8,
+			       IMG_UINT8 * psPVRSRVPDumpCommentOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *psPVRSRVPDumpCommentOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT *) IMG_OFFSET_ADDR(psPVRSRVPDumpCommentOUT_UI8,
+								     0);
+
+	IMG_CHAR *uiCommentInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPVRSRVPDumpCommentIN->ui32CommentSize > PVRSRV_PDUMP_MAX_COMMENT_SIZE))
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PVRSRVPDumpComment_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPVRSRVPDumpCommentIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPVRSRVPDumpCommentIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PVRSRVPDumpComment_exit;
+			}
+		}
+	}
+
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize != 0)
+	{
+		uiCommentInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiCommentInt, (const void __user *)psPVRSRVPDumpCommentIN->puiComment,
+		     psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPVRSRVPDumpCommentOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PVRSRVPDumpComment_exit;
+		}
+		((IMG_CHAR *)
+		 uiCommentInt)[(psPVRSRVPDumpCommentIN->ui32CommentSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPVRSRVPDumpCommentOUT->eError =
+	    PDumpCommentKM(psConnection, OSGetDevNode(psConnection),
+			   psPVRSRVPDumpCommentIN->ui32CommentSize,
+			   uiCommentInt, psPVRSRVPDumpCommentIN->ui32Flags);
+
+PVRSRVPDumpComment_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPVRSRVPDumpCommentOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpSetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *psPVRSRVPDumpSetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpSetFrameOUT_UI8,
+								      0);
+
+	psPVRSRVPDumpSetFrameOUT->eError =
+	    PDumpSetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    psPVRSRVPDumpSetFrameIN->ui32Frame);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePDumpDataDescriptor(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPDumpDataDescriptorIN_UI8,
+				IMG_UINT8 * psPDumpDataDescriptorOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *psPDumpDataDescriptorOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR *) IMG_OFFSET_ADDR(psPDumpDataDescriptorOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psPDumpDataDescriptorIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPDumpDataDescriptorIN->ui32StringSize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PDumpDataDescriptor_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPDumpDataDescriptorIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPDumpDataDescriptorIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PDumpDataDescriptor_exit;
+			}
+		}
+	}
+
+	if (psPDumpDataDescriptorIN->ui32StringSize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPDumpDataDescriptorIN->puiFileName,
+		     psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPDumpDataDescriptorOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PDumpDataDescriptor_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPDumpDataDescriptorIN->ui32StringSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPDumpDataDescriptorOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psPDumpDataDescriptorOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PDumpDataDescriptor_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPDumpDataDescriptorOUT->eError =
+	    DevmemIntPDumpDataDescriptor(psConnection, OSGetDevNode(psConnection),
+					 psDevmemCtxInt,
+					 psPDumpDataDescriptorIN->ui32StringSize,
+					 uiFileNameInt,
+					 psPDumpDataDescriptorIN->sDataDevAddr,
+					 psPDumpDataDescriptorIN->ui32DataSize,
+					 psPDumpDataDescriptorIN->ui32HeaderType,
+					 psPDumpDataDescriptorIN->ui32ElementType,
+					 psPDumpDataDescriptorIN->ui32ElementCount,
+					 psPDumpDataDescriptorIN->ui32PDumpFlags);
+
+PDumpDataDescriptor_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPDumpDataDescriptorOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPBridge(void);
+void DeinitPDUMPBridge(void);
+
+/*
+ * Register all PDUMP functions with services
+ */
+PVRSRV_ERROR InitPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR,
+			      PVRSRVBridgePDumpImageDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPIMAGEDESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPIMAGEDESCRIPTOR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT,
+			      PVRSRVBridgePVRSRVPDumpComment, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPCOMMENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPCOMMENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME,
+			      PVRSRVBridgePVRSRVPDumpSetFrame, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETFRAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR,
+			      PVRSRVBridgePDumpDataDescriptor, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPDATADESCRIPTOR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPDATADESCRIPTOR));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdump functions with services
+ */
+void DeinitPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPIMAGEDESCRIPTOR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPCOMMENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PVRSRVPDUMPSETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMP, PVRSRV_BRIDGE_PDUMP_PDUMPDATADESCRIPTOR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..a2ba27b88b30
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_bridge.h
@@ -0,0 +1,74 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPCTRL_BRIDGE_H
+#define CLIENT_PDUMPCTRL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpctrl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge);
+
+#endif /* CLIENT_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
new file mode 100644
index 000000000000..a2bc30360aea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.c
@@ -0,0 +1,109 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpctrl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpctrl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "pdump_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetState(IMG_HANDLE hBridge, IMG_UINT64 * pui64State)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpGetStateKM(pui64State);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpGetFrame(IMG_HANDLE hBridge, IMG_UINT32 * pui32Frame)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpGetFrameKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), pui32Frame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpSetDefaultCaptureParams(IMG_HANDLE hBridge,
+								   IMG_UINT32 ui32Mode,
+								   IMG_UINT32 ui32Start,
+								   IMG_UINT32 ui32End,
+								   IMG_UINT32 ui32Interval,
+								   IMG_UINT32 ui32MaxParamFileSize,
+								   IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PDumpSetDefaultCaptureParamsKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32Mode,
+					   ui32Start,
+					   ui32End,
+					   ui32Interval, ui32MaxParamFileSize, ui32AutoTermTimeout);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpIsLastCaptureFrame(IMG_HANDLE hBridge,
+							      IMG_BOOL * pbpbIsLastCaptureFrame)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PDumpIsLastCaptureFrameKM(pbpbIsLastCaptureFrame);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVPDumpForceCaptureStop(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PDumpForceCaptureStopKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge));
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/common_pdumpctrl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/common_pdumpctrl_bridge.h
new file mode 100644
index 000000000000..f710e4c06131
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/common_pdumpctrl_bridge.h
@@ -0,0 +1,150 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPCTRL_BRIDGE_H
+#define COMMON_PDUMPCTRL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP			PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPCTRL_CMD_LAST			(PVRSRV_BRIDGE_PDUMPCTRL_CMD_FIRST+4)
+
+/*******************************************
+            PVRSRVPDumpGetState
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE;
+
+/* Bridge out structure for PVRSRVPDumpGetState */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE_TAG
+{
+	IMG_UINT64 ui64State;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE;
+
+/*******************************************
+            PVRSRVPDumpGetFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME;
+
+/* Bridge out structure for PVRSRVPDumpGetFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Frame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME;
+
+/*******************************************
+            PVRSRVPDumpSetDefaultCaptureParams
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	IMG_UINT32 ui32AutoTermTimeout;
+	IMG_UINT32 ui32End;
+	IMG_UINT32 ui32Interval;
+	IMG_UINT32 ui32MaxParamFileSize;
+	IMG_UINT32 ui32Mode;
+	IMG_UINT32 ui32Start;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/* Bridge out structure for PVRSRVPDumpSetDefaultCaptureParams */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS;
+
+/*******************************************
+            PVRSRVPDumpIsLastCaptureFrame
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/* Bridge out structure for PVRSRVPDumpIsLastCaptureFrame */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bpbIsLastCaptureFrame;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME;
+
+/*******************************************
+            PVRSRVPDumpForceCaptureStop
+ *******************************************/
+
+/* Bridge in structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP;
+
+/* Bridge out structure for PVRSRVPDumpForceCaptureStop */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP;
+
+#endif /* COMMON_PDUMPCTRL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/server_pdumpctrl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/server_pdumpctrl_bridge.c
new file mode 100644
index 000000000000..2d5578cf83f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpctrl_bridge/server_pdumpctrl_bridge.c
@@ -0,0 +1,252 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpctrl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpctrl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pdump_km.h"
+
+#include "common_pdumpctrl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetState(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetStateIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetStateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *psPVRSRVPDumpGetStateOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetStateOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetStateIN);
+
+	psPVRSRVPDumpGetStateOUT->eError = PDumpGetStateKM(&psPVRSRVPDumpGetStateOUT->ui64State);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpGetFrame(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameIN_UI8,
+				IMG_UINT8 * psPVRSRVPDumpGetFrameOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *psPVRSRVPDumpGetFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME *) IMG_OFFSET_ADDR(psPVRSRVPDumpGetFrameOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpGetFrameIN);
+
+	psPVRSRVPDumpGetFrameOUT->eError =
+	    PDumpGetFrameKM(psConnection, OSGetDevNode(psConnection),
+			    &psPVRSRVPDumpGetFrameOUT->ui32Frame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams(IMG_UINT32 ui32DispatchTableEntry,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8,
+					       IMG_UINT8 *
+					       psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8,
+					       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *psPVRSRVPDumpSetDefaultCaptureParamsIN
+	    =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS
+	    *psPVRSRVPDumpSetDefaultCaptureParamsOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpSetDefaultCaptureParamsOUT_UI8, 0);
+
+	psPVRSRVPDumpSetDefaultCaptureParamsOUT->eError =
+	    PDumpSetDefaultCaptureParamsKM(psConnection, OSGetDevNode(psConnection),
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Mode,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Start,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32End,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->ui32Interval,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32MaxParamFileSize,
+					   psPVRSRVPDumpSetDefaultCaptureParamsIN->
+					   ui32AutoTermTimeout);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameIN_UI8,
+					  IMG_UINT8 * psPVRSRVPDumpIsLastCaptureFrameOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *psPVRSRVPDumpIsLastCaptureFrameOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpIsLastCaptureFrameOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpIsLastCaptureFrameIN);
+
+	psPVRSRVPDumpIsLastCaptureFrameOUT->eError =
+	    PDumpIsLastCaptureFrameKM(&psPVRSRVPDumpIsLastCaptureFrameOUT->bpbIsLastCaptureFrame);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePVRSRVPDumpForceCaptureStop(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopIN_UI8,
+					IMG_UINT8 * psPVRSRVPDumpForceCaptureStopOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *psPVRSRVPDumpForceCaptureStopOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP *)
+	    IMG_OFFSET_ADDR(psPVRSRVPDumpForceCaptureStopOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psPVRSRVPDumpForceCaptureStopIN);
+
+	psPVRSRVPDumpForceCaptureStopOUT->eError =
+	    PDumpForceCaptureStopKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pPDUMPCTRLBridgeLock;
+
+PVRSRV_ERROR InitPDUMPCTRLBridge(void);
+void DeinitPDUMPCTRLBridge(void);
+
+/*
+ * Register all PDUMPCTRL functions with services
+ */
+PVRSRV_ERROR InitPDUMPCTRLBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pPDUMPCTRLBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE,
+			      PVRSRVBridgePVRSRVPDumpGetState, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETSTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL, PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME,
+			      PVRSRVBridgePVRSRVPDumpGetFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPGETFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS,
+			      PVRSRVBridgePVRSRVPDumpSetDefaultCaptureParams, pPDUMPCTRLBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME,
+			      PVRSRVBridgePVRSRVPDumpIsLastCaptureFrame, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPISLASTCAPTUREFRAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+			      PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP,
+			      PVRSRVBridgePVRSRVPDumpForceCaptureStop, pPDUMPCTRLBridgeLock, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVPDUMPFORCECAPTURESTOP));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpctrl functions with services
+ */
+void DeinitPDUMPCTRLBridge(void)
+{
+	OSLockDestroy(pPDUMPCTRLBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETSTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPGETFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPSETDEFAULTCAPTUREPARAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPISLASTCAPTUREFRAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPCTRL,
+				PVRSRV_BRIDGE_PDUMPCTRL_PVRSRVPDUMPFORCECAPTURESTOP);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_bridge.h
new file mode 100644
index 000000000000..a4368477e225
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_bridge.h
@@ -0,0 +1,125 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PDUMPMM_BRIDGE_H
+#define CLIENT_PDUMPMM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pdumpmm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
new file mode 100644
index 000000000000..1358f75a7537
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/client_pdumpmm_direct_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pdumpmm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pdumpmm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMem(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT32 ui32PDumpFlags, IMG_BOOL bbZero)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMem(psPMRInt, uiOffset, uiSize, ui32PDumpFlags, bbZero);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue32(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT32 ui32Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue32(psPMRInt, uiOffset, ui32Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpLoadMemValue64(IMG_HANDLE hBridge,
+						       IMG_HANDLE hPMR,
+						       IMG_DEVMEM_OFFSET_T uiOffset,
+						       IMG_UINT64 ui64Value,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpLoadMemValue64(psPMRInt, uiOffset, ui64Value, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSaveToFile(IMG_HANDLE hBridge,
+						   IMG_HANDLE hPMR,
+						   IMG_DEVMEM_OFFSET_T uiOffset,
+						   IMG_DEVMEM_SIZE_T uiSize,
+						   IMG_UINT32 ui32ArraySize,
+						   const IMG_CHAR * puiFileName,
+						   IMG_UINT32 ui32uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       uiOffset, uiSize, ui32ArraySize, puiFileName, ui32uiFileOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpSymbolicAddr(IMG_HANDLE hBridge,
+						     IMG_HANDLE hPMR,
+						     IMG_DEVMEM_OFFSET_T uiOffset,
+						     IMG_UINT32 ui32MemspaceNameLen,
+						     IMG_CHAR * puiMemspaceName,
+						     IMG_UINT32 ui32SymbolicAddrLen,
+						     IMG_CHAR * puiSymbolicAddr,
+						     IMG_DEVMEM_OFFSET_T * puiNewOffset,
+						     IMG_DEVMEM_OFFSET_T * puiNextSymName)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  uiOffset,
+				  ui32MemspaceNameLen,
+				  puiMemspaceName,
+				  ui32SymbolicAddrLen,
+				  puiSymbolicAddr, puiNewOffset, puiNextSymName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpPol32(IMG_HANDLE hBridge,
+					      IMG_HANDLE hPMR,
+					      IMG_DEVMEM_OFFSET_T uiOffset,
+					      IMG_UINT32 ui32Value,
+					      IMG_UINT32 ui32Mask,
+					      PDUMP_POLL_OPERATOR eOperator,
+					      IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpPol32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCheck32(IMG_HANDLE hBridge,
+						IMG_HANDLE hPMR,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_UINT32 ui32Value,
+						IMG_UINT32 ui32Mask,
+						PDUMP_POLL_OPERATOR eOperator,
+						IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    PMRPDumpCheck32(psPMRInt, uiOffset, ui32Value, ui32Mask, eOperator, ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRPDumpCBP(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_DEVMEM_OFFSET_T uiReadOffset,
+					    IMG_DEVMEM_OFFSET_T uiWriteOffset,
+					    IMG_DEVMEM_SIZE_T uiPacketSize,
+					    IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRPDumpCBP(psPMRInt, uiReadOffset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntPDumpSaveToFileVirtual(IMG_HANDLE hBridge,
+								IMG_HANDLE hDevmemServerContext,
+								IMG_DEV_VIRTADDR sAddress,
+								IMG_DEVMEM_SIZE_T uiSize,
+								IMG_UINT32 ui32ArraySize,
+								const IMG_CHAR * puiFileName,
+								IMG_UINT32 ui32FileOffset,
+								IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError =
+	    DevmemIntPDumpSaveToFileVirtual(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    psDevmemServerContextInt,
+					    sAddress,
+					    uiSize,
+					    ui32ArraySize,
+					    puiFileName, ui32FileOffset, ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/common_pdumpmm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/common_pdumpmm_bridge.h
new file mode 100644
index 000000000000..2020cbf5b30b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/common_pdumpmm_bridge.h
@@ -0,0 +1,259 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PDUMPMM_BRIDGE_H
+#define COMMON_PDUMPMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL			PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_PDUMPMM_CMD_LAST			(PVRSRV_BRIDGE_PDUMPMM_CMD_FIRST+8)
+
+/*******************************************
+            PMRPDumpLoadMem
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_BOOL bbZero;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM;
+
+/* Bridge out structure for PMRPDumpLoadMem */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM;
+
+/*******************************************
+            PMRPDumpLoadMemValue32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32;
+
+/* Bridge out structure for PMRPDumpLoadMemValue32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32;
+
+/*******************************************
+            PMRPDumpLoadMemValue64
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	IMG_UINT64 ui64Value;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64;
+
+/* Bridge out structure for PMRPDumpLoadMemValue64 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64;
+
+/*******************************************
+            PMRPDumpSaveToFile
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32uiFileOffset;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE;
+
+/* Bridge out structure for PMRPDumpSaveToFile */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE;
+
+/*******************************************
+            PMRPDumpSymbolicAddr
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	IMG_UINT32 ui32MemspaceNameLen;
+	IMG_UINT32 ui32SymbolicAddrLen;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR;
+
+/* Bridge out structure for PMRPDumpSymbolicAddr */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiNewOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_CHAR *puiMemspaceName;
+	IMG_CHAR *puiSymbolicAddr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR;
+
+/*******************************************
+            PMRPDumpPol32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPPOL32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPPOL32;
+
+/* Bridge out structure for PMRPDumpPol32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32;
+
+/*******************************************
+            PMRPDumpCheck32
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32_TAG
+{
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_HANDLE hPMR;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32;
+
+/* Bridge out structure for PMRPDumpCheck32 */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32;
+
+/*******************************************
+            PMRPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_PMRPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiReadOffset;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRPDUMPCBP;
+
+/* Bridge out structure for PMRPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_PMRPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRPDUMPCBP;
+
+/*******************************************
+            DevmemIntPDumpSaveToFileVirtual
+ *******************************************/
+
+/* Bridge in structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hDevmemServerContext;
+	const IMG_CHAR *puiFileName;
+	IMG_UINT32 ui32ArraySize;
+	IMG_UINT32 ui32FileOffset;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+/* Bridge out structure for DevmemIntPDumpSaveToFileVirtual */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL;
+
+#endif /* COMMON_PDUMPMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/server_pdumpmm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/server_pdumpmm_bridge.c
new file mode 100644
index 000000000000..38e1a6610fd5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pdumpmm_bridge/server_pdumpmm_bridge.c
@@ -0,0 +1,958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pdumpmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pdumpmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdump_physmem.h"
+
+#include "common_pdumpmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpLoadMemIN_UI8,
+			    IMG_UINT8 * psPMRPDumpLoadMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *psPMRPDumpLoadMemIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *psPMRPDumpLoadMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM *) IMG_OFFSET_ADDR(psPMRPDumpLoadMemOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemOUT->eError =
+	    PMRPDumpLoadMem(psPMRInt,
+			    psPMRPDumpLoadMemIN->uiOffset,
+			    psPMRPDumpLoadMemIN->uiSize,
+			    psPMRPDumpLoadMemIN->ui32PDumpFlags, psPMRPDumpLoadMemIN->bbZero);
+
+PMRPDumpLoadMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue32(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue32OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *psPMRPDumpLoadMemValue32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue32OUT->eError =
+	    PMRPDumpLoadMemValue32(psPMRInt,
+				   psPMRPDumpLoadMemValue32IN->uiOffset,
+				   psPMRPDumpLoadMemValue32IN->ui32Value,
+				   psPMRPDumpLoadMemValue32IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpLoadMemValue64(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64IN_UI8,
+				   IMG_UINT8 * psPMRPDumpLoadMemValue64OUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *psPMRPDumpLoadMemValue64OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64 *)
+	    IMG_OFFSET_ADDR(psPMRPDumpLoadMemValue64OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpLoadMemValue64IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpLoadMemValue64OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpLoadMemValue64_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpLoadMemValue64OUT->eError =
+	    PMRPDumpLoadMemValue64(psPMRInt,
+				   psPMRPDumpLoadMemValue64IN->uiOffset,
+				   psPMRPDumpLoadMemValue64IN->ui64Value,
+				   psPMRPDumpLoadMemValue64IN->ui32PDumpFlags);
+
+PMRPDumpLoadMemValue64_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSaveToFile(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRPDumpSaveToFileIN_UI8,
+			       IMG_UINT8 * psPMRPDumpSaveToFileOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *psPMRPDumpSaveToFileOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE *) IMG_OFFSET_ADDR(psPMRPDumpSaveToFileOUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSaveToFileIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPMRPDumpSaveToFileIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSaveToFile_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSaveToFileIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSaveToFileIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSaveToFile_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt, (const void __user *)psPMRPDumpSaveToFileIN->puiFileName,
+		     psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPMRPDumpSaveToFileOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSaveToFile_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psPMRPDumpSaveToFileIN->ui32ArraySize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSaveToFileOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSaveToFileOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSaveToFile_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSaveToFileOUT->eError =
+	    PMRPDumpSaveToFile(psPMRInt,
+			       psPMRPDumpSaveToFileIN->uiOffset,
+			       psPMRPDumpSaveToFileIN->uiSize,
+			       psPMRPDumpSaveToFileIN->ui32ArraySize,
+			       uiFileNameInt, psPMRPDumpSaveToFileIN->ui32uiFileOffset);
+
+PMRPDumpSaveToFile_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSaveToFileOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePMRPDumpSymbolicAddr(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrIN_UI8,
+				 IMG_UINT8 * psPMRPDumpSymbolicAddrOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR *) IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *psPMRPDumpSymbolicAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR *)
+	    IMG_OFFSET_ADDR(psPMRPDumpSymbolicAddrOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpSymbolicAddrIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *puiMemspaceNameInt = NULL;
+	IMG_CHAR *puiSymbolicAddrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) + 0;
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen > PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen > PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	psPMRPDumpSymbolicAddrOUT->puiMemspaceName = psPMRPDumpSymbolicAddrIN->puiMemspaceName;
+	psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr = psPMRPDumpSymbolicAddrIN->puiSymbolicAddr;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPMRPDumpSymbolicAddrIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPMRPDumpSymbolicAddrIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PMRPDumpSymbolicAddr_exit;
+			}
+		}
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen != 0)
+	{
+		puiMemspaceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR);
+	}
+
+	if (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen != 0)
+	{
+		puiSymbolicAddrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpSymbolicAddrOUT->eError =
+	    PMR_PDumpSymbolicAddr(psPMRInt,
+				  psPMRPDumpSymbolicAddrIN->uiOffset,
+				  psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen,
+				  puiMemspaceNameInt,
+				  psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen,
+				  puiSymbolicAddrInt,
+				  &psPMRPDumpSymbolicAddrOUT->uiNewOffset,
+				  &psPMRPDumpSymbolicAddrOUT->uiNextSymName);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRPDumpSymbolicAddrOUT->eError != PVRSRV_OK))
+	{
+		goto PMRPDumpSymbolicAddr_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiMemspaceNameInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiMemspaceName,
+		      puiMemspaceNameInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32MemspaceNameLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiSymbolicAddrInt) &&
+	    ((psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPMRPDumpSymbolicAddrOUT->puiSymbolicAddr,
+		      puiSymbolicAddrInt,
+		      (psPMRPDumpSymbolicAddrIN->ui32SymbolicAddrLen * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psPMRPDumpSymbolicAddrOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PMRPDumpSymbolicAddr_exit;
+		}
+	}
+
+PMRPDumpSymbolicAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPMRPDumpSymbolicAddrOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpPol32(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psPMRPDumpPol32IN_UI8,
+			  IMG_UINT8 * psPMRPDumpPol32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *psPMRPDumpPol32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *psPMRPDumpPol32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32 *) IMG_OFFSET_ADDR(psPMRPDumpPol32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpPol32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpPol32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpPol32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpPol32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpPol32OUT->eError =
+	    PMRPDumpPol32(psPMRInt,
+			  psPMRPDumpPol32IN->uiOffset,
+			  psPMRPDumpPol32IN->ui32Value,
+			  psPMRPDumpPol32IN->ui32Mask,
+			  psPMRPDumpPol32IN->eOperator, psPMRPDumpPol32IN->ui32PDumpFlags);
+
+PMRPDumpPol32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCheck32(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psPMRPDumpCheck32IN_UI8,
+			    IMG_UINT8 * psPMRPDumpCheck32OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *psPMRPDumpCheck32IN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *psPMRPDumpCheck32OUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32 *) IMG_OFFSET_ADDR(psPMRPDumpCheck32OUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCheck32IN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCheck32OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCheck32OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCheck32_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCheck32OUT->eError =
+	    PMRPDumpCheck32(psPMRInt,
+			    psPMRPDumpCheck32IN->uiOffset,
+			    psPMRPDumpCheck32IN->ui32Value,
+			    psPMRPDumpCheck32IN->ui32Mask,
+			    psPMRPDumpCheck32IN->eOperator, psPMRPDumpCheck32IN->ui32PDumpFlags);
+
+PMRPDumpCheck32_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRPDumpCBPIN_UI8,
+			IMG_UINT8 * psPMRPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRPDUMPCBP *psPMRPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *psPMRPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRPDUMPCBP *) IMG_OFFSET_ADDR(psPMRPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRPDumpCBPIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRPDumpCBPOUT->eError =
+	    PMRPDumpCBP(psPMRInt,
+			psPMRPDumpCBPIN->uiReadOffset,
+			psPMRPDumpCBPIN->uiWriteOffset,
+			psPMRPDumpCBPIN->uiPacketSize, psPMRPDumpCBPIN->uiBufferSize);
+
+PMRPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PDUMP_MAX_FILENAME_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRV_PDUMP_MAX_FILENAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualIN_UI8,
+					    IMG_UINT8 * psDevmemIntPDumpSaveToFileVirtualOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *psDevmemIntPDumpSaveToFileVirtualOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL *)
+	    IMG_OFFSET_ADDR(psDevmemIntPDumpSaveToFileVirtualOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerContext = psDevmemIntPDumpSaveToFileVirtualIN->hDevmemServerContext;
+	DEVMEMINT_CTX *psDevmemServerContextInt = NULL;
+	IMG_CHAR *uiFileNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (unlikely
+	    (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize > PVRSRV_PDUMP_MAX_FILENAME_SIZE))
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevmemIntPDumpSaveToFileVirtualIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevmemIntPDumpSaveToFileVirtualIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevmemIntPDumpSaveToFileVirtual_exit;
+			}
+		}
+	}
+
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize != 0)
+	{
+		uiFileNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileNameInt,
+		     (const void __user *)psDevmemIntPDumpSaveToFileVirtualIN->puiFileName,
+		     psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psDevmemIntPDumpSaveToFileVirtualOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevmemIntPDumpSaveToFileVirtual_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileNameInt)[(psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize *
+				 sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerContextInt,
+				       hDevmemServerContext,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntPDumpSaveToFileVirtualOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntPDumpSaveToFileVirtual_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntPDumpSaveToFileVirtualOUT->eError =
+	    DevmemIntPDumpSaveToFileVirtual(psConnection, OSGetDevNode(psConnection),
+					    psDevmemServerContextInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->sAddress,
+					    psDevmemIntPDumpSaveToFileVirtualIN->uiSize,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32ArraySize,
+					    uiFileNameInt,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32FileOffset,
+					    psDevmemIntPDumpSaveToFileVirtualIN->ui32PDumpFlags);
+
+DevmemIntPDumpSaveToFileVirtual_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevmemIntPDumpSaveToFileVirtualOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPDUMPMMBridge(void);
+void DeinitPDUMPMMBridge(void);
+
+/*
+ * Register all PDUMPMM functions with services
+ */
+PVRSRV_ERROR InitPDUMPMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM,
+			      PVRSRVBridgePMRPDumpLoadMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32,
+			      PVRSRVBridgePMRPDumpLoadMemValue32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64,
+			      PVRSRVBridgePMRPDumpLoadMemValue64, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPLOADMEMVALUE64),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPLOADMEMVALUE64));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE,
+			      PVRSRVBridgePMRPDumpSaveToFile, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSAVETOFILE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSAVETOFILE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR,
+			      PVRSRVBridgePMRPDumpSymbolicAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPSYMBOLICADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPSYMBOLICADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32,
+			      PVRSRVBridgePMRPDumpPol32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPPOL32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPPOL32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32,
+			      PVRSRVBridgePMRPDumpCheck32, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCHECK32),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCHECK32));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP,
+			      PVRSRVBridgePMRPDumpCBP, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+			      PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL,
+			      PVRSRVBridgeDevmemIntPDumpSaveToFileVirtual, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTPDUMPSAVETOFILEVIRTUAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTPDUMPSAVETOFILEVIRTUAL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pdumpmm functions with services
+ */
+void DeinitPDUMPMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPLOADMEMVALUE64);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSAVETOFILE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPSYMBOLICADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPPOL32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCHECK32);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM, PVRSRV_BRIDGE_PDUMPMM_PMRPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PDUMPMM,
+				PVRSRV_BRIDGE_PDUMPMM_DEVMEMINTPDUMPSAVETOFILEVIRTUAL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_bridge.h
new file mode 100644
index 000000000000..2cfafd5a83e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PVRTL_BRIDGE_H
+#define CLIENT_PVRTL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pvrtl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams,
+						  IMG_UINT32 * pui32NumFound);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin,
+						IMG_UINT32 * pui32Available);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data);
+
+#endif /* CLIENT_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_direct_bridge.c
new file mode 100644
index 000000000000..fa2fbed9595b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/client_pvrtl_direct_bridge.c
@@ -0,0 +1,175 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pvrtl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pvrtl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#include "tlserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerOpenStreamKM(puiName, ui32Mode, &psSDInt, &psTLPMRInt);
+
+	*phSD = psSDInt;
+	*phTLPMR = psTLPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCloseStreamKM(psSDInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerAcquireDataKM(psSDInt, pui32ReadOffset, pui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerReleaseDataKM(psSDInt, ui32ReadOffset, ui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams, IMG_UINT32 * pui32NumFound)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerDiscoverStreamsKM(puiNamePattern, ui32Size, puiStreams, pui32NumFound);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin, IMG_UINT32 * pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    pui32BufferOffset, ui32Size, ui32SizeMin, pui32Available);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCommitStreamKM(psSDInt, ui32ReqSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerWriteDataKM(psSDInt, ui32Size, pui8Data);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/common_pvrtl_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/common_pvrtl_bridge.h
new file mode 100644
index 000000000000..edc822375222
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/common_pvrtl_bridge.h
@@ -0,0 +1,214 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PVRTL_BRIDGE_H
+#define COMMON_PVRTL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#define PVRSRV_BRIDGE_PVRTL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PVRTL_TLWRITEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PVRTL_CMD_LAST			(PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7)
+
+/*******************************************
+            TLOpenStream
+ *******************************************/
+
+/* Bridge in structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_IN_TLOPENSTREAM_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_UINT32 ui32Mode;
+} __packed PVRSRV_BRIDGE_IN_TLOPENSTREAM;
+
+/* Bridge out structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLOPENSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_HANDLE hTLPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLOPENSTREAM;
+
+/*******************************************
+            TLCloseStream
+ *******************************************/
+
+/* Bridge in structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCLOSESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLCLOSESTREAM;
+
+/* Bridge out structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCLOSESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCLOSESTREAM;
+
+/*******************************************
+            TLAcquireData
+ *******************************************/
+
+/* Bridge in structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_IN_TLACQUIREDATA_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLACQUIREDATA;
+
+/* Bridge out structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_OUT_TLACQUIREDATA_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLACQUIREDATA;
+
+/*******************************************
+            TLReleaseData
+ *******************************************/
+
+/* Bridge in structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_IN_TLRELEASEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_IN_TLRELEASEDATA;
+
+/* Bridge out structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_OUT_TLRELEASEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLRELEASEDATA;
+
+/*******************************************
+            TLDiscoverStreams
+ *******************************************/
+
+/* Bridge in structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS_TAG
+{
+	const IMG_CHAR *puiNamePattern;
+	IMG_CHAR *puiStreams;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS;
+
+/* Bridge out structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS_TAG
+{
+	IMG_CHAR *puiStreams;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumFound;
+} __packed PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS;
+
+/*******************************************
+            TLReserveStream
+ *******************************************/
+
+/* Bridge in structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_IN_TLRESERVESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32SizeMin;
+} __packed PVRSRV_BRIDGE_IN_TLRESERVESTREAM;
+
+/* Bridge out structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLRESERVESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Available;
+	IMG_UINT32 ui32BufferOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLRESERVESTREAM;
+
+/*******************************************
+            TLCommitStream
+ *******************************************/
+
+/* Bridge in structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCOMMITSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReqSize;
+} __packed PVRSRV_BRIDGE_IN_TLCOMMITSTREAM;
+
+/* Bridge out structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM;
+
+/*******************************************
+            TLWriteData
+ *******************************************/
+
+/* Bridge in structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_IN_TLWRITEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLWRITEDATA;
+
+/* Bridge out structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_OUT_TLWRITEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLWRITEDATA;
+
+#endif /* COMMON_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/server_pvrtl_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/server_pvrtl_bridge.c
new file mode 100644
index 000000000000..9facf794011e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/pvrtl_bridge/server_pvrtl_bridge.c
@@ -0,0 +1,827 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "tlserver.h"
+
+#include "common_pvrtl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _TLOpenStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = TLServerCloseStreamKM((TL_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLOpenStream(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psTLOpenStreamIN_UI8,
+			 IMG_UINT8 * psTLOpenStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLOPENSTREAM *psTLOpenStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLOPENSTREAM *psTLOpenStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamOUT_UI8, 0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	psTLOpenStreamOUT->hSD = NULL;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLOpenStreamOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLOpenStream_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLOpenStreamIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLOpenStreamIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLOpenStreamOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLOpenStream_exit;
+			}
+		}
+	}
+
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psTLOpenStreamIN->puiName,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLOpenStreamOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLOpenStream_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) - 1] =
+		    '\0';
+	}
+
+	psTLOpenStreamOUT->eError =
+	    TLServerOpenStreamKM(uiNameInt, psTLOpenStreamIN->ui32Mode, &psSDInt, &psTLPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		goto TLOpenStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psTLOpenStreamOUT->hSD,
+							      (void *)psSDInt,
+							      PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _TLOpenStreampsSDIntRelease);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								 &psTLOpenStreamOUT->hTLPMR,
+								 (void *)psTLPMRInt,
+								 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 psTLOpenStreamOUT->hSD);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLOpenStream_exit:
+
+	if (psTLOpenStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psTLOpenStreamOUT->hSD)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psTLOpenStreamOUT->hSD,
+							     PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSDInt)
+		{
+			TLServerCloseStreamKM(psSDInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLOpenStreamOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCloseStream(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLCloseStreamIN_UI8,
+			  IMG_UINT8 * psTLCloseStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCLOSESTREAM *psTLCloseStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *psTLCloseStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLCloseStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psTLCloseStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	if (unlikely((psTLCloseStreamOUT->eError != PVRSRV_OK) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psTLCloseStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCloseStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLCloseStream_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLAcquireData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLAcquireDataIN_UI8,
+			  IMG_UINT8 * psTLAcquireDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLACQUIREDATA *psTLAcquireDataIN =
+	    (PVRSRV_BRIDGE_IN_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLACQUIREDATA *psTLAcquireDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLAcquireDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLAcquireDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLAcquireDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLAcquireData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLAcquireDataOUT->eError =
+	    TLServerAcquireDataKM(psSDInt,
+				  &psTLAcquireDataOUT->ui32ReadOffset,
+				  &psTLAcquireDataOUT->ui32ReadLen);
+
+TLAcquireData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReleaseData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLReleaseDataIN_UI8,
+			  IMG_UINT8 * psTLReleaseDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRELEASEDATA *psTLReleaseDataIN =
+	    (PVRSRV_BRIDGE_IN_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRELEASEDATA *psTLReleaseDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReleaseDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReleaseDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReleaseDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReleaseData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReleaseDataOUT->eError =
+	    TLServerReleaseDataKM(psSDInt,
+				  psTLReleaseDataIN->ui32ReadOffset,
+				  psTLReleaseDataIN->ui32ReadLen);
+
+TLReleaseData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLDiscoverStreams(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psTLDiscoverStreamsIN_UI8,
+			      IMG_UINT8 * psTLDiscoverStreamsOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *psTLDiscoverStreamsIN =
+	    (PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *psTLDiscoverStreamsOUT =
+	    (PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsOUT_UI8, 0);
+
+	IMG_CHAR *uiNamePatternInt = NULL;
+	IMG_CHAR *puiStreamsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) + 0;
+
+	if (psTLDiscoverStreamsIN->ui32Size > PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLDiscoverStreams_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psTLDiscoverStreamsOUT->puiStreams = psTLDiscoverStreamsIN->puiStreams;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLDiscoverStreams_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLDiscoverStreamsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLDiscoverStreamsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLDiscoverStreams_exit;
+			}
+		}
+	}
+
+	{
+		uiNamePatternInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNamePatternInt,
+		     (const void __user *)psTLDiscoverStreamsIN->puiNamePattern,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+		((IMG_CHAR *) uiNamePatternInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) -
+						1] = '\0';
+	}
+	if (psTLDiscoverStreamsIN->ui32Size != 0)
+	{
+		puiStreamsInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR);
+	}
+
+	psTLDiscoverStreamsOUT->eError =
+	    TLServerDiscoverStreamsKM(uiNamePatternInt,
+				      psTLDiscoverStreamsIN->ui32Size,
+				      puiStreamsInt, &psTLDiscoverStreamsOUT->ui32NumFound);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLDiscoverStreamsOUT->eError != PVRSRV_OK))
+	{
+		goto TLDiscoverStreams_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamsInt) && ((psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psTLDiscoverStreamsOUT->puiStreams, puiStreamsInt,
+		      (psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+	}
+
+TLDiscoverStreams_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLDiscoverStreamsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReserveStream(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psTLReserveStreamIN_UI8,
+			    IMG_UINT8 * psTLReserveStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRESERVESTREAM *psTLReserveStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *psTLReserveStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReserveStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReserveStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReserveStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReserveStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReserveStreamOUT->eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    &psTLReserveStreamOUT->ui32BufferOffset,
+				    psTLReserveStreamIN->ui32Size,
+				    psTLReserveStreamIN->ui32SizeMin,
+				    &psTLReserveStreamOUT->ui32Available);
+
+TLReserveStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCommitStream(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psTLCommitStreamIN_UI8,
+			   IMG_UINT8 * psTLCommitStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *psTLCommitStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *psTLCommitStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLCommitStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLCommitStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLCommitStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCommitStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLCommitStreamOUT->eError =
+	    TLServerCommitStreamKM(psSDInt, psTLCommitStreamIN->ui32ReqSize);
+
+TLCommitStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLWriteData(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psTLWriteDataIN_UI8,
+			IMG_UINT8 * psTLWriteDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLWRITEDATA *psTLWriteDataIN =
+	    (PVRSRV_BRIDGE_IN_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLWRITEDATA *psTLWriteDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLWriteDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psTLWriteDataIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLWriteData_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLWriteData_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLWriteDataIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLWriteDataIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLWriteDataOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLWriteData_exit;
+			}
+		}
+	}
+
+	if (psTLWriteDataIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psTLWriteDataIN->pui8Data,
+		     psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psTLWriteDataOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLWriteData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLWriteDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLWriteDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLWriteData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLWriteDataOUT->eError =
+	    TLServerWriteDataKM(psSDInt, psTLWriteDataIN->ui32Size, ui8DataInt);
+
+TLWriteData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLWriteDataOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+
+/*
+ * Register all PVRTL functions with services
+ */
+PVRSRV_ERROR InitPVRTLBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM,
+			      PVRSRVBridgeTLOpenStream, NULL, sizeof(PVRSRV_BRIDGE_IN_TLOPENSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLOPENSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM,
+			      PVRSRVBridgeTLCloseStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCLOSESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCLOSESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA,
+			      PVRSRVBridgeTLAcquireData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLACQUIREDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLACQUIREDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA,
+			      PVRSRVBridgeTLReleaseData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRELEASEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRELEASEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS,
+			      PVRSRVBridgeTLDiscoverStreams, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM,
+			      PVRSRVBridgeTLReserveStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRESERVESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRESERVESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM,
+			      PVRSRVBridgeTLCommitStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCOMMITSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA,
+			      PVRSRVBridgeTLWriteData, NULL, sizeof(PVRSRV_BRIDGE_IN_TLWRITEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLWRITEDATA));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pvrtl functions with services
+ */
+void DeinitPVRTLBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/common_rgxcmp_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/common_rgxcmp_bridge.h
new file mode 100644
index 000000000000..aa1d2b73985a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/common_rgxcmp_bridge.h
@@ -0,0 +1,309 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXCMP_BRIDGE_H
+#define COMMON_RGXCMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXCMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXCMP_CMD_LAST			(PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11)
+
+/*******************************************
+            RGXCreateComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticComputeContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+	IMG_UINT32 ui32StaticComputeContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT;
+
+/* Bridge out structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT;
+
+/*******************************************
+            RGXDestroyComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT;
+
+/* Bridge out structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT;
+
+/*******************************************
+            RGXFlushComputeData
+ *******************************************/
+
+/* Bridge in structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA;
+
+/* Bridge out structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA;
+
+/*******************************************
+            RGXSendCancelCmd
+ *******************************************/
+
+/* Bridge in structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32FirstIntJobRefToCancel;
+	IMG_INT32 i32LastIntJobRefToCancel;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD;
+
+/* Bridge out structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD;
+
+/*******************************************
+            RGXSetComputeContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/*******************************************
+            RGXNotifyComputeWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXSetComputeContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/*******************************************
+            RGXGetLastDeviceError
+ *******************************************/
+
+/* Bridge in structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR;
+
+/* Bridge out structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Error;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR;
+
+/*******************************************
+            RGXKickTimestampQuery
+ *******************************************/
+
+/* Bridge in structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY;
+
+/* Bridge out structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY;
+
+/*******************************************
+            RGXKickCDM
+ *******************************************/
+
+/* Bridge in structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKCDM_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 *pui32ClientUpdateOffset;
+	IMG_UINT32 *pui32ClientUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phClientUpdateUFOSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumOfWorkgroups;
+	IMG_UINT32 ui32NumOfWorkitems;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKCDM;
+
+/* Bridge out structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKCDM_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	IMG_UINT32 ui32IntJobRef;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKCDM;
+
+/*******************************************
+            RGXCDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXCDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY;
+
+#endif /* COMMON_RGXCMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/server_rgxcmp_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/server_rgxcmp_bridge.c
new file mode 100644
index 000000000000..8910cfb5ba6a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxcmp_bridge/server_rgxcmp_bridge.c
@@ -0,0 +1,1580 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxcompute.h"
+
+#include "common_rgxcmp_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateComputeContextpsComputeContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyComputeContextKM((RGX_SERVER_COMPUTE_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_COMPUTECONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_COMPUTECONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXCreateComputeContextIN_UI8,
+				    IMG_UINT8 * psRGXCreateComputeContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateComputeContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticComputeContextStateInt = NULL;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateComputeContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize >
+	     RGXFWIF_STATIC_COMPUTECONTEXT_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateComputeContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateComputeContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateComputeContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateComputeContextIN->pui8FrameworkCmd,
+		     psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize != 0)
+	{
+		ui8StaticComputeContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		    sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticComputeContextStateInt,
+		     (const void __user *)psRGXCreateComputeContextIN->
+		     pui8StaticComputeContextState,
+		     psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVRGXCreateComputeContextKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXCreateComputeContextIN->i32Priority,
+					    psRGXCreateComputeContextIN->ui32FrameworkCmdSize,
+					    ui8FrameworkCmdInt,
+					    hPrivDataInt,
+					    psRGXCreateComputeContextIN->
+					    ui32StaticComputeContextStateSize,
+					    ui8StaticComputeContextStateInt,
+					    psRGXCreateComputeContextIN->ui32PackedCCBSizeU88,
+					    psRGXCreateComputeContextIN->ui32ContextFlags,
+					    psRGXCreateComputeContextIN->ui64RobustnessAddress,
+					    psRGXCreateComputeContextIN->ui32MaxDeadlineMS,
+					    &psComputeContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psRGXCreateComputeContextOUT->
+									 hComputeContext,
+									 (void *)
+									 psComputeContextInt,
+									 PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _RGXCreateComputeContextpsComputeContextIntRelease);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateComputeContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateComputeContextOUT->eError != PVRSRV_OK)
+	{
+		if (psComputeContextInt)
+		{
+			PVRSRVRGXDestroyComputeContextKM(psComputeContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateComputeContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXDestroyComputeContextIN_UI8,
+				     IMG_UINT8 * psRGXDestroyComputeContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXDestroyComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXDestroyComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyComputeContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyComputeContextIN->
+					      hComputeContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyComputeContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyComputeContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyComputeContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyComputeContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFlushComputeData(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXFlushComputeDataIN_UI8,
+				IMG_UINT8 * psRGXFlushComputeDataOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataIN =
+	    (PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataOUT_UI8,
+								      0);
+
+	IMG_HANDLE hComputeContext = psRGXFlushComputeDataIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXFlushComputeDataOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXFlushComputeData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXFlushComputeDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXFlushComputeDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXFlushComputeData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXFlushComputeDataOUT->eError = PVRSRVRGXFlushComputeDataKM(psComputeContextInt);
+
+RGXFlushComputeData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendCancelCmd(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSendCancelCmdIN_UI8,
+			     IMG_UINT8 * psRGXSendCancelCmdOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *psRGXSendCancelCmdIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *psRGXSendCancelCmdOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSendCancelCmdIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSendCancelCmdOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSendCancelCmd_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendCancelCmdOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendCancelCmd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVRGXSendCancelCmdKM(psComputeContextInt,
+				     psRGXSendCancelCmdIN->i32FirstIntJobRefToCancel,
+				     psRGXSendCancelCmdIN->i32LastIntJobRefToCancel);
+
+RGXSendCancelCmd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPriorityIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVRGXSetComputeContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						 psComputeContextInt,
+						 psRGXSetComputeContextPriorityIN->i32Priority);
+
+RGXSetComputeContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXNotifyComputeWriteOffsetUpdateIN_UI8,
+					      IMG_UINT8 *
+					      psRGXNotifyComputeWriteOffsetUpdateOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXNotifyComputeWriteOffsetUpdateIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXNotifyComputeWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXNotifyComputeWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXNotifyComputeWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXNotifyComputeWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(psComputeContextInt);
+
+RGXNotifyComputeWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPropertyIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVRGXSetComputeContextPropertyKM(psComputeContextInt,
+						 psRGXSetComputeContextPropertyIN->ui32Property,
+						 psRGXSetComputeContextPropertyIN->ui64Input,
+						 &psRGXSetComputeContextPropertyOUT->ui64Output);
+
+RGXSetComputeContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetLastDeviceError(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorIN_UI8,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXGetLastDeviceErrorOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXGetLastDeviceError_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetLastDeviceErrorIN);
+
+	psRGXGetLastDeviceErrorOUT->eError =
+	    PVRSRVRGXGetLastDeviceErrorKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetLastDeviceErrorOUT->ui32Error);
+
+RGXGetLastDeviceError_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTimestampQuery(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXKickTimestampQueryIN_UI8,
+				  IMG_UINT8 * psRGXKickTimestampQueryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickTimestampQueryIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXKickTimestampQueryIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTimestampQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTimestampQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTimestampQuery_exit;
+			}
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTimestampQueryIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickTimestampQueryIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickTimestampQueryIN->pui8DMCmd,
+		     psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTimestampQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTimestampQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVRGXKickTimestampQueryKM(psComputeContextInt,
+					  psRGXKickTimestampQueryIN->hCheckFenceFd,
+					  psRGXKickTimestampQueryIN->hUpdateTimeline,
+					  &psRGXKickTimestampQueryOUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXKickTimestampQueryIN->ui32CmdSize,
+					  ui8DMCmdInt, psRGXKickTimestampQueryIN->ui32ExtJobRef);
+
+RGXKickTimestampQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTimestampQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickCDM(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRGXKickCDMIN_UI8,
+		       IMG_UINT8 * psRGXKickCDMOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKCDM *psRGXKickCDMIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKCDM *psRGXKickCDMOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickCDMIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientUpdateOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickCDMIN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickCDM_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickCDMIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickCDMIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickCDMOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		psClientUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXKickCDMIN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickCDMIN->phClientUpdateUFOSyncPrimBlock,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateOffsetInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateOffset,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateValueInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateValue,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickCDMIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickCDMIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickCDMIN->pui8DMCmd,
+		     psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickCDMIN->pui32SyncPMRFlags,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0, psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickCDMIN->phSyncPMRs,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickCDMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientUpdateUFOSyncPrimBlockInt[i],
+						       hClientUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickCDMOUT->eError =
+	    PVRSRVRGXKickCDMKM(psComputeContextInt,
+			       psRGXKickCDMIN->ui32ClientUpdateCount,
+			       psClientUpdateUFOSyncPrimBlockInt,
+			       ui32ClientUpdateOffsetInt,
+			       ui32ClientUpdateValueInt,
+			       psRGXKickCDMIN->hCheckFenceFd,
+			       psRGXKickCDMIN->hUpdateTimeline,
+			       &psRGXKickCDMOUT->hUpdateFence,
+			       uiUpdateFenceNameInt,
+			       psRGXKickCDMIN->hExportFenceToSignal,
+			       psRGXKickCDMIN->ui32CmdSize,
+			       ui8DMCmdInt,
+			       psRGXKickCDMIN->ui32PDumpFlags,
+			       psRGXKickCDMIN->ui32ExtJobRef,
+			       psRGXKickCDMIN->ui32SyncPMRCount,
+			       ui32SyncPMRFlagsInt,
+			       psSyncPMRsInt,
+			       psRGXKickCDMIN->ui32NumOfWorkgroups,
+			       psRGXKickCDMIN->ui32NumOfWorkitems,
+			       psRGXKickCDMIN->ui64DeadlineInus, &psRGXKickCDMOUT->ui32IntJobRef);
+
+RGXKickCDM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+
+	if (hClientUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientUpdateUFOSyncPrimBlockInt
+			    && psClientUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickCDMOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXCDMGetSharedMemoryIN);
+
+	psRGXCDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXCDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXCDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMGetSharedMemory_exit:
+
+	if (psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXCDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+
+/*
+ * Register all RGXCMP functions with services
+ */
+PVRSRV_ERROR InitRGXCMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT,
+			      PVRSRVBridgeRGXCreateComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT,
+			      PVRSRVBridgeRGXDestroyComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA,
+			      PVRSRVBridgeRGXFlushComputeData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD,
+			      PVRSRVBridgeRGXSendCancelCmd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetComputeContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetComputeContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR,
+			      PVRSRVBridgeRGXGetLastDeviceError, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY,
+			      PVRSRVBridgeRGXKickTimestampQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM,
+			      PVRSRVBridgeRGXKickCDM, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKCDM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKCDM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxcmp functions with services
+ */
+void DeinitRGXCMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
new file mode 100644
index 000000000000..534b8e9536ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/common_rgxfwdbg_bridge.h
@@ -0,0 +1,339 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXFWDBG_BRIDGE_H
+#define COMMON_RGXFWDBG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "rgx_bridge.h"
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST			(PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15)
+
+/*******************************************
+            RGXFWDebugSetFWLog
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG_TAG
+{
+	IMG_UINT32 ui32RGXFWLogType;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG;
+
+/* Bridge out structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG;
+
+/*******************************************
+            RGXFWDebugDumpFreelistPageList
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/* Bridge out structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/*******************************************
+            RGXFWDebugPowerOff
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF;
+
+/* Bridge out structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF;
+
+/*******************************************
+            RGXFWDebugPowerOn
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON;
+
+/* Bridge out structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON;
+
+/*******************************************
+            RGXFWDebugSetVzConnectionCooldownPeriodInSec
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	IMG_UINT32 ui32VzConne;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/* Bridge out structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/*******************************************
+            RGXFWDebugSetHCSDeadline
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	IMG_UINT32 ui32RGXHCSDeadline;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE;
+
+/* Bridge out structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE;
+
+/*******************************************
+            RGXFWDebugSetDriverPriority
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/* Bridge out structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSlice
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32TSPercentage;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSliceInterval
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	IMG_UINT32 ui32TSIntervalMs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/*******************************************
+            RGXFWDebugSetDriverIsolationGroup
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32IsolationGroup;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/* Bridge out structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/*******************************************
+            RGXFWDebugSetOSNewOnlineState
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32OSNewState;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/* Bridge out structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/*******************************************
+            RGXFWDebugMapGuestHeap
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	IMG_UINT64 ui64ui64GuestHeapBase;
+	IMG_UINT32 ui32DriverID;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP;
+
+/* Bridge out structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP;
+
+/*******************************************
+            RGXFWDebugPHRConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32PHRMode;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE;
+
+/*******************************************
+            RGXFWDebugWdgConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32WdgPeriodUs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE;
+
+/*******************************************
+            RGXCurrentTime
+ *******************************************/
+
+/* Bridge in structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_IN_RGXCURRENTTIME_TAG
+{
+	IMG_UINT8 ui8TimerType;
+} __packed PVRSRV_BRIDGE_IN_RGXCURRENTTIME;
+
+/* Bridge out structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCURRENTTIME_TAG
+{
+	IMG_UINT64 ui64Time;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCURRENTTIME;
+
+/*******************************************
+            RGXFWDebugInjectFault
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT;
+
+/* Bridge out structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT;
+
+#endif /* COMMON_RGXFWDBG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
new file mode 100644
index 000000000000..e5646490aa2d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxfwdbg_bridge/server_rgxfwdbg_bridge.c
@@ -0,0 +1,542 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "rgxfwdbg.h"
+#include "pmr.h"
+#include "rgxtimecorr.h"
+
+#include "common_rgxfwdbg_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetFWLog(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogOUT_UI8,
+								     0);
+
+	psRGXFWDebugSetFWLogOUT->eError =
+	    PVRSRVRGXFWDebugSetFWLogKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXFWDebugSetFWLogIN->ui32RGXFWLogType);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugDumpFreelistPageList(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListIN_UI8,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugDumpFreelistPageListIN);
+
+	psRGXFWDebugDumpFreelistPageListOUT->eError =
+	    PVRSRVRGXFWDebugDumpFreelistPageListKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOff(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugPowerOffIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugPowerOffOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOffIN);
+
+	psRGXFWDebugPowerOffOUT->eError =
+	    PVRSRVRGXFWDebugPowerOffKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOn(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXFWDebugPowerOnIN_UI8,
+			      IMG_UINT8 * psRGXFWDebugPowerOnOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOnIN);
+
+	psRGXFWDebugPowerOnOUT->eError =
+	    PVRSRVRGXFWDebugPowerOnKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec(IMG_UINT32 ui32DispatchTableEntry,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8,
+							 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8, 0);
+
+	psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT->eError =
+	    PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(psConnection,
+								 OSGetDevNode(psConnection),
+								 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN->
+								 ui32VzConne);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetHCSDeadline(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineIN_UI8,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineOUT_UI8, 0);
+
+	psRGXFWDebugSetHCSDeadlineOUT->eError =
+	    PVRSRVRGXFWDebugSetHCSDeadlineKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXFWDebugSetHCSDeadlineIN->ui32RGXHCSDeadline);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityIN_UI8,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverPriorityOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRGXFWDebugSetDriverPriorityIN->ui32DriverID,
+						psRGXFWDebugSetDriverPriorityIN->ui32Priority);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSlice(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceIN_UI8,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceKM(psConnection, OSGetDevNode(psConnection),
+						 psRGXFWDebugSetDriverTimeSliceIN->ui32DriverID,
+						 psRGXFWDebugSetDriverTimeSliceIN->
+						 ui32TSPercentage);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval(IMG_UINT32 ui32DispatchTableEntry,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8,
+						 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceIntervalOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(psConnection, OSGetDevNode(psConnection),
+							 psRGXFWDebugSetDriverTimeSliceIntervalIN->
+							 ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXFWDebugSetDriverIsolationGroupIN_UI8,
+					      IMG_UINT8 *
+					      psRGXFWDebugSetDriverIsolationGroupOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverIsolationGroupOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverIsolationGroupKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32DriverID,
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32IsolationGroup);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetOSNewOnlineState(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateIN_UI8,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateOUT_UI8, 0);
+
+	psRGXFWDebugSetOSNewOnlineStateOUT->eError =
+	    PVRSRVRGXFWDebugSetOSNewOnlineStateKM(psConnection, OSGetDevNode(psConnection),
+						  psRGXFWDebugSetOSNewOnlineStateIN->ui32DriverID,
+						  psRGXFWDebugSetOSNewOnlineStateIN->
+						  ui32OSNewState);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugMapGuestHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapOUT_UI8, 0);
+
+	psRGXFWDebugMapGuestHeapOUT->eError =
+	    PVRSRVRGXFWDebugMapGuestHeapKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugMapGuestHeapIN->ui32DriverID,
+					   psRGXFWDebugMapGuestHeapIN->ui64ui64GuestHeapBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPHRConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureOUT_UI8, 0);
+
+	psRGXFWDebugPHRConfigureOUT->eError =
+	    PVRSRVRGXFWDebugPHRConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugPHRConfigureIN->ui32ui32PHRMode);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugWdgConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureOUT_UI8, 0);
+
+	psRGXFWDebugWdgConfigureOUT->eError =
+	    PVRSRVRGXFWDebugWdgConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugWdgConfigureIN->ui32ui32WdgPeriodUs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCurrentTime(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psRGXCurrentTimeIN_UI8,
+			   IMG_UINT8 * psRGXCurrentTimeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCURRENTTIME *psRGXCurrentTimeIN =
+	    (PVRSRV_BRIDGE_IN_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *psRGXCurrentTimeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeOUT_UI8, 0);
+
+	psRGXCurrentTimeOUT->eError =
+	    PVRSRVRGXCurrentTime(psConnection, OSGetDevNode(psConnection),
+				 psRGXCurrentTimeIN->ui8TimerType, &psRGXCurrentTimeOUT->ui64Time);
+
+	return 0;
+}
+
+#define PVRSRVBridgeRGXFWDebugInjectFault NULL
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+
+/*
+ * Register all RGXFWDBG functions with services
+ */
+PVRSRV_ERROR InitRGXFWDBGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG,
+			      PVRSRVBridgeRGXFWDebugSetFWLog, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST,
+			      PVRSRVBridgeRGXFWDebugDumpFreelistPageList, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF,
+			      PVRSRVBridgeRGXFWDebugPowerOff, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON,
+			      PVRSRVBridgeRGXFWDebugPowerOn, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC,
+			      PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC),
+			      sizeof
+			      (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE,
+			      PVRSRVBridgeRGXFWDebugSetHCSDeadline, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY,
+			      PVRSRVBridgeRGXFWDebugSetDriverPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSlice, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP,
+			      PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE,
+			      PVRSRVBridgeRGXFWDebugSetOSNewOnlineState, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP,
+			      PVRSRVBridgeRGXFWDebugMapGuestHeap, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugPHRConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugWdgConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME,
+			      PVRSRVBridgeRGXCurrentTime, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCURRENTTIME),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCURRENTTIME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT,
+			      PVRSRVBridgeRGXFWDebugInjectFault, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxfwdbg functions with services
+ */
+void DeinitRGXFWDBGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/common_rgxhwperf_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/common_rgxhwperf_bridge.h
new file mode 100644
index 000000000000..7940464c7177
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/common_rgxhwperf_bridge.h
@@ -0,0 +1,246 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXHWPERF_BRIDGE_H
+#define COMMON_RGXHWPERF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_hwperf.h"
+
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST			(PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9)
+
+/*******************************************
+            RGXGetConfiguredHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/*******************************************
+            RGXGetEnabledHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS;
+
+/* Bridge out structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32BlockCount;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetHWPerfTimeStamp
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP;
+
+/* Bridge out structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT64 ui64TimeStamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP;
+
+/*******************************************
+            RGXCtrlHWPerf
+ *******************************************/
+
+/* Bridge in structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_IN_RGXCTRLHWPERF_TAG
+{
+	IMG_UINT64 ui64Mask;
+	IMG_UINT32 ui32StreamId;
+	IMG_BOOL bToggle;
+} __packed PVRSRV_BRIDGE_IN_RGXCTRLHWPERF;
+
+/* Bridge out structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF;
+
+/*******************************************
+            RGXGetHWPerfBvncFeatureFlags
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/* Bridge out structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	RGX_HWPERF_BVNC sBVNC;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/*******************************************
+            RGXControlHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	IMG_UINT16 *pui16BlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_BOOL bEnable;
+} __packed PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS;
+
+/* Bridge out structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS;
+
+/*******************************************
+            RGXConfigureHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_UINT32 ui32CtrlWord;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS;
+
+/* Bridge out structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS;
+
+/*******************************************
+            RGXOpenHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXOpenHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXCloseHWPerfClientStream
+ *******************************************/
+
+/* Bridge in structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/* Bridge out structure for RGXCloseHWPerfClientStream */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM;
+
+/*******************************************
+            RGXWriteHWPerfClientEvent
+ *******************************************/
+
+/* Bridge in structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT;
+
+/* Bridge out structure for RGXWriteHWPerfClientEvent */
+typedef struct PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT;
+
+#endif /* COMMON_RGXHWPERF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/server_rgxhwperf_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/server_rgxhwperf_bridge.c
new file mode 100644
index 000000000000..97aa4f234367
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxhwperf_bridge/server_rgxhwperf_bridge.c
@@ -0,0 +1,919 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxhwperf.h"
+#include "rgx_fwif_km.h"
+
+#include "common_rgxhwperf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersIN_UI8,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters =
+	    psRGXGetConfiguredHWPerfCountersIN->psConfiguredCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredCountersInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+						   psRGXGetConfiguredHWPerfCountersIN->ui32BlockID,
+						   psConfiguredCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters,
+		      psConfiguredCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetEnabledHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksIN_UI8,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT32 *pui32EnabledBlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) + 0;
+
+	psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs =
+	    psRGXGetEnabledHWPerfBlocksIN->pui32EnabledBlockIDs;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetEnabledHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXGetEnabledHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetEnabledHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		pui32EnabledBlockIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32);
+	}
+
+	psRGXGetEnabledHWPerfBlocksOUT->eError =
+	    PVRSRVRGXGetEnabledHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					      psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen,
+					      &psRGXGetEnabledHWPerfBlocksOUT->ui32BlockCount,
+					      pui32EnabledBlockIDsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetEnabledHWPerfBlocksOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32EnabledBlockIDsInt) &&
+	    ((psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs,
+		      pui32EnabledBlockIDsInt,
+		      (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetEnabledHWPerfBlocks_exit;
+		}
+	}
+
+RGXGetEnabledHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetEnabledHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfTimeStamp(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampIN_UI8,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfTimeStampIN);
+
+	psRGXGetHWPerfTimeStampOUT->eError =
+	    PVRSRVRGXGetHWPerfTimeStampKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetHWPerfTimeStampOUT->ui64TimeStamp);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCtrlHWPerf(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXCtrlHWPerfIN_UI8,
+			  IMG_UINT8 * psRGXCtrlHWPerfOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *psRGXCtrlHWPerfIN =
+	    (PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *psRGXCtrlHWPerfOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfOUT_UI8, 0);
+
+	psRGXCtrlHWPerfOUT->eError =
+	    PVRSRVRGXCtrlHWPerfKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXCtrlHWPerfIN->ui32StreamId,
+				  psRGXCtrlHWPerfIN->bToggle, psRGXCtrlHWPerfIN->ui64Mask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsIN_UI8,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfBvncFeatureFlagsIN);
+
+	psRGXGetHWPerfBvncFeatureFlagsOUT->eError =
+	    PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(psConnection, OSGetDevNode(psConnection),
+						 &psRGXGetHWPerfBvncFeatureFlagsOUT->sBVNC);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXControlHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksIN_UI8,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT16 *ui16BlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) + 0;
+
+	if (unlikely(psRGXControlHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXControlHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXControlHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXControlHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		ui16BlockIDsInt = (IMG_UINT16 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16);
+	}
+
+	/* Copy the data over */
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui16BlockIDsInt,
+		     (const void __user *)psRGXControlHWPerfBlocksIN->pui16BlockIDs,
+		     psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) != PVRSRV_OK)
+		{
+			psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXControlHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXControlHWPerfBlocksOUT->eError =
+	    PVRSRVRGXControlHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXControlHWPerfBlocksIN->bEnable,
+					   psRGXControlHWPerfBlocksIN->ui32ArrayLen,
+					   ui16BlockIDsInt);
+
+RGXControlHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXControlHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX + 3 <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX+3 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigureHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksIN_UI8,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigureHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX + 3))
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigureHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigureHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigureHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigureHWPerfBlocksIN->psBlockConfigs,
+		     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigureHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXConfigureHWPerfBlocksOUT->eError =
+	    PVRSRVRGXConfigureHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXConfigureHWPerfBlocksIN->ui32CtrlWord,
+					     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen,
+					     psBlockConfigsInt);
+
+RGXConfigureHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigureHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXOpenHWPerfClientStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCloseHWPerfClientStreamKM((HWPERF_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXOpenHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamIN_UI8,
+				      IMG_UINT8 * psRGXOpenHWPerfClientStreamOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *psRGXOpenHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXOpenHWPerfClientStreamOUT_UI8, 0);
+
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psRGXOpenHWPerfClientStreamIN);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVRGXOpenHWPerfClientStreamKM(psConnection, OSGetDevNode(psConnection), &psSDInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXOpenHWPerfClientStreamOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXOpenHWPerfClientStreamOUT->hSD, (void *)psSDInt,
+				      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXOpenHWPerfClientStreampsSDIntRelease);
+	if (unlikely(psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXOpenHWPerfClientStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXOpenHWPerfClientStream_exit:
+
+	if (psRGXOpenHWPerfClientStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psSDInt)
+		{
+			PVRSRVRGXCloseHWPerfClientStreamKM(psSDInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCloseHWPerfClientStream(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamIN_UI8,
+				       IMG_UINT8 * psRGXCloseHWPerfClientStreamOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *psRGXCloseHWPerfClientStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM *)
+	    IMG_OFFSET_ADDR(psRGXCloseHWPerfClientStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCloseHWPerfClientStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCloseHWPerfClientStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	if (unlikely((psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_OK) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCloseHWPerfClientStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCloseHWPerfClientStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCloseHWPerfClientStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCloseHWPerfClientStream_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXWriteHWPerfClientEvent(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventIN_UI8,
+				      IMG_UINT8 * psRGXWriteHWPerfClientEventOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventIN =
+	    (PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *psRGXWriteHWPerfClientEventOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT *)
+	    IMG_OFFSET_ADDR(psRGXWriteHWPerfClientEventOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psRGXWriteHWPerfClientEventIN->hSD;
+	HWPERF_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXWriteHWPerfClientEventIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXWriteHWPerfClientEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXWriteHWPerfClientEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXWriteHWPerfClientEvent_exit;
+			}
+		}
+	}
+
+	if (psRGXWriteHWPerfClientEventIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psRGXWriteHWPerfClientEventIN->pui8Data,
+		     psRGXWriteHWPerfClientEventIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXWriteHWPerfClientEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXWriteHWPerfClientEvent_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD, IMG_TRUE);
+	if (unlikely(psRGXWriteHWPerfClientEventOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXWriteHWPerfClientEvent_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXWriteHWPerfClientEventOUT->eError =
+	    PVRSRVRGXWriteHWPerfClientEventKM(psSDInt,
+					      psRGXWriteHWPerfClientEventIN->ui32Size, ui8DataInt);
+
+RGXWriteHWPerfClientEvent_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXWriteHWPerfClientEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+
+/*
+ * Register all RGXHWPERF functions with services
+ */
+PVRSRV_ERROR InitRGXHWPERFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS,
+			      PVRSRVBridgeRGXGetEnabledHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP,
+			      PVRSRVBridgeRGXGetHWPerfTimeStamp, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF,
+			      PVRSRVBridgeRGXCtrlHWPerf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCTRLHWPERF),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS,
+			      PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS,
+			      PVRSRVBridgeRGXControlHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS,
+			      PVRSRVBridgeRGXConfigureHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXOpenHWPerfClientStream, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXOPENHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM,
+			      PVRSRVBridgeRGXCloseHWPerfClientStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCLOSEHWPERFCLIENTSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLOSEHWPERFCLIENTSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT,
+			      PVRSRVBridgeRGXWriteHWPerfClientEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXWRITEHWPERFCLIENTEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXWRITEHWPERFCLIENTEVENT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxhwperf functions with services
+ */
+void DeinitRGXHWPERFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXOPENHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCLOSEHWPERFCLIENTSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXWRITEHWPERFCLIENTEVENT);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/common_rgxkicksync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/common_rgxkicksync_bridge.h
new file mode 100644
index 000000000000..afd882ca7529
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/common_rgxkicksync_bridge.h
@@ -0,0 +1,143 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXKICKSYNC_BRIDGE_H
+#define COMMON_RGXKICKSYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST			(PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3)
+
+/*******************************************
+            RGXCreateKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXDestroyKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXKickSync2
+ *******************************************/
+
+/* Bridge in structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKSYNC2_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 *pui32UpdateDevVarOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phUpdateUFODevVarBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hTimelineFenceFD;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKSYNC2;
+
+/* Bridge out structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKSYNC2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFenceFD;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKSYNC2;
+
+/*******************************************
+            RGXSetKickSyncContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+#endif /* COMMON_RGXKICKSYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/server_rgxkicksync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/server_rgxkicksync_bridge.c
new file mode 100644
index 000000000000..8d943cd536e2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxkicksync_bridge/server_rgxkicksync_bridge.c
@@ -0,0 +1,593 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxkicksync.h"
+
+#include "common_rgxkicksync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateKickSyncContextpsKickSyncContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyKickSyncContextKM((RGX_SERVER_KICKSYNC_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateKickSyncContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateKickSyncContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXCreateKickSyncContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVRGXCreateKickSyncContextKM(psConnection, OSGetDevNode(psConnection),
+					     hPrivDataInt,
+					     psRGXCreateKickSyncContextIN->ui32PackedCCBSizeU88,
+					     psRGXCreateKickSyncContextIN->ui32ContextFlags,
+					     &psKickSyncContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateKickSyncContextOUT->hKickSyncContext,
+				      (void *)psKickSyncContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateKickSyncContextpsKickSyncContextIntRelease);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateKickSyncContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK)
+	{
+		if (psKickSyncContextInt)
+		{
+			PVRSRVRGXDestroyKickSyncContextKM(psKickSyncContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyKickSyncContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyKickSyncContextIN->
+					      hKickSyncContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyKickSyncContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyKickSyncContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyKickSyncContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyKickSyncContext_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_DEV_VARS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_DEV_VARS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickSync2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickSync2IN_UI8,
+			 IMG_UINT8 * psRGXKickSync2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *psRGXKickSync2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *psRGXKickSync2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2OUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXKickSync2IN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFODevVarBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFODevVarBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateDevVarOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRGXKickSync2IN->ui32ClientUpdateCount > PVRSRV_MAX_DEV_VARS))
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickSync2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickSync2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickSync2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickSync2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickSync2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFODevVarBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFODevVarBlockInt, 0,
+			       psRGXKickSync2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFODevVarBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFODevVarBlockInt2,
+		     (const void __user *)psRGXKickSync2IN->phUpdateUFODevVarBlock,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateDevVarOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateDevVarOffsetInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateDevVarOffset,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateValue,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickSync2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickSync2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickSync2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickSync2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFODevVarBlockInt[i],
+						       hUpdateUFODevVarBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickSync2OUT->eError =
+	    PVRSRVRGXKickSyncKM(psKickSyncContextInt,
+				psRGXKickSync2IN->ui32ClientUpdateCount,
+				psUpdateUFODevVarBlockInt,
+				ui32UpdateDevVarOffsetInt,
+				ui32UpdateValueInt,
+				psRGXKickSync2IN->hCheckFenceFD,
+				psRGXKickSync2IN->hTimelineFenceFD,
+				&psRGXKickSync2OUT->hUpdateFenceFD,
+				uiUpdateFenceNameInt, psRGXKickSync2IN->ui32ExtJobRef);
+
+RGXKickSync2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+
+	if (hUpdateUFODevVarBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFODevVarBlockInt && psUpdateUFODevVarBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFODevVarBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickSync2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetKickSyncContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXSetKickSyncContextPropertyIN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetKickSyncContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetKickSyncContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVRGXSetKickSyncContextPropertyKM(psKickSyncContextInt,
+						  psRGXSetKickSyncContextPropertyIN->ui32Property,
+						  psRGXSetKickSyncContextPropertyIN->ui64Input,
+						  &psRGXSetKickSyncContextPropertyOUT->ui64Output);
+
+RGXSetKickSyncContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+
+/*
+ * Register all RGXKICKSYNC functions with services
+ */
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXCreateKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXDestroyKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2,
+			      PVRSRVBridgeRGXKickSync2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKSYNC2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKSYNC2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetKickSyncContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxkicksync functions with services
+ */
+void DeinitRGXKICKSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY);
+
+}
+#else /* SUPPORT_RGXKICKSYNC_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXKICKSYNC_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXKICKSYNCBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXKICKSYNCBridge()
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_bridge.h
new file mode 100644
index 000000000000..ed5a664ef17e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_bridge.h
@@ -0,0 +1,73 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RGXPDUMP_BRIDGE_H
+#define CLIENT_RGXPDUMP_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_rgxpdump_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags);
+
+#endif /* CLIENT_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
new file mode 100644
index 000000000000..7ba5126053dc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/client_rgxpdump_direct_bridge.c
@@ -0,0 +1,118 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for rgxpdump
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_rgxpdump_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "rgx_bridge.h"
+
+#include "rgxpdump.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpTraceBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpTraceBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpSignatureBuffer(IMG_HANDLE hBridge, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpSignatureBufferKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					 ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpComputeCRCSignatureCheck(IMG_HANDLE hBridge,
+							      IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpCRCSignatureCheck(IMG_HANDLE hBridge,
+						       IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					   ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPreCommand(IMG_HANDLE hBridge,
+							IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPreCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    ui32PDumpFlags);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePDumpValCheckPostCommand(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPDumpValCheckPostCommandKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					     ui32PDumpFlags);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/common_rgxpdump_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/common_rgxpdump_bridge.h
new file mode 100644
index 000000000000..37317acb4707
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/common_rgxpdump_bridge.h
@@ -0,0 +1,161 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXPDUMP_BRIDGE_H
+#define COMMON_RGXPDUMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND			PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXPDUMP_CMD_LAST			(PVRSRV_BRIDGE_RGXPDUMP_CMD_FIRST+5)
+
+/*******************************************
+            PDumpTraceBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER;
+
+/* Bridge out structure for PDumpTraceBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER;
+
+/*******************************************
+            PDumpSignatureBuffer
+ *******************************************/
+
+/* Bridge in structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER;
+
+/* Bridge out structure for PDumpSignatureBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER;
+
+/*******************************************
+            PDumpComputeCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpComputeCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpCRCSignatureCheck
+ *******************************************/
+
+/* Bridge in structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK;
+
+/* Bridge out structure for PDumpCRCSignatureCheck */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK;
+
+/*******************************************
+            PDumpValCheckPreCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND;
+
+/* Bridge out structure for PDumpValCheckPreCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND;
+
+/*******************************************
+            PDumpValCheckPostCommand
+ *******************************************/
+
+/* Bridge in structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND;
+
+/* Bridge out structure for PDumpValCheckPostCommand */
+typedef struct PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND;
+
+#endif /* COMMON_RGXPDUMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/server_rgxpdump_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/server_rgxpdump_bridge.c
new file mode 100644
index 000000000000..58ed4ebb037f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxpdump_bridge/server_rgxpdump_bridge.c
@@ -0,0 +1,239 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxpdump
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxpdump
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxpdump.h"
+
+#include "common_rgxpdump_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePDumpTraceBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psPDumpTraceBufferIN_UI8,
+			     IMG_UINT8 * psPDumpTraceBufferOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *psPDumpTraceBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *psPDumpTraceBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER *) IMG_OFFSET_ADDR(psPDumpTraceBufferOUT_UI8, 0);
+
+	psPDumpTraceBufferOUT->eError =
+	    PVRSRVPDumpTraceBufferKM(psConnection, OSGetDevNode(psConnection),
+				     psPDumpTraceBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpSignatureBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPDumpSignatureBufferIN_UI8,
+				 IMG_UINT8 * psPDumpSignatureBufferOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER *) IMG_OFFSET_ADDR(psPDumpSignatureBufferIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *psPDumpSignatureBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER *)
+	    IMG_OFFSET_ADDR(psPDumpSignatureBufferOUT_UI8, 0);
+
+	psPDumpSignatureBufferOUT->eError =
+	    PVRSRVPDumpSignatureBufferKM(psConnection, OSGetDevNode(psConnection),
+					 psPDumpSignatureBufferIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+#define PVRSRVBridgePDumpComputeCRCSignatureCheck NULL
+
+static IMG_INT
+PVRSRVBridgePDumpCRCSignatureCheck(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckIN_UI8,
+				   IMG_UINT8 * psPDumpCRCSignatureCheckOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *psPDumpCRCSignatureCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK *)
+	    IMG_OFFSET_ADDR(psPDumpCRCSignatureCheckOUT_UI8, 0);
+
+	psPDumpCRCSignatureCheckOUT->eError =
+	    PVRSRVPDumpCRCSignatureCheckKM(psConnection, OSGetDevNode(psConnection),
+					   psPDumpCRCSignatureCheckIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPreCommand(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psPDumpValCheckPreCommandIN_UI8,
+				    IMG_UINT8 * psPDumpValCheckPreCommandOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *psPDumpValCheckPreCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPreCommandOUT_UI8, 0);
+
+	psPDumpValCheckPreCommandOUT->eError =
+	    PVRSRVPDumpValCheckPreCommandKM(psConnection, OSGetDevNode(psConnection),
+					    psPDumpValCheckPreCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePDumpValCheckPostCommand(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPDumpValCheckPostCommandIN_UI8,
+				     IMG_UINT8 * psPDumpValCheckPostCommandOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandIN =
+	    (PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *psPDumpValCheckPostCommandOUT =
+	    (PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND *)
+	    IMG_OFFSET_ADDR(psPDumpValCheckPostCommandOUT_UI8, 0);
+
+	psPDumpValCheckPostCommandOUT->eError =
+	    PVRSRVPDumpValCheckPostCommandKM(psConnection, OSGetDevNode(psConnection),
+					     psPDumpValCheckPostCommandIN->ui32PDumpFlags);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXPDUMPBridge(void);
+void DeinitRGXPDUMPBridge(void);
+
+/*
+ * Register all RGXPDUMP functions with services
+ */
+PVRSRV_ERROR InitRGXPDUMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER,
+			      PVRSRVBridgePDumpTraceBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPTRACEBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPTRACEBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER,
+			      PVRSRVBridgePDumpSignatureBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPSIGNATUREBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPSIGNATUREBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpComputeCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCOMPUTECRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCOMPUTECRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK,
+			      PVRSRVBridgePDumpCRCSignatureCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPCRCSIGNATURECHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPCRCSIGNATURECHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND,
+			      PVRSRVBridgePDumpValCheckPreCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPRECOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPRECOMMAND));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+			      PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND,
+			      PVRSRVBridgePDumpValCheckPostCommand, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PDUMPVALCHECKPOSTCOMMAND),
+			      sizeof(PVRSRV_BRIDGE_OUT_PDUMPVALCHECKPOSTCOMMAND));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxpdump functions with services
+ */
+void DeinitRGXPDUMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP, PVRSRV_BRIDGE_RGXPDUMP_PDUMPTRACEBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPSIGNATUREBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCOMPUTECRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPCRCSIGNATURECHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPRECOMMAND);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXPDUMP,
+				PVRSRV_BRIDGE_RGXPDUMP_PDUMPVALCHECKPOSTCOMMAND);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/common_rgxray_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/common_rgxray_bridge.h
new file mode 100644
index 000000000000..8a5ecd699492
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/common_rgxray_bridge.h
@@ -0,0 +1,133 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxray
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxray
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXRAY_BRIDGE_H
+#define COMMON_RGXRAY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXRAY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXRAY_RGXCREATERAYCONTEXT			PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXRAY_RGXDESTROYRAYCONTEXT			PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXRAY_RGXKICKRDM			PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXRAY_CMD_LAST			(PVRSRV_BRIDGE_RGXRAY_CMD_FIRST+2)
+
+/*******************************************
+            RGXCreateRayContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateRayContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8sStaticRayContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32StaticRayContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT;
+
+/* Bridge out structure for RGXCreateRayContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT_TAG
+{
+	IMG_HANDLE hRayContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT;
+
+/*******************************************
+            RGXDestroyRayContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyRayContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT_TAG
+{
+	IMG_HANDLE hRayContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT;
+
+/* Bridge out structure for RGXDestroyRayContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT;
+
+/*******************************************
+            RGXKickRDM
+ *******************************************/
+
+/* Bridge in structure for RGXKickRDM */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKRDM_TAG
+{
+	IMG_UINT64 ui64ui64DeadlineInus;
+	IMG_HANDLE hRayContext;
+	IMG_UINT32 *pui32ClientUpdateOffset;
+	IMG_UINT32 *pui32ClientUpdateValue;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phClientUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32ui32AccStructSizeInBytes;
+	IMG_UINT32 ui32ui32DispatchSize;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKRDM;
+
+/* Bridge out structure for RGXKickRDM */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKRDM_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKRDM;
+
+#endif /* COMMON_RGXRAY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/server_rgxray_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/server_rgxray_bridge.c
new file mode 100644
index 000000000000..c537779917bf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxray_bridge/server_rgxray_bridge.c
@@ -0,0 +1,634 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxray
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxray
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxray.h"
+
+#include "common_rgxray_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateRayContextpsRayContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyRayContextKM((RGX_SERVER_RAY_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_STATIC_RAYCONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_RAYCONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateRayContext(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXCreateRayContextIN_UI8,
+				IMG_UINT8 * psRGXCreateRayContextOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT *psRGXCreateRayContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT *) IMG_OFFSET_ADDR(psRGXCreateRayContextIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT *psRGXCreateRayContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT *) IMG_OFFSET_ADDR(psRGXCreateRayContextOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPrivData = psRGXCreateRayContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8sStaticRayContextStateInt = NULL;
+	RGX_SERVER_RAY_CONTEXT *psRayContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateRayContextIN->ui32StaticRayContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely
+	    (psRGXCreateRayContextIN->ui32StaticRayContextStateSize >
+	     RGXFWIF_STATIC_RAYCONTEXT_SIZE))
+	{
+		psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRayContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateRayContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateRayContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateRayContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateRayContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateRayContextIN->ui32StaticRayContextStateSize != 0)
+	{
+		ui8sStaticRayContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRayContextIN->ui32StaticRayContextStateSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRayContextIN->ui32StaticRayContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8sStaticRayContextStateInt,
+		     (const void __user *)psRGXCreateRayContextIN->pui8sStaticRayContextState,
+		     psRGXCreateRayContextIN->ui32StaticRayContextStateSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateRayContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRayContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateRayContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateRayContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRayContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRayContextOUT->eError =
+	    PVRSRVRGXCreateRayContextKM(psConnection, OSGetDevNode(psConnection),
+					psRGXCreateRayContextIN->i32Priority,
+					hPrivDataInt,
+					psRGXCreateRayContextIN->ui32ContextFlags,
+					psRGXCreateRayContextIN->ui32StaticRayContextStateSize,
+					ui8sStaticRayContextStateInt,
+					psRGXCreateRayContextIN->ui64RobustnessAddress,
+					psRGXCreateRayContextIN->ui32MaxDeadlineMS,
+					&psRayContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateRayContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateRayContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRayContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRGXCreateRayContextOUT->
+								     hRayContext,
+								     (void *)psRayContextInt,
+								     PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RGXCreateRayContextpsRayContextIntRelease);
+	if (unlikely(psRGXCreateRayContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRayContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateRayContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateRayContextOUT->eError != PVRSRV_OK)
+	{
+		if (psRayContextInt)
+		{
+			PVRSRVRGXDestroyRayContextKM(psRayContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateRayContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyRayContext(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXDestroyRayContextIN_UI8,
+				 IMG_UINT8 * psRGXDestroyRayContextOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT *psRGXDestroyRayContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT *) IMG_OFFSET_ADDR(psRGXDestroyRayContextIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT *psRGXDestroyRayContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRayContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyRayContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyRayContextIN->hRayContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT);
+	if (unlikely((psRGXDestroyRayContextOUT->eError != PVRSRV_OK) &&
+		     (psRGXDestroyRayContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXDestroyRayContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyRayContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyRayContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyRayContext_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickRDM(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRGXKickRDMIN_UI8,
+		       IMG_UINT8 * psRGXKickRDMOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKRDM *psRGXKickRDMIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKRDM *) IMG_OFFSET_ADDR(psRGXKickRDMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKRDM *psRGXKickRDMOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKRDM *) IMG_OFFSET_ADDR(psRGXKickRDMOUT_UI8, 0);
+
+	IMG_HANDLE hRayContext = psRGXKickRDMIN->hRayContext;
+	RGX_SERVER_RAY_CONTEXT *psRayContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientUpdateOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXKickRDMIN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickRDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickRDM_exit;
+	}
+
+	if (unlikely(psRGXKickRDMIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickRDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickRDM_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickRDMOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickRDM_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickRDMIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickRDMIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickRDMOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickRDM_exit;
+			}
+		}
+	}
+
+	if (psRGXKickRDMIN->ui32ClientUpdateCount != 0)
+	{
+		psClientUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXKickRDMIN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickRDMIN->phClientUpdateUFOSyncPrimBlock,
+		     psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+	if (psRGXKickRDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateOffsetInt,
+		     (const void __user *)psRGXKickRDMIN->pui32ClientUpdateOffset,
+		     psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+	if (psRGXKickRDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateValueInt,
+		     (const void __user *)psRGXKickRDMIN->pui32ClientUpdateValue,
+		     psRGXKickRDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickRDMIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickRDMIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickRDMIN->pui8DMCmd,
+		     psRGXKickRDMIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickRDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickRDM_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickRDMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRayContextInt,
+				       hRayContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickRDMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickRDM_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickRDMIN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickRDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientUpdateUFOSyncPrimBlockInt[i],
+						       hClientUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickRDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickRDM_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickRDMOUT->eError =
+	    PVRSRVRGXKickRDMKM(psRayContextInt,
+			       psRGXKickRDMIN->ui32ClientUpdateCount,
+			       psClientUpdateUFOSyncPrimBlockInt,
+			       ui32ClientUpdateOffsetInt,
+			       ui32ClientUpdateValueInt,
+			       psRGXKickRDMIN->hCheckFenceFd,
+			       psRGXKickRDMIN->hUpdateTimeline,
+			       &psRGXKickRDMOUT->hUpdateFence,
+			       uiUpdateFenceNameInt,
+			       psRGXKickRDMIN->ui32CmdSize,
+			       ui8DMCmdInt,
+			       psRGXKickRDMIN->ui32PDumpFlags,
+			       psRGXKickRDMIN->ui32ExtJobRef,
+			       psRGXKickRDMIN->ui32ui32AccStructSizeInBytes,
+			       psRGXKickRDMIN->ui32ui32DispatchSize,
+			       psRGXKickRDMIN->ui64ui64DeadlineInus);
+
+RGXKickRDM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRayContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRayContext, PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT);
+	}
+
+	if (hClientUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickRDMIN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientUpdateUFOSyncPrimBlockInt
+			    && psClientUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickRDMOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXRAYBridge(void);
+void DeinitRGXRAYBridge(void);
+
+/*
+ * Register all RGXRAY functions with services
+ */
+PVRSRV_ERROR InitRGXRAYBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXCREATERAYCONTEXT,
+			      PVRSRVBridgeRGXCreateRayContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATERAYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATERAYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXDESTROYRAYCONTEXT,
+			      PVRSRVBridgeRGXDestroyRayContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYRAYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYRAYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXKICKRDM,
+			      PVRSRVBridgeRGXKickRDM, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKRDM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKRDM));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxray functions with services
+ */
+void DeinitRGXRAYBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXCREATERAYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXDESTROYRAYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXRAY, PVRSRV_BRIDGE_RGXRAY_RGXKICKRDM);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/common_rgxregconfig_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/common_rgxregconfig_bridge.h
new file mode 100644
index 000000000000..942b7e4b06d0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/common_rgxregconfig_bridge.h
@@ -0,0 +1,146 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXREGCONFIG_BRIDGE_H
+#define COMMON_RGXREGCONFIG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST			(PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetRegConfigType
+ *******************************************/
+
+/* Bridge in structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE_TAG
+{
+	IMG_UINT8 ui8RegPowerIsland;
+} __packed PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE;
+
+/* Bridge out structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE;
+
+/*******************************************
+            RGXAddRegconfig
+ *******************************************/
+
+/* Bridge in structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXADDREGCONFIG_TAG
+{
+	IMG_UINT64 ui64RegMask;
+	IMG_UINT64 ui64RegValue;
+	IMG_UINT32 ui32RegAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXADDREGCONFIG;
+
+/* Bridge out structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG;
+
+/*******************************************
+            RGXClearRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG;
+
+/* Bridge out structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG;
+
+/*******************************************
+            RGXEnableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG;
+
+/* Bridge out structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG;
+
+/*******************************************
+            RGXDisableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG;
+
+/* Bridge out structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG;
+
+#endif /* COMMON_RGXREGCONFIG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/server_rgxregconfig_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/server_rgxregconfig_bridge.c
new file mode 100644
index 000000000000..548401470d09
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxregconfig_bridge/server_rgxregconfig_bridge.c
@@ -0,0 +1,246 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxregconfig.h"
+
+#include "common_rgxregconfig_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetRegConfigType(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXSetRegConfigTypeIN_UI8,
+				IMG_UINT8 * psRGXSetRegConfigTypeOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeOUT_UI8,
+								      0);
+
+	psRGXSetRegConfigTypeOUT->eError =
+	    PVRSRVRGXSetRegConfigTypeKM(psConnection, OSGetDevNode(psConnection),
+					psRGXSetRegConfigTypeIN->ui8RegPowerIsland);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXAddRegconfig(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRGXAddRegconfigIN_UI8,
+			    IMG_UINT8 * psRGXAddRegconfigOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *psRGXAddRegconfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *psRGXAddRegconfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigOUT_UI8, 0);
+
+	psRGXAddRegconfigOUT->eError =
+	    PVRSRVRGXAddRegConfigKM(psConnection, OSGetDevNode(psConnection),
+				    psRGXAddRegconfigIN->ui32RegAddr,
+				    psRGXAddRegconfigIN->ui64RegValue,
+				    psRGXAddRegconfigIN->ui64RegMask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXClearRegConfigIN_UI8,
+			      IMG_UINT8 * psRGXClearRegConfigOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *psRGXClearRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *psRGXClearRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXClearRegConfigIN);
+
+	psRGXClearRegConfigOUT->eError =
+	    PVRSRVRGXClearRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXEnableRegConfigIN_UI8,
+			       IMG_UINT8 * psRGXEnableRegConfigOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *psRGXEnableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *psRGXEnableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEnableRegConfigIN);
+
+	psRGXEnableRegConfigOUT->eError =
+	    PVRSRVRGXEnableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXDisableRegConfigIN_UI8,
+				IMG_UINT8 * psRGXDisableRegConfigOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *psRGXDisableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *psRGXDisableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXDisableRegConfigIN);
+
+	psRGXDisableRegConfigOUT->eError =
+	    PVRSRVRGXDisableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+
+/*
+ * Register all RGXREGCONFIG functions with services
+ */
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE,
+			      PVRSRVBridgeRGXSetRegConfigType, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG,
+			      PVRSRVBridgeRGXAddRegconfig, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXADDREGCONFIG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG,
+			      PVRSRVBridgeRGXClearRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG,
+			      PVRSRVBridgeRGXEnableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG,
+			      PVRSRVBridgeRGXDisableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxregconfig functions with services
+ */
+void DeinitRGXREGCONFIGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG);
+
+}
+#else /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXREGCONFIG_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXREGCONFIGBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXREGCONFIGBridge()
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/common_rgxta3d_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/common_rgxta3d_bridge.h
new file mode 100644
index 000000000000..9157c6895073
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/common_rgxta3d_bridge.h
@@ -0,0 +1,425 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTA3D_BRIDGE_H
+#define COMMON_RGXTA3D_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_fwif_shared.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_LAST			(PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+15)
+
+/*******************************************
+            RGXDestroyHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET_TAG
+{
+	IMG_HANDLE hKmHwRTDataSet;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET;
+
+/* Bridge out structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET;
+
+/*******************************************
+            RGXCreateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER;
+
+/* Bridge out structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferMemDesc;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER;
+
+/* Bridge out structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER;
+
+/*******************************************
+            RGXPopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+} __packed PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXUnpopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+} __packed PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST;
+
+/* Bridge out structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST;
+
+/*******************************************
+            RGXDestroyRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT;
+
+/* Bridge out structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT;
+
+/*******************************************
+            RGXSendZSStoreDisable
+ *******************************************/
+
+/* Bridge in structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32ExtJobRefToDisableZSStore;
+	IMG_BOOL bDisableDepthStore;
+	IMG_BOOL bDisableStencilStore;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE;
+
+/* Bridge out structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE;
+
+/*******************************************
+            RGXSetRenderContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXRenderContextStalled
+ *******************************************/
+
+/* Bridge in structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED_TAG
+{
+	IMG_HANDLE hRenderContext;
+} __packed PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED;
+
+/* Bridge out structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED;
+
+/*******************************************
+            RGXKickTA3D2
+ *******************************************/
+
+/* Bridge in structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTA3D2_TAG
+{
+	IMG_UINT64 ui64Deadline;
+	IMG_HANDLE hKMHWRTDataSet;
+	IMG_HANDLE hMSAAScratchBuffer;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock;
+	IMG_HANDLE hRenderContext;
+	IMG_HANDLE hZSBuffer;
+	IMG_UINT32 *pui32Client3DUpdateSyncOffset;
+	IMG_UINT32 *pui32Client3DUpdateValue;
+	IMG_UINT32 *pui32ClientTAFenceSyncOffset;
+	IMG_UINT32 *pui32ClientTAFenceValue;
+	IMG_UINT32 *pui32ClientTAUpdateSyncOffset;
+	IMG_UINT32 *pui32ClientTAUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui83DCmd;
+	IMG_BYTE *pui83DPRCmd;
+	IMG_BYTE *pui8TACmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_CHAR *puiUpdateFenceName3D;
+	IMG_HANDLE *phClient3DUpdateSyncPrimBlock;
+	IMG_HANDLE *phClientTAFenceSyncPrimBlock;
+	IMG_HANDLE *phClientTAUpdateSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFence;
+	PVRSRV_FENCE hCheckFence3D;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	PVRSRV_TIMELINE hUpdateTimeline3D;
+	IMG_UINT32 ui323DCmdSize;
+	IMG_UINT32 ui323DPRCmdSize;
+	IMG_UINT32 ui32Client3DUpdateCount;
+	IMG_UINT32 ui32ClientTAFenceCount;
+	IMG_UINT32 ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumberOfDrawCalls;
+	IMG_UINT32 ui32NumberOfIndices;
+	IMG_UINT32 ui32NumberOfMRTs;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PRFenceUFOSyncOffset;
+	IMG_UINT32 ui32PRFenceValue;
+	IMG_UINT32 ui32RenderTargetSize;
+	IMG_UINT32 ui32SyncPMRCount;
+	IMG_UINT32 ui32TACmdSize;
+	IMG_BOOL bbAbort;
+	IMG_BOOL bbKick3D;
+	IMG_BOOL bbKickPR;
+	IMG_BOOL bbKickTA;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTA3D2;
+
+/* Bridge out structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTA3D2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	PVRSRV_FENCE hUpdateFence3D;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTA3D2;
+
+/*******************************************
+            RGXSetRenderContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hRenderContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXCreateHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_DEV_VIRTADDR sVHeapTableDevVAddr;
+	IMG_DEV_VIRTADDR *psPMDataAddr;
+	IMG_DEV_VIRTADDR *psPMSecureDataAddr;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET;
+
+/* Bridge out structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET;
+
+/*******************************************
+            RGXCreateFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hFreeListAndStateReservation;
+	IMG_HANDLE hMemCtxPrivData;
+	IMG_HANDLE hsGlobalFreeList;
+	IMG_UINT32 ui32GrowFLPages;
+	IMG_UINT32 ui32GrowParamThreshold;
+	IMG_UINT32 ui32InitFLPages;
+	IMG_UINT32 ui32MaxFLPages;
+	IMG_BOOL bbFreeListCheck;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEFREELIST;
+
+/* Bridge out structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST;
+
+/*******************************************
+            RGXCreateRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticRenderContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32Max3DDeadlineMS;
+	IMG_UINT32 ui32MaxTADeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+	IMG_UINT32 ui32StaticRenderContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT;
+
+/* Bridge out structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_HANDLE hRenderContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT;
+
+/*******************************************
+            RGXCreateHWRTDataSet2
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_DEV_VIRTADDR sVHeapTableDevVAddr;
+	IMG_HANDLE hMListsReservation;
+	IMG_HANDLE hPMSecureStatesReservation;
+	IMG_HANDLE hPMStatesReservation;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2;
+
+/* Bridge out structure for RGXCreateHWRTDataSet2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2;
+
+#endif /* COMMON_RGXTA3D_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/server_rgxta3d_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/server_rgxta3d_bridge.c
new file mode 100644
index 000000000000..dbb87e97fb26
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxta3d_bridge/server_rgxta3d_bridge.c
@@ -0,0 +1,2842 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxta3d.h"
+
+#include "common_rgxta3d_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetIN_UI8,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyHWRTDataSetOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyHWRTDataSetIN->
+					      hKmHwRTDataSet,
+					      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+	if (unlikely
+	    ((psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyHWRTDataSetOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyHWRTDataSet_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyHWRTDataSet_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateZSBufferpssZSBufferKMIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyZSBufferKM((RGX_ZSBUFFER_DATA *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateZSBufferIN_UI8,
+			      IMG_UINT8 * psRGXCreateZSBufferOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *psRGXCreateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *psRGXCreateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psRGXCreateZSBufferIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psRGXCreateZSBufferIN->hPMR;
+	PMR *psPMRInt = NULL;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError =
+	    RGXCreateZSBufferKM(psConnection, OSGetDevNode(psConnection),
+				psReservationInt,
+				psPMRInt, psRGXCreateZSBufferIN->uiMapFlags, &pssZSBufferKMInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateZSBufferOUT->
+								   hsZSBufferKM,
+								   (void *)pssZSBufferKMInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateZSBufferpssZSBufferKMIntRelease);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssZSBufferKMInt)
+		{
+			RGXDestroyZSBufferKM(pssZSBufferKMInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyZSBufferIN_UI8,
+			       IMG_UINT8 * psRGXDestroyZSBufferOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyZSBufferIN->
+					      hsZSBufferMemDesc,
+					      PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	if (unlikely
+	    ((psRGXDestroyZSBufferOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyZSBuffer_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXPopulateZSBufferpssPopulationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXUnpopulateZSBufferKM((RGX_POPULATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXPopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXPopulateZSBufferIN_UI8,
+				IMG_UINT8 * psRGXPopulateZSBufferOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferOUT_UI8,
+								      0);
+
+	IMG_HANDLE hsZSBufferKM = psRGXPopulateZSBufferIN->hsZSBufferKM;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+	RGX_POPULATION *pssPopulationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXPopulateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pssZSBufferKMInt,
+				       hsZSBufferKM,
+				       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError =
+	    RGXPopulateZSBufferKM(pssZSBufferKMInt, &pssPopulationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRGXPopulateZSBufferOUT->
+								     hsPopulation,
+								     (void *)pssPopulationInt,
+								     PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RGXPopulateZSBufferpssPopulationIntRelease);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXPopulateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pssZSBufferKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hsZSBufferKM, PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXPopulateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssPopulationInt)
+		{
+			RGXUnpopulateZSBufferKM(pssPopulationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXUnpopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferIN_UI8,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXUnpopulateZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXUnpopulateZSBufferIN->hsPopulation,
+					      PVRSRV_HANDLE_TYPE_RGX_POPULATION);
+	if (unlikely((psRGXUnpopulateZSBufferOUT->eError != PVRSRV_OK) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXUnpopulateZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXUnpopulateZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXUnpopulateZSBuffer_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyFreeListIN_UI8,
+			       IMG_UINT8 * psRGXDestroyFreeListOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *psRGXDestroyFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *psRGXDestroyFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyFreeListOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyFreeListIN->hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+	if (unlikely((psRGXDestroyFreeListOUT->eError != PVRSRV_OK) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyFreeListOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyFreeList_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyFreeList_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXDestroyRenderContextIN_UI8,
+				    IMG_UINT8 * psRGXDestroyRenderContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyRenderContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyRenderContextIN->
+					      hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyRenderContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyRenderContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyRenderContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyRenderContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendZSStoreDisable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXSendZSStoreDisableIN_UI8,
+				  IMG_UINT8 * psRGXSendZSStoreDisableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSendZSStoreDisableIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendZSStoreDisableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendZSStoreDisable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVRGXSendZSStoreDisableKM(psConnection, OSGetDevNode(psConnection),
+					  psRenderContextInt,
+					  psRGXSendZSStoreDisableIN->bDisableDepthStore,
+					  psRGXSendZSStoreDisableIN->bDisableStencilStore,
+					  psRGXSendZSStoreDisableIN->i32ExtJobRefToDisableZSStore);
+
+RGXSendZSStoreDisable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPriorityIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPriorityIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVRGXSetRenderContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRenderContextInt,
+						psRGXSetRenderContextPriorityIN->i32Priority);
+
+RGXSetRenderContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXRenderContextStalled(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXRenderContextStalledIN_UI8,
+				    IMG_UINT8 * psRGXRenderContextStalledOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledIN =
+	    (PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXRenderContextStalledIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXRenderContextStalledOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXRenderContextStalledOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXRenderContextStalled_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXRenderContextStalledOUT->eError = RGXRenderContextStalledKM(psRenderContextInt);
+
+RGXRenderContextStalled_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTA3D2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickTA3D2IN_UI8,
+			 IMG_UINT8 * psRGXKickTA3D2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *psRGXKickTA3D2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *psRGXKickTA3D2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2OUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXKickTA3D2IN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAFenceSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAFenceSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAFenceSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAFenceValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClient3DUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClient3DUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32Client3DUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32Client3DUpdateValueInt = NULL;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock = psRGXKickTA3D2IN->hPRFenceUFOSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *psPRFenceUFOSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_CHAR *uiUpdateFenceName3DInt = NULL;
+	IMG_BYTE *ui8TACmdInt = NULL;
+	IMG_BYTE *ui83DPRCmdInt = NULL;
+	IMG_BYTE *ui83DCmdInt = NULL;
+	IMG_HANDLE hKMHWRTDataSet = psRGXKickTA3D2IN->hKMHWRTDataSet;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSetInt = NULL;
+	IMG_HANDLE hZSBuffer = psRGXKickTA3D2IN->hZSBuffer;
+	RGX_ZSBUFFER_DATA *psZSBufferInt = NULL;
+	IMG_HANDLE hMSAAScratchBuffer = psRGXKickTA3D2IN->hMSAAScratchBuffer;
+	RGX_ZSBUFFER_DATA *psMSAAScratchBufferInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAFenceCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32Client3DUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32TACmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DPRCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTA3D2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTA3D2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTA3D2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		psClientTAFenceSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAFenceSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAFenceSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAFenceSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAFenceSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceValue,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		psClientTAUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateValue,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		psClient3DUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClient3DUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClient3DUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClient3DUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClient3DUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateValue,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	{
+		uiUpdateFenceName3DInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceName3DInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName3D,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceName3DInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						      1] = '\0';
+	}
+	if (psRGXKickTA3D2IN->ui32TACmdSize != 0)
+	{
+		ui8TACmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8TACmdInt, (const void __user *)psRGXKickTA3D2IN->pui8TACmd,
+		     psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize != 0)
+	{
+		ui83DPRCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DPRCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DPRCmd,
+		     psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DCmdSize != 0)
+	{
+		ui83DCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DCmd,
+		     psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32SyncPMRFlags,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickTA3D2IN->phSyncPMRs,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psClientTAFenceSyncPrimBlockInt[i],
+						       hClientTAFenceSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientTAUpdateSyncPrimBlockInt[i],
+						       hClientTAUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClient3DUpdateSyncPrimBlockInt[i],
+						       hClient3DUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPRFenceUFOSyncPrimBlockInt,
+				       hPRFenceUFOSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psKMHWRTDataSetInt,
+					       hKMHWRTDataSet,
+					       PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psZSBufferInt,
+					       hZSBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psMSAAScratchBufferInt,
+					       hMSAAScratchBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVRGXKickTA3DKM(psRenderContextInt,
+				psRGXKickTA3D2IN->ui32ClientTAFenceCount,
+				psClientTAFenceSyncPrimBlockInt,
+				ui32ClientTAFenceSyncOffsetInt,
+				ui32ClientTAFenceValueInt,
+				psRGXKickTA3D2IN->ui32ClientTAUpdateCount,
+				psClientTAUpdateSyncPrimBlockInt,
+				ui32ClientTAUpdateSyncOffsetInt,
+				ui32ClientTAUpdateValueInt,
+				psRGXKickTA3D2IN->ui32Client3DUpdateCount,
+				psClient3DUpdateSyncPrimBlockInt,
+				ui32Client3DUpdateSyncOffsetInt,
+				ui32Client3DUpdateValueInt,
+				psPRFenceUFOSyncPrimBlockInt,
+				psRGXKickTA3D2IN->ui32PRFenceUFOSyncOffset,
+				psRGXKickTA3D2IN->ui32PRFenceValue,
+				psRGXKickTA3D2IN->hCheckFence,
+				psRGXKickTA3D2IN->hUpdateTimeline,
+				&psRGXKickTA3D2OUT->hUpdateFence,
+				uiUpdateFenceNameInt,
+				psRGXKickTA3D2IN->hCheckFence3D,
+				psRGXKickTA3D2IN->hUpdateTimeline3D,
+				&psRGXKickTA3D2OUT->hUpdateFence3D,
+				uiUpdateFenceName3DInt,
+				psRGXKickTA3D2IN->ui32TACmdSize,
+				ui8TACmdInt,
+				psRGXKickTA3D2IN->ui323DPRCmdSize,
+				ui83DPRCmdInt,
+				psRGXKickTA3D2IN->ui323DCmdSize,
+				ui83DCmdInt,
+				psRGXKickTA3D2IN->ui32ExtJobRef,
+				psRGXKickTA3D2IN->bbKickTA,
+				psRGXKickTA3D2IN->bbKickPR,
+				psRGXKickTA3D2IN->bbKick3D,
+				psRGXKickTA3D2IN->bbAbort,
+				psRGXKickTA3D2IN->ui32PDumpFlags,
+				psKMHWRTDataSetInt,
+				psZSBufferInt,
+				psMSAAScratchBufferInt,
+				psRGXKickTA3D2IN->ui32SyncPMRCount,
+				ui32SyncPMRFlagsInt,
+				psSyncPMRsInt,
+				psRGXKickTA3D2IN->ui32RenderTargetSize,
+				psRGXKickTA3D2IN->ui32NumberOfDrawCalls,
+				psRGXKickTA3D2IN->ui32NumberOfIndices,
+				psRGXKickTA3D2IN->ui32NumberOfMRTs, psRGXKickTA3D2IN->ui64Deadline);
+
+RGXKickTA3D2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+
+	if (hClientTAFenceSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAFenceSyncPrimBlockInt && psClientTAFenceSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAFenceSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClientTAUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAUpdateSyncPrimBlockInt && psClientTAUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClient3DUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClient3DUpdateSyncPrimBlockInt && psClient3DUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClient3DUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPRFenceUFOSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPRFenceUFOSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psKMHWRTDataSetInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hKMHWRTDataSet,
+						    PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psZSBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hZSBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psMSAAScratchBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hMSAAScratchBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTA3D2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPropertyIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPropertyOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPropertyIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVRGXSetRenderContextPropertyKM(psRenderContextInt,
+						psRGXSetRenderContextPropertyIN->ui32Property,
+						psRGXSetRenderContextPropertyIN->ui64Input,
+						&psRGXSetRenderContextPropertyOUT->ui64Output);
+
+RGXSetRenderContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetIN_UI8,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *) IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetOUT_UI8, 0);
+
+	IMG_DEV_VIRTADDR *sPMDataAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sPMSecureDataAddrInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet = psRGXCreateHWRTDataSetIN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSetIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSetIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+
+	{
+		sPMDataAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sPMDataAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psPMDataAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sPMSecureDataAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sPMSecureDataAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psPMSecureDataAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSetOUT->eError =
+	    RGXCreateHWRTDataSet(psConnection, OSGetDevNode(psConnection),
+				 psRGXCreateHWRTDataSetIN->sVHeapTableDevVAddr,
+				 sPMDataAddrInt,
+				 sPMSecureDataAddrInt,
+				 psapsFreeListsInt,
+				 sTailPtrsDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSetOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateFreeListpsCleanupCookieIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyFreeList((RGX_FREELIST *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateFreeListIN_UI8,
+			      IMG_UINT8 * psRGXCreateFreeListOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *psRGXCreateFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *psRGXCreateFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListOUT_UI8, 0);
+
+	IMG_HANDLE hMemCtxPrivData = psRGXCreateFreeListIN->hMemCtxPrivData;
+	IMG_HANDLE hMemCtxPrivDataInt = NULL;
+	IMG_HANDLE hsGlobalFreeList = psRGXCreateFreeListIN->hsGlobalFreeList;
+	RGX_FREELIST *pssGlobalFreeListInt = NULL;
+	IMG_HANDLE hFreeListAndStateReservation =
+	    psRGXCreateFreeListIN->hFreeListAndStateReservation;
+	DEVMEMINT_RESERVATION *psFreeListAndStateReservationInt = NULL;
+	RGX_FREELIST *psCleanupCookieInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hMemCtxPrivDataInt,
+				       hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+		/* Look up the address from the handle */
+		psRGXCreateFreeListOUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&pssGlobalFreeListInt,
+					       hsGlobalFreeList,
+					       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+		if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXCreateFreeList_exit;
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psFreeListAndStateReservationInt,
+				       hFreeListAndStateReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError =
+	    RGXCreateFreeList(psConnection, OSGetDevNode(psConnection),
+			      hMemCtxPrivDataInt,
+			      psRGXCreateFreeListIN->ui32MaxFLPages,
+			      psRGXCreateFreeListIN->ui32InitFLPages,
+			      psRGXCreateFreeListIN->ui32GrowFLPages,
+			      psRGXCreateFreeListIN->ui32GrowParamThreshold,
+			      pssGlobalFreeListInt,
+			      psRGXCreateFreeListIN->bbFreeListCheck,
+			      psFreeListAndStateReservationInt, &psCleanupCookieInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateFreeListOUT->
+								   hCleanupCookie,
+								   (void *)psCleanupCookieInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateFreeListpsCleanupCookieIntRelease);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateFreeList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hMemCtxPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (pssGlobalFreeListInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hsGlobalFreeList,
+						    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psFreeListAndStateReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hFreeListAndStateReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateFreeListOUT->eError != PVRSRV_OK)
+	{
+		if (psCleanupCookieInt)
+		{
+			RGXDestroyFreeList(psCleanupCookieInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateRenderContextpsRenderContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyRenderContextKM((RGX_SERVER_RENDER_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_RENDERCONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_RENDERCONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXCreateRenderContextIN_UI8,
+				   IMG_UINT8 * psRGXCreateRenderContextOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateRenderContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticRenderContextStateInt = NULL;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateRenderContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize >
+	     RGXFWIF_STATIC_RENDERCONTEXT_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateRenderContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateRenderContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateRenderContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8FrameworkCmd,
+		     psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize != 0)
+	{
+		ui8StaticRenderContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticRenderContextStateInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8StaticRenderContextState,
+		     psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVRGXCreateRenderContextKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXCreateRenderContextIN->i32Priority,
+					   psRGXCreateRenderContextIN->ui32FrameworkCmdSize,
+					   ui8FrameworkCmdInt,
+					   hPrivDataInt,
+					   psRGXCreateRenderContextIN->
+					   ui32StaticRenderContextStateSize,
+					   ui8StaticRenderContextStateInt,
+					   psRGXCreateRenderContextIN->ui32PackedCCBSizeU8888,
+					   psRGXCreateRenderContextIN->ui32ContextFlags,
+					   psRGXCreateRenderContextIN->ui64RobustnessAddress,
+					   psRGXCreateRenderContextIN->ui32MaxTADeadlineMS,
+					   psRGXCreateRenderContextIN->ui32Max3DDeadlineMS,
+					   &psRenderContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psRGXCreateRenderContextOUT->
+									hRenderContext,
+									(void *)psRenderContextInt,
+									PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_RGXCreateRenderContextpsRenderContextIntRelease);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateRenderContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateRenderContextOUT->eError != PVRSRV_OK)
+	{
+		if (psRenderContextInt)
+		{
+			PVRSRVRGXDestroyRenderContextKM(psRenderContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateRenderContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2IN_UI8,
+				  IMG_UINT8 * psRGXCreateHWRTDataSet2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2IN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *psRGXCreateHWRTDataSet2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2 *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSet2OUT_UI8, 0);
+
+	IMG_HANDLE hMListsReservation = psRGXCreateHWRTDataSet2IN->hMListsReservation;
+	DEVMEMINT_RESERVATION *psMListsReservationInt = NULL;
+	IMG_HANDLE hPMStatesReservation = psRGXCreateHWRTDataSet2IN->hPMStatesReservation;
+	DEVMEMINT_RESERVATION *psPMStatesReservationInt = NULL;
+	IMG_HANDLE hPMSecureStatesReservation =
+	    psRGXCreateHWRTDataSet2IN->hPMSecureStatesReservation;
+	DEVMEMINT_RESERVATION *psPMSecureStatesReservationInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet = psRGXCreateHWRTDataSet2IN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSet2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSet2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSet2IN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psMListsReservationInt,
+				       hMListsReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMStatesReservationInt,
+				       hPMStatesReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMSecureStatesReservationInt,
+				       hPMSecureStatesReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSet2OUT->eError =
+	    RGXCreateHWRTDataSet2(psConnection, OSGetDevNode(psConnection),
+				  psRGXCreateHWRTDataSet2IN->sVHeapTableDevVAddr,
+				  psMListsReservationInt,
+				  psPMStatesReservationInt,
+				  psPMSecureStatesReservationInt,
+				  psapsFreeListsInt,
+				  sTailPtrsDevVAddrInt,
+				  psRGXCreateHWRTDataSet2IN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet2_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSet2OUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSet2psKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet2_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSet2OUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSet2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet2_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psMListsReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hMListsReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMStatesReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMStatesReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMSecureStatesReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMSecureStatesReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSet2OUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSet2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+
+/*
+ * Register all RGXTA3D functions with services
+ */
+PVRSRV_ERROR InitRGXTA3DBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET,
+			      PVRSRVBridgeRGXDestroyHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER,
+			      PVRSRVBridgeRGXCreateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER,
+			      PVRSRVBridgeRGXDestroyZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXPopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXUnpopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST,
+			      PVRSRVBridgeRGXDestroyFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT,
+			      PVRSRVBridgeRGXDestroyRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE,
+			      PVRSRVBridgeRGXSendZSStoreDisable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetRenderContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED,
+			      PVRSRVBridgeRGXRenderContextStalled, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2,
+			      PVRSRVBridgeRGXKickTA3D2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKTA3D2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTA3D2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetRenderContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET,
+			      PVRSRVBridgeRGXCreateHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST,
+			      PVRSRVBridgeRGXCreateFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT,
+			      PVRSRVBridgeRGXCreateRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2,
+			      PVRSRVBridgeRGXCreateHWRTDataSet2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET2));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxta3d functions with services
+ */
+void DeinitRGXTA3DBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET2);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
new file mode 100644
index 000000000000..34d7c2c56a2c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/common_rgxtimerquery_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTIMERQUERY_BRIDGE_H
+#define COMMON_RGXTIMERQUERY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST			(PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2)
+
+/*******************************************
+            RGXBeginTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY;
+
+/* Bridge out structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY;
+
+/*******************************************
+            RGXEndTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY;
+
+/* Bridge out structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY;
+
+/*******************************************
+            RGXQueryTimer
+ *******************************************/
+
+/* Bridge in structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_IN_RGXQUERYTIMER_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXQUERYTIMER;
+
+/* Bridge out structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXQUERYTIMER_TAG
+{
+	IMG_UINT64 ui64EndTime;
+	IMG_UINT64 ui64StartTime;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXQUERYTIMER;
+
+#endif /* COMMON_RGXTIMERQUERY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
new file mode 100644
index 000000000000..4af3cfc35cba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtimerquery_bridge/server_rgxtimerquery_bridge.c
@@ -0,0 +1,171 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtimerquery.h"
+
+#include "common_rgxtimerquery_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXBeginTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXBeginTimerQueryIN_UI8,
+			       IMG_UINT8 * psRGXBeginTimerQueryOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryOUT_UI8,
+								     0);
+
+	psRGXBeginTimerQueryOUT->eError =
+	    PVRSRVRGXBeginTimerQueryKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXBeginTimerQueryIN->ui32QueryId);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEndTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXEndTimerQueryIN_UI8,
+			     IMG_UINT8 * psRGXEndTimerQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *psRGXEndTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *psRGXEndTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEndTimerQueryIN);
+
+	psRGXEndTimerQueryOUT->eError =
+	    PVRSRVRGXEndTimerQueryKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXQueryTimer(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXQueryTimerIN_UI8,
+			  IMG_UINT8 * psRGXQueryTimerOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXQUERYTIMER *psRGXQueryTimerIN =
+	    (PVRSRV_BRIDGE_IN_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *psRGXQueryTimerOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerOUT_UI8, 0);
+
+	psRGXQueryTimerOUT->eError =
+	    PVRSRVRGXQueryTimerKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXQueryTimerIN->ui32QueryId,
+				  &psRGXQueryTimerOUT->ui64StartTime,
+				  &psRGXQueryTimerOUT->ui64EndTime);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+
+/*
+ * Register all RGXTIMERQUERY functions with services
+ */
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY,
+			      PVRSRVBridgeRGXBeginTimerQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY,
+			      PVRSRVBridgeRGXEndTimerQuery, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER, PVRSRVBridgeRGXQueryTimer,
+			      NULL, sizeof(PVRSRV_BRIDGE_IN_RGXQUERYTIMER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXQUERYTIMER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtimerquery functions with services
+ */
+void DeinitRGXTIMERQUERYBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/common_rgxtq2_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/common_rgxtq2_bridge.h
new file mode 100644
index 000000000000..f20fca015dc0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/common_rgxtq2_bridge.h
@@ -0,0 +1,264 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ2_BRIDGE_H
+#define COMMON_RGXTQ2_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8)
+
+/*******************************************
+            RGXTDMCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXTDMNotifyWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXTDMSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXTDMSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXTDMSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ2_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/server_rgxtq2_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/server_rgxtq2_bridge.c
new file mode 100644
index 000000000000..1129b824d850
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/rgxtq2_bridge/server_rgxtq2_bridge.c
@@ -0,0 +1,1610 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtdmtransfer.h"
+
+#include "common_rgxtq2_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXTDMCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMDestroyTransferContextKM((RGX_SERVER_TQ_TDM_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXTDMCreateTransferContextIN_UI8,
+					IMG_UINT8 * psRGXTDMCreateTransferContextOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXTDMCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXTDMCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMCreateTransferContextOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXTDMCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVRGXTDMCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+						psRGXTDMCreateTransferContextIN->i32Priority,
+						psRGXTDMCreateTransferContextIN->
+						ui32FrameworkCmdSize, ui8FrameworkCmdInt,
+						hPrivDataInt,
+						psRGXTDMCreateTransferContextIN->
+						ui32PackedCCBSizeU88,
+						psRGXTDMCreateTransferContextIN->ui32ContextFlags,
+						psRGXTDMCreateTransferContextIN->
+						ui64RobustnessAddress,
+						psRGXTDMCreateTransferContextIN->ui32MaxDeadlineMS,
+						&psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXTDMCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXTDMCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXTDMDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextIN_UI8,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMDestroyTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMDestroyTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	if (unlikely
+	    ((psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psRGXTDMDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						     psTransferContextInt,
+						     psRGXTDMSetTransferContextPriorityIN->
+						     i32Priority);
+
+RGXTDMSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateIN_UI8,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMNotifyWriteOffsetUpdateIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMNotifyWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMNotifyWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMNotifyWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMNotifyWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(psTransferContextInt,
+						  psRGXTDMNotifyWriteOffsetUpdateIN->
+						  ui32PDumpFlags);
+
+RGXTDMNotifyWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer2IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVRGXTDMSubmitTransferKM(psTransferContextInt,
+					 psRGXTDMSubmitTransfer2IN->ui32PDumpFlags,
+					 psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount,
+					 psUpdateUFOSyncPrimBlockInt,
+					 ui32UpdateSyncOffsetInt,
+					 ui32UpdateValueInt,
+					 psRGXTDMSubmitTransfer2IN->hCheckFenceFD,
+					 psRGXTDMSubmitTransfer2IN->hUpdateTimeline,
+					 &psRGXTDMSubmitTransfer2OUT->hUpdateFence,
+					 uiUpdateFenceNameInt,
+					 psRGXTDMSubmitTransfer2IN->ui32CommandSize,
+					 ui8FWCommandInt,
+					 psRGXTDMSubmitTransfer2IN->ui32ExternalJobReference,
+					 psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount,
+					 ui32SyncPMRFlagsInt,
+					 psSyncPMRsInt,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic1,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic2,
+					 psRGXTDMSubmitTransfer2IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTDMGetSharedMemoryIN);
+
+	psRGXTDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXTDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMGetSharedMemory_exit:
+
+	if (psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPropertyKM(psTransferContextInt,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui32Property,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui64Input,
+						     &psRGXTDMSetTransferContextPropertyOUT->
+						     ui64Output);
+
+RGXTDMSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer3IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVRGXTDMSubmitTransfer3KM(psTransferContextInt,
+					  psRGXTDMSubmitTransfer3IN->ui32PDumpFlags,
+					  psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount,
+					  psUpdateUFOSyncPrimBlockInt,
+					  ui32UpdateSyncOffsetInt,
+					  ui32UpdateValueInt,
+					  psRGXTDMSubmitTransfer3IN->hCheckFenceFD,
+					  psRGXTDMSubmitTransfer3IN->hUpdateTimeline,
+					  &psRGXTDMSubmitTransfer3OUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXTDMSubmitTransfer3IN->hExportFenceToSignal,
+					  psRGXTDMSubmitTransfer3IN->ui32CommandSize,
+					  ui8FWCommandInt,
+					  psRGXTDMSubmitTransfer3IN->ui32ExternalJobReference,
+					  psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount,
+					  ui32SyncPMRFlagsInt,
+					  psSyncPMRsInt,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic1,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic2,
+					  psRGXTDMSubmitTransfer3IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+
+/*
+ * Register all RGXTQ2 functions with services
+ */
+PVRSRV_ERROR InitRGXTQ2Bridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXTDMSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXTDMSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXTDMSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXTDMSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq2 functions with services
+ */
+void DeinitRGXTQ2Bridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_bridge.h
new file mode 100644
index 000000000000..c419e21e9d07
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_bridge.h
@@ -0,0 +1,88 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RI_BRIDGE_H
+#define CLIENT_RI_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_ri_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner);
+
+#endif /* CLIENT_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_direct_bridge.c
new file mode 100644
index 000000000000..de037ee78be5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/client_ri_direct_bridge.c
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for ri
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_ri_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "ri_typedefs.h"
+
+#include "ri_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError =
+	    RIWriteMEMDESCEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psPMRHandleInt,
+				  ui32TextBSize,
+				  puiTextB, ui64Offset, ui64Size, ui64Flags, &psRIHandleInt);
+
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	eError =
+	    RIWriteProcListEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   ui32TextBSize, puiTextB, ui64Size, ui64DevVAddr, &psRIHandleInt);
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIUpdateMEMDESCAddrKM(psRIHandleInt, sAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIDeleteMEMDESCEntryKM(psRIHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIDumpListKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpAllKM();
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpProcessKM(ui32Pid);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRHandleInt, ui32Owner);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/common_ri_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/common_ri_bridge.h
new file mode 100644
index 000000000000..baa41adccdfe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/common_ri_bridge.h
@@ -0,0 +1,224 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RI_BRIDGE_H
+#define COMMON_RI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "ri_typedefs.h"
+
+#define PVRSRV_BRIDGE_RI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR			PVRSRV_BRIDGE_RI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RI_RIDUMPLIST			PVRSRV_BRIDGE_RI_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RI_RIDUMPALL			PVRSRV_BRIDGE_RI_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RI_RIDUMPPROCESS			PVRSRV_BRIDGE_RI_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER			PVRSRV_BRIDGE_RI_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RI_CMD_LAST			(PVRSRV_BRIDGE_RI_CMD_FIRST+8)
+
+/*******************************************
+            RIWritePMREntry
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY;
+
+/* Bridge out structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY;
+
+/*******************************************
+            RIWriteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_UINT64 ui64Flags;
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRHandle;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY;
+
+/* Bridge out structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY;
+
+/*******************************************
+            RIWriteProcListEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_UINT64 ui64DevVAddr;
+	IMG_UINT64 ui64Size;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY;
+
+/* Bridge out structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY;
+
+/*******************************************
+            RIUpdateMEMDESCAddr
+ *******************************************/
+
+/* Bridge in structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR_TAG
+{
+	IMG_DEV_VIRTADDR sAddr;
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR;
+
+/* Bridge out structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR;
+
+/*******************************************
+            RIDeleteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY;
+
+/* Bridge out structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY;
+
+/*******************************************
+            RIDumpList
+ *******************************************/
+
+/* Bridge in structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPLIST_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPLIST;
+
+/* Bridge out structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPLIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPLIST;
+
+/*******************************************
+            RIDumpAll
+ *******************************************/
+
+/* Bridge in structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPALL_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPALL;
+
+/* Bridge out structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPALL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPALL;
+
+/*******************************************
+            RIDumpProcess
+ *******************************************/
+
+/* Bridge in structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPPROCESS_TAG
+{
+	IMG_PID ui32Pid;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPPROCESS;
+
+/* Bridge out structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPPROCESS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPPROCESS;
+
+/*******************************************
+            RIWritePMREntryWithOwner
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	IMG_HANDLE hPMRHandle;
+	IMG_PID ui32Owner;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER;
+
+/* Bridge out structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER;
+
+#endif /* COMMON_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/server_ri_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/server_ri_bridge.c
new file mode 100644
index 000000000000..f8a54d70ef0c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/ri_bridge/server_ri_bridge.c
@@ -0,0 +1,761 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "ri_server.h"
+
+#include "common_ri_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntry(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRIWritePMREntryIN_UI8,
+			    IMG_UINT8 * psRIWritePMREntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *psRIWritePMREntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *psRIWritePMREntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryOUT->eError = RIWritePMREntryKM(psPMRHandleInt);
+
+RIWritePMREntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteMEMDESCEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIWriteMEMDESCEntryIN_UI8,
+				IMG_UINT8 * psRIWriteMEMDESCEntryOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMRHandle = psRIWriteMEMDESCEntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteMEMDESCEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteMEMDESCEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteMEMDESCEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteMEMDESCEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteMEMDESCEntryIN->puiTextB,
+		     psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteMEMDESCEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWriteMEMDESCEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError =
+	    RIWriteMEMDESCEntryKM(psConnection, OSGetDevNode(psConnection),
+				  psPMRHandleInt,
+				  psRIWriteMEMDESCEntryIN->ui32TextBSize,
+				  uiTextBInt,
+				  psRIWriteMEMDESCEntryIN->ui64Offset,
+				  psRIWriteMEMDESCEntryIN->ui64Size,
+				  psRIWriteMEMDESCEntryIN->ui64Flags, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRIWriteMEMDESCEntryOUT->
+								     hRIHandle,
+								     (void *)psRIHandleInt,
+								     PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RIWriteMEMDESCEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteMEMDESCEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteMEMDESCEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteProcListEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteProcListEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIWriteProcListEntryIN_UI8,
+				 IMG_UINT8 * psRIWriteProcListEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *) IMG_OFFSET_ADDR(psRIWriteProcListEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *)
+	    IMG_OFFSET_ADDR(psRIWriteProcListEntryOUT_UI8, 0);
+
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteProcListEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteProcListEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteProcListEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteProcListEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteProcListEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteProcListEntryIN->puiTextB,
+		     psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteProcListEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psRIWriteProcListEntryOUT->eError =
+	    RIWriteProcListEntryKM(psConnection, OSGetDevNode(psConnection),
+				   psRIWriteProcListEntryIN->ui32TextBSize,
+				   uiTextBInt,
+				   psRIWriteProcListEntryIN->ui64Size,
+				   psRIWriteProcListEntryIN->ui64DevVAddr, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteProcListEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRIWriteProcListEntryOUT->
+								      hRIHandle,
+								      (void *)psRIHandleInt,
+								      PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RIWriteProcListEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteProcListEntry_exit:
+
+	if (psRIWriteProcListEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteProcListEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIUpdateMEMDESCAddr(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrIN_UI8,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrIN =
+	    (PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrOUT_UI8,
+								      0);
+
+	IMG_HANDLE hRIHandle = psRIUpdateMEMDESCAddrIN->hRIHandle;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRIHandleInt,
+				       hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE, IMG_TRUE);
+	if (unlikely(psRIUpdateMEMDESCAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIUpdateMEMDESCAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    RIUpdateMEMDESCAddrKM(psRIHandleInt, psRIUpdateMEMDESCAddrIN->sAddr);
+
+RIUpdateMEMDESCAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRIHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDeleteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryIN_UI8,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *)
+	    IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIDeleteMEMDESCEntryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRIDeleteMEMDESCEntryIN->hRIHandle,
+					      PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	if (unlikely((psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_OK) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRIDeleteMEMDESCEntryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDeleteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIDeleteMEMDESCEntry_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpList(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRIDumpListIN_UI8,
+		       IMG_UINT8 * psRIDumpListOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPLIST *psRIDumpListIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPLIST *psRIDumpListOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIDumpListIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIDumpListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIDumpListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDumpList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIDumpListOUT->eError = RIDumpListKM(psPMRHandleInt);
+
+RIDumpList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpAll(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psRIDumpAllIN_UI8,
+		      IMG_UINT8 * psRIDumpAllOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPALL *psRIDumpAllIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPALL *psRIDumpAllOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psRIDumpAllIN);
+
+	psRIDumpAllOUT->eError = RIDumpAllKM();
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpProcess(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRIDumpProcessIN_UI8,
+			  IMG_UINT8 * psRIDumpProcessOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPPROCESS *psRIDumpProcessIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *psRIDumpProcessOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psRIDumpProcessOUT->eError = RIDumpProcessKM(psRIDumpProcessIN->ui32Pid);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntryWithOwner(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerIN_UI8,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryWithOwnerIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryWithOwnerOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntryWithOwner_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    RIWritePMREntryWithOwnerKM(psPMRHandleInt, psRIWritePMREntryWithOwnerIN->ui32Owner);
+
+RIWritePMREntryWithOwner_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+
+/*
+ * Register all RI functions with services
+ */
+PVRSRV_ERROR InitRIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY,
+			      PVRSRVBridgeRIWritePMREntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY,
+			      PVRSRVBridgeRIWriteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY,
+			      PVRSRVBridgeRIWriteProcListEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR,
+			      PVRSRVBridgeRIUpdateMEMDESCAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY,
+			      PVRSRVBridgeRIDeleteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST, PVRSRVBridgeRIDumpList,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPLIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPLIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL, PVRSRVBridgeRIDumpAll,
+			      NULL, 0, sizeof(PVRSRV_BRIDGE_OUT_RIDUMPALL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS,
+			      PVRSRVBridgeRIDumpProcess, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPPROCESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPPROCESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER,
+			      PVRSRVBridgeRIWritePMREntryWithOwner, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all ri functions with services
+ */
+void DeinitRIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/common_smm_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/common_smm_bridge.h
new file mode 100644
index 000000000000..7d6603dfbe8b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/common_smm_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SMM_BRIDGE_H
+#define COMMON_SMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR			PVRSRV_BRIDGE_SMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SMM_CMD_LAST			(PVRSRV_BRIDGE_SMM_CMD_FIRST+2)
+
+/*******************************************
+            PMRSecureExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR;
+
+/* Bridge out structure for PMRSecureExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR;
+
+/*******************************************
+            PMRSecureUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR;
+
+/* Bridge out structure for PMRSecureUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR;
+
+/*******************************************
+            PMRSecureImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_SECURE_TYPE Export;
+} __packed PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR;
+
+/* Bridge out structure for PMRSecureImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR;
+
+#endif /* COMMON_SMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/server_smm_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/server_smm_bridge.c
new file mode 100644
index 000000000000..5641b1ea12f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/smm_bridge/server_smm_bridge.c
@@ -0,0 +1,281 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for smm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for smm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "secure_export.h"
+
+#include "common_smm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgePMRSecureExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureExportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureExportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *psPMRSecureExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *psPMRSecureExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureExportPMROUT_UI8,
+								     0);
+
+	IMG_HANDLE hPMR = psPMRSecureExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR *psPMROutInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRSecureExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRSecureExportPMROUT->eError =
+	    PMRSecureExportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRInt,
+			       &psPMRSecureExportPMROUT->Export, &psPMROutInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureExportPMR_exit;
+	}
+
+PMRSecureExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRSecureExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMROutInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRSecureUnexportPMR(psPMROutInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psPMRSecureUnexportPMRIN_UI8,
+				 IMG_UINT8 * psPMRSecureUnexportPMROUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureUnexportPMRIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *psPMRSecureUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREUNEXPORTPMR *)
+	    IMG_OFFSET_ADDR(psPMRSecureUnexportPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRSecureUnexportPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_SECURE_EXPORT);
+	if (unlikely((psPMRSecureUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRSecureUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRSecureUnexportPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureUnexportPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRSecureImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRSecureImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPMRSecureImportPMRIN_UI8,
+			       IMG_UINT8 * psPMRSecureImportPMROUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *psPMRSecureImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *psPMRSecureImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRSECUREIMPORTPMR *) IMG_OFFSET_ADDR(psPMRSecureImportPMROUT_UI8,
+								     0);
+
+	PMR *psPMRInt = NULL;
+
+	psPMRSecureImportPMROUT->eError =
+	    PMRSecureImportPMR(psConnection, OSGetDevNode(psConnection),
+			       psPMRSecureImportPMRIN->Export,
+			       &psPMRInt,
+			       &psPMRSecureImportPMROUT->uiSize, &psPMRSecureImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRSecureImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psPMRSecureImportPMROUT->hPMR,
+								    (void *)psPMRInt,
+								    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _PMRSecureImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRSecureImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRSecureImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRSecureImportPMR_exit:
+
+	if (psPMRSecureImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSMMBridge(void);
+void DeinitSMMBridge(void);
+
+/*
+ * Register all SMM functions with services
+ */
+PVRSRV_ERROR InitSMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR,
+			      PVRSRVBridgePMRSecureExportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR,
+			      PVRSRVBridgePMRSecureUnexportPMR, NULL);
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR,
+			      PVRSRVBridgePMRSecureImportPMR, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all smm functions with services
+ */
+void DeinitSMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SMM, PVRSRV_BRIDGE_SMM_PMRSECUREIMPORTPMR);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/common_srvcore_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/common_srvcore_bridge.h
new file mode 100644
index 000000000000..603ba5b1d89e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/common_srvcore_bridge.h
@@ -0,0 +1,369 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SRVCORE_BRIDGE_H
+#define COMMON_SRVCORE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_device_types.h"
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_SRVCORE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SRVCORE_CONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SRVCORE_DISCONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16
+#define PVRSRV_BRIDGE_SRVCORE_CMD_LAST			(PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16)
+
+/*******************************************
+            Connect
+ *******************************************/
+
+/* Bridge in structure for Connect */
+typedef struct PVRSRV_BRIDGE_IN_CONNECT_TAG
+{
+	IMG_UINT32 ui32ClientBuildOptions;
+	IMG_UINT32 ui32ClientDDKBuild;
+	IMG_UINT32 ui32ClientDDKVersion;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_CONNECT;
+
+/* Bridge out structure for Connect */
+typedef struct PVRSRV_BRIDGE_OUT_CONNECT_TAG
+{
+	IMG_UINT64 ui64PackedBvnc;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CapabilityFlags;
+	IMG_UINT8 ui8KernelArch;
+} __packed PVRSRV_BRIDGE_OUT_CONNECT;
+
+/*******************************************
+            Disconnect
+ *******************************************/
+
+/* Bridge in structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_IN_DISCONNECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DISCONNECT;
+
+/* Bridge out structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_OUT_DISCONNECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DISCONNECT;
+
+/*******************************************
+            AcquireGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT;
+
+/* Bridge out structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT;
+
+/*******************************************
+            ReleaseGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+} __packed PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT;
+
+/* Bridge out structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT;
+
+/*******************************************
+            EventObjectOpen
+ *******************************************/
+
+/* Bridge in structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hEventObject;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN;
+
+/* Bridge out structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN;
+
+/*******************************************
+            EventObjectWait
+ *******************************************/
+
+/* Bridge in structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT;
+
+/* Bridge out structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT;
+
+/*******************************************
+            EventObjectClose
+ *******************************************/
+
+/* Bridge in structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE;
+
+/* Bridge out structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE;
+
+/*******************************************
+            DumpDebugInfo
+ *******************************************/
+
+/* Bridge in structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_IN_DUMPDEBUGINFO_TAG
+{
+	IMG_UINT32 ui32VerbLevel;
+} __packed PVRSRV_BRIDGE_IN_DUMPDEBUGINFO;
+
+/* Bridge out structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO;
+
+/*******************************************
+            GetDevClockSpeed
+ *******************************************/
+
+/* Bridge in structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED;
+
+/* Bridge out structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ClockSpeed;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED;
+
+/*******************************************
+            HWOpTimeout
+ *******************************************/
+
+/* Bridge in structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_IN_HWOPTIMEOUT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HWOPTIMEOUT;
+
+/* Bridge out structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_HWOPTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HWOPTIMEOUT;
+
+/*******************************************
+            AlignmentCheck
+ *******************************************/
+
+/* Bridge in structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_IN_ALIGNMENTCHECK_TAG
+{
+	IMG_UINT32 *pui32AlignChecks;
+	IMG_UINT32 ui32AlignChecksSize;
+} __packed PVRSRV_BRIDGE_IN_ALIGNMENTCHECK;
+
+/* Bridge out structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK;
+
+/*******************************************
+            GetDeviceStatus
+ *******************************************/
+
+/* Bridge in structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVICESTATUS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVICESTATUS;
+
+/* Bridge out structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVICESTATUS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceSatus;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVICESTATUS;
+
+/*******************************************
+            GetMultiCoreInfo
+ *******************************************/
+
+/* Bridge in structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_IN_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	IMG_UINT32 ui32CapsSize;
+} __packed PVRSRV_BRIDGE_IN_GETMULTICOREINFO;
+
+/* Bridge out structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_OUT_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumCores;
+} __packed PVRSRV_BRIDGE_OUT_GETMULTICOREINFO;
+
+/*******************************************
+            EventObjectWaitTimeout
+ *******************************************/
+
+/* Bridge in structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	IMG_UINT64 ui64uiTimeoutus;
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT;
+
+/* Bridge out structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT;
+
+/*******************************************
+            FindProcessMemStats
+ *******************************************/
+
+/* Bridge in structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	IMG_UINT32 ui32ArrSize;
+	IMG_UINT32 ui32PID;
+	IMG_BOOL bbAllProcessStats;
+} __packed PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS;
+
+/* Bridge out structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS;
+
+/*******************************************
+            AcquireInfoPage
+ *******************************************/
+
+/* Bridge in structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE;
+
+/* Bridge out structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE;
+
+/*******************************************
+            ReleaseInfoPage
+ *******************************************/
+
+/* Bridge in structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_RELEASEINFOPAGE;
+
+/* Bridge out structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE;
+
+#endif /* COMMON_SRVCORE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/server_srvcore_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/server_srvcore_bridge.c
new file mode 100644
index 000000000000..d40e53b244f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/srvcore_bridge/server_srvcore_bridge.c
@@ -0,0 +1,1080 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "srvcore.h"
+#include "info_page.h"
+#include "proc_stats.h"
+#include "rgx_fwif_alignchecks.h"
+
+#include "common_srvcore_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeConnect(IMG_UINT32 ui32DispatchTableEntry,
+		    IMG_UINT8 * psConnectIN_UI8,
+		    IMG_UINT8 * psConnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CONNECT *psConnectIN =
+	    (PVRSRV_BRIDGE_IN_CONNECT *) IMG_OFFSET_ADDR(psConnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CONNECT *psConnectOUT =
+	    (PVRSRV_BRIDGE_OUT_CONNECT *) IMG_OFFSET_ADDR(psConnectOUT_UI8, 0);
+
+	psConnectOUT->eError =
+	    PVRSRVConnectKM(psConnection, OSGetDevNode(psConnection),
+			    psConnectIN->ui32Flags,
+			    psConnectIN->ui32ClientBuildOptions,
+			    psConnectIN->ui32ClientDDKVersion,
+			    psConnectIN->ui32ClientDDKBuild,
+			    &psConnectOUT->ui8KernelArch,
+			    &psConnectOUT->ui32CapabilityFlags, &psConnectOUT->ui64PackedBvnc);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDisconnect(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDisconnectIN_UI8,
+		       IMG_UINT8 * psDisconnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DISCONNECT *psDisconnectIN =
+	    (PVRSRV_BRIDGE_IN_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DISCONNECT *psDisconnectOUT =
+	    (PVRSRV_BRIDGE_OUT_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDisconnectIN);
+
+	psDisconnectOUT->eError = PVRSRVDisconnectKM();
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireGlobalEventObjecthGlobalEventObjectIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseGlobalEventObjectKM((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psAcquireGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psAcquireGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectOUT_UI8, 0);
+
+	IMG_HANDLE hGlobalEventObjectInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireGlobalEventObjectIN);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAcquireGlobalEventObjectKM(&hGlobalEventObjectInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psAcquireGlobalEventObjectOUT->hGlobalEventObject,
+				      (void *)hGlobalEventObjectInt,
+				      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _AcquireGlobalEventObjecthGlobalEventObjectIntRelease);
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AcquireGlobalEventObject_exit:
+
+	if (psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK)
+	{
+		if (hGlobalEventObjectInt)
+		{
+			PVRSRVReleaseGlobalEventObjectKM(hGlobalEventObjectInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psReleaseGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psReleaseGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psReleaseGlobalEventObjectOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psReleaseGlobalEventObjectIN->
+					      hGlobalEventObject,
+					      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	if (unlikely
+	    ((psReleaseGlobalEventObjectOUT->eError != PVRSRV_OK)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psReleaseGlobalEventObjectOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto ReleaseGlobalEventObject_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+ReleaseGlobalEventObject_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _EventObjectOpenhOSEventIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = OSEventObjectClose((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectOpen(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectOpenIN_UI8,
+			    IMG_UINT8 * psEventObjectOpenOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *psEventObjectOpenIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *psEventObjectOpenOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenOUT_UI8, 0);
+
+	IMG_HANDLE hEventObject = psEventObjectOpenIN->hEventObject;
+	IMG_HANDLE hEventObjectInt = NULL;
+	IMG_HANDLE hOSEventInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectOpenOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hEventObjectInt,
+				       hEventObject,
+				       PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT, IMG_TRUE);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = OSEventObjectOpen(hEventObjectInt, &hOSEventInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		goto EventObjectOpen_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psEventObjectOpenOUT->hOSEvent,
+								 (void *)hOSEventInt,
+								 PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _EventObjectOpenhOSEventIntRelease);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectOpen_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hEventObjectInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hEventObject, PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psEventObjectOpenOUT->eError != PVRSRV_OK)
+	{
+		if (hOSEventInt)
+		{
+			OSEventObjectClose(hOSEventInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectWaitIN_UI8,
+			    IMG_UINT8 * psEventObjectWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *psEventObjectWaitIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *psEventObjectWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitOUT->eError = OSEventObjectWait(hOSEventKMInt);
+
+EventObjectWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectClose(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psEventObjectCloseIN_UI8,
+			     IMG_UINT8 * psEventObjectCloseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *psEventObjectCloseIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *psEventObjectCloseOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectCloseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psEventObjectCloseIN->hOSEventKM,
+					      PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	if (unlikely((psEventObjectCloseOUT->eError != PVRSRV_OK) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psEventObjectCloseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectClose_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectClose_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDumpDebugInfo(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDumpDebugInfoIN_UI8,
+			  IMG_UINT8 * psDumpDebugInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *psDumpDebugInfoIN =
+	    (PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *psDumpDebugInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoOUT_UI8, 0);
+
+	psDumpDebugInfoOUT->eError =
+	    PVRSRVDumpDebugInfoKM(psConnection, OSGetDevNode(psConnection),
+				  psDumpDebugInfoIN->ui32VerbLevel);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDevClockSpeed(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetDevClockSpeedIN_UI8,
+			     IMG_UINT8 * psGetDevClockSpeedOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *psGetDevClockSpeedIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *psGetDevClockSpeedOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDevClockSpeedIN);
+
+	psGetDevClockSpeedOUT->eError =
+	    PVRSRVGetDevClockSpeedKM(psConnection, OSGetDevNode(psConnection),
+				     &psGetDevClockSpeedOUT->ui32ClockSpeed);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHWOpTimeout(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psHWOpTimeoutIN_UI8,
+			IMG_UINT8 * psHWOpTimeoutOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HWOPTIMEOUT *psHWOpTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *psHWOpTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHWOpTimeoutIN);
+
+	psHWOpTimeoutOUT->eError = PVRSRVHWOpTimeoutKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static_assert(RGXFW_ALIGN_CHECKS_UM_MAX <= IMG_UINT32_MAX,
+	      "RGXFW_ALIGN_CHECKS_UM_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeAlignmentCheck(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psAlignmentCheckIN_UI8,
+			   IMG_UINT8 * psAlignmentCheckOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *psAlignmentCheckIN =
+	    (PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *psAlignmentCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AlignChecksInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psAlignmentCheckIN->ui32AlignChecksSize > RGXFW_ALIGN_CHECKS_UM_MAX))
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto AlignmentCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto AlignmentCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psAlignmentCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psAlignmentCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psAlignmentCheckOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto AlignmentCheck_exit;
+			}
+		}
+	}
+
+	if (psAlignmentCheckIN->ui32AlignChecksSize != 0)
+	{
+		ui32AlignChecksInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AlignChecksInt,
+		     (const void __user *)psAlignmentCheckIN->pui32AlignChecks,
+		     psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psAlignmentCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto AlignmentCheck_exit;
+		}
+	}
+
+	psAlignmentCheckOUT->eError =
+	    PVRSRVAlignmentCheckKM(psConnection, OSGetDevNode(psConnection),
+				   psAlignmentCheckIN->ui32AlignChecksSize, ui32AlignChecksInt);
+
+AlignmentCheck_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psAlignmentCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDeviceStatus(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psGetDeviceStatusIN_UI8,
+			    IMG_UINT8 * psGetDeviceStatusOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVICESTATUS *psGetDeviceStatusIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *psGetDeviceStatusOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDeviceStatusIN);
+
+	psGetDeviceStatusOUT->eError =
+	    PVRSRVGetDeviceStatusKM(psConnection, OSGetDevNode(psConnection),
+				    &psGetDeviceStatusOUT->ui32DeviceSatus);
+
+	return 0;
+}
+
+static_assert(8 <= IMG_UINT32_MAX, "8 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeGetMultiCoreInfo(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetMultiCoreInfoIN_UI8,
+			     IMG_UINT8 * psGetMultiCoreInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETMULTICOREINFO *psGetMultiCoreInfoIN =
+	    (PVRSRV_BRIDGE_IN_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *psGetMultiCoreInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoOUT_UI8, 0);
+
+	IMG_UINT64 *pui64CapsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize > 8)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	psGetMultiCoreInfoOUT->pui64Caps = psGetMultiCoreInfoIN->pui64Caps;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psGetMultiCoreInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psGetMultiCoreInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto GetMultiCoreInfo_exit;
+			}
+		}
+	}
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize != 0)
+	{
+		pui64CapsInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64);
+	}
+
+	psGetMultiCoreInfoOUT->eError =
+	    PVRSRVGetMultiCoreInfoKM(psConnection, OSGetDevNode(psConnection),
+				     psGetMultiCoreInfoIN->ui32CapsSize,
+				     &psGetMultiCoreInfoOUT->ui32NumCores, pui64CapsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psGetMultiCoreInfoOUT->eError != PVRSRV_OK))
+	{
+		goto GetMultiCoreInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64CapsInt) && ((psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psGetMultiCoreInfoOUT->pui64Caps, pui64CapsInt,
+		      (psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto GetMultiCoreInfo_exit;
+		}
+	}
+
+GetMultiCoreInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psGetMultiCoreInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWaitTimeout(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psEventObjectWaitTimeoutIN_UI8,
+				   IMG_UINT8 * psEventObjectWaitTimeoutOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitTimeoutIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitTimeoutOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitTimeoutOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWaitTimeout_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitTimeoutOUT->eError =
+	    OSEventObjectWaitTimeout(hOSEventKMInt, psEventObjectWaitTimeoutIN->ui64uiTimeoutus);
+
+EventObjectWaitTimeout_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PROCESS_STAT_TYPE_COUNT <= IMG_UINT32_MAX,
+	      "PVRSRV_PROCESS_STAT_TYPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeFindProcessMemStats(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psFindProcessMemStatsIN_UI8,
+				IMG_UINT8 * psFindProcessMemStatsOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *psFindProcessMemStatsIN =
+	    (PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *psFindProcessMemStatsOUT =
+	    (PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsOUT_UI8,
+								      0);
+
+	IMG_UINT64 *pui64MemStatsArrayInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psFindProcessMemStatsIN->ui32ArrSize > PVRSRV_PROCESS_STAT_TYPE_COUNT)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto FindProcessMemStats_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psFindProcessMemStatsOUT->pui64MemStatsArray = psFindProcessMemStatsIN->pui64MemStatsArray;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto FindProcessMemStats_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psFindProcessMemStatsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psFindProcessMemStatsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto FindProcessMemStats_exit;
+			}
+		}
+	}
+
+	if (psFindProcessMemStatsIN->ui32ArrSize != 0)
+	{
+		pui64MemStatsArrayInt =
+		    (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64);
+	}
+
+	psFindProcessMemStatsOUT->eError =
+	    PVRSRVFindProcessMemStatsKM(psFindProcessMemStatsIN->ui32PID,
+					psFindProcessMemStatsIN->ui32ArrSize,
+					psFindProcessMemStatsIN->bbAllProcessStats,
+					pui64MemStatsArrayInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psFindProcessMemStatsOUT->eError != PVRSRV_OK))
+	{
+		goto FindProcessMemStats_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64MemStatsArrayInt) &&
+	    ((psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psFindProcessMemStatsOUT->pui64MemStatsArray,
+		      pui64MemStatsArrayInt,
+		      (psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto FindProcessMemStats_exit;
+		}
+	}
+
+FindProcessMemStats_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psFindProcessMemStatsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireInfoPagepsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseInfoPageKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psAcquireInfoPageIN_UI8,
+			    IMG_UINT8 * psAcquireInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *psAcquireInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *psAcquireInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageOUT_UI8, 0);
+
+	PMR *psPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireInfoPageIN);
+
+	psAcquireInfoPageOUT->eError = PVRSRVAcquireInfoPageKM(&psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psAcquireInfoPageOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psAcquireInfoPageOUT->hPMR, (void *)psPMRInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _AcquireInfoPagepsPMRIntRelease);
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+AcquireInfoPage_exit:
+
+	if (psAcquireInfoPageOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			PVRSRVReleaseInfoPageKM(psPMRInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psReleaseInfoPageIN_UI8,
+			    IMG_UINT8 * psReleaseInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *psReleaseInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *psReleaseInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psReleaseInfoPageOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psReleaseInfoPageIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psReleaseInfoPageOUT->eError != PVRSRV_OK) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psReleaseInfoPageOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto ReleaseInfoPage_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+ReleaseInfoPage_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+
+/*
+ * Register all SRVCORE functions with services
+ */
+PVRSRV_ERROR InitSRVCOREBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT,
+			      PVRSRVBridgeConnect, NULL, sizeof(PVRSRV_BRIDGE_IN_CONNECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_CONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT,
+			      PVRSRVBridgeDisconnect, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DISCONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT,
+			      PVRSRVBridgeAcquireGlobalEventObject, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT,
+			      PVRSRVBridgeReleaseGlobalEventObject, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN,
+			      PVRSRVBridgeEventObjectOpen, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT,
+			      PVRSRVBridgeEventObjectWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE,
+			      PVRSRVBridgeEventObjectClose, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO,
+			      PVRSRVBridgeDumpDebugInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DUMPDEBUGINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED,
+			      PVRSRVBridgeGetDevClockSpeed, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT,
+			      PVRSRVBridgeHWOpTimeout, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HWOPTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK,
+			      PVRSRVBridgeAlignmentCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_ALIGNMENTCHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS,
+			      PVRSRVBridgeGetDeviceStatus, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVICESTATUS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO,
+			      PVRSRVBridgeGetMultiCoreInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_GETMULTICOREINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_GETMULTICOREINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT,
+			      PVRSRVBridgeEventObjectWaitTimeout, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS,
+			      PVRSRVBridgeFindProcessMemStats, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS),
+			      sizeof(PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE,
+			      PVRSRVBridgeAcquireInfoPage, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE,
+			      PVRSRVBridgeReleaseInfoPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEINFOPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all srvcore functions with services
+ */
+void DeinitSRVCOREBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_bridge.h
new file mode 100644
index 000000000000..19f1b0ece5b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_bridge.h
@@ -0,0 +1,102 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNC_BRIDGE_H
+#define CLIENT_SYNC_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_sync_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence);
+
+#endif /* CLIENT_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_direct_bridge.c
new file mode 100644
index 000000000000..d631aea421ad
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/client_sync_direct_bridge.c
@@ -0,0 +1,262 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for sync
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_sync_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    &psSyncHandleInt,
+					    pui32SyncPrimVAddr,
+					    pui32SyncPrimBlockSize, &pshSyncPMRInt);
+
+	*phSyncHandle = psSyncHandleInt;
+	*phhSyncPMR = pshSyncPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimSetKM(psSyncHandleInt, ui32Index, ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpKM(psSyncHandleInt, ui32Offset);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt, ui32Offset, ui32Value);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     ui32Offset, ui32Value, ui32Mask, eOperator, uiPDumpFlags);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     ui32Offset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVSyncAllocEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   bServerSync, ui32FWAddr, ui32ClassNameSize, puiClassName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVSyncFreeEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32FWAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PVRSRVSyncCheckpointSignalledPDumpPolKM(hFence);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hFence);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/common_sync_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/common_sync_bridge.h
new file mode 100644
index 000000000000..753d40b64f12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/common_sync_bridge.h
@@ -0,0 +1,254 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNC_BRIDGE_H
+#define COMMON_SYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#define PVRSRV_BRIDGE_SYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMSET			PVRSRV_BRIDGE_SYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE			PVRSRV_BRIDGE_SYNC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNC_CMD_LAST			(PVRSRV_BRIDGE_SYNC_CMD_FIRST+9)
+
+/*******************************************
+            AllocSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_HANDLE hhSyncPMR;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32SyncPrimBlockSize;
+	IMG_UINT32 ui32SyncPrimVAddr;
+} __packed PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            FreeSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+} __packed PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            SyncPrimSet
+ *******************************************/
+
+/* Bridge in structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMSET_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMSET;
+
+/* Bridge out structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMSET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMSET;
+
+/*******************************************
+            SyncPrimPDump
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP;
+
+/* Bridge out structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP;
+
+/*******************************************
+            SyncPrimPDumpValue
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE;
+
+/* Bridge out structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE;
+
+/*******************************************
+            SyncPrimPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+	PDUMP_FLAGS_T uiPDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL;
+
+/* Bridge out structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL;
+
+/*******************************************
+            SyncPrimPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP;
+
+/* Bridge out structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP;
+
+/*******************************************
+            SyncAllocEvent
+ *******************************************/
+
+/* Bridge in structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCALLOCEVENT_TAG
+{
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32FWAddr;
+	IMG_BOOL bServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCALLOCEVENT;
+
+/* Bridge out structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT;
+
+/*******************************************
+            SyncFreeEvent
+ *******************************************/
+
+/* Bridge in structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFREEEVENT_TAG
+{
+	IMG_UINT32 ui32FWAddr;
+} __packed PVRSRV_BRIDGE_IN_SYNCFREEEVENT;
+
+/* Bridge out structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFREEEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFREEEVENT;
+
+/*******************************************
+            SyncCheckpointSignalledPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_FENCE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+/* Bridge out structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+#endif /* COMMON_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/server_sync_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/server_sync_bridge.c
new file mode 100644
index 000000000000..a0d9210086c9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/sync_bridge/server_sync_bridge.c
@@ -0,0 +1,755 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+#include "common_sync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _AllocSyncPrimitiveBlockpsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVFreeSyncPrimitiveBlockKM((SYNC_PRIMITIVE_BLOCK *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAllocSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockIN_UI8,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockOUT_UI8, 0);
+
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAllocSyncPrimitiveBlockIN);
+
+	psAllocSyncPrimitiveBlockOUT->hSyncHandle = NULL;
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(psConnection, OSGetDevNode(psConnection),
+					    &psSyncHandleInt,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimVAddr,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimBlockSize,
+					    &pshSyncPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAllocSyncPrimitiveBlockOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psAllocSyncPrimitiveBlockOUT->
+									 hSyncHandle,
+									 (void *)psSyncHandleInt,
+									 PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _AllocSyncPrimitiveBlockpsSyncHandleIntRelease);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psAllocSyncPrimitiveBlockOUT->hhSyncPMR,
+					 (void *)pshSyncPMRInt,
+					 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psAllocSyncPrimitiveBlockOUT->hSyncHandle);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AllocSyncPrimitiveBlock_exit:
+
+	if (psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK)
+	{
+		if (psAllocSyncPrimitiveBlockOUT->hSyncHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psAllocSyncPrimitiveBlockOUT->
+							     hSyncHandle,
+							     PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSyncHandleInt)
+		{
+			PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeFreeSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockIN_UI8,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psFreeSyncPrimitiveBlockOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psFreeSyncPrimitiveBlockIN->hSyncHandle,
+					      PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	if (unlikely((psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_OK) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psFreeSyncPrimitiveBlockOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto FreeSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+FreeSyncPrimitiveBlock_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncPrimSet(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psSyncPrimSetIN_UI8,
+			IMG_UINT8 * psSyncPrimSetOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMSET *psSyncPrimSetIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMSET *psSyncPrimSetOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimSetIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimSetOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimSetOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimSet_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimSetOUT->eError =
+	    PVRSRVSyncPrimSetKM(psSyncHandleInt,
+				psSyncPrimSetIN->ui32Index, psSyncPrimSetIN->ui32Value);
+
+SyncPrimSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDump(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncPrimPDumpIN_UI8,
+			  IMG_UINT8 * psSyncPrimPDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *psSyncPrimPDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *psSyncPrimPDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVSyncPrimPDumpKM(psSyncHandleInt, psSyncPrimPDumpIN->ui32Offset);
+
+SyncPrimPDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDump NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpValue(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncPrimPDumpValueIN_UI8,
+			       IMG_UINT8 * psSyncPrimPDumpValueOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueOUT_UI8,
+								     0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpValueIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpValueOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpValue_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt,
+				       psSyncPrimPDumpValueIN->ui32Offset,
+				       psSyncPrimPDumpValueIN->ui32Value);
+
+SyncPrimPDumpValue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpValue NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpPolIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpPolOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpPolIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpPolOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpPol_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     psSyncPrimPDumpPolIN->ui32Offset,
+				     psSyncPrimPDumpPolIN->ui32Value,
+				     psSyncPrimPDumpPolIN->ui32Mask,
+				     psSyncPrimPDumpPolIN->eOperator,
+				     psSyncPrimPDumpPolIN->uiPDumpFlags);
+
+SyncPrimPDumpPol_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpPol NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpCBPIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpCBPIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     psSyncPrimPDumpCBPIN->ui32Offset,
+				     psSyncPrimPDumpCBPIN->uiWriteOffset,
+				     psSyncPrimPDumpCBPIN->uiPacketSize,
+				     psSyncPrimPDumpCBPIN->uiBufferSize);
+
+SyncPrimPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpCBP NULL
+#endif
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncAllocEvent(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncAllocEventIN_UI8,
+			   IMG_UINT8 * psSyncAllocEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *psSyncAllocEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *psSyncAllocEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventOUT_UI8, 0);
+
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncAllocEventIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncAllocEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncAllocEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncAllocEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncAllocEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncAllocEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncAllocEvent_exit;
+			}
+		}
+	}
+
+	if (psSyncAllocEventIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncAllocEventIN->puiClassName,
+		     psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncAllocEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncAllocEvent_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psSyncAllocEventOUT->eError =
+	    PVRSRVSyncAllocEventKM(psConnection, OSGetDevNode(psConnection),
+				   psSyncAllocEventIN->bServerSync,
+				   psSyncAllocEventIN->ui32FWAddr,
+				   psSyncAllocEventIN->ui32ClassNameSize, uiClassNameInt);
+
+SyncAllocEvent_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncAllocEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFreeEvent(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncFreeEventIN_UI8,
+			  IMG_UINT8 * psSyncFreeEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFREEEVENT *psSyncFreeEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *psSyncFreeEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventOUT_UI8, 0);
+
+	psSyncFreeEventOUT->eError =
+	    PVRSRVSyncFreeEventKM(psConnection, OSGetDevNode(psConnection),
+				  psSyncFreeEventIN->ui32FWAddr);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncCheckpointSignalledPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolIN_UI8,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psSyncCheckpointSignalledPDumpPolOUT->eError =
+	    PVRSRVSyncCheckpointSignalledPDumpPolKM(psSyncCheckpointSignalledPDumpPolIN->hFence);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncCheckpointSignalledPDumpPol NULL
+#endif
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+
+/*
+ * Register all SYNC functions with services
+ */
+PVRSRV_ERROR InitSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeAllocSyncPrimitiveBlock, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeFreeSyncPrimitiveBlock, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK),
+			      sizeof(PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET,
+			      PVRSRVBridgeSyncPrimSet, NULL, sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMSET),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMSET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP,
+			      PVRSRVBridgeSyncPrimPDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE,
+			      PVRSRVBridgeSyncPrimPDumpValue, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL,
+			      PVRSRVBridgeSyncPrimPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP,
+			      PVRSRVBridgeSyncPrimPDumpCBP, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT,
+			      PVRSRVBridgeSyncAllocEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCALLOCEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT,
+			      PVRSRVBridgeSyncFreeEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFREEEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFREEEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+			      PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL,
+			      PVRSRVBridgeSyncCheckpointSignalledPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all sync functions with services
+ */
+void DeinitSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+				PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/common_syncfallback_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/common_syncfallback_bridge.h
new file mode 100644
index 000000000000..87016c4706c7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/common_syncfallback_bridge.h
@@ -0,0 +1,357 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCFALLBACK_BRIDGE_H
+#define COMMON_SYNCFALLBACK_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE			PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SYNCFALLBACK_CMD_LAST			(PVRSRV_BRIDGE_SYNCFALLBACK_CMD_FIRST+15)
+
+/*******************************************
+            SyncFbTimelineCreatePVR
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR;
+
+/* Bridge out structure for SyncFbTimelineCreatePVR */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR;
+
+/*******************************************
+            SyncFbTimelineRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE;
+
+/* Bridge out structure for SyncFbTimelineRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE;
+
+/*******************************************
+            SyncFbFenceDup
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hInFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP;
+
+/* Bridge out structure for SyncFbFenceDup */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP;
+
+/*******************************************
+            SyncFbFenceMerge
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hInFence1;
+	IMG_HANDLE hInFence2;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE;
+
+/* Bridge out structure for SyncFbFenceMerge */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE_TAG
+{
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE;
+
+/*******************************************
+            SyncFbFenceRelease
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE;
+
+/* Bridge out structure for SyncFbFenceRelease */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE;
+
+/*******************************************
+            SyncFbFenceWait
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT_TAG
+{
+	IMG_HANDLE hFence;
+	IMG_UINT32 ui32Timeout;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT;
+
+/* Bridge out structure for SyncFbFenceWait */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT;
+
+/*******************************************
+            SyncFbFenceDump
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP_TAG
+{
+	IMG_HANDLE hFence;
+	const IMG_CHAR *puiDescStr;
+	const IMG_CHAR *puiFileStr;
+	const IMG_CHAR *puiModuleStr;
+	IMG_UINT32 ui32DescStrLength;
+	IMG_UINT32 ui32FileStrLength;
+	IMG_UINT32 ui32Line;
+	IMG_UINT32 ui32ModuleStrLength;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP;
+
+/* Bridge out structure for SyncFbFenceDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP;
+
+/*******************************************
+            SyncFbTimelineCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW_TAG
+{
+	const IMG_CHAR *puiTimelineName;
+	IMG_UINT32 ui32TimelineNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW;
+
+/* Bridge out structure for SyncFbTimelineCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW;
+
+/*******************************************
+            SyncFbFenceCreateSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW_TAG
+{
+	IMG_HANDLE hTimeline;
+	const IMG_CHAR *puiFenceName;
+	IMG_UINT32 ui32FenceNameSize;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW;
+
+/* Bridge out structure for SyncFbFenceCreateSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	IMG_HANDLE hOutFence;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW;
+
+/*******************************************
+            SyncFbTimelineAdvanceSW
+ *******************************************/
+
+/* Bridge in structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_HANDLE hTimeline;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW;
+
+/* Bridge out structure for SyncFbTimelineAdvanceSW */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW_TAG
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW;
+
+/*******************************************
+            SyncFbFenceExportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroyInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroyInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE;
+
+/*******************************************
+            SyncFbFenceImportInsecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hImport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE;
+
+/* Bridge out structure for SyncFbFenceImportInsecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE;
+
+/*******************************************
+            SyncFbFenceExportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_HANDLE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceExportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Export;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE;
+
+/*******************************************
+            SyncFbFenceExportDestroySecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	IMG_HANDLE hExport;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/* Bridge out structure for SyncFbFenceExportDestroySecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE;
+
+/*******************************************
+            SyncFbFenceImportSecure
+ *******************************************/
+
+/* Bridge in structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_SECURE_TYPE Import;
+} __packed PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE;
+
+/* Bridge out structure for SyncFbFenceImportSecure */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE;
+
+#endif /* COMMON_SYNCFALLBACK_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/server_syncfallback_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/server_syncfallback_bridge.c
new file mode 100644
index 000000000000..68e650d89602
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/syncfallback_bridge/server_syncfallback_bridge.c
@@ -0,0 +1,1958 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for syncfallback
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for syncfallback
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync_fallback_server.h"
+#include "pvrsrv_sync_server.h"
+
+#include "common_syncfallback_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleaseExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _SyncFbTimelineCreatePVRpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_TIMELINE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_TIMELINE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreatePVR(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVRIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineCreatePVROUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVRIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *psSyncFbTimelineCreatePVROUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreatePVROUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize > SYNC_FB_TIMELINE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreatePVRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreatePVRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreatePVR_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreatePVRIN->puiTimelineName,
+		     psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreatePVROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreatePVR_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    SyncFbTimelineCreatePVR(psSyncFbTimelineCreatePVRIN->ui32TimelineNameSize,
+				    uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreatePVROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreatePVROUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreatePVRpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreatePVR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreatePVR_exit:
+
+	if (psSyncFbTimelineCreatePVROUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreatePVROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineRelease(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psSyncFbTimelineReleaseIN_UI8,
+				  IMG_UINT8 * psSyncFbTimelineReleaseOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *psSyncFbTimelineReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineReleaseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbTimelineReleaseIN->hTimeline,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	if (unlikely((psSyncFbTimelineReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbTimelineReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbTimelineReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineRelease_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceDuppsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDup(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncFbFenceDupIN_UI8,
+			   IMG_UINT8 * psSyncFbFenceDupOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *psSyncFbFenceDupIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *psSyncFbFenceDupOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP *) IMG_OFFSET_ADDR(psSyncFbFenceDupOUT_UI8, 0);
+
+	IMG_HANDLE hInFence = psSyncFbFenceDupIN->hInFence;
+	PVRSRV_FENCE_SERVER *psInFenceInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFenceInt,
+				       hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError = SyncFbFenceDup(psInFenceInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDupOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceDupOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceDuppsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceDupOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDup_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceDup_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceDupOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceMergepsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceMerge(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncFbFenceMergeIN_UI8,
+			     IMG_UINT8 * psSyncFbFenceMergeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *psSyncFbFenceMergeIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *psSyncFbFenceMergeOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE *) IMG_OFFSET_ADDR(psSyncFbFenceMergeOUT_UI8, 0);
+
+	IMG_HANDLE hInFence1 = psSyncFbFenceMergeIN->hInFence1;
+	PVRSRV_FENCE_SERVER *psInFence1Int = NULL;
+	IMG_HANDLE hInFence2 = psSyncFbFenceMergeIN->hInFence2;
+	PVRSRV_FENCE_SERVER *psInFence2Int = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceMergeIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceMerge_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceMergeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceMergeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceMerge_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt, (const void __user *)psSyncFbFenceMergeIN->puiFenceName,
+		     psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceMergeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceMerge_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceMergeIN->ui32FenceNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence1Int,
+				       hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Look up the address from the handle */
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psInFence2Int,
+				       hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    SyncFbFenceMerge(psInFence1Int,
+			     psInFence2Int,
+			     psSyncFbFenceMergeIN->ui32FenceNameSize,
+			     uiFenceNameInt, &psOutFenceInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceMergeOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceMergeOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _SyncFbFenceMergepsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceMergeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceMerge_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceMerge_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psInFence1Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence1, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psInFence2Int)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hInFence2, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceMergeOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceMergeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceRelease(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncFbFenceReleaseIN_UI8,
+			       IMG_UINT8 * psSyncFbFenceReleaseOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *psSyncFbFenceReleaseIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *psSyncFbFenceReleaseOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE *) IMG_OFFSET_ADDR(psSyncFbFenceReleaseOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceReleaseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceReleaseIN->hFence,
+					      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	if (unlikely((psSyncFbFenceReleaseOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncFbFenceReleaseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncFbFenceReleaseOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceRelease_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceRelease_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceWaitIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *psSyncFbFenceWaitIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *psSyncFbFenceWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT *) IMG_OFFSET_ADDR(psSyncFbFenceWaitOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceWaitIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceWaitOUT->eError =
+	    SyncFbFenceWait(psFenceInt, psSyncFbFenceWaitIN->ui32Timeout);
+
+SyncFbFenceWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+static_assert((SYNC_FB_FILE_STRING_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_FILE_STRING_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_MODULE_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_MODULE_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+static_assert((SYNC_FB_DESC_STRING_LEN_MAX + 1) <= IMG_UINT32_MAX,
+	      "(SYNC_FB_DESC_STRING_LEN_MAX+1) must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceDump(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psSyncFbFenceDumpIN_UI8,
+			    IMG_UINT8 * psSyncFbFenceDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *psSyncFbFenceDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *psSyncFbFenceDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP *) IMG_OFFSET_ADDR(psSyncFbFenceDumpOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceDumpIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	IMG_CHAR *uiFileStrInt = NULL;
+	IMG_CHAR *uiModuleStrInt = NULL;
+	IMG_CHAR *uiDescStrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32FileStrLength > (SYNC_FB_FILE_STRING_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely
+	    (psSyncFbFenceDumpIN->ui32ModuleStrLength > (SYNC_FB_MODULE_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (unlikely(psSyncFbFenceDumpIN->ui32DescStrLength > (SYNC_FB_DESC_STRING_LEN_MAX + 1)))
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceDump_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceDump_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceDumpIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceDumpIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceDump_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceDumpIN->ui32FileStrLength != 0)
+	{
+		uiFileStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFileStrInt, (const void __user *)psSyncFbFenceDumpIN->puiFileStr,
+		     psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiFileStrInt)[(psSyncFbFenceDumpIN->ui32FileStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength != 0)
+	{
+		uiModuleStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiModuleStrInt, (const void __user *)psSyncFbFenceDumpIN->puiModuleStr,
+		     psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiModuleStrInt)[(psSyncFbFenceDumpIN->ui32ModuleStrLength * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+	if (psSyncFbFenceDumpIN->ui32DescStrLength != 0)
+	{
+		uiDescStrInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiDescStrInt, (const void __user *)psSyncFbFenceDumpIN->puiDescStr,
+		     psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceDumpOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceDump_exit;
+		}
+		((IMG_CHAR *)
+		 uiDescStrInt)[(psSyncFbFenceDumpIN->ui32DescStrLength * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceDumpOUT->eError =
+	    SyncFbFenceDump(psFenceInt,
+			    psSyncFbFenceDumpIN->ui32Line,
+			    psSyncFbFenceDumpIN->ui32FileStrLength,
+			    uiFileStrInt,
+			    psSyncFbFenceDumpIN->ui32ModuleStrLength,
+			    uiModuleStrInt, psSyncFbFenceDumpIN->ui32DescStrLength, uiDescStrInt);
+
+SyncFbFenceDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceDumpOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbTimelineCreateSWpsTimelineIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbTimelineRelease((PVRSRV_TIMELINE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWIN_UI8,
+				   IMG_UINT8 * psSyncFbTimelineCreateSWOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *psSyncFbTimelineCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineCreateSWOUT_UI8, 0);
+
+	IMG_CHAR *uiTimelineNameInt = NULL;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbTimelineCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbTimelineCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbTimelineCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize != 0)
+	{
+		uiTimelineNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTimelineNameInt,
+		     (const void __user *)psSyncFbTimelineCreateSWIN->puiTimelineName,
+		     psSyncFbTimelineCreateSWIN->ui32TimelineNameSize * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psSyncFbTimelineCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbTimelineCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiTimelineNameInt)[(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize *
+				     sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    SyncFbTimelineCreateSW(psSyncFbTimelineCreateSWIN->ui32TimelineNameSize,
+				   uiTimelineNameInt, &psTimelineInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbTimelineCreateSWOUT->hTimeline,
+				      (void *)psTimelineInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbTimelineCreateSWpsTimelineIntRelease);
+	if (unlikely(psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbTimelineCreateSW_exit:
+
+	if (psSyncFbTimelineCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psTimelineInt)
+		{
+			SyncFbTimelineRelease(psTimelineInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbTimelineCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceCreateSWpsOutFenceIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static_assert(SYNC_FB_FENCE_MAX_LENGTH <= IMG_UINT32_MAX,
+	      "SYNC_FB_FENCE_MAX_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceCreateSW(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psSyncFbFenceCreateSWIN_UI8,
+				IMG_UINT8 * psSyncFbFenceCreateSWOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *psSyncFbFenceCreateSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW *) IMG_OFFSET_ADDR(psSyncFbFenceCreateSWOUT_UI8,
+								      0);
+
+	IMG_HANDLE hTimeline = psSyncFbFenceCreateSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+	IMG_CHAR *uiFenceNameInt = NULL;
+	PVRSRV_FENCE_SERVER *psOutFenceInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncFbFenceCreateSWIN->ui32FenceNameSize > SYNC_FB_FENCE_MAX_LENGTH))
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncFbFenceCreateSWIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncFbFenceCreateSWIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncFbFenceCreateSW_exit;
+			}
+		}
+	}
+
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize != 0)
+	{
+		uiFenceNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiFenceNameInt,
+		     (const void __user *)psSyncFbFenceCreateSWIN->puiFenceName,
+		     psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncFbFenceCreateSWOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncFbFenceCreateSW_exit;
+		}
+		((IMG_CHAR *)
+		 uiFenceNameInt)[(psSyncFbFenceCreateSWIN->ui32FenceNameSize * sizeof(IMG_CHAR)) -
+				 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    SyncFbFenceCreateSW(psConnection, OSGetDevNode(psConnection),
+				psTimelineInt,
+				psSyncFbFenceCreateSWIN->ui32FenceNameSize,
+				uiFenceNameInt,
+				&psOutFenceInt, &psSyncFbFenceCreateSWOUT->ui64SyncPtIdx);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceCreateSWOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceCreateSWOUT->hOutFence, (void *)psOutFenceInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceCreateSWpsOutFenceIntRelease);
+	if (unlikely(psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceCreateSW_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceCreateSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceCreateSWOUT->eError != PVRSRV_OK)
+	{
+		if (psOutFenceInt)
+		{
+			SyncFbFenceRelease(psOutFenceInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncFbFenceCreateSWOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbTimelineAdvanceSW(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWIN_UI8,
+				    IMG_UINT8 * psSyncFbTimelineAdvanceSWOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *psSyncFbTimelineAdvanceSWOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW *)
+	    IMG_OFFSET_ADDR(psSyncFbTimelineAdvanceSWOUT_UI8, 0);
+
+	IMG_HANDLE hTimeline = psSyncFbTimelineAdvanceSWIN->hTimeline;
+	PVRSRV_TIMELINE_SERVER *psTimelineInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psTimelineInt,
+				       hTimeline,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbTimelineAdvanceSWOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbTimelineAdvanceSW_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbTimelineAdvanceSWOUT->eError =
+	    SyncFbTimelineAdvanceSW(psTimelineInt, &psSyncFbTimelineAdvanceSWOUT->ui64SyncPtIdx);
+
+SyncFbTimelineAdvanceSW_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTimelineInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hTimeline, PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceExportInsecurepsExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceExportDestroyInsecure((PVRSRV_FENCE_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceExportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *psSyncFbFenceExportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportInsecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    SyncFbFenceExportInsecure(psFenceInt, &psExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase, &hExportInt,
+				      (void *)psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceExportInsecurepsExportIntRelease);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportInsecureOUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+									   &psSyncFbFenceExportInsecureOUT->
+									   hExport,
+									   (void *)psExportInt,
+									   PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+									   PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+									   (PFN_HANDLE_RELEASE) &
+									   ReleaseExport);
+	if (unlikely(psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportInsecure_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncFbFenceExportInsecureOUT->hExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE)
+							     psSyncFbFenceExportInsecureOUT->
+							     hExport,
+							     PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hExportInt,
+							PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psExportInt)
+		{
+			SyncFbFenceExportDestroyInsecure(psExportInt);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroyInsecure(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureIN_UI8,
+					     IMG_UINT8 * psSyncFbFenceExportDestroyInsecureOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *psSyncFbFenceExportDestroyInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroyInsecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	IMG_HANDLE hExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psExportInt,
+				       (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->hExport,
+				       PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_FALSE);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hExportInt,
+				     psExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely(psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT(psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hExportInt, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK) &&
+		     (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+		     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+	}
+	PVR_ASSERT((psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_OK) ||
+		   (psSyncFbFenceExportDestroyInsecureOUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceExportDestroyInsecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroyInsecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroyInsecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroyInsecureOUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceExportDestroyInsecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+SyncFbFenceExportDestroyInsecure_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceExportDestroyInsecure NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _SyncFbFenceImportInsecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportInsecure(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureIN_UI8,
+				      IMG_UINT8 * psSyncFbFenceImportInsecureOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *psSyncFbFenceImportInsecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportInsecureOUT_UI8, 0);
+
+	IMG_HANDLE hImport = psSyncFbFenceImportInsecureIN->hImport;
+	PVRSRV_FENCE_EXPORT *psImportInt = NULL;
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psImportInt,
+				       hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT, IMG_TRUE);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    SyncFbFenceImportInsecure(psConnection, OSGetDevNode(psConnection),
+				      psImportInt, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportInsecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportInsecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportInsecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportInsecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportInsecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psImportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hImport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psSyncFbFenceImportInsecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncFbFenceImportInsecure NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceExportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceExportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *psSyncFbFenceExportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportSecureOUT_UI8, 0);
+
+	IMG_HANDLE hFence = psSyncFbFenceExportSecureIN->hFence;
+	PVRSRV_FENCE_SERVER *psFenceInt = NULL;
+	PVRSRV_FENCE_EXPORT *psExportInt = NULL;
+	CONNECTION_DATA *psSecureConnection;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncFbFenceExportSecureOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psFenceInt,
+				       hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER, IMG_TRUE);
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceExportSecure_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceExportSecureOUT->eError =
+	    SyncFbFenceExportSecure(psConnection, OSGetDevNode(psConnection),
+				    psFenceInt,
+				    &psSyncFbFenceExportSecureOUT->Export,
+				    &psExportInt, &psSecureConnection);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceExportSecure_exit;
+	}
+
+SyncFbFenceExportSecure_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psFenceInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hFence, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psSyncFbFenceExportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psExportInt)
+		{
+			SyncFbFenceExportDestroySecure(psExportInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceExportDestroySecure(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureIN_UI8,
+					   IMG_UINT8 * psSyncFbFenceExportDestroySecureOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *psSyncFbFenceExportDestroySecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceExportDestroySecureOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncFbFenceExportDestroySecureOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncFbFenceExportDestroySecureIN->
+					      hExport, PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT);
+	if (unlikely
+	    ((psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_OK)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psSyncFbFenceExportDestroySecureOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psSyncFbFenceExportDestroySecureOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncFbFenceExportDestroySecure_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncFbFenceExportDestroySecure_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncFbFenceImportSecurepsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFbFenceImportSecure(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psSyncFbFenceImportSecureIN_UI8,
+				    IMG_UINT8 * psSyncFbFenceImportSecureOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *psSyncFbFenceImportSecureOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE *)
+	    IMG_OFFSET_ADDR(psSyncFbFenceImportSecureOUT_UI8, 0);
+
+	PVRSRV_FENCE_SERVER *psSyncHandleInt = NULL;
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    SyncFbFenceImportSecure(psConnection, OSGetDevNode(psConnection),
+				    psSyncFbFenceImportSecureIN->Import, &psSyncHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psSyncFbFenceImportSecureOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psSyncFbFenceImportSecureOUT->hSyncHandle,
+				      (void *)psSyncHandleInt,
+				      PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _SyncFbFenceImportSecurepsSyncHandleIntRelease);
+	if (unlikely(psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto SyncFbFenceImportSecure_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+SyncFbFenceImportSecure_exit:
+
+	if (psSyncFbFenceImportSecureOUT->eError != PVRSRV_OK)
+	{
+		if (psSyncHandleInt)
+		{
+			SyncFbFenceRelease(psSyncHandleInt);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void);
+void DeinitSYNCFALLBACKBridge(void);
+
+/*
+ * Register all SYNCFALLBACK functions with services
+ */
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR,
+			      PVRSRVBridgeSyncFbTimelineCreatePVR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATEPVR),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATEPVR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE,
+			      PVRSRVBridgeSyncFbTimelineRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK, PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP,
+			      PVRSRVBridgeSyncFbFenceDup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE,
+			      PVRSRVBridgeSyncFbFenceMerge, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEMERGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEMERGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE,
+			      PVRSRVBridgeSyncFbFenceRelease, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCERELEASE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCERELEASE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT,
+			      PVRSRVBridgeSyncFbFenceWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP,
+			      PVRSRVBridgeSyncFbFenceDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW,
+			      PVRSRVBridgeSyncFbTimelineCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW,
+			      PVRSRVBridgeSyncFbFenceCreateSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCECREATESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCECREATESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW,
+			      PVRSRVBridgeSyncFbTimelineAdvanceSW, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBTIMELINEADVANCESW),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBTIMELINEADVANCESW));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroyInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE,
+			      PVRSRVBridgeSyncFbFenceImportInsecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTINSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTINSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceExportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE,
+			      PVRSRVBridgeSyncFbFenceExportDestroySecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEEXPORTDESTROYSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEEXPORTDESTROYSECURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+			      PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE,
+			      PVRSRVBridgeSyncFbFenceImportSecure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFBFENCEIMPORTSECURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFBFENCEIMPORTSECURE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all syncfallback functions with services
+ */
+void DeinitSYNCFALLBACKBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATEPVR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEMERGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCERELEASE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCECREATESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBTIMELINEADVANCESW);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTINSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEEXPORTDESTROYSECURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCFALLBACK,
+				PVRSRV_BRIDGE_SYNCFALLBACK_SYNCFBFENCEIMPORTSECURE);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_bridge.h
new file mode 100644
index 000000000000..544efd962834
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_bridge.h
@@ -0,0 +1,68 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNCTRACKING_BRIDGE_H
+#define CLIENT_SYNCTRACKING_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_synctracking_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName);
+
+#endif /* CLIENT_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_direct_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_direct_bridge.c
new file mode 100644
index 000000000000..baeb89a6a0ae
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/client_synctracking_direct_bridge.c
@@ -0,0 +1,92 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for synctracking
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_synctracking_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "sync.h"
+#include "sync_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	pshRecordInt = (SYNC_RECORD_HANDLE) hhRecord;
+
+	eError = PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt;
+
+	pshServerSyncPrimBlockInt = (SYNC_PRIMITIVE_BLOCK *) hhServerSyncPrimBlock;
+
+	eError =
+	    PVRSRVSyncRecordAddKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  ui32ui32FwBlockAddr,
+				  ui32ui32SyncOffset,
+				  bbServerSync, ui32ClassNameSize, puiClassName);
+
+	*phhRecord = pshRecordInt;
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/common_synctracking_bridge.h b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/common_synctracking_bridge.h
new file mode 100644
index 000000000000..9fd9fdb796a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/common_synctracking_bridge.h
@@ -0,0 +1,97 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCTRACKING_BRIDGE_H
+#define COMMON_SYNCTRACKING_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST			(PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1)
+
+/*******************************************
+            SyncRecordRemoveByHandle
+ *******************************************/
+
+/* Bridge in structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	IMG_HANDLE hhRecord;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE;
+
+/* Bridge out structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE;
+
+/*******************************************
+            SyncRecordAdd
+ *******************************************/
+
+/* Bridge in structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhServerSyncPrimBlock;
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32ui32FwBlockAddr;
+	IMG_UINT32 ui32ui32SyncOffset;
+	IMG_BOOL bbServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDADD;
+
+/* Bridge out structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhRecord;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDADD;
+
+#endif /* COMMON_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/server_synctracking_bridge.c b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/server_synctracking_bridge.c
new file mode 100644
index 000000000000..06757e5ef8d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/generated/volcanic/synctracking_bridge/server_synctracking_bridge.c
@@ -0,0 +1,329 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+
+#include "common_synctracking_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeSyncRecordRemoveByHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleIN_UI8,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordRemoveByHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncRecordRemoveByHandleIN->hhRecord,
+					      PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE);
+	if (unlikely((psSyncRecordRemoveByHandleOUT->eError != PVRSRV_OK) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncRecordRemoveByHandleOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordRemoveByHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordRemoveByHandle_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncRecordAddpshRecordIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVSyncRecordRemoveByHandleKM((SYNC_RECORD_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncRecordAdd(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncRecordAddIN_UI8,
+			  IMG_UINT8 * psSyncRecordAddOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDADD *psSyncRecordAddIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDADD *psSyncRecordAddOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddOUT_UI8, 0);
+
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	IMG_HANDLE hhServerSyncPrimBlock = psSyncRecordAddIN->hhServerSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncRecordAddIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncRecordAdd_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncRecordAdd_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncRecordAddIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncRecordAddIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncRecordAddOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncRecordAdd_exit;
+			}
+		}
+	}
+
+	if (psSyncRecordAddIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncRecordAddIN->puiClassName,
+		     psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncRecordAddOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncRecordAdd_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncRecordAddOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pshServerSyncPrimBlockInt,
+				       hhServerSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError =
+	    PVRSRVSyncRecordAddKM(psConnection, OSGetDevNode(psConnection),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  psSyncRecordAddIN->ui32ui32FwBlockAddr,
+				  psSyncRecordAddIN->ui32ui32SyncOffset,
+				  psSyncRecordAddIN->bbServerSync,
+				  psSyncRecordAddIN->ui32ClassNameSize, uiClassNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psSyncRecordAddOUT->hhRecord,
+							       (void *)pshRecordInt,
+							       PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+							       PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+							       (PFN_HANDLE_RELEASE) &
+							       _SyncRecordAddpshRecordIntRelease);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordAdd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pshServerSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hhServerSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psSyncRecordAddOUT->eError != PVRSRV_OK)
+	{
+		if (pshRecordInt)
+		{
+			PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncRecordAddOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+
+/*
+ * Register all SYNCTRACKING functions with services
+ */
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+			      PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE,
+			      PVRSRVBridgeSyncRecordRemoveByHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING, PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD,
+			      PVRSRVBridgeSyncRecordAdd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDADD),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDADD));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all synctracking functions with services
+ */
+void DeinitSYNCTRACKINGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.2.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.2.30.h
new file mode 100644
index 000000000000..88b123174c49
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.2.30.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.2.30 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_2_30_H
+#define RGXCONFIG_KM_1_V_2_30_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 30
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_2_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.19.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.19.h
new file mode 100644
index 000000000000..5f60752630c0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.19.h
@@ -0,0 +1,82 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.4.19 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_4_19_H
+#define RGXCONFIG_KM_1_V_4_19_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 19
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_4_19_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.5.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.5.h
new file mode 100644
index 000000000000..1af637872cf9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_1.V.4.5.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.4.5 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_4_5_H
+#define RGXCONFIG_KM_1_V_4_5_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 5
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_4_5_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_15.V.1.64.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_15.V.1.64.h
new file mode 100644
index 000000000000..3ad2665eb9ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_15.V.1.64.h
@@ -0,0 +1,87 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 15.V.1.64 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_15_V_1_64_H
+#define RGXCONFIG_KM_15_V_1_64_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 15
+#define RGX_BNC_KM_N 1
+#define RGX_BNC_KM_C 64
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP217)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_15_V_1_64_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.18.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.18.h
new file mode 100644
index 000000000000..4594c6429129
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.18.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.104.18 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_104_18_H
+#define RGXCONFIG_KM_22_V_104_18_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 18
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_104_18_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.218.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.218.h
new file mode 100644
index 000000000000..e271c768760d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.104.218.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.104.218 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_104_218_H
+#define RGXCONFIG_KM_22_V_104_218_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 218
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_104_218_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.208.318.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.208.318.h
new file mode 100644
index 000000000000..8b76f2aa6cf0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.208.318.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.208.318 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_208_318_H
+#define RGXCONFIG_KM_22_V_208_318_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 318
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_208_318_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.21.16.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.21.16.h
new file mode 100644
index 000000000000..9d54fdd6a578
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.21.16.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.21.16 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_21_16_H
+#define RGXCONFIG_KM_22_V_21_16_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 21
+#define RGX_BNC_KM_C 16
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (8U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_21_16_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.25.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.25.h
new file mode 100644
index 000000000000..ffe2926153ff
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.25.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.25 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_25_H
+#define RGXCONFIG_KM_22_V_54_25_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 25
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_25_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.30.h
new file mode 100644
index 000000000000..d59101b73ad6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.30.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.30 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_30_H
+#define RGXCONFIG_KM_22_V_54_30_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 30
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.330.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.330.h
new file mode 100644
index 000000000000..fd7547bca34e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.330.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.330 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_330_H
+#define RGXCONFIG_KM_22_V_54_330_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 330
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_330_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.38.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.38.h
new file mode 100644
index 000000000000..6db0a9272d48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_22.V.54.38.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.38 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_38_H
+#define RGXCONFIG_KM_22_V_54_38_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 38
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_38_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.104.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.104.504.h
new file mode 100644
index 000000000000..e79b894f1299
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.104.504.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.104.504 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_104_504_H
+#define RGXCONFIG_KM_24_V_104_504_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 504
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_104_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.504.h
new file mode 100644
index 000000000000..2b92c75a2d80
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.504.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.208.504 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_208_504_H
+#define RGXCONFIG_KM_24_V_208_504_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 504
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_208_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.505.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.505.h
new file mode 100644
index 000000000000..d14c991870e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.208.505.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.208.505 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_208_505_H
+#define RGXCONFIG_KM_24_V_208_505_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 505
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_208_505_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.54.204.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.54.204.h
new file mode 100644
index 000000000000..0274380eafdd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_24.V.54.204.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.54.204 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_54_204_H
+#define RGXCONFIG_KM_24_V_54_204_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 204
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_54_204_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.108.208.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.108.208.h
new file mode 100644
index 000000000000..bae4231b4f9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.108.208.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 29.V.108.208 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_29_V_108_208_H
+#define RGXCONFIG_KM_29_V_108_208_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 29
+#define RGX_BNC_KM_N 108
+#define RGX_BNC_KM_C 208
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_29_V_108_208_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.52.202.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.52.202.h
new file mode 100644
index 000000000000..805e6df66ab3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_29.V.52.202.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 29.V.52.202 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_29_V_52_202_H
+#define RGXCONFIG_KM_29_V_52_202_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 29
+#define RGX_BNC_KM_N 52
+#define RGX_BNC_KM_C 202
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_29_V_52_202_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.11.3.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.11.3.h
new file mode 100644
index 000000000000..19155570aa2e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.11.3.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 33.V.11.3 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_33_V_11_3_H
+#define RGXCONFIG_KM_33_V_11_3_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 33
+#define RGX_BNC_KM_N 11
+#define RGX_BNC_KM_C 3
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (1U)
+#define RGX_FEATURE_NUM_OSIDS (2U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_33_V_11_3_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.22.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.22.1.h
new file mode 100644
index 000000000000..a375f749cf46
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_33.V.22.1.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 33.V.22.1 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_33_V_22_1_H
+#define RGXCONFIG_KM_33_V_22_1_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 33
+#define RGX_BNC_KM_N 22
+#define RGX_BNC_KM_C 1
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_PM_MMU_VFP
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_33_V_22_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.182.h
new file mode 100644
index 000000000000..48084d57d188
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.182.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_182_H
+#define RGXCONFIG_KM_36_V_104_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.183.h
new file mode 100644
index 000000000000..dc4a3590f0d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.183.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.183 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_183_H
+#define RGXCONFIG_KM_36_V_104_183_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 183
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.796.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.796.h
new file mode 100644
index 000000000000..d4f87c4ffa4d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.104.796.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.796 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_796_H
+#define RGXCONFIG_KM_36_V_104_796_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 796
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (2U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SAFETY_IRQ
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_796_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.52.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.52.182.h
new file mode 100644
index 000000000000..049939fbb452
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.52.182.h
@@ -0,0 +1,105 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.52.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_52_182_H
+#define RGXCONFIG_KM_36_V_52_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 52
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_52_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.103.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.103.h
new file mode 100644
index 000000000000..3f8b4ffab362
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.103.h
@@ -0,0 +1,103 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.103 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_103_H
+#define RGXCONFIG_KM_36_V_54_103_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 103
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_36_V_54_103_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.182.h
new file mode 100644
index 000000000000..78b53d52cf4a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.182.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_182_H
+#define RGXCONFIG_KM_36_V_54_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.183.h
new file mode 100644
index 000000000000..d2f5dd4bab1b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.183.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.183 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_183_H
+#define RGXCONFIG_KM_36_V_54_183_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 183
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.280.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.280.h
new file mode 100644
index 000000000000..30bbaa68dcb5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_36.V.54.280.h
@@ -0,0 +1,110 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.280 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_280_H
+#define RGXCONFIG_KM_36_V_54_280_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 280
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (11U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_280_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.51.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.51.h
new file mode 100644
index 000000000000..28114cad8c97
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.51.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.2.51 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_2_51_H
+#define RGXCONFIG_KM_4_V_2_51_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 51
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_2_51_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.58.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.58.h
new file mode 100644
index 000000000000..99ee53e0c197
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.2.58.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.2.58 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_2_58_H
+#define RGXCONFIG_KM_4_V_2_58_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 58
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_2_58_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.4.55.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.4.55.h
new file mode 100644
index 000000000000..c27adf56b55b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.4.55.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.4.55 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_4_55_H
+#define RGXCONFIG_KM_4_V_4_55_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 55
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_4_55_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.6.62.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.6.62.h
new file mode 100644
index 000000000000..2d4aa9599a2b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_4.V.6.62.h
@@ -0,0 +1,93 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.6.62 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_6_62_H
+#define RGXCONFIG_KM_4_V_6_62_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 6
+#define RGX_BNC_KM_C 62
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_6_62_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.102.389.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.102.389.h
new file mode 100644
index 000000000000..a8eab1e08e4d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.102.389.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 46.V.102.389 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_46_V_102_389_H
+#define RGXCONFIG_KM_46_V_102_389_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 46
+#define RGX_BNC_KM_N 102
+#define RGX_BNC_KM_C 389
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (20U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_VOLCANIC_TB
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_46_V_102_389_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.204.390.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.204.390.h
new file mode 100644
index 000000000000..96544ed32b48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_46.V.204.390.h
@@ -0,0 +1,114 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 46.V.204.390 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_46_V_204_390_H
+#define RGXCONFIG_KM_46_V_204_390_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 46
+#define RGX_BNC_KM_N 204
+#define RGX_BNC_KM_C 390
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (20U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_VOLCANIC_TB
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_46_V_204_390_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_5.V.1.46.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_5.V.1.46.h
new file mode 100644
index 000000000000..1c9b7db878d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_5.V.1.46.h
@@ -0,0 +1,85 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 5.V.1.46 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_5_V_1_46_H
+#define RGXCONFIG_KM_5_V_1_46_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 5
+#define RGX_BNC_KM_N 1
+#define RGX_BNC_KM_C 46
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP217)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (1U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_5_V_1_46_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_6.V.4.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_6.V.4.35.h
new file mode 100644
index 000000000000..cbf5589ce79c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/configs/rgxconfig_km_6.V.4.35.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 6.V.4.35 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_6_V_4_35_H
+#define RGXCONFIG_KM_6_V_4_35_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 6
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 35
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_6_V_4_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.39.4.19.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.39.4.19.h
new file mode 100644
index 000000000000..a64cb4f379e9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.39.4.19.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.39.4.19
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_39_4_19_H
+#define RGXCORE_KM_1_39_4_19_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2784771 */
+
+/******************************************************************************
+ * BVNC = 1.39.4.19
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 39
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 19
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344
+#define FIX_HW_BRN_42321
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_39_4_19_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.75.2.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.75.2.30.h
new file mode 100644
index 000000000000..b33edeafe674
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.75.2.30.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.75.2.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_75_2_30_H
+#define RGXCORE_KM_1_75_2_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2309075 */
+
+/******************************************************************************
+ * BVNC = 1.75.2.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 75
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_42321
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_75_2_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.82.4.5.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.82.4.5.h
new file mode 100644
index 000000000000..ae53d7ceef27
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_1.82.4.5.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.82.4.5
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_82_4_5_H
+#define RGXCORE_KM_1_82_4_5_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2503111 */
+
+/******************************************************************************
+ * BVNC = 1.82.4.5
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 82
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 5
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_82_4_5_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_15.5.1.64.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_15.5.1.64.h
new file mode 100644
index 000000000000..e354f5b8c8fe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_15.5.1.64.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 15.5.1.64
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_15_5_1_64_H
+#define RGXCORE_KM_15_5_1_64_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3846532 */
+
+/******************************************************************************
+ * BVNC = 15.5.1.64
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 15
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 1
+#define RGX_BVNC_KM_C 64
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_15_5_1_64_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.102.54.38.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.102.54.38.h
new file mode 100644
index 000000000000..ee279a63a255
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.102.54.38.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.102.54.38
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_102_54_38_H
+#define RGXCORE_KM_22_102_54_38_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4831550 */
+
+/******************************************************************************
+ * BVNC = 22.102.54.38
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 102
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 38
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_102_54_38_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.104.208.318.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.104.208.318.h
new file mode 100644
index 000000000000..58472828a1b0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.104.208.318.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.104.208.318
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_104_208_318_H
+#define RGXCORE_KM_22_104_208_318_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5124208 */
+
+/******************************************************************************
+ * BVNC = 22.104.208.318
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 104
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 318
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_104_208_318_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.105.208.318.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.105.208.318.h
new file mode 100644
index 000000000000..d175ac4a8a87
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.105.208.318.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.105.208.318
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_105_208_318_H
+#define RGXCORE_KM_22_105_208_318_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5324554 */
+
+/******************************************************************************
+ * BVNC = 22.105.208.318
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 105
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 318
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_105_208_318_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.30.54.25.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.30.54.25.h
new file mode 100644
index 000000000000..34d2c176900a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.30.54.25.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.30.54.25
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_30_54_25_H
+#define RGXCORE_KM_22_30_54_25_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4086500 */
+
+/******************************************************************************
+ * BVNC = 22.30.54.25
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 30
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 25
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_30_54_25_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.40.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.40.54.30.h
new file mode 100644
index 000000000000..f9dfea7676f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.40.54.30.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.40.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_40_54_30_H
+#define RGXCORE_KM_22_40_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4094817 */
+
+/******************************************************************************
+ * BVNC = 22.40.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 40
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_40_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.46.54.330.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.46.54.330.h
new file mode 100644
index 000000000000..f269fc0c8a12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.46.54.330.h
@@ -0,0 +1,78 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.46.54.330
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_46_54_330_H
+#define RGXCORE_KM_22_46_54_330_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4136505 */
+
+/******************************************************************************
+ * BVNC = 22.46.54.330
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 46
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 330
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_22_46_54_330_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.49.21.16.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.49.21.16.h
new file mode 100644
index 000000000000..f1cc4df22992
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.49.21.16.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.49.21.16
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_49_21_16_H
+#define RGXCORE_KM_22_49_21_16_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4158766 */
+
+/******************************************************************************
+ * BVNC = 22.49.21.16
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 49
+#define RGX_BVNC_KM_N 21
+#define RGX_BVNC_KM_C 16
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_49_21_16_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.67.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.67.54.30.h
new file mode 100644
index 000000000000..c60d20b87718
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.67.54.30.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.67.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_67_54_30_H
+#define RGXCORE_KM_22_67_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4339986 */
+
+/******************************************************************************
+ * BVNC = 22.67.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 67
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_67_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.68.54.30.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.68.54.30.h
new file mode 100644
index 000000000000..b5f8459a1442
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.68.54.30.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.68.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_68_54_30_H
+#define RGXCORE_KM_22_68_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4339984 */
+
+/******************************************************************************
+ * BVNC = 22.68.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 68
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_68_54_30_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.86.104.218.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.86.104.218.h
new file mode 100644
index 000000000000..a0c3001f1147
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.86.104.218.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.86.104.218
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_86_104_218_H
+#define RGXCORE_KM_22_86_104_218_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4665024 */
+
+/******************************************************************************
+ * BVNC = 22.86.104.218
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 86
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 218
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_86_104_218_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.87.104.18.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.87.104.18.h
new file mode 100644
index 000000000000..2a2c66ad95d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_22.87.104.18.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.87.104.18
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_87_104_18_H
+#define RGXCORE_KM_22_87_104_18_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4658768 */
+
+/******************************************************************************
+ * BVNC = 22.87.104.18
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 87
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 18
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_87_104_18_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.50.208.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.50.208.504.h
new file mode 100644
index 000000000000..1aca3f49a04c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.50.208.504.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.50.208.504
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_50_208_504_H
+#define RGXCORE_KM_24_50_208_504_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5086680 */
+
+/******************************************************************************
+ * BVNC = 24.50.208.504
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 50
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 504
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_50_208_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.56.208.505.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.56.208.505.h
new file mode 100644
index 000000000000..26060f932d47
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.56.208.505.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.56.208.505
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_56_208_505_H
+#define RGXCORE_KM_24_56_208_505_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5203837 */
+
+/******************************************************************************
+ * BVNC = 24.56.208.505
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 56
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 505
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_56_208_505_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.66.54.204.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.66.54.204.h
new file mode 100644
index 000000000000..4fb0ef0395b3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.66.54.204.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.66.54.204
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_66_54_204_H
+#define RGXCORE_KM_24_66_54_204_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5200207 */
+
+/******************************************************************************
+ * BVNC = 24.66.54.204
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 66
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 204
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_66_54_204_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.67.104.504.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.67.104.504.h
new file mode 100644
index 000000000000..4266f63edac9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_24.67.104.504.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.67.104.504
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_67_104_504_H
+#define RGXCORE_KM_24_67_104_504_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5203838 */
+
+/******************************************************************************
+ * BVNC = 24.67.104.504
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 67
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 504
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_67_104_504_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.14.108.208.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.14.108.208.h
new file mode 100644
index 000000000000..6636f54022c7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.14.108.208.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 29.14.108.208
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_29_14_108_208_H
+#define RGXCORE_KM_29_14_108_208_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5682134 */
+
+/******************************************************************************
+ * BVNC = 29.14.108.208
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 29
+#define RGX_BVNC_KM_V 14
+#define RGX_BVNC_KM_N 108
+#define RGX_BVNC_KM_C 208
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_68186
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_29_14_108_208_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.19.52.202.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.19.52.202.h
new file mode 100644
index 000000000000..01683cb721ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_29.19.52.202.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 29.19.52.202
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_29_19_52_202_H
+#define RGXCORE_KM_29_19_52_202_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5691248 */
+
+/******************************************************************************
+ * BVNC = 29.19.52.202
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 29
+#define RGX_BVNC_KM_V 19
+#define RGX_BVNC_KM_N 52
+#define RGX_BVNC_KM_C 202
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_68186
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_29_19_52_202_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.15.11.3.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.15.11.3.h
new file mode 100644
index 000000000000..9762fa901bda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.15.11.3.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 33.15.11.3
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_33_15_11_3_H
+#define RGXCORE_KM_33_15_11_3_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5820045 */
+
+/******************************************************************************
+ * BVNC = 33.15.11.3
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 33
+#define RGX_BVNC_KM_V 15
+#define RGX_BVNC_KM_N 11
+#define RGX_BVNC_KM_C 3
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_33_15_11_3_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.8.22.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.8.22.1.h
new file mode 100644
index 000000000000..272e922e0191
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_33.8.22.1.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 33.8.22.1
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_33_8_22_1_H
+#define RGXCORE_KM_33_8_22_1_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5544349 */
+
+/******************************************************************************
+ * BVNC = 33.8.22.1
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 33
+#define RGX_BVNC_KM_V 8
+#define RGX_BVNC_KM_N 22
+#define RGX_BVNC_KM_C 1
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_33_8_22_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.29.52.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.29.52.182.h
new file mode 100644
index 000000000000..7a5a87c84f7b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.29.52.182.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.29.52.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_29_52_182_H
+#define RGXCORE_KM_36_29_52_182_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908879 */
+
+/******************************************************************************
+ * BVNC = 36.29.52.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 29
+#define RGX_BVNC_KM_N 52
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_29_52_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.50.54.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.50.54.182.h
new file mode 100644
index 000000000000..0eccc97305ab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.50.54.182.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.50.54.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_50_54_182_H
+#define RGXCORE_KM_36_50_54_182_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5849603 */
+
+/******************************************************************************
+ * BVNC = 36.50.54.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 50
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_50_54_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.52.104.182.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.52.104.182.h
new file mode 100644
index 000000000000..d14b662b9730
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.52.104.182.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.52.104.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_52_104_182_H
+#define RGXCORE_KM_36_52_104_182_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5849605 */
+
+/******************************************************************************
+ * BVNC = 36.52.104.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 52
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_52_104_182_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.53.104.796.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.53.104.796.h
new file mode 100644
index 000000000000..c3ec305d3830
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.53.104.796.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.53.104.796
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_53_104_796_H
+#define RGXCORE_KM_36_53_104_796_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5896094 */
+
+/******************************************************************************
+ * BVNC = 36.53.104.796
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 53
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 796
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_53_104_796_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.54.54.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.54.54.183.h
new file mode 100644
index 000000000000..6cac1b26d084
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.54.54.183.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.54.54.183
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_54_54_183_H
+#define RGXCORE_KM_36_54_54_183_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908021 */
+
+/******************************************************************************
+ * BVNC = 36.54.54.183
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 54
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 183
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_54_54_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.55.54.103.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.55.54.103.h
new file mode 100644
index 000000000000..809d5808d3ea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.55.54.103.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.55.54.103
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_55_54_103_H
+#define RGXCORE_KM_36_55_54_103_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908021 */
+
+/******************************************************************************
+ * BVNC = 36.55.54.103
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 55
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 103
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_55_54_103_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.56.104.183.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.56.104.183.h
new file mode 100644
index 000000000000..96e875e3b1f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.56.104.183.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.56.104.183
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_56_104_183_H
+#define RGXCORE_KM_36_56_104_183_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5942195 */
+
+/******************************************************************************
+ * BVNC = 36.56.104.183
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 56
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 183
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_56_104_183_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.60.54.280.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.60.54.280.h
new file mode 100644
index 000000000000..5f46e75d4f06
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_36.60.54.280.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.60.54.280
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_60_54_280_H
+#define RGXCORE_KM_36_60_54_280_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @6040409 */
+
+/******************************************************************************
+ * BVNC = 36.60.54.280
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 60
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 280
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_60_54_280_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.31.4.55.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.31.4.55.h
new file mode 100644
index 000000000000..a39603ae35a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.31.4.55.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.31.4.55
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_31_4_55_H
+#define RGXCORE_KM_4_31_4_55_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2919104 */
+
+/******************************************************************************
+ * BVNC = 4.31.4.55
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 31
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 55
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_31_4_55_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.40.2.51.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.40.2.51.h
new file mode 100644
index 000000000000..931fa0cd445c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.40.2.51.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.40.2.51
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_40_2_51_H
+#define RGXCORE_KM_4_40_2_51_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3254374 */
+
+/******************************************************************************
+ * BVNC = 4.40.2.51
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 40
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 51
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_40_2_51_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.43.6.62.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.43.6.62.h
new file mode 100644
index 000000000000..90d35a8360bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.43.6.62.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.43.6.62
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_43_6_62_H
+#define RGXCORE_KM_4_43_6_62_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3253129 */
+
+/******************************************************************************
+ * BVNC = 4.43.6.62
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 43
+#define RGX_BVNC_KM_N 6
+#define RGX_BVNC_KM_C 62
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_43_6_62_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.45.2.58.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.45.2.58.h
new file mode 100644
index 000000000000..00e6b90f6207
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.45.2.58.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.45.2.58
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_45_2_58_H
+#define RGXCORE_KM_4_45_2_58_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3547765 */
+
+/******************************************************************************
+ * BVNC = 4.45.2.58
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 45
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 58
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_45_2_58_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.46.6.62.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.46.6.62.h
new file mode 100644
index 000000000000..909198f233b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_4.46.6.62.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.46.6.62
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_46_6_62_H
+#define RGXCORE_KM_4_46_6_62_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4015666 */
+
+/******************************************************************************
+ * BVNC = 4.46.6.62
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 46
+#define RGX_BVNC_KM_N 6
+#define RGX_BVNC_KM_C 62
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_46_6_62_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.72.102.389.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.72.102.389.h
new file mode 100644
index 000000000000..66eb259c0ebf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.72.102.389.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 46.72.102.389
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_46_72_102_389_H
+#define RGXCORE_KM_46_72_102_389_H
+
+/* Automatically generated file (07/02/2023 09:11:39): Do not edit manually */
+/* CS: @6385487 */
+
+/******************************************************************************
+ * BVNC = 46.72.102.389
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 46
+#define RGX_BVNC_KM_V 72
+#define RGX_BVNC_KM_N 102
+#define RGX_BVNC_KM_C 389
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_66927
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_46_72_102_389_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.94.204.390.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.94.204.390.h
new file mode 100644
index 000000000000..548989eb518d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_46.94.204.390.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 46.94.204.390
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_46_94_204_390_H
+#define RGXCORE_KM_46_94_204_390_H
+
+/* Automatically generated file (02/03/2023 09:11:42): Do not edit manually */
+/* CS: @6389604 */
+
+/******************************************************************************
+ * BVNC = 46.94.204.390
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 46
+#define RGX_BVNC_KM_V 94
+#define RGX_BVNC_KM_N 204
+#define RGX_BVNC_KM_C 390
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_66927
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_46_94_204_390_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_5.9.1.46.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_5.9.1.46.h
new file mode 100644
index 000000000000..cba592e7b327
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_5.9.1.46.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 5.9.1.46
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_5_9_1_46_H
+#define RGXCORE_KM_5_9_1_46_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2967148 */
+
+/******************************************************************************
+ * BVNC = 5.9.1.46
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 5
+#define RGX_BVNC_KM_V 9
+#define RGX_BVNC_KM_N 1
+#define RGX_BVNC_KM_C 46
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_5_9_1_46_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_6.34.4.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_6.34.4.35.h
new file mode 100644
index 000000000000..b45ed9e1547c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/cores/rgxcore_km_6.34.4.35.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 6.34.4.35
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_6_34_4_35_H
+#define RGXCORE_KM_6_34_4_35_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3533654 */
+
+/******************************************************************************
+ * BVNC = 6.34.4.35
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 6
+#define RGX_BVNC_KM_V 34
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 35
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_6_34_4_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_defs_km.h
new file mode 100644
index 000000000000..73c44b77f498
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_defs_km.h
@@ -0,0 +1,341 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_defs_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_DEFS_KM_H
+#define RGX_BVNC_DEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(RGX_BVNC_DEFS_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_defs_um.h"
+#endif
+
+#define BVNC_FIELD_WIDTH (16U)
+
+#define PVR_ARCH_NAME "rogue"
+
+
+/******************************************************************************
+ * Mask and bit-position macros for features without values
+ *****************************************************************************/
+
+#define RGX_FEATURE_AXI_ACELITE_POS                                     (0U)
+#define RGX_FEATURE_AXI_ACELITE_BIT_MASK                                (IMG_UINT64_C(0x0000000000000001))
+
+#define RGX_FEATURE_CLUSTER_GROUPING_POS                                (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK                           (IMG_UINT64_C(0x0000000000000002))
+
+#define RGX_FEATURE_COMPUTE_POS                                         (2U)
+#define RGX_FEATURE_COMPUTE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000000004))
+
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_POS                          (3U)
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_BIT_MASK                     (IMG_UINT64_C(0x0000000000000008))
+
+#define RGX_FEATURE_COMPUTE_OVERLAP_POS                                 (4U)
+#define RGX_FEATURE_COMPUTE_OVERLAP_BIT_MASK                            (IMG_UINT64_C(0x0000000000000010))
+
+#define RGX_FEATURE_COREID_PER_OS_POS                                   (5U)
+#define RGX_FEATURE_COREID_PER_OS_BIT_MASK                              (IMG_UINT64_C(0x0000000000000020))
+
+#define RGX_FEATURE_DYNAMIC_DUST_POWER_POS                              (6U)
+#define RGX_FEATURE_DYNAMIC_DUST_POWER_BIT_MASK                         (IMG_UINT64_C(0x0000000000000040))
+
+#define RGX_FEATURE_FASTRENDER_DM_POS                                   (7U)
+#define RGX_FEATURE_FASTRENDER_DM_BIT_MASK                              (IMG_UINT64_C(0x0000000000000080))
+
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_POS                           (8U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_BIT_MASK                      (IMG_UINT64_C(0x0000000000000100))
+
+#define RGX_FEATURE_GPU_VIRTUALISATION_POS                              (9U)
+#define RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK                         (IMG_UINT64_C(0x0000000000000200))
+
+#define RGX_FEATURE_GS_RTA_SUPPORT_POS                                  (10U)
+#define RGX_FEATURE_GS_RTA_SUPPORT_BIT_MASK                             (IMG_UINT64_C(0x0000000000000400))
+
+#define RGX_FEATURE_IRQ_PER_OS_POS                                      (11U)
+#define RGX_FEATURE_IRQ_PER_OS_BIT_MASK                                 (IMG_UINT64_C(0x0000000000000800))
+
+#define RGX_FEATURE_MIPS_POS                                            (12U)
+#define RGX_FEATURE_MIPS_BIT_MASK                                       (IMG_UINT64_C(0x0000000000001000))
+
+#define RGX_FEATURE_PBE2_IN_XE_POS                                      (13U)
+#define RGX_FEATURE_PBE2_IN_XE_BIT_MASK                                 (IMG_UINT64_C(0x0000000000002000))
+
+#define RGX_FEATURE_PBE_CHECKSUM_2D_POS                                 (14U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D_BIT_MASK                            (IMG_UINT64_C(0x0000000000004000))
+
+#define RGX_FEATURE_PBVNC_COREID_REG_POS                                (15U)
+#define RGX_FEATURE_PBVNC_COREID_REG_BIT_MASK                           (IMG_UINT64_C(0x0000000000008000))
+
+#define RGX_FEATURE_PERFBUS_POS                                         (16U)
+#define RGX_FEATURE_PERFBUS_BIT_MASK                                    (IMG_UINT64_C(0x0000000000010000))
+
+#define RGX_FEATURE_PERF_COUNTER_BATCH_POS                              (17U)
+#define RGX_FEATURE_PERF_COUNTER_BATCH_BIT_MASK                         (IMG_UINT64_C(0x0000000000020000))
+
+#define RGX_FEATURE_PM_MMU_VFP_POS                                      (18U)
+#define RGX_FEATURE_PM_MMU_VFP_BIT_MASK                                 (IMG_UINT64_C(0x0000000000040000))
+
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_POS                              (19U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_BIT_MASK                         (IMG_UINT64_C(0x0000000000080000))
+
+#define RGX_FEATURE_ROGUEXE_POS                                         (20U)
+#define RGX_FEATURE_ROGUEXE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000100000))
+
+#define RGX_FEATURE_SAFETY_IRQ_POS                                      (21U)
+#define RGX_FEATURE_SAFETY_IRQ_BIT_MASK                                 (IMG_UINT64_C(0x0000000000200000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_POS                (22U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_BIT_MASK           (IMG_UINT64_C(0x0000000000400000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1_POS             (23U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1_BIT_MASK        (IMG_UINT64_C(0x0000000000800000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2_POS             (24U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2_BIT_MASK        (IMG_UINT64_C(0x0000000001000000))
+
+#define RGX_FEATURE_SINGLE_BIF_POS                                      (25U)
+#define RGX_FEATURE_SINGLE_BIF_BIT_MASK                                 (IMG_UINT64_C(0x0000000002000000))
+
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE_POS                           (26U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE_BIT_MASK                      (IMG_UINT64_C(0x0000000004000000))
+
+#define RGX_FEATURE_SOC_TIMER_POS                                       (27U)
+#define RGX_FEATURE_SOC_TIMER_BIT_MASK                                  (IMG_UINT64_C(0x0000000008000000))
+
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_POS                            (28U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_BIT_MASK                       (IMG_UINT64_C(0x0000000010000000))
+
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_POS                                (29U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_BIT_MASK                           (IMG_UINT64_C(0x0000000020000000))
+
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION_POS                          (30U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK                     (IMG_UINT64_C(0x0000000040000000))
+
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT_POS                           (31U)
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK                      (IMG_UINT64_C(0x0000000080000000))
+
+#define RGX_FEATURE_TFBC_NATIVE_YUV10_POS                               (32U)
+#define RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK                          (IMG_UINT64_C(0x0000000100000000))
+
+#define RGX_FEATURE_TILE_REGION_PROTECTION_POS                          (33U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION_BIT_MASK                     (IMG_UINT64_C(0x0000000200000000))
+
+#define RGX_FEATURE_TLA_POS                                             (34U)
+#define RGX_FEATURE_TLA_BIT_MASK                                        (IMG_UINT64_C(0x0000000400000000))
+
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_POS             (35U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_BIT_MASK        (IMG_UINT64_C(0x0000000800000000))
+
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_POS                         (36U)
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_BIT_MASK                    (IMG_UINT64_C(0x0000001000000000))
+
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_POS                      (37U)
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK                 (IMG_UINT64_C(0x0000002000000000))
+
+#define RGX_FEATURE_VOLCANIC_TB_POS                                     (38U)
+#define RGX_FEATURE_VOLCANIC_TB_BIT_MASK                                (IMG_UINT64_C(0x0000004000000000))
+
+#define RGX_FEATURE_WATCHDOG_TIMER_POS                                  (39U)
+#define RGX_FEATURE_WATCHDOG_TIMER_BIT_MASK                             (IMG_UINT64_C(0x0000008000000000))
+
+#define RGX_FEATURE_WORKGROUP_PROTECTION_POS                            (40U)
+#define RGX_FEATURE_WORKGROUP_PROTECTION_BIT_MASK                       (IMG_UINT64_C(0x0000010000000000))
+
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY_POS                             (41U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY_BIT_MASK                        (IMG_UINT64_C(0x0000020000000000))
+
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE_POS                           (42U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE_BIT_MASK                      (IMG_UINT64_C(0x0000040000000000))
+
+
+/******************************************************************************
+ * Defines for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX             (2U)
+#define RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX                              (3U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX                  (2U)
+#define RGX_FEATURE_FBCDC_MAX_VALUE_IDX                                 (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX                       (6U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX                    (4U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX           (2U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX             (2U)
+#define RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_META_MAX_VALUE_IDX                                  (4U)
+#define RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX                     (3U)
+#define RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX                          (5U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX                     (9U)
+#define RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX                            (2U)
+#define RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX                             (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX                        (4U)
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX       (3U)
+#define RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX                             (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX              (2U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX                 (6U)
+#define RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX                          (2U)
+#define RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX                          (4U)
+#define RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX            (2U)
+#define RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX                       (2U)
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX           (2U)
+#define RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX                        (2U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX                (2U)
+
+/******************************************************************************
+ * Features with values indexes
+ *****************************************************************************/
+
+typedef enum _RGX_FEATURE_WITH_VALUE_INDEX_ {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_IDX,
+	RGX_FEATURE_ECC_RAMS_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_IDX,
+	RGX_FEATURE_FBCDC_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_IDX,
+	RGX_FEATURE_LAYOUT_MARS_IDX,
+	RGX_FEATURE_META_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX,
+	RGX_FEATURE_NUM_MEMBUS_IDX,
+	RGX_FEATURE_NUM_OSIDS_IDX,
+	RGX_FEATURE_NUM_RASTER_PIPES_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_IDX,
+	RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_IDX,
+	RGX_FEATURE_SLC_BANKS_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_IDX,
+	RGX_FEATURE_TFBC_VERSION_IDX,
+	RGX_FEATURE_TILE_SIZE_X_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_IDX,
+	RGX_FEATURE_XE_ARCHITECTURE_IDX,
+	RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_IDX,
+	RGX_FEATURE_XPU_MAX_SLAVES_IDX,
+	RGX_FEATURE_XPU_REGISTER_BROADCAST_IDX,
+	RGX_FEATURE_WITH_VALUES_MAX_IDX,
+} RGX_FEATURE_WITH_VALUE_INDEX;
+
+
+/******************************************************************************
+ * Mask and bit-position macros for ERNs and BRNs
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344_POS                                            (0U)
+#define FIX_HW_BRN_38344_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000001))
+
+#define HW_ERN_42290_POS                                                (1U)
+#define HW_ERN_42290_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000002))
+
+#define FIX_HW_BRN_42321_POS                                            (2U)
+#define FIX_HW_BRN_42321_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000004))
+
+#define HW_ERN_42606_POS                                                (3U)
+#define HW_ERN_42606_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000008))
+
+#define HW_ERN_47025_POS                                                (4U)
+#define HW_ERN_47025_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000010))
+
+#define FIX_HW_BRN_50767_POS                                            (5U)
+#define FIX_HW_BRN_50767_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000020))
+
+#define HW_ERN_57596_POS                                                (6U)
+#define HW_ERN_57596_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000040))
+
+#define FIX_HW_BRN_60084_POS                                            (7U)
+#define FIX_HW_BRN_60084_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000080))
+
+#define HW_ERN_61389_POS                                                (8U)
+#define HW_ERN_61389_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000100))
+
+#define FIX_HW_BRN_61450_POS                                            (9U)
+#define FIX_HW_BRN_61450_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000200))
+
+#define FIX_HW_BRN_63142_POS                                            (10U)
+#define FIX_HW_BRN_63142_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000400))
+
+#define FIX_HW_BRN_63553_POS                                            (11U)
+#define FIX_HW_BRN_63553_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000800))
+
+#define FIX_HW_BRN_64502_POS                                            (12U)
+#define FIX_HW_BRN_64502_BIT_MASK                                       (IMG_UINT64_C(0x0000000000001000))
+
+#define FIX_HW_BRN_65101_POS                                            (13U)
+#define FIX_HW_BRN_65101_BIT_MASK                                       (IMG_UINT64_C(0x0000000000002000))
+
+#define FIX_HW_BRN_65273_POS                                            (14U)
+#define FIX_HW_BRN_65273_BIT_MASK                                       (IMG_UINT64_C(0x0000000000004000))
+
+#define HW_ERN_66622_POS                                                (15U)
+#define HW_ERN_66622_BIT_MASK                                           (IMG_UINT64_C(0x0000000000008000))
+
+#define FIX_HW_BRN_66927_POS                                            (16U)
+#define FIX_HW_BRN_66927_BIT_MASK                                       (IMG_UINT64_C(0x0000000000010000))
+
+#define FIX_HW_BRN_68186_POS                                            (17U)
+#define FIX_HW_BRN_68186_BIT_MASK                                       (IMG_UINT64_C(0x0000000000020000))
+
+#define FIX_HW_BRN_71242_POS                                            (18U)
+#define FIX_HW_BRN_71242_BIT_MASK                                       (IMG_UINT64_C(0x0000000000040000))
+
+#define FIX_HW_BRN_71317_POS                                            (19U)
+#define FIX_HW_BRN_71317_BIT_MASK                                       (IMG_UINT64_C(0x0000000000080000))
+
+/* Macro used for padding the unavailable values for features with values */
+#define RGX_FEATURE_VALUE_INVALID	(0xFFFFFFFEU)
+
+/* Macro used for marking a feature with value as disabled for a specific bvnc */
+#define RGX_FEATURE_VALUE_DISABLED	(0xFFFFFFFFU)
+
+#endif /* RGX_BVNC_DEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_table_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_table_km.h
new file mode 100644
index 000000000000..d10a0a1ab22d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_bvnc_table_km.h
@@ -0,0 +1,458 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_table_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_TABLE_KM_H
+#define RGX_BVNC_TABLE_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+
+#ifndef RGXBVNC_C
+#error "This file should only be included from rgxbvnc.c"
+#endif
+
+#if defined(RGX_BVNC_TABLE_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_table_um.h"
+#endif
+
+
+/******************************************************************************
+ * Arrays for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values[RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_ECC_RAMS_values[RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values[RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_values[RGX_FEATURE_FBCDC_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, 4, 50, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ALGORITHM_values[RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 50, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values[RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 7, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_LAYOUT_MARS_values[RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_values[RGX_FEATURE_META_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, LTP217, LTP218, MTP218, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_COREMEM_SIZE_values[RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_CLUSTERS_values[RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, 6, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values[RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 6, 7, 8, 12, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_MEMBUS_values[RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_OSIDS_values[RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_RASTER_PIPES_values[RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values[RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 32, 36, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_values[RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_BANKS_values[RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values[RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 512, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values[RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 8, 16, 64, 128, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TB_GPU_COUNT_values[RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TFBC_VERSION_values[RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 10, 11, 20, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_X_values[RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_Y_values[RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values[RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XE_ARCHITECTURE_values[RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_values[RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 19, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_MAX_SLAVES_values[RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_REGISTER_BROADCAST_values[RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+
+/******************************************************************************
+ * Table contains pointers to each feature value array for features that have
+ * values.
+ * Indexed using enum RGX_FEATURE_WITH_VALUE_INDEX from rgx_bvnc_defs_km.h
+ *****************************************************************************/
+
+static const void * const gaFeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX] = {
+	aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values,
+	aui16_RGX_FEATURE_ECC_RAMS_values,
+	aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values,
+	aui16_RGX_FEATURE_FBCDC_values,
+	aui16_RGX_FEATURE_FBCDC_ALGORITHM_values,
+	aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_LAYOUT_MARS_values,
+	aui16_RGX_FEATURE_META_values,
+	aui16_RGX_FEATURE_META_COREMEM_SIZE_values,
+	aui16_RGX_FEATURE_NUM_CLUSTERS_values,
+	aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values,
+	aui16_RGX_FEATURE_NUM_MEMBUS_values,
+	aui16_RGX_FEATURE_NUM_OSIDS_values,
+	aui16_RGX_FEATURE_NUM_RASTER_PIPES_values,
+	aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values,
+	aui16_RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_values,
+	aui16_RGX_FEATURE_SLC_BANKS_values,
+	aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values,
+	aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values,
+	aui16_RGX_FEATURE_TB_GPU_COUNT_values,
+	aui16_RGX_FEATURE_TFBC_VERSION_values,
+	aui16_RGX_FEATURE_TILE_SIZE_X_values,
+	aui16_RGX_FEATURE_TILE_SIZE_Y_values,
+	aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values,
+	aui16_RGX_FEATURE_XE_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_values,
+	aui16_RGX_FEATURE_XPU_MAX_SLAVES_values,
+	aui16_RGX_FEATURE_XPU_REGISTER_BROADCAST_values,
+};
+
+
+/******************************************************************************
+ * Array containing the lengths of the arrays containing the values.
+ * Used for indexing the aui16_<FEATURE>_values defined upwards
+ *****************************************************************************/
+
+
+static const IMG_UINT16 gaFeaturesValuesMaxIndexes[] = {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX,
+	RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX,
+	RGX_FEATURE_META_MAX_VALUE_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX,
+	RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX,
+};
+
+#define RGX_FEATURE_VALUE_TYPE_UINT16 (0x0000U)
+#define RGX_FEATURE_VALUE_TYPE_UINT32 (0x8000U)
+#define RGX_FEATURE_TYPE_BIT_SHIFT 14
+
+/******************************************************************************
+ * Bit-positions for features with values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16FeaturesWithValuesBitPositions[] = {
+	(0U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_POS */
+	(2U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_ECC_RAMS_POS */
+	(4U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FAULT_DECODE_VERSION_POS */
+	(6U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_POS */
+	(9U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ALGORITHM_POS */
+	(12U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ARCHITECTURE_POS */
+	(15U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_POS */
+	(17U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_POS */
+	(19U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_LAYOUT_MARS_POS */
+	(21U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_POS */
+	(24U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_COREMEM_SIZE_POS */
+	(26U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_CLUSTERS_POS */
+	(29U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_ISP_IPP_PIPES_POS */
+	(33U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_MEMBUS_POS */
+	(35U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_OSIDS_POS */
+	(37U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_RASTER_PIPES_POS */
+	(39U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PHYS_BUS_WIDTH_POS */
+	(42U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_POS */
+	(44U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_BANKS_POS */
+	(47U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_POS */
+	(49U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_POS */
+	(52U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TB_GPU_COUNT_POS */
+	(54U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TFBC_VERSION_POS */
+	(57U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_X_POS */
+	(59U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_Y_POS */
+	(61U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_POS */
+	(64U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XE_ARCHITECTURE_POS */
+	(66U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_POS */
+	(68U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_MAX_SLAVES_POS */
+	(70U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_REGISTER_BROADCAST_POS */
+};
+
+
+/******************************************************************************
+ * Bit-masks for features with values
+ *****************************************************************************/
+
+static const IMG_UINT64 aui64FeaturesWithValuesBitMasks[] = {
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_ECC_RAMS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_FAULT_DECODE_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000001C0)), /* RGX_FEATURE_FBCDC_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000E00)), /* RGX_FEATURE_FBCDC_ALGORITHM_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000007000)), /* RGX_FEATURE_FBCDC_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000018000)), /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000060000)), /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000180000)), /* RGX_FEATURE_LAYOUT_MARS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000E00000)), /* RGX_FEATURE_META_BIT_MASK */
+	(IMG_UINT64_C(0x0000000003000000)), /* RGX_FEATURE_META_COREMEM_SIZE_BIT_MASK */
+	(IMG_UINT64_C(0x000000001C000000)), /* RGX_FEATURE_NUM_CLUSTERS_BIT_MASK */
+	(IMG_UINT64_C(0x00000001E0000000)), /* RGX_FEATURE_NUM_ISP_IPP_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000000600000000)), /* RGX_FEATURE_NUM_MEMBUS_BIT_MASK */
+	(IMG_UINT64_C(0x0000001800000000)), /* RGX_FEATURE_NUM_OSIDS_BIT_MASK */
+	(IMG_UINT64_C(0x0000006000000000)), /* RGX_FEATURE_NUM_RASTER_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000038000000000)), /* RGX_FEATURE_PHYS_BUS_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x00000C0000000000)), /* RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000700000000000)), /* RGX_FEATURE_SLC_BANKS_BIT_MASK */
+	(IMG_UINT64_C(0x0001800000000000)), /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x000E000000000000)), /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_BIT_MASK */
+	(IMG_UINT64_C(0x0030000000000000)), /* RGX_FEATURE_TB_GPU_COUNT_BIT_MASK */
+	(IMG_UINT64_C(0x01C0000000000000)), /* RGX_FEATURE_TFBC_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0600000000000000)), /* RGX_FEATURE_TILE_SIZE_X_BIT_MASK */
+	(IMG_UINT64_C(0x1800000000000000)), /* RGX_FEATURE_TILE_SIZE_Y_BIT_MASK */
+	(IMG_UINT64_C(0x6000000000000000)), /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_XE_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_XPU_MAX_SLAVES_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000000C0)), /* RGX_FEATURE_XPU_REGISTER_BROADCAST_BIT_MASK */
+};
+
+
+/******************************************************************************
+ * Table mapping bitmasks for features and features with values
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaFeatures[][4]=
+{
+	{ IMG_UINT64_C(0x0001000000040005), IMG_UINT64_C(0x0000000400010414), IMG_UINT64_C(0x340a81a06d6a9211), IMG_UINT64_C(0x0000000000000000) },	/* 1.0.4.5 */
+	{ IMG_UINT64_C(0x0001000000040013), IMG_UINT64_C(0x0000000400010415), IMG_UINT64_C(0x340a81a06d6a9211), IMG_UINT64_C(0x0000000000000000) },	/* 1.0.4.19 */
+	{ IMG_UINT64_C(0x0004000000020033), IMG_UINT64_C(0x0000042c0803045f), IMG_UINT64_C(0x340a81a0ea4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.2.51 */
+	{ IMG_UINT64_C(0x000400000002003a), IMG_UINT64_C(0x0000042c0803065f), IMG_UINT64_C(0x340a81b0ea4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.2.58 */
+	{ IMG_UINT64_C(0x0004000000040037), IMG_UINT64_C(0x0000042c0803045e), IMG_UINT64_C(0x340a81a0ee4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.4.55 */
+	{ IMG_UINT64_C(0x000400000006003e), IMG_UINT64_C(0x0000042c0803065f), IMG_UINT64_C(0x340ab1d0f24aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.6.62 */
+	{ IMG_UINT64_C(0x000500000001002e), IMG_UINT64_C(0x0000000008110445), IMG_UINT64_C(0x340691a0252a9411), IMG_UINT64_C(0x0000000000000001) },	/* 5.0.1.46 */
+	{ IMG_UINT64_C(0x0006000000040023), IMG_UINT64_C(0x0000042c0803045f), IMG_UINT64_C(0x340a81a0ee4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 6.0.4.35 */
+	{ IMG_UINT64_C(0x000f000000010040), IMG_UINT64_C(0x0000000008110645), IMG_UINT64_C(0x340891b0452a9411), IMG_UINT64_C(0x0000000000000001) },	/* 15.0.1.64 */
+	{ IMG_UINT64_C(0x0016000000150010), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0494b044080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.21.16 */
+	{ IMG_UINT64_C(0x0016000000360019), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b064080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.25 */
+	{ IMG_UINT64_C(0x001600000036001e), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b084080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.30 */
+	{ IMG_UINT64_C(0x0016000000360026), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a08953084080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.38 */
+	{ IMG_UINT64_C(0x001600000036014a), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b0840a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.330 */
+	{ IMG_UINT64_C(0x0016000000680012), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a089530c4080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.104.18 */
+	{ IMG_UINT64_C(0x00160000006800da), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a089530c40a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.104.218 */
+	{ IMG_UINT64_C(0x0016000000d0013e), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a08a551080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.208.318 */
+	{ IMG_UINT64_C(0x00180000003600cc), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a089930840a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.54.204 */
+	{ IMG_UINT64_C(0x00180000006801f8), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a089930a40a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.104.504 */
+	{ IMG_UINT64_C(0x0018000000d001f8), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a0aa951080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.208.504 */
+	{ IMG_UINT64_C(0x0018000000d001f9), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a0aa951080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.208.505 */
+	{ IMG_UINT64_C(0x001d0000003400ca), IMG_UINT64_C(0x000002183951feb5), IMG_UINT64_C(0x2a069930440a9891), IMG_UINT64_C(0x0000000000000001) },	/* 29.0.52.202 */
+	{ IMG_UINT64_C(0x001d0000006c00d0), IMG_UINT64_C(0x000002183951feb5), IMG_UINT64_C(0x2a0aa950e80a9891), IMG_UINT64_C(0x0000000000000001) },	/* 29.0.108.208 */
+	{ IMG_UINT64_C(0x00210000000b0003), IMG_UINT64_C(0x000002001551ba25), IMG_UINT64_C(0x2a02992824080011), IMG_UINT64_C(0x0000000000000001) },	/* 33.0.11.3 */
+	{ IMG_UINT64_C(0x0021000000160001), IMG_UINT64_C(0x000002181955be35), IMG_UINT64_C(0x2a06993044080011), IMG_UINT64_C(0x0000000000000001) },	/* 33.0.22.1 */
+	{ IMG_UINT64_C(0x00240000003400b6), IMG_UINT64_C(0x000002001d51bf35), IMG_UINT64_C(0x2a4699304412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.52.182 */
+	{ IMG_UINT64_C(0x0024000000360067), IMG_UINT64_C(0x000002181d51b735), IMG_UINT64_C(0x2a469930840abad1), IMG_UINT64_C(0x0000000000000001) },	/* 36.0.54.103 */
+	{ IMG_UINT64_C(0x00240000003600b6), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a4699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.182 */
+	{ IMG_UINT64_C(0x00240000003600b7), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a4699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.183 */
+	{ IMG_UINT64_C(0x0024000000360118), IMG_UINT64_C(0x00000219dd51bf35), IMG_UINT64_C(0x2a8699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.280 */
+	{ IMG_UINT64_C(0x00240000006800b6), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a469930a412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.182 */
+	{ IMG_UINT64_C(0x00240000006800b7), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a469930a412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.183 */
+	{ IMG_UINT64_C(0x002400000068031c), IMG_UINT64_C(0x0000039a1d79af35), IMG_UINT64_C(0x2a469930a412bad9), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.796 */
+	{ IMG_UINT64_C(0x002e000000660185), IMG_UINT64_C(0x00000241dd51bf35), IMG_UINT64_C(0x2ad899324412bad5), IMG_UINT64_C(0x0000000000000055) },	/* 46.0.102.389 */
+	{ IMG_UINT64_C(0x002e000000cc0186), IMG_UINT64_C(0x00000259dd51bf35), IMG_UINT64_C(0x2ad89932c412bad5), IMG_UINT64_C(0x0000000000000055) },	/* 46.0.204.390 */
+};
+
+/******************************************************************************
+ * Table mapping bitmasks for ERNs/BRNs
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaErnsBrns[][2]=
+{
+	{ IMG_UINT64_C(0x0001002700040013), IMG_UINT64_C(0x0000000000000005) },	/* 1.39.4.19 */
+	{ IMG_UINT64_C(0x0001005200040005), IMG_UINT64_C(0x0000000000000000) },	/* 1.82.4.5 */
+	{ IMG_UINT64_C(0x0004001f00040037), IMG_UINT64_C(0x000000000000042a) },	/* 4.31.4.55 */
+	{ IMG_UINT64_C(0x0004002800020033), IMG_UINT64_C(0x000000000000042a) },	/* 4.40.2.51 */
+	{ IMG_UINT64_C(0x0004002b0006003e), IMG_UINT64_C(0x000000000000142a) },	/* 4.43.6.62 */
+	{ IMG_UINT64_C(0x0004002d0002003a), IMG_UINT64_C(0x000000000000140a) },	/* 4.45.2.58 */
+	{ IMG_UINT64_C(0x0004002e0006003e), IMG_UINT64_C(0x000000000000142a) },	/* 4.46.6.62 */
+	{ IMG_UINT64_C(0x000500090001002e), IMG_UINT64_C(0x0000000000000001) },	/* 5.9.1.46 */
+	{ IMG_UINT64_C(0x0006002200040023), IMG_UINT64_C(0x000000000000040a) },	/* 6.34.4.35 */
+	{ IMG_UINT64_C(0x000f000500010040), IMG_UINT64_C(0x0000000000001008) },	/* 15.5.1.64 */
+	{ IMG_UINT64_C(0x0016001e00360019), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.30.54.25 */
+	{ IMG_UINT64_C(0x001600280036001e), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.40.54.30 */
+	{ IMG_UINT64_C(0x0016002e0036014a), IMG_UINT64_C(0x0000000000087a8a) },	/* 22.46.54.330 */
+	{ IMG_UINT64_C(0x0016003100150010), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.49.21.16 */
+	{ IMG_UINT64_C(0x001600430036001e), IMG_UINT64_C(0x00000000000859c8) },	/* 22.67.54.30 */
+	{ IMG_UINT64_C(0x001600440036001e), IMG_UINT64_C(0x0000000000085948) },	/* 22.68.54.30 */
+	{ IMG_UINT64_C(0x00160056006800da), IMG_UINT64_C(0x0000000000083908) },	/* 22.86.104.218 */
+	{ IMG_UINT64_C(0x0016005700680012), IMG_UINT64_C(0x0000000000081948) },	/* 22.87.104.18 */
+	{ IMG_UINT64_C(0x0016006600360026), IMG_UINT64_C(0x0000000000081948) },	/* 22.102.54.38 */
+	{ IMG_UINT64_C(0x0016006800d0013e), IMG_UINT64_C(0x000000000008390a) },	/* 22.104.208.318 */
+	{ IMG_UINT64_C(0x0018003200d001f8), IMG_UINT64_C(0x000000000008884a) },	/* 24.50.208.504 */
+	{ IMG_UINT64_C(0x0018003800d001f9), IMG_UINT64_C(0x000000000008884a) },	/* 24.56.208.505 */
+	{ IMG_UINT64_C(0x00180042003600cc), IMG_UINT64_C(0x000000000008884a) },	/* 24.66.54.204 */
+	{ IMG_UINT64_C(0x00180043006801f8), IMG_UINT64_C(0x000000000008884a) },	/* 24.67.104.504 */
+	{ IMG_UINT64_C(0x001d000e006c00d0), IMG_UINT64_C(0x00000000000a885a) },	/* 29.14.108.208 */
+	{ IMG_UINT64_C(0x001d0013003400ca), IMG_UINT64_C(0x00000000000a885a) },	/* 29.19.52.202 */
+	{ IMG_UINT64_C(0x0021000800160001), IMG_UINT64_C(0x000000000008085a) },	/* 33.8.22.1 */
+	{ IMG_UINT64_C(0x0021000f000b0003), IMG_UINT64_C(0x00000000000c085a) },	/* 33.15.11.3 */
+	{ IMG_UINT64_C(0x0024001d003400b6), IMG_UINT64_C(0x000000000008085a) },	/* 36.29.52.182 */
+	{ IMG_UINT64_C(0x00240032003600b6), IMG_UINT64_C(0x00000000000c085a) },	/* 36.50.54.182 */
+	{ IMG_UINT64_C(0x00240034006800b6), IMG_UINT64_C(0x00000000000c085a) },	/* 36.52.104.182 */
+	{ IMG_UINT64_C(0x002400350068031c), IMG_UINT64_C(0x000000000008005a) },	/* 36.53.104.796 */
+	{ IMG_UINT64_C(0x00240036003600b7), IMG_UINT64_C(0x000000000008085a) },	/* 36.54.54.183 */
+	{ IMG_UINT64_C(0x0024003700360067), IMG_UINT64_C(0x000000000008085a) },	/* 36.55.54.103 */
+	{ IMG_UINT64_C(0x00240038006800b7), IMG_UINT64_C(0x000000000008085a) },	/* 36.56.104.183 */
+	{ IMG_UINT64_C(0x0024003c00360118), IMG_UINT64_C(0x000000000008085a) },	/* 36.60.54.280 */
+	{ IMG_UINT64_C(0x002e004800660185), IMG_UINT64_C(0x000000000009085a) },	/* 46.72.102.389 */
+	{ IMG_UINT64_C(0x002e005e00cc0186), IMG_UINT64_C(0x000000000009085a) },	/* 46.94.204.390 */
+};
+
+#if defined(DEBUG)
+
+#define FEATURE_NO_VALUES_NAMES_MAX_IDX (43U)
+
+static const IMG_CHAR * const gaszFeaturesNoValuesNames[FEATURE_NO_VALUES_NAMES_MAX_IDX] =
+{
+	"AXI_ACELITE",
+	"CLUSTER_GROUPING",
+	"COMPUTE",
+	"COMPUTE_MORTON_CAPABLE",
+	"COMPUTE_OVERLAP",
+	"COREID_PER_OS",
+	"DYNAMIC_DUST_POWER",
+	"FASTRENDER_DM",
+	"GPU_MULTICORE_SUPPORT",
+	"GPU_VIRTUALISATION",
+	"GS_RTA_SUPPORT",
+	"IRQ_PER_OS",
+	"MIPS",
+	"PBE2_IN_XE",
+	"PBE_CHECKSUM_2D",
+	"PBVNC_COREID_REG",
+	"PERFBUS",
+	"PERF_COUNTER_BATCH",
+	"PM_MMU_VFP",
+	"RISCV_FW_PROCESSOR",
+	"ROGUEXE",
+	"SAFETY_IRQ",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT_V1",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT_V2",
+	"SINGLE_BIF",
+	"SLC_SIZE_CONFIGURABLE",
+	"SOC_TIMER",
+	"SYS_BUS_SECURE_RESET",
+	"TDM_PDS_CHECKSUM",
+	"TFBC_DELTA_CORRELATION",
+	"TFBC_LOSSY_37_PERCENT",
+	"TFBC_NATIVE_YUV10",
+	"TILE_REGION_PROTECTION",
+	"TLA",
+	"TPU_CEM_DATAMASTER_GLOBAL_REGISTERS",
+	"TPU_DM_GLOBAL_REGISTERS",
+	"TPU_FILTERING_MODE_CONTROL",
+	"VOLCANIC_TB",
+	"WATCHDOG_TIMER",
+	"WORKGROUP_PROTECTION",
+	"XE_MEMORY_HIERARCHY",
+	"XT_TOP_INFRASTRUCTURE",
+};
+
+#define ERNSBRNS_IDS_MAX_IDX (20U)
+
+static const IMG_UINT32 gaui64ErnsBrnsIDs[ERNSBRNS_IDS_MAX_IDX] =
+{
+	38344,
+	42290,
+	42321,
+	42606,
+	47025,
+	50767,
+	57596,
+	60084,
+	61389,
+	61450,
+	63142,
+	63553,
+	64502,
+	65101,
+	65273,
+	66622,
+	66927,
+	68186,
+	71242,
+	71317,
+};
+
+#endif /* defined(DEBUG) */
+#endif /* RGX_BVNC_TABLE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_cr_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_cr_defs_km.h
new file mode 100644
index 000000000000..1326877b50be
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgx_cr_defs_km.h
@@ -0,0 +1,8472 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_cr_defs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ */
+
+
+#ifndef RGX_CR_DEFS_KM_H
+#define RGX_CR_DEFS_KM_H
+
+#if !defined(IMG_EXPLICIT_INCLUDE_HWDEFS)
+#error This file may only be included if explicitly defined
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGX_CR_DEFS_KM_REVISION 1
+
+/*
+    Register RGX_CR_RASTERISATION_INDIRECT
+*/
+#define RGX_CR_RASTERISATION_INDIRECT                                               (0x8238U)
+#define RGX_CR_RASTERISATION_INDIRECT_MASKFULL                                      (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_RASTERISATION_INDIRECT_ADDRESS_SHIFT                                 (0U)
+#define RGX_CR_RASTERISATION_INDIRECT_ADDRESS_CLRMSK                                (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_USC_INDIRECT
+*/
+#define RGX_CR_USC_INDIRECT                                                         (0x8000U)
+#define RGX_CR_USC_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_USC_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_USC_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_PBE_INDIRECT
+*/
+#define RGX_CR_PBE_INDIRECT                                                         (0x83E0U)
+#define RGX_CR_PBE_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_PBE_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_PBE_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_PBE_PERF_INDIRECT
+*/
+#define RGX_CR_PBE_PERF_INDIRECT                                                    (0x83D8U)
+#define RGX_CR_PBE_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_PBE_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_PBE_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_TPU_PERF_INDIRECT
+*/
+#define RGX_CR_TPU_PERF_INDIRECT                                                    (0x83F0U)
+#define RGX_CR_TPU_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TPU_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_TPU_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_INDIRECT
+*/
+#define RGX_CR_RASTERISATION_PERF_INDIRECT                                          (0x8318U)
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_MASKFULL                                 (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_ADDRESS_SHIFT                            (0U)
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_ADDRESS_CLRMSK                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_INDIRECT
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT                                             (0x8028U)
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_ADDRESS_SHIFT                               (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_ADDRESS_CLRMSK                              (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_USC_PERF_INDIRECT
+*/
+#define RGX_CR_USC_PERF_INDIRECT                                                    (0x8030U)
+#define RGX_CR_USC_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_USC_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_USC_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_INDIRECT
+*/
+#define RGX_CR_BLACKPEARL_INDIRECT                                                  (0x8388U)
+#define RGX_CR_BLACKPEARL_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BLACKPEARL_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BLACKPEARL_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_INDIRECT
+*/
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT                                             (0x83F8U)
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_ADDRESS_SHIFT                               (0U)
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_ADDRESS_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_TEXAS3_PERF_INDIRECT
+*/
+#define RGX_CR_TEXAS3_PERF_INDIRECT                                                 (0x83D0U)
+#define RGX_CR_TEXAS3_PERF_INDIRECT_MASKFULL                                        (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TEXAS3_PERF_INDIRECT_ADDRESS_SHIFT                                   (0U)
+#define RGX_CR_TEXAS3_PERF_INDIRECT_ADDRESS_CLRMSK                                  (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_INDIRECT
+*/
+#define RGX_CR_TEXAS_PERF_INDIRECT                                                  (0x8288U)
+#define RGX_CR_TEXAS_PERF_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_TEXAS_PERF_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_TEXAS_PERF_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_BX_TU_PERF_INDIRECT
+*/
+#define RGX_CR_BX_TU_PERF_INDIRECT                                                  (0xC900U)
+#define RGX_CR_BX_TU_PERF_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BX_TU_PERF_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BX_TU_PERF_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_CLK_CTRL
+*/
+#define RGX_CR_CLK_CTRL                                                             (0x0000U)
+#define RGX_CR_CLK_CTRL__PBE2_XE__MASKFULL                                          (IMG_UINT64_C(0xFFFFFF003F3FFFFF))
+#define RGX_CR_CLK_CTRL__S7_INFRA__MASKFULL                                         (IMG_UINT64_C(0xCFCF03000F3F3F0F))
+#define RGX_CR_CLK_CTRL_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFF003F3FFFFF))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_SHIFT                                             (62U)
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_CLRMSK                                            (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_ON                                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_AUTO                                              (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__ON                                     (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__AUTO                                   (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__OFF                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__ON                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__AUTO                                    (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_SHIFT                                                   (60U)
+#define RGX_CR_CLK_CTRL_IPP_CLRMSK                                                  (IMG_UINT64_C(0xCFFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_IPP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_ON                                                      (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_AUTO                                                    (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__ON                                           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__ON                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_FBC_SHIFT                                                   (58U)
+#define RGX_CR_CLK_CTRL_FBC_CLRMSK                                                  (IMG_UINT64_C(0xF3FFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FBC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC_ON                                                      (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC_AUTO                                                    (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_SHIFT                                                  (56U)
+#define RGX_CR_CLK_CTRL_FBDC_CLRMSK                                                 (IMG_UINT64_C(0xFCFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FBDC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_ON                                                     (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_AUTO                                                   (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__ON                                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__AUTO                                        (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__ON                                           (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__AUTO                                         (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_SHIFT                                            (54U)
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_CLRMSK                                           (IMG_UINT64_C(0xFF3FFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_ON                                               (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_AUTO                                             (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__ON                                    (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__AUTO                                  (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__ON                                     (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__AUTO                                   (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_USCS_SHIFT                                                  (52U)
+#define RGX_CR_CLK_CTRL_USCS_CLRMSK                                                 (IMG_UINT64_C(0xFFCFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_USCS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS_ON                                                     (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS_AUTO                                                   (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__ON                                          (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__AUTO                                        (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__ON                                           (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__AUTO                                         (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_PBE_SHIFT                                                   (50U)
+#define RGX_CR_CLK_CTRL_PBE_CLRMSK                                                  (IMG_UINT64_C(0xFFF3FFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_PBE_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE_ON                                                      (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE_AUTO                                                    (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__ON                                           (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__ON                                            (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_SHIFT                                                (48U)
+#define RGX_CR_CLK_CTRL_MCU_L1_CLRMSK                                               (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_MCU_L1_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_ON                                                   (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_AUTO                                                 (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__ON                                        (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__AUTO                                      (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__ON                                         (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__AUTO                                       (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_CDM_SHIFT                                                   (46U)
+#define RGX_CR_CLK_CTRL_CDM_CLRMSK                                                  (IMG_UINT64_C(0xFFFF3FFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_CDM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM_ON                                                      (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM_AUTO                                                    (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_SHIFT                                              (44U)
+#define RGX_CR_CLK_CTRL_SIDEKICK_CLRMSK                                             (IMG_UINT64_C(0xFFFFCFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_SIDEKICK_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_ON                                                 (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_AUTO                                               (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__OFF                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__ON                                      (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__AUTO                                    (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__ON                                       (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__AUTO                                     (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_SHIFT                                          (42U)
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_CLRMSK                                         (IMG_UINT64_C(0xFFFFF3FFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_ON                                             (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_AUTO                                           (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__ON                                  (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__AUTO                                (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__OFF                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__ON                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__AUTO                                 (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SHIFT                                                   (40U)
+#define RGX_CR_CLK_CTRL_BIF_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_ON                                                      (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF_AUTO                                                    (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_SHIFT                                         (28U)
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_ON                                            (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_AUTO                                          (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__OFF                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__ON                                 (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__AUTO                               (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__ON                                  (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__AUTO                                (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_SHIFT                                                (26U)
+#define RGX_CR_CLK_CTRL_MCU_L0_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL_MCU_L0_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_ON                                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_AUTO                                                 (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__ON                                        (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__AUTO                                      (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__ON                                         (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__AUTO                                       (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_TPU_SHIFT                                                   (24U)
+#define RGX_CR_CLK_CTRL_TPU_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL_TPU_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_ON                                                      (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU_AUTO                                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_USC_SHIFT                                                   (20U)
+#define RGX_CR_CLK_CTRL_USC_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL_USC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC_ON                                                      (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC_AUTO                                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_TLA_SHIFT                                                   (18U)
+#define RGX_CR_CLK_CTRL_TLA_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL_TLA_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA_ON                                                      (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA_AUTO                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_SLC_SHIFT                                                   (16U)
+#define RGX_CR_CLK_CTRL_SLC_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL_SLC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC_ON                                                      (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC_AUTO                                                    (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_UVS_SHIFT                                                   (14U)
+#define RGX_CR_CLK_CTRL_UVS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_CLK_CTRL_UVS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS_ON                                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS_AUTO                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_PDS_SHIFT                                                   (12U)
+#define RGX_CR_CLK_CTRL_PDS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL_PDS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS_ON                                                      (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS_AUTO                                                    (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_VDM_SHIFT                                                   (10U)
+#define RGX_CR_CLK_CTRL_VDM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL_VDM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM_ON                                                      (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM_AUTO                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_PM_SHIFT                                                    (8U)
+#define RGX_CR_CLK_CTRL_PM_CLRMSK                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL_PM_OFF                                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM_ON                                                       (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM_AUTO                                                     (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__ON                                            (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__AUTO                                          (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__ON                                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__AUTO                                           (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_GPP_SHIFT                                                   (6U)
+#define RGX_CR_CLK_CTRL_GPP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_CLK_CTRL_GPP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP_ON                                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP_AUTO                                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_TE_SHIFT                                                    (4U)
+#define RGX_CR_CLK_CTRL_TE_CLRMSK                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL_TE_OFF                                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE_ON                                                       (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE_AUTO                                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__ON                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__AUTO                                          (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__ON                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__AUTO                                           (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TSP_SHIFT                                                   (2U)
+#define RGX_CR_CLK_CTRL_TSP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL_TSP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP_ON                                                      (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP_AUTO                                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_ISP_SHIFT                                                   (0U)
+#define RGX_CR_CLK_CTRL_ISP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL_ISP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP_ON                                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP_AUTO                                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS
+*/
+#define RGX_CR_CLK_STATUS                                                           (0x0008U)
+#define RGX_CR_CLK_STATUS__PBE2_XE__MASKFULL                                        (IMG_UINT64_C(0x00000001FFF077FF))
+#define RGX_CR_CLK_STATUS__S7_INFRA__MASKFULL                                       (IMG_UINT64_C(0x00000001B3101773))
+#define RGX_CR_CLK_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x00000001FFF077FF))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_SHIFT                                            (32U)
+#define RGX_CR_CLK_STATUS_MCU_FBTC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_RUNNING                                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__S7_INFRA__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__S7_INFRA__RUNNING                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__PBE2_XE__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__PBE2_XE__RUNNING                                (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_SHIFT                                           (31U)
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_RUNNING                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__S7_INFRA__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__S7_INFRA__RUNNING                              (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__PBE2_XE__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__PBE2_XE__RUNNING                               (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_IPP_SHIFT                                                 (30U)
+#define RGX_CR_CLK_STATUS_IPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_CLK_STATUS_IPP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP_RUNNING                                               (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_IPP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_IPP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_FBC_SHIFT                                                 (29U)
+#define RGX_CR_CLK_STATUS_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_CLK_STATUS_FBC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC_RUNNING                                               (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBDC_SHIFT                                                (28U)
+#define RGX_CR_CLK_STATUS_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_CLK_STATUS_FBDC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC_RUNNING                                              (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FBDC__S7_INFRA__GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC__S7_INFRA__RUNNING                                   (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FBDC__PBE2_XE__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC__PBE2_XE__RUNNING                                    (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_SHIFT                                          (27U)
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_RUNNING                                        (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__S7_INFRA__GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__S7_INFRA__RUNNING                             (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__PBE2_XE__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__PBE2_XE__RUNNING                              (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_USCS_SHIFT                                                (26U)
+#define RGX_CR_CLK_STATUS_USCS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_CLK_STATUS_USCS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS_RUNNING                                              (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_USCS__S7_INFRA__GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS__S7_INFRA__RUNNING                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_USCS__PBE2_XE__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS__PBE2_XE__RUNNING                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_PBE_SHIFT                                                 (25U)
+#define RGX_CR_CLK_STATUS_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_CLK_STATUS_PBE_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE_RUNNING                                               (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_PBE__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_PBE__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_MCU_L1_SHIFT                                              (24U)
+#define RGX_CR_CLK_STATUS_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_CLK_STATUS_MCU_L1_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1_RUNNING                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__S7_INFRA__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__S7_INFRA__RUNNING                                 (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__PBE2_XE__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__PBE2_XE__RUNNING                                  (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_CDM_SHIFT                                                 (23U)
+#define RGX_CR_CLK_STATUS_CDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_CLK_STATUS_CDM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM_RUNNING                                               (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_CDM__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_CDM__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_SIDEKICK_SHIFT                                            (22U)
+#define RGX_CR_CLK_STATUS_SIDEKICK_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CLK_STATUS_SIDEKICK_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK_RUNNING                                          (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__S7_INFRA__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__S7_INFRA__RUNNING                               (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__PBE2_XE__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__PBE2_XE__RUNNING                                (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_SHIFT                                        (21U)
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_RUNNING                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__S7_INFRA__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__S7_INFRA__RUNNING                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__PBE2_XE__GATED                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__PBE2_XE__RUNNING                            (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SHIFT                                                 (20U)
+#define RGX_CR_CLK_STATUS_BIF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_RUNNING                                               (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_BIF__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_BIF__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_SHIFT                                       (14U)
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_RUNNING                                     (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__S7_INFRA__GATED                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__S7_INFRA__RUNNING                          (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__PBE2_XE__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__PBE2_XE__RUNNING                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_MCU_L0_SHIFT                                              (13U)
+#define RGX_CR_CLK_STATUS_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_CLK_STATUS_MCU_L0_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0_RUNNING                                            (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_MCU_L0__S7_INFRA__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0__S7_INFRA__RUNNING                                 (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_MCU_L0__PBE2_XE__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0__PBE2_XE__RUNNING                                  (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_TPU_SHIFT                                                 (12U)
+#define RGX_CR_CLK_STATUS_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS_TPU_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_RUNNING                                               (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_TPU__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_TPU__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_USC_SHIFT                                                 (10U)
+#define RGX_CR_CLK_STATUS_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS_USC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC_RUNNING                                               (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_USC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_USC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_TLA_SHIFT                                                 (9U)
+#define RGX_CR_CLK_STATUS_TLA_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS_TLA_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA_RUNNING                                               (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_TLA__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_TLA__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_SLC_SHIFT                                                 (8U)
+#define RGX_CR_CLK_STATUS_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS_SLC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC_RUNNING                                               (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_SLC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_SLC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_UVS_SHIFT                                                 (7U)
+#define RGX_CR_CLK_STATUS_UVS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_STATUS_UVS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS_RUNNING                                               (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_UVS__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_UVS__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_PDS_SHIFT                                                 (6U)
+#define RGX_CR_CLK_STATUS_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS_PDS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS_RUNNING                                               (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_PDS__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_PDS__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_VDM_SHIFT                                                 (5U)
+#define RGX_CR_CLK_STATUS_VDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_STATUS_VDM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM_RUNNING                                               (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_VDM__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_VDM__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_PM_SHIFT                                                  (4U)
+#define RGX_CR_CLK_STATUS_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS_PM_GATED                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM_RUNNING                                                (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_PM__S7_INFRA__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM__S7_INFRA__RUNNING                                     (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_PM__PBE2_XE__GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM__PBE2_XE__RUNNING                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_GPP_SHIFT                                                 (3U)
+#define RGX_CR_CLK_STATUS_GPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_STATUS_GPP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP_RUNNING                                               (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_GPP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_GPP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_TE_SHIFT                                                  (2U)
+#define RGX_CR_CLK_STATUS_TE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS_TE_GATED                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE_RUNNING                                                (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TE__S7_INFRA__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE__S7_INFRA__RUNNING                                     (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TE__PBE2_XE__GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE__PBE2_XE__RUNNING                                      (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TSP_SHIFT                                                 (1U)
+#define RGX_CR_CLK_STATUS_TSP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS_TSP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP_RUNNING                                               (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_TSP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_TSP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_ISP_SHIFT                                                 (0U)
+#define RGX_CR_CLK_STATUS_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS_ISP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP_RUNNING                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS_ISP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS_ISP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID__PBVNC                                                       (0x0020U)
+#define RGX_CR_CORE_ID__PBVNC__MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__BRANCH_ID_SHIFT                                      (48U)
+#define RGX_CR_CORE_ID__PBVNC__BRANCH_ID_CLRMSK                                     (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__VERSION_ID_SHIFT                                     (32U)
+#define RGX_CR_CORE_ID__PBVNC__VERSION_ID_CLRMSK                                    (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_SHIFT                       (16U)
+#define RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_CORE_ID__PBVNC__CONFIG_ID_SHIFT                                      (0U)
+#define RGX_CR_CORE_ID__PBVNC__CONFIG_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID                                                              (0x0018U)
+#define RGX_CR_CORE_ID_MASKFULL                                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CORE_ID_ID_SHIFT                                                     (16U)
+#define RGX_CR_CORE_ID_ID_CLRMSK                                                    (0x0000FFFFU)
+#define RGX_CR_CORE_ID_CONFIG_SHIFT                                                 (0U)
+#define RGX_CR_CORE_ID_CONFIG_CLRMSK                                                (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_CORE_REVISION
+*/
+#define RGX_CR_CORE_REVISION                                                        (0x0020U)
+#define RGX_CR_CORE_REVISION_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CORE_REVISION_DESIGNER_SHIFT                                         (24U)
+#define RGX_CR_CORE_REVISION_DESIGNER_CLRMSK                                        (0x00FFFFFFU)
+#define RGX_CR_CORE_REVISION_MAJOR_SHIFT                                            (16U)
+#define RGX_CR_CORE_REVISION_MAJOR_CLRMSK                                           (0xFF00FFFFU)
+#define RGX_CR_CORE_REVISION_MINOR_SHIFT                                            (8U)
+#define RGX_CR_CORE_REVISION_MINOR_CLRMSK                                           (0xFFFF00FFU)
+#define RGX_CR_CORE_REVISION_MAINTENANCE_SHIFT                                      (0U)
+#define RGX_CR_CORE_REVISION_MAINTENANCE_CLRMSK                                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_DESIGNER_REV_FIELD1
+*/
+#define RGX_CR_DESIGNER_REV_FIELD1                                                  (0x0028U)
+#define RGX_CR_DESIGNER_REV_FIELD1_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_DESIGNER_REV_FIELD1_DESIGNER_REV_FIELD1_SHIFT                        (0U)
+#define RGX_CR_DESIGNER_REV_FIELD1_DESIGNER_REV_FIELD1_CLRMSK                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_DESIGNER_REV_FIELD2
+*/
+#define RGX_CR_DESIGNER_REV_FIELD2                                                  (0x0030U)
+#define RGX_CR_DESIGNER_REV_FIELD2_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_DESIGNER_REV_FIELD2_DESIGNER_REV_FIELD2_SHIFT                        (0U)
+#define RGX_CR_DESIGNER_REV_FIELD2_DESIGNER_REV_FIELD2_CLRMSK                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_CHANGESET_NUMBER
+*/
+#define RGX_CR_CHANGESET_NUMBER                                                     (0x0040U)
+#define RGX_CR_CHANGESET_NUMBER_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CHANGESET_NUMBER_CHANGESET_NUMBER_SHIFT                              (0U)
+#define RGX_CR_CHANGESET_NUMBER_CHANGESET_NUMBER_CLRMSK                             (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_GRAY
+*/
+#define RGX_CR_SOC_TIMER_GRAY                                                       (0x00E0U)
+#define RGX_CR_SOC_TIMER_GRAY_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_SHIFT                                           (0U)
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_CLRMSK                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_BINARY
+*/
+#define RGX_CR_SOC_TIMER_BINARY                                                     (0x00E8U)
+#define RGX_CR_SOC_TIMER_BINARY_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_CLRMSK                                        (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_CLK_XTPLUS_CTRL
+*/
+#define RGX_CR_CLK_XTPLUS_CTRL                                                      (0x0080U)
+#define RGX_CR_CLK_XTPLUS_CTRL_MASKFULL                                             (IMG_UINT64_C(0x0000003FFFFF0000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_SHIFT                                            (36U)
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_ON                                               (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_AUTO                                             (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_SHIFT                                           (34U)
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFF3FFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_OFF                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_ON                                              (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_AUTO                                            (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_SHIFT                                            (32U)
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFCFFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_ON                                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_AUTO                                             (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_SHIFT                                        (30U)
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFF3FFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_ON                                           (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_AUTO                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_SHIFT                                          (28U)
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_ON                                             (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_AUTO                                           (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_SHIFT                                         (26U)
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_ON                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_AUTO                                          (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_SHIFT                                          (24U)
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_ON                                             (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_AUTO                                           (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_SHIFT                                     (22U)
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFF3FFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_ON                                        (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_AUTO                                      (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_SHIFT                                 (20U)
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_ON                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_AUTO                                  (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_SHIFT                                     (18U)
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_ON                                        (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_AUTO                                      (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_SHIFT                                       (16U)
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_ON                                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_AUTO                                        (IMG_UINT64_C(0x0000000000020000))
+
+
+/*
+    Register RGX_CR_CLK_XTPLUS_STATUS
+*/
+#define RGX_CR_CLK_XTPLUS_STATUS                                                    (0x0088U)
+#define RGX_CR_CLK_XTPLUS_STATUS_MASKFULL                                           (IMG_UINT64_C(0x00000000000007FF))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_SHIFT                                          (10U)
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_RUNNING                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_SHIFT                                          (9U)
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_RUNNING                                        (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_SHIFT                                      (8U)
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_RUNNING                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_SHIFT                                         (7U)
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_RUNNING                                       (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_SHIFT                                        (6U)
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_RUNNING                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_SHIFT                                       (5U)
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_RUNNING                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_SHIFT                                        (4U)
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_RUNNING                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_SHIFT                                   (3U)
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_RUNNING                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_SHIFT                               (2U)
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_RUNNING                             (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_SHIFT                                   (1U)
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_RUNNING                                 (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_SHIFT                                     (0U)
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_RUNNING                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET
+*/
+#define RGX_CR_SOFT_RESET                                                           (0x0100U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL                                        (IMG_UINT64_C(0xFFEFFFFFFFFFFC3D))
+#define RGX_CR_SOFT_RESET_MASKFULL                                                  (IMG_UINT64_C(0x00E7FFFFFFFFFC3D))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_SHIFT                             (63U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_CLRMSK                            (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_EN                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_SHIFT                             (62U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_SHIFT                             (61U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_CLRMSK                            (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_EN                                (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_SHIFT                                (60U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_CLRMSK                               (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_EN                                   (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_SHIFT                               (59U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_CLRMSK                              (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_EN                                  (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_SHIFT                                       (58U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_CLRMSK                                      (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_EN                                          (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_SHIFT                                       (57U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_CLRMSK                                      (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_EN                                          (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_SHIFT                                       (56U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_CLRMSK                                      (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_EN                                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SOFT_RESET_DPX1_CORE_SHIFT                                           (55U)
+#define RGX_CR_SOFT_RESET_DPX1_CORE_CLRMSK                                          (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DPX1_CORE_EN                                              (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SOFT_RESET_DPX0_CORE_SHIFT                                           (54U)
+#define RGX_CR_SOFT_RESET_DPX0_CORE_CLRMSK                                          (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DPX0_CORE_EN                                              (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SOFT_RESET_FBA_SHIFT                                                 (53U)
+#define RGX_CR_SOFT_RESET_FBA_CLRMSK                                                (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBA_EN                                                    (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_SHIFT                                    (51U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_CLRMSK                                   (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_EN                                       (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SOFT_RESET_SH_SHIFT                                                  (50U)
+#define RGX_CR_SOFT_RESET_SH_CLRMSK                                                 (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_SH_EN                                                     (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SOFT_RESET_VRDM_SHIFT                                                (49U)
+#define RGX_CR_SOFT_RESET_VRDM_CLRMSK                                               (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_VRDM_EN                                                   (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SOFT_RESET_MCU_FBTC_SHIFT                                            (48U)
+#define RGX_CR_SOFT_RESET_MCU_FBTC_CLRMSK                                           (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_MCU_FBTC_EN                                               (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_SHIFT                                       (47U)
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_EN                                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_SHIFT                                       (46U)
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_EN                                          (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_SHIFT                                       (45U)
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_EN                                          (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_SHIFT                                       (44U)
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_EN                                          (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SOFT_RESET_IPP_SHIFT                                                 (43U)
+#define RGX_CR_SOFT_RESET_IPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_IPP_EN                                                    (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_SHIFT                                           (42U)
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_EN                                              (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_SHIFT                                        (41U)
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_EN                                           (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_SHIFT                                         (40U)
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_EN                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_SHIFT                                         (39U)
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_EN                                            (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_SHIFT                                         (38U)
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_EN                                            (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_SHIFT                                         (37U)
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_EN                                            (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_SHIFT                                         (36U)
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_EN                                            (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_SHIFT                                         (35U)
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_EN                                            (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SOFT_RESET_MMU_SHIFT                                                 (34U)
+#define RGX_CR_SOFT_RESET_MMU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SOFT_RESET_MMU_EN                                                    (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SOFT_RESET_BIF1_SHIFT                                                (33U)
+#define RGX_CR_SOFT_RESET_BIF1_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF1_EN                                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SOFT_RESET_GARTEN_SHIFT                                              (32U)
+#define RGX_CR_SOFT_RESET_GARTEN_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GARTEN_EN                                                 (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SOFT_RESET_CPU_SHIFT                                                 (32U)
+#define RGX_CR_SOFT_RESET_CPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SOFT_RESET_CPU_EN                                                    (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_SHIFT                                         (31U)
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_EN                                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_SHIFT                                         (30U)
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_EN                                            (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_SHIFT                                         (29U)
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_EN                                            (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_SHIFT                                          (28U)
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_EN                                             (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SOFT_RESET_SLC_SHIFT                                                 (27U)
+#define RGX_CR_SOFT_RESET_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SOFT_RESET_SLC_EN                                                    (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SOFT_RESET_TLA_SHIFT                                                 (26U)
+#define RGX_CR_SOFT_RESET_TLA_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SOFT_RESET_TLA_EN                                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SOFT_RESET_UVS_SHIFT                                                 (25U)
+#define RGX_CR_SOFT_RESET_UVS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SOFT_RESET_UVS_EN                                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SOFT_RESET_TE_SHIFT                                                  (24U)
+#define RGX_CR_SOFT_RESET_TE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SOFT_RESET_TE_EN                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SOFT_RESET_GPP_SHIFT                                                 (23U)
+#define RGX_CR_SOFT_RESET_GPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SOFT_RESET_GPP_EN                                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SOFT_RESET_FBDC_SHIFT                                                (22U)
+#define RGX_CR_SOFT_RESET_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SOFT_RESET_FBDC_EN                                                   (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SOFT_RESET_FBC_SHIFT                                                 (21U)
+#define RGX_CR_SOFT_RESET_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SOFT_RESET_FBC_EN                                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SOFT_RESET_PM_SHIFT                                                  (20U)
+#define RGX_CR_SOFT_RESET_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SOFT_RESET_PM_EN                                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SOFT_RESET_PBE_SHIFT                                                 (19U)
+#define RGX_CR_SOFT_RESET_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SOFT_RESET_PBE_EN                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SOFT_RESET_USC_SHARED_SHIFT                                          (18U)
+#define RGX_CR_SOFT_RESET_USC_SHARED_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SOFT_RESET_USC_SHARED_EN                                             (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SOFT_RESET_MCU_L1_SHIFT                                              (17U)
+#define RGX_CR_SOFT_RESET_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SOFT_RESET_MCU_L1_EN                                                 (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SOFT_RESET_BIF_SHIFT                                                 (16U)
+#define RGX_CR_SOFT_RESET_BIF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SOFT_RESET_BIF_EN                                                    (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SOFT_RESET_CDM_SHIFT                                                 (15U)
+#define RGX_CR_SOFT_RESET_CDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SOFT_RESET_CDM_EN                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SOFT_RESET_VDM_SHIFT                                                 (14U)
+#define RGX_CR_SOFT_RESET_VDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SOFT_RESET_VDM_EN                                                    (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SOFT_RESET_TESS_SHIFT                                                (13U)
+#define RGX_CR_SOFT_RESET_TESS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SOFT_RESET_TESS_EN                                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SOFT_RESET_PDS_SHIFT                                                 (12U)
+#define RGX_CR_SOFT_RESET_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SOFT_RESET_PDS_EN                                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SOFT_RESET_ISP_SHIFT                                                 (11U)
+#define RGX_CR_SOFT_RESET_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SOFT_RESET_ISP_EN                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SOFT_RESET_TSP_SHIFT                                                 (10U)
+#define RGX_CR_SOFT_RESET_TSP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SOFT_RESET_TSP_EN                                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SOFT_RESET_SYSARB_SHIFT                                              (5U)
+#define RGX_CR_SOFT_RESET_SYSARB_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SOFT_RESET_SYSARB_EN                                                 (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_SHIFT                                       (4U)
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_EN                                          (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SOFT_RESET_MCU_L0_SHIFT                                              (3U)
+#define RGX_CR_SOFT_RESET_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SOFT_RESET_MCU_L0_EN                                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SOFT_RESET_TPU_SHIFT                                                 (2U)
+#define RGX_CR_SOFT_RESET_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SOFT_RESET_TPU_EN                                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SOFT_RESET_USC_SHIFT                                                 (0U)
+#define RGX_CR_SOFT_RESET_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SOFT_RESET_USC_EN                                                    (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET2
+*/
+#define RGX_CR_SOFT_RESET2                                                          (0x0108U)
+#define RGX_CR_SOFT_RESET2_MASKFULL                                                 (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_SOFT_RESET2_SPFILTER_SHIFT                                           (12U)
+#define RGX_CR_SOFT_RESET2_SPFILTER_CLRMSK                                          (0xFFE00FFFU)
+#define RGX_CR_SOFT_RESET2_TDM_SHIFT                                                (11U)
+#define RGX_CR_SOFT_RESET2_TDM_CLRMSK                                               (0xFFFFF7FFU)
+#define RGX_CR_SOFT_RESET2_TDM_EN                                                   (0x00000800U)
+#define RGX_CR_SOFT_RESET2_ASTC_SHIFT                                               (10U)
+#define RGX_CR_SOFT_RESET2_ASTC_CLRMSK                                              (0xFFFFFBFFU)
+#define RGX_CR_SOFT_RESET2_ASTC_EN                                                  (0x00000400U)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_SHIFT                                         (9U)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_CLRMSK                                        (0xFFFFFDFFU)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_EN                                            (0x00000200U)
+#define RGX_CR_SOFT_RESET2_USCPS_SHIFT                                              (8U)
+#define RGX_CR_SOFT_RESET2_USCPS_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_SOFT_RESET2_USCPS_EN                                                 (0x00000100U)
+#define RGX_CR_SOFT_RESET2_IPF_SHIFT                                                (7U)
+#define RGX_CR_SOFT_RESET2_IPF_CLRMSK                                               (0xFFFFFF7FU)
+#define RGX_CR_SOFT_RESET2_IPF_EN                                                   (0x00000080U)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_SHIFT                                           (6U)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_EN                                              (0x00000040U)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_SHIFT                                         (5U)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_EN                                            (0x00000020U)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_SHIFT                                         (4U)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_EN                                            (0x00000010U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_SHIFT                                     (3U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_EN                                        (0x00000008U)
+#define RGX_CR_SOFT_RESET2_PIXEL_SHIFT                                              (2U)
+#define RGX_CR_SOFT_RESET2_PIXEL_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_SOFT_RESET2_PIXEL_EN                                                 (0x00000004U)
+#define RGX_CR_SOFT_RESET2_CDM_SHIFT                                                (1U)
+#define RGX_CR_SOFT_RESET2_CDM_CLRMSK                                               (0xFFFFFFFDU)
+#define RGX_CR_SOFT_RESET2_CDM_EN                                                   (0x00000002U)
+#define RGX_CR_SOFT_RESET2_VERTEX_SHIFT                                             (0U)
+#define RGX_CR_SOFT_RESET2_VERTEX_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_CR_SOFT_RESET2_VERTEX_EN                                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_ENABLE
+*/
+#define RGX_CR_EVENT_ENABLE                                                         (0x0128U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__MASKFULL                                      (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_ENABLE__SIGNALS__MASKFULL                                      (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_ENABLE_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_SHIFT                                 (28U)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_EN                                    (0x10000000U)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_SHIFT                                (27U)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_CLRMSK                               (0xF7FFFFFFU)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_EN                                   (0x08000000U)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_SHIFT                                 (26U)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_EN                                    (0x04000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_SHIFT                                  (25U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_CLRMSK                                 (0xFDFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_EN                                     (0x02000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_SHIFT                                  (24U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_CLRMSK                                 (0xFEFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_EN                                     (0x01000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_SHIFT                                  (23U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_CLRMSK                                 (0xFF7FFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_EN                                     (0x00800000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_SHIFT                                  (22U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_EN                                     (0x00400000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_SHIFT                                  (21U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_CLRMSK                                 (0xFFDFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_EN                                     (0x00200000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_SHIFT                                  (20U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_SHIFT                                  (20U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_SHIFT                                  (19U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_CLRMSK                                 (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_EN                                     (0x00080000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_SHIFT                               (19U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_EN                                  (0x00080000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_SHIFT                                  (18U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_CLRMSK                                 (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_EN                                     (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_SHIFT                                      (17U)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_CLRMSK                                     (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_EN                                         (0x00020000U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_SHIFT                                      (14U)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_CLRMSK                                     (0xFFFFBFFFU)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_EN                                         (0x00004000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_SHIFT                                    (8U)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_EN                                       (0x00000100U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_SHIFT                                (3U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_EN                                   (0x00000008U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_SHIFT                                      (0U)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_STATUS
+*/
+#define RGX_CR_EVENT_STATUS                                                         (0x0130U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__MASKFULL                                      (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_STATUS__SIGNALS__MASKFULL                                      (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_STATUS_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_SHIFT                                 (28U)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_EN                                    (0x10000000U)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_SHIFT                                (27U)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_CLRMSK                               (0xF7FFFFFFU)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_EN                                   (0x08000000U)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_SHIFT                                 (26U)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_EN                                    (0x04000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_SHIFT                                  (25U)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_CLRMSK                                 (0xFDFFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_EN                                     (0x02000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_SHIFT                                  (24U)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_CLRMSK                                 (0xFEFFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_EN                                     (0x01000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_SHIFT                                  (23U)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_CLRMSK                                 (0xFF7FFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_EN                                     (0x00800000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_SHIFT                                  (22U)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_EN                                     (0x00400000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_SHIFT                                  (21U)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_CLRMSK                                 (0xFFDFFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_EN                                     (0x00200000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_SHIFT                                  (20U)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_SHIFT                                  (20U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_SHIFT                                  (19U)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_CLRMSK                                 (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_EN                                     (0x00080000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_SHIFT                               (19U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_EN                                  (0x00080000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_SHIFT                                  (18U)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_CLRMSK                                 (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_EN                                     (0x00040000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_SHIFT                                      (17U)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_CLRMSK                                     (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_EN                                         (0x00020000U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_SHIFT                                      (14U)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_CLRMSK                                     (0xFFFFBFFFU)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_EN                                         (0x00004000U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_SHIFT                                    (8U)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_EN                                       (0x00000100U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_SHIFT                                (3U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_EN                                   (0x00000008U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_SHIFT                                      (0U)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_CLEAR
+*/
+#define RGX_CR_EVENT_CLEAR                                                          (0x0138U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__MASKFULL                                       (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_CLEAR__SIGNALS__MASKFULL                                       (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_CLEAR_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_SHIFT                                 (31U)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_CLRMSK                                (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_EN                                    (0x80000000U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_SHIFT                                   (30U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_CLRMSK                                  (0xBFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_EN                                      (0x40000000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_SHIFT                             (29U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_CLRMSK                            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_EN                                (0x20000000U)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_SHIFT                                  (28U)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_CLRMSK                                 (0xEFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_EN                                     (0x10000000U)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_SHIFT                                 (27U)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_CLRMSK                                (0xF7FFFFFFU)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_EN                                    (0x08000000U)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_SHIFT                                  (26U)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_CLRMSK                                 (0xFBFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_EN                                     (0x04000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_SHIFT                                   (25U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_CLRMSK                                  (0xFDFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_EN                                      (0x02000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_SHIFT                                   (24U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_CLRMSK                                  (0xFEFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_EN                                      (0x01000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_SHIFT                                   (23U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_CLRMSK                                  (0xFF7FFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_EN                                      (0x00800000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_SHIFT                                   (22U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_CLRMSK                                  (0xFFBFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_EN                                      (0x00400000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_SHIFT                                   (21U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_CLRMSK                                  (0xFFDFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_EN                                      (0x00200000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_SHIFT                                   (20U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_EN                                      (0x00100000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_SHIFT                                   (20U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_EN                                      (0x00100000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_SHIFT                                   (19U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_CLRMSK                                  (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_EN                                      (0x00080000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_SHIFT                                (19U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_CLRMSK                               (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_EN                                   (0x00080000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_SHIFT                                   (18U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_CLRMSK                                  (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_EN                                      (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT               (18U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK              (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                  (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT               (18U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK              (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                  (0x00040000U)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_SHIFT                                       (17U)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_CLRMSK                                      (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_EN                                          (0x00020000U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                   (17U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                  (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                      (0x00020000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_SHIFT                               (16U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_EN                                  (0x00010000U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_SHIFT                                        (15U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_CLRMSK                                       (0xFFFF7FFFU)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_EN                                           (0x00008000U)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_SHIFT                                       (14U)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_CLRMSK                                      (0xFFFFBFFFU)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_EN                                          (0x00004000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_SHIFT                                           (13U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_CLRMSK                                          (0xFFFFDFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_EN                                              (0x00002000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_SHIFT                                           (12U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_CLRMSK                                          (0xFFFFEFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_EN                                              (0x00001000U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_SHIFT                                        (11U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_CLRMSK                                       (0xFFFFF7FFU)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_EN                                           (0x00000800U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_SHIFT                                     (10U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_CLRMSK                                    (0xFFFFFBFFU)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_EN                                        (0x00000400U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_SHIFT                                     (9U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_CLRMSK                                    (0xFFFFFDFFU)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_EN                                        (0x00000200U)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_SHIFT                                     (8U)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_CLRMSK                                    (0xFFFFFEFFU)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_EN                                        (0x00000100U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_SHIFT                                   (7U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_CLRMSK                                  (0xFFFFFF7FU)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_EN                                      (0x00000080U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_SHIFT                                       (6U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_CLRMSK                                      (0xFFFFFFBFU)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_EN                                          (0x00000040U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_SHIFT                                        (5U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_EN                                           (0x00000020U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_SHIFT                                  (4U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_EN                                     (0x00000010U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_SHIFT                                 (3U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_EN                                    (0x00000008U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_SHIFT                                   (2U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_EN                                      (0x00000004U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_SHIFT                                    (1U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_EN                                       (0x00000002U)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_SHIFT                                       (0U)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_TIMER
+*/
+#define RGX_CR_TIMER                                                                (0x0160U)
+#define RGX_CR_TIMER_MASKFULL                                                       (IMG_UINT64_C(0x8000FFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_SHIFT                                                    (63U)
+#define RGX_CR_TIMER_BIT31_CLRMSK                                                   (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_EN                                                       (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_TIMER_VALUE_SHIFT                                                    (0U)
+#define RGX_CR_TIMER_VALUE_CLRMSK                                                   (IMG_UINT64_C(0xFFFF000000000000))
+
+
+/*
+    Register RGX_CR_TLA_STATUS
+*/
+#define RGX_CR_TLA_STATUS                                                           (0x0178U)
+#define RGX_CR_TLA_STATUS_MASKFULL                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_TLA_STATUS_BLIT_COUNT_SHIFT                                          (39U)
+#define RGX_CR_TLA_STATUS_BLIT_COUNT_CLRMSK                                         (IMG_UINT64_C(0x0000007FFFFFFFFF))
+#define RGX_CR_TLA_STATUS_REQUEST_SHIFT                                             (7U)
+#define RGX_CR_TLA_STATUS_REQUEST_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF800000007F))
+#define RGX_CR_TLA_STATUS_FIFO_FULLNESS_SHIFT                                       (1U)
+#define RGX_CR_TLA_STATUS_FIFO_FULLNESS_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFF81))
+#define RGX_CR_TLA_STATUS_BUSY_SHIFT                                                (0U)
+#define RGX_CR_TLA_STATUS_BUSY_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_TLA_STATUS_BUSY_EN                                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_PM_PARTIAL_RENDER_ENABLE
+*/
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE                                             (0x0338U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_SHIFT                                    (0U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_SIDEKICK_IDLE
+*/
+#define RGX_CR_SIDEKICK_IDLE                                                        (0x03C8U)
+#define RGX_CR_SIDEKICK_IDLE_MASKFULL                                               (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_SHIFT                                           (6U)
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_EN                                              (0x00000040U)
+#define RGX_CR_SIDEKICK_IDLE_MMU_SHIFT                                              (5U)
+#define RGX_CR_SIDEKICK_IDLE_MMU_CLRMSK                                             (0xFFFFFFDFU)
+#define RGX_CR_SIDEKICK_IDLE_MMU_EN                                                 (0x00000020U)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_SHIFT                                           (4U)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_CLRMSK                                          (0xFFFFFFEFU)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_EN                                              (0x00000010U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_SHIFT                                              (3U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_CLRMSK                                             (0xFFFFFFF7U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_EN                                                 (0x00000008U)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_SHIFT                                           (2U)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_EN                                              (0x00000004U)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_SHIFT                                           (1U)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_EN                                              (0x00000002U)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_SHIFT                                            (0U)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_MARS_IDLE
+*/
+#define RGX_CR_MARS_IDLE                                                            (0x08F8U)
+#define RGX_CR_MARS_IDLE_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_SHIFT                                           (2U)
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_EN                                              (0x00000004U)
+#define RGX_CR_MARS_IDLE_CPU_SHIFT                                                  (1U)
+#define RGX_CR_MARS_IDLE_CPU_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_MARS_IDLE_CPU_EN                                                     (0x00000002U)
+#define RGX_CR_MARS_IDLE_SOCIF_SHIFT                                                (0U)
+#define RGX_CR_MARS_IDLE_SOCIF_CLRMSK                                               (0xFFFFFFFEU)
+#define RGX_CR_MARS_IDLE_SOCIF_EN                                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS                                             (0x0430U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_MASKFULL                                    (IMG_UINT64_C(0x00000000000000F3))
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_LAST_PIPE_SHIFT                             (4U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_LAST_PIPE_CLRMSK                            (0xFFFFFF0FU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_SHIFT                           (1U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_CLRMSK                          (0xFFFFFFFDU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_EN                              (0x00000002U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_SHIFT                              (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK0
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0                                              (0x0438U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_MASKFULL                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE1_SHIFT                             (32U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE1_CLRMSK                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE0_SHIFT                             (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE0_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK1
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1                                              (0x0440U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_PDS_STATE2_SHIFT                             (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_PDS_STATE2_CLRMSK                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK2
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2                                              (0x0448U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_MASKFULL                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT2_SHIFT                            (32U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT2_CLRMSK                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT1_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT1_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK0
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0                                             (0x0450U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_MASKFULL                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE1_SHIFT                            (32U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE1_CLRMSK                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE0_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE0_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK1
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1                                             (0x0458U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_PDS_STATE2_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_PDS_STATE2_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK2
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2                                             (0x0460U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_MASKFULL                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT2_SHIFT                           (32U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT2_CLRMSK                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT1_SHIFT                           (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT1_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS                                             (0x04A0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_SHIFT                           (1U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_CLRMSK                          (0xFFFFFFFDU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_EN                              (0x00000002U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_SHIFT                              (0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_PDS0                                                     (0x04A8U)
+#define RGX_CR_CDM_CONTEXT_PDS0_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_SHIFT                                     (36U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_CLRMSK                                    (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSIZE                                 (16U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_SHIFT                                     (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_PDS1                                                     (0x04B0U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__MASKFULL                                (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_MASKFULL                                            (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                       (30U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                      (0xBFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                          (0x40000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_SHIFT                                   (29U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_CLRMSK                                  (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_EN                                      (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                       (29U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                      (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                          (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_SHIFT                                   (28U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_CLRMSK                                  (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_EN                                      (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_SHIFT                            (28U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_CLRMSK                           (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_EN                               (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_SHIFT                                        (27U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_CLRMSK                                       (0xF7FFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_EN                                           (0x08000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                      (22U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                     (0xF03FFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_SHIFT                                  (21U)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_CLRMSK                                 (0xF81FFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                     (21U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK                    (0xFFDFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_EN                        (0x00200000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_SHIFT                                 (20U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_CLRMSK                                (0xFFEFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_EN                                    (0x00100000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                       (12U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                      (0xFFE00FFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SIZE_SHIFT                                   (11U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SIZE_CLRMSK                                  (0xFFF007FFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_SHIFT                                     (7U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_CLRMSK                                    (0xFFFFF87FU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                         (7U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                        (0xFFFFF07FU)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_SHIFT                                     (1U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_CLRMSK                                    (0xFFFFFF81U)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_SHIFT                                         (0U)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS
+*/
+#define RGX_CR_CDM_TERMINATE_PDS                                                    (0x04B8U)
+#define RGX_CR_CDM_TERMINATE_PDS_MASKFULL                                           (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_SHIFT                                    (36U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_CLRMSK                                   (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSIZE                                (16U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_SHIFT                                    (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSIZE                                (16U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS1
+*/
+#define RGX_CR_CDM_TERMINATE_PDS1                                                   (0x04C0U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__MASKFULL                              (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_MASKFULL                                          (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                     (30U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                    (0xBFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                        (0x40000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_SHIFT                                 (29U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_CLRMSK                                (0xDFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_EN                                    (0x20000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                     (29U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                    (0xDFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                        (0x20000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_SHIFT                                 (28U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_EN                                    (0x10000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_SHIFT                          (28U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_CLRMSK                         (0xEFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_EN                             (0x10000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_SHIFT                                      (27U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_CLRMSK                                     (0xF7FFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_EN                                         (0x08000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                    (22U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                   (0xF03FFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_SHIFT                                (21U)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_CLRMSK                               (0xF81FFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                   (21U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK                  (0xFFDFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_EN                      (0x00200000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_SHIFT                               (20U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_CLRMSK                              (0xFFEFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_EN                                  (0x00100000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                     (12U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                    (0xFFE00FFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SIZE_SHIFT                                 (11U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SIZE_CLRMSK                                (0xFFF007FFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_SHIFT                                   (7U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_CLRMSK                                  (0xFFFFF87FU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                       (7U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                      (0xFFFFF07FU)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_SHIFT                                   (1U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_CLRMSK                                  (0xFFFFFF81U)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_SHIFT                                       (0U)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0                                                (0x04D8U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_SHIFT                                (36U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_CLRMSK                               (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSIZE                            (16U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_SHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSIZE                            (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1                                                (0x04E0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__MASKFULL                           (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                  (30U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                 (0xBFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                     (0x40000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_SHIFT                              (29U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_EN                                 (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                  (29U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                 (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                     (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_SHIFT                              (28U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_EN                                 (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_SHIFT                       (28U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_CLRMSK                      (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_EN                          (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_SHIFT                                   (27U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_CLRMSK                                  (0xF7FFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_EN                                      (0x08000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                 (22U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                (0xF03FFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_SHIFT                             (21U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_CLRMSK                            (0xF81FFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                (21U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK               (0xFFDFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_EN                   (0x00200000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_SHIFT                            (20U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_CLRMSK                           (0xFFEFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_EN                               (0x00100000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                  (12U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                 (0xFFE00FFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SIZE_SHIFT                              (11U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SIZE_CLRMSK                             (0xFFF007FFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_SHIFT                                (7U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_CLRMSK                               (0xFFFFF87FU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                    (7U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                   (0xFFFFF07FU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_SHIFT                                (1U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_CLRMSK                               (0xFFFFFF81U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_SHIFT                                    (0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_CONFIG
+*/
+#define RGX_CR_MIPS_WRAPPER_CONFIG                                                  (0x0810U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_MASKFULL                                         (IMG_UINT64_C(0x000001030F01FFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_SHIFT                             (40U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_CLRMSK                            (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_EN                                (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_SHIFT                               (33U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_EN                                  (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_SHIFT                               (32U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_EN                                  (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_OS_ID_SHIFT                                      (25U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_OS_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFF1FFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_SHIFT                                    (24U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_EN                                       (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_SHIFT                              (16U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MIPS32                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_REGBANK_BASE_ADDR_SHIFT                          (0U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_REGBANK_BASE_ADDR_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP1_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1                                             (0x0818U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP1_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2                                             (0x0820U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP2_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1                                             (0x0828U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP2_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2                                             (0x0830U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP3_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1                                             (0x0838U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP3_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2                                             (0x0840U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP4_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1                                             (0x0848U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP4_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2                                             (0x0850U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP5_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1                                             (0x0858U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP5_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2                                             (0x0860U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS                                      (0x0868U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_MASKFULL                             (IMG_UINT64_C(0x00000001FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_SHIFT                          (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_EN                             (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_ADDRESS_SHIFT                        (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_ADDRESS_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR                                       (0x0870U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_MASKFULL                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG                                         (0x0878U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MASKFULL                                (IMG_UINT64_C(0xFFFFFFF7FFFFFFBF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ADDR_OUT_SHIFT                          (36U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ADDR_OUT_CLRMSK                         (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_OS_ID_SHIFT                             (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_OS_ID_CLRMSK                            (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_BASE_ADDR_IN_SHIFT                      (12U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_BASE_ADDR_IN_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_SHIFT                           (11U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_EN                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_SHIFT                       (7U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFF87F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4KB                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_16KB                        (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64KB                        (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256KB                       (IMG_UINT64_C(0x0000000000000180))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_1MB                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4MB                         (IMG_UINT64_C(0x0000000000000280))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_16MB                        (IMG_UINT64_C(0x0000000000000300))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64MB                        (IMG_UINT64_C(0x0000000000000380))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256MB                       (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ENTRY_SHIFT                             (1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ENTRY_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFC1))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_EN                          (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_READ
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ                                           (0x0880U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_MASKFULL                                  (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_ENTRY_SHIFT                               (1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_ENTRY_CLRMSK                              (0xFFFFFFC1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_SHIFT                             (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA                                           (0x0888U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFF7FFFFFF81))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_SHIFT                            (36U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_CLRMSK                           (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_OS_ID_SHIFT                               (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_OS_ID_CLRMSK                              (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_SHIFT                        (12U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_SHIFT                             (11U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_EN                                (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_SHIFT                         (7U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFF87F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_SHIFT                         (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_EN                            (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_ENABLE
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE                                              (0x08A0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_STATUS
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS                                              (0x08A8U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_CLEAR
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR                                               (0x08B0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_MASKFULL                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_SHIFT                                   (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_NMI_ENABLE
+*/
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE                                              (0x08B8U)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT                                               (0x08C0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_MASKFULL                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_SHIFT                                 (0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_DEBUG_CONFIG
+*/
+#define RGX_CR_MIPS_DEBUG_CONFIG                                                    (0x08C8U)
+#define RGX_CR_MIPS_DEBUG_CONFIG_MASKFULL                                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_SHIFT                          (0U)
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_CLRMSK                         (0xFFFFFFFEU)
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_EN                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_EXCEPTION_STATUS
+*/
+#define RGX_CR_MIPS_EXCEPTION_STATUS                                                (0x08D0U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_SHIFT                                 (5U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_CLRMSK                                (0xFFFFFFDFU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_EN                                    (0x00000020U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_SHIFT                             (4U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_CLRMSK                            (0xFFFFFFEFU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN                                (0x00000010U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_SHIFT                              (3U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_CLRMSK                             (0xFFFFFFF7U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_EN                                 (0x00000008U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_SHIFT                              (2U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_EN                                 (0x00000004U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_SHIFT                                   (1U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_EN                                      (0x00000002U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_SHIFT                                   (0U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_STATUS
+*/
+#define RGX_CR_MIPS_WRAPPER_STATUS                                                  (0x08E8U)
+#define RGX_CR_MIPS_WRAPPER_STATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MIPS_WRAPPER_STATUS_OUTSTANDING_REQUESTS_SHIFT                       (0U)
+#define RGX_CR_MIPS_WRAPPER_STATUS_OUTSTANDING_REQUESTS_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_XPU_BROADCAST
+*/
+#define RGX_CR_XPU_BROADCAST                                                        (0x0890U)
+#define RGX_CR_XPU_BROADCAST_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_XPU_BROADCAST_MASK_SHIFT                                             (0U)
+#define RGX_CR_XPU_BROADCAST_MASK_CLRMSK                                            (0xFFFFFE00U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX                                                    (0x0A00U)
+#define RGX_CR_META_SP_MSLVDATAX_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT                                                    (0x0A08U)
+#define RGX_CR_META_SP_MSLVDATAT_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0                                                    (0x0A10U)
+#define RGX_CR_META_SP_MSLVCTRL0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_SHIFT                                         (2U)
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_CLRMSK                                        (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_SHIFT                                     (1U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN                                        (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_SHIFT                                           (0U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1                                                    (0x0A18U)
+#define RGX_CR_META_SP_MSLVCTRL1_MASKFULL                                           (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_SHIFT                                 (30U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_CLRMSK                                (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_SHIFT                              (29U)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_EN                                 (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_SHIFT                             (28U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_CLRMSK                            (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_EN                                (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_SHIFT                                 (26U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN                                    (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_SHIFT                                 (25U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_CLRMSK                                (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_EN                                    (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_SHIFT                                        (24U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_CLRMSK                                       (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_EN                                           (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_SHIFT                                     (21U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_CLRMSK                                    (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_SHIFT                                       (20U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_CLRMSK                                      (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_EN                                          (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_SHIFT                                    (18U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_CLRMSK                                   (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_EN                                       (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_SHIFT                                       (4U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_CLRMSK                                      (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_CLRMSK                                  (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_SHIFT                                   (0U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_CLRMSK                                  (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE                                                 (0x0A50U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_MASKFULL                                        (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_SHIFT                                     (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK                                                   (0x0A80U)
+#define RGX_CR_META_SP_MSLVT0KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI                                                  (0x0A88U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK                                                   (0x0A90U)
+#define RGX_CR_META_SP_MSLVT1KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI                                                  (0x0A98U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK                                                   (0x0AA0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI                                                  (0x0AA8U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK                                                   (0x0AB0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI                                                  (0x0AB8U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST                                                      (0x0AC0U)
+#define RGX_CR_META_SP_MSLVRST_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS                                                (0x0AC8U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_SHIFT                                (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_EN                                   (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_SHIFT                                (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN                                   (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE                                                (0x0AD0U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_SHIFT                                   (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_CLRMSK                                  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_EN                                      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_EN                                      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL                                                 (0x0AD8U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE
+*/
+#define RGX_CR_MTS_SCHEDULE                                                         (0x0B00U)
+#define RGX_CR_MTS_SCHEDULE_MASKFULL                                                (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE_HOST_SHIFT                                              (8U)
+#define RGX_CR_MTS_SCHEDULE_HOST_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE_HOST_BG_TIMER                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_HOST_HOST                                               (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_SHIFT                                          (6U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_CLRMSK                                         (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT0                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT1                                           (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT2                                           (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT3                                           (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_SHIFT                                           (5U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_CLRMSK                                          (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_BGCTX                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_INTCTX                                          (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE_TASK_SHIFT                                              (4U)
+#define RGX_CR_MTS_SCHEDULE_TASK_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED                                        (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_TASK_COUNTED                                            (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE_DM_SHIFT                                                (0U)
+#define RGX_CR_MTS_SCHEDULE_DM_CLRMSK                                               (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM0                                                  (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM1                                                  (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM2                                                  (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM3                                                  (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM4                                                  (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM5                                                  (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM6                                                  (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM7                                                  (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM_ALL                                               (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE1
+*/
+#define RGX_CR_MTS_SCHEDULE1                                                        (0x10B00U)
+#define RGX_CR_MTS_SCHEDULE1_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE1_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE1_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE1_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE1_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE2
+*/
+#define RGX_CR_MTS_SCHEDULE2                                                        (0x20B00U)
+#define RGX_CR_MTS_SCHEDULE2_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE2_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE2_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE2_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE2_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE3
+*/
+#define RGX_CR_MTS_SCHEDULE3                                                        (0x30B00U)
+#define RGX_CR_MTS_SCHEDULE3_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE3_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE3_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE3_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE3_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE4
+*/
+#define RGX_CR_MTS_SCHEDULE4                                                        (0x40B00U)
+#define RGX_CR_MTS_SCHEDULE4_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE4_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE4_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE4_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE4_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE5
+*/
+#define RGX_CR_MTS_SCHEDULE5                                                        (0x50B00U)
+#define RGX_CR_MTS_SCHEDULE5_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE5_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE5_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE5_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE5_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE6
+*/
+#define RGX_CR_MTS_SCHEDULE6                                                        (0x60B00U)
+#define RGX_CR_MTS_SCHEDULE6_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE6_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE6_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE6_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE6_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE7
+*/
+#define RGX_CR_MTS_SCHEDULE7                                                        (0x70B00U)
+#define RGX_CR_MTS_SCHEDULE7_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE7_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE7_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE7_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE7_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC                                           (0x0B30U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC                                           (0x0B38U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC                                          (0x0B40U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC                                          (0x0B48U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_GARTEN_WRAPPER_CONFIG
+*/
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG                                            (0x0B50U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__MASKFULL                        (IMG_UINT64_C(0x000FF0FFFFFFF701))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_MASKFULL                                   (IMG_UINT64_C(0x0000FFFFFFFFF001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT                        (44U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK                       (IMG_UINT64_C(0xFFFF0FFFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT             (44U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_CLRMSK            (IMG_UINT64_C(0xFFF00FFFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT                             (40U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_CLRMSK                            (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_ADDR_SHIFT                           (12U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_ADDR_CLRMSK                          (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PERSISTENCE_SHIFT         (9U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PERSISTENCE_CLRMSK        (IMG_UINT64_C(0xFFFFFFFFFFFFF9FF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_SHIFT        (8U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_CLRMSK       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_EN           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_SHIFT                            (0U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_MTS                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S7_INFRA__META                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S7_INFRA__MTS                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MTS_DM0_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE                                             (0x0B58U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM1_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE                                             (0x0B60U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM2_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE                                             (0x0B68U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM3_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE                                             (0x0B70U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM4_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE                                             (0x0B78U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM5_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE                                             (0x0B80U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX
+*/
+#define RGX_CR_MTS_INTCTX                                                           (0x0B98U)
+#define RGX_CR_MTS_INTCTX_MASKFULL                                                  (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_SHIFT                                    (22U)
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_CLRMSK                                   (0xC03FFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_PTR_SHIFT                                              (18U)
+#define RGX_CR_MTS_INTCTX_DM_PTR_CLRMSK                                             (0xFFC3FFFFU)
+#define RGX_CR_MTS_INTCTX_THREAD_ACTIVE_SHIFT                                       (16U)
+#define RGX_CR_MTS_INTCTX_THREAD_ACTIVE_CLRMSK                                      (0xFFFCFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_SHIFT                                   (8U)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_CLRMSK                                  (0xFFFF00FFU)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX
+*/
+#define RGX_CR_MTS_BGCTX                                                            (0x0BA0U)
+#define RGX_CR_MTS_BGCTX_MASKFULL                                                   (IMG_UINT64_C(0x0000000000003FFF))
+#define RGX_CR_MTS_BGCTX_DM_PTR_SHIFT                                               (10U)
+#define RGX_CR_MTS_BGCTX_DM_PTR_CLRMSK                                              (0xFFFFC3FFU)
+#define RGX_CR_MTS_BGCTX_THREAD_ACTIVE_SHIFT                                        (8U)
+#define RGX_CR_MTS_BGCTX_THREAD_ACTIVE_CLRMSK                                       (0xFFFFFCFFU)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE
+*/
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE                                           (0x0BA8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_SHIFT                                 (56U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_CLRMSK                                (IMG_UINT64_C(0x00FFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_SHIFT                                 (48U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_CLRMSK                                (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_SHIFT                                 (40U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_CLRMSK                                (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_SHIFT                                 (32U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_CLRMSK                                (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_SHIFT                                 (24U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_SHIFT                                 (16U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_SHIFT                                 (8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_SHIFT                                 (0U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MTS_GPU_INT_STATUS
+*/
+#define RGX_CR_MTS_GPU_INT_STATUS                                                   (0x0BB0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_SHIFT                                      (0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE_ENABLE
+*/
+#define RGX_CR_MTS_SCHEDULE_ENABLE                                                  (0x0BC8U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_SHIFT                                       (0U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_CLRMSK                                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS0_EVENT_STATUS                                                 (0x0BD8U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR                                                  (0x0BE8U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS1_EVENT_STATUS                                                 (0x10BD8U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR                                                  (0x10BE8U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS2_EVENT_STATUS                                                 (0x20BD8U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR                                                  (0x20BE8U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS3_EVENT_STATUS                                                 (0x30BD8U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR                                                  (0x30BE8U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS4_EVENT_STATUS                                                 (0x40BD8U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR                                                  (0x40BE8U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS5_EVENT_STATUS                                                 (0x50BD8U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR                                                  (0x50BE8U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS6_EVENT_STATUS                                                 (0x60BD8U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR                                                  (0x60BE8U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS7_EVENT_STATUS                                                 (0x70BD8U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR                                                  (0x70BE8U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_BOOT
+*/
+#define RGX_CR_META_BOOT                                                            (0x0BF8U)
+#define RGX_CR_META_BOOT_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_BOOT_MODE_SHIFT                                                 (0U)
+#define RGX_CR_META_BOOT_MODE_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_META_BOOT_MODE_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_GARTEN_SLC
+*/
+#define RGX_CR_GARTEN_SLC                                                           (0x0BB8U)
+#define RGX_CR_GARTEN_SLC_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_SHIFT                                     (0U)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_PPP
+*/
+#define RGX_CR_PPP                                                                  (0x0CD0U)
+#define RGX_CR_PPP_MASKFULL                                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CHECKSUM_SHIFT                                                   (0U)
+#define RGX_CR_PPP_CHECKSUM_CLRMSK                                                  (0x00000000U)
+
+
+#define RGX_CR_ISP_RENDER_DIR_TYPE_MASK                                             (0x00000003U)
+/*
+Top-left to bottom-right */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_TL2BR                                            (0x00000000U)
+/*
+Top-right to bottom-left */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_TR2BL                                            (0x00000001U)
+/*
+Bottom-left to top-right */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_BL2TR                                            (0x00000002U)
+/*
+Bottom-right to top-left */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_BR2TL                                            (0x00000003U)
+
+
+#define RGX_CR_ISP_RENDER_MODE_TYPE_MASK                                            (0x00000003U)
+/*
+Normal render     */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_NORM                                            (0x00000000U)
+/*
+Fast 2D render    */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_FAST_2D                                         (0x00000002U)
+/*
+Fast scale render */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_FAST_SCALE                                      (0x00000003U)
+
+
+/*
+    Register RGX_CR_ISP_RENDER
+*/
+#define RGX_CR_ISP_RENDER                                                           (0x0F08U)
+#define RGX_CR_ISP_RENDER_MASKFULL                                                  (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_SHIFT                           (8U)
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_CLRMSK                          (0xFFFFFEFFU)
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_EN                              (0x00000100U)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_SHIFT                             (7U)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_CLRMSK                            (0xFFFFFF7FU)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_EN                                (0x00000080U)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_SHIFT                           (6U)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_CLRMSK                          (0xFFFFFFBFU)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_EN                              (0x00000040U)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_SHIFT                                        (5U)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_EN                                           (0x00000020U)
+#define RGX_CR_ISP_RENDER_RESUME_SHIFT                                              (4U)
+#define RGX_CR_ISP_RENDER_RESUME_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_ISP_RENDER_RESUME_EN                                                 (0x00000010U)
+#define RGX_CR_ISP_RENDER_DIR_SHIFT                                                 (2U)
+#define RGX_CR_ISP_RENDER_DIR_CLRMSK                                                (0xFFFFFFF3U)
+#define RGX_CR_ISP_RENDER_DIR_TL2BR                                                 (0x00000000U)
+#define RGX_CR_ISP_RENDER_DIR_TR2BL                                                 (0x00000004U)
+#define RGX_CR_ISP_RENDER_DIR_BL2TR                                                 (0x00000008U)
+#define RGX_CR_ISP_RENDER_DIR_BR2TL                                                 (0x0000000CU)
+#define RGX_CR_ISP_RENDER_MODE_SHIFT                                                (0U)
+#define RGX_CR_ISP_RENDER_MODE_CLRMSK                                               (0xFFFFFFFCU)
+#define RGX_CR_ISP_RENDER_MODE_NORM                                                 (0x00000000U)
+#define RGX_CR_ISP_RENDER_MODE_FAST_2D                                              (0x00000002U)
+#define RGX_CR_ISP_RENDER_MODE_FAST_SCALE                                           (0x00000003U)
+
+
+/*
+    Register RGX_CR_ISP_CTL
+*/
+#define RGX_CR_ISP_CTL                                                              (0x0F38U)
+#define RGX_CR_ISP_CTL_MASKFULL                                                     (IMG_UINT64_C(0x00000000FFFFF3FF))
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_SHIFT                                         (31U)
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_CLRMSK                                        (0x7FFFFFFFU)
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_EN                                            (0x80000000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SHIFT                                             (30U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_CLRMSK                                            (0xBFFFFFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_EN                                                (0x40000000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_SHIFT                                         (29U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_CLRMSK                                        (0xDFFFFFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_EN                                            (0x20000000U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_SHIFT                                        (28U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_CLRMSK                                       (0xEFFFFFFFU)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_EN                                           (0x10000000U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_SHIFT                                             (27U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_CLRMSK                                            (0xF7FFFFFFU)
+#define RGX_CR_ISP_CTL_PAIR_TILES_EN                                                (0x08000000U)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_SHIFT                                            (26U)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_CLRMSK                                           (0xFBFFFFFFU)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_EN                                               (0x04000000U)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_SHIFT                                            (25U)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_CLRMSK                                           (0xFDFFFFFFU)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_EN                                               (0x02000000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_SHIFT                                    (23U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_CLRMSK                                   (0xFE7FFFFFU)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_DX9                                      (0x00000000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_DX10                                     (0x00800000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_OGL                                      (0x01000000U)
+#define RGX_CR_ISP_CTL_NUM_TILES_PER_USC_SHIFT                                      (21U)
+#define RGX_CR_ISP_CTL_NUM_TILES_PER_USC_CLRMSK                                     (0xFF9FFFFFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_SHIFT                                           (20U)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_CLRMSK                                          (0xFFEFFFFFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_EN                                              (0x00100000U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_SHIFT                                     (19U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_CLRMSK                                    (0xFFF7FFFFU)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_EN                                        (0x00080000U)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_SHIFT                               (18U)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_EN                                  (0x00040000U)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_SHIFT                                    (17U)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_CLRMSK                                   (0xFFFDFFFFU)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_EN                                       (0x00020000U)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_SHIFT                                             (16U)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_CLRMSK                                            (0xFFFEFFFFU)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_EN                                                (0x00010000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_SHIFT                                            (12U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_CLRMSK                                           (0xFFFF0FFFU)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_ONE                                         (0x00000000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TWO                                         (0x00001000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_THREE                                       (0x00002000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FOUR                                        (0x00003000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FIVE                                        (0x00004000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SIX                                         (0x00005000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SEVEN                                       (0x00006000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_EIGHT                                       (0x00007000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_NINE                                        (0x00008000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TEN                                         (0x00009000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_ELEVEN                                      (0x0000A000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TWELVE                                      (0x0000B000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_THIRTEEN                                    (0x0000C000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FOURTEEN                                    (0x0000D000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FIFTEEN                                     (0x0000E000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SIXTEEN                                     (0x0000F000U)
+#define RGX_CR_ISP_CTL_VALID_ID_SHIFT                                               (4U)
+#define RGX_CR_ISP_CTL_VALID_ID_CLRMSK                                              (0xFFFFFC0FU)
+#define RGX_CR_ISP_CTL_UPASS_START_SHIFT                                            (0U)
+#define RGX_CR_ISP_CTL_UPASS_START_CLRMSK                                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_ISP_STATUS
+*/
+#define RGX_CR_ISP_STATUS                                                           (0x1038U)
+#define RGX_CR_ISP_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_SHIFT                                           (2U)
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_EN                                              (0x00000004U)
+#define RGX_CR_ISP_STATUS_ACTIVE_SHIFT                                              (1U)
+#define RGX_CR_ISP_STATUS_ACTIVE_CLRMSK                                             (0xFFFFFFFDU)
+#define RGX_CR_ISP_STATUS_ACTIVE_EN                                                 (0x00000002U)
+#define RGX_CR_ISP_STATUS_EOR_SHIFT                                                 (0U)
+#define RGX_CR_ISP_STATUS_EOR_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_ISP_STATUS_EOR_EN                                                    (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_ISP_XTP_RESUME, with 64 repeats
+*/
+#define RGX_CR_ISP_XTP_RESUME_REPEATCOUNT                                           (64U)
+/*
+    Register RGX_CR_ISP_XTP_RESUME0
+*/
+#define RGX_CR_ISP_XTP_RESUME0                                                      (0x3A00U)
+#define RGX_CR_ISP_XTP_RESUME0_MASKFULL                                             (IMG_UINT64_C(0x00000000003FF3FF))
+#define RGX_CR_ISP_XTP_RESUME0_TILE_X_SHIFT                                         (12U)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_X_CLRMSK                                        (0xFFC00FFFU)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_Y_SHIFT                                         (0U)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_Y_CLRMSK                                        (0xFFFFFC00U)
+
+
+/*
+    Register group: RGX_CR_ISP_XTP_STORE, with 32 repeats
+*/
+#define RGX_CR_ISP_XTP_STORE_REPEATCOUNT                                            (32U)
+/*
+    Register RGX_CR_ISP_XTP_STORE0
+*/
+#define RGX_CR_ISP_XTP_STORE0                                                       (0x3C00U)
+#define RGX_CR_ISP_XTP_STORE0_MASKFULL                                              (IMG_UINT64_C(0x000000007F3FF3FF))
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_SHIFT                                          (30U)
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_CLRMSK                                         (0xBFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_EN                                             (0x40000000U)
+#define RGX_CR_ISP_XTP_STORE0_EOR_SHIFT                                             (29U)
+#define RGX_CR_ISP_XTP_STORE0_EOR_CLRMSK                                            (0xDFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_EOR_EN                                                (0x20000000U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_SHIFT                                       (28U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_CLRMSK                                      (0xEFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_EN                                          (0x10000000U)
+#define RGX_CR_ISP_XTP_STORE0_MT_SHIFT                                              (24U)
+#define RGX_CR_ISP_XTP_STORE0_MT_CLRMSK                                             (0xF0FFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_X_SHIFT                                          (12U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_X_CLRMSK                                         (0xFFC00FFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_Y_SHIFT                                          (0U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_Y_CLRMSK                                         (0xFFFFFC00U)
+
+
+/*
+    Register group: RGX_CR_BIF_CAT_BASE, with 8 repeats
+*/
+#define RGX_CR_BIF_CAT_BASE_REPEATCOUNT                                             (8U)
+/*
+    Register RGX_CR_BIF_CAT_BASE0
+*/
+#define RGX_CR_BIF_CAT_BASE0                                                        (0x1200U)
+#define RGX_CR_BIF_CAT_BASE0_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE1
+*/
+#define RGX_CR_BIF_CAT_BASE1                                                        (0x1208U)
+#define RGX_CR_BIF_CAT_BASE1_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE1_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE1_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE1_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE1_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE2
+*/
+#define RGX_CR_BIF_CAT_BASE2                                                        (0x1210U)
+#define RGX_CR_BIF_CAT_BASE2_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE2_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE2_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE2_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE2_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE3
+*/
+#define RGX_CR_BIF_CAT_BASE3                                                        (0x1218U)
+#define RGX_CR_BIF_CAT_BASE3_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE3_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE3_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE3_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE3_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE4
+*/
+#define RGX_CR_BIF_CAT_BASE4                                                        (0x1220U)
+#define RGX_CR_BIF_CAT_BASE4_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE4_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE4_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE4_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE4_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE5
+*/
+#define RGX_CR_BIF_CAT_BASE5                                                        (0x1228U)
+#define RGX_CR_BIF_CAT_BASE5_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE5_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE5_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE5_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE5_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE6
+*/
+#define RGX_CR_BIF_CAT_BASE6                                                        (0x1230U)
+#define RGX_CR_BIF_CAT_BASE6_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE6_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE6_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE6_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE6_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE7
+*/
+#define RGX_CR_BIF_CAT_BASE7                                                        (0x1238U)
+#define RGX_CR_BIF_CAT_BASE7_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE7_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE7_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE7_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE7_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE_INDEX
+*/
+#define RGX_CR_BIF_CAT_BASE_INDEX                                                   (0x1240U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_MASKFULL                                          (IMG_UINT64_C(0x00070707073F0707))
+#define RGX_CR_BIF_CAT_BASE_INDEX_RVTX_SHIFT                                        (48U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_RVTX_CLRMSK                                       (IMG_UINT64_C(0xFFF8FFFFFFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_RAY_SHIFT                                         (40U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_RAY_CLRMSK                                        (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_HOST_SHIFT                                        (32U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_HOST_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TLA_SHIFT                                         (24U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TLA_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF8FFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TDM_SHIFT                                         (19U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TDM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFC7FFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_CDM_SHIFT                                         (16U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_CDM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFF8FFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_PIXEL_SHIFT                                       (8U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_PIXEL_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFF8FF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TA_SHIFT                                          (0U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF8))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_VCE0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0                                                 (0x1248U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_MASKFULL                                        (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_INIT_PAGE_SHIFT                                 (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_INIT_PAGE_CLRMSK                                (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_ADDR_SHIFT                                      (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_SHIFT                                      (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_EN                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_SHIFT                                     (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_EN                                        (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_TE0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_TE0                                                  (0x1250U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_MASKFULL                                         (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_INIT_PAGE_SHIFT                                  (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_INIT_PAGE_CLRMSK                                 (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_ADDR_SHIFT                                       (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_ADDR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_SHIFT                                       (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_SHIFT                                      (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_ALIST0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0                                               (0x1260U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_MASKFULL                                      (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_INIT_PAGE_SHIFT                               (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_INIT_PAGE_CLRMSK                              (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_ADDR_SHIFT                                    (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_SHIFT                                    (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_EN                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_SHIFT                                   (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_VCE1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1                                                 (0x1268U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_MASKFULL                                        (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_INIT_PAGE_SHIFT                                 (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_INIT_PAGE_CLRMSK                                (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_ADDR_SHIFT                                      (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_SHIFT                                      (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_EN                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_SHIFT                                     (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_EN                                        (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_TE1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_TE1                                                  (0x1270U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_MASKFULL                                         (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_INIT_PAGE_SHIFT                                  (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_INIT_PAGE_CLRMSK                                 (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_ADDR_SHIFT                                       (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_ADDR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_SHIFT                                       (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_SHIFT                                      (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_ALIST1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1                                               (0x1280U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_MASKFULL                                      (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_INIT_PAGE_SHIFT                               (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_INIT_PAGE_CLRMSK                              (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_ADDR_SHIFT                                    (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_SHIFT                                    (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_EN                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_SHIFT                                   (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_MMU_ENTRY_STATUS
+*/
+#define RGX_CR_BIF_MMU_ENTRY_STATUS                                                 (0x1288U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF0F3))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_SHIFT                                   (12U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_CLRMSK                                  (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_SHIFT                                  (4U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFF0F))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_DATA_TYPE_SHIFT                                 (0U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_DATA_TYPE_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_BIF_MMU_ENTRY
+*/
+#define RGX_CR_BIF_MMU_ENTRY                                                        (0x1290U)
+#define RGX_CR_BIF_MMU_ENTRY_MASKFULL                                               (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_SHIFT                                           (1U)
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_EN                                              (0x00000002U)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_SHIFT                                          (0U)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_CTRL_INVAL
+*/
+#define RGX_CR_BIF_CTRL_INVAL                                                       (0x12A0U)
+#define RGX_CR_BIF_CTRL_INVAL_MASKFULL                                              (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_SHIFT                                            (3U)
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_EN                                               (0x00000008U)
+#define RGX_CR_BIF_CTRL_INVAL_PC_SHIFT                                              (2U)
+#define RGX_CR_BIF_CTRL_INVAL_PC_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_BIF_CTRL_INVAL_PC_EN                                                 (0x00000004U)
+#define RGX_CR_BIF_CTRL_INVAL_PD_SHIFT                                              (1U)
+#define RGX_CR_BIF_CTRL_INVAL_PD_CLRMSK                                             (0xFFFFFFFDU)
+#define RGX_CR_BIF_CTRL_INVAL_PD_EN                                                 (0x00000002U)
+#define RGX_CR_BIF_CTRL_INVAL_PT_SHIFT                                              (0U)
+#define RGX_CR_BIF_CTRL_INVAL_PT_CLRMSK                                             (0xFFFFFFFEU)
+#define RGX_CR_BIF_CTRL_INVAL_PT_EN                                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_CTRL
+*/
+#define RGX_CR_BIF_CTRL                                                             (0x12A8U)
+#define RGX_CR_BIF_CTRL__XE_MEM__MASKFULL                                           (IMG_UINT64_C(0x000000000000033F))
+#define RGX_CR_BIF_CTRL_MASKFULL                                                    (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_SHIFT                                (9U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_CLRMSK                               (0xFFFFFDFFU)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_EN                                   (0x00000200U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_SHIFT                               (8U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_CLRMSK                              (0xFFFFFEFFU)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_EN                                  (0x00000100U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_SHIFT                               (7U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_CLRMSK                              (0xFFFFFF7FU)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_EN                                  (0x00000080U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_SHIFT                              (6U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_EN                                 (0x00000040U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_SHIFT                                        (5U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_EN                                           (0x00000020U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_SHIFT                                        (4U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_CLRMSK                                       (0xFFFFFFEFU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_EN                                           (0x00000010U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_SHIFT                                            (3U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_EN                                               (0x00000008U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_SHIFT                                          (2U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_EN                                             (0x00000004U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_SHIFT                                        (1U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_CLRMSK                                       (0xFFFFFFFDU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_EN                                           (0x00000002U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_SHIFT                                        (0U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK0_MMU_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS                                           (0x12B0U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_MASKFULL                                  (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT                            (12U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK                           (0xFFFF0FFFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT                           (8U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK                          (0xFFFFF8FFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT                           (5U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK                          (0xFFFFFF9FU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_SHIFT                            (4U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN                               (0x00000010U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_SHIFT                    (2U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN                       (0x00000004U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_SHIFT                               (0U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK0_REQ_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS                                           (0x12B8U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__MASKFULL                         (IMG_UINT64_C(0x001FFFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_MASKFULL                                  (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_SHIFT                        (52U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_CLRMSK                       (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN                           (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_SHIFT                                 (50U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_CLRMSK                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT                     (46U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK                    (IMG_UINT64_C(0xFFF03FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT                              (44U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK                             (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT                              (40U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT                     (40U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK                    (IMG_UINT64_C(0xFFFFC0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT                             (4U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK                            (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT                        (4U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSIZE                         (16U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK1_MMU_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS                                           (0x12C0U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_MASKFULL                                  (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_CAT_BASE_SHIFT                            (12U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_CAT_BASE_CLRMSK                           (0xFFFF0FFFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_PAGE_SIZE_SHIFT                           (8U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_PAGE_SIZE_CLRMSK                          (0xFFFFF8FFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_DATA_TYPE_SHIFT                           (5U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_DATA_TYPE_CLRMSK                          (0xFFFFFF9FU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_SHIFT                            (4U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_EN                               (0x00000010U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_SHIFT                    (2U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_EN                       (0x00000004U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_SHIFT                               (0U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK1_REQ_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS                                           (0x12C8U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_MASKFULL                                  (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_SHIFT                                 (50U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_CLRMSK                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_EN                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_SB_SHIFT                              (44U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_SB_CLRMSK                             (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_ID_SHIFT                              (40U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_SHIFT                             (4U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_CLRMSK                            (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_ALIGNSHIFT                        (4U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_ALIGNSIZE                         (16U)
+
+
+/*
+    Register RGX_CR_BIF_MMU_STATUS
+*/
+#define RGX_CR_BIF_MMU_STATUS                                                       (0x12D0U)
+#define RGX_CR_BIF_MMU_STATUS__XE_MEM__MASKFULL                                     (IMG_UINT64_C(0x000000001FFFFFF7))
+#define RGX_CR_BIF_MMU_STATUS_MASKFULL                                              (IMG_UINT64_C(0x000000001FFFFFF7))
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_SHIFT                                        (28U)
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_CLRMSK                                       (0xEFFFFFFFU)
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_EN                                           (0x10000000U)
+#define RGX_CR_BIF_MMU_STATUS_PC_DATA_SHIFT                                         (20U)
+#define RGX_CR_BIF_MMU_STATUS_PC_DATA_CLRMSK                                        (0xF00FFFFFU)
+#define RGX_CR_BIF_MMU_STATUS_PD_DATA_SHIFT                                         (12U)
+#define RGX_CR_BIF_MMU_STATUS_PD_DATA_CLRMSK                                        (0xFFF00FFFU)
+#define RGX_CR_BIF_MMU_STATUS_PT_DATA_SHIFT                                         (4U)
+#define RGX_CR_BIF_MMU_STATUS_PT_DATA_CLRMSK                                        (0xFFFFF00FU)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_SHIFT                                         (2U)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_CLRMSK                                        (0xFFFFFFFBU)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_EN                                            (0x00000004U)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_SHIFT                                          (1U)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_EN                                             (0x00000002U)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_SHIFT                                            (0U)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_EN                                               (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_BIF_TILING_CFG, with 8 repeats
+*/
+#define RGX_CR_BIF_TILING_CFG_REPEATCOUNT                                           (8U)
+/*
+    Register RGX_CR_BIF_TILING_CFG0
+*/
+#define RGX_CR_BIF_TILING_CFG0                                                      (0x12D8U)
+#define RGX_CR_BIF_TILING_CFG0_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG0_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG1
+*/
+#define RGX_CR_BIF_TILING_CFG1                                                      (0x12E0U)
+#define RGX_CR_BIF_TILING_CFG1_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG1_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG2
+*/
+#define RGX_CR_BIF_TILING_CFG2                                                      (0x12E8U)
+#define RGX_CR_BIF_TILING_CFG2_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG2_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG3
+*/
+#define RGX_CR_BIF_TILING_CFG3                                                      (0x12F0U)
+#define RGX_CR_BIF_TILING_CFG3_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG3_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG4
+*/
+#define RGX_CR_BIF_TILING_CFG4                                                      (0x12F8U)
+#define RGX_CR_BIF_TILING_CFG4_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG4_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG5
+*/
+#define RGX_CR_BIF_TILING_CFG5                                                      (0x1300U)
+#define RGX_CR_BIF_TILING_CFG5_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG5_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG6
+*/
+#define RGX_CR_BIF_TILING_CFG6                                                      (0x1308U)
+#define RGX_CR_BIF_TILING_CFG6_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG6_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG7
+*/
+#define RGX_CR_BIF_TILING_CFG7                                                      (0x1310U)
+#define RGX_CR_BIF_TILING_CFG7_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG7_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_READS_EXT_STATUS
+*/
+#define RGX_CR_BIF_READS_EXT_STATUS                                                 (0x1320U)
+#define RGX_CR_BIF_READS_EXT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_BIF_READS_EXT_STATUS_MMU_SHIFT                                       (16U)
+#define RGX_CR_BIF_READS_EXT_STATUS_MMU_CLRMSK                                      (0xF000FFFFU)
+#define RGX_CR_BIF_READS_EXT_STATUS_BANK1_SHIFT                                     (0U)
+#define RGX_CR_BIF_READS_EXT_STATUS_BANK1_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_READS_INT_STATUS
+*/
+#define RGX_CR_BIF_READS_INT_STATUS                                                 (0x1328U)
+#define RGX_CR_BIF_READS_INT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000007FFFFFF))
+#define RGX_CR_BIF_READS_INT_STATUS_MMU_SHIFT                                       (16U)
+#define RGX_CR_BIF_READS_INT_STATUS_MMU_CLRMSK                                      (0xF800FFFFU)
+#define RGX_CR_BIF_READS_INT_STATUS_BANK1_SHIFT                                     (0U)
+#define RGX_CR_BIF_READS_INT_STATUS_BANK1_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_READS_INT_STATUS
+*/
+#define RGX_CR_BIFPM_READS_INT_STATUS                                               (0x1330U)
+#define RGX_CR_BIFPM_READS_INT_STATUS_MASKFULL                                      (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIFPM_READS_INT_STATUS_BANK0_SHIFT                                   (0U)
+#define RGX_CR_BIFPM_READS_INT_STATUS_BANK0_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_READS_EXT_STATUS
+*/
+#define RGX_CR_BIFPM_READS_EXT_STATUS                                               (0x1338U)
+#define RGX_CR_BIFPM_READS_EXT_STATUS_MASKFULL                                      (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIFPM_READS_EXT_STATUS_BANK0_SHIFT                                   (0U)
+#define RGX_CR_BIFPM_READS_EXT_STATUS_BANK0_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_STATUS_MMU
+*/
+#define RGX_CR_BIFPM_STATUS_MMU                                                     (0x1350U)
+#define RGX_CR_BIFPM_STATUS_MMU_MASKFULL                                            (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIFPM_STATUS_MMU_REQUESTS_SHIFT                                      (0U)
+#define RGX_CR_BIFPM_STATUS_MMU_REQUESTS_CLRMSK                                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_STATUS_MMU
+*/
+#define RGX_CR_BIF_STATUS_MMU                                                       (0x1358U)
+#define RGX_CR_BIF_STATUS_MMU_MASKFULL                                              (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIF_STATUS_MMU_REQUESTS_SHIFT                                        (0U)
+#define RGX_CR_BIF_STATUS_MMU_REQUESTS_CLRMSK                                       (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_READ
+*/
+#define RGX_CR_BIF_FAULT_READ                                                       (0x13E0U)
+#define RGX_CR_BIF_FAULT_READ_MASKFULL                                              (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_SHIFT                                         (4U)
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_ALIGNSHIFT                                    (4U)
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_ALIGNSIZE                                     (16U)
+
+
+/*
+    Register RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS
+*/
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS                                     (0x1430U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_MASKFULL                            (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT                      (12U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK                     (0xFFFF0FFFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT                     (8U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK                    (0xFFFFF8FFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT                     (5U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK                    (0xFFFFFF9FU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_SHIFT                      (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_CLRMSK                     (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN                         (0x00000010U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_SHIFT              (2U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_CLRMSK             (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN                 (0x00000004U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_SHIFT                         (0U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_EN                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS
+*/
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS                                     (0x1438U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_MASKFULL                            (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_SHIFT                           (50U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_CLRMSK                          (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN                              (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT                        (44U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK                       (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT                        (40U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK                       (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT                       (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK                      (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT                  (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSIZE                   (16U)
+
+
+/*
+    Register RGX_CR_TFBC_COMPRESSION_CONTROL
+*/
+#define RGX_CR_TFBC_COMPRESSION_CONTROL                                             (0x14A0U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_MASKFULL                                    (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_SHIFT            (8U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_CLRMSK           (0xFFFFFEFFU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN               (0x00000100U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_SHIFT                        (7U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN                           (0x00000080U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_SHIFT_SHIFT                         (4U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_SHIFT_CLRMSK                        (0xFFFFFF8FU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_SHIFT                        (3U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_CLRMSK                       (0xFFFFFFF7U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_EN                           (0x00000008U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT                                (1U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK                               (0xFFFFFFF9U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_DEFAULT                              (0x00000000U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD_AND_CORRELATION  (0x00000002U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD                  (0x00000004U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_RESERVED                             (0x00000006U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT                         (0U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_0                       (0x00000000U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_1                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_MCU_FENCE
+*/
+#define RGX_CR_MCU_FENCE                                                            (0x1740U)
+#define RGX_CR_MCU_FENCE_MASKFULL                                                   (IMG_UINT64_C(0x000007FFFFFFFFE0))
+#define RGX_CR_MCU_FENCE_DM_SHIFT                                                   (40U)
+#define RGX_CR_MCU_FENCE_DM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_MCU_FENCE_DM_VERTEX                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MCU_FENCE_DM_PIXEL                                                   (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MCU_FENCE_DM_COMPUTE                                                 (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MCU_FENCE_DM_RAY_VERTEX                                              (IMG_UINT64_C(0x0000030000000000))
+#define RGX_CR_MCU_FENCE_DM_RAY                                                     (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_MCU_FENCE_DM_FASTRENDER                                              (IMG_UINT64_C(0x0000050000000000))
+#define RGX_CR_MCU_FENCE_ADDR_SHIFT                                                 (5U)
+#define RGX_CR_MCU_FENCE_ADDR_CLRMSK                                                (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_CR_MCU_FENCE_ADDR_ALIGNSHIFT                                            (5U)
+#define RGX_CR_MCU_FENCE_ADDR_ALIGNSIZE                                             (32U)
+
+
+/*
+    Register group: RGX_CR_SCRATCH, with 16 repeats
+*/
+#define RGX_CR_SCRATCH_REPEATCOUNT                                                  (16U)
+/*
+    Register RGX_CR_SCRATCH0
+*/
+#define RGX_CR_SCRATCH0                                                             (0x1A00U)
+#define RGX_CR_SCRATCH0_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH0_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH0_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH1
+*/
+#define RGX_CR_SCRATCH1                                                             (0x1A08U)
+#define RGX_CR_SCRATCH1_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH1_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH1_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH2
+*/
+#define RGX_CR_SCRATCH2                                                             (0x1A10U)
+#define RGX_CR_SCRATCH2_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH2_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH2_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH3
+*/
+#define RGX_CR_SCRATCH3                                                             (0x1A18U)
+#define RGX_CR_SCRATCH3_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH3_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH3_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH4
+*/
+#define RGX_CR_SCRATCH4                                                             (0x1A20U)
+#define RGX_CR_SCRATCH4_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH4_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH4_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH5
+*/
+#define RGX_CR_SCRATCH5                                                             (0x1A28U)
+#define RGX_CR_SCRATCH5_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH5_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH5_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH6
+*/
+#define RGX_CR_SCRATCH6                                                             (0x1A30U)
+#define RGX_CR_SCRATCH6_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH6_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH6_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH7
+*/
+#define RGX_CR_SCRATCH7                                                             (0x1A38U)
+#define RGX_CR_SCRATCH7_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH7_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH7_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH8
+*/
+#define RGX_CR_SCRATCH8                                                             (0x1A40U)
+#define RGX_CR_SCRATCH8_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH8_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH8_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH9
+*/
+#define RGX_CR_SCRATCH9                                                             (0x1A48U)
+#define RGX_CR_SCRATCH9_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH9_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH9_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH10
+*/
+#define RGX_CR_SCRATCH10                                                            (0x1A50U)
+#define RGX_CR_SCRATCH10_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH10_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH10_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH11
+*/
+#define RGX_CR_SCRATCH11                                                            (0x1A58U)
+#define RGX_CR_SCRATCH11_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH11_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH11_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH12
+*/
+#define RGX_CR_SCRATCH12                                                            (0x1A60U)
+#define RGX_CR_SCRATCH12_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH12_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH12_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH13
+*/
+#define RGX_CR_SCRATCH13                                                            (0x1A68U)
+#define RGX_CR_SCRATCH13_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH13_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH13_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH14
+*/
+#define RGX_CR_SCRATCH14                                                            (0x1A70U)
+#define RGX_CR_SCRATCH14_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH14_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH14_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH15
+*/
+#define RGX_CR_SCRATCH15                                                            (0x1A78U)
+#define RGX_CR_SCRATCH15_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH15_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH15_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS0_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS0_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS0_SCRATCH0
+*/
+#define RGX_CR_OS0_SCRATCH0                                                         (0x1A80U)
+#define RGX_CR_OS0_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH1
+*/
+#define RGX_CR_OS0_SCRATCH1                                                         (0x1A88U)
+#define RGX_CR_OS0_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH2
+*/
+#define RGX_CR_OS0_SCRATCH2                                                         (0x1A90U)
+#define RGX_CR_OS0_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS0_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH3
+*/
+#define RGX_CR_OS0_SCRATCH3                                                         (0x1A98U)
+#define RGX_CR_OS0_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS0_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS1_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS1_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS1_SCRATCH0
+*/
+#define RGX_CR_OS1_SCRATCH0                                                         (0x11A80U)
+#define RGX_CR_OS1_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH1
+*/
+#define RGX_CR_OS1_SCRATCH1                                                         (0x11A88U)
+#define RGX_CR_OS1_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH2
+*/
+#define RGX_CR_OS1_SCRATCH2                                                         (0x11A90U)
+#define RGX_CR_OS1_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS1_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH3
+*/
+#define RGX_CR_OS1_SCRATCH3                                                         (0x11A98U)
+#define RGX_CR_OS1_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS1_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS2_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS2_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS2_SCRATCH0
+*/
+#define RGX_CR_OS2_SCRATCH0                                                         (0x21A80U)
+#define RGX_CR_OS2_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH1
+*/
+#define RGX_CR_OS2_SCRATCH1                                                         (0x21A88U)
+#define RGX_CR_OS2_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH2
+*/
+#define RGX_CR_OS2_SCRATCH2                                                         (0x21A90U)
+#define RGX_CR_OS2_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS2_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH3
+*/
+#define RGX_CR_OS2_SCRATCH3                                                         (0x21A98U)
+#define RGX_CR_OS2_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS2_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS3_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS3_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS3_SCRATCH0
+*/
+#define RGX_CR_OS3_SCRATCH0                                                         (0x31A80U)
+#define RGX_CR_OS3_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH1
+*/
+#define RGX_CR_OS3_SCRATCH1                                                         (0x31A88U)
+#define RGX_CR_OS3_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH2
+*/
+#define RGX_CR_OS3_SCRATCH2                                                         (0x31A90U)
+#define RGX_CR_OS3_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS3_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH3
+*/
+#define RGX_CR_OS3_SCRATCH3                                                         (0x31A98U)
+#define RGX_CR_OS3_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS3_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS4_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS4_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS4_SCRATCH0
+*/
+#define RGX_CR_OS4_SCRATCH0                                                         (0x41A80U)
+#define RGX_CR_OS4_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH1
+*/
+#define RGX_CR_OS4_SCRATCH1                                                         (0x41A88U)
+#define RGX_CR_OS4_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH2
+*/
+#define RGX_CR_OS4_SCRATCH2                                                         (0x41A90U)
+#define RGX_CR_OS4_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS4_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH3
+*/
+#define RGX_CR_OS4_SCRATCH3                                                         (0x41A98U)
+#define RGX_CR_OS4_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS4_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS5_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS5_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS5_SCRATCH0
+*/
+#define RGX_CR_OS5_SCRATCH0                                                         (0x51A80U)
+#define RGX_CR_OS5_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH1
+*/
+#define RGX_CR_OS5_SCRATCH1                                                         (0x51A88U)
+#define RGX_CR_OS5_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH2
+*/
+#define RGX_CR_OS5_SCRATCH2                                                         (0x51A90U)
+#define RGX_CR_OS5_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS5_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH3
+*/
+#define RGX_CR_OS5_SCRATCH3                                                         (0x51A98U)
+#define RGX_CR_OS5_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS5_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS6_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS6_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS6_SCRATCH0
+*/
+#define RGX_CR_OS6_SCRATCH0                                                         (0x61A80U)
+#define RGX_CR_OS6_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH1
+*/
+#define RGX_CR_OS6_SCRATCH1                                                         (0x61A88U)
+#define RGX_CR_OS6_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH2
+*/
+#define RGX_CR_OS6_SCRATCH2                                                         (0x61A90U)
+#define RGX_CR_OS6_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS6_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH3
+*/
+#define RGX_CR_OS6_SCRATCH3                                                         (0x61A98U)
+#define RGX_CR_OS6_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS6_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS7_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS7_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS7_SCRATCH0
+*/
+#define RGX_CR_OS7_SCRATCH0                                                         (0x71A80U)
+#define RGX_CR_OS7_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH1
+*/
+#define RGX_CR_OS7_SCRATCH1                                                         (0x71A88U)
+#define RGX_CR_OS7_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH2
+*/
+#define RGX_CR_OS7_SCRATCH2                                                         (0x71A90U)
+#define RGX_CR_OS7_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS7_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH3
+*/
+#define RGX_CR_OS7_SCRATCH3                                                         (0x71A98U)
+#define RGX_CR_OS7_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS7_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_SPFILTER_SIGNAL_DESCR
+*/
+#define RGX_CR_SPFILTER_SIGNAL_DESCR                                                (0x2700U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MASKFULL                                       (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_SHIFT                                     (0U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_CLRMSK                                    (0xFFFF0000U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_ALIGNSHIFT                                (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_SPFILTER_SIGNAL_DESCR_MIN
+*/
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN                                            (0x2708U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_SHIFT                                 (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_ALIGNSHIFT                            (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_ALIGNSIZE                             (16U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_ADDR_REMAP_CONFIG, with 16 repeats
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG_REPEATCOUNT                                 (16U)
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG0
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0                                            (0x3000U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG1
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1                                            (0x3008U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG2
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2                                            (0x3010U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG3
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3                                            (0x3018U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG4
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4                                            (0x3020U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG5
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5                                            (0x3028U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG6
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6                                            (0x3030U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG7
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7                                            (0x3038U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG8
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8                                            (0x3040U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG9
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9                                            (0x3048U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG10
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10                                           (0x3050U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG11
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11                                           (0x3058U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG12
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12                                           (0x3060U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG13
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13                                           (0x3068U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG14
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14                                           (0x3070U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG15
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15                                           (0x3078U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_BOOT
+*/
+#define RGX_CR_FWCORE_BOOT                                                          (0x3090U)
+#define RGX_CR_FWCORE_BOOT_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_BOOT_ENABLE_SHIFT                                             (0U)
+#define RGX_CR_FWCORE_BOOT_ENABLE_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_BOOT_ENABLE_EN                                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_RESET_ADDR
+*/
+#define RGX_CR_FWCORE_RESET_ADDR                                                    (0x3098U)
+#define RGX_CR_FWCORE_RESET_ADDR_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_SHIFT                                         (1U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_CLRMSK                                        (0x00000001U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_ALIGNSHIFT                                    (1U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_ALIGNSIZE                                     (2U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_ADDR
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR                                              (0x30A0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_SHIFT                                   (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_CLRMSK                                  (0x00000001U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_ALIGNSHIFT                              (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_ALIGNSIZE                               (2U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT                                             (0x30A8U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_SHIFT                            (0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS                                          (0x30B0U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_MASKFULL                                 (IMG_UINT64_C(0x000000000000F771))
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_CAT_BASE_SHIFT                           (12U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_CAT_BASE_CLRMSK                          (0xFFFF0FFFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_SHIFT                          (8U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_CLRMSK                         (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_DATA_TYPE_SHIFT                          (5U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_DATA_TYPE_CLRMSK                         (0xFFFFFF9FU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_SHIFT                           (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_CLRMSK                          (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_EN                              (0x00000010U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_SHIFT                              (0U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS                                          (0x30B8U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_MASKFULL                                 (IMG_UINT64_C(0x001FFFFFFFFFFFF0))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_SHIFT                                (52U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_CLRMSK                               (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN                                   (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT                             (46U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK                            (IMG_UINT64_C(0xFFF03FFFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT                             (40U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK                            (IMG_UINT64_C(0xFFFFC0FFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT                            (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK                           (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT                       (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSIZE                        (16U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CTRL_INVAL
+*/
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL                                                (0x30C0U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_MASKFULL                                       (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_SHIFT                                      (3U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_EN                                         (0x00000008U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_SHIFT                                       (2U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_CLRMSK                                      (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_EN                                          (0x00000004U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_SHIFT                                       (1U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_EN                                          (0x00000002U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_SHIFT                                       (0U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_MMU_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_MMU_STATUS                                                (0x30C8U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000FFFFFF7))
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PC_DATA_SHIFT                                  (20U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PC_DATA_CLRMSK                                 (0xF00FFFFFU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PD_DATA_SHIFT                                  (12U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PD_DATA_CLRMSK                                 (0xFFF00FFFU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PT_DATA_SHIFT                                  (4U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PT_DATA_CLRMSK                                 (0xFFFFF00FU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_SHIFT                                  (2U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_EN                                     (0x00000004U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_SHIFT                                   (1U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_EN                                      (0x00000002U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_READS_EXT_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS                                          (0x30D8U)
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MMU_SHIFT                                (0U)
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MMU_CLRMSK                               (0xFFFFF000U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_READS_INT_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS                                          (0x30E0U)
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x00000000000007FF))
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MMU_SHIFT                                (0U)
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MMU_CLRMSK                               (0xFFFFF800U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_FENCE
+*/
+#define RGX_CR_FWCORE_WRAPPER_FENCE                                                 (0x30E8U)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_EN                                           (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_MEM_CAT_BASE, with 8 repeats
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE_REPEATCOUNT                                      (8U)
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE0
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE0                                                 (0x30F0U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE1
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE1                                                 (0x30F8U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE2
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE2                                                 (0x3100U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE3
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE3                                                 (0x3108U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE4
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE4                                                 (0x3110U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE5
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE5                                                 (0x3118U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE6
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE6                                                 (0x3120U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE7
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE7                                                 (0x3128U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_RESET
+*/
+#define RGX_CR_FWCORE_WDT_RESET                                                     (0x3130U)
+#define RGX_CR_FWCORE_WDT_RESET_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WDT_RESET_EN_SHIFT                                            (0U)
+#define RGX_CR_FWCORE_WDT_RESET_EN_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_RESET_EN_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_CTRL
+*/
+#define RGX_CR_FWCORE_WDT_CTRL                                                      (0x3138U)
+#define RGX_CR_FWCORE_WDT_CTRL_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFF1F01))
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_SHIFT                                           (16U)
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_CLRMSK                                          (0x0000FFFFU)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_SHIFT                                      (8U)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_CLRMSK                                     (0xFFFFE0FFU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_COUNT
+*/
+#define RGX_CR_FWCORE_WDT_COUNT                                                     (0x3140U)
+#define RGX_CR_FWCORE_WDT_COUNT_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED0, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED0_REPEATCOUNT                                     (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED00
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00                                                (0x3400U)
+#define RGX_CR_FWCORE_DMI_RESERVED00_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED01
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED01                                                (0x3408U)
+#define RGX_CR_FWCORE_DMI_RESERVED01_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED02
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED02                                                (0x3410U)
+#define RGX_CR_FWCORE_DMI_RESERVED02_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED03
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED03                                                (0x3418U)
+#define RGX_CR_FWCORE_DMI_RESERVED03_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA0
+*/
+#define RGX_CR_FWCORE_DMI_DATA0                                                     (0x3420U)
+#define RGX_CR_FWCORE_DMI_DATA0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA1
+*/
+#define RGX_CR_FWCORE_DMI_DATA1                                                     (0x3428U)
+#define RGX_CR_FWCORE_DMI_DATA1_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED1, with 10 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED1_REPEATCOUNT                                     (10U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED10
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10                                                (0x3430U)
+#define RGX_CR_FWCORE_DMI_RESERVED10_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED11
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED11                                                (0x3438U)
+#define RGX_CR_FWCORE_DMI_RESERVED11_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED12
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED12                                                (0x3440U)
+#define RGX_CR_FWCORE_DMI_RESERVED12_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED13
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED13                                                (0x3448U)
+#define RGX_CR_FWCORE_DMI_RESERVED13_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED14
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED14                                                (0x3450U)
+#define RGX_CR_FWCORE_DMI_RESERVED14_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMCONTROL
+*/
+#define RGX_CR_FWCORE_DMI_DMCONTROL                                                 (0x3480U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_MASKFULL                                        (IMG_UINT64_C(0x00000000D0000003))
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_SHIFT                                   (31U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_CLRMSK                                  (0x7FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN                                      (0x80000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_SHIFT                                 (30U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_CLRMSK                                (0xBFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN                                    (0x40000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_SHIFT                              (28U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_EN                                 (0x10000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_SHIFT                                  (1U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_EN                                     (0x00000002U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMSTATUS
+*/
+#define RGX_CR_FWCORE_DMI_DMSTATUS                                                  (0x3488U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000004FFFFF))
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_SHIFT                                  (22U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_EN                                     (0x00400000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_SHIFT                               (19U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_EN                                  (0x00080000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_SHIFT                               (18U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_EN                                  (0x00040000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_SHIFT                               (17U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_CLRMSK                              (0xFFFDFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN                                  (0x00020000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_SHIFT                               (16U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_EN                                  (0x00010000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_SHIFT                             (15U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_EN                                (0x00008000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_SHIFT                             (14U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_CLRMSK                            (0xFFFFBFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_EN                                (0x00004000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_SHIFT                                 (13U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_CLRMSK                                (0xFFFFDFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_EN                                    (0x00002000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_SHIFT                                 (12U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_CLRMSK                                (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_EN                                    (0x00001000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_SHIFT                                 (11U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_CLRMSK                                (0xFFFFF7FFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_EN                                    (0x00000800U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_SHIFT                                 (10U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_CLRMSK                                (0xFFFFFBFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_EN                                    (0x00000400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_SHIFT                                  (9U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_CLRMSK                                 (0xFFFFFDFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN                                     (0x00000200U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_SHIFT                                  (8U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_EN                                     (0x00000100U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_SHIFT                              (7U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_CLRMSK                             (0xFFFFFF7FU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_EN                                 (0x00000080U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_SHIFT                                   (6U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_CLRMSK                                  (0xFFFFFFBFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_EN                                      (0x00000040U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_SHIFT                            (5U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_CLRMSK                           (0xFFFFFFDFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_EN                               (0x00000020U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_SHIFT                            (4U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_EN                               (0x00000010U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_SHIFT                                    (0U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_CLRMSK                                   (0xFFFFFFF0U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED2, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED2_REPEATCOUNT                                     (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED20
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20                                                (0x3490U)
+#define RGX_CR_FWCORE_DMI_RESERVED20_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED21
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED21                                                (0x3498U)
+#define RGX_CR_FWCORE_DMI_RESERVED21_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED22
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED22                                                (0x34A0U)
+#define RGX_CR_FWCORE_DMI_RESERVED22_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED23
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED23                                                (0x34A8U)
+#define RGX_CR_FWCORE_DMI_RESERVED23_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_ABSTRACTCS
+*/
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS                                                (0x34B0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_MASKFULL                                       (IMG_UINT64_C(0x000000001F00170F))
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_SHIFT                              (24U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_CLRMSK                             (0xE0FFFFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_SHIFT                                     (12U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_CLRMSK                                    (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN                                        (0x00001000U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT                                   (8U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK                                  (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_SHIFT                                (0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_CLRMSK                               (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_COMMAND
+*/
+#define RGX_CR_FWCORE_DMI_COMMAND                                                   (0x34B8U)
+#define RGX_CR_FWCORE_DMI_COMMAND_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT                                     (24U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_CLRMSK                                    (0x00FFFFFFU)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_CLRMSK                                    (0xFF000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED3, with 32 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED3_REPEATCOUNT                                     (32U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED30
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30                                                (0x34C0U)
+#define RGX_CR_FWCORE_DMI_RESERVED30_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED31
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED31                                                (0x34C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED31_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBCS
+*/
+#define RGX_CR_FWCORE_DMI_SBCS                                                      (0x35C0U)
+#define RGX_CR_FWCORE_DMI_SBCS_MASKFULL                                             (IMG_UINT64_C(0x00000000E07FFFFF))
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_SHIFT                                      (29U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_CLRMSK                                     (0x1FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_SHIFT                                    (22U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_CLRMSK                                   (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_EN                                       (0x00400000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_SHIFT                                         (21U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN                                            (0x00200000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_SHIFT                                   (20U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN                                      (0x00100000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT                                       (17U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_CLRMSK                                      (0xFFF1FFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_SHIFT                                (16U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_CLRMSK                               (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_EN                                   (0x00010000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_SHIFT                                   (15U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_CLRMSK                                  (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_EN                                      (0x00008000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT                                        (12U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK                                       (0xFFFF8FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_SHIFT                                        (5U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_CLRMSK                                       (0xFFFFF01FU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_SHIFT                                    (4U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_EN                                       (0x00000010U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_SHIFT                                     (3U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_EN                                        (0x00000008U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_SHIFT                                     (2U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_EN                                        (0x00000004U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_SHIFT                                     (1U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_EN                                        (0x00000002U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_SHIFT                                      (0U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBADDRESS0
+*/
+#define RGX_CR_FWCORE_DMI_SBADDRESS0                                                (0x35C8U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_SBDATA, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA_REPEATCOUNT                                        (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA0
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA0                                                   (0x35E0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA0_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA1
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA1                                                   (0x35E8U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA1_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA2
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA2                                                   (0x35F0U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA2_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA3
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA3                                                   (0x35F8U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA3_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_HALTSUM0
+*/
+#define RGX_CR_FWCORE_DMI_HALTSUM0                                                  (0x3600U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_SLC_CTRL_MISC
+*/
+#define RGX_CR_SLC_CTRL_MISC                                                        (0x3800U)
+#define RGX_CR_SLC_CTRL_MISC_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFF03FF010F))
+#define RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS_SHIFT                                    (32U)
+#define RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS_CLRMSK                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_SHIFT                             (25U)
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_EN                                (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_SHIFT                                  (24U)
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_EN                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SHIFT                                 (16U)
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_INTERLEAVED_64_BYTE                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_INTERLEAVED_128_BYTE                  (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SIMPLE_HASH1                          (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SIMPLE_HASH2                          (IMG_UINT64_C(0x0000000000110000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH1                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH2_SCRAMBLE                    (IMG_UINT64_C(0x0000000000210000))
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_SHIFT                                            (8U)
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_EN                                               (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_SHIFT                                    (3U)
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_EN                                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_SHIFT                            (2U)
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_EN                               (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_SHIFT                          (1U)
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_SHIFT                            (0U)
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_EN                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC_CTRL_FLUSH_INVAL
+*/
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL                                                 (0x3818U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_MASKFULL                                        (IMG_UINT64_C(0x0000000080000FFF))
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_SHIFT                                      (31U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_CLRMSK                                     (0x7FFFFFFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_EN                                         (0x80000000U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_SHIFT                             (11U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_CLRMSK                            (0xFFFFF7FFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_EN                                (0x00000800U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_SHIFT                             (10U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_CLRMSK                            (0xFFFFFBFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_EN                                (0x00000400U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_SHIFT                                    (9U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_EN                                       (0x00000200U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_SHIFT                                    (8U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_EN                                       (0x00000100U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_SHIFT                                    (7U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_CLRMSK                                   (0xFFFFFF7FU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_EN                                       (0x00000080U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_SHIFT                                    (6U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_CLRMSK                                   (0xFFFFFFBFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_EN                                       (0x00000040U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_SHIFT                              (5U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_EN                                 (0x00000020U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_SHIFT                                    (4U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_EN                                       (0x00000010U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_SHIFT                                (3U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_EN                                   (0x00000008U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_SHIFT                                  (2U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_EN                                     (0x00000004U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_SHIFT                                     (1U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_EN                                        (0x00000002U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_SHIFT                                       (0U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS0
+*/
+#define RGX_CR_SLC_STATUS0                                                          (0x3820U)
+#define RGX_CR_SLC_STATUS0_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_SHIFT                                (2U)
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_EN                                   (0x00000004U)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_SHIFT                                      (1U)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_EN                                         (0x00000002U)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_SHIFT                                      (0U)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_CTRL_BYPASS
+*/
+#define RGX_CR_SLC_CTRL_BYPASS                                                      (0x3828U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__MASKFULL                                    (IMG_UINT64_C(0x0FFFFFFFFFFF7FFF))
+#define RGX_CR_SLC_CTRL_BYPASS_MASKFULL                                             (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_SHIFT                     (59U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_CLRMSK                    (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_EN                        (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_SHIFT            (58U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_CLRMSK           (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_EN               (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_SHIFT            (57U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_CLRMSK           (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_EN               (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_SHIFT              (56U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_CLRMSK             (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_EN                 (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_SHIFT              (55U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_CLRMSK             (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_EN                 (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_SHIFT                     (54U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_CLRMSK                    (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_EN                        (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_SHIFT                    (53U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_CLRMSK                   (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_EN                       (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_SHIFT                      (52U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_CLRMSK                     (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_EN                         (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_SHIFT                        (51U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_CLRMSK                       (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_EN                           (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_SHIFT                               (50U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_CLRMSK                              (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_EN                                  (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_SHIFT                          (49U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_CLRMSK                         (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_EN                             (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_SHIFT                          (48U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_CLRMSK                         (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_EN                             (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_SHIFT                          (47U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_CLRMSK                         (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_EN                             (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_SHIFT                          (46U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_CLRMSK                         (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_EN                             (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_SHIFT                               (45U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_CLRMSK                              (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_EN                                  (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_SHIFT                               (44U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_CLRMSK                              (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_EN                                  (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_SHIFT                               (43U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_CLRMSK                              (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_EN                                  (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_SHIFT                                (42U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_CLRMSK                               (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_EN                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_SHIFT                               (41U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_EN                                  (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_SHIFT                               (40U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_EN                                  (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_SHIFT                    (39U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_CLRMSK                   (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_EN                       (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_SHIFT                           (38U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_CLRMSK                          (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_EN                              (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_SHIFT                      (37U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_CLRMSK                     (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_EN                         (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_SHIFT                               (36U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_EN                                  (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_SHIFT                     (35U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_CLRMSK                    (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_EN                        (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_SHIFT                     (34U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_EN                        (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_SHIFT                            (33U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_EN                               (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_SHIFT                            (32U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_EN                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_SHIFT                         (31U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_EN                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_SHIFT                           (30U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_EN                              (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_SHIFT                              (29U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_EN                                 (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_SHIFT                             (28U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_EN                                (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_SHIFT                                  (27U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_EN                                     (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_SHIFT                                         (26U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_EN                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_SHIFT                                    (25U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_EN                                       (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_SHIFT                                        (24U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_EN                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_SHIFT                                       (23U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_EN                                          (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_SHIFT                                        (22U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_EN                                           (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_SHIFT                                       (21U)
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_EN                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_SHIFT                                         (20U)
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_EN                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_SHIFT                                        (19U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_EN                                           (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_SHIFT                                        (18U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_EN                                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_SHIFT                                        (17U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_EN                                           (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_SHIFT                                     (16U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_EN                                        (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_SHIFT                                    (15U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN                                       (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_SHIFT                                        (14U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_EN                                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_SHIFT                                       (13U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_EN                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_SHIFT                                       (12U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_EN                                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_SHIFT                                     (11U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_EN                                        (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_SHIFT                                     (10U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_EN                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_SHIFT                                     (9U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_EN                                        (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_SHIFT                                         (8U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_EN                                            (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_SHIFT                                         (7U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_EN                                            (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_SHIFT                                         (6U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_EN                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_SHIFT                                   (5U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_EN                                      (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_SHIFT                                         (4U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_EN                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_SHIFT                                     (3U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_EN                                        (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_SHIFT                                       (2U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_EN                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_SHIFT                                          (1U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_EN                                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_SHIFT                                            (0U)
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_EN                                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC_STATUS1
+*/
+#define RGX_CR_SLC_STATUS1                                                          (0x3870U)
+#define RGX_CR_SLC_STATUS1_MASKFULL                                                 (IMG_UINT64_C(0x800003FF03FFFFFF))
+#define RGX_CR_SLC_STATUS1_PAUSED_SHIFT                                             (63U)
+#define RGX_CR_SLC_STATUS1_PAUSED_CLRMSK                                            (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_PAUSED_EN                                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_SLC_STATUS1_READS1_SHIFT                                             (32U)
+#define RGX_CR_SLC_STATUS1_READS1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_CR_SLC_STATUS1_READS0_SHIFT                                             (16U)
+#define RGX_CR_SLC_STATUS1_READS0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_CR_SLC_STATUS1_READS1_EXT_SHIFT                                         (8U)
+#define RGX_CR_SLC_STATUS1_READS1_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_SLC_STATUS1_READS0_EXT_SHIFT                                         (0U)
+#define RGX_CR_SLC_STATUS1_READS0_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_SLC_IDLE
+*/
+#define RGX_CR_SLC_IDLE                                                             (0x3898U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MASKFULL                                           (IMG_UINT64_C(0x00000000000003FF))
+#define RGX_CR_SLC_IDLE_MASKFULL                                                    (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_SHIFT                                   (9U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_CLRMSK                                  (0xFFFFFDFFU)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_EN                                      (0x00000200U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_SHIFT                                   (8U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_CLRMSK                                  (0xFFFFFEFFU)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_EN                                      (0x00000100U)
+#define RGX_CR_SLC_IDLE_IMGBV4_SHIFT                                                (7U)
+#define RGX_CR_SLC_IDLE_IMGBV4_CLRMSK                                               (0xFFFFFF7FU)
+#define RGX_CR_SLC_IDLE_IMGBV4_EN                                                   (0x00000080U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_SHIFT                                           (6U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_EN                                              (0x00000040U)
+#define RGX_CR_SLC_IDLE_RBOFIFO_SHIFT                                               (5U)
+#define RGX_CR_SLC_IDLE_RBOFIFO_CLRMSK                                              (0xFFFFFFDFU)
+#define RGX_CR_SLC_IDLE_RBOFIFO_EN                                                  (0x00000020U)
+#define RGX_CR_SLC_IDLE_FRC_CONV_SHIFT                                              (4U)
+#define RGX_CR_SLC_IDLE_FRC_CONV_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_SLC_IDLE_FRC_CONV_EN                                                 (0x00000010U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_SHIFT                                              (3U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_CLRMSK                                             (0xFFFFFFF7U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_EN                                                 (0x00000008U)
+#define RGX_CR_SLC_IDLE_VXD_CONV_SHIFT                                              (2U)
+#define RGX_CR_SLC_IDLE_VXD_CONV_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_SLC_IDLE_VXD_CONV_EN                                                 (0x00000004U)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_SHIFT                                             (1U)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_CLRMSK                                            (0xFFFFFFFDU)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_EN                                                (0x00000002U)
+#define RGX_CR_SLC_IDLE_CBAR_SHIFT                                                  (0U)
+#define RGX_CR_SLC_IDLE_CBAR_CLRMSK                                                 (0xFFFFFFFEU)
+#define RGX_CR_SLC_IDLE_CBAR_EN                                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS2
+*/
+#define RGX_CR_SLC_STATUS2                                                          (0x3908U)
+#define RGX_CR_SLC_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x000003FF03FFFFFF))
+#define RGX_CR_SLC_STATUS2_READS3_SHIFT                                             (32U)
+#define RGX_CR_SLC_STATUS2_READS3_CLRMSK                                            (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_CR_SLC_STATUS2_READS2_SHIFT                                             (16U)
+#define RGX_CR_SLC_STATUS2_READS2_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_CR_SLC_STATUS2_READS3_EXT_SHIFT                                         (8U)
+#define RGX_CR_SLC_STATUS2_READS3_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_SLC_STATUS2_READS2_EXT_SHIFT                                         (0U)
+#define RGX_CR_SLC_STATUS2_READS2_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_SLC_CTRL_MISC2
+*/
+#define RGX_CR_SLC_CTRL_MISC2                                                       (0x3930U)
+#define RGX_CR_SLC_CTRL_MISC2_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC2_SCRAMBLE_BITS_SHIFT                                   (0U)
+#define RGX_CR_SLC_CTRL_MISC2_SCRAMBLE_BITS_CLRMSK                                  (0x00000000U)
+
+
+/*
+    Register RGX_CR_SLC_CROSSBAR_LOAD_BALANCE
+*/
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE                                            (0x3938U)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_MASKFULL                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_SHIFT                               (0U)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_SIZE_IN_KB
+*/
+#define RGX_CR_SLC_SIZE_IN_KB                                                       (0x3970U)
+#define RGX_CR_SLC_SIZE_IN_KB_MASKFULL                                              (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_SLC_SIZE_IN_KB_SIZE_SHIFT                                            (0U)
+#define RGX_CR_SLC_SIZE_IN_KB_SIZE_CLRMSK                                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_USC_TIMER
+*/
+#define RGX_CR_USC_TIMER                                                            (0x46C8U)
+#define RGX_CR_USC_TIMER_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_TIMER_CNT_SHIFT                                                  (0U)
+#define RGX_CR_USC_TIMER_CNT_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_TIMER_CNT
+*/
+#define RGX_CR_USC_TIMER_CNT                                                        (0x46D0U)
+#define RGX_CR_USC_TIMER_CNT_MASKFULL                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_USC_TIMER_CNT_RESET_SHIFT                                            (0U)
+#define RGX_CR_USC_TIMER_CNT_RESET_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_USC_TIMER_CNT_RESET_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_USC_UVS0_CHECKSUM
+*/
+#define RGX_CR_USC_UVS0_CHECKSUM                                                    (0x5000U)
+#define RGX_CR_USC_UVS0_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS0_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS0_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS1_CHECKSUM
+*/
+#define RGX_CR_USC_UVS1_CHECKSUM                                                    (0x5008U)
+#define RGX_CR_USC_UVS1_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS1_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS1_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS2_CHECKSUM
+*/
+#define RGX_CR_USC_UVS2_CHECKSUM                                                    (0x5010U)
+#define RGX_CR_USC_UVS2_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS2_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS2_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS3_CHECKSUM
+*/
+#define RGX_CR_USC_UVS3_CHECKSUM                                                    (0x5018U)
+#define RGX_CR_USC_UVS3_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS3_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS3_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_SIGNATURE
+*/
+#define RGX_CR_PPP_SIGNATURE                                                        (0x5020U)
+#define RGX_CR_PPP_SIGNATURE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_SIGNATURE_VALUE_SHIFT                                            (0U)
+#define RGX_CR_PPP_SIGNATURE_VALUE_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_SIGNATURE
+*/
+#define RGX_CR_TE_SIGNATURE                                                         (0x5028U)
+#define RGX_CR_TE_SIGNATURE_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_SIGNATURE_VALUE_SHIFT                                             (0U)
+#define RGX_CR_TE_SIGNATURE_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_CHECKSUM
+*/
+#define RGX_CR_TE_CHECKSUM                                                          (0x5110U)
+#define RGX_CR_TE_CHECKSUM_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_CHECKSUM_VALUE_SHIFT                                              (0U)
+#define RGX_CR_TE_CHECKSUM_VALUE_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVB_CHECKSUM
+*/
+#define RGX_CR_USC_UVB_CHECKSUM                                                     (0x5118U)
+#define RGX_CR_USC_UVB_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_VCE_CHECKSUM
+*/
+#define RGX_CR_VCE_CHECKSUM                                                         (0x5030U)
+#define RGX_CR_VCE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VCE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_VCE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_PDS_CHECKSUM
+*/
+#define RGX_CR_ISP_PDS_CHECKSUM                                                     (0x5038U)
+#define RGX_CR_ISP_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_TPF_CHECKSUM
+*/
+#define RGX_CR_ISP_TPF_CHECKSUM                                                     (0x5040U)
+#define RGX_CR_ISP_TPF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_PLANE0_CHECKSUM
+*/
+#define RGX_CR_TFPU_PLANE0_CHECKSUM                                                 (0x5048U)
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_VALUE_SHIFT                                     (0U)
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_VALUE_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_PLANE1_CHECKSUM
+*/
+#define RGX_CR_TFPU_PLANE1_CHECKSUM                                                 (0x5050U)
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_VALUE_SHIFT                                     (0U)
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_VALUE_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM
+*/
+#define RGX_CR_PBE_CHECKSUM                                                         (0x5058U)
+#define RGX_CR_PBE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_PBE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_PDS_DOUTM_STM_SIGNATURE
+*/
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE                                              (0x5060U)
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_VALUE_SHIFT                                  (0U)
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_VALUE_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_IFPU_ISP_CHECKSUM
+*/
+#define RGX_CR_IFPU_ISP_CHECKSUM                                                    (0x5068U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS4_CHECKSUM
+*/
+#define RGX_CR_USC_UVS4_CHECKSUM                                                    (0x5100U)
+#define RGX_CR_USC_UVS4_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS4_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS4_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS5_CHECKSUM
+*/
+#define RGX_CR_USC_UVS5_CHECKSUM                                                    (0x5108U)
+#define RGX_CR_USC_UVS5_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS5_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS5_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_CLIP_CHECKSUM
+*/
+#define RGX_CR_PPP_CLIP_CHECKSUM                                                    (0x5120U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_PHASE
+*/
+#define RGX_CR_PERF_TA_PHASE                                                        (0x6008U)
+#define RGX_CR_PERF_TA_PHASE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_PHASE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_TA_PHASE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_PHASE
+*/
+#define RGX_CR_PERF_3D_PHASE                                                        (0x6010U)
+#define RGX_CR_PERF_3D_PHASE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_PHASE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_3D_PHASE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_COMPUTE_PHASE
+*/
+#define RGX_CR_PERF_COMPUTE_PHASE                                                   (0x6018U)
+#define RGX_CR_PERF_COMPUTE_PHASE_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_COMPUTE_PHASE_COUNT_SHIFT                                       (0U)
+#define RGX_CR_PERF_COMPUTE_PHASE_COUNT_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_CYCLE
+*/
+#define RGX_CR_PERF_TA_CYCLE                                                        (0x6020U)
+#define RGX_CR_PERF_TA_CYCLE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_CYCLE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_TA_CYCLE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_CYCLE
+*/
+#define RGX_CR_PERF_3D_CYCLE                                                        (0x6028U)
+#define RGX_CR_PERF_3D_CYCLE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_CYCLE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_3D_CYCLE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_COMPUTE_CYCLE
+*/
+#define RGX_CR_PERF_COMPUTE_CYCLE                                                   (0x6030U)
+#define RGX_CR_PERF_COMPUTE_CYCLE_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_COMPUTE_CYCLE_COUNT_SHIFT                                       (0U)
+#define RGX_CR_PERF_COMPUTE_CYCLE_COUNT_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_OR_3D_CYCLE
+*/
+#define RGX_CR_PERF_TA_OR_3D_CYCLE                                                  (0x6038U)
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_COUNT_SHIFT                                      (0U)
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_COUNT_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_INITIAL_TA_CYCLE
+*/
+#define RGX_CR_PERF_INITIAL_TA_CYCLE                                                (0x6040U)
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_READ_STALL
+*/
+#define RGX_CR_PERF_SLC0_READ_STALL                                                 (0x60B8U)
+#define RGX_CR_PERF_SLC0_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC0_WRITE_STALL                                                (0x60C0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_READ_STALL
+*/
+#define RGX_CR_PERF_SLC1_READ_STALL                                                 (0x60E0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC1_WRITE_STALL                                                (0x60E8U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_READ_STALL
+*/
+#define RGX_CR_PERF_SLC2_READ_STALL                                                 (0x6158U)
+#define RGX_CR_PERF_SLC2_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC2_WRITE_STALL                                                (0x6160U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_READ_STALL
+*/
+#define RGX_CR_PERF_SLC3_READ_STALL                                                 (0x6180U)
+#define RGX_CR_PERF_SLC3_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC3_WRITE_STALL                                                (0x6188U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_SPINUP
+*/
+#define RGX_CR_PERF_3D_SPINUP                                                       (0x6220U)
+#define RGX_CR_PERF_3D_SPINUP_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_SPINUP_CYCLES_SHIFT                                          (0U)
+#define RGX_CR_PERF_3D_SPINUP_CYCLES_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_AXI_ACE_LITE_CONFIGURATION
+*/
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION                                           (0x38C0U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_MASKFULL                                  (IMG_UINT64_C(0x00003FFFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT                    (45U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_CLRMSK                   (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_EN                       (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_OSID_SECURITY_SHIFT                       (37U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_OSID_SECURITY_CLRMSK                      (IMG_UINT64_C(0xFFFFE01FFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT    (36U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_CLRMSK   (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_EN       (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_SHIFT              (35U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_CLRMSK             (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_EN                 (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_SHIFT               (34U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_CLRMSK              (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_EN                  (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT           (30U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_CLRMSK          (IMG_UINT64_C(0xFFFFFFFC3FFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT                    (26U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFC3FFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT                    (22U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFC3FFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_BARRIER_SHIFT                    (20U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_BARRIER_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_BARRIER_SHIFT                    (18U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_BARRIER_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT          (16U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_CLRMSK         (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT                   (14U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT                   (12U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT               (10U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT               (8U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_NON_SNOOPING_SHIFT                (4U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_NON_SNOOPING_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFF0F))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_NON_SNOOPING_SHIFT                (0U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_NON_SNOOPING_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF0))
+
+
+/*
+    Register RGX_CR_POWER_ESTIMATE_RESULT
+*/
+#define RGX_CR_POWER_ESTIMATE_RESULT                                                (0x6328U)
+#define RGX_CR_POWER_ESTIMATE_RESULT_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_POWER_ESTIMATE_RESULT_VALUE_SHIFT                                    (0U)
+#define RGX_CR_POWER_ESTIMATE_RESULT_VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF
+*/
+#define RGX_CR_TA_PERF                                                              (0x7600U)
+#define RGX_CR_TA_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TA_PERF_CLR_3_SHIFT                                                  (4U)
+#define RGX_CR_TA_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define RGX_CR_TA_PERF_CLR_3_EN                                                     (0x00000010U)
+#define RGX_CR_TA_PERF_CLR_2_SHIFT                                                  (3U)
+#define RGX_CR_TA_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_TA_PERF_CLR_2_EN                                                     (0x00000008U)
+#define RGX_CR_TA_PERF_CLR_1_SHIFT                                                  (2U)
+#define RGX_CR_TA_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define RGX_CR_TA_PERF_CLR_1_EN                                                     (0x00000004U)
+#define RGX_CR_TA_PERF_CLR_0_SHIFT                                                  (1U)
+#define RGX_CR_TA_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_TA_PERF_CLR_0_EN                                                     (0x00000002U)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT0
+*/
+#define RGX_CR_TA_PERF_SELECT0                                                      (0x7608U)
+#define RGX_CR_TA_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT1
+*/
+#define RGX_CR_TA_PERF_SELECT1                                                      (0x7610U)
+#define RGX_CR_TA_PERF_SELECT1_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT1_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT1_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT1_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT1_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT1_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT2
+*/
+#define RGX_CR_TA_PERF_SELECT2                                                      (0x7618U)
+#define RGX_CR_TA_PERF_SELECT2_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT2_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT2_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT2_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT2_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT2_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT3
+*/
+#define RGX_CR_TA_PERF_SELECT3                                                      (0x7620U)
+#define RGX_CR_TA_PERF_SELECT3_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT3_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT3_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT3_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT3_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT3_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECTED_BITS
+*/
+#define RGX_CR_TA_PERF_SELECTED_BITS                                                (0x7648U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG3_SHIFT                                     (48U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG3_CLRMSK                                    (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG2_SHIFT                                     (32U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG2_CLRMSK                                    (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG1_SHIFT                                     (16U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG1_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG0_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG0_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_0
+*/
+#define RGX_CR_TA_PERF_COUNTER_0                                                    (0x7650U)
+#define RGX_CR_TA_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_1
+*/
+#define RGX_CR_TA_PERF_COUNTER_1                                                    (0x7658U)
+#define RGX_CR_TA_PERF_COUNTER_1_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_1_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_1_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_2
+*/
+#define RGX_CR_TA_PERF_COUNTER_2                                                    (0x7660U)
+#define RGX_CR_TA_PERF_COUNTER_2_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_2_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_2_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_3
+*/
+#define RGX_CR_TA_PERF_COUNTER_3                                                    (0x7668U)
+#define RGX_CR_TA_PERF_COUNTER_3_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_3_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_3_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF
+*/
+#define RGX_CR_RASTERISATION_PERF                                                   (0x7700U)
+#define RGX_CR_RASTERISATION_PERF_MASKFULL                                          (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_RASTERISATION_PERF_CLR_3_SHIFT                                       (4U)
+#define RGX_CR_RASTERISATION_PERF_CLR_3_CLRMSK                                      (0xFFFFFFEFU)
+#define RGX_CR_RASTERISATION_PERF_CLR_3_EN                                          (0x00000010U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_SHIFT                                       (3U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_CLRMSK                                      (0xFFFFFFF7U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_EN                                          (0x00000008U)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_SHIFT                                       (2U)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_CLRMSK                                      (0xFFFFFFFBU)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_EN                                          (0x00000004U)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_SHIFT                                       (1U)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_EN                                          (0x00000002U)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_SHIFT                                 (0U)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_SELECT0
+*/
+#define RGX_CR_RASTERISATION_PERF_SELECT0                                           (0x7708U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MASKFULL                                  (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MAX_SHIFT                           (48U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MAX_CLRMSK                          (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MIN_SHIFT                           (32U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MIN_CLRMSK                          (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_SHIFT                                (21U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_GROUP_SELECT_SHIFT                        (16U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_GROUP_SELECT_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BIT_SELECT_SHIFT                          (0U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BIT_SELECT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_COUNTER_0
+*/
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0                                         (0x7750U)
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_REG_SHIFT                               (0U)
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_REG_CLRMSK                              (0x00000000U)
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF                                                  (0x7800U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_MASKFULL                                         (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_SHIFT                                      (4U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_CLRMSK                                     (0xFFFFFFEFU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_EN                                         (0x00000010U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_SHIFT                                      (3U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_EN                                         (0x00000008U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_SHIFT                                      (2U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_CLRMSK                                     (0xFFFFFFFBU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_EN                                         (0x00000004U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_SHIFT                                      (1U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_EN                                         (0x00000002U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_SHIFT                                (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0                                          (0x7808U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MASKFULL                                 (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MAX_SHIFT                          (48U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MAX_CLRMSK                         (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MIN_SHIFT                          (32U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MIN_CLRMSK                         (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_SHIFT                               (21U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_EN                                  (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_GROUP_SELECT_SHIFT                       (16U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_GROUP_SELECT_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BIT_SELECT_SHIFT                         (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BIT_SELECT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0                                        (0x7850U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_REG_SHIFT                              (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_REG_CLRMSK                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF
+*/
+#define RGX_CR_TPU_MCU_L0_PERF                                                      (0x7900U)
+#define RGX_CR_TPU_MCU_L0_PERF_MASKFULL                                             (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_SHIFT                                          (4U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_EN                                             (0x00000010U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_SHIFT                                          (3U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_EN                                             (0x00000008U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_SHIFT                                          (2U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_EN                                             (0x00000004U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_SHIFT                                          (1U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_EN                                             (0x00000002U)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_SHIFT                                    (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_SELECT0
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0                                              (0x7908U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MASKFULL                                     (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MAX_SHIFT                              (48U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MAX_CLRMSK                             (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MIN_SHIFT                              (32U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MIN_CLRMSK                             (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_SHIFT                                   (21U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_EN                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_GROUP_SELECT_SHIFT                           (16U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_GROUP_SELECT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BIT_SELECT_SHIFT                             (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BIT_SELECT_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_COUNTER_0
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0                                            (0x7950U)
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_REG_SHIFT                                  (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_REG_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_PERF
+*/
+#define RGX_CR_USC_PERF                                                             (0x8100U)
+#define RGX_CR_USC_PERF_MASKFULL                                                    (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_USC_PERF_CLR_3_SHIFT                                                 (4U)
+#define RGX_CR_USC_PERF_CLR_3_CLRMSK                                                (0xFFFFFFEFU)
+#define RGX_CR_USC_PERF_CLR_3_EN                                                    (0x00000010U)
+#define RGX_CR_USC_PERF_CLR_2_SHIFT                                                 (3U)
+#define RGX_CR_USC_PERF_CLR_2_CLRMSK                                                (0xFFFFFFF7U)
+#define RGX_CR_USC_PERF_CLR_2_EN                                                    (0x00000008U)
+#define RGX_CR_USC_PERF_CLR_1_SHIFT                                                 (2U)
+#define RGX_CR_USC_PERF_CLR_1_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_USC_PERF_CLR_1_EN                                                    (0x00000004U)
+#define RGX_CR_USC_PERF_CLR_0_SHIFT                                                 (1U)
+#define RGX_CR_USC_PERF_CLR_0_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_USC_PERF_CLR_0_EN                                                    (0x00000002U)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_SHIFT                                           (0U)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_USC_PERF_SELECT0
+*/
+#define RGX_CR_USC_PERF_SELECT0                                                     (0x8108U)
+#define RGX_CR_USC_PERF_SELECT0_MASKFULL                                            (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MAX_SHIFT                                     (48U)
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MAX_CLRMSK                                    (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MIN_SHIFT                                     (32U)
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MIN_CLRMSK                                    (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_MODE_SHIFT                                          (21U)
+#define RGX_CR_USC_PERF_SELECT0_MODE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_MODE_EN                                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_USC_PERF_SELECT0_GROUP_SELECT_SHIFT                                  (16U)
+#define RGX_CR_USC_PERF_SELECT0_GROUP_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_USC_PERF_SELECT0_BIT_SELECT_SHIFT                                    (0U)
+#define RGX_CR_USC_PERF_SELECT0_BIT_SELECT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_USC_PERF_COUNTER_0
+*/
+#define RGX_CR_USC_PERF_COUNTER_0                                                   (0x8150U)
+#define RGX_CR_USC_PERF_COUNTER_0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_PERF_COUNTER_0_REG_SHIFT                                         (0U)
+#define RGX_CR_USC_PERF_COUNTER_0_REG_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_JONES_IDLE
+*/
+#define RGX_CR_JONES_IDLE                                                           (0x8328U)
+#define RGX_CR_JONES_IDLE_MASKFULL                                                  (IMG_UINT64_C(0x0000000000007FFF))
+#define RGX_CR_JONES_IDLE_TDM_SHIFT                                                 (14U)
+#define RGX_CR_JONES_IDLE_TDM_CLRMSK                                                (0xFFFFBFFFU)
+#define RGX_CR_JONES_IDLE_TDM_EN                                                    (0x00004000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_SHIFT                                          (13U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_EN                                             (0x00002000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_SHIFT                                              (12U)
+#define RGX_CR_JONES_IDLE_FB_CDC_CLRMSK                                             (0xFFFFEFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_EN                                                 (0x00001000U)
+#define RGX_CR_JONES_IDLE_MMU_SHIFT                                                 (11U)
+#define RGX_CR_JONES_IDLE_MMU_CLRMSK                                                (0xFFFFF7FFU)
+#define RGX_CR_JONES_IDLE_MMU_EN                                                    (0x00000800U)
+#define RGX_CR_JONES_IDLE_TLA_SHIFT                                                 (10U)
+#define RGX_CR_JONES_IDLE_TLA_CLRMSK                                                (0xFFFFFBFFU)
+#define RGX_CR_JONES_IDLE_TLA_EN                                                    (0x00000400U)
+#define RGX_CR_JONES_IDLE_GARTEN_SHIFT                                              (9U)
+#define RGX_CR_JONES_IDLE_GARTEN_CLRMSK                                             (0xFFFFFDFFU)
+#define RGX_CR_JONES_IDLE_GARTEN_EN                                                 (0x00000200U)
+#define RGX_CR_JONES_IDLE_HOSTIF_SHIFT                                              (8U)
+#define RGX_CR_JONES_IDLE_HOSTIF_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_JONES_IDLE_HOSTIF_EN                                                 (0x00000100U)
+#define RGX_CR_JONES_IDLE_SOCIF_SHIFT                                               (7U)
+#define RGX_CR_JONES_IDLE_SOCIF_CLRMSK                                              (0xFFFFFF7FU)
+#define RGX_CR_JONES_IDLE_SOCIF_EN                                                  (0x00000080U)
+#define RGX_CR_JONES_IDLE_TILING_SHIFT                                              (6U)
+#define RGX_CR_JONES_IDLE_TILING_CLRMSK                                             (0xFFFFFFBFU)
+#define RGX_CR_JONES_IDLE_TILING_EN                                                 (0x00000040U)
+#define RGX_CR_JONES_IDLE_IPP_SHIFT                                                 (5U)
+#define RGX_CR_JONES_IDLE_IPP_CLRMSK                                                (0xFFFFFFDFU)
+#define RGX_CR_JONES_IDLE_IPP_EN                                                    (0x00000020U)
+#define RGX_CR_JONES_IDLE_USCS_SHIFT                                                (4U)
+#define RGX_CR_JONES_IDLE_USCS_CLRMSK                                               (0xFFFFFFEFU)
+#define RGX_CR_JONES_IDLE_USCS_EN                                                   (0x00000010U)
+#define RGX_CR_JONES_IDLE_PM_SHIFT                                                  (3U)
+#define RGX_CR_JONES_IDLE_PM_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_JONES_IDLE_PM_EN                                                     (0x00000008U)
+#define RGX_CR_JONES_IDLE_CDM_SHIFT                                                 (2U)
+#define RGX_CR_JONES_IDLE_CDM_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_JONES_IDLE_CDM_EN                                                    (0x00000004U)
+#define RGX_CR_JONES_IDLE_VDM_SHIFT                                                 (1U)
+#define RGX_CR_JONES_IDLE_VDM_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_JONES_IDLE_VDM_EN                                                    (0x00000002U)
+#define RGX_CR_JONES_IDLE_BIF_SHIFT                                                 (0U)
+#define RGX_CR_JONES_IDLE_BIF_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_JONES_IDLE_BIF_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_TORNADO_PERF
+*/
+#define RGX_CR_TORNADO_PERF                                                         (0x8228U)
+#define RGX_CR_TORNADO_PERF_MASKFULL                                                (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TORNADO_PERF_CLR_3_SHIFT                                             (4U)
+#define RGX_CR_TORNADO_PERF_CLR_3_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_TORNADO_PERF_CLR_3_EN                                                (0x00000010U)
+#define RGX_CR_TORNADO_PERF_CLR_2_SHIFT                                             (3U)
+#define RGX_CR_TORNADO_PERF_CLR_2_CLRMSK                                            (0xFFFFFFF7U)
+#define RGX_CR_TORNADO_PERF_CLR_2_EN                                                (0x00000008U)
+#define RGX_CR_TORNADO_PERF_CLR_1_SHIFT                                             (2U)
+#define RGX_CR_TORNADO_PERF_CLR_1_CLRMSK                                            (0xFFFFFFFBU)
+#define RGX_CR_TORNADO_PERF_CLR_1_EN                                                (0x00000004U)
+#define RGX_CR_TORNADO_PERF_CLR_0_SHIFT                                             (1U)
+#define RGX_CR_TORNADO_PERF_CLR_0_CLRMSK                                            (0xFFFFFFFDU)
+#define RGX_CR_TORNADO_PERF_CLR_0_EN                                                (0x00000002U)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_SHIFT                                       (0U)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_TORNADO_PERF_SELECT0
+*/
+#define RGX_CR_TORNADO_PERF_SELECT0                                                 (0x8230U)
+#define RGX_CR_TORNADO_PERF_SELECT0_MASKFULL                                        (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MAX_SHIFT                                 (48U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MAX_CLRMSK                                (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MIN_SHIFT                                 (32U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MIN_CLRMSK                                (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_SHIFT                                      (21U)
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_EN                                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TORNADO_PERF_SELECT0_GROUP_SELECT_SHIFT                              (16U)
+#define RGX_CR_TORNADO_PERF_SELECT0_GROUP_SELECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BIT_SELECT_SHIFT                                (0U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BIT_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TORNADO_PERF_COUNTER_0
+*/
+#define RGX_CR_TORNADO_PERF_COUNTER_0                                               (0x8268U)
+#define RGX_CR_TORNADO_PERF_COUNTER_0_MASKFULL                                      (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TORNADO_PERF_COUNTER_0_REG_SHIFT                                     (0U)
+#define RGX_CR_TORNADO_PERF_COUNTER_0_REG_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF
+*/
+#define RGX_CR_TEXAS_PERF                                                           (0x8290U)
+#define RGX_CR_TEXAS_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_TEXAS_PERF_CLR_5_SHIFT                                               (6U)
+#define RGX_CR_TEXAS_PERF_CLR_5_CLRMSK                                              (0xFFFFFFBFU)
+#define RGX_CR_TEXAS_PERF_CLR_5_EN                                                  (0x00000040U)
+#define RGX_CR_TEXAS_PERF_CLR_4_SHIFT                                               (5U)
+#define RGX_CR_TEXAS_PERF_CLR_4_CLRMSK                                              (0xFFFFFFDFU)
+#define RGX_CR_TEXAS_PERF_CLR_4_EN                                                  (0x00000020U)
+#define RGX_CR_TEXAS_PERF_CLR_3_SHIFT                                               (4U)
+#define RGX_CR_TEXAS_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_PERF_CLR_3_EN                                                  (0x00000010U)
+#define RGX_CR_TEXAS_PERF_CLR_2_SHIFT                                               (3U)
+#define RGX_CR_TEXAS_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define RGX_CR_TEXAS_PERF_CLR_2_EN                                                  (0x00000008U)
+#define RGX_CR_TEXAS_PERF_CLR_1_SHIFT                                               (2U)
+#define RGX_CR_TEXAS_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_PERF_CLR_1_EN                                                  (0x00000004U)
+#define RGX_CR_TEXAS_PERF_CLR_0_SHIFT                                               (1U)
+#define RGX_CR_TEXAS_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_TEXAS_PERF_CLR_0_EN                                                  (0x00000002U)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_SELECT0
+*/
+#define RGX_CR_TEXAS_PERF_SELECT0                                                   (0x8298U)
+#define RGX_CR_TEXAS_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF803FFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_SHIFT                                        (31U)
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_TEXAS_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define RGX_CR_TEXAS_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFC0FFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_COUNTER_0
+*/
+#define RGX_CR_TEXAS_PERF_COUNTER_0                                                 (0x82D8U)
+#define RGX_CR_TEXAS_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TEXAS_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define RGX_CR_TEXAS_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_JONES_PERF
+*/
+#define RGX_CR_JONES_PERF                                                           (0x8330U)
+#define RGX_CR_JONES_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_JONES_PERF_CLR_3_SHIFT                                               (4U)
+#define RGX_CR_JONES_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define RGX_CR_JONES_PERF_CLR_3_EN                                                  (0x00000010U)
+#define RGX_CR_JONES_PERF_CLR_2_SHIFT                                               (3U)
+#define RGX_CR_JONES_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define RGX_CR_JONES_PERF_CLR_2_EN                                                  (0x00000008U)
+#define RGX_CR_JONES_PERF_CLR_1_SHIFT                                               (2U)
+#define RGX_CR_JONES_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define RGX_CR_JONES_PERF_CLR_1_EN                                                  (0x00000004U)
+#define RGX_CR_JONES_PERF_CLR_0_SHIFT                                               (1U)
+#define RGX_CR_JONES_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_JONES_PERF_CLR_0_EN                                                  (0x00000002U)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_PERF_SELECT0
+*/
+#define RGX_CR_JONES_PERF_SELECT0                                                   (0x8338U)
+#define RGX_CR_JONES_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_MODE_SHIFT                                        (21U)
+#define RGX_CR_JONES_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_JONES_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define RGX_CR_JONES_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define RGX_CR_JONES_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_JONES_PERF_COUNTER_0
+*/
+#define RGX_CR_JONES_PERF_COUNTER_0                                                 (0x8368U)
+#define RGX_CR_JONES_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_JONES_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define RGX_CR_JONES_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF
+*/
+#define RGX_CR_BLACKPEARL_PERF                                                      (0x8400U)
+#define RGX_CR_BLACKPEARL_PERF_MASKFULL                                             (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_SHIFT                                          (6U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_CLRMSK                                         (0xFFFFFFBFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_EN                                             (0x00000040U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_SHIFT                                          (5U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_EN                                             (0x00000020U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_SHIFT                                          (4U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_EN                                             (0x00000010U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_SHIFT                                          (3U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_EN                                             (0x00000008U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_SHIFT                                          (2U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_EN                                             (0x00000004U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_SHIFT                                          (1U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_EN                                             (0x00000002U)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_SHIFT                                    (0U)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_SELECT0
+*/
+#define RGX_CR_BLACKPEARL_PERF_SELECT0                                              (0x8408U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MASKFULL                                     (IMG_UINT64_C(0x3FFF3FFF803FFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MAX_SHIFT                              (48U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MAX_CLRMSK                             (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MIN_SHIFT                              (32U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MIN_CLRMSK                             (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_SHIFT                                   (31U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_EN                                      (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_GROUP_SELECT_SHIFT                           (16U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_GROUP_SELECT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFC0FFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BIT_SELECT_SHIFT                             (0U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BIT_SELECT_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_COUNTER_0
+*/
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0                                            (0x8448U)
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_REG_SHIFT                                  (0U)
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_REG_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_PERF
+*/
+#define RGX_CR_PBE_PERF                                                             (0x8478U)
+#define RGX_CR_PBE_PERF_MASKFULL                                                    (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_PBE_PERF_CLR_3_SHIFT                                                 (4U)
+#define RGX_CR_PBE_PERF_CLR_3_CLRMSK                                                (0xFFFFFFEFU)
+#define RGX_CR_PBE_PERF_CLR_3_EN                                                    (0x00000010U)
+#define RGX_CR_PBE_PERF_CLR_2_SHIFT                                                 (3U)
+#define RGX_CR_PBE_PERF_CLR_2_CLRMSK                                                (0xFFFFFFF7U)
+#define RGX_CR_PBE_PERF_CLR_2_EN                                                    (0x00000008U)
+#define RGX_CR_PBE_PERF_CLR_1_SHIFT                                                 (2U)
+#define RGX_CR_PBE_PERF_CLR_1_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_PBE_PERF_CLR_1_EN                                                    (0x00000004U)
+#define RGX_CR_PBE_PERF_CLR_0_SHIFT                                                 (1U)
+#define RGX_CR_PBE_PERF_CLR_0_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_PBE_PERF_CLR_0_EN                                                    (0x00000002U)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_SHIFT                                           (0U)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_PBE_PERF_SELECT0
+*/
+#define RGX_CR_PBE_PERF_SELECT0                                                     (0x8480U)
+#define RGX_CR_PBE_PERF_SELECT0_MASKFULL                                            (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MAX_SHIFT                                     (48U)
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MAX_CLRMSK                                    (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MIN_SHIFT                                     (32U)
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MIN_CLRMSK                                    (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_MODE_SHIFT                                          (21U)
+#define RGX_CR_PBE_PERF_SELECT0_MODE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_MODE_EN                                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_PBE_PERF_SELECT0_GROUP_SELECT_SHIFT                                  (16U)
+#define RGX_CR_PBE_PERF_SELECT0_GROUP_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BIT_SELECT_SHIFT                                    (0U)
+#define RGX_CR_PBE_PERF_SELECT0_BIT_SELECT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_PBE_PERF_COUNTER_0
+*/
+#define RGX_CR_PBE_PERF_COUNTER_0                                                   (0x84B0U)
+#define RGX_CR_PBE_PERF_COUNTER_0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_PERF_COUNTER_0_REG_SHIFT                                         (0U)
+#define RGX_CR_PBE_PERF_COUNTER_0_REG_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_OCP_REVINFO
+*/
+#define RGX_CR_OCP_REVINFO                                                          (0x9000U)
+#define RGX_CR_OCP_REVINFO_MASKFULL                                                 (IMG_UINT64_C(0x00000007FFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_SYSBUS_SHIFT                                      (33U)
+#define RGX_CR_OCP_REVINFO_HWINFO_SYSBUS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF9FFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_SHIFT                                      (32U)
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_EN                                         (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_OCP_REVINFO_REVISION_SHIFT                                           (0U)
+#define RGX_CR_OCP_REVINFO_REVISION_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_OCP_SYSCONFIG
+*/
+#define RGX_CR_OCP_SYSCONFIG                                                        (0x9010U)
+#define RGX_CR_OCP_SYSCONFIG_MASKFULL                                               (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_CR_OCP_SYSCONFIG_DUST2_STANDBY_MODE_SHIFT                               (10U)
+#define RGX_CR_OCP_SYSCONFIG_DUST2_STANDBY_MODE_CLRMSK                              (0xFFFFF3FFU)
+#define RGX_CR_OCP_SYSCONFIG_DUST1_STANDBY_MODE_SHIFT                               (8U)
+#define RGX_CR_OCP_SYSCONFIG_DUST1_STANDBY_MODE_CLRMSK                              (0xFFFFFCFFU)
+#define RGX_CR_OCP_SYSCONFIG_DUST0_STANDBY_MODE_SHIFT                               (6U)
+#define RGX_CR_OCP_SYSCONFIG_DUST0_STANDBY_MODE_CLRMSK                              (0xFFFFFF3FU)
+#define RGX_CR_OCP_SYSCONFIG_RASCAL_STANDBYMODE_SHIFT                               (4U)
+#define RGX_CR_OCP_SYSCONFIG_RASCAL_STANDBYMODE_CLRMSK                              (0xFFFFFFCFU)
+#define RGX_CR_OCP_SYSCONFIG_STANDBY_MODE_SHIFT                                     (2U)
+#define RGX_CR_OCP_SYSCONFIG_STANDBY_MODE_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_OCP_SYSCONFIG_IDLE_MODE_SHIFT                                        (0U)
+#define RGX_CR_OCP_SYSCONFIG_IDLE_MODE_CLRMSK                                       (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_0
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_0                                                  (0x9020U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_SHIFT                        (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_1
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_1                                                  (0x9028U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_SHIFT                      (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_CLRMSK                     (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_EN                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_2
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_2                                                  (0x9030U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_SHIFT                                (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_0
+*/
+#define RGX_CR_OCP_IRQSTATUS_0                                                      (0x9038U)
+#define RGX_CR_OCP_IRQSTATUS_0_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_SHIFT                         (0U)
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_EN                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_1
+*/
+#define RGX_CR_OCP_IRQSTATUS_1                                                      (0x9040U)
+#define RGX_CR_OCP_IRQSTATUS_1_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_SHIFT                       (0U)
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_CLRMSK                      (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_EN                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_2
+*/
+#define RGX_CR_OCP_IRQSTATUS_2                                                      (0x9048U)
+#define RGX_CR_OCP_IRQSTATUS_2_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_SHIFT                                 (0U)
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_0
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_0                                                  (0x9050U)
+#define RGX_CR_OCP_IRQENABLE_SET_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_SHIFT                     (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_CLRMSK                    (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_EN                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_1
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_1                                                  (0x9058U)
+#define RGX_CR_OCP_IRQENABLE_SET_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_SHIFT                   (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_2
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_2                                                  (0x9060U)
+#define RGX_CR_OCP_IRQENABLE_SET_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_SHIFT                             (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_0
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_0                                                  (0x9068U)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_SHIFT                    (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_CLRMSK                   (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_EN                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_1
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_1                                                  (0x9070U)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_SHIFT                  (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_CLRMSK                 (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_EN                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_2
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_2                                                  (0x9078U)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_SHIFT                            (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQ_EVENT
+*/
+#define RGX_CR_OCP_IRQ_EVENT                                                        (0x9080U)
+#define RGX_CR_OCP_IRQ_EVENT_MASKFULL                                               (IMG_UINT64_C(0x00000000000FFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_SHIFT                    (19U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_EN                       (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_SHIFT                    (18U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_EN                       (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_SHIFT                    (17U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_EN                       (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_SHIFT                    (16U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_EN                       (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (15U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_SHIFT                         (14U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_SHIFT                          (13U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_SHIFT                        (12U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (11U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_SHIFT                         (10U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_SHIFT                          (9U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_SHIFT                        (8U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (7U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_SHIFT                         (6U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_SHIFT                          (5U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_SHIFT                        (4U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (3U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_SHIFT                         (2U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_SHIFT                          (1U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_SHIFT                        (0U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_OCP_DEBUG_CONFIG
+*/
+#define RGX_CR_OCP_DEBUG_CONFIG                                                     (0x9088U)
+#define RGX_CR_OCP_DEBUG_CONFIG_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_SHIFT                                           (0U)
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_DEBUG_STATUS
+*/
+#define RGX_CR_OCP_DEBUG_STATUS                                                     (0x9090U)
+#define RGX_CR_OCP_DEBUG_STATUS_MASKFULL                                            (IMG_UINT64_C(0x001F1F77FFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SDISCACK_SHIFT                              (51U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SDISCACK_CLRMSK                             (IMG_UINT64_C(0xFFE7FFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_SHIFT                              (50U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_EN                                 (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_MCONNECT_SHIFT                              (48U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_MCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SDISCACK_SHIFT                              (43U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SDISCACK_CLRMSK                             (IMG_UINT64_C(0xFFFFE7FFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_SHIFT                              (42U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_EN                                 (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_MCONNECT_SHIFT                              (40U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_MCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_SHIFT                                  (38U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_EN                                     (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_SHIFT                         (37U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_EN                            (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_SHIFT                           (36U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_CLRMSK                          (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_EN                              (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_SHIFT                                  (34U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_EN                                     (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_SHIFT                         (33U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_EN                            (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_SHIFT                           (32U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_EN                              (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_SHIFT                                (31U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_EN                                   (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_SHIFT                                   (30U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_EN                                      (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_SHIFT                                (29U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCACK_SHIFT                                (27U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFE7FFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_SHIFT                                (26U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MCONNECT_SHIFT                                (24U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_SHIFT                                (23U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_SHIFT                                   (22U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_SHIFT                                (21U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCACK_SHIFT                                (19U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE7FFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_SHIFT                                (18U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MCONNECT_SHIFT                                (16U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_SHIFT                                (15U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_SHIFT                                   (14U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_SHIFT                                (13U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCACK_SHIFT                                (11U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFE7FF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_SHIFT                                (10U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MCONNECT_SHIFT                                (8U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_SHIFT                                (7U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_SHIFT                                   (6U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_SHIFT                                (5U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCACK_SHIFT                                (3U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFE7))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_SHIFT                                (2U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MCONNECT_SHIFT                                (0U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_SHIFT                                     (6U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_CLRMSK                                    (0xFFFFFFBFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_EN                                        (0x00000040U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_SHIFT                                         (5U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_EN                                            (0x00000020U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_SHIFT                                         (4U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_EN                                            (0x00000010U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_SHIFT                                       (3U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_CLRMSK                                      (0xFFFFFFF7U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_EN                                          (0x00000008U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_SHIFT                                        (2U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_CLRMSK                                       (0xFFFFFFFBU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_EN                                           (0x00000004U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_SHIFT                                       (1U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_EN                                          (0x00000002U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_SHIFT                                          (0U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_EN                                             (0x00000001U)
+
+
+#define RGX_CR_BIF_TRUST_DM_MASK                                                    (0x0000007FU)
+
+
+/*
+    Register RGX_CR_BIF_TRUST
+*/
+#define RGX_CR_BIF_TRUST                                                            (0xA000U)
+#define RGX_CR_BIF_TRUST_MASKFULL                                                   (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_SHIFT                          (20U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_CLRMSK                         (0xFFEFFFFFU)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_EN                             (0x00100000U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_SHIFT                            (19U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_CLRMSK                           (0xFFF7FFFFU)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_EN                               (0x00080000U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_SHIFT                                 (18U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_CLRMSK                                (0xFFFBFFFFU)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_EN                                    (0x00040000U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_SHIFT                                   (17U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_CLRMSK                                  (0xFFFDFFFFU)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_EN                                      (0x00020000U)
+#define RGX_CR_BIF_TRUST_ENABLE_SHIFT                                               (16U)
+#define RGX_CR_BIF_TRUST_ENABLE_CLRMSK                                              (0xFFFEFFFFU)
+#define RGX_CR_BIF_TRUST_ENABLE_EN                                                  (0x00010000U)
+#define RGX_CR_BIF_TRUST_DM_TRUSTED_SHIFT                                           (9U)
+#define RGX_CR_BIF_TRUST_DM_TRUSTED_CLRMSK                                          (0xFFFF01FFU)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_SHIFT                             (8U)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_CLRMSK                            (0xFFFFFEFFU)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_EN                                (0x00000100U)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_SHIFT                               (7U)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_CLRMSK                              (0xFFFFFF7FU)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_EN                                  (0x00000080U)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_SHIFT                               (6U)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_CLRMSK                              (0xFFFFFFBFU)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_EN                                  (0x00000040U)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_CLRMSK                              (0xFFFFFFDFU)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_EN                                  (0x00000020U)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_SHIFT                                 (4U)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_EN                                    (0x00000010U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_SHIFT                                 (3U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_EN                                    (0x00000008U)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_SHIFT                              (2U)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_EN                                 (0x00000004U)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_SHIFT                                (1U)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_EN                                   (0x00000002U)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_SHIFT                                (0U)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_SYS_BUS_SECURE
+*/
+#define RGX_CR_SYS_BUS_SECURE                                                       (0xA100U)
+#define RGX_CR_SYS_BUS_SECURE__SECR__MASKFULL                                       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_MASKFULL                                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_SHIFT                                          (0U)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_FBA_FC0_CHECKSUM
+*/
+#define RGX_CR_FBA_FC0_CHECKSUM                                                     (0xD170U)
+#define RGX_CR_FBA_FC0_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC0_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC0_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC1_CHECKSUM
+*/
+#define RGX_CR_FBA_FC1_CHECKSUM                                                     (0xD178U)
+#define RGX_CR_FBA_FC1_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC1_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC1_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC2_CHECKSUM
+*/
+#define RGX_CR_FBA_FC2_CHECKSUM                                                     (0xD180U)
+#define RGX_CR_FBA_FC2_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC2_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC2_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC3_CHECKSUM
+*/
+#define RGX_CR_FBA_FC3_CHECKSUM                                                     (0xD188U)
+#define RGX_CR_FBA_FC3_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC3_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC3_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_CLK_CTRL2
+*/
+#define RGX_CR_CLK_CTRL2                                                            (0xD200U)
+#define RGX_CR_CLK_CTRL2_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000F33))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_SHIFT                                             (10U)
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_ON                                                (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_AUTO                                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL2_VRDM_SHIFT                                                 (8U)
+#define RGX_CR_CLK_CTRL2_VRDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL2_VRDM_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_VRDM_ON                                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL2_VRDM_AUTO                                                  (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL2_SH_SHIFT                                                   (4U)
+#define RGX_CR_CLK_CTRL2_SH_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL2_SH_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_SH_ON                                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL2_SH_AUTO                                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL2_FBA_SHIFT                                                  (0U)
+#define RGX_CR_CLK_CTRL2_FBA_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL2_FBA_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_FBA_ON                                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL2_FBA_AUTO                                                   (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS2
+*/
+#define RGX_CR_CLK_STATUS2                                                          (0xD208U)
+#define RGX_CR_CLK_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000015))
+#define RGX_CR_CLK_STATUS2_VRDM_SHIFT                                               (4U)
+#define RGX_CR_CLK_STATUS2_VRDM_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS2_VRDM_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_VRDM_RUNNING                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS2_SH_SHIFT                                                 (2U)
+#define RGX_CR_CLK_STATUS2_SH_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS2_SH_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_SH_RUNNING                                               (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS2_FBA_SHIFT                                                (0U)
+#define RGX_CR_CLK_STATUS2_FBA_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS2_FBA_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_FBA_RUNNING                                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL
+*/
+#define RGX_CR_RPM_SHF_FPL                                                          (0xD520U)
+#define RGX_CR_RPM_SHF_FPL_MASKFULL                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFC))
+#define RGX_CR_RPM_SHF_FPL_SIZE_SHIFT                                               (40U)
+#define RGX_CR_RPM_SHF_FPL_SIZE_CLRMSK                                              (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_CR_RPM_SHF_FPL_BASE_SHIFT                                               (2U)
+#define RGX_CR_RPM_SHF_FPL_BASE_CLRMSK                                              (IMG_UINT64_C(0xFFFFFF0000000003))
+#define RGX_CR_RPM_SHF_FPL_BASE_ALIGNSHIFT                                          (2U)
+#define RGX_CR_RPM_SHF_FPL_BASE_ALIGNSIZE                                           (4U)
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL_READ
+*/
+#define RGX_CR_RPM_SHF_FPL_READ                                                     (0xD528U)
+#define RGX_CR_RPM_SHF_FPL_READ_MASKFULL                                            (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_SHIFT                                        (22U)
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_CLRMSK                                       (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_EN                                           (0x00400000U)
+#define RGX_CR_RPM_SHF_FPL_READ_OFFSET_SHIFT                                        (0U)
+#define RGX_CR_RPM_SHF_FPL_READ_OFFSET_CLRMSK                                       (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL_WRITE
+*/
+#define RGX_CR_RPM_SHF_FPL_WRITE                                                    (0xD530U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_MASKFULL                                           (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_SHIFT                                       (22U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_EN                                          (0x00400000U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_OFFSET_SHIFT                                       (0U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_OFFSET_CLRMSK                                      (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL
+*/
+#define RGX_CR_RPM_SHG_FPL                                                          (0xD538U)
+#define RGX_CR_RPM_SHG_FPL_MASKFULL                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFC))
+#define RGX_CR_RPM_SHG_FPL_SIZE_SHIFT                                               (40U)
+#define RGX_CR_RPM_SHG_FPL_SIZE_CLRMSK                                              (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_CR_RPM_SHG_FPL_BASE_SHIFT                                               (2U)
+#define RGX_CR_RPM_SHG_FPL_BASE_CLRMSK                                              (IMG_UINT64_C(0xFFFFFF0000000003))
+#define RGX_CR_RPM_SHG_FPL_BASE_ALIGNSHIFT                                          (2U)
+#define RGX_CR_RPM_SHG_FPL_BASE_ALIGNSIZE                                           (4U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL_READ
+*/
+#define RGX_CR_RPM_SHG_FPL_READ                                                     (0xD540U)
+#define RGX_CR_RPM_SHG_FPL_READ_MASKFULL                                            (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_SHIFT                                        (22U)
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_CLRMSK                                       (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_EN                                           (0x00400000U)
+#define RGX_CR_RPM_SHG_FPL_READ_OFFSET_SHIFT                                        (0U)
+#define RGX_CR_RPM_SHG_FPL_READ_OFFSET_CLRMSK                                       (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL_WRITE
+*/
+#define RGX_CR_RPM_SHG_FPL_WRITE                                                    (0xD548U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_MASKFULL                                           (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_SHIFT                                       (22U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_EN                                          (0x00400000U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_OFFSET_SHIFT                                       (0U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_OFFSET_CLRMSK                                      (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_SH_PERF
+*/
+#define RGX_CR_SH_PERF                                                              (0xD5F8U)
+#define RGX_CR_SH_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_SH_PERF_CLR_3_SHIFT                                                  (4U)
+#define RGX_CR_SH_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define RGX_CR_SH_PERF_CLR_3_EN                                                     (0x00000010U)
+#define RGX_CR_SH_PERF_CLR_2_SHIFT                                                  (3U)
+#define RGX_CR_SH_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_SH_PERF_CLR_2_EN                                                     (0x00000008U)
+#define RGX_CR_SH_PERF_CLR_1_SHIFT                                                  (2U)
+#define RGX_CR_SH_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define RGX_CR_SH_PERF_CLR_1_EN                                                     (0x00000004U)
+#define RGX_CR_SH_PERF_CLR_0_SHIFT                                                  (1U)
+#define RGX_CR_SH_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_SH_PERF_CLR_0_EN                                                     (0x00000002U)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_SH_PERF_SELECT0
+*/
+#define RGX_CR_SH_PERF_SELECT0                                                      (0xD600U)
+#define RGX_CR_SH_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define RGX_CR_SH_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SH_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_SH_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_SH_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_SH_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SH_PERF_COUNTER_0
+*/
+#define RGX_CR_SH_PERF_COUNTER_0                                                    (0xD628U)
+#define RGX_CR_SH_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SH_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define RGX_CR_SH_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_SHG_CHECKSUM
+*/
+#define RGX_CR_SHF_SHG_CHECKSUM                                                     (0xD1C0U)
+#define RGX_CR_SHF_SHG_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_SHG_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SHF_SHG_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_VERTEX_BIF_CHECKSUM
+*/
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM                                              (0xD1C8U)
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_VALUE_SHIFT                                  (0U)
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_VALUE_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_VARY_BIF_CHECKSUM
+*/
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM                                                (0xD1D0U)
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_VALUE_SHIFT                                    (0U)
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_RPM_BIF_CHECKSUM
+*/
+#define RGX_CR_RPM_BIF_CHECKSUM                                                     (0xD1D8U)
+#define RGX_CR_RPM_BIF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_RPM_BIF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_RPM_BIF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHG_BIF_CHECKSUM
+*/
+#define RGX_CR_SHG_BIF_CHECKSUM                                                     (0xD1E0U)
+#define RGX_CR_SHG_BIF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHG_BIF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SHG_BIF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHG_FE_BE_CHECKSUM
+*/
+#define RGX_CR_SHG_FE_BE_CHECKSUM                                                   (0xD1E8U)
+#define RGX_CR_SHG_FE_BE_CHECKSUM_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHG_FE_BE_CHECKSUM_VALUE_SHIFT                                       (0U)
+#define RGX_CR_SHG_FE_BE_CHECKSUM_VALUE_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register DPX_CR_BF_PERF
+*/
+#define DPX_CR_BF_PERF                                                              (0xC458U)
+#define DPX_CR_BF_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BF_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_BF_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_BF_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_BF_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_BF_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_BF_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_BF_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_BF_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_BF_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_BF_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_BF_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_BF_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_BF_PERF_SELECT0
+*/
+#define DPX_CR_BF_PERF_SELECT0                                                      (0xC460U)
+#define DPX_CR_BF_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_BF_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BF_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_BF_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BF_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_BF_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BF_PERF_COUNTER_0
+*/
+#define DPX_CR_BF_PERF_COUNTER_0                                                    (0xC488U)
+#define DPX_CR_BF_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BF_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_BF_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_BT_PERF
+*/
+#define DPX_CR_BT_PERF                                                              (0xC3D0U)
+#define DPX_CR_BT_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BT_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_BT_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_BT_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_BT_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_BT_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_BT_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_BT_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_BT_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_BT_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_BT_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_BT_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_BT_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_BT_PERF_SELECT0
+*/
+#define DPX_CR_BT_PERF_SELECT0                                                      (0xC3D8U)
+#define DPX_CR_BT_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_BT_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BT_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_BT_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BT_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_BT_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BT_PERF_COUNTER_0
+*/
+#define DPX_CR_BT_PERF_COUNTER_0                                                    (0xC420U)
+#define DPX_CR_BT_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BT_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_BT_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_RQ_USC_DEBUG
+*/
+#define DPX_CR_RQ_USC_DEBUG                                                         (0xC110U)
+#define DPX_CR_RQ_USC_DEBUG_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RQ_USC_DEBUG_CHECKSUM_SHIFT                                          (0U)
+#define DPX_CR_RQ_USC_DEBUG_CHECKSUM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register DPX_CR_BIF_FAULT_BANK_MMU_STATUS
+*/
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS                                            (0xC5C8U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_MASKFULL                                   (IMG_UINT64_C(0x000000000000F775))
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_CAT_BASE_SHIFT                             (12U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_CAT_BASE_CLRMSK                            (0xFFFF0FFFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_PAGE_SIZE_SHIFT                            (8U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_PAGE_SIZE_CLRMSK                           (0xFFFFF8FFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_DATA_TYPE_SHIFT                            (5U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_DATA_TYPE_CLRMSK                           (0xFFFFFF9FU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_SHIFT                             (4U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_CLRMSK                            (0xFFFFFFEFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_EN                                (0x00000010U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_SHIFT                     (2U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                    (0xFFFFFFFBU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_EN                        (0x00000004U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_SHIFT                                (0U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_CLRMSK                               (0xFFFFFFFEU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_EN                                   (0x00000001U)
+
+
+/*
+    Register DPX_CR_BIF_FAULT_BANK_REQ_STATUS
+*/
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS                                            (0xC5D0U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_MASKFULL                                   (IMG_UINT64_C(0x03FFFFFFFFFFFFF0))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_SHIFT                                  (57U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_CLRMSK                                 (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_EN                                     (IMG_UINT64_C(0x0200000000000000))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_SB_SHIFT                               (44U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_SB_CLRMSK                              (IMG_UINT64_C(0xFE000FFFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_ID_SHIFT                               (40U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_ID_CLRMSK                              (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_SHIFT                              (4U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_CLRMSK                             (IMG_UINT64_C(0xFFFFFF000000000F))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_ALIGNSHIFT                         (4U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_ALIGNSIZE                          (16U)
+
+
+/*
+    Register DPX_CR_BIF_MMU_STATUS
+*/
+#define DPX_CR_BIF_MMU_STATUS                                                       (0xC5D8U)
+#define DPX_CR_BIF_MMU_STATUS_MASKFULL                                              (IMG_UINT64_C(0x000000000FFFFFF7))
+#define DPX_CR_BIF_MMU_STATUS_PC_DATA_SHIFT                                         (20U)
+#define DPX_CR_BIF_MMU_STATUS_PC_DATA_CLRMSK                                        (0xF00FFFFFU)
+#define DPX_CR_BIF_MMU_STATUS_PD_DATA_SHIFT                                         (12U)
+#define DPX_CR_BIF_MMU_STATUS_PD_DATA_CLRMSK                                        (0xFFF00FFFU)
+#define DPX_CR_BIF_MMU_STATUS_PT_DATA_SHIFT                                         (4U)
+#define DPX_CR_BIF_MMU_STATUS_PT_DATA_CLRMSK                                        (0xFFFFF00FU)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_SHIFT                                         (2U)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_CLRMSK                                        (0xFFFFFFFBU)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_EN                                            (0x00000004U)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_SHIFT                                          (1U)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_CLRMSK                                         (0xFFFFFFFDU)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_EN                                             (0x00000002U)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_SHIFT                                            (0U)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_RT_PERF
+*/
+#define DPX_CR_RT_PERF                                                              (0xC700U)
+#define DPX_CR_RT_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_RT_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_RT_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_RT_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_RT_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_RT_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_RT_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_RT_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_RT_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_RT_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_RT_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_RT_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_RT_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_RT_PERF_SELECT0
+*/
+#define DPX_CR_RT_PERF_SELECT0                                                      (0xC708U)
+#define DPX_CR_RT_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_RT_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_RT_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_RT_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_RT_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_RT_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_RT_PERF_COUNTER_0
+*/
+#define DPX_CR_RT_PERF_COUNTER_0                                                    (0xC730U)
+#define DPX_CR_RT_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RT_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_RT_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_BX_TU_PERF
+*/
+#define DPX_CR_BX_TU_PERF                                                           (0xC908U)
+#define DPX_CR_BX_TU_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BX_TU_PERF_CLR_3_SHIFT                                               (4U)
+#define DPX_CR_BX_TU_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define DPX_CR_BX_TU_PERF_CLR_3_EN                                                  (0x00000010U)
+#define DPX_CR_BX_TU_PERF_CLR_2_SHIFT                                               (3U)
+#define DPX_CR_BX_TU_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define DPX_CR_BX_TU_PERF_CLR_2_EN                                                  (0x00000008U)
+#define DPX_CR_BX_TU_PERF_CLR_1_SHIFT                                               (2U)
+#define DPX_CR_BX_TU_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define DPX_CR_BX_TU_PERF_CLR_1_EN                                                  (0x00000004U)
+#define DPX_CR_BX_TU_PERF_CLR_0_SHIFT                                               (1U)
+#define DPX_CR_BX_TU_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define DPX_CR_BX_TU_PERF_CLR_0_EN                                                  (0x00000002U)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register DPX_CR_BX_TU_PERF_SELECT0
+*/
+#define DPX_CR_BX_TU_PERF_SELECT0                                                   (0xC910U)
+#define DPX_CR_BX_TU_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_SHIFT                                        (21U)
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BX_TU_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define DPX_CR_BX_TU_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BX_TU_PERF_COUNTER_0
+*/
+#define DPX_CR_BX_TU_PERF_COUNTER_0                                                 (0xC938U)
+#define DPX_CR_BX_TU_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BX_TU_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define DPX_CR_BX_TU_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register DPX_CR_RS_PDS_RR_CHECKSUM
+*/
+#define DPX_CR_RS_PDS_RR_CHECKSUM                                                   (0xC0F0U)
+#define DPX_CR_RS_PDS_RR_CHECKSUM_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RS_PDS_RR_CHECKSUM_VALUE_SHIFT                                       (0U)
+#define DPX_CR_RS_PDS_RR_CHECKSUM_VALUE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT                                            (0xE140U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_MASKFULL                                   (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_SHIFT                                   (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_CLRMSK                                  (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING                                                    (0xE148U)
+#define RGX_CR_MMU_CBASE_MAPPING_MASKFULL                                           (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT                                    (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK                                   (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT                               (12U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS
+*/
+#define RGX_CR_MMU_FAULT_STATUS                                                     (0xE150U)
+#define RGX_CR_MMU_FAULT_STATUS_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_ADDRESS_SHIFT                                       (28U)
+#define RGX_CR_MMU_FAULT_STATUS_ADDRESS_CLRMSK                                      (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_CONTEXT_SHIFT                                       (20U)
+#define RGX_CR_MMU_FAULT_STATUS_CONTEXT_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFF00FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_TAG_SB_SHIFT                                        (12U)
+#define RGX_CR_MMU_FAULT_STATUS_TAG_SB_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFF00FFF))
+#define RGX_CR_MMU_FAULT_STATUS_REQ_ID_SHIFT                                        (6U)
+#define RGX_CR_MMU_FAULT_STATUS_REQ_ID_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_CR_MMU_FAULT_STATUS_LEVEL_SHIFT                                         (4U)
+#define RGX_CR_MMU_FAULT_STATUS_LEVEL_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_MMU_FAULT_STATUS_RNW_SHIFT                                           (3U)
+#define RGX_CR_MMU_FAULT_STATUS_RNW_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_RNW_EN                                              (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT                                          (1U)
+#define RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_SHIFT                                         (0U)
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_EN                                            (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS_META
+*/
+#define RGX_CR_MMU_FAULT_STATUS_META                                                (0xE158U)
+#define RGX_CR_MMU_FAULT_STATUS_META_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT                                  (28U)
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK                                 (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT                                  (20U)
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF00FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_SHIFT                                   (12U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFF00FFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT                                   (6U)
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT                                    (4U)
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_SHIFT                                      (3U)
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_EN                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT                                     (1U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_SHIFT                                    (0U)
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN                                       (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC3_CTRL_MISC
+*/
+#define RGX_CR_SLC3_CTRL_MISC                                                       (0xE200U)
+#define RGX_CR_SLC3_CTRL_MISC_MASKFULL                                              (IMG_UINT64_C(0x0000000000000107))
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_SHIFT                                  (8U)
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_EN                                     (0x00000100U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_SHIFT                                (0U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_CLRMSK                               (0xFFFFFFF8U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_LINEAR                               (0x00000000U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_IN_PAGE_HASH                         (0x00000001U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_FIXED_PVR_HASH                       (0x00000002U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_SCRAMBLE_PVR_HASH                    (0x00000003U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_WEAVED_HASH                          (0x00000004U)
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE
+*/
+#define RGX_CR_SLC3_SCRAMBLE                                                        (0xE208U)
+#define RGX_CR_SLC3_SCRAMBLE_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE_BITS_SHIFT                                             (0U)
+#define RGX_CR_SLC3_SCRAMBLE_BITS_CLRMSK                                            (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE2
+*/
+#define RGX_CR_SLC3_SCRAMBLE2                                                       (0xE210U)
+#define RGX_CR_SLC3_SCRAMBLE2_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE2_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE2_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE3
+*/
+#define RGX_CR_SLC3_SCRAMBLE3                                                       (0xE218U)
+#define RGX_CR_SLC3_SCRAMBLE3_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE3_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE3_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE4
+*/
+#define RGX_CR_SLC3_SCRAMBLE4                                                       (0xE260U)
+#define RGX_CR_SLC3_SCRAMBLE4_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE4_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE4_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_STATUS
+*/
+#define RGX_CR_SLC3_STATUS                                                          (0xE220U)
+#define RGX_CR_SLC3_STATUS_MASKFULL                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_STATUS_WRITES1_SHIFT                                            (48U)
+#define RGX_CR_SLC3_STATUS_WRITES1_CLRMSK                                           (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_SLC3_STATUS_WRITES0_SHIFT                                            (32U)
+#define RGX_CR_SLC3_STATUS_WRITES0_CLRMSK                                           (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_SLC3_STATUS_READS1_SHIFT                                             (16U)
+#define RGX_CR_SLC3_STATUS_READS1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_SLC3_STATUS_READS0_SHIFT                                             (0U)
+#define RGX_CR_SLC3_STATUS_READS0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SLC3_IDLE
+*/
+#define RGX_CR_SLC3_IDLE                                                            (0xE228U)
+#define RGX_CR_SLC3_IDLE_MASKFULL                                                   (IMG_UINT64_C(0x00000000000FFFFF))
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST2_SHIFT                                         (18U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST2_CLRMSK                                        (0xFFF3FFFFU)
+#define RGX_CR_SLC3_IDLE_MMU_SHIFT                                                  (17U)
+#define RGX_CR_SLC3_IDLE_MMU_CLRMSK                                                 (0xFFFDFFFFU)
+#define RGX_CR_SLC3_IDLE_MMU_EN                                                     (0x00020000U)
+#define RGX_CR_SLC3_IDLE_RDI_SHIFT                                                  (16U)
+#define RGX_CR_SLC3_IDLE_RDI_CLRMSK                                                 (0xFFFEFFFFU)
+#define RGX_CR_SLC3_IDLE_RDI_EN                                                     (0x00010000U)
+#define RGX_CR_SLC3_IDLE_IMGBV4_SHIFT                                               (12U)
+#define RGX_CR_SLC3_IDLE_IMGBV4_CLRMSK                                              (0xFFFF0FFFU)
+#define RGX_CR_SLC3_IDLE_CACHE_BANKS_SHIFT                                          (4U)
+#define RGX_CR_SLC3_IDLE_CACHE_BANKS_CLRMSK                                         (0xFFFFF00FU)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST_SHIFT                                          (2U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST_CLRMSK                                         (0xFFFFFFF3U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_SHIFT                                         (1U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_EN                                            (0x00000002U)
+#define RGX_CR_SLC3_IDLE_XBAR_SHIFT                                                 (0U)
+#define RGX_CR_SLC3_IDLE_XBAR_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_SLC3_IDLE_XBAR_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC3_FAULT_STOP_STATUS
+*/
+#define RGX_CR_SLC3_FAULT_STOP_STATUS                                               (0xE248U)
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_MASKFULL                                      (IMG_UINT64_C(0x0000000000001FFF))
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_BIF_SHIFT                                     (0U)
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_BIF_CLRMSK                                    (0xFFFFE000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_MODE
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_MODE                                               (0xF048U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MASKFULL                                      (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_SHIFT                                    (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_CLRMSK                                   (0xFFFFFFFCU)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_INDEX                                    (0x00000000U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_INSTANCE                                 (0x00000001U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_LIST                                     (0x00000002U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING0
+*/
+#define RGX_CR_CONTEXT_MAPPING0                                                     (0xF078U)
+#define RGX_CR_CONTEXT_MAPPING0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING0_2D_SHIFT                                            (24U)
+#define RGX_CR_CONTEXT_MAPPING0_2D_CLRMSK                                           (0x00FFFFFFU)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_SHIFT                                           (16U)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_CLRMSK                                          (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING0_3D_SHIFT                                            (8U)
+#define RGX_CR_CONTEXT_MAPPING0_3D_CLRMSK                                           (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING0_TA_SHIFT                                            (0U)
+#define RGX_CR_CONTEXT_MAPPING0_TA_CLRMSK                                           (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING1
+*/
+#define RGX_CR_CONTEXT_MAPPING1                                                     (0xF080U)
+#define RGX_CR_CONTEXT_MAPPING1_MASKFULL                                            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_CONTEXT_MAPPING1_HOST_SHIFT                                          (8U)
+#define RGX_CR_CONTEXT_MAPPING1_HOST_CLRMSK                                         (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING1_TLA_SHIFT                                           (0U)
+#define RGX_CR_CONTEXT_MAPPING1_TLA_CLRMSK                                          (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING2
+*/
+#define RGX_CR_CONTEXT_MAPPING2                                                     (0xF088U)
+#define RGX_CR_CONTEXT_MAPPING2_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING3
+*/
+#define RGX_CR_CONTEXT_MAPPING3                                                     (0xF090U)
+#define RGX_CR_CONTEXT_MAPPING3_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_JONES_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ                                           (0xF098U)
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_COUNTER_SHIFT                             (0U)
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_COUNTER_CLRMSK                            (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ                                      (0xF0A0U)
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_MASKFULL                             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_COUNTER_SHIFT                        (0U)
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_COUNTER_CLRMSK                       (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_DUST_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ                                            (0xF0A8U)
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_MASKFULL                                   (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_COUNTER_SHIFT                              (0U)
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_COUNTER_CLRMSK                             (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_JONES_FIX
+*/
+#define RGX_CR_JONES_FIX                                                            (0xF0C0U)
+#define RGX_CR_JONES_FIX__ROGUE3__MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX_MASKFULL                                                   (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX_DISABLE_SHIFT                                              (0U)
+#define RGX_CR_JONES_FIX_DISABLE_CLRMSK                                             (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING4
+*/
+#define RGX_CR_CONTEXT_MAPPING4                                                     (0xF210U)
+#define RGX_CR_CONTEXT_MAPPING4_MASKFULL                                            (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_SHIFT                                  (40U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_SHIFT                                    (32U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_SHIFT                                     (24U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_SHIFT                                  (16U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_SHIFT                                    (8U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_SHIFT                                     (0U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MULTICORE_GPU
+*/
+#define RGX_CR_MULTICORE_GPU                                                        (0xF300U)
+#define RGX_CR_MULTICORE_GPU_MASKFULL                                               (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_SHIFT                              (6U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_EN                                 (0x00000040U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_SHIFT                              (5U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_EN                                 (0x00000020U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_SHIFT                               (4U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_CLRMSK                              (0xFFFFFFEFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_EN                                  (0x00000010U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_SHIFT                               (3U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_CLRMSK                              (0xFFFFFFF7U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN                                  (0x00000008U)
+#define RGX_CR_MULTICORE_GPU_ID_SHIFT                                               (0U)
+#define RGX_CR_MULTICORE_GPU_ID_CLRMSK                                              (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_MULTICORE_SYSTEM
+*/
+#define RGX_CR_MULTICORE_SYSTEM                                                     (0xF308U)
+#define RGX_CR_MULTICORE_SYSTEM_MASKFULL                                            (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT                                     (0U)
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_CLRMSK                                    (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON                                       (0xF310U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT          (8U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK         (0xC00000FFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON                                       (0xF320U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT          (8U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK         (0xC00000FFU)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON                                        (0xF330U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                    (30U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                   (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT           (8U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK          (0xC00000FFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_ERR_INJ
+*/
+#define RGX_CR_ECC_RAM_ERR_INJ                                                      (0xF340U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MASKFULL                                             (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_SHIFT                                   (4U)
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_CLRMSK                                  (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_EN                                      (0x00000010U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_SHIFT                                            (3U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_EN                                               (0x00000008U)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_SHIFT                                     (2U)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_EN                                        (0x00000004U)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_SHIFT                                         (1U)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_EN                                            (0x00000002U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_SHIFT                                           (0U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_INIT_KICK
+*/
+#define RGX_CR_ECC_RAM_INIT_KICK                                                    (0xF348U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MASKFULL                                           (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_SHIFT                                 (4U)
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_EN                                    (0x00000010U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_SHIFT                                          (3U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_EN                                             (0x00000008U)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_SHIFT                                   (2U)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_EN                                      (0x00000004U)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_SHIFT                                       (1U)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_EN                                          (0x00000002U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_SHIFT                                         (0U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_INIT_DONE
+*/
+#define RGX_CR_ECC_RAM_INIT_DONE                                                    (0xF350U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MASKFULL                                           (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_SHIFT                                 (4U)
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_EN                                    (0x00000010U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_SHIFT                                          (3U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_EN                                             (0x00000008U)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_SHIFT                                   (2U)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_EN                                      (0x00000004U)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_SHIFT                                       (1U)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_EN                                          (0x00000002U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_SHIFT                                         (0U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_ENABLE
+*/
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE                                         (0xF390U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL                               (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_SHIFT                       (7U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_CLRMSK                      (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_EN                          (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_SHIFT                   (6U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_EN                      (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                (5U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK               (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_EN                   (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                 (4U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_EN                    (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_SHIFT                         (3U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_CLRMSK                        (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_EN                            (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_SHIFT                         (2U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_EN                            (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_SHIFT                        (1U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_EN                           (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_SHIFT                   (0U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_STATUS
+*/
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE                                         (0xF398U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__MASKFULL                               (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_SHIFT                       (7U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_CLRMSK                      (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_EN                          (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_SHIFT                   (6U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_EN                      (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                (5U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK               (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_EN                   (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                 (4U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_EN                    (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_SHIFT                         (3U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_CLRMSK                        (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_EN                            (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_SHIFT                         (2U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_EN                            (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_SHIFT                        (1U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_EN                           (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_SHIFT                   (0U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_CLEAR
+*/
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE                                          (0xF3A0U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__MASKFULL                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_SHIFT                        (7U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_EN                           (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_SHIFT                    (6U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                   (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_EN                       (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                 (5U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK                (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_EN                    (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                  (4U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                 (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_EN                     (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_SHIFT                          (3U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_CLRMSK                         (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_EN                             (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_SHIFT                          (2U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_CLRMSK                         (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_EN                             (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_SHIFT                         (1U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_EN                            (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_SHIFT                    (0U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                   (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_EN                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_STATUS
+*/
+#define RGX_CR_FAULT_FW_STATUS                                                      (0xF3B0U)
+#define RGX_CR_FAULT_FW_STATUS_MASKFULL                                             (IMG_UINT64_C(0x0000000000010001))
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_SHIFT                                    (16U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_CLRMSK                                   (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_EN                                       (0x00010000U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_SHIFT                                     (0U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_CLEAR
+*/
+#define RGX_CR_FAULT_FW_CLEAR                                                       (0xF3B8U)
+#define RGX_CR_FAULT_FW_CLEAR_MASKFULL                                              (IMG_UINT64_C(0x0000000000010001))
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_SHIFT                                     (16U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_CLRMSK                                    (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_EN                                        (0x00010000U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_SHIFT                                      (0U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_SAFETY_EVENT_ENABLE
+*/
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE                                     (0xF3D8U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL                           (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_SHIFT                   (7U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_CLRMSK                  (0xFFFFFF7FU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_EN                      (0x00000080U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_SHIFT               (6U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_CLRMSK              (0xFFFFFFBFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_EN                  (0x00000040U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT            (5U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK           (0xFFFFFFDFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_EN               (0x00000020U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT             (4U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_EN                (0x00000010U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_SHIFT                     (3U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_CLRMSK                    (0xFFFFFFF7U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_EN                        (0x00000008U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_SHIFT                     (2U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_CLRMSK                    (0xFFFFFFFBU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_EN                        (0x00000004U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_SHIFT                    (1U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_CLRMSK                   (0xFFFFFFFDU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_EN                       (0x00000002U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_SHIFT               (0U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_CLRMSK              (0xFFFFFFFEU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_EN                  (0x00000001U)
+
+
+#endif /* RGX_CR_DEFS_KM_H */
+/*****************************************************************************
+ End of file (rgx_cr_defs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxdefs_km.h
new file mode 100644
index 000000000000..818166dff902
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxdefs_km.h
@@ -0,0 +1,381 @@
+/*************************************************************************/ /*!
+@Title          Rogue hw definitions (kernel mode)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXDEFS_KM_H
+#define RGXDEFS_KM_H
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+#include RGX_BVNC_CORE_KM_HEADER
+#include RGX_BNC_CONFIG_KM_HEADER
+#endif
+
+#define IMG_EXPLICIT_INCLUDE_HWDEFS
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#include "rgx_cr_defs_km.h"
+#endif
+#undef IMG_EXPLICIT_INCLUDE_HWDEFS
+
+#include "rgx_heap_firmware.h"
+
+/* The following Macros are picked up through BVNC headers for no hardware
+ * operations to be compatible with old build infrastructure.
+ */
+#if defined(NO_HARDWARE)
+/******************************************************************************
+ * Check for valid B.X.N.C
+ *****************************************************************************/
+#if !defined(RGX_BVNC_KM_B) || !defined(RGX_BVNC_KM_V) || !defined(RGX_BVNC_KM_N) || !defined(RGX_BVNC_KM_C)
+#error "Need to specify BVNC (RGX_BVNC_KM_B, RGX_BVNC_KM_V, RGX_BVNC_KM_N and RGX_BVNC_C)"
+#endif
+
+/* Check core/config compatibility */
+#if (RGX_BVNC_KM_B != RGX_BNC_KM_B) || (RGX_BVNC_KM_N != RGX_BNC_KM_N) || (RGX_BVNC_KM_C != RGX_BNC_KM_C)
+#error "BVNC headers are mismatching (KM core/config)"
+#endif
+#endif
+
+/******************************************************************************
+ * RGX Version name
+ *****************************************************************************/
+#define RGX_BVNC_KM_ST2(S)	#S
+#define RGX_BVNC_KM_ST(S)	RGX_BVNC_KM_ST2(S)
+#define RGX_BVNC_KM			RGX_BVNC_KM_ST(RGX_BVNC_KM_B) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_V) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_N) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_C)
+#define RGX_BVNC_KM_V_ST	RGX_BVNC_KM_ST(RGX_BVNC_KM_V)
+
+/* Maximum string size is [bb.vvvp.nnnn.cccc\0], includes null char */
+#define RGX_BVNC_STR_SIZE_MAX (2U+1U+4U+1U+4U+1U+4U+1U)
+#define RGX_BVNC_STR_FMTSPEC  "%u.%u.%u.%u"
+#define RGX_BVNC_STRP_FMTSPEC "%u.%up.%u.%u"
+
+
+/******************************************************************************
+ * RGX Defines
+ *****************************************************************************/
+
+#define BVNC_FIELD_MASK     ((1UL << BVNC_FIELD_WIDTH) - 1U)
+#define C_POSITION          (0U)
+#define N_POSITION          ((C_POSITION) + (BVNC_FIELD_WIDTH))
+#define V_POSITION          ((N_POSITION) + (BVNC_FIELD_WIDTH))
+#define B_POSITION          ((V_POSITION) + (BVNC_FIELD_WIDTH))
+
+#define B_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (B_POSITION)))
+#define V_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (V_POSITION)))
+#define N_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (N_POSITION)))
+#define C_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (C_POSITION)))
+
+#define GET_B(x)            (((x) & (B_POSTION_MASK)) >> (B_POSITION))
+#define GET_V(x)            (((x) & (V_POSTION_MASK)) >> (V_POSITION))
+#define GET_N(x)            (((x) & (N_POSTION_MASK)) >> (N_POSITION))
+#define GET_C(x)            (((x) & (C_POSTION_MASK)) >> (C_POSITION))
+
+#define BVNC_PACK(B,V,N,C)  (((((IMG_UINT64)(B))) << (B_POSITION)) | \
+                             ((((IMG_UINT64)(V))) << (V_POSITION)) | \
+                             ((((IMG_UINT64)(N))) << (N_POSITION)) | \
+                             ((((IMG_UINT64)(C))) << (C_POSITION)) \
+                            )
+
+#define RGX_CR_CORE_ID_CONFIG_N_SHIFT    (8U)
+#define RGX_CR_CORE_ID_CONFIG_C_SHIFT    (0U)
+
+#define RGX_CR_CORE_ID_CONFIG_N_CLRMSK   (0XFFFF00FFU)
+#define RGX_CR_CORE_ID_CONFIG_C_CLRMSK   (0XFFFFFF00U)
+
+#if defined(RGX_CR_CORE_ID__PBVNC)
+#define GET_PBVNC_B(X)	((X & ~RGX_CR_CORE_ID__PBVNC__BRANCH_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__BRANCH_ID_SHIFT)
+#define GET_PBVNC_V(X)	((X & ~RGX_CR_CORE_ID__PBVNC__VERSION_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__VERSION_ID_SHIFT)
+#define GET_PBVNC_N(X)	((X & ~RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_SHIFT)
+#define GET_PBVNC_C(X)	((X & ~RGX_CR_CORE_ID__PBVNC__CONFIG_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__CONFIG_ID_SHIFT)
+#endif
+
+#if defined(RGX_FEATURE_NUM_OSIDS)
+#define RGXFW_MAX_NUM_OSIDS                               (RGX_FEATURE_NUM_OSIDS)
+#else
+#define RGXFW_MAX_NUM_OSIDS                               (8U)
+#endif
+
+#define RGXFW_HOST_DRIVER_ID                              (0U)
+#define RGXFW_GUEST_DRIVER_ID_START                       (RGXFW_HOST_DRIVER_ID + 1U)
+
+#define RGXFW_THREAD_0                                    (0U)
+#define RGXFW_THREAD_1                                    (1U)
+
+/* META cores (required for the RGX_FEATURE_META) */
+#define MTP218   (1U)
+#define MTP219   (2U)
+#define LTP218   (3U)
+#define LTP217   (4U)
+
+/* META Core memory feature depending on META variants */
+#define RGX_META_COREMEM_32K      (32*1024)
+#define RGX_META_COREMEM_48K      (48*1024)
+#define RGX_META_COREMEM_64K      (64*1024)
+#define RGX_META_COREMEM_96K      (96*1024)
+#define RGX_META_COREMEM_128K     (128*1024)
+#define RGX_META_COREMEM_256K     (256*1024)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+#if (!defined(SUPPORT_TRUSTED_DEVICE) || defined(RGX_FEATURE_META_DMA)) && \
+    (defined(RGX_FEATURE_META_COREMEM_SIZE) && RGX_FEATURE_META_COREMEM_SIZE != 0)
+#define RGX_META_COREMEM_SIZE     (RGX_FEATURE_META_COREMEM_SIZE*1024U)
+#define RGX_META_COREMEM          (1)
+#define RGX_META_COREMEM_CODE     (1)
+#if !defined(FIX_HW_BRN_50767) && (!defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1))
+#define RGX_META_COREMEM_DATA     (1)
+#endif
+#else
+#undef SUPPORT_META_COREMEM
+#undef RGX_FEATURE_META_COREMEM_SIZE
+#undef RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_COREMEM_SIZE (0)
+#define RGX_META_COREMEM_SIZE         (0)
+#endif
+#endif
+
+#define GET_ROGUE_CACHE_LINE_SIZE(x)    ((((IMG_UINT32)(x)) > 0U) ? ((IMG_UINT32)(x)/8U) : (0U))
+
+#if defined(RGX_FEATURE_META_DMA)
+#define RGX_META_DMA_BLOCK_SIZE (32U)
+#else
+#define RGX_META_DMA_BLOCK_SIZE (0U)
+#endif
+
+#if defined(SUPPORT_AGP)
+#define MAX_HW_TA3DCONTEXTS	3U
+#else
+#define MAX_HW_TA3DCONTEXTS	2U
+#endif
+
+#define RGX_CR_CLK_CTRL_ALL_ON          (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL_MASKFULL)
+#define RGX_CR_CLK_CTRL_ALL_AUTO        (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_ON         (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_AUTO       (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_XTPLUS_CTRL_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_XTPLUS_CTRL_MASKFULL)
+#define RGX_CR_CLK_XTPLUS_CTRL_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_XTPLUS_CTRL_MASKFULL)
+#define DPX_CR_DPX_CLK_CTRL_ALL_ON      (IMG_UINT64_C(0x5555555555555555)&DPX_CR_DPX_CLK_CTRL_MASKFULL)
+#define DPX_CR_DPX_CLK_CTRL_ALL_AUTO    (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&DPX_CR_DPX_CLK_CTRL_MASKFULL)
+
+#define RGX_CR_SOFT_RESET_DUST_n_CORE_EN	(RGX_CR_SOFT_RESET_DUST_A_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_B_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_C_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_D_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_E_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_F_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_G_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_H_CORE_EN)
+
+/* SOFT_RESET Rascal and DUSTs bits */
+#define RGX_CR_SOFT_RESET_RASCALDUSTS_EN	(RGX_CR_SOFT_RESET_RASCAL_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_n_CORE_EN)
+
+
+
+
+/* SOFT_RESET steps as defined in the TRM */
+#define RGX_S7_SOFT_RESET_DUSTS (RGX_CR_SOFT_RESET_DUST_n_CORE_EN)
+
+#define RGX_S7_SOFT_RESET_JONES (RGX_CR_SOFT_RESET_PM_EN  | \
+                                 RGX_CR_SOFT_RESET_VDM_EN | \
+                                 RGX_CR_SOFT_RESET_ISP_EN)
+
+#define RGX_S7_SOFT_RESET_JONES_ALL (RGX_S7_SOFT_RESET_JONES  | \
+                                     RGX_CR_SOFT_RESET_BIF_EN | \
+                                     RGX_CR_SOFT_RESET_SLC_EN | \
+                                     RGX_CR_SOFT_RESET_GARTEN_EN)
+
+#define RGX_S7_SOFT_RESET2 (RGX_CR_SOFT_RESET2_BLACKPEARL_EN | \
+                            RGX_CR_SOFT_RESET2_PIXEL_EN | \
+                            RGX_CR_SOFT_RESET2_CDM_EN | \
+                            RGX_CR_SOFT_RESET2_VERTEX_EN)
+
+
+/* PM interactive mode enabled default for all rogue cores */
+#define PM_INTERACTIVE_MODE
+
+#define RGX_MLIST_ENTRY_STRIDE	(4U) /* 4 bytes */
+#define RGX_NUM_PM_ADDR_SPACES	(2U) /* VCE & TE share virtual space and Alist */
+#define RGX_PM_MAX_PB_VIRT_ADDR_SPACE  (IMG_UINT64_C(0x400000000)) /* PM Maximum addressable limit */
+
+#define RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT		(12U)
+#define RGX_BIF_PM_PHYSICAL_PAGE_SIZE			(1UL << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT		(14U)
+#define RGX_BIF_PM_VIRTUAL_PAGE_SIZE			(1UL << RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE	(16U)
+
+/* To get the number of required Dusts, divide the number of
+ * clusters by 2 and round up
+ */
+#define RGX_REQ_NUM_DUSTS(CLUSTERS)    (((CLUSTERS) + 1U) / 2U)
+
+/* To get the number of required Bernado/Phantom(s), divide
+ * the number of clusters by 4 and round up
+ */
+#define RGX_REQ_NUM_PHANTOMS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+#define RGX_REQ_NUM_BERNADOS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+#define RGX_REQ_NUM_BLACKPEARLS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+# define RGX_NUM_PHANTOMS (RGX_REQ_NUM_PHANTOMS(RGX_FEATURE_NUM_CLUSTERS))
+#endif
+
+
+/* META second thread feature depending on META variants and
+ * available CoreMem
+ */
+#if defined(RGX_FEATURE_META) && (RGX_FEATURE_META == MTP218 || RGX_FEATURE_META == MTP219) && defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && (RGX_FEATURE_META_COREMEM_SIZE == 256)
+#define RGXFW_META_SUPPORT_2ND_THREAD
+#endif
+
+/*
+ * Utility macros to calculate CAT_BASE register addresses
+ */
+#define BIF_CAT_BASEx(n) \
+	(RGX_CR_BIF_CAT_BASE0 + ((n) * (RGX_CR_BIF_CAT_BASE1 - RGX_CR_BIF_CAT_BASE0)))
+
+#define FWCORE_MEM_CAT_BASEx(n) \
+	(RGX_CR_FWCORE_MEM_CAT_BASE0 + ((n) * (RGX_CR_FWCORE_MEM_CAT_BASE1 - RGX_CR_FWCORE_MEM_CAT_BASE0)))
+
+/*
+ * FWCORE wrapper register defines
+ */
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT   RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK  RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK
+#define FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT     (12U)
+
+/******************************************************************************
+ * WA HWBRNs
+ *****************************************************************************/
+
+#if defined(RGX_CR_JONES_IDLE_MASKFULL)
+/* Workaround for HW BRN 57289 */
+#if (RGX_CR_JONES_IDLE_MASKFULL != 0x0000000000007FFF)
+#error This WA must be updated if RGX_CR_JONES_IDLE is expanded!!!
+#endif
+#undef RGX_CR_JONES_IDLE_MASKFULL
+#undef RGX_CR_JONES_IDLE_TDM_SHIFT
+#undef RGX_CR_JONES_IDLE_TDM_CLRMSK
+#undef RGX_CR_JONES_IDLE_TDM_EN
+#define RGX_CR_JONES_IDLE_MASKFULL                        (IMG_UINT64_C(0x0000000000003FFF))
+#endif
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+
+#if defined(RGX_FEATURE_ROGUEXE)
+#define RGX_NUM_RASTERISATION_MODULES	RGX_FEATURE_NUM_CLUSTERS
+#else
+#define RGX_NUM_RASTERISATION_MODULES	RGX_NUM_PHANTOMS
+#endif
+
+#endif /* !(defined(__KERNEL__) || defined(TEE_DDK)) */
+
+/* GPU CR timer tick in GPU cycles */
+#define RGX_CRTIME_TICK_IN_CYCLES					(256U)
+#define RGX_CRTIME_TICK_IN_CYCLES_SHIFT				(8U)
+
+/* for nohw multicore return max cores possible to client */
+#define RGX_MULTICORE_MAX_NOHW_CORES               (4U)
+
+/*
+  If the size of the SLC is less than this value then the TPU bypasses the SLC.
+ */
+#define RGX_TPU_CACHED_SLC_SIZE_THRESHOLD_KB			(128U)
+
+/*
+ * If the size of the SLC is bigger than this value then the TCU must not be bypassed in the SLC.
+ * In XE_MEMORY_HIERARCHY cores, the TCU is bypassed by default.
+ */
+#define RGX_TCU_CACHED_SLC_SIZE_THRESHOLD_KB			(32U)
+
+/*
+ * Register used by the FW to track the current boot stage (not used in MIPS)
+ */
+#define RGX_FW_BOOT_STAGE_REGISTER     (RGX_CR_POWER_ESTIMATE_RESULT)
+
+/*
+ * Virtualisation definitions
+ */
+#define RGX_VIRTUALISATION_REG_SIZE_PER_OS (RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE)
+
+/*
+ * Renaming MTS sideband bitfields to emphasize that the Register Bank number
+ * of the MTS register used identifies a specific Driver/VM rather than the OSID tag
+ * emitted on bus memory transactions.
+ */
+#define RGX_MTS_SBDATA_DRIVERID_CLRMSK RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_CLRMSK
+#define RGX_MTS_SBDATA_DRIVERID_SHIFT RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_SHIFT
+
+/*
+ * Register Bank containing registers secured against host access
+ */
+#define RGX_HOST_SECURE_REGBANK_OFFSET				(0xF0000U)
+#define RGX_HOST_SECURE_REGBANK_SIZE				(0x10000U)
+
+/*
+	Maximum number of render targets in array
+*/
+#define RGX_MAX_TA_RENDER_TARGETS					(2048U)
+
+/*
+ * Macro used to indicate which version of HWPerf is active
+ */
+#define RGX_FEATURE_HWPERF_ROGUE
+
+/*
+ * Maximum number of cores supported by TRP
+ */
+#define RGX_TRP_MAX_NUM_CORES                           (4U)
+
+/*
+ * Maximum number of cores supported by WGP
+ */
+#define RGX_WGP_MAX_NUM_CORES                           (8U)
+
+#if defined(RGX_FEATURE_VOLCANIC_TB)
+#define SUPPORT_VOLCANIC_TB
+#endif
+#define RGX_FEATURE_SECURITY_ROGUE
+
+/* Typically the PCI bus returns this value on error */
+#define RGX_PCI_ERROR_VALUE_BYTE (0xFFU)
+#define RGX_PCI_ERROR_VALUE_DWORD (0xFFFFFFFFU)
+
+#endif /* RGXDEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmhdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmhdefs_km.h
new file mode 100644
index 000000000000..663001dd9ebc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmhdefs_km.h
@@ -0,0 +1,286 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmhdefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_mh.def
+ */
+
+
+#ifndef RGXMHDEFS_KM_H
+#define RGXMHDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMHDEFS_KM_REVISION 0
+
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_FENCE                            (0x00000000U)
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_CONTEXT                          (0x00000001U)
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_QUEUE                            (0x00000002U)
+
+
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTL_STREAM                       (0x00000000U)
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTX_BUFFER                       (0x00000001U)
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_QUEUE_CTL                        (0x00000002U)
+
+
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAFSTACK                              (0x00000008U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMLIST                               (0x00000009U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DFSTACK                              (0x0000000aU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMLIST                               (0x0000000bU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX0                                (0x0000000cU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX1                                (0x0000002dU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_UFSTACK                               (0x0000000fU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMMUSTACK                            (0x00000012U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMMUSTACK                            (0x00000013U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAUFSTACK                             (0x00000016U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DUFSTACK                             (0x00000017U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DVFP                                 (0x00000019U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAVFP                                 (0x0000001aU)
+
+
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAFSTACK                              (0x00000000U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMLIST                               (0x00000001U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DFSTACK                              (0x00000002U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMLIST                               (0x00000003U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX0                                (0x00000004U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX1                                (0x00000025U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_MAVP                                  (0x00000006U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_UFSTACK                               (0x00000007U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMMUSTACK                            (0x00000008U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMMUSTACK                            (0x00000009U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAUFSTACK                             (0x00000014U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DUFSTACK                             (0x00000015U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAVFP                                 (0x00000018U)
+
+
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PPP                                        (0x00000008U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_VCERTC                                     (0x00000007U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_TEACRTC                                    (0x00000006U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGRTC                                     (0x00000005U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGR                                       (0x00000004U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGS                                       (0x00000003U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_TPC                                        (0x00000002U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_VCE                                        (0x00000001U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_PREQ00                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_PREQ01                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_RREQ                     (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_DBSC                     (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CPF                      (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_DELTA                    (0x00000007U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ02                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ03                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ00                   (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ01                   (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ02                   (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ03                   (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_RREQ                     (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_DBSC                     (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CPF                      (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_DELTA                    (0x0000000bU)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ02                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ03                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ04                   (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ05                   (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ06                   (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ00                   (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ01                   (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ02                   (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ03                   (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ04                   (0x0000000bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ05                   (0x0000000cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ06                   (0x0000000dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_RREQ                     (0x0000000eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_DBSC                     (0x0000000fU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CPF                      (0x00000010U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_DELTA                    (0x00000011U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ00                  (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ01                  (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ02                  (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ03                  (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ04                  (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ05                  (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ06                  (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ07                  (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ08                  (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ09                  (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ10                  (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ11                  (0x0000000bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ12                  (0x0000000cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ13                  (0x0000000dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ00                  (0x0000000eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ01                  (0x0000000fU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ02                  (0x00000010U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ03                  (0x00000011U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ04                  (0x00000012U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ05                  (0x00000013U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ06                  (0x00000014U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ07                  (0x00000015U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ08                  (0x00000016U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ09                  (0x00000017U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ10                  (0x00000018U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ11                  (0x00000019U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ12                  (0x0000001aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ13                  (0x0000001bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_RREQ                    (0x0000001cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_DBSC                    (0x0000001dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CPF                     (0x0000001eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_DELTA                   (0x0000001fU)
+
+
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_PDS_STATE                                (0x00000000U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DEPTH_BIAS                               (0x00000001U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_FLOOR_OFFSET_DATA                        (0x00000002U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DELTA_DATA                               (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_ISP_ENCODING_ISP_TAG_ZLS                                      (0x00000000U)
+#define RGX_MH_TAG_SB_ISP_ENCODING_ISP_TAG_DS                                       (0x00000001U)
+
+
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTROL                                  (0x00000000U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STATE                                    (0x00000001U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_INDEX                                    (0x00000002U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STACK                                    (0x00000004U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTEXT                                  (0x00000008U)
+
+
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTROL_STREAM                           (0x00000000U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_INDIRECT_DATA                            (0x00000001U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_EVENT_DATA                               (0x00000002U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTEXT_STATE                            (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_OPCODE_FETCH                           (0x00000002U)
+#define RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_DATA_ACCESS                            (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PT_REQUEST                               (0x00000000U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PD_REQUEST                               (0x00000001U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PC_REQUEST                               (0x00000002U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PT_REQUEST                            (0x00000003U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_REQUEST                            (0x00000004U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_REQUEST                            (0x00000005U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_WREQUEST                           (0x00000006U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_WREQUEST                           (0x00000007U)
+
+
+#define RGX_MH_TAG_ENCODING_MH_TAG_MMU                                              (0x00000000U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU                                          (0x00000001U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU                                          (0x00000002U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU                                          (0x00000003U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MIPS                                             (0x00000004U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0                                         (0x00000005U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1                                         (0x00000006U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2                                         (0x00000007U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3                                         (0x00000008U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0                                         (0x00000009U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1                                         (0x0000000aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2                                         (0x0000000bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3                                         (0x0000000cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4                                         (0x0000000dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PDS_0                                            (0x0000000eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PDS_1                                            (0x0000000fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCA                                         (0x00000010U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCB                                         (0x00000011U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCC                                         (0x00000012U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCD                                         (0x00000013U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCA                                     (0x00000014U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCB                                     (0x00000015U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCC                                     (0x00000016U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCD                                     (0x00000017U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDSRW                                        (0x00000018U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TCU_0                                            (0x00000019U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TCU_1                                            (0x0000001aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_0                                          (0x0000001bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_1                                          (0x0000001cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_2                                          (0x0000001dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_3                                          (0x0000001eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_USC                                              (0x0000001fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_ISP_ZLS                                          (0x00000020U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_ISP_DS                                           (0x00000021U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF                                              (0x00000022U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS                                     (0x00000023U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF                                          (0x00000024U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ                                         (0x00000025U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS                                       (0x00000026U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG5                                         (0x00000027U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PPP                                           (0x00000028U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPWRTC                                        (0x00000029U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TEACRTC                                       (0x0000002aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGRTC                                        (0x0000002bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGREGION                                     (0x0000002cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGSTREAM                                     (0x0000002dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPW                                           (0x0000002eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPC                                           (0x0000002fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PM_ALLOC                                         (0x00000030U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PM_DEALLOC                                       (0x00000031U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TDM_DMA                                          (0x00000032U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TDM_CTL                                          (0x00000033U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE0                                             (0x00000034U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE1                                             (0x00000035U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE2                                             (0x00000036U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE3                                             (0x00000037U)
+
+
+#endif /* RGXMHDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmhdefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmmudefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmmudefs_km.h
new file mode 100644
index 000000000000..14a6a73c84bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/rogue/km/rgxmmudefs_km.h
@@ -0,0 +1,216 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmmudefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_bif.def
+ */
+
+
+#ifndef RGXMMUDEFS_KM_H
+#define RGXMMUDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMMUDEFS_KM_REVISION 0
+
+#define RGX_BIF_DM_ENCODING_VERTEX                                                  (0x00000000U)
+#define RGX_BIF_DM_ENCODING_PIXEL                                                   (0x00000001U)
+#define RGX_BIF_DM_ENCODING_COMPUTE                                                 (0x00000002U)
+#define RGX_BIF_DM_ENCODING_TLA                                                     (0x00000003U)
+#define RGX_BIF_DM_ENCODING_PB_VCE                                                  (0x00000004U)
+#define RGX_BIF_DM_ENCODING_PB_TE                                                   (0x00000005U)
+#define RGX_BIF_DM_ENCODING_META                                                    (0x00000007U)
+#define RGX_BIF_DM_ENCODING_HOST                                                    (0x00000008U)
+#define RGX_BIF_DM_ENCODING_PM_ALIST                                                (0x00000009U)
+
+
+#define RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT                                            (30U)
+#define RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF003FFFFFFF))
+#define RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT                                            (21U)
+#define RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFC01FFFFF))
+#define RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT                                            (12U)
+#define RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFE00FFF))
+
+
+#define RGX_MMUCTRL_ENTRIES_PC_VALUE                                                (0x00000400U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PD_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PT_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PC_VALUE                                             (0x00000020U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PD_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PT_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_PAGE_SIZE_MASK                                                  (0x00000007U)
+#define RGX_MMUCTRL_PAGE_SIZE_4KB                                                   (0x00000000U)
+#define RGX_MMUCTRL_PAGE_SIZE_16KB                                                  (0x00000001U)
+#define RGX_MMUCTRL_PAGE_SIZE_64KB                                                  (0x00000002U)
+#define RGX_MMUCTRL_PAGE_SIZE_256KB                                                 (0x00000003U)
+#define RGX_MMUCTRL_PAGE_SIZE_1MB                                                   (0x00000004U)
+#define RGX_MMUCTRL_PAGE_SIZE_2MB                                                   (0x00000005U)
+
+
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT                                            (12U)
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_SHIFT                                           (14U)
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF0000003FFF))
+
+
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_SHIFT                                           (16U)
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_SHIFT                                          (18U)
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFF000003FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_SHIFT                                            (20U)
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00000FFFFF))
+
+
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_SHIFT                                            (21U)
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00001FFFFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_SHIFT                                         (12U)
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_SHIFT                                        (10U)
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000003FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_SHIFT                                        (8U)
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000000FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_SHIFT                                       (6U)
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF000000003F))
+
+
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_SHIFT                                   (62U)
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_CLRMSK                                  (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_SHIFT                                        (40U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_CLRMSK                                       (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PAGE_SHIFT                                              (12U)
+#define RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_SHIFT                                        (6U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_SHIFT                                     (5U)
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000000000000020))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_SHIFT                                            (4U)
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_EN                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_SHIFT                                   (3U)
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN                                      (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PT_DATA_CC_SHIFT                                                (2U)
+#define RGX_MMUCTRL_PT_DATA_CC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_MMUCTRL_PT_DATA_CC_EN                                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_EN                                            (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PT_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PT_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PT_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_SHIFT                                     (40U)
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_SHIFT                                           (5U)
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF1))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB                                         (IMG_UINT64_C(0x0000000000000006))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB                                           (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB                                           (IMG_UINT64_C(0x000000000000000a))
+#define RGX_MMUCTRL_PD_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PD_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PD_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT                                           (4U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK                                          (0x0000000FU)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT                                      (12U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSIZE                                       (4096U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_SHIFT                                     (1U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN                                        (0x00000002U)
+#define RGX_MMUCTRL_PC_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PC_DATA_VALID_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_MMUCTRL_PC_DATA_VALID_EN                                                (0x00000001U)
+
+
+#endif /* RGXMMUDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmmudefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_27.V.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_27.V.254.2.h
new file mode 100644
index 000000000000..3d7bc8345c05
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_27.V.254.2.h
@@ -0,0 +1,136 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 27.V.254.2 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_27_V_254_2_H
+#define RGXCONFIG_KM_27_V_254_2_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 27
+#define RGX_BNC_KM_N 254
+#define RGX_BNC_KM_C 2
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (3U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_MMU_VERSION (3U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (2U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_27_V_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.1632.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.1632.1.h
new file mode 100644
index 000000000000..caafb57427d6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.1632.1.h
@@ -0,0 +1,140 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 30.V.1632.1 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_30_V_1632_1_H
+#define RGXCONFIG_KM_30_V_1632_1_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 30
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 1
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (512U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_30_V_1632_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.408.101.h
new file mode 100644
index 000000000000..7e9a1f7b3cd0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.408.101.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 30.V.408.101 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_30_V_408_101_H
+#define RGXCONFIG_KM_30_V_408_101_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 30
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 101
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_30_V_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.816.20.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.816.20.h
new file mode 100644
index 000000000000..f09e158a5ab8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_30.V.816.20.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 30.V.816.20 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_30_V_816_20_H
+#define RGXCONFIG_KM_30_V_816_20_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 30
+#define RGX_BNC_KM_N 816
+#define RGX_BNC_KM_C 20
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (256U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_30_V_816_20_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.21.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.21.h
new file mode 100644
index 000000000000..b0faae645326
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.21.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.21 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_21_H
+#define RGXCONFIG_KM_35_V_1632_21_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 21
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (512U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_21_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.23.h
new file mode 100644
index 000000000000..07fd015bc7a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.23.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.23 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_23_H
+#define RGXCONFIG_KM_35_V_1632_23_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 23
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (512U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.34.h
new file mode 100644
index 000000000000..18e0cf4ba89b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.34.h
@@ -0,0 +1,153 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.34 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_34_H
+#define RGXCONFIG_KM_35_V_1632_34_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 34
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.35.h
new file mode 100644
index 000000000000..d359272a63f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.1632.35.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.1632.35 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_1632_35_H
+#define RGXCONFIG_KM_35_V_1632_35_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 1632
+#define RGX_BNC_KM_C 35
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (16U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_1632_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.101.h
new file mode 100644
index 000000000000..ab5fde00d4d5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.101.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.101 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_101_H
+#define RGXCONFIG_KM_35_V_408_101_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 101
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (512U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (1U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (16384U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.23.h
new file mode 100644
index 000000000000..718e12c8ed69
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.23.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.23 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_23_H
+#define RGXCONFIG_KM_35_V_408_23_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 23
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.33.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.33.h
new file mode 100644
index 000000000000..df85d8147f21
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.33.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.33 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_33_H
+#define RGXCONFIG_KM_35_V_408_33_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 33
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_33_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.34.h
new file mode 100644
index 000000000000..8fb939b2a65b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_35.V.408.34.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 35.V.408.34 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_35_V_408_34_H
+#define RGXCONFIG_KM_35_V_408_34_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 35
+#define RGX_BNC_KM_N 408
+#define RGX_BNC_KM_C 34
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (1U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (1U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (1U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (1U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (1U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (1U)
+#define RGX_FEATURE_SCALABLE_VCE (1U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_35_V_408_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_38.V.2448.402.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_38.V.2448.402.h
new file mode 100644
index 000000000000..4d4d7c7f9c1d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_38.V.2448.402.h
@@ -0,0 +1,153 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 38.V.2448.402 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_38_V_2448_402_H
+#define RGXCONFIG_KM_38_V_2448_402_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 38
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 402
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (1U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (2U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_38_V_2448_402_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1041.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1041.h
new file mode 100644
index 000000000000..3732194dda31
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1041.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.1041 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_1041_H
+#define RGXCONFIG_KM_70_V_2448_1041_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1041
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_1041_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1042.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1042.h
new file mode 100644
index 000000000000..3c8dbf11a303
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1042.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.1042 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_1042_H
+#define RGXCONFIG_KM_70_V_2448_1042_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1042
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_1042_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1360.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1360.h
new file mode 100644
index 000000000000..ec915beeaaf1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.1360.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.1360 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_1360_H
+#define RGXCONFIG_KM_70_V_2448_1360_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1360
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_1360_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.418.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.418.h
new file mode 100644
index 000000000000..045e17306351
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_70.V.2448.418.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 70.V.2448.418 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_70_V_2448_418_H
+#define RGXCONFIG_KM_70_V_2448_418_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 70
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 418
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (4U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (2U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (2U)
+#define RGX_FEATURE_META (MTP219)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (4U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (2U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (0U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (3U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SIGNAL_SNOOPING
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (2U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_70_V_2448_418_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1210.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1210.h
new file mode 100644
index 000000000000..bc75857064f1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1210.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.2448.1210 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_2448_1210_H
+#define RGXCONFIG_KM_71_V_2448_1210_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1210
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (3U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_2448_1210_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1211.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1211.h
new file mode 100644
index 000000000000..67b2fa73b0a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1211.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.2448.1211 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_2448_1211_H
+#define RGXCONFIG_KM_71_V_2448_1211_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1211
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_2448_1211_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1212.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1212.h
new file mode 100644
index 000000000000..5d896d60fca8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.2448.1212.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.2448.1212 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_2448_1212_H
+#define RGXCONFIG_KM_71_V_2448_1212_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 2448
+#define RGX_BNC_KM_C 1212
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (24U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (2U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (2U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (1536U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_2448_1212_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2080.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2080.h
new file mode 100644
index 000000000000..9b6265e018d8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2080.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2080 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2080_H
+#define RGXCONFIG_KM_71_V_3672_2080_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2080
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (1U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (4U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2080_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2081.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2081.h
new file mode 100644
index 000000000000..28ca12ce156f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2081.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2081 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2081_H
+#define RGXCONFIG_KM_71_V_3672_2081_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2081
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2081_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2082.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2082.h
new file mode 100644
index 000000000000..f7c2d401daf1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2082.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2082 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2082_H
+#define RGXCONFIG_KM_71_V_3672_2082_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2082
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2082_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2083.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2083.h
new file mode 100644
index 000000000000..2fc12a05bcad
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2083.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2083 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2083_H
+#define RGXCONFIG_KM_71_V_3672_2083_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2083
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2083_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2084.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2084.h
new file mode 100644
index 000000000000..7ecc97a74886
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/configs/rgxconfig_km_71.V.3672.2084.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 71.V.3672.2084 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_71_V_3672_2084_H
+#define RGXCONFIG_KM_71_V_3672_2084_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 71
+#define RGX_BNC_KM_N 3672
+#define RGX_BNC_KM_C 2084
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_AXI_ACE
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (5U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL (4U)
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (5U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (5U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (6U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (2048U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (32U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_HOST_SECURITY_VERSION (4U)
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU (3U)
+#define RGX_FEATURE_META_COREMEM_BANKS (8U)
+#define RGX_FEATURE_META_COREMEM_SIZE (96U)
+#define RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT (2U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES
+#define RGX_FEATURE_MMU_VERSION (4U)
+#define RGX_FEATURE_NUM_CLUSTERS (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (36U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU (3U)
+#define RGX_FEATURE_NUM_MEMBUS (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_SPU (3U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBE_PER_SPU (1U)
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_PDS_TEMPSIZE8
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION (2U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES
+#define RGX_FEATURE_POWER_ISLAND_VERSION (6U)
+#define RGX_FEATURE_RAY_TRACING_ARCH (0U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX (32768U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_RT_RAC_PER_SPU
+#define RGX_FEATURE_S7_CACHE_HIERARCHY
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE
+#define RGX_FEATURE_SCALABLE_TE_ARCH (2U)
+#define RGX_FEATURE_SCALABLE_VCE (3U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP
+#define RGX_FEATURE_SLC_BANKS (8U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (1024U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT /* Specifies the SLC is */
+                                        /* customer-configurable. True SLC */
+                                        /* size must be sourced from */
+                                        /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2048U)
+#define RGX_FEATURE_SLC_VIVT
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SPU0_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT (1U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT (0U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (2U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT (3U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS
+#define RGX_FEATURE_TESSELLATION
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE
+#define RGX_FEATURE_USC_TIMER
+#define RGX_FEATURE_VDM_DRAWINDIRECT
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_ZLS_CHECKSUM
+
+#endif /* RGXCONFIG_KM_71_V_3672_2084_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.5.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.5.254.2.h
new file mode 100644
index 000000000000..e0b7dfc02471
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.5.254.2.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 27.5.254.2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_27_5_254_2_H
+#define RGXCORE_KM_27_5_254_2_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5262350 */
+
+/******************************************************************************
+ * BVNC = 27.5.254.2
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 27
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 254
+#define RGX_BVNC_KM_C 2
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71422
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+
+
+
+#endif /* RGXCORE_KM_27_5_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.7.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.7.254.2.h
new file mode 100644
index 000000000000..38624611a499
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.7.254.2.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 27.7.254.2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_27_7_254_2_H
+#define RGXCORE_KM_27_7_254_2_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5941981 */
+
+/******************************************************************************
+ * BVNC = 27.7.254.2
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 27
+#define RGX_BVNC_KM_V 7
+#define RGX_BVNC_KM_N 254
+#define RGX_BVNC_KM_C 2
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+
+
+
+#endif /* RGXCORE_KM_27_7_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.8.254.2.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.8.254.2.h
new file mode 100644
index 000000000000..6719e1c5e96c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_27.8.254.2.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 27.8.254.2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_27_8_254_2_H
+#define RGXCORE_KM_27_8_254_2_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5975910 */
+
+/******************************************************************************
+ * BVNC = 27.8.254.2
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 27
+#define RGX_BVNC_KM_V 8
+#define RGX_BVNC_KM_N 254
+#define RGX_BVNC_KM_C 2
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+
+
+
+#endif /* RGXCORE_KM_27_8_254_2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.408.101.h
new file mode 100644
index 000000000000..f05575d4ef65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.408.101.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 30.3.408.101
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_30_3_408_101_H
+#define RGXCORE_KM_30_3_408_101_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5653222 */
+
+/******************************************************************************
+ * BVNC = 30.3.408.101
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 30
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 101
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_30_3_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.816.20.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.816.20.h
new file mode 100644
index 000000000000..7319e86fb7bf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.3.816.20.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 30.3.816.20
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_30_3_816_20_H
+#define RGXCORE_KM_30_3_816_20_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5690709 */
+
+/******************************************************************************
+ * BVNC = 30.3.816.20
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 30
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 816
+#define RGX_BVNC_KM_C 20
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_30_3_816_20_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.5.1632.1.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.5.1632.1.h
new file mode 100644
index 000000000000..0fbbef658811
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_30.5.1632.1.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 30.5.1632.1
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_30_5_1632_1_H
+#define RGXCORE_KM_30_5_1632_1_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @5621518 */
+
+/******************************************************************************
+ * BVNC = 30.5.1632.1
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 30
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 1
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_30_5_1632_1_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.21.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.21.h
new file mode 100644
index 000000000000..f2e489cfbab7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.21.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.21
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_21_H
+#define RGXCORE_KM_35_2_1632_21_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5744712 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.21
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 21
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71157
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_21_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.23.h
new file mode 100644
index 000000000000..9c19267ac08a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.23.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_23_H
+#define RGXCORE_KM_35_2_1632_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5928760 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71157
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.34.h
new file mode 100644
index 000000000000..a4c3741fb3e4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.34.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.34
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_34_H
+#define RGXCORE_KM_35_2_1632_34_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6097610 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.34
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 34
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.35.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.35.h
new file mode 100644
index 000000000000..9a959fe90e65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.1632.35.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.1632.35
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_1632_35_H
+#define RGXCORE_KM_35_2_1632_35_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6327132 */
+
+/******************************************************************************
+ * BVNC = 35.2.1632.35
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 35
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_1632_35_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.33.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.33.h
new file mode 100644
index 000000000000..6e97c78a9226
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.33.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.408.33
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_408_33_H
+#define RGXCORE_KM_35_2_408_33_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @6220991 */
+
+/******************************************************************************
+ * BVNC = 35.2.408.33
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 33
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_408_33_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.34.h
new file mode 100644
index 000000000000..ecd899c933b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.2.408.34.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.2.408.34
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_2_408_34_H
+#define RGXCORE_KM_35_2_408_34_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6252022 */
+
+/******************************************************************************
+ * BVNC = 35.2.408.34
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 34
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_2_408_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.1632.23.h
new file mode 100644
index 000000000000..9726e2ca1fb4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.1632.23.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.1632.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_1632_23_H
+#define RGXCORE_KM_35_3_1632_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5906056 */
+
+/******************************************************************************
+ * BVNC = 35.3.1632.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.101.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.101.h
new file mode 100644
index 000000000000..074fa3d600f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.101.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.408.101
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_408_101_H
+#define RGXCORE_KM_35_3_408_101_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5653222 */
+
+/******************************************************************************
+ * BVNC = 35.3.408.101
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 101
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_408_101_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.33.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.33.h
new file mode 100644
index 000000000000..a17f409b5644
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.33.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.408.33
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_408_33_H
+#define RGXCORE_KM_35_3_408_33_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6249306 */
+
+/******************************************************************************
+ * BVNC = 35.3.408.33
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 33
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_408_33_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.34.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.34.h
new file mode 100644
index 000000000000..c44fe2243220
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.3.408.34.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.3.408.34
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_3_408_34_H
+#define RGXCORE_KM_35_3_408_34_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6285885 */
+
+/******************************************************************************
+ * BVNC = 35.3.408.34
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 34
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_3_408_34_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.4.1632.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.4.1632.23.h
new file mode 100644
index 000000000000..02899da3d8fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.4.1632.23.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.4.1632.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_4_1632_23_H
+#define RGXCORE_KM_35_4_1632_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5952322 */
+
+/******************************************************************************
+ * BVNC = 35.4.1632.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 4
+#define RGX_BVNC_KM_N 1632
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_4_1632_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.5.408.23.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.5.408.23.h
new file mode 100644
index 000000000000..a5048e951f47
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_35.5.408.23.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 35.5.408.23
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_35_5_408_23_H
+#define RGXCORE_KM_35_5_408_23_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6080671 */
+
+/******************************************************************************
+ * BVNC = 35.5.408.23
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 35
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 408
+#define RGX_BVNC_KM_C 23
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_35_5_408_23_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.4.2448.402.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.4.2448.402.h
new file mode 100644
index 000000000000..7d9e4e4ac323
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.4.2448.402.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 38.4.2448.402
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_38_4_2448_402_H
+#define RGXCORE_KM_38_4_2448_402_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @5976185 */
+
+/******************************************************************************
+ * BVNC = 38.4.2448.402
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 38
+#define RGX_BVNC_KM_V 4
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 402
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72143
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_38_4_2448_402_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.6.2448.402.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.6.2448.402.h
new file mode 100644
index 000000000000..7bf85accef8a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_38.6.2448.402.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 38.6.2448.402
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_38_6_2448_402_H
+#define RGXCORE_KM_38_6_2448_402_H
+
+/* Automatically generated file (31/03/2023 09:11:38): Do not edit manually */
+/* CS: @6068975 */
+
+/******************************************************************************
+ * BVNC = 38.6.2448.402
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 38
+#define RGX_BVNC_KM_V 6
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 402
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71960
+#define FIX_HW_BRN_72143
+#define FIX_HW_BRN_72144
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_38_6_2448_402_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1041.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1041.h
new file mode 100644
index 000000000000..c44b6d4d1e5f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1041.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.1041
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_1041_H
+#define RGXCORE_KM_70_2_2448_1041_H
+
+/* Automatically generated file (06/05/2022 11:58:47): Do not edit manually */
+/* CS: @6243633 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.1041
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1041
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_1041_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1042.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1042.h
new file mode 100644
index 000000000000..de04ac9fde5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1042.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.1042
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_1042_H
+#define RGXCORE_KM_70_2_2448_1042_H
+
+/* Automatically generated file (05/09/2023 09:10:40): Do not edit manually */
+/* CS: @6236243 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.1042
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1042
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_1042_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1360.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1360.h
new file mode 100644
index 000000000000..95c2bbad0095
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.1360.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.1360
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_1360_H
+#define RGXCORE_KM_70_2_2448_1360_H
+
+/* Automatically generated file (08/06/2022 13:43:04): Do not edit manually */
+/* CS: @6208659 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.1360
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1360
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_1360_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.418.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.418.h
new file mode 100644
index 000000000000..80335f71e1aa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.2.2448.418.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.2.2448.418
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_2_2448_418_H
+#define RGXCORE_KM_70_2_2448_418_H
+
+/* Automatically generated file (23/03/2022 09:02:26): Do not edit manually */
+/* CS: @6208659 */
+
+/******************************************************************************
+ * BVNC = 70.2.2448.418
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 418
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_2_2448_418_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1042.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1042.h
new file mode 100644
index 000000000000..490f350681f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1042.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.3.2448.1042
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_3_2448_1042_H
+#define RGXCORE_KM_70_3_2448_1042_H
+
+/* Automatically generated file (14/09/2023 09:10:35): Do not edit manually */
+/* CS: @6458228 */
+
+/******************************************************************************
+ * BVNC = 70.3.2448.1042
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1042
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_3_2448_1042_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1360.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1360.h
new file mode 100644
index 000000000000..c07d2e70b8ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_70.3.2448.1360.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 70.3.2448.1360
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_70_3_2448_1360_H
+#define RGXCORE_KM_70_3_2448_1360_H
+
+/* Automatically generated file (28/06/2022 10:24:53): Do not edit manually */
+/* CS: @6266740 */
+
+/******************************************************************************
+ * BVNC = 70.3.2448.1360
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 70
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1360
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_70_3_2448_1360_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1210.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1210.h
new file mode 100644
index 000000000000..e70690a29eb8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1210.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.2448.1210
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_2448_1210_H
+#define RGXCORE_KM_71_2_2448_1210_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6317660 */
+
+/******************************************************************************
+ * BVNC = 71.2.2448.1210
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1210
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_2448_1210_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1211.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1211.h
new file mode 100644
index 000000000000..4bd717024ef8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1211.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.2448.1211
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_2448_1211_H
+#define RGXCORE_KM_71_2_2448_1211_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6347253 */
+
+/******************************************************************************
+ * BVNC = 71.2.2448.1211
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1211
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_2448_1211_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1212.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1212.h
new file mode 100644
index 000000000000..4bf4a7bb6fda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.2448.1212.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.2448.1212
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_2448_1212_H
+#define RGXCORE_KM_71_2_2448_1212_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6450859 */
+
+/******************************************************************************
+ * BVNC = 71.2.2448.1212
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 2448
+#define RGX_BVNC_KM_C 1212
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_2448_1212_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2080.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2080.h
new file mode 100644
index 000000000000..1a17786ee245
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2080.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2080
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2080_H
+#define RGXCORE_KM_71_2_3672_2080_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6375375 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2080
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2080
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2080_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2081.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2081.h
new file mode 100644
index 000000000000..3b0113c6a5c5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2081.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2081
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2081_H
+#define RGXCORE_KM_71_2_3672_2081_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6422248 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2081
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2081
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2081_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2082.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2082.h
new file mode 100644
index 000000000000..651bad6b7c97
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2082.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2082
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2082_H
+#define RGXCORE_KM_71_2_3672_2082_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6449305 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2082
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2082
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2082_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2083.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2083.h
new file mode 100644
index 000000000000..e98d838c03f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.2.3672.2083.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.2.3672.2083
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_2_3672_2083_H
+#define RGXCORE_KM_71_2_3672_2083_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6464565 */
+
+/******************************************************************************
+ * BVNC = 71.2.3672.2083
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 2
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2083
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_2_3672_2083_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2083.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2083.h
new file mode 100644
index 000000000000..bdc923c78d0e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2083.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.3.3672.2083
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_3_3672_2083_H
+#define RGXCORE_KM_71_3_3672_2083_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6473124 */
+
+/******************************************************************************
+ * BVNC = 71.3.3672.2083
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2083
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_3_3672_2083_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2084.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2084.h
new file mode 100644
index 000000000000..f287f89ce147
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/cores/rgxcore_km_71.3.3672.2084.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 71.3.3672.2084
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_71_3_3672_2084_H
+#define RGXCORE_KM_71_3_3672_2084_H
+
+/* Automatically generated file (30/11/2023 09:10:38): Do not edit manually */
+/* CS: @6495448 */
+
+/******************************************************************************
+ * BVNC = 71.3.3672.2084
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 71
+#define RGX_BVNC_KM_V 3
+#define RGX_BVNC_KM_N 3672
+#define RGX_BVNC_KM_C 2084
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_74812
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_65104
+#define HW_ERN_69700
+
+
+
+#endif /* RGXCORE_KM_71_3_3672_2084_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_defs_km.h
new file mode 100644
index 000000000000..769165e76a2e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_defs_km.h
@@ -0,0 +1,397 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_defs_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_DEFS_KM_H
+#define RGX_BVNC_DEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(RGX_BVNC_DEFS_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_defs_um.h"
+#endif
+
+#define BVNC_FIELD_WIDTH (16U)
+
+#define PVR_ARCH_NAME "volcanic"
+
+
+/******************************************************************************
+ * Mask and bit-position macros for features without values
+ *****************************************************************************/
+
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE_POS                     (0U)
+#define RGX_FEATURE_ALBIORIX_TOP_INFRASTRUCTURE_BIT_MASK                (IMG_UINT64_C(0x0000000000000001))
+
+#define RGX_FEATURE_AXI_ACE_POS                                         (1U)
+#define RGX_FEATURE_AXI_ACE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000000002))
+
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE_POS                       (2U)
+#define RGX_FEATURE_BARREX_TOP_INFRASTRUCTURE_BIT_MASK                  (IMG_UINT64_C(0x0000000000000004))
+
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE_POS                (3U)
+#define RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE_BIT_MASK           (IMG_UINT64_C(0x0000000000000008))
+
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE_POS                      (4U)
+#define RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE_BIT_MASK                 (IMG_UINT64_C(0x0000000000000010))
+
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE_POS                  (5U)
+#define RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE_BIT_MASK             (IMG_UINT64_C(0x0000000000000020))
+
+#define RGX_FEATURE_CLUSTER_GROUPING_POS                                (6U)
+#define RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK                           (IMG_UINT64_C(0x0000000000000040))
+
+#define RGX_FEATURE_COMPUTE_POS                                         (7U)
+#define RGX_FEATURE_COMPUTE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000000080))
+
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_POS                          (8U)
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_BIT_MASK                     (IMG_UINT64_C(0x0000000000000100))
+
+#define RGX_FEATURE_COMPUTE_ONLY_POS                                    (9U)
+#define RGX_FEATURE_COMPUTE_ONLY_BIT_MASK                               (IMG_UINT64_C(0x0000000000000200))
+
+#define RGX_FEATURE_COMPUTE_OVERLAP_POS                                 (10U)
+#define RGX_FEATURE_COMPUTE_OVERLAP_BIT_MASK                            (IMG_UINT64_C(0x0000000000000400))
+
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS_POS                   (11U)
+#define RGX_FEATURE_COMPUTE_OVERLAP_WITH_BARRIERS_BIT_MASK              (IMG_UINT64_C(0x0000000000000800))
+
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS_POS                  (12U)
+#define RGX_FEATURE_COMPUTE_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK             (IMG_UINT64_C(0x0000000000001000))
+
+#define RGX_FEATURE_COREID_PER_OS_POS                                   (13U)
+#define RGX_FEATURE_COREID_PER_OS_BIT_MASK                              (IMG_UINT64_C(0x0000000000002000))
+
+#define RGX_FEATURE_CR_PARITY_POS                                       (14U)
+#define RGX_FEATURE_CR_PARITY_BIT_MASK                                  (IMG_UINT64_C(0x0000000000004000))
+
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE_POS                       (15U)
+#define RGX_FEATURE_DIVANO_TOP_INFRASTRUCTURE_BIT_MASK                  (IMG_UINT64_C(0x0000000000008000))
+
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7_POS                            (16U)
+#define RGX_FEATURE_DUST_POWER_ISLAND_S7_BIT_MASK                       (IMG_UINT64_C(0x0000000000010000))
+
+#define RGX_FEATURE_ERYX_TOP_INFRASTRUCTURE_POS                         (17U)
+#define RGX_FEATURE_ERYX_TOP_INFRASTRUCTURE_BIT_MASK                    (IMG_UINT64_C(0x0000000000020000))
+
+#define RGX_FEATURE_FASTRENDER_DM_POS                                   (18U)
+#define RGX_FEATURE_FASTRENDER_DM_BIT_MASK                              (IMG_UINT64_C(0x0000000000040000))
+
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS_POS                     (19U)
+#define RGX_FEATURE_FRAG_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK                (IMG_UINT64_C(0x0000000000080000))
+
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER_POS                            (20U)
+#define RGX_FEATURE_GEOMETRY_BIF_ARBITER_BIT_MASK                       (IMG_UINT64_C(0x0000000000100000))
+
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS_POS                     (21U)
+#define RGX_FEATURE_GEOM_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK                (IMG_UINT64_C(0x0000000000200000))
+
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_POS                           (22U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_BIT_MASK                      (IMG_UINT64_C(0x0000000000400000))
+
+#define RGX_FEATURE_GPU_VIRTUALISATION_POS                              (23U)
+#define RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK                         (IMG_UINT64_C(0x0000000000800000))
+
+#define RGX_FEATURE_GS_RTA_SUPPORT_POS                                  (24U)
+#define RGX_FEATURE_GS_RTA_SUPPORT_BIT_MASK                             (IMG_UINT64_C(0x0000000001000000))
+
+#define RGX_FEATURE_IDLE_CYCLE_STEALING_POS                             (25U)
+#define RGX_FEATURE_IDLE_CYCLE_STEALING_BIT_MASK                        (IMG_UINT64_C(0x0000000002000000))
+
+#define RGX_FEATURE_META_DMA_POS                                        (26U)
+#define RGX_FEATURE_META_DMA_BIT_MASK                                   (IMG_UINT64_C(0x0000000004000000))
+
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES_POS                 (27U)
+#define RGX_FEATURE_META_REGISTER_UNPACKED_ACCESSES_BIT_MASK            (IMG_UINT64_C(0x0000000008000000))
+
+#define RGX_FEATURE_MH_PARITY_POS                                       (28U)
+#define RGX_FEATURE_MH_PARITY_BIT_MASK                                  (IMG_UINT64_C(0x0000000010000000))
+
+#define RGX_FEATURE_PBE_CHECKSUM_2D_POS                                 (29U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D_BIT_MASK                            (IMG_UINT64_C(0x0000000020000000))
+
+#define RGX_FEATURE_PBVNC_COREID_REG_POS                                (30U)
+#define RGX_FEATURE_PBVNC_COREID_REG_BIT_MASK                           (IMG_UINT64_C(0x0000000040000000))
+
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE_POS           (31U)
+#define RGX_FEATURE_PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE_BIT_MASK      (IMG_UINT64_C(0x0000000080000000))
+
+#define RGX_FEATURE_PDS_TEMPSIZE8_POS                                   (32U)
+#define RGX_FEATURE_PDS_TEMPSIZE8_BIT_MASK                              (IMG_UINT64_C(0x0000000100000000))
+
+#define RGX_FEATURE_PERFBUS_POS                                         (33U)
+#define RGX_FEATURE_PERFBUS_BIT_MASK                                    (IMG_UINT64_C(0x0000000200000000))
+
+#define RGX_FEATURE_PERF_COUNTER_BATCH_POS                              (34U)
+#define RGX_FEATURE_PERF_COUNTER_BATCH_BIT_MASK                         (IMG_UINT64_C(0x0000000400000000))
+
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES_POS                  (35U)
+#define RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES_BIT_MASK             (IMG_UINT64_C(0x0000000800000000))
+
+#define RGX_FEATURE_RISCV_DUAL_LOCKSTEP_POS                             (36U)
+#define RGX_FEATURE_RISCV_DUAL_LOCKSTEP_BIT_MASK                        (IMG_UINT64_C(0x0000001000000000))
+
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_POS                              (37U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_BIT_MASK                         (IMG_UINT64_C(0x0000002000000000))
+
+#define RGX_FEATURE_RT_RAC_PER_SPU_POS                                  (38U)
+#define RGX_FEATURE_RT_RAC_PER_SPU_BIT_MASK                             (IMG_UINT64_C(0x0000004000000000))
+
+#define RGX_FEATURE_S7_CACHE_HIERARCHY_POS                              (39U)
+#define RGX_FEATURE_S7_CACHE_HIERARCHY_BIT_MASK                         (IMG_UINT64_C(0x0000008000000000))
+
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE_POS                           (40U)
+#define RGX_FEATURE_S7_TOP_INFRASTRUCTURE_BIT_MASK                      (IMG_UINT64_C(0x0000010000000000))
+
+#define RGX_FEATURE_SAFETY_IRQ_POS                                      (41U)
+#define RGX_FEATURE_SAFETY_IRQ_BIT_MASK                                 (IMG_UINT64_C(0x0000020000000000))
+
+#define RGX_FEATURE_SAFETY_SELF_TEST_POS                                (42U)
+#define RGX_FEATURE_SAFETY_SELF_TEST_BIT_MASK                           (IMG_UINT64_C(0x0000040000000000))
+
+#define RGX_FEATURE_SCALABLE_VDM_GPP_POS                                (43U)
+#define RGX_FEATURE_SCALABLE_VDM_GPP_BIT_MASK                           (IMG_UINT64_C(0x0000080000000000))
+
+#define RGX_FEATURE_SIGNAL_SNOOPING_POS                                 (44U)
+#define RGX_FEATURE_SIGNAL_SNOOPING_BIT_MASK                            (IMG_UINT64_C(0x0000100000000000))
+
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT_POS                             (45U)
+#define RGX_FEATURE_SLC_SIZE_ADJUSTMENT_BIT_MASK                        (IMG_UINT64_C(0x0000200000000000))
+
+#define RGX_FEATURE_SLC_VIVT_POS                                        (46U)
+#define RGX_FEATURE_SLC_VIVT_BIT_MASK                                   (IMG_UINT64_C(0x0000400000000000))
+
+#define RGX_FEATURE_SOC_TIMER_POS                                       (47U)
+#define RGX_FEATURE_SOC_TIMER_BIT_MASK                                  (IMG_UINT64_C(0x0000800000000000))
+
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING_POS                           (48U)
+#define RGX_FEATURE_SPU_ARCH_CLOCK_GATING_BIT_MASK                      (IMG_UINT64_C(0x0001000000000000))
+
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_POS                            (49U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_BIT_MASK                       (IMG_UINT64_C(0x0002000000000000))
+
+#define RGX_FEATURE_SYS_PARITY_POS                                      (50U)
+#define RGX_FEATURE_SYS_PARITY_BIT_MASK                                 (IMG_UINT64_C(0x0004000000000000))
+
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_POS                                (51U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_BIT_MASK                           (IMG_UINT64_C(0x0008000000000000))
+
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS_POS                      (52U)
+#define RGX_FEATURE_TDM_SLC_MMU_AUTO_CACHE_OPS_BIT_MASK                 (IMG_UINT64_C(0x0010000000000000))
+
+#define RGX_FEATURE_TESSELLATION_POS                                    (53U)
+#define RGX_FEATURE_TESSELLATION_BIT_MASK                               (IMG_UINT64_C(0x0020000000000000))
+
+#define RGX_FEATURE_TILE_REGION_PROTECTION_POS                          (54U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION_BIT_MASK                     (IMG_UINT64_C(0x0040000000000000))
+
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_POS             (55U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_BIT_MASK        (IMG_UINT64_C(0x0080000000000000))
+
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_POS                         (56U)
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_BIT_MASK                    (IMG_UINT64_C(0x0100000000000000))
+
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE_POS           (57U)
+#define RGX_FEATURE_USC_INSTRUCTION_CACHE_AUTO_INVALIDATE_BIT_MASK      (IMG_UINT64_C(0x0200000000000000))
+
+#define RGX_FEATURE_USC_TIMER_POS                                       (58U)
+#define RGX_FEATURE_USC_TIMER_BIT_MASK                                  (IMG_UINT64_C(0x0400000000000000))
+
+#define RGX_FEATURE_VDM_DRAWINDIRECT_POS                                (59U)
+#define RGX_FEATURE_VDM_DRAWINDIRECT_BIT_MASK                           (IMG_UINT64_C(0x0800000000000000))
+
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS_POS                            (60U)
+#define RGX_FEATURE_VDM_OBJECT_LEVEL_LLS_BIT_MASK                       (IMG_UINT64_C(0x1000000000000000))
+
+#define RGX_FEATURE_WATCHDOG_TIMER_POS                                  (61U)
+#define RGX_FEATURE_WATCHDOG_TIMER_BIT_MASK                             (IMG_UINT64_C(0x2000000000000000))
+
+#define RGX_FEATURE_WORKGROUP_PROTECTION_POS                            (62U)
+#define RGX_FEATURE_WORKGROUP_PROTECTION_BIT_MASK                       (IMG_UINT64_C(0x4000000000000000))
+
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP_POS                        (63U)
+#define RGX_FEATURE_WORKGROUP_PROTECTION_SMP_BIT_MASK                   (IMG_UINT64_C(0x8000000000000000))
+
+#define RGX_FEATURE_ZLS_CHECKSUM_POS                                    (64U)
+#define RGX_FEATURE_ZLS_CHECKSUM_BIT_MASK                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/******************************************************************************
+ * Defines for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX             (3U)
+#define RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_MAX_VALUE_IDX               (3U)
+#define RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX                              (3U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX                  (4U)
+#define RGX_FEATURE_FBCDC_MAX_VALUE_IDX                                 (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX                       (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX                    (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX           (3U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX             (3U)
+#define RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX                 (5U)
+#define RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX                           (2U)
+#define RGX_FEATURE_MAX_TPU_PER_SPU_MAX_VALUE_IDX                       (4U)
+#define RGX_FEATURE_META_MAX_VALUE_IDX                                  (2U)
+#define RGX_FEATURE_META_COREMEM_BANKS_MAX_VALUE_IDX                    (2U)
+#define RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX                     (2U)
+#define RGX_FEATURE_META_DMA_CHANNEL_COUNT_MAX_VALUE_IDX                (3U)
+#define RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX                          (9U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX                     (9U)
+#define RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX                       (4U)
+#define RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX                            (5U)
+#define RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX                             (2U)
+#define RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX                               (5U)
+#define RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX                        (2U)
+#define RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX         (4U)
+#define RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX                  (7U)
+#define RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_RENDER_TARGET_XY_MAX_MAX_VALUE_IDX                  (3U)
+#define RGX_FEATURE_SCALABLE_TE_ARCH_MAX_VALUE_IDX                      (4U)
+#define RGX_FEATURE_SCALABLE_VCE_MAX_VALUE_IDX                          (5U)
+#define RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX                             (6U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX              (2U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX                 (7U)
+#define RGX_FEATURE_SPU0_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_SPU1_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_SPU2_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_SPU3_RAC_PRESENT_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX                          (4U)
+#define RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX             (4U)
+#define RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX                           (2U)
+#define RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX                           (2U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX            (2U)
+
+/******************************************************************************
+ * Features with values indexes
+ *****************************************************************************/
+
+typedef enum _RGX_FEATURE_WITH_VALUE_INDEX_ {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_IDX,
+	RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_IDX,
+	RGX_FEATURE_ECC_RAMS_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_IDX,
+	RGX_FEATURE_FBCDC_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_IDX,
+	RGX_FEATURE_HOST_SECURITY_VERSION_IDX,
+	RGX_FEATURE_LAYOUT_MARS_IDX,
+	RGX_FEATURE_MAX_TPU_PER_SPU_IDX,
+	RGX_FEATURE_META_IDX,
+	RGX_FEATURE_META_COREMEM_BANKS_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_IDX,
+	RGX_FEATURE_META_DMA_CHANNEL_COUNT_IDX,
+	RGX_FEATURE_MMU_VERSION_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX,
+	RGX_FEATURE_NUM_ISP_PER_SPU_IDX,
+	RGX_FEATURE_NUM_MEMBUS_IDX,
+	RGX_FEATURE_NUM_OSIDS_IDX,
+	RGX_FEATURE_NUM_SPU_IDX,
+	RGX_FEATURE_PBE_PER_SPU_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_IDX,
+	RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_IDX,
+	RGX_FEATURE_POWER_ISLAND_VERSION_IDX,
+	RGX_FEATURE_RAY_TRACING_ARCH_IDX,
+	RGX_FEATURE_RENDER_TARGET_XY_MAX_IDX,
+	RGX_FEATURE_SCALABLE_TE_ARCH_IDX,
+	RGX_FEATURE_SCALABLE_VCE_IDX,
+	RGX_FEATURE_SLC_BANKS_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_IDX,
+	RGX_FEATURE_SPU0_RAC_PRESENT_IDX,
+	RGX_FEATURE_SPU1_RAC_PRESENT_IDX,
+	RGX_FEATURE_SPU2_RAC_PRESENT_IDX,
+	RGX_FEATURE_SPU3_RAC_PRESENT_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_IDX,
+	RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_IDX,
+	RGX_FEATURE_TILE_SIZE_X_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_IDX,
+	RGX_FEATURE_WITH_VALUES_MAX_IDX,
+} RGX_FEATURE_WITH_VALUE_INDEX;
+
+
+/******************************************************************************
+ * Mask and bit-position macros for ERNs and BRNs
+ *****************************************************************************/
+
+#define HW_ERN_65104_POS                                                (0U)
+#define HW_ERN_65104_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000001))
+
+#define HW_ERN_69700_POS                                                (1U)
+#define HW_ERN_69700_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000002))
+
+#define FIX_HW_BRN_71157_POS                                            (2U)
+#define FIX_HW_BRN_71157_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000004))
+
+#define FIX_HW_BRN_71422_POS                                            (3U)
+#define FIX_HW_BRN_71422_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000008))
+
+#define FIX_HW_BRN_71960_POS                                            (4U)
+#define FIX_HW_BRN_71960_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000010))
+
+#define FIX_HW_BRN_72143_POS                                            (5U)
+#define FIX_HW_BRN_72143_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000020))
+
+#define FIX_HW_BRN_72144_POS                                            (6U)
+#define FIX_HW_BRN_72144_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000040))
+
+#define FIX_HW_BRN_74812_POS                                            (7U)
+#define FIX_HW_BRN_74812_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000080))
+
+/* Macro used for padding the unavailable values for features with values */
+#define RGX_FEATURE_VALUE_INVALID	(0xFFFFFFFEU)
+
+/* Macro used for marking a feature with value as disabled for a specific bvnc */
+#define RGX_FEATURE_VALUE_DISABLED	(0xFFFFFFFFU)
+
+#endif /* RGX_BVNC_DEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_table_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_table_km.h
new file mode 100644
index 000000000000..38ca0805c203
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_bvnc_table_km.h
@@ -0,0 +1,530 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_table_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_TABLE_KM_H
+#define RGX_BVNC_TABLE_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+
+#ifndef RGXBVNC_C
+#error "This file should only be included from rgxbvnc.c"
+#endif
+
+#if defined(RGX_BVNC_TABLE_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_table_um.h"
+#endif
+
+
+/******************************************************************************
+ * Arrays for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values[RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 5, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_values[RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_ECC_RAMS_values[RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values[RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_values[RGX_FEATURE_FBCDC_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 5, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ALGORITHM_values[RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 5, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values[RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 6, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 512, 2048, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_HOST_SECURITY_VERSION_values[RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_LAYOUT_MARS_values[RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_MAX_TPU_PER_SPU_values[RGX_FEATURE_MAX_TPU_PER_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_values[RGX_FEATURE_META_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, MTP219, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_COREMEM_BANKS_values[RGX_FEATURE_META_COREMEM_BANKS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_COREMEM_SIZE_values[RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 96, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_DMA_CHANNEL_COUNT_values[RGX_FEATURE_META_DMA_CHANNEL_COUNT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_MMU_VERSION_values[RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_CLUSTERS_values[RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 6, 8, 9, 12, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values[RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 4, 8, 12, 16, 24, 32, 36, 48, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_ISP_PER_SPU_values[RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_MEMBUS_values[RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_OSIDS_values[RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_SPU_values[RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PBE_PER_SPU_values[RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values[RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_values[RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_POWER_ISLAND_VERSION_values[RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 5, 6, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_RAY_TRACING_ARCH_values[RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_RENDER_TARGET_XY_MAX_values[RGX_FEATURE_RENDER_TARGET_XY_MAX_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16384, 32768, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SCALABLE_TE_ARCH_values[RGX_FEATURE_SCALABLE_TE_ARCH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SCALABLE_VCE_values[RGX_FEATURE_SCALABLE_VCE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_BANKS_values[RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, 6, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values[RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1024, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values[RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 128, 256, 512, 1024, 1536, 2048, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU0_RAC_PRESENT_values[RGX_FEATURE_SPU0_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU1_RAC_PRESENT_values[RGX_FEATURE_SPU1_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU2_RAC_PRESENT_values[RGX_FEATURE_SPU2_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SPU3_RAC_PRESENT_values[RGX_FEATURE_SPU3_RAC_PRESENT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TB_GPU_COUNT_values[RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_values[RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_X_values[RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_Y_values[RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values[RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 40, };
+
+
+/******************************************************************************
+ * Table contains pointers to each feature value array for features that have
+ * values.
+ * Indexed using enum RGX_FEATURE_WITH_VALUE_INDEX from rgx_bvnc_defs_km.h
+ *****************************************************************************/
+
+static const void * const gaFeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX] = {
+	aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values,
+	aui16_RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_values,
+	aui16_RGX_FEATURE_ECC_RAMS_values,
+	aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values,
+	aui16_RGX_FEATURE_FBCDC_values,
+	aui16_RGX_FEATURE_FBCDC_ALGORITHM_values,
+	aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_HOST_SECURITY_VERSION_values,
+	aui16_RGX_FEATURE_LAYOUT_MARS_values,
+	aui16_RGX_FEATURE_MAX_TPU_PER_SPU_values,
+	aui16_RGX_FEATURE_META_values,
+	aui16_RGX_FEATURE_META_COREMEM_BANKS_values,
+	aui16_RGX_FEATURE_META_COREMEM_SIZE_values,
+	aui16_RGX_FEATURE_META_DMA_CHANNEL_COUNT_values,
+	aui16_RGX_FEATURE_MMU_VERSION_values,
+	aui16_RGX_FEATURE_NUM_CLUSTERS_values,
+	aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values,
+	aui16_RGX_FEATURE_NUM_ISP_PER_SPU_values,
+	aui16_RGX_FEATURE_NUM_MEMBUS_values,
+	aui16_RGX_FEATURE_NUM_OSIDS_values,
+	aui16_RGX_FEATURE_NUM_SPU_values,
+	aui16_RGX_FEATURE_PBE_PER_SPU_values,
+	aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values,
+	aui16_RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_values,
+	aui16_RGX_FEATURE_POWER_ISLAND_VERSION_values,
+	aui16_RGX_FEATURE_RAY_TRACING_ARCH_values,
+	aui16_RGX_FEATURE_RENDER_TARGET_XY_MAX_values,
+	aui16_RGX_FEATURE_SCALABLE_TE_ARCH_values,
+	aui16_RGX_FEATURE_SCALABLE_VCE_values,
+	aui16_RGX_FEATURE_SLC_BANKS_values,
+	aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values,
+	aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values,
+	aui16_RGX_FEATURE_SPU0_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_SPU1_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_SPU2_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_SPU3_RAC_PRESENT_values,
+	aui16_RGX_FEATURE_TB_GPU_COUNT_values,
+	aui16_RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_values,
+	aui16_RGX_FEATURE_TILE_SIZE_X_values,
+	aui16_RGX_FEATURE_TILE_SIZE_Y_values,
+	aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values,
+};
+
+
+/******************************************************************************
+ * Array containing the lengths of the arrays containing the values.
+ * Used for indexing the aui16_<FEATURE>_values defined upwards
+ *****************************************************************************/
+
+
+static const IMG_UINT16 gaFeaturesValuesMaxIndexes[] = {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX,
+	RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_MAX_VALUE_IDX,
+	RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX,
+	RGX_FEATURE_MAX_TPU_PER_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_META_MAX_VALUE_IDX,
+	RGX_FEATURE_META_COREMEM_BANKS_MAX_VALUE_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX,
+	RGX_FEATURE_META_DMA_CHANNEL_COUNT_MAX_VALUE_IDX,
+	RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX,
+	RGX_FEATURE_RENDER_TARGET_XY_MAX_MAX_VALUE_IDX,
+	RGX_FEATURE_SCALABLE_TE_ARCH_MAX_VALUE_IDX,
+	RGX_FEATURE_SCALABLE_VCE_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU0_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU1_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU2_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_SPU3_RAC_PRESENT_MAX_VALUE_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX,
+	RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX,
+};
+
+#define RGX_FEATURE_VALUE_TYPE_UINT16 (0x0000U)
+#define RGX_FEATURE_VALUE_TYPE_UINT32 (0x8000U)
+#define RGX_FEATURE_TYPE_BIT_SHIFT 14
+
+/******************************************************************************
+ * Bit-positions for features with values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16FeaturesWithValuesBitPositions[] = {
+	(0U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_POS */
+	(2U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_POS */
+	(4U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_ECC_RAMS_POS */
+	(6U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FAULT_DECODE_VERSION_POS */
+	(9U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_POS */
+	(11U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ALGORITHM_POS */
+	(13U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ARCHITECTURE_POS */
+	(15U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_POS */
+	(17U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_POS */
+	(19U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_HOST_SECURITY_VERSION_POS */
+	(22U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_LAYOUT_MARS_POS */
+	(24U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_MAX_TPU_PER_SPU_POS */
+	(27U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_POS */
+	(29U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_COREMEM_BANKS_POS */
+	(31U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_COREMEM_SIZE_POS */
+	(33U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_DMA_CHANNEL_COUNT_POS */
+	(35U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_MMU_VERSION_POS */
+	(37U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_CLUSTERS_POS */
+	(41U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_ISP_IPP_PIPES_POS */
+	(45U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_ISP_PER_SPU_POS */
+	(48U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_MEMBUS_POS */
+	(51U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_OSIDS_POS */
+	(53U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_SPU_POS */
+	(56U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PBE_PER_SPU_POS */
+	(58U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PHYS_BUS_WIDTH_POS */
+	(60U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_POS */
+	(64U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_POWER_ISLAND_VERSION_POS */
+	(67U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_RAY_TRACING_ARCH_POS */
+	(69U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_RENDER_TARGET_XY_MAX_POS */
+	(71U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SCALABLE_TE_ARCH_POS */
+	(74U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SCALABLE_VCE_POS */
+	(77U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_BANKS_POS */
+	(80U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_POS */
+	(82U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_POS */
+	(85U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU0_RAC_PRESENT_POS */
+	(87U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU1_RAC_PRESENT_POS */
+	(89U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU2_RAC_PRESENT_POS */
+	(91U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SPU3_RAC_PRESENT_POS */
+	(93U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TB_GPU_COUNT_POS */
+	(96U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_POS */
+	(99U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_X_POS */
+	(101U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_Y_POS */
+	(103U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_POS */
+};
+
+
+/******************************************************************************
+ * Bit-masks for features with values
+ *****************************************************************************/
+
+static const IMG_UINT64 aui64FeaturesWithValuesBitMasks[] = {
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_CONTEXT_SWITCH_3D_LEVEL_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_ECC_RAMS_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000001C0)), /* RGX_FEATURE_FAULT_DECODE_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000600)), /* RGX_FEATURE_FBCDC_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000001800)), /* RGX_FEATURE_FBCDC_ALGORITHM_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000006000)), /* RGX_FEATURE_FBCDC_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000018000)), /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000060000)), /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000380000)), /* RGX_FEATURE_HOST_SECURITY_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000C00000)), /* RGX_FEATURE_LAYOUT_MARS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000007000000)), /* RGX_FEATURE_MAX_TPU_PER_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0000000018000000)), /* RGX_FEATURE_META_BIT_MASK */
+	(IMG_UINT64_C(0x0000000060000000)), /* RGX_FEATURE_META_COREMEM_BANKS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000180000000)), /* RGX_FEATURE_META_COREMEM_SIZE_BIT_MASK */
+	(IMG_UINT64_C(0x0000000600000000)), /* RGX_FEATURE_META_DMA_CHANNEL_COUNT_BIT_MASK */
+	(IMG_UINT64_C(0x0000001800000000)), /* RGX_FEATURE_MMU_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x000001E000000000)), /* RGX_FEATURE_NUM_CLUSTERS_BIT_MASK */
+	(IMG_UINT64_C(0x00001E0000000000)), /* RGX_FEATURE_NUM_ISP_IPP_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000E00000000000)), /* RGX_FEATURE_NUM_ISP_PER_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0007000000000000)), /* RGX_FEATURE_NUM_MEMBUS_BIT_MASK */
+	(IMG_UINT64_C(0x0018000000000000)), /* RGX_FEATURE_NUM_OSIDS_BIT_MASK */
+	(IMG_UINT64_C(0x00E0000000000000)), /* RGX_FEATURE_NUM_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0300000000000000)), /* RGX_FEATURE_PBE_PER_SPU_BIT_MASK */
+	(IMG_UINT64_C(0x0C00000000000000)), /* RGX_FEATURE_PHYS_BUS_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x7000000000000000)), /* RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000007)), /* RGX_FEATURE_POWER_ISLAND_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000018)), /* RGX_FEATURE_RAY_TRACING_ARCH_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000060)), /* RGX_FEATURE_RENDER_TARGET_XY_MAX_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000380)), /* RGX_FEATURE_SCALABLE_TE_ARCH_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000001C00)), /* RGX_FEATURE_SCALABLE_VCE_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000E000)), /* RGX_FEATURE_SLC_BANKS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000030000)), /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x00000000001C0000)), /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000600000)), /* RGX_FEATURE_SPU0_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000001800000)), /* RGX_FEATURE_SPU1_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000006000000)), /* RGX_FEATURE_SPU2_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000018000000)), /* RGX_FEATURE_SPU3_RAC_PRESENT_BIT_MASK */
+	(IMG_UINT64_C(0x00000000E0000000)), /* RGX_FEATURE_TB_GPU_COUNT_BIT_MASK */
+	(IMG_UINT64_C(0x0000000700000000)), /* RGX_FEATURE_TDM_CONTROL_STREAM_FORMAT_BIT_MASK */
+	(IMG_UINT64_C(0x0000001800000000)), /* RGX_FEATURE_TILE_SIZE_X_BIT_MASK */
+	(IMG_UINT64_C(0x0000006000000000)), /* RGX_FEATURE_TILE_SIZE_Y_BIT_MASK */
+	(IMG_UINT64_C(0x0000018000000000)), /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_BIT_MASK */
+};
+
+
+/******************************************************************************
+ * Table mapping bitmasks for features and features with values
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaFeatures[][4]=
+{
+	{ IMG_UINT64_C(0x001b000000fe0002), IMG_UINT64_C(0x1daad98f65952dca), IMG_UINT64_C(0x1629424ca94aaa55), IMG_UINT64_C(0x000000a92aa524a9) },	/* 27.0.254.2 */
+	{ IMG_UINT64_C(0x001e000001980065), IMG_UINT64_C(0x1fbaf98fedbd3dcb), IMG_UINT64_C(0x15292234a94aaa59), IMG_UINT64_C(0x000000a92aa524a9) },	/* 30.0.408.101 */
+	{ IMG_UINT64_C(0x001e000003300014), IMG_UINT64_C(0x1fbaf98fedbd3dcb), IMG_UINT64_C(0x15294454aa4aaa59), IMG_UINT64_C(0x000000a92aa944a9) },	/* 30.0.816.20 */
+	{ IMG_UINT64_C(0x0023000001980017), IMG_UINT64_C(0x1fbaf98fedfd3dcf), IMG_UINT64_C(0x15292234a94d2a59), IMG_UINT64_C(0x000000aa6aa524ca) },	/* 35.0.408.23 */
+	{ IMG_UINT64_C(0x0023000001980021), IMG_UINT64_C(0xfffaf9afedfd3dcf), IMG_UINT64_C(0x15292232a14d2aa9), IMG_UINT64_C(0x000000aa2aa524ca) },	/* 35.0.408.33 */
+	{ IMG_UINT64_C(0x0023000001980022), IMG_UINT64_C(0xfffaf9afedfd3dcf), IMG_UINT64_C(0x15292232a14d2aa9), IMG_UINT64_C(0x000000aa4aa524ca) },	/* 35.0.408.34 */
+	{ IMG_UINT64_C(0x0023000001980065), IMG_UINT64_C(0x1fbaf98fedbd3dcb), IMG_UINT64_C(0x15292234a94aaa59), IMG_UINT64_C(0x000000a92aa524a9) },	/* 35.0.408.101 */
+	{ IMG_UINT64_C(0x0023000006600015), IMG_UINT64_C(0x1fbaf98fedfd3dcf), IMG_UINT64_C(0x154a4894aa4d2a59), IMG_UINT64_C(0x000000aa4aad694a) },	/* 35.0.1632.21 */
+	{ IMG_UINT64_C(0x0023000006600017), IMG_UINT64_C(0x1fbaf98fedfd3dcf), IMG_UINT64_C(0x154a4894aa4d2a59), IMG_UINT64_C(0x000000aa6aad694a) },	/* 35.0.1632.23 */
+	{ IMG_UINT64_C(0x0023000006600022), IMG_UINT64_C(0x7ffaf9afedfd3dcf), IMG_UINT64_C(0x154a4892a24d2aa9), IMG_UINT64_C(0x000000aa4ab9694a) },	/* 35.0.1632.34 */
+	{ IMG_UINT64_C(0x0023000006600023), IMG_UINT64_C(0xfffaf9afedfd3dcf), IMG_UINT64_C(0x154a4892a24d2aa9), IMG_UINT64_C(0x000000aa6ab9694a) },	/* 35.0.1632.35 */
+	{ IMG_UINT64_C(0x0026000009900192), IMG_UINT64_C(0x1fbbf9cfedfd3ddf), IMG_UINT64_C(0x156c4ab4aa4d2a59), IMG_UINT64_C(0x000000aa2d59ad52) },	/* 38.0.2448.402 */
+	{ IMG_UINT64_C(0x00460000099001a2), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156c4ab4aa553459), IMG_UINT64_C(0x000000aa2d59ad53) },	/* 70.0.2448.418 */
+	{ IMG_UINT64_C(0x0046000009900411), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156b4ab4aa5d3459), IMG_UINT64_C(0x000000aa2ab58d4b) },	/* 70.0.2448.1041 */
+	{ IMG_UINT64_C(0x0046000009900412), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156b4ab4aa5d3459), IMG_UINT64_C(0x000000aa2ab58d4b) },	/* 70.0.2448.1042 */
+	{ IMG_UINT64_C(0x0046000009900550), IMG_UINT64_C(0x1fbbf9cfedfd3dff), IMG_UINT64_C(0x156c4ab4aa5d3459), IMG_UINT64_C(0x000000aa2d59ad53) },	/* 70.0.2448.1360 */
+	{ IMG_UINT64_C(0x00470000099004ba), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x254b6ab2a35d345a), IMG_UINT64_C(0x000000ab2b558954) },	/* 71.0.2448.1210 */
+	{ IMG_UINT64_C(0x00470000099004bb), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x254b6ab2a365345a), IMG_UINT64_C(0x000000ab2b558954) },	/* 71.0.2448.1211 */
+	{ IMG_UINT64_C(0x00470000099004bc), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x254b6ab2a365345a), IMG_UINT64_C(0x000000ab2b558954) },	/* 71.0.2448.1212 */
+	{ IMG_UINT64_C(0x004700000e580820), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x256c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4c) },	/* 71.0.3672.2080 */
+	{ IMG_UINT64_C(0x004700000e580821), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2081 */
+	{ IMG_UINT64_C(0x004700000e580822), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2082 */
+	{ IMG_UINT64_C(0x004700000e580823), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2083 */
+	{ IMG_UINT64_C(0x004700000e580824), IMG_UINT64_C(0x1fbbe9efedfdbdff), IMG_UINT64_C(0x356c6ef2a365345a), IMG_UINT64_C(0x000000ab4d59ad4e) },	/* 71.0.3672.2084 */
+};
+
+/******************************************************************************
+ * Table mapping bitmasks for ERNs/BRNs
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaErnsBrns[][2]=
+{
+	{ IMG_UINT64_C(0x001b000500fe0002), IMG_UINT64_C(0x0000000000000009) },	/* 27.5.254.2 */
+	{ IMG_UINT64_C(0x001b000700fe0002), IMG_UINT64_C(0x0000000000000001) },	/* 27.7.254.2 */
+	{ IMG_UINT64_C(0x001b000800fe0002), IMG_UINT64_C(0x0000000000000001) },	/* 27.8.254.2 */
+	{ IMG_UINT64_C(0x001e000301980065), IMG_UINT64_C(0x0000000000000053) },	/* 30.3.408.101 */
+	{ IMG_UINT64_C(0x001e000303300014), IMG_UINT64_C(0x0000000000000053) },	/* 30.3.816.20 */
+	{ IMG_UINT64_C(0x0023000201980022), IMG_UINT64_C(0x0000000000000053) },	/* 35.2.408.34 */
+	{ IMG_UINT64_C(0x0023000206600015), IMG_UINT64_C(0x0000000000000057) },	/* 35.2.1632.21 */
+	{ IMG_UINT64_C(0x0023000206600017), IMG_UINT64_C(0x0000000000000057) },	/* 35.2.1632.23 */
+	{ IMG_UINT64_C(0x0023000206600022), IMG_UINT64_C(0x0000000000000053) },	/* 35.2.1632.34 */
+	{ IMG_UINT64_C(0x0023000206600023), IMG_UINT64_C(0x0000000000000053) },	/* 35.2.1632.35 */
+	{ IMG_UINT64_C(0x0023000301980021), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.408.33 */
+	{ IMG_UINT64_C(0x0023000301980022), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.408.34 */
+	{ IMG_UINT64_C(0x0023000301980065), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.408.101 */
+	{ IMG_UINT64_C(0x0023000306600017), IMG_UINT64_C(0x0000000000000053) },	/* 35.3.1632.23 */
+	{ IMG_UINT64_C(0x0023000406600017), IMG_UINT64_C(0x0000000000000053) },	/* 35.4.1632.23 */
+	{ IMG_UINT64_C(0x0023000501980017), IMG_UINT64_C(0x0000000000000053) },	/* 35.5.408.23 */
+	{ IMG_UINT64_C(0x0026000409900192), IMG_UINT64_C(0x0000000000000073) },	/* 38.4.2448.402 */
+	{ IMG_UINT64_C(0x0026000609900192), IMG_UINT64_C(0x0000000000000073) },	/* 38.6.2448.402 */
+	{ IMG_UINT64_C(0x00460002099001a2), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.418 */
+	{ IMG_UINT64_C(0x0046000209900411), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.1041 */
+	{ IMG_UINT64_C(0x0046000209900412), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.1042 */
+	{ IMG_UINT64_C(0x0046000209900550), IMG_UINT64_C(0x0000000000000003) },	/* 70.2.2448.1360 */
+	{ IMG_UINT64_C(0x0046000309900412), IMG_UINT64_C(0x0000000000000003) },	/* 70.3.2448.1042 */
+	{ IMG_UINT64_C(0x0046000309900550), IMG_UINT64_C(0x0000000000000003) },	/* 70.3.2448.1360 */
+	{ IMG_UINT64_C(0x00470002099004ba), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.2448.1210 */
+	{ IMG_UINT64_C(0x00470002099004bb), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.2448.1211 */
+	{ IMG_UINT64_C(0x00470002099004bc), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.2448.1212 */
+	{ IMG_UINT64_C(0x004700020e580820), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2080 */
+	{ IMG_UINT64_C(0x004700020e580821), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2081 */
+	{ IMG_UINT64_C(0x004700020e580822), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2082 */
+	{ IMG_UINT64_C(0x004700020e580823), IMG_UINT64_C(0x0000000000000083) },	/* 71.2.3672.2083 */
+	{ IMG_UINT64_C(0x004700030e580823), IMG_UINT64_C(0x0000000000000083) },	/* 71.3.3672.2083 */
+	{ IMG_UINT64_C(0x004700030e580824), IMG_UINT64_C(0x0000000000000083) },	/* 71.3.3672.2084 */
+};
+
+#if defined(DEBUG)
+
+#define FEATURE_NO_VALUES_NAMES_MAX_IDX (65U)
+
+static const IMG_CHAR * const gaszFeaturesNoValuesNames[FEATURE_NO_VALUES_NAMES_MAX_IDX] =
+{
+	"ALBIORIX_TOP_INFRASTRUCTURE",
+	"AXI_ACE",
+	"BARREX_TOP_INFRASTRUCTURE",
+	"BINDLESS_IMAGE_AND_TEXTURE_STATE",
+	"CATURIX_TOP_INFRASTRUCTURE",
+	"CATURIX_XTP_TOP_INFRASTRUCTURE",
+	"CLUSTER_GROUPING",
+	"COMPUTE",
+	"COMPUTE_MORTON_CAPABLE",
+	"COMPUTE_ONLY",
+	"COMPUTE_OVERLAP",
+	"COMPUTE_OVERLAP_WITH_BARRIERS",
+	"COMPUTE_SLC_MMU_AUTO_CACHE_OPS",
+	"COREID_PER_OS",
+	"CR_PARITY",
+	"DIVANO_TOP_INFRASTRUCTURE",
+	"DUST_POWER_ISLAND_S7",
+	"ERYX_TOP_INFRASTRUCTURE",
+	"FASTRENDER_DM",
+	"FRAG_SLC_MMU_AUTO_CACHE_OPS",
+	"GEOMETRY_BIF_ARBITER",
+	"GEOM_SLC_MMU_AUTO_CACHE_OPS",
+	"GPU_MULTICORE_SUPPORT",
+	"GPU_VIRTUALISATION",
+	"GS_RTA_SUPPORT",
+	"IDLE_CYCLE_STEALING",
+	"META_DMA",
+	"META_REGISTER_UNPACKED_ACCESSES",
+	"MH_PARITY",
+	"PBE_CHECKSUM_2D",
+	"PBVNC_COREID_REG",
+	"PDS_INSTRUCTION_CACHE_AUTO_INVALIDATE",
+	"PDS_TEMPSIZE8",
+	"PERFBUS",
+	"PERF_COUNTER_BATCH",
+	"PM_BYTE_ALIGNED_BASE_ADDRESSES",
+	"RISCV_DUAL_LOCKSTEP",
+	"RISCV_FW_PROCESSOR",
+	"RT_RAC_PER_SPU",
+	"S7_CACHE_HIERARCHY",
+	"S7_TOP_INFRASTRUCTURE",
+	"SAFETY_IRQ",
+	"SAFETY_SELF_TEST",
+	"SCALABLE_VDM_GPP",
+	"SIGNAL_SNOOPING",
+	"SLC_SIZE_ADJUSTMENT",
+	"SLC_VIVT",
+	"SOC_TIMER",
+	"SPU_ARCH_CLOCK_GATING",
+	"SYS_BUS_SECURE_RESET",
+	"SYS_PARITY",
+	"TDM_PDS_CHECKSUM",
+	"TDM_SLC_MMU_AUTO_CACHE_OPS",
+	"TESSELLATION",
+	"TILE_REGION_PROTECTION",
+	"TPU_CEM_DATAMASTER_GLOBAL_REGISTERS",
+	"TPU_DM_GLOBAL_REGISTERS",
+	"USC_INSTRUCTION_CACHE_AUTO_INVALIDATE",
+	"USC_TIMER",
+	"VDM_DRAWINDIRECT",
+	"VDM_OBJECT_LEVEL_LLS",
+	"WATCHDOG_TIMER",
+	"WORKGROUP_PROTECTION",
+	"WORKGROUP_PROTECTION_SMP",
+	"ZLS_CHECKSUM",
+};
+
+#define ERNSBRNS_IDS_MAX_IDX (8U)
+
+static const IMG_UINT32 gaui64ErnsBrnsIDs[ERNSBRNS_IDS_MAX_IDX] =
+{
+	65104,
+	69700,
+	71157,
+	71422,
+	71960,
+	72143,
+	72144,
+	74812,
+};
+
+#endif /* defined(DEBUG) */
+#endif /* RGX_BVNC_TABLE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_cr_defs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_cr_defs_km.h
new file mode 100644
index 000000000000..a9edab2f32e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgx_cr_defs_km.h
@@ -0,0 +1,10332 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_cr_defs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ */
+
+
+#ifndef RGX_CR_DEFS_KM_H
+#define RGX_CR_DEFS_KM_H
+
+#if !defined(IMG_EXPLICIT_INCLUDE_HWDEFS)
+#error This file may only be included if explicitly defined
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGX_CR_DEFS_KM_REVISION 217
+
+/*
+    Register RGX_CR_USC_INDIRECT
+*/
+#define RGX_CR_USC_INDIRECT                                                         (0x8000U)
+#define RGX_CR_USC_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_USC_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_USC_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_MERCER_INDIRECT
+*/
+#define RGX_CR_MERCER_INDIRECT                                                      (0x8238U)
+#define RGX_CR_MERCER_INDIRECT_MASKFULL                                             (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MERCER_INDIRECT_ADDRESS_SHIFT                                        (0U)
+#define RGX_CR_MERCER_INDIRECT_ADDRESS_CLRMSK                                       (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_PBE_INDIRECT
+*/
+#define RGX_CR_PBE_INDIRECT                                                         (0x83E0U)
+#define RGX_CR_PBE_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_PBE_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_PBE_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_PBE_SHARED_INDIRECT
+*/
+#define RGX_CR_PBE_SHARED_INDIRECT                                                  (0x8388U)
+#define RGX_CR_PBE_SHARED_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_PBE_SHARED_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_PBE_SHARED_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFE0U)
+
+
+/*
+    Register RGX_CR_ISP_INDIRECT
+*/
+#define RGX_CR_ISP_INDIRECT                                                         (0x83F8U)
+#define RGX_CR_ISP_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_ISP_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_ISP_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_TPU_INDIRECT
+*/
+#define RGX_CR_TPU_INDIRECT                                                         (0x83E8U)
+#define RGX_CR_TPU_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_TPU_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_TPU_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_SWIFT_INDIRECT
+*/
+#define RGX_CR_SWIFT_INDIRECT                                                       (0x8308U)
+#define RGX_CR_SWIFT_INDIRECT_MASKFULL                                              (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_SWIFT_INDIRECT_ADDRESS_SHIFT                                         (0U)
+#define RGX_CR_SWIFT_INDIRECT_ADDRESS_CLRMSK                                        (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_TEXAS_INDIRECT
+*/
+#define RGX_CR_TEXAS_INDIRECT                                                       (0x8390U)
+#define RGX_CR_TEXAS_INDIRECT_MASKFULL                                              (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TEXAS_INDIRECT_ADDRESS_SHIFT                                         (0U)
+#define RGX_CR_TEXAS_INDIRECT_ADDRESS_CLRMSK                                        (0xFFFFFFE0U)
+
+
+/*
+    Register RGX_CR_RAC_INDIRECT
+*/
+#define RGX_CR_RAC_INDIRECT                                                         (0x8398U)
+#define RGX_CR_RAC_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_RAC_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_RAC_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_CLK_CTRL0
+*/
+#define RGX_CR_CLK_CTRL0                                                            (0x0000U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__MASKFULL                                      (IMG_UINT64_C(0xFFCF0303FF3F333F))
+#define RGX_CR_CLK_CTRL0_MASKFULL                                                   (IMG_UINT64_C(0xFFCF0303FF3F3303))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_SHIFT                                            (62U)
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_CLRMSK                                           (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_ON                                               (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS_AUTO                                             (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS__PIPEDM_GT0__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS__PIPEDM_GT0__ON                                  (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_TEXAS__PIPEDM_GT0__AUTO                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE_SHIFT                                              (60U)
+#define RGX_CR_CLK_CTRL0_FBCACHE_CLRMSK                                             (IMG_UINT64_C(0xCFFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBCACHE_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE_ON                                                 (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE_AUTO                                               (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE__PIPEDM_GT0__OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE__PIPEDM_GT0__ON                                    (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL0_FBCACHE__PIPEDM_GT0__AUTO                                  (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL0_FBC_SHIFT                                                  (58U)
+#define RGX_CR_CLK_CTRL0_FBC_CLRMSK                                                 (IMG_UINT64_C(0xF3FFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBC_ON                                                     (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL0_FBC_AUTO                                                   (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL0_FBC__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBC__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL0_FBC__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC_SHIFT                                                 (56U)
+#define RGX_CR_CLK_CTRL0_FBDC_CLRMSK                                                (IMG_UINT64_C(0xFCFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBDC_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC_ON                                                    (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC_AUTO                                                  (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC__PIPEDM_GT0__OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC__PIPEDM_GT0__ON                                       (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL0_FBDC__PIPEDM_GT0__AUTO                                     (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL0_FBM_SHIFT                                                  (54U)
+#define RGX_CR_CLK_CTRL0_FBM_CLRMSK                                                 (IMG_UINT64_C(0xFF3FFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_FBM_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBM_ON                                                     (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL0_FBM_AUTO                                                   (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL0_FBM__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_FBM__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL0_FBM__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL0_PBE_SHIFT                                                  (50U)
+#define RGX_CR_CLK_CTRL0_PBE_CLRMSK                                                 (IMG_UINT64_C(0xFFF3FFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_PBE_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PBE_ON                                                     (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL0_PBE_AUTO                                                   (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL0_PBE__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PBE__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL0_PBE__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1_SHIFT                                               (48U)
+#define RGX_CR_CLK_CTRL0_MCU_L1_CLRMSK                                              (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_MCU_L1_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1_ON                                                  (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1_AUTO                                                (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1__PIPEDM_GT0__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1__PIPEDM_GT0__ON                                     (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L1__PIPEDM_GT0__AUTO                                   (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_SHIFT                                                  (40U)
+#define RGX_CR_CLK_CTRL0_BIF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_BIF_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF_ON                                                     (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL0_BIF_AUTO                                                   (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL0_BIF__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_BIF__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL0_BIF__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL0_TAG_SHIFT                                                  (32U)
+#define RGX_CR_CLK_CTRL0_TAG_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFCFFFFFFFF))
+#define RGX_CR_CLK_CTRL0_TAG_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TAG_ON                                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_CTRL0_TAG_AUTO                                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_CTRL0_TAG__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TAG__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_CTRL0_TAG__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_CTRL0_MADD_SHIFT                                                 (30U)
+#define RGX_CR_CLK_CTRL0_MADD_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFF3FFFFFFF))
+#define RGX_CR_CLK_CTRL0_MADD_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MADD_ON                                                    (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_CTRL0_MADD_AUTO                                                  (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_CTRL0_MADD__PIPEDM_GT0__OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MADD__PIPEDM_GT0__ON                                       (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_CTRL0_MADD__PIPEDM_GT0__AUTO                                     (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_CTRL0_TF_SHIFT                                                   (28U)
+#define RGX_CR_CLK_CTRL0_TF_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL0_TF_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TF_ON                                                      (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL0_TF_AUTO                                                    (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL0_TF__PIPEDM_GT0__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TF__PIPEDM_GT0__ON                                         (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL0_TF__PIPEDM_GT0__AUTO                                       (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0_SHIFT                                               (26U)
+#define RGX_CR_CLK_CTRL0_MCU_L0_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL0_MCU_L0_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0_ON                                                  (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0_AUTO                                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0__PIPEDM_GT0__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0__PIPEDM_GT0__ON                                     (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL0_MCU_L0__PIPEDM_GT0__AUTO                                   (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL0_TPU_SHIFT                                                  (24U)
+#define RGX_CR_CLK_CTRL0_TPU_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL0_TPU_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TPU_ON                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL0_TPU_AUTO                                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL0_TPU__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_TPU__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL0_TPU__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL0_USC_SHIFT                                                  (20U)
+#define RGX_CR_CLK_CTRL0_USC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL0_USC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_USC_ON                                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL0_USC_AUTO                                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL0_USC__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_USC__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL0_USC__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_SHIFT                                             (18U)
+#define RGX_CR_CLK_CTRL0_SLC_BANK_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_ON                                                (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK_AUTO                                              (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK__PIPEDM_GT0__OFF                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK__PIPEDM_GT0__ON                                   (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL0_SLC_BANK__PIPEDM_GT0__AUTO                                 (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL0_SLC_SHIFT                                                  (16U)
+#define RGX_CR_CLK_CTRL0_SLC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL0_SLC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC_ON                                                     (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL0_SLC_AUTO                                                   (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL0_SLC__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_SLC__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL0_SLC__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL0_PDS_SHIFT                                                  (12U)
+#define RGX_CR_CLK_CTRL0_PDS_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL0_PDS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PDS_ON                                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL0_PDS_AUTO                                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL0_PDS__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PDS__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL0_PDS__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL0_PM_SHIFT                                                   (8U)
+#define RGX_CR_CLK_CTRL0_PM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL0_PM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PM_ON                                                      (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL0_PM_AUTO                                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL0_PM__PIPEDM_GT0__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_PM__PIPEDM_GT0__ON                                         (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL0_PM__PIPEDM_GT0__AUTO                                       (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_SHIFT                              (4U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_OFF                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_ON                                 (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__TPU2_LCTRL_AUTO                               (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_SHIFT                                  (2U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_ON                                     (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_BE_AUTO                                   (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL0_ISP_SHIFT                                                  (0U)
+#define RGX_CR_CLK_CTRL0_ISP_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL0_ISP_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_ISP_ON                                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL0_ISP_AUTO                                                   (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL0_ISP__PIPEDM_GT0__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0_ISP__PIPEDM_GT0__ON                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL0_ISP__PIPEDM_GT0__AUTO                                      (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_SHIFT                                  (0U)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_ON                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ISP_FE_AUTO                                   (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS0
+*/
+#define RGX_CR_CLK_STATUS0                                                          (0x0008U)
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__MASKFULL                                    (IMG_UINT64_C(0x00000001BF107F53))
+#define RGX_CR_CLK_STATUS0_MASKFULL                                                 (IMG_UINT64_C(0x00000001BF107F51))
+#define RGX_CR_CLK_STATUS0_MCU_L0_SHIFT                                             (32U)
+#define RGX_CR_CLK_STATUS0_MCU_L0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CLK_STATUS0_MCU_L0_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L0_RUNNING                                           (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS0_MCU_L0__PIPEDM_GT0__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L0__PIPEDM_GT0__RUNNING                              (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_SHIFT                                          (31U)
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS_RUNNING                                        (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS__PIPEDM_GT0__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF_TEXAS__PIPEDM_GT0__RUNNING                           (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE_SHIFT                                            (29U)
+#define RGX_CR_CLK_STATUS0_FBCACHE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_CLK_STATUS0_FBCACHE_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE_RUNNING                                          (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE__PIPEDM_GT0__GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBCACHE__PIPEDM_GT0__RUNNING                             (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS0_FBC_SHIFT                                                (28U)
+#define RGX_CR_CLK_STATUS0_FBC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_CLK_STATUS0_FBC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBC_RUNNING                                              (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS0_FBC__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBC__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS0_FBDC_SHIFT                                               (27U)
+#define RGX_CR_CLK_STATUS0_FBDC_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_CLK_STATUS0_FBDC_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBDC_RUNNING                                             (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS0_FBDC__PIPEDM_GT0__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBDC__PIPEDM_GT0__RUNNING                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS0_FBM_SHIFT                                                (26U)
+#define RGX_CR_CLK_STATUS0_FBM_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_CLK_STATUS0_FBM_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBM_RUNNING                                              (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS0_FBM__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_FBM__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS0_PBE_SHIFT                                                (25U)
+#define RGX_CR_CLK_STATUS0_PBE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_CLK_STATUS0_PBE_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PBE_RUNNING                                              (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS0_PBE__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PBE__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1_SHIFT                                             (24U)
+#define RGX_CR_CLK_STATUS0_MCU_L1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_CLK_STATUS0_MCU_L1_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1_RUNNING                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1__PIPEDM_GT0__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MCU_L1__PIPEDM_GT0__RUNNING                              (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS0_BIF_SHIFT                                                (20U)
+#define RGX_CR_CLK_STATUS0_BIF_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_CLK_STATUS0_BIF_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF_RUNNING                                              (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS0_BIF__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_BIF__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS0_TF_SHIFT                                                 (14U)
+#define RGX_CR_CLK_STATUS0_TF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_CLK_STATUS0_TF_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TF_RUNNING                                               (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS0_TF__PIPEDM_GT0__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TF__PIPEDM_GT0__RUNNING                                  (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS0_MADD_SHIFT                                               (13U)
+#define RGX_CR_CLK_STATUS0_MADD_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_CLK_STATUS0_MADD_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MADD_RUNNING                                             (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS0_MADD__PIPEDM_GT0__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_MADD__PIPEDM_GT0__RUNNING                                (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS0_TPU_SHIFT                                                (12U)
+#define RGX_CR_CLK_STATUS0_TPU_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS0_TPU_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TPU_RUNNING                                              (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS0_TPU__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TPU__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS0_TAG_SHIFT                                                (11U)
+#define RGX_CR_CLK_STATUS0_TAG_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_CLK_STATUS0_TAG_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TAG_RUNNING                                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_STATUS0_TAG__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_TAG__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_STATUS0_USC_SHIFT                                                (10U)
+#define RGX_CR_CLK_STATUS0_USC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS0_USC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_USC_RUNNING                                              (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS0_USC__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_USC__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS0_SLC_BANK_SHIFT                                           (9U)
+#define RGX_CR_CLK_STATUS0_SLC_BANK_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS0_SLC_BANK_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC_BANK_RUNNING                                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS0_SLC_BANK__PIPEDM_GT0__GATED                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC_BANK__PIPEDM_GT0__RUNNING                            (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS0_SLC_SHIFT                                                (8U)
+#define RGX_CR_CLK_STATUS0_SLC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS0_SLC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC_RUNNING                                              (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS0_SLC__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_SLC__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS0_PDS_SHIFT                                                (6U)
+#define RGX_CR_CLK_STATUS0_PDS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS0_PDS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PDS_RUNNING                                              (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS0_PDS__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PDS__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS0_PM_SHIFT                                                 (4U)
+#define RGX_CR_CLK_STATUS0_PM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS0_PM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PM_RUNNING                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS0_PM__PIPEDM_GT0__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_PM__PIPEDM_GT0__RUNNING                                  (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_SHIFT                                (1U)
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_BE_RUNNING                              (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS0_ISP_SHIFT                                                (0U)
+#define RGX_CR_CLK_STATUS0_ISP_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS0_ISP_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_ISP_RUNNING                                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS0_ISP__PIPEDM_GT0__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0_ISP__PIPEDM_GT0__RUNNING                                 (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_SHIFT                                (0U)
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS0__PIPEDM_GT0__ISP_FE_RUNNING                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID                                                              (0x0020U)
+#define RGX_CR_CORE_ID_MASKFULL                                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CORE_ID_BRANCH_ID_SHIFT                                              (48U)
+#define RGX_CR_CORE_ID_BRANCH_ID_CLRMSK                                             (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CORE_ID_VERSION_ID_SHIFT                                             (32U)
+#define RGX_CR_CORE_ID_VERSION_ID_CLRMSK                                            (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_SHIFT                               (16U)
+#define RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_CORE_ID_CONFIG_ID_SHIFT                                              (0U)
+#define RGX_CR_CORE_ID_CONFIG_ID_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SPU_ENABLE
+*/
+#define RGX_CR_SPU_ENABLE                                                           (0x0050U)
+#define RGX_CR_SPU_ENABLE_MASKFULL                                                  (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SPU_ENABLE_ENABLE_SHIFT                                              (0U)
+#define RGX_CR_SPU_ENABLE_ENABLE_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_SOC_TIMER_GRAY
+*/
+#define RGX_CR_SOC_TIMER_GRAY                                                       (0x00E0U)
+#define RGX_CR_SOC_TIMER_GRAY_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_SHIFT                                           (0U)
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_CLRMSK                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_BINARY
+*/
+#define RGX_CR_SOC_TIMER_BINARY                                                     (0x00E8U)
+#define RGX_CR_SOC_TIMER_BINARY_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_CLRMSK                                        (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_CLK_CTRL1
+*/
+#define RGX_CR_CLK_CTRL1                                                            (0x0080U)
+#define RGX_CR_CLK_CTRL1_MASKFULL                                                   (IMG_UINT64_C(0xFFFC3F3FFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_BSC_SHIFT                                                  (62U)
+#define RGX_CR_CLK_CTRL1_BSC_CLRMSK                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_BSC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_BSC_ON                                                     (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL1_BSC_AUTO                                                   (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_SHIFT                                         (60U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_CLRMSK                                        (IMG_UINT64_C(0xCFFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_ON                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SAP_AUTO                                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_SHIFT                                     (58U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_CLRMSK                                    (IMG_UINT64_C(0xF3FFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_ON                                        (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_PAP_CMN_AUTO                                      (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_SHIFT                                         (56U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_CLRMSK                                        (IMG_UINT64_C(0xFCFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_ON                                            (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TPX_AUTO                                          (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_SHIFT                                      (54U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_CLRMSK                                     (IMG_UINT64_C(0xFF3FFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_ON                                         (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMIPSB_AUTO                                       (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL1_PSB_SHIFT                                                  (52U)
+#define RGX_CR_CLK_CTRL1_PSB_CLRMSK                                                 (IMG_UINT64_C(0xFFCFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_PSB_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_PSB_ON                                                     (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL1_PSB_AUTO                                                   (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_SHIFT                                       (50U)
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_CLRMSK                                      (IMG_UINT64_C(0xFFF3FFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_ON                                          (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL1_TPU_USC_SELECT_AUTO                                        (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_SHIFT                                             (44U)
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_CLRMSK                                            (IMG_UINT64_C(0xFFFFCFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_ON                                                (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL1_CDM_PIPE_AUTO                                              (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_SHIFT                                         (42U)
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_CLRMSK                                        (IMG_UINT64_C(0xFFFFF3FFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_ON                                            (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL1_USC_L2ICACHE_AUTO                                          (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL1_TCU_L1_SHIFT                                               (40U)
+#define RGX_CR_CLK_CTRL1_TCU_L1_CLRMSK                                              (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_TCU_L1_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TCU_L1_ON                                                  (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL1_TCU_L1_AUTO                                                (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL1_TDM_SHIFT                                                  (36U)
+#define RGX_CR_CLK_CTRL1_TDM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_TDM_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TDM_ON                                                     (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_CLK_CTRL1_TDM_AUTO                                                   (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_CLK_CTRL1_ASTC_SHIFT                                                 (34U)
+#define RGX_CR_CLK_CTRL1_ASTC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFF3FFFFFFFF))
+#define RGX_CR_CLK_CTRL1_ASTC_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_ASTC_ON                                                    (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_CLK_CTRL1_ASTC_AUTO                                                  (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_CLK_CTRL1_IPF_SHIFT                                                  (32U)
+#define RGX_CR_CLK_CTRL1_IPF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFCFFFFFFFF))
+#define RGX_CR_CLK_CTRL1_IPF_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_IPF_ON                                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_CTRL1_IPF_AUTO                                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_CTRL1_COMPUTE_SHIFT                                              (30U)
+#define RGX_CR_CLK_CTRL1_COMPUTE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFF3FFFFFFF))
+#define RGX_CR_CLK_CTRL1_COMPUTE_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_COMPUTE_ON                                                 (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_CTRL1_COMPUTE_AUTO                                               (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_CTRL1_PIXEL_SHIFT                                                (28U)
+#define RGX_CR_CLK_CTRL1_PIXEL_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL1_PIXEL_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_PIXEL_ON                                                   (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL1_PIXEL_AUTO                                                 (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL1_VERTEX_SHIFT                                               (26U)
+#define RGX_CR_CLK_CTRL1_VERTEX_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL1_VERTEX_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_VERTEX_ON                                                  (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL1_VERTEX_AUTO                                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL1_TPF_SHIFT                                                  (24U)
+#define RGX_CR_CLK_CTRL1_TPF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL1_TPF_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_TPF_ON                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL1_TPF_AUTO                                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_SHIFT                                           (22U)
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFF3FFFFF))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_OFF                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_ON                                              (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_CTRL1_GEO_VERTEX_AUTO                                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_SHIFT                                         (20U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_ON                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_LMS_AUTO                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_SHIFT                                           (18U)
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_OFF                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_ON                                              (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL1_GEO_SHARED_AUTO                                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_SHIFT                                             (16U)
+#define RGX_CR_CLK_CTRL1_GEO_TESS_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_ON                                                (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL1_GEO_TESS_AUTO                                              (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_SHIFT                                         (14U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_ON                                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_SMP_AUTO                                          (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_SHIFT                                         (12U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_ON                                            (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_DMA_AUTO                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_SHIFT                                         (10U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_ON                                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_TAP_AUTO                                          (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_SHIFT                                          (8U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_ON                                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_AP_AUTO                                           (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_SHIFT                                         (6U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_ON                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_EMI_AUTO                                          (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_SHIFT                                         (4U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_ON                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_ITR_AUTO                                          (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_SHIFT                                         (2U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_ON                                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_CPX_AUTO                                          (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_SHIFT                                         (0U)
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_ON                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL1_USC_PIPE_MOV_AUTO                                          (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS1
+*/
+#define RGX_CR_CLK_STATUS1                                                          (0x0088U)
+#define RGX_CR_CLK_STATUS1_MASKFULL                                                 (IMG_UINT64_C(0x00000007FFFF77FB))
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_SHIFT                                         (34U)
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TPU2_LCTRL_RUNNING                                       (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_CLK_STATUS1_TILING_SHIFT                                             (33U)
+#define RGX_CR_CLK_STATUS1_TILING_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_CLK_STATUS1_TILING_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TILING_RUNNING                                           (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_STATUS1_GEO_VDM_SHIFT                                            (32U)
+#define RGX_CR_CLK_STATUS1_GEO_VDM_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CLK_STATUS1_GEO_VDM_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_VDM_RUNNING                                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_SHIFT                                       (31U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SAP_RUNNING                                     (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_SHIFT                                   (30U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_PAP_CMN_RUNNING                                 (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_SHIFT                                       (29U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TPX_RUNNING                                     (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_SHIFT                                    (28U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMIPSB_RUNNING                                  (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS1_BSC_SHIFT                                                (27U)
+#define RGX_CR_CLK_STATUS1_BSC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_CLK_STATUS1_BSC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_BSC_RUNNING                                              (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS1_PSB_SHIFT                                                (26U)
+#define RGX_CR_CLK_STATUS1_PSB_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_CLK_STATUS1_PSB_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_PSB_RUNNING                                              (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_SHIFT                                     (25U)
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TPU_USC_SELECT_RUNNING                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_SHIFT                                       (24U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_SMP_RUNNING                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_SHIFT                                       (23U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_DMA_RUNNING                                     (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_SHIFT                                       (22U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_TAP_RUNNING                                     (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_SHIFT                                        (21U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_AP_RUNNING                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_SHIFT                                       (20U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_EMI_RUNNING                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_SHIFT                                       (19U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_ITR_RUNNING                                     (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_SHIFT                                       (18U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_CPX_RUNNING                                     (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_SHIFT                                       (17U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_MOV_RUNNING                                     (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_SHIFT                                       (16U)
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_PIPE_LMS_RUNNING                                     (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_SHIFT                                           (14U)
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_CDM_PIPE_RUNNING                                         (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS1_TCU_L1_SHIFT                                             (13U)
+#define RGX_CR_CLK_STATUS1_TCU_L1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_CLK_STATUS1_TCU_L1_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TCU_L1_RUNNING                                           (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_SHIFT                                       (12U)
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_USC_L2ICACHE_RUNNING                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS1_TDM_SHIFT                                                (10U)
+#define RGX_CR_CLK_STATUS1_TDM_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS1_TDM_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TDM_RUNNING                                              (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS1_IPF_SHIFT                                                (9U)
+#define RGX_CR_CLK_STATUS1_IPF_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS1_IPF_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_IPF_RUNNING                                              (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS1_COMPUTE_SHIFT                                            (8U)
+#define RGX_CR_CLK_STATUS1_COMPUTE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS1_COMPUTE_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_COMPUTE_RUNNING                                          (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS1_ASTC_SHIFT                                               (7U)
+#define RGX_CR_CLK_STATUS1_ASTC_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_STATUS1_ASTC_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_ASTC_RUNNING                                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS1_PIXEL_SHIFT                                              (6U)
+#define RGX_CR_CLK_STATUS1_PIXEL_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS1_PIXEL_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_PIXEL_RUNNING                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS1_VERTEX_SHIFT                                             (5U)
+#define RGX_CR_CLK_STATUS1_VERTEX_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_STATUS1_VERTEX_GATED                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_VERTEX_RUNNING                                           (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS1_TPF_SHIFT                                                (4U)
+#define RGX_CR_CLK_STATUS1_TPF_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS1_TPF_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_TPF_RUNNING                                              (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_SHIFT                                         (3U)
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_SHARED_RUNNING                                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_SHIFT                                         (1U)
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_VERTEX_RUNNING                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS1_GEO_TESS_SHIFT                                           (0U)
+#define RGX_CR_CLK_STATUS1_GEO_TESS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS1_GEO_TESS_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS1_GEO_TESS_RUNNING                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_CLK_CTRL2
+*/
+#define RGX_CR_CLK_CTRL2                                                            (0x0090U)
+#define RGX_CR_CLK_CTRL2_MASKFULL                                                   (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CLK_CTRL2_TILING_SHIFT                                               (28U)
+#define RGX_CR_CLK_CTRL2_TILING_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL2_TILING_OFF                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_TILING_ON                                                  (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL2_TILING_AUTO                                                (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_SHIFT                                              (26U)
+#define RGX_CR_CLK_CTRL2_GEO_VDM_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_ON                                                 (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL2_GEO_VDM_AUTO                                               (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL2_RTS_SHIFT                                                  (24U)
+#define RGX_CR_CLK_CTRL2_RTS_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL2_RTS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RTS_ON                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL2_RTS_AUTO                                                   (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL2_PCG_SHIFT                                                  (22U)
+#define RGX_CR_CLK_CTRL2_PCG_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFF3FFFFF))
+#define RGX_CR_CLK_CTRL2_PCG_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_PCG_ON                                                     (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_CTRL2_PCG_AUTO                                                   (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_CTRL2_BPS_SHIFT                                                  (20U)
+#define RGX_CR_CLK_CTRL2_BPS_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL2_BPS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_BPS_ON                                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL2_BPS_AUTO                                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL2_RS_SHIFT                                                   (18U)
+#define RGX_CR_CLK_CTRL2_RS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL2_RS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RS_ON                                                      (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL2_RS_AUTO                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL2_RRC_SHIFT                                                  (16U)
+#define RGX_CR_CLK_CTRL2_RRC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL2_RRC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RRC_ON                                                     (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL2_RRC_AUTO                                                   (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL2_URI_SHIFT                                                  (14U)
+#define RGX_CR_CLK_CTRL2_URI_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_CLK_CTRL2_URI_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_URI_ON                                                     (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL2_URI_AUTO                                                   (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL2_RCE_SHIFT                                                  (12U)
+#define RGX_CR_CLK_CTRL2_RCE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL2_RCE_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_RCE_ON                                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL2_RCE_AUTO                                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL2_ASC_SHIFT                                                  (10U)
+#define RGX_CR_CLK_CTRL2_ASC_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL2_ASC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_ASC_ON                                                     (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL2_ASC_AUTO                                                   (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL2_FP_SHIFT                                                   (8U)
+#define RGX_CR_CLK_CTRL2_FP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL2_FP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_FP_ON                                                      (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL2_FP_AUTO                                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL2_INT_SHIFT                                                  (6U)
+#define RGX_CR_CLK_CTRL2_INT_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_CLK_CTRL2_INT_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_INT_ON                                                     (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL2_INT_AUTO                                                   (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL2_YUV_SHIFT                                                  (4U)
+#define RGX_CR_CLK_CTRL2_YUV_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL2_YUV_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_YUV_ON                                                     (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL2_YUV_AUTO                                                   (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_SHIFT                                    (2U)
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_ON                                       (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_DOT8_AUTO                                     (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_SHIFT                                     (0U)
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_ON                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL2_USC_PIPE_PAP_FOP_AUTO                                      (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS2
+*/
+#define RGX_CR_CLK_STATUS2                                                          (0x0098U)
+#define RGX_CR_CLK_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x0000000000001FFF))
+#define RGX_CR_CLK_STATUS2_RTS_SHIFT                                                (12U)
+#define RGX_CR_CLK_STATUS2_RTS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS2_RTS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RTS_RUNNING                                              (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS2_PCG_SHIFT                                                (11U)
+#define RGX_CR_CLK_STATUS2_PCG_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_CLK_STATUS2_PCG_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_PCG_RUNNING                                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_STATUS2_BPS_SHIFT                                                (10U)
+#define RGX_CR_CLK_STATUS2_BPS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS2_BPS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_BPS_RUNNING                                              (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS2_RS_SHIFT                                                 (9U)
+#define RGX_CR_CLK_STATUS2_RS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS2_RS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RS_RUNNING                                               (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS2_RRC_SHIFT                                                (8U)
+#define RGX_CR_CLK_STATUS2_RRC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS2_RRC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RRC_RUNNING                                              (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS2_URI_SHIFT                                                (7U)
+#define RGX_CR_CLK_STATUS2_URI_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_STATUS2_URI_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_URI_RUNNING                                              (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS2_RCE_SHIFT                                                (6U)
+#define RGX_CR_CLK_STATUS2_RCE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS2_RCE_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_RCE_RUNNING                                              (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS2_ASC_SHIFT                                                (5U)
+#define RGX_CR_CLK_STATUS2_ASC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_STATUS2_ASC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_ASC_RUNNING                                              (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS2_FP_SHIFT                                                 (4U)
+#define RGX_CR_CLK_STATUS2_FP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS2_FP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_FP_RUNNING                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS2_INT_SHIFT                                                (3U)
+#define RGX_CR_CLK_STATUS2_INT_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_STATUS2_INT_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_INT_RUNNING                                              (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS2_YUV_SHIFT                                                (2U)
+#define RGX_CR_CLK_STATUS2_YUV_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS2_YUV_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_YUV_RUNNING                                              (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_SHIFT                                  (1U)
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_DOT8_RUNNING                                (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_SHIFT                                   (0U)
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_USC_PIPE_PAP_FOP_RUNNING                                 (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET
+*/
+#define RGX_CR_SOFT_RESET                                                           (0x0100U)
+#define RGX_CR_SOFT_RESET_MASKFULL                                                  (IMG_UINT64_C(0x1FFFFFE0007BDEFF))
+#define RGX_CR_SOFT_RESET_TPU2_LCTRL_SHIFT                                          (60U)
+#define RGX_CR_SOFT_RESET_TPU2_LCTRL_CLRMSK                                         (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TPU2_LCTRL_EN                                             (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_SOFT_RESET_TILING_SHIFT                                              (59U)
+#define RGX_CR_SOFT_RESET_TILING_CLRMSK                                             (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TILING_EN                                                 (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SOFT_RESET_GEO_VDM_SHIFT                                             (58U)
+#define RGX_CR_SOFT_RESET_GEO_VDM_CLRMSK                                            (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GEO_VDM_EN                                                (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SOFT_RESET_RAC_SHIFT                                                 (57U)
+#define RGX_CR_SOFT_RESET_RAC_CLRMSK                                                (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_RAC_EN                                                    (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SOFT_RESET_GEO_TESS_SHIFT                                            (56U)
+#define RGX_CR_SOFT_RESET_GEO_TESS_CLRMSK                                           (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GEO_TESS_EN                                               (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SOFT_RESET_INT_SHIFT                                                 (55U)
+#define RGX_CR_SOFT_RESET_INT_CLRMSK                                                (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_INT_EN                                                    (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SOFT_RESET_FP_SHIFT                                                  (54U)
+#define RGX_CR_SOFT_RESET_FP_CLRMSK                                                 (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FP_EN                                                     (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SOFT_RESET_YUV_SHIFT                                                 (53U)
+#define RGX_CR_SOFT_RESET_YUV_CLRMSK                                                (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_YUV_EN                                                    (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SOFT_RESET_PSB_SHIFT                                                 (52U)
+#define RGX_CR_SOFT_RESET_PSB_CLRMSK                                                (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PSB_EN                                                    (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_SOFT_RESET_ASC_SHIFT                                                 (51U)
+#define RGX_CR_SOFT_RESET_ASC_CLRMSK                                                (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_ASC_EN                                                    (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SOFT_RESET_RCE_SHIFT                                                 (50U)
+#define RGX_CR_SOFT_RESET_RCE_CLRMSK                                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_RCE_EN                                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SOFT_RESET_BSC_SHIFT                                                 (49U)
+#define RGX_CR_SOFT_RESET_BSC_CLRMSK                                                (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BSC_EN                                                    (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SOFT_RESET_TPU_USC_SELECT_SHIFT                                      (48U)
+#define RGX_CR_SOFT_RESET_TPU_USC_SELECT_CLRMSK                                     (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TPU_USC_SELECT_EN                                         (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SOFT_RESET_USC_L2ICACHE_SHIFT                                        (47U)
+#define RGX_CR_SOFT_RESET_USC_L2ICACHE_CLRMSK                                       (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_USC_L2ICACHE_EN                                           (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SOFT_RESET_TCU_L1_SHIFT                                              (46U)
+#define RGX_CR_SOFT_RESET_TCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TCU_L1_EN                                                 (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_SHIFT                                           (45U)
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_EN                                              (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SOFT_RESET_BIF_JONES_SHIFT                                           (44U)
+#define RGX_CR_SOFT_RESET_BIF_JONES_CLRMSK                                          (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF_JONES_EN                                              (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SOFT_RESET_SLC_SHIFT                                                 (43U)
+#define RGX_CR_SOFT_RESET_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_SLC_EN                                                    (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SOFT_RESET_FBCACHE_SHIFT                                             (42U)
+#define RGX_CR_SOFT_RESET_FBCACHE_CLRMSK                                            (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBCACHE_EN                                                (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SOFT_RESET_FBM_SHIFT                                                 (41U)
+#define RGX_CR_SOFT_RESET_FBM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBM_EN                                                    (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SOFT_RESET_FBDC_SHIFT                                                (40U)
+#define RGX_CR_SOFT_RESET_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBDC_EN                                                   (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SOFT_RESET_FBC_SHIFT                                                 (39U)
+#define RGX_CR_SOFT_RESET_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBC_EN                                                    (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SOFT_RESET_PM_SHIFT                                                  (38U)
+#define RGX_CR_SOFT_RESET_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PM_EN                                                     (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SOFT_RESET_GARTEN_SHIFT                                              (37U)
+#define RGX_CR_SOFT_RESET_GARTEN_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GARTEN_EN                                                 (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SOFT_RESET_TAG_SHIFT                                                 (22U)
+#define RGX_CR_SOFT_RESET_TAG_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SOFT_RESET_TAG_EN                                                    (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SOFT_RESET_TF_SHIFT                                                  (21U)
+#define RGX_CR_SOFT_RESET_TF_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SOFT_RESET_TF_EN                                                     (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SOFT_RESET_MADD_SHIFT                                                (20U)
+#define RGX_CR_SOFT_RESET_MADD_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SOFT_RESET_MADD_EN                                                   (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SOFT_RESET_PBE_SHIFT                                                 (19U)
+#define RGX_CR_SOFT_RESET_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SOFT_RESET_PBE_EN                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SOFT_RESET_MCU_L1_SHIFT                                              (17U)
+#define RGX_CR_SOFT_RESET_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SOFT_RESET_MCU_L1_EN                                                 (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SOFT_RESET_CDM_PIPE_SHIFT                                            (16U)
+#define RGX_CR_SOFT_RESET_CDM_PIPE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SOFT_RESET_CDM_PIPE_EN                                               (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SOFT_RESET_TDM_SHIFT                                                 (15U)
+#define RGX_CR_SOFT_RESET_TDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SOFT_RESET_TDM_EN                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SOFT_RESET_ASTC_SHIFT                                                (14U)
+#define RGX_CR_SOFT_RESET_ASTC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SOFT_RESET_ASTC_EN                                                   (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SOFT_RESET_PDS_SHIFT                                                 (12U)
+#define RGX_CR_SOFT_RESET_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SOFT_RESET_PDS_EN                                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SOFT_RESET_ISP_SHIFT                                                 (11U)
+#define RGX_CR_SOFT_RESET_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SOFT_RESET_ISP_EN                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SOFT_RESET_TPF_SHIFT                                                 (10U)
+#define RGX_CR_SOFT_RESET_TPF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SOFT_RESET_TPF_EN                                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SOFT_RESET_IPF_SHIFT                                                 (9U)
+#define RGX_CR_SOFT_RESET_IPF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SOFT_RESET_IPF_EN                                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SOFT_RESET_GEO_SHARED_SHIFT                                          (7U)
+#define RGX_CR_SOFT_RESET_GEO_SHARED_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SOFT_RESET_GEO_SHARED_EN                                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SOFT_RESET_GEO_VERTEX_SHIFT                                          (6U)
+#define RGX_CR_SOFT_RESET_GEO_VERTEX_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SOFT_RESET_GEO_VERTEX_EN                                             (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SOFT_RESET_PIXEL_SHIFT                                               (5U)
+#define RGX_CR_SOFT_RESET_PIXEL_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SOFT_RESET_PIXEL_EN                                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SOFT_RESET_COMPUTE_SHIFT                                             (4U)
+#define RGX_CR_SOFT_RESET_COMPUTE_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SOFT_RESET_COMPUTE_EN                                                (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SOFT_RESET_MCU_L0_SHIFT                                              (3U)
+#define RGX_CR_SOFT_RESET_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SOFT_RESET_MCU_L0_EN                                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SOFT_RESET_TPU_SHIFT                                                 (2U)
+#define RGX_CR_SOFT_RESET_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SOFT_RESET_TPU_EN                                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SOFT_RESET_VERTEX_SHIFT                                              (1U)
+#define RGX_CR_SOFT_RESET_VERTEX_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SOFT_RESET_VERTEX_EN                                                 (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SOFT_RESET_USC_SHIFT                                                 (0U)
+#define RGX_CR_SOFT_RESET_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SOFT_RESET_USC_EN                                                    (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET_SPU
+*/
+#define RGX_CR_SOFT_RESET_SPU                                                       (0x0108U)
+#define RGX_CR_SOFT_RESET_SPU_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SOFT_RESET_SPU_SPU31_SHIFT                                           (31U)
+#define RGX_CR_SOFT_RESET_SPU_SPU31_CLRMSK                                          (0x7FFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU31_EN                                              (0x80000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU30_SHIFT                                           (30U)
+#define RGX_CR_SOFT_RESET_SPU_SPU30_CLRMSK                                          (0xBFFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU30_EN                                              (0x40000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU29_SHIFT                                           (29U)
+#define RGX_CR_SOFT_RESET_SPU_SPU29_CLRMSK                                          (0xDFFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU29_EN                                              (0x20000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU28_SHIFT                                           (28U)
+#define RGX_CR_SOFT_RESET_SPU_SPU28_CLRMSK                                          (0xEFFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU28_EN                                              (0x10000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU27_SHIFT                                           (27U)
+#define RGX_CR_SOFT_RESET_SPU_SPU27_CLRMSK                                          (0xF7FFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU27_EN                                              (0x08000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU26_SHIFT                                           (26U)
+#define RGX_CR_SOFT_RESET_SPU_SPU26_CLRMSK                                          (0xFBFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU26_EN                                              (0x04000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU25_SHIFT                                           (25U)
+#define RGX_CR_SOFT_RESET_SPU_SPU25_CLRMSK                                          (0xFDFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU25_EN                                              (0x02000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU24_SHIFT                                           (24U)
+#define RGX_CR_SOFT_RESET_SPU_SPU24_CLRMSK                                          (0xFEFFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU24_EN                                              (0x01000000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU23_SHIFT                                           (23U)
+#define RGX_CR_SOFT_RESET_SPU_SPU23_CLRMSK                                          (0xFF7FFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU23_EN                                              (0x00800000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU22_SHIFT                                           (22U)
+#define RGX_CR_SOFT_RESET_SPU_SPU22_CLRMSK                                          (0xFFBFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU22_EN                                              (0x00400000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU21_SHIFT                                           (21U)
+#define RGX_CR_SOFT_RESET_SPU_SPU21_CLRMSK                                          (0xFFDFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU21_EN                                              (0x00200000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU20_SHIFT                                           (20U)
+#define RGX_CR_SOFT_RESET_SPU_SPU20_CLRMSK                                          (0xFFEFFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU20_EN                                              (0x00100000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU19_SHIFT                                           (19U)
+#define RGX_CR_SOFT_RESET_SPU_SPU19_CLRMSK                                          (0xFFF7FFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU19_EN                                              (0x00080000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU18_SHIFT                                           (18U)
+#define RGX_CR_SOFT_RESET_SPU_SPU18_CLRMSK                                          (0xFFFBFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU18_EN                                              (0x00040000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU17_SHIFT                                           (17U)
+#define RGX_CR_SOFT_RESET_SPU_SPU17_CLRMSK                                          (0xFFFDFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU17_EN                                              (0x00020000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU16_SHIFT                                           (16U)
+#define RGX_CR_SOFT_RESET_SPU_SPU16_CLRMSK                                          (0xFFFEFFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU16_EN                                              (0x00010000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU15_SHIFT                                           (15U)
+#define RGX_CR_SOFT_RESET_SPU_SPU15_CLRMSK                                          (0xFFFF7FFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU15_EN                                              (0x00008000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU14_SHIFT                                           (14U)
+#define RGX_CR_SOFT_RESET_SPU_SPU14_CLRMSK                                          (0xFFFFBFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU14_EN                                              (0x00004000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU13_SHIFT                                           (13U)
+#define RGX_CR_SOFT_RESET_SPU_SPU13_CLRMSK                                          (0xFFFFDFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU13_EN                                              (0x00002000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU12_SHIFT                                           (12U)
+#define RGX_CR_SOFT_RESET_SPU_SPU12_CLRMSK                                          (0xFFFFEFFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU12_EN                                              (0x00001000U)
+#define RGX_CR_SOFT_RESET_SPU_SPU11_SHIFT                                           (11U)
+#define RGX_CR_SOFT_RESET_SPU_SPU11_CLRMSK                                          (0xFFFFF7FFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU11_EN                                              (0x00000800U)
+#define RGX_CR_SOFT_RESET_SPU_SPU10_SHIFT                                           (10U)
+#define RGX_CR_SOFT_RESET_SPU_SPU10_CLRMSK                                          (0xFFFFFBFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU10_EN                                              (0x00000400U)
+#define RGX_CR_SOFT_RESET_SPU_SPU9_SHIFT                                            (9U)
+#define RGX_CR_SOFT_RESET_SPU_SPU9_CLRMSK                                           (0xFFFFFDFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU9_EN                                               (0x00000200U)
+#define RGX_CR_SOFT_RESET_SPU_SPU8_SHIFT                                            (8U)
+#define RGX_CR_SOFT_RESET_SPU_SPU8_CLRMSK                                           (0xFFFFFEFFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU8_EN                                               (0x00000100U)
+#define RGX_CR_SOFT_RESET_SPU_SPU7_SHIFT                                            (7U)
+#define RGX_CR_SOFT_RESET_SPU_SPU7_CLRMSK                                           (0xFFFFFF7FU)
+#define RGX_CR_SOFT_RESET_SPU_SPU7_EN                                               (0x00000080U)
+#define RGX_CR_SOFT_RESET_SPU_SPU6_SHIFT                                            (6U)
+#define RGX_CR_SOFT_RESET_SPU_SPU6_CLRMSK                                           (0xFFFFFFBFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU6_EN                                               (0x00000040U)
+#define RGX_CR_SOFT_RESET_SPU_SPU5_SHIFT                                            (5U)
+#define RGX_CR_SOFT_RESET_SPU_SPU5_CLRMSK                                           (0xFFFFFFDFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU5_EN                                               (0x00000020U)
+#define RGX_CR_SOFT_RESET_SPU_SPU4_SHIFT                                            (4U)
+#define RGX_CR_SOFT_RESET_SPU_SPU4_CLRMSK                                           (0xFFFFFFEFU)
+#define RGX_CR_SOFT_RESET_SPU_SPU4_EN                                               (0x00000010U)
+#define RGX_CR_SOFT_RESET_SPU_SPU3_SHIFT                                            (3U)
+#define RGX_CR_SOFT_RESET_SPU_SPU3_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_SOFT_RESET_SPU_SPU3_EN                                               (0x00000008U)
+#define RGX_CR_SOFT_RESET_SPU_SPU2_SHIFT                                            (2U)
+#define RGX_CR_SOFT_RESET_SPU_SPU2_CLRMSK                                           (0xFFFFFFFBU)
+#define RGX_CR_SOFT_RESET_SPU_SPU2_EN                                               (0x00000004U)
+#define RGX_CR_SOFT_RESET_SPU_SPU1_SHIFT                                            (1U)
+#define RGX_CR_SOFT_RESET_SPU_SPU1_CLRMSK                                           (0xFFFFFFFDU)
+#define RGX_CR_SOFT_RESET_SPU_SPU1_EN                                               (0x00000002U)
+#define RGX_CR_SOFT_RESET_SPU_SPU0_SHIFT                                            (0U)
+#define RGX_CR_SOFT_RESET_SPU_SPU0_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SOFT_RESET_SPU_SPU0_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_MULTICORE_EVENT_REDUCE
+*/
+#define RGX_CR_MULTICORE_EVENT_REDUCE                                               (0x2428U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__MASKFULL                         (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MASKFULL                                      (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_FINISHED_SHIFT                            (31U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_FINISHED_CLRMSK                           (0x7FFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_FINISHED_EN                               (0x80000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE1_SHIFT                (31U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE1_CLRMSK               (0x7FFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE1_EN                   (0x80000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_BUFFER_STALL_SHIFT                        (30U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_BUFFER_STALL_CLRMSK                       (0xBFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TDM_BUFFER_STALL_EN                           (0x40000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE0_SHIFT                (30U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE0_CLRMSK               (0xBFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TWO_D_DONE0_EN                   (0x40000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_SIGNAL_FAILURE_SHIFT                  (29U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_SIGNAL_FAILURE_CLRMSK                 (0xDFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_SIGNAL_FAILURE_EN                     (0x20000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE1_SHIFT                  (29U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE1_CLRMSK                 (0xDFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE1_EN                     (0x20000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_FINISHED_SHIFT                            (28U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_FINISHED_CLRMSK                           (0xEFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_FINISHED_EN                               (0x10000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE0_SHIFT                  (28U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE0_CLRMSK                 (0xEFFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__RAY_DONE0_EN                     (0x10000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_BARRIER_HIT_SHIFT                         (27U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_BARRIER_HIT_CLRMSK                        (0xF7FFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_RCE_BARRIER_HIT_EN                            (0x08000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT            (26U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK           (0xFBFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TDM_BARRIER_HIT_EN               (0x04000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT          (24U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK         (0xFEFFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_CORRECTABLE_EN             (0x01000000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_LATENT_SHIFT               (22U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_LATENT_CLRMSK              (0xFFBFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FAULT_LATENT_EN                  (0x00400000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE1_SHIFT              (21U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK             (0xFFDFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE1_EN                 (0x00200000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE0_SHIFT              (20U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK             (0xFFEFFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_BE_DONE0_EN                 (0x00100000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_WDT_TIMEOUT_SHIFT                             (19U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_WDT_TIMEOUT_CLRMSK                            (0xFFF7FFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_WDT_TIMEOUT_EN                                (0x00080000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT        (19U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK       (0xFFF7FFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__IPP_END_RENDER_SENT_EN           (0x00080000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_CDM_BARRIER_HIT_SHIFT                         (18U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_CDM_BARRIER_HIT_CLRMSK                        (0xFFFBFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_CDM_BARRIER_HIT_EN                            (0x00040000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__WDT_TIMEOUT_SHIFT                (18U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__WDT_TIMEOUT_CLRMSK               (0xFFFBFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__WDT_TIMEOUT_EN                   (0x00040000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT            (17U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK           (0xFFFDFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__CDM_BARRIER_HIT_EN               (0x00020000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_BUFFER_STALL_SHIFT                    (16U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_BUFFER_STALL_CLRMSK                   (0xFFFEFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_BUFFER_STALL_EN                       (0x00010000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT     (16U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK    (0xFFFEFFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN        (0x00010000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_USC_TRIGGER_SHIFT                             (15U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_USC_TRIGGER_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_USC_TRIGGER_EN                                (0x00008000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_SHIFT                                (14U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_CLRMSK                               (0xFFFFBFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_EN                                   (0x00004000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_ACK_SHIFT                                (13U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_ACK_CLRMSK                               (0xFFFFDFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_ACK_EN                                   (0x00002000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_REQ_SHIFT                                (12U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_REQ_CLRMSK                               (0xFFFFEFFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_GPIO_REQ_EN                                   (0x00001000U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_ABORT_SHIFT                             (11U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_ABORT_CLRMSK                            (0xFFFFF7FFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_ABORT_EN                                (0x00000800U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_COMPLETE_SHIFT                          (10U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_COMPLETE_CLRMSK                         (0xFFFFFBFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_POWER_COMPLETE_EN                             (0x00000400U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MMU_PAGE_FAULT_SHIFT                          (9U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MMU_PAGE_FAULT_CLRMSK                         (0xFFFFFDFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_MMU_PAGE_FAULT_EN                             (0x00000200U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_FRAG_DONE_SHIFT                            (8U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_FRAG_DONE_CLRMSK                           (0xFFFFFEFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_FRAG_DONE_EN                               (0x00000100U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT           (8U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK          (0xFFFFFEFFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE1_EN              (0x00000100U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_OUT_OF_MEMORY_SHIFT                        (7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_OUT_OF_MEMORY_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_PM_OUT_OF_MEMORY_EN                           (0x00000080U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT           (7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK          (0xFFFFFF7FU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__FRAG_PM_BE_DONE0_EN              (0x00000080U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_TERMINATE_SHIFT                            (6U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_TERMINATE_CLRMSK                           (0xFFFFFFBFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_TERMINATE_EN                               (0x00000040U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT           (6U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK          (0xFFFFFFBFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__PM_OUT_OF_MEMORY_EN              (0x00000040U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_FINISHED_SHIFT                             (5U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_FINISHED_CLRMSK                            (0xFFFFFFDFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TA_FINISHED_EN                                (0x00000020U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TA_TERMINATE_SHIFT               (5U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TA_TERMINATE_CLRMSK              (0xFFFFFFDFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__TA_TERMINATE_EN                  (0x00000020U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_IPP_END_RENDER_SENT_SHIFT                     (4U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_IPP_END_RENDER_SENT_CLRMSK                    (0xFFFFFFEFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_IPP_END_RENDER_SENT_EN                        (0x00000010U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE1_SHIFT                 (4U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE1_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE1_EN                    (0x00000010U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_ISP_END_RENDER_SHIFT                          (3U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_ISP_END_RENDER_CLRMSK                         (0xFFFFFFF7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_ISP_END_RENDER_EN                             (0x00000008U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE0_SHIFT                 (3U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE0_CLRMSK                (0xFFFFFFF7U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__GEOM_DONE0_EN                    (0x00000008U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_FINISHED_SHIFT                        (2U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_FINISHED_CLRMSK                       (0xFFFFFFFBU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_COMPUTE_FINISHED_EN                           (0x00000004U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE1_SHIFT              (2U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE1_CLRMSK             (0xFFFFFFFBU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE1_EN                 (0x00000004U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TE_END_SHIFT                                  (1U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TE_END_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_TE_END_EN                                     (0x00000002U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE0_SHIFT              (1U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE0_CLRMSK             (0xFFFFFFFDU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE__PIPEDM_GT0__COMPUTE_DONE0_EN                 (0x00000002U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_SHIFT                               (0U)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON                                       (0x24B0U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_EXECUTE_COUNT_SHIFT                   (8U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_EXECUTE_COUNT_CLRMSK                  (0xC00000FFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_TDM_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON                                            (0x24B8U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                        (30U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                       (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_EXECUTE_COUNT_SHIFT                        (8U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_EXECUTE_COUNT_CLRMSK                       (0xC00000FFU)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_GPU_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MULTICORE_TDM_CTRL_COMMON_GPU_ENABLE_CLRMSK                          (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON                                        (0x24C0U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                    (30U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                   (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_EXECUTE_COUNT_SHIFT                    (8U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_EXECUTE_COUNT_CLRMSK                   (0xC00000FFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_BROADCAST
+*/
+#define RGX_CR_MULTICORE_BROADCAST                                                  (0x24E0U)
+#define RGX_CR_MULTICORE_BROADCAST_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MULTICORE_BROADCAST_PROGRAMMER_MASK_SHIFT                            (8U)
+#define RGX_CR_MULTICORE_BROADCAST_PROGRAMMER_MASK_CLRMSK                           (0xFFFF00FFU)
+#define RGX_CR_MULTICORE_BROADCAST_FIRMWARE_MASK_SHIFT                              (0U)
+#define RGX_CR_MULTICORE_BROADCAST_FIRMWARE_MASK_CLRMSK                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE
+*/
+#define RGX_CR_MULTICORE                                                            (0x24E8U)
+#define RGX_CR_MULTICORE_MASKFULL                                                   (IMG_UINT64_C(0x00000007FFFFFFFF))
+#define RGX_CR_MULTICORE_DOMAIN_GPU_MASK_SHIFT                                      (27U)
+#define RGX_CR_MULTICORE_DOMAIN_GPU_MASK_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF807FFFFFF))
+#define RGX_CR_MULTICORE_PRIMARY_CORE_ID_SHIFT                                      (3U)
+#define RGX_CR_MULTICORE_PRIMARY_CORE_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFF8000007))
+#define RGX_CR_MULTICORE_ID_SHIFT                                                   (0U)
+#define RGX_CR_MULTICORE_ID_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF8))
+
+
+/*
+    Register RGX_CR_MULTICORE_SYSTEM
+*/
+#define RGX_CR_MULTICORE_SYSTEM                                                     (0x24F0U)
+#define RGX_CR_MULTICORE_SYSTEM_MASKFULL                                            (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT                                     (0U)
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_CLRMSK                                    (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_MULTICORE_DOMAIN
+*/
+#define RGX_CR_MULTICORE_DOMAIN                                                     (0x24F8U)
+#define RGX_CR_MULTICORE_DOMAIN_MASKFULL                                            (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_SHIFT                                     (0U)
+#define RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_CLRMSK                                    (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_MMU_PT_PARITY_CLEAR
+*/
+#define RGX_CR_MMU_PT_PARITY_CLEAR                                                  (0x0110U)
+#define RGX_CR_MMU_PT_PARITY_CLEAR_MASKFULL                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_PT_PARITY_CLEAR_CTXTN_SHIFT                                      (0U)
+#define RGX_CR_MMU_PT_PARITY_CLEAR_CTXTN_CLRMSK                                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_MMU_PT_PARITY_STATUS
+*/
+#define RGX_CR_MMU_PT_PARITY_STATUS                                                 (0x0120U)
+#define RGX_CR_MMU_PT_PARITY_STATUS_MASKFULL                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_PT_PARITY_STATUS_CTXTN_SHIFT                                     (0U)
+#define RGX_CR_MMU_PT_PARITY_STATUS_CTXTN_CLRMSK                                    (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_EVENT_ENABLE
+*/
+#define RGX_CR_EVENT_ENABLE                                                         (0x0128U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__MASKFULL                                   (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__MASKFULL                    (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__MASKFULL                    (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_EVENT_ENABLE_MASKFULL                                                (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_SHIFT          (31U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_CLRMSK         (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_EN             (0x80000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_SHIFT           (31U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_CLRMSK          (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_EN              (0x80000000U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_SHIFT           (30U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_CLRMSK          (0xBFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_EN              (0x40000000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_SHIFT             (29U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_CLRMSK            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_EN                (0x20000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_SHIFT          (28U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_CLRMSK         (0xEFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_EN             (0x10000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_SHIFT             (28U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_CLRMSK            (0xEFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_EN                (0x10000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT       (26U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK      (0xFBFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_EN          (0x04000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT     (24U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK    (0xFEFFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_EN        (0x01000000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_SHIFT          (22U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_CLRMSK         (0xFFBFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_EN             (0x00400000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_SHIFT         (21U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK        (0xFFDFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_EN            (0x00200000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_SHIFT         (20U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK        (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_EN            (0x00100000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT           (19U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_CLRMSK          (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN              (0x00080000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT   (19U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK  (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_EN      (0x00080000U)
+#define RGX_CR_EVENT_ENABLE_TDM_CONTEXT_STORE_FINISHED_SHIFT                        (18U)
+#define RGX_CR_EVENT_ENABLE_TDM_CONTEXT_STORE_FINISHED_CLRMSK                       (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_CONTEXT_STORE_FINISHED_EN                           (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_SHIFT       (18U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_EN          (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT           (18U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_CLRMSK          (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_EN              (0x00040000U)
+#define RGX_CR_EVENT_ENABLE_SPFILTER_SIGNAL_UPDATE_SHIFT                            (17U)
+#define RGX_CR_EVENT_ENABLE_SPFILTER_SIGNAL_UPDATE_CLRMSK                           (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE_SPFILTER_SIGNAL_UPDATE_EN                               (0x00020000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT       (17U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK      (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_EN          (0x00020000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT (16U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK (0xFFFEFFFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN   (0x00010000U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_ENABLE_FAULT_FW_SHIFT                                          (14U)
+#define RGX_CR_EVENT_ENABLE_FAULT_FW_CLRMSK                                         (0xFFFFBFFFU)
+#define RGX_CR_EVENT_ENABLE_FAULT_FW_EN                                             (0x00004000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_ENABLE_PM_FRAG_DONE_SHIFT                                      (8U)
+#define RGX_CR_EVENT_ENABLE_PM_FRAG_DONE_CLRMSK                                     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_ENABLE_PM_FRAG_DONE_EN                                         (0x00000100U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT      (8U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_EN         (0x00000100U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT      (7U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK     (0xFFFFFF7FU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_EN         (0x00000080U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT      (6U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_EN         (0x00000040U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_SHIFT          (5U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_CLRMSK         (0xFFFFFFDFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_EN             (0x00000020U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__IPP_END_RENDER_SENT_SHIFT                  (4U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__IPP_END_RENDER_SENT_CLRMSK                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__IPP_END_RENDER_SENT_EN                     (0x00000010U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_SHIFT   (4U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_CLRMSK  (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_EN      (0x00000010U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_SHIFT            (4U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_CLRMSK           (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_EN               (0x00000010U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_RENDER_SHIFT                                    (3U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_RENDER_CLRMSK                                   (0xFFFFFFF7U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_RENDER_EN                                       (0x00000008U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_SHIFT            (3U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_CLRMSK           (0xFFFFFFF7U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_EN               (0x00000008U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_SHIFT         (2U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_CLRMSK        (0xFFFFFFFBU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_EN            (0x00000004U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__TE_END_SHIFT                               (1U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__TE_END_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ1__TE_END_EN                                  (0x00000002U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_SHIFT                (1U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_CLRMSK               (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_EN                   (0x00000002U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_SHIFT         (1U)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_CLRMSK        (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_EN            (0x00000002U)
+#define RGX_CR_EVENT_ENABLE_FAULT_GPU_SHIFT                                         (0U)
+#define RGX_CR_EVENT_ENABLE_FAULT_GPU_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_EVENT_ENABLE_FAULT_GPU_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_STATUS
+*/
+#define RGX_CR_EVENT_STATUS                                                         (0x0130U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__MASKFULL                                   (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__MASKFULL                    (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__MASKFULL                    (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_EVENT_STATUS_MASKFULL                                                (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_SHIFT          (31U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_CLRMSK         (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_EN             (0x80000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_SHIFT           (31U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_CLRMSK          (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_EN              (0x80000000U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_SHIFT           (30U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_CLRMSK          (0xBFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_EN              (0x40000000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_SHIFT             (29U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_CLRMSK            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_EN                (0x20000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_SHIFT          (28U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_CLRMSK         (0xEFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_EN             (0x10000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_SHIFT             (28U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_CLRMSK            (0xEFFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_EN                (0x10000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_SHIFT       (27U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_CLRMSK      (0xF7FFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_EN          (0x08000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT       (26U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK      (0xFBFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_EN          (0x04000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT     (24U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK    (0xFEFFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_EN        (0x01000000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_SHIFT          (22U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_CLRMSK         (0xFFBFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_EN             (0x00400000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_SHIFT         (21U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK        (0xFFDFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_EN            (0x00200000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_SHIFT         (20U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK        (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_EN            (0x00100000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT           (19U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_CLRMSK          (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN              (0x00080000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT   (19U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK  (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_EN      (0x00080000U)
+#define RGX_CR_EVENT_STATUS_TDM_CONTEXT_STORE_FINISHED_SHIFT                        (18U)
+#define RGX_CR_EVENT_STATUS_TDM_CONTEXT_STORE_FINISHED_CLRMSK                       (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_CONTEXT_STORE_FINISHED_EN                           (0x00040000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_SHIFT       (18U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_EN          (0x00040000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT           (18U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_CLRMSK          (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_EN              (0x00040000U)
+#define RGX_CR_EVENT_STATUS_SPFILTER_SIGNAL_UPDATE_SHIFT                            (17U)
+#define RGX_CR_EVENT_STATUS_SPFILTER_SIGNAL_UPDATE_CLRMSK                           (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS_SPFILTER_SIGNAL_UPDATE_EN                               (0x00020000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT       (17U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK      (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_EN          (0x00020000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT (16U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK (0xFFFEFFFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN   (0x00010000U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_STATUS_FAULT_FW_SHIFT                                          (14U)
+#define RGX_CR_EVENT_STATUS_FAULT_FW_CLRMSK                                         (0xFFFFBFFFU)
+#define RGX_CR_EVENT_STATUS_FAULT_FW_EN                                             (0x00004000U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_STATUS_PM_FRAG_DONE_SHIFT                                      (8U)
+#define RGX_CR_EVENT_STATUS_PM_FRAG_DONE_CLRMSK                                     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_STATUS_PM_FRAG_DONE_EN                                         (0x00000100U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT      (8U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK     (0xFFFFFEFFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_EN         (0x00000100U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT      (7U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK     (0xFFFFFF7FU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_EN         (0x00000080U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT      (6U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_EN         (0x00000040U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_SHIFT          (5U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_CLRMSK         (0xFFFFFFDFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_EN             (0x00000020U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__IPP_END_RENDER_SENT_SHIFT                  (4U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__IPP_END_RENDER_SENT_CLRMSK                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__IPP_END_RENDER_SENT_EN                     (0x00000010U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_SHIFT   (4U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_CLRMSK  (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_EN      (0x00000010U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_SHIFT            (4U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_CLRMSK           (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_EN               (0x00000010U)
+#define RGX_CR_EVENT_STATUS_ISP_END_RENDER_SHIFT                                    (3U)
+#define RGX_CR_EVENT_STATUS_ISP_END_RENDER_CLRMSK                                   (0xFFFFFFF7U)
+#define RGX_CR_EVENT_STATUS_ISP_END_RENDER_EN                                       (0x00000008U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_SHIFT            (3U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_CLRMSK           (0xFFFFFFF7U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_EN               (0x00000008U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_SHIFT         (2U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_CLRMSK        (0xFFFFFFFBU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_EN            (0x00000004U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__TE_END_SHIFT                               (1U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__TE_END_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ1__TE_END_EN                                  (0x00000002U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_SHIFT                (1U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_CLRMSK               (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_EN                   (0x00000002U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_SHIFT         (1U)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_CLRMSK        (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_EN            (0x00000002U)
+#define RGX_CR_EVENT_STATUS_FAULT_GPU_SHIFT                                         (0U)
+#define RGX_CR_EVENT_STATUS_FAULT_GPU_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_EVENT_STATUS_FAULT_GPU_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_CLEAR
+*/
+#define RGX_CR_EVENT_CLEAR                                                          (0x0138U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__MASKFULL                                    (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__MASKFULL                     (IMG_UINT64_C(0x00000000F80FFFFF))
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__MASKFULL                     (IMG_UINT64_C(0x00000000FD7FFFFF))
+#define RGX_CR_EVENT_CLEAR_MASKFULL                                                 (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_SHIFT                                 (31U)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_CLRMSK                                (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_EN                                    (0x80000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_SHIFT           (31U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_CLRMSK          (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TDM_FINISHED_EN              (0x80000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_SHIFT            (31U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_CLRMSK           (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE1_EN               (0x80000000U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_SHIFT                                   (30U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_CLRMSK                                  (0xBFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_EN                                      (0x40000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_SHIFT            (30U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_CLRMSK           (0xBFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TWO_D_DONE0_EN               (0x40000000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_SHIFT                             (29U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_CLRMSK                            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_EN                                (0x20000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_SHIFT              (29U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_CLRMSK             (0xDFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE1_EN                 (0x20000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_SHIFT           (28U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_CLRMSK          (0xEFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_FINISHED_EN              (0x10000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_SHIFT              (28U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_CLRMSK             (0xEFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RAY_DONE0_EN                 (0x10000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_SHIFT        (27U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_CLRMSK       (0xF7FFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__RCE_BARRIER_HIT_EN           (0x08000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_SHIFT        (27U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_CLRMSK       (0xF7FFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__RCE_BARRIER_HIT_EN           (0x08000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_SHIFT        (26U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_CLRMSK       (0xFBFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TDM_BARRIER_HIT_EN           (0x04000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_SHIFT      (24U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_CLRMSK     (0xFEFFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_CORRECTABLE_EN         (0x01000000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_SHIFT           (22U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_CLRMSK          (0xFFBFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FAULT_LATENT_EN              (0x00400000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_SHIFT          (21U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_CLRMSK         (0xFFDFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE1_EN             (0x00200000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_SHIFT          (20U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_CLRMSK         (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_BE_DONE0_EN             (0x00100000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT            (19U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_CLRMSK           (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN               (0x00080000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_SHIFT    (19U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_CLRMSK   (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__IPP_END_RENDER_SENT_EN       (0x00080000U)
+#define RGX_CR_EVENT_CLEAR_TDM_CONTEXT_STORE_FINISHED_SHIFT                         (18U)
+#define RGX_CR_EVENT_CLEAR_TDM_CONTEXT_STORE_FINISHED_CLRMSK                        (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_CONTEXT_STORE_FINISHED_EN                            (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_SHIFT        (18U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_CLRMSK       (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__CDM_BARRIER_HIT_EN           (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT            (18U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_CLRMSK           (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_EN               (0x00040000U)
+#define RGX_CR_EVENT_CLEAR_SPFILTER_SIGNAL_UPDATE_SHIFT                             (17U)
+#define RGX_CR_EVENT_CLEAR_SPFILTER_SIGNAL_UPDATE_CLRMSK                            (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR_SPFILTER_SIGNAL_UPDATE_EN                                (0x00020000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_SHIFT        (17U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_CLRMSK       (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__CDM_BARRIER_HIT_EN           (0x00020000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_SHIFT                               (16U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_EN                                  (0x00010000U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_SHIFT (16U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_CLRMSK (0xFFFEFFFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__SPFILTER_SIGNAL_UPDATE_EN    (0x00010000U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_SHIFT                                        (15U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_CLRMSK                                       (0xFFFF7FFFU)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_EN                                           (0x00008000U)
+#define RGX_CR_EVENT_CLEAR_FAULT_FW_SHIFT                                           (14U)
+#define RGX_CR_EVENT_CLEAR_FAULT_FW_CLRMSK                                          (0xFFFFBFFFU)
+#define RGX_CR_EVENT_CLEAR_FAULT_FW_EN                                              (0x00004000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_SHIFT                                           (13U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_CLRMSK                                          (0xFFFFDFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_EN                                              (0x00002000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_SHIFT                                           (12U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_CLRMSK                                          (0xFFFFEFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_EN                                              (0x00001000U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_SHIFT                                        (11U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_CLRMSK                                       (0xFFFFF7FFU)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_EN                                           (0x00000800U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_SHIFT                                     (10U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_CLRMSK                                    (0xFFFFFBFFU)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_EN                                        (0x00000400U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_SHIFT                                     (9U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_CLRMSK                                    (0xFFFFFDFFU)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_EN                                        (0x00000200U)
+#define RGX_CR_EVENT_CLEAR_PM_FRAG_DONE_SHIFT                                       (8U)
+#define RGX_CR_EVENT_CLEAR_PM_FRAG_DONE_CLRMSK                                      (0xFFFFFEFFU)
+#define RGX_CR_EVENT_CLEAR_PM_FRAG_DONE_EN                                          (0x00000100U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_SHIFT       (8U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_CLRMSK      (0xFFFFFEFFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE1_EN          (0x00000100U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_SHIFT                                   (7U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_CLRMSK                                  (0xFFFFFF7FU)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_EN                                      (0x00000080U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_SHIFT       (7U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_CLRMSK      (0xFFFFFF7FU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__FRAG_PM_BE_DONE0_EN          (0x00000080U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_SHIFT                                       (6U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_CLRMSK                                      (0xFFFFFFBFU)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_EN                                          (0x00000040U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_SHIFT       (6U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_CLRMSK      (0xFFFFFFBFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__PM_OUT_OF_MEMORY_EN          (0x00000040U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_SHIFT                                        (5U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_EN                                           (0x00000020U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_SHIFT           (5U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_CLRMSK          (0xFFFFFFDFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__TA_TERMINATE_EN              (0x00000020U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_SHIFT                                  (4U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_EN                                     (0x00000010U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__IPP_END_RENDER_SENT_SHIFT                   (4U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__IPP_END_RENDER_SENT_CLRMSK                  (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__IPP_END_RENDER_SENT_EN                      (0x00000010U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_SHIFT    (4U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_CLRMSK   (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__IPP_END_RENDER_SENT_EN       (0x00000010U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_SHIFT             (4U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_CLRMSK            (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE1_EN                (0x00000010U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_RENDER_SHIFT                                     (3U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_RENDER_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_RENDER_EN                                        (0x00000008U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_SHIFT             (3U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_CLRMSK            (0xFFFFFFF7U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__GEOM_DONE0_EN                (0x00000008U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_SHIFT                                   (2U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_EN                                      (0x00000004U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_SHIFT          (2U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_CLRMSK         (0xFFFFFFFBU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE1_EN             (0x00000004U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_SHIFT                                    (1U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_EN                                       (0x00000002U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__TE_END_SHIFT                                (1U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__TE_END_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ1__TE_END_EN                                   (0x00000002U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_SHIFT                 (1U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_CLRMSK                (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_EQ0__TE_END_EN                    (0x00000002U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_SHIFT          (1U)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_CLRMSK         (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR__AXT_IF_EQ2_AND_PIPEDM_GT0__COMPUTE_DONE0_EN             (0x00000002U)
+#define RGX_CR_EVENT_CLEAR_FAULT_GPU_SHIFT                                          (0U)
+#define RGX_CR_EVENT_CLEAR_FAULT_GPU_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_EVENT_CLEAR_FAULT_GPU_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_TIMER
+*/
+#define RGX_CR_TIMER                                                                (0x0160U)
+#define RGX_CR_TIMER_MASKFULL                                                       (IMG_UINT64_C(0x8000FFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_SHIFT                                                    (63U)
+#define RGX_CR_TIMER_BIT31_CLRMSK                                                   (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_EN                                                       (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_TIMER_VALUE_SHIFT                                                    (0U)
+#define RGX_CR_TIMER_VALUE_CLRMSK                                                   (IMG_UINT64_C(0xFFFF000000000000))
+
+
+/*
+    Register RGX_CR_FAULT_FW_STATUS
+*/
+#define RGX_CR_FAULT_FW_STATUS                                                      (0x0170U)
+#define RGX_CR_FAULT_FW_STATUS_MASKFULL                                             (IMG_UINT64_C(0x00000000000F007F))
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_CORRECT_SHIFT                              (19U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_CORRECT_CLRMSK                             (0xFFF7FFFFU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_CORRECT_EN                                 (0x00080000U)
+#define RGX_CR_FAULT_FW_STATUS_META_CORRECT_SHIFT                                   (18U)
+#define RGX_CR_FAULT_FW_STATUS_META_CORRECT_CLRMSK                                  (0xFFFBFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_META_CORRECT_EN                                      (0x00040000U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CORRECT_SHIFT                                    (17U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CORRECT_CLRMSK                                   (0xFFFDFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CORRECT_EN                                       (0x00020000U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_SHIFT                                    (16U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_CLRMSK                                   (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_EN                                       (0x00010000U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_DETECT_SHIFT                               (6U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_DETECT_CLRMSK                              (0xFFFFFFBFU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_CHEST_DETECT_EN                                  (0x00000040U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_SHIFT                          (5U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_CLRMSK                         (0xFFFFFFDFU)
+#define RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_EN                             (0x00000020U)
+#define RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_SHIFT                                  (4U)
+#define RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_EN                                     (0x00000010U)
+#define RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_SHIFT                                (3U)
+#define RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_EN                                   (0x00000008U)
+#define RGX_CR_FAULT_FW_STATUS_META_DETECT_SHIFT                                    (2U)
+#define RGX_CR_FAULT_FW_STATUS_META_DETECT_CLRMSK                                   (0xFFFFFFFBU)
+#define RGX_CR_FAULT_FW_STATUS_META_DETECT_EN                                       (0x00000004U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_DETECT_SHIFT                                     (1U)
+#define RGX_CR_FAULT_FW_STATUS_SLC_DETECT_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FAULT_FW_STATUS_SLC_DETECT_EN                                        (0x00000002U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_DETECT_SHIFT                                     (0U)
+#define RGX_CR_FAULT_FW_STATUS_MMU_DETECT_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_STATUS_MMU_DETECT_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_CLEAR
+*/
+#define RGX_CR_FAULT_FW_CLEAR                                                       (0x0178U)
+#define RGX_CR_FAULT_FW_CLEAR_MASKFULL                                              (IMG_UINT64_C(0x00000000000F007F))
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_CORRECT_SHIFT                               (19U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_CORRECT_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_CORRECT_EN                                  (0x00080000U)
+#define RGX_CR_FAULT_FW_CLEAR_META_CORRECT_SHIFT                                    (18U)
+#define RGX_CR_FAULT_FW_CLEAR_META_CORRECT_CLRMSK                                   (0xFFFBFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_META_CORRECT_EN                                       (0x00040000U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CORRECT_SHIFT                                     (17U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CORRECT_CLRMSK                                    (0xFFFDFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CORRECT_EN                                        (0x00020000U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_CORRECT_SHIFT                                     (16U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_CORRECT_CLRMSK                                    (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_CORRECT_EN                                        (0x00010000U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_DETECT_SHIFT                                (6U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_DETECT_CLRMSK                               (0xFFFFFFBFU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_CHEST_DETECT_EN                                   (0x00000040U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_PTE_PARITY_DETECT_SHIFT                           (5U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_PTE_PARITY_DETECT_CLRMSK                          (0xFFFFFFDFU)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_PTE_PARITY_DETECT_EN                              (0x00000020U)
+#define RGX_CR_FAULT_FW_CLEAR_PARITY_DETECT_SHIFT                                   (4U)
+#define RGX_CR_FAULT_FW_CLEAR_PARITY_DETECT_CLRMSK                                  (0xFFFFFFEFU)
+#define RGX_CR_FAULT_FW_CLEAR_PARITY_DETECT_EN                                      (0x00000010U)
+#define RGX_CR_FAULT_FW_CLEAR_LOCKSTEP_DETECT_SHIFT                                 (3U)
+#define RGX_CR_FAULT_FW_CLEAR_LOCKSTEP_DETECT_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_FAULT_FW_CLEAR_LOCKSTEP_DETECT_EN                                    (0x00000008U)
+#define RGX_CR_FAULT_FW_CLEAR_META_DETECT_SHIFT                                     (2U)
+#define RGX_CR_FAULT_FW_CLEAR_META_DETECT_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_FAULT_FW_CLEAR_META_DETECT_EN                                        (0x00000004U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_DETECT_SHIFT                                      (1U)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_DETECT_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_FAULT_FW_CLEAR_SLC_DETECT_EN                                         (0x00000002U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_DETECT_SHIFT                                      (0U)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_DETECT_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_CLEAR_MMU_DETECT_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_GPU_STATUS
+*/
+#define RGX_CR_FAULT_GPU_STATUS                                                     (0x0180U)
+#define RGX_CR_FAULT_GPU_STATUS_MASKFULL                                            (IMG_UINT64_C(0x000000000001000F))
+#define RGX_CR_FAULT_GPU_STATUS_REST_CORRECT_SHIFT                                  (16U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_CORRECT_CLRMSK                                 (0xFFFEFFFFU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_CORRECT_EN                                     (0x00010000U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_FAULT_SHIFT                             (3U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_FAULT_CLRMSK                            (0xFFFFFFF7U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_FAULT_EN                                (0x00000008U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_PARITY_SHIFT                                   (2U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_PARITY_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_PARITY_EN                                      (0x00000004U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_SHIFT                                   (1U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_LATENT_EN                                      (0x00000002U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_DETECT_SHIFT                                   (0U)
+#define RGX_CR_FAULT_GPU_STATUS_REST_DETECT_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_FAULT_GPU_STATUS_REST_DETECT_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_GPU_CLEAR
+*/
+#define RGX_CR_FAULT_GPU_CLEAR                                                      (0x0188U)
+#define RGX_CR_FAULT_GPU_CLEAR_MASKFULL                                             (IMG_UINT64_C(0x000000000001000F))
+#define RGX_CR_FAULT_GPU_CLEAR_REST_CORRECT_SHIFT                                   (16U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_CORRECT_CLRMSK                                  (0xFFFEFFFFU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_CORRECT_EN                                      (0x00010000U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_FAULT_SHIFT                              (3U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_FAULT_CLRMSK                             (0xFFFFFFF7U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_FAULT_EN                                 (0x00000008U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_PARITY_SHIFT                                    (2U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_PARITY_CLRMSK                                   (0xFFFFFFFBU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_PARITY_EN                                       (0x00000004U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_SHIFT                                    (1U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_LATENT_EN                                       (0x00000002U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_DETECT_SHIFT                                    (0U)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_DETECT_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_FAULT_GPU_CLEAR_REST_DETECT_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS
+*/
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS                                    (0x1220U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MASKFULL                           (IMG_UINT64_C(0x0000003FFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CDM_FAULT_DS_SHIFT                 (37U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CDM_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CDM_FAULT_DS_EN                    (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_FAULT_DS_SHIFT                 (36U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_FAULT_DS_EN                    (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_FAULT_DS_SHIFT                 (35U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_FAULT_DS_EN                    (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_IPP_FAULT_DS_SHIFT                 (34U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_IPP_FAULT_DS_CLRMSK                (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_IPP_FAULT_DS_EN                    (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TILING_FAULT_DS_SHIFT              (33U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TILING_FAULT_DS_CLRMSK             (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TILING_FAULT_DS_EN                 (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_RED_SHIFT                 (32U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_RED_CLRMSK                (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_RED_EN                    (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_PARITY_SHIFT                 (31U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_PARITY_EN                    (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_BUS_PARITY_SHIFT             (30U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_BUS_PARITY_CLRMSK            (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_CR_BUS_PARITY_EN                (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_BUS_PARITY_SHIFT                (29U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_BUS_PARITY_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_BUS_PARITY_EN                   (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GPU_IRQ_CR_PARITY_SHIFT            (28U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GPU_IRQ_CR_PARITY_CLRMSK           (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GPU_IRQ_CR_PARITY_EN               (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_FILTER_PARITY_SHIFT            (27U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_FILTER_PARITY_CLRMSK           (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_FILTER_PARITY_EN               (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_PARITY_SHIFT                 (26U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_PARITY_EN                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_EXTERNAL_AXI_MEM_BUS_PARITY_SHIFT  (25U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_EXTERNAL_AXI_MEM_BUS_PARITY_CLRMSK (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_EXTERNAL_AXI_MEM_BUS_PARITY_EN     (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_PARITY_SHIFT                   (24U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_PARITY_EN                      (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_PARITY_SHIFT                   (23U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_PARITY_EN                      (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DFU_PARITY_SHIFT                   (22U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DFU_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DFU_PARITY_EN                      (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_SHIFT           (21U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_CLRMSK          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_EN              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_AXI_BUS_PARITY_SHIFT               (20U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_AXI_BUS_PARITY_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_AXI_BUS_PARITY_EN                  (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BUS_PARITY_SHIFT                   (19U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BUS_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BUS_PARITY_EN                      (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_AXI_BUS_PARITY_SHIFT    (18U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_AXI_BUS_PARITY_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_AXI_BUS_PARITY_EN       (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_BUS_PARITY_SHIFT        (17U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_BUS_PARITY_CLRMSK       (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_IRQ_BUS_PARITY_EN           (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_PARITY_SHIFT                    (16U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_PARITY_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CR_PARITY_EN                       (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_CR_PARITY_SHIFT             (15U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_CR_PARITY_CLRMSK            (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SAFETY_CR_PARITY_EN                (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_SHIFT                     (14U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_LOCKSTEP_EN                        (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TEXAS_SHIFT                        (13U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TEXAS_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TEXAS_EN                           (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_DMA_SHIFT                       (12U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_DMA_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FW_DMA_EN                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBSC_SHIFT                         (11U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBSC_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBSC_EN                            (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBHC_SHIFT                         (10U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBHC_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_FBHC_EN                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_SHIFT                          (9U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_DCE_EN                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_SHIFT                          (8U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TDM_EN                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_SHIFT                          (7U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_BIF_EN                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GARTEN_SHIFT                       (6U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GARTEN_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_GARTEN_EN                          (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TE3_SHIFT                          (5U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TE3_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_TE3_EN                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MMU_SHIFT                          (4U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MMU_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_MMU_EN                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_ACE_SHIFT                      (3U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_ACE_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_ACE_EN                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_SHIFT                          (2U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_SLC_EN                             (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_FBCDC_SHIFT                  (1U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_FBCDC_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_FBCDC_EN                     (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_SLC_SHIFT                    (0U)
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_SLC_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_FAULT_JONES_REDUCER_DETECT_STATUS_CHEST_SLC_EN                       (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FAULT_JONES_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE                                          (0x1230U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x0000003FFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CDM_FAULT_DS_SHIFT                       (37U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CDM_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CDM_FAULT_DS_EN                          (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_FAULT_DS_SHIFT                       (36U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_FAULT_DS_EN                          (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_FAULT_DS_SHIFT                       (35U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_FAULT_DS_EN                          (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_IPP_FAULT_DS_SHIFT                       (34U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_IPP_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_IPP_FAULT_DS_EN                          (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TILING_FAULT_DS_SHIFT                    (33U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TILING_FAULT_DS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TILING_FAULT_DS_EN                       (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_RED_SHIFT                       (32U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_RED_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_RED_EN                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_PARITY_SHIFT                       (31U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_PARITY_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_PARITY_EN                          (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_BUS_PARITY_SHIFT                   (30U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_BUS_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_CR_BUS_PARITY_EN                      (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_BUS_PARITY_SHIFT                      (29U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_BUS_PARITY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_BUS_PARITY_EN                         (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GPU_IRQ_CR_PARITY_SHIFT                  (28U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GPU_IRQ_CR_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GPU_IRQ_CR_PARITY_EN                     (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_FILTER_PARITY_SHIFT                  (27U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_FILTER_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_FILTER_PARITY_EN                     (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_PARITY_SHIFT                       (26U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_PARITY_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_PARITY_EN                          (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_EXTERNAL_AXI_MEM_BUS_PARITY_SHIFT        (25U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_EXTERNAL_AXI_MEM_BUS_PARITY_CLRMSK       (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_EXTERNAL_AXI_MEM_BUS_PARITY_EN           (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_PARITY_SHIFT                         (24U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_PARITY_EN                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_PARITY_SHIFT                         (23U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_PARITY_EN                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DFU_PARITY_SHIFT                         (22U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DFU_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DFU_PARITY_EN                            (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                 (21U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_AXI_BUS_PARITY_SHIFT                     (20U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_AXI_BUS_PARITY_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_AXI_BUS_PARITY_EN                        (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BUS_PARITY_SHIFT                         (19U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BUS_PARITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BUS_PARITY_EN                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_AXI_BUS_PARITY_SHIFT          (18U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_AXI_BUS_PARITY_CLRMSK         (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_AXI_BUS_PARITY_EN             (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_BUS_PARITY_SHIFT              (17U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_BUS_PARITY_CLRMSK             (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_IRQ_BUS_PARITY_EN                 (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_PARITY_SHIFT                          (16U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_PARITY_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CR_PARITY_EN                             (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_CR_PARITY_SHIFT                   (15U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_CR_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SAFETY_CR_PARITY_EN                      (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_SHIFT                           (14U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_LOCKSTEP_EN                              (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TEXAS_SHIFT                              (13U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TEXAS_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TEXAS_EN                                 (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_DMA_SHIFT                             (12U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_DMA_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FW_DMA_EN                                (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBSC_SHIFT                               (11U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBSC_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBSC_EN                                  (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBHC_SHIFT                               (10U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBHC_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_FBHC_EN                                  (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_SHIFT                                (9U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_DCE_EN                                   (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_SHIFT                                (8U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TDM_EN                                   (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_SHIFT                                (7U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_BIF_EN                                   (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GARTEN_SHIFT                             (6U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GARTEN_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_GARTEN_EN                                (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TE3_SHIFT                                (5U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TE3_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_TE3_EN                                   (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MMU_SHIFT                                (4U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MMU_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_MMU_EN                                   (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_ACE_SHIFT                            (3U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_ACE_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_ACE_EN                               (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_SHIFT                                (2U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_SLC_EN                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_FBCDC_SHIFT                        (1U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_FBCDC_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_FBCDC_EN                           (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_SLC_SHIFT                          (0U)
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_SLC_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_FAULT_JONES_REDUCER_DISABLE_CHEST_SLC_EN                             (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FAULT_TEXAS_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE                                          (0x1248U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x0000077FFFFFFC0F))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_FAULT_DS_SHIFT                       (42U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_FAULT_DS_CLRMSK                      (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_FAULT_DS_EN                          (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CDM_PIPE_FAULT_DS_SHIFT                  (41U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CDM_PIPE_FAULT_DS_CLRMSK                 (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CDM_PIPE_FAULT_DS_EN                     (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                 (40U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                    (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_PARITY_SHIFT                 (38U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_PARITY_CLRMSK                (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_PARITY_EN                    (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_PARITY_SHIFT                      (37U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_PARITY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_PARITY_EN                         (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_PARITY_SHIFT                  (36U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_PARITY_EN                     (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_PARITY_SHIFT                  (35U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_PARITY_CLRMSK                 (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_PARITY_EN                     (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_BUS_PARITY_SHIFT                   (34U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_BUS_PARITY_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_BUS_PARITY_EN                      (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_BUS_PARITY_SHIFT              (33U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_BUS_PARITY_CLRMSK             (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_BUS_PARITY_EN                 (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_BUS_PARITY_SHIFT                     (32U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_BUS_PARITY_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_BUS_PARITY_EN                        (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CR_PARITY_SHIFT                      (31U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CR_PARITY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CR_PARITY_EN                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_CR_PARITY_SHIFT               (30U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_CR_PARITY_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHARED_CR_PARITY_EN                  (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_CR_PARITY_SHIFT                    (29U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_CR_PARITY_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TEXAS_CR_PARITY_EN                       (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SWIFT_SHIFT                              (28U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SWIFT_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_SWIFT_EN                                 (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MERCER_SHIFT                             (27U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MERCER_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MERCER_EN                                (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_SHIFT                         (26U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS0_EN                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_SHIFT                         (25U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_BIF_TEXAS1_EN                            (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_SHIFT                        (24U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDSRW_CACHE_EN                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_SHIFT                             (23U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_MCU_L1_EN                                (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VDM_SHIFT                                (22U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VDM_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VDM_EN                                   (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFA_SHIFT                                (21U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFA_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFA_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_UVB_SHIFT                                (20U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_UVB_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_UVB_EN                                   (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PPP_SHIFT                                (19U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PPP_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PPP_EN                                   (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CLIPPER_SHIFT                            (18U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CLIPPER_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_CLIPPER_EN                               (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBG_SHIFT                                (17U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBG_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBG_EN                                   (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBS_SHIFT                                (16U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBS_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VBS_EN                                   (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VCE_SHIFT                                (15U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VCE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_VCE_EN                                   (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PRIM_BUFFER_SHIFT                        (14U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PRIM_BUFFER_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PRIM_BUFFER_EN                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TESS_SHIFT                               (13U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TESS_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TESS_EN                                  (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IPF_SHIFT                                (12U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IPF_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IPF_EN                                   (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFPU_SHIFT                               (11U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFPU_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_IFPU_EN                                  (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TPF_SHIFT                                (10U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TPF_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_TPF_EN                                   (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_SHIFT                                (3U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PDS_EN                                   (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_SHIFT                                (2U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PBE_EN                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_USC_L2ICACHE_SHIFT                       (1U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_USC_L2ICACHE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_USC_L2ICACHE_EN                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PSB_SHIFT                                (0U)
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PSB_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_FAULT_TEXAS_REDUCER_DISABLE_PSB_EN                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FAULT_SWIFT_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE                                          (0x1260U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x00000000FFC007FF))
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BUS_PARITY_SHIFT                         (31U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BUS_PARITY_CLRMSK                        (0x7FFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BUS_PARITY_EN                            (0x80000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SWIFT_CR_PARITY_SHIFT                    (30U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SWIFT_CR_PARITY_CLRMSK                   (0xBFFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SWIFT_CR_PARITY_EN                       (0x40000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_CR_PARITY_SHIFT                      (29U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_CR_PARITY_CLRMSK                     (0xDFFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_CR_PARITY_EN                         (0x20000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CR_PARITY_SHIFT                      (28U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CR_PARITY_CLRMSK                     (0xEFFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CR_PARITY_EN                         (0x10000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_LATENT_SHIFT                             (27U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_LATENT_CLRMSK                            (0xF7FFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_LATENT_EN                                (0x08000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ICS_FAULT_SHIFT                          (26U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ICS_FAULT_CLRMSK                         (0xFBFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ICS_FAULT_EN                             (0x04000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_PARITY_SHIFT                         (25U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_PARITY_CLRMSK                        (0xFDFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_PARITY_EN                            (0x02000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_L1_PARITY_SHIFT                      (24U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_L1_PARITY_CLRMSK                     (0xFEFFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_L1_PARITY_EN                         (0x01000000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_DEMUX_PARITY_SHIFT                       (23U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_DEMUX_PARITY_CLRMSK                      (0xFF7FFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_DEMUX_PARITY_EN                          (0x00800000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                 (22U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK                (0xFFBFFFFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                    (0x00400000U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_SHIFT                                (10U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_CLRMSK                               (0xFFFFFBFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TCU_EN                                   (0x00000400U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TF_SHIFT                             (9U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TF_CLRMSK                            (0xFFFFFDFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TF_EN                                (0x00000200U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TAG_SHIFT                            (8U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TAG_CLRMSK                           (0xFFFFFEFFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_TAG_EN                               (0x00000100U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_MADD_SHIFT                           (7U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_MADD_CLRMSK                          (0xFFFFFF7FU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_TPU_MADD_EN                              (0x00000080U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_SHIFT                                (6U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_CLRMSK                               (0xFFFFFFBFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_BSC_EN                                   (0x00000040U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_IDEC_SHIFT                           (5U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_IDEC_CLRMSK                          (0xFFFFFFDFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_IDEC_EN                              (0x00000020U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CT_SHIFT                             (4U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CT_CLRMSK                            (0xFFFFFFEFU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_CT_EN                                (0x00000010U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_DEPTHSORT_SHIFT                      (3U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_DEPTHSORT_CLRMSK                     (0xFFFFFFF7U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_DEPTHSORT_EN                         (0x00000008U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_LUT_SHIFT                    (2U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_LUT_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_LUT_EN                       (0x00000004U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_BUFFER_SHIFT                 (1U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_BUFFER_CLRMSK                (0xFFFFFFFDU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_TAGSORT_BUFFER_EN                    (0x00000002U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_ZLS_SHIFT                            (0U)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_ZLS_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FAULT_SWIFT_REDUCER_DISABLE_ISP_ZLS_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS
+*/
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS                                   (0x1268U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MASKFULL                          (IMG_UINT64_C(0x00000000F000001F))
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_BUS_PARITY_SHIFT                  (31U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_BUS_PARITY_CLRMSK                 (0x7FFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_BUS_PARITY_EN                     (0x80000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_SHIFT          (30U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_CLRMSK         (0xBFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_SOCIF_DELAY_PARITY_EN             (0x40000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_CR_PARITY_SHIFT                   (29U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_CR_PARITY_CLRMSK                  (0xDFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_CR_PARITY_EN                      (0x20000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MH_PARITY_SHIFT                   (28U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MH_PARITY_CLRMSK                  (0xEFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MH_PARITY_EN                      (0x10000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_LATENT_SHIFT              (4U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_LATENT_CLRMSK             (0xFFFFFFEFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_LATENT_EN                 (0x00000010U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_FAULT_SHIFT               (3U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_FAULT_CLRMSK              (0xFFFFFFF7U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ICS_FAULT_EN                  (0x00000008U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ECC_SHIFT                     (2U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ECC_CLRMSK                    (0xFFFFFFFBU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_ECC_EN                        (0x00000004U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_SHIFT                         (1U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_USC_EN                            (0x00000002U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MCU_L0_SHIFT                      (0U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MCU_L0_CLRMSK                     (0xFFFFFFFEU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DETECT_STATUS_MCU_L0_EN                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_MERCER_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE                                         (0x1278U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MASKFULL                                (IMG_UINT64_C(0x00000000F000001F))
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_BUS_PARITY_SHIFT                        (31U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_BUS_PARITY_CLRMSK                       (0x7FFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_BUS_PARITY_EN                           (0x80000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_SOCIF_DELAY_PARITY_SHIFT                (30U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_SOCIF_DELAY_PARITY_CLRMSK               (0xBFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_SOCIF_DELAY_PARITY_EN                   (0x40000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_CR_PARITY_SHIFT                         (29U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_CR_PARITY_CLRMSK                        (0xDFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_CR_PARITY_EN                            (0x20000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MH_PARITY_SHIFT                         (28U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MH_PARITY_CLRMSK                        (0xEFFFFFFFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MH_PARITY_EN                            (0x10000000U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_LATENT_SHIFT                    (4U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_LATENT_CLRMSK                   (0xFFFFFFEFU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_LATENT_EN                       (0x00000010U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_FAULT_SHIFT                     (3U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_FAULT_CLRMSK                    (0xFFFFFFF7U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ICS_FAULT_EN                        (0x00000008U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ECC_SHIFT                           (2U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ECC_CLRMSK                          (0xFFFFFFFBU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_ECC_EN                              (0x00000004U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_SHIFT                               (1U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_USC_EN                                  (0x00000002U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MCU_L0_SHIFT                            (0U)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MCU_L0_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FAULT_MERCER_REDUCER_DISABLE_MCU_L0_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS
+*/
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS                                      (0x1280U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MASKFULL                             (IMG_UINT64_C(0x00000000FFFC1FFF))
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_BUS_PARITY_SHIFT                     (31U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_BUS_PARITY_CLRMSK                    (0x7FFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_BUS_PARITY_EN                        (0x80000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CR_PARITY_SHIFT                      (30U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CR_PARITY_CLRMSK                     (0xBFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CR_PARITY_EN                         (0x40000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_LATENT_SHIFT                     (29U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_LATENT_CLRMSK                    (0xDFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_LATENT_EN                        (0x20000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_LATENT_SHIFT                     (28U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_LATENT_CLRMSK                    (0xEFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_LATENT_EN                        (0x10000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_LATENT_SHIFT                     (27U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_LATENT_CLRMSK                    (0xF7FFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_LATENT_EN                        (0x08000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_LATENT_SHIFT                     (26U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_LATENT_CLRMSK                    (0xFBFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_LATENT_EN                        (0x04000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_ICS_FAULT_SHIFT                  (25U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_ICS_FAULT_CLRMSK                 (0xFDFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PAP_ICS_FAULT_EN                     (0x02000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_ICS_FAULT_SHIFT                  (24U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_ICS_FAULT_CLRMSK                 (0xFEFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SAP_ICS_FAULT_EN                     (0x01000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_ICS_FAULT_SHIFT                  (23U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_ICS_FAULT_CLRMSK                 (0xFF7FFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CPX_ICS_FAULT_EN                     (0x00800000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_ICS_FAULT_SHIFT                  (22U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_ICS_FAULT_CLRMSK                 (0xFFBFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_MOV_ICS_FAULT_EN                     (0x00400000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_PARITY_FAULT_SHIFT               (21U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_PARITY_FAULT_CLRMSK              (0xFFDFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_PARITY_FAULT_EN                  (0x00200000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_PARITY_FAULT_SHIFT                (20U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_PARITY_FAULT_CLRMSK               (0xFFEFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_PARITY_FAULT_EN                   (0x00100000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_FAULT_DS_SHIFT                 (19U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_FAULT_DS_CLRMSK                (0xFFF7FFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_FAULT_DS_EN                    (0x00080000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_FAULT_DS_SHIFT                 (18U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_FAULT_DS_CLRMSK                (0xFFFBFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_FAULT_DS_EN                    (0x00040000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PRED_STORE_SHIFT                     (12U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PRED_STORE_CLRMSK                    (0xFFFFEFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_PRED_STORE_EN                        (0x00001000U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_SHIFT                            (11U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_CLRMSK                           (0xFFFFF7FFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_DMX_EN                               (0x00000800U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_TS_SHIFT                         (10U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_TS_CLRMSK                        (0xFFFFFBFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_TS_EN                            (0x00000400U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_CRB_SHIFT                        (9U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_CRB_CLRMSK                       (0xFFFFFDFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_TPX_CRB_EN                           (0x00000200U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_IF_SHIFT                       (8U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_IF_CLRMSK                      (0xFFFFFEFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ICTRL_IF_EN                          (0x00000100U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_SHIFT                             (7U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_CLRMSK                            (0xFFFFFF7FU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_US_EN                                (0x00000080U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CEU_SHIFT                            (6U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CEU_CLRMSK                           (0xFFFFFFBFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_CEU_EN                               (0x00000040U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_TS_SHIFT                       (5U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_TS_CLRMSK                      (0xFFFFFFDFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SCHED_TS_EN                          (0x00000020U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_IRS_SHIFT                            (4U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_IRS_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_IRS_EN                               (0x00000010U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_LMS_SHIFT                            (3U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_LMS_CLRMSK                           (0xFFFFFFF7U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_LMS_EN                               (0x00000008U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SHS_SHIFT                            (2U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SHS_CLRMSK                           (0xFFFFFFFBU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_SHS_EN                               (0x00000004U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_COEFF_STORE_SHIFT                    (1U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_COEFF_STORE_CLRMSK                   (0xFFFFFFFDU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_COEFF_STORE_EN                       (0x00000002U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ITR_SHIFT                            (0U)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ITR_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FAULT_USC_REDUCER_DETECT_STATUS_ITR_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_USC_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE                                            (0x1290U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFC1FFF))
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_BUS_PARITY_SHIFT                           (31U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_BUS_PARITY_CLRMSK                          (0x7FFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_BUS_PARITY_EN                              (0x80000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CR_PARITY_SHIFT                            (30U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CR_PARITY_CLRMSK                           (0xBFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CR_PARITY_EN                               (0x40000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_LATENT_SHIFT                           (29U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_LATENT_CLRMSK                          (0xDFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_LATENT_EN                              (0x20000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_LATENT_SHIFT                           (28U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_LATENT_CLRMSK                          (0xEFFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_LATENT_EN                              (0x10000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_LATENT_SHIFT                           (27U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_LATENT_CLRMSK                          (0xF7FFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_LATENT_EN                              (0x08000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_LATENT_SHIFT                           (26U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_LATENT_CLRMSK                          (0xFBFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_LATENT_EN                              (0x04000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_ICS_FAULT_SHIFT                        (25U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_ICS_FAULT_CLRMSK                       (0xFDFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PAP_ICS_FAULT_EN                           (0x02000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_ICS_FAULT_SHIFT                        (24U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_ICS_FAULT_CLRMSK                       (0xFEFFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SAP_ICS_FAULT_EN                           (0x01000000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_ICS_FAULT_SHIFT                        (23U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_ICS_FAULT_CLRMSK                       (0xFF7FFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CPX_ICS_FAULT_EN                           (0x00800000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_ICS_FAULT_SHIFT                        (22U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_ICS_FAULT_CLRMSK                       (0xFFBFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_MOV_ICS_FAULT_EN                           (0x00400000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_PARITY_FAULT_SHIFT                     (21U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_PARITY_FAULT_CLRMSK                    (0xFFDFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_PARITY_FAULT_EN                        (0x00200000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_PARITY_FAULT_SHIFT                      (20U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_PARITY_FAULT_CLRMSK                     (0xFFEFFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_PARITY_FAULT_EN                         (0x00100000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_FAULT_DS_SHIFT                       (19U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_FAULT_DS_CLRMSK                      (0xFFF7FFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_FAULT_DS_EN                          (0x00080000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_FAULT_DS_SHIFT                       (18U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_FAULT_DS_CLRMSK                      (0xFFFBFFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_FAULT_DS_EN                          (0x00040000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PRED_STORE_SHIFT                           (12U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PRED_STORE_CLRMSK                          (0xFFFFEFFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_PRED_STORE_EN                              (0x00001000U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_SHIFT                                  (11U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_CLRMSK                                 (0xFFFFF7FFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_DMX_EN                                     (0x00000800U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_TS_SHIFT                               (10U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_TS_CLRMSK                              (0xFFFFFBFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_TS_EN                                  (0x00000400U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_CRB_SHIFT                              (9U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_CRB_CLRMSK                             (0xFFFFFDFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_TPX_CRB_EN                                 (0x00000200U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_IF_SHIFT                             (8U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_IF_CLRMSK                            (0xFFFFFEFFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ICTRL_IF_EN                                (0x00000100U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_SHIFT                                   (7U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_CLRMSK                                  (0xFFFFFF7FU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_US_EN                                      (0x00000080U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CEU_SHIFT                                  (6U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CEU_CLRMSK                                 (0xFFFFFFBFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_CEU_EN                                     (0x00000040U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_TS_SHIFT                             (5U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_TS_CLRMSK                            (0xFFFFFFDFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SCHED_TS_EN                                (0x00000020U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_IRS_SHIFT                                  (4U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_IRS_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_IRS_EN                                     (0x00000010U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_LMS_SHIFT                                  (3U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_LMS_CLRMSK                                 (0xFFFFFFF7U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_LMS_EN                                     (0x00000008U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SHS_SHIFT                                  (2U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SHS_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_SHS_EN                                     (0x00000004U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_COEFF_STORE_SHIFT                          (1U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_COEFF_STORE_CLRMSK                         (0xFFFFFFFDU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_COEFF_STORE_EN                             (0x00000002U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ITR_SHIFT                                  (0U)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ITR_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_FAULT_USC_REDUCER_DISABLE_ITR_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_CHEST_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE                                          (0x12A8U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_MASKFULL                                 (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_BUS_PARITY_SHIFT                      (3U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_BUS_PARITY_CLRMSK                     (0xFFFFFFF7U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_BUS_PARITY_EN                         (0x00000008U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_PARITY_SHIFT                          (2U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_PARITY_CLRMSK                         (0xFFFFFFFBU)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_CR_PARITY_EN                             (0x00000004U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_BANK_PARITY_SHIFT                        (1U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_BANK_PARITY_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_BANK_PARITY_EN                           (0x00000002U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_OWDB_PARITY_SHIFT                        (0U)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_OWDB_PARITY_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_CR_FAULT_CHEST_REDUCER_DISABLE_OWDB_PARITY_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_SLC_REDUCER_DISABLE
+*/
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE                                            (0x1328U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MASKFULL                                   (IMG_UINT64_C(0x00000000C00007FF))
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_BUS_PARITY_SHIFT                           (31U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_BUS_PARITY_CLRMSK                          (0x7FFFFFFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_BUS_PARITY_EN                              (0x80000000U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_CR_PARITY_SHIFT                            (30U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_CR_PARITY_CLRMSK                           (0xBFFFFFFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_CR_PARITY_EN                               (0x40000000U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPD_CACHE_SHIFT                       (10U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPD_CACHE_CLRMSK                      (0xFFFFFBFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPD_CACHE_EN                          (0x00000400U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPC_CACHE_SHIFT                       (9U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPC_CACHE_CLRMSK                      (0xFFFFFDFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPC_CACHE_EN                          (0x00000200U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPT_CACHE_SHIFT                       (8U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPT_CACHE_CLRMSK                      (0xFFFFFEFFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_HMMU_HPT_CACHE_EN                          (0x00000100U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_DATA_STORE_SHIFT                   (7U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_DATA_STORE_CLRMSK                  (0xFFFFFF7FU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_DATA_STORE_EN                      (0x00000080U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_CAM_SHIFT                          (6U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_CAM_CLRMSK                         (0xFFFFFFBFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PT_CAM_EN                             (0x00000040U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_INST_QUEUE_SHIFT                      (5U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_INST_QUEUE_CLRMSK                     (0xFFFFFFDFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_INST_QUEUE_EN                         (0x00000020U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_DATA_SHIFT                       (4U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_DATA_CLRMSK                      (0xFFFFFFEFU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_DATA_EN                          (0x00000010U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_CAM_SHIFT                        (3U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_CAM_CLRMSK                       (0xFFFFFFF7U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_MMU4_PDPC_CAM_EN                           (0x00000008U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_SPFILTER_SHIFT                         (2U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_SPFILTER_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_SPFILTER_EN                            (0x00000004U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_CCM_SHIFT                              (1U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_CCM_CLRMSK                             (0xFFFFFFFDU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_CCM_EN                                 (0x00000002U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_IWDB_SHIFT                             (0U)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_IWDB_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_FAULT_SLC_REDUCER_DISABLE_SLC_IWDB_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_REDUCER_CLEAR
+*/
+#define RGX_CR_FAULT_REDUCER_CLEAR                                                  (0x1330U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_FAULT_REDUCER_CLEAR_SLC_SHIFT                                        (6U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SLC_CLRMSK                                       (0xFFFFFFBFU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SLC_EN                                           (0x00000040U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_CHEST_SHIFT                                      (5U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_CHEST_CLRMSK                                     (0xFFFFFFDFU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_CHEST_EN                                         (0x00000020U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_JONES_SHIFT                                      (4U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_JONES_CLRMSK                                     (0xFFFFFFEFU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_JONES_EN                                         (0x00000010U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_TEXAS_SHIFT                                      (3U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_TEXAS_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_TEXAS_EN                                         (0x00000008U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SWIFT_SHIFT                                      (2U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SWIFT_CLRMSK                                     (0xFFFFFFFBU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_SWIFT_EN                                         (0x00000004U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MERCER_SHIFT                                     (1U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MERCER_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_MERCER_EN                                        (0x00000002U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_USC_SHIFT                                        (0U)
+#define RGX_CR_FAULT_REDUCER_CLEAR_USC_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_FAULT_REDUCER_CLEAR_USC_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_FIRMWARE_PROCESSOR_LS
+*/
+#define RGX_CR_FIRMWARE_PROCESSOR_LS                                                (0x01A0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__MASKFULL                                 (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_MASKFULL                                       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__SELF_TEST_ENABLE_SHIFT                   (2U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__SELF_TEST_ENABLE_CLRMSK                  (0xFFFFFFF3U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_ENABLE_SHIFT                                   (0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_ENABLE_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__ENABLE_SHIFT                             (0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__ENABLE_CLRMSK                            (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS
+*/
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS                                         (0x01A8U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS_MASKFULL                                (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS_SELF_TEST_DONE_SHIFT                    (0U)
+#define RGX_CR_FIRMWARE_PROCESSOR_LS_STATUS_SELF_TEST_DONE_CLRMSK                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_JONES_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN                                            (0x8C30U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_MASKFULL                                   (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CHEST_SHIFT                            (22U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CHEST_CLRMSK                           (0xFFBFFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CHEST_EN                               (0x00400000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_RED_SHIFT                             (21U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_RED_CLRMSK                            (0xFFDFFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_RED_EN                                (0x00200000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_PRI_SHIFT                             (20U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_PRI_CLRMSK                            (0xFFEFFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DCLS_PRI_EN                                (0x00100000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_FILTER_SHIFT                           (19U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_FILTER_CLRMSK                          (0xFFF7FFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_FILTER_EN                              (0x00080000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PT_PARITY_SHIFT                            (18U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PT_PARITY_CLRMSK                           (0xFFFBFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PT_PARITY_EN                               (0x00040000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_JONES_REGBANK_SHIFT                        (17U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_JONES_REGBANK_CLRMSK                       (0xFFFDFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_JONES_REGBANK_EN                           (0x00020000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SI_AXI2IMG_SHIFT                           (16U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SI_AXI2IMG_CLRMSK                          (0xFFFEFFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SI_AXI2IMG_EN                              (0x00010000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                              (15U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                             (0xFFFF7FFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_REDUCER_EN                                 (0x00008000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI_R_SHIFT                                (14U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI_R_CLRMSK                               (0xFFFFBFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI_R_EN                                   (0x00004000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DFU_SHIFT                                  (13U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DFU_CLRMSK                                 (0xFFFFDFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_DFU_EN                                     (0x00002000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SAFETY_IRQ_SHIFT                           (12U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SAFETY_IRQ_CLRMSK                          (0xFFFFEFFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SAFETY_IRQ_EN                              (0x00001000U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GPU_IRQ_SHIFT                              (11U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GPU_IRQ_CLRMSK                             (0xFFFFF7FFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GPU_IRQ_EN                                 (0x00000800U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI2IMG_SHIFT                              (10U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI2IMG_CLRMSK                             (0xFFFFFBFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_AXI2IMG_EN                                 (0x00000400U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SOCIF_SHIFT                                (9U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SOCIF_CLRMSK                               (0xFFFFFDFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SOCIF_EN                                   (0x00000200U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GARTEN_SHIFT                               (8U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GARTEN_CLRMSK                              (0xFFFFFEFFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_GARTEN_EN                                  (0x00000100U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_TDM_SHIFT                                  (7U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_TDM_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_TDM_EN                                     (0x00000080U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_VERTEX_SHIFT                               (6U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_VERTEX_CLRMSK                              (0xFFFFFFBFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_VERTEX_EN                                  (0x00000040U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PIXEL_SHIFT                                (5U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PIXEL_CLRMSK                               (0xFFFFFFDFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PIXEL_EN                                   (0x00000020U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_COMPUTE_SHIFT                              (4U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_COMPUTE_CLRMSK                             (0xFFFFFFEFU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_COMPUTE_EN                                 (0x00000010U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_FBCDC_SHIFT                                (3U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_FBCDC_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_FBCDC_EN                                   (0x00000008U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PM_SHIFT                                   (2U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PM_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_PM_EN                                      (0x00000004U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_SHIFT                                  (1U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_BIF_EN                                     (0x00000002U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_SHIFT                                  (0U)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_JONES_SAFETY_SELF_TEST_EN_SLC_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN                                            (0x8C38U)
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN_SPU_SHIFT                                  (0U)
+#define RGX_CR_TEXAS_SAFETY_SELF_TEST_EN_SPU_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_SWIFT_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN                                            (0x8C40U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU31_SWIFT0_SHIFT                         (31U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU31_SWIFT0_CLRMSK                        (0x7FFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU31_SWIFT0_EN                            (0x80000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU30_SWIFT0_SHIFT                         (30U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU30_SWIFT0_CLRMSK                        (0xBFFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU30_SWIFT0_EN                            (0x40000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU29_SWIFT0_SHIFT                         (29U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU29_SWIFT0_CLRMSK                        (0xDFFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU29_SWIFT0_EN                            (0x20000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU28_SWIFT0_SHIFT                         (28U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU28_SWIFT0_CLRMSK                        (0xEFFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU28_SWIFT0_EN                            (0x10000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU27_SWIFT0_SHIFT                         (27U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU27_SWIFT0_CLRMSK                        (0xF7FFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU27_SWIFT0_EN                            (0x08000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU26_SWIFT0_SHIFT                         (26U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU26_SWIFT0_CLRMSK                        (0xFBFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU26_SWIFT0_EN                            (0x04000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU25_SWIFT0_SHIFT                         (25U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU25_SWIFT0_CLRMSK                        (0xFDFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU25_SWIFT0_EN                            (0x02000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU24_SWIFT0_SHIFT                         (24U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU24_SWIFT0_CLRMSK                        (0xFEFFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU24_SWIFT0_EN                            (0x01000000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU23_SWIFT0_SHIFT                         (23U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU23_SWIFT0_CLRMSK                        (0xFF7FFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU23_SWIFT0_EN                            (0x00800000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU22_SWIFT0_SHIFT                         (22U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU22_SWIFT0_CLRMSK                        (0xFFBFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU22_SWIFT0_EN                            (0x00400000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU21_SWIFT0_SHIFT                         (21U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU21_SWIFT0_CLRMSK                        (0xFFDFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU21_SWIFT0_EN                            (0x00200000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU20_SWIFT0_SHIFT                         (20U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU20_SWIFT0_CLRMSK                        (0xFFEFFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU20_SWIFT0_EN                            (0x00100000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU19_SWIFT0_SHIFT                         (19U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU19_SWIFT0_CLRMSK                        (0xFFF7FFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU19_SWIFT0_EN                            (0x00080000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU18_SWIFT0_SHIFT                         (18U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU18_SWIFT0_CLRMSK                        (0xFFFBFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU18_SWIFT0_EN                            (0x00040000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU17_SWIFT0_SHIFT                         (17U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU17_SWIFT0_CLRMSK                        (0xFFFDFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU17_SWIFT0_EN                            (0x00020000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU16_SWIFT0_SHIFT                         (16U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU16_SWIFT0_CLRMSK                        (0xFFFEFFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU16_SWIFT0_EN                            (0x00010000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU15_SWIFT0_SHIFT                         (15U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU15_SWIFT0_CLRMSK                        (0xFFFF7FFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU15_SWIFT0_EN                            (0x00008000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU14_SWIFT0_SHIFT                         (14U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU14_SWIFT0_CLRMSK                        (0xFFFFBFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU14_SWIFT0_EN                            (0x00004000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU13_SWIFT0_SHIFT                         (13U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU13_SWIFT0_CLRMSK                        (0xFFFFDFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU13_SWIFT0_EN                            (0x00002000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU12_SWIFT0_SHIFT                         (12U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU12_SWIFT0_CLRMSK                        (0xFFFFEFFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU12_SWIFT0_EN                            (0x00001000U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU11_SWIFT0_SHIFT                         (11U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU11_SWIFT0_CLRMSK                        (0xFFFFF7FFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU11_SWIFT0_EN                            (0x00000800U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU10_SWIFT0_SHIFT                         (10U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU10_SWIFT0_CLRMSK                        (0xFFFFFBFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU10_SWIFT0_EN                            (0x00000400U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU9_SWIFT0_SHIFT                          (9U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU9_SWIFT0_CLRMSK                         (0xFFFFFDFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU9_SWIFT0_EN                             (0x00000200U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU8_SWIFT0_SHIFT                          (8U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU8_SWIFT0_CLRMSK                         (0xFFFFFEFFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU8_SWIFT0_EN                             (0x00000100U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU7_SWIFT0_SHIFT                          (7U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU7_SWIFT0_CLRMSK                         (0xFFFFFF7FU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU7_SWIFT0_EN                             (0x00000080U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU6_SWIFT0_SHIFT                          (6U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU6_SWIFT0_CLRMSK                         (0xFFFFFFBFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU6_SWIFT0_EN                             (0x00000040U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU5_SWIFT0_SHIFT                          (5U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU5_SWIFT0_CLRMSK                         (0xFFFFFFDFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU5_SWIFT0_EN                             (0x00000020U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU4_SWIFT0_SHIFT                          (4U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU4_SWIFT0_CLRMSK                         (0xFFFFFFEFU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU4_SWIFT0_EN                             (0x00000010U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU3_SWIFT0_SHIFT                          (3U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU3_SWIFT0_CLRMSK                         (0xFFFFFFF7U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU3_SWIFT0_EN                             (0x00000008U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU2_SWIFT0_SHIFT                          (2U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU2_SWIFT0_CLRMSK                         (0xFFFFFFFBU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU2_SWIFT0_EN                             (0x00000004U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU1_SWIFT0_SHIFT                          (1U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU1_SWIFT0_CLRMSK                         (0xFFFFFFFDU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU1_SWIFT0_EN                             (0x00000002U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU0_SWIFT0_SHIFT                          (0U)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU0_SWIFT0_CLRMSK                         (0xFFFFFFFEU)
+#define RGX_CR_SWIFT_SAFETY_SELF_TEST_EN_SPU0_SWIFT0_EN                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_MERCER_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN                                           (0x8C48U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_MASKFULL                                  (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER2_SHIFT                       (62U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER2_CLRMSK                      (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER2_EN                          (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER1_SHIFT                       (61U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER1_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER1_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER0_SHIFT                       (60U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER0_CLRMSK                      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU20_MERCER0_EN                          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER2_SHIFT                       (59U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER2_CLRMSK                      (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER2_EN                          (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER1_SHIFT                       (58U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER1_EN                          (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER0_SHIFT                       (57U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU19_MERCER0_EN                          (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER2_SHIFT                       (56U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER2_EN                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER1_SHIFT                       (55U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER1_EN                          (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER0_SHIFT                       (54U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU18_MERCER0_EN                          (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER2_SHIFT                       (53U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER2_EN                          (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER1_SHIFT                       (52U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER1_EN                          (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER0_SHIFT                       (51U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU17_MERCER0_EN                          (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER2_SHIFT                       (50U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER2_EN                          (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER1_SHIFT                       (49U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER1_EN                          (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER0_SHIFT                       (48U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU16_MERCER0_EN                          (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER2_SHIFT                       (47U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER2_EN                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER1_SHIFT                       (46U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER1_EN                          (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER0_SHIFT                       (45U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU15_MERCER0_EN                          (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER2_SHIFT                       (44U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER2_EN                          (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER1_SHIFT                       (43U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER1_EN                          (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER0_SHIFT                       (42U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU14_MERCER0_EN                          (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER2_SHIFT                       (41U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER2_EN                          (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER1_SHIFT                       (40U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER1_EN                          (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER0_SHIFT                       (39U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU13_MERCER0_EN                          (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER2_SHIFT                       (38U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER2_EN                          (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER1_SHIFT                       (37U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER1_EN                          (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER0_SHIFT                       (36U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU12_MERCER0_EN                          (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER2_SHIFT                       (35U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER2_EN                          (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER1_SHIFT                       (34U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER1_EN                          (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER0_SHIFT                       (33U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU11_MERCER0_EN                          (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER2_SHIFT                       (32U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER2_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER2_EN                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER1_SHIFT                       (31U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER1_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER1_EN                          (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER0_SHIFT                       (30U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER0_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU10_MERCER0_EN                          (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER2_SHIFT                        (29U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER2_EN                           (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER1_SHIFT                        (28U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER1_EN                           (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER0_SHIFT                        (27U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU9_MERCER0_EN                           (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER2_SHIFT                        (26U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER2_EN                           (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER1_SHIFT                        (25U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER1_EN                           (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER0_SHIFT                        (24U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU8_MERCER0_EN                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER2_SHIFT                        (23U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER2_EN                           (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER1_SHIFT                        (22U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER1_EN                           (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER0_SHIFT                        (21U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU7_MERCER0_EN                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER2_SHIFT                        (20U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER2_EN                           (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER1_SHIFT                        (19U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER1_EN                           (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER0_SHIFT                        (18U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU6_MERCER0_EN                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER2_SHIFT                        (17U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER2_EN                           (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER1_SHIFT                        (16U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER1_EN                           (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER0_SHIFT                        (15U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU5_MERCER0_EN                           (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER2_SHIFT                        (14U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER2_EN                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER1_SHIFT                        (13U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER1_EN                           (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER0_SHIFT                        (12U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU4_MERCER0_EN                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER2_SHIFT                        (11U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER1_SHIFT                        (10U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER0_SHIFT                        (9U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU3_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER2_SHIFT                        (8U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER1_SHIFT                        (7U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER0_SHIFT                        (6U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU2_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER2_SHIFT                        (5U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER1_SHIFT                        (4U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER0_SHIFT                        (3U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU1_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER2_SHIFT                        (2U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER2_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER2_EN                           (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER1_SHIFT                        (1U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER1_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER1_EN                           (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER0_SHIFT                        (0U)
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER0_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MERCER_SAFETY_SELF_TEST_EN_SPU0_MERCER0_EN                           (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_USC3_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN                                             (0x8C60U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                               (1U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                              (0xFFFFFFFDU)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REGBANK_EN                                  (0x00000002U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                               (0U)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_USC3_SAFETY_SELF_TEST_EN_REDUCER_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_TPU_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN                                              (0x8C68U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_MASKFULL                                     (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                                (1U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REGBANK_EN                                   (0x00000002U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                                (0U)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_TPU_SAFETY_SELF_TEST_EN_REDUCER_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_ISP_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN                                              (0x8C70U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_MASKFULL                                     (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                                (1U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REGBANK_EN                                   (0x00000002U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                                (0U)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_ISP_SAFETY_SELF_TEST_EN_REDUCER_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_SAFETY_SELF_TEST_EN
+*/
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN                                              (0x8C78U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_MASKFULL                                     (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REGBANK_SHIFT                                (1U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REGBANK_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REGBANK_EN                                   (0x00000002U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REDUCER_SHIFT                                (0U)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REDUCER_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_SLC_SAFETY_SELF_TEST_EN_REDUCER_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE
+*/
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE                                        (0x2458U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_MASKFULL                               (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REDUCER_SHIFT                    (3U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REDUCER_CLRMSK                   (0xFFFFFFF7U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REDUCER_EN                       (0x00000008U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REGBANK_SHIFT                    (2U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REGBANK_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_CHEST_REGBANK_EN                       (0x00000004U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_BANK_SHIFT                             (1U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_BANK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_BANK_EN                                (0x00000002U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_OWDB_SHIFT                             (0U)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_OWDB_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_CHEST_PARITY_SELF_TEST_ENABLE_OWDB_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_RAM_STATUS
+*/
+#define RGX_CR_JONES_RAM_STATUS                                                     (0x1148U)
+#define RGX_CR_JONES_RAM_STATUS_MASKFULL                                            (IMG_UINT64_C(0x00000000000003FF))
+#define RGX_CR_JONES_RAM_STATUS_SLC_CHEST_SHIFT                                     (9U)
+#define RGX_CR_JONES_RAM_STATUS_SLC_CHEST_CLRMSK                                    (0xFFFFFDFFU)
+#define RGX_CR_JONES_RAM_STATUS_SLC_CHEST_EN                                        (0x00000200U)
+#define RGX_CR_JONES_RAM_STATUS_GARTEN_SHIFT                                        (8U)
+#define RGX_CR_JONES_RAM_STATUS_GARTEN_CLRMSK                                       (0xFFFFFEFFU)
+#define RGX_CR_JONES_RAM_STATUS_GARTEN_EN                                           (0x00000100U)
+#define RGX_CR_JONES_RAM_STATUS_TDM_SHIFT                                           (7U)
+#define RGX_CR_JONES_RAM_STATUS_TDM_CLRMSK                                          (0xFFFFFF7FU)
+#define RGX_CR_JONES_RAM_STATUS_TDM_EN                                              (0x00000080U)
+#define RGX_CR_JONES_RAM_STATUS_VERTEX_SHIFT                                        (6U)
+#define RGX_CR_JONES_RAM_STATUS_VERTEX_CLRMSK                                       (0xFFFFFFBFU)
+#define RGX_CR_JONES_RAM_STATUS_VERTEX_EN                                           (0x00000040U)
+#define RGX_CR_JONES_RAM_STATUS_PIXEL_SHIFT                                         (5U)
+#define RGX_CR_JONES_RAM_STATUS_PIXEL_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_JONES_RAM_STATUS_PIXEL_EN                                            (0x00000020U)
+#define RGX_CR_JONES_RAM_STATUS_COMPUTE_SHIFT                                       (4U)
+#define RGX_CR_JONES_RAM_STATUS_COMPUTE_CLRMSK                                      (0xFFFFFFEFU)
+#define RGX_CR_JONES_RAM_STATUS_COMPUTE_EN                                          (0x00000010U)
+#define RGX_CR_JONES_RAM_STATUS_FBCDC_SHIFT                                         (3U)
+#define RGX_CR_JONES_RAM_STATUS_FBCDC_CLRMSK                                        (0xFFFFFFF7U)
+#define RGX_CR_JONES_RAM_STATUS_FBCDC_EN                                            (0x00000008U)
+#define RGX_CR_JONES_RAM_STATUS_PM_SHIFT                                            (2U)
+#define RGX_CR_JONES_RAM_STATUS_PM_CLRMSK                                           (0xFFFFFFFBU)
+#define RGX_CR_JONES_RAM_STATUS_PM_EN                                               (0x00000004U)
+#define RGX_CR_JONES_RAM_STATUS_BIF_SHIFT                                           (1U)
+#define RGX_CR_JONES_RAM_STATUS_BIF_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_JONES_RAM_STATUS_BIF_EN                                              (0x00000002U)
+#define RGX_CR_JONES_RAM_STATUS_SLC_SHIFT                                           (0U)
+#define RGX_CR_JONES_RAM_STATUS_SLC_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_JONES_RAM_STATUS_SLC_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_RAM_INIT_KICK
+*/
+#define RGX_CR_JONES_RAM_INIT_KICK                                                  (0x1158U)
+#define RGX_CR_JONES_RAM_INIT_KICK_MASKFULL                                         (IMG_UINT64_C(0x00000000000003FF))
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_SHIFT                                  (9U)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_CLRMSK                                 (0xFFFFFDFFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_EN                                     (0x00000200U)
+#define RGX_CR_JONES_RAM_INIT_KICK_GARTEN_SHIFT                                     (8U)
+#define RGX_CR_JONES_RAM_INIT_KICK_GARTEN_CLRMSK                                    (0xFFFFFEFFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_GARTEN_EN                                        (0x00000100U)
+#define RGX_CR_JONES_RAM_INIT_KICK_TDM_SHIFT                                        (7U)
+#define RGX_CR_JONES_RAM_INIT_KICK_TDM_CLRMSK                                       (0xFFFFFF7FU)
+#define RGX_CR_JONES_RAM_INIT_KICK_TDM_EN                                           (0x00000080U)
+#define RGX_CR_JONES_RAM_INIT_KICK_VERTEX_SHIFT                                     (6U)
+#define RGX_CR_JONES_RAM_INIT_KICK_VERTEX_CLRMSK                                    (0xFFFFFFBFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_VERTEX_EN                                        (0x00000040U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PIXEL_SHIFT                                      (5U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PIXEL_CLRMSK                                     (0xFFFFFFDFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_PIXEL_EN                                         (0x00000020U)
+#define RGX_CR_JONES_RAM_INIT_KICK_COMPUTE_SHIFT                                    (4U)
+#define RGX_CR_JONES_RAM_INIT_KICK_COMPUTE_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_JONES_RAM_INIT_KICK_COMPUTE_EN                                       (0x00000010U)
+#define RGX_CR_JONES_RAM_INIT_KICK_FBCDC_SHIFT                                      (3U)
+#define RGX_CR_JONES_RAM_INIT_KICK_FBCDC_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_JONES_RAM_INIT_KICK_FBCDC_EN                                         (0x00000008U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PM_SHIFT                                         (2U)
+#define RGX_CR_JONES_RAM_INIT_KICK_PM_CLRMSK                                        (0xFFFFFFFBU)
+#define RGX_CR_JONES_RAM_INIT_KICK_PM_EN                                            (0x00000004U)
+#define RGX_CR_JONES_RAM_INIT_KICK_BIF_SHIFT                                        (1U)
+#define RGX_CR_JONES_RAM_INIT_KICK_BIF_CLRMSK                                       (0xFFFFFFFDU)
+#define RGX_CR_JONES_RAM_INIT_KICK_BIF_EN                                           (0x00000002U)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_SHIFT                                        (0U)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_JONES_RAM_INIT_KICK_SLC_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_ARCH_CLK_GATE_ENABLE
+*/
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG                            (0x1208U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__MASKFULL                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__SACG_AND_NOT_CXTP_INFRA__MASKFULL              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_SHIFT                (14U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_ON                   (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU7_AUTO                 (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_SHIFT                (12U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_ON                   (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU6_AUTO                 (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_SHIFT                (10U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_ON                   (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU5_AUTO                 (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_SHIFT                (8U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_ON                   (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU4_AUTO                 (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_SHIFT                (6U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_ON                   (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU3_AUTO                 (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_SHIFT                (4U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_ON                   (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU2_AUTO                 (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_SHIFT                (2U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_ON                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU1_AUTO                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_SHIFT                (0U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_OFF                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_ON                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__CXTP_INFRA_AND_SACG__SPU0_AUTO                 (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__SACG_AND_NOT_CXTP_INFRA__CLUSTER_SHIFT         (0U)
+#define RGX_CR_ARCH_CLK_GATE_ENABLE__SACG_AND_NOT_CXTP_INFRA__CLUSTER_CLRMSK        (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_PM_PARTIAL_RENDER_ENABLE
+*/
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE                                             (0x0338U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_SHIFT                                    (0U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5                                (0x04A0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__MASKFULL                      (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__MASKFULL                      (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__NEED_RESUME_SHIFT             (1U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__NEED_RESUME_CLRMSK            (0xFFFFFFFDU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__NEED_RESUME_EN                (0x00000002U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__NEED_RESUME_SHIFT             (1U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__NEED_RESUME_CLRMSK            (0xFFFFFFFDU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_GT4__NEED_RESUME_EN                (0x00000002U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__COMPLETE_SHIFT                (0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__COMPLETE_CLRMSK               (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5__COMPLETE_EN                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_PDS0                                                     (0x04A8U)
+#define RGX_CR_CDM_CONTEXT_PDS0_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_SHIFT                                     (36U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_CLRMSK                                    (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSIZE                                 (16U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_SHIFT                                     (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_PDS1                                                     (0x04B0U)
+#define RGX_CR_CDM_CONTEXT_PDS1_MASKFULL                                            (IMG_UINT64_C(0x00000001FFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_SHIFT                                   (32U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_EN                                      (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_SHIFT                                   (31U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_EN                                      (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_SHIFT                                        (30U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_EN                                           (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_SHIFT                                  (23U)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC07FFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_ALIGNSHIFT                             (1U)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_ALIGNSIZE                              (2U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_TYPE_SHIFT                                (22U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_TYPE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_TYPE_EN                                   (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_SHIFT                                (11U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFC007FF))
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_ALLOC_SIZE_ALIGNSIZE                            (16U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_SHIFT                                     (6U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_ALIGNSHIFT                                (1U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_ALIGNSIZE                                 (2U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_SHIFT                                     (0U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFC0))
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_ALIGNSHIFT                                (2U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_ALIGNSIZE                                 (4U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0                                                (0x04D8U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_SHIFT                                (36U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_CLRMSK                               (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSIZE                            (16U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_SHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSIZE                            (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1                                                (0x04E0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_MASKFULL                                       (IMG_UINT64_C(0x00000001BFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_SHIFT                              (32U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_EN                                 (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_SHIFT                              (31U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_EN                                 (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_SHIFT                             (23U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFC07FFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_ALIGNSHIFT                        (1U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_ALIGNSIZE                         (2U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_TYPE_SHIFT                           (22U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_TYPE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_TYPE_EN                              (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_SHIFT                           (11U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFC007FF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_ALIGNSHIFT                      (6U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_ALLOC_SIZE_ALIGNSIZE                       (64U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_SHIFT                                (6U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_ALIGNSHIFT                           (1U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_ALIGNSIZE                            (2U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_SHIFT                                (0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFC0))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_ALIGNSHIFT                           (2U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_ALIGNSIZE                            (4U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS
+*/
+#define RGX_CR_CDM_TERMINATE_PDS                                                    (0x04B8U)
+#define RGX_CR_CDM_TERMINATE_PDS_MASKFULL                                           (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_SHIFT                                    (36U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_CLRMSK                                   (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSIZE                                (16U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_SHIFT                                    (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSIZE                                (16U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS1
+*/
+#define RGX_CR_CDM_TERMINATE_PDS1                                                   (0x04C0U)
+#define RGX_CR_CDM_TERMINATE_PDS1_MASKFULL                                          (IMG_UINT64_C(0x00000001BFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_SHIFT                                 (32U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_EN                                    (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_SHIFT                                 (31U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_EN                                    (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_SHIFT                                (23U)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFC07FFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_ALIGNSHIFT                           (1U)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_ALIGNSIZE                            (2U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_TYPE_SHIFT                              (22U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_TYPE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_TYPE_EN                                 (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_SHIFT                              (11U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFC007FF))
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_ALIGNSHIFT                         (6U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_ALLOC_SIZE_ALIGNSIZE                          (64U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_SHIFT                                   (6U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_ALIGNSHIFT                              (1U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_ALIGNSIZE                               (2U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_SHIFT                                   (0U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFC0))
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_ALIGNSHIFT                              (2U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_ALIGNSIZE                               (4U)
+
+
+/*
+    Register group: RGX_CR_SCRATCH, with 16 repeats
+*/
+#define RGX_CR_SCRATCH_REPEATCOUNT                                                  (16U)
+/*
+    Register RGX_CR_SCRATCH0
+*/
+#define RGX_CR_SCRATCH0                                                             (0x0800U)
+#define RGX_CR_SCRATCH0_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH0_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH0_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH1
+*/
+#define RGX_CR_SCRATCH1                                                             (0x0808U)
+#define RGX_CR_SCRATCH1_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH1_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH1_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH2
+*/
+#define RGX_CR_SCRATCH2                                                             (0x0810U)
+#define RGX_CR_SCRATCH2_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH2_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH2_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH3
+*/
+#define RGX_CR_SCRATCH3                                                             (0x0818U)
+#define RGX_CR_SCRATCH3_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH3_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH3_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH4
+*/
+#define RGX_CR_SCRATCH4                                                             (0x0820U)
+#define RGX_CR_SCRATCH4_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH4_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH4_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH5
+*/
+#define RGX_CR_SCRATCH5                                                             (0x0828U)
+#define RGX_CR_SCRATCH5_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH5_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH5_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH6
+*/
+#define RGX_CR_SCRATCH6                                                             (0x0830U)
+#define RGX_CR_SCRATCH6_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH6_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH6_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH7
+*/
+#define RGX_CR_SCRATCH7                                                             (0x0838U)
+#define RGX_CR_SCRATCH7_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH7_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH7_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH8
+*/
+#define RGX_CR_SCRATCH8                                                             (0x0840U)
+#define RGX_CR_SCRATCH8_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH8_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH8_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH9
+*/
+#define RGX_CR_SCRATCH9                                                             (0x0848U)
+#define RGX_CR_SCRATCH9_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH9_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH9_DATA_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH10
+*/
+#define RGX_CR_SCRATCH10                                                            (0x0850U)
+#define RGX_CR_SCRATCH10_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH10_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH10_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH11
+*/
+#define RGX_CR_SCRATCH11                                                            (0x0858U)
+#define RGX_CR_SCRATCH11_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH11_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH11_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH12
+*/
+#define RGX_CR_SCRATCH12                                                            (0x0860U)
+#define RGX_CR_SCRATCH12_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH12_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH12_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH13
+*/
+#define RGX_CR_SCRATCH13                                                            (0x0868U)
+#define RGX_CR_SCRATCH13_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH13_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH13_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH14
+*/
+#define RGX_CR_SCRATCH14                                                            (0x0870U)
+#define RGX_CR_SCRATCH14_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH14_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH14_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SCRATCH15
+*/
+#define RGX_CR_SCRATCH15                                                            (0x0878U)
+#define RGX_CR_SCRATCH15_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SCRATCH15_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH15_DATA_CLRMSK                                                (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_OS0_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS0_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS0_SCRATCH0
+*/
+#define RGX_CR_OS0_SCRATCH0                                                         (0x0880U)
+#define RGX_CR_OS0_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH1
+*/
+#define RGX_CR_OS0_SCRATCH1                                                         (0x0888U)
+#define RGX_CR_OS0_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH2
+*/
+#define RGX_CR_OS0_SCRATCH2                                                         (0x0890U)
+#define RGX_CR_OS0_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH3
+*/
+#define RGX_CR_OS0_SCRATCH3                                                         (0x0898U)
+#define RGX_CR_OS0_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS1_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS1_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS1_SCRATCH0
+*/
+#define RGX_CR_OS1_SCRATCH0                                                         (0x10880U)
+#define RGX_CR_OS1_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH1
+*/
+#define RGX_CR_OS1_SCRATCH1                                                         (0x10888U)
+#define RGX_CR_OS1_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH2
+*/
+#define RGX_CR_OS1_SCRATCH2                                                         (0x10890U)
+#define RGX_CR_OS1_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH3
+*/
+#define RGX_CR_OS1_SCRATCH3                                                         (0x10898U)
+#define RGX_CR_OS1_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS2_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS2_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS2_SCRATCH0
+*/
+#define RGX_CR_OS2_SCRATCH0                                                         (0x20880U)
+#define RGX_CR_OS2_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH1
+*/
+#define RGX_CR_OS2_SCRATCH1                                                         (0x20888U)
+#define RGX_CR_OS2_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH2
+*/
+#define RGX_CR_OS2_SCRATCH2                                                         (0x20890U)
+#define RGX_CR_OS2_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH3
+*/
+#define RGX_CR_OS2_SCRATCH3                                                         (0x20898U)
+#define RGX_CR_OS2_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS3_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS3_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS3_SCRATCH0
+*/
+#define RGX_CR_OS3_SCRATCH0                                                         (0x30880U)
+#define RGX_CR_OS3_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH1
+*/
+#define RGX_CR_OS3_SCRATCH1                                                         (0x30888U)
+#define RGX_CR_OS3_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH2
+*/
+#define RGX_CR_OS3_SCRATCH2                                                         (0x30890U)
+#define RGX_CR_OS3_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH3
+*/
+#define RGX_CR_OS3_SCRATCH3                                                         (0x30898U)
+#define RGX_CR_OS3_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS4_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS4_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS4_SCRATCH0
+*/
+#define RGX_CR_OS4_SCRATCH0                                                         (0x40880U)
+#define RGX_CR_OS4_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH1
+*/
+#define RGX_CR_OS4_SCRATCH1                                                         (0x40888U)
+#define RGX_CR_OS4_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH2
+*/
+#define RGX_CR_OS4_SCRATCH2                                                         (0x40890U)
+#define RGX_CR_OS4_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH3
+*/
+#define RGX_CR_OS4_SCRATCH3                                                         (0x40898U)
+#define RGX_CR_OS4_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS5_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS5_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS5_SCRATCH0
+*/
+#define RGX_CR_OS5_SCRATCH0                                                         (0x50880U)
+#define RGX_CR_OS5_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH1
+*/
+#define RGX_CR_OS5_SCRATCH1                                                         (0x50888U)
+#define RGX_CR_OS5_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH2
+*/
+#define RGX_CR_OS5_SCRATCH2                                                         (0x50890U)
+#define RGX_CR_OS5_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH3
+*/
+#define RGX_CR_OS5_SCRATCH3                                                         (0x50898U)
+#define RGX_CR_OS5_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS6_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS6_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS6_SCRATCH0
+*/
+#define RGX_CR_OS6_SCRATCH0                                                         (0x60880U)
+#define RGX_CR_OS6_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH1
+*/
+#define RGX_CR_OS6_SCRATCH1                                                         (0x60888U)
+#define RGX_CR_OS6_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH2
+*/
+#define RGX_CR_OS6_SCRATCH2                                                         (0x60890U)
+#define RGX_CR_OS6_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH3
+*/
+#define RGX_CR_OS6_SCRATCH3                                                         (0x60898U)
+#define RGX_CR_OS6_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS7_SCRATCH, with 4 repeats
+*/
+#define RGX_CR_OS7_SCRATCH_REPEATCOUNT                                              (4U)
+/*
+    Register RGX_CR_OS7_SCRATCH0
+*/
+#define RGX_CR_OS7_SCRATCH0                                                         (0x70880U)
+#define RGX_CR_OS7_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH1
+*/
+#define RGX_CR_OS7_SCRATCH1                                                         (0x70888U)
+#define RGX_CR_OS7_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH2
+*/
+#define RGX_CR_OS7_SCRATCH2                                                         (0x70890U)
+#define RGX_CR_OS7_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH2_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH3
+*/
+#define RGX_CR_OS7_SCRATCH3                                                         (0x70898U)
+#define RGX_CR_OS7_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH3_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA                        (0xF3000U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAX_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAX_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A00U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAX_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAX_CLRMSK  (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA                        (0x3000U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAX_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAX_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX                                                    (0x0A00U)
+#define RGX_CR_META_SP_MSLVDATAX_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA                        (0xF3040U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA__MSLVDATAT_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A08U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAT_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVDATAT_CLRMSK  (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA                        (0x3040U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA__MSLVDATAT_CLRMSK      (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT                                                    (0x0A08U)
+#define RGX_CR_META_SP_MSLVDATAT_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA                        (0xF3080U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__ADDR_SHIFT            (2U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__ADDR_CLRMSK           (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__AUTOINCR_SHIFT        (1U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__AUTOINCR_CLRMSK       (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__AUTOINCR_EN           (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_SHIFT              (0U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_CLRMSK             (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A10U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__ADDR_SHIFT        (2U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__ADDR_CLRMSK       (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__AUTOINCR_SHIFT    (1U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__AUTOINCR_CLRMSK   (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__AUTOINCR_EN       (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__RD_SHIFT          (0U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__RD_CLRMSK         (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_NOT_MRUA__RD_EN             (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA                        (0x3080U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__ADDR_SHIFT            (2U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__ADDR_CLRMSK           (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__AUTOINCR_SHIFT        (1U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__AUTOINCR_CLRMSK       (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__AUTOINCR_EN           (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_SHIFT              (0U)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_CLRMSK             (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0                                                    (0x0A10U)
+#define RGX_CR_META_SP_MSLVCTRL0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_SHIFT                                         (2U)
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_CLRMSK                                        (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_SHIFT                                     (1U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN                                        (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_SHIFT                                           (0U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA                        (0xF30C0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRTHREAD_SHIFT    (30U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRTHREAD_CLRMSK   (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__LOCK2_INTERLOCK_SHIFT (29U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__LOCK2_INTERLOCK_CLRMSK (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__LOCK2_INTERLOCK_EN    (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__ATOMIC_INTERLOCK_SHIFT (28U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__ATOMIC_INTERLOCK_CLRMSK (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__ATOMIC_INTERLOCK_EN   (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_SHIFT    (26U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_CLRMSK   (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN       (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__COREMEM_IDLE_SHIFT    (25U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__COREMEM_IDLE_CLRMSK   (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__COREMEM_IDLE_EN       (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_SHIFT           (24U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_CLRMSK          (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN              (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRID_SHIFT        (21U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERRID_CLRMSK       (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERR_SHIFT          (20U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERR_CLRMSK         (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__DEFERR_EN             (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__WR_ACTIVE_SHIFT       (18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__WR_ACTIVE_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__WR_ACTIVE_EN          (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__THREAD_SHIFT          (4U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__THREAD_CLRMSK         (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__TRANS_SIZE_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__TRANS_SIZE_CLRMSK     (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__BYTE_ROUND_SHIFT      (0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__BYTE_ROUND_CLRMSK     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA                    (0xF0A18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL          (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRTHREAD_SHIFT (30U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRTHREAD_CLRMSK (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__LOCK2_INTERLOCK_SHIFT (29U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__LOCK2_INTERLOCK_CLRMSK (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__LOCK2_INTERLOCK_EN (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__ATOMIC_INTERLOCK_SHIFT (28U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__ATOMIC_INTERLOCK_CLRMSK (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__ATOMIC_INTERLOCK_EN (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__GBLPORT_IDLE_SHIFT (26U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__GBLPORT_IDLE_CLRMSK (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__GBLPORT_IDLE_EN   (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__COREMEM_IDLE_SHIFT (25U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__COREMEM_IDLE_CLRMSK (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__COREMEM_IDLE_EN   (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__READY_SHIFT       (24U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__READY_CLRMSK      (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__READY_EN          (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRID_SHIFT    (21U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERRID_CLRMSK   (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERR_SHIFT      (20U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERR_CLRMSK     (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__DEFERR_EN         (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__WR_ACTIVE_SHIFT   (18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__WR_ACTIVE_CLRMSK  (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__WR_ACTIVE_EN      (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__THREAD_SHIFT      (4U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__THREAD_CLRMSK     (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__TRANS_SIZE_SHIFT  (2U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__TRANS_SIZE_CLRMSK (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__BYTE_ROUND_SHIFT  (0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_NOT_MRUA__BYTE_ROUND_CLRMSK (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA                        (0x30C0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL              (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRTHREAD_SHIFT    (30U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRTHREAD_CLRMSK   (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__LOCK2_INTERLOCK_SHIFT (29U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__LOCK2_INTERLOCK_CLRMSK (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__LOCK2_INTERLOCK_EN    (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__ATOMIC_INTERLOCK_SHIFT (28U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__ATOMIC_INTERLOCK_CLRMSK (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__ATOMIC_INTERLOCK_EN   (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_SHIFT    (26U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_CLRMSK   (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN       (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__COREMEM_IDLE_SHIFT    (25U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__COREMEM_IDLE_CLRMSK   (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__COREMEM_IDLE_EN       (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_SHIFT           (24U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_CLRMSK          (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN              (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRID_SHIFT        (21U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERRID_CLRMSK       (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERR_SHIFT          (20U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERR_CLRMSK         (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__DEFERR_EN             (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__WR_ACTIVE_SHIFT       (18U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__WR_ACTIVE_CLRMSK      (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__WR_ACTIVE_EN          (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__THREAD_SHIFT          (4U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__THREAD_CLRMSK         (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__TRANS_SIZE_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__TRANS_SIZE_CLRMSK     (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__BYTE_ROUND_SHIFT      (0U)
+#define RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__BYTE_ROUND_CLRMSK     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1                                                    (0x0A18U)
+#define RGX_CR_META_SP_MSLVCTRL1_MASKFULL                                           (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_SHIFT                                 (30U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_CLRMSK                                (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_SHIFT                              (29U)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_EN                                 (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_SHIFT                             (28U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_CLRMSK                            (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_EN                                (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_SHIFT                                 (26U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN                                    (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_SHIFT                                 (25U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_CLRMSK                                (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_EN                                    (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_SHIFT                                        (24U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_CLRMSK                                       (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_EN                                           (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_SHIFT                                     (21U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_CLRMSK                                    (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_SHIFT                                       (20U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_CLRMSK                                      (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_EN                                          (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_SHIFT                                    (18U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_CLRMSK                                   (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_EN                                       (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_SHIFT                                       (4U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_CLRMSK                                      (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_CLRMSK                                  (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_SHIFT                                   (0U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_CLRMSK                                  (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA                     (0xF3280U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__INPUT_SHIFT        (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__INPUT_CLRMSK       (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__OUTPUT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_MRUA__OUTPUT_CLRMSK      (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA                 (0xF0A50U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL       (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__INPUT_SHIFT    (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__INPUT_CLRMSK   (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__OUTPUT_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_GT1_AND_NOT_MRUA__OUTPUT_CLRMSK  (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA                     (0x3280U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__INPUT_SHIFT        (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__INPUT_CLRMSK       (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__OUTPUT_SHIFT       (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE__HOST_SECURITY_EQ1_AND_MRUA__OUTPUT_CLRMSK      (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE                                                 (0x0A50U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_MASKFULL                                        (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_SHIFT                                     (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3400U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0A80U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3400U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT0KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK                                                   (0x0A80U)
+#define RGX_CR_META_SP_MSLVT0KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF3440U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT0KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0A88U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT0KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x3440U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT0KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI                                                  (0x0A88U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3480U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0A90U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3480U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT1KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK                                                   (0x0A90U)
+#define RGX_CR_META_SP_MSLVT1KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF34C0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT1KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0A98U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT1KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x34C0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT1KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI                                                  (0x0A98U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3500U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0AA0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3500U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT2KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK                                                   (0x0AA0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF3540U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT2KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0AA8U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT2KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x3540U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT2KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI                                                  (0x0AA8U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA                       (0xF3580U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA                   (0xF0AB0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICK_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICK_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA                       (0x3580U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICK_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVT3KICK__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICK_CLRMSK    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK                                                   (0x0AB0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA                      (0xF35C0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_MRUA__MSLVT3KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA                  (0xF0AB8U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICKI_SHIFT (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_GT1_AND_NOT_MRUA__MSLVT3KICKI_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA                      (0x35C0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICKI_SHIFT   (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI__HOST_SECURITY_EQ1_AND_MRUA__MSLVT3KICKI_CLRMSK  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI                                                  (0x0AB8U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA                          (0xF3600U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__MASKFULL                (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__SOFTRESET_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__SOFTRESET_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_MRUA__SOFTRESET_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA                      (0xF0AC0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__SOFTRESET_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__SOFTRESET_CLRMSK    (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_GT1_AND_NOT_MRUA__SOFTRESET_EN        (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA                          (0x3600U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL                (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__SOFTRESET_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__SOFTRESET_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST__HOST_SECURITY_EQ1_AND_MRUA__SOFTRESET_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST                                                      (0x0AC0U)
+#define RGX_CR_META_SP_MSLVRST_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA                    (0xF3640U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT3_SHIFT   (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT3_CLRMSK  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT3_EN      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT2_SHIFT   (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT2_CLRMSK  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA__TRIGVECT2_EN      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA                (0xF0AC8U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL      (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT3_SHIFT (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT3_CLRMSK (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT3_EN  (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT2_SHIFT (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT2_CLRMSK (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_NOT_MRUA__TRIGVECT2_EN  (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA                    (0x3640U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT3_SHIFT   (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT3_CLRMSK  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT3_EN      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT2_SHIFT   (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT2_CLRMSK  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA__TRIGVECT2_EN      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS                                                (0x0AC8U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_SHIFT                                (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_EN                                   (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_SHIFT                                (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN                                   (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA                    (0xF3680U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT1_SHIFT      (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT1_CLRMSK     (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT1_EN         (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT0_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT0_CLRMSK     (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA__EVENT0_EN         (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA                (0xF0AD0U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL      (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT1_SHIFT  (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT1_CLRMSK (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT1_EN     (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT0_SHIFT  (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT0_CLRMSK (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_NOT_MRUA__EVENT0_EN     (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA                    (0x3680U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL          (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT1_SHIFT      (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT1_CLRMSK     (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT1_EN         (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT0_SHIFT      (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT0_CLRMSK     (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA__EVENT0_EN         (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE                                                (0x0AD0U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_SHIFT                                   (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_CLRMSK                                  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_EN                                      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_EN                                      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA                     (0xF36C0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MODE_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MODE_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA__MODE_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA                 (0xF0AD8U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MASKFULL       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MODE_SHIFT     (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MODE_CLRMSK    (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_NOT_MRUA__MODE_EN        (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA                     (0x36C0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MASKFULL           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MODE_SHIFT         (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MODE_CLRMSK        (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA__MODE_EN            (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL                                                 (0x0AD8U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_EN                                         (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4, with 32 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4_REPEATCOUNT                (32U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED00
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4                            (0xF8FC0U)
+#define RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED0, with 32 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED0_REPEATCOUNT                                     (32U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED00
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00                                                (0x8FC0U)
+#define RGX_CR_FWCORE_DMI_RESERVED00_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED01
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED01__HOST_SECURITY_GEQ4                            (0xF8FC8U)
+#define RGX_CR_FWCORE_DMI_RESERVED01__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED01
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED01                                                (0x8FC8U)
+#define RGX_CR_FWCORE_DMI_RESERVED01_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED02
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED02__HOST_SECURITY_GEQ4                            (0xF8FD0U)
+#define RGX_CR_FWCORE_DMI_RESERVED02__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED02
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED02                                                (0x8FD0U)
+#define RGX_CR_FWCORE_DMI_RESERVED02_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED03
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED03__HOST_SECURITY_GEQ4                            (0xF8FD8U)
+#define RGX_CR_FWCORE_DMI_RESERVED03__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED03
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED03                                                (0x8FD8U)
+#define RGX_CR_FWCORE_DMI_RESERVED03_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA0
+*/
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4                                 (0xF90C0U)
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4__MASKFULL                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4__VAL_SHIFT                      (0U)
+#define RGX_CR_FWCORE_DMI_DATA0__HOST_SECURITY_GEQ4__VAL_CLRMSK                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA0
+*/
+#define RGX_CR_FWCORE_DMI_DATA0                                                     (0x90C0U)
+#define RGX_CR_FWCORE_DMI_DATA0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4_REPEATCOUNT                (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED10
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4                            (0xF90C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED10__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED1, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED1_REPEATCOUNT                                     (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED10
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10                                                (0x90C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED10_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED11
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED11__HOST_SECURITY_GEQ4                            (0xF90D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED11__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED11
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED11                                                (0x90D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED11_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED12
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED12__HOST_SECURITY_GEQ4                            (0xF90D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED12__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED12
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED12                                                (0x90D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED12_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED13
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED13__HOST_SECURITY_GEQ4                            (0xF90E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED13__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED13
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED13                                                (0x90E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED13_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED14
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED14__HOST_SECURITY_GEQ4                            (0xF90E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED14__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED14
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED14                                                (0x90E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED14_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED15
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED15__HOST_SECURITY_GEQ4                            (0xF90F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED15__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED15
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED15                                                (0x90F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED15_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED16
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED16__HOST_SECURITY_GEQ4                            (0xF90F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED16__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED16
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED16                                                (0x90F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED16_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA1
+*/
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4                                 (0xF9100U)
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4__MASKFULL                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4__VAL_SHIFT                      (0U)
+#define RGX_CR_FWCORE_DMI_DATA1__HOST_SECURITY_GEQ4__VAL_CLRMSK                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA1
+*/
+#define RGX_CR_FWCORE_DMI_DATA1                                                     (0x9100U)
+#define RGX_CR_FWCORE_DMI_DATA1_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4_REPEATCOUNT                (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED20
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4                            (0xF9108U)
+#define RGX_CR_FWCORE_DMI_RESERVED20__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED2, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED2_REPEATCOUNT                                     (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED20
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20                                                (0x9108U)
+#define RGX_CR_FWCORE_DMI_RESERVED20_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED21
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED21__HOST_SECURITY_GEQ4                            (0xF9110U)
+#define RGX_CR_FWCORE_DMI_RESERVED21__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED21
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED21                                                (0x9110U)
+#define RGX_CR_FWCORE_DMI_RESERVED21_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED22
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED22__HOST_SECURITY_GEQ4                            (0xF9118U)
+#define RGX_CR_FWCORE_DMI_RESERVED22__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED22
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED22                                                (0x9118U)
+#define RGX_CR_FWCORE_DMI_RESERVED22_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED23
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED23__HOST_SECURITY_GEQ4                            (0xF9120U)
+#define RGX_CR_FWCORE_DMI_RESERVED23__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED23
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED23                                                (0x9120U)
+#define RGX_CR_FWCORE_DMI_RESERVED23_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4, with 80 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4_REPEATCOUNT                (80U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED30
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4                            (0xF9140U)
+#define RGX_CR_FWCORE_DMI_RESERVED30__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED3, with 80 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED3_REPEATCOUNT                                     (80U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED30
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30                                                (0x9140U)
+#define RGX_CR_FWCORE_DMI_RESERVED30_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED31
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED31__HOST_SECURITY_GEQ4                            (0xF9148U)
+#define RGX_CR_FWCORE_DMI_RESERVED31__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED31
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED31                                                (0x9148U)
+#define RGX_CR_FWCORE_DMI_RESERVED31_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED32
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED32__HOST_SECURITY_GEQ4                            (0xF9150U)
+#define RGX_CR_FWCORE_DMI_RESERVED32__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED32
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED32                                                (0x9150U)
+#define RGX_CR_FWCORE_DMI_RESERVED32_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED33
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED33__HOST_SECURITY_GEQ4                            (0xF9158U)
+#define RGX_CR_FWCORE_DMI_RESERVED33__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED33
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED33                                                (0x9158U)
+#define RGX_CR_FWCORE_DMI_RESERVED33_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED34
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED34__HOST_SECURITY_GEQ4                            (0xF9160U)
+#define RGX_CR_FWCORE_DMI_RESERVED34__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED34
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED34                                                (0x9160U)
+#define RGX_CR_FWCORE_DMI_RESERVED34_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED35
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED35__HOST_SECURITY_GEQ4                            (0xF9168U)
+#define RGX_CR_FWCORE_DMI_RESERVED35__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED35
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED35                                                (0x9168U)
+#define RGX_CR_FWCORE_DMI_RESERVED35_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED36
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED36__HOST_SECURITY_GEQ4                            (0xF9170U)
+#define RGX_CR_FWCORE_DMI_RESERVED36__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED36
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED36                                                (0x9170U)
+#define RGX_CR_FWCORE_DMI_RESERVED36_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED37
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED37__HOST_SECURITY_GEQ4                            (0xF9178U)
+#define RGX_CR_FWCORE_DMI_RESERVED37__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED37
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED37                                                (0x9178U)
+#define RGX_CR_FWCORE_DMI_RESERVED37_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED38
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED38__HOST_SECURITY_GEQ4                            (0xF9180U)
+#define RGX_CR_FWCORE_DMI_RESERVED38__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED38
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED38                                                (0x9180U)
+#define RGX_CR_FWCORE_DMI_RESERVED38_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED39
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED39__HOST_SECURITY_GEQ4                            (0xF9188U)
+#define RGX_CR_FWCORE_DMI_RESERVED39__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED39
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED39                                                (0x9188U)
+#define RGX_CR_FWCORE_DMI_RESERVED39_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED310
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED310__HOST_SECURITY_GEQ4                           (0xF9190U)
+#define RGX_CR_FWCORE_DMI_RESERVED310__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED310
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED310                                               (0x9190U)
+#define RGX_CR_FWCORE_DMI_RESERVED310_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED311
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED311__HOST_SECURITY_GEQ4                           (0xF9198U)
+#define RGX_CR_FWCORE_DMI_RESERVED311__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED311
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED311                                               (0x9198U)
+#define RGX_CR_FWCORE_DMI_RESERVED311_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED312
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED312__HOST_SECURITY_GEQ4                           (0xF91A0U)
+#define RGX_CR_FWCORE_DMI_RESERVED312__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED312
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED312                                               (0x91A0U)
+#define RGX_CR_FWCORE_DMI_RESERVED312_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED313
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED313__HOST_SECURITY_GEQ4                           (0xF91A8U)
+#define RGX_CR_FWCORE_DMI_RESERVED313__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED313
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED313                                               (0x91A8U)
+#define RGX_CR_FWCORE_DMI_RESERVED313_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED314
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED314__HOST_SECURITY_GEQ4                           (0xF91B0U)
+#define RGX_CR_FWCORE_DMI_RESERVED314__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED314
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED314                                               (0x91B0U)
+#define RGX_CR_FWCORE_DMI_RESERVED314_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED315
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED315__HOST_SECURITY_GEQ4                           (0xF91B8U)
+#define RGX_CR_FWCORE_DMI_RESERVED315__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED315
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED315                                               (0x91B8U)
+#define RGX_CR_FWCORE_DMI_RESERVED315_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED316
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED316__HOST_SECURITY_GEQ4                           (0xF91C0U)
+#define RGX_CR_FWCORE_DMI_RESERVED316__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED316
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED316                                               (0x91C0U)
+#define RGX_CR_FWCORE_DMI_RESERVED316_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED317
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED317__HOST_SECURITY_GEQ4                           (0xF91C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED317__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED317
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED317                                               (0x91C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED317_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED318
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED318__HOST_SECURITY_GEQ4                           (0xF91D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED318__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED318
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED318                                               (0x91D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED318_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED319
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED319__HOST_SECURITY_GEQ4                           (0xF91D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED319__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED319
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED319                                               (0x91D8U)
+#define RGX_CR_FWCORE_DMI_RESERVED319_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED320
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED320__HOST_SECURITY_GEQ4                           (0xF91E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED320__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED320
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED320                                               (0x91E0U)
+#define RGX_CR_FWCORE_DMI_RESERVED320_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED321
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED321__HOST_SECURITY_GEQ4                           (0xF91E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED321__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED321
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED321                                               (0x91E8U)
+#define RGX_CR_FWCORE_DMI_RESERVED321_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED322
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED322__HOST_SECURITY_GEQ4                           (0xF91F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED322__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED322
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED322                                               (0x91F0U)
+#define RGX_CR_FWCORE_DMI_RESERVED322_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED323
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED323__HOST_SECURITY_GEQ4                           (0xF91F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED323__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED323
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED323                                               (0x91F8U)
+#define RGX_CR_FWCORE_DMI_RESERVED323_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED324
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED324__HOST_SECURITY_GEQ4                           (0xF9200U)
+#define RGX_CR_FWCORE_DMI_RESERVED324__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED324
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED324                                               (0x9200U)
+#define RGX_CR_FWCORE_DMI_RESERVED324_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED325
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED325__HOST_SECURITY_GEQ4                           (0xF9208U)
+#define RGX_CR_FWCORE_DMI_RESERVED325__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED325
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED325                                               (0x9208U)
+#define RGX_CR_FWCORE_DMI_RESERVED325_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED326
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED326__HOST_SECURITY_GEQ4                           (0xF9210U)
+#define RGX_CR_FWCORE_DMI_RESERVED326__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED326
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED326                                               (0x9210U)
+#define RGX_CR_FWCORE_DMI_RESERVED326_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED327
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED327__HOST_SECURITY_GEQ4                           (0xF9218U)
+#define RGX_CR_FWCORE_DMI_RESERVED327__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED327
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED327                                               (0x9218U)
+#define RGX_CR_FWCORE_DMI_RESERVED327_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED328
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED328__HOST_SECURITY_GEQ4                           (0xF9220U)
+#define RGX_CR_FWCORE_DMI_RESERVED328__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED328
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED328                                               (0x9220U)
+#define RGX_CR_FWCORE_DMI_RESERVED328_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED329
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED329__HOST_SECURITY_GEQ4                           (0xF9228U)
+#define RGX_CR_FWCORE_DMI_RESERVED329__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED329
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED329                                               (0x9228U)
+#define RGX_CR_FWCORE_DMI_RESERVED329_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED330
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED330__HOST_SECURITY_GEQ4                           (0xF9230U)
+#define RGX_CR_FWCORE_DMI_RESERVED330__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED330
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED330                                               (0x9230U)
+#define RGX_CR_FWCORE_DMI_RESERVED330_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED331
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED331__HOST_SECURITY_GEQ4                           (0xF9238U)
+#define RGX_CR_FWCORE_DMI_RESERVED331__HOST_SECURITY_GEQ4__MASKFULL                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED331
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED331                                               (0x9238U)
+#define RGX_CR_FWCORE_DMI_RESERVED331_MASKFULL                                      (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMCONTROL
+*/
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4                             (0xF93C0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__MASKFULL                   (IMG_UINT64_C(0x00000000D0000003))
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__HALTREQ_SHIFT              (31U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__HALTREQ_CLRMSK             (0x7FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__HALTREQ_EN                 (0x80000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__RESUMEREQ_SHIFT            (30U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__RESUMEREQ_CLRMSK           (0xBFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__RESUMEREQ_EN               (0x40000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__ACKHAVERESET_SHIFT         (28U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__ACKHAVERESET_CLRMSK        (0xEFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__ACKHAVERESET_EN            (0x10000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__NDMRESET_SHIFT             (1U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__NDMRESET_CLRMSK            (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__NDMRESET_EN                (0x00000002U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_SHIFT             (0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_CLRMSK            (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_EN                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMCONTROL
+*/
+#define RGX_CR_FWCORE_DMI_DMCONTROL                                                 (0x93C0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_MASKFULL                                        (IMG_UINT64_C(0x00000000D0000003))
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_SHIFT                                   (31U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_CLRMSK                                  (0x7FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN                                      (0x80000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_SHIFT                                 (30U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_CLRMSK                                (0xBFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN                                    (0x40000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_SHIFT                              (28U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_EN                                 (0x10000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_SHIFT                                  (1U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_EN                                     (0x00000002U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN                                     (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4_REPEATCOUNT                (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED40
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4                            (0xF93C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED40__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED4, with 7 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED4_REPEATCOUNT                                     (7U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED40
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED40                                                (0x93C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED40_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED41
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED41__HOST_SECURITY_GEQ4                            (0xF93D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED41__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED41
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED41                                                (0x93D0U)
+#define RGX_CR_FWCORE_DMI_RESERVED41_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMSTATUS
+*/
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4                              (0xF9400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__MASKFULL                    (IMG_UINT64_C(0x00000000004FFFFF))
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__IMPEBREAK_SHIFT             (22U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__IMPEBREAK_CLRMSK            (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__IMPEBREAK_EN                (0x00400000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHAVERESET_SHIFT          (19U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHAVERESET_CLRMSK         (0xFFF7FFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHAVERESET_EN             (0x00080000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHAVERESET_SHIFT          (18U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHAVERESET_CLRMSK         (0xFFFBFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHAVERESET_EN             (0x00040000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRESUMEACK_SHIFT          (17U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRESUMEACK_CLRMSK         (0xFFFDFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRESUMEACK_EN             (0x00020000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRESUMEACK_SHIFT          (16U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRESUMEACK_CLRMSK         (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRESUMEACK_EN             (0x00010000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLNONEXISTENT_SHIFT        (15U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLNONEXISTENT_CLRMSK       (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLNONEXISTENT_EN           (0x00008000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYNONEXISTENT_SHIFT        (14U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYNONEXISTENT_CLRMSK       (0xFFFFBFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYNONEXISTENT_EN           (0x00004000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLUNAVAIL_SHIFT            (13U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLUNAVAIL_CLRMSK           (0xFFFFDFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLUNAVAIL_EN               (0x00002000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYUNAVAIL_SHIFT            (12U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYUNAVAIL_CLRMSK           (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYUNAVAIL_EN               (0x00001000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRUNNING_SHIFT            (11U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRUNNING_CLRMSK           (0xFFFFF7FFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLRUNNING_EN               (0x00000800U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRUNNING_SHIFT            (10U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRUNNING_CLRMSK           (0xFFFFFBFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYRUNNING_EN               (0x00000400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHALTED_SHIFT             (9U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHALTED_CLRMSK            (0xFFFFFDFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ALLHALTED_EN                (0x00000200U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHALTED_SHIFT             (8U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHALTED_CLRMSK            (0xFFFFFEFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__ANYHALTED_EN                (0x00000100U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHENTICATED_SHIFT         (7U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHENTICATED_CLRMSK        (0xFFFFFF7FU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHENTICATED_EN            (0x00000080U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHBUSY_SHIFT              (6U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHBUSY_CLRMSK             (0xFFFFFFBFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__AUTHBUSY_EN                 (0x00000040U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__HASRESETHALTREQ_SHIFT       (5U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__HASRESETHALTREQ_CLRMSK      (0xFFFFFFDFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__HASRESETHALTREQ_EN          (0x00000020U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__CONFSTRPTRVALID_SHIFT       (4U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__CONFSTRPTRVALID_CLRMSK      (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__CONFSTRPTRVALID_EN          (0x00000010U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__VERSION_SHIFT               (0U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS__HOST_SECURITY_GEQ4__VERSION_CLRMSK              (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMSTATUS
+*/
+#define RGX_CR_FWCORE_DMI_DMSTATUS                                                  (0x9400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000004FFFFF))
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_SHIFT                                  (22U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_EN                                     (0x00400000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_SHIFT                               (19U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_EN                                  (0x00080000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_SHIFT                               (18U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_EN                                  (0x00040000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_SHIFT                               (17U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_CLRMSK                              (0xFFFDFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN                                  (0x00020000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_SHIFT                               (16U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_EN                                  (0x00010000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_SHIFT                             (15U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_EN                                (0x00008000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_SHIFT                             (14U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_CLRMSK                            (0xFFFFBFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_EN                                (0x00004000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_SHIFT                                 (13U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_CLRMSK                                (0xFFFFDFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_EN                                    (0x00002000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_SHIFT                                 (12U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_CLRMSK                                (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_EN                                    (0x00001000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_SHIFT                                 (11U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_CLRMSK                                (0xFFFFF7FFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_EN                                    (0x00000800U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_SHIFT                                 (10U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_CLRMSK                                (0xFFFFFBFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_EN                                    (0x00000400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_SHIFT                                  (9U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_CLRMSK                                 (0xFFFFFDFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN                                     (0x00000200U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_SHIFT                                  (8U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_EN                                     (0x00000100U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_SHIFT                              (7U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_CLRMSK                             (0xFFFFFF7FU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_EN                                 (0x00000080U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_SHIFT                                   (6U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_CLRMSK                                  (0xFFFFFFBFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_EN                                      (0x00000040U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_SHIFT                            (5U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_CLRMSK                           (0xFFFFFFDFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_EN                               (0x00000020U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_SHIFT                            (4U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_EN                               (0x00000010U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_SHIFT                                    (0U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_CLRMSK                                   (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_ABSTRACTCS
+*/
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4                            (0xF9540U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x000000001F00170F))
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__PROGBUFSIZE_SHIFT         (24U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__PROGBUFSIZE_CLRMSK        (0xE0FFFFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__BUSY_SHIFT                (12U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__BUSY_CLRMSK               (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__BUSY_EN                   (0x00001000U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__CMDERR_SHIFT              (8U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__CMDERR_CLRMSK             (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__DATACOUNT_SHIFT           (0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS__HOST_SECURITY_GEQ4__DATACOUNT_CLRMSK          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_ABSTRACTCS
+*/
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS                                                (0x9540U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_MASKFULL                                       (IMG_UINT64_C(0x000000001F00170F))
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_SHIFT                              (24U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_CLRMSK                             (0xE0FFFFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_SHIFT                                     (12U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_CLRMSK                                    (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN                                        (0x00001000U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT                                   (8U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK                                  (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_SHIFT                                (0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_CLRMSK                               (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_COMMAND
+*/
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4                               (0xF9580U)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CMDTYPE_SHIFT                (24U)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CMDTYPE_CLRMSK               (0x00FFFFFFU)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CONTROL_SHIFT                (0U)
+#define RGX_CR_FWCORE_DMI_COMMAND__HOST_SECURITY_GEQ4__CONTROL_CLRMSK               (0xFF000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_COMMAND
+*/
+#define RGX_CR_FWCORE_DMI_COMMAND                                                   (0x9580U)
+#define RGX_CR_FWCORE_DMI_COMMAND_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT                                     (24U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_CLRMSK                                    (0x00FFFFFFU)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_CLRMSK                                    (0xFF000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBCS
+*/
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4                                  (0xF9DC0U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__MASKFULL                        (IMG_UINT64_C(0x00000000E07FFFFF))
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBVERSION_SHIFT                 (29U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBVERSION_CLRMSK                (0x1FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSYERROR_SHIFT               (22U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSYERROR_CLRMSK              (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSYERROR_EN                  (0x00400000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSY_SHIFT                    (21U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSY_CLRMSK                   (0xFFDFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBBUSY_EN                       (0x00200000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONADDR_SHIFT              (20U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONADDR_CLRMSK             (0xFFEFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONADDR_EN                 (0x00100000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS_SHIFT                  (17U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS_CLRMSK                 (0xFFF1FFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBAUTOINCREMENT_SHIFT           (16U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBAUTOINCREMENT_CLRMSK          (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBAUTOINCREMENT_EN              (0x00010000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONDATA_SHIFT              (15U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONDATA_CLRMSK             (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBREADONDATA_EN                 (0x00008000U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBERROR_SHIFT                   (12U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBERROR_CLRMSK                  (0xFFFF8FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBASIZE_SHIFT                   (5U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBASIZE_CLRMSK                  (0xFFFFF01FU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS128_SHIFT               (4U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS128_CLRMSK              (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS128_EN                  (0x00000010U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS64_SHIFT                (3U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS64_CLRMSK               (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS64_EN                   (0x00000008U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS32_SHIFT                (2U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS32_CLRMSK               (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS32_EN                   (0x00000004U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS16_SHIFT                (1U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS16_CLRMSK               (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS16_EN                   (0x00000002U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS8_SHIFT                 (0U)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS8_CLRMSK                (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_SBCS__HOST_SECURITY_GEQ4__SBACCESS8_EN                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBCS
+*/
+#define RGX_CR_FWCORE_DMI_SBCS                                                      (0x9DC0U)
+#define RGX_CR_FWCORE_DMI_SBCS_MASKFULL                                             (IMG_UINT64_C(0x00000000E07FFFFF))
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_SHIFT                                      (29U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_CLRMSK                                     (0x1FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_SHIFT                                    (22U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_CLRMSK                                   (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_EN                                       (0x00400000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_SHIFT                                         (21U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN                                            (0x00200000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_SHIFT                                   (20U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN                                      (0x00100000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT                                       (17U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_CLRMSK                                      (0xFFF1FFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_SHIFT                                (16U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_CLRMSK                               (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_EN                                   (0x00010000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_SHIFT                                   (15U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_CLRMSK                                  (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_EN                                      (0x00008000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT                                        (12U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK                                       (0xFFFF8FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_SHIFT                                        (5U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_CLRMSK                                       (0xFFFFF01FU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_SHIFT                                    (4U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_EN                                       (0x00000010U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_SHIFT                                     (3U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_EN                                        (0x00000008U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_SHIFT                                     (2U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_EN                                        (0x00000004U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_SHIFT                                     (1U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_EN                                        (0x00000002U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_SHIFT                                      (0U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBADDRESS0
+*/
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4                            (0xF9E00U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4__MASKFULL                  (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4__ADDRESS_SHIFT             (0U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0__HOST_SECURITY_GEQ4__ADDRESS_CLRMSK            (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBADDRESS0
+*/
+#define RGX_CR_FWCORE_DMI_SBADDRESS0                                                (0x9E00U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA0
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA0__HOST_SECURITY_GEQ4                               (0xF9EC0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA0
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA0                                                   (0x9EC0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA1
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA1__HOST_SECURITY_GEQ4                               (0xF9F00U)
+#define RGX_CR_FWCORE_DMI_SBDATA1__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA1
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA1                                                   (0x9F00U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA2
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA2__HOST_SECURITY_GEQ4                               (0xF9F40U)
+#define RGX_CR_FWCORE_DMI_SBDATA2__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA2
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA2                                                   (0x9F40U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA3
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA3__HOST_SECURITY_GEQ4                               (0xF9F80U)
+#define RGX_CR_FWCORE_DMI_SBDATA3__HOST_SECURITY_GEQ4__MASKFULL                     (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA3
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA3                                                   (0x9F80U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_MASKFULL                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_HALTSUM0
+*/
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4                              (0xF9FC0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4__MASKFULL                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4__VAL_SHIFT                   (0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0__HOST_SECURITY_GEQ4__VAL_CLRMSK                  (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_HALTSUM0
+*/
+#define RGX_CR_FWCORE_DMI_HALTSUM0                                                  (0x9FC0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE
+*/
+#define RGX_CR_MTS_SCHEDULE                                                         (0x0B00U)
+#define RGX_CR_MTS_SCHEDULE_MASKFULL                                                (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE_HOST_SHIFT                                              (8U)
+#define RGX_CR_MTS_SCHEDULE_HOST_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE_HOST_BG_TIMER                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_HOST_HOST                                               (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_SHIFT                                          (6U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_CLRMSK                                         (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT0                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT1                                           (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT2                                           (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT3                                           (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_SHIFT                                           (5U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_CLRMSK                                          (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_BGCTX                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_INTCTX                                          (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE_TASK_SHIFT                                              (4U)
+#define RGX_CR_MTS_SCHEDULE_TASK_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED                                        (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_TASK_COUNTED                                            (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE_DM_SHIFT                                                (0U)
+#define RGX_CR_MTS_SCHEDULE_DM_CLRMSK                                               (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM0                                                  (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM1                                                  (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM2                                                  (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM3                                                  (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM4                                                  (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM5                                                  (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM6                                                  (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM7                                                  (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM_ALL                                               (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE1
+*/
+#define RGX_CR_MTS_SCHEDULE1                                                        (0x10B00U)
+#define RGX_CR_MTS_SCHEDULE1_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE1_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE1_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE1_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE1_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE2
+*/
+#define RGX_CR_MTS_SCHEDULE2                                                        (0x20B00U)
+#define RGX_CR_MTS_SCHEDULE2_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE2_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE2_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE2_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE2_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE3
+*/
+#define RGX_CR_MTS_SCHEDULE3                                                        (0x30B00U)
+#define RGX_CR_MTS_SCHEDULE3_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE3_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE3_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE3_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE3_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE4
+*/
+#define RGX_CR_MTS_SCHEDULE4                                                        (0x40B00U)
+#define RGX_CR_MTS_SCHEDULE4_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE4_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE4_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE4_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE4_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE5
+*/
+#define RGX_CR_MTS_SCHEDULE5                                                        (0x50B00U)
+#define RGX_CR_MTS_SCHEDULE5_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE5_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE5_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE5_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE5_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE6
+*/
+#define RGX_CR_MTS_SCHEDULE6                                                        (0x60B00U)
+#define RGX_CR_MTS_SCHEDULE6_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE6_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE6_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE6_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE6_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE7
+*/
+#define RGX_CR_MTS_SCHEDULE7                                                        (0x70B00U)
+#define RGX_CR_MTS_SCHEDULE7_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE7_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE7_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE7_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE7_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC                                           (0x0B30U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC                                           (0x0B38U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC                                          (0x0B40U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC                                          (0x0B48U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_GARTEN_WRAPPER_CONFIG
+*/
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG                                            (0x0B50U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S8CPR__MASKFULL                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_MASKFULL                                   (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_SLC_COHERENT_SHIFT                   (1U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_SLC_COHERENT_CLRMSK                  (0xFFFFFFFDU)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_SLC_COHERENT_EN                      (0x00000002U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_SHIFT                            (0U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META                             (0x00000000U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_MTS                              (0x00000001U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S8CPR__META                     (0x00000000U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S8CPR__MTS                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_DM0_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE                                             (0x0B58U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM1_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE                                             (0x0B60U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM2_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE                                             (0x0B68U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM3_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE                                             (0x0B70U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM4_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE                                             (0x0B78U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM5_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE                                             (0x0B80U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM6_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE                                             (0xD138U)
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM6_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM7_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE                                             (0xD140U)
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM7_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX
+*/
+#define RGX_CR_MTS_INTCTX                                                           (0x0B98U)
+#define RGX_CR_MTS_INTCTX_MASKFULL                                                  (IMG_UINT64_C(0x000000003FC0FFFF))
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_SHIFT                                    (22U)
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_CLRMSK                                   (0xC03FFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_SHIFT                                   (8U)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_CLRMSK                                  (0xFFFF00FFU)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX
+*/
+#define RGX_CR_MTS_BGCTX                                                            (0x0BA0U)
+#define RGX_CR_MTS_BGCTX_MASKFULL                                                   (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE
+*/
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE                                           (0x0BA8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_SHIFT                                 (56U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_CLRMSK                                (IMG_UINT64_C(0x00FFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_SHIFT                                 (48U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_CLRMSK                                (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_SHIFT                                 (40U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_CLRMSK                                (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_SHIFT                                 (32U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_CLRMSK                                (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_SHIFT                                 (24U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_SHIFT                                 (16U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_SHIFT                                 (8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_SHIFT                                 (0U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MTS_GPU_INT_STATUS
+*/
+#define RGX_CR_MTS_GPU_INT_STATUS                                                   (0x0BB0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_SHIFT                                      (0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS0_EVENT_STATUS                                                 (0x0BD0U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR                                                  (0x0BE0U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS1_EVENT_STATUS                                                 (0x10BD0U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR                                                  (0x10BE0U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS2_EVENT_STATUS                                                 (0x20BD0U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR                                                  (0x20BE0U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS3_EVENT_STATUS                                                 (0x30BD0U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR                                                  (0x30BE0U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS4_EVENT_STATUS                                                 (0x40BD0U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR                                                  (0x40BE0U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS5_EVENT_STATUS                                                 (0x50BD0U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR                                                  (0x50BE0U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS6_EVENT_STATUS                                                 (0x60BD0U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR                                                  (0x60BE0U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS7_EVENT_STATUS                                                 (0x70BD0U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR                                                  (0x70BE0U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE_ENABLE
+*/
+#define RGX_CR_MTS_SCHEDULE_ENABLE                                                  (0x0BD8U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_SHIFT                                       (0U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_CLRMSK                                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_FWCORE_BOOT
+*/
+#define RGX_CR_FWCORE_BOOT                                                          (0x70C0U)
+#define RGX_CR_FWCORE_BOOT_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_BOOT_MODE_SHIFT                                               (0U)
+#define RGX_CR_FWCORE_BOOT_MODE_CLRMSK                                              (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_BOOT_MODE_EN                                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_BOOT
+*/
+#define RGX_CR_META_BOOT                                                            (0x0BF8U)
+#define RGX_CR_META_BOOT_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_BOOT_MODE_SHIFT                                                 (0U)
+#define RGX_CR_META_BOOT_MODE_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_META_BOOT_MODE_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_GARTEN_SLC
+*/
+#define RGX_CR_GARTEN_SLC                                                           (0x0BB8U)
+#define RGX_CR_GARTEN_SLC_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_SHIFT                                     (0U)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_ISP_RENDER
+*/
+#define RGX_CR_ISP_RENDER                                                           (0x0F08U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MASKFULL                             (IMG_UINT64_C(0x000000000003FFFF))
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__MASKFULL                       (IMG_UINT64_C(0x000000000007FF53))
+#define RGX_CR_ISP_RENDER_MASKFULL                                                  (IMG_UINT64_C(0x000000000003FFF0))
+#define RGX_CR_ISP_RENDER_TILES_PER_ISP_SHIFT                                       (16U)
+#define RGX_CR_ISP_RENDER_TILES_PER_ISP_CLRMSK                                      (0xFFFCFFFFU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__TILES_PER_ISP_SHIFT            (16U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__TILES_PER_ISP_CLRMSK           (0xFFF8FFFFU)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_HIGH_SHIFT                                     (12U)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_HIGH_CLRMSK                                    (0xFFFF0FFFU)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_LOW_SHIFT                                      (8U)
+#define RGX_CR_ISP_RENDER_TILE_LIMIT_LOW_CLRMSK                                     (0xFFFFF0FFU)
+#define RGX_CR_ISP_RENDER_TILE_STARVATION_SHIFT                                     (7U)
+#define RGX_CR_ISP_RENDER_TILE_STARVATION_CLRMSK                                    (0xFFFFFF7FU)
+#define RGX_CR_ISP_RENDER_TILE_STARVATION_EN                                        (0x00000080U)
+#define RGX_CR_ISP_RENDER_PROCESS_EMPTY_TILES_SHIFT                                 (6U)
+#define RGX_CR_ISP_RENDER_PROCESS_EMPTY_TILES_CLRMSK                                (0xFFFFFFBFU)
+#define RGX_CR_ISP_RENDER_PROCESS_EMPTY_TILES_EN                                    (0x00000040U)
+#define RGX_CR_ISP_RENDER_RESUME_SHIFT                                              (4U)
+#define RGX_CR_ISP_RENDER_RESUME_CLRMSK                                             (0xFFFFFFCFU)
+#define RGX_CR_ISP_RENDER_RESUME_CONTEXT_NONE                                       (0x00000000U)
+#define RGX_CR_ISP_RENDER_RESUME_CONTEXT_TILE                                       (0x00000010U)
+#define RGX_CR_ISP_RENDER_RESUME_CONTEXT_PBLK                                       (0x00000030U)
+#define RGX_CR_ISP_RENDER_RESUME__CS3DL_EQ4_AND_PIPEDM_EQ0__CONTEXT_NONE            (0x00000000U)
+#define RGX_CR_ISP_RENDER_RESUME__CS3DL_EQ4_AND_PIPEDM_EQ0__CONTEXT_RESUME          (0x00000010U)
+#define RGX_CR_ISP_RENDER_RESUME__IFR_AND_PIPEDM_EQ0__CONTEXT_NONE                  (0x00000000U)
+#define RGX_CR_ISP_RENDER_RESUME__IFR_AND_PIPEDM_EQ0__CONTEXT_TILE                  (0x00000010U)
+#define RGX_CR_ISP_RENDER_RESUME__IFR_AND_PIPEDM_EQ0__CONTEXT_PBLK                  (0x00000030U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_SHIFT                   (4U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_CLRMSK                  (0xFFFFFFEFU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_CONTEXT_NONE            (0x00000000U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__RESUME_CONTEXT_RESUME          (0x00000010U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_SHIFT                            (2U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_CLRMSK                           (0xFFFFFFF3U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_TL2BR                            (0x00000000U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_TR2BL                            (0x00000004U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_BL2TR                            (0x00000008U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__DIR_BR2TL                            (0x0000000CU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_PROTECTED_TILES_SHIFT  (1U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_PROTECTED_TILES_CLRMSK (0xFFFFFFFDU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_PROTECTED_TILES_EN     (0x00000002U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_SHIFT                           (0U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_CLRMSK                          (0xFFFFFFFCU)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_NORM                            (0x00000000U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_FAST_2D                         (0x00000002U)
+#define RGX_CR_ISP_RENDER__IFR_AND_PIPEDM_EQ0__MODE_FAST_SCALE                      (0x00000003U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_UNPROTECTED_TILES_SHIFT (0U)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_UNPROTECTED_TILES_CLRMSK (0xFFFFFFFEU)
+#define RGX_CR_ISP_RENDER__CS3DL_EQ4_AND_PIPEDM_EQ0__PROCESS_UNPROTECTED_TILES_EN   (0x00000001U)
+
+
+/*
+    Register RGX_CR_ISP_CTL
+*/
+#define RGX_CR_ISP_CTL                                                              (0x0FB0U)
+#define RGX_CR_ISP_CTL_MASKFULL                                                     (IMG_UINT64_C(0x00000000007BF8FF))
+#define RGX_CR_ISP_CTL_DBUFFER_COUNT_SHIFT                                          (20U)
+#define RGX_CR_ISP_CTL_DBUFFER_COUNT_CLRMSK                                         (0xFF8FFFFFU)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_SHIFT                                     (19U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_CLRMSK                                    (0xFFF7FFFFU)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_EN                                        (0x00080000U)
+#define RGX_CR_ISP_CTL_UPFRONT_DEPTH_DISABLE_SHIFT                                  (17U)
+#define RGX_CR_ISP_CTL_UPFRONT_DEPTH_DISABLE_CLRMSK                                 (0xFFFDFFFFU)
+#define RGX_CR_ISP_CTL_UPFRONT_DEPTH_DISABLE_EN                                     (0x00020000U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ONE_SHIFT                                        (16U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ONE_CLRMSK                                       (0xFFFEFFFFU)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ONE_EN                                           (0x00010000U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ZERO_SHIFT                                       (15U)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ZERO_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_ISP_CTL_DEPTH_CLAMP_ZERO_EN                                          (0x00008000U)
+#define RGX_CR_ISP_CTL_LINE_SAMPLE_SHIFT                                            (14U)
+#define RGX_CR_ISP_CTL_LINE_SAMPLE_CLRMSK                                           (0xFFFFBFFFU)
+#define RGX_CR_ISP_CTL_LINE_SAMPLE_EN                                               (0x00004000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SHIFT                                             (13U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_CLRMSK                                            (0xFFFFDFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_EN                                                (0x00002000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SINGLE_PIXEL_SHIFT                                (12U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SINGLE_PIXEL_CLRMSK                               (0xFFFFEFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SINGLE_PIXEL_EN                                   (0x00001000U)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_SHIFT                                           (11U)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_CLRMSK                                          (0xFFFFF7FFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_EN                                              (0x00000800U)
+#define RGX_CR_ISP_CTL_UPASS_START_SHIFT                                            (0U)
+#define RGX_CR_ISP_CTL_UPASS_START_CLRMSK                                           (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_MEM_TILING_CFG, with 8 repeats
+*/
+#define RGX_CR_MEM_TILING_CFG_REPEATCOUNT                                           (8U)
+/*
+    Register RGX_CR_MEM_TILING_CFG0
+*/
+#define RGX_CR_MEM_TILING_CFG0                                                      (0x12D8U)
+#define RGX_CR_MEM_TILING_CFG0_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG0_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG0_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG0_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG0_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG1
+*/
+#define RGX_CR_MEM_TILING_CFG1                                                      (0x12E0U)
+#define RGX_CR_MEM_TILING_CFG1_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG1_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG1_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG1_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG1_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG2
+*/
+#define RGX_CR_MEM_TILING_CFG2                                                      (0x12E8U)
+#define RGX_CR_MEM_TILING_CFG2_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG2_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG2_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG2_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG2_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG3
+*/
+#define RGX_CR_MEM_TILING_CFG3                                                      (0x12F0U)
+#define RGX_CR_MEM_TILING_CFG3_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG3_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG3_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG3_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG3_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG4
+*/
+#define RGX_CR_MEM_TILING_CFG4                                                      (0x12F8U)
+#define RGX_CR_MEM_TILING_CFG4_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG4_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG4_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG4_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG4_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG5
+*/
+#define RGX_CR_MEM_TILING_CFG5                                                      (0x1300U)
+#define RGX_CR_MEM_TILING_CFG5_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG5_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG5_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG5_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG5_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG6
+*/
+#define RGX_CR_MEM_TILING_CFG6                                                      (0x1308U)
+#define RGX_CR_MEM_TILING_CFG6_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG6_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG6_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG6_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG6_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MEM_TILING_CFG7
+*/
+#define RGX_CR_MEM_TILING_CFG7                                                      (0x1310U)
+#define RGX_CR_MEM_TILING_CFG7_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_MEM_TILING_CFG7_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_MEM_TILING_CFG7_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG7_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_MEM_TILING_CFG7_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_USC_TIMER
+*/
+#define RGX_CR_USC_TIMER                                                            (0x46C8U)
+#define RGX_CR_USC_TIMER_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_USC_TIMER_CNT_SHIFT                                                  (0U)
+#define RGX_CR_USC_TIMER_CNT_CLRMSK                                                 (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_USC_TIMER_CNT
+*/
+#define RGX_CR_USC_TIMER_CNT                                                        (0x46D0U)
+#define RGX_CR_USC_TIMER_CNT_MASKFULL                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_USC_TIMER_CNT_RESET_SHIFT                                            (0U)
+#define RGX_CR_USC_TIMER_CNT_RESET_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_USC_TIMER_CNT_RESET_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_PAPSAG_FAULT_INJECT
+*/
+#define RGX_CR_PAPSAG_FAULT_INJECT                                                  (0x4D00U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_SHIFT                                      (0U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_CLRMSK                                     (0xFFFFFFFCU)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_DISABLE                                    (0x00000000U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_STUCK0                                     (0x00000001U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_STUCK1                                     (0x00000002U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_VALUE_RESERVED                                   (0x00000003U)
+
+
+/*
+    Register RGX_CR_PAPSAG_FAULT_INJECT_SAFETY
+*/
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY                                           (0x4D08U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_MASKFULL                                  (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_SHIFT                               (0U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_CLRMSK                              (0xFFFFFFFCU)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_DISABLE                             (0x00000000U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_STUCK0                              (0x00000001U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_STUCK1                              (0x00000002U)
+#define RGX_CR_PAPSAG_FAULT_INJECT_SAFETY_VALUE_RESERVED                            (0x00000003U)
+
+
+/*
+    Register RGX_CR_TE_CHECKSUM
+*/
+#define RGX_CR_TE_CHECKSUM                                                          (0x5110U)
+#define RGX_CR_TE_CHECKSUM_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_CHECKSUM_VALUE_SHIFT                                              (0U)
+#define RGX_CR_TE_CHECKSUM_VALUE_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVB_CHECKSUM
+*/
+#define RGX_CR_USC_UVB_CHECKSUM                                                     (0x5118U)
+#define RGX_CR_USC_UVB_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_TMA_CHECKSUM
+*/
+#define RGX_CR_TE_TMA_CHECKSUM                                                      (0x5128U)
+#define RGX_CR_TE_TMA_CHECKSUM_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_TMA_CHECKSUM_VALUE_SHIFT                                          (0U)
+#define RGX_CR_TE_TMA_CHECKSUM_VALUE_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_CDM_PDS_CHECKSUM
+*/
+#define RGX_CR_CDM_PDS_CHECKSUM                                                     (0x5130U)
+#define RGX_CR_CDM_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CDM_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_CDM_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_VCE_CHECKSUM
+*/
+#define RGX_CR_VCE_CHECKSUM                                                         (0x5030U)
+#define RGX_CR_VCE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VCE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_VCE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_PDS_CHECKSUM
+*/
+#define RGX_CR_ISP_PDS_CHECKSUM                                                     (0x5038U)
+#define RGX_CR_ISP_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_TPF_CHECKSUM
+*/
+#define RGX_CR_ISP_TPF_CHECKSUM                                                     (0x5040U)
+#define RGX_CR_ISP_TPF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_CHECKSUM
+*/
+#define RGX_CR_TFPU_CHECKSUM                                                        (0x5048U)
+#define RGX_CR_TFPU_CHECKSUM_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_CHECKSUM_VALUE_SHIFT                                            (0U)
+#define RGX_CR_TFPU_CHECKSUM_VALUE_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_ZLS_CHECKSUM
+*/
+#define RGX_CR_ZLS_CHECKSUM                                                         (0x5050U)
+#define RGX_CR_ZLS_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ZLS_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_ZLS_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM_3D
+*/
+#define RGX_CR_PBE_CHECKSUM_3D__PC2                                                 (0x5058U)
+#define RGX_CR_PBE_CHECKSUM_3D__PC2__MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_3D__PC2__VALUE_SHIFT                                    (0U)
+#define RGX_CR_PBE_CHECKSUM_3D__PC2__VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM
+*/
+#define RGX_CR_PBE_CHECKSUM                                                         (0x5058U)
+#define RGX_CR_PBE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_PBE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_PDS_DOUTM_STM_CHECKSUM
+*/
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM                                               (0x5060U)
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM_MASKFULL                                      (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM_VALUE_SHIFT                                   (0U)
+#define RGX_CR_PDS_DOUTM_STM_CHECKSUM_VALUE_CLRMSK                                  (0x00000000U)
+
+
+/*
+    Register RGX_CR_IFPU_ISP_CHECKSUM
+*/
+#define RGX_CR_IFPU_ISP_CHECKSUM                                                    (0x5068U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_CLIP_CHECKSUM
+*/
+#define RGX_CR_PPP_CLIP_CHECKSUM                                                    (0x5120U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_VCE_PRIM_CHECKSUM
+*/
+#define RGX_CR_VCE_PRIM_CHECKSUM                                                    (0x5140U)
+#define RGX_CR_VCE_PRIM_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VCE_PRIM_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_VCE_PRIM_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_TDM_PDS_CHECKSUM
+*/
+#define RGX_CR_TDM_PDS_CHECKSUM                                                     (0x5148U)
+#define RGX_CR_TDM_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TDM_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_TDM_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM_2D
+*/
+#define RGX_CR_PBE_CHECKSUM_2D                                                      (0x5158U)
+#define RGX_CR_PBE_CHECKSUM_2D_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_2D_VALUE_SHIFT                                          (0U)
+#define RGX_CR_PBE_CHECKSUM_2D_VALUE_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_GEOM
+*/
+#define RGX_CR_PERF_PHASE_GEOM                                                      (0x6008U)
+#define RGX_CR_PERF_PHASE_GEOM__PIPEDM_EQ0__MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_GEOM_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_GEOM_COUNT_SHIFT                                          (0U)
+#define RGX_CR_PERF_PHASE_GEOM_COUNT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_FRAG
+*/
+#define RGX_CR_PERF_PHASE_FRAG                                                      (0x6010U)
+#define RGX_CR_PERF_PHASE_FRAG__PIPEDM_EQ0__MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_FRAG_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_FRAG_COUNT_SHIFT                                          (0U)
+#define RGX_CR_PERF_PHASE_FRAG_COUNT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_COMP
+*/
+#define RGX_CR_PERF_PHASE_COMP                                                      (0x6018U)
+#define RGX_CR_PERF_PHASE_COMP__PIPEDM_EQ0__MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_COMP_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_COMP_COUNT_SHIFT                                          (0U)
+#define RGX_CR_PERF_PHASE_COMP_COUNT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_GEOM_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL                                                (0x6020U)
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_CYCLE_GEOM_TOTAL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_FRAG_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL                                                (0x6028U)
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_CYCLE_FRAG_TOTAL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_COMP_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL                                                (0x6030U)
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_CYCLE_COMP_TOTAL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL                                        (0x6038U)
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL_COUNT_SHIFT                            (0U)
+#define RGX_CR_PERF_CYCLE_GEOM_OR_FRAG_TOTAL_COUNT_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_PHASE_2D
+*/
+#define RGX_CR_PERF_PHASE_2D                                                        (0x6050U)
+#define RGX_CR_PERF_PHASE_2D__PIPEDM_EQ0__MASKFULL                                  (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_2D_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_PHASE_2D_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_PHASE_2D_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_2D_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_2D_TOTAL                                                  (0x6058U)
+#define RGX_CR_PERF_CYCLE_2D_TOTAL_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_2D_TOTAL_COUNT_SHIFT                                      (0U)
+#define RGX_CR_PERF_CYCLE_2D_TOTAL_COUNT_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_READ_STALL
+*/
+#define RGX_CR_PERF_SLC0_READ_STALL                                                 (0x60B8U)
+#define RGX_CR_PERF_SLC0_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC0_WRITE_STALL                                                (0x60C0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_READ_STALL
+*/
+#define RGX_CR_PERF_SLC1_READ_STALL                                                 (0x60E0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC1_WRITE_STALL                                                (0x60E8U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_READ_STALL
+*/
+#define RGX_CR_PERF_SLC2_READ_STALL                                                 (0x6158U)
+#define RGX_CR_PERF_SLC2_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC2_WRITE_STALL                                                (0x6160U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_READ_STALL
+*/
+#define RGX_CR_PERF_SLC3_READ_STALL                                                 (0x6180U)
+#define RGX_CR_PERF_SLC3_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC3_WRITE_STALL                                                (0x6188U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL
+*/
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL                                       (0x6408U)
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL_COUNT_SHIFT                           (0U)
+#define RGX_CR_PERF_CYCLE_GEOM_AND_FRAG_TOTAL_COUNT_CLRMSK                          (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG0
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4                        (0xF4000U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register group: RGX_CR_FWCORE_ADDR_REMAP_CONFIG, with 16 repeats
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG_REPEATCOUNT                                 (16U)
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG0
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0                                            (0x7020U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG1
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4                        (0xF4008U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG1
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1                                            (0x7028U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG2
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4                        (0xF4010U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG2
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2                                            (0x7030U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG3
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4                        (0xF4018U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG3
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3                                            (0x7038U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG4
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4                        (0xF4020U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG4
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4                                            (0x7040U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG5
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4                        (0xF4028U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG5
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5                                            (0x7048U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG6
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4                        (0xF4030U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG6
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6                                            (0x7050U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG7
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4                        (0xF4038U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG7
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7                                            (0x7058U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG8
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4                        (0xF4040U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG8
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8                                            (0x7060U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG9
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4                        (0xF4048U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__MASKFULL              (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT   (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK  (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN      (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT        (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK       (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__FETCH_EN_EN           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__SIZE_SHIFT            (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__SIZE_CLRMSK           (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK       (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT     (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG9
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9                                            (0x7068U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MASKFULL                                   (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MMU_CONTEXT_SHIFT                          (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MMU_CONTEXT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG10
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4                       (0xF4050U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG10
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10                                           (0x7070U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG11
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4                       (0xF4058U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG11
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11                                           (0x7078U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG12
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4                       (0xF4060U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG12
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12                                           (0x7080U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG13
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4                       (0xF4068U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG13
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13                                           (0x7088U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG14
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4                       (0xF4070U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG14
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14                                           (0x7090U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG15
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4                       (0xF4078U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__MASKFULL             (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__LOAD_STORE_EN_SHIFT  (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__LOAD_STORE_EN_CLRMSK (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__LOAD_STORE_EN_EN     (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__FETCH_EN_SHIFT       (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__FETCH_EN_CLRMSK      (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__FETCH_EN_EN          (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__SIZE_SHIFT           (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__SIZE_CLRMSK          (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__DEVVADDR_SHIFT       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__DEVVADDR_CLRMSK      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__MMU_CONTEXT_SHIFT    (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15__HOST_SECURITY_GEQ4__MMU_CONTEXT_CLRMSK   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG15
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15                                           (0x7098U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MASKFULL                                  (IMG_UINT64_C(0x3FFFF0FFFFFFF0FF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MMU_CONTEXT_SHIFT                         (0U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MMU_CONTEXT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_ENABLE
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE                                            (0x70A0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_MASKFULL                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_EVENT_EN_SHIFT                             (0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_EVENT_EN_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ENABLE_EVENT_EN_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT                                             (0x70A8U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_SHIFT                            (0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_RESET_VECTOR
+*/
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR                                          (0x70B0U)
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR_MASKFULL                                 (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR_ADDR_SHIFT                               (1U)
+#define RGX_CR_FWCORE_WRAPPER_RESET_VECTOR_ADDR_CLRMSK                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_VECTOR
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR                                            (0x70B8U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR_ADDR_SHIFT                                 (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_VECTOR_ADDR_CLRMSK                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_IDLE
+*/
+#define RGX_CR_JONES_IDLE                                                           (0x8328U)
+#define RGX_CR_JONES_IDLE_MASKFULL                                                  (IMG_UINT64_C(0x00000000000FFEFF))
+#define RGX_CR_JONES_IDLE_SI_AXI2IMG_SHIFT                                          (19U)
+#define RGX_CR_JONES_IDLE_SI_AXI2IMG_CLRMSK                                         (0xFFF7FFFFU)
+#define RGX_CR_JONES_IDLE_SI_AXI2IMG_EN                                             (0x00080000U)
+#define RGX_CR_JONES_IDLE_ASC_SHIFT                                                 (18U)
+#define RGX_CR_JONES_IDLE_ASC_CLRMSK                                                (0xFFFBFFFFU)
+#define RGX_CR_JONES_IDLE_ASC_EN                                                    (0x00040000U)
+#define RGX_CR_JONES_IDLE_RCE_SHIFT                                                 (17U)
+#define RGX_CR_JONES_IDLE_RCE_CLRMSK                                                (0xFFFDFFFFU)
+#define RGX_CR_JONES_IDLE_RCE_EN                                                    (0x00020000U)
+#define RGX_CR_JONES_IDLE_AXI2IMG_SHIFT                                             (16U)
+#define RGX_CR_JONES_IDLE_AXI2IMG_CLRMSK                                            (0xFFFEFFFFU)
+#define RGX_CR_JONES_IDLE_AXI2IMG_EN                                                (0x00010000U)
+#define RGX_CR_JONES_IDLE_SLC_SHIFT                                                 (15U)
+#define RGX_CR_JONES_IDLE_SLC_CLRMSK                                                (0xFFFF7FFFU)
+#define RGX_CR_JONES_IDLE_SLC_EN                                                    (0x00008000U)
+#define RGX_CR_JONES_IDLE_TDM_SHIFT                                                 (14U)
+#define RGX_CR_JONES_IDLE_TDM_CLRMSK                                                (0xFFFFBFFFU)
+#define RGX_CR_JONES_IDLE_TDM_EN                                                    (0x00004000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_SHIFT                                          (13U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_EN                                             (0x00002000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_SHIFT                                              (12U)
+#define RGX_CR_JONES_IDLE_FB_CDC_CLRMSK                                             (0xFFFFEFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_EN                                                 (0x00001000U)
+#define RGX_CR_JONES_IDLE_MMU_SHIFT                                                 (11U)
+#define RGX_CR_JONES_IDLE_MMU_CLRMSK                                                (0xFFFFF7FFU)
+#define RGX_CR_JONES_IDLE_MMU_EN                                                    (0x00000800U)
+#define RGX_CR_JONES_IDLE_DFU_SHIFT                                                 (10U)
+#define RGX_CR_JONES_IDLE_DFU_CLRMSK                                                (0xFFFFFBFFU)
+#define RGX_CR_JONES_IDLE_DFU_EN                                                    (0x00000400U)
+#define RGX_CR_JONES_IDLE_GARTEN_SHIFT                                              (9U)
+#define RGX_CR_JONES_IDLE_GARTEN_CLRMSK                                             (0xFFFFFDFFU)
+#define RGX_CR_JONES_IDLE_GARTEN_EN                                                 (0x00000200U)
+#define RGX_CR_JONES_IDLE_SOCIF_SHIFT                                               (7U)
+#define RGX_CR_JONES_IDLE_SOCIF_CLRMSK                                              (0xFFFFFF7FU)
+#define RGX_CR_JONES_IDLE_SOCIF_EN                                                  (0x00000080U)
+#define RGX_CR_JONES_IDLE_TILING_SHIFT                                              (6U)
+#define RGX_CR_JONES_IDLE_TILING_CLRMSK                                             (0xFFFFFFBFU)
+#define RGX_CR_JONES_IDLE_TILING_EN                                                 (0x00000040U)
+#define RGX_CR_JONES_IDLE_IPP_SHIFT                                                 (5U)
+#define RGX_CR_JONES_IDLE_IPP_CLRMSK                                                (0xFFFFFFDFU)
+#define RGX_CR_JONES_IDLE_IPP_EN                                                    (0x00000020U)
+#define RGX_CR_JONES_IDLE_USC_GMUTEX_SHIFT                                          (4U)
+#define RGX_CR_JONES_IDLE_USC_GMUTEX_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_JONES_IDLE_USC_GMUTEX_EN                                             (0x00000010U)
+#define RGX_CR_JONES_IDLE_PM_SHIFT                                                  (3U)
+#define RGX_CR_JONES_IDLE_PM_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_JONES_IDLE_PM_EN                                                     (0x00000008U)
+#define RGX_CR_JONES_IDLE_CDM_SHIFT                                                 (2U)
+#define RGX_CR_JONES_IDLE_CDM_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_JONES_IDLE_CDM_EN                                                    (0x00000004U)
+#define RGX_CR_JONES_IDLE_DCE_SHIFT                                                 (1U)
+#define RGX_CR_JONES_IDLE_DCE_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_JONES_IDLE_DCE_EN                                                    (0x00000002U)
+#define RGX_CR_JONES_IDLE_BIF_SHIFT                                                 (0U)
+#define RGX_CR_JONES_IDLE_BIF_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_JONES_IDLE_BIF_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_SYS_BUS_SECURE
+*/
+#define RGX_CR_SYS_BUS_SECURE                                                       (0xA100U)
+#define RGX_CR_SYS_BUS_SECURE__SBSR__MASKFULL                                       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_MASKFULL                                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_SHIFT                                          (0U)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6            (0xF0000U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__MASKFULL  (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_CLRMSK (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4 (0xF0000U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__MASKFULL (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_SHIFT (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_CLRMSK (0xFFFFFFE0U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6            (0xF0000U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6__MASKFULL  (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_GT6__ID_CLRMSK (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT                                            (0xE140U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__MASKFULL (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_EQ6__MASKFULL  (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_MASKFULL                                   (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_SHIFT                                   (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_CLRMSK                                  (0xFFFFFF00U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_SHIFT (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_CLRMSK (0xFFFFFFE0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_SHIFT (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4__ID_CLRMSK (0xFFFFFFE0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_EQ6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_EQ1_AND_MHPW_EQ6__ID_CLRMSK (0xFFFFFFC0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_SHIFT  (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_EQ6__ID_CLRMSK (0xFFFFFFC0U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1                                 (0xF0008U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__MASKFULL                       (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_SHIFT                  (28U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_CLRMSK                 (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_EN                     (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT                (0U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK               (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT           (12U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSIZE            (4096U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU                         (0xF0008U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__MASKFULL               (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__INVALID_SHIFT          (28U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__INVALID_CLRMSK         (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__INVALID_EN             (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_SHIFT        (0U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_CLRMSK       (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_ALIGNSHIFT   (12U)
+#define RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1_AND_VPU__BASE_ADDR_ALIGNSIZE    (4096U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING__VPU                                               (0x1E010U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__MASKFULL                                     (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__INVALID_SHIFT                                (28U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__INVALID_CLRMSK                               (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__INVALID_EN                                   (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_SHIFT                              (0U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_CLRMSK                             (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_ALIGNSHIFT                         (12U)
+#define RGX_CR_MMU_CBASE_MAPPING__VPU__BASE_ADDR_ALIGNSIZE                          (4096U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING                                                    (0xE148U)
+#define RGX_CR_MMU_CBASE_MAPPING_MASKFULL                                           (IMG_UINT64_C(0x000000001FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING_INVALID_SHIFT                                      (28U)
+#define RGX_CR_MMU_CBASE_MAPPING_INVALID_CLRMSK                                     (0xEFFFFFFFU)
+#define RGX_CR_MMU_CBASE_MAPPING_INVALID_EN                                         (0x10000000U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT                                    (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK                                   (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT                               (12U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS1
+*/
+#define RGX_CR_MMU_FAULT_STATUS1                                                    (0xE150U)
+#define RGX_CR_MMU_FAULT_STATUS1_MASKFULL                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_LEVEL_SHIFT                                        (62U)
+#define RGX_CR_MMU_FAULT_STATUS1_LEVEL_CLRMSK                                       (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_REQ_ID_SHIFT                                       (56U)
+#define RGX_CR_MMU_FAULT_STATUS1_REQ_ID_CLRMSK                                      (IMG_UINT64_C(0xC0FFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT                                      (48U)
+#define RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK                                     (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT                                      (4U)
+#define RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK                                     (IMG_UINT64_C(0xFFFF00000000000F))
+#define RGX_CR_MMU_FAULT_STATUS1_RNW_SHIFT                                          (3U)
+#define RGX_CR_MMU_FAULT_STATUS1_RNW_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS1_RNW_EN                                             (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT                                         (1U)
+#define RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS1_FAULT_SHIFT                                        (0U)
+#define RGX_CR_MMU_FAULT_STATUS1_FAULT_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS1_FAULT_EN                                           (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS2
+*/
+#define RGX_CR_MMU_FAULT_STATUS2                                                    (0xE158U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__MASKFULL                               (IMG_UINT64_C(0x00000000003FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS2_MASKFULL                                           (IMG_UINT64_C(0x000000003FFF07FF))
+#define RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_SHIFT                                    (29U)
+#define RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_CLRMSK                                   (0xDFFFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_EN                                       (0x20000000U)
+#define RGX_CR_MMU_FAULT_STATUS2_CLEANUNIQUE_SHIFT                                  (28U)
+#define RGX_CR_MMU_FAULT_STATUS2_CLEANUNIQUE_CLRMSK                                 (0xEFFFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_CLEANUNIQUE_EN                                     (0x10000000U)
+#define RGX_CR_MMU_FAULT_STATUS2_BANK_SHIFT                                         (24U)
+#define RGX_CR_MMU_FAULT_STATUS2_BANK_CLRMSK                                        (0xF0FFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_SHIFT                        (21U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_CLRMSK                       (0xFFDFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_EN                           (0x00200000U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_SHIFT                        (20U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_CLRMSK                       (0xFFEFFFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_EN                           (0x00100000U)
+#define RGX_CR_MMU_FAULT_STATUS2_TLB_ENTRY_SHIFT                                    (16U)
+#define RGX_CR_MMU_FAULT_STATUS2_TLB_ENTRY_CLRMSK                                   (0xFF00FFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_SHIFT                           (12U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_CLRMSK                          (0xFFF00FFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_SHIFT                                    (10U)
+#define RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_EN                                       (0x00000400U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BANK_SHIFT                             (8U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BANK_CLRMSK                            (0xFFFFF0FFU)
+#define RGX_CR_MMU_FAULT_STATUS2_BIF_ID_SHIFT                                       (0U)
+#define RGX_CR_MMU_FAULT_STATUS2_BIF_ID_CLRMSK                                      (0xFFFFFC00U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__ACTIVE_ID_SHIFT                        (0U)
+#define RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__ACTIVE_ID_CLRMSK                       (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS_META
+*/
+#define RGX_CR_MMU_FAULT_STATUS_META                                                (0xE160U)
+#define RGX_CR_MMU_FAULT_STATUS_META_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT                                    (62U)
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK                                   (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT                                   (56U)
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK                                  (IMG_UINT64_C(0xC0FFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT                                  (48U)
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK                                 (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT                                  (4U)
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK                                 (IMG_UINT64_C(0xFFFF00000000000F))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_SHIFT                                      (3U)
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_EN                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT                                     (1U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_SHIFT                                    (0U)
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN                                       (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS2_META
+*/
+#define RGX_CR_MMU_FAULT_STATUS2_META                                               (0xE198U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__MASKFULL                          (IMG_UINT64_C(0x0000000000001FFF))
+#define RGX_CR_MMU_FAULT_STATUS2_META_MASKFULL                                      (IMG_UINT64_C(0x0000000000003FFF))
+#define RGX_CR_MMU_FAULT_STATUS2_META_WRITEBACK_SHIFT                               (13U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_WRITEBACK_CLRMSK                              (0xFFFFDFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META_WRITEBACK_EN                                  (0x00002000U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_CLEANUNIQUE_SHIFT                             (12U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_CLEANUNIQUE_CLRMSK                            (0xFFFFEFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META_CLEANUNIQUE_EN                                (0x00001000U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__WRITEBACK_SHIFT                   (12U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__WRITEBACK_CLRMSK                  (0xFFFFEFFFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__WRITEBACK_EN                      (0x00001000U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_BANK_SHIFT                                    (8U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_BANK_CLRMSK                                   (0xFFFFF0FFU)
+#define RGX_CR_MMU_FAULT_STATUS2_META_TLB_ENTRY_SHIFT                               (0U)
+#define RGX_CR_MMU_FAULT_STATUS2_META_TLB_ENTRY_CLRMSK                              (0xFFFFFF00U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__ACTIVE_ID_SHIFT                   (0U)
+#define RGX_CR_MMU_FAULT_STATUS2_META__AXT_INFRA__ACTIVE_ID_CLRMSK                  (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS_PM
+*/
+#define RGX_CR_MMU_FAULT_STATUS_PM                                                  (0xE130U)
+#define RGX_CR_MMU_FAULT_STATUS_PM__PM_RECYCLE__MASKFULL                            (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_MASKFULL                                         (IMG_UINT64_C(0x0000000007FFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_DM_SHIFT                                         (24U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_DM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF8FFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM__PM_RECYCLE__DM_SHIFT                            (24U)
+#define RGX_CR_MMU_FAULT_STATUS_PM__PM_RECYCLE__DM_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFF0FFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_RNW_SHIFT                                        (23U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_RNW_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_PM_RNW_EN                                           (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_MMU_FAULT_STATUS_PM_ADDRESS_SHIFT                                    (3U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFF800007))
+#define RGX_CR_MMU_FAULT_STATUS_PM_LEVEL_SHIFT                                      (1U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_LEVEL_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_PM_FAULT_SHIFT                                      (0U)
+#define RGX_CR_MMU_FAULT_STATUS_PM_FAULT_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_PM_FAULT_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_STATUS
+*/
+#define RGX_CR_MMU_STATUS__VPU                                                      (0x10288U)
+#define RGX_CR_MMU_STATUS__VPU__MASKFULL                                            (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__MMU_STALLED_SHIFT                                   (40U)
+#define RGX_CR_MMU_STATUS__VPU__MMU_STALLED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__MMU_STALLED_EN                                      (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MMU_STATUS__VPU__PM_WRITES_SHIFT                                     (38U)
+#define RGX_CR_MMU_STATUS__VPU__PM_WRITES_CLRMSK                                    (IMG_UINT64_C(0xFFFFFF3FFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__PM_READS_SHIFT                                      (36U)
+#define RGX_CR_MMU_STATUS__VPU__PM_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__PC_READS_SHIFT                                      (24U)
+#define RGX_CR_MMU_STATUS__VPU__PC_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_MMU_STATUS__VPU__PD_READS_SHIFT                                      (12U)
+#define RGX_CR_MMU_STATUS__VPU__PD_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_MMU_STATUS__VPU__PT_READS_SHIFT                                      (0U)
+#define RGX_CR_MMU_STATUS__VPU__PT_READS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_MMU_STATUS
+*/
+#define RGX_CR_MMU_STATUS                                                           (0xE170U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MASKFULL                                      (IMG_UINT64_C(0x000003FFFFFFFFFF))
+#define RGX_CR_MMU_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MMU_STALLED_SHIFT                             (41U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MMU_STALLED_CLRMSK                            (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__MMU_STALLED_EN                                (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MMU_STATUS_MMU_STALLED_SHIFT                                         (40U)
+#define RGX_CR_MMU_STATUS_MMU_STALLED_CLRMSK                                        (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MMU_STATUS_MMU_STALLED_EN                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_WRITES_SHIFT                               (39U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_WRITES_CLRMSK                              (IMG_UINT64_C(0xFFFFFE7FFFFFFFFF))
+#define RGX_CR_MMU_STATUS_PM_WRITES_SHIFT                                           (38U)
+#define RGX_CR_MMU_STATUS_PM_WRITES_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF3FFFFFFFFF))
+#define RGX_CR_MMU_STATUS_PM_READS_SHIFT                                            (36U)
+#define RGX_CR_MMU_STATUS_PM_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_READS_SHIFT                                (36U)
+#define RGX_CR_MMU_STATUS__CXT_INFRA__PM_READS_CLRMSK                               (IMG_UINT64_C(0xFFFFFF8FFFFFFFFF))
+#define RGX_CR_MMU_STATUS_PC_READS_SHIFT                                            (24U)
+#define RGX_CR_MMU_STATUS_PC_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_MMU_STATUS_PD_READS_SHIFT                                            (12U)
+#define RGX_CR_MMU_STATUS_PD_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_MMU_STATUS_PT_READS_SHIFT                                            (0U)
+#define RGX_CR_MMU_STATUS_PT_READS_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_MMU_ENTRY_STATUS
+*/
+#define RGX_CR_MMU_ENTRY_STATUS__VPU                                                (0x1E028U)
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__MASKFULL                                      (IMG_UINT64_C(0x000000FFFFFF80FF))
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__ADDRESS_SHIFT                                 (15U)
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__ADDRESS_CLRMSK                                (IMG_UINT64_C(0xFFFFFF0000007FFF))
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__CONTEXT_ID_SHIFT                              (0U)
+#define RGX_CR_MMU_ENTRY_STATUS__VPU__CONTEXT_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MMU_ENTRY_STATUS
+*/
+#define RGX_CR_MMU_ENTRY_STATUS                                                     (0xE178U)
+#define RGX_CR_MMU_ENTRY_STATUS_MASKFULL                                            (IMG_UINT64_C(0x000000FFFFFF80FF))
+#define RGX_CR_MMU_ENTRY_STATUS_ADDRESS_SHIFT                                       (15U)
+#define RGX_CR_MMU_ENTRY_STATUS_ADDRESS_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000007FFF))
+#define RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_SHIFT                                    (0U)
+#define RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MMU_ENTRY
+*/
+#define RGX_CR_MMU_ENTRY__VPU                                                       (0x1E030U)
+#define RGX_CR_MMU_ENTRY__VPU__MASKFULL                                             (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_MMU_ENTRY__VPU__ENABLE_SHIFT                                         (1U)
+#define RGX_CR_MMU_ENTRY__VPU__ENABLE_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_MMU_ENTRY__VPU__ENABLE_EN                                            (0x00000002U)
+#define RGX_CR_MMU_ENTRY__VPU__PENDING_SHIFT                                        (0U)
+#define RGX_CR_MMU_ENTRY__VPU__PENDING_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_MMU_ENTRY__VPU__PENDING_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_MMU_ENTRY
+*/
+#define RGX_CR_MMU_ENTRY                                                            (0xE180U)
+#define RGX_CR_MMU_ENTRY_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_MMU_ENTRY_ENABLE_SHIFT                                               (1U)
+#define RGX_CR_MMU_ENTRY_ENABLE_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_MMU_ENTRY_ENABLE_EN                                                  (0x00000002U)
+#define RGX_CR_MMU_ENTRY_PENDING_SHIFT                                              (0U)
+#define RGX_CR_MMU_ENTRY_PENDING_CLRMSK                                             (0xFFFFFFFEU)
+#define RGX_CR_MMU_ENTRY_PENDING_EN                                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_ONE
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE                                              (0xE350U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_MASKFULL                                     (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT                              (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_CLRMSK                             (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_SHIFT                               (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_CLRMSK                              (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT                          (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSIZE                           (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_SHIFT                              (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT                         (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSIZE                          (2097152U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_TWO
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO                                              (0xE358U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_MASKFULL                                     (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_PAGE_SIZE_SHIFT                              (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_PAGE_SIZE_CLRMSK                             (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_SHIFT                               (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_CLRMSK                              (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_ALIGNSHIFT                          (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_END_ADDR_ALIGNSIZE                           (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_SHIFT                              (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_ALIGNSHIFT                         (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_TWO_BASE_ADDR_ALIGNSIZE                          (2097152U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_THREE
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE                                            (0xE360U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_MASKFULL                                   (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_PAGE_SIZE_SHIFT                            (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_PAGE_SIZE_CLRMSK                           (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_SHIFT                             (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_ALIGNSHIFT                        (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_END_ADDR_ALIGNSIZE                         (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_SHIFT                            (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_ALIGNSHIFT                       (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_THREE_BASE_ADDR_ALIGNSIZE                        (2097152U)
+
+
+/*
+    Register RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR
+*/
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR                                             (0xE368U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_MASKFULL                                    (IMG_UINT64_C(0x000001FFFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_PAGE_SIZE_SHIFT                             (38U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_PAGE_SIZE_CLRMSK                            (IMG_UINT64_C(0xFFFFFE3FFFFFFFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_SHIFT                              (19U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFC00007FFFF))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_ALIGNSHIFT                         (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_END_ADDR_ALIGNSIZE                          (2097152U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_SHIFT                             (0U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFF80000))
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_ALIGNSHIFT                        (21U)
+#define RGX_CR_MMU_PAGE_SIZE_RANGE_FOUR_BASE_ADDR_ALIGNSIZE                         (2097152U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS1
+*/
+#define RGX_CR_SLC_STATUS1                                                          (0xE210U)
+#define RGX_CR_SLC_STATUS1_MASKFULL                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_XBAR_CFI_TIMEOUTS_SHIFT                                  (48U)
+#define RGX_CR_SLC_STATUS1_XBAR_CFI_TIMEOUTS_CLRMSK                                 (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_WRITES_SHIFT                            (36U)
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFF000FFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_WRITES_SHIFT                            (24U)
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_READS_SHIFT                             (12U)
+#define RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_READS_SHIFT                             (0U)
+#define RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_SLC_STATUS2
+*/
+#define RGX_CR_SLC_STATUS2                                                          (0xE218U)
+#define RGX_CR_SLC_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_SHIFT                                     (48U)
+#define RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_CLRMSK                                    (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_WRITES_SHIFT                            (36U)
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFF000FFFFFFFFF))
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_WRITES_SHIFT                            (24U)
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_WRITES_CLRMSK                           (IMG_UINT64_C(0xFFFFFFF000FFFFFF))
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_READS_SHIFT                             (12U)
+#define RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFF000FFF))
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_READS_SHIFT                             (0U)
+#define RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_READS_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFF000))
+
+
+/*
+    Register RGX_CR_SLC_IDLE
+*/
+#define RGX_CR_SLC_IDLE                                                             (0xE230U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__MASKFULL                               (IMG_UINT64_C(0x0000000001FFFFFF))
+#define RGX_CR_SLC_IDLE_MASKFULL                                                    (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__FBCDC_ARB_SHIFT                        (24U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__FBCDC_ARB_CLRMSK                       (0xFEFFFFFFU)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__FBCDC_ARB_EN                           (0x01000000U)
+#define RGX_CR_SLC_IDLE_FBCDC_ARB_SHIFT                                             (20U)
+#define RGX_CR_SLC_IDLE_FBCDC_ARB_CLRMSK                                            (0xFFEFFFFFU)
+#define RGX_CR_SLC_IDLE_FBCDC_ARB_EN                                                (0x00100000U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__OWDB_SHIFT                             (20U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__OWDB_CLRMSK                            (0xFF0FFFFFU)
+#define RGX_CR_SLC_IDLE_OWDB_SHIFT                                                  (16U)
+#define RGX_CR_SLC_IDLE_OWDB_CLRMSK                                                 (0xFFF0FFFFU)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__ACE_CLBS_SHIFT                         (16U)
+#define RGX_CR_SLC_IDLE__GCC_AND_AXT_IF_GT0__ACE_CLBS_CLRMSK                        (0xFFF0FFFFU)
+#define RGX_CR_SLC_IDLE_ACE_CONVERTERS_SHIFT                                        (12U)
+#define RGX_CR_SLC_IDLE_ACE_CONVERTERS_CLRMSK                                       (0xFFFF0FFFU)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_SHIFT                                           (4U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_CLRMSK                                          (0xFFFFF00FU)
+#define RGX_CR_SLC_IDLE_MMU_SHIFT                                                   (3U)
+#define RGX_CR_SLC_IDLE_MMU_CLRMSK                                                  (0xFFFFFFF7U)
+#define RGX_CR_SLC_IDLE_MMU_EN                                                      (0x00000008U)
+#define RGX_CR_SLC_IDLE_CCM_SHIFT                                                   (2U)
+#define RGX_CR_SLC_IDLE_CCM_CLRMSK                                                  (0xFFFFFFFBU)
+#define RGX_CR_SLC_IDLE_CCM_EN                                                      (0x00000004U)
+#define RGX_CR_SLC_IDLE_RDI_SHIFT                                                   (1U)
+#define RGX_CR_SLC_IDLE_RDI_CLRMSK                                                  (0xFFFFFFFDU)
+#define RGX_CR_SLC_IDLE_RDI_EN                                                      (0x00000002U)
+#define RGX_CR_SLC_IDLE_XBAR_SHIFT                                                  (0U)
+#define RGX_CR_SLC_IDLE_XBAR_CLRMSK                                                 (0xFFFFFFFEU)
+#define RGX_CR_SLC_IDLE_XBAR_EN                                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS3
+*/
+#define RGX_CR_SLC_STATUS3                                                          (0xE238U)
+#define RGX_CR_SLC_STATUS3_MASKFULL                                                 (IMG_UINT64_C(0x0FFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS3_MAX_CRITICAL_QOS_READ_LATENCY_SHIFT                      (50U)
+#define RGX_CR_SLC_STATUS3_MAX_CRITICAL_QOS_READ_LATENCY_CLRMSK                     (IMG_UINT64_C(0xF003FFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS3_MAX_LOW_QOS_READ_LATENCY_SHIFT                           (40U)
+#define RGX_CR_SLC_STATUS3_MAX_LOW_QOS_READ_LATENCY_CLRMSK                          (IMG_UINT64_C(0xFFFC00FFFFFFFFFF))
+#define RGX_CR_SLC_STATUS3_AVG_CRITICAL_QOS_READ_LATENCY_SHIFT                      (30U)
+#define RGX_CR_SLC_STATUS3_AVG_CRITICAL_QOS_READ_LATENCY_CLRMSK                     (IMG_UINT64_C(0xFFFFFF003FFFFFFF))
+#define RGX_CR_SLC_STATUS3_AVG_LOW_QOS_READ_LATENCY_SHIFT                           (20U)
+#define RGX_CR_SLC_STATUS3_AVG_LOW_QOS_READ_LATENCY_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFC00FFFFF))
+#define RGX_CR_SLC_STATUS3_MIN_CRITICAL_QOS_READ_LATENCY_SHIFT                      (10U)
+#define RGX_CR_SLC_STATUS3_MIN_CRITICAL_QOS_READ_LATENCY_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFF003FF))
+#define RGX_CR_SLC_STATUS3_MIN_LOW_QOS_READ_LATENCY_SHIFT                           (0U)
+#define RGX_CR_SLC_STATUS3_MIN_LOW_QOS_READ_LATENCY_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFC00))
+
+
+/*
+    Register RGX_CR_SLC_FAULT_STOP_STATUS
+*/
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU                                           (0x1E240U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU__MASKFULL                                 (IMG_UINT64_C(0x000000000001FFFF))
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU__BIF_SHIFT                                (0U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS__VPU__BIF_CLRMSK                               (0xFFFE0000U)
+
+
+/*
+    Register RGX_CR_SLC_FAULT_STOP_STATUS
+*/
+#define RGX_CR_SLC_FAULT_STOP_STATUS                                                (0xE240U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000001FFFF))
+#define RGX_CR_SLC_FAULT_STOP_STATUS_BIF_SHIFT                                      (0U)
+#define RGX_CR_SLC_FAULT_STOP_STATUS_BIF_CLRMSK                                     (0xFFFE0000U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS_DEBUG
+*/
+#define RGX_CR_SLC_STATUS_DEBUG__VPU                                                (0x1E260U)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__MASKFULL                                      (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_COH_REQ_SHIFT                             (16U)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_COH_REQ_CLRMSK                            (0x0000FFFFU)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_ADDR_ALIAS_SHIFT                          (0U)
+#define RGX_CR_SLC_STATUS_DEBUG__VPU__ERR_ADDR_ALIAS_CLRMSK                         (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS_DEBUG
+*/
+#define RGX_CR_SLC_STATUS_DEBUG                                                     (0xE260U)
+#define RGX_CR_SLC_STATUS_DEBUG_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_COH_REQ_SHIFT                                   (16U)
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_COH_REQ_CLRMSK                                  (0x0000FFFFU)
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_ADDR_ALIAS_SHIFT                                (0U)
+#define RGX_CR_SLC_STATUS_DEBUG_ERR_ADDR_ALIAS_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_HMMU_OSID_PAGE_SIZE
+*/
+#define RGX_CR_HMMU_OSID_PAGE_SIZE                                                  (0x80000U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_MASKFULL                                         (IMG_UINT64_C(0x0000000077777777))
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_7_SHIFT                                     (28U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_7_CLRMSK                                    (0x8FFFFFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_6_SHIFT                                     (24U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_6_CLRMSK                                    (0xF8FFFFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_5_SHIFT                                     (20U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_5_CLRMSK                                    (0xFF8FFFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_4_SHIFT                                     (16U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_4_CLRMSK                                    (0xFFF8FFFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_3_SHIFT                                     (12U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_3_CLRMSK                                    (0xFFFF8FFFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_2_SHIFT                                     (8U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_2_CLRMSK                                    (0xFFFFF8FFU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_1_SHIFT                                     (4U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_1_CLRMSK                                    (0xFFFFFF8FU)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_0_SHIFT                                     (0U)
+#define RGX_CR_HMMU_OSID_PAGE_SIZE_OSID_0_CLRMSK                                    (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_HMMU_BYPASS
+*/
+#define RGX_CR_HMMU_BYPASS                                                          (0x80008U)
+#define RGX_CR_HMMU_BYPASS_MASKFULL                                                 (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_HMMU_BYPASS_EN_SHIFT                                                 (0U)
+#define RGX_CR_HMMU_BYPASS_EN_CLRMSK                                                (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_HMMU_INVAL
+*/
+#define RGX_CR_HMMU_INVAL                                                           (0x80010U)
+#define RGX_CR_HMMU_INVAL_MASKFULL                                                  (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_HMMU_INVAL_OS_ID_SHIFT                                               (4U)
+#define RGX_CR_HMMU_INVAL_OS_ID_CLRMSK                                              (0xFFFFFF8FU)
+#define RGX_CR_HMMU_INVAL_ALL_OS_IDS_SHIFT                                          (3U)
+#define RGX_CR_HMMU_INVAL_ALL_OS_IDS_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_HMMU_INVAL_ALL_OS_IDS_EN                                             (0x00000008U)
+#define RGX_CR_HMMU_INVAL_HPC_SHIFT                                                 (2U)
+#define RGX_CR_HMMU_INVAL_HPC_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_HMMU_INVAL_HPC_EN                                                    (0x00000004U)
+#define RGX_CR_HMMU_INVAL_HPD_SHIFT                                                 (1U)
+#define RGX_CR_HMMU_INVAL_HPD_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_HMMU_INVAL_HPD_EN                                                    (0x00000002U)
+#define RGX_CR_HMMU_INVAL_HPT_SHIFT                                                 (0U)
+#define RGX_CR_HMMU_INVAL_HPT_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_HMMU_INVAL_HPT_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING0
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0                                               (0x80018U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR1_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR1_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID1_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID1_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID1_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR0_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_ADDR0_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID0_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID0_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING0_VALID0_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING1
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1                                               (0x80020U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR3_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR3_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID3_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID3_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID3_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR2_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_ADDR2_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID2_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID2_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING1_VALID2_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING2
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2                                               (0x80028U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR5_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR5_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID5_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID5_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID5_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR4_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_ADDR4_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID4_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID4_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING2_VALID4_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_HPC_BASE_MAPPING3
+*/
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3                                               (0x80030U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_MASKFULL                                      (IMG_UINT64_C(0xFFFFFFF1FFFFFFF1))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR7_SHIFT                                   (36U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR7_CLRMSK                                  (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID7_SHIFT                                  (32U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID7_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID7_EN                                     (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR6_SHIFT                                   (4U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_ADDR6_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID6_SHIFT                                  (0U)
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID6_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HPC_BASE_MAPPING3_VALID6_EN                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register group: RGX_CR_HMMU_PAGE_FAULT_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO_REPEATCOUNT                                     (8U)
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO0
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0                                                (0x80038U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO0_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO1
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1                                                (0x80040U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO1_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO2
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2                                                (0x80048U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO2_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO3
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3                                                (0x80050U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO3_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO4
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4                                                (0x80058U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO4_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO5
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5                                                (0x80060U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO5_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO6
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6                                                (0x80068U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO6_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PAGE_FAULT_INFO7
+*/
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7                                                (0x80070U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_ADDR_SHIFT                                     (2U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_LEVEL_SHIFT                                    (0U)
+#define RGX_CR_HMMU_PAGE_FAULT_INFO7_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register group: RGX_CR_HMMU_PENDING_ENTRY_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO_REPEATCOUNT                                  (8U)
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO0
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0                                             (0x800C0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO0_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO1
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1                                             (0x800C8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO1_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO2
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2                                             (0x800D0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO2_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO3
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3                                             (0x800D8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO3_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO4
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4                                             (0x800E0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO4_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO5
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5                                             (0x800E8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO5_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO6
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6                                             (0x800F0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO6_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY_INFO7
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7                                             (0x800F8U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_MASKFULL                                    (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_ADDR_SHIFT                                  (2U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_ADDR_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFC0000003))
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_LEVEL_SHIFT                                 (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_INFO7_LEVEL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_HMMU_HOST_IRQ_ENABLE
+*/
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE                                                 (0x80100U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_MASKFULL                                        (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_SHIFT                            (3U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_EN                               (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_PM_SHIFT                         (2U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_PM_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_READONLY_FAULT_PM_EN                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PAGE_FAULT_SHIFT                                (1U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PAGE_FAULT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PAGE_FAULT_EN                                   (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PENDING_ENTRY_SHIFT                             (0U)
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PENDING_ENTRY_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_HOST_IRQ_ENABLE_PENDING_ENTRY_EN                                (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_HMMU_PENDING_ENTRY
+*/
+#define RGX_CR_HMMU_PENDING_ENTRY                                                   (0x80108U)
+#define RGX_CR_HMMU_PENDING_ENTRY_MASKFULL                                          (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_HMMU_PENDING_ENTRY_ENABLE_SHIFT                                      (0U)
+#define RGX_CR_HMMU_PENDING_ENTRY_ENABLE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_HMMU_PENDING_ENTRY_ENABLE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_HMMU_FAULT_STATUS
+*/
+#define RGX_CR_HMMU_FAULT_STATUS                                                    (0x80120U)
+#define RGX_CR_HMMU_FAULT_STATUS_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID7_SHIFT                      (31U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID7_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID7_EN                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID6_SHIFT                      (30U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID6_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID6_EN                         (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID5_SHIFT                      (29U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID5_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID5_EN                         (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID4_SHIFT                      (28U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID4_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID4_EN                         (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID3_SHIFT                      (27U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID3_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID3_EN                         (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID2_SHIFT                      (26U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID2_EN                         (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID1_SHIFT                      (25U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID1_EN                         (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID0_SHIFT                      (24U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_PM_OSID0_EN                         (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID7_SHIFT                         (23U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID7_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID7_EN                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID6_SHIFT                         (22U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID6_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID6_EN                            (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID5_SHIFT                         (21U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID5_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID5_EN                            (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID4_SHIFT                         (20U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID4_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID4_EN                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID3_SHIFT                         (19U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID3_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID3_EN                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID2_SHIFT                         (18U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID2_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID2_EN                            (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID1_SHIFT                         (17U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID1_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID1_EN                            (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID0_SHIFT                         (16U)
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID0_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_READONLY_FAULT_OSID0_EN                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID7_SHIFT                          (15U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID7_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID7_EN                             (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID6_SHIFT                          (14U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID6_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID6_EN                             (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID5_SHIFT                          (13U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID5_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID5_EN                             (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID4_SHIFT                          (12U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID4_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID4_EN                             (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID3_SHIFT                          (11U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID3_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID3_EN                             (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID2_SHIFT                          (10U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID2_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID2_EN                             (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID1_SHIFT                          (9U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID1_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID1_EN                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID0_SHIFT                          (8U)
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID0_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_HMMU_FAULT_STATUS_PENDING_ENTRY_OSID0_EN                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID7_SHIFT                             (7U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID7_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID7_EN                                (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID6_SHIFT                             (6U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID6_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID6_EN                                (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID5_SHIFT                             (5U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID5_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID5_EN                                (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID4_SHIFT                             (4U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID4_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID4_EN                                (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID3_SHIFT                             (3U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID3_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID3_EN                                (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID2_SHIFT                             (2U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID2_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID2_EN                                (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID1_SHIFT                             (1U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID1_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID1_EN                                (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID0_SHIFT                             (0U)
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID0_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_HMMU_FAULT_STATUS_PAGE_FAULT_OSID0_EN                                (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register group: RGX_CR_HMMU_READONLY_FAULT_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO_REPEATCOUNT                                 (8U)
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO0
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0                                            (0x80190U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO0_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO1
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1                                            (0x80198U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO1_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO2
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2                                            (0x801A0U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO2_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO3
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3                                            (0x801A8U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO3_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO4
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4                                            (0x801B0U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO4_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO5
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5                                            (0x801B8U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO5_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO6
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6                                            (0x801C0U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO6_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_INFO7
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7                                            (0x801C8U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7_ADDR_SHIFT                                 (4U)
+#define RGX_CR_HMMU_READONLY_FAULT_INFO7_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+
+
+/*
+    Register group: RGX_CR_HMMU_READONLY_FAULT_PM_INFO, with 8 repeats
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO_REPEATCOUNT                              (8U)
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO0
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0                                         (0x801D0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO0_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO1
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1                                         (0x801D8U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO1_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO2
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2                                         (0x801E0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO2_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO3
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3                                         (0x801E8U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO3_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO4
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4                                         (0x801F0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO4_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO5
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5                                         (0x801F8U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO5_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO6
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6                                         (0x80200U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO6_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_HMMU_READONLY_FAULT_PM_INFO7
+*/
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7                                         (0x80208U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7_MASKFULL                                (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7_ADDR_SHIFT                              (0U)
+#define RGX_CR_HMMU_READONLY_FAULT_PM_INFO7_ADDR_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFF0000000))
+
+
+/*
+    Register RGX_CR_ACE_CTRL
+*/
+#define RGX_CR_ACE_CTRL__VPU                                                        (0x1E320U)
+#define RGX_CR_ACE_CTRL__VPU__MASKFULL                                              (IMG_UINT64_C(0x00000000007FCFFF))
+#define RGX_CR_ACE_CTRL__VPU__CLB_AXQOS_SHIFT                                       (19U)
+#define RGX_CR_ACE_CTRL__VPU__CLB_AXQOS_CLRMSK                                      (0xFF87FFFFU)
+#define RGX_CR_ACE_CTRL__VPU__PM_MMU_AXCACHE_SHIFT                                  (15U)
+#define RGX_CR_ACE_CTRL__VPU__PM_MMU_AXCACHE_CLRMSK                                 (0xFFF87FFFU)
+#define RGX_CR_ACE_CTRL__VPU__ENABLE_NONSECURE_PROT_MATCH_SHIFT                     (14U)
+#define RGX_CR_ACE_CTRL__VPU__ENABLE_NONSECURE_PROT_MATCH_CLRMSK                    (0xFFFFBFFFU)
+#define RGX_CR_ACE_CTRL__VPU__ENABLE_NONSECURE_PROT_MATCH_EN                        (0x00004000U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_SHIFT                                     (8U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_CLRMSK                                    (0xFFFFF0FFU)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_DEVICE_BUFFERABLE                         (0x00000100U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_NORMAL_NC_NON_BUFFERABLE                  (0x00000200U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_NORMAL_NC_BUFFERABLE                      (0x00000300U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_THROUGH_NO_ALLOCATE                 (0x00000600U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_THROUGH_WRITE_ALLOCATE              (0x00000E00U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_BACK_NO_ALLOCATE                    (0x00000700U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE                 (0x00000F00U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_SHIFT                                     (4U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_CLRMSK                                    (0xFFFFFF0FU)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_DEVICE_BUFFERABLE                         (0x00000010U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_NORMAL_NC_NON_BUFFERABLE                  (0x00000020U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_NORMAL_NC_BUFFERABLE                      (0x00000030U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_THROUGH_NO_ALLOCATE                 (0x000000A0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_THROUGH_READ_ALLOCATE               (0x000000E0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_BACK_NO_ALLOCATE                    (0x000000B0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE                  (0x000000F0U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_DOMAIN_SHIFT                                      (2U)
+#define RGX_CR_ACE_CTRL__VPU__MMU_DOMAIN_CLRMSK                                     (0xFFFFFFF3U)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_SHIFT                                      (1U)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_INNER_SHAREABLE                            (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__COH_DOMAIN_OUTER_SHAREABLE                            (0x00000002U)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_SHIFT                                  (0U)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_NON_SHAREABLE                          (0x00000000U)
+#define RGX_CR_ACE_CTRL__VPU__NON_COH_DOMAIN_SYSTEM                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_ACE_CTRL
+*/
+#define RGX_CR_ACE_CTRL                                                             (0xE320U)
+#define RGX_CR_ACE_CTRL_MASKFULL                                                    (IMG_UINT64_C(0x0000000000FFCFFF))
+#define RGX_CR_ACE_CTRL_DISABLE_EMPTY_BURST_REMOVAL_SHIFT                           (23U)
+#define RGX_CR_ACE_CTRL_DISABLE_EMPTY_BURST_REMOVAL_CLRMSK                          (0xFF7FFFFFU)
+#define RGX_CR_ACE_CTRL_DISABLE_EMPTY_BURST_REMOVAL_EN                              (0x00800000U)
+#define RGX_CR_ACE_CTRL_CLB_AXQOS_SHIFT                                             (19U)
+#define RGX_CR_ACE_CTRL_CLB_AXQOS_CLRMSK                                            (0xFF87FFFFU)
+#define RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_SHIFT                                        (15U)
+#define RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_CLRMSK                                       (0xFFF87FFFU)
+#define RGX_CR_ACE_CTRL_ENABLE_NONSECURE_PROT_MATCH_SHIFT                           (14U)
+#define RGX_CR_ACE_CTRL_ENABLE_NONSECURE_PROT_MATCH_CLRMSK                          (0xFFFFBFFFU)
+#define RGX_CR_ACE_CTRL_ENABLE_NONSECURE_PROT_MATCH_EN                              (0x00004000U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_SHIFT                                           (8U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_CLRMSK                                          (0xFFFFF0FFU)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_DEVICE_NON_BUFFERABLE                           (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_DEVICE_BUFFERABLE                               (0x00000100U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_NORMAL_NC_NON_BUFFERABLE                        (0x00000200U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_NORMAL_NC_BUFFERABLE                            (0x00000300U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_THROUGH_NO_ALLOCATE                       (0x00000600U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_THROUGH_WRITE_ALLOCATE                    (0x00000E00U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_NO_ALLOCATE                          (0x00000700U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE                       (0x00000F00U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__DEVICE_BUFFERABLE                         (0x00000100U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__NORMAL_NC_NON_BUFFERABLE                  (0x00000200U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__NORMAL_NC_BUFFERABLE                      (0x00000300U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_THROUGH_NO_ALLOCATE                 (0x00000600U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_THROUGH_WRITE_ALLOCATE              (0x00000E00U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_BACK_NO_ALLOCATE                    (0x00000700U)
+#define RGX_CR_ACE_CTRL_MMU_AWCACHE__VPU__WRITE_BACK_WRITE_ALLOCATE                 (0x00000F00U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_SHIFT                                           (4U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_CLRMSK                                          (0xFFFFFF0FU)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_DEVICE_NON_BUFFERABLE                           (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_DEVICE_BUFFERABLE                               (0x00000010U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_NORMAL_NC_NON_BUFFERABLE                        (0x00000020U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_NORMAL_NC_BUFFERABLE                            (0x00000030U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_THROUGH_NO_ALLOCATE                       (0x000000A0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_THROUGH_READ_ALLOCATE                     (0x000000E0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_NO_ALLOCATE                          (0x000000B0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE                        (0x000000F0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__DEVICE_NON_BUFFERABLE                     (0x00000000U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__DEVICE_BUFFERABLE                         (0x00000010U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__NORMAL_NC_NON_BUFFERABLE                  (0x00000020U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__NORMAL_NC_BUFFERABLE                      (0x00000030U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_THROUGH_NO_ALLOCATE                 (0x000000A0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_THROUGH_READ_ALLOCATE               (0x000000E0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_BACK_NO_ALLOCATE                    (0x000000B0U)
+#define RGX_CR_ACE_CTRL_MMU_ARCACHE__VPU__WRITE_BACK_READ_ALLOCATE                  (0x000000F0U)
+#define RGX_CR_ACE_CTRL_MMU_DOMAIN_SHIFT                                            (2U)
+#define RGX_CR_ACE_CTRL_MMU_DOMAIN_CLRMSK                                           (0xFFFFFFF3U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_SHIFT                                            (1U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_CLRMSK                                           (0xFFFFFFFDU)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_INNER_SHAREABLE                                  (0x00000000U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN_OUTER_SHAREABLE                                  (0x00000002U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN__VPU__INNER_SHAREABLE                            (0x00000000U)
+#define RGX_CR_ACE_CTRL_COH_DOMAIN__VPU__OUTER_SHAREABLE                            (0x00000002U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_SHIFT                                        (0U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_NON_SHAREABLE                                (0x00000000U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN_SYSTEM                                       (0x00000001U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN__VPU__NON_SHAREABLE                          (0x00000000U)
+#define RGX_CR_ACE_CTRL_NON_COH_DOMAIN__VPU__SYSTEM                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_SOC_AXI
+*/
+#define RGX_CR_SOC_AXI                                                              (0xE338U)
+#define RGX_CR_SOC_AXI_MASKFULL                                                     (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_SOC_AXI_NON_COHERENT_128_BYTE_BURST_SUPPORT_SHIFT                    (3U)
+#define RGX_CR_SOC_AXI_NON_COHERENT_128_BYTE_BURST_SUPPORT_CLRMSK                   (0xFFFFFFF7U)
+#define RGX_CR_SOC_AXI_NON_COHERENT_128_BYTE_BURST_SUPPORT_EN                       (0x00000008U)
+#define RGX_CR_SOC_AXI_COHERENT_128_BYTE_BURST_SUPPORT_SHIFT                        (2U)
+#define RGX_CR_SOC_AXI_COHERENT_128_BYTE_BURST_SUPPORT_CLRMSK                       (0xFFFFFFFBU)
+#define RGX_CR_SOC_AXI_COHERENT_128_BYTE_BURST_SUPPORT_EN                           (0x00000004U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_SHIFT                                      (0U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_CLRMSK                                     (0xFFFFFFFCU)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY                               (0x00000000U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY                         (0x00000001U)
+#define RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY                         (0x00000002U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING0
+*/
+#define RGX_CR_CONTEXT_MAPPING0                                                     (0xF078U)
+#define RGX_CR_CONTEXT_MAPPING0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING0_2D_SHIFT                                            (24U)
+#define RGX_CR_CONTEXT_MAPPING0_2D_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING0_CDM_SHIFT                                           (16U)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_CONTEXT_MAPPING0_3D_SHIFT                                            (8U)
+#define RGX_CR_CONTEXT_MAPPING0_3D_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_CONTEXT_MAPPING0_GEOM_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING0_GEOM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING2
+*/
+#define RGX_CR_CONTEXT_MAPPING2                                                     (0xF088U)
+#define RGX_CR_CONTEXT_MAPPING2_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING3
+*/
+#define RGX_CR_CONTEXT_MAPPING3                                                     (0xF090U)
+#define RGX_CR_CONTEXT_MAPPING3_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_OUTSTANDING_READ                                                 (0xF098U)
+#define RGX_CR_BIF_OUTSTANDING_READ_MASKFULL                                        (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_OUTSTANDING_READ_COUNTER_SHIFT                                   (0U)
+#define RGX_CR_BIF_OUTSTANDING_READ_COUNTER_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS1_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ                                          (0xF0A0U)
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ_COUNTER_SHIFT                            (0U)
+#define RGX_CR_BIF_TEXAS1_OUTSTANDING_READ_COUNTER_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS0_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ                                          (0xF0A8U)
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ_COUNTER_SHIFT                            (0U)
+#define RGX_CR_BIF_TEXAS0_OUTSTANDING_READ_COUNTER_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_PFS
+*/
+#define RGX_CR_BIF_PFS                                                              (0xF0B0U)
+#define RGX_CR_BIF_PFS_MASKFULL                                                     (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_BIF_PFS_SLC_STALLING_SHIFT                                           (8U)
+#define RGX_CR_BIF_PFS_SLC_STALLING_CLRMSK                                          (0xFFFFFEFFU)
+#define RGX_CR_BIF_PFS_SLC_STALLING_EN                                              (0x00000100U)
+#define RGX_CR_BIF_PFS_TILING_IDLE_SHIFT                                            (7U)
+#define RGX_CR_BIF_PFS_TILING_IDLE_CLRMSK                                           (0xFFFFFF7FU)
+#define RGX_CR_BIF_PFS_TILING_IDLE_EN                                               (0x00000080U)
+#define RGX_CR_BIF_PFS_ARB_IDLE_SHIFT                                               (6U)
+#define RGX_CR_BIF_PFS_ARB_IDLE_CLRMSK                                              (0xFFFFFFBFU)
+#define RGX_CR_BIF_PFS_ARB_IDLE_EN                                                  (0x00000040U)
+#define RGX_CR_BIF_PFS_RTN_MEM_IF_IDLE_SHIFT                                        (5U)
+#define RGX_CR_BIF_PFS_RTN_MEM_IF_IDLE_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_BIF_PFS_RTN_MEM_IF_IDLE_EN                                           (0x00000020U)
+#define RGX_CR_BIF_PFS_NONMETA_REQ_COMPLETE_SHIFT                                   (4U)
+#define RGX_CR_BIF_PFS_NONMETA_REQ_COMPLETE_CLRMSK                                  (0xFFFFFFEFU)
+#define RGX_CR_BIF_PFS_NONMETA_REQ_COMPLETE_EN                                      (0x00000010U)
+#define RGX_CR_BIF_PFS_NONMETA_WDATA_COMPLETE_SHIFT                                 (3U)
+#define RGX_CR_BIF_PFS_NONMETA_WDATA_COMPLETE_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_BIF_PFS_NONMETA_WDATA_COMPLETE_EN                                    (0x00000008U)
+#define RGX_CR_BIF_PFS_WDATA_IP_IDLE_SHIFT                                          (2U)
+#define RGX_CR_BIF_PFS_WDATA_IP_IDLE_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BIF_PFS_WDATA_IP_IDLE_EN                                             (0x00000004U)
+#define RGX_CR_BIF_PFS_REQUEST_IP_IDLE_SHIFT                                        (1U)
+#define RGX_CR_BIF_PFS_REQUEST_IP_IDLE_CLRMSK                                       (0xFFFFFFFDU)
+#define RGX_CR_BIF_PFS_REQUEST_IP_IDLE_EN                                           (0x00000002U)
+#define RGX_CR_BIF_PFS_STALL_COMPLETE_SHIFT                                         (0U)
+#define RGX_CR_BIF_PFS_STALL_COMPLETE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_BIF_PFS_STALL_COMPLETE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS0_PFS
+*/
+#define RGX_CR_BIF_TEXAS0_PFS                                                       (0xF0B8U)
+#define RGX_CR_BIF_TEXAS0_PFS_MASKFULL                                              (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_INFLIGHT_SHIFT                              (6U)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_INFLIGHT_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_INFLIGHT_EN                                 (0x00000040U)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_IDLE_SHIFT                                  (5U)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_IDLE_CLRMSK                                 (0xFFFFFFDFU)
+#define RGX_CR_BIF_TEXAS0_PFS_JONES_CFI_IDLE_EN                                     (0x00000020U)
+#define RGX_CR_BIF_TEXAS0_PFS_SLC_STALLING_SHIFT                                    (4U)
+#define RGX_CR_BIF_TEXAS0_PFS_SLC_STALLING_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_BIF_TEXAS0_PFS_SLC_STALLING_EN                                       (0x00000010U)
+#define RGX_CR_BIF_TEXAS0_PFS_TILING_IDLE_SHIFT                                     (3U)
+#define RGX_CR_BIF_TEXAS0_PFS_TILING_IDLE_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_BIF_TEXAS0_PFS_TILING_IDLE_EN                                        (0x00000008U)
+#define RGX_CR_BIF_TEXAS0_PFS_ARB_IDLE_SHIFT                                        (2U)
+#define RGX_CR_BIF_TEXAS0_PFS_ARB_IDLE_CLRMSK                                       (0xFFFFFFFBU)
+#define RGX_CR_BIF_TEXAS0_PFS_ARB_IDLE_EN                                           (0x00000004U)
+#define RGX_CR_BIF_TEXAS0_PFS_RTN_MEM_IF_IDLE_SHIFT                                 (1U)
+#define RGX_CR_BIF_TEXAS0_PFS_RTN_MEM_IF_IDLE_CLRMSK                                (0xFFFFFFFDU)
+#define RGX_CR_BIF_TEXAS0_PFS_RTN_MEM_IF_IDLE_EN                                    (0x00000002U)
+#define RGX_CR_BIF_TEXAS0_PFS_STALL_COMPLETE_SHIFT                                  (0U)
+#define RGX_CR_BIF_TEXAS0_PFS_STALL_COMPLETE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_BIF_TEXAS0_PFS_STALL_COMPLETE_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_TEXAS1_PFS
+*/
+#define RGX_CR_BIF_TEXAS1_PFS                                                       (0xF0C8U)
+#define RGX_CR_BIF_TEXAS1_PFS_MASKFULL                                              (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_PDSRW_SHIFT                        (6U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_PDSRW_CLRMSK                       (0xFFFFFFBFU)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_PDSRW_EN                           (0x00000040U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_MCU_SHIFT                          (5U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_MCU_CLRMSK                         (0xFFFFFFDFU)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_ACCUM_STALLING_MCU_EN                             (0x00000020U)
+#define RGX_CR_BIF_TEXAS1_PFS_SLC_STALLING_SHIFT                                    (4U)
+#define RGX_CR_BIF_TEXAS1_PFS_SLC_STALLING_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_BIF_TEXAS1_PFS_SLC_STALLING_EN                                       (0x00000010U)
+#define RGX_CR_BIF_TEXAS1_PFS_TILING_IDLE_SHIFT                                     (3U)
+#define RGX_CR_BIF_TEXAS1_PFS_TILING_IDLE_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_BIF_TEXAS1_PFS_TILING_IDLE_EN                                        (0x00000008U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_IDLE_SHIFT                                        (2U)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_IDLE_CLRMSK                                       (0xFFFFFFFBU)
+#define RGX_CR_BIF_TEXAS1_PFS_ARB_IDLE_EN                                           (0x00000004U)
+#define RGX_CR_BIF_TEXAS1_PFS_RTN_MEM_IF_IDLE_SHIFT                                 (1U)
+#define RGX_CR_BIF_TEXAS1_PFS_RTN_MEM_IF_IDLE_CLRMSK                                (0xFFFFFFFDU)
+#define RGX_CR_BIF_TEXAS1_PFS_RTN_MEM_IF_IDLE_EN                                    (0x00000002U)
+#define RGX_CR_BIF_TEXAS1_PFS_STALL_COMPLETE_SHIFT                                  (0U)
+#define RGX_CR_BIF_TEXAS1_PFS_STALL_COMPLETE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_BIF_TEXAS1_PFS_STALL_COMPLETE_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_FIX
+*/
+#define RGX_CR_JONES_FIX__ROGUE3                                                    (0xF0C0U)
+#define RGX_CR_JONES_FIX__ROGUE3__MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX__ROGUE3__DISABLE_SHIFT                                     (0U)
+#define RGX_CR_JONES_FIX__ROGUE3__DISABLE_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_FBCDC_STATUS
+*/
+#define RGX_CR_FBCDC_STATUS                                                         (0xF600U)
+#define RGX_CR_FBCDC_STATUS_MASKFULL                                                (IMG_UINT64_C(0x000000000F0F0F0F))
+#define RGX_CR_FBCDC_STATUS_STATE_CHECK_FAIL_SHIFT                                  (24U)
+#define RGX_CR_FBCDC_STATUS_STATE_CHECK_FAIL_CLRMSK                                 (0xF0FFFFFFU)
+#define RGX_CR_FBCDC_STATUS_UFOD_DECODER_ERROR_SHIFT                                (16U)
+#define RGX_CR_FBCDC_STATUS_UFOD_DECODER_ERROR_CLRMSK                               (0xFFF0FFFFU)
+#define RGX_CR_FBCDC_STATUS_HEADER_CHECK_FAIL_SHIFT                                 (8U)
+#define RGX_CR_FBCDC_STATUS_HEADER_CHECK_FAIL_CLRMSK                                (0xFFFFF0FFU)
+#define RGX_CR_FBCDC_STATUS_TILE_LENGTH_CHECK_FAIL_SHIFT                            (0U)
+#define RGX_CR_FBCDC_STATUS_TILE_LENGTH_CHECK_FAIL_CLRMSK                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FBCDC_SIGNATURE_STATUS
+*/
+#define RGX_CR_FBCDC_SIGNATURE_STATUS                                               (0xF618U)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_MASKFULL                                      (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBM_ERROR_SHIFT                               (4U)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBM_ERROR_CLRMSK                              (0xFFFFFF0FU)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBDC_ERROR_SHIFT                              (0U)
+#define RGX_CR_FBCDC_SIGNATURE_STATUS_FBDC_ERROR_CLRMSK                             (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING4
+*/
+#define RGX_CR_CONTEXT_MAPPING4                                                     (0xF210U)
+#define RGX_CR_CONTEXT_MAPPING4_MASKFULL                                            (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_SHIFT                                  (40U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_SHIFT                                    (32U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_SHIFT                                     (24U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_SHIFT                                  (16U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_SHIFT                                    (8U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_SHIFT                                     (0U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_FBCDC_IDLE
+*/
+#define RGX_CR_FBCDC_IDLE                                                           (0xF218U)
+#define RGX_CR_FBCDC_IDLE_MASKFULL                                                  (IMG_UINT64_C(0x0000000000003FFF))
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF3_SHIFT                                          (13U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF3_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF3_EN                                             (0x00002000U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF3_SHIFT                                         (12U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF3_CLRMSK                                        (0xFFFFEFFFU)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF3_EN                                            (0x00001000U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF3_SHIFT                                          (11U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF3_CLRMSK                                         (0xFFFFF7FFU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF3_EN                                             (0x00000800U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF2_SHIFT                                          (10U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF2_CLRMSK                                         (0xFFFFFBFFU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF2_EN                                             (0x00000400U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF2_SHIFT                                         (9U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF2_CLRMSK                                        (0xFFFFFDFFU)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF2_EN                                            (0x00000200U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF2_SHIFT                                          (8U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF2_CLRMSK                                         (0xFFFFFEFFU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF2_EN                                             (0x00000100U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF1_SHIFT                                          (7U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF1_CLRMSK                                         (0xFFFFFF7FU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF1_EN                                             (0x00000080U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF1_SHIFT                                         (6U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF1_CLRMSK                                        (0xFFFFFFBFU)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF1_EN                                            (0x00000040U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF1_SHIFT                                          (5U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF1_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF1_EN                                             (0x00000020U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF0_SHIFT                                          (4U)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF0_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_FBCDC_IDLE_FBC_MEMIF0_EN                                             (0x00000010U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF0_SHIFT                                         (3U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF0_CLRMSK                                        (0xFFFFFFF7U)
+#define RGX_CR_FBCDC_IDLE_FBDC_MEMIF0_EN                                            (0x00000008U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF0_SHIFT                                          (2U)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF0_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_FBCDC_IDLE_FBM_MEMIF0_EN                                             (0x00000004U)
+#define RGX_CR_FBCDC_IDLE_FBHC_SHIFT                                                (1U)
+#define RGX_CR_FBCDC_IDLE_FBHC_CLRMSK                                               (0xFFFFFFFDU)
+#define RGX_CR_FBCDC_IDLE_FBHC_EN                                                   (0x00000002U)
+#define RGX_CR_FBCDC_IDLE_FBSC_SHIFT                                                (0U)
+#define RGX_CR_FBCDC_IDLE_FBSC_CLRMSK                                               (0xFFFFFFFEU)
+#define RGX_CR_FBCDC_IDLE_FBSC_EN                                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_MERCER_SOFT_RESET
+*/
+#define RGX_CR_MERCER_SOFT_RESET                                                    (0x0630U)
+#define RGX_CR_MERCER_SOFT_RESET_MASKFULL                                           (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_SHIFT                                (62U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_CLRMSK                               (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_EN                                   (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_SHIFT                                (61U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_CLRMSK                               (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_EN                                   (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_SHIFT                                (60U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_CLRMSK                               (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_EN                                   (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_SHIFT                                (59U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_CLRMSK                               (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_EN                                   (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_SHIFT                                (58U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_EN                                   (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_SHIFT                                (57U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_EN                                   (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_SHIFT                                (56U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_EN                                   (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_SHIFT                                (55U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_EN                                   (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_SHIFT                                (54U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_EN                                   (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_SHIFT                                (53U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_EN                                   (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_SHIFT                                (52U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_EN                                   (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_SHIFT                                (51U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_EN                                   (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_SHIFT                                (50U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_EN                                   (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_SHIFT                                (49U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_EN                                   (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_SHIFT                                (48U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_EN                                   (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_SHIFT                                (47U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_EN                                   (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_SHIFT                                (46U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_EN                                   (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_SHIFT                                (45U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_EN                                   (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_SHIFT                                (44U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_EN                                   (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_SHIFT                                (43U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_EN                                   (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_SHIFT                                (42U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_EN                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_SHIFT                                (41U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_EN                                   (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_SHIFT                                (40U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_EN                                   (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_SHIFT                                (39U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_EN                                   (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_SHIFT                                (38U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_EN                                   (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_SHIFT                                (37U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_EN                                   (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_SHIFT                                (36U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_EN                                   (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_SHIFT                                (35U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_EN                                   (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_SHIFT                                (34U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_EN                                   (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_SHIFT                                (33U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_EN                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_SHIFT                                (32U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_EN                                   (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_SHIFT                                (31U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_EN                                   (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_SHIFT                                (30U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_EN                                   (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_SHIFT                                 (29U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_EN                                    (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_SHIFT                                 (28U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_EN                                    (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_SHIFT                                 (27U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_EN                                    (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_SHIFT                                 (26U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_EN                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_SHIFT                                 (25U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_EN                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_SHIFT                                 (24U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_EN                                    (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_SHIFT                                 (23U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_SHIFT                                 (22U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_SHIFT                                 (21U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_SHIFT                                 (20U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_SHIFT                                 (19U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_SHIFT                                 (18U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_SHIFT                                 (17U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_SHIFT                                 (16U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_SHIFT                                 (15U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_SHIFT                                 (14U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_SHIFT                                 (13U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_SHIFT                                 (12U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_SHIFT                                 (11U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_SHIFT                                 (10U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_SHIFT                                 (9U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_SHIFT                                 (8U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_SHIFT                                 (7U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_SHIFT                                 (6U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_SHIFT                                 (5U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_SHIFT                                 (4U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_SHIFT                                 (3U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_SHIFT                                 (2U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_EN                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_SHIFT                                 (1U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_EN                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_SHIFT                                 (0U)
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_EN                                    (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_TEXAS_SOFT_RESET
+*/
+#define RGX_CR_TEXAS_SOFT_RESET                                                     (0x0640U)
+#define RGX_CR_TEXAS_SOFT_RESET_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TEXAS_SOFT_RESET_SPU31_SHIFT                                         (31U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU31_CLRMSK                                        (0x7FFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU31_EN                                            (0x80000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU30_SHIFT                                         (30U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU30_CLRMSK                                        (0xBFFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU30_EN                                            (0x40000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU29_SHIFT                                         (29U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU29_CLRMSK                                        (0xDFFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU29_EN                                            (0x20000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU28_SHIFT                                         (28U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU28_CLRMSK                                        (0xEFFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU28_EN                                            (0x10000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU27_SHIFT                                         (27U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU27_CLRMSK                                        (0xF7FFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU27_EN                                            (0x08000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU26_SHIFT                                         (26U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU26_CLRMSK                                        (0xFBFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU26_EN                                            (0x04000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU25_SHIFT                                         (25U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU25_CLRMSK                                        (0xFDFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU25_EN                                            (0x02000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU24_SHIFT                                         (24U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU24_CLRMSK                                        (0xFEFFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU24_EN                                            (0x01000000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU23_SHIFT                                         (23U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU23_CLRMSK                                        (0xFF7FFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU23_EN                                            (0x00800000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU22_SHIFT                                         (22U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU22_CLRMSK                                        (0xFFBFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU22_EN                                            (0x00400000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU21_SHIFT                                         (21U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU21_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU21_EN                                            (0x00200000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU20_SHIFT                                         (20U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU20_CLRMSK                                        (0xFFEFFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU20_EN                                            (0x00100000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU19_SHIFT                                         (19U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU19_CLRMSK                                        (0xFFF7FFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU19_EN                                            (0x00080000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU18_SHIFT                                         (18U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU18_CLRMSK                                        (0xFFFBFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU18_EN                                            (0x00040000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU17_SHIFT                                         (17U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU17_CLRMSK                                        (0xFFFDFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU17_EN                                            (0x00020000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU16_SHIFT                                         (16U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU16_CLRMSK                                        (0xFFFEFFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU16_EN                                            (0x00010000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU15_SHIFT                                         (15U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU15_CLRMSK                                        (0xFFFF7FFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU15_EN                                            (0x00008000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU14_SHIFT                                         (14U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU14_CLRMSK                                        (0xFFFFBFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU14_EN                                            (0x00004000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU13_SHIFT                                         (13U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU13_CLRMSK                                        (0xFFFFDFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU13_EN                                            (0x00002000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU12_SHIFT                                         (12U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU12_CLRMSK                                        (0xFFFFEFFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU12_EN                                            (0x00001000U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU11_SHIFT                                         (11U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU11_CLRMSK                                        (0xFFFFF7FFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU11_EN                                            (0x00000800U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU10_SHIFT                                         (10U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU10_CLRMSK                                        (0xFFFFFBFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU10_EN                                            (0x00000400U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU9_SHIFT                                          (9U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU9_CLRMSK                                         (0xFFFFFDFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU9_EN                                             (0x00000200U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU8_SHIFT                                          (8U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU8_CLRMSK                                         (0xFFFFFEFFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU8_EN                                             (0x00000100U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU7_SHIFT                                          (7U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU7_CLRMSK                                         (0xFFFFFF7FU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU7_EN                                             (0x00000080U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU6_SHIFT                                          (6U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU6_CLRMSK                                         (0xFFFFFFBFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU6_EN                                             (0x00000040U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU5_SHIFT                                          (5U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU5_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU5_EN                                             (0x00000020U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU4_SHIFT                                          (4U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU4_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU4_EN                                             (0x00000010U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU3_SHIFT                                          (3U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU3_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU3_EN                                             (0x00000008U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU2_SHIFT                                          (2U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU2_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU2_EN                                             (0x00000004U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU1_SHIFT                                          (1U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU1_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU1_EN                                             (0x00000002U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU0_SHIFT                                          (0U)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU0_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_SOFT_RESET_SPU0_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_SWIFT_SOFT_RESET
+*/
+#define RGX_CR_SWIFT_SOFT_RESET                                                     (0x0650U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__MASKFULL                               (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT2_SHIFT                     (62U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT2_EN                        (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT1_SHIFT                     (61U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT1_EN                        (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT0_SHIFT                     (60U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU20_SWIFT0_EN                        (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT2_SHIFT                     (59U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT2_EN                        (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT1_SHIFT                     (58U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT1_EN                        (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT0_SHIFT                     (57U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU19_SWIFT0_EN                        (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT2_SHIFT                     (56U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT2_EN                        (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT1_SHIFT                     (55U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT1_EN                        (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT0_SHIFT                     (54U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU18_SWIFT0_EN                        (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT2_SHIFT                     (53U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT2_EN                        (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT1_SHIFT                     (52U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT1_EN                        (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT0_SHIFT                     (51U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU17_SWIFT0_EN                        (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT2_SHIFT                     (50U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT2_EN                        (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT1_SHIFT                     (49U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT1_EN                        (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT0_SHIFT                     (48U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU16_SWIFT0_EN                        (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT2_SHIFT                     (47U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT2_EN                        (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT1_SHIFT                     (46U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT1_EN                        (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT0_SHIFT                     (45U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU15_SWIFT0_EN                        (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT2_SHIFT                     (44U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT2_EN                        (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT1_SHIFT                     (43U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT1_EN                        (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT0_SHIFT                     (42U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU14_SWIFT0_EN                        (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT2_SHIFT                     (41U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT2_EN                        (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT1_SHIFT                     (40U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT1_EN                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT0_SHIFT                     (39U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU13_SWIFT0_EN                        (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT2_SHIFT                     (38U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT2_EN                        (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT1_SHIFT                     (37U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT1_EN                        (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT0_SHIFT                     (36U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU12_SWIFT0_EN                        (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT2_SHIFT                     (35U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT2_EN                        (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT1_SHIFT                     (34U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT1_EN                        (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT0_SHIFT                     (33U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU11_SWIFT0_EN                        (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT2_SHIFT                     (32U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT2_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT2_EN                        (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU31_SHIFT                                         (31U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU31_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU31_EN                                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT1_SHIFT                     (31U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT1_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT1_EN                        (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU30_SHIFT                                         (30U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU30_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU30_EN                                            (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT0_SHIFT                     (30U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT0_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU10_SWIFT0_EN                        (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU29_SHIFT                                         (29U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU29_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU29_EN                                            (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT2_SHIFT                      (29U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT2_EN                         (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU28_SHIFT                                         (28U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU28_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU28_EN                                            (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT1_SHIFT                      (28U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT1_EN                         (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU27_SHIFT                                         (27U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU27_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU27_EN                                            (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT0_SHIFT                      (27U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU9_SWIFT0_EN                         (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU26_SHIFT                                         (26U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU26_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU26_EN                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT2_SHIFT                      (26U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT2_EN                         (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU25_SHIFT                                         (25U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU25_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU25_EN                                            (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT1_SHIFT                      (25U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT1_EN                         (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU24_SHIFT                                         (24U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU24_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU24_EN                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT0_SHIFT                      (24U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU8_SWIFT0_EN                         (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU23_SHIFT                                         (23U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU23_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU23_EN                                            (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT2_SHIFT                      (23U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU22_SHIFT                                         (22U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU22_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU22_EN                                            (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT1_SHIFT                      (22U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU21_SHIFT                                         (21U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU21_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU21_EN                                            (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT0_SHIFT                      (21U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU7_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU20_SHIFT                                         (20U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU20_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU20_EN                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT2_SHIFT                      (20U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU19_SHIFT                                         (19U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU19_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU19_EN                                            (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT1_SHIFT                      (19U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU18_SHIFT                                         (18U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU18_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU18_EN                                            (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT0_SHIFT                      (18U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU6_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU17_SHIFT                                         (17U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU17_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU17_EN                                            (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT2_SHIFT                      (17U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU16_SHIFT                                         (16U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU16_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU16_EN                                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT1_SHIFT                      (16U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU15_SHIFT                                         (15U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU15_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU15_EN                                            (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT0_SHIFT                      (15U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU5_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU14_SHIFT                                         (14U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU14_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU14_EN                                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT2_SHIFT                      (14U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU13_SHIFT                                         (13U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU13_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU13_EN                                            (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT1_SHIFT                      (13U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU12_SHIFT                                         (12U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU12_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU12_EN                                            (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT0_SHIFT                      (12U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU4_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU11_SHIFT                                         (11U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU11_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU11_EN                                            (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT2_SHIFT                      (11U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU10_SHIFT                                         (10U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU10_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU10_EN                                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT1_SHIFT                      (10U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU9_SHIFT                                          (9U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU9_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU9_EN                                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT0_SHIFT                      (9U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU3_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU8_SHIFT                                          (8U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU8_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU8_EN                                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT2_SHIFT                      (8U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU7_SHIFT                                          (7U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU7_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU7_EN                                             (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT1_SHIFT                      (7U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU6_SHIFT                                          (6U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU6_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU6_EN                                             (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT0_SHIFT                      (6U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU2_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU5_SHIFT                                          (5U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU5_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU5_EN                                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT2_SHIFT                      (5U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU4_SHIFT                                          (4U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU4_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU4_EN                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT1_SHIFT                      (4U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU3_SHIFT                                          (3U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU3_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU3_EN                                             (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT0_SHIFT                      (3U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU1_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU2_SHIFT                                          (2U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU2_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU2_EN                                             (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT2_SHIFT                      (2U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT2_EN                         (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU1_SHIFT                                          (1U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU1_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU1_EN                                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT1_SHIFT                      (1U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT1_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT1_EN                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU0_SHIFT                                          (0U)
+#define RGX_CR_SWIFT_SOFT_RESET_SPU0_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SWIFT_SOFT_RESET_SPU0_EN                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT0_SHIFT                      (0U)
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT0_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SWIFT_SOFT_RESET__AXT_IF_GT0__SPU0_SWIFT0_EN                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_RAC_SOFT_RESET
+*/
+#define RGX_CR_RAC_SOFT_RESET                                                       (0x0660U)
+#define RGX_CR_RAC_SOFT_RESET_MASKFULL                                              (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC2_SHIFT                                      (62U)
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC2_CLRMSK                                     (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC2_EN                                         (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC1_SHIFT                                      (61U)
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC1_CLRMSK                                     (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC1_EN                                         (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC0_SHIFT                                      (60U)
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC0_CLRMSK                                     (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU20_RAC0_EN                                         (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC2_SHIFT                                      (59U)
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC2_CLRMSK                                     (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC2_EN                                         (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC1_SHIFT                                      (58U)
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC1_EN                                         (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC0_SHIFT                                      (57U)
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU19_RAC0_EN                                         (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC2_SHIFT                                      (56U)
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC2_EN                                         (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC1_SHIFT                                      (55U)
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC1_EN                                         (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC0_SHIFT                                      (54U)
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU18_RAC0_EN                                         (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC2_SHIFT                                      (53U)
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC2_EN                                         (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC1_SHIFT                                      (52U)
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC1_EN                                         (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC0_SHIFT                                      (51U)
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU17_RAC0_EN                                         (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC2_SHIFT                                      (50U)
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC2_EN                                         (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC1_SHIFT                                      (49U)
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC1_EN                                         (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC0_SHIFT                                      (48U)
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU16_RAC0_EN                                         (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC2_SHIFT                                      (47U)
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC2_EN                                         (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC1_SHIFT                                      (46U)
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC1_EN                                         (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC0_SHIFT                                      (45U)
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU15_RAC0_EN                                         (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC2_SHIFT                                      (44U)
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC2_EN                                         (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC1_SHIFT                                      (43U)
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC1_EN                                         (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC0_SHIFT                                      (42U)
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU14_RAC0_EN                                         (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC2_SHIFT                                      (41U)
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC2_EN                                         (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC1_SHIFT                                      (40U)
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC1_EN                                         (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC0_SHIFT                                      (39U)
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU13_RAC0_EN                                         (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC2_SHIFT                                      (38U)
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC2_EN                                         (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC1_SHIFT                                      (37U)
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC1_EN                                         (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC0_SHIFT                                      (36U)
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU12_RAC0_EN                                         (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC2_SHIFT                                      (35U)
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC2_EN                                         (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC1_SHIFT                                      (34U)
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC1_EN                                         (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC0_SHIFT                                      (33U)
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU11_RAC0_EN                                         (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC2_SHIFT                                      (32U)
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC2_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC2_EN                                         (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC1_SHIFT                                      (31U)
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC1_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC1_EN                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC0_SHIFT                                      (30U)
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC0_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU10_RAC0_EN                                         (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC2_SHIFT                                       (29U)
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC2_EN                                          (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC1_SHIFT                                       (28U)
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC1_EN                                          (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC0_SHIFT                                       (27U)
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU9_RAC0_EN                                          (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC2_SHIFT                                       (26U)
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC2_EN                                          (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC1_SHIFT                                       (25U)
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC1_EN                                          (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC0_SHIFT                                       (24U)
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU8_RAC0_EN                                          (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC2_SHIFT                                       (23U)
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC2_EN                                          (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC1_SHIFT                                       (22U)
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC1_EN                                          (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC0_SHIFT                                       (21U)
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU7_RAC0_EN                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC2_SHIFT                                       (20U)
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC2_EN                                          (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC1_SHIFT                                       (19U)
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC1_EN                                          (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC0_SHIFT                                       (18U)
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU6_RAC0_EN                                          (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC2_SHIFT                                       (17U)
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC2_EN                                          (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC1_SHIFT                                       (16U)
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC1_EN                                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC0_SHIFT                                       (15U)
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU5_RAC0_EN                                          (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC2_SHIFT                                       (14U)
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC2_EN                                          (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC1_SHIFT                                       (13U)
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC1_EN                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC0_SHIFT                                       (12U)
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU4_RAC0_EN                                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC2_SHIFT                                       (11U)
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC1_SHIFT                                       (10U)
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC0_SHIFT                                       (9U)
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU3_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC2_SHIFT                                       (8U)
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC1_SHIFT                                       (7U)
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC0_SHIFT                                       (6U)
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_RAC_SOFT_RESET_SPU2_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC2_SHIFT                                       (5U)
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC1_SHIFT                                       (4U)
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC0_SHIFT                                       (3U)
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_RAC_SOFT_RESET_SPU1_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC2_SHIFT                                       (2U)
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC2_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC2_EN                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC1_SHIFT                                       (1U)
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC1_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC1_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC0_SHIFT                                       (0U)
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC0_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_RAC_SOFT_RESET_SPU0_RAC0_EN                                          (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_RESET
+*/
+#define RGX_CR_FWCORE_WDT_RESET                                                     (0x4500U)
+#define RGX_CR_FWCORE_WDT_RESET_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WDT_RESET_EN_SHIFT                                            (0U)
+#define RGX_CR_FWCORE_WDT_RESET_EN_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_RESET_EN_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_CTRL
+*/
+#define RGX_CR_FWCORE_WDT_CTRL                                                      (0x4508U)
+#define RGX_CR_FWCORE_WDT_CTRL_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFF1F01))
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_SHIFT                                           (16U)
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_CLRMSK                                          (0x0000FFFFU)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_SHIFT                                      (8U)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_CLRMSK                                     (0xFFFFE0FFU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_COUNT
+*/
+#define RGX_CR_FWCORE_WDT_COUNT                                                     (0x4510U)
+#define RGX_CR_FWCORE_WDT_COUNT_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_CLRMSK                                        (0x00000000U)
+
+
+#endif /* RGX_CR_DEFS_KM_H */
+/*****************************************************************************
+ End of file (rgx_cr_defs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxdefs_km.h
new file mode 100644
index 000000000000..3342fceb3a12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxdefs_km.h
@@ -0,0 +1,440 @@
+/*************************************************************************/ /*!
+@Title          Rogue hw definitions (kernel mode)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXDEFS_KM_H
+#define RGXDEFS_KM_H
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+#include RGX_BVNC_CORE_KM_HEADER
+#include RGX_BNC_CONFIG_KM_HEADER
+#endif
+
+#define IMG_EXPLICIT_INCLUDE_HWDEFS
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#include "rgx_cr_defs_km.h"
+#endif
+#undef IMG_EXPLICIT_INCLUDE_HWDEFS
+
+#include "rgx_heap_firmware.h"
+
+/* The following Macros are picked up through BVNC headers for no hardware
+ * operations to be compatible with old build infrastructure.
+ */
+#if defined(NO_HARDWARE)
+/******************************************************************************
+ * Check for valid B.X.N.C
+ *****************************************************************************/
+#if !defined(RGX_BVNC_KM_B) || !defined(RGX_BVNC_KM_V) || !defined(RGX_BVNC_KM_N) || !defined(RGX_BVNC_KM_C)
+#error "Need to specify BVNC (RGX_BVNC_KM_B, RGX_BVNC_KM_V, RGX_BVNC_KM_N and RGX_BVNC_C)"
+#endif
+
+/* Check core/config compatibility */
+#if (RGX_BVNC_KM_B != RGX_BNC_KM_B) || (RGX_BVNC_KM_N != RGX_BNC_KM_N) || (RGX_BVNC_KM_C != RGX_BNC_KM_C)
+#error "BVNC headers are mismatching (KM core/config)"
+#endif
+#endif
+
+/******************************************************************************
+ * RGX Version name
+ *****************************************************************************/
+#define RGX_BVNC_KM_ST2(S)	#S
+#define RGX_BVNC_KM_ST(S)	RGX_BVNC_KM_ST2(S)
+#define RGX_BVNC_KM			RGX_BVNC_KM_ST(RGX_BVNC_KM_B) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_V) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_N) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_C)
+#define RGX_BVNC_KM_V_ST	RGX_BVNC_KM_ST(RGX_BVNC_KM_V)
+
+/* Maximum string size is [bb.vvvp.nnnn.cccc\0], includes null char */
+#define RGX_BVNC_STR_SIZE_MAX (2+1+4+1+4+1+4+1)
+#define RGX_BVNC_STR_FMTSPEC  "%u.%u.%u.%u"
+#define RGX_BVNC_STRP_FMTSPEC "%u.%up.%u.%u"
+
+
+/******************************************************************************
+ * RGX Defines
+ *****************************************************************************/
+
+#define BVNC_FIELD_MASK     ((1 << BVNC_FIELD_WIDTH) - 1)
+#define C_POSITION          (0)
+#define N_POSITION          ((C_POSITION) + (BVNC_FIELD_WIDTH))
+#define V_POSITION          ((N_POSITION) + (BVNC_FIELD_WIDTH))
+#define B_POSITION          ((V_POSITION) + (BVNC_FIELD_WIDTH))
+
+#define B_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (B_POSITION)))
+#define V_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (V_POSITION)))
+#define N_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (N_POSITION)))
+#define C_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (C_POSITION)))
+
+#define GET_B(x)            (((x) & (B_POSTION_MASK)) >> (B_POSITION))
+#define GET_V(x)            (((x) & (V_POSTION_MASK)) >> (V_POSITION))
+#define GET_N(x)            (((x) & (N_POSTION_MASK)) >> (N_POSITION))
+#define GET_C(x)            (((x) & (C_POSTION_MASK)) >> (C_POSITION))
+
+#define BVNC_PACK(B,V,N,C)  ((((IMG_UINT64)B)) << (B_POSITION) | \
+                             (((IMG_UINT64)V)) << (V_POSITION) | \
+                             (((IMG_UINT64)N)) << (N_POSITION) | \
+                             (((IMG_UINT64)C)) << (C_POSITION) \
+                            )
+
+#define RGX_CR_CORE_ID_CONFIG_N_SHIFT    (8U)
+#define RGX_CR_CORE_ID_CONFIG_C_SHIFT    (0U)
+
+#define RGX_CR_CORE_ID_CONFIG_N_CLRMSK   (0XFFFF00FFU)
+#define RGX_CR_CORE_ID_CONFIG_C_CLRMSK   (0XFFFFFF00U)
+
+/* The default number of OSID is 1, higher number implies VZ enabled firmware */
+#if !defined(RGXFW_NATIVE) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED + 1U > 1U)
+#define RGXFW_NUM_OS RGX_NUM_DRIVERS_SUPPORTED
+#else
+#define RGXFW_NUM_OS 1U
+#endif
+
+#if defined(RGX_FEATURE_NUM_OSIDS)
+#define RGXFW_MAX_NUM_OSIDS                               (RGX_FEATURE_NUM_OSIDS)
+#else
+#define RGXFW_MAX_NUM_OSIDS                               (8U)
+#endif
+
+#define RGXFW_HOST_DRIVER_ID                              (0U)
+#define RGXFW_GUEST_DRIVER_ID_START                       (RGXFW_HOST_DRIVER_ID + 1U)
+
+#define RGXFW_THREAD_0                                    (0U)
+#define RGXFW_THREAD_1                                    (1U)
+
+/* META cores (required for the RGX_FEATURE_META) */
+#define MTP218   (1)
+#define MTP219   (2)
+#define LTP218   (3)
+#define LTP217   (4)
+
+/* META Core memory feature depending on META variants */
+#define RGX_META_COREMEM_32K      (32*1024)
+#define RGX_META_COREMEM_48K      (48*1024)
+#define RGX_META_COREMEM_64K      (64*1024)
+#define RGX_META_COREMEM_96K      (96*1024)
+#define RGX_META_COREMEM_128K     (128*1024)
+#define RGX_META_COREMEM_256K     (256*1024)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(RGX_FEATURE_META_DMA)
+#undef SUPPORT_META_COREMEM
+#undef RGX_FEATURE_META_COREMEM_SIZE
+#define RGX_FEATURE_META_COREMEM_SIZE (0)
+#define RGX_META_COREMEM_SIZE         (0)
+#elif defined(RGX_FEATURE_META_COREMEM_SIZE)
+#define RGX_META_COREMEM_SIZE         (RGX_FEATURE_META_COREMEM_SIZE*1024U)
+#else
+#define RGX_META_COREMEM_SIZE         (0)
+#endif
+
+#if RGX_META_COREMEM_SIZE != 0
+#define RGX_META_COREMEM
+#define RGX_META_COREMEM_CODE
+#define RGX_META_COREMEM_DATA
+#endif
+#endif
+
+#define GET_ROGUE_CACHE_LINE_SIZE(x)    ((((IMG_INT32)x) > 0) ? ((x)/8) : (0))
+
+#if defined(RGX_FEATURE_META_DMA)
+#define RGX_META_DMA_BLOCK_SIZE (32U)
+#else
+#define RGX_META_DMA_BLOCK_SIZE (0U)
+#endif
+
+#if defined(SUPPORT_AGP)
+#if defined(SUPPORT_AGP4)
+#define MAX_HW_TA3DCONTEXTS	5U
+#else
+#define MAX_HW_TA3DCONTEXTS	3U
+#endif
+#else
+#define MAX_HW_TA3DCONTEXTS	2U
+#endif
+
+#define RGX_CR_CLK_CTRL0_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL0_MASKFULL)
+#define RGX_CR_CLK_CTRL0_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL0_MASKFULL)
+#define RGX_CR_CLK_CTRL1_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL1_MASKFULL)
+#define RGX_CR_CLK_CTRL1_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL1_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL2_MASKFULL)
+
+#if defined(__KERNEL__)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL0__PIPEDM_GT0__MASKFULL)
+#define RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL0__PIPEDM_GT0__MASKFULL)
+#endif
+
+#define RGX_CR_MERCER0_SOFT_RESET_SPU_EN (RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER0_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER0_EN)
+
+#define RGX_CR_MERCER1_SOFT_RESET_SPU_EN (RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER1_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER1_EN)
+
+#define RGX_CR_MERCER2_SOFT_RESET_SPU_EN (RGX_CR_MERCER_SOFT_RESET_SPU0_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU1_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU2_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU3_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU4_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU5_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU6_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU7_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU8_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU9_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU10_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU11_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU12_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU13_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU14_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU15_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU16_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU17_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU18_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU19_MERCER2_EN | \
+										  RGX_CR_MERCER_SOFT_RESET_SPU20_MERCER2_EN)
+
+
+/* SOFT_RESET steps as defined in the TRM */
+#define RGX_SOFT_RESET_JONES             (RGX_CR_SOFT_RESET_PM_EN | \
+                                          RGX_CR_SOFT_RESET_ISP_EN)
+#define RGX_SOFT_RESET_JONES_ALL         (RGX_SOFT_RESET_JONES | \
+                                          RGX_CR_SOFT_RESET_BIF_TEXAS_EN | \
+                                          RGX_CR_SOFT_RESET_BIF_JONES_EN | \
+										  RGX_CR_SOFT_RESET_SLC_EN | \
+                                          RGX_CR_SOFT_RESET_GARTEN_EN)
+#define RGX_SOFT_RESET_EXTRA             (RGX_CR_SOFT_RESET_PIXEL_EN | \
+                                          RGX_CR_SOFT_RESET_VERTEX_EN |  \
+                                          RGX_CR_SOFT_RESET_GEO_VERTEX_EN | \
+                                          RGX_CR_SOFT_RESET_GEO_SHARED_EN | \
+                                          RGX_CR_SOFT_RESET_COMPUTE_EN | \
+                                          RGX_CR_SOFT_RESET_TDM_EN)
+#define RGX_SOFT_RESET_FROM_WITHIN_CORE  (RGX_CR_SOFT_RESET_MASKFULL ^ \
+                                          (RGX_CR_SOFT_RESET_GARTEN_EN | \
+                                           RGX_CR_SOFT_RESET_BIF_JONES_EN | \
+                                           RGX_CR_SOFT_RESET_SLC_EN))
+
+#define RGX_MLIST_ENTRY_STRIDE	(4U) /* 4 bytes */
+#define RGX_NUM_PM_ADDR_SPACES	(2U) /* VCE & TE share virtual space and Alist */
+#define RGX_PM_MAX_PB_VIRT_ADDR_SPACE  (IMG_UINT64_C(0x400000000)) /* PM Maximum addressable limit */
+
+
+#define RGX_PM_MAX_RSTATE_SIZE_DWORDS (46U)
+#define RGX_PM_MLIST_BASE_ADDR_MAX_ALIGNSIZE (32U)
+#define RGX_PM_VHEAP_BASE_ADDR_MAX_ALIGNSIZE (32U)
+#define _RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_)[2] = ((_x_) & (IMG_UINT64_C(0x00000000fffffff0)))); \
+                                                          ((_ft_)[3] = (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32)); }
+#define _RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(_ft_,_x_) { ((_ft_)[6] = ((_x_) & (IMG_UINT64_C(0x00000000fffffff0)))); \
+                                                          ((_ft_)[7] = (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32)); }
+
+#define RGX_PM_RENDERSTATE_BASE_ADDR_ALIGNSIZE (32U)
+
+#define RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT		(12U)
+#define RGX_BIF_PM_PHYSICAL_PAGE_SIZE			(1U << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT		(14U)
+#define RGX_BIF_PM_VIRTUAL_PAGE_SIZE			(1U << RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE	(32U)
+
+/* To get the number of required Bernado/Phantom(s), divide
+ * the number of clusters by 4 and round up
+ */
+#define RGX_REQ_NUM_PHANTOMS(CLUSTERS) ((CLUSTERS + 3U) / 4U)
+#define RGX_REQ_NUM_BERNADOS(CLUSTERS) ((CLUSTERS + 3U) / 4U)
+#define RGX_REQ_NUM_BLACKPEARLS(CLUSTERS) ((CLUSTERS + 3U) / 4U)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+# define RGX_NUM_PHANTOMS (RGX_REQ_NUM_PHANTOMS(RGX_FEATURE_NUM_CLUSTERS))
+#endif
+
+/* for nohw multicore, true max number of cores returned to client */
+#define RGX_MULTICORE_MAX_NOHW_CORES                (8U)
+
+/*
+ * META second thread feature depending on META variants and
+ * available CoreMem
+ */
+#if defined(RGX_FEATURE_META) && (RGX_FEATURE_META == MTP218 || RGX_FEATURE_META == MTP219) && (RGX_FEATURE_META_COREMEM_SIZE >= 96) && (RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION == 0)
+#define RGXFW_META_SUPPORT_2ND_THREAD
+#endif
+
+/*
+ * FWCORE wrapper register defines
+ */
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT   RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK  RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK
+#define FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT     (12U)
+
+
+/*
+ * FBC clear color register defaults based on HW defaults
+ * non-YUV clear colour 0: 0x00000000 (encoded as ch3,2,1,0)
+ * non-YUV clear colour 1: 0x01000000 (encoded as ch3,2,1,0)
+ * YUV clear colour 0: 0x000 000 (encoded as UV Y)
+ * YUV clear colour 1: 0x000 3FF (encoded as UV Y)
+ */
+#define RGX_FBC_CC_DEFAULT (0x0100000000000000)
+#define RGX_FBC_CC_YUV_DEFAULT (0x000003FF00000000)
+
+/*
+ * Virtualisation definitions
+ */
+
+#define RGX_VIRTUALISATION_REG_SIZE_PER_OS (RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE)
+
+/*
+ * Renaming MTS sideband bitfields to emphasize that the Register Bank number
+ * of the MTS register used identifies a specific Driver/VM rather than the OSID tag
+ * emitted on bus memory transactions.
+ */
+#define RGX_MTS_SBDATA_DRIVERID_CLRMSK RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_CLRMSK
+#define RGX_MTS_SBDATA_DRIVERID_SHIFT RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_SHIFT
+
+/* Register Bank containing registers secured against host access */
+#define RGX_HOST_SECURE_REGBANK_OFFSET				(0xF0000U)
+#define RGX_HOST_SECURE_REGBANK_SIZE				(0x10000U)
+
+/* GPU CR timer tick in GPU cycles */
+#define RGX_CRTIME_TICK_IN_CYCLES					(256U)
+#define RGX_CRTIME_TICK_IN_CYCLES_SHIFT				(8U)
+
+#if defined(FIX_HW_BRN_71840)
+#define ROGUE_RENDERSIZE_MAXX						(16384U)
+#define ROGUE_RENDERSIZE_MAXY						(16384U)
+#else
+#define ROGUE_RENDERSIZE_MAXX						(RGX_FEATURE_RENDER_TARGET_XY_MAX)
+#define ROGUE_RENDERSIZE_MAXY						(RGX_FEATURE_RENDER_TARGET_XY_MAX)
+#endif
+
+/*
+ * Register used by the FW to track the current boot stage (not used in MIPS)
+ */
+#define RGX_FW_BOOT_STAGE_REGISTER     (RGX_CR_SCRATCH14)
+
+/*
+ * Define used to determine whether or not SLC range-based flush/invalidate
+ * interface is supported.
+ */
+#define RGX_SRV_SLC_RANGEBASED_CFI_SUPPORTED 1
+
+/*
+ * Macro used to indicate which version of HWPerf is active
+ */
+#define RGX_FEATURE_HWPERF_VOLCANIC
+
+/*
+ * Maximum number of cores supported by TRP
+ */
+#define RGX_TRP_MAX_NUM_CORES                           (8U)
+
+/*
+ * Maximum number of cores supported by WGP
+ */
+#define RGX_WGP_MAX_NUM_CORES                           (8U)
+
+/*
+ * Supports command to invalidate FBCDC descriptor state cache
+ */
+#define RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED 1
+
+#if defined(FIX_HW_BRN_71422)
+/*
+ * The BRN71422 software workaround requires a target physical address on
+ * the hardware platform with a low latency response time and which will
+ * not suffer from delays of DRAM hardware operations such as refresh and
+ * recalibration. Only with that address defined will the workaround be used.
+ */
+#if !defined(PDUMP)
+//#define RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR  (IMG_UINT64_C(0x0000000000))
+#endif
+#define RGX_BRN71422_WORKAROUND_READ_SIZE           (32U)
+#endif
+
+#define SUPPORT_VOLCANIC_TB
+#define RGX_FEATURE_SECURITY_VOLCANIC
+
+/* Typically the PCI bus returns this value on error */
+#define RGX_PCI_ERROR_VALUE_BYTE (0xFFU)
+#define RGX_PCI_ERROR_VALUE_DWORD (0xFFFFFFFFU)
+
+
+/*
+	Maximum number of render targets in array.
+	This derives from the maximum value for RGX_FEATURE_RTA_INDICES across all cores.
+*/
+#define RGX_MAX_RTA_INDICES (2048U)
+
+#endif /* RGXDEFS_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxmmudefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxmmudefs_km.h
new file mode 100644
index 000000000000..4dedb33467e6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxmmudefs_km.h
@@ -0,0 +1,278 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmmudefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_bif.def
+ */
+
+
+#ifndef RGXMMUDEFS_KM_H
+#define RGXMMUDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMMUDEFS_KM_REVISION 0
+
+#define OSID_CTXT_MAPPING_REGISTERS_CONSTANTS_OSID_CTXT_MAPPING_NUM_ENTRIES_VALUE   (0x00000010U)
+
+
+#define OSID_CTXT_MAPPING_REGISTERS_CONSTANTS_OSID_CTXT_MAPPING_PER_OS_SHIFT_VALUE  (0x00000004U)
+
+
+#define OSID_CTXT_MAPPING_REGISTERS_CONSTANTS_OSID_CTXT_MAPPING_PER_OS_MASK_VALUE   (0x00000007U)
+
+
+#define RGX_BIF_DM_ENCODING_VERTEX                                                  (0x00000000U)
+#define RGX_BIF_DM_ENCODING_PIXEL                                                   (0x00000001U)
+#define RGX_BIF_DM_ENCODING_COMPUTE                                                 (0x00000002U)
+#define RGX_BIF_DM_ENCODING_TLA                                                     (0x00000003U)
+#define RGX_BIF_DM_ENCODING_PB_VCE                                                  (0x00000004U)
+#define RGX_BIF_DM_ENCODING_PB_TE                                                   (0x00000005U)
+#define RGX_BIF_DM_ENCODING_META                                                    (0x00000007U)
+#define RGX_BIF_DM_ENCODING_HOST                                                    (0x00000008U)
+#define RGX_BIF_DM_ENCODING_PM_ALIST                                                (0x00000009U)
+
+
+#define RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT                                            (30U)
+#define RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF003FFFFFFF))
+#define RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT                                            (21U)
+#define RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFC01FFFFF))
+#define RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT                                            (12U)
+#define RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFE00FFF))
+
+
+#define RGX_MMUCTRL_ENTRIES_PC_VALUE                                                (0x00000400U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PD_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PT_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PC_VALUE                                             (0x00000020U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PD_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PT_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_PAGE_SIZE_MASK                                                  (0x00000007U)
+#define RGX_MMUCTRL_PAGE_SIZE_4KB                                                   (0x00000000U)
+#define RGX_MMUCTRL_PAGE_SIZE_16KB                                                  (0x00000001U)
+#define RGX_MMUCTRL_PAGE_SIZE_64KB                                                  (0x00000002U)
+#define RGX_MMUCTRL_PAGE_SIZE_256KB                                                 (0x00000003U)
+#define RGX_MMUCTRL_PAGE_SIZE_1MB                                                   (0x00000004U)
+#define RGX_MMUCTRL_PAGE_SIZE_2MB                                                   (0x00000005U)
+
+
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT                                            (12U)
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_SHIFT                                           (14U)
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF0000003FFF))
+
+
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_SHIFT                                           (16U)
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_SHIFT                                          (18U)
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFF000003FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_SHIFT                                            (20U)
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00000FFFFF))
+
+
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_SHIFT                                            (21U)
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00001FFFFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_SHIFT                                         (12U)
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_SHIFT                                        (10U)
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000003FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_SHIFT                                        (8U)
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000000FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_SHIFT                                       (6U)
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF000000003F))
+
+
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_AXCACHE_MASK                                                    (0x0000000FU)
+/*
+Device Non-bufferable */
+#define RGX_MMUCTRL_AXCACHE_DEVNONBUFF                                              (0x00000000U)
+/*
+Device Bufferable */
+#define RGX_MMUCTRL_AXCACHE_DEVBUFF                                                 (0x00000001U)
+/*
+Normal Non-cacheable Non-bufferable */
+#define RGX_MMUCTRL_AXCACHE_NORMNONBUFF                                             (0x00000002U)
+/*
+Normal Non-cacheable Bufferable */
+#define RGX_MMUCTRL_AXCACHE_NORMBUFF                                                (0x00000003U)
+/*
+Write-through No-allocate*/
+#define RGX_MMUCTRL_AXCACHE_WTNOALLOC                                               (0x00000006U)
+/*
+Write-back No-allocate*/
+#define RGX_MMUCTRL_AXCACHE_WBNOALLOC                                               (0x00000007U)
+/*
+Write-through Read-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WTRALLOC                                                (0x00000008U)
+/*
+Write-back Read-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WBRALLOC                                                (0x00000009U)
+/*
+Write-through Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WTWALLOC                                                (0x0000000aU)
+/*
+Write-back Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WBWALLOC                                                (0x0000000bU)
+/*
+Write-through Read/Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WTRWALLOC                                               (0x0000000eU)
+/*
+Write-back Read/Write-Allocate */
+#define RGX_MMUCTRL_AXCACHE_WBRWALLOC                                               (0x0000000fU)
+
+
+#define RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT                                         (63U)
+#define RGX_MMUCTRL_PT_DATA_PT_PARITY_CLRMSK                                        (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PT_PARITY_EN                                            (IMG_UINT64_C(0x8000000000000000))
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_SHIFT                                   (62U)
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_CLRMSK                                  (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_SHIFT                                           (58U)
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_CLRMSK                                          (IMG_UINT64_C(0xC3FFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_DEVNONBUFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_DEVBUFF                                         (IMG_UINT64_C(0x0400000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_NORMNONBUFF                                     (IMG_UINT64_C(0x0800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_NORMBUFF                                        (IMG_UINT64_C(0x0c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTNOALLOC                                       (IMG_UINT64_C(0x1800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBNOALLOC                                       (IMG_UINT64_C(0x1c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTRALLOC                                        (IMG_UINT64_C(0x2000000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBRALLOC                                        (IMG_UINT64_C(0x2400000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTWALLOC                                        (IMG_UINT64_C(0x2800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBWALLOC                                        (IMG_UINT64_C(0x2c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WTRWALLOC                                       (IMG_UINT64_C(0x3800000000000000))
+#define RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC                                       (IMG_UINT64_C(0x3c00000000000000))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_SHIFT                                        (40U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_CLRMSK                                       (IMG_UINT64_C(0xFC0000FFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PAGE_SHIFT                                              (12U)
+#define RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_SHIFT                                        (6U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_SHIFT                                     (5U)
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000000000000020))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_SHIFT                                            (4U)
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_EN                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_MMUCTRL_PT_DATA_CC_SHIFT                                                (2U)
+#define RGX_MMUCTRL_PT_DATA_CC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_MMUCTRL_PT_DATA_CC_EN                                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_EN                                            (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PT_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PT_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PT_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_SHIFT                                     (40U)
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_SHIFT                                           (5U)
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF1))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB                                         (IMG_UINT64_C(0x0000000000000006))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB                                           (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB                                           (IMG_UINT64_C(0x000000000000000a))
+#define RGX_MMUCTRL_PD_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PD_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PD_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT                                           (4U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK                                          (0x0000000FU)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT                                      (12U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSIZE                                       (4096U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_SHIFT                                     (1U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN                                        (0x00000002U)
+#define RGX_MMUCTRL_PC_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PC_DATA_VALID_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_MMUCTRL_PC_DATA_VALID_EN                                                (0x00000001U)
+
+
+#endif /* RGXMMUDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmmudefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxtbdefs_km.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxtbdefs_km.h
new file mode 100644
index 000000000000..c265ef0059b2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/km/rgxtbdefs_km.h
@@ -0,0 +1,1899 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxtbdefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ */
+
+
+#ifndef RGXTBDEFS_KM_H
+#define RGXTBDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXTBDEFS_KM_REVISION 1
+
+/*
+    Register RGX_TB_SOFT_RESET
+*/
+#define RGX_TB_SOFT_RESET                                                           (0x0000U)
+#define RGX_TB_SOFT_RESET_MASKFULL                                                  (IMG_UINT64_C(0x00000000FFFF0107))
+#define RGX_TB_SOFT_RESET_SPU_SHIFT                                                 (16U)
+#define RGX_TB_SOFT_RESET_SPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_TB_SOFT_RESET_JONES_SHIFT                                               (8U)
+#define RGX_TB_SOFT_RESET_JONES_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_TB_SOFT_RESET_JONES_EN                                                  (IMG_UINT64_C(0x0000000000000100))
+#define RGX_TB_SOFT_RESET_SYS_SHIFT                                                 (2U)
+#define RGX_TB_SOFT_RESET_SYS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_TB_SOFT_RESET_SYS_EN                                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_TB_SOFT_RESET_MEM_SHIFT                                                 (1U)
+#define RGX_TB_SOFT_RESET_MEM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_TB_SOFT_RESET_MEM_EN                                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_TB_SOFT_RESET_CORE_SHIFT                                                (0U)
+#define RGX_TB_SOFT_RESET_CORE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_SOFT_RESET_CORE_EN                                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_PCI_MASTER
+*/
+#define RGX_TB_PCI_MASTER                                                           (0x0008U)
+#define RGX_TB_PCI_MASTER_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_PCI_MASTER_MODE_SHIFT                                                (0U)
+#define RGX_TB_PCI_MASTER_MODE_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_PCI_MASTER_MODE_EN                                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_MEM_ARBITER
+*/
+#define RGX_TB_MEM_ARBITER                                                          (0x0088U)
+#define RGX_TB_MEM_ARBITER_MASKFULL                                                 (IMG_UINT64_C(0x0000000000010F11))
+#define RGX_TB_MEM_ARBITER_LIMIT_BW_SHIFT                                           (16U)
+#define RGX_TB_MEM_ARBITER_LIMIT_BW_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_TB_MEM_ARBITER_LIMIT_BW_EN                                              (IMG_UINT64_C(0x0000000000010000))
+#define RGX_TB_MEM_ARBITER_PRI_SKEW_SHIFT                                           (8U)
+#define RGX_TB_MEM_ARBITER_PRI_SKEW_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFF0FF))
+#define RGX_TB_MEM_ARBITER_PRI_RNW_SHIFT                                            (4U)
+#define RGX_TB_MEM_ARBITER_PRI_RNW_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_TB_MEM_ARBITER_PRI_RNW_EN                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_TB_MEM_ARBITER_ENABLE_SHIFT                                             (0U)
+#define RGX_TB_MEM_ARBITER_ENABLE_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_MEM_ARBITER_ENABLE_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_QOS_RD_LATENCY
+*/
+#define RGX_TB_QOS_RD_LATENCY                                                       (0x0090U)
+#define RGX_TB_QOS_RD_LATENCY_MASKFULL                                              (IMG_UINT64_C(0xFFFF3FFF3FFF3FFF))
+#define RGX_TB_QOS_RD_LATENCY_DIST_SHIFT                                            (62U)
+#define RGX_TB_QOS_RD_LATENCY_DIST_CLRMSK                                           (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_TB_QOS_RD_LATENCY_MAX_15_SHIFT                                          (48U)
+#define RGX_TB_QOS_RD_LATENCY_MAX_15_CLRMSK                                         (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_TB_QOS_RD_LATENCY_MIN_15_SHIFT                                          (32U)
+#define RGX_TB_QOS_RD_LATENCY_MIN_15_CLRMSK                                         (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_TB_QOS_RD_LATENCY_MAX_0_SHIFT                                           (16U)
+#define RGX_TB_QOS_RD_LATENCY_MAX_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFC000FFFF))
+#define RGX_TB_QOS_RD_LATENCY_MIN_0_SHIFT                                           (0U)
+#define RGX_TB_QOS_RD_LATENCY_MIN_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFC000))
+
+
+/*
+    Register RGX_TB_QOS_WR_LATENCY
+*/
+#define RGX_TB_QOS_WR_LATENCY                                                       (0x0098U)
+#define RGX_TB_QOS_WR_LATENCY_MASKFULL                                              (IMG_UINT64_C(0xFFFF3FFF3FFF3FFF))
+#define RGX_TB_QOS_WR_LATENCY_DIST_SHIFT                                            (62U)
+#define RGX_TB_QOS_WR_LATENCY_DIST_CLRMSK                                           (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_TB_QOS_WR_LATENCY_MAX_15_SHIFT                                          (48U)
+#define RGX_TB_QOS_WR_LATENCY_MAX_15_CLRMSK                                         (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_TB_QOS_WR_LATENCY_MIN_15_SHIFT                                          (32U)
+#define RGX_TB_QOS_WR_LATENCY_MIN_15_CLRMSK                                         (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_TB_QOS_WR_LATENCY_MAX_0_SHIFT                                           (16U)
+#define RGX_TB_QOS_WR_LATENCY_MAX_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFC000FFFF))
+#define RGX_TB_QOS_WR_LATENCY_MIN_0_SHIFT                                           (0U)
+#define RGX_TB_QOS_WR_LATENCY_MIN_0_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFC000))
+
+
+/*
+    Register RGX_TB_MAX_ID_OUTSTANDING
+*/
+#define RGX_TB_MAX_ID_OUTSTANDING                                                   (0x00B0U)
+#define RGX_TB_MAX_ID_OUTSTANDING_MASKFULL                                          (IMG_UINT64_C(0x000003FF03FF03FF))
+#define RGX_TB_MAX_ID_OUTSTANDING_RD_WR_SHIFT                                       (32U)
+#define RGX_TB_MAX_ID_OUTSTANDING_RD_WR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_TB_MAX_ID_OUTSTANDING_WRITE_SHIFT                                       (16U)
+#define RGX_TB_MAX_ID_OUTSTANDING_WRITE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_TB_MAX_ID_OUTSTANDING_READ_SHIFT                                        (0U)
+#define RGX_TB_MAX_ID_OUTSTANDING_READ_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFC00))
+
+
+/*
+    Register RGX_TB_COHERENT_MEM_REGION
+*/
+#define RGX_TB_COHERENT_MEM_REGION                                                  (0x00C0U)
+#define RGX_TB_COHERENT_MEM_REGION_MASKFULL                                         (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_TB_COHERENT_MEM_REGION_START_ADDR_SHIFT                                 (12U)
+#define RGX_TB_COHERENT_MEM_REGION_START_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+/*
+    Register RGX_TB_LMA_MEM_REGION
+*/
+#define RGX_TB_LMA_MEM_REGION                                                       (0x00C8U)
+#define RGX_TB_LMA_MEM_REGION_MASKFULL                                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_LMA_MEM_REGION_SIZE_SHIFT                                            (0U)
+#define RGX_TB_LMA_MEM_REGION_SIZE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_UMA_MEM_REGION
+*/
+#define RGX_TB_UMA_MEM_REGION                                                       (0x00D0U)
+#define RGX_TB_UMA_MEM_REGION_MASKFULL                                              (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_TB_UMA_MEM_REGION_START_ADDR_SHIFT                                      (12U)
+#define RGX_TB_UMA_MEM_REGION_START_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+/*
+    Register RGX_TB_SYSTEM_STATUS
+*/
+#define RGX_TB_SYSTEM_STATUS                                                        (0x00E0U)
+#define RGX_TB_SYSTEM_STATUS_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFF3FF700FF))
+#define RGX_TB_SYSTEM_STATUS_SPU_ISON_SHIFT                                         (48U)
+#define RGX_TB_SYSTEM_STATUS_SPU_ISON_CLRMSK                                        (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_SPU_POWER_SHIFT                                        (32U)
+#define RGX_TB_SYSTEM_STATUS_SPU_POWER_CLRMSK                                       (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_DAVY_ISON_SHIFT                                        (29U)
+#define RGX_TB_SYSTEM_STATUS_DAVY_ISON_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_DAVY_ISON_EN                                           (IMG_UINT64_C(0x0000000020000000))
+#define RGX_TB_SYSTEM_STATUS_DAVY_POWER_SHIFT                                       (28U)
+#define RGX_TB_SYSTEM_STATUS_DAVY_POWER_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_DAVY_POWER_EN                                          (IMG_UINT64_C(0x0000000010000000))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_ISON_SHIFT                                 (27U)
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_ISON_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_ISON_EN                                    (IMG_UINT64_C(0x0000000008000000))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_POWER_SHIFT                                (26U)
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_POWER_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_CHEST_ENOLA_POWER_EN                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_ABORT_SHIFT                           (25U)
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_ABORT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_ABORT_EN                              (IMG_UINT64_C(0x0000000002000000))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_COMPLETE_SHIFT                        (24U)
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_COMPLETE_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_TB_SYSTEM_STATUS_HOST_POWER_EVENT_COMPLETE_EN                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_TB_SYSTEM_STATUS_GPU_STATE_SHIFT                                        (20U)
+#define RGX_TB_SYSTEM_STATUS_GPU_STATE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFF0FFFFF))
+#define RGX_TB_SYSTEM_STATUS_SYSTEM_IRQ_SHIFT                                       (18U)
+#define RGX_TB_SYSTEM_STATUS_SYSTEM_IRQ_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_TB_SYSTEM_STATUS_SYSTEM_IRQ_EN                                          (IMG_UINT64_C(0x0000000000040000))
+#define RGX_TB_SYSTEM_STATUS_TRIGGER_SHIFT                                          (17U)
+#define RGX_TB_SYSTEM_STATUS_TRIGGER_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_TB_SYSTEM_STATUS_TRIGGER_EN                                             (IMG_UINT64_C(0x0000000000020000))
+#define RGX_TB_SYSTEM_STATUS_HMMU_IRQ_SHIFT                                         (16U)
+#define RGX_TB_SYSTEM_STATUS_HMMU_IRQ_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_TB_SYSTEM_STATUS_HMMU_IRQ_EN                                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_TB_SYSTEM_STATUS_IRQ_SHIFT                                              (0U)
+#define RGX_TB_SYSTEM_STATUS_IRQ_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_TB_SYSTEM_CONFIG
+*/
+#define RGX_TB_SYSTEM_CONFIG                                                        (0x00F0U)
+#define RGX_TB_SYSTEM_CONFIG_MASKFULL                                               (IMG_UINT64_C(0x0000000007007737))
+#define RGX_TB_SYSTEM_CONFIG_SOC_AXI_FEATURE_SHIFT                                  (24U)
+#define RGX_TB_SYSTEM_CONFIG_SOC_AXI_FEATURE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF8FFFFFF))
+#define RGX_TB_SYSTEM_CONFIG_ICE_CABLEIF256_SHIFT                                   (14U)
+#define RGX_TB_SYSTEM_CONFIG_ICE_CABLEIF256_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_TB_SYSTEM_CONFIG_ICE_CABLEIF256_EN                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_TBX_SHIFT                                       (13U)
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_TBX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_TBX_EN                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_ICE_SHIFT                                       (12U)
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_ICE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_TB_SYSTEM_CONFIG_VELOCE_ICE_EN                                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_AVIP_SHIFT                                     (10U)
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_AVIP_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_AVIP_EN                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_TBA_SHIFT                                      (9U)
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_TBA_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_TBA_EN                                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_ICE_SHIFT                                      (8U)
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_ICE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_TB_SYSTEM_CONFIG_CADENCE_ICE_EN                                         (IMG_UINT64_C(0x0000000000000100))
+#define RGX_TB_SYSTEM_CONFIG_EMU_UMA_SHIFT                                          (5U)
+#define RGX_TB_SYSTEM_CONFIG_EMU_UMA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_TB_SYSTEM_CONFIG_EMU_UMA_EN                                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_TB_SYSTEM_CONFIG_EMU_BUILD_SHIFT                                        (4U)
+#define RGX_TB_SYSTEM_CONFIG_EMU_BUILD_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_TB_SYSTEM_CONFIG_EMU_BUILD_EN                                           (IMG_UINT64_C(0x0000000000000010))
+#define RGX_TB_SYSTEM_CONFIG_NS_NOC_SHIFT                                           (2U)
+#define RGX_TB_SYSTEM_CONFIG_NS_NOC_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_TB_SYSTEM_CONFIG_NS_NOC_EN                                              (IMG_UINT64_C(0x0000000000000004))
+#define RGX_TB_SYSTEM_CONFIG_IMG_NOC_SHIFT                                          (1U)
+#define RGX_TB_SYSTEM_CONFIG_IMG_NOC_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_TB_SYSTEM_CONFIG_IMG_NOC_EN                                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_TB_SYSTEM_CONFIG_TB_NONCOHERENT_SHIFT                                   (0U)
+#define RGX_TB_SYSTEM_CONFIG_TB_NONCOHERENT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_SYSTEM_CONFIG_TB_NONCOHERENT_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_TB_DOMAINSPLIT_TYPE_RESERVED_SHIFT                                      (9U)
+#define RGX_TB_DOMAINSPLIT_TYPE_RESERVED_CLRMSK                                     (0xFFFF01FFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU7_SHIFT                                          (8U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU7_CLRMSK                                         (0xFFFFFEFFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU7_EN                                             (0x00000100U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU6_SHIFT                                          (7U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU6_CLRMSK                                         (0xFFFFFF7FU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU6_EN                                             (0x00000080U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU5_SHIFT                                          (6U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU5_CLRMSK                                         (0xFFFFFFBFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU5_EN                                             (0x00000040U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU4_SHIFT                                          (5U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU4_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU4_EN                                             (0x00000020U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU3_SHIFT                                          (4U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU3_EN                                             (0x00000010U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU2_SHIFT                                          (3U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU2_EN                                             (0x00000008U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU1_SHIFT                                          (2U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU1_EN                                             (0x00000004U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU0_SHIFT                                          (1U)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_TB_DOMAINSPLIT_TYPE_SPU0_EN                                             (0x00000002U)
+#define RGX_TB_DOMAINSPLIT_TYPE_JONES_SHIFT                                         (0U)
+#define RGX_TB_DOMAINSPLIT_TYPE_JONES_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_TB_DOMAINSPLIT_TYPE_JONES_EN                                            (0x00000001U)
+
+
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_MASK                                        (0xFFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER31_SHIFT                             (31U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER31_CLRMSK                            (0x7FFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER31_EN                                (0x80000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER30_SHIFT                             (30U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER30_CLRMSK                            (0xBFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER30_EN                                (0x40000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER29_SHIFT                             (29U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER29_CLRMSK                            (0xDFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER29_EN                                (0x20000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER28_SHIFT                             (28U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER28_CLRMSK                            (0xEFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER28_EN                                (0x10000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER27_SHIFT                             (27U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER27_CLRMSK                            (0xF7FFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER27_EN                                (0x08000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER26_SHIFT                             (26U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER26_CLRMSK                            (0xFBFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER26_EN                                (0x04000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER25_SHIFT                             (25U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER25_CLRMSK                            (0xFDFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER25_EN                                (0x02000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER24_SHIFT                             (24U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER24_CLRMSK                            (0xFEFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER24_EN                                (0x01000000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER23_SHIFT                             (23U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER23_CLRMSK                            (0xFF7FFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER23_EN                                (0x00800000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER22_SHIFT                             (22U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER22_CLRMSK                            (0xFFBFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER22_EN                                (0x00400000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER21_SHIFT                             (21U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER21_CLRMSK                            (0xFFDFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER21_EN                                (0x00200000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER20_SHIFT                             (20U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER20_CLRMSK                            (0xFFEFFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER20_EN                                (0x00100000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER19_SHIFT                             (19U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER19_CLRMSK                            (0xFFF7FFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER19_EN                                (0x00080000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER18_SHIFT                             (18U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER18_CLRMSK                            (0xFFFBFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER18_EN                                (0x00040000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER17_SHIFT                             (17U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER17_CLRMSK                            (0xFFFDFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER17_EN                                (0x00020000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER16_SHIFT                             (16U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER16_CLRMSK                            (0xFFFEFFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER16_EN                                (0x00010000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER15_SHIFT                             (15U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER15_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER15_EN                                (0x00008000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER14_SHIFT                             (14U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER14_CLRMSK                            (0xFFFFBFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER14_EN                                (0x00004000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER13_SHIFT                             (13U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER13_CLRMSK                            (0xFFFFDFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER13_EN                                (0x00002000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER12_SHIFT                             (12U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER12_CLRMSK                            (0xFFFFEFFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER12_EN                                (0x00001000U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER11_SHIFT                             (11U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER11_CLRMSK                            (0xFFFFF7FFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER11_EN                                (0x00000800U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER10_SHIFT                             (10U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER10_CLRMSK                            (0xFFFFFBFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER10_EN                                (0x00000400U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER9_SHIFT                              (9U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER9_CLRMSK                             (0xFFFFFDFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER9_EN                                 (0x00000200U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER8_SHIFT                              (8U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER8_CLRMSK                             (0xFFFFFEFFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER8_EN                                 (0x00000100U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER7_SHIFT                              (7U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER7_CLRMSK                             (0xFFFFFF7FU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER7_EN                                 (0x00000080U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER6_SHIFT                              (6U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER6_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER6_EN                                 (0x00000040U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER5_SHIFT                              (5U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER5_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER5_EN                                 (0x00000020U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER4_SHIFT                              (4U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER4_CLRMSK                             (0xFFFFFFEFU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER4_EN                                 (0x00000010U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER3_SHIFT                              (3U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER3_CLRMSK                             (0xFFFFFFF7U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER3_EN                                 (0x00000008U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER2_SHIFT                              (2U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER2_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER2_EN                                 (0x00000004U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER1_SHIFT                              (1U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER1_CLRMSK                             (0xFFFFFFFDU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER1_EN                                 (0x00000002U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER0_SHIFT                              (0U)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER0_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_TB_DOMAINSPLIT_CLUSTER_TYPE_CLUSTER0_EN                                 (0x00000001U)
+
+
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC31_SHIFT                                     (31U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC31_CLRMSK                                    (0x7FFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC31_EN                                        (0x80000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC30_SHIFT                                     (30U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC30_CLRMSK                                    (0xBFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC30_EN                                        (0x40000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC29_SHIFT                                     (29U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC29_CLRMSK                                    (0xDFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC29_EN                                        (0x20000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC28_SHIFT                                     (28U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC28_CLRMSK                                    (0xEFFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC28_EN                                        (0x10000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC27_SHIFT                                     (27U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC27_CLRMSK                                    (0xF7FFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC27_EN                                        (0x08000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC26_SHIFT                                     (26U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC26_CLRMSK                                    (0xFBFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC26_EN                                        (0x04000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC25_SHIFT                                     (25U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC25_CLRMSK                                    (0xFDFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC25_EN                                        (0x02000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC24_SHIFT                                     (24U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC24_CLRMSK                                    (0xFEFFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC24_EN                                        (0x01000000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC23_SHIFT                                     (23U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC23_CLRMSK                                    (0xFF7FFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC23_EN                                        (0x00800000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC22_SHIFT                                     (22U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC22_CLRMSK                                    (0xFFBFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC22_EN                                        (0x00400000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC21_SHIFT                                     (21U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC21_CLRMSK                                    (0xFFDFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC21_EN                                        (0x00200000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC20_SHIFT                                     (20U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC20_CLRMSK                                    (0xFFEFFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC20_EN                                        (0x00100000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC19_SHIFT                                     (19U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC19_CLRMSK                                    (0xFFF7FFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC19_EN                                        (0x00080000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC18_SHIFT                                     (18U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC18_CLRMSK                                    (0xFFFBFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC18_EN                                        (0x00040000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC17_SHIFT                                     (17U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC17_CLRMSK                                    (0xFFFDFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC17_EN                                        (0x00020000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC16_SHIFT                                     (16U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC16_CLRMSK                                    (0xFFFEFFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC16_EN                                        (0x00010000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC15_SHIFT                                     (15U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC15_CLRMSK                                    (0xFFFF7FFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC15_EN                                        (0x00008000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC14_SHIFT                                     (14U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC14_CLRMSK                                    (0xFFFFBFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC14_EN                                        (0x00004000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC13_SHIFT                                     (13U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC13_CLRMSK                                    (0xFFFFDFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC13_EN                                        (0x00002000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC12_SHIFT                                     (12U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC12_CLRMSK                                    (0xFFFFEFFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC12_EN                                        (0x00001000U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC11_SHIFT                                     (11U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC11_CLRMSK                                    (0xFFFFF7FFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC11_EN                                        (0x00000800U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC10_SHIFT                                     (10U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC10_CLRMSK                                    (0xFFFFFBFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC10_EN                                        (0x00000400U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC9_SHIFT                                      (9U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC9_CLRMSK                                     (0xFFFFFDFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC9_EN                                         (0x00000200U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC8_SHIFT                                      (8U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC8_CLRMSK                                     (0xFFFFFEFFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC8_EN                                         (0x00000100U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC7_SHIFT                                      (7U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC7_CLRMSK                                     (0xFFFFFF7FU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC7_EN                                         (0x00000080U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC6_SHIFT                                      (6U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC6_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC6_EN                                         (0x00000040U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC5_SHIFT                                      (5U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC5_CLRMSK                                     (0xFFFFFFDFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC5_EN                                         (0x00000020U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC4_SHIFT                                      (4U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC4_CLRMSK                                     (0xFFFFFFEFU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC4_EN                                         (0x00000010U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC3_SHIFT                                      (3U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC3_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC3_EN                                         (0x00000008U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC2_SHIFT                                      (2U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC2_CLRMSK                                     (0xFFFFFFFBU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC2_EN                                         (0x00000004U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC1_SHIFT                                      (1U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC1_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC1_EN                                         (0x00000002U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC0_SHIFT                                      (0U)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC0_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_TB_DOMAINSPLIT_RAC_TYPE_RAC0_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT
+*/
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3                                  (0x0100U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__MASKFULL                        (IMG_UINT64_C(0xFFFFFFFF0000FF03))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_SHIFT            (32U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLRMSK           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER31_SHIFT  (63U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER31_CLRMSK (IMG_UINT64_C(0x7fffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER30_SHIFT  (62U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER30_CLRMSK (IMG_UINT64_C(0xbfffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER29_SHIFT  (61U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER29_CLRMSK (IMG_UINT64_C(0xdfffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER28_SHIFT  (60U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER28_CLRMSK (IMG_UINT64_C(0xefffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER27_SHIFT  (59U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER27_CLRMSK (IMG_UINT64_C(0xf7ffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER26_SHIFT  (58U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER26_CLRMSK (IMG_UINT64_C(0xfbffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER25_SHIFT  (57U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER25_CLRMSK (IMG_UINT64_C(0xfdffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER24_SHIFT  (56U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER24_CLRMSK (IMG_UINT64_C(0xfeffffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER23_SHIFT  (55U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER23_CLRMSK (IMG_UINT64_C(0xff7fffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER22_SHIFT  (54U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER22_CLRMSK (IMG_UINT64_C(0xffbfffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER21_SHIFT  (53U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER21_CLRMSK (IMG_UINT64_C(0xffdfffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER20_SHIFT  (52U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER20_CLRMSK (IMG_UINT64_C(0xffefffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER19_SHIFT  (51U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER19_CLRMSK (IMG_UINT64_C(0xfff7ffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER18_SHIFT  (50U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER18_CLRMSK (IMG_UINT64_C(0xfffbffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER17_SHIFT  (49U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER17_CLRMSK (IMG_UINT64_C(0xfffdffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER16_SHIFT  (48U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER16_CLRMSK (IMG_UINT64_C(0xfffeffffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER15_SHIFT  (47U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER15_CLRMSK (IMG_UINT64_C(0xffff7fffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER14_SHIFT  (46U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER14_CLRMSK (IMG_UINT64_C(0xffffbfffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER13_SHIFT  (45U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER13_CLRMSK (IMG_UINT64_C(0xffffdfffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER12_SHIFT  (44U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER12_CLRMSK (IMG_UINT64_C(0xffffefffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER11_SHIFT  (43U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER11_CLRMSK (IMG_UINT64_C(0xfffff7ffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER10_SHIFT  (42U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER10_CLRMSK (IMG_UINT64_C(0xfffffbffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER9_SHIFT   (41U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER9_CLRMSK  (IMG_UINT64_C(0xfffffdffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER8_SHIFT   (40U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER8_CLRMSK  (IMG_UINT64_C(0xfffffeffffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER7_SHIFT   (39U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER7_CLRMSK  (IMG_UINT64_C(0xffffff7fffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER6_SHIFT   (38U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER6_CLRMSK  (IMG_UINT64_C(0xffffffbfffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER5_SHIFT   (37U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER5_CLRMSK  (IMG_UINT64_C(0xffffffdfffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER4_SHIFT   (36U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER4_CLRMSK  (IMG_UINT64_C(0xffffffefffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER3_SHIFT   (35U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER3_CLRMSK  (IMG_UINT64_C(0xfffffff7ffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER2_SHIFT   (34U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER2_CLRMSK  (IMG_UINT64_C(0xfffffffbffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER1_SHIFT   (33U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER1_CLRMSK  (IMG_UINT64_C(0xfffffffdffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER0_SHIFT   (32U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__DOMAIN_CLUSTER_CLUSTER0_CLRMSK  (IMG_UINT64_C(0xfffffffeffffffff))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__GPU_MASK_SHIFT                  (8U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__GPU_MASK_CLRMSK                 (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__REQ_SHIFT                       (1U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__REQ_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__REQ_EN                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_SHIFT                      (0U)
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_POWER_DOWN                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_TB_HOST_POWER_EVENT__POWER_ISLAND_GEQ3__TYPE_POWER_UP                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_GPU_CONTROL
+*/
+#define RGX_TB_GPU_CONTROL                                                          (0x0110U)
+#define RGX_TB_GPU_CONTROL_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000007))
+#define RGX_TB_GPU_CONTROL_FW_LOG_DISABLE_SHIFT                                     (2U)
+#define RGX_TB_GPU_CONTROL_FW_LOG_DISABLE_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_TB_GPU_CONTROL_FW_LOG_DISABLE_EN                                        (0x00000004U)
+#define RGX_TB_GPU_CONTROL_DXT_BC_ENABLE_SHIFT                                      (1U)
+#define RGX_TB_GPU_CONTROL_DXT_BC_ENABLE_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_TB_GPU_CONTROL_DXT_BC_ENABLE_EN                                         (0x00000002U)
+#define RGX_TB_GPU_CONTROL_ASTC_ENABLE_SHIFT                                        (0U)
+#define RGX_TB_GPU_CONTROL_ASTC_ENABLE_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_CONTROL_ASTC_ENABLE_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT_JONES
+*/
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5      (0x0118U)
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__MASKFULL (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__DOMAIN_SHIFT (0U)
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__DOMAIN_CLRMSK (0xFFFFFFFEU)
+#define RGX_TB_HOST_POWER_EVENT_JONES__POWER_ISLAND_GEQ2_AND_POWER_ISLAND_LEQ5__DOMAIN_EN (0x00000001U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT_TEXAS
+*/
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5      (0x0168U)
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__MASKFULL (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_SHIFT (0U)
+#define RGX_TB_HOST_POWER_EVENT_TEXAS__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_CLRMSK (0xFFFF0000U)
+
+
+/*
+    Register RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA
+*/
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5 (0x0170U)
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__MASKFULL (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_SHIFT (0U)
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_CLRMSK (0xFFFFFFFEU)
+#define RGX_TB_HOST_POWER_EVENT_CHEST_ENOLA__POWER_ISLAND_GEQ3_AND_POWER_ISLAND_LEQ5__DOMAIN_EN (0x00000001U)
+
+
+/*
+    Register RGX_TB_TRIGGER_SHORT
+*/
+#define RGX_TB_TRIGGER_SHORT                                                        (0x1500U)
+#define RGX_TB_TRIGGER_SHORT_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_SHORT_TIMEOUT_SHIFT                                          (0U)
+#define RGX_TB_TRIGGER_SHORT_TIMEOUT_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRIGGER_MEDIUM
+*/
+#define RGX_TB_TRIGGER_MEDIUM                                                       (0x1508U)
+#define RGX_TB_TRIGGER_MEDIUM_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_MEDIUM_TIMEOUT_SHIFT                                         (0U)
+#define RGX_TB_TRIGGER_MEDIUM_TIMEOUT_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRIGGER_HIGH
+*/
+#define RGX_TB_TRIGGER_HIGH                                                         (0x1510U)
+#define RGX_TB_TRIGGER_HIGH_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_HIGH_TIMEOUT_SHIFT                                           (0U)
+#define RGX_TB_TRIGGER_HIGH_TIMEOUT_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRIGGER_MAX
+*/
+#define RGX_TB_TRIGGER_MAX                                                          (0x1518U)
+#define RGX_TB_TRIGGER_MAX_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_TRIGGER_MAX_TIMEOUT_SHIFT                                            (0U)
+#define RGX_TB_TRIGGER_MAX_TIMEOUT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_TB_RDATA_CORRUPT_ENABLE
+*/
+#define RGX_TB_RDATA_CORRUPT_ENABLE                                                 (0x1560U)
+#define RGX_TB_RDATA_CORRUPT_ENABLE_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_RDATA_CORRUPT_ENABLE_ENABLE_SHIFT                                    (0U)
+#define RGX_TB_RDATA_CORRUPT_ENABLE_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_TB_RDATA_CORRUPT_ENABLE_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_TB_RDATA_CORRUPT_MASK
+*/
+#define RGX_TB_RDATA_CORRUPT_MASK                                                   (0x1568U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_RDATA_CORRUPT_MASK_MMU_SHIFT                                         (31U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MMU_CLRMSK                                        (0x7FFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_MMU_EN                                            (0x80000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_UPS_SHIFT                                         (30U)
+#define RGX_TB_RDATA_CORRUPT_MASK_UPS_CLRMSK                                        (0xBFFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_UPS_EN                                            (0x40000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBM_SHIFT                                         (29U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBM_CLRMSK                                        (0xDFFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBM_EN                                            (0x20000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TUL_SHIFT                                         (28U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TUL_CLRMSK                                        (0xEFFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TUL_EN                                            (0x10000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHR_SHIFT                                         (27U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHR_CLRMSK                                        (0xF7FFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHR_EN                                            (0x08000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBA_SHIFT                                         (26U)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBA_CLRMSK                                        (0xFBFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_FBA_EN                                            (0x04000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VDM_SHIFT                                         (25U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VDM_CLRMSK                                        (0xFDFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_VDM_EN                                            (0x02000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_USC_L2_SHIFT                                      (24U)
+#define RGX_TB_RDATA_CORRUPT_MASK_USC_L2_CLRMSK                                     (0xFEFFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_USC_L2_EN                                         (0x01000000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDS_SHIFT                                         (23U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDS_CLRMSK                                        (0xFF7FFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDS_EN                                            (0x00800000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDSRW_SHIFT                                       (22U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDSRW_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PDSRW_EN                                          (0x00400000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TPF_SHIFT                                         (21U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TPF_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TPF_EN                                            (0x00200000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHF_SHIFT                                         (20U)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHF_CLRMSK                                        (0xFFEFFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_SHF_EN                                            (0x00100000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_AMC_SHIFT                                         (19U)
+#define RGX_TB_RDATA_CORRUPT_MASK_AMC_CLRMSK                                        (0xFFF7FFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_AMC_EN                                            (0x00080000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RAC_SHIFT                                         (18U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RAC_CLRMSK                                        (0xFFFBFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_RAC_EN                                            (0x00040000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_RTC_SHIFT                                     (17U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_RTC_CLRMSK                                    (0xFFFDFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_RTC_EN                                        (0x00020000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_ISP_SHIFT                                         (16U)
+#define RGX_TB_RDATA_CORRUPT_MASK_ISP_CLRMSK                                        (0xFFFEFFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_ISP_EN                                            (0x00010000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PPP_SHIFT                                         (15U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PPP_CLRMSK                                        (0xFFFF7FFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PPP_EN                                            (0x00008000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPF_SHIFT                                         (14U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPF_CLRMSK                                        (0xFFFFBFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPF_EN                                            (0x00004000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_SHIFT                                         (13U)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_CLRMSK                                        (0xFFFFDFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_VCE_EN                                            (0x00002000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PBE_SHIFT                                         (12U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PBE_CLRMSK                                        (0xFFFFEFFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_PBE_EN                                            (0x00001000U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TCU_SHIFT                                         (11U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TCU_CLRMSK                                        (0xFFFFF7FFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TCU_EN                                            (0x00000800U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MCU_SHIFT                                         (10U)
+#define RGX_TB_RDATA_CORRUPT_MASK_MCU_CLRMSK                                        (0xFFFFFBFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_MCU_EN                                            (0x00000400U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RPM_SHIFT                                         (9U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RPM_CLRMSK                                        (0xFFFFFDFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_RPM_EN                                            (0x00000200U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RTU_SHIFT                                         (8U)
+#define RGX_TB_RDATA_CORRUPT_MASK_RTU_CLRMSK                                        (0xFFFFFEFFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_RTU_EN                                            (0x00000100U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TILING_SHIFT                                      (7U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TILING_CLRMSK                                     (0xFFFFFF7FU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TILING_EN                                         (0x00000080U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_DMA_SHIFT                                    (6U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_DMA_CLRMSK                                   (0xFFFFFFBFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_DMA_EN                                       (0x00000040U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_SHIFT                                        (5U)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_META_EN                                           (0x00000020U)
+#define RGX_TB_RDATA_CORRUPT_MASK_CDM_SHIFT                                         (4U)
+#define RGX_TB_RDATA_CORRUPT_MASK_CDM_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_TB_RDATA_CORRUPT_MASK_CDM_EN                                            (0x00000010U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PM_SHIFT                                          (3U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PM_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_TB_RDATA_CORRUPT_MASK_PM_EN                                             (0x00000008U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TDM_SHIFT                                         (2U)
+#define RGX_TB_RDATA_CORRUPT_MASK_TDM_CLRMSK                                        (0xFFFFFFFBU)
+#define RGX_TB_RDATA_CORRUPT_MASK_TDM_EN                                            (0x00000004U)
+#define RGX_TB_RDATA_CORRUPT_MASK_DCE_SHIFT                                         (1U)
+#define RGX_TB_RDATA_CORRUPT_MASK_DCE_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_TB_RDATA_CORRUPT_MASK_DCE_EN                                            (0x00000002U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPP_SHIFT                                         (0U)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPP_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_TB_RDATA_CORRUPT_MASK_IPP_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_TB_RDATA_CORRUPT_FREQ
+*/
+#define RGX_TB_RDATA_CORRUPT_FREQ                                                   (0x1570U)
+#define RGX_TB_RDATA_CORRUPT_FREQ_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_RDATA_CORRUPT_FREQ_FREQ_SHIFT                                        (0U)
+#define RGX_TB_RDATA_CORRUPT_FREQ_FREQ_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE
+*/
+#define RGX_TB_TRUSTED_DEVICE                                                       (0x2000U)
+#define RGX_TB_TRUSTED_DEVICE_MASKFULL                                              (IMG_UINT64_C(0x000000000000001F))
+#define RGX_TB_TRUSTED_DEVICE_ALLOW_SECURE_READS_SHIFT                              (4U)
+#define RGX_TB_TRUSTED_DEVICE_ALLOW_SECURE_READS_CLRMSK                             (0xFFFFFFEFU)
+#define RGX_TB_TRUSTED_DEVICE_ALLOW_SECURE_READS_EN                                 (0x00000010U)
+#define RGX_TB_TRUSTED_DEVICE_HWCONFIG_SHIFT                                        (2U)
+#define RGX_TB_TRUSTED_DEVICE_HWCONFIG_CLRMSK                                       (0xFFFFFFF3U)
+#define RGX_TB_TRUSTED_DEVICE_OSID_DISABLE_SHIFT                                    (1U)
+#define RGX_TB_TRUSTED_DEVICE_OSID_DISABLE_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_TB_TRUSTED_DEVICE_OSID_DISABLE_EN                                       (0x00000002U)
+#define RGX_TB_TRUSTED_DEVICE_ENABLE_SHIFT                                          (0U)
+#define RGX_TB_TRUSTED_DEVICE_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_TB_TRUSTED_DEVICE_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_FAULT_STATUS
+*/
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS                                          (0x2008U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x01FFFFFFFFFF1771))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_AXPROT_SHIFT                             (56U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_AXPROT_CLRMSK                            (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_AXPROT_EN                                (IMG_UINT64_C(0x0100000000000000))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_ADDR_SHIFT                               (16U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_ADDR_CLRMSK                              (IMG_UINT64_C(0xFF0000000000FFFF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_RNW_SHIFT                                (12U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_RNW_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_RNW_EN                                   (IMG_UINT64_C(0x0000000000001000))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_OS_ID_SHIFT                              (8U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_OS_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFF8FF))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_BANK_SHIFT                               (4U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_BANK_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFF8F))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_VALID_SHIFT                              (0U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_VALID_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_STATUS_VALID_EN                                 (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR
+*/
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR                                           (0x2010U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_MASKFULL                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_PULSE_SHIFT                               (0U)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_PULSE_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_TB_TRUSTED_DEVICE_FAULT_CLEAR_PULSE_EN                                  (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0                                       (0x2018U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1                                       (0x2020U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2                                       (0x2028U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3                                       (0x2030U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4                                       (0x2038U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5                                       (0x2040U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6                                       (0x2048U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7                                       (0x2050U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MIN_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0                                       (0x2058U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1                                       (0x2060U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2                                       (0x2068U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3                                       (0x2070U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4                                       (0x2078U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5                                       (0x2080U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6                                       (0x2088U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7                                       (0x2090U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION0_MAX_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0                                       (0x2098U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1                                       (0x20A0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2                                       (0x20A8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3                                       (0x20B0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4                                       (0x20B8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5                                       (0x20C0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6                                       (0x20C8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7                                       (0x20D0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MIN_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register group: RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS, with 8 repeats
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS_REPEATCOUNT                            (8U)
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0                                       (0x20D8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS0_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1                                       (0x20E0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS1_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2                                       (0x20E8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS2_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3                                       (0x20F0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS3_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4                                       (0x20F8U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS4_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5                                       (0x2100U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS5_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6                                       (0x2108U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS6_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7
+*/
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7                                       (0x2110U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7_MASKFULL                              (IMG_UINT64_C(0x000000FFFFFFFFFF))
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7_ADDR_SHIFT                            (0U)
+#define RGX_TB_TRUSTED_DEVICE_REGION1_MAX_OS7_ADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000000))
+
+
+/*
+    Register RGX_TB_BW_LIMITER
+*/
+#define RGX_TB_BW_LIMITER                                                           (0x2118U)
+#define RGX_TB_BW_LIMITER_MASKFULL                                                  (IMG_UINT64_C(0x00000000007707FF))
+#define RGX_TB_BW_LIMITER_DROPN_EXT_SHIFT                                           (20U)
+#define RGX_TB_BW_LIMITER_DROPN_EXT_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFF8FFFFF))
+#define RGX_TB_BW_LIMITER_PERIOD_EXT_SHIFT                                          (16U)
+#define RGX_TB_BW_LIMITER_PERIOD_EXT_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFF8FFFF))
+#define RGX_TB_BW_LIMITER_DROPN_SHIFT                                               (6U)
+#define RGX_TB_BW_LIMITER_DROPN_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFF83F))
+#define RGX_TB_BW_LIMITER_PERIOD_SHIFT                                              (1U)
+#define RGX_TB_BW_LIMITER_PERIOD_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFC1))
+#define RGX_TB_BW_LIMITER_ENABLE_SHIFT                                              (0U)
+#define RGX_TB_BW_LIMITER_ENABLE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_BW_LIMITER_ENABLE_EN                                                 (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_TRUSTED_DEVICE_ACECONFIG
+*/
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG                                             (0x2120U)
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_MASKFULL                                    (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_OSID_SECURITY_SHIFT                         (1U)
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_OSID_SECURITY_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFE01))
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_ENABLE_SHIFT                                (0U)
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_ENABLE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_TRUSTED_DEVICE_ACECONFIG_ENABLE_EN                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_DRAM_CROSSBAR
+*/
+#define RGX_TB_DRAM_CROSSBAR                                                        (0x2128U)
+#define RGX_TB_DRAM_CROSSBAR_MASKFULL                                               (IMG_UINT64_C(0x0000000000003301))
+#define RGX_TB_DRAM_CROSSBAR_CHANNELS_SHIFT                                         (12U)
+#define RGX_TB_DRAM_CROSSBAR_CHANNELS_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_TB_DRAM_CROSSBAR_SEL_MODE_SHIFT                                         (8U)
+#define RGX_TB_DRAM_CROSSBAR_SEL_MODE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_TB_DRAM_CROSSBAR_ENABLE_SHIFT                                           (0U)
+#define RGX_TB_DRAM_CROSSBAR_ENABLE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_DRAM_CROSSBAR_ENABLE_EN                                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_LOCKUP
+*/
+#define RGX_TB_LOCKUP                                                               (0x2130U)
+#define RGX_TB_LOCKUP_MASKFULL                                                      (IMG_UINT64_C(0xFFFFFFFF00000001))
+#define RGX_TB_LOCKUP_CYCLES_BEFORE_SHIFT                                           (32U)
+#define RGX_TB_LOCKUP_CYCLES_BEFORE_CLRMSK                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_TB_LOCKUP_ENABLE_DETECT_SHIFT                                           (0U)
+#define RGX_TB_LOCKUP_ENABLE_DETECT_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_TB_LOCKUP_ENABLE_DETECT_EN                                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_TB_SOC_TIMER
+*/
+#define RGX_TB_SOC_TIMER                                                            (0x2140U)
+#define RGX_TB_SOC_TIMER_MASKFULL                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_TB_SOC_TIMER_COUNT_SHIFT                                                (0U)
+#define RGX_TB_SOC_TIMER_COUNT_CLRMSK                                               (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_TB_PROGRAMABLE_CLK_DIV
+*/
+#define RGX_TB_PROGRAMABLE_CLK_DIV                                                  (0x2150U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MASKFULL                                         (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MODE_SHIFT                                       (11U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MODE_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_MODE_EN                                          (0x00000800U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_PROFILE_SEL_SHIFT                                (9U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_PROFILE_SEL_CLRMSK                               (0xFFFFF9FFU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DIV_SHIFT                                        (5U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DIV_CLRMSK                                       (0xFFFFFE1FU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DELAY_SHIFT                                      (1U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_DELAY_CLRMSK                                     (0xFFFFFFE1U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_EVENT_SHIFT                                      (0U)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_EVENT_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_TB_PROGRAMABLE_CLK_DIV_EVENT_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPIO_FREQ_CTRL
+*/
+#define RGX_TB_GPIO_FREQ_CTRL                                                       (0x2160U)
+#define RGX_TB_GPIO_FREQ_CTRL_MASKFULL                                              (IMG_UINT64_C(0x000000000000001F))
+#define RGX_TB_GPIO_FREQ_CTRL_COUNT_SHIFT                                           (1U)
+#define RGX_TB_GPIO_FREQ_CTRL_COUNT_CLRMSK                                          (0xFFFFFFE1U)
+#define RGX_TB_GPIO_FREQ_CTRL_ENABLE_SHIFT                                          (0U)
+#define RGX_TB_GPIO_FREQ_CTRL_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_TB_GPIO_FREQ_CTRL_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPIO_MODE
+*/
+#define RGX_TB_GPIO_MODE                                                            (0x2170U)
+#define RGX_TB_GPIO_MODE_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_GPIO_MODE_PROTOCOL_SHIFT                                             (0U)
+#define RGX_TB_GPIO_MODE_PROTOCOL_CLRMSK                                            (0xFFFFFFFCU)
+
+
+/*
+    Register group: RGX_TB_SAFETY_STATUS, with 8 repeats
+*/
+#define RGX_TB_SAFETY_STATUS_REPEATCOUNT                                            (8U)
+/*
+    Register RGX_TB_SAFETY_STATUS0
+*/
+#define RGX_TB_SAFETY_STATUS0                                                       (0x6000U)
+#define RGX_TB_SAFETY_STATUS0_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS0_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS0_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS1
+*/
+#define RGX_TB_SAFETY_STATUS1                                                       (0x6008U)
+#define RGX_TB_SAFETY_STATUS1_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS1_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS1_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS2
+*/
+#define RGX_TB_SAFETY_STATUS2                                                       (0x6010U)
+#define RGX_TB_SAFETY_STATUS2_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS2_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS2_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS3
+*/
+#define RGX_TB_SAFETY_STATUS3                                                       (0x6018U)
+#define RGX_TB_SAFETY_STATUS3_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS3_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS3_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS4
+*/
+#define RGX_TB_SAFETY_STATUS4                                                       (0x6020U)
+#define RGX_TB_SAFETY_STATUS4_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS4_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS4_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS5
+*/
+#define RGX_TB_SAFETY_STATUS5                                                       (0x6028U)
+#define RGX_TB_SAFETY_STATUS5_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS5_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS5_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS6
+*/
+#define RGX_TB_SAFETY_STATUS6                                                       (0x6030U)
+#define RGX_TB_SAFETY_STATUS6_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS6_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS6_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_STATUS7
+*/
+#define RGX_TB_SAFETY_STATUS7                                                       (0x6038U)
+#define RGX_TB_SAFETY_STATUS7_MASKFULL                                              (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_STATUS7_SAFETY_IRQ_SHIFT                                      (0U)
+#define RGX_TB_SAFETY_STATUS7_SAFETY_IRQ_CLRMSK                                     (0xFFFFFFFCU)
+
+
+/*
+    Register group: RGX_TB_SAFETY_IRQ_ENABLE, with 8 repeats
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE_REPEATCOUNT                                        (8U)
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE0
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE0                                                   (0x6040U)
+#define RGX_TB_SAFETY_IRQ_ENABLE0_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE0_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE0_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE1
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE1                                                   (0x6048U)
+#define RGX_TB_SAFETY_IRQ_ENABLE1_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE1_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE1_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE2
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE2                                                   (0x6050U)
+#define RGX_TB_SAFETY_IRQ_ENABLE2_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE2_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE2_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE3
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE3                                                   (0x6058U)
+#define RGX_TB_SAFETY_IRQ_ENABLE3_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE3_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE3_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE4
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE4                                                   (0x6060U)
+#define RGX_TB_SAFETY_IRQ_ENABLE4_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE4_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE4_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE5
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE5                                                   (0x6068U)
+#define RGX_TB_SAFETY_IRQ_ENABLE5_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE5_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE5_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE6
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE6                                                   (0x6070U)
+#define RGX_TB_SAFETY_IRQ_ENABLE6_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE6_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE6_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_SAFETY_IRQ_ENABLE7
+*/
+#define RGX_TB_SAFETY_IRQ_ENABLE7                                                   (0x6078U)
+#define RGX_TB_SAFETY_IRQ_ENABLE7_MASKFULL                                          (IMG_UINT64_C(0x0000000000000003))
+#define RGX_TB_SAFETY_IRQ_ENABLE7_SAFETY_IRQ_EN_SHIFT                               (0U)
+#define RGX_TB_SAFETY_IRQ_ENABLE7_SAFETY_IRQ_EN_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_TB_FAULT_MASK
+*/
+#define RGX_TB_FAULT_MASK                                                           (0x6080U)
+#define RGX_TB_FAULT_MASK_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_TB_FAULT_MASK_MASK_FAULT_EVENT_SHIFT                                    (0U)
+#define RGX_TB_FAULT_MASK_MASK_FAULT_EVENT_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_TB_FAULT_MASK_MASK_FAULT_EVENT_EN                                       (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_GPU_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL_REPEATCOUNT                                    (8U)
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0                                               (0x60A0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL0_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1                                               (0x60A8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL1_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2                                               (0x60B0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL2_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3                                               (0x60B8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL3_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4                                               (0x60C0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL4_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5                                               (0x60C8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL5_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6                                               (0x60D0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL6_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_TB_GPU_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7                                               (0x60D8U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_COUNT_SHIFT                         (16U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_COUNT_CLRMSK                        (0xFF00FFFFU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_CHECK_SHIFT                         (1U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_CHECK_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_CHECK_EN                            (0x00000002U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_INSERT_SHIFT                        (0U)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_INSERT_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_TB_GPU_PARITY_ERROR_CTRL7_GPU_ERROR_INSERT_EN                           (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_SFTY_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL_REPEATCOUNT                                   (8U)
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0                                              (0x60E0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL0_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1                                              (0x60E8U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL1_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2                                              (0x60F0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL2_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3                                              (0x60F8U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL3_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4                                              (0x6100U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL4_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5                                              (0x6108U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL5_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6                                              (0x6110U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL6_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_TB_SFTY_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7                                              (0x6118U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_MASKFULL                                     (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_COUNT_SHIFT                            (16U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_COUNT_CLRMSK                           (0xFF00FFFFU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_CHECK_SHIFT                            (1U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_CHECK_CLRMSK                           (0xFFFFFFFDU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_CHECK_EN                               (0x00000002U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_INSERT_SHIFT                           (0U)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_INSERT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_TB_SFTY_PARITY_ERROR_CTRL7_ERROR_INSERT_EN                              (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_XPU_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL_REPEATCOUNT                                    (8U)
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0                                               (0x6120U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL0_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1                                               (0x6128U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL1_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2                                               (0x6130U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL2_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3                                               (0x6138U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL3_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4                                               (0x6140U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL4_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5                                               (0x6148U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL5_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6                                               (0x6150U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL6_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_XPU_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7                                               (0x6158U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_XPU_PARITY_ERROR_CTRL7_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register group: RGX_TB_MEM_PARITY_ERROR_CTRL, with 8 repeats
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL_REPEATCOUNT                                    (8U)
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL0
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0                                               (0x6160U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL0_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL1
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1                                               (0x6168U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL1_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL2
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2                                               (0x6170U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL2_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL3
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3                                               (0x6178U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL3_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL4
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4                                               (0x6180U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL4_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL5
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5                                               (0x6188U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL5_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL6
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6                                               (0x6190U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL6_ERROR_INSERT_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_TB_MEM_PARITY_ERROR_CTRL7
+*/
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7                                               (0x6198U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_MASKFULL                                      (IMG_UINT64_C(0x0000000000FF0003))
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_COUNT_SHIFT                             (16U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_COUNT_CLRMSK                            (0xFF00FFFFU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_CHECK_SHIFT                             (1U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_CHECK_CLRMSK                            (0xFFFFFFFDU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_CHECK_EN                                (0x00000002U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_INSERT_SHIFT                            (0U)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_INSERT_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_TB_MEM_PARITY_ERROR_CTRL7_ERROR_INSERT_EN                               (0x00000001U)
+
+
+#endif /* RGXTBDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxtbdefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/rgxpmdefs.h b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/rgxpmdefs.h
new file mode 100644
index 000000000000..5db7c2585356
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/hwdefs/volcanic/rgxpmdefs.h
@@ -0,0 +1,5019 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxpmdefs.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_pm.def: #15
+ */
+
+
+#ifndef RGXPMDEFS_H
+#define RGXPMDEFS_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXPMDEFS_REVISION 15
+
+/*
+The mini PB size on a per-RT basis
+*/
+typedef struct PM_DATA_MINI_PB_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_MINI_PB;
+
+/*
+
+*/
+#define PM_DATA_MINI_PB_SIZE_WOFF                         (0U)
+#define PM_DATA_MINI_PB_SIZE_SHIFT                        (0U)
+#define PM_DATA_MINI_PB_SIZE_CLRMSK                       (0xFFFFFC00U)
+#define PM_DATA_MINI_PB_SET_SIZE(_ft_,_x_)                ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_MINI_PB_SIZE_CLRMSK ) | (((_x_) & (0x000003ffU))  <<  (PM_DATA_MINI_PB_SIZE_SHIFT))))
+#define PM_DATA_MINI_PB_GET_SIZE(_ft_)                    (((_ft_).u32_0  >>  (PM_DATA_MINI_PB_SIZE_SHIFT)) & 0x000003ffU)
+
+
+/*
+The minimum PB size for the WDDM driver only. It is consistent with the OPENGL/OPENGLES. However, it is breaking down as two parts: the pagable memory and non pagable memory.
+*/
+typedef struct PM_DATA_WDDM_MINI_PB_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_WDDM_MINI_PB;
+
+/*
+
+*/
+#define PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_WOFF        (0U)
+#define PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_SHIFT       (10U)
+#define PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_CLRMSK      (0xFFF003FFU)
+#define PM_DATA_WDDM_MINI_PB_SET_NON_PAGABLE_SIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_CLRMSK ) | (((_x_) & (0x000003ffU))  <<  (PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_SHIFT))))
+#define PM_DATA_WDDM_MINI_PB_GET_NON_PAGABLE_SIZE(_ft_)   (((_ft_).u32_0  >>  (PM_DATA_WDDM_MINI_PB_NON_PAGABLE_SIZE_SHIFT)) & 0x000003ffU)
+/*
+
+*/
+#define PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_WOFF            (0U)
+#define PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_SHIFT           (0U)
+#define PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_CLRMSK          (0xFFFFFC00U)
+#define PM_DATA_WDDM_MINI_PB_SET_PAGABLE_SIZE(_ft_,_x_)   ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_CLRMSK ) | (((_x_) & (0x000003ffU))  <<  (PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_SHIFT))))
+#define PM_DATA_WDDM_MINI_PB_GET_PAGABLE_SIZE(_ft_)       (((_ft_).u32_0  >>  (PM_DATA_WDDM_MINI_PB_PAGABLE_SIZE_SHIFT)) & 0x000003ffU)
+
+
+/*
+the mini number of the reserve pages when only the local free list is used */
+#define PM_DATA_PM_RESERVE_PAGES_MIN_SIZE                 (0x00000007U)
+
+
+/*
+the mini number of the reserve pages when unified free list is present */
+#define PM_DATA_PM_RESERVE_PAGES_MIN_UNIFIED_SIZE         (0x0000000bU)
+
+
+/*
+This defines the format of entries in the FSTACK, UFSTACK and MMUSTACK
+*/
+typedef struct PM_DATA_FSTACK_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_FSTACK_ENTRY;
+
+/*
+Reserved for future use
+*/
+#define PM_DATA_FSTACK_ENTRY_RSV_WOFF                     (0U)
+#define PM_DATA_FSTACK_ENTRY_RSV_SHIFT                    (28U)
+#define PM_DATA_FSTACK_ENTRY_RSV_CLRMSK                   (0x0FFFFFFFU)
+#define PM_DATA_FSTACK_ENTRY_SET_RSV(_ft_,_x_)            ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_FSTACK_ENTRY_RSV_CLRMSK ) | (((_x_) & (0x0000000fU))  <<  (PM_DATA_FSTACK_ENTRY_RSV_SHIFT))))
+#define PM_DATA_FSTACK_ENTRY_GET_RSV(_ft_)                (((_ft_).u32_0  >>  (PM_DATA_FSTACK_ENTRY_RSV_SHIFT)) & 0x0000000fU)
+/*
+Address of 4 kB physical page
+*/
+#define PM_DATA_FSTACK_ENTRY_PPAGE_WOFF                   (0U)
+#define PM_DATA_FSTACK_ENTRY_PPAGE_SHIFT                  (0U)
+#define PM_DATA_FSTACK_ENTRY_PPAGE_CLRMSK                 (0xF0000000U)
+#define PM_DATA_FSTACK_ENTRY_SET_PPAGE(_ft_,_x_)          ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_FSTACK_ENTRY_PPAGE_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_FSTACK_ENTRY_PPAGE_SHIFT))))
+#define PM_DATA_FSTACK_ENTRY_GET_PPAGE(_ft_)              (((_ft_).u32_0  >>  (PM_DATA_FSTACK_ENTRY_PPAGE_SHIFT)) & 0x0fffffffU)
+
+
+/*
+This defines the format of an ALIST (Allocation List) entry
+*/
+typedef struct PM_DATA_ALIST_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+} PM_DATA_ALIST_ENTRY;
+
+/*
+Valid bit.  Indicates whether this ALIST entry is valid.
+*/
+#define PM_DATA_ALIST_ENTRY_VAL_WOFF                      (1U)
+#define PM_DATA_ALIST_ENTRY_VAL_SHIFT                     (31U)
+#define PM_DATA_ALIST_ENTRY_VAL_CLRMSK                    (0x7FFFFFFFU)
+#define PM_DATA_ALIST_ENTRY_SET_VAL(_ft_,_x_)             ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_VAL_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_ALIST_ENTRY_VAL_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_VAL(_ft_)                 (((_ft_).u32_1  >>  (PM_DATA_ALIST_ENTRY_VAL_SHIFT)) & 0x00000001U)
+/*
+The "data master" of the virtual page.  0=VCE, 1=TE, 2,3=reserved.
+*/
+#define PM_DATA_ALIST_ENTRY_DM_INDEX_WOFF                 (1U)
+#define PM_DATA_ALIST_ENTRY_DM_INDEX_SHIFT                (26U)
+#define PM_DATA_ALIST_ENTRY_DM_INDEX_CLRMSK               (0xF3FFFFFFU)
+#define PM_DATA_ALIST_ENTRY_SET_DM_INDEX(_ft_,_x_)        ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_DM_INDEX_CLRMSK ) | (((_x_) & (0x00000003U))  <<  (PM_DATA_ALIST_ENTRY_DM_INDEX_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_DM_INDEX(_ft_)            (((_ft_).u32_1  >>  (PM_DATA_ALIST_ENTRY_DM_INDEX_SHIFT)) & 0x00000003U)
+/*
+Render Target Array index.  Up to 2 k Render Target Arrays are supported.
+*/
+#define PM_DATA_ALIST_ENTRY_RTA_INDEX_WOFF                (1U)
+#define PM_DATA_ALIST_ENTRY_RTA_INDEX_SHIFT               (14U)
+#define PM_DATA_ALIST_ENTRY_RTA_INDEX_CLRMSK              (0xFE003FFFU)
+#define PM_DATA_ALIST_ENTRY_SET_RTA_INDEX(_ft_,_x_)       ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_RTA_INDEX_CLRMSK ) | (((_x_) & (0x000007ffU))  <<  (PM_DATA_ALIST_ENTRY_RTA_INDEX_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_RTA_INDEX(_ft_)           (((_ft_).u32_1  >>  (PM_DATA_ALIST_ENTRY_RTA_INDEX_SHIFT)) & 0x000007ffU)
+/*
+The virtual page number (16 kB virtual page).
+*/
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_WOFF             (0U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_WOFF             (1U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_SHIFT            (16U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_SHIFT            (0U)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_CLRMSK           (0x0000FFFFU)
+#define PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_CLRMSK           (0xFFFFFFF0U)
+#define PM_DATA_ALIST_ENTRY_SET_VRP_PPAGE(_ft_,_x_)       { ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_ALIST_ENTRY_VRP_PPAGE_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000ffff)))  <<  16))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_ALIST_ENTRY_VRP_PPAGE_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000f0000)))  >>  16))); }
+#define PM_DATA_ALIST_ENTRY_GET_VRP_PPAGE(_ft_)           (((_ft_).u32_0  >>  (16)) | ((IMG_UINT64)((_ft_).u32_1 & 0x0000000fU ) << (16)))
+/*
+The 16-bit macrotile mask.  Indicates which macrotile(s) are using this 16 kB page
+*/
+#define PM_DATA_ALIST_ENTRY_MTILE_MASK_WOFF               (0U)
+#define PM_DATA_ALIST_ENTRY_MTILE_MASK_SHIFT              (0U)
+#define PM_DATA_ALIST_ENTRY_MTILE_MASK_CLRMSK             (0xFFFF0000U)
+#define PM_DATA_ALIST_ENTRY_SET_MTILE_MASK(_ft_,_x_)      ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_ALIST_ENTRY_MTILE_MASK_CLRMSK ) | (((_x_) & (0x0000ffffU))  <<  (PM_DATA_ALIST_ENTRY_MTILE_MASK_SHIFT))))
+#define PM_DATA_ALIST_ENTRY_GET_MTILE_MASK(_ft_)          (((_ft_).u32_0  >>  (PM_DATA_ALIST_ENTRY_MTILE_MASK_SHIFT)) & 0x0000ffffU)
+
+
+/*
+This defines the format of entries in the MLIST
+*/
+typedef struct PM_DATA_MLIST_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_MLIST_ENTRY;
+
+/*
+Original source of the MMU page:
+0=Page was allocated from the FSTACK,
+1=Page was allocated from the UFSTACK.
+This bit is ignored when RGX_CR_PM_MMU_STACK_POLICY=1
+*/
+#define PM_DATA_MLIST_ENTRY_SRC_WOFF                      (0U)
+#define PM_DATA_MLIST_ENTRY_SRC_SHIFT                     (31U)
+#define PM_DATA_MLIST_ENTRY_SRC_CLRMSK                    (0x7FFFFFFFU)
+#define PM_DATA_MLIST_ENTRY_SET_SRC(_ft_,_x_)             ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_MLIST_ENTRY_SRC_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_MLIST_ENTRY_SRC_SHIFT))))
+#define PM_DATA_MLIST_ENTRY_GET_SRC(_ft_)                 (((_ft_).u32_0  >>  (PM_DATA_MLIST_ENTRY_SRC_SHIFT)) & 0x00000001U)
+/*
+Address of Physical Page allocated to MMU
+*/
+#define PM_DATA_MLIST_ENTRY_PPAGE_WOFF                    (0U)
+#define PM_DATA_MLIST_ENTRY_PPAGE_SHIFT                   (0U)
+#define PM_DATA_MLIST_ENTRY_PPAGE_CLRMSK                  (0xF0000000U)
+#define PM_DATA_MLIST_ENTRY_SET_PPAGE(_ft_,_x_)           ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_MLIST_ENTRY_PPAGE_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_MLIST_ENTRY_PPAGE_SHIFT))))
+#define PM_DATA_MLIST_ENTRY_GET_PPAGE(_ft_)               (((_ft_).u32_0  >>  (PM_DATA_MLIST_ENTRY_PPAGE_SHIFT)) & 0x0fffffffU)
+
+
+/*
+This defines the format of entries in the VFP Table
+*/
+typedef struct PM_DATA_VFP_TABLE_ENTRY_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+} PM_DATA_VFP_TABLE_ENTRY;
+
+/*
+Valid bit.  0=VFP is unmapped, 1=VFP is mapped.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_VALID_WOFF                (1U)
+#define PM_DATA_VFP_TABLE_ENTRY_VALID_SHIFT               (31U)
+#define PM_DATA_VFP_TABLE_ENTRY_VALID_CLRMSK              (0x7FFFFFFFU)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_VALID(_ft_,_x_)       ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VFP_TABLE_ENTRY_VALID_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VFP_TABLE_ENTRY_VALID_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_VALID(_ft_)           (((_ft_).u32_1  >>  (PM_DATA_VFP_TABLE_ENTRY_VALID_SHIFT)) & 0x00000001U)
+/*
+Address of MMU Page Table Entry.  8 Byte Granular.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_WOFF              (1U)
+#define PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_SHIFT             (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_CLRMSK            (0x80000000U)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_PTE_PTR(_ft_,_x_)     ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_PTE_PTR(_ft_)         (((_ft_).u32_1  >>  (PM_DATA_VFP_TABLE_ENTRY_PTE_PTR_SHIFT)) & 0x7fffffffU)
+/*
+Reserved for future use.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_RSV_WOFF                  (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_RSV_SHIFT                 (28U)
+#define PM_DATA_VFP_TABLE_ENTRY_RSV_CLRMSK                (0x0FFFFFFFU)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_RSV(_ft_,_x_)         ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VFP_TABLE_ENTRY_RSV_CLRMSK ) | (((_x_) & (0x0000000fU))  <<  (PM_DATA_VFP_TABLE_ENTRY_RSV_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_RSV(_ft_)             (((_ft_).u32_0  >>  (PM_DATA_VFP_TABLE_ENTRY_RSV_SHIFT)) & 0x0000000fU)
+/*
+Address of 1 kB Physical Page. 1 TB addressable.
+*/
+#define PM_DATA_VFP_TABLE_ENTRY_PPAGE_WOFF                (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_PPAGE_SHIFT               (0U)
+#define PM_DATA_VFP_TABLE_ENTRY_PPAGE_CLRMSK              (0xF0000000U)
+#define PM_DATA_VFP_TABLE_ENTRY_SET_PPAGE(_ft_,_x_)       ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VFP_TABLE_ENTRY_PPAGE_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VFP_TABLE_ENTRY_PPAGE_SHIFT))))
+#define PM_DATA_VFP_TABLE_ENTRY_GET_PPAGE(_ft_)           (((_ft_).u32_0  >>  (PM_DATA_VFP_TABLE_ENTRY_PPAGE_SHIFT)) & 0x0fffffffU)
+
+
+/*
+PerPipe Segment SIZE, it has a fixed mapping as follows:
+PIPE Number - Segment Size
+1             16G
+2             8G
+4             4G
+8             2G
+*/
+typedef struct PM_DATA_PERPIPE_SEGSIZE_TAG {
+     IMG_UINT32 u32_0;
+} PM_DATA_PERPIPE_SEGSIZE;
+
+/*
+PerSegment Size 2G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_SHIFT      (3U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_CLRMSK     (0xFFFFFFF7U)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE8_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE8_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE8_SEGSZIZE_SHIFT)) & 0x00000001U)
+/*
+PerSegment Size 4G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_SHIFT      (2U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_CLRMSK     (0xFFFFFFFBU)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE4_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE4_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE4_SEGSZIZE_SHIFT)) & 0x00000001U)
+/*
+PerSegment Size 8G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_SHIFT      (1U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_CLRMSK     (0xFFFFFFFDU)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE2_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE2_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE2_SEGSZIZE_SHIFT)) & 0x00000001U)
+/*
+PerSegment Size 16G
+*/
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_WOFF       (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_SHIFT      (0U)
+#define PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_CLRMSK     (0xFFFFFFFEU)
+#define PM_DATA_PERPIPE_SEGSIZE_SET_PIPE1_SEGSZIZE(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_SHIFT))))
+#define PM_DATA_PERPIPE_SEGSIZE_GET_PIPE1_SEGSZIZE(_ft_)  (((_ft_).u32_0  >>  (PM_DATA_PERPIPE_SEGSIZE_PIPE1_SEGSZIZE_SHIFT)) & 0x00000001U)
+
+
+/*
+PM Virtual Heap Buffer Offset. This buffer contains all the meta-data associated with each render target.
+size is 11904/8 = 1488 Bytes.
+
+Natively the buffer supports up to 8-VCEs and 8-TEs scaling without changing HW.
+
+In case relevant PIPE N is not present, the corresponding space is just reserved.
+*/
+typedef struct PM_DATA_VHEAP_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+     IMG_UINT32 u32_46;
+     IMG_UINT32 u32_47;
+     IMG_UINT32 u32_48;
+     IMG_UINT32 u32_49;
+     IMG_UINT32 u32_50;
+     IMG_UINT32 u32_51;
+     IMG_UINT32 u32_52;
+     IMG_UINT32 u32_53;
+     IMG_UINT32 u32_54;
+     IMG_UINT32 u32_55;
+     IMG_UINT32 u32_56;
+     IMG_UINT32 u32_57;
+     IMG_UINT32 u32_58;
+     IMG_UINT32 u32_59;
+     IMG_UINT32 u32_60;
+     IMG_UINT32 u32_61;
+     IMG_UINT32 u32_62;
+     IMG_UINT32 u32_63;
+     IMG_UINT32 u32_64;
+     IMG_UINT32 u32_65;
+     IMG_UINT32 u32_66;
+     IMG_UINT32 u32_67;
+     IMG_UINT32 u32_68;
+     IMG_UINT32 u32_69;
+     IMG_UINT32 u32_70;
+     IMG_UINT32 u32_71;
+     IMG_UINT32 u32_72;
+     IMG_UINT32 u32_73;
+     IMG_UINT32 u32_74;
+     IMG_UINT32 u32_75;
+     IMG_UINT32 u32_76;
+     IMG_UINT32 u32_77;
+     IMG_UINT32 u32_78;
+     IMG_UINT32 u32_79;
+     IMG_UINT32 u32_80;
+     IMG_UINT32 u32_81;
+     IMG_UINT32 u32_82;
+     IMG_UINT32 u32_83;
+     IMG_UINT32 u32_84;
+     IMG_UINT32 u32_85;
+     IMG_UINT32 u32_86;
+     IMG_UINT32 u32_87;
+     IMG_UINT32 u32_88;
+     IMG_UINT32 u32_89;
+     IMG_UINT32 u32_90;
+     IMG_UINT32 u32_91;
+     IMG_UINT32 u32_92;
+     IMG_UINT32 u32_93;
+     IMG_UINT32 u32_94;
+     IMG_UINT32 u32_95;
+     IMG_UINT32 u32_96;
+     IMG_UINT32 u32_97;
+     IMG_UINT32 u32_98;
+     IMG_UINT32 u32_99;
+     IMG_UINT32 u32_100;
+     IMG_UINT32 u32_101;
+     IMG_UINT32 u32_102;
+     IMG_UINT32 u32_103;
+     IMG_UINT32 u32_104;
+     IMG_UINT32 u32_105;
+     IMG_UINT32 u32_106;
+     IMG_UINT32 u32_107;
+     IMG_UINT32 u32_108;
+     IMG_UINT32 u32_109;
+     IMG_UINT32 u32_110;
+     IMG_UINT32 u32_111;
+     IMG_UINT32 u32_112;
+     IMG_UINT32 u32_113;
+     IMG_UINT32 u32_114;
+     IMG_UINT32 u32_115;
+     IMG_UINT32 u32_116;
+     IMG_UINT32 u32_117;
+     IMG_UINT32 u32_118;
+     IMG_UINT32 u32_119;
+     IMG_UINT32 u32_120;
+     IMG_UINT32 u32_121;
+     IMG_UINT32 u32_122;
+     IMG_UINT32 u32_123;
+     IMG_UINT32 u32_124;
+     IMG_UINT32 u32_125;
+     IMG_UINT32 u32_126;
+     IMG_UINT32 u32_127;
+     IMG_UINT32 u32_128;
+     IMG_UINT32 u32_129;
+     IMG_UINT32 u32_130;
+     IMG_UINT32 u32_131;
+     IMG_UINT32 u32_132;
+     IMG_UINT32 u32_133;
+     IMG_UINT32 u32_134;
+     IMG_UINT32 u32_135;
+     IMG_UINT32 u32_136;
+     IMG_UINT32 u32_137;
+     IMG_UINT32 u32_138;
+     IMG_UINT32 u32_139;
+     IMG_UINT32 u32_140;
+     IMG_UINT32 u32_141;
+     IMG_UINT32 u32_142;
+     IMG_UINT32 u32_143;
+     IMG_UINT32 u32_144;
+     IMG_UINT32 u32_145;
+     IMG_UINT32 u32_146;
+     IMG_UINT32 u32_147;
+     IMG_UINT32 u32_148;
+     IMG_UINT32 u32_149;
+     IMG_UINT32 u32_150;
+     IMG_UINT32 u32_151;
+     IMG_UINT32 u32_152;
+     IMG_UINT32 u32_153;
+     IMG_UINT32 u32_154;
+     IMG_UINT32 u32_155;
+     IMG_UINT32 u32_156;
+     IMG_UINT32 u32_157;
+     IMG_UINT32 u32_158;
+     IMG_UINT32 u32_159;
+     IMG_UINT32 u32_160;
+     IMG_UINT32 u32_161;
+     IMG_UINT32 u32_162;
+     IMG_UINT32 u32_163;
+     IMG_UINT32 u32_164;
+     IMG_UINT32 u32_165;
+     IMG_UINT32 u32_166;
+     IMG_UINT32 u32_167;
+     IMG_UINT32 u32_168;
+     IMG_UINT32 u32_169;
+     IMG_UINT32 u32_170;
+     IMG_UINT32 u32_171;
+     IMG_UINT32 u32_172;
+     IMG_UINT32 u32_173;
+     IMG_UINT32 u32_174;
+     IMG_UINT32 u32_175;
+     IMG_UINT32 u32_176;
+     IMG_UINT32 u32_177;
+     IMG_UINT32 u32_178;
+     IMG_UINT32 u32_179;
+     IMG_UINT32 u32_180;
+     IMG_UINT32 u32_181;
+     IMG_UINT32 u32_182;
+     IMG_UINT32 u32_183;
+     IMG_UINT32 u32_184;
+     IMG_UINT32 u32_185;
+     IMG_UINT32 u32_186;
+     IMG_UINT32 u32_187;
+     IMG_UINT32 u32_188;
+     IMG_UINT32 u32_189;
+     IMG_UINT32 u32_190;
+     IMG_UINT32 u32_191;
+     IMG_UINT32 u32_192;
+     IMG_UINT32 u32_193;
+     IMG_UINT32 u32_194;
+     IMG_UINT32 u32_195;
+     IMG_UINT32 u32_196;
+     IMG_UINT32 u32_197;
+     IMG_UINT32 u32_198;
+     IMG_UINT32 u32_199;
+     IMG_UINT32 u32_200;
+     IMG_UINT32 u32_201;
+     IMG_UINT32 u32_202;
+     IMG_UINT32 u32_203;
+     IMG_UINT32 u32_204;
+     IMG_UINT32 u32_205;
+     IMG_UINT32 u32_206;
+     IMG_UINT32 u32_207;
+     IMG_UINT32 u32_208;
+     IMG_UINT32 u32_209;
+     IMG_UINT32 u32_210;
+     IMG_UINT32 u32_211;
+     IMG_UINT32 u32_212;
+     IMG_UINT32 u32_213;
+     IMG_UINT32 u32_214;
+     IMG_UINT32 u32_215;
+     IMG_UINT32 u32_216;
+     IMG_UINT32 u32_217;
+     IMG_UINT32 u32_218;
+     IMG_UINT32 u32_219;
+     IMG_UINT32 u32_220;
+     IMG_UINT32 u32_221;
+     IMG_UINT32 u32_222;
+     IMG_UINT32 u32_223;
+     IMG_UINT32 u32_224;
+     IMG_UINT32 u32_225;
+     IMG_UINT32 u32_226;
+     IMG_UINT32 u32_227;
+     IMG_UINT32 u32_228;
+     IMG_UINT32 u32_229;
+     IMG_UINT32 u32_230;
+     IMG_UINT32 u32_231;
+     IMG_UINT32 u32_232;
+     IMG_UINT32 u32_233;
+     IMG_UINT32 u32_234;
+     IMG_UINT32 u32_235;
+     IMG_UINT32 u32_236;
+     IMG_UINT32 u32_237;
+     IMG_UINT32 u32_238;
+     IMG_UINT32 u32_239;
+     IMG_UINT32 u32_240;
+     IMG_UINT32 u32_241;
+     IMG_UINT32 u32_242;
+     IMG_UINT32 u32_243;
+     IMG_UINT32 u32_244;
+     IMG_UINT32 u32_245;
+     IMG_UINT32 u32_246;
+     IMG_UINT32 u32_247;
+     IMG_UINT32 u32_248;
+     IMG_UINT32 u32_249;
+     IMG_UINT32 u32_250;
+     IMG_UINT32 u32_251;
+     IMG_UINT32 u32_252;
+     IMG_UINT32 u32_253;
+     IMG_UINT32 u32_254;
+     IMG_UINT32 u32_255;
+     IMG_UINT32 u32_256;
+     IMG_UINT32 u32_257;
+     IMG_UINT32 u32_258;
+     IMG_UINT32 u32_259;
+     IMG_UINT32 u32_260;
+     IMG_UINT32 u32_261;
+     IMG_UINT32 u32_262;
+     IMG_UINT32 u32_263;
+     IMG_UINT32 u32_264;
+     IMG_UINT32 u32_265;
+     IMG_UINT32 u32_266;
+     IMG_UINT32 u32_267;
+     IMG_UINT32 u32_268;
+     IMG_UINT32 u32_269;
+     IMG_UINT32 u32_270;
+     IMG_UINT32 u32_271;
+     IMG_UINT32 u32_272;
+     IMG_UINT32 u32_273;
+     IMG_UINT32 u32_274;
+     IMG_UINT32 u32_275;
+     IMG_UINT32 u32_276;
+     IMG_UINT32 u32_277;
+     IMG_UINT32 u32_278;
+     IMG_UINT32 u32_279;
+     IMG_UINT32 u32_280;
+     IMG_UINT32 u32_281;
+     IMG_UINT32 u32_282;
+     IMG_UINT32 u32_283;
+     IMG_UINT32 u32_284;
+     IMG_UINT32 u32_285;
+     IMG_UINT32 u32_286;
+     IMG_UINT32 u32_287;
+     IMG_UINT32 u32_288;
+     IMG_UINT32 u32_289;
+     IMG_UINT32 u32_290;
+     IMG_UINT32 u32_291;
+     IMG_UINT32 u32_292;
+     IMG_UINT32 u32_293;
+     IMG_UINT32 u32_294;
+     IMG_UINT32 u32_295;
+     IMG_UINT32 u32_296;
+     IMG_UINT32 u32_297;
+     IMG_UINT32 u32_298;
+     IMG_UINT32 u32_299;
+     IMG_UINT32 u32_300;
+     IMG_UINT32 u32_301;
+     IMG_UINT32 u32_302;
+     IMG_UINT32 u32_303;
+     IMG_UINT32 u32_304;
+     IMG_UINT32 u32_305;
+     IMG_UINT32 u32_306;
+     IMG_UINT32 u32_307;
+     IMG_UINT32 u32_308;
+     IMG_UINT32 u32_309;
+     IMG_UINT32 u32_310;
+     IMG_UINT32 u32_311;
+     IMG_UINT32 u32_312;
+     IMG_UINT32 u32_313;
+     IMG_UINT32 u32_314;
+     IMG_UINT32 u32_315;
+     IMG_UINT32 u32_316;
+     IMG_UINT32 u32_317;
+     IMG_UINT32 u32_318;
+     IMG_UINT32 u32_319;
+     IMG_UINT32 u32_320;
+     IMG_UINT32 u32_321;
+     IMG_UINT32 u32_322;
+     IMG_UINT32 u32_323;
+     IMG_UINT32 u32_324;
+     IMG_UINT32 u32_325;
+     IMG_UINT32 u32_326;
+     IMG_UINT32 u32_327;
+     IMG_UINT32 u32_328;
+     IMG_UINT32 u32_329;
+     IMG_UINT32 u32_330;
+     IMG_UINT32 u32_331;
+     IMG_UINT32 u32_332;
+     IMG_UINT32 u32_333;
+     IMG_UINT32 u32_334;
+     IMG_UINT32 u32_335;
+     IMG_UINT32 u32_336;
+     IMG_UINT32 u32_337;
+     IMG_UINT32 u32_338;
+     IMG_UINT32 u32_339;
+     IMG_UINT32 u32_340;
+     IMG_UINT32 u32_341;
+     IMG_UINT32 u32_342;
+     IMG_UINT32 u32_343;
+     IMG_UINT32 u32_344;
+     IMG_UINT32 u32_345;
+     IMG_UINT32 u32_346;
+     IMG_UINT32 u32_347;
+     IMG_UINT32 u32_348;
+     IMG_UINT32 u32_349;
+     IMG_UINT32 u32_350;
+     IMG_UINT32 u32_351;
+     IMG_UINT32 u32_352;
+     IMG_UINT32 u32_353;
+     IMG_UINT32 u32_354;
+     IMG_UINT32 u32_355;
+     IMG_UINT32 u32_356;
+     IMG_UINT32 u32_357;
+     IMG_UINT32 u32_358;
+     IMG_UINT32 u32_359;
+     IMG_UINT32 u32_360;
+     IMG_UINT32 u32_361;
+     IMG_UINT32 u32_362;
+     IMG_UINT32 u32_363;
+     IMG_UINT32 u32_364;
+     IMG_UINT32 u32_365;
+     IMG_UINT32 u32_366;
+     IMG_UINT32 u32_367;
+     IMG_UINT32 u32_368;
+     IMG_UINT32 u32_369;
+     IMG_UINT32 u32_370;
+     IMG_UINT32 u32_371;
+} PM_DATA_VHEAP_BUFFER;
+
+/*
+TE7 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_WOFF   (371U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_371 = (((_ft_).u32_371 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_371  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE7 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_WOFF (370U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_WOFF (371U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_370 = (((_ft_).u32_370 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_371 = (((_ft_).u32_371 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_370  >>  (20)) | ((IMG_UINT64)((_ft_).u32_371 & 0x0000ffffU ) << (12)))
+/*
+TE7 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_WOFF (369U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_WOFF (370U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_369 = (((_ft_).u32_369 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_370 = (((_ft_).u32_370 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_369  >>  (24)) | ((IMG_UINT64)((_ft_).u32_370 & 0x000fffffU ) << (8)))
+/*
+TE7 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_WOFF (368U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_WOFF (369U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_368 = (((_ft_).u32_368 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_369 = (((_ft_).u32_369 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_368  >>  (28)) | ((IMG_UINT64)((_ft_).u32_369 & 0x00ffffffU ) << (4)))
+/*
+TE7 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_WOFF    (368U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_368 = (((_ft_).u32_368 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_368  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE7 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_WOFF      (364U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_364 = (((_ft_).u32_364 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_364  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE6 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_WOFF   (363U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_363 = (((_ft_).u32_363 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_363  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE6 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_WOFF (362U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_WOFF (363U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_362 = (((_ft_).u32_362 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_363 = (((_ft_).u32_363 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_362  >>  (20)) | ((IMG_UINT64)((_ft_).u32_363 & 0x0000ffffU ) << (12)))
+/*
+TE6 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_WOFF (361U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_WOFF (362U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_361 = (((_ft_).u32_361 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_362 = (((_ft_).u32_362 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_361  >>  (24)) | ((IMG_UINT64)((_ft_).u32_362 & 0x000fffffU ) << (8)))
+/*
+TE6 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_WOFF (360U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_WOFF (361U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_360 = (((_ft_).u32_360 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_361 = (((_ft_).u32_361 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_360  >>  (28)) | ((IMG_UINT64)((_ft_).u32_361 & 0x00ffffffU ) << (4)))
+/*
+TE6 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_WOFF    (360U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_360 = (((_ft_).u32_360 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_360  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE6 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_WOFF      (356U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_356 = (((_ft_).u32_356 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_356  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE5 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_WOFF   (355U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_355 = (((_ft_).u32_355 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_355  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE5 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_WOFF (354U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_WOFF (355U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_354 = (((_ft_).u32_354 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_355 = (((_ft_).u32_355 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_354  >>  (20)) | ((IMG_UINT64)((_ft_).u32_355 & 0x0000ffffU ) << (12)))
+/*
+TE5 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_WOFF (353U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_WOFF (354U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_353 = (((_ft_).u32_353 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_354 = (((_ft_).u32_354 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_353  >>  (24)) | ((IMG_UINT64)((_ft_).u32_354 & 0x000fffffU ) << (8)))
+/*
+TE5 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_WOFF (352U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_WOFF (353U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_352 = (((_ft_).u32_352 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_353 = (((_ft_).u32_353 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_352  >>  (28)) | ((IMG_UINT64)((_ft_).u32_353 & 0x00ffffffU ) << (4)))
+/*
+TE5 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_WOFF    (352U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_352 = (((_ft_).u32_352 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_352  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE5 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_WOFF      (348U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_348 = (((_ft_).u32_348 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_348  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE4 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_WOFF   (347U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_347 = (((_ft_).u32_347 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_347  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE4 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_WOFF (346U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_WOFF (347U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_346 = (((_ft_).u32_346 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_347 = (((_ft_).u32_347 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_346  >>  (20)) | ((IMG_UINT64)((_ft_).u32_347 & 0x0000ffffU ) << (12)))
+/*
+TE4 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_WOFF (345U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_WOFF (346U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_345 = (((_ft_).u32_345 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_346 = (((_ft_).u32_346 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_345  >>  (24)) | ((IMG_UINT64)((_ft_).u32_346 & 0x000fffffU ) << (8)))
+/*
+TE4 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_WOFF (344U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_WOFF (345U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_344 = (((_ft_).u32_344 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_345 = (((_ft_).u32_345 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_344  >>  (28)) | ((IMG_UINT64)((_ft_).u32_345 & 0x00ffffffU ) << (4)))
+/*
+TE4 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_WOFF    (344U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_344 = (((_ft_).u32_344 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_344  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE4 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_WOFF      (340U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_340 = (((_ft_).u32_340 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_340  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE3 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_WOFF   (339U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_339 = (((_ft_).u32_339 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_339  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE3 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_WOFF (338U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_WOFF (339U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_338 = (((_ft_).u32_338 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_339 = (((_ft_).u32_339 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_338  >>  (20)) | ((IMG_UINT64)((_ft_).u32_339 & 0x0000ffffU ) << (12)))
+/*
+TE3 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_WOFF (337U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_WOFF (338U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_337 = (((_ft_).u32_337 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_338 = (((_ft_).u32_338 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_337  >>  (24)) | ((IMG_UINT64)((_ft_).u32_338 & 0x000fffffU ) << (8)))
+/*
+TE3 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_WOFF (336U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_WOFF (337U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_336 = (((_ft_).u32_336 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_337 = (((_ft_).u32_337 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_336  >>  (28)) | ((IMG_UINT64)((_ft_).u32_337 & 0x00ffffffU ) << (4)))
+/*
+TE3 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_WOFF    (336U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_336 = (((_ft_).u32_336 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_336  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE3 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_WOFF      (332U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_332 = (((_ft_).u32_332 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_332  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE2 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_WOFF   (331U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_331 = (((_ft_).u32_331 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_331  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE2 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_WOFF (330U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_WOFF (331U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_330 = (((_ft_).u32_330 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_331 = (((_ft_).u32_331 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_330  >>  (20)) | ((IMG_UINT64)((_ft_).u32_331 & 0x0000ffffU ) << (12)))
+/*
+TE2 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_WOFF (329U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_WOFF (330U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_329 = (((_ft_).u32_329 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_330 = (((_ft_).u32_330 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_329  >>  (24)) | ((IMG_UINT64)((_ft_).u32_330 & 0x000fffffU ) << (8)))
+/*
+TE2 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_WOFF (328U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_WOFF (329U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_328 = (((_ft_).u32_328 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_329 = (((_ft_).u32_329 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_328  >>  (28)) | ((IMG_UINT64)((_ft_).u32_329 & 0x00ffffffU ) << (4)))
+/*
+TE2 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_WOFF    (328U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_328 = (((_ft_).u32_328 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_328  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE2 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_WOFF      (324U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_324 = (((_ft_).u32_324 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_324  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE1 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_WOFF   (323U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_323 = (((_ft_).u32_323 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_323  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE1 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_WOFF (322U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_WOFF (323U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_322 = (((_ft_).u32_322 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_323 = (((_ft_).u32_323 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_322  >>  (20)) | ((IMG_UINT64)((_ft_).u32_323 & 0x0000ffffU ) << (12)))
+/*
+TE1 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_WOFF (321U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_WOFF (322U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_321 = (((_ft_).u32_321 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_322 = (((_ft_).u32_322 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_321  >>  (24)) | ((IMG_UINT64)((_ft_).u32_322 & 0x000fffffU ) << (8)))
+/*
+TE1 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_WOFF (320U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_WOFF (321U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_320 = (((_ft_).u32_320 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_321 = (((_ft_).u32_321 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_320  >>  (28)) | ((IMG_UINT64)((_ft_).u32_321 & 0x00ffffffU ) << (4)))
+/*
+TE1 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_WOFF    (320U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_320 = (((_ft_).u32_320 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_320  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE1 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_WOFF      (316U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_316 = (((_ft_).u32_316 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_316  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+TE0 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_WOFF   (315U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_SHIFT  (16U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_315 = (((_ft_).u32_315 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_315  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+TE0 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_WOFF (314U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_WOFF (315U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_314 = (((_ft_).u32_314 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_315 = (((_ft_).u32_315 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_314  >>  (20)) | ((IMG_UINT64)((_ft_).u32_315 & 0x0000ffffU ) << (12)))
+/*
+TE0 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_WOFF (313U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_WOFF (314U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_313 = (((_ft_).u32_313 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_314 = (((_ft_).u32_314 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_313  >>  (24)) | ((IMG_UINT64)((_ft_).u32_314 & 0x000fffffU ) << (8)))
+/*
+TE0 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_WOFF (312U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_WOFF (313U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_312 = (((_ft_).u32_312 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_313 = (((_ft_).u32_313 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_312  >>  (28)) | ((IMG_UINT64)((_ft_).u32_313 & 0x00ffffffU ) << (4)))
+/*
+TE0 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_WOFF    (312U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_SHIFT   (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_CLRMSK  (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_312 = (((_ft_).u32_312 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_312  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+TE0 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_WOFF      (308U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_SHIFT     (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_CLRMSK    (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_308 = (((_ft_).u32_308 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_308  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_WOFF  (307U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_307 = (((_ft_).u32_307 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_307  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_WOFF (306U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_WOFF (307U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_306 = (((_ft_).u32_306 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_307 = (((_ft_).u32_307 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_306  >>  (20)) | ((IMG_UINT64)((_ft_).u32_307 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_WOFF (305U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_WOFF (306U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_305 = (((_ft_).u32_305 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_306 = (((_ft_).u32_306 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_305  >>  (24)) | ((IMG_UINT64)((_ft_).u32_306 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_WOFF (304U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_WOFF (305U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_304 = (((_ft_).u32_304 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_305 = (((_ft_).u32_305 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_304  >>  (28)) | ((IMG_UINT64)((_ft_).u32_305 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_WOFF   (304U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_304 = (((_ft_).u32_304 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_304  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_WOFF     (300U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_300 = (((_ft_).u32_300 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_300  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_WOFF  (299U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_299 = (((_ft_).u32_299 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_299  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_WOFF (298U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_WOFF (299U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_298 = (((_ft_).u32_298 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_299 = (((_ft_).u32_299 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_298  >>  (20)) | ((IMG_UINT64)((_ft_).u32_299 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_WOFF (297U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_WOFF (298U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_297 = (((_ft_).u32_297 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_298 = (((_ft_).u32_298 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_297  >>  (24)) | ((IMG_UINT64)((_ft_).u32_298 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_WOFF (296U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_WOFF (297U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_296 = (((_ft_).u32_296 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_297 = (((_ft_).u32_297 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_296  >>  (28)) | ((IMG_UINT64)((_ft_).u32_297 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_WOFF   (296U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_296 = (((_ft_).u32_296 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_296  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_WOFF     (292U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_292 = (((_ft_).u32_292 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_292  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_WOFF  (291U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_291 = (((_ft_).u32_291 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_291  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_WOFF (290U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_WOFF (291U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_290 = (((_ft_).u32_290 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_291 = (((_ft_).u32_291 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_290  >>  (20)) | ((IMG_UINT64)((_ft_).u32_291 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_WOFF (289U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_WOFF (290U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_289 = (((_ft_).u32_289 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_290 = (((_ft_).u32_290 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_289  >>  (24)) | ((IMG_UINT64)((_ft_).u32_290 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_WOFF (288U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_WOFF (289U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_288 = (((_ft_).u32_288 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_289 = (((_ft_).u32_289 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_288  >>  (28)) | ((IMG_UINT64)((_ft_).u32_289 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_WOFF   (288U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_288 = (((_ft_).u32_288 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_288  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_WOFF     (284U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_284 = (((_ft_).u32_284 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_284  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE7 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_WOFF  (283U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_283 = (((_ft_).u32_283 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_283  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE7 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_WOFF (282U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_WOFF (283U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_282 = (((_ft_).u32_282 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_283 = (((_ft_).u32_283 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_282  >>  (20)) | ((IMG_UINT64)((_ft_).u32_283 & 0x0000ffffU ) << (12)))
+/*
+VCE7 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_WOFF (281U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_WOFF (282U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_281 = (((_ft_).u32_281 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_282 = (((_ft_).u32_282 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_281  >>  (24)) | ((IMG_UINT64)((_ft_).u32_282 & 0x000fffffU ) << (8)))
+/*
+VCE7 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_WOFF (280U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_WOFF (281U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_280 = (((_ft_).u32_280 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_281 = (((_ft_).u32_281 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_280  >>  (28)) | ((IMG_UINT64)((_ft_).u32_281 & 0x00ffffffU ) << (4)))
+/*
+VCE7 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_WOFF   (280U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_280 = (((_ft_).u32_280 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_280  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE7 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_WOFF     (276U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_276 = (((_ft_).u32_276 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_276  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE6 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_WOFF  (259U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_259 = (((_ft_).u32_259 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_259  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE6 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_WOFF (258U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_WOFF (259U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_258 = (((_ft_).u32_258 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_259 = (((_ft_).u32_259 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_258  >>  (20)) | ((IMG_UINT64)((_ft_).u32_259 & 0x0000ffffU ) << (12)))
+/*
+VCE6 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_WOFF (257U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_WOFF (258U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_257 = (((_ft_).u32_257 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_258 = (((_ft_).u32_258 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_257  >>  (24)) | ((IMG_UINT64)((_ft_).u32_258 & 0x000fffffU ) << (8)))
+/*
+VCE6 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_WOFF (256U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_WOFF (257U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_256 = (((_ft_).u32_256 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_257 = (((_ft_).u32_257 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_256  >>  (28)) | ((IMG_UINT64)((_ft_).u32_257 & 0x00ffffffU ) << (4)))
+/*
+VCE6 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_WOFF   (256U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_256 = (((_ft_).u32_256 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_256  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE6 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_WOFF     (252U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_252 = (((_ft_).u32_252 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_252  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE6 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_WOFF  (251U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_251 = (((_ft_).u32_251 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_251  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE6 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_WOFF (250U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_WOFF (251U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_250 = (((_ft_).u32_250 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_251 = (((_ft_).u32_251 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_250  >>  (20)) | ((IMG_UINT64)((_ft_).u32_251 & 0x0000ffffU ) << (12)))
+/*
+VCE6 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_WOFF (249U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_WOFF (250U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_249 = (((_ft_).u32_249 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_250 = (((_ft_).u32_250 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_249  >>  (24)) | ((IMG_UINT64)((_ft_).u32_250 & 0x000fffffU ) << (8)))
+/*
+VCE6 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_WOFF (248U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_WOFF (249U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_248 = (((_ft_).u32_248 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_249 = (((_ft_).u32_249 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_248  >>  (28)) | ((IMG_UINT64)((_ft_).u32_249 & 0x00ffffffU ) << (4)))
+/*
+VCE6 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_WOFF   (248U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_248 = (((_ft_).u32_248 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_248  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE6 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_WOFF     (244U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_244 = (((_ft_).u32_244 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_244  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_WOFF  (243U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_243 = (((_ft_).u32_243 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_243  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_WOFF (242U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_WOFF (243U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_242 = (((_ft_).u32_242 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_243 = (((_ft_).u32_243 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_242  >>  (20)) | ((IMG_UINT64)((_ft_).u32_243 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_WOFF (241U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_WOFF (242U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_241 = (((_ft_).u32_241 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_242 = (((_ft_).u32_242 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_241  >>  (24)) | ((IMG_UINT64)((_ft_).u32_242 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_WOFF (240U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_WOFF (241U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_240 = (((_ft_).u32_240 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_241 = (((_ft_).u32_241 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_240  >>  (28)) | ((IMG_UINT64)((_ft_).u32_241 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_WOFF   (240U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_240 = (((_ft_).u32_240 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_240  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_WOFF     (236U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_236 = (((_ft_).u32_236 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_236  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_WOFF  (235U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_235 = (((_ft_).u32_235 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_235  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_WOFF (234U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_WOFF (235U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_234 = (((_ft_).u32_234 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_235 = (((_ft_).u32_235 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_234  >>  (20)) | ((IMG_UINT64)((_ft_).u32_235 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_WOFF (233U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_WOFF (234U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_233 = (((_ft_).u32_233 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_234 = (((_ft_).u32_234 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_233  >>  (24)) | ((IMG_UINT64)((_ft_).u32_234 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_WOFF (232U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_WOFF (233U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_232 = (((_ft_).u32_232 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_233 = (((_ft_).u32_233 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_232  >>  (28)) | ((IMG_UINT64)((_ft_).u32_233 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_WOFF   (232U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_232 = (((_ft_).u32_232 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_232  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_WOFF     (228U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_228 = (((_ft_).u32_228 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_228  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_WOFF  (227U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_227 = (((_ft_).u32_227 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_227  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_WOFF (226U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_WOFF (227U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_226 = (((_ft_).u32_226 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_227 = (((_ft_).u32_227 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_226  >>  (20)) | ((IMG_UINT64)((_ft_).u32_227 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_WOFF (225U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_WOFF (226U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_225 = (((_ft_).u32_225 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_226 = (((_ft_).u32_226 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_225  >>  (24)) | ((IMG_UINT64)((_ft_).u32_226 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_WOFF (224U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_WOFF (225U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_224 = (((_ft_).u32_224 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_225 = (((_ft_).u32_225 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_224  >>  (28)) | ((IMG_UINT64)((_ft_).u32_225 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_WOFF   (224U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_224 = (((_ft_).u32_224 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_224  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_WOFF     (220U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_220 = (((_ft_).u32_220 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_220  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE5 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_WOFF  (219U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_219 = (((_ft_).u32_219 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_219  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE5 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_WOFF (218U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_WOFF (219U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_218 = (((_ft_).u32_218 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_219 = (((_ft_).u32_219 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_218  >>  (20)) | ((IMG_UINT64)((_ft_).u32_219 & 0x0000ffffU ) << (12)))
+/*
+VCE5 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_WOFF (217U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_WOFF (218U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_217 = (((_ft_).u32_217 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_218 = (((_ft_).u32_218 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_217  >>  (24)) | ((IMG_UINT64)((_ft_).u32_218 & 0x000fffffU ) << (8)))
+/*
+VCE5 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_WOFF (216U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_WOFF (217U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_216 = (((_ft_).u32_216 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_217 = (((_ft_).u32_217 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_216  >>  (28)) | ((IMG_UINT64)((_ft_).u32_217 & 0x00ffffffU ) << (4)))
+/*
+VCE5 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_WOFF   (216U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_216 = (((_ft_).u32_216 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_216  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE5 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_WOFF     (212U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_212 = (((_ft_).u32_212 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_212  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_WOFF  (211U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_211 = (((_ft_).u32_211 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_211  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_WOFF (210U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_WOFF (211U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_210 = (((_ft_).u32_210 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_211 = (((_ft_).u32_211 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_210  >>  (20)) | ((IMG_UINT64)((_ft_).u32_211 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_WOFF (209U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_WOFF (210U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_209 = (((_ft_).u32_209 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_210 = (((_ft_).u32_210 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_209  >>  (24)) | ((IMG_UINT64)((_ft_).u32_210 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_WOFF (208U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_WOFF (209U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_208 = (((_ft_).u32_208 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_209 = (((_ft_).u32_209 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_208  >>  (28)) | ((IMG_UINT64)((_ft_).u32_209 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_WOFF   (208U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_208 = (((_ft_).u32_208 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_208  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_WOFF     (204U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_204 = (((_ft_).u32_204 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_204  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_WOFF  (203U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_203 = (((_ft_).u32_203 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_203  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_WOFF (202U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_WOFF (203U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_202 = (((_ft_).u32_202 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_203 = (((_ft_).u32_203 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_202  >>  (20)) | ((IMG_UINT64)((_ft_).u32_203 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_WOFF (201U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_WOFF (202U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_201 = (((_ft_).u32_201 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_202 = (((_ft_).u32_202 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_201  >>  (24)) | ((IMG_UINT64)((_ft_).u32_202 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_WOFF (200U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_WOFF (201U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_200 = (((_ft_).u32_200 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_201 = (((_ft_).u32_201 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_200  >>  (28)) | ((IMG_UINT64)((_ft_).u32_201 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_WOFF   (200U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_200 = (((_ft_).u32_200 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_200  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_WOFF     (196U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_196 = (((_ft_).u32_196 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_196  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_WOFF  (195U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_195 = (((_ft_).u32_195 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_195  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_WOFF (194U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_WOFF (195U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_194 = (((_ft_).u32_194 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_195 = (((_ft_).u32_195 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_194  >>  (20)) | ((IMG_UINT64)((_ft_).u32_195 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_WOFF (193U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_WOFF (194U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_193 = (((_ft_).u32_193 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_194 = (((_ft_).u32_194 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_193  >>  (24)) | ((IMG_UINT64)((_ft_).u32_194 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_WOFF (192U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_WOFF (193U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_192 = (((_ft_).u32_192 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_193 = (((_ft_).u32_193 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_192  >>  (28)) | ((IMG_UINT64)((_ft_).u32_193 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_WOFF   (192U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_192 = (((_ft_).u32_192 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_192  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_WOFF     (188U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_188 = (((_ft_).u32_188 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_188  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE4 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_WOFF  (187U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_187 = (((_ft_).u32_187 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_187  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE4 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_WOFF (186U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_WOFF (187U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_186 = (((_ft_).u32_186 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_187 = (((_ft_).u32_187 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_186  >>  (20)) | ((IMG_UINT64)((_ft_).u32_187 & 0x0000ffffU ) << (12)))
+/*
+VCE4 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_WOFF (185U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_WOFF (186U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_185 = (((_ft_).u32_185 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_186 = (((_ft_).u32_186 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_185  >>  (24)) | ((IMG_UINT64)((_ft_).u32_186 & 0x000fffffU ) << (8)))
+/*
+VCE4 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_WOFF (184U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_WOFF (185U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_184 = (((_ft_).u32_184 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_185 = (((_ft_).u32_185 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_184  >>  (28)) | ((IMG_UINT64)((_ft_).u32_185 & 0x00ffffffU ) << (4)))
+/*
+VCE4 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_WOFF   (184U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_184 = (((_ft_).u32_184 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_184  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE4 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_WOFF     (180U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_180 = (((_ft_).u32_180 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_180  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_WOFF  (179U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_179 = (((_ft_).u32_179 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_179  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_WOFF (178U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_WOFF (179U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_178 = (((_ft_).u32_178 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_179 = (((_ft_).u32_179 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_178  >>  (20)) | ((IMG_UINT64)((_ft_).u32_179 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_WOFF (177U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_WOFF (178U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_177 = (((_ft_).u32_177 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_178 = (((_ft_).u32_178 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_177  >>  (24)) | ((IMG_UINT64)((_ft_).u32_178 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_WOFF (176U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_WOFF (177U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_176 = (((_ft_).u32_176 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_177 = (((_ft_).u32_177 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_176  >>  (28)) | ((IMG_UINT64)((_ft_).u32_177 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_WOFF   (176U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_176 = (((_ft_).u32_176 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_176  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_WOFF     (172U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_172 = (((_ft_).u32_172 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_172  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_WOFF  (171U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_171 = (((_ft_).u32_171 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_171  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_WOFF (170U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_WOFF (171U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_170 = (((_ft_).u32_170 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_171 = (((_ft_).u32_171 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_170  >>  (20)) | ((IMG_UINT64)((_ft_).u32_171 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_WOFF (169U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_WOFF (170U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_169 = (((_ft_).u32_169 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_170 = (((_ft_).u32_170 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_169  >>  (24)) | ((IMG_UINT64)((_ft_).u32_170 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_WOFF (168U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_WOFF (169U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_168 = (((_ft_).u32_168 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_169 = (((_ft_).u32_169 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_168  >>  (28)) | ((IMG_UINT64)((_ft_).u32_169 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_WOFF   (168U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_168 = (((_ft_).u32_168 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_168  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_WOFF     (164U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_164 = (((_ft_).u32_164 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_164  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_WOFF  (163U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_163 = (((_ft_).u32_163 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_163  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_WOFF (162U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_WOFF (163U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_162 = (((_ft_).u32_162 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_163 = (((_ft_).u32_163 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_162  >>  (20)) | ((IMG_UINT64)((_ft_).u32_163 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_WOFF (161U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_WOFF (162U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_161 = (((_ft_).u32_161 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_162 = (((_ft_).u32_162 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_161  >>  (24)) | ((IMG_UINT64)((_ft_).u32_162 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_WOFF (160U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_WOFF (161U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_160 = (((_ft_).u32_160 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_161 = (((_ft_).u32_161 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_160  >>  (28)) | ((IMG_UINT64)((_ft_).u32_161 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_WOFF   (160U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_160 = (((_ft_).u32_160 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_160  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_WOFF     (156U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_156 = (((_ft_).u32_156 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_156  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE3 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_WOFF  (155U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_155 = (((_ft_).u32_155 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_155  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE3 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_WOFF (154U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_WOFF (155U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_154 = (((_ft_).u32_154 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_155 = (((_ft_).u32_155 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_154  >>  (20)) | ((IMG_UINT64)((_ft_).u32_155 & 0x0000ffffU ) << (12)))
+/*
+VCE3 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_WOFF (153U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_WOFF (154U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_153 = (((_ft_).u32_153 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_154 = (((_ft_).u32_154 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_153  >>  (24)) | ((IMG_UINT64)((_ft_).u32_154 & 0x000fffffU ) << (8)))
+/*
+VCE3 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_WOFF (152U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_WOFF (153U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_152 = (((_ft_).u32_152 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_153 = (((_ft_).u32_153 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_152  >>  (28)) | ((IMG_UINT64)((_ft_).u32_153 & 0x00ffffffU ) << (4)))
+/*
+VCE3 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_WOFF   (152U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_152 = (((_ft_).u32_152 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_152  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE3 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_WOFF     (148U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_148 = (((_ft_).u32_148 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_148  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_WOFF  (147U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_147 = (((_ft_).u32_147 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_147  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_WOFF (146U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_WOFF (147U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_146 = (((_ft_).u32_146 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_147 = (((_ft_).u32_147 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_146  >>  (20)) | ((IMG_UINT64)((_ft_).u32_147 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_WOFF (145U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_WOFF (146U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_145 = (((_ft_).u32_145 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_146 = (((_ft_).u32_146 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_145  >>  (24)) | ((IMG_UINT64)((_ft_).u32_146 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_WOFF (144U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_WOFF (145U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_144 = (((_ft_).u32_144 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_145 = (((_ft_).u32_145 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_144  >>  (28)) | ((IMG_UINT64)((_ft_).u32_145 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_WOFF   (144U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_144 = (((_ft_).u32_144 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_144  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_WOFF     (140U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_140 = (((_ft_).u32_140 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_140  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_WOFF  (139U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_139 = (((_ft_).u32_139 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_139  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_WOFF (138U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_WOFF (139U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_138 = (((_ft_).u32_138 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_139 = (((_ft_).u32_139 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_138  >>  (20)) | ((IMG_UINT64)((_ft_).u32_139 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_WOFF (137U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_WOFF (138U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_137 = (((_ft_).u32_137 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_138 = (((_ft_).u32_138 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_137  >>  (24)) | ((IMG_UINT64)((_ft_).u32_138 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_WOFF (136U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_WOFF (137U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_136 = (((_ft_).u32_136 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_137 = (((_ft_).u32_137 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_136  >>  (28)) | ((IMG_UINT64)((_ft_).u32_137 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_WOFF   (136U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_136 = (((_ft_).u32_136 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_136  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_WOFF     (132U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_132 = (((_ft_).u32_132 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_132  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_WOFF  (131U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_131 = (((_ft_).u32_131 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_131  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_WOFF (130U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_WOFF (131U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_130 = (((_ft_).u32_130 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_131 = (((_ft_).u32_131 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_130  >>  (20)) | ((IMG_UINT64)((_ft_).u32_131 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_WOFF (129U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_WOFF (130U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_129 = (((_ft_).u32_129 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_130 = (((_ft_).u32_130 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_129  >>  (24)) | ((IMG_UINT64)((_ft_).u32_130 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_WOFF (128U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_WOFF (129U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_128 = (((_ft_).u32_128 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_129 = (((_ft_).u32_129 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_128  >>  (28)) | ((IMG_UINT64)((_ft_).u32_129 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_WOFF   (128U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_128 = (((_ft_).u32_128 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_128  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_WOFF     (124U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_124 = (((_ft_).u32_124 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_124  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE2 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_WOFF  (123U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_123 = (((_ft_).u32_123 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_123  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE2 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_WOFF (122U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_WOFF (123U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_122 = (((_ft_).u32_122 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_123 = (((_ft_).u32_123 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_122  >>  (20)) | ((IMG_UINT64)((_ft_).u32_123 & 0x0000ffffU ) << (12)))
+/*
+VCE2 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_WOFF (121U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_WOFF (122U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_121 = (((_ft_).u32_121 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_122 = (((_ft_).u32_122 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_121  >>  (24)) | ((IMG_UINT64)((_ft_).u32_122 & 0x000fffffU ) << (8)))
+/*
+VCE2 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_WOFF (120U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_WOFF (121U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_120 = (((_ft_).u32_120 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_121 = (((_ft_).u32_121 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_120  >>  (28)) | ((IMG_UINT64)((_ft_).u32_121 & 0x00ffffffU ) << (4)))
+/*
+VCE2 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_WOFF   (120U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_120 = (((_ft_).u32_120 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_120  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE2 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_WOFF     (116U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_116 = (((_ft_).u32_116 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_116  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_WOFF  (115U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_115 = (((_ft_).u32_115 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_115  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_WOFF (114U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_WOFF (115U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_114 = (((_ft_).u32_114 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_115 = (((_ft_).u32_115 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_114  >>  (20)) | ((IMG_UINT64)((_ft_).u32_115 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_WOFF (113U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_WOFF (114U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_113 = (((_ft_).u32_113 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_114 = (((_ft_).u32_114 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_113  >>  (24)) | ((IMG_UINT64)((_ft_).u32_114 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_WOFF (112U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_WOFF (113U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_112 = (((_ft_).u32_112 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_113 = (((_ft_).u32_113 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_112  >>  (28)) | ((IMG_UINT64)((_ft_).u32_113 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_WOFF   (112U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_112 = (((_ft_).u32_112 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_112  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_WOFF     (108U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_108 = (((_ft_).u32_108 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_108  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_WOFF  (107U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_107 = (((_ft_).u32_107 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_107  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_WOFF (106U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_WOFF (107U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_106 = (((_ft_).u32_106 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_107 = (((_ft_).u32_107 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_106  >>  (20)) | ((IMG_UINT64)((_ft_).u32_107 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_WOFF (105U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_WOFF (106U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_105 = (((_ft_).u32_105 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_106 = (((_ft_).u32_106 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_105  >>  (24)) | ((IMG_UINT64)((_ft_).u32_106 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_WOFF (104U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_WOFF (105U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_104 = (((_ft_).u32_104 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_105 = (((_ft_).u32_105 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_104  >>  (28)) | ((IMG_UINT64)((_ft_).u32_105 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_WOFF   (104U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_104 = (((_ft_).u32_104 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_104  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_WOFF     (100U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_100 = (((_ft_).u32_100 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_100  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_WOFF  (99U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_99 = (((_ft_).u32_99 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_99  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_WOFF (98U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_WOFF (99U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_98 = (((_ft_).u32_98 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_99 = (((_ft_).u32_99 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_98  >>  (20)) | ((IMG_UINT64)((_ft_).u32_99 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_WOFF (97U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_WOFF (98U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_97 = (((_ft_).u32_97 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_98 = (((_ft_).u32_98 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_97  >>  (24)) | ((IMG_UINT64)((_ft_).u32_98 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_WOFF (96U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_WOFF (97U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_96 = (((_ft_).u32_96 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_97 = (((_ft_).u32_97 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_96  >>  (28)) | ((IMG_UINT64)((_ft_).u32_97 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_WOFF   (96U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_96 = (((_ft_).u32_96 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_96  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_WOFF     (92U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_92 = (((_ft_).u32_92 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_92  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE1 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_WOFF  (91U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_91 = (((_ft_).u32_91 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_91  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE1 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_WOFF (90U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_WOFF (91U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_90 = (((_ft_).u32_90 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_91 = (((_ft_).u32_91 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_90  >>  (20)) | ((IMG_UINT64)((_ft_).u32_91 & 0x0000ffffU ) << (12)))
+/*
+VCE1 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_WOFF (89U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_WOFF (90U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_89 = (((_ft_).u32_89 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_90 = (((_ft_).u32_90 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_89  >>  (24)) | ((IMG_UINT64)((_ft_).u32_90 & 0x000fffffU ) << (8)))
+/*
+VCE1 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_WOFF (88U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_WOFF (89U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_88 = (((_ft_).u32_88 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_89 = (((_ft_).u32_89 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_88  >>  (28)) | ((IMG_UINT64)((_ft_).u32_89 & 0x00ffffffU ) << (4)))
+/*
+VCE1 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_WOFF   (88U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_88 = (((_ft_).u32_88 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_88  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE1 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_WOFF     (84U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_84 = (((_ft_).u32_84 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_84  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page3 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_WOFF  (83U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_UFSTACK(_ft_,_x_) ((_ft_).u32_83 = (((_ft_).u32_83 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_UFSTACK(_ft_) (((_ft_).u32_83  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page3 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_WOFF (82U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_WOFF (83U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE3(_ft_,_x_) { ((_ft_).u32_82 = (((_ft_).u32_82 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_83 = (((_ft_).u32_83 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE3(_ft_) (((_ft_).u32_82  >>  (20)) | ((IMG_UINT64)((_ft_).u32_83 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page3 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_WOFF (81U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_WOFF (82U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE2(_ft_,_x_) { ((_ft_).u32_81 = (((_ft_).u32_81 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_82 = (((_ft_).u32_82 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE2(_ft_) (((_ft_).u32_81  >>  (24)) | ((IMG_UINT64)((_ft_).u32_82 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page3 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_WOFF (80U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_WOFF (81U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE1(_ft_,_x_) { ((_ft_).u32_80 = (((_ft_).u32_80 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_81 = (((_ft_).u32_81 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE1(_ft_) (((_ft_).u32_80  >>  (28)) | ((IMG_UINT64)((_ft_).u32_81 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page3 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_WOFF   (80U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PPAGE0(_ft_,_x_) ((_ft_).u32_80 = (((_ft_).u32_80 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PPAGE0(_ft_) (((_ft_).u32_80  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page3 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_WOFF     (76U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT3_PAGE(_ft_,_x_) ((_ft_).u32_76 = (((_ft_).u32_76 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT3_PAGE(_ft_) (((_ft_).u32_76  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT3_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page2 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_WOFF  (75U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_UFSTACK(_ft_,_x_) ((_ft_).u32_75 = (((_ft_).u32_75 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_UFSTACK(_ft_) (((_ft_).u32_75  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page2 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_WOFF (74U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_WOFF (75U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE3(_ft_,_x_) { ((_ft_).u32_74 = (((_ft_).u32_74 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_75 = (((_ft_).u32_75 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE3(_ft_) (((_ft_).u32_74  >>  (20)) | ((IMG_UINT64)((_ft_).u32_75 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page2 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_WOFF (73U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_WOFF (74U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE2(_ft_,_x_) { ((_ft_).u32_73 = (((_ft_).u32_73 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_74 = (((_ft_).u32_74 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE2(_ft_) (((_ft_).u32_73  >>  (24)) | ((IMG_UINT64)((_ft_).u32_74 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page2 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_WOFF (72U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_WOFF (73U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE1(_ft_,_x_) { ((_ft_).u32_72 = (((_ft_).u32_72 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_73 = (((_ft_).u32_73 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE1(_ft_) (((_ft_).u32_72  >>  (28)) | ((IMG_UINT64)((_ft_).u32_73 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page2 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_WOFF   (72U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PPAGE0(_ft_,_x_) ((_ft_).u32_72 = (((_ft_).u32_72 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PPAGE0(_ft_) (((_ft_).u32_72  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page2 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_WOFF     (68U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT2_PAGE(_ft_,_x_) ((_ft_).u32_68 = (((_ft_).u32_68 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT2_PAGE(_ft_) (((_ft_).u32_68  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT2_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page1 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_WOFF  (67U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_UFSTACK(_ft_,_x_) ((_ft_).u32_67 = (((_ft_).u32_67 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_UFSTACK(_ft_) (((_ft_).u32_67  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page1 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_WOFF (66U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_WOFF (67U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE3(_ft_,_x_) { ((_ft_).u32_66 = (((_ft_).u32_66 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_67 = (((_ft_).u32_67 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE3(_ft_) (((_ft_).u32_66  >>  (20)) | ((IMG_UINT64)((_ft_).u32_67 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page1 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_WOFF (65U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_WOFF (66U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE2(_ft_,_x_) { ((_ft_).u32_65 = (((_ft_).u32_65 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_66 = (((_ft_).u32_66 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE2(_ft_) (((_ft_).u32_65  >>  (24)) | ((IMG_UINT64)((_ft_).u32_66 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page1 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_WOFF (64U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_WOFF (65U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE1(_ft_,_x_) { ((_ft_).u32_64 = (((_ft_).u32_64 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_65 = (((_ft_).u32_65 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE1(_ft_) (((_ft_).u32_64  >>  (28)) | ((IMG_UINT64)((_ft_).u32_65 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page1 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_WOFF   (64U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PPAGE0(_ft_,_x_) ((_ft_).u32_64 = (((_ft_).u32_64 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PPAGE0(_ft_) (((_ft_).u32_64  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page1 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_WOFF     (60U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT1_PAGE(_ft_,_x_) ((_ft_).u32_60 = (((_ft_).u32_60 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT1_PAGE(_ft_) (((_ft_).u32_60  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT1_PAGE_SHIFT)) & 0x000fffffU)
+/*
+VCE0 opened page0 struct: unified stack bit
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_WOFF  (59U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_SHIFT (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_CLRMSK (0xFFFEFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_UFSTACK(_ft_,_x_) ((_ft_).u32_59 = (((_ft_).u32_59 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_UFSTACK(_ft_) (((_ft_).u32_59  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_UFSTACK_SHIFT)) & 0x00000001U)
+/*
+VCE0 opened page0 struct: physical page 3
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_WOFF (58U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_WOFF (59U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_SHIFT (20U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_CLRMSK (0x000FFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_CLRMSK (0xFFFF0000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE3(_ft_,_x_) { ((_ft_).u32_58 = (((_ft_).u32_58 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000000000fff)))  <<  20))); \
+                                                          ((_ft_).u32_59 = (((_ft_).u32_59 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE3_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffff000)))  >>  12))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE3(_ft_) (((_ft_).u32_58  >>  (20)) | ((IMG_UINT64)((_ft_).u32_59 & 0x0000ffffU ) << (12)))
+/*
+VCE0 opened page0 struct: physical page 2
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_WOFF (57U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_WOFF (58U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_SHIFT (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_CLRMSK (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_CLRMSK (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE2(_ft_,_x_) { ((_ft_).u32_57 = (((_ft_).u32_57 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000000000ff)))  <<  24))); \
+                                                          ((_ft_).u32_58 = (((_ft_).u32_58 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE2_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffff00)))  >>  8))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE2(_ft_) (((_ft_).u32_57  >>  (24)) | ((IMG_UINT64)((_ft_).u32_58 & 0x000fffffU ) << (8)))
+/*
+VCE0 opened page0 struct: physical page 1
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_WOFF (56U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_WOFF (57U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_SHIFT (28U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_SHIFT (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_CLRMSK (0x0FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_CLRMSK (0xFF000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE1(_ft_,_x_) { ((_ft_).u32_56 = (((_ft_).u32_56 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000000000f)))  <<  28))); \
+                                                          ((_ft_).u32_57 = (((_ft_).u32_57 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE1_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000ffffff0)))  >>  4))); }
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE1(_ft_) (((_ft_).u32_56  >>  (28)) | ((IMG_UINT64)((_ft_).u32_57 & 0x00ffffffU ) << (4)))
+/*
+VCE0 opened page0 struct: physical page 0
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_WOFF   (56U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_SHIFT  (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_CLRMSK (0xF0000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PPAGE0(_ft_,_x_) ((_ft_).u32_56 = (((_ft_).u32_56 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PPAGE0(_ft_) (((_ft_).u32_56  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PPAGE0_SHIFT)) & 0x0fffffffU)
+/*
+VCE0 opened page0 struct: virtual page number
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_WOFF     (52U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_SHIFT    (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_CLRMSK   (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT0_PAGE(_ft_,_x_) ((_ft_).u32_52 = (((_ft_).u32_52 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT0_PAGE(_ft_) (((_ft_).u32_52  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT0_PAGE_SHIFT)) & 0x000fffffU)
+/*
+Rsv2 area
+*/
+#define PM_DATA_VHEAP_BUFFER_RESV2_W0_WOFF                (42U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W0_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W1_WOFF                (43U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W1_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W2_WOFF                (44U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W2_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W3_WOFF                (45U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W3_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W4_WOFF                (46U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W4_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W5_WOFF                (47U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W5_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W6_WOFF                (48U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W6_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W7_WOFF                (49U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W7_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W8_WOFF                (50U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W8_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W9_WOFF                (51U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W9_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W0_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W1_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W2_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W3_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W4_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W5_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W6_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W7_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W8_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV2_W9_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W0(_ft_,_x_)       ((_ft_).u32_42 = (((_ft_).u32_42 & PM_DATA_VHEAP_BUFFER_RESV2_W0_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W0_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W0(_ft_)           (((_ft_).u32_42  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W0_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W1(_ft_,_x_)       ((_ft_).u32_43 = (((_ft_).u32_43 & PM_DATA_VHEAP_BUFFER_RESV2_W1_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W1_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W1(_ft_)           (((_ft_).u32_43  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W1_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W2(_ft_,_x_)       ((_ft_).u32_44 = (((_ft_).u32_44 & PM_DATA_VHEAP_BUFFER_RESV2_W2_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W2_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W2(_ft_)           (((_ft_).u32_44  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W2_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W3(_ft_,_x_)       ((_ft_).u32_45 = (((_ft_).u32_45 & PM_DATA_VHEAP_BUFFER_RESV2_W3_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W3_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W3(_ft_)           (((_ft_).u32_45  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W3_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W4(_ft_,_x_)       ((_ft_).u32_46 = (((_ft_).u32_46 & PM_DATA_VHEAP_BUFFER_RESV2_W4_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W4_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W4(_ft_)           (((_ft_).u32_46  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W4_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W5(_ft_,_x_)       ((_ft_).u32_47 = (((_ft_).u32_47 & PM_DATA_VHEAP_BUFFER_RESV2_W5_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W5_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W5(_ft_)           (((_ft_).u32_47  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W5_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W6(_ft_,_x_)       ((_ft_).u32_48 = (((_ft_).u32_48 & PM_DATA_VHEAP_BUFFER_RESV2_W6_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W6_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W6(_ft_)           (((_ft_).u32_48  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W6_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W7(_ft_,_x_)       ((_ft_).u32_49 = (((_ft_).u32_49 & PM_DATA_VHEAP_BUFFER_RESV2_W7_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W7_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W7(_ft_)           (((_ft_).u32_49  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W7_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W8(_ft_,_x_)       ((_ft_).u32_50 = (((_ft_).u32_50 & PM_DATA_VHEAP_BUFFER_RESV2_W8_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W8_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W8(_ft_)           (((_ft_).u32_50  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W8_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV2_W9(_ft_,_x_)       ((_ft_).u32_51 = (((_ft_).u32_51 & PM_DATA_VHEAP_BUFFER_RESV2_W9_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV2_W9_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV2_W9(_ft_)           (((_ft_).u32_51  >>  (PM_DATA_VHEAP_BUFFER_RESV2_W9_SHIFT)) & 0xffffffffU)
+/*
+Number of pages allocated to TE7 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_SHIFT       (24U)
+#define PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_CLRMSK      (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE7_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE7_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE7_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE6 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_SHIFT       (16U)
+#define PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_CLRMSK      (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE6_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE6_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE6_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE5 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_SHIFT       (8U)
+#define PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_CLRMSK      (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE5_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE5_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE5_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE4 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_WOFF        (41U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_SHIFT       (0U)
+#define PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_CLRMSK      (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE4_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE4_INFLIGHT_CNT(_ft_)   (((_ft_).u32_41  >>  (PM_DATA_VHEAP_BUFFER_TE4_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE3 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_SHIFT       (24U)
+#define PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_CLRMSK      (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE3_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE3_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE3_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE2 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_SHIFT       (16U)
+#define PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_CLRMSK      (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE2_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE2_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE2_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE1 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_SHIFT       (8U)
+#define PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_CLRMSK      (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_TE1_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE1_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE1_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to TE0 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_WOFF        (40U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_SHIFT       (0U)
+#define PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_CLRMSK      (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_TE0_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_TE0_INFLIGHT_CNT(_ft_)   (((_ft_).u32_40  >>  (PM_DATA_VHEAP_BUFFER_TE0_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE7 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_SHIFT      (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_CLRMSK     (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE7_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE7_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE7_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE6 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_SHIFT      (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_CLRMSK     (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE6_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE6_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE6_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE5 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_SHIFT      (8U)
+#define PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_CLRMSK     (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE5_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE5_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE5_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE4 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_WOFF       (39U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_SHIFT      (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_CLRMSK     (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE4_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_39 = (((_ft_).u32_39 & PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE4_INFLIGHT_CNT(_ft_)  (((_ft_).u32_39  >>  (PM_DATA_VHEAP_BUFFER_VCE4_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE3 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_SHIFT      (24U)
+#define PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_CLRMSK     (0x00FFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE3_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE3_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE3_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE2 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_SHIFT      (16U)
+#define PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_CLRMSK     (0xFF00FFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE2_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE2_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE2_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE1 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_SHIFT      (8U)
+#define PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_CLRMSK     (0xFFFF00FFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE1_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE1_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE1_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+Number of pages allocated to VCE0 but not yet closed
+*/
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_WOFF       (38U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_SHIFT      (0U)
+#define PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_CLRMSK     (0xFFFFFF00U)
+#define PM_DATA_VHEAP_BUFFER_SET_VCE0_INFLIGHT_CNT(_ft_,_x_) ((_ft_).u32_38 = (((_ft_).u32_38 & PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_CLRMSK ) | (((_x_) & (0x000000ffU))  <<  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VCE0_INFLIGHT_CNT(_ft_)  (((_ft_).u32_38  >>  (PM_DATA_VHEAP_BUFFER_VCE0_INFLIGHT_CNT_SHIFT)) & 0x000000ffU)
+/*
+1=The PM ran out of memory during processing
+*/
+#define PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_WOFF           (37U)
+#define PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_PM_OUTOFMEM_R(_ft_,_x_)  ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_PM_OUTOFMEM_R(_ft_)      (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_PM_OUTOFMEM_R_SHIFT)) & 0x00000001U)
+/*
+A copy of rgx_cr_pm_outofmem_abortall (at the point the VHEAP buffer was written)
+*/
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_WOFF          (37U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_SHIFT         (30U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_CLRMSK        (0xBFFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_OUTOFMEM_ABORT(_ft_,_x_) ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_OUTOFMEM_ABORT(_ft_)     (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_ABORT_SHIFT)) & 0x00000001U)
+/*
+When running out of memory, indicates which of the free stacks have run out of memory.
+If bit 2 is set, MMUSTACK has run out of memory.
+Bit 2 is reserved.
+If bit 1 is set, UFSTACK has run out of memory.
+If bit 0 is set, FSTACK has run out of memory.
+*/
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_WOFF            (37U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_SHIFT           (2U)
+#define PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_CLRMSK          (0xFFFFFFE3U)
+#define PM_DATA_VHEAP_BUFFER_SET_OUTOFMEM_SRC(_ft_,_x_)   ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_CLRMSK ) | (((_x_) & (0x00000007U))  <<  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_OUTOFMEM_SRC(_ft_)       (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_OUTOFMEM_SRC_SHIFT)) & 0x00000007U)
+/*
+When running out of memory, indicates the source of the request that caused the OOM event
+If bit 1 is set, TE caused the OOM.
+If bit 0 is set, VCE caused the OOM.
+*/
+#define PM_DATA_VHEAP_BUFFER_REQ_SRC_WOFF                 (37U)
+#define PM_DATA_VHEAP_BUFFER_REQ_SRC_SHIFT                (0U)
+#define PM_DATA_VHEAP_BUFFER_REQ_SRC_CLRMSK               (0xFFFFFFFCU)
+#define PM_DATA_VHEAP_BUFFER_SET_REQ_SRC(_ft_,_x_)        ((_ft_).u32_37 = (((_ft_).u32_37 & PM_DATA_VHEAP_BUFFER_REQ_SRC_CLRMSK ) | (((_x_) & (0x00000003U))  <<  (PM_DATA_VHEAP_BUFFER_REQ_SRC_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_REQ_SRC(_ft_)            (((_ft_).u32_37  >>  (PM_DATA_VHEAP_BUFFER_REQ_SRC_SHIFT)) & 0x00000003U)
+/*
+MAX RTA index dword in TA stream
+*/
+#define PM_DATA_VHEAP_BUFFER_MAX_RTA_WOFF                 (36U)
+#define PM_DATA_VHEAP_BUFFER_MAX_RTA_SHIFT                (0U)
+#define PM_DATA_VHEAP_BUFFER_MAX_RTA_CLRMSK               (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_MAX_RTA(_ft_,_x_)        ((_ft_).u32_36 = (((_ft_).u32_36 & PM_DATA_VHEAP_BUFFER_MAX_RTA_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (PM_DATA_VHEAP_BUFFER_MAX_RTA_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_MAX_RTA(_ft_)            (((_ft_).u32_36  >>  (PM_DATA_VHEAP_BUFFER_MAX_RTA_SHIFT)) & 0xffffffffU)
+/*
+Rsv1 area
+*/
+#define PM_DATA_VHEAP_BUFFER_RESV1_W0_WOFF                (20U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W0_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W1_WOFF                (21U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W1_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W2_WOFF                (22U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W2_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W3_WOFF                (23U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W3_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W4_WOFF                (24U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W4_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W5_WOFF                (25U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W5_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W6_WOFF                (26U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W6_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W7_WOFF                (27U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W7_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W8_WOFF                (28U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W8_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W9_WOFF                (29U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W9_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W10_WOFF               (30U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W10_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W11_WOFF               (31U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W11_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W12_WOFF               (32U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W12_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W13_WOFF               (33U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W13_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W14_WOFF               (34U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W14_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W15_WOFF               (35U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W15_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W0_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W1_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W2_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W3_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W4_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W5_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W6_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W7_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W8_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W9_CLRMSK              (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W10_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W11_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W12_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W13_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W14_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_RESV1_W15_CLRMSK             (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W0(_ft_,_x_)       ((_ft_).u32_20 = (((_ft_).u32_20 & PM_DATA_VHEAP_BUFFER_RESV1_W0_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W0_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W0(_ft_)           (((_ft_).u32_20  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W0_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W1(_ft_,_x_)       ((_ft_).u32_21 = (((_ft_).u32_21 & PM_DATA_VHEAP_BUFFER_RESV1_W1_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W1_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W1(_ft_)           (((_ft_).u32_21  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W1_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W2(_ft_,_x_)       ((_ft_).u32_22 = (((_ft_).u32_22 & PM_DATA_VHEAP_BUFFER_RESV1_W2_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W2_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W2(_ft_)           (((_ft_).u32_22  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W2_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W3(_ft_,_x_)       ((_ft_).u32_23 = (((_ft_).u32_23 & PM_DATA_VHEAP_BUFFER_RESV1_W3_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W3_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W3(_ft_)           (((_ft_).u32_23  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W3_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W4(_ft_,_x_)       ((_ft_).u32_24 = (((_ft_).u32_24 & PM_DATA_VHEAP_BUFFER_RESV1_W4_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W4_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W4(_ft_)           (((_ft_).u32_24  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W4_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W5(_ft_,_x_)       ((_ft_).u32_25 = (((_ft_).u32_25 & PM_DATA_VHEAP_BUFFER_RESV1_W5_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W5_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W5(_ft_)           (((_ft_).u32_25  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W5_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W6(_ft_,_x_)       ((_ft_).u32_26 = (((_ft_).u32_26 & PM_DATA_VHEAP_BUFFER_RESV1_W6_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W6_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W6(_ft_)           (((_ft_).u32_26  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W6_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W7(_ft_,_x_)       ((_ft_).u32_27 = (((_ft_).u32_27 & PM_DATA_VHEAP_BUFFER_RESV1_W7_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W7_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W7(_ft_)           (((_ft_).u32_27  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W7_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W8(_ft_,_x_)       ((_ft_).u32_28 = (((_ft_).u32_28 & PM_DATA_VHEAP_BUFFER_RESV1_W8_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W8_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W8(_ft_)           (((_ft_).u32_28  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W8_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W9(_ft_,_x_)       ((_ft_).u32_29 = (((_ft_).u32_29 & PM_DATA_VHEAP_BUFFER_RESV1_W9_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W9_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W9(_ft_)           (((_ft_).u32_29  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W9_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W10(_ft_,_x_)      ((_ft_).u32_30 = (((_ft_).u32_30 & PM_DATA_VHEAP_BUFFER_RESV1_W10_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W10_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W10(_ft_)          (((_ft_).u32_30  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W10_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W11(_ft_,_x_)      ((_ft_).u32_31 = (((_ft_).u32_31 & PM_DATA_VHEAP_BUFFER_RESV1_W11_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W11_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W11(_ft_)          (((_ft_).u32_31  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W11_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W12(_ft_,_x_)      ((_ft_).u32_32 = (((_ft_).u32_32 & PM_DATA_VHEAP_BUFFER_RESV1_W12_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W12_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W12(_ft_)          (((_ft_).u32_32  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W12_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W13(_ft_,_x_)      ((_ft_).u32_33 = (((_ft_).u32_33 & PM_DATA_VHEAP_BUFFER_RESV1_W13_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W13_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W13(_ft_)          (((_ft_).u32_33  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W13_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W14(_ft_,_x_)      ((_ft_).u32_34 = (((_ft_).u32_34 & PM_DATA_VHEAP_BUFFER_RESV1_W14_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W14_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W14(_ft_)          (((_ft_).u32_34  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W14_SHIFT)) & 0xffffffffU)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV1_W15(_ft_,_x_)      ((_ft_).u32_35 = (((_ft_).u32_35 & PM_DATA_VHEAP_BUFFER_RESV1_W15_CLRMSK ) |  (((_x_) & (0xffffffffU))  <<  PM_DATA_VHEAP_BUFFER_RESV1_W15_SHIFT)))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV1_W15(_ft_)          (((_ft_).u32_35  >>  (PM_DATA_VHEAP_BUFFER_RESV1_W15_SHIFT)) & 0xffffffffU)
+/*
+Rsv0 area
+*/
+#define PM_DATA_VHEAP_BUFFER_RESV0_WOFF                   (19U)
+#define PM_DATA_VHEAP_BUFFER_RESV0_SHIFT                  (0U)
+#define PM_DATA_VHEAP_BUFFER_RESV0_CLRMSK                 (0x00000000U)
+#define PM_DATA_VHEAP_BUFFER_SET_RESV0(_ft_,_x_)          ((_ft_).u32_19 = (((_ft_).u32_19 & PM_DATA_VHEAP_BUFFER_RESV0_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (PM_DATA_VHEAP_BUFFER_RESV0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_RESV0(_ft_)              (((_ft_).u32_19  >>  (PM_DATA_VHEAP_BUFFER_RESV0_SHIFT)) & 0xffffffffU)
+/*
+Init Bit Sent Flag for TE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_WOFF           (18U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE7_INIT(_ft_,_x_)  ((_ft_).u32_18 = (((_ft_).u32_18 & PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE7_INIT(_ft_)      (((_ft_).u32_18  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_WOFF                (18U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE7_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE7(_ft_,_x_)       ((_ft_).u32_18 = (((_ft_).u32_18 & PM_DATA_VHEAP_BUFFER_VPTR_TE7_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE7(_ft_)           (((_ft_).u32_18  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE7_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_WOFF          (17U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE7_INIT(_ft_,_x_) ((_ft_).u32_17 = (((_ft_).u32_17 & PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE7_INIT(_ft_)     (((_ft_).u32_17  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE7
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_WOFF               (17U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE7_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE7(_ft_,_x_)      ((_ft_).u32_17 = (((_ft_).u32_17 & PM_DATA_VHEAP_BUFFER_VPTR_VCE7_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE7(_ft_)          (((_ft_).u32_17  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE7_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_WOFF           (16U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE6_INIT(_ft_,_x_)  ((_ft_).u32_16 = (((_ft_).u32_16 & PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE6_INIT(_ft_)      (((_ft_).u32_16  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_WOFF                (16U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE6_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE6(_ft_,_x_)       ((_ft_).u32_16 = (((_ft_).u32_16 & PM_DATA_VHEAP_BUFFER_VPTR_TE6_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE6(_ft_)           (((_ft_).u32_16  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE6_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_WOFF          (15U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE6_INIT(_ft_,_x_) ((_ft_).u32_15 = (((_ft_).u32_15 & PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE6_INIT(_ft_)     (((_ft_).u32_15  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE6
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_WOFF               (15U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE6_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE6(_ft_,_x_)      ((_ft_).u32_15 = (((_ft_).u32_15 & PM_DATA_VHEAP_BUFFER_VPTR_VCE6_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE6(_ft_)          (((_ft_).u32_15  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE6_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_WOFF           (14U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE5_INIT(_ft_,_x_)  ((_ft_).u32_14 = (((_ft_).u32_14 & PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE5_INIT(_ft_)      (((_ft_).u32_14  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_WOFF                (14U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE5_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE5(_ft_,_x_)       ((_ft_).u32_14 = (((_ft_).u32_14 & PM_DATA_VHEAP_BUFFER_VPTR_TE5_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE5(_ft_)           (((_ft_).u32_14  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE5_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_WOFF          (13U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE5_INIT(_ft_,_x_) ((_ft_).u32_13 = (((_ft_).u32_13 & PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE5_INIT(_ft_)     (((_ft_).u32_13  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE5
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_WOFF               (13U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE5_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE5(_ft_,_x_)      ((_ft_).u32_13 = (((_ft_).u32_13 & PM_DATA_VHEAP_BUFFER_VPTR_VCE5_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE5(_ft_)          (((_ft_).u32_13  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE5_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_WOFF           (12U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE4_INIT(_ft_,_x_)  ((_ft_).u32_12 = (((_ft_).u32_12 & PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE4_INIT(_ft_)      (((_ft_).u32_12  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_WOFF                (12U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE4_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE4(_ft_,_x_)       ((_ft_).u32_12 = (((_ft_).u32_12 & PM_DATA_VHEAP_BUFFER_VPTR_TE4_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE4(_ft_)           (((_ft_).u32_12  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE4_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_WOFF          (11U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE4_INIT(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE4_INIT(_ft_)     (((_ft_).u32_11  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE4
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_WOFF               (11U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE4_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE4(_ft_,_x_)      ((_ft_).u32_11 = (((_ft_).u32_11 & PM_DATA_VHEAP_BUFFER_VPTR_VCE4_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE4(_ft_)          (((_ft_).u32_11  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE4_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_WOFF           (10U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE3_INIT(_ft_,_x_)  ((_ft_).u32_10 = (((_ft_).u32_10 & PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE3_INIT(_ft_)      (((_ft_).u32_10  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_WOFF                (10U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE3_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE3(_ft_,_x_)       ((_ft_).u32_10 = (((_ft_).u32_10 & PM_DATA_VHEAP_BUFFER_VPTR_TE3_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE3(_ft_)           (((_ft_).u32_10  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE3_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_WOFF          (9U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE3_INIT(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE3_INIT(_ft_)     (((_ft_).u32_9  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE3
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_WOFF               (9U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE3_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE3(_ft_,_x_)      ((_ft_).u32_9 = (((_ft_).u32_9 & PM_DATA_VHEAP_BUFFER_VPTR_VCE3_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE3(_ft_)          (((_ft_).u32_9  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE3_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_WOFF           (8U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE2_INIT(_ft_,_x_)  ((_ft_).u32_8 = (((_ft_).u32_8 & PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE2_INIT(_ft_)      (((_ft_).u32_8  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_WOFF                (8U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE2_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE2(_ft_,_x_)       ((_ft_).u32_8 = (((_ft_).u32_8 & PM_DATA_VHEAP_BUFFER_VPTR_TE2_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE2(_ft_)           (((_ft_).u32_8  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE2_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_WOFF          (7U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE2_INIT(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE2_INIT(_ft_)     (((_ft_).u32_7  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE2
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_WOFF               (7U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE2_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE2(_ft_,_x_)      ((_ft_).u32_7 = (((_ft_).u32_7 & PM_DATA_VHEAP_BUFFER_VPTR_VCE2_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE2(_ft_)          (((_ft_).u32_7  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE2_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_WOFF           (6U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE1_INIT(_ft_,_x_)  ((_ft_).u32_6 = (((_ft_).u32_6 & PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE1_INIT(_ft_)      (((_ft_).u32_6  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_WOFF                (6U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE1_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE1(_ft_,_x_)       ((_ft_).u32_6 = (((_ft_).u32_6 & PM_DATA_VHEAP_BUFFER_VPTR_TE1_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE1(_ft_)           (((_ft_).u32_6  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE1_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_WOFF          (5U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE1_INIT(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE1_INIT(_ft_)     (((_ft_).u32_5  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE1
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_WOFF               (5U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE1_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE1(_ft_,_x_)      ((_ft_).u32_5 = (((_ft_).u32_5 & PM_DATA_VHEAP_BUFFER_VPTR_VCE1_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE1(_ft_)          (((_ft_).u32_5  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE1_SHIFT)) & 0x000fffffU)
+/*
+4KB aligned  top pointer for MMU
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_MMU_WOFF                (4U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_MMU_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_MMU_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_MMU(_ft_,_x_)       ((_ft_).u32_4 = (((_ft_).u32_4 & PM_DATA_VHEAP_BUFFER_VPTR_MMU_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_MMU_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_MMU(_ft_)           (((_ft_).u32_4  >>  (PM_DATA_VHEAP_BUFFER_VPTR_MMU_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for ALIST
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_WOFF         (3U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_SHIFT        (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_CLRMSK       (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_ALIST_INIT(_ft_,_x_) ((_ft_).u32_3 = (((_ft_).u32_3 & PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_ALIST_INIT(_ft_)    (((_ft_).u32_3  >>  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for ALIST
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_WOFF              (3U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_SHIFT             (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_ALIST_CLRMSK            (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_ALIST(_ft_,_x_)     ((_ft_).u32_3 = (((_ft_).u32_3 & PM_DATA_VHEAP_BUFFER_VPTR_ALIST_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_ALIST(_ft_)         (((_ft_).u32_3  >>  (PM_DATA_VHEAP_BUFFER_VPTR_ALIST_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_WOFF           (1U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_SHIFT          (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_CLRMSK         (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE0_INIT(_ft_,_x_)  ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE0_INIT(_ft_)      (((_ft_).u32_1  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_WOFF                (1U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_SHIFT               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_TE0_CLRMSK              (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_TE0(_ft_,_x_)       ((_ft_).u32_1 = (((_ft_).u32_1 & PM_DATA_VHEAP_BUFFER_VPTR_TE0_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_TE0(_ft_)           (((_ft_).u32_1  >>  (PM_DATA_VHEAP_BUFFER_VPTR_TE0_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for VCE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_WOFF          (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_SHIFT         (31U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_CLRMSK        (0x7FFFFFFFU)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE0_INIT(_ft_,_x_) ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE0_INIT(_ft_)     (((_ft_).u32_0  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE0
+*/
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_WOFF               (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_SHIFT              (0U)
+#define PM_DATA_VHEAP_BUFFER_VPTR_VCE0_CLRMSK             (0xFFF00000U)
+#define PM_DATA_VHEAP_BUFFER_SET_VPTR_VCE0(_ft_,_x_)      ((_ft_).u32_0 = (((_ft_).u32_0 & PM_DATA_VHEAP_BUFFER_VPTR_VCE0_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_SHIFT))))
+#define PM_DATA_VHEAP_BUFFER_GET_VPTR_VCE0(_ft_)          (((_ft_).u32_0  >>  (PM_DATA_VHEAP_BUFFER_VPTR_VCE0_SHIFT)) & 0x000fffffU)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+The PM FreeListState Buffer Layout - this will apply to 3 resources - FSTACK, UFSTACK and MMUSTACK
+*/
+typedef struct RGX_PM_FREELISTSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+} RGX_PM_FREELISTSTATE_BUFFER;
+
+/*
+Reserved field word 2
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_WOFF    (7U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_2(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_2(_ft_) (((_ft_).u32_7  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 1
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_WOFF    (6U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_1(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_1(_ft_) (((_ft_).u32_6  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 0
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_WOFF    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_0(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_0(_ft_) (((_ft_).u32_5  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT)) & 0xffffffffU)
+/*
+The number of pages consumed for the MMU Page Table. Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_WOFF   (4U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT  (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_MMUPAGE_STATUS(_ft_) (((_ft_).u32_4  >>  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The total number of pages consumed from the free stack. Must be initialised to zero.  This field is unused in the MMUSTACK.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_WOFF      (3U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK    (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(_ft_,_x_) ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_PAGE_STATUS(_ft_) (((_ft_).u32_3  >>  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Stack pointer for the free stack - the location of the next free page relative to the BaseAddr, in number of DWORDs.  Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_WOFF        (2U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT       (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK      (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(_ft_,_x_) ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_STACK_PTR(_ft_)   (((_ft_).u32_2  >>  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT)) & 0xffffffffU)
+/*
+Base address of the free stack - points to the bottom of the stack.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_WOFF     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_SHIFT    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_WOFF     (1U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_SHIFT    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK   (0x0000001FU)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK   (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_BASE_ADDR(_ft_)   (((_ft_).u32_0  >>  (5)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (27)))
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+The PM FreeListState Buffer Layout - this will apply to 3 resources - FSTACK, UFSTACK and MMUSTACK
+*/
+typedef struct RGX_PM_FREELISTSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+} RGX_PM_FREELISTSTATE_BUFFER;
+
+/*
+Reserved field word 2
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_WOFF    (7U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_2(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_2(_ft_) (((_ft_).u32_7  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_2_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 1
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_WOFF    (6U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_1(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_1(_ft_) (((_ft_).u32_6  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_1_SHIFT)) & 0xffffffffU)
+/*
+Reserved field word 0
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_WOFF    (5U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT   (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK  (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_RSV_STUFF32_0(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_RSV_STUFF32_0(_ft_) (((_ft_).u32_5  >>  (RGX_PM_FREELISTSTATE_BUFFER_RSV_STUFF32_0_SHIFT)) & 0xffffffffU)
+/*
+The number of pages consumed for the MMU Page Table. Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_WOFF   (4U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT  (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_MMUPAGE_STATUS(_ft_) (((_ft_).u32_4  >>  (RGX_PM_FREELISTSTATE_BUFFER_MMUPAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The total number of pages consumed from the free stack. Must be initialised to zero.  This field is unused in the MMUSTACK.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_WOFF      (3U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK    (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(_ft_,_x_) ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_PAGE_STATUS(_ft_) (((_ft_).u32_3  >>  (RGX_PM_FREELISTSTATE_BUFFER_PAGE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Stack pointer for the free stack - the location of the next free page relative to the BaseAddr, in number of DWORDs.  Must be initialised to zero.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_WOFF        (2U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT       (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK      (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(_ft_,_x_) ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT))))
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_STACK_PTR(_ft_)   (((_ft_).u32_2  >>  (RGX_PM_FREELISTSTATE_BUFFER_STACK_PTR_SHIFT)) & 0xffffffffU)
+/*
+Base address of the free stack - points to the bottom of the stack.
+*/
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_WOFF     (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_SHIFT    (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_WOFF     (1U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_SHIFT    (0U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK   (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK   (0x00000000U)
+#define RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_FREELISTSTATE_BUFFER_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_FREELISTSTATE_BUFFER_GET_BASE_ADDR(_ft_)   (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (32)))
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+256-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT         (5U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSIZE          (32U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR          (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_LOWER    (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_UPPER    (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT         (4U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSIZE          (16U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR          (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_LOWER    (0U)
+#define RGX_PM_FREELISTSTATE_BASE_ADDR_BASE_ADDR_UPPER    (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+/*
+Maximum range supported by hardware is 23 bits.
+*/
+#define RGX_PM_FREELISTSTATE_STACK_PTR_STACK_PTR          (0U)
+#define RGX_PM_FREELISTSTATE_STACK_PTR_STACK_PTR_LOWER    (0U)
+#define RGX_PM_FREELISTSTATE_STACK_PTR_STACK_PTR_UPPER    (16777215U)
+
+
+/*
+Maximum range supported by hardware is 23 bits.
+*/
+#define RGX_PM_FREELISTSTATE_PAGE_STATUS_PAGE_STATUS      (0U)
+#define RGX_PM_FREELISTSTATE_PAGE_STATUS_PAGE_STATUS_LOWER (0U)
+#define RGX_PM_FREELISTSTATE_PAGE_STATUS_PAGE_STATUS_UPPER (16777215U)
+
+
+/*
+Maximum range supported by hardware is 23 bits.
+*/
+#define RGX_PM_FREELISTSTATE_MMUPAGE_STATUS_MMUPAGE_STATUS (0U)
+#define RGX_PM_FREELISTSTATE_MMUPAGE_STATUS_MMUPAGE_STATUS_LOWER (0U)
+#define RGX_PM_FREELISTSTATE_MMUPAGE_STATUS_MMUPAGE_STATUS_UPPER (16777215U)
+
+
+#if defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+The base address of the Virtual-Physical Page Translation Table.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_WOFF   (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_SHIFT  (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_WOFF   (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_SHIFT  (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK (0x0000000FU)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VFP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffffff)))  <<  4))); \
+                                                          ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0ffffffff0000000)))  >>  28))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VFP_BASE_ADDR(_ft_) (((_ft_).u32_10  >>  (4)) | ((IMG_UINT64)((_ft_).u32_11 & 0xffffffffU ) << (28)))
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The base address of the VHEAP buffer.
+Must be initialised to point to the location of the VHEAP buffer in memory.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_SHIFT (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_WOFF (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_SHIFT (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK (0x0000000FU)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x000000000fffffff)))  <<  4))); \
+                                                          ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0ffffffff0000000)))  >>  28))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VHEAP_BASE_ADDR(_ft_) (((_ft_).u32_6  >>  (4)) | ((IMG_UINT64)((_ft_).u32_7 & 0xffffffffU ) << (28)))
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x0000001FU)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (5)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (27)))
+/*
+The number of entries on the ALIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (32)))
+#endif /* defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE) */
+
+
+#if defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+The base address of the Virtual-Physical Page Translation Table.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_WOFF   (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_WOFF   (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VFP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VFP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VFP_BASE_ADDR(_ft_) (((_ft_).u32_10  >>  (0)) | ((IMG_UINT64)((_ft_).u32_11 & 0xffffffffU ) << (32)))
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+Must be initialised to zero.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+The base address of the VHEAP buffer.
+Must be initialised to point to the location of the VHEAP buffer in memory.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_WOFF (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_VHEAP_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VHEAP_BASE_ADDR(_ft_) (((_ft_).u32_6  >>  (0)) | ((IMG_UINT64)((_ft_).u32_7 & 0xffffffffU ) << (32)))
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (0)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (32)))
+/*
+The number of entries on the ALIST.  Must be initialised to zero, meaning no pages allocated.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0xffffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0xffffffffU ) << (32)))
+#endif /* defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE) */
+
+
+#if defined(RGX_FEATURE_SINGLE_TE_VSPACE)
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+MMU catalogue base address for VCE pipe 3 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_WOFF (37U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_,_x_) ((_ft_).u32_37 = (((_ft_).u32_37 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_) (((_ft_).u32_37  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 3 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 3 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_WOFF (35U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_,_x_) ((_ft_).u32_35 = (((_ft_).u32_35 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_) (((_ft_).u32_35  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 2 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_WOFF (33U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_,_x_) ((_ft_).u32_33 = (((_ft_).u32_33 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_) (((_ft_).u32_33  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 1 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_WOFF (30U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_30 = (((_ft_).u32_30 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_30  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_WOFF (29U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_29 = (((_ft_).u32_29 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_) (((_ft_).u32_29  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for VCE pipe 0 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 0 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_WOFF (26U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_26 = (((_ft_).u32_26 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_26  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_WOFF (25U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_25 = (((_ft_).u32_25 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_) (((_ft_).u32_25  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for TE MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_WOFF (18U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_,_x_) ((_ft_).u32_18 = (((_ft_).u32_18 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_) (((_ft_).u32_18  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_WOFF (17U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_ADDR(_ft_,_x_) ((_ft_).u32_17 = (((_ft_).u32_17 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_ADDR(_ft_) (((_ft_).u32_17  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for ALIST MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_MAPPED(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_MAPPED(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for ALIST INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_WOFF      (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0_INIT(_ft_,_x_) ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0_INIT(_ft_) (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_WOFF           (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0(_ft_,_x_)  ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0(_ft_)      (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_WOFF     (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3_INIT(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3_INIT(_ft_) (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_WOFF          (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3(_ft_)     (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_WOFF     (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2_INIT(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2_INIT(_ft_) (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_WOFF          (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2(_ft_)     (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_WOFF     (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1_INIT(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1_INIT(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_WOFF          (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1(_ft_)     (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_WOFF     (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0_INIT(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0_INIT(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_WOFF          (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0(_ft_)     (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT)) & 0x7fffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_7  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_6  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x0000001FU)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (5)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (27)))
+/*
+Init bit sent flag for ALIST
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_WOFF         (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT        (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK       (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_INIT(_ft_,_x_) ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_INIT(_ft_)    (((_ft_).u32_1  >>  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT)) & 0x00000001U)
+/*
+The number of entries on the ALIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x7fffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0x7fffffffU ) << (32)))
+#endif /* defined(RGX_FEATURE_SINGLE_TE_VSPACE) */
+
+
+#if !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_SINGLE_TE_VSPACE))
+/*
+The PM Render Context Buffer Layout
+*/
+typedef struct RGX_PM_RENDERSTATE_BUFFER_TAG {
+     IMG_UINT32 u32_0;
+     IMG_UINT32 u32_1;
+     IMG_UINT32 u32_2;
+     IMG_UINT32 u32_3;
+     IMG_UINT32 u32_4;
+     IMG_UINT32 u32_5;
+     IMG_UINT32 u32_6;
+     IMG_UINT32 u32_7;
+     IMG_UINT32 u32_8;
+     IMG_UINT32 u32_9;
+     IMG_UINT32 u32_10;
+     IMG_UINT32 u32_11;
+     IMG_UINT32 u32_12;
+     IMG_UINT32 u32_13;
+     IMG_UINT32 u32_14;
+     IMG_UINT32 u32_15;
+     IMG_UINT32 u32_16;
+     IMG_UINT32 u32_17;
+     IMG_UINT32 u32_18;
+     IMG_UINT32 u32_19;
+     IMG_UINT32 u32_20;
+     IMG_UINT32 u32_21;
+     IMG_UINT32 u32_22;
+     IMG_UINT32 u32_23;
+     IMG_UINT32 u32_24;
+     IMG_UINT32 u32_25;
+     IMG_UINT32 u32_26;
+     IMG_UINT32 u32_27;
+     IMG_UINT32 u32_28;
+     IMG_UINT32 u32_29;
+     IMG_UINT32 u32_30;
+     IMG_UINT32 u32_31;
+     IMG_UINT32 u32_32;
+     IMG_UINT32 u32_33;
+     IMG_UINT32 u32_34;
+     IMG_UINT32 u32_35;
+     IMG_UINT32 u32_36;
+     IMG_UINT32 u32_37;
+     IMG_UINT32 u32_38;
+     IMG_UINT32 u32_39;
+     IMG_UINT32 u32_40;
+     IMG_UINT32 u32_41;
+     IMG_UINT32 u32_42;
+     IMG_UINT32 u32_43;
+     IMG_UINT32 u32_44;
+     IMG_UINT32 u32_45;
+} RGX_PM_RENDERSTATE_BUFFER;
+
+/*
+MMU catalogue base address for VCE pipe 3 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_WOFF (45U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_,_x_) ((_ft_).u32_45 = (((_ft_).u32_45 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE(_ft_) (((_ft_).u32_45  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 3 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_WOFF (44U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_,_x_) ((_ft_).u32_44 = (((_ft_).u32_44 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_MAPPED(_ft_) (((_ft_).u32_44  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 3 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_WOFF (44U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_,_x_) ((_ft_).u32_44 = (((_ft_).u32_44 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE(_ft_) (((_ft_).u32_44  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE3_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_WOFF (43U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_,_x_) ((_ft_).u32_43 = (((_ft_).u32_43 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE(_ft_) (((_ft_).u32_43  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 2 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_WOFF (42U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_,_x_) ((_ft_).u32_42 = (((_ft_).u32_42 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_MAPPED(_ft_) (((_ft_).u32_42  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 2 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_WOFF (42U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_,_x_) ((_ft_).u32_42 = (((_ft_).u32_42 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE(_ft_) (((_ft_).u32_42  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE2_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_WOFF (41U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_,_x_) ((_ft_).u32_41 = (((_ft_).u32_41 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE(_ft_) (((_ft_).u32_41  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 1 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_WOFF (40U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_MAPPED(_ft_) (((_ft_).u32_40  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 1 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_WOFF (40U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_,_x_) ((_ft_).u32_40 = (((_ft_).u32_40 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE(_ft_) (((_ft_).u32_40  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE1_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 3 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_WOFF (37U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE(_ft_,_x_) ((_ft_).u32_37 = (((_ft_).u32_37 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE(_ft_) (((_ft_).u32_37  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 3 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE3_MAPPED(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE3_MAPPED(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 3 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_WOFF (36U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE(_ft_,_x_) ((_ft_).u32_36 = (((_ft_).u32_36 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE(_ft_) (((_ft_).u32_36  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE3_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 2 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_WOFF (35U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE(_ft_,_x_) ((_ft_).u32_35 = (((_ft_).u32_35 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE(_ft_) (((_ft_).u32_35  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 2 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE2_MAPPED(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE2_MAPPED(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 2 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_WOFF (34U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE(_ft_,_x_) ((_ft_).u32_34 = (((_ft_).u32_34 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE(_ft_) (((_ft_).u32_34  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE2_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 1 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_WOFF (33U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE(_ft_,_x_) ((_ft_).u32_33 = (((_ft_).u32_33 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE(_ft_) (((_ft_).u32_33  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 1 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE1_MAPPED(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE1_MAPPED(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 1 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_WOFF (32U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE(_ft_,_x_) ((_ft_).u32_32 = (((_ft_).u32_32 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE(_ft_) (((_ft_).u32_32  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE1_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_WOFF (30U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_30 = (((_ft_).u32_30 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_30  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for VCE pipe 0 ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_WOFF (29U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_29 = (((_ft_).u32_29 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_ADDR(_ft_) (((_ft_).u32_29  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for VCE pipe 0 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_MAPPED(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for VCE pipe 0 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_WOFF (28U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_28 = (((_ft_).u32_28 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_28  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_VCE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 0 LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_WOFF (26U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_,_x_) ((_ft_).u32_26 = (((_ft_).u32_26 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE(_ft_) (((_ft_).u32_26  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for TE  pipe 0 ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_WOFF (25U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_,_x_) ((_ft_).u32_25 = (((_ft_).u32_25 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_ADDR(_ft_) (((_ft_).u32_25  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for TE  pipe 0 MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_MAPPED(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for TE  pipe 0 INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_WOFF (24U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_,_x_) ((_ft_).u32_24 = (((_ft_).u32_24 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE(_ft_) (((_ft_).u32_24  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_TE_PIPE0_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST LAST_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_WOFF (18U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_,_x_) ((_ft_).u32_18 = (((_ft_).u32_18 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_LAST_PAGE(_ft_) (((_ft_).u32_18  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_LAST_PAGE_SHIFT)) & 0x000fffffU)
+/*
+MMU catalogue base address for ALIST ADDR
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_WOFF (17U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK (0xF0000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_ADDR(_ft_,_x_) ((_ft_).u32_17 = (((_ft_).u32_17 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_CLRMSK ) | (((_x_) & (0x0fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_ADDR(_ft_) (((_ft_).u32_17  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_ADDR_SHIFT)) & 0x0fffffffU)
+/*
+MMU catalogue base address for ALIST MAPPED
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_MAPPED(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_MAPPED(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_MAPPED_SHIFT)) & 0x00000001U)
+/*
+MMU catalogue base address for ALIST INIT_PAGE
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_WOFF (16U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK (0xFFF00000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_,_x_) ((_ft_).u32_16 = (((_ft_).u32_16 & RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_CLRMSK ) | (((_x_) & (0x000fffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MMU_CAT_BASE_ALIST_INIT_PAGE(_ft_) (((_ft_).u32_16  >>  (RGX_PM_RENDERSTATE_BUFFER_MMU_CAT_BASE_ALIST_INIT_PAGE_SHIFT)) & 0x000fffffU)
+/*
+Init Bit Sent Flag for TE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_WOFF      (15U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE3_INIT(_ft_,_x_) ((_ft_).u32_15 = (((_ft_).u32_15 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE3_INIT(_ft_) (((_ft_).u32_15  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_WOFF           (15U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE3(_ft_,_x_)  ((_ft_).u32_15 = (((_ft_).u32_15 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE3(_ft_)      (((_ft_).u32_15  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE3_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_WOFF     (14U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3_INIT(_ft_,_x_) ((_ft_).u32_14 = (((_ft_).u32_14 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3_INIT(_ft_) (((_ft_).u32_14  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE3
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_WOFF          (14U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE3(_ft_,_x_) ((_ft_).u32_14 = (((_ft_).u32_14 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE3(_ft_)     (((_ft_).u32_14  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE3_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for TE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_WOFF      (13U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE2_INIT(_ft_,_x_) ((_ft_).u32_13 = (((_ft_).u32_13 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE2_INIT(_ft_) (((_ft_).u32_13  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_WOFF           (13U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE2(_ft_,_x_)  ((_ft_).u32_13 = (((_ft_).u32_13 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE2(_ft_)      (((_ft_).u32_13  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE2_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_WOFF     (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2_INIT(_ft_,_x_) ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2_INIT(_ft_) (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE2
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_WOFF          (12U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE2(_ft_,_x_) ((_ft_).u32_12 = (((_ft_).u32_12 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE2(_ft_)     (((_ft_).u32_12  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE2_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for TE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_WOFF      (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE1_INIT(_ft_,_x_) ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE1_INIT(_ft_) (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_WOFF           (11U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE1(_ft_,_x_)  ((_ft_).u32_11 = (((_ft_).u32_11 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE1(_ft_)      (((_ft_).u32_11  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE1_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_WOFF     (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1_INIT(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1_INIT(_ft_) (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE1
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_WOFF          (10U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE1(_ft_,_x_) ((_ft_).u32_10 = (((_ft_).u32_10 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE1(_ft_)     (((_ft_).u32_10  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE1_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_WOFF      (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT     (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK    (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0_INIT(_ft_,_x_) ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0_INIT(_ft_) (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for TE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_WOFF           (9U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT          (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK         (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_TE0(_ft_,_x_)  ((_ft_).u32_9 = (((_ft_).u32_9 & RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_TE0(_ft_)      (((_ft_).u32_9  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_TE0_SHIFT)) & 0x7fffffffU)
+/*
+Init Bit Sent Flag for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_WOFF     (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT    (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK   (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0_INIT(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0_INIT(_ft_) (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_INIT_SHIFT)) & 0x00000001U)
+/*
+16KB aligned  virtual top pointer for VCE0
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_WOFF          (8U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT         (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK        (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_VPTR_VCE0(_ft_,_x_) ((_ft_).u32_8 = (((_ft_).u32_8 & RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_CLRMSK ) | (((_x_) & (0x7fffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_VPTR_VCE0(_ft_)     (((_ft_).u32_8  >>  (RGX_PM_RENDERSTATE_BUFFER_VPTR_VCE0_SHIFT)) & 0x7fffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the ISP.
+A '1' in a bit position indicates that the ISP has signalled that the corresponding macrotile can be freed.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_WOFF   (7U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT  (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MTILEFREE_STATUS(_ft_,_x_) ((_ft_).u32_7 = (((_ft_).u32_7 & RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MTILEFREE_STATUS(_ft_) (((_ft_).u32_7  >>  (RGX_PM_RENDERSTATE_BUFFER_MTILEFREE_STATUS_SHIFT)) & 0xffffffffU)
+/*
+A 16-bit macrotile mask indicating which macrotiles have been freed by the PM.
+A '1' in a bit position indicates that the corresponding macrotile has been freed, and its pages released back to the appropriate free stack.
+Only the least-significant 16 bits are valid.
+Only used in the 3D phase.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_WOFF (6U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_DEALLOC_MASK_STATUS(_ft_,_x_) ((_ft_).u32_6 = (((_ft_).u32_6 & RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_DEALLOC_MASK_STATUS(_ft_) (((_ft_).u32_6  >>  (RGX_PM_RENDERSTATE_BUFFER_DEALLOC_MASK_STATUS_SHIFT)) & 0xffffffffU)
+/*
+Reserved bits, un-used.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_WOFF        (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT       (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK      (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_RSV_STUFF32(_ft_,_x_) ((_ft_).u32_5 = (((_ft_).u32_5 & RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_RSV_STUFF32(_ft_)   (((_ft_).u32_5  >>  (RGX_PM_RENDERSTATE_BUFFER_RSV_STUFF32_SHIFT)) & 0xffffffffU)
+/*
+The number of entries on the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_WOFF         (4U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT        (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK       (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_TAIL(_ft_,_x_) ((_ft_).u32_4 = (((_ft_).u32_4 & RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_CLRMSK ) | (((_x_) & (0xffffffffU))  <<  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_TAIL(_ft_)    (((_ft_).u32_4  >>  (RGX_PM_RENDERSTATE_BUFFER_MLIST_TAIL_SHIFT)) & 0xffffffffU)
+/*
+The base address of the MLIST.
+Must be initialised to point to a block of memory where the PM can write the MLIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_WOFF (2U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_WOFF (3U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_SHIFT (5U)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK (0x0000001FU)
+#define RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(_ft_,_x_) { ((_ft_).u32_2 = (((_ft_).u32_2 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x0000000007ffffff)))  <<  5))); \
+                                                          ((_ft_).u32_3 = (((_ft_).u32_3 & RGX_PM_RENDERSTATE_BUFFER_MLIST_BASE_ADDR_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x07fffffff8000000)))  >>  27))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_MLIST_BASE_ADDR(_ft_) (((_ft_).u32_2  >>  (5)) | ((IMG_UINT64)((_ft_).u32_3 & 0xffffffffU ) << (27)))
+/*
+Init bit sent flag for ALIST
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_WOFF         (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT        (31U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK       (0x7FFFFFFFU)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_INIT(_ft_,_x_) ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_CLRMSK ) | (((_x_) & (0x00000001U))  <<  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT))))
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_INIT(_ft_)    (((_ft_).u32_1  >>  (RGX_PM_RENDERSTATE_BUFFER_ALIST_INIT_SHIFT)) & 0x00000001U)
+/*
+The number of entries on the ALIST.
+*/
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_WOFF      (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_WOFF      (1U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_SHIFT     (0U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK    (0x00000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK    (0x80000000U)
+#define RGX_PM_RENDERSTATE_BUFFER_SET_ALIST_TAIL(_ft_,_x_) { ((_ft_).u32_0 = (((_ft_).u32_0 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W0_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x00000000ffffffff)))  <<  0))); \
+                                                          ((_ft_).u32_1 = (((_ft_).u32_1 & RGX_PM_RENDERSTATE_BUFFER_ALIST_TAIL_W1_CLRMSK ) |  (((_x_) & (IMG_UINT64_C(0x7fffffff00000000)))  >>  32))); }
+#define RGX_PM_RENDERSTATE_BUFFER_GET_ALIST_TAIL(_ft_)    (((_ft_).u32_0  >>  (0)) | ((IMG_UINT64)((_ft_).u32_1 & 0x7fffffffU ) << (32)))
+#endif /* !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_PM_REGISTER_CONFIG_MODE) && !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) && !defined(RGX_FEATURE_SINGLE_TE_VSPACE)) && !(defined(RGX_FEATURE_SINGLE_TE_VSPACE)) */
+
+
+/*
+Maximum range supported by hardware is 33 bits.
+*/
+#define RGX_PM_RENDERSTATE_ALIST_TAIL_ALIST_TAIL          (0U)
+#define RGX_PM_RENDERSTATE_ALIST_TAIL_ALIST_TAIL_LOWER    (0U)
+#define RGX_PM_RENDERSTATE_ALIST_TAIL_ALIST_TAIL_UPPER    (8589934591ULL)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+256-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSHIFT     (5U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSIZE      (32U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSHIFT     (4U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_ALIGNSIZE      (16U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_MLIST_BASE_ADDR_MLIST_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+/*
+Maximum range supported by hardware is 33 bits.
+*/
+#define RGX_PM_RENDERSTATE_MLIST_TAIL_MLIST_TAIL          (0U)
+#define RGX_PM_RENDERSTATE_MLIST_TAIL_MLIST_TAIL_LOWER    (0U)
+#define RGX_PM_RENDERSTATE_MLIST_TAIL_MLIST_TAIL_UPPER    (8589934591ULL)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSHIFT     (4U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSIZE      (16U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSHIFT     (4U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_ALIGNSIZE      (16U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VHEAP_BASE_ADDR_VHEAP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+/*
+Only the 16 least-significant bits are used
+*/
+#define RGX_PM_RENDERSTATE_DEALLOC_MASK_STATUS_DEALLOC_MASK_STATUS (0U)
+#define RGX_PM_RENDERSTATE_DEALLOC_MASK_STATUS_DEALLOC_MASK_STATUS_LOWER (0U)
+#define RGX_PM_RENDERSTATE_DEALLOC_MASK_STATUS_DEALLOC_MASK_STATUS_UPPER (65535U)
+
+
+/*
+Only the 16 least-significant bits are used
+*/
+#define RGX_PM_RENDERSTATE_MTILEFREE_STATUS_MTILEFREE_STATUS (0U)
+#define RGX_PM_RENDERSTATE_MTILEFREE_STATUS_MTILEFREE_STATUS_LOWER (0U)
+#define RGX_PM_RENDERSTATE_MTILEFREE_STATUS_MTILEFREE_STATUS_UPPER (65535U)
+
+
+#if defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit granular, lower bits ignored.
+Maximum addressable range supported by hardware is 1 TB.
+*/
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSHIFT       (4U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSIZE        (16U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR    (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#if !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES)
+/*
+128-bit aligned.
+Maximum addressable range supported by hardware is 1 TB.
+The 40-bit, 16-byte-aligned address is packed into bits 35:0 of the two DWORDs.
+*/
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSHIFT       (4U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_ALIGNSIZE        (16U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR    (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_LOWER (0U)
+#define RGX_PM_RENDERSTATE_VFP_BASE_ADDR_VFP_BASE_ADDR_UPPER (68719476735ULL)
+#endif /* !defined(RGX_FEATURE_PM_BYTE_ALIGNED_BASE_ADDRESSES) */
+
+
+#endif /* RGXPMDEFS_H */
+/*****************************************************************************
+ End of file (rgxpmdefs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/cache_ops.h b/drivers/gpu/drm/img/img-volcanic/include/cache_ops.h
new file mode 100644
index 000000000000..a1d714519d59
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/cache_ops.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services cache management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines for cache management which are visible internally
+                and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef CACHE_OPS_H
+#define CACHE_OPS_H
+#include "img_types.h"
+/*!
+* @Defgroup CPUCacheAPIs
+* @{
+*/
+#define CACHE_BATCH_MAX (8U)
+#define MAX_DMA_OPS (34)
+typedef IMG_UINT32 PVRSRV_CACHE_OP;				/*!< Type represents cache maintenance operation */
+#define PVRSRV_CACHE_OP_NONE				0x0U	/*!< No operation */
+#define PVRSRV_CACHE_OP_CLEAN				0x1U	/*!< Flush w/o invalidate */
+#define PVRSRV_CACHE_OP_INVALIDATE			0x2U	/*!< Invalidate w/o flush */
+#define PVRSRV_CACHE_OP_FLUSH				0x3U	/*!< Flush w/ invalidate */
+/*! @} End of Defgroup CPUCacheAPIs */
+
+#endif	/* CACHE_OPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/dc_external.h b/drivers/gpu/drm/img/img-volcanic/include/dc_external.h
new file mode 100644
index 000000000000..2773ba0a833b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/dc_external.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          Display class external
+@Description    Defines DC specific structures which are externally visible
+                (i.e. visible to clients of services), but are also required
+                within services.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DC_EXTERNAL_H
+#define DC_EXTERNAL_H
+
+#include "img_types.h"
+
+/*!
+ * Maximum size of the display name in DC_DISPLAY_INFO
+ */
+#define DC_NAME_SIZE	50
+
+/*!
+ * Maximum planes supported by DC interface based display drivers.
+ */
+#define DC_MAX_PLANES		(4)
+/*!
+ * This contains information about a display.
+ * The structure can be queried by services from the display driver via a
+ * registered callback.
+ *
+ *   Structure: #DC_DISPLAY_INFO_TAG
+ *   Typedef: ::DC_DISPLAY_INFO
+ */
+typedef struct DC_DISPLAY_INFO_TAG
+{
+	IMG_CHAR		szDisplayName[DC_NAME_SIZE];	/*!< Display identifier string */
+	IMG_UINT32		ui32MinDisplayPeriod;			/*!< Minimum number of VSync periods */
+	IMG_UINT32		ui32MaxDisplayPeriod;			/*!< Maximum number of VSync periods */
+	IMG_UINT32		ui32MaxPipes;					/*!< Maximum number of pipes for this display */
+	IMG_BOOL		bUnlatchedSupported;			/*!< Can the device be unlatched? */
+} DC_DISPLAY_INFO;
+
+/*!
+ * When services imports a buffer from the display driver it has to fill
+ * this structure to inform services about the buffer properties.
+ *
+ *   Structure: #DC_BUFFER_IMPORT_INFO_TAG
+ *   Typedef: ::DC_BUFFER_IMPORT_INFO
+ */
+typedef struct DC_BUFFER_IMPORT_INFO_TAG
+{
+	IMG_UINT32		ePixFormat;			/*!< Enum value of type IMG_PIXFMT for the pixel format */
+	IMG_UINT32		ui32BPP;			/*!< Bits per pixel */
+	IMG_UINT32		ui32Width[3];		/*!< Width of the different channels (defined by ePixFormat) */
+	IMG_UINT32		ui32Height[3];		/*!< Height of the different channels (defined by ePixFormat) */
+	IMG_UINT32		ui32ByteStride[3];	/*!< Byte stride of the different channels (defined by ePixFormat) */
+	IMG_UINT32		ui32PrivData[3];	/*!< Private data of the display for each of the channels */
+} DC_BUFFER_IMPORT_INFO;
+
+/* DC_BUFFER_IMPORT_INFO is passed over the bridge so we need to ensure
+ * it has proper size */
+static_assert((sizeof(DC_BUFFER_IMPORT_INFO) % 4U) == 0U, "invalid size of DC_BUFFER_IMPORT_INFO");
+
+/*!
+ * Configuration details of the frame buffer compression module
+ *
+ *   Structure: #DC_FBC_CREATE_INFO_TAG
+ *   Typedef: ::DC_FBC_CREATE_INFO
+ */
+typedef struct DC_FBC_CREATE_INFO_TAG
+{
+	IMG_UINT32		ui32FBCWidth;	/*!< Pixel width that the FBC module is working on */
+	IMG_UINT32		ui32FBCHeight;	/*!< Pixel height that the FBC module is working on */
+	IMG_UINT32		ui32FBCStride;	/*!< Pixel stride that the FBC module is working on */
+	IMG_UINT32		ui32Size;		/*!< Size of the buffer to create */
+} DC_FBC_CREATE_INFO;
+
+/*!
+ * DC buffer details like frame buffer compression and surface properties
+ *
+ *   Structure: #DC_BUFFER_CREATE_INFO_TAG
+ *   Typedef: ::DC_BUFFER_CREATE_INFO
+ */
+typedef struct DC_BUFFER_CREATE_INFO_TAG
+{
+	PVRSRV_SURFACE_INFO		sSurface;	/*!< Surface properties, specified by user */
+	IMG_UINT32				ui32BPP;	/*!< Bits per pixel */
+	DC_FBC_CREATE_INFO		sFBC;		/*!< Frame buffer compressed specific data */
+} DC_BUFFER_CREATE_INFO;
+
+/* DC_BUFFER_CREATE_INFO is passed over the bridge so we need to ensure
+ * it has proper size */
+static_assert((sizeof(DC_BUFFER_CREATE_INFO) % 4U) == 0U, "invalid size of DC_BUFFER_CREATE_INFO");
+
+#endif /* DC_EXTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/devicemem_typedefs.h b/drivers/gpu/drm/img/img-volcanic/include/devicemem_typedefs.h
new file mode 100644
index 000000000000..d51c099eed7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/devicemem_typedefs.h
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Client side part of device memory management -- this file
+                is forked from new_devmem_allocation.h as this one has to
+                reside in the top level include so that client code is able
+                to make use of the typedefs.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_TYPEDEFS_H
+#define DEVICEMEM_TYPEDEFS_H
+
+#include <powervr/mem_types.h>
+#include "img_types.h"
+#include "pvrsrv_memallocflags.h"
+
+typedef struct DEVMEM_CONTEXT_TAG DEVMEM_CONTEXT;     /*!< Convenience typedef for struct DEVMEM_CONTEXT_TAG */
+typedef struct DEVMEM_HEAP_TAG DEVMEM_HEAP;           /*!< Convenience typedef for struct DEVMEM_HEAP_TAG */
+typedef struct DEVMEM_MEMDESC_TAG DEVMEM_MEMDESC;     /*!< Convenience typedef for struct DEVMEM_MEMDESC_TAG */
+typedef struct DEVMEM_PAGELIST_TAG DEVMEM_PAGELIST;   /*!< Convenience typedef for struct DEVMEM_PAGELIST_TAG */
+
+typedef IMG_HANDLE DEVMEM_EXPORTHANDLE;             /*!< Typedef for DeviceMem Export Handle */
+typedef IMG_UINT64 DEVMEM_EXPORTKEY;                /*!< Typedef for DeviceMem Export Key */
+typedef IMG_DEVMEM_SIZE_T DEVMEM_SIZE_T;            /*!< Typedef for DeviceMem SIZE_T */
+typedef IMG_DEVMEM_LOG2ALIGN_T DEVMEM_LOG2ALIGN_T;  /*!< Typedef for DeviceMem LOG2 Alignment */
+
+typedef struct DEVMEMX_PHYS_MEMDESC_TAG DEVMEMX_PHYSDESC;    /*!< Convenience typedef for DevmemX physical */
+typedef struct DEVMEMX_VIRT_MEMDESC_TAG DEVMEMX_VIRTDESC;    /*!< Convenience typedef for DevmemX virtual */
+
+/*! calling code needs all the info in this struct, to be able to pass it around */
+typedef struct
+{
+    /*! A handle to the PMR. */
+    IMG_HANDLE hPMRExportHandle;
+    /*! The "key" to prove we have authorisation to use this PMR */
+    IMG_UINT64 uiPMRExportPassword;
+    /*! Size and alignment properties for this PMR.  Note, these
+       numbers are not trusted in kernel, but we need to cache them
+       client-side in order to allocate from the VM arena.  The kernel
+       will know the actual alignment and size of the PMR and thus
+       would prevent client code from breaching security here.  Ditto
+       for physmem granularity (aka page size) if this is different
+       from alignment */
+    IMG_DEVMEM_SIZE_T uiSize;
+    /*! We call this "contiguity guarantee" to be more precise than
+       calling it "alignment" or "page size", terms which may seem
+       similar but have different emphasis.  The number reported here
+       is the minimum contiguity guarantee from the creator of the
+       PMR.  Now, there is no requirement to allocate that coarsely
+       from the RA.  The alignment given to the RA simply needs to be
+       at least as coarse as the device page size for the heap we
+       ultimately intend to map into.  What is important is that the
+       device MMU data page size is not greater than the minimum
+       contiguity guarantee from the PMR.  This value is reported to
+       the client in order that it can choose to make early checks and
+       perhaps decide which heap (in a variable page size scenario) it
+       would be safe to map this PMR into.  For convenience, the
+       client may choose to use this argument as the alignment of the
+       virtual range he chooses to allocate, but this is _not_
+       necessary and in many cases would be able to get away with a
+       finer alignment, should the heap into which this PMR will be
+       mapped support it. */
+    IMG_DEVMEM_LOG2ALIGN_T uiLog2ContiguityGuarantee;
+} DEVMEM_EXPORTCOOKIE;
+
+/* Enum that describes the operation associated with changing sparse memory */
+typedef IMG_UINT32 SPARSE_MEM_RESIZE_FLAGS;
+#define SPARSE_RESIZE_NONE 0U
+
+	/* This should be set to indicate the change needs allocation */
+#define SPARSE_RESIZE_ALLOC 1U
+
+	/* This should be set to indicate the change needs free */
+#define SPARSE_RESIZE_FREE 2U
+
+#define SPARSE_RESIZE_BOTH (SPARSE_RESIZE_ALLOC | SPARSE_RESIZE_FREE)
+
+/* To use with DevmemSubAllocate() as the default factor if no over-allocation
+ * is desired.
+ */
+#define DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER (1U)
+
+/* Defines the max length for PMR, MemDesc, Device memory History and RI debug
+ * annotations stored in memory, including the null terminator.
+ */
+#define DEVMEM_ANNOTATION_MAX_LEN ((IMG_UINT32)PVR_ANNOTATION_MAX_LEN + 1U)
+
+
+/* Reserved VA space of a heap must always be multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+ * Granularity has been chosen to support the max possible practically used OS page size.
+ */
+#define DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY        0x10000 /* 64KB is MAX anticipated OS page size */
+
+#endif /* #ifndef DEVICEMEM_TYPEDEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/dllist.h b/drivers/gpu/drm/img/img-volcanic/include/dllist.h
new file mode 100644
index 000000000000..f59c26356a87
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/dllist.h
@@ -0,0 +1,423 @@
+/*************************************************************************/ /*!
+@File
+@Title          Double linked list header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Double linked list interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DLLIST_H
+#define DLLIST_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+/*!
+	Pointer to a linked list node
+*/
+typedef struct DLLIST_NODE_TAG	*PDLLIST_NODE;
+
+
+/*!
+	Node in a linked list.
+	A list is comprised of a single list head and 0 to n nodes.
+	The head and nodes are all represented by `DLLIST_NODE`'s.
+	The head node is a special sentinel and should not be associated with
+	any elements in the list.
+
+	For example, the following list of fruits has 3 elements:
+	`... <--> "banana" <--> "apple" <--> head <--> "orange" <--> ...`
+	Therefore, using dllist_foreach_*(head) will iterate just the 3 fruits.
+
+	This is important because a `dllist_is_empty` is when the number of
+	elements == 0 is equivalent to when the head points to itself.
+	Proper use of the dllist_* functions requires a head.
+
+	For example, the following list is improper:
+	`... <--> "banana" <--> "apple" <--> "orange" <--> ...`
+	as one element must be treated as the head, and therefore is ignored.
+*/
+/*
+ * Note: the following structure's size is architecture-dependent and clients
+ * may need to create a mirror of the structure definition if it needs to be
+ * used in a structure shared between host and device.
+ * Consider such clients if any changes are made to this structure.
+ */
+typedef struct DLLIST_NODE_TAG
+{
+	struct DLLIST_NODE_TAG	*psPrevNode;
+	struct DLLIST_NODE_TAG	*psNextNode;
+} DLLIST_NODE;
+
+
+/*!
+	Static initialiser
+*/
+#define DECLARE_DLLIST(n) \
+DLLIST_NODE (n) = {&(n), &(n)}
+
+/*************************************************************************/ /*!
+@Function       dllist_foreach_node
+
+@Description    Walk through all the nodes on the list.
+                Safe against removal of (node).
+
+@Input          list_head              List node to start the operation
+@Input          node                   Current list node
+@Input          next                   Node after the current one
+
+*/
+/*****************************************************************************/
+#define dllist_foreach_node(list_head, node, next)						\
+	for ((node) = (list_head)->psNextNode, (next) = (node)->psNextNode;		\
+		 (node) != (list_head);											\
+		 (node) = (next), (next) = (node)->psNextNode)
+
+#define dllist_foreach_node_backwards(list_head, node, prev)			\
+	for ((node) = (list_head)->psPrevNode, (prev) = (node)->psPrevNode;		\
+		 (node) != (list_head);											\
+		 (node) = (prev), (prev) = (node)->psPrevNode)
+
+
+/*************************************************************************/ /*!
+@Function       dllist_foreach
+
+@Description    Simplification of dllist_foreach_node.
+                Walk through all the nodes on the list.
+                Safe against removal of currently-iterated node.
+
+                Adds utility-macro dllist_cur() to typecast the current node.
+
+@Input          list_head              List node to start the operation
+
+*/
+/*****************************************************************************/
+#define dllist_foreach(list_head)	\
+	for (DLLIST_NODE *DllCurNode = (list_head).psNextNode, *DllNextNode = DllCurNode->psNextNode;		\
+		 DllCurNode != &(list_head);																\
+		 DllCurNode = DllNextNode, DllNextNode = DllCurNode->psNextNode)
+
+#define dllist_foreach_backwards(list_head)	\
+	for (DLLIST_NODE *DllCurNode = (list_head).psPrevNode, *DllPrevNode = DllCurNode->psPrevNode;		\
+		 DllCurNode != &(list_head);																\
+		 DllCurNode = DllPrevNode, DllPrevNode = DllCurNode->psPrevNode)
+
+#define dllist_cur(type, member)	IMG_CONTAINER_OF(DllCurNode, type, member)
+
+/*************************************************************************/ /*!
+@Function       dllist_init
+
+@Description    Initialize a new double linked list
+
+@Input          psListHead             List head Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_init(PDLLIST_NODE psListHead)
+{
+	psListHead->psPrevNode = psListHead;
+	psListHead->psNextNode = psListHead;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_is_empty
+
+@Description    Returns whether the list is empty
+
+@Input          psListHead             List head Node
+
+*/
+/*****************************************************************************/
+static INLINE
+bool dllist_is_empty(const DLLIST_NODE *const psListHead)
+{
+	return (psListHead->psPrevNode == psListHead);
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_add_to_head
+
+@Description    Add psNewNode to head of list psListHead
+
+@Input          psListHead             Head Node
+@Input          psNewNode              New Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_add_to_head(PDLLIST_NODE psListHead, PDLLIST_NODE psNewNode)
+{
+	PDLLIST_NODE psTmp;
+
+	psTmp = psListHead->psNextNode;
+
+	psListHead->psNextNode = psNewNode;
+	psNewNode->psNextNode = psTmp;
+
+	psTmp->psPrevNode = psNewNode;
+	psNewNode->psPrevNode = psListHead;
+}
+
+
+/*************************************************************************/ /*!
+@Function       dllist_add_to_tail
+
+@Description    Add psNewNode to tail of list psListHead
+
+@Input          psListHead             Head Node
+@Input          psNewNode              New Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_add_to_tail(PDLLIST_NODE psListHead, PDLLIST_NODE psNewNode)
+{
+	PDLLIST_NODE psTmp;
+
+	psTmp = psListHead->psPrevNode;
+
+	psListHead->psPrevNode = psNewNode;
+	psNewNode->psPrevNode = psTmp;
+
+	psTmp->psNextNode = psNewNode;
+	psNewNode->psNextNode = psListHead;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_node_is_in_list
+
+@Description    Returns true if psNode is in a list
+
+@Input          psNode                 List node
+
+*/
+/*****************************************************************************/
+static INLINE
+bool dllist_node_is_in_list(const DLLIST_NODE *const psNode)
+{
+	return (psNode->psNextNode != NULL);
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_get_next_node
+
+@Description    Returns the list node after psListHead or NULL psListHead is
+                the only element in the list.
+
+@Input          psListHead             List node to start the operation
+
+*/
+/*****************************************************************************/
+static INLINE
+PDLLIST_NODE dllist_get_next_node(PDLLIST_NODE psListHead)
+{
+	if (psListHead->psNextNode == psListHead)
+	{
+		return NULL;
+	}
+	else
+	{
+		return psListHead->psNextNode;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_get_prev_node
+
+@Description    Returns the list node preceding psListHead or NULL if
+                psListHead is the only element in the list.
+
+@Input          psListHead             List node to start the operation
+
+*/
+/*****************************************************************************/
+static INLINE
+PDLLIST_NODE dllist_get_prev_node(PDLLIST_NODE psListHead)
+{
+	if (psListHead->psPrevNode == psListHead)
+	{
+		return NULL;
+	}
+	else
+	{
+		return psListHead->psPrevNode;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_remove_node
+
+@Description    Removes psListNode from the list where it currently belongs
+
+@Input          psListNode             List node to be removed
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_remove_node(PDLLIST_NODE psListNode)
+{
+	psListNode->psNextNode->psPrevNode = psListNode->psPrevNode;
+	psListNode->psPrevNode->psNextNode = psListNode->psNextNode;
+
+	/* Clear the node to show it's not in a list */
+	psListNode->psPrevNode = NULL;
+	psListNode->psNextNode = NULL;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_replace_head
+
+@Description    Moves the list from psOldHead to psNewHead
+
+@Input          psOldHead              List node to be replaced. Will become a
+                                       head node of an empty list.
+@Input          psNewHead              List node to be inserted. Must be an
+                                       empty list head.
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_replace_head(PDLLIST_NODE psOldHead, PDLLIST_NODE psNewHead)
+{
+	if (dllist_is_empty(psOldHead))
+	{
+		psNewHead->psNextNode = psNewHead;
+		psNewHead->psPrevNode = psNewHead;
+	}
+	else
+	{
+		/* Change the neighbouring nodes */
+		psOldHead->psNextNode->psPrevNode = psNewHead;
+		psOldHead->psPrevNode->psNextNode = psNewHead;
+
+		/* Copy the old data to the new node */
+		psNewHead->psNextNode = psOldHead->psNextNode;
+		psNewHead->psPrevNode = psOldHead->psPrevNode;
+
+		/* Remove links to the previous list */
+		psOldHead->psNextNode = psOldHead;
+		psOldHead->psPrevNode = psOldHead;
+	}
+}
+
+/**************************************************************************/ /*!
+@Function       dllist_insert_list_at_head
+
+@Description    Inserts psInHead list into the head of the psOutHead list.
+                After this operation psOutHead will contain psInHead at the
+                head of the list and the remaining elements that were
+                already in psOutHead will be places after the psInList (so
+                at a tail of the original list).
+
+@Input          psOutHead       List node psInHead will be inserted to.
+@Input          psInHead        List node to be inserted to psOutHead.
+                                After this operation this becomes an empty list.
+*/ /***************************************************************************/
+static INLINE
+void dllist_insert_list_at_head(PDLLIST_NODE psOutHead, PDLLIST_NODE psInHead)
+{
+	PDLLIST_NODE psInHeadNextNode = psInHead->psNextNode;
+	PDLLIST_NODE psOutHeadNextNode = psOutHead->psNextNode;
+
+	if (!dllist_is_empty(psInHead))
+	{
+		psOutHead->psNextNode = psInHeadNextNode;
+		psInHeadNextNode->psPrevNode = psOutHead;
+
+		psInHead->psPrevNode->psNextNode = psOutHeadNextNode;
+		psOutHeadNextNode->psPrevNode = psInHead->psPrevNode;
+
+		dllist_init(psInHead);
+	}
+ }
+
+/*************************************************************************/ /*!
+@Description    Pointer to a dllist comparison callback function.
+@Input          psNode  Pointer to a node in a dllist.
+@Input          psNext  Pointer to psNode's next neighbour.
+*/ /**************************************************************************/
+typedef bool (*DLLIST_CMP_CB)(const DLLIST_NODE *psNode, const DLLIST_NODE *psNext);
+
+/*************************************************************************/ /*!
+@Function       dllist_sort
+
+@Description    Insert-sorts the List in place
+                The cmpr function passes the current and next node,
+                From which the user writes the function responsible
+                for choosing to swap order or not.
+                The function returns true if a swap is required
+
+@Input          psListHead              List Head to be sorted.
+
+@Input          cmpr                    Function pointer to use for sorting
+
+*/
+/*****************************************************************************/
+static INLINE void dllist_sort(PDLLIST_NODE psListHead,
+                        DLLIST_CMP_CB cmpr)
+{
+	DLLIST_NODE *node, *next;
+	DLLIST_NODE sTempHead;
+
+	dllist_init(&sTempHead);
+
+	dllist_foreach_node(psListHead, node, next)
+	{
+		dllist_remove_node(node);
+		dllist_add_to_head(&sTempHead, node);
+	}
+
+	while (!dllist_is_empty(&sTempHead))
+	{
+		DLLIST_NODE *psSmallestNode = NULL;
+
+		dllist_foreach_node(&sTempHead, node, next)
+		{
+			if (!psSmallestNode || cmpr(psSmallestNode, node))
+			{
+				psSmallestNode = node;
+			}
+		}
+
+		dllist_remove_node(psSmallestNode);
+		dllist_add_to_tail(psListHead, psSmallestNode);
+	}
+}
+
+#endif /* DLLIST_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/netlink.h b/drivers/gpu/drm/img/img-volcanic/include/drm/netlink.h
new file mode 100644
index 000000000000..7b0a71fbaab3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/netlink.h
@@ -0,0 +1,147 @@
+/*
+ * @File
+ * @Title       Nulldisp/Netlink interface definition
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+
+#ifndef __NETLINK_H__
+#define __NETLINK_H__
+
+/* For multi-plane pixel formats */
+#define NLPVRDPY_MAX_NUM_PLANES 3
+
+enum nlpvrdpy_cmd {
+	__NLPVRDPY_CMD_INVALID,
+	NLPVRDPY_CMD_CONNECT,
+	NLPVRDPY_CMD_CONNECTED,
+	NLPVRDPY_CMD_DISCONNECT,
+	NLPVRDPY_CMD_FLIP,
+	NLPVRDPY_CMD_FLIPPED,
+	NLPVRDPY_CMD_COPY,
+	NLPVRDPY_CMD_COPIED,
+	__NLPVRDPY_CMD_MAX
+};
+#define NLPVRDPY_CMD_MAX (__NLPVRDPY_CMD_MAX - 1)
+
+enum nlpvrdpy_attr {
+	__NLPVRDPY_ATTR_INVALID,
+	NLPVRDPY_ATTR_NAME,
+	NLPVRDPY_ATTR_MINOR,
+	NLPVRDPY_ATTR_NUM_PLANES,
+	NLPVRDPY_ATTR_WIDTH,
+	NLPVRDPY_ATTR_HEIGHT,
+	NLPVRDPY_ATTR_PIXFMT,
+	NLPVRDPY_ATTR_YUV_CSC,
+	NLPVRDPY_ATTR_YUV_BPP,
+	NLPVRDPY_ATTR_PLANE0_ADDR,
+	NLPVRDPY_ATTR_PLANE0_SIZE,
+	NLPVRDPY_ATTR_PLANE0_OFFSET,
+	NLPVRDPY_ATTR_PLANE0_PITCH,
+	NLPVRDPY_ATTR_PLANE0_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_PLANE1_ADDR,
+	NLPVRDPY_ATTR_PLANE1_SIZE,
+	NLPVRDPY_ATTR_PLANE1_OFFSET,
+	NLPVRDPY_ATTR_PLANE1_PITCH,
+	NLPVRDPY_ATTR_PLANE1_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_PLANE2_ADDR,
+	NLPVRDPY_ATTR_PLANE2_SIZE,
+	NLPVRDPY_ATTR_PLANE2_OFFSET,
+	NLPVRDPY_ATTR_PLANE2_PITCH,
+	NLPVRDPY_ATTR_PLANE2_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_FB_MODIFIER,
+	NLPVRDPY_ATTR_NAMING_REQUIRED,
+	NLPVRDPY_ATTR_PAD,
+	__NLPVRDPY_ATTR_MAX
+};
+#define NLPVRDPY_ATTR_MAX  (__NLPVRDPY_ATTR_MAX - 1)
+
+static struct nla_policy __attribute__((unused))
+nlpvrdpy_policy[NLPVRDPY_ATTR_MAX + 1] = {
+	[NLPVRDPY_ATTR_NAME]                = { .type = NLA_STRING },
+	[NLPVRDPY_ATTR_MINOR]               = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_NUM_PLANES]          = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_WIDTH]               = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_HEIGHT]              = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PIXFMT]              = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_YUV_CSC]             = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_YUV_BPP]             = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_PLANE0_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PLANE1_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PLANE2_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_FB_MODIFIER]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_NAMING_REQUIRED]     = { .type = NLA_FLAG },
+};
+
+#define NLPVRDPY_ATTR_PLANE(index, type)				\
+	({								\
+		enum nlpvrdpy_attr __retval;				\
+									\
+		switch (index) {					\
+		case 0:							\
+			__retval = NLPVRDPY_ATTR_PLANE0_ ## type;	\
+			break;						\
+		case 1:							\
+			__retval = NLPVRDPY_ATTR_PLANE1_ ## type;	\
+			break;						\
+		case 2:							\
+			__retval = NLPVRDPY_ATTR_PLANE2_ ## type;	\
+			break;						\
+		default:						\
+			__retval = __NLPVRDPY_ATTR_INVALID;		\
+			break;						\
+		};							\
+									\
+		__retval;						\
+	})
+
+#endif /* __NETLINK_H__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/nulldisp_drm.h b/drivers/gpu/drm/img/img-volcanic/include/drm/nulldisp_drm.h
new file mode 100644
index 000000000000..8403fb56f1a5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/nulldisp_drm.h
@@ -0,0 +1,105 @@
+/*
+ * @File
+ * @Title       Nulldisp DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__NULLDISP_DRM_H__)
+#define __NULLDISP_DRM_H__
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+struct drm_nulldisp_gem_create {
+	__u64 size;   /* in */
+	__u32 flags;  /* in */
+	__u32 handle; /* out */
+};
+
+struct drm_nulldisp_gem_mmap {
+	__u32 handle; /* in */
+	__u32 pad;
+	__u64 offset; /* out */
+};
+
+#define NULLDISP_GEM_CPU_PREP_READ   (1 << 0)
+#define NULLDISP_GEM_CPU_PREP_WRITE  (1 << 1)
+#define NULLDISP_GEM_CPU_PREP_NOWAIT (1 << 2)
+
+struct drm_nulldisp_gem_cpu_prep {
+	__u32 handle; /* in */
+	__u32 flags;  /* in */
+};
+
+struct drm_nulldisp_gem_cpu_fini {
+	__u32 handle; /* in */
+	__u32 pad;
+};
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+#define DRM_NULLDISP_GEM_CREATE   0x00
+#define DRM_NULLDISP_GEM_MMAP     0x01
+#define DRM_NULLDISP_GEM_CPU_PREP 0x02
+#define DRM_NULLDISP_GEM_CPU_FINI 0x03
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define DRM_IOCTL_NULLDISP_GEM_CREATE \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CREATE, \
+		 struct drm_nulldisp_gem_create)
+
+#define DRM_IOCTL_NULLDISP_GEM_MMAP \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_MMAP, \
+		 struct drm_nulldisp_gem_mmap)
+
+#define DRM_IOCTL_NULLDISP_GEM_CPU_PREP \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CPU_PREP, \
+		struct drm_nulldisp_gem_cpu_prep)
+
+#define DRM_IOCTL_NULLDISP_GEM_CPU_FINI \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CPU_FINI, \
+		struct drm_nulldisp_gem_cpu_fini)
+
+#endif /* defined(__NULLDISP_DRM_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/pdp_drm.h b/drivers/gpu/drm/img/img-volcanic/include/drm/pdp_drm.h
new file mode 100644
index 000000000000..f5d747d3ad10
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/pdp_drm.h
@@ -0,0 +1,105 @@
+/*
+ * @File
+ * @Title       PDP DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_DRM_H__)
+#define __PDP_DRM_H__
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+struct drm_pdp_gem_create {
+	__u64 size;	/* in */
+	__u32 flags;	/* in */
+	__u32 handle;	/* out */
+};
+
+struct drm_pdp_gem_mmap {
+	__u32 handle;	/* in */
+	__u32 pad;
+	__u64 offset;	/* out */
+};
+
+#define PDP_GEM_CPU_PREP_READ	(1 << 0)
+#define PDP_GEM_CPU_PREP_WRITE	(1 << 1)
+#define PDP_GEM_CPU_PREP_NOWAIT	(1 << 2)
+
+struct drm_pdp_gem_cpu_prep {
+	__u32 handle;	/* in */
+	__u32 flags;	/* in */
+};
+
+struct drm_pdp_gem_cpu_fini {
+	__u32 handle;	/* in */
+	__u32 pad;
+};
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+#define DRM_PDP_GEM_CREATE		0x00
+#define DRM_PDP_GEM_MMAP		0x01
+#define DRM_PDP_GEM_CPU_PREP		0x02
+#define DRM_PDP_GEM_CPU_FINI		0x03
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define DRM_IOCTL_PDP_GEM_CREATE \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_CREATE, \
+		 struct drm_pdp_gem_create)
+
+#define DRM_IOCTL_PDP_GEM_MMAP\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_MMAP, \
+		 struct drm_pdp_gem_mmap)
+
+#define DRM_IOCTL_PDP_GEM_CPU_PREP \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_PREP, \
+		struct drm_pdp_gem_cpu_prep)
+
+#define DRM_IOCTL_PDP_GEM_CPU_FINI \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_FINI, \
+		struct drm_pdp_gem_cpu_fini)
+
+#endif /* defined(__PDP_DRM_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/drm/pvr_drm.h b/drivers/gpu/drm/img/img-volcanic/include/drm/pvr_drm.h
new file mode 100644
index 000000000000..bc76ddf9b597
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/drm/pvr_drm.h
@@ -0,0 +1,164 @@
+/*
+ * @File
+ * @Title       PVR DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_DRM_H__)
+#define __PVR_DRM_H__
+
+#include <linux/types.h>
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+/*
+ * IMPORTANT:
+ * All structures below are designed to be the same size when compiled for 32
+ * and/or 64 bit architectures, i.e. there should be no compiler inserted
+ * padding. This is achieved by sticking to the following rules:
+ * 1) only use fixed width types
+ * 2) always naturally align fields by arranging them appropriately and by using
+ *    padding fields when necessary
+ *
+ * These rules should _always_ be followed when modifying or adding new
+ * structures to this file.
+ */
+
+struct drm_pvr_srvkm_cmd {
+	__u32 bridge_id;
+	__u32 bridge_func_id;
+	__u64 in_data_ptr;
+	__u64 out_data_ptr;
+	__u32 in_data_size;
+	__u32 out_data_size;
+};
+
+struct pvr_sync_rename_ioctl_data {
+	char szName[32];
+};
+
+struct pvr_sw_sync_create_fence_data {
+	char name[32];
+	__s32 fence;
+	__u32 pad;
+	__u64 sync_pt_idx;
+};
+
+typedef struct pvr_sw_sync_create_fence_data pvr_exp_sync_create_fence_data_t;
+
+struct pvr_sw_timeline_advance_data {
+	__u64 sync_pt_idx;
+};
+
+#define PVR_SRVKM_SERVICES_INIT  1
+#define PVR_SRVKM_SYNC_INIT 2
+#define PVR_SRVKM_SYNC_EXP_FENCE_INIT 3
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#define PVR_SRVKM_SERVICES_PAGE_MIGRATE_INIT 4
+#endif
+struct drm_pvr_srvkm_init_data {
+	__u32 init_module;
+};
+
+/* Values used to configure the PVRSRV_DEVICE_INIT_MODE tunable (Linux-only) */
+#define PVRSRV_LINUX_DEV_INIT_ON_PROBE   1
+#define PVRSRV_LINUX_DEV_INIT_ON_OPEN    2
+#define PVRSRV_LINUX_DEV_INIT_ON_CONNECT 3
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+
+/* PVR Services command */
+#define DRM_PVR_SRVKM_CMD			0
+
+/* PVR Sync commands */
+#define DRM_PVR_SYNC_RENAME_CMD			1
+#define DRM_PVR_SYNC_FORCE_SW_ONLY_CMD		2
+
+/* PVR Software Sync commands */
+#define DRM_PVR_SW_SYNC_CREATE_FENCE_CMD	3
+#define DRM_PVR_SW_SYNC_INC_CMD			4
+
+/* PVR Services Render Device Init command */
+#define DRM_PVR_SRVKM_INIT		5
+
+/* PVR Services Export Fence Sync commands */
+#define DRM_PVR_EXP_FENCE_SYNC_FORCE_CMD	6
+#define DRM_PVR_SYNC_CREATE_EXPORT_FENCE_CMD	7
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define	DRM_IOCTL_PVR_SRVKM_CMD	\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SRVKM_CMD, \
+		 struct drm_pvr_srvkm_cmd)
+
+#define DRM_IOCTL_PVR_SYNC_RENAME_CMD \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PVR_SYNC_RENAME_CMD, \
+		struct pvr_sync_rename_ioctl_data)
+
+#define DRM_IOCTL_PVR_SYNC_FORCE_SW_ONLY_CMD \
+	DRM_IO(DRM_COMMAND_BASE + DRM_PVR_SYNC_FORCE_SW_ONLY_CMD)
+
+#define	DRM_IOCTL_PVR_SW_SYNC_CREATE_FENCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SW_SYNC_CREATE_FENCE_CMD, \
+		struct pvr_sw_sync_create_fence_data)
+
+#define DRM_IOCTL_PVR_SW_SYNC_INC_CMD \
+	DRM_IOR(DRM_COMMAND_BASE + DRM_PVR_SW_SYNC_INC_CMD, \
+		struct pvr_sw_timeline_advance_data)
+
+#define DRM_IOCTL_PVR_SRVKM_INIT \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PVR_SRVKM_INIT, \
+		struct drm_pvr_srvkm_init_data)
+
+#define DRM_IOCTL_PVR_EXP_FENCE_SYNC_FORCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_EXP_FENCE_SYNC_FORCE_CMD, \
+		pvr_exp_sync_create_fence_data_t)
+
+#define DRM_IOCTL_PVR_SYNC_CREATE_EXPORT_FENCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SYNC_CREATE_EXPORT_FENCE_CMD, \
+		pvr_exp_sync_create_fence_data_t)
+
+#endif /* defined(__PVR_DRM_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_3dtypes.h b/drivers/gpu/drm/img/img-volcanic/include/img_3dtypes.h
new file mode 100644
index 000000000000..102c6eeda959
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_3dtypes.h
@@ -0,0 +1,248 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global 3D types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines 3D types for use by IMG APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_3DTYPES_H
+#define IMG_3DTYPES_H
+
+#include <powervr/buffer_attribs.h>
+#include "img_types.h"
+#include "img_defs.h"
+
+/**
+ * Comparison functions
+ * This comparison function is defined as:
+ * A {CmpFunc} B
+ * A is a reference value, e.g., incoming depth etc.
+ * B is the sample value, e.g., value in depth buffer.
+ */
+typedef enum _IMG_COMPFUNC_
+{
+	IMG_COMPFUNC_NEVER,			/**< The comparison never succeeds */
+	IMG_COMPFUNC_LESS,			/**< The comparison is a less-than operation */
+	IMG_COMPFUNC_EQUAL,			/**< The comparison is an equal-to operation */
+	IMG_COMPFUNC_LESS_EQUAL,	/**< The comparison is a less-than or equal-to
+									 operation */
+	IMG_COMPFUNC_GREATER,		/**< The comparison is a greater-than operation
+								*/
+	IMG_COMPFUNC_NOT_EQUAL,		/**< The comparison is a no-equal-to operation
+								*/
+	IMG_COMPFUNC_GREATER_EQUAL,	/**< The comparison is a greater-than or
+									 equal-to operation */
+	IMG_COMPFUNC_ALWAYS,		/**< The comparison always succeeds */
+} IMG_COMPFUNC;
+
+/**
+ * Stencil op functions
+ */
+typedef enum _IMG_STENCILOP_
+{
+	IMG_STENCILOP_KEEP,		/**< Keep original value */
+	IMG_STENCILOP_ZERO,		/**< Set stencil to 0 */
+	IMG_STENCILOP_REPLACE,	/**< Replace stencil entry */
+	IMG_STENCILOP_INCR_SAT,	/**< Increment stencil entry, clamping to max */
+	IMG_STENCILOP_DECR_SAT,	/**< Decrement stencil entry, clamping to zero */
+	IMG_STENCILOP_INVERT,	/**< Invert bits in stencil entry */
+	IMG_STENCILOP_INCR,		/**< Increment stencil entry,
+								 wrapping if necessary */
+	IMG_STENCILOP_DECR,		/**< Decrement stencil entry,
+								 wrapping if necessary */
+} IMG_STENCILOP;
+
+/**
+ * Alpha blending allows colours and textures on one surface
+ * to be blended with transparency onto another surface.
+ * These definitions apply to both source and destination blending
+ * states
+ */
+typedef enum _IMG_BLEND_
+{
+	IMG_BLEND_ZERO = 0,        /**< Blend factor is (0,0,0,0) */
+	IMG_BLEND_ONE,             /**< Blend factor is (1,1,1,1) */
+	IMG_BLEND_SRC_COLOUR,      /**< Blend factor is the source colour */
+	IMG_BLEND_INV_SRC_COLOUR,  /**< Blend factor is the inverted source colour
+									(i.e. 1-src_col) */
+	IMG_BLEND_SRC_ALPHA,       /**< Blend factor is the source alpha */
+	IMG_BLEND_INV_SRC_ALPHA,   /**< Blend factor is the inverted source alpha
+									(i.e. 1-src_alpha) */
+	IMG_BLEND_DEST_ALPHA,      /**< Blend factor is the destination alpha */
+	IMG_BLEND_INV_DEST_ALPHA,  /**< Blend factor is the inverted destination
+									alpha */
+	IMG_BLEND_DEST_COLOUR,     /**< Blend factor is the destination colour */
+	IMG_BLEND_INV_DEST_COLOUR, /**< Blend factor is the inverted destination
+									colour */
+	IMG_BLEND_SRC_ALPHASAT,    /**< Blend factor is the alpha saturation (the
+									minimum of (Src alpha,
+									1 - destination alpha)) */
+	IMG_BLEND_BLEND_FACTOR,    /**< Blend factor is a constant */
+	IMG_BLEND_INVBLEND_FACTOR, /**< Blend factor is a constant (inverted)*/
+	IMG_BLEND_SRC1_COLOUR,     /**< Blend factor is the colour outputted from
+									the pixel shader */
+	IMG_BLEND_INV_SRC1_COLOUR, /**< Blend factor is the inverted colour
+									outputted from the pixel shader */
+	IMG_BLEND_SRC1_ALPHA,      /**< Blend factor is the alpha outputted from
+									the pixel shader */
+	IMG_BLEND_INV_SRC1_ALPHA   /**< Blend factor is the inverted alpha
+									outputted from the pixel shader */
+} IMG_BLEND;
+
+/**
+ * The arithmetic operation to perform when blending
+ */
+typedef enum _IMG_BLENDOP_
+{
+	IMG_BLENDOP_ADD = 0,      /**< Result = (Source + Destination) */
+	IMG_BLENDOP_SUBTRACT,     /**< Result = (Source - Destination) */
+	IMG_BLENDOP_REV_SUBTRACT, /**< Result = (Destination - Source) */
+	IMG_BLENDOP_MIN,          /**< Result = min (Source, Destination) */
+	IMG_BLENDOP_MAX           /**< Result = max (Source, Destination) */
+} IMG_BLENDOP;
+
+/**
+ * Logical operation to perform when logic ops are enabled
+ */
+typedef enum _IMG_LOGICOP_
+{
+	IMG_LOGICOP_CLEAR = 0,     /**< Result = 0 */
+	IMG_LOGICOP_SET,           /**< Result = -1 */
+	IMG_LOGICOP_COPY,          /**< Result = Source */
+	IMG_LOGICOP_COPY_INVERTED, /**< Result = ~Source */
+	IMG_LOGICOP_NOOP,          /**< Result = Destination */
+	IMG_LOGICOP_INVERT,        /**< Result = ~Destination */
+	IMG_LOGICOP_AND,           /**< Result = Source & Destination */
+	IMG_LOGICOP_NAND,          /**< Result = ~(Source & Destination) */
+	IMG_LOGICOP_OR,            /**< Result = Source | Destination */
+	IMG_LOGICOP_NOR,           /**< Result = ~(Source | Destination) */
+	IMG_LOGICOP_XOR,           /**< Result = Source ^ Destination */
+	IMG_LOGICOP_EQUIV,         /**< Result = ~(Source ^ Destination) */
+	IMG_LOGICOP_AND_REVERSE,   /**< Result = Source & ~Destination */
+	IMG_LOGICOP_AND_INVERTED,  /**< Result = ~Source & Destination */
+	IMG_LOGICOP_OR_REVERSE,    /**< Result = Source | ~Destination */
+	IMG_LOGICOP_OR_INVERTED    /**< Result = ~Source | Destination */
+} IMG_LOGICOP;
+
+/**
+ * Type of fog blending supported
+ */
+typedef enum _IMG_FOGMODE_
+{
+	IMG_FOGMODE_NONE, /**< No fog blending - fog calculations are
+					   *   based on the value output from the vertex phase */
+	IMG_FOGMODE_LINEAR, /**< Linear interpolation */
+	IMG_FOGMODE_EXP, /**< Exponential */
+	IMG_FOGMODE_EXP2, /**< Exponential squaring */
+} IMG_FOGMODE;
+
+/**
+ * Types of filtering
+ */
+typedef enum _IMG_FILTER_
+{
+	IMG_FILTER_DONTCARE,	/**< Any filtering mode is acceptable */
+	IMG_FILTER_POINT,		/**< Point filtering */
+	IMG_FILTER_LINEAR,		/**< Bi-linear filtering */
+	IMG_FILTER_BICUBIC,		/**< Bi-cubic filtering */
+} IMG_FILTER;
+
+/**
+ * Addressing modes for textures
+ */
+typedef enum _IMG_ADDRESSMODE_
+{
+	IMG_ADDRESSMODE_DONTCARE,
+	IMG_ADDRESSMODE_REPEAT,	/**< Texture repeats continuously */
+	IMG_ADDRESSMODE_FLIP, /**< Texture flips on odd integer part */
+	IMG_ADDRESSMODE_CLAMP, /**< Texture clamped at 0 or 1 */
+	IMG_ADDRESSMODE_FLIPCLAMP, /**< Flipped once, then clamp */
+	IMG_ADDRESSMODE_CLAMPBORDER,
+	IMG_ADDRESSMODE_OGL_CLAMP,
+	IMG_ADDRESSMODE_OVG_TILEFILL,
+} IMG_ADDRESSMODE;
+
+/**
+ * Culling based on winding order of triangle.
+ */
+typedef enum _IMG_CULLMODE_
+{
+	IMG_CULLMODE_NONE,			/**< Don't cull */
+	IMG_CULLMODE_FRONTFACING,	/**< Front facing triangles */
+	IMG_CULLMODE_BACKFACING,	/**< Back facing triangles */
+} IMG_CULLMODE;
+
+/**
+ * Colour for clearing surfaces.
+ *  The four elements of the 4 x 32 bit array will map to colour
+ *  R,G,B,A components, in order.
+ *  For YUV colour space the order is Y,U,V.
+ *  For Depth and Stencil formats D maps to R and S maps to G.
+ */
+typedef union IMG_CLEAR_COLOUR_TAG {
+	IMG_UINT32        aui32[4];
+	IMG_INT32         ai32[4];
+	IMG_FLOAT         af32[4];
+} IMG_CLEAR_COLOUR;
+
+static_assert(sizeof(IMG_FLOAT) == sizeof(IMG_INT32), "Size of IMG_FLOAT is not 32 bits.");
+
+/*! ************************************************************************//**
+@brief          Specifies the MSAA resolve operation.
+*/ /**************************************************************************/
+typedef enum _IMG_RESOLVE_OP_
+{
+	IMG_RESOLVE_BLEND   = 0,          /*!< box filter on the samples */
+	IMG_RESOLVE_MIN     = 1,          /*!< minimum of the samples */
+	IMG_RESOLVE_MAX     = 2,          /*!< maximum of the samples */
+	IMG_RESOLVE_SAMPLE0 = 3,          /*!< choose sample 0 */
+	IMG_RESOLVE_SAMPLE1 = 4,          /*!< choose sample 1 */
+	IMG_RESOLVE_SAMPLE2 = 5,          /*!< choose sample 2 */
+	IMG_RESOLVE_SAMPLE3 = 6,          /*!< choose sample 3 */
+	IMG_RESOLVE_SAMPLE4 = 7,          /*!< choose sample 4 */
+	IMG_RESOLVE_SAMPLE5 = 8,          /*!< choose sample 5 */
+	IMG_RESOLVE_SAMPLE6 = 9,          /*!< choose sample 6 */
+	IMG_RESOLVE_SAMPLE7 = 10,         /*!< choose sample 7 */
+} IMG_RESOLVE_OP;
+
+
+#endif /* IMG_3DTYPES_H */
+/******************************************************************************
+ End of file (img_3dtypes.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_defs.h b/drivers/gpu/drm/img/img-volcanic/include/img_defs.h
new file mode 100644
index 000000000000..4cb065edb788
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_defs.h
@@ -0,0 +1,629 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common header containing type definitions for portability
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains variable and structure definitions. Any platform
+                specific types should be defined in this file.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DEFS_H
+#define IMG_DEFS_H
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include <linux/types.h>
+#else
+#include <stddef.h>
+#endif
+#if !(defined(__linux__) && defined(__KERNEL__))
+#if defined(__riscv)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wundef"
+#endif
+#include <assert.h>
+#if defined(__riscv)
+#pragma GCC diagnostic pop
+#endif
+#endif
+
+#include "img_types.h"
+
+#if defined(NO_INLINE_FUNCS)
+	#define	INLINE
+	#define	FORCE_INLINE
+#else
+#if defined(__cplusplus) || defined(INTEGRITY_OS)
+	#if	!defined(INLINE)
+		#define INLINE				inline
+	#endif
+	#define	FORCE_INLINE			static inline
+#else
+#if	!defined(INLINE)
+	#define	INLINE					__inline
+#endif
+#if (defined(UNDER_WDDM) || defined(WINDOWS_WDF)) && defined(_X86_)
+	#define	FORCE_INLINE			__forceinline
+#else
+	#define	FORCE_INLINE			static __inline
+#endif
+#endif
+#endif
+
+/* True if the GCC version is at least the given version. False for older
+ * versions of GCC, or other compilers.
+ */
+#if defined(__GNUC__)
+#define GCC_VERSION_AT_LEAST(major, minor) \
+	(__GNUC__ > (major) || \
+	(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#define GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if defined(__clang__)
+#define CLANG_VERSION_AT_LEAST(major) \
+	(__clang_major__ >= (major))
+#else
+#define CLANG_VERSION_AT_LEAST(major) 0
+#endif
+
+/* Use Clang's __has_extension and __has_builtin macros if available. */
+#if defined(__has_extension)
+#define has_clang_extension(e) __has_extension(e)
+#else
+#define has_clang_extension(e) 0
+#endif
+
+#if defined(__has_builtin)
+#define has_clang_builtin(e) __has_builtin(e)
+#else
+#define has_clang_builtin(e) 0
+#endif
+
+/* Use this in any file, or use attributes under GCC - see below */
+#ifndef PVR_UNREFERENCED_PARAMETER
+#define	PVR_UNREFERENCED_PARAMETER(param) ((void)(param))
+#endif
+
+/* static_assert(condition, "message to print if it fails");
+ *
+ * Assert something at compile time. If the assertion fails, try to print
+ * the message, otherwise do nothing. static_assert is available if:
+ *
+ * - It's already defined as a macro (e.g. by <assert.h> in C11)
+ * - We're using MSVC which exposes static_assert unconditionally
+ * - We're using a C++ compiler that supports C++11
+ * - We're using GCC 4.6 and up in C mode (in which case it's available as
+ *   _Static_assert)
+ *
+ * In all other cases, fall back to an equivalent that makes an invalid
+ * declaration.
+ */
+#if !defined(static_assert) && !defined(_MSC_VER) && \
+		(!defined(__cplusplus) || __cplusplus < 201103L) || defined(__KLOCWORK__)
+	/* static_assert isn't already available */
+	#if !defined(__cplusplus) && (GCC_VERSION_AT_LEAST(4, 6) || \
+								  (defined(__clang__) && has_clang_extension(c_static_assert)))
+		#define static_assert _Static_assert
+	#else
+		#define static_assert(expr, message) \
+			extern int static_assert_failed[(expr) ? 1 : -1] __attribute__((unused))
+	#endif
+#endif
+
+/*
+ * unreachable("explanation") can be used to indicate to the compiler that
+ * some parts of the code can never be reached, like the default branch
+ * of a switch that covers all real-world possibilities, even though there
+ * are other ints that exist for instance.
+ *
+ * The message will be printed as an assert() when debugging.
+ *
+ * Note: there is no need to add a 'return' or any error handling after
+ * calling unreachable(), as this call will never return.
+ */
+#if defined(__linux__) && defined(__KERNEL__)
+/* Kernel has its own unreachable(), which is a simple infinite loop */
+#elif GCC_VERSION_AT_LEAST(4, 5) || has_clang_builtin(__builtin_unreachable)
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			__builtin_unreachable(); \
+		} while (false)
+#elif defined(_MSC_VER)
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			__assume(0); \
+		} while (false)
+#else
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			while (1); \
+		} while (false)
+#endif
+
+/*
+ * assume(x > 2 && x <= 7) works like an assert(), except it hints to the
+ * compiler what it can assume to optimise the code, like a limited range
+ * of parameter values.
+ */
+#if has_clang_builtin(__builtin_assume)
+	#define assume(expr) \
+		do { \
+			assert(expr); \
+			__builtin_assume(expr); \
+		} while (false)
+#elif defined(_MSC_VER)
+	#define assume(expr) \
+		do { \
+			assert(expr); \
+			__assume(expr); \
+		} while (false)
+#elif defined(__linux__) && defined(__KERNEL__)
+	#define assume(expr) ((void)(expr))
+#elif GCC_VERSION_AT_LEAST(4, 5) || has_clang_builtin(__builtin_unreachable)
+	#define assume(expr) \
+		do { \
+			if (unlikely(!(expr))) \
+				unreachable("Assumption isn't true: " # expr); \
+		} while (false)
+#else
+	#define assume(expr) assert(expr)
+#endif
+
+/*! Macro to calculate the n-byte aligned value from that supplied rounding up.
+ * n must be a power of two.
+ */
+#define PVR_ALIGN(_x, _n)	((((_x)+((_n)-1U))|((_n)-1U))^((_n)-1U))
+
+#if defined(_WIN32)
+
+#if defined(WINDOWS_WDF)
+
+	/*
+	 * For WINDOWS_WDF drivers we don't want these defines to overwrite calling conventions propagated through the build system.
+	 * This 'empty' choice helps to resolve all the calling conv issues.
+	 *
+	 */
+	#define IMG_CALLCONV
+	#define C_CALLCONV
+
+	#define IMG_INTERNAL
+	#define IMG_RESTRICT __restrict
+
+	/*
+	 * The proper way of dll linking under MS compilers is made of two things:
+	 * - decorate implementation with __declspec(dllexport)
+	 *   this decoration helps compiler with making the so called
+	 *   'export library'
+	 * - decorate forward-declaration (in a source dependent on a dll) with
+	 *   __declspec(dllimport), this decoration helps the compiler to make
+	 *   faster and smaller code in terms of calling dll-imported functions
+	 *
+	 * Usually these decorations are performed by having a single macro define
+	 * making that expands to a proper __declspec() depending on the
+	 * translation unit, dllexport inside the dll source and dllimport outside
+	 * the dll source. Having IMG_EXPORT and IMG_IMPORT resolving to the same
+	 * __declspec() makes no sense, but at least works.
+	 */
+	#define IMG_IMPORT __declspec(dllexport)
+	#define IMG_EXPORT __declspec(dllexport)
+
+#else
+
+	#define IMG_CALLCONV __stdcall
+	#define IMG_INTERNAL
+	#define	IMG_EXPORT	__declspec(dllexport)
+	#define IMG_RESTRICT __restrict
+	#define C_CALLCONV	__cdecl
+
+	/*
+	 * IMG_IMPORT is defined as IMG_EXPORT so that headers and implementations
+	 * match. Some compilers require the header to be declared IMPORT, while
+	 * the implementation is declared EXPORT.
+	 */
+	#define	IMG_IMPORT	IMG_EXPORT
+
+#endif
+
+#if defined(UNDER_WDDM)
+	#ifndef	_INC_STDLIB
+		#if defined(__mips)
+			/* do nothing */
+		#elif defined(UNDER_MSBUILD)
+			/* do nothing */
+		#else
+			_CRTIMP void __cdecl abort(void);
+		#endif
+	#endif
+#endif /* UNDER_WDDM */
+#else
+	#if (defined(__linux__) || defined(__QNXNTO__)) && defined(__KERNEL__)
+		#define IMG_INTERNAL
+		#define IMG_EXPORT
+		#define IMG_CALLCONV
+	#elif defined(__linux__) || defined(__METAG) || defined(__mips) || defined(__QNXNTO__) || defined(__riscv) || defined(__APPLE__) || defined(TEE_DDK)
+		#define IMG_CALLCONV
+		#define C_CALLCONV
+
+		#if defined(__METAG)
+			#define IMG_INTERNAL
+		#else
+			#define IMG_INTERNAL    __attribute__((visibility("hidden")))
+		#endif
+
+		#define IMG_EXPORT      __attribute__((visibility("default")))
+		#define IMG_RESTRICT    __restrict__
+	#elif defined(INTEGRITY_OS)
+		#define IMG_CALLCONV
+		#define IMG_INTERNAL
+		#define IMG_EXPORT
+		#define IMG_RESTRICT
+		#define C_CALLCONV
+		#define __cdecl
+
+		#ifndef USE_CODE
+			#define IMG_ABORT() printf("IMG_ABORT was called.\n")
+		#endif
+	#else
+		#error("define an OS")
+	#endif
+
+#endif
+
+/* Use default definition if not overridden */
+#ifndef IMG_ABORT
+	#if defined(EXIT_ON_ABORT)
+		#define IMG_ABORT()	exit(1)
+	#else
+		#define IMG_ABORT()	abort()
+	#endif
+#endif
+
+/* The best way to suppress unused parameter warnings using GCC is to use a
+ * variable attribute. Place the __maybe_unused between the type and name of an
+ * unused parameter in a function parameter list e.g. 'int __maybe_unused var'.
+ * This should only be used in GCC build environments, for example, in files
+ * that compile only on Linux.
+ * Other files should use PVR_UNREFERENCED_PARAMETER
+ */
+
+/* Kernel macros for compiler attributes */
+/* Note: param positions start at 1 */
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/compiler.h>
+
+	#if !defined(__fallthrough)
+		#if (GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)) && !defined(__CHECKER__)
+			#define __fallthrough __attribute__((__fallthrough__))
+		#else
+			#define __fallthrough
+		#endif
+	#endif
+#elif defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES)
+	#define __must_check       __attribute__((warn_unused_result))
+	#define __maybe_unused     __attribute__((unused))
+	#define __malloc           __attribute__((malloc))
+
+	/* Bionic's <sys/cdefs.h> might have defined these already */
+	/* See https://android.googlesource.com/platform/bionic.git/+/master/libc/include/sys/cdefs.h */
+	#if !defined(__packed)
+		#define __packed           __attribute__((packed))
+	#endif
+	#if !defined(__aligned)
+		#define __aligned(n)       __attribute__((aligned(n)))
+	#endif
+	#if !defined(__noreturn)
+		#define __noreturn         __attribute__((noreturn))
+	#endif
+
+	/* That one compiler that supports attributes but doesn't support
+	 * the printf attribute... */
+	#if defined(__GNUC__)
+		#if defined(__MINGW32__)
+		    #define __printf(fmt, va)  __attribute__((format(gnu_printf, (fmt), (va))))
+		#else
+			#define __printf(fmt, va)  __attribute__((format(printf, (fmt), (va))))
+		#endif
+	#else
+		#define __printf(fmt, va)
+	#endif /* defined(__GNUC__) */
+
+	#if defined(__cplusplus) && (__cplusplus >= 201703L)
+		#define __fallthrough [[fallthrough]]
+	#elif GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+		#define __fallthrough __attribute__((__fallthrough__))
+	#else
+		#define __fallthrough
+	#endif
+
+	#define __user
+	#define __force
+	#define __iomem
+#else
+	/* Silently ignore those attributes */
+	#define __printf(fmt, va)
+	#define __packed
+	#define __aligned(n)
+	#define __must_check
+	#define __maybe_unused
+	#define __malloc
+
+	#if defined(_MSC_VER) || defined(CC_ARM)
+		#define __noreturn __declspec(noreturn)
+	#else
+		#define __noreturn
+	#endif
+
+	/* This may already been defined, e.g. by SAL (Source Annotation Language) */
+	#if !defined(__fallthrough)
+		#define __fallthrough
+	#endif
+
+	#define __user
+	#define __force
+	#define __iomem
+#endif
+
+
+/* Other attributes, following the same style */
+#if defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES)
+	#define __const_function      __attribute__((const))
+#else
+	#define __const_function
+#endif
+
+
+/* GCC builtins */
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/compiler.h>
+#elif defined(__GNUC__) || defined(INTEGRITY_OS)
+
+/* Klocwork does not support __builtin_expect, which makes the actual condition
+ * expressions hidden during analysis, affecting it negatively. */
+#if !defined(__KLOCWORK__) && !defined(INTEGRITY_OS) && !defined(DEBUG)
+	#define likely(x)   __builtin_expect(!!(x), 1)
+	#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+
+	/* Compiler memory barrier to prevent reordering */
+	#define barrier() __asm__ __volatile__("": : :"memory")
+#else
+	#define barrier() static_assert(0, "barrier() isn't supported by your compiler");
+#endif
+
+/* That one OS that defines one but not the other... */
+#ifndef likely
+	#define likely(x)   (x)
+#endif
+#ifndef unlikely
+	#define unlikely(x) (x)
+#endif
+
+#if !defined(BITS_PER_BYTE)
+#define BITS_PER_BYTE (8)
+#endif /* BITS_PER_BYTE */
+
+/* These two macros are also provided by the kernel */
+#ifndef BIT
+#define BIT(b) (1UL << (b))
+#endif
+
+#ifndef BIT_ULL
+#define BIT_ULL(b) (1ULL << (b))
+#endif
+
+#define BIT_SET(f, b)     BITMASK_SET((f),    BIT(b))
+#define BIT_UNSET(f, b)   BITMASK_UNSET((f),  BIT(b))
+#define BIT_TOGGLE(f, b)  BITMASK_TOGGLE((f), BIT(b))
+#define BIT_ISSET(f, b)   BITMASK_HAS((f),    BIT(b))
+
+#define BITMASK_SET(f, m)     do { ((f) |= (m)); } while (false)
+#define BITMASK_UNSET(f, m)   do { ((f) &= ~(m)); } while (false)
+#define BITMASK_TOGGLE(f, m)  do { ((f) ^= (m)); } while (false)
+#define BITMASK_HAS(f, m)     (((f) & (m)) == (m)) /* the bits from the mask are all set */
+#define BITMASK_ANY(f, m)     (((f) & (m)) != 0U)  /* any bit from the mask is set */
+
+#ifndef MAX
+#define MAX(a ,b)	(((a) > (b)) ? (a) : (b))
+#endif
+
+#ifndef MIN
+#define MIN(a, b)	(((a) < (b)) ? (a) : (b))
+#endif
+
+#ifndef CLAMP
+#define CLAMP(min, max, n)  ((n) < (min) ? (min) : ((n) > (max) ? (max) : (n)))
+#endif
+
+#define SWAP(X, Y) (X) ^= (Y); (Y) ^= (X); (X) ^= (Y);
+
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/kernel.h>
+	#include <linux/bug.h>
+#endif
+
+/* Get a structure's address from the address of a member */
+#define IMG_CONTAINER_OF(ptr, type, member) \
+	(type *) ((uintptr_t) (ptr) - offsetof(type, member))
+
+/* Get a new pointer with an offset (in bytes) from a base address, useful
+ * when traversing byte buffers and accessing data in buffers through struct
+ * pointers.
+ * Note, this macro is not equivalent to or replacing offsetof() */
+#define IMG_OFFSET_ADDR(addr, offset_in_bytes) \
+	(void*)&(((IMG_UINT8*)(void*)(addr))[offset_in_bytes])
+
+/* Get a new pointer with an offset (in bytes) from a base address, version
+ * for volatile memory.
+ */
+#define IMG_OFFSET_ADDR_VOLATILE(addr, offset_in_bytes) \
+	(volatile void*)&(((volatile IMG_UINT8*)(volatile void*)(addr))[offset_in_bytes])
+
+/* Get a new pointer with an offset (in dwords) from a base address, useful
+ * when traversing byte buffers and accessing data in buffers through struct
+ * pointers.
+ * Note, this macro is not equivalent to or replacing offsetof() */
+#define IMG_OFFSET_ADDR_DW(addr, offset_in_dwords) \
+	(void*)(((IMG_UINT32*)(void*)(addr)) + (offset_in_dwords))
+
+/* The number of elements in a fixed-sized array */
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(ARR) (sizeof(ARR) / sizeof((ARR)[0]))
+#endif
+
+/* To guarantee that __func__ can be used, define it as a macro here if it
+   isn't already provided by the compiler. */
+#if defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus < 201103L)
+#define __func__ __FUNCTION__
+#endif
+
+#if defined(__cplusplus)
+/* C++ Specific:
+ * Disallow use of copy and assignment operator within a class.
+ * Should be placed under private. */
+#define IMG_DISALLOW_COPY_AND_ASSIGN(C) \
+	C(const C&); \
+	void operator=(const C&)
+#endif
+
+#if defined(SUPPORT_PVR_VALGRIND) && !defined(__METAG) && !defined(__mips) && !defined(__riscv)
+	#include "/usr/include/valgrind/memcheck.h"
+
+	#define VG_MARK_INITIALIZED(pvData,ui32Size) VALGRIND_MAKE_MEM_DEFINED(pvData,ui32Size)
+	#define VG_MARK_NOACCESS(pvData,ui32Size) VALGRIND_MAKE_MEM_NOACCESS(pvData,ui32Size)
+	#define VG_MARK_ACCESS(pvData,ui32Size) VALGRIND_MAKE_MEM_UNDEFINED(pvData,ui32Size)
+	#define VG_ASSERT_DEFINED(pvData,ui32Size) VALGRIND_CHECK_MEM_IS_DEFINED(pvData,ui32Size)
+#else
+	#if defined(_MSC_VER)
+	#	define PVR_MSC_SUPPRESS_4127 __pragma(warning(suppress:4127))
+	#else
+	#	define PVR_MSC_SUPPRESS_4127
+	#endif
+
+	#define VG_MARK_INITIALIZED(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_MARK_NOACCESS(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_MARK_ACCESS(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_ASSERT_DEFINED(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+#endif
+
+#define IMG_STRINGIFY_IMPL(x) # x
+#define IMG_STRINGIFY(x) IMG_STRINGIFY_IMPL(x)
+
+#define IMG_CONCATENATE_IMPL(x,y) x ## y
+#define IMG_CONCATENATE(x,y) IMG_CONCATENATE_IMPL(x,y)
+
+#if defined(DEBUG) && !defined(INTEGRITY_OS)
+#define IMG_PAGESLOG2BYTES(_tcast, _npages, _log2) \
+	({ \
+		PVR_ASSERT( ((IMG_UINT64)(1ULL) << (sizeof(_tcast)*8UL)) >= ((IMG_UINT64)(_npages) << (_log2)) ); \
+		(_tcast)(_npages) << (_log2); \
+	})
+#else
+#define IMG_PAGESLOG2BYTES(_tcast, _npages, _log2) ((_npages) << (_log2))
+#endif
+
+#define IMG_PAGE2BYTES32(logsize) IMG_PAGESLOG2BYTES(IMG_UINT32,IMG_UINT32_C(1),logsize)
+#define IMG_PAGE2BYTES64(logsize) ((IMG_UINT64)IMG_UINT64_C(1) << (logsize))
+
+#define IMG_PAGES2BYTES32(pages,logsize) IMG_PAGESLOG2BYTES(IMG_UINT32,pages,logsize)
+#define IMG_PAGES2BYTES64(pages,logsize) ((IMG_UINT64)(pages) << (logsize))
+
+#define IMG_PAGE_SHIFT_4KB   12U
+#define IMG_PAGE_SHIFT_16KB  14U
+#define IMG_PAGE_SHIFT_64KB  16U
+#define IMG_PAGE_SHIFT_256KB 18U
+#define IMG_PAGE_SHIFT_1MB   20U
+#define IMG_PAGE_SHIFT_2MB   21U
+
+#if defined(INTEGRITY_OS)
+	/* Definitions not present in INTEGRITY. */
+	#define PATH_MAX	200
+#endif
+
+#if defined(__clang__) || defined(__GNUC__)
+	/* __SIZEOF_POINTER__ is defined already by these compilers */
+#elif defined(INTEGRITY_OS)
+	#if defined(__Ptr_Is_64)
+		#define __SIZEOF_POINTER__ 8
+	#else
+		#define __SIZEOF_POINTER__ 4
+	#endif
+#elif defined(_WIN32)
+	#define __SIZEOF_POINTER__ sizeof(char *)
+#else
+	#warning Unknown OS - using default method to determine whether CPU arch is 64-bit.
+	#define __SIZEOF_POINTER__ sizeof(char *)
+#endif
+
+/* RDI8567: gcc/clang/llvm load/store optimisations may cause issues with
+ * uncached device memory allocations. Some pointers are made 'volatile'
+ * to prevent those optimisations being applied to writes through those
+ * pointers.
+ */
+#if (GCC_VERSION_AT_LEAST(7, 0) || defined(__clang__)) && (defined(__arm64__) || defined(__aarch64__))
+#define NOLDSTOPT volatile
+/* after applying 'volatile' to a pointer, we may need to cast it to 'void *'
+ * to keep it compatible with its existing uses.
+ */
+#define NOLDSTOPT_VOID (void *)
+
+#define NOLDSTOPT_REQUIRED 1
+#else
+#define NOLDSTOPT
+#define NOLDSTOPT_VOID
+#endif
+
+#define PVR_PRE_DPF (void) printf
+
+/* C STD >= C99 */
+#if !defined(INTEGRITY_OS) && !defined(__CHECKER__) && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+#define IMG_FLEX_ARRAY_MEMBER
+#define IMG_FLEX_ARRAY_SIZE(size, count) ((size) * (count))
+#else
+/* In C STD prior to C99 flexible array members are an extension feature and syntax requires alternative approach */
+#define IMG_FLEX_ARRAY_MEMBER (1)
+#define IMG_FLEX_ARRAY_SIZE(size, count) ((size) * ((count) - 1))
+#endif
+
+#endif /* IMG_DEFS_H */
+/*****************************************************************************
+ End of file (img_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_drm_fourcc_internal.h b/drivers/gpu/drm/img/img-volcanic/include/img_drm_fourcc_internal.h
new file mode 100644
index 000000000000..ffde181fcb48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_drm_fourcc_internal.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          Wrapper around drm_fourcc.h
+@Description    FourCCs and the DRM framebuffer modifiers should be added here
+                unless they are used by kernel code or a known user outside of
+                the DDK. If FourCCs or DRM framebuffer modifiers are required
+                outside of the DDK, they shall be moved to the corresponding
+                public header.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DRM_FOURCC_INTERNAL_H
+#define IMG_DRM_FOURCC_INTERNAL_H
+
+#include <powervr/img_drm_fourcc.h>
+
+/*
+ * Modifier names are structured using the following convention,
+ * with underscores (_) between items:
+ * - prefix: DRM_FORMAT_MOD
+ * - identifier for our driver: PVR
+ * - category: FBCDC
+ *   - compression tile dimension: 8x8, 16x4, 32x2
+ *   - FBDC version: V0, V1, V2, V3, V7, V8, V10, V12, V13, V14
+ */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0      fourcc_mod_code(PVR, 1)
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0_FIX  fourcc_mod_code(PVR, 2) /* Fix for HW_BRN_37464 */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V1 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V2      fourcc_mod_code(PVR, 4)
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V3      fourcc_mod_code(PVR, 5)
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V7 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V8      fourcc_mod_code(PVR, 18)
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V10 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V13 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0     fourcc_mod_code(PVR, 7)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0_FIX fourcc_mod_code(PVR, 8) /* Fix for HW_BRN_37464 */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V1 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V2     fourcc_mod_code(PVR, 10)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V3     fourcc_mod_code(PVR, 11)
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V7 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V8     fourcc_mod_code(PVR, 19)
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V10 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V13 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V1     fourcc_mod_code(PVR, 13)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V3     fourcc_mod_code(PVR, 14)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V8     fourcc_mod_code(PVR, 20)
+/* DRM_FORMAT_MOD_PVR_FBCDC_32x2_V10 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V12    fourcc_mod_code(PVR, 17)
+
+#endif /* IMG_DRM_FOURCC_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_elf.h b/drivers/gpu/drm/img/img-volcanic/include/img_elf.h
new file mode 100644
index 000000000000..8837d9592599
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_elf.h
@@ -0,0 +1,111 @@
+/*************************************************************************/ /*!
+@File           img_elf.h
+@Title          IMG ELF file definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    Definitions for ELF file structures used in the DDK.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(IMG_ELF_H)
+#define IMG_ELF_H
+
+#include "img_types.h"
+
+/* ELF format defines */
+#define ELF_PT_LOAD     (0x1U)   /* Program header identifier as Load */
+#define ELF_SHT_SYMTAB  (0x2U)   /* Section identifier as Symbol Table */
+#define ELF_SHT_STRTAB  (0x3U)   /* Section identifier as String Table */
+#define MAX_STRTAB_NUM  (0x8U)   /* Maximum number of string table in the ELF file */
+
+/* Redefined structs of ELF format */
+typedef struct
+{
+	IMG_UINT8    ui32Eident[16];
+	IMG_UINT16   ui32Etype;
+	IMG_UINT16   ui32Emachine;
+	IMG_UINT32   ui32Eversion;
+	IMG_UINT32   ui32Eentry;
+	IMG_UINT32   ui32Ephoff;
+	IMG_UINT32   ui32Eshoff;
+	IMG_UINT32   ui32Eflags;
+	IMG_UINT16   ui32Eehsize;
+	IMG_UINT16   ui32Ephentsize;
+	IMG_UINT16   ui32Ephnum;
+	IMG_UINT16   ui32Eshentsize;
+	IMG_UINT16   ui32Eshnum;
+	IMG_UINT16   ui32Eshtrndx;
+} IMG_ELF_HDR;
+
+typedef struct
+{
+	IMG_UINT32   ui32Stname;
+	IMG_UINT32   ui32Stvalue;
+	IMG_UINT32   ui32Stsize;
+	IMG_UINT8    ui32Stinfo;
+	IMG_UINT8    ui32Stother;
+	IMG_UINT16   ui32Stshndx;
+} IMG_ELF_SYM;
+
+typedef struct
+{
+	IMG_UINT32   ui32Shname;
+	IMG_UINT32   ui32Shtype;
+	IMG_UINT32   ui32Shflags;
+	IMG_UINT32   ui32Shaddr;
+	IMG_UINT32   ui32Shoffset;
+	IMG_UINT32   ui32Shsize;
+	IMG_UINT32   ui32Shlink;
+	IMG_UINT32   ui32Shinfo;
+	IMG_UINT32   ui32Shaddralign;
+	IMG_UINT32   ui32Shentsize;
+} IMG_ELF_SHDR;
+
+typedef struct
+{
+	IMG_UINT32   ui32Ptype;
+	IMG_UINT32   ui32Poffset;
+	IMG_UINT32   ui32Pvaddr;
+	IMG_UINT32   ui32Ppaddr;
+	IMG_UINT32   ui32Pfilesz;
+	IMG_UINT32   ui32Pmemsz;
+	IMG_UINT32   ui32Pflags;
+	IMG_UINT32   ui32Palign;
+} IMG_ELF_PROGRAM_HDR;
+
+#endif /* IMG_ELF_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/img_types.h b/drivers/gpu/drm/img/img-volcanic/include/img_types.h
new file mode 100644
index 000000000000..c18c5da47da5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/img_types.h
@@ -0,0 +1,333 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines type aliases for use by IMG APIs.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_TYPES_H
+#define IMG_TYPES_H
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* To use C99 types and definitions, there are two special cases we need to
+ * cater for:
+ *
+ * - Visual Studio: in VS2010 or later, some standard headers are available,
+ *   and MSVC has its own built-in sized types. We can define the C99 types
+ *   in terms of these.
+ *
+ * - Linux kernel code: C99 sized types are defined in <linux/types.h>, but
+ *   some other features (like macros for constants or printf format
+ *   strings) are missing, so we need to fill in the gaps ourselves.
+ *
+ * For other cases (userspace code under Linux, Android or Neutrino, or
+ * firmware code), we can include the standard headers.
+ */
+#if defined(_MSC_VER)
+	#include <stdbool.h>		/* bool */
+	#include "msvc_types.h"
+#elif defined(__linux__) && defined(__KERNEL__)
+	#include <linux/version.h>
+	#include <linux/types.h>
+	#include "kernel_types.h"
+#elif defined(__linux__) || defined(__METAG) || defined(__MINGW32__) || \
+	defined(__QNXNTO__) || defined(INTEGRITY_OS) || defined(__riscv) || \
+	defined(__APPLE__) || defined(TEE_DDK)
+	#include <stddef.h>			/* NULL */
+	#include <stdint.h>
+#if defined(__riscv)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wundef"
+#endif
+	#include <inttypes.h>		/* intX_t/uintX_t, format specifiers */
+	#include <limits.h>			/* INT_MIN, etc */
+	#include <sys/types.h>		/* ssize_t */
+#if defined(__riscv)
+#pragma GCC diagnostic pop
+#endif
+	#include <stdbool.h>		/* bool */
+#elif defined(__mips)
+	#include <stddef.h>			/* NULL */
+	#include <inttypes.h>		/* intX_t/uintX_t, format specifiers */
+	#include <stdbool.h>		/* bool */
+#else
+	#error C99 support not set up for this build
+#endif
+
+/*
+ * Due to a Klocwork bug, 'true'/'false' constants are not recognized to be of
+ * boolean type. This results in large number of false-positives being reported
+ * (MISRA.ETYPE.ASSIGN.2012: "An expression value of essential type 'signed char'
+ * is assigned to an object of essential type 'bool'"). Work around this by
+ * redefining those constants with cast to bool added.
+ */
+#if defined(__KLOCWORK__) && !defined(__cplusplus)
+#undef true
+#undef false
+#define true ((bool) 1)
+#define false ((bool) 0)
+#endif
+
+typedef unsigned int	IMG_UINT;
+typedef int				IMG_INT;
+
+typedef uint8_t			IMG_UINT8,	*IMG_PUINT8;
+typedef uint8_t			IMG_BYTE,	*IMG_PBYTE;
+typedef int8_t			IMG_INT8;
+typedef char			IMG_CHAR,	*IMG_PCHAR;
+
+typedef uint16_t		IMG_UINT16,	*IMG_PUINT16;
+typedef int16_t			IMG_INT16;
+typedef uint32_t		IMG_UINT32,	*IMG_PUINT32;
+typedef int32_t			IMG_INT32,	*IMG_PINT32;
+#if defined(INTEGRITY_OS)
+#if __INT_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## U))
+#elif __LONG_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## UL))
+#elif defined(__LLONG_BIT) && __LLONG_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## ULL))
+#endif
+#else /* defined(INTEGRITY_OS) */
+#define IMG_UINT32_C(c) ((IMG_UINT32)UINT32_C(c))
+#endif /* defined(INTEGRITY_OS) */
+
+typedef uint64_t		IMG_UINT64,	*IMG_PUINT64;
+typedef int64_t			IMG_INT64;
+#define IMG_INT64_C(c)	INT64_C(c)
+#if defined(INTEGRITY_OS)
+#if __INT_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## U)
+#elif defined(__LONG_BIT) && __LONG_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## UL)
+#elif defined(__LLONG_BIT) && __LLONG_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## ULL)
+#endif
+#else /* defined(INTEGRITY_OS) */
+#define IMG_UINT64_C(c)	UINT64_C(c)
+#endif /* defined(INTEGRITY_OS) */
+#define IMG_UINT16_C(c)	UINT16_C(c)
+#define IMG_UINT64_FMTSPEC PRIu64
+#define IMG_UINT64_FMTSPECX PRIX64
+#define IMG_UINT64_FMTSPECx PRIx64
+#define IMG_UINT64_FMTSPECo PRIo64
+#define IMG_INT64_FMTSPECd PRId64
+
+#define IMG_UINT8_MAX	UINT8_MAX
+#define IMG_UINT16_MAX	UINT16_MAX
+#define IMG_UINT32_MAX	UINT32_MAX
+#define IMG_UINT64_MAX	UINT64_MAX
+
+#define IMG_INT_MIN		INT_MIN
+#define IMG_INT_MAX		INT_MAX
+#define IMG_INT8_MAX	INT8_MAX
+#define IMG_INT16_MAX	INT16_MAX
+#define IMG_INT32_MAX	INT32_MAX
+#define IMG_INT64_MAX	INT64_MAX
+
+/* Linux kernel mode does not use floating point */
+typedef float			IMG_FLOAT,	*IMG_PFLOAT;
+typedef double			IMG_DOUBLE;
+
+typedef union
+{
+	IMG_UINT32 ui32;
+	IMG_FLOAT f;
+} IMG_UINT32_FLOAT;
+
+typedef int				IMG_SECURE_TYPE;
+
+typedef bool      IMG_BOOL;
+typedef bool*     IMG_PBOOL;
+#define IMG_FALSE ((bool) 0)
+#define IMG_TRUE  ((bool) 1)
+
+typedef IMG_CHAR const* IMG_PCCHAR;
+
+/* Format specifiers for 'size_t' type */
+#if defined(_MSC_VER)
+#define IMG_SIZE_FMTSPEC  "%Iu"
+#define IMG_SIZE_FMTSPECX "%Ix"
+#else
+#define IMG_SIZE_FMTSPEC  "%zu"
+#define IMG_SIZE_FMTSPECX "%zx"
+#endif
+
+#if defined(__linux__) && defined(__KERNEL__)
+/* prints the function name when used with printk */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#define IMG_PFN_FMTSPEC "%ps"
+#else
+#define IMG_PFN_FMTSPEC "%pf"
+#endif
+#else
+#define IMG_PFN_FMTSPEC "%p"
+#endif
+
+typedef void           *IMG_HANDLE;
+
+/* Process IDs */
+typedef IMG_UINT32      IMG_PID;
+
+/* OS connection type */
+typedef int             IMG_OS_CONNECTION;
+
+
+/*
+ * Address types.
+ * All types used to refer to a block of memory are wrapped in structures
+ * to enforce some degree of type safety, i.e. a IMG_DEV_VIRTADDR cannot
+ * be assigned to a variable of type IMG_DEV_PHYADDR because they are not the
+ * same thing.
+ *
+ * There is an assumption that the system contains at most one non-cpu mmu,
+ * and a memory block is only mapped by the MMU once.
+ *
+ * Different devices could have offset views of the physical address space.
+ *
+ */
+
+
+/*
+ *
+ * +------------+    +------------+      +------------+        +------------+
+ * |    CPU     |    |    DEV     |      |    DEV     |        |    DEV     |
+ * +------------+    +------------+      +------------+        +------------+
+ *       |                 |                   |                     |
+ *       | void *          |IMG_DEV_VIRTADDR   |IMG_DEV_VIRTADDR     |
+ *       |                 \-------------------/                     |
+ *       |                          |                                |
+ * +------------+             +------------+                         |
+ * |    MMU     |             |    MMU     |                         |
+ * +------------+             +------------+                         |
+ *       |                          |                                |
+ *       |                          |                                |
+ *       |                          |                                |
+ *   +--------+                +---------+                      +--------+
+ *   | Offset |                | (Offset)|                      | Offset |
+ *   +--------+                +---------+                      +--------+
+ *       |                          |                IMG_DEV_PHYADDR |
+ *       |                          |                                |
+ *       |                          | IMG_DEV_PHYADDR                |
+ * +---------------------------------------------------------------------+
+ * |                         System Address bus                          |
+ * +---------------------------------------------------------------------+
+ *
+ */
+
+#define IMG_DEV_VIRTADDR_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_SIZE_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_ALIGN_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_OFFSET_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+
+/* cpu physical address */
+typedef struct
+{
+#if defined(UNDER_WDDM) || defined(WINDOWS_WDF)
+	uintptr_t uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(uintptr_t)(var)
+#define CPUPHYADDR_FMTARG(var)				(IMG_UINT64)(var)
+#define CPUPHYADDR_UINT_FMTSPEC "0x%016" IMG_UINT64_FMTSPECx
+#elif defined(__linux__) && defined(__KERNEL__)
+	phys_addr_t uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(phys_addr_t)(var)
+#define CPUPHYADDR_FMTARG(var)				(&var)
+#define CPUPHYADDR_UINT_FMTSPEC "%pa"
+#else
+	IMG_UINT64 uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(IMG_UINT64)(var)
+#define CPUPHYADDR_FMTARG(var)				(var)
+#define CPUPHYADDR_UINT_FMTSPEC "0x%016" IMG_UINT64_FMTSPECx
+#endif
+} IMG_CPU_PHYADDR;
+
+/* device physical address */
+typedef struct
+{
+	IMG_UINT64 uiAddr;
+} IMG_DEV_PHYADDR;
+
+/* dma address */
+typedef struct
+{
+	IMG_UINT64 uiAddr;
+} IMG_DMA_ADDR;
+
+/*
+	rectangle structure
+*/
+typedef struct
+{
+	IMG_INT32	x0;
+	IMG_INT32	y0;
+	IMG_INT32	x1;
+	IMG_INT32	y1;
+} IMG_RECT;
+
+typedef struct
+{
+	IMG_INT16	x0;
+	IMG_INT16	y0;
+	IMG_INT16	x1;
+	IMG_INT16	y1;
+} IMG_RECT_16;
+
+/*
+ * box structure
+ */
+typedef struct
+{
+	IMG_INT32	x0;
+	IMG_INT32	y0;
+	IMG_INT32	z0;
+	IMG_INT32	x1;
+	IMG_INT32	y1;
+	IMG_INT32	z1;
+} IMG_BOX;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* IMG_TYPES_H */
+/******************************************************************************
+ End of file (img_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/kernel_types.h b/drivers/gpu/drm/img/img-volcanic/include/kernel_types.h
new file mode 100644
index 000000000000..c3305102fc20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/kernel_types.h
@@ -0,0 +1,137 @@
+/*************************************************************************/ /*!
+@Title          C99-compatible types and definitions for Linux kernel code
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/kernel.h>
+
+/* Limits of specified-width integer types */
+
+/* S8_MIN, etc were added in kernel version 3.14. The other versions are for
+ * earlier kernels. They can be removed once older kernels don't need to be
+ * supported.
+ */
+#ifdef S8_MIN
+	#define INT8_MIN	S8_MIN
+#else
+	#define INT8_MIN	(-128)
+#endif
+
+#ifdef S8_MAX
+	#define INT8_MAX	S8_MAX
+#else
+	#define INT8_MAX	127
+#endif
+
+#ifdef U8_MAX
+	#define UINT8_MAX	U8_MAX
+#else
+	#define UINT8_MAX	0xFF
+#endif
+
+#ifdef S16_MIN
+	#define INT16_MIN	S16_MIN
+#else
+	#define INT16_MIN	(-32768)
+#endif
+
+#ifdef S16_MAX
+	#define INT16_MAX	S16_MAX
+#else
+	#define INT16_MAX	32767
+#endif
+
+#ifdef U16_MAX
+	#define UINT16_MAX	U16_MAX
+#else
+	#define UINT16_MAX	0xFFFF
+#endif
+
+#ifdef S32_MIN
+	#define INT32_MIN	S32_MIN
+#else
+	#define INT32_MIN	(-2147483647 - 1)
+#endif
+
+#ifdef S32_MAX
+	#define INT32_MAX	S32_MAX
+#else
+	#define INT32_MAX	2147483647
+#endif
+
+#ifdef U32_MAX
+	#define UINT32_MAX	U32_MAX
+#else
+	#define UINT32_MAX	0xFFFFFFFF
+#endif
+
+#ifdef S64_MIN
+	#define INT64_MIN	S64_MIN
+#else
+	#define INT64_MIN	(-9223372036854775807LL)
+#endif
+
+#ifdef S64_MAX
+	#define INT64_MAX	S64_MAX
+#else
+	#define INT64_MAX	9223372036854775807LL
+#endif
+
+#ifdef U64_MAX
+	#define UINT64_MAX	U64_MAX
+#else
+	#define UINT64_MAX	0xFFFFFFFFFFFFFFFFULL
+#endif
+
+/* Macros for integer constants */
+#define INT8_C			S8_C
+#define UINT8_C			U8_C
+#define INT16_C			S16_C
+#define UINT16_C		U16_C
+#define INT32_C			S32_C
+#define UINT32_C		U32_C
+#define INT64_C			S64_C
+#define UINT64_C		U64_C
+
+/* Format conversion of integer types <inttypes.h> */
+
+#define PRIX64		"llX"
+#define PRIx64		"llx"
+#define PRIu64		"llu"
+#define PRId64		"lld"
diff --git a/drivers/gpu/drm/img/img-volcanic/include/linux_sw_sync.h b/drivers/gpu/drm/img/img-volcanic/include/linux_sw_sync.h
new file mode 100644
index 000000000000..c12c650294a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/linux_sw_sync.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef _UAPI_LINUX_PVR_SW_SYNC_H
+#define _UAPI_LINUX_PVR_SW_SYNC_H
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+
+#include <linux/types.h>
+
+#include "pvrsrv_sync_km.h"
+#include "pvr_drm.h"
+
+#endif /* defined(SUPPORT_NATIVE_FENCE_SYNC) */
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/lock_types.h b/drivers/gpu/drm/img/img-volcanic/include/lock_types.h
new file mode 100644
index 000000000000..bf7167b554f6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/lock_types.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@File           lock_types.h
+@Title          Locking types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Locking specific enums, defines and structures
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOCK_TYPES_H
+#define LOCK_TYPES_H
+
+/* In Linux kernel mode we are using the kernel mutex implementation directly
+ * with macros. This allows us to use the kernel lockdep feature for lock
+ * debugging. */
+#if defined(__linux__) && defined(__KERNEL__)
+
+#include <linux/types.h>
+#include <linux/mutex.h>
+/* The mutex is defined as a pointer to be compatible with the other code. This
+ * isn't ideal and usually you wouldn't do that in kernel code. */
+typedef struct mutex *POS_LOCK;
+typedef struct rw_semaphore *POSWR_LOCK;
+typedef spinlock_t *POS_SPINLOCK;
+typedef atomic_t ATOMIC_T;
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+#include "img_types.h" /* needed for IMG_INT */
+typedef struct OS_LOCK_TAG *POS_LOCK;
+
+#if defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+typedef struct OSWR_LOCK_TAG *POSWR_LOCK;
+#else /* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+typedef struct OSWR_LOCK_TAG {
+	IMG_UINT32 ui32Unused;
+} *POSWR_LOCK;
+#endif /* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+
+#if defined(__linux__) || defined(__APPLE__)
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(__QNXNTO__)
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(_WIN32)
+	/*
+	 * Placeholder definition. WDDM doesn't use Services, but some headers
+	 * still have to be shared. This is one such case.
+	 */
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(INTEGRITY_OS)
+	/* Only lower 32bits are used in OS ATOMIC APIs to have consistent behaviour across all OS */
+	typedef struct OS_ATOMIC_TAG {IMG_INT64 counter;} ATOMIC_T;
+#else
+	#error "Please type-define an atomic lock for this environment"
+#endif
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#endif /* LOCK_TYPES_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/log2.h b/drivers/gpu/drm/img/img-volcanic/include/log2.h
new file mode 100644
index 000000000000..795660400850
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/log2.h
@@ -0,0 +1,417 @@
+/*************************************************************************/ /*!
+@Title          Integer log2 and related functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOG2_H
+#define LOG2_H
+
+#include "img_defs.h"
+
+/*************************************************************************/ /*!
+@Description    Determine if a number is a power of two.
+@Input          n
+@Return         True if n is a power of 2, false otherwise. True if n == 0.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL __const_function IsPower2(uint32_t n)
+{
+	/* C++ needs this cast. */
+	return (IMG_BOOL)((n & (n - 1U)) == 0U);
+}
+
+/*************************************************************************/ /*!
+@Description    Determine if a number is a power of two.
+@Input          n
+@Return         True if n is a power of 2, false otherwise. True if n == 0.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL __const_function IsPower2_64(uint64_t n)
+{
+	/* C++ needs this cast. */
+	return (IMG_BOOL)((n & (n - 1U)) == 0U);
+}
+
+/* Code using GNU GCC intrinsics */
+#if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER))
+
+/* CHAR_BIT is typically found in <limits.h>. For all the platforms where
+ * CHAR_BIT is not available, defined it here with the assumption that there
+ * are 8 bits in a byte */
+#ifndef CHAR_BIT
+#define CHAR_BIT 8U
+#endif
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2(uint32_t n)
+{
+	if (unlikely(n == 0U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+		return uNumBits - (uint32_t)__builtin_clz(n) - 1U;
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2_64(uint64_t n)
+{
+	if (unlikely(n == 0U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+		return uNumBits - (uint32_t)__builtin_clzll(n) - 1U;
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2(uint32_t n)
+{
+	if (unlikely(n == 0U || n == 1U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+
+		n--; /* Handle powers of 2 */
+		return uNumBits - (uint32_t)__builtin_clz(n);
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2_64(uint64_t n)
+{
+	if (unlikely(n == 0U || n == 1U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+
+		n--; /* Handle powers of 2 */
+		return uNumBits - (uint32_t)__builtin_clzll(n);
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2(uint32_t n)
+{
+	return (uint32_t)CHAR_BIT * (uint32_t)sizeof(n) - (uint32_t)__builtin_clz(n) - 1U;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2_64(uint64_t n)
+{
+	return (uint32_t)CHAR_BIT * (uint32_t)sizeof(n) - (uint32_t)__builtin_clzll(n) - 1U;
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function RoundUpToNextPowerOfTwo(uint32_t n)
+{
+	/* Cases with n greater than 2^31 needs separate handling
+	 * as result of (1<<32) is undefined. */
+	if (unlikely( n == 0U || n > (uint32_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - 1U)))
+	{
+		return 0;
+	}
+
+	/* Return n if it is already a power of 2 */
+	if ((IMG_BOOL)((n & (n - 1U)) == 0U))
+	{
+		return n;
+	}
+
+	return (uint32_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - (uint32_t)__builtin_clz(n));
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint64_t __const_function RoundUpToNextPowerOfTwo_64(uint64_t n)
+{
+	/* Cases with n greater than 2^63 needs separate handling
+	 * as result of (1<<64) is undefined. */
+	if (unlikely( n == 0U || n > (uint64_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - 1U)))
+	{
+		return 0;
+	}
+
+	/* Return n if it is already a power of 2 */
+	if ((IMG_BOOL)((n & (n - 1U)) == 0U))
+	{
+		return n;
+	}
+
+	return (uint64_t)1 << ((uint64_t)CHAR_BIT * sizeof(n) - (uint64_t)__builtin_clzll(n));
+}
+
+#else /* #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER)) */
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function RoundUpToNextPowerOfTwo(uint32_t n)
+{
+	n--;
+	n |= n >> 1;  /* handle  2 bit numbers */
+	n |= n >> 2;  /* handle  4 bit numbers */
+	n |= n >> 4;  /* handle  8 bit numbers */
+	n |= n >> 8;  /* handle 16 bit numbers */
+	n |= n >> 16; /* handle 32 bit numbers */
+	n++;
+
+	return n;
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint64_t __const_function RoundUpToNextPowerOfTwo_64(uint64_t n)
+{
+	n--;
+	n |= n >> 1;  /* handle  2 bit numbers */
+	n |= n >> 2;  /* handle  4 bit numbers */
+	n |= n >> 4;  /* handle  8 bit numbers */
+	n |= n >> 8;  /* handle 16 bit numbers */
+	n |= n >> 16; /* handle 32 bit numbers */
+	n |= n >> 32; /* handle 64 bit numbers */
+	n++;
+
+	return n;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2(uint32_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	for (n >>= 1; n != 0U; n >>= 1)
+	{
+		ui32log2++;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2_64(uint64_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	for (n >>= 1; n != 0U; n >>= 1)
+	{
+		ui32log2++;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2(uint32_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	if (n == 0U)
+	{
+		return 0;
+	}
+
+	n--; /* Handle powers of 2 */
+
+	while (n != 0U)
+	{
+		ui32log2++;
+		n >>= 1;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2_64(uint64_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	if (n == 0U)
+	{
+		return 0;
+	}
+
+	n--; /* Handle powers of 2 */
+
+	while (n != 0U)
+	{
+		ui32log2++;
+		n >>= 1;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2(uint32_t n)
+{
+	static const uint32_t b[] =
+		{0xAAAAAAAAU, 0xCCCCCCCCU, 0xF0F0F0F0U, 0xFF00FF00U, 0xFFFF0000U};
+	uint32_t r = (((n & b[0]) != 0U) ? 1U : 0U);
+
+	r |= (uint32_t) (((n & b[4]) != 0U) ? (1U << 4) : 0U);
+	r |= (uint32_t) (((n & b[3]) != 0U) ? (1U << 3) : 0U);
+	r |= (uint32_t) (((n & b[2]) != 0U) ? (1U << 2) : 0U);
+	r |= (uint32_t) (((n & b[1]) != 0U) ? (1U << 1) : 0U);
+
+	return r;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2_64(uint64_t n)
+{
+	static const uint64_t b[] =
+		{0xAAAAAAAAAAAAAAAAUL, 0xCCCCCCCCCCCCCCCCUL,
+		 0xF0F0F0F0F0F0F0F0UL, 0xFF00FF00FF00FF00UL,
+		 0xFFFF0000FFFF0000UL, 0xFFFFFFFF00000000UL};
+	uint32_t r = (((n & b[0]) != 0U) ? 1U : 0U);
+
+	r |= (uint32_t) (((n & b[5]) != 0U) ? (1U << 5) : 0U);
+	r |= (uint32_t) (((n & b[4]) != 0U) ? (1U << 4) : 0U);
+	r |= (uint32_t) (((n & b[3]) != 0U) ? (1U << 3) : 0U);
+	r |= (uint32_t) (((n & b[2]) != 0U) ? (1U << 2) : 0U);
+	r |= (uint32_t) (((n & b[1]) != 0U) ? (1U << 1) : 0U);
+
+	return r;
+}
+
+#endif /* #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER)) */
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(size)) , where size is the max of 3 sizes
+				This is almost always the ONLY EVER valid use of FloorLog2.
+				Usually CeilLog2() should be used instead.
+				For a 5x5x1 texture, the 3 miplevels are:
+					0:  5x5x1
+					1:	2x2x1
+					2:	1x1x1
+
+				For an 8x8x1 texture, the 4 miplevels are:
+					0:  8x8x1
+					1:	4x4x1
+					2:  2x2x1
+					3:  1x1x1
+
+
+@Input          sizeX, sizeY, sizeZ
+@Return         Count of mipmap levels for given dimensions
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function NumMipLevels(uint32_t sizeX, uint32_t sizeY, uint32_t sizeZ)
+{
+
+	uint32_t maxSize = MAX(MAX(sizeX, sizeY), sizeZ);
+	return FloorLog2(maxSize) + 1U;
+}
+
+#endif /* LOG2_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/multicore_defs.h b/drivers/gpu/drm/img/img-volcanic/include/multicore_defs.h
new file mode 100644
index 000000000000..b410f57075f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/multicore_defs.h
@@ -0,0 +1,53 @@
+/**************************************************************************/ /*!
+@File
+@Title          RGX Multicore Information flags
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_MULTICORE_DEFS_H
+#define RGX_MULTICORE_DEFS_H
+
+/* Capability bits returned to client in RGXGetMultiCoreInfo */
+#define RGX_MULTICORE_CAPABILITY_FRAGMENT_EN    (0x00000040U)
+#define RGX_MULTICORE_CAPABILITY_GEOMETRY_EN    (0x00000020U)
+#define RGX_MULTICORE_CAPABILITY_COMPUTE_EN     (0x00000010U)
+#define RGX_MULTICORE_CAPABILITY_PRIMARY_EN     (0x00000008U)
+#define RGX_MULTICORE_ID_CLRMSK                 (0xFFFFFFF8U)
+
+#endif /* RGX_MULTICORE_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/osfunc_common.h b/drivers/gpu/drm/img/img-volcanic/include/osfunc_common.h
new file mode 100644
index 000000000000..d93cb6d61aba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/osfunc_common.h
@@ -0,0 +1,274 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS functions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSFUNC_COMMON_H
+/*! @cond Doxygen_Suppress */
+#define OSFUNC_COMMON_H
+/*! @endcond */
+
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/string.h>
+#else
+#include <string.h>
+#endif
+
+#include "img_types.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/**************************************************************************/ /*!
+@Function       DeviceMemSet
+@Description    Set memory, whose mapping may be uncached, to a given value.
+                Safe implementation for all architectures for uncached mapping,
+                optimised for speed where supported by tool chains.
+                In such cases, OSDeviceMemSet() is defined as a call to this
+                function.
+@Input          pvDest     void pointer to the memory to be set
+@Input          ui8Value   byte containing the value to be set
+@Input          ui32Size   the number of bytes to be set to the given value
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemSet(void *pvDest, IMG_UINT8 ui8Value, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemCopy
+@Description    Copy values from one area of memory. Safe implementation for
+                all architectures for uncached mapping, of either the source
+                or destination, optimised for speed where supported by tool
+                chains. In such cases, OSDeviceMemCopy() is defined as a call
+                to this function.
+@Input          pvDst      void pointer to the destination memory
+@Input          pvSrc      void pointer to the source memory
+@Input          ui32Size   the number of bytes to be copied
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemCopy(void *pvDst, const void *pvSrc, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemSetBytes
+@Description    Potentially very slow (but safe) memset fallback for non-GNU C
+                compilers for arm64/aarch64
+@Input          pvDest     void pointer to the memory to be set
+@Input          ui8Value   byte containing the value to be set
+@Input          ui32Size   the number of bytes to be set to the given value
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemSetBytes(void *pvDest, IMG_UINT8 ui8Value, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemCopyBytes
+@Description    Potentially very slow (but safe) memcpy fallback for non-GNU C
+                compilers for arm64/aarch64
+@Input          pvDst      void pointer to the destination memory
+@Input          pvSrc      void pointer to the source memory
+@Input          ui32Size   the number of bytes to be copied
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemCopyBytes(void *pvDst, const void *pvSrc, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSStringSafeCopy
+@Description    Copy at most uDataSize-1 bytes from pszSrc to pszDest. pszDest
+                is always null-terminated. If no null byte ('\0') is contained
+                within the first uDataSize-1 characters of the source string,
+                the destination will contain the truncated source string,
+                otherwise pszDest will contain the entire source string.
+@Input          pszDest    char pointer to the destination string
+@Input          pszSrc     const char pointer to the source string
+@Input          uDataSize  the maximum number of bytes to be copied; this
+                           should normally be the size of the destination
+                           buffer
+@Return         Number of bytes copied if uDataSize > strlen(pszSrc), negative
+                value otherwise
+ */ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__) && !defined(DEBUG)
+static inline ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	return strscpy(pszDest, pszSrc, uDataSize);
+}
+#else
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize);
+#endif
+
+#if defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)
+#if defined(__GNUC__)
+/* Workarounds for assumptions made that memory will not be mapped uncached
+ * in kernel or user address spaces on arm64 platforms (or other testing).
+ */
+
+#define OSDeviceMemSet(a,b,c)  DeviceMemSet((a), (b), (c))
+#define OSDeviceMemCopy(a,b,c) DeviceMemCopy((a), (b), (c))
+
+#else /* defined __GNUC__ */
+
+#define OSDeviceMemSet(a,b,c)  DeviceMemSetBytes((a), (b), (c))
+#define OSDeviceMemCopy(a,b,c) DeviceMemCopyBytes((a), (b), (c))
+
+#endif /* defined __GNUC__ */
+
+#else /* (defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)) */
+
+/* Everything else */
+
+/**************************************************************************/ /*!
+@Function       OSDeviceMemSet
+@Description    Set memory, whose mapping may be uncached, to a given value.
+                On some architectures, additional processing may be needed
+                if the mapping is uncached.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSDeviceMemSet(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memset((a), (b), (c)); \
+			(void) *(volatile IMG_UINT32*)((void*)(a)); \
+		} \
+	} while (false)
+
+/**************************************************************************/ /*!
+@Function       OSDeviceMemCopy
+@Description    Copy values from one area of memory, to another, when one
+                or both mappings may be uncached.
+                On some architectures, additional processing may be needed
+                if mappings are uncached.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSDeviceMemCopy(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			memcpy((a), (b), (c)); \
+			(void) *(volatile IMG_UINT32*)((void*)(a)); \
+		} \
+	} while (false)
+
+#endif /* (defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)) */
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemSet
+@Description    Set memory, where the mapping is known to be cached, to a
+                given value. This function exists to allow an optimal memset
+                to be performed when memory is known to be cached.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemSet(a,b,c)  (void) memset((a), (b), (c))
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemCopy
+@Description    Copy values from one area of memory, to another, when both
+                mappings are known to be cached.
+                This function exists to allow an optimal memcpy to be
+                performed when memory is known to be cached.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemCopy(a,b,c) memcpy((a), (b), (c))
+
+#if defined(__KERNEL__)
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemSetWMB
+@Description    Set memory, where the mapping is known to be cached or
+                write-combine, to a given value and issue a write memory barrier
+                after. This
+                function exists to allow an optimal memset to be performed when
+                memory is known to be cached or write-combine.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemSetWMB(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memset((a), (b), (c)); \
+			OSWriteMemoryBarrier(a); \
+		} \
+	} while (false)
+/**************************************************************************/ /*!
+@Function       OSCachedMemCopy
+@Description    Copy values from one area of memory, to another, when both
+                mappings are known to be cached or write-combine and issue
+                a write memory barrier after.
+                This function exists to allow an optimal memcpy to be
+                performed when memory is known to be cached or write-combine.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemCopyWMB(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memcpy((a), (b), (c)); \
+			OSWriteMemoryBarrier(a); \
+		} \
+	} while (false)
+#endif /* defined(__KERNEL__) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OSFUNC_COMMON_H */
+
+/******************************************************************************
+ End of file (osfunc_common.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pdumpdefs.h b/drivers/gpu/drm/img/img-volcanic/include/pdumpdefs.h
new file mode 100644
index 000000000000..7cea362a4152
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pdumpdefs.h
@@ -0,0 +1,268 @@
+/*************************************************************************/ /*!
+@File
+@Title          PDUMP definitions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    PDUMP definitions header
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PDUMPDEFS_H
+#define PDUMPDEFS_H
+
+/*! PDump Pixel Format Enumeration */
+typedef enum _PDUMP_PIXEL_FORMAT_
+{
+	PVRSRV_PDUMP_PIXEL_FORMAT_UNSUPPORTED = 0,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB8 = 1,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB332 = 2,
+	PVRSRV_PDUMP_PIXEL_FORMAT_KRGB555 = 3,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB565 = 4,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB4444 = 5,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB1555 = 6,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB888 = 7,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB8888 = 8,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV8 = 9,
+/*	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV4444 = 10, */
+	PVRSRV_PDUMP_PIXEL_FORMAT_VY0UY1_8888 = 11,
+	PVRSRV_PDUMP_PIXEL_FORMAT_UY0VY1_8888 = 12,
+	PVRSRV_PDUMP_PIXEL_FORMAT_Y0UY1V_8888 = 13,
+	PVRSRV_PDUMP_PIXEL_FORMAT_Y0VY1U_8888 = 14,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV888 = 15,
+	PVRSRV_PDUMP_PIXEL_FORMAT_UYVY10101010 = 16,
+	PVRSRV_PDUMP_PIXEL_FORMAT_VYAUYA8888 = 17,
+	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV8888 = 18,
+	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV2101010 = 19,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV101010 = 20,
+	PVRSRV_PDUMP_PIXEL_FORMAT_PL12Y8 = 21,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_IMC2 = 22,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV12 = 23,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_PL8 = 24,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_PL12 = 25,
+	PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV8 = 26,
+	PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV8 = 27,
+	PVRSRV_PDUMP_PIXEL_FORMAT_PL12Y10 = 28,
+	PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV10 = 29,
+	PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV10 = 30,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR8888 = 31,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA8888 = 32,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB8332 = 33,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB555 = 34,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16 = 35,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32 = 36,
+	PVRSRV_PDUMP_PIXEL_FORMAT_L16 = 37,
+	PVRSRV_PDUMP_PIXEL_FORMAT_L32 = 38,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA8888 = 39,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR4444 = 40,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA4444 = 41,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA4444 = 42,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR1555 = 43,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA5551 = 44,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA5551 = 45,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGR565 = 46,
+	PVRSRV_PDUMP_PIXEL_FORMAT_A8 = 47,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16F16F16 = 49,
+	PVRSRV_PDUMP_PIXEL_FORMAT_A4 = 50,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB2101010 = 51,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RSGSBS888 = 52,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32F32F32 = 53,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16 = 54,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32 = 55,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16F16 = 56,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32F32 = 57,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U8 = 58,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U8U8 = 59,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16 = 60,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16U16 = 61,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16U16U16U16 = 62,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32 = 63,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32U32 = 64,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32U32U32U32 = 65,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV32 = 66,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR10_PACK16 = 67,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB10_PACK16 = 68,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA10_PACK16 = 69,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA10_PACK16 = 70,
+
+	PVRSRV_PDUMP_PIXEL_FORMAT_FORCE_I32 = 0x7fffffff
+
+} PDUMP_PIXEL_FORMAT;
+
+typedef enum _PDUMP_FBC_SWIZZLE_
+{
+	PVRSRV_PDUMP_FBC_SWIZZLE_ARGB = 0x0,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ARBG = 0x1,
+	PVRSRV_PDUMP_FBC_SWIZZLE_AGRB = 0x2,
+	PVRSRV_PDUMP_FBC_SWIZZLE_AGBR = 0x3,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ABGR = 0x4,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ABRG = 0x5,
+	PVRSRV_PDUMP_FBC_SWIZZLE_RGBA = 0x8,
+	PVRSRV_PDUMP_FBC_SWIZZLE_RBGA = 0x9,
+	PVRSRV_PDUMP_FBC_SWIZZLE_GRBA = 0xA,
+	PVRSRV_PDUMP_FBC_SWIZZLE_GBRA = 0xB,
+	PVRSRV_PDUMP_FBC_SWIZZLE_BGRA = 0xC,
+	PVRSRV_PDUMP_FBC_SWIZZLE_BRGA = 0xD,
+} PDUMP_FBC_SWIZZLE;
+
+/*! PDump addrmode */
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT			0
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_MASK			0x000000FF
+
+#define PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_SHIFT			8
+#define PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_NEGATIVE		(1U << PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT		11
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_MASK		0x00000800
+
+#define PVRSRV_PDUMP_ADDRMODE_BIFTILE_MODE_SHIFT		12
+#define PVRSRV_PDUMP_ADDRMODE_BIFTILE_MODE_MASK			0x000FF000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT				20
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_MASK				0x00F00000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCDECOR_SHIFT			24
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_SHIFT			25
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT		26
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_MASK		0x0C000000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT			28
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_MASK			0xF0000000
+
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_STRIDE			(0U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE1 (1U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE2 (2U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE3 (3U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE4 (4U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE5 (5U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE6 (6U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE7 (7U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_TWIDDLED		(9U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_PAGETILED		(11U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_ZTWIDDLED		(12U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_25_50_75	(0U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_25_37_50	(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_NONE				(0U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_DIRECT		(1U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_DIRECT		(2U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_32X2_DIRECT		(3U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_INDIRECT		(4U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_INDIRECT		(5U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_INDIRECT_4TILE	(6U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_INDIRECT_4TILE	(7U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_NONE		(0U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_75		(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_37		(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_50		(2U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_25		(3U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBC_DECOR					(1U << PVRSRV_PDUMP_ADDRMODE_FBCDECOR_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBC_LOSSY					(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_BASE			(1U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_ENHANCED		(2U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V2				(3U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_SURFACE		(4U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_RESOURCE		(5U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_1_SURFACE	(6U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_1_RESOURCE	(7U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V4				(8U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V4PLUS			(9U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_TFBCDC			(10U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+
+/*! PDump Poll Operator */
+typedef enum _PDUMP_POLL_OPERATOR
+{
+	PDUMP_POLL_OPERATOR_EQUAL = 0,
+	PDUMP_POLL_OPERATOR_LESS = 1,
+	PDUMP_POLL_OPERATOR_LESSEQUAL = 2,
+	PDUMP_POLL_OPERATOR_GREATER = 3,
+	PDUMP_POLL_OPERATOR_GREATEREQUAL = 4,
+	PDUMP_POLL_OPERATOR_NOTEQUAL = 5,
+} PDUMP_POLL_OPERATOR;
+
+
+#define PVRSRV_PDUMP_MAX_FILENAME_SIZE			75  /*!< Max length of a pdump log file name */
+#define PVRSRV_PDUMP_MAX_COMMENT_SIZE			350 /*!< Max length of a pdump comment */
+
+/*!
+	PDump MMU type
+*/
+typedef enum
+{
+	PDUMP_MMU_TYPE_4KPAGE_32BIT_STDTILE  = 1,
+	PDUMP_MMU_TYPE_VARPAGE_32BIT_STDTILE = 2,
+	PDUMP_MMU_TYPE_4KPAGE_36BIT_EXTTILE  = 3,
+	PDUMP_MMU_TYPE_4KPAGE_32BIT_EXTTILE  = 4,
+	PDUMP_MMU_TYPE_4KPAGE_36BIT_STDTILE  = 5,
+	PDUMP_MMU_TYPE_VARPAGE_40BIT         = 6,
+	PDUMP_MMU_TYPE_VIDEO_40BIT_STDTILE   = 7,
+	PDUMP_MMU_TYPE_VIDEO_40BIT_EXTTILE   = 8,
+	PDUMP_MMU_TYPE_MIPS_MICROAPTIV       = 9,
+	PDUMP_MMU_TYPE_LAST
+} PDUMP_MMU_TYPE;
+
+/*!
+	PDump states
+	These values are used by the bridge call PVRSRVPDumpGetState
+*/
+#define PDUMP_STATE_CAPTURE_FRAME		(1U)		/*!< Flag represents the PDump being in capture range or not*/
+#define PDUMP_STATE_CONNECTED			(2U)		/*!< Flag represents the PDump Client App being connected on not */
+#define PDUMP_STATE_SUSPENDED			(4U)		/*!< Flag represents the PDump being suspended or not */
+#define PDUMP_STATE_CAPTURE_IN_INTERVAL	(8U)		/*!< Flag represents the PDump being in a capture range interval */
+#define PDUMP_STATE_APP_TERMINATED		(16U)		/*!< Flag represents the PDump captured app has been terminated */
+
+/*!
+	PDump Capture modes
+	Values used with calls to PVRSRVPDumpSetDefaultCaptureParams
+*/
+#define PDUMP_CAPMODE_UNSET                     0x00000000UL
+#define PDUMP_CAPMODE_FRAMED                    0x00000001UL
+#define PDUMP_CAPMODE_CONTINUOUS                0x00000002UL
+#define PDUMP_CAPMODE_BLOCKED                   0x00000003UL
+
+#define PDUMP_CAPMODE_MAX                       PDUMP_CAPMODE_BLOCKED
+
+#endif /* PDUMPDEFS_H */
+
+/*****************************************************************************
+ End of file (pdumpdefs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pdumpdesc.h b/drivers/gpu/drm/img/img-volcanic/include/pdumpdesc.h
new file mode 100644
index 000000000000..273cde910e7f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pdumpdesc.h
@@ -0,0 +1,230 @@
+/*************************************************************************/ /*!
+@File           pdumpdesc.h
+@Title          PDump Descriptor format
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Describes PDump descriptors that may be passed to the
+                extraction routines (SAB).
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PDUMPDESC_H)
+#define PDUMPDESC_H
+
+#include "pdumpdefs.h"
+
+/*
+ * Common fields
+ */
+#define HEADER_WORD0_TYPE_SHIFT				(0)
+#define HEADER_WORD0_TYPE_CLRMSK			(0xFFFFFFFFU)
+
+#define HEADER_WORD1_SIZE_SHIFT				(0)
+#define HEADER_WORD1_SIZE_CLRMSK			(0x0000FFFFU)
+#define HEADER_WORD1_VERSION_SHIFT			(16)
+#define HEADER_WORD1_VERSION_CLRMSK			(0xFFFF0000U)
+
+#define HEADER_WORD2_DATA_SIZE_SHIFT		(0)
+#define HEADER_WORD2_DATA_SIZE_CLRMSK		(0xFFFFFFFFU)
+
+
+/*
+ * The image type descriptor
+ */
+
+/*
+ * Header type (IMGBv3) - 'IMGB' in hex + VERSION 3
+ * Header size - 64 bytes
+ */
+#define IMAGE_HEADER_TYPE					(0x42474D49)
+#define IMAGE_HEADER_SIZE					(64)
+#define IMAGE_HEADER_VERSION				(3)
+
+/*
+ * Image type-specific fields
+ */
+#define IMAGE_HEADER_WORD3_LOGICAL_WIDTH_SHIFT		(0)
+#define IMAGE_HEADER_WORD3_LOGICAL_WIDTH_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_SHIFT		(0)
+#define IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD5_FORMAT_SHIFT				(0)
+#define IMAGE_HEADER_WORD5_FORMAT_CLRMSK			(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_SHIFT		(0)
+#define IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_SHIFT	(0)
+#define IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD8_TWIDDLING_SHIFT			(0)
+#define IMAGE_HEADER_WORD8_TWIDDLING_CLRMSK			(0x000000FFU)
+#define IMAGE_HEADER_WORD8_TWIDDLING_STRIDED		(0 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+#define IMAGE_HEADER_WORD8_TWIDDLING_NTWIDDLE		(9 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+#define IMAGE_HEADER_WORD8_TWIDDLING_ZTWIDDLE		(12 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+
+
+#define IMAGE_HEADER_WORD8_STRIDE_SHIFT				(8)
+#define IMAGE_HEADER_WORD8_STRIDE_CLRMSK			(0x0000FF00U)
+#define IMAGE_HEADER_WORD8_STRIDE_POSITIVE			(0 << IMAGE_HEADER_WORD8_STRIDE_SHIFT)
+#define IMAGE_HEADER_WORD8_STRIDE_NEGATIVE			(1 << IMAGE_HEADER_WORD8_STRIDE_SHIFT)
+
+#define IMAGE_HEADER_WORD8_BIFTYPE_SHIFT			(16)
+#define IMAGE_HEADER_WORD8_BIFTYPE_CLRMSK			(0x00FF0000U)
+#define IMAGE_HEADER_WORD8_BIFTYPE_NONE				(0 << IMAGE_HEADER_WORD8_BIFTYPE_SHIFT)
+
+#define IMAGE_HEADER_WORD8_FBCTYPE_SHIFT			(24)
+#define IMAGE_HEADER_WORD8_FBCTYPE_CLRMSK			(0xFF000000U)
+#define IMAGE_HEADER_WORD8_FBCTYPE_8X8				(1 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+#define IMAGE_HEADER_WORD8_FBCTYPE_16x4				(2 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+#define IMAGE_HEADER_WORD8_FBCTYPE_32x2				(3 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+
+#define IMAGE_HEADER_WORD9_FBCDECOR_SHIFT			(0)
+#define IMAGE_HEADER_WORD9_FBCDECOR_CLRMSK			(0x000000FFU)
+#define IMAGE_HEADER_WORD9_FBCDECOR_ENABLE			(1 << IMAGE_HEADER_WORD9_FBCDECOR_SHIFT)
+
+/* Align with fbcomp_export_c.h in pdump_tools branch */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT			(8)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_CLRMSK			(0x0000FF00U)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_SAME_AS_GPU	(0 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_BASE			(1 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_TWIDDLED_EN	(2 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* TWIDDLED_ENHANCED */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V2				(3 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT1   (4 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT2   (5 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* V30_WITH_HEADER_REMAP */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT1   (6 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT2   (7 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* V31_WITH_HEADER_REMAP */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V4				(8 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V4_PLUS		(9 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_TFBC			(10 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+
+#define IMAGE_HEADER_WORD9_LOSSY_SHIFT				(16)
+#define IMAGE_HEADER_WORD9_LOSSY_CLRMSK				(0x00FF0000U)
+/* Non-TFBC */
+#define IMAGE_HEADER_WORD9_LOSSY_ON					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+
+/* TFBC */
+#define IMAGE_HEADER_WORD9_LOSSY_75					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_37					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_50					(2 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_25					(3 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_OFF				(0 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+
+#define IMAGE_HEADER_WORD9_SWIZZLE_SHIFT			(24)
+#define IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK			(0xFF000000U)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ARGB		(0x0 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ARBG		(0x1 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_AGRB		(0x2 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_AGBR		(0x3 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ABGR		(0x4 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ABRG		(0x5 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_RGBA		(0x8 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_RBGA		(0x9 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_GRBA		(0xA << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_GBRA		(0xB << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_BGRA		(0xC << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_BRGA		(0xD << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+
+#define IMAGE_HEADER_WORD10_FBCCLEAR_CH0_SHIFT		(0)
+#define IMAGE_HEADER_WORD10_FBCCLEAR_CH0_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD11_FBCCLEAR_CH1_SHIFT		(0)
+#define IMAGE_HEADER_WORD11_FBCCLEAR_CH1_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD12_FBCCLEAR_CH2_SHIFT		(0)
+#define IMAGE_HEADER_WORD12_FBCCLEAR_CH2_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD13_FBCCLEAR_CH3_SHIFT		(0)
+#define IMAGE_HEADER_WORD13_FBCCLEAR_CH3_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT		(0)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_CLRMSK		(0x000000FFU)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75		(0 << IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_25_37_50		(1 << IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT)
+
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT		(8)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_CLRMSK		(0x0000FF00U)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_ALL			(0 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_CORR	(1 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY	(2 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_PTC_ONLY	(3 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_SHIFT	(16)
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_CLRMSK	(0x00FF0000U)
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_EN		(1 << IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_SHIFT) /* Treat YUV10 optimal formats as 8 bits */
+
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_SHIFT	(24)
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_CLRMSK	(0xFF000000U)
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN		(1 << IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_SHIFT) /* Override lossy min channel setting */
+
+/* IMAGE_HEADER_WORD15_RESERVED2 */
+
+/*
+ * The data type descriptor
+ */
+
+/*
+ * Header type (IMGCv1) - 'IMGC' in hex + VERSION 0
+ * Header size - 20 bytes (5 x 32 bit WORDS)
+ */
+#define DATA_HEADER_TYPE                    (0x43474D49)
+#define DATA_HEADER_SIZE                    (20)
+#define DATA_HEADER_VERSION                 (0)
+
+/*
+ * The IBIN type descriptor
+ */
+
+/*
+ * Header type (IBIN) - 'IBIN' in hex + VERSION 0
+ * Header size - 12 bytes (3 x 32 bit WORDS)
+ */
+#define IBIN_HEADER_TYPE                    (0x4e494249)
+#define IBIN_HEADER_SIZE                    (12)
+#define IBIN_HEADER_VERSION                 (0)
+
+/*
+ * Data type-specific fields
+ */
+#define DATA_HEADER_WORD3_ELEMENT_TYPE_SHIFT        (0)
+#define DATA_HEADER_WORD3_ELEMENT_TYPE_CLRMSK       (0xFFFFFFFFU)
+
+#define DATA_HEADER_WORD4_ELEMENT_COUNT_SHIFT       (0)
+#define DATA_HEADER_WORD4_ELEMENT_COUNT_CLRMSK      (0xFFFFFFFFU)
+
+#endif /* PDUMPDESC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/buffer_attribs.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/buffer_attribs.h
new file mode 100644
index 000000000000..41eaaaecd19e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/buffer_attribs.h
@@ -0,0 +1,193 @@
+/*************************************************************************/ /*!
+@File
+@Title          3D types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef POWERVR_BUFFER_ATTRIBS_H
+#define POWERVR_BUFFER_ATTRIBS_H
+
+/*!
+ * Memory layouts
+ * Defines how pixels are laid out within a surface.
+ */
+typedef enum
+{
+	IMG_MEMLAYOUT_STRIDED,       /**< Resource is strided, one row at a time */
+	IMG_MEMLAYOUT_TWIDDLED,      /**< Resource is 2D twiddled to match HW */
+	IMG_MEMLAYOUT_3DTWIDDLED,    /**< Resource is 3D twiddled, classic style */
+	IMG_MEMLAYOUT_TILED,         /**< Resource is tiled, tiling config specified elsewhere. */
+	IMG_MEMLAYOUT_PAGETILED,     /**< Resource is pagetiled */
+	IMG_MEMLAYOUT_INVNTWIDDLED,  /**< Resource is 2D twiddled !N style */
+} IMG_MEMLAYOUT;
+
+/*!
+ * Rotation types
+ */
+typedef enum
+{
+	IMG_ROTATION_0DEG = 0,
+	IMG_ROTATION_90DEG = 1,
+	IMG_ROTATION_180DEG = 2,
+	IMG_ROTATION_270DEG = 3,
+	IMG_ROTATION_FLIP_Y = 4,
+
+	IMG_ROTATION_BAD = 255,
+} IMG_ROTATION;
+
+/*!
+ * Alpha types.
+ */
+typedef enum
+{
+	IMG_COLOURSPACE_FORMAT_UNKNOWN                 =  0x0UL << 16,
+	IMG_COLOURSPACE_FORMAT_LINEAR                  =  0x1UL << 16,
+	IMG_COLOURSPACE_FORMAT_SRGB                    =  0x2UL << 16,
+	IMG_COLOURSPACE_FORMAT_SCRGB                   =  0x3UL << 16,
+	IMG_COLOURSPACE_FORMAT_SCRGB_LINEAR            =  0x4UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3_LINEAR       =  0x5UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3              =  0x6UL << 16,
+	IMG_COLOURSPACE_FORMAT_BT2020_PQ               =  0x7UL << 16,
+	IMG_COLOURSPACE_FORMAT_BT2020_LINEAR           =  0x8UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3_PASSTHROUGH  =  0x9UL << 16,
+	IMG_COLOURSPACE_FORMAT_MASK                    =  0xFUL << 16,
+} IMG_COLOURSPACE_FORMAT;
+
+/*!
+ * Determines if FB Compression is Lossy
+ */
+#define IS_FBCDC_LOSSY(mode)			((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_TRUE : IMG_FALSE)
+
+/*!
+ * Determines if FB Compression is Packed
+ */
+#define IS_FBCDC_PACKED(mode)			((mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8) ? IMG_TRUE : IMG_FALSE)
+
+/*!
+ * Returns type of FB Compression
+ */
+#define GET_FBCDC_BLOCK_TYPE(mode)		((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8)   ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : mode)
+
+/*!
+ * Adds Packing compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_PACKING(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_PACKED_8x8 : mode)
+
+/*!
+ * Removes Packing compression setting from mode
+ */
+#define FBCDC_MODE_REMOVE_PACKING(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8 : mode)
+
+/*!
+ * Adds Lossy25 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY25(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2 : mode)
+
+/*!
+ * Adds Lossy37 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY37(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2 : mode)
+
+/*!
+ * Adds Lossy50 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY50(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2 : mode)
+
+/*!
+ * Adds Lossy75 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY75(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2 : mode)
+
+/*!
+ * Removes Lossy compression setting from mode
+ */
+#define FBCDC_MODE_REMOVE_LOSSY(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : mode)
+
+/*!
+ * Types of framebuffer compression
+ */
+typedef enum
+{
+	IMG_FB_COMPRESSION_NONE,
+	IMG_FB_COMPRESSION_DIRECT_8x8,
+	IMG_FB_COMPRESSION_DIRECT_16x4,
+	IMG_FB_COMPRESSION_DIRECT_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2,
+	IMG_FB_COMPRESSION_DIRECT_PACKED_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2,
+} IMG_FB_COMPRESSION;
+
+
+#endif /* POWERVR_BUFFER_ATTRIBS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/img_drm_fourcc.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/img_drm_fourcc.h
new file mode 100644
index 000000000000..d4717db1cccb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/img_drm_fourcc.h
@@ -0,0 +1,161 @@
+/*************************************************************************/ /*!
+@File
+@Title          Wrapper around drm_fourcc.h
+@Description    FourCCs and DRM framebuffer modifiers that are not in the
+                Kernel's and libdrm's drm_fourcc.h can be added here.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DRM_FOURCC_H
+#define IMG_DRM_FOURCC_H
+
+#if defined(__KERNEL__)
+#include <drm/drm_fourcc.h>
+#else
+/*
+ * Include types.h to workaround versions of libdrm older than 2.4.68
+ * not including the correct headers.
+ */
+#include <linux/types.h>
+
+#include <drm_fourcc.h>
+#endif
+
+/*
+ * Don't get too inspired by this example :)
+ * ADF doesn't support DRM modifiers, so the memory layout had to be
+ * included in the fourcc name, but the proper way to specify information
+ * additional to pixel formats is to use DRM modifiers.
+ *
+ * See upstream drm_fourcc.h for the proper naming convention.
+ */
+#ifndef DRM_FORMAT_BGRA8888_DIRECT_16x4
+#define DRM_FORMAT_BGRA8888_DIRECT_16x4 fourcc_code('I', 'M', 'G', '0')
+#endif
+
+#if !defined(__KERNEL__)
+/*
+ * A definition for the same format was added in Linux kernel 5.2 in commit
+ * 88ab9c76d191ad8645b483f31e2b394b0f3e280e. As such, this definition has been
+ * deprecated and the DRM_FORMAT_ABGR16161616F kernel define should be used
+ * instead of this one.
+ */
+#define DRM_FORMAT_ABGR16_IMG_DEPRECATED fourcc_code('I', 'M', 'G', '1')
+#endif
+
+/*
+ * Upstream does not have a packed 10 Bits Per Channel YVU format yet,
+ * so let`s make one up.
+ * Note: at the moment this format is not intended to be used with
+ *       a framebuffer, so the kernels core DRM doesn`t need to know
+ *       about this format. This means that the kernel doesn`t need
+ *       to be patched.
+ */
+#if !defined(__KERNEL__)
+#define DRM_FORMAT_YVU444_PACK10_IMG fourcc_code('I', 'M', 'G', '2')
+#define DRM_FORMAT_YUV422_2PLANE_PACK10_IMG fourcc_code('I', 'M', 'G', '3')
+#define DRM_FORMAT_YUV420_2PLANE_PACK10_IMG fourcc_code('I', 'M', 'G', '4')
+#endif
+
+/*
+ * Value chosen in the middle of 255 pool to minimise the chance of hitting
+ * the same value potentially defined by other vendors in the drm_fourcc.h
+ */
+#define DRM_FORMAT_MOD_VENDOR_PVR 0x92
+
+#ifndef DRM_FORMAT_MOD_VENDOR_NONE
+#define DRM_FORMAT_MOD_VENDOR_NONE 0
+#endif
+
+#ifndef DRM_FORMAT_RESERVED
+#define DRM_FORMAT_RESERVED ((1ULL << 56) - 1)
+#endif
+
+#define img_fourcc_mod_combine(uiModHi, uiModLo) \
+	((__u64) ((__u32) (uiModHi)) << 32 | (__u64) ((__u32) (uiModLo)))
+
+#define img_fourcc_mod_hi(ui64Mod) \
+	((__u32) ((__u64) (ui64Mod) >> 32))
+
+#define img_fourcc_mod_lo(ui64Mod) \
+	((__u32) ((__u64) (ui64Mod)) & 0xffffffff)
+
+#ifndef fourcc_mod_code
+#define fourcc_mod_code(vendor, val) \
+	((((__u64)DRM_FORMAT_MOD_VENDOR_## vendor) << 56) | (val & 0x00ffffffffffffffULL))
+#endif
+
+#ifndef DRM_FORMAT_MOD_INVALID
+#define DRM_FORMAT_MOD_INVALID fourcc_mod_code(NONE, DRM_FORMAT_RESERVED)
+#endif
+
+#ifndef DRM_FORMAT_MOD_LINEAR
+#define DRM_FORMAT_MOD_LINEAR fourcc_mod_code(NONE, 0)
+#endif
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V1      fourcc_mod_code(PVR, 3)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V1     fourcc_mod_code(PVR, 9)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V7      fourcc_mod_code(PVR, 6)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V7     fourcc_mod_code(PVR, 12)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V10     fourcc_mod_code(PVR, 21)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V10    fourcc_mod_code(PVR, 22)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V10    fourcc_mod_code(PVR, 23)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12     fourcc_mod_code(PVR, 15)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12    fourcc_mod_code(PVR, 16)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V13             fourcc_mod_code(PVR, 24)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V13     fourcc_mod_code(PVR, 25)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V13     fourcc_mod_code(PVR, 26)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V13     fourcc_mod_code(PVR, 27)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V13            fourcc_mod_code(PVR, 28)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V13    fourcc_mod_code(PVR, 29)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V13    fourcc_mod_code(PVR, 30)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V13    fourcc_mod_code(PVR, 31)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V13     fourcc_mod_code(PVR, 32)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V13    fourcc_mod_code(PVR, 33)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V14             fourcc_mod_code(PVR, 34)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V14     fourcc_mod_code(PVR, 35)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V14     fourcc_mod_code(PVR, 36)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V14     fourcc_mod_code(PVR, 37)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V14     fourcc_mod_code(PVR, 38)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V14            fourcc_mod_code(PVR, 39)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V14    fourcc_mod_code(PVR, 40)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V14    fourcc_mod_code(PVR, 41)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V14    fourcc_mod_code(PVR, 42)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V14    fourcc_mod_code(PVR, 43)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V14            fourcc_mod_code(PVR, 44)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_32x2_V14    fourcc_mod_code(PVR, 45)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_32x2_V14    fourcc_mod_code(PVR, 46)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_32x2_V14    fourcc_mod_code(PVR, 47)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_32x2_V14    fourcc_mod_code(PVR, 48)
+
+#endif /* IMG_DRM_FOURCC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/imgpixfmts.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/imgpixfmts.h
new file mode 100644
index 000000000000..127752640e92
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/imgpixfmts.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@File           imgpixfmts.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ **
+ ** WARNING: File is autogenerated by parsesystable.py - DO NOT EDIT.
+ ** Use fmts_systable.txt to add new formats.
+ **
+ *****************************************************************************/
+
+#if !defined(IMGPIXFMTS_H)
+#define IMGPIXFMTS_H
+
+typedef enum _IMG_PIXFMT_
+{
+	IMG_PIXFMT_UNKNOWN = 0,
+	IMG_PIXFMT_RESERVED_1 = 1,
+	IMG_PIXFMT_RESERVED_2 = 2,
+	IMG_PIXFMT_RESERVED_3 = 3,
+	IMG_PIXFMT_RESERVED_4 = 4,
+	IMG_PIXFMT_RESERVED_5 = 5,
+	IMG_PIXFMT_RESERVED_6 = 6,
+	IMG_PIXFMT_RESERVED_7 = 7,
+	IMG_PIXFMT_RESERVED_8 = 8,
+	IMG_PIXFMT_RESERVED_9 = 9,
+	IMG_PIXFMT_RESERVED_10 = 10,
+	IMG_PIXFMT_RESERVED_11 = 11,
+	IMG_PIXFMT_RESERVED_12 = 12,
+	IMG_PIXFMT_RESERVED_13 = 13,
+	IMG_PIXFMT_RESERVED_14 = 14,
+	IMG_PIXFMT_RESERVED_15 = 15,
+	IMG_PIXFMT_RESERVED_16 = 16,
+	IMG_PIXFMT_RESERVED_17 = 17,
+	IMG_PIXFMT_RESERVED_18 = 18,
+	IMG_PIXFMT_RESERVED_19 = 19,
+	IMG_PIXFMT_RESERVED_20 = 20,
+	IMG_PIXFMT_RESERVED_21 = 21,
+	IMG_PIXFMT_RESERVED_22 = 22,
+	IMG_PIXFMT_RESERVED_23 = 23,
+	IMG_PIXFMT_RESERVED_24 = 24,
+	IMG_PIXFMT_R10G10B10A2_UNORM = 25,
+	IMG_PIXFMT_RESERVED_26 = 26,
+	IMG_PIXFMT_RESERVED_27 = 27,
+	IMG_PIXFMT_B10G10R10A2_UNORM = 28,
+	IMG_PIXFMT_RESERVED_29 = 29,
+	IMG_PIXFMT_RESERVED_30 = 30,
+	IMG_PIXFMT_RESERVED_31 = 31,
+	IMG_PIXFMT_R8G8B8A8_UNORM = 32,
+	IMG_PIXFMT_R8G8B8A8_UNORM_SRGB = 33,
+	IMG_PIXFMT_RESERVED_34 = 34,
+	IMG_PIXFMT_RESERVED_35 = 35,
+	IMG_PIXFMT_RESERVED_36 = 36,
+	IMG_PIXFMT_R8G8B8X8_UNORM = 37,
+	IMG_PIXFMT_RESERVED_38 = 38,
+	IMG_PIXFMT_RESERVED_39 = 39,
+	IMG_PIXFMT_RESERVED_40 = 40,
+	IMG_PIXFMT_RESERVED_41 = 41,
+	IMG_PIXFMT_RESERVED_42 = 42,
+	IMG_PIXFMT_RESERVED_43 = 43,
+	IMG_PIXFMT_RESERVED_44 = 44,
+	IMG_PIXFMT_RESERVED_45 = 45,
+	IMG_PIXFMT_RESERVED_46 = 46,
+	IMG_PIXFMT_RESERVED_47 = 47,
+	IMG_PIXFMT_RESERVED_48 = 48,
+	IMG_PIXFMT_RESERVED_49 = 49,
+	IMG_PIXFMT_RESERVED_50 = 50,
+	IMG_PIXFMT_RESERVED_51 = 51,
+	IMG_PIXFMT_D32_FLOAT = 52,
+	IMG_PIXFMT_RESERVED_53 = 53,
+	IMG_PIXFMT_RESERVED_54 = 54,
+	IMG_PIXFMT_RESERVED_55 = 55,
+	IMG_PIXFMT_RESERVED_56 = 56,
+	IMG_PIXFMT_RESERVED_57 = 57,
+	IMG_PIXFMT_RESERVED_58 = 58,
+	IMG_PIXFMT_D24_UNORM_X8_TYPELESS = 59,
+	IMG_PIXFMT_RESERVED_60 = 60,
+	IMG_PIXFMT_RESERVED_61 = 61,
+	IMG_PIXFMT_RESERVED_62 = 62,
+	IMG_PIXFMT_R8G8_UNORM = 63,
+	IMG_PIXFMT_RESERVED_64 = 64,
+	IMG_PIXFMT_RESERVED_65 = 65,
+	IMG_PIXFMT_RESERVED_66 = 66,
+	IMG_PIXFMT_RESERVED_67 = 67,
+	IMG_PIXFMT_RESERVED_68 = 68,
+	IMG_PIXFMT_RESERVED_69 = 69,
+	IMG_PIXFMT_D16_UNORM = 70,
+	IMG_PIXFMT_RESERVED_71 = 71,
+	IMG_PIXFMT_RESERVED_72 = 72,
+	IMG_PIXFMT_RESERVED_73 = 73,
+	IMG_PIXFMT_RESERVED_74 = 74,
+	IMG_PIXFMT_RESERVED_75 = 75,
+	IMG_PIXFMT_RESERVED_76 = 76,
+	IMG_PIXFMT_R8_UNORM = 77,
+	IMG_PIXFMT_RESERVED_78 = 78,
+	IMG_PIXFMT_RESERVED_79 = 79,
+	IMG_PIXFMT_RESERVED_80 = 80,
+	IMG_PIXFMT_RESERVED_81 = 81,
+	IMG_PIXFMT_S8_UINT = 82,
+	IMG_PIXFMT_RESERVED_83 = 83,
+	IMG_PIXFMT_RESERVED_84 = 84,
+	IMG_PIXFMT_RESERVED_85 = 85,
+	IMG_PIXFMT_B5G6R5_UNORM = 86,
+	IMG_PIXFMT_R5G6B5_UNORM = 87,
+	IMG_PIXFMT_B5G5R5A1_UNORM = 88,
+	IMG_PIXFMT_B5G5R5X1_UNORM = 89,
+	IMG_PIXFMT_B8G8R8A8_UNORM = 90,
+	IMG_PIXFMT_B8G8R8X8_UNORM = 91,
+	IMG_PIXFMT_B8G8R8A8_UINT = 92,
+	IMG_PIXFMT_B8G8R8A8_SNORM = 93,
+	IMG_PIXFMT_B8G8R8A8_SINT = 94,
+	IMG_PIXFMT_B8G8R8A8_UNORM_SRGB = 95,
+	IMG_PIXFMT_RESERVED_96 = 96,
+	IMG_PIXFMT_RESERVED_97 = 97,
+	IMG_PIXFMT_RESERVED_98 = 98,
+	IMG_PIXFMT_RESERVED_99 = 99,
+	IMG_PIXFMT_RESERVED_100 = 100,
+	IMG_PIXFMT_RESERVED_101 = 101,
+	IMG_PIXFMT_RESERVED_102 = 102,
+	IMG_PIXFMT_RESERVED_103 = 103,
+	IMG_PIXFMT_RESERVED_104 = 104,
+	IMG_PIXFMT_RESERVED_105 = 105,
+	IMG_PIXFMT_RESERVED_106 = 106,
+	IMG_PIXFMT_RESERVED_107 = 107,
+	IMG_PIXFMT_RESERVED_108 = 108,
+	IMG_PIXFMT_RESERVED_109 = 109,
+	IMG_PIXFMT_RESERVED_110 = 110,
+	IMG_PIXFMT_RESERVED_111 = 111,
+	IMG_PIXFMT_RESERVED_112 = 112,
+	IMG_PIXFMT_RESERVED_113 = 113,
+	IMG_PIXFMT_RESERVED_114 = 114,
+	IMG_PIXFMT_RESERVED_115 = 115,
+	IMG_PIXFMT_RESERVED_116 = 116,
+	IMG_PIXFMT_RESERVED_117 = 117,
+	IMG_PIXFMT_RESERVED_118 = 118,
+	IMG_PIXFMT_RESERVED_119 = 119,
+	IMG_PIXFMT_RESERVED_120 = 120,
+	IMG_PIXFMT_RESERVED_121 = 121,
+	IMG_PIXFMT_RESERVED_122 = 122,
+	IMG_PIXFMT_RESERVED_123 = 123,
+	IMG_PIXFMT_RESERVED_124 = 124,
+	IMG_PIXFMT_RESERVED_125 = 125,
+	IMG_PIXFMT_RESERVED_126 = 126,
+	IMG_PIXFMT_RESERVED_127 = 127,
+	IMG_PIXFMT_RESERVED_128 = 128,
+	IMG_PIXFMT_RESERVED_129 = 129,
+	IMG_PIXFMT_RESERVED_130 = 130,
+	IMG_PIXFMT_RESERVED_131 = 131,
+	IMG_PIXFMT_RESERVED_132 = 132,
+	IMG_PIXFMT_RESERVED_133 = 133,
+	IMG_PIXFMT_RESERVED_134 = 134,
+	IMG_PIXFMT_RESERVED_135 = 135,
+	IMG_PIXFMT_RESERVED_136 = 136,
+	IMG_PIXFMT_RESERVED_137 = 137,
+	IMG_PIXFMT_RESERVED_138 = 138,
+	IMG_PIXFMT_L8_UNORM = 139,
+	IMG_PIXFMT_RESERVED_140 = 140,
+	IMG_PIXFMT_L8A8_UNORM = 141,
+	IMG_PIXFMT_RESERVED_142 = 142,
+	IMG_PIXFMT_RESERVED_143 = 143,
+	IMG_PIXFMT_RESERVED_144 = 144,
+	IMG_PIXFMT_RESERVED_145 = 145,
+	IMG_PIXFMT_RESERVED_146 = 146,
+	IMG_PIXFMT_RESERVED_147 = 147,
+	IMG_PIXFMT_B4G4R4A4_UNORM = 148,
+	IMG_PIXFMT_RESERVED_149 = 149,
+	IMG_PIXFMT_RESERVED_150 = 150,
+	IMG_PIXFMT_RESERVED_151 = 151,
+	IMG_PIXFMT_RESERVED_152 = 152,
+	IMG_PIXFMT_RESERVED_153 = 153,
+	IMG_PIXFMT_RESERVED_154 = 154,
+	IMG_PIXFMT_RESERVED_155 = 155,
+	IMG_PIXFMT_RESERVED_156 = 156,
+	IMG_PIXFMT_RESERVED_157 = 157,
+	IMG_PIXFMT_RESERVED_158 = 158,
+	IMG_PIXFMT_RESERVED_159 = 159,
+	IMG_PIXFMT_RESERVED_160 = 160,
+	IMG_PIXFMT_RESERVED_161 = 161,
+	IMG_PIXFMT_RESERVED_162 = 162,
+	IMG_PIXFMT_R8G8B8_UNORM = 163,
+	IMG_PIXFMT_R8G8B8_UNORM_SRGB = 164,
+	IMG_PIXFMT_RESERVED_165 = 165,
+	IMG_PIXFMT_RESERVED_166 = 166,
+	IMG_PIXFMT_RESERVED_167 = 167,
+	IMG_PIXFMT_RESERVED_168 = 168,
+	IMG_PIXFMT_RESERVED_169 = 169,
+	IMG_PIXFMT_RESERVED_170 = 170,
+	IMG_PIXFMT_RESERVED_171 = 171,
+	IMG_PIXFMT_RESERVED_172 = 172,
+	IMG_PIXFMT_RESERVED_173 = 173,
+	IMG_PIXFMT_UYVY = 174,
+	IMG_PIXFMT_VYUY = 175,
+	IMG_PIXFMT_YUYV = 176,
+	IMG_PIXFMT_YVYU = 177,
+	IMG_PIXFMT_YVU420_2PLANE = 178,
+	IMG_PIXFMT_YUV420_2PLANE = 179,
+	IMG_PIXFMT_YVU420_2PLANE_MACRO_BLOCK = 180,
+	IMG_PIXFMT_YUV420_3PLANE = 181,
+	IMG_PIXFMT_YVU420_3PLANE = 182,
+	IMG_PIXFMT_RESERVED_183 = 183,
+	IMG_PIXFMT_RESERVED_184 = 184,
+	IMG_PIXFMT_RESERVED_185 = 185,
+	IMG_PIXFMT_RESERVED_186 = 186,
+	IMG_PIXFMT_RESERVED_187 = 187,
+	IMG_PIXFMT_RESERVED_188 = 188,
+	IMG_PIXFMT_RESERVED_189 = 189,
+	IMG_PIXFMT_RESERVED_190 = 190,
+	IMG_PIXFMT_RESERVED_191 = 191,
+	IMG_PIXFMT_RESERVED_192 = 192,
+	IMG_PIXFMT_RESERVED_193 = 193,
+	IMG_PIXFMT_RESERVED_194 = 194,
+	IMG_PIXFMT_RESERVED_195 = 195,
+	IMG_PIXFMT_RESERVED_196 = 196,
+	IMG_PIXFMT_RESERVED_197 = 197,
+	IMG_PIXFMT_RESERVED_198 = 198,
+	IMG_PIXFMT_RESERVED_199 = 199,
+	IMG_PIXFMT_RESERVED_200 = 200,
+	IMG_PIXFMT_RESERVED_201 = 201,
+	IMG_PIXFMT_RESERVED_202 = 202,
+	IMG_PIXFMT_RESERVED_203 = 203,
+	IMG_PIXFMT_YVU8_422_2PLANE_PACK8 = 204,
+	IMG_PIXFMT_RESERVED_205 = 205,
+	IMG_PIXFMT_YVU10_444_1PLANE_PACK10 = 206,
+	IMG_PIXFMT_YVU10_420_2PLANE_PACK16 = 207,
+	IMG_PIXFMT_RESERVED_208 = 208,
+	IMG_PIXFMT_RESERVED_209 = 209,
+	IMG_PIXFMT_YUV8_422_2PLANE_PACK8 = 210,
+	IMG_PIXFMT_YUV8_444_3PLANE_PACK8 = 211,
+	IMG_PIXFMT_RESERVED_212 = 212,
+	IMG_PIXFMT_RESERVED_213 = 213,
+	IMG_PIXFMT_RESERVED_214 = 214,
+	IMG_PIXFMT_RESERVED_215 = 215,
+	IMG_PIXFMT_RESERVED_216 = 216,
+	IMG_PIXFMT_RESERVED_217 = 217,
+	IMG_PIXFMT_RESERVED_218 = 218,
+	IMG_PIXFMT_RESERVED_219 = 219,
+	IMG_PIXFMT_RESERVED_220 = 220,
+	IMG_PIXFMT_RESERVED_221 = 221,
+	IMG_PIXFMT_RESERVED_222 = 222,
+	IMG_PIXFMT_RESERVED_223 = 223,
+	IMG_PIXFMT_RESERVED_224 = 224,
+	IMG_PIXFMT_RESERVED_225 = 225,
+	IMG_PIXFMT_RESERVED_226 = 226,
+	IMG_PIXFMT_RESERVED_227 = 227,
+	IMG_PIXFMT_RESERVED_228 = 228,
+	IMG_PIXFMT_RESERVED_229 = 229,
+	IMG_PIXFMT_RESERVED_230 = 230,
+	IMG_PIXFMT_RESERVED_231 = 231,
+	IMG_PIXFMT_RESERVED_232 = 232,
+	IMG_PIXFMT_RESERVED_233 = 233,
+	IMG_PIXFMT_RESERVED_234 = 234,
+	IMG_PIXFMT_RESERVED_235 = 235,
+	IMG_PIXFMT_RESERVED_236 = 236,
+	IMG_PIXFMT_RESERVED_237 = 237,
+	IMG_PIXFMT_RESERVED_238 = 238,
+	IMG_PIXFMT_RESERVED_239 = 239,
+	IMG_PIXFMT_RESERVED_240 = 240,
+	IMG_PIXFMT_RESERVED_241 = 241,
+	IMG_PIXFMT_RESERVED_242 = 242,
+	IMG_PIXFMT_RESERVED_243 = 243,
+	IMG_PIXFMT_RESERVED_244 = 244,
+	IMG_PIXFMT_RESERVED_245 = 245,
+	IMG_PIXFMT_RESERVED_246 = 246,
+	IMG_PIXFMT_RESERVED_247 = 247,
+	IMG_PIXFMT_YVU8_420_2PLANE_PACK8_P = 248,
+	IMG_PIXFMT_RESERVED_249 = 249,
+	IMG_PIXFMT_RESERVED_250 = 250,
+	IMG_PIXFMT_RESERVED_251 = 251,
+	IMG_PIXFMT_YUV8_420_2PLANE_PACK8_P = 252,
+	IMG_PIXFMT_RESERVED_253 = 253,
+	IMG_PIXFMT_RESERVED_254 = 254,
+	IMG_PIXFMT_UYVY10_422_1PLANE_PACK10_CUST1 = 255,
+	IMG_PIXFMT_RESERVED_256 = 256,
+	IMG_PIXFMT_RESERVED_257 = 257,
+	IMG_PIXFMT_RESERVED_258 = 258,
+	IMG_PIXFMT_RESERVED_259 = 259,
+	IMG_PIXFMT_RESERVED_260 = 260,
+	IMG_PIXFMT_RESERVED_261 = 261,
+	IMG_PIXFMT_RESERVED_262 = 262,
+	IMG_PIXFMT_RESERVED_263 = 263,
+	IMG_PIXFMT_RESERVED_264 = 264,
+	IMG_PIXFMT_RESERVED_265 = 265,
+	IMG_PIXFMT_RESERVED_266 = 266,
+	IMG_PIXFMT_RESERVED_267 = 267,
+	IMG_PIXFMT_R10G10B10X2_UNORM = 268,
+#define IMG_PIXFMT_ENUM_COUNT 269
+} IMG_PIXFMT;
+
+#endif /* IMGPIXFMTS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/mem_types.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/mem_types.h
new file mode 100644
index 000000000000..a6dce8fe9889
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/mem_types.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File
+@Title          Public types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef POWERVR_TYPES_H
+#define POWERVR_TYPES_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(_MSC_VER)
+	#include "msvc_types.h"
+#elif defined(__linux__) && defined(__KERNEL__)
+	#include <linux/types.h>
+	#include <linux/compiler.h>
+#else
+	#include <stdint.h>
+	#define __iomem
+#endif
+
+typedef void *IMG_CPU_VIRTADDR;
+
+/* device virtual address */
+typedef struct
+{
+	uint64_t uiAddr;
+#define IMG_CAST_TO_DEVVADDR_UINT(var)		(uint64_t)(var)
+
+} IMG_DEV_VIRTADDR;
+
+typedef uint64_t IMG_DEVMEM_SIZE_T;
+typedef uint64_t IMG_DEVMEM_ALIGN_T;
+typedef uint64_t IMG_DEVMEM_OFFSET_T;
+typedef uint32_t IMG_DEVMEM_LOG2ALIGN_T;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/public/powervr/pvrsrv_sync_ext.h b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/pvrsrv_sync_ext.h
new file mode 100644
index 000000000000..933212e03c17
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/public/powervr/pvrsrv_sync_ext.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services external synchronisation interface header
+@Description    Defines synchronisation structures that are visible internally
+                and externally
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef POWERVR_SYNC_EXT_H
+#define POWERVR_SYNC_EXT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*!
+ * Number of sync prims still used internally in operations
+ */
+#define PVRSRV_MAX_SYNC_PRIMS 4U
+
+/*!
+ * Maximum number of dev var updates passed in a kick call
+ */
+#define PVRSRV_MAX_DEV_VARS 13U
+
+/*!
+ * Number of UFOs in operations
+ */
+#define	PVRSRV_MAX_SYNCS (PVRSRV_MAX_SYNC_PRIMS + PVRSRV_MAX_DEV_VARS)
+
+/*! Implementation independent types for passing fence/timeline to Services.
+ */
+typedef int32_t PVRSRV_FENCE;
+typedef int32_t PVRSRV_TIMELINE;
+typedef int32_t PVRSRV_EXP_FENCE_CTX;
+
+/*! Maximum length for an annotation name string for fence sync model objects.
+ */
+#define PVRSRV_SYNC_NAME_LENGTH 32U
+
+/* Macros for API callers using the fence sync model
+ */
+#define PVRSRV_NO_TIMELINE      ((PVRSRV_TIMELINE) -1)
+#define PVRSRV_NO_FENCE         ((PVRSRV_FENCE)    -1)
+#define PVRSRV_NO_EXP_FENCE_CTX ((PVRSRV_EXP_FENCE_CTX) -1)
+#define PVRSRV_NO_FENCE_PTR     NULL
+#define PVRSRV_NO_TIMELINE_PTR  NULL
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_buffer_sync_shared.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_buffer_sync_shared.h
new file mode 100644
index 000000000000..7a110910dbd6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_buffer_sync_shared.h
@@ -0,0 +1,57 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR buffer sync shared
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared definitions between client and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BUFFER_SYNC_SHARED_H
+#define PVR_BUFFER_SYNC_SHARED_H
+
+#define PVR_BUFFER_FLAG_READ		(1U << 0)
+#define PVR_BUFFER_FLAG_WRITE		(1U << 1)
+#define PVR_BUFFER_FLAG_MASK		(PVR_BUFFER_FLAG_READ | \
+									 PVR_BUFFER_FLAG_WRITE)
+
+/* Maximum number of PMRs passed
+ * in a kick when using buffer sync
+ */
+#define PVRSRV_MAX_BUFFERSYNC_PMRS 32
+
+#endif /* PVR_BUFFER_SYNC_SHARED_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_debug.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_debug.h
new file mode 100644
index 000000000000..46033a0da47e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_debug.h
@@ -0,0 +1,1071 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Debug Declarations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides debug functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DEBUG_H
+#define PVR_DEBUG_H
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/* If the kernel pre defined macro is present, we will use the definition of HTBLOGK
+ * in htbserver.h with the format strings specified in ht_buffer_sf.h. Otherwise,
+ * HTBLOGK needs to be defined as empty since it is not used by user-mode code.
+ */
+#if defined(__KERNEL__)
+#	include "htbserver.h"
+#	include "htbuffer_sf.h"
+#else
+#	define HTBLOGK(...) ((void) 0)
+#endif
+
+/*! @cond Doxygen_Suppress */
+#if defined(_MSC_VER)
+#	define MSC_SUPPRESS_4127 __pragma(warning(suppress:4127))
+#else
+#	define MSC_SUPPRESS_4127
+#endif
+/*! @endcond */
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ && __KERNEL__*/
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define PVR_MAX_DEBUG_MESSAGE_LEN	(512)   /*!< Max length of a Debug Message */
+
+/* These are privately used by pvr_debug, use the PVR_DBG_ defines instead */
+#define DBGPRIV_FATAL     0x001U  /*!< Debug-Fatal. Privately used by pvr_debug. */
+#define DBGPRIV_ERROR     0x002U  /*!< Debug-Error. Privately used by pvr_debug. */
+#define DBGPRIV_WARNING   0x004U  /*!< Debug-Warning. Privately used by pvr_debug. */
+#define DBGPRIV_MESSAGE   0x008U  /*!< Debug-Message. Privately used by pvr_debug. */
+#define DBGPRIV_VERBOSE   0x010U  /*!< Debug-Verbose. Privately used by pvr_debug. */
+#define DBGPRIV_CALLTRACE 0x020U  /*!< Debug-CallTrace. Privately used by pvr_debug. */
+#define DBGPRIV_ALLOC     0x040U  /*!< Debug-Alloc. Privately used by pvr_debug. */
+#define DBGPRIV_BUFFERED  0x080U  /*!< Debug-Buffered. Privately used by pvr_debug. */
+#define DBGPRIV_DEBUG     0x100U  /*!< Debug-AdHoc-Debug. Never submitted. Privately used by pvr_debug. */
+#define DBGPRIV_LAST      0x100U  /*!< Always set to highest mask value. Privately used by pvr_debug. */
+
+/* Enable DPF logging for locally from some make targets */
+#if defined(PVRSRV_NEED_PVR_DPF_LOCAL)
+#undef PVRSRV_NEED_PVR_DPF
+#define PVRSRV_NEED_PVR_DPF
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_ASSERT) && defined(DEBUG)
+#define PVRSRV_NEED_PVR_ASSERT
+#endif
+
+#if defined(PVRSRV_NEED_PVR_ASSERT) && !defined(PVRSRV_NEED_PVR_DPF)
+#define PVRSRV_NEED_PVR_DPF
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_TRACE) && (defined(DEBUG) || defined(TIMING))
+#define PVRSRV_NEED_PVR_TRACE
+#endif
+
+#if !defined(DOXYGEN)
+/*************************************************************************/ /*
+PVRSRVGetErrorString
+Returns a string describing the provided PVRSRV_ERROR code
+NB No doxygen comments provided as this function does not require porting
+   for other operating systems
+*/ /**************************************************************************/
+const IMG_CHAR *PVRSRVGetErrorString(PVRSRV_ERROR eError);
+#define PVRSRVGETERRORSTRING PVRSRVGetErrorString
+#endif
+
+/* PVR_ASSERT() and PVR_DBG_BREAK handling */
+
+#if defined(__KLOCWORK__)
+/* A dummy no-return function to be used under Klocwork to mark unreachable
+   paths instead of abort() in order to avoid MISRA.STDLIB.ABORT issues. */
+__noreturn void klocwork_abort(void);
+#endif
+
+#if defined(PVRSRV_NEED_PVR_ASSERT) || defined(DOXYGEN)
+
+/* Unfortunately the Klocwork static analysis checker doesn't understand our
+ * ASSERT macros. Thus it reports lots of false positive. Defining our Assert
+ * macros in a special way when the code is analysed by Klocwork avoids
+ * them.
+ */
+#if defined(__KLOCWORK__)
+#define PVR_ASSERT(x) do { if (!(x)) {klocwork_abort();} } while (false)
+#else /* ! __KLOCWORKS__ */
+
+#if defined(_WIN32)
+#define PVR_ASSERT(expr) do										\
+	{															\
+		MSC_SUPPRESS_4127										\
+		if (unlikely(!(expr)))								\
+		{														\
+			PVRSRVDebugPrintf(DBGPRIV_FATAL, __FILE__, __LINE__,\
+					  "*** Debug assertion failed!");			\
+			__debugbreak();										\
+		}														\
+	MSC_SUPPRESS_4127											\
+	} while (false)
+
+#else
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include <linux/kernel.h>
+#include <linux/bug.h>
+
+/* In Linux kernel mode, use WARN_ON() directly. This produces the
+ * correct filename and line number in the warning message.
+ */
+#define PVR_ASSERT(EXPR) do											\
+	{																\
+		if (unlikely(!(EXPR)))										\
+		{															\
+			PVRSRVDebugPrintf(DBGPRIV_FATAL, __FILE__, __LINE__,	\
+							  "Debug assertion failed!");			\
+			WARN_ON(1);												\
+		}															\
+	} while (false)
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugAssertFail
+@Description    Indicate to the user that a debug assertion has failed and
+                prevent the program from continuing.
+                Invoked from the macro PVR_ASSERT().
+@Input          pszFile       The name of the source file where the assertion failed
+@Input          ui32Line      The line number of the failed assertion
+@Input          pszAssertion  String describing the assertion
+@Return         NEVER!
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV __noreturn
+PVRSRVDebugAssertFail(const IMG_CHAR *pszFile,
+                      IMG_UINT32 ui32Line,
+                      const IMG_CHAR *pszAssertion);
+
+#define PVR_ASSERT(EXPR) do										\
+	{															\
+		if (unlikely(!(EXPR)))									\
+		{														\
+			PVRSRVDebugAssertFail(__FILE__, __LINE__, #EXPR);	\
+		}														\
+	} while (false)
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+#endif /* defined(_WIN32) */
+#endif /* defined(__KLOCWORK__) */
+
+#if defined(__KLOCWORK__)
+	#define PVR_DBG_BREAK do { klocwork_abort(); } while (false)
+#else
+	#if defined(WIN32)
+		#define PVR_DBG_BREAK __debugbreak()   /*!< Implementation of PVR_DBG_BREAK for (non-WinCE) Win32 */
+	#else
+		#if defined(PVR_DBG_BREAK_ASSERT_FAIL)
+		/*!< Implementation of PVR_DBG_BREAK that maps onto PVRSRVDebugAssertFail */
+			#if defined(_WIN32)
+				#define PVR_DBG_BREAK	DBG_BREAK
+			#else
+				#if defined(__linux__) && defined(__KERNEL__)
+					#define PVR_DBG_BREAK BUG()
+				#else
+					#define PVR_DBG_BREAK	PVRSRVDebugAssertFail(__FILE__, __LINE__, "PVR_DBG_BREAK")
+				#endif
+			#endif
+		#else
+			/*!< Null Implementation of PVR_DBG_BREAK (does nothing) */
+			#define PVR_DBG_BREAK
+		#endif
+	#endif
+#endif
+
+
+#else /* defined(PVRSRV_NEED_PVR_ASSERT) */
+	/* Unfortunately the Klocwork static analysis checker doesn't understand our
+	* ASSERT macros. Thus it reports lots of false positive. Defining our Assert
+	* macros in a special way when the code is analysed by Klocwork avoids
+	* them.
+	*/
+	#if defined(__KLOCWORK__)
+		#define PVR_ASSERT(EXPR) do { if (!(EXPR)) {klocwork_abort();} } while (false)
+	#else
+		#define PVR_ASSERT(EXPR) (void)(EXPR) /*!< Null Implementation of PVR_ASSERT (does nothing) */
+	#endif
+
+	#define PVR_DBG_BREAK    /*!< Null Implementation of PVR_DBG_BREAK (does nothing) */
+
+#endif /* defined(PVRSRV_NEED_PVR_ASSERT) */
+
+
+/* PVR_DPF() handling */
+
+#if defined(PVRSRV_NEED_PVR_DPF) || defined(DOXYGEN)
+
+	/* New logging mechanism */
+	#define PVR_DBG_FATAL     DBGPRIV_FATAL     /*!< Debug level passed to PVRSRVDebugPrintf() for fatal errors. */
+	#define PVR_DBG_ERROR     DBGPRIV_ERROR     /*!< Debug level passed to PVRSRVDebugPrintf() for non-fatal errors. */
+	#define PVR_DBG_WARNING   DBGPRIV_WARNING   /*!< Debug level passed to PVRSRVDebugPrintf() for warnings. */
+	#define PVR_DBG_MESSAGE   DBGPRIV_MESSAGE   /*!< Debug level passed to PVRSRVDebugPrintf() for information only. */
+	#define PVR_DBG_VERBOSE   DBGPRIV_VERBOSE   /*!< Debug level passed to PVRSRVDebugPrintf() for very low-priority debug. */
+	#define PVR_DBG_CALLTRACE DBGPRIV_CALLTRACE /*!< Debug level passed to PVRSRVDebugPrintf() for function tracing purposes. */
+	#define PVR_DBG_ALLOC     DBGPRIV_ALLOC     /*!< Debug level passed to PVRSRVDebugPrintf() for tracking some of drivers memory operations. */
+	#define PVR_DBG_BUFFERED  DBGPRIV_BUFFERED  /*!< Debug level passed to PVRSRVDebugPrintf() when debug should be written to the debug circular buffer. */
+	#define PVR_DBG_DEBUG     DBGPRIV_DEBUG     /*!< Debug level passed to PVRSRVDebugPrintf() for debug messages. */
+
+	/* These levels are always on with PVRSRV_NEED_PVR_DPF */
+	/*! @cond Doxygen_Suppress */
+	#define PVR_DPF_0x001U(...) PVRSRVDebugPrintf(DBGPRIV_FATAL, __VA_ARGS__)
+	#define PVR_DPF_0x002U(...) PVRSRVDebugPrintf(DBGPRIV_ERROR, __VA_ARGS__)
+	#define PVR_DPF_0x080U(...) PVRSRVDebugPrintf(DBGPRIV_BUFFERED, __VA_ARGS__)
+
+	/*
+	 * The AdHoc-Debug level is only supported when enabled in the local
+	 * build environment and may need to be used in both debug and release
+	 * builds. An error is generated in the formal build if it is checked in.
+	 */
+#if defined(PVR_DPF_ADHOC_DEBUG_ON)
+	#define PVR_DPF_0x100U(...) PVRSRVDebugPrintf(DBGPRIV_DEBUG, __VA_ARGS__)
+#else
+	/* Use an undefined token here to stop compilation dead in the offending module */
+	#define PVR_DPF_0x100U(...) __ERROR__PVR_DBG_DEBUG_is_in_use_but_has_not_been_enabled__Note_Debug_DPF_must_not_be_checked_in__Define_PVR_DPF_ADHOC_DEBUG_ON_for_testing
+#endif
+
+	/* Some are compiled out completely in release builds */
+#if defined(DEBUG) || defined(DOXYGEN)
+	#define PVR_DPF_0x004U(...) PVRSRVDebugPrintf(DBGPRIV_WARNING, __VA_ARGS__)
+	#define PVR_DPF_0x008U(...) PVRSRVDebugPrintf(DBGPRIV_MESSAGE, __VA_ARGS__)
+	#define PVR_DPF_0x010U(...) PVRSRVDebugPrintf(DBGPRIV_VERBOSE, __VA_ARGS__)
+	#define PVR_DPF_0x020U(...) PVRSRVDebugPrintf(DBGPRIV_CALLTRACE, __VA_ARGS__)
+	#define PVR_DPF_0x040U(...) PVRSRVDebugPrintf(DBGPRIV_ALLOC, __VA_ARGS__)
+#else
+	#define PVR_DPF_0x004U(...)
+	#define PVR_DPF_0x008U(...)
+	#define PVR_DPF_0x010U(...)
+	#define PVR_DPF_0x020U(...)
+	#define PVR_DPF_0x040U(...)
+#endif
+
+	/* Translate the different log levels to separate macros
+	 * so they can each be compiled out.
+	 */
+#if defined(DEBUG)
+	#define PVR_DPF_EX(lvl, ...) PVR_DPF_ ## lvl (__FILE__, __LINE__, __VA_ARGS__)
+#else
+	#define PVR_DPF_EX(lvl, ...) PVR_DPF_ ## lvl ("", __LINE__, __VA_ARGS__)
+#endif
+	/*! @endcond */
+
+	/* Get rid of the double bracketing */
+	#define PVR_DPF(x) PVR_DPF_EX x
+
+	#define PVR_LOG_ERROR(_rc, _call) do \
+		{ \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+		} while (false)
+
+	#define PVR_LOG_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_WARN_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_WARNING, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_NOMEM(_expr, _call) do \
+		{ \
+			if (unlikely(_expr == NULL)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_OUT_OF_MEMORY, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()", _call, __func__)); \
+				return PVRSRV_ERROR_OUT_OF_MEMORY; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_NOMEM(_expr, _err, _go) do \
+		{ \
+			if (unlikely(_expr == NULL)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_OUT_OF_MEMORY, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()", #_expr, __func__)); \
+				_err = PVRSRV_ERROR_OUT_OF_MEMORY; \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				return; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR(_rc, _call, _go) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_WITH_ERROR(_call, _err, _rc, _go) do \
+		{ \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			_err = _rc; \
+			goto _go; \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_IF_FALSE(_expr, _msg) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_FALSE(_expr, _msg, _rc) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_FALSE(_expr, _msg) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_FALSE(_expr, _msg, _go) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE(_expr, _msg, _rc) do \
+		{ \
+			if (unlikely((_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_T, PVRSRV_ERROR_UNEXPECTED_TRUE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_INVALID_PARAM(_expr, _param) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_INVALID_PARAMS, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s invalid in %s()", _param, __func__)); \
+				return PVRSRV_ERROR_INVALID_PARAMS; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do \
+		{ if (unlikely(!(_expr))) { \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_INVALID_PARAMS, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s invalid in %s()", #_expr, __func__)); \
+			_err = PVRSRV_ERROR_INVALID_PARAMS; \
+			goto _go; } \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_MSG(_lvl, _msg) do \
+		{ \
+			if (_lvl == PVR_DBG_ERROR) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+			} \
+			else if (_lvl == PVR_DBG_WARNING) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+			} \
+			PVR_DPF((_lvl, ("In %s() "_msg), __func__)); \
+		} while (false)
+
+	#define PVR_LOG_VA(_lvl, _msg, ...) do \
+		{ \
+			if (_lvl == PVR_DBG_ERROR) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+			} \
+			else if (_lvl == PVR_DBG_WARNING) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+			} \
+			PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+		} while (false)
+
+	#define PVR_LOG_IF_ERROR_VA(_lvl, _rc, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				if (_lvl == PVR_DBG_ERROR) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				} \
+				else if (_lvl == PVR_DBG_WARNING) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_WARNING, _rc, HTB_FILE_NAME, __LINE__); \
+				} \
+				PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_IF_FALSE_VA(_lvl, _expr, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				if (_lvl == PVR_DBG_ERROR) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+				} \
+				else if (_lvl == PVR_DBG_WARNING) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+				} \
+				PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR_VA(_rc, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR_VA(_rc, _go, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_FALSE_VA(_expr, _rc, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("At %s: "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_FALSE_VA(_expr, _go, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE_VA(_expr, _rc, _msg, ...) do \
+		{ \
+			if (unlikely((_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("At %s: "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+#else /* defined(PVRSRV_NEED_PVR_DPF) */
+
+	#define PVR_DPF(X)  /*!< Null Implementation of PowerVR Debug Printf (does nothing) */
+
+	#define PVR_LOG_MSG(_lvl, _msg)
+	#define PVR_LOG_VA(_lvl, _msg, ...)
+	#define PVR_LOG_ERROR(_rc, _call) (void)(_rc)
+	#define PVR_LOG_IF_ERROR(_rc, _call) (void)(_rc)
+	#define PVR_WARN_IF_ERROR(_rc, _call) (void)(_rc)
+
+	#define PVR_LOG_IF_ERROR_VA(_lvl, _rc, _msg, ...) (void)(_rc)
+	#define PVR_LOG_IF_FALSE_VA(_lvl, _expr, _msg, ...) (void)(_expr)
+
+	#define PVR_LOG_RETURN_IF_NOMEM(_expr, _call) do { if (unlikely(_expr == NULL)) { return PVRSRV_ERROR_OUT_OF_MEMORY; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_NOMEM(_expr, _err, _go) do { if (unlikely(_expr == NULL)) { _err = PVRSRV_ERROR_OUT_OF_MEMORY; goto _go; } MSC_SUPPRESS_4127	} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR(_rc, _call) do { if (unlikely(_rc != PVRSRV_OK)) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_ERROR_VA(_rc, _msg, ...) do { if (unlikely(_rc != PVRSRV_OK)) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_VOID_IF_ERROR(_rc, _call) do { if (unlikely(_rc != PVRSRV_OK)) { return; } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR(_rc, _call, _go) do { if (unlikely(_rc != PVRSRV_OK)) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_ERROR_VA(_rc, _go, _msg, ...) do { if (unlikely(_rc != PVRSRV_OK)) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_WITH_ERROR(_call, _err, _rc, _go) do { _err = _rc; goto _go; MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_IF_FALSE(_expr, _msg) (void)(_expr)
+	#define PVR_LOG_RETURN_IF_FALSE(_expr, _msg, _rc) do { if (unlikely(!(_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_FALSE_VA(_expr, _rc, _msg, ...) do { if (unlikely(!(_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE(_expr, _msg, _rc) do { if (unlikely((_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_TRUE_VA(_expr, _rc, _msg, ...) do { if (unlikely((_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_FALSE(_expr, _msg) do { if (unlikely(!(_expr))) { return; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_FALSE(_expr, _msg, _go) do { if (unlikely(!(_expr))) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_FALSE_VA(_expr, _go, _msg, ...) do { if (unlikely(!(_expr))) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_IF_INVALID_PARAM(_expr, _param) do { if (unlikely(!(_expr))) { return PVRSRV_ERROR_INVALID_PARAMS; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do { if (unlikely(!(_expr))) { _err = PVRSRV_ERROR_INVALID_PARAMS; goto _go; } MSC_SUPPRESS_4127 } while (false)
+
+	#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#endif /* defined(PVRSRV_NEED_PVR_DPF) */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintf
+@Description    Output a debug message to the user, using an OS-specific
+                method, to a log or console which can be read by developers
+                Invoked from the macro PVR_DPF().
+@Input          ui32DebugLevel   The debug level of the message. This can
+                                 be used to restrict the output of debug
+                                 messages based on their severity.
+                                 If this is PVR_DBG_BUFFERED, the message
+                                 should be written into a debug circular
+                                 buffer instead of being output immediately
+                                 (useful when performance would otherwise
+                                 be adversely affected).
+                                 The debug circular buffer shall only be
+                                 output when PVRSRVDebugPrintfDumpCCB() is
+                                 called.
+@Input          pszFileName      The source file containing the code that is
+                                 generating the message
+@Input          ui32Line         The line number in the source file
+@Input          pszFormat        The formatted message string
+@Input          ...              Zero or more arguments for use by the
+                                 formatted string
+@Return         None
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVDebugPrintf(IMG_UINT32 ui32DebugLevel,
+                                               const IMG_CHAR *pszFileName,
+                                               IMG_UINT32 ui32Line,
+                                               const IMG_CHAR *pszFormat,
+                                               ...) __printf(4, 5);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintfDumpCCB
+@Description    When PVRSRVDebugPrintf() is called with the ui32DebugLevel
+                specified as DBGPRIV_BUFFERED, the debug shall be written to
+                the debug circular buffer instead of being output immediately.
+                (This could be used to obtain debug without incurring a
+                performance hit by printing it at that moment).
+                This function shall dump the contents of that debug circular
+                buffer to be output in an OS-specific method to a log or
+                console which can be read by developers.
+@Return         None
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVDebugPrintfDumpCCB(void);
+
+#if !defined(DOXYGEN)
+#define PVR_DPF_FUNC__(lvl, message, ...) PVR_DPF((lvl, "%s: " message, __func__, ##__VA_ARGS__))
+#define PVR_DPF_FUNC(x) PVR_DPF_FUNC__ x
+#endif
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_ERROR macro.
+ */
+#define PVR_RETURN_IF_ERROR(_rc) do \
+	{ if (unlikely(_rc != PVRSRV_OK)) { \
+		return _rc; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_FALSE macro.
+ */
+#define PVR_RETURN_IF_FALSE(_expr, _rc) do \
+	{ if (unlikely(!(_expr))) { \
+		return _rc; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_INVALID_PARAM macro.
+ */
+#define PVR_RETURN_IF_INVALID_PARAM(_expr) do \
+	{ if (unlikely(!(_expr))) { \
+		return PVRSRV_ERROR_INVALID_PARAMS; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_NOMEM macro.
+ */
+#define PVR_RETURN_IF_NOMEM(_expr) do \
+	{ if (unlikely(!(_expr))) { \
+		return PVRSRV_ERROR_OUT_OF_MEMORY; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_NOMEM macro.
+ */
+#define PVR_GOTO_IF_NOMEM(_expr, _err, _go) do \
+	{ if (unlikely(_expr == NULL)) { \
+		_err = PVRSRV_ERROR_OUT_OF_MEMORY; \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_INVALID_PARAM macro.
+ */
+#define PVR_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do \
+	{ if (unlikely(!(_expr))) { \
+		_err = PVRSRV_ERROR_INVALID_PARAMS; \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_FALSE macro.
+ */
+#define PVR_GOTO_IF_FALSE(_expr, _go) do \
+	{ if (unlikely(!(_expr))) { \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_ERROR macro.
+ */
+#define PVR_GOTO_IF_ERROR(_rc, _go) do \
+	{ if (unlikely(_rc != PVRSRV_OK)) { \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_WITH_ERROR macro.
+ */
+#define PVR_GOTO_WITH_ERROR(_err, _rc, _go) do \
+	{ _err = _rc; goto _go; \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/*! @cond Doxygen_Suppress */
+#if defined(PVR_DPF_FUNCTION_TRACE_ON)
+
+	#define PVR_DPF_ENTERED \
+	PVR_DPF((PVR_DBG_CALLTRACE, "|-> %s:%d entered", __func__, __LINE__))
+
+	#define PVR_DPF_ENTERED1(p1) \
+		PVR_DPF((PVR_DBG_CALLTRACE, "|-> %s:%d entered (0x%lx)", __func__, __LINE__, ((unsigned long)p1)))
+
+	#define PVR_DPF_RETURN_RC(a) \
+	do { int _r = (a); PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned %d", __func__, __LINE__, (_r))); return (_r); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_RC1(a,p1) \
+		do { int _r = (a); PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned %d (0x%lx)", __func__, __LINE__, (_r), ((unsigned long)p1))); return (_r); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_VAL(a) \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned with value", __func__, __LINE__)); return (a); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_OK \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned ok", __func__, __LINE__)); return PVRSRV_OK; MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned", __func__, __LINE__)); return; MSC_SUPPRESS_4127 } while (false)
+
+	#if !defined(DEBUG)
+	#error PVR DPF Function trace enabled in release build, rectify
+	#endif
+
+#else /* defined(PVR_DPF_FUNCTION_TRACE_ON) */
+
+	#define PVR_DPF_ENTERED
+	#define PVR_DPF_ENTERED1(p1)
+	#define PVR_DPF_RETURN_RC(a)     return (a)
+	#define PVR_DPF_RETURN_RC1(a,p1) return (a)
+	#define PVR_DPF_RETURN_VAL(a)    return (a)
+	#define PVR_DPF_RETURN_OK        return PVRSRV_OK
+	#define PVR_DPF_RETURN           return
+
+#endif /* defined(PVR_DPF_FUNCTION_TRACE_ON) */
+/*! @endcond */
+
+#if defined(__KERNEL__) || defined(DOXYGEN) || defined(__QNXNTO__)
+/*Use PVR_DPF() unless message is necessary in release build */
+#define PVR_LOG(X) PVRSRVReleasePrintf X
+
+/*************************************************************************/ /*!
+@Function       PVRSRVReleasePrintf
+@Description    Output an important message, using an OS-specific method,
+                to the Server log or console which will always be output in
+                both release and debug builds.
+                Invoked from the macro PVR_LOG(). Used in Services Server only.
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+@Return         None
+*/ /**************************************************************************/
+void IMG_CALLCONV PVRSRVReleasePrintf(const IMG_CHAR *pszFormat, ...) __printf(1, 2);
+
+ /*************************************************************************/ /*!
+ @Function       PVRSRVReleasePrintfVArgs
+ @Description    Output an important message, using an OS-specific method,
+                 to the Server log or console which will always be output in
+                 both release and debug builds.
+                 Calls to va_start and va_end should wrap this function when
+                 passing in va_list args.
+ @Input          pszFormat   The message format string
+ @Input          vaArgs      va_list arguments to print using pszFormat.
+ @Return         None
+ */ /**************************************************************************/
+void IMG_CALLCONV PVRSRVReleasePrintfVArgs(const IMG_CHAR *pszFormat, va_list vaArgs);
+#endif
+
+/* PVR_TRACE() handling */
+
+#if defined(PVRSRV_NEED_PVR_TRACE) || defined(DOXYGEN)
+
+	#define PVR_TRACE(X)	PVRSRVTrace X    /*!< PowerVR Debug Trace Macro */
+	/* Empty string implementation that is -O0 build friendly */
+	#define PVR_TRACE_EMPTY_LINE()	PVR_TRACE(("%s", ""))
+
+/*************************************************************************/ /*!
+@Function       PVRTrace
+@Description    Output a debug message to the user
+                Invoked from the macro PVR_TRACE().
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVTrace(const IMG_CHAR* pszFormat, ... )
+	__printf(1, 2);
+
+#else /* defined(PVRSRV_NEED_PVR_TRACE) */
+	/*! Null Implementation of PowerVR Debug Trace Macro (does nothing) */
+	#define PVR_TRACE(X)
+
+#endif /* defined(PVRSRV_NEED_PVR_TRACE) */
+
+
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_64BITS_TO_32BITS)
+#endif
+	INLINE static IMG_UINT32 TRUNCATE_64BITS_TO_32BITS(IMG_UINT64 uiInput)
+	{
+		IMG_UINT32 uiTruncated;
+
+		uiTruncated = (IMG_UINT32)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_64BITS_TO_SIZE_T)
+#endif
+	INLINE static size_t TRUNCATE_64BITS_TO_SIZE_T(IMG_UINT64 uiInput)
+	{
+		size_t uiTruncated;
+
+		uiTruncated = (size_t)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_SIZE_T_TO_32BITS)
+#endif
+	INLINE static IMG_UINT32 TRUNCATE_SIZE_T_TO_32BITS(size_t uiInput)
+	{
+		IMG_UINT32 uiTruncated;
+
+		uiTruncated = (IMG_UINT32)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#else /* defined(PVRSRV_NEED_PVR_ASSERT) */
+	#define TRUNCATE_64BITS_TO_32BITS(expr) ((IMG_UINT32)(expr))
+	#define TRUNCATE_64BITS_TO_SIZE_T(expr) ((size_t)(expr))
+	#define TRUNCATE_SIZE_T_TO_32BITS(expr) ((IMG_UINT32)(expr))
+#endif /* defined(PVRSRV_NEED_PVR_ASSERT) */
+
+/*! @cond Doxygen_Suppress */
+/* Macros used to trace calls */
+#if defined(DEBUG)
+	#define PVR_DBG_FILELINE , (__FILE__), (__LINE__)
+	#define PVR_DBG_FILELINE_PARAM , const IMG_CHAR *pszaFile, IMG_UINT32 ui32Line
+	#define PVR_DBG_FILELINE_ARG , pszaFile, ui32Line
+	#define PVR_DBG_FILELINE_FMT " %s:%u"
+	#define PVR_DBG_FILELINE_UNREF() do { PVR_UNREFERENCED_PARAMETER(pszaFile); \
+				PVR_UNREFERENCED_PARAMETER(ui32Line); } while (false)
+#else
+	#define PVR_DBG_FILELINE
+	#define PVR_DBG_FILELINE_PARAM
+	#define PVR_DBG_FILELINE_ARG
+	#define PVR_DBG_FILELINE_FMT
+	#define PVR_DBG_FILELINE_UNREF()
+#endif
+/*! @endcond */
+
+#if defined(__cplusplus)
+}
+#endif
+
+/*!
+    @def PVR_ASSERT
+    @brief Aborts the program if assertion fails.
+
+    The macro will be defined only when PVRSRV_NEED_PVR_ASSERT macro is
+    enabled. It's ignored otherwise.
+
+    @def PVR_DPF
+    @brief PowerVR Debug Printf logging macro used throughout the driver.
+
+    The macro allows to print logging messages to appropriate log. The
+    destination log is based on the component (user space / kernel space) and
+    operating system (Linux, Android, etc.).
+
+    The macro also supports severity levels that allow to turn on/off messages
+    based on their importance.
+
+    This macro will print messages with severity level higher that error only
+    if PVRSRV_NEED_PVR_DPF macro is defined.
+
+    @def PVR_LOG_ERROR
+    @brief Logs error.
+
+    @def PVR_LOG_IF_ERROR
+    @brief Logs error if not PVRSRV_OK.
+
+    @def PVR_WARN_IF_ERROR
+    @brief Logs warning if not PVRSRV_OK.
+
+    @def PVR_LOG_RETURN_IF_NOMEM
+    @brief Logs error if expression is NULL and returns PVRSRV_ERROR_OUT_OF_MEMORY.
+
+    @def PVR_LOG_GOTO_IF_NOMEM
+    @brief Logs error if expression is NULL and jumps to given label.
+
+    @def PVR_LOG_RETURN_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and returns the error.
+
+    @def PVR_LOG_RETURN_VOID_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and returns (used in function that return void).
+
+    @def PVR_LOG_GOTO_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and jumps to label.
+
+    @def PVR_LOG_GOTO_WITH_ERROR
+    @brief Logs error, goes to a label and sets the error code.
+
+    @def PVR_LOG_IF_FALSE
+    @brief Prints error message if expression is false.
+
+    @def PVR_LOG_RETURN_IF_FALSE
+    @brief Prints error message if expression is false and returns given error.
+
+    @def PVR_LOG_RETURN_VOID_IF_FALSE
+    @brief Prints error message if expression is false and returns (used in function that return void).
+
+    @def PVR_LOG_GOTO_IF_FALSE
+    @brief Prints error message if expression is false and jumps to label.
+
+    @def PVR_LOG_RETURN_IF_TRUE
+    @brief Prints error message if expression is true and returns given error.
+
+    @def PVR_LOG_RETURN_IF_INVALID_PARAM
+    @brief Prints error message if expression is false and returns PVRSRV_ERROR_INVALID_PARAMS.
+
+    @def PVR_LOG_GOTO_IF_INVALID_PARAM
+    @brief Prints error message if expression is false and jumps to label.
+
+    @def PVR_RETURN_IF_ERROR
+    @brief Returns passed error code if it's different than PVRSRV_OK;
+
+    @def PVR_RETURN_IF_FALSE
+    @brief Returns passed error code if expression is false.
+
+    @def PVR_RETURN_IF_INVALID_PARAM
+    @brief Returns PVRSRV_ERROR_INVALID_PARAMS if expression is false.
+
+    @def PVR_RETURN_IF_NOMEM
+    @brief Returns PVRSRV_ERROR_OUT_OF_MEMORY if expression is NULL.
+
+    @def PVR_GOTO_IF_NOMEM
+    @brief Goes to a label if expression is NULL.
+
+    @def PVR_GOTO_IF_INVALID_PARAM
+    @brief Goes to a label if expression is false.
+
+    @def PVR_GOTO_IF_FALSE
+    @brief Goes to a label if expression is false.
+
+    @def PVR_GOTO_IF_ERROR
+    @brief Goes to a label if the error code is different than PVRSRV_OK;
+
+    @def PVR_GOTO_WITH_ERROR
+    @brief Goes to a label and sets the error code.
+
+    @def PVR_LOG
+    @brief Prints message to a log unconditionally.
+
+    This macro will print messages only if PVRSRV_NEED_PVR_LOG macro is defined.
+    @def PVR_LOG_MSG
+    @brief Prints message to a log with the given log-level.
+
+    @def PVR_LOG_VA
+    @brief Prints message with var-args to a log with the given log-level.
+
+    @def PVR_LOG_IF_ERROR_VA
+    @brief Prints message with var-args to a log if the error code is different than PVRSRV_OK.
+
+    @def PVR_LOG_IF_FALSE_VA
+    @brief Prints message with var-args if expression is false.
+
+    @def PVR_LOG_RETURN_IF_ERROR_VA
+    @brief Prints message with var-args to a log and returns the error code.
+
+    @def PVR_LOG_GOTO_IF_ERROR_VA
+    @brief Prints message with var-args to a log and goes to a label if the error code is different than PVRSRV_OK.
+
+    @def PVR_LOG_RETURN_IF_FALSE_VA
+    @brief Logs the error message with var-args if the expression is false and returns the error code.
+
+    @def PVR_LOG_GOTO_IF_FALSE_VA
+    @brief Logs the error message with var-args and goes to a label if the expression is false.
+
+    @def PVR_LOG_RETURN_IF_TRUE_VA
+    @brief Logs the error message with var-args if the expression is true and returns the error code.
+
+    @def PVR_TRACE_EMPTY_LINE
+    @brief Prints empty line to a log (PVRSRV_NEED_PVR_LOG must be defined).
+
+    @def TRUNCATE_64BITS_TO_32BITS
+    @brief Truncates 64 bit value to 32 bit value (with possible precision loss).
+
+    @def TRUNCATE_64BITS_TO_SIZE_T
+    @brief Truncates 64 bit value to size_t value (with possible precision loss).
+
+    @def TRUNCATE_SIZE_T_TO_32BITS
+    @brief Truncates size_t value to 32 bit value (with possible precision loss).
+ */
+
+#endif /* PVR_DEBUG_H */
+
+/******************************************************************************
+ End of file (pvr_debug.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_fd_sync_kernel.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_fd_sync_kernel.h
new file mode 100644
index 000000000000..3645e29079b1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_fd_sync_kernel.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File           pvr_fd_sync_kernel.h
+@Title          Kernel/userspace interface definitions to use the kernel sync
+                driver
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef _PVR_FD_SYNC_KERNEL_H_
+#define _PVR_FD_SYNC_KERNEL_H_
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+#include "pvr_drm.h"
+
+#define PVR_SYNC_MAX_QUERY_FENCE_POINTS 14
+
+struct pvr_sync_pt_info {
+	/* Output */
+	__u32 id;
+	__u32 ui32FWAddr;
+	__u32 ui32CurrOp;
+	__u32 ui32NextOp;
+	__u32 ui32TlTaken;
+} __attribute__((packed, aligned(8)));
+
+#endif /* _PVR_FD_SYNC_KERNEL_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvr_intrinsics.h b/drivers/gpu/drm/img/img-volcanic/include/pvr_intrinsics.h
new file mode 100644
index 000000000000..410a2f5a50b5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvr_intrinsics.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File
+@Title          Intrinsics definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_INTRINSICS_H
+#define PVR_INTRINSICS_H
+
+/* PVR_CTZLL:
+ * Count the number of trailing zeroes in a long long integer
+ */
+
+#if defined(__GNUC__)
+#if defined(__x86_64__)
+
+	#define PVR_CTZLL __builtin_ctzll
+#endif
+#endif
+
+/* PVR_CLZLL:
+ * Count the number of leading zeroes in a long long integer
+ */
+
+#if defined(__GNUC__)
+#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || \
+					defined(__arm__) || defined(__mips)
+
+#define PVR_CLZLL __builtin_clzll
+
+#endif
+#endif
+
+#endif /* PVR_INTRINSICS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrmodule.h b/drivers/gpu/drm/img/img-volcanic/include/pvrmodule.h
new file mode 100644
index 000000000000..267c7b687487
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrmodule.h
@@ -0,0 +1,48 @@
+/*************************************************************************/ /*!
+@Title          Module Author and License.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef	_PVRMODULE_H_
+#define	_PVRMODULE_H_
+
+MODULE_AUTHOR("Imagination Technologies Ltd. <gpl-support@imgtec.com>");
+MODULE_LICENSE("Dual MIT/GPL");
+
+#endif	/* _PVRMODULE_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_device_types.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_device_types.h
new file mode 100644
index 000000000000..20e0d5ce3ed5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_device_types.h
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR device type definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVRSRV_DEVICE_TYPES_H)
+#define PVRSRV_DEVICE_TYPES_H
+
+#include "img_types.h"
+
+#if !defined(PVRSRV_MAX_DEVICES)
+#error "PVRSRV_MAX_DEVICES must be defined in the DDK build environment"
+#endif	/* !defined(PVRSRV_NUM_DEVICES) */
+#define PVRSRV_HOST_DEVICE_ID	255U /*!< Device ID used for host (non-GPU) device. */
+
+static_assert(PVRSRV_MAX_DEVICES < PVRSRV_HOST_DEVICE_ID, "Invalid host device ID.");
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#endif /* PVRSRV_DEVICE_TYPES_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_devvar.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_devvar.h
new file mode 100644
index 000000000000..a8c64e309fda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_devvar.h
@@ -0,0 +1,291 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Device Variable interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for device variables
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_DEVVAR_H
+#define PVRSRV_DEVVAR_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define DEVVAR_MAX_NAME_LEN 32
+
+typedef struct SYNC_PRIM_CONTEXT_TAG *PDEVVARCTX;
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_TAG *PDEVVAR;
+
+typedef struct PVRSRV_DEV_VAR_UPDATE_TAG
+{
+	PDEVVAR					psDevVar;			/*!< Pointer to the dev var */
+	IMG_UINT32				ui32UpdateValue;	/*!< the update value */
+} PVRSRV_DEV_VAR_UPDATE;
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarContextCreate
+
+@Description    Create a new device variable context
+
+@Input          psDevConnection         Device to create the device
+                                        variable context on
+
+@Output         phDevVarContext         Handle to the created device
+                                        variable context
+
+@Return         PVRSRV_OK if the device variable context was successfully
+                created
+*/
+/*****************************************************************************/
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVDevVarContextCreate(const PVRSRV_DEV_CONNECTION *psDevConnection,
+                          PDEVVARCTX                  *phDevVarContext);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarContextDestroy
+
+@Description    Destroy a device variable context
+
+@Input          hDevVarContext          Handle to the device variable
+                                        context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarContextDestroy(PDEVVARCTX hDevVarContext);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarAlloc
+
+@Description    Allocate a new device variable on the specified device
+                variable context. The device variable's value is initialised
+                with the value passed in ui32InitialValue.
+
+@Input          hDevVarContext          Handle to the device variable
+                                        context
+@Input          ui32InitialValue        Value to initially assign to the
+                                        new variable
+@Input          pszDevVarName           Name assigned to the device variable
+                                        (for debug purposes)
+
+@Output         ppsDevVar               Created device variable
+
+@Return         PVRSRV_OK if the device variable was successfully created
+*/
+/*****************************************************************************/
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVDevVarAllocI(PDEVVARCTX hDevVarContext,
+                   PDEVVAR *ppsDevVar,
+                   IMG_UINT32 ui32InitialValue,
+                   const IMG_CHAR *pszDevVarName
+                   PVR_DBG_FILELINE_PARAM);
+#define PVRSRVDevVarAlloc(hDevVarContext, ppsDevVar, ui32InitialValue, pszDevVarName) \
+	PVRSRVDevVarAllocI( (hDevVarContext), (ppsDevVar), (ui32InitialValue), (pszDevVarName) \
+	                    PVR_DBG_FILELINE )
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarFree
+
+@Description    Free a device variable
+
+@Input          psDevVar                The device variable to free
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarFree(PDEVVAR psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarSet
+
+@Description    Set the device variable to a value
+
+@Input          psDevVar                The device variable to set
+
+@Input          ui32Value               Value to set it to
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarSet(PDEVVAR		psDevVar,
+                IMG_UINT32	ui32Value);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarGet
+
+@Description    Get the current value of the device variable
+
+@Input          psDevVar                The device variable to get the
+                                        value of
+
+@Return         Value of the variable
+*/
+/*****************************************************************************/
+IMG_EXPORT IMG_UINT32
+PVRSRVDevVarGet(PDEVVAR		psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarGetFirmwareAddr
+
+@Description    Returns the address of the associated firmware value for a
+                specified device integer (not exposed to client)
+
+@Input          psDevVar               The device variable to resolve
+
+@Return         The firmware address of the device variable
+*/
+/*****************************************************************************/
+IMG_EXPORT IMG_UINT32
+PVRSRVDevVarGetFirmwareAddr(PDEVVAR	psDevVar);
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDump
+
+@Description    PDump the current value of the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDump(PDEVVAR psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDumpPol
+
+@Description    Do a PDump poll of the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Input          ui32Value               Value to Poll for
+
+@Input          ui32Mask                PDump mask operator
+
+@Input          ui32PDumpFlags          PDump flags
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDumpPol(PDEVVAR psDevVar,
+                     IMG_UINT32 ui32Value,
+                     IMG_UINT32 ui32Mask,
+                     PDUMP_POLL_OPERATOR eOperator,
+                     IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDumpCBP
+
+@Description    Do a PDump CB poll using the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Input          uiWriteOffset           Current write offset of buffer
+
+@Input          uiPacketSize            Size of the packet to write into CB
+
+@Input          uiBufferSize            Size of the CB
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDumpCBP(PDEVVAR psDevVar,
+                     IMG_UINT64 uiWriteOffset,
+                     IMG_UINT64 uiPacketSize,
+                     IMG_UINT64 uiBufferSize);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDump)
+#endif
+static INLINE void
+PVRSRVDevVarPDump(PDEVVAR psDevVar)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevVar);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDumpPol)
+#endif
+static INLINE void
+PVRSRVDevVarPDumpPol(PDEVVAR psDevVar,
+                     IMG_UINT32 ui32Value,
+                     IMG_UINT32 ui32Mask,
+                     PDUMP_POLL_OPERATOR eOperator,
+                     IMG_UINT32 ui32PDumpFlags)
+{
+	 PVR_UNREFERENCED_PARAMETER(psDevVar);
+	 PVR_UNREFERENCED_PARAMETER(ui32Value);
+	 PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	 PVR_UNREFERENCED_PARAMETER(eOperator);
+	 PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDumpCBP)
+#endif
+static INLINE void
+PVRSRVDevVarPDumpCBP(PDEVVAR psDevVar,
+                     IMG_UINT64 uiWriteOffset,
+                     IMG_UINT64 uiPacketSize,
+                     IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevVar);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+}
+#endif /* PDUMP */
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* PVRSRV_DEVVAR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_error.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_error.h
new file mode 100644
index 000000000000..120c42d87155
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_error.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_error.h
+@Title          services error enumerant
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines error codes used by any/all services modules
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVRSRV_ERROR_H)
+#define PVRSRV_ERROR_H
+
+/*!
+ *****************************************************************************
+ * Error values
+ *****************************************************************************/
+typedef enum PVRSRV_ERROR_TAG
+{
+	PVRSRV_OK,
+
+#define PVRE(x) x,
+#include "pvrsrv_errors.h"
+#undef PVRE
+
+	PVRSRV_ERROR_FORCE_I32 = 0x7fffffff
+
+} PVRSRV_ERROR;
+
+/*!
+ * @Function PVRSRVIsRetryError
+ * @Description Checks if error code is one of the errors that require retry
+ *              from the caller.
+ * @Input eError Error code.
+ * @Return IMG_TRUE if eError is one of the error codes that require the caller
+ *         to retry.
+ */
+#define PVRSRVIsRetryError(eError) \
+	(((eError == PVRSRV_ERROR_RETRY) || (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)) ? \
+	 IMG_TRUE : IMG_FALSE)
+
+#endif /* !defined(PVRSRV_ERROR_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_errors.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_errors.h
new file mode 100644
index 000000000000..4c88e675d0d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_errors.h
@@ -0,0 +1,429 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_errors.h
+@Title          services error codes
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines error codes used by any/all services modules
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Don't add include guards to this file! */
+
+PVRE(PVRSRV_ERROR_OUT_OF_MEMORY)
+PVRE(PVRSRV_ERROR_TOO_FEW_BUFFERS)
+PVRE(PVRSRV_ERROR_INVALID_PARAMS)
+PVRE(PVRSRV_ERROR_INIT_FAILURE)
+PVRE(PVRSRV_ERROR_CANT_REGISTER_CALLBACK)
+PVRE(PVRSRV_ERROR_INVALID_DEVICE)
+PVRE(PVRSRV_ERROR_NOT_OWNER)
+PVRE(PVRSRV_ERROR_BAD_MAPPING)
+PVRE(PVRSRV_ERROR_TIMEOUT)
+PVRE(PVRSRV_ERROR_NOT_IMPLEMENTED)
+PVRE(PVRSRV_ERROR_FLIP_CHAIN_EXISTS)
+PVRE(PVRSRV_ERROR_INVALID_SWAPINTERVAL)
+PVRE(PVRSRV_ERROR_SCENE_INVALID)
+PVRE(PVRSRV_ERROR_STREAM_ERROR)
+PVRE(PVRSRV_ERROR_FAILED_DEPENDENCIES)
+PVRE(PVRSRV_ERROR_CMD_NOT_PROCESSED)
+PVRE(PVRSRV_ERROR_CMD_TOO_BIG)
+PVRE(PVRSRV_ERROR_DEVICE_REGISTER_FAILED)
+PVRE(PVRSRV_ERROR_TOOMANYBUFFERS)
+PVRE(PVRSRV_ERROR_NOT_SUPPORTED)
+PVRE(PVRSRV_ERROR_PROCESSING_BLOCKED)
+PVRE(PVRSRV_ERROR_CANNOT_FLUSH_QUEUE)
+PVRE(PVRSRV_ERROR_CANNOT_GET_QUEUE_SPACE)
+PVRE(PVRSRV_ERROR_CANNOT_GET_RENDERDETAILS)
+PVRE(PVRSRV_ERROR_RETRY)
+PVRE(PVRSRV_ERROR_DDK_VERSION_MISMATCH)
+PVRE(PVRSRV_ERROR_DDK_BUILD_MISMATCH)
+PVRE(PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH)
+PVRE(PVRSRV_ERROR_BVNC_MISMATCH)
+PVRE(PVRSRV_ERROR_FWPROCESSOR_MISMATCH)
+PVRE(PVRSRV_ERROR_UPLOAD_TOO_BIG)
+PVRE(PVRSRV_ERROR_INVALID_FLAGS)
+PVRE(PVRSRV_ERROR_FAILED_TO_REGISTER_PROCESS)
+PVRE(PVRSRV_ERROR_UNABLE_TO_LOAD_LIBRARY)
+PVRE(PVRSRV_ERROR_UNABLE_GET_FUNC_ADDR)
+PVRE(PVRSRV_ERROR_UNLOAD_LIBRARY_FAILED)
+PVRE(PVRSRV_ERROR_BRIDGE_CALL_FAILED)
+PVRE(PVRSRV_ERROR_IOCTL_CALL_FAILED)
+PVRE(PVRSRV_ERROR_MMU_API_PROTOCOL_ERROR)
+PVRE(PVRSRV_ERROR_MMU_CONFIG_IS_WRONG)
+PVRE(PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_ALLOCATE_PAGETABLES)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_CREATE_HEAP)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_UNMAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE)
+PVRE(PVRSRV_ERROR_MMU_LIVE_ALLOCATIONS_IN_HEAP)
+PVRE(PVRSRV_ERROR_MMU_RESERVATION_NOT_INSIDE_HEAP)
+PVRE(PVRSRV_ERROR_PMR_NEW_MEMORY)
+PVRE(PVRSRV_ERROR_PMR_STILL_REFERENCED)
+PVRE(PVRSRV_ERROR_PMR_CLIENT_NOT_TRUSTED)
+PVRE(PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES)
+PVRE(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY)
+PVRE(PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES)
+PVRE(PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE)
+PVRE(PVRSRV_ERROR_PMR_NOT_PERMITTED)
+PVRE(PVRSRV_ERROR_PMR_ALREADY_OCCUPIED)
+PVRE(PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR)
+PVRE(PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR)
+PVRE(PVRSRV_ERROR_PMR_WRONG_PMR_TYPE)
+PVRE(PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS)
+PVRE(PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE)
+PVRE(PVRSRV_ERROR_PMR_BAD_CHUNK_SIZE)
+PVRE(PVRSRV_ERROR_PMR_MAPPINGTABLE_MISMATCH)
+PVRE(PVRSRV_ERROR_PMR_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING)
+PVRE(PVRSRV_ERROR_PMR_EMPTY)
+PVRE(PVRSRV_ERROR_PMR_NO_CPU_MAP_FOUND)
+PVRE(PVRSRV_ERROR_PMR_CPU_PAGE_UNMAP_FAILED)
+PVRE(PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED)
+PVRE(PVRSRV_ERROR_PMR_PAGE_POISONING_FAILED)
+PVRE(PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY)
+PVRE(PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_PMR_TOO_LARGE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP)
+PVRE(PVRSRV_ERROR_DEVICEMEM_BAD_IMPORT_SIZE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX)
+PVRE(PVRSRV_ERROR_DEVICEMEM_MAP_FAILED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_NON_ZERO_USAGE_COUNT)
+PVRE(PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA)
+PVRE(PVRSRV_ERROR_DEVICEMEM_OUT_OF_DEVICE_VM)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_NO_MAPPING)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_LMA_HEAP)
+PVRE(PVRSRV_ERROR_INVALID_MMU_TYPE)
+PVRE(PVRSRV_ERROR_BUFFER_DEVICE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_BUFFER_DEVICE_ALREADY_PRESENT)
+PVRE(PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_PCI_CALL_FAILED)
+PVRE(PVRSRV_ERROR_PCI_REGION_TOO_SMALL)
+PVRE(PVRSRV_ERROR_PCI_REGION_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH)
+PVRE(PVRSRV_ERROR_REGISTER_BASE_NOT_SET)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_USER_MEM)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_VP_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_SHARED_PBDESC)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_KERNELVIRTUAL)
+PVRE(PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_VIRT_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES)
+PVRE(PVRSRV_ERROR_FAILED_TO_FREE_PAGES)
+PVRE(PVRSRV_ERROR_FAILED_TO_COPY_PAGES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_LOCK_PAGES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNLOCK_PAGES)
+PVRE(PVRSRV_ERROR_STILL_MAPPED)
+PVRE(PVRSRV_ERROR_MAPPING_NOT_FOUND)
+PVRE(PVRSRV_ERROR_PHYS_ADDRESS_EXCEEDS_32BIT)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_INVALID_SEGMENT_BLOCK)
+PVRE(PVRSRV_ERROR_INVALID_GFXDEVDEVDATA)
+PVRE(PVRSRV_ERROR_INVALID_DEVINFO)
+PVRE(PVRSRV_ERROR_INVALID_MEMINFO)
+PVRE(PVRSRV_ERROR_INVALID_MISCINFO)
+PVRE(PVRSRV_ERROR_UNKNOWN_IOCTL)
+PVRE(PVRSRV_ERROR_INVALID_CONTEXT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT)
+PVRE(PVRSRV_ERROR_INVALID_HEAP)
+PVRE(PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE)
+PVRE(PVRSRV_ERROR_INVALID_KERNELINFO)
+PVRE(PVRSRV_ERROR_UNKNOWN_POWER_STATE)
+PVRE(PVRSRV_ERROR_INVALID_HANDLE_TYPE)
+PVRE(PVRSRV_ERROR_INVALID_WRAP_TYPE)
+PVRE(PVRSRV_ERROR_INVALID_PHYS_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_CPU_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_HEAPINFO)
+PVRE(PVRSRV_ERROR_INVALID_PERPROC)
+PVRE(PVRSRV_ERROR_FAILED_TO_RETRIEVE_HEAPINFO)
+PVRE(PVRSRV_ERROR_INVALID_MAP_REQUEST)
+PVRE(PVRSRV_ERROR_INVALID_UNMAP_REQUEST)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FIND_MAPPING_HEAP)
+PVRE(PVRSRV_ERROR_MAPPING_STILL_IN_USE)
+PVRE(PVRSRV_ERROR_EXCEEDED_HW_LIMITS)
+PVRE(PVRSRV_ERROR_NO_STAGING_BUFFER_ALLOCATED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_PERPROC_AREA)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CLOSE_THREAD)
+PVRE(PVRSRV_ERROR_THREAD_READ_ERROR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNINSTALL_ISR)
+PVRE(PVRSRV_ERROR_ISR_ALREADY_INSTALLED)
+PVRE(PVRSRV_ERROR_ISR_NOT_INSTALLED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INITIALISE_INTERRUPT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RETRIEVE_INFO)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DO_BACKWARDS_BLIT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CLOSE_SERVICES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_CONTEXT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_RESOURCE)
+PVRE(PVRSRV_ERROR_INVALID_CCB_COMMAND)
+PVRE(PVRSRV_ERROR_KERNEL_CCB_FULL)
+PVRE(PVRSRV_ERROR_FLIP_FAILED)
+PVRE(PVRSRV_ERROR_UNBLANK_DISPLAY_FAILED)
+PVRE(PVRSRV_ERROR_TIMEOUT_POLLING_FOR_VALUE)
+PVRE(PVRSRV_ERROR_TIMEOUT_WAITING_FOR_CLIENT_CCB)
+PVRE(PVRSRV_ERROR_CREATE_RENDER_CONTEXT_FAILED)
+PVRE(PVRSRV_ERROR_UNKNOWN_PRIMARY_FRAG)
+PVRE(PVRSRV_ERROR_UNEXPECTED_SECONDARY_FRAG)
+PVRE(PVRSRV_ERROR_UNEXPECTED_PRIMARY_FRAG)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSERT_FENCE_ID)
+PVRE(PVRSRV_ERROR_BLIT_SETUP_FAILED)
+PVRE(PVRSRV_ERROR_SUBMIT_NEEDED)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_AVAILABLE)
+PVRE(PVRSRV_ERROR_PDUMP_BUFFER_FULL)
+PVRE(PVRSRV_ERROR_PDUMP_BUF_OVERFLOW)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_ACTIVE)
+PVRE(PVRSRV_ERROR_INCOMPLETE_LINE_OVERLAPS_PAGES)
+PVRE(PVRSRV_ERROR_MUTEX_DESTROY_FAILED)
+PVRE(PVRSRV_ERROR_MUTEX_INTERRUPTIBLE_ERROR)
+PVRE(PVRSRV_ERROR_INSUFFICIENT_SPACE_FOR_COMMAND)
+PVRE(PVRSRV_ERROR_PROCESS_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_PROCESS_NOT_FOUND)
+PVRE(PVRSRV_ERROR_SRV_CONNECT_FAILED)
+PVRE(PVRSRV_ERROR_SRV_DISCONNECT_FAILED)
+PVRE(PVRSRV_ERROR_DEINT_PHASE_FAILED)
+PVRE(PVRSRV_ERROR_INIT2_PHASE_FAILED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FIND_RESOURCE)
+PVRE(PVRSRV_ERROR_NO_DC_DEVICES_FOUND)
+PVRE(PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE)
+PVRE(PVRSRV_ERROR_DC_INVALID_MAXDEPTH)
+PVRE(PVRSRV_ERROR_UNABLE_TO_OPEN_DC_DEVICE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNREGISTER_DEVICE)
+PVRE(PVRSRV_ERROR_NO_DEVICEDATA_FOUND)
+PVRE(PVRSRV_ERROR_NO_DEVICENODE_FOUND)
+PVRE(PVRSRV_ERROR_NO_CLIENTNODE_FOUND)
+PVRE(PVRSRV_ERROR_FAILED_TO_PROCESS_QUEUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INIT_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SCHEDULE_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_KILL_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_TIMER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DISABLE_TIMER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REMOVE_TIMER)
+PVRE(PVRSRV_ERROR_UNKNOWN_PIXEL_FORMAT)
+PVRE(PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_ALLOCATED)
+PVRE(PVRSRV_ERROR_HANDLE_TYPE_MISMATCH)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_SHAREABLE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_INVALID_SUBHANDLE)
+PVRE(PVRSRV_ERROR_HANDLE_BATCH_IN_USE)
+PVRE(PVRSRV_ERROR_HANDLE_BATCH_COMMIT_FAILURE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_HASH_TABLE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REMOVE_HASH_VALUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSERT_HASH_VALUE)
+PVRE(PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_BACKING_STORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_BM_HEAP)
+PVRE(PVRSRV_ERROR_UNKNOWN_INIT_SERVER_STATE)
+PVRE(PVRSRV_ERROR_NO_FREE_DEVICEIDS_AVAILABLE)
+PVRE(PVRSRV_ERROR_INVALID_DEVICEID)
+PVRE(PVRSRV_ERROR_DEVICEID_NOT_FOUND)
+PVRE(PVRSRV_ERROR_MEMORY_TEST_FAILED)
+PVRE(PVRSRV_ERROR_CPUPADDR_TEST_FAILED)
+PVRE(PVRSRV_ERROR_COPY_TEST_FAILED)
+PVRE(PVRSRV_ERROR_SEMAPHORE_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RELEASE_CLOCK)
+PVRE(PVRSRV_ERROR_CLOCK_REQUEST_FAILED)
+PVRE(PVRSRV_ERROR_DISABLE_CLOCK_FAILURE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SET_CLOCK_RATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ROUND_CLOCK_RATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_PARENT_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_SYSTEM_CLOCK)
+PVRE(PVRSRV_ERROR_UNKNOWN_SGL_ERROR)
+PVRE(PVRSRV_ERROR_SYSTEM_POWER_CHANGE_FAILURE)
+PVRE(PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE)
+PVRE(PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+PVRE(PVRSRV_ERROR_BAD_SYNC_STATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_MMUCONTEXT_ID)
+PVRE(PVRSRV_ERROR_PARAMETER_BUFFER_INVALID_ALIGNMENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ACQUIRE_CONNECTION)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RELEASE_CONNECTION)
+PVRE(PVRSRV_ERROR_PHYSHEAP_ID_IN_USE)
+PVRE(PVRSRV_ERROR_PHYSHEAP_ID_INVALID)
+PVRE(PVRSRV_ERROR_PHYSHEAP_CONFIG)
+PVRE(PVRSRV_ERROR_HP_REQUEST_TOO_LONG)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_PRIM)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_PRIM_OP)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_CONTEXT)
+PVRE(PVRSRV_ERROR_BP_NOT_SET)
+PVRE(PVRSRV_ERROR_BP_ALREADY_SET)
+PVRE(PVRSRV_ERROR_FEATURE_DISABLED)
+PVRE(PVRSRV_ERROR_REG_CONFIG_ENABLED)
+PVRE(PVRSRV_ERROR_REG_CONFIG_FULL)
+PVRE(PVRSRV_ERROR_REG_CONFIG_INVALID_TYPE)
+PVRE(PVRSRV_ERROR_MEMORY_ACCESS)
+PVRE(PVRSRV_ERROR_NO_SYSTEM_BUFFER)
+PVRE(PVRSRV_ERROR_DC_INVALID_CONFIG)
+PVRE(PVRSRV_ERROR_DC_INVALID_CROP_RECT)
+PVRE(PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT)
+PVRE(PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS)
+PVRE(PVRSRV_ERROR_DC_INVALID_TRANSFORM)
+PVRE(PVRSRV_ERROR_DC_INVALID_SCALE)
+PVRE(PVRSRV_ERROR_DC_INVALID_CUSTOM)
+PVRE(PVRSRV_ERROR_DC_TOO_MANY_PIPES)
+PVRE(PVRSRV_ERROR_DC_INVALID_PLANE_ALPHA)
+PVRE(PVRSRV_ERROR_NOT_READY)
+PVRE(PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_MEMORY_LAYOUT)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_DIMS)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ADD_TIMER)
+PVRE(PVRSRV_ERROR_NOT_FOUND)
+PVRE(PVRSRV_ERROR_ALREADY_OPEN)
+PVRE(PVRSRV_ERROR_STREAM_MISUSE)
+PVRE(PVRSRV_ERROR_STREAM_FULL)
+PVRE(PVRSRV_ERROR_STREAM_READLIMIT_REACHED)
+PVRE(PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE)
+PVRE(PVRSRV_ERROR_PHYSMEM_NOT_ALLOCATED)
+PVRE(PVRSRV_ERROR_PBSIZE_ALREADY_MAX)
+PVRE(PVRSRV_ERROR_PBSIZE_ALREADY_MIN)
+PVRE(PVRSRV_ERROR_INVALID_PB_CONFIG)
+PVRE(PVRSRV_ERROR_META_THREAD0_NOT_ENABLED)
+PVRE(PVRSRV_ERROR_NOT_AUTHENTICATED)
+PVRE(PVRSRV_ERROR_REQUEST_TDFWMEM_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_INIT_TDFWMEM_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_REQUEST_TDSECUREBUF_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_INIT_TDSECUREBUF_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_MUTEX_ALREADY_CREATED)
+PVRE(PVRSRV_ERROR_DBGTABLE_ALREADY_REGISTERED)
+PVRE(PVRSRV_ERROR_ALREADY_EXISTS)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SEND_PULSE)
+PVRE(PVRSRV_ERROR_TASK_FAILED)
+PVRE(PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+PVRE(PVRSRV_ERROR_INVALID_GPU_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_OFFSET)
+PVRE(PVRSRV_ERROR_CCCB_STALLED)
+PVRE(PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_NOT_ENABLED)
+PVRE(PVRSRV_ERROR_SYSTEM_LOCAL_MEMORY_INIT_FAIL)
+PVRE(PVRSRV_ERROR_FW_IMAGE_MISMATCH)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+PVRE(PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL)
+PVRE(PVRSRV_ERROR_RPM_PBSIZE_ALREADY_MAX)
+PVRE(PVRSRV_ERROR_NONZERO_REFCOUNT)
+PVRE(PVRSRV_ERROR_SETAFFINITY_FAILED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_COMPILE_PDS)
+PVRE(PVRSRV_ERROR_INTERNAL_ERROR)
+PVRE(PVRSRV_ERROR_BRIDGE_EFAULT)
+PVRE(PVRSRV_ERROR_BRIDGE_EINVAL)
+PVRE(PVRSRV_ERROR_BRIDGE_ENOMEM)
+PVRE(PVRSRV_ERROR_BRIDGE_ERANGE)
+PVRE(PVRSRV_ERROR_BRIDGE_EPERM)
+PVRE(PVRSRV_ERROR_BRIDGE_ENOTTY)
+PVRE(PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)
+PVRE(PVRSRV_ERROR_PROBE_DEFER)
+PVRE(PVRSRV_ERROR_INVALID_ALIGNMENT)
+PVRE(PVRSRV_ERROR_CLOSE_FAILED)
+PVRE(PVRSRV_ERROR_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_CONVERSION_FAILED)
+PVRE(PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+PVRE(PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL)
+PVRE(PVRSRV_ERROR_RA_OUT_OF_RESOURCE)
+PVRE(PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS)
+PVRE(PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED)
+PVRE(PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED)
+PVRE(PVRSRV_ERROR_RA_FREE_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_OBJECT_STILL_REFERENCED)
+PVRE(PVRSRV_ERROR_BVNC_UNSUPPORTED)
+PVRE(PVRSRV_ERROR_INVALID_BVNC_PARAMS)
+PVRE(PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT)
+PVRE(PVRSRV_ERROR_PID_ALREADY_REGISTERED)
+PVRE(PVRSRV_ERROR_PID_NOT_REGISTERED)
+PVRE(PVRSRV_ERROR_SIGNAL_FAILED)
+PVRE(PVRSRV_ERROR_INVALID_NOTIF_STREAM)
+PVRE(PVRSRV_ERROR_INVALID_SPU_MASK)
+PVRE(PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED)
+PVRE(PVRSRV_ERROR_INVALID_PVZ_CONFIG)
+PVRE(PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED)
+PVRE(PVRSRV_ERROR_NOT_SW_TIMELINE)
+PVRE(PVRSRV_ERROR_SW_TIMELINE_AT_LATEST_POINT)
+PVRE(PVRSRV_ERROR_INVALID_PVZ_OSID)
+PVRE(PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE)
+PVRE(PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG)
+PVRE(PVRSRV_ERROR_INTERRUPTED)
+PVRE(PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+PVRE(PVRSRV_ERROR_PDUMP_INVALID_BLOCKLEN)
+PVRE(PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF)
+PVRE(PVRSRV_ERROR_MULTIPLE_SECURITY_PDUMPS)
+PVRE(PVRSRV_ERROR_BAD_PARAM_SIZE)
+PVRE(PVRSRV_ERROR_INVALID_REQUEST)
+PVRE(PVRSRV_ERROR_FAILED_TO_ACQUIRE_PAGES)
+PVRE(PVRSRV_ERROR_TEST_FAILED)
+PVRE(PVRSRV_ERROR_SYNC_PRIM_OP_NOT_SUPPORTED)
+PVRE(PVRSRV_ERROR_FAILED_TO_GET_VIRT_ADDR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FREE_RESOURCE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_SEMAPHORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_SEMAPHORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_SEMAPHORE)
+PVRE(PVRSRV_ERROR_TOO_MANY_SYNCS)
+PVRE(PVRSRV_ERROR_ION_NO_CLIENT)
+PVRE(PVRSRV_ERROR_ION_FAILED_TO_ALLOC)
+PVRE(PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+PVRE(PVRSRV_ERROR_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_OUT_OF_APP_POOL_MEMORY)
+PVRE(PVRSRV_ERROR_REFCOUNT_OVERFLOW)
+PVRE(PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+PVRE(PVRSRV_ERROR_UNEXPECTED_TRUE_EXPR)
+PVRE(PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR)
+PVRE(PVRSRV_ERROR_KERNEL_CCB_OFFSET)
+PVRE(PVRSRV_ERROR_PMB_NOT_PAGE_MULTIPLE)
+PVRE(PVRSRV_ERROR_ATOMIC_OVERFLOW)
+PVRE(PVRSRV_ERROR_ATOMIC_UNDERFLOW)
+PVRE(PVRSRV_ERROR_DMABUF_LOCK)
+PVRE(PVRSRV_ERROR_DMABUF_ATTACH)
+PVRE(PVRSRV_ERROR_DMABUF_ATTACHMENT_MAPPING)
+PVRE(PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED)
+
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memalloc_physheap.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memalloc_physheap.h
new file mode 100644
index 000000000000..3c17d8b4495d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memalloc_physheap.h
@@ -0,0 +1,212 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_memalloc_physheap.h
+@Title          Services Phys Heap types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Used in creating and allocating from Physical Heaps.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVRSRV_MEMALLOC_PHYSHEAP_H
+#define PVRSRV_MEMALLOC_PHYSHEAP_H
+
+#include "img_defs.h"
+
+/*
+ * These IDs are replicated in the Device Memory allocation flags to allow
+ * allocations to be made in terms of their locality/use to ensure the correct
+ * physical heap is accessed for the given system/platform configuration.
+ * A system Phys Heap Config is linked to one or more Phys Heaps. When a heap
+ * is not present in the system configuration the allocation will fallback to
+ * the default GPU_LOCAL physical heap which all systems must define.
+ * See PVRSRV_MEMALLOCFLAGS_*_MAPPABLE_MASK.
+ *
+ * NOTE: Enum order important, table in physheap.c must change if order changed.
+ */
+#define PHYS_HEAP_LIST                                                                                                                           \
+	X(DEFAULT)       /* Client: default phys heap for device memory allocations */                                                               \
+	X(CPU_LOCAL)     /* Client: used for buffers with more CPU access than GPU */                                                                \
+	X(GPU_LOCAL)     /* Client: used for buffers with more GPU access than CPU */                                                                \
+	X(GPU_PRIVATE)   /* Client: used for buffers that only required GPU read/write access, not visible to the CPU. */                            \
+	X(FW_MAIN)       /* Internal: runtime data, e.g. CCBs, sync objects */                                                                       \
+	X(EXTERNAL)      /* Internal: used by some PMR import/export factories where the physical memory heap is not managed by the pvrsrv driver */ \
+	X(GPU_COHERENT)  /* Internal: used for a cache coherent region */                                                                            \
+	X(GPU_SECURE)    /* Internal: used by security validation */                                                                                 \
+	X(FW_CONFIG)     /* Internal: subheap of FW_MAIN, configuration data for FW init */                                                          \
+	X(FW_CODE)       /* Internal: used by security validation or dedicated fw */                                                                 \
+	X(FW_PRIV_DATA)  /* Internal: internal FW data (like the stack, FW control data structures, etc.) */                                         \
+	X(FW_PREMAP_PT)  /* Internal: page tables for premapped firmware memory */                                                                   \
+	X(FW_PREMAP0)    /* Internal: Host OS premap fw heap */                                                                                      \
+	X(FW_PREMAP1)    /* Internal: Guest OS 1 premap fw heap */                                                                                   \
+	X(FW_PREMAP2)    /* Internal: Guest OS 2 premap fw heap */                                                                                   \
+	X(FW_PREMAP3)    /* Internal: Guest OS 3 premap fw heap */                                                                                   \
+	X(FW_PREMAP4)    /* Internal: Guest OS 4 premap fw heap */                                                                                   \
+	X(FW_PREMAP5)    /* Internal: Guest OS 5 premap fw heap */                                                                                   \
+	X(FW_PREMAP6)    /* Internal: Guest OS 6 premap fw heap */                                                                                   \
+	X(FW_PREMAP7)    /* Internal: Guest OS 7 premap fw heap */                                                                                   \
+	X(WRAP)          /* External: Wrap memory */                                                                                                 \
+	X(DISPLAY)       /* External: Display memory */                                                                                              \
+	X(LAST)
+
+typedef enum _PVRSRV_PHYS_HEAP_
+{
+#define X(_name) PVRSRV_PHYS_HEAP_ ## _name,
+	PHYS_HEAP_LIST
+#undef X
+
+	PVRSRV_PHYS_HEAP_INVALID = 0x7FFFFFFF
+} PVRSRV_PHYS_HEAP;
+
+/* Defines the number of user mode physheaps. These physheaps are: DEFAULT, GPU_LOCAL,
+ * CPU_LOCAL, GPU_PRIVATE, GPU_SECURE. */
+#define MAX_USER_MODE_ALLOC_PHYS_HEAPS 5
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= (0x1FU + 1U), "Ensure enum fits in memalloc flags bitfield.");
+
+/*! Type conveys the class of physical heap to instantiate within Services
+ * for the physical pool of memory. */
+#define PHYS_HEAP_TYPE_LIST                                                                                                                                                                                                                                           \
+	X(UNKNOWN)              /* Not a valid value for any config */                                                                                                                                                                                                    \
+	X(UMA)                  /* Heap represents OS managed physical memory heap i.e. system RAM. Unified Memory Architecture physmem_osmem PMR factory */                                                                                                              \
+	X(LMA)                  /* Heap represents physical memory pool managed by Services i.e. carve out from system RAM or local card memory. Local Memory Architecture physmem_lma PMR factory */                                                                     \
+	X(DLM)                  /* Heap represents local card memory. Used in a DLM heap (Dedicated Local Memory) system. */                                                                                                                                              \
+	X(IMA)                  /* Heap represents phys heap that imports PMBs from a DLM heap.*/                                                                                                                                                                         \
+	X(DMA)                  /* Heap represents a physical memory pool managed by Services, alias of LMA and is only used on VZ non-native system configurations for a heap used for allocations tagged with PVRSRV_PHYS_HEAP_FW_MAIN or PVRSRV_PHYS_HEAP_FW_CONFIG */ \
+	X(WRAP)                 /* Heap used to group UM buffers given to Services. Integrity OS port only. */                                                                                                                                                            \
+	X(LAST)                                                                                                                                                                                                                                                           \
+
+typedef enum _PHYS_HEAP_TYPE_
+{
+#define X(_name) PHYS_HEAP_TYPE_ ## _name,
+	PHYS_HEAP_TYPE_LIST
+#undef X
+
+} PHYS_HEAP_TYPE;
+
+/* Defines used when interpreting the ui32PhysHeapFlags in PHYS_HEAP_MEM_STATS
+     0x000000000000000d
+     d = is this the default heap? (1=yes, 0=no)
+*/
+#define PVRSRV_PHYS_HEAP_FLAGS_IS_DEFAULT (0x1U)
+
+/* Force PHYS_HEAP_MEM_STATS size to be a multiple of 8 bytes
+ * (as type is a parameter in bridge calls)
+ */
+typedef struct PHYS_HEAP_MEM_STATS_V1_TAG
+{
+	IMG_UINT64 ui64TotalSize;     /*!< Total number of bytes in the heap. */
+	IMG_UINT64 ui64FreeSize;      /*!< Remaining number of bytes free for allocation. */
+	IMG_UINT32 ui32PhysHeapFlags; /*!< Flags associated within the heap. */
+	PHYS_HEAP_TYPE ePhysHeapType; /*!< The type of physheap. */
+} PHYS_HEAP_MEM_STATS_V1;
+typedef struct PHYS_HEAP_MEM_STATS_V2_TAG
+{
+	IMG_UINT64 ui64TotalSize;     /*!< Total number of bytes in the heap. */
+	IMG_UINT64 ui64FreeSize;      /*!< Remaining number of bytes free for allocation. */
+	IMG_UINT32 ui32PhysHeapFlags; /*!< Flags associated within the heap. */
+	PHYS_HEAP_TYPE ePhysHeapType; /*!< The type of physheap. */
+	IMG_UINT64 ui64DevicesInSPAS; /*!< A bitmap of devices that are linked to the heap via a SPAS.
+	                                   Where a device is encoded as a bit using the following:
+	                                   (1 << psDevNode->sDevId.ui32InternalID) */
+} PHYS_HEAP_MEM_STATS_V2;
+typedef PHYS_HEAP_MEM_STATS_V2 PHYS_HEAP_MEM_STATS, *PHYS_HEAP_MEM_STATS_PTR;
+
+#if defined(PHYSHEAP_STRINGS)
+
+static const char *const _pszPhysHeapStrings[] = {
+#define X(_name) #_name,
+	PHYS_HEAP_LIST
+#undef X
+};
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetClientPhysHeapTypeName
+@Description    Returns the phys heap type as a string.
+
+@Input          ePhysHeapType   The physheap type.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetClientPhysHeapTypeName(PHYS_HEAP_TYPE ePhysHeapType)
+{
+	switch (ePhysHeapType)
+	{
+#define X(_name) case PHYS_HEAP_TYPE_ ## _name: return "PHYS_HEAP_TYPE_" # _name;
+		PHYS_HEAP_TYPE_LIST
+#undef X
+		default:
+			return "Unknown Heap Type";
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetPhysHeapName
+@Description    Returns the name of a PhysHeap.
+
+@Input          ePhysHeap   The enum value of the physheap.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetPhysHeapName(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	if (ePhysHeap < 0 || ePhysHeap >= PVRSRV_PHYS_HEAP_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszPhysHeapStrings[ePhysHeap];
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetClientPhysHeapName
+@Description    Returns the name of a client PhysHeap.
+
+@Input          ePhysHeap   The enum value of the physheap.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetClientPhysHeapName(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+    if (ePhysHeap > PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+    {
+        return "Unknown Heap";
+    }
+
+    return PVRSRVGetPhysHeapName(ePhysHeap);
+}
+#endif /* PHYSHEAP_STRINGS */
+
+#endif /* PVRSRV_MEMALLOC_PHYSHEAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags.h
new file mode 100644
index 000000000000..d0703131e62d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags.h
@@ -0,0 +1,1121 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines flags used on memory allocations and mappings
+                These flags are relevant throughout the memory management
+                software stack and are specified by users of services and
+                understood by all levels of the memory management in both
+                client and server.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_MEMALLOCFLAGS_H
+#define PVRSRV_MEMALLOCFLAGS_H
+
+#include "img_types.h"
+#include "pvrsrv_memalloc_physheap.h"
+
+/*!
+  Type for specifying memory allocation flags.
+ */
+
+typedef IMG_UINT64 PVRSRV_MEMALLOCFLAGS_T;
+#define PVRSRV_MEMALLOCFLAGS_FMTSPEC IMG_UINT64_FMTSPECx
+
+#if defined(__KERNEL__)
+#include "pvrsrv_memallocflags_internal.h"
+#endif /* __KERNEL__ */
+
+/*
+ * --- MAPPING FLAGS      0..14 (15-bits) ---
+ * | 0-3    | 4-7    | 8-10        | 11-13       | 14          |
+ * | GPU-RW | CPU-RW | GPU-Caching | CPU-Caching | KM-Mappable |
+ *
+ * --- MISC FLAGS         15..20 (9-bits) ---
+ * | 15    | 16        | 17  | 18         | 19              | 20      |
+ * | Defer | Reserved  | SVM | Scratch-Pg | CPU-Cache-Clean | Zero-Pg |
+ *
+ * --- RI FLAGS  21..23 (3-bits) ---
+ * | 21     | 22       | 23        |
+ * | Import | Suballoc | FW alloc  |
+ *
+ * --- DEV CONTROL FLAGS  26..27 (2-bits) ---
+ * | 24-25 | 26-27        |
+ * | ...   | Device-Flags |
+ *
+ * --- MISC FLAGS         28..31 (4-bits) ---
+ * | 28             | 29             | 30          | 31            |
+ * | No-Cache-Align | Poison-On-Free | P.-On-Alloc | Zero-On-Alloc |
+ *
+ * --- VALIDATION FLAGS ---
+ * | 35             |
+ * | Shared-buffer  |
+ *
+ * --- OS SPECIFIC FLAGS ---
+ * | 36             | 37            | 38              |
+ * | Linux Pref CMA | Linux Movable | Linux Deny Move |
+ *
+ * --- IPA Policy ---
+ * | 53-55      |
+ * | IPA Policy |
+ *
+ * --- PHYS HEAP HINTS ---
+ * | 56            | 57-58 | 59-63          |
+ * | Mandate Heap  |       | PhysHeap Hints |
+ *
+ */
+
+/*
+ *  **********************************************************
+ *  *                                                        *
+ *  *                      MAPPING FLAGS                     *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*!
+ * This flag affects the device MMU protection flags, and specifies
+ * that the memory may be read by the GPU.
+ *
+ * Typically all device memory allocations would specify this flag.
+ *
+ * At the moment, memory allocations without this flag are not supported
+ *
+ * This flag will live with the PMR, thus subsequent mappings would
+ * honour this flag.
+ *
+ * This is a dual purpose flag.  It specifies that memory is permitted
+ * to be read by the GPU, and also requests that the allocation is
+ * mapped into the GPU as a readable mapping
+ *
+ * To be clear:
+ * - When used as an argument on PMR creation; it specifies
+ *       that GPU readable mappings will be _permitted_
+ * - When used as an argument to a "map" function: it specifies
+ *       that a GPU readable mapping is _desired_
+ * - When used as an argument to "AllocDeviceMem": it specifies
+ *       that the PMR will be created with permission to be mapped
+ *       with a GPU readable mapping, _and_ that this PMR will be
+ *       mapped with a GPU readable mapping.
+ * This distinction becomes important when (a) we export allocations;
+ * and (b) when we separate the creation of the PMR from the mapping.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_READABLE		(IMG_UINT64_C(1)<<0)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_READABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_READABLE(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_READABLE) != 0U)
+
+/*!
+ * This flag affects the device MMU protection flags, and specifies
+ * that the memory may be written by the GPU
+ *
+ * Using this flag on an allocation signifies that the allocation is
+ * intended to be written by the GPU.
+ *
+ * Omitting this flag causes a read-only mapping.
+ *
+ * This flag will live with the PMR, thus subsequent mappings would
+ * honour this flag.
+ *
+ * This is a dual purpose flag.  It specifies that memory is permitted
+ * to be written by the GPU, and also requests that the allocation is
+ * mapped into the GPU as a writable mapping (see note above about
+ * permission vs. mapping mode, and why this flag causes permissions
+ * to be inferred from mapping mode on first allocation)
+ *
+ * N.B.  This flag has no relevance to the CPU's MMU mapping, if any,
+ * and would therefore not enforce read-only mapping on CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE       (IMG_UINT64_C(1)<<1)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITEABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as GPU readable in another GPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED  (IMG_UINT64_C(1)<<2)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_READ_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as GPU writable in another GPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED (IMG_UINT64_C(1)<<3)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITE_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED) != 0U)
+
+/*!
+  The flag indicates that an allocation is mapped as readable to the CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_READABLE        (IMG_UINT64_C(1)<<4)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_READABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_READABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_READABLE) != 0U)
+
+/*!
+  The flag indicates that an allocation is mapped as writable to the CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE       (IMG_UINT64_C(1)<<5)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITEABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as CPU readable in another CPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED  (IMG_UINT64_C(1)<<6)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_READ_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as CPU writable in another CPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED (IMG_UINT64_C(1)<<7)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITE_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED) != 0U)
+
+
+/*
+ *  **********************************************************
+ *  *                                                        *
+ *  *                   CACHE CONTROL FLAGS                  *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*
+	GPU domain
+	==========
+
+	The following defines are used to control the GPU cache bit field.
+	The defines are mutually exclusive.
+
+	A helper macro, PVRSRV_GPU_CACHE_MODE, is provided to obtain just the GPU
+	cache bit field from the flags. This should be used whenever the GPU cache
+	mode needs to be determined.
+*/
+
+/*!
+  GPU domain. Flag indicating uncached memory. This means that any writes to memory
+  allocated with this flag are written straight to memory and thus are
+  coherent for any device in the system.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_UNCACHED				(IMG_UINT64_C(1)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_UNCACHED(uiFlags)				(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_UNCACHED)
+
+/*!
+   GPU domain. Use write combiner (if supported) to combine sequential writes
+   together to reduce memory access by doing burst writes.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC			(IMG_UINT64_C(0)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags)			(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC)
+
+/*!
+    GPU domain. This flag affects the GPU MMU protection flags.
+    The allocation will be cached.
+    Services will try to set the coherent bit in the GPU MMU tables so the
+    GPU cache is snooping the CPU cache. If coherency is not supported the
+    caller is responsible to ensure the caches are up to date.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT			(IMG_UINT64_C(2)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags)		(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+
+/*!
+   GPU domain. Request cached memory, but not coherent (i.e. no cache
+   snooping). Services will flush the GPU internal caches after every GPU
+   task so no cache maintenance requests from the users are necessary.
+
+    Note: We reserve 3 bits in the CPU/GPU cache mode to allow for future
+    expansion.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT		(IMG_UINT64_C(3)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags)		(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT)
+
+/*!
+    GPU domain. This flag is for internal use only and is used to indicate
+    that the underlying allocation should be cached on the GPU after all
+    the snooping and coherent checks have been done
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHED					(IMG_UINT64_C(7)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHED(uiFlags)				(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHED)
+
+/*!
+    GPU domain. GPU cache mode mask.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK			(IMG_UINT64_C(7)<<8)
+
+/*!
+  @Description    A helper macro to obtain just the GPU	cache bit field from the flags.
+                  This should be used whenever the GPU cache mode needs to be determined.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the GPU cache bit field.
+ */
+#define PVRSRV_GPU_CACHE_MODE(uiFlags)					((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK)
+
+
+/*
+	CPU domain
+	==========
+
+	The following defines are used to control the CPU cache bit field.
+	The defines are mutually exclusive.
+
+	A helper macro, PVRSRV_CPU_CACHE_MODE, is provided to obtain just the CPU
+	cache bit field from the flags. This should be used whenever the CPU cache
+	mode needs to be determined.
+*/
+
+/*!
+   CPU domain. Use write combiner (if supported) to combine sequential writes
+   together to reduce memory access by doing burst writes.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC			(IMG_UINT64_C(0)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)			(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC)
+
+/*!
+    CPU domain. This flag affects the CPU MMU protection flags.
+    The allocation will be cached.
+    Services will try to set the coherent bit in the CPU MMU tables so the
+    CPU cache is snooping the GPU cache. If coherency is not supported the
+    caller is responsible to ensure the caches are up to date.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT			(IMG_UINT64_C(2)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags)		(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+
+/*!
+    CPU domain. Request cached memory, but not coherent (i.e. no cache
+    snooping). This means that if the allocation needs to transition from
+    one device to another services has to be informed so it can
+    flush/invalidate the appropriate caches.
+
+    Note: We reserve 3 bits in the CPU/GPU cache mode to allow for future
+    expansion.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT		(IMG_UINT64_C(3)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags)		(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT)
+
+/*!
+    CPU domain. This flag is for internal use only and is used to indicate
+    that the underlying allocation should be cached on the CPU
+    after all the snooping and coherent checks have been done
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHED					(IMG_UINT64_C(7)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHED(uiFlags)				(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHED)
+
+/*!
+	CPU domain. CPU cache mode mask
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK			(IMG_UINT64_C(7)<<11)
+
+/*!
+  @Description    A helper macro to obtain just the CPU	cache bit field from the flags.
+                  This should be used whenever the CPU cache mode needs to be determined.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the CPU cache bit field.
+ */
+#define PVRSRV_CPU_CACHE_MODE(uiFlags)					((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)
+
+/* Helper flags for usual cases */
+
+/*!
+ * Memory will be write-combined on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_UNCACHED_WC		(PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC | PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_WRITE_COMBINE(uiFlags)				(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_UNCACHED_WC)
+
+/*!
+ * Memory will be cached.
+ * Services will try to set the correct flags in the MMU tables.
+ * In case there is no coherency support the caller has to ensure caches are up to date */
+#define PVRSRV_MEMALLOCFLAG_CACHE_COHERENT				(PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT | PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CACHE_COHERENT(uiFlags)			(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CACHE_COHERENT)
+
+/*!
+ * Memory will be cache-incoherent on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT			(PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CACHE_INCOHERENT(uiFlags)			(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT)
+
+/*!
+	Cache mode mask
+*/
+#define PVRSRV_CACHE_MODE(uiFlags)						(PVRSRV_GPU_CACHE_MODE(uiFlags) | PVRSRV_CPU_CACHE_MODE(uiFlags))
+
+
+/*!
+   CPU MMU Flags mask -- intended for use internal to services only
+ */
+#define PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK  (PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+												PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+												PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)
+
+/*!
+   MMU Flags mask -- intended for use internal to services only - used for
+   partitioning the flags bits and determining which flags to pass down to
+   mmu_common.c
+ */
+#define PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK  (PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK)
+
+/*!
+    Indicates that the PMR created due to this allocation will support
+    in-kernel CPU mappings.  Only privileged processes may use this flag as
+    it may cause wastage of precious kernel virtual memory on some platforms.
+ */
+#define PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE			(IMG_UINT64_C(1)<<14)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_KERNEL_CPU_MAPPABLE(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE) != 0U)
+
+
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                   ALLOC MEMORY FLAGS                   *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*!	----- Bit 15
+
+    Indicates when the allocation of physical memory pages backing the PMR
+    is carried out. When set, pages are not allocated at PMR creation but are
+    instead deferred until they are first needed, i.e. "on demand".
+    When unset, the pages may be allocated at the same time the PMR is created
+    or deferred (at the KM/Server's discretion).
+ */
+#define PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC			(IMG_UINT64_C(1)<<15)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_ON_DEMAND(uiFlags)					(((uiFlags) & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC) != 0U)
+
+/*! ----- Bit 16
+
+    This flag is unused but kept for compatibility reasons. Once not a concern
+    the flag can be removed and but reused.
+ */
+#define PVRSRV_MEMALLOCFLAG_PHYS_RESERVED				(IMG_UINT64_C(1)<<16)
+
+/*! ----- Bit 17
+
+    Indicates that the allocation will be accessed by the CPU and GPU using
+    the same virtual address, i.e. for all SVM allocs,
+    IMG_CPU_VIRTADDR == IMG_DEV_VIRTADDR
+ */
+#define PVRSRV_MEMALLOCFLAG_SVM_ALLOC					(IMG_UINT64_C(1)<<17)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_SVM_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_SVM_ALLOC(uiFlags)					(((uiFlags) & PVRSRV_MEMALLOCFLAG_SVM_ALLOC) != 0U)
+
+/*! ----- Bit 18
+
+    Indicates the particular memory that's being allocated is sparse and the
+    sparse regions should not be backed by scratch page
+ */
+#define PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING		(IMG_UINT64_C(1) << 18)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING) == 0U)
+
+/*! ----- Bit 19
+
+  Used to force Services to carry out at least one CPU cache flush and
+  invalidate on a CPU cached buffer during allocation of the memory. Applicable
+  to incoherent systems, it must be used for buffers which are CPU cached and
+  which will not be 100% written to by the CPU before the GPU accesses it. For
+  performance reasons, avoid usage if the whole buffer that is allocated is
+  written to by the CPU anyway before the next GPU kick, or if the system is
+  coherent.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN				(IMG_UINT64_C(1)<<19)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN) != 0U)
+
+/*! ----- Bit 20
+
+    Indicates the particular memory region should be backed by zero page.
+    This is different with zero on alloc flag such that only physically unbacked
+    pages are backed by zero page at the time of mapping.
+    The zero backed page is always with read only attribute irrespective of its
+    original attributes.
+ */
+#define PVRSRV_MEMALLOCFLAG_ZERO_BACKING			(IMG_UINT64_C(1) << 20)
+#define PVRSRV_IS_ZERO_BACKING_REQUIRED(uiFlags)		(((uiFlags) & \
+			PVRSRV_MEMALLOCFLAG_ZERO_BACKING) == PVRSRV_MEMALLOCFLAG_ZERO_BACKING)
+
+/*
+ ************************************************************
+ *                         RI flags                         *
+ ************************************************************
+ * The Flags are used exclusively by the RI (Reference Info)
+ * server for tracking KM allocations by various processes
+ *
+ * Import - Handle imported from other process
+ * Suballoc - Handle Suballocation of existing PMR
+ * FW Alloc - Used by FW during driver initialisation
+ *
+ * --- RI FLAGS  21..23 (3-bits) ---
+ * | 21     | 22       | 23        |
+ * | Import | Suballoc | FW alloc  |
+ *
+ */
+
+/*! ----- Bit 21
+  Used by RI server to register Allocation as an import from other process
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_IMPORT					(IMG_UINT64_C(1)<<21)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_IMPORT flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_IMPORT(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_IMPORT) != 0U)
+
+/*! ----- Bit 22
+  Used by RI server to register Allocation as a suballocation of existing PMR
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_SUBALLOC					(IMG_UINT64_C(1)<<22)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_SUBALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_SUBALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_SUBALLOC) != 0U)
+
+/*! ----- Bit 23
+  Used by RI server to register Allocation as FW/System process
+  'FW Alloc' also means the allocation is considered to belong to the SYS process
+  (ie that it will have a lifetime longer than the process which allocated it)
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC				(IMG_UINT64_C(1)<<23)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC) != 0U)
+
+/*!
+  @Description    Macro passing the PVRSRV_MEMALLOCFLAG_RI_ Flags.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the RI server bit field
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_MASK(uiFlags)			((uiFlags) & (IMG_UINT64_C(7)<<21))
+
+
+/*! ----- Bit 24
+  *
+    Not used.
+ */
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *           MEMORY ZEROING AND POISONING FLAGS           *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * Zero / Poison, on alloc/free
+ *
+ * We think the following usecases are required:
+ *
+ *  don't poison or zero on alloc or free
+ *     (normal operation, also most efficient)
+ *  poison on alloc
+ *     (for helping to highlight bugs)
+ *  poison on alloc and free
+ *     (for helping to highlight bugs)
+ *  zero on alloc
+ *     (avoid highlighting security issues in other uses of memory)
+ *  zero on alloc and poison on free
+ *     (avoid highlighting security issues in other uses of memory, while
+ *      helping to highlight a subset of bugs e.g. memory freed prematurely)
+ *
+ * Since there are more than 4, we can't encode this in just two bits,
+ * so we might as well have a separate flag for each of the three
+ * actions.
+ */
+
+/*!
+    Ensures that the memory allocated is initialised with zeroes.
+ */
+#define PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC				(IMG_UINT64_C(1)<<31)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) != 0U)
+
+/*!
+    Scribbles over the allocated memory with a poison value
+
+    Not compatible with ZERO_ON_ALLOC
+
+    Poisoning is very deliberately _not_ reflected in PDump as we want
+    a simulation to cry loudly if the initialised data propagates to a
+    result.
+ */
+#define PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC				(IMG_UINT64_C(1)<<30)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC) != 0U)
+
+#if defined(DEBUG)
+/*!
+    Causes memory to be trashed when freed, used when debugging only, not to be used
+    as a security measure.
+ */
+#define PVRSRV_MEMALLOCFLAG_POISON_ON_FREE				(IMG_UINT64_C(1)<<29)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_POISON_ON_FREE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_POISON_ON_FREE(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_POISON_ON_FREE) != 0U)
+#endif /* DEBUG */
+
+/*!
+    Avoid address alignment to a CPU or GPU cache line size.
+ */
+#define PVRSRV_MEMALLOCFLAG_NO_CACHE_LINE_ALIGN			(IMG_UINT64_C(1)<<28)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_CHECK_NO_CACHE_LINE_ALIGN flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_NO_CACHE_LINE_ALIGN(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_NO_CACHE_LINE_ALIGN) != 0U)
+
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                Device specific MMU flags               *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 26 to 27)
+ *
+ * Some services controlled devices have device specific control bits in
+ * their page table entries, we need to allow these flags to be passed down
+ * the memory management layers so the user can control these bits.
+ * For example, RGX device has the file rgx_memallocflags.h
+ */
+
+/*!
+ * Offset of device specific MMU flags.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET		26
+
+/*!
+ * Mask for retrieving device specific MMU flags.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK		(IMG_UINT64_C(3) << PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET)
+
+/*!
+  @Description    Helper macro for setting device specific MMU flags.
+  @Input  uiFlags Flag index.
+  @Return         Flag vector with the specified bit set.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(uiFlags)	\
+			(((PVRSRV_MEMALLOCFLAGS_T)(uiFlags) << PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET) & \
+			PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                 Secure validation flags                *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bit 35)
+ *
+ */
+
+/*!
+    PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER
+ */
+
+#define PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER           (IMG_UINT64_C(1)<<35)
+#define PVRSRV_CHECK_SHARED_BUFFER(uiFlags)             (((uiFlags) & PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER) != 0U)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                OS Specific alloc flags                 *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 36 to 38)
+ *
+ */
+#define PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_OFFSET 36
+#define PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK          (IMG_UINT64_C(7) << PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_OFFSET)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA        (IMG_UINT64_C(1)<<36)
+#define PVRSRV_CHECK_OS_LINUX_PREFER_CMA(uiFlags)      (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA) != 0U)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE           (IMG_UINT64_C(1)<<37)
+#define PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags)         (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE) != 0U)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE         (IMG_UINT64_C(1)<<38)
+#define PVRSRV_CHECK_OS_LINUX_DENY_MOVE(uiFlags)       (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE) != 0U)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                       IPA Policy                       *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 53 to 55)
+ *
+ */
+
+/*!
+ * Offset of Intermediate Physical Address (IPA) policy.
+ */
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET      53
+
+/*!
+ * Mask for retrieving IPA policy.
+ */
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK        (IMG_UINT64_C(7) << PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET)
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY(uiFlags)    (((uiFlags) & PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK) >> PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                 Phys Heap Hints                        *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 56 to 63)
+ *
+ */
+
+/*!
+ * Ensures Physheap isn't reassigned when considered favourable by driver under a OOM condition.
+ */
+#define PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP            (IMG_UINT64_C(1)<<56)
+#define PVRSRV_CHECK_MANDATED_PHYSHEAP(uiFlags)         (((uiFlags) & PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP) != 0U)
+
+/*!
+ * Value of enum PVRSRV_PHYS_HEAP stored in memalloc flags. If not set
+ * i.e. PVRSRV_PHYS_HEAP_DEFAULT (value 0) used, the system layer defined default physical heap is used.
+ */
+#define PVRSRV_PHYS_HEAP_HINT_SHIFT        (59)
+#define PVRSRV_PHYS_HEAP_HINT_MASK         (IMG_UINT64_C(0x1F) << PVRSRV_PHYS_HEAP_HINT_SHIFT)
+
+
+/*!
+  @Description    Macro extracting the Phys Heap hint from memalloc flag value.
+  @Input uiFlags  Allocation flags
+  @Return         returns the value of the PHYS_HEAP_HINT bitfield
+ */
+#define PVRSRV_GET_PHYS_HEAP_HINT(uiFlags)      ((PVRSRV_PHYS_HEAP)(((uiFlags) & PVRSRV_PHYS_HEAP_HINT_MASK) \
+                                                 >> PVRSRV_PHYS_HEAP_HINT_SHIFT))
+
+/*!
+  @Description    Macro converting a Phys Heap value into a memalloc bitfield
+  @Input uiFlags  Device Phys Heap
+  @Return         returns a shifted bitfield with the Device Phys Heap value
+ */
+#define PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(PhysHeap)      ((((PVRSRV_MEMALLOCFLAGS_T)PVRSRV_PHYS_HEAP_ ## PhysHeap) << \
+                                                            PVRSRV_PHYS_HEAP_HINT_SHIFT) \
+                                                           & PVRSRV_PHYS_HEAP_HINT_MASK)
+/*!
+  @Description    Macro to replace an existing phys heap hint value in flags.
+  @Input PhysHeap Phys Heap Macro
+  @Input uiFlags  Allocation flags
+  @Return         N/A
+ */
+#define PVRSRV_SET_PHYS_HEAP_HINT(PhysHeap, uiFlags)	  (uiFlags) = ((uiFlags) & ~PVRSRV_PHYS_HEAP_HINT_MASK) | \
+                                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(PhysHeap)
+
+/*!
+  @Description    Macro to replace an existing phys heap hint value using Phys Heap value.
+  @Input PhysHeap Phys Heap Value
+  @Input uiFlags  Allocation flags
+  @Return         N/A
+ */
+#define PVRSRV_CHANGE_PHYS_HEAP_HINT(Physheap, uiFlags)	  (uiFlags) = ((uiFlags) & ~PVRSRV_PHYS_HEAP_HINT_MASK) | \
+                                                           (((PVRSRV_MEMALLOCFLAGS_T)(Physheap) << \
+                                                            PVRSRV_PHYS_HEAP_HINT_SHIFT) \
+                                                           & PVRSRV_PHYS_HEAP_HINT_MASK)
+
+/*!
+  @Description    Macros checking if a Phys Heap hint is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the hint is set, false otherwise
+ */
+#define PVRSRV_CHECK_PHYS_HEAP(PhysHeap, uiFlags) (PVRSRV_PHYS_HEAP_ ## PhysHeap == PVRSRV_GET_PHYS_HEAP_HINT(uiFlags))
+
+#define PVRSRV_CHECK_FW_MAIN(uiFlags)            (PVRSRV_CHECK_PHYS_HEAP(FW_MAIN, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_CONFIG, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_CODE, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PRIV_DATA, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP0, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP1, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP2, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP3, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP4, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP5, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP6, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP7, uiFlags))
+
+/*!
+ * Secure buffer mask -- Flags in the mask are allowed for secure buffers
+ * because they are not related to CPU mappings.
+ */
+#define PVRSRV_MEMALLOCFLAGS_SECBUFMASK  ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                           PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED)
+
+/*!
+ * Trusted device mask -- Flags in the mask are allowed for trusted device
+ * because the driver cannot access the memory
+ */
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_TDFWMASK    ~(PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                           PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                           PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+#else
+#define PVRSRV_MEMALLOCFLAGS_TDFWMASK    ~(PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                           PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+#endif
+
+/*!
+  PMR flags mask -- for internal services use only.  This is the set of flags
+  that will be passed down and stored with the PMR, this also includes the
+  MMU flags which the PMR has to pass down to mm_common.c at PMRMap time.
+*/
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK  (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                            PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                            PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER | \
+                                            PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                            PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                            PVRSRV_PHYS_HEAP_HINT_MASK)
+#else
+#define PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK  (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                            PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER | \
+                                            PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP  | \
+                                            PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                            PVRSRV_PHYS_HEAP_HINT_MASK)
+#endif
+
+/*!
+ * CPU mappable mask -- Any flag set in the mask requires memory to be CPU mappable
+ */
+#define PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK (PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                                PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                                PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)
+/*!
+  RA differentiation mask
+
+  for use internal to services
+
+  this is the set of flags bits that are able to determine whether a pair of
+  allocations are permitted to live in the same page table. Allocations
+  whose flags differ in any of these places would be allocated from separate
+  RA Imports and therefore would never coexist in the same page.
+  Special cases are zeroing and poisoning of memory. The caller is responsible
+  to set the sub-allocations to the value he wants it to be. To differentiate
+  between zeroed and poisoned RA Imports does not make sense because the
+  memory might be reused.
+
+*/
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK (PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK \
+                                                      & \
+                                                      ~(PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC   | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_FREE))
+#else
+#define PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK (PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK \
+                                                      & \
+                                                      ~(PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC   | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC))
+#endif
+/*!
+  Flags that affect _allocation_
+*/
+#define PVRSRV_MEMALLOCFLAGS_PERALLOCFLAGSMASK (0xFFFFFFFFU)
+
+/*!
+  Flags that affect _mapping_
+*/
+#define PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK   (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                                    PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                                    PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+
+#if ((~(PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK) & PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK) != 0U)
+#error PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK is not a subset of PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK
+#endif
+
+
+/*!
+  Flags that affect _physical allocations_ in the DevMemX API
+ */
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_PHYSICAL_MASK (PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                                    PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                                    PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE | \
+                                                    PVRSRV_PHYS_HEAP_HINT_MASK)
+#else
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_PHYSICAL_MASK (PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                                    PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA | \
+                                                    PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE | \
+                                                    PVRSRV_PHYS_HEAP_HINT_MASK)
+#endif
+
+/*!
+  Flags that affect _virtual allocations_ in the DevMemX API
+ */
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK  (PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED)
+
+#endif /* PVRSRV_MEMALLOCFLAGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags_internal.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags_internal.h
new file mode 100644
index 000000000000..4fee3d49b927
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_memallocflags_internal.h
@@ -0,0 +1,78 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management allocation flags for internal Services
+                use only
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines flags used on memory allocations and mappings
+                These flags are relevant throughout the memory management
+                software stack and are specified by users of services and
+                understood by all levels of the memory management in the server
+                and in special cases in the client.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_MEMALLOCFLAGS_INTERNAL_H
+#define PVRSRV_MEMALLOCFLAGS_INTERNAL_H
+
+/*!
+   CPU domain. Request uncached memory. This means that any writes to memory
+   allocated with this flag are written straight to memory and thus are
+   coherent for any device in the system.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_UNCACHED (1ULL<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_UNCACHED(uiFlags) (PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_UNCACHED)
+
+/*!
+ * Memory will be uncached on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_UNCACHED (PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | PVRSRV_MEMALLOCFLAG_CPU_UNCACHED)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_UNCACHED(uiFlags) (PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_UNCACHED)
+
+#endif /* PVRSRV_MEMALLOCFLAGS_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_sync_km.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_sync_km.h
new file mode 100644
index 000000000000..04611f9f7cee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_sync_km.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File
+@Title         PVR synchronisation interface
+@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description   Types for server side code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVRSRV_SYNC_KM_H
+#define PVRSRV_SYNC_KM_H
+
+#include <powervr/pvrsrv_sync_ext.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define SYNC_FB_FILE_STRING_MAX			256
+#define SYNC_FB_MODULE_STRING_LEN_MAX	(32)
+#define	SYNC_FB_DESC_STRING_LEN_MAX		(32)
+
+/* By default, fence-sync module emits into HWPerf (of course, if enabled) and
+ * considers a process (sleepable) context */
+#define PVRSRV_FENCE_FLAG_NONE             (0U)
+#define PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT (1U << 0)
+#define PVRSRV_FENCE_FLAG_CTX_ATOMIC       (1U << 1)
+
+#if defined(__cplusplus)
+}
+#endif
+#endif	/* PVRSRV_SYNC_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlcommon.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlcommon.h
new file mode 100644
index 000000000000..9bc65fe05f98
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlcommon.h
@@ -0,0 +1,260 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Transport Layer common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common types and definitions included into
+                both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_TLCOMMON_H
+#define PVR_TLCOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+
+/*! Handle type for stream descriptor objects as created by this API */
+typedef IMG_HANDLE PVRSRVTL_SD;
+
+/*! Maximum stream name length including the null byte */
+#define PRVSRVTL_MAX_STREAM_NAME_SIZE	40U
+
+/*! Maximum number of streams expected to exist */
+#define PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER (32*PRVSRVTL_MAX_STREAM_NAME_SIZE)
+
+/*! Packet lengths are always rounded up to a multiple of 8 bytes */
+#define PVRSRVTL_PACKET_ALIGNMENT		8U
+#define PVRSRVTL_ALIGN(x)				PVR_ALIGN(x, PVRSRVTL_PACKET_ALIGNMENT)
+
+
+/*! A packet is made up of a header structure followed by the data bytes.
+ * There are 3 types of packet: normal (has data), data lost and padding,
+ * see packet flags. Header kept small to reduce data overhead.
+ *
+ * if the ORDER of the structure members is changed, please UPDATE the
+ * PVRSRVTL_PACKET_FLAG_OFFSET macro.
+ *
+ * Layout of uiTypeSize member is :
+ *
+ * |<---------------------------32-bits------------------------------>|
+ * |<----8---->|<-----1----->|<----7--->|<------------16------------->|
+ * |    Type   | Drop-Oldest |  UNUSED  |             Size            |
+ *
+ */
+typedef struct
+{
+	IMG_UINT32 uiTypeSize;	/*!< Type, Drop-Oldest flag & number of bytes following header */
+	IMG_UINT32 uiReserved;	/*!< Reserve, packets and data must be 8 byte aligned */
+
+	/* First bytes of TL packet data follow header ... */
+} PVRSRVTL_PACKETHDR, *PVRSRVTL_PPACKETHDR;
+
+/* Structure must always be a size multiple of 8 as stream buffer
+ * still an array of IMG_UINT32s.
+ */
+static_assert((sizeof(PVRSRVTL_PACKETHDR) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(PVRSRVTL_PACKETHDR) must be a multiple of 8");
+
+/*! Packet header reserved word fingerprint "TLP1" */
+#define PVRSRVTL_PACKETHDR_RESERVED 0x31504C54U
+
+/*! Packet header mask used to extract the size from the uiTypeSize member.
+ * Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_SIZE_MASK    0x0000FFFFU
+#define PVRSRVTL_MAX_PACKET_SIZE        (PVRSRVTL_PACKETHDR_SIZE_MASK & ~0xFU)
+
+
+/*! Packet header mask used to extract the type from the uiTypeSize member.
+ * Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_TYPE_MASK    0xFF000000U
+#define PVRSRVTL_PACKETHDR_TYPE_OFFSET  24U
+
+/*! Packet header mask used to check if packets before this one were dropped
+ * or not. Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_OLDEST_DROPPED_MASK    0x00800000U
+#define PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET    23U
+
+/*! Packet type enumeration.
+ */
+typedef IMG_UINT32 PVRSRVTL_PACKETTYPE;
+
+/*! Undefined packet */
+#define PVRSRVTL_PACKETTYPE_UNDEF 0U
+
+/*! Normal packet type. Indicates data follows the header.
+ */
+#define PVRSRVTL_PACKETTYPE_DATA 1U
+
+/*! When seen this packet type indicates that at this moment in the stream
+ * packet(s) were not able to be accepted due to space constraints and
+ * that recent data may be lost - depends on how the producer handles the
+ * error. Such packets have no data, data length is 0.
+ */
+#define PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED 2U
+
+/*! Packets with this type set are padding packets that contain undefined
+ * data and must be ignored/skipped by the client. They are used when the
+ * circular stream buffer wraps around and there is not enough space for
+ * the data at the end of the buffer. Such packets have a length of 0 or
+ * more.
+ */
+#define PVRSRVTL_PACKETTYPE_PADDING 3U
+
+/*! This packet type conveys to the stream consumer that the stream
+ * producer has reached the end of data for that data sequence. The
+ * TLDaemon has several options for processing these packets that can
+ * be selected on a per stream basis.
+ */
+#define PVRSRVTL_PACKETTYPE_MARKER_EOS 4U
+
+/*! This is same as PVRSRVTL_PACKETTYPE_MARKER_EOS but additionally removes
+ * old data record output file before opening new/next one
+ */
+#define PVRSRVTL_PACKETTYPE_MARKER_EOS_REMOVEOLD 5U
+
+/*! Packet emitted on first stream opened by writer. Packet carries a name
+ * of the opened stream in a form of null-terminated string.
+ */
+#define PVRSRVTL_PACKETTYPE_STREAM_OPEN_FOR_WRITE 6U
+
+/*! Packet emitted on last stream closed by writer. Packet carries a name
+ * of the closed stream in a form of null-terminated string.
+ */
+#define PVRSRVTL_PACKETTYPE_STREAM_CLOSE_FOR_WRITE 7U
+
+#define PVRSRVTL_PACKETTYPE_LAST 8U
+
+/* The SET_PACKET_* macros rely on the order the PVRSRVTL_PACKETHDR members are declared:
+ * uiFlags is the upper half of a structure consisting of 2 uint16 quantities.
+ */
+#define PVRSRVTL_SET_PACKET_DATA(len)       (len) | (PVRSRVTL_PACKETTYPE_DATA                     << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_PADDING(len)    (len) | (PVRSRVTL_PACKETTYPE_PADDING                  << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_WRITE_FAILED    (0U)   | (PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_HDR(len, type)  (len) | ((type)                                       << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+
+/*! Returns the number of bytes of data in the packet.
+ * p may be any address type.
+ */
+#define GET_PACKET_DATA_LEN(p)	\
+	((IMG_UINT32) ((PVRSRVTL_PPACKETHDR) (void *) (p))->uiTypeSize & PVRSRVTL_PACKETHDR_SIZE_MASK)
+
+
+/*! Returns a IMG_BYTE* pointer to the first byte of data in the packet */
+#define GET_PACKET_DATA_PTR(p)	\
+	(((IMG_UINT8 *) (void *) (p)) + sizeof(PVRSRVTL_PACKETHDR))
+
+/*! Turns the packet address p into a PVRSRVTL_PPACKETHDR pointer type.
+ */
+#define GET_PACKET_HDR(p)		((PVRSRVTL_PPACKETHDR) ((void *) (p)))
+
+/*! Given a PVRSRVTL_PPACKETHDR address, return the address of the next pack
+ *  It is up to the caller to determine if the new address is within the
+ *  packet buffer.
+ */
+#define GET_NEXT_PACKET_ADDR(p) \
+	GET_PACKET_HDR( \
+		GET_PACKET_DATA_PTR(p) + \
+		( \
+			(GET_PACKET_DATA_LEN(p) + (PVRSRVTL_PACKET_ALIGNMENT-1U)) & \
+			(~(PVRSRVTL_PACKET_ALIGNMENT-1U)) \
+		) \
+	)
+
+/*! Get the type of the packet. p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define GET_PACKET_TYPE(p)		(((p)->uiTypeSize & PVRSRVTL_PACKETHDR_TYPE_MASK)>>PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+
+/*! Set PACKETS_DROPPED flag in packet header as a part of uiTypeSize.
+ * p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define SET_PACKETS_DROPPED(p)		(((p)->uiTypeSize) | (1UL << PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET))
+
+/*! Check if packets were dropped before this packet.
+ * p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define CHECK_PACKETS_DROPPED(p)	((((p)->uiTypeSize & PVRSRVTL_PACKETHDR_OLDEST_DROPPED_MASK)>>PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET) != 0U)
+
+/*! Flags for use with PVRSRVTLOpenStream
+ * 0x01 - Do not block in PVRSRVTLAcquireData() when no bytes are available
+ * 0x02 - When the stream does not exist wait for a bit (2s) in
+ *        PVRSRVTLOpenStream() and then exit with a timeout error if it still
+ *        does not exist.
+ * 0x04 - Open stream for write only operations.
+ *        If flag is not used stream is opened as read-only. This flag is
+ *        required if one wants to call reserve/commit/write function on the
+ *        stream descriptor. Read from on the stream descriptor opened
+ *        with this flag will fail.
+ * 0x08 - Disable Producer Callback.
+ *        If this flag is set and the stream becomes empty, do not call any
+ *        associated producer callback to generate more data from the reader
+ *        context.
+ * 0x10 - Reset stream on open.
+ *        When this flag is used the stream will drop all of the stored data.
+ * 0x20 - Limit read position to the write position at time the stream
+ *        was opened. Hence this flag will freeze the content read to that
+ *        produced before the stream was opened for reading.
+ * 0x40 - Ignore Open Callback.
+ *        When this flag is set ignore any OnReaderOpenCallback setting for
+ *        the stream. This allows access to the stream to be made without
+ *        generating any extra packets into the stream.
+ */
+
+#define PVRSRV_STREAM_FLAG_NONE                        (0U)
+#define PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING         (1U<<0)
+#define PVRSRV_STREAM_FLAG_OPEN_WAIT                   (1U<<1)
+#define PVRSRV_STREAM_FLAG_OPEN_WO                     (1U<<2)
+#define PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK   (1U<<3)
+#define PVRSRV_STREAM_FLAG_RESET_ON_OPEN               (1U<<4)
+#define PVRSRV_STREAM_FLAG_READ_LIMIT                  (1U<<5)
+#define PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK        (1U<<6)
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_TLCOMMON_H */
+/******************************************************************************
+ End of file (pvrsrv_tlcommon.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlstreams.h b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlstreams.h
new file mode 100644
index 000000000000..624b8eb1f47d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrsrv_tlstreams.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Transport Layer stream names
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common types and definitions included into
+                both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_TLSTREAMS_H
+#define PVRSRV_TLSTREAMS_H
+
+#define PVRSRV_TL_CTRL_STREAM "tlctrl"
+
+#define PVRSRV_TL_HWPERF_RGX_FW_STREAM      "hwperf_fw_"
+#define PVRSRV_TL_HWPERF_HOST_SERVER_STREAM "hwperf_host_"
+
+#define PVRSRV_TL_FTRACE_RGX_FW_STREAM      "ftrace_fw_"
+
+/* Host HWPerf client stream names are of the form 'hwperf_client_<pid>' */
+#define PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM         "hwperf_client_"
+#define PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC "hwperf_client_%u_%u"
+
+#endif /* PVRSRV_TLSTREAMS_H */
+
+/******************************************************************************
+ End of file (pvrsrv_tlstreams.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/pvrversion.h b/drivers/gpu/drm/img/img-volcanic/include/pvrversion.h
new file mode 100644
index 000000000000..4d17f6e55594
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/pvrversion.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File           pvrversion.h
+@Title          PowerVR version numbers and strings.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Version numbers and strings for PowerVR components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRVERSION_H
+#define PVRVERSION_H
+
+#define PVRVERSION_MAJ               24U
+#define PVRVERSION_MIN               2U
+
+#define PVRVERSION_FAMILY           "rogueddk"
+#define PVRVERSION_BRANCHNAME       "24.2"
+#define PVRVERSION_BUILD             6643903
+#define PVRVERSION_BSCONTROL        "Rogue_DDK_Linux_WS"
+
+#define PVRVERSION_STRING           "Rogue_DDK_Linux_WS rogueddk 24.2@6643903"
+#define PVRVERSION_STRING_SHORT     "24.2@6643903"
+
+#define COPYRIGHT_TXT               "Copyright (c) Imagination Technologies Ltd. All Rights Reserved."
+
+#define PVRVERSION_BUILD_HI          664
+#define PVRVERSION_BUILD_LO          3903
+#define PVRVERSION_STRING_NUMERIC   "24.2.664.3903"
+
+#define PVRVERSION_PACK(MAJOR,MINOR) (((IMG_UINT32)((IMG_UINT32)(MAJOR) & 0xFFFFU) << 16U) | (((MINOR) & 0xFFFFU) << 0U))
+#define PVRVERSION_UNPACK_MAJ(VERSION) (((VERSION) >> 16U) & 0xFFFFU)
+#define PVRVERSION_UNPACK_MIN(VERSION) (((VERSION) >> 0U) & 0xFFFFU)
+
+#endif /* PVRVERSION_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_common.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_common.h
new file mode 100644
index 000000000000..55b0ed450e1c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_common.h
@@ -0,0 +1,262 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Common Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common types and definitions for RGX software
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_COMMON_H
+#define RGX_COMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+/* Included to get the BVNC_KM_N defined and other feature defs */
+#include "km/rgxdefs_km.h"
+
+#include "rgx_common_asserts.h"
+
+typedef struct RGXFWIF_DEV_VIRTADDR_
+{
+	IMG_UINT32	ui32Addr;
+} RGXFWIF_DEV_VIRTADDR;
+
+/* Virtualisation validation builds are meant to test the VZ-related hardware without a fully virtualised platform.
+ * As such a driver can support either the vz-validation code or real virtualisation. */
+#if defined(SUPPORT_GPUVIRT_VALIDATION) && (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+#error "Invalid build configuration: Virtualisation support (RGX_NUM_DRIVERS_SUPPORTED > 1) and virtualisation validation code (SUPPORT_GPUVIRT_VALIDATION) are mutually exclusive."
+#endif
+
+/* The RGXFWIF_DM defines assume only one of RGX_FEATURE_TLA or
+ * RGX_FEATURE_FASTRENDER_DM is present. Ensure this with a compile-time check.
+ */
+#if defined(RGX_FEATURE_TLA) && defined(RGX_FEATURE_FASTRENDER_DM)
+#error "Both RGX_FEATURE_TLA and RGX_FEATURE_FASTRENDER_DM defined. Fix code to handle this!"
+#endif
+
+#define FW_OSID							(0U)
+#define MMU_CONTEXT_MAPPING_FWPRIV		(0U) /* FW code/private data */
+
+#if (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID) || defined(RGX_FEATURE_MIPS)
+/* The Firmware accesses its private code & data and the interface
+ * memory it shares with the KM drivers using the same MMU context */
+#define MMU_CONTEXT_MAPPING_FWIF		MMU_CONTEXT_MAPPING_FWPRIV
+#else
+/* The Firmware accesses the interface memory it shares
+ * with the KM drivers using a reserved MMU context */
+#define MMU_CONTEXT_MAPPING_FWIF		(7U)
+#endif
+
+/*! The master definition for data masters known to the firmware of RGX.
+ * When a new DM is added to this list, relevant entry should be added to
+ * RGX_HWPERF_DM enum list.
+ * The DM in a V1 HWPerf packet uses this definition. */
+
+typedef IMG_UINT32 RGXFWIF_DM;
+
+#define	RGXFWIF_DM_GP			IMG_UINT32_C(0)
+/* Either TDM or 2D DM is present. The above build time error is present to verify this */
+#define	RGXFWIF_DM_2D			IMG_UINT32_C(1) /* when RGX_FEATURE_TLA defined */
+#define	RGXFWIF_DM_TDM			IMG_UINT32_C(1) /* when RGX_FEATURE_FASTRENDER_DM defined */
+
+#define	RGXFWIF_DM_GEOM			IMG_UINT32_C(2)
+#define	RGXFWIF_DM_3D			IMG_UINT32_C(3)
+#define	RGXFWIF_DM_CDM			IMG_UINT32_C(4)
+#define	RGXFWIF_DM_RAY			IMG_UINT32_C(5)
+#define	RGXFWIF_DM_GEOM2		IMG_UINT32_C(6)
+#define	RGXFWIF_DM_GEOM3		IMG_UINT32_C(7)
+#define	RGXFWIF_DM_GEOM4		IMG_UINT32_C(8)
+
+#define	RGXFWIF_DM_LAST			RGXFWIF_DM_GEOM4
+
+typedef IMG_UINT32 RGX_KICK_TYPE_DM;
+#define RGX_KICK_TYPE_DM_GP		IMG_UINT32_C(0x001)
+#define RGX_KICK_TYPE_DM_TDM_2D	IMG_UINT32_C(0x002)
+#define RGX_KICK_TYPE_DM_TA		IMG_UINT32_C(0x004)
+#define RGX_KICK_TYPE_DM_3D		IMG_UINT32_C(0x008)
+#define RGX_KICK_TYPE_DM_CDM	IMG_UINT32_C(0x010)
+#define RGX_KICK_TYPE_DM_RTU	IMG_UINT32_C(0x020)
+#define RGX_KICK_TYPE_DM_SHG	IMG_UINT32_C(0x040)
+#define RGX_KICK_TYPE_DM_TQ2D	IMG_UINT32_C(0x080)
+#define RGX_KICK_TYPE_DM_TQ3D	IMG_UINT32_C(0x100)
+#define RGX_KICK_TYPE_DM_RAY	IMG_UINT32_C(0x200)
+#define RGX_KICK_TYPE_DM_LAST	IMG_UINT32_C(0x400)
+
+/* Maximum number of DM in use: GP, 2D/TDM, GEOM, 3D, CDM, RDM, GEOM2, GEOM3, GEOM4 */
+#define RGXFWIF_DM_MAX			(RGXFWIF_DM_LAST + 1U)
+/* The set of DMs for gathering stats on GPU utilisation excludes GP */
+#define RGXFWIF_GPU_UTIL_DM_MAX (RGXFWIF_DM_MAX - 1U)
+
+/*
+ * Data Master Tags to be appended to resources created on behalf of each RGX
+ * Context.
+ */
+#define RGX_RI_DM_TAG_KS   'K'
+#define RGX_RI_DM_TAG_CDM  'C'
+#define RGX_RI_DM_TAG_RC   'R' /* To be removed once TA/3D Timelines are split */
+#define RGX_RI_DM_TAG_TA   'V'
+#define RGX_RI_DM_TAG_GEOM 'V'
+#define RGX_RI_DM_TAG_3D   'P'
+#define RGX_RI_DM_TAG_TDM  'T'
+#define RGX_RI_DM_TAG_TQ2D '2'
+#define RGX_RI_DM_TAG_TQ3D 'Q'
+#define RGX_RI_DM_TAG_RAY  'r'
+
+/*
+ * Client API Tags to be appended to resources created on behalf of each
+ * Client API.
+ */
+#define RGX_RI_CLIENT_API_GLES1    '1'
+#define RGX_RI_CLIENT_API_GLES3    '3'
+#define RGX_RI_CLIENT_API_VULKAN   'V'
+#define RGX_RI_CLIENT_API_EGL      'E'
+#define RGX_RI_CLIENT_API_OPENCL   'C'
+#define RGX_RI_CLIENT_API_OPENGL   'G'
+#define RGX_RI_CLIENT_API_SERVICES 'S'
+#define RGX_RI_CLIENT_API_WSEGL    'W'
+#define RGX_RI_CLIENT_API_ANDROID  'A'
+#define RGX_RI_CLIENT_API_LWS      'L'
+
+/*
+ * Format a RI annotation for a given RGX Data Master context
+ */
+#define RGX_RI_FORMAT_DM_ANNOTATION(annotation, dmTag, clientAPI) do         \
+	{                                                                        \
+		(annotation)[0] = (dmTag);                                           \
+		(annotation)[1] = (clientAPI);                                       \
+		(annotation)[2] = '\0';                                              \
+	} while (false)
+
+/*!
+ ******************************************************************************
+ * RGXFW Compiler alignment definitions
+ *****************************************************************************/
+#if defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES) || defined(INTEGRITY_OS)
+#define RGXFW_ALIGN			__attribute__ ((aligned (8)))
+#define	RGXFW_ALIGN_DCACHEL		__attribute__((aligned (64)))
+#elif defined(_MSC_VER)
+#define RGXFW_ALIGN			__declspec(align(8))
+#define	RGXFW_ALIGN_DCACHEL		__declspec(align(64))
+#pragma warning (disable : 4324)
+#else
+#error "Align MACROS need to be defined for this compiler"
+#endif
+
+/*!
+ ******************************************************************************
+ * Force 8-byte alignment for structures allocated uncached.
+ *****************************************************************************/
+#define UNCACHED_ALIGN      RGXFW_ALIGN
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation states
+ *****************************************************************************/
+#define RGXFWIF_GPU_UTIL_STATE_IDLE      (0U)
+#define RGXFWIF_GPU_UTIL_STATE_ACTIVE    (1U)
+#define RGXFWIF_GPU_UTIL_STATE_BLOCKED   (2U)
+#define RGXFWIF_GPU_UTIL_STATE_NUM       (3U)
+/* the state below "combines" IDLE and BLOCKED
+ * and is used when we only care about GPU being in ACTIVE or not */
+#define RGXFWIF_GPU_UTIL_STATE_INACTIVE  (0U)
+/* when we combine IDLE and BLOCKED we end up with one state less */
+#define RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM       (RGXFWIF_GPU_UTIL_STATE_NUM-1U)
+#define RGXFWIF_GPU_UTIL_STATE_MASK      IMG_UINT64_C(0x0000000000000003)
+#define RGXFWIF_GPU_UTIL_STATE_MASK32    IMG_UINT32_C(0x00000003)
+
+
+/*
+ * Maximum amount of register writes that can be done by the register
+ * programmer (FW or META DMA). This is not a HW limitation, it is only
+ * a protection against malformed inputs to the register programmer.
+ */
+#define RGX_MAX_NUM_REGISTER_PROGRAMMER_WRITES  (256U)
+
+/* FW common context priority. */
+/*!
+ * @AddToGroup WorkloadContexts
+ * @{
+ */
+#define RGX_CTX_PRIORITY_REALTIME  (INT32_MAX)
+#define RGX_CTX_PRIORITY_HIGH      (2) /*!< HIGH priority */
+#define RGX_CTX_PRIORITY_MEDIUM    (1) /*!< MEDIUM priority */
+#define RGX_CTX_PRIORITY_LOW       (0) /*!< LOW priority */
+/*!
+ * @} End of AddToGroup WorkloadContexts
+ */
+
+
+/*
+ *   Use of the 32-bit context property flags mask
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *                                   0
+ *                                   |
+ *    -------------------------------x
+ */
+/*
+ * Context creation flags
+ * (specify a context's properties at creation time)
+ */
+#define RGX_CONTEXT_FLAG_DISABLESLR					(1UL << 0) /*!< Disable SLR */
+
+/* Bitmask of context flags allowed to be modified after context create. */
+#define RGX_CONTEXT_FLAGS_WRITEABLE_MASK            (RGX_CONTEXT_FLAG_DISABLESLR)
+
+/* List of attributes that may be set for a context */
+typedef IMG_UINT32 RGX_CONTEXT_PROPERTY;
+#define RGX_CONTEXT_PROPERTY_FLAGS 0U /*!< Context flags */
+
+/* MMU4 supported number of ranges */
+#define RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES (4U)
+#define RGX_MMU_RANGE_NON4KHEAP          (0U)
+#define RGX_MMU_RANGE_GLOBAL             (RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES - 1)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_COMMON_H */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_common_asserts.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_common_asserts.h
new file mode 100644
index 000000000000..c571cc6f008e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_common_asserts.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Common Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common types and definitions for RGX software
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_COMMON_ASSERTS_H
+#define RGX_COMMON_ASSERTS_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*! This macro represents a mask of LSBs that must be zero on data structure
+ * sizes and offsets to ensure they are 8-byte granular on types shared between
+ * the FW and host driver */
+#define RGX_FW_ALIGNMENT_LSB (7U)
+
+/*! Macro to test structure size alignment */
+#define RGX_FW_STRUCT_SIZE_ASSERT(_a)	\
+	static_assert((sizeof(_a) & RGX_FW_ALIGNMENT_LSB) == 0U,	\
+				  "Size of " #_a " is not properly aligned")
+
+/*! Macro to test structure member alignment */
+#define RGX_FW_STRUCT_OFFSET_ASSERT(_a, _b)	\
+	static_assert((offsetof(_a, _b) & RGX_FW_ALIGNMENT_LSB) == 0U,	\
+				  "Offset of " #_a "." #_b " is not properly aligned")
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_COMMON_ASSERTS_H */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_compat_bvnc.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_compat_bvnc.h
new file mode 100644
index 000000000000..c3e1333cdb0f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_compat_bvnc.h
@@ -0,0 +1,140 @@
+/*************************************************************************/ /*!
+@File           rgx_compat_bvnc.h
+@Title          BVNC compatibility check utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used for packing BNC and V.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_COMPAT_BVNC_H)
+#define RGX_COMPAT_BVNC_H
+
+#include "img_types.h"
+
+#if defined(RGX_FIRMWARE)               /* Services firmware */
+# include "rgxfw_utils.h"
+# define PVR_COMPAT_ASSERT RGXFW_ASSERT
+#elif !defined(RGX_BUILD_BINARY)        /* Services host driver code */
+# include "pvr_debug.h"
+# define PVR_COMPAT_ASSERT PVR_ASSERT
+#else                                   /* FW user-mode tools */
+# include <assert.h>
+# define PVR_COMPAT_ASSERT assert
+#endif
+
+/* 64bit endian conversion macros */
+#if defined(__BIG_ENDIAN__)
+#define RGX_INT64_TO_BE(N) (N)
+#define RGX_INT64_FROM_BE(N) (N)
+#define RGX_INT32_TO_BE(N) (N)
+#define RGX_INT32_FROM_BE(N) (N)
+#else
+#define RGX_INT64_TO_BE(N)        \
+	((((N) >> 56)   & 0xff)       \
+	 | (((N) >> 40) & 0xff00)     \
+	 | (((N) >> 24) & 0xff0000)   \
+	 | (((N) >> 8)  & 0xff000000U) \
+	 | ((N)                << 56) \
+	 | (((N) & 0xff00)     << 40) \
+	 | (((N) & 0xff0000)   << 24) \
+	 | (((N) & 0xff000000U) << 8))
+#define RGX_INT64_FROM_BE(N) RGX_INT64_TO_BE(N)
+
+#define RGX_INT32_TO_BE(N)   \
+	((((N) >> 24)  & 0xff)   \
+	 | (((N) >> 8) & 0xff00) \
+	 | ((N)           << 24) \
+	 | ((((N) & 0xff00) << 8)))
+#define RGX_INT32_FROM_BE(N) RGX_INT32_TO_BE(N)
+#endif
+
+/******************************************************************************
+ * RGX Version packed into 64-bit (BVNC) to be used by Compatibility Check
+ *****************************************************************************/
+
+#define RGX_BVNC_PACK_SHIFT_B 48
+#define RGX_BVNC_PACK_SHIFT_V 32
+#define RGX_BVNC_PACK_SHIFT_N 16
+#define RGX_BVNC_PACK_SHIFT_C 0
+
+#define RGX_BVNC_PACK_MASK_B (IMG_UINT64_C(0xFFFF000000000000))
+#define RGX_BVNC_PACK_MASK_V (IMG_UINT64_C(0x0000FFFF00000000))
+#define RGX_BVNC_PACK_MASK_N (IMG_UINT64_C(0x00000000FFFF0000))
+#define RGX_BVNC_PACK_MASK_C (IMG_UINT64_C(0x000000000000FFFF))
+
+#define RGX_BVNC_PACKED_EXTR_B(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_B) >> RGX_BVNC_PACK_SHIFT_B))
+#define RGX_BVNC_PACKED_EXTR_V(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_V) >> RGX_BVNC_PACK_SHIFT_V))
+#define RGX_BVNC_PACKED_EXTR_N(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_N) >> RGX_BVNC_PACK_SHIFT_N))
+#define RGX_BVNC_PACKED_EXTR_C(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_C) >> RGX_BVNC_PACK_SHIFT_C))
+
+#define RGX_BVNC_EQUAL(L,R,all,version,bvnc) do {															\
+										(bvnc) = IMG_FALSE;													\
+										(version) = ((L).ui32LayoutVersion == (R).ui32LayoutVersion);		\
+										if (version)														\
+										{																	\
+											(bvnc) = ((L).ui64BVNC == (R).ui64BVNC);						\
+										}																	\
+										(all) = (version) && (bvnc);										\
+									} while (false)
+
+
+/**************************************************************************//**
+ * Utility function for packing BVNC
+ *****************************************************************************/
+static inline IMG_UINT64 rgx_bvnc_pack(IMG_UINT32 ui32B, IMG_UINT32 ui32V, IMG_UINT32 ui32N, IMG_UINT32 ui32C)
+{
+	/*
+	 * Test for input B, V, N and C exceeding max bit width.
+	 */
+	PVR_COMPAT_ASSERT((ui32B & (~(RGX_BVNC_PACK_MASK_B >> RGX_BVNC_PACK_SHIFT_B))) == 0U);
+	PVR_COMPAT_ASSERT((ui32V & (~(RGX_BVNC_PACK_MASK_V >> RGX_BVNC_PACK_SHIFT_V))) == 0U);
+	PVR_COMPAT_ASSERT((ui32N & (~(RGX_BVNC_PACK_MASK_N >> RGX_BVNC_PACK_SHIFT_N))) == 0U);
+	PVR_COMPAT_ASSERT((ui32C & (~(RGX_BVNC_PACK_MASK_C >> RGX_BVNC_PACK_SHIFT_C))) == 0U);
+
+	return (((IMG_UINT64)ui32B << RGX_BVNC_PACK_SHIFT_B) |
+			((IMG_UINT64)ui32V << RGX_BVNC_PACK_SHIFT_V) |
+			((IMG_UINT64)ui32N << RGX_BVNC_PACK_SHIFT_N) |
+			((IMG_UINT64)ui32C << RGX_BVNC_PACK_SHIFT_C));
+}
+
+
+#endif /* RGX_COMPAT_BVNC_H */
+
+/******************************************************************************
+ End of file (rgx_compat_bvnc.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_resetframework.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_resetframework.h
new file mode 100644
index 000000000000..e60bafd84536
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_resetframework.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_resetframework.h
+@Title          Post-reset work-around framework FW interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_RESETFRAMEWORK_H)
+#define RGX_FWIF_RESETFRAMEWORK_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+
+typedef struct
+{
+	union
+	{
+		IMG_UINT64	uCDMReg_CDM_CB_BASE;			//  defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+		IMG_UINT64	uCDMReg_CDM_CTRL_STREAM_BASE;	// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+	};
+	IMG_UINT64	uCDMReg_CDM_CB_QUEUE;				// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+	IMG_UINT64	uCDMReg_CDM_CB;						// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+} RGXFWIF_RF_REGISTERS;
+
+typedef struct
+{
+	/* THIS MUST BE THE LAST MEMBER OF THE CONTAINING STRUCTURE */
+	RGXFWIF_RF_REGISTERS RGXFW_ALIGN sFWRegisters;
+
+} RGXFWIF_RF_CMD;
+
+/* to opaquely allocate and copy in the kernel */
+#define RGXFWIF_RF_CMD_SIZE  sizeof(RGXFWIF_RF_CMD)
+
+#endif /* RGX_FWIF_RESETFRAMEWORK_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_sf.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_sf.h
new file mode 100644
index 000000000000..a190875b5e6d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_fwif_sf.h
@@ -0,0 +1,1067 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_sf.h
+@Title          RGX firmware interface string format specifiers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the rgx firmware logging messages. The following
+                list are the messages the firmware prints. Changing anything
+                but the first column or spelling mistakes in the strings will
+                break compatibility with log files created with older/newer
+                firmware versions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_SF_H
+#define RGX_FWIF_SF_H
+
+/******************************************************************************
+ * *DO*NOT* rearrange or delete lines in SFIDLIST or SFGROUPLIST or you
+ *           WILL BREAK fw tracing message compatibility with previous
+ *           fw versions. Only add new ones, if so required.
+ *****************************************************************************/
+/* Available log groups */
+#define RGXFW_LOG_SFGROUPLIST       \
+	X(RGXFW_GROUP_NULL,NULL)        \
+	X(RGXFW_GROUP_MAIN,MAIN)        \
+	X(RGXFW_GROUP_CLEANUP,CLEANUP)  \
+	X(RGXFW_GROUP_CSW,CSW)          \
+	X(RGXFW_GROUP_PM, PM)           \
+	X(RGXFW_GROUP_RTD,RTD)          \
+	X(RGXFW_GROUP_SPM,SPM)          \
+	X(RGXFW_GROUP_MTS,MTS)          \
+	X(RGXFW_GROUP_BIF,BIF)          \
+	X(RGXFW_GROUP_MISC,MISC)        \
+	X(RGXFW_GROUP_POW,POW)          \
+	X(RGXFW_GROUP_HWR,HWR)          \
+	X(RGXFW_GROUP_HWP,HWP)          \
+	X(RGXFW_GROUP_RPM,RPM)          \
+	X(RGXFW_GROUP_DMA,DMA)          \
+	X(RGXFW_GROUP_DBG,DBG)          \
+	X(RGXFW_GROUP_VZ,VZ)            \
+	X(RGXFW_GROUP_SAFETY,SAFETY)    \
+	X(RGXFW_GROUP_VERBOSE,VERBOSE)  \
+	X(RGXFW_GROUP_CUSTOMER,CUSTOMER)
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief FW Trace log groups(GID) list
+ */
+enum RGXFW_LOG_SFGROUPS {
+#define X(A,B) A,
+	RGXFW_LOG_SFGROUPLIST
+#undef X
+};
+
+#define IMG_SF_STRING_MAX_SIZE 256U
+
+typedef struct {
+	IMG_UINT32 ui32Id;
+	IMG_CHAR sName[IMG_SF_STRING_MAX_SIZE];
+} RGXFW_STID_FMT; /*  pair of string format id and string formats */
+
+typedef struct {
+	IMG_UINT32 ui32Id;
+	const IMG_CHAR *psName;
+} RGXKM_STID_FMT; /*  pair of string format id and string formats */
+
+/* Table of String Format specifiers, the group they belong and the number of
+ * arguments each expects. Xmacro styled macros are used to generate what is
+ * needed without requiring hand editing.
+ *
+ * id		: id within a group
+ * gid		: group id
+ * Sym name	: name of enumerations used to identify message strings
+ * String	: Actual string
+ * #args	: number of arguments the string format requires
+ */
+#define RGXFW_LOG_SFIDLIST \
+/*id, gid,              id name,        string,                           # arguments */ \
+X(  0, RGXFW_GROUP_NULL, RGXFW_SF_FIRST, "You should not use this string", 0) \
+\
+X(  1, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D_DEPRECATED, "Kick 3D: FWCtx 0x%08.8x @ %u, RTD 0x%08x. Partial render:%u, CSW resume:%u, prio:%d", 6) \
+X(  2, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_FINISHED, "3D finished, HWRTData0State=%x, HWRTData1State=%x", 2) \
+X(  3, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK3D_TQ_DEPRECATED, "Kick 3D TQ: FWCtx 0x%08.8x @ %u, CSW resume:%u, prio: %d", 4) \
+X(  4, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_TQ_FINISHED, "3D Transfer finished", 0) \
+X(  5, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_DEPRECATED, "Kick Compute: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X(  6, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_FINISHED, "Compute finished", 0) \
+X(  7, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA_DEPRECATED, "Kick TA: FWCtx 0x%08.8x @ %u, RTD 0x%08x. First kick:%u, Last kick:%u, CSW resume:%u, prio:%d", 7) \
+X(  8, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_FINISHED, "TA finished", 0) \
+X(  9, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_RESTART_AFTER_PRENDER, "Restart TA after partial render", 0) \
+X( 10, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_RESUME_WOUT_PRENDER, "Resume TA without partial render", 0) \
+X( 11, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OOM, "Out of memory! Context 0x%08x, HWRTData 0x%x", 2) \
+X( 12, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TLA_DEPRECATED, "Kick TLA: FWCtx 0x%08.8x @ %u, prio:%d", 3) \
+X( 13, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TLA_FINISHED, "TLA finished", 0) \
+X( 14, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CCCB_WOFF_UPDATE, "cCCB Woff update = %u, DM = %u, FWCtx = 0x%08.8x", 3) \
+X( 16, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_START, "UFO Checks for FWCtx 0x%08.8x @ %u", 2) \
+X( 17, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK, "UFO Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 18, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_SUCCEEDED, "UFO Checks succeeded", 0) \
+X( 19, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_PR_CHECK, "UFO PR-Check: [0x%08.8x] is 0x%08.8x requires >= 0x%08.8x", 3) \
+X( 20, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK_START, "UFO SPM PR-Checks for FWCtx 0x%08.8x", 1) \
+X( 21, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK_DEPRECATED, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires >= ????????, [0x%08.8x] is ???????? requires 0x%08.8x", 4) \
+X( 22, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_UPDATE_START, "UFO Updates for FWCtx 0x%08.8x @ %u", 2) \
+X( 23, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_UPDATE, "UFO Update: [0x%08.8x] = 0x%08.8x", 2) \
+X( 24, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ASSERT_FAILED, "ASSERT Failed: line %u of:", 1) \
+X( 25, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_LOCKUP_DEPRECATED, "HWR: Lockup detected on DM%u, FWCtx: 0x%08.8x", 2) \
+X( 26, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_RESET_FW_DEPRECATED, "HWR: Reset fw state for DM%u, FWCtx: 0x%08.8x, MemCtx: 0x%08.8x", 3) \
+X( 27, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_RESET_HW_DEPRECATED, "HWR: Reset HW", 0) \
+X( 28, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_TERMINATED_DEPRECATED, "HWR: Lockup recovered.", 0) \
+X( 29, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_FALSE_LOCKUP_DEPRECATED, "HWR: False lockup detected for DM%u", 1) \
+X( 30, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ALIGN_FAILED, "Alignment check %u failed: host = 0x%x, fw = 0x%x", 3) \
+X( 31, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GP_USC_TRIGGERED, "GP USC triggered", 0) \
+X( 32, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_OVERALLOC_REGS, "Overallocating %u temporary registers and %u shared registers for breakpoint handler", 2) \
+X( 33, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET_DEPRECATED, "Setting breakpoint: Addr 0x%08.8x", 1) \
+X( 34, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_STORE, "Store breakpoint state", 0) \
+X( 35, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_UNSET, "Unsetting BP Registers", 0) \
+X( 36, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NONZERO_RT, "Active RTs expected to be zero, actually %u", 1) \
+X( 37, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTC_PRESENT, "RTC present, %u active render targets", 1) \
+X( 38, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_EST_POWER_DEPRECATED, "Estimated Power 0x%x", 1) \
+X( 39, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTA_TARGET, "RTA render target %u", 1) \
+X( 40, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTA_KICK_RENDER, "Kick RTA render %u of %u", 2) \
+X( 41, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_SIZES_CHECK_DEPRECATED, "HWR sizes check %u failed: addresses = %u, sizes = %u", 3) \
+X( 42, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_ENABLE_DEPRECATED, "Pow: DUSTS_ENABLE = 0x%x", 1) \
+X( 43, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_HWREQ_DEPRECATED, "Pow: On(1)/Off(0): %u, Units: 0x%08.8x", 2) \
+X( 44, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_CHANGE_DEPRECATED, "Pow: Changing number of dusts from %u to %u", 2) \
+X( 45, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_SIDEKICK_IDLE_DEPRECATED, "Pow: Sidekick ready to be powered down", 0) \
+X( 46, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_CHANGE_REQ_DEPRECATED, "Pow: Request to change num of dusts to %u (bPowRascalDust=%u)", 2) \
+X( 47, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PARTIALRENDER_WITHOUT_ZSBUFFER_STORE, "No ZS Buffer used for partial render (store)", 0) \
+X( 48, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PARTIALRENDER_WITHOUT_ZSBUFFER_LOAD, "No Depth/Stencil Buffer used for partial render (load)", 0) \
+X( 49, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_SET_LOCKUP_DEPRECATED, "HWR: Lock-up DM%u FWCtx: 0x%08.8x", 2) \
+X( 50, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_REG_VALUE_DEPRECATED, "MLIST%u checker: CatBase TE=0x%08x (%u Pages), VCE=0x%08x (%u Pages), ALIST=0x%08x, IsTA=%u", 7) \
+X( 51, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_MLIST_VALUE, "MLIST%u checker: MList[%u] = 0x%08x", 3) \
+X( 52, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_OK, "MLIST%u OK", 1) \
+X( 53, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_EMPTY, "MLIST%u is empty", 1) \
+X( 54, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_REG_VALUE, "MLIST%u checker: CatBase TE=0x%08x%08x, VCE=0x%08x%08x, ALIST=0x%08x%08x, IsTA=%u", 8) \
+X( 55, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_40480KICK, "3D OQ flush kick", 0) \
+X( 56, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWP_UNSUPPORTED_BLOCK, "HWPerf block ID (0x%x) unsupported by device", 1) \
+X( 57, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET_DEPRECATED2, "Setting breakpoint: Addr 0x%08.8x DM%u", 2) \
+X( 58, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED, "Kick RTU: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X( 59, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTU_FINISHED_DEPRECATED, "RDM finished on context %u", 1) \
+X( 60, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_SHG_DEPRECATED, "Kick SHG: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X( 61, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SHG_FINISHED_DEPRECATED, "SHG finished", 0) \
+X( 62, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBA_FINISHED_DEPRECATED, "FBA finished on context %u", 1) \
+X( 63, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_FAILED, "UFO Checks failed", 0) \
+X( 64, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_START, "Kill DM%u start", 1) \
+X( 65, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_COMPLETE, "Kill DM%u complete", 1) \
+X( 66, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FC_CCB_UPDATE_DEPRECATED, "FC%u cCCB Woff update = %u", 2) \
+X( 67, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED2, "Kick RTU: FWCtx 0x%08.8x @ %u, prio: %d, Frame Context: %u", 4) \
+X( 68, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_INIT, "GPU init", 0) \
+X( 69, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNITS_INIT, "GPU Units init (# mask: 0x%x)", 1) \
+X( 70, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGTIMES, "Register access cycles: read: %u cycles, write: %u cycles, iterations: %u", 3) \
+X( 71, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGCONFIG_ADD, "Register configuration added. Address: 0x%x Value: 0x%x%x", 3) \
+X( 72, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGCONFIG_SET, "Register configuration applied to type %u. (0:pow on, 1:Rascal/dust init, 2-5: TA,3D,CDM,TLA, 6:All)", 1) \
+X( 73, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TPC_FLUSH, "Perform TPC flush.", 0) \
+X( 74, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP_DEPRECATED, "GPU has locked up (see HWR logs for more info)", 0) \
+X( 75, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_OUTOFTIME, "HWR has been triggered - GPU has overrun its deadline (see HWR logs)", 0) \
+X( 76, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_POLLFAILURE_DEPRECATED, "HWR has been triggered - GPU has failed a poll (see HWR logs)", 0) \
+X( 77, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DOPPLER_OOM_DEPRECATED, "Doppler out of memory event for FC %u", 1) \
+X( 78, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK1, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires >= 0x%08.8x", 3) \
+X( 79, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK2, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 80, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIMESTAMP, "TIMESTAMP -> [0x%08.8x]", 1) \
+X( 81, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_RMW_UPDATE_START, "UFO RMW Updates for FWCtx 0x%08.8x @ %u", 2) \
+X( 82, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_RMW_UPDATE, "UFO Update: [0x%08.8x] = 0x%08.8x", 2) \
+X( 83, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NULLCMD, "Kick Null cmd: FWCtx 0x%08.8x @ %u", 2) \
+X( 84, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RPM_OOM_DEPRECATED, "RPM Out of memory! Context 0x%08x, SH requestor %u", 2) \
+X( 85, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTU_ABORT_DISCARD_DEPRECATED, "Discard RTU due to RPM abort: FWCtx 0x%08.8x @ %u, prio: %d, Frame Context: %u", 4) \
+X( 86, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED, "Deferring DM%u from running context 0x%08x @ %u (deferred DMs = 0x%08x)", 4) \
+X( 87, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED_WAITING_TURN_DEPRECATED, "Deferring DM%u from running context 0x%08x @ %u to let other deferred DMs run (deferred DMs = 0x%08x)", 4) \
+X( 88, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED_NO_LONGER, "No longer deferring DM%u from running context = 0x%08x @ %u (deferred DMs = 0x%08x)", 4) \
+X( 89, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_FWCCB_DEPRECATED, "FWCCB for DM%u is full, we will have to wait for space! (Roff = %u, Woff = %u)", 3) \
+X( 90, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_FWCCB, "FWCCB for Driver ID %u is full, we will have to wait for space! (Roff = %u, Woff = %u)", 3) \
+X( 91, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SYNC_PART, "Host Sync Partition marker: %u", 1) \
+X( 92, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SYNC_PART_RPT, "Host Sync Partition repeat: %u", 1) \
+X( 93, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CLOCK_SPEED_CHANGE, "Core clock set to %u Hz", 1) \
+X( 94, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_OFFSETS, "Compute Queue: FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 7) \
+X( 95, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_DEPRECATED, "Signal check failed, Required Data: 0x%x, Address: 0x%08x%08x", 3) \
+X( 96, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE_DEPRECATED, "Signal update, Snoop Filter: %u, MMU Ctx: %u, Signal Id: %u, Signals Base: 0x%08x%08x", 5) \
+X( 97, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWCONTEXT_SIGNALED, "Signalled the previously waiting FWCtx: 0x%08.8x, OSId: %u, Signal Address: 0x%08x%08x", 4) \
+X( 98, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_STALLED_DEPRECATED, "Compute stalled", 0) \
+X( 99, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_STALLED, "Compute stalled (Roff = %u, Woff = %u, Size = %u)", 3) \
+X(100, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_RESUMED_FROM_STALL, "Compute resumed (Roff = %u, Woff = %u, Size = %u)", 3) \
+X(101, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOTIFY_SIGNAL_UPDATE, "Signal update notification from the host, PC Physical Address: 0x%08x%08x, Signal Virtual Address: 0x%08x%08x", 4) \
+X(102, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE_OSID_DM_DEPRECATED, "Signal update from DM: %u, OSId: %u, PC Physical Address: 0x%08x%08x", 4) \
+X(103, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_DM_DEPRECATED, "DM: %u signal check failed", 1) \
+X(104, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED, "Kick TDM: FWCtx 0x%08.8x @ %u, prio:%d", 3) \
+X(105, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_FINISHED, "TDM finished", 0) \
+X(106, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TE_PIPE_STATUS_DEPRECATED, "MMU_PM_CAT_BASE_TE[%u]_PIPE[%u]:  0x%08x 0x%08x)", 4) \
+X(107, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_HIT_DEPRECATED, "BRN 54141 HIT", 0) \
+X(108, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_APPLYING_DUMMY_TA_DEPRECATED, "BRN 54141 Dummy TA kicked", 0) \
+X(109, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_RESUME_TA_DEPRECATED, "BRN 54141 resume TA", 0) \
+X(110, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_DOUBLE_HIT_DEPRECATED, "BRN 54141 double hit after applying WA", 0) \
+X(111, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_DUMMY_TA_VDM_BASE_DEPRECATED, "BRN 54141 Dummy TA VDM base address: 0x%08x%08x", 2) \
+X(112, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_WITH_CURRENT, "Signal check failed, Required Data: 0x%x, Current Data: 0x%x, Address: 0x%08x%08x", 4) \
+X(113, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BUFFER_STALL_DEPRECATED, "TDM stalled (Roff = %u, Woff = %u)", 2) \
+X(114, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOTIFY_WRITE_OFFSET_UPDATE, "Write Offset update notification for stalled FWCtx 0x%08.8x", 1) \
+X(115, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED, "Changing OSid %u's priority from %u to %u", 3) \
+X(116, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_RESUMED, "Compute resumed", 0) \
+X(117, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TLA, "Kick TLA: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(118, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED3, "Kick TDM: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(119, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA, "Kick TA: FWCtx 0x%08.8x @ %u, RTD 0x%08x, First kick:%u, Last kick:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(120, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D, "Kick 3D: FWCtx 0x%08.8x @ %u, RTD 0x%08x, Partial render:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 10) \
+X(121, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3DTQ, "Kick 3D TQ: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(122, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_DEPRECATED2, "Kick Compute: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 6) \
+X(123, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED3, "Kick RTU: FWCtx 0x%08.8x @ %u, Frame Context:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(124, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_SHG_DEPRECATED2, "Kick SHG: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(125, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CSRM_RECONFIG, "Reconfigure CSRM: special coeff support enable %u.", 1) \
+X(127, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_REQ_MAX_COEFFS, "TA requires max coeff mode, deferring: %u.", 1) \
+X(128, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_REQ_MAX_COEFFS, "3D requires max coeff mode, deferring: %u.", 1) \
+X(129, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_FAILED, "Kill DM%u failed", 1) \
+X(130, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_QUEUE, "Thread Queue is full, we will have to wait for space! (Roff = %u, Woff = %u)", 2) \
+X(131, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_QUEUE_FENCE, "Thread Queue is fencing, we are waiting for Roff = %u (Roff = %u, Woff = %u)", 3) \
+X(132, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_HCS_TRIGGERED_DEPRECATED, "DM %u failed to Context Switch on time. Triggered HCS (see HWR logs).", 1) \
+X(133, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HCS_SET_DEPRECATED, "HCS changed to %u ms", 1) \
+X(134, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UPDATE_TILES_IN_FLIGHT_DEPRECATED, "Updating Tiles In Flight (Dusts=%u, PartitionMask=0x%08x, ISPCtl=0x%08x%08x)", 4) \
+X(135, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_TILES_IN_FLIGHT, "  Phantom %u: USCTiles=%u", 2) \
+X(136, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ISOLATION_CONF_OFF_DEPRECATED, "Isolation grouping is disabled", 0) \
+X(137, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ISOLATION_CONF_DEPRECATED, "Isolation group configured with a priority threshold of %u", 1) \
+X(138, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_ONLINE_DEPRECATED, "OS %u has come online", 1) \
+X(139, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_OFFLINE_DEPRECATED, "OS %u has gone offline", 1) \
+X(140, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWCONTEXT_SIGNAL_REKICK, "Signalled the previously stalled FWCtx: 0x%08.8x, OSId: %u, Signal Address: 0x%08x%08x", 4) \
+X(141, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSETS_DEPRECATED, "TDM Queue: FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 7) \
+X(142, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSET_READ_RESET, "Reset TDM Queue Read Offset: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u becomes 0, Woff = %u, Size = %u)", 6) \
+X(143, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UMQ_MISMATCHED_READ_OFFSET, "User Mode Queue mismatched stream start: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u, StreamStartOffset = %u)", 5) \
+X(144, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_DEINIT, "GPU deinit", 0) \
+X(145, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNITS_DEINIT, "GPU units deinit", 0) \
+X(146, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CONFIG_DEPRECATED, "Initialised OS %u with config flags 0x%08x", 2) \
+X(147, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_LIMIT, "UFO limit exceeded %u/%u", 2) \
+X(148, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_62850KICK, "3D Dummy stencil store", 0) \
+X(149, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CONFIG_DEPRECATED2, "Initialised OS %u with config flags 0x%08x and extended config flags 0x%08x", 3) \
+X(150, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_COMMAND_DEPRECATED, "Unknown Command (eCmdType=0x%08x)", 1) \
+X(151, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE, "UFO forced update: FWCtx 0x%08.8x @ %u [0x%08.8x] = 0x%08.8x", 4) \
+X(152, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE_NOP_DEPRECATED, "UFO forced update NOP: FWCtx 0x%08.8x @ %u [0x%08.8x] = 0x%08.8x, reason %u", 5) \
+X(153, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN66075_CHECK, "TDM context switch check: Roff %u points to 0x%08x, Match=%u", 3) \
+X(154, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CCBS_DEPRECATED, "Driver ID %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x", 6) \
+X(155, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWIRQ, "FW IRQ # %u @ %u", 2) \
+X(156, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET, "Setting breakpoint: Addr 0x%08.8x DM%u usc_breakpoint_ctrl_dm = %u", 3) \
+X(157, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KERNEL_CCB_DEPRECATED, "Invalid KCCB setup for OSid %u: KCCB 0x%08x, KCCB Ctrl 0x%08x", 3) \
+X(158, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KERNEL_CCB_CMD, "Invalid KCCB cmd (%u) for Driver ID %u @ KCCB 0x%08x", 3) \
+X(159, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_FAULT_DEPRECATED, "FW FAULT: At line %u in file 0x%08x%08x, additional data=0x%08x", 4) \
+X(160, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_INVALID, "Invalid breakpoint: MemCtx 0x%08x Addr 0x%08.8x DM%u usc_breakpoint_ctrl_dm = %u", 4) \
+X(161, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FLUSHINVAL_CMD_INVALID_DEPRECATED, "Discarding invalid SLC flushinval command for OSid %u: DM %u, FWCtx 0x%08x", 3) \
+X(162, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NOTIFY_WRITE_OFFSET_UPDATE_DEPRECATED, "Invalid Write Offset update notification from OSid %u to DM %u: FWCtx 0x%08x, MemCtx 0x%08x", 4) \
+X(163, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KCCB_KICK_CMD_DEPRECATED, "Null FWCtx in KCCB kick cmd for OSid %u: KCCB 0x%08x, ROff %u, WOff %u", 4) \
+X(164, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FULL_CHPTCCB, "Checkpoint CCB for Driver ID %u is full, signalling host for full check state (Roff = %u, Woff = %u)", 3) \
+X(165, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CCBS_DEPRECATED2, "OSid %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x chptCCBCtl@0x%x chptCCB@0x%x", 8) \
+X(166, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_STATE_CHANGE_DEPRECATED, "Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-offloading 4-cooldown). Status %u (1-ok 0-fail)", 4) \
+X(167, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_STALE_KCCB_CMDS, "Driver ID %u has %u stale commands in its KCCB", 2) \
+X(168, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_VCE_PAUSE, "Applying VCE pause", 0) \
+X(169, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KCCB_UPDATE_RTN_SLOT_DEPRECATED, "OSid %u KCCB slot %u value updated to %u", 3) \
+X(170, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_KCCB_COMMAND, "Unknown KCCB Command: KCCBCtl=0x%08x, KCCB=0x%08x, Roff=%u, Woff=%u, Wrap=%u, Cmd=0x%08x, CmdType=0x%08x", 7) \
+X(171, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_CCB_COMMAND1, "Unknown Client CCB Command processing fences: FWCtx=0x%08x, CCBCtl=0x%08x, CCB=0x%08x, Roff=%u, Doff=%u, Woff=%u, Wrap=%u, CmdHdr=0x%08x, CmdType=0x%08x, CmdSize=%u", 10) \
+X(172, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_CCB_COMMAND2, "Unknown Client CCB Command executing kick: FWCtx=0x%08x, CCBCtl=0x%08x, CCB=0x%08x, Roff=%u, Doff=%u, Woff=%u, Wrap=%u, CmdHdr=0x%08x, CmdType=0x%08x, CmdSize=%u", 10) \
+X(173, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KCCB_KICK_CMD, "Null FWCtx in KCCB kick cmd for Driver ID %u with WOff %u", 2) \
+X(174, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FLUSHINVAL_CMD_INVALID, "Discarding invalid SLC flushinval command for Driver ID %u, FWCtx 0x%08x", 2) \
+X(175, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NOTIFY_WRITE_OFFSET_UPDATE, "Invalid Write Offset update notification from Driver ID %u: FWCtx 0x%08x, MemCtx 0x%08x", 3) \
+X(176, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_INIT_CONFIG, "Initialised Firmware with config flags 0x%08x and extended config flags 0x%08x", 2) \
+X(177, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_CONFIG, "Set Periodic Hardware Reset Mode: %u", 1) \
+X(179, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_TRIG, "PHR mode %u, FW state: 0x%08x, HWR flags: 0x%08x", 3) \
+X(180, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_RESET_DEPRECATED, "PHR mode %u triggered a reset", 1) \
+X(181, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE, "Signal update, Snoop Filter: %u, Signal Id: %u", 2) \
+X(182, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEV_SERIES8_DEPRECATED, "WARNING: Skipping FW KCCB Cmd type %u which is not yet supported on Series8.", 1) \
+X(183, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INCONSISTENT_MMU_FLAGS, "MMU context cache data NULL, but cache flags=0x%x (sync counter=%u, update value=%u) OSId=%u", 4) \
+X(184, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SLC_FLUSH, "SLC range based flush: Context=%u VAddr=0x%02x%08x, Size=0x%08x, Invalidate=%u", 5) \
+X(185, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBSC_INVAL_DEPRECATED, "FBSC invalidate for Context Set [0x%08x]: Entry mask 0x%08x%08x.", 3) \
+X(186, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN66284_UPDATE, "TDM context switch check: Roff %u was not valid for kick starting at %u, moving back to %u", 3) \
+X(187, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SPFILTER_UPDATES, "Signal updates: FIFO: %u, Signals: 0x%08x", 2) \
+X(188, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_FBSC_CMD, "Invalid FBSC cmd: FWCtx 0x%08x, MemCtx 0x%08x", 2) \
+X(189, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN68497_BLIT, "Insert BRN68497 WA blit after TDM Context store.", 0) \
+X(190, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PENDING_UFO_UPDATE_START, "UFO Updates for previously finished FWCtx 0x%08.8x", 1) \
+X(191, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTC_RTA_PRESENT, "RTC with RTA present, %u active render targets", 1) \
+X(192, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NULL_RTAS, "Invalid RTA Set-up. The ValidRenderTargets array in RTACtl is Null!", 0) \
+X(193, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_COUNTER, "Block 0x%x / Counter 0x%x INVALID and ignored", 2) \
+X(194, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ECC_FAULT_DEPRECATED, "ECC fault GPU=0x%08x FW=0x%08x", 2) \
+X(195, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PROCESS_XPU_EVENT, "Processing XPU event on DM = %u", 1) \
+X(196, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_VZ_WDG_TRIGGER_DEPRECATED, "Driver ID %u failed to respond to the virtualisation watchdog in time. Timestamp of its last input = %u", 2) \
+X(197, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP, "GPU-%u has locked up (see HWR logs for more info)", 1) \
+X(198, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UPDATE_TILES_IN_FLIGHT, "Updating Tiles In Flight (Dusts=%u, PartitionMask=0x%08x, ISPCtl=0x%08x)", 3) \
+X(199, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP_DM, "GPU has locked up (see HWR logs for more info)", 0) \
+X(200, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REPROCESS_XPU_EVENTS, "Reprocessing outstanding XPU events from cores 0x%02x", 1) \
+X(201, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SECONDARY_XPU_EVENT, "Secondary XPU event on DM=%u, CoreMask=0x%02x, Raised=0x%02x", 3) \
+X(202, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSETS, "TDM Queue: Core %u, FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 8) \
+X(203, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BUFFER_STALL, "TDM stalled Core %u (Roff = %u, Woff = %u)", 3) \
+X(204, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_OFFSETS, "Compute Queue: Core %u, FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 8) \
+X(205, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_STALLED, "Compute stalled core %u (Roff = %u, Woff = %u, Size = %u)", 4) \
+X(206, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UMQ_MISMATCHED_CORE_READ_OFFSET, "User Mode Queue mismatched stream start: Core %u, FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u, StreamStartOffset = %u)", 6) \
+X(207, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_RESUMED_FROM_STALL, "TDM resumed core %u (Roff = %u, Woff = %u)", 3) \
+X(208, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_RESUMED_FROM_STALL, "Compute resumed core %u (Roff = %u, Woff = %u, Size = %u)", 4) \
+X(209, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_MTS_PERMISSION_CHANGED_DEPRECATED, " Updated permission for Driver ID %u to perform MTS kicks: %u (1 = allowed, 0 = not allowed)", 2) \
+X(210, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TEST1, "Mask = 0x%X, mask2 = 0x%X", 2) \
+X(211, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TEST2, "  core %u, reg = %u, mask = 0x%X)", 3) \
+X(212, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ECC_FAULT_SAFETY_BUS, "ECC fault received from safety bus: 0x%08x", 1) \
+X(213, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SAFETY_WDG_CONFIG, "Safety Watchdog threshold period set to 0x%x clock cycles", 1) \
+X(214, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SAFETY_WDG_TRIGGER, "MTS Safety Event triggered by the safety watchdog.", 0) \
+X(215, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_USC_TASKS_RANGE, "DM%u USC tasks range limit 0 - %u, stride %u", 3) \
+X(216, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_ECC_FAULT, "ECC fault GPU=0x%08x", 1) \
+X(217, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_SAFETY_RESET, "GPU Hardware units reset to prevent transient faults.", 0) \
+X(218, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ABORTCMD, "Kick Abort cmd: FWCtx 0x%08.8x @ %u", 2) \
+X(219, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_DEPRECATED, "Kick Ray: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7)\
+X(220, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_FINISHED_DEPRECATED, "Ray finished", 0) \
+X(221, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWDATA_INIT_STATUS, "State of firmware's private data at boot time: %u (0 = uninitialised, 1 = initialised); Fw State Flags = 0x%08X", 2) \
+X(222, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CFI_TIMEOUT, "CFI Timeout detected (%u increasing to %u)", 2) \
+X(223, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CFI_TIMEOUT_FBM, "CFI Timeout detected for FBM (%u increasing to %u)", 2) \
+X(224, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GEOM_OOM_DISALLOWED, "Geom OOM event not allowed", 0) \
+X(225, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED2, "Changing Driver ID %u's priority from %u to %u; Isolation = %u (0 = off; 1 = on)", 4) \
+X(226, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SKIP_ALREADY_RUN_GEOM, "Skipping already executed TA FWCtx 0x%08.8x @ %u", 2) \
+X(227, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ATTEMPT_TO_RUN_AHEAD_GEOM, "Attempt to execute TA FWCtx 0x%08.8x @ %u ahead of time on other GEOM", 2) \
+X(228, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED2, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(229, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA_PIPELINE, "Kick TA: Kick ID %u FWCtx 0x%08.8x @ %u, RTD 0x%08x, First kick:%u, Last kick:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 12) \
+X(230, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D_PIPELINE, "Kick 3D: Kick ID %u FWCtx 0x%08.8x @ %u, RTD 0x%08x, Partial render:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(231, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_PIPELINE_DEPRECATED, "Kick Compute: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 7) \
+X(232, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_FINISHED_PIPELINE, "TDM finished: Kick ID %u", 1) \
+X(233, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_FINISHED_PIPELINE, "TA finished: Kick ID %u", 1) \
+X(234, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_FINISHED_PIPELINE, "3D finished: Kick ID %u, HWRTData0State=%x, HWRTData1State=%x", 3) \
+X(235, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_FINISHED_PIPELINE, "Compute finished: Kick ID %u", 1) \
+X(236, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_PIPELINE_DEPRECATED, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u, Base 0x%08x%08x. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 10) \
+X(237, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_PIPELINE_DEPRECATED, "Kick Ray: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8)\
+X(238, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_FINISHED_PIPELINE, "Ray finished: Kick ID %u", 1) \
+X(239, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RT_UNITS_INIT, "GPU RT Units init (# mask: 0x%08x%08x)", 2) \
+X(240, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_PENDING_PASS, "UFO Check: [0x%08.8x] is pending update to 0x%08.8x and therefore passes", 2) \
+X(241, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_PR_CHECK_PENDING_PASS, "UFO PR-Check: [0x%08.8x] is pending update to 0x%08.8x and therefore passes", 2) \
+X(242, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DELAY_DM_TO_OVERLAP_PIPES, "Holding kick of DM %u pipe %u to encourage pipeline overlap", 2) \
+X(243, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RELEASE_DM_PIPE, "Releasing kick for DM %u pipe %u", 2) \
+X(244, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED3, "Changing Driver ID %u's priority from %u to %u", 3) \
+X(245, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_ISOLATION_GROUP_CHANGE_DEPRECATED, "Changing Driver ID %u's isolation group from %u to %u", 3) \
+X(246, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_VK_TIMESTAMP, "VK Timestamp: addr=0x%08x%08x, avail=0x%08x%08x stamp=0x%08x%08x", 6) \
+X(247, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_HCS_TRIGGERED, "DM %u failed to Context Switch on time (Current time: 0x%08x%08x, deadline: 0x%08x%08x). Triggered HCS (see HWR logs).", 5) \
+X(248, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_POLLFAILURE, "HWR has been triggered - GPU has failed a poll, RGX_CR_EVENT_STATUS=0x%08x (see HWR logs)", 1) \
+X(249, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBCDC_FAILURE_DETECTED_DEPRECATED, "FBCDC signature failure detected so block scheduling more work", 0) \
+X(250, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBCDC_FAILURE_CLEARED_DEPRECATED, "FBCDC signature cleared which unlocks scheduling more work", 0) \
+X(251, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_FAULT, "FW FAULT: At line %u in file 0x%08x%08x, additional data=0x%08x%08x", 5) \
+X(252, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE_NOP, "UFO forced update NOP: FWCtx 0x%08.8x @ %u, reason %u", 3) \
+X(253, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_CONTEXT_STORED, "TDM FWCtx:0x%08.8x stored", 1) \
+X(254, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CDM_CONTEXT_STORED, "CDM FWCtx:0x%08.8x stored, resumeMask:0x%08x", 2) \
+X(255, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GEOM_CONTEXT_STORED, "GEOM FWCtx:0x%08.8x stored", 1) \
+X(256, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_CONTEXT_STORED, "3D FWCtx:0x%08.8x stored, resumeMask:0x%08x", 2) \
+X(257, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RDM_CONTEXT_STORED, "RAY FWCtx:0x%08.8x stored", 1) \
+X(258, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM, "Kick TDM: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(259, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE, "Kick Compute: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 7) \
+X(260, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_PIPELINE, "Kick Compute: Kick ID %u FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 8) \
+X(261, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_PIPELINE, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u, Base 0x%08x%08x, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(262, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_PIPELINE, "Kick Ray: Kick ID %u FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 9)\
+X(263, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DEPTH, "3D Disable Depth. ExtJobRef = 0x%08x", 1) \
+X(264, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_STENCIL, "3D Disable Stencil. ExtJobRef = 0x%08x", 1) \
+X(265, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DS_IN_3D_RUNNING, "3D Disable DS in 3D running. RenderContext 0x%08.8x ExtJobRef 0x%08x", 2) \
+X(266, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DS_IN_KICK_3D, "3D Disable DS in kick 3D. RenderContext 0x%08.8x ExtJobRef 0x%08x", 2) \
+X(267, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ADD_DISABLE_DS_QUEUE, "Add disable DS in queue. RenderContext 0x%08.8x DisableJobRef 0x%08x Uncheck %u", 3) \
+X(268, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOT_FIND_USABLE_DS_IN_QUEUE, "Not find usable DS in queue. RenderContext 0x%08.8x ExtJobRef 0x%08x DisableJobRef 0x%08x Uncheck %u index %u", 5) \
+X(269, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_LIST_FULL, "Unable to set breakpoint for MemCtx 0x%08x as the breakpoint list is full.", 1) \
+X(270, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NUM_LOG_PARAMS, "Invalid number of log parameters passed! (Group:%u ID:%u Params:%u Passed:%u)", 4) \
+X(271, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_CANCEL_PIPELINE, "TDM cancelled: Kick ID %u", 1) \
+X(272, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_CANCEL_PIPELINE, "TA cancelled: Kick ID %u", 1) \
+X(273, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_CANCEL_PIPELINE, "3D cancelled: Kick ID %u", 1) \
+X(274, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CDM_CANCEL_PIPELINE, "Compute cancelled: Kick ID %u", 1) \
+X(275, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_CANCEL_PIPELINE, "Ray cancelled: Kick ID %u", 1) \
+X(276, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_MAX_DEPRECATED, "Time Slice: Update Failed, Max total allocations exceeded 100pc", 0)\
+X(277, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_DYNAMIC_LOW_DEPRECATED, "Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 0)\
+X(278, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_UPDATE_SUCCESS_DEPRECATED, "Time Slice: Updated Successfully", 0)\
+X(279, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISCARD_FWCCB, "FWCCB for Driver ID %u is full, discarding command! (Roff = %u, Woff = %u)", 3) \
+X(280, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_RECEIVED_DEPRECATED, "Cancelling jobs with intjobref<0x%08x, DM = %u, FWCtx = 0x%08.8x", 3) \
+X(281, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_RECEIVED, "Cancelling jobs with intjobref from 0x%08x to 0x%08x, DM = %u, FWCtx = 0x%08.8x", 4) \
+X(282, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_MAX_DEPRECATED2, "Time Slice: Update Failed, Max total allocations exceeded 100pc", 0)\
+X(283, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_DYNAMIC_LOW_DEPRECATED2, "Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 0)\
+X(284, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_UPDATE_SUCCESS_DEPRECATED2, "Time Slice: Updated Successfully", 0)\
+X(285, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_STATE_CHANGE_DEPRECATED2, "Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-graceful_offloading 4-forced_offloading 5-cooldown). Status %u (1-ok 0-fail)", 4) \
+X(286, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBSC_INVAL, "FBSC invalidate for Context Set [0x%08x]", 1) \
+X(287, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_DROPPED, "Dropped cancel request with intjobref from 0x%08x to 0x%08x, DM = %u, FWCtx = 0x%08.8x", 4) \
+X(288, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ZERO_LM, "Zeroing local memory after context storing FWCtx = 0x%08.8x", 1) \
+X(289, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FAILED_IRQ_CLEAR, "Host failed to clear IRQ DriverID: %u Reg: 0x%08x",2) \
+X(290, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_ABORTED, "Kill DM%u aborted", 1) \
+X(291, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SOC_CLOCK_SPEED_CHANGE, "SOC clock set to %u Hz", 1) \
+\
+X(  1, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK_DEPRECATED, "Bg Task DM = %u, counted = %u", 2) \
+X(  2, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_COMPLETE_DEPRECATED, "Bg Task complete DM = %u", 1) \
+X(  3, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_KICK_DEPRECATED, "Irq Task DM = %u, Breq = %u, SBIrq = 0x%x", 3) \
+X(  4, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_COMPLETE_DEPRECATED, "Irq Task complete DM = %u", 1) \
+X(  5, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KICK_MTS_BG_ALL_DEPRECATED, "Kick MTS Bg task DM=All", 0) \
+X(  6, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KICK_MTS_IRQ, "Kick MTS Irq task DM=%u", 1) \
+X(  7, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYCELLTYPE_DEPRECATED, "Ready queue debug DM = %u, celltype = %u", 2) \
+X(  8, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYTORUN_DEPRECATED, "Ready-to-run debug DM = %u, item = 0x%x", 2) \
+X(  9, RGXFW_GROUP_MTS, RGXFW_SF_MTS_CMDHEADER, "Client command header DM = %u, client CCB = 0x%x, cmd = 0x%x", 3) \
+X( 10, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYTORUN, "Ready-to-run debug Driver ID = %u, DM = %u, item = 0x%x", 3) \
+X( 11, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYCELLTYPE_DEPRECATED2, "Ready queue debug DM = %u, celltype = %u, OSid = %u", 3) \
+X( 12, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK_DEPRECATED2, "Bg Task DM = %u, counted = %u, OSid = %u", 3) \
+X( 13, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_COMPLETE, "Bg Task complete DM Bitfield: %u", 1) \
+X( 14, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_COMPLETE, "Irq Task complete.", 0) \
+X( 15, RGXFW_GROUP_MTS, RGXFW_SF_MTS_CMD_DISCARD, "Discarded Command Type: %u OS ID = %u PID = %u context = 0x%08x cccb ROff = 0x%x, due to USC breakpoint hit by OS ID = %u PID = %u.", 7) \
+X( 16, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_EXEC_DEPRECATED, "KCCB Slot %u: DM=%u, Cmd=0x%08x, OSid=%u", 4) \
+X( 17, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_RTN_VALUE, "KCCB Slot %u: Return value %u", 2) \
+X( 18, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK, "Bg Task Driver ID = %u", 1) \
+X( 19, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_EXEC, "KCCB Slot %u: Cmd=0x%08x, Driver ID=%u", 3) \
+X( 20, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_KICK, "Irq Task (EVENT_STATUS=0x%08x)", 1) \
+X( 21, RGXFW_GROUP_MTS, RGXFW_SF_MTS_VZ_SIDEBAND, "VZ sideband test, kicked with DriverID=%u from MTS, OSid for test=%u", 2) \
+\
+X(  1, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FWCTX_CLEANUP, "FwCommonContext [0x%08x] cleaned", 1) \
+X(  2, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FWCTX_BUSY, "FwCommonContext [0x%08x] is busy: ReadOffset = %u, WriteOffset = %u", 3) \
+X(  3, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANUP_DEPRECATED, "HWRTData [0x%08x] for DM=%u, received cleanup request", 2) \
+X(  4, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED_FOR_DM_DEPRECATED, "HWRTData [0x%08x] HW Context cleaned for DM%u, executed commands = %u", 3) \
+X(  5, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY_DEPRECATED, "HWRTData [0x%08x] HW Context for DM%u is busy", 2) \
+X(  6, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED_DEPRECATED, "HWRTData [0x%08x] HW Context %u cleaned", 2) \
+X(  7, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FL_CLEANED, "Freelist [0x%08x] cleaned", 1) \
+X(  8, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_ZSBUFFER_CLEANED, "ZSBuffer [0x%08x] cleaned", 1) \
+X(  9, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_ZSBUFFER_BUSY, "ZSBuffer [0x%08x] is busy: submitted = %u, executed = %u", 3) \
+X( 10, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY_DEPRECATED2, "HWRTData [0x%08x] HW Context for DM%u is busy: submitted = %u, executed = %u", 4) \
+X( 11, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANUP_DEPRECATED, "HW Ray Frame data [0x%08x] for DM=%u, received cleanup request", 2) \
+X( 12, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANED_FOR_DM_DEPRECATED, "HW Ray Frame Data [0x%08x] cleaned for DM%u, executed commands = %u", 3) \
+X( 13, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_BUSY_DEPRECATED, "HW Ray Frame Data [0x%08x] for DM%u is busy: submitted = %u, executed = %u", 4) \
+X( 14, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANED_DEPRECATED, "HW Ray Frame Data [0x%08x] HW Context %u cleaned", 2) \
+X( 15, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_INVALID_REQUEST, "Discarding invalid cleanup request of type 0x%x", 1) \
+X( 16, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANUP, "Received cleanup request for HWRTData [0x%08x]", 1) \
+X( 17, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY, "HWRTData [0x%08x] HW Context is busy: submitted = %u, executed = %u", 3) \
+X( 18, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED, "HWRTData [0x%08x] HW Context %u cleaned, executed commands = %u", 3) \
+\
+X(  1, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_NEEDS_RESUME, "CDM FWCtx 0x%08.8x needs resume", 1) \
+X(  2, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_RESUME_DEPRECATED, "*** CDM FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X(  3, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_SHARED, "CDM FWCtx shared alloc size load 0x%x", 1) \
+X(  4, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_COMPLETE, "*** CDM FWCtx store complete", 0) \
+X(  5, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_START, "*** CDM FWCtx store start", 0) \
+X(  6, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_SOFT_RESET, "CDM Soft Reset", 0) \
+X(  7, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_NEEDS_RESUME, "3D FWCtx 0x%08.8x needs resume", 1) \
+X(  8, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME, "*** 3D FWCtx 0x%08.8x resume", 1) \
+X(  9, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_COMPLETE, "*** 3D context store complete", 0) \
+X( 10, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_DEPRECATED, "3D context store pipe state: 0x%08.8x 0x%08.8x 0x%08.8x", 3) \
+X( 11, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_START, "*** 3D context store start", 0) \
+X( 12, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_TQ_RESUME, "*** 3D TQ FWCtx 0x%08.8x resume", 1) \
+X( 13, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_NEEDS_RESUME, "TA FWCtx 0x%08.8x needs resume", 1) \
+X( 14, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_RESUME, "*** TA FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X( 15, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_SHARED, "TA context shared alloc size store 0x%x, load 0x%x", 2) \
+X( 16, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_COMPLETE, "*** TA context store complete", 0) \
+X( 17, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_START, "*** TA context store start", 0) \
+X( 18, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM_DEPRECATED, "Higher priority context requests DM %u, old prio:%d, new prio:%d", 3) \
+X( 19, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SET_CONTEXT_PRIORITY, "Set FWCtx 0x%x priority to %u", 2) \
+X( 20, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_DEPRECATED2, "3D context store pipe%u state: 0x%08.8x", 2) \
+X( 21, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE_DEPRECATED, "3D context resume pipe%u state: 0x%08.8x", 2) \
+X( 22, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_NEEDS_RESUME_DEPRECATED, "SHG FWCtx 0x%08.8x needs resume", 1) \
+X( 23, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_RESUME_DEPRECATED, "*** SHG FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X( 24, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_SHARED_DEPRECATED, "SHG context shared alloc size store 0x%x, load 0x%x", 2) \
+X( 25, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_STORE_COMPLETE_DEPRECATED, "*** SHG context store complete", 0) \
+X( 26, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_STORE_START_DEPRECATED, "*** SHG context store start", 0) \
+X( 27, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_PIPE_INDIRECT, "Performing TA indirection, last used pipe %u", 1) \
+X( 28, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_CTRL_STREAM_TERMINATE, "CDM context store hit ctrl stream terminate. Skip resume.", 0) \
+X( 29, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_RESUME_AB_BUFFER, "*** CDM FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x, shader state %u", 4) \
+X( 30, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STATE_BUFFER_FLIP, "TA PDS/USC state buffer flip (%u->%u)", 2) \
+X( 31, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_52563_HIT_DEPRECATED, "TA context store hit BRN 52563: vertex store tasks outstanding", 0) \
+X( 32, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_USC_POLL_FAILED, "TA USC poll failed (USC vertex task count: %u)", 1) \
+X( 33, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_DEFERRED_DEPRECATED, "TA context store deferred due to BRN 54141.", 0) \
+X( 34, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM_DEPRECATED2, "Higher priority context requests DM %u. Prios (OSid, OSid Prio, Context Prio): Current: %u, %u, %u New: %u, %u, %u", 7) \
+X( 35, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_START, "*** TDM context store start", 0) \
+X( 36, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_COMPLETE, "*** TDM context store complete", 0) \
+X( 37, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_NEEDS_RESUME_DEPRECATED, "TDM context needs resume, header [0x%08.8x, 0x%08.8x]", 2) \
+X( 38, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM, "Higher priority context requests DM %u. Prios (Driver ID, Driver ID Prio, Context Prio): Current: %u, %u, %u New: %u, %u, %u. Hard Context Switching: %u", 8) \
+X( 39, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE, "3D context store pipe %2d (%2d) state: 0x%08.8x", 3) \
+X( 40, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE, "3D context resume pipe %2d (%2d) state: 0x%08.8x", 3) \
+X( 41, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_START_VOLCANIC, "*** 3D context store start version %u (1=IPP_TILE, 2=ISP_TILE)", 1) \
+X( 42, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_VOLCANIC, "3D context store pipe%u state: 0x%08.8x%08x", 3) \
+X( 43, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE_VOLCANIC, "3D context resume pipe%u state: 0x%08.8x%08x", 3) \
+X( 44, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_IPP_STATE,  "3D context resume IPP state: 0x%08.8x%08x", 2) \
+X( 45, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_PIPES_EMPTY, "All 3D pipes empty after ISP tile mode store! IPP_status: 0x%08x", 1) \
+X( 46, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_RESUME_PIPE_STATE_DEPRECATED, "TDM context resume pipe%u state: 0x%08.8x%08x", 3) \
+X( 47, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_LEVEL4_STORE_START, "*** 3D context store start version 4", 0) \
+X( 48, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RESUME_MULTICORE, "Multicore context resume on DM%u active core mask 0x%04.4x", 2) \
+X( 49, RGXFW_GROUP_CSW, RGXFW_SF_CSW_STORE_MULTICORE, "Multicore context store on DM%u active core mask 0x%04.4x", 2) \
+X( 50, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_RESUME_PIPE_STATE, "TDM context resume Core %u, pipe%u state: 0x%08.8x%08x%08x", 5) \
+X( 51, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_STORE_COMPLETE, "*** RDM FWCtx store complete", 0) \
+X( 52, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_STORE_START, "*** RDM FWCtx store start", 0) \
+X( 53, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_NEEDS_RESUME, "RDM FWCtx 0x%08.8x needs resume", 1) \
+X( 54, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_RESUME, "RDM FWCtx 0x%08.8x resume", 1) \
+\
+X(  1, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_BIFREQ_DEPRECATED, "Activate MemCtx=0x%08x BIFreq=%u secure=%u", 3) \
+X(  2, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE, "Deactivate MemCtx=0x%08x", 1) \
+X(  3, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCREG_ALLOC_DEPRECATED, "Alloc PC reg %u", 1) \
+X(  4, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_GRAB, "Grab reg set %u refcount now %u", 2) \
+X(  5, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_UNGRAB_DEPRECATED, "Ungrab reg set %u refcount now %u", 2) \
+X(  6, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_BIFREQ_DEPRECATED, "Setup reg=%u BIFreq=%u, expect=0x%08x%08x, actual=0x%08x%08x", 6) \
+X(  7, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TRUST_DEPRECATED, "Trust enabled:%u, for BIFreq=%u", 2) \
+X(  8, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TILECFG_DEPRECATED, "BIF Tiling Cfg %u base 0x%08x%08x len 0x%08x%08x enable %u stride %u --> 0x%08x%08x", 9) \
+X(  9, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSID0, "Wrote the Value %u to OSID0, Cat Base %u, Register's contents are now 0x%08x 0x%08x", 4) \
+X( 10, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSID1, "Wrote the Value %u to OSID1, Context  %u, Register's contents are now 0x%04x", 3) \
+X( 11, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSIDx_DEPRECATED, "ui32OSid = %u, Catbase = %u, Reg Address = 0x%x, Reg index = %u, Bitshift index = %u, Val = 0x%08x%08x", 7) \
+X( 12, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY_BIFREQ_DEPRECATED, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u, BIFREQ %u", 5) \
+X( 13, RGXFW_GROUP_BIF, RGXFW_SF_BIF_UNMAP_GPU_MEMORY, "Unmap GPU memory (event status 0x%x)", 1) \
+X( 14, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_DM, "Activate MemCtx=0x%08x DM=%u secure=%u", 3) \
+X( 15, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM_DEPRECATED, "Setup reg=%u DM=%u, expect=0x%08x%08x, actual=0x%08x%08x", 6) \
+X( 16, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u", 4) \
+X( 17, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TRUST_DM, "Trust enabled:%u, for DM=%u", 2) \
+X( 18, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY_DM, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u, DM %u", 5) \
+X( 19, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM, "Setup register set=%u DM=%u, PC address=0x%08x%08x, OSid=%u, NewPCRegRequired=%u", 6) \
+X( 20, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_ALLOC, "Alloc PC set %u as register range [%u - %u]", 3) \
+X( 21, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSIDx, "ui32CoreID = %u, ui32OSid = %u, Catbase = %u, Reg Address = 0x%x, Reg index = %u, Bitshift index = %u, Val = 0x%08x%08x", 8) \
+X( 22, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM_SECURE, "Setup secure register=%u DM=%u, PC address=0x%08x%08x, OSid=%u, NewContext=%u", 6) \
+X( 23, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_DM_DEPRECATED, "Activate MemCtx=0x%08x DM=%u secure=%u CtxFlags=0x%08x", 4) \
+X( 24, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE_DEPRECATED, "Deactivate MemCtx=0x%08x CtxFlags=0x%08x", 2) \
+X( 25, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE_AND_UNGRAB_PCSET, "Deactivate MemCtx=0x%08x, ungrab reg set %u refcount now %u", 3) \
+\
+X(  1, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_WRITE, "GPIO write 0x%02x", 1) \
+X(  2, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_READ, "GPIO read 0x%02x", 1) \
+X(  3, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_ENABLED, "GPIO enabled", 0) \
+X(  4, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_DISABLED, "GPIO disabled", 0) \
+X(  5, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_STATUS, "GPIO status=%u (0=OK, 1=Disabled)", 1) \
+X(  6, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_READ, "GPIO_AP: Read address=0x%02x (%u byte(s))", 2) \
+X(  7, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_WRITE, "GPIO_AP: Write address=0x%02x (%u byte(s))", 2) \
+X(  8, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_TIMEOUT, "GPIO_AP timeout!", 0) \
+X(  9, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_ERROR, "GPIO_AP error. GPIO status=%u (0=OK, 1=Disabled)", 1) \
+X( 10, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_ALREADY_READ, "GPIO already read 0x%02x", 1) \
+X( 11, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_CHECK_BUFFER_AVAILABLE, "SR: Check buffer %u available returned %u", 2) \
+X( 12, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_WAITING_BUFFER_AVAILABLE, "SR: Waiting for buffer %u", 1) \
+X( 13, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_WAIT_BUFFER_TIMEOUT, "SR: Timeout waiting for buffer %u (after %u ticks)", 2) \
+X( 14, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_FRAME_CHECK, "SR: Skip frame check for strip %u returned %u (0=No skip, 1=Skip frame)", 2) \
+X( 15, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_REMAINING_STRIPS, "SR: Skip remaining strip %u in frame", 1) \
+X( 16, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_FRAME_SKIP_NEW_FRAME, "SR: Inform HW that strip %u is a new frame", 1) \
+X( 17, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_FRAME_TIMEOUT, "SR: Timeout waiting for INTERRUPT_FRAME_SKIP (after %u ticks)", 1) \
+X( 18, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_STRIP_MODE, "SR: Strip mode is %u", 1) \
+X( 19, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_STRIP_INDEX, "SR: Strip Render start (strip %u)", 1) \
+X( 20, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_BUFFER_RENDERED, "SR: Strip Render complete (buffer %u)", 1) \
+X( 21, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_BUFFER_FAULT, "SR: Strip Render fault (buffer %u)", 1) \
+X( 22, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_DEPRECATED2, "TRP state: %u", 1) \
+X( 23, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_FAILURE_DEPRECATED, "TRP failure: %u", 1) \
+X( 24, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SW_TRP_STATE_DEPRECATED, "SW TRP State: %u", 1) \
+X( 25, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SW_TRP_FAILURE_DEPRECATED2, "SW TRP failure: %u", 1) \
+X( 26, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HW_KICK_DEPRECATED, "HW kick event (%u)", 1) \
+X( 27, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WGP_CHECKSUMS_DEPRECATED, "GPU core (%u/%u): checksum 0x%08x vs. 0x%08x", 4) \
+X( 28, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WGP_UNIT_CHECKSUMS_DEPRECATED, "GPU core (%u/%u), unit (%u,%u): checksum 0x%08x vs. 0x%08x", 6) \
+X( 29, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_CHECK_REG_DEPRECATED, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 30, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_USC_SLOTS_CHECK_DEPRECATED, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X( 31, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_USC_REG_CHECK_DEPRECATED, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 32, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_HWRTDATA_DEPRECATED, "TRP HWRTData: 0x%08x, state: %u", 2) \
+X( 33, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_CNTX_DEPRECATED, "TRP Context: 0x%08x, state: %u", 2) \
+X( 34, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_FAILURE_CNTX_DEPRECATED, "TRP Context: 0x%08x, failure: %u", 2) \
+X( 35, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DEPRECATED, "Memory dump: Addr=0x%02x%08x, Size=%d, ContextId=%u, DM=%u", 5) \
+X( 36, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS4_DEPRECATED, "  0x%02x%08x  %08x %08x %08x %08x", 6) \
+X( 37, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS3_DEPRECATED, "  0x%02x%08x  %08x %08x %08x", 5) \
+X( 38, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS2_DEPRECATED, "  0x%02x%08x  %08x %08x", 4) \
+X( 39, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS1_DEPRECATED, "  0x%02x%08x  %08x", 3) \
+X( 40, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DEPRECATED, "Memory dump: Addr=0x%08x, Size=%d", 2) \
+X( 41, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS4_DEPRECATED, "  0x%08x  %08x %08x %08x %08x", 5) \
+X( 42, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS3_DEPRECATED, "  0x%08x  %08x %08x %08x", 4) \
+X( 43, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS2_DEPRECATED, "  0x%08x  %08x %08x", 3) \
+X( 44, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS1_DEPRECATED, "  0x%08x  %08x", 2) \
+X( 45, RGXFW_GROUP_MISC, RGXFW_SF_MISC_FBCDC_FAILURE_STATUS_DEPRECATED, "FBCDC: Core=0x%08x, Status=0x%08x, Signature status=0x%08x", 3) \
+X( 46, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WORK_CYCLES_PIPEDM_EN, "FWCtx 0x%08.8x, PipeDM state %04x, (start) %08x, (end) %08x, elapsed %08x", 5) \
+X( 47, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WORK_CYCLES, "FWCtx 0x%08.8x, elapsed %08x", 2) \
+X( 48, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_UNEXPECTED_EVENT_DEPRECATED, "Unexpected TRP pass completion on DM: %u",1) \
+X( 49, RGXFW_GROUP_MISC, RGXFW_SF_MISC_FAILED_IRQ_CLEAR_DEPRECATED, "Host failed to clear IRQ, Reg: 0x%08x",1) \
+\
+X(  1, RGXFW_GROUP_PM, RGXFW_SF_PM_AMLIST, "ALIST%u SP = %u, MLIST%u SP = %u (VCE 0x%08x%08x, TE 0x%08x%08x, ALIST 0x%08x%08x)", 10) \
+X(  2, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED_DEPRECATED, "Is TA: %u, finished: %u on HW %u (HWRTData = 0x%08x, MemCtx = 0x%08x). FL different between TA/3D: global:%u, local:%u, mmu:%u", 8) \
+X(  3, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE_DEPRECATED, "UFL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), MFL-3D-Base: 0x%08x%08x (SP = %u, 4PT = %u)", 14) \
+X(  4, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE_DEPRECATED, "UFL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), MFL-TA-Base: 0x%08x%08x (SP = %u, 4PT = %u)", 14) \
+X(  5, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_GROW_COMPLETE_DEPRECATED, "Freelist grow completed [0x%08x]: added pages 0x%08x, total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 5) \
+X(  6, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_GROW_DENIED_DEPRECATED, "Grow for freelist ID=0x%08x denied by host", 1) \
+X(  7, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_COMPLETE, "Freelist update completed [0x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 5) \
+X(  8, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_RECONSTRUCTION_FAILED_DEPRECATED, "Reconstruction of freelist ID=0x%08x failed", 1) \
+X(  9, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_WARNING, "Ignored attempt to pause or unpause the DM while there is no relevant operation in progress (0-TA,1-3D): %u, operation(0-unpause, 1-pause): %u", 2) \
+X( 10, RGXFW_GROUP_PM, RGXFW_SF_PM_3D_TIMEOUT_STATUS, "Force free 3D Context memory, FWCtx: 0x%08x, status(1:success, 0:fail): %u", 2)\
+X( 11, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_ALLOC, "PM pause TA ALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 12, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_UNPAUSE_ALLOC, "PM unpause TA ALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 13, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_DALLOC, "PM pause 3D DALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 14, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_UNPAUSE_DALLOC, "PM unpause 3D DALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 15, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_FAILED, "PM ALLOC/DALLOC change was not actioned: PM_PAGE_MANAGEOP_STATUS=0x%x", 1) \
+X( 16, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED, "Is TA: %u, finished: %u on HW %u (HWRTData = 0x%08x, MemCtx = 0x%08x). FL different between TA/3D: global:%u, local:%u", 7) \
+X( 17, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE, "UFL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 18, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE, "UFL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 19, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_COMPLETE_VOLCANIC, "Freelist update completed [0x%08x / FL State 0x%08x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 7) \
+X( 20, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_FAILED, "Freelist update failed [0x%08x / FL State 0x%08x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 7) \
+X( 21, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE_VOLCANIC, "UFL-3D-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 22, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE_VOLCANIC, "UFL-TA-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 23, RGXFW_GROUP_PM, RGXFW_SF_PM_CHECK_FL_BASEADDR, "Freelist 0x%08x base address from HW: 0x%02x%08x (expected value: 0x%02x%08x)", 5) \
+X( 24, RGXFW_GROUP_PM, RGXFW_SF_PM_ANALYSE_FL_GROW, "Analysis of FL grow: Pause=(%u,%u) Paused+Valid(%u,%u) PMStateBuffer=0x%x", 5) \
+X( 25, RGXFW_GROUP_PM, RGXFW_SF_PM_ATTEMPT_FL_GROW, "Attempt FL grow for FL: 0x%08x, new dev address: 0x%02x%08x, new page count: %u, new ready count: %u", 5) \
+X( 26, RGXFW_GROUP_PM, RGXFW_SF_PM_DEFER_FL_GROW, "Deferring FL grow for non-loaded FL: 0x%08x, new dev address: 0x%02x%08x, new page count: %u, new ready count: %u", 5) \
+X( 27, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED_ALBIORIX, "Is GEOM: %u, finished: %u (HWRTData = 0x%08x, MemCtx = 0x%08x)", 4) \
+X( 28, RGXFW_GROUP_PM, RGXFW_SF_PM_3D_TIMEOUT, "3D Timeout Now for FWCtx 0x%08.8x", 1) \
+X( 29, RGXFW_GROUP_PM, RGXFW_SF_PM_RECYCLE, "GEOM PM Recycle for FWCtx 0x%08.8x", 1) \
+X( 30, RGXFW_GROUP_PM, RGXFW_SF_PM_PRIMARY_CONFIG, "PM running primary config (Core %u)", 1) \
+X( 31, RGXFW_GROUP_PM, RGXFW_SF_PM_SECONDARY_CONFIG, "PM running secondary config (Core %u)", 1) \
+X( 32, RGXFW_GROUP_PM, RGXFW_SF_PM_TERTIARY_CONFIG, "PM running tertiary config (Core %u)", 1) \
+X( 33, RGXFW_GROUP_PM, RGXFW_SF_PM_QUATERNARY_CONFIG, "PM running quaternary config (Core %u)", 1) \
+X( 34, RGXFW_GROUP_PM, RGXFW_SF_PM_REVERT_CONFIG, "PM reverting to previous config (Core %u)", 1) \
+\
+X(  1, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GLL_DYNAMIC_STATUS_DEPRECATED, "Global link list dynamic page count: vertex 0x%x, varying 0x%x, node 0x%x", 3) \
+X(  2, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GLL_STATIC_STATUS_DEPRECATED, "Global link list static page count: vertex 0x%x, varying 0x%x, node 0x%x", 3) \
+X(  3, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_WAIT_FOR_GROW_DEPRECATED, "RPM request failed. Waiting for freelist grow.", 0) \
+X(  4, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_ABORT_DEPRECATED, "RPM request failed. Aborting the current frame.", 0) \
+X(  5, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_WAIT_FOR_PENDING_GROW_DEPRECATED, "RPM waiting for pending grow on freelist 0x%08x", 1) \
+X(  6, RGXFW_GROUP_RPM, RGXFW_SF_RPM_REQUEST_HOST_GROW_DEPRECATED, "Request freelist grow [0x%08x] current pages %u, grow size %u", 3) \
+X(  7, RGXFW_GROUP_RPM, RGXFW_SF_RPM_FREELIST_LOAD_DEPRECATED, "Freelist load: SHF = 0x%08x, SHG = 0x%08x", 2) \
+X(  8, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_DEPRECATED, "SHF FPL register: 0x%08x.0x%08x", 2) \
+X(  9, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_DEPRECATED, "SHG FPL register: 0x%08x.0x%08x", 2) \
+X( 10, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_FREELIST_DEPRECATED, "Kernel requested RPM grow on freelist (type %u) at 0x%08x from current size %u to new size %u, RPM restart: %u (1=Yes)", 5) \
+X( 11, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_RESTART_DEPRECATED, "Restarting SHG", 0) \
+X( 12, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_ABORTED_DEPRECATED, "Grow failed, aborting the current frame.", 0) \
+X( 13, RGXFW_GROUP_RPM, RGXFW_SF_RPM_ABORT_COMPLETE_DEPRECATED, "RPM abort complete on HWFrameData [0x%08x].", 1) \
+X( 14, RGXFW_GROUP_RPM, RGXFW_SF_RPM_CLEANUP_NEEDS_ABORT_DEPRECATED, "RPM freelist cleanup [0x%08x] requires abort to proceed.", 1) \
+X( 15, RGXFW_GROUP_RPM, RGXFW_SF_RPM_RPM_PT_DEPRECATED, "RPM page table base register: 0x%08x.0x%08x", 2) \
+X( 16, RGXFW_GROUP_RPM, RGXFW_SF_RPM_OOM_ABORT_DEPRECATED, "Issuing RPM abort.", 0) \
+X( 17, RGXFW_GROUP_RPM, RGXFW_SF_RPM_OOM_TOGGLE_CHECK_FULL_DEPRECATED, "RPM OOM received but toggle bits indicate free pages available", 0) \
+X( 18, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_HW_TIMEOUT_DEPRECATED, "RPM hardware timeout. Unable to process OOM event.", 0) \
+X( 19, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_LOAD_DEPRECATED_DEPRECATED, "SHF FL (0x%08x) load, FPL: 0x%08x.0x%08x, roff: 0x%08x, woff: 0x%08x", 5) \
+X( 20, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_LOAD_DEPRECATED, "SHG FL (0x%08x) load, FPL: 0x%08x.0x%08x, roff: 0x%08x, woff: 0x%08x", 5) \
+X( 21, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_STORE_DEPRECATED, "SHF FL (0x%08x) store, roff: 0x%08x, woff: 0x%08x", 3) \
+X( 22, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_STORE_DEPRECATED, "SHG FL (0x%08x) store, roff: 0x%08x, woff: 0x%08x", 3) \
+\
+X(  1, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_FINISHED, "3D RTData 0x%08x finished on HW context %u", 2) \
+X(  2, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_READY, "3D RTData 0x%08x ready on HW context %u", 2) \
+X(  3, RGXFW_GROUP_RTD, RGXFW_SF_RTD_PB_SET_TO_DEPRECATED, "CONTEXT_PB_BASE set to 0x%x, FL different between TA/3D: local: %u, global: %u, mmu: %u", 4) \
+X(  4, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOADVFP_3D_DEPRECATED, "Loading VFP table 0x%08x%08x for 3D", 2) \
+X(  5, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOADVFP_TA_DEPRECATED, "Loading VFP table 0x%08x%08x for TA", 2) \
+X(  6, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_DEPRECATED, "Load Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: TotalPMPages = %u, FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 10) \
+X(  7, RGXFW_GROUP_RTD, RGXFW_SF_RTD_VHEAP_STORE, "Perform VHEAP table store", 0) \
+X(  8, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_MATCH_FOUND, "RTData 0x%08x: found match in Context=%u: Load=No, Store=No", 2) \
+X(  9, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_NULL_FOUND, "RTData 0x%08x: found NULL in Context=%u: Load=Yes, Store=No", 2) \
+X( 10, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_3D_FINISHED, "RTData 0x%08x: found state 3D finished (0x%08x) in Context=%u: Load=Yes, Store=Yes", 3) \
+X( 11, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_TA_FINISHED, "RTData 0x%08x: found state TA finished (0x%08x) in Context=%u: Load=Yes, Store=Yes", 3) \
+X( 12, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_STACK_POINTERS, "Loading stack-pointers for %u (0:MidTA,1:3D) on context %u, MLIST = 0x%08x, ALIST = 0x%08x%08x", 5) \
+X( 13, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_PB_DEPRECATED, "Store Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: TotalPMPages = %u, FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 10) \
+X( 14, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_FINISHED, "TA RTData 0x%08x finished on HW context %u", 2) \
+X( 15, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_LOADED, "TA RTData 0x%08x loaded on HW context %u", 2) \
+X( 16, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_PB_DEPRECATED2, "Store Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 17, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_DEPRECATED2, "Load Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 18, RGXFW_GROUP_RTD, RGXFW_SF_RTD_DEBUG_DEPRECATED, "Freelist 0x%x RESET!!!!!!!!", 1) \
+X( 19, RGXFW_GROUP_RTD, RGXFW_SF_RTD_DEBUG2_DEPRECATED, "Freelist 0x%x stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 5) \
+X( 20, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_DEPRECATED, "Request reconstruction of Freelist 0x%x type: %u (0:local,1:global,2:mmu) on HW context %u", 3) \
+X( 21, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_ACK_DEPRECATED, "Freelist reconstruction ACK from host (HWR state :%u)", 1) \
+X( 22, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_ACK_DEPRECATED2, "Freelist reconstruction completed", 0) \
+X( 23, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_LOADED_DEPRECATED, "TA RTData 0x%08x loaded on HW context %u HWRTDataNeedsLoading=%u", 3) \
+X( 24, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TE_RGNHDR_INFO, "TE Region headers base 0x%08x%08x (RGNHDR Init: %u)", 3) \
+X( 25, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_BUFFER_ADDRS_DEPRECATED, "TA Buffers: FWCtx 0x%08x, RT 0x%08x, RTData 0x%08x, VHeap 0x%08x%08x, TPC 0x%08x%08x (MemCtx 0x%08x)", 8) \
+X( 26, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_LOADED_DEPRECATED, "3D RTData 0x%08x loaded on HW context %u", 2) \
+X( 27, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS_DEPRECATED, "3D Buffers: FWCtx 0x%08x, RT 0x%08x, RTData 0x%08x (MemCtx 0x%08x)", 4) \
+X( 28, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RESTART_AFTER_PR_EXECUTED, "Restarting TA after partial render, HWRTData0State=0x%x, HWRTData1State=0x%x", 2) \
+X( 29, RGXFW_GROUP_RTD, RGXFW_SF_RTD_PB_SET_TO, "CONTEXT_PB_BASE set to 0x%x, FL different between TA/3D: local: %u, global: %u", 3) \
+X( 30, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_FL, "Store Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 31, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL, "Load Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 32, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS_DEPRECATED2, "3D Buffers: FWCtx 0x%08x, parent RT 0x%08x, RTData 0x%08x on ctx %u, (MemCtx 0x%08x)", 5) \
+X( 33, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_BUFFER_ADDRS, "TA Buffers: FWCtx 0x%08x, RTData 0x%08x, VHeap 0x%08x%08x, TPC 0x%08x%08x (MemCtx 0x%08x)", 7) \
+X( 34, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS, "3D Buffers: FWCtx 0x%08x, RTData 0x%08x on ctx %u, (MemCtx 0x%08x)", 4) \
+X( 35, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_V2, "Load Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u)", 6) \
+X( 36, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILLED_TA, "TA RTData 0x%08x marked as killed.", 1) \
+X( 37, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILLED_3D, "3D RTData 0x%08x marked as killed.", 1) \
+X( 38, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILL_TA_AFTER_RESTART, "RTData 0x%08x will be killed after TA restart.", 1) \
+X( 39, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RENDERSTATE_RESET, "RTData 0x%08x Render State Buffer 0x%02x%08x will be reset.", 3) \
+X( 40, RGXFW_GROUP_RTD, RGXFW_SF_RTD_GEOM_RENDERSTATE, "GEOM RTData 0x%08x using Render State Buffer 0x%02x%08x.", 3) \
+X( 41, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FRAG_RENDERSTATE, "FRAG RTData 0x%08x using Render State Buffer 0x%02x%08x.", 3) \
+\
+X(  1, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZLOAD_DEPRECATED, "Force Z-Load for partial render", 0) \
+X(  2, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSTORE_DEPRECATED, "Force Z-Store for partial render", 0) \
+X(  3, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_LOCAL_DEPRECATED, "3D MemFree: Local FL 0x%08x", 1) \
+X(  4, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_MMU_DEPRECATED, "3D MemFree: MMU FL 0x%08x", 1) \
+X(  5, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_GLOBAL_DEPRECATED, "3D MemFree: Global FL 0x%08x", 1) \
+X(  6, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_DEPRECATED, "OOM TA/3D PR Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x, HardwareSync Fence [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 6) \
+X(  7, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_UN_FL, "OOM TA_cmd=0x%08x, U-FL 0x%08x, N-FL 0x%08x", 3) \
+X(  8, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_UN_MMU_FL_DEPRECATED, "OOM TA_cmd=0x%08x, OOM MMU:%u, U-FL 0x%08x, N-FL 0x%08x, MMU-FL 0x%08x", 5) \
+X(  9, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_AVOIDED_DEPRECATED, "Partial render avoided", 0) \
+X( 10, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_DISCARDED_DEPRECATED, "Partial render discarded", 0) \
+X( 11, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_FINISHED, "Partial Render finished", 0) \
+X( 12, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_3DBG_DEPRECATED, "SPM Owner = 3D-BG", 0) \
+X( 13, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_3DIRQ_DEPRECATED, "SPM Owner = 3D-IRQ", 0) \
+X( 14, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_NONE_DEPRECATED, "SPM Owner = NONE", 0) \
+X( 15, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_TABG_DEPRECATED, "SPM Owner = TA-BG", 0) \
+X( 16, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_TAIRQ_DEPRECATED, "SPM Owner = TA-IRQ", 0) \
+X( 17, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSTORE_ADDRESS, "ZStore address 0x%08x%08x", 2) \
+X( 18, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SSTORE_ADDRESS, "SStore address 0x%08x%08x", 2) \
+X( 19, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZLOAD_ADDRESS, "ZLoad address 0x%08x%08x", 2) \
+X( 20, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SLOAD_ADDRESS, "SLoad address 0x%08x%08x", 2) \
+X( 21, RGXFW_GROUP_SPM, RGXFW_SF_SPM_NO_DEFERRED_ZSBUFFER_DEPRECATED, "No deferred ZS Buffer provided", 0) \
+X( 22, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_POPULATED, "ZS Buffer successfully populated (ID=0x%08x)", 1) \
+X( 23, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_POP_UNNEEDED_DEPRECATED, "No need to populate ZS Buffer (ID=0x%08x)", 1) \
+X( 24, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNPOPULATED, "ZS Buffer successfully unpopulated (ID=0x%08x)", 1) \
+X( 25, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNPOP_UNNEEDED_DEPRECATED, "No need to unpopulate ZS Buffer (ID=0x%08x)", 1) \
+X( 26, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_DEPRECATED, "Send ZS-Buffer backing request to host (ID=0x%08x)", 1) \
+X( 27, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNBACKING_REQUEST_DEPRECATED, "Send ZS-Buffer unbacking request to host (ID=0x%08x)", 1) \
+X( 28, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_PENDING_DEPRECATED, "Don't send ZS-Buffer backing request. Previous request still pending (ID=0x%08x)", 1) \
+X( 29, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNBACKING_REQUEST_PENDING_DEPRECATED, "Don't send ZS-Buffer unbacking request. Previous request still pending (ID=0x%08x)", 1) \
+X( 30, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZBUFFER_NOT_READY_DEPRECATED, "Partial Render waiting for ZBuffer to be backed (ID=0x%08x)", 1) \
+X( 31, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SBUFFER_NOT_READY_DEPRECATED, "Partial Render waiting for SBuffer to be backed (ID=0x%08x)", 1) \
+X( 32, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_NONE, "SPM State = none", 0) \
+X( 33, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_BLOCKED, "SPM State = PR blocked", 0) \
+X( 34, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_GROW, "SPM State = wait for grow", 0) \
+X( 35, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_HW, "SPM State = wait for HW", 0) \
+X( 36, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_RUNNING, "SPM State = PR running", 0) \
+X( 37, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_AVOIDED, "SPM State = PR avoided", 0) \
+X( 38, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_EXECUTED, "SPM State = PR executed", 0) \
+X( 39, RGXFW_GROUP_SPM, RGXFW_SF_SPM_FREELIST_MATCH, "3DMemFree matches freelist 0x%08x (FL type = %u)", 2) \
+X( 40, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_FLAG_SET, "Raise the 3DMemFreeDetected flag", 0) \
+X( 41, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_PENDING_GROW, "Wait for pending grow on Freelist 0x%08x", 1) \
+X( 42, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_FAILED, "ZS Buffer failed to be populated (ID=0x%08x)", 1) \
+X( 43, RGXFW_GROUP_SPM, RGXFW_SF_SPM_FL_GROW_DEBUG, "Grow update inconsistency: FL addr: 0x%02x%08x, curr pages: %u, ready: %u, new: %u", 5) \
+X( 44, RGXFW_GROUP_SPM, RGXFW_SF_SPM_RESUMED_TA_WITH_SP, "OOM: Resumed TA with ready pages, FL addr: 0x%02x%08x, current pages: %u, SP : %u", 4) \
+X( 45, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ACK_GROW_UPDATE_DEPRECATED, "Received grow update, FL addr: 0x%02x%08x, current pages: %u, ready pages: %u, threshold: %u", 5) \
+X( 46, RGXFW_GROUP_SPM, RGXFW_SF_SPM_NO_DEFERRED_PRBUFFER, "No deferred partial render FW (Type=%u) Buffer provided", 1) \
+X( 47, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_POP_UNNEEDED, "No need to populate PR Buffer (ID=0x%08x)", 1) \
+X( 48, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNPOP_UNNEEDED, "No need to unpopulate PR Buffer (ID=0x%08x)", 1) \
+X( 49, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_REQUEST, "Send PR Buffer backing request to host (ID=0x%08x)", 1) \
+X( 50, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNBACKING_REQUEST, "Send PR Buffer unbacking request to host (ID=0x%08x)", 1) \
+X( 51, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_REQUEST_PENDING, "Don't send PR Buffer backing request. Previous request still pending (ID=0x%08x)", 1) \
+X( 52, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNBACKING_REQUEST_PENDING, "Don't send PR Buffer unbacking request. Previous request still pending (ID=0x%08x)", 1) \
+X( 53, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_NOT_READY, "Partial Render waiting for Buffer %u type to be backed (ID=0x%08x)", 2) \
+X( 54, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ACK_GROW_UPDATE, "Received grow update, FL addr: 0x%02x%08x, new pages: %u, ready pages: %u", 4) \
+X( 66, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD, "OOM TA/3D PR Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 67, RGXFW_GROUP_SPM, RGXFW_SF_SPM_RESUMED_TA, "OOM: Resumed TA with ready pages, FL addr: 0x%02x%08x, current pages: %u", 3) \
+X( 68, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PR_DEADLOCK_UNBLOCKED, "OOM TA/3D PR deadlock unblocked reordering DM%u runlist head from Context 0x%08x to 0x%08x", 3) \
+X( 69, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_FORCEFREE, "SPM State = PR force free", 0) \
+X( 70, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_FAILED, "Failure to back PR Buffer", 0) \
+\
+X(  1, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK_DEPRECATED, "Check Pow state DM%u int: 0x%x, ext: 0x%x, pow flags: 0x%x", 4) \
+X(  2, RGXFW_GROUP_POW, RGXFW_SF_POW_GPU_IDLE, "GPU idle (might be powered down). Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X(  3, RGXFW_GROUP_POW, RGXFW_SF_POW_OSREQ_DEPRECATED, "OS requested pow off (forced = %u), DM%u, pow flags: 0x%x", 3) \
+X(  4, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_DEPRECATED, "Initiate powoff query. Inactive DMs: %u %u %u %u", 4) \
+X(  5, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECKOFF_DEPRECATED, "Any RD-DM pending? %u, Any RD-DM Active? %u", 2) \
+X(  6, RGXFW_GROUP_POW, RGXFW_SF_POW_GPU_OFF, "GPU ready to be powered down. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X(  7, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ, "HW Request On(1)/Off(0): %u, Units: 0x%08.8x", 2) \
+X(  8, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE_REQ, "Request to change num of dusts to %u (Power flags=%u)", 2) \
+X(  9, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE, "Changing number of dusts from %u to %u", 2) \
+X( 11, RGXFW_GROUP_POW, RGXFW_SF_POW_SIDEKICK_INIT_DEPRECATED, "Sidekick init", 0) \
+X( 12, RGXFW_GROUP_POW, RGXFW_SF_POW_RD_INIT_DEPRECATED, "Rascal+Dusts init (# dusts mask: 0x%x)", 1) \
+X( 13, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_RD, "Initiate powoff query for RD-DMs.", 0) \
+X( 14, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_TLA, "Initiate powoff query for TLA-DM.", 0) \
+X( 15, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_RD, "Any RD-DM pending? %u, Any RD-DM Active? %u", 2) \
+X( 16, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_TLA, "TLA-DM pending? %u, TLA-DM Active? %u", 2) \
+X( 17, RGXFW_GROUP_POW, RGXFW_SF_POW_BRN37270_DEPRECATED, "Request power up due to BRN37270. Pow stat int: 0x%x", 1) \
+X( 18, RGXFW_GROUP_POW, RGXFW_SF_POW_REQ_CANCEL, "Cancel power off request int: 0x%x, ext: 0x%x, pow flags: 0x%x", 3) \
+X( 19, RGXFW_GROUP_POW, RGXFW_SF_POW_FORCED_IDLE, "OS requested forced IDLE, pow flags: 0x%x", 1) \
+X( 20, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_FORCED_IDLE, "OS cancelled forced IDLE, pow flags: 0x%x", 1) \
+X( 21, RGXFW_GROUP_POW, RGXFW_SF_POW_IDLE_TIMER, "Idle timer start. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 22, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_IDLE_TIMER, "Cancel idle timer. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 23, RGXFW_GROUP_POW, RGXFW_SF_POW_APM_LATENCY_CHANGE, "Active PM latency set to %ums. Core clock: %u Hz", 2) \
+X( 24, RGXFW_GROUP_POW, RGXFW_SF_POW_CDM_CLUSTERS, "Compute cluster mask change to 0x%x, %u dusts powered.", 2) \
+X( 25, RGXFW_GROUP_POW, RGXFW_SF_POW_NULL_CMD_INIOFF_RD, "Null command executed, repeating initiate powoff query for RD-DMs.", 0) \
+X( 26, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_ENERGY, "Power monitor: Estimate of dynamic energy %u", 1) \
+X( 27, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK_DEPRECATED2, "Check Pow state: Int: 0x%x, Ext: 0x%x, Pow flags: 0x%x", 3) \
+X( 28, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_NEW_DEADLINE, "Proactive DVFS: New deadline, time = 0x%08x%08x", 2) \
+X( 29, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_NEW_WORKLOAD, "Proactive DVFS: New workload, cycles = 0x%08x%08x", 2) \
+X( 30, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_CALCULATE, "Proactive DVFS: Proactive frequency calculated = %u", 1) \
+X( 31, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UTILISATION, "Proactive DVFS: Reactive utilisation = %u percent", 1) \
+X( 32, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_REACT, "Proactive DVFS: Reactive frequency calculated = %u.%u", 2) \
+X( 33, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_SEND_DEPRECATED, "Proactive DVFS: OPP Point Sent = 0x%x", 1) \
+X( 34, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DEADLINE_REMOVED, "Proactive DVFS: Deadline removed = 0x%08x%08x", 2) \
+X( 35, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_WORKLOAD_REMOVED, "Proactive DVFS: Workload removed = 0x%08x%08x", 2) \
+X( 36, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_THROTTLE, "Proactive DVFS: Throttle to a maximum = 0x%x", 1) \
+X( 37, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_FAILURE, "Proactive DVFS: Failed to pass OPP point via GPIO.", 0) \
+X( 38, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_INVALID_NODE_DEPRECATED, "Proactive DVFS: Invalid node passed to function.", 0) \
+X( 39, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GUEST_BAD_ACCESS_DEPRECATED, "Proactive DVFS: Guest OS attempted to do a privileged action. OSid = %u", 1) \
+X( 40, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UNPROFILED_STARTED, "Proactive DVFS: Unprofiled work started. Total unprofiled work present: %u", 1) \
+X( 41, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UNPROFILED_FINISHED, "Proactive DVFS: Unprofiled work finished. Total unprofiled work present: %u", 1) \
+X( 42, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DISABLED, "Proactive DVFS: Disabled: Not enabled by host.", 0) \
+X( 43, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ_RESULT, "HW Request Completed(1)/Aborted(0): %u, Ticks: %u", 2) \
+X( 44, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE_FIX_59042_DEPRECATED, "Allowed number of dusts is %u due to BRN59042.", 1) \
+X( 45, RGXFW_GROUP_POW, RGXFW_SF_POW_HOST_TIMEOUT_NOTIFICATION, "Host timed out while waiting for a forced idle state. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 46, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK, "Check Pow state: Int: 0x%x, Ext: 0x%x, Pow flags: 0x%x, Fence Counters: Check: %u - Update: %u", 5) \
+X( 47, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_SEND, "Custom/Proactive DVFS: OPP Point Sent = 0x%x, Success = 0x%x", 2) \
+X( 48, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_TO_IDLE, "Proactive DVFS: GPU transitioned to idle", 0) \
+X( 49, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_TO_ACTIVE, "Proactive DVFS: GPU transitioned to active", 0) \
+X( 50, RGXFW_GROUP_POW, RGXFW_SF_POW_POWDUMP_BUFFER_SIZE, "Power counter dumping: Data truncated writing register %u. Buffer too small.", 1) \
+X( 51, RGXFW_GROUP_POW, RGXFW_SF_POW_POWCTRL_ABORT, "Power controller returned ABORT for last request so retrying.", 0) \
+X( 52, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_POWER_REQUEST_DEPRECATED, "Discarding invalid power request: type 0x%x, DM %u", 2) \
+X( 53, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_FORCED_IDLE_NOT_IDLE, "Detected attempt to cancel forced idle while not forced idle (pow state 0x%x, pow flags 0x%x)", 2) \
+X( 54, RGXFW_GROUP_POW, RGXFW_SF_POW_FORCED_POW_OFF_NOT_IDLE, "Detected attempt to force power off while not forced idle (pow state 0x%x, pow flags 0x%x)", 2) \
+X( 55, RGXFW_GROUP_POW, RGXFW_SF_POW_NUMDUST_CHANGE_NOT_IDLE, "Detected attempt to change dust count while not forced idle (pow state 0x%x)", 1) \
+X( 56, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_RESULT, "Power monitor: Type = %u (0 = power, 1 = energy), Estimate result = 0x%08x%08x", 3) \
+X( 57, RGXFW_GROUP_POW, RGXFW_SF_POW_MINMAX_CONFLICT, "Conflicting clock frequency range: OPP min = %u, max = %u", 2) \
+X( 58, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_FLOOR, "Proactive DVFS: Set floor to a minimum = 0x%x", 1) \
+X( 59, RGXFW_GROUP_POW, RGXFW_SF_POW_OSREQ, "OS requested pow off (forced = %u), pow flags: 0x%x", 2) \
+X( 60, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_POWER_REQUEST, "Discarding invalid power request: type 0x%x", 1) \
+X( 61, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_STATE_CHANGE_REQ, "Request to change SPU power state mask from 0x%x to 0x%x. Pow flags: 0x%x", 3) \
+X( 62, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_STATE_CHANGE, "Changing SPU power state mask from 0x%x to 0x%x", 2) \
+X( 63, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_CHANGE_NOT_IDLE, "Detected attempt to change SPU power state mask while not forced idle (pow state 0x%x)", 1) \
+X( 64, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_SPU_POWER_MASK, "Invalid SPU power mask 0x%x! Changing to 1", 1) \
+X( 65, RGXFW_GROUP_POW, RGXFW_SF_POW_CLKDIV_UPDATE, "Proactive DVFS: Send OPP %u with clock divider value %u", 2) \
+X( 66, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_PERF_MODE, "Power counters in raw/validation mode.", 0) \
+X( 67, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_RESET, "Reset PPA block state %u (1=reset, 0=recalculate).", 1) \
+X( 68, RGXFW_GROUP_POW, RGXFW_SF_POW_POWCTRL_ABORT_WITH_CORE, "Power controller returned ABORT for Core-%u last request so retrying.", 1) \
+X( 69, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ64BIT, "HW Request On(1)/Off(0): %u, Units: 0x%08x%08x", 3) \
+X( 70, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_RAC_POW_STATE_CHANGE_REQ, "Request to change SPU power state mask from 0x%x to 0x%x and RAC from 0x%x to 0x%x. Pow flags: 0x%x", 5) \
+X( 71, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_RAC_POW_STATE_CHANGE, "Changing SPU power state mask from 0x%x to 0x%x and RAC from 0x%x to 0x%x", 4) \
+X( 72, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_RAC, "RAC pending? %u, RAC Active? %u", 2) \
+X( 73, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_RAC, "Initiate powoff query for RAC.", 0) \
+X( 74, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DEFER_REACTIVE_UPDATE, "Proactive DVFS: Defer reactive update to meet next deadline 0x%08x%08x", 2) \
+\
+X(  1, RGXFW_GROUP_HWR, RGXFW_SF_HWR_LOCKUP_DEPRECATED, "Lockup detected on DM%u, FWCtx: 0x%08.8x", 2) \
+X(  2, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_FW_DEPRECATED, "Reset fw state for DM%u, FWCtx: 0x%08.8x, MemCtx: 0x%08.8x", 3) \
+X(  3, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED, "Reset HW", 0) \
+X(  4, RGXFW_GROUP_HWR, RGXFW_SF_HWR_TERMINATED_DEPRECATED, "Lockup recovered.", 0) \
+X(  5, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DEPRECATED, "Lock-up DM%u FWCtx: 0x%08.8x", 2) \
+X(  6, RGXFW_GROUP_HWR, RGXFW_SF_HWR_LOCKUP_DETECTED_DEPRECATED, "Lockup detected: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X(  7, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EARLY_FAULT_DETECTION_DEPRECATED, "Early fault detection: GLB(%u->%u), PER-DM(0x%08x)", 3) \
+X(  8, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_LOCKUP_DEPRECATED, "Hold scheduling due lockup: GLB(%u), PER-DM(0x%08x->0x%08x)", 3) \
+X(  9, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FALSE_LOCKUP_DEPRECATED, "False lockup detected: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X( 10, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BRN37729_DEPRECATED, "BRN37729: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X( 11, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FREELISTS_RECONSTRUCTED_DEPRECATED, "Freelists reconstructed: GLB(%u->%u), PER-DM(0x%08x)", 3) \
+X( 12, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RECONSTRUCTING_FREELISTS_DEPRECATED, "Reconstructing freelists: %u (0-No, 1-Yes): GLB(%u->%u), PER-DM(0x%08x)", 4) \
+X( 13, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FAILED_HW_POLL, "HW poll %u (0-Unset 1-Set) failed (reg:0x%08x val:0x%08x)", 3) \
+X( 14, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_DISCARDED_DEPRECATED, "Discarded cmd on DM%u FWCtx=0x%08x", 2) \
+X( 15, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_DISCARDED, "Discarded cmd on DM%u (reason=%u) HWRTData=0x%08x (st: %u), FWCtx 0x%08x @ %u", 6) \
+X( 16, RGXFW_GROUP_HWR, RGXFW_SF_HWR_PM_FENCE_DEPRECATED, "PM fence WA could not be applied, Valid TA Setup: %u, RD powered off: %u", 2) \
+X( 17, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_SNAPSHOT, "FL snapshot RTD 0x%08.8x - local (0x%08.8x): %u, global (0x%08.8x): %u", 5) \
+X( 18, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_CHECK, "FL check RTD 0x%08.8x, discard: %u - local (0x%08.8x): s%u?=c%u, global (0x%08.8x): s%u?=c%u", 8) \
+X( 19, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_DEPRECATED, "FL reconstruction 0x%08.8x c%u", 2) \
+X( 20, RGXFW_GROUP_HWR, RGXFW_SF_HWR_3D_CHECK, "3D check: missing TA FWCtx 0x%08.8x @ %u, RTD 0x%08x.", 3) \
+X( 21, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED2, "Reset HW (mmu:%u, extmem: %u)", 2) \
+X( 22, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ZERO_TA_CACHES, "Zero TA caches for FWCtx: 0x%08.8x (TPC addr: 0x%08x%08x, size: %u bytes)", 4) \
+X( 23, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FREELISTS_RECONSTRUCTED_DEPRECATED2, "Recovery DM%u: Freelists reconstructed. New R-Flags=0x%08x", 2) \
+X( 24, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SKIPPED_CMD, "Recovery DM%u: FWCtx 0x%08x skipped to command @ %u. PR=%u. New R-Flags=0x%08x", 5) \
+X( 25, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_RECOVERED, "Recovery DM%u: DM fully recovered", 1) \
+X( 26, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_LOCKUP, "DM%u: Hold scheduling due to R-Flag = 0x%08x", 2) \
+X( 27, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_RECONSTRUCTION, "Analysis: Need freelist reconstruction", 0) \
+X( 28, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_SKIP, "Analysis DM%u: Lockup FWCtx: 0x%08.8x. Need to skip to next command", 2) \
+X( 29, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_SKIP_OOM_TA, "Analysis DM%u: Lockup while TA is OOM FWCtx: 0x%08.8x. Need to skip to next command", 2) \
+X( 30, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_PR_CLEANUP, "Analysis DM%u: Lockup while partial render FWCtx: 0x%08.8x. Need PR cleanup", 2) \
+X( 31, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DEPRECATED2, "GPU has locked up", 0) \
+X( 32, RGXFW_GROUP_HWR, RGXFW_SF_HWR_READY, "DM%u ready for HWR", 1) \
+X( 33, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_UPDATE_RECOVERY, "Recovery DM%u: Updated Recovery counter. New R-Flags=0x%08x", 2) \
+X( 34, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BRN37729_DEPRECATED2, "Analysis: BRN37729 detected, reset TA and re-kicked 0x%08x)", 1) \
+X( 35, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_TIMED_OUT_DEPRECATED, "DM%u timed out", 1) \
+X( 36, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EVENT_STATUS_REG, "RGX_CR_EVENT_STATUS=0x%08x", 1) \
+X( 37, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_FALSE_LOCKUP, "DM%u lockup falsely detected, R-Flags=0x%08x", 2) \
+X( 38, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_OUTOFTIME, "GPU has overrun its deadline", 0) \
+X( 39, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_POLLFAILURE, "GPU has failed a poll", 0) \
+X( 40, RGXFW_GROUP_HWR, RGXFW_SF_HWR_PERF_PHASE_REG, "RGX DM%u phase count=0x%08x", 2) \
+X( 41, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED3, "Reset HW (loop:%u, poll failures: 0x%08x)", 2) \
+X( 42, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_FAULT_EVENT, "MMU fault event: 0x%08x", 1) \
+X( 43, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BIF1_FAULT, "BIF1 page fault detected (Bank1 MMU Status: 0x%08x)", 1) \
+X( 44, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK_TRUE_DEPRECATED, "Fast CRC Failed. Proceeding to full register checking (DM: %u).", 1) \
+X( 45, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_META_FAULT, "Meta MMU page fault detected (Meta MMU Status: 0x%08x%08x)", 2) \
+X( 46, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK_DEPRECATED, "Fast CRC Check result for DM%u is HWRNeeded=%u", 2) \
+X( 47, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FULL_CHECK_DEPRECATED, "Full Signature Check result for DM%u is HWRNeeded=%u", 2) \
+X( 48, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FINAL_RESULT, "Final result for DM%u is HWRNeeded=%u with HWRChecksToGo=%u", 3) \
+X( 49, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_CHECK_DEPRECATED, "USC Slots result for DM%u is HWRNeeded=%u USCSlotsUsedByDM=%u", 3) \
+X( 50, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DEADLINE_CHECK_DEPRECATED, "Deadline counter for DM%u is HWRDeadline=%u", 2) \
+X( 51, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_FREELIST_DEPRECATED, "Holding Scheduling on OSid %u due to pending freelist reconstruction", 1) \
+X( 52, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_REQUEST, "Requesting reconstruction for freelist 0x%x (ID=%u)", 2) \
+X( 53, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_PASSED, "Reconstruction of freelist ID=%u complete", 1) \
+X( 54, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global,2:mmu) on HW context %u", 4) \
+X( 55, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_FAILED, "Reconstruction of freelist ID=%u failed", 1) \
+X( 56, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESTRICTING_PDS_TASKS, "Restricting PDS Tasks to help other stalling DMs (RunningMask=0x%02x, StallingMask=0x%02x, PDS_CTRL=0x%08x%08x)", 4) \
+X( 57, RGXFW_GROUP_HWR, RGXFW_SF_HWR_UNRESTRICTING_PDS_TASKS, "Unrestricting PDS Tasks again (RunningMask=0x%02x, StallingMask=0x%02x, PDS_CTRL=0x%08x%08x)", 4) \
+X( 58, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_USED, "USC slots: %u used by DM%u", 2) \
+X( 59, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_EMPTY, "USC slots: %u empty", 1) \
+X( 60, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HCS_FIRE, "HCS DM%u's Context Switch failed to meet deadline. Current time: 0x%08x%08x, deadline: 0x%08x%08x", 5) \
+X( 61, RGXFW_GROUP_HWR, RGXFW_SF_HWR_START_HW_RESET, "Begin hardware reset (HWR Counter=%u)", 1) \
+X( 62, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FINISH_HW_RESET, "Finished hardware reset (HWR Counter=%u)", 1) \
+X( 63, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_FREELIST, "Holding Scheduling on DM %u for Driver ID %u due to pending freelist reconstruction", 2) \
+X( 64, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_UMQ_READ_OFFSET, "User Mode Queue ROff reset: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u becomes StreamStartOffset = %u)", 5) \
+X( 65, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED2, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global) on HW context %u", 4) \
+X( 66, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MIPS_FAULT, "Mips page fault detected (BadVAddr: 0x%08x, EntryLo0: 0x%08x, EntryLo1: 0x%08x)", 3) \
+X( 67, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ANOTHER_CHANCE, "At least one other DM is running okay so DM%u will get another chance", 1) \
+X( 68, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_FW, "Reconstructing in FW, FL: 0x%x (ID=%u)", 2) \
+X( 69, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ZERO_RTC, "Zero RTC for FWCtx: 0x%08.8x (RTC addr: 0x%08x%08x, size: %u bytes)", 4) \
+X( 70, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED3, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global) phase: %u (0:TA, 1:3D) on HW context %u", 5) \
+X( 71, RGXFW_GROUP_HWR, RGXFW_SF_HWR_START_LONG_HW_POLL, "Start long HW poll %u (0-Unset 1-Set) for (reg:0x%08x val:0x%08x)", 3) \
+X( 72, RGXFW_GROUP_HWR, RGXFW_SF_HWR_END_LONG_HW_POLL, "End long HW poll (result=%u)", 1) \
+X( 73, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DEADLINE_CHECK, "DM%u has taken %u ticks and deadline is %u ticks", 3) \
+X( 74, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WATCHDOG_CHECK_DEPRECATED, "USC Watchdog result for DM%u is HWRNeeded=%u Status=%u USCs={0x%x} with HWRChecksToGo=%u", 5) \
+X( 75, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED, "Reconstruction needed for freelist 0x%x (ID=%u) Driver ID: %u type: %u (0:local,1:global) phase: %u (0:TA, 1:3D) on HW context %u", 6) \
+X( 76, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP, "GPU-%u has locked up", 1) \
+X( 77, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DM, "DM%u has locked up", 1) \
+X( 78, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_EVENT_STATUS_REG, "Core %u RGX_CR_EVENT_STATUS=0x%08x", 2) \
+X( 79, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MULTICORE_EVENT_STATUS_REG, "RGX_CR_MULTICORE_EVENT_STATUS%u=0x%08x", 2) \
+X( 80, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_BIF0_FAULT, "BIF0 page fault detected (Core %u MMU Status: 0x%08x%08x Req Status: 0x%08x%08x)", 5) \
+X( 81, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_MMU_FAULT_S7, "MMU page fault detected (Core %u MMU Status: 0x%08x%08x)", 3) \
+X( 82, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_MMU_FAULT, "MMU page fault detected (Core %u MMU Status: 0x%08x%08x 0x%08x)", 4) \
+X( 83, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW, "Reset HW (core:%u of %u, loop:%u, poll failures: 0x%08x)", 4) \
+X( 84, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK, "Fast CRC Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 85, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FULL_CHECK, "Full Signature Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 86, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_CHECK, "USC Slots result for Core%u, DM%u is HWRNeeded=%u USCSlotsUsedByDM=%u", 4) \
+X( 87, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WATCHDOG_CHECK, "USC Watchdog result for Core%u DM%u is HWRNeeded=%u Status=%u USCs={0x%x} with HWRChecksToGo=%u", 6) \
+X( 88, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_RISCV_FAULT, "RISC-V MMU page fault detected (FWCORE MMU Status 0x%08x Req Status 0x%08x%08x)", 3) \
+X( 89, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_TEXAS1_PFS_DEPRECATED, "TEXAS1_PFS poll failed on core %u with value 0x%08x", 2) \
+X( 90, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_PFS, "BIF_PFS poll failed on core %u with value 0x%08x", 2) \
+X( 91, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SET_ABORT_PM_STATUS, "MMU_ABORT_PM_STATUS set poll failed on core %u with value 0x%08x", 2) \
+X( 92, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_UNSET_ABORT_PM_STATUS, "MMU_ABORT_PM_STATUS unset poll failed on core %u with value 0x%08x", 2) \
+X( 93, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SLC_INVAL, "MMU_CTRL_INVAL poll (all but fw) failed on core %u with value 0x%08x", 2) \
+X( 94, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SLCMMU_INVAL, "MMU_CTRL_INVAL poll (all) failed on core %u with value 0x%08x", 2) \
+X( 95, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_TEXAS_PFS, "TEXAS%u_PFS poll failed on core %u with value 0x%08x", 3) \
+X( 96, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EXTRA_CHECK, "Extra Registers Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 97, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WRITE_TO_GPU_READONLY_ADDR, "FW attempted to write to read-only GPU address 0x%08x", 1) \
+X( 98, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_TIMED_OUT, "DM%u timed out (phase count=0x%08x)", 2) \
+\
+X(  1, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CFGBLK, "Block 0x%x mapped to Config Idx %u", 2) \
+X(  2, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_OMTBLK, "Block 0x%x omitted from event - not enabled in HW", 1) \
+X(  3, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INCBLK, "Block 0x%x included in event - enabled in HW", 1) \
+X(  4, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_SELREG, "Select register state hi_0x%x lo_0x%x", 2) \
+X(  5, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CSBHDR, "Counter stream block header word 0x%x", 1) \
+X(  6, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTROFF, "Counter register offset 0x%x", 1) \
+X(  7, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CFGSKP, "Block 0x%x config unset, skipping", 1) \
+X(  8, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INDBLK, "Accessing Indirect block 0x%x", 1) \
+X(  9, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DIRBLK, "Accessing Direct block 0x%x", 1) \
+X( 10, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CNTPRG, "Programmed counter select register at offset 0x%x", 1) \
+X( 11, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKPRG, "Block register offset 0x%x and value 0x%x", 2) \
+X( 12, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKCG, "Reading config block from driver 0x%x", 1) \
+X( 13, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKRG, "Reading block range 0x%x to 0x%x", 2) \
+X( 14, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKREC, "Recording block 0x%x config from driver", 1) \
+X( 15, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKED, "Finished reading config block from driver", 0) \
+X( 16, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CUSTOM_COUNTER, "Custom Counter offset: 0x%x  value: 0x%x", 2) \
+X( 17, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_SELECT_CNTR, "Select counter n:%u  ID:0x%x", 2) \
+X( 18, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_SELECT_PACK, "The counter ID 0x%x is not allowed. The package [b:%u, n:%u] will be discarded", 3) \
+X( 19, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHANGE_FILTER_STATUS_CUSTOM, "Custom Counters filter status %u", 1) \
+X( 20, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_WRONG_BLOCK, "The Custom block %u is not allowed. Use only blocks lower than %u. The package will be discarded", 2) \
+X( 21, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_TOO_MANY_ID, "The package will be discarded because it contains %u counters IDs while the upper limit is %u", 2) \
+X( 22, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHECK_FILTER, "Check Filter 0x%x is 0x%x ?", 2) \
+X( 23, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_RESET_CUSTOM_BLOCK, "The custom block %u is reset", 1) \
+X( 24, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INVALID_CMD_DEPRECATED, "Encountered an invalid command (%u)", 1) \
+X( 25, RGXFW_GROUP_HWP, RGXFW_SF_HWP_WAITING_FOR_QUEUE_DEPRECATED, "HWPerf Queue is full, we will have to wait for space! (Roff = %u, Woff = %u)", 2) \
+X( 26, RGXFW_GROUP_HWP, RGXFW_SF_HWP_WAITING_FOR_QUEUE_FENCE_DEPRECATED, "HWPerf Queue is fencing, we are waiting for Roff = %u (Roff = %u, Woff = %u)", 3) \
+X( 27, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CUSTOM_BLOCK, "Custom Counter block: %u", 1) \
+X( 28, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKENA, "Block 0x%x ENABLED", 1) \
+X( 29, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKDIS, "Block 0x%x DISABLED", 1) \
+X( 30, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INDBLK_INSTANCE, "Accessing Indirect block 0x%x, instance %u", 2) \
+X( 31, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTRVAL, "Counter register 0x%x, Value 0x%x", 2) \
+X( 32, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHANGE_FILTER_STATUS, "Counters filter status %u", 1) \
+X( 33, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTLBLK, "Block 0x%x mapped to Ctl Idx %u", 2) \
+X( 34, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_WORKEST_EN, "Block(s) in use for workload estimation.", 0) \
+X( 35, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CYCCTR, "GPU %u Cycle counter 0x%x, Value 0x%x", 3) \
+X( 36, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CYCMAX, "GPU Mask 0x%x Cycle counter 0x%x, Value 0x%x", 3) \
+X( 37, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_IGNORE_BLOCKS, "Blocks IGNORED for GPU %u", 1) \
+\
+X(  1, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_REQUEST_DEPRECATED, "Transfer 0x%02x request: 0x%02x%08x -> 0x%08x, size %u", 5) \
+X(  2, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_COMPLETE, "Transfer of type 0x%02x expected on channel %u, 0x%02x found, status %u", 4) \
+X(  3, RGXFW_GROUP_DMA, RGXFW_SF_DMA_INT_REG, "DMA Interrupt register 0x%08x", 1) \
+X(  4, RGXFW_GROUP_DMA, RGXFW_SF_DMA_WAIT, "Waiting for transfer of type 0x%02x completion...", 1) \
+X(  5, RGXFW_GROUP_DMA, RGXFW_SF_DMA_CCB_LOADING_FAILED, "Loading of cCCB data from FW common context 0x%08x (offset: %u, size: %u) failed", 3) \
+X(  6, RGXFW_GROUP_DMA, RGXFW_SF_DMA_CCB_LOAD_INVALID, "Invalid load of cCCB data from FW common context 0x%08x (offset: %u, size: %u)", 3) \
+X(  7, RGXFW_GROUP_DMA, RGXFW_SF_DMA_POLL_FAILED, "Transfer 0x%02x request poll failure", 1) \
+X(  8, RGXFW_GROUP_DMA, RGXFW_SF_DMA_BOOT_TRANSFER_FAILED_DEPRECATED, "Boot transfer(s) failed (code? %u, data? %u), used slower memcpy instead", 2) \
+X(  9, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_REQUEST, "Transfer 0x%02x request on ch. %u: system 0x%02x%08x, coremem 0x%08x, flags 0x%x, size %u", 7) \
+\
+X(  1, RGXFW_GROUP_DBG, RGXFW_SF_DBG_INTPAIR, "0x%08x 0x%08x", 2) \
+X(  2, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1HEX, "0x%08x", 1) \
+X(  3, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2HEX, "0x%08x 0x%08x", 2) \
+X(  4, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3HEX, "0x%08x 0x%08x 0x%08x", 3) \
+X(  5, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4HEX, "0x%08x 0x%08x 0x%08x 0x%08x", 4) \
+X(  6, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 5) \
+X(  7, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 6) \
+X(  8, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 7) \
+X(  9, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 8) \
+X( 10, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1SIGNED, "%d", 1) \
+X( 11, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2SIGNED, "%d %d", 2) \
+X( 12, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3SIGNED, "%d %d %d", 3) \
+X( 13, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4SIGNED, "%d %d %d %d", 4) \
+X( 14, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5SIGNED, "%d %d %d %d %d", 5) \
+X( 15, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6SIGNED, "%d %d %d %d %d %d", 6) \
+X( 16, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7SIGNED, "%d %d %d %d %d %d %d", 7) \
+X( 17, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8SIGNED, "%d %d %d %d %d %d %d %d", 8) \
+X( 18, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1UNSIGNED, "%u", 1) \
+X( 19, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2UNSIGNED, "%u %u", 2) \
+X( 20, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3UNSIGNED, "%u %u %u", 3) \
+X( 21, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4UNSIGNED, "%u %u %u %u", 4) \
+X( 22, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5UNSIGNED, "%u %u %u %u %u", 5) \
+X( 23, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6UNSIGNED, "%u %u %u %u %u %u", 6) \
+X( 24, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7UNSIGNED, "%u %u %u %u %u %u %u", 7) \
+X( 25, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8UNSIGNED, "%u %u %u %u %u %u %u %u", 8) \
+\
+X(  1, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_STATE_CHANGE, "[Host time %09u%09u%09u] Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-graceful_offloading 4-forced_offloading 5-cooldown). Status %u (1-ok 0-fail)", 7) \
+X(  2, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_MTS_PERMISSION_CHANGED, "[Host time %09u%09u%09u] Updated permission for Driver ID %u to perform MTS kicks: %u (1 = allowed, 0 = not allowed)", 5) \
+X(  3, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_INIT_CCBS, "[Host time %09u%09u%09u] Driver ID %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x", 9) \
+X(  4, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_INIT_CONFIG, "[Host time %09u%09u%09u] Initialised OS %u with config flags 0x%08x", 5) \
+X(  5, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_MAX, "[Host time %09u%09u%09u] Time Slice: Update Failed, Max total allocations exceeded 100pc", 3)\
+X(  6, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_DYNAMIC_LOW, "[Host time %09u%09u%09u] Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 3)\
+X(  7, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_UPDATE_SUCCESS, "[Host time %09u%09u%09u] Time Slice: Updated Successfully", 3)\
+X(  8, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_ISOLATION_GROUP_CHANGE, "[Host time %09u%09u%09u] Changing Driver ID %u's isolation group from %u to %u", 6) \
+X(  9, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_PRIORITY_CHANGE, "[Host time %09u%09u%09u] Changing Driver ID %u's priority from %u to %u", 6) \
+X( 10, RGXFW_GROUP_VZ, RGXFW_SF_VZ_WDG_TRIGGER, "[Host time %09u%09u%09u] Driver ID %u failed to respond to the virtualisation watchdog in time. Timestamp of its last input = %u", 5) \
+\
+\
+X(  1, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_STATUS, "FBCDC: Core=0x%08x, Status=0x%08x, Signature status=0x%08x", 3) \
+X(  2, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_DETECTED, "FBCDC signature failure detected so block scheduling more work", 0) \
+X(  3, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_CLEARED, "FBCDC signature cleared which unlocks scheduling more work", 0) \
+X(  4, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_FAILURE, "TRP failure: %u", 1) \
+X(  5, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SW_TRP_STATE, "SW TRP State: %u", 1) \
+X(  6, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_HW_KICK, "HW kick event (%u)", 1) \
+X(  7, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_CHECKSUMS, "GPU core (%u/%u): checksum 0x%08x vs. 0x%08x", 4) \
+X(  8, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_UNIT_CHECKSUMS, "GPU core (%u/%u), unit (%u,%u): checksum 0x%08x vs. 0x%08x", 6) \
+X(  9, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_CHECK_REG, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 10, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_USC_SLOTS_CHECK, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X( 11, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_USC_REG_CHECK, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 12, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_STATE_HWRTDATA_DEPRECATED, "TRP HWRTData: 0x%08x, state: %u", 2) \
+X( 13, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_STATE_CNTX_DEPRECATED, "TRP Context: 0x%08x, state: %u", 2) \
+X( 14, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_FAILURE_CNTX, "TRP Context: 0x%08x, failure: %u", 2) \
+X( 15, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_UNEXPECTED_EVENT_DEPRECATED, "Unexpected TRP pass completion on DM: %u",1) \
+X( 16, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_ENABLED, "ICS enabled for DM: %u",1) \
+X( 17, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_INTERVAL,"ICS: interval: %u cycles, threshold: %u cycles",2) \
+X( 18, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_FAULT,"ICS fault detected, GPU cores mask 0x%08x",1) \
+X( 19, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_MODE,"Set ICS mode to: %u ",1) \
+X( 20, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_TESTS_COMPLETION_FAILURE,"ICS tests not completed in mode %u",1) \
+X( 21, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_TESTS_START_FAILURE,"ICS tests failed to start",0) \
+X( 22, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SELF_TEST_START,"Start Safety self-test",0) \
+X( 23, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SELF_TEST_END,"Safety self-test completed",0) \
+\
+X(  1, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_CHECK_REG, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X(  2, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_USC_SLOTS_CHECK, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X(  3, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_USC_REG_CHECK, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X(  4, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU, "Memory dump: Addr=0x%02x%08x, Size=%d, ContextId=%u, DM=%u", 5) \
+X(  5, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS4, "  0x%02x%08x  %08x %08x %08x %08x", 6) \
+X(  6, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS3, "  0x%02x%08x  %08x %08x %08x", 5) \
+X(  7, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS2, "  0x%02x%08x  %08x %08x", 4) \
+X(  8, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS1, "  0x%02x%08x  %08x", 3) \
+X(  9, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW, "Memory dump: Addr=0x%08x, Size=%d", 2) \
+X(  0, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS4, "  0x%08x  %08x %08x %08x %08x", 5) \
+X( 11, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS3, "  0x%08x  %08x %08x %08x", 4) \
+X( 12, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS2, "  0x%08x  %08x %08x", 3) \
+X( 13, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS1, "  0x%08x  %08x", 2) \
+\
+X(65535, RGXFW_GROUP_NULL, RGXFW_SF_LAST, "You should not use this string", 15)
+
+
+/*  The symbolic names found in the table above are assigned an ui32 value of
+ *  the following format:
+ *  31 30 28 26      20   19  16    15  12      11  10           0   bits
+ *  -   ---   --- ----     ----      ----        -   --- ---- ----
+ *        0-10: id number
+ *    12-15,11: group id number (bits 0-3 and bit 4)
+ *       16-19: number of parameters
+ *       20-26: unused
+ *       28-30: active: identify SF packet, otherwise regular int32
+ *          31: reserved for signed/unsigned compatibility
+ *
+ *   The following macro assigns those values to the enum generated SF ids list.
+ */
+#define RGXFW_LOG_IDMARKER			(0x70000000U)
+#define RGXFW_LOG_CREATESFID(a,b,e) ((IMG_UINT32)(a) | (((IMG_UINT32)(b)&0x10U)<<7U) | (((IMG_UINT32)(b)&0xFU)<<12U) | ((IMG_UINT32)(e)<<16U)) | RGXFW_LOG_IDMARKER
+
+#define RGXFW_LOG_IDMASK			(0xFFF00000U)
+#define RGXFW_LOG_VALIDID(I)		(((I) & RGXFW_LOG_IDMASK) == RGXFW_LOG_IDMARKER)
+
+typedef enum {
+#define X(a, b, c, d, e) c = RGXFW_LOG_CREATESFID(a,b,e),
+	RGXFW_LOG_SFIDLIST
+#undef X
+} RGXFW_LOG_SFids;
+
+/* Return the group id number that the given (enum generated) id belongs to */
+#define RGXFW_SF_GID(x) ((((IMG_UINT32)(x)>>12) & 0xfU)|(((IMG_UINT32)(x)>>7) & 0x10U))
+/* Return the id number that the given (enum generated) id belongs to */
+#define RGXFW_SF_ID(x) ((IMG_UINT32)(x) & 0x7ffU)
+/* Returns how many arguments the SF(string format) for the given (enum generated) id requires */
+#define RGXFW_SF_PARAMNUM(x) (((IMG_UINT32)(x)>>16) & 0xfU)
+
+#endif /* RGX_FWIF_SF_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_heap_firmware.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_heap_firmware.h
new file mode 100644
index 000000000000..2628cd82e38e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_heap_firmware.h
@@ -0,0 +1,135 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX FW heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAP_FIRMWARE_H)
+#define RGX_HEAP_FIRMWARE_H
+
+/* Start at 903GiB. Size of 32MB per OSID (see rgxheapconfig.h)
+ * NOTE:
+ *      The firmware heaps bases and sizes are defined here to
+ *      simplify #include dependencies, see rgxheapconfig.h
+ *      for the full RGX virtual address space layout.
+ */
+
+/*
+ * The Config heap holds initialisation data shared between the
+ * the driver and firmware (e.g. pointers to the KCCB and FWCCB).
+ * The Main Firmware heap size is adjusted accordingly but most
+ * of the map / unmap functions must take into consideration
+ * the entire range (i.e. main and config heap).
+ */
+#define RGX_FIRMWARE_NUMBER_OF_FW_HEAPS              (IMG_UINT32_C(2))
+#define RGX_FIRMWARE_HEAP_SHIFT                      RGX_FW_HEAP_SHIFT
+#define RGX_FIRMWARE_RAW_HEAP_BASE                   (IMG_UINT64_C(0xE1C0000000))
+#define RGX_FIRMWARE_RAW_HEAP_SIZE                   (IMG_UINT32_C(1) << RGX_FIRMWARE_HEAP_SHIFT)
+
+/* To enable the firmware to compute the exact address of structures allocated by the KM
+ * in the Fw Config subheap, regardless of the KM's page size (and PMR granularity),
+ * objects allocated consecutively but from different PMRs (due to differing memalloc flags)
+ * are allocated with a 64kb offset. This way, all structures will be located at the same base
+ * addresses when the KM is running with a page size of 4k, 16k or 64k.  */
+#define RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY    (IMG_UINT32_C(0x10000))
+
+/* Ensure the heap can hold 3 PMRs of maximum supported granularity (192KB):
+ * 1st PMR: RGXFWIF_CONNECTION_CTL
+ * 2nd PMR: RGXFWIF_OSINIT
+ * 3rd PMR: RGXFWIF_SYSINIT */
+#define RGX_FIRMWARE_CONFIG_HEAP_SIZE                (IMG_UINT32_C(3)*RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY)
+
+#define RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE          (RGX_FIRMWARE_RAW_HEAP_SIZE - RGX_FIRMWARE_CONFIG_HEAP_SIZE)
+/*
+ * MIPS FW needs space in the Main heap to map GPU memory.
+ * This space is taken from the MAIN heap, to avoid creating a new heap.
+ */
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL       (IMG_UINT32_C(0x100000)) /* 1MB */
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101     (IMG_UINT32_C(0x400000)) /* 4MB */
+
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL      (RGX_FIRMWARE_RAW_HEAP_SIZE -  RGX_FIRMWARE_CONFIG_HEAP_SIZE - \
+                                                           RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL)
+
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101    (RGX_FIRMWARE_RAW_HEAP_SIZE -  RGX_FIRMWARE_CONFIG_HEAP_SIZE - \
+                                                           RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101)
+
+#if !defined(__KERNEL__)
+#if defined(FIX_HW_BRN_65101)
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE      RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE        RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101
+
+#include "img_defs.h"
+static_assert((RGX_FIRMWARE_RAW_HEAP_SIZE) >= IMG_UINT32_C(0x800000), "MIPS GPU map size cannot be increased due to BRN65101 with a small FW heap");
+
+#else
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE      RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE        RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL
+#endif
+#endif /* !defined(__KERNEL__) */
+
+#define RGX_FIRMWARE_MAIN_HEAP_BASE             RGX_FIRMWARE_RAW_HEAP_BASE
+#define RGX_FIRMWARE_CONFIG_HEAP_BASE           (RGX_FIRMWARE_MAIN_HEAP_BASE + \
+                                                 RGX_FIRMWARE_RAW_HEAP_SIZE - \
+                                                 RGX_FIRMWARE_CONFIG_HEAP_SIZE)
+
+
+/* 1 Mb can hold the maximum amount of page tables for the memory shared between the firmware and all KM drivers:
+ *  MAX(RAW_HEAP_SIZE) = 32 Mb; MAX(NUMBER_OS) = 8; Total shared memory = 256 Mb;
+ *  MMU objects required: 65536 PTEs; 16 PDEs; 1 PCE; */
+#define RGX_FIRMWARE_MAX_PAGETABLE_SIZE (1 * 1024 * 1024)
+
+#define RGX_FW_CONFIG_HEAP_SIZE (1 << RGX_FW_HEAP_SHIFT)
+#define RGX_FW_MAX_HEAP_SIZE (1 << 28)
+/*
+ * The maximum configurable size via RGX_FW_HEAP_SHIFT is 256MiB (1<<28) and
+ * the minimum is 4MiB (1<<22); the firmware heap size is dependent
+ * on the number of drivers supported.
+ */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	#if defined(RGX_FW_HEAP_SHIFT) && (RGX_FW_HEAP_SHIFT < 22 || (RGX_FW_CONFIG_HEAP_SIZE > (RGX_FW_MAX_HEAP_SIZE / RGX_NUM_DRIVERS_SUPPORTED)))
+	#error "RGX_FW_HEAP_SHIFT is outside valid range"
+	#endif
+#else
+	#if defined(RGX_FW_HEAP_SHIFT) && (RGX_FW_HEAP_SHIFT < 22 || RGX_FW_HEAP_SHIFT > 28)
+	#error "RGX_FW_HEAP_SHIFT is outside valid range [22,28]"
+	#endif
+#endif
+
+
+#endif /* RGX_HEAP_FIRMWARE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_hwperf_common.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_hwperf_common.h
new file mode 100644
index 000000000000..070bfede23d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_hwperf_common.h
@@ -0,0 +1,1645 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_COMMON_H_
+#define RGX_HWPERF_COMMON_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_common_asserts.h"
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_sync_km.h"
+
+
+/******************************************************************************
+ * Packet Event Type Enumerations
+ *****************************************************************************/
+
+/*! Type used to encode the event that generated the packet.
+ * NOTE: When this type is updated the corresponding hwperfbin2json tool
+ * source needs to be updated as well. The RGX_HWPERF_EVENT_MASK_* macros will
+ * also need updating when adding new types.
+ *
+ * @par
+ * The event type values are incrementing integers for use as a shift ordinal
+ * in the event filtering process at the point events are generated.
+ * This scheme thus implies a limit of 63 event types.
+ */
+
+typedef IMG_UINT32 RGX_HWPERF_EVENT_TYPE;
+
+#define RGX_HWPERF_INVALID				0x00U /*!< Invalid. Reserved value. */
+
+/*! FW types 0x01..0x06 */
+#define RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE	0x01U
+
+#define RGX_HWPERF_FW_BGSTART			0x01U /*!< Background task processing start */
+#define RGX_HWPERF_FW_BGEND				0x02U /*!< Background task end */
+#define RGX_HWPERF_FW_IRQSTART			0x03U /*!< IRQ task processing start */
+
+#define RGX_HWPERF_FW_IRQEND			0x04U /*!< IRQ task end */
+#define RGX_HWPERF_FW_DBGSTART			0x05U /*!< Debug event start */
+#define RGX_HWPERF_FW_DBGEND			0x06U /*!< Debug event end */
+
+#define RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE		0x06U
+
+/*! HW types 0x07..0x19 */
+#define RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE	0x07U
+
+#define RGX_HWPERF_HW_PMOOM_TAPAUSE		0x07U /*!< TA Pause at PM Out of Memory */
+
+#define RGX_HWPERF_HW_TAKICK			0x08U /*!< TA task started */
+#define RGX_HWPERF_HW_TAFINISHED		0x09U /*!< TA task finished */
+#define RGX_HWPERF_HW_3DTQKICK			0x0AU /*!< 3D TQ started */
+#define RGX_HWPERF_HW_3DKICK			0x0BU /*!< 3D task started */
+#define RGX_HWPERF_HW_3DFINISHED		0x0CU /*!< 3D task finished */
+#define RGX_HWPERF_HW_CDMKICK			0x0DU /*!< CDM task started */
+#define RGX_HWPERF_HW_CDMFINISHED		0x0EU /*!< CDM task finished */
+#define RGX_HWPERF_HW_TLAKICK			0x0FU /*!< TLA task started */
+#define RGX_HWPERF_HW_TLAFINISHED		0x10U /*!< TLS task finished */
+#define RGX_HWPERF_HW_3DSPMKICK			0x11U /*!< 3D SPM task started */
+#define RGX_HWPERF_HW_PERIODIC			0x12U /*!< Periodic event with updated HW counters */
+#define RGX_HWPERF_HW_RTUKICK			0x13U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_RTUFINISHED		0x14U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_SHGKICK			0x15U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_SHGFINISHED		0x16U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_3DTQFINISHED		0x17U /*!< 3D TQ finished */
+#define RGX_HWPERF_HW_3DSPMFINISHED		0x18U /*!< 3D SPM task finished */
+
+#define RGX_HWPERF_HW_PMOOM_TARESUME	0x19U /*!< TA Resume after PM Out of Memory */
+
+/*! HW_EVENT_RANGE0 used up. Use next empty range below to add new hardware events */
+#define RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE	0x19U
+
+/*! other types 0x1A..0x1F */
+#define RGX_HWPERF_CLKS_CHG				0x1AU /*!< Clock speed change in GPU */
+#define RGX_HWPERF_GPU_STATE_CHG		0x1BU /*!< GPU work state change */
+
+/*! power types 0x20..0x27 */
+#define RGX_HWPERF_PWR_EST_RANGE_FIRST_TYPE	0x20U
+#define RGX_HWPERF_PWR_EST_REQUEST		0x20U /*!< Power estimate requested (via GPIO) */
+#define RGX_HWPERF_PWR_EST_READY		0x21U /*!< Power estimate inputs ready */
+#define RGX_HWPERF_PWR_EST_RESULT		0x22U /*!< Power estimate result calculated */
+#define RGX_HWPERF_PWR_EST_RANGE_LAST_TYPE	0x22U
+
+#define RGX_HWPERF_PWR_CHG				0x23U /*!< Power state change */
+
+/*! HW_EVENT_RANGE1 0x28..0x2F, for accommodating new hardware events */
+#define RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE	0x28U
+
+#define RGX_HWPERF_HW_TDMKICK			0x28U /*!< TDM task started */
+#define RGX_HWPERF_HW_TDMFINISHED		0x29U /*!< TDM task finished */
+#define RGX_HWPERF_HW_NULLKICK			0x2AU /*!< NULL event */
+
+#define RGX_HWPERF_HW_EVENT_RANGE1_LAST_TYPE 0x2AU
+#define RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD		0x2BU
+
+/*! context switch types 0x30..0x31 */
+#define RGX_HWPERF_CSW_START			0x30U /*!< HW context store started */
+#define RGX_HWPERF_CSW_FINISHED			0x31U /*!< HW context store finished */
+
+/*! DVFS events */
+#define RGX_HWPERF_DVFS					0x32U /*!< Dynamic voltage/frequency scaling events */
+
+/*! firmware misc 0x38..0x39 */
+#define RGX_HWPERF_UFO					0x38U /*!< FW UFO Check / Update */
+#define RGX_HWPERF_FWACT				0x39U /*!< FW Activity notification */
+
+/*! last */
+#define RGX_HWPERF_LAST_TYPE			0x3BU
+
+/*! This enumeration must have a value that is a power of two as it is
+ * used in masks and a filter bit field (currently 64 bits long).
+ */
+#define RGX_HWPERF_MAX_TYPE				0x40U
+
+static_assert(RGX_HWPERF_LAST_TYPE < RGX_HWPERF_MAX_TYPE, "Too many HWPerf event types");
+
+/*! Macro used to check if an event type ID is present in the known set of hardware type events */
+#define HWPERF_PACKET_IS_HW_TYPE(_etype)	(((_etype) >= RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE && (_etype) <= RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE) || \
+											 ((_etype) >= RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE && (_etype) <= RGX_HWPERF_HW_EVENT_RANGE1_LAST_TYPE))
+
+/*! Macro used to check if an event type ID is present in the known set of firmware type events */
+#define HWPERF_PACKET_IS_FW_TYPE(_etype)					\
+	((_etype) >= RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE &&	\
+	 (_etype) <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE)
+
+
+typedef enum {
+	RGX_HWPERF_HOST_INVALID   = 0x00,           /*!< Invalid, do not use. */
+	RGX_HWPERF_HOST_ENQ       = 0x01,           /*!< ``0x01`` Kernel driver has queued GPU work.
+	                                             See RGX_HWPERF_HOST_ENQ_DATA */
+	RGX_HWPERF_HOST_UFO       = 0x02,           /*!< ``0x02`` UFO updated by the driver.
+	                                             See RGX_HWPERF_HOST_UFO_DATA */
+	RGX_HWPERF_HOST_ALLOC     = 0x03,           /*!< ``0x03`` Resource allocated.
+	                                             See RGX_HWPERF_HOST_ALLOC_DATA */
+	RGX_HWPERF_HOST_CLK_SYNC  = 0x04,           /*!< ``0x04`` GPU / Host clocks correlation data.
+	                                             See RGX_HWPERF_HOST_CLK_SYNC_DATA */
+	RGX_HWPERF_HOST_FREE      = 0x05,           /*!< ``0x05`` Resource freed,
+	                                             See RGX_HWPERF_HOST_FREE_DATA */
+	RGX_HWPERF_HOST_MODIFY    = 0x06,           /*!< ``0x06`` Resource modified / updated.
+	                                             See RGX_HWPERF_HOST_MODIFY_DATA */
+	RGX_HWPERF_HOST_DEV_INFO  = 0x07,           /*!< ``0x07`` Device Health status.
+	                                             See RGX_HWPERF_HOST_DEV_INFO_DATA */
+	RGX_HWPERF_HOST_INFO      = 0x08,           /*!< ``0x08`` Device memory usage information.
+	                                             See RGX_HWPERF_HOST_INFO_DATA */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT = 0x09,     /*!< ``0x09`` Wait for sync event.
+	                                             See RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA */
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE  = 0x0A, /*!< ``0x0A`` Software timeline advanced.
+	                                             See RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA */
+	RGX_HWPERF_HOST_CLIENT_INFO = 0x0B,			/*!< ``0x0B`` Additional client info.
+	                                             See RGX_HWPERF_HOST_CLIENT_INFO_DATA */
+
+	/*! last */
+	RGX_HWPERF_HOST_LAST_TYPE,
+
+	/*! This enumeration must have a value that is a power of two as it is
+	 * used in masks and a filter bit field (currently 32 bits long).
+	 */
+	RGX_HWPERF_HOST_MAX_TYPE  = 0x20
+} RGX_HWPERF_HOST_EVENT_TYPE;
+
+/*!< The event type values are incrementing integers for use as a shift ordinal
+ * in the event filtering process at the point events are generated.
+ * This scheme thus implies a limit of 31 event types.
+ */
+static_assert(RGX_HWPERF_HOST_LAST_TYPE < RGX_HWPERF_HOST_MAX_TYPE, "Too many HWPerf host event types");
+
+/*! Define containing bit position for 32bit feature flags used in hwperf and api */
+typedef IMG_UINT32 RGX_HWPERF_FEATURE_FLAGS;
+#define RGX_HWPERF_FEATURE_PERFBUS_FLAG                    0x00000001U
+#define RGX_HWPERF_FEATURE_S7_TOP_INFRASTRUCTURE_FLAG      0x00000002U
+#define RGX_HWPERF_FEATURE_XT_TOP_INFRASTRUCTURE_FLAG      0x00000004U
+#define RGX_HWPERF_FEATURE_PERF_COUNTER_BATCH_FLAG         0x00000008U
+#define RGX_HWPERF_FEATURE_ROGUEXE_FLAG                    0x00000010U
+#define RGX_HWPERF_FEATURE_DUST_POWER_ISLAND_S7_FLAG       0x00000020U
+#define RGX_HWPERF_FEATURE_PBE2_IN_XE_FLAG                 0x00000040U
+#define RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION             0x00000080U
+#define RGX_HWPERF_FEATURE_MULTICORE_FLAG                  0x00000100U
+#define RGX_HWPERF_FEATURE_RAYTRACING_FLAG                 0x00000200U
+#define RGX_HWPERF_FEATURE_CXT_TOP_INFRASTRUCTURE_FLAG     0x00000400U
+#define RGX_HWPERF_FEATURE_VOLCANIC_FLAG                   0x00000800U
+#define RGX_HWPERF_FEATURE_ROGUE_FLAG                      0x00001000U
+#define RGX_HWPERF_FEATURE_RESERVED1_FLAG                  0x00002000U
+#define RGX_HWPERF_FEATURE_CXT_XTP_TOP_INFRASTRUCTURE_FLAG 0x00004000U
+#define RGX_HWPERF_FEATURE_AX_TOP_INFRASTRUCTURE_FLAG      0x00008000U
+#define RGX_HWPERF_FEATURE_BX_TOP_INFRASTRUCTURE_FLAG      0x00010000U
+#define RGX_HWPERF_FEATURE_DX_TOP_INFRASTRUCTURE_FLAG      0x00020000U
+
+/* ! Define for RGX_HWPERF_DM type. The values are architecture specific */
+typedef IMG_UINT32 RGX_HWPERF_DM;
+
+/******************************************************************************
+ * Packet Header Format Version 2 Types
+ *****************************************************************************/
+
+/*! Major version number of the protocol in operation
+ */
+#define RGX_HWPERF_V2_FORMAT 2
+
+/*! Signature ASCII pattern 'HWP2' found in the first word of a HWPerfV2 packet
+ */
+#define HWPERF_PACKET_V2_SIG		0x48575032
+
+/*! Signature ASCII pattern 'HWPA' found in the first word of a HWPerfV2a packet
+ */
+#define HWPERF_PACKET_V2A_SIG		0x48575041
+
+/*! Signature ASCII pattern 'HWPB' found in the first word of a HWPerfV2b packet
+ */
+#define HWPERF_PACKET_V2B_SIG		0x48575042
+
+/*! Signature ASCII pattern 'HWPC' found in the first word of a HWPerfV2c packet
+ */
+#define HWPERF_PACKET_V2C_SIG		0x48575043
+
+#define HWPERF_PACKET_ISVALID(_val) (((_val) == HWPERF_PACKET_V2_SIG) || ((_val) == HWPERF_PACKET_V2A_SIG) || ((_val) == HWPERF_PACKET_V2B_SIG) || ((_val) == HWPERF_PACKET_V2C_SIG))
+/*!< Checks that the packet signature is one of the supported versions */
+
+/*! Type defines the HWPerf packet header common to all events. */
+typedef struct
+{
+	IMG_UINT32  ui32Sig;        /*!< Always the value HWPERF_PACKET_SIG */
+	IMG_UINT32  ui32Size;       /*!< Overall packet size in bytes */
+	IMG_UINT32  eTypeId;        /*!< Event type information field */
+	IMG_UINT32  ui32Ordinal;    /*!< Sequential number of the packet */
+	IMG_UINT64  ui64Timestamp;  /*!< Event timestamp */
+} RGX_HWPERF_V2_PACKET_HDR, *RGX_PHWPERF_V2_PACKET_HDR;
+
+RGX_FW_STRUCT_OFFSET_ASSERT(RGX_HWPERF_V2_PACKET_HDR, ui64Timestamp);
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_V2_PACKET_HDR);
+
+
+/*! Mask for use with the IMG_UINT32 ui32Size header field */
+#define RGX_HWPERF_SIZE_MASK         0xFFFFU
+
+/*! This macro defines an upper limit to which the size of the largest variable
+ * length HWPerf packet must fall within, currently 3KB. This constant may be
+ * used to allocate a buffer to hold one packet.
+ * This upper limit is policed by packet producing code.
+ */
+#define RGX_HWPERF_MAX_PACKET_SIZE   0xC00U
+
+/*! Defines an upper limit to the size of a variable length packet payload.
+ */
+#define RGX_HWPERF_MAX_PAYLOAD_SIZE	 ((IMG_UINT32)(RGX_HWPERF_MAX_PACKET_SIZE-\
+	sizeof(RGX_HWPERF_V2_PACKET_HDR)))
+
+/*! Macro which takes a structure name and provides the packet size for
+ * a fixed size payload packet, rounded up to 8 bytes to align packets
+ * for 64 bit architectures. */
+#define RGX_HWPERF_MAKE_SIZE_FIXED(_struct)       ((IMG_UINT32)(RGX_HWPERF_SIZE_MASK&(sizeof(RGX_HWPERF_V2_PACKET_HDR)+PVR_ALIGN(sizeof(_struct), PVRSRVTL_PACKET_ALIGNMENT))))
+
+/*! Macro which takes the number of bytes written in the data payload of a
+ * packet for a variable size payload packet, rounded up to 8 bytes to
+ * align packets for 64 bit architectures. */
+#define RGX_HWPERF_MAKE_SIZE_VARIABLE(_size)      ((IMG_UINT32)(RGX_HWPERF_SIZE_MASK&((IMG_UINT32)sizeof(RGX_HWPERF_V2_PACKET_HDR)+PVR_ALIGN((_size), PVRSRVTL_PACKET_ALIGNMENT))))
+
+/*! Macro to obtain the size of the packet */
+#define RGX_HWPERF_GET_SIZE(_packet_addr)         ((IMG_UINT16)(((_packet_addr)->ui32Size) & RGX_HWPERF_SIZE_MASK))
+
+/*! Macro to obtain the size of the packet data */
+#define RGX_HWPERF_GET_DATA_SIZE(_packet_addr)    (RGX_HWPERF_GET_SIZE(_packet_addr) - sizeof(RGX_HWPERF_V2_PACKET_HDR))
+
+/*! Masks for use with the IMG_UINT32 eTypeId header field */
+#define RGX_HWPERF_TYPEID_MASK			0x0007FFFFU
+#define RGX_HWPERF_TYPEID_EVENT_MASK	0x00007FFFU
+#define RGX_HWPERF_TYPEID_THREAD_MASK	0x00008000U
+#define RGX_HWPERF_TYPEID_STREAM_MASK	0x00070000U
+#define RGX_HWPERF_TYPEID_META_DMA_MASK	0x00080000U
+#define RGX_HWPERF_TYPEID_M_CORE_MASK	0x00100000U
+#define RGX_HWPERF_TYPEID_PIPEDM_MASK	0x00200000U
+#define RGX_HWPERF_TYPEID_OSID_MASK		0x1f000000U
+
+/*! Meta thread macros for encoding the ID into the type field of a packet */
+#define RGX_HWPERF_META_THREAD_SHIFT	15U
+#define RGX_HWPERF_META_THREAD_ID0		0x0U  /*!< Meta Thread 0 ID */
+#define RGX_HWPERF_META_THREAD_ID1		0x1U  /*!< Meta Thread 1 ID */
+/*! Obsolete, kept for source compatibility */
+#define RGX_HWPERF_META_THREAD_MASK		0x1U
+/*! Stream ID macros for encoding the ID into the type field of a packet */
+#define RGX_HWPERF_STREAM_SHIFT			16U
+/*! Meta DMA macro for encoding how the packet was generated into the type field of a packet */
+#define RGX_HWPERF_META_DMA_SHIFT		19U
+/*! Bit-shift macro used for encoding multi-core data into the type field of a packet */
+#define RGX_HWPERF_M_CORE_SHIFT			20U
+/*! Bit-shift macro used for encoding Pipeline DM data into the type field of a packet */
+#define RGX_HWPERF_PIPEDM_SHIFT			21U
+/*! OSID bit-shift macro used for encoding OSID into type field of a packet */
+#define RGX_HWPERF_OSID_SHIFT			24U
+
+/*! HWPerf Stream ID type definition. Maximum of 32bits. */
+typedef IMG_UINT32 RGX_HWPERF_STREAM_ID;
+/*! Events from the Firmware/GPU */
+#define RGX_HWPERF_STREAM_ID0_FW		0U
+/*! Events from the Server host driver component */
+#define RGX_HWPERF_STREAM_ID1_HOST		1U
+/*! Events from the Client host driver component */
+#define RGX_HWPERF_STREAM_ID2_CLIENT	2U
+#define RGX_HWPERF_STREAM_ID_LAST		3U
+
+/* Checks if all stream IDs can fit under RGX_HWPERF_TYPEID_STREAM_MASK. */
+static_assert(((IMG_UINT32)RGX_HWPERF_STREAM_ID_LAST - 1U) < (RGX_HWPERF_TYPEID_STREAM_MASK >> RGX_HWPERF_STREAM_SHIFT),
+		"Too many HWPerf stream IDs.");
+
+/*! Compile-time value used to seed the Multi-Core (MC) bit in the typeID field.
+ *  Only set by RGX_FIRMWARE builds.
+ */
+#if defined(RGX_FIRMWARE)
+# if defined(RGX_FEATURE_GPU_MULTICORE_SUPPORT)
+#define RGX_HWPERF_M_CORE_VALUE 1U  /*!< 1 => Multi-core supported */
+# else
+#define RGX_HWPERF_M_CORE_VALUE 0U  /*!< 0 => Multi-core not supported */
+# endif
+# if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION) && (RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION > 0)
+#define RGX_HWPERF_PIPEDM_VALUE 1U  /*!< 1 => Pipeline DM supported */
+# else
+#define RGX_HWPERF_PIPEDM_VALUE 0U  /*!< 0 => Pipeline DM not supported */
+# endif
+#else
+#define RGX_HWPERF_M_CORE_VALUE 0U  /*!< 0 => Multi-core not supported */
+#define RGX_HWPERF_PIPEDM_VALUE 0U  /*!< 0 => Pipeline DM not supported */
+#endif
+
+/*! Macros used to set the packet type and encode meta thread ID (0|1),
+ * HWPerf stream ID, multi-core capability and OSID within the typeID */
+#define RGX_HWPERF_MAKE_TYPEID(_stream, _type, _thread, _metadma, _osid)\
+		((IMG_UINT32) ((RGX_HWPERF_TYPEID_STREAM_MASK&((IMG_UINT32)(_stream) << RGX_HWPERF_STREAM_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_THREAD_MASK & ((IMG_UINT32)(_thread) << RGX_HWPERF_META_THREAD_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_EVENT_MASK & (IMG_UINT32)(_type)) | \
+		(RGX_HWPERF_TYPEID_META_DMA_MASK & ((IMG_UINT32)(_metadma) << RGX_HWPERF_META_DMA_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_OSID_MASK & ((IMG_UINT32)(_osid) << RGX_HWPERF_OSID_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_M_CORE_MASK & ((IMG_UINT32)(RGX_HWPERF_M_CORE_VALUE) << RGX_HWPERF_M_CORE_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_PIPEDM_MASK & ((IMG_UINT32)(RGX_HWPERF_PIPEDM_VALUE) << RGX_HWPERF_PIPEDM_SHIFT))))
+
+/*! Obtains the event type that generated the packet */
+#define RGX_HWPERF_GET_TYPE(_packet_addr)            (((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_EVENT_MASK)
+
+/*! Obtains the META Thread number that generated the packet */
+#define RGX_HWPERF_GET_THREAD_ID(_packet_addr)       (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_THREAD_MASK) >> RGX_HWPERF_META_THREAD_SHIFT))
+
+/*! Determines if the packet generated contains multi-core data */
+#define RGX_HWPERF_GET_M_CORE(_packet_addr)          (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_M_CORE_MASK) >> RGX_HWPERF_M_CORE_SHIFT)
+
+/*! Determines if the packet generated contains multi-core data */
+#define RGX_HWPERF_GET_PIPEDM(_packet_addr)          (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_PIPEDM_MASK) >> RGX_HWPERF_PIPEDM_SHIFT)
+
+/*! Obtains the guest OSID which resulted in packet generation */
+#define RGX_HWPERF_GET_OSID(_packet_addr)            (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_OSID_MASK) >> RGX_HWPERF_OSID_SHIFT)
+
+/*! Obtain stream id */
+#define RGX_HWPERF_GET_STREAM_ID(_packet_addr)       (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_STREAM_MASK) >> RGX_HWPERF_STREAM_SHIFT))
+
+/*! Obtain information about how the packet was generated, which might affect payload total size */
+#define RGX_HWPERF_GET_META_DMA_INFO(_packet_addr)   (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_META_DMA_MASK) >> RGX_HWPERF_META_DMA_SHIFT))
+
+/*! Obtains a typed pointer to a packet given a buffer address */
+#define RGX_HWPERF_GET_PACKET(_buffer_addr)            ((RGX_HWPERF_V2_PACKET_HDR *)(void *)  (_buffer_addr))
+/*! Obtains a typed pointer to a data structure given a packet address */
+#define RGX_HWPERF_GET_PACKET_DATA_BYTES(_packet_addr) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR)))
+/*! Obtains a typed pointer to the next packet given a packet address */
+#define RGX_HWPERF_GET_NEXT_PACKET(_packet_addr)       ((RGX_HWPERF_V2_PACKET_HDR *)  (IMG_OFFSET_ADDR((_packet_addr), RGX_HWPERF_SIZE_MASK&((_packet_addr)->ui32Size))))
+
+/*! Obtains a typed pointer to a packet header given the packet data address */
+#define RGX_HWPERF_GET_PACKET_HEADER(_packet_addr)     ((RGX_HWPERF_V2_PACKET_HDR *)  (IMG_OFFSET_ADDR((_packet_addr), -(IMG_INT32)sizeof(RGX_HWPERF_V2_PACKET_HDR))))
+
+
+/******************************************************************************
+ * Other Common Defines
+ *****************************************************************************/
+
+/*! Used to align structs using a flexible array member as the definition of the member
+ * size can change between C standards */
+#if (IMG_FLEX_ARRAY_MEMBER + 0)
+#define RGX_HWPERF_FLEX_ARRAY_ALIGN(_align_var) _align_var
+#else
+#define RGX_HWPERF_FLEX_ARRAY_ALIGN(...)
+#endif
+
+/*! Masks for use with the IMG_UINT32 ui32BlkInfo field */
+#define RGX_HWPERF_BLKINFO_BLKCOUNT_MASK	0xFFFF0000U
+#define RGX_HWPERF_BLKINFO_BLKOFFSET_MASK	0x0000FFFFU
+
+/*! Shift for the NumBlocks and counter block offset field in ui32BlkInfo */
+#define RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT	16U
+#define RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT 0U
+
+/*! Macro used to set the block info word as a combination of two 16-bit integers */
+#define RGX_HWPERF_MAKE_BLKINFO(_numblks, _blkoffset) ((IMG_UINT32) ((RGX_HWPERF_BLKINFO_BLKCOUNT_MASK&((_numblks) << RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT)) | (RGX_HWPERF_BLKINFO_BLKOFFSET_MASK&((_blkoffset) << RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT))))
+
+/*! Macro used to obtain the number of counter blocks present in the packet */
+#define RGX_HWPERF_GET_BLKCOUNT(_blkinfo)            (((_blkinfo) & RGX_HWPERF_BLKINFO_BLKCOUNT_MASK) >> RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT)
+
+/*! Obtains the offset of the counter block stream in the packet */
+#define RGX_HWPERF_GET_BLKOFFSET(_blkinfo)           (((_blkinfo) & RGX_HWPERF_BLKINFO_BLKOFFSET_MASK) >> RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT)
+
+/*! This macro gets the number of blocks depending on the packet version */
+#define RGX_HWPERF_GET_NUMBLKS(_sig, _packet_data, _numblocks) \
+	do { \
+		if (HWPERF_PACKET_V2B_SIG == (_sig) || HWPERF_PACKET_V2C_SIG == (_sig)) \
+		{ \
+			(_numblocks) = RGX_HWPERF_GET_BLKCOUNT((_packet_data)->ui32BlkInfo);\
+		} \
+		else \
+		{ \
+			IMG_UINT32 ui32VersionOffset = (((_sig) == HWPERF_PACKET_V2_SIG) ? 1 : 3);\
+			(_numblocks) = *(IMG_UINT16 *)(IMG_OFFSET_ADDR(&(_packet_data)->ui32WorkTarget, ui32VersionOffset)); \
+		} \
+	} while (0)
+
+/*! This macro gets the counter stream pointer depending on the packet version */
+#define RGX_HWPERF_GET_CNTSTRM(_sig, _hw_packet_data, _cntstream_ptr) \
+{ \
+	if (HWPERF_PACKET_V2B_SIG == (_sig) || HWPERF_PACKET_V2C_SIG == (_sig)) \
+	{ \
+		(_cntstream_ptr) = (IMG_UINT32 *)(IMG_OFFSET_ADDR((_hw_packet_data), RGX_HWPERF_GET_BLKOFFSET((_hw_packet_data)->ui32BlkInfo))); \
+	} \
+	else \
+	{ \
+		IMG_UINT32 ui32BlkStreamOffsetInWords = (((_sig) == HWPERF_PACKET_V2_SIG) ? 6 : 8); \
+		(_cntstream_ptr) = (IMG_UINT32 *)(IMG_OFFSET_ADDR_DW((_hw_packet_data), ui32BlkStreamOffsetInWords)); \
+	} \
+}
+
+/*! Masks for use with the IMG_UINT32 ui32KickInfo field */
+#define RGX_HWPERF_KICKINFO_STARTBE_MASK	0xFFFFFF00U
+#define RGX_HWPERF_KICKINFO_KICKID_MASK		0x000000FFU
+
+/*! Shift for the Kick ID field in ui32KickInfo */
+#define RGX_HWPERF_KICKINFO_STARTBE_SHIFT 8U
+#define RGX_HWPERF_KICKINFO_KICKID_SHIFT  0U
+
+/*! Macro used to set the kick info field. */
+#define RGX_HWPERF_MAKE_KICKINFO(_startbe, _kickid) \
+        ((IMG_UINT32) (RGX_HWPERF_KICKINFO_STARTBE_MASK&((_startbe) << RGX_HWPERF_KICKINFO_STARTBE_SHIFT))| \
+                      (RGX_HWPERF_KICKINFO_KICKID_MASK&((_kickid) << RGX_HWPERF_KICKINFO_KICKID_SHIFT)))
+
+/*! Macro used to obtain the lowest 24 bits of START_BE if present in the packet */
+#define RGX_HWPERF_GET_STARTBE(_kickinfo)           (((_kickinfo) & RGX_HWPERF_KICKINFO_STARTBE_MASK) >> RGX_HWPERF_KICKINFO_STARTBE_SHIFT)
+
+/*! Macro used to obtain the Kick ID if present in the packet */
+#define RGX_HWPERF_GET_KICKID(_kickinfo)            (((_kickinfo) & RGX_HWPERF_KICKINFO_KICKID_MASK) >> RGX_HWPERF_KICKINFO_KICKID_SHIFT)
+
+/*! Masks for use with the RGX_HWPERF_UFO_EV eEvType field */
+#define RGX_HWPERF_UFO_STREAMSIZE_MASK 0xFFFF0000U
+#define RGX_HWPERF_UFO_STREAMOFFSET_MASK 0x0000FFFFU
+
+/*! Shift for the UFO count and data stream fields */
+#define RGX_HWPERF_UFO_STREAMSIZE_SHIFT 16U
+#define RGX_HWPERF_UFO_STREAMOFFSET_SHIFT 0U
+
+/*! Macro used to set UFO stream info word as a combination of two 16-bit integers */
+#define RGX_HWPERF_MAKE_UFOPKTINFO(_ssize, _soff) \
+        ((IMG_UINT32) ((RGX_HWPERF_UFO_STREAMSIZE_MASK&((_ssize) << RGX_HWPERF_UFO_STREAMSIZE_SHIFT)) | \
+        (RGX_HWPERF_UFO_STREAMOFFSET_MASK&((_soff) << RGX_HWPERF_UFO_STREAMOFFSET_SHIFT))))
+
+/*! Macro used to obtain UFO count*/
+#define RGX_HWPERF_GET_UFO_STREAMSIZE(_streaminfo) \
+        (((_streaminfo) & RGX_HWPERF_UFO_STREAMSIZE_MASK) >> RGX_HWPERF_UFO_STREAMSIZE_SHIFT)
+
+/*! Obtains the offset of the UFO stream in the packet */
+#define RGX_HWPERF_GET_UFO_STREAMOFFSET(_streaminfo) \
+        (((_streaminfo) & RGX_HWPERF_UFO_STREAMOFFSET_MASK) >> RGX_HWPERF_UFO_STREAMOFFSET_SHIFT)
+
+
+/*! This structure holds the data of a firmware packet. */
+typedef struct
+{
+	RGX_HWPERF_DM eDM;             /*!< DataMaster identifier, see RGX_HWPERF_DM */
+	IMG_UINT32 ui32TxtActCyc;      /*!< Meta TXTACTCYC register value */
+	IMG_UINT32 ui32FWPerfCount0;   /*!< Meta/MIPS PERF_COUNT0 register */
+	IMG_UINT32 ui32FWPerfCount1;   /*!< Meta/MIPS PERF_COUNT1 register */
+	IMG_UINT32 ui32TimeCorrIndex;  /*!< Internal field */
+	IMG_UINT32 ui32Padding;        /*!< Reserved */
+} RGX_HWPERF_FW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FW_DATA);
+
+
+/*! This structure holds the data of a gpu work period packet. */
+typedef struct
+{
+	IMG_UINT64 ui64GPUWorkPeriodStartTime;
+	IMG_UINT32 ui32UID;               /*!< Process identifier */
+	IMG_UINT32 ui32TimeCorrIndex;     /*!< Index to the time correlation at the beginning of gpu work period */
+	IMG_UINT32 ui32StartTimeCorrIndex;     /*!< Index to the time correlation at the time the packet was generated */
+	IMG_UINT32 ui32Padding2;          /*!< Reserved. To ensure correct alignment (not written in the packet) */
+} RGX_HWPERF_GPU_WORK_PERIOD_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_GPU_WORK_PERIOD_DATA);
+
+/*! This structure holds the data of a hardware packet, including counters. */
+typedef struct
+{
+	union
+	{
+		IMG_UINT32 ui32DMCyc;         /*!< DataMaster cycle count register, 0 if none */
+		IMG_UINT32 ui32KickStartTime; /*!< Front End start time for Pipeline DMs */
+	};
+	IMG_UINT32 ui32FrameNum;          /*!< Frame number, undefined on some DataMasters */
+	IMG_UINT32 ui32PID;               /*!< Process identifier */
+	IMG_UINT32 ui32DMContext;         /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32WorkTarget;        /*!< RenderTarget for a TA,3D; Frame context for RTU, 0x0 otherwise */
+	IMG_UINT32 ui32ExtJobRef;         /*!< Client driver context job reference used for tracking/debugging */
+	IMG_UINT32 ui32IntJobRef;         /*!< RGX Data master context job reference used for tracking/debugging */
+	IMG_UINT32 ui32TimeCorrIndex;     /*!< Index to the time correlation at the time the packet was generated */
+	IMG_UINT32 ui32BlkInfo;           /*!< <31..16> NumBlocks <15..0> Counter block stream offset */
+	IMG_UINT32 ui32WorkCtx;           /*!< Work context: Render Context for TA/3D; RayTracing Context for RTU/SHG; 0x0 otherwise */
+	IMG_UINT32 ui32CtxPriority;       /*!< Context priority */
+	IMG_UINT32 ui32GPUIdMask;         /*!< GPU IDs active within this event */
+	IMG_UINT32 ui32KickInfo;          /*!< <31..8> Back End start time lowest 24 bits <7..0> GPU Pipeline DM kick ID, 0 if not using Pipeline DMs */
+	IMG_UINT32 ui32KickEndTime;       /*!< Back End finish time for Pipeline DMs */
+	IMG_UINT32 aui32CountBlksStream[IMG_FLEX_ARRAY_MEMBER]; /*!< Optional variable length Counter data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);          /*!< Reserved. To ensure correct alignment (not written in the packet) */
+} RGX_HWPERF_HW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_HW_DATA);
+RGX_FW_STRUCT_OFFSET_ASSERT(RGX_HWPERF_HW_DATA, aui32CountBlksStream);
+
+/*! Mask for use with the aui32CountBlksStream field when decoding the
+ * counter block ID and mask word. */
+#define RGX_HWPERF_CNTBLK_ID_MASK	0xFFFF0000U
+#define RGX_HWPERF_CNTBLK_ID_SHIFT	16U
+
+/*! MAX value used in server handling of counter config arrays */
+#define RGX_CNTBLK_COUNTERS_MAX PVRSRV_HWPERF_COUNTERS_PERBLK
+
+/*! Obtains the counter block ID word from an aui32CountBlksStream field.
+ * The word combines Control bits (15-12), GPU-Id (11-8), Group (7-4), Unit
+ * within group (3-0) */
+#define RGX_HWPERF_GET_CNTBLK_IDW(_word)           ((IMG_UINT16)(((_word)&RGX_HWPERF_CNTBLK_ID_MASK)>>RGX_HWPERF_CNTBLK_ID_SHIFT))
+
+/*! Obtains the counter block ID from the supplied RGX_HWPERF_HW_DATA address
+ * and stream index. May be used in decoding the counter block stream words of
+ * a RGX_HWPERF_HW_DATA structure. */
+#define RGX_HWPERF_GET_CNTBLK_ID(_data_addr, _idx) RGX_HWPERF_GET_CNTBLK_IDW((_data_addr)->aui32CountBlksStream[(_idx)])
+
+/*! Obtains the GPU ID from the supplied RGX_HWPERF_HW_DATA CNTBLK_IDW */
+#define RGX_HWPERF_GET_CNTBLK_GPUW(_word)          ((IMG_UINT16)(((_word)&RGX_CNTBLK_ID_MC_GPU_MASK)>>RGX_CNTBLK_ID_MC_GPU_SHIFT))
+
+#define RGX_HWPERF_GET_CNT_MASKW(_word)           ((IMG_UINT16)((_word)&(~RGX_HWPERF_CNTBLK_ID_MASK)))
+
+/*! Obtains the counter mask from the supplied RGX_HWPERF_HW_DATA address
+ * and stream index. May be used in decoding the counter block stream words
+ * of a RGX_HWPERF_HW_DATA structure. */
+#define RGX_HWPERF_GET_CNT_MASK(_data_addr, _idx) RGX_HWPERF_GET_CNT_MASKW((_data_addr)->aui32CountBlksStream[(_idx)])
+
+
+/*! Context switch packet event */
+typedef struct
+{
+	RGX_HWPERF_DM	eDM;					/*!< DataMaster identifier, see RGX_HWPERF_DM */
+	IMG_UINT32		ui32DMContext;			/*!< GPU Data Master (FW) Context */
+	IMG_UINT32		ui32FrameNum;			/*!< Client Frame number (TA, 3D only) */
+	IMG_UINT32		ui32TxtActCyc;			/*!< Meta TXTACTCYC register value */
+	IMG_UINT32		ui32PerfCycle;			/*!< Cycle count. Used to measure HW context store latency */
+	IMG_UINT32		ui32PerfPhase;			/*!< Phase. Used to determine geometry content */
+	IMG_UINT32		ui32GPUIdMask;			/*!< GPU IDs active within this event */
+	IMG_UINT32		ui32Padding;			/*!< Padding to 8 DWords */
+} RGX_HWPERF_CSW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CSW_DATA);
+
+/*! Enumeration of clocks supporting this event */
+typedef enum
+{
+	RGX_HWPERF_CLKS_CHG_INVALID = 0,
+
+	RGX_HWPERF_CLKS_CHG_NAME_CORE = 1,
+
+	RGX_HWPERF_CLKS_CHG_LAST,
+} RGX_HWPERF_CLKS_CHG_NAME;
+
+/*! This structure holds the data of a clocks change packet. */
+typedef struct
+{
+	IMG_UINT64                ui64NewClockSpeed;         /*!< New Clock Speed (in Hz) */
+	RGX_HWPERF_CLKS_CHG_NAME  eClockName;                /*!< Clock name */
+	IMG_UINT32                ui32CalibratedClockSpeed;  /*!< Calibrated new GPU clock speed (in Hz) */
+	IMG_UINT64                ui64OSMonoTimeStamp;       /*!< OSTimeStamp sampled by the host (monotonic) */
+	IMG_UINT64                ui64CRTimeStamp;           /*!< CRTimeStamp sampled by the host and
+	                                                          correlated to OSTimeStamp */
+	IMG_UINT64                ui64OSSecondaryTimeStamp;  /*!< OSTimeStamp sampled by the host */
+} RGX_HWPERF_CLKS_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CLKS_CHG_DATA);
+
+/*! Enumeration of GPU utilisation states supported by this event */
+typedef IMG_UINT32 RGX_HWPERF_GPU_STATE;
+
+/*! This structure holds the data of a GPU utilisation state change packet. */
+typedef struct
+{
+	RGX_HWPERF_GPU_STATE	eState;		/*!< New GPU utilisation state */
+	IMG_UINT32				uiUnused1;	/*!< Padding */
+	IMG_UINT32				uiUnused2;	/*!< Padding */
+	IMG_UINT32				uiUnused3;	/*!< Padding */
+} RGX_HWPERF_GPU_STATE_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_GPU_STATE_CHG_DATA);
+
+
+/*! Signature pattern 'HPE1' found in the first word of a PWR_EST packet data */
+#define HWPERF_PWR_EST_V1_SIG	0x48504531
+
+/*! Macros to obtain a component field from a counter ID word */
+#define RGX_HWPERF_GET_PWR_EST_HIGH_FLAG(_word) (((_word)&0x80000000)>>31)
+#define RGX_HWPERF_GET_PWR_EST_GPUID(_word)     (((_word)&0x70000000)>>28)
+/*!< Obtains the GPU ID from a counter ID word */
+#define RGX_HWPERF_GET_PWR_EST_UNIT(_word)      (((_word)&0x0F000000)>>24)
+#define RGX_HWPERF_GET_PWR_EST_NUMBER(_word)    ((_word)&0x0000FFFF)
+
+#define RGX_HWPERF_PWR_EST_HIGH_OFFSET		(31)
+#define RGX_HWPERF_PWR_EST_GPUID_OFFSET		(28)
+#define RGX_HWPERF_PWR_EST_GPUID_MASK		(0x7U)
+#define RGX_HWPERF_PWR_EST_UNIT_OFFSET		(24)
+#define RGX_HWPERF_PWR_EST_UNIT_MASK		(0xFU)
+#define RGX_HWPERF_PWR_EST_VALUE_MASK		(0xFFFFU)
+
+/*! This macro constructs a counter ID for a power estimate data stream from
+ * the component parts of: high word flag, unit id, GPU id, counter number */
+#define RGX_HWPERF_MAKE_PWR_EST_COUNTERID(_high, _unit, _core, _number) \
+			((IMG_UINT32)(((IMG_UINT32)((IMG_UINT32)(_high)&0x1U)<<RGX_HWPERF_PWR_EST_HIGH_OFFSET) | \
+			((IMG_UINT32) ((IMG_UINT32)(_unit)&RGX_HWPERF_PWR_EST_UNIT_MASK)<<RGX_HWPERF_PWR_EST_UNIT_OFFSET) | \
+			((IMG_UINT32) ((IMG_UINT32)(_core)&RGX_HWPERF_PWR_EST_GPUID_MASK)<<RGX_HWPERF_PWR_EST_GPUID_OFFSET) | \
+			                           ((_number)&RGX_HWPERF_PWR_EST_VALUE_MASK)))
+
+/*! This structure holds the data for a power estimate packet. */
+typedef struct
+{
+	IMG_UINT32  ui32StreamVersion;  /*!< Version word, HWPERF_PWR_EST_V1_SIG */
+	IMG_UINT32  ui32StreamSize;     /*!< Size of array in bytes of stream data
+	                                     held in the aui32StreamData member */
+	IMG_UINT32  aui32StreamData[IMG_FLEX_ARRAY_MEMBER]; /*!< Counter data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved); /*!< Reserved. To ensure correct alignment */
+} RGX_HWPERF_PWR_EST_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_PWR_EST_DATA);
+
+/*! Enumeration of the kinds of power change events that can occur */
+typedef enum
+{
+	RGX_HWPERF_PWR_UNDEFINED    = 0,
+	RGX_HWPERF_PWR_ON           = 1, /*!< Whole device powered on */
+	RGX_HWPERF_PWR_OFF          = 2, /*!< Whole device powered off */
+	RGX_HWPERF_PWR_UP           = 3, /*!< Power turned on to a HW domain */
+	RGX_HWPERF_PWR_DOWN         = 4, /*!< Power turned off to a HW domain */
+	RGX_HWPERF_PWR_SAFETY_RESET = 5, /*!< Resetting the GPU HW units for safety reasons */
+	RGX_HWPERF_PWR_PHR_FULL     = 6, /*!< Periodic HW full GPU Reset */
+	RGX_HWPERF_PWR_RD_UP        = 7, /*!< Power turned on to a Rascal+Dust */
+	RGX_HWPERF_PWR_RD_DOWN      = 8, /*!< Power turned off to a Rascal+Dust */
+	RGX_HWPERF_PWR_SPU_UP       = 9, /*!< Power turned on to a SPU */
+	RGX_HWPERF_PWR_SPU_DOWN     = 10, /*!< Power turned off to a SPU */
+	RGX_HWPERF_PWR_CLUSTER_UP   = 11, /*!< Power turned on to a cluster */
+	RGX_HWPERF_PWR_CLUSTER_DOWN = 12, /*!< Power turned off to a cluster */
+	RGX_HWPERF_PWR_RAC_UP       = 13, /*!< Power turned on to a RAC */
+	RGX_HWPERF_PWR_RAC_DOWN     = 14, /*!< Power turned off to a RAC */
+
+	RGX_HWPERF_PWR_LAST,
+} RGX_HWPERF_PWR;
+
+/*! This structure holds the data of a power packet. */
+typedef struct
+{
+	RGX_HWPERF_PWR eChange;                  /*!< Defines the type of power change */
+	IMG_UINT32     ui32Domains;              /*!< HW Domains affected */
+	IMG_UINT64     ui64OSMonoTimeStamp;      /*!< OSTimeStamp sampled by the host (monotonic) */
+	IMG_UINT64     ui64CRTimeStamp;          /*!< CRTimeStamp sampled by the host and
+	                                              correlated to OSTimeStamp */
+	IMG_UINT32     ui32CalibratedClockSpeed; /*!< GPU clock speed (in Hz) at the time
+	                                              the two timers were correlated */
+	IMG_UINT32     ui32Unused1;              /*!< Padding */
+	IMG_UINT64     ui64OSSecondaryTimeStamp; /*!< OSTimeStamp sampled by the host */
+} RGX_HWPERF_PWR_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_PWR_CHG_DATA);
+
+/*
+ * PDVFS, GPU clock frequency changes and workload estimation profiling
+ * data.
+ */
+/*! DVFS and work estimation events. */
+typedef enum
+{
+	RGX_HWPERF_DVFS_EV_INVALID,                 /*!< Invalid value. */
+	RGX_HWPERF_DVFS_EV_PROACTIVE_EST_START,     /*!< Proactive DVFS estimate start */
+	RGX_HWPERF_DVFS_EV_PROACTIVE_EST_FINISHED,  /*!< Proactive DVFS estimate finished */
+	RGX_HWPERF_DVFS_EV_REACTIVE_EST_START,      /*!< Reactive DVFS estimate start */
+	RGX_HWPERF_DVFS_EV_REACTIVE_EST_FINISHED,   /*!< Reactive DVFS estimate finished */
+	/* workload estimation */
+	RGX_HWPERF_DVFS_EV_WORK_EST_START,          /*!< Workload estimation start */
+	RGX_HWPERF_DVFS_EV_WORK_EST_FINISHED,       /*!< Workload estimation finished */
+	RGX_HWPERF_DVFS_EV_FREQ_CHG,                /*!< DVFS OPP/clock frequency change */
+
+	RGX_HWPERF_DVFS_EV_LAST               /*!< Number of element. */
+} RGX_HWPERF_DVFS_EV;
+
+/*! Enumeration of DVFS transitions that can occur */
+typedef enum
+{
+	RGX_HWPERF_DVFS_OPP_NONE        = 0x0,  /*!< No OPP change, already operating at required freq */
+	RGX_HWPERF_DVFS_OPP_IDLE        = 0x1,  /*!< GPU is idle, defer the OPP change */
+	/* 0x2 to 0xF reserved */
+	RGX_HWPERF_DVFS_OPP_UPDATE      = 0x10, /*!< OPP change, new point is encoded in bits [3:0] */
+	RGX_HWPERF_DVFS_OPP_LAST        = 0x20,
+} RGX_HWPERF_DVFS_OPP;
+
+typedef union
+{
+	/*! This structure holds the data of a proactive DVFS calculation packet. */
+	struct
+	{
+		IMG_UINT64     ui64DeadlineInus;         /*!< Next deadline in microseconds */
+		IMG_UINT32     ui32Frequency;            /*!< Required freq to meet deadline at 90% utilisation */
+		IMG_UINT32     ui32WorkloadCycles;       /*!< Current workload estimate in cycles */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sProDVFSCalc;
+
+	/*! This structure holds the data of a reactive DVFS calculation packet. */
+	struct
+	{
+		IMG_UINT32     ui32Frequency;            /*!< Required freq to achieve average 90% utilisation */
+		IMG_UINT32     ui32Utilisation;          /*!< GPU utilisation since last update in percentage points (0..100%) */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sDVFSCalc;
+
+	/*! This structure holds the data of a work estimation packet. */
+	struct
+	{
+		IMG_UINT32     ui32CyclesPrediction;     /*!< Predicted cycle count for this workload */
+		IMG_UINT32     ui32CyclesTaken;          /*!< Actual cycle count for this workload */
+		RGX_HWPERF_DM  eDM;                      /*!< Target DM */
+		IMG_UINT32     ui32ReturnDataIndex;      /*!< Index into workload estimation table */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sWorkEst;
+
+	/*! This structure holds the data of an Operating Performance Points (OPP) clock frequency transition packet. */
+	struct
+	{
+		IMG_UINT32     ui32OPPData;              /*!< New Operating Performance Point (OPP) */
+	} sOPP;
+
+} RGX_HWPERF_DVFS_DETAIL;
+
+/*! DVFS sub-event data structure */
+typedef struct {
+	RGX_HWPERF_DVFS_EV      eEventType;          /*!< DVFS sub-event type */
+	RGX_HWPERF_DVFS_DETAIL  uData;               /*!< DVFS sub-event data */
+} RGX_HWPERF_DVFS_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_DVFS_DATA);
+
+typedef enum {
+	RGX_HWPERF_UFO_EV_UPDATE,          /*!< Update on the UFO objects. */
+	RGX_HWPERF_UFO_EV_CHECK_SUCCESS,   /*!< Successful check on UFO objects. */
+	RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS, /*!< Successful partial render check on UFO objects. */
+	RGX_HWPERF_UFO_EV_CHECK_FAIL,      /*!< Unsuccessful check on UFO objects. */
+	RGX_HWPERF_UFO_EV_PRCHECK_FAIL,    /*!< Unsuccessful partial render check on UFO objects. */
+	RGX_HWPERF_UFO_EV_FORCE_UPDATE,    /*!< Forced erroring of the UFO objects. */
+
+	RGX_HWPERF_UFO_EV_LAST             /*!< Reserved. Do not use. */
+} RGX_HWPERF_UFO_EV;
+
+/*! Data stream tuple. */
+typedef union
+{
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32Value;         /*!< Value of the UFO object */
+	} sCheckSuccess;
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32Value;         /*!< Value of the UFO object */
+		IMG_UINT32 ui32Required;      /*!< Value of the UFO object required by the fence */
+	} sCheckFail;
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32OldValue;      /*!< Value of UFO object before update */
+		IMG_UINT32 ui32NewValue;      /*!< Value of UFO object after update */
+	} sUpdate;
+} RGX_HWPERF_UFO_DATA_ELEMENT;
+
+/*! This structure holds the packet payload data for UFO event. */
+typedef struct
+{
+	RGX_HWPERF_UFO_EV eEvType;     /*!< Subtype of the event. See RGX_HWPERF_UFO_EV */
+	IMG_UINT32 ui32TimeCorrIndex;  /*!< Index to the timer correlation data
+	                                 at the time the packet was generated.
+	                                 Used to approximate Host timestamps for
+	                                 these events. */
+	IMG_UINT32 ui32PID;            /*!< Client process identifier */
+	IMG_UINT32 ui32ExtJobRef;      /*!< Reference used by callers of the RGX
+	                                 API to track submitted work (for
+	                                 debugging/trace purposes) */
+	IMG_UINT32 ui32IntJobRef;      /*!< Internal reference used to track
+	                                 submitted work (for debugging / trace
+	                                 purposes) */
+	IMG_UINT32 ui32DMContext;      /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32StreamInfo;     /*!< Encoded number of elements in the
+	                                 stream and stream data offset in the
+	                                 payload */
+	RGX_HWPERF_DM eDM;             /*!< Data Master number, see RGX_HWPERF_DM */
+	IMG_UINT32 aui32StreamData[IMG_FLEX_ARRAY_MEMBER];  /*!< Series of tuples holding UFO objects data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);        /*!< Unused, reserved */
+} RGX_HWPERF_UFO_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_UFO_DATA);
+
+
+/*!
+ * RGX_HWPERF_KICK_TYPE describes the type of kick for events received / sent
+ * between KICK_START / KICK_END inclusively for all event types.
+ */
+typedef enum
+{
+	RGX_HWPERF_KICK_TYPE_RESERVED_0,  /*!< Replaced by separate TA and 3D types (Deprecated) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_1,  /*!< Compute Data Master Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_CDM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_2,  /*!< Ray Store Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_RS) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_3,  /*!< Scene Hierarchy Generator Kick (Deprecated) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_4,  /*!< TQ 2D Data Master Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_TQTDM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_5,  /*!< Sync Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_SYNC) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_6,  /*!< TA Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_GEOM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_7,  /*!< 3D Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_3D) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_8,
+	RGX_HWPERF_KICK_TYPE_RESERVED_9,
+
+	RGX_HWPERF_KICK_TYPE_UNIFIED = 0x10,
+
+	RGX_HWPERF_KICK_TYPE2_TQ2D,  /*!< 2D TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_TQ3D,  /*!< 3D TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_TQTDM, /*!< 2D Data Master TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_CDM,   /*!< Compute Kick */
+	RGX_HWPERF_KICK_TYPE2_GEOM,  /*!< GEOM Kick */
+	RGX_HWPERF_KICK_TYPE2_3D,    /*!< 3D Kick */
+	RGX_HWPERF_KICK_TYPE2_SYNC,  /*!< Sync Kick */
+	RGX_HWPERF_KICK_TYPE2_RS,    /*!< Ray Store Kick */
+	RGX_HWPERF_KICK_TYPE2_LAST,
+
+	RGX_HWPERF_KICK_TYPE_FORCE_32BIT = 0x7fffffff
+} RGX_HWPERF_KICK_TYPE;
+
+typedef enum
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_NONE,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_DEFAULT_FRAMEBUFFER,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_OFFSCREEN_FB_ATTACHMENTS,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_TILE_LIFETIME_DATA,
+
+	RGX_HWPERF_RESOURCE_TYPE_COUNT
+} RGX_HWPERF_RESOURCE_CAPTURE_TYPE;
+
+typedef struct
+{
+	IMG_UINT32 ui32Height;
+	IMG_UINT32 ui32Width;
+	IMG_UINT32 ui32BPP;
+	IMG_UINT32 ui32PixFormat;
+} RGX_RESOURCE_PER_SURFACE_INFO, *PRGX_RESOURCE_PER_SURFACE_INFO;
+
+typedef struct
+{
+	IMG_INT32  i32XOffset;        /*!< render surface X shift */
+	IMG_INT32  i32YOffset;        /*!< render surface Y shift */
+	IMG_UINT32 ui32WidthInTiles;  /*!< number of TLT data points in X */
+	IMG_UINT32 ui32HeightInTiles; /*!< number of TLT data points in Y */
+} RGX_RESOURCE_PER_TLT_BUFFER_INFO, *PRGX_RESOURCE_PER_TLT_BUFFER_INFO;
+
+typedef union
+{
+	struct RGX_RESOURCE_CAPTURE_RENDER_SURFACES
+	{
+		IMG_UINT32 ui32RenderSurfaceCount;
+		RGX_RESOURCE_PER_SURFACE_INFO sSurface[IMG_FLEX_ARRAY_MEMBER];
+	} sRenderSurfaces;
+
+	struct RGX_RESOURCE_CAPTURE_TILE_LIFETIME_BUFFERS
+	{
+		IMG_UINT32 uiReserved;
+		RGX_RESOURCE_PER_TLT_BUFFER_INFO sTLTBufInfo[IMG_FLEX_ARRAY_MEMBER];
+	} sTLTBuffers;
+} RGX_RESOURCE_CAPTURE_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE eType;
+	IMG_PID uPID;
+	IMG_UINT32 ui32ContextID;
+	IMG_UINT32 ui32FrameNum;
+	IMG_UINT32 ui32CapturedTaskJobRef;	/* The job ref of the HW task that emitted the data */
+	IMG_INT32 eClientModule;			/* RGX_HWPERF_CLIENT_API - ID that the capture is originating from. */
+	RGX_RESOURCE_CAPTURE_DETAIL uDetail; /* eType determines the value of the union */
+} RGX_RESOURCE_CAPTURE_INFO, *PRGX_RESOURCE_CAPTURE_INFO;
+
+#define RGX_RESOURCE_CAPTURE_INFO_BASE_SIZE() offsetof(RGX_RESOURCE_CAPTURE_INFO, uDetail)
+
+/*! Tile Lifetime Tracking header size. Only available if
+ * RGX_FEATURE_ISP_TILE_LIFETIME_TRACKING is present and enabled via
+ * SUPPORT_TLT_PERF
+ */
+#define RGX_TLT_HARDWARE_HDR_SIZE   (16U)
+
+/* PVRSRVGetHWPerfResourceCaptureResult */
+typedef enum
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_NONE = 0,
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK,					/* We got data ok, expect more packets for this request. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_NOT_READY,			/* Signals a timeout on the connection - no data available yet. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_COMPLETE_SUCCESS,	/* The request completed successfully, signals the end of packets for the request. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_COMPLETE_FAILURE		/* The request failed, signals the end of packets for the request. */
+} RGX_HWPERF_RESOURCE_CAPTURE_RESULT_STATUS;
+
+typedef struct
+{
+	IMG_PID uPID;						/* In case of a failed request pass the caller the PID and context ID. */
+	IMG_UINT32 ui32CtxID;
+	RGX_RESOURCE_CAPTURE_INFO *psInfo;	/* Various meta-data regarding the captured resource which aid the requester when,
+											unpacking the resource data, valid if RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK is returned. */
+	IMG_BYTE *pbData;					/* Buffer containing the captured resource data, valid if RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK is returned. */
+} RGX_RESOURCE_CAPTURE_RESULT;
+
+/*! Firmware Activity event. */
+typedef enum
+{
+	RGX_HWPERF_FWACT_EV_INVALID,            /*!< Invalid value. */
+	RGX_HWPERF_FWACT_EV_REGS_SET,           /*!< Registers set. */
+	RGX_HWPERF_FWACT_EV_HWR_DETECTED,       /*!< HWR detected. */
+	RGX_HWPERF_FWACT_EV_HWR_RESET_REQUIRED, /*!< Reset required. */
+	RGX_HWPERF_FWACT_EV_HWR_RECOVERED,      /*!< HWR recovered. */
+	RGX_HWPERF_FWACT_EV_HWR_FREELIST_READY, /*!< Freelist ready. */
+	RGX_HWPERF_FWACT_EV_FEATURES,           /*!< Features present */
+	RGX_HWPERF_FWACT_EV_FILTER_SET,         /*!< Event filter set. */
+	RGX_HWPERF_FWACT_EV_KICK_CANCEL,        /*!< A previous pipelined kick cancel. */
+
+	RGX_HWPERF_FWACT_EV_LAST                /*!< Number of element. */
+} RGX_HWPERF_FWACT_EV;
+
+/*! Cause of the HWR event. */
+typedef enum
+{
+	RGX_HWPERF_HWR_REASON_INVALID,              /*!< Invalid value. */
+	RGX_HWPERF_HWR_REASON_LOCKUP,               /*!< Lockup. */
+	RGX_HWPERF_HWR_REASON_PAGEFAULT,            /*!< Page fault. */
+	RGX_HWPERF_HWR_REASON_POLLFAIL,             /*!< Poll fail. */
+	RGX_HWPERF_HWR_REASON_DEADLINE_OVERRUN,     /*!< Deadline overrun. */
+	RGX_HWPERF_HWR_REASON_CSW_DEADLINE_OVERRUN, /*!< Hard Context Switch deadline overrun. */
+
+	RGX_HWPERF_HWR_REASON_LAST                  /*!< Number of elements. */
+} RGX_HWPERF_HWR_REASON;
+
+
+/* Fixed size for BVNC string so it does not alter packet data format
+ * Check it is large enough against official BVNC string length maximum
+ */
+#define RGX_HWPERF_MAX_BVNC_LEN (24)
+static_assert((RGX_HWPERF_MAX_BVNC_LEN >= RGX_BVNC_STR_SIZE_MAX),
+			  "Space inside HWPerf packet data for BVNC string insufficient");
+
+#define RGX_HWPERF_MAX_BVNC_BLOCK_LEN (20U)
+
+/*! BVNC Features */
+typedef struct
+{
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of counters in this block type */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Number of blocks of this type */
+	IMG_UINT16 ui16NumBlocks;
+
+	/*! Reserved for future use */
+	IMG_UINT16 ui16Reserved;
+} RGX_HWPERF_BVNC_BLOCK;
+
+/*! BVNC Features */
+typedef struct
+{
+	IMG_CHAR aszBvncString[RGX_HWPERF_MAX_BVNC_LEN]; /*!< BVNC string */
+	IMG_UINT32 ui32BvncKmFeatureFlags;               /*!< See RGX_HWPERF_FEATURE_FLAGS */
+	IMG_UINT16 ui16BvncBlocks;                       /*!< Number of blocks described in aBvncBlocks */
+	IMG_UINT16 ui16BvncGPUCores;                     /*!< Number of GPU cores present */
+	RGX_HWPERF_BVNC_BLOCK aBvncBlocks[RGX_HWPERF_MAX_BVNC_BLOCK_LEN];  /*!< Supported Performance Blocks for BVNC. See RGX_HWPERF_BVNC_BLOCK */
+} RGX_HWPERF_BVNC;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_BVNC);
+
+/*! Performance Counter Configuration data element. */
+typedef struct
+{
+	IMG_UINT32 ui32BlockID;               /*!< Counter Block ID. See RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT32 ui32NumCounters;           /*!< Number of counters configured */
+	IMG_UINT32 ui32CounterVals[RGX_CNTBLK_COUNTERS_MAX];  /*!< Counters configured (ui32NumCounters worth of entries) */
+} RGX_HWPERF_COUNTER_CFG_DATA_EL;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_COUNTER_CFG_DATA_EL);
+
+/*! Performance Counter Configuration data. */
+typedef struct
+{
+	IMG_UINT32 ui32EnabledBlocks;          /*!< Number of Enabled Blocks. */
+	RGX_HWPERF_COUNTER_CFG_DATA_EL uData;  /*!< Start of variable length data. See RGX_HWPERF_COUNTER_CFG_DATA_EL */
+	IMG_UINT32 ui32Padding;                /*!< reserved */
+} RGX_HWPERF_COUNTER_CFG;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_COUNTER_CFG);
+
+/*! Sub-event's data. */
+typedef union
+{
+	struct
+	{
+		RGX_HWPERF_DM eDM;				/*!< Data Master ID. */
+		RGX_HWPERF_HWR_REASON eReason;	/*!< Reason of the HWR. */
+		IMG_UINT32 ui32DMContext;		/*!< FW render context */
+	} sHWR;								/*!< HWR sub-event data. */
+
+	RGX_HWPERF_BVNC sBVNC;              /*!< BVNC Features. See RGX_HWPERF_BVNC */
+	struct
+	{
+		IMG_UINT32 ui32EvMaskLo;		/*!< Low order 32 bits of Filter Mask */
+		IMG_UINT32 ui32EvMaskHi;		/*!< High order 32 bits of Filter Mask */
+	} sEvMsk;							/*!< HW Filter Mask */
+	RGX_HWPERF_COUNTER_CFG sPCC;        /*!< Performance Counter Config. See RGX_HWPERF_COUNTER_CFG */
+
+	struct
+	{
+		RGX_HWPERF_DM eDM;				/*!< Data Master ID. */
+		IMG_UINT32 ui32DMContext;		/*!< FW context */
+		IMG_UINT32 ui32GPUIdMask;		/*!< Multicore mask. */
+		IMG_UINT32 ui32KickID;			/*!< Kick Id cancelled. */
+	} sKickCancel;						/*!< Kick cancel sub-event data. */
+} RGX_HWPERF_FWACT_DETAIL;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FWACT_DETAIL);
+
+/*! This structure holds the data of a FW activity event packet */
+typedef struct
+{
+	RGX_HWPERF_FWACT_EV eEvType;           /*!< Event type. */
+	RGX_HWPERF_FWACT_DETAIL uFwActDetail;  /*!< Data of the sub-event. */
+	IMG_UINT32 ui32Padding;                /*!< Reserved. */
+} RGX_HWPERF_FWACT_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FWACT_DATA);
+
+typedef struct
+{
+	RGX_HWPERF_KICK_TYPE ui32EnqType; /*!< Workload type sent to FW for
+	                                   scheduling on GPU hardware.
+	                                   See RGX_HWPERF_KICK_TYPE */
+	IMG_UINT32 ui32PID;               /*!< Client process identifier */
+	IMG_UINT32 ui32ExtJobRef;         /*!< Reference used by callers of the RGX API
+	                                   to track submitted work (for debugging /
+	                                   trace purposes) */
+	IMG_UINT32 ui32IntJobRef;         /*!< internal reference used to track submitted
+	                                   work (for debugging / trace purposes) */
+	IMG_UINT32 ui32DMContext;         /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32Padding;           /*!< Unused, reserved */
+	IMG_UINT64 ui64CheckFence_UID;    /*!< ID of fence gating work execution on GPU */
+	IMG_UINT64 ui64UpdateFence_UID;   /*!< ID of fence triggered after work completes on GPU */
+	IMG_UINT64 ui64DeadlineInus;      /*!< Workload deadline in system monotonic time */
+	IMG_UINT32 ui32CycleEstimate;     /*!< Estimated cycle time for the workload */
+	PVRSRV_FENCE hCheckFence;         /*!< Fence this enqueue task waits for, before starting */
+	PVRSRV_FENCE hUpdateFence;        /*!< Fence this enqueue task signals, on completion */
+	PVRSRV_TIMELINE hUpdateTimeline;  /*!< Timeline on which the above hUpdateFence is created */
+
+	/* Align structure size to 8 bytes */
+} RGX_HWPERF_HOST_ENQ_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_ENQ_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_ENQ_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef struct
+{
+	RGX_HWPERF_UFO_EV eEvType;   /*!< Subtype of the event */
+	IMG_UINT32 ui32StreamInfo;   /*!< Encoded number of elements in the stream and
+	                              stream data offset in the payload */
+#ifdef __CHECKER__
+	/* Since we're not conforming to the C99 standard by not using a flexible
+	 * array member need to add a special case for Smatch static code analyser. */
+	IMG_UINT32 aui32StreamData[];
+#else
+	IMG_UINT32 aui32StreamData[IMG_FLEX_ARRAY_MEMBER];       /*!< Series of tuples holding UFO objects data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);      /*!< Reserved, align structure size to 8 bytes */
+#endif
+} RGX_HWPERF_HOST_UFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_UFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_UFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*!
+ * RGX_HWPERF_HOST_RESOURCE_TYPE describes the type of resource which has been
+ * Allocated, Freed or Modified. The values are used to determine which event
+ * data structure to use to decode the data from the event stream
+ */
+typedef enum
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE_INVALID,   /*!< Invalid */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC,      /*!< SyncPrim */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_TIMELINE_DEPRECATED,
+	                                         /*!< Timeline resource packets are
+	                                           now emitted in client hwperf buffer */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR, /*!< Fence for use on GPU (SYNC_CP backed) */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP,   /*!< Sync Checkpoint */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW,  /*!< Fence created on SW timeline */
+
+	RGX_HWPERF_HOST_RESOURCE_TYPE_LAST       /*!< End of enumeration */
+} RGX_HWPERF_HOST_RESOURCE_TYPE;
+
+typedef union
+{
+	/*! Data for TYPE_TIMELINE (*Deprecated*). This sub-event is no longer
+     *  generated in the HOST stream. Timeline data is now provided in the
+     *  CLIENT stream instead.
+     */
+	struct
+	{
+		IMG_UINT32 uiPid;             /*!< Identifier of owning process */
+		IMG_UINT64 ui64Timeline_UID1; /*!< Unique identifier for timeline resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sTimelineAlloc;
+
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_PID uiPID;                /*!< Identifier of owning process */
+		PVRSRV_FENCE hFence;          /*!< Unique identifier for the fence resource */
+		IMG_UINT32 ui32CheckPt_FWAddr; /*!< Unique identifier of the check point
+		                                backing this fence on the GPU */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+	} sFenceAlloc;
+
+	/*! Data for TYPE_SYNC_CP */
+	struct
+	{
+		IMG_UINT32 ui32CheckPt_FWAddr; /*!< Unique identifier for the check point resource */
+		PVRSRV_TIMELINE hTimeline;     /*!< Unique identifier for the timeline resource */
+		IMG_PID uiPID;                 /*!< Identifier of owning process */
+		PVRSRV_FENCE hFence;           /*!< Unique identifier for the fence resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSyncCheckPointAlloc;
+
+	/*! Data for TYPE_FENCE_SW */
+	struct
+	{
+		IMG_PID uiPID;                 /*!< Identifier of owning process */
+		PVRSRV_FENCE hSWFence;         /*!< Unique identifier for the SWFence resource */
+		PVRSRV_TIMELINE hSWTimeline;   /*!< Unique identifier for the timeline resource */
+		IMG_UINT64 ui64SyncPtIndex;    /*!< Sync-pt index where this SW timeline has reached */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSWFenceAlloc;
+
+	/*! Data for TYPE_SYNC */
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;         /*!< Identifier of sync resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSyncAlloc;
+} RGX_HWPERF_HOST_ALLOC_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32AllocType;
+	                                 /*!< This describes the type of the resource
+	                                  allocated in the driver. See
+	                                  RGX_HWPERF_HOST_RESOURCE_TYPE */
+	RGX_HWPERF_HOST_ALLOC_DETAIL RGXFW_ALIGN uAllocDetail;
+	                                 /*!< Union of structures providing further
+	                                  data regarding the resource allocated.
+	                                  Size of data varies with union member that
+	                                  is present, check ``ui32AllocType`` value
+	                                  to decode */
+} RGX_HWPERF_HOST_ALLOC_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_ALLOC_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_ALLOC_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef union
+{
+	/*! Data for TYPE_TIMELINE (*Deprecated*) */
+	struct
+	{
+		IMG_UINT32 uiPid;             /*!< Identifier of owning process */
+		IMG_UINT64 ui64Timeline_UID1; /*!< Unique identifier for the timeline resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sTimelineDestroy;
+
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_UINT64 ui64Fence_UID;     /*!< Unique identifier for the fence resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. */
+	} sFenceDestroy;
+
+	/*! Data for TYPE_SYNC_CP */
+	struct
+	{
+		IMG_UINT32 ui32CheckPt_FWAddr;  /*!< Unique identifier for the check point resource */
+	} sSyncCheckPointFree;
+
+	/*! Data for TYPE_SYNC */
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< Unique identifier for the sync resource */
+	} sSyncFree;
+} RGX_HWPERF_HOST_FREE_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32FreeType;
+	                              /*!< This describes the type of the resource
+	                               freed or released by the driver. See
+	                               RGX_HWPERF_HOST_RESOURCE_TYPE */
+	RGX_HWPERF_HOST_FREE_DETAIL uFreeDetail;
+	                              /*!< Union of structures providing further data
+	                               regarding the resource freed. Size of data
+	                               varies with union member that is present,
+	                               check ``ui32FreeType`` value to decode */
+	IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+} RGX_HWPERF_HOST_FREE_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_FREE_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_FREE_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef struct
+{
+	IMG_UINT64 ui64CRTimestamp;          /*!< CR timer value from the latest entry of
+	                                      the time domains correlation table */
+	IMG_UINT64 ui64OSMonoTimestamp;      /*!< OS timestamp (monotonic) in nanoseconds from the
+	                                      latest entry of the time domains correlation table */
+	IMG_UINT32 ui32ClockSpeed;           /*!< GPU clock speed from the latest entry of
+	                                      the time domains correlation table */
+	IMG_UINT32 ui32Padding;              /*!< Reserved, align structure size to 8 bytes */
+	IMG_UINT64 ui64OSSecondaryTimeStamp; /*!< OS timestamp in nanoseconds from the latest entry
+	                                      of the time domains correlation table */
+} RGX_HWPERF_HOST_CLK_SYNC_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_CLK_SYNC_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_CLK_SYNC_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef union
+{
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_UINT64 ui64NewFence_UID;  /*!< Unique identifier for the new merged fence
+		                               resource that has been created */
+		IMG_UINT64 ui64InFence1_UID;  /*!< Unique identifier for the fence resource */
+		IMG_UINT64 ui64InFence2_UID;  /*!< Unique identifier of the check point backing
+		                               the fence on the GPU */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sFenceMerge;
+} RGX_HWPERF_HOST_MODIFY_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32ModifyType;
+	                                /*!< Describes the type of the resource
+	                                 modified by the driver. See
+	                                 RGX_HWPERF_HOST_RESOURCE_TYPE */
+
+	RGX_HWPERF_HOST_MODIFY_DETAIL uModifyDetail;
+	                                /*!< Union of structures providing further
+	                                 data regarding the resource modified.
+	                                 Size of data varies with union member that
+	                                 is present.
+	                                 Check ``uiModifyType`` value to decode */
+} RGX_HWPERF_HOST_MODIFY_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_MODIFY_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_MODIFY_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef enum
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED = 0, /*!< Invalid */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_OK,            /*!< Device OK */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_NOT_RESPONDING,/*!< Device not responding to requests */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_DEAD,          /*!< Device not responding */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_FAULT,         /*!< Device has faulted */
+
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_LAST
+} RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS;
+
+typedef enum
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_UNDEFINED = 0,     /*!< Invalid */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_NONE,              /*!< No underlying health reason. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_ASSERTED,          /*!< Device has asserted. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_POLL_FAILING,      /*!< Device poll has failed. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_TIMEOUTS,          /*!< Device timeout has fired. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_CORRUPT,     /*!< Queue has become corrupt. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_STALLED,     /*!< Queue has stalled. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_IDLING,            /*!< Device is idling. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_RESTARTING,        /*!< Device restarting. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS,/*!< Interrupts have been discarded. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_PCI_ERROR,         /*!< PCI error detected. */
+
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_LAST
+} RGX_HWPERF_HOST_DEVICE_HEALTH_REASON;
+
+/*! Data for device status event */
+typedef struct
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS eDeviceHealthStatus;
+	                                 /*!< Device's health status */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON eDeviceHealthReason;
+	                                 /*!< Reason for device's health status */
+} RGX_HWPERF_HOST_DEVICE_HEALTH;
+
+/*! RGX_HWPERF_DEV_INFO_EV values */
+typedef enum
+{
+	RGX_HWPERF_DEV_INFO_EV_HEALTH,      /*!< Health sub-event */
+	RGX_HWPERF_DEV_INFO_EV_FEATURES,    /*!< Features sub-event */
+
+	RGX_HWPERF_DEV_INFO_EV_LAST         /*!< Last enumeration value */
+} RGX_HWPERF_DEV_INFO_EV;
+
+/*! RGX_HWPERF_HOST_DEV_INFO_DETAIL is a union of structures providing
+ *  further data regarding the device's status
+ */
+typedef union
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH sDeviceStatus;    /*!< Device health status */
+	RGX_HWPERF_BVNC               sBVNC;            /*!< Device features */
+} RGX_HWPERF_HOST_DEV_INFO_DETAIL;
+
+/*! RGX_HWPERF_HOST_DEV_INFO_DATA contains device health status information */
+typedef struct
+{
+	IMG_UINT32                      ui32Padding;
+	                            /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_DEV_INFO_EV          eEvType;
+	                            /*!< Type of the sub-event. See
+	                              RGX_HWPERF_DEV_INFO_EV */
+	RGX_HWPERF_HOST_DEV_INFO_DETAIL uDevInfoDetail;
+	                            /*!< Union of structures providing further data
+	                              regarding the device's status. Size of data
+	                              varies with union member that is present,
+	                              check ``eEvType`` value to decode */
+} RGX_HWPERF_HOST_DEV_INFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_DEV_INFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			 "sizeof(RGX_HWPERF_HOST_DEV_INFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! RGX_HWPERF_INFO_EV event subtype for RGX_HWPERF_HOST_INFO_DATA events */
+typedef enum
+{
+	RGX_HWPERF_INFO_EV_RESERVED_0,
+	RGX_HWPERF_INFO_EV_MEM64_USAGE,     /*!< 64-bit Memory usage event */
+	RGX_HWPERF_INFO_EV_LAST             /*!< End of enumeration */
+} RGX_HWPERF_INFO_EV;
+
+/*! RGX_HWPERF_HOST_INFO_DETAIL contains the data payload for the
+ * RGX_HWPERF_HOST_INFO_DATA event.
+ */
+typedef union
+{
+	/*! Host Memory usage statistics */
+	struct
+	{
+		IMG_UINT64 ui64TotalMemoryUsage;   /*!< Total memory usage (bytes) */
+		/*! Detailed memory usage */
+		struct _RGX_HWPERF_HOST_INFO_PER_PROC_USAGE_
+		{
+			IMG_UINT32 ui32Pid;              /*!< Process ID */
+			IMG_UINT32 ui32Padding;          /*!< Padding */
+			IMG_UINT64 ui64KernelMemUsage;   /*!< Kernel memory usage (bytes) */
+			IMG_UINT64 ui64GraphicsMemUsage; /*!< GPU memory usage (bytes) */
+		} sPerProcessUsage[IMG_FLEX_ARRAY_MEMBER];
+	} sMemUsageStats;
+} RGX_HWPERF_HOST_INFO_DETAIL;
+
+/*! RGX_HWPERF_HOST_INFO_DATA. Host Info data event payload contains device
+ * memory usage information.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_INFO_EV eEvType;   /*!< Type of subevent. See RGX_HWPERF_INFO_EV */
+	RGX_HWPERF_HOST_INFO_DETAIL uInfoDetail;
+	                              /*!< Union of structures providing further data
+	                               regarding memory usage. Size varies with union
+	                               member that is present, check ``eEvType``
+	                               value to decode */
+} RGX_HWPERF_HOST_INFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_INFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_INFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! FENCE_WAIT_TYPE definitions */
+typedef enum
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN = 0,    /*!< Begin */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END,          /*!< End */
+
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_LAST,         /*!< Do not use */
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE;
+
+/*! FENCE_WAIT_RESULT definitions */
+typedef enum
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_INVALID = 0, /*!< Invalid */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_TIMEOUT,     /*!< Timed Out */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_PASSED,      /*!< Passed */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_ERROR,       /*!< Errored */
+
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_LAST,        /*!< Do not use */
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT;
+
+/*! FENCE_WAIT_DETAIL Event Payload */
+typedef union
+{
+/*! Data for SYNC_FENCE_WAIT_TYPE_BEGIN */
+	struct
+	{
+		IMG_UINT32 ui32TimeoutInMs;                     /*!< Wait timeout (ms) */
+	} sBegin;
+
+	/*! Data for SYNC_FENCE_WAIT_TYPE_END */
+	struct
+	{
+		RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT eResult; /*!< Wait result */
+	} sEnd;
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DETAIL;
+
+/*! RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA Event Payload. This data structure
+ * is received whenever the host driver handles a wait for sync event request.
+ */
+typedef struct
+{
+	IMG_PID uiPID;          /*!< Identifier of the owning process */
+	PVRSRV_FENCE hFence;    /*!< Unique identifier for the fence resource */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType;
+	                        /*!< Type of the subevent, see
+	                         RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DETAIL uDetail;
+	                        /*!< Union of structures providing further data
+	                         regarding device's status. Size of data varies with
+	                         union member that is present, check ``eType`` value
+	                         to decode */
+
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA;
+
+static_assert((sizeof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA.
+ * Software Timeline Advanced Event Payload. This data structure is received
+ * whenever the host driver processes a Software Timeline Advanced event.
+ */
+typedef struct
+{
+	IMG_PID uiPID;                /*!< Identifier of the owning process */
+	PVRSRV_TIMELINE hTimeline;    /*!< Unique identifier for the timeline resource */
+	IMG_UINT64 ui64SyncPtIndex;   /*!< Index of the sync point to which the
+	                               timeline has advanced */
+
+} RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA;
+
+static_assert((sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef enum
+{
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_INVALID = 0,  /*!< Invalid */
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME, /*!< Process Name */
+
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_LAST,         /*!< Do not use */
+} RGX_HWPERF_HOST_CLIENT_INFO_TYPE;
+
+typedef struct
+{
+	IMG_PID uiClientPID; /*!< Client process identifier */
+	IMG_UINT32 ui32Length;  /*!< Number of bytes present in ``acName`` */
+	IMG_CHAR acName[IMG_FLEX_ARRAY_MEMBER]; /*!< Process name string, null terminated */
+} RGX_HWPERF_HOST_CLIENT_PROC_NAME;
+
+#define RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen) \
+	((IMG_UINT32)(offsetof(RGX_HWPERF_HOST_CLIENT_PROC_NAME, acName) + (ui32NameLen)))
+
+typedef union
+{
+	struct
+	{
+		IMG_UINT32 ui32Count; /*!< Number of elements in ``asProcNames`` */
+		RGX_HWPERF_HOST_CLIENT_PROC_NAME asProcNames[IMG_FLEX_ARRAY_MEMBER];
+	} sProcName;
+} RGX_HWPERF_HOST_CLIENT_INFO_DETAIL;
+
+typedef struct
+{
+	IMG_UINT32 uiReserved; /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE eType;
+	                        /*!< Type of the subevent, see
+	                         RGX_HWPERF_HOST_CLIENT_INFO_TYPE */
+	RGX_HWPERF_HOST_CLIENT_INFO_DETAIL uDetail;
+	                        /*!< Union of structures. Size of data
+	                         varies with union member that is present,
+	                         check ``eType`` value to decode */
+} RGX_HWPERF_HOST_CLIENT_INFO_DATA;
+
+/*! This type is a union of packet payload data structures associated with
+ * various FW and Host events */
+typedef union
+{
+	RGX_HWPERF_FW_DATA             sFW;           /*!< Firmware event packet data,
+	                                               events ``0x01-0x06`` */
+	RGX_HWPERF_HW_DATA             sHW;           /*!< Hardware event packet data,
+	                                               events ``0x07-0x19``, ``0x28-0x29``
+	                                               See RGX_HWPERF_HW_DATA */
+	RGX_HWPERF_GPU_WORK_PERIOD_DATA sGPUWorkPeriod;
+
+	RGX_HWPERF_CLKS_CHG_DATA       sCLKSCHG;      /*!< Clock change event packet
+	                                               data, events ``0x1A`` */
+	RGX_HWPERF_GPU_STATE_CHG_DATA  sGPUSTATECHG;  /*!< GPU utilisation state
+	                                               change event packet data,
+	                                               events ``0x1B`` */
+	RGX_HWPERF_PWR_EST_DATA        sPWREST;       /*!< Power estimate event
+	                                               packet data,
+	                                               events ``0x20-0x22`` */
+	RGX_HWPERF_PWR_CHG_DATA        sPWR;          /*!< Power event packet data,
+	                                               events ``0x23`` */
+	RGX_HWPERF_CSW_DATA            sCSW;          /*!< Context switch packet data,
+	                                               events ``0x30-0x31`` */
+	RGX_HWPERF_DVFS_DATA           sDVFS;         /*!< DVFS activity data,
+	                                               events ``0x32`` */
+	RGX_HWPERF_UFO_DATA            sUFO;          /*!< UFO data, events ``0x38`` */
+	RGX_HWPERF_FWACT_DATA          sFWACT;        /*!< Firmware activity event
+	                                               packet data,
+	                                               events ``0x39`` */
+	/* */
+	RGX_HWPERF_HOST_ENQ_DATA       sENQ;          /*!< Host ENQ data,
+	                                               events ``0x01`` (Host) */
+	RGX_HWPERF_HOST_UFO_DATA       sHUFO;         /*!< Host UFO data,
+	                                               events ``0x02`` (Host) */
+	RGX_HWPERF_HOST_ALLOC_DATA     sHALLOC;       /*!< Host Alloc data,
+	                                               events ``0x03`` (Host) */
+	RGX_HWPERF_HOST_CLK_SYNC_DATA  sHCLKSYNC;     /*!< Host CLK_SYNC data,
+	                                               events ``0x04`` (Host) */
+	RGX_HWPERF_HOST_FREE_DATA      sHFREE;        /*!< Host Free data,
+	                                               events ``0x05`` (Host) */
+	RGX_HWPERF_HOST_MODIFY_DATA    sHMOD;         /*!< Host Modify data,
+	                                               events ``0x06`` (Host) */
+	RGX_HWPERF_HOST_DEV_INFO_DATA  sHDEVINFO;     /*!< Host device info data,
+	                                               events ``0x07`` (Host) */
+	RGX_HWPERF_HOST_INFO_DATA      sHINFO;        /*!< Host info data,
+	                                               events ``0x08`` (Host) */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA sWAIT;   /*!< Host fence-wait data,
+	                                               events ``0x09`` (Host) */
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA sSWTLADV; /*!< Host SW-timeline advance
+	                                               data, events ``0x0A`` (Host) */
+	RGX_HWPERF_HOST_CLIENT_INFO_DATA sHClientInfo; /*!< Host client info,
+	                                               events ``0x0B`` (Host) */
+} RGX_HWPERF_V2_PACKET_DATA, *RGX_PHWPERF_V2_PACKET_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_V2_PACKET_DATA);
+
+#define RGX_HWPERF_GET_PACKET_DATA(_packet_addr) ((RGX_PHWPERF_V2_PACKET_DATA) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR))))
+
+#define RGX_HWPERF_GET_DVFS_EVENT_TYPE_PTR(_packet_addr)	\
+	((RGX_HWPERF_DVFS_EV*) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR) + offsetof(RGX_HWPERF_DVFS_DATA,eEventType))))
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_COMMON_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_meta.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_meta.h
new file mode 100644
index 000000000000..1e6214c9ba1c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_meta.h
@@ -0,0 +1,389 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX META definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX META helper definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_META_H)
+#define RGX_META_H
+
+
+/***** The META HW register definitions in the file are updated manually *****/
+
+
+#include "img_defs.h"
+#include "km/rgxdefs_km.h"
+
+
+/******************************************************************************
+* META registers and MACROS
+******************************************************************************/
+#define META_CR_CTRLREG_BASE(T)					(0x04800000U + (0x1000U*(T)))
+
+#define META_CR_TXPRIVEXT						(0x048000E8)
+#define META_CR_TXPRIVEXT_MINIM_EN				(IMG_UINT32_C(0x1) << 7)
+
+#define META_CR_SYSC_JTAG_THREAD				(0x04830030)
+#define META_CR_SYSC_JTAG_THREAD_PRIV_EN		(0x00000004)
+
+#define META_CR_PERF_COUNT0						(0x0480FFE0)
+#define META_CR_PERF_COUNT1						(0x0480FFE8)
+#define META_CR_PERF_COUNT_CTRL_SHIFT			(28)
+#define META_CR_PERF_COUNT_CTRL_MASK			(0xF0000000)
+#define META_CR_PERF_COUNT_CTRL_DCACHEHITS		(IMG_UINT32_C(0x8) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICACHEHITS		(IMG_UINT32_C(0x9) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICACHEMISS		(IMG_UINT32_C(0xA) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICORE			(IMG_UINT32_C(0xD) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_THR_SHIFT			(24)
+#define META_CR_PERF_COUNT_THR_MASK				(0x0F000000)
+#define META_CR_PERF_COUNT_THR_0				(IMG_UINT32_C(0x1) << META_CR_PERF_COUNT_THR_SHIFT)
+#define META_CR_PERF_COUNT_THR_1				(IMG_UINT32_C(0x2) << META_CR_PERF_COUNT_THR_SHIFT)
+
+#define META_CR_TxVECINT_BHALT					(0x04820500)
+#define META_CR_PERF_ICORE0						(0x0480FFD0)
+#define META_CR_PERF_ICORE1						(0x0480FFD8)
+#define META_CR_PERF_ICORE_DCACHEMISS			(0x8)
+
+#define META_CR_PERF_COUNT(CTRL, THR)			((META_CR_PERF_COUNT_CTRL_##CTRL << META_CR_PERF_COUNT_CTRL_SHIFT) | \
+												 (THR << META_CR_PERF_COUNT_THR_SHIFT))
+
+#define META_CR_TXUXXRXDT_OFFSET				(META_CR_CTRLREG_BASE(0U) + 0x0000FFF0U)
+#define META_CR_TXUXXRXRQ_OFFSET				(META_CR_CTRLREG_BASE(0U) + 0x0000FFF8U)
+
+#define META_CR_TXUXXRXRQ_DREADY_BIT			(0x80000000U)	/* Poll for done */
+#define META_CR_TXUXXRXRQ_RDnWR_BIT				(0x00010000U)	/* Set for read  */
+#define META_CR_TXUXXRXRQ_TX_S					(12)
+#define META_CR_TXUXXRXRQ_RX_S					(4)
+#define META_CR_TXUXXRXRQ_UXX_S					(0)
+
+#define META_CR_TXUIN_ID						(0x0)			/* Internal ctrl regs */
+#define META_CR_TXUD0_ID						(0x1)			/* Data unit regs */
+#define META_CR_TXUD1_ID						(0x2)			/* Data unit regs */
+#define META_CR_TXUA0_ID						(0x3)			/* Address unit regs */
+#define META_CR_TXUA1_ID						(0x4)			/* Address unit regs */
+#define META_CR_TXUPC_ID						(0x5)			/* PC registers */
+
+/* Macros to calculate register access values */
+#define META_CR_CORE_REG(Thr, RegNum, Unit)	(((IMG_UINT32)(Thr)		<< META_CR_TXUXXRXRQ_TX_S) | \
+											 ((IMG_UINT32)(RegNum)	<< META_CR_TXUXXRXRQ_RX_S) | \
+											 ((IMG_UINT32)(Unit)	<< META_CR_TXUXXRXRQ_UXX_S))
+
+#define META_CR_THR0_PC		META_CR_CORE_REG(0, 0, META_CR_TXUPC_ID)
+#define META_CR_THR0_PCX	META_CR_CORE_REG(0, 1, META_CR_TXUPC_ID)
+#define META_CR_THR0_SP		META_CR_CORE_REG(0, 0, META_CR_TXUA0_ID)
+
+#define META_CR_THR1_PC		META_CR_CORE_REG(1, 0, META_CR_TXUPC_ID)
+#define META_CR_THR1_PCX	META_CR_CORE_REG(1, 1, META_CR_TXUPC_ID)
+#define META_CR_THR1_SP		META_CR_CORE_REG(1, 0, META_CR_TXUA0_ID)
+
+#define SP_ACCESS(Thread)	META_CR_CORE_REG(Thread, 0, META_CR_TXUA0_ID)
+#define PC_ACCESS(Thread)	META_CR_CORE_REG(Thread, 0, META_CR_TXUPC_ID)
+
+#define META_CR_COREREG_ENABLE			(0x0000000U)
+#define META_CR_COREREG_STATUS			(0x0000010U)
+#define META_CR_COREREG_DEFR			(0x00000A0U)
+#define META_CR_COREREG_PRIVEXT			(0x00000E8U)
+
+#define META_CR_T0ENABLE_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_ENABLE)
+#define META_CR_T0STATUS_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_STATUS)
+#define META_CR_T0DEFR_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_DEFR)
+#define META_CR_T0PRIVEXT_OFFSET		(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_PRIVEXT)
+
+#define META_CR_T1ENABLE_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_ENABLE)
+#define META_CR_T1STATUS_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_STATUS)
+#define META_CR_T1DEFR_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_DEFR)
+#define META_CR_T1PRIVEXT_OFFSET		(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_PRIVEXT)
+
+#define META_CR_TXENABLE_ENABLE_BIT		(0x00000001U)   /* Set if running */
+#define META_CR_TXSTATUS_PRIV			(0x00020000U)
+#define META_CR_TXPRIVEXT_MINIM			(0x00000080U)
+
+#define META_MEM_GLOBAL_RANGE_BIT		(0x80000000U)
+
+#define META_CR_TXCLKCTRL          (0x048000B0)
+#define META_CR_TXCLKCTRL_ALL_ON   (0x55111111)
+#define META_CR_TXCLKCTRL_ALL_AUTO (0xAA222222)
+
+
+/******************************************************************************
+* META LDR Format
+******************************************************************************/
+/* Block header structure */
+typedef struct
+{
+	IMG_UINT32	ui32DevID;
+	IMG_UINT32	ui32SLCode;
+	IMG_UINT32	ui32SLData;
+	IMG_UINT16	ui16PLCtrl;
+	IMG_UINT16	ui16CRC;
+
+} RGX_META_LDR_BLOCK_HDR;
+
+/* High level data stream block structure */
+typedef struct
+{
+	IMG_UINT16	ui16Cmd;
+	IMG_UINT16	ui16Length;
+	IMG_UINT32	ui32Next;
+	IMG_UINT32	aui32CmdData[4];
+
+} RGX_META_LDR_L1_DATA_BLK;
+
+/* High level data stream block structure */
+typedef struct
+{
+	IMG_UINT16	ui16Tag;
+	IMG_UINT16	ui16Length;
+	IMG_UINT32	aui32BlockData[4];
+
+} RGX_META_LDR_L2_DATA_BLK;
+
+/* Config command structure */
+typedef struct
+{
+	IMG_UINT32	ui32Type;
+	IMG_UINT32	aui32BlockData[4];
+
+} RGX_META_LDR_CFG_BLK;
+
+/* Block type definitions */
+#define RGX_META_LDR_COMMENT_TYPE_MASK			(0x0010U)
+#define RGX_META_LDR_BLK_IS_COMMENT(X)			((X & RGX_META_LDR_COMMENT_TYPE_MASK) != 0U)
+
+/* Command definitions
+ *  Value   Name            Description
+ *  0       LoadMem         Load memory with binary data.
+ *  1       LoadCore        Load a set of core registers.
+ *  2       LoadMMReg       Load a set of memory mapped registers.
+ *  3       StartThreads    Set each thread PC and SP, then enable threads.
+ *  4       ZeroMem         Zeros a memory region.
+ *  5       Config          Perform a configuration command.
+ */
+#define RGX_META_LDR_CMD_MASK				(0x000FU)
+
+#define RGX_META_LDR_CMD_LOADMEM			(0x0000U)
+#define RGX_META_LDR_CMD_LOADCORE			(0x0001U)
+#define RGX_META_LDR_CMD_LOADMMREG			(0x0002U)
+#define RGX_META_LDR_CMD_START_THREADS		(0x0003U)
+#define RGX_META_LDR_CMD_ZEROMEM			(0x0004U)
+#define RGX_META_LDR_CMD_CONFIG			(0x0005U)
+
+/* Config Command definitions
+ *  Value   Name        Description
+ *  0       Pause       Pause for x times 100 instructions
+ *  1       Read        Read a value from register - No value return needed.
+ *                      Utilises effects of issuing reads to certain registers
+ *  2       Write       Write to mem location
+ *  3       MemSet      Set mem to value
+ *  4       MemCheck    check mem for specific value.
+ */
+#define RGX_META_LDR_CFG_PAUSE			(0x0000)
+#define RGX_META_LDR_CFG_READ			(0x0001)
+#define RGX_META_LDR_CFG_WRITE			(0x0002)
+#define RGX_META_LDR_CFG_MEMSET			(0x0003)
+#define RGX_META_LDR_CFG_MEMCHECK		(0x0004)
+
+
+/******************************************************************************
+* RGX FW segmented MMU definitions
+******************************************************************************/
+/* All threads can access the segment */
+#define RGXFW_SEGMMU_ALLTHRS	(IMG_UINT32_C(0xf) << 8U)
+/* Writable */
+#define RGXFW_SEGMMU_WRITEABLE	(0x1U << 1U)
+/* All threads can access and writable */
+#define RGXFW_SEGMMU_ALLTHRS_WRITEABLE	(RGXFW_SEGMMU_ALLTHRS | RGXFW_SEGMMU_WRITEABLE)
+
+/* Direct map region 8 optionally used for custom mappings - max 8MB */
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_ID			(8U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_ADDR_START	(0x06000000U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_MAX_SIZE		(0x00800000U)
+
+/* Direct map region 9 optionally used for custom mappings - max 8MB */
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_ID			(9U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_ADDR_START	(0x06800000U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_MAX_SIZE		(0x00800000U)
+
+/* Direct map region 10 used for mapping GPU memory - max 8MB */
+#define RGXFW_SEGMMU_DMAP_GPU_ID			(10U)
+#define RGXFW_SEGMMU_DMAP_GPU_ADDR_START	(0x07000000U)
+#define RGXFW_SEGMMU_DMAP_GPU_MAX_SIZE		(0x00800000U)
+
+/* Segment IDs */
+#define RGXFW_SEGMMU_DATA_ID			(1U)
+#define RGXFW_SEGMMU_BOOTLDR_ID			(2U)
+#define RGXFW_SEGMMU_TEXT_ID			(RGXFW_SEGMMU_BOOTLDR_ID)
+
+/*
+ * SLC caching strategy in S7 and volcanic is emitted through the segment MMU.
+ * All the segments configured through the macro RGXFW_SEGMMU_OUTADDR_TOP are
+ * CACHED in the SLC.
+ * The interface has been kept the same to simplify the code changes.
+ * The bifdm argument is ignored (no longer relevant) in S7 and volcanic.
+ */
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(pers, slc_policy, mmu_ctx)      ((((IMG_UINT64) ((pers)    & 0x3U))  << 52) | \
+                                                                           (((IMG_UINT64) ((mmu_ctx) & 0xFFU)) << 44) | \
+                                                                           (((IMG_UINT64) ((slc_policy) & 0x1U))  << 40))
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(mmu_ctx)      RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(0x3U, 0x0U, mmu_ctx)
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED(mmu_ctx)    RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(0x0U, 0x1U, mmu_ctx)
+
+/* To configure the Page Catalog and BIF-DM fed into the BIF for Garten
+ * accesses through this segment
+ */
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC(pc, bifdm)              (((IMG_UINT64)((IMG_UINT64)(pc)    & 0xFU) << 44U) | \
+                                                              ((IMG_UINT64)((IMG_UINT64)(bifdm) & 0xFU) << 40U))
+
+#define RGXFW_SEGMMU_META_BIFDM_ID   (0x7U)
+#if !(defined(__KERNEL__) || defined(TEE_DDK)) && defined(RGX_FEATURE_META)
+#if defined(RGX_FEATURE_SLC_VIVT)
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED    RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_UNCACHED  RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED
+#define RGXFW_SEGMMU_OUTADDR_TOP_META          RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED
+#else
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED    RGXFW_SEGMMU_OUTADDR_TOP_SLC
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_UNCACHED  RGXFW_SEGMMU_OUTADDR_TOP_SLC
+#define RGXFW_SEGMMU_OUTADDR_TOP_META(pc)      RGXFW_SEGMMU_OUTADDR_TOP_SLC(pc, RGXFW_SEGMMU_META_BIFDM_ID)
+#endif
+#endif
+
+/* META segments have 4kB minimum size */
+#define RGXFW_SEGMMU_ALIGN			(0x1000U)
+
+/* Segmented MMU registers (n = segment id) */
+#define META_CR_MMCU_SEGMENTn_BASE(n)			(0x04850000U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_LIMIT(n)			(0x04850004U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_OUTA0(n)			(0x04850008U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_OUTA1(n)			(0x0485000CU + ((n)*0x10U))
+
+/* The following defines must be recalculated if the Meta MMU segments used
+ * to access Host-FW data are changed
+ * Current combinations are:
+ * - SLC uncached, META cached,   FW base address 0x70000000
+ * - SLC uncached, META uncached, FW base address 0xF0000000
+ * - SLC cached,   META cached,   FW base address 0x10000000
+ * - SLC cached,   META uncached, FW base address 0x90000000
+ */
+#define RGXFW_SEGMMU_DATA_BASE_ADDRESS        (0x10000000U)
+#define RGXFW_SEGMMU_DATA_META_CACHED         (0x0U)
+#define RGXFW_SEGMMU_DATA_META_UNCACHED       (META_MEM_GLOBAL_RANGE_BIT) // 0x80000000
+#define RGXFW_SEGMMU_DATA_META_CACHE_MASK     (META_MEM_GLOBAL_RANGE_BIT)
+/* For non-VIVT SLCs the cacheability of the FW data in the SLC is selected in
+ * the PTEs for the FW data, not in the Meta Segment MMU, which means these
+ * defines have no real effect in those cases
+ */
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED     (0x0U)
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED   (0x60000000U)
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK (0x60000000U)
+
+/******************************************************************************
+* RGX FW Bootloader defaults
+******************************************************************************/
+#define RGXFW_BOOTLDR_META_ADDR		(0x40000000U)
+#define RGXFW_BOOTLDR_DEVV_ADDR_0	(0xC0000000U)
+#define RGXFW_BOOTLDR_DEVV_ADDR_1	(0x000000E1)
+#define RGXFW_BOOTLDR_DEVV_ADDR		((((IMG_UINT64) RGXFW_BOOTLDR_DEVV_ADDR_1) << 32) | RGXFW_BOOTLDR_DEVV_ADDR_0)
+#define RGXFW_BOOTLDR_LIMIT		(0x1FFFF000)
+#define RGXFW_MAX_BOOTLDR_OFFSET	(0x1000)
+
+/* Bootloader configuration offset is in dwords (512 bytes) */
+#define RGXFW_BOOTLDR_CONF_OFFSET	(0x80)
+
+
+/******************************************************************************
+* RGX META Stack
+******************************************************************************/
+#define RGX_META_STACK_SIZE  (0x1000U)
+
+/******************************************************************************
+ RGX META Core memory
+******************************************************************************/
+/* code and data both map to the same physical memory */
+#define RGX_META_COREMEM_CODE_ADDR   (0x80000000U)
+#define RGX_META_COREMEM_DATA_ADDR   (0x82000000U)
+#define RGX_META_COREMEM_OFFSET_MASK (0x01ffffffU)
+
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#define RGX_META_IS_COREMEM_CODE(A, B)  (((A) >= RGX_META_COREMEM_CODE_ADDR) && ((A) < (RGX_META_COREMEM_CODE_ADDR + (B))))
+#define RGX_META_IS_COREMEM_DATA(A, B)  (((A) >= RGX_META_COREMEM_DATA_ADDR) && ((A) < (RGX_META_COREMEM_DATA_ADDR + (B))))
+#endif
+
+/******************************************************************************
+* 2nd thread
+******************************************************************************/
+#define RGXFW_THR1_PC		(0x18930000)
+#define RGXFW_THR1_SP		(0x78890000)
+
+/******************************************************************************
+* META compatibility
+******************************************************************************/
+
+#define META_CR_CORE_ID			(0x04831000)
+#define META_CR_CORE_ID_VER_SHIFT	(16U)
+#define META_CR_CORE_ID_VER_CLRMSK	(0XFF00FFFFU)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK)) && defined(RGX_FEATURE_META)
+
+	#if (RGX_FEATURE_META == MTP218)
+	#define RGX_CR_META_CORE_ID_VALUE 0x19
+	#elif (RGX_FEATURE_META == MTP219)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1E
+	#elif (RGX_FEATURE_META == LTP218)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1C
+	#elif (RGX_FEATURE_META == LTP217)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1F
+	#else
+	#error "Unknown META ID"
+	#endif
+#else
+
+	#define RGX_CR_META_MTP218_CORE_ID_VALUE 0x19
+	#define RGX_CR_META_MTP219_CORE_ID_VALUE 0x1E
+	#define RGX_CR_META_LTP218_CORE_ID_VALUE 0x1C
+	#define RGX_CR_META_LTP217_CORE_ID_VALUE 0x1F
+
+#endif
+#define RGXFW_PROCESSOR_META        "META"
+
+
+#endif /* RGX_META_H */
+
+/******************************************************************************
+ End of file (rgx_meta.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_mips.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_mips.h
new file mode 100644
index 000000000000..a80d2f66f0b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_mips.h
@@ -0,0 +1,406 @@
+/*************************************************************************/ /*!
+@File           rgx_mips.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    RGX MIPS definitions, kernel/user space
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_MIPS_H)
+#define RGX_MIPS_H
+
+/*
+ * Utility defines for memory management
+ */
+#define RGXMIPSFW_LOG2_PAGE_SIZE_4K              (12)
+#define RGXMIPSFW_PAGE_SIZE_4K                   (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+#define RGXMIPSFW_PAGE_MASK_4K                   (RGXMIPSFW_PAGE_SIZE_4K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_64K             (16)
+#define RGXMIPSFW_PAGE_SIZE_64K                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_64K)
+#define RGXMIPSFW_PAGE_MASK_64K                  (RGXMIPSFW_PAGE_SIZE_64K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_256K            (18)
+#define RGXMIPSFW_PAGE_SIZE_256K                 (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_256K)
+#define RGXMIPSFW_PAGE_MASK_256K                 (RGXMIPSFW_PAGE_SIZE_256K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_1MB             (20)
+#define RGXMIPSFW_PAGE_SIZE_1MB                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_1MB)
+#define RGXMIPSFW_PAGE_MASK_1MB                  (RGXMIPSFW_PAGE_SIZE_1MB - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_4MB             (22)
+#define RGXMIPSFW_PAGE_SIZE_4MB                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_4MB)
+#define RGXMIPSFW_PAGE_MASK_4MB                  (RGXMIPSFW_PAGE_SIZE_4MB - 1)
+#define RGXMIPSFW_LOG2_PTE_ENTRY_SIZE            (2)
+/* log2 page table sizes dependent on FW heap size and page size (for each OS) */
+#define RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K         (RGX_FIRMWARE_HEAP_SHIFT - RGXMIPSFW_LOG2_PAGE_SIZE_4K + RGXMIPSFW_LOG2_PTE_ENTRY_SIZE)
+#define RGXMIPSFW_LOG2_PAGETABLE_SIZE_64K        (RGX_FIRMWARE_HEAP_SHIFT - RGXMIPSFW_LOG2_PAGE_SIZE_64K + RGXMIPSFW_LOG2_PTE_ENTRY_SIZE)
+/* Maximum number of page table pages (both Host and MIPS pages) */
+#define RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES        (4)
+/* Total number of TLB entries */
+#define RGXMIPSFW_NUMBER_OF_TLB_ENTRIES          (16)
+/* "Uncached" caching policy */
+#define RGXMIPSFW_UNCACHED_CACHE_POLICY          (0X00000002U)
+/* "Write-back write-allocate" caching policy */
+#define RGXMIPSFW_WRITEBACK_CACHE_POLICY         (0X00000003)
+/* "Write-through no write-allocate" caching policy */
+#define RGXMIPSFW_WRITETHROUGH_CACHE_POLICY      (0X00000001)
+/* Cached policy used by MIPS in case of physical bus on 32 bit */
+#define RGXMIPSFW_CACHED_POLICY                  (RGXMIPSFW_WRITEBACK_CACHE_POLICY)
+/* Cached policy used by MIPS in case of physical bus on more than 32 bit */
+#define RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT      (RGXMIPSFW_WRITETHROUGH_CACHE_POLICY)
+/* Total number of Remap entries */
+#define RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES        (2 * RGXMIPSFW_NUMBER_OF_TLB_ENTRIES)
+
+
+/*
+ * MIPS EntryLo/PTE format
+ */
+
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_SHIFT     (31U)
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_CLRMSK    (0X7FFFFFFF)
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN        (0X80000000U)
+
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_SHIFT     (30U)
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_CLRMSK    (0XBFFFFFFF)
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN        (0X40000000U)
+
+/* Page Frame Number */
+#define RGXMIPSFW_ENTRYLO_PFN_SHIFT              (6)
+#define RGXMIPSFW_ENTRYLO_PFN_ALIGNSHIFT         (12)
+/* Mask used for the MIPS Page Table in case of physical bus on 32 bit */
+#define RGXMIPSFW_ENTRYLO_PFN_MASK               (0x03FFFFC0)
+#define RGXMIPSFW_ENTRYLO_PFN_SIZE               (20)
+/* Mask used for the MIPS Page Table in case of physical bus on more than 32 bit */
+#define RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT   (0x3FFFFFC0U)
+#define RGXMIPSFW_ENTRYLO_PFN_SIZE_ABOVE_32BIT   (24)
+#define RGXMIPSFW_ADDR_TO_ENTRYLO_PFN_RSHIFT     (RGXMIPSFW_ENTRYLO_PFN_ALIGNSHIFT - \
+                                                  RGXMIPSFW_ENTRYLO_PFN_SHIFT)
+
+#define RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT     (3U)
+#define RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK    (0XFFFFFFC7U)
+
+#define RGXMIPSFW_ENTRYLO_DIRTY_SHIFT            (2U)
+#define RGXMIPSFW_ENTRYLO_DIRTY_CLRMSK           (0XFFFFFFFB)
+#define RGXMIPSFW_ENTRYLO_DIRTY_EN               (0X00000004U)
+
+#define RGXMIPSFW_ENTRYLO_VALID_SHIFT            (1U)
+#define RGXMIPSFW_ENTRYLO_VALID_CLRMSK           (0XFFFFFFFD)
+#define RGXMIPSFW_ENTRYLO_VALID_EN               (0X00000002U)
+
+#define RGXMIPSFW_ENTRYLO_GLOBAL_SHIFT           (0U)
+#define RGXMIPSFW_ENTRYLO_GLOBAL_CLRMSK          (0XFFFFFFFE)
+#define RGXMIPSFW_ENTRYLO_GLOBAL_EN              (0X00000001U)
+
+#define RGXMIPSFW_ENTRYLO_DVG                    (RGXMIPSFW_ENTRYLO_DIRTY_EN | \
+                                                  RGXMIPSFW_ENTRYLO_VALID_EN | \
+                                                  RGXMIPSFW_ENTRYLO_GLOBAL_EN)
+#define RGXMIPSFW_ENTRYLO_UNCACHED               (RGXMIPSFW_UNCACHED_CACHE_POLICY << \
+                                                  RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT)
+#define RGXMIPSFW_ENTRYLO_DVG_UNCACHED           (RGXMIPSFW_ENTRYLO_DVG | RGXMIPSFW_ENTRYLO_UNCACHED)
+
+
+/* Remap Range Config Addr Out */
+/* These defines refer to the upper half of the Remap Range Config register */
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_MASK      (0x0FFFFFF0)
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_SHIFT     (4)  /* wrt upper half of the register */
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_ALIGNSHIFT (12)
+#define RGXMIPSFW_ADDR_TO_RR_ADDR_OUT_RSHIFT     (RGXMIPSFW_REMAP_RANGE_ADDR_OUT_ALIGNSHIFT - \
+                                                  RGXMIPSFW_REMAP_RANGE_ADDR_OUT_SHIFT)
+/*
+ * Pages to trampoline problematic physical addresses:
+ *   - RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN : 0x1FC0_0000
+ *   - RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN : 0x1FC0_1000
+ *   - RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN : 0x1FC0_2000
+ *   - (benign trampoline)               : 0x1FC0_3000
+ * that would otherwise be erroneously remapped by the MIPS wrapper
+ * (see "Firmware virtual layout and remap configuration" section below)
+ */
+
+#define RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES       (2)
+#define RGXMIPSFW_TRAMPOLINE_NUMPAGES            (1U << RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES)
+#define RGXMIPSFW_TRAMPOLINE_SIZE                (RGXMIPSFW_TRAMPOLINE_NUMPAGES << RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+#define RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE   (RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES + RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+
+#define RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR    (RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN)
+#define RGXMIPSFW_TRAMPOLINE_OFFSET(a)           (a - RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN)
+
+#define RGXMIPSFW_SENSITIVE_ADDR(a)              (RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN == (~((1UL << RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE)-1U) & a))
+
+#define RGXMIPSFW_C0_PAGEMASK_4K                 (0x00001800)
+#define RGXMIPSFW_C0_PAGEMASK_16K                (0x00007800)
+#define RGXMIPSFW_C0_PAGEMASK_64K                (0x0001F800)
+#define RGXMIPSFW_C0_PAGEMASK_256K               (0x0007F800)
+#define RGXMIPSFW_C0_PAGEMASK_1MB                (0x001FF800)
+#define RGXMIPSFW_C0_PAGEMASK_4MB                (0x007FF800)
+
+#if defined(RGX_FEATURE_GPU_MULTICORE_SUPPORT)
+/* GPU_COUNT: number of physical cores in the system
+ * NUM_OF_REGBANKS = GPU_COUNT + 1 //XPU BROADCAST BANK
+ * RGXMIPSFW_REGISTERS_PAGE_SIZE = NUM_OF_REGBANKS * REGBANK_SIZE(64KB) * NUM_OF_OSID(8)
+ * For RGXMIPSFW_REGISTERS_PAGE_SIZE = 4MB, NUM_OF_REGBANKS = 8 so supports upto GPU_COUNT = 7 cores
+ */
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_4MB)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_4MB)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4MB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED == 1)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_64K)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_64K)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64KB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED <= 4)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_256K)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_256K)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256KB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED <= 8)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_1MB)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_1MB)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_1MB)
+#else
+#error "MIPS TLB invalid params"
+#endif
+
+#define RGXMIPSFW_DECODE_REMAP_CONFIG_REGION_SIZE(r)       ((1U << (((r >> 7) + 1U) << 1U))*0x400)
+
+/*
+ * Firmware virtual layout and remap configuration
+ */
+/*
+ * For each remap region we define:
+ * - the virtual base used by the Firmware to access code/data through that region
+ * - the microAptivAP physical address correspondent to the virtual base address,
+ *   used as input address and remapped to the actual physical address
+ * - log2 of size of the region remapped by the MIPS wrapper, i.e. number of bits from
+ *   the bottom of the base input address that survive onto the output address
+ *   (this defines both the alignment and the maximum size of the remapped region)
+ * - one or more code/data segments within the remapped region
+ */
+
+/* Boot remap setup */
+#define RGXMIPSFW_BOOT_REMAP_VIRTUAL_BASE        (0xBFC00000)
+#define RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN        (0x1FC00000U)
+#define RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_BOOT_NMI_CODE_VIRTUAL_BASE     (RGXMIPSFW_BOOT_REMAP_VIRTUAL_BASE)
+
+/* Data remap setup */
+#define RGXMIPSFW_DATA_REMAP_VIRTUAL_BASE        (0xBFC01000)
+#define RGXMIPSFW_DATA_CACHED_REMAP_VIRTUAL_BASE (0x9FC01000)
+#define RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN        (0x1FC01000U)
+#define RGXMIPSFW_DATA_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_BOOT_NMI_DATA_VIRTUAL_BASE     (RGXMIPSFW_DATA_REMAP_VIRTUAL_BASE)
+
+/* Code remap setup */
+#define RGXMIPSFW_CODE_REMAP_VIRTUAL_BASE        (0x9FC02000)
+#define RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN        (0x1FC02000U)
+#define RGXMIPSFW_CODE_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_EXCEPTIONS_VIRTUAL_BASE        (RGXMIPSFW_CODE_REMAP_VIRTUAL_BASE)
+
+/* Permanent mappings setup */
+#define RGXMIPSFW_PT_VIRTUAL_BASE                (0xCF000000)
+#define RGXMIPSFW_REGISTERS_VIRTUAL_BASE         (0xCF800000)
+#define RGXMIPSFW_STACK_VIRTUAL_BASE             (0xCF600000)
+#define RGXMIPSFW_MIPS_STATE_VIRTUAL_BASE        (RGXMIPSFW_REGISTERS_VIRTUAL_BASE + RGXMIPSFW_REGISTERS_PAGE_SIZE)
+
+/* Offset inside the bootloader data page where the general_exception handler saves the error state.
+ * The error value is then copied by the NMI handler to the MipsState struct in shared memory.
+ * This is done because it's difficult to obtain the address of MipsState inside the general
+ * exception handler. */
+#define RGXMIPSFW_ERROR_STATE_BASE                        (0x100)
+
+/*
+ * Bootloader configuration data
+ */
+/* Bootloader configuration offset (where RGXMIPSFW_BOOT_DATA lives)
+ * within the bootloader/NMI data page */
+#define RGXMIPSFW_BOOTLDR_CONF_OFFSET                         (0x0U)
+
+/*
+ * MIPS boot stage
+ */
+#define RGXMIPSFW_BOOT_STAGE_OFFSET                           (0x400)
+
+/*
+ * MIPS private data in the bootloader data page.
+ * Memory below this offset is used by the FW only, no interface data allowed.
+ */
+#define RGXMIPSFW_PRIVATE_DATA_OFFSET                         (0x800)
+
+
+/* The things that follow are excluded when compiling assembly sources */
+#if !defined(RGXMIPSFW_ASSEMBLY_CODE)
+#include "img_types.h"
+#include "km/rgxdefs_km.h"
+
+typedef struct
+{
+	IMG_UINT64 ui64StackPhyAddr;
+	IMG_UINT64 ui64RegBase;
+	IMG_UINT64 aui64PTPhyAddr[RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES];
+	IMG_UINT32 ui32PTLog2PageSize;
+	IMG_UINT32 ui32PTNumPages;
+	IMG_UINT32 ui32Reserved1;
+	IMG_UINT32 ui32Reserved2;
+} RGXMIPSFW_BOOT_DATA;
+
+#define RGXMIPSFW_GET_OFFSET_IN_DWORDS(offset)                (offset / sizeof(IMG_UINT32))
+#define RGXMIPSFW_GET_OFFSET_IN_QWORDS(offset)                (offset / sizeof(IMG_UINT64))
+
+/* Used for compatibility checks */
+#define RGXMIPSFW_ARCHTYPE_VER_CLRMSK                         (0xFFFFE3FFU)
+#define RGXMIPSFW_ARCHTYPE_VER_SHIFT                          (10U)
+#define RGXMIPSFW_CORE_ID_VALUE                               (0x001U)
+#define RGXFW_PROCESSOR_MIPS                                  "MIPS"
+
+/* microAptivAP cache line size */
+#define RGXMIPSFW_MICROAPTIVEAP_CACHELINE_SIZE                (16U)
+
+/* The SOCIF transactions are identified with the top 16 bits of the physical address emitted by the MIPS */
+#define RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN           (16U)
+
+/* Values to put in the MIPS selectors for performance counters */
+#define RGXMIPSFW_PERF_COUNT_CTRL_ICACHE_ACCESSES_C0          (9U)   /* Icache accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_ICACHE_MISSES_C1            (9U)   /* Icache misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_DCACHE_ACCESSES_C0          (10U)  /* Dcache accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_DCACHE_MISSES_C1            (11U) /* Dcache misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_ITLB_INSTR_ACCESSES_C0      (5U)  /* ITLB instruction accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_JTLB_INSTR_MISSES_C1        (7U)  /* JTLB instruction accesses misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_INSTR_COMPLETED_C0          (1U)  /* Instructions completed in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_JTLB_DATA_MISSES_C1         (8U)  /* JTLB data misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_EVENT_SHIFT                 (5U)  /* Shift for the Event field in the MIPS perf ctrl registers */
+/* Additional flags for performance counters. See MIPS manual for further reference */
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_USER_MODE             (8U)
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_KERNEL_MODE           (2U)
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_EXL                   (1U)
+
+
+#define RGXMIPSFW_C0_NBHWIRQ	8
+
+/* Macros to decode C0_Cause register */
+#define RGXMIPSFW_C0_CAUSE_EXCCODE(CAUSE)       (((CAUSE) & 0x7cU) >> 2U)
+#define RGXMIPSFW_C0_CAUSE_EXCCODE_FWERROR      9
+/* Use only when Coprocessor Unusable exception */
+#define RGXMIPSFW_C0_CAUSE_UNUSABLE_UNIT(CAUSE) (((CAUSE) >> 28U) & 0x3U)
+#define RGXMIPSFW_C0_CAUSE_PENDING_HWIRQ(CAUSE) (((CAUSE) & 0x3fc00) >> 10)
+#define RGXMIPSFW_C0_CAUSE_FDCIPENDING          (1UL << 21)
+#define RGXMIPSFW_C0_CAUSE_IV                   (1UL << 23)
+#define RGXMIPSFW_C0_CAUSE_IC                   (1UL << 25)
+#define RGXMIPSFW_C0_CAUSE_PCIPENDING           (1UL << 26)
+#define RGXMIPSFW_C0_CAUSE_TIPENDING            (1UL << 30)
+#define RGXMIPSFW_C0_CAUSE_BRANCH_DELAY         (1UL << 31)
+
+/* Macros to decode C0_Debug register */
+#define RGXMIPSFW_C0_DEBUG_EXCCODE(DEBUG) (((DEBUG) >> 10U) & 0x1fU)
+#define RGXMIPSFW_C0_DEBUG_DSS            (1UL << 0)
+#define RGXMIPSFW_C0_DEBUG_DBP            (1UL << 1)
+#define RGXMIPSFW_C0_DEBUG_DDBL           (1UL << 2)
+#define RGXMIPSFW_C0_DEBUG_DDBS           (1UL << 3)
+#define RGXMIPSFW_C0_DEBUG_DIB            (1UL << 4)
+#define RGXMIPSFW_C0_DEBUG_DINT           (1UL << 5)
+#define RGXMIPSFW_C0_DEBUG_DIBIMPR        (1UL << 6)
+#define RGXMIPSFW_C0_DEBUG_DDBLIMPR       (1UL << 18)
+#define RGXMIPSFW_C0_DEBUG_DDBSIMPR       (1UL << 19)
+#define RGXMIPSFW_C0_DEBUG_IEXI           (1UL << 20)
+#define RGXMIPSFW_C0_DEBUG_DBUSEP         (1UL << 21)
+#define RGXMIPSFW_C0_DEBUG_CACHEEP        (1UL << 22)
+#define RGXMIPSFW_C0_DEBUG_MCHECKP        (1UL << 23)
+#define RGXMIPSFW_C0_DEBUG_IBUSEP         (1UL << 24)
+#define RGXMIPSFW_C0_DEBUG_DM             (1UL << 30)
+#define RGXMIPSFW_C0_DEBUG_DBD            (1UL << 31)
+
+/* Macros to decode TLB entries */
+#define RGXMIPSFW_TLB_GET_MASK(PAGE_MASK)       (((PAGE_MASK) >> 13) & 0XFFFFU)
+#define RGXMIPSFW_TLB_GET_PAGE_SIZE(PAGE_MASK)  ((((PAGE_MASK) | 0x1FFFU) + 1U) >> 11U) /* page size in KB */
+#define RGXMIPSFW_TLB_GET_PAGE_MASK(PAGE_SIZE)  ((((PAGE_SIZE) << 11) - 1) & ~0x7FF) /* page size in KB */
+#define RGXMIPSFW_TLB_GET_VPN2(ENTRY_HI)        ((ENTRY_HI) >> 13)
+#define RGXMIPSFW_TLB_GET_COHERENCY(ENTRY_LO)   (((ENTRY_LO) >> 3) & 0x7U)
+#define RGXMIPSFW_TLB_GET_PFN(ENTRY_LO)         (((ENTRY_LO) >> 6) & 0XFFFFFU)
+/* GET_PA uses a non-standard PFN mask for 36 bit addresses */
+#define RGXMIPSFW_TLB_GET_PA(ENTRY_LO)          (((IMG_UINT64)(ENTRY_LO) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) << 6)
+#define RGXMIPSFW_TLB_GET_INHIBIT(ENTRY_LO)     (((ENTRY_LO) >> 30) & 0x3U)
+#define RGXMIPSFW_TLB_GET_DGV(ENTRY_LO)         ((ENTRY_LO) & 0x7U)
+#define RGXMIPSFW_TLB_GLOBAL                    (1U)
+#define RGXMIPSFW_TLB_VALID                     (1U << 1)
+#define RGXMIPSFW_TLB_DIRTY                     (1U << 2)
+#define RGXMIPSFW_TLB_XI                        (1U << 30)
+#define RGXMIPSFW_TLB_RI                        (1U << 31)
+
+typedef struct {
+	IMG_UINT32 ui32TLBPageMask;
+	IMG_UINT32 ui32TLBHi;
+	IMG_UINT32 ui32TLBLo0;
+	IMG_UINT32 ui32TLBLo1;
+} RGX_MIPS_TLB_ENTRY;
+
+typedef struct {
+	IMG_UINT32 ui32RemapAddrIn;     /* always 4k aligned */
+	IMG_UINT32 ui32RemapAddrOut;    /* always 4k aligned */
+	IMG_UINT32 ui32RemapRegionSize;
+} RGX_MIPS_REMAP_ENTRY;
+
+typedef struct {
+	IMG_UINT32 ui32ErrorState; /* This must come first in the structure */
+	IMG_UINT32 ui32Sync;
+	IMG_UINT32 ui32ErrorEPC;
+	IMG_UINT32 ui32StatusRegister;
+	IMG_UINT32 ui32CauseRegister;
+	IMG_UINT32 ui32BadRegister;
+	IMG_UINT32 ui32EPC;
+	IMG_UINT32 ui32SP;
+	IMG_UINT32 ui32Debug;
+	IMG_UINT32 ui32DEPC;
+	IMG_UINT32 ui32BadInstr;
+	IMG_UINT32 ui32UnmappedAddress;
+	RGX_MIPS_TLB_ENTRY asTLB[RGXMIPSFW_NUMBER_OF_TLB_ENTRIES];
+	IMG_UINT64 aui64Remap[RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES];
+} RGX_MIPS_STATE;
+
+static_assert(offsetof(RGX_MIPS_STATE, ui32ErrorState) == 0,
+				"ui32ErrorState is not the first member of the RGX_MIPS_STATE struct");
+
+#if defined(SUPPORT_MIPS_64K_PAGE_SIZE)
+static_assert(RGXMIPSFW_REGISTERS_PAGE_SIZE >= RGXMIPSFW_PAGE_SIZE_64K,
+				"Register page size must be greater or equal to MIPS page size");
+#else
+static_assert(RGXMIPSFW_REGISTERS_PAGE_SIZE >= RGXMIPSFW_PAGE_SIZE_4K,
+				"Register page size must be greater or equal to MIPS page size");
+#endif
+
+
+#endif /* RGXMIPSFW_ASSEMBLY_CODE */
+
+#endif /* RGX_MIPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgx_riscv.h b/drivers/gpu/drm/img/img-volcanic/include/rgx_riscv.h
new file mode 100644
index 000000000000..f98b77658d8f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgx_riscv.h
@@ -0,0 +1,258 @@
+/*************************************************************************/ /*!
+@File           rgx_riscv.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    RGX RISCV definitions, kernel/user space
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_RISCV_H)
+#define RGX_RISCV_H
+
+#include "km/rgxdefs_km.h"
+
+
+/* Utility defines to convert regions to virtual addresses and remaps */
+#define RGXRISCVFW_NUM_REGIONS_LOG2             IMG_UINT32_C(4)
+#define RGXRISCVFW_REGION_SIZE_LOG2             IMG_UINT32_C(28)
+#define RGXRISCVFW_NUM_REGIONS                  (IMG_UINT32_C(1) << RGXRISCVFW_NUM_REGIONS_LOG2)
+#define RGXRISCVFW_GET_REGION_BASE(r)           ((r) << RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_GET_REGION(a)                ((a) >> RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_MAX_REGION_SIZE              (IMG_UINT32_C(1) << RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_GET_REMAP(r)                 (RGX_CR_FWCORE_ADDR_REMAP_CONFIG0 + ((r) * 8U))
+
+#define RGXRISCVFW_REGION_MASK                  (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_NUM_REGIONS - IMG_UINT32_C(1)))
+
+#if !defined(RGXRISCVFW_ASSEMBLY_CODE)
+static_assert(RGXRISCVFW_NUM_REGIONS_LOG2 + RGXRISCVFW_REGION_SIZE_LOG2 == 32,
+              "Mismatch between number of remap regions and remap size");
+#endif
+
+/* RISCV remap output is aligned to 4K */
+#define RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN  (0x1000U)
+
+/*
+ * FW bootloader defines
+ */
+#define RGXRISCVFW_BOOTLDR_CODE_REGION          (IMG_UINT32_C(0xC0000000) >> RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_BOOTLDR_DATA_REGION          IMG_UINT32_C(0x5)
+#define RGXRISCVFW_BOOTLDR_CODE_BASE            (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_BASE            (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_BOOTLDR_DATA_REGION))
+#define RGXRISCVFW_BOOTLDR_CODE_REMAP           (RGXRISCVFW_GET_REMAP(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_REMAP           (RGXRISCVFW_GET_REMAP(RGXRISCVFW_BOOTLDR_DATA_REGION))
+
+/* Bootloader data offset in dwords from the beginning of the FW data allocation */
+#define RGXRISCVFW_BOOTLDR_CONF_OFFSET          (0x0)
+
+/*
+ * FW coremem region defines
+ */
+#define RGXRISCVFW_COREMEM_REGION               IMG_UINT32_C(0x8)
+#define RGXRISCVFW_COREMEM_MAX_SIZE             IMG_UINT32_C(0x10000000) /* 256 MB */
+#define RGXRISCVFW_COREMEM_BASE                 (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_COREMEM_REGION))
+#define RGXRISCVFW_COREMEM_END                  (RGXRISCVFW_COREMEM_BASE + RGXRISCVFW_COREMEM_MAX_SIZE - 1U)
+
+
+/*
+ * Host-FW shared data defines
+ */
+#define RGXRISCVFW_SHARED_CACHED_DATA_REGION    (0x0UL)
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_REGION  (0x1UL)
+#define RGXRISCVFW_SHARED_CACHED_DATA_BASE      (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SHARED_CACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_BASE    (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SHARED_UNCACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_CACHED_DATA_REMAP     (RGXRISCVFW_GET_REMAP(RGXRISCVFW_SHARED_CACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_REMAP   (RGXRISCVFW_GET_REMAP(RGXRISCVFW_SHARED_UNCACHED_DATA_REGION))
+
+
+/*
+ * GPU SOCIF access defines
+ */
+#define RGXRISCVFW_SOCIF_REGION                 (0x2U)
+#define RGXRISCVFW_SOCIF_BASE                   (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SOCIF_REGION))
+
+
+/* The things that follow are excluded when compiling assembly sources */
+#if !defined(RGXRISCVFW_ASSEMBLY_CODE)
+#include "img_types.h"
+
+#define RGXFW_PROCESSOR_RISCV       "RISCV"
+#define RGXRISCVFW_CORE_ID_VALUE    (0x00450B02U)
+#define RGXRISCVFW_MISA_ADDR        (0x301U)
+#define RGXRISCVFW_MISA_VALUE       (0x40001104U)
+#define RGXRISCVFW_MSCRATCH_ADDR    (0x340U)
+
+typedef struct
+{
+	IMG_UINT64 ui64CorememCodeDevVAddr;
+	IMG_UINT64 ui64CorememDataDevVAddr;
+	IMG_UINT32 ui32CorememCodeFWAddr;
+	IMG_UINT32 ui32CorememDataFWAddr;
+	IMG_UINT32 ui32CorememCodeSize;
+	IMG_UINT32 ui32CorememDataSize;
+	IMG_UINT32 ui32Flags;
+	IMG_UINT32 ui32Reserved;
+} RGXRISCVFW_BOOT_DATA;
+
+/*
+ * List of registers to be printed in debug dump.
+ * First column:  register names (general purpose or control/status registers)
+ * Second column: register number to be used in abstract access register command
+ * (see RISC-V debug spec v0.13)
+ */
+#define RGXRISCVFW_DEBUG_DUMP_REGISTERS \
+	X(pc,        0x7b1) /* dpc */ \
+	X(ra,       0x1001) \
+	X(sp,       0x1002) \
+	X(mepc,      0x341) \
+	X(mcause,    0x342) \
+	X(mdseac,    0xfc0) \
+	X(mstatus,   0x300) \
+	X(mie,       0x304) \
+	X(mip,       0x344) \
+	X(mscratch,  0x340) \
+	X(mbvnc0,    0xffe) \
+	X(mbvnc1,    0xfff) \
+	X(micect,    0x7f0) \
+	X(mdcect,    0x7f3) \
+	X(mdcrfct,   0x7f4) \
+
+typedef struct
+{
+#define X(name, address) \
+	IMG_UINT32 name;
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+} RGXRISCVFW_STATE;
+
+
+#define RGXRISCVFW_MCAUSE_INTERRUPT  (1U << 31)
+
+#define RGXRISCVFW_MCAUSE_TABLE \
+	X(0x00000000U, IMG_FALSE, "NMI pin assertion") /* Also reset value */ \
+	X(0x00000001U, IMG_TRUE,  "Instruction access fault") \
+	X(0x00000002U, IMG_TRUE,  "Illegal instruction") \
+	X(0x00000003U, IMG_TRUE,  "Breakpoint") \
+	X(0x00000004U, IMG_TRUE,  "Load address misaligned") \
+	X(0x00000005U, IMG_TRUE,  "Load access fault") \
+	X(0x00000006U, IMG_TRUE,  "Store/AMO address misaligned") \
+	X(0x00000007U, IMG_TRUE,  "Store/AMO access fault") \
+	X(0x0000000BU, IMG_TRUE,  "Environment call from M-mode (FW assert)") \
+	X(0x80000007U, IMG_FALSE, "Machine timer interrupt") \
+	X(0x8000000BU, IMG_FALSE, "Machine external interrupt") \
+	X(0x8000001EU, IMG_FALSE, "Machine correctable error local interrupt") \
+	X(0xF0000000U, IMG_TRUE,  "Machine D-bus store error NMI") \
+	X(0xF0000001U, IMG_TRUE,  "Machine D-bus non-blocking load error NMI") \
+	X(0xF0000002U, IMG_TRUE,  "dCache unrecoverable NMI")
+
+
+/* Debug module HW defines */
+#define RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER (0U)
+#define RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY   (2U)
+#define RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT   (2UL << 20)
+#define RGXRISCVFW_DMI_COMMAND_WRITE           (1UL << 16)
+#define RGXRISCVFW_DMI_COMMAND_READ            (0UL << 16)
+#define RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT     (2U)
+
+/* Abstract command error codes (descriptions from RISC-V debug spec v0.13) */
+typedef IMG_UINT32 RGXRISCVFW_ABSTRACT_CMD_ERR;
+
+/* No error. */
+#define RISCV_ABSTRACT_CMD_NO_ERROR 0U
+
+/*
+ * An abstract command was executing while command, abstractcs, or abstractauto
+ * was written, or when one of the data or progbuf registers was read or
+ * written. This status is only written if cmderr contains 0.
+ */
+#define RISCV_ABSTRACT_CMD_BUSY 1U
+
+/*
+ * The requested command is not supported, regardless of whether
+ * the hart is running or not.
+ */
+#define RISCV_ABSTRACT_CMD_NOT_SUPPORTED 2U
+
+/*
+ * An exception occurred while executing the command
+ * (e.g. while executing the Program Buffer).
+ */
+#define RISCV_ABSTRACT_CMD_EXCEPTION 3U
+
+/*
+ * The abstract command couldn't execute because the hart wasn't in the required
+ * state (running/halted), or unavailable.
+ */
+#define RISCV_ABSTRACT_CMD_HALT_RESUME 4U
+
+/*
+ * The abstract command failed due to a bus error
+ * (e.g. alignment, access size, or timeout).
+ */
+#define RISCV_ABSTRACT_CMD_BUS_ERROR 5U
+
+/* The command failed for another reason. */
+#define RISCV_ABSTRACT_CMD_OTHER_ERROR 7U
+
+
+/* System Bus error codes (descriptions from RISC-V debug spec v0.13) */
+typedef IMG_UINT32 RGXRISCVFW_SYSBUS_ERR;
+
+/* There was no bus error. */
+#define RISCV_SYSBUS_NO_ERROR 0U
+
+/* There was a timeout. */
+#define RISCV_SYSBUS_TIMEOUT 1U
+
+/* A bad address was accessed. */
+#define RISCV_SYSBUS_BAD_ADDRESS 2U
+
+/* There was an alignment error. */
+#define RISCV_SYSBUS_BAD_ALIGNMENT 3U
+
+/* An access of unsupported size was requested. */
+#define RISCV_SYSBUS_UNSUPPORTED_SIZE 4U
+
+/* Other. */
+#define RISCV_SYSBUS_OTHER_ERROR 7U
+
+
+#endif /* RGXRISCVFW_ASSEMBLY_CODE */
+
+#endif /* RGX_RISCV_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rgxfw_log_helper.h b/drivers/gpu/drm/img/img-volcanic/include/rgxfw_log_helper.h
new file mode 100644
index 000000000000..275b63aca46b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rgxfw_log_helper.h
@@ -0,0 +1,79 @@
+/*************************************************************************/ /*!
+@File           rgxfw_log_helper.h
+@Title          Firmware TBI logging helper function
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       Generic
+@Description    This file contains some helper code to make TBI logging possible
+                Specifically, it uses the SFIDLIST xmacro to trace ids back to
+                the original strings.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGXFW_LOG_HELPER_H
+#define RGXFW_LOG_HELPER_H
+
+#include "rgx_fwif_sf.h"
+
+static const IMG_CHAR *const groups[]= {
+#define X(A,B) #B,
+	RGXFW_LOG_SFGROUPLIST
+#undef X
+};
+
+/*  idToStringID : Search SFs tuples {id,string} for a matching id.
+ *   return index to array if found or RGXFW_SF_LAST if none found.
+ *   bsearch could be used as ids are in increasing order. */
+#if defined(RGX_FIRMWARE)
+static IMG_UINT32 idToStringID(IMG_UINT32 ui32CheckData, const RGXFW_STID_FMT *const psSFs)
+#else
+static IMG_UINT32 idToStringID(IMG_UINT32 ui32CheckData, const RGXKM_STID_FMT *const psSFs)
+#endif
+{
+	IMG_UINT32 i = 0, ui32Id = (IMG_UINT32)RGXFW_SF_LAST;
+
+	for ( i = 0 ; psSFs[i].ui32Id != (IMG_UINT32)RGXFW_SF_LAST ; i++)
+	{
+		if ( ui32CheckData == psSFs[i].ui32Id )
+		{
+			ui32Id = i;
+			break;
+		}
+	}
+	return ui32Id;
+}
+
+#endif /* RGXFW_LOG_HELPER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/ri_typedefs.h b/drivers/gpu/drm/img/img-volcanic/include/ri_typedefs.h
new file mode 100644
index 000000000000..77be10e2ab03
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/ri_typedefs.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Information (RI) Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Client side part of RI management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RI_TYPEDEFS_H
+#define RI_TYPEDEFS_H
+
+#include "img_types.h"
+
+typedef struct RI_SUBLIST_ENTRY RI_ENTRY;
+typedef RI_ENTRY* RI_HANDLE;
+
+#endif /* #ifndef RI_TYPEDEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/pvrsrv_surface.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/pvrsrv_surface.h
new file mode 100644
index 000000000000..ae80f3cf1793
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/pvrsrv_surface.h
@@ -0,0 +1,218 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR Services Surface
+@Description    Surface definitions and utilities that are externally visible
+                (i.e. visible to clients of services), but are also required
+                within services.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_SURFACE_H
+#define PVRSRV_SURFACE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_SURFACE_TRANSFORM_NONE      (0U << 0)		/*!< No transformation */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_H    (1U << 0)		/*!< Flip horizontally */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_V    (1U << 1)		/*!< Flip vertically */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_90    (1U << 2)                          /*!< Rotate  90 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_180   ((1U << 0) + (1U << 1))             /*!< Rotate 180 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_270   ((1U << 0) + (1U << 1) + (1U << 2))  /*!< Rotate 270 degree clockwise */
+
+#define PVRSRV_SURFACE_BLENDING_NONE       0	/*!< Use no blending */
+#define PVRSRV_SURFACE_BLENDING_PREMULT    1	/*!< Use blending with pre-multiplier */
+#define PVRSRV_SURFACE_BLENDING_COVERAGE   2	/*!< Use coverage blending */
+
+/*!
+ * Modes of memory layouts for surfaces.
+ *
+ *   Enum: #_PVRSRV_SURFACE_MEMLAYOUT_
+ *   Typedef: ::PVRSRV_SURFACE_MEMLAYOUT
+ */
+typedef enum _PVRSRV_SURFACE_MEMLAYOUT_
+{
+	PVRSRV_SURFACE_MEMLAYOUT_STRIDED = 0,		/*!< Strided memory buffer */
+	PVRSRV_SURFACE_MEMLAYOUT_FBC,				/*!< Compressed frame buffer */
+} PVRSRV_SURFACE_MEMLAYOUT;
+
+/*!
+ * Frame Buffer Compression layout.
+ * Defines the FBC mode to use.
+ *
+ *   Structure: #PVRSRV_SURFACE_FBC_LAYOUT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FBC_LAYOUT
+ */
+typedef struct PVRSRV_SURFACE_FBC_LAYOUT_TAG
+{
+	/*! The compression mode for this surface */
+	IMG_FB_COMPRESSION	eFBCompressionMode;
+} PVRSRV_SURFACE_FBC_LAYOUT;
+
+/*!
+ * Pixel and memory format of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_FORMAT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FORMAT
+ */
+typedef struct PVRSRV_SURFACE_FORMAT_TAG
+{
+	/*! Enum value of type IMG_PIXFMT for the pixel format */
+	IMG_UINT32					ePixFormat;
+
+	/*! Enum surface memory layout */
+	PVRSRV_SURFACE_MEMLAYOUT	eMemLayout;
+
+	/*! Special layout options for the surface.
+	 * Needs services support.
+	 * Depends on eMemLayout.*/
+	union {
+		PVRSRV_SURFACE_FBC_LAYOUT	sFBCLayout;
+	} u;
+} PVRSRV_SURFACE_FORMAT;
+
+/*!
+ * Width and height of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_DIMS_TAG
+ *   Typedef: ::PVRSRV_SURFACE_DIMS
+ */
+typedef struct PVRSRV_SURFACE_DIMS_TAG
+{
+	IMG_UINT32		ui32Width;		/*!< Width in pixels */
+	IMG_UINT32		ui32Height;		/*!< Height in pixels */
+} PVRSRV_SURFACE_DIMS;
+
+/*!
+ * Dimension and format details of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_INFO
+ */
+typedef struct PVRSRV_SURFACE_INFO_TAG
+{
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Width and height */
+	PVRSRV_SURFACE_FORMAT	sFormat;	/*!< Memory format */
+
+	/*! Needed for FBCv3.
+	 * Indicates offset from allocation start
+	 * to the actual data, skipping the header.
+	 */
+	IMG_UINT32				ui32FBCBaseOffset;
+} PVRSRV_SURFACE_INFO;
+
+/*!
+ * Defines a rectangle on a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_RECT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_RECT
+ */
+typedef struct PVRSRV_SURFACE_RECT_TAG
+{
+	IMG_INT32				i32XOffset;	/*!< X offset from origin in pixels */
+	IMG_INT32				i32YOffset;	/*!< Y offset from origin in pixels */
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Rectangle dimensions */
+} PVRSRV_SURFACE_RECT;
+
+/*!
+ * Surface configuration details
+ *
+ *   Structure: #PVRSRV_SURFACE_CONFIG_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_CONFIG_INFO
+ */
+typedef struct PVRSRV_SURFACE_CONFIG_INFO_TAG
+{
+	/*! Crop applied to surface (BEFORE transformation) */
+	PVRSRV_SURFACE_RECT		sCrop;
+
+	/*! Region of screen to display surface in (AFTER scaling) */
+	PVRSRV_SURFACE_RECT		sDisplay;
+
+	/*! Surface transformation none/flip/rotate.
+	 * Use PVRSRV_SURFACE_TRANSFORM_xxx macros
+	 */
+	IMG_UINT32				ui32Transform;
+
+	/*! Alpha blending mode e.g. none/premult/coverage.
+	 * Use PVRSRV_SURFACE_BLENDING_xxx macros
+	 */
+	IMG_UINT32				eBlendType;
+
+	/*! Custom data for the display engine */
+	IMG_UINT32				ui32Custom;
+
+	/*! Alpha value for this plane */
+	IMG_UINT8				ui8PlaneAlpha;
+
+	/*! Reserved for later use */
+	IMG_UINT8				ui8Reserved1[3];
+} PVRSRV_SURFACE_CONFIG_INFO;
+
+/*!
+ * Contains information about a panel
+ */
+typedef struct PVRSRV_PANEL_INFO_TAG
+{
+	PVRSRV_SURFACE_INFO sSurfaceInfo;		/*!< Panel surface details */
+	IMG_UINT32			ui32RefreshRate;	/*!< Panel refresh rate in Hz */
+	IMG_UINT32			ui32XDpi;			/*!< Panel DPI in x direction */
+	IMG_UINT32			ui32YDpi;			/*!< Panel DPI in y direction */
+} PVRSRV_PANEL_INFO;
+
+/*!
+ * Helper function to create a Config Info based on a Surface Info
+ * to do a flip with no scale, transformation etc.
+ */
+static INLINE void SurfaceConfigFromSurfInfo(const PVRSRV_SURFACE_INFO *psSurfaceInfo,
+                                             PVRSRV_SURFACE_CONFIG_INFO *psConfigInfo)
+{
+	psConfigInfo->sCrop.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sCrop.i32XOffset = 0;
+	psConfigInfo->sCrop.i32YOffset = 0;
+	psConfigInfo->sDisplay.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sDisplay.i32XOffset = 0;
+	psConfigInfo->sDisplay.i32YOffset = 0;
+	psConfigInfo->ui32Transform = PVRSRV_SURFACE_TRANSFORM_NONE;
+	psConfigInfo->eBlendType = PVRSRV_SURFACE_BLENDING_NONE;
+	psConfigInfo->ui32Custom = 0;
+	psConfigInfo->ui8PlaneAlpha = 0xff;
+}
+
+#endif /* PVRSRV_SURFACE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_alignchecks.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_alignchecks.h
new file mode 100644
index 000000000000..a33b6d541c65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_alignchecks.h
@@ -0,0 +1,192 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX fw interface alignment checks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Checks to avoid disalignment in RGX fw data structures
+                shared with the host
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_ALIGNCHECKS_H)
+#define RGX_FWIF_ALIGNCHECKS_H
+
+/* for the offsetof macro */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/*!
+ ******************************************************************************
+ * Alignment UM/FW checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_UM_MAX 128U
+
+#define RGXFW_ALIGN_CHECKS_INIT0								\
+		(IMG_UINT32)sizeof(RGXFWIF_TRACEBUF),								\
+		offsetof(RGXFWIF_TRACEBUF, ui32LogType),				\
+		offsetof(RGXFWIF_TRACEBUF, sTraceBuf),					\
+		offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),	\
+		offsetof(RGXFWIF_TRACEBUF, ui32TracebufFlags),			\
+																\
+		(IMG_UINT32)sizeof(RGXFWIF_SYSDATA),								\
+		offsetof(RGXFWIF_SYSDATA, ePowState),					\
+		offsetof(RGXFWIF_SYSDATA, ui32HWPerfDropCount),			\
+		offsetof(RGXFWIF_SYSDATA, ui32LastDropOrdinal),			\
+		offsetof(RGXFWIF_SYSDATA, ui32FWFaults),				\
+		offsetof(RGXFWIF_SYSDATA, ui32HWRStateFlags),			\
+																\
+		(IMG_UINT32)sizeof(RGXFWIF_OSDATA),									\
+		offsetof(RGXFWIF_OSDATA, ui32HostSyncCheckMark),		\
+		offsetof(RGXFWIF_OSDATA, ui32KCCBCmdsExecuted),			\
+																	\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRINFOBUF),									\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmLockedUpCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmOverranCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmRecoveredCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmFalseDetectCount),	\
+																	\
+		/* RGXFWIF_CMDTA checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTA),							\
+		offsetof(RGXFWIF_CMDTA, sGeomRegs),				\
+														\
+		/* RGXFWIF_CMD3D checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_CMD3D),							\
+		offsetof(RGXFWIF_CMD3D, s3DRegs),				\
+														\
+		/* RGXFWIF_CMDTRANSFER checks */				\
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTRANSFER),					\
+		offsetof(RGXFWIF_CMDTRANSFER, sTransRegs),		\
+														\
+														\
+		/* RGXFWIF_CMD_COMPUTE checks */				\
+		(IMG_UINT32)sizeof(RGXFWIF_CMD_COMPUTE),					\
+		offsetof(RGXFWIF_CMD_COMPUTE, sCDMRegs),		\
+														\
+		/* RGXFWIF_FREELIST checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_FREELIST),						\
+		offsetof(RGXFWIF_FREELIST, psFreeListDevVAddr),	\
+		offsetof(RGXFWIF_FREELIST, ui32MaxPages),		\
+		offsetof(RGXFWIF_FREELIST, ui32CurrentPages),	\
+														\
+		/* RGXFWIF_HWRTDATA checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRTDATA),							\
+		offsetof(RGXFWIF_HWRTDATA, psVHeapTableDevVAddr),	\
+		offsetof(RGXFWIF_HWRTDATA, psPMMListDevVAddr),		\
+		offsetof(RGXFWIF_HWRTDATA, apsFreeLists),			\
+		offsetof(RGXFWIF_HWRTDATA, ui64VCECatBase),			\
+		offsetof(RGXFWIF_HWRTDATA, eState),					\
+															\
+		/* RGXFWIF_HWRTDATA_COMMON checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRTDATA_COMMON),						\
+		offsetof(RGXFWIF_HWRTDATA_COMMON, bTACachesNeedZeroing),\
+																\
+		/* RGXFWIF_HWPERF_CTL_BLK checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_HWPERF_CTL_BLK),						\
+		offsetof(RGXFWIF_HWPERF_CTL_BLK, aui64CounterCfg),	\
+															\
+		/* RGXFWIF_HWPERF_CTL checks */			\
+		(IMG_UINT32)sizeof(RGXFWIF_HWPERF_CTL),				\
+		offsetof(RGXFWIF_HWPERF_CTL, SelCntr)
+
+#if defined(RGX_FEATURE_TLA)
+#define RGXFW_ALIGN_CHECKS_INIT1                   \
+		RGXFW_ALIGN_CHECKS_INIT0,                  \
+		/* RGXFWIF_CMD2D checks */                 \
+		(IMG_UINT32)sizeof(RGXFWIF_CMD2D),                     \
+		offsetof(RGXFWIF_CMD2D, s2DRegs)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT1		RGXFW_ALIGN_CHECKS_INIT0
+#endif	/* RGX_FEATURE_TLA */
+
+
+#if defined(RGX_FEATURE_FASTRENDER_DM)
+#define RGXFW_ALIGN_CHECKS_INIT                    \
+		RGXFW_ALIGN_CHECKS_INIT1,                  \
+		/* RGXFWIF_CMDTDM checks */                \
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTDM),                    \
+		offsetof(RGXFWIF_CMDTDM, sTDMRegs)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT		RGXFW_ALIGN_CHECKS_INIT1
+#endif /* ! RGX_FEATURE_FASTRENDER_DM */
+
+
+
+/*!
+ ******************************************************************************
+ * Alignment KM checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_INIT_KM                                   \
+		(IMG_UINT32)sizeof(RGXFWIF_SYSINIT),                                     \
+		offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr),                   \
+		offsetof(RGXFWIF_SYSINIT, sPDSExecBase),                     \
+		offsetof(RGXFWIF_SYSINIT, sUSCExecBase),                     \
+		offsetof(RGXFWIF_SYSINIT, aui32TPUTrilinearFracMask),        \
+		offsetof(RGXFWIF_SYSINIT, sTraceBufCtl),                     \
+		offsetof(RGXFWIF_SYSINIT, sFwSysData),                       \
+		(IMG_UINT32)sizeof(RGXFWIF_OSINIT),                                      \
+		offsetof(RGXFWIF_OSINIT, psKernelCCBCtl),                    \
+		offsetof(RGXFWIF_OSINIT, psKernelCCB),                       \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCBCtl),                  \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCB),                     \
+		offsetof(RGXFWIF_OSINIT, sFwOsData),                         \
+		offsetof(RGXFWIF_OSINIT, sRGXCompChecks),                    \
+		                                                             \
+		/* RGXFWIF_FWRENDERCONTEXT checks */                         \
+		(IMG_UINT32)sizeof(RGXFWIF_FWRENDERCONTEXT),                             \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),               \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),               \
+		                                                             \
+		(IMG_UINT32)sizeof(RGXFWIF_FWCOMMONCONTEXT),                             \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psFWMemContext),           \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, sRunNode),                 \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psCCB),                    \
+		                                                             \
+		(IMG_UINT32)sizeof(RGXFWIF_MMUCACHEDATA),                                \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32CacheFlags),              \
+		offsetof(RGXFWIF_MMUCACHEDATA, sMMUCacheSync),               \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32MMUCacheSyncUpdateValue)
+
+#endif /* RGX_FWIF_ALIGNCHECKS_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_alignchecks.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_hwperf.h
new file mode 100644
index 000000000000..ec2d9bb071e7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_hwperf.h
@@ -0,0 +1,253 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_hwperf.h
+@Title          RGX HWPerf support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared header between RGX firmware and Init process
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_HWPERF_H
+#define RGX_FWIF_HWPERF_H
+
+#include "rgx_fwif_shared.h"
+#include "rgx_hwperf.h"
+#include "rgxdefs_km.h"
+
+
+/*****************************************************************************/
+
+/* Structure to hold a block's parameters for passing between the BG context
+ * and the IRQ context when applying a configuration request. */
+typedef struct
+{
+	/* Few members could be booleans but padded to IMG_UINT32
+	 * to workaround pdump alignment requirements */
+	IMG_UINT32              ui32Valid;
+	IMG_UINT32              ui32Enabled;
+	IMG_UINT32              eBlockID;
+	IMG_UINT32              uiCounterMask;
+	IMG_UINT64  RGXFW_ALIGN aui64CounterCfg[RGX_CNTBLK_MUX_COUNTERS_MAX];
+}  RGXFWIF_HWPERF_CTL_BLK;
+
+/* Structure used to hold the configuration of the non-mux counters blocks */
+typedef struct
+{
+	IMG_UINT32            ui32NumSelectedCounters;
+	IMG_UINT32            aui32SelectedCountersIDs[RGX_HWPERF_MAX_CUSTOM_CNTRS];
+} RGXFW_HWPERF_SELECT;
+
+/* Structure used to hold a Direct-Addressable block's parameters for passing
+ * between the BG context and the IRQ context when applying a configuration
+ * request. HWPERF_UNIFIED use only.
+ */
+typedef struct
+{
+	IMG_UINT32               uiEnabled;
+	IMG_UINT32               uiNumCounters;
+	IMG_UINT32               eBlockID;
+	RGXFWIF_DEV_VIRTADDR     psModel;
+	IMG_UINT32               aui32Counters[RGX_CNTBLK_COUNTERS_MAX];
+} RGXFWIF_HWPERF_DA_BLK;
+
+
+/* Structure to hold the whole configuration request details for all blocks
+ * The block masks and counts are used to optimise reading of this data. */
+typedef struct
+{
+	IMG_UINT32                         ui32HWPerfCtlFlags;
+
+	IMG_UINT32                         ui32SelectedCountersBlockMask;
+	RGXFW_HWPERF_SELECT RGXFW_ALIGN    SelCntr[RGX_HWPERF_MAX_CUSTOM_BLKS];
+
+	IMG_UINT32                         ui32EnabledMUXBlksCount;
+	RGXFWIF_HWPERF_CTL_BLK RGXFW_ALIGN sBlkCfg[RGX_HWPERF_MAX_MUX_BLKS];
+} UNCACHED_ALIGN RGXFWIF_HWPERF_CTL;
+
+/* NOTE: The switch statement in this function must be kept in alignment with
+ * the enumeration RGX_HWPERF_CNTBLK_ID defined in rgx_hwperf.h. ASSERTs may
+ * result if not.
+ * The function provides a hash lookup to get a handle on the global store for
+ * a block's configuration store from it's block ID.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(rgxfw_hwperf_get_block_ctl)
+#endif
+static INLINE RGXFWIF_HWPERF_CTL_BLK *rgxfw_hwperf_get_block_ctl(
+		RGX_HWPERF_CNTBLK_ID eBlockID, RGXFWIF_HWPERF_CTL *psHWPerfInitData)
+{
+	IMG_UINT32 ui32Idx;
+
+	/* Hash the block ID into a control configuration array index */
+	switch (eBlockID)
+	{
+		case RGX_CNTBLK_ID_TA:
+		case RGX_CNTBLK_ID_RASTER:
+		case RGX_CNTBLK_ID_HUB:
+		case RGX_CNTBLK_ID_TORNADO:
+		case RGX_CNTBLK_ID_JONES:
+		{
+			ui32Idx = eBlockID;
+			break;
+		}
+		case RGX_CNTBLK_ID_TPU_MCU0:
+		case RGX_CNTBLK_ID_TPU_MCU1:
+		case RGX_CNTBLK_ID_TPU_MCU2:
+		case RGX_CNTBLK_ID_TPU_MCU3:
+		case RGX_CNTBLK_ID_TPU_MCU4:
+		case RGX_CNTBLK_ID_TPU_MCU5:
+		case RGX_CNTBLK_ID_TPU_MCU6:
+		case RGX_CNTBLK_ID_TPU_MCU7:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_USC0:
+		case RGX_CNTBLK_ID_USC1:
+		case RGX_CNTBLK_ID_USC2:
+		case RGX_CNTBLK_ID_USC3:
+		case RGX_CNTBLK_ID_USC4:
+		case RGX_CNTBLK_ID_USC5:
+		case RGX_CNTBLK_ID_USC6:
+		case RGX_CNTBLK_ID_USC7:
+		case RGX_CNTBLK_ID_USC8:
+		case RGX_CNTBLK_ID_USC9:
+		case RGX_CNTBLK_ID_USC10:
+		case RGX_CNTBLK_ID_USC11:
+		case RGX_CNTBLK_ID_USC12:
+		case RGX_CNTBLK_ID_USC13:
+		case RGX_CNTBLK_ID_USC14:
+		case RGX_CNTBLK_ID_USC15:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_TEXAS0:
+		case RGX_CNTBLK_ID_TEXAS1:
+		case RGX_CNTBLK_ID_TEXAS2:
+		case RGX_CNTBLK_ID_TEXAS3:
+		case RGX_CNTBLK_ID_TEXAS4:
+		case RGX_CNTBLK_ID_TEXAS5:
+		case RGX_CNTBLK_ID_TEXAS6:
+		case RGX_CNTBLK_ID_TEXAS7:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_RASTER0:
+		case RGX_CNTBLK_ID_RASTER1:
+		case RGX_CNTBLK_ID_RASTER2:
+		case RGX_CNTBLK_ID_RASTER3:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_BLACKPEARL0:
+		case RGX_CNTBLK_ID_BLACKPEARL1:
+		case RGX_CNTBLK_ID_BLACKPEARL2:
+		case RGX_CNTBLK_ID_BLACKPEARL3:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(RASTER, 3) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_PBE0:
+		case RGX_CNTBLK_ID_PBE1:
+		case RGX_CNTBLK_ID_PBE2:
+		case RGX_CNTBLK_ID_PBE3:
+		case RGX_CNTBLK_ID_PBE4:
+		case RGX_CNTBLK_ID_PBE5:
+		case RGX_CNTBLK_ID_PBE6:
+		case RGX_CNTBLK_ID_PBE7:
+		case RGX_CNTBLK_ID_PBE8:
+		case RGX_CNTBLK_ID_PBE9:
+		case RGX_CNTBLK_ID_PBE10:
+		case RGX_CNTBLK_ID_PBE11:
+		case RGX_CNTBLK_ID_PBE12:
+		case RGX_CNTBLK_ID_PBE13:
+		case RGX_CNTBLK_ID_PBE14:
+		case RGX_CNTBLK_ID_PBE15:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(RASTER, 3) +
+						RGX_CNTBLK_INDIRECT_COUNT(BLACKPEARL, 3) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		default:
+		{
+			ui32Idx = RGX_HWPERF_MAX_DEFINED_BLKS;
+			break;
+		}
+	}
+	if (ui32Idx >= RGX_HWPERF_MAX_DEFINED_BLKS)
+	{
+		return NULL;
+	}
+	return &psHWPerfInitData->sBlkCfg[ui32Idx];
+}
+
+/* Stub routine for rgxfw_hwperf_get_da_block_ctl(). Just return a NULL.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(rgxfw_hwperf_get_da_block_ctl)
+#endif
+static INLINE RGXFWIF_HWPERF_DA_BLK* rgxfw_hwperf_get_da_block_ctl(
+		RGX_HWPERF_CNTBLK_ID eBlockID, RGXFWIF_HWPERF_CTL *psHWPerfInitData)
+{
+	PVR_UNREFERENCED_PARAMETER(eBlockID);
+	PVR_UNREFERENCED_PARAMETER(psHWPerfInitData);
+
+	return NULL;
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_km.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_km.h
new file mode 100644
index 000000000000..22da7ec4cc9d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_km.h
@@ -0,0 +1,1888 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures used by pvrsrvkm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures used by pvrsrvkm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_KM_H)
+#define RGX_FWIF_KM_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+#include "rgxdefs_km.h"
+#include "dllist.h"
+#include "rgx_hwperf.h"
+#include "rgx_mips.h"
+#include "rgxheapconfig.h"
+
+/************************************************************************
+* RGX FW signature checks
+************************************************************************/
+#define RGXFW_SIG_BUFFER_SIZE_MIN       (8192)
+
+#define RGXFWIF_TIMEDIFF_ID			((0x1UL << 28) | RGX_CR_TIMER)
+
+#define RGXFW_POLL_TYPE_SET 0x80000000U
+
+/* Firmware per-DM HWR states */
+#define RGXFWIF_DM_STATE_WORKING					(0x00U)		/*!< DM is working if all flags are cleared */
+#define RGXFWIF_DM_STATE_READY_FOR_HWR				(IMG_UINT32_C(0x1) << 0)	/*!< DM is idle and ready for HWR */
+#define RGXFWIF_DM_STATE_NEEDS_SKIP					(IMG_UINT32_C(0x1) << 2)	/*!< DM need to skip to next cmd before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP			(IMG_UINT32_C(0x1) << 3)	/*!< DM need partial render cleanup before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR			(IMG_UINT32_C(0x1) << 4)	/*!< DM need to increment Recovery Count once fully recovered */
+#define RGXFWIF_DM_STATE_GUILTY_LOCKUP				(IMG_UINT32_C(0x1) << 5)	/*!< DM was identified as locking up and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_LOCKUP			(IMG_UINT32_C(0x1) << 6)	/*!< DM was innocently affected by another lockup which caused HWR */
+#define RGXFWIF_DM_STATE_GUILTY_OVERRUNING			(IMG_UINT32_C(0x1) << 7)	/*!< DM was identified as over-running and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_OVERRUNING		(IMG_UINT32_C(0x1) << 8)	/*!< DM was innocently affected by another DM over-running which caused HWR */
+#define RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH		(IMG_UINT32_C(0x1) << 9)	/*!< DM was forced into HWR as it delayed more important workloads */
+#define RGXFWIF_DM_STATE_GPU_ECC_HWR				(IMG_UINT32_C(0x1) << 10)	/*!< DM was forced into HWR due to an uncorrected GPU ECC error */
+#define RGXFWIF_DM_STATE_GPU_PARITY_HWR				(IMG_UINT32_C(0x1) << 11)	/*!< DM was forced into HWR due to an uncorrected GPU PARITY error */
+#define RGXFWIF_DM_STATE_GPU_LATENT_HWR				(IMG_UINT32_C(0x1) << 12)	/*!< DM was forced into HWR due to an uncorrected GPU LATENT error */
+
+
+/* Firmware's connection state */
+typedef IMG_UINT32 RGXFWIF_CONNECTION_FW_STATE;
+#define RGXFW_CONNECTION_FW_OFFLINE				0U	/*!< Firmware is offline */
+#define RGXFW_CONNECTION_FW_READY				1U	/*!< Firmware is initialised */
+#define RGXFW_CONNECTION_FW_ACTIVE				2U	/*!< Firmware connection is fully established */
+#define RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD	3U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_FORCED_OFFLOAD		4U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_COOLDOWN			5U	/*!< Firmware connection is in cooldown period */
+#define RGXFW_CONNECTION_FW_STATE_COUNT			6U
+
+#define RGXFW_FORCED_OFFLOAD_HCS_DEADLINE_MS	2U		/*!< Workloads of Guest being forcefully offloaded must be stopped quickly */
+#define RGXFW_GRACEFUL_OFFLOAD_HCS_DEADLINE_MS	1000U	/*!< Workloads of Guest being gracefully offloaded are allowed more time to finish */
+
+/* OS' connection state */
+typedef enum
+{
+	RGXFW_CONNECTION_OS_OFFLINE = 0,	/*!< OS is offline */
+	RGXFW_CONNECTION_OS_READY,			/*!< OS's KM driver is setup and waiting */
+	RGXFW_CONNECTION_OS_ACTIVE,			/*!< OS connection is fully established */
+	RGXFW_CONNECTION_OS_STATE_COUNT
+} RGXFWIF_CONNECTION_OS_STATE;
+
+#define RGXFWIF_CTXSWITCH_PROFILE_FAST_EN		(IMG_UINT32_C(0x1))
+#define RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN		(IMG_UINT32_C(0x2))
+#define RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN		(IMG_UINT32_C(0x3))
+#define RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN	(IMG_UINT32_C(0x4))
+
+#define RGXFWIF_ISP_SCHEDMODE_VER1_IPP			(IMG_UINT32_C(0x1))
+#define RGXFWIF_ISP_SCHEDMODE_VER2_ISP			(IMG_UINT32_C(0x2))
+/*!
+ ******************************************************************************
+ * RGX firmware Init Config Data
+ *****************************************************************************/
+
+/* Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_CTXSWITCH_MODE_RAND				(IMG_UINT32_C(0x1) << 0)	/*!< Randomise context switch requests */
+#define RGXFWIF_INICFG_CTXSWITCH_SRESET_EN				(IMG_UINT32_C(0x1) << 1)
+#define RGXFWIF_INICFG_HWPERF_EN						(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN				(IMG_UINT32_C(0x1) << 3)	/*!< Randomise DM-killing requests */
+#define RGXFWIF_INICFG_POW_RASCALDUST					(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_SPU_CLOCK_GATE					(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_FBCDC_V3_1_EN					(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_CHECK_MLIST_EN					(IMG_UINT32_C(0x1) << 7)
+#define RGXFWIF_INICFG_DISABLE_CLKGATING_EN				(IMG_UINT32_C(0x1) << 8)
+#define RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN	(IMG_UINT32_C(0x1) << 9)
+#define RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN		(IMG_UINT32_C(0x1) << 10)
+/* 11 unused */
+#define RGXFWIF_INICFG_REGCONFIG_EN						(IMG_UINT32_C(0x1) << 12)
+#define RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY			(IMG_UINT32_C(0x1) << 13)
+#define RGXFWIF_INICFG_HWP_DISABLE_FILTER				(IMG_UINT32_C(0x1) << 14)
+/* 15 unused */
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT			(16)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST			(RGXFWIF_CTXSWITCH_PROFILE_FAST_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM			(RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW			(RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY		(RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK			(IMG_UINT32_C(0x7) << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_DISABLE_DM_OVERLAP				(IMG_UINT32_C(0x1) << 19)
+#define RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER			(IMG_UINT32_C(0x1) << 20)
+#define RGXFWIF_INICFG_INJECT_ICS_FAULT					(IMG_UINT32_C(0x1) << 21)
+#define RGXFWIF_INICFG_VALIDATE_IRQ						(IMG_UINT32_C(0x1) << 22)
+#define RGXFWIF_INICFG_DISABLE_PDP_EN					(IMG_UINT32_C(0x1) << 23)
+#define RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN	(IMG_UINT32_C(0x1) << 24)
+#define RGXFWIF_INICFG_WORKEST							(IMG_UINT32_C(0x1) << 25)
+#define RGXFWIF_INICFG_PDVFS							(IMG_UINT32_C(0x1) << 26)
+/* 27 unused */
+/* 28 unused */
+#define RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT				(29)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_NONE				(0)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP			(RGXFWIF_ISP_SCHEDMODE_VER1_IPP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP			(RGXFWIF_ISP_SCHEDMODE_VER2_ISP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_MASK				(RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP |\
+                                                         RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP)
+#define RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER			(IMG_UINT32_C(0x1) << 31)
+
+#define RGXFWIF_INICFG_ALL								(0xFFFFFFFFU)
+
+/* Extended Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT			(0)
+/* [8]   Lossy min channel override
+ * [7]   YUV10 override
+ * [6:4] Quality
+ * [3]   Quality enable
+ * [2:1] Compression scheme
+ * [0]   Lossy group */
+#define RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK			(IMG_UINT32_C(0x1FF)) /* RGX_CR_TFBC_COMPRESSION_CONTROL_MASKFULL */
+#define RGXFWIF_INICFG_EXT_ALL							(RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK)
+
+#define RGXFWIF_INICFG_SYS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_CTXSWITCH_MODE_RAND | \
+														  RGXFWIF_INICFG_CTXSWITCH_SRESET_EN)
+
+/* Flag definitions affecting only workloads submitted by a particular OS */
+
+/*!
+ * @AddToGroup ContextSwitching
+ * @{
+ * @Name Per-OS DM context switch configuration flags
+ * @{
+ */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN				(IMG_UINT32_C(0x1) << 0) /*!< Enables TDM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN				(IMG_UINT32_C(0x1) << 1) /*!< Enables GEOM DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN				(IMG_UINT32_C(0x1) << 2) /*!< Enables FRAG DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN				(IMG_UINT32_C(0x1) << 3) /*!< Enables CDM context switch */
+
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM				(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM				(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D				(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM				(IMG_UINT32_C(0x1) << 7)
+
+#define RGXFWIF_INICFG_OS_ALL							(0xFFU)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL				(RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL)
+
+/*!
+ * @} End of Per-OS Context switch configuration flags
+ * @} End of AddToGroup ContextSwitching
+ */
+
+#define RGXFWIF_FILTCFG_TRUNCATE_HALF					(IMG_UINT32_C(0x1) << 3)
+#define RGXFWIF_FILTCFG_TRUNCATE_INT					(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_FILTCFG_NEW_FILTER_MODE					(IMG_UINT32_C(0x1) << 1)
+
+typedef IMG_UINT32 RGX_ACTIVEPM_CONF;
+#define RGX_ACTIVEPM_FORCE_OFF	0U
+#define RGX_ACTIVEPM_FORCE_ON	1U
+#define RGX_ACTIVEPM_DEFAULT	2U
+
+typedef IMG_UINT32 RGX_RD_POWER_ISLAND_CONF;
+#define RGX_RD_POWER_ISLAND_FORCE_OFF	0U
+#define RGX_RD_POWER_ISLAND_FORCE_ON	1U
+#define RGX_RD_POWER_ISLAND_DEFAULT		2U
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+/* Unused registers re-purposed for storing counters of the Firmware's
+ * interrupts for each OS
+ */
+#if (RGXFW_MAX_NUM_OSIDS == 8)
+#define IRQ_COUNTER_STORAGE_REGS                        \
+		0x2028U, /* RGX_CR_PM_TA_MMU_FSTACK         */  \
+		0x2050U, /* RGX_CR_PM_3D_MMU_FSTACK         */  \
+		0x2030U, /* RGX_CR_PM_START_OF_MMU_TACONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/
+#elif (RGXFW_MAX_NUM_OSIDS == 2)
+#define IRQ_COUNTER_STORAGE_REGS                        \
+		0x2028U, /* RGX_CR_PM_TA_MMU_FSTACK         */  \
+		0x2050U, /* RGX_CR_PM_3D_MMU_FSTACK         */
+#else
+#error Unsupported number of IRQ_COUNTER_STORAGE_REGS registers!
+#endif
+#endif
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SIGBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TRACEBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SYSDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_OSDATA;
+#if defined(SUPPORT_TBI_INTERFACE)
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TBIBUF;
+#endif
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERFBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRINFOBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RUNTIME_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_GPU_UTIL_FW;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_REG_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERF_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_MUX_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_SELECT_CUSTOM_CNTRS;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_RTN_SLOTS;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWCOMMONCONTEXT;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_ZSBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CORE_CLK_RATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COUNTERBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FIRMWAREGCOVBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRTDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TIMESTAMP_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*!
+ * @Brief Buffer to store KM active client contexts
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+} RGXFWIF_ACTIVE_CONTEXT_BUF_DATA;
+#endif
+
+/*!
+ * This number is used to represent an invalid page catalogue physical address
+ */
+#define RGXFWIF_INVALID_PC_PHYADDR 0xFFFFFFFFFFFFFFFFLLU
+
+/*!
+ * This number is used to represent an unallocated set of page catalog base registers
+ */
+#define RGXFW_BIF_INVALID_PCSET 0xFFFFFFFFU
+
+/*!
+ * This number is used to represent an invalid OS ID for the purpose of tracking PC set ownership
+ */
+#define RGXFW_BIF_INVALID_OSID 0xFFFFFFFFU
+
+/*!
+ * Firmware memory context.
+ */
+typedef struct
+{
+	IMG_DEV_PHYADDR			RGXFW_ALIGN sPCDevPAddr;	/*!< device physical address of context's page catalogue */
+	IMG_UINT32				uiPageCatBaseRegSet;	/*!< associated page catalog base register (RGXFW_BIF_INVALID_PCSET == unallocated) */
+	IMG_UINT32				uiBreakpointAddr; /*!< breakpoint address */
+	IMG_UINT32				uiBPHandlerAddr; /*!< breakpoint handler address */
+	IMG_UINT32				uiBreakpointCtl; /*!< DM and enable control for BP */
+	IMG_UINT32				ui32FwMemCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32              ui32OSid;
+	IMG_BOOL                bOSidAxiProt;
+#endif
+
+} UNCACHED_ALIGN RGXFWIF_FWMEMCONTEXT;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWMEMCONTEXT) == 32,
+				"RGXFWIF_FWMEMCONTEXT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * FW context state flags
+ */
+#define RGXFWIF_CONTEXT_FLAGS_NEED_RESUME				(0x00000001U)
+#define RGXFWIF_CONTEXT_FLAGS_MC_NEED_RESUME_MASKFULL	(0x000000FFU)
+#define RGXFWIF_CONTEXT_FLAGS_TDM_HEADER_STALE			(0x00000100U)
+#define RGXFWIF_CONTEXT_FLAGS_LAST_KICK_SECURE			(0x00000200U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend state (per GEOM core)
+ */
+typedef struct
+{
+	/* FW-accessible TA state which must be written out to memory on context store */
+	IMG_UINT64	RGXFW_ALIGN uTAReg_VDM_CALL_STACK_POINTER;		 /*!< VDM control stream stack pointer, to store in mid-TA */
+	IMG_UINT64	RGXFW_ALIGN uTAReg_VDM_CALL_STACK_POINTER_Init;	 /*!< Initial value of VDM control stream stack pointer (in case is 'lost' due to a lock-up) */
+	IMG_UINT16	ui16TACurrentIdx;
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE_PER_GEOM;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_GEOM_CORE_SIZE 4
+
+static_assert(RGX_NUM_GEOM_CORES <= MAX_GEOM_CORE_SIZE,
+				"RGX_NUM_GEOM_CORES is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend states for all GEOM cores
+ */
+typedef struct
+{
+	/*! FW-accessible TA state which must be written out to memory on context store */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[MAX_GEOM_CORE_SIZE];
+#else
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[RGX_NUM_GEOM_CORES];
+#endif
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_TACTX_STATE) == 160,
+				"RGXFWIF_TACTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware FRAG/3D context suspend state
+ */
+typedef struct
+{
+	/* FW-accessible ISP state which must be written out to memory on context store */
+	IMG_UINT32	u3DReg_PM_DEALLOCATED_MASK_STATUS; /*!< PM deallocation status */
+	IMG_UINT32	u3DReg_PM_PDS_MTILEFREE_STATUS; /*!< Macro-tiles (MTs) finished status */
+	IMG_UINT32	ui32CtxStateFlags;	/*!< Compatibility and other flags */
+	/* au3DReg_ISP_STORE should be the last element of the structure
+	 * as this is an array whose size is determined at runtime
+	 * after detecting the RGX core */
+	IMG_UINT32	au3DReg_ISP_STORE[]; /*!< ISP state (per-pipe) */
+} UNCACHED_ALIGN RGXFWIF_3DCTX_STATE;
+
+static_assert(sizeof(RGXFWIF_3DCTX_STATE) <= 16U,
+              "Size of structure RGXFWIF_3DCTX_STATE exceeds maximum expected size.");
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_3DCTX_STATE) == 16,
+				"RGXFWIF_3DCTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_CTX_USING_BUFFER_A		(0)
+#define RGXFWIF_CTX_USING_BUFFER_B		(1U)
+
+typedef struct
+{
+	IMG_UINT32	ui32CtxStateFlags; /*!< Target buffer and other flags */
+} RGXFWIF_COMPUTECTX_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_COMPUTECTX_STATE) == 4,
+				"RGXFWIF_COMPUTECTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_STATS_PENDING   (1U << 0)
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT (1U << 1)
+
+typedef struct
+{
+	IMG_UINT32	ui32ExtJobRefToDisableZSStore;
+	IMG_BOOL	bDisableZStore;
+	IMG_BOOL	bDisableSStore;
+} RGXFWIF_DISABLE_ZSSTORE;
+
+#define MAX_ZSSTORE_DISABLE 8
+
+typedef struct
+{
+	bool       bSaved;
+	IMG_UINT64 ui64CheckSum[4];
+}RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY;
+
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_TQ[RGX_TRP_MAX_NUM_CORES][1];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_2D[RGX_TRP_MAX_NUM_CORES][2];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_3D[RGX_TRP_MAX_NUM_CORES][4];
+typedef RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY RGXFWIF_TRP_CHECKSUM_GEOM[RGX_TRP_MAX_NUM_CORES];
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware render context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTAContext;				/*!< Firmware context for the TA */
+	RGXFWIF_FWCOMMONCONTEXT	s3DContext;				/*!< Firmware context for the 3D */
+
+	RGXFWIF_STATIC_RENDERCONTEXT_STATE sStaticRenderContextState;
+
+	RGXFWIF_DISABLE_ZSSTORE sDisableZSStoreQueue[MAX_ZSSTORE_DISABLE];
+
+	IMG_UINT32			ui32ZSStoreQueueCount;
+	IMG_UINT32			ui32WriteOffsetOfDisableZSStore;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32			ui32FwRenderCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_3D		aui64TRPChecksums3D;	/*!< Used by Firmware to store checksums during 3D WRR */
+	RGXFWIF_TRP_CHECKSUM_GEOM	aui64TRPChecksumsGeom;	/*!< Used by Firmware to store checksums during TA WRR */
+	RGXFWIF_DM			eTRPGeomCoreAffinity; /* !< Represent the DM affinity for pending 2nd TRP pass of GEOM otherwise points RGXFWIF_DM_MAX. */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWRENDERCONTEXT;
+
+/*!
+	Firmware compute context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT sCDMContext;				/*!< Firmware context for the CDM */
+
+	RGXFWIF_STATIC_COMPUTECONTEXT_STATE sStaticComputeContextState;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32 ui32ComputeCtxFlags; /*!< Compatibility and other flags */
+
+	IMG_UINT32		aui32WGPChecksum[RGX_WGP_MAX_NUM_CORES];
+} UNCACHED_ALIGN RGXFWIF_FWCOMPUTECONTEXT;
+
+/*!
+	Firmware TDM context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTDMContext;				/*!< Firmware context for the TDM */
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+} UNCACHED_ALIGN RGXFWIF_FWTDMCONTEXT;
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware transfer context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTQContext;			 /*!< Firmware context for TQ3D */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_TQ aui64TRPChecksumsTQ;/*!< Used by Firmware to store checksums during TQ WRR */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWTRANSFERCONTEXT;
+
+/*!
+ ******************************************************************************
+ * Defines for CMD_TYPE corruption detection and forward compatibility check
+ *****************************************************************************/
+
+/* CMD_TYPE 32bit contains:
+ * 31:16	Reserved for magic value to detect corruption (16 bits)
+ * 15		Reserved for RGX_CCB_TYPE_TASK (1 bit)
+ * 14:0		Bits available for CMD_TYPEs (15 bits) */
+
+
+/* Magic value to detect corruption */
+#define RGX_CMD_MAGIC_DWORD			IMG_UINT32_C(0x2ABC)
+#define RGX_CMD_MAGIC_DWORD_MASK	(0xFFFF0000U)
+#define RGX_CMD_MAGIC_DWORD_SHIFT	(16U)
+#define RGX_CMD_MAGIC_DWORD_SHIFTED	(RGX_CMD_MAGIC_DWORD << RGX_CMD_MAGIC_DWORD_SHIFT)
+
+/*!
+ * @InGroup KCCBTypes ClientCCBTypes
+ * @Brief Generic CCB control structure
+ */
+typedef struct
+{
+	volatile IMG_UINT32		ui32WriteOffset;		/*!< write offset into array of commands (MUST be aligned to 16 bytes!) */
+	volatile IMG_UINT32		ui32ReadOffset;			/*!< read offset into array of commands */
+	IMG_UINT32				ui32WrapMask;			/*!< Offset wrapping mask (Total capacity of the CCB - 1) */
+} UNCACHED_ALIGN RGXFWIF_CCB_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCB_CTL) == 16,
+				"RGXFWIF_CCB_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @Defgroup KCCBTypes Kernel CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the KCCB functionality
+ * @{
+ */
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PT      (0x1U) /* MMU_CTRL_INVAL_PT_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PD      (0x2U) /* MMU_CTRL_INVAL_PD_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PC      (0x4U) /* MMU_CTRL_INVAL_PC_EN */
+
+#if !defined(__KERNEL__)
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB   (0x10U) /* can't use PM_TLB0 bit from BIFPM_CTRL reg because it collides with PT bit from BIF_CTRL reg */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB | 0x8U) /* BIF_CTRL_INVAL_TLB1_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x0U) /* not used */
+
+#else
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB   (0x10U) /* can't use PM_TLB0 bit from BIFPM_CTRL reg because it collides with PT bit from BIF_CTRL reg */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB | 0x8U) /* BIF_CTRL_INVAL_TLB1_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x800U) /* MMU_CTRL_INVAL_ALL_CONTEXTS_EN */
+#endif
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT (0x4000000U) /* indicates FW should interrupt the host */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_MMUCACHE type command
+ */
+typedef struct
+{
+	IMG_UINT32            ui32CacheFlags;
+	RGXFWIF_DEV_VIRTADDR  sMMUCacheSync;
+	IMG_UINT32            ui32MMUCacheSyncUpdateValue;
+} RGXFWIF_MMUCACHEDATA;
+
+#define RGXFWIF_BPDATA_FLAGS_ENABLE (1U << 0)
+#define RGXFWIF_BPDATA_FLAGS_WRITE  (1U << 1)
+#define RGXFWIF_BPDATA_FLAGS_CTL    (1U << 2)
+#define RGXFWIF_BPDATA_FLAGS_REGS   (1U << 3)
+
+typedef struct
+{
+	PRGXFWIF_FWMEMCONTEXT		psFWMemContext;		/*!< Memory context */
+	IMG_UINT32					ui32BPAddr;			/*!< Breakpoint address */
+	IMG_UINT32					ui32HandlerAddr;	/*!< Breakpoint handler */
+	IMG_UINT32					ui32BPDM;			/*!< Breakpoint control */
+	IMG_UINT32					ui32BPDataFlags;
+	IMG_UINT32					ui32TempRegs;		/*!< Number of temporary registers to overallocate */
+	IMG_UINT32					ui32SharedRegs;		/*!< Number of shared registers to overallocate */
+	IMG_UINT64 RGXFW_ALIGN		ui64SpillAddr;
+	RGXFWIF_DM					eDM;				/*!< DM associated with the breakpoint */
+} RGXFWIF_BPDATA;
+
+#define RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS (RGXFWIF_PRBUFFER_MAXSUPPORTED + 1U) /* +1 is RTDATASET cleanup */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_KICK type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CWoffUpdate;	/*!< Client CCB woff update */
+	IMG_UINT32					ui32CWrapMaskUpdate; /*!< Client CCB wrap mask update after CCCB growth */
+	IMG_UINT32					ui32NumCleanupCtl;		/*!< number of CleanupCtl pointers attached */
+	PRGXFWIF_CLEANUP_CTL		apsCleanupCtl[RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS]; /*!< CleanupCtl structures associated with command */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32					ui32WorkEstCmdHeaderOffset; /*!< offset to the CmdHeader which houses the workload estimation kick data. */
+#endif
+} RGXFWIF_KCCB_CMD_KICK_DATA;
+
+/*!
+ * @Brief Command data for @Ref RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK type command
+ */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_KICK_DATA	sTACmdKickData; /*!< GEOM DM kick command data */
+	RGXFWIF_KCCB_CMD_KICK_DATA	s3DCmdKickData; /*!< FRAG DM kick command data */
+} RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FORCE_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CCBFenceOffset;	/*!< Client CCB fence offset */
+} RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;
+	RGXFWIF_DISABLE_ZSSTORE		sDisableZSStore;
+} RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA;
+
+/*!
+ * @Brief Resource types supported by \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef enum
+{
+	RGXFWIF_CLEANUP_FWCOMMONCONTEXT,		/*!< FW common context cleanup */
+	RGXFWIF_CLEANUP_HWRTDATA,				/*!< FW HW RT data cleanup */
+	RGXFWIF_CLEANUP_FREELIST,				/*!< FW freelist cleanup */
+	RGXFWIF_CLEANUP_ZSBUFFER,				/*!< FW ZS Buffer cleanup */
+} RGXFWIF_CLEANUP_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef struct
+{
+	RGXFWIF_CLEANUP_TYPE			eCleanupType;			/*!< Cleanup type */
+	union {
+		PRGXFWIF_FWCOMMONCONTEXT	psContext;				/*!< FW common context to cleanup */
+		PRGXFWIF_HWRTDATA			psHWRTData;				/*!< HW RT to cleanup */
+		PRGXFWIF_FREELIST			psFreelist;				/*!< Freelist to cleanup */
+		PRGXFWIF_ZSBUFFER			psZSBuffer;				/*!< ZS Buffer to cleanup */
+	} uCleanupData;
+} RGXFWIF_CLEANUP_REQUEST;
+
+/*!
+ * @Brief Type of power requests supported in \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef enum
+{
+	RGXFWIF_POW_OFF_REQ = 1,           /*!< GPU power-off request */
+	RGXFWIF_POW_FORCED_IDLE_REQ,       /*!< Force-idle related request */
+	RGXFWIF_POW_NUM_UNITS_CHANGE,      /*!< Request to change default powered scalable units */
+	RGXFWIF_POW_APM_LATENCY_CHANGE     /*!< Request to change the APM latency period */
+} RGXFWIF_POWER_TYPE;
+
+/*!
+ * @Brief Supported force-idle related requests with \ref RGXFWIF_POW_FORCED_IDLE_REQ type request
+ */
+typedef enum
+{
+	RGXFWIF_POWER_FORCE_IDLE = 1,      /*!< Request to force-idle GPU */
+	RGXFWIF_POWER_CANCEL_FORCED_IDLE,  /*!< Request to cancel a previously successful force-idle transition */
+	RGXFWIF_POWER_HOST_TIMEOUT,        /*!< Notification that host timed-out waiting for force-idle state */
+} RGXFWIF_POWER_FORCE_IDLE_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef struct
+{
+	RGXFWIF_POWER_TYPE					ePowType;					/*!< Type of power request */
+	union
+	{
+		IMG_UINT32						ui32PowUnits;			/*!< Number of active Dusts */
+		IMG_BOOL						bForced;				/*!< If the operation is mandatory */
+		RGXFWIF_POWER_FORCE_IDLE_TYPE	ePowRequestType;		/*!< Type of Request. Consolidating Force Idle, Cancel Forced Idle, Host Timeout */
+	} uPowerReqData;
+} RGXFWIF_POWER_REQUEST;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_SLCFLUSHINVAL type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext; /*!< Context to fence on (only useful when bDMContext == TRUE) */
+	IMG_BOOL    bInval;                 /*!< Invalidate the cache as well as flushing */
+	IMG_BOOL    bDMContext;             /*!< The data to flush/invalidate belongs to a specific DM context */
+	IMG_UINT64	RGXFW_ALIGN ui64Address;	/*!< Optional address of range (only useful when bDMContext == FALSE) */
+	IMG_UINT64	RGXFW_ALIGN ui64Size;		/*!< Optional size of range (only useful when bDMContext == FALSE) */
+} RGXFWIF_SLCFLUSHINVALDATA;
+
+typedef enum
+{
+	RGXFWIF_HWPERF_CTRL_TOGGLE = 0,
+	RGXFWIF_HWPERF_CTRL_SET    = 1,
+	RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV = 2
+} RGXFWIF_HWPERF_UPDATE_CONFIG;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG type command
+ */
+typedef struct
+{
+	RGXFWIF_HWPERF_UPDATE_CONFIG eOpCode; /*!< Control operation code */
+	IMG_UINT64	RGXFW_ALIGN	ui64Mask;   /*!< Mask of events to toggle */
+} RGXFWIF_HWPERF_CTRL;
+
+typedef struct
+{
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_MUX_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_MUX_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_MUX_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS type command
+ */
+typedef struct
+{
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_DA_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE type command
+ */
+typedef struct
+{
+	IMG_UINT32	ui32NewClockSpeed;			/*!< New clock speed */
+} RGXFWIF_CORECLKSPEEDCHANGE_DATA;
+
+#define RGXFWIF_HWPERF_CTRL_BLKS_MAX	16U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS type command
+ */
+typedef struct
+{
+	bool		bEnable;
+	IMG_UINT32	ui32NumBlocks;                              /*!< Number of block IDs in the array */
+	IMG_UINT16	aeBlockIDs[RGXFWIF_HWPERF_CTRL_BLKS_MAX];   /*!< Array of RGX_HWPERF_CNTBLK_ID values */
+} RGXFWIF_HWPERF_CTRL_BLKS;
+
+
+typedef struct
+{
+	IMG_UINT16                      ui16CustomBlock;
+	IMG_UINT16                      ui16NumCounters;
+	PRGX_HWPERF_SELECT_CUSTOM_CNTRS sCustomCounterIDs;
+} RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE & \ref RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE type commands
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;				/*!< ZS-Buffer FW address */
+	IMG_BOOL				bDone;								/*!< action backing/unbacking succeeded */
+} RGXFWIF_ZSBUFFER_BACKING_DATA;
+
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE type command
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;				/*!< Freelist FW address */
+	IMG_UINT32				ui32DeltaPages;						/*!< Amount of the Freelist change */
+	IMG_UINT32				ui32NewPages;						/*!< New amount of pages on the freelist (including ready pages) */
+	IMG_UINT32              ui32ReadyPages;                     /*!< Number of ready pages to be held in reserve until OOM */
+} RGXFWIF_FREELIST_GS_DATA;
+
+#define RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT         (MAX_HW_TA3DCONTEXTS * RGXFW_MAX_FREELISTS * 2U)
+#define RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG 0x80000000U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE type command
+ */
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+} RGXFWIF_FREELISTS_RECONSTRUCTION_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT  psContext; /*!< Context to that may need to be resumed following write offset update */
+} UNCACHED_ALIGN RGXFWIF_WRITE_OFFSET_UPDATE_DATA;
+
+/*!
+ ******************************************************************************
+ * Proactive DVFS Structures
+ *****************************************************************************/
+#define NUM_OPP_LEVELS 16
+
+typedef struct
+{
+	IMG_UINT32			ui32Volt; /* V  */
+	IMG_UINT32			ui32Freq; /* Hz */
+} UNCACHED_ALIGN OPP_LEVEL;
+
+typedef struct
+{
+	OPP_LEVEL		asOPPValues[NUM_OPP_LEVELS];
+#if defined(DEBUG)
+	IMG_UINT32		ui32MinOPPPoint;
+#endif
+	IMG_UINT32		ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_OPP_INFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MAX_FREQ_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32MinOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MIN_FREQ_DATA;
+
+/*!
+ ******************************************************************************
+ * Register configuration structures
+ *****************************************************************************/
+
+#define RGXFWIF_REG_CFG_MAX_SIZE 512
+
+typedef enum
+{
+	RGXFWIF_REGCFG_CMD_ADD				= 101,
+	RGXFWIF_REGCFG_CMD_CLEAR			= 102,
+	RGXFWIF_REGCFG_CMD_ENABLE			= 103,
+	RGXFWIF_REGCFG_CMD_DISABLE			= 104
+} RGXFWIF_REGDATA_CMD_TYPE;
+
+typedef struct
+{
+	IMG_UINT64		ui64Addr;
+	IMG_UINT64		ui64Mask;
+	IMG_UINT64		ui64Value;
+} RGXFWIF_REG_CFG_REC;
+
+typedef struct
+{
+	RGXFWIF_REGDATA_CMD_TYPE         eCmdType;
+	RGXFWIF_REG_CFG_TYPE             eRegConfigType;
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  sRegConfig;
+
+} RGXFWIF_REGCONFIG_DATA;
+
+typedef struct
+{
+	/**
+	 * PDump WRW command write granularity is 32 bits.
+	 * Add padding to ensure array size is 32 bit granular.
+	 */
+	IMG_UINT8           RGXFW_ALIGN  aui8NumRegsType[PVR_ALIGN((IMG_UINT32)RGXFWIF_REG_CFG_TYPE_ALL,sizeof(IMG_UINT32))];
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  asRegConfigs[RGXFWIF_REG_CFG_MAX_SIZE];
+} UNCACHED_ALIGN RGXFWIF_REG_CFG;
+
+typedef enum
+{
+	RGXFWIF_OS_ONLINE = 1,
+	RGXFWIF_OS_OFFLINE
+} RGXFWIF_OS_STATE_CHANGE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE type command
+ */
+typedef struct
+{
+	IMG_UINT32 ui32DriverID;
+	RGXFWIF_OS_STATE_CHANGE eNewOSState;
+} UNCACHED_ALIGN RGXFWIF_OS_STATE_CHANGE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext;
+	IMG_UINT32               ui32FirstIntJobRefToCancel;
+	IMG_UINT32               ui32LastIntJobRefToCancel;
+} UNCACHED_ALIGN RGXFWIF_CANCEL_WORK_DATA;
+
+typedef enum
+{
+	RGXFWIF_PWR_COUNTER_DUMP_START = 1,
+	RGXFWIF_PWR_COUNTER_DUMP_STOP,
+	RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+} RGXFWIF_COUNTER_DUMP_REQUEST;
+
+typedef struct
+{
+	RGXFWIF_COUNTER_DUMP_REQUEST eCounterDumpRequest;
+}  RGXFW_ALIGN RGXFWIF_COUNTER_DUMP_DATA;
+
+/*!
+ * @Brief List of command types supported by the Kernel CCB
+ */
+typedef enum
+{
+	/* Common commands */
+	RGXFWIF_KCCB_CMD_KICK								= 101U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< DM workload kick command */
+	RGXFWIF_KCCB_CMD_MMUCACHE							= 102U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< MMU cache invalidation request */
+	RGXFWIF_KCCB_CMD_BP									= 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_SLCFLUSHINVAL						= 104U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< SLC flush and invalidation request */
+	RGXFWIF_KCCB_CMD_CLEANUP							= 105U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Requests cleanup of a FW resource (type specified in the command data) */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE			= 106U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Backing for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE			= 107U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Unbacking for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE				= 108U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelist Grow done */
+	RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE	= 109U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelists Reconstruction done */
+	RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE			= 110U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the firmware that the host has added more data to a CDM2 Circular Buffer */
+	RGXFWIF_KCCB_CMD_HEALTH_CHECK						= 111U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Health check request */
+	RGXFWIF_KCCB_CMD_FORCE_UPDATE						= 112U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Forcing signalling of all unmet UFOs for a given CCB offset */
+	RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK				= 113U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< There is a TA and a 3D command in this single kick */
+	RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE			= 114U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW that a Guest OS has come online / offline. */
+	RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE					= 115U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW to disable zs store of a running 3D or add it to queue of render context. */
+	RGXFWIF_KCCB_CMD_CANCEL_WORK						= 116U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Cancel all work up to and including a given intjobref for a given context */
+	/* Commands only permitted to the native or host OS */
+	RGXFWIF_KCCB_CMD_POW								= 200U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Power request */
+	RGXFWIF_KCCB_CMD_REGCONFIG							= 201U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE					= 202U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Core clock speed change event */
+	RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE						= 203U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Ask the firmware to update its cached ui32LogType value from the (shared) tracebuf control structure */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ				= 204U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a maximum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE				= 205U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the priority/group for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL					= 206U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set or clear firmware state flags */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MIN_FREQ				= 207U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a minimum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_PHR_CFG							= 208U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Periodic Hardware Reset behaviour */
+	RGXFWIF_KCCB_CMD_WDG_CFG							= 210U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Safety Firmware Watchdog */
+	RGXFWIF_KCCB_CMD_COUNTER_DUMP						= 211U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Controls counter dumping in the FW */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE					= 213U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL			= 214U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice interval for all drivers. It can only be serviced for the Host DDK */
+
+	/* HWPerf commands */
+	RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG				= 300U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure HWPerf events (to be generated) and HWPerf buffer address (if required) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS					= 301U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure directly addressable counters for HWPerf */
+	RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS					= 302U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Enable or disable multiple HWPerf blocks (reusing existing configuration) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS			= 303U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure, clear and enable multiple HWPerf blocks */
+	RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS			= 304U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure the custom counters for HWPerf */
+
+} RGXFWIF_KCCB_CMD_TYPE;
+
+#define RGXFWIF_LAST_ALLOWED_GUEST_KCCB_CMD (RGXFWIF_KCCB_CMD_POW - 1)
+
+/*! @Brief Kernel CCB command packet */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_TYPE  eCmdType;      /*!< Command type */
+	IMG_UINT32             ui32KCCBFlags; /*!< Compatibility and other flags */
+
+	/* NOTE: Make sure that uCmdData is the last member of this struct
+	 * This is to calculate actual command size for device mem copy.
+	 * (Refer RGXGetCmdMemCopySize())
+	 * */
+	union
+	{
+		RGXFWIF_KCCB_CMD_KICK_DATA			sCmdKickData;			/*!< Data for Kick command */
+		RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA	sCombinedTA3DCmdKickData;	/*!< Data for combined TA/3D Kick command */
+		RGXFWIF_MMUCACHEDATA				sMMUCacheData;			/*!< Data for MMU cache command */
+		RGXFWIF_BPDATA						sBPData;				/*!< Data for Breakpoint Commands */
+		RGXFWIF_SLCFLUSHINVALDATA			sSLCFlushInvalData;		/*!< Data for SLC Flush/Inval commands */
+		RGXFWIF_CLEANUP_REQUEST				sCleanupData;			/*!< Data for cleanup commands */
+		RGXFWIF_POWER_REQUEST				sPowData;				/*!< Data for power request commands */
+		RGXFWIF_HWPERF_CTRL					sHWPerfCtrl;			/*!< Data for HWPerf control command */
+		RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS	sHWPerfCfgEnableBlks;	/*!< Data for HWPerf configure, clear and enable performance counter block command */
+		RGXFWIF_HWPERF_CTRL_BLKS			sHWPerfCtrlBlks;		/*!< Data for HWPerf enable or disable performance counter block commands */
+		RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS	sHWPerfSelectCstmCntrs;	/*!< Data for HWPerf configure the custom counters to read */
+		RGXFWIF_HWPERF_CONFIG_DA_BLKS		sHWPerfCfgDABlks;		/*!< Data for HWPerf configure Directly Addressable blocks */
+		RGXFWIF_CORECLKSPEEDCHANGE_DATA		sCoreClkSpeedChangeData;/*!< Data for core clock speed change */
+		RGXFWIF_ZSBUFFER_BACKING_DATA		sZSBufferBackingData;	/*!< Feedback for Z/S Buffer backing/unbacking */
+		RGXFWIF_FREELIST_GS_DATA			sFreeListGSData;		/*!< Feedback for Freelist grow/shrink */
+		RGXFWIF_FREELISTS_RECONSTRUCTION_DATA	sFreeListsReconstructionData;	/*!< Feedback for Freelists reconstruction */
+		RGXFWIF_REGCONFIG_DATA				sRegConfigData;			/*!< Data for custom register configuration */
+		RGXFWIF_WRITE_OFFSET_UPDATE_DATA    sWriteOffsetUpdateData; /*!< Data for informing the FW about the write offset update */
+		RGXFWIF_PDVFS_MAX_FREQ_DATA			sPDVFSMaxFreqData;		/*!< Data for setting the max frequency/OPP */
+		RGXFWIF_PDVFS_MIN_FREQ_DATA			sPDVFSMinFreqData;		/*!< Data for setting the min frequency/OPP */
+		RGXFWIF_OS_STATE_CHANGE_DATA        sCmdOSOnlineStateData;  /*!< Data for updating the Guest Online states */
+		RGXFWIF_DEV_VIRTADDR                sTBIBuffer;             /*!< Dev address for TBI buffer allocated on demand */
+		RGXFWIF_COUNTER_DUMP_DATA			sCounterDumpConfigData; /*!< Data for dumping of register ranges */
+		RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA	sDisableZSStoreData;	/*!< Data for disabling zs store of a 3D workload */
+		RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA  sForceUpdateData;       /*!< Data for signalling all unmet fences for a given CCB */
+		RGXFWIF_CANCEL_WORK_DATA			sCancelWorkData;		/*!< Data for cancelling work */
+	} UNCACHED_ALIGN uCmdData;
+} UNCACHED_ALIGN RGXFWIF_KCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_KCCB_CMD);
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_KCCB_CMD) == 64,
+				"RGXFWIF_KCCB_CMD is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of KCCBTypes */
+
+/*!
+ * @Defgroup FWCCBTypes Firmware CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the Firmware CCB functionality
+ * @{
+ */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING and the
+ * \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING Firmware CCB commands
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32ZSBufferID; /*!< ZS buffer ID */
+} RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELIST_GROW Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32FreelistID; /*!< Freelist ID */
+} RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;                                     /*!< Freelists count */
+	IMG_UINT32			ui32HwrCounter;                                         /*!< HWR counter */
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT]; /*!< Array of freelist IDs to reconstruct */
+} RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA;
+
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF			(1U<<0)	/*!< 1 if a page fault happened */
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS	(1U<<1)	/*!< 1 if applicable to all contexts */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32						ui32ServerCommonContextID;	/*!< Context affected by the reset */
+	RGX_CONTEXT_RESET_REASON		eResetReason;				/*!< Reason for reset */
+	RGXFWIF_DM						eDM;						/*!< Data Master affected by the reset */
+	IMG_UINT32						ui32ResetJobRef;			/*!< Job ref running at the time of reset */
+	IMG_UINT32						ui32Flags;					/*!< RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG bitfield */
+	IMG_UINT64 RGXFW_ALIGN			ui64PCAddress;				/*!< At what page catalog address */
+	IMG_DEV_VIRTADDR RGXFW_ALIGN	sFaultAddress;				/*!< Page fault address (only when applicable) */
+} RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;	/*!< Page fault address */
+} RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA;
+
+/*!
+ ******************************************************************************
+ * List of command types supported by the Firmware CCB
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING              = 101U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be backed with physical pages
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING            = 102U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be unbacked
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_FREELIST_GROW                 = 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand freelist grow
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA */
+	RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION      = 104U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests freelists reconstruction
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA */
+	RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION    = 105U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a HWR event on a context
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA */
+	RGXFWIF_FWCCB_CMD_DEBUG_DUMP                    = 106U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand debug dump
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS                  = 107U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand update on process stats
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA */
+
+	RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE          = 108U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART           = 109U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests GPU restart
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION    = 112U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a FW pagefault
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA */
+} RGXFWIF_FWCCB_CMD_TYPE;
+
+/*!
+ ******************************************************************************
+ * List of the various stats of the process to update/increment
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS=1,		/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumPartialRenders stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumOutOfMemory stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTAStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32Num3DStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumCDMStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTDMStores stat */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_UPDATE_STATS Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE		eElementToUpdate;			/*!< Element to update */
+	IMG_PID									pidOwner;					/*!< The pid of the process whose stats are being updated */
+	IMG_INT32								i32AdjustmentValue;			/*!< Adjustment to be made to the statistic */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32CoreClkRate;
+} UNCACHED_ALIGN RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA;
+
+
+/*!
+ ******************************************************************************
+ * @Brief Firmware CCB command structure
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_TYPE  eCmdType;       /*!< Command type */
+	IMG_UINT32              ui32FWCCBFlags; /*!< Compatibility and other flags */
+
+	union
+	{
+		RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA				sCmdZSBufferBacking;			/*!< Data for Z/S-Buffer on-demand (un)backing*/
+		RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA					sCmdFreeListGS;					/*!< Data for on-demand freelist grow/shrink */
+		RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA		sCmdFreeListsReconstruction;	/*!< Data for freelists reconstruction */
+		RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA				sCmdContextResetNotification;	/*!< Data for context reset notification */
+		RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA					sCmdUpdateStatsData;			/*!< Data for updating process stats */
+		RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA			sCmdCoreClkRateChange;
+		RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA					sCmdFWPagefault;				/*!< Data for context reset notification */
+	} RGXFW_ALIGN uCmdData;
+} RGXFW_ALIGN RGXFWIF_FWCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_FWCCB_CMD);
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWCCB_CMD) == 48,
+				"RGXFWIF_FWCCB_CMD is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of FWCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Workload estimation Firmware CCB command structure for RGX
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT16 ui16ReturnDataIndex; /*!< Index for return data array */
+	IMG_UINT16 ui16CyclesTakenHigh;	/*!< Not used on Rogue GPU cores. */
+	IMG_UINT32 ui32CyclesTaken;     /*!< The cycles the workload took on the hardware */
+} RGXFWIF_WORKEST_FWCCB_CMD;
+
+/*!
+ * @Defgroup ClientCCBTypes Client CCB data interface
+ * @Brief Types grouping data structures and defines used in realising Client CCB commands/functionality
+ * @{
+ */
+
+/* Required memory alignment for 64-bit variables accessible by Meta
+  (The gcc meta aligns 64-bit variables to 64-bit; therefore, memory shared
+   between the host and meta that contains 64-bit variables has to maintain
+   this alignment) */
+#define RGXFWIF_FWALLOC_ALIGN	sizeof(IMG_UINT64)
+
+#define RGX_CCB_TYPE_TASK			(IMG_UINT32_C(1) << 15)
+#define RGX_CCB_FWALLOC_ALIGN(size)	(PVR_ALIGN(size, RGXFWIF_FWALLOC_ALIGN))
+
+typedef IMG_UINT32 RGXFWIF_CCB_CMD_TYPE;
+
+/*!
+ * @Name Client CCB command types
+ * @{
+ */
+#define RGXFWIF_CCB_CMD_TYPE_GEOM			(201U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TA DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_3D			(202U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_3D				(203U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command */
+#define RGXFWIF_CCB_CMD_TYPE_3D_PR			(204U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for Partial render */
+#define RGXFWIF_CCB_CMD_TYPE_CDM			(205U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Compute DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_TDM			(206U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TDM command */
+#define RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE (207U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_TQ_2D			(208U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 2D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP	(209U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_NULL			(210U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_ABORT			(211U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+
+/* Leave a gap between CCB specific commands and generic commands */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE          (212U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a command */
+#define RGXFWIF_CCB_CMD_TYPE_UPDATE         (213U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates of a command */
+#define RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE     (214U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates related to workload resources */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE_PR       (215U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a PR command */
+#define RGXFWIF_CCB_CMD_TYPE_PRIORITY       (216U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Context priority update command */
+/* Pre and Post timestamp commands are supposed to sandwich the DM cmd. The
+   padding code with the CCB wrap upsets the FW if we don't have the task type
+   bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+*/
+#define RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP (217U | RGX_CMD_MAGIC_DWORD_SHIFTED)
+/* UNFENCED type is not supported but keeping defines as it is for now */
+
+
+#define RGXFWIF_CCB_CMD_TYPE_PADDING	(221U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Skip without action type command */
+#define RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP	(223U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Process a vulkan timestamp */
+/*! @} End of Client CCB command types */
+
+
+#define RGXFWIF_TRP_STATUS_UNKNOWN				0x000U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_OK			0x001U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_ERROR		0x002U
+
+#define RGXFWIF_CR_TRP_SIGNATURE_STATUS			(RGX_CR_SCRATCH10)
+
+
+typedef struct
+{
+	/* Index for the KM Workload estimation return data array */
+	IMG_UINT16 RGXFW_ALIGN         ui16ReturnDataIndex;
+	/* Predicted time taken to do the work in cycles */
+	IMG_UINT32 RGXFW_ALIGN         ui32CyclesPrediction;
+	/* Deadline for the workload (in usecs) */
+	IMG_UINT64 RGXFW_ALIGN         ui64Deadline;
+} RGXFWIF_WORKEST_KICK_DATA;
+
+/*! @Brief Command header of a command in the client CCB buffer.
+ *
+ *  Followed by this header is the command-data specific to the
+ *  command-type as specified in the header.
+ */
+typedef struct
+{
+	RGXFWIF_CCB_CMD_TYPE					eCmdType;      /*!< Command data type following this command header */
+	IMG_UINT32								ui32CmdSize;   /*!< Size of the command following this header */
+	IMG_UINT32								ui32ExtJobRef; /*!< external job reference - provided by client and used in debug for tracking submitted work */
+	IMG_UINT32								ui32IntJobRef; /*!< internal job reference - generated by services and used in debug for tracking submitted work */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_WORKEST_KICK_DATA RGXFW_ALIGN	sWorkEstKickData; /*!< Workload Estimation - Workload Estimation Data */
+#endif
+} RGXFWIF_CCB_CMD_HEADER;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCB_CMD_HEADER) == 16,
+				"RGXFWIF_CCB_CMD_HEADER is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*
+ ******************************************************************************
+ * Client CCB commands which are only required by the kernel
+ *****************************************************************************/
+
+/*! @Brief Command data for \ref RGXFWIF_CCB_CMD_TYPE_PRIORITY type client CCB command */
+typedef struct
+{
+	IMG_INT32              i32Priority; /*!< Priority level */
+} RGXFWIF_CMD_PRIORITY;
+
+/*! @} End of ClientCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Signature and Checksums Buffer
+ *****************************************************************************/
+typedef struct
+{
+	PRGXFWIF_SIGBUFFER		sBuffer;			/*!< Ptr to Signature Buffer memory */
+	IMG_UINT32				ui32LeftSizeInRegs;	/*!< Amount of space left for storing regs in the buffer */
+} UNCACHED_ALIGN RGXFWIF_SIGBUF_CTL;
+
+typedef struct
+{
+	PRGXFWIF_COUNTERBUFFER	sBuffer;			/*!< Ptr to counter dump buffer */
+	IMG_UINT32				ui32SizeInDwords;	/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_COUNTER_DUMP_CTL;
+
+typedef struct
+{
+	PRGXFWIF_FIRMWAREGCOVBUFFER	sBuffer;		/*!< Ptr to firmware gcov buffer */
+	IMG_UINT32					ui32Size;		/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_FIRMWARE_GCOV_CTL;
+
+/*!
+ *****************************************************************************
+ * RGX Compatibility checks
+ *****************************************************************************/
+
+/* WARNING: Whenever the layout of RGXFWIF_COMPCHECKS_BVNC changes, the
+	following define should be increased by 1 to indicate to the
+	compatibility logic that layout has changed. */
+#define RGXFWIF_COMPCHECKS_LAYOUT_VERSION 3
+
+typedef struct
+{
+	IMG_UINT32	ui32LayoutVersion; /* WARNING: This field must be defined as first one in this structure */
+	IMG_UINT64	RGXFW_ALIGN ui64BVNC;
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS_BVNC;
+
+typedef struct
+{
+	IMG_UINT8	ui8OsCountSupport;
+} UNCACHED_ALIGN RGXFWIF_INIT_OPTIONS;
+
+#define RGXFWIF_COMPCHECKS_BVNC_DECLARE_AND_INIT(name) \
+	RGXFWIF_COMPCHECKS_BVNC (name) = { \
+		RGXFWIF_COMPCHECKS_LAYOUT_VERSION, \
+		0, \
+	}
+#define RGXFWIF_COMPCHECKS_BVNC_INIT(name) \
+	do { \
+		(name).ui32LayoutVersion = RGXFWIF_COMPCHECKS_LAYOUT_VERSION; \
+		(name).ui64BVNC = 0; \
+	} while (false)
+
+typedef struct
+{
+	RGXFWIF_COMPCHECKS_BVNC		sHWBVNC;				/*!< hardware BVNC (from the RGX registers) */
+	RGXFWIF_COMPCHECKS_BVNC		sFWBVNC;				/*!< firmware BVNC */
+	IMG_UINT32					ui32DDKVersion;			/*!< software DDK version */
+	IMG_UINT32					ui32DDKBuild;			/*!< software DDK build no. */
+	IMG_UINT32					ui32BuildOptions;		/*!< build options bit-field */
+	RGXFWIF_INIT_OPTIONS		sInitOptions;			/*!< initialisation options bit-field */
+	IMG_BOOL					bUpdated;				/*!< Information is valid */
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS;
+
+typedef struct
+{
+	IMG_UINT32                 ui32NumCores;
+	IMG_UINT64 RGXFW_ALIGN     aui64MultiCoreCapabilities[RGX_MULTICORE_MAX_NOHW_CORES];
+} UNCACHED_ALIGN RGXFWIF_MULTICORE_INFO;
+
+/*! @Brief Firmware Runtime configuration data \ref RGXFWIF_RUNTIME_CFG
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	IMG_UINT32         ui32ActivePMLatencyms;                    /*!< APM latency in ms before signalling IDLE to the host */
+	IMG_UINT32         ui32RuntimeCfgFlags;                      /*!< Compatibility and other flags */
+	IMG_BOOL           bActivePMLatencyPersistant;               /*!< If set, APM latency does not reset to system default each GPU power transition */
+	IMG_UINT32         ui32CoreClockSpeed;                       /*!< Core clock speed, currently only used to calculate timer ticks */
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32         ui32SOCClockSpeed;                        /*!< SOC clock speed, used for deadline scheduling */
+#endif
+	IMG_UINT32         ui32PowUnitsState;                        /*!< Last number of dusts change requested by the host */
+	IMG_UINT32         ui32PHRMode;                              /*!< Periodic Hardware Reset configuration values */
+	IMG_UINT32         ui32HCSDeadlineMS;                        /*!< New number of milliseconds C/S is allowed to last */
+	IMG_UINT32         ui32WdgPeriodUs;                          /*!< The watchdog period in microseconds */
+	IMG_INT32          ai32DriverPriority[RGXFW_MAX_NUM_OSIDS];  /*!< Array of priorities per OS */
+	IMG_UINT32         aui32DriverIsolationGroup[RGXFW_MAX_NUM_OSIDS]; /*!< Array of isolation groups per OS */
+	IMG_UINT32         aui32TSPercentage[RGXFW_MAX_NUM_OSIDS];   /*!< Array of time slice per OS */
+	IMG_UINT32         ui32TSIntervalMs;                         /*!< Time slice interval */
+	IMG_UINT32         ui32VzConnectionCooldownPeriodInSec;      /*!< Vz Connection Cooldown period in secs */
+
+	PRGXFWIF_HWPERFBUF sHWPerfBuf;                               /*!< On-demand allocated HWPerf buffer address, to be passed to the FW */
+	RGXFWIF_DMA_ADDR   sHWPerfDMABuf;
+	RGXFWIF_DMA_ADDR   sHWPerfCtlDMABuf;
+} RGXFWIF_RUNTIME_CFG;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_RUNTIME_CFG) == 68,
+				"RGXFWIF_RUNTIME_CFG is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ *****************************************************************************
+ * Control data for RGX
+ *****************************************************************************/
+
+#define RGXFWIF_HWR_DEBUG_DUMP_ALL (99999U)
+
+#if defined(PDUMP)
+
+#define RGXFWIF_PID_FILTER_MAX_NUM_PIDS 32U
+
+typedef enum
+{
+	RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+	RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT
+} RGXFWIF_PID_FILTER_MODE;
+
+typedef struct
+{
+	IMG_PID uiPID;
+	IMG_UINT32 ui32DriverID;
+} RGXFW_ALIGN RGXFWIF_PID_FILTER_ITEM;
+
+typedef struct
+{
+	RGXFWIF_PID_FILTER_MODE eMode;
+	/* each process in the filter list is specified by a PID and OS ID pair.
+	 * each PID and OS pair is an item in the items array (asItems).
+	 * if the array contains less than RGXFWIF_PID_FILTER_MAX_NUM_PIDS entries
+	 * then it must be terminated by an item with pid of zero.
+	 */
+	RGXFWIF_PID_FILTER_ITEM asItems[RGXFWIF_PID_FILTER_MAX_NUM_PIDS];
+} RGXFW_ALIGN RGXFWIF_PID_FILTER;
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA  (0x1U << 0)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE  (0x1U << 1)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE  (0x1U << 2)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE     (0x1U << 3)
+#endif
+
+typedef enum
+{
+	RGXFWIF_TPU_DM_PDM = 0,
+	RGXFWIF_TPU_DM_VDM = 1,
+	RGXFWIF_TPU_DM_CDM = 2,
+	RGXFWIF_TPU_DM_TDM = 3,
+	RGXFWIF_TPU_DM_LAST
+} RGXFWIF_TPU_DM;
+
+typedef enum
+{
+	RGXFWIF_GPIO_VAL_OFF           = 0, /*!< No GPIO validation */
+	RGXFWIF_GPIO_VAL_GENERAL       = 1, /*!< Simple test case that
+	                                         initiates by sending data via the
+	                                         GPIO and then sends back any data
+	                                         received over the GPIO */
+	RGXFWIF_GPIO_VAL_AP            = 2, /*!< More complex test case that writes
+	                                         and reads data across the entire
+	                                         GPIO AP address range.*/
+#if defined(SUPPORT_STRIP_RENDERING)
+	RGXFWIF_GPIO_VAL_SR_BASIC      = 3, /*!< Strip Rendering AP based basic test.*/
+	RGXFWIF_GPIO_VAL_SR_COMPLEX    = 4, /*!< Strip Rendering AP based complex test.*/
+#endif
+	RGXFWIF_GPIO_VAL_TESTBENCH     = 5, /*!< Validates the GPIO Testbench. */
+	RGXFWIF_GPIO_VAL_LOOPBACK      = 6, /*!< Send and then receive each byte
+	                                         in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LOOPBACK_LITE = 7, /*!< Send and then receive each power-of-2
+	                                         byte in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LAST
+} RGXFWIF_GPIO_VAL_MODE;
+
+typedef IMG_UINT32 FW_PERF_CONF;
+#define FW_PERF_CONF_NONE			0U
+#define FW_PERF_CONF_ICACHE			1U
+#define FW_PERF_CONF_DCACHE			2U
+#define FW_PERF_CONF_JTLB_INSTR		5U
+#define FW_PERF_CONF_INSTRUCTIONS	6U
+
+typedef enum
+{
+	FW_BOOT_STAGE_TLB_INIT_FAILURE = -2,
+	FW_BOOT_STAGE_NOT_AVAILABLE = -1,
+	FW_BOOT_NOT_STARTED = 0,
+	FW_BOOT_BLDR_STARTED = 1,
+	FW_BOOT_CACHE_DONE,
+	FW_BOOT_TLB_DONE,
+	FW_BOOT_MAIN_STARTED,
+	FW_BOOT_ALIGNCHECKS_DONE,
+	FW_BOOT_INIT_DONE,
+} FW_BOOT_STAGE;
+
+/*!
+ * @AddToGroup KCCBTypes
+ * @{
+ * @Name Kernel CCB return slot responses
+ * @{
+ * Usage of bit-fields instead of bare integers
+ * allows FW to possibly pack-in several responses for each single kCCB command.
+ */
+
+#define RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED   (1U << 0) /*!< Command executed (return status from FW) */
+#define RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY   (1U << 1) /*!< A cleanup was requested but resource busy */
+#define RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE   (1U << 2) /*!< Poll failed in FW for a HW operation to complete */
+
+#define RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE            0x0U      /*!< Reset value of a kCCB return slot (set by host) */
+/*!
+ * @} End of Name Kernel CCB return slot responses
+ * @} End of AddToGroup KCCBTypes
+ */
+
+/*! @Brief OS connection data \ref RGXFWIF_CONNECTION_CTL allocated
+ * by services and used to track OS state in Firmware and Services
+ **/
+typedef struct
+{
+	/* Fw-Os connection states */
+	volatile RGXFWIF_CONNECTION_FW_STATE eConnectionFwState;    /*!< Firmware-OS connection state */
+	volatile RGXFWIF_CONNECTION_OS_STATE eConnectionOsState;    /*!< Services-OS connection state */
+	volatile IMG_UINT32                  ui32AliveFwToken;      /*!< OS Alive token updated by Firmware */
+	volatile IMG_UINT32                  ui32AliveOsToken;      /*!< OS Alive token updated by Services */
+} UNCACHED_ALIGN RGXFWIF_CONNECTION_CTL;
+
+/*! @Brief Firmware OS Initialization data \ref RGXFWIF_OSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	/* Kernel CCB */
+	PRGXFWIF_CCB_CTL        psKernelCCBCtl; /*!< Kernel CCB Control */
+	PRGXFWIF_CCB            psKernelCCB; /*!<  Kernel CCB */
+	PRGXFWIF_CCB_RTN_SLOTS  psKernelCCBRtnSlots; /*!<  Kernel CCB return slots */
+
+	/* Firmware CCB */
+	PRGXFWIF_CCB_CTL        psFirmwareCCBCtl; /*!<  Firmware CCB control */
+	PRGXFWIF_CCB            psFirmwareCCB; /*!<  Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	PRGXFWIF_CCB_CTL        psWorkEstFirmwareCCBCtl; /*!<  Workload estimation control */
+	PRGXFWIF_CCB            psWorkEstFirmwareCCB; /*!<  Workload estimation buffer */
+
+	PRGXFWIF_HWRINFOBUF     sRGXFWIfHWRInfoBufCtl; /*!<  HWRecoveryInfo control */
+
+	IMG_UINT32              ui32HWRDebugDumpLimit; /*!< Firmware debug dump maximum limit */
+
+	PRGXFWIF_OSDATA         sFwOsData; /*!<  Firmware per-os shared data */
+
+	RGXFWIF_COMPCHECKS      sRGXCompChecks; /*!< Compatibility checks to be populated by the Firmware */
+	RGXFWIF_MULTICORE_INFO  sRGXMulticoreInfo; /*! < Multicore capability info */
+
+} UNCACHED_ALIGN RGXFWIF_OSINIT;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_OSINIT) == 104,
+				"RGXFWIF_OSINIT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @Brief Firmware System Initialization data \ref RGXFWIF_SYSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	RGX_MIPS_STATE          sMIPSState; /*!< MIPS Debug Data; this must be the first member in the structure */
+
+	IMG_DEV_PHYADDR         RGXFW_ALIGN sFaultPhysAddr; /*!< Fault read address */
+
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sPDSExecBase; /*!< PDS execution base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sUSCExecBase; /*!< USC execution base */
+
+	IMG_UINT64              RGXFW_ALIGN ui64HWPerfFilter; /*! Event filter for Firmware events */
+
+	IMG_UINT32              RGXFW_ALIGN aui32TPUTrilinearFracMask[RGXFWIF_TPU_DM_LAST];
+
+#if defined(PDUMP)
+	RGXFWIF_SIGBUF_CTL      asSigBufCtl[RGXFWIF_DM_MAX]; /*!< Signature and Checksum Buffers for DMs */
+#endif
+
+	RGXFWIF_DMA_ADDR        sCorememDataStore; /*!< Firmware coremem data */
+
+	RGXFWIF_COUNTER_DUMP_CTL sCounterDumpCtl;
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	RGXFWIF_FIRMWARE_GCOV_CTL sFirmwareGcovCtl; /*!< Firmware gcov buffer control */
+#endif
+
+	IMG_UINT32              ui32FilterFlags;
+
+	PRGXFWIF_RUNTIME_CFG    sRuntimeCfg; /*!<  Firmware Runtime configuration */
+
+	PRGXFWIF_TRACEBUF       sTraceBufCtl; /*!<  Firmware Trace buffer control */
+	PRGXFWIF_SYSDATA        sFwSysData; /*!< Firmware System shared data */
+#if defined(SUPPORT_TBI_INTERFACE)
+	PRGXFWIF_TBIBUF         sTBIBuf; /*!< Tbi log buffer */
+#endif
+
+	PRGXFWIF_GPU_UTIL_FW    sGpuUtilFWCtl; /*!< GPU utilization buffer */
+	PRGXFWIF_REG_CFG        sRegCfg; /*!< Firmware register user configuration */
+	PRGXFWIF_HWPERF_CTL     sHWPerfCtl; /*!< HWPerf counter block configuration.*/
+
+	RGXFWIF_DEV_VIRTADDR    sAlignChecks; /*!< Array holding Server structures alignment data */
+
+	IMG_UINT32              ui32InitialCoreClockSpeed; /*!< Core clock speed at FW boot time */
+
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32              ui32InitialSOCClockSpeed; /*!< System/SOC clock speed at FW boot time */
+#endif
+
+	IMG_UINT32              ui32InitialActivePMLatencyms; /*!< APM latency in ms before signalling IDLE to the host */
+
+	IMG_BOOL                bFirmwareStarted; /*!< Flag to be set by the Firmware after successful start */
+
+	IMG_UINT32              ui32MarkerVal; /*!< Host/FW Trace synchronisation Partition Marker */
+
+	IMG_UINT32              ui32FirmwareStartedTimeStamp; /*!< Firmware initialization complete time */
+
+	FW_PERF_CONF            eFirmwarePerf; /*!< Firmware performance counter config */
+
+#if defined(SUPPORT_FW_OPP_TABLE)
+	RGXFWIF_OPP_INFO        RGXFW_ALIGN sOPPInfo;
+
+	/**
+	 * FW Pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory when running on non primary FW thread
+	 * to communicate to host driver.
+	 */
+	PRGXFWIF_CORE_CLK_RATE  RGXFW_ALIGN sCoreClockRate;
+#endif
+
+#if defined(PDUMP)
+	RGXFWIF_PID_FILTER      sPIDFilter;
+#endif
+
+	RGXFWIF_GPIO_VAL_MODE   eGPIOValidationMode;
+
+	RGX_HWPERF_BVNC         sBvncKmFeatureFlags; /*!< Used in HWPerf for decoding BVNC Features*/
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32              ui32SecurityTestFlags;
+	RGXFWIF_DEV_VIRTADDR    pbSecureBuffer;
+	RGXFWIF_DEV_VIRTADDR    pbNonSecureBuffer;
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	RGXFWIF_DEV_VIRTADDR    sActiveContextBufBase; /*!< Active context buffer base */
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	/*
+	 * Used when validation is enabled to allow the host to check
+	 * that MTS sent the correct sideband in response to a kick
+	 * from a given OSes schedule register.
+	 * Testing is enabled if RGXFWIF_KICK_TEST_ENABLED_BIT is set
+	 *
+	 * Set by the host to:
+	 * (osid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT
+	 * reset to 0 by FW when kicked by the given OSid
+	 */
+	IMG_UINT32              ui32OSKickTest;
+#endif
+
+	/* Value to write into RGX_CR_TFBC_COMPRESSION_CONTROL */
+	IMG_UINT32              ui32TFBCCompressionControl;
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_UINT32              ui32VzWdgPeriod;
+#endif
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	/* notify firmware power-up on host-side recovery */
+	IMG_BOOL                bFwHostRecoveryMode;
+#endif
+} UNCACHED_ALIGN RGXFWIF_SYSINIT;
+
+static_assert(offsetof(RGXFWIF_SYSINIT, sMIPSState) == 0,
+				"sMIPSState is not the first member of the RGXFWIF_SYSINIT struct");
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+#define RGXFWIF_KICK_TEST_ENABLED_BIT  0x1
+#define RGXFWIF_KICK_TEST_OSID_SHIFT   0x1
+#endif
+
+/*!
+ *****************************************************************************
+ * Timer correlation shared data and defines
+ *****************************************************************************/
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64OSMonoTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeStamp;
+
+	/* Utility variable used to convert CR timer deltas to OS timer deltas (nS),
+	 * where the deltas are relative to the timestamps above:
+	 * deltaOS = (deltaCR * K) >> decimal_shift, see full explanation below */
+	IMG_UINT64 RGXFW_ALIGN ui64CRDeltaToOSDeltaKNs;
+
+	IMG_UINT32             ui32CoreClockSpeed;
+	IMG_UINT32             ui32Reserved;
+} UNCACHED_ALIGN RGXFWIF_TIME_CORR;
+
+
+/* The following macros are used to help converting FW timestamps to the Host
+ * time domain. On the FW the RGX_CR_TIMER counter is used to keep track of
+ * time; it increments by 1 every 256 GPU clock ticks, so the general
+ * formula to perform the conversion is:
+ *
+ * [ GPU clock speed in Hz, if (scale == 10^9) then deltaOS is in nS,
+ *   otherwise if (scale == 10^6) then deltaOS is in uS ]
+ *
+ *             deltaCR * 256                                   256 * scale
+ *  deltaOS = --------------- * scale = deltaCR * K    [ K = --------------- ]
+ *             GPUclockspeed                                  GPUclockspeed
+ *
+ * The actual K is multiplied by 2^20 (and deltaCR * K is divided by 2^20)
+ * to get some better accuracy and to avoid returning 0 in the integer
+ * division 256000000/GPUfreq if GPUfreq is greater than 256MHz.
+ * This is the same as keeping K as a decimal number.
+ *
+ * The maximum deltaOS is slightly more than 5hrs for all GPU frequencies
+ * (deltaCR * K is more or less a constant), and it's relative to the base
+ * OS timestamp sampled as a part of the timer correlation data.
+ * This base is refreshed on GPU power-on, DVFS transition and periodic
+ * frequency calibration (executed every few seconds if the FW is doing
+ * some work), so as long as the GPU is doing something and one of these
+ * events is triggered then deltaCR * K will not overflow and deltaOS will be
+ * correct.
+ */
+
+#define RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT  (20)
+
+#define RGXFWIF_GET_DELTA_OSTIME_NS(deltaCR, K) \
+	(((deltaCR) * (K)) >> RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT)
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation
+ *****************************************************************************/
+
+/* See rgx_common.h for a list of GPU states */
+#define RGXFWIF_GPU_UTIL_TIME_MASK       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_TIME_MASK32     (IMG_UINT32_C(0xFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+#define RGXFWIF_GPU_UTIL_GET_TIME(word)    ((word) & RGXFWIF_GPU_UTIL_TIME_MASK)
+#define RGXFWIF_GPU_UTIL_GET_STATE(word)   ((word) & RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_GET_TIME32(word)  ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_TIME_MASK32)
+#define RGXFWIF_GPU_UTIL_GET_STATE32(word) ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+/* The OS timestamps computed by the FW are approximations of the real time,
+ * which means they could be slightly behind or ahead the real timer on the Host.
+ * In some cases we can perform subtractions between FW approximated
+ * timestamps and real OS timestamps, so we need a form of protection against
+ * negative results if for instance the FW one is a bit ahead of time.
+ */
+#define RGXFWIF_GPU_UTIL_GET_PERIOD(newtime,oldtime) \
+	(((newtime) > (oldtime)) ? ((newtime) - (oldtime)) : 0U)
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME(time) | RGXFWIF_GPU_UTIL_GET_STATE(state))
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD32(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME32(time) | RGXFWIF_GPU_UTIL_GET_STATE32(state))
+
+
+/* The timer correlation array must be big enough to ensure old entries won't be
+ * overwritten before all the HWPerf events linked to those entries are processed
+ * by the MISR. The update frequency of this array depends on how fast the system
+ * can change state (basically how small the APM latency is) and perform DVFS transitions.
+ *
+ * The minimum size is 2 (not 1) to avoid race conditions between the FW reading
+ * an entry while the Host is updating it. With 2 entries in the worst case the FW
+ * will read old data, which is still quite ok if the Host is updating the timer
+ * correlation at that time.
+ */
+#define RGXFWIF_TIME_CORR_ARRAY_SIZE            256U
+#define RGXFWIF_TIME_CORR_CURR_INDEX(seqcount)  ((seqcount) % RGXFWIF_TIME_CORR_ARRAY_SIZE)
+
+/* Make sure the timer correlation array size is a power of 2 */
+static_assert((RGXFWIF_TIME_CORR_ARRAY_SIZE & (RGXFWIF_TIME_CORR_ARRAY_SIZE - 1U)) == 0U,
+			  "RGXFWIF_TIME_CORR_ARRAY_SIZE must be a power of two");
+
+/* The time is stored in DM state time-stamps, and as a result in DMs states counters, in "approximately microseconds",
+ * dividing the time originally obtained in nanoseconds by 2^10 for the sake of reducing coremem usage */
+#define RGXFWIF_DM_OS_TIMESTAMP_SHIFT    10U
+
+typedef struct
+{
+	/* Last GPU DM per-OS states + OS time of the last state update */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWord[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* DMs time-stamps are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWordWrap[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* Counters for the amount of time the GPU DMs were active or inactive(idle or blocked) */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSStatsCounters[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	/* DMs Counters are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSCountersWrap[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+} RGXFWIF_GPU_STATS;
+
+typedef struct
+{
+	RGXFWIF_TIME_CORR      sTimeCorr[RGXFWIF_TIME_CORR_ARRAY_SIZE];
+	IMG_UINT32             ui32TimeCorrSeqCount;
+
+	/* Compatibility and other flags */
+	IMG_UINT32             ui32GpuUtilFlags;
+
+	/* Last GPU state + OS time of the last state update */
+	IMG_UINT64 RGXFW_ALIGN ui64GpuLastWord;
+	/* Counters for the amount of time the GPU was active/idle/blocked */
+	IMG_UINT64 RGXFW_ALIGN aui64GpuStatsCounters[RGXFWIF_GPU_UTIL_STATE_NUM];
+	/* Device off period timestamp offset */
+	IMG_INT64 RGXFW_ALIGN i64DeviceTimestampOffset;
+	/* Stats per OSID/DriverID */
+	RGXFWIF_GPU_STATS sStats[RGXFW_MAX_NUM_OSIDS];
+} UNCACHED_ALIGN RGXFWIF_GPU_UTIL_FW;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_GPU_UTIL_FW) == 12584,
+				"RGXFWIF_GPU_UTIL_FW is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES      RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES      (RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      PVR_ALIGN((RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES), DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY)
+#else
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      0
+#endif
+
+#define RGXFWIF_TDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES)
+#define RGXFWIF_CDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES)
+
+/*!
+ ******************************************************************************
+ * Virtualisation and Security
+ *****************************************************************************/
+#if defined(SECURE_FW_CODE_OSID)
+/* software workaround for SoCs without fw_code, fw_priv_data signals, MIPS only */
+#if defined(RGX_FEATURE_META)
+#error "SECURE_FW_CODE_OSID is not supported on META cores"
+#elif defined(RGX_FEATURE_RISCV_FW_PROCESSOR)
+#error "SECURE_FW_CODE_OSID is not supported on RISC-V cores"
+#elif (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#error "SECURE_FW_CODE_OSID is not supported on virtualization drivers"
+#elif (SECURE_FW_CODE_OSID + 1 > 2)
+#define MIPS_FW_CODE_OSID				(SECURE_FW_CODE_OSID)
+#else
+#define MIPS_FW_CODE_OSID				(1U)
+#endif
+#endif /* defined(SECURE_FW_CODE_OSID) */
+
+static_assert((RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID),
+			  " Invalid RGX_FW_HEAP_OSID_ASSIGNMENT value. Rogue cores support only the RGX_FW_HEAP_USES_FIRMWARE_OSID config");
+
+/* Firmware and Host driver share the same OSID */
+#define FW_HEAP_OSID					(FW_OSID)
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* virtualization without security support */
+#define DRIVER_ID(osid)					(osid)
+#define OSID(did)						(did)
+/* Time slice support */
+/* Bits 30 and 31 reserved by FW private driver priority */
+#define RGXFW_VZ_PRIORITY_MAX_SHIFT	(30U)
+#define RGXFW_VZ_PRIORITY_MASK		((1U << RGXFW_VZ_PRIORITY_MAX_SHIFT) - 1U)
+#define RGXFW_VZ_TIME_SLICE_MAX		(100U)
+#define RGXFW_VZ_TIME_SLICE_MIN		(5U)
+#else
+#define DRIVER_ID(osid)					(0U)
+#define OSID(did)						(did)
+#define RGXFW_VZ_PRIORITY_MASK			(0U)
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) < RGX_NUM_DRIVERS_SUPPORTED; (did)++)
+
+#if defined(__KERNEL__)
+/* Driver implementation */
+#define FOREACH_ACTIVE_DRIVER(devinfo, did)        RGXFwSharedMemCacheOpValue(psFwSysData->asOsRuntimeFlagsMirror[RGXFW_HOST_DRIVER_ID], \
+                                                                              INVALIDATE); \
+                                                   FOREACH_SUPPORTED_DRIVER(did)                                \
+                                                   {                                                            \
+                                                   if (devinfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[did].bfOsState != RGXFW_CONNECTION_FW_ACTIVE) continue;
+
+#define END_FOREACH_ACTIVE_DRIVER                  }
+
+#else
+/* Firmware implementation */
+#define FOREACH_ACTIVE_DRIVER(did)                 do {                                                                      \
+                                                   unsigned int idx;                                                         \
+                                                   for (idx = 0; idx < RGXFW_NUM_ACTIVE_DRIVERS; idx++)                      \
+                                                   {                                                                         \
+                                                       (did) = gsRGXFWCtl.aui32ActiveDrivers[idx];                           \
+                                                       {
+
+#define END_FOREACH_ACTIVE_DRIVER                  }}} while (false);
+#endif /* defined(__KERNEL__) */
+
+
+#else
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) <= RGXFW_HOST_DRIVER_ID; (did)++)
+
+#define FOREACH_ACTIVE_DRIVER(did)                 FOREACH_SUPPORTED_DRIVER(did)
+#define END_FOREACH_ACTIVE_DRIVER
+
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#define FOREACH_VALIDATION_OSID(osid)              for ((osid)=0; (osid) < GPUVIRT_VALIDATION_NUM_OS; (osid)++)
+#define FOREACH_HW_OSID(osid)                      for ((osid)=0; (osid) < RGXFW_MAX_NUM_OSIDS; (osid)++)
+#define FOREACH_DRIVER_RAW_HEAP(did, _struct, dev) for ((did)=RGX_FIRST_RAW_HEAP_DRIVER_ID; (did) < ((PVRSRV_VZ_MODE_IS(NATIVE, _struct, dev) ? 1 : RGX_NUM_DRIVERS_SUPPORTED)); (did)++)
+
+#endif /* RGX_FWIF_KM_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_shared.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_shared.h
new file mode 100644
index 000000000000..ccab45ae8fc0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_fwif_shared.h
@@ -0,0 +1,1140 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures shared by both host client
+                and host server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_SHARED_H)
+#define RGX_FWIF_SHARED_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+#include "devicemem_typedefs.h"
+
+/* Indicates the number of RTDATAs per RTDATASET */
+#if defined(SUPPORT_AGP)
+#define RGXMKIF_NUM_RTDATAS           4U
+#define RGXMKIF_NUM_GEOMDATAS         4U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  12U /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (2U)
+#else
+#define RGXMKIF_NUM_RTDATAS           2U
+#define RGXMKIF_NUM_GEOMDATAS         1U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  2U  /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (1U)
+#endif
+
+/* Maximum number of UFOs in a CCB command.
+ * The number is based on having 32 sync prims (as originally), plus 32 sync
+ * checkpoints.
+ * Once the use of sync prims is no longer supported, we will retain
+ * the same total (64) as the number of sync checkpoints which may be
+ * supporting a fence is not visible to the client driver and has to
+ * allow for the number of different timelines involved in fence merges.
+ */
+#define RGXFWIF_CCB_CMD_MAX_UFOS			(32U+32U)
+
+/*
+ * This is a generic limit imposed on any DM (TA,3D,CDM,TDM,2D,TRANSFER)
+ * command passed through the bridge.
+ * Just across the bridge in the server, any incoming kick command size is
+ * checked against this maximum limit.
+ * In case the incoming command size is larger than the specified limit,
+ * the bridge call is retired with error.
+ */
+#define RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE	(1024U)
+
+typedef struct
+{
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN psDevVirtAddr;
+	RGXFWIF_DEV_VIRTADDR    pbyFWAddr;
+} UNCACHED_ALIGN RGXFWIF_DMA_ADDR;
+
+typedef IMG_UINT8	RGXFWIF_CCCB;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_UFO_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CLEANUP_CTL;
+
+
+/*!
+ * @InGroup ClientCCBTypes
+ * @Brief Command data for fence & update types Client CCB commands.
+ */
+typedef struct
+{
+	PRGXFWIF_UFO_ADDR	puiAddrUFO; /*!< Address to be checked/updated */
+	IMG_UINT32			ui32Value;  /*!< Value to check-against/update-to */
+} RGXFWIF_UFO;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Track pending and executed workloads of HWRTDATA and ZSBUFFER
+ */
+typedef struct
+{
+	IMG_UINT32			ui32SubmittedCommands;	/*!< Number of commands received by the FW */
+	IMG_UINT32			ui32ExecutedCommands;	/*!< Number of commands executed by the FW */
+} UNCACHED_ALIGN RGXFWIF_CLEANUP_CTL;
+
+#define	RGXFWIF_PRBUFFER_START        IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_ZSBUFFER     IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_MSAABUFFER   IMG_UINT32_C(1)
+#define	RGXFWIF_PRBUFFER_MAXSUPPORTED IMG_UINT32_C(2)
+
+typedef IMG_UINT32 RGXFWIF_PRBUFFER_TYPE;
+
+typedef enum
+{
+	RGXFWIF_PRBUFFER_UNBACKED = 0,
+	RGXFWIF_PRBUFFER_BACKED,
+	RGXFWIF_PRBUFFER_BACKING_PENDING,
+	RGXFWIF_PRBUFFER_UNBACKING_PENDING,
+}RGXFWIF_PRBUFFER_STATE;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief OnDemand Z/S/MSAA Buffers
+ */
+typedef struct
+{
+	IMG_UINT32		ui32BufferID;		/*!< Buffer ID*/
+	IMG_BOOL		bOnDemand;		/*!< Needs On-demand Z/S/MSAA Buffer allocation */
+	RGXFWIF_PRBUFFER_STATE	eState;			/*!< Z/S/MSAA -Buffer state */
+	RGXFWIF_CLEANUP_CTL	sCleanupState;		/*!< Cleanup state */
+	IMG_UINT32		ui32PRBufferFlags;	/*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_PRBUFFER;
+
+/*
+ * Used to share frame numbers across UM-KM-FW,
+ * frame number is set in UM,
+ * frame number is required in both KM for HTB and FW for FW trace.
+ *
+ * May be used to house Kick flags in the future.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32FrameNum; /*!< associated frame number */
+} CMD_COMMON;
+
+/*
+ * TA and 3D commands require set of firmware addresses that are stored in the
+ * Kernel. Client has handle(s) to Kernel containers storing these addresses,
+ * instead of raw addresses. We have to patch/write these addresses in KM to
+ * prevent UM from controlling FW addresses directly.
+ * Typedefs for TA and 3D commands are shared between Client and Firmware (both
+ * single-BVNC). Kernel is implemented in a multi-BVNC manner, so it can't use
+ * TA|3D CMD type definitions directly. Therefore we have a SHARED block that
+ * is shared between UM-KM-FW across all BVNC configurations.
+ */
+typedef struct
+{
+	CMD_COMMON           sCmn;      /*!< Common command attributes */
+	RGXFWIF_DEV_VIRTADDR sHWRTData; /* RTData associated with this command,
+									   this is used for context selection and for storing out HW-context,
+									   when TA is switched out for continuing later */
+
+	RGXFWIF_DEV_VIRTADDR asPRBuffer[RGXFWIF_PRBUFFER_MAXSUPPORTED];	/* Supported PR Buffers like Z/S/MSAA Scratch */
+
+} CMDTA3D_SHARED;
+
+/*!
+ * Client Circular Command Buffer (CCCB) control structure.
+ * This is shared between the Server and the Firmware and holds byte offsets
+ * into the CCCB as well as the wrapping mask to aid wrap around. A given
+ * snapshot of this queue with Cmd 1 running on the GPU might be:
+ *
+ *          Roff                           Doff                 Woff
+ * [..........|-1----------|=2===|=3===|=4===|~5~~~~|~6~~~~|~7~~~~|..........]
+ *            <      runnable commands       ><   !ready to run   >
+ *
+ * Cmd 1    : Currently executing on the GPU data master.
+ * Cmd 2,3,4: Fence dependencies met, commands runnable.
+ * Cmd 5... : Fence dependency not met yet.
+ */
+typedef struct
+{
+	IMG_UINT32  ui32WriteOffset;    /*!< Host write offset into CCB. This
+	                                 *    must be aligned to 16 bytes. */
+	IMG_UINT32  ui32ReadOffset;     /*!< Firmware read offset into CCB.
+	                                      Points to the command that is
+	                                 *    runnable on GPU, if R!=W */
+	IMG_UINT32  ui32DepOffset;      /*!< Firmware fence dependency offset.
+	                                 *    Points to commands not ready, i.e.
+	                                 *    fence dependencies are not met. */
+	IMG_UINT32  ui32WrapMask;       /*!< Offset wrapping mask, total capacity
+	                                      in bytes of the CCB-1 */
+
+	IMG_UINT32  ui32ReadOffset2;     /*!< Firmware 2nd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R2!=W */
+	IMG_UINT32  ui32ReadOffset3;     /*!< Firmware 3rd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R3!=W */
+	IMG_UINT32  ui32ReadOffset4;     /*!< Firmware 4th read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R4!=W */
+
+} UNCACHED_ALIGN RGXFWIF_CCCB_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCCB_CTL) == 32,
+				"RGXFWIF_CCCB_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef IMG_UINT32 RGXFW_FREELIST_TYPE;
+
+#define RGXFW_LOCAL_FREELIST     IMG_UINT32_C(0)
+#define RGXFW_GLOBAL_FREELIST    IMG_UINT32_C(1)
+#if defined(SUPPORT_AGP)
+#define RGXFW_GLOBAL2_FREELIST   IMG_UINT32_C(2)
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL2_FREELIST + 1U)
+#else
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL_FREELIST + 1U)
+#endif
+#define RGXFW_MAX_HWFREELISTS    (2U)
+
+/*!
+ * @Defgroup ContextSwitching Context switching data interface
+ * @Brief Types grouping data structures and defines used in realising the Context Switching (CSW) functionality
+ * @{
+ */
+
+/*!
+ * @Brief GEOM DM or TA register controls for context switch
+ */
+typedef struct
+{
+	IMG_UINT64	uTAReg_VDM_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the VDM's context state buffer */
+	IMG_UINT64	uTAReg_VDM_CONTEXT_STATE_RESUME_ADDR;
+	IMG_UINT64	uTAReg_TA_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the TA's context state buffer */
+
+	struct
+	{
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK0; /*!< VDM context store task 0 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK1; /*!< VDM context store task 1 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK2; /*!< VDM context store task 2 */
+
+		/* VDM resume state update controls */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK0; /*!< VDM context resume task 0 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK1; /*!< VDM context resume task 1 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK2; /*!< VDM context resume task 2 */
+	} asTAState[2];
+
+} RGXFWIF_TAREGISTERS_CSWITCH;
+/*! @} End of Defgroup ContextSwitching */
+
+#define RGXFWIF_TAREGISTERS_CSWITCH_SIZE sizeof(RGXFWIF_TAREGISTERS_CSWITCH)
+
+typedef struct
+{
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS1;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS1;
+
+	/* CDM resume controls */
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0_B;
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0_B;
+
+} RGXFWIF_CDM_REGISTERS_CSWITCH;
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Render context static register controls for context switch
+ */
+typedef struct
+{
+	RGXFWIF_TAREGISTERS_CSWITCH	RGXFW_ALIGN asCtxSwitch_GeomRegs[RGX_NUM_GEOM_CORES];	/*!< Geom registers for ctx switch */
+} RGXFWIF_STATIC_RENDERCONTEXT_STATE;
+
+#define RGXFWIF_STATIC_RENDERCONTEXT_SIZE sizeof(RGXFWIF_STATIC_RENDERCONTEXT_STATE)
+
+typedef struct
+{
+	RGXFWIF_CDM_REGISTERS_CSWITCH	RGXFW_ALIGN sCtxSwitch_Regs;	/*!< CDM registers for ctx switch */
+} RGXFWIF_STATIC_COMPUTECONTEXT_STATE;
+
+#define RGXFWIF_STATIC_COMPUTECONTEXT_SIZE sizeof(RGXFWIF_STATIC_COMPUTECONTEXT_STATE)
+
+/*!
+	@Brief Context reset reason. Last reset reason for a reset context.
+*/
+typedef enum
+{
+	RGX_CONTEXT_RESET_REASON_NONE                = 0,	/*!< No reset reason recorded */
+	RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP       = 1,	/*!< Caused a reset due to locking up */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP     = 2,	/*!< Affected by another context locking up */
+	RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING   = 3,	/*!< Overran the global deadline */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING = 4,	/*!< Affected by another context overrunning */
+	RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH = 5,	/*!< Forced reset to ensure scheduling requirements */
+	RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM        = 6,	/*!< CDM Mission/safety checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM        = 7,	/*!< TRP checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_OK          = 8,	/*!< GPU ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR         = 9,	/*!< GPU ECC error (uncorrected, HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_OK           = 10,	/*!< FW ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_ERR          = 11,	/*!< FW ECC error (uncorrected, ERR) */
+	RGX_CONTEXT_RESET_REASON_FW_WATCHDOG         = 12,	/*!< FW Safety watchdog triggered */
+	RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT        = 13,	/*!< FW page fault (no HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR         = 14,	/*!< FW execution error (GPU reset requested) */
+	RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR     = 15,	/*!< Host watchdog detected FW error */
+	RGX_CONTEXT_GEOM_OOM_DISABLED                = 16,	/*!< Geometry DM OOM event is not allowed */
+	RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH         = 17,	/*!< PVRIC Signature mismatch */
+	RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR   = 18,	/*!< Parity error in MMU Page Table Entry */
+	RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR       = 19,	/*!< Parity error in MH, system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR      = 20,	/*!< Parity error in system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR      = 21,	/*!< Latent/ICS signature mismatch error */
+	RGX_CONTEXT_RESET_REASON_DCLS_ERR            = 22,	/*!< Dual Core Lock Step FW error detected */
+} RGX_CONTEXT_RESET_REASON;
+
+/*!
+	@Brief Context reset data shared with the host
+*/
+typedef struct
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reset reason */
+	IMG_UINT32 ui32ResetExtJobRef;  /*!< External Job ID */
+} RGX_CONTEXT_RESET_REASON_DATA;
+
+#define RGX_HEAP_UM_PDS_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_PDS_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_PDS_RESERVED_TOTAL_SIZE            RGX_HEAP_UM_PDS_RESERVED_SIZE
+
+#define RGX_HEAP_UM_USC_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_USC_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET      RGX_HEAP_UM_USC_RESERVED_SIZE
+
+#define RGX_HEAP_UM_GENERAL_RESERVED_SIZE           DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_GENERAL_RESERVED_REGION_OFFSET  0
+#define RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET  RGX_HEAP_UM_GENERAL_RESERVED_SIZE
+
+/*************************************************************************/ /*!
+ Logging type
+*/ /**************************************************************************/
+#define RGXFWIF_LOG_TYPE_NONE			0x00000000U
+#define RGXFWIF_LOG_TYPE_TRACE			0x00000001U
+#define RGXFWIF_LOG_TYPE_GROUP_MAIN		0x00000002U
+#define RGXFWIF_LOG_TYPE_GROUP_MTS		0x00000004U
+#define RGXFWIF_LOG_TYPE_GROUP_CLEANUP	0x00000008U
+#define RGXFWIF_LOG_TYPE_GROUP_CSW		0x00000010U
+#define RGXFWIF_LOG_TYPE_GROUP_BIF		0x00000020U
+#define RGXFWIF_LOG_TYPE_GROUP_PM		0x00000040U
+#define RGXFWIF_LOG_TYPE_GROUP_RTD		0x00000080U
+#define RGXFWIF_LOG_TYPE_GROUP_SPM		0x00000100U
+#define RGXFWIF_LOG_TYPE_GROUP_POW		0x00000200U
+#define RGXFWIF_LOG_TYPE_GROUP_HWR		0x00000400U
+#define RGXFWIF_LOG_TYPE_GROUP_HWP		0x00000800U
+#define RGXFWIF_LOG_TYPE_GROUP_RPM		0x00001000U
+#define RGXFWIF_LOG_TYPE_GROUP_DMA		0x00002000U
+#define RGXFWIF_LOG_TYPE_GROUP_MISC		0x00004000U
+#define RGXFWIF_LOG_TYPE_GROUP_VZ		0x00008000U
+#define RGXFWIF_LOG_TYPE_GROUP_SAFETY	0x00010000U
+#define RGXFWIF_LOG_TYPE_GROUP_VERBOSE	0x00020000U
+#define RGXFWIF_LOG_TYPE_GROUP_CUSTOMER	0x00040000U
+#define RGXFWIF_LOG_TYPE_GROUP_DEBUG	0x80000000U
+#define RGXFWIF_LOG_TYPE_GROUP_MASK		0x8007FFFEU
+#define RGXFWIF_LOG_TYPE_MASK			0x8007FFFFU
+
+/* String used in pvrdebug -h output */
+#define RGXFWIF_LOG_GROUPS_STRING_LIST   "main,mts,cleanup,csw,bif,pm,rtd,spm,pow,hwr,hwp,rpm,dma,misc,vz,safety,verbose,customer,debug"
+
+/* Table entry to map log group strings to log type value */
+typedef struct {
+	const IMG_CHAR* pszLogGroupName;
+	IMG_UINT32      ui32LogGroupType;
+} RGXFWIF_LOG_GROUP_MAP_ENTRY;
+
+/*
+  Macro for use with the RGXFWIF_LOG_GROUP_MAP_ENTRY type to create a lookup
+  table where needed. Keep log group names short, no more than 20 chars.
+*/
+#define RGXFWIF_LOG_GROUP_NAME_VALUE_MAP { "none",    RGXFWIF_LOG_TYPE_NONE }, \
+                                         { "main",    RGXFWIF_LOG_TYPE_GROUP_MAIN }, \
+                                         { "mts",     RGXFWIF_LOG_TYPE_GROUP_MTS }, \
+                                         { "cleanup", RGXFWIF_LOG_TYPE_GROUP_CLEANUP }, \
+                                         { "csw",     RGXFWIF_LOG_TYPE_GROUP_CSW }, \
+                                         { "bif",     RGXFWIF_LOG_TYPE_GROUP_BIF }, \
+                                         { "pm",      RGXFWIF_LOG_TYPE_GROUP_PM }, \
+                                         { "rtd",     RGXFWIF_LOG_TYPE_GROUP_RTD }, \
+                                         { "spm",     RGXFWIF_LOG_TYPE_GROUP_SPM }, \
+                                         { "pow",     RGXFWIF_LOG_TYPE_GROUP_POW }, \
+                                         { "hwr",     RGXFWIF_LOG_TYPE_GROUP_HWR }, \
+                                         { "hwp",     RGXFWIF_LOG_TYPE_GROUP_HWP }, \
+                                         { "rpm",     RGXFWIF_LOG_TYPE_GROUP_RPM }, \
+                                         { "dma",     RGXFWIF_LOG_TYPE_GROUP_DMA }, \
+                                         { "misc",    RGXFWIF_LOG_TYPE_GROUP_MISC }, \
+                                         { "vz",      RGXFWIF_LOG_TYPE_GROUP_VZ }, \
+                                         { "safety",  RGXFWIF_LOG_TYPE_GROUP_SAFETY }, \
+                                         { "verbose", RGXFWIF_LOG_TYPE_GROUP_VERBOSE }, \
+                                         { "customer",RGXFWIF_LOG_TYPE_GROUP_CUSTOMER }, \
+                                         { "debug",   RGXFWIF_LOG_TYPE_GROUP_DEBUG }
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s"
+
+/* Used in a print statement to display log group state, one per group */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST(types)  ((((types) & RGXFWIF_LOG_TYPE_GROUP_MAIN) != 0U)	?("main ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MTS) != 0U)		?("mts ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CLEANUP) != 0U)	?("cleanup ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CSW) != 0U)		?("csw ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_BIF) != 0U)		?("bif ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_PM) != 0U)		?("pm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RTD) != 0U)		?("rtd ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SPM) != 0U)		?("spm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_POW) != 0U)		?("pow ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWR) != 0U)		?("hwr ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWP) != 0U)		?("hwp ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RPM) != 0U)		?("rpm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DMA) != 0U)		?("dma ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MISC) != 0U)	?("misc ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VZ) != 0U)		?("vz ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SAFETY) != 0U)	?("safety ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VERBOSE) != 0U)	?("verbose ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CUSTOMER) != 0U)?("customer ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DEBUG) != 0U)	?("debug ")		:(""))
+
+/*!
+ ******************************************************************************
+ * Trace Buffer
+ *****************************************************************************/
+
+/*! Min, Max, and Default size of RGXFWIF_TRACEBUF_SPACE in DWords */
+#define RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS      8192U  /*  32KB */
+#define RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS 12000U  /* ~48KB */
+#define RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS     32768U  /* 128KB */
+
+#define RGXFW_TRACE_BUFFER_ASSERT_SIZE 200U
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+#define RGXFW_THREAD_NUM 2U
+#else
+#define RGXFW_THREAD_NUM 1U
+#endif
+
+typedef struct
+{
+	IMG_CHAR	szPath[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_CHAR	szInfo[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_UINT32	ui32LineNum;
+} UNCACHED_ALIGN RGXFWIF_FILE_INFO_BUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FILE_INFO_BUF) == 408,
+				"RGXFWIF_FILE_INFO_BUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @Defgroup SRVAndFWTracing Services and Firmware Tracing data interface
+ * @Brief The document groups/lists the data structures and the interfaces related to Services and Firmware Tracing
+ * @{
+ */
+
+/*!
+ * @Brief Firmware trace buffer details
+ */
+typedef struct
+{
+	IMG_UINT32                         ui32TracePointer;          /*!< Trace pointer (write index into Trace Buffer) */
+	IMG_UINT32                         ui32WrapCount;             /*!< Number of times the Trace Buffer has wrapped */
+
+	RGXFWIF_DEV_VIRTADDR               pui32RGXFWIfTraceBuffer;   /*!< Trace buffer address (FW address), to be used by firmware for writing into trace buffer */
+
+	RGXFWIF_FILE_INFO_BUF RGXFW_ALIGN  sAssertBuf;
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF_SPACE;
+
+/*! @} End of Defgroup SRVAndFWTracing */
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief Firmware trace control data
+ */
+typedef struct
+{
+	IMG_UINT32              ui32LogType;                  /*!< FW trace log group configuration */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[MAX_THREAD_NUM];  /*!< FW Trace buffer */
+#else
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[RGXFW_THREAD_NUM];  /*!< FW Trace buffer */
+#endif
+	IMG_UINT32              ui32TraceBufSizeInDWords;     /*!< FW Trace buffer size in dwords, Member initialised only when sTraceBuf is actually allocated
+															(in RGXTraceBufferInitOnDemandResources) */
+	IMG_UINT32              ui32TracebufFlags;            /*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_TRACEBUF) == 880,
+				"RGXFWIF_TRACEBUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Debug-info sub-fields */
+/* Bit 0: RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT bit from RGX_CR_EVENT_STATUS register */
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT        (0U)
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET          (1U << RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT)
+
+/* Bit 1: RGX_CR_BIF_MMU_ENTRY_PENDING bit from RGX_CR_BIF_MMU_ENTRY register */
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT     (1U)
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET       (1U << RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT)
+
+/* Bit 2: RGX_CR_SLAVE_EVENT register is non-zero */
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT          (2U)
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET            (1U << RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT)
+
+/* Bit 3-15: Unused bits */
+
+#define RGXFWT_DEBUG_INFO_STR_MAXLEN                  64
+#define RGXFWT_DEBUG_INFO_STR_PREPEND                 " (debug info: "
+#define RGXFWT_DEBUG_INFO_STR_APPEND                  ")"
+
+/* Table of debug info sub-field's masks and corresponding message strings
+ * to be appended to firmware trace
+ *
+ * Mask     : 16 bit mask to be applied to debug-info field
+ * String   : debug info message string
+ */
+
+#define RGXFWT_DEBUG_INFO_MSKSTRLIST \
+/*Mask,                                           String*/ \
+X(RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET,      "mmu pf") \
+X(RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET,   "mmu pending") \
+X(RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET,        "slave events")
+
+/* Firmware trace time-stamp field breakup */
+
+/* RGX_CR_TIMER register read (48 bits) value*/
+#define RGXFWT_TIMESTAMP_TIME_SHIFT                   (0U)
+#define RGXFWT_TIMESTAMP_TIME_CLRMSK                  (IMG_UINT64_C(0xFFFF000000000000))
+
+/* Extra debug-info (16 bits) */
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT             (48U)
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK            ~RGXFWT_TIMESTAMP_TIME_CLRMSK
+
+typedef struct
+{
+	IMG_UINT			bfOsState		: 3;
+	IMG_UINT			bfFLOk			: 1;
+	IMG_UINT			bfFLGrowPending	: 1;
+	IMG_UINT			bfReserved		: 27;
+} RGXFWIF_OS_RUNTIME_FLAGS;
+
+#define RGXFWIF_FWFAULTINFO_MAX		(8U)			/* Total number of FW fault logs stored */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64CRTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64OSTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64Data;
+	RGXFWIF_FILE_INFO_BUF	sFaultBuf;
+} UNCACHED_ALIGN RGX_FWFAULTINFO;
+
+#define RGXFWIF_POW_STATES \
+  X(RGXFWIF_POW_OFF)			/* idle and handshaked with the host (ready to full power down) */ \
+  X(RGXFWIF_POW_ON)				/* running HW commands */ \
+  X(RGXFWIF_POW_FORCED_IDLE)	/* forced idle */ \
+  X(RGXFWIF_POW_IDLE)			/* idle waiting for host handshake */
+
+typedef enum
+{
+#define X(NAME) NAME,
+	RGXFWIF_POW_STATES
+#undef X
+} RGXFWIF_POW_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_THREAD_NUM 2
+
+static_assert(RGXFW_THREAD_NUM <= MAX_THREAD_NUM,
+				"RGXFW_THREAD_NUM is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Firmware HWR states */
+#define RGXFWIF_HWR_HARDWARE_OK			(IMG_UINT32_C(0x1) << 0U)	/*!< The HW state is ok or locked up */
+#define RGXFWIF_HWR_RESET_IN_PROGRESS	(IMG_UINT32_C(0x1) << 1U)	/*!< Tells if a HWR reset is in progress */
+#define RGXFWIF_HWR_GENERAL_LOCKUP		(IMG_UINT32_C(0x1) << 3U)	/*!< A DM unrelated lockup has been detected */
+#define RGXFWIF_HWR_DM_RUNNING_OK		(IMG_UINT32_C(0x1) << 4U)	/*!< At least one DM is running without being close to a lockup */
+#define RGXFWIF_HWR_DM_STALLING			(IMG_UINT32_C(0x1) << 5U)	/*!< At least one DM is close to lockup */
+#define RGXFWIF_HWR_FW_FAULT			(IMG_UINT32_C(0x1) << 6U)	/*!< The FW has faulted and needs to restart */
+#define RGXFWIF_HWR_RESTART_REQUESTED	(IMG_UINT32_C(0x1) << 7U)	/*!< The FW has requested the host to restart it */
+
+#define RGXFWIF_PHR_MODE_OFF			(0UL)
+#define RGXFWIF_PHR_MODE_RD_RESET		(1UL)
+
+typedef IMG_UINT32 RGXFWIF_HWR_STATEFLAGS;
+
+typedef IMG_UINT32 RGXFWIF_HWR_RECOVERYFLAGS;
+
+/*! @Brief Firmware system data shared with the Host driver */
+typedef struct
+{
+	IMG_UINT32                 ui32ConfigFlags;                       /*!< Configuration flags from host */
+	IMG_UINT32                 ui32ConfigFlagsExt;                    /*!< Extended configuration flags from host */
+	volatile RGXFWIF_POW_STATE ePowState;
+	struct {
+		volatile IMG_UINT32        ui32HWPerfRIdx;
+		volatile IMG_UINT32        ui32HWPerfWIdx;
+		volatile IMG_UINT32        ui32HWPerfWrapCount;
+	} sHWPerfCtrl; /* Struct used to inval/flush HWPerfCtrl members */
+	IMG_UINT32                 ui32HWPerfSize;                        /*!< Constant after setup, needed in FW */
+	IMG_UINT32                 ui32HWPerfDropCount;                   /*!< The number of times the FW drops a packet due to buffer full */
+
+	/* ui32HWPerfUt, ui32FirstDropOrdinal, ui32LastDropOrdinal only valid when FW is built with
+	 * RGX_HWPERF_UTILIZATION & RGX_HWPERF_DROP_TRACKING defined in rgxfw_hwperf.c */
+	IMG_UINT32                 ui32HWPerfUt;                          /*!< Buffer utilisation, high watermark of bytes in use */
+	IMG_UINT32                 ui32FirstDropOrdinal;                  /*!< The ordinal of the first packet the FW dropped */
+	IMG_UINT32                 ui32LastDropOrdinal;                   /*!< The ordinal of the last packet the FW dropped */
+	RGXFWIF_OS_RUNTIME_FLAGS   asOsRuntimeFlagsMirror[RGXFW_MAX_NUM_OSIDS];/*!< State flags for each Operating System mirrored from Fw coremem */
+	RGX_FWFAULTINFO            sFaultInfo[RGXFWIF_FWFAULTINFO_MAX];   /*!< Firmware fault info */
+	IMG_UINT32                 ui32FWFaults;                          /*!< Firmware faults count */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32                 aui32CrPollAddr[MAX_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[MAX_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[MAX_THREAD_NUM];    /*!< Failed poll count */
+#else
+	IMG_UINT32                 aui32CrPollAddr[RGXFW_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[RGXFW_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[RGXFW_THREAD_NUM];    /*!< Failed poll count */
+#endif
+	IMG_UINT64 RGXFW_ALIGN     ui64StartIdleTime;
+	IMG_UINT32                 aui32TSMirror[RGXFW_MAX_NUM_OSIDS];    /*!< Array of time slice per OS Mirrored from the FW */
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define RGXFWIF_STATS_FRAMEWORK_LINESIZE    (8)
+#define RGXFWIF_STATS_FRAMEWORK_MAX         (2048*RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+	IMG_UINT32 RGXFW_ALIGN     aui32FWStatsBuf[RGXFWIF_STATS_FRAMEWORK_MAX];
+#endif
+	RGXFWIF_HWR_STATEFLAGS     ui32HWRStateFlags; /*!< Firmware's Current HWR state */
+	RGXFWIF_HWR_RECOVERYFLAGS  aui32HWRRecoveryFlags[RGXFWIF_DM_MAX]; /*!< Each DM's HWR state */
+	IMG_UINT32                 ui32FwSysDataFlags;                      /*!< Compatibility and other flags */
+	IMG_UINT32                 ui32McConfig;                            /*!< Identify whether MC config is P-P or P-S */
+	IMG_UINT32                 ui32MemFaultCheck;                       /*!< Device mem fault check on PCI systems */
+} UNCACHED_ALIGN RGXFWIF_SYSDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_SYSDATA) == 3624,
+				"RGXFWIF_SYSDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER == 3624");
+#endif
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+#define PVR_SLR_LOG_ENTRIES 10U
+#define PVR_SLR_LOG_STRLEN  30 /*!< MAX_CLIENT_CCB_NAME not visible to this header */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64Timestamp;
+	IMG_UINT32				ui32FWCtxAddr;
+	IMG_UINT32				ui32NumUFOs;
+	IMG_CHAR				aszCCBName[PVR_SLR_LOG_STRLEN];
+} UNCACHED_ALIGN RGXFWIF_SLR_ENTRY;
+#endif
+
+#define RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS (128U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware per-os data and configuration
+ */
+typedef struct
+{
+	IMG_UINT32                 ui32FwOsConfigFlags;                   /*!< Configuration flags from an OS */
+	IMG_UINT32                 ui32FWSyncCheckMark;                   /*!< Markers to signal that the host should perform a full sync check */
+	IMG_UINT32                 ui32HostSyncCheckMark;                  /*!< Markers to signal that the Firmware should perform a full sync check */
+#if defined(PVRSRV_STALLED_CCB_ACTION) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32                 ui32ForcedUpdatesRequested;
+	IMG_UINT8                  ui8SLRLogWp;
+	RGXFWIF_SLR_ENTRY          sSLRLogFirst;
+	RGXFWIF_SLR_ENTRY          sSLRLog[PVR_SLR_LOG_ENTRIES];
+	IMG_UINT64 RGXFW_ALIGN     ui64LastForcedUpdateTime;
+#endif
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	volatile IMG_UINT32        aui32InterruptCount[MAX_THREAD_NUM]; /*!< Interrupt count from Threads > */
+#else
+	volatile IMG_UINT32        aui32InterruptCount[RGXFW_THREAD_NUM]; /*!< Interrupt count from Threads > */
+#endif
+	IMG_UINT32                 ui32KCCBCmdsExecuted;                  /*!< Executed Kernel CCB command count */
+	RGXFWIF_DEV_VIRTADDR       sPowerSync;                            /*!< Sync prim used to signal the host the power off state */
+	IMG_UINT32                 ui32FwOsDataFlags;                       /*!< Compatibility and other flags */
+	IMG_UINT32                 aui32OfflineBuffer[RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS];
+} UNCACHED_ALIGN RGXFWIF_OSDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_OSDATA) == 584,
+				"RGXFWIF_OSDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+/*!
+ ******************************************************************************
+ * HWR Data
+ *****************************************************************************/
+/*!
+ * @Defgroup HWRInfo FW HWR shared data interface
+ * @Brief Types grouping data structures and defines used in realising the HWR record.
+ * @{
+ */
+
+#define RGXFW_PROCESS_NAME_LEN	(16)
+
+/*! @Brief HWR Lockup types */
+typedef enum
+{
+	RGX_HWRTYPE_UNKNOWNFAILURE = 0, /*!< Unknown failure */
+	RGX_HWRTYPE_OVERRUN        = 1, /*!< DM overrun */
+	RGX_HWRTYPE_POLLFAILURE    = 2, /*!< Poll failure */
+	RGX_HWRTYPE_BIF0FAULT      = 3, /*!< BIF0 fault */
+	RGX_HWRTYPE_BIF1FAULT      = 4, /*!< BIF1 fault */
+	RGX_HWRTYPE_TEXASBIF0FAULT = 5, /*!< TEXASBIF0 fault */
+	RGX_HWRTYPE_MMUFAULT       = 6, /*!< MMU fault */
+	RGX_HWRTYPE_MMUMETAFAULT   = 7, /*!< MMU META fault */
+	RGX_HWRTYPE_MIPSTLBFAULT   = 8, /*!< MIPS TLB fault */
+	RGX_HWRTYPE_ECCFAULT       = 9, /*!< ECC fault */
+	RGX_HWRTYPE_MMURISCVFAULT  = 10, /*!< MMU RISCV fault */
+} RGX_HWRTYPE;
+
+#define RGXFWIF_HWRTYPE_BIF_BANK_GET(eHWRType) (((eHWRType) == RGX_HWRTYPE_BIF0FAULT) ? 0 : 1)
+
+#define RGXFWIF_HWRTYPE_PAGE_FAULT_GET(eHWRType) ((((eHWRType) == RGX_HWRTYPE_BIF0FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_BIF1FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_TEXASBIF0FAULT) ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUFAULT)       ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUMETAFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MIPSTLBFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMURISCVFAULT)) ? true : false)
+
+/************************
+ *  GPU HW error codes  *
+ ************************/
+typedef enum
+{
+	RGX_HW_ERR_NA = 0x0,
+	RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL = 0x101,
+} RGX_HW_ERR;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFReqStatus; /*!< BIF request status */
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFMMUStatus; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_BIFINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32FaultGPU; /*!< ECC fault in GPU */
+} RGX_ECCINFO;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		aui64MMUStatus[2]; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_MMUINFO;
+
+typedef struct
+{
+	IMG_UINT32	ui32ThreadNum; /*!< Thread ID performing poll operation */
+	IMG_UINT32	ui32CrPollAddr; /*!< CR Poll Address */
+	IMG_UINT32	ui32CrPollMask; /*!< CR Poll mask */
+	IMG_UINT32	ui32CrPollLastValue; /*!< CR Poll last value */
+	IMG_UINT64	RGXFW_ALIGN ui64Reserved;
+} UNCACHED_ALIGN RGX_POLLINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32BadVAddr; /*!< VA address */
+	IMG_UINT32 ui32EntryLo;
+} RGX_TLBINFO;
+
+/*! @Brief Structure to keep information specific to a lockup e.g. DM, timer, lockup type etc. */
+typedef struct
+{
+	union
+	{
+		RGX_BIFINFO  sBIFInfo; /*!< BIF failure details */
+		RGX_MMUINFO  sMMUInfo; /*!< MMU failure details */
+		RGX_POLLINFO sPollInfo; /*!< Poll failure details */
+		RGX_TLBINFO  sTLBInfo; /*!< TLB failure details */
+		RGX_ECCINFO  sECCInfo; /*!< ECC failure details */
+	} uHWRData;
+
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimer; /*!< Timer value at the time of lockup */
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimer; /*!< OS timer value at the time of lockup */
+	IMG_UINT32             ui32FrameNum; /*!< Frame number of the workload */
+	IMG_UINT32             ui32PID; /*!< PID belonging to the workload */
+	IMG_UINT32             ui32ActiveHWRTData; /*!< HWRT data of the workload */
+	IMG_UINT32             ui32HWRNumber; /*!< HWR number */
+	IMG_UINT32             ui32EventStatus; /*!< Core specific event status register at the time of lockup */
+	IMG_UINT32             ui32HWRRecoveryFlags; /*!< DM state flags */
+	RGX_HWRTYPE            eHWRType; /*!< Type of lockup */
+	RGXFWIF_DM             eDM; /*!< Recovery triggered for the DM */
+	IMG_UINT32             ui32CoreID; /*!< Core ID of the GPU */
+	RGX_HW_ERR             eHWErrorCode; /*!< Error code used to determine HW fault */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeOfKick; /*!< Workload kick time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetStart; /*!< HW reset start time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetFinish; /*!< HW reset stop time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeFreelistReady; /*!< freelist ready time on the last HWR */
+	IMG_CHAR   RGXFW_ALIGN szProcName[RGXFW_PROCESS_NAME_LEN]; /*!< User process name */
+} UNCACHED_ALIGN RGX_HWRINFO;
+
+#define RGXFWIF_HWINFO_MAX_FIRST 8U							/* Number of first HWR logs recorded (never overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX_LAST 8U							/* Number of latest HWR logs (older logs are overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX (RGXFWIF_HWINFO_MAX_FIRST + RGXFWIF_HWINFO_MAX_LAST)	/* Total number of HWR logs stored in a buffer */
+#define RGXFWIF_HWINFO_LAST_INDEX (RGXFWIF_HWINFO_MAX - 1U)	/* Index of the last log in the HWR log buffer */
+
+/*! @Brief Firmware HWR information structure allocated by the Services and used by the Firmware to update recovery information. */
+typedef struct
+{
+	RGX_HWRINFO sHWRInfo[RGXFWIF_HWINFO_MAX]; /*!< Max number of recovery record */
+	IMG_UINT32  ui32HwrCounter; /*!< HWR counter used in FL reconstruction */
+	IMG_UINT32  ui32WriteIndex; /*!< Index for updating recovery information in sHWRInfo */
+	IMG_UINT32  ui32DDReqCount; /*!< Count of DebugDump requested to the host after recovery */
+	IMG_UINT32  ui32HWRInfoBufFlags; /* Compatibility and other flags */
+	IMG_UINT32  aui32HwrDmLockedUpCount[RGXFWIF_DM_MAX]; /*!< Lockup count for each DM */
+	IMG_UINT32  aui32HwrDmOverranCount[RGXFWIF_DM_MAX]; /*!< Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmRecoveredCount[RGXFWIF_DM_MAX]; /*!< Lockup + Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmFalseDetectCount[RGXFWIF_DM_MAX]; /*!< False lockup detection count for each DM */
+} UNCACHED_ALIGN RGXFWIF_HWRINFOBUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRINFOBUF) == 2336,
+				"RGXFWIF_HWRINFOBUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of HWRInfo */
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COMMONCTX_STATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RF_CMD;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWMEMCONTEXT;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Freelist holding usage state of the Parameter Buffers
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN psFreeListDevVAddr;	/*!< Freelist page table base */
+	IMG_UINT64		RGXFW_ALIGN ui64CurrentDevVAddr;/*!< Freelist page table entry for current free page */
+	IMG_UINT32		ui32CurrentStackTop;		/*!< Freelist current free page */
+	IMG_UINT32		ui32MaxPages;			/*!< Max no. of pages can be added to the freelist */
+	IMG_UINT32		ui32GrowPages;			/*!< No pages to add in each freelist grow */
+	IMG_UINT32		ui32CurrentPages;		/*!< Total no. of pages made available to the PM HW */
+	IMG_UINT32		ui32AllocatedPageCount;		/*!< No. of pages allocated by PM HW */
+	IMG_UINT32		ui32AllocatedMMUPageCount;	/*!< No. of pages allocated for GPU MMU for PM*/
+#if defined(SUPPORT_SHADOW_FREELISTS) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32		ui32HWRCounter;
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;
+#endif
+	IMG_UINT32		ui32FreeListID;			/*!< Unique Freelist ID */
+	IMG_BOOL		bGrowPending;			/*!< Freelist grow is pending */
+	IMG_UINT32		ui32ReadyPages;			/*!< Reserved pages to be used only on PM OOM event */
+	IMG_UINT32		ui32FreelistFlags;		/*!< Compatibility and other flags */
+#if defined(SUPPORT_AGP) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32		ui32PmGlobalPb;			/*!< PM Global PB on which Freelist is loaded */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FREELIST;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FREELIST) == 64,
+				"RGXFWIF_FREELIST is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct {RGXFWIF_DEV_VIRTADDR sNext;
+                RGXFWIF_DEV_VIRTADDR sPrev;}	RGXFW_DLLIST_NODE;
+
+#define RGXFWIF_MAX_NUM_CANCEL_REQUESTS		(8U)		/* Maximum number of workload cancellation requests */
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware Common Context (or FWCC)
+ */
+typedef struct RGXFWIF_FWCOMMONCONTEXT_
+{
+	/* CCB details for this firmware context */
+	PRGXFWIF_CCCB_CTL		psCCBCtl;				/*!< CCB control */
+	PRGXFWIF_CCCB			psCCB;					/*!< CCB base */
+
+	/* Context suspend state */
+	PRGXFWIF_COMMONCTX_STATE	RGXFW_ALIGN psContextState;		/*!< TA/3D context suspend state, read/written by FW */
+
+	/* Flags e.g. for context switching */
+	IMG_UINT32				ui32FWComCtxFlags;
+	IMG_INT32				i32Priority;  /*!< Priority level */
+	IMG_UINT32				ui32PrioritySeqNum;
+
+	/* Framework state */
+	PRGXFWIF_RF_CMD			RGXFW_ALIGN psRFCmd;		/*!< Register updates for Framework */
+
+	/* Misc and compatibility flags */
+	IMG_UINT32				ui32CompatFlags;
+
+	/* Statistic updates waiting to be passed back to the host... */
+	IMG_INT32				i32StatsNumStores;		/*!< Number of stores on this context since last update */
+	IMG_INT32				i32StatsNumOutOfMemory;		/*!< Number of OOMs on this context since last update */
+	IMG_INT32				i32StatsNumPartialRenders;	/*!< Number of PRs on this context since last update */
+	RGXFWIF_DM				eDM;				/*!< Data Master type */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN  sBufStalledNode;			/*!< List entry for the buffer stalled list */
+	IMG_UINT64				RGXFW_ALIGN  ui64CBufQueueCtrlAddr;	/*!< Address of the circular buffer queue pointers */
+
+	IMG_UINT64				RGXFW_ALIGN  ui64RobustnessAddress;
+	IMG_UINT32				ui32MaxDeadlineMS;			/*!< Max HWR deadline limit in ms */
+	bool					bReadOffsetNeedsReset;			/*!< Following HWR circular buffer read-offset needs resetting */
+
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sWaitingNode;		/*!< List entry for the waiting list */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sRunNode;			/*!< List entry for the run list */
+	RGXFWIF_UFO				sLastFailedUFO;			/*!< UFO that last failed (or NULL) */
+
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;					/*!< Memory context */
+
+	/* References to the host side originators */
+	IMG_UINT32				ui32ServerCommonContextID;	/*!< the Server Common Context */
+	IMG_UINT32				ui32PID;			/*!< associated process ID */
+
+	IMG_BOOL				bGeomOOMDisabled;		/*!< True when Geom DM OOM is not allowed */
+	IMG_CHAR				szProcName[RGXFW_PROCESS_NAME_LEN];	/*!< User process name */
+	IMG_UINT32				ui32DeferCount;		/*!< Number of context defers before forced scheduling of context */
+	IMG_UINT32				aui32FirstIntJobRefToCancel[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the beginning of range of IntJobRefs at and above which workloads will be discarded */
+	IMG_UINT32				aui32FirstValidIntJobRef[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the end of range of IntJobRef below which workloads will be discarded */
+	IMG_BOOL				bCancelRangesActive;	/*!< True if any active ranges in aui32FirstIntJobRefToCancel and aui32FirstValidIntJobRef arrays */
+	IMG_BOOL				bLastKickedCmdWasSafetyOnly;
+	IMG_UINT32				ui32UID;			/*!< associated process UID used in FW managed gpu work period hwperf events */
+} UNCACHED_ALIGN RGXFWIF_FWCOMMONCONTEXT;
+
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) <= 256U,
+              "Size of structure RGXFWIF_FWCOMMONCONTEXT exceeds maximum expected size.");
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) == 168,
+				"RGXFWIF_FWCOMMONCONTEXT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ ******************************************************************************
+ * HWRTData
+ *****************************************************************************/
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FREELIST;
+
+/* HWRTData flags */
+/* Deprecated flags 1:0 */
+#define HWRTDATA_HAS_LAST_TA              (IMG_UINT32_C(1) << 2)
+#define HWRTDATA_PARTIAL_RENDERED         (IMG_UINT32_C(1) << 3)
+#define HWRTDATA_DISABLE_TILE_REORDERING  (IMG_UINT32_C(1) << 4)
+#define HWRTDATA_NEED_BRN65101_BLIT       (IMG_UINT32_C(1) << 5)
+#define HWRTDATA_FIRST_BRN65101_STRIP     (IMG_UINT32_C(1) << 6)
+#define HWRTDATA_NEED_BRN67182_2ND_RENDER (IMG_UINT32_C(1) << 7)
+#if defined(SUPPORT_AGP)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT0    (IMG_UINT32_C(1) << 8)
+#if defined(SUPPORT_AGP4)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT1    (IMG_UINT32_C(1) << 9)
+#endif
+#define HWRTDATA_GEOM_NEEDS_RESUME        (IMG_UINT32_C(1) << 10)
+#endif
+#if defined(SUPPORT_TRP)
+#define HWRTDATA_GEOM_TRP_IN_PROGRESS          (IMG_UINT32_C(1) << 9)
+#endif
+
+typedef enum
+{
+	RGXFWIF_RTDATA_STATE_NONE = 0,
+	RGXFWIF_RTDATA_STATE_KICKTA,
+	RGXFWIF_RTDATA_STATE_KICKTAFIRST,
+	RGXFWIF_RTDATA_STATE_TAFINISHED,
+	RGXFWIF_RTDATA_STATE_KICK3D,
+	RGXFWIF_RTDATA_STATE_3DFINISHED,
+	RGXFWIF_RTDATA_STATE_3DCONTEXTSTORED,
+	RGXFWIF_RTDATA_STATE_TAOUTOFMEM,
+	RGXFWIF_RTDATA_STATE_PARTIALRENDERFINISHED,
+	/* In case of HWR, we can't set the RTDATA state to NONE,
+	 * as this will cause any TA to become a first TA.
+	 * To ensure all related TA's are skipped, we use the HWR state */
+	RGXFWIF_RTDATA_STATE_HWR,
+	RGXFWIF_RTDATA_STATE_UNKNOWN = 0x7FFFFFFFU
+} RGXFWIF_RTDATA_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_FREELISTS_SIZE 3
+
+static_assert(RGXFW_MAX_FREELISTS <= MAX_FREELISTS_SIZE,
+				"RGXFW_MAX_FREELISTS is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct
+{
+	IMG_BOOL							bTACachesNeedZeroing;
+
+	IMG_UINT32							ui32ScreenPixelMax;
+	IMG_UINT64							RGXFW_ALIGN ui64MultiSampleCtl;
+	IMG_UINT64							ui64FlippedMultiSampleCtl;
+	IMG_UINT32							ui32TPCStride;
+	IMG_UINT32							ui32TPCSize;
+	IMG_UINT32							ui32TEScreen;
+	IMG_UINT32							ui32MTileStride;
+	IMG_UINT32							ui32TEAA;
+	IMG_UINT32							ui32TEMTILE1;
+	IMG_UINT32							ui32TEMTILE2;
+	IMG_UINT32							ui32ISPMergeLowerX;
+	IMG_UINT32							ui32ISPMergeLowerY;
+	IMG_UINT32							ui32ISPMergeUpperX;
+	IMG_UINT32							ui32ISPMergeUpperY;
+	IMG_UINT32							ui32ISPMergeScaleX;
+	IMG_UINT32							ui32ISPMergeScaleY;
+	IMG_UINT32							uiRgnHeaderSize;
+	IMG_UINT32							ui32ISPMtileSize;
+} UNCACHED_ALIGN RGXFWIF_HWRTDATA_COMMON;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRTDATA_COMMON) == 88,
+				"RGXFWIF_HWRTDATA_COMMON is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct
+{
+	IMG_UINT32           ui32RenderTargetIndex;		//Render number
+	IMG_UINT32           ui32CurrentRenderTarget;	//index in RTA
+	IMG_UINT32           ui32ActiveRenderTargets;	//total active RTs
+	IMG_UINT32           ui32CumulActiveRenderTargets;   //total active RTs from the first TA kick, for OOM
+	RGXFWIF_DEV_VIRTADDR sValidRenderTargets;  //Array of valid RT indices
+	RGXFWIF_DEV_VIRTADDR sRTANumPartialRenders;  //Array of number of occurred partial renders per render target
+	IMG_UINT32           ui32MaxRTs;   //Number of render targets in the array
+	IMG_UINT32           ui32RTACtlFlags; /* Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_RTA_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_RTA_CTL) == 32,
+				"RGXFWIF_RTA_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Render Target data i.e. HWRTDATA used to hold the PM context
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psPMMListDevVAddr;			/*!< MList Data Store */
+
+	IMG_UINT64			RGXFW_ALIGN ui64VCECatBase[1];			/*!< VCE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64VCELastCatBase[1];
+	IMG_UINT64			RGXFW_ALIGN ui64TECatBase[1];			/*!< TE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64TELastCatBase[1];
+	IMG_UINT64			RGXFW_ALIGN ui64AlistCatBase;			/*!< Alist Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64AlistLastCatBase;
+
+	IMG_UINT64			RGXFW_ALIGN ui64PMAListStackPointer;		/*!< Freelist page table entry for current Mlist page */
+	IMG_UINT32			ui32PMMListStackPointer;			/*!< Current Mlist page */
+
+	RGXFWIF_DEV_VIRTADDR		sHWRTDataCommonFwAddr;				/*!< Render target dimension dependent data */
+
+	IMG_UINT32			ui32HWRTDataFlags;
+	RGXFWIF_RTDATA_STATE		eState;						/*!< Current workload processing state of HWRTDATA */
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[MAX_FREELISTS_SIZE];	/*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[MAX_FREELISTS_SIZE];
+#else
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[RGXFW_MAX_FREELISTS];	/*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[RGXFW_MAX_FREELISTS];
+#endif
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psVHeapTableDevVAddr;		/*!< VHeap table base */
+
+	RGXFWIF_RTA_CTL			sRTACtl;					/*!< Render target array data */
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sTailPtrsDevVAddr;			/*!< Tail pointers base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sMacrotileArrayDevVAddr;		/*!< Macrotiling array base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sRgnHeaderDevVAddr;			/*!< Region headers base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sRTCDevVAddr;			/*!< Render target cache base */
+#if defined(RGX_FIRMWARE)
+	struct RGXFWIF_FWCOMMONCONTEXT_* RGXFW_ALIGN psOwnerGeom;
+#else
+	RGXFWIF_DEV_VIRTADDR		RGXFW_ALIGN pui32OwnerGeomNotUsedByHost;
+#endif
+#if defined(SUPPORT_TRP) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32			ui32KickFlagsCopy;
+	IMG_UINT32			ui32TEPageCopy;
+	IMG_UINT32			ui32VCEPageCopy;
+#endif
+#if defined(SUPPORT_AGP) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_BOOL			bTACachesNeedZeroing;
+#endif
+
+	RGXFWIF_CLEANUP_CTL		RGXFW_ALIGN_DCACHEL sCleanupState;					/*!< Render target clean up state */
+} RGXFW_ALIGN_DCACHEL RGXFWIF_HWRTDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRTDATA) == 256,
+				"RGXFWIF_HWRTDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Sync_checkpoint firmware object.
+ * This is the FW-addressable structure used to hold the sync checkpoint's
+ * state and other information which needs to be accessed by the firmware.
+ */
+typedef struct
+{
+	IMG_UINT32	ui32State;          /*!< Holds the current state of the sync checkpoint */
+	IMG_UINT32	ui32FwRefCount;     /*!< Holds the FW reference count (num of fences/updates processed) */
+} SYNC_CHECKPOINT_FW_OBJ;
+
+/* Bit mask Firmware can use to test if a checkpoint has signalled or errored */
+#define SYNC_CHECKPOINT_SIGNALLED_MASK (0x1 << 0)
+
+typedef enum
+{
+	RGXFWIF_REG_CFG_TYPE_PWR_ON=0,      /* Sidekick power event */
+	RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,   /* Rascal / dust power event */
+	RGXFWIF_REG_CFG_TYPE_TA,            /* TA kick */
+	RGXFWIF_REG_CFG_TYPE_3D,            /* 3D kick */
+	RGXFWIF_REG_CFG_TYPE_CDM,           /* Compute kick */
+	RGXFWIF_REG_CFG_TYPE_TLA,           /* TLA kick */
+	RGXFWIF_REG_CFG_TYPE_TDM,           /* TDM kick */
+	RGXFWIF_REG_CFG_TYPE_ALL            /* Applies to all types. Keep as last element */
+} RGXFWIF_REG_CFG_TYPE;
+
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_OFFSET_BYTES                 RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES               (1U << 19)
+
+#define RGX_HEAP_KM_USC_RESERVED_SIZE                              RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES
+#define RGX_HEAP_USC_RESERVED_TOTAL_SIZE                           RGX_HEAP_UM_USC_RESERVED_SIZE + RGX_HEAP_KM_USC_RESERVED_SIZE
+
+#endif /*  RGX_FWIF_SHARED_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_shared.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_heaps.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_heaps.h
new file mode 100644
index 000000000000..f8fcfe1fe27e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_heaps.h
@@ -0,0 +1,67 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAPS_H)
+#define RGX_HEAPS_H
+
+/*
+  Identify heaps by their names
+*/
+#define RGX_GENERAL_SVM_HEAP_IDENT          "General SVM"               /*!< RGX General SVM (shared virtual memory) Heap Identifier */
+#define RGX_GENERAL_HEAP_IDENT              "General"                   /*!< RGX General Heap Identifier */
+#define RGX_GENERAL_NON4K_HEAP_IDENT        "General NON-4K"            /*!< RGX General non-4K Heap Identifier */
+#define RGX_PDSCODEDATA_HEAP_IDENT          "PDS Code and Data"         /*!< RGX PDS Code/Data Heap Identifier */
+#define RGX_USCCODE_HEAP_IDENT              "USC Code"                  /*!< RGX USC Code Heap Identifier */
+#define RGX_VK_CAPT_REPLAY_HEAP_IDENT       "Vulkan Capture Replay"     /*!< RGX Vulkan capture replay buffer Heap Identifier */
+#define RGX_FBCDC_HEAP_IDENT                "FBCDC"                     /*!< RGX FBCDC State Table Heap Identifier */
+#define RGX_FBCDC_LARGE_HEAP_IDENT          "Large FBCDC"               /*!< RGX Large FBCDC State Table Heap Identifier */
+#define RGX_CMP_MISSION_RMW_HEAP_IDENT      "Compute Mission RMW"       /*!< Compute Mission RMW Heap Identifier */
+#define RGX_CMP_SAFETY_RMW_HEAP_IDENT       "Compute Safety RMW"        /*!< Compute Safety RMW Heap Identifier */
+#define RGX_TEXTURE_STATE_HEAP_IDENT        "Texture State"             /*!< Texture State Heap Identifier */
+#define RGX_VISIBILITY_TEST_HEAP_IDENT      "Visibility Test"           /*!< Visibility Test Heap Identifier */
+
+/* Services client internal heap identification */
+#define RGX_RGNHDR_BRN_63142_HEAP_IDENT     "RgnHdr BRN63142"           /*!< RGX RgnHdr BRN63142 Heap Identifier */
+#define RGX_TQ3DPARAMETERS_HEAP_IDENT       "TQ3DParameters"            /*!< RGX TQ 3D Parameters Heap Identifier */
+#define RGX_MMU_INIA_BRN_65273_HEAP_IDENT   "MMU INIA BRN65273"         /*!< MMU BRN65273 Heap A Identifier */
+#define RGX_MMU_INIB_BRN_65273_HEAP_IDENT   "MMU INIB BRN65273"         /*!< MMU BRN65273 Heap B Identifier */
+#endif /* RGX_HEAPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_hwperf.h
new file mode 100644
index 000000000000..d164c20a2417
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_hwperf.h
@@ -0,0 +1,483 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_H_
+#define RGX_HWPERF_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_hwperf_common.h"
+
+#if !defined(__KERNEL__)
+/* User-mode and Firmware definitions only */
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+
+/* HWPerf interface assumption checks */
+static_assert(RGX_FEATURE_NUM_CLUSTERS <= 16U, "Cluster count too large for HWPerf protocol definition");
+
+/*! The number of indirectly addressable TPU_MSC blocks in the GPU */
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST MAX(((IMG_UINT32)RGX_FEATURE_NUM_CLUSTERS >> 1), 1U)
+
+/*! The number of indirectly addressable USC blocks in the GPU */
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER (RGX_FEATURE_NUM_CLUSTERS)
+
+# if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+
+ /*! Defines the number of performance counter blocks that are directly
+  * addressable in the RGX register map for S. */
+#  define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      1 /* JONES */
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       (RGX_NUM_PHANTOMS)
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      1 /* BLACKPEARL */
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         2 /* TPU, TEXAS */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 2 /* USC, PBE */
+
+# elif defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+
+  /*! Defines the number of performance counter blocks that are directly
+   * addressable in the RGX register map. */
+#   define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS    2 /* TORNADO, TA */
+
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       (RGX_NUM_PHANTOMS)
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      2 /* RASTER, TEXAS */
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         1 /* TPU */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 1 /* USC */
+
+# else /* !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && !defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) i.e. S6 */
+
+ /*! Defines the number of performance counter blocks that are
+  * addressable in the RGX register map for Series 6. */
+#  define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      3 /* TA, RASTER, HUB */
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       0 /* PHANTOM is not there in Rogue1. Just using it to keep naming same as later series (RogueXT n Rogue XT+) */
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      0
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         1 /* TPU */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 1 /* USC */
+
+# endif
+
+/*! The number of performance counters in each layout block defined for UM/FW code */
+#if defined(RGX_FEATURE_CLUSTER_GROUPING)
+  #define RGX_HWPERF_CNTRS_IN_BLK 6
+ #else
+  #define RGX_HWPERF_CNTRS_IN_BLK 4
+#endif
+
+#endif /* #if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER) */
+#else /* defined(__KERNEL__) */
+/* Kernel/server definitions - not used, hence invalid definitions */
+
+# define RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC 0xFF
+
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST    RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+
+# define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_INDIRECT_BY_PHANTOM       RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_NONDUST_BLKS      RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_DUST_BLKS         RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+
+#endif
+
+/*! The number of custom non-mux counter blocks supported */
+#define RGX_HWPERF_MAX_CUSTOM_BLKS 5U
+
+/*! The number of counters supported in each non-mux counter block */
+#define RGX_HWPERF_MAX_CUSTOM_CNTRS 8U
+
+
+/******************************************************************************
+ * Data Stream Common Types
+ *****************************************************************************/
+
+/*! All the Data Masters HWPerf is aware of. When a new DM is added to this
+ * list, it should be appended at the end to maintain backward compatibility
+ * of HWPerf data.
+ */
+#define	RGX_HWPERF_DM_GP                  0x00000000U
+#define	RGX_HWPERF_DM_2D                  0x00000001U
+#define	RGX_HWPERF_DM_TA                  0x00000002U
+#define	RGX_HWPERF_DM_3D                  0x00000003U
+#define	RGX_HWPERF_DM_CDM                 0x00000004U
+#define	RGX_HWPERF_DM_RTU                 0x00000005U
+#define	RGX_HWPERF_DM_SHG                 0x00000006U
+#define	RGX_HWPERF_DM_TDM                 0x00000007U
+
+#define	RGX_HWPERF_DM_LAST                0x00000008U
+
+#define	RGX_HWPERF_DM_INVALID             0x1FFFFFFFU
+
+/******************************************************************************
+ * API Types
+ *****************************************************************************/
+
+/*! Counter block IDs for all the hardware blocks with counters.
+ * Directly addressable blocks must have a value between 0..15 [0..0xF].
+ * Indirect groups have following encoding:
+ * First hex digit (LSB) represents a unit number within the group
+ * and the second hex digit represents the group number.
+ * Group 0 is the direct group, all others are indirect groups.
+ */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_ID;
+
+/*! Directly addressable counter blocks */
+#if defined(DOXYGEN)
+/*! _RGX_HWPERF_CNTBLK_ID */
+#endif
+#define	RGX_CNTBLK_ID_TA			 0x0000U
+#define	RGX_CNTBLK_ID_RASTER		 0x0001U /*!< Non-cluster grouping cores */
+#define	RGX_CNTBLK_ID_HUB			 0x0002U /*!< Non-cluster grouping cores */
+#define	RGX_CNTBLK_ID_TORNADO		 0x0003U /*!< XT cores */
+#define	RGX_CNTBLK_ID_JONES			 0x0004U /*!< S7 cores */
+#define	RGX_CNTBLK_ID_DIRECT_LAST	 0x0005U /*!< Indirect blocks start from here */
+#define	RGX_CNTBLK_ID_BF_DEPRECATED	 0x0005U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_BT_DEPRECATED	 0x0006U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_RT_DEPRECATED	 0x0007U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_SH_DEPRECATED	 0x0008U /*!< Ray tracing unit (DEPRECATED) */
+
+
+/*! Indirectly addressable counter blocks. DA blocks indicate counter blocks
+ *  where the counter registers are directly accessible
+ */
+#define	RGX_CNTBLK_ID_TPU_MCU0		 0x0010U /*!< Addressable by Dust */
+#define RGX_CNTBLK_ID_TPU_MCU0_DA	 0x8010U
+#define	RGX_CNTBLK_ID_TPU_MCU1		 0x0011U
+#define RGX_CNTBLK_ID_TPU_MCU1_DA	 0x8011U
+#define	RGX_CNTBLK_ID_TPU_MCU2		 0x0012U
+#define RGX_CNTBLK_ID_TPU_MCU2_DA	 0x8012U
+#define	RGX_CNTBLK_ID_TPU_MCU3		 0x0013U
+#define RGX_CNTBLK_ID_TPU_MCU3_DA	 0x8013U
+#define	RGX_CNTBLK_ID_TPU_MCU4		 0x0014U
+#define RGX_CNTBLK_ID_TPU_MCU4_DA	 0x8014U
+#define	RGX_CNTBLK_ID_TPU_MCU5		 0x0015U
+#define RGX_CNTBLK_ID_TPU_MCU5_DA	 0x8015U
+#define	RGX_CNTBLK_ID_TPU_MCU6		 0x0016U
+#define RGX_CNTBLK_ID_TPU_MCU6_DA	 0x8016U
+#define	RGX_CNTBLK_ID_TPU_MCU7		 0x0017U
+#define RGX_CNTBLK_ID_TPU_MCU7_DA	 0x8017U
+#define	RGX_CNTBLK_ID_TPU_MCU_ALL	 0x4010U
+#define	RGX_CNTBLK_ID_TPU_MCU_ALL_DA	 0xC010U
+
+#define	RGX_CNTBLK_ID_USC0			 0x0020U /*!< Addressable by Cluster */
+#define RGX_CNTBLK_ID_USC0_DA		 0x8020U
+#define	RGX_CNTBLK_ID_USC1			 0x0021U
+#define RGX_CNTBLK_ID_USC1_DA		 0x8021U
+#define	RGX_CNTBLK_ID_USC2			 0x0022U
+#define RGX_CNTBLK_ID_USC2_DA		 0x8022U
+#define	RGX_CNTBLK_ID_USC3			 0x0023U
+#define RGX_CNTBLK_ID_USC3_DA		 0x8023U
+#define	RGX_CNTBLK_ID_USC4			 0x0024U
+#define RGX_CNTBLK_ID_USC4_DA		 0x8024U
+#define	RGX_CNTBLK_ID_USC5			 0x0025U
+#define RGX_CNTBLK_ID_USC5_DA		 0x8025U
+#define	RGX_CNTBLK_ID_USC6			 0x0026U
+#define RGX_CNTBLK_ID_USC6_DA		 0x8026U
+#define	RGX_CNTBLK_ID_USC7			 0x0027U
+#define RGX_CNTBLK_ID_USC7_DA		 0x8027U
+#define	RGX_CNTBLK_ID_USC8			 0x0028U
+#define RGX_CNTBLK_ID_USC8_DA		 0x8028U
+#define	RGX_CNTBLK_ID_USC9			 0x0029U
+#define RGX_CNTBLK_ID_USC9_DA		 0x8029U
+#define	RGX_CNTBLK_ID_USC10			 0x002AU
+#define RGX_CNTBLK_ID_USC10_DA		 0x802AU
+#define	RGX_CNTBLK_ID_USC11			 0x002BU
+#define RGX_CNTBLK_ID_USC11_DA		 0x802BU
+#define	RGX_CNTBLK_ID_USC12			 0x002CU
+#define RGX_CNTBLK_ID_USC12_DA		 0x802CU
+#define	RGX_CNTBLK_ID_USC13			 0x002DU
+#define RGX_CNTBLK_ID_USC13_DA		 0x802DU
+#define	RGX_CNTBLK_ID_USC14			 0x002EU
+#define RGX_CNTBLK_ID_USC14_DA		 0x802EU
+#define	RGX_CNTBLK_ID_USC15			 0x002FU
+#define RGX_CNTBLK_ID_USC15_DA		 0x802FU
+#define	RGX_CNTBLK_ID_USC_ALL		 0x4020U
+#define	RGX_CNTBLK_ID_USC_ALL_DA	 0xC020U
+
+#define	RGX_CNTBLK_ID_TEXAS0		 0x0030U /*!< Addressable by Phantom in XT, Dust in S7 */
+#define	RGX_CNTBLK_ID_TEXAS1		 0x0031U
+#define	RGX_CNTBLK_ID_TEXAS2		 0x0032U
+#define	RGX_CNTBLK_ID_TEXAS3		 0x0033U
+#define	RGX_CNTBLK_ID_TEXAS4		 0x0034U
+#define	RGX_CNTBLK_ID_TEXAS5		 0x0035U
+#define	RGX_CNTBLK_ID_TEXAS6		 0x0036U
+#define	RGX_CNTBLK_ID_TEXAS7		 0x0037U
+#define	RGX_CNTBLK_ID_TEXAS_ALL		 0x4030U
+
+#define	RGX_CNTBLK_ID_RASTER0		 0x0040U /*!< Addressable by Phantom, XT only */
+#define	RGX_CNTBLK_ID_RASTER1		 0x0041U
+#define	RGX_CNTBLK_ID_RASTER2		 0x0042U
+#define	RGX_CNTBLK_ID_RASTER3		 0x0043U
+#define	RGX_CNTBLK_ID_RASTER_ALL	 0x4040U
+
+#define	RGX_CNTBLK_ID_BLACKPEARL0	 0x0050U /*!< Addressable by Phantom, S7, only */
+#define	RGX_CNTBLK_ID_BLACKPEARL1	 0x0051U
+#define	RGX_CNTBLK_ID_BLACKPEARL2	 0x0052U
+#define	RGX_CNTBLK_ID_BLACKPEARL3	 0x0053U
+#define	RGX_CNTBLK_ID_BLACKPEARL_ALL 0x4050U
+
+#define	RGX_CNTBLK_ID_PBE0			 0x0060U /*!< Addressable by Cluster in S7 and PBE2_IN_XE */
+#define	RGX_CNTBLK_ID_PBE1			 0x0061U
+#define	RGX_CNTBLK_ID_PBE2			 0x0062U
+#define	RGX_CNTBLK_ID_PBE3			 0x0063U
+#define	RGX_CNTBLK_ID_PBE4			 0x0064U
+#define	RGX_CNTBLK_ID_PBE5			 0x0065U
+#define	RGX_CNTBLK_ID_PBE6			 0x0066U
+#define	RGX_CNTBLK_ID_PBE7			 0x0067U
+#define	RGX_CNTBLK_ID_PBE8			 0x0068U
+#define	RGX_CNTBLK_ID_PBE9			 0x0069U
+#define	RGX_CNTBLK_ID_PBE10			 0x006AU
+#define	RGX_CNTBLK_ID_PBE11			 0x006BU
+#define	RGX_CNTBLK_ID_PBE12			 0x006CU
+#define	RGX_CNTBLK_ID_PBE13			 0x006DU
+#define	RGX_CNTBLK_ID_PBE14			 0x006EU
+#define	RGX_CNTBLK_ID_PBE15			 0x006FU
+#define	RGX_CNTBLK_ID_PBE_ALL		 0x4060U
+
+#define	RGX_CNTBLK_ID_LAST			 0x0070U /*!< End of PBE block */
+
+#define	RGX_CNTBLK_ID_BX_TU0_DEPRECATED		 0x0070U /*!< Doppler unit, DEPRECATED */
+#define	RGX_CNTBLK_ID_BX_TU1_DEPRECATED		 0x0071U
+#define	RGX_CNTBLK_ID_BX_TU2_DEPRECATED		 0x0072U
+#define	RGX_CNTBLK_ID_BX_TU3_DEPRECATED		 0x0073U
+#define	RGX_CNTBLK_ID_BX_TU_ALL_DEPRECATED	 0x4070U
+
+#define	RGX_CNTBLK_ID_CUSTOM0		 0x70F0U
+#define	RGX_CNTBLK_ID_CUSTOM1		 0x70F1U
+#define	RGX_CNTBLK_ID_CUSTOM2		 0x70F2U
+#define	RGX_CNTBLK_ID_CUSTOM3		 0x70F3U
+#define	RGX_CNTBLK_ID_CUSTOM4_FW	 0x70F4U /*!< Custom block used for getting statistics held in the FW */
+#define	RGX_CNTBLK_ID_CUSTOM_MASK	 0x70FFU
+
+
+/* Masks for the counter block ID*/
+#define	RGX_CNTBLK_ID_UNIT_MASK      (0x000FU)
+#define	RGX_CNTBLK_ID_GROUP_MASK     (0x00F0U)
+#define	RGX_CNTBLK_ID_GROUP_SHIFT    (4U)
+#define	RGX_CNTBLK_ID_MC_GPU_MASK    (0x0F00U)
+#define	RGX_CNTBLK_ID_MC_GPU_SHIFT   (8U)
+#define	RGX_CNTBLK_ID_UNIT_ALL_MASK  (0x4000U)
+#define	RGX_CNTBLK_ID_DA_MASK        (0x8000U) /*!< Block with directly accessible counter registers */
+
+#define RGX_CNTBLK_INDIRECT_COUNT(_class, _n) ((IMG_UINT32)(RGX_CNTBLK_ID_ ## _class ## _n) - (IMG_UINT32)(RGX_CNTBLK_ID_ ## _class ## 0) + 1u)
+
+/*! The number of layout blocks defined with configurable multiplexed
+ * performance counters, hence excludes custom counter blocks.
+ */
+#define RGX_HWPERF_MAX_DEFINED_BLKS  (\
+	(IMG_UINT32)RGX_CNTBLK_ID_DIRECT_LAST    +\
+	RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU,     7)+\
+	RGX_CNTBLK_INDIRECT_COUNT(USC,        15)+\
+	RGX_CNTBLK_INDIRECT_COUNT(TEXAS,       7)+\
+	RGX_CNTBLK_INDIRECT_COUNT(RASTER,      3)+\
+	RGX_CNTBLK_INDIRECT_COUNT(BLACKPEARL,  3)+\
+	RGX_CNTBLK_INDIRECT_COUNT(PBE,        15) )
+#define RGX_HWPERF_MAX_MUX_BLKS      (\
+    RGX_HWPERF_MAX_DEFINED_BLKS      )
+
+static_assert(
+	((RGX_CNTBLK_ID_DIRECT_LAST + ((RGX_CNTBLK_ID_LAST & RGX_CNTBLK_ID_GROUP_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT)) <= RGX_HWPERF_MAX_BVNC_BLOCK_LEN),
+	"RGX_HWPERF_MAX_BVNC_BLOCK_LEN insufficient");
+
+#define RGX_HWPERF_EVENT_MASK_VALUE(e)      (IMG_UINT64_C(1) << (IMG_UINT32)(e))
+
+#define RGX_CUSTOM_FW_CNTRS	\
+                X(TA_LOCAL_FL_SIZE,    0x0, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(TA_GLOBAL_FL_SIZE,   0x1, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(3D_LOCAL_FL_SIZE,    0x2, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(3D_GLOBAL_FL_SIZE,   0x3, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(ISP_TILES_IN_FLIGHT, 0x4, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DSPMKICK))
+
+/*! Counter IDs for the firmware held statistics */
+typedef enum
+{
+#define X(ctr, id, allow_mask)	RGX_CUSTOM_FW_CNTR_##ctr = id,
+	RGX_CUSTOM_FW_CNTRS
+#undef X
+
+	/* always the last entry in the list */
+	RGX_CUSTOM_FW_CNTR_LAST
+} RGX_HWPERF_CUSTOM_FW_CNTR_ID;
+
+/*! Identifier for each counter in a performance counting module */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_COUNTER_ID;
+
+#define	RGX_CNTBLK_COUNTER0_ID 0U
+#define	RGX_CNTBLK_COUNTER1_ID 1U
+#define	RGX_CNTBLK_COUNTER2_ID 2U
+#define	RGX_CNTBLK_COUNTER3_ID 3U
+#define	RGX_CNTBLK_COUNTER4_ID 4U
+#define	RGX_CNTBLK_COUNTER5_ID 5U
+	/* MAX value used in server handling of counter config arrays */
+#define	RGX_CNTBLK_MUX_COUNTERS_MAX 6U
+
+
+/* sets all the bits from bit _b1 to _b2, in a IMG_UINT64 type */
+#define MASK_RANGE_IMPL(b1, b2)	((IMG_UINT64)((IMG_UINT64_C(1) << ((IMG_UINT32)(b2)-(IMG_UINT32)(b1) + 1U)) - 1U) << (IMG_UINT32)(b1))
+#define MASK_RANGE(R)			MASK_RANGE_IMPL(R##_FIRST_TYPE, R##_LAST_TYPE)
+#define RGX_HWPERF_HOST_EVENT_MASK_VALUE(e) (IMG_UINT32_C(1) << (e))
+
+/*! Mask macros for use with RGXCtrlHWPerf() API.
+ */
+#define RGX_HWPERF_EVENT_MASK_NONE          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_HWPERF_EVENT_MASK_DEFAULT       RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_FWACT) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG)
+#define RGX_HWPERF_EVENT_MASK_ALL           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+
+/*! HWPerf Firmware event masks
+ * @par
+ * All FW Start/End/Debug (SED) events. */
+#define RGX_HWPERF_EVENT_MASK_FW_SED    (MASK_RANGE(RGX_HWPERF_FW_EVENT_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_FW_UFO    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO))
+#define RGX_HWPERF_EVENT_MASK_FW_CSW    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_START) |\
+                                          RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_FINISHED))
+/*! All FW events. */
+#define RGX_HWPERF_EVENT_MASK_ALL_FW    (RGX_HWPERF_EVENT_MASK_FW_SED |\
+                                          RGX_HWPERF_EVENT_MASK_FW_UFO |\
+                                          RGX_HWPERF_EVENT_MASK_FW_CSW)
+
+/*! HW Periodic events (1ms interval). */
+#define RGX_HWPERF_EVENT_MASK_HW_PERIODIC   (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PERIODIC))
+/*! All HW Kick/Finish events. */
+#define RGX_HWPERF_EVENT_MASK_HW_KICKFINISH ((MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE0) |\
+                                               MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE1)) &\
+                                              ~(RGX_HWPERF_EVENT_MASK_HW_PERIODIC))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_HW        (RGX_HWPERF_EVENT_MASK_HW_KICKFINISH |\
+                                              RGX_HWPERF_EVENT_MASK_HW_PERIODIC)
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR_EST   (MASK_RANGE(RGX_HWPERF_PWR_EST_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR       (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_GPU_STATE_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG))
+
+/*! HWPerf Host event masks
+ */
+#define RGX_HWPERF_EVENT_MASK_HOST_WORK_ENQ  (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_ENQ))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_UFO   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_UFO))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_PWR   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_CLK_SYNC))
+
+
+/*! Type used in the RGX API RGXConfigMuxHWPerfCounters() */
+typedef struct
+{
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! 4 or 6 LSBs used to select counters to configure in this block. */
+	IMG_UINT8  ui8CounterSelect;
+
+	/*! 4 or 6 LSBs used as MODE bits for the counters in the group. */
+	IMG_UINT8  ui8Mode;
+
+	/*! 5 or 6 LSBs used as the GROUP_SELECT value for the counter. */
+	IMG_UINT8  aui8GroupSelect[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 16 LSBs used as the BIT_SELECT value for the counter. */
+	IMG_UINT16 aui16BitSelect[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 14 LSBs used as the BATCH_MAX value for the counter. */
+	IMG_UINT32 aui32BatchMax[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 14 LSBs used as the BATCH_MIN value for the counter. */
+	IMG_UINT32 aui32BatchMin[RGX_CNTBLK_MUX_COUNTERS_MAX];
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_MUX_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+
+/*! Type used in the RGX API RGXConfigHWPerfCounters() */
+typedef struct
+{
+	/*! Reserved for future use */
+	IMG_UINT32 ui32Reserved;
+
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of configured counters within this block */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Counter register values */
+	IMG_UINT16 ui16Counters[RGX_CNTBLK_COUNTERS_MAX];
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_CNTBLK);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_options.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_options.h
new file mode 100644
index 000000000000..df9c949f4a1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgx_options.h
@@ -0,0 +1,350 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX build options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Each build option listed here is packed into a dword which provides up to
+ *  log2(RGX_BUILD_OPTIONS_MASK_KM + 1) flags for KM and
+ *  (32 - log2(RGX_BUILD_OPTIONS_MASK_KM + 1)) flags for UM.
+ * The corresponding bit is set if the build option was enabled at compile
+ * time.
+ *
+ * IMPORTANT: add new options to unused bits or define a new dword
+ * (e.g. RGX_BUILD_OPTIONS_KM2 or RGX_BUILD_OPTIONS2) so that the bitfield
+ * remains backwards compatible.
+ */
+
+#ifndef RGX_OPTIONS_H
+#define RGX_OPTIONS_H
+
+#define OPTIONS_OPEN_SOURCE_EN					(0x1UL << 0)
+#define OPTIONS_PDUMP_EN						(0x1UL << 1)
+#define OPTIONS_UNUSED1_EN						(0x1UL << 2)
+#define OPTIONS_SECURE_ALLOC_KM_EN				(0x1UL << 3)
+#define OPTIONS_RGX_EN							(0x1UL << 4)
+#define OPTIONS_SECURE_EXPORT_EN				(0x1UL << 5)
+#define OPTIONS_INSECURE_EXPORT_EN				(0x1UL << 6)
+#define OPTIONS_VFP_EN							(0x1UL << 7)
+#define OPTIONS_WORKLOAD_ESTIMATION_EN			(0x1UL << 8)
+#define OPTIONS_PDVFS_EN						(0x1UL << 9)
+#define OPTIONS_DEBUG_EN						(0x1UL << 10)
+#define OPTIONS_BUFFER_SYNC_EN					(0x1UL << 11)
+#define OPTIONS_AUTOVZ_EN						(0x1UL << 12)
+#define OPTIONS_AUTOVZ_HW_REGS_EN				(0x1UL << 13)
+#define OPTIONS_FW_IRQ_REG_COUNTERS_EN			(0x1UL << 14)
+#define OPTIONS_VALIDATION_EN					(0x1UL << 15)
+#define OPTIONS_NO_HARDWARE_EN					(0x1UL << 16)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN                  (0x1UL << 17)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_MASK                      (0x1fUL << 18)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT                     (18)
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_PERCONTEXT_FREELIST_EN			(0x1UL << 31)
+
+#define RGX_BUILD_OPTIONS_MASK_KM				  \
+			(OPTIONS_OPEN_SOURCE_EN				| \
+			 OPTIONS_PDUMP_EN					| \
+			 OPTIONS_SECURE_ALLOC_KM_EN			| \
+			 OPTIONS_RGX_EN						| \
+			 OPTIONS_SECURE_EXPORT_EN			| \
+			 OPTIONS_INSECURE_EXPORT_EN			| \
+			 OPTIONS_VFP_EN						| \
+			 OPTIONS_WORKLOAD_ESTIMATION_EN		| \
+			 OPTIONS_PDVFS_EN					| \
+			 OPTIONS_DEBUG_EN					| \
+			 OPTIONS_BUFFER_SYNC_EN				| \
+			 OPTIONS_AUTOVZ_EN					| \
+			 OPTIONS_AUTOVZ_HW_REGS_EN			| \
+			 OPTIONS_FW_IRQ_REG_COUNTERS_EN		| \
+			 OPTIONS_VALIDATION_EN				| \
+			 OPTIONS_NO_HARDWARE_EN				| \
+			 OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN)
+
+#define RGX_BUILD_OPTIONS_MASK_FW \
+			(RGX_BUILD_OPTIONS_MASK_KM & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_MASK & \
+			 ~OPTIONS_BUFFER_SYNC_EN)
+
+/* Build options that the FW must have if the present on the KM */
+#define FW_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the UM must have if the present on the KM */
+#define UM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the KM must have if the present on the UM */
+#define KM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN					| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_BUFFER_SYNC_EN))
+
+#define OPEN_SOURCE_OPTION	"OPEN_SOURCE_DRIVER  "
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	#define OPTIONS_BIT0		OPTIONS_OPEN_SOURCE_EN
+	#if OPTIONS_BIT0 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT0		0x0UL
+#endif /* SUPPORT_OPEN_SOURCE_DRIVER */
+
+#define PDUMP_OPTION	"PDUMP  "
+#if defined(PDUMP)
+	#define OPTIONS_BIT1		OPTIONS_PDUMP_EN
+	#if OPTIONS_BIT1 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT1		0x0UL
+#endif /* PDUMP */
+
+/* No longer used */
+#define INTERNAL_UNUSED1_OPTION	"INTERNAL_UNUSED1  "
+#if defined(INTERNAL_UNUSED1)
+	#define OPTIONS_BIT2		OPTIONS_UNUSED1_EN
+	#if OPTIONS_BIT2 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT2		0x0UL
+#endif
+
+#define SECURE_ALLOC_KM_OPTION "SECURE_ALLOC_KM  "
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	#define OPTIONS_BIT3		OPTIONS_SECURE_ALLOC_KM_EN
+	#if OPTIONS_BIT3 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT3 0x0UL
+#endif /* SUPPORT_SECURE_ALLOC_KM */
+
+#define RGX_OPTION	" "
+#if defined(SUPPORT_RGX)
+	#define OPTIONS_BIT4		OPTIONS_RGX_EN
+	#if OPTIONS_BIT4 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT4		0x0UL
+#endif /* SUPPORT_RGX */
+
+#define SECURE_EXPORT_OPTION	"SECURE_EXPORTS  "
+#if defined(SUPPORT_SECURE_EXPORT)
+	#define OPTIONS_BIT5		OPTIONS_SECURE_EXPORT_EN
+	#if OPTIONS_BIT5 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT5		0x0UL
+#endif /* SUPPORT_SECURE_EXPORT */
+
+#define INSECURE_EXPORT_OPTION	"INSECURE_EXPORTS  "
+#if defined(SUPPORT_INSECURE_EXPORT)
+	#define OPTIONS_BIT6		OPTIONS_INSECURE_EXPORT_EN
+	#if OPTIONS_BIT6 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT6		0x0UL
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+#define VFP_OPTION	"VFP  "
+#if defined(SUPPORT_VFP)
+	#define OPTIONS_BIT7		OPTIONS_VFP_EN
+	#if OPTIONS_BIT7 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT7		0x0UL
+#endif /* SUPPORT_VFP */
+
+#define WORKLOAD_ESTIMATION_OPTION	"WORKLOAD_ESTIMATION  "
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	#define OPTIONS_BIT8		OPTIONS_WORKLOAD_ESTIMATION_EN
+	#if OPTIONS_BIT8 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT8		0x0UL
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+
+#define PDVFS_OPTION	"PDVFS  "
+#if defined(SUPPORT_PDVFS)
+	#define OPTIONS_BIT9		OPTIONS_PDVFS_EN
+	#if OPTIONS_BIT9 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT9		0x0UL
+#endif /* SUPPORT_PDVFS */
+
+#define DEBUG_OPTION	"DEBUG  "
+#if defined(DEBUG)
+	#define OPTIONS_BIT10		OPTIONS_DEBUG_EN
+	#if OPTIONS_BIT10 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT10		0x0UL
+#endif /* DEBUG */
+
+#define BUFFER_SYNC_OPTION	"BUFFER_SYNC  "
+#if defined(SUPPORT_BUFFER_SYNC)
+	#define OPTIONS_BIT11		OPTIONS_BUFFER_SYNC_EN
+	#if OPTIONS_BIT11 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT11		0x0UL
+#endif /* SUPPORT_BUFFER_SYNC */
+
+#define AUTOVZ_OPTION	"AUTOVZ  "
+#if defined(SUPPORT_AUTOVZ)
+	#define OPTIONS_BIT12		OPTIONS_AUTOVZ_EN
+	#if OPTIONS_BIT12 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT12		0x0UL
+#endif /* SUPPORT_AUTOVZ */
+
+#define AUTOVZ_HW_REGS_OPTION	"AUTOVZ_HW_REGS  "
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+	#define OPTIONS_BIT13		OPTIONS_AUTOVZ_HW_REGS_EN
+	#if OPTIONS_BIT13 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT13		0x0UL
+#endif /* SUPPORT_AUTOVZ_HW_REGS */
+
+#define RGX_FW_IRQ_OS_COUNTERS_OPTION	"FW_IRQ_OS_COUNTERS  "
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	#define OPTIONS_BIT14		OPTIONS_FW_IRQ_REG_COUNTERS_EN
+	#if OPTIONS_BIT14 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT14		0x0UL
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+#define VALIDATION_OPTION	"VALIDATION  "
+	#define OPTIONS_BIT15		0x0UL
+
+#define NO_HARDWARE_OPTION	"NO_HARDWARE  "
+#if defined(NO_HARDWARE)
+	#define OPTIONS_BIT16		OPTIONS_NO_HARDWARE_EN
+	#if OPTIONS_BIT16 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT16		0x0UL
+#endif /* NO_HARDWARE */
+
+#define NUM_DRIVERS_SUPPORTED_CHECK_OPTION	"NUM_DRIVERS_SUPPORTED_CHECK  "
+	#define OPTIONS_BIT17		OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN
+	#if OPTIONS_BIT17 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif /* NUM_DRIVERS_SUPPORTED_CHECK */
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_BIT31		OPTIONS_PERCONTEXT_FREELIST_EN
+#if OPTIONS_BIT31 <= RGX_BUILD_OPTIONS_MASK_KM
+#error "Bit exceeds reserved range"
+#endif
+
+#define RGX_BUILD_OPTIONS_KM	\
+	(OPTIONS_BIT0				|\
+	 OPTIONS_BIT1				|\
+	 OPTIONS_BIT2				|\
+	 OPTIONS_BIT3				|\
+	 OPTIONS_BIT4				|\
+	 OPTIONS_BIT5				|\
+	 OPTIONS_BIT6				|\
+	 OPTIONS_BIT7				|\
+	 OPTIONS_BIT8				|\
+	 OPTIONS_BIT9				|\
+	 OPTIONS_BIT10				|\
+	 OPTIONS_BIT11				|\
+	 OPTIONS_BIT12				|\
+	 OPTIONS_BIT13				|\
+	 OPTIONS_BIT14				|\
+	 OPTIONS_BIT15				|\
+	 OPTIONS_BIT16				|\
+	 OPTIONS_BIT17)
+
+#define RGX_BUILD_OPTIONS (RGX_BUILD_OPTIONS_KM | OPTIONS_BIT31)
+
+#define RGX_BUILD_OPTIONS_LIST			\
+	{									\
+		OPEN_SOURCE_OPTION,				\
+		PDUMP_OPTION,					\
+		INTERNAL_UNUSED1_OPTION,		\
+		SECURE_ALLOC_KM_OPTION,			\
+		RGX_OPTION,						\
+		SECURE_EXPORT_OPTION,			\
+		INSECURE_EXPORT_OPTION,			\
+		VFP_OPTION,						\
+		WORKLOAD_ESTIMATION_OPTION,		\
+		PDVFS_OPTION,					\
+		DEBUG_OPTION,					\
+		BUFFER_SYNC_OPTION,				\
+		AUTOVZ_OPTION,					\
+		AUTOVZ_HW_REGS_OPTION,			\
+		RGX_FW_IRQ_OS_COUNTERS_OPTION,	\
+		VALIDATION_OPTION,				\
+		NO_HARDWARE_OPTION,				\
+		NUM_DRIVERS_SUPPORTED_CHECK_OPTION				\
+	}
+
+#endif /* RGX_OPTIONS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig.h
new file mode 100644
index 000000000000..6825da5caa3b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig.h
@@ -0,0 +1,288 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_H
+#define RGXHEAPCONFIG_H
+
+#include "rgxdefs_km.h"
+
+#define RGX_HEAP_SIZE_32KiB      IMG_UINT64_C(0x0000008000)
+#define RGX_HEAP_SIZE_2MiB       IMG_UINT64_C(0x0000200000)
+#define RGX_HEAP_SIZE_4MiB       IMG_UINT64_C(0x0000400000)
+#define RGX_HEAP_SIZE_16MiB      IMG_UINT64_C(0x0001000000)
+#define RGX_HEAP_SIZE_32MiB      IMG_UINT64_C(0x0002000000)
+#define RGX_HEAP_SIZE_256MiB     IMG_UINT64_C(0x0010000000)
+
+#define RGX_HEAP_SIZE_1GiB       IMG_UINT64_C(0x0040000000)
+#define RGX_HEAP_SIZE_2GiB       IMG_UINT64_C(0x0080000000)
+#define RGX_HEAP_SIZE_4GiB       IMG_UINT64_C(0x0100000000)
+#define RGX_HEAP_SIZE_16GiB      IMG_UINT64_C(0x0400000000)
+#define RGX_HEAP_SIZE_32GiB      IMG_UINT64_C(0x0800000000)
+#define RGX_HEAP_SIZE_64GiB      IMG_UINT64_C(0x1000000000)
+#define RGX_HEAP_SIZE_128GiB     IMG_UINT64_C(0x2000000000)
+#define RGX_HEAP_SIZE_256GiB     IMG_UINT64_C(0x4000000000)
+#define RGX_HEAP_SIZE_512GiB     IMG_UINT64_C(0x8000000000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address heaps that are used in
+	application memory contexts. It also shows where the Firmware memory heap
+	fits into this, but the firmware heap is only ever created in the
+	Services KM/server component.
+
+	RGX_PDSCODEDATA_HEAP_BASE and RGX_USCCODE_HEAP_BASE will be programmed,
+	on a global basis, into RGX_CR_PDS_EXEC_BASE and RGX_CR_USC_CODE_BASE_*
+	respectively. Therefore if clients use multiple configs they must still
+	be consistent with their definitions for these heaps.
+
+	Shared virtual memory (GENERAL_SVM) support requires half of the address
+	space (512 GiB) be reserved for SVM allocations to mirror application CPU
+	addresses. However, if BRN_65273 WA is active in which case the SVM heap
+	is disabled. This is reflected in the device connection capability bits
+	returned to user space.
+
+	The GENERAL non-SVM region is 512 GiB to 768 GiB and is shared between the
+	general (4KiB) heap and the general non-4K heap. The first 128 GiB is used
+	for the GENERAL_HEAP (4KiB) and the last 32 GiB is used for the
+	GENERAL_NON4K_HEAP. This heap has a default page-size of 16K.
+	AppHint PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE can be used to forced it
+	to these values: 4K,64K,256K,1M,2M.
+
+	The heaps defined for BRN_65273 _replace_ the non-BRN equivalents below
+	when this BRN WA is active on affected cores. This is different to most
+	other BRNs and hence has been given its own header file for clarity,
+	see below. This	is a special case, other BRNs that need 1 or 2 additional
+	heaps should be added to this file, like BRN_63142 below.
+	NOTE: All regular heaps below greater than 1GB require a BRN_65273 WA heap.
+
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range. Often this is the reserved
+	size of the heap within the range.
+*/
+
+/* This BRN requires a different virtual memory map from the standard one
+ * defined in this file below. Hence the alternative heap definitions for this
+ * BRN are provided in a separate file for clarity. */
+#include "rgxheapconfig_65273.h"
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0000_0000 - 0x00_0020_0000 **/
+	/* 0 MiB to 2 MiB, size of 2 MiB : RESERVED (only when General SVM
+	 *                                           doesn't exist) **/
+
+	/* BRN_65273 TQ3DPARAMETERS base 0x0000010000 */
+	/* BRN_65273 GENERAL base        0x65C0000000 */
+	/* BRN_65273 GENERAL_NON4K base  0x73C0000000 */
+
+/* 0x00_0000_8000 - 0x7F_FFFF_8000 **/
+	/* MAX(32 KiB, PAGE_SIZE) to 512 GiB, size of 512 GiB less MAX(32 KiB, PAGE_SIZE) : GENERAL_SVM_HEAP **/
+
+	/* The MAX is determined at runtime (PAGE_SIZE isn't available on all platforms)
+	 * so the #define's must NOT be used directly. Use the heap config after initialisation. */
+	#define RGX_GENERAL_SVM_HEAP_BASE           IMG_UINT64_C(0x0000008000)
+	#define RGX_GENERAL_SVM_HEAP_SIZE           (RGX_HEAP_SIZE_512GiB - RGX_HEAP_SIZE_32KiB)
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0x80_0000_0000 - 0x9F_FFFF_FFFF **/
+	/* 512 GiB to 640 GiB, size of 128 GiB : GENERAL_HEAP **/
+	#define RGX_GENERAL_HEAP_BASE               IMG_UINT64_C(0x8000000000)
+	#define RGX_GENERAL_HEAP_SIZE               RGX_HEAP_SIZE_128GiB
+
+	/* BRN_65273 PDSCODEDATA base    0xA800000000 */
+
+/* 0xA0_0000_0000 - 0xAF_FFFF_FFFF **/
+	/* 640 GiB to 704 GiB, size of 64 GiB : FREE **/
+
+/* B0_0000_0000 - 0xB7_FFFF_FFFF **/
+	/* 704 GiB to 736 GiB, size of 32 GiB : FREE **/
+
+	/* BRN_65273 USCCODE base        0xBA00000000 */
+
+/* 0xB8_0000_0000 - 0xBF_FFFF_FFFF **/
+	/* 736 GiB to 768 GiB, size of 32 GiB : GENERAL_NON4K_HEAP **/
+	#define RGX_GENERAL_NON4K_HEAP_BASE         IMG_UINT64_C(0xB800000000)
+	#define RGX_GENERAL_NON4K_HEAP_SIZE         RGX_HEAP_SIZE_32GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xC0_0000_0000 - 0xD9_FFFF_FFFF **/
+	/* 768 GiB to 872 GiB, size of 104 GiB : FREE **/
+
+/* 0xDA_0000_0000 - 0xDA_FFFF_FFFF **/
+	/* 872 GiB to 876 GiB, size of 4 GiB : PDSCODEDATA_HEAP **/
+	#define RGX_PDSCODEDATA_HEAP_BASE           IMG_UINT64_C(0xDA00000000)
+	#define RGX_PDSCODEDATA_HEAP_SIZE           RGX_HEAP_SIZE_4GiB
+
+/* 0xDB_0000_0000 - 0xDB_FFFF_FFFF **/
+	/* 876 GiB to 880 GiB, size of 256 MiB (reserved 4GiB) : BRN **/
+	/* HWBRN63142 workaround requires Region Header memory to be at the top
+	   of a 16GiB aligned range. This is so when masked with 0x03FFFFFFFF the
+	   address will avoid aliasing PB addresses. Start at 879.75GiB. Size of 256MiB. */
+	#define RGX_RGNHDR_BRN_63142_HEAP_BASE      IMG_UINT64_C(0xDBF0000000)
+	#define RGX_RGNHDR_BRN_63142_HEAP_SIZE      RGX_HEAP_SIZE_256MiB
+
+/* 0xDC_0000_0000 - 0xDF_FFFF_FFFF **/
+	/* 880 GiB to 896 GiB, size of 16 GiB : FREE **/
+
+/* 0xE0_0000_0000 - 0xE0_FDFF_FFFF **/
+	/* 896 GiB to 900 GiB, size of 4 GiB less 32 MiB : USCCODE_HEAP **/
+	#define RGX_USCCODE_HEAP_BASE               IMG_UINT64_C(0xE000000000)
+	#define RGX_USCCODE_HEAP_SIZE               (RGX_HEAP_SIZE_4GiB - RGX_HEAP_SIZE_32MiB)
+
+/* 0xE0_FE00_0000 - 0xE0_FFFF_FFFF **/
+	/* 900 GiB less 32 MiB to 900 GiB, size of 32 MiB : RESERVED VOLCANIC **/
+
+/* 0xE1_0000_0000 - 0xE1_BFFF_FFFF **/
+	/* 900 GiB to 903 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xE1_C000_000 - 0xE1_FFFF_FFFF **/
+	/* 903 GiB to 904 GiB, reserved 1 GiB, : FIRMWARE_HEAP **/
+
+	/* Firmware heaps defined in rgx_heap_firmware.h as they are not present in
+	   application memory contexts, see:
+	    RGX_FIRMWARE_RAW_HEAP_BASE
+	    RGX_FIRMWARE_RAW_HEAP_SIZE
+	   See header for other sub-heaps details
+	*/
+
+/* 0xE2_0000_0000 - 0xE3_FFFF_FFFF **/
+	/* 904 GiB to 912 GiB, size of 8 GiB : FREE **/
+
+	/* BRN_65273 VISIBILITY_TEST base 0xE400000000 */
+
+/* 0xE4_0000_0000 - 0xE7_FFFF_FFFF **/
+	/* 912 GiB to 928 GiB, size 16 GiB : TQ3DPARAMETERS_HEAP **/
+	/* Aligned to match RGX_CR_ISP_PIXEL_BASE at 16 GiB */
+	#define RGX_TQ3DPARAMETERS_HEAP_BASE        IMG_UINT64_C(0xE400000000)
+	#define RGX_TQ3DPARAMETERS_HEAP_SIZE        RGX_HEAP_SIZE_16GiB
+
+/* 0xE8_0000_0000 - 0xE8_FFFF_FFFF **/
+	/* 928 GiB to 932 GiB, size of 4 GiB : FREE **/
+
+/* 0xE9_0000_0000 - 0xE9_3FFF_FFFF **/
+	/* 932 GiB to 933 GiB, size of 1 GiB : VK_CAPT_REPLAY_HEAP **/
+	#define RGX_VK_CAPT_REPLAY_HEAP_BASE        IMG_UINT64_C(0xE900000000)
+	#define RGX_VK_CAPT_REPLAY_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xE9_4000_0000 - 0xE9_FFFF_FFFF **/
+	/* 933 GiB to 936 GiB, size of 3 GiB : FREE **/
+
+/* 0xEA_0000_0000 - 0xEA_001F_FFFF **/
+	/* 936 GiB to 937 GiB, size of 1 GiB : FREE **/
+
+/* 0xEA_4000_0000 - 0xEA_FFFF_FFFF **/
+	/* 937 GiB to 940 GiB, size of 3 GiB : FREE **/
+
+/* 0xEB_0000_0000 - 0xEB_FFFF_FFFF **/
+	/* 940 GiB to 944 GiB, size of 4 GiB : RESERVED VOLCANIC  **/
+
+/* 0xEC_0000_0000 - 0xEC_001F_FFFF **/
+	/* 944 GiB to 945 GiB, size 2 MiB : FBCDC_HEAP **/
+	#define RGX_FBCDC_HEAP_BASE                 IMG_UINT64_C(0xEC00000000)
+	#define RGX_FBCDC_HEAP_SIZE                 RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_4000_0000 - 0xEC_401F_FFFF **/
+	/* 945 GiB to 946 GiB, size 2 MiB : FBCDC_LARGE_HEAP **/
+	#define RGX_FBCDC_LARGE_HEAP_BASE           IMG_UINT64_C(0xEC40000000)
+	#define RGX_FBCDC_LARGE_HEAP_SIZE           RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_8000_0000 - 0xED_FFFF_FFFF **/
+	/* 946 GiB to 952 GiB, size of 6 GiB : RESERVED VOLCANIC  **/
+
+/* 0xEE_0000_0000 - 0xEE_3FFF_FFFF **/
+	/* 952 GiB to 953 GiB, size of 1 GiB : CMP_MISSION_RMW_HEAP **/
+	#define RGX_CMP_MISSION_RMW_HEAP_BASE       IMG_UINT64_C(0xEE00000000)
+	#define RGX_CMP_MISSION_RMW_HEAP_SIZE       RGX_HEAP_SIZE_1GiB
+
+/* 0xEE_4000_0000 - 0xEE_FFFF_FFFF **/
+	/* 953 GiB to 956 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xEF_0000_0000 - 0xEF_3FFF_FFFF **/
+	/* 956 GiB to 957 GiB, size of 1 GiB : CMP_SAFETY_RMW_HEAP **/
+	#define RGX_CMP_SAFETY_RMW_HEAP_BASE        IMG_UINT64_C(0xEF00000000)
+	#define RGX_CMP_SAFETY_RMW_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xEF_4000_0000 - 0xEF_FFFF_FFFF **/
+	/* 957 GiB to 960 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xF0_0000_0000 - 0xF0_FFFF_FFFF **/
+	/* 960 GiB to 964 GiB, size of 4 GiB : TEXTURE_STATE_HEAP (36-bit aligned) */
+	#define RGX_TEXTURE_STATE_HEAP_BASE         IMG_UINT64_C(0xF000000000)
+	#define RGX_TEXTURE_STATE_HEAP_SIZE         RGX_HEAP_SIZE_4GiB
+
+/* 0xF1_0000_0000 - 0xF1_FFFF_FFFF **/
+	/* 964 GiB to 968 GiB, size of 4 GiB : FREE **/
+
+/* 0xF2_0000_0000 - 0xF2_001F_FFFF **/
+	/* 968 GiB to 969 GiB, size of 2 MiB : VISIBILITY_TEST_HEAP **/
+	#define RGX_VISIBILITY_TEST_HEAP_BASE       IMG_UINT64_C(0xF200000000)
+	#define RGX_VISIBILITY_TEST_HEAP_SIZE       RGX_HEAP_SIZE_2MiB
+
+/* 0xF2_4000_0000 - 0xF2_FFFF_FFFF **/
+	/* 969 GiB to 972 GiB, size of 3 GiB : FREE **/
+
+	/* BRN_65273 MMU_INIA base 0xF800000000 */
+	/* BRN_65273 MMU_INIB base 0xF900000000 */
+
+/* 0xF3_0000_0000 - 0xFF_FFFF_FFFF **/
+	/* 972 GiB to 1024 GiB, size of 52 GiB : FREE **/
+
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig_65273.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig_65273.h
new file mode 100644
index 000000000000..66f44dc8c18b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/rgxheapconfig_65273.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map for BRN_65273.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_65273_H
+#define RGXHEAPCONFIG_65273_H
+
+#define RGX_HEAP_SIZE_64KiB      IMG_UINT64_C(0x0000010000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address replacement heaps that are used
+	in application memory contexts when BRN_65273 is present. The addresses
+	used are specifically crafted to avoid the BRN - under no circumstances
+	can they be moved, altered in size or additional heaps added.
+
+	The heaps defined for BRN_65273 _replace_ the non-BRN equivalents when this
+	BRN WA is active on affected cores. This is different to most other BRNs
+	and hence has been given its own header file for clarity. The SVM_HEAP is
+	also disabled and unavailable when the WA is active. This is reflected
+	in the device connection capability bits returned to user space.
+
+	NOTE: All regular heaps in rgxheapconfig.h greater than 1GB require
+	      a BRN_65273 WA heap.
+
+	Base addresses must have to be a multiple of 4MiB
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range.
+*/
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0001_0000 - 0x00_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires TQ memory to start at 64 KiB and use a
+	 * unique single 0.99GiB PCE entry. */
+	#define RGX_TQ3DPARAMETERS_BRN_65273_HEAP_BASE  IMG_UINT64_C(0x0000010000)
+	#define RGX_TQ3DPARAMETERS_BRN_65273_HEAP_SIZE  (RGX_HEAP_SIZE_1GiB - RGX_HEAP_SIZE_64KiB)
+
+/* 0x65_C000_0000 - 0x66_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires General Heap to use a unique PCE entry for each GiB in range */
+	#define RGX_GENERAL_BRN_65273_HEAP_BASE         IMG_UINT64_C(0x65C0000000)
+	#define RGX_GENERAL_BRN_65273_HEAP_SIZE         RGX_HEAP_SIZE_2GiB
+
+/* 0x73_C000_0000 - 0x74_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires Non4K memory to use a unique PCE entry for each GiB in range */
+	#define RGX_GENERAL_NON4K_BRN_65273_HEAP_BASE   IMG_UINT64_C(0x73C0000000)
+	#define RGX_GENERAL_NON4K_BRN_65273_HEAP_SIZE   RGX_HEAP_SIZE_2GiB
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0xA8_0000_0000 - 0xA8_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires PDS memory to use a unique single 1GiB PCE entry. */
+	#define RGX_PDSCODEDATA_BRN_65273_HEAP_BASE     IMG_UINT64_C(0xA800000000)
+	#define RGX_PDSCODEDATA_BRN_65273_HEAP_SIZE     RGX_HEAP_SIZE_1GiB
+
+/* 0xBA_0000_0000 - 0xBA_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires USC memory to use a unique single 1GiB PCE entry. */
+	#define RGX_USCCODE_BRN_65273_HEAP_BASE         IMG_UINT64_C(0xBA00000000)
+	#define RGX_USCCODE_BRN_65273_HEAP_SIZE         RGX_HEAP_SIZE_1GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xE4_0000_0000 - 0xE4_001F_FFFF **/
+	/* HWBRN65273 workaround requires USC memory to use a unique single 1GiB PCE entry. */
+	#define RGX_VISIBILITY_TEST_BRN_65273_HEAP_BASE IMG_UINT64_C(0xE400000000)
+	#define RGX_VISIBILITY_TEST_BRN_65273_HEAP_SIZE RGX_HEAP_SIZE_2MiB
+
+/* 0xF8_0000_0000 - 0xF9_FFFF_FFFF **/
+	/* HWBRN65273 workaround requires two Region Header buffers 4GiB apart. */
+	#define RGX_MMU_INIA_BRN_65273_HEAP_BASE        IMG_UINT64_C(0xF800000000)
+	#define RGX_MMU_INIA_BRN_65273_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+	#define RGX_MMU_INIB_BRN_65273_HEAP_BASE        IMG_UINT64_C(0xF900000000)
+	#define RGX_MMU_INIB_BRN_65273_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_65273_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig_65273.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/rogue/system/rgx_tc/tc_clocks.h b/drivers/gpu/drm/img/img-volcanic/include/rogue/system/rgx_tc/tc_clocks.h
new file mode 100644
index 000000000000..431273de0827
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/rogue/system/rgx_tc/tc_clocks.h
@@ -0,0 +1,158 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TC_CLOCKS_H)
+#define TC_CLOCKS_H
+
+/*
+ * The core clock speed is passed through a multiplier depending on the TC
+ * version.
+ *
+ * On TC_ES1: Multiplier = x3, final speed = 270MHz
+ * On TC_ES2: Multiplier = x6, final speed = 540MHz
+ * On TCF5:   Multiplier = 1x final speed = 45MHz
+ *
+ *
+ * The base (unmultiplied speed) can be adjusted using a module parameter
+ * called "sys_core_clk_speed", a number in Hz.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_core_clk_speed=60000000" /etc/init.d/rc.pvr start
+ *
+ * would result in a core speed of 60MHz xMultiplier.
+ *
+ *
+ * The memory clock is unmultiplied and can be adjusted using a module
+ * parameter called "sys_mem_clk_speed", this should be the number in Hz for
+ * the memory clock speed.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=100000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the memory clock speed set to 100MHz.
+ *
+ *
+ * Same applies to the system interface clock speed, "sys_sysif_clk_speed".
+ * Needed for TCF5 but not for TC_ES2/ES1.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_sysif_clk_speed=45000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the system clock speed set to 45MHz.
+ *
+ *
+ * All parameters can be specified at once, e.g.,
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=MEMORY_SPEED sys_core_clk_speed=CORE_SPEED sys_sysif_clk_speed=SYSIF_SPEED" /etc/init.d/rc.pvr start
+ */
+
+#define RGX_TC_SYS_CLOCK_SPEED		(25000000) /*< At the moment just used for TCF5 */
+#define RGX_TC_CLOCK_MULTIPLEX    (1)
+
+#if defined(TC_APOLLO_TCF5_22_46_54_330)
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_22_49_21_16) || \
+      defined(TC_APOLLO_TCF5_22_60_22_29) || \
+      defined(TC_APOLLO_TCF5_22_75_22_25)
+ #define RGX_TC_CORE_CLOCK_SPEED	(20000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+#elif defined(TC_APOLLO_TCF5_22_67_54_30)
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_22_89_204_18)
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_22_86_104_218)
+ #define RGX_TC_CORE_CLOCK_SPEED	(30000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_88_104_318)
+ #define RGX_TC_CORE_CLOCK_SPEED	(28000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_98_54_230)
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_102_54_38)
+ #define RGX_TC_CORE_CLOCK_SPEED	(80000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_BVNC_NOT_SUPPORTED)
+ /* TC TCF5 (22.*) fallback frequencies */
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(20000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_33_8_22_1)
+ #define RGX_TC_CORE_CLOCK_SPEED	(25000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_REFERENCE)
+ /* TC TCF5 (Reference bitfile) */
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_BONNIE)
+ /* TC Bonnie */
+ #define RGX_TC_CORE_CLOCK_SPEED	(18000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(65000000)
+#elif defined(TC_APOLLO_ES2)
+ /* TC ES2 */
+ #define RGX_TC_CORE_CLOCK_SPEED	(90000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(104000000)
+#elif defined(TC_ORION)
+ #define RGX_TC_CORE_CLOCK_SPEED	(40000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(100000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_29_19_52_202)
+ #define RGX_TC_CORE_CLOCK_SPEED	(25000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_29_18_204_508)
+ #define RGX_TC_CORE_CLOCK_SPEED	(15000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(35000000)
+#else
+ /* TC ES1 */
+ #define RGX_TC_CORE_CLOCK_SPEED	(90000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(65000000)
+#endif
+
+#endif /* if !defined(TC_CLOCKS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/services_km.h b/drivers/gpu/drm/img/img-volcanic/include/services_km.h
new file mode 100644
index 000000000000..8903b86a76a9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/services_km.h
@@ -0,0 +1,180 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services API Kernel mode Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported services API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SERVICES_KM_H
+#define SERVICES_KM_H
+
+#include "img_types.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+/*! 4k page size definition */
+#define PVRSRV_4K_PAGE_SIZE					4096U      /*!< Size of a 4K Page */
+#define PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT		12          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 16k page size definition */
+#define PVRSRV_16K_PAGE_SIZE					16384U      /*!< Size of a 16K Page */
+#define PVRSRV_16K_PAGE_SIZE_ALIGNSHIFT		14          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 64k page size definition */
+#define PVRSRV_64K_PAGE_SIZE					65536U      /*!< Size of a 64K Page */
+#define PVRSRV_64K_PAGE_SIZE_ALIGNSHIFT		16          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 256k page size definition */
+#define PVRSRV_256K_PAGE_SIZE					262144U      /*!< Size of a 256K Page */
+#define PVRSRV_256K_PAGE_SIZE_ALIGNSHIFT		18          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 1MB page size definition */
+#define PVRSRV_1M_PAGE_SIZE					1048576U      /*!< Size of a 1M Page */
+#define PVRSRV_1M_PAGE_SIZE_ALIGNSHIFT		20          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 2MB page size definition */
+#define PVRSRV_2M_PAGE_SIZE					2097152U      /*!< Size of a 2M Page */
+#define PVRSRV_2M_PAGE_SIZE_ALIGNSHIFT		21          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+
+/*!
+ * @AddToGroup SRVConnectInterfaces
+ * @{
+ */
+
+#ifndef PVRSRV_DEV_CONNECTION_TYPEDEF
+#define PVRSRV_DEV_CONNECTION_TYPEDEF
+/*!
+ * Forward declaration (look on connection.h)
+ */
+typedef struct PVRSRV_DEV_CONNECTION_TAG PVRSRV_DEV_CONNECTION;
+#endif
+
+/*!
+ * @Anchor SRV_FLAGS
+ * @Name SRV_FLAGS: Services connection flags
+ * Allows to define per-client policy for Services.
+ * @{
+ */
+
+/*
+ *   Use of the 32-bit connection flags mask
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *   31  27     20             6 4   0
+ *    |   |      |             | |   |
+ *    X---XXXXXXXX-------------XXX----
+ */
+
+#define SRV_NO_HWPERF_CLIENT_STREAM     (1UL << 4)  /*!< Don't create HWPerf for this connection */
+#define SRV_FLAGS_CLIENT_64BIT_COMPAT   (1UL << 5)  /*!< This flags gets set if the client is 64 Bit compatible. */
+#define SRV_FLAGS_CLIENT_SLR_DISABLED   (1UL << 6)  /*!< This flag is set if the client does not want Sync Lockup Recovery (SLR) enabled. */
+#define SRV_FLAGS_PDUMPCTRL             (1UL << 31) /*!< PDump Ctrl client flag */
+
+/*! @} SRV_FLAGS */
+
+/*! @} End of SRVConnectInterfaces */
+
+/*
+ * Bits 20 - 27 are used to pass information needed for validation
+ * of the GPU Virtualisation Validation mechanism. In particular:
+ *
+ * Bits:
+ * [20 - 22]: OSid of the memory region that will be used for allocations
+ * [23 - 25]: OSid that will be emitted by the Firmware for all memory accesses
+ *            regarding that memory context.
+ *      [26]: If the AXI Protection register will be set to secure for that OSid
+ *      [27]: If the Emulator Wrapper Register checking for protection violation
+ *            will be set to secure for that OSid
+ */
+
+#define VIRTVAL_FLAG_OSID_SHIFT        (20)
+#define SRV_VIRTVAL_FLAG_OSID_MASK     (7U << VIRTVAL_FLAG_OSID_SHIFT)
+
+#define VIRTVAL_FLAG_OSIDREG_SHIFT     (23)
+#define SRV_VIRTVAL_FLAG_OSIDREG_MASK  (7U << VIRTVAL_FLAG_OSIDREG_SHIFT)
+
+#define VIRTVAL_FLAG_AXIPREG_SHIFT     (26)
+#define SRV_VIRTVAL_FLAG_AXIPREG_MASK  (1U << VIRTVAL_FLAG_AXIPREG_SHIFT)
+
+#define VIRTVAL_FLAG_AXIPTD_SHIFT      (27)
+#define SRV_VIRTVAL_FLAG_AXIPTD_MASK   (1U << VIRTVAL_FLAG_AXIPTD_SHIFT)
+
+
+/* Size of pointer on a 64 bit machine */
+#define	POINTER_SIZE_64BIT	(8U)
+
+
+/*
+    Pdump flags which are accessible to Services clients
+*/
+#define PDUMP_NONE          0x00000000U /*<! No flags */
+
+#define PDUMP_BLKDATA       0x10000000U /*<! This flag indicates block-mode PDump data to be recorded in
+                                                          Block script stream in addition to Main script stream,
+                                                          if capture mode is set to BLOCKED */
+
+#define PDUMP_CONT          0x40000000U /*<! Output this entry always regardless of framed capture range,
+                                                          used by client applications being dumped. */
+#define PDUMP_PERSIST       0x80000000U /*<! Output this entry always regardless of app and range,
+                                                          used by persistent resources created after
+                                                          driver initialisation that must appear in
+                                                          all PDump captures in that session. */
+
+/* Valid range of values for pdump block length in Block mode of PDump */
+#define PDUMP_BLOCKLEN_MIN          10
+#define PDUMP_BLOCKLEN_MAX          1000
+
+#define PDUMP_FRAME_MIN             0
+#define PDUMP_FRAME_MAX             (IMG_UINT32_MAX - 1)
+#define PDUMP_FRAME_UNSET           IMG_UINT32_MAX
+
+/* Status of the device. */
+typedef IMG_UINT32 PVRSRV_DEVICE_STATUS;
+#define PVRSRV_DEVICE_STATUS_UNKNOWN        0U /* status of the device is unknown */
+#define PVRSRV_DEVICE_STATUS_OK             1U /* the device is operational */
+#define PVRSRV_DEVICE_STATUS_NOT_RESPONDING 2U /* the device is not responding */
+#define PVRSRV_DEVICE_STATUS_DEVICE_ERROR   3U /* the device is not operational */
+
+#endif /* SERVICES_KM_H */
+/**************************************************************************//**
+End of file (services_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/servicesext.h b/drivers/gpu/drm/img/img-volcanic/include/servicesext.h
new file mode 100644
index 000000000000..f2887d48aed7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/servicesext.h
@@ -0,0 +1,134 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services definitions required by external drivers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides services data structures, defines and prototypes
+                required by external drivers
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SERVICESEXT_H)
+#define SERVICESEXT_H
+
+/* include/ */
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_3dtypes.h"
+#include "pvrsrv_device_types.h"
+
+/*
+ * Lock buffer read/write flags
+ */
+#define PVRSRV_LOCKFLG_READONLY		(1)		/*!< The locking process will only read the locked surface */
+
+/*!
+ *****************************************************************************
+ *	Services State
+ *****************************************************************************/
+typedef enum _PVRSRV_SERVICES_STATE_
+{
+	PVRSRV_SERVICES_STATE_UNDEFINED = 0,
+	PVRSRV_SERVICES_STATE_OK,
+	PVRSRV_SERVICES_STATE_BAD,
+} PVRSRV_SERVICES_STATE;
+
+
+/*!
+ *****************************************************************************
+ *	States for power management
+ *****************************************************************************/
+
+/*!
+  System Power State Enum
+ */
+
+#define _PVRSRV_SYS_POWER_STATES \
+	X(Unspecified, -1) /*!< Unspecified : Uninitialised */ \
+	X(OFF,          0) /*!< Off */ \
+	X(ON,           1) /*!< On */
+
+typedef enum _PVRSRV_SYS_POWER_STATE_
+{
+#define X(name, value) PVRSRV_SYS_POWER_STATE_##name = value,
+	_PVRSRV_SYS_POWER_STATES
+#undef X
+
+	PVRSRV_SYS_POWER_STATE_FORCE_I32 = 0x7fffffff	/*!< Force enum to be at least 32-bits wide */
+
+} PVRSRV_SYS_POWER_STATE, *PPVRSRV_SYS_POWER_STATE; /*!< Typedef for ptr to PVRSRV_SYS_POWER_STATE */
+
+/*!
+  Device Power State Enum
+ */
+typedef IMG_INT32 PVRSRV_DEV_POWER_STATE;
+typedef IMG_INT32 *PPVRSRV_DEV_POWER_STATE;	/*!< Typedef for ptr to PVRSRV_DEV_POWER_STATE */ /* PRQA S 3205 */
+#define PVRSRV_DEV_POWER_STATE_DEFAULT	-1	/*!< Default state for the device */
+#define PVRSRV_DEV_POWER_STATE_OFF		 0	/*!< Unpowered */
+#define PVRSRV_DEV_POWER_STATE_ON		 1	/*!< Running */
+
+/*!
+  Power Flags Enum
+ */
+typedef IMG_UINT32 PVRSRV_POWER_FLAGS;
+#define PVRSRV_POWER_FLAGS_NONE             0U      /*!< No flags */
+#define PVRSRV_POWER_FLAGS_FORCED           (1U << 0) /*!< Power the transition should not fail */
+#define PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ (1U << 1) /*!< Indicates an OS Power Management
+                                                           transition (S3/S4) has been requested.
+                                                           Allows system modules to save VRAM */
+#define PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ  (1U << 2) /*!< Indicates an OS Power Management
+                                                           transition has been requested.
+                                                           Allows system modules to load VRAM */
+
+/*!
+ *****************************************************************************
+ * This structure is used for OS independent registry (profile) access
+ *****************************************************************************/
+
+typedef struct PVRSRV_REGISTRY_INFO_TAG
+{
+	IMG_UINT32	ui32DevCookie;
+	IMG_PCHAR	pszKey;
+	IMG_PCHAR	pszValue;
+	IMG_PCHAR	pszBuf;
+	IMG_UINT32	ui32BufSize;
+} PVRSRV_REGISTRY_INFO, *PPVRSRV_REGISTRY_INFO;
+
+#endif /* SERVICESEXT_H */
+/******************************************************************************
+ End of file (servicesext.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/sync_checkpoint_external.h b/drivers/gpu/drm/img/img-volcanic/include/sync_checkpoint_external.h
new file mode 100644
index 000000000000..bfab937a1809
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/sync_checkpoint_external.h
@@ -0,0 +1,88 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services external synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines synchronisation checkpoint structures that are visible
+				internally and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_EXTERNAL_H
+#define SYNC_CHECKPOINT_EXTERNAL_H
+
+#include "img_types.h"
+
+#ifndef CHECKPOINT_TYPES
+#define CHECKPOINT_TYPES
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG *PSYNC_CHECKPOINT_CONTEXT;
+
+typedef struct SYNC_CHECKPOINT_TAG *PSYNC_CHECKPOINT;
+#endif
+
+/* PVRSRV_SYNC_CHECKPOINT states.
+ * The OS native sync implementation should call pfnIsSignalled() to determine if a
+ * PVRSRV_SYNC_CHECKPOINT has signalled (which will return an IMG_BOOL), but can set the
+ * state for a PVRSRV_SYNC_CHECKPOINT (which is currently in the NOT_SIGNALLED state)
+ * where that PVRSRV_SYNC_CHECKPOINT is representing a foreign sync.
+ */
+typedef IMG_UINT32 PVRSRV_SYNC_CHECKPOINT_STATE;
+
+#define PVRSRV_SYNC_CHECKPOINT_UNDEF         0x000U
+#define PVRSRV_SYNC_CHECKPOINT_ACTIVE        0xac1U  /*!< checkpoint has not signalled */
+#define PVRSRV_SYNC_CHECKPOINT_SIGNALLED     0x519U  /*!< checkpoint has signalled */
+#define PVRSRV_SYNC_CHECKPOINT_ERRORED       0xeffU   /*!< checkpoint has been errored */
+
+
+#define PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(fwaddr)	(((fwaddr) & 0x1U) != 0U)
+#define PVRSRV_UFO_IS_SYNC_CHECKPOINT(ufoptr)			(PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR((ufoptr)->puiAddrUFO.ui32Addr))
+
+/* Maximum number of sync checkpoints the firmware supports in one fence */
+#define MAX_SYNC_CHECKPOINTS_PER_FENCE 32U
+
+/*!
+ * Define to be used with SyncCheckpointAlloc() to indicate a checkpoint which
+ * represents a foreign sync point or collection of foreign sync points.
+ */
+#define SYNC_CHECKPOINT_FOREIGN_CHECKPOINT ((PVRSRV_TIMELINE) - 2U)
+/*!
+ * Define to be used with SyncCheckpointAlloc() to indicate a checkpoint which
+ * represents a mirrored cross-device foreign sync point
+ */
+#define SYNC_CHECKPOINT_MIRRORED_CHECKPOINT ((PVRSRV_TIMELINE) - 3U)
+
+#endif /* SYNC_CHECKPOINT_EXTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/sync_prim_internal.h b/drivers/gpu/drm/img/img-volcanic/include/sync_prim_internal.h
new file mode 100644
index 000000000000..c4dec49e9ca4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/sync_prim_internal.h
@@ -0,0 +1,83 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation typedef header
+@Description    Defines synchronisation types that are used internally
+                only
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_INTERNAL_H
+#define SYNC_INTERNAL_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include <powervr/mem_types.h>
+
+/* These are included here as the typedefs are required
+ * internally.
+ */
+
+typedef struct SYNC_PRIM_CONTEXT_TAG *PSYNC_PRIM_CONTEXT;
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_TAG
+{
+	volatile uint32_t __iomem *pui32LinAddr;	/*!< User pointer to the primitive */
+} PVRSRV_CLIENT_SYNC_PRIM;
+
+/*!
+ * Bundled information for a sync prim operation
+ *
+ *   Structure: #PVRSRV_CLIENT_SYNC_PRIM_OP
+ *   Typedef: ::PVRSRV_CLIENT_SYNC_PRIM_OP
+ */
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_OP_TAG
+{
+	#define PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK	(1U << 0)
+	#define PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE	(1U << 1)
+	uint32_t                    ui32Flags;       /*!< Operation flags: PVRSRV_CLIENT_SYNC_PRIM_OP_XXX */
+	PVRSRV_CLIENT_SYNC_PRIM    *psSync;          /*!< Pointer to the client sync primitive */
+	uint32_t                    ui32FenceValue;  /*!< The Fence value (only used if PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK is set) */
+	uint32_t                    ui32UpdateValue; /*!< The Update value (only used if PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE is set) */
+} PVRSRV_CLIENT_SYNC_PRIM_OP;
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* SYNC_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/apollo_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/apollo_regs.h
new file mode 100644
index 000000000000..4081e2123ac7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/apollo_regs.h
@@ -0,0 +1,108 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(APOLLO_REGS_H)
+#define APOLLO_REGS_H
+
+#include "tc_clocks.h"
+
+/* TC TCF5 */
+#define TC5_SYS_APOLLO_REG_PCI_BASENUM (1)
+#define TC5_SYS_APOLLO_REG_PDP2_OFFSET (0x800000)
+#define TC5_SYS_APOLLO_REG_PDP2_SIZE   (0x7C4)
+
+#define TC5_SYS_APOLLO_REG_PDP2_FBDC_OFFSET (0xA00000)
+#define TC5_SYS_APOLLO_REG_PDP2_FBDC_SIZE   (0x14)
+
+#define TC5_SYS_APOLLO_REG_HDMI_OFFSET (0xC00000)
+#define TC5_SYS_APOLLO_REG_HDMI_SIZE   (0x1C)
+
+/* TC ES2 */
+#define TCF_TEMP_SENSOR_SPI_OFFSET	0xe
+#define TCF_TEMP_SENSOR_TO_C(raw)	(((raw) * 248 / 4096) - 54)
+
+/* Number of bytes that are broken */
+#define SYS_DEV_MEM_BROKEN_BYTES	(1024 * 1024)
+#define SYS_DEV_MEM_REGION_SIZE		(0x40000000 - SYS_DEV_MEM_BROKEN_BYTES)
+
+/* Apollo reg on base register 0 */
+#define SYS_APOLLO_REG_PCI_BASENUM	(0)
+#define SYS_APOLLO_REG_REGION_SIZE	(0x00010000)
+
+#define SYS_APOLLO_REG_SYS_OFFSET	(0x0000)
+#define SYS_APOLLO_REG_SYS_SIZE		(0x0400)
+
+#define SYS_APOLLO_REG_PLL_OFFSET	(0x1000)
+#define SYS_APOLLO_REG_PLL_SIZE		(0x0400)
+
+#define SYS_APOLLO_REG_HOST_OFFSET	(0x4050)
+#define SYS_APOLLO_REG_HOST_SIZE	(0x0014)
+
+#define SYS_APOLLO_REG_PDP1_OFFSET	(0xC000)
+#define SYS_APOLLO_REG_PDP1_SIZE	(0x2000)
+
+/* Offsets for flashing Apollo PROMs from base 0 */
+#define APOLLO_FLASH_STAT_OFFSET	(0x4058)
+#define APOLLO_FLASH_DATA_WRITE_OFFSET	(0x4050)
+#define APOLLO_FLASH_RESET_OFFSET	(0x4060)
+
+#define APOLLO_FLASH_FIFO_STATUS_MASK	 (0xF)
+#define APOLLO_FLASH_FIFO_STATUS_SHIFT	 (0)
+#define APOLLO_FLASH_PROGRAM_STATUS_MASK (0xF)
+#define APOLLO_FLASH_PROGRAM_STATUS_SHIFT (16)
+
+#define APOLLO_FLASH_PROG_COMPLETE_BIT	(0x1)
+#define APOLLO_FLASH_PROG_PROGRESS_BIT	(0x2)
+#define APOLLO_FLASH_PROG_FAILED_BIT	(0x4)
+#define APOLLO_FLASH_INV_FILETYPE_BIT	(0x8)
+
+#define APOLLO_FLASH_FIFO_SIZE		(8)
+
+/* RGX reg on base register 1 */
+#define SYS_RGX_REG_PCI_BASENUM		(1)
+#define SYS_RGX_REG_REGION_SIZE		(0x7FFFF)
+
+/* Device memory (including HP mapping) on base register 2 */
+#define SYS_DEV_MEM_PCI_BASENUM		(2)
+
+#endif /* APOLLO_REGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/bonnie_tcf.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/bonnie_tcf.h
new file mode 100644
index 000000000000..fc87ec790df9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/bonnie_tcf.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* bonnie_tcf.h - Bonnie TCF register definitions */
+
+/* tab size 4 */
+
+#ifndef BONNIE_TCF_DEFS_H
+#define BONNIE_TCF_DEFS_H
+
+#define BONNIE_TCF_OFFSET_BONNIETC_REGBANK							0x00000000
+#define BONNIE_TCF_OFFSET_TC_IFACE_COUNTERS							0x00004000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_IMGV4_RTM_TOP				0x00008000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_TCF_SCRATCH_PAD_SECN		0x0000C000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_TCF_SCRATCH_PAD_DBG		0x00010000
+#define BONNIE_TCF_OFFSET_MULTI_CLK_ALIGN							0x00014000
+#define BONNIE_TCF_OFFSET_ALIGN_DATA_TX								0x00018000
+#define BONNIE_TCF_OFFSET_SAI_RX_1									0x0001C000
+#define BONNIE_TCF_OFFSET_SAI_RX_SDR								0x00040000
+#define BONNIE_TCF_OFFSET_SAI_TX_1									0x00044000
+#define BONNIE_TCF_OFFSET_SAI_TX_SDR								0x00068000
+
+#define BONNIE_TCF_OFFSET_SAI_RX_DELTA								0x00004000
+#define BONNIE_TCF_OFFSET_SAI_TX_DELTA								0x00004000
+
+#define BONNIE_TCF_OFFSET_SAI_CLK_TAPS								0x0000000C
+#define BONNIE_TCF_OFFSET_SAI_EYES									0x00000010
+#define BONNIE_TCF_OFFSET_SAI_TRAIN_ACK								0x00000018
+
+
+#endif /* BONNIE_TCF_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_defs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_defs.h
new file mode 100644
index 000000000000..10ca9b6d4382
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_defs.h
@@ -0,0 +1,338 @@
+/****************************************************************************
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin Memory Map - View from PCIe
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+****************************************************************************/
+
+#ifndef _ODIN_DEFS_H_
+#define _ODIN_DEFS_H_
+
+/* These defines have not been autogenerated */
+
+#define PCI_VENDOR_ID_ODIN                  (0x1AEE)
+#define DEVICE_ID_ODIN                      (0x1010)
+#define DEVICE_ID_VALI                      (0x2010)
+#define DEVICE_ID_TBA                       (0x1CF2)
+
+/* PCI BAR 0 contains the PDP regs and the Odin system regs */
+#define ODN_SYS_BAR                         0
+#define ODN_SYS_REGION_SIZE                 0x000800000 /* 8MB */
+
+#define ODN_SYS_REGS_OFFSET                 0
+#define ODN_SYS_REGS_SIZE                   0x000400000 /* 4MB */
+
+#define ODN_PDP_REGS_OFFSET                 0x000440000
+#define ODN_PDP_REGS_SIZE                   0x000040000 /* 256k */
+
+#define ODN_PDP2_REGS_OFFSET                0x000480000
+#define ODN_PDP2_REGS_SIZE                  0x000040000 /* 256k */
+
+#define ODN_PDP2_PFIM_OFFSET                0x000500000
+#define ODN_PDP2_PFIM_SIZE                  0x000040000 /* 256k */
+
+#define ODIN_DMA_REGS_OFFSET                0x0004C0000
+#define ODIN_DMA_REGS_SIZE                  0x000040000 /* 256k */
+
+#define ODIN_DMA_CHAN_REGS_SIZE             0x000001000 /*   4k */
+
+/* PCI BAR 2 contains the Device Under Test SOCIF 64MB region */
+#define ODN_DUT_SOCIF_BAR                   2
+#define ODN_DUT_SOCIF_OFFSET                0x000000000
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* Reserve only the strictly required register range for each OSID */
+#define ODN_DUT_SOCIF_SIZE                  0x000010000 /* 64k */
+#else
+/* Grant the driver access to the entire register IO range */
+#define ODN_DUT_SOCIF_SIZE                  0x004000000 /* 64MB */
+#endif
+
+/* PCI BAR 4 contains the on-board 1GB DDR memory */
+#define ODN_DDR_BAR                         4
+#define ODN_DDR_MEM_OFFSET                  0x000000000
+#define ODN_DDR_MEM_SIZE                    0x040000000 /* 1GB */
+
+/* Odin system register banks */
+#define ODN_REG_BANK_CORE                   0x00000
+#define ODN_REG_BANK_TCF_SPI_MASTER         0x02000
+#define ODN_REG_BANK_ODN_CLK_BLK            0x0A000
+#define ODN_REG_BANK_ODN_MCU_COMMUNICATOR   0x0C000
+#define ODN_REG_BANK_DB_TYPE_ID             0x0C200
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_TCFVUOCTA   0x000000C6U
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_MASK   0x000000C0U
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_SHIFT  0x6
+#define ODN_REG_BANK_ODN_I2C                0x0E000
+#define ODN_REG_BANK_MULTI_CLK_ALIGN        0x20000
+#define ODN_REG_BANK_ALIGN_DATA_TX          0x22000
+#define ODN_REG_BANK_SAI_RX_DDR_0           0x24000
+#define ODN_REG_BANK_SAI_RX_DDR(n)          (ODN_REG_BANK_SAI_RX_DDR_0 + (0x02000*n))
+#define ODN_REG_BANK_SAI_TX_DDR_0           0x3A000
+#define ODN_REG_BANK_SAI_TX_DDR(n)          (ODN_REG_BANK_SAI_TX_DDR_0 + (0x02000*n))
+#define ODN_REG_BANK_SAI_TX_SDR             0x4E000
+
+/* Odin SPI regs */
+#define ODN_SPI_MST_ADDR_RDNWR              0x0000
+#define ODN_SPI_MST_WDATA                   0x0004
+#define ODN_SPI_MST_RDATA                   0x0008
+#define ODN_SPI_MST_STATUS                  0x000C
+#define ODN_SPI_MST_GO                      0x0010
+
+
+/*
+   Odin CLK regs - the odn_clk_blk module defs are not auto generated
+ */
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG1           0x620
+#define ODN_PDP_PCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define ODN_PDP_PCLK_ODIV1_LO_TIME_SHIFT         0
+#define ODN_PDP_PCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define ODN_PDP_PCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG2           0x624
+#define ODN_PDP_PCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define ODN_PDP_PCLK_ODIV2_NOCOUNT_SHIFT         6
+#define ODN_PDP_PCLK_ODIV2_EDGE_MASK             0x00000080U
+#define ODN_PDP_PCLK_ODIV2_EDGE_SHIFT            7
+
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG3           0x61C
+
+#define ODN_PDP_M_CLK_OUT_DIVIDER_REG1           0x628
+#define ODN_PDP_MCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define ODN_PDP_MCLK_ODIV1_LO_TIME_SHIFT         0
+#define ODN_PDP_MCLK_ODIV1_HI_TIME_MASK	         0x00000FC0U
+#define ODN_PDP_MCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define ODN_PDP_M_CLK_OUT_DIVIDER_REG2           0x62C
+#define ODN_PDP_MCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define ODN_PDP_MCLK_ODIV2_NOCOUNT_SHIFT         6
+#define ODN_PDP_MCLK_ODIV2_EDGE_MASK             0x00000080U
+#define ODN_PDP_MCLK_ODIV2_EDGE_SHIFT            7
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG1            0x650
+#define ODN_PDP_PCLK_MUL1_LO_TIME_MASK           0x0000003FU
+#define ODN_PDP_PCLK_MUL1_LO_TIME_SHIFT          0
+#define ODN_PDP_PCLK_MUL1_HI_TIME_MASK           0x00000FC0U
+#define ODN_PDP_PCLK_MUL1_HI_TIME_SHIFT          6
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG2            0x654
+#define ODN_PDP_PCLK_MUL2_NOCOUNT_MASK           0x00000040U
+#define ODN_PDP_PCLK_MUL2_NOCOUNT_SHIFT          6
+#define ODN_PDP_PCLK_MUL2_EDGE_MASK              0x00000080U
+#define ODN_PDP_PCLK_MUL2_EDGE_SHIFT             7
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG3            0x64C
+
+#define ODN_PDP_P_CLK_IN_DIVIDER_REG             0x658
+#define ODN_PDP_PCLK_IDIV_LO_TIME_MASK           0x0000003FU
+#define ODN_PDP_PCLK_IDIV_LO_TIME_SHIFT          0
+#define ODN_PDP_PCLK_IDIV_HI_TIME_MASK           0x00000FC0U
+#define ODN_PDP_PCLK_IDIV_HI_TIME_SHIFT          6
+#define ODN_PDP_PCLK_IDIV_NOCOUNT_MASK           0x00001000U
+#define ODN_PDP_PCLK_IDIV_NOCOUNT_SHIFT          12
+#define ODN_PDP_PCLK_IDIV_EDGE_MASK              0x00002000U
+#define ODN_PDP_PCLK_IDIV_EDGE_SHIFT             13
+
+/*
+ * DUT core clock input divider, multiplier and out divider.
+ */
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1                (0x0028)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  (0)
+
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2                (0x002C)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_MASK      (0x00000080U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_SHIFT     (7)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_MASK   (0x00000040U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  (6)
+
+#define ODN_DUT_CORE_CLK_MULTIPLIER1                 (0x0050)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME_MASK    (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME_SHIFT   (6)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME_MASK    (0x0000003FU)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME_SHIFT   (0)
+
+#define ODN_DUT_CORE_CLK_MULTIPLIER2                 (0x0054)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_MASK       (0x00007000U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_SHIFT      (12)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_EN_MASK    (0x00000800U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_EN_SHIFT   (11)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE_MASK       (0x00000080U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE_SHIFT      (7)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT_MASK    (0x00000040U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT_SHIFT   (6)
+
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1                 (0x0058)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE_MASK       (0x00002000U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE_SHIFT      (13)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT_MASK    (0x00001000U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT_SHIFT   (12)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME_MASK    (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME_SHIFT   (6)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME_MASK    (0x0000003FU)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME_SHIFT   (0)
+
+/*
+ * DUT interface clock input divider, multiplier and out divider.
+ */
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1               (0x0220)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME_MASK  (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT (6)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME_MASK  (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT (0)
+
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2               (0x0224)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE_MASK     (0x00000080U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE_SHIFT    (7)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT_MASK  (0x00000040U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT (6)
+
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1                (0x0250)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME_SHIFT  (0)
+
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2                (0x0254)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_MASK      (0x00007000U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_SHIFT     (12)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_EN_MASK   (0x00000800U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_EN_SHIFT  (11)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE_MASK      (0x00000080U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE_SHIFT     (7)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT_MASK   (0x00000040U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT_SHIFT  (6)
+
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1                (0x0258)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE_MASK      (0x00002000U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE_SHIFT     (13)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT_MASK   (0x00001000U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT_SHIFT  (12)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME_SHIFT  (0)
+
+
+/*
+ * Min max values from Xilinx Virtex7 data sheet DS183, for speed grade 2
+ * All in Hz
+ */
+#define ODN_INPUT_CLOCK_SPEED                        (100000000U)
+#define ODN_INPUT_CLOCK_SPEED_MIN                    (10000000U)
+#define ODN_INPUT_CLOCK_SPEED_MAX                    (933000000U)
+#define ODN_OUTPUT_CLOCK_SPEED_MIN                   (4690000U)
+#define ODN_OUTPUT_CLOCK_SPEED_MAX                   (933000000U)
+#define ODN_VCO_MIN                                  (600000000U)
+#define ODN_VCO_MAX                                  (1440000000U)
+#define ODN_PFD_MIN                                  (10000000U)
+#define ODN_PFD_MAX                                  (500000000U)
+
+/*
+ * Max values that can be set in DRP registers
+ */
+#define ODN_OREG_VALUE_MAX                            (126.875f)
+#define ODN_MREG_VALUE_MAX                            (126.875f)
+#define ODN_DREG_VALUE_MAX                            (126U)
+
+
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE                (0x00000001U)
+#define ODN_MMCM_LOCK_STATUS_DUT_IF                  (0x00000002U)
+#define ODN_MMCM_LOCK_STATUS_PDPP                    (0x00000008U)
+
+/*
+    Odin interrupt flags
+*/
+#define ODN_INTERRUPT_ENABLE_PDP1           (1 << ODN_INTERRUPT_ENABLE_PDP1_SHIFT)
+#define ODN_INTERRUPT_ENABLE_PDP2           (1 << ODN_INTERRUPT_ENABLE_PDP2_SHIFT)
+#define ODN_INTERRUPT_ENABLE_DUT            (1 << ODN_INTERRUPT_ENABLE_DUT_SHIFT)
+#define ODN_INTERRUPT_STATUS_PDP1           (1 << ODN_INTERRUPT_STATUS_PDP1_SHIFT)
+#define ODN_INTERRUPT_STATUS_PDP2           (1 << ODN_INTERRUPT_STATUS_PDP2_SHIFT)
+#define ODN_INTERRUPT_STATUS_DUT            (1 << ODN_INTERRUPT_STATUS_DUT_SHIFT)
+#define ODN_INTERRUPT_CLEAR_PDP1            (1 << ODN_INTERRUPT_CLR_PDP1_SHIFT)
+#define ODN_INTERRUPT_CLEAR_PDP2            (1 << ODN_INTERRUPT_CLR_PDP2_SHIFT)
+#define ODN_INTERRUPT_CLEAR_DUT             (1 << ODN_INTERRUPT_CLR_DUT_SHIFT)
+
+#define ODN_INTERRUPT_ENABLE_CDMA           (1 << ODN_INTERRUPT_ENABLE_CDMA_SHIFT)
+#define ODN_INTERRUPT_STATUS_CDMA           (1 << ODN_INTERRUPT_STATUS_CDMA_SHIFT)
+#define ODN_INTERRUPT_CLEAR_CDMA            (1 << ODN_INTERRUPT_CLR_CDMA_SHIFT)
+
+#define ODN_INTERRUPT_ENABLE_CDMA2          (1 << (ODN_INTERRUPT_ENABLE_CDMA_SHIFT + 1))
+#define ODN_INTERRUPT_STATUS_CDMA2          (1 << (ODN_INTERRUPT_STATUS_CDMA_SHIFT + 1))
+#define ODN_INTERRUPT_CLEAR_CDMA2           (1 << (ODN_INTERRUPT_CLR_CDMA_SHIFT + 1))
+
+#define ODN_INTERRUPT_ENABLE_OSID(n)        (1 << (ODN_INTERRUPT_ENABLE_OS_IRQ_SHIFT + (n)))
+#define ODN_INTERRUPT_STATUS_OSID(n)        (1 << (ODN_INTERRUPT_STATUS_OS_IRQ_SHIFT + (n)))
+#define ODN_INTERRUPT_CLEAR_OSID(n)         (1 << (ODN_INTERRUPT_CLR_OS_IRQ_SHIFT + (n)))
+
+/*
+   Other defines
+*/
+#define ODN_STREAM_OFF                      0
+#define ODN_STREAM_ON                       1
+#define ODN_SYNC_GEN_DISABLE                0
+#define ODN_SYNC_GEN_ENABLE                 1
+#define ODN_INTERLACE_DISABLE               0
+#define ODN_INTERLACE_ENABLE                1
+#define ODN_PIXEL_CLOCK_INVERTED            1
+#define ODN_HSYNC_POLARITY_ACTIVE_HIGH      1
+
+#define ODN_PDP_INTCLR_ALL                  0x000FFFFFU
+#define	ODN_PDP_INTSTAT_ALL_OURUN_MASK      0x000FFFF0U
+
+/*
+   DMA defs
+*/
+#define ODN_CDMA_ADDR_WIDTH                35
+#define ODN_DMA_HW_DESC_HEAP_SIZE          0x100000
+#define ODN_DMA_CHAN_RX                    0
+#define ODN_DMA_CHAN_TX                    1
+
+#define ODIN_DMA_TX_CHAN_NAME              "tx"
+#define ODIN_DMA_RX_CHAN_NAME              "rx"
+
+/*
+   FBC defs
+*/
+#define ODIN_PFIM_RELNUM                   (005U)
+
+#endif /* _ODIN_DEFS_H_ */
+
+/*****************************************************************************
+ End of file (odn_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_pdp_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_pdp_regs.h
new file mode 100644
index 000000000000..da47a253db31
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_pdp_regs.h
@@ -0,0 +1,8540 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* tab size 4 */
+
+#ifndef ODN_PDP_REGS_H
+#define ODN_PDP_REGS_H
+
+/* Odin-PDP hardware register definitions */
+
+
+#define ODN_PDP_GRPH1SURF_OFFSET					(0x0000)
+
+/* PDP, GRPH1SURF, GRPH1PIXFMT
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USEGAMMA
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USECSC
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USELUT
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2SURF_OFFSET					(0x0004)
+
+/* PDP, GRPH2SURF, GRPH2PIXFMT
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USEGAMMA
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USECSC
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USELUT
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3SURF_OFFSET					(0x0008)
+
+/* PDP, GRPH3SURF, GRPH3PIXFMT
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USEGAMMA
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USECSC
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USELUT
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4SURF_OFFSET					(0x000C)
+
+/* PDP, GRPH4SURF, GRPH4PIXFMT
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USEGAMMA
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USECSC
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USELUT
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SURF_OFFSET						(0x0010)
+
+/* PDP, VID1SURF, VID1PIXFMT
+*/
+#define ODN_PDP_VID1SURF_VID1PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEGAMMA
+*/
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USECSC
+*/
+#define ODN_PDP_VID1SURF_VID1USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID1SURF_VID1USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USECSC_SHIFT			(25)
+#define ODN_PDP_VID1SURF_VID1USECSC_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEI2P
+*/
+#define ODN_PDP_VID1SURF_VID1USEI2P_MASK			(0x01000000)
+#define ODN_PDP_VID1SURF_VID1USEI2P_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEI2P_SHIFT			(24)
+#define ODN_PDP_VID1SURF_VID1USEI2P_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USEI2P_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1COSITED
+*/
+#define ODN_PDP_VID1SURF_VID1COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID1SURF_VID1COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1COSITED_SHIFT			(23)
+#define ODN_PDP_VID1SURF_VID1COSITED_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEHQCD
+*/
+#define ODN_PDP_VID1SURF_VID1USEHQCD_MASK			(0x00400000)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_SHIFT			(22)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEINSTREAM
+*/
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_MASK		(0x00200000)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_LSBMASK	(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_SHIFT		(21)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_LENGTH		(1)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2SURF_OFFSET						(0x0014)
+
+/* PDP, VID2SURF, VID2PIXFMT
+*/
+#define ODN_PDP_VID2SURF_VID2PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2COSITED
+*/
+#define ODN_PDP_VID2SURF_VID2COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID2SURF_VID2COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SURF_VID2COSITED_SHIFT			(23)
+#define ODN_PDP_VID2SURF_VID2COSITED_LENGTH			(1)
+#define ODN_PDP_VID2SURF_VID2COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USEGAMMA
+*/
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USECSC
+*/
+#define ODN_PDP_VID2SURF_VID2USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID2SURF_VID2USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2SURF_VID2USECSC_SHIFT			(25)
+#define ODN_PDP_VID2SURF_VID2USECSC_LENGTH			(1)
+#define ODN_PDP_VID2SURF_VID2USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SURF_OFFSET						(0x0018)
+
+/* PDP, VID3SURF, VID3PIXFMT
+*/
+#define ODN_PDP_VID3SURF_VID3PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3COSITED
+*/
+#define ODN_PDP_VID3SURF_VID3COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID3SURF_VID3COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SURF_VID3COSITED_SHIFT			(23)
+#define ODN_PDP_VID3SURF_VID3COSITED_LENGTH			(1)
+#define ODN_PDP_VID3SURF_VID3COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USEGAMMA
+*/
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USECSC
+*/
+#define ODN_PDP_VID3SURF_VID3USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID3SURF_VID3USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3SURF_VID3USECSC_SHIFT			(25)
+#define ODN_PDP_VID3SURF_VID3USECSC_LENGTH			(1)
+#define ODN_PDP_VID3SURF_VID3USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SURF_OFFSET						(0x001C)
+
+/* PDP, VID4SURF, VID4PIXFMT
+*/
+#define ODN_PDP_VID4SURF_VID4PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4COSITED
+*/
+#define ODN_PDP_VID4SURF_VID4COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID4SURF_VID4COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4SURF_VID4COSITED_SHIFT			(23)
+#define ODN_PDP_VID4SURF_VID4COSITED_LENGTH			(1)
+#define ODN_PDP_VID4SURF_VID4COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USEGAMMA
+*/
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USECSC
+*/
+#define ODN_PDP_VID4SURF_VID4USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID4SURF_VID4USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4SURF_VID4USECSC_SHIFT			(25)
+#define ODN_PDP_VID4SURF_VID4USECSC_LENGTH			(1)
+#define ODN_PDP_VID4SURF_VID4USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1CTRL_OFFSET					(0x0020)
+
+/* PDP, GRPH1CTRL, GRPH1STREN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_SHIFT			(31)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_LENGTH			(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYEN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYSRC
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLEND
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLENDPOS
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1DITHEREN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CTRL_OFFSET					(0x0024)
+
+/* PDP, GRPH2CTRL, GRPH2STREN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_SHIFT			(31)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_LENGTH			(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYEN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYSRC
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLEND
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLENDPOS
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2DITHEREN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CTRL_OFFSET					(0x0028)
+
+/* PDP, GRPH3CTRL, GRPH3STREN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_SHIFT			(31)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_LENGTH			(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYEN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYSRC
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLEND
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLENDPOS
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3DITHEREN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CTRL_OFFSET					(0x002C)
+
+/* PDP, GRPH4CTRL, GRPH4STREN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_SHIFT			(31)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_LENGTH			(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYEN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYSRC
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLEND
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLENDPOS
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4DITHEREN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1CTRL_OFFSET						(0x0030)
+
+/* PDP, VID1CTRL, VID1STREN
+*/
+#define ODN_PDP_VID1CTRL_VID1STREN_MASK				(0x80000000)
+#define ODN_PDP_VID1CTRL_VID1STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1STREN_SHIFT			(31)
+#define ODN_PDP_VID1CTRL_VID1STREN_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYEN
+*/
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYSRC
+*/
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLEND
+*/
+#define ODN_PDP_VID1CTRL_VID1BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID1CTRL_VID1BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID1CTRL_VID1BLEND_SHIFT			(27)
+#define ODN_PDP_VID1CTRL_VID1BLEND_LENGTH			(2)
+#define ODN_PDP_VID1CTRL_VID1BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLENDPOS
+*/
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1DITHEREN
+*/
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CTRL_OFFSET						(0x0034)
+
+/* PDP, VID2CTRL, VID2STREN
+*/
+#define ODN_PDP_VID2CTRL_VID2STREN_MASK				(0x80000000)
+#define ODN_PDP_VID2CTRL_VID2STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2STREN_SHIFT			(31)
+#define ODN_PDP_VID2CTRL_VID2STREN_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYEN
+*/
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYSRC
+*/
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLEND
+*/
+#define ODN_PDP_VID2CTRL_VID2BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID2CTRL_VID2BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID2CTRL_VID2BLEND_SHIFT			(27)
+#define ODN_PDP_VID2CTRL_VID2BLEND_LENGTH			(2)
+#define ODN_PDP_VID2CTRL_VID2BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLENDPOS
+*/
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2DITHEREN
+*/
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CTRL_OFFSET						(0x0038)
+
+/* PDP, VID3CTRL, VID3STREN
+*/
+#define ODN_PDP_VID3CTRL_VID3STREN_MASK				(0x80000000)
+#define ODN_PDP_VID3CTRL_VID3STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3STREN_SHIFT			(31)
+#define ODN_PDP_VID3CTRL_VID3STREN_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYEN
+*/
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYSRC
+*/
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLEND
+*/
+#define ODN_PDP_VID3CTRL_VID3BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID3CTRL_VID3BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID3CTRL_VID3BLEND_SHIFT			(27)
+#define ODN_PDP_VID3CTRL_VID3BLEND_LENGTH			(2)
+#define ODN_PDP_VID3CTRL_VID3BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLENDPOS
+*/
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3DITHEREN
+*/
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CTRL_OFFSET						(0x003C)
+
+/* PDP, VID4CTRL, VID4STREN
+*/
+#define ODN_PDP_VID4CTRL_VID4STREN_MASK				(0x80000000)
+#define ODN_PDP_VID4CTRL_VID4STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4STREN_SHIFT			(31)
+#define ODN_PDP_VID4CTRL_VID4STREN_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYEN
+*/
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYSRC
+*/
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLEND
+*/
+#define ODN_PDP_VID4CTRL_VID4BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID4CTRL_VID4BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID4CTRL_VID4BLEND_SHIFT			(27)
+#define ODN_PDP_VID4CTRL_VID4BLEND_LENGTH			(2)
+#define ODN_PDP_VID4CTRL_VID4BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLENDPOS
+*/
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4DITHEREN
+*/
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1UCTRL_OFFSET					(0x0050)
+
+/* PDP, VID1UCTRL, VID1UVHALFSTR
+*/
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2UCTRL_OFFSET					(0x0054)
+
+/* PDP, VID2UCTRL, VID2UVHALFSTR
+*/
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3UCTRL_OFFSET					(0x0058)
+
+/* PDP, VID3UCTRL, VID3UVHALFSTR
+*/
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4UCTRL_OFFSET					(0x005C)
+
+/* PDP, VID4UCTRL, VID4UVHALFSTR
+*/
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1STRIDE_OFFSET					(0x0060)
+
+/* PDP, GRPH1STRIDE, GRPH1STRIDE
+*/
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2STRIDE_OFFSET					(0x0064)
+
+/* PDP, GRPH2STRIDE, GRPH2STRIDE
+*/
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3STRIDE_OFFSET					(0x0068)
+
+/* PDP, GRPH3STRIDE, GRPH3STRIDE
+*/
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4STRIDE_OFFSET					(0x006C)
+
+/* PDP, GRPH4STRIDE, GRPH4STRIDE
+*/
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1STRIDE_OFFSET					(0x0070)
+
+/* PDP, VID1STRIDE, VID1STRIDE
+*/
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_SHIFT			(22)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_LENGTH		(10)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2STRIDE_OFFSET					(0x0074)
+
+/* PDP, VID2STRIDE, VID2STRIDE
+*/
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_SHIFT			(22)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_LENGTH		(10)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3STRIDE_OFFSET					(0x0078)
+
+/* PDP, VID3STRIDE, VID3STRIDE
+*/
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_SHIFT			(22)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_LENGTH		(10)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4STRIDE_OFFSET					(0x007C)
+
+/* PDP, VID4STRIDE, VID4STRIDE
+*/
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_SHIFT			(22)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_LENGTH		(10)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1SIZE_OFFSET					(0x0080)
+
+/* PDP, GRPH1SIZE, GRPH1WIDTH
+*/
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SIZE, GRPH1HEIGHT
+*/
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2SIZE_OFFSET					(0x0084)
+
+/* PDP, GRPH2SIZE, GRPH2WIDTH
+*/
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SIZE, GRPH2HEIGHT
+*/
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3SIZE_OFFSET					(0x0088)
+
+/* PDP, GRPH3SIZE, GRPH3WIDTH
+*/
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SIZE, GRPH3HEIGHT
+*/
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4SIZE_OFFSET					(0x008C)
+
+/* PDP, GRPH4SIZE, GRPH4WIDTH
+*/
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SIZE, GRPH4HEIGHT
+*/
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SIZE_OFFSET						(0x0090)
+
+/* PDP, VID1SIZE, VID1WIDTH
+*/
+#define ODN_PDP_VID1SIZE_VID1WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_SHIFT			(16)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_LENGTH			(12)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1SIZE, VID1HEIGHT
+*/
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SIZE_OFFSET						(0x0094)
+
+/* PDP, VID2SIZE, VID2WIDTH
+*/
+#define ODN_PDP_VID2SIZE_VID2WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_SHIFT			(16)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_LENGTH			(12)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2SIZE, VID2HEIGHT
+*/
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SIZE_OFFSET						(0x0098)
+
+/* PDP, VID3SIZE, VID3WIDTH
+*/
+#define ODN_PDP_VID3SIZE_VID3WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_SHIFT			(16)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_LENGTH			(12)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3SIZE, VID3HEIGHT
+*/
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SIZE_OFFSET						(0x009C)
+
+/* PDP, VID4SIZE, VID4WIDTH
+*/
+#define ODN_PDP_VID4SIZE_VID4WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_SHIFT			(16)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_LENGTH			(12)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4SIZE, VID4HEIGHT
+*/
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1POSN_OFFSET					(0x00A0)
+
+/* PDP, GRPH1POSN, GRPH1XSTART
+*/
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1POSN, GRPH1YSTART
+*/
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2POSN_OFFSET					(0x00A4)
+
+/* PDP, GRPH2POSN, GRPH2XSTART
+*/
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2POSN, GRPH2YSTART
+*/
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3POSN_OFFSET					(0x00A8)
+
+/* PDP, GRPH3POSN, GRPH3XSTART
+*/
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3POSN, GRPH3YSTART
+*/
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4POSN_OFFSET					(0x00AC)
+
+/* PDP, GRPH4POSN, GRPH4XSTART
+*/
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4POSN, GRPH4YSTART
+*/
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1POSN_OFFSET						(0x00B0)
+
+/* PDP, VID1POSN, VID1XSTART
+*/
+#define ODN_PDP_VID1POSN_VID1XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID1POSN_VID1XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1XSTART_SHIFT			(16)
+#define ODN_PDP_VID1POSN_VID1XSTART_LENGTH			(12)
+#define ODN_PDP_VID1POSN_VID1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSN, VID1YSTART
+*/
+#define ODN_PDP_VID1POSN_VID1YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1YSTART_SHIFT			(0)
+#define ODN_PDP_VID1POSN_VID1YSTART_LENGTH			(12)
+#define ODN_PDP_VID1POSN_VID1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2POSN_OFFSET						(0x00B4)
+
+/* PDP, VID2POSN, VID2XSTART
+*/
+#define ODN_PDP_VID2POSN_VID2XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID2POSN_VID2XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2XSTART_SHIFT			(16)
+#define ODN_PDP_VID2POSN_VID2XSTART_LENGTH			(12)
+#define ODN_PDP_VID2POSN_VID2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSN, VID2YSTART
+*/
+#define ODN_PDP_VID2POSN_VID2YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2YSTART_SHIFT			(0)
+#define ODN_PDP_VID2POSN_VID2YSTART_LENGTH			(12)
+#define ODN_PDP_VID2POSN_VID2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3POSN_OFFSET						(0x00B8)
+
+/* PDP, VID3POSN, VID3XSTART
+*/
+#define ODN_PDP_VID3POSN_VID3XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID3POSN_VID3XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3XSTART_SHIFT			(16)
+#define ODN_PDP_VID3POSN_VID3XSTART_LENGTH			(12)
+#define ODN_PDP_VID3POSN_VID3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSN, VID3YSTART
+*/
+#define ODN_PDP_VID3POSN_VID3YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3YSTART_SHIFT			(0)
+#define ODN_PDP_VID3POSN_VID3YSTART_LENGTH			(12)
+#define ODN_PDP_VID3POSN_VID3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4POSN_OFFSET						(0x00BC)
+
+/* PDP, VID4POSN, VID4XSTART
+*/
+#define ODN_PDP_VID4POSN_VID4XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID4POSN_VID4XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4XSTART_SHIFT			(16)
+#define ODN_PDP_VID4POSN_VID4XSTART_LENGTH			(12)
+#define ODN_PDP_VID4POSN_VID4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSN, VID4YSTART
+*/
+#define ODN_PDP_VID4POSN_VID4YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4YSTART_SHIFT			(0)
+#define ODN_PDP_VID4POSN_VID4YSTART_LENGTH			(12)
+#define ODN_PDP_VID4POSN_VID4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1GALPHA_OFFSET					(0x00C0)
+
+/* PDP, GRPH1GALPHA, GRPH1GALPHA
+*/
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2GALPHA_OFFSET					(0x00C4)
+
+/* PDP, GRPH2GALPHA, GRPH2GALPHA
+*/
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3GALPHA_OFFSET					(0x00C8)
+
+/* PDP, GRPH3GALPHA, GRPH3GALPHA
+*/
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4GALPHA_OFFSET					(0x00CC)
+
+/* PDP, GRPH4GALPHA, GRPH4GALPHA
+*/
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1GALPHA_OFFSET					(0x00D0)
+
+/* PDP, VID1GALPHA, VID1GALPHA
+*/
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_SHIFT			(0)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_LENGTH		(10)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2GALPHA_OFFSET					(0x00D4)
+
+/* PDP, VID2GALPHA, VID2GALPHA
+*/
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_SHIFT			(0)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_LENGTH		(10)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3GALPHA_OFFSET					(0x00D8)
+
+/* PDP, VID3GALPHA, VID3GALPHA
+*/
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_SHIFT			(0)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_LENGTH		(10)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4GALPHA_OFFSET				    (0x00DC)
+
+/* PDP, VID4GALPHA, VID4GALPHA
+*/
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_SHIFT			(0)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_LENGTH		(10)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1CKEY_R_OFFSET					(0x00E0)
+
+/* PDP, GRPH1CKEY_R, GRPH1CKEY_R
+*/
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1CKEY_GB_OFFSET			        (0x00E4)
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_G
+*/
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_B
+*/
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CKEY_R_OFFSET					(0x00E8)
+
+/* PDP, GRPH2CKEY_R, GRPH2CKEY_R
+*/
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CKEY_GB_OFFSET					(0x00EC)
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_G
+*/
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_B
+*/
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CKEY_R_OFFSET					(0x00F0)
+
+/* PDP, GRPH3CKEY_R, GRPH3CKEY_R
+*/
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CKEY_GB_OFFSET					(0x00F4)
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_G
+*/
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_B
+*/
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CKEY_R_OFFSET					(0x00F8)
+
+/* PDP, GRPH4CKEY_R, GRPH4CKEY_R
+*/
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CKEY_GB_OFFSET					(0x00FC)
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_G
+*/
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_B
+*/
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1CKEY_R_OFFSET					(0x0100)
+
+/* PDP, VID1CKEY_R, VID1CKEY_R
+*/
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1CKEY_GB_OFFSET					(0x0104)
+
+/* PDP, VID1CKEY_GB, VID1CKEY_G
+*/
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CKEY_GB, VID1CKEY_B
+*/
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CKEY_R_OFFSET					(0x0108)
+
+/* PDP, VID2CKEY_R, VID2CKEY_R
+*/
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CKEY_GB_OFFSET					(0x010C)
+
+/* PDP, VID2CKEY_GB, VID2CKEY_G
+*/
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CKEY_GB, VID2CKEY_B
+*/
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CKEY_R_OFFSET					(0x0110)
+
+/* PDP, VID3CKEY_R, VID3CKEY_R
+*/
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CKEY_GB_OFFSET					(0x0114)
+
+/* PDP, VID3CKEY_GB, VID3CKEY_G
+*/
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CKEY_GB, VID3CKEY_B
+*/
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CKEY_R_OFFSET					(0x0118)
+
+/* PDP, VID4CKEY_R, VID4CKEY_R
+*/
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CKEY_GB_OFFSET					(0x011C)
+
+/* PDP, VID4CKEY_GB, VID4CKEY_G
+*/
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CKEY_GB, VID4CKEY_B
+*/
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1BLND2_R_OFFSET					(0x0120)
+
+/* PDP, GRPH1BLND2_R, GRPH1PIXDBL
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1LINDBL
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1CKEYMASK_R
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1BLND2_GB_OFFSET				(0x0124)
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_G
+*/
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_B
+*/
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BLND2_R_OFFSET					(0x0128)
+
+/* PDP, GRPH2BLND2_R, GRPH2PIXDBL
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2LINDBL
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2CKEYMASK_R
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BLND2_GB_OFFSET				(0x012C)
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_G
+*/
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_B
+*/
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BLND2_R_OFFSET					(0x0130)
+
+/* PDP, GRPH3BLND2_R, GRPH3PIXDBL
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3LINDBL
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3CKEYMASK_R
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BLND2_GB_OFFSET				(0x0134)
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_G
+*/
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_B
+*/
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BLND2_R_OFFSET					(0x0138)
+
+/* PDP, GRPH4BLND2_R, GRPH4PIXDBL
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4LINDBL
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4CKEYMASK_R
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BLND2_GB_OFFSET				(0x013C)
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_G
+*/
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_B
+*/
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1BLND2_R_OFFSET					(0x0140)
+
+/* PDP, VID1BLND2_R, VID1CKEYMASK_R
+*/
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1BLND2_GB_OFFSET					(0x0144)
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_G
+*/
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_B
+*/
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2BLND2_R_OFFSET		            (0x0148)
+
+/* PDP, VID2BLND2_R, VID2CKEYMASK_R
+*/
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2BLND2_GB_OFFSET		            (0x014C)
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_G
+*/
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_B
+*/
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3BLND2_R_OFFSET		            (0x0150)
+
+/* PDP, VID3BLND2_R, VID3CKEYMASK_R
+*/
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3BLND2_GB_OFFSET		            (0x0154)
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_G
+*/
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_B
+*/
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4BLND2_R_OFFSET		            (0x0158)
+
+/* PDP, VID4BLND2_R, VID4CKEYMASK_R
+*/
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4BLND2_GB_OFFSET		            (0x015C)
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_G
+*/
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_B
+*/
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_OFFSET		    (0x0160)
+
+/* PDP, GRPH1INTERLEAVE_CTRL, GRPH1INTFIELD
+*/
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_MASK	(0x00000001)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_OFFSET		    (0x0164)
+
+/* PDP, GRPH2INTERLEAVE_CTRL, GRPH2INTFIELD
+*/
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_OFFSET		    (0x0168)
+
+/* PDP, GRPH3INTERLEAVE_CTRL, GRPH3INTFIELD
+*/
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_OFFSET		    (0x016C)
+
+/* PDP, GRPH4INTERLEAVE_CTRL, GRPH4INTFIELD
+*/
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1INTERLEAVE_CTRL_OFFSET		    (0x0170)
+
+/* PDP, VID1INTERLEAVE_CTRL, VID1INTFIELD
+*/
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SHIFT (0)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LENGTH (1)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2INTERLEAVE_CTRL_OFFSET		    (0x0174)
+
+/* PDP, VID2INTERLEAVE_CTRL, VID2INTFIELD
+*/
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SHIFT (0)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LENGTH (1)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3INTERLEAVE_CTRL_OFFSET		    (0x0178)
+
+/* PDP, VID3INTERLEAVE_CTRL, VID3INTFIELD
+*/
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SHIFT (0)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LENGTH (1)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4INTERLEAVE_CTRL_OFFSET		    (0x017C)
+
+/* PDP, VID4INTERLEAVE_CTRL, VID4INTFIELD
+*/
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SHIFT (0)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LENGTH (1)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1BASEADDR_OFFSET		        (0x0180)
+
+/* PDP, GRPH1BASEADDR, GRPH1BASEADDR
+*/
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BASEADDR_OFFSET		        (0x0184)
+
+/* PDP, GRPH2BASEADDR, GRPH2BASEADDR
+*/
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BASEADDR_OFFSET		        (0x0188)
+
+/* PDP, GRPH3BASEADDR, GRPH3BASEADDR
+*/
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BASEADDR_OFFSET		        (0x018C)
+
+/* PDP, GRPH4BASEADDR, GRPH4BASEADDR
+*/
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1BASEADDR_OFFSET		            (0x0190)
+
+/* PDP, VID1BASEADDR, VID1BASEADDR
+*/
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2BASEADDR_OFFSET		            (0x0194)
+
+/* PDP, VID2BASEADDR, VID2BASEADDR
+*/
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3BASEADDR_OFFSET		            (0x0198)
+
+/* PDP, VID3BASEADDR, VID3BASEADDR
+*/
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4BASEADDR_OFFSET		            (0x019C)
+
+/* PDP, VID4BASEADDR, VID4BASEADDR
+*/
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1UBASEADDR_OFFSET				(0x01B0)
+
+/* PDP, VID1UBASEADDR, VID1UBASEADDR
+*/
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_SHIFT	(5)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH	(27)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2UBASEADDR_OFFSET		        (0x01B4)
+
+/* PDP, VID2UBASEADDR, VID2UBASEADDR
+*/
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3UBASEADDR_OFFSET		        (0x01B8)
+
+/* PDP, VID3UBASEADDR, VID3UBASEADDR
+*/
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4UBASEADDR_OFFSET		        (0x01BC)
+
+/* PDP, VID4UBASEADDR, VID4UBASEADDR
+*/
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VBASEADDR_OFFSET		        (0x01D0)
+
+/* PDP, VID1VBASEADDR, VID1VBASEADDR
+*/
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VBASEADDR_OFFSET		        (0x01D4)
+
+/* PDP, VID2VBASEADDR, VID2VBASEADDR
+*/
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VBASEADDR_OFFSET		        (0x01D8)
+
+/* PDP, VID3VBASEADDR, VID3VBASEADDR
+*/
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VBASEADDR_OFFSET		(0x01DC)
+
+/* PDP, VID4VBASEADDR, VID4VBASEADDR
+*/
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1POSTSKIPCTRL_OFFSET		(0x0230)
+
+/* PDP, VID1POSTSKIPCTRL, VID1HPOSTCLIP
+*/
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSTSKIPCTRL, VID1VPOSTCLIP
+*/
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2POSTSKIPCTRL_OFFSET		(0x0234)
+
+/* PDP, VID2POSTSKIPCTRL, VID2HPOSTCLIP
+*/
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSTSKIPCTRL, VID2VPOSTCLIP
+*/
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3POSTSKIPCTRL_OFFSET		(0x0238)
+
+/* PDP, VID3POSTSKIPCTRL, VID3HPOSTCLIP
+*/
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSTSKIPCTRL, VID3VPOSTCLIP
+*/
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4POSTSKIPCTRL_OFFSET		(0x023C)
+
+/* PDP, VID4POSTSKIPCTRL, VID4HPOSTCLIP
+*/
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSTSKIPCTRL, VID4VPOSTCLIP
+*/
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1DECIMATE_CTRL_OFFSET		(0x0240)
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_EN
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2DECIMATE_CTRL_OFFSET		(0x0244)
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_EN
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3DECIMATE_CTRL_OFFSET		(0x0248)
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_EN
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4DECIMATE_CTRL_OFFSET		(0x024C)
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_EN
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1DECIMATE_CTRL_OFFSET		(0x0250)
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_EN
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2DECIMATE_CTRL_OFFSET		(0x0254)
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_EN
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3DECIMATE_CTRL_OFFSET		(0x0258)
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_EN
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4DECIMATE_CTRL_OFFSET		(0x025C)
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_EN
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SKIPCTRL_OFFSET		(0x0270)
+
+/* PDP, VID1SKIPCTRL, VID1HSKIP
+*/
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_SHIFT		(16)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_LENGTH		(12)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SKIPCTRL, VID1VSKIP
+*/
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_SHIFT		(0)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_LENGTH		(12)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SKIPCTRL_OFFSET		(0x0274)
+
+/* PDP, VID2SKIPCTRL, VID2HSKIP
+*/
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_SHIFT		(16)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_LENGTH		(12)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SKIPCTRL, VID2VSKIP
+*/
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_SHIFT		(0)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_LENGTH		(12)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SKIPCTRL_OFFSET		(0x0278)
+
+/* PDP, VID3SKIPCTRL, VID3HSKIP
+*/
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_SHIFT		(16)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_LENGTH		(12)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SKIPCTRL, VID3VSKIP
+*/
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_SHIFT		(0)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_LENGTH		(12)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SKIPCTRL_OFFSET		(0x027C)
+
+/* PDP, VID4SKIPCTRL, VID4HSKIP
+*/
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_SHIFT		(16)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_LENGTH		(12)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SKIPCTRL, VID4VSKIP
+*/
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_SHIFT		(0)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_LENGTH		(12)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SCALECTRL_OFFSET		(0x0460)
+
+/* PDP, VID1SCALECTRL, VID1HSCALEBP
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSCALEBP
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1HSBEFOREVS
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSURUNCTRL
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1PAN_EN
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VORDER
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_SHIFT		(16)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_LENGTH		(2)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VPITCH
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_SHIFT		(0)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_LENGTH		(16)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VSINIT_OFFSET		(0x0464)
+
+/* PDP, VID1VSINIT, VID1VINITIAL1
+*/
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1VSINIT, VID1VINITIAL0
+*/
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF0_OFFSET		(0x0468)
+
+/* PDP, VID1VCOEFF0, VID1VCOEFF0
+*/
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF1_OFFSET		(0x046C)
+
+/* PDP, VID1VCOEFF1, VID1VCOEFF1
+*/
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF2_OFFSET		(0x0470)
+
+/* PDP, VID1VCOEFF2, VID1VCOEFF2
+*/
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF3_OFFSET		(0x0474)
+
+/* PDP, VID1VCOEFF3, VID1VCOEFF3
+*/
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF4_OFFSET		(0x0478)
+
+/* PDP, VID1VCOEFF4, VID1VCOEFF4
+*/
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF5_OFFSET		(0x047C)
+
+/* PDP, VID1VCOEFF5, VID1VCOEFF5
+*/
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF6_OFFSET		(0x0480)
+
+/* PDP, VID1VCOEFF6, VID1VCOEFF6
+*/
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF7_OFFSET		(0x0484)
+
+/* PDP, VID1VCOEFF7, VID1VCOEFF7
+*/
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF8_OFFSET		(0x0488)
+
+/* PDP, VID1VCOEFF8, VID1VCOEFF8
+*/
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HSINIT_OFFSET		(0x048C)
+
+/* PDP, VID1HSINIT, VID1HINITIAL
+*/
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1HSINIT, VID1HPITCH
+*/
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_SHIFT		(0)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_LENGTH		(16)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF0_OFFSET		(0x0490)
+
+/* PDP, VID1HCOEFF0, VID1HCOEFF0
+*/
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF1_OFFSET		(0x0494)
+
+/* PDP, VID1HCOEFF1, VID1HCOEFF1
+*/
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF2_OFFSET		(0x0498)
+
+/* PDP, VID1HCOEFF2, VID1HCOEFF2
+*/
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF3_OFFSET		(0x049C)
+
+/* PDP, VID1HCOEFF3, VID1HCOEFF3
+*/
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF4_OFFSET		(0x04A0)
+
+/* PDP, VID1HCOEFF4, VID1HCOEFF4
+*/
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF5_OFFSET		(0x04A4)
+
+/* PDP, VID1HCOEFF5, VID1HCOEFF5
+*/
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF6_OFFSET		(0x04A8)
+
+/* PDP, VID1HCOEFF6, VID1HCOEFF6
+*/
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF7_OFFSET		(0x04AC)
+
+/* PDP, VID1HCOEFF7, VID1HCOEFF7
+*/
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF8_OFFSET		(0x04B0)
+
+/* PDP, VID1HCOEFF8, VID1HCOEFF8
+*/
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF9_OFFSET		(0x04B4)
+
+/* PDP, VID1HCOEFF9, VID1HCOEFF9
+*/
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF10_OFFSET		(0x04B8)
+
+/* PDP, VID1HCOEFF10, VID1HCOEFF10
+*/
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF11_OFFSET		(0x04BC)
+
+/* PDP, VID1HCOEFF11, VID1HCOEFF11
+*/
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF12_OFFSET		(0x04C0)
+
+/* PDP, VID1HCOEFF12, VID1HCOEFF12
+*/
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF13_OFFSET		(0x04C4)
+
+/* PDP, VID1HCOEFF13, VID1HCOEFF13
+*/
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF14_OFFSET		(0x04C8)
+
+/* PDP, VID1HCOEFF14, VID1HCOEFF14
+*/
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF15_OFFSET		(0x04CC)
+
+/* PDP, VID1HCOEFF15, VID1HCOEFF15
+*/
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF16_OFFSET		(0x04D0)
+
+/* PDP, VID1HCOEFF16, VID1HCOEFF16
+*/
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SCALESIZE_OFFSET		(0x04D4)
+
+/* PDP, VID1SCALESIZE, VID1SCALEWIDTH
+*/
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALESIZE, VID1SCALEHEIGHT
+*/
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CORE_ID_OFFSET		(0x04E0)
+
+/* PDP, PVR_ODN_PDP_CORE_ID, GROUP_ID
+*/
+#define ODN_PDP_CORE_ID_GROUP_ID_MASK		(0xFF000000)
+#define ODN_PDP_CORE_ID_GROUP_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_ID_GROUP_ID_SHIFT		(24)
+#define ODN_PDP_CORE_ID_GROUP_ID_LENGTH		(8)
+#define ODN_PDP_CORE_ID_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_ID, CORE_ID
+*/
+#define ODN_PDP_CORE_ID_CORE_ID_MASK		(0x00FF0000)
+#define ODN_PDP_CORE_ID_CORE_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_ID_CORE_ID_SHIFT		(16)
+#define ODN_PDP_CORE_ID_CORE_ID_LENGTH		(8)
+#define ODN_PDP_CORE_ID_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_ID, CONFIG_ID
+*/
+#define ODN_PDP_CORE_ID_CONFIG_ID_MASK		(0x0000FFFF)
+#define ODN_PDP_CORE_ID_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_CORE_ID_CONFIG_ID_SHIFT		(0)
+#define ODN_PDP_CORE_ID_CONFIG_ID_LENGTH		(16)
+#define ODN_PDP_CORE_ID_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CORE_REV_OFFSET		(0x04F0)
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MAJOR_REV
+*/
+#define ODN_PDP_CORE_REV_MAJOR_REV_MASK		(0x00FF0000)
+#define ODN_PDP_CORE_REV_MAJOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAJOR_REV_SHIFT		(16)
+#define ODN_PDP_CORE_REV_MAJOR_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MINOR_REV
+*/
+#define ODN_PDP_CORE_REV_MINOR_REV_MASK		(0x0000FF00)
+#define ODN_PDP_CORE_REV_MINOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MINOR_REV_SHIFT		(8)
+#define ODN_PDP_CORE_REV_MINOR_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MAINT_REV
+*/
+#define ODN_PDP_CORE_REV_MAINT_REV_MASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAINT_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAINT_REV_SHIFT		(0)
+#define ODN_PDP_CORE_REV_MAINT_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SCALECTRL_OFFSET		(0x0500)
+
+/* PDP, VID2SCALECTRL, VID2HSCALEBP
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSCALEBP
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2HSBEFOREVS
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSURUNCTRL
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2PAN_EN
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VORDER
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_SHIFT		(16)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_LENGTH		(2)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VPITCH
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_SHIFT		(0)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_LENGTH		(16)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VSINIT_OFFSET		(0x0504)
+
+/* PDP, VID2VSINIT, VID2VINITIAL1
+*/
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2VSINIT, VID2VINITIAL0
+*/
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF0_OFFSET		(0x0508)
+
+/* PDP, VID2VCOEFF0, VID2VCOEFF0
+*/
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF1_OFFSET		(0x050C)
+
+/* PDP, VID2VCOEFF1, VID2VCOEFF1
+*/
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF2_OFFSET		(0x0510)
+
+/* PDP, VID2VCOEFF2, VID2VCOEFF2
+*/
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF3_OFFSET		(0x0514)
+
+/* PDP, VID2VCOEFF3, VID2VCOEFF3
+*/
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF4_OFFSET		(0x0518)
+
+/* PDP, VID2VCOEFF4, VID2VCOEFF4
+*/
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF5_OFFSET		(0x051C)
+
+/* PDP, VID2VCOEFF5, VID2VCOEFF5
+*/
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF6_OFFSET		(0x0520)
+
+/* PDP, VID2VCOEFF6, VID2VCOEFF6
+*/
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF7_OFFSET		(0x0524)
+
+/* PDP, VID2VCOEFF7, VID2VCOEFF7
+*/
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF8_OFFSET		(0x0528)
+
+/* PDP, VID2VCOEFF8, VID2VCOEFF8
+*/
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HSINIT_OFFSET		(0x052C)
+
+/* PDP, VID2HSINIT, VID2HINITIAL
+*/
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2HSINIT, VID2HPITCH
+*/
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_SHIFT		(0)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_LENGTH		(16)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF0_OFFSET		(0x0530)
+
+/* PDP, VID2HCOEFF0, VID2HCOEFF0
+*/
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF1_OFFSET		(0x0534)
+
+/* PDP, VID2HCOEFF1, VID2HCOEFF1
+*/
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF2_OFFSET		(0x0538)
+
+/* PDP, VID2HCOEFF2, VID2HCOEFF2
+*/
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF3_OFFSET		(0x053C)
+
+/* PDP, VID2HCOEFF3, VID2HCOEFF3
+*/
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF4_OFFSET		(0x0540)
+
+/* PDP, VID2HCOEFF4, VID2HCOEFF4
+*/
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF5_OFFSET		(0x0544)
+
+/* PDP, VID2HCOEFF5, VID2HCOEFF5
+*/
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF6_OFFSET		(0x0548)
+
+/* PDP, VID2HCOEFF6, VID2HCOEFF6
+*/
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF7_OFFSET		(0x054C)
+
+/* PDP, VID2HCOEFF7, VID2HCOEFF7
+*/
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF8_OFFSET		(0x0550)
+
+/* PDP, VID2HCOEFF8, VID2HCOEFF8
+*/
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF9_OFFSET		(0x0554)
+
+/* PDP, VID2HCOEFF9, VID2HCOEFF9
+*/
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF10_OFFSET		(0x0558)
+
+/* PDP, VID2HCOEFF10, VID2HCOEFF10
+*/
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF11_OFFSET		(0x055C)
+
+/* PDP, VID2HCOEFF11, VID2HCOEFF11
+*/
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF12_OFFSET		(0x0560)
+
+/* PDP, VID2HCOEFF12, VID2HCOEFF12
+*/
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF13_OFFSET		(0x0564)
+
+/* PDP, VID2HCOEFF13, VID2HCOEFF13
+*/
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF14_OFFSET		(0x0568)
+
+/* PDP, VID2HCOEFF14, VID2HCOEFF14
+*/
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF15_OFFSET		(0x056C)
+
+/* PDP, VID2HCOEFF15, VID2HCOEFF15
+*/
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF16_OFFSET		(0x0570)
+
+/* PDP, VID2HCOEFF16, VID2HCOEFF16
+*/
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SCALESIZE_OFFSET		(0x0574)
+
+/* PDP, VID2SCALESIZE, VID2SCALEWIDTH
+*/
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALESIZE, VID2SCALEHEIGHT
+*/
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SCALECTRL_OFFSET		(0x0578)
+
+/* PDP, VID3SCALECTRL, VID3HSCALEBP
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSCALEBP
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3HSBEFOREVS
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSURUNCTRL
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3PAN_EN
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VORDER
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_SHIFT		(16)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_LENGTH		(2)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VPITCH
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_SHIFT		(0)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_LENGTH		(16)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VSINIT_OFFSET		(0x057C)
+
+/* PDP, VID3VSINIT, VID3VINITIAL1
+*/
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3VSINIT, VID3VINITIAL0
+*/
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF0_OFFSET		(0x0580)
+
+/* PDP, VID3VCOEFF0, VID3VCOEFF0
+*/
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF1_OFFSET		(0x0584)
+
+/* PDP, VID3VCOEFF1, VID3VCOEFF1
+*/
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF2_OFFSET		(0x0588)
+
+/* PDP, VID3VCOEFF2, VID3VCOEFF2
+*/
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF3_OFFSET		(0x058C)
+
+/* PDP, VID3VCOEFF3, VID3VCOEFF3
+*/
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF4_OFFSET		(0x0590)
+
+/* PDP, VID3VCOEFF4, VID3VCOEFF4
+*/
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF5_OFFSET		(0x0594)
+
+/* PDP, VID3VCOEFF5, VID3VCOEFF5
+*/
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF6_OFFSET		(0x0598)
+
+/* PDP, VID3VCOEFF6, VID3VCOEFF6
+*/
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF7_OFFSET		(0x059C)
+
+/* PDP, VID3VCOEFF7, VID3VCOEFF7
+*/
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF8_OFFSET		(0x05A0)
+
+/* PDP, VID3VCOEFF8, VID3VCOEFF8
+*/
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HSINIT_OFFSET		(0x05A4)
+
+/* PDP, VID3HSINIT, VID3HINITIAL
+*/
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3HSINIT, VID3HPITCH
+*/
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_SHIFT		(0)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_LENGTH		(16)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF0_OFFSET		(0x05A8)
+
+/* PDP, VID3HCOEFF0, VID3HCOEFF0
+*/
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF1_OFFSET		(0x05AC)
+
+/* PDP, VID3HCOEFF1, VID3HCOEFF1
+*/
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF2_OFFSET		(0x05B0)
+
+/* PDP, VID3HCOEFF2, VID3HCOEFF2
+*/
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF3_OFFSET		(0x05B4)
+
+/* PDP, VID3HCOEFF3, VID3HCOEFF3
+*/
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF4_OFFSET		(0x05B8)
+
+/* PDP, VID3HCOEFF4, VID3HCOEFF4
+*/
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF5_OFFSET		(0x05BC)
+
+/* PDP, VID3HCOEFF5, VID3HCOEFF5
+*/
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF6_OFFSET		(0x05C0)
+
+/* PDP, VID3HCOEFF6, VID3HCOEFF6
+*/
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF7_OFFSET		(0x05C4)
+
+/* PDP, VID3HCOEFF7, VID3HCOEFF7
+*/
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF8_OFFSET		(0x05C8)
+
+/* PDP, VID3HCOEFF8, VID3HCOEFF8
+*/
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF9_OFFSET		(0x05CC)
+
+/* PDP, VID3HCOEFF9, VID3HCOEFF9
+*/
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF10_OFFSET		(0x05D0)
+
+/* PDP, VID3HCOEFF10, VID3HCOEFF10
+*/
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_LENGTH	(32)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF11_OFFSET		(0x05D4)
+
+/* PDP, VID3HCOEFF11, VID3HCOEFF11
+*/
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_LENGTH	(32)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF12_OFFSET		(0x05D8)
+
+/* PDP, VID3HCOEFF12, VID3HCOEFF12
+*/
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF13_OFFSET		(0x05DC)
+
+/* PDP, VID3HCOEFF13, VID3HCOEFF13
+*/
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF14_OFFSET		(0x05E0)
+
+/* PDP, VID3HCOEFF14, VID3HCOEFF14
+*/
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF15_OFFSET		(0x05E4)
+
+/* PDP, VID3HCOEFF15, VID3HCOEFF15
+*/
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF16_OFFSET		(0x05E8)
+
+/* PDP, VID3HCOEFF16, VID3HCOEFF16
+*/
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SCALESIZE_OFFSET		(0x05EC)
+
+/* PDP, VID3SCALESIZE, VID3SCALEWIDTH
+*/
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALESIZE, VID3SCALEHEIGHT
+*/
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SCALECTRL_OFFSET		(0x05F0)
+
+/* PDP, VID4SCALECTRL, VID4HSCALEBP
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_SHIFT	(31)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSCALEBP
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_SHIFT	(30)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4HSBEFOREVS
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_MASK	(0x20000000)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_SHIFT	(29)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSURUNCTRL
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_MASK	(0x08000000)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_SHIFT	(27)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4PAN_EN
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VORDER
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_LSBMASK	(0x00000003)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_SHIFT		(16)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_LENGTH		(2)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VPITCH
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_LSBMASK	(0x0000FFFF)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_SHIFT		(0)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_LENGTH		(16)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VSINIT_OFFSET			(0x05F4)
+
+/* PDP, VID4VSINIT, VID4VINITIAL1
+*/
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4VSINIT, VID4VINITIAL0
+*/
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF0_OFFSET			(0x05F8)
+
+/* PDP, VID4VCOEFF0, VID4VCOEFF0
+*/
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF1_OFFSET			(0x05FC)
+
+/* PDP, VID4VCOEFF1, VID4VCOEFF1
+*/
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF2_OFFSET			(0x0600)
+
+/* PDP, VID4VCOEFF2, VID4VCOEFF2
+*/
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF3_OFFSET			(0x0604)
+
+/* PDP, VID4VCOEFF3, VID4VCOEFF3
+*/
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF4_OFFSET			(0x0608)
+
+/* PDP, VID4VCOEFF4, VID4VCOEFF4
+*/
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF5_OFFSET			(0x060C)
+
+/* PDP, VID4VCOEFF5, VID4VCOEFF5
+*/
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF6_OFFSET			(0x0610)
+
+/* PDP, VID4VCOEFF6, VID4VCOEFF6
+*/
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF7_OFFSET			(0x0614)
+
+/* PDP, VID4VCOEFF7, VID4VCOEFF7
+*/
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF8_OFFSET			(0x0618)
+
+/* PDP, VID4VCOEFF8, VID4VCOEFF8
+*/
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HSINIT_OFFSET			(0x061C)
+
+/* PDP, VID4HSINIT, VID4HINITIAL
+*/
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4HSINIT, VID4HPITCH
+*/
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_SHIFT		(0)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_LENGTH		(16)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF0_OFFSET			(0x0620)
+
+/* PDP, VID4HCOEFF0, VID4HCOEFF0
+*/
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF1_OFFSET			(0x0624)
+
+/* PDP, VID4HCOEFF1, VID4HCOEFF1
+*/
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF2_OFFSET			(0x0628)
+
+/* PDP, VID4HCOEFF2, VID4HCOEFF2
+*/
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF3_OFFSET			(0x062C)
+
+/* PDP, VID4HCOEFF3, VID4HCOEFF3
+*/
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF4_OFFSET			(0x0630)
+
+/* PDP, VID4HCOEFF4, VID4HCOEFF4
+*/
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF5_OFFSET			(0x0634)
+
+/* PDP, VID4HCOEFF5, VID4HCOEFF5
+*/
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF6_OFFSET			(0x0638)
+
+/* PDP, VID4HCOEFF6, VID4HCOEFF6
+*/
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF7_OFFSET			(0x063C)
+
+/* PDP, VID4HCOEFF7, VID4HCOEFF7
+*/
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF8_OFFSET			(0x0640)
+
+/* PDP, VID4HCOEFF8, VID4HCOEFF8
+*/
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF9_OFFSET			(0x0644)
+
+/* PDP, VID4HCOEFF9, VID4HCOEFF9
+*/
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF10_OFFSET			(0x0648)
+
+/* PDP, VID4HCOEFF10, VID4HCOEFF10
+*/
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF11_OFFSET			(0x064C)
+
+/* PDP, VID4HCOEFF11, VID4HCOEFF11
+*/
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF12_OFFSET			(0x0650)
+
+/* PDP, VID4HCOEFF12, VID4HCOEFF12
+*/
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF13_OFFSET			(0x0654)
+
+/* PDP, VID4HCOEFF13, VID4HCOEFF13
+*/
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF14_OFFSET			(0x0658)
+
+/* PDP, VID4HCOEFF14, VID4HCOEFF14
+*/
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF15_OFFSET			(0x065C)
+
+/* PDP, VID4HCOEFF15, VID4HCOEFF15
+*/
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF16_OFFSET			(0x0660)
+
+/* PDP, VID4HCOEFF16, VID4HCOEFF16
+*/
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_LSBMASK	(0x000000FF)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_LENGTH	(8)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SCALESIZE_OFFSET			(0x0664)
+
+/* PDP, VID4SCALESIZE, VID4SCALEWIDTH
+*/
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALESIZE, VID4SCALEHEIGHT
+*/
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND0_OFFSET				(0x0668)
+
+/* PDP, PORTER_BLND0, BLND0BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND0, BLND0PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND1_OFFSET				(0x066C)
+
+/* PDP, PORTER_BLND1, BLND1BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND1, BLND1PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND2_OFFSET				(0x0670)
+
+/* PDP, PORTER_BLND2, BLND2BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND2, BLND2PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND3_OFFSET				(0x0674)
+
+/* PDP, PORTER_BLND3, BLND3BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND3, BLND3PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND4_OFFSET				(0x0678)
+
+/* PDP, PORTER_BLND4, BLND4BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND4, BLND4PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND5_OFFSET				(0x067C)
+
+/* PDP, PORTER_BLND5, BLND5BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND5, BLND5PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND6_OFFSET				(0x0680)
+
+/* PDP, PORTER_BLND6, BLND6BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND6, BLND6PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND7_OFFSET				(0x0684)
+
+/* PDP, PORTER_BLND7, BLND7BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND7, BLND7PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06C8)
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06CC)
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMAX
+*/
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMIN
+*/
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_C_RG_OFFSET		(0x06D0)
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_R
+*/
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_G
+*/
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_C_B_OFFSET		(0x06D4)
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYEN
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYC_B
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06D8)
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_OFFSET				(0x06DC)
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMAX
+*/
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMIN
+*/
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_C_RG_OFFSET				(0x06E0)
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_R
+*/
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_G
+*/
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_C_B_OFFSET				(0x06E4)
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYEN
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYC_B
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06E8)
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_OFFSET			(0x06EC)
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMAX
+*/
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMIN
+*/
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_C_RG_OFFSET		(0x06F0)
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_R
+*/
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_G
+*/
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_C_B_OFFSET		(0x06F4)
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYEN
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYC_B
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06F8)
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06FC)
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMAX
+*/
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMIN
+*/
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_C_RG_OFFSET			(0x0700)
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_R
+*/
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_G
+*/
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_C_B_OFFSET		(0x0704)
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_MASK	(0x20000000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SHIFT	(29)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LENGTH	(1)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYEN
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYC_B
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF0_OFFSET			(0x0708)
+
+/* PDP, CSCCOEFF0, CSCCOEFFRU
+*/
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF0, CSCCOEFFRY
+*/
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF1_OFFSET			(0x070C)
+
+/* PDP, CSCCOEFF1, CSCCOEFFGY
+*/
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF1, CSCCOEFFRV
+*/
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF2_OFFSET			(0x0710)
+
+/* PDP, CSCCOEFF2, CSCCOEFFGV
+*/
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF2, CSCCOEFFGU
+*/
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF3_OFFSET			(0x0714)
+
+/* PDP, CSCCOEFF3, CSCCOEFFBU
+*/
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF3, CSCCOEFFBY
+*/
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF4_OFFSET			(0x0718)
+
+/* PDP, CSCCOEFF4, CSCCOEFFBV
+*/
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BGNDCOL_AR_OFFSET			(0x071C)
+
+/* PDP, BGNDCOL_AR, BGNDCOL_A
+*/
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_MASK		(0x03FF0000)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_SHIFT		(16)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_AR, BGNDCOL_R
+*/
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_MASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_SHIFT		(0)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BGNDCOL_GB_OFFSET			(0x0720)
+
+/* PDP, BGNDCOL_GB, BGNDCOL_G
+*/
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_MASK		(0x03FF0000)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_SHIFT		(16)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_GB, BGNDCOL_B
+*/
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_MASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_SHIFT		(0)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BORDCOL_R_OFFSET			(0x0724)
+
+/* PDP, BORDCOL_R, BORDCOL_R
+*/
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_MASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_SHIFT		(0)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_LENGTH		(10)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BORDCOL_GB_OFFSET			(0x0728)
+
+/* PDP, BORDCOL_GB, BORDCOL_G
+*/
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_MASK		(0x03FF0000)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_SHIFT		(16)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_LENGTH		(10)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BORDCOL_GB, BORDCOL_B
+*/
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_MASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_SHIFT		(0)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_LENGTH		(10)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_LINESTAT_OFFSET				(0x0734)
+
+/* PDP, LINESTAT, LINENO
+*/
+#define ODN_PDP_LINESTAT_LINENO_MASK			(0x00001FFF)
+#define ODN_PDP_LINESTAT_LINENO_LSBMASK			(0x00001FFF)
+#define ODN_PDP_LINESTAT_LINENO_SHIFT			(0)
+#define ODN_PDP_LINESTAT_LINENO_LENGTH			(13)
+#define ODN_PDP_LINESTAT_LINENO_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_OFFSET	(0x0738)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C11C12, CR_PROCAMP_C12
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C11C12, CR_PROCAMP_C11
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_OFFSET		(0x073C)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C13C21, CR_PROCAMP_C21
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C13C21, CR_PROCAMP_C13
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_OFFSET		(0x0740)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C22C23, CR_PROCAMP_C23
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C22C23, CR_PROCAMP_C22
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_OFFSET		(0x0744)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C31C32, CR_PROCAMP_C32
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C31C32, CR_PROCAMP_C31
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_OFFSET		(0x0748)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_C33
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_LSBMASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_RANGE
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_MASK		(0x00000030)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LSBMASK		(0x00000003)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SHIFT		(4)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LENGTH		(2)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_EN
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_MASK		(0x00000001)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_LENGTH		(1)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_OFFSET		(0x074C)
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_G
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_MASK		(0x0FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_B
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_MASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_OFFSET		(0x0750)
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_R, CR_PROCAMP_OUTOFF_R
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_MASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_OFFSET		(0x0754)
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_G
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_MASK		(0x03FF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_B
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_MASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_OFFSET		(0x0758)
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_R, CR_PROCAMP_INOFF_R
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_MASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SIGNAT_R_OFFSET		(0x075C)
+
+/* PDP, SIGNAT_R, SIGNATURE_R
+*/
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_MASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_SHIFT		(0)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_LENGTH		(10)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SIGNAT_GB_OFFSET		(0x0760)
+
+/* PDP, SIGNAT_GB, SIGNATURE_G
+*/
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_MASK		(0x03FF0000)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_SHIFT		(16)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_LENGTH		(10)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SIGNAT_GB, SIGNATURE_B
+*/
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_MASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_SHIFT		(0)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_LENGTH		(10)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGISTER_UPDATE_CTRL_OFFSET		(0x0764)
+
+/* PDP, REGISTER_UPDATE_CTRL, BYPASS_DOUBLE_BUFFERING
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_MASK		(0x00000004)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SHIFT		(2)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, REGISTERS_VALID
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_MASK		(0x00000002)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SHIFT		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, USE_VBLANK
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_MASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SHIFT		(0)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGISTER_UPDATE_STATUS_OFFSET		(0x0768)
+
+/* PDP, REGISTER_UPDATE_STATUS, REGISTERS_UPDATED
+*/
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_MASK		(0x00000002)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SHIFT		(1)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGCTRL_OFFSET		(0x076C)
+
+/* PDP, DBGCTRL, DBG_READ
+*/
+#define ODN_PDP_DBGCTRL_DBG_READ_MASK		(0x00000002)
+#define ODN_PDP_DBGCTRL_DBG_READ_LSBMASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_READ_SHIFT		(1)
+#define ODN_PDP_DBGCTRL_DBG_READ_LENGTH		(1)
+#define ODN_PDP_DBGCTRL_DBG_READ_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGCTRL, DBG_ENAB
+*/
+#define ODN_PDP_DBGCTRL_DBG_ENAB_MASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_LSBMASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_SHIFT		(0)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_LENGTH		(1)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGDATA_R_OFFSET		(0x0770)
+
+/* PDP, DBGDATA_R, DBG_DATA_R
+*/
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_MASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_SHIFT		(0)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_LENGTH		(10)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGDATA_GB_OFFSET		(0x0774)
+
+/* PDP, DBGDATA_GB, DBG_DATA_G
+*/
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_MASK		(0x03FF0000)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_SHIFT		(16)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_LENGTH		(10)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGDATA_GB, DBG_DATA_B
+*/
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_MASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_SHIFT		(0)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_LENGTH		(10)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGSIDE_OFFSET				(0x0778)
+
+/* PDP, DBGSIDE, DBG_VAL
+*/
+#define ODN_PDP_DBGSIDE_DBG_VAL_MASK			(0x00000008)
+#define ODN_PDP_DBGSIDE_DBG_VAL_LSBMASK			(0x00000001)
+#define ODN_PDP_DBGSIDE_DBG_VAL_SHIFT			(3)
+#define ODN_PDP_DBGSIDE_DBG_VAL_LENGTH			(1)
+#define ODN_PDP_DBGSIDE_DBG_VAL_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, DBGSIDE, DBG_SIDE
+*/
+#define ODN_PDP_DBGSIDE_DBG_SIDE_MASK			(0x00000007)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_LSBMASK		(0x00000007)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_SHIFT			(0)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_LENGTH			(3)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_OUTPUT_OFFSET				(0x077C)
+
+/* PDP, OUTPUT, EIGHT_BIT_OUTPUT
+*/
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_MASK		(0x00000002)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_LSBMASK		(0x00000001)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_SHIFT		(1)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_LENGTH		(1)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OUTPUT, OUTPUT_CONFIG
+*/
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_MASK		(0x00000001)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_LSBMASK		(0x00000001)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_SHIFT		(0)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_LENGTH		(1)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SYNCCTRL_OFFSET				(0x0780)
+
+/* PDP, SYNCCTRL, SYNCACTIVE
+*/
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_MASK		(0x80000000)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_SHIFT		(31)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, ODN_PDP_RST
+*/
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_MASK		(0x20000000)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_SHIFT		(29)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, POWERDN
+*/
+#define ODN_PDP_SYNCCTRL_POWERDN_MASK			(0x10000000)
+#define ODN_PDP_SYNCCTRL_POWERDN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_POWERDN_SHIFT			(28)
+#define ODN_PDP_SYNCCTRL_POWERDN_LENGTH			(1)
+#define ODN_PDP_SYNCCTRL_POWERDN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, LOWPWRMODE
+*/
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_MASK		(0x08000000)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_SHIFT		(27)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDSYNCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_MASK		(0x04000000)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_SHIFT		(26)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDINTCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_MASK		(0x02000000)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_SHIFT		(25)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDCTRL_MASK			(0x01000000)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_SHIFT			(24)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_LENGTH			(1)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDWAIT
+*/
+#define ODN_PDP_SYNCCTRL_UPDWAIT_MASK			(0x000F0000)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_LSBMASK		(0x0000000F)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_SHIFT			(16)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_LENGTH			(4)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, FIELD_EN
+*/
+#define ODN_PDP_SYNCCTRL_FIELD_EN_MASK			(0x00002000)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_SHIFT			(13)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, CSYNC_EN
+*/
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_MASK			(0x00001000)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_SHIFT			(12)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, CLKPOL
+*/
+#define ODN_PDP_SYNCCTRL_CLKPOL_MASK		(0x00000800)
+#define ODN_PDP_SYNCCTRL_CLKPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_CLKPOL_SHIFT		(11)
+#define ODN_PDP_SYNCCTRL_CLKPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_CLKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VS_SLAVE
+*/
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_MASK		(0x00000080)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_SHIFT		(7)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HS_SLAVE
+*/
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_MASK		(0x00000040)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_SHIFT		(6)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKPOL
+*/
+#define ODN_PDP_SYNCCTRL_BLNKPOL_MASK		(0x00000020)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_SHIFT		(5)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKDIS
+*/
+#define ODN_PDP_SYNCCTRL_BLNKDIS_MASK		(0x00000010)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_SHIFT		(4)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSPOL
+*/
+#define ODN_PDP_SYNCCTRL_VSPOL_MASK		(0x00000008)
+#define ODN_PDP_SYNCCTRL_VSPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VSPOL_SHIFT		(3)
+#define ODN_PDP_SYNCCTRL_VSPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSDIS
+*/
+#define ODN_PDP_SYNCCTRL_VSDIS_MASK		(0x00000004)
+#define ODN_PDP_SYNCCTRL_VSDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VSDIS_SHIFT		(2)
+#define ODN_PDP_SYNCCTRL_VSDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VSDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSPOL
+*/
+#define ODN_PDP_SYNCCTRL_HSPOL_MASK		(0x00000002)
+#define ODN_PDP_SYNCCTRL_HSPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSPOL_SHIFT		(1)
+#define ODN_PDP_SYNCCTRL_HSPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSDIS
+*/
+#define ODN_PDP_SYNCCTRL_HSDIS_MASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSDIS_SHIFT		(0)
+#define ODN_PDP_SYNCCTRL_HSDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HSDIS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_HSYNC1_OFFSET			(0x0784)
+
+/* PDP, HSYNC1, HBPS
+*/
+#define ODN_PDP_HSYNC1_HBPS_MASK		(0x1FFF0000)
+#define ODN_PDP_HSYNC1_HBPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC1_HBPS_SHIFT		(16)
+#define ODN_PDP_HSYNC1_HBPS_LENGTH		(13)
+#define ODN_PDP_HSYNC1_HBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC1, HT
+*/
+#define ODN_PDP_HSYNC1_HT_MASK			(0x00001FFF)
+#define ODN_PDP_HSYNC1_HT_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC1_HT_SHIFT			(0)
+#define ODN_PDP_HSYNC1_HT_LENGTH		(13)
+#define ODN_PDP_HSYNC1_HT_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_HSYNC2_OFFSET			(0x0788)
+
+/* PDP, HSYNC2, HAS
+*/
+#define ODN_PDP_HSYNC2_HAS_MASK			(0x1FFF0000)
+#define ODN_PDP_HSYNC2_HAS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HAS_SHIFT		(16)
+#define ODN_PDP_HSYNC2_HAS_LENGTH		(13)
+#define ODN_PDP_HSYNC2_HAS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC2, HLBS
+*/
+#define ODN_PDP_HSYNC2_HLBS_MASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HLBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HLBS_SHIFT		(0)
+#define ODN_PDP_HSYNC2_HLBS_LENGTH		(13)
+#define ODN_PDP_HSYNC2_HLBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_HSYNC3_OFFSET			(0x078C)
+
+/* PDP, HSYNC3, HFPS
+*/
+#define ODN_PDP_HSYNC3_HFPS_MASK		(0x1FFF0000)
+#define ODN_PDP_HSYNC3_HFPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HFPS_SHIFT		(16)
+#define ODN_PDP_HSYNC3_HFPS_LENGTH		(13)
+#define ODN_PDP_HSYNC3_HFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC3, HRBS
+*/
+#define ODN_PDP_HSYNC3_HRBS_MASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HRBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HRBS_SHIFT		(0)
+#define ODN_PDP_HSYNC3_HRBS_LENGTH		(13)
+#define ODN_PDP_HSYNC3_HRBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VSYNC1_OFFSET			(0x0790)
+
+/* PDP, VSYNC1, VBPS
+*/
+#define ODN_PDP_VSYNC1_VBPS_MASK		(0x1FFF0000)
+#define ODN_PDP_VSYNC1_VBPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC1_VBPS_SHIFT		(16)
+#define ODN_PDP_VSYNC1_VBPS_LENGTH		(13)
+#define ODN_PDP_VSYNC1_VBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC1, VT
+*/
+#define ODN_PDP_VSYNC1_VT_MASK			(0x00001FFF)
+#define ODN_PDP_VSYNC1_VT_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC1_VT_SHIFT			(0)
+#define ODN_PDP_VSYNC1_VT_LENGTH		(13)
+#define ODN_PDP_VSYNC1_VT_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_VSYNC2_OFFSET			(0x0794)
+
+/* PDP, VSYNC2, VAS
+*/
+#define ODN_PDP_VSYNC2_VAS_MASK			(0x1FFF0000)
+#define ODN_PDP_VSYNC2_VAS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VAS_SHIFT		(16)
+#define ODN_PDP_VSYNC2_VAS_LENGTH		(13)
+#define ODN_PDP_VSYNC2_VAS_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VSYNC2, VTBS
+*/
+#define ODN_PDP_VSYNC2_VTBS_MASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VTBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VTBS_SHIFT		(0)
+#define ODN_PDP_VSYNC2_VTBS_LENGTH		(13)
+#define ODN_PDP_VSYNC2_VTBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VSYNC3_OFFSET			(0x0798)
+
+/* PDP, VSYNC3, VFPS
+*/
+#define ODN_PDP_VSYNC3_VFPS_MASK		(0x1FFF0000)
+#define ODN_PDP_VSYNC3_VFPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VFPS_SHIFT		(16)
+#define ODN_PDP_VSYNC3_VFPS_LENGTH		(13)
+#define ODN_PDP_VSYNC3_VFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC3, VBBS
+*/
+#define ODN_PDP_VSYNC3_VBBS_MASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VBBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VBBS_SHIFT		(0)
+#define ODN_PDP_VSYNC3_VBBS_LENGTH		(13)
+#define ODN_PDP_VSYNC3_VBBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTSTAT_OFFSET			(0x079C)
+
+/* PDP, INTSTAT, INTS_VID4ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID4URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK1
+*/
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK0
+*/
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK1
+*/
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK0
+*/
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTENAB_OFFSET				(0x07A0)
+
+/* PDP, INTENAB, INTEN_VID4ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID4URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK1
+*/
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK0
+*/
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK1
+*/
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK0
+*/
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTCLR_OFFSET		(0x07A4)
+
+/* PDP, INTCLR, INTCLR_VID4ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID4URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK1
+*/
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK0
+*/
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK1
+*/
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK0
+*/
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_MEMCTRL_OFFSET		(0x07A8)
+
+/* PDP, MEMCTRL, MEMREFRESH
+*/
+#define ODN_PDP_MEMCTRL_MEMREFRESH_MASK		(0xC0000000)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_LSBMASK		(0x00000003)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_SHIFT		(30)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_LENGTH		(2)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEMCTRL, BURSTLEN
+*/
+#define ODN_PDP_MEMCTRL_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_MEMCTRL_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_MEMCTRL_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_MEMCTRL_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_MEMCTRL_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_MEM_THRESH_OFFSET		(0x07AC)
+
+/* PDP, MEM_THRESH, UVTHRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, YTHRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, THRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_ALTERNATE_3D_CTRL_OFFSET		(0x07B0)
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_ON
+*/
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_MASK		(0x00000010)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LSBMASK		(0x00000001)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SHIFT		(4)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LENGTH		(1)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_BLENDSEL
+*/
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_MASK		(0x00000007)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LSBMASK		(0x00000007)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SHIFT		(0)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LENGTH		(3)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA0_R_OFFSET		(0x07B4)
+
+/* PDP, GAMMA0_R, GAMMA0_R
+*/
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_SHIFT		(0)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_LENGTH		(10)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA0_GB_OFFSET		(0x07B8)
+
+/* PDP, GAMMA0_GB, GAMMA0_G
+*/
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_SHIFT		(16)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_LENGTH		(10)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA0_GB, GAMMA0_B
+*/
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_SHIFT		(0)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_LENGTH		(10)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA1_R_OFFSET		(0x07BC)
+
+/* PDP, GAMMA1_R, GAMMA1_R
+*/
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_SHIFT		(0)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_LENGTH		(10)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA1_GB_OFFSET		(0x07C0)
+
+/* PDP, GAMMA1_GB, GAMMA1_G
+*/
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_SHIFT		(16)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_LENGTH		(10)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA1_GB, GAMMA1_B
+*/
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_SHIFT		(0)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_LENGTH		(10)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA2_R_OFFSET		(0x07C4)
+
+/* PDP, GAMMA2_R, GAMMA2_R
+*/
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_SHIFT		(0)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_LENGTH		(10)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA2_GB_OFFSET		(0x07C8)
+
+/* PDP, GAMMA2_GB, GAMMA2_G
+*/
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_SHIFT		(16)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_LENGTH		(10)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA2_GB, GAMMA2_B
+*/
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_SHIFT		(0)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_LENGTH		(10)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA3_R_OFFSET		(0x07CC)
+
+/* PDP, GAMMA3_R, GAMMA3_R
+*/
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_SHIFT		(0)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_LENGTH		(10)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA3_GB_OFFSET		(0x07D0)
+
+/* PDP, GAMMA3_GB, GAMMA3_G
+*/
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_SHIFT		(16)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_LENGTH		(10)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA3_GB, GAMMA3_B
+*/
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_SHIFT		(0)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_LENGTH		(10)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA4_R_OFFSET		(0x07D4)
+
+/* PDP, GAMMA4_R, GAMMA4_R
+*/
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_SHIFT		(0)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_LENGTH		(10)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA4_GB_OFFSET		(0x07D8)
+
+/* PDP, GAMMA4_GB, GAMMA4_G
+*/
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_SHIFT		(16)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_LENGTH		(10)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA4_GB, GAMMA4_B
+*/
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_SHIFT		(0)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_LENGTH		(10)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA5_R_OFFSET		(0x07DC)
+
+/* PDP, GAMMA5_R, GAMMA5_R
+*/
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_SHIFT		(0)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_LENGTH		(10)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA5_GB_OFFSET		(0x07E0)
+
+/* PDP, GAMMA5_GB, GAMMA5_G
+*/
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_SHIFT		(16)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_LENGTH		(10)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA5_GB, GAMMA5_B
+*/
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_SHIFT		(0)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_LENGTH		(10)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA6_R_OFFSET		(0x07E4)
+
+/* PDP, GAMMA6_R, GAMMA6_R
+*/
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_SHIFT		(0)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_LENGTH		(10)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA6_GB_OFFSET		(0x07E8)
+
+/* PDP, GAMMA6_GB, GAMMA6_G
+*/
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_SHIFT		(16)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_LENGTH		(10)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA6_GB, GAMMA6_B
+*/
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_SHIFT		(0)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_LENGTH		(10)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA7_R_OFFSET		(0x07EC)
+
+/* PDP, GAMMA7_R, GAMMA7_R
+*/
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_SHIFT		(0)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_LENGTH		(10)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA7_GB_OFFSET		(0x07F0)
+
+/* PDP, GAMMA7_GB, GAMMA7_G
+*/
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_SHIFT		(16)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_LENGTH		(10)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA7_GB, GAMMA7_B
+*/
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_SHIFT		(0)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_LENGTH		(10)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA8_R_OFFSET		(0x07F4)
+
+/* PDP, GAMMA8_R, GAMMA8_R
+*/
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_SHIFT		(0)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_LENGTH		(10)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA8_GB_OFFSET		(0x07F8)
+
+/* PDP, GAMMA8_GB, GAMMA8_G
+*/
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_SHIFT		(16)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_LENGTH		(10)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA8_GB, GAMMA8_B
+*/
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_SHIFT		(0)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_LENGTH		(10)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA9_R_OFFSET		(0x07FC)
+
+/* PDP, GAMMA9_R, GAMMA9_R
+*/
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_SHIFT		(0)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_LENGTH		(10)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA9_GB_OFFSET		(0x0800)
+
+/* PDP, GAMMA9_GB, GAMMA9_G
+*/
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_SHIFT		(16)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_LENGTH		(10)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA9_GB, GAMMA9_B
+*/
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_SHIFT		(0)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_LENGTH		(10)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA10_R_OFFSET		(0x0804)
+
+/* PDP, GAMMA10_R, GAMMA10_R
+*/
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_SHIFT		(0)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_LENGTH		(10)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA10_GB_OFFSET		(0x0808)
+
+/* PDP, GAMMA10_GB, GAMMA10_G
+*/
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_SHIFT		(16)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_LENGTH		(10)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA10_GB, GAMMA10_B
+*/
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_SHIFT		(0)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_LENGTH		(10)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA11_R_OFFSET		(0x080C)
+
+/* PDP, GAMMA11_R, GAMMA11_R
+*/
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_SHIFT		(0)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_LENGTH		(10)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA11_GB_OFFSET		(0x0810)
+
+/* PDP, GAMMA11_GB, GAMMA11_G
+*/
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_SHIFT		(16)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_LENGTH		(10)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA11_GB, GAMMA11_B
+*/
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_SHIFT		(0)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_LENGTH		(10)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA12_R_OFFSET		(0x0814)
+
+/* PDP, GAMMA12_R, GAMMA12_R
+*/
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_SHIFT		(0)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_LENGTH		(10)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA12_GB_OFFSET		(0x0818)
+
+/* PDP, GAMMA12_GB, GAMMA12_G
+*/
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_SHIFT		(16)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_LENGTH		(10)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA12_GB, GAMMA12_B
+*/
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_SHIFT		(0)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_LENGTH		(10)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA13_R_OFFSET		(0x081C)
+
+/* PDP, GAMMA13_R, GAMMA13_R
+*/
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_SHIFT		(0)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_LENGTH		(10)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA13_GB_OFFSET		(0x0820)
+
+/* PDP, GAMMA13_GB, GAMMA13_G
+*/
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_SHIFT		(16)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_LENGTH		(10)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA13_GB, GAMMA13_B
+*/
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_SHIFT		(0)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_LENGTH		(10)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA14_R_OFFSET		(0x0824)
+
+/* PDP, GAMMA14_R, GAMMA14_R
+*/
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_SHIFT		(0)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_LENGTH		(10)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA14_GB_OFFSET		(0x0828)
+
+/* PDP, GAMMA14_GB, GAMMA14_G
+*/
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_SHIFT		(16)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_LENGTH		(10)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA14_GB, GAMMA14_B
+*/
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_SHIFT		(0)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_LENGTH		(10)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA15_R_OFFSET		(0x082C)
+
+/* PDP, GAMMA15_R, GAMMA15_R
+*/
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_SHIFT		(0)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_LENGTH		(10)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA15_GB_OFFSET		(0x0830)
+
+/* PDP, GAMMA15_GB, GAMMA15_G
+*/
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_SHIFT		(16)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_LENGTH		(10)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA15_GB, GAMMA15_B
+*/
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_SHIFT		(0)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_LENGTH		(10)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA16_R_OFFSET		(0x0834)
+
+/* PDP, GAMMA16_R, GAMMA16_R
+*/
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_SHIFT		(0)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_LENGTH		(10)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA16_GB_OFFSET		(0x0838)
+
+/* PDP, GAMMA16_GB, GAMMA16_G
+*/
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_SHIFT		(16)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_LENGTH		(10)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA16_GB, GAMMA16_B
+*/
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_SHIFT		(0)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_LENGTH		(10)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA17_R_OFFSET		(0x083C)
+
+/* PDP, GAMMA17_R, GAMMA17_R
+*/
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_SHIFT		(0)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_LENGTH		(10)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA17_GB_OFFSET		(0x0840)
+
+/* PDP, GAMMA17_GB, GAMMA17_G
+*/
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_SHIFT		(16)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_LENGTH		(10)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA17_GB, GAMMA17_B
+*/
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_SHIFT		(0)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_LENGTH		(10)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA18_R_OFFSET		(0x0844)
+
+/* PDP, GAMMA18_R, GAMMA18_R
+*/
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_SHIFT		(0)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_LENGTH		(10)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA18_GB_OFFSET		(0x0848)
+
+/* PDP, GAMMA18_GB, GAMMA18_G
+*/
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_SHIFT		(16)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_LENGTH		(10)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA18_GB, GAMMA18_B
+*/
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_SHIFT		(0)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_LENGTH		(10)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA19_R_OFFSET		(0x084C)
+
+/* PDP, GAMMA19_R, GAMMA19_R
+*/
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_SHIFT		(0)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_LENGTH		(10)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA19_GB_OFFSET		(0x0850)
+
+/* PDP, GAMMA19_GB, GAMMA19_G
+*/
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_SHIFT		(16)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_LENGTH		(10)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA19_GB, GAMMA19_B
+*/
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_SHIFT		(0)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_LENGTH		(10)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA20_R_OFFSET		(0x0854)
+
+/* PDP, GAMMA20_R, GAMMA20_R
+*/
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_SHIFT		(0)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_LENGTH		(10)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA20_GB_OFFSET		(0x0858)
+
+/* PDP, GAMMA20_GB, GAMMA20_G
+*/
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_SHIFT		(16)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_LENGTH		(10)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA20_GB, GAMMA20_B
+*/
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_SHIFT		(0)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_LENGTH		(10)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA21_R_OFFSET		(0x085C)
+
+/* PDP, GAMMA21_R, GAMMA21_R
+*/
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_SHIFT		(0)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_LENGTH		(10)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA21_GB_OFFSET		(0x0860)
+
+/* PDP, GAMMA21_GB, GAMMA21_G
+*/
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_SHIFT		(16)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_LENGTH		(10)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA21_GB, GAMMA21_B
+*/
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_SHIFT		(0)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_LENGTH		(10)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA22_R_OFFSET		(0x0864)
+
+/* PDP, GAMMA22_R, GAMMA22_R
+*/
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_SHIFT		(0)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_LENGTH		(10)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA22_GB_OFFSET		(0x0868)
+
+/* PDP, GAMMA22_GB, GAMMA22_G
+*/
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_SHIFT		(16)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_LENGTH		(10)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA22_GB, GAMMA22_B
+*/
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_SHIFT		(0)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_LENGTH		(10)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA23_R_OFFSET		(0x086C)
+
+/* PDP, GAMMA23_R, GAMMA23_R
+*/
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_SHIFT		(0)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_LENGTH		(10)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA23_GB_OFFSET		(0x0870)
+
+/* PDP, GAMMA23_GB, GAMMA23_G
+*/
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_SHIFT		(16)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_LENGTH		(10)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA23_GB, GAMMA23_B
+*/
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_SHIFT		(0)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_LENGTH		(10)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA24_R_OFFSET		(0x0874)
+
+/* PDP, GAMMA24_R, GAMMA24_R
+*/
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_SHIFT		(0)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_LENGTH		(10)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA24_GB_OFFSET		(0x0878)
+
+/* PDP, GAMMA24_GB, GAMMA24_G
+*/
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_SHIFT		(16)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_LENGTH		(10)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA24_GB, GAMMA24_B
+*/
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_SHIFT		(0)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_LENGTH		(10)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA25_R_OFFSET		(0x087C)
+
+/* PDP, GAMMA25_R, GAMMA25_R
+*/
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_SHIFT		(0)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_LENGTH		(10)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA25_GB_OFFSET		(0x0880)
+
+/* PDP, GAMMA25_GB, GAMMA25_G
+*/
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_SHIFT		(16)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_LENGTH		(10)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA25_GB, GAMMA25_B
+*/
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_SHIFT		(0)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_LENGTH		(10)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA26_R_OFFSET		(0x0884)
+
+/* PDP, GAMMA26_R, GAMMA26_R
+*/
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_SHIFT		(0)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_LENGTH		(10)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA26_GB_OFFSET		(0x0888)
+
+/* PDP, GAMMA26_GB, GAMMA26_G
+*/
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_SHIFT		(16)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_LENGTH		(10)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA26_GB, GAMMA26_B
+*/
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_SHIFT		(0)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_LENGTH		(10)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA27_R_OFFSET		(0x088C)
+
+/* PDP, GAMMA27_R, GAMMA27_R
+*/
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_SHIFT		(0)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_LENGTH		(10)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA27_GB_OFFSET		(0x0890)
+
+/* PDP, GAMMA27_GB, GAMMA27_G
+*/
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_SHIFT		(16)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_LENGTH		(10)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA27_GB, GAMMA27_B
+*/
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_SHIFT		(0)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_LENGTH		(10)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA28_R_OFFSET		(0x0894)
+
+/* PDP, GAMMA28_R, GAMMA28_R
+*/
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_SHIFT		(0)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_LENGTH		(10)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA28_GB_OFFSET		(0x0898)
+
+/* PDP, GAMMA28_GB, GAMMA28_G
+*/
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_SHIFT		(16)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_LENGTH		(10)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA28_GB, GAMMA28_B
+*/
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_SHIFT		(0)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_LENGTH		(10)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA29_R_OFFSET		(0x089C)
+
+/* PDP, GAMMA29_R, GAMMA29_R
+*/
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_SHIFT		(0)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_LENGTH		(10)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA29_GB_OFFSET		(0x08A0)
+
+/* PDP, GAMMA29_GB, GAMMA29_G
+*/
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_SHIFT		(16)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_LENGTH		(10)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA29_GB, GAMMA29_B
+*/
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_SHIFT		(0)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_LENGTH		(10)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA30_R_OFFSET		(0x08A4)
+
+/* PDP, GAMMA30_R, GAMMA30_R
+*/
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_SHIFT		(0)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_LENGTH		(10)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA30_GB_OFFSET		(0x08A8)
+
+/* PDP, GAMMA30_GB, GAMMA30_G
+*/
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_SHIFT		(16)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_LENGTH		(10)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA30_GB, GAMMA30_B
+*/
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_SHIFT		(0)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_LENGTH		(10)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA31_R_OFFSET		(0x08AC)
+
+/* PDP, GAMMA31_R, GAMMA31_R
+*/
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_SHIFT		(0)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_LENGTH		(10)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA31_GB_OFFSET		(0x08B0)
+
+/* PDP, GAMMA31_GB, GAMMA31_G
+*/
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_SHIFT		(16)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_LENGTH		(10)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA31_GB, GAMMA31_B
+*/
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_SHIFT		(0)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_LENGTH		(10)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA32_R_OFFSET			(0x08B4)
+
+/* PDP, GAMMA32_R, GAMMA32_R
+*/
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_SHIFT		(0)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_LENGTH		(10)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA32_GB_OFFSET			(0x08B8)
+
+/* PDP, GAMMA32_GB, GAMMA32_G
+*/
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_SHIFT		(16)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_LENGTH		(10)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA32_GB, GAMMA32_B
+*/
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_SHIFT		(0)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_LENGTH		(10)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VEVENT_OFFSET				(0x08BC)
+
+/* PDP, VEVENT, VEVENT
+*/
+#define ODN_PDP_VEVENT_VEVENT_MASK			(0x1FFF0000)
+#define ODN_PDP_VEVENT_VEVENT_LSBMASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VEVENT_SHIFT			(16)
+#define ODN_PDP_VEVENT_VEVENT_LENGTH			(13)
+#define ODN_PDP_VEVENT_VEVENT_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VEVENT, VFETCH
+*/
+#define ODN_PDP_VEVENT_VFETCH_MASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VFETCH_LSBMASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VFETCH_SHIFT			(0)
+#define ODN_PDP_VEVENT_VFETCH_LENGTH			(13)
+#define ODN_PDP_VEVENT_VFETCH_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_HDECTRL_OFFSET				(0x08C0)
+
+/* PDP, HDECTRL, HDES
+*/
+#define ODN_PDP_HDECTRL_HDES_MASK		(0x1FFF0000)
+#define ODN_PDP_HDECTRL_HDES_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDES_SHIFT		(16)
+#define ODN_PDP_HDECTRL_HDES_LENGTH		(13)
+#define ODN_PDP_HDECTRL_HDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HDECTRL, HDEF
+*/
+#define ODN_PDP_HDECTRL_HDEF_MASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDEF_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDEF_SHIFT		(0)
+#define ODN_PDP_HDECTRL_HDEF_LENGTH		(13)
+#define ODN_PDP_HDECTRL_HDEF_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VDECTRL_OFFSET			(0x08C4)
+
+/* PDP, VDECTRL, VDES
+*/
+#define ODN_PDP_VDECTRL_VDES_MASK		(0x1FFF0000)
+#define ODN_PDP_VDECTRL_VDES_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDES_SHIFT		(16)
+#define ODN_PDP_VDECTRL_VDES_LENGTH		(13)
+#define ODN_PDP_VDECTRL_VDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VDECTRL, VDEF
+*/
+#define ODN_PDP_VDECTRL_VDEF_MASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDEF_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDEF_SHIFT		(0)
+#define ODN_PDP_VDECTRL_VDEF_LENGTH		(13)
+#define ODN_PDP_VDECTRL_VDEF_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_OPMASK_R_OFFSET			(0x08C8)
+
+/* PDP, OPMASK_R, MASKLEVEL
+*/
+#define ODN_PDP_OPMASK_R_MASKLEVEL_MASK		(0x80000000)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_LSBMASK		(0x00000001)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_SHIFT		(31)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_LENGTH		(1)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, BLANKLEVEL
+*/
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_MASK		(0x40000000)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_LSBMASK		(0x00000001)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_SHIFT		(30)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_LENGTH		(1)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, MASKR
+*/
+#define ODN_PDP_OPMASK_R_MASKR_MASK		(0x000003FF)
+#define ODN_PDP_OPMASK_R_MASKR_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_R_MASKR_SHIFT		(0)
+#define ODN_PDP_OPMASK_R_MASKR_LENGTH		(10)
+#define ODN_PDP_OPMASK_R_MASKR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_OPMASK_GB_OFFSET		(0x08CC)
+
+/* PDP, OPMASK_GB, MASKG
+*/
+#define ODN_PDP_OPMASK_GB_MASKG_MASK		(0x03FF0000)
+#define ODN_PDP_OPMASK_GB_MASKG_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKG_SHIFT		(16)
+#define ODN_PDP_OPMASK_GB_MASKG_LENGTH		(10)
+#define ODN_PDP_OPMASK_GB_MASKG_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_GB, MASKB
+*/
+#define ODN_PDP_OPMASK_GB_MASKB_MASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKB_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKB_SHIFT		(0)
+#define ODN_PDP_OPMASK_GB_MASKB_LENGTH		(10)
+#define ODN_PDP_OPMASK_GB_MASKB_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_ADDR_CTRL_OFFSET		(0x08D0)
+
+/* PDP, REGLD_ADDR_CTRL, REGLD_ADDRIN
+*/
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_MASK		(0xFFFFFFF0)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LSBMASK		(0x0FFFFFFF)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SHIFT		(4)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LENGTH		(28)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_ADDR_STAT_OFFSET		(0x08D4)
+
+/* PDP, REGLD_ADDR_STAT, REGLD_ADDROUT
+*/
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_MASK		(0xFFFFFFF0)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LSBMASK		(0x0FFFFFFF)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SHIFT		(4)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LENGTH		(28)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_STAT_OFFSET		(0x08D8)
+
+/* PDP, REGLD_STAT, REGLD_ADDREN
+*/
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_MASK		(0x00800000)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_LSBMASK		(0x00000001)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_SHIFT		(23)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_LENGTH		(1)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_CTRL_OFFSET		(0x08DC)
+
+/* PDP, REGLD_CTRL, REGLD_ADDRLEN
+*/
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_MASK		(0xFF000000)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_LSBMASK	(0x000000FF)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_SHIFT		(24)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_LENGTH		(8)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGLD_CTRL, REGLD_VAL
+*/
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_MASK		(0x00800000)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_LSBMASK		(0x00000001)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_SHIFT		(23)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_LENGTH		(1)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_UPDCTRL_OFFSET			(0x08E0)
+
+/* PDP, UPDCTRL, UPDFIELD
+*/
+#define ODN_PDP_UPDCTRL_UPDFIELD_MASK		(0x00000001)
+#define ODN_PDP_UPDCTRL_UPDFIELD_LSBMASK	(0x00000001)
+#define ODN_PDP_UPDCTRL_UPDFIELD_SHIFT		(0)
+#define ODN_PDP_UPDCTRL_UPDFIELD_LENGTH		(1)
+#define ODN_PDP_UPDCTRL_UPDFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTCTRL_OFFSET			(0x08E4)
+
+/* PDP, PVR_ODN_PDP_INTCTRL, HBLNK_LINE
+*/
+#define ODN_PDP_INTCTRL_HBLNK_LINE_MASK		(0x00010000)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_LSBMASK	(0x00000001)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_SHIFT	(16)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_LENGTH	(1)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_INTCTRL, HBLNK_LINENO
+*/
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_MASK	(0x00001FFF)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_LSBMASK	(0x00001FFF)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_SHIFT	(0)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_LENGTH	(13)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_PDISETUP_OFFSET		(0x0900)
+
+/* PDP, PDISETUP, PDI_BLNKLVL
+*/
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_MASK		(0x00000040)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_SHIFT		(6)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_BLNK
+*/
+#define ODN_PDP_PDISETUP_PDI_BLNK_MASK		(0x00000020)
+#define ODN_PDP_PDISETUP_PDI_BLNK_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_BLNK_SHIFT		(5)
+#define ODN_PDP_PDISETUP_PDI_BLNK_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_BLNK_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_PWR
+*/
+#define ODN_PDP_PDISETUP_PDI_PWR_MASK		(0x00000010)
+#define ODN_PDP_PDISETUP_PDI_PWR_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_PWR_SHIFT		(4)
+#define ODN_PDP_PDISETUP_PDI_PWR_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_PWR_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_EN
+*/
+#define ODN_PDP_PDISETUP_PDI_EN_MASK		(0x00000008)
+#define ODN_PDP_PDISETUP_PDI_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_EN_SHIFT		(3)
+#define ODN_PDP_PDISETUP_PDI_EN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_GDEN
+*/
+#define ODN_PDP_PDISETUP_PDI_GDEN_MASK		(0x00000004)
+#define ODN_PDP_PDISETUP_PDI_GDEN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_GDEN_SHIFT		(2)
+#define ODN_PDP_PDISETUP_PDI_GDEN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_GDEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_NFEN
+*/
+#define ODN_PDP_PDISETUP_PDI_NFEN_MASK		(0x00000002)
+#define ODN_PDP_PDISETUP_PDI_NFEN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_NFEN_SHIFT		(1)
+#define ODN_PDP_PDISETUP_PDI_NFEN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_NFEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_CR
+*/
+#define ODN_PDP_PDISETUP_PDI_CR_MASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_CR_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_CR_SHIFT		(0)
+#define ODN_PDP_PDISETUP_PDI_CR_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_CR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDITIMING0_OFFSET		(0x0904)
+
+/* PDP, PDITIMING0, PDI_PWRSVGD
+*/
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_MASK		(0x0F000000)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_LSBMASK		(0x0000000F)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_SHIFT		(24)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_LENGTH		(4)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_LSDEL
+*/
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_MASK		(0x007F0000)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_LSBMASK		(0x0000007F)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_SHIFT		(16)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_LENGTH		(7)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_PWRSV2GD2
+*/
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_MASK		(0x000003FF)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_LSBMASK		(0x000003FF)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_SHIFT		(0)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_LENGTH		(10)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDITIMING1_OFFSET		(0x0908)
+
+/* PDP, PDITIMING1, PDI_NLDEL
+*/
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_MASK		(0x000F0000)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_LSBMASK		(0x0000000F)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_SHIFT		(16)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_LENGTH		(4)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING1, PDI_ACBDEL
+*/
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_MASK		(0x000003FF)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_LSBMASK		(0x000003FF)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_SHIFT		(0)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_LENGTH		(10)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDICOREID_OFFSET		(0x090C)
+
+/* PDP, PDICOREID, PDI_GROUP_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_MASK		(0xFF000000)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_SHIFT		(24)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_LENGTH		(8)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CORE_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_MASK		(0x00FF0000)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_SHIFT		(16)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_LENGTH		(8)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CONFIG_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_MASK		(0x0000FFFF)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_SHIFT		(0)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_LENGTH		(16)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDICOREREV_OFFSET		(0x0910)
+
+/* PDP, PDICOREREV, PDI_MAJOR_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_MASK		(0x00FF0000)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_SHIFT		(16)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MINOR_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_MASK		(0x0000FF00)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_SHIFT		(8)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MAINT_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_MASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_SHIFT		(0)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX2_OFFSET		(0x0920)
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_MASK		(0x000000C0)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_MASK		(0x00000030)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_MASK		(0x0000000C)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SHIFT		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_MASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX4_0_OFFSET		(0x0924)
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_MASK		(0xF0000000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SHIFT		(28)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_MASK		(0x0F000000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_MASK		(0x00F00000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SHIFT		(20)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_MASK		(0x000F0000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SHIFT		(16)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_MASK		(0x0000F000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_MASK		(0x00000F00)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SHIFT		(8)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_MASK		(0x000000F0)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_MASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX4_1_OFFSET		(0x0928)
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_MASK		(0xF0000000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SHIFT		(28)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_MASK		(0x0F000000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_MASK		(0x00F00000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SHIFT		(20)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_MASK		(0x000F0000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SHIFT		(16)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_MASK		(0x0000F000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_MASK		(0x00000F00)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SHIFT		(8)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_MASK		(0x000000F0)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_MASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_0_OFFSET		(0x092C)
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X4Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X3Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X2Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_1_OFFSET		(0x0930)
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X7Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X6Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X5Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_2_OFFSET		(0x0934)
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X6Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X5Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X4Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X3Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X2Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_3_OFFSET		(0x0938)
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X3Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X2Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X1Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X0Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X7Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_4_OFFSET		(0x093C)
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X0Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X7Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X6Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X5Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X4Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_5_OFFSET		(0x0940)
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X5Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X4Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X3Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X2Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X1Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_6_OFFSET		(0x0944)
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X2Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X1Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X0Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X7Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X6Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_7_OFFSET		(0x0948)
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X7Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X6Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X5Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X4Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X3Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_8_OFFSET		(0x094C)
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X4Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X3Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X2Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X1Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X0Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_9_OFFSET		(0x0950)
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X1Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X0Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X7Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X6Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X5Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_10_OFFSET		(0x0954)
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X6Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X5Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X4Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X3Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X2Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_11_OFFSET		(0x0958)
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X3Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X2Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X1Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X0Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X7Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_12_OFFSET		(0x095C)
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X7Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X6Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X5Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X4Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_MEMCTRL_OFFSET		(0x0960)
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_BURSTLEN
+*/
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_MEM_THRESH_OFFSET		(0x0964)
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_THRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_MEMCTRL_OFFSET		(0x0968)
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_BURSTLEN
+*/
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_MEM_THRESH_OFFSET		(0x096C)
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_THRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_MEMCTRL_OFFSET		(0x0970)
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_BURSTLEN
+*/
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_MEM_THRESH_OFFSET		(0x0974)
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_THRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_MEMCTRL_OFFSET		(0x0978)
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_BURSTLEN
+*/
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_MEM_THRESH_OFFSET		(0x097C)
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_THRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_MEMCTRL_OFFSET		(0x0980)
+
+/* PDP, VID1_MEMCTRL, VID1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEMCTRL, VID1_BURSTLEN
+*/
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_MEM_THRESH_OFFSET		(0x0984)
+
+/* PDP, VID1_MEM_THRESH, VID1_UVTHRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_YTHRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_THRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_MEMCTRL_OFFSET		(0x0988)
+
+/* PDP, VID2_MEMCTRL, VID2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEMCTRL, VID2_BURSTLEN
+*/
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_MEM_THRESH_OFFSET		(0x098C)
+
+/* PDP, VID2_MEM_THRESH, VID2_UVTHRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_YTHRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_THRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_MEMCTRL_OFFSET		(0x0990)
+
+/* PDP, VID3_MEMCTRL, VID3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEMCTRL, VID3_BURSTLEN
+*/
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_MEM_THRESH_OFFSET		(0x0994)
+
+/* PDP, VID3_MEM_THRESH, VID3_UVTHRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_YTHRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_THRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_MEMCTRL_OFFSET		(0x0998)
+
+/* PDP, VID4_MEMCTRL, VID4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEMCTRL, VID4_BURSTLEN
+*/
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_MEM_THRESH_OFFSET		(0x099C)
+
+/* PDP, VID4_MEM_THRESH, VID4_UVTHRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_YTHRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_THRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_PANIC_THRESH_OFFSET		(0x09A0)
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_PANIC_THRESH_OFFSET		(0x09A4)
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_PANIC_THRESH_OFFSET		(0x09A8)
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_PANIC_THRESH_OFFSET		(0x09AC)
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_PANIC_THRESH_OFFSET		(0x09B0)
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_PANIC_THRESH_OFFSET		(0x09B4)
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_PANIC_THRESH_OFFSET		(0x09B8)
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_PANIC_THRESH_OFFSET		(0x09BC)
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BURST_BOUNDARY_OFFSET		(0x09C0)
+
+/* PDP, BURST_BOUNDARY, BURST_BOUNDARY
+*/
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_MASK		(0x0000003F)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_LSBMASK		(0x0000003F)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_SHIFT		(0)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_LENGTH		(6)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_SIGNED_FIELD	IMG_FALSE
+
+
+/* ---------------------- End of register definitions ---------------------- */
+
+/* NUMREG defines the extent of register address space.
+*/
+
+#define		ODN_PDP_NUMREG	   ((0x09C0 >> 2)+1)
+
+/* Info about video plane addresses */
+#define ODN_PDP_YADDR_BITS		(ODN_PDP_VID1BASEADDR_VID1BASEADDR_LENGTH)
+#define ODN_PDP_YADDR_ALIGN		5
+#define ODN_PDP_UADDR_BITS		(ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH)
+#define ODN_PDP_UADDR_ALIGN		5
+#define ODN_PDP_VADDR_BITS		(ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH)
+#define ODN_PDP_VADDR_ALIGN		5
+
+#define ODN_PDP_YSTRIDE_BITS	(ODN_PDP_VID1STRIDE_VID1STRIDE_LENGTH)
+#define ODN_PDP_YSTRIDE_ALIGN	5
+
+#define ODN_PDP_MAX_INPUT_WIDTH (ODN_PDP_VID1SIZE_VID1WIDTH_LSBMASK + 1)
+#define ODN_PDP_MAX_INPUT_HEIGHT (ODN_PDP_VID1SIZE_VID1HEIGHT_LSBMASK + 1)
+
+/* Maximum 6 bytes per pixel for RGB161616 */
+#define ODN_PDP_MAX_IMAGE_BYTES (ODN_PDP_MAX_INPUT_WIDTH * ODN_PDP_MAX_INPUT_HEIGHT * 6)
+
+/* Round up */
+#define ODN_PDP_MAX_IMAGE_PAGES ((ODN_PDP_MAX_IMAGE_BYTES+PAGE_SIZE-1)/PAGE_SIZE)
+
+#define ODN_PDP_YADDR_MAX		(((1 << ODN_PDP_YADDR_BITS) - 1) << ODN_PDP_YADDR_ALIGN)
+#define ODN_PDP_UADDR_MAX		(((1 << ODN_PDP_UADDR_BITS) - 1) << ODN_PDP_UADDR_ALIGN)
+#define ODN_PDP_VADDR_MAX		(((1 << ODN_PDP_VADDR_BITS) - 1) << ODN_PDP_VADDR_ALIGN)
+#define ODN_PDP_YSTRIDE_MAX		((1 << ODN_PDP_YSTRIDE_BITS) << ODN_PDP_YSTRIDE_ALIGN)
+#define ODN_PDP_YADDR_ALIGNMASK		((1 << ODN_PDP_YADDR_ALIGN) - 1)
+#define ODN_PDP_UADDR_ALIGNMASK		((1 << ODN_PDP_UADDR_ALIGN) - 1)
+#define ODN_PDP_VADDR_ALIGNMASK		((1 << ODN_PDP_VADDR_ALIGN) - 1)
+#define ODN_PDP_YSTRIDE_ALIGNMASK	((1 << ODN_PDP_YSTRIDE_ALIGN) - 1)
+
+/* Field Values (some are reserved for future use) */
+#define ODN_PDP_SURF_PIXFMT_RGB332					0x3
+#define ODN_PDP_SURF_PIXFMT_ARGB4444				0x4
+#define ODN_PDP_SURF_PIXFMT_ARGB1555				0x5
+#define ODN_PDP_SURF_PIXFMT_RGB888					0x6
+#define ODN_PDP_SURF_PIXFMT_RGB565					0x7
+#define ODN_PDP_SURF_PIXFMT_ARGB8888				0x8
+#define ODN_PDP_SURF_PIXFMT_420_PL8					0x9
+#define ODN_PDP_SURF_PIXFMT_420_PL8IVU				0xA
+#define ODN_PDP_SURF_PIXFMT_420_PL8IUV				0xB
+#define ODN_PDP_SURF_PIXFMT_422_UY0VY1_8888			0xC
+#define ODN_PDP_SURF_PIXFMT_422_VY0UY1_8888			0xD
+#define ODN_PDP_SURF_PIXFMT_422_Y0UY1V_8888			0xE
+#define ODN_PDP_SURF_PIXFMT_422_Y0VY1U_8888			0xF
+#define ODN_PDP_SURF_PIXFMT_AYUV8888				0x10
+#define ODN_PDP_SURF_PIXFMT_YUV101010				0x15
+#define ODN_PDP_SURF_PIXFMT_RGB101010				0x17
+#define ODN_PDP_SURF_PIXFMT_420_PL10IUV				0x18
+#define ODN_PDP_SURF_PIXFMT_420_PL10IVU				0x19
+#define ODN_PDP_SURF_PIXFMT_422_PL10IUV				0x1A
+#define ODN_PDP_SURF_PIXFMT_422_PL10IVU				0x1B
+#define ODN_PDP_SURF_PIXFMT_RGB121212				0x1E
+#define ODN_PDP_SURF_PIXFMT_RGB161616				0x1F
+
+#define ODN_PDP_CTRL_CKEYSRC_PREV					0x0
+#define ODN_PDP_CTRL_CKEYSRC_CUR					0x1
+
+#define ODN_PDP_MEMCTRL_MEMREFRESH_ALWAYS			0x0
+#define ODN_PDP_MEMCTRL_MEMREFRESH_HBLNK			0x1
+#define ODN_PDP_MEMCTRL_MEMREFRESH_VBLNK			0x2
+#define ODN_PDP_MEMCTRL_MEMREFRESH_BOTH				0x3
+
+#define ODN_PDP_3D_CTRL_BLENDSEL_BGND_WITH_POS0		0x0
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS0_WITH_POS1		0x1
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS1_WITH_POS2		0x2
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS2_WITH_POS3		0x3
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS3_WITH_POS4		0x4
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS4_WITH_POS5		0x5
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS5_WITH_POS6		0x6
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS6_WITH_POS7		0x7
+
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_Y_STRIDE	0x0
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_DOUBLE_Y_STRIDE 0x1
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_HALF_Y_STRIDE 0x2
+
+#define ODN_PDP_PROCAMP_OUTPUT_OFFSET_FRACTIONAL_BITS 1
+#define ODN_PDP_PROCAMP_COEFFICIENT_FRACTIONAL_BITS	10
+
+/*---------------------------------------------------------------------------*/
+
+#endif /* ODN_PDP_REGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_regs.h
new file mode 100644
index 000000000000..5d5821623ac7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/odin_regs.h
@@ -0,0 +1,1026 @@
+/******************************************************************************
+@Title          Odin system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin FPGA register defs for IMG 3rd generation TCF
+
+	Auto generated headers, eg. odn_core.h:
+		regconv -d . -a 8 odn_core.def
+
+	Source files :
+		odn_core.def
+		mca_debug.def
+		sai_rx_debug.def
+		sai_tx_debug.def
+		ad_tx.def
+
+	Changes:
+		Removed obsolete copyright dates
+		Changed lower case to upper case
+			(eg. odn_core changed to ODN_CORE)
+		Changed PVR5__ to ODN_
+		Merged multiple .def files into one header
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+/* tab size 4 */
+
+#ifndef _ODIN_REGS_H_
+#define _ODIN_REGS_H_
+
+/******************************
+  Generated from: odn_core.def
+*******************************/
+
+/*
+	Register ID
+*/
+#define ODN_CORE_ID                             0x0000
+#define ODN_ID_VARIANT_MASK                     0x0000FFFFU
+#define ODN_ID_VARIANT_SHIFT                    0
+#define ODN_ID_VARIANT_SIGNED                   0
+
+#define ODN_ID_ID_MASK                          0xFFFF0000U
+#define ODN_ID_ID_SHIFT                         16
+#define ODN_ID_ID_SIGNED                        0
+
+/*
+	Register REL
+*/
+#define ODN_CORE_REL                            0x0004
+#define ODN_REL_MINOR_MASK                      0x0000FFFFU
+#define ODN_REL_MINOR_SHIFT                     0
+#define ODN_REL_MINOR_SIGNED                    0
+
+#define ODN_REL_MAJOR_MASK                      0xFFFF0000U
+#define ODN_REL_MAJOR_SHIFT                     16
+#define ODN_REL_MAJOR_SIGNED                    0
+
+/*
+	Register CHANGE_SET
+*/
+#define ODN_CORE_CHANGE_SET                     0x0008
+#define ODN_CHANGE_SET_SET_MASK                 0xFFFFFFFFU
+#define ODN_CHANGE_SET_SET_SHIFT                0
+#define ODN_CHANGE_SET_SET_SIGNED               0
+
+/*
+	Register USER_ID
+*/
+#define ODN_CORE_USER_ID                        0x000C
+#define ODN_USER_ID_ID_MASK                     0x000000FFU
+#define ODN_USER_ID_ID_SHIFT                    0
+#define ODN_USER_ID_ID_SIGNED                   0
+
+/*
+	Register USER_BUILD
+*/
+#define ODN_CORE_USER_BUILD                     0x0010
+#define ODN_USER_BUILD_BUILD_MASK               0xFFFFFFFFU
+#define ODN_USER_BUILD_BUILD_SHIFT              0
+#define ODN_USER_BUILD_BUILD_SIGNED             0
+
+/*
+	Register SW_IF_VERSION
+*/
+#define ODN_CORE_SW_IF_VERSION                  0x0014
+#define ODN_SW_IF_VERSION_VERSION_MASK          0x0000FFFFU
+#define ODN_SW_IF_VERSION_VERSION_SHIFT         0
+#define ODN_SW_IF_VERSION_VERSION_SIGNED        0
+
+/*
+	Register INTERNAL_RESETN
+*/
+#define ODN_CORE_INTERNAL_RESETN                0x0080
+#define ODN_INTERNAL_RESETN_DDR_MASK            0x00000001U
+#define ODN_INTERNAL_RESETN_DDR_SHIFT           0
+#define ODN_INTERNAL_RESETN_DDR_SIGNED          0
+
+#define ODN_INTERNAL_RESETN_MIG0_MASK           0x00000002U
+#define ODN_INTERNAL_RESETN_MIG0_SHIFT          1
+#define ODN_INTERNAL_RESETN_MIG0_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_MIG1_MASK           0x00000004U
+#define ODN_INTERNAL_RESETN_MIG1_SHIFT          2
+#define ODN_INTERNAL_RESETN_MIG1_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PDP1_MASK           0x00000008U
+#define ODN_INTERNAL_RESETN_PDP1_SHIFT          3
+#define ODN_INTERNAL_RESETN_PDP1_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PDP2_MASK           0x00000010U
+#define ODN_INTERNAL_RESETN_PDP2_SHIFT          4
+#define ODN_INTERNAL_RESETN_PDP2_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PERIP_MASK          0x00000020U
+#define ODN_INTERNAL_RESETN_PERIP_SHIFT         5
+#define ODN_INTERNAL_RESETN_PERIP_SIGNED        0
+
+#define ODN_INTERNAL_RESETN_GIST_MASK           0x00000040U
+#define ODN_INTERNAL_RESETN_GIST_SHIFT          6
+#define ODN_INTERNAL_RESETN_GIST_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PIKE_MASK           0x00000080U
+#define ODN_INTERNAL_RESETN_PIKE_SHIFT          7
+#define ODN_INTERNAL_RESETN_PIKE_SIGNED         0
+
+/*
+	Register EXTERNAL_RESETN
+*/
+#define ODN_CORE_EXTERNAL_RESETN                0x0084
+#define ODN_EXTERNAL_RESETN_DUT_MASK            0x00000001U
+#define ODN_EXTERNAL_RESETN_DUT_SHIFT           0
+#define ODN_EXTERNAL_RESETN_DUT_SIGNED          0
+
+#define ODN_EXTERNAL_RESETN_DUT_SPI_MASK        0x00000002U
+#define ODN_EXTERNAL_RESETN_DUT_SPI_SHIFT       1
+#define ODN_EXTERNAL_RESETN_DUT_SPI_SIGNED      0
+
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_MASK    0x00000004U
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_SHIFT   2
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_SIGNED  0
+
+#define ODN_EXTERNAL_RESETN_DUT_IF_MASK         0x00000008U
+#define ODN_EXTERNAL_RESETN_DUT_IF_SHIFT        3
+#define ODN_EXTERNAL_RESETN_DUT_IF_SIGNED       0
+
+#define ODN_EXTERNAL_RESETN_DUT1_MASK           0x00000010U
+#define ODN_EXTERNAL_RESETN_DUT1_SHIFT          4
+#define ODN_EXTERNAL_RESETN_DUT1_SIGNED         0
+
+#define ODN_EXTERNAL_RESETN_DUT2_MASK           0x00000020U
+#define ODN_EXTERNAL_RESETN_DUT2_SHIFT          5
+#define ODN_EXTERNAL_RESETN_DUT2_SIGNED         0
+
+/*
+	Register EXTERNAL_RESET
+*/
+#define ODN_CORE_EXTERNAL_RESET                 0x0088
+#define ODN_EXTERNAL_RESET_PVT_CAL_MASK         0x00000001U
+#define ODN_EXTERNAL_RESET_PVT_CAL_SHIFT        0
+#define ODN_EXTERNAL_RESET_PVT_CAL_SIGNED       0
+
+#define ODN_EXTERNAL_RESET_PLL_MASK             0x00000002U
+#define ODN_EXTERNAL_RESET_PLL_SHIFT            1
+#define ODN_EXTERNAL_RESET_PLL_SIGNED           0
+
+/*
+	Register INTERNAL_AUTO_RESETN
+*/
+#define ODN_CORE_INTERNAL_AUTO_RESETN           0x008C
+#define ODN_INTERNAL_AUTO_RESETN_AUX_MASK       0x00000001U
+#define ODN_INTERNAL_AUTO_RESETN_AUX_SHIFT      0
+#define ODN_INTERNAL_AUTO_RESETN_AUX_SIGNED     0
+
+/*
+	Register CLK_GEN_RESET
+*/
+#define ODN_CORE_CLK_GEN_RESET                  0x0090
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_MASK    0x00000001U
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_SHIFT   0
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_SIGNED  0
+
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_MASK      0x00000002U
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_SHIFT     1
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_SIGNED    0
+
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_MASK       0x00000004U
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_SHIFT      2
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_SIGNED     0
+
+#define ODN_CLK_GEN_RESET_PDP_MMCM_MASK         0x00000008U
+#define ODN_CLK_GEN_RESET_PDP_MMCM_SHIFT        3
+#define ODN_CLK_GEN_RESET_PDP_MMCM_SIGNED       0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define ODN_CORE_INTERRUPT_STATUS               0x0100
+#define ODN_INTERRUPT_STATUS_DUT_MASK           0x00000001U
+#define ODN_INTERRUPT_STATUS_DUT_SHIFT          0
+#define ODN_INTERRUPT_STATUS_DUT_SIGNED         0
+
+#define ODN_INTERRUPT_STATUS_PDP1_MASK          0x00000002U
+#define ODN_INTERRUPT_STATUS_PDP1_SHIFT         1
+#define ODN_INTERRUPT_STATUS_PDP1_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_PDP2_MASK          0x00000004U
+#define ODN_INTERRUPT_STATUS_PDP2_SHIFT         2
+#define ODN_INTERRUPT_STATUS_PDP2_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_PERIP_MASK         0x00000008U
+#define ODN_INTERRUPT_STATUS_PERIP_SHIFT        3
+#define ODN_INTERRUPT_STATUS_PERIP_SIGNED       0
+
+#define ODN_INTERRUPT_STATUS_UART_MASK          0x00000010U
+#define ODN_INTERRUPT_STATUS_UART_SHIFT         4
+#define ODN_INTERRUPT_STATUS_UART_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_MASK 0x00000020U
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_MASK 0x00000040U
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_SHIFT 6
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_MASK 0x00000100U
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_DUT2_MASK          0x00000200U
+#define ODN_INTERRUPT_STATUS_DUT2_SHIFT         9
+#define ODN_INTERRUPT_STATUS_DUT2_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_CDMA_MASK          0x00001800U
+#define ODN_INTERRUPT_STATUS_CDMA_SHIFT         11
+#define ODN_INTERRUPT_STATUS_CDMA_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_OS_IRQ_MASK        0x001FE000U
+#define ODN_INTERRUPT_STATUS_OS_IRQ_SHIFT       13
+#define ODN_INTERRUPT_STATUS_OS_IRQ_SIGNED      0
+
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_MASK      0x40000000U
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_SHIFT     30
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_SIGNED    0
+
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_MASK 0x80000000U
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_SHIFT 31
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_SIGNED 0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define ODN_CORE_INTERRUPT_ENABLE               0x0104
+#define ODN_INTERRUPT_ENABLE_DUT_MASK           0x00000001U
+#define ODN_INTERRUPT_ENABLE_DUT_SHIFT          0
+#define ODN_INTERRUPT_ENABLE_DUT_SIGNED         0
+
+#define ODN_INTERRUPT_ENABLE_PDP1_MASK          0x00000002U
+#define ODN_INTERRUPT_ENABLE_PDP1_SHIFT         1
+#define ODN_INTERRUPT_ENABLE_PDP1_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_PDP2_MASK          0x00000004U
+#define ODN_INTERRUPT_ENABLE_PDP2_SHIFT         2
+#define ODN_INTERRUPT_ENABLE_PDP2_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_PERIP_MASK         0x00000008U
+#define ODN_INTERRUPT_ENABLE_PERIP_SHIFT        3
+#define ODN_INTERRUPT_ENABLE_PERIP_SIGNED       0
+
+#define ODN_INTERRUPT_ENABLE_UART_MASK          0x00000010U
+#define ODN_INTERRUPT_ENABLE_UART_SHIFT         4
+#define ODN_INTERRUPT_ENABLE_UART_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_MASK 0x00000020U
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_MASK 0x00000040U
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_SHIFT 6
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_MASK 0x00000100U
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_DUT2_MASK          0x00000200U
+#define ODN_INTERRUPT_ENABLE_DUT2_SHIFT         9
+#define ODN_INTERRUPT_ENABLE_DUT2_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_CDMA_MASK          0x00001800U
+#define ODN_INTERRUPT_ENABLE_CDMA_SHIFT         11
+#define ODN_INTERRUPT_ENABLE_CDMA_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_MASK        0x001FE000U
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_SHIFT       13
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_SIGNED      0
+
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_MASK      0x40000000U
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_SHIFT     30
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_SIGNED    0
+
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_MASK 0x80000000U
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_SHIFT 31
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_SIGNED 0
+
+/*
+	Register INTERRUPT_CLR
+*/
+#define ODN_CORE_INTERRUPT_CLR                  0x010C
+#define ODN_INTERRUPT_CLR_DUT_MASK              0x00000001U
+#define ODN_INTERRUPT_CLR_DUT_SHIFT             0
+#define ODN_INTERRUPT_CLR_DUT_SIGNED            0
+
+#define ODN_INTERRUPT_CLR_PDP1_MASK             0x00000002U
+#define ODN_INTERRUPT_CLR_PDP1_SHIFT            1
+#define ODN_INTERRUPT_CLR_PDP1_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_PDP2_MASK             0x00000004U
+#define ODN_INTERRUPT_CLR_PDP2_SHIFT            2
+#define ODN_INTERRUPT_CLR_PDP2_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_PERIP_MASK            0x00000008U
+#define ODN_INTERRUPT_CLR_PERIP_SHIFT           3
+#define ODN_INTERRUPT_CLR_PERIP_SIGNED          0
+
+#define ODN_INTERRUPT_CLR_UART_MASK             0x00000010U
+#define ODN_INTERRUPT_CLR_UART_SHIFT            4
+#define ODN_INTERRUPT_CLR_UART_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_MASK  0x00000020U
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_MASK   0x00000040U
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_SHIFT  6
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_MASK  0x00000100U
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_DUT2_MASK             0x00000200U
+#define ODN_INTERRUPT_CLR_DUT2_SHIFT            9
+#define PVR5__INTERRUPT_CLR_DUT2_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_CDMA_MASK             0x00001800U
+#define ODN_INTERRUPT_CLR_CDMA_SHIFT            11
+#define ODN_INTERRUPT_CLR_CDMA_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_OS_IRQ_MASK           0x001FE000U
+#define ODN_INTERRUPT_CLR_OS_IRQ_SHIFT          13
+#define ODN_INTERRUPT_CLR_OS_IRQ_SIGNED         0
+
+#define ODN_INTERRUPT_CLR_IRQ_TEST_MASK         0x40000000U
+#define ODN_INTERRUPT_CLR_IRQ_TEST_SHIFT        30
+#define ODN_INTERRUPT_CLR_IRQ_TEST_SIGNED       0
+
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_MASK     0x80000000U
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_SHIFT    31
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_SIGNED   0
+
+/*
+	Register INTERRUPT_TEST
+*/
+#define ODN_CORE_INTERRUPT_TEST                 0x0110
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_MASK  0x00000001U
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_SHIFT 0
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT_CLR
+*/
+#define ODN_CORE_INTERRUPT_TIMEOUT_CLR      0x0114
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_MASK 0x00000002U
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SHIFT 1
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SIGNED 0
+
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_MASK 0x00000001U
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SHIFT 0
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT
+*/
+#define ODN_CORE_INTERRUPT_TIMEOUT          0x0118
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_MASK 0xFFFFFFFFU
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SHIFT 0
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SIGNED 0
+/*
+	Register SYSTEM_ID
+*/
+#define ODN_CORE_SYSTEM_ID                      0x011C
+#define ODN_SYSTEM_ID_ID_MASK                   0x0000FFFFU
+#define ODN_SYSTEM_ID_ID_SHIFT                  0
+#define ODN_SYSTEM_ID_ID_SIGNED                 0
+
+/*
+	Register SUPPORTED_FEATURES
+*/
+#define ODN_CORE_SUPPORTED_FEATURES             0x0120
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_MASK 0xFFFFFFFEU
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_SHIFT 1
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_SIGNED 0
+
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_MASK 0x00000001U
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_SHIFT 0
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_SIGNED 0
+
+/*
+	Register NUM_GPIO
+*/
+#define ODN_CORE_NUM_GPIO                       0x0180
+#define ODN_NUM_GPIO_NUMBER_MASK                0x0000000FU
+#define ODN_NUM_GPIO_NUMBER_SHIFT               0
+#define ODN_NUM_GPIO_NUMBER_SIGNED              0
+
+/*
+	Register GPIO_EN
+*/
+#define ODN_CORE_GPIO_EN                        0x0184
+#define ODN_GPIO_EN_DIRECTION_MASK              0x000000FFU
+#define ODN_GPIO_EN_DIRECTION_SHIFT             0
+#define ODN_GPIO_EN_DIRECTION_SIGNED            0
+
+/*
+	Register GPIO
+*/
+#define ODN_CORE_GPIO                           0x0188
+#define ODN_GPIO_GPIO_MASK                      0x000000FFU
+#define ODN_GPIO_GPIO_SHIFT                     0
+#define ODN_GPIO_GPIO_SIGNED                    0
+
+/*
+	Register NUM_DUT_CTRL
+*/
+#define ODN_CORE_NUM_DUT_CTRL                   0x0190
+#define ODN_NUM_DUT_CTRL_NUM_PINS_MASK          0xFFFFFFFFU
+#define ODN_NUM_DUT_CTRL_NUM_PINS_SHIFT         0
+#define ODN_NUM_DUT_CTRL_NUM_PINS_SIGNED        0
+
+/*
+	Register DUT_CTRL1
+*/
+#define ODN_CORE_DUT_CTRL1                      0x0194
+#define ODN_DUT_CTRL1_CONTROL1_MASK             0x3FFFFFFFU
+#define ODN_DUT_CTRL1_CONTROL1_SHIFT            0
+#define ODN_DUT_CTRL1_CONTROL1_SIGNED           0
+
+#define ODN_DUT_CTRL1_FBDC_BYPASS_MASK          0x40000000U
+#define ODN_DUT_CTRL1_FBDC_BYPASS_SHIFT         30
+#define ODN_DUT_CTRL1_FBDC_BYPASS_SIGNED        0
+
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_MASK       0x80000000U
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_SHIFT      31
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_SIGNED     0
+
+/*
+	Register DUT_CTRL2
+*/
+#define ODN_CORE_DUT_CTRL2                      0x0198
+#define ODN_DUT_CTRL2_CONTROL2_MASK             0xFFFFFFFFU
+#define ODN_DUT_CTRL2_CONTROL2_SHIFT            0
+#define ODN_DUT_CTRL2_CONTROL2_SIGNED           0
+
+/*
+	Register NUM_DUT_STAT
+*/
+#define ODN_CORE_NUM_DUT_STAT                   0x019C
+#define ODN_NUM_DUT_STAT_NUM_PINS_MASK          0xFFFFFFFFU
+#define ODN_NUM_DUT_STAT_NUM_PINS_SHIFT         0
+#define ODN_NUM_DUT_STAT_NUM_PINS_SIGNED        0
+
+/*
+	Register DUT_STAT1
+*/
+#define ODN_CORE_DUT_STAT1                      0x01A0
+#define ODN_DUT_STAT1_STATUS1_MASK              0xFFFFFFFFU
+#define ODN_DUT_STAT1_STATUS1_SHIFT             0
+#define ODN_DUT_STAT1_STATUS1_SIGNED            0
+
+/*
+	Register DUT_STAT2
+*/
+#define ODN_CORE_DUT_STAT2                      0x01A4
+#define ODN_DUT_STAT2_STATUS2_MASK              0xFFFFFFFFU
+#define ODN_DUT_STAT2_STATUS2_SHIFT             0
+#define ODN_DUT_STAT2_STATUS2_SIGNED            0
+
+/*
+	Register DASH_LEDS
+*/
+#define ODN_CORE_DASH_LEDS                      0x01A8
+#define ODN_DASH_LEDS_REPA_MASK                 0xFFF00000U
+#define ODN_DASH_LEDS_REPA_SHIFT                20
+#define ODN_DASH_LEDS_REPA_SIGNED               0
+
+#define ODN_DASH_LEDS_PIKE_MASK                 0x00000FFFU
+#define ODN_DASH_LEDS_PIKE_SHIFT                0
+#define ODN_DASH_LEDS_PIKE_SIGNED               0
+
+/*
+	Register DUT_CLK_INFO
+*/
+#define ODN_CORE_DUT_CLK_INFO                   0x01B0
+#define ODN_DUT_CLK_INFO_CORE_MASK              0x0000FFFFU
+#define ODN_DUT_CLK_INFO_CORE_SHIFT             0
+#define ODN_DUT_CLK_INFO_CORE_SIGNED            0
+
+#define ODN_DUT_CLK_INFO_MEM_MASK               0xFFFF0000U
+#define ODN_DUT_CLK_INFO_MEM_SHIFT              16
+#define ODN_DUT_CLK_INFO_MEM_SIGNED             0
+
+/*
+	Register DUT_CLK_PHSE
+*/
+#define ODN_CORE_DUT_CLK_PHSE                   0x01B4
+#define ODN_DUT_CLK_PHSE_MEM_REQ_MASK           0x0000FFFFU
+#define ODN_DUT_CLK_PHSE_MEM_REQ_SHIFT          0
+#define ODN_DUT_CLK_PHSE_MEM_REQ_SIGNED         0
+
+#define ODN_DUT_CLK_PHSE_MEM_RD_MASK            0xFFFF0000U
+#define ODN_DUT_CLK_PHSE_MEM_RD_SHIFT           16
+#define ODN_DUT_CLK_PHSE_MEM_RD_SIGNED          0
+
+/*
+	Register CORE_STATUS
+*/
+#define ODN_CORE_CORE_STATUS                    0x0200
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_MASK   0x00000001U
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_SHIFT  0
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_MASK 0x00000010U
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_SHIFT 4
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_MASK 0x00000020U
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_SHIFT 5
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_MASK 0x00000040U
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_SHIFT 6
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_MASK 0x00000080U
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_SHIFT 7
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_MASK 0x00000100U
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_SHIFT 8
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_SIGNED 0
+
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_MASK 0x00000200U
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_SHIFT 9
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_SIGNED 0
+
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_MASK 0x00001000U
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SHIFT 12
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_MASK 0x00002000U
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SHIFT 13
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+/*
+	Register CORE_CONTROL
+*/
+#define ODN_CORE_CORE_CONTROL                   0x0204
+#define ODN_CORE_CONTROL_BAR4_OFFSET_MASK       0x0000001FU
+#define ODN_CORE_CONTROL_BAR4_OFFSET_SHIFT      0
+#define ODN_CORE_CONTROL_BAR4_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_MASK 0x00000300U
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SHIFT 8
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SIGNED 0
+
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_MASK    0x00001C00U
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_SHIFT   10
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_SIGNED  0
+
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_MASK 0x00002000U
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SHIFT 13
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SIGNED 0
+
+#define ODN_CORE_CONTROL_PDP1_OFFSET_MASK       0x00070000U
+#define ODN_CORE_CONTROL_PDP1_OFFSET_SHIFT      16
+#define ODN_CORE_CONTROL_PDP1_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_PDP2_OFFSET_MASK       0x00700000U
+#define ODN_CORE_CONTROL_PDP2_OFFSET_SHIFT      20
+#define ODN_CORE_CONTROL_PDP2_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_DUT_OFFSET_MASK        0x07000000U
+#define ODN_CORE_CONTROL_DUT_OFFSET_SHIFT       24
+#define ODN_CORE_CONTROL_DUT_OFFSET_SIGNED      0
+
+/*
+	Register REG_BANK_STATUS
+*/
+#define ODN_CORE_REG_BANK_STATUS                0x0208
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_MASK 0xFFFFFFFFU
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SHIFT 0
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SIGNED 0
+
+/*
+	Register MMCM_LOCK_STATUS
+*/
+#define ODN_CORE_MMCM_LOCK_STATUS               0x020C
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_MASK      0x00000001U
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_SHIFT     0
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_SIGNED    0
+
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_MASK        0x00000002U
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_SHIFT       1
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_SIGNED      0
+
+#define ODN_MMCM_LOCK_STATUS_MULTI_MASK         0x00000004U
+#define ODN_MMCM_LOCK_STATUS_MULTI_SHIFT        2
+#define ODN_MMCM_LOCK_STATUS_MULTI_SIGNED       0
+
+#define ODN_MMCM_LOCK_STATUS_PDPP_MASK          0x00000008U
+#define ODN_MMCM_LOCK_STATUS_PDPP_SHIFT         3
+#define ODN_MMCM_LOCK_STATUS_PDPP_SIGNED        0
+
+/*
+	Register GIST_STATUS
+*/
+#define ODN_CORE_GIST_STATUS                    0x0210
+#define ODN_GIST_STATUS_MST_MASK                0x000001FFU
+#define ODN_GIST_STATUS_MST_SHIFT               0
+#define ODN_GIST_STATUS_MST_SIGNED              0
+
+#define ODN_GIST_STATUS_SLV_MASK                0x001FF000U
+#define ODN_GIST_STATUS_SLV_SHIFT               12
+#define ODN_GIST_STATUS_SLV_SIGNED              0
+
+#define ODN_GIST_STATUS_SLV_OUT_MASK            0x03000000U
+#define ODN_GIST_STATUS_SLV_OUT_SHIFT           24
+#define ODN_GIST_STATUS_SLV_OUT_SIGNED          0
+
+#define ODN_GIST_STATUS_MST_OUT_MASK            0x70000000U
+#define ODN_GIST_STATUS_MST_OUT_SHIFT           28
+#define ODN_GIST_STATUS_MST_OUT_SIGNED          0
+
+/*
+	Register DUT_MST_ADD
+*/
+#define ODN_CORE_DUT_MST_ADD                    0x0214
+#define ODN_DUT_MST_ADD_SLV_OUT_MASK            0x0000003FU
+#define ODN_DUT_MST_ADD_SLV_OUT_SHIFT           0
+#define ODN_DUT_MST_ADD_SLV_OUT_SIGNED          0
+
+/*
+	Register DUT_MULTIPLX_INFO
+*/
+#define ODN_CORE_DUT_MULTIPLX_INFO              0x0218
+#define ODN_DUT_MULTIPLX_INFO_MEM_MASK          0x000000FFU
+#define ODN_DUT_MULTIPLX_INFO_MEM_SHIFT         0
+#define ODN_DUT_MULTIPLX_INFO_MEM_SIGNED        0
+
+/****************************
+  Generated from: ad_tx.def
+*****************************/
+
+/*
+	Register ADT_CONTROL
+*/
+#define ODN_AD_TX_DEBUG_ADT_CONTROL             0x0000
+#define ODN_SET_ADTX_READY_MASK                 0x00000004U
+#define ODN_SET_ADTX_READY_SHIFT                2
+#define ODN_SET_ADTX_READY_SIGNED               0
+
+#define ODN_SEND_ALIGN_DATA_MASK                0x00000002U
+#define ODN_SEND_ALIGN_DATA_SHIFT               1
+#define ODN_SEND_ALIGN_DATA_SIGNED              0
+
+#define ODN_ENABLE_FLUSHING_MASK                0x00000001U
+#define ODN_ENABLE_FLUSHING_SHIFT               0
+#define ODN_ENABLE_FLUSHING_SIGNED              0
+
+/*
+	Register ADT_STATUS
+*/
+#define ODN_AD_TX_DEBUG_ADT_STATUS              0x0004
+#define ODN_REQUEST_COMPLETE_MASK               0x00000001U
+#define ODN_REQUEST_COMPLETE_SHIFT              0
+#define ODN_REQUEST_COMPLETE_SIGNED             0
+
+
+/******************************
+ Generated from: mca_debug.def
+*******************************/
+
+/*
+	Register MCA_CONTROL
+*/
+#define ODN_MCA_DEBUG_MCA_CONTROL               0x0000
+#define ODN_ALIGN_START_MASK                    0x00000001U
+#define ODN_ALIGN_START_SHIFT                   0
+#define ODN_ALIGN_START_SIGNED                  0
+
+/*
+	Register MCA_STATUS
+*/
+#define ODN_MCA_DEBUG_MCA_STATUS                0x0004
+#define ODN_TCHECK_SDEBUG_MASK                  0x40000000U
+#define ODN_TCHECK_SDEBUG_SHIFT                 30
+#define ODN_TCHECK_SDEBUG_SIGNED                0
+
+#define ODN_CHECK_SDEBUG_MASK                   0x20000000U
+#define ODN_CHECK_SDEBUG_SHIFT                  29
+#define ODN_CHECK_SDEBUG_SIGNED                 0
+
+#define ODN_ALIGN_SDEBUG_MASK                   0x10000000U
+#define ODN_ALIGN_SDEBUG_SHIFT                  28
+#define ODN_ALIGN_SDEBUG_SIGNED                 0
+
+#define ODN_FWAIT_SDEBUG_MASK                   0x08000000U
+#define ODN_FWAIT_SDEBUG_SHIFT                  27
+#define ODN_FWAIT_SDEBUG_SIGNED                 0
+
+#define ODN_IDLE_SDEBUG_MASK                    0x04000000U
+#define ODN_IDLE_SDEBUG_SHIFT                   26
+#define ODN_IDLE_SDEBUG_SIGNED                  0
+
+#define ODN_FIFO_FULL_MASK                      0x03FF0000U
+#define ODN_FIFO_FULL_SHIFT                     16
+#define ODN_FIFO_FULL_SIGNED                    0
+
+#define ODN_FIFO_EMPTY_MASK                     0x0000FFC0U
+#define ODN_FIFO_EMPTY_SHIFT                    6
+#define ODN_FIFO_EMPTY_SIGNED                   0
+
+#define ODN_TAG_CHECK_ERROR_MASK                0x00000020U
+#define ODN_TAG_CHECK_ERROR_SHIFT               5
+#define ODN_TAG_CHECK_ERROR_SIGNED              0
+
+#define ODN_ALIGN_CHECK_ERROR_MASK              0x00000010U
+#define ODN_ALIGN_CHECK_ERROR_SHIFT             4
+#define ODN_ALIGN_CHECK_ERROR_SIGNED            0
+
+#define ODN_ALIGN_ERROR_MASK                    0x00000008U
+#define ODN_ALIGN_ERROR_SHIFT                   3
+#define ODN_ALIGN_ERROR_SIGNED                  0
+
+#define ODN_TAG_CHECKING_OK_MASK                0x00000004U
+#define ODN_TAG_CHECKING_OK_SHIFT               2
+#define ODN_TAG_CHECKING_OK_SIGNED              0
+
+#define ODN_ALIGN_CHECK_OK_MASK                 0x00000002U
+#define ODN_ALIGN_CHECK_OK_SHIFT                1
+#define ODN_ALIGN_CHECK_OK_SIGNED               0
+
+#define ODN_ALIGNMENT_FOUND_MASK                0x00000001U
+#define ODN_ALIGNMENT_FOUND_SHIFT               0
+#define ODN_ALIGNMENT_FOUND_SIGNED              0
+
+
+/*********************************
+ Generated from: sai_rx_debug.def
+**********************************/
+
+/*
+	Register SIG_RESULT
+*/
+#define ODN_SAI_RX_DEBUG_SIG_RESULT             0x0000
+#define ODN_SIG_RESULT_VALUE_MASK               0xFFFFFFFFU
+#define ODN_SIG_RESULT_VALUE_SHIFT              0
+#define ODN_SIG_RESULT_VALUE_SIGNED             0
+
+/*
+	Register INIT_SIG
+*/
+#define ODN_SAI_RX_DEBUG_INIT_SIG               0x0004
+#define ODN_INIT_SIG_VALUE_MASK                 0x00000001U
+#define ODN_INIT_SIG_VALUE_SHIFT                0
+#define ODN_INIT_SIG_VALUE_SIGNED               0
+
+/*
+	Register SAI_BYPASS
+*/
+#define ODN_SAI_RX_DEBUG_SAI_BYPASS             0x0008
+#define ODN_BYPASS_CLK_TAPS_VALUE_MASK          0x000003FFU
+#define ODN_BYPASS_CLK_TAPS_VALUE_SHIFT         0
+#define ODN_BYPASS_CLK_TAPS_VALUE_SIGNED        0
+
+#define ODN_BYPASS_SET_MASK                     0x00010000U
+#define ODN_BYPASS_SET_SHIFT                    16
+#define ODN_BYPASS_SET_SIGNED                   0
+
+#define ODN_BYPASS_EN_MASK                      0x00100000U
+#define ODN_BYPASS_EN_SHIFT                     20
+#define ODN_BYPASS_EN_SIGNED                    0
+
+#define ODN_EN_STATUS_MASK                      0x01000000U
+#define ODN_EN_STATUS_SHIFT                     24
+#define ODN_EN_STATUS_SIGNED                    0
+
+/*
+	Register SAI_CLK_TAPS
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CLK_TAPS           0x000C
+#define ODN_CLK_TAPS_VALUE_MASK                 0x000003FFU
+#define ODN_CLK_TAPS_VALUE_SHIFT                0
+#define ODN_CLK_TAPS_VALUE_SIGNED               0
+
+#define ODN_TRAINING_COMPLETE_MASK              0x00010000U
+#define ODN_TRAINING_COMPLETE_SHIFT             16
+#define ODN_TRAINING_COMPLETE_SIGNED            0
+
+/*
+	Register SAI_EYES
+*/
+#define ODN_SAI_RX_DEBUG_SAI_EYES               0x0010
+#define ODN_MIN_EYE_END_MASK                    0x0000FFFFU
+#define ODN_MIN_EYE_END_SHIFT                   0
+#define ODN_MIN_EYE_END_SIGNED                  0
+
+#define ODN_MAX_EYE_START_MASK                  0xFFFF0000U
+#define ODN_MAX_EYE_START_SHIFT                 16
+#define ODN_MAX_EYE_START_SIGNED                0
+
+/*
+	Register SAI_DDR_INVERT
+*/
+#define ODN_SAI_RX_DEBUG_SAI_DDR_INVERT         0x0014
+#define ODN_DDR_INVERT_MASK                     0x00000001U
+#define ODN_DDR_INVERT_SHIFT                    0
+#define ODN_DDR_INVERT_SIGNED                   0
+
+#define ODN_OVERIDE_VALUE_MASK                  0x00010000U
+#define ODN_OVERIDE_VALUE_SHIFT                 16
+#define ODN_OVERIDE_VALUE_SIGNED                0
+
+#define ODN_INVERT_OVERIDE_MASK                 0x00100000U
+#define ODN_INVERT_OVERIDE_SHIFT                20
+#define ODN_INVERT_OVERIDE_SIGNED               0
+
+/*
+	Register SAI_TRAIN_ACK
+*/
+#define ODN_SAI_RX_DEBUG_SAI_TRAIN_ACK          0x0018
+#define ODN_TRAIN_ACK_FAIL_MASK                 0x00000001U
+#define ODN_TRAIN_ACK_FAIL_SHIFT                0
+#define ODN_TRAIN_ACK_FAIL_SIGNED               0
+
+#define ODN_TRAIN_ACK_FAIL_COUNT_MASK           0x000000F0U
+#define ODN_TRAIN_ACK_FAIL_COUNT_SHIFT          4
+#define ODN_TRAIN_ACK_FAIL_COUNT_SIGNED         0
+
+#define ODN_TRAIN_ACK_COMPLETE_MASK             0x00000100U
+#define ODN_TRAIN_ACK_COMPLETE_SHIFT            8
+#define ODN_TRAIN_ACK_COMPLETE_SIGNED           0
+
+#define ODN_TRAIN_ACK_OVERIDE_MASK              0x00001000U
+#define ODN_TRAIN_ACK_OVERIDE_SHIFT             12
+#define ODN_TRAIN_ACK_OVERIDE_SIGNED            0
+
+/*
+	Register SAI_TRAIN_ACK_COUNT
+*/
+#define ODN_SAI_RX_DEBUG_SAI_TRAIN_ACK_COUNT    0x001C
+#define ODN_TRAIN_COUNT_MASK                    0xFFFFFFFFU
+#define ODN_TRAIN_COUNT_SHIFT                   0
+#define ODN_TRAIN_COUNT_SIGNED                  0
+
+/*
+	Register SAI_CHANNEL_NUMBER
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_NUMBER     0x0020
+#define ODN_CHANNEL_NUMBER_MASK                 0x0000FFFFU
+#define ODN_CHANNEL_NUMBER_SHIFT                0
+#define ODN_CHANNEL_NUMBER_SIGNED               0
+
+/*
+	Register SAI_CHANNEL_EYE_START
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_START  0x0024
+#define ODN_CHANNEL_EYE_START_MASK              0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_START_SHIFT             0
+#define ODN_CHANNEL_EYE_START_SIGNED            0
+
+/*
+	Register SAI_CHANNEL_EYE_END
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_END    0x0028
+#define ODN_CHANNEL_EYE_END_MASK                0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_END_SHIFT               0
+#define ODN_CHANNEL_EYE_END_SIGNED              0
+
+/*
+	Register SAI_CHANNEL_EYE_PATTERN
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_PATTERN 0x002C
+#define ODN_CHANNEL_EYE_PATTERN_MASK            0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_PATTERN_SHIFT           0
+#define ODN_CHANNEL_EYE_PATTERN_SIGNED          0
+
+/*
+	Register SAI_CHANNEL_EYE_DEBUG
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_DEBUG  0x0030
+#define ODN_CHANNEL_EYE_SENSE_MASK              0x00000001U
+#define ODN_CHANNEL_EYE_SENSE_SHIFT             0
+#define ODN_CHANNEL_EYE_SENSE_SIGNED            0
+
+#define ODN_CHANNEL_EYE_COMPLETE_MASK           0x00000002U
+#define ODN_CHANNEL_EYE_COMPLETE_SHIFT          1
+#define ODN_CHANNEL_EYE_COMPLETE_SIGNED         0
+
+
+/*********************************
+ Generated from: sai_tx_debug.def
+**********************************/
+
+/*
+	Register SIG_RESULT
+*/
+#define ODN_SAI_TX_DEBUG_SIG_RESULT             0x0000
+#define ODN_TX_SIG_RESULT_VALUE_MASK            0xFFFFFFFFU
+#define ODN_TX_SIG_RESULT_VALUE_SHIFT           0
+#define ODN_TX_SIG_RESULT_VALUE_SIGNED          0
+
+/*
+	Register INIT_SIG
+*/
+#define ODN_SAI_TX_DEBUG_INIT_SIG               0x0004
+#define ODN_TX_INIT_SIG_VALUE_MASK              0x00000001U
+#define ODN_TX_INIT_SIG_VALUE_SHIFT             0
+#define ODN_TX_INIT_SIG_VALUE_SIGNED            0
+
+/*
+	Register SAI_BYPASS
+*/
+#define ODN_SAI_TX_DEBUG_SAI_BYPASS             0x0008
+#define ODN_TX_BYPASS_EN_MASK                   0x00000001U
+#define ODN_TX_BYPASS_EN_SHIFT                  0
+#define ODN_TX_BYPASS_EN_SIGNED                 0
+
+#define ODN_TX_ACK_RESEND_MASK                  0x00000002U
+#define ODN_TX_ACK_RESEND_SHIFT                 1
+#define ODN_TX_ACK_RESEND_SIGNED                0
+
+#define ODN_TX_DISABLE_ACK_SEND_MASK            0x00000004U
+#define ODN_TX_DISABLE_ACK_SEND_SHIFT           2
+#define ODN_TX_DISABLE_ACK_SEND_SIGNED          0
+
+/*
+	Register SAI_STATUS
+*/
+#define ODN_SAI_TX_DEBUG_SAI_STATUS             0x000C
+#define ODN_TX_TRAINING_COMPLETE_MASK           0x00000001U
+#define ODN_TX_TRAINING_COMPLETE_SHIFT          0
+#define ODN_TX_TRAINING_COMPLETE_SIGNED         0
+
+#define ODN_TX_TRAINING_ACK_COMPLETE_MASK       0x00000002U
+#define ODN_TX_TRAINING_ACK_COMPLETE_SHIFT      1
+#define ODN_TX_TRAINING_ACK_COMPLETE_SIGNED     0
+
+
+
+#endif /* _ODIN_REGS_H_ */
+
+/******************************************************************************
+ End of file (odin_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_defs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_defs.h
new file mode 100644
index 000000000000..1691151de58d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_defs.h
@@ -0,0 +1,183 @@
+/****************************************************************************
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Orion Memory Map - View from PCIe
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+****************************************************************************/
+
+#ifndef _ORION_DEFS_H_
+#define _ORION_DEFS_H_
+
+/*
+ * These defines have not been autogenerated
+ * Only values different from Odin will be included here
+ */
+
+#define DEVICE_ID_ORION                          0x1020
+
+/* Odin system register banks */
+#define SRS_REG_BANK_ODN_CLK_BLK                 0x02000
+
+/*
+ * Orion CLK regs - the srs_clk_blk module defs are not auto generated
+ */
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG1           0x620
+#define SRS_PDP_PCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define SRS_PDP_PCLK_ODIV1_LO_TIME_SHIFT         0
+#define SRS_PDP_PCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define SRS_PDP_PCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG2           0x624
+#define SRS_PDP_PCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define SRS_PDP_PCLK_ODIV2_NOCOUNT_SHIFT         6
+#define SRS_PDP_PCLK_ODIV2_EDGE_MASK             0x00000080U
+#define SRS_PDP_PCLK_ODIV2_EDGE_SHIFT            7
+#define SRS_PDP_PCLK_ODIV2_FRAC_MASK             0x00007C00U
+#define SRS_PDP_PCLK_ODIV2_FRAC_SHIFT            10
+
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG3           0x61C
+
+#define SRS_PDP_M_CLK_OUT_DIVIDER_REG1           0x628
+#define SRS_PDP_MCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define SRS_PDP_MCLK_ODIV1_LO_TIME_SHIFT         0
+#define SRS_PDP_MCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define SRS_PDP_MCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define SRS_PDP_M_CLK_OUT_DIVIDER_REG2           0x62C
+#define SRS_PDP_MCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define SRS_PDP_MCLK_ODIV2_NOCOUNT_SHIFT         6
+#define SRS_PDP_MCLK_ODIV2_EDGE_MASK             0x00000080U
+#define SRS_PDP_MCLK_ODIV2_EDGE_SHIFT            7
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG1            0x650
+#define SRS_PDP_PCLK_MUL1_LO_TIME_MASK           0x0000003FU
+#define SRS_PDP_PCLK_MUL1_LO_TIME_SHIFT          0
+#define SRS_PDP_PCLK_MUL1_HI_TIME_MASK           0x00000FC0U
+#define SRS_PDP_PCLK_MUL1_HI_TIME_SHIFT          6
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG2            0x654
+#define SRS_PDP_PCLK_MUL2_NOCOUNT_MASK           0x00000040U
+#define SRS_PDP_PCLK_MUL2_NOCOUNT_SHIFT          6
+#define SRS_PDP_PCLK_MUL2_EDGE_MASK              0x00000080U
+#define SRS_PDP_PCLK_MUL2_EDGE_SHIFT             7
+#define SRS_PDP_PCLK_MUL2_FRAC_MASK              0x00007C00U
+#define SRS_PDP_PCLK_MUL2_FRAC_SHIFT             10
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG3            0x64C
+
+#define SRS_PDP_P_CLK_IN_DIVIDER_REG             0x658
+#define SRS_PDP_PCLK_IDIV_LO_TIME_MASK           0x0000003FU
+#define SRS_PDP_PCLK_IDIV_LO_TIME_SHIFT          0
+#define SRS_PDP_PCLK_IDIV_HI_TIME_MASK           0x00000FC0U
+#define SRS_PDP_PCLK_IDIV_HI_TIME_SHIFT          6
+#define SRS_PDP_PCLK_IDIV_NOCOUNT_MASK           0x00001000U
+#define SRS_PDP_PCLK_IDIV_NOCOUNT_SHIFT          12
+#define SRS_PDP_PCLK_IDIV_EDGE_MASK              0x00002000U
+#define SRS_PDP_PCLK_IDIV_EDGE_SHIFT             13
+
+/*
+ * DUT core clock input divider, DUT reference clock input divider
+ */
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1                0x0020
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_MASK   0x00000FC0U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  6
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_MASK   0x0000003FU
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  0
+
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2                0x0024
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_MASK      0x00000080U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_SHIFT     7
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_MASK   0x00000040U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  6
+
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1                 0x0028
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_HI_TIME_MASK    0x00000FC0U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_HI_TIME_SHIFT   6
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_LO_TIME_MASK    0x0000003FU
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_LO_TIME_SHIFT   0
+
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2                 0x002C
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_EDGE_MASK       0x00000080U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_EDGE_SHIFT      7
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_NOCOUNT_MASK    0x00000040U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT   6
+
+/*
+ * DUT interface reference clock input divider
+ */
+
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1                0x0228
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_HI_TIME_MASK   0x00000FC0U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  6
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_LO_TIME_MASK   0x0000003FU
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  0
+
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2                0x022C
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_EDGE_MASK      0x00000080U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_EDGE_SHIFT     7
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_NOCOUNT_MASK   0x00000040U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  6
+
+/*
+ * Min max values from Xilinx Virtex Ultrascale data sheet DS893,
+ * for speed grade 1. All in Hz.
+ */
+#define SRS_INPUT_CLOCK_SPEED                       100000000U
+#define SRS_INPUT_CLOCK_SPEED_MIN                   10000000U
+#define SRS_INPUT_CLOCK_SPEED_MAX                   800000000U
+#define SRS_OUTPUT_CLOCK_SPEED_MIN                  4690000U
+#define SRS_OUTPUT_CLOCK_SPEED_MAX                  630000000U
+#define SRS_VCO_MIN                                 600000000U
+#define SRS_VCO_MAX                                 1200000000U
+#define SRS_PFD_MIN                                 10000000U
+#define SRS_PFD_MAX                                 450000000U
+
+/*
+ * Orion interrupt flags
+ */
+#define SRS_INTERRUPT_ENABLE_PDP1          (1 << SRS_INTERRUPT_ENABLE_PDP_SHIFT)
+#define SRS_INTERRUPT_ENABLE_DUT           (1 << SRS_INTERRUPT_ENABLE_DUT_SHIFT)
+#define SRS_INTERRUPT_STATUS_PDP1          (1 << SRS_INTERRUPT_STATUS_PDP_SHIFT)
+#define SRS_INTERRUPT_STATUS_DUT           (1 << SRS_INTERRUPT_STATUS_DUT_SHIFT)
+#define SRS_INTERRUPT_CLEAR_PDP1           (1 << SRS_INTERRUPT_CLR_PDP_SHIFT)
+#define SRS_INTERRUPT_CLEAR_DUT            (1 << SRS_INTERRUPT_CLR_DUT_SHIFT)
+
+#endif /* _ORION_DEFS_H_ */
+
+/*****************************************************************************
+ End of file (orion_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_regs.h
new file mode 100644
index 000000000000..2a626bd2b70a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/orion_regs.h
@@ -0,0 +1,439 @@
+/******************************************************************************
+@Title          Orion system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Orion FPGA register defs for Sirius RTL
+@Author         Autogenerated
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef _OUT_DRV_H_
+#define _OUT_DRV_H_
+
+/*
+	Register ID
+*/
+#define SRS_CORE_ID                         0x0000
+#define SRS_ID_VARIANT_MASK                     0x0000FFFFU
+#define SRS_ID_VARIANT_SHIFT                    0
+#define SRS_ID_VARIANT_SIGNED                   0
+
+#define SRS_ID_ID_MASK                          0xFFFF0000U
+#define SRS_ID_ID_SHIFT                         16
+#define SRS_ID_ID_SIGNED                        0
+
+/*
+	Register REVISION
+*/
+#define SRS_CORE_REVISION                   0x0004
+#define SRS_REVISION_MINOR_MASK                 0x000000FFU
+#define SRS_REVISION_MINOR_SHIFT                0
+#define SRS_REVISION_MINOR_SIGNED               0
+
+#define SRS_REVISION_MAJOR_MASK                 0x00000F00U
+#define SRS_REVISION_MAJOR_SHIFT                8
+#define SRS_REVISION_MAJOR_SIGNED               0
+
+/*
+	Register CHANGE_SET
+*/
+#define SRS_CORE_CHANGE_SET                 0x0008
+#define SRS_CHANGE_SET_SET_MASK                 0xFFFFFFFFU
+#define SRS_CHANGE_SET_SET_SHIFT                0
+#define SRS_CHANGE_SET_SET_SIGNED               0
+
+/*
+	Register USER_ID
+*/
+#define SRS_CORE_USER_ID                    0x000C
+#define SRS_USER_ID_ID_MASK                     0x0000000FU
+#define SRS_USER_ID_ID_SHIFT                    0
+#define SRS_USER_ID_ID_SIGNED                   0
+
+/*
+	Register USER_BUILD
+*/
+#define SRS_CORE_USER_BUILD                 0x0010
+#define SRS_USER_BUILD_BUILD_MASK               0xFFFFFFFFU
+#define SRS_USER_BUILD_BUILD_SHIFT              0
+#define SRS_USER_BUILD_BUILD_SIGNED             0
+
+/*
+	Register SOFT_RESETN
+*/
+#define SRS_CORE_SOFT_RESETN                0x0080
+#define SRS_SOFT_RESETN_DDR_MASK                0x00000001U
+#define SRS_SOFT_RESETN_DDR_SHIFT               0
+#define SRS_SOFT_RESETN_DDR_SIGNED              0
+
+#define SRS_SOFT_RESETN_USB_MASK                0x00000002U
+#define SRS_SOFT_RESETN_USB_SHIFT               1
+#define SRS_SOFT_RESETN_USB_SIGNED              0
+
+#define SRS_SOFT_RESETN_PDP_MASK                0x00000004U
+#define SRS_SOFT_RESETN_PDP_SHIFT               2
+#define SRS_SOFT_RESETN_PDP_SIGNED              0
+
+#define SRS_SOFT_RESETN_GIST_MASK               0x00000008U
+#define SRS_SOFT_RESETN_GIST_SHIFT              3
+#define SRS_SOFT_RESETN_GIST_SIGNED             0
+
+/*
+	Register DUT_SOFT_RESETN
+*/
+#define SRS_CORE_DUT_SOFT_RESETN            0x0084
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_MASK       0x00000001U
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_SHIFT      0
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_SIGNED     0
+
+/*
+	Register SOFT_AUTO_RESETN
+*/
+#define SRS_CORE_SOFT_AUTO_RESETN           0x0088
+#define SRS_SOFT_AUTO_RESETN_CFG_MASK           0x00000001U
+#define SRS_SOFT_AUTO_RESETN_CFG_SHIFT          0
+#define SRS_SOFT_AUTO_RESETN_CFG_SIGNED         0
+
+/*
+	Register CLK_GEN_RESET
+*/
+#define SRS_CORE_CLK_GEN_RESET              0x0090
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_MASK    0x00000001U
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_SHIFT   0
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_SIGNED  0
+
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_MASK      0x00000002U
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_SHIFT     1
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_SIGNED    0
+
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_MASK       0x00000004U
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_SHIFT      2
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_SIGNED     0
+
+#define SRS_CLK_GEN_RESET_PDP_MMCM_MASK         0x00000008U
+#define SRS_CLK_GEN_RESET_PDP_MMCM_SHIFT        3
+#define SRS_CLK_GEN_RESET_PDP_MMCM_SIGNED       0
+
+/*
+	Register DUT_MEM
+*/
+#define SRS_CORE_DUT_MEM                    0x0120
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_MASK  0x0000FFFFU
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_SHIFT 0
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_SIGNED 0
+
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_MASK 0xFFFF0000U
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_SHIFT 16
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_SIGNED 0
+
+/*
+	Register APM
+*/
+#define SRS_CORE_APM                        0x0150
+#define SRS_APM_RESET_EVENT_MASK                0x00000001U
+#define SRS_APM_RESET_EVENT_SHIFT               0
+#define SRS_APM_RESET_EVENT_SIGNED              0
+
+#define SRS_APM_CAPTURE_EVENT_MASK              0x00000002U
+#define SRS_APM_CAPTURE_EVENT_SHIFT             1
+#define SRS_APM_CAPTURE_EVENT_SIGNED            0
+
+/*
+	Register NUM_GPIO
+*/
+#define SRS_CORE_NUM_GPIO                   0x0180
+#define SRS_NUM_GPIO_NUMBER_MASK                0x0000000FU
+#define SRS_NUM_GPIO_NUMBER_SHIFT               0
+#define SRS_NUM_GPIO_NUMBER_SIGNED              0
+
+/*
+	Register GPIO_EN
+*/
+#define SRS_CORE_GPIO_EN                    0x0184
+#define SRS_GPIO_EN_DIRECTION_MASK              0x000000FFU
+#define SRS_GPIO_EN_DIRECTION_SHIFT             0
+#define SRS_GPIO_EN_DIRECTION_SIGNED            0
+
+/*
+	Register GPIO
+*/
+#define SRS_CORE_GPIO                       0x0188
+#define SRS_GPIO_GPIO_MASK                      0x000000FFU
+#define SRS_GPIO_GPIO_SHIFT                     0
+#define SRS_GPIO_GPIO_SIGNED                    0
+
+/*
+	Register SPI_MASTER_IFACE
+*/
+#define SRS_CORE_SPI_MASTER_IFACE           0x018C
+#define SRS_SPI_MASTER_IFACE_ENABLE_MASK        0x00000001U
+#define SRS_SPI_MASTER_IFACE_ENABLE_SHIFT       0
+#define SRS_SPI_MASTER_IFACE_ENABLE_SIGNED      0
+
+/*
+	Register SRS_IP_STATUS
+*/
+#define SRS_CORE_SRS_IP_STATUS              0x0200
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_MASK 0x00000001U
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_SHIFT 0
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_MASK 0x00000002U
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_SHIFT 1
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_MASK 0x00000004U
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SHIFT 2
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_MASK 0x00000008U
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SHIFT 3
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+/*
+	Register CORE_CONTROL
+*/
+#define SRS_CORE_CORE_CONTROL               0x0204
+#define SRS_CORE_CONTROL_BAR4_OFFSET_MASK       0x0000001FU
+#define SRS_CORE_CONTROL_BAR4_OFFSET_SHIFT      0
+#define SRS_CORE_CONTROL_BAR4_OFFSET_SIGNED     0
+
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_MASK 0x00000300U
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SHIFT 8
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SIGNED 0
+
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_MASK    0x00001C00U
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_SHIFT   10
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_SIGNED  0
+
+/*
+	Register REG_BANK_STATUS
+*/
+#define SRS_CORE_REG_BANK_STATUS            0x0208
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_MASK 0xFFFFFFFFU
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SHIFT 0
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SIGNED 0
+
+/*
+	Register MMCM_LOCK_STATUS
+*/
+#define SRS_CORE_MMCM_LOCK_STATUS           0x020C
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_MASK      0x00000001U
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_SHIFT     0
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_SIGNED    0
+
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_MASK        0x00000002U
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_SHIFT       1
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_SIGNED      0
+
+#define SRS_MMCM_LOCK_STATUS_MULTI_MASK         0x00000004U
+#define SRS_MMCM_LOCK_STATUS_MULTI_SHIFT        2
+#define SRS_MMCM_LOCK_STATUS_MULTI_SIGNED       0
+
+#define SRS_MMCM_LOCK_STATUS_PDP_MASK           0x00000008U
+#define SRS_MMCM_LOCK_STATUS_PDP_SHIFT          3
+#define SRS_MMCM_LOCK_STATUS_PDP_SIGNED         0
+
+/*
+	Register GIST_STATUS
+*/
+#define SRS_CORE_GIST_STATUS                0x0210
+#define SRS_GIST_STATUS_MST_MASK                0x000001FFU
+#define SRS_GIST_STATUS_MST_SHIFT               0
+#define SRS_GIST_STATUS_MST_SIGNED              0
+
+#define SRS_GIST_STATUS_SLV_MASK                0x001FF000U
+#define SRS_GIST_STATUS_SLV_SHIFT               12
+#define SRS_GIST_STATUS_SLV_SIGNED              0
+
+#define SRS_GIST_STATUS_SLV_OUT_MASK            0x03000000U
+#define SRS_GIST_STATUS_SLV_OUT_SHIFT           24
+#define SRS_GIST_STATUS_SLV_OUT_SIGNED          0
+
+#define SRS_GIST_STATUS_MST_OUT_MASK            0x70000000U
+#define SRS_GIST_STATUS_MST_OUT_SHIFT           28
+#define SRS_GIST_STATUS_MST_OUT_SIGNED          0
+
+/*
+	Register SENSOR_BOARD
+*/
+#define SRS_CORE_SENSOR_BOARD               0x0214
+#define SRS_SENSOR_BOARD_ID_MASK                0x00000003U
+#define SRS_SENSOR_BOARD_ID_SHIFT               0
+#define SRS_SENSOR_BOARD_ID_SIGNED              0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define SRS_CORE_INTERRUPT_STATUS           0x0218
+#define SRS_INTERRUPT_STATUS_DUT_MASK           0x00000001U
+#define SRS_INTERRUPT_STATUS_DUT_SHIFT          0
+#define SRS_INTERRUPT_STATUS_DUT_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_PDP_MASK           0x00000002U
+#define SRS_INTERRUPT_STATUS_PDP_SHIFT          1
+#define SRS_INTERRUPT_STATUS_PDP_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_I2C_MASK           0x00000004U
+#define SRS_INTERRUPT_STATUS_I2C_SHIFT          2
+#define SRS_INTERRUPT_STATUS_I2C_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_SPI_MASK           0x00000008U
+#define SRS_INTERRUPT_STATUS_SPI_SHIFT          3
+#define SRS_INTERRUPT_STATUS_SPI_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_APM_MASK           0x00000010U
+#define SRS_INTERRUPT_STATUS_APM_SHIFT          4
+#define SRS_INTERRUPT_STATUS_APM_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_OS_IRQ_MASK        0x00001FE0U
+#define SRS_INTERRUPT_STATUS_OS_IRQ_SHIFT       5
+#define SRS_INTERRUPT_STATUS_OS_IRQ_SIGNED      0
+
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_MASK      0x40000000U
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_SHIFT     30
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_SIGNED    0
+
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_MASK 0x80000000U
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_SHIFT 31
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_SIGNED 0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define SRS_CORE_INTERRUPT_ENABLE           0x021C
+#define SRS_INTERRUPT_ENABLE_DUT_MASK           0x00000001U
+#define SRS_INTERRUPT_ENABLE_DUT_SHIFT          0
+#define SRS_INTERRUPT_ENABLE_DUT_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_PDP_MASK           0x00000002U
+#define SRS_INTERRUPT_ENABLE_PDP_SHIFT          1
+#define SRS_INTERRUPT_ENABLE_PDP_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_I2C_MASK           0x00000004U
+#define SRS_INTERRUPT_ENABLE_I2C_SHIFT          2
+#define SRS_INTERRUPT_ENABLE_I2C_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_SPI_MASK           0x00000008U
+#define SRS_INTERRUPT_ENABLE_SPI_SHIFT          3
+#define SRS_INTERRUPT_ENABLE_SPI_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_APM_MASK           0x00000010U
+#define SRS_INTERRUPT_ENABLE_APM_SHIFT          4
+#define SRS_INTERRUPT_ENABLE_APM_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_MASK        0x00001FE0U
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_SHIFT       5
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_SIGNED      0
+
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_MASK      0x40000000U
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_SHIFT     30
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_SIGNED    0
+
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_MASK 0x80000000U
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_SHIFT 31
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_SIGNED 0
+
+/*
+	Register INTERRUPT_CLR
+*/
+#define SRS_CORE_INTERRUPT_CLR              0x0220
+#define SRS_INTERRUPT_CLR_DUT_MASK              0x00000001U
+#define SRS_INTERRUPT_CLR_DUT_SHIFT             0
+#define SRS_INTERRUPT_CLR_DUT_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_PDP_MASK              0x00000002U
+#define SRS_INTERRUPT_CLR_PDP_SHIFT             1
+#define SRS_INTERRUPT_CLR_PDP_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_I2C_MASK              0x00000004U
+#define SRS_INTERRUPT_CLR_I2C_SHIFT             2
+#define SRS_INTERRUPT_CLR_I2C_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_SPI_MASK              0x00000008U
+#define SRS_INTERRUPT_CLR_SPI_SHIFT             3
+#define SRS_INTERRUPT_CLR_SPI_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_APM_MASK              0x00000010U
+#define SRS_INTERRUPT_CLR_APM_SHIFT             4
+#define SRS_INTERRUPT_CLR_APM_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_OS_IRQ_MASK           0x00001FE0U
+#define SRS_INTERRUPT_CLR_OS_IRQ_SHIFT          5
+#define SRS_INTERRUPT_CLR_OS_IRQ_SIGNED         0
+
+#define SRS_INTERRUPT_CLR_IRQ_TEST_MASK         0x40000000U
+#define SRS_INTERRUPT_CLR_IRQ_TEST_SHIFT        30
+#define SRS_INTERRUPT_CLR_IRQ_TEST_SIGNED       0
+
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_MASK     0x80000000U
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_SHIFT    31
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_SIGNED   0
+
+/*
+	Register INTERRUPT_TEST
+*/
+#define SRS_CORE_INTERRUPT_TEST             0x0224
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_MASK  0x00000001U
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_SHIFT 0
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT_CLR
+*/
+#define SRS_CORE_INTERRUPT_TIMEOUT_CLR      0x0228
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_MASK 0x00000002U
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SHIFT 1
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SIGNED 0
+
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_MASK 0x00000001U
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SHIFT 0
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT
+*/
+#define SRS_CORE_INTERRUPT_TIMEOUT          0x022C
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_MASK 0xFFFFFFFFU
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SHIFT 0
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SIGNED 0
+
+#endif /* _OUT_DRV_H_ */
+
+/******************************************************************************
+ End of file (orion_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pdp_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pdp_regs.h
new file mode 100644
index 000000000000..bd26b0617f95
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pdp_regs.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__PDP_REGS_H__)
+#define __PDP_REGS_H__
+
+/*************************************************************************/ /*!
+ PCI Device Information
+*/ /**************************************************************************/
+
+#define DCPDP_VENDOR_ID_POWERVR			(0x1010)
+
+#define DCPDP_DEVICE_ID_PCI_APOLLO_FPGA		(0x1CF1)
+#define DCPDP_DEVICE_ID_PCIE_APOLLO_FPGA	(0x1CF2)
+
+/*************************************************************************/ /*!
+ PCI Device Base Address Information
+*/ /**************************************************************************/
+
+/* PLL and PDP registers on base address register 0 */
+#define DCPDP_REG_PCI_BASENUM			(0)
+
+#define DCPDP_PCI_PLL_REG_OFFSET		(0x1000)
+#define DCPDP_PCI_PLL_REG_SIZE			(0x0400)
+
+#define DCPDP_PCI_PDP_REG_OFFSET		(0xC000)
+#define DCPDP_PCI_PDP_REG_SIZE			(0x2000)
+
+/*************************************************************************/ /*!
+ Misc register information
+*/ /**************************************************************************/
+
+/* This information isn't captured in tcf_rgbpdp_regs.h so define it here */
+#define DCPDP_STR1SURF_FORMAT_ARGB8888		(0xE)
+#define DCPDP_STR1ADDRCTRL_BASE_ADDR_SHIFT	(4)
+#define DCPDP_STR1POSN_STRIDE_SHIFT		(4)
+
+#endif /* !defined(__PDP_REGS_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_defs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_defs.h
new file mode 100644
index 000000000000..d39c06f703bd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_defs.h
@@ -0,0 +1,69 @@
+/******************************************************************************
+@Title          Odin PFIM definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin register defs for PDP-FBDC Interface Module
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef _PFIM_DEFS_H_
+#define _PFIM_DEFS_H_
+
+/* Supported FBC modes */
+#define ODIN_PFIM_MOD_LINEAR       (0x00)
+#define ODIN_PFIM_FBCDC_8X8_V12    (0x01)
+#define ODIN_PFIM_FBCDC_16X4_V12   (0x02)
+#define ODIN_PFIM_FBCDC_MAX        (0x03)
+
+/* Supported pixel formats */
+#define ODN_PFIM_PIXFMT_NONE       (0x00)
+#define ODN_PFIM_PIXFMT_ARGB8888   (0x0C)
+#define ODN_PFIM_PIXFMT_RGB565     (0x05)
+
+/* Tile types */
+#define ODN_PFIM_TILETYPE_8X8      (0x01)
+#define ODN_PFIM_TILETYPE_16X4     (0x02)
+#define ODN_PFIM_TILETYPE_32x2     (0x03)
+
+#define PFIM_ROUNDUP(X, Y)         (((X) + ((Y) - 1U)) & ~((Y) - 1U))
+#define PFIM_RND_TAG               (0x10)
+
+#endif /* _PFIM_DEFS_H_ */
+
+/******************************************************************************
+ End of file (pfim_defs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_regs.h
new file mode 100644
index 000000000000..4b8ff82138b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/pfim_regs.h
@@ -0,0 +1,265 @@
+/******************************************************************************
+@Title          Odin PFIM control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin register defs for PDP-FBDC Interface Module
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+#ifndef _PFIM_REGS_H_
+#define _PFIM_REGS_H_
+
+/*
+	Register CR_PFIM_NUM_TILES
+*/
+#define CR_PFIM_NUM_TILES 0x0000
+#define CR_PFIM_NUM_TILES_MASK 0x007FFFFFU
+#define CR_PFIM_NUM_TILES_SHIFT 0
+#define CR_PFIM_NUM_TILES_SIGNED 0
+
+/*
+	Register CR_PFIM_TILES_PER_LINE
+*/
+#define CR_PFIM_TILES_PER_LINE 0x0004
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_MASK 0x000000FFU
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_SHIFT 0
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB
+*/
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB 0x0008
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB
+*/
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB 0x000C
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_MASK 0x00000003U
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_UV_BASE_ADDR_LSB
+*/
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB 0x0010
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_UV_BASE_ADDR_MSB
+*/
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB 0x0014
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_MASK 0x00000003U
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_PDP_Y_BASE_ADDR
+*/
+#define CR_PFIM_PDP_Y_BASE_ADDR 0x0018
+#define CR_PFIM_PDP_Y_BASE_ADDR_MASK 0xFFFFFFFFU
+#define CR_PFIM_PDP_Y_BASE_ADDR_SHIFT 0
+#define CR_PFIM_PDP_Y_BASE_ADDR_SIGNED 0
+
+/*
+	Register CR_PFIM_PDP_UV_BASE_ADDR
+*/
+#define CR_PFIM_PDP_UV_BASE_ADDR 0x001C
+#define CR_PFIM_PDP_UV_BASE_ADDR_MASK 0xFFFFFFFFU
+#define CR_PFIM_PDP_UV_BASE_ADDR_SHIFT 0
+#define CR_PFIM_PDP_UV_BASE_ADDR_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_CONTEXT
+*/
+#define CR_PFIM_FBDC_REQ_CONTEXT 0x0020
+#define CR_PFIM_FBDC_REQ_CONTEXT_MASK 0x00000007U
+#define CR_PFIM_FBDC_REQ_CONTEXT_SHIFT 0
+#define CR_PFIM_FBDC_REQ_CONTEXT_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_TAG
+*/
+#define CR_PFIM_FBDC_REQ_TAG 0x0024
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_MASK     0x00000003U
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_SHIFT    0
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_SIGNED   0
+
+#define CR_PFIM_FBDC_REQ_TAG_UV_MASK        0x00000030U
+#define CR_PFIM_FBDC_REQ_TAG_UV_SHIFT       4
+#define CR_PFIM_FBDC_REQ_TAG_UV_SIGNED      0
+
+/*
+	Register CR_PFIM_FBDC_REQ_SB_TAG
+*/
+#define CR_PFIM_FBDC_REQ_SB_TAG 0x0028
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_MASK  0x00000003U
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_SHIFT 0
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_SIGNED 0
+
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_MASK     0x00000030U
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_SHIFT    4
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_SIGNED   0
+
+/*
+	Register CR_PFIM_FBDC_HDR_INVAL_REQ
+*/
+#define CR_PFIM_FBDC_HDR_INVAL_REQ 0x002C
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_MASK 0x00000001U
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_SHIFT 0
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_PIX_FORMAT
+*/
+#define CR_PFIM_FBDC_PIX_FORMAT 0x0030
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_MASK 0x0000007FU
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_SHIFT 0
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_CH0123_VAL0
+*/
+#define CR_PFIM_FBDC_CR_CH0123_VAL0 0x0034
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_CH0123_VAL1
+*/
+#define CR_PFIM_FBDC_CR_CH0123_VAL1 0x0038
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_Y_VAL0
+*/
+#define CR_PFIM_FBDC_CR_Y_VAL0 0x003C
+#define CR_PFIM_FBDC_CR_Y_VAL0_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_Y_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_Y_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_UV_VAL0
+*/
+#define CR_PFIM_FBDC_CR_UV_VAL0 0x0040
+#define CR_PFIM_FBDC_CR_UV_VAL0_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_UV_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_UV_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_Y_VAL1
+*/
+#define CR_PFIM_FBDC_CR_Y_VAL1 0x0044
+#define CR_PFIM_FBDC_CR_Y_VAL1_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_Y_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_Y_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_UV_VAL1
+*/
+#define CR_PFIM_FBDC_CR_UV_VAL1 0x0048
+#define CR_PFIM_FBDC_CR_UV_VAL1_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_UV_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_UV_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_ENABLE
+*/
+#define CR_PFIM_FBDC_FILTER_ENABLE 0x004C
+#define CR_PFIM_FBDC_FILTER_ENABLE_MASK 0x00000001U
+#define CR_PFIM_FBDC_FILTER_ENABLE_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_ENABLE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_STATUS
+*/
+#define CR_PFIM_FBDC_FILTER_STATUS 0x0050
+#define CR_PFIM_FBDC_FILTER_STATUS_MASK 0x0000000FU
+#define CR_PFIM_FBDC_FILTER_STATUS_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_STATUS_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_CLEAR
+*/
+#define CR_PFIM_FBDC_FILTER_CLEAR 0x0054
+#define CR_PFIM_FBDC_FILTER_CLEAR_MASK 0x0000000FU
+#define CR_PFIM_FBDC_FILTER_CLEAR_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_CLEAR_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_TILE_TYPE
+*/
+#define CR_PFIM_FBDC_TILE_TYPE 0x0058
+#define CR_PFIM_FBDC_TILE_TYPE_MASK 0x00000003U
+#define CR_PFIM_FBDC_TILE_TYPE_SHIFT 0
+#define CR_PFIM_FBDC_TILE_TYPE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CLEAR_COLOUR_LSB
+*/
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB 0x005C
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CLEAR_COLOUR_MSB
+*/
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB 0x0060
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_LOSSY
+*/
+#define CR_PFIM_FBDC_REQ_LOSSY 0x0064
+#define CR_PFIM_FBDC_REQ_LOSSY_MASK 0x00000001U
+#define CR_PFIM_FBDC_REQ_LOSSY_SHIFT 0
+#define CR_PFIM_FBDC_REQ_LOSSY_SIGNED 0
+
+#endif /* _PFIM_REGS_H_ */
+
+/******************************************************************************
+ End of file (pfim_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_clk_ctrl.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_clk_ctrl.h
new file mode 100644
index 000000000000..cc7b10fd8116
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_clk_ctrl.h
@@ -0,0 +1,1018 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from: tcf_clk_ctrl.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_CLK_CTRL_H_)
+#define _TCF_CLK_CTRL_H_
+
+/*
+ * The following register definitions are valid if register 0x28 has value 0.
+ */
+
+/*
+	Register FPGA_ID_REG
+*/
+#define TCF_CLK_CTRL_FPGA_ID_REG            0x0000
+#define FPGA_ID_REG_CORE_CFG_MASK           0x0000FFFFU
+#define FPGA_ID_REG_CORE_CFG_SHIFT          0
+#define FPGA_ID_REG_CORE_CFG_SIGNED         0
+
+#define FPGA_ID_REG_CORE_ID_MASK            0xFFFF0000U
+#define FPGA_ID_REG_CORE_ID_SHIFT           16
+#define FPGA_ID_REG_CORE_ID_SIGNED          0
+
+/*
+	Register FPGA_REV_REG
+*/
+#define TCF_CLK_CTRL_FPGA_REV_REG           0x0008
+#define FPGA_REV_REG_MAINT_MASK             0x000000FFU
+#define FPGA_REV_REG_MAINT_SHIFT            0
+#define FPGA_REV_REG_MAINT_SIGNED           0
+
+#define FPGA_REV_REG_MINOR_MASK             0x0000FF00U
+#define FPGA_REV_REG_MINOR_SHIFT            8
+#define FPGA_REV_REG_MINOR_SIGNED           0
+
+#define FPGA_REV_REG_MAJOR_MASK             0x00FF0000U
+#define FPGA_REV_REG_MAJOR_SHIFT            16
+#define FPGA_REV_REG_MAJOR_SIGNED           0
+
+#define FPGA_REV_REG_DESIGNER_MASK          0xFF000000U
+#define FPGA_REV_REG_DESIGNER_SHIFT         24
+#define FPGA_REV_REG_DESIGNER_SIGNED        0
+
+/*
+	Register FPGA_DES_REV_1
+*/
+#define TCF_CLK_CTRL_FPGA_DES_REV_1         0x0010
+#define FPGA_DES_REV_1_MASK                 0xFFFFFFFFU
+#define FPGA_DES_REV_1_SHIFT                0
+#define FPGA_DES_REV_1_SIGNED               0
+
+/*
+	Register FPGA_DES_REV_2
+*/
+#define TCF_CLK_CTRL_FPGA_DES_REV_2         0x0018
+#define FPGA_DES_REV_2_MASK                 0xFFFFFFFFU
+#define FPGA_DES_REV_2_SHIFT                0
+#define FPGA_DES_REV_2_SIGNED               0
+
+/*
+	Register TCF_CORE_ID_REG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_ID_REG        0x0020
+#define TCF_CORE_ID_REG_CORE_CFG_MASK       0x0000FFFFU
+#define TCF_CORE_ID_REG_CORE_CFG_SHIFT      0
+#define TCF_CORE_ID_REG_CORE_CFG_SIGNED     0
+
+#define TCF_CORE_ID_REG_CORE_ID_MASK        0xFFFF0000U
+#define TCF_CORE_ID_REG_CORE_ID_SHIFT       16
+#define TCF_CORE_ID_REG_CORE_ID_SIGNED      0
+
+/*
+	Register TCF_CORE_REV_REG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_REV_REG       0x0028
+#define TCF_CORE_REV_REG_MAINT_MASK         0x000000FFU
+#define TCF_CORE_REV_REG_MAINT_SHIFT        0
+#define TCF_CORE_REV_REG_MAINT_SIGNED       0
+
+#define TCF_CORE_REV_REG_MINOR_MASK         0x0000FF00U
+#define TCF_CORE_REV_REG_MINOR_SHIFT        8
+#define TCF_CORE_REV_REG_MINOR_SIGNED       0
+
+#define TCF_CORE_REV_REG_MAJOR_MASK         0x00FF0000U
+#define TCF_CORE_REV_REG_MAJOR_SHIFT        16
+#define TCF_CORE_REV_REG_MAJOR_SIGNED       0
+
+#define TCF_CORE_REV_REG_DESIGNER_MASK      0xFF000000U
+#define TCF_CORE_REV_REG_DESIGNER_SHIFT     24
+#define TCF_CORE_REV_REG_DESIGNER_SIGNED    0
+
+/*
+	Register TCF_CORE_DES_REV_1
+*/
+#define TCF_CLK_CTRL_TCF_CORE_DES_REV_1     0x0030
+#define TCF_CORE_DES_REV_1_MASK             0xFFFFFFFFU
+#define TCF_CORE_DES_REV_1_SHIFT            0
+#define TCF_CORE_DES_REV_1_SIGNED           0
+
+/*
+	Register TCF_CORE_DES_REV_2
+*/
+#define TCF_CLK_CTRL_TCF_CORE_DES_REV_2     0x0038
+#define TCF_CORE_DES_REV_2_MASK             0xFFFFFFFFU
+#define TCF_CORE_DES_REV_2_SHIFT            0
+#define TCF_CORE_DES_REV_2_SIGNED           0
+
+
+/*
+ * The following register definitions are valid if register 0x28 has value 1.
+ */
+
+/*
+	Register ID
+*/
+#define TCF_CLK_CTRL_ID                     0x0000
+#define VARIANT_MASK                        0x0000FFFFU
+#define VARIANT_SHIFT                       0
+#define VARIANT_SIGNED                      0
+
+#define ID_MASK                             0xFFFF0000U
+#define ID_SHIFT                            16
+#define ID_SIGNED                           0
+
+/*
+	Register REL
+*/
+#define TCF_CLK_CTRL_REL                    0x0008
+#define MINOR_MASK                          0x0000FFFFU
+#define MINOR_SHIFT                         0
+#define MINOR_SIGNED                        0
+
+#define MAJOR_MASK                          0xFFFF0000U
+#define MAJOR_SHIFT                         16
+#define MAJOR_SIGNED                        0
+
+/*
+	Register CHANGE_SET
+*/
+#define TCF_CLK_CTRL_CHANGE_SET             0x0010
+#define SET_MASK                            0xFFFFFFFFU
+#define SET_SHIFT                           0
+#define SET_SIGNED                          0
+
+/*
+	Register USER_ID
+*/
+#define TCF_CLK_CTRL_USER_ID                0x0018
+#define USER_ID_MASK                        0x0000000FU
+#define USER_ID_SHIFT                       0
+#define USER_ID_SIGNED                      0
+
+/*
+	Register USER_BUILD
+*/
+#define TCF_CLK_CTRL_USER_BUILD             0x0020
+#define BUILD_MASK                          0xFFFFFFFFU
+#define BUILD_SHIFT                         0
+#define BUILD_SIGNED                        0
+
+/*
+	Register SW_IF_VERSION
+*/
+#define TCF_CLK_CTRL_SW_IF_VERSION          0x0028
+#define VERSION_MASK                        0x0000FFFFU
+#define VERSION_SHIFT                       0
+#define VERSION_SIGNED                      0
+
+/*
+ * The following register definitions are valid for all Apollo builds,
+ * even if some of the registers are not available for certain cores.
+ */
+
+/*
+	Register SCB_GENERAL_CONTROL
+*/
+#define TCF_CLK_CTRL_SCB_GENERAL_CONTROL    0x0040
+#define SCB_GC_TRANS_HALT_MASK              0x00000200U
+#define SCB_GC_TRANS_HALT_SHIFT             9
+#define SCB_GC_TRANS_HALT_SIGNED            0
+
+#define SCB_GC_CKD_REGS_MASK                0x00000100U
+#define SCB_GC_CKD_REGS_SHIFT               8
+#define SCB_GC_CKD_REGS_SIGNED              0
+
+#define SCB_GC_CKD_SLAVE_MASK               0x00000080U
+#define SCB_GC_CKD_SLAVE_SHIFT              7
+#define SCB_GC_CKD_SLAVE_SIGNED             0
+
+#define SCB_GC_CKD_MASTER_MASK              0x00000040U
+#define SCB_GC_CKD_MASTER_SHIFT             6
+#define SCB_GC_CKD_MASTER_SIGNED            0
+
+#define SCB_GC_CKD_XDATA_MASK               0x00000020U
+#define SCB_GC_CKD_XDATA_SHIFT              5
+#define SCB_GC_CKD_XDATA_SIGNED             0
+
+#define SCB_GC_SFR_REG_MASK                 0x00000010U
+#define SCB_GC_SFR_REG_SHIFT                4
+#define SCB_GC_SFR_REG_SIGNED               0
+
+#define SCB_GC_SFR_SLAVE_MASK               0x00000008U
+#define SCB_GC_SFR_SLAVE_SHIFT              3
+#define SCB_GC_SFR_SLAVE_SIGNED             0
+
+#define SCB_GC_SFR_MASTER_MASK              0x00000004U
+#define SCB_GC_SFR_MASTER_SHIFT             2
+#define SCB_GC_SFR_MASTER_SIGNED            0
+
+#define SCB_GC_SFR_DET_DATA_MASK            0x00000002U
+#define SCB_GC_SFR_DET_DATA_SHIFT           1
+#define SCB_GC_SFR_DET_DATA_SIGNED          0
+
+#define SCB_GC_SFR_GEN_DATA_MASK            0x00000001U
+#define SCB_GC_SFR_GEN_DATA_SHIFT           0
+#define SCB_GC_SFR_GEN_DATA_SIGNED          0
+
+/*
+	Register SCB_MASTER_READ_COUNT
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_READ_COUNT  0x0048
+#define MASTER_READ_COUNT_MASK              0x0000FFFFU
+#define MASTER_READ_COUNT_SHIFT             0
+#define MASTER_READ_COUNT_SIGNED            0
+
+/*
+	Register SCB_MASTER_READ_DATA
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_READ_DATA   0x0050
+#define MASTER_READ_DATA_MASK               0x000000FFU
+#define MASTER_READ_DATA_SHIFT              0
+#define MASTER_READ_DATA_SIGNED             0
+
+/*
+	Register SCB_MASTER_ADDRESS
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_ADDRESS     0x0058
+#define SCB_MASTER_ADDRESS_MASK             0x000003FFU
+#define SCB_MASTER_ADDRESS_SHIFT            0
+#define SCB_MASTER_ADDRESS_SIGNED           0
+
+/*
+	Register SCB_MASTER_WRITE_DATA
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_WRITE_DATA  0x0060
+#define MASTER_WRITE_DATA_MASK              0x000000FFU
+#define MASTER_WRITE_DATA_SHIFT             0
+#define MASTER_WRITE_DATA_SIGNED            0
+
+/*
+	Register SCB_MASTER_WRITE_COUNT
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_WRITE_COUNT 0x0068
+#define MASTER_WRITE_COUNT_MASK             0x0000FFFFU
+#define MASTER_WRITE_COUNT_SHIFT            0
+#define MASTER_WRITE_COUNT_SIGNED           0
+
+/*
+	Register SCB_BUS_SELECT
+*/
+#define TCF_CLK_CTRL_SCB_BUS_SELECT         0x0070
+#define BUS_SELECT_MASK                     0x00000003U
+#define BUS_SELECT_SHIFT                    0
+#define BUS_SELECT_SIGNED                   0
+
+/*
+	Register SCB_MASTER_FILL_STATUS
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_FILL_STATUS 0x0078
+#define MASTER_WRITE_FIFO_EMPTY_MASK        0x00000008U
+#define MASTER_WRITE_FIFO_EMPTY_SHIFT       3
+#define MASTER_WRITE_FIFO_EMPTY_SIGNED      0
+
+#define MASTER_WRITE_FIFO_FULL_MASK         0x00000004U
+#define MASTER_WRITE_FIFO_FULL_SHIFT        2
+#define MASTER_WRITE_FIFO_FULL_SIGNED       0
+
+#define MASTER_READ_FIFO_EMPTY_MASK         0x00000002U
+#define MASTER_READ_FIFO_EMPTY_SHIFT        1
+#define MASTER_READ_FIFO_EMPTY_SIGNED       0
+
+#define MASTER_READ_FIFO_FULL_MASK          0x00000001U
+#define MASTER_READ_FIFO_FULL_SHIFT         0
+#define MASTER_READ_FIFO_FULL_SIGNED        0
+
+/*
+	Register CLK_AND_RST_CTRL
+*/
+#define TCF_CLK_CTRL_CLK_AND_RST_CTRL       0x0080
+#define GLB_CLKG_EN_MASK                    0x00020000U
+#define GLB_CLKG_EN_SHIFT                   17
+#define GLB_CLKG_EN_SIGNED                  0
+
+#define CLK_GATE_CNTL_MASK                  0x00010000U
+#define CLK_GATE_CNTL_SHIFT                 16
+#define CLK_GATE_CNTL_SIGNED                0
+
+#define DUT_DCM_RESETN_MASK                 0x00000400U
+#define DUT_DCM_RESETN_SHIFT                10
+#define DUT_DCM_RESETN_SIGNED               0
+
+#define MEM_RESYNC_BYPASS_MASK              0x00000200U
+#define MEM_RESYNC_BYPASS_SHIFT             9
+#define MEM_RESYNC_BYPASS_SIGNED            0
+
+#define SYS_RESYNC_BYPASS_MASK              0x00000100U
+#define SYS_RESYNC_BYPASS_SHIFT             8
+#define SYS_RESYNC_BYPASS_SIGNED            0
+
+#define SCB_RESETN_MASK                     0x00000010U
+#define SCB_RESETN_SHIFT                    4
+#define SCB_RESETN_SIGNED                   0
+
+#define PDP2_RESETN_MASK                    0x00000008U
+#define PDP2_RESETN_SHIFT                   3
+#define PDP2_RESETN_SIGNED                  0
+
+#define PDP1_RESETN_MASK                    0x00000004U
+#define PDP1_RESETN_SHIFT                   2
+#define PDP1_RESETN_SIGNED                  0
+
+#define DDR_RESETN_MASK                     0x00000002U
+#define DDR_RESETN_SHIFT                    1
+#define DDR_RESETN_SIGNED                   0
+
+#define DUT_RESETN_MASK                     0x00000001U
+#define DUT_RESETN_SHIFT                    0
+#define DUT_RESETN_SIGNED                   0
+
+/*
+	Register TEST_REG_OUT
+*/
+#define TCF_CLK_CTRL_TEST_REG_OUT           0x0088
+#define TEST_REG_OUT_MASK                   0xFFFFFFFFU
+#define TEST_REG_OUT_SHIFT                  0
+#define TEST_REG_OUT_SIGNED                 0
+
+/*
+	Register TEST_REG_IN
+*/
+#define TCF_CLK_CTRL_TEST_REG_IN            0x0090
+#define TEST_REG_IN_MASK                    0xFFFFFFFFU
+#define TEST_REG_IN_SHIFT                   0
+#define TEST_REG_IN_SIGNED                  0
+
+/*
+	Register TEST_CTRL
+*/
+#define TCF_CLK_CTRL_TEST_CTRL              0x0098
+#define PCI_TEST_OFFSET_MASK                0xF8000000U
+#define PCI_TEST_OFFSET_SHIFT               27
+#define PCI_TEST_OFFSET_SIGNED              0
+
+#define PDP1_HOST_MEM_SELECT_MASK           0x00000200U
+#define PDP1_HOST_MEM_SELECT_SHIFT          9
+#define PDP1_HOST_MEM_SELECT_SIGNED         0
+
+#define HOST_PHY_MODE_MASK                  0x00000100U
+#define HOST_PHY_MODE_SHIFT                 8
+#define HOST_PHY_MODE_SIGNED                0
+
+#define HOST_ONLY_MODE_MASK                 0x00000080U
+#define HOST_ONLY_MODE_SHIFT                7
+#define HOST_ONLY_MODE_SIGNED               0
+
+#define PCI_TEST_MODE_MASK                  0x00000040U
+#define PCI_TEST_MODE_SHIFT                 6
+#define PCI_TEST_MODE_SIGNED                0
+
+#define TURN_OFF_DDR_MASK                   0x00000020U
+#define TURN_OFF_DDR_SHIFT                  5
+#define TURN_OFF_DDR_SIGNED                 0
+
+#define SYS_RD_CLK_INV_MASK                 0x00000010U
+#define SYS_RD_CLK_INV_SHIFT                4
+#define SYS_RD_CLK_INV_SIGNED               0
+
+#define MEM_REQ_CLK_INV_MASK                0x00000008U
+#define MEM_REQ_CLK_INV_SHIFT               3
+#define MEM_REQ_CLK_INV_SIGNED              0
+
+#define BURST_SPLIT_MASK                    0x00000004U
+#define BURST_SPLIT_SHIFT                   2
+#define BURST_SPLIT_SIGNED                  0
+
+#define CLK_INVERSION_MASK                  0x00000002U
+#define CLK_INVERSION_SHIFT                 1
+#define CLK_INVERSION_SIGNED                0
+
+#define ADDRESS_FORCE_MASK                  0x00000001U
+#define ADDRESS_FORCE_SHIFT                 0
+#define ADDRESS_FORCE_SIGNED                0
+
+/*
+	Register CLEAR_HOST_MEM_SIG
+*/
+#define TCF_CLK_CTRL_CLEAR_HOST_MEM_SIG     0x00A0
+#define SIGNATURE_TAG_ID_MASK               0x00000F00U
+#define SIGNATURE_TAG_ID_SHIFT              8
+#define SIGNATURE_TAG_ID_SIGNED             0
+
+#define CLEAR_HOST_MEM_SIGNATURE_MASK       0x00000001U
+#define CLEAR_HOST_MEM_SIGNATURE_SHIFT      0
+#define CLEAR_HOST_MEM_SIGNATURE_SIGNED     0
+
+/*
+	Register HOST_MEM_SIGNATURE
+*/
+#define TCF_CLK_CTRL_HOST_MEM_SIGNATURE     0x00A8
+#define HOST_MEM_SIGNATURE_MASK             0xFFFFFFFFU
+#define HOST_MEM_SIGNATURE_SHIFT            0
+#define HOST_MEM_SIGNATURE_SIGNED           0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define TCF_CLK_CTRL_INTERRUPT_STATUS       0x00C8
+#define INTERRUPT_MASTER_STATUS_MASK        0x80000000U
+#define INTERRUPT_MASTER_STATUS_SHIFT       31
+#define INTERRUPT_MASTER_STATUS_SIGNED      0
+
+#define OTHER_INTS_MASK                     0x7FFE0000U
+#define OTHER_INTS_SHIFT                    17
+#define OTHER_INTS_SIGNED                   0
+
+#define HOST_MST_NORESPONSE_MASK            0x00010000U
+#define HOST_MST_NORESPONSE_SHIFT           16
+#define HOST_MST_NORESPONSE_SIGNED          0
+
+#define PDP2_INT_MASK                       0x00008000U
+#define PDP2_INT_SHIFT                      15
+#define PDP2_INT_SIGNED                     0
+
+#define PDP1_INT_MASK                       0x00004000U
+#define PDP1_INT_SHIFT                      14
+#define PDP1_INT_SIGNED                     0
+
+#define EXT_INT_MASK                        0x00002000U
+#define EXT_INT_SHIFT                       13
+#define EXT_INT_SIGNED                      0
+
+#define SCB_MST_HLT_BIT_MASK                0x00001000U
+#define SCB_MST_HLT_BIT_SHIFT               12
+#define SCB_MST_HLT_BIT_SIGNED              0
+
+#define SCB_SLV_EVENT_MASK                  0x00000800U
+#define SCB_SLV_EVENT_SHIFT                 11
+#define SCB_SLV_EVENT_SIGNED                0
+
+#define SCB_TDONE_RX_MASK                   0x00000400U
+#define SCB_TDONE_RX_SHIFT                  10
+#define SCB_TDONE_RX_SIGNED                 0
+
+#define SCB_SLV_WT_RD_DAT_MASK              0x00000200U
+#define SCB_SLV_WT_RD_DAT_SHIFT             9
+#define SCB_SLV_WT_RD_DAT_SIGNED            0
+
+#define SCB_SLV_WT_PRV_RD_MASK              0x00000100U
+#define SCB_SLV_WT_PRV_RD_SHIFT             8
+#define SCB_SLV_WT_PRV_RD_SIGNED            0
+
+#define SCB_SLV_WT_WR_DAT_MASK              0x00000080U
+#define SCB_SLV_WT_WR_DAT_SHIFT             7
+#define SCB_SLV_WT_WR_DAT_SIGNED            0
+
+#define SCB_MST_WT_RD_DAT_MASK              0x00000040U
+#define SCB_MST_WT_RD_DAT_SHIFT             6
+#define SCB_MST_WT_RD_DAT_SIGNED            0
+
+#define SCB_ADD_ACK_ERR_MASK                0x00000020U
+#define SCB_ADD_ACK_ERR_SHIFT               5
+#define SCB_ADD_ACK_ERR_SIGNED              0
+
+#define SCB_WR_ACK_ERR_MASK                 0x00000010U
+#define SCB_WR_ACK_ERR_SHIFT                4
+#define SCB_WR_ACK_ERR_SIGNED               0
+
+#define SCB_SDAT_LO_TIM_MASK                0x00000008U
+#define SCB_SDAT_LO_TIM_SHIFT               3
+#define SCB_SDAT_LO_TIM_SIGNED              0
+
+#define SCB_SCLK_LO_TIM_MASK                0x00000004U
+#define SCB_SCLK_LO_TIM_SHIFT               2
+#define SCB_SCLK_LO_TIM_SIGNED              0
+
+#define SCB_UNEX_START_BIT_MASK             0x00000002U
+#define SCB_UNEX_START_BIT_SHIFT            1
+#define SCB_UNEX_START_BIT_SIGNED           0
+
+#define SCB_BUS_INACTIVE_MASK               0x00000001U
+#define SCB_BUS_INACTIVE_SHIFT              0
+#define SCB_BUS_INACTIVE_SIGNED             0
+
+/*
+	Register INTERRUPT_OP_CFG
+*/
+#define TCF_CLK_CTRL_INTERRUPT_OP_CFG       0x00D0
+#define PULSE_NLEVEL_MASK                   0x80000000U
+#define PULSE_NLEVEL_SHIFT                  31
+#define PULSE_NLEVEL_SIGNED                 0
+
+#define INT_SENSE_MASK                      0x40000000U
+#define INT_SENSE_SHIFT                     30
+#define INT_SENSE_SIGNED                    0
+
+#define INTERRUPT_DEST_MASK                 0x0000000FU
+#define INTERRUPT_DEST_SHIFT                0
+#define INTERRUPT_DEST_SIGNED               0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define TCF_CLK_CTRL_INTERRUPT_ENABLE       0x00D8
+#define INTERRUPT_MASTER_ENABLE_MASK        0x80000000U
+#define INTERRUPT_MASTER_ENABLE_SHIFT       31
+#define INTERRUPT_MASTER_ENABLE_SIGNED      0
+
+#define INTERRUPT_ENABLE_MASK               0x7FFFFFFFU
+#define INTERRUPT_ENABLE_SHIFT              0
+#define INTERRUPT_ENABLE_SIGNED             0
+
+/*
+	Register INTERRUPT_CLEAR
+*/
+#define TCF_CLK_CTRL_INTERRUPT_CLEAR        0x00E0
+#define INTERRUPT_MASTER_CLEAR_MASK         0x80000000U
+#define INTERRUPT_MASTER_CLEAR_SHIFT        31
+#define INTERRUPT_MASTER_CLEAR_SIGNED       0
+
+#define INTERRUPT_CLEAR_MASK                0x7FFFFFFFU
+#define INTERRUPT_CLEAR_SHIFT               0
+#define INTERRUPT_CLEAR_SIGNED              0
+
+/*
+	Register YCC_RGB_CTRL
+*/
+#define TCF_CLK_CTRL_YCC_RGB_CTRL           0x00E8
+#define RGB_CTRL1_MASK                      0x000001FFU
+#define RGB_CTRL1_SHIFT                     0
+#define RGB_CTRL1_SIGNED                    0
+
+#define RGB_CTRL2_MASK                      0x01FF0000U
+#define RGB_CTRL2_SHIFT                     16
+#define RGB_CTRL2_SIGNED                    0
+
+/*
+	Register EXP_BRD_CTRL
+*/
+#define TCF_CLK_CTRL_EXP_BRD_CTRL           0x00F8
+#define PDP1_DATA_EN_MASK                   0x00000003U
+#define PDP1_DATA_EN_SHIFT                  0
+#define PDP1_DATA_EN_SIGNED                 0
+
+#define PDP2_DATA_EN_MASK                   0x00000030U
+#define PDP2_DATA_EN_SHIFT                  4
+#define PDP2_DATA_EN_SIGNED                 0
+
+#define EXP_BRD_OUTPUT_MASK                 0xFFFFFF00U
+#define EXP_BRD_OUTPUT_SHIFT                8
+#define EXP_BRD_OUTPUT_SIGNED               0
+
+/*
+	Register HOSTIF_CONTROL
+*/
+#define TCF_CLK_CTRL_HOSTIF_CONTROL         0x0100
+#define HOSTIF_CTRL_MASK                    0x000000FFU
+#define HOSTIF_CTRL_SHIFT                   0
+#define HOSTIF_CTRL_SIGNED                  0
+
+/*
+	Register DUT_CONTROL_1
+*/
+#define TCF_CLK_CTRL_DUT_CONTROL_1          0x0108
+#define DUT_CTRL_1_MASK                     0xFFFFFFFFU
+#define DUT_CTRL_1_SHIFT                    0
+#define DUT_CTRL_1_SIGNED                   0
+
+/* TC ES2 additional needs those: */
+#define DUT_CTRL_TEST_MODE_SHIFT            0
+#define DUT_CTRL_TEST_MODE_MASK             0x3
+
+#define DUT_CTRL_VCC_0V9EN                  (1<<12)
+#define DUT_CTRL_VCC_1V8EN                  (1<<13)
+#define DUT_CTRL_VCC_IO_INH                 (1<<14)
+#define DUT_CTRL_VCC_CORE_INH               (1<<15)
+
+/*
+	Register DUT_STATUS_1
+*/
+#define TCF_CLK_CTRL_DUT_STATUS_1           0x0110
+#define DUT_STATUS_1_MASK                   0xFFFFFFFFU
+#define DUT_STATUS_1_SHIFT                  0
+#define DUT_STATUS_1_SIGNED                 0
+
+/*
+	Register DUT_CTRL_NOT_STAT_1
+*/
+#define TCF_CLK_CTRL_DUT_CTRL_NOT_STAT_1    0x0118
+#define DUT_STAT_NOT_CTRL_1_MASK            0xFFFFFFFFU
+#define DUT_STAT_NOT_CTRL_1_SHIFT           0
+#define DUT_STAT_NOT_CTRL_1_SIGNED          0
+
+/*
+	Register DUT_CONTROL_2
+*/
+#define TCF_CLK_CTRL_DUT_CONTROL_2          0x0120
+#define DUT_CTRL_2_MASK                     0xFFFFFFFFU
+#define DUT_CTRL_2_SHIFT                    0
+#define DUT_CTRL_2_SIGNED                   0
+
+/*
+	Register DUT_STATUS_2
+*/
+#define TCF_CLK_CTRL_DUT_STATUS_2           0x0128
+#define DUT_STATUS_2_MASK                   0xFFFFFFFFU
+#define DUT_STATUS_2_SHIFT                  0
+#define DUT_STATUS_2_SIGNED                 0
+
+/*
+	Register DUT_CTRL_NOT_STAT_2
+*/
+#define TCF_CLK_CTRL_DUT_CTRL_NOT_STAT_2    0x0130
+#define DUT_CTRL_NOT_STAT_2_MASK            0xFFFFFFFFU
+#define DUT_CTRL_NOT_STAT_2_SHIFT           0
+#define DUT_CTRL_NOT_STAT_2_SIGNED          0
+
+/*
+	Register BUS_CAP_BASE_ADDR
+*/
+#define TCF_CLK_CTRL_BUS_CAP_BASE_ADDR      0x0138
+#define BUS_CAP_BASE_ADDR_MASK              0xFFFFFFFFU
+#define BUS_CAP_BASE_ADDR_SHIFT             0
+#define BUS_CAP_BASE_ADDR_SIGNED            0
+
+/*
+	Register BUS_CAP_ENABLE
+*/
+#define TCF_CLK_CTRL_BUS_CAP_ENABLE         0x0140
+#define BUS_CAP_ENABLE_MASK                 0x00000001U
+#define BUS_CAP_ENABLE_SHIFT                0
+#define BUS_CAP_ENABLE_SIGNED               0
+
+/*
+	Register BUS_CAP_COUNT
+*/
+#define TCF_CLK_CTRL_BUS_CAP_COUNT          0x0148
+#define BUS_CAP_COUNT_MASK                  0xFFFFFFFFU
+#define BUS_CAP_COUNT_SHIFT                 0
+#define BUS_CAP_COUNT_SIGNED                0
+
+/*
+	Register DCM_LOCK_STATUS
+*/
+#define TCF_CLK_CTRL_DCM_LOCK_STATUS        0x0150
+#define DCM_LOCK_STATUS_MASK                0x00000007U
+#define DCM_LOCK_STATUS_SHIFT               0
+#define DCM_LOCK_STATUS_SIGNED              0
+
+/*
+	Register AUX_DUT_RESETNS
+*/
+#define TCF_CLK_CTRL_AUX_DUT_RESETNS        0x0158
+#define AUX_DUT_RESETNS_MASK                0x0000000FU
+#define AUX_DUT_RESETNS_SHIFT               0
+#define AUX_DUT_RESETNS_SIGNED              0
+
+/*
+	Register TCF_SPI_MST_ADDR_RDNWR
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR 0x0160
+#define TCF_SPI_MST_ADDR_MASK               0x0003FFFFU
+#define TCF_SPI_MST_ADDR_SHIFT              0
+#define TCF_SPI_MST_ADDR_SIGNED             0
+
+#define TCF_SPI_MST_RDNWR_MASK              0x00040000U
+#define TCF_SPI_MST_RDNWR_SHIFT             18
+#define TCF_SPI_MST_RDNWR_SIGNED            0
+
+#define TCF_SPI_MST_SLAVE_ID_MASK           0x00080000U
+#define TCF_SPI_MST_SLAVE_ID_SHIFT          19
+#define TCF_SPI_MST_SLAVE_ID_SIGNED         0
+
+#define TCF_SPI_MST_MASTER_ID_MASK          0x00300000U
+#define TCF_SPI_MST_MASTER_ID_SHIFT         20
+#define TCF_SPI_MST_MASTER_ID_SIGNED        0
+
+/*
+	Register TCF_SPI_MST_WDATA
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_WDATA      0x0168
+#define TCF_SPI_MST_WDATA_MASK              0xFFFFFFFFU
+#define TCF_SPI_MST_WDATA_SHIFT             0
+#define TCF_SPI_MST_WDATA_SIGNED            0
+
+/*
+	Register TCF_SPI_MST_RDATA
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_RDATA      0x0170
+#define TCF_SPI_MST_RDATA_MASK              0xFFFFFFFFU
+#define TCF_SPI_MST_RDATA_SHIFT             0
+#define TCF_SPI_MST_RDATA_SIGNED            0
+
+/*
+	Register TCF_SPI_MST_STATUS
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_STATUS     0x0178
+#define TCF_SPI_MST_STATUS_MASK             0x0000000FU
+#define TCF_SPI_MST_STATUS_SHIFT            0
+#define TCF_SPI_MST_STATUS_SIGNED           0
+
+/*
+	Register TCF_SPI_MST_GO
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_GO         0x0180
+#define TCF_SPI_MST_GO_MASK                 0x00000001U
+#define TCF_SPI_MST_GO_SHIFT                0
+#define TCF_SPI_MST_GO_SIGNED               0
+
+/*
+	Register EXT_SIG_CTRL
+*/
+#define TCF_CLK_CTRL_EXT_SIG_CTRL           0x0188
+#define EXT_SYS_REQ_SIG_START_MASK          0x00000001U
+#define EXT_SYS_REQ_SIG_START_SHIFT         0
+#define EXT_SYS_REQ_SIG_START_SIGNED        0
+
+#define EXT_SYS_RD_SIG_START_MASK           0x00000002U
+#define EXT_SYS_RD_SIG_START_SHIFT          1
+#define EXT_SYS_RD_SIG_START_SIGNED         0
+
+#define EXT_MEM_REQ_SIG_START_MASK          0x00000004U
+#define EXT_MEM_REQ_SIG_START_SHIFT         2
+#define EXT_MEM_REQ_SIG_START_SIGNED        0
+
+#define EXT_MEM_RD_SIG_START_MASK           0x00000008U
+#define EXT_MEM_RD_SIG_START_SHIFT          3
+#define EXT_MEM_RD_SIG_START_SIGNED         0
+
+/*
+	Register EXT_SYS_REQ_SIG
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_SIG        0x0190
+#define EXT_SYS_REQ_SIG_MASK                0xFFFFFFFFU
+#define EXT_SYS_REQ_SIG_SHIFT               0
+#define EXT_SYS_REQ_SIG_SIGNED              0
+
+/*
+	Register EXT_SYS_RD_SIG
+*/
+#define TCF_CLK_CTRL_EXT_SYS_RD_SIG         0x0198
+#define EXT_SYS_RD_SIG_MASK                 0xFFFFFFFFU
+#define EXT_SYS_RD_SIG_SHIFT                0
+#define EXT_SYS_RD_SIG_SIGNED               0
+
+/*
+	Register EXT_MEM_REQ_SIG
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_SIG        0x01A0
+#define EXT_MEM_REQ_SIG_MASK                0xFFFFFFFFU
+#define EXT_MEM_REQ_SIG_SHIFT               0
+#define EXT_MEM_REQ_SIG_SIGNED              0
+
+/*
+	Register EXT_MEM_RD_SIG
+*/
+#define TCF_CLK_CTRL_EXT_MEM_RD_SIG         0x01A8
+#define EXT_MEM_RD_SIG_MASK                 0xFFFFFFFFU
+#define EXT_MEM_RD_SIG_SHIFT                0
+#define EXT_MEM_RD_SIG_SIGNED               0
+
+/*
+	Register EXT_SYS_REQ_WR_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_WR_CNT     0x01B0
+#define EXT_SYS_REQ_WR_CNT_MASK             0xFFFFFFFFU
+#define EXT_SYS_REQ_WR_CNT_SHIFT            0
+#define EXT_SYS_REQ_WR_CNT_SIGNED           0
+
+/*
+	Register EXT_SYS_REQ_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_RD_CNT     0x01B8
+#define EXT_SYS_REQ_RD_CNT_MASK             0xFFFFFFFFU
+#define EXT_SYS_REQ_RD_CNT_SHIFT            0
+#define EXT_SYS_REQ_RD_CNT_SIGNED           0
+
+/*
+	Register EXT_SYS_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_RD_CNT         0x01C0
+#define EXT_SYS_RD_CNT_MASK                 0xFFFFFFFFU
+#define EXT_SYS_RD_CNT_SHIFT                0
+#define EXT_SYS_RD_CNT_SIGNED               0
+
+/*
+	Register EXT_MEM_REQ_WR_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_WR_CNT     0x01C8
+#define EXT_MEM_REQ_WR_CNT_MASK             0xFFFFFFFFU
+#define EXT_MEM_REQ_WR_CNT_SHIFT            0
+#define EXT_MEM_REQ_WR_CNT_SIGNED           0
+
+/*
+	Register EXT_MEM_REQ_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_RD_CNT     0x01D0
+#define EXT_MEM_REQ_RD_CNT_MASK             0xFFFFFFFFU
+#define EXT_MEM_REQ_RD_CNT_SHIFT            0
+#define EXT_MEM_REQ_RD_CNT_SIGNED           0
+
+/*
+	Register EXT_MEM_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_RD_CNT         0x01D8
+#define EXT_MEM_RD_CNT_MASK                 0xFFFFFFFFU
+#define EXT_MEM_RD_CNT_SHIFT                0
+#define EXT_MEM_RD_CNT_SIGNED               0
+
+/*
+	Register TCF_CORE_TARGET_BUILD_CFG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_TARGET_BUILD_CFG 0x01E0
+#define TCF_CORE_TARGET_BUILD_ID_MASK       0x000000FFU
+#define TCF_CORE_TARGET_BUILD_ID_SHIFT      0
+#define TCF_CORE_TARGET_BUILD_ID_SIGNED     0
+
+/*
+	Register MEM_THROUGH_SYS
+*/
+#define TCF_CLK_CTRL_MEM_THROUGH_SYS        0x01E8
+#define MEM_THROUGH_SYS_MASK                0x00000001U
+#define MEM_THROUGH_SYS_SHIFT               0
+#define MEM_THROUGH_SYS_SIGNED              0
+
+/*
+	Register HOST_PHY_OFFSET
+*/
+#define TCF_CLK_CTRL_HOST_PHY_OFFSET        0x01F0
+#define HOST_PHY_OFFSET_MASK                0xFFFFFFFFU
+#define HOST_PHY_OFFSET_SHIFT               0
+#define HOST_PHY_OFFSET_SIGNED              0
+
+/*
+	Register DEBUG_REG_SEL
+*/
+#define TCF_CLK_CTRL_DEBUG_REG_SEL          0x01F8
+#define DEBUG_REG_SELECT_MASK               0xFFFFFFFFU
+#define DEBUG_REG_SELECT_SHIFT              0
+#define DEBUG_REG_SELECT_SIGNED             0
+
+/*
+	Register DEBUG_REG
+*/
+#define TCF_CLK_CTRL_DEBUG_REG              0x0200
+#define DEBUG_REG_VALUE_MASK                0xFFFFFFFFU
+#define DEBUG_REG_VALUE_SHIFT               0
+#define DEBUG_REG_VALUE_SIGNED              0
+
+/*
+	Register JTAG_CTRL
+*/
+#define TCF_CLK_CTRL_JTAG_CTRL              0x0208
+#define JTAG_TRST_MASK                      0x00000001U
+#define JTAG_TRST_SHIFT                     0
+#define JTAG_TRST_SIGNED                    0
+
+#define JTAG_TMS_MASK                       0x00000002U
+#define JTAG_TMS_SHIFT                      1
+#define JTAG_TMS_SIGNED                     0
+
+#define JTAG_TCK_MASK                       0x00000004U
+#define JTAG_TCK_SHIFT                      2
+#define JTAG_TCK_SIGNED                     0
+
+#define JTAG_TDO_MASK                       0x00000008U
+#define JTAG_TDO_SHIFT                      3
+#define JTAG_TDO_SIGNED                     0
+
+#define JTAG_TDI_MASK                       0x00000010U
+#define JTAG_TDI_SHIFT                      4
+#define JTAG_TDI_SIGNED                     0
+
+#define JTAG_DASH_N_REG_MASK                0x40000000U
+#define JTAG_DASH_N_REG_SHIFT               30
+#define JTAG_DASH_N_REG_SIGNED              0
+
+#define JTAG_DISABLE_MASK                   0x80000000U
+#define JTAG_DISABLE_SHIFT                  31
+#define JTAG_DISABLE_SIGNED                 0
+
+/*
+	Register SAI_DEBUG_RDNWR
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_RDNWR        0x0300
+#define SAI_DEBUG_REG_ADDR_MASK             0x000001FFU
+#define SAI_DEBUG_REG_ADDR_SHIFT            0
+#define SAI_DEBUG_REG_ADDR_SIGNED           0
+
+#define SAI_DEBUG_REG_RDNWR_MASK            0x00000200U
+#define SAI_DEBUG_REG_RDNWR_SHIFT           9
+#define SAI_DEBUG_REG_RDNWR_SIGNED          0
+
+/*
+	Register SAI_DEBUG_WDATA
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_WDATA        0x0308
+#define SAI_DEBUG_REG_WDATA_MASK            0xFFFFFFFFU
+#define SAI_DEBUG_REG_WDATA_SHIFT           0
+#define SAI_DEBUG_REG_WDATA_SIGNED          0
+
+/*
+	Register SAI_DEBUG_RDATA
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_RDATA        0x0310
+#define SAI_DEBUG_REG_RDATA_MASK            0xFFFFFFFFU
+#define SAI_DEBUG_REG_RDATA_SHIFT           0
+#define SAI_DEBUG_REG_RDATA_SIGNED          0
+
+/*
+	Register SAI_DEBUG_GO
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_GO           0x0318
+#define SAI_DEBUG_REG_GO_MASK               0x00000001U
+#define SAI_DEBUG_REG_GO_SHIFT              0
+#define SAI_DEBUG_REG_GO_SIGNED             0
+
+/*
+	Register AUX_DUT_RESETS
+*/
+#define TCF_CLK_CTRL_AUX_DUT_RESETS         0x0320
+#define AUX_DUT_RESETS_MASK                 0x0000000FU
+#define AUX_DUT_RESETS_SHIFT                0
+#define AUX_DUT_RESETS_SIGNED               0
+
+/*
+	Register DUT_CLK_CTRL
+*/
+#define TCF_CLK_CTRL_DUT_CLK_CTRL           0x0328
+#define MEM_REQ_PHSE_MASK                   0x0000FFFFU
+#define MEM_REQ_PHSE_SHIFT                  0
+#define MEM_REQ_PHSE_SIGNED                 0
+
+/*
+	Register DUT_CLK_STATUS
+*/
+#define TCF_CLK_CTRL_DUT_CLK_STATUS         0x0330
+#define MEM_REQ_PHSE_SET_MASK               0x00000003U
+#define MEM_REQ_PHSE_SET_SHIFT              0
+#define MEM_REQ_PHSE_SET_SIGNED             0
+
+/*
+	Register DUT_CLK_INFO
+*/
+#define TCF_CLK_CTRL_DUT_CLK_INFO           0x0340
+#define CORE_MASK                           0x0000FFFFU
+#define CORE_SHIFT                          0
+#define CORE_SIGNED                         0
+
+#define MEM_MASK                            0xFFFF0000U
+#define MEM_SHIFT                           16
+#define MEM_SIGNED                          0
+
+/*
+	Register DUT_CLK_PHSE
+*/
+#define TCF_CLK_CTRL_DUT_CLK_PHSE           0x0348
+#define MEM_REQ_MASK                        0x0000FFFFU
+#define MEM_REQ_SHIFT                       0
+#define MEM_REQ_SIGNED                      0
+
+#define MEM_RD_MASK                         0xFFFF0000U
+#define MEM_RD_SHIFT                        16
+#define MEM_RD_SIGNED                       0
+
+#endif /* !defined(_TCF_CLK_CTRL_H_) */
+
+/*****************************************************************************
+ End of file (tcf_clk_ctrl.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_pll.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_pll.h
new file mode 100644
index 000000000000..71eaf924bbd6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_pll.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework PDP register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from tcf_pll.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_PLL_H_)
+#define _TCF_PLL_H_
+
+/*
+	Register PLL_DDR2_CLK0
+*/
+#define TCF_PLL_PLL_DDR2_CLK0               0x0000
+#define DDR2_PLL_CLK0_PHS_MASK              0x00300000U
+#define DDR2_PLL_CLK0_PHS_SHIFT             20
+#define DDR2_PLL_CLK0_PHS_SIGNED            0
+
+#define DDR2_PLL_CLK0_MS_MASK               0x00030000U
+#define DDR2_PLL_CLK0_MS_SHIFT              16
+#define DDR2_PLL_CLK0_MS_SIGNED             0
+
+#define DDR2_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define DDR2_PLL_CLK0_FREQ_SHIFT            0
+#define DDR2_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_DDR2_CLK1TO5
+*/
+#define TCF_PLL_PLL_DDR2_CLK1TO5            0x0008
+#define DDR2_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define DDR2_PLL_CLK1TO5_PHS_SHIFT          20
+#define DDR2_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define DDR2_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define DDR2_PLL_CLK1TO5_MS_SHIFT           10
+#define DDR2_PLL_CLK1TO5_MS_SIGNED          0
+
+#define DDR2_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define DDR2_PLL_CLK1TO5_FREQ_SHIFT         0
+#define DDR2_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_DDR2_DRP_GO
+*/
+#define TCF_PLL_PLL_DDR2_DRP_GO             0x0010
+#define PLL_DDR2_DRP_GO_MASK                0x00000001U
+#define PLL_DDR2_DRP_GO_SHIFT               0
+#define PLL_DDR2_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_PDP_CLK0
+*/
+#define TCF_PLL_PLL_PDP_CLK0                0x0018
+#define PDP_PLL_CLK0_PHS_MASK               0x00300000U
+#define PDP_PLL_CLK0_PHS_SHIFT              20
+#define PDP_PLL_CLK0_PHS_SIGNED             0
+
+#define PDP_PLL_CLK0_MS_MASK                0x00030000U
+#define PDP_PLL_CLK0_MS_SHIFT               16
+#define PDP_PLL_CLK0_MS_SIGNED              0
+
+#define PDP_PLL_CLK0_FREQ_MASK              0x000001FFU
+#define PDP_PLL_CLK0_FREQ_SHIFT             0
+#define PDP_PLL_CLK0_FREQ_SIGNED            0
+
+/*
+	Register PLL_PDP_CLK1TO5
+*/
+#define TCF_PLL_PLL_PDP_CLK1TO5             0x0020
+#define PDP_PLL_CLK1TO5_PHS_MASK            0x3FF00000U
+#define PDP_PLL_CLK1TO5_PHS_SHIFT           20
+#define PDP_PLL_CLK1TO5_PHS_SIGNED          0
+
+#define PDP_PLL_CLK1TO5_MS_MASK             0x000FFC00U
+#define PDP_PLL_CLK1TO5_MS_SHIFT            10
+#define PDP_PLL_CLK1TO5_MS_SIGNED           0
+
+#define PDP_PLL_CLK1TO5_FREQ_MASK           0x000003FFU
+#define PDP_PLL_CLK1TO5_FREQ_SHIFT          0
+#define PDP_PLL_CLK1TO5_FREQ_SIGNED         0
+
+/*
+	Register PLL_PDP_DRP_GO
+*/
+#define TCF_PLL_PLL_PDP_DRP_GO              0x0028
+#define PLL_PDP_DRP_GO_MASK                 0x00000001U
+#define PLL_PDP_DRP_GO_SHIFT                0
+#define PLL_PDP_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_PDP2_CLK0
+*/
+#define TCF_PLL_PLL_PDP2_CLK0               0x0030
+#define PDP2_PLL_CLK0_PHS_MASK              0x00300000U
+#define PDP2_PLL_CLK0_PHS_SHIFT             20
+#define PDP2_PLL_CLK0_PHS_SIGNED            0
+
+#define PDP2_PLL_CLK0_MS_MASK               0x00030000U
+#define PDP2_PLL_CLK0_MS_SHIFT              16
+#define PDP2_PLL_CLK0_MS_SIGNED             0
+
+#define PDP2_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define PDP2_PLL_CLK0_FREQ_SHIFT            0
+#define PDP2_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_PDP2_CLK1TO5
+*/
+#define TCF_PLL_PLL_PDP2_CLK1TO5            0x0038
+#define PDP2_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define PDP2_PLL_CLK1TO5_PHS_SHIFT          20
+#define PDP2_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define PDP2_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define PDP2_PLL_CLK1TO5_MS_SHIFT           10
+#define PDP2_PLL_CLK1TO5_MS_SIGNED          0
+
+#define PDP2_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define PDP2_PLL_CLK1TO5_FREQ_SHIFT         0
+#define PDP2_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_PDP2_DRP_GO
+*/
+#define TCF_PLL_PLL_PDP2_DRP_GO             0x0040
+#define PLL_PDP2_DRP_GO_MASK                0x00000001U
+#define PLL_PDP2_DRP_GO_SHIFT               0
+#define PLL_PDP2_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_CORE_CLK0
+*/
+#define TCF_PLL_PLL_CORE_CLK0               0x0048
+#define CORE_PLL_CLK0_PHS_MASK              0x00300000U
+#define CORE_PLL_CLK0_PHS_SHIFT             20
+#define CORE_PLL_CLK0_PHS_SIGNED            0
+
+#define CORE_PLL_CLK0_MS_MASK               0x00030000U
+#define CORE_PLL_CLK0_MS_SHIFT              16
+#define CORE_PLL_CLK0_MS_SIGNED             0
+
+#define CORE_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define CORE_PLL_CLK0_FREQ_SHIFT            0
+#define CORE_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_CORE_CLK1TO5
+*/
+#define TCF_PLL_PLL_CORE_CLK1TO5            0x0050
+#define CORE_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define CORE_PLL_CLK1TO5_PHS_SHIFT          20
+#define CORE_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define CORE_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define CORE_PLL_CLK1TO5_MS_SHIFT           10
+#define CORE_PLL_CLK1TO5_MS_SIGNED          0
+
+#define CORE_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define CORE_PLL_CLK1TO5_FREQ_SHIFT         0
+#define CORE_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_CORE_DRP_GO
+*/
+#define TCF_PLL_PLL_CORE_DRP_GO             0x0058
+#define PLL_CORE_DRP_GO_MASK                0x00000001U
+#define PLL_CORE_DRP_GO_SHIFT               0
+#define PLL_CORE_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_SYSIF_CLK0
+*/
+#define TCF_PLL_PLL_SYSIF_CLK0              0x0060
+#define SYSIF_PLL_CLK0_PHS_MASK             0x00300000U
+#define SYSIF_PLL_CLK0_PHS_SHIFT            20
+#define SYSIF_PLL_CLK0_PHS_SIGNED           0
+
+#define SYSIF_PLL_CLK0_MS_MASK              0x00030000U
+#define SYSIF_PLL_CLK0_MS_SHIFT             16
+#define SYSIF_PLL_CLK0_MS_SIGNED            0
+
+#define SYSIF_PLL_CLK0_FREQ_MASK            0x000001FFU
+#define SYSIF_PLL_CLK0_FREQ_SHIFT           0
+#define SYSIF_PLL_CLK0_FREQ_SIGNED          0
+
+/*
+	Register PLL_SYSIF_CLK1TO5
+*/
+#define TCF_PLL_PLL_SYSIF_CLK1TO5           0x0068
+#define SYSIF_PLL_CLK1TO5_PHS_MASK          0x3FF00000U
+#define SYSIF_PLL_CLK1TO5_PHS_SHIFT         20
+#define SYSIF_PLL_CLK1TO5_PHS_SIGNED        0
+
+#define SYSIF_PLL_CLK1TO5_MS_MASK           0x000FFC00U
+#define SYSIF_PLL_CLK1TO5_MS_SHIFT          10
+#define SYSIF_PLL_CLK1TO5_MS_SIGNED         0
+
+#define SYSIF_PLL_CLK1TO5_FREQ_MASK         0x000003FFU
+#define SYSIF_PLL_CLK1TO5_FREQ_SHIFT        0
+#define SYSIF_PLL_CLK1TO5_FREQ_SIGNED       0
+
+/*
+	Register PLL_SYS_DRP_GO
+*/
+#define TCF_PLL_PLL_SYS_DRP_GO              0x0070
+#define PLL_SYS_DRP_GO_MASK                 0x00000001U
+#define PLL_SYS_DRP_GO_SHIFT                0
+#define PLL_SYS_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_MEMIF_CLK0
+*/
+#define TCF_PLL_PLL_MEMIF_CLK0              0x0078
+#define MEMIF_PLL_CLK0_PHS_MASK             0x00300000U
+#define MEMIF_PLL_CLK0_PHS_SHIFT            20
+#define MEMIF_PLL_CLK0_PHS_SIGNED           0
+
+#define MEMIF_PLL_CLK0_MS_MASK              0x00030000U
+#define MEMIF_PLL_CLK0_MS_SHIFT             16
+#define MEMIF_PLL_CLK0_MS_SIGNED            0
+
+#define MEMIF_PLL_CLK0_FREQ_MASK            0x000001FFU
+#define MEMIF_PLL_CLK0_FREQ_SHIFT           0
+#define MEMIF_PLL_CLK0_FREQ_SIGNED          0
+
+/*
+	Register PLL_MEMIF_CLK1TO5
+*/
+#define TCF_PLL_PLL_MEMIF_CLK1TO5           0x0080
+#define MEMIF_PLL_CLK1TO5_PHS_MASK          0x3FF00000U
+#define MEMIF_PLL_CLK1TO5_PHS_SHIFT         20
+#define MEMIF_PLL_CLK1TO5_PHS_SIGNED        0
+
+#define MEMIF_PLL_CLK1TO5_MS_MASK           0x000FFC00U
+#define MEMIF_PLL_CLK1TO5_MS_SHIFT          10
+#define MEMIF_PLL_CLK1TO5_MS_SIGNED         0
+
+#define MEMIF_PLL_CLK1TO5_FREQ_MASK         0x000003FFU
+#define MEMIF_PLL_CLK1TO5_FREQ_SHIFT        0
+#define MEMIF_PLL_CLK1TO5_FREQ_SIGNED       0
+
+/*
+	Register PLL_MEM_DRP_GO
+*/
+#define TCF_PLL_PLL_MEM_DRP_GO              0x0088
+#define PLL_MEM_DRP_GO_MASK                 0x00000001U
+#define PLL_MEM_DRP_GO_SHIFT                0
+#define PLL_MEM_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_ALL_DRP_GO
+*/
+#define TCF_PLL_PLL_ALL_DRP_GO              0x0090
+#define PLL_ALL_DRP_GO_MASK                 0x00000001U
+#define PLL_ALL_DRP_GO_SHIFT                0
+#define PLL_ALL_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_DRP_STATUS
+*/
+#define TCF_PLL_PLL_DRP_STATUS              0x0098
+#define PLL_LOCKS_MASK                      0x00003F00U
+#define PLL_LOCKS_SHIFT                     8
+#define PLL_LOCKS_SIGNED                    0
+
+#define PLL_DRP_GOOD_MASK                   0x0000003FU
+#define PLL_DRP_GOOD_SHIFT                  0
+#define PLL_DRP_GOOD_SIGNED                 0
+
+#endif /* !defined(_TCF_PLL_H_) */
+
+/*****************************************************************************
+ End of file (tcf_pll.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_rgbpdp_regs.h b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_rgbpdp_regs.h
new file mode 100644
index 000000000000..e87ba6152411
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/system/rgx_tc/tcf_rgbpdp_regs.h
@@ -0,0 +1,559 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework PDP register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from: tcf_rgbpdp_regs.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_RGBPDP_REGS_H_)
+#define _TCF_RGBPDP_REGS_H_
+
+/*
+	Register PVR_TCF_RGBPDP_STR1SURF
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STR1SURF  0x0000
+#define STR1HEIGHT_MASK                     0x000007FFU
+#define STR1HEIGHT_SHIFT                    0
+#define STR1HEIGHT_SIGNED                   0
+
+#define STR1WIDTH_MASK                      0x003FF800U
+#define STR1WIDTH_SHIFT                     11
+#define STR1WIDTH_SIGNED                    0
+
+#define STR1PIXFMT_MASK                     0x0F000000U
+#define STR1PIXFMT_SHIFT                    24
+#define STR1PIXFMT_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_STR1ADDRCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL 0x0004
+#define STR1BASE_MASK                       0x03FFFFFFU
+#define STR1BASE_SHIFT                      0
+#define STR1BASE_SIGNED                     0
+
+#define STR1INTFIELD_MASK                   0x40000000U
+#define STR1INTFIELD_SHIFT                  30
+#define STR1INTFIELD_SIGNED                 0
+
+#define STR1STREN_MASK                      0x80000000U
+#define STR1STREN_SHIFT                     31
+#define STR1STREN_SIGNED                    0
+
+/*
+	Register PVR_PDP_STR1POSN
+*/
+#define TCF_RGBPDP_PVR_PDP_STR1POSN         0x0008
+#define STR1STRIDE_MASK                     0x000003FFU
+#define STR1STRIDE_SHIFT                    0
+#define STR1STRIDE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_MEMCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_MEMCTRL   0x000C
+#define MEMREFRESH_MASK                     0xC0000000U
+#define MEMREFRESH_SHIFT                    30
+#define MEMREFRESH_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_STRCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL   0x0010
+#define BURSTLEN_GFX_MASK                   0x000000FFU
+#define BURSTLEN_GFX_SHIFT                  0
+#define BURSTLEN_GFX_SIGNED                 0
+
+#define THRESHOLD_GFX_MASK                  0x0000FF00U
+#define THRESHOLD_GFX_SHIFT                 8
+#define THRESHOLD_GFX_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_SYNCCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL  0x0014
+#define HSDIS_MASK                          0x00000001U
+#define HSDIS_SHIFT                         0
+#define HSDIS_SIGNED                        0
+
+#define HSPOL_MASK                          0x00000002U
+#define HSPOL_SHIFT                         1
+#define HSPOL_SIGNED                        0
+
+#define VSDIS_MASK                          0x00000004U
+#define VSDIS_SHIFT                         2
+#define VSDIS_SIGNED                        0
+
+#define VSPOL_MASK                          0x00000008U
+#define VSPOL_SHIFT                         3
+#define VSPOL_SIGNED                        0
+
+#define BLNKDIS_MASK                        0x00000010U
+#define BLNKDIS_SHIFT                       4
+#define BLNKDIS_SIGNED                      0
+
+#define BLNKPOL_MASK                        0x00000020U
+#define BLNKPOL_SHIFT                       5
+#define BLNKPOL_SIGNED                      0
+
+#define HS_SLAVE_MASK                       0x00000040U
+#define HS_SLAVE_SHIFT                      6
+#define HS_SLAVE_SIGNED                     0
+
+#define VS_SLAVE_MASK                       0x00000080U
+#define VS_SLAVE_SHIFT                      7
+#define VS_SLAVE_SIGNED                     0
+
+#define INTERLACE_MASK                      0x00000100U
+#define INTERLACE_SHIFT                     8
+#define INTERLACE_SIGNED                    0
+
+#define FIELDPOL_MASK                       0x00000200U
+#define FIELDPOL_SHIFT                      9
+#define FIELDPOL_SIGNED                     0
+
+#define CLKPOL_MASK                         0x00000800U
+#define CLKPOL_SHIFT                        11
+#define CLKPOL_SIGNED                       0
+
+#define CSYNC_EN_MASK                       0x00001000U
+#define CSYNC_EN_SHIFT                      12
+#define CSYNC_EN_SIGNED                     0
+
+#define FIELD_EN_MASK                       0x00002000U
+#define FIELD_EN_SHIFT                      13
+#define FIELD_EN_SIGNED                     0
+
+#define UPDWAIT_MASK                        0x000F0000U
+#define UPDWAIT_SHIFT                       16
+#define UPDWAIT_SIGNED                      0
+
+#define UPDCTRL_MASK                        0x01000000U
+#define UPDCTRL_SHIFT                       24
+#define UPDCTRL_SIGNED                      0
+
+#define UPDINTCTRL_MASK                     0x02000000U
+#define UPDINTCTRL_SHIFT                    25
+#define UPDINTCTRL_SIGNED                   0
+
+#define UPDSYNCTRL_MASK                     0x04000000U
+#define UPDSYNCTRL_SHIFT                    26
+#define UPDSYNCTRL_SIGNED                   0
+
+#define POWERDN_MASK                        0x10000000U
+#define POWERDN_SHIFT                       28
+#define POWERDN_SIGNED                      0
+
+#define DISP_RST_MASK                       0x20000000U
+#define DISP_RST_SHIFT                      29
+#define DISP_RST_SIGNED                     0
+
+#define SYNCACTIVE_MASK                     0x80000000U
+#define SYNCACTIVE_SHIFT                    31
+#define SYNCACTIVE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_BORDCOL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_BORDCOL   0x0018
+#define BORDCOL_MASK                        0x00FFFFFFU
+#define BORDCOL_SHIFT                       0
+#define BORDCOL_SIGNED                      0
+
+/*
+	Register PVR_TCF_RGBPDP_UPDCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_UPDCTRL   0x001C
+#define UPDFIELD_MASK                       0x00000001U
+#define UPDFIELD_SHIFT                      0
+#define UPDFIELD_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC1
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC1    0x0020
+#define HT_MASK                             0x00000FFFU
+#define HT_SHIFT                            0
+#define HT_SIGNED                           0
+
+#define HBPS_MASK                           0x0FFF0000U
+#define HBPS_SHIFT                          16
+#define HBPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC2
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC2    0x0024
+#define HLBS_MASK                           0x00000FFFU
+#define HLBS_SHIFT                          0
+#define HLBS_SIGNED                         0
+
+#define HAS_MASK                            0x0FFF0000U
+#define HAS_SHIFT                           16
+#define HAS_SIGNED                          0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC3
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC3    0x0028
+#define HRBS_MASK                           0x00000FFFU
+#define HRBS_SHIFT                          0
+#define HRBS_SIGNED                         0
+
+#define HFPS_MASK                           0x0FFF0000U
+#define HFPS_SHIFT                          16
+#define HFPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC1
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC1    0x002C
+#define VT_MASK                             0x00000FFFU
+#define VT_SHIFT                            0
+#define VT_SIGNED                           0
+
+#define VBPS_MASK                           0x0FFF0000U
+#define VBPS_SHIFT                          16
+#define VBPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC2
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC2    0x0030
+#define VTBS_MASK                           0x00000FFFU
+#define VTBS_SHIFT                          0
+#define VTBS_SIGNED                         0
+
+#define VAS_MASK                            0x0FFF0000U
+#define VAS_SHIFT                           16
+#define VAS_SIGNED                          0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC3
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC3    0x0034
+#define VBBS_MASK                           0x00000FFFU
+#define VBBS_SHIFT                          0
+#define VBBS_SIGNED                         0
+
+#define VFPS_MASK                           0x0FFF0000U
+#define VFPS_SHIFT                          16
+#define VFPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_HDECTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HDECTRL   0x0038
+#define HDEF_MASK                           0x00000FFFU
+#define HDEF_SHIFT                          0
+#define HDEF_SIGNED                         0
+
+#define HDES_MASK                           0x0FFF0000U
+#define HDES_SHIFT                          16
+#define HDES_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VDECTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VDECTRL   0x003C
+#define VDEF_MASK                           0x00000FFFU
+#define VDEF_SHIFT                          0
+#define VDEF_SIGNED                         0
+
+#define VDES_MASK                           0x0FFF0000U
+#define VDES_SHIFT                          16
+#define VDES_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VEVENT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VEVENT    0x0040
+#define VFETCH_MASK                         0x00000FFFU
+#define VFETCH_SHIFT                        0
+#define VFETCH_SIGNED                       0
+
+#define VEVENT_MASK                         0x0FFF0000U
+#define VEVENT_SHIFT                        16
+#define VEVENT_SIGNED                       0
+
+/*
+	Register PVR_TCF_RGBPDP_OPMASK
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_OPMASK    0x0044
+#define MASKR_MASK                          0x000000FFU
+#define MASKR_SHIFT                         0
+#define MASKR_SIGNED                        0
+
+#define MASKG_MASK                          0x0000FF00U
+#define MASKG_SHIFT                         8
+#define MASKG_SIGNED                        0
+
+#define MASKB_MASK                          0x00FF0000U
+#define MASKB_SHIFT                         16
+#define MASKB_SIGNED                        0
+
+#define BLANKLEVEL_MASK                     0x40000000U
+#define BLANKLEVEL_SHIFT                    30
+#define BLANKLEVEL_SIGNED                   0
+
+#define MASKLEVEL_MASK                      0x80000000U
+#define MASKLEVEL_SHIFT                     31
+#define MASKLEVEL_SIGNED                    0
+
+/*
+	Register PVR_TCF_RGBPDP_INTSTAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTSTAT   0x0048
+#define INTS_HBLNK0_MASK                    0x00000001U
+#define INTS_HBLNK0_SHIFT                   0
+#define INTS_HBLNK0_SIGNED                  0
+
+#define INTS_HBLNK1_MASK                    0x00000002U
+#define INTS_HBLNK1_SHIFT                   1
+#define INTS_HBLNK1_SIGNED                  0
+
+#define INTS_VBLNK0_MASK                    0x00000004U
+#define INTS_VBLNK0_SHIFT                   2
+#define INTS_VBLNK0_SIGNED                  0
+
+#define INTS_VBLNK1_MASK                    0x00000008U
+#define INTS_VBLNK1_SHIFT                   3
+#define INTS_VBLNK1_SIGNED                  0
+
+#define INTS_STR1URUN_MASK                  0x00000010U
+#define INTS_STR1URUN_SHIFT                 4
+#define INTS_STR1URUN_SIGNED                0
+
+#define INTS_STR1ORUN_MASK                  0x00000020U
+#define INTS_STR1ORUN_SHIFT                 5
+#define INTS_STR1ORUN_SIGNED                0
+
+#define INTS_DISPURUN_MASK                  0x00000040U
+#define INTS_DISPURUN_SHIFT                 6
+#define INTS_DISPURUN_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_INTENAB
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB   0x004C
+#define INTEN_HBLNK0_MASK                   0x00000001U
+#define INTEN_HBLNK0_SHIFT                  0
+#define INTEN_HBLNK0_SIGNED                 0
+
+#define INTEN_HBLNK1_MASK                   0x00000002U
+#define INTEN_HBLNK1_SHIFT                  1
+#define INTEN_HBLNK1_SIGNED                 0
+
+#define INTEN_VBLNK0_MASK                   0x00000004U
+#define INTEN_VBLNK0_SHIFT                  2
+#define INTEN_VBLNK0_SIGNED                 0
+
+#define INTEN_VBLNK1_MASK                   0x00000008U
+#define INTEN_VBLNK1_SHIFT                  3
+#define INTEN_VBLNK1_SIGNED                 0
+
+#define INTEN_STR1URUN_MASK                 0x00000010U
+#define INTEN_STR1URUN_SHIFT                4
+#define INTEN_STR1URUN_SIGNED               0
+
+#define INTEN_STR1ORUN_MASK                 0x00000020U
+#define INTEN_STR1ORUN_SHIFT                5
+#define INTEN_STR1ORUN_SIGNED               0
+
+#define INTEN_DISPURUN_MASK                 0x00000040U
+#define INTEN_DISPURUN_SHIFT                6
+#define INTEN_DISPURUN_SIGNED               0
+
+/*
+	Register PVR_TCF_RGBPDP_INTCLEAR
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTCLEAR  0x0050
+#define INTCLR_HBLNK0_MASK                  0x00000001U
+#define INTCLR_HBLNK0_SHIFT                 0
+#define INTCLR_HBLNK0_SIGNED                0
+
+#define INTCLR_HBLNK1_MASK                  0x00000002U
+#define INTCLR_HBLNK1_SHIFT                 1
+#define INTCLR_HBLNK1_SIGNED                0
+
+#define INTCLR_VBLNK0_MASK                  0x00000004U
+#define INTCLR_VBLNK0_SHIFT                 2
+#define INTCLR_VBLNK0_SIGNED                0
+
+#define INTCLR_VBLNK1_MASK                  0x00000008U
+#define INTCLR_VBLNK1_SHIFT                 3
+#define INTCLR_VBLNK1_SIGNED                0
+
+#define INTCLR_STR1URUN_MASK                0x00000010U
+#define INTCLR_STR1URUN_SHIFT               4
+#define INTCLR_STR1URUN_SIGNED              0
+
+#define INTCLR_STR1ORUN_MASK                0x00000020U
+#define INTCLR_STR1ORUN_SHIFT               5
+#define INTCLR_STR1ORUN_SIGNED              0
+
+#define INTCLR_DISPURUN_MASK                0x00000040U
+#define INTCLR_DISPURUN_SHIFT               6
+#define INTCLR_DISPURUN_SIGNED              0
+
+/*
+	Register PVR_TCF_RGBPDP_INTCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTCTRL   0x0054
+#define HBLNK_LINENO_MASK                   0x00000FFFU
+#define HBLNK_LINENO_SHIFT                  0
+#define HBLNK_LINENO_SIGNED                 0
+
+#define HBLNK_LINE_MASK                     0x00010000U
+#define HBLNK_LINE_SHIFT                    16
+#define HBLNK_LINE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_SIGNAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_SIGNAT    0x0058
+#define SIGNATURE_MASK                      0xFFFFFFFFU
+#define SIGNATURE_SHIFT                     0
+#define SIGNATURE_SIGNED                    0
+
+/*
+	Register PVR_TCF_RGBPDP_LINESTAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_LINESTAT  0x005C
+#define LINENO_MASK                         0x00000FFFU
+#define LINENO_SHIFT                        0
+#define LINENO_SIGNED                       0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGCTRL   0x0060
+#define DBG_ENAB_MASK                       0x00000001U
+#define DBG_ENAB_SHIFT                      0
+#define DBG_ENAB_SIGNED                     0
+
+#define DBG_READ_MASK                       0x00000002U
+#define DBG_READ_SHIFT                      1
+#define DBG_READ_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGDATA
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGDATA   0x0064
+#define DBG_DATA_MASK                       0x00FFFFFFU
+#define DBG_DATA_SHIFT                      0
+#define DBG_DATA_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGSIDE
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGSIDE   0x0068
+#define DBG_SIDE_MASK                       0x00000007U
+#define DBG_SIDE_SHIFT                      0
+#define DBG_SIDE_SIGNED                     0
+
+#define DBG_VAL_MASK                        0x00000008U
+#define DBG_VAL_SHIFT                       3
+#define DBG_VAL_SIGNED                      0
+
+/*
+	Register PVR_TCF_RGBPDP_REGLD_STAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_REGLD_STAT 0x0070
+#define REGLD_ADDROUT_MASK                  0x00FFFFFFU
+#define REGLD_ADDROUT_SHIFT                 0
+#define REGLD_ADDROUT_SIGNED                0
+
+#define REGLD_ADDREN_MASK                   0x80000000U
+#define REGLD_ADDREN_SHIFT                  31
+#define REGLD_ADDREN_SIGNED                 0
+
+/*
+	Register PVR_TCF_RGBPDP_REGLD_CTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_REGLD_CTRL 0x0074
+#define REGLD_ADDRIN_MASK                   0x00FFFFFFU
+#define REGLD_ADDRIN_SHIFT                  0
+#define REGLD_ADDRIN_SIGNED                 0
+
+#define REGLD_VAL_MASK                      0x01000000U
+#define REGLD_VAL_SHIFT                     24
+#define REGLD_VAL_SIGNED                    0
+
+#define REGLD_ADDRLEN_MASK                  0xFE000000U
+#define REGLD_ADDRLEN_SHIFT                 25
+#define REGLD_ADDRLEN_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_CORE_ID
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_CORE_ID   0x0078
+#define CONFIG_ID_MASK                      0x0000FFFFU
+#define CONFIG_ID_SHIFT                     0
+#define CONFIG_ID_SIGNED                    0
+
+#define CORE_ID_MASK                        0x00FF0000U
+#define CORE_ID_SHIFT                       16
+#define CORE_ID_SIGNED                      0
+
+#define GROUP_ID_MASK                       0xFF000000U
+#define GROUP_ID_SHIFT                      24
+#define GROUP_ID_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_CORE_REV
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_CORE_REV  0x007C
+#define MAINT_REV_MASK                      0x000000FFU
+#define MAINT_REV_SHIFT                     0
+#define MAINT_REV_SIGNED                    0
+
+#define MINOR_REV_MASK                      0x0000FF00U
+#define MINOR_REV_SHIFT                     8
+#define MINOR_REV_SIGNED                    0
+
+#define MAJOR_REV_MASK                      0x00FF0000U
+#define MAJOR_REV_SHIFT                     16
+#define MAJOR_REV_SIGNED                    0
+
+#endif /* !defined(_TCF_RGBPDP_REGS_H_) */
+
+/*****************************************************************************
+ End of file (tcf_rgbpdp_regs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/virt_validation_defs.h b/drivers/gpu/drm/img/img-volcanic/include/virt_validation_defs.h
new file mode 100644
index 000000000000..5b8908f712c0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/virt_validation_defs.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Definitions for virtualization
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services shared header for virtualization definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRV_VIRT_DEFS_H
+#define SRV_VIRT_DEFS_H
+
+#if !defined(GPUVIRT_VALIDATION_MAX_STRING_LENGTH)
+	#define GPUVIRT_VALIDATION_MAX_STRING_LENGTH 512
+#endif
+
+#define GPUVIRT_VALIDATION_MAX_OS 8
+
+#define GPUVIRT_VALIDATION_NUM_REGIONS 2
+#define GPUVIRT_VAL_REGION_SECURE 0
+#define GPUVIRT_VAL_REGION_SHARED 1
+
+/* Shared region 1MB */
+#define GPUVIRT_SIZEOF_SHARED 0x100000
+
+/* Min region size 64MB */
+#define GPUVIRT_MIN_SIZE 0x4000000
+
+#endif /* SRV_VIRT_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/pvrsrv_surface.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/pvrsrv_surface.h
new file mode 100644
index 000000000000..4bb87e0dfff8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/pvrsrv_surface.h
@@ -0,0 +1,212 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR Services Surface
+@Description    Surface definitions and utilities that are externally visible
+                (i.e. visible to clients of services), but are also required
+                within services.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_SURFACE_H
+#define PVRSRV_SURFACE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/buffer_attribs.h>
+
+#define PVRSRV_SURFACE_TRANSFORM_NONE      (0 << 0)		/*!< No transformation */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_H    (1 << 0)		/*!< Flip horizontally */
+#define PVRSRV_SURFACE_TRANSFORM_FLIP_V    (1 << 1)		/*!< Flip vertically */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_90    (1 << 2)                          /*!< Rotate  90 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_180   ((1 << 0) + (1 << 1))             /*!< Rotate 180 degree clockwise */
+#define PVRSRV_SURFACE_TRANSFORM_ROT_270   ((1 << 0) + (1 << 1) + (1 << 2))  /*!< Rotate 270 degree clockwise */
+
+#define PVRSRV_SURFACE_BLENDING_NONE       0	/*!< Use no blending */
+#define PVRSRV_SURFACE_BLENDING_PREMULT    1	/*!< Use blending with pre-multiplier */
+#define PVRSRV_SURFACE_BLENDING_COVERAGE   2	/*!< Use coverage blending */
+
+/*!
+ * Modes of memory layouts for surfaces.
+ *
+ *   Enum: #_PVRSRV_SURFACE_MEMLAYOUT_
+ *   Typedef: ::PVRSRV_SURFACE_MEMLAYOUT
+ */
+typedef enum _PVRSRV_SURFACE_MEMLAYOUT_
+{
+	PVRSRV_SURFACE_MEMLAYOUT_STRIDED = 0,		/*!< Strided memory buffer */
+	PVRSRV_SURFACE_MEMLAYOUT_FBC,				/*!< Compressed frame buffer */
+} PVRSRV_SURFACE_MEMLAYOUT;
+
+/*!
+ * Frame Buffer Compression layout.
+ * Defines the FBC mode to use.
+ *
+ *   Structure: #PVRSRV_SURFACE_FBC_LAYOUT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FBC_LAYOUT
+ */
+typedef struct PVRSRV_SURFACE_FBC_LAYOUT_TAG
+{
+	/*! The compression mode for this surface */
+	IMG_FB_COMPRESSION	eFBCompressionMode;
+} PVRSRV_SURFACE_FBC_LAYOUT;
+
+/*!
+ * Pixel and memory format of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_FORMAT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_FORMAT
+ */
+typedef struct PVRSRV_SURFACE_FORMAT_TAG
+{
+	/*! Enum value of type IMG_PIXFMT for the pixel format */
+	IMG_UINT32					ePixFormat;
+
+	/*! Enum surface memory layout */
+	PVRSRV_SURFACE_MEMLAYOUT	eMemLayout;
+
+	/*! Special layout options for the surface.
+	 * Needs services support.
+	 * Depends on eMemLayout.*/
+	union {
+		PVRSRV_SURFACE_FBC_LAYOUT	sFBCLayout;
+	} u;
+} PVRSRV_SURFACE_FORMAT;
+
+/*!
+ * Width and height of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_DIMS_TAG
+ *   Typedef: ::PVRSRV_SURFACE_DIMS
+ */
+typedef struct PVRSRV_SURFACE_DIMS_TAG
+{
+	IMG_UINT32		ui32Width;		/*!< Width in pixels */
+	IMG_UINT32		ui32Height;		/*!< Height in pixels */
+} PVRSRV_SURFACE_DIMS;
+
+/*!
+ * Dimension and format details of a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_INFO
+ */
+typedef struct PVRSRV_SURFACE_INFO_TAG
+{
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Width and height */
+	PVRSRV_SURFACE_FORMAT	sFormat;	/*!< Memory format */
+} PVRSRV_SURFACE_INFO;
+
+/*!
+ * Defines a rectangle on a surface
+ *
+ *   Structure: #PVRSRV_SURFACE_RECT_TAG
+ *   Typedef: ::PVRSRV_SURFACE_RECT
+ */
+typedef struct PVRSRV_SURFACE_RECT_TAG
+{
+	IMG_INT32				i32XOffset;	/*!< X offset from origin in pixels */
+	IMG_INT32				i32YOffset;	/*!< Y offset from origin in pixels */
+	PVRSRV_SURFACE_DIMS		sDims;		/*!< Rectangle dimensions */
+} PVRSRV_SURFACE_RECT;
+
+/*!
+ * Surface configuration details
+ *
+ *   Structure: #PVRSRV_SURFACE_CONFIG_INFO_TAG
+ *   Typedef: ::PVRSRV_SURFACE_CONFIG_INFO
+ */
+typedef struct PVRSRV_SURFACE_CONFIG_INFO_TAG
+{
+	/*! Crop applied to surface (BEFORE transformation) */
+	PVRSRV_SURFACE_RECT		sCrop;
+
+	/*! Region of screen to display surface in (AFTER scaling) */
+	PVRSRV_SURFACE_RECT		sDisplay;
+
+	/*! Surface transformation none/flip/rotate.
+	 * Use PVRSRV_SURFACE_TRANSFORM_xxx macros
+	 */
+	IMG_UINT32				ui32Transform;
+
+	/*! Alpha blending mode e.g. none/premult/coverage.
+	 * Use PVRSRV_SURFACE_BLENDING_xxx macros
+	 */
+	IMG_UINT32				eBlendType;
+
+	/*! Custom data for the display engine */
+	IMG_UINT32				ui32Custom;
+
+	/*! Alpha value for this plane */
+	IMG_UINT8				ui8PlaneAlpha;
+
+	/*! Reserved for later use */
+	IMG_UINT8				ui8Reserved1[3];
+} PVRSRV_SURFACE_CONFIG_INFO;
+
+/*!
+ * Contains information about a panel
+ */
+typedef struct PVRSRV_PANEL_INFO_TAG
+{
+	PVRSRV_SURFACE_INFO sSurfaceInfo;		/*!< Panel surface details */
+	IMG_UINT32			ui32RefreshRate;	/*!< Panel refresh rate in Hz */
+	IMG_UINT32			ui32XDpi;			/*!< Panel DPI in x direction */
+	IMG_UINT32			ui32YDpi;			/*!< Panel DPI in y direction */
+} PVRSRV_PANEL_INFO;
+
+/*!
+ * Helper function to create a Config Info based on a Surface Info
+ * to do a flip with no scale, transformation etc.
+ */
+static INLINE void SurfaceConfigFromSurfInfo(const PVRSRV_SURFACE_INFO *psSurfaceInfo,
+                                             PVRSRV_SURFACE_CONFIG_INFO *psConfigInfo)
+{
+	psConfigInfo->sCrop.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sCrop.i32XOffset = 0;
+	psConfigInfo->sCrop.i32YOffset = 0;
+	psConfigInfo->sDisplay.sDims = psSurfaceInfo->sDims;
+	psConfigInfo->sDisplay.i32XOffset = 0;
+	psConfigInfo->sDisplay.i32YOffset = 0;
+	psConfigInfo->ui32Transform = PVRSRV_SURFACE_TRANSFORM_NONE;
+	psConfigInfo->eBlendType = PVRSRV_SURFACE_BLENDING_NONE;
+	psConfigInfo->ui32Custom = 0;
+	psConfigInfo->ui8PlaneAlpha = 0xff;
+}
+
+#endif /* PVRSRV_SURFACE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_alignchecks.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_alignchecks.h
new file mode 100644
index 000000000000..411e984aaa9e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_alignchecks.h
@@ -0,0 +1,190 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX fw interface alignment checks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Checks to avoid disalignment in RGX fw data structures
+                shared with the host
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_ALIGNCHECKS_H)
+#define RGX_FWIF_ALIGNCHECKS_H
+
+/* for the offsetof macro */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/*!
+ ******************************************************************************
+ * Alignment UM/FW checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_UM_MAX 128U
+
+#if defined(PM_INTERACTIVE_MODE)
+#define HWRTDATA_PM_OFFSET  offsetof(RGXFWIF_HWRTDATA, sPMMListDevVAddr),
+#define HWRTDATA_HEAPTABLE_OFFSET offsetof(RGXFWIF_HWRTDATA, psVHeapTableDevVAddr),
+#else
+#define HWRTDATA_PM_OFFSET  offsetof(RGXFWIF_HWRTDATA, sPMRenderStateDevVAddr),
+#define HWRTDATA_HEAPTABLE_OFFSET
+#endif
+
+#define RGXFW_ALIGN_CHECKS_INIT0									\
+		sizeof(RGXFWIF_TRACEBUF),									\
+		offsetof(RGXFWIF_TRACEBUF, ui32LogType),					\
+		offsetof(RGXFWIF_TRACEBUF, sTraceBuf),						\
+		offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),		\
+		offsetof(RGXFWIF_TRACEBUF, ui32TracebufFlags),				\
+																	\
+		sizeof(RGXFWIF_SYSDATA),									\
+		offsetof(RGXFWIF_SYSDATA, ePowState),						\
+		offsetof(RGXFWIF_SYSDATA, ui32HWPerfDropCount),				\
+		offsetof(RGXFWIF_SYSDATA, ui32LastDropOrdinal),				\
+		offsetof(RGXFWIF_SYSDATA, ui32FWFaults),					\
+		offsetof(RGXFWIF_SYSDATA, ui32HWRStateFlags),				\
+																	\
+		sizeof(RGXFWIF_OSDATA),										\
+		offsetof(RGXFWIF_OSDATA, ui32HostSyncCheckMark),			\
+		offsetof(RGXFWIF_OSDATA, ui32KCCBCmdsExecuted),				\
+																	\
+		sizeof(RGXFWIF_HWRINFOBUF),									\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmLockedUpCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmOverranCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmRecoveredCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmFalseDetectCount),	\
+																	\
+		/* RGXFWIF_CMDTA checks */									\
+		sizeof(RGXFWIF_CMDTA),										\
+		offsetof(RGXFWIF_CMDTA, sGeomRegs),							\
+																	\
+		/* RGXFWIF_CMD3D checks */									\
+		sizeof(RGXFWIF_CMD3D),										\
+		offsetof(RGXFWIF_CMD3D, s3DRegs),							\
+																	\
+		/* RGXFWIF_CMD_COMPUTE checks */							\
+		sizeof(RGXFWIF_CMD_COMPUTE),								\
+		offsetof(RGXFWIF_CMD_COMPUTE, sCDMRegs),					\
+																	\
+		/* RGXFWIF_FREELIST checks */								\
+		sizeof(RGXFWIF_FREELIST), \
+		offsetof(RGXFWIF_FREELIST, sFreeListBaseDevVAddr),\
+		offsetof(RGXFWIF_FREELIST, sFreeListStateDevVAddr),\
+		offsetof(RGXFWIF_FREELIST, sFreeListLastGrowDevVAddr),\
+		offsetof(RGXFWIF_FREELIST, ui32MaxPages),\
+		offsetof(RGXFWIF_FREELIST, ui32CurrentPages),\
+									\
+		/* RGXFWIF_HWRTDATA checks */					\
+		sizeof(RGXFWIF_HWRTDATA), \
+		HWRTDATA_PM_OFFSET \
+		HWRTDATA_HEAPTABLE_OFFSET \
+		offsetof(RGXFWIF_HWRTDATA, apsFreeLists),\
+		/*offsetof(RGXFWIF_HWRTDATA, ui64VCECatBase),*/ \
+		offsetof(RGXFWIF_HWRTDATA, eState), \
+							\
+\
+		sizeof(RGXFWIF_HWPERF_CTL), \
+		offsetof(RGXFWIF_HWPERF_CTL, sBlkCfg), \
+		sizeof(RGXFWIF_CMDTDM), \
+		offsetof(RGXFWIF_CMDTDM, sTDMRegs)
+
+#define RGXFW_ALIGN_CHECKS_INIT		RGXFW_ALIGN_CHECKS_INIT0
+
+
+
+/*!
+ ******************************************************************************
+ * Alignment KM checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_INIT_KM0                         \
+		sizeof(RGXFWIF_SYSINIT),                            \
+		offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr),          \
+		offsetof(RGXFWIF_SYSINIT, sPDSExecBase),            \
+		offsetof(RGXFWIF_SYSINIT, sUSCExecBase),            \
+		offsetof(RGXFWIF_SYSINIT, aui32TPUTrilinearFracMask),\
+		offsetof(RGXFWIF_SYSINIT, sTraceBufCtl),            \
+		offsetof(RGXFWIF_SYSINIT, sFwSysData),              \
+		                                                    \
+		sizeof(RGXFWIF_OSINIT),                             \
+		offsetof(RGXFWIF_OSINIT, psKernelCCBCtl),           \
+		offsetof(RGXFWIF_OSINIT, psKernelCCB),              \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCBCtl),         \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCB),            \
+		offsetof(RGXFWIF_OSINIT, sFwOsData),                \
+		offsetof(RGXFWIF_OSINIT, sRGXCompChecks),           \
+		                                                    \
+		/* RGXFWIF_FWRENDERCONTEXT checks */                \
+		sizeof(RGXFWIF_FWRENDERCONTEXT),                    \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),      \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),      \
+		                                                    \
+		sizeof(RGXFWIF_FWCOMPUTECONTEXT),                   \
+		offsetof(RGXFWIF_FWCOMPUTECONTEXT, sCDMContext),    \
+		offsetof(RGXFWIF_FWCOMPUTECONTEXT, sStaticComputeContextState),\
+		offsetof(RGXFWIF_FWCOMPUTECONTEXT, ui32WorkEstCCBSubmitted),\
+															\
+		sizeof(RGXFWIF_FWTDMCONTEXT),                       \
+		offsetof(RGXFWIF_FWTDMCONTEXT, sTDMContext),        \
+		offsetof(RGXFWIF_FWTDMCONTEXT, ui32WorkEstCCBSubmitted),\
+															\
+		sizeof(RGXFWIF_FWCOMMONCONTEXT),                    \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psFWMemContext),  \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, sRunNode),        \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psCCB),           \
+		                                                    \
+		sizeof(RGXFWIF_MMUCACHEDATA),                       \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32CacheFlags),     \
+		offsetof(RGXFWIF_MMUCACHEDATA, sMMUCacheSync),      \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32MMUCacheSyncUpdateValue)
+
+#if defined(SUPPORT_TRP)
+#define RGXFW_ALIGN_CHECKS_INIT_KM                          \
+		RGXFW_ALIGN_CHECKS_INIT_KM0,                        \
+		offsetof(RGXFWIF_FWTDMCONTEXT, aui64TRPChecksums2D)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT_KM RGXFW_ALIGN_CHECKS_INIT_KM0
+#endif
+
+#endif /* RGX_FWIF_ALIGNCHECKS_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_alignchecks.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_customer.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_customer.h
new file mode 100644
index 000000000000..60a9273570fe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_customer.h
@@ -0,0 +1,51 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures used by pvrsrvkm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures used by pvrsrvkm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_CUSTOMER_H)
+#define RGX_FWIF_CUSTOMER_H
+
+/*
+ * Custom firmware data structures here
+ */
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_hwperf.h
new file mode 100644
index 000000000000..e2287375fdc2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_hwperf.h
@@ -0,0 +1,125 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_hwperf.h
+@Title          RGX HWPerf support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared header between RGX firmware and Init process
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_HWPERF_H
+#define RGX_FWIF_HWPERF_H
+
+#include "rgx_fwif_shared.h"
+#include "rgx_hwperf.h"
+#include "rgxdefs_km.h"
+
+/* Server and Firmware definitions only */
+
+/*! The number of HWPerf blocks in the GPU */
+
+#if defined(RGX_FIRMWARE)
+#define RGX_HWPERF_NUM_SPU ((RGX_FEATURE_NUM_SPU))
+#define RGX_HWPERF_NUM_USC ((RGX_FEATURE_NUM_CLUSTERS))
+#define RGX_HWPERF_NUM_ISP_PER_SPU ((RGX_FEATURE_NUM_ISP_PER_SPU))
+#define RGX_HWPERF_NUM_PBE ((RGX_FEATURE_PBE_PER_SPU) * (RGX_FEATURE_NUM_SPU))
+#define RGX_HWPERF_NUM_MERCER ((RGX_FEATURE_NUM_CLUSTERS))
+#define RGX_HWPERF_NUM_PBE_SHARED ((RGX_FEATURE_NUM_SPU))
+#define RGX_HWPERF_NUM_SWIFT ((RGX_FEATURE_NUM_SPU * RGX_FEATURE_MAX_TPU_PER_SPU))
+#define RGX_HWPERF_NUM_TEXAS ((RGX_FEATURE_NUM_SPU))
+#if (RGX_FEATURE_RAY_TRACING_ARCH > 2) && (RGX_FEATURE_SPU0_RAC_PRESENT > 0)
+#define RGX_HWPERF_NUM_RAC   ((RGX_NUM_RAC))
+#else
+#define RGX_HWPERF_NUM_RAC   ((0))
+#endif
+#define RGX_HWPERF_NUM_TPU   ((RGX_FEATURE_NUM_SPU * RGX_FEATURE_MAX_TPU_PER_SPU))
+#define RGX_HWPERF_NUM_ISP   ((RGX_FEATURE_NUM_CLUSTERS))
+
+#define RGX_CNTBLK_INDIRECT_COUNT(_class) ((RGX_HWPERF_NUM_ ## _class))
+
+/*! The number of layout blocks defined with configurable
+ * performance counters. Compile time constants.
+ * This is for the Series 8XT+ layout.
+ */
+#define RGX_HWPERF_MAX_DEFINED_BLKS (\
+	(IMG_UINT32)RGX_CNTBLK_ID_DIRECT_LAST  +\
+	RGX_CNTBLK_INDIRECT_COUNT(ISP)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(MERCER)      +\
+	RGX_CNTBLK_INDIRECT_COUNT(PBE)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(PBE_SHARED)  +\
+	RGX_CNTBLK_INDIRECT_COUNT(USC)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(TPU)         +\
+	RGX_CNTBLK_INDIRECT_COUNT(SWIFT)       +\
+	RGX_CNTBLK_INDIRECT_COUNT(TEXAS)       +\
+	RGX_CNTBLK_INDIRECT_COUNT(RAC))
+
+#endif	/* RGX_FIRMWARE */
+
+/*****************************************************************************/
+
+/* Structure used in the FW's global control data to hold the performance
+ * counters provisioned for a given block. */
+typedef struct
+{
+	IMG_UINT32           uiBlockID;
+	IMG_UINT32           uiNumCounters;    // Number of counters held
+	                                       // in aui32CounterCfg
+	                                       // [0..RGX_CNTBLK_COUNTERS_MAX)
+	IMG_UINT32           uiEnabled;        // 1 => enabled, 0=> disabled
+	RGXFWIF_DEV_VIRTADDR psModel;          // link to model table for uiBlockID
+	IMG_UINT32           aui32CounterCfg[RGX_CNTBLK_COUNTERS_MAX];
+} RGXFWIF_HWPERF_CTL_BLK;
+
+
+/*!
+ *****************************************************************************
+ * Structure used in the FW's global RGXFW_CTL store, holding HWPerf counter
+ * block configuration. It is written to by the Server on FW initialisation
+ * (PDUMP=1) and by the FW BG kCCB command processing code. It is read by
+ * the FW IRQ register programming and HWPerf event generation routines.
+ * Size of the sBlkCfg[] array must be consistent between KM/UM and FW.
+ * FW will ASSERT if the sizes are different
+ * (ui32NumBlocks != RGX_HWPERF_MAX_DEFINED_BLKS)
+ ****************************************************************************/
+typedef struct
+{
+	IMG_UINT32                         ui32Reserved;
+	IMG_UINT32                         ui32CtrlWord;
+	IMG_UINT32                         ui32EnabledBlksCount;
+	IMG_UINT32                         ui32NumBlocks;
+	RGXFWIF_HWPERF_CTL_BLK RGXFW_ALIGN sBlkCfg[1];	// First array entry
+} UNCACHED_ALIGN RGXFWIF_HWPERF_CTL;
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_km.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_km.h
new file mode 100644
index 000000000000..45cdb2d4ca88
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_km.h
@@ -0,0 +1,1949 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures used by pvrsrvkm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures used by pvrsrvkm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_KM_H)
+#define RGX_FWIF_KM_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+#include "rgxdefs_km.h"
+#include "dllist.h"
+#include "rgx_hwperf.h"
+#include "rgxheapconfig.h"
+#include "rgx_fwif_customer.h"
+
+/************************************************************************
+* RGX FW signature checks
+************************************************************************/
+#define RGXFW_SIG_BUFFER_SIZE_MIN       (8192)
+
+#define RGXFWIF_TIMEDIFF_ID			((0x1UL << 28) | RGX_CR_TIMER)
+
+#define RGXFW_POLL_TYPE_SET 0x80000000U
+
+/* Firmware per-DM HWR states */
+#define RGXFWIF_DM_STATE_WORKING					(0x00U)		/*!< DM is working if all flags are cleared */
+#define RGXFWIF_DM_STATE_READY_FOR_HWR				(IMG_UINT32_C(0x1) << 0)	/*!< DM is idle and ready for HWR */
+#define RGXFWIF_DM_STATE_NEEDS_SKIP					(IMG_UINT32_C(0x1) << 2)	/*!< DM need to skip to next cmd before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP			(IMG_UINT32_C(0x1) << 3)	/*!< DM need partial render cleanup before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR			(IMG_UINT32_C(0x1) << 4)	/*!< DM need to increment Recovery Count once fully recovered */
+#define RGXFWIF_DM_STATE_GUILTY_LOCKUP				(IMG_UINT32_C(0x1) << 5)	/*!< DM was identified as locking up and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_LOCKUP			(IMG_UINT32_C(0x1) << 6)	/*!< DM was innocently affected by another lockup which caused HWR */
+#define RGXFWIF_DM_STATE_GUILTY_OVERRUNING			(IMG_UINT32_C(0x1) << 7)	/*!< DM was identified as over-running and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_OVERRUNING		(IMG_UINT32_C(0x1) << 8)	/*!< DM was innocently affected by another DM over-running which caused HWR */
+#define RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH		(IMG_UINT32_C(0x1) << 9)	/*!< DM was forced into HWR as it delayed more important workloads */
+#define RGXFWIF_DM_STATE_GPU_ECC_HWR				(IMG_UINT32_C(0x1) << 10)	/*!< DM was forced into HWR due to an uncorrected GPU ECC error */
+#define RGXFWIF_DM_STATE_GPU_PARITY_HWR				(IMG_UINT32_C(0x1) << 11)	/*!< DM was forced into HWR due to an uncorrected GPU PARITY error */
+#define RGXFWIF_DM_STATE_GPU_LATENT_HWR				(IMG_UINT32_C(0x1) << 12)	/*!< DM was forced into HWR due to an uncorrected GPU LATENT error */
+#define RGXFWIF_DM_STATE_ICS_HWR					(IMG_UINT32_C(0x1) << 13)	/*!< DM was forced into HWR due to a fault detected by Idle Cycle Stealing */
+
+/* Firmware's connection state */
+typedef IMG_UINT32 RGXFWIF_CONNECTION_FW_STATE;
+#define RGXFW_CONNECTION_FW_OFFLINE				0U	/*!< Firmware is offline */
+#define RGXFW_CONNECTION_FW_READY				1U	/*!< Firmware is initialised */
+#define RGXFW_CONNECTION_FW_ACTIVE				2U	/*!< Firmware connection is fully established */
+#define RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD	3U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_FORCED_OFFLOAD		4U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_COOLDOWN			5U	/*!< Firmware connection is in cooldown period */
+#define RGXFW_CONNECTION_FW_STATE_COUNT			6U
+
+#define RGXFW_FORCED_OFFLOAD_HCS_DEADLINE_MS	2U		/*!< Workloads of Guest being forcefully offloaded must be stopped quickly */
+#define RGXFW_GRACEFUL_OFFLOAD_HCS_DEADLINE_MS	1000U	/*!< Workloads of Guest being gracefully offloaded are allowed more time to finish */
+
+/* OS' connection state */
+typedef enum
+{
+	RGXFW_CONNECTION_OS_OFFLINE = 0,	/*!< OS is offline */
+	RGXFW_CONNECTION_OS_READY,			/*!< OS's KM driver is setup and waiting */
+	RGXFW_CONNECTION_OS_ACTIVE,			/*!< OS connection is fully established */
+	RGXFW_CONNECTION_OS_STATE_COUNT
+} RGXFWIF_CONNECTION_OS_STATE;
+
+#define RGXFWIF_CTXSWITCH_PROFILE_FAST_EN		(IMG_UINT32_C(0x1))
+#define RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN		(IMG_UINT32_C(0x2))
+#define RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN		(IMG_UINT32_C(0x3))
+#define RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN	(IMG_UINT32_C(0x4))
+
+#define RGXFWIF_ISP_SCHEDMODE_VER1_IPP			(IMG_UINT32_C(0x1))
+#define RGXFWIF_ISP_SCHEDMODE_VER2_ISP			(IMG_UINT32_C(0x2))
+/*!
+ ******************************************************************************
+ * RGX firmware Init Config Data
+ *****************************************************************************/
+
+/* Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_CTXSWITCH_MODE_RAND				(IMG_UINT32_C(0x1) << 0)	/*!< Randomise context switch requests */
+#define RGXFWIF_INICFG_CTXSWITCH_SRESET_EN				(IMG_UINT32_C(0x1) << 1)
+#define RGXFWIF_INICFG_HWPERF_EN						(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN				(IMG_UINT32_C(0x1) << 3)	/*!< Randomise DM-killing requests */
+#define RGXFWIF_INICFG_POW_RASCALDUST					(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_SPU_CLOCK_GATE					(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_FBCDC_V3_1_EN					(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_CHECK_MLIST_EN					(IMG_UINT32_C(0x1) << 7)
+#define RGXFWIF_INICFG_DISABLE_CLKGATING_EN				(IMG_UINT32_C(0x1) << 8)
+#define RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN	(IMG_UINT32_C(0x1) << 9)
+#define RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN		(IMG_UINT32_C(0x1) << 10)
+/* 11 unused */
+#define RGXFWIF_INICFG_REGCONFIG_EN						(IMG_UINT32_C(0x1) << 12)
+#define RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY			(IMG_UINT32_C(0x1) << 13)
+#define RGXFWIF_INICFG_HWP_DISABLE_FILTER				(IMG_UINT32_C(0x1) << 14)
+/* 15 unused */
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT			(16)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST			(RGXFWIF_CTXSWITCH_PROFILE_FAST_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM			(RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW			(RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY		(RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK			(IMG_UINT32_C(0x7) << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_DISABLE_DM_OVERLAP				(IMG_UINT32_C(0x1) << 19)
+#define RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER			(IMG_UINT32_C(0x1) << 20)
+#define RGXFWIF_INICFG_INJECT_ICS_FAULT					(IMG_UINT32_C(0x1) << 21)
+#define RGXFWIF_INICFG_VALIDATE_IRQ						(IMG_UINT32_C(0x1) << 22)
+#define RGXFWIF_INICFG_DISABLE_PDP_EN					(IMG_UINT32_C(0x1) << 23)
+#define RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN	(IMG_UINT32_C(0x1) << 24)
+#define RGXFWIF_INICFG_WORKEST							(IMG_UINT32_C(0x1) << 25)
+#define RGXFWIF_INICFG_PDVFS							(IMG_UINT32_C(0x1) << 26)
+/* 27 unused */
+/* 28 unused */
+#define RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT				(29)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_NONE				(0)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP			(RGXFWIF_ISP_SCHEDMODE_VER1_IPP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP			(RGXFWIF_ISP_SCHEDMODE_VER2_ISP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_MASK				(RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP |\
+                                                         RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP)
+#define RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER			(IMG_UINT32_C(0x1) << 31)
+#define RGXFWIF_INICFG_ALL								(0xFFFFF7FFU)
+
+/* Extended Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_EXT_ALL							(0x0U)
+
+#define RGXFWIF_INICFG_SYS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_CTXSWITCH_MODE_RAND | \
+														  RGXFWIF_INICFG_CTXSWITCH_SRESET_EN)
+
+/* Flag definitions affecting only workloads submitted by a particular OS */
+
+/*!
+ * @AddToGroup ContextSwitching
+ * @{
+ * @Name Per-OS DM context switch configuration flags
+ * @{
+ */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN				(IMG_UINT32_C(0x1) << 0) /*!< Enables TDM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN				(IMG_UINT32_C(0x1) << 1) /*!< Enables GEOM-TA and GEOM-SHG context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN				(IMG_UINT32_C(0x1) << 2) /*!< Enables FRAG DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN				(IMG_UINT32_C(0x1) << 3) /*!< Enables CDM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN				(IMG_UINT32_C(0x1) << 4)
+
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM				(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM				(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D				(IMG_UINT32_C(0x1) << 7)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM				(IMG_UINT32_C(0x1) << 8)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM				(IMG_UINT32_C(0x1) << 9)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL				(RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL)
+
+/*!
+ * @} End of Per-OS Context switch configuration flags
+ * @} End of AddToGroup ContextSwitching
+ */
+/*!
+ * @AddToGroup IdleCycleStealing
+ * @{
+ * @Name Per-OS DM Idle Cycle Stealing configuration flags
+ * @{
+ */
+#define RGXFWIF_INICFG_OS_ICS_TDM_EN				(IMG_UINT32_C(0x1) << 10) /*!< Enables ICS for TDM */
+#define RGXFWIF_INICFG_OS_ICS_GEOM_EN				(IMG_UINT32_C(0x1) << 11) /*!< Enables ICS for GEOM */
+#define RGXFWIF_INICFG_OS_ICS_3D_EN					(IMG_UINT32_C(0x1) << 12) /*!< Enables ICS for FRAG */
+#define RGXFWIF_INICFG_OS_ICS_CDM_EN				(IMG_UINT32_C(0x1) << 13) /*!< Enables ICS for CDM */
+#define RGXFWIF_INICFG_OS_ICS_RDM_EN				(IMG_UINT32_C(0x1) << 14) /*!< Enables ICS for RDM */
+
+#define RGXFWIF_INICFG_OS_ICS_DM_ALL				(RGXFWIF_INICFG_OS_ICS_TDM_EN | \
+													 RGXFWIF_INICFG_OS_ICS_GEOM_EN | \
+													 RGXFWIF_INICFG_OS_ICS_3D_EN | \
+													 RGXFWIF_INICFG_OS_ICS_CDM_EN | \
+													 RGXFWIF_INICFG_OS_ICS_RDM_EN)
+
+#define RGXFWIF_INICFG_OS_ICS_DM_ALL_SHIFT			(10U)
+#define RGXFWIF_INICFG_OS_ICS_DM_APPHINT			(RGXFWIF_INICFG_OS_ICS_CDM_EN >> RGXFWIF_INICFG_OS_ICS_DM_ALL_SHIFT)
+
+#define RGXFWIF_INICFG_OS_ICS_CLRMSK				~(RGXFWIF_INICFG_OS_ICS_DM_ALL)
+/*!
+ * @} End of Per-OS Idle Cycle Stealing configuration flags
+ * @} End of AddToGroup IdleCycleStealing
+ */
+#define RGXFWIF_INICFG_OS_FDTI_PROFILE_LONG			(IMG_UINT32_C(0x1))
+#define RGXFWIF_INICFG_OS_FDTI_PROFILE_MEDIUM		(IMG_UINT32_C(0x2))
+#define RGXFWIF_INICFG_OS_FDTI_PROFILE_SHORT		(IMG_UINT32_C(0x3))
+
+#define RGXFWIF_INICFG_FDTI_PROFILE_SHIFT			(16)
+#define RGXFWIF_INICFG_FDTI_PROFILE_MASK			(IMG_UINT32_C(0x3) << RGXFWIF_INICFG_FDTI_PROFILE_SHIFT)
+
+#define RGXFWIF_INICFG_OS_ALL						(0x3FFFFU)
+
+#define RGXFWIF_FILTCFG_TRUNCATE_HALF					(IMG_UINT32_C(0x1) << 3)
+#define RGXFWIF_FILTCFG_TRUNCATE_INT					(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_FILTCFG_NEW_FILTER_MODE					(IMG_UINT32_C(0x1) << 1)
+
+typedef IMG_UINT32 RGX_ACTIVEPM_CONF;
+#define RGX_ACTIVEPM_FORCE_OFF	0U
+#define RGX_ACTIVEPM_FORCE_ON	1U
+#define RGX_ACTIVEPM_DEFAULT	2U
+
+typedef IMG_UINT32 RGX_RD_POWER_ISLAND_CONF;
+#define RGX_RD_POWER_ISLAND_FORCE_OFF	0U
+#define RGX_RD_POWER_ISLAND_FORCE_ON	1U
+#define RGX_RD_POWER_ISLAND_DEFAULT		2U
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SIGBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TRACEBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SYSDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_OSDATA;
+#if defined(SUPPORT_TBI_INTERFACE)
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TBIBUF;
+#endif
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERFBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRINFOBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RUNTIME_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_GPU_UTIL_FW;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_REG_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERF_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_RTN_SLOTS;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWCOMMONCONTEXT;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_ZSBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CORE_CLK_RATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COUNTERBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FIRMWAREGCOVBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRTDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TIMESTAMP_ADDR;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*!
+ * @Brief Buffer to store KM active client contexts
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+} RGXFWIF_ACTIVE_CONTEXT_BUF_DATA;
+#endif
+
+/*!
+ * This number is used to represent an invalid page catalogue physical address
+ */
+#define RGXFWIF_INVALID_PC_PHYADDR 0xFFFFFFFFFFFFFFFFLLU
+
+/*!
+ * This number is used to represent an unallocated set of page catalog base registers
+ */
+#define RGXFW_BIF_INVALID_PCSET 0xFFFFFFFFU
+
+/*!
+ * This number is used to represent an invalid OS ID for the purpose of tracking PC set ownership
+ */
+#define RGXFW_BIF_INVALID_OSID 0xFFFFFFFFU
+
+#define RGXFWIF_FWMEMCONTEXT_FLAGS_USED_TESS		(0x00000001U)
+
+/*!
+ * Firmware memory context.
+ */
+typedef struct
+{
+	IMG_DEV_PHYADDR			RGXFW_ALIGN sPCDevPAddr;	/*!< device physical address of context's page catalogue */
+	IMG_UINT32				uiPageCatBaseRegSet;		/*!< index of the associated set of page catalog base registers (RGXFW_BIF_INVALID_PCSET == unallocated) */
+	IMG_UINT32				uiBreakpointAddr; /*!< breakpoint address */
+	IMG_UINT32				uiBPHandlerAddr; /*!< breakpoint handler address */
+	IMG_UINT32				uiBreakpointCtl; /*!< DM and enable control for BP */
+	IMG_UINT64				RGXFW_ALIGN ui64FBCStateIDMask;	/*!< FBCDC state descriptor IDs (non-zero means defer on mem context activation) */
+	IMG_UINT64				RGXFW_ALIGN ui64SpillAddr;
+	IMG_UINT32				ui32FwMemCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	IMG_UINT32              ui32OSid;
+	IMG_BOOL                bOSidAxiProt;
+#endif
+
+} UNCACHED_ALIGN RGXFWIF_FWMEMCONTEXT;
+
+/*!
+ * FW context state flags
+ */
+#define RGXFWIF_CONTEXT_FLAGS_NEED_RESUME			(0x00000001U)
+#define RGXFWIF_CONTEXT_FLAGS_TDM_HEADER_STALE		(0x00000002U)
+#define RGXFWIF_CONTEXT_FLAGS_LAST_KICK_SECURE		(0x00000200U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend state (per GEOM core)
+ */
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN uTAReg_DCE_CMD0;
+	IMG_UINT32				uTAReg_DCE_CMD1;
+	IMG_UINT32				uTAReg_DCE_WRITE;
+	IMG_UINT64	RGXFW_ALIGN uTAReg_DCE_DRAW0;
+	IMG_UINT64	RGXFW_ALIGN uTAReg_DCE_DRAW1;
+	IMG_UINT64				ui64EnabledUnitsMask;
+	IMG_UINT32				uTAReg_GTA_SO_PRIM[4];
+	IMG_UINT16	ui16TACurrentIdx;
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE_PER_GEOM;
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend states for all GEOM cores
+ */
+typedef struct
+{
+	/*! FW-accessible TA state which must be written out to memory on context store */
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[RGX_NUM_GEOM_CORES];
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE;
+
+/* The following defines need to be auto generated using the HW defines
+ * rather than hard coding it */
+#define RGXFWIF_ISP_PIPE_COUNT_MAX		(48)
+#define RGXFWIF_PIPE_COUNT_PER_ISP		(2)
+#define RGXFWIF_IPP_RESUME_REG_COUNT	(1)
+
+#if !defined(__KERNEL__)
+#define RGXFWIF_ISP_COUNT		(RGX_FEATURE_NUM_SPU * RGX_FEATURE_NUM_ISP_PER_SPU)
+#define RGXFWIF_ISP_PIPE_COUNT	(RGXFWIF_ISP_COUNT * RGXFWIF_PIPE_COUNT_PER_ISP)
+#if RGXFWIF_ISP_PIPE_COUNT > RGXFWIF_ISP_PIPE_COUNT_MAX
+#error RGXFWIF_ISP_PIPE_COUNT should not be greater than RGXFWIF_ISP_PIPE_COUNT_MAX
+#endif
+#endif /* !defined(__KERNEL__) */
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware FRAG/3D context suspend state
+ */
+typedef struct
+{
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT32	RGXFW_ALIGN u3DReg_PM_DEALLOCATED_MASK_STATUS;	/*!< Managed by PM HW in the non-interactive mode */
+#endif
+	IMG_UINT32	ui32CtxStateFlags;	/*!< Compatibility and other flags */
+
+	/* FW-accessible ISP state which must be written out to memory on context store */
+	/* au3DReg_ISP_STORE should be the last element of the structure
+	 * as this is an array whose size is determined at runtime
+	 * after detecting the RGX core */
+	IMG_UINT64	RGXFW_ALIGN au3DReg_ISP_STORE[]; /*!< ISP state (per-pipe) */
+} UNCACHED_ALIGN RGXFWIF_3DCTX_STATE;
+
+#define RGXFWIF_CTX_USING_BUFFER_A		(0)
+#define RGXFWIF_CTX_USING_BUFFER_B		(1U)
+
+typedef struct
+{
+	IMG_UINT32  ui32CtxStateFlags;		/*!< Target buffer and other flags */
+	IMG_UINT64  ui64EnabledUnitsMask;
+} RGXFWIF_COMPUTECTX_STATE;
+
+typedef struct
+{
+	bool       bSaved;
+	IMG_UINT64 ui64CheckSum[4];
+}RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY;
+
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_STATS_PENDING   (1U << 0)
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT (1U << 1)
+
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_2D[RGX_TRP_MAX_NUM_CORES][2];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_3D[RGX_TRP_MAX_NUM_CORES][4];
+typedef RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY RGXFWIF_TRP_CHECKSUM_GEOM[RGX_TRP_MAX_NUM_CORES];
+
+typedef struct
+{
+	IMG_UINT32	ui32ExtJobRefToDisableZSStore;
+	IMG_BOOL	bDisableZStore;
+	IMG_BOOL	bDisableSStore;
+} RGXFWIF_DISABLE_ZSSTORE;
+
+#define MAX_ZSSTORE_DISABLE 8
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware render context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTAContext;				/*!< Firmware context for the TA */
+	RGXFWIF_FWCOMMONCONTEXT	s3DContext;				/*!< Firmware context for the 3D */
+
+	RGXFWIF_STATIC_RENDERCONTEXT_STATE sStaticRenderContextState;
+
+	RGXFWIF_DISABLE_ZSSTORE sDisableZSStoreQueue[MAX_ZSSTORE_DISABLE];
+
+	IMG_UINT32			ui32ZSStoreQueueCount;
+	IMG_UINT32			ui32WriteOffsetOfDisableZSStore;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32			ui32FwRenderCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_3D		aui64TRPChecksums3D;	/*!< Used by Firmware to store checksums during 3D WRR */
+	RGXFWIF_TRP_CHECKSUM_GEOM	aui64TRPChecksumsGeom;	/*!< Used by Firmware to store checksums during TA WRR */
+	RGXFWIF_DM			eTRPGeomCoreAffinity; /* !< Represent the DM affinity for pending 2nd TRP pass of GEOM otherwise points RGXFWIF_DM_MAX. */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWRENDERCONTEXT;
+
+/*!
+	Firmware compute context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT sCDMContext;				/*!< Firmware context for the CDM */
+
+	RGXFWIF_STATIC_COMPUTECONTEXT_STATE sStaticComputeContextState;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32 ui32ComputeCtxFlags; /*!< Compatibility and other flags */
+
+	IMG_UINT32		aui32WGPChecksum[RGX_WGP_MAX_NUM_CORES];
+} UNCACHED_ALIGN RGXFWIF_FWCOMPUTECONTEXT;
+
+/*!
+	Firmware ray context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT sRDMContext;				/*!< Firmware context for the RDM */
+	RGXFWIF_STATIC_RAYCONTEXT_STATE sStaticRayContextState;
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+} UNCACHED_ALIGN RGXFWIF_FWRAYCONTEXT;
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware TDM context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTDMContext;				/*!< Firmware context for the TDM */
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_2D	RGXFW_ALIGN	aui64TRPChecksums2D; /*!< Used by Firmware to store checksums during TDM WRR */
+#endif
+
+} UNCACHED_ALIGN RGXFWIF_FWTDMCONTEXT;
+
+/*!
+ ******************************************************************************
+ * Defines for CMD_TYPE corruption detection and forward compatibility check
+ *****************************************************************************/
+
+/* CMD_TYPE 32bit contains:
+ * 31:16	Reserved for magic value to detect corruption (16 bits)
+ * 15		Reserved for RGX_CCB_TYPE_TASK (1 bit)
+ * 14:0		Bits available for CMD_TYPEs (15 bits) */
+
+
+/* Magic value to detect corruption */
+#define RGX_CMD_MAGIC_DWORD			IMG_UINT32_C(0x2ABC)
+#define RGX_CMD_MAGIC_DWORD_MASK	(0xFFFF0000U)
+#define RGX_CMD_MAGIC_DWORD_SHIFT	(16U)
+#define RGX_CMD_MAGIC_DWORD_SHIFTED	(RGX_CMD_MAGIC_DWORD << RGX_CMD_MAGIC_DWORD_SHIFT)
+
+/*!
+ * @InGroup KCCBTypes ClientCCBTypes
+ * @Brief Generic CCB control structure
+ */
+typedef struct
+{
+	volatile IMG_UINT32		ui32WriteOffset;		/*!< write offset into array of commands (MUST be aligned to 16 bytes!) */
+	volatile IMG_UINT32		ui32ReadOffset;			/*!< read offset into array of commands */
+	IMG_UINT32				ui32WrapMask;			/*!< Offset wrapping mask (Total capacity of the CCB - 1) */
+} UNCACHED_ALIGN RGXFWIF_CCB_CTL;
+
+/*!
+ * @Defgroup KCCBTypes Kernel CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the KCCB functionality
+ * @{
+ */
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PT      (0x1U) /* MMU_CTRL_INVAL_PT_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PD      (0x2U) /* MMU_CTRL_INVAL_PD_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PC      (0x4U) /* MMU_CTRL_INVAL_PC_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x800U) /* MMU_CTRL_INVAL_ALL_CONTEXTS_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (0x0) /* not used */
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT (0x4000000U) /* indicates FW should interrupt the host */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_MMUCACHE type command
+ */
+typedef struct
+{
+	IMG_UINT32            ui32CacheFlags;
+	RGXFWIF_DEV_VIRTADDR  sMMUCacheSync;
+	IMG_UINT32            ui32MMUCacheSyncUpdateValue;
+} RGXFWIF_MMUCACHEDATA;
+
+#define RGXFWIF_BPDATA_FLAGS_ENABLE (1U << 0)
+#define RGXFWIF_BPDATA_FLAGS_WRITE  (1U << 1)
+#define RGXFWIF_BPDATA_FLAGS_CTL    (1U << 2)
+#define RGXFWIF_BPDATA_FLAGS_REGS   (1U << 3)
+
+typedef struct
+{
+	PRGXFWIF_FWMEMCONTEXT		psFWMemContext;		/*!< Memory context */
+	IMG_UINT32					ui32BPAddr;			/*!< Breakpoint address */
+	IMG_UINT32					ui32HandlerAddr;	/*!< Breakpoint handler */
+	IMG_UINT32					ui32BPDM;			/*!< Breakpoint control */
+	IMG_UINT32					ui32BPDataFlags;
+	IMG_UINT32					ui32TempRegs;		/*!< Number of temporary registers to overallocate */
+	IMG_UINT32					ui32SharedRegs;		/*!< Number of shared registers to overallocate */
+	IMG_UINT64 RGXFW_ALIGN		ui64SpillAddr;
+	RGXFWIF_DM					eDM;				/*!< DM associated with the breakpoint */
+} RGXFWIF_BPDATA;
+
+#define RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS (RGXFWIF_PRBUFFER_MAXSUPPORTED + 1U) /* +1 is RTDATASET cleanup */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_KICK type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CWoffUpdate;	/*!< Client CCB woff update */
+	IMG_UINT32					ui32CWrapMaskUpdate; /*!< Client CCB wrap mask update after CCCB growth */
+	IMG_UINT32					ui32NumCleanupCtl;		/*!< number of CleanupCtl pointers attached */
+	PRGXFWIF_CLEANUP_CTL		apsCleanupCtl[RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS]; /*!< CleanupCtl structures associated with command */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32					ui32WorkEstCmdHeaderOffset; /*!< offset to the CmdHeader which houses the workload estimation kick data. */
+#endif
+} RGXFWIF_KCCB_CMD_KICK_DATA;
+
+/*!
+ * @Brief Command data for @Ref RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK type command
+ */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_KICK_DATA	sTACmdKickData; /*!< GEOM DM kick command data */
+	RGXFWIF_KCCB_CMD_KICK_DATA	s3DCmdKickData; /*!< FRAG DM kick command data */
+} RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FORCE_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CCBFenceOffset;	/*!< Client CCB fence offset */
+} RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;
+	RGXFWIF_DISABLE_ZSSTORE		sDisableZSStore;
+} RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA;
+
+/*!
+ * @Brief Resource types supported by \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef enum
+{
+	RGXFWIF_CLEANUP_FWCOMMONCONTEXT,		/*!< FW common context cleanup */
+	RGXFWIF_CLEANUP_HWRTDATA,				/*!< FW HW RT data cleanup */
+	RGXFWIF_CLEANUP_FREELIST,				/*!< FW freelist cleanup */
+	RGXFWIF_CLEANUP_ZSBUFFER,				/*!< FW ZS Buffer cleanup */
+} RGXFWIF_CLEANUP_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef struct
+{
+	RGXFWIF_CLEANUP_TYPE			eCleanupType;			/*!< Cleanup type */
+	union {
+		PRGXFWIF_FWCOMMONCONTEXT	psContext;				/*!< FW common context to cleanup */
+		PRGXFWIF_HWRTDATA			psHWRTData;				/*!< HW RT to cleanup */
+		PRGXFWIF_FREELIST			psFreelist;				/*!< Freelist to cleanup */
+		PRGXFWIF_ZSBUFFER			psZSBuffer;				/*!< ZS Buffer to cleanup */
+	} uCleanupData;
+} RGXFWIF_CLEANUP_REQUEST;
+
+/*!
+ * @Brief Type of power requests supported in \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef enum
+{
+	RGXFWIF_POW_OFF_REQ = 1,           /*!< GPU power-off request */
+	RGXFWIF_POW_FORCED_IDLE_REQ,       /*!< Force-idle related request */
+	RGXFWIF_POW_NUM_UNITS_CHANGE,      /*!< Request to change default powered scalable units */
+	RGXFWIF_POW_APM_LATENCY_CHANGE     /*!< Request to change the APM latency period */
+} RGXFWIF_POWER_TYPE;
+
+/*!
+ * @Brief Supported force-idle related requests with \ref RGXFWIF_POW_FORCED_IDLE_REQ type request
+ */
+typedef enum
+{
+	RGXFWIF_POWER_FORCE_IDLE = 1,      /*!< Request to force-idle GPU */
+	RGXFWIF_POWER_CANCEL_FORCED_IDLE,  /*!< Request to cancel a previously successful force-idle transition */
+	RGXFWIF_POWER_HOST_TIMEOUT,        /*!< Notification that host timed-out waiting for force-idle state */
+} RGXFWIF_POWER_FORCE_IDLE_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef struct
+{
+	RGXFWIF_POWER_TYPE					ePowType;					/*!< Type of power request */
+	union
+	{
+		struct
+		{
+			IMG_UINT32					ui32PowUnits;	/*!< New power units state mask */
+			IMG_UINT32					ui32RACUnits;	/*!< New RAC state mask */
+		};
+		IMG_BOOL						bForced;				/*!< If the operation is mandatory */
+		RGXFWIF_POWER_FORCE_IDLE_TYPE	ePowRequestType;		/*!< Type of Request. Consolidating Force Idle, Cancel Forced Idle, Host Timeout */
+	} uPowerReqData;
+} RGXFWIF_POWER_REQUEST;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_SLCFLUSHINVAL type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext; /*!< Context to fence on (only useful when bDMContext == TRUE) */
+	IMG_BOOL    bInval;                 /*!< Invalidate the cache as well as flushing */
+	IMG_BOOL    bDMContext;             /*!< The data to flush/invalidate belongs to a specific DM context */
+	IMG_UINT64	RGXFW_ALIGN ui64Address;	/*!< Optional address of range (only useful when bDMContext == FALSE) */
+	IMG_UINT64	RGXFW_ALIGN ui64Size;		/*!< Optional size of range (only useful when bDMContext == FALSE) */
+} RGXFWIF_SLCFLUSHINVALDATA;
+
+typedef enum
+{
+	RGXFWIF_HWPERF_CTRL_TOGGLE = 0,
+	RGXFWIF_HWPERF_CTRL_SET    = 1,
+	RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV = 2
+} RGXFWIF_HWPERF_UPDATE_CONFIG;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG type command
+ */
+typedef struct
+{
+	RGXFWIF_HWPERF_UPDATE_CONFIG eOpCode; /*!< Control operation code */
+	IMG_UINT64	RGXFW_ALIGN	ui64Mask;   /*!< Mask of events to toggle */
+} RGXFWIF_HWPERF_CTRL;
+
+typedef enum
+{
+	RGXFWIF_HWPERF_CNTR_NOOP    = 0,   /* No-Op */
+	RGXFWIF_HWPERF_CNTR_ENABLE  = 1,   /* Enable Counters */
+	RGXFWIF_HWPERF_CNTR_DISABLE = 2    /* Disable Counters */
+} RGXFWIF_HWPERF_CNTR_CONFIG;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS type command
+ */
+typedef struct
+{
+	IMG_UINT32                ui32CtrlWord;
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE type command
+ */
+typedef struct
+{
+	IMG_UINT32	ui32NewClockSpeed;			/*!< New clock speed */
+} RGXFWIF_CORECLKSPEEDCHANGE_DATA;
+
+#define RGXFWIF_HWPERF_CTRL_BLKS_MAX	16U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS type command
+ */
+typedef struct
+{
+	bool		bEnable;
+	IMG_UINT32	ui32NumBlocks;                              /*!< Number of block IDs in the array */
+	IMG_UINT16	aeBlockIDs[RGXFWIF_HWPERF_CTRL_BLKS_MAX];   /*!< Array of RGX_HWPERF_CNTBLK_ID values */
+} RGXFWIF_HWPERF_CTRL_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE & \ref RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE type commands
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;				/*!< ZS-Buffer FW address */
+	IMG_BOOL				bDone;								/*!< action backing/unbacking succeeded */
+} RGXFWIF_ZSBUFFER_BACKING_DATA;
+
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE type command
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;				/*!< Freelist FW address */
+	IMG_UINT32				ui32DeltaPages;						/*!< Amount of the Freelist change */
+	IMG_UINT32				ui32NewPages;						/*!< New amount of pages on the freelist (including ready pages) */
+	IMG_UINT32              ui32ReadyPages;                     /*!< Number of ready pages to be held in reserve until OOM */
+} RGXFWIF_FREELIST_GS_DATA;
+
+/* Max freelists must include freelists loaded (for all kick IDs) and freelists being setup. */
+#define RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT         (MAX_HW_TA3DCONTEXTS * RGXFW_MAX_FREELISTS * 3U)
+#define RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG 0x80000000U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE type command
+ */
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+} RGXFWIF_FREELISTS_RECONSTRUCTION_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT  psContext; /*!< Context to that may need to be resumed following write offset update */
+} UNCACHED_ALIGN RGXFWIF_WRITE_OFFSET_UPDATE_DATA;
+
+/*!
+ ******************************************************************************
+ * Proactive DVFS Structures
+ *****************************************************************************/
+#define NUM_OPP_LEVELS 16
+
+typedef struct
+{
+	IMG_UINT32			ui32Volt; /* V  */
+	IMG_UINT32			ui32Freq; /* Hz */
+} UNCACHED_ALIGN OPP_LEVEL;
+
+typedef struct
+{
+	OPP_LEVEL		asOPPValues[NUM_OPP_LEVELS];
+#if defined(DEBUG)
+	IMG_UINT32		ui32MinOPPPoint;
+#endif
+	IMG_UINT32		ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_OPP_INFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MAX_FREQ_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32MinOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MIN_FREQ_DATA;
+
+/*!
+ ******************************************************************************
+ * Register configuration structures
+ *****************************************************************************/
+
+#define RGXFWIF_REG_CFG_MAX_SIZE 512
+
+typedef enum
+{
+	RGXFWIF_REGCFG_CMD_ADD				= 101,
+	RGXFWIF_REGCFG_CMD_CLEAR			= 102,
+	RGXFWIF_REGCFG_CMD_ENABLE			= 103,
+	RGXFWIF_REGCFG_CMD_DISABLE			= 104
+} RGXFWIF_REGDATA_CMD_TYPE;
+
+typedef struct
+{
+	IMG_UINT64		ui64Addr;
+	IMG_UINT64		ui64Mask;
+	IMG_UINT64		ui64Value;
+} RGXFWIF_REG_CFG_REC;
+
+typedef struct
+{
+	RGXFWIF_REGDATA_CMD_TYPE         eCmdType;
+	RGXFWIF_REG_CFG_TYPE             eRegConfigType;
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  sRegConfig;
+
+} RGXFWIF_REGCONFIG_DATA;
+
+typedef struct
+{
+	/**
+	 * PDump WRW command write granularity is 32 bits.
+	 * Add padding to ensure array size is 32 bit granular.
+	 */
+	IMG_UINT8           RGXFW_ALIGN  aui8NumRegsType[PVR_ALIGN((IMG_UINT32)RGXFWIF_REG_CFG_TYPE_ALL,sizeof(IMG_UINT32))];
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  asRegConfigs[RGXFWIF_REG_CFG_MAX_SIZE];
+} UNCACHED_ALIGN RGXFWIF_REG_CFG;
+
+typedef enum
+{
+	RGXFWIF_OS_ONLINE = 1,
+	RGXFWIF_OS_OFFLINE
+} RGXFWIF_OS_STATE_CHANGE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE type command
+ */
+typedef struct
+{
+	IMG_UINT32 ui32DriverID;
+	RGXFWIF_OS_STATE_CHANGE eNewOSState;
+} UNCACHED_ALIGN RGXFWIF_OS_STATE_CHANGE_DATA;
+
+typedef enum
+{
+	RGXFWIF_PWR_COUNTER_DUMP_START = 1,
+	RGXFWIF_PWR_COUNTER_DUMP_STOP,
+	RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+} RGXFWIF_COUNTER_DUMP_REQUEST;
+
+typedef struct
+{
+	RGXFWIF_COUNTER_DUMP_REQUEST eCounterDumpRequest;
+}  RGXFW_ALIGN RGXFWIF_COUNTER_DUMP_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext;
+	IMG_UINT32               ui32FirstIntJobRefToCancel;
+	IMG_UINT32               ui32LastIntJobRefToCancel;
+} UNCACHED_ALIGN RGXFWIF_CANCEL_WORK_DATA;
+
+/*!
+ ******************************************************************************
+ * Platform configuration structures
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32 ui32PlatformCmd;
+
+	/* !! Customisable properties/values follow !! */
+	IMG_UINT32 ui32ClockFrequencyExample;
+} UNCACHED_ALIGN RGXFWIF_PLATFORM_DATA;
+
+
+/*!
+ * @Brief List of command types supported by the Kernel CCB
+ */
+typedef enum
+{
+	/* Common commands */
+	RGXFWIF_KCCB_CMD_KICK								= 101U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< DM workload kick command */
+	RGXFWIF_KCCB_CMD_MMUCACHE							= 102U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< MMU cache invalidation request */
+	RGXFWIF_KCCB_CMD_BP									= 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_SLCFLUSHINVAL						= 104U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< SLC flush and invalidation request */
+	RGXFWIF_KCCB_CMD_CLEANUP							= 105U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Requests cleanup of a FW resource (type specified in the command data) */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE			= 106U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Backing for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE			= 107U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Unbacking for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE				= 108U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelist Grow done */
+	RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE	= 109U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelists Reconstruction done */
+	RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE			= 110U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the firmware that the host has added more data to a CDM2 Circular Buffer */
+	RGXFWIF_KCCB_CMD_HEALTH_CHECK						= 111U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Health check request */
+	RGXFWIF_KCCB_CMD_FORCE_UPDATE						= 112U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Forcing signalling of all unmet UFOs for a given CCB offset */
+	RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK				= 113U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< There is a TA and a 3D command in this single kick */
+	RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE			= 114U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW that a Guest OS has come online / offline. */
+	RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE					= 115U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW to disable zs store of a running 3D or add it to queue of render context. */
+	RGXFWIF_KCCB_CMD_CANCEL_WORK						= 116U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Cancel all work up to and including a given intjobref for a given context */
+	/* Commands only permitted to the native or host OS */
+	RGXFWIF_KCCB_CMD_POW								= 200U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Power request (type specified in the command data) */
+	RGXFWIF_KCCB_CMD_REGCONFIG							= 201U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE					= 202U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Core clock speed change event */
+	RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE						= 203U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Ask the firmware to update its cached ui32LogType value from the (shared) tracebuf control structure */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ				= 204U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a maximum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE				= 205U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the priority/group for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL					= 206U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set or clear firmware state flags */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MIN_FREQ				= 207U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a minimum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_PHR_CFG							= 208U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Periodic Hardware Reset behaviour */
+	RGXFWIF_KCCB_CMD_WDG_CFG							= 210U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Safety Firmware Watchdog */
+	RGXFWIF_KCCB_CMD_COUNTER_DUMP						= 211U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Controls counter dumping in the FW */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE					= 213U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL			= 214U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice interval for all drivers. It can only be serviced for the Host DDK */
+
+	/* HWPerf commands */
+	RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG				= 300U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure HWPerf events (to be generated) and HWPerf buffer address (if required) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS					= 301U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure, clear and enable multiple HWPerf blocks */
+	RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS					= 302U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Enable or disable multiple HWPerf blocks (reusing existing configuration) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS_DIRECT	= 303U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure, clear and enable multiple HWPerf blocks during the init process */
+
+	/* Indirect platform command */
+	/* !! Do not add internal commands to the 4xx range !! */
+	RGXFWIF_KCCB_CMD_PLATFORM_CMD						= 400U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Send a platform specific command */
+
+} RGXFWIF_KCCB_CMD_TYPE;
+
+#define RGXFWIF_LAST_ALLOWED_GUEST_KCCB_CMD (RGXFWIF_KCCB_CMD_POW - 1)
+
+/*! @Brief Kernel CCB command packet */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_TYPE  eCmdType;      /*!< Command type */
+	IMG_UINT32             ui32KCCBFlags; /*!< Compatibility and other flags */
+
+	/* NOTE: Make sure that uCmdData is the last member of this struct
+	 * This is to calculate actual command size for device mem copy.
+	 * (Refer RGXGetCmdMemCopySize())
+	 * */
+	union
+	{
+		RGXFWIF_KCCB_CMD_KICK_DATA			sCmdKickData;			/*!< Data for Kick command */
+		RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA	sCombinedTA3DCmdKickData;	/*!< Data for combined TA/3D Kick command */
+		RGXFWIF_MMUCACHEDATA				sMMUCacheData;			/*!< Data for MMU cache command */
+		RGXFWIF_BPDATA						sBPData;				/*!< Data for Breakpoint Commands */
+		RGXFWIF_SLCFLUSHINVALDATA			sSLCFlushInvalData;		/*!< Data for SLC Flush/Inval commands */
+		RGXFWIF_CLEANUP_REQUEST				sCleanupData;			/*!< Data for cleanup commands */
+		RGXFWIF_POWER_REQUEST				sPowData;				/*!< Data for power request commands */
+		RGXFWIF_HWPERF_CTRL					sHWPerfCtrl;			/*!< Data for HWPerf control command */
+		RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS	sHWPerfCfgEnableBlks;	/*!< Data for HWPerf configure, clear and enable performance counter block command */
+		RGXFWIF_HWPERF_CTRL_BLKS			sHWPerfCtrlBlks;		/*!< Data for HWPerf enable or disable performance counter block commands */
+		RGXFWIF_CORECLKSPEEDCHANGE_DATA		sCoreClkSpeedChangeData;/*!< Data for core clock speed change */
+		RGXFWIF_ZSBUFFER_BACKING_DATA		sZSBufferBackingData;	/*!< Feedback for Z/S Buffer backing/unbacking */
+		RGXFWIF_FREELIST_GS_DATA			sFreeListGSData;		/*!< Feedback for Freelist grow/shrink */
+		RGXFWIF_FREELISTS_RECONSTRUCTION_DATA	sFreeListsReconstructionData;	/*!< Feedback for Freelists reconstruction */
+		RGXFWIF_REGCONFIG_DATA				sRegConfigData;			/*!< Data for custom register configuration */
+		RGXFWIF_WRITE_OFFSET_UPDATE_DATA    sWriteOffsetUpdateData; /*!< Data for informing the FW about the write offset update */
+#if defined(SUPPORT_PDVFS)
+		RGXFWIF_PDVFS_MAX_FREQ_DATA			sPDVFSMaxFreqData;
+		RGXFWIF_PDVFS_MIN_FREQ_DATA			sPDVFSMinFreqData;		/*!< Data for setting the min frequency/OPP */
+#endif
+		RGXFWIF_OS_STATE_CHANGE_DATA        sCmdOSOnlineStateData;  /*!< Data for updating the Guest Online states */
+		RGXFWIF_DEV_VIRTADDR                sTBIBuffer;             /*!< Dev address for TBI buffer allocated on demand */
+		RGXFWIF_COUNTER_DUMP_DATA			sCounterDumpConfigData; /*!< Data for dumping of register ranges */
+		RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA	sDisableZSStoreData;	/*!< Data for disabling zs store of a 3D workload */
+		RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA  sForceUpdateData;       /*!< Data for signalling all unmet fences for a given CCB */
+		RGXFWIF_CANCEL_WORK_DATA			sCancelWorkData;		/*!< Data for cancelling work */
+		RGXFWIF_PLATFORM_DATA               sPlatformData;          /*!< Data for sending platform specific parameters */
+	} UNCACHED_ALIGN uCmdData;
+} UNCACHED_ALIGN RGXFWIF_KCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_KCCB_CMD);
+
+/*! @} End of KCCBTypes */
+
+/*!
+ * @Defgroup FWCCBTypes Firmware CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the Firmware CCB functionality
+ * @{
+ */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING and the
+ * \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING Firmware CCB commands
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32ZSBufferID; /*!< ZS buffer ID */
+} RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELIST_GROW Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32FreelistID; /*!< Freelist ID */
+} RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;                                     /*!< Freelists count */
+	IMG_UINT32			ui32HwrCounter;                                         /*!< HWR counter */
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT]; /*!< Array of freelist IDs to reconstruct */
+} RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA;
+
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF			(1U<<0)	/*!< 1 if a page fault happened */
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS	(1U<<1)	/*!< 1 if applicable to all contexts */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32						ui32ServerCommonContextID;	/*!< Context affected by the reset */
+	RGX_CONTEXT_RESET_REASON		eResetReason;				/*!< Reason for reset */
+	RGXFWIF_DM						eDM;						/*!< Data Master affected by the reset */
+	IMG_UINT32						ui32ResetJobRef;			/*!< Job ref running at the time of reset */
+	IMG_UINT32						ui32Flags;					/*!< RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG bitfield */
+	IMG_UINT64 RGXFW_ALIGN			ui64PCAddress;				/*!< At what page catalog address */
+	IMG_DEV_VIRTADDR RGXFW_ALIGN	sFaultAddress;				/*!< Page fault address (only when applicable) */
+} RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;	/*!< Page fault address */
+} RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA;
+
+/*!
+ ******************************************************************************
+ * List of command types supported by the Firmware CCB
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING              = 101U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be backed with physical pages
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING            = 102U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be unbacked
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_FREELIST_GROW                 = 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand freelist grow
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA */
+	RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION      = 104U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests freelists reconstruction
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA */
+	RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION    = 105U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a HWR event on a context
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA */
+	RGXFWIF_FWCCB_CMD_DEBUG_DUMP                    = 106U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand debug dump
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS                  = 107U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand update on process stats
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA */
+	RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE          = 108U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART           = 109U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests GPU restart
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION    = 112U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a FW pagefault
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA */
+} RGXFWIF_FWCCB_CMD_TYPE;
+
+/*!
+ ******************************************************************************
+ * List of the various stats of the process to update/increment
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS=1,		/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumPartialRenders stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumOutOfMemory stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTAStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32Num3DStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumCDMStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTDMStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_RAY_STORES				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumRayStores stat */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_UPDATE_STATS Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE		eElementToUpdate;			/*!< Element to update */
+	IMG_PID									pidOwner;					/*!< The pid of the process whose stats are being updated */
+	IMG_INT32								i32AdjustmentValue;			/*!< Adjustment to be made to the statistic */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32CoreClkRate;
+} UNCACHED_ALIGN RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA;
+
+
+/*!
+ ******************************************************************************
+ * @Brief Firmware CCB command structure
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_TYPE  eCmdType;       /*!< Command type */
+	IMG_UINT32              ui32FWCCBFlags; /*!< Compatibility and other flags */
+
+	union
+	{
+		RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA				sCmdZSBufferBacking;			/*!< Data for Z/S-Buffer on-demand (un)backing*/
+		RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA					sCmdFreeListGS;					/*!< Data for on-demand freelist grow/shrink */
+		RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA		sCmdFreeListsReconstruction;	/*!< Data for freelists reconstruction */
+		RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA				sCmdContextResetNotification;	/*!< Data for context reset notification */
+		RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA					sCmdUpdateStatsData;			/*!< Data for updating process stats */
+		RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA			sCmdCoreClkRateChange;
+		RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA					sCmdFWPagefault;				/*!< Data for context reset notification */
+	} RGXFW_ALIGN uCmdData;
+} RGXFW_ALIGN RGXFWIF_FWCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_FWCCB_CMD);
+
+/*! @} End of FWCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Workload estimation Firmware CCB command structure for RGX
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT16 ui16ReturnDataIndex; /*!< Index for return data array */
+	IMG_UINT16 ui16CyclesTakenHigh;	/*!< [39:32] from the cycle counter, which has up to 40-bit range. */
+	IMG_UINT32 ui32CyclesTaken;     /*!< The cycles the workload took on the hardware */
+} RGXFWIF_WORKEST_FWCCB_CMD;
+
+/*!
+ * @Defgroup ClientCCBTypes Client CCB data interface
+ * @Brief Types grouping data structures and defines used in realising Client CCB commands/functionality
+ * @{
+ */
+
+/* Required memory alignment for 64-bit variables accessible by Meta
+  (The gcc meta aligns 64-bit variables to 64-bit; therefore, memory shared
+   between the host and meta that contains 64-bit variables has to maintain
+   this alignment) */
+#define RGXFWIF_FWALLOC_ALIGN	sizeof(IMG_UINT64)
+
+#define RGX_CCB_TYPE_TASK			(IMG_UINT32_C(1) << 15)
+#define RGX_CCB_FWALLOC_ALIGN(size)	(PVR_ALIGN(size, RGXFWIF_FWALLOC_ALIGN))
+
+typedef IMG_UINT32 RGXFWIF_CCB_CMD_TYPE;
+
+/*!
+ * @Name Client CCB command types
+ * @{
+ */
+#define RGXFWIF_CCB_CMD_TYPE_GEOM			(201U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TA DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_3D			(202U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_3D				(203U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command */
+#define RGXFWIF_CCB_CMD_TYPE_3D_PR			(204U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for Partial render */
+#define RGXFWIF_CCB_CMD_TYPE_CDM			(205U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Compute DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_TDM			(206U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TDM command */
+#define RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE (207U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_TQ_2D			(208U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 2D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP	(209U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_NULL			(210U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_ABORT			(211U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+
+/* Leave a gap between CCB specific commands and generic commands */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE          (212U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a command */
+#define RGXFWIF_CCB_CMD_TYPE_UPDATE         (213U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates of a command */
+#define RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE     (214U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates related to workload resources */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE_PR       (215U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a PR command */
+#define RGXFWIF_CCB_CMD_TYPE_PRIORITY       (216U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Context priority update command */
+/* Pre and Post timestamp commands are supposed to sandwich the DM cmd. The
+   padding code with the CCB wrap upsets the FW if we don't have the task type
+   bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+*/
+#define RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP (217U | RGX_CMD_MAGIC_DWORD_SHIFTED)
+/* UNFENCED type is not supported but keeping defines as it is for now */
+
+
+#define RGXFWIF_CCB_CMD_TYPE_PADDING	(221U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Skip without action type command */
+#define RGXFWIF_CCB_CMD_TYPE_RAY		(222U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP	(223U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Process a vulkan timestamp */
+/*! @} End of Client CCB command types */
+
+
+#define RGXFWIF_TRP_STATUS_UNKNOWN				0x000U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_OK			0x001U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_ERROR		0x002U
+
+#define RGXFWIF_CR_TRP_SIGNATURE_STATUS			(RGX_CR_SCRATCH10)
+
+
+typedef struct
+{
+	/* Index for the KM Workload estimation return data array */
+	IMG_UINT16 RGXFW_ALIGN         ui16ReturnDataIndex;
+	/* Predicted time taken to do the work in cycles */
+	IMG_UINT32 RGXFW_ALIGN         ui32CyclesPrediction;
+	/* Deadline for the workload (in usecs) */
+	IMG_UINT64 RGXFW_ALIGN         ui64Deadline;
+} RGXFWIF_WORKEST_KICK_DATA;
+
+/*! @Brief Command header of a command in the client CCB buffer.
+ *
+ *  Followed by this header is the command-data specific to the
+ *  command-type as specified in the header.
+ */
+typedef struct
+{
+	RGXFWIF_CCB_CMD_TYPE					eCmdType;      /*!< Command data type following this command header */
+	IMG_UINT32								ui32CmdSize;   /*!< Size of the command following this header */
+	IMG_UINT32								ui32ExtJobRef; /*!< external job reference - provided by client and used in debug for tracking submitted work */
+	IMG_UINT32								ui32IntJobRef; /*!< internal job reference - generated by services and used in debug for tracking submitted work */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA RGXFW_ALIGN	sWorkEstKickData; /*!< Workload Estimation - Workload Estimation Data */
+#endif
+} RGXFWIF_CCB_CMD_HEADER;
+
+/*
+ ******************************************************************************
+ * Client CCB commands which are only required by the kernel
+ *****************************************************************************/
+
+/*! @Brief Command data for \ref RGXFWIF_CCB_CMD_TYPE_PRIORITY type client CCB command */
+typedef struct
+{
+	IMG_INT32              i32Priority; /*!< Priority level */
+} RGXFWIF_CMD_PRIORITY;
+
+/*! @} End of ClientCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Signature and Checksums Buffer
+ *****************************************************************************/
+typedef struct
+{
+	PRGXFWIF_SIGBUFFER		sBuffer;			/*!< Ptr to Signature Buffer memory */
+	IMG_UINT32				ui32LeftSizeInRegs;	/*!< Amount of space left for storing regs in the buffer */
+} UNCACHED_ALIGN RGXFWIF_SIGBUF_CTL;
+
+typedef struct
+{
+	PRGXFWIF_COUNTERBUFFER	sBuffer;			/*!< Ptr to counter dump buffer */
+	IMG_UINT32				ui32SizeInDwords;	/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_COUNTER_DUMP_CTL;
+
+typedef struct
+{
+	PRGXFWIF_FIRMWAREGCOVBUFFER	sBuffer;		/*!< Ptr to firmware gcov buffer */
+	IMG_UINT32					ui32Size;		/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_FIRMWARE_GCOV_CTL;
+
+/*!
+ *****************************************************************************
+ * RGX Compatibility checks
+ *****************************************************************************/
+
+/* WARNING: Whenever the layout of RGXFWIF_COMPCHECKS_BVNC changes, the
+	following define should be increased by 1 to indicate to the
+	compatibility logic that layout has changed. */
+#define RGXFWIF_COMPCHECKS_LAYOUT_VERSION 3
+
+typedef struct
+{
+	IMG_UINT32	ui32LayoutVersion; /* WARNING: This field must be defined as first one in this structure */
+	IMG_UINT64	RGXFW_ALIGN ui64BVNC;
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS_BVNC;
+
+typedef struct
+{
+	IMG_UINT8	ui8OsCountSupport;
+} UNCACHED_ALIGN RGXFWIF_INIT_OPTIONS;
+
+#define RGXFWIF_COMPCHECKS_BVNC_DECLARE_AND_INIT(name) \
+	RGXFWIF_COMPCHECKS_BVNC (name) = { \
+		RGXFWIF_COMPCHECKS_LAYOUT_VERSION, \
+		0, \
+	}
+#define RGXFWIF_COMPCHECKS_BVNC_INIT(name) \
+	do { \
+		(name).ui32LayoutVersion = RGXFWIF_COMPCHECKS_LAYOUT_VERSION; \
+		(name).ui64BVNC = 0; \
+	} while (false)
+
+typedef struct
+{
+	RGXFWIF_COMPCHECKS_BVNC		sHWBVNC;				/*!< hardware BVNC (from the RGX registers) */
+	RGXFWIF_COMPCHECKS_BVNC		sFWBVNC;				/*!< firmware BVNC */
+	IMG_UINT32					ui32DDKVersion;			/*!< software DDK version */
+	IMG_UINT32					ui32DDKBuild;			/*!< software DDK build no. */
+	IMG_UINT32					ui32BuildOptions;		/*!< build options bit-field */
+	RGXFWIF_INIT_OPTIONS		sInitOptions;			/*!< initialisation options bit-field */
+	IMG_BOOL					bUpdated;				/*!< Information is valid */
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS;
+
+typedef struct
+{
+	IMG_UINT32                 ui32NumCores;
+	IMG_UINT32                 ui32MulticoreInfo;
+} UNCACHED_ALIGN RGXFWIF_MULTICORE_INFO;
+
+/*! @Brief Firmware Runtime configuration data \ref RGXFWIF_RUNTIME_CFG
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	IMG_UINT32         ui32ActivePMLatencyms;                    /*!< APM latency in ms before signalling IDLE to the host */
+	IMG_UINT32         ui32RuntimeCfgFlags;                      /*!< Compatibility and other flags */
+	IMG_BOOL           bActivePMLatencyPersistant;               /*!< If set, APM latency does not reset to system default each GPU power transition */
+	IMG_UINT32         ui32CoreClockSpeed;                       /*!< Core clock speed, currently only used to calculate timer ticks */
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32         ui32SOCClockSpeed;                        /*!< SOC clock speed, used for deadline scheduling */
+#endif
+	IMG_UINT32         ui32PowUnitsState;                        /*!< Power Unit state mask set by the host */
+	IMG_UINT32         ui32RACUnitsState;                        /*!< RAC state mask set by the host */
+	IMG_UINT32         ui32PHRMode;                              /*!< Periodic Hardware Reset configuration values */
+	IMG_UINT32         ui32HCSDeadlineMS;                        /*!< New number of milliseconds C/S is allowed to last */
+	IMG_UINT32         ui32WdgPeriodUs;                          /*!< The watchdog period in microseconds */
+	IMG_INT32          ai32DriverPriority[RGXFW_MAX_NUM_OSIDS];  /*!< Array of priorities per OS */
+	IMG_UINT32         aui32DriverIsolationGroup[RGXFW_MAX_NUM_OSIDS]; /*!< Array of isolation groups per OS */
+	IMG_UINT32         aui32TSPercentage[RGXFW_MAX_NUM_OSIDS];   /*!< Array of time slice per OS */
+	IMG_UINT32         ui32TSIntervalMs;                         /*!< Time slice interval */
+	IMG_UINT32         ui32VzConnectionCooldownPeriodInSec;      /*!< Vz Connection Cooldown period in secs */
+
+	PRGXFWIF_HWPERFBUF sHWPerfBuf;                               /*!< On-demand allocated HWPerf buffer address, to be passed to the FW */
+	RGXFWIF_DMA_ADDR   sHWPerfDMABuf;
+	RGXFWIF_DMA_ADDR   sHWPerfCtlDMABuf;
+} RGXFWIF_RUNTIME_CFG;
+
+/*!
+ *****************************************************************************
+ * Control data for RGX
+ *****************************************************************************/
+
+#define RGXFWIF_HWR_DEBUG_DUMP_ALL (99999U)
+
+#if defined(PDUMP)
+
+#define RGXFWIF_PID_FILTER_MAX_NUM_PIDS 32U
+
+typedef enum
+{
+	RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+	RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT
+} RGXFWIF_PID_FILTER_MODE;
+
+typedef struct
+{
+	IMG_PID uiPID;
+	IMG_UINT32 ui32DriverID;
+} RGXFW_ALIGN RGXFWIF_PID_FILTER_ITEM;
+
+typedef struct
+{
+	RGXFWIF_PID_FILTER_MODE eMode;
+	/* each process in the filter list is specified by a PID and OS ID pair.
+	 * each PID and OS pair is an item in the items array (asItems).
+	 * if the array contains less than RGXFWIF_PID_FILTER_MAX_NUM_PIDS entries
+	 * then it must be terminated by an item with pid of zero.
+	 */
+	RGXFWIF_PID_FILTER_ITEM asItems[RGXFWIF_PID_FILTER_MAX_NUM_PIDS];
+} RGXFW_ALIGN RGXFWIF_PID_FILTER;
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA  (0x1U << 0)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE  (0x1U << 1)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE  (0x1U << 2)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE     (0x1U << 3)
+#endif
+
+typedef enum
+{
+	RGXFWIF_USRM_DM_VDM = 0,
+	RGXFWIF_USRM_DM_DDM = 1,
+	RGXFWIF_USRM_DM_CDM = 2,
+	RGXFWIF_USRM_DM_PDM = 3,
+	RGXFWIF_USRM_DM_TDM = 4,
+	RGXFWIF_USRM_DM_LAST
+} RGXFWIF_USRM_DM;
+
+typedef enum
+{
+	RGXFWIF_UVBRM_DM_VDM = 0,
+	RGXFWIF_UVBRM_DM_DDM = 1,
+	RGXFWIF_UVBRM_DM_LAST
+} RGXFWIF_UVBRM_DM;
+
+typedef enum
+{
+	RGXFWIF_TPU_DM_PDM = 0,
+	RGXFWIF_TPU_DM_VDM = 1,
+	RGXFWIF_TPU_DM_CDM = 2,
+	RGXFWIF_TPU_DM_TDM = 3,
+	RGXFWIF_TPU_DM_RDM = 4,
+	RGXFWIF_TPU_DM_LAST
+} RGXFWIF_TPU_DM;
+
+typedef enum
+{
+	RGXFWIF_GPIO_VAL_OFF           = 0, /*!< No GPIO validation */
+	RGXFWIF_GPIO_VAL_GENERAL       = 1, /*!< Simple test case that
+	                                         initiates by sending data via the
+	                                         GPIO and then sends back any data
+	                                         received over the GPIO */
+	RGXFWIF_GPIO_VAL_AP            = 2, /*!< More complex test case that writes
+	                                         and reads data across the entire
+	                                         GPIO AP address range.*/
+#if defined(SUPPORT_STRIP_RENDERING)
+	RGXFWIF_GPIO_VAL_SR_BASIC      = 3, /*!< Strip Rendering AP based basic test.*/
+	RGXFWIF_GPIO_VAL_SR_COMPLEX    = 4, /*!< Strip Rendering AP based complex test.*/
+#endif
+	RGXFWIF_GPIO_VAL_TESTBENCH     = 5, /*!< Validates the GPIO Testbench. */
+	RGXFWIF_GPIO_VAL_LOOPBACK      = 6, /*!< Send and then receive each byte
+	                                         in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LOOPBACK_LITE = 7, /*!< Send and then receive each power-of-2
+	                                         byte in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LAST
+} RGXFWIF_GPIO_VAL_MODE;
+
+typedef IMG_UINT32 FW_PERF_CONF;
+#define FW_PERF_CONF_NONE			0U
+#define FW_PERF_CONF_ICACHE			1U
+#define FW_PERF_CONF_DCACHE			2U
+#define FW_PERF_CONF_JTLB_INSTR		5U
+#define FW_PERF_CONF_INSTRUCTIONS	6U
+
+typedef enum
+{
+	FW_BOOT_STAGE_TLB_INIT_FAILURE = -2,
+	FW_BOOT_STAGE_NOT_AVAILABLE = -1,
+	FW_BOOT_NOT_STARTED = 0,
+	FW_BOOT_BLDR_STARTED = 1,
+	FW_BOOT_CACHE_DONE,
+	FW_BOOT_TLB_DONE,
+	FW_BOOT_MAIN_STARTED,
+	FW_BOOT_ALIGNCHECKS_DONE,
+	FW_BOOT_INIT_DONE,
+} FW_BOOT_STAGE;
+
+/*!
+ * @AddToGroup KCCBTypes
+ * @{
+ * @Name Kernel CCB return slot responses
+ * @{
+ * Usage of bit-fields instead of bare integers
+ * allows FW to possibly pack-in several responses for each single kCCB command.
+ */
+
+#define RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED   (1U << 0) /*!< Command executed (return status from FW) */
+#define RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY   (1U << 1) /*!< A cleanup was requested but resource busy */
+#define RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE   (1U << 2) /*!< Poll failed in FW for a HW operation to complete */
+
+#define RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE            0x0U      /*!< Reset value of a kCCB return slot (set by host) */
+/*!
+ * @} End of Name Kernel CCB return slot responses
+ * @} End of AddToGroup KCCBTypes
+ */
+
+/*! @Brief OS connection data \ref RGXFWIF_CONNECTION_CTL allocated
+ * by services and used to track OS state in Firmware and Services
+ **/
+typedef struct
+{
+	/* Fw-Os connection states */
+	volatile RGXFWIF_CONNECTION_FW_STATE eConnectionFwState;    /*!< Firmware-OS connection state */
+	volatile RGXFWIF_CONNECTION_OS_STATE eConnectionOsState;    /*!< Services-OS connection state */
+	volatile IMG_UINT32                  ui32AliveFwToken;      /*!< OS Alive token updated by Firmware */
+	volatile IMG_UINT32                  ui32AliveOsToken;      /*!< OS Alive token updated by Services */
+} UNCACHED_ALIGN RGXFWIF_CONNECTION_CTL;
+
+/*! @Brief Firmware OS Initialization data \ref RGXFWIF_OSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	/* Kernel CCB */
+	PRGXFWIF_CCB_CTL        psKernelCCBCtl; /*!< Kernel CCB Control */
+	PRGXFWIF_CCB            psKernelCCB; /*!<  Kernel CCB */
+	PRGXFWIF_CCB_RTN_SLOTS  psKernelCCBRtnSlots; /*!<  Kernel CCB return slots */
+
+	/* Firmware CCB */
+	PRGXFWIF_CCB_CTL        psFirmwareCCBCtl; /*!<  Firmware CCB control */
+	PRGXFWIF_CCB            psFirmwareCCB; /*!<  Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	PRGXFWIF_CCB_CTL        psWorkEstFirmwareCCBCtl; /*!<  Workload estimation control */
+	PRGXFWIF_CCB            psWorkEstFirmwareCCB; /*!<  Workload estimation buffer */
+
+	PRGXFWIF_HWRINFOBUF     sRGXFWIfHWRInfoBufCtl; /*!<  HWRecoveryInfo control */
+
+	IMG_UINT32              ui32HWRDebugDumpLimit; /*!< Firmware debug dump maximum limit */
+
+	PRGXFWIF_OSDATA         sFwOsData; /*!<  Firmware per-os shared data */
+
+	RGXFWIF_COMPCHECKS      sRGXCompChecks; /*!< Compatibility checks to be populated by the Firmware */
+	RGXFWIF_MULTICORE_INFO  sRGXMulticoreInfo; /*! < Multicore capability info */
+
+} UNCACHED_ALIGN RGXFWIF_OSINIT;
+
+/*! @Brief Firmware System Initialization data \ref RGXFWIF_SYSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	IMG_DEV_PHYADDR         RGXFW_ALIGN sFaultPhysAddr; /*!< Fault read address */
+
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sPDSExecBase; /*!< PDS execution base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sUSCExecBase; /*!< USC execution base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sFBCDCStateTableBase; /*!< FBCDC bindless texture state table base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sFBCDCLargeStateTableBase;
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sTextureHeapBase; /*!< Texture state base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sPDSIndirectHeapBase; /* Pixel Indirect State base */
+
+	IMG_UINT64              RGXFW_ALIGN ui64HWPerfFilter; /*! Event filter for Firmware events */
+
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sSLC3FenceDevVAddr; /*!< Address to use as a fence when issuing SLC3_CFI */
+
+	IMG_UINT64				RGXFW_ALIGN aui64UVBRMNumRegions[RGXFWIF_UVBRM_DM_LAST];
+	IMG_UINT32              RGXFW_ALIGN aui32TPUTrilinearFracMask[RGXFWIF_TPU_DM_LAST];
+	IMG_UINT32				RGXFW_ALIGN aui32USRMNumRegions[RGXFWIF_USRM_DM_LAST];
+
+	IMG_UINT64				ui64ClkCtrl0;
+	IMG_UINT64				ui64ClkCtrl1;
+	IMG_UINT32				ui32ClkCtrl2;
+
+	IMG_UINT32              ui32FilterFlags;
+
+#if defined(PDUMP)
+	RGXFWIF_SIGBUF_CTL      asSigBufCtl[RGXFWIF_DM_MAX]; /*!< Signature and Checksum Buffers for DMs */
+#endif
+
+	PRGXFWIF_RUNTIME_CFG    sRuntimeCfg; /*!<  Firmware Runtime configuration */
+
+	PRGXFWIF_TRACEBUF       sTraceBufCtl; /*!<  Firmware Trace buffer control */
+	PRGXFWIF_SYSDATA        sFwSysData; /*!< Firmware System shared data */
+#if defined(SUPPORT_TBI_INTERFACE)
+	PRGXFWIF_TBIBUF         sTBIBuf; /*!< Tbi log buffer */
+#endif
+
+	PRGXFWIF_GPU_UTIL_FW    sGpuUtilFWCtl; /*!< GPU utilization buffer */
+	PRGXFWIF_REG_CFG        sRegCfg; /*!< Firmware register user configuration */
+	PRGXFWIF_HWPERF_CTL     sHWPerfCtl; /*!< HWPerf counter block configuration.*/
+
+	RGXFWIF_COUNTER_DUMP_CTL sCounterDumpCtl;
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	RGXFWIF_FIRMWARE_GCOV_CTL sFirmwareGcovCtl; /*!< Firmware gcov buffer control */
+#endif
+
+	RGXFWIF_DEV_VIRTADDR    sAlignChecks; /*!< Array holding Server structures alignment data */
+
+	IMG_UINT32              ui32InitialCoreClockSpeed; /*!< Core clock speed at FW boot time */
+
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32              ui32InitialSOCClockSpeed; /*!< System/SOC clock speed at FW boot time */
+#endif
+
+	IMG_UINT32              ui32InitialActivePMLatencyms; /*!< APM latency in ms before signalling IDLE to the host */
+
+	IMG_BOOL                bFirmwareStarted; /*!< Flag to be set by the Firmware after successful start */
+
+	IMG_UINT32              ui32MarkerVal; /*!< Host/FW Trace synchronisation Partition Marker */
+
+	IMG_UINT32              ui32FirmwareStartedTimeStamp; /*!< Firmware initialization complete time */
+
+	RGXFWIF_DMA_ADDR        sCorememDataStore; /*!< Firmware coremem data */
+
+	FW_PERF_CONF            eFirmwarePerf; /*!< Firmware performance counter config */
+
+#if defined(SUPPORT_FW_OPP_TABLE)
+	RGXFWIF_OPP_INFO        RGXFW_ALIGN sOPPInfo;
+
+	/**
+	 * FW Pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory when running on non primary FW thread
+	 * to communicate to host driver.
+	 */
+	PRGXFWIF_CORE_CLK_RATE  RGXFW_ALIGN sCoreClockRate;
+#endif
+
+#if defined(PDUMP)
+	RGXFWIF_PID_FILTER      sPIDFilter;
+#endif
+
+	RGXFWIF_GPIO_VAL_MODE   eGPIOValidationMode;
+
+	RGX_HWPERF_BVNC         sBvncKmFeatureFlags; /*!< Used in HWPerf for decoding BVNC Features*/
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32              ui32SecurityTestFlags;
+	RGXFWIF_DEV_VIRTADDR    pbSecureBuffer;
+	RGXFWIF_DEV_VIRTADDR    pbNonSecureBuffer;
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	RGXFWIF_DEV_VIRTADDR    sActiveContextBufBase; /*!< Active context buffer base */
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	/*
+	 * Used when validation is enabled to allow the host to check
+	 * that MTS sent the correct sideband in response to a kick
+	 * from a given OSes schedule register.
+	 * Testing is enabled if RGXFWIF_KICK_TEST_ENABLED_BIT is set
+	 *
+	 * Set by the host to:
+	 * (osid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT
+	 * reset to 0 by FW when kicked by the given OSid
+	 */
+	IMG_UINT32              ui32OSKickTest;
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_UINT32              ui32VzWdgPeriod;
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	/* notify firmware power-up on host-side recovery */
+	IMG_BOOL                bFwHostRecoveryMode;
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	RGXFWIF_DEV_VIRTADDR    pbFwScratchBuf;
+#endif
+#if defined(SUPPORT_ICS)
+	IMG_UINT32 ui32FDTI;
+	IMG_UINT32 ui32ICSThreshold;
+	IMG_BOOL   bTestModeOn;
+#endif
+} UNCACHED_ALIGN RGXFWIF_SYSINIT;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+#define RGXFWIF_KICK_TEST_ENABLED_BIT  0x1
+#define RGXFWIF_KICK_TEST_OSID_SHIFT   0x1
+#endif
+
+/*!
+ *****************************************************************************
+ * Timer correlation shared data and defines
+ *****************************************************************************/
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64OSMonoTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeStamp;
+
+	/* Utility variable used to convert CR timer deltas to OS timer deltas (nS),
+	 * where the deltas are relative to the timestamps above:
+	 * deltaOS = (deltaCR * K) >> decimal_shift, see full explanation below */
+	IMG_UINT64 RGXFW_ALIGN ui64CRDeltaToOSDeltaKNs;
+
+	IMG_UINT32             ui32CoreClockSpeed;
+	IMG_UINT32             ui32Reserved;
+} UNCACHED_ALIGN RGXFWIF_TIME_CORR;
+
+
+/* The following macros are used to help converting FW timestamps to the Host
+ * time domain. On the FW the RGX_CR_TIMER counter is used to keep track of
+ * time; it increments by 1 every 256 GPU clock ticks, so the general
+ * formula to perform the conversion is:
+ *
+ * [ GPU clock speed in Hz, if (scale == 10^9) then deltaOS is in nS,
+ *   otherwise if (scale == 10^6) then deltaOS is in uS ]
+ *
+ *             deltaCR * 256                                   256 * scale
+ *  deltaOS = --------------- * scale = deltaCR * K    [ K = --------------- ]
+ *             GPUclockspeed                                  GPUclockspeed
+ *
+ * The actual K is multiplied by 2^20 (and deltaCR * K is divided by 2^20)
+ * to get some better accuracy and to avoid returning 0 in the integer
+ * division 256000000/GPUfreq if GPUfreq is greater than 256MHz.
+ * This is the same as keeping K as a decimal number.
+ *
+ * The maximum deltaOS is slightly more than 5hrs for all GPU frequencies
+ * (deltaCR * K is more or less a constant), and it's relative to the base
+ * OS timestamp sampled as a part of the timer correlation data.
+ * This base is refreshed on GPU power-on, DVFS transition and periodic
+ * frequency calibration (executed every few seconds if the FW is doing
+ * some work), so as long as the GPU is doing something and one of these
+ * events is triggered then deltaCR * K will not overflow and deltaOS will be
+ * correct.
+ */
+
+#define RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT  (20)
+
+#define RGXFWIF_GET_DELTA_OSTIME_NS(deltaCR, K) \
+	(((deltaCR) * (K)) >> RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT)
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation
+ *****************************************************************************/
+
+/* See rgx_common.h for a list of GPU states */
+#define RGXFWIF_GPU_UTIL_TIME_MASK       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_TIME_MASK32     (IMG_UINT32_C(0xFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+#define RGXFWIF_GPU_UTIL_GET_TIME(word)    ((word) & RGXFWIF_GPU_UTIL_TIME_MASK)
+#define RGXFWIF_GPU_UTIL_GET_STATE(word)   ((word) & RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_GET_TIME32(word)  ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_TIME_MASK32)
+#define RGXFWIF_GPU_UTIL_GET_STATE32(word) ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+/* The OS timestamps computed by the FW are approximations of the real time,
+ * which means they could be slightly behind or ahead the real timer on the Host.
+ * In some cases we can perform subtractions between FW approximated
+ * timestamps and real OS timestamps, so we need a form of protection against
+ * negative results if for instance the FW one is a bit ahead of time.
+ */
+#define RGXFWIF_GPU_UTIL_GET_PERIOD(newtime,oldtime) \
+	(((newtime) > (oldtime)) ? ((newtime) - (oldtime)) : 0U)
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME(time) | RGXFWIF_GPU_UTIL_GET_STATE(state))
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD32(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME32(time) | RGXFWIF_GPU_UTIL_GET_STATE32(state))
+
+
+/* The timer correlation array must be big enough to ensure old entries won't be
+ * overwritten before all the HWPerf events linked to those entries are processed
+ * by the MISR. The update frequency of this array depends on how fast the system
+ * can change state (basically how small the APM latency is) and perform DVFS transitions.
+ *
+ * The minimum size is 2 (not 1) to avoid race conditions between the FW reading
+ * an entry while the Host is updating it. With 2 entries in the worst case the FW
+ * will read old data, which is still quite ok if the Host is updating the timer
+ * correlation at that time.
+ */
+#define RGXFWIF_TIME_CORR_ARRAY_SIZE            256U
+#define RGXFWIF_TIME_CORR_CURR_INDEX(seqcount)  ((seqcount) % RGXFWIF_TIME_CORR_ARRAY_SIZE)
+
+/* Make sure the timer correlation array size is a power of 2 */
+static_assert((RGXFWIF_TIME_CORR_ARRAY_SIZE & (RGXFWIF_TIME_CORR_ARRAY_SIZE - 1U)) == 0U,
+			  "RGXFWIF_TIME_CORR_ARRAY_SIZE must be a power of two");
+
+/* The time is stored in DM state counters in "approximately microseconds",
+ * dividing the time originally obtained in nanoseconds by 2^10 for the sake of reducing coremem usage */
+#define RGXFWIF_DM_OS_TIMESTAMP_SHIFT    10U
+
+typedef struct
+{
+	/* Last GPU DM per-OS states + OS time of the last state update */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWord[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* DMs time-stamps are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWordWrap[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* Counters for the amount of time the GPU DMs were active or inactive(idle or blocked) */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSStatsCounters[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	/* DMs Counters are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSCountersWrap[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+} RGXFWIF_GPU_STATS;
+
+typedef struct
+{
+	RGXFWIF_TIME_CORR      sTimeCorr[RGXFWIF_TIME_CORR_ARRAY_SIZE];
+	IMG_UINT32             ui32TimeCorrSeqCount;
+
+	/* Compatibility and other flags */
+	IMG_UINT32             ui32GpuUtilFlags;
+
+	/* Last GPU state + OS time of the last state update */
+	IMG_UINT64 RGXFW_ALIGN ui64GpuLastWord;
+	/* Counters for the amount of time the GPU was active/idle/blocked */
+	IMG_UINT64 RGXFW_ALIGN aui64GpuStatsCounters[RGXFWIF_GPU_UTIL_STATE_NUM];
+	/* Device off period timestamp offset */
+	IMG_INT64 RGXFW_ALIGN i64DeviceTimestampOffset;
+	/* Stats per OSID/DriverID */
+	RGXFWIF_GPU_STATS sStats[RGXFW_MAX_NUM_OSIDS];
+} UNCACHED_ALIGN RGXFWIF_GPU_UTIL_FW;
+
+/*!
+ ******************************************************************************
+ * Virtualisation and Security
+ *****************************************************************************/
+
+#define RGX_GP_SUPPORTS_SECURITY		(0U)
+#define RGX_TDM_SUPPORTS_SECURITY		(1U)
+#define RGX_GEOM_SUPPORTS_SECURITY		(0U)
+#define RGX_FRAG_SUPPORTS_SECURITY		(1U)
+#define RGX_COMPUTE_SUPPORTS_SECURITY	(1U)
+#define RGX_RAY_SUPPORTS_SECURITY		(0U)
+
+#define RGX_DMS_WITH_SECURITY_COUNT		(RGX_GP_SUPPORTS_SECURITY		+ \
+										 RGX_TDM_SUPPORTS_SECURITY		+ \
+										 RGX_GEOM_SUPPORTS_SECURITY		+ \
+										 RGX_FRAG_SUPPORTS_SECURITY		+ \
+										 RGX_COMPUTE_SUPPORTS_SECURITY	+ \
+										 RGX_RAY_SUPPORTS_SECURITY)
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#if (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_DEDICATED_OSID) || (defined(RGX_LEGACY_SECURE_OSID_SCHEME))
+/* OSIDs 0 and 1 reserved for Firmware */
+#define DRIVER_OSID_START_OFFSET		(FW_OSID+2)
+#else
+/* OSIDs 1 reserved for Firmware */
+#define DRIVER_OSID_START_OFFSET		(FW_OSID+1)
+#endif
+
+#else
+/* Firmware and Host driver share the same OSID */
+#define DRIVER_OSID_START_OFFSET		(FW_OSID)
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+#if (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_DEDICATED_OSID)
+/* Firmware heaps reside in a dedicated non-secure IPA space. */
+#define FW_HEAP_OSID					(FW_OSID+1)
+#elif (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_HOST_OSID)
+/* Firmware heaps reside in the Host driver's non-secure IPA space. */
+#define FW_HEAP_OSID					(DRIVER_OSID_START_OFFSET)
+#elif (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID)
+/* Firmware heaps reside in the IPA space as the Firmware. */
+#define FW_HEAP_OSID					(FW_OSID)
+#else
+#error "RGX_FW_HEAP_OSID_ASSIGNMENT not configured correctly."
+#endif
+
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) && !defined(__KERNEL__)
+/* placeholder define for UM tools including this header */
+#define RGX_NUM_DRIVERS_SUPPORTED (1U)
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+
+#if defined(RGX_LEGACY_SECURE_OSID_SCHEME)
+/* keep reverse compatibility with original scheme */
+static_assert((RGX_NUM_DRIVERS_SUPPORTED == 1),
+			  "RGX_LEGACY_SECURE_OSID_SCHEME only applicable on native drivers (no VZ support)");
+
+#define DRIVER_ID(osid)					(0U)
+#define OSID_SECURE(did)				(1U)
+#define OSID(did)						(2U)
+
+#else /* defined(RGX_LEGACY_SECURE_OSID_SCHEME) */
+/* native and virtualized security support */
+#define LAST_OSID						(RGXFW_MAX_NUM_OSIDS - 1)
+#define DRIVER_ID(osid)					(osid - DRIVER_OSID_START_OFFSET)
+#define OSID(did)						(did + DRIVER_OSID_START_OFFSET)
+#define OSID_SECURE(did) \
+			(did==0 ? (DRIVER0_SECURITY_SUPPORT ? (LAST_OSID) : OSID(did)) : \
+			(did==1 ? (DRIVER1_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==2 ? (DRIVER2_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==3 ? (DRIVER3_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==4 ? (DRIVER4_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==5 ? (DRIVER5_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT - DRIVER4_SECURITY_SUPPORT) : OSID(did)) : \
+			(did==6 ? (DRIVER6_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT - DRIVER4_SECURITY_SUPPORT - DRIVER5_SECURITY_SUPPORT) : OSID(did)) : \
+			          (DRIVER7_SECURITY_SUPPORT ? (LAST_OSID - DRIVER0_SECURITY_SUPPORT - DRIVER1_SECURITY_SUPPORT - DRIVER2_SECURITY_SUPPORT - DRIVER3_SECURITY_SUPPORT - DRIVER4_SECURITY_SUPPORT - DRIVER5_SECURITY_SUPPORT - DRIVER6_SECURITY_SUPPORT) : OSID(did)))))))))
+
+static_assert((RGX_NUM_DRIVERS_SUPPORTED + DRIVER_OSID_START_OFFSET +
+				DRIVER0_SECURITY_SUPPORT + DRIVER1_SECURITY_SUPPORT + DRIVER2_SECURITY_SUPPORT + DRIVER3_SECURITY_SUPPORT +
+				DRIVER4_SECURITY_SUPPORT + DRIVER5_SECURITY_SUPPORT + DRIVER6_SECURITY_SUPPORT + DRIVER7_SECURITY_SUPPORT) <= RGXFW_MAX_NUM_OSIDS,
+				"The GPU hardware is not equipped with enough hardware OSIDs to satisfy the requirements.");
+#endif /* defined(RGX_LEGACY_SECURE_OSID_SCHEME) */
+
+/* Time slice support */
+/* Bits 30 and 31 reserved by FW private driver priority */
+#define RGXFW_VZ_PRIORITY_MAX_SHIFT		(30U)
+#define RGXFW_VZ_PRIORITY_MASK			((1U << RGXFW_VZ_PRIORITY_MAX_SHIFT) - 1U)
+#define RGXFW_VZ_TIME_SLICE_MAX			(100U)
+#define RGXFW_VZ_TIME_SLICE_MIN			(5U)
+
+#elif defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* virtualization without security support */
+#define DRIVER_ID(osid)					(osid - DRIVER_OSID_START_OFFSET)
+#define OSID(did)						(did + DRIVER_OSID_START_OFFSET)
+
+/* Time slice support */
+/* Bits 30 and 31 reserved by FW private driver priority */
+#define RGXFW_VZ_PRIORITY_MAX_SHIFT		(30U)
+#define RGXFW_VZ_PRIORITY_MASK			((1U << RGXFW_VZ_PRIORITY_MAX_SHIFT) - 1U)
+#define RGXFW_VZ_TIME_SLICE_MAX			(100U)
+#define RGXFW_VZ_TIME_SLICE_MIN			(5U)
+#else
+/* native without security support */
+#define DRIVER_ID(osid)					(0U)
+#define OSID(did)						(did)
+#define RGXFW_VZ_PRIORITY_MASK			(0U)
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) < RGX_NUM_DRIVERS_SUPPORTED; (did)++)
+
+#if defined(__KERNEL__)
+/* Driver implementation */
+#define FOREACH_ACTIVE_DRIVER(devinfo, did)        FOREACH_SUPPORTED_DRIVER(did)                                \
+                                                   {                                                            \
+                                                   if (devinfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[did].bfOsState != RGXFW_CONNECTION_FW_ACTIVE) continue;
+
+#define END_FOREACH_ACTIVE_DRIVER                  }
+
+#else
+/* Firmware implementation */
+#define FOREACH_ACTIVE_DRIVER(did)                 do {                                                                      \
+                                                   unsigned int idx;                                                         \
+                                                   for (idx = 0; idx < RGXFW_NUM_ACTIVE_DRIVERS; idx++)                      \
+                                                   {                                                                         \
+                                                       (did) = gsRGXFWCtl.aui32ActiveDrivers[idx];                           \
+                                                       {
+
+#define END_FOREACH_ACTIVE_DRIVER                  }}} while (false);
+#endif /* defined(__KERNEL__) */
+
+
+#else
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) <= RGXFW_HOST_DRIVER_ID; (did)++)
+
+#define FOREACH_ACTIVE_DRIVER(did)                 FOREACH_SUPPORTED_DRIVER(did)
+#define END_FOREACH_ACTIVE_DRIVER
+
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#define FOREACH_VALIDATION_OSID(osid)              for ((osid)=0; (osid) < GPUVIRT_VALIDATION_NUM_OS; (osid)++)
+#define FOREACH_HW_OSID(osid)                      for ((osid)=0; (osid) < RGXFW_MAX_NUM_OSIDS; (osid)++)
+#define FOREACH_DRIVER_RAW_HEAP(did, _struct, dev) for ((did)=RGX_FIRST_RAW_HEAP_DRIVER_ID; (did) < ((PVRSRV_VZ_MODE_IS(NATIVE, _struct, dev) ? 1 : RGX_NUM_DRIVERS_SUPPORTED)); (did)++)
+
+#define RGXFWIF_TDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES)
+#define RGXFWIF_CDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES)
+#define RGXFWIF_3D_SECURE_IPP_BUF_VADDR                            (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_OFFSET_BYTES)
+#define RGXFWIF_CDM_SECURE_SR_BUF_VADDR                            PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_OFFSET_BYTES), 1024ULL)
+#define RGXFWIF_CDM_SECURE_SR_B_BUF_VADDR                          PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_OFFSET_BYTES), 1024ULL)
+#define RGXFWIF_CDM_SECURE_CONTEXT_STATE_VADDR                     PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_OFFSET_BYTES), 128ULL)
+#define RGXFWIF_TDM_SECURE_CONTEXT_STATE_VADDR                     PVR_ALIGN((RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_OFFSET_BYTES), 128ULL)
+
+#define SAFETY_SELF_TEST_STEP_CLEAR     1
+#define SAFETY_SELF_TEST_STEP_ENABLE    2
+#define SAFETY_SELF_TEST_STEP_TEST      3
+#define SAFETY_SELF_TEST_STEP_DISABLE   4
+
+#endif /* RGX_FWIF_KM_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_shared.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_shared.h
new file mode 100644
index 000000000000..061f718cec62
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_fwif_shared.h
@@ -0,0 +1,1159 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures shared by both host client
+                and host server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_SHARED_H)
+#define RGX_FWIF_SHARED_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+#include "devicemem_typedefs.h"
+
+/* Indicates the number of RTDATAs per RTDATASET */
+#if defined(SUPPORT_AGP)
+#if defined(SUPPORT_AGP4)
+#define RGXMKIF_NUM_RTDATAS           4U
+#define RGXMKIF_NUM_GEOMDATAS         4U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  20U /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (4U)
+#else
+#define RGXMKIF_NUM_RTDATAS           4U
+#define RGXMKIF_NUM_GEOMDATAS         4U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  12U /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (2U)
+#endif
+#else
+#define RGXMKIF_NUM_RTDATAS           2U
+#define RGXMKIF_NUM_GEOMDATAS         1U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  2U  /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (1U)
+#endif
+
+/* Maximum number of UFOs in a CCB command.
+ * The number is based on having 32 sync prims (as originally), plus 32 sync
+ * checkpoints.
+ * Once the use of sync prims is no longer supported, we will retain
+ * the same total (64) as the number of sync checkpoints which may be
+ * supporting a fence is not visible to the client driver and has to
+ * allow for the number of different timelines involved in fence merges.
+ */
+#define RGXFWIF_CCB_CMD_MAX_UFOS			(32U+32U)
+
+/*
+ * This is a generic limit imposed on any DM (TA,3D,CDM,TDM,2D,TRANSFER)
+ * command passed through the bridge.
+ * Just across the bridge in the server, any incoming kick command size is
+ * checked against this maximum limit.
+ * In case the incoming command size is larger than the specified limit,
+ * the bridge call is retired with error.
+ */
+#define RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE	(1024U)
+
+typedef struct
+{
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN psDevVirtAddr;
+	RGXFWIF_DEV_VIRTADDR    pbyFWAddr;
+} UNCACHED_ALIGN RGXFWIF_DMA_ADDR;
+
+typedef IMG_UINT8	RGXFWIF_CCCB;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_UFO_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CLEANUP_CTL;
+
+
+/*!
+ * @InGroup ClientCCBTypes
+ * @Brief Command data for fence & update types Client CCB commands.
+ */
+typedef struct
+{
+	PRGXFWIF_UFO_ADDR	puiAddrUFO; /*!< Address to be checked/updated */
+	IMG_UINT32			ui32Value;  /*!< Value to check-against/update-to */
+} RGXFWIF_UFO;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Track pending and completed workloads of HWRTDATA and ZSBUFFER
+ */
+typedef struct
+{
+	IMG_UINT32			ui32SubmittedCommands;	/*!< Number of commands received by the FW */
+	IMG_UINT32			ui32ExecutedCommands;	/*!< Number of commands executed by the FW */
+} UNCACHED_ALIGN RGXFWIF_CLEANUP_CTL;
+
+#define	RGXFWIF_PRBUFFER_START        IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_ZSBUFFER     IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_MSAABUFFER   IMG_UINT32_C(1)
+#define	RGXFWIF_PRBUFFER_MAXSUPPORTED IMG_UINT32_C(2)
+
+typedef IMG_UINT32 RGXFWIF_PRBUFFER_TYPE;
+
+typedef enum
+{
+	RGXFWIF_PRBUFFER_UNBACKED = 0,
+	RGXFWIF_PRBUFFER_BACKED,
+	RGXFWIF_PRBUFFER_BACKING_PENDING,
+	RGXFWIF_PRBUFFER_UNBACKING_PENDING,
+}RGXFWIF_PRBUFFER_STATE;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief OnDemand Z/S/MSAA Buffers
+ */
+typedef struct
+{
+	IMG_UINT32		ui32BufferID;		/*!< Buffer ID*/
+	IMG_BOOL		bOnDemand;		/*!< Needs On-demand Z/S/MSAA Buffer allocation */
+	RGXFWIF_PRBUFFER_STATE	eState;			/*!< Z/S/MSAA -Buffer state */
+	RGXFWIF_CLEANUP_CTL	sCleanupState;		/*!< Cleanup state */
+	IMG_UINT32		ui32PRBufferFlags;	/*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_PRBUFFER;
+
+/*
+ * Used to share frame numbers across UM-KM-FW,
+ * frame number is set in UM,
+ * frame number is required in both KM for HTB and FW for FW trace.
+ *
+ * May be used to house Kick flags in the future.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32FrameNum; /*!< associated frame number */
+} CMD_COMMON;
+
+/*
+ * TA and 3D commands require set of firmware addresses that are stored in the
+ * Kernel. Client has handle(s) to Kernel containers storing these addresses,
+ * instead of raw addresses. We have to patch/write these addresses in KM to
+ * prevent UM from controlling FW addresses directly.
+ * Typedefs for TA and 3D commands are shared between Client and Firmware (both
+ * single-BVNC). Kernel is implemented in a multi-BVNC manner, so it can't use
+ * TA|3D CMD type definitions directly. Therefore we have a SHARED block that
+ * is shared between UM-KM-FW across all BVNC configurations.
+ */
+typedef struct
+{
+	CMD_COMMON           sCmn;      /*!< Common command attributes */
+	RGXFWIF_DEV_VIRTADDR sHWRTData; /* RTData associated with this command,
+									   this is used for context selection and for storing out HW-context,
+									   when TA is switched out for continuing later */
+
+	RGXFWIF_DEV_VIRTADDR asPRBuffer[RGXFWIF_PRBUFFER_MAXSUPPORTED];	/* Supported PR Buffers like Z/S/MSAA Scratch */
+
+} CMDTA3D_SHARED;
+
+/*!
+ * Client Circular Command Buffer (CCCB) control structure.
+ * This is shared between the Server and the Firmware and holds byte offsets
+ * into the CCCB as well as the wrapping mask to aid wrap around. A given
+ * snapshot of this queue with Cmd 1 running on the GPU might be:
+ *
+ *          Roff                           Doff                 Woff
+ * [..........|-1----------|=2===|=3===|=4===|~5~~~~|~6~~~~|~7~~~~|..........]
+ *            <      runnable commands       ><   !ready to run   >
+ *
+ * Cmd 1    : Currently executing on the GPU data master.
+ * Cmd 2,3,4: Fence dependencies met, commands runnable.
+ * Cmd 5... : Fence dependency not met yet.
+ */
+typedef struct
+{
+	IMG_UINT32  ui32WriteOffset;    /*!< Host write offset into CCB. This
+	                                 *    must be aligned to 16 bytes. */
+	IMG_UINT32  ui32ReadOffset;     /*!< Firmware read offset into CCB.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R!=W */
+	IMG_UINT32  ui32DepOffset;      /*!< Firmware fence dependency offset.
+	                                      Points to commands not ready, i.e.
+	                                      fence dependencies are not met. */
+	IMG_UINT32  ui32WrapMask;       /*!< Offset wrapping mask, total capacity
+	                                      in bytes of the CCB-1 */
+
+	IMG_UINT32  ui32ReadOffset2;     /*!< Firmware 2nd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R2!=W */
+	IMG_UINT32  ui32ReadOffset3;     /*!< Firmware 3rd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R3!=W */
+	IMG_UINT32  ui32ReadOffset4;     /*!< Firmware 4th read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R4!=W */
+
+} UNCACHED_ALIGN RGXFWIF_CCCB_CTL;
+
+
+typedef IMG_UINT32 RGXFW_FREELIST_TYPE;
+
+#define RGXFW_LOCAL_FREELIST     IMG_UINT32_C(0)
+#define RGXFW_GLOBAL_FREELIST    IMG_UINT32_C(1)
+#if defined(SUPPORT_AGP4)
+#define RGXFW_GLOBAL2_FREELIST   IMG_UINT32_C(2)
+#define RGXFW_GLOBAL3_FREELIST   IMG_UINT32_C(3)
+#define RGXFW_GLOBAL4_FREELIST   IMG_UINT32_C(4)
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL4_FREELIST + 1U)
+#elif defined(SUPPORT_AGP)
+#define RGXFW_GLOBAL2_FREELIST   IMG_UINT32_C(2)
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL2_FREELIST + 1U)
+#else
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL_FREELIST + 1U)
+#endif
+#define RGXFW_MAX_HWFREELISTS    (2U)
+
+/*!
+ * @Defgroup ContextSwitching Context switching data interface
+ * @Brief Types grouping data structures and defines used in realising the Context Switching (CSW) functionality
+ * @{
+ */
+
+/*!
+ * @Brief GEOM DM or TA register controls for context switch
+ */
+typedef struct
+{
+	IMG_UINT64	uTAReg_DCE_CONTEXT_STATE_BASE_ADDR;
+	IMG_UINT64	uTAReg_TA_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the TA's context state buffer */
+
+	struct
+	{
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_VDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_VDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_VDM2;
+
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_DDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_DDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_DDM2;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_STORE_TASK_XFB;
+
+		/* VDM resume state update controls */
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_VDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_VDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_VDM2;
+
+
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_DDM0;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_DDM1;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_DDM2;
+		IMG_UINT64	uTAReg_DCE_CONTEXT_RESUME_TASK_XFB;
+	} asTAState[2];
+
+} RGXFWIF_TAREGISTERS_CSWITCH;
+/*! @} End of Defgroup ContextSwitching */
+
+typedef struct
+{
+	IMG_UINT64	u3DReg_IPP_CONTEXT_ADDR;
+} RGXFWIF_3DREGISTERS_CSWITCH;
+
+typedef struct
+{
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS1;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS1;
+
+	/* CDM resume controls */
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0_B;
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0_B;
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_SECURE_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_SECURE_PDS0_B;
+
+	IMG_UINT64	uCDMReg_CDM_RESUME_SECURE_PDS0;
+	IMG_UINT64	uCDMReg_CDM_RESUME_SECURE_PDS0_B;
+
+	IMG_DEV_VIRTADDR	uCDMReg_CDM_CONTEXT_sSRBuffer;
+	IMG_DEV_VIRTADDR	uCDMReg_CDM_CONTEXT_sSRBuffer_B;
+
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_STATE_BASE_ADDR;
+#endif
+} RGXFWIF_CDM_REGISTERS_CSWITCH;
+
+static_assert((sizeof(RGXFWIF_CDM_REGISTERS_CSWITCH) % 8U) == 0U,
+			"the size of the structure must be multiple of 8");
+
+#define RGXFWIF_CDM_REGISTERS_CSWITCH_SIZE sizeof(RGXFWIF_CDM_REGISTERS_CSWITCH)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Render context static register controls for context switch
+ */
+typedef struct
+{
+	RGXFWIF_TAREGISTERS_CSWITCH	RGXFW_ALIGN asCtxSwitch_GeomRegs[RGX_NUM_GEOM_CORES];
+	RGXFWIF_3DREGISTERS_CSWITCH RGXFW_ALIGN sCtxSwitch_3DRegs;  /*!< 3D registers for ctx switch */
+} RGXFWIF_STATIC_RENDERCONTEXT_STATE;
+
+#define RGXFWIF_STATIC_RENDERCONTEXT_SIZE sizeof(RGXFWIF_STATIC_RENDERCONTEXT_STATE)
+
+typedef struct
+{
+	RGXFWIF_CDM_REGISTERS_CSWITCH	RGXFW_ALIGN sCtxSwitch_Regs;	/*!< CDM registers for ctx switch */
+} RGXFWIF_STATIC_COMPUTECONTEXT_STATE;
+
+#define RGXFWIF_STATIC_COMPUTECONTEXT_SIZE sizeof(RGXFWIF_STATIC_COMPUTECONTEXT_STATE)
+
+typedef struct
+{
+	IMG_UINT64	uRDMReg_RDM_CONTEXT_STATE_BASE_ADDR;
+} RGXFWIF_RDM_REGISTERS_CSWITCH;
+
+static_assert((sizeof(RGXFWIF_RDM_REGISTERS_CSWITCH) % 8U) == 0U,
+			"the size of the structure must be multiple of 8");
+
+#define RGXFWIF_RDM_REGISTERS_CSWITCH_SIZE sizeof(RGXFWIF_RDM_REGISTERS_CSWITCH)
+
+typedef struct
+{
+	RGXFWIF_RDM_REGISTERS_CSWITCH	RGXFW_ALIGN sCtxSwitch_Regs;	/*!< RDM registers for ctx switch */
+} RGXFWIF_STATIC_RAYCONTEXT_STATE;
+
+#define RGXFWIF_STATIC_RAYCONTEXT_SIZE sizeof(RGXFWIF_STATIC_RAYCONTEXT_STATE)
+
+/*!
+	@Brief Context reset reason. Last reset reason for a reset context.
+*/
+typedef enum
+{
+	RGX_CONTEXT_RESET_REASON_NONE                = 0,	/*!< No reset reason recorded */
+	RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP       = 1,	/*!< Caused a reset due to locking up */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP     = 2,	/*!< Affected by another context locking up */
+	RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING   = 3,	/*!< Overran the global deadline */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING = 4,	/*!< Affected by another context overrunning */
+	RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH = 5,	/*!< Forced reset to ensure scheduling requirements */
+	RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM        = 6,	/*!< CDM Mission/safety checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM        = 7,	/*!< TRP checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_OK          = 8,	/*!< GPU ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR         = 9,	/*!< GPU ECC error (uncorrected, HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_OK           = 10,	/*!< FW ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_ERR          = 11,	/*!< FW ECC error (uncorrected, ERR) */
+	RGX_CONTEXT_RESET_REASON_FW_WATCHDOG         = 12,	/*!< FW Safety watchdog triggered */
+	RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT        = 13,	/*!< FW page fault (no HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR         = 14,	/*!< FW execution error (GPU reset requested) */
+	RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR     = 15,	/*!< Host watchdog detected FW error */
+	RGX_CONTEXT_GEOM_OOM_DISABLED                = 16,	/*!< Geometry DM OOM event is not allowed */
+	RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH         = 17,	/*!< PVRIC Signature mismatch */
+	RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR   = 18,	/*!< Parity error in MMU Page Table Entry */
+	RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR       = 19,	/*!< Parity error in MH, system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR      = 20,	/*!< Parity error in system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR      = 21,	/*!< Latent/ICS signature mismatch error */
+	RGX_CONTEXT_RESET_REASON_DCLS_ERR            = 22,	/*!< Dual Core Lock Step FW error detected */
+	RGX_CONTEXT_RESET_REASON_ICS_HWR             = 23,	/*!< ICS fault */
+} RGX_CONTEXT_RESET_REASON;
+
+/*!
+	@Brief Context reset data shared with the host
+*/
+typedef struct
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reset reason */
+	IMG_UINT32 ui32ResetExtJobRef;  /*!< External Job ID */
+} RGX_CONTEXT_RESET_REASON_DATA;
+
+#define RGX_HEAP_UM_PDS_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_PDS_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_PDS_RESERVED_TOTAL_SIZE            RGX_HEAP_UM_PDS_RESERVED_SIZE
+
+#define RGX_HEAP_UM_USC_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_USC_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET      RGX_HEAP_UM_USC_RESERVED_SIZE
+
+#define RGX_HEAP_UM_GENERAL_RESERVED_SIZE           DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_GENERAL_RESERVED_REGION_OFFSET  0
+#define RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET  RGX_HEAP_UM_GENERAL_RESERVED_SIZE
+
+/*
+ * 4 dwords reserved for shared register management.
+ * The first dword is the number of shared register blocks to reload.
+ * Should be a multiple of 4 dwords, size in bytes.
+ */
+#define		RGX_LLS_SHARED_REGS_RESERVE_SIZE	(16U)
+
+/*************************************************************************/ /*!
+ Logging type
+*/ /**************************************************************************/
+#define RGXFWIF_LOG_TYPE_NONE			0x00000000U
+#define RGXFWIF_LOG_TYPE_TRACE			0x00000001U
+#define RGXFWIF_LOG_TYPE_GROUP_MAIN		0x00000002U
+#define RGXFWIF_LOG_TYPE_GROUP_MTS		0x00000004U
+#define RGXFWIF_LOG_TYPE_GROUP_CLEANUP	0x00000008U
+#define RGXFWIF_LOG_TYPE_GROUP_CSW		0x00000010U
+#define RGXFWIF_LOG_TYPE_GROUP_BIF		0x00000020U
+#define RGXFWIF_LOG_TYPE_GROUP_PM		0x00000040U
+#define RGXFWIF_LOG_TYPE_GROUP_RTD		0x00000080U
+#define RGXFWIF_LOG_TYPE_GROUP_SPM		0x00000100U
+#define RGXFWIF_LOG_TYPE_GROUP_POW		0x00000200U
+#define RGXFWIF_LOG_TYPE_GROUP_HWR		0x00000400U
+#define RGXFWIF_LOG_TYPE_GROUP_HWP		0x00000800U
+#define RGXFWIF_LOG_TYPE_GROUP_RPM		0x00001000U
+#define RGXFWIF_LOG_TYPE_GROUP_DMA		0x00002000U
+#define RGXFWIF_LOG_TYPE_GROUP_MISC		0x00004000U
+#define RGXFWIF_LOG_TYPE_GROUP_VZ		0x00008000U
+#define RGXFWIF_LOG_TYPE_GROUP_SAFETY	0x00010000U
+#define RGXFWIF_LOG_TYPE_GROUP_VERBOSE	0x00020000U
+#define RGXFWIF_LOG_TYPE_GROUP_CUSTOMER	0x00040000U
+#define RGXFWIF_LOG_TYPE_GROUP_DEBUG	0x80000000U
+#define RGXFWIF_LOG_TYPE_GROUP_MASK		0x8007FFFEU
+#define RGXFWIF_LOG_TYPE_MASK			0x8007FFFFU
+
+/* String used in pvrdebug -h output */
+#define RGXFWIF_LOG_GROUPS_STRING_LIST   "main,mts,cleanup,csw,bif,pm,rtd,spm,pow,hwr,hwp,rpm,dma,misc,vz,safety,verbose,customer,debug"
+
+/* Table entry to map log group strings to log type value */
+typedef struct {
+	const IMG_CHAR* pszLogGroupName;
+	IMG_UINT32      ui32LogGroupType;
+} RGXFWIF_LOG_GROUP_MAP_ENTRY;
+
+/*
+  Macro for use with the RGXFWIF_LOG_GROUP_MAP_ENTRY type to create a lookup
+  table where needed. Keep log group names short, no more than 20 chars.
+*/
+#define RGXFWIF_LOG_GROUP_NAME_VALUE_MAP { "none",    RGXFWIF_LOG_TYPE_NONE }, \
+                                         { "main",    RGXFWIF_LOG_TYPE_GROUP_MAIN }, \
+                                         { "mts",     RGXFWIF_LOG_TYPE_GROUP_MTS }, \
+                                         { "cleanup", RGXFWIF_LOG_TYPE_GROUP_CLEANUP }, \
+                                         { "csw",     RGXFWIF_LOG_TYPE_GROUP_CSW }, \
+                                         { "bif",     RGXFWIF_LOG_TYPE_GROUP_BIF }, \
+                                         { "pm",      RGXFWIF_LOG_TYPE_GROUP_PM }, \
+                                         { "rtd",     RGXFWIF_LOG_TYPE_GROUP_RTD }, \
+                                         { "spm",     RGXFWIF_LOG_TYPE_GROUP_SPM }, \
+                                         { "pow",     RGXFWIF_LOG_TYPE_GROUP_POW }, \
+                                         { "hwr",     RGXFWIF_LOG_TYPE_GROUP_HWR }, \
+                                         { "hwp",     RGXFWIF_LOG_TYPE_GROUP_HWP }, \
+                                         { "rpm",     RGXFWIF_LOG_TYPE_GROUP_RPM }, \
+                                         { "dma",     RGXFWIF_LOG_TYPE_GROUP_DMA }, \
+                                         { "misc",    RGXFWIF_LOG_TYPE_GROUP_MISC }, \
+                                         { "vz",      RGXFWIF_LOG_TYPE_GROUP_VZ }, \
+                                         { "safety",  RGXFWIF_LOG_TYPE_GROUP_SAFETY }, \
+                                         { "verbose", RGXFWIF_LOG_TYPE_GROUP_VERBOSE }, \
+                                         { "customer",RGXFWIF_LOG_TYPE_GROUP_CUSTOMER }, \
+                                         { "debug",   RGXFWIF_LOG_TYPE_GROUP_DEBUG }
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s"
+
+/* Used in a print statement to display log group state, one per group */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST(types)  ((((types) & RGXFWIF_LOG_TYPE_GROUP_MAIN) != 0U)	?("main ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MTS) != 0U)		?("mts ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CLEANUP) != 0U)	?("cleanup ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CSW) != 0U)		?("csw ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_BIF) != 0U)		?("bif ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_PM) != 0U)		?("pm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RTD) != 0U)		?("rtd ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SPM) != 0U)		?("spm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_POW) != 0U)		?("pow ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWR) != 0U)		?("hwr ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWP) != 0U)		?("hwp ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RPM) != 0U)		?("rpm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DMA) != 0U)		?("dma ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MISC) != 0U)	?("misc ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VZ) != 0U)		?("vz ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SAFETY) != 0U)	?("safety ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VERBOSE) != 0U)	?("verbose ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CUSTOMER) != 0U)?("customer ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DEBUG) != 0U)	?("debug ")		:(""))
+
+/*!
+ ******************************************************************************
+ * Trace Buffer
+ *****************************************************************************/
+
+/*! Min, Max, and Default size of RGXFWIF_TRACEBUF_SPACE in DWords */
+#define RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS      8192U  /*  32KB */
+#define RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS 12000U  /* ~48KB */
+#define RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS     32768U  /* 128KB */
+
+#define RGXFW_TRACE_BUFFER_ASSERT_SIZE 200U
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+#define RGXFW_THREAD_NUM 2U
+#else
+#define RGXFW_THREAD_NUM 1U
+#endif
+
+typedef struct
+{
+	IMG_CHAR	szPath[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_CHAR	szInfo[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_UINT32	ui32LineNum;
+} UNCACHED_ALIGN RGXFWIF_FILE_INFO_BUF;
+
+/*!
+ * @Defgroup SRVAndFWTracing Services and Firmware Tracing data interface
+ * @Brief The document groups/lists the data structures and the interfaces related to Services and Firmware Tracing
+ * @{
+ */
+
+/*!
+ * @Brief Firmware trace buffer details
+ */
+typedef struct
+{
+	IMG_UINT32                         ui32TracePointer;          /*!< Trace pointer (write index into Trace Buffer) */
+	IMG_UINT32                         ui32WrapCount;             /*!< Number of times the Trace Buffer has wrapped */
+
+	RGXFWIF_DEV_VIRTADDR               pui32RGXFWIfTraceBuffer;   /*!< Trace buffer address (FW address), to be used by firmware for writing into trace buffer */
+
+	RGXFWIF_FILE_INFO_BUF RGXFW_ALIGN  sAssertBuf;
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF_SPACE;
+
+/*! @} End of Defgroup SRVAndFWTracing */
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief Firmware trace control data
+ */
+typedef struct
+{
+	IMG_UINT32              ui32LogType;                  /*!< FW trace log group configuration */
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[RGXFW_THREAD_NUM];  /*!< FW Trace buffer */
+	IMG_UINT32              ui32TraceBufSizeInDWords;     /*!< FW Trace buffer size in dwords, Member initialised only when sTraceBuf is actually allocated
+															(in RGXTraceBufferInitOnDemandResources) */
+	IMG_UINT32              ui32TracebufFlags;            /*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF;
+
+/* Debug-info sub-fields */
+/* Bit 0: RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT bit from RGX_CR_EVENT_STATUS register */
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT        (0U)
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET          (1U << RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT)
+
+/* Bit 1: RGX_CR_BIF_MMU_ENTRY_PENDING bit from RGX_CR_BIF_MMU_ENTRY register */
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT     (1U)
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET       (1U << RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT)
+
+/* Bit 2: RGX_CR_SLAVE_EVENT register is non-zero */
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT          (2U)
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET            (1U << RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT)
+
+/* Bit 3-15: Unused bits */
+
+#define RGXFWT_DEBUG_INFO_STR_MAXLEN                  64
+#define RGXFWT_DEBUG_INFO_STR_PREPEND                 " (debug info: "
+#define RGXFWT_DEBUG_INFO_STR_APPEND                  ")"
+
+/* Table of debug info sub-field's masks and corresponding message strings
+ * to be appended to firmware trace
+ *
+ * Mask     : 16 bit mask to be applied to debug-info field
+ * String   : debug info message string
+ */
+
+#define RGXFWT_DEBUG_INFO_MSKSTRLIST \
+/*Mask,                                           String*/ \
+X(RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET,      "mmu pf") \
+X(RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET,   "mmu pending") \
+X(RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET,        "slave events")
+
+/* Firmware trace time-stamp field breakup */
+
+/* RGX_CR_TIMER register read (48 bits) value*/
+#define RGXFWT_TIMESTAMP_TIME_SHIFT                   (0U)
+#define RGXFWT_TIMESTAMP_TIME_CLRMSK                  (IMG_UINT64_C(0xFFFF000000000000))
+
+/* Extra debug-info (16 bits) */
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT             (48U)
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK            ~RGXFWT_TIMESTAMP_TIME_CLRMSK
+
+typedef struct
+{
+	IMG_UINT			bfOsState		: 3;
+	IMG_UINT			bfFLOk			: 1;
+	IMG_UINT			bfFLGrowPending	: 1;
+	IMG_UINT			bfReserved		: 27;
+} RGXFWIF_OS_RUNTIME_FLAGS;
+
+#define RGXFWIF_FWFAULTINFO_MAX		(8U)			/* Total number of FW fault logs stored */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64CRTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64OSTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64Data;
+	RGXFWIF_FILE_INFO_BUF	sFaultBuf;
+} UNCACHED_ALIGN RGX_FWFAULTINFO;
+
+#define RGXFWIF_POW_STATES \
+  X(RGXFWIF_POW_OFF)			/* idle and handshaked with the host (ready to full power down) */ \
+  X(RGXFWIF_POW_ON)				/* running HW commands */ \
+  X(RGXFWIF_POW_FORCED_IDLE)	/* forced idle */ \
+  X(RGXFWIF_POW_IDLE)			/* idle waiting for host handshake */
+
+typedef enum
+{
+#define X(NAME) NAME,
+	RGXFWIF_POW_STATES
+#undef X
+} RGXFWIF_POW_STATE;
+
+/* Firmware HWR states */
+#define RGXFWIF_HWR_HARDWARE_OK			(IMG_UINT32_C(0x1) << 0U)	/*!< The HW state is ok or locked up */
+#define RGXFWIF_HWR_RESET_IN_PROGRESS	(IMG_UINT32_C(0x1) << 1U)	/*!< Tells if a HWR reset is in progress */
+#define RGXFWIF_HWR_GENERAL_LOCKUP		(IMG_UINT32_C(0x1) << 3U)	/*!< A DM unrelated lockup has been detected */
+#define RGXFWIF_HWR_DM_RUNNING_OK		(IMG_UINT32_C(0x1) << 4U)	/*!< At least one DM is running without being close to a lockup */
+#define RGXFWIF_HWR_DM_STALLING			(IMG_UINT32_C(0x1) << 5U)	/*!< At least one DM is close to lockup */
+#define RGXFWIF_HWR_FW_FAULT			(IMG_UINT32_C(0x1) << 6U)	/*!< The FW has faulted and needs to restart */
+#define RGXFWIF_HWR_RESTART_REQUESTED	(IMG_UINT32_C(0x1) << 7U)	/*!< The FW has requested the host to restart it */
+
+#define RGXFWIF_PHR_MODE_OFF			(0UL)
+#define RGXFWIF_PHR_MODE_RD_RESET		(1UL)
+
+typedef IMG_UINT32 RGXFWIF_HWR_STATEFLAGS;
+
+typedef IMG_UINT32 RGXFWIF_HWR_RECOVERYFLAGS;
+
+/*! @Brief Firmware system data shared with the Host driver */
+typedef struct
+{
+	IMG_UINT32                 ui32ConfigFlags;                       /*!< Configuration flags from host */
+	IMG_UINT32                 ui32ConfigFlagsExt;                    /*!< Extended configuration flags from host */
+	volatile RGXFWIF_POW_STATE ePowState;
+	struct {
+		volatile IMG_UINT32        ui32HWPerfRIdx;
+		volatile IMG_UINT32        ui32HWPerfWIdx;
+		volatile IMG_UINT32        ui32HWPerfWrapCount;
+	} sHWPerfCtrl; /* Struct used to inval/flush HWPerfCtrl members */
+	IMG_UINT32                 ui32HWPerfSize;                        /*!< Constant after setup, needed in FW */
+	IMG_UINT32                 ui32HWPerfDropCount;                   /*!< The number of times the FW drops a packet due to buffer full */
+
+	/* ui32HWPerfUt, ui32FirstDropOrdinal, ui32LastDropOrdinal only valid when FW is built with
+	 * RGX_HWPERF_UTILIZATION & RGX_HWPERF_DROP_TRACKING defined in rgxfw_hwperf.c */
+	IMG_UINT32                 ui32HWPerfUt;                          /*!< Buffer utilisation, high watermark of bytes in use */
+	IMG_UINT32                 ui32FirstDropOrdinal;                  /*!< The ordinal of the first packet the FW dropped */
+	IMG_UINT32                 ui32LastDropOrdinal;                   /*!< The ordinal of the last packet the FW dropped */
+	RGXFWIF_OS_RUNTIME_FLAGS   asOsRuntimeFlagsMirror[RGXFW_MAX_NUM_OSIDS];/*!< State flags for each Operating System mirrored from Fw coremem */
+	RGX_FWFAULTINFO            sFaultInfo[RGXFWIF_FWFAULTINFO_MAX];   /*!< Firmware fault info */
+	IMG_UINT32                 ui32FWFaults;                          /*!< Firmware faults count */
+	IMG_UINT32                 aui32CrPollAddr[RGXFW_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[RGXFW_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[RGXFW_THREAD_NUM];    /*!< Failed poll count */
+	IMG_UINT64 RGXFW_ALIGN     ui64StartIdleTime;
+	IMG_UINT32                 aui32TSMirror[RGXFW_MAX_NUM_OSIDS];    /*!< Array of time slice per OS Mirrored from the FW */
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK)
+#define RGXFWIF_STATS_FRAMEWORK_LINESIZE    (8)
+#define RGXFWIF_STATS_FRAMEWORK_MAX         (2048*RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+	IMG_UINT32 RGXFW_ALIGN     aui32FWStatsBuf[RGXFWIF_STATS_FRAMEWORK_MAX];
+#endif
+	RGXFWIF_HWR_STATEFLAGS     ui32HWRStateFlags; /*!< Firmware's Current HWR state */
+	RGXFWIF_HWR_RECOVERYFLAGS  aui32HWRRecoveryFlags[RGXFWIF_DM_MAX]; /*!< Each DM's HWR state */
+	IMG_UINT32                 ui32FwSysDataFlags;                      /*!< Compatibility and other flags */
+	IMG_UINT32                 ui32MemFaultCheck;                       /*!< Device mem fault check on PCI systems */
+} UNCACHED_ALIGN RGXFWIF_SYSDATA;
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+#define PVR_SLR_LOG_ENTRIES 10U
+#define PVR_SLR_LOG_STRLEN  30 /*!< MAX_CLIENT_CCB_NAME not visible to this header */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64Timestamp;
+	IMG_UINT32				ui32FWCtxAddr;
+	IMG_UINT32				ui32NumUFOs;
+	IMG_CHAR				aszCCBName[PVR_SLR_LOG_STRLEN];
+} UNCACHED_ALIGN RGXFWIF_SLR_ENTRY;
+#endif
+
+#define RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS (128U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware per-os data and configuration
+ */
+typedef struct
+{
+	IMG_UINT32                 ui32FwOsConfigFlags;                   /*!< Configuration flags from an OS */
+	IMG_UINT32                 ui32FWSyncCheckMark;                   /*!< Markers to signal that the host should perform a full sync check */
+	IMG_UINT32                 ui32HostSyncCheckMark;                  /*!< Markers to signal that the Firmware should perform a full sync check */
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+	IMG_UINT32                 ui32ForcedUpdatesRequested;
+	IMG_UINT8                  ui8SLRLogWp;
+	RGXFWIF_SLR_ENTRY          sSLRLogFirst;
+	RGXFWIF_SLR_ENTRY          sSLRLog[PVR_SLR_LOG_ENTRIES];
+	IMG_UINT64 RGXFW_ALIGN     ui64LastForcedUpdateTime;
+#endif
+	volatile IMG_UINT32        aui32InterruptCount[RGXFW_THREAD_NUM]; /*!< Interrupt count from Threads > */
+	IMG_UINT32                 ui32KCCBCmdsExecuted;                  /*!< Executed Kernel CCB command count */
+	RGXFWIF_DEV_VIRTADDR       sPowerSync;                            /*!< Sync prim used to signal the host the power off state */
+	IMG_UINT32                 ui32FwOsDataFlags;                       /*!< Compatibility and other flags */
+	IMG_UINT32                 aui32OfflineBuffer[RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS];
+} UNCACHED_ALIGN RGXFWIF_OSDATA;
+
+/*!
+ ******************************************************************************
+ * HWR Data
+ *****************************************************************************/
+/*!
+ * @Defgroup HWRInfo FW HWR shared data interface
+ * @Brief Types grouping data structures and defines used in realising the HWR record.
+ * @{
+ */
+
+#define RGXFW_PROCESS_NAME_LEN	(16)
+
+/*! @Brief HWR Lockup types */
+typedef enum
+{
+	RGX_HWRTYPE_UNKNOWNFAILURE = 0, /*!< Unknown failure */
+	RGX_HWRTYPE_OVERRUN        = 1, /*!< DM overrun */
+	RGX_HWRTYPE_POLLFAILURE    = 2, /*!< Poll failure */
+	RGX_HWRTYPE_BIF0FAULT      = 3, /*!< BIF0 fault */
+	RGX_HWRTYPE_BIF1FAULT      = 4, /*!< BIF1 fault */
+	RGX_HWRTYPE_TEXASBIF0FAULT = 5, /*!< TEXASBIF0 fault */
+	RGX_HWRTYPE_MMUFAULT       = 6, /*!< MMU fault */
+	RGX_HWRTYPE_MMUMETAFAULT   = 7, /*!< MMU META fault */
+	RGX_HWRTYPE_MIPSTLBFAULT   = 8, /*!< MIPS TLB fault */
+	RGX_HWRTYPE_ECCFAULT       = 9, /*!< ECC fault */
+	RGX_HWRTYPE_MMURISCVFAULT  = 10, /*!< MMU RISCV fault */
+	RGX_HWRTYPE_ICSFAULT       = 11, /*!< Idle Cycle Stealing fault */
+} RGX_HWRTYPE;
+
+#define RGXFWIF_HWRTYPE_BIF_BANK_GET(eHWRType) (((eHWRType) == RGX_HWRTYPE_BIF0FAULT) ? 0 : 1)
+
+#define RGXFWIF_HWRTYPE_PAGE_FAULT_GET(eHWRType) ((((eHWRType) == RGX_HWRTYPE_BIF0FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_BIF1FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_TEXASBIF0FAULT) ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUFAULT)       ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUMETAFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MIPSTLBFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMURISCVFAULT)) ? true : false)
+
+/************************
+ *  GPU HW error codes  *
+ ************************/
+typedef enum
+{
+	RGX_HW_ERR_NA = 0x0,
+	RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL = 0x101,
+} RGX_HW_ERR;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFReqStatus; /*!< BIF request status */
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFMMUStatus; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_BIFINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32FaultGPU; /*!< ECC fault in GPU */
+} RGX_ECCINFO;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		aui64MMUStatus[2]; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_MMUINFO;
+
+typedef struct
+{
+	IMG_UINT32	ui32ThreadNum; /*!< Thread ID performing poll operation */
+	IMG_UINT32	ui32CrPollAddr; /*!< CR Poll Address */
+	IMG_UINT32	ui32CrPollMask; /*!< CR Poll mask */
+	IMG_UINT32	ui32CrPollLastValue; /*!< CR Poll last value */
+	IMG_UINT64	RGXFW_ALIGN ui64Reserved;
+} UNCACHED_ALIGN RGX_POLLINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32BadVAddr; /*!< VA address */
+	IMG_UINT32 ui32EntryLo;
+} RGX_TLBINFO;
+
+/*! @Brief Structure to keep information specific to a lockup e.g. DM, timer, lockup type etc. */
+typedef struct
+{
+	union
+	{
+		RGX_BIFINFO  sBIFInfo; /*!< BIF failure details */
+		RGX_MMUINFO  sMMUInfo; /*!< MMU failure details */
+		RGX_POLLINFO sPollInfo; /*!< Poll failure details */
+		RGX_TLBINFO  sTLBInfo; /*!< TLB failure details */
+		RGX_ECCINFO  sECCInfo; /*!< ECC failure details */
+	} uHWRData;
+
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimer; /*!< Timer value at the time of lockup */
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimer; /*!< OS timer value at the time of lockup */
+	IMG_UINT32             ui32FrameNum; /*!< Frame number of the workload */
+	IMG_UINT32             ui32PID; /*!< PID belonging to the workload */
+	IMG_UINT32             ui32ActiveHWRTData; /*!< HWRT data of the workload */
+	IMG_UINT32             ui32HWRNumber; /*!< HWR number */
+	IMG_UINT32             ui32EventStatus; /*!< Core specific event status register at the time of lockup */
+	IMG_UINT32             ui32HWRRecoveryFlags; /*!< DM state flags */
+	RGX_HWRTYPE            eHWRType; /*!< Type of lockup */
+	RGXFWIF_DM             eDM; /*!< Recovery triggered for the DM */
+	IMG_UINT32             ui32CoreID; /*!< Core ID of the GPU */
+	RGX_HW_ERR             eHWErrorCode; /*!< Error code used to determine HW fault */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeOfKick; /*!< Workload kick time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetStart; /*!< HW reset start time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetFinish; /*!< HW reset stop time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeFreelistReady; /*!< freelist ready time on the last HWR */
+	IMG_CHAR   RGXFW_ALIGN szProcName[RGXFW_PROCESS_NAME_LEN]; /*!< User process name */
+	IMG_UINT32 RGXFW_ALIGN ui32PDSStalledDMs; /*!< DMs stalled on PDS Store space */
+	IMG_UINT32             ui32PDSActiveDMUSCs; /*!< Per-DM USC PDS activity */
+} UNCACHED_ALIGN RGX_HWRINFO;
+
+#define RGXFWIF_HWINFO_MAX_FIRST 8U							/* Number of first HWR logs recorded (never overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX_LAST 8U							/* Number of latest HWR logs (older logs are overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX (RGXFWIF_HWINFO_MAX_FIRST + RGXFWIF_HWINFO_MAX_LAST)	/* Total number of HWR logs stored in a buffer */
+#define RGXFWIF_HWINFO_LAST_INDEX (RGXFWIF_HWINFO_MAX - 1U)	/* Index of the last log in the HWR log buffer */
+
+/*! @Brief Firmware HWR information structure allocated by the Services and used by the Firmware to update recovery information. */
+typedef struct
+{
+	RGX_HWRINFO sHWRInfo[RGXFWIF_HWINFO_MAX]; /*!< Max number of recovery record */
+	IMG_UINT32  ui32HwrCounter; /*!< HWR counter used in FL reconstruction */
+	IMG_UINT32  ui32WriteIndex; /*!< Index for updating recovery information in sHWRInfo */
+	IMG_UINT32  ui32DDReqCount; /*!< Count of DebugDump requested to the host after recovery */
+	IMG_UINT32  ui32HWRInfoBufFlags; /* Compatibility and other flags */
+	IMG_UINT32  aui32HwrDmLockedUpCount[RGXFWIF_DM_MAX]; /*!< Lockup count for each DM */
+	IMG_UINT32  aui32HwrDmOverranCount[RGXFWIF_DM_MAX]; /*!< Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmRecoveredCount[RGXFWIF_DM_MAX]; /*!< Lockup + Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmFalseDetectCount[RGXFWIF_DM_MAX]; /*!< False lockup detection count for each DM */
+} UNCACHED_ALIGN RGXFWIF_HWRINFOBUF;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COMMONCTX_STATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RF_CMD;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWMEMCONTEXT;
+
+/*! @} End of HWRInfo */
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Freelist holding usage state of the Parameter Buffers
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN sFreeListBaseDevVAddr;	/*!< Freelist page table base address */
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN sFreeListStateDevVAddr;	/*!< Freelist state buffer base address */
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN sFreeListLastGrowDevVAddr;	/*!< Freelist base address at last grow */
+
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT64		RGXFW_ALIGN ui64CurrentDevVAddr;/*!< Freelist page table entry for current free page */
+	IMG_UINT32		ui32CurrentStackTop;		/*!< Freelist current free page */
+#endif
+
+	IMG_UINT32		ui32MaxPages;			/*!< Max no. of pages can be added to the freelist */
+	IMG_UINT32		ui32GrowPages;			/*!< No pages to add in each freelist grow */
+	IMG_UINT32		ui32CurrentPages;		/*!< Total no. of pages made available to the PM HW */
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT32		ui32AllocatedPageCount;		/*!< No. of pages allocated by PM HW */
+	IMG_UINT32		ui32AllocatedMMUPageCount;	/*!< No. of pages allocated for GPU MMU for PM*/
+#endif
+#if defined(SUPPORT_SHADOW_FREELISTS)
+	IMG_UINT32		ui32HWRCounter;
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;
+#endif
+	IMG_UINT32		ui32FreeListID;			/*!< Unique Freelist ID */
+	IMG_BOOL		bGrowPending;			/*!< Freelist grow is pending */
+	IMG_UINT32		ui32ReadyPages;			/*!< Reserved pages to be used only on PM OOM event */
+	IMG_UINT32		ui32FreelistFlags;		/*!< Compatibility and other flags */
+
+	IMG_BOOL		bUpdatePending;
+	IMG_UINT32		ui32UpdateNewPages;
+	IMG_UINT32		ui32UpdateNewReadyPages;
+} UNCACHED_ALIGN RGXFWIF_FREELIST;
+
+typedef struct {RGXFWIF_DEV_VIRTADDR sNext;
+                RGXFWIF_DEV_VIRTADDR sPrev;}	RGXFW_DLLIST_NODE;
+
+#define RGXFWIF_MAX_NUM_CANCEL_REQUESTS		(8U)		/* Maximum number of workload cancellation requests */
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware Common Context (or FWCC)
+ */
+typedef struct RGXFWIF_FWCOMMONCONTEXT_
+{
+	/* CCB details for this firmware context */
+	PRGXFWIF_CCCB_CTL		psCCBCtl;				/*!< CCB control */
+	PRGXFWIF_CCCB			psCCB;					/*!< CCB base */
+	RGXFWIF_DMA_ADDR		sCCBMetaDMAAddr;
+
+	/* Context suspend state */
+	PRGXFWIF_COMMONCTX_STATE	RGXFW_ALIGN psContextState;		/*!< TA/3D context suspend state, read/written by FW */
+
+	/* Flags e.g. for context switching */
+	IMG_UINT32				ui32FWComCtxFlags;
+	IMG_INT32				i32Priority;  /*!< Priority level */
+	IMG_UINT32				ui32PrioritySeqNum;
+
+	/* Framework state */
+	PRGXFWIF_RF_CMD			RGXFW_ALIGN psRFCmd;		/*!< Register updates for Framework */
+
+	/* Misc and compatibility flags */
+	IMG_UINT32				ui32CompatFlags;
+
+	/* Statistic updates waiting to be passed back to the host... */
+	IMG_INT32				i32StatsNumStores;		/*!< Number of stores on this context since last update */
+	IMG_INT32				i32StatsNumOutOfMemory;		/*!< Number of OOMs on this context since last update */
+	IMG_INT32				i32StatsNumPartialRenders;	/*!< Number of PRs on this context since last update */
+	RGXFWIF_DM				eDM;				/*!< Data Master type */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN  sBufStalledNode;			/*!< List entry for the buffer stalled list */
+	IMG_UINT64				RGXFW_ALIGN  ui64CBufQueueCtrlAddr;	/*!< Address of the circular buffer queue pointers */
+
+	IMG_UINT64				RGXFW_ALIGN  ui64RobustnessAddress;
+	IMG_UINT32				ui32MaxDeadlineMS;			/*!< Max HWR deadline limit in ms */
+	bool					bReadOffsetNeedsReset;			/*!< Following HWR circular buffer read-offset needs resetting */
+	IMG_BOOL				bIPPContextNeedsReset;			/*!< Following HWR during 3D, IPP context needs resetting */
+
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sWaitingNode;		/*!< List entry for the waiting list */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sRunNode;			/*!< List entry for the run list */
+	RGXFWIF_UFO				sLastFailedUFO;			/*!< UFO that last failed (or NULL) */
+
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;					/*!< Memory context */
+
+	/* References to the host side originators */
+	IMG_UINT32				ui32ServerCommonContextID;	/*!< the Server Common Context */
+	IMG_UINT32				ui32PID;			/*!< associated process ID */
+
+	IMG_BOOL				bGeomOOMDisabled;		/*!< True when Geom DM OOM is not allowed */
+
+	IMG_UINT32				ui32PipelinedKicks;		/*!< Number of kick from this CCB currently submitted to the DM pipeline */
+	IMG_CHAR				szProcName[RGXFW_PROCESS_NAME_LEN];	/*!< User process name */
+
+	IMG_UINT32				ui32DeferCount;		/*!< Number of context defers before forced scheduling of context */
+	IMG_UINT32				aui32FirstIntJobRefToCancel[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the beginning of range of IntJobRefs at and above which workloads will be discarded */
+	IMG_UINT32				aui32FirstValidIntJobRef[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the end of range of IntJobRef below which workloads will be discarded */
+	IMG_BOOL				bCancelRangesActive;	/*!< True if any active ranges in aui32FirstIntJobRefToCancel and aui32FirstValidIntJobRef arrays */
+	IMG_BOOL				bLastKickedCmdWasSafetyOnly;
+	IMG_UINT32				ui32UID;			/*!< associated process UID used in FW managed gpu work period hwperf events */
+} UNCACHED_ALIGN RGXFWIF_FWCOMMONCONTEXT;
+
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) <= 256U,
+              "Size of structure RGXFWIF_FWCOMMONCONTEXT exceeds maximum expected size.");
+
+/*!
+ ******************************************************************************
+ * HWRTData
+ *****************************************************************************/
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FREELIST;
+
+/* HWRTData flags */
+/* Deprecated flags 1:0 */
+#define HWRTDATA_HAS_LAST_TA              (IMG_UINT32_C(1) << 2)
+#define HWRTDATA_PARTIAL_RENDERED         (IMG_UINT32_C(1) << 3)
+#define HWRTDATA_KILLED                   (IMG_UINT32_C(1) << 4)
+#define HWRTDATA_KILL_AFTER_TARESTART     (IMG_UINT32_C(1) << 5)
+#if defined(SUPPORT_AGP)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT0    (IMG_UINT32_C(1) << 6)
+#if defined(SUPPORT_AGP4)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT1    (IMG_UINT32_C(1) << 7)
+#endif
+#define HWRTDATA_GEOM_NEEDS_RESUME        (IMG_UINT32_C(1) << 8)
+#endif
+#if defined(SUPPORT_TRP)
+#define HWRTDATA_GEOM_TRP_IN_PROGRESS          (IMG_UINT32_C(1) << 9)
+#endif
+
+typedef enum
+{
+	RGXFWIF_RTDATA_STATE_NONE = 0,
+	RGXFWIF_RTDATA_STATE_KICKTA,
+	RGXFWIF_RTDATA_STATE_KICKTAFIRST,
+	RGXFWIF_RTDATA_STATE_TAFINISHED,
+	RGXFWIF_RTDATA_STATE_KICK3D,
+	RGXFWIF_RTDATA_STATE_3DFINISHED,
+	RGXFWIF_RTDATA_STATE_3DCONTEXTSTORED,
+	RGXFWIF_RTDATA_STATE_TAOUTOFMEM,
+	RGXFWIF_RTDATA_STATE_PARTIALRENDERFINISHED,
+	/* In case of HWR, we can't set the RTDATA state to NONE,
+	 * as this will cause any TA to become a first TA.
+	 * To ensure all related TA's are skipped, we use the HWR state */
+	RGXFWIF_RTDATA_STATE_HWR,
+	RGXFWIF_RTDATA_STATE_UNKNOWN = 0x7FFFFFFFU
+} RGXFWIF_RTDATA_STATE;
+
+typedef struct
+{
+    IMG_UINT32							ui32ScreenPixelMax;
+	IMG_UINT64							RGXFW_ALIGN ui64PPPMultiSampleCtl;
+	IMG_UINT32							ui32TEStride;
+	IMG_UINT32							ui32TPCSize;
+	IMG_UINT32							ui32TEScreen;
+	IMG_UINT32							ui32TEAA;
+	IMG_UINT32							ui32TEMTILE1;
+	IMG_UINT32							ui32TEMTILE2;
+	IMG_UINT32							ui32RgnStride;
+	IMG_UINT32							ui32ISPMergeLowerX;
+	IMG_UINT32							ui32ISPMergeLowerY;
+	IMG_UINT32							ui32ISPMergeUpperX;
+	IMG_UINT32							ui32ISPMergeUpperY;
+	IMG_UINT32							ui32ISPMergeScaleX;
+	IMG_UINT32							ui32ISPMergeScaleY;
+} UNCACHED_ALIGN RGXFWIF_HWRTDATA_COMMON;
+
+typedef struct
+{
+	IMG_UINT32           ui32RenderTargetIndex;		//Render number
+	IMG_UINT32           ui32CurrentRenderTarget;	//index in RTA
+	IMG_UINT32           ui32ActiveRenderTargets;	//total active RTs
+	RGXFWIF_DEV_VIRTADDR sValidRenderTargets;  //Array of valid RT indices
+	RGXFWIF_DEV_VIRTADDR sRTANumPartialRenders;  //Array of number of occurred partial renders per render target
+	IMG_UINT32           ui32MaxRTs;   //Number of render targets in the array
+	IMG_UINT32           ui32RTACtlFlags; /* Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_RTA_CTL;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Render Target data i.e. HWRTDATA used to hold the PM context
+ */
+typedef struct
+{
+	IMG_UINT64			RGXFW_ALIGN ui64VCECatBase[2];			/*!< VCE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64VCELastCatBase[2];
+	IMG_UINT64			RGXFW_ALIGN ui64TECatBase[2];			/*!< TE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64TELastCatBase[2];
+	IMG_UINT64			RGXFW_ALIGN ui64AlistCatBase;			/*!< Alist Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64AlistLastCatBase;
+
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psVHeapTableDevVAddr;		/*!< VHeap table base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sPMMListDevVAddr;			/*!< Mlist table base */
+#else
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sPMRenderStateDevVAddr;		/*!< Series8 PM State buffers */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sPMSecureRenderStateDevVAddr;
+#endif
+
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[RGXFW_MAX_FREELISTS]; /*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[RGXFW_MAX_FREELISTS];
+	IMG_BOOL			bRenderStateNeedsReset;
+
+	IMG_UINT32			ui32HWRTDataFlags;
+	RGXFWIF_RTDATA_STATE		eState;						/*!< Current workload processing state of HWRTDATA */
+
+	RGXFWIF_RTA_CTL			sRTACtl;					/*!< Render target array data */
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sTailPtrsDevVAddr;			/*!< Tail pointers base */
+#if defined(RGX_FIRMWARE)
+	struct RGXFWIF_FWCOMMONCONTEXT_* psOwnerGeom;
+#else
+	RGXFWIF_DEV_VIRTADDR		pui32OwnerGeomNotUsedByHost;
+#endif
+
+#if defined(PM_INTERACTIVE_MODE)
+	IMG_UINT64			RGXFW_ALIGN ui64PMAListStackPointer;		/*!< Freelist page table entry for current Mlist page */
+	IMG_UINT32			ui32PMMListStackPointer;			/*!< Current Mlist page */
+#endif
+#if defined(SUPPORT_TRP)
+	IMG_UINT32			ui32KickFlagsCopy;
+#endif
+
+	RGXFWIF_CLEANUP_CTL		RGXFW_ALIGN_DCACHEL sCleanupState;					/*!< Render target clean up state */
+} RGXFW_ALIGN_DCACHEL RGXFWIF_HWRTDATA;
+
+/* Sync_checkpoint firmware object.
+ * This is the FW-addressable structure used to hold the sync checkpoint's
+ * state and other information which needs to be accessed by the firmware.
+ */
+typedef struct
+{
+	IMG_UINT32	ui32State;          /*!< Holds the current state of the sync checkpoint */
+	IMG_UINT32	ui32FwRefCount;     /*!< Holds the FW reference count (num of fences/updates processed) */
+} SYNC_CHECKPOINT_FW_OBJ;
+
+/* Bit mask Firmware can use to test if a checkpoint has signalled or errored */
+#define SYNC_CHECKPOINT_SIGNALLED_MASK (0x1 << 0)
+
+typedef enum
+{
+	RGXFWIF_REG_CFG_TYPE_PWR_ON=0,      /* Sidekick power event */
+	RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,   /* Rascal / dust power event */
+	RGXFWIF_REG_CFG_TYPE_TA,            /* TA kick */
+	RGXFWIF_REG_CFG_TYPE_3D,            /* 3D kick */
+	RGXFWIF_REG_CFG_TYPE_CDM,           /* Compute kick */
+	RGXFWIF_REG_CFG_TYPE_TDM,           /* TDM kick */
+	RGXFWIF_REG_CFG_TYPE_ALL            /* Applies to all types. Keep as last element */
+} RGXFWIF_REG_CFG_TYPE;
+
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_OFFSET_BYTES                 RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES               (1U << 21)
+
+#define RGX_HEAP_KM_USC_RESERVED_SIZE                              RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES
+#define RGX_HEAP_USC_RESERVED_TOTAL_SIZE                           RGX_HEAP_UM_USC_RESERVED_SIZE + RGX_HEAP_KM_USC_RESERVED_SIZE
+
+#define RGXFW_SCRATCH_BUF_SIZE             (1048576U)
+
+#define RGX_MAX_SECURE_ALLOC_NUM_CORES     (8U)
+
+#define RGX_MAX_SECURE_ALLOC_NUM_CLUSTERS  (12U)
+
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES      RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES      (RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#define RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_OFFSET_BYTES     (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_MAX_SIZE_BYTES   8192U
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_OFFSET_BYTES     (RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES   (0x00004000U * RGX_MAX_SECURE_ALLOC_NUM_CORES *  RGX_MAX_SECURE_ALLOC_NUM_CLUSTERS + 48U + 1023U)
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_OFFSET_BYTES   (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_MAX_SIZE_BYTES RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_OFFSET_BYTES    (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_MAX_SIZE_BYTES  (((0x00000080U + 127U) * RGX_MAX_SECURE_ALLOC_NUM_CORES) + 127U)
+
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_OFFSET_BYTES    (RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_MAX_SIZE_BYTES  (((0x00000180U + 127U) * RGX_MAX_SECURE_ALLOC_NUM_CORES) + 127U)
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      PVR_ALIGN((RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_3D_SECURE_IPP_BUF_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_BUF_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_SR_B_BUF_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_CONTEXT_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_CONTEXT_MAX_SIZE_BYTES), DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY)
+#else
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      0
+#endif
+
+#endif /*  RGX_FWIF_SHARED_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_shared.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_heaps.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_heaps.h
new file mode 100644
index 000000000000..4e72d8d1a7bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_heaps.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAPS_H)
+#define RGX_HEAPS_H
+
+/*
+  Identify heaps by their names
+*/
+#define RGX_GENERAL_SVM_HEAP_IDENT          "General SVM"               /*!< SVM (shared virtual memory) Heap Identifier */
+#define RGX_GENERAL_HEAP_IDENT              "General"                   /*!< RGX General Heap Identifier */
+#define RGX_GENERAL_NON4K_HEAP_IDENT        "General NON-4K"            /*!< RGX General non-4K Heap Identifier */
+#define RGX_PDSCODEDATA_HEAP_IDENT          "PDS Code and Data"         /*!< RGX PDS Code/Data Heap Identifier */
+#define RGX_USCCODE_HEAP_IDENT              "USC Code"                  /*!< RGX USC Code Heap Identifier */
+#define RGX_USCCODE_BPH_HEAP_IDENT          "BP Handler USC Code"       /*!< RGX USC Code for breakpoint handlers Heap Identifier */
+#define RGX_VK_CAPT_REPLAY_HEAP_IDENT       "Vulkan Capture Replay"     /*!< RGX vulkan capture replay buffer Heap Identifier */
+#define RGX_COMPONENT_CTRL_HEAP_IDENT       "Component Control"         /*!< RGX DCE Component Control Heap Identifier */
+#define RGX_FBCDC_HEAP_IDENT                "FBCDC"                     /*!< RGX FBCDC State Table Heap Identifier */
+#define RGX_FBCDC_LARGE_HEAP_IDENT          "Large FBCDC"               /*!< RGX Large FBCDC State Table Heap Identifier */
+#define RGX_PDS_INDIRECT_STATE_HEAP_IDENT   "PDS Indirect State"        /*!< PDS Indirect State Table Heap Identifier */
+#define RGX_CMP_MISSION_RMW_HEAP_IDENT      "Compute Mission RMW"       /*!< Compute Mission RMW Heap Identifier */
+#define RGX_CMP_SAFETY_RMW_HEAP_IDENT       "Compute Safety RMW"        /*!< Compute Safety RMW Heap Identifier */
+#define RGX_TEXTURE_STATE_HEAP_IDENT        "Texture State"             /*!< Texture State Heap Identifier */
+#define RGX_VISIBILITY_TEST_HEAP_IDENT      "Visibility Test"           /*!< Visibility Test Heap Identifier */
+
+#endif /* RGX_HEAPS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf.h
new file mode 100644
index 000000000000..d0d4014a117a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf.h
@@ -0,0 +1,343 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_H_
+#define RGX_HWPERF_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_hwperf_common.h"
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+/* HWPerf interface assumption checks */
+static_assert(RGX_FEATURE_NUM_CLUSTERS <= 16U,
+			  "Cluster count too large for HWPerf protocol definition");
+#endif
+
+/*! Perf counter control words */
+#define RGX_HWPERF_CTRL_NOP						(0)			/*!< only update HW counters */
+#define RGX_HWPERF_CTRL_STATE_UPDATE_EN			(1U << 31)	/*!< persistent state update; see other flags below */
+#define RGX_HWPERF_CTRL_GEOM_FULLRANGE			(1U)		/*!< selectable geom and 3D counters are full range */
+#define RGX_HWPERF_CTRL_COMP_FULLRANGE			(2U)		/*!< selectable compute counters are full range */
+#define RGX_HWPERF_CTRL_TDM_FULLRANGE			(4U)		/*!< selectable TDM counters are full range */
+
+
+/******************************************************************************
+ * Data Stream Common Types
+ *****************************************************************************/
+
+/*! All the Data Masters HWPerf is aware of. When a new DM is added to this
+ * list, it should be appended at the end to maintain backward compatibility
+ * of HWPerf data.
+ */
+#define RGX_HWPERF_DM_GP                  0x00000000U
+#define	RGX_HWPERF_DM_TDM                 0x00000001U
+#define	RGX_HWPERF_DM_GEOM                0x00000002U
+#define	RGX_HWPERF_DM_3D                  0x00000003U
+#define	RGX_HWPERF_DM_CDM                 0x00000004U
+#define	RGX_HWPERF_DM_RTU                 0x00000005U
+
+#define	RGX_HWPERF_DM_LAST                0x00000006U
+
+#define	RGX_HWPERF_DM_INVALID             0x1FFFFFFFU
+
+/******************************************************************************
+ * API Types
+ *****************************************************************************/
+
+/*! Counter block IDs for all the hardware blocks with counters.
+ * Directly addressable blocks must have a value between 0..15 [0..0xF].
+ * Indirect groups have following encoding:
+ * First hex digit (LSB) represents a unit number within the group
+ * and the second hex digit represents the group number.
+ * Group 0 is the direct group, all others are indirect groups.
+ */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_ID;
+
+/*! Directly addressable non bank-switched counter blocks */
+#define RGX_CNTBLK_ID_JONES          0x0000U
+#define RGX_CNTBLK_ID_SLC            0x0001U	/*!< SLC-specific counter control */
+#define RGX_CNTBLK_ID_FBCDC          0x0002U
+#define RGX_CNTBLK_ID_FW_CUSTOM      0x0003U	/*!< Custom FW provided counters */
+
+/*! Directly addressable SLC counter blocks - presence depends on GPU. */
+#define RGX_CNTBLK_ID_SLCBANK0       0x0004U	/*!< SLCBANK0 counter control */
+#define RGX_CNTBLK_ID_SLCBANK1       0x0005U	/*!< SLCBANK1 counter control */
+#define RGX_CNTBLK_ID_SLCBANK2       0x0006U	/*!< SLCBANK2 counter control */
+#define RGX_CNTBLK_ID_SLCBANK3       0x0007U	/*!< SLCBANK3 counter control */
+#define RGX_CNTBLK_ID_SLCBANK_ALL    0x4004U	/*!< SLC ALL block ID */
+
+#define RGX_CNTBLK_ID_DIRECT_LAST    0x0008U	/*!< Indirect blocks start from here */
+
+/*! Indirectly addressable counter blocks */
+#define RGX_CNTBLK_ID_ISP0           0x0010U	/*!< ISP 1..N ISP */
+#define RGX_CNTBLK_ID_ISP1           0x0011U
+#define RGX_CNTBLK_ID_ISP2           0x0012U
+#define RGX_CNTBLK_ID_ISP3           0x0013U
+#define RGX_CNTBLK_ID_ISP4           0x0014U
+#define RGX_CNTBLK_ID_ISP5           0x0015U
+#define RGX_CNTBLK_ID_ISP6           0x0016U
+#define RGX_CNTBLK_ID_ISP7           0x0017U
+#define RGX_CNTBLK_ID_ISP8           0x0018U
+#define RGX_CNTBLK_ID_ISP9           0x0019U
+#define RGX_CNTBLK_ID_ISP_ALL        0x4010U
+
+#define RGX_CNTBLK_ID_MERCER0        0x0020U	/*!< MERCER 1..N MERCER */
+#define RGX_CNTBLK_ID_MERCER1        0x0021U
+#define RGX_CNTBLK_ID_MERCER2        0x0022U
+#define RGX_CNTBLK_ID_MERCER3        0x0023U
+#define RGX_CNTBLK_ID_MERCER4        0x0024U
+#define RGX_CNTBLK_ID_MERCER5        0x0025U
+#define RGX_CNTBLK_ID_MERCER6        0x0026U
+#define RGX_CNTBLK_ID_MERCER7        0x0027U
+#define RGX_CNTBLK_ID_MERCER8        0x0028U
+#define RGX_CNTBLK_ID_MERCER9        0x0029U
+#define RGX_CNTBLK_ID_MERCER_ALL     0x4020U
+
+#define RGX_CNTBLK_ID_PBE0           0x0030U	/*!< PBE 1..N PBE_PER_SPU x N SPU */
+#define RGX_CNTBLK_ID_PBE1           0x0031U
+#define RGX_CNTBLK_ID_PBE2           0x0032U
+#define RGX_CNTBLK_ID_PBE3           0x0033U
+#define RGX_CNTBLK_ID_PBE4           0x0034U
+#define RGX_CNTBLK_ID_PBE5           0x0035U
+#define RGX_CNTBLK_ID_PBE6           0x0036U
+#define RGX_CNTBLK_ID_PBE7           0x0037U
+#define RGX_CNTBLK_ID_PBE_ALL        0x4030U
+
+#define RGX_CNTBLK_ID_PBE_SHARED0    0x0040U	/*!< PBE_SHARED 1..N SPU */
+#define RGX_CNTBLK_ID_PBE_SHARED1    0x0041U
+#define RGX_CNTBLK_ID_PBE_SHARED2    0x0042U
+#define RGX_CNTBLK_ID_PBE_SHARED3    0x0043U
+#define RGX_CNTBLK_ID_PBE_SHARED_ALL 0x4040U
+
+#define RGX_CNTBLK_ID_USC0           0x0050U	/*!< USC 1..N USC */
+#define RGX_CNTBLK_ID_USC1           0x0051U
+#define RGX_CNTBLK_ID_USC2           0x0052U
+#define RGX_CNTBLK_ID_USC3           0x0053U
+#define RGX_CNTBLK_ID_USC4           0x0054U
+#define RGX_CNTBLK_ID_USC5           0x0055U
+#define RGX_CNTBLK_ID_USC6           0x0056U
+#define RGX_CNTBLK_ID_USC7           0x0057U
+#define RGX_CNTBLK_ID_USC8           0x0058U
+#define RGX_CNTBLK_ID_USC9           0x0059U
+#define RGX_CNTBLK_ID_USC_ALL        0x4050U
+
+#define RGX_CNTBLK_ID_TPU0           0x0060U	/*!< TPU 1..N TPU */
+#define RGX_CNTBLK_ID_TPU1           0x0061U
+#define RGX_CNTBLK_ID_TPU2           0x0062U
+#define RGX_CNTBLK_ID_TPU3           0x0063U
+#define RGX_CNTBLK_ID_TPU4           0x0064U
+#define RGX_CNTBLK_ID_TPU5           0x0065U
+#define RGX_CNTBLK_ID_TPU6           0x0066U
+#define RGX_CNTBLK_ID_TPU7           0x0067U
+#define RGX_CNTBLK_ID_TPU8           0x0068U
+#define RGX_CNTBLK_ID_TPU9           0x0069U
+#define RGX_CNTBLK_ID_TPU_ALL        0x4060U
+
+#define RGX_CNTBLK_ID_SWIFT0         0x0070U	/*!< SWIFT 1..N SWIFT */
+#define RGX_CNTBLK_ID_SWIFT1         0x0071U
+#define RGX_CNTBLK_ID_SWIFT2         0x0072U
+#define RGX_CNTBLK_ID_SWIFT3         0x0073U
+#define RGX_CNTBLK_ID_SWIFT4         0x0074U
+#define RGX_CNTBLK_ID_SWIFT5         0x0075U
+#define RGX_CNTBLK_ID_SWIFT6         0x0076U
+#define RGX_CNTBLK_ID_SWIFT7         0x0077U
+#define RGX_CNTBLK_ID_SWIFT8         0x0078U
+#define RGX_CNTBLK_ID_SWIFT9         0x0079U
+#define RGX_CNTBLK_ID_SWIFT_ALL      0x4070U
+
+#define RGX_CNTBLK_ID_TEXAS0         0x0080U	/*!< TEXAS 1..N TEXAS */
+#define RGX_CNTBLK_ID_TEXAS1         0x0081U
+#define RGX_CNTBLK_ID_TEXAS2         0x0082U
+#define RGX_CNTBLK_ID_TEXAS3         0x0083U
+#define RGX_CNTBLK_ID_TEXAS_ALL      0x4080U
+
+#define RGX_CNTBLK_ID_RAC0           0x0090U	/*!< RAC 1..N RAC */
+#define RGX_CNTBLK_ID_RAC1           0x0091U
+#define RGX_CNTBLK_ID_RAC2           0x0092U
+#define RGX_CNTBLK_ID_RAC3           0x0093U
+#define RGX_CNTBLK_ID_RAC_ALL        0x4090U
+
+#define RGX_CNTBLK_ID_LAST           0x0094U	/*!< End of RAC block */
+
+/*! Masks for the counter block ID*/
+#define RGX_CNTBLK_ID_UNIT_MASK      (0x000FU)	/*!< Unit within group */
+#define RGX_CNTBLK_ID_GROUP_MASK     (0x00F0U)	/*!< Group value */
+#define RGX_CNTBLK_ID_GROUP_SHIFT    (4U)
+#define RGX_CNTBLK_ID_MC_GPU_MASK    (0x0F00U)	/*!< GPU ID for MC use */
+#define RGX_CNTBLK_ID_MC_GPU_SHIFT   (8U)
+#define RGX_CNTBLK_ID_UNIT_ALL_MASK  (0x4000U)	/*!< Program all units within a group */
+
+static_assert(
+	((RGX_CNTBLK_ID_DIRECT_LAST + ((RGX_CNTBLK_ID_LAST & RGX_CNTBLK_ID_GROUP_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT)) <= RGX_HWPERF_MAX_BVNC_BLOCK_LEN),
+	"RGX_HWPERF_MAX_BVNC_BLOCK_LEN insufficient");
+
+#define RGX_HWPERF_EVENT_MASK_VALUE(e)      (IMG_UINT64_C(1) << (e))
+
+/* When adding new counters here, make sure changes are made to rgxfw_hwperf_fwblk_valid() as well */
+#define RGX_CUSTOM_FW_CNTRS	\
+                X(TA_LOCAL_FL_SIZE,    0x0, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(TA_GLOBAL_FL_SIZE,   0x1, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(3D_LOCAL_FL_SIZE,    0x2, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(3D_GLOBAL_FL_SIZE,   0x3, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(ISP_TILES_IN_FLIGHT, 0x4, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DSPMKICK))
+
+/*! Counter IDs for the firmware held statistics */
+typedef enum
+{
+#define X(ctr, id, allow_mask)	RGX_CUSTOM_FW_CNTR_##ctr = id,
+	RGX_CUSTOM_FW_CNTRS
+#undef X
+
+	/* always the last entry in the list */
+	RGX_CUSTOM_FW_CNTR_LAST
+} RGX_HWPERF_CUSTOM_FW_CNTR_ID;
+
+/*! Identifier for each counter in a performance counting module */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_COUNTER_ID;
+
+/* sets all the bits from bit _b1 to _b2, in a IMG_UINT64 type */
+#define MASK_RANGE_IMPL(b1, b2)	((IMG_UINT64)((IMG_UINT64_C(1) << ((IMG_UINT32)(b2)-(IMG_UINT32)(b1) + 1U)) - 1U) << (IMG_UINT32)b1)
+#define MASK_RANGE(R)			MASK_RANGE_IMPL(R##_FIRST_TYPE, R##_LAST_TYPE)
+#define RGX_HWPERF_HOST_EVENT_MASK_VALUE(e) (IMG_UINT32_C(1) << (e))
+
+/*! Mask macros for use with RGXCtrlHWPerf() API.
+ */
+#define RGX_HWPERF_EVENT_MASK_NONE          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_HWPERF_EVENT_MASK_DEFAULT       RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_FWACT) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG)
+#define RGX_HWPERF_EVENT_MASK_ALL           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+
+/*! HWPerf Firmware event masks
+ * @par
+ * All FW Start/End/Debug (SED) events. */
+#define RGX_HWPERF_EVENT_MASK_FW_SED    (MASK_RANGE(RGX_HWPERF_FW_EVENT_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_FW_UFO    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO))
+#define RGX_HWPERF_EVENT_MASK_FW_CSW    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_START) |\
+                                          RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_FINISHED))
+/*! All FW events. */
+#define RGX_HWPERF_EVENT_MASK_ALL_FW    (RGX_HWPERF_EVENT_MASK_FW_SED |\
+                                          RGX_HWPERF_EVENT_MASK_FW_UFO |\
+                                          RGX_HWPERF_EVENT_MASK_FW_CSW)
+
+/*! HW Periodic events (1ms interval). */
+#define RGX_HWPERF_EVENT_MASK_HW_PERIODIC   (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PERIODIC))
+/*! All HW Kick/Finish events. */
+#define RGX_HWPERF_EVENT_MASK_HW_KICKFINISH ((MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE0) |\
+                                               MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE1)) &\
+                                              ~(RGX_HWPERF_EVENT_MASK_HW_PERIODIC))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_HW        (RGX_HWPERF_EVENT_MASK_HW_KICKFINISH |\
+                                              RGX_HWPERF_EVENT_MASK_HW_PERIODIC)
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR_EST   (MASK_RANGE(RGX_HWPERF_PWR_EST_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR       (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_GPU_STATE_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG))
+
+/*! HWPerf Host event masks
+ */
+#define RGX_HWPERF_EVENT_MASK_HOST_WORK_ENQ  (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_ENQ))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_UFO   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_UFO))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_PWR   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_CLK_SYNC))
+
+
+/*! Type used in the RGX API RGXConfigHWPerfCounters() */
+typedef struct
+{
+	/*! Reserved for future use */
+	IMG_UINT32 ui32Reserved;
+
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of configured counters within this block */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Counter register values */
+	IMG_UINT16 ui16Counters[RGX_CNTBLK_COUNTERS_MAX];
+
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_CNTBLK);
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf_table.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf_table.h
new file mode 100644
index 000000000000..9364d500e011
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_hwperf_table.h
@@ -0,0 +1,513 @@
+/*************************************************************************/ /*!
+@File
+@Title          HWPerf counter table header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_HWPERF_TABLE_H
+#define RGX_HWPERF_TABLE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+
+/*****************************************************************************/
+
+/* Forward declaration */
+typedef struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_ RGXFW_HWPERF_CNTBLK_TYPE_MODEL;
+
+/* Function pointer type for functions to check dynamic power state of
+ * counter block instance. Used only in firmware. */
+typedef IMG_BOOL (*PFN_RGXFW_HWPERF_CNTBLK_POWERED)(
+		RGX_HWPERF_CNTBLK_ID eBlkType,
+		IMG_UINT8 ui8UnitId);
+
+/* Counter block run-time info */
+typedef struct
+{
+	IMG_UINT32 uiNumUnits;             /* Number of instances of this block type in the core */
+} RGX_HWPERF_CNTBLK_RT_INFO;
+
+/* Function pointer type for functions to check block is valid and present
+ * on that RGX Device at runtime. It may have compile logic or run-time
+ * logic depending on where the code executes: server, srvinit or firmware.
+ * Values in the psRtInfo output parameter are only valid if true returned.
+ */
+typedef IMG_BOOL (*PFN_RGXFW_HWPERF_CNTBLK_PRESENT)(
+		const struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_* psBlkTypeDesc,
+		const void *pvDev_km,
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo);
+
+/* This structure encodes properties of a type of performance counter block.
+ * The structure is sometimes referred to as a block type descriptor. These
+ * properties contained in this structure represent the columns in the block
+ * type model table variable below. These values vary depending on the build
+ * BVNC and core type.
+ * Each direct block has a unique type descriptor and each indirect group has
+ * a type descriptor.
+ */
+struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_
+{
+	IMG_UINT32 uiCntBlkIdBase;         /* The starting block id for this block type */
+	IMG_UINT32 uiIndirectReg;          /* 0 if direct type otherwise the indirect register value to select indirect unit */
+	IMG_UINT32 uiNumUnits;             /* Number of instances of this block type in the core (compile time use) */
+	const IMG_CHAR *pszBlockNameComment;             /* Name of the PERF register. Used while dumping the perf counters to pdumps */
+	PFN_RGXFW_HWPERF_CNTBLK_POWERED pfnIsBlkPowered; /* A function to determine dynamic power state for the block type */
+	PFN_RGXFW_HWPERF_CNTBLK_PRESENT pfnIsBlkPresent; /* A function to determine presence on RGX Device at run-time */
+	IMG_UINT16 *pszBlkCfgValid;        /* Array of supported counters per block type */
+};
+
+/*****************************************************************************/
+
+/* Shared compile-time context ASSERT macro */
+#if defined(RGX_FIRMWARE)
+/*  firmware context */
+#	define DBG_ASSERT(_c) RGXFW_ASSERT((_c))
+#else
+/*  host client/server context */
+#	define DBG_ASSERT(_c) PVR_ASSERT((_c))
+#endif
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered()
+
+ Referenced in gasCntBlkTypeModel[] table below and only called from
+ RGX_FIRMWARE run-time context. Therefore compile time configuration is used.
+ *****************************************************************************/
+
+#if defined(RGX_FIRMWARE) && defined(RGX_FEATURE_PERFBUS)
+#	include "rgxfw_pow.h"
+#	include "rgxfw_utils.h"
+
+static inline IMG_BOOL rgxfw_hwperf_pow_st_direct(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId);
+static inline IMG_BOOL rgxfw_hwperf_pow_st_direct(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+
+	switch (eBlkType)
+	{
+		case RGX_CNTBLK_ID_JONES:
+		case RGX_CNTBLK_ID_SLC:
+		case RGX_CNTBLK_ID_SLCBANK0:
+		case RGX_CNTBLK_ID_FBCDC:
+		case RGX_CNTBLK_ID_FW_CUSTOM:
+			return IMG_TRUE;
+
+#if !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE)
+		case RGX_CNTBLK_ID_SLCBANK1:
+			if (RGX_FEATURE_NUM_MEMBUS > 1U)
+			{
+				return IMG_TRUE;
+			}
+			else
+			{
+				return IMG_FALSE;
+			}
+
+		case RGX_CNTBLK_ID_SLCBANK2:
+		case RGX_CNTBLK_ID_SLCBANK3:
+			if (RGX_FEATURE_NUM_MEMBUS > 2U)
+			{
+				return IMG_TRUE;
+			}
+			else
+			{
+				return IMG_FALSE;
+			}
+#endif /* !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE) */
+
+		default:
+			return IMG_FALSE;
+	}
+}
+
+/* Only use conditional compilation when counter blocks appear in different
+ * islands for different Rogue families.
+ */
+static inline IMG_BOOL rgxfw_hwperf_pow_st_indirect(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId);
+static inline IMG_BOOL rgxfw_hwperf_pow_st_indirect(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+
+	IMG_UINT32 ui32NumDustsEnabled = rgxfw_pow_get_enabled_units();
+
+	// We don't have any Dusts Enabled until first DC opens the GPU. This makes
+	// setting the PDump HWPerf trace buffers very difficult.
+	// To work around this we special-case some of the 'have to be there'
+	// indirect registers (e.g., TPU0)
+
+	switch (eBlkType)
+	{
+		case RGX_CNTBLK_ID_TPU0:
+			return IMG_TRUE;
+			/*NOTREACHED*/
+			break;
+		default:
+			if (rgxfw_pow_is_rd_on() && (ui32NumDustsEnabled > 0U))
+			{
+				return IMG_TRUE;
+			}
+			else
+			{
+				return IMG_FALSE;
+			}
+			/*NOTREACHED*/
+			break;
+	}
+	return IMG_TRUE;
+}
+
+#else /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+# define rgxfw_hwperf_pow_st_direct   ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+# define rgxfw_hwperf_pow_st_indirect ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+
+#endif /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered() end
+ *****************************************************************************/
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() start
+
+ Referenced in gasCntBlkTypeModel[] table below and called from all build
+ contexts:
+ RGX_FIRMWARE, PVRSRVCTL (UM) and PVRSRVKM (Server).
+
+ Therefore each function has two implementations, one for compile time and one
+ run time configuration depending on the context. The functions will inform the
+ caller whether this block is valid for this particular RGX device. Other
+ run-time dependent data is returned in psRtInfo for the caller to use.
+ *****************************************************************************/
+
+
+/* Used for all block types: Direct and Indirect */
+static inline IMG_BOOL rgx_hwperf_blk_present(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo)
+{
+#if defined(__KERNEL__)	/* Server context -- Run-time Only */
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)pvDev_km;
+	PVRSRV_DEVICE_NODE *psNode;
+	IMG_UINT32	ui32MaxTPUPerSPU;
+	IMG_UINT32	ui32NumMemBus;
+	IMG_UINT32	ui32RTArchVal;
+
+	DBG_ASSERT(psDevInfo != NULL);
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT(psRtInfo != NULL);
+
+	if (((psDevInfo == NULL) || (psBlkTypeDesc == NULL)) || (psRtInfo == NULL))
+	{
+		return IMG_FALSE;
+	}
+
+	psNode = psDevInfo->psDeviceNode;
+	DBG_ASSERT(psNode != NULL);
+
+	if (psNode == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	ui32MaxTPUPerSPU =
+		PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, MAX_TPU_PER_SPU);
+
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psNode, CATURIX_TOP_INFRASTRUCTURE))
+	{
+		ui32NumMemBus = 1U;
+	}
+	else
+	{
+		ui32NumMemBus =
+			PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_MEMBUS);
+	}
+
+	ui32RTArchVal =
+		PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, RAY_TRACING_ARCH);
+
+	switch (psBlkTypeDesc->uiCntBlkIdBase)
+	{
+		case RGX_CNTBLK_ID_JONES:
+		case RGX_CNTBLK_ID_SLC:
+		case RGX_CNTBLK_ID_SLCBANK0:
+		case RGX_CNTBLK_ID_FBCDC:
+		case RGX_CNTBLK_ID_FW_CUSTOM:
+			psRtInfo->uiNumUnits = 1;
+			break;
+
+#if !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE)
+		case RGX_CNTBLK_ID_SLCBANK1:
+			if (ui32NumMemBus >= 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		case RGX_CNTBLK_ID_SLCBANK2:
+		case RGX_CNTBLK_ID_SLCBANK3:
+			if (ui32NumMemBus > 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+#endif /* !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE) */
+
+		case RGX_CNTBLK_ID_TPU0:
+		case RGX_CNTBLK_ID_SWIFT0:
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			psRtInfo->uiNumUnits *= ui32MaxTPUPerSPU;
+			break;
+
+		case RGX_CNTBLK_ID_TEXAS0:
+		case RGX_CNTBLK_ID_PBE_SHARED0:
+
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			break;
+
+		case RGX_CNTBLK_ID_RAC0:
+			if (ui32RTArchVal > 2U)
+			{
+				psRtInfo->uiNumUnits =
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU0_RAC_PRESENT) +
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU1_RAC_PRESENT) +
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU2_RAC_PRESENT) +
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, SPU3_RAC_PRESENT);
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		case RGX_CNTBLK_ID_USC0:
+		case RGX_CNTBLK_ID_MERCER0:
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_CLUSTERS);
+			break;
+
+		case RGX_CNTBLK_ID_PBE0:
+
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, PBE_PER_SPU);
+			psRtInfo->uiNumUnits *=
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			break;
+
+		case RGX_CNTBLK_ID_ISP0:
+
+			psRtInfo->uiNumUnits =
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_ISP_PER_SPU);
+			/* Adjust by NUM_SPU */
+
+			psRtInfo->uiNumUnits *=
+				PVRSRV_GET_DEVICE_FEATURE_VALUE(psNode, NUM_SPU);
+			break;
+
+		default:
+			return IMG_FALSE;
+	}
+	/* Verify that we have at least one unit present */
+	if (psRtInfo->uiNumUnits > 0U)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#else	/* FW context -- Compile-time only */
+	IMG_UINT32	ui32NumMemBus;
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+
+	if (unlikely(psBlkTypeDesc == NULL))
+	{
+		return IMG_FALSE;
+	}
+
+#if !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE)
+	ui32NumMemBus = RGX_FEATURE_NUM_MEMBUS;
+#else
+	ui32NumMemBus = 1U;
+#endif /* !defined(RGX_FEATURE_CATURIX_TOP_INFRASTRUCTURE) */
+
+	switch (psBlkTypeDesc->uiCntBlkIdBase)
+	{
+		/* Handle the dynamic-sized SLC blocks which are only present if
+		 * RGX_FEATURE_NUM_MEMBUS is appropriately set.
+		 */
+		case RGX_CNTBLK_ID_SLCBANK1:
+			if (ui32NumMemBus >= 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		case RGX_CNTBLK_ID_SLCBANK2:
+		case RGX_CNTBLK_ID_SLCBANK3:
+			if (ui32NumMemBus > 2U)
+			{
+				psRtInfo->uiNumUnits = 1;
+			}
+			else
+			{
+				psRtInfo->uiNumUnits = 0;
+			}
+			break;
+
+		default:
+			psRtInfo->uiNumUnits = psBlkTypeDesc->uiNumUnits;
+			break;
+	}
+	if (psRtInfo->uiNumUnits > 0U)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#endif	/* defined(__KERNEL__) */
+}
+
+#if !defined(__KERNEL__) /* Firmware or User-mode context */
+
+/* Used to instantiate a null row in the block type model table below where the
+ * block is not supported for a given build BVNC in firmware/user mode context.
+ * This is needed as the blockid to block type lookup uses the table as well
+ * and clients may try to access blocks not in the hardware. */
+#define RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(_blkid) X(_blkid, 0, 0, #_blkid, NULL, NULL, NULL)
+
+#endif
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() end
+ *****************************************************************************/
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] table
+
+ This table holds the entries for the performance counter block type model.
+ Where the block is not present on an RGX device in question the
+ pfnIsBlkPresent() returns false, if valid and present it returns true.
+ Columns in the table with a ** indicate the value is a default and the value
+ returned in RGX_HWPERF_CNTBLK_RT_INFO when calling pfnIsBlkPresent()should
+ be used at runtime by the caller. These columns are only valid for compile
+ time BVNC configured contexts.
+
+ Order of table rows must match order of counter block IDs in the enumeration
+ RGX_HWPERF_CNTBLK_ID.
+
+ Table contains Xmacro styled entries. Each includer of this file must define
+ a gasCntBlkTypeModel[] structure which is local to itself. Only the layout is
+ defined here.
+
+ uiCntBlkIdBase      : Block-ID
+ uiIndirectReg       : 0 => Direct, non-zero => INDIRECT register address
+ uiNumUnits          : Number of units present on the GPU
+ pszBlockNameComment : Name of the Performance Block
+ pfnIsBlkPowered     : Function to determine power state of block
+ pfnIsBlkPresent     : Function to determine block presence on the core
+ pszBlkCfgValid      : Array of counters valid within this block type
+ *****************************************************************************/
+
+	// Furian 8XT V2 layout:
+
+	/*   uiCntBlkIdBase,  uiIndirectReg,   uiNumUnits**,   pszBlockNameComment, pfnIsBlkPowered,    pfnIsBlkPresent */
+
+	/* RGX_CNTBLK_ID_JONES */
+#if defined(RGX_FIRMWARE) || defined(__KERNEL__)
+
+/* Furian 8XT Direct Performance counter blocks */
+
+#define	RGX_CNT_BLK_TYPE_MODEL_DIRECT_LIST \
+	/* uiCntBlkIdBase,  uiIndirectReg,  uiNumUnits**, pszBlockNameComment, pfnIsBlkPowered, pfnIsBlkPresent */ \
+X(RGX_CNTBLK_ID_JONES, 0, 1, "PERF_BLK_JONES", rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiJONES), \
+X(RGX_CNTBLK_ID_SLC,   0, 1, "PERF_BLK_SLC",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC), \
+X(RGX_CNTBLK_ID_FBCDC, 0, 1, "PERF_BLK_FBCDC", rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiFBCDC), \
+X(RGX_CNTBLK_ID_FW_CUSTOM, 0, 1, "PERF_BLK_FW_CUSTOM", rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiFWCUSTOM), \
+X(RGX_CNTBLK_ID_SLCBANK0,   0, 1, "PERF_BLK_SLC0",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC0), \
+X(RGX_CNTBLK_ID_SLCBANK1,   0, 1, "PERF_BLK_SLC1",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC1), \
+X(RGX_CNTBLK_ID_SLCBANK2,   0, 1, "PERF_BLK_SLC2",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC2), \
+X(RGX_CNTBLK_ID_SLCBANK3,   0, 1, "PERF_BLK_SLC3",   rgxfw_hwperf_pow_st_direct, rgx_hwperf_blk_present, g_auiSLC3)
+
+/* Furian 8XT Indirect Performance counter blocks */
+
+#if !defined(RGX_CR_RAC_INDIRECT)
+#define RGX_CR_RAC_INDIRECT (0x8398U)
+#endif
+
+#define RGX_CNT_BLK_TYPE_MODEL_INDIRECT_LIST \
+	/* uiCntBlkIdBase,  uiIndirectReg,  uiNumUnits**, pszBlockNameComment, pfnIsBlkPowered, pfnIsBlkPresent */ \
+X(RGX_CNTBLK_ID_ISP0, RGX_CR_ISP_INDIRECT, RGX_HWPERF_NUM_SPU * RGX_HWPERF_NUM_ISP_PER_SPU, "PERF_BLK_ISP", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiISP), \
+X(RGX_CNTBLK_ID_MERCER0, RGX_CR_MERCER_INDIRECT, RGX_HWPERF_NUM_MERCER, "PERF_BLK_MERCER", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiMERCER), \
+X(RGX_CNTBLK_ID_PBE0, RGX_CR_PBE_INDIRECT, RGX_HWPERF_NUM_PBE, "PERF_BLK_PBE", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiPBE), \
+X(RGX_CNTBLK_ID_PBE_SHARED0, RGX_CR_PBE_SHARED_INDIRECT, RGX_HWPERF_NUM_PBE_SHARED, "PERF_BLK_PBE_SHARED", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiPBE_SHARED), \
+X(RGX_CNTBLK_ID_USC0, RGX_CR_USC_INDIRECT, RGX_HWPERF_NUM_USC, "PERF_BLK_USC", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiUSC), \
+X(RGX_CNTBLK_ID_TPU0, RGX_CR_TPU_INDIRECT, RGX_HWPERF_NUM_TPU, "PERF_BLK_TPU", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiTPU), \
+X(RGX_CNTBLK_ID_SWIFT0, RGX_CR_SWIFT_INDIRECT, RGX_HWPERF_NUM_SWIFT, "PERF_BLK_SWIFT", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiSWIFT), \
+X(RGX_CNTBLK_ID_TEXAS0, RGX_CR_TEXAS_INDIRECT, RGX_HWPERF_NUM_TEXAS, "PERF_BLK_TEXAS", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiTEXAS), \
+X(RGX_CNTBLK_ID_RAC0, RGX_CR_RAC_INDIRECT, RGX_HWPERF_NUM_RAC, "PERF_BLK_RAC", rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present, g_auiRAC)
+
+#else /* !defined(RGX_FIRMWARE) && !defined(__KERNEL__) */
+
+#error "RGX_FIRMWARE or __KERNEL__ *MUST* be defined"
+
+#endif	/* defined(RGX_FIRMWARE) || defined(__KERNEL__) */
+
+#endif /* RGX_HWPERF_TABLE_H */
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_options.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_options.h
new file mode 100644
index 000000000000..3f489fc7fdee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgx_options.h
@@ -0,0 +1,350 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX build options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Each build option listed here is packed into a dword which provides up to
+ *  log2(RGX_BUILD_OPTIONS_MASK_KM + 1) flags for KM and
+ *  (32 - log2(RGX_BUILD_OPTIONS_MASK_KM + 1)) flags for UM.
+ * The corresponding bit is set if the build option was enabled at compile
+ * time.
+ *
+ * IMPORTANT: add new options to unused bits or define a new dword
+ * (e.g. RGX_BUILD_OPTIONS_KM2 or RGX_BUILD_OPTIONS2) so that the bitfield
+ * remains backwards compatible.
+ */
+
+#ifndef RGX_OPTIONS_H
+#define RGX_OPTIONS_H
+
+#define OPTIONS_OPEN_SOURCE_EN					(0x1UL << 0)
+#define OPTIONS_PDUMP_EN						(0x1UL << 1)
+#define OPTIONS_SECURE_CONTEXT_SWITCH_EN		(0x1UL << 2)
+#define OPTIONS_SECURE_ALLOC_KM_EN				(0x1UL << 3)
+#define OPTIONS_RGX_EN							(0x1UL << 4)
+#define OPTIONS_SECURE_EXPORT_EN				(0x1UL << 5)
+#define OPTIONS_INSECURE_EXPORT_EN				(0x1UL << 6)
+#define OPTIONS_VFP_EN							(0x1UL << 7)
+#define OPTIONS_WORKLOAD_ESTIMATION_EN			(0x1UL << 8)
+#define OPTIONS_PDVFS_EN						(0x1UL << 9)
+#define OPTIONS_DEBUG_EN						(0x1UL << 10)
+#define OPTIONS_BUFFER_SYNC_EN					(0x1UL << 11)
+#define OPTIONS_AUTOVZ_EN						(0x1UL << 12)
+#define OPTIONS_AUTOVZ_HW_REGS_EN				(0x1UL << 13)
+#define OPTIONS_UNUSED2_EN						(0x1UL << 14)
+#define OPTIONS_VALIDATION_EN					(0x1UL << 15)
+#define OPTIONS_NO_HARDWARE_EN					(0x1UL << 16)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN                  (0x1UL << 17)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_MASK                      (0x1fUL << 18)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT                     (18)
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_PERCONTEXT_FREELIST_EN			(0x1UL << 31)
+
+#define RGX_BUILD_OPTIONS_MASK_KM				  \
+			(OPTIONS_OPEN_SOURCE_EN				| \
+			 OPTIONS_PDUMP_EN					| \
+			 OPTIONS_SECURE_CONTEXT_SWITCH_EN	| \
+			 OPTIONS_SECURE_ALLOC_KM_EN			| \
+			 OPTIONS_RGX_EN						| \
+			 OPTIONS_SECURE_EXPORT_EN			| \
+			 OPTIONS_INSECURE_EXPORT_EN			| \
+			 OPTIONS_VFP_EN						| \
+			 OPTIONS_WORKLOAD_ESTIMATION_EN		| \
+			 OPTIONS_PDVFS_EN					| \
+			 OPTIONS_DEBUG_EN					| \
+			 OPTIONS_BUFFER_SYNC_EN				| \
+			 OPTIONS_AUTOVZ_EN					| \
+			 OPTIONS_AUTOVZ_HW_REGS_EN			| \
+			 OPTIONS_VALIDATION_EN				| \
+			 OPTIONS_NO_HARDWARE_EN				| \
+			 OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN)
+
+#define RGX_BUILD_OPTIONS_MASK_FW \
+			(RGX_BUILD_OPTIONS_MASK_KM & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_MASK & \
+			 ~OPTIONS_BUFFER_SYNC_EN)
+
+/* Build options that the FW must have if the present on the KM */
+#define FW_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the UM must have if the present on the KM */
+#define UM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the KM must have if the present on the UM */
+#define KM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN					| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_BUFFER_SYNC_EN))
+
+#define OPEN_SOURCE_OPTION	"OPEN_SOURCE_DRIVER  "
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	#define OPTIONS_BIT0		OPTIONS_OPEN_SOURCE_EN
+	#if OPTIONS_BIT0 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT0		0x0UL
+#endif /* SUPPORT_OPEN_SOURCE_DRIVER */
+
+#define PDUMP_OPTION	"PDUMP  "
+#if defined(PDUMP)
+	#define OPTIONS_BIT1		OPTIONS_PDUMP_EN
+	#if OPTIONS_BIT1 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT1		0x0UL
+#endif /* PDUMP */
+
+#define SECURE_CONTEXT_SWITCH_OPTION "SECURE_CONTEXT_SWITCH "
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	#define OPTIONS_BIT2		OPTIONS_SECURE_CONTEXT_SWITCH_EN
+	#if OPTIONS_BIT2 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT2		0x0UL
+#endif /* SUPPORT_SECURE_CONTEXT_SWITCH */
+
+#define SECURE_ALLOC_KM_OPTION "SECURE_ALLOC_KM  "
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	#define OPTIONS_BIT3		OPTIONS_SECURE_ALLOC_KM_EN
+	#if OPTIONS_BIT3 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT3 0x0UL
+#endif /* SUPPORT_SECURE_ALLOC_KM */
+
+#define RGX_OPTION	" "
+#if defined(SUPPORT_RGX)
+	#define OPTIONS_BIT4		OPTIONS_RGX_EN
+	#if OPTIONS_BIT4 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT4		0x0UL
+#endif /* SUPPORT_RGX */
+
+#define SECURE_EXPORT_OPTION	"SECURE_EXPORTS  "
+#if defined(SUPPORT_SECURE_EXPORT)
+	#define OPTIONS_BIT5		OPTIONS_SECURE_EXPORT_EN
+	#if OPTIONS_BIT5 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT5		0x0UL
+#endif /* SUPPORT_SECURE_EXPORT */
+
+#define INSECURE_EXPORT_OPTION	"INSECURE_EXPORTS  "
+#if defined(SUPPORT_INSECURE_EXPORT)
+	#define OPTIONS_BIT6		OPTIONS_INSECURE_EXPORT_EN
+	#if OPTIONS_BIT6 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT6		0x0UL
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+#define VFP_OPTION	"VFP  "
+#if defined(SUPPORT_VFP)
+	#define OPTIONS_BIT7		OPTIONS_VFP_EN
+	#if OPTIONS_BIT7 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT7		0x0UL
+#endif /* SUPPORT_VFP */
+
+#define WORKLOAD_ESTIMATION_OPTION	"WORKLOAD_ESTIMATION  "
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	#define OPTIONS_BIT8		OPTIONS_WORKLOAD_ESTIMATION_EN
+	#if OPTIONS_BIT8 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT8		0x0UL
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+
+#define PDVFS_OPTION	"PDVFS  "
+#if defined(SUPPORT_PDVFS)
+	#define OPTIONS_BIT9		OPTIONS_PDVFS_EN
+	#if OPTIONS_BIT9 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT9		0x0UL
+#endif /* SUPPORT_PDVFS */
+
+#define DEBUG_OPTION	"DEBUG  "
+#if defined(DEBUG)
+	#define OPTIONS_BIT10		OPTIONS_DEBUG_EN
+	#if OPTIONS_BIT10 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT10		0x0UL
+#endif /* DEBUG */
+
+#define BUFFER_SYNC_OPTION	"BUFFER_SYNC  "
+#if defined(SUPPORT_BUFFER_SYNC)
+	#define OPTIONS_BIT11		OPTIONS_BUFFER_SYNC_EN
+	#if OPTIONS_BIT11 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT11		0x0UL
+#endif /* SUPPORT_BUFFER_SYNC */
+
+#define AUTOVZ_OPTION	"AUTOVZ  "
+#if defined(SUPPORT_AUTOVZ)
+	#define OPTIONS_BIT12		OPTIONS_AUTOVZ_EN
+	#if OPTIONS_BIT12 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT12		0x0UL
+#endif /* SUPPORT_AUTOVZ */
+
+#define AUTOVZ_HW_REGS_OPTION	"AUTOVZ_HW_REGS  "
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+	#define OPTIONS_BIT13		OPTIONS_AUTOVZ_HW_REGS_EN
+	#if OPTIONS_BIT13 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT13		0x0UL
+#endif /* SUPPORT_AUTOVZ_HW_REGS */
+
+/* UNUSED bit, kept for rogue/volcanic compatibility */
+#define INTERNAL_UNUSED2_OPTION	"INTERNAL_UNUSED2  "
+#if defined(INTERNAL_UNUSED2)
+	#define OPTIONS_BIT14		OPTIONS_UNUSED2_EN
+	#if OPTIONS_BIT14 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT14		0x0UL
+#endif
+
+#define VALIDATION_OPTION	"VALIDATION  "
+	#define OPTIONS_BIT15		0x0UL
+
+#define NO_HARDWARE_OPTION	"NO_HARDWARE  "
+#if defined(NO_HARDWARE)
+	#define OPTIONS_BIT16		OPTIONS_NO_HARDWARE_EN
+	#if OPTIONS_BIT16 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT16		0x0UL
+#endif /* NO_HARDWARE */
+
+#define NUM_DRIVERS_SUPPORTED_CHECK_OPTION	"NUM_DRIVERS_SUPPORTED_CHECK  "
+	#define OPTIONS_BIT17		OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN
+	#if OPTIONS_BIT17 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif /* NUM_DRIVERS_SUPPORTED_CHECK */
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_BIT31		OPTIONS_PERCONTEXT_FREELIST_EN
+#if OPTIONS_BIT31 <= RGX_BUILD_OPTIONS_MASK_KM
+#error "Bit exceeds reserved range"
+#endif
+
+#define RGX_BUILD_OPTIONS_KM	\
+	(OPTIONS_BIT0				|\
+	 OPTIONS_BIT1				|\
+	 OPTIONS_BIT2				|\
+	 OPTIONS_BIT3				|\
+	 OPTIONS_BIT4				|\
+	 OPTIONS_BIT5				|\
+	 OPTIONS_BIT6				|\
+	 OPTIONS_BIT7				|\
+	 OPTIONS_BIT8				|\
+	 OPTIONS_BIT9				|\
+	 OPTIONS_BIT10				|\
+	 OPTIONS_BIT11				|\
+	 OPTIONS_BIT12				|\
+	 OPTIONS_BIT13				|\
+	 OPTIONS_BIT14				|\
+	 OPTIONS_BIT15				|\
+	 OPTIONS_BIT16				|\
+	 OPTIONS_BIT17)
+
+#define RGX_BUILD_OPTIONS (RGX_BUILD_OPTIONS_KM | OPTIONS_BIT31)
+
+#define RGX_BUILD_OPTIONS_LIST			\
+	{									\
+		OPEN_SOURCE_OPTION,				\
+		PDUMP_OPTION,					\
+		SECURE_CONTEXT_SWITCH_OPTION,	\
+		SECURE_ALLOC_KM_OPTION,			\
+		RGX_OPTION,						\
+		SECURE_EXPORT_OPTION,			\
+		INSECURE_EXPORT_OPTION,			\
+		VFP_OPTION,						\
+		WORKLOAD_ESTIMATION_OPTION,		\
+		PDVFS_OPTION,					\
+		DEBUG_OPTION,					\
+		BUFFER_SYNC_OPTION,				\
+		AUTOVZ_OPTION,					\
+		AUTOVZ_HW_REGS_OPTION,			\
+		INTERNAL_UNUSED2_OPTION,		\
+		VALIDATION_OPTION,				\
+		NO_HARDWARE_OPTION,				\
+		NUM_DRIVERS_SUPPORTED_CHECK_OPTION				\
+	}
+
+#endif /* RGX_OPTIONS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgxheapconfig.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgxheapconfig.h
new file mode 100644
index 000000000000..00801fcaae84
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/rgxheapconfig.h
@@ -0,0 +1,294 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_H
+#define RGXHEAPCONFIG_H
+
+#include "rgxdefs_km.h"
+
+#define RGX_HEAP_SIZE_32KiB      IMG_UINT64_C(0x0000008000)
+#define RGX_HEAP_SIZE_2MiB       IMG_UINT64_C(0x0000200000)
+#define RGX_HEAP_SIZE_4MiB       IMG_UINT64_C(0x0000400000)
+#define RGX_HEAP_SIZE_16MiB      IMG_UINT64_C(0x0001000000)
+#define RGX_HEAP_SIZE_32MiB      IMG_UINT64_C(0x0002000000)
+#define RGX_HEAP_SIZE_256MiB     IMG_UINT64_C(0x0010000000)
+
+#define RGX_HEAP_SIZE_1GiB       IMG_UINT64_C(0x0040000000)
+#define RGX_HEAP_SIZE_2GiB       IMG_UINT64_C(0x0080000000)
+#define RGX_HEAP_SIZE_4GiB       IMG_UINT64_C(0x0100000000)
+#define RGX_HEAP_SIZE_16GiB      IMG_UINT64_C(0x0400000000)
+#define RGX_HEAP_SIZE_32GiB      IMG_UINT64_C(0x0800000000)
+#define RGX_HEAP_SIZE_64GiB      IMG_UINT64_C(0x1000000000)
+#define RGX_HEAP_SIZE_128GiB     IMG_UINT64_C(0x2000000000)
+#define RGX_HEAP_SIZE_256GiB     IMG_UINT64_C(0x4000000000)
+#define RGX_HEAP_SIZE_512GiB     IMG_UINT64_C(0x8000000000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address heaps that are used in
+	application memory contexts. It also shows where the Firmware memory heap
+	fits into this, but the firmware heap is only ever created in the
+	Services KM/server component.
+
+	RGX_PDSCODEDATA_HEAP_BASE and RGX_USCCODE_HEAP_BASE will be programmed,
+	on a global basis, into RGX_CR_PDS_EXEC_BASE and RGX_CR_USC_CODE_BASE_*
+	respectively. Therefore if clients use multiple configs they must still
+	be consistent with their definitions for these heaps.
+
+	Shared virtual memory (GENERAL_SVM) support requires half of the address
+	space (512 GiB) be reserved for SVM allocations to mirror application CPU
+	addresses.
+
+	The GENERAL non-SVM region is 512 GiB to 768 GiB and is shared between the
+	general (4KiB) heap and the general non-4K heap. The first 128 GiB is used
+	for the GENERAL_HEAP (4KiB) and the last 32 GiB is used for the
+	GENERAL_NON4K_HEAP. This heap has a default page-size of 16K.
+	AppHint PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE can be used to forced it
+	to these values: 4K,64K,256K,1M,2M.
+
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range. Often this is the reserved
+	size of the heap within the range.
+*/
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0000_0000 - 0x00_0020_0000 **/
+	/* 0 MiB to 2 MiB, size of 2 MiB : RESERVED (only when General SVM
+	 *                                           doesn't exist) **/
+
+/* 0x00_0000_8000 - 0x7F_FFFF_8000 **/
+	/* MAX(32 KiB, PAGE_SIZE) to 512 GiB, size of 512 GiB less MAX(32 KiB, PAGE_SIZE) : GENERAL_SVM_HEAP **/
+
+	/* The MAX is determined at runtime (PAGE_SIZE isn't available on all platforms)
+	 * so the #define's must NOT be used directly. Use the heap config after initialisation. */
+	#define RGX_GENERAL_SVM_HEAP_BASE           IMG_UINT64_C(0x0000008000)
+	#define RGX_GENERAL_SVM_HEAP_SIZE           (RGX_HEAP_SIZE_512GiB - RGX_HEAP_SIZE_32KiB)
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0x80_0000_0000 - 0x9F_FFFF_FFFF **/
+	/* 512 GiB to 640 GiB, size of 128 GiB : GENERAL_HEAP **/
+	#define RGX_GENERAL_HEAP_BASE               IMG_UINT64_C(0x8000000000)
+	#define RGX_GENERAL_HEAP_SIZE               RGX_HEAP_SIZE_128GiB
+
+/* 0xA0_0000_0000 - 0xAF_FFFF_FFFF **/
+	/* 640 GiB to 704 GiB, size of 64 GiB : FREE **/
+
+/* 0xB0_0000_0000 - 0xB1_FFFF_FFFF **/
+	/* 704 GiB to 720 GiB, size of 16 GiB : RESERVED ROGUE **/
+
+/* B4_0000_0000 - 0xB7_FFFF_FFFF **/
+	/* 720 GiB to 736 GiB, size of 16 GiB : FREE **/
+
+/* 0xB8_0000_0000 - 0xBF_FFFF_FFFF **/
+	/* 736 GiB to 768 GiB, size of 32 GiB : GENERAL_NON4K_HEAP **/
+	#define RGX_GENERAL_NON4K_HEAP_BASE         IMG_UINT64_C(0xB800000000)
+	#define RGX_GENERAL_NON4K_HEAP_SIZE         RGX_HEAP_SIZE_32GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xC0_0000_0000 - 0xD9_FFFF_FFFF **/
+	/* 768 GiB to 872 GiB, size of 104 GiB : FREE **/
+
+/* 0xDA_0000_0000 - 0xDA_FFFF_FFFF **/
+	/* 872 GiB to 876 GiB, size of 4 GiB : PDSCODEDATA_HEAP **/
+	#define RGX_PDSCODEDATA_HEAP_BASE           IMG_UINT64_C(0xDA00000000)
+	#define RGX_PDSCODEDATA_HEAP_SIZE           RGX_HEAP_SIZE_4GiB
+
+/* 0xDB_0000_0000 - 0xDC_FFFF_FFFF **/
+	/* 876 GiB to 884 GiB, size of 8 GiB : RESERVED ROGUE **/
+
+/* 0xDD_0000_0000 - 0xDF_FFFF_FFFF **/
+	/* 884 GiB to 896 GiB, size of 12 GiB : FREE **/
+
+	/*
+	 * The breakpoint handler code heap must share the same 4GiB address
+	 * range as the USC shader code heap. The address space split is
+	 * configurable.
+	 *
+	 * The breakpoint handler register is fixed, so the following parts
+	 * of the BP handler address are static:
+	 * [31:24] = 0xFE for 4064MiB offset after the USC code heap.
+	 *   [5:0] = 0x00 i.e. aligned to 64 Byte boundary.
+	 *
+	 * The remaining part of the BP handler is dynamic and encoded in
+	 * the USC instruction BABP_target_addr. The BP handler thus
+	 * allows a range of 16 MiB with granularity of 64 Bytes.
+	 */
+
+/* 0xE0_0000_0000 - 0xE0_FDFF_FFFF **/
+	/* 896 GiB to 900 GiB, size of 4 GiB less 32 MiB : USCCODE_HEAP **/
+	#define RGX_USCCODE_HEAP_BASE               IMG_UINT64_C(0xE000000000)
+	#define RGX_USCCODE_HEAP_SIZE               (RGX_HEAP_SIZE_4GiB - RGX_HEAP_SIZE_32MiB)
+
+/* 0xE0_FE00_0000 - 0xE0_FEFF_FFFF **/
+	/* 900 GiB less 32 MiB to 900 GiB less 16 MiB, size of 16 MiB : USCCODE_BPH_HEAP **/
+	#define RGX_USCCODE_BPH_HEAP_BASE           (IMG_UINT64_C(0xE100000000) - RGX_HEAP_SIZE_32MiB)
+	#define RGX_USCCODE_BPH_HEAP_SIZE           RGX_HEAP_SIZE_16MiB
+
+/* 0xE0_FF00_0000 - 0xE0_FFFF_FFFF **/
+	/* 900 GiB less 16 MiB to 900 GiB, size of 16 MiB : RESERVED **/
+
+/* 0xE1_0000_0000 - 0xE1_BFFF_FFFF **/
+	/* 900 GiB to 903 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xE1_C000_0000 - 0xE1_FFFF_FFFF **/
+	/* 903 GiB to 904 GiB, reserved 1 GiB, : FIRMWARE_HEAP **/
+
+	/* Firmware heaps defined in rgx_heap_firmware.h as they are not present in
+	   application memory contexts, see:
+	    RGX_FIRMWARE_RAW_HEAP_BASE
+	    RGX_FIRMWARE_RAW_HEAP_SIZE
+	   See header for other sub-heaps details
+	*/
+
+/* 0xE2_0000_0000 - 0xE2_FFFF_FFFF **/
+	/* 904 GiB to 908 GiB, size of 4GiB : RESERVED ROGUE **/
+
+/* 0xE3_0000_0000 - 0xE3_FFFF_FFFF **/
+	/* 908 GiB to 912 GiB, size of 4 GiB : FREE **/
+
+/* 0xE4_0000_0000 - 0xE7_FFFF_FFFF **/
+	/* 912 GiB to 928 GiB, size 16 GiB : RESERVED_ROGUE **/
+
+/* 0xE8_0000_0000 - 0xE8_FFFF_FFFF **/
+	/* 928 GiB to 932 GiB, size of 4 GiB : FREE **/
+
+/* 0xE9_0000_0000 - 0xE9_3FFF_FFFF **/
+	/* 932 GiB to 933 GiB, size of 1 GiB : VK_CAPT_REPLAY_HEAP **/
+	#define RGX_VK_CAPT_REPLAY_HEAP_BASE    IMG_UINT64_C(0xE900000000)
+	#define RGX_VK_CAPT_REPLAY_HEAP_SIZE    RGX_HEAP_SIZE_1GiB
+
+/* 0xE9_4000_0000 - 0xE9_FFFF_FFFF **/
+	/* 933 GiB to 936 GiB, size of 3 GiB : FREE **/
+
+/* 0xEA_0000_0000 - 0xEA_001F_FFFF **/
+	/* 936 GiB to 937 GiB, size of 1 GiB : FREE **/
+
+/* 0xEA_4000_0000 - 0xEA_FFFF_FFFF **/
+	/* 937 GiB to 940 GiB, size of 3 GiB : FREE **/
+
+/* 0xEB_0000_0000 - 0xEB_FFFF_FFFF **/
+	/* 940 GiB to 944 GiB, size 4 GiB : COMPONENT_CTRL_HEAP **/
+	#define RGX_COMPONENT_CTRL_HEAP_BASE        IMG_UINT64_C(0xEB00000000)
+	#define RGX_COMPONENT_CTRL_HEAP_SIZE        RGX_HEAP_SIZE_4GiB
+
+/* 0xEC_0000_0000 - 0xEC_001F_FFFF **/
+	/* 944 GiB to 945 GiB, size 2 MiB : FBCDC_HEAP **/
+	#define RGX_FBCDC_HEAP_BASE                 IMG_UINT64_C(0xEC00000000)
+	#define RGX_FBCDC_HEAP_SIZE                 RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_4000_0000 - 0xEC_401F_FFFF **/
+	/* 945 GiB to 946 GiB, size 2 MiB : FBCDC_LARGE_HEAP **/
+	#define RGX_FBCDC_LARGE_HEAP_BASE           IMG_UINT64_C(0xEC40000000)
+	#define RGX_FBCDC_LARGE_HEAP_SIZE           RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_8000_0000 - 0xEC_FFFF_FFFF **/
+	/* 946 GiB to 948 GiB, size of 3 GiB : FREE **/
+
+/* 0xED_0000_0000 - 0xED_00FF_FFFF */
+	/* 948 GiB to 949 GiB, size 16 MiB : PDS_INDIRECT_STATE_HEAP */
+	#define RGX_PDS_INDIRECT_STATE_HEAP_BASE    IMG_UINT64_C(0xED00000000)
+	#define RGX_PDS_INDIRECT_STATE_HEAP_SIZE    RGX_HEAP_SIZE_16MiB
+
+/* 0xED_4000_0000 - 0xED_FFFF_FFFF **/
+	/* 949 GiB to 952 GiB, size of 3 GiB : FREE **/
+
+/* 0xEE_0000_0000 - 0xEE_3FFF_FFFF **/
+	/* 952 GiB to 953 GiB, size of 1 GiB : CMP_MISSION_RMW_HEAP **/
+	#define RGX_CMP_MISSION_RMW_HEAP_BASE       IMG_UINT64_C(0xEE00000000)
+	#define RGX_CMP_MISSION_RMW_HEAP_SIZE       RGX_HEAP_SIZE_1GiB
+
+/* 0xEE_4000_0000 - 0xEE_FFFF_FFFF **/
+	/* 953 GiB to 956 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xEF_0000_0000 - 0xEF_3FFF_FFFF **/
+	/* 956 GiB to 957 GiB, size of 1 GiB : CMP_SAFETY_RMW_HEAP **/
+	#define RGX_CMP_SAFETY_RMW_HEAP_BASE        IMG_UINT64_C(0xEF00000000)
+	#define RGX_CMP_SAFETY_RMW_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xEF_4000_0000 - 0xEF_FFFF_FFFF **/
+	/* 957 GiB to 960 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xF0_0000_0000 - 0xF0_FFFF_FFFF **/
+	/* 960 GiB to 964 GiB, size of 4 GiB : TEXTURE_STATE_HEAP (36-bit aligned) */
+	#define RGX_TEXTURE_STATE_HEAP_BASE         IMG_UINT64_C(0xF000000000)
+	#define RGX_TEXTURE_STATE_HEAP_SIZE         RGX_HEAP_SIZE_4GiB
+
+/* 0xF1_0000_0000 - 0xF1_FFFF_FFFF **/
+	/* 964 GiB to 968 GiB, size of 4 GiB : FREE **/
+
+/* 0xF2_0000_0000 - 0xF2_001F_FFFF **/
+	/* 968 GiB to 969 GiB, size of 2 MiB : VISIBILITY_TEST_HEAP **/
+	#define RGX_VISIBILITY_TEST_HEAP_BASE       IMG_UINT64_C(0xF200000000)
+	#define RGX_VISIBILITY_TEST_HEAP_SIZE       RGX_HEAP_SIZE_2MiB
+
+/* 0xF2_4000_0000 - 0xF2_FFFF_FFFF **/
+	/* 969 GiB to 972 GiB, size of 3 GiB : FREE **/
+
+/* 0xF3_0000_0000 - 0xF7_FFFF_FFFF **/
+	/* 972 GiB to 992 GiB, size of 20 GiB : FREE **/
+
+/* 0xF8_0000_0000 - 0xF9_FFFF_FFFF **/
+	/* 992 GiB to 1000 GiB, size 8 GiB : RESERVED ROGUE **/
+
+/* 0xFA_0000_0000 - 0xFF_FFFF_FFFF **/
+	/* 1000 GiB to 1024 GiB, size of 24 GiB : FREE **/
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/include/volcanic/system/rgx_tc/tc_clocks.h b/drivers/gpu/drm/img/img-volcanic/include/volcanic/system/rgx_tc/tc_clocks.h
new file mode 100644
index 000000000000..9463830fa1c4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/include/volcanic/system/rgx_tc/tc_clocks.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TC_CLOCKS_H)
+#define TC_CLOCKS_H
+
+/*
+ * The core clock speed is passed through a multiplier depending on the TC
+ * version.
+ *
+ * On TC_ES1: Multiplier = x3, final speed = 270MHz
+ * On TC_ES2: Multiplier = x6, final speed = 540MHz
+ * On TCF5:   Multiplier = 1x final speed = 45MHz
+ *
+ *
+ * The base (unmultiplied speed) can be adjusted using a module parameter
+ * called "sys_core_clk_speed", a number in Hz.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_core_clk_speed=60000000" /etc/init.d/rc.pvr start
+ *
+ * would result in a core speed of 60MHz xMultiplier.
+ *
+ *
+ * The memory clock is unmultiplied and can be adjusted using a module
+ * parameter called "sys_mem_clk_speed", this should be the number in Hz for
+ * the memory clock speed.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=100000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the memory clock speed set to 100MHz.
+ *
+ *
+ * Same applies to the system interface clock speed, "sys_sysif_clk_speed".
+ * Needed for TCF5 but not for TC_ES2/ES1.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_sysif_clk_speed=45000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the system clock speed set to 45MHz.
+ *
+ *
+ * All parameters can be specified at once, e.g.,
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=MEMORY_SPEED sys_core_clk_speed=CORE_SPEED sys_sysif_clk_speed=SYSIF_SPEED" /etc/init.d/rc.pvr start
+ */
+
+#define RGX_TC_SYS_CLOCK_SPEED		(45000000) /*< Unused */
+
+#if defined(TC_ODIN_27_5_254_2)
+ #define RGX_TC_CORE_CLOCK_SPEED	(94000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+ #define RGX_TC_CLOCK_MULTIPLEX    (16)
+#else
+ /* FPGA tcfvuquad with Odin */
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000) /* 3.125MHz */
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000) /* 3.75MHz */
+ #define RGX_TC_CLOCK_MULTIPLEX    (1)
+#endif
+
+#endif /* if !defined(TC_CLOCKS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/common/env/linux/dc_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/common/env/linux/dc_osfuncs.c
new file mode 100644
index 000000000000..3d6ce091d727
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/common/env/linux/dc_osfuncs.c
@@ -0,0 +1,464 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include "pvr_notifier.h"
+#include "srvkm.h"
+#include "dc_osfuncs.h"
+#include <linux/version.h>
+#include <linux/bug.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/workqueue.h>
+#include <asm/page.h>
+
+#if defined(CONFIG_X86)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,2,0))
+#include <asm/fpu/internal.h>
+#else
+#include <asm/i387.h>
+#endif
+#endif
+
+#define MAX_DRVNAME 128
+static IMG_CHAR g_szDrvName[MAX_DRVNAME + 1];
+
+typedef struct WORK_ITEM_TAG
+{
+	struct work_struct	sWork;
+	PFN_WORK_PROCESSOR	pfnProcessor;
+	void			*pvData;
+} WORK_ITEM;
+
+void DC_OSSetDrvName(const IMG_CHAR *pszDrvName)
+{
+	strncpy(g_szDrvName, pszDrvName, MAX_DRVNAME);
+	g_szDrvName[MAX_DRVNAME] = '\0'; /* ensure null term (not guaranteed by strncpy) */
+}
+
+void DC_OSAbort(const IMG_CHAR *pszFile, IMG_UINT32 ui32Line)
+{
+	printk(KERN_EMERG "%s - Assertion failed at %s:%d\n", g_szDrvName, pszFile, ui32Line);
+	BUG();
+}
+
+void DC_OSDebugPrintf(DC_OS_DEBUG_LEVEL eDebugLevel, const IMG_CHAR *pszFormat, ...)
+{
+	va_list args;
+
+	switch (eDebugLevel)
+	{
+		case DBGLVL_FATAL:
+		{
+			printk(KERN_EMERG " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_ALERT:
+		{
+			printk(KERN_ALERT " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_ERROR:
+		{
+			printk(KERN_ERR " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_WARNING:
+		{
+			printk(KERN_WARNING " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_NOTICE:
+		{
+			printk(KERN_NOTICE " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_INFO:
+		{
+			printk(KERN_INFO " %s", g_szDrvName);
+			break;
+		}
+		case DBGLVL_DEBUG:
+		{
+			printk(KERN_DEBUG " %s", g_szDrvName);
+			break;
+		}
+	}
+
+	va_start(args, pszFormat);
+	vprintk(pszFormat, args);
+	va_end(args);
+}
+
+IMG_CHAR *DC_OSStringNCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uiLength)
+{
+	return strncpy(pszDest, pszSrc, uiLength);
+}
+
+IMG_INT DC_OSSNPrintf(IMG_CHAR *pszDest, size_t uiLength, const IMG_CHAR *pszFormat, ...)
+{
+	va_list args;
+	IMG_INT iRetVal;
+
+	va_start(args, pszFormat);
+	iRetVal = vsnprintf(pszDest, uiLength, pszFormat, args);
+	va_end(args);
+
+	return iRetVal;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0))
+IMG_INT64 DC_OSClockns(void)
+{
+	struct timespec64 now;
+
+	ktime_get_raw_ts64(&now);
+	return now.tv_sec * 1000000000 + now.tv_nsec;
+}
+#else
+IMG_INT64 DC_OSClockns(void)
+{
+	struct timespec now;
+	ktime_get_ts(&now);
+	return now.tv_sec * 1000000000ULL + now.tv_nsec;
+}
+#endif
+
+IMG_UINT32 DC_OSGetPageSize(void)
+{
+	return PAGE_SIZE;
+}
+
+IMG_UINT32 DC_OSGetPageShift(void)
+{
+	return PAGE_SHIFT;
+}
+
+IMG_UINT32 DC_OSGetPageMask(void)
+{
+	return (IMG_UINT32)PAGE_MASK;
+}
+
+void *DC_OSAllocMem(size_t uiSize)
+{
+	return kmalloc(uiSize, GFP_KERNEL);
+}
+
+void *DC_OSCallocMem(size_t uiSize)
+{
+	return kzalloc(uiSize, GFP_KERNEL);
+}
+
+void DC_OSFreeMem(void *pvMem)
+{
+	kfree(pvMem);
+}
+
+void DC_OSMemSet(void *pvDest, IMG_UINT8 ui8Value, size_t uiSize)
+{
+	memset(pvDest, ui8Value, uiSize);
+}
+
+IMG_UINT32 DC_OSAddrRangeStart(void *pvDevice, IMG_UINT8 ui8BaseNum)
+{
+	struct device *psDev = pvDevice;
+
+	DC_ASSERT(dev_is_pci(psDev));
+
+	return pci_resource_start(to_pci_dev(psDev), ui8BaseNum);
+}
+
+void *DC_OSRequestAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size, IMG_CHAR *pszRequestorName)
+{
+	return request_mem_region(sCpuPAddr.uiAddr, ui32Size, pszRequestorName);
+}
+
+void DC_OSReleaseAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size)
+{
+	release_mem_region(sCpuPAddr.uiAddr, ui32Size);
+}
+
+IMG_CPU_VIRTADDR DC_OSMapPhysAddr(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size)
+{
+	return ioremap(sCpuPAddr.uiAddr, ui32Size);
+}
+
+void DC_OSUnmapPhysAddr(IMG_CPU_VIRTADDR pvCpuVAddr, IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	iounmap(pvCpuVAddr);
+}
+
+IMG_UINT32 DC_OSReadReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset)
+{
+	return ioread32(pvRegCpuVBase + ui32Offset);
+}
+
+void DC_OSWriteReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	iowrite32(ui32Value, (pvRegCpuVBase + ui32Offset));
+}
+
+IMG_UINT32 DC_OSDiv64(IMG_UINT64 ui64Numerator, IMG_UINT32 ui32Denominator)
+{
+	(void)do_div(ui64Numerator, ui32Denominator);
+
+	return (IMG_UINT32)ui64Numerator;
+}
+
+void DC_OSFloatingPointBegin(void)
+{
+#if defined(CONFIG_X86)
+	kernel_fpu_begin();
+#else
+	DC_OSDebugPrintf(DBGLVL_WARNING, " - %s: Unsupported architecture\n", __func__);
+#endif
+}
+
+void DC_OSFloatingPointEnd(void)
+{
+#if defined(CONFIG_X86)
+	kernel_fpu_end();
+#else
+	DC_OSDebugPrintf(DBGLVL_WARNING, " - %s: Unsupported architecture\n", __func__);
+#endif
+}
+
+PVRSRV_ERROR DC_OSMutexCreate(void **ppvMutex)
+{
+	struct mutex *psLock;
+
+	psLock = DC_OSAllocMem(sizeof(*psLock));
+	if (psLock == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	mutex_init(psLock);
+
+	*ppvMutex = (void *)psLock;
+
+	return PVRSRV_OK;
+}
+
+void DC_OSMutexDestroy(void *pvMutex)
+{
+	DC_OSFreeMem(pvMutex);
+}
+
+void DC_OSMutexLock(void *pvMutex)
+{
+	mutex_lock((struct mutex *)pvMutex);
+}
+
+void DC_OSMutexUnlock(void *pvMutex)
+{
+	mutex_unlock((struct mutex *)pvMutex);
+}
+
+void DC_OSDelayus(IMG_UINT32 ui32Timeus)
+{
+	udelay(ui32Timeus);
+}
+
+PVRSRV_ERROR DC_OSPVRServicesConnectionOpen(IMG_HANDLE *phPVRServicesConnection)
+{
+	*phPVRServicesConnection = (IMG_HANDLE)NULL;
+
+	return PVRSRV_OK;
+}
+
+void DC_OSPVRServicesConnectionClose(IMG_HANDLE hPVRServicesConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRServicesConnection);
+
+	DC_ASSERT(hPVRServicesConnection == (IMG_HANDLE)NULL);
+}
+
+PVRSRV_ERROR DC_OSPVRServicesSetupFuncs(IMG_HANDLE hPVRServicesConnection, DC_SERVICES_FUNCS *psServicesFuncs)
+{
+	DC_ASSERT(hPVRServicesConnection == (IMG_HANDLE)NULL);
+
+	if (psServicesFuncs == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psServicesFuncs->pfnDCRegisterDevice = DCRegisterDevice;
+	psServicesFuncs->pfnDCUnregisterDevice = DCUnregisterDevice;
+	psServicesFuncs->pfnDCDisplayConfigurationRetired = DCDisplayConfigurationRetired;
+	psServicesFuncs->pfnDCImportBufferAcquire = DCImportBufferAcquire;
+	psServicesFuncs->pfnDCImportBufferRelease = DCImportBufferRelease;
+
+	psServicesFuncs->pfnPhysHeapAcquireByUsage = PhysHeapAcquireByUsage;
+	psServicesFuncs->pfnPhysHeapRelease = PhysHeapRelease;
+	psServicesFuncs->pfnPhysHeapGetType = PhysHeapGetType;
+	psServicesFuncs->pfnPhysHeapGetSize = PhysHeapGetSize;
+	psServicesFuncs->pfnPhysHeapGetCpuPAddr = PhysHeapGetCpuPAddr;
+	psServicesFuncs->pfnPhysHeapCpuPAddrToDevPAddr = PhysHeapCpuPAddrToDevPAddr;
+
+	psServicesFuncs->pfnSysInstallDeviceLISR = PVRSRVSystemInstallDeviceLISR;
+	psServicesFuncs->pfnSysUninstallDeviceLISR = PVRSRVSystemUninstallDeviceLISR;
+
+	psServicesFuncs->pfnCheckStatus = PVRSRVCheckStatus;
+	psServicesFuncs->pfnGetErrorString = PVRSRVGetErrorString;
+	psServicesFuncs->pfnGetDeviceInstance = PVRSRVGetDeviceInstance;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueCreate(IMG_HANDLE *phQueue, IMG_UINT32 ui32Length)
+{
+	struct workqueue_struct *psQueue;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Length);
+
+	if (phQueue == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psQueue = alloc_ordered_workqueue("%s", WQ_FREEZABLE | WQ_MEM_RECLAIM, g_szDrvName);
+	if (psQueue == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	*phQueue = (IMG_HANDLE)psQueue;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueDestroy(IMG_HANDLE hQueue)
+{
+	struct workqueue_struct *psQueue = (struct workqueue_struct *)hQueue;
+
+	DC_ASSERT(psQueue);
+
+	destroy_workqueue(psQueue);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueFlush(IMG_HANDLE hQueue)
+{
+	struct workqueue_struct *psQueue = (struct workqueue_struct *)hQueue;
+
+	DC_ASSERT(psQueue);
+
+	flush_workqueue(psQueue);
+
+	return PVRSRV_OK;
+}
+
+static void WorkQueueProcessorWrapper(struct work_struct *psWork)
+{
+	WORK_ITEM *psWorkItem = container_of(psWork, WORK_ITEM, sWork);
+
+#if defined(DEBUG)
+	DC_ASSERT(psWorkItem && psWorkItem->pfnProcessor);
+#else
+	// klocwork workaround
+	if (psWorkItem == NULL || psWorkItem->pfnProcessor == NULL)
+	{
+		DC_OSAbort(__FILE__, __LINE__);
+		return;
+	}
+#endif
+
+	(void)psWorkItem->pfnProcessor(psWorkItem->pvData);
+}
+
+PVRSRV_ERROR DC_OSWorkQueueCreateWorkItem(IMG_HANDLE *phWorkItem, PFN_WORK_PROCESSOR pfnProcessor, void *pvProcessorData)
+{
+	WORK_ITEM *psWorkItem;
+
+	if (phWorkItem == NULL || pfnProcessor == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psWorkItem = DC_OSAllocMem(sizeof(*psWorkItem));
+	if (psWorkItem == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_WORK(&psWorkItem->sWork, WorkQueueProcessorWrapper);
+
+	psWorkItem->pfnProcessor = pfnProcessor;
+	psWorkItem->pvData = pvProcessorData;
+
+	*phWorkItem = (IMG_HANDLE)psWorkItem;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueDestroyWorkItem(IMG_HANDLE phWorkItem)
+{
+	WORK_ITEM *psWorkItem = (WORK_ITEM *)phWorkItem;
+
+	DC_ASSERT(psWorkItem);
+
+	DC_OSFreeMem(psWorkItem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DC_OSWorkQueueAddWorkItem(IMG_HANDLE hQueue, IMG_HANDLE hWorkItem)
+{
+	struct workqueue_struct *psQueue = (struct workqueue_struct *)hQueue;
+	WORK_ITEM *psWorkItem = (WORK_ITEM *)hWorkItem;
+
+	DC_ASSERT(psQueue && psWorkItem);
+
+	if (!queue_work(psQueue, &psWorkItem->sWork))
+	{
+		printk(KERN_WARNING " %s - %s: Cannot queue work that's already queued\n",
+		       g_szDrvName, __func__);
+		return PVRSRV_ERROR_UNABLE_TO_SCHEDULE_TASK;
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/common/dc_example.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/common/dc_example.c
new file mode 100644
index 000000000000..893b5f0fef36
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/common/dc_example.c
@@ -0,0 +1,1675 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Services headers */
+#include <powervr/imgpixfmts.h>
+#include <powervr/buffer_attribs.h>
+#include "dc_osfuncs.h"
+#include "dc_example.h"
+#include "img_defs.h"
+#include "kerneldisplay.h"
+
+/*
+	Enable to track contexts and buffers
+*/
+/* #define DCEX_DEBUG 1 */
+
+/*
+	Enable to get more debug. Only supported on UMA
+*/
+/* #define DCEX_VERBOSE 1*/
+
+#if defined(DCEX_DEBUG)
+	#define DCEX_DEBUG_PRINT(fmt, ...) \
+		DC_OSDebugPrintf(DBGLVL_WARNING, fmt, __VA_ARGS__)
+#else
+	#define DCEX_DEBUG_PRINT(fmt, ...)
+#endif
+
+/*
+	The number of inflight commands this display driver can handle
+*/
+#define MAX_COMMANDS_INFLIGHT 2
+
+#ifdef NO_HARDWARE
+#define MAX_PIPES 1
+#else
+#define MAX_PIPES 8
+#endif
+
+static IMG_UINT32 ui32ByteStride;
+static IMG_UINT32 ePixelFormat;
+
+static IMG_BOOL CheckBufferDimensions(void)
+{
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams;
+	IMG_UINT32 ui32BytesPP = 0;
+
+	psModuleParams = DCExampleGetModuleParameters();
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return IMG_FALSE;
+	}
+
+	if (psModuleParams->ui32Width == 0 ||
+	    psModuleParams->ui32Height == 0 ||
+	    psModuleParams->ui32Depth == 0)
+	{
+		DC_OSDebugPrintf(
+			DBGLVL_WARNING,
+			": Illegal module parameters (width %u, height %u, depth %u)\n",
+			psModuleParams->ui32Width,
+			psModuleParams->ui32Height,
+			psModuleParams->ui32Depth);
+
+		return IMG_FALSE;
+	}
+
+	switch (psModuleParams->ui32Depth)
+	{
+		case 32:
+			switch (psModuleParams->ui32Format)
+			{
+				case 0:
+					ePixelFormat = IMG_PIXFMT_B8G8R8A8_UNORM;
+					break;
+				case 1:
+					ePixelFormat = IMG_PIXFMT_B10G10R10A2_UNORM;
+					break;
+				default:
+					ePixelFormat = IMG_PIXFMT_UNKNOWN;
+					break;
+			}
+
+			ui32BytesPP = 4;
+			break;
+		case 16:
+			switch (psModuleParams->ui32Format)
+			{
+				case 0:
+					ePixelFormat = IMG_PIXFMT_B5G6R5_UNORM;
+					break;
+				default:
+					ePixelFormat = IMG_PIXFMT_UNKNOWN;
+					break;
+			}
+
+			ui32BytesPP = 2;
+			break;
+		default:
+			DC_OSDebugPrintf(DBGLVL_WARNING, ": Display depth %lu not supported\n", psModuleParams->ui32Depth);
+
+			ePixelFormat = IMG_PIXFMT_UNKNOWN;
+			break;
+	}
+
+	if (ePixelFormat == IMG_PIXFMT_UNKNOWN)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING,
+						 ": Display format %lu not supported for depth %lu\n",
+						 psModuleParams->ui32Format, psModuleParams->ui32Depth);
+		return IMG_FALSE;
+	}
+
+	if ((psModuleParams->ui32MemLayout == 1 && psModuleParams->ui32FBCFormat == 0) ||
+	    (psModuleParams->ui32MemLayout != 1 && psModuleParams->ui32FBCFormat != 0))
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR,
+				 ": Invalid memory layout/FBC mode combination (memory layout %u, FBC format %u)\n",
+				 psModuleParams->ui32MemLayout,
+				 psModuleParams->ui32FBCFormat);
+
+		return IMG_FALSE;
+	}
+
+	ui32ByteStride = psModuleParams->ui32Width * ui32BytesPP;
+
+	DC_OSDebugPrintf(DBGLVL_INFO, " Width: %u pixels\n", psModuleParams->ui32Width);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Height: %u pixels\n", psModuleParams->ui32Height);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Stride: %u bytes\n", ui32ByteStride);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Depth: %u bits\n", psModuleParams->ui32Depth);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Format: %u\n", psModuleParams->ui32Format);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Memory layout: %u\n", psModuleParams->ui32MemLayout);
+	DC_OSDebugPrintf(DBGLVL_INFO, " FBC format: %u\n", psModuleParams->ui32FBCFormat);
+	DC_OSDebugPrintf(DBGLVL_INFO, " X-Dpi: %u\n", psModuleParams->ui32XDpi);
+	DC_OSDebugPrintf(DBGLVL_INFO, " Y-Dpi: %u\n", psModuleParams->ui32YDpi);
+
+	return IMG_TRUE;
+}
+
+
+typedef struct _DCEX_DEVICE_
+{
+	IMG_HANDLE		hPVRServicesConnection;
+	IMG_HANDLE		hPVRServicesDevice;
+	DC_SERVICES_FUNCS	sPVRServicesFuncs;
+
+	IMG_HANDLE		hSrvHandle;
+#if defined(LMA)
+	PHYS_HEAP		*psPhysHeap;
+	IMG_CPU_PHYADDR		sDispStartAddr;
+	IMG_UINT64		uiDispMemSize;
+	IMG_UINT32		ui32BufferSize;
+	IMG_UINT32		ui32BufferCount;
+	IMG_UINT32		ui32BufferUseMask;
+#endif
+	DLLIST_NODE		sListNode;      /* List of all device nodes */
+	void			*hBufListLock;  /* Lock for accessing sBufListNode */
+	DLLIST_NODE		sBufListNode;   /* List of Buffers for this device */
+	/* Per-device context information */
+	IMG_HANDLE		hConfigData[MAX_COMMANDS_INFLIGHT];
+	IMG_UINT32		ui32Head;
+	IMG_UINT32		ui32Tail;
+	IMG_BOOL		b_DisplayContextActive;
+} DCEX_DEVICE;
+
+typedef enum
+{
+	DCEX_BUFFER_SOURCE_ALLOC,
+	DCEX_BUFFER_SOURCE_IMPORT,
+} DCEX_BUFFER_SOURCE;
+
+typedef struct _DCEX_BUFFER_
+{
+	ATOMIC_T		i32RefCount;	/* Only required for system buffer */
+	IMG_BOOL		bIsSysBuffer;	/* Set for system buffer */
+	IMG_UINT32		ePixFormat;
+	IMG_UINT32		ui32Width;
+	IMG_UINT32		ui32Height;
+	IMG_UINT32		ui32ByteStride;
+	IMG_UINT32		ui32Size;
+	IMG_UINT32		ui32PageCount;
+	IMG_HANDLE		hImport;
+	IMG_DEV_PHYADDR		*pasDevPAddr;
+	DCEX_DEVICE		*psDevice;
+#if defined(LMA)
+	IMG_UINT64		uiAllocHandle;
+#else
+	void			*pvAllocHandle;
+#endif
+	DCEX_BUFFER_SOURCE eSource;
+	DLLIST_NODE		sListNode;     /* List of all buffers one per device */
+} DCEX_BUFFER;
+
+static void *g_hDevDataListLock;
+
+static DLLIST_NODE g_sDeviceDataListHead;
+
+static INLINE void DCExampleConfigPush(DCEX_DEVICE *psDevice,
+                                       IMG_HANDLE hConfigData)
+{
+	IMG_UINT32 ui32Head;
+
+	DC_ASSERT(psDevice != NULL);
+	ui32Head = psDevice->ui32Head;
+	psDevice->hConfigData[ui32Head] = hConfigData;
+	ui32Head++;
+
+	if (ui32Head >= MAX_COMMANDS_INFLIGHT)
+	{
+		ui32Head = 0;
+	}
+
+	psDevice->ui32Head = ui32Head;
+}
+
+static INLINE IMG_HANDLE DCExampleConfigPop(DCEX_DEVICE *psDevice)
+{
+	IMG_UINT32 ui32Tail;
+	IMG_HANDLE hConfigData;
+
+	DC_ASSERT(psDevice != NULL);
+	ui32Tail = psDevice->ui32Tail;
+	hConfigData = psDevice->hConfigData[ui32Tail];
+	ui32Tail++;
+
+	if (ui32Tail >= MAX_COMMANDS_INFLIGHT)
+	{
+		ui32Tail = 0;
+	}
+	psDevice->ui32Tail = ui32Tail;
+
+	return hConfigData;
+}
+
+static INLINE IMG_BOOL DCExampleConfigIsEmpty(DCEX_DEVICE *psDevice)
+{
+	DC_ASSERT(psDevice != NULL);
+	return ((psDevice->ui32Tail == psDevice->ui32Head) ? IMG_TRUE : IMG_FALSE);
+}
+
+#if defined(LMA)
+/*
+	Simple unit size allocator
+*/
+static
+IMG_UINT64 _DCExampleAllocLMABuffer(DCEX_DEVICE *psDevice)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 pvRet = 0;
+
+	for (i = 0; i < psDevice->ui32BufferCount; i++)
+	{
+		if ((psDevice->ui32BufferUseMask & (1UL << i)) == 0)
+		{
+			pvRet = psDevice->sDispStartAddr.uiAddr +
+					(i * psDevice->ui32BufferSize);
+			psDevice->ui32BufferUseMask |= (1UL << i);
+			break;
+		}
+	}
+
+	return pvRet;
+}
+
+static
+void _DCExampleFreeLMABuffer(DCEX_DEVICE *psDevice, IMG_UINT64 uiAddr)
+{
+	IMG_UINT64 ui64Offset;
+
+	DC_ASSERT(uiAddr >= psDevice->sDispStartAddr.uiAddr);
+
+	DCEX_DEBUG_PRINT("Freeing %p, Size %x, UseMask %x", (void *)uiAddr,
+	                  psDevice->ui32BufferSize, psDevice->ui32BufferUseMask);
+
+	ui64Offset = uiAddr - psDevice->sDispStartAddr.uiAddr;
+	ui64Offset = DC_OSDiv64(ui64Offset, psDevice->ui32BufferSize);
+	DC_ASSERT(ui64Offset <= psDevice->ui32BufferCount);
+	psDevice->ui32BufferUseMask &= ~(1UL << ui64Offset);
+}
+#endif /* LMA */
+
+static
+void DCExampleGetInfo(IMG_HANDLE hDeviceData,
+						  DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/*
+		Copy our device name
+	*/
+	DC_OSStringNCopy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	/*
+		Report what our minimum and maximum display period is.
+	*/
+	psDisplayInfo->ui32MinDisplayPeriod	= 0;
+	psDisplayInfo->ui32MaxDisplayPeriod	= 1;
+	psDisplayInfo->ui32MaxPipes			= MAX_PIPES;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static
+PVRSRV_ERROR DCExamplePanelQueryCount(IMG_HANDLE hDeviceData,
+										 IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+	/*
+		If you know the panel count at compile time just hardcode it, if it's
+		dynamic you should probe it here
+	*/
+	*pui32NumPanels = 1;
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExamplePanelQuery(IMG_HANDLE hDeviceData,
+									IMG_UINT32 ui32PanelsArraySize,
+									IMG_UINT32 *pui32NumPanels,
+									PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/*
+		If we have hotplug displays then there is a chance a display could
+		have been removed so return the number of panels we have queried
+	*/
+	*pui32NumPanels = 1;
+
+	/*
+		Either hard code the values here or probe each panel here. If a new
+		panel has been hotplugged then ignore it as we've not been given
+		room to store its data
+	*/
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = ePixelFormat;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width = psModuleParams->ui32Width;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height = psModuleParams->ui32Height;
+
+	psPanelInfo[0].ui32RefreshRate = psModuleParams->ui32RefreshRate;
+	psPanelInfo[0].ui32XDpi = psModuleParams->ui32XDpi;
+	psPanelInfo[0].ui32YDpi = psModuleParams->ui32YDpi;
+
+	switch (psModuleParams->ui32MemLayout)
+	{
+		case 0:
+			psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+			break;
+		case 1:
+			psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_FBC;
+			break;
+		default:
+			DC_OSDebugPrintf(DBGLVL_ERROR, ": Unknown memory layout %ld\n", psModuleParams->ui32MemLayout);
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	switch (psModuleParams->ui32FBCFormat)
+	{
+		case 0:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+			break;
+		case 1:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_8x8;
+			break;
+		case 2:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_16x4;
+			break;
+		case 5:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_32x2;
+			break;
+		case 6:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8;
+			break;
+		case 7:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4;
+			break;
+		case 8:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2;
+			break;
+		case 9:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8;
+			break;
+		case 10:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4;
+			break;
+		case 11:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2;
+			break;
+		case 12:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8;
+			break;
+		case 13:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4;
+			break;
+		case 14:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2;
+			break;
+		case 15:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8;
+			break;
+		case 16:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4;
+			break;
+		case 17:
+			psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2;
+			break;
+		default:
+			DC_OSDebugPrintf(DBGLVL_ERROR, ": Unknown FBC format %ld\n", psModuleParams->ui32FBCFormat);
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DC_ASSERT((psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout == PVRSRV_SURFACE_MEMLAYOUT_FBC &&
+		   psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode != IMG_FB_COMPRESSION_NONE) ||
+		  (psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout != PVRSRV_SURFACE_MEMLAYOUT_FBC &&
+		   psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode == IMG_FB_COMPRESSION_NONE));
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExampleFormatQuery(IMG_HANDLE hDeviceData,
+									IMG_UINT32 ui32NumFormats,
+									PVRSRV_SURFACE_FORMAT *pasFormat,
+									IMG_UINT32 *pui32Supported)
+{
+	IMG_UINT32 i;
+
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		/*
+			If the display controller has multiple display pipes (DMA engines)
+			each one should be checked to see if it supports the specified
+			format.
+		*/
+		if ((pasFormat[i].ePixFormat == IMG_PIXFMT_B10G10R10A2_UNORM) ||
+		    (pasFormat[i].ePixFormat == IMG_PIXFMT_B8G8R8A8_UNORM) ||
+		    (pasFormat[i].ePixFormat == IMG_PIXFMT_B5G6R5_UNORM))
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExampleDimQuery(IMG_HANDLE hDeviceData,
+								 IMG_UINT32 ui32NumDims,
+								 PVRSRV_SURFACE_DIMS *psDim,
+								 IMG_UINT32 *pui32Supported)
+{
+	IMG_UINT32 i;
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		/*
+			If the display controller has multiple display pipes (DMA engines)
+			each one should be checked to see if it supports the specified
+			dimension.
+		*/
+		if ((psDim[i].ui32Width == psModuleParams->ui32Width)
+		&&  (psDim[i].ui32Height == psModuleParams->ui32Height))
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DCExampleBufferSystemAcquire(IMG_HANDLE hDeviceData,
+											IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+											IMG_UINT32 *pui32PageCount,
+											IMG_UINT32 *pui32ByteStride,
+											IMG_HANDLE *phSystemBuffer)
+{
+	DCEX_BUFFER *psBuffer;
+	DCEX_DEVICE *psDevice = (DCEX_DEVICE *)hDeviceData;
+	DLLIST_NODE *psNode, *psNext;
+
+	/*
+		This function is optional. It provides a method for services
+		to acquire a display buffer which it didn't setup but was created
+		by the OS (e.g. Linux frame buffer).
+		If the OS should trigger a mode change then it's not allowed to free
+		the previous buffer until services has released it via BufferSystemRelease
+	*/
+
+	/*
+		Protect the access to the system buffer list
+	 */
+	DC_OSMutexLock(psDevice->hBufListLock);
+
+	/*
+		Take a reference to the system buffer 1st to make sure it isn't freed
+	*/
+	dllist_foreach_node(&psDevice->sBufListNode, psNode, psNext)
+	{
+		psBuffer = IMG_CONTAINER_OF(psNode, DCEX_BUFFER, sListNode);
+
+		if ((psBuffer->psDevice == psDevice) && psBuffer->bIsSysBuffer)
+		{
+
+			DCEX_DEBUG_PRINT("%s: psBuffer %p Ref Count %d\n", __func__,
+			                 psBuffer, DC_OSAtomicRead(&psBuffer->i32RefCount));
+
+			DC_OSAtomicIncrement(&psBuffer->i32RefCount);
+
+			*puiLog2PageSize = DC_OSGetPageShift();
+			*pui32PageCount = psBuffer->ui32Size >> DC_OSGetPageShift();
+			*pui32ByteStride = psBuffer->ui32ByteStride;
+			*phSystemBuffer = psBuffer;
+
+			DC_OSMutexUnlock(psDevice->hBufListLock);
+			return PVRSRV_OK;
+		}
+	}
+
+	DC_OSMutexUnlock(psDevice->hBufListLock);
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+static
+void DCExampleBufferSystemRelease(IMG_HANDLE hSystemBuffer)
+{
+	DCEX_BUFFER *psBuffer = hSystemBuffer;
+
+	/*
+		Acquire the buffer mutex to prevent alteration while we are
+		releasing the system buffer.
+	*/
+	DC_OSMutexLock(psBuffer->psDevice->hBufListLock);
+
+	if (!psBuffer->bIsSysBuffer)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, "%s: Unexpected non system-buffer found\n",
+		                __func__);
+		DC_OSMutexUnlock(psBuffer->psDevice->hBufListLock);
+		return;
+	}
+
+	DC_OSAtomicDecrement(&psBuffer->i32RefCount);
+
+	/*
+		If the system buffer has changed and we've just dropped the last
+		refcount then free the buffer
+	*/
+	if (!dllist_node_is_in_list(&psBuffer->sListNode) && (DC_OSAtomicRead(&psBuffer->i32RefCount) == 0))
+	{
+		/* Free the buffer and it's memory (if the memory was allocated) */
+		DCEX_DEBUG_PRINT("%s: Dropping %p", __func__, psBuffer);
+	}
+
+	/*
+		Release buffer mutex now that we've released it.
+	*/
+	DC_OSMutexUnlock(psBuffer->psDevice->hBufListLock);
+}
+
+static
+PVRSRV_ERROR DCExampleContextCreate(IMG_HANDLE hDeviceData,
+									  IMG_HANDLE *hDisplayContext)
+{
+	DCEX_DEVICE *psDevice = hDeviceData;
+
+	if (psDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/*
+		The creation of a display context is a software concept and
+		it's an "agreement" between the services client and the DC driver
+		as to what this means (if anything)
+	*/
+	if (psDevice->b_DisplayContextActive != IMG_TRUE)
+	{
+		*hDisplayContext = hDeviceData;
+
+		psDevice->b_DisplayContextActive = IMG_TRUE;
+		DCEX_DEBUG_PRINT("Create context (%p)\n", *hDisplayContext);
+		return PVRSRV_OK;
+	}
+
+	return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+}
+
+static
+PVRSRV_ERROR DCExampleContextConfigureCheck(IMG_HANDLE hDisplayContext,
+											  IMG_UINT32 ui32PipeCount,
+											  PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+											  IMG_HANDLE *ahBuffers)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+
+	PVR_UNREFERENCED_PARAMETER(hDisplayContext);
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/*
+		The display engine might have a limit on the number of discretely
+		configurable pipes. In such a case an error should be returned.
+	*/
+	if (ui32PipeCount > MAX_PIPES)
+	{
+		eError = PVRSRV_ERROR_DC_TOO_MANY_PIPES;
+		goto fail_max_pipes;
+	}
+
+	/*
+		This optional function allows the display driver to check if the display
+		configuration passed in is valid.
+		It's possible that due to HW constraints that although the client has
+		honoured the DimQuery and FormatQuery results the configuration it
+		has requested is still not possible (e.g. there isn't enough space in
+		the display controller's MMU, or due restrictions on display pipes.
+	*/
+
+	for (i = 0; i < ui32PipeCount; i++)
+	{
+		DCEX_BUFFER *psBuffer = ahBuffers[i];
+
+		if (pasSurfAttrib[i].sCrop.sDims.ui32Width  != psModuleParams->ui32Width  ||
+		    pasSurfAttrib[i].sCrop.sDims.ui32Height != psModuleParams->ui32Height ||
+		    pasSurfAttrib[i].sCrop.i32XOffset != 0 ||
+		    pasSurfAttrib[i].sCrop.i32YOffset != 0)
+		{
+			eError = PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+			break;
+		}
+
+		if (pasSurfAttrib[i].sDisplay.sDims.ui32Width !=
+			pasSurfAttrib[i].sCrop.sDims.ui32Width ||
+			pasSurfAttrib[i].sDisplay.sDims.ui32Height !=
+			pasSurfAttrib[i].sCrop.sDims.ui32Height ||
+			pasSurfAttrib[i].sDisplay.i32XOffset !=
+			pasSurfAttrib[i].sCrop.i32XOffset ||
+			pasSurfAttrib[i].sDisplay.i32YOffset !=
+			pasSurfAttrib[i].sCrop.i32YOffset)
+		{
+			eError = PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+			break;
+		}
+
+		if (psBuffer->ui32Width != psModuleParams->ui32Width
+		||  psBuffer->ui32Height != psModuleParams->ui32Height)
+		{
+			eError = PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+			break;
+		}
+	}
+
+fail_max_pipes:
+	return eError;
+}
+
+static
+void DCExampleContextConfigure(IMG_HANDLE hDisplayContext,
+									 IMG_UINT32 ui32PipeCount,
+									 PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+									 IMG_HANDLE *ahBuffers,
+									 IMG_UINT32 ui32DisplayPeriod,
+									 IMG_HANDLE hConfigData)
+{
+	DCEX_DEVICE *psDeviceData = hDisplayContext;
+	IMG_UINT32 i;
+
+	/*
+		As we have no HW and thus no VSync IRQ we just activate the
+		new config here
+	*/
+	for (i = 0; i < ui32PipeCount; i++)
+	{
+#if defined(DCEX_DEBUG)
+		DCEX_BUFFER *psBuffer = ahBuffers[i];
+		/*
+			There is no checking to be done here as we can't fail,
+			any checking should have been DCExampleContextConfigureCheck.
+		*/
+
+		/*
+			If the display controller supports scaling it should set it up
+			here.
+		*/
+
+		/*
+			Setup the DMA from the display buffer
+		*/
+
+		/*
+			Save the config data as we need to pass it back once this
+			configuration gets retired
+		*/
+		DCEX_DEBUG_PRINT("Display buffer (%p), Ref Count %d\n", psBuffer,
+		                 DC_OSAtomicRead(&psBuffer->i32RefCount));
+#endif /* DCEX_DEBUG */
+	}
+
+	/*
+		As we have no HW and thus no VSync IRQ we just retire the
+		previous config as soon as we get a new one
+	*/
+	if (DCExampleConfigIsEmpty(psDeviceData) == IMG_FALSE)
+	{
+		/* Retire the current config */
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(DCExampleConfigPop(psDeviceData));
+	}
+
+	if (ui32PipeCount != 0)
+	{
+		/* Save our new config data */
+		DCExampleConfigPush(psDeviceData, hConfigData);
+	}
+	else
+	{
+		/*
+			When the client requests the display context to be destroyed
+			services will issue a "NULL" flip to us so we can retire
+			the current configuration.
+
+			We need to pop the current (and last) configuration off the our
+			stack and retire it which we're already done above as it's our
+			default behaviour to retire the previous config immediately when we
+			get a new one. We also need to "retire" the NULL flip so the DC core
+			knows that we've done whatever we need to do so it can start to
+			tear down the display context
+
+			In real devices we could have a number of configurations in flight,
+			all these configurations _and the NULL flip_ need to be processed
+			in order with the NULL flip being completed when it's active rather
+			then when it will be retired (as it will never be retired)
+
+			At this point there is nothing that the display is being asked to
+			display by services and it's the DC driver implementation decision
+			as to what it should then do. Typically, for systems that have a
+			system surface the DC would switch back to displaying that.
+		*/
+		DCEX_DEBUG_PRINT("Display flushed (%p)\n", hDisplayContext);
+
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(hConfigData);
+	}
+}
+
+static
+void DCExampleContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DCEX_DEVICE *psDevice = hDisplayContext;
+
+	DC_ASSERT(DCExampleConfigIsEmpty(psDevice) == IMG_TRUE);
+
+	/*
+		Counterpart to ContextCreate. Any buffers created/imported on
+		this display context will have been freed before this call so
+		all the display driver needs to do is release any resources
+		allocated at ContextCreate time.
+	*/
+	psDevice->b_DisplayContextActive = IMG_FALSE;
+	DCEX_DEBUG_PRINT("Destroy display context (%p)\n", hDisplayContext);
+}
+
+static
+PVRSRV_ERROR DCExampleBufferAlloc(IMG_HANDLE hDisplayContext,
+									DC_BUFFER_CREATE_INFO *psCreateInfo,
+									IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+									IMG_UINT32 *pui32PageCount,
+									IMG_UINT32 *pui32ByteStride,
+									IMG_HANDLE *phBuffer)
+{
+	DCEX_BUFFER *psBuffer;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	PVRSRV_ERROR eError;
+	DCEX_DEVICE *psDevice = hDisplayContext;
+
+	/*
+		Allocate the buffer control structure
+	*/
+
+	psBuffer = DC_OSCallocMem(sizeof(DCEX_BUFFER));
+
+	if (psBuffer == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_bufferalloc;
+	}
+
+	dllist_init(&psBuffer->sListNode);
+
+	/* Common setup */
+	psBuffer->eSource = DCEX_BUFFER_SOURCE_ALLOC;
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 1);
+	psBuffer->ePixFormat = psSurfInfo->sFormat.ePixFormat;
+	psBuffer->bIsSysBuffer = IMG_FALSE;
+
+	/*
+		Store the display buffer size, that is the pixel width and height that
+		will be displayed which might be less than the pixel width and height
+		of the buffer
+	*/
+	psBuffer->ui32Width = psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height = psSurfInfo->sDims.ui32Height;
+
+	switch (psSurfInfo->sFormat.eMemLayout)
+	{
+		case PVRSRV_SURFACE_MEMLAYOUT_STRIDED:
+			/*
+				As we're been asked to allocate this buffer we decide what it's
+				stride should be.
+			*/
+			psBuffer->ui32ByteStride = psSurfInfo->sDims.ui32Width * psCreateInfo->ui32BPP;
+			psBuffer->ui32Size = psBuffer->ui32Height * psBuffer->ui32ByteStride;
+			break;
+		case PVRSRV_SURFACE_MEMLAYOUT_FBC:
+			psBuffer->ui32ByteStride = psCreateInfo->sFBC.ui32FBCStride * psCreateInfo->ui32BPP;
+			psBuffer->ui32Size = psCreateInfo->sFBC.ui32Size;
+
+			/*
+				Here we should program the FBC registers in the display
+				controller according to the information we have in
+				psSurfInfo->sFBC
+			*/
+			break;
+
+		default:
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+			goto fail_memlayout;
+			break;
+	}
+
+	psBuffer->psDevice = psDevice;
+
+#if defined(LMA)
+	/*
+		Check that the buffer size calculated is compatible with what was allocated during initialisation.
+	*/
+	if (psBuffer->ui32Size > psDevice->ui32BufferSize)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, "%s: Buffer size of 0x%0x is too large for allocation of 0x%0x!\n",
+		                 __func__, psBuffer->ui32Size, psDevice->ui32BufferSize);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_buffersize;
+	}
+	else if (psBuffer->ui32Size != psDevice->ui32BufferSize)
+	{
+		DCEX_DEBUG_PRINT("%s: Buffer size of 0x%0x is not same as allocated 0x%0x\n",
+		                 __func__, psBuffer->ui32Size, psDevice->ui32BufferSize);
+		psBuffer->ui32Size = psDevice->ui32BufferSize;
+	}
+
+	/*
+		Allocate display addressable memory. We only need physical addresses
+		at this stage.
+
+		Note: This could be deferred until the 1st map or acquire call.
+	*/
+	psBuffer->uiAllocHandle = _DCExampleAllocLMABuffer(psDevice);
+	if (psBuffer->uiAllocHandle == 0)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_buffermemalloc;
+	}
+#else
+	psBuffer->pvAllocHandle = DCExampleVirtualAllocUncached(psBuffer->ui32Size);
+	if (psBuffer->pvAllocHandle == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_buffermemalloc;
+	}
+#endif
+	*pui32ByteStride = psBuffer->ui32ByteStride;
+	*puiLog2PageSize = DC_OSGetPageShift();
+	*pui32PageCount = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+	*phBuffer = psBuffer;
+
+	DCEX_DEBUG_PRINT("Allocate buffer (%p)\n", psBuffer);
+
+	DC_OSMutexLock(psDevice->hBufListLock);
+	dllist_add_to_tail(&psDevice->sBufListNode, &psBuffer->sListNode);
+	DC_OSMutexUnlock(psDevice->hBufListLock);
+
+	return PVRSRV_OK;
+
+#if defined(LMA)
+fail_buffersize:
+#endif
+fail_memlayout:
+fail_buffermemalloc:
+	DC_OSFreeMem(psBuffer);
+
+fail_bufferalloc:
+	return eError;
+}
+
+#if !defined(LMA)
+static
+PVRSRV_ERROR DCExampleBufferImport(IMG_HANDLE hDisplayContext,
+									 IMG_UINT32 ui32NumPlanes,
+									 IMG_HANDLE **paphImport,
+									 DC_BUFFER_IMPORT_INFO *psSurfAttrib,
+									 IMG_HANDLE *phBuffer)
+{
+	/*
+		This it optional and should only be provided if the display controller
+		can access "general" memory (e.g. the memory doesn't have to contiguous)
+	*/
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams = DCExampleGetModuleParameters();
+	DCEX_BUFFER *psBuffer;
+	DCEX_DEVICE *psDevice = hDisplayContext;
+
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/*
+		Check to see if our display hardware supports this buffer
+	*/
+	if ((psSurfAttrib->ePixFormat != IMG_PIXFMT_B8G8R8A8_UNORM) ||
+		(psSurfAttrib->ui32Width[0] != psModuleParams->ui32Width) ||
+		(psSurfAttrib->ui32Height[0] != psModuleParams->ui32Height) ||
+		(psSurfAttrib->ui32ByteStride[0] != ui32ByteStride))
+	{
+		return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	psBuffer = DC_OSCallocMem(sizeof(DCEX_BUFFER));
+
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psBuffer->eSource = DCEX_BUFFER_SOURCE_IMPORT;
+	psBuffer->ui32Width = psSurfAttrib->ui32Width[0];
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 1);
+	psBuffer->ePixFormat = psSurfAttrib->ePixFormat;
+	psBuffer->ui32ByteStride = psSurfAttrib->ui32ByteStride[0];
+	psBuffer->ui32Width = psSurfAttrib->ui32Width[0];
+	psBuffer->ui32Height = psSurfAttrib->ui32Height[0];
+	psBuffer->psDevice = psDevice;
+
+	/*
+		If the display controller supports mapping "general" memory, but has
+		limitations (e.g. if it doesn't have full range addressing) these
+		should be checked here by calling DCImportBufferAcquire. In this case
+		it lock down the physical address of the buffer at this strange rather
+		then being able to defer it to map time.
+	*/
+	psBuffer->hImport = paphImport[0];
+
+	*phBuffer = psBuffer;
+	DCEX_DEBUG_PRINT("Import buffer (%p)\n", psBuffer);
+	return PVRSRV_OK;
+}
+#endif
+
+static
+PVRSRV_ERROR DCExampleBufferAcquire(IMG_HANDLE hBuffer,
+									IMG_DEV_PHYADDR *pasDevPAddr,
+									void **ppvLinAddr)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+	PVRSRV_ERROR eError;
+#if defined(LMA)
+	IMG_UINT32 i;
+	unsigned long ulPages = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+	PHYS_HEAP *psPhysHeap = psBuffer->psDevice->psPhysHeap;
+	IMG_CPU_PHYADDR sCpuPAddr;
+#endif
+	/*
+		If we didn't allocate the display memory at buffer alloc time
+		we would have to do it here.
+	*/
+
+	/*
+		Fill in the array of addresses we were passed
+	*/
+#if defined(LMA)
+	sCpuPAddr.uiAddr = psBuffer->uiAllocHandle;
+	DCEX_DEBUG_PRINT("Acquire buffer (%p) memory, npages = %d, Start = %p, size = 0x%x\n",
+	                 psBuffer, ulPages, sCpuPAddr.uiAddr, psBuffer->ui32Size);
+	for (i = 0; i < ulPages; i++)
+	{
+		psBuffer->psDevice->sPVRServicesFuncs.pfnPhysHeapCpuPAddrToDevPAddr(
+				psPhysHeap,
+				1,
+				&pasDevPAddr[i],
+				&sCpuPAddr);
+
+		/* Only display first and last mappings */
+		if ((i == 0) || i == (ulPages - 1))
+		{
+			DCEX_DEBUG_PRINT("Acquire buffer (%p) page %d, DevPAddr %p, CPUPAddr %p\n",
+			                 psBuffer, i, pasDevPAddr[i].uiAddr, sCpuPAddr.uiAddr);
+		}
+		sCpuPAddr.uiAddr += DC_OSGetPageSize();
+	}
+	*ppvLinAddr = NULL;
+	eError      = PVRSRV_OK;
+#else
+	eError = DCExampleGetDevPAddrs(psBuffer->pvAllocHandle, pasDevPAddr, 0, psBuffer->ui32Size);
+	if (eError == PVRSRV_OK)
+	{
+		eError = DCExampleGetLinAddr(psBuffer->pvAllocHandle, ppvLinAddr);
+	}
+#endif
+
+	DCEX_DEBUG_PRINT("Acquire buffer (%p) memory\n", psBuffer);
+	return eError;
+}
+
+static
+void DCExampleBufferRelease(IMG_HANDLE hBuffer)
+{
+#if defined(DCEX_DEBUG)
+	DCEX_BUFFER *psBuffer = hBuffer;
+#endif
+	/*
+		We could release the display memory here (assuming it wasn't
+		still mapped into the display controller).
+
+		As the buffer hasn't been freed the contents must be preserved, i.e.
+		in the next call to Acquire different physical pages can be returned,
+		but they must have the same contents as the old pages had at Release
+		time.
+	*/
+	DCEX_DEBUG_PRINT("Release buffer (%p) memory, Ref Count = %d\n",
+	                 psBuffer, DC_OSAtomicRead(&psBuffer->i32RefCount));
+}
+
+static
+void DCExampleBufferFree(IMG_HANDLE hBuffer)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+
+	DCEX_DEBUG_PRINT("Free buffer (%p)\n", psBuffer);
+	if (psBuffer->eSource == DCEX_BUFFER_SOURCE_ALLOC)
+	{
+#if defined(LMA)
+		DCEX_DEBUG_PRINT("Free buffer memory (%llu/[0x%llx])\n",
+		                 psBuffer->uiAllocHandle, psBuffer->uiAllocHandle);
+		_DCExampleFreeLMABuffer(psBuffer->psDevice, psBuffer->uiAllocHandle);
+#else
+		DCEX_DEBUG_PRINT("Free buffer memory (%p)\n", psBuffer->pvAllocHandle);
+		DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+	}
+
+	DC_OSMutexLock(psBuffer->psDevice->hBufListLock);
+	dllist_remove_node(&psBuffer->sListNode);
+	DC_OSMutexUnlock(psBuffer->psDevice->hBufListLock);
+	DC_OSFreeMem(psBuffer);
+}
+
+static
+PVRSRV_ERROR DCExampleBufferMap(IMG_HANDLE hBuffer)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+	IMG_UINT32 ui32PageCount;
+	PVRSRV_ERROR eError;
+#if defined(DCEX_VERBOSE)
+	IMG_UINT32 i;
+#endif
+	/*
+		If the display controller needs memory to be mapped into it
+		(e.g. it has an MMU) and didn't do it in the alloc and import then it
+		should provide this function.
+	*/
+
+	if (psBuffer->hImport)
+	{
+		IMG_DEV_PHYADDR *pasDevPAddr;
+		/*
+			In the case of an import buffer we didn't allocate the buffer and
+			so need to ask for it's pages
+		*/
+		eError = psBuffer->psDevice->sPVRServicesFuncs.pfnDCImportBufferAcquire(
+					psBuffer->hImport,
+					DC_OSGetPageShift(),
+					&ui32PageCount,
+					&pasDevPAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_import;
+		}
+#if defined(DCEX_VERBOSE)
+		for (i = 0; i < ui32PageCount; i++)
+		{
+			DCEX_DEBUG_PRINT(": DCExampleBufferMap: DCExample map address 0x%016llx\n", pasDevPAddr[i].uiAddr);
+		}
+#endif
+		psBuffer->pasDevPAddr = pasDevPAddr;
+		psBuffer->ui32PageCount = ui32PageCount;
+	}
+#if defined(DCEX_VERBOSE)
+	else
+	{
+		unsigned long ulPages = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+		IMG_DEV_PHYADDR sDevPAddr;
+
+		for (i = 0; i < ulPages; i++)
+		{
+			eError = DCExampleGetDevPAddrs(psBuffer->pvAllocHandle, &sDevPAddr, i, DC_OSGetPageSize());
+			if (PVRSRV_OK == eError)
+			{
+				DC_OSDebugPrintf(DBGLVL_INFO, ": DCExampleBufferMap: DCExample map address 0x%016llx\n", sDevPAddr.uiAddr);
+			}
+		}
+	}
+#endif
+	DCEX_DEBUG_PRINT("Map buffer (%p) into display\n", psBuffer);
+	return PVRSRV_OK;
+
+fail_import:
+	return eError;
+}
+
+static
+void DCExampleBufferUnmap(IMG_HANDLE hBuffer)
+{
+	DCEX_BUFFER *psBuffer = hBuffer;
+#if defined(DCEX_VERBOSE)
+	IMG_UINT32 i;
+#endif
+	/*
+		If the display controller provided buffer map then it must provide
+		this function
+	*/
+
+	/*
+		Unmap the memory from the display controller's MMU
+	*/
+	if (psBuffer->hImport)
+	{
+#if defined(DCEX_VERBOSE)
+		for (i = 0; i < psBuffer->ui32PageCount; i++)
+		{
+			DC_OSDebugPrintf(DBGLVL_INFO, ": DCExampleBufferUnmap: DCExample unmap address 0x%016llx\n", psBuffer->pasDevPAddr[i].uiAddr);
+		}
+#endif
+		/*
+			As this was an imported buffer we need to release it
+		*/
+		psBuffer->psDevice->sPVRServicesFuncs.pfnDCImportBufferRelease(
+				psBuffer->hImport,
+				psBuffer->pasDevPAddr);
+	}
+#if defined(DCEX_VERBOSE)
+	else
+	{
+		unsigned long ulPages = DC_OS_BYTES_TO_PAGES(psBuffer->ui32Size);
+		IMG_DEV_PHYADDR sDevPAddr;
+
+		for (i = 0; i < ulPages; i++)
+		{
+			PVRSRV_ERROR eError = DCExampleGetDevPAddrs(psBuffer->pvAllocHandle, &sDevPAddr, i, DC_OSGetPageSize());
+			if (PVRSRV_OK == eError)
+			{
+				DC_OSDebugPrintf(DBGLVL_INFO, ": DCExampleBufferUnmap: DCExample unmap address 0x%016llx\n", sDevPAddr.uiAddr);
+			}
+		}
+	}
+#endif
+	DCEX_DEBUG_PRINT("Unmap buffer (%p) from display\n", psBuffer);
+}
+
+/*
+	In this example driver we provide (almost) the full range of functions
+*/
+
+static DC_DEVICE_FUNCTIONS sDCFunctions;
+
+/*
+    Release all driver data allocated during DCExampleInit()
+*/
+static void DCExampleReleaseDriverData(const IMG_BOOL bFromDeInit)
+{
+	DLLIST_NODE *psNode, *psNext;
+	DCEX_DEVICE *psDeviceData;
+	DCEX_BUFFER *psBuffer;
+
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		DLLIST_NODE *psBufNode, *psBufNext;
+
+		psDeviceData = IMG_CONTAINER_OF(psNode, DCEX_DEVICE, sListNode);
+
+		if (bFromDeInit)
+		{
+			/* Unregister the device before freeing */
+			psDeviceData->sPVRServicesFuncs.pfnDCUnregisterDevice(psDeviceData->hSrvHandle);
+		}
+
+		/* Release all Heaps */
+#if defined(LMA)
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+#endif
+
+		/* Close all connection handles */
+		DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+
+		/* Release all associated buffers for this device */
+		DC_OSMutexLock(psDeviceData->hBufListLock);
+		dllist_foreach_node(&psDeviceData->sBufListNode, psBufNode, psBufNext)
+		{
+			psBuffer = IMG_CONTAINER_OF(psBufNode, DCEX_BUFFER, sListNode);
+
+			/* Free all LMA / Virtual Buffer allocations */
+#if defined(LMA)
+			_DCExampleFreeLMABuffer(psBuffer->psDevice, psBuffer->uiAllocHandle);
+#else
+			DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+			dllist_remove_node(psBufNode);
+
+			DC_OSFreeMem(psBuffer);
+		}
+		DC_OSMutexUnlock(psDeviceData->hBufListLock);
+
+		/* Release all OS memory */
+		DC_OSMutexLock(g_hDevDataListLock);
+
+		dllist_remove_node(psNode);
+
+		DC_OSMutexUnlock(g_hDevDataListLock);
+
+		DC_ASSERT(dllist_is_empty(&psDeviceData->sBufListNode));
+
+		DC_OSMutexDestroy(psDeviceData->hBufListLock);
+		DC_OSFreeMem(psDeviceData);
+	}
+
+	DC_ASSERT(dllist_is_empty(&g_sDeviceDataListHead));
+}
+
+#if defined(INTEGRITY_OS)
+static PVRSRV_ERROR DCExampleAcquireKernelMappingData(IMG_HANDLE hBuffer, IMG_HANDLE *phMapping, void **ppPhysAddr)
+{
+	DCEX_BUFFER *psBuffer = (DCEX_BUFFER *)hBuffer;
+	if (!psBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return DCExampleOSAcquireKernelMappingData(psBuffer->pvAllocHandle, phMapping, ppPhysAddr);
+}
+#endif
+
+/*
+	functions exported by kernel services for use by 3rd party kernel display
+	class device driver
+*/
+PVRSRV_ERROR DCExampleInit(IMG_UINT32 *puiNumDevices)
+{
+	DCEX_BUFFER *psBuffer;
+	DCEX_DEVICE *psDeviceData;
+	PVRSRV_ERROR eError;
+#if defined(LMA)
+	PPVRSRV_DEVICE_NODE psDevNode;
+	IMG_UINT64 ui64BufferCount;
+#endif
+	const DC_EXAMPLE_MODULE_PARAMETERS *psModuleParams;
+	IMG_UINT32 uiCurDev;
+	IMG_UINT32 uiNumDevicesConfigured;
+
+	DC_OSSetDrvName(DRVNAME);
+	dllist_init(&g_sDeviceDataListHead);
+
+	eError = DC_OSMutexCreate(&g_hDevDataListLock);
+	if (PVRSRV_OK != eError)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create g_hDevDataListLock (%d)\n", __func__, eError);
+		goto fail_nolocks;
+	}
+
+	/* Check the module params and setup global buffer size state */
+	if (!CheckBufferDimensions())
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psModuleParams = DCExampleGetModuleParameters();
+	if (NULL == psModuleParams)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, ": Cannot fetch module parameters\n");
+		return PVRSRV_ERROR_NO_DEVICEDATA_FOUND;
+	}
+
+	/* Get the number of physically registered devices in the system. */
+	uiNumDevicesConfigured = psModuleParams->ui32NumDevices;
+
+	DCEX_DEBUG_PRINT("%s: Num Registered = %u\n", __func__, uiNumDevicesConfigured);
+
+	for (uiCurDev = 0; uiCurDev < uiNumDevicesConfigured; uiCurDev++)
+	{
+
+		/*
+			If the display controller hasn't already been initialised elsewhere
+			in the system then it should be initialised here.
+
+			Create the private data structure (psDeviceData) and store all the
+			device specific data we will need later (e.g. pointer to mapped
+			registered)
+			This device specific private data will be passed into our callbacks
+			so we don't need global data and can have more than one display
+			controller driven by the same driver (we would just create an
+			"instance" of a device
+			by registering the same callbacks with different private data)
+		*/
+
+		DCEX_DEBUG_PRINT("%s: Initialising device %u\n", __func__, uiCurDev);
+
+		psDeviceData = DC_OSCallocMem(sizeof(DCEX_DEVICE));
+		if (psDeviceData == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_devicealloc;
+		}
+
+		dllist_init(&psDeviceData->sListNode);
+		dllist_init(&psDeviceData->sBufListNode);
+
+		eError = DC_OSMutexCreate(&psDeviceData->hBufListLock);
+		if (PVRSRV_OK != eError)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create hBufListLock (%d)\n",  __func__, eError);
+			goto fail_servicesconnectionclose;
+		}
+
+		eError = DC_OSPVRServicesConnectionOpen(&psDeviceData->hPVRServicesConnection);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to open connection to PVR Services (%d)\n", __func__, eError);
+			goto fail_servicesconnectionclose;
+		}
+
+		eError = DC_OSPVRServicesSetupFuncs(psDeviceData->hPVRServicesConnection, &psDeviceData->sPVRServicesFuncs);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to setup PVR Services function table (%d)\n", __func__, eError);
+			goto fail_servicesconnectionclose;
+		}
+
+#if defined(LMA)
+		psDevNode = psDeviceData->sPVRServicesFuncs.pfnGetDeviceInstance(uiCurDev);
+		PVR_GOTO_IF_INVALID_PARAM(psDevNode != NULL, eError, fail_devicealloc);
+		/*
+			If the display is using card memory then we need to know
+			where that memory is so we have to acquire the heap we want
+			to use (a carveout of the card memory) so we can get it's address
+		*/
+		eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapAcquireByUsage(
+					PHYS_HEAP_USAGE_DISPLAY,
+					psDevNode,
+					&psDeviceData->psPhysHeap);
+
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_heapacquire;
+		}
+
+		/* Verify we're operating on a LMA heap */
+		DC_ASSERT(psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetType(psDeviceData->psPhysHeap) == PHYS_HEAP_TYPE_LMA);
+
+		eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetCpuPAddr(
+					psDeviceData->psPhysHeap,
+					&psDeviceData->sDispStartAddr);
+
+		DC_ASSERT(eError == PVRSRV_OK);
+
+		eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetSize(
+					psDeviceData->psPhysHeap,
+					&psDeviceData->uiDispMemSize);
+
+		DC_ASSERT(eError == PVRSRV_OK);
+#endif
+		/*
+			If the display driver has a system surface create the buffer structure
+			that describes it here.
+
+			Note:
+			All this data and the buffer should be queried from the OS, but in this
+			example we don't have the ability to fetch this data from an OS driver,
+			so we create the data.
+		*/
+		psBuffer = DC_OSCallocMem(sizeof(DCEX_BUFFER));
+
+		if (psBuffer == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_bufferalloc;
+		}
+
+		dllist_init(&psBuffer->sListNode);
+
+		DC_OSAtomicWrite(&psBuffer->i32RefCount, 1);
+		psBuffer->ePixFormat = ePixelFormat;
+		psBuffer->ui32Width = psModuleParams->ui32Width;
+		psBuffer->ui32Height = psModuleParams->ui32Height;
+		psBuffer->ui32ByteStride = ui32ByteStride;
+		psBuffer->ui32Size = psBuffer->ui32Height * psBuffer->ui32ByteStride;
+		psBuffer->ui32Size += (psBuffer->ui32Size >> 4); /* Speculative extra 6.25% for FBC overhead. */
+		psBuffer->ui32Size = (psBuffer->ui32Size + DC_OSGetPageSize() - 1) & DC_OSGetPageMask();
+		psBuffer->psDevice = psDeviceData;
+		psBuffer->bIsSysBuffer = IMG_TRUE;
+
+#if defined(LMA)
+		/* Simple allocator, assume all buffers are going to be the same size. */
+		ui64BufferCount = psDeviceData->uiDispMemSize;
+		ui64BufferCount = DC_OSDiv64(ui64BufferCount, psBuffer->ui32Size);
+
+		psDeviceData->ui32BufferCount = (IMG_UINT32) ui64BufferCount;
+		DC_ASSERT((IMG_UINT32) ui64BufferCount == psDeviceData->ui32BufferCount);
+
+		if (psDeviceData->ui32BufferCount > 32)
+		{
+			psDeviceData->ui32BufferCount = 32;
+		}
+
+		psDeviceData->ui32BufferSize = psBuffer->ui32Size;
+		psDeviceData->ui32BufferUseMask = 0;
+
+		DC_OSDebugPrintf(DBGLVL_INFO, " Buffers: %u (%u bytes each)\n",
+		                 psDeviceData->ui32BufferCount, psDeviceData->ui32BufferSize);
+
+		psBuffer->uiAllocHandle = _DCExampleAllocLMABuffer(psDeviceData);
+		if (psBuffer->uiAllocHandle == 0)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_buffermemalloc;
+		}
+		DCEX_DEBUG_PRINT("Allocate system buffer handle = %p\n", psBuffer->uiAllocHandle);
+#else
+		psBuffer->pvAllocHandle = DCExampleVirtualAllocUncached(psBuffer->ui32Size);
+		if (psBuffer->pvAllocHandle == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_buffermemalloc;
+		}
+#endif
+		DCEX_DEBUG_PRINT("Allocate system buffer = %p\n", psBuffer);
+
+		/* Initialise DC Function Table */
+		DC_OSMemSet(&sDCFunctions, 0, sizeof(sDCFunctions));
+
+		sDCFunctions.pfnGetInfo					= DCExampleGetInfo;
+		sDCFunctions.pfnPanelQueryCount			= DCExamplePanelQueryCount;
+		sDCFunctions.pfnPanelQuery				= DCExamplePanelQuery;
+		sDCFunctions.pfnFormatQuery				= DCExampleFormatQuery;
+		sDCFunctions.pfnDimQuery				= DCExampleDimQuery;
+		sDCFunctions.pfnSetBlank				= NULL;
+		sDCFunctions.pfnSetVSyncReporting		= NULL;
+		sDCFunctions.pfnLastVSyncQuery			= NULL;
+		sDCFunctions.pfnContextCreate			= DCExampleContextCreate;
+		sDCFunctions.pfnContextDestroy			= DCExampleContextDestroy;
+		sDCFunctions.pfnContextConfigure		= DCExampleContextConfigure;
+		sDCFunctions.pfnContextConfigureCheck	= DCExampleContextConfigureCheck;
+		sDCFunctions.pfnBufferAlloc				= DCExampleBufferAlloc;
+		sDCFunctions.pfnBufferAcquire			= DCExampleBufferAcquire;
+		sDCFunctions.pfnBufferRelease			= DCExampleBufferRelease;
+		sDCFunctions.pfnBufferFree				= DCExampleBufferFree;
+#if !defined(LMA)
+		sDCFunctions.pfnBufferImport			= DCExampleBufferImport;
+#endif
+		sDCFunctions.pfnBufferMap				= DCExampleBufferMap;
+		sDCFunctions.pfnBufferUnmap				= DCExampleBufferUnmap;
+		sDCFunctions.pfnBufferSystemAcquire		= DCExampleBufferSystemAcquire;
+		sDCFunctions.pfnBufferSystemRelease		= DCExampleBufferSystemRelease;
+#if defined(INTEGRITY_OS)
+		sDCFunctions.pfnAcquireKernelMappingData = DCExampleAcquireKernelMappingData;
+#endif
+
+		/*
+			Register our DC driver with services
+		*/
+		eError = psDeviceData->sPVRServicesFuncs.pfnDCRegisterDevice(
+					&sDCFunctions,
+					MAX_COMMANDS_INFLIGHT,
+					psDeviceData,
+					&psDeviceData->hSrvHandle);
+
+		/*
+		    We may fail to register if we've had more instances enabled than
+		    there are physical devices available. If this is not the first
+		    device we should allow access to all of the devices currently
+		    registered.
+		 */
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_WARNING,
+			                 "pfnDCRegisterDevice failed (%d). %u devices registered\n",
+			                  eError, uiCurDev);
+			if (uiCurDev > 0)
+			{
+				/* Free psSystemBuffer and psDeviceData */
+#if defined(LMA)
+				_DCExampleFreeLMABuffer(psBuffer->psDevice,
+				                        psBuffer->uiAllocHandle);
+				psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+#else
+				DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+				DC_OSFreeMem(psBuffer);
+				DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+				DC_OSFreeMem(psDeviceData);
+				/* Allow access to successfully discovered devices */
+				if (puiNumDevices)
+				{
+					*puiNumDevices = uiCurDev;
+				}
+				return PVRSRV_OK;
+			}
+			else
+			{
+				goto fail_register;
+			}
+		}
+
+		/* Save the device data somewhere we can retrieve it */
+		DC_OSMutexLock(g_hDevDataListLock);
+		dllist_add_to_tail(&g_sDeviceDataListHead, &psDeviceData->sListNode);
+		DC_OSMutexUnlock(g_hDevDataListLock);
+
+		/* Store the system buffer on the device hook */
+		DC_OSMutexLock(psDeviceData->hBufListLock);
+		dllist_add_to_tail(&psDeviceData->sBufListNode, &psBuffer->sListNode);
+		DC_OSMutexUnlock(psDeviceData->hBufListLock);
+	}	/* End of loop over # devices to create */
+
+	if (puiNumDevices)
+	{
+		*puiNumDevices = uiNumDevicesConfigured;
+	}
+	return PVRSRV_OK;
+
+fail_register:
+	/*
+	 * Warning: Here we are aborting the initialisation and we may have
+	 * partially allocated some data. On entry psDeviceData refers to the
+	 * most recent 'in-progress' device allocation as does 'psBuffer' refer
+	 * to the in-progress buffer allocation.
+	 * We need to free these first, and then clean up any already established
+	 * contents which will be hanging from the DLLIST_NODEs
+	 */
+	if (!dllist_node_is_in_list(&psBuffer->sListNode))
+	{
+#if defined(LMA)
+		_DCExampleFreeLMABuffer(psBuffer->psDevice, psBuffer->uiAllocHandle);
+#else
+		DCExampleVirtualFree(psBuffer->pvAllocHandle);
+#endif
+	}
+
+fail_buffermemalloc:
+	/* Free all buffer allocations */
+	if (!dllist_node_is_in_list(&psBuffer->sListNode))
+	{
+		DC_OSFreeMem(psBuffer);
+	}
+
+fail_bufferalloc:
+	/* Release all Heaps */
+#if defined(LMA)
+	if (!dllist_node_is_in_list(&psDeviceData->sListNode))
+	{
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+	}
+fail_heapacquire:
+#endif
+
+	/* Close all connection handles */
+	if (!dllist_node_is_in_list(&psDeviceData->sListNode))
+	{
+		DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+	}
+
+fail_servicesconnectionclose:
+	/* Release all OS memory */
+	if (!dllist_node_is_in_list(&psDeviceData->sListNode))
+	{
+		DC_OSFreeMem(psDeviceData);
+	}
+
+	/*
+	 * Now we must clean up any previously completed allocations. These will,
+	 * if present, be on the DLLIST_NODE lists for g_sDeviceDataListHead and
+	 * the incorporated sBufListNode entries within.
+	 */
+	DCExampleReleaseDriverData(IMG_FALSE);
+
+fail_devicealloc:
+	DC_OSMutexDestroy(g_hDevDataListLock);
+
+fail_nolocks:
+	return eError;
+}
+
+void DCExampleDeinit(void)
+{
+
+	DCExampleReleaseDriverData(IMG_TRUE);
+
+	/* Free the global list locks. */
+	DC_OSMutexDestroy(g_hDevDataListLock);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..22ad43c75345
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Kbuild.mk
@@ -0,0 +1,57 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-y += \
+ -I$(TOP)/services/display/include \
+ -I$(TOP)/services/display/dc_example/include
+ifeq ($(PVR_ARCH),rogue)
+ccflags-y += \
+ -I$(TOP)/include/public \
+ -I$(TOP)/include/$(PVR_ARCH)/public
+endif
+
+dc_example-y += \
+ services/display/dc_example/common/dc_example.o \
+ services/display/dc_example/env/linux/ex_osfuncs.o \
+ services/display/common/env/linux/dc_osfuncs.o
+
+ifneq ($(W),1)
+CFLAGS_services/display/dc_example/common/dc_example.o := -Werror
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Linux.mk
new file mode 100644
index 000000000000..ad36f6c8f267
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := dc_example
+
+dc_example_type := kernel_module
+dc_example_target := dc_example.ko
+dc_example_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/ex_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/ex_osfuncs.c
new file mode 100644
index 000000000000..fdc77551feee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/env/linux/ex_osfuncs.c
@@ -0,0 +1,407 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+#include <linux/version.h>
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#include "dc_osfuncs.h"
+#include "dc_example.h"
+
+#ifndef DC_EXAMPLE_WIDTH
+#define DC_EXAMPLE_WIDTH 640
+#endif
+
+#ifndef DC_EXAMPLE_HEIGHT
+#define DC_EXAMPLE_HEIGHT 480
+#endif
+
+#ifndef DC_EXAMPLE_BIT_DEPTH
+#define DC_EXAMPLE_BIT_DEPTH 32
+#endif
+
+#ifndef DC_EXAMPLE_FORMAT
+#define DC_EXAMPLE_FORMAT 0
+#endif
+
+#ifndef DC_EXAMPLE_MEMORY_LAYOUT
+#define DC_EXAMPLE_MEMORY_LAYOUT 0
+#endif
+
+#ifndef DC_EXAMPLE_FBC_FORMAT
+#define DC_EXAMPLE_FBC_FORMAT 0
+#endif
+
+#ifndef DC_EXAMPLE_REFRESH_RATE
+#define DC_EXAMPLE_REFRESH_RATE 60
+#endif
+
+#ifndef DC_EXAMPLE_DPI
+#define DC_EXAMPLE_DPI 160
+#endif
+
+#ifndef DC_EXAMPLE_NUM_DEVICES
+#define DC_EXAMPLE_NUM_DEVICES 1
+#endif
+
+#define DC_EXAMPLE_DEBUGFS_NUM_DEVICES "num_devices"
+
+static struct dentry	*g_psDebugFSEntryDir;	/* Top-level handle */
+static struct dentry	*g_psNumDevicesEntry;	/* 'num_devices' entry */
+
+static IMG_UINT32 g_uiNumDCEXDevs;			/* Number of valid devices */
+
+DC_EXAMPLE_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32Width = DC_EXAMPLE_WIDTH,
+	.ui32Height = DC_EXAMPLE_HEIGHT,
+	.ui32Depth = DC_EXAMPLE_BIT_DEPTH,
+	.ui32Format = DC_EXAMPLE_FORMAT,
+	.ui32MemLayout = DC_EXAMPLE_MEMORY_LAYOUT,
+	.ui32FBCFormat = DC_EXAMPLE_FBC_FORMAT,
+	.ui32RefreshRate = DC_EXAMPLE_REFRESH_RATE,
+	.ui32XDpi = DC_EXAMPLE_DPI,
+	.ui32YDpi = DC_EXAMPLE_DPI,
+	.ui32NumDevices = DC_EXAMPLE_NUM_DEVICES
+};
+
+module_param_named(width,	sModuleParams.ui32Width, uint, S_IRUGO);
+module_param_named(height,	sModuleParams.ui32Height, uint, S_IRUGO);
+module_param_named(depth,	sModuleParams.ui32Depth, uint, S_IRUGO);
+module_param_named(format,	sModuleParams.ui32Format, uint, S_IRUGO);
+module_param_named(memlayout,	sModuleParams.ui32MemLayout, uint, S_IRUGO);
+module_param_named(fbcformat,	sModuleParams.ui32FBCFormat, uint, S_IRUGO);
+module_param_named(refreshrate,	sModuleParams.ui32RefreshRate, uint, S_IRUGO);
+module_param_named(xdpi,	sModuleParams.ui32XDpi, uint, S_IRUGO);
+module_param_named(ydpi,	sModuleParams.ui32YDpi, uint, S_IRUGO);
+module_param_named(num_devices,	sModuleParams.ui32NumDevices, uint, S_IRUGO);
+MODULE_PARM_DESC(num_devices,
+        "Number of display devices to register (default: 1 - max: 16)");
+
+const DC_EXAMPLE_MODULE_PARAMETERS *DCExampleGetModuleParameters(void)
+{
+	return &sModuleParams;
+}
+
+static int NumDevicesOpen(struct inode *psInode, struct file *psFile)
+{
+	psFile->private_data = psInode->i_private;
+
+	return 0;
+}
+
+static ssize_t NumDevicesRead(struct file *psFile,
+                              char __user *psUserBuffer,
+                              size_t uiCount, loff_t *puiPosition)
+{
+	char aszBuffer[]="XX\n";
+	loff_t uiPosition = *puiPosition;
+	size_t uiBufferSize = ARRAY_SIZE(aszBuffer);
+	size_t uiLocCount;
+
+	PVR_UNREFERENCED_PARAMETER(psFile);
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	(void) DC_OSSNPrintf(aszBuffer, sizeof(aszBuffer), "%u", g_uiNumDCEXDevs);
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+
+	if (copy_to_user(psUserBuffer, &aszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static const struct file_operations gsDCEXNumDevicesFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = NumDevicesOpen,
+	.read = NumDevicesRead,
+	.llseek = default_llseek
+};
+
+typedef struct _DC_EXAMPLE_PAGE_MAPPING
+{
+	void         *pvVirtAddr;
+	unsigned int  uiPageCnt;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	struct page **ppsPageArray;
+#endif
+} DC_EXAMPLE_PAGE_MAPPING;
+
+static void pvr_vfree(DC_EXAMPLE_PAGE_MAPPING *psMap)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	unsigned int i;
+#endif
+
+	if (!psMap)
+	{
+		return;
+	}
+
+	vfree(psMap->pvVirtAddr);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	/* Free up all the pages. */
+	for (i = 0; i < psMap->uiPageCnt; i++)
+	{
+		if (psMap->ppsPageArray[i])
+		{
+			__free_page(psMap->ppsPageArray[i]);
+		}
+	}
+
+	kfree(psMap->ppsPageArray);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) */
+
+	kfree(psMap);
+}
+
+static void *pvr_vmalloc(size_t uiSize, gfp_t gfp_mask, pgprot_t prot)
+{
+	DC_EXAMPLE_PAGE_MAPPING *psMap;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	unsigned int i;
+#endif
+
+	psMap = kzalloc(sizeof(*psMap), GFP_KERNEL);
+	if (!psMap)
+	{
+		return NULL;
+	}
+
+	psMap->uiPageCnt = DIV_ROUND_UP(uiSize, PAGE_SIZE);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	psMap->ppsPageArray = kcalloc(psMap->uiPageCnt, sizeof(*psMap->ppsPageArray), GFP_KERNEL);
+	if (!psMap->ppsPageArray)
+	{
+		goto err_pvr_vfree;
+	}
+
+	/* Allocate kernel pages 1 by 1 - Keep reference in the "pages" struct. */
+	for (i = 0; i < psMap->uiPageCnt; i++)
+	{
+		psMap->ppsPageArray[i] = alloc_page(gfp_mask);
+		if (!psMap->ppsPageArray[i])
+		{
+			goto err_pvr_vfree;
+		}
+	}
+
+	/* Map the pages into contiguous VM space. */
+	psMap->pvVirtAddr = vmap(psMap->ppsPageArray, (uiSize >> PAGE_SHIFT), VM_MAP, prot);
+#else
+	psMap->pvVirtAddr = __vmalloc(uiSize, gfp_mask, prot);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) */
+
+	if (!psMap->pvVirtAddr)
+	{
+		goto err_pvr_vfree;
+	}
+
+	return (void *) psMap;
+
+err_pvr_vfree:
+	pvr_vfree(psMap);
+	return NULL;
+}
+
+void *DCExampleVirtualAllocUncached(size_t uiSize)
+{
+	unsigned int gfp_flags = GFP_USER | __GFP_NOWARN | __GFP_NOMEMALLOC;
+
+#if defined(CONFIG_X86_64)
+	gfp_flags |= __GFP_DMA32;
+#else
+#if !defined(CONFIG_X86_PAE)
+	gfp_flags |= __GFP_HIGHMEM;
+#endif
+#endif
+
+	return pvr_vmalloc(uiSize, gfp_flags, pgprot_noncached(PAGE_KERNEL));
+}
+
+IMG_BOOL DCExampleVirtualFree(void *pvAllocHandle)
+{
+	pvr_vfree((DC_EXAMPLE_PAGE_MAPPING *) pvAllocHandle);
+
+	/* vfree does not return a value, so all we can do is hard code IMG_TRUE */
+	return IMG_TRUE;
+}
+
+#define	VMALLOC_TO_PAGE_PHYS(vAddr) page_to_phys(vmalloc_to_page(vAddr))
+
+PVRSRV_ERROR DCExampleGetLinAddr(void *pvAllocHandle, IMG_CPU_VIRTADDR *ppvLinAddr)
+{
+	DC_EXAMPLE_PAGE_MAPPING *psMap     = (DC_EXAMPLE_PAGE_MAPPING *) pvAllocHandle;
+	IMG_CPU_VIRTADDR         pvLinAddr = psMap->pvVirtAddr;
+
+	if (ppvLinAddr != NULL)
+	{
+		*ppvLinAddr = pvLinAddr;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DCExampleGetDevPAddrs(void *pvAllocHandle, IMG_DEV_PHYADDR *pasDevPAddr,
+				   uint32_t uiPageNo, size_t uiSize)
+{
+	DC_EXAMPLE_PAGE_MAPPING *psMap = (DC_EXAMPLE_PAGE_MAPPING *) pvAllocHandle;
+	unsigned long ulPages      = DC_OS_BYTES_TO_PAGES(uiSize);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	IMG_CPU_VIRTADDR pvLinAddr = (IMG_CPU_VIRTADDR) IMG_OFFSET_ADDR(psMap->pvVirtAddr, uiPageNo * PAGE_SIZE);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)) */
+	int           i;
+
+	if (ulPages > psMap->uiPageCnt)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	for (i = 0; i < ulPages; i++)
+	{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+		pasDevPAddr[i].uiAddr = VMALLOC_TO_PAGE_PHYS(pvLinAddr);
+		pvLinAddr += PAGE_SIZE;
+#else
+		pasDevPAddr[i].uiAddr = page_to_phys(psMap->ppsPageArray[uiPageNo + i]);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)) */
+	}
+
+	return PVRSRV_OK;
+}
+
+static int __init dc_example_init(void)
+{
+	if (DCExampleInit(&g_uiNumDCEXDevs) != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+	/* Create a debugfs entry to return the number of devices discovered */
+	g_psDebugFSEntryDir = debugfs_create_dir(MODNAME, NULL);
+	if (IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs root directory "
+		       "(debugfs entries won't be available)\n", __func__, MODNAME);
+		g_psDebugFSEntryDir = NULL;
+	}
+
+	g_psNumDevicesEntry = debugfs_create_file(DC_EXAMPLE_DEBUGFS_NUM_DEVICES,
+	                                          S_IFREG | S_IRUGO,
+	                                          g_psDebugFSEntryDir,
+	                                          &sModuleParams.ui32NumDevices,
+	                                          &gsDCEXNumDevicesFileOps);
+
+	if (IS_ERR_OR_NULL(g_psNumDevicesEntry))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs file "
+		       "(debugfs entries won't be available)\n", __func__,
+		       DC_EXAMPLE_DEBUGFS_NUM_DEVICES);
+		g_psNumDevicesEntry = NULL;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ Function Name: dc_example_deinit
+ Description  : Remove the driver from the kernel.
+
+                __exit places the function in a special memory section that
+                the kernel frees once the function has been run.  Refer also
+                to module_exit() macro call below.
+******************************************************************************/
+static void __exit dc_example_deinit(void)
+{
+	DCExampleDeinit();
+
+	/* Remove the debugfs entry for the number of devices */
+	if (g_psNumDevicesEntry != NULL)
+	{
+		debugfs_remove(g_psNumDevicesEntry);
+		g_psNumDevicesEntry = NULL;
+	}
+
+	if (g_psDebugFSEntryDir != NULL)
+	{
+		debugfs_remove(g_psDebugFSEntryDir);
+		g_psDebugFSEntryDir = NULL;
+	}
+}
+
+module_init(dc_example_init);
+module_exit(dc_example_deinit);
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/include/dc_example.h b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/include/dc_example.h
new file mode 100644
index 000000000000..e74362249791
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_example/include/dc_example.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_EXAMPLE_H)
+#define DC_EXAMPLE_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(LMA)
+	#define DRVNAME	"dc_example_LMA"
+#else
+	#define DRVNAME	"dc_example_UMA"
+#endif
+
+#define MODNAME "dc_example"
+
+/******************************************************************************
+ * dc_example OS functions
+ *****************************************************************************/
+
+/* Module parameters */
+typedef struct _DC_EXAMPLE_MODULE_PARAMETERS_
+{
+	IMG_UINT32 ui32Width;
+	IMG_UINT32 ui32Height;
+	IMG_UINT32 ui32Depth;
+	IMG_UINT32 ui32Format;
+	IMG_UINT32 ui32MemLayout;
+	IMG_UINT32 ui32FBCFormat;
+	IMG_UINT32 ui32RefreshRate;
+	IMG_UINT32 ui32XDpi;
+	IMG_UINT32 ui32YDpi;
+	IMG_UINT32 ui32NumDevices;
+} DC_EXAMPLE_MODULE_PARAMETERS;
+
+const DC_EXAMPLE_MODULE_PARAMETERS *DCExampleGetModuleParameters(void);
+
+void        *DCExampleVirtualAllocUncached(size_t uiSize);
+IMG_BOOL     DCExampleVirtualFree(void *pvAllocHandle);
+PVRSRV_ERROR DCExampleGetLinAddr(void *pvAllocHandle, IMG_CPU_VIRTADDR *ppvLinAddr);
+PVRSRV_ERROR DCExampleGetDevPAddrs(void *pvAllocHandle, IMG_DEV_PHYADDR *pasDevPAddr,
+				   uint32_t uiPageNo, size_t uiSize);
+#if defined(INTEGRITY_OS)
+PVRSRV_ERROR DCExampleOSAcquireKernelMappingData(void *pvAllocHandle, IMG_HANDLE *phMapping, void **ppPhysAddr);
+#endif
+
+/******************************************************************************
+ * dc_example functions
+ *****************************************************************************/
+
+PVRSRV_ERROR DCExampleInit(IMG_UINT32 *puiNumDevices);
+void DCExampleDeinit(void);
+
+#endif /* !defined(DC_EXAMPLE_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Kbuild.mk
new file mode 100644
index 000000000000..10b32e2870d1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Kbuild.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-y += \
+ -I$(TOP)/services/display/dc_fbdev
+
+dc_fbdev-y += \
+	services/display/dc_fbdev/dc_fbdev.o
+
+ifneq ($(W),1)
+#CFLAGS_services/display/dc_fbdev/dc_fbdev.o := -Werror
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Linux.mk
new file mode 100644
index 000000000000..a26c521e050b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := dc_fbdev
+
+dc_fbdev_type := kernel_module
+dc_fbdev_target := dc_fbdev.ko
+dc_fbdev_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/dc_fbdev.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/dc_fbdev.c
new file mode 100644
index 000000000000..5bc2a8eb35b8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_fbdev/dc_fbdev.c
@@ -0,0 +1,941 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/console.h>
+#include <linux/module.h>
+#include <linux/fb.h>
+
+#include "kerneldisplay.h"
+#include "powervr/imgpixfmts.h"
+#include "pvrmodule.h" /* for MODULE_LICENSE() */
+
+#if !defined(CONFIG_FB)
+#error dc_fbdev needs Linux framebuffer support. Enable it in your kernel.
+#endif
+
+#define DRVNAME					"dc_fbdev"
+#define MAX_COMMANDS_IN_FLIGHT	2
+
+#if defined(DC_FBDEV_NUM_PREFERRED_BUFFERS)
+#define NUM_PREFERRED_BUFFERS	DC_FBDEV_NUM_PREFERRED_BUFFERS
+#else
+#define NUM_PREFERRED_BUFFERS	2
+#endif
+
+#define FALLBACK_REFRESH_RATE		60
+#define FALLBACK_DPI				160
+
+typedef struct
+{
+	IMG_HANDLE			hSrvHandle;
+	IMG_UINT32			ePixFormat;
+	struct fb_info		*psLINFBInfo;
+	bool				bCanFlip;
+}
+DC_FBDEV_DEVICE;
+
+typedef struct
+{
+	DC_FBDEV_DEVICE		*psDeviceData;
+	IMG_HANDLE			hLastConfigData;
+	IMG_UINT32		ui32AllocUseMask;
+}
+DC_FBDEV_CONTEXT;
+
+typedef struct
+{
+	DC_FBDEV_CONTEXT	*psDeviceContext;
+	IMG_UINT32			ui32Width;
+	IMG_UINT32			ui32Height;
+	IMG_UINT32			ui32ByteStride;
+	IMG_UINT32			ui32BufferID;
+}
+DC_FBDEV_BUFFER;
+
+static IMG_UINT32 gui32fb_devminor = 0;
+module_param_named(fb_devminor, gui32fb_devminor, uint, S_IRUGO | S_IWUSR);
+
+static DC_FBDEV_DEVICE *gpsDeviceData;
+
+#if defined(DC_FBDEV_FORCE_CONTEXT_CLEAN)
+static
+void DC_FBDEV_Clean(IMG_HANDLE hDisplayContext, IMG_HANDLE *ahBuffers)
+{
+	DC_FBDEV_DEVICE *psDeviceData;
+	DC_FBDEV_BUFFER *psBuffer;
+	IMG_CPU_PHYADDR sCPUPhysStart;
+	void *pvOSDevice, *pvVirtStart;
+	IMG_UINT32 ui32ByteSize, *pbPhysEnd;
+	uintptr_t uiStartAddr, uiEndAddr, uiMaxLen;
+
+	/* validate incoming parameters */
+	if (!hDisplayContext ||
+		!ahBuffers       ||
+		!((DC_FBDEV_CONTEXT *)hDisplayContext)->psDeviceData ||
+		!((DC_FBDEV_CONTEXT *)hDisplayContext)->psDeviceData->psLINFBInfo)
+	{
+		return;
+	}
+
+	/* lookup display context data, buffer and kernel device handle */
+	psDeviceData = ((DC_FBDEV_CONTEXT *)hDisplayContext)->psDeviceData;
+	pvOSDevice = gpsDeviceData->psLINFBInfo->dev;
+	PVR_UNREFERENCED_PARAMETER(pvOSDevice);
+	psBuffer = ahBuffers[0];
+
+	/* calc. framebuffer start/end addresses */
+	ui32ByteSize = psBuffer->ui32ByteStride * psBuffer->ui32Height;
+	uiStartAddr  = (uintptr_t)psDeviceData->psLINFBInfo->screen_base;
+	uiStartAddr += psBuffer->ui32BufferID * ui32ByteSize;
+	uiMaxLen     = psDeviceData->psLINFBInfo->fix.smem_len;
+	uiMaxLen    -= psBuffer->ui32BufferID * ui32ByteSize;
+	uiEndAddr    = uiStartAddr + ui32MaxLen;
+
+	/* outer framebuffer inter-page loop */
+	pvVirtStart  = (IMG_BYTE *)((uintptr_t)uiStartAddr & (~(PAGE_SIZE-1)));
+	while ((uintptr_t)pvVirtStart < uiEndAddr)
+	{
+#if !(defined(ARM64) && defined(X86))
+		/* crossing (possibly discontiguous) page boundary, translate virt -> phys page */
+		sCPUPhysStart.uiAddr =
+		page_to_phys(vmalloc_to_page(pvVirtStart));
+		pbPhysEnd = (IMG_UINT32 *)((uintptr_t)sCpuPAddr.uiAddr & (~(PAGE_SIZE-1)));
+#endif
+
+		/* inner intra-page loop */
+		while (pvVirtStart < (pvVirtStart + PAGE_SIZE))
+		{
+			/* per-arch d-cache flush mechanism */
+#if defined(CONFIG_X86)
+			asm volatile("clflush %0" : "+m" (*pvVirtStart));
+#elif defined(CONFIG_ARM64)
+			asm volatile ("dc civac, %0" :: "r" (pvVirtStart));
+#elif defined(CONFIG_ARM)
+			unsigned long uiLockFlags = {0};
+			spinlock_t spinlock = {0};
+			flush_cache_all();
+#if defined(CONFIG_OUTER_CACHE)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0))
+			spin_lock_init(&spinlock);
+			spin_lock_irqsave(&spinlock, uiLockFlags);
+			outer_clean_range(0, ULONG_MAX);
+			spin_unlock_irqrestore(&spinlock, uiLockFlags);
+#else
+			outer_clean_range(0, ULONG_MAX)
+#endif
+#endif
+			PVR_UNREFERENCED_PARAMETER(uiLockFlags);
+			PVR_UNREFERENCED_PARAMETER(spinlock);
+			return;
+#elif defined(CONFIG_MIPS)
+			dma_cache_sync(pvOSDevice, (void *)pvVirtStart, PAGE_SIZE, DMA_TO_DEVICE);
+#elif defined(CONFIG_META)
+			writeback_dcache_region((void *)pbPhysEnd, PAGE_SIZE);
+#else
+			PVR_ASSERT(0);
+#endif
+
+			/* move to next d-cache line/page */
+#if defined(ARM64) || defined(X86)
+			pvVirtStart += cache_line_size();
+#else
+			pvVirtStart += PAGE_SIZE;
+#endif
+		}
+	}
+}
+#endif
+
+static
+void DC_FBDEV_GetInfo(IMG_HANDLE hDeviceData,
+                      DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	strncpy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	psDisplayInfo->ui32MinDisplayPeriod	= 0;
+	psDisplayInfo->ui32MaxDisplayPeriod	= 1;
+	psDisplayInfo->ui32MaxPipes			= 1;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_PanelQueryCount(IMG_HANDLE hDeviceData,
+									  IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+	*pui32NumPanels = 1;
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_PanelQuery(IMG_HANDLE hDeviceData,
+								 IMG_UINT32 ui32PanelsArraySize,
+								 IMG_UINT32 *pui32NumPanels,
+								 PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	DC_FBDEV_DEVICE *psDeviceData = hDeviceData;
+	struct fb_var_screeninfo *psVar = &psDeviceData->psLINFBInfo->var;
+	struct fb_var_screeninfo sVar = { .pixclock = 0 };
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+		return PVRSRV_ERROR_RETRY;
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	*pui32NumPanels = 1;
+
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = psDeviceData->ePixFormat;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width    = psVar->xres;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height   = psVar->yres;
+	psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+	psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+
+	/* Conformant fbdev drivers should have 'var' and mode in sync by now,
+	 * but some don't (like drmfb), so try a couple of different ways to
+	 * get the info before falling back to the default.
+	 */
+	if (psVar->xres > 0 && psVar->yres > 0 && psVar->pixclock > 0)
+		sVar = *psVar;
+	else if (psDeviceData->psLINFBInfo->mode)
+		fb_videomode_to_var(&sVar, psDeviceData->psLINFBInfo->mode);
+
+	/* Override the refresh rate when defined. */
+#ifdef DC_FBDEV_REFRESH
+	psPanelInfo[0].ui32RefreshRate = DC_FBDEV_REFRESH;
+#else
+	if (sVar.xres > 0 && sVar.yres > 0 && sVar.pixclock > 0)
+	{
+		psPanelInfo[0].ui32RefreshRate = 1000000000LU /
+			((sVar.upper_margin + sVar.lower_margin +
+			  sVar.yres + sVar.vsync_len) *
+			 (sVar.left_margin  + sVar.right_margin +
+			  sVar.xres + sVar.hsync_len) *
+			 (sVar.pixclock / 1000));
+	}
+	else
+		psPanelInfo[0].ui32RefreshRate = FALLBACK_REFRESH_RATE;
+#endif
+
+	psPanelInfo[0].ui32XDpi =
+		((int)sVar.width > 0) ? (254000 / sVar.width * psVar->xres / 10000) : FALLBACK_DPI;
+
+	psPanelInfo[0].ui32YDpi	=
+		((int)sVar.height > 0) ? 254000 / sVar.height * psVar->yres / 10000 : FALLBACK_DPI;
+
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_FormatQuery(IMG_HANDLE hDeviceData,
+								  IMG_UINT32 ui32NumFormats,
+								  PVRSRV_SURFACE_FORMAT *pasFormat,
+								  IMG_UINT32 *pui32Supported)
+{
+	DC_FBDEV_DEVICE *psDeviceData = hDeviceData;
+	int i;
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (pasFormat[i].ePixFormat == psDeviceData->ePixFormat)
+			pui32Supported[i]++;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_DimQuery(IMG_HANDLE hDeviceData,
+							   IMG_UINT32 ui32NumDims,
+							   PVRSRV_SURFACE_DIMS *psDim,
+							   IMG_UINT32 *pui32Supported)
+{
+	DC_FBDEV_DEVICE *psDeviceData = hDeviceData;
+	struct fb_var_screeninfo *psVar = &psDeviceData->psLINFBInfo->var;
+	int i;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+		return PVRSRV_ERROR_RETRY;
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (psDim[i].ui32Width  == psVar->xres &&
+		    psDim[i].ui32Height == psVar->yres)
+		    pui32Supported[i]++;
+	}
+
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_ContextCreate(IMG_HANDLE hDeviceData,
+									IMG_HANDLE *hDisplayContext)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	psDeviceContext = kzalloc(sizeof(DC_FBDEV_CONTEXT), GFP_KERNEL);
+	if (!psDeviceContext)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psDeviceContext->psDeviceData = hDeviceData;
+	*hDisplayContext = psDeviceContext;
+
+err_out:
+	return eError;
+}
+
+static PVRSRV_ERROR
+DC_FBDEV_ContextConfigureCheck(IMG_HANDLE hDisplayContext,
+							   IMG_UINT32 ui32PipeCount,
+							   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+							   IMG_HANDLE *ahBuffers)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+	DC_FBDEV_DEVICE *psDeviceData = psDeviceContext->psDeviceData;
+	struct fb_var_screeninfo *psVar = &psDeviceData->psLINFBInfo->var;
+	DC_FBDEV_BUFFER *psBuffer;
+	PVRSRV_ERROR eError;
+
+	if (ui32PipeCount != 1)
+	{
+		eError = PVRSRV_ERROR_DC_TOO_MANY_PIPES;
+		goto err_out;
+	}
+
+	if (!ahBuffers)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_CONFIG;
+		goto err_out;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+	{
+		eError = PVRSRV_ERROR_RETRY;
+		goto err_out;
+	}
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	psBuffer = ahBuffers[0];
+
+	if (pasSurfAttrib[0].sCrop.sDims.ui32Width  != psVar->xres ||
+		pasSurfAttrib[0].sCrop.sDims.ui32Height != psVar->yres ||
+		pasSurfAttrib[0].sCrop.i32XOffset != 0 ||
+		pasSurfAttrib[0].sCrop.i32YOffset != 0)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+		goto err_unlock;
+	}
+
+	if (pasSurfAttrib[0].sDisplay.sDims.ui32Width !=
+		pasSurfAttrib[0].sCrop.sDims.ui32Width ||
+		pasSurfAttrib[0].sDisplay.sDims.ui32Height !=
+		pasSurfAttrib[0].sCrop.sDims.ui32Height ||
+		pasSurfAttrib[0].sDisplay.i32XOffset !=
+		pasSurfAttrib[0].sCrop.i32XOffset ||
+		pasSurfAttrib[0].sDisplay.i32YOffset !=
+		pasSurfAttrib[0].sCrop.i32YOffset)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+		goto err_unlock;
+	}
+
+	if (psBuffer->ui32Width  != psVar->xres &&
+		psBuffer->ui32Height != psVar->yres)
+	{
+		eError = PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+		goto err_unlock;
+	}
+
+	eError = PVRSRV_OK;
+err_unlock:
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+err_out:
+	return eError;
+}
+
+static
+void DC_FBDEV_ContextConfigure(IMG_HANDLE hDisplayContext,
+								   IMG_UINT32 ui32PipeCount,
+								   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+								   IMG_HANDLE *ahBuffers,
+								   IMG_UINT32 ui32DisplayPeriod,
+								   IMG_HANDLE hConfigData)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+	DC_FBDEV_DEVICE *psDeviceData = psDeviceContext->psDeviceData;
+	struct fb_var_screeninfo sVar = psDeviceData->psLINFBInfo->var;
+	int err;
+
+	PVR_UNREFERENCED_PARAMETER(ui32PipeCount);
+	PVR_UNREFERENCED_PARAMETER(pasSurfAttrib);
+	PVR_UNREFERENCED_PARAMETER(ui32DisplayPeriod);
+
+	sVar.yoffset = 0;
+
+	if (ui32PipeCount != 0)
+	{
+		BUG_ON(ahBuffers == NULL);
+
+		if (psDeviceData->bCanFlip)
+		{
+			DC_FBDEV_BUFFER *psBuffer = ahBuffers[0];
+			sVar.yoffset = sVar.yres * psBuffer->ui32BufferID;
+		}
+	}
+
+#if defined(DC_FBDEV_FORCE_CONTEXT_CLEAN)
+	DC_FBDEV_Clean(hDisplayContext, ahBuffers);
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psDeviceData->psLINFBInfo))
+	{
+		return;
+	}
+#else
+	lock_fb_info(psDeviceData->psLINFBInfo);
+#endif
+
+	console_lock();
+
+	/* If we're supposed to be able to flip, but the yres_virtual has been
+	 * changed to an unsupported (smaller) value, we need to change it back
+	 * (this is a workaround for some Linux fbdev drivers that seem to lose any
+	 * modifications to yres_virtual after a blank.)
+	 */
+	if (psDeviceData->bCanFlip &&
+		sVar.yres_virtual < sVar.yres * NUM_PREFERRED_BUFFERS)
+	{
+		sVar.activate = FB_ACTIVATE_NOW;
+		sVar.yres_virtual = sVar.yres * NUM_PREFERRED_BUFFERS;
+
+		err = fb_set_var(psDeviceData->psLINFBInfo, &sVar);
+		if (err)
+			pr_err("fb_set_var failed (err=%d)\n", err);
+	}
+	else
+	{
+		err = fb_pan_display(psDeviceData->psLINFBInfo, &sVar);
+		if (err)
+			pr_err("fb_pan_display failed (err=%d)\n", err);
+	}
+
+	console_unlock();
+	unlock_fb_info(psDeviceData->psLINFBInfo);
+
+	if (psDeviceContext->hLastConfigData)
+	{
+		DCDisplayConfigurationRetired(psDeviceContext->hLastConfigData);
+	}
+
+	if (ui32PipeCount != 0)
+	{
+		psDeviceContext->hLastConfigData = hConfigData;
+	}
+	else
+	{
+		/* If the pipe count is zero, we're tearing down. Don't record
+		 * any new configurations.
+		 */
+		psDeviceContext->hLastConfigData = NULL;
+
+		/* We still need to "retire" this NULL flip as that signals back to
+		 * the DC core that we've finished doing what we need to do and it
+		 * can destroy the display context
+		 */
+		DCDisplayConfigurationRetired(hConfigData);
+	}
+}
+
+static
+void DC_FBDEV_ContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+
+	BUG_ON(psDeviceContext->hLastConfigData != NULL);
+	kfree(psDeviceContext);
+}
+
+static
+IMG_BOOL DC_FBDEV_GetBufferID(DC_FBDEV_CONTEXT *psDeviceContext, IMG_UINT32 *pui32BufferID)
+{
+	IMG_UINT32 ui32IDLimit;
+	IMG_UINT32 ui32BufferID;
+
+	if (psDeviceContext->psDeviceData->bCanFlip)
+	{
+		ui32IDLimit = NUM_PREFERRED_BUFFERS;
+	}
+	else
+	{
+		ui32IDLimit = 1;
+	}
+
+	for (ui32BufferID = 0; ui32BufferID < ui32IDLimit; ++ui32BufferID)
+	{
+		if ((psDeviceContext->ui32AllocUseMask & (1UL << ui32BufferID)) == 0)
+		{
+			psDeviceContext->ui32AllocUseMask |= (1UL << ui32BufferID);
+
+			*pui32BufferID = ui32BufferID;
+
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+
+static
+void DC_FBDEV_PutBufferID(DC_FBDEV_CONTEXT *psDeviceContext, IMG_UINT32 ui32BufferID)
+{
+	psDeviceContext->ui32AllocUseMask &= ~(1UL << ui32BufferID);
+}
+
+#define BYTE_TO_PAGES(range) (((range) + (PAGE_SIZE - 1)) >> PAGE_SHIFT)
+
+static
+PVRSRV_ERROR DC_FBDEV_BufferAlloc(IMG_HANDLE hDisplayContext,
+								  DC_BUFFER_CREATE_INFO *psCreateInfo,
+								  IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+								  IMG_UINT32 *pui32PageCount,
+								  IMG_UINT32 *pui32ByteStride,
+								  IMG_HANDLE *phBuffer)
+{
+	DC_FBDEV_CONTEXT *psDeviceContext = hDisplayContext;
+	DC_FBDEV_DEVICE *psDeviceData = psDeviceContext->psDeviceData;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	PVRSRV_ERROR eError;
+	DC_FBDEV_BUFFER *psBuffer;
+	IMG_UINT32 ui32ByteSize;
+
+	if (psSurfInfo->sFormat.ePixFormat != psDeviceData->ePixFormat)
+	{
+		eError = PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+		goto err_out;
+	}
+
+	psBuffer = kmalloc(sizeof(DC_FBDEV_BUFFER), GFP_KERNEL);
+	if (!psBuffer)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psBuffer->psDeviceContext = psDeviceContext;
+	psBuffer->ui32ByteStride =
+		psSurfInfo->sDims.ui32Width * psCreateInfo->ui32BPP;
+
+	psBuffer->ui32Width = psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height = psSurfInfo->sDims.ui32Height;
+
+	if (!DC_FBDEV_GetBufferID(psDeviceContext, &psBuffer->ui32BufferID))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_free;
+	}
+
+	ui32ByteSize = psBuffer->ui32ByteStride * psBuffer->ui32Height;
+
+	*puiLog2PageSize = PAGE_SHIFT;
+	*pui32PageCount	 = BYTE_TO_PAGES(ui32ByteSize);
+	*pui32ByteStride = psBuffer->ui32ByteStride;
+	*phBuffer	 = psBuffer;
+
+	return PVRSRV_OK;
+
+err_free:
+	kfree(psBuffer);
+
+err_out:
+	return eError;
+}
+
+static
+PVRSRV_ERROR DC_FBDEV_BufferAcquire(IMG_HANDLE hBuffer,
+									IMG_DEV_PHYADDR *pasDevPAddr,
+									void **ppvLinAddr)
+{
+	DC_FBDEV_BUFFER *psBuffer = hBuffer;
+	DC_FBDEV_DEVICE *psDeviceData = psBuffer->psDeviceContext->psDeviceData;
+	IMG_UINT32 ui32ByteSize = psBuffer->ui32ByteStride * psBuffer->ui32Height;
+	uintptr_t uiStartAddr;
+	IMG_UINT32 i, ui32MaxLen;
+
+#if defined(DC_FBDEV_USE_SCREEN_BASE)
+	uiStartAddr = (uintptr_t)psDeviceData->psLINFBInfo->screen_base;
+#else
+	uiStartAddr = psDeviceData->psLINFBInfo->fix.smem_start;
+#endif
+
+	uiStartAddr += psBuffer->ui32BufferID * ui32ByteSize;
+	ui32MaxLen = psDeviceData->psLINFBInfo->fix.smem_len -
+				 psBuffer->ui32BufferID * ui32ByteSize;
+
+	for (i = 0; i < BYTE_TO_PAGES(ui32ByteSize); i++)
+	{
+		BUG_ON(i * PAGE_SIZE >= ui32MaxLen);
+		pasDevPAddr[i].uiAddr = uiStartAddr + (i * PAGE_SIZE);
+#if defined(DC_FBDEV_USE_SCREEN_BASE)
+		pasDevPAddr[i].uiAddr =
+		page_to_phys(vmalloc_to_page((void *)(uintptr_t)pasDevPAddr[i].uiAddr));
+#endif
+	}
+
+	/* We're UMA, so services will do the right thing and make
+	 * its own CPU virtual address mapping for the buffer.
+	 */
+	*ppvLinAddr = NULL;
+
+	return PVRSRV_OK;
+}
+
+static void DC_FBDEV_BufferRelease(IMG_HANDLE hBuffer)
+{
+	PVR_UNREFERENCED_PARAMETER(hBuffer);
+}
+
+static void DC_FBDEV_BufferFree(IMG_HANDLE hBuffer)
+{
+	DC_FBDEV_BUFFER *psBuffer = hBuffer;
+
+	DC_FBDEV_PutBufferID(psBuffer->psDeviceContext, psBuffer->ui32BufferID);
+
+	kfree(psBuffer);
+}
+
+/* If we can flip, we need to make sure we have the memory to do so.
+ *
+ * We'll assume that the fbdev device provides extra space in
+ * yres_virtual for panning; xres_virtual is theoretically supported,
+ * but it involves more work.
+ *
+ * If the fbdev device doesn't have yres_virtual > yres, we'll try
+ * requesting it before bailing. Userspace applications commonly do
+ * this with an FBIOPUT_VSCREENINFO ioctl().
+ *
+ * Another problem is with a limitation in the services DC -- it
+ * needs framebuffers to be page aligned (this is a SW limitation,
+ * the HW can support non-page-aligned buffers). So we have to
+ * check that stride * height for a single buffer is page aligned.
+ */
+
+static bool DC_FBDEV_FlipPossible(struct fb_info *psLINFBInfo)
+{
+	struct fb_var_screeninfo sVar = psLINFBInfo->var;
+	int err;
+
+	if (!psLINFBInfo->fix.xpanstep && !psLINFBInfo->fix.ypanstep &&
+		!psLINFBInfo->fix.ywrapstep)
+	{
+		pr_err("The fbdev device detected does not support ypan/ywrap. "
+			   "Flipping disabled.\n");
+		return false;
+	}
+
+	if ((psLINFBInfo->fix.line_length * sVar.yres) % PAGE_SIZE != 0)
+	{
+		pr_err("Line length (in bytes) x yres is not a multiple of "
+			   "page size. Flipping disabled.\n");
+		return false;
+	}
+
+	/* We might already have enough space */
+	if (sVar.yres * NUM_PREFERRED_BUFFERS <= sVar.yres_virtual)
+		return true;
+
+	pr_err("No buffer space for flipping; asking for more.\n");
+
+	sVar.activate = FB_ACTIVATE_NOW;
+	sVar.yres_virtual = sVar.yres * NUM_PREFERRED_BUFFERS;
+
+	err = fb_set_var(psLINFBInfo, &sVar);
+	if (err)
+	{
+		pr_err("fb_set_var failed (err=%d). Flipping disabled.\n", err);
+		return false;
+	}
+
+	if (sVar.yres * NUM_PREFERRED_BUFFERS > sVar.yres_virtual)
+	{
+		pr_err("Failed to obtain additional buffer space. "
+			   "Flipping disabled.\n");
+		return false;
+	}
+
+	/* Some fbdev drivers allow the yres_virtual modification through,
+	 * but don't actually update the fix. We need the fix to be updated
+	 * and more memory allocated, so we can actually take advantage of
+	 * the increased yres_virtual.
+	 */
+	if (psLINFBInfo->fix.smem_len < psLINFBInfo->fix.line_length * sVar.yres_virtual)
+	{
+		pr_err("'fix' not re-allocated with sufficient buffer space. "
+			   "Flipping disabled.\n");
+		return false;
+	}
+
+	return true;
+}
+
+static int __init DC_FBDEV_init(void)
+{
+	static DC_DEVICE_FUNCTIONS sDCFunctions =
+	{
+		.pfnGetInfo					= DC_FBDEV_GetInfo,
+		.pfnPanelQueryCount			= DC_FBDEV_PanelQueryCount,
+		.pfnPanelQuery				= DC_FBDEV_PanelQuery,
+		.pfnFormatQuery				= DC_FBDEV_FormatQuery,
+		.pfnDimQuery				= DC_FBDEV_DimQuery,
+		.pfnSetBlank				= NULL,
+		.pfnSetVSyncReporting		= NULL,
+		.pfnLastVSyncQuery			= NULL,
+		.pfnContextCreate			= DC_FBDEV_ContextCreate,
+		.pfnContextDestroy			= DC_FBDEV_ContextDestroy,
+		.pfnContextConfigure		= DC_FBDEV_ContextConfigure,
+		.pfnContextConfigureCheck	= DC_FBDEV_ContextConfigureCheck,
+		.pfnBufferAlloc				= DC_FBDEV_BufferAlloc,
+		.pfnBufferAcquire			= DC_FBDEV_BufferAcquire,
+		.pfnBufferRelease			= DC_FBDEV_BufferRelease,
+		.pfnBufferFree				= DC_FBDEV_BufferFree,
+	};
+
+	struct fb_info *psLINFBInfo;
+	IMG_PIXFMT ePixFormat;
+	int err = -ENODEV;
+
+	if (gui32fb_devminor >= FB_MAX)
+	{
+		pr_err("Invalid Linux framebuffer device minor number!\n"
+			   "fb_devminor (%u) >= FB_MAX (%u)",
+				gui32fb_devminor,
+				FB_MAX);
+		goto err_out;
+	}
+
+	psLINFBInfo = registered_fb[gui32fb_devminor];
+	if (!psLINFBInfo)
+	{
+		pr_err("No Linux framebuffer (/dev/fbdev%u) device is registered!\n"
+			   "Check you have a framebuffer driver compiled into your "
+			   "kernel\nand that it is enabled on the cmdline.\n",
+			   gui32fb_devminor);
+		goto err_out;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	if (!lock_fb_info(psLINFBInfo))
+		goto err_out;
+#else
+	lock_fb_info(psLINFBInfo);
+#endif
+
+	console_lock();
+
+	/* Filter out broken FB devices */
+	if (!psLINFBInfo->fix.smem_len || !psLINFBInfo->fix.line_length)
+	{
+		pr_err("The fbdev device detected had a zero smem_len or "
+			   "line_length,\nwhich suggests it is a broken driver.\n");
+		goto err_unlock;
+	}
+
+	if (psLINFBInfo->fix.type != FB_TYPE_PACKED_PIXELS ||
+		psLINFBInfo->fix.visual != FB_VISUAL_TRUECOLOR)
+	{
+		pr_err("The fbdev device detected is not truecolor with packed "
+			   "pixels.\n");
+		goto err_unlock;
+	}
+
+	if (psLINFBInfo->var.bits_per_pixel == 32)
+	{
+		if (psLINFBInfo->var.red.length   != 8  ||
+			psLINFBInfo->var.green.length != 8  ||
+			psLINFBInfo->var.blue.length  != 8  ||
+			psLINFBInfo->var.red.offset   != 16 ||
+			psLINFBInfo->var.green.offset != 8  ||
+			psLINFBInfo->var.blue.offset  != 0)
+		{
+			pr_err("The fbdev device detected uses an unrecognized "
+				   "32bit pixel format (%u/%u/%u, %u/%u/%u)\n",
+				   psLINFBInfo->var.red.length,
+				   psLINFBInfo->var.green.length,
+				   psLINFBInfo->var.blue.length,
+				   psLINFBInfo->var.red.offset,
+				   psLINFBInfo->var.green.offset,
+				   psLINFBInfo->var.blue.offset);
+			goto err_unlock;
+		}
+#if defined(DC_FBDEV_FORCE_XRGB8888)
+		ePixFormat = IMG_PIXFMT_B8G8R8X8_UNORM;
+#else
+		ePixFormat = IMG_PIXFMT_B8G8R8A8_UNORM;
+#endif
+	}
+	else if (psLINFBInfo->var.bits_per_pixel == 16)
+	{
+		if (psLINFBInfo->var.red.length   != 5  ||
+			psLINFBInfo->var.green.length != 6  ||
+			psLINFBInfo->var.blue.length  != 5  ||
+			psLINFBInfo->var.red.offset   != 11 ||
+			psLINFBInfo->var.green.offset != 5  ||
+			psLINFBInfo->var.blue.offset  != 0)
+		{
+			pr_err("The fbdev device detected uses an unrecognized "
+				   "16bit pixel format (%u/%u/%u, %u/%u/%u)\n",
+				   psLINFBInfo->var.red.length,
+				   psLINFBInfo->var.green.length,
+				   psLINFBInfo->var.blue.length,
+				   psLINFBInfo->var.red.offset,
+				   psLINFBInfo->var.green.offset,
+				   psLINFBInfo->var.blue.offset);
+			goto err_unlock;
+		}
+		ePixFormat = IMG_PIXFMT_B5G6R5_UNORM;
+	}
+	else
+	{
+		pr_err("The fbdev device detected uses an unsupported "
+			   "bpp (%u).\n", psLINFBInfo->var.bits_per_pixel);
+		goto err_unlock;
+	}
+
+	if (!try_module_get(psLINFBInfo->fbops->owner))
+	{
+		pr_err("try_module_get() failed");
+		goto err_unlock;
+	}
+
+	if (psLINFBInfo->fbops->fb_open &&
+		psLINFBInfo->fbops->fb_open(psLINFBInfo, 0) != 0)
+	{
+		pr_err("fb_open() failed");
+		goto err_module_put;
+	}
+
+	gpsDeviceData = kmalloc(sizeof(DC_FBDEV_DEVICE), GFP_KERNEL);
+	if (!gpsDeviceData)
+		goto err_module_put;
+
+	gpsDeviceData->psLINFBInfo = psLINFBInfo;
+	gpsDeviceData->ePixFormat = ePixFormat;
+
+	if (DCRegisterDevice(&sDCFunctions,
+						 MAX_COMMANDS_IN_FLIGHT,
+						 gpsDeviceData,
+						 &gpsDeviceData->hSrvHandle) != PVRSRV_OK)
+		goto err_kfree;
+
+	gpsDeviceData->bCanFlip = DC_FBDEV_FlipPossible(psLINFBInfo);
+
+	pr_info("Found usable fbdev device (%s):\n"
+#if defined(DC_FBDEV_USE_SCREEN_BASE)
+			"range (virtual)  = 0x%lx-0x%lx\n"
+#else
+			"range (physical) = 0x%lx-0x%lx\n"
+#endif
+			"size (bytes)     = 0x%x\n"
+			"xres x yres      = %ux%u\n"
+			"xres x yres (v)  = %ux%u\n"
+			"img pix fmt      = %u\n"
+			"flipping?        = %d\n",
+			psLINFBInfo->fix.id,
+			psLINFBInfo->fix.smem_start,
+			psLINFBInfo->fix.smem_start + psLINFBInfo->fix.smem_len,
+			psLINFBInfo->fix.smem_len,
+			psLINFBInfo->var.xres, psLINFBInfo->var.yres,
+			psLINFBInfo->var.xres_virtual, psLINFBInfo->var.yres_virtual,
+			ePixFormat, gpsDeviceData->bCanFlip);
+	err = 0;
+err_unlock:
+	console_unlock();
+	unlock_fb_info(psLINFBInfo);
+err_out:
+	return err;
+err_kfree:
+	kfree(gpsDeviceData);
+err_module_put:
+	module_put(psLINFBInfo->fbops->owner);
+	goto err_unlock;
+}
+
+static void __exit DC_FBDEV_exit(void)
+{
+	DC_FBDEV_DEVICE *psDeviceData = gpsDeviceData;
+	struct fb_info *psLINFBInfo = psDeviceData->psLINFBInfo;
+
+	lock_fb_info(psLINFBInfo);
+	console_lock();
+
+	if (psLINFBInfo->fbops->fb_release)
+	   psLINFBInfo->fbops->fb_release(psLINFBInfo, 0);
+
+	module_put(psLINFBInfo->fbops->owner);
+
+	console_unlock();
+	unlock_fb_info(psLINFBInfo);
+
+	DCUnregisterDevice(psDeviceData->hSrvHandle);
+	kfree(psDeviceData);
+}
+
+module_init(DC_FBDEV_init);
+module_exit(DC_FBDEV_exit);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/common/dc_pdp_common.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/common/dc_pdp_common.c
new file mode 100644
index 000000000000..eac70858205a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/common/dc_pdp_common.c
@@ -0,0 +1,1847 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "dc_pdp.h"
+#include "img_defs.h"
+#include "pdp2_regs.h"
+#include "pdp2_mmu_regs.h"
+
+#if defined(__linux__)
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#endif
+
+/*******************************************************************************
+ * Debug/print macros
+ ******************************************************************************/
+#if defined(PDP_DEBUG)
+	#define PDP_CHECKPOINT PDP_DEBUG_PRINT(" CP: - %s\n", __func__);
+	#define PDP_DEBUG_PRINT(fmt, ...) \
+		DC_OSDebugPrintf(DBGLVL_WARNING, fmt, __VA_ARGS__)
+#else
+	#define PDP_CHECKPOINT  do {} while (0)
+	#define PDP_DEBUG_PRINT(fmt, ...) do {} while (0)
+#endif
+
+#define PDP_ERROR_PRINT(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_ERROR, fmt, __VA_ARGS__)
+
+#if defined(PLATO_DISPLAY_PDUMP)
+
+#undef DC_OSWriteReg32
+#define DC_OSWriteReg32 PDP_OSWriteHWReg32
+
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	plato_pdump_pol(base,reg,val,msk, DRVNAME); \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			PDP_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#else
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			PDP_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#endif // PDUMP
+
+#define pol(base,reg,val,msk) polpr(base,reg,val,msk,10,10)
+
+#if defined(PDP_DEBUG)
+static DLLIST_NODE gsPdpDeviceListHead;  /* List of discovered / configured PDP */
+static IMG_UINT32 gNumPdpDevices;        /* Number of discovered / configured PDP */
+void PDPDebugCtrl(void)
+{
+	int k = 0;
+	int i = 0;
+	DCPDP_DEVICE *psDevice;
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsPdpDeviceListHead, psNode, psNext)
+	{
+		psDevice = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sDbgListNode);
+
+		if (gNumPdpDevices > 1)
+			printk("\n-----PDP Register dump [device %d] ------\n", i+1);
+		else
+			printk("\n------PDP Register dump -------\n");
+
+		for (k = 0; k < PDP_NUMREG; k++)
+		{
+			PDP_DEBUG_PRINT(" Offset %x: %x\n", k*4, DC_OSReadReg32(psDevice->pvPDPRegCpuVAddr, k * 4));
+		}
+
+		if (gNumPdpDevices > 1)
+			printk("\n-----PDP MMU Reg dump [device %d] ------\n", i+1);
+		else
+			printk("\n------PDP MMU Reg dump -------\n");
+
+		for (k = 0; k < PDP_BIF_VERSION_OFFSET; k += 4)
+		{
+			PDP_DEBUG_PRINT(" BIF Offset %x: %x\n", k, DC_OSReadReg32(psDevice->pvPDPBifRegCpuVAddr, k));
+		}
+		++i;
+	}
+	/*
+
+	static int i = 0;
+	IMG_CPU_VIRTADDR pvBufferCpuVAddr;
+	IMG_UINT32 *pui32Pixel;
+	IMG_UINT32 ui32SizeInPixels;
+	IMG_UINT32 j;
+
+	gpsBuffer[2] = gPdpDevice->psSystemBuffer;
+
+	for (b = 0; b < 3; b++)
+	{
+		pvBufferCpuVAddr = DC_OSMapPhysAddr(gpsBuffer[b]->sCpuPAddr, gpsBuffer[b]->ui32SizeInBytes);
+		if (pvBufferCpuVAddr != NULL)
+		{
+			pui32Pixel = (IMG_UINT32 *)pvBufferCpuVAddr;
+			ui32SizeInPixels = 10; //gpsBuffer[i]->ui32Width;// * gpsBuffer[i]->ui32Height;
+
+			for (j = 0; j < ui32SizeInPixels; j++)
+			{
+				DC_OSDebugPrintf(DBGLVL_WARNING, " - %s: Buffer %d Pixel[%u]: 0x%08X\n", __func__, b, j, pui32Pixel[j]);
+			}
+
+			DC_OSUnmapPhysAddr(pvBufferCpuVAddr, gpsBuffer[b]->ui32SizeInBytes);
+		}
+	}
+	i = (i + 1) % 2;
+	*/
+}
+#endif /* PDP_DEBUG */
+
+static IMG_BOOL BufferAcquireMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psDeviceData->ui32BufferCount; i++)
+	{
+		if ((psDeviceData->ui32BufferUseMask & (1UL << i)) == 0)
+		{
+			psDeviceData->ui32BufferUseMask |= (1UL << i);
+
+			psBuffer->sCpuPAddr.uiAddr = psDeviceData->sDispMemCpuPAddr.uiAddr + (i * psDeviceData->ui32BufferSize);
+
+			psDeviceData->sPVRServicesFuncs.pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap,
+										      1,
+										      &psBuffer->sDevPAddr,
+										      &psBuffer->sCpuPAddr);
+			if (psBuffer->sDevPAddr.uiAddr == psBuffer->sCpuPAddr.uiAddr)
+			{
+				/* Because the device and CPU addresses are the same we assume that we have an
+				   incompatible PDP device address. Calculate the correct PDP device address. */
+				psBuffer->sDevPAddr.uiAddr -= psDeviceData->sDispMemCpuPAddr.uiAddr;
+			}
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+
+static void BufferReleaseMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT64 uiOffset;
+
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr >= psDeviceData->sDispMemCpuPAddr.uiAddr);
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr < (psDeviceData->sDispMemCpuPAddr.uiAddr + psDeviceData->uiDispMemSize));
+
+	uiOffset = psBuffer->sCpuPAddr.uiAddr - psDeviceData->sDispMemCpuPAddr.uiAddr;
+	uiOffset = DC_OSDiv64(uiOffset, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferUseMask &= ~(1UL << uiOffset);
+
+	psBuffer->sDevPAddr.uiAddr = 0;
+	psBuffer->sCpuPAddr.uiAddr = 0;
+}
+
+static void EnableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET);
+	SET_FIELD(ui32InterruptEnable, PDP_INTENAB_INTEN_VBLNK0, 1);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET, ui32InterruptEnable);
+}
+
+static void DisableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET);
+	SET_FIELD(ui32InterruptEnable, PDP_INTENAB_INTEN_VBLNK0, 0);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET, ui32InterruptEnable);
+}
+
+static void PDPUpdateRegisters(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32Value = 0;
+	DC_ASSERT(psDeviceData != NULL);
+	/* To be set when register settings should be adopted by the PDP */
+	SET_FIELD(ui32Value, PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID, 0x01);
+	/* To indicate whether or not the PDP should adopt the updated register settings during */
+	SET_FIELD(ui32Value, PDP_REGISTER_UPDATE_CTRL_USE_VBLANK, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_REGISTER_UPDATE_CTRL_OFFSET, ui32Value); // 0x00000003
+}
+
+static void PDPResetMMU(DCPDP_DEVICE * psDeviceData)
+{
+	IMG_UINT32 ui32Value = 0;
+
+	/* Display pipeline software reset: Software reset. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_PDP_RST, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x20000000
+
+	ui32Value = 0;
+	/* Display pipeline software reset: Normal operation. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_PDP_RST, 0x00);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x00000000
+
+}
+
+static INLINE void Flip(DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 ui32Value = 0;
+	IMG_UINT32 ui32Width = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	IMG_UINT32 ui32Height = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	ui32Value = 0;
+	SET_FIELD(ui32Value, PDP_GRPH1SIZE_GRPH1WIDTH, ui32Width - 1);
+	SET_FIELD(ui32Value, PDP_GRPH1SIZE_GRPH1HEIGHT, ui32Height - 1);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1SIZE_OFFSET, ui32Value);
+
+	ui32Value = 0;
+	SET_FIELD(ui32Value, PDP_GRPH1POSN_GRPH1XSTART, 0);
+	SET_FIELD(ui32Value, PDP_GRPH1POSN_GRPH1YSTART, 0);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1POSN_OFFSET, ui32Value);
+
+	ui32Value = 0;
+	switch (psBuffer->ePixelFormat)
+	{
+		//case IMG_PIXFMT_B5G6R5_UNORM: // XXX Not supported technically we only have R5G6B5
+		//{
+		//	SET_FIELD(ui32Value, PDP_GRPH1SURF_GRPH1PIXFMT, DCPDP_PIXEL_FORMAT_RGB565);
+		//	break;
+		//}
+		//case IMG_PIXFMT_B8G8R8X8_UNORM: // XXX Not supported technically we only have RGBA8
+		case IMG_PIXFMT_B8G8R8A8_UNORM:
+		//case IMG_PIXFMT_R8G8B8A8_UNORM:
+		//case IMG_PIXFMT_R8G8B8X8_UNORM:
+		{
+			SET_FIELD(ui32Value, PDP_GRPH1SURF_GRPH1PIXFMT, DCPDP_PIXEL_FORMAT_ARGB8);
+			break;
+		}
+		default:
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Unrecognised pixel format\n");
+			DC_ASSERT(0);
+		}
+	}
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1SURF_OFFSET, ui32Value);
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1STRIDE_OFFSET,
+			PLACE_FIELD(PDP_GRPH1STRIDE_GRPH1STRIDE, (psBuffer->ui32ByteStride >> PDP_GRPH1STRIDE_DIVIDE) - 1));
+
+	DC_ASSERT(psBuffer);
+	/* Check if the address is a correct 40-bit address */
+	DC_ASSERT((psBuffer->sDevPAddr.uiAddr & 0xffffff0000000000) == 0);
+
+	/* Write 32 lsb of the address to stream address register */
+	ui32Value = 0;
+	ui32Value = DC_OSReadReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1BASEADDR_OFFSET);
+	ui32Value = (psBuffer->sDevPAddr.uiAddr & PDP_GRPH1BASEADDR_GRPH1BASEADDR_MASK);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1BASEADDR_OFFSET, ui32Value);
+
+	/* Write 8 msb of the address to address extension bits in the PDP MMU control register */
+	ui32Value = DC_OSReadReg32(psBuffer->psDeviceData->pvPDPBifRegCpuVAddr, PDP_BIF_ADDRESS_CONTROL_OFFSET);
+	SET_FIELD(ui32Value, PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED, (psBuffer->sDevPAddr.uiAddr >> 32));
+	SET_FIELD(ui32Value, PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING, 0x00);
+	SET_FIELD(ui32Value, PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS, 0x01);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPBifRegCpuVAddr, PDP_BIF_ADDRESS_CONTROL_OFFSET, ui32Value);
+
+	PDPUpdateRegisters(psBuffer->psDeviceData);
+
+#if defined(VIRTUAL_PLATFORM)
+	/* Kick pdp sim render */
+	DC_OSDebugPrintf(DBGLVL_DEBUG, "%s:%d: Kicking PDP render\n", __func__, __LINE__);
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, PDP_INTCLR_OFFSET, 0x11111111);
+#endif
+}
+
+static INLINE void FlipConfigQueueAdvanceIndex(DCPDP_FLIP_CONTEXT *psContext, IMG_UINT32 *pui32Index)
+{
+	(*pui32Index)++;
+
+	if (*pui32Index >= DCPDP_MAX_COMMANDS_INFLIGHT)
+	{
+		*pui32Index = 0;
+	}
+}
+
+static INLINE IMG_BOOL FlipConfigQueueEmptyNoLock(DCPDP_FLIP_CONTEXT *psContext)
+{
+	return (IMG_BOOL)(psContext->ui32FlipConfigInsertIndex == psContext->ui32FlipConfigRemoveIndex);
+}
+
+static IMG_BOOL FlipConfigQueueEmpty(DCPDP_FLIP_CONTEXT *psContext)
+{
+	unsigned long ulSpinlockFlags;
+	IMG_BOOL bFlipConfigQueueEmpty;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+	bFlipConfigQueueEmpty = FlipConfigQueueEmptyNoLock(psContext);
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	return bFlipConfigQueueEmpty;
+}
+
+static void FlipConfigQueueAdd(DCPDP_FLIP_CONTEXT *psContext,
+				   IMG_HANDLE hConfigData,
+				   DCPDP_BUFFER *psBuffer,
+				   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				   IMG_UINT32 ui32DisplayPeriod)
+{
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Get the next inactive flip config and update the insert index */
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigInsertIndex];
+	FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigInsertIndex);
+
+	DC_ASSERT(psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_INACTIVE);
+
+	/* Fill out the flip config */
+	psFlipConfig->psContext		= psContext;
+	psFlipConfig->hConfigData	= hConfigData;
+	psFlipConfig->psBuffer		= psBuffer;
+	psFlipConfig->ui32DisplayPeriod	= ui32DisplayPeriod;
+	psFlipConfig->pasSurfAttrib	= pasSurfAttrib;
+
+	/* Should be updated last */
+	psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_PENDING;
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+}
+
+static void SetModeRegisters(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_TIMING_DATA *psTimingData = &psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex];
+	IMG_UINT32 ui32Value;
+
+	DC_ASSERT(psDeviceData->psSystemBuffer);
+
+	PDP_CHECKPOINT;
+
+	ui32Value = 0;
+	/* Memory refreshes enabled during horizontal and vertical blanking. */
+	SET_FIELD(ui32Value, PDP_MEMCTRL_MEMREFRESH, 0x02);
+	/* Specifies number of consecutive words issued from the current base address before moving onto the next. */
+	/* Issues with NoC sending interleaved read responses to PDP require burst to be 1 */
+	SET_FIELD(ui32Value, PDP_MEMCTRL_BURSTLEN, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_MEMCTRL_OFFSET, ui32Value);
+
+	PDPResetMMU(psDeviceData);
+
+	ui32Value = 0;
+	/* XXX Test these inputs, this may be configured incorrectly */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_OPMASK_R_OFFSET , 0x00000000);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_OPMASK_GB_OFFSET, 0x00000000);
+
+	ui32Value = 0;
+	/* Power down mode: Output valid sync signals, and substitute active data with black pixels. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_POWERDN, 0x01);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x10000000
+
+	/* Clear all interrupt enable bits, so that none of them are enabled for any plane */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTENAB_OFFSET  , 0x00000000);
+
+	ui32Value = 0;
+	/* Power down mode: Output valid sync signals, and substitute active data with black pixels. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_POWERDN, 0x01);
+	/* Controls Blanking signal polarity: Blanking signal is active low */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_BLNKPOL, 0x01);
+	/* Controls Vertical Sync polarity: Vertical Sync is active low */
+	/* VSync polarity is active high (0), should come from hdmi driver */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_VSPOL, psTimingData->ui8VSyncPolarity);
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_HSPOL, psTimingData->ui8HSyncPolarity);
+	//SET_FIELD(ui32Value, PDP_SYNCCTRL_HSDIS, 1);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET , ui32Value); // 0x10000028
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VSYNC1_OFFSET,
+			PLACE_FIELD(PDP_VSYNC1_VBPS, psTimingData->ui32VBackPorch) | PLACE_FIELD(PDP_VSYNC1_VT, psTimingData->ui32VTotal));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VSYNC2_OFFSET,
+			PLACE_FIELD(PDP_VSYNC2_VAS, psTimingData->ui32VActiveStart) | PLACE_FIELD(PDP_VSYNC2_VTBS, psTimingData->ui32VTopBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VSYNC3_OFFSET,
+			PLACE_FIELD(PDP_VSYNC3_VFPS, psTimingData->ui32VFrontPorch) | PLACE_FIELD(PDP_VSYNC3_VBBS, psTimingData->ui32VBottomBorder));
+
+	if (psTimingData->bReducedBlanking)
+	{
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VEVENT_OFFSET,
+			PLACE_FIELD(PDP_VEVENT_VEVENT, psTimingData->ui32VBottomBorder + DCPDP_REDUCED_BLANKING_VEVENT) |
+			PLACE_FIELD(PDP_VEVENT_VFETCH, psTimingData->ui32VBackPorch / 2));
+	}
+	else
+	{
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VEVENT_OFFSET,
+			PLACE_FIELD(PDP_VEVENT_VEVENT, 0) | PLACE_FIELD(PDP_VEVENT_VFETCH, psTimingData->ui32VBackPorch));
+	}
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_VDECTRL_OFFSET,
+			PLACE_FIELD(PDP_VDECTRL_VDES, psTimingData->ui32VActiveStart) | PLACE_FIELD(PDP_VDECTRL_VDEF, psTimingData->ui32VBottomBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HSYNC1_OFFSET,
+			PLACE_FIELD(PDP_HSYNC1_HBPS, psTimingData->ui32HBackPorch) | PLACE_FIELD(PDP_HSYNC1_HT, psTimingData->ui32HTotal));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HSYNC2_OFFSET,
+			PLACE_FIELD(PDP_HSYNC2_HAS, psTimingData->ui32HActiveStart) | PLACE_FIELD(PDP_HSYNC2_HLBS, psTimingData->ui32HLeftBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HSYNC3_OFFSET,
+			PLACE_FIELD(PDP_HSYNC3_HFPS, psTimingData->ui32HFrontPorch) | PLACE_FIELD(PDP_HSYNC3_HRBS, psTimingData->ui32HRightBorder));
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_HDECTRL_OFFSET,
+			PLACE_FIELD(PDP_HDECTRL_HDES, psTimingData->ui32HActiveStart) | PLACE_FIELD(PDP_HDECTRL_HDEF, psTimingData->ui32HRightBorder));
+
+	/* Enable GRPH1 */
+	DCPDPEnableMemoryRequest(psDeviceData, IMG_TRUE);
+
+	ui32Value = 0;
+	/* Set the global alpha to 1023, i.e. fully saturated */
+	SET_FIELD(ui32Value, PDP_GRPH1GALPHA_GRPH1GALPHA, 0x03FF);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1GALPHA_OFFSET, ui32Value); // 0x000003FF
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1_MEM_THRESH_OFFSET, 0x780f80f8);
+
+	ui32Value = 0;
+	/* Starts Sync Generator. */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_SYNCACTIVE, 0x01);
+	/* Controls polarity of pixel clock: Pixel clock is inverted */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_CLKPOL, 0x01);
+	/* Controls Blanking signal polarity: Blanking signal is active low */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_BLNKPOL, 0x01);
+	/* Controls Vertical Sync polarity: Vertical Sync is active low */
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_VSPOL, psTimingData->ui8VSyncPolarity);
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_HSPOL, psTimingData->ui8HSyncPolarity);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET, ui32Value); // 0x80000828
+}
+
+static IMG_UINT32 GetTimingIndex(DCPDP_DEVICE *psDeviceData, IMG_UINT32 width, IMG_UINT32 height)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 index = -1;
+	for (i = 0; i < psDeviceData->uiTimingDataSize; i++)
+	{
+		if (psDeviceData->pasTimingData[i].ui32HDisplay == width &&
+		    psDeviceData->pasTimingData[i].ui32VDisplay == height)
+		{
+			index = i;
+		}
+	}
+
+	return index;
+}
+
+static void FlipConfigQueueProcess(DCPDP_FLIP_CONTEXT *psContext, IMG_BOOL bFlipped, IMG_BOOL bFlush)
+{
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+	IMG_HANDLE ahRetiredConfigs[DCPDP_MAX_COMMANDS_INFLIGHT] = {0};
+	IMG_UINT32 i, ui32RetireConfigIndex = 0;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSMutexLock(psContext->pvMutex);
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+	pasSurfAttrib = psFlipConfig->pasSurfAttrib;
+
+	do
+	{
+		switch (psFlipConfig->eStatus)
+		{
+			case DCPDP_FLIP_CONFIG_PENDING:
+				if (!bFlipped)
+				{
+					Flip(psFlipConfig->psBuffer);
+
+					bFlipped = IMG_TRUE;
+				}
+
+				/* If required, set the mode after flipping */
+				if (pasSurfAttrib != NULL)
+				{
+					if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay ||
+					    pasSurfAttrib[0].sDisplay.sDims.ui32Height != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay)
+					{
+						psDeviceData->uiTimingDataIndex = GetTimingIndex(psDeviceData,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Width,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+						SetModeRegisters(psDeviceData);
+						DCPDPEnableMemoryRequest(psDeviceData, IMG_TRUE);
+					}
+				}
+
+				psFlipConfig->eStatus = DCPDP_FLIP_CONFIG_ACTIVE;
+
+				/* Now that a new flip config is active we can retire the previous one (if one exists) */
+				if (psContext->hConfigDataToRetire != NULL)
+				{
+					ahRetiredConfigs[ui32RetireConfigIndex++] = psContext->hConfigDataToRetire;
+					psContext->hConfigDataToRetire = NULL;
+				}
+
+				__fallthrough;
+			case DCPDP_FLIP_CONFIG_ACTIVE:
+				/* We can retire this config only when it's buffer has been
+				   displayed for at least the same number of frames as the
+				   display period or if we're flushing the queue */
+				if (psFlipConfig->ui32DisplayPeriod != 0)
+				{
+					psFlipConfig->ui32DisplayPeriod--;
+				}
+
+				if (psFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+				{
+					DCPDP_FLIP_CONFIG *psNextFlipConfig;
+
+					/* There should never be an outstanding config to retire at this point */
+					DC_ASSERT(psContext->hConfigDataToRetire == NULL);
+
+					/* Now that this config has been active for the minimum number of frames, i.e. the
+					   display period, we can mark the flip config queue item as inactive. However, we
+					   can only retire the config once a new config has been made active. Therefore,
+					   store the Services config data handle so that we can retire it later. */
+					psContext->hConfigDataToRetire = psFlipConfig->hConfigData;
+
+					/* Reset the flip config data */
+					psFlipConfig->hConfigData	= 0;
+					psFlipConfig->psBuffer		= NULL;
+					psFlipConfig->ui32DisplayPeriod	= 0;
+					psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_INACTIVE;
+
+					/* Move on the queue remove index */
+					FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigRemoveIndex);
+
+					psNextFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+					if (psNextFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+					{
+						psFlipConfig = psNextFlipConfig;
+						bFlipped = IMG_FALSE;
+					}
+				}
+				break;
+			case DCPDP_FLIP_CONFIG_INACTIVE:
+				if (FlipConfigQueueEmptyNoLock(psContext))
+				{
+					break;
+				}
+				DC_OSDebugPrintf(DBGLVL_ERROR, " - Tried to process an inactive config when one wasn't expected\n");
+
+				__fallthrough;
+			default:
+				DC_ASSERT(0);
+		}
+	} while (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING);
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Call into Services to retire any configs that were ready to be
+	 * retired. */
+	for (i = 0; i < ui32RetireConfigIndex; i++)
+	{
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(ahRetiredConfigs[i]);
+	}
+
+	DC_OSMutexUnlock(psContext->pvMutex);
+}
+
+static PVRSRV_ERROR VSyncProcessor(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = pvData;
+
+	psDeviceData->sPVRServicesFuncs.pfnCheckStatus(NULL);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FlipConfigProcessor(void *pvData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)pvData;
+
+	DC_ASSERT(psContext);
+
+	FlipConfigQueueProcess(psContext, IMG_TRUE, IMG_FALSE);
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPGetInfo(IMG_HANDLE hDeviceData, DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Copy our device name */
+	DC_OSStringNCopy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	/* Report what our minimum and maximum display period is. */
+	psDisplayInfo->ui32MinDisplayPeriod	= DCPDP_MIN_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxDisplayPeriod	= DCPDP_MAX_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxPipes		= 1;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static PVRSRV_ERROR DCPDPPanelQueryCount(IMG_HANDLE hDeviceData, IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPPanelQuery(IMG_HANDLE hDeviceData,
+				    IMG_UINT32 ui32PanelsArraySize,
+				    IMG_UINT32 *pui32NumPanels,
+				    PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	PDP_CHECKPOINT;
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	DC_OSMemSet(psPanelInfo, 0, sizeof(*psPanelInfo));
+
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = psDeviceData->ePixelFormat;
+	psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+	psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	psPanelInfo[0].ui32RefreshRate = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh;
+	psPanelInfo[0].ui32XDpi = DCPDP_DPI;
+	psPanelInfo[0].ui32YDpi = DCPDP_DPI;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPFormatQuery(IMG_HANDLE hDeviceData,
+				     IMG_UINT32 ui32NumFormats,
+				     PVRSRV_SURFACE_FORMAT *pasFormat,
+				     IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (pasFormat[i].ePixFormat == psDeviceData->ePixelFormat)
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPDimQuery(IMG_HANDLE hDeviceData,
+				  IMG_UINT32 ui32NumDims,
+				  PVRSRV_SURFACE_DIMS *psDim,
+				  IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+	IMG_UINT32 j;
+
+	PDP_CHECKPOINT;
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		for (j = 0; j < psDeviceData->uiTimingDataSize; j++)
+		{
+			if ((psDim[i].ui32Width == psDeviceData->pasTimingData[j].ui32HDisplay) &&
+			    (psDim[i].ui32Height == psDeviceData->pasTimingData[j].ui32VDisplay))
+			{
+				pui32Supported[i]++;
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetBlank(IMG_HANDLE hDeviceData,
+								  IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	IMG_UINT32 ui32Value;
+
+	PDP_CHECKPOINT;
+
+	ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET);
+
+	SET_FIELD(ui32Value, PDP_SYNCCTRL_POWERDN, bEnabled ? 1 : 0);
+	psDeviceData->bVSyncEnabled = !bEnabled;
+
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_SYNCCTRL_OFFSET, ui32Value);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetVSyncReporting(IMG_HANDLE hDeviceData,
+										   IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	psDeviceData->bVSyncReporting = bEnabled;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPLastVSyncQuery(IMG_HANDLE hDeviceData,
+										IMG_INT64 *pi64Timestamp)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	*pi64Timestamp = psDeviceData->i64LastVSyncTimeStamp;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPContextCreate(IMG_HANDLE hDeviceData,
+				       IMG_HANDLE *hDisplayContext)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	DCPDP_FLIP_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	if (psDeviceData->psFlipContext)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Cannot create a context when one is already active\n");
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psContext = DC_OSCallocMem(sizeof(*psContext));
+	if (psContext == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Not enough memory to allocate a display context\n");
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* We don't want to process the flip config work queue in interrupt context so
+	   create a work queue which we can use to defer the work */
+	eError = DC_OSWorkQueueCreate(&psContext->hFlipConfigWorkQueue, DCPDP_MAX_COMMANDS_INFLIGHT);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Couldn't create a work queue (%s)\n",
+				 psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPDPOSFreeMemContext;
+	}
+
+	/* Create work items to place on the flip config work queue when we service a vsync interrupt */
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		eError = DC_OSWorkQueueCreateWorkItem(&psContext->asFlipConfigQueue[i].hWorkItem, FlipConfigProcessor, psContext);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create a flip config work item (%s)\n",
+					 psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+			goto ErrorDC_OSWorkQueueDestroyWorkItems;
+		}
+	}
+
+	eError = DC_OSMutexCreate(&psContext->pvMutex);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create flip config mutex (%s)\n",
+				 psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDC_OSWorkQueueDestroyWorkItems;
+	}
+
+	DC_OSSpinLockCreate(&psContext->sSpinLock);
+
+	psContext->psDeviceData		= psDeviceData;
+	psDeviceData->psFlipContext	= psContext;
+	*hDisplayContext		= (IMG_HANDLE)psContext;
+
+	return PVRSRV_OK;
+
+ErrorDC_OSWorkQueueDestroyWorkItems:
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+
+ErrorPDPOSFreeMemContext:
+	DC_OSFreeMem(psContext);
+
+	return eError;
+}
+
+static void DCPDPContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	/* Make sure there are no work items on the queue before it's destroyed */
+	DC_OSWorkQueueFlush(psContext->hFlipConfigWorkQueue);
+
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+	DC_OSMutexDestroy(psContext->pvMutex);
+	DC_OSSpinLockDestroy(&psContext->sSpinLock);
+
+	psDeviceData->psFlipContext = NULL;
+	DC_OSFreeMem(psContext);
+
+}
+
+static void DCPDPContextConfigure(IMG_HANDLE hDisplayContext,
+				  IMG_UINT32 ui32PipeCount,
+				  PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				  IMG_HANDLE *ahBuffers,
+				  IMG_UINT32 ui32DisplayPeriod,
+				  IMG_HANDLE hConfigData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_BOOL bQueueWasEmpty;
+
+	if (ui32DisplayPeriod > DCPDP_MAX_DISPLAY_PERIOD)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Requested display period of %u is larger than maximum supported display period of %u (clamping)\n",
+				 ui32DisplayPeriod, DCPDP_MAX_DISPLAY_PERIOD);
+		DC_ASSERT(ui32DisplayPeriod <= DCPDP_MAX_DISPLAY_PERIOD);
+
+		/* Clamp to the maximum so that we can continue */
+		ui32DisplayPeriod = DCPDP_MAX_DISPLAY_PERIOD;
+	}
+
+	/* Add new config to the queue */
+	if (ui32PipeCount != 0)
+	{
+		/* Check if the queue is empty before adding a new flip config */
+		bQueueWasEmpty = FlipConfigQueueEmpty(psContext);
+
+		/* We have a regular config so add it as normal */
+		FlipConfigQueueAdd(psContext, hConfigData, (DCPDP_BUFFER *)ahBuffers[0], pasSurfAttrib, ui32DisplayPeriod);
+
+		/* Check to see if vsync unlocked flipping should be done. This is determined by the
+		   display period being 0 and the flip config queue being empty before we added the
+		   new flip config. If it wasn't empty then we need to allow the remaining vsync locked
+		   flip configs to be processed before we can start processing the queue here.
+		   Also immediately flip if the current vsync interrupt is disabled.
+		   This makes sure that even when the PDP is turned off we progress. */
+		if ((ui32DisplayPeriod == 0 && bQueueWasEmpty) ||
+			 psDeviceData->bVSyncEnabled == IMG_FALSE)
+		{
+			FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_FALSE);
+		}
+	}
+	else
+	{
+		/* We have a 'NULL' config, which indicates the display context is
+		   about to be destroyed. Queue a flip back to the system buffer */
+		FlipConfigQueueAdd(psContext, hConfigData, psDeviceData->psSystemBuffer, pasSurfAttrib, 0);
+
+		/* Flush the remaining configs (this will result in the system buffer being displayed) */
+		FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_TRUE);
+
+		DC_OSMutexLock(psContext->pvMutex);
+
+		/* Immediately retire the system buffer config data */
+		psDeviceData->sPVRServicesFuncs.pfnDCDisplayConfigurationRetired(hConfigData);
+		psContext->hConfigDataToRetire = NULL;
+
+		DC_OSMutexUnlock(psContext->pvMutex);
+	}
+}
+
+static PVRSRV_ERROR DCPDPContextConfigureCheck(IMG_HANDLE hDisplayContext,
+					       IMG_UINT32 ui32PipeCount,
+					       PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+					       IMG_HANDLE *ahBuffers)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_BUFFER *psBuffer;
+	IMG_UINT32 ui32NewVESATimingIndex;
+	ui32NewVESATimingIndex = GetTimingIndex(psDeviceData, pasSurfAttrib[0].sDisplay.sDims.ui32Width, pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+
+	if (ui32NewVESATimingIndex == -1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+
+	if (ui32PipeCount != 1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+	/* new crop dimensions, new mode dimensions */
+	else if (pasSurfAttrib[0].sCrop.sDims.ui32Width != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32HDisplay ||
+		 pasSurfAttrib[0].sCrop.sDims.ui32Height != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32VDisplay ||
+		 pasSurfAttrib[0].sCrop.i32XOffset != 0 ||
+		 pasSurfAttrib[0].sCrop.i32YOffset != 0)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+	}
+	/* new mode dimensions, new crop dimensions */
+	else if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != pasSurfAttrib[0].sCrop.sDims.ui32Width ||
+		 pasSurfAttrib[0].sDisplay.sDims.ui32Height != pasSurfAttrib[0].sCrop.sDims.ui32Height ||
+		 pasSurfAttrib[0].sDisplay.i32XOffset != pasSurfAttrib[0].sCrop.i32XOffset ||
+		 pasSurfAttrib[0].sDisplay.i32YOffset != pasSurfAttrib[0].sCrop.i32YOffset)
+	{
+		return PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+	}
+
+	psBuffer = ahBuffers[0];
+
+	/* new buffer dimensions, current mode dimensions */
+	if (psBuffer->ui32Width  < pasSurfAttrib[0].sDisplay.sDims.ui32Width ||
+	    psBuffer->ui32Height < pasSurfAttrib[0].sDisplay.sDims.ui32Height)
+	{
+		return PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferSystemAcquire(IMG_HANDLE hDeviceData,
+					     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+					     IMG_UINT32 *pui32PageCount,
+					     IMG_UINT32 *pui32ByteStride,
+					     IMG_HANDLE *phSystemBuffer)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	PDP_CHECKPOINT;
+
+	OSAtomicIncrement(&psDeviceData->psSystemBuffer->i32RefCount);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psDeviceData->psSystemBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psDeviceData->psSystemBuffer->ui32ByteStride;
+	*phSystemBuffer		= psDeviceData->psSystemBuffer;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferSystemRelease(IMG_HANDLE hSystemBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hSystemBuffer;
+
+	PDP_CHECKPOINT;
+
+	OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static PVRSRV_ERROR DCPDPBufferAlloc(IMG_HANDLE hDisplayContext,
+				     DC_BUFFER_CREATE_INFO *psCreateInfo,
+				     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+				     IMG_UINT32 *pui32PageCount,
+				     IMG_UINT32 *pui32ByteStride,
+				     IMG_HANDLE *phBuffer)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	DCPDP_BUFFER *psBuffer;
+
+	PDP_CHECKPOINT;
+
+	/* XXX: This need to change because we support more formats */
+	DC_ASSERT(psCreateInfo->ui32BPP == DCPDP_PIXEL_FORMAT_BPP);
+
+	if (psSurfInfo->sFormat.eMemLayout != PVRSRV_SURFACE_MEMLAYOUT_STRIDED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	switch (psSurfInfo->sFormat.ePixFormat)
+	{
+		case DCPDP_PIXEL_FORMAT: // hard coded one supported format
+			break;
+		default:
+			return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	if (psSurfInfo->sFormat.ePixFormat != psDeviceData->ePixelFormat)
+	{
+		return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psSurfInfo->sFormat.ePixFormat;
+	psBuffer->ui32Width			= psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height		= psSurfInfo->sDims.ui32Height;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * psCreateInfo->ui32BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32)DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height, DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psBuffer->ui32ByteStride;
+	*phBuffer		= psBuffer;
+
+	PDP_DEBUG_PRINT("- %s: Acquired buffer at address %lx\n", __func__, psBuffer->sCpuPAddr.uiAddr);
+
+#if defined(PDP_DEBUG)
+	dllist_init(&psBuffer->sDbgListNode);
+	dllist_add_to_tail(&psDeviceData->sDbgBufListHead, &psBuffer->sDbgListNode);
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferAcquire(IMG_HANDLE hBuffer, IMG_DEV_PHYADDR *pasDevPAddr, void **ppvLinAddr)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+	DCPDP_DEVICE *psDeviceData = psBuffer->psDeviceData;
+	IMG_CPU_PHYADDR sCpuPAddr = psBuffer->sCpuPAddr;
+	IMG_UINT32 i;
+
+	PDP_CHECKPOINT;
+
+	for (i = 0; i < psBuffer->ui32SizeInPages; i++)
+	{
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap, 1,
+									      &pasDevPAddr[i], &sCpuPAddr);
+		sCpuPAddr.uiAddr += DC_OSGetPageSize();
+	}
+
+	OSAtomicIncrement(&psBuffer->i32RefCount);
+
+	*ppvLinAddr = NULL;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferRelease(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+	OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static void DCPDPBufferFree(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+	PDP_CHECKPOINT;
+
+#if defined(DEBUG)
+	if (OSAtomicRead(&psBuffer->i32RefCount) != 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING, " - Freeing buffer that still has %d references\n", OSAtomicRead(&psBuffer->i32RefCount));
+	}
+#endif
+
+	BufferReleaseMemory(psBuffer->psDeviceData, psBuffer);
+
+#if defined(PDP_DEBUG)
+	dllist_remove_node(&psBuffer->sDbgListNode);
+#endif
+
+	DC_OSFreeMem(psBuffer);
+}
+
+static DC_DEVICE_FUNCTIONS g_sDCFunctions =
+{
+	DCPDPGetInfo,				/* pfnGetInfo */
+	DCPDPPanelQueryCount,		/* pfnPanelQueryCount */
+	DCPDPPanelQuery,			/* pfnPanelQuery */
+	DCPDPFormatQuery,			/* pfnFormatQuery */
+	DCPDPDimQuery,				/* pfnDimQuery */
+	DCPDPSetBlank,				/* pfnSetBlank */
+	DCPDPSetVSyncReporting,		/* pfnSetVSyncReporting */
+	DCPDPLastVSyncQuery,		/* pfnLastVSyncQuery */
+	DCPDPContextCreate,			/* pfnContextCreate */
+	DCPDPContextDestroy,		/* pfnContextDestroy */
+	DCPDPContextConfigure,		/* pfnContextConfigure */
+	DCPDPContextConfigureCheck,	/* pfnContextConfigureCheck */
+	DCPDPBufferAlloc,			/* pfnBufferAlloc */
+	DCPDPBufferAcquire,			/* pfnBufferAcquire */
+	DCPDPBufferRelease,			/* pfnBufferRelease */
+	DCPDPBufferFree,			/* pfnBufferFree */
+	NULL,					/* pfnBufferImport */
+	NULL,					/* pfnBufferMap */
+	NULL,					/* pfnBufferUnmap */
+	DCPDPBufferSystemAcquire,	/* pfnBufferSystemAcquire */
+	DCPDPBufferSystemRelease,	/* pfnBufferSystemRelease */
+};
+
+static IMG_BOOL DisplayLISRHandler(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)pvData;
+	IMG_UINT32 ui32InterruptStatus;
+	PVRSRV_ERROR eError;
+
+	ui32InterruptStatus = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTSTAT_OFFSET);
+	if (ui32InterruptStatus & PDP_INTSTAT_INTS_VBLNK0_MASK)
+	{
+		/* If a previous register update has completed, clear the status bit */
+		if (DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_REGISTER_UPDATE_STATUS_OFFSET) & PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_MASK)
+		{
+			DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_REGISTER_UPDATE_CTRL_OFFSET, 0);
+		}
+
+		/* Get the timestamp for this interrupt. */
+		psDeviceData->i64LastVSyncTimeStamp = DC_OSClockns();
+		if (psDeviceData->bVSyncReporting)
+		{
+			eError = DC_OSWorkQueueAddWorkItem(psDeviceData->hVSyncWorkQueue, psDeviceData->hVSyncWorkItem);
+			if (eError != PVRSRV_OK)
+			{
+				DC_OSDebugPrintf(DBGLVL_WARNING,
+						 " - %s: Couldn't queue work item (%s)\n",
+						 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+
+			}
+		}
+
+		if (psDeviceData->psFlipContext)
+		{
+			DCPDP_FLIP_CONTEXT *psContext = psDeviceData->psFlipContext;
+			DCPDP_FLIP_CONFIG *psFlipConfig;
+
+			DC_OSSpinLock(&psContext->sSpinLock);
+
+			psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+
+			if (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING)
+			{
+				Flip(psFlipConfig->psBuffer);
+			}
+
+			if (psFlipConfig->eStatus != DCPDP_FLIP_CONFIG_INACTIVE)
+			{
+				eError = DC_OSWorkQueueAddWorkItem(psContext->hFlipConfigWorkQueue, psFlipConfig->hWorkItem);
+				if (eError != PVRSRV_OK)
+				{
+					DC_OSDebugPrintf(DBGLVL_WARNING,
+							 " - %s: Couldn't queue work item (%s)\n",
+							 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+				}
+			}
+
+			DC_OSSpinUnlock(&psContext->sSpinLock);
+		}
+		/* Clear the vsync status bit to show that the vsync interrupt has been handled */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTCLR_OFFSET, (0x1UL << PDP_INTSTAT_INTS_VBLNK0_SHIFT));
+		/* For some reason, first interrupt always contains HBLNK, so clear all */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_INTCLR_OFFSET, 0xFFFFFFFF);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static void DeInitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+	{
+		if (OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) != 0)
+		{
+			printk(KERN_INFO "%s: References to psSystemBuffer still exist"
+		   " (%d)", __func__, OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount));
+		}
+		OSAtomicWrite(&psDeviceData->psSystemBuffer->i32RefCount, 0);
+	}
+	else
+#else
+	{
+		DC_ASSERT(OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) == 0);
+	}
+#endif
+
+	PDP_CHECKPOINT;
+
+	if (psDeviceData->psSystemBuffer != NULL)
+	{
+		BufferReleaseMemory(psDeviceData, psDeviceData->psSystemBuffer);
+
+		DC_OSFreeMem(psDeviceData->psSystemBuffer);
+		psDeviceData->psSystemBuffer = NULL;
+	}
+}
+
+static PVRSRV_ERROR InitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_BUFFER *psBuffer;
+	IMG_CPU_VIRTADDR pvBufferCpuVAddr;
+	IMG_UINT32 *pui32Pixel;
+	IMG_UINT8 ui8Red, ui8Green[4], ui8Blue;
+	IMG_UINT32 i, j;
+	IMG_UINT64 ui64BufferCount;
+
+	/* On a reconfigure, system buffer will not be NULL */
+	if (psDeviceData->psSystemBuffer != NULL)
+	{
+		DeInitSystemBuffer(psDeviceData);
+		psDeviceData->psSystemBuffer = NULL;
+	}
+
+	/* Setup simple buffer allocator */
+	psDeviceData->ui32BufferSize = (IMG_UINT32)DC_ALIGN(psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay *
+							    psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay *
+							    DCPDP_PIXEL_FORMAT_BPP,
+							    DC_OSGetPageSize());
+
+	ui64BufferCount = psDeviceData->uiDispMemSize;
+	ui64BufferCount = DC_OSDiv64(ui64BufferCount, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferCount = (IMG_UINT32)ui64BufferCount;
+	if (psDeviceData->ui32BufferCount == 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Not enough space for the framebuffer\n", __func__);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	else if (psDeviceData->ui32BufferCount > DCPDP_MAX_BUFFERS)
+	{
+		psDeviceData->ui32BufferCount = DCPDP_MAX_BUFFERS;
+	}
+
+	psDeviceData->ui32BufferUseMask = 0;
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psDeviceData->ePixelFormat;
+	psBuffer->ui32Width			= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psBuffer->ui32Height		= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * DCPDP_PIXEL_FORMAT_BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	PDP_DEBUG_PRINT("- %s: System buffer width: %d, height: %d, starting addr: %lx\n", __func__, psBuffer->ui32Width, psBuffer->ui32Height, psBuffer->sCpuPAddr.uiAddr);
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32)DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height, DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	/* Initialise the system buffer to a nice rainbow. */
+	pvBufferCpuVAddr = DC_OSMapPhysAddr(psBuffer->sCpuPAddr, psBuffer->ui32SizeInBytes);
+	if (pvBufferCpuVAddr != NULL)
+	{
+		pui32Pixel = (IMG_UINT32 *)pvBufferCpuVAddr;
+
+#if (DCPDP_PIXEL_FORMAT_BPP == 4)
+		for (i = 0; i < psBuffer->ui32Height; ++i)
+		{
+			for (j = 0; j < psBuffer->ui32Width; ++j)
+			{
+				IMG_UINT32 index = i * psBuffer->ui32Width + j;
+				ui8Red   = 0xFF - (((i + 1) * 0xFF) / psBuffer->ui32Height);
+				ui8Green[0] = ((j + 1) * 0xFF) / psBuffer->ui32Width;
+				ui8Blue  = ((i + 1) * 0xFF) / psBuffer->ui32Height;
+				pui32Pixel[index] = 0xFF000000 | (ui8Red << 16) | (ui8Green[0] << 8) | (ui8Blue << 0);
+			}
+		}
+#endif
+
+#if (DCPDP_PIXEL_FORMAT_BPP == 3)
+		/* Optimisation for faster upload on emu, i varies over the height, j the width, and k the pixel buffer */
+		/* Relies on the width being a factor of 4 */
+		for (i = 0; i < psBuffer->ui32Height; ++i)
+		{
+			for (j = k = 0; j < psBuffer->ui32Width; j += 4, k += 3)
+			{
+				index = i * ((psBuffer->ui32Width * 3) / 4) + k;
+				ui8Red      = 0xFF - (((i + 1) * 0xFF) / psBuffer->ui32Height);
+				ui8Green[0] = ((j + 1) * 0xFF) / psBuffer->ui32Width;
+				ui8Green[1] = ((j + 2) * 0xFF) / psBuffer->ui32Width;
+				ui8Green[2] = ((j + 3) * 0xFF) / psBuffer->ui32Width;
+				ui8Green[3] = ((j + 4) * 0xFF) / psBuffer->ui32Width;
+				ui8Blue     = ((i + 1) * 0xFF) / psBuffer->ui32Height;
+				pui32Pixel[index + 0] = (ui8Blue << 24)     | (ui8Red << 16)      | (ui8Green[0] << 8) | (ui8Blue);
+				pui32Pixel[index + 1] = (ui8Green[1] << 24) | (ui8Blue << 16)     | (ui8Red << 8)      | (ui8Green[2]);
+				pui32Pixel[index + 2] = (ui8Red << 24)      | (ui8Green[3] << 16) | (ui8Blue << 8)     | (ui8Red);
+			}
+		}
+#endif
+		DC_OSUnmapPhysAddr(pvBufferCpuVAddr, psBuffer->ui32SizeInBytes);
+	}
+
+	psDeviceData->psSystemBuffer = psBuffer;
+
+	return PVRSRV_OK;
+}
+
+
+static PVRSRV_ERROR InitTimingData(DCPDP_DEVICE * pvDevice)
+{
+	const DCPDP_MODULE_PARAMETERS * pvModuleParameters;
+	DCPDP_TIMING_DATA * psTimingData;
+
+	pvDevice->uiTimingDataSize = 1;
+	pvDevice->pasTimingData = DC_OSAllocMem(sizeof(DCPDP_TIMING_DATA) * pvDevice->uiTimingDataSize);
+	if (pvDevice->pasTimingData	== NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	psTimingData = &pvDevice->pasTimingData[pvDevice->uiTimingDataIndex];
+
+	pvModuleParameters = DCPDPGetModuleParameters();
+
+	psTimingData->ui32HDisplay      = pvModuleParameters->ui32PDPWidth;
+	psTimingData->ui32HTotal        = psTimingData->ui32HDisplay;
+	psTimingData->ui32HBackPorch    = 0;
+	psTimingData->ui32HFrontPorch   = psTimingData->ui32HTotal;
+	psTimingData->ui32HLeftBorder   = psTimingData->ui32HBackPorch;
+	psTimingData->ui32HActiveStart  = psTimingData->ui32HLeftBorder;
+	psTimingData->ui32HRightBorder  = psTimingData->ui32HActiveStart + psTimingData->ui32HDisplay;
+	psTimingData->ui8HSyncPolarity  = 0;
+
+	psTimingData->ui32VDisplay      = pvModuleParameters->ui32PDPHeight;
+	psTimingData->ui32VTotal        = psTimingData->ui32VDisplay;
+	psTimingData->ui32VBackPorch    = 0;
+	psTimingData->ui32VFrontPorch   = psTimingData->ui32VTotal;
+	psTimingData->ui32VTopBorder    = psTimingData->ui32VBackPorch;
+	psTimingData->ui32VActiveStart  = psTimingData->ui32VTopBorder;
+	psTimingData->ui32VBottomBorder = psTimingData->ui32VActiveStart + psTimingData->ui32VDisplay;
+	psTimingData->ui8VSyncPolarity  = 0;
+	psTimingData->ui32VRefresh   = 60;
+
+	PDP_DEBUG_PRINT("- %s: HDisplay: %d, HActive: %d, HLeftBorder: %d, HTotal: %d, HFrontPorch: %d, HBackPorch: %d, HRightBorder: %d\n", __func__, psTimingData->ui32HDisplay, psTimingData->ui32HActiveStart,
+	psTimingData->ui32HLeftBorder, psTimingData->ui32HTotal, psTimingData->ui32HFrontPorch, psTimingData->ui32HBackPorch, psTimingData->ui32HRightBorder);
+
+	PDP_DEBUG_PRINT("- %s: VDisplay: %d, VActive: %d, VTopBorder: %d, VTotal: %d, VFrontPorch: %d, VBackPorch: %d, VBottomBorder: %d\n", __func__, psTimingData->ui32VDisplay, psTimingData->ui32VActiveStart,
+	psTimingData->ui32VTopBorder, psTimingData->ui32VTotal, psTimingData->ui32VFrontPorch, psTimingData->ui32VBackPorch, psTimingData->ui32VBottomBorder);
+
+	return PVRSRV_OK;
+}
+
+static void UpdateTimingData(DCPDP_DEVICE * pvDevice)
+{
+	DCPDP_TIMING_DATA * psTimingData = &pvDevice->pasTimingData[pvDevice->uiTimingDataIndex];
+	DTD * dtd;
+
+	if (pvDevice->videoParams.mDtdActiveIndex > HDMI_DTD_MAX)
+	{
+		PDP_ERROR_PRINT(" - %s: DTD Active index is greater than max (%d), resetting to 0\n", __func__, HDMI_DTD_MAX);
+		pvDevice->videoParams.mDtdActiveIndex = 0;
+	}
+
+	dtd = &pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex];
+
+	dtd->mHBackPorchWidth = dtd->mHFrontPorchWidth;
+	dtd->mVBackPorchWidth = dtd->mVFrontPorchWidth;
+
+	psTimingData->ui32HDisplay      = dtd->mHActive;
+	psTimingData->ui32HTotal        = dtd->mHActive + dtd->mHBlanking + 2 * dtd->mHBorder;
+	psTimingData->ui32HBackPorch    = dtd->mHSyncPulseWidth;
+	psTimingData->ui32HFrontPorch   = psTimingData->ui32HTotal - dtd->mHFrontPorchWidth;
+	psTimingData->ui32HLeftBorder   = psTimingData->ui32HBackPorch + dtd->mHBackPorchWidth;
+	psTimingData->ui32HActiveStart  = psTimingData->ui32HLeftBorder + dtd->mHBorder;
+	psTimingData->ui32HRightBorder  = psTimingData->ui32HActiveStart + psTimingData->ui32HDisplay;
+	psTimingData->ui8HSyncPolarity  = !dtd->mHSyncPolarity; // For PDP, 0 means active high
+
+	psTimingData->ui32VDisplay      = dtd->mVActive;
+	psTimingData->ui32VTotal        = dtd->mVActive + dtd->mVBlanking + 2 * dtd->mVBorder;
+	psTimingData->ui32VBackPorch    = dtd->mVSyncPulseWidth;
+	psTimingData->ui32VFrontPorch   = psTimingData->ui32VTotal - dtd->mVFrontPorchWidth;
+	psTimingData->ui32VTopBorder    = psTimingData->ui32VBackPorch + dtd->mVBackPorchWidth; //dtd->mVBlanking - dtd->mVFrontPorchWidth;
+	psTimingData->ui32VActiveStart  = psTimingData->ui32VTopBorder + dtd->mVBorder;
+	psTimingData->ui32VBottomBorder = psTimingData->ui32VActiveStart + psTimingData->ui32VDisplay;
+	psTimingData->ui8VSyncPolarity  = !dtd->mVSyncPolarity; // For PDP, 0 means active high
+
+	if (dtd->mHSyncPolarity == 1 && dtd->mVSyncPolarity == 0)
+	{
+		psTimingData->bReducedBlanking = IMG_TRUE;
+	}
+
+	PDP_DEBUG_PRINT("- %s: HDisplay: %d, HActive: %d, HLeftBorder: %d, HTotal: %d, HFrontPorch: %d, HBackPorch: %d, HRightBorder: %d\n", __func__, psTimingData->ui32HDisplay, psTimingData->ui32HActiveStart,
+	psTimingData->ui32HLeftBorder, psTimingData->ui32HTotal, psTimingData->ui32HFrontPorch, psTimingData->ui32HBackPorch, psTimingData->ui32HRightBorder);
+
+	PDP_DEBUG_PRINT("- %s: VDisplay: %d, VActive: %d, VTopBorder: %d, VTotal: %d, VFrontPorch: %d, VBackPorch: %d, VBottomBorder: %d\n", __func__, psTimingData->ui32VDisplay, psTimingData->ui32VActiveStart,
+	psTimingData->ui32VTopBorder, psTimingData->ui32VTotal, psTimingData->ui32VFrontPorch, psTimingData->ui32VBackPorch, psTimingData->ui32VBottomBorder);
+
+	psTimingData->ui32VRefresh   = 60;
+
+}
+
+PVRSRV_ERROR DCPDPInit(void *pvDevice, DCPDP_DEVICE **ppsDeviceData, IMG_UINT32 ui32Instance)
+{
+	DCPDP_DEVICE *psDeviceData;
+	PVRSRV_ERROR eError;
+	PPVRSRV_DEVICE_NODE psDevNode;
+#if defined(PDP_DEBUG)
+	static IMG_BOOL bFirstTime = IMG_TRUE;
+#endif
+
+	if (ppsDeviceData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DC_OSSetDrvName(DRVNAME);
+	PDP_CHECKPOINT;
+
+	psDeviceData = DC_OSCallocMem(sizeof(*psDeviceData));
+	if (psDeviceData == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to allocate device data\n", __func__);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psDeviceData->pvDevice = pvDevice;
+	psDeviceData->ePixelFormat = DCPDP_PIXEL_FORMAT;
+#if defined(PDP_DEBUG)
+	/* Initialise our debug hooks for tracking each device and the buffers
+	 * allocated for that device. Device tracking is hooked from our local
+	 * gsPdpDeviceListHead and the buffers are attached to the per-device
+	 * sDbgBufListHead DLLIST. These are used by PDPDebugCtrl().
+	 */
+	dllist_init(&psDeviceData->sDbgListNode);
+	dllist_init(&psDeviceData->sDbgBufListHead);
+	/* Initialise the tracking list if this is our first call. We assume that
+	 * we are called from a single-threaded __init() routine from the OS
+	 * driver layer so we do not need to implement mutex locking to handle
+	 * multiple simultaneous calls. */
+	if (bFirstTime)
+	{
+		dllist_init(&gsPdpDeviceListHead);
+		bFirstTime = IMG_FALSE;
+	}
+	dllist_add_to_tail(&gsPdpDeviceListHead, &psDeviceData->sDbgListNode);
+	gNumPdpDevices++;
+#endif
+
+	eError = DC_OSPVRServicesConnectionOpen(&psDeviceData->hPVRServicesConnection);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to open connection to PVR Services (%d)\n",
+				 __func__, eError);
+		goto ErrorFreeDeviceData;
+	}
+
+	eError = DC_OSPVRServicesSetupFuncs(psDeviceData->hPVRServicesConnection, &psDeviceData->sPVRServicesFuncs);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to setup PVR Services function table (%d)\n",
+				 __func__, eError);
+		goto ErrorPDPOSPVRServicesConnectionClose;
+	}
+
+	psDevNode = psDeviceData->sPVRServicesFuncs.pfnGetDeviceInstance(ui32Instance);
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode != NULL, eError, ErrorPDPOSPVRServicesConnectionClose);
+
+
+	/* Services manages the card memory so we need to acquire the display controller
+	   heap (which is a carve out of the card memory) so we can allocate our own buffers. */
+	eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapAcquireByUsage(PHYS_HEAP_USAGE_DISPLAY,
+								    psDevNode,
+								    &psDeviceData->psPhysHeap);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to acquire heap (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPDPOSPVRServicesConnectionClose;
+	}
+
+	/* The PDP can only access local memory so make sure the heap is of the correct type */
+	if (psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetType(psDeviceData->psPhysHeap) != PHYS_HEAP_TYPE_LMA)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Heap is of the wrong type\n", __func__);
+		eError = PVRSRV_ERROR_INVALID_HEAP;
+		goto ErrorPhysHeapRelease;
+	}
+
+	/* Get the CPU base address for the display heap */
+	eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetCpuPAddr(psDeviceData->psPhysHeap,
+								       &psDeviceData->sDispMemCpuPAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get display memory base address (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	eError = psDeviceData->sPVRServicesFuncs.pfnPhysHeapGetSize(psDeviceData->psPhysHeap,
+								    &psDeviceData->uiDispMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get the display memory size (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	DC_OSDebugPrintf(DBGLVL_DEBUG, " - %s: Display memory size (0x%016lX)\n",
+			 __func__, psDeviceData->uiDispMemSize);
+
+	/* Reserve and map the PDP registers */
+	psDeviceData->sPDPRegCpuPAddr.uiAddr =
+		DC_OSAddrRangeStart(psDeviceData->pvDevice, DCPDP2_REG_PCI_BASENUM) +
+		DCPDP2_PCI_PDP_REG_OFFSET;
+
+	if (DC_OSRequestAddrRegion(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE, DRVNAME) == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to reserve the PDP registers\n", __func__);
+
+		psDeviceData->sPDPRegCpuPAddr.uiAddr = 0;
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+
+		goto ErrorPhysHeapRelease;
+	}
+
+	psDeviceData->pvPDPRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE);
+	if (psDeviceData->pvPDPRegCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to map PDP registers\n", __func__);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+		goto ErrorReleaseMemRegion;
+	}
+
+	{
+		/*
+		 * 1GB Plato boards have issue with PDP initialisation. When PDP fails to
+		 * properly initialize, PDP registers are inaccessible.
+		 *
+		 * We can detect this by reading a register that has a known
+		 * and unchanging value.
+		 */
+		#define PLATO_PDP_CORE_ID (0x07010003)
+
+		IMG_UINT32 ui32CoreId;
+
+		ui32CoreId =
+			DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_CORE_ID_OFFSET);
+
+		if (ui32CoreId == PLATO_PDP_CORE_ID)
+		{
+
+			DC_OSDebugPrintf(DBGLVL_INFO, "%s: PDP initialized! (CoreId = 0x%x)",
+					__func__, ui32CoreId);
+		}
+		else
+		{
+			DC_OSDebugPrintf(DBGLVL_INFO, "%s: PDP failed to initialize! (CoreId = 0x%x)"
+					, __func__, ui32CoreId);
+
+			goto ErrorPDPOSUnmapPDPRegisters;
+		}
+	}
+
+	psDeviceData->pvPDPBifRegCpuVAddr = psDeviceData->pvPDPRegCpuVAddr + DCPDP2_BIF_REGS_OFFSET;
+
+	PDP_DEBUG_PRINT(" - %s: Register base CpuPAddr: %lx\n", __func__, psDeviceData->sPDPRegCpuPAddr.uiAddr);
+	PDP_DEBUG_PRINT(" - %s: Register base CpuVAddr: %p\n", __func__, psDeviceData->pvPDPRegCpuVAddr);
+
+	/* Initialize timing data to defaults */
+	eError = InitTimingData(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialize timing data\n", __func__);
+		goto ErrorPDPOSUnmapPDPRegisters;
+	}
+
+	/* Create a system buffer */
+	eError = InitSystemBuffer(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialise the system buffer (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorFreeTimingData;
+	}
+
+	eError = psDeviceData->sPVRServicesFuncs.pfnDCRegisterDevice(&g_sDCFunctions,
+								     DCPDP_MAX_COMMANDS_INFLIGHT,
+								     psDeviceData,
+								     &psDeviceData->hPVRServicesDevice);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to register the display device (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDeinitSysBuffer;
+	}
+
+	/* Register our handler */
+	eError = psDeviceData->sPVRServicesFuncs.pfnSysInstallDeviceLISR(pvDevice,
+									 PLATO_IRQ_PDP,
+									 DRVNAME,
+									 DisplayLISRHandler,
+									 psDeviceData,
+									 &psDeviceData->hLISRData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to install the display device interrupt handler (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDCUnregisterDevice;
+	}
+
+	eError = DC_OSWorkQueueCreate(&psDeviceData->hVSyncWorkQueue, 1);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work queue (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDCUninstallLISR;
+	}
+
+	eError = DC_OSWorkQueueCreateWorkItem(&psDeviceData->hVSyncWorkItem, VSyncProcessor, psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work item (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		goto ErrorDCDestroyWorkQueue;
+	}
+
+	*ppsDeviceData = psDeviceData;
+
+	return PVRSRV_OK;
+
+ErrorDCDestroyWorkQueue:
+	DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+ErrorDCUninstallLISR:
+	psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+
+ErrorDCUnregisterDevice:
+	psDeviceData->sPVRServicesFuncs.pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+
+ErrorDeinitSysBuffer:
+	DeInitSystemBuffer(psDeviceData);
+
+ErrorFreeTimingData:
+	DC_OSFreeMem(psDeviceData->pasTimingData);
+
+ErrorPDPOSUnmapPDPRegisters:
+	DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+ErrorReleaseMemRegion:
+	DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+ErrorPhysHeapRelease:
+	psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+ErrorPDPOSPVRServicesConnectionClose:
+	DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+
+ErrorFreeDeviceData:
+	DC_OSFreeMem(psDeviceData);
+
+	return eError;
+
+}
+
+PVRSRV_ERROR DCPDPStart(DCPDP_DEVICE *psDeviceData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	UpdateTimingData(psDeviceData);
+
+	/* Create a system buffer */
+	eError = InitSystemBuffer(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialise the system buffer (%s)\n",
+				 __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError));
+		return eError;
+	}
+
+	SetModeRegisters(psDeviceData);
+	psDeviceData->bVSyncEnabled = IMG_TRUE;
+
+	Flip(psDeviceData->psSystemBuffer);
+
+	EnableVSyncInterrupt(psDeviceData);
+
+	/* Print some useful information */
+	DC_OSDebugPrintf(DBGLVL_INFO, " using mode %ux%u@%uHz (phys: %ux%u)\n",
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh,
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay * 1000) / DCPDP_DPI * 254) / 10000),
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay * 1000) / DCPDP_DPI * 254) / 10000));
+	DC_OSDebugPrintf(DBGLVL_INFO, " register base: 0x%llxK\n", psDeviceData->sPDPRegCpuPAddr.uiAddr);
+	DC_OSDebugPrintf(DBGLVL_INFO, " memory base: 0x%llxK\n", psDeviceData->sDispMemCpuPAddr.uiAddr);
+
+	return PVRSRV_OK;
+
+}
+
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, void **ppvDevice)
+{
+	PDP_CHECKPOINT;
+
+	if (psDeviceData)
+	{
+		DC_OSWorkQueueDestroyWorkItem(psDeviceData->hVSyncWorkItem);
+		DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+		DisableVSyncInterrupt(psDeviceData);
+
+		psDeviceData->sPVRServicesFuncs.pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+		{
+			psDeviceData->psFlipContext = NULL;
+		}
+		else
+#else
+		{
+			DC_ASSERT(psDeviceData->psFlipContext == NULL);
+		}
+#endif
+		(void)psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+
+		DeInitSystemBuffer(psDeviceData);
+
+		/* Disable stream 1 to ensure PDP2 doesn't try to access on card memory */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1CTRL_OFFSET, PLACE_FIELD(PDP_GRPH1CTRL_GRPH1STREN, 0));
+
+		DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+		DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, DCPDP2_PCI_PDP_REG_SIZE);
+
+		psDeviceData->sPVRServicesFuncs.pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+		DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+
+		if (ppvDevice != NULL)
+		{
+			*ppvDevice = psDeviceData->pvDevice;
+		}
+
+		PDP_CHECKPOINT;
+
+		DC_OSFreeMem(psDeviceData->pasTimingData);
+#if defined(PDP_DEBUG)
+		/* Remove this device from the list of known PDP devices */
+		dllist_remove_node(&psDeviceData->sDbgListNode);
+		gNumPdpDevices--;
+#endif
+		DC_OSFreeMem(psDeviceData);
+	}
+}
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable)
+{
+	if (psDeviceData)
+	{
+		IMG_UINT32 ui32Value;
+		ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1CTRL_OFFSET);
+		SET_FIELD(ui32Value, PDP_GRPH1CTRL_GRPH1STREN, bEnable ? 1 : 0);
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, PDP_GRPH1CTRL_OFFSET, ui32Value);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..12a9dfb5780b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Kbuild.mk
@@ -0,0 +1,55 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-$(CONFIG_X86) += -mno-soft-float
+
+ccflags-y += \
+ -I$(TOP)/services/display/include \
+ -I$(TOP)/services/display/dc_pdp2/include \
+ -I$(TOP)/services/shared/include \
+ -I$(TOP)/services/system/$(PVR_SYSTEM) \
+ -I$(TOP)/services/system/$(PVR_ARCH)/$(PVR_SYSTEM) \
+ -I$(TOP)/include/public \
+ -I$(TOP)/include/$(PVR_ARCH)/public
+
+dc_pdp2-y += \
+ services/display/dc_pdp2/common/dc_pdp_common.o \
+ services/display/dc_pdp2/env/linux/pdp_osfuncs.o \
+ services/display/common/env/linux/dc_osfuncs.o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Linux.mk
new file mode 100644
index 000000000000..4ff6a1dcd739
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := dc_pdp2
+
+dc_pdp2_type := kernel_module
+dc_pdp2_target := dc_pdp2.ko
+dc_pdp2_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/pdp_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/pdp_osfuncs.c
new file mode 100644
index 000000000000..5a3d1ab5931f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/env/linux/pdp_osfuncs.c
@@ -0,0 +1,620 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#include "dc_osfuncs.h"
+#include "dc_pdp.h"
+
+#if defined(ENABLE_PLATO_HDMI)
+/* APIs needed from plato_hdmi */
+EXPORT_SYMBOL(PDPInitialize);
+#endif
+
+#define DCPDP_WIDTH_MIN			(640)
+#define DCPDP_WIDTH_MAX			(1920)
+#define DCPDP_HEIGHT_MIN		(480)
+#define DCPDP_HEIGHT_MAX		(1080)
+#define DCPDP_WIDTH_DEFAULT		(640)
+#define DCPDP_HEIGHT_DEFAULT	(480)
+
+#define DCPDP_DEBUGFS_DISPLAY_ENABLED   "display_enabled"
+#define DCPDP_DEBUGFS_NUM_DEVICES       "num_devices"
+
+#if defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_WIDTH defined but DCPDP_HEIGHT not defined
+#elif !defined(DCPDP_WIDTH) && defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_HEIGHT defined but DCPDP_WIDTH not defined
+#elif !defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#define DCPDP_WIDTH			DCPDP_WIDTH_DEFAULT
+#define DCPDP_HEIGHT		DCPDP_HEIGHT_DEFAULT
+#elif (DCPDP_WIDTH > DCPDP_WIDTH_MAX)
+#error ERROR: DCPDP_WIDTH too large (max: 1920)
+#elif (DCPDP_WIDTH < DCPDP_WIDTH_MIN)
+#error ERROR: DCPDP_WIDTH too small (min: 640)
+#elif (DCPDP_HEIGHT > DCPDP_HEIGHT_MAX)
+#error ERROR: DCPDP_HEIGHT too large (max: 1080)
+#elif (DCPDP_HEIGHT < DCPDP_HEIGHT_MIN)
+#error ERROR: DCPDP_HEIGHT too small (max: 480)
+#endif
+
+static DLLIST_NODE g_sDeviceDataListHead;    /* List head for devices supported
+                                                by this driver. */
+static IMG_UINT32 g_uiNumPDPDevs;            /* Number of devices found. */
+
+static struct dentry    *g_psDebugFSEntryDir; /* Top-level 'dc_pdp2' entry */
+static struct dentry    *g_psNumDevicesEntry; /* 'num_devices' entry */
+static struct dentry    *g_psDispEnabledEntry; /* 'display_enabled' entry */
+
+/* PDP module parameters */
+DCPDP_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32PDPEnabled = 1,
+	.ui32PDPWidth   = DCPDP_WIDTH,
+	.ui32PDPHeight  = DCPDP_HEIGHT
+};
+module_param_named(mem_en, sModuleParams.ui32PDPEnabled, uint, S_IRUGO | S_IWUSR);
+module_param_named(width,  sModuleParams.ui32PDPWidth,   uint, S_IRUGO | S_IWUSR);
+module_param_named(height, sModuleParams.ui32PDPHeight,  uint, S_IRUGO | S_IWUSR);
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void)
+{
+	return &sModuleParams;
+}
+
+static int DisplayEnabledOpen(struct inode *psINode, struct file *psFile)
+{
+	psFile->private_data = psINode->i_private;
+
+	return 0;
+}
+
+static ssize_t NumDevicesRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	char aszBuffer[] = "XX\n";
+	loff_t uiPosition = *puiPosition;
+	size_t uiBufferSize = ARRAY_SIZE(aszBuffer);
+	size_t uiLocCount;
+
+	PVR_UNREFERENCED_PARAMETER(psFile);
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	(void) DC_OSSNPrintf(aszBuffer, sizeof(aszBuffer), "%u", g_uiNumPDPDevs);
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+	if (copy_to_user(psUserBuffer, &aszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static ssize_t DisplayEnabledRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)psFile->private_data;
+	IMG_BOOL bPDPEnabled;
+	loff_t uiPosition = *puiPosition;
+	char pszBuffer[] = "N\n";
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	/* Handle the top-level 'meta' entry which reports the values of all of
+	 * the underlying unit(s) using logical AND of all values.
+	 */
+	if (psDeviceData == NULL)
+	{
+		/* Iterate across all present units and construct the top-level value.
+		 * IMG_TRUE => all devices are enabled.
+		 * IMG_FALSE => at least one device is disabled.
+		 */
+		DLLIST_NODE *psNode, *psNext;
+
+		bPDPEnabled = IMG_TRUE;
+		dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+		{
+			psDeviceData = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+
+			bPDPEnabled &= psDeviceData->bPDPEnabled;
+		}
+	}
+	else
+	{
+		bPDPEnabled = psDeviceData->bPDPEnabled;
+	}
+
+	if (bPDPEnabled == IMG_TRUE)
+	{
+		pszBuffer[0] = 'Y';
+	}
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+	if (copy_to_user(psUserBuffer, &pszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static ssize_t DisplayEnabledWrite(struct file *psFile,
+				   const char __user *psUserBuffer,
+				   size_t uiCount,
+				   loff_t *puiPosition)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)(psFile->private_data);
+	char pszBuffer[3];
+	bool bPDPEnabled;
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	uiLocCount = min(uiCount, (uiBufferSize - 1U));
+
+	if (copy_from_user(pszBuffer, psUserBuffer, uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	pszBuffer[uiLocCount] = '\0';
+
+	if (strtobool(pszBuffer, &bPDPEnabled) == 0)
+	{
+		/* If we have no device-specific data we iterate over all units */
+		if (psDeviceData == NULL)
+		{
+			DLLIST_NODE *psNode, *psNext;
+			dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+			{
+				psDeviceData = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+				psDeviceData->bPDPEnabled = bPDPEnabled ? IMG_TRUE : IMG_FALSE;
+
+#if defined(PDP_DEBUG)
+				printk(KERN_INFO DRVNAME " - %s: Setting display %u %s\n", __func__,
+				       psDeviceData->ui32Instance, bPDPEnabled ? "Enabled" : "Disabled");
+#endif
+
+				DCPDPEnableMemoryRequest(psDeviceData, psDeviceData->bPDPEnabled);
+			}
+		}
+		else
+		{
+			psDeviceData->bPDPEnabled = bPDPEnabled ? IMG_TRUE : IMG_FALSE;
+
+#if defined(PDP_DEBUG)
+			printk(KERN_INFO DRVNAME " - %s: Setting display %u %s\n", __func__,
+			       psDeviceData->ui32Instance, bPDPEnabled ? "Enabled" : "Disabled");
+#endif
+
+			DCPDPEnableMemoryRequest(psDeviceData, psDeviceData->bPDPEnabled);
+		}
+	}
+
+	return (ssize_t)uiLocCount;
+}
+
+#if defined(PDP_DEBUG)
+static long DisplayEnabledIoctl(struct file *file, unsigned int ioctl, unsigned long argp)
+{
+	PDPDebugCtrl();
+	return 0;
+}
+#endif /* PDP_DEBUG */
+
+static const struct file_operations gsDisplayEnabledFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = DisplayEnabledOpen,
+	.read = DisplayEnabledRead,
+	.write = DisplayEnabledWrite,
+	.llseek = default_llseek,
+#if defined(PDP_DEBUG)
+	.unlocked_ioctl = DisplayEnabledIoctl
+#endif /* PDP_DEBUG */
+};
+
+static const struct file_operations gsNumDevicesFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = DisplayEnabledOpen,
+	.read = NumDevicesRead,
+	.llseek = default_llseek
+};
+
+PVRSRV_ERROR PDPInitialize(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psNode, *psNext;
+
+	/* Iterate over all devices and initialise them. May need to be changed to
+	 * support different video params per device ... TBD
+	 */
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		DCPDP_DEVICE *psDeviceData = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+		if (psDeviceData->ui32Instance == uiInstance)
+		{
+			(void) printk(KERN_INFO DRVNAME " - %s: Matched Device %u\n",
+			              __func__, psDeviceData->ui32Instance);
+
+			if (psDeviceData->bPreInitDone == IMG_FALSE)
+			{
+				(void) printk(KERN_ERR DRVNAME
+				              "-%s: Called before PreInit is complete\n",
+				              __func__);
+				return -ENODEV;
+			}
+
+			psDeviceData->videoParams = *pVideoParams;
+
+			eError = DCPDPStart(psDeviceData);
+			if (eError != PVRSRV_OK)
+			{
+				(void) printk(KERN_ERR DRVNAME
+				              " - %s: Failed to initialise device (%d)\n",
+				              __func__, eError);
+				return -ENODEV;
+			}
+
+			/* Enable / Disable display according to default modparam value */
+			if (sModuleParams.ui32PDPEnabled == 1)
+			{
+				if (!psDeviceData->bPDPEnabled)
+				{
+#if defined(PDP_DEBUG)
+					printk(KERN_WARNING DRVNAME " - %s: Enabling display %u\n",
+					       __func__, psDeviceData->ui32Instance);
+#endif
+					psDeviceData->bPDPEnabled = IMG_TRUE;
+					DCPDPEnableMemoryRequest(psDeviceData, IMG_TRUE);
+				}
+			}
+			else
+			{
+				if (psDeviceData->bPDPEnabled)
+				{
+#if defined(PDP_DEBUG)
+					printk(KERN_WARNING DRVNAME " - %s: Disabling display %u\n",
+					       __func__, psDeviceData->ui32Instance);
+#endif
+					psDeviceData->bPDPEnabled = IMG_FALSE;
+					DCPDPEnableMemoryRequest(psDeviceData, IMG_FALSE);
+				}
+			}
+		}
+	}
+
+	return eError;
+}
+
+static struct pci_dev *dc_pdp_getdevice(struct pci_dev *fromDev)
+{
+	struct pci_dev *psPCIDev;
+
+	psPCIDev = pci_get_device(DCPDP2_VENDOR_ID_PLATO,
+	                          DCPDP2_DEVICE_ID_PCI_PLATO,
+	                          fromDev);
+
+	return psPCIDev;
+}
+
+static int __init dc_pdp_init(void)
+{
+	DCPDP_DEVICE *psDevicePriv;
+	struct pci_dev *psPCIDev, *fromDev = NULL;
+	PVRSRV_ERROR eError;
+	int error;
+	IMG_CHAR aszDevName[4];	/* Unit number of device "0" .. "N" */
+
+	/* Iterate over all PCI instances */
+	dllist_init(&g_sDeviceDataListHead);
+	while ((psPCIDev = dc_pdp_getdevice(fromDev)) != NULL)
+	{
+		fromDev = psPCIDev;
+
+		error = pci_enable_device(psPCIDev);
+		if (error != 0)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to enable PCI device (%d)\n", __func__, error);
+			continue;  /* Go to next potential instance. Drop reference */
+		}
+
+		eError = DCPDPInit(&psPCIDev->dev, &psDevicePriv, g_uiNumPDPDevs);
+		if (eError != PVRSRV_OK)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to initialise device (%d)\n", __func__, eError);
+			continue;  /* Go to next potential instance. Drop reference */
+		}
+
+		dllist_init(&psDevicePriv->sListNode);
+
+		/* To prevent possible problems with system suspend/resume, we don't
+		   keep the device enabled, but rely on the fact that the Rogue driver
+		   will have done a pci_enable_device. */
+		pci_disable_device(psPCIDev);
+
+		/* We need to create a debugFS top-level entry for the dc_pdp2 device
+		 * once we've found the first PCI device. This is used as the root
+		 * debugFS directory for device-specific entries and holds the number
+		 * of devices found entry in 'num_devices'. Only do this on the first
+		 * iteration of the discovery loop.
+		 * Note: The creation of debugFS style entries may be disabled due to
+		 *       OS configuration. If so, we will continue with the driver
+		 *       being initialised but there will not be a way of observing
+		 *       the values provided by the debugFS entries.
+		 */
+		if (g_uiNumPDPDevs == 0U)
+		{
+			/* Top-level directory entry. All devices have a sub-directory */
+			g_psDebugFSEntryDir = debugfs_create_dir(DRVNAME, NULL);
+			if (IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME
+				       " - %s: Failed to create '%s' debugfs root directory "
+				       "(debugfs entries won't be available)\n", __func__,
+				       DRVNAME);
+				g_psDebugFSEntryDir = NULL;
+			}
+			else
+			{
+				/* Create the 'num_devices' top-level file. This will report the
+				 * current value of g_uiNumPDPDevs. Varies from 1 .. N.
+				 */
+				g_psNumDevicesEntry = debugfs_create_file(DCPDP_DEBUGFS_NUM_DEVICES,
+				                                          S_IFREG | S_IRUGO,
+				                                          g_psDebugFSEntryDir,
+				                                          NULL,	/* Top device */
+				                                          &gsNumDevicesFileOps);
+
+				if (IS_ERR_OR_NULL(g_psNumDevicesEntry))
+				{
+					printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s'"
+					       " debugfs file "
+					       "(debugfs entries won't be available)\n", __func__,
+					       DCPDP_DEBUGFS_NUM_DEVICES);
+					g_psNumDevicesEntry = NULL;
+				}
+			}
+
+			/* Create top-level 'display_enabled' file. This reports the
+			 * combined status of all device display_enabled settings and sets
+			 * all devices to the same status.
+			 */
+			g_psDispEnabledEntry = debugfs_create_file(DCPDP_DEBUGFS_DISPLAY_ENABLED,
+			                                              S_IFREG | S_IRUGO | S_IWUSR,
+			                                              g_psDebugFSEntryDir,
+			                                              NULL,	/* Top device */
+			                                              &gsDisplayEnabledFileOps);
+			if (IS_ERR_OR_NULL(g_psDispEnabledEntry))
+			{
+				printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s'"
+				       " debugfs file "
+				       "(debugfs entries won't be available)\n", __func__,
+				       DCPDP_DEBUGFS_DISPLAY_ENABLED);
+				g_psDispEnabledEntry = NULL;
+			}
+		}
+
+		/* Create device-specific directory name. This is the unit / instance
+		 * number of this device (i.e., the current value of g_uiNumPDPDevs as
+		 * this is incremented for each device we enumerate). */
+		if (DC_OSSNPrintf(aszDevName, sizeof(aszDevName), "%u", g_uiNumPDPDevs) >= sizeof(aszDevName))
+		{
+			printk(KERN_WARNING DRVNAME " - %s: DC_OSSNPrintf of '%u' failed"
+			       " Returned '%s'\n",
+			       __func__, g_uiNumPDPDevs, aszDevName);
+		};
+
+		if (!IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+		{
+			psDevicePriv->psDebugFSEntryDir = debugfs_create_dir(aszDevName, g_psDebugFSEntryDir);
+			if (IS_ERR_OR_NULL(psDevicePriv->psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME
+				       " - %s: Failed to create '%s/%s' debugfs device directory "
+				       "(debugfs entries won't be available)\n", __func__, DRVNAME,
+				       aszDevName);
+				psDevicePriv->psDebugFSEntryDir = NULL;
+			}
+			else
+			{
+
+				psDevicePriv->psDisplayEnabledEntry = debugfs_create_file(DCPDP_DEBUGFS_DISPLAY_ENABLED,
+				                                                          S_IFREG | S_IRUGO | S_IWUSR,
+				                                                          psDevicePriv->psDebugFSEntryDir,
+				                                                          psDevicePriv,	/* This device */
+				                                                          &gsDisplayEnabledFileOps);
+				if (IS_ERR_OR_NULL(psDevicePriv->psDisplayEnabledEntry))
+				{
+					printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs entry\n",
+					         __func__, DCPDP_DEBUGFS_DISPLAY_ENABLED);
+					         psDevicePriv->psDisplayEnabledEntry = NULL;
+				}
+			}
+		}
+
+#if defined(PDP_DEBUG)
+		printk(KERN_INFO DRVNAME " - %s: Initialised instance %u\n",
+		       __func__, g_uiNumPDPDevs);
+#endif
+
+		psDevicePriv->ui32Instance = g_uiNumPDPDevs++;
+		psDevicePriv->bPreInitDone = IMG_TRUE;
+		/* Add to the list of known devices */
+		dllist_add_to_tail(&g_sDeviceDataListHead, &psDevicePriv->sListNode);
+	}
+
+#if defined(PDP_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: Found %u devices\n", __func__, g_uiNumPDPDevs);
+#endif
+	return (g_uiNumPDPDevs > 0) ? 0 : -ENODEV;
+}
+
+static void __exit dc_pdp_deinit(void)
+{
+	DCPDP_DEVICE *psDevicePriv;
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		struct pci_dev *psPCIDev;
+		psDevicePriv = IMG_CONTAINER_OF(psNode, DCPDP_DEVICE, sListNode);
+
+		if (psDevicePriv->psDebugFSEntryDir)
+		{
+			if (psDevicePriv->psDisplayEnabledEntry)
+			{
+				debugfs_remove(psDevicePriv->psDisplayEnabledEntry);
+				psDevicePriv->psDisplayEnabledEntry = NULL;
+			}
+
+			debugfs_remove(psDevicePriv->psDebugFSEntryDir);
+			psDevicePriv->psDebugFSEntryDir = NULL;
+		}
+
+#if defined(PDP_DEBUG)
+		printk(KERN_WARNING DRVNAME " - %s: DeInit'ing instance %u\n",
+		       __func__, psDevicePriv->ui32Instance);
+#endif
+		dllist_remove_node(&psDevicePriv->sListNode);
+		DCPDPDeInit(psDevicePriv, (void *)&psPCIDev);
+
+		g_uiNumPDPDevs--;
+	}
+
+	/* Now clean up the top-level debugfs entries */
+	if (g_psNumDevicesEntry)
+	{
+		debugfs_remove(g_psNumDevicesEntry);
+		g_psNumDevicesEntry = NULL;
+	}
+
+	if (g_psDispEnabledEntry)
+	{
+		debugfs_remove(g_psDispEnabledEntry);
+		g_psDispEnabledEntry = NULL;
+	}
+
+	if (g_psDebugFSEntryDir)
+	{
+		debugfs_remove(g_psDebugFSEntryDir);
+		g_psDebugFSEntryDir = NULL;
+	}
+
+	if (!dllist_is_empty(&g_sDeviceDataListHead))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Still have devices initialised\n",
+		       __func__);
+	}
+}
+
+#if defined(PLATO_DISPLAY_PDUMP)
+
+void PDP_OSWriteHWReg32(void *pvLinRegBaseAddr, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	OSWriteHWReg32(pvLinRegBaseAddr, ui32Offset, ui32Value);
+	if (pvLinRegBaseAddr == g_psDeviceData->pvPDPRegCpuVAddr)
+	{
+		plato_pdump_reg32(pvLinRegBaseAddr, ui32Offset, ui32Value, DRVNAME);
+	}
+	else if (pvLinRegBaseAddr == g_psDeviceData->pvPDPBifRegCpuVAddr)
+	{
+		plato_pdump_reg32(pvLinRegBaseAddr, ui32Offset, ui32Value, "pdp_bif");
+	}
+}
+#endif
+
+module_init(dc_pdp_init);
+module_exit(dc_pdp_deinit);
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/include/dc_pdp.h b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/include/dc_pdp.h
new file mode 100644
index 000000000000..aee6b96ca92a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/dc_pdp2/include/dc_pdp.h
@@ -0,0 +1,322 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_PDP_H)
+#define DC_PDP_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "powervr/imgpixfmts.h"
+#include "pvrsrv.h"
+#include "dc_osfuncs.h"
+#include "plato.h"
+
+#if defined(PLATO_DISPLAY_PDUMP)
+#include "plato_pdump.h"
+#endif
+
+#define DRVNAME	"dc_pdp2"
+
+
+/*************************************************************************/ /*!
+ PCI Device Information
+*/ /**************************************************************************/
+
+#define DCPDP2_VENDOR_ID_PLATO			(0x1AEE)
+#define DCPDP2_DEVICE_ID_PCI_PLATO		(0x0003)
+
+/*************************************************************************/ /*!
+ PCI Device Base Address Information
+*/ /**************************************************************************/
+
+/* PDP registers on base address register 1 */
+#define DCPDP2_REG_PCI_BASENUM			(1)
+
+#define DCPDP2_PCI_PDP_REG_OFFSET		(0x200000)
+#define DCPDP2_PCI_PDP_REG_SIZE			(0x20000)
+
+#define DCPDP2_BIF_REGS_OFFSET			(0x1000)
+/*************************************************************************/ /*!
+ dc_pdp OS functions
+*/ /**************************************************************************/
+
+typedef struct DCPDP_MODULE_PARAMETERS_TAG
+{
+	IMG_UINT32  ui32PDPEnabled;
+	IMG_UINT32  ui32PDPWidth;
+	IMG_UINT32  ui32PDPHeight;
+} DCPDP_MODULE_PARAMETERS;
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void);
+
+/*************************************************************************/ /*!
+ dc_pdp Timing definitions
+*/ /**************************************************************************/
+#define DCPDP_MAX_COMMANDS_INFLIGHT	(2)
+
+/* This has to be less than or equal to the size of the type
+   used for ui32BufferUseMask in the DCPDP_DEVICE structure */
+#define DCPDP_MAX_BUFFERS		(32)
+
+#define DCPDP_MIN_DISPLAY_PERIOD	(0)
+#define DCPDP_MAX_DISPLAY_PERIOD	(2)
+
+#define DCPDP_PIXEL_FORMAT		IMG_PIXFMT_B8G8R8A8_UNORM
+#define DCPDP_PIXEL_FORMAT_BPP		(4)
+
+// Graphics/video pixel format:
+// 00000b 8-bit indexed (greyscale if LUT turned off) [RGB]
+// 00100b 4-bit alpha + 12-bit rgb444 [ARGB]
+// 00101b 1-bit alpha + 15-bit rgb555 [ARGB]
+// 00110b 24-bit rgb888 [RGB]
+// 00111b 16-bit rgb565 [RGB]
+// 01000b 8-bit alpha + 24-bit rgb888 [RGBA]
+// 10000b 8-bit alpha + 24-bit yuv888 [AYUV]
+// 10101b 30-bit YUV 4:4:4 YUV101010 [YUV]
+// 10110b 32-bit RGBX [RGBA]
+#define DCPDP_PIXEL_FORMAT_G		(0x00)
+#define DCPDP_PIXEL_FORMAT_ARGB4	(0x04)
+#define DCPDP_PIXEL_FORMAT_ARGB1555	(0x05)
+#define DCPDP_PIXEL_FORMAT_RGB8		(0x06)
+#define DCPDP_PIXEL_FORMAT_RGB565	(0x07)
+#define DCPDP_PIXEL_FORMAT_ARGB8	(0x08)
+#define DCPDP_PIXEL_FORMAT_AYUV8	(0x10)
+#define DCPDP_PIXEL_FORMAT_YUV10	(0x15)
+#define DCPDP_PIXEL_FORMAT_RGBA8	(0x16)
+
+/* Divide by 32, even though spec says to divide by 16 */
+#define PDP_GRPH1STRIDE_DIVIDE 5
+
+#ifndef DCPDP_DPI
+#define DCPDP_DPI			(160)
+#endif
+
+/* For reduced blanking displays, VEVENT needs to be right after VBBS */
+#define DCPDP_REDUCED_BLANKING_VEVENT	(1)
+
+/*************************************************************************/ /*!
+ dc_pdp Data structures
+*/ /**************************************************************************/
+typedef struct DCPDP_BUFFER_TAG DCPDP_BUFFER;
+typedef struct DCPDP_FLIP_CONFIG_TAG DCPDP_FLIP_CONFIG;
+typedef struct DCPDP_FLIP_CONTEXT_TAG DCPDP_FLIP_CONTEXT;
+typedef struct DCPDP_DEVICE_TAG DCPDP_DEVICE;
+
+typedef struct DCPDP_TIMING_DATA_TAG
+{
+	IMG_UINT32		ui32HDisplay;
+	IMG_UINT32		ui32HBackPorch;
+	IMG_UINT32		ui32HTotal;
+	IMG_UINT32		ui32HActiveStart;
+	IMG_UINT32		ui32HLeftBorder;
+	IMG_UINT32		ui32HRightBorder;
+	IMG_UINT32		ui32HFrontPorch;
+
+	IMG_UINT32		ui32VDisplay;
+	IMG_UINT32		ui32VBackPorch;
+	IMG_UINT32		ui32VTotal;
+	IMG_UINT32		ui32VActiveStart;
+	IMG_UINT32		ui32VTopBorder;
+	IMG_UINT32		ui32VBottomBorder;
+	IMG_UINT32		ui32VFrontPorch;
+
+	IMG_UINT8		ui8VSyncPolarity;
+	IMG_UINT8		ui8HSyncPolarity;
+
+	IMG_UINT32		ui32VRefresh;
+	IMG_UINT32		ui32ClockFreq;
+
+	IMG_BOOL		bReducedBlanking;
+} DCPDP_TIMING_DATA;
+
+struct DCPDP_BUFFER_TAG
+{
+	ATOMIC_T		i32RefCount;
+
+	DCPDP_DEVICE		*psDeviceData;
+
+	IMG_UINT32		ui32Width;
+	IMG_UINT32		ui32Height;
+	IMG_UINT32		ui32ByteStride;
+	IMG_UINT32		ui32SizeInBytes;
+	IMG_UINT32		ui32SizeInPages;
+	IMG_PIXFMT		ePixelFormat;
+
+	IMG_CPU_PHYADDR		sCpuPAddr;
+	IMG_DEV_PHYADDR		sDevPAddr;
+
+#if defined(PDP_DEBUG)
+	DLLIST_NODE		sDbgListNode;		/* Linked list of buffers per-device */
+#endif
+};
+
+typedef enum DCPDP_FLIP_CONFIG_STATUS_TAG
+{
+	DCPDP_FLIP_CONFIG_INACTIVE = 0,
+	DCPDP_FLIP_CONFIG_PENDING,
+	DCPDP_FLIP_CONFIG_ACTIVE,
+} DCPDP_FLIP_CONFIG_STATUS;
+
+
+/* Flip config structure used for queuing of flips */
+struct DCPDP_FLIP_CONFIG_TAG
+{
+	/* Context with which this config is associated */
+	DCPDP_FLIP_CONTEXT	*psContext;
+
+	/* Services config data */
+	IMG_HANDLE		hConfigData;
+
+	/* Current status of the config */
+	DCPDP_FLIP_CONFIG_STATUS eStatus;
+
+	/* Buffer to be displayed when the config is made active */
+	DCPDP_BUFFER		*psBuffer;
+
+	/* Number of frames for which this config should remain active */
+	IMG_UINT32		ui32DisplayPeriod;
+
+	/* New mode to possibly switch to after flip */
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+
+	/* Handle to a work item that processes the flip config queue */
+	IMG_HANDLE		hWorkItem;
+};
+
+struct DCPDP_FLIP_CONTEXT_TAG
+{
+	/* Device on which the context has been created */
+	DCPDP_DEVICE		*psDeviceData;
+
+	/* Lock protecting the queue of flip configs */
+	DC_SPINLOCK		sSpinLock;
+
+	/* Mutex to serialise calls back into Services */
+	void			*pvMutex;
+
+	/* Queue of configs that need processing */
+	DCPDP_FLIP_CONFIG	asFlipConfigQueue[DCPDP_MAX_COMMANDS_INFLIGHT];
+	IMG_UINT32		ui32FlipConfigInsertIndex;
+	IMG_UINT32		ui32FlipConfigRemoveIndex;
+
+	/* Handle to a work queue used to defer the processing of the
+	   flip config queue when a vsync interrupt is serviced */
+	IMG_HANDLE		hFlipConfigWorkQueue;
+
+	/* Services config data to retire */
+	IMG_HANDLE		hConfigDataToRetire;
+};
+
+struct DCPDP_DEVICE_TAG
+{
+	IMG_HANDLE              hPVRServicesConnection;
+	IMG_HANDLE              hPVRServicesDevice;
+	DC_SERVICES_FUNCS       sPVRServicesFuncs;
+
+	void                    *pvDevice;
+
+	IMG_HANDLE              hLISRData;
+
+	PHYS_HEAP               *psPhysHeap;
+
+	IMG_CPU_PHYADDR         sPDPRegCpuPAddr;
+	IMG_CPU_VIRTADDR        pvPDPRegCpuVAddr;
+
+	IMG_CPU_VIRTADDR        pvPDPBifRegCpuVAddr;
+
+	IMG_CPU_PHYADDR         sDispMemCpuPAddr;
+	IMG_UINT64              uiDispMemSize;
+
+	IMG_UINT32              uiTimingDataIndex;
+	DCPDP_TIMING_DATA       *pasTimingData;
+	IMG_UINT32              uiTimingDataSize;
+	IMG_PIXFMT              ePixelFormat;
+
+	IMG_UINT32              ui32BufferSize;
+	IMG_UINT32              ui32BufferCount;
+	IMG_UINT32              ui32BufferUseMask;
+
+	DCPDP_BUFFER            *psSystemBuffer;
+
+	DCPDP_FLIP_CONTEXT      *psFlipContext;
+
+	IMG_BOOL                bVSyncEnabled;
+	IMG_BOOL                bVSyncReporting;
+	IMG_INT64               i64LastVSyncTimeStamp;
+	IMG_HANDLE              hVSyncWorkQueue;
+	IMG_HANDLE              hVSyncWorkItem;
+
+	IMG_BOOL                bPreInitDone;
+
+#if defined(ENABLE_PLATO_HDMI)
+	VIDEO_PARAMS            videoParams;
+#endif
+#if defined(PDP_DEBUG)
+	DLLIST_NODE             sDbgListNode;            /* Linked list of debug devices
+	                                                    used by common layer */
+	DLLIST_NODE             sDbgBufListHead;         /* Head of linked buffers
+	                                                    for this device */
+#endif
+	DLLIST_NODE             sListNode;               /* Linked list of devices
+	                                                    used by OS driver layer */
+	IMG_UINT32              ui32Instance;            /* Device instance # */
+	struct dentry           *psDebugFSEntryDir;      /* Device-specific debugFS directory */
+	struct dentry           *psDisplayEnabledEntry;  /* 'display_enabled' debugFSentry for device */
+	IMG_BOOL                bPDPEnabled;             /* State of 'display_enabled' setting */
+};
+
+/*******************************************************************************
+ * dc_pdp common functions
+ ******************************************************************************/
+
+PVRSRV_ERROR DCPDPInit(void *pvDevice, DCPDP_DEVICE **ppsDeviceData, IMG_UINT32 ui32Instance);
+PVRSRV_ERROR DCPDPStart(DCPDP_DEVICE *psDeviceData);
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, void **ppvDevice);
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable);
+#if defined(PDP_DEBUG)
+void PDPDebugCtrl(void);
+#endif /* PDP_DEBUG */
+
+#if defined(PLATO_DISPLAY_PDUMP)
+void PDP_OSWriteHWReg32(void *pvLinRegBaseAddr, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+#endif
+
+#endif /* !defined(DC_PDP_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/include/dc_osfuncs.h b/drivers/gpu/drm/img/img-volcanic/services/display/include/dc_osfuncs.h
new file mode 100644
index 000000000000..ce41da1dce38
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/include/dc_osfuncs.h
@@ -0,0 +1,258 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_OSFUNCS_H)
+#define DC_OSFUNCS_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "physheap.h"
+#include "kerneldisplay.h"
+#include "pvrsrv.h"
+#include "syscommon.h"
+
+#if defined(DEBUG)
+#define DC_ASSERT(EXPR)							\
+	do								\
+	{								\
+		if (!(EXPR))						\
+		{							\
+			DC_OSAbort(__FILE__, __LINE__);			\
+		}							\
+	} while (0)
+#else
+#define DC_ASSERT(EXPR) (void)(EXPR) /* Null implementation of ASSERT (does nothing) */
+#endif
+
+#define DC_ALIGN(value, alignment) (((value) + ((alignment) - 1)) & ~((alignment) - 1))
+
+
+/******************************************************************************
+ * DC OS functions
+ *****************************************************************************/
+
+/* Services display class function pointers */
+typedef PVRSRV_ERROR (*PFN_DC_REGISTER_DEVICE)(DC_DEVICE_FUNCTIONS *psFuncTable, IMG_UINT32 ui32MaxConfigsInFlight, IMG_HANDLE hDeviceData, IMG_HANDLE *phSrvHandle);
+typedef void (*PFN_DC_UNREGISTER_DEVICE)(IMG_HANDLE hSrvHandle);
+typedef void (*PFN_DC_DISPLAY_CONFIGURATION_RETIRED)(IMG_HANDLE hConfigData);
+typedef PVRSRV_ERROR (*PFN_DC_IMPORT_BUFFER_ACQUIRE)(IMG_HANDLE hImport, IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize, IMG_UINT32 *pui32PageCount, IMG_DEV_PHYADDR **ppasDevPAddr);
+typedef void (*PFN_DC_IMPORT_BUFFER_RELEASE)(IMG_HANDLE hImport, IMG_DEV_PHYADDR *pasDevPAddr);
+
+/* Services physical heap function pointers */
+typedef PVRSRV_ERROR (*PFN_PHYS_HEAP_ACQUIRE_BY_USAGE)(PHYS_HEAP_USAGE_FLAGS ui32UsageFlag, PPVRSRV_DEVICE_NODE psDevNode, PHYS_HEAP **ppsPhysHeap);
+typedef void (*PFN_PHYS_HEAP_RELEASE)(PHYS_HEAP *psPhysHeap);
+typedef PHYS_HEAP_TYPE (*PFN_PHYS_HEAP_GET_TYPE)(PHYS_HEAP *psPhysHeap);
+typedef PVRSRV_ERROR (*PFN_PHYS_HEAP_GET_SIZE)(PHYS_HEAP *psPhysHeap, IMG_UINT64 *puiSize);
+typedef PVRSRV_ERROR (*PFN_PHYS_HEAP_GET_ADDRESS)(PHYS_HEAP *psPhysHeap, IMG_CPU_PHYADDR *psCpuPAddr);
+typedef void (*PFN_PHYS_HEAP_CPU_PADDR_TO_DEV_PADDR)(PHYS_HEAP *psPhysHeap, IMG_UINT32 ui32NumOfAddr, IMG_DEV_PHYADDR *psDevPAddr, IMG_CPU_PHYADDR *psCpuPAddr);
+
+/* Services system functions */
+typedef PVRSRV_ERROR (*PFN_SYS_INSTALL_DEVICE_LISR)(void *pvOSDevice, IMG_UINT32 ui32IRQ, const IMG_CHAR *pszName, PFN_LISR pfnLISR, void *pvData, IMG_HANDLE *phLISRData);
+typedef PVRSRV_ERROR (*PFN_SYS_UNINSTALL_DEVICE_LISR)(IMG_HANDLE hLISRData);
+
+/* Other service related functions */
+typedef void (*PFN_CHECK_STATUS)(void *hCmdCompCallerHandle);
+typedef const IMG_CHAR *(*PFN_GET_ERROR_STRING)(PVRSRV_ERROR eError);
+typedef PVRSRV_DEVICE_NODE *(*PFN_GET_DEVICE_INST)(IMG_UINT32 ui32Instance);
+
+#define DC_OS_BYTES_TO_PAGES(range)	(((range) + (DC_OSGetPageSize() - 1)) >> DC_OSGetPageShift())
+
+typedef struct DC_SERVICES_FUNCS_TAG
+{
+	/* Display class functions */
+	PFN_DC_REGISTER_DEVICE			pfnDCRegisterDevice;
+	PFN_DC_UNREGISTER_DEVICE		pfnDCUnregisterDevice;
+	PFN_DC_DISPLAY_CONFIGURATION_RETIRED	pfnDCDisplayConfigurationRetired;
+	PFN_DC_IMPORT_BUFFER_ACQUIRE		pfnDCImportBufferAcquire;
+	PFN_DC_IMPORT_BUFFER_RELEASE		pfnDCImportBufferRelease;
+
+	/* Physical heap functions */
+	PFN_PHYS_HEAP_ACQUIRE_BY_USAGE	pfnPhysHeapAcquireByUsage;
+	PFN_PHYS_HEAP_RELEASE			pfnPhysHeapRelease;
+	PFN_PHYS_HEAP_GET_TYPE			pfnPhysHeapGetType;
+	PFN_PHYS_HEAP_GET_SIZE			pfnPhysHeapGetSize;
+	PFN_PHYS_HEAP_GET_ADDRESS		pfnPhysHeapGetCpuPAddr;
+	PFN_PHYS_HEAP_CPU_PADDR_TO_DEV_PADDR	pfnPhysHeapCpuPAddrToDevPAddr;
+
+	/* System functions */
+	PFN_SYS_INSTALL_DEVICE_LISR		pfnSysInstallDeviceLISR;
+	PFN_SYS_UNINSTALL_DEVICE_LISR		pfnSysUninstallDeviceLISR;
+
+	/* Other service related functions */
+	PFN_CHECK_STATUS			pfnCheckStatus;
+	PFN_GET_ERROR_STRING			pfnGetErrorString;
+	PFN_GET_DEVICE_INST				pfnGetDeviceInstance;
+
+} DC_SERVICES_FUNCS;
+
+typedef enum
+{
+	DBGLVL_FATAL = 0,
+	DBGLVL_ALERT,
+	DBGLVL_ERROR,
+	DBGLVL_WARNING,
+	DBGLVL_NOTICE,
+	DBGLVL_INFO,
+	DBGLVL_DEBUG,
+} DC_OS_DEBUG_LEVEL;
+
+/* Logging & other misc stuff */
+void DC_OSSetDrvName(const IMG_CHAR *pszDrvName);
+void DC_OSAbort(const IMG_CHAR *pszFile, IMG_UINT32 ui32Line);
+
+void DC_OSDebugPrintf(DC_OS_DEBUG_LEVEL eDebugLevel, const IMG_CHAR *pszFormat, ...);
+
+IMG_CHAR *DC_OSStringNCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uiLength);
+IMG_INT DC_OSSNPrintf(IMG_CHAR *pszDest, size_t uiLength, const IMG_CHAR *pszFormat, ...);
+
+IMG_INT64 DC_OSClockns(void);
+
+/* Memory management */
+IMG_UINT32 DC_OSGetPageSize(void);
+IMG_UINT32 DC_OSGetPageShift(void);
+IMG_UINT32 DC_OSGetPageMask(void);
+
+void *DC_OSAllocMem(size_t uiSize);
+void *DC_OSCallocMem(size_t uiSize);
+void DC_OSFreeMem(void *pvMem);
+void DC_OSMemSet(void *pvDest, IMG_UINT8 ui8Value, size_t uiSize);
+
+IMG_UINT32 DC_OSAddrRangeStart(void *pvDevice, IMG_UINT8 ui8BaseNum);
+void *DC_OSRequestAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size, IMG_CHAR *pszRequestorName);
+void DC_OSReleaseAddrRegion(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size);
+
+IMG_CPU_VIRTADDR DC_OSMapPhysAddr(IMG_CPU_PHYADDR sCpuPAddr, IMG_UINT32 ui32Size);
+void DC_OSUnmapPhysAddr(IMG_CPU_VIRTADDR pvCpuVAddr, IMG_UINT32 ui32Size);
+
+/* Register access */
+IMG_UINT32 DC_OSReadReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset);
+void DC_OSWriteReg32(IMG_CPU_VIRTADDR pvRegCpuVBase, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+
+/* Floating-point support */
+IMG_UINT32 DC_OSDiv64(IMG_UINT64 ui64Numerator, IMG_UINT32 ui32Denominator);
+void DC_OSFloatingPointBegin(void);
+void DC_OSFloatingPointEnd(void);
+
+/* Inter-thread synchronisation */
+PVRSRV_ERROR DC_OSMutexCreate(void **ppvMutex);
+void DC_OSMutexDestroy(void *pvMutex);
+void DC_OSMutexLock(void *pvMutex);
+void DC_OSMutexUnlock(void *pvMutex);
+
+/* Locks that can be held in interrupt handlers */
+#if defined(__linux__)
+#include <linux/spinlock.h>
+typedef spinlock_t DC_SPINLOCK;
+# define DC_OSSpinLockCreate(psLock) spin_lock_init(psLock)
+# define DC_OSSpinLockDestroy(psLock) /* no need to destroy a spinlock */
+/* used outside hardirqs */
+# define DC_OSSpinLockIRQSave(psLock, ulIRQFlags) spin_lock_irqsave(psLock, ulIRQFlags)
+# define DC_OSSpinUnlockIRQRestore(psLock, ulIRQFlags) spin_unlock_irqrestore(psLock, ulIRQFlags)
+/* used inside hardirqs */
+# define DC_OSSpinLock(psLock) spin_lock(psLock)
+# define DC_OSSpinUnlock(psLock) spin_unlock(psLock)
+
+#elif defined(__QNXNTO__)
+#include <pthread.h>
+typedef pthread_spinlock_t DC_SPINLOCK;
+# define DC_OSSpinLockCreate(psLock) pthread_spin_init(psLock, PTHREAD_PROCESS_PRIVATE)
+# define DC_OSSpinLockDestroy(psLock) pthread_spin_destroy(psLock)
+# define DC_OSSpinLockIRQSave(psLock, ulIRQFlags) PVR_UNREFERENCED_PARAMETER(ulIRQFlags); pthread_spin_lock(psLock)
+# define DC_OSSpinUnlockIRQRestore(psLock, ulIRQFlags) PVR_UNREFERENCED_PARAMETER(ulIRQFlags); pthread_spin_unlock(psLock)
+
+# define DC_OSSpinLock(psLock) pthread_spin_lock(psLock)
+# define DC_OSSpinUnlock(psLock) pthread_spin_unlock(psLock)
+
+#elif defined(INTEGRITY_OS)
+typedef struct DC_SPINLOCK_TAG
+{
+	IMG_UINT64 ui64Lock;
+} DC_SPINLOCK;
+IMG_BOOL DC_OSSpinLockCreate(void **pplMutex);
+IMG_BOOL DC_OSSpinLockDestroy(void *plMutex);
+IMG_BOOL DC_OSSpinLock(void *plMutex);
+IMG_BOOL DC_OSSpinUnlock(void *plMutex);
+#define DC_OSSpinLockIRQSave(plMutex) DC_OSSpinLock(plMutex)
+#define DC_OSSpinUnlockIRQRestore(plMutex) DC_OSSpinUnlock(plMutex)
+#else
+#error "define an OS"
+#endif
+
+void DC_OSDelayus(IMG_UINT32 ui32Timeus);
+
+/* Services access */
+PVRSRV_ERROR DC_OSPVRServicesConnectionOpen(IMG_HANDLE *phPVRServicesConnection);
+void DC_OSPVRServicesConnectionClose(IMG_HANDLE hPVRServicesConnection);
+PVRSRV_ERROR DC_OSPVRServicesSetupFuncs(IMG_HANDLE hPVRServicesConnection, DC_SERVICES_FUNCS *psServicesFuncs);
+
+/* Workqueue functionality */
+/* OS work queue interface function pointer */
+typedef PVRSRV_ERROR (*PFN_WORK_PROCESSOR)(void *pvData);
+
+PVRSRV_ERROR DC_OSWorkQueueCreate(IMG_HANDLE *phQueue, IMG_UINT32 ui32Length);
+PVRSRV_ERROR DC_OSWorkQueueDestroy(IMG_HANDLE hQueue);
+PVRSRV_ERROR DC_OSWorkQueueFlush(IMG_HANDLE hQueue);
+
+PVRSRV_ERROR DC_OSWorkQueueCreateWorkItem(IMG_HANDLE *phWorkItem, PFN_WORK_PROCESSOR pfnProcessor, void *pvProcessorData);
+
+PVRSRV_ERROR DC_OSWorkQueueDestroyWorkItem(IMG_HANDLE phWorkItem);
+PVRSRV_ERROR DC_OSWorkQueueAddWorkItem(IMG_HANDLE hQueue, IMG_HANDLE hWorkItem);
+
+/* Atomic ops */
+
+#if defined(__linux__)
+
+#include <linux/atomic.h>
+
+#define DC_OSAtomicRead(pCounter) atomic_read(pCounter)
+#define DC_OSAtomicWrite(pCounter, i) atomic_set(pCounter, i)
+#define DC_OSAtomicIncrement(pCounter) atomic_inc_return(pCounter)
+#define DC_OSAtomicDecrement(pCounter) atomic_dec_return(pCounter)
+
+#else
+
+IMG_INT DC_OSAtomicRead(const ATOMIC_T *pCounter);
+void DC_OSAtomicWrite(ATOMIC_T *pCounter, IMG_INT v);
+IMG_INT DC_OSAtomicIncrement(ATOMIC_T *pCounter);
+IMG_INT DC_OSAtomicDecrement(ATOMIC_T *pCounter);
+
+#endif
+
+#endif /* !defined(DC_OSFUNCS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/common/dc_pdp_common.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/common/dc_pdp_common.c
new file mode 100644
index 000000000000..3747e0e9a26b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/common/dc_pdp_common.c
@@ -0,0 +1,1855 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "dc_pdp.h"
+#include "dc_osfuncs.h"
+#include "img_defs.h"
+#include "tcf_pll.h"
+#include "tcf_rgbpdp_regs.h"
+#include "powervr/imgpixfmts.h"
+
+#include "pvrsrv.h"
+
+#if defined(__linux__)
+#include <linux/spinlock.h>
+#endif
+
+#define DCPDP_MAX_COMMANDS_INFLIGHT	(2)
+
+/* This has to be less than or equal to the size of the type
+   used for ui32BufferUseMask in the DCPDP_DEVICE structure */
+#define DCPDP_MAX_BUFFERS		(32)
+
+#define DCPDP_MIN_DISPLAY_PERIOD	(0)
+#define DCPDP_MAX_DISPLAY_PERIOD	(2)
+
+#define DCPDP_PIXEL_FORMAT		IMG_PIXFMT_B8G8R8A8_UNORM
+#define DCPDP_PIXEL_FORMAT_BPP		(4)
+
+#ifndef DCPDP_DPI
+#define DCPDP_DPI			(160)
+#endif
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+#define	fGTF_MARGIN			1.8f	/* Top & bottom margin % */
+#define fGTF_HSYNC_MARGIN		8.0f	/* HSync margin % */
+#define fGTF_MIN_VSYNCBP		550.0f	/* Min VSync + Back Porch (us) */
+#define	fGTF_MIN_PORCH			1.0f	/* Minimum porch line / char cell */
+#define fGTF_CELL_GRAN			8.0f	/* assumed pixel granularity (cell) */
+#define fGTF_CELL_GRAN2			16.0f	/* assumed pixel granularity * 2 (cell) */
+
+#define fGTF_M				600.0f	/* Blanking Formula Gradient %/KHz */
+#define	fGTF_C				40.0f	/* Blanking Formula Offset % */
+#define fGTF_K				128.0f	/* Blanking Formula scale factor */
+#define fGTF_J				20.0f	/* Blanking Formula weighting % */
+
+#define GTF_VSYNC_WIDTH			7	/* Width of VSync lines */
+#define	GTF_MIN_PORCH			1	/* Minimum porch line / char cell */
+#define	GTF_MIN_PORCH_PIXELS		8	/* Minimum porch Pixels cell */
+#define GTF_CELL_GRAN			8	/* assumed pixel granularity (cell) */
+#define GTF_CELL_GRAN2			16	/* assumed pixel granularity * 2 (cell) */
+#endif
+
+typedef struct DCPDP_BUFFER_TAG DCPDP_BUFFER;
+typedef struct DCPDP_FLIP_CONFIG_TAG DCPDP_FLIP_CONFIG;
+typedef struct DCPDP_FLIP_CONTEXT_TAG DCPDP_FLIP_CONTEXT;
+
+typedef struct DCPDP_TIMING_DATA_TAG
+{
+	IMG_UINT32		ui32HDisplay;
+	IMG_UINT32		ui32HBackPorch;
+	IMG_UINT32		ui32HTotal;
+	IMG_UINT32		ui32HActiveStart;
+	IMG_UINT32		ui32HLeftBorder;
+	IMG_UINT32		ui32HRightBorder;
+	IMG_UINT32		ui32HFrontPorch;
+
+	IMG_UINT32		ui32VDisplay;
+	IMG_UINT32		ui32VBackPorch;
+	IMG_UINT32		ui32VTotal;
+	IMG_UINT32		ui32VActiveStart;
+	IMG_UINT32		ui32VTopBorder;
+	IMG_UINT32		ui32VBottomBorder;
+	IMG_UINT32		ui32VFrontPorch;
+
+	IMG_UINT32		ui32VRefresh;
+	IMG_UINT32		ui32ClockFreq;
+} DCPDP_TIMING_DATA;
+
+struct DCPDP_BUFFER_TAG
+{
+	ATOMIC_T		i32RefCount;
+
+	DCPDP_DEVICE		*psDeviceData;
+
+	IMG_UINT32		ui32Width;
+	IMG_UINT32		ui32Height;
+	IMG_UINT32		ui32ByteStride;
+	IMG_UINT32		ui32SizeInBytes;
+	IMG_UINT32		ui32SizeInPages;
+	IMG_PIXFMT		ePixelFormat;
+
+	IMG_CPU_PHYADDR		sCpuPAddr;
+	IMG_DEV_PHYADDR		sDevPAddr;
+	IMG_CPU_PHYADDR		sPDPPAddr;
+};
+
+typedef enum DCPDP_FLIP_CONFIG_STATUS_TAG
+{
+	DCPDP_FLIP_CONFIG_INACTIVE = 0,
+	DCPDP_FLIP_CONFIG_PENDING,
+	DCPDP_FLIP_CONFIG_ACTIVE,
+} DCPDP_FLIP_CONFIG_STATUS;
+
+
+/* Flip config structure used for queuing of flips */
+struct DCPDP_FLIP_CONFIG_TAG
+{
+	/* Context with which this config is associated */
+	DCPDP_FLIP_CONTEXT	*psContext;
+
+	/* Services config data */
+	IMG_HANDLE		hConfigData;
+
+	/* Current status of the config */
+	DCPDP_FLIP_CONFIG_STATUS eStatus;
+
+	/* Buffer to be displayed when the config is made active */
+	DCPDP_BUFFER		*psBuffer;
+
+	/* Number of frames for which this config should remain active */
+	IMG_UINT32		ui32DisplayPeriod;
+
+	/* New mode to possibly switch to after flip */
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+
+	/* Handle to a work item that processes the flip config queue */
+	IMG_HANDLE		hWorkItem;
+};
+
+struct DCPDP_FLIP_CONTEXT_TAG
+{
+	/* Device on which the context has been created */
+	DCPDP_DEVICE		*psDeviceData;
+
+	/* Lock protecting the queue of flip configs */
+	DC_SPINLOCK		sSpinLock;
+
+	/* Mutex to serialise calls back into Services */
+	void			*pvMutex;
+
+	/* Queue of configs that need processing */
+	DCPDP_FLIP_CONFIG	asFlipConfigQueue[DCPDP_MAX_COMMANDS_INFLIGHT];
+	IMG_UINT32		ui32FlipConfigInsertIndex;
+	IMG_UINT32		ui32FlipConfigRemoveIndex;
+
+	/* Handle to a work queue used to defer the processing of the
+	   flip config queue when a vsync interrupt is serviced */
+	IMG_HANDLE		hFlipConfigWorkQueue;
+
+	/* Services config data to retire */
+	IMG_HANDLE		hConfigDataToRetire;
+};
+
+struct DCPDP_DEVICE_TAG
+{
+	IMG_HANDLE		hPVRServicesDevice;
+	const DC_SERVICES_FUNCS	*psPVRServicesFuncs;
+
+	DCPDP_DEVICE_PRIV	*psDevicePriv;
+
+	PHYS_HEAP		*psPhysHeap;
+
+	IMG_CPU_PHYADDR		sPDPRegCpuPAddr;
+	IMG_CPU_VIRTADDR	pvPDPRegCpuVAddr;
+	IMG_UINT32		ui32PDPRegSize;
+
+	IMG_CPU_PHYADDR		sLMACPUPhysBaseAddr;
+	IMG_CPU_PHYADDR		sDispMemCpuPAddr;
+	IMG_UINT64		uiDispMemSize;
+
+	IMG_UINT32		uiTimingDataIndex;
+	DCPDP_TIMING_DATA      *pasTimingData;
+	IMG_UINT32		uiTimingDataSize;
+	IMG_PIXFMT		ePixelFormat;
+
+	IMG_UINT32		ui32BufferSize;
+	IMG_UINT32		ui32BufferCount;
+	IMG_UINT32		ui32BufferUseMask;
+
+	DCPDP_BUFFER		*psSystemBuffer;
+
+	DCPDP_FLIP_CONTEXT	*psFlipContext;
+
+	IMG_BOOL		bVSyncEnabled;
+	IMG_BOOL		bVSyncReporting;
+	IMG_INT64		i64LastVSyncTimeStamp;
+	IMG_HANDLE		hVSyncWorkQueue;
+	IMG_HANDLE		hVSyncWorkItem;
+};
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+static struct VESA_STANDARD_MODES
+{
+        IMG_UINT32 uiWidth;
+        IMG_UINT32 uiHeight;
+} gasVesaModes[] =
+{
+        { 640, 480},
+        { 800, 600},
+        { 1024, 768},
+        { 1280, 720},
+        { 1280, 768},
+        { 1280, 800},
+        { 1280, 1024},
+};
+#endif
+
+static IMG_BOOL BufferAcquireMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psDeviceData->ui32BufferCount; i++)
+	{
+		if ((psDeviceData->ui32BufferUseMask & (1UL << i)) == 0)
+		{
+			psDeviceData->ui32BufferUseMask |= (1UL << i);
+
+			psBuffer->sCpuPAddr.uiAddr = psDeviceData->sDispMemCpuPAddr.uiAddr + (i * psDeviceData->ui32BufferSize);
+
+			psDeviceData->psPVRServicesFuncs->pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap,
+											1,
+											&psBuffer->sDevPAddr,
+											&psBuffer->sCpuPAddr);
+
+			/* Calculate the correct PDP device address.
+			 * PDP device can only access local memory but not host memory.
+			 * The PDP device address starts at 0 and extends to the size of
+			 * local memory.
+			 */
+
+			/* psBuffer->psDeviceData->sLMACPUPhysBaseAddr.uiAddr always points
+			 * to the start of CPU mapped physical address of the local memory.
+			 *
+			 * Display heap is a carve out from the local memory after LMA local heap.
+			 * Any buffers used for GPU/PDP are offsets from start of LMA heap
+			 */
+			psBuffer->sPDPPAddr.uiAddr = psBuffer->sCpuPAddr.uiAddr - psBuffer->psDeviceData->sLMACPUPhysBaseAddr.uiAddr;
+
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+
+static void BufferReleaseMemory(DCPDP_DEVICE *psDeviceData, DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT64 uiOffset;
+
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr >= psDeviceData->sDispMemCpuPAddr.uiAddr);
+	DC_ASSERT(psBuffer->sCpuPAddr.uiAddr < (psDeviceData->sDispMemCpuPAddr.uiAddr + psDeviceData->uiDispMemSize));
+
+	uiOffset = psBuffer->sCpuPAddr.uiAddr - psDeviceData->sDispMemCpuPAddr.uiAddr;
+	uiOffset = DC_OSDiv64(uiOffset, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferUseMask &= ~(1UL << uiOffset);
+
+	psBuffer->sDevPAddr.uiAddr = 0;
+	psBuffer->sCpuPAddr.uiAddr = 0;
+}
+
+static void ClearSystemBuffer(DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 *puiPixel, uiSizeInPixels, uiGray, i;
+
+	IMG_CPU_VIRTADDR pvBufferCpuVAddr = DC_OSMapPhysAddr(psBuffer->sCpuPAddr,
+	                                                     psBuffer->ui32SizeInBytes);
+	if (pvBufferCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING, "Request for clearing system buffer but no "
+		                 "buffer provided");
+		return;
+	}
+
+	puiPixel = (IMG_UINT32 *) pvBufferCpuVAddr;
+	uiSizeInPixels = psBuffer->ui32Width * psBuffer->ui32Height;
+
+	for (i = 0; i < uiSizeInPixels; i++)
+	{
+		uiGray = (IMG_UINT32) (0x5A * i / uiSizeInPixels);
+		puiPixel[i] = 0xFF000000 + uiGray + (uiGray << 8) + (uiGray << 16);
+	}
+
+	DC_OSUnmapPhysAddr(pvBufferCpuVAddr, psBuffer->ui32SizeInBytes);
+}
+
+static void EnableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB);
+	ui32InterruptEnable |= (0x1 << INTEN_VBLNK1_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB, ui32InterruptEnable);
+}
+
+static void DisableVSyncInterrupt(DCPDP_DEVICE *psDeviceData)
+{
+	IMG_UINT32 ui32InterruptEnable;
+
+	ui32InterruptEnable = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB);
+	ui32InterruptEnable &= ~(INTEN_VBLNK1_MASK);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB, ui32InterruptEnable);
+}
+
+static void SetModeRegisters(DCPDP_DEVICE *psDeviceData);
+
+static INLINE void Flip(DCPDP_BUFFER *psBuffer)
+{
+	IMG_UINT32 ui32Str1AddrCtrl;
+	IMG_UINT32 ui32Stride;
+	IMG_UINT32 ui32Value;
+	IMG_UINT32 ui32Width = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	IMG_UINT32 ui32Height = psBuffer->psDeviceData->pasTimingData[psBuffer->psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	ui32Value = (ui32Width - 1) << STR1WIDTH_SHIFT;
+	ui32Value |= (ui32Height - 1) << STR1HEIGHT_SHIFT;
+
+	switch (psBuffer->psDeviceData->ePixelFormat)
+	{
+		case IMG_PIXFMT_B8G8R8A8_UNORM:
+		{
+			ui32Value |= DCPDP_STR1SURF_FORMAT_ARGB8888 << STR1PIXFMT_SHIFT;
+			break;
+		}
+		default:
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Unrecognised pixel format\n");
+			DC_ASSERT(0);
+		}
+	}
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1SURF, ui32Value);
+
+	ui32Stride = ui32Width * DCPDP_PIXEL_FORMAT_BPP;
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_PDP_STR1POSN, (ui32Stride >> DCPDP_STR1POSN_STRIDE_SHIFT) - 1);
+
+	DC_ASSERT(psBuffer);
+
+	ui32Str1AddrCtrl = DC_OSReadReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL);
+	ui32Str1AddrCtrl &= ~(STR1BASE_MASK);
+	ui32Str1AddrCtrl |= ((psBuffer->sPDPPAddr.uiAddr >> DCPDP_STR1ADDRCTRL_BASE_ADDR_SHIFT) & STR1BASE_MASK);
+
+	DC_OSWriteReg32(psBuffer->psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, ui32Str1AddrCtrl);
+}
+
+static INLINE void FlipConfigQueueAdvanceIndex(DCPDP_FLIP_CONTEXT *psContext, IMG_UINT32 *pui32Index)
+{
+	(*pui32Index)++;
+
+	if (*pui32Index >= DCPDP_MAX_COMMANDS_INFLIGHT)
+	{
+		*pui32Index = 0;
+	}
+}
+
+static INLINE IMG_BOOL FlipConfigQueueEmptyNoLock(DCPDP_FLIP_CONTEXT *psContext)
+{
+	return (IMG_BOOL)(psContext->ui32FlipConfigInsertIndex == psContext->ui32FlipConfigRemoveIndex);
+}
+
+static IMG_BOOL FlipConfigQueueEmpty(DCPDP_FLIP_CONTEXT *psContext)
+{
+	unsigned long ulSpinlockFlags;
+	IMG_BOOL bFlipConfigQueueEmpty;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+	bFlipConfigQueueEmpty = FlipConfigQueueEmptyNoLock(psContext);
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	return bFlipConfigQueueEmpty;
+}
+
+static void FlipConfigQueueAdd(DCPDP_FLIP_CONTEXT *psContext,
+				   IMG_HANDLE hConfigData,
+				   DCPDP_BUFFER *psBuffer,
+				   PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				   IMG_UINT32 ui32DisplayPeriod)
+{
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Get the next inactive flip config and update the insert index */
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigInsertIndex];
+	FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigInsertIndex);
+
+	DC_ASSERT(psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_INACTIVE);
+
+	/* Fill out the flip config */
+	psFlipConfig->psContext		= psContext;
+	psFlipConfig->hConfigData	= hConfigData;
+	psFlipConfig->psBuffer		= psBuffer;
+	psFlipConfig->ui32DisplayPeriod	= ui32DisplayPeriod;
+	psFlipConfig->pasSurfAttrib	= pasSurfAttrib;
+
+	/* Should be updated last */
+	psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_PENDING;
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+}
+
+static IMG_UINT32 GetTimingIndex(DCPDP_DEVICE *psDeviceData, IMG_UINT32 width, IMG_UINT32 height);
+
+static void FlipConfigQueueProcess(DCPDP_FLIP_CONTEXT *psContext, IMG_BOOL bFlipped, IMG_BOOL bFlush)
+{
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_FLIP_CONFIG *psFlipConfig;
+	PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib;
+	IMG_HANDLE ahRetiredConfigs[DCPDP_MAX_COMMANDS_INFLIGHT] = {0};
+	IMG_UINT32 i, ui32RetireConfigIndex = 0;
+	unsigned long ulSpinlockFlags;
+
+	DC_OSMutexLock(psContext->pvMutex);
+	DC_OSSpinLockIRQSave(&psContext->sSpinLock, ulSpinlockFlags);
+
+	psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+	pasSurfAttrib = psFlipConfig->pasSurfAttrib;
+
+	do
+	{
+		switch (psFlipConfig->eStatus)
+		{
+			case DCPDP_FLIP_CONFIG_PENDING:
+				if (!bFlipped)
+				{
+					Flip(psFlipConfig->psBuffer);
+
+					bFlipped = IMG_TRUE;
+				}
+
+				/* If required, set the mode after flipping */
+				if (pasSurfAttrib != NULL)
+				{
+					if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay ||
+					    pasSurfAttrib[0].sDisplay.sDims.ui32Height != psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay)
+					{
+						psDeviceData->uiTimingDataIndex = GetTimingIndex(psDeviceData,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Width,
+												 pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+						SetModeRegisters(psDeviceData);
+					}
+				}
+
+				psFlipConfig->eStatus = DCPDP_FLIP_CONFIG_ACTIVE;
+
+				/* Now that a new flip config is active we can retire the previous one (if one exists) */
+				if (psContext->hConfigDataToRetire != NULL)
+				{
+					ahRetiredConfigs[ui32RetireConfigIndex++] = psContext->hConfigDataToRetire;
+					psContext->hConfigDataToRetire = NULL;
+				}
+
+				__fallthrough;
+			case DCPDP_FLIP_CONFIG_ACTIVE:
+				/* We can retire this config only when it's buffer has been
+				   displayed for at least the same number of frames as the
+				   display period or if we're flushing the queue */
+				if (psFlipConfig->ui32DisplayPeriod != 0)
+				{
+					psFlipConfig->ui32DisplayPeriod--;
+				}
+
+				if (psFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+				{
+					DCPDP_FLIP_CONFIG *psNextFlipConfig;
+
+					/* There should never be an outstanding config to retire at this point */
+					DC_ASSERT(psContext->hConfigDataToRetire == NULL);
+
+					/* Now that this config has been active for the minimum number of frames, i.e. the
+					   display period, we can mark the flip config queue item as inactive. However, we
+					   can only retire the config once a new config has been made active. Therefore,
+					   store the Services config data handle so that we can retire it later. */
+					psContext->hConfigDataToRetire = psFlipConfig->hConfigData;
+
+					/* Reset the flip config data */
+					psFlipConfig->hConfigData	= 0;
+					psFlipConfig->psBuffer		= NULL;
+					psFlipConfig->ui32DisplayPeriod	= 0;
+					psFlipConfig->eStatus		= DCPDP_FLIP_CONFIG_INACTIVE;
+
+					/* Move on the queue remove index */
+					FlipConfigQueueAdvanceIndex(psContext, &psContext->ui32FlipConfigRemoveIndex);
+
+					psNextFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+					if (psNextFlipConfig->ui32DisplayPeriod == 0 || bFlush)
+					{
+						psFlipConfig = psNextFlipConfig;
+						bFlipped = IMG_FALSE;
+					}
+				}
+				break;
+			case DCPDP_FLIP_CONFIG_INACTIVE:
+				if (FlipConfigQueueEmptyNoLock(psContext))
+				{
+					break;
+				}
+				DC_OSDebugPrintf(DBGLVL_ERROR, " - Tried to process an inactive config when one wasn't expected\n");
+
+				__fallthrough;
+			default:
+				DC_ASSERT(0);
+		}
+	} while (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING);
+
+	DC_OSSpinUnlockIRQRestore(&psContext->sSpinLock, ulSpinlockFlags);
+
+	/* Call into Services to retire any configs that were ready to be
+	 * retired. */
+	for (i = 0; i < ui32RetireConfigIndex; i++)
+	{
+		psDeviceData->psPVRServicesFuncs->pfnDCDisplayConfigurationRetired(ahRetiredConfigs[i]);
+	}
+
+	DC_OSMutexUnlock(psContext->pvMutex);
+}
+
+static PVRSRV_ERROR VSyncProcessor(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = pvData;
+
+	psDeviceData->psPVRServicesFuncs->pfnCheckStatus(NULL);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FlipConfigProcessor(void *pvData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)pvData;
+
+	DC_ASSERT(psContext);
+
+	FlipConfigQueueProcess(psContext, IMG_TRUE, IMG_FALSE);
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPGetInfo(IMG_HANDLE hDeviceData, DC_DISPLAY_INFO *psDisplayInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Copy our device name */
+	DC_OSStringNCopy(psDisplayInfo->szDisplayName, DRVNAME " 1", DC_NAME_SIZE);
+
+	/* Report what our minimum and maximum display period is. */
+	psDisplayInfo->ui32MinDisplayPeriod	= DCPDP_MIN_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxDisplayPeriod	= DCPDP_MAX_DISPLAY_PERIOD;
+	psDisplayInfo->ui32MaxPipes		= 1;
+	psDisplayInfo->bUnlatchedSupported	= IMG_FALSE;
+}
+
+static PVRSRV_ERROR DCPDPPanelQueryCount(IMG_HANDLE hDeviceData, IMG_UINT32 *pui32NumPanels)
+{
+	PVR_UNREFERENCED_PARAMETER(hDeviceData);
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPPanelQuery(IMG_HANDLE hDeviceData,
+				    IMG_UINT32 ui32PanelsArraySize,
+				    IMG_UINT32 *pui32NumPanels,
+				    PVRSRV_PANEL_INFO *psPanelInfo)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	/* Don't support hot plug or multiple displays so hard code the value */
+	*pui32NumPanels = 1;
+
+	DC_OSMemSet(psPanelInfo, 0, sizeof(*psPanelInfo));
+
+	psPanelInfo[0].sSurfaceInfo.sFormat.ePixFormat = psDeviceData->ePixelFormat;
+	psPanelInfo[0].sSurfaceInfo.sFormat.eMemLayout = PVRSRV_SURFACE_MEMLAYOUT_STRIDED;
+	psPanelInfo[0].sSurfaceInfo.sFormat.u.sFBCLayout.eFBCompressionMode = IMG_FB_COMPRESSION_NONE;
+
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Width = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psPanelInfo[0].sSurfaceInfo.sDims.ui32Height = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+
+	psPanelInfo[0].ui32RefreshRate = psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh;
+	psPanelInfo[0].ui32XDpi = DCPDP_DPI;
+	psPanelInfo[0].ui32YDpi = DCPDP_DPI;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPFormatQuery(IMG_HANDLE hDeviceData,
+				     IMG_UINT32 ui32NumFormats,
+				     PVRSRV_SURFACE_FORMAT *pasFormat,
+				     IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32NumFormats; i++)
+	{
+		pui32Supported[i] = 0;
+
+		if (pasFormat[i].ePixFormat == psDeviceData->ePixelFormat)
+		{
+			pui32Supported[i]++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPDimQuery(IMG_HANDLE hDeviceData,
+				  IMG_UINT32 ui32NumDims,
+				  PVRSRV_SURFACE_DIMS *psDim,
+				  IMG_UINT32 *pui32Supported)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	IMG_UINT32 i;
+	IMG_UINT32 j;
+
+	for (i = 0; i < ui32NumDims; i++)
+	{
+		pui32Supported[i] = 0;
+
+		for (j = 0; j < psDeviceData->uiTimingDataSize; j++)
+		{
+			if ((psDim[i].ui32Width == psDeviceData->pasTimingData[j].ui32HDisplay) &&
+			    (psDim[i].ui32Height == psDeviceData->pasTimingData[j].ui32VDisplay))
+			{
+				pui32Supported[i]++;
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetBlank(IMG_HANDLE hDeviceData,
+								  IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	IMG_UINT32 ui32Value;
+
+	ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL);
+	if (bEnabled)
+	{
+		ui32Value |= 0x1 << POWERDN_SHIFT;
+		psDeviceData->bVSyncEnabled = IMG_FALSE;
+	}
+	else
+	{
+		ui32Value &= ~(POWERDN_MASK);
+		psDeviceData->bVSyncEnabled = IMG_TRUE;
+	}
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, ui32Value);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPSetVSyncReporting(IMG_HANDLE hDeviceData,
+										   IMG_BOOL bEnabled)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	psDeviceData->bVSyncReporting = bEnabled;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPLastVSyncQuery(IMG_HANDLE hDeviceData,
+										IMG_INT64 *pi64Timestamp)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	*pi64Timestamp = psDeviceData->i64LastVSyncTimeStamp;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPContextCreate(IMG_HANDLE hDeviceData,
+				       IMG_HANDLE *hDisplayContext)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+	DCPDP_FLIP_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	if (psDeviceData->psFlipContext)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Cannot create a context when one is already active\n");
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psContext = DC_OSCallocMem(sizeof(*psContext));
+	if (psContext == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Not enough memory to allocate a display context\n");
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* We don't want to process the flip config work queue in interrupt context so
+	   create a work queue which we can use to defer the work */
+	eError = DC_OSWorkQueueCreate(&psContext->hFlipConfigWorkQueue, DCPDP_MAX_COMMANDS_INFLIGHT);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Couldn't create a work queue (%s)\n",
+				 psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorPDPOSFreeMemContext;
+	}
+
+	/* Create work items to place on the flip config work queue when we service a vsync interrupt */
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		eError = DC_OSWorkQueueCreateWorkItem(&psContext->asFlipConfigQueue[i].hWorkItem, FlipConfigProcessor, psContext);
+		if (eError != PVRSRV_OK)
+		{
+			DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create a flip config work item (%s)\n",
+					 psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+			goto ErrorDC_OSWorkQueueDestroyWorkItems;
+		}
+	}
+
+	eError = DC_OSMutexCreate(&psContext->pvMutex);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Failed to create flip config mutex (%s)\n",
+				 psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDC_OSWorkQueueDestroyWorkItems;
+	}
+
+	DC_OSSpinLockCreate(&psContext->sSpinLock);
+
+	psContext->psDeviceData		= psDeviceData;
+	psDeviceData->psFlipContext	= psContext;
+	*hDisplayContext		= (IMG_HANDLE)psContext;
+
+	EnableVSyncInterrupt(psDeviceData);
+
+	return PVRSRV_OK;
+
+ErrorDC_OSWorkQueueDestroyWorkItems:
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+
+ErrorPDPOSFreeMemContext:
+	DC_OSFreeMem(psContext);
+
+	return eError;
+}
+
+static void DCPDPContextDestroy(IMG_HANDLE hDisplayContext)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_UINT32 i;
+
+	DisableVSyncInterrupt(psDeviceData);
+
+	/* Make sure there are no work items on the queue before it's destroyed */
+	DC_OSWorkQueueFlush(psContext->hFlipConfigWorkQueue);
+
+	for (i = 0; i < DCPDP_MAX_COMMANDS_INFLIGHT; i++)
+	{
+		if (psContext->asFlipConfigQueue[i].hWorkItem != (IMG_HANDLE)NULL)
+		{
+			DC_OSWorkQueueDestroyWorkItem(psContext->asFlipConfigQueue[i].hWorkItem);
+		}
+	}
+
+	DC_OSWorkQueueDestroy(psContext->hFlipConfigWorkQueue);
+	DC_OSMutexDestroy(psContext->pvMutex);
+	DC_OSSpinLockDestroy(&psContext->sSpinLock);
+
+	psDeviceData->psFlipContext = NULL;
+	DC_OSFreeMem(psContext);
+}
+
+static IMG_UINT32 GetTimingIndex(DCPDP_DEVICE *psDeviceData, IMG_UINT32 width, IMG_UINT32 height)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 index = -1;
+	for (i = 0; i < psDeviceData->uiTimingDataSize; i++)
+	{
+		if (psDeviceData->pasTimingData[i].ui32HDisplay == width &&
+		   psDeviceData->pasTimingData[i].ui32VDisplay == height)
+		{
+			index = i;
+		}
+	}
+
+	return index;
+}
+
+static void DCPDPContextConfigure(IMG_HANDLE hDisplayContext,
+				  IMG_UINT32 ui32PipeCount,
+				  PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+				  IMG_HANDLE *ahBuffers,
+				  IMG_UINT32 ui32DisplayPeriod,
+				  IMG_HANDLE hConfigData)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	IMG_BOOL bQueueWasEmpty;
+
+	if (ui32DisplayPeriod > DCPDP_MAX_DISPLAY_PERIOD)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - Requested display period of %u is larger than maximum supported display period of %u (clamping)\n",
+				 ui32DisplayPeriod, DCPDP_MAX_DISPLAY_PERIOD);
+		DC_ASSERT(ui32DisplayPeriod <= DCPDP_MAX_DISPLAY_PERIOD);
+
+		/* Clamp to the maximum so that we can continue */
+		ui32DisplayPeriod = DCPDP_MAX_DISPLAY_PERIOD;
+	}
+
+	/* Add new config to the queue */
+	if (ui32PipeCount != 0)
+	{
+		/* Check if the queue is empty before adding a new flip config */
+		bQueueWasEmpty = FlipConfigQueueEmpty(psContext);
+
+		/* We have a regular config so add it as normal */
+		FlipConfigQueueAdd(psContext, hConfigData, (DCPDP_BUFFER *)ahBuffers[0], pasSurfAttrib, ui32DisplayPeriod);
+
+		/* Check to see if vsync unlocked flipping should be done. This is determined by the
+		   display period being 0 and the flip config queue being empty before we added the
+		   new flip config. If it wasn't empty then we need to allow the remaining vsync locked
+		   flip configs to be processed before we can start processing the queue here.
+		   Also immediately flip if the current vsync interrupt is disabled.
+		   This makes sure that even when the PDP is turned off we progress. */
+		if ((!ui32DisplayPeriod && bQueueWasEmpty) || !psDeviceData->bVSyncEnabled)
+		{
+			FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_FALSE);
+		}
+	}
+	else
+	{
+		/* We have a 'NULL' config, which indicates the display context is
+		   about to be destroyed. Queue a flip back to the system buffer */
+		FlipConfigQueueAdd(psContext, hConfigData, psDeviceData->psSystemBuffer, pasSurfAttrib, 0);
+
+		/* Flush the remaining configs (this will result in the system buffer being displayed) */
+		FlipConfigQueueProcess(psContext, IMG_FALSE, IMG_TRUE);
+
+		DC_OSMutexLock(psContext->pvMutex);
+
+		/* Immediately retire the system buffer config data */
+		psDeviceData->psPVRServicesFuncs->pfnDCDisplayConfigurationRetired(hConfigData);
+		psContext->hConfigDataToRetire = NULL;
+
+		DC_OSMutexUnlock(psContext->pvMutex);
+	}
+}
+
+static PVRSRV_ERROR DCPDPContextConfigureCheck(IMG_HANDLE hDisplayContext,
+					       IMG_UINT32 ui32PipeCount,
+					       PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+					       IMG_HANDLE *ahBuffers)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	DCPDP_BUFFER *psBuffer;
+	IMG_UINT32 ui32NewVESATimingIndex;
+	ui32NewVESATimingIndex = GetTimingIndex(psDeviceData,pasSurfAttrib[0].sDisplay.sDims.ui32Width, pasSurfAttrib[0].sDisplay.sDims.ui32Height);
+
+	if (ui32NewVESATimingIndex == -1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+
+	if (ui32PipeCount != 1)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CONFIG;
+	}
+	/* new crop dimensions, new mode dimensions */
+	else if (pasSurfAttrib[0].sCrop.sDims.ui32Width != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32HDisplay ||
+		 pasSurfAttrib[0].sCrop.sDims.ui32Height != psDeviceData->pasTimingData[ui32NewVESATimingIndex].ui32VDisplay ||
+		 pasSurfAttrib[0].sCrop.i32XOffset != 0 ||
+		 pasSurfAttrib[0].sCrop.i32YOffset != 0)
+	{
+		return PVRSRV_ERROR_DC_INVALID_CROP_RECT;
+	}
+	/* new mode dimensions, new crop dimensions */
+	else if (pasSurfAttrib[0].sDisplay.sDims.ui32Width != pasSurfAttrib[0].sCrop.sDims.ui32Width ||
+		 pasSurfAttrib[0].sDisplay.sDims.ui32Height != pasSurfAttrib[0].sCrop.sDims.ui32Height ||
+		 pasSurfAttrib[0].sDisplay.i32XOffset != pasSurfAttrib[0].sCrop.i32XOffset ||
+		 pasSurfAttrib[0].sDisplay.i32YOffset != pasSurfAttrib[0].sCrop.i32YOffset)
+	{
+		return PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT;
+	}
+
+	psBuffer = ahBuffers[0];
+
+	/* new buffer dimensions, current mode dimensions */
+	if (psBuffer->ui32Width  < pasSurfAttrib[0].sDisplay.sDims.ui32Width ||
+	    psBuffer->ui32Height < pasSurfAttrib[0].sDisplay.sDims.ui32Height)
+	{
+		return PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferSystemAcquire(IMG_HANDLE hDeviceData,
+					     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+					     IMG_UINT32 *pui32PageCount,
+					     IMG_UINT32 *pui32ByteStride,
+					     IMG_HANDLE *phSystemBuffer)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)hDeviceData;
+
+	DC_OSAtomicIncrement(&psDeviceData->psSystemBuffer->i32RefCount);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psDeviceData->psSystemBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psDeviceData->psSystemBuffer->ui32ByteStride;
+	*phSystemBuffer		= psDeviceData->psSystemBuffer;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferSystemRelease(IMG_HANDLE hSystemBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hSystemBuffer;
+
+	DC_OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static PVRSRV_ERROR DCPDPBufferAlloc(IMG_HANDLE hDisplayContext,
+				     DC_BUFFER_CREATE_INFO *psCreateInfo,
+				     IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+				     IMG_UINT32 *pui32PageCount,
+				     IMG_UINT32 *pui32ByteStride,
+				     IMG_HANDLE *phBuffer)
+{
+	DCPDP_FLIP_CONTEXT *psContext = (DCPDP_FLIP_CONTEXT *)hDisplayContext;
+	DCPDP_DEVICE *psDeviceData = psContext->psDeviceData;
+	PVRSRV_SURFACE_INFO *psSurfInfo = &psCreateInfo->sSurface;
+	DCPDP_BUFFER *psBuffer;
+
+	DC_ASSERT(psCreateInfo->ui32BPP == DCPDP_PIXEL_FORMAT_BPP);
+
+	if (psSurfInfo->sFormat.eMemLayout != PVRSRV_SURFACE_MEMLAYOUT_STRIDED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psSurfInfo->sFormat.ePixFormat != psDeviceData->ePixelFormat)
+	{
+		return PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT;
+	}
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psSurfInfo->sFormat.ePixFormat;
+	psBuffer->ui32Width		= psSurfInfo->sDims.ui32Width;
+	psBuffer->ui32Height		= psSurfInfo->sDims.ui32Height;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * psCreateInfo->ui32BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32)DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height, DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	*puiLog2PageSize	= DC_OSGetPageShift();
+	*pui32PageCount		= psBuffer->ui32SizeInPages;
+	*pui32ByteStride	= psBuffer->ui32ByteStride;
+	*phBuffer		= psBuffer;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DCPDPBufferAcquire(IMG_HANDLE hBuffer, IMG_DEV_PHYADDR *pasDevPAddr, void **ppvLinAddr)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+	DCPDP_DEVICE *psDeviceData = psBuffer->psDeviceData;
+	IMG_CPU_PHYADDR sCpuPAddr = psBuffer->sCpuPAddr;
+	IMG_UINT32 i;
+
+	for (i = 0; i < psBuffer->ui32SizeInPages; i++)
+	{
+		psDeviceData->psPVRServicesFuncs->pfnPhysHeapCpuPAddrToDevPAddr(psDeviceData->psPhysHeap,
+										1,
+										&pasDevPAddr[i], &sCpuPAddr);
+		sCpuPAddr.uiAddr += DC_OSGetPageSize();
+	}
+
+	DC_OSAtomicIncrement(&psBuffer->i32RefCount);
+
+	*ppvLinAddr = NULL;
+
+	return PVRSRV_OK;
+}
+
+static void DCPDPBufferRelease(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+	DC_OSAtomicDecrement(&psBuffer->i32RefCount);
+}
+
+static void DCPDPBufferFree(IMG_HANDLE hBuffer)
+{
+	DCPDP_BUFFER *psBuffer = (DCPDP_BUFFER *)hBuffer;
+
+#if defined(DEBUG)
+	if (DC_OSAtomicRead(&psBuffer->i32RefCount) != 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_WARNING, " - Freeing buffer that still has %d references\n", DC_OSAtomicRead(&psBuffer->i32RefCount));
+	}
+#endif
+
+	BufferReleaseMemory(psBuffer->psDeviceData, psBuffer);
+
+	DC_OSFreeMem(psBuffer);
+}
+
+static PVRSRV_ERROR DCPDPResetDevice(IMG_HANDLE hDeviceData)
+{
+	DCPDP_DEVICE *psDeviceData = hDeviceData;
+
+	ClearSystemBuffer(psDeviceData->psSystemBuffer);
+
+	SetModeRegisters(psDeviceData);
+	Flip(psDeviceData->psSystemBuffer);
+
+	return PVRSRV_OK;
+}
+
+static DC_DEVICE_FUNCTIONS g_sDCFunctions =
+{
+	DCPDPGetInfo,				/* pfnGetInfo */
+	DCPDPPanelQueryCount,		/* pfnPanelQueryCount */
+	DCPDPPanelQuery,			/* pfnPanelQuery */
+	DCPDPFormatQuery,			/* pfnFormatQuery */
+	DCPDPDimQuery,				/* pfnDimQuery */
+	DCPDPSetBlank,				/* pfnSetBlank */
+	DCPDPSetVSyncReporting,		/* pfnSetVSyncReporting */
+	DCPDPLastVSyncQuery,		/* pfnLastVSyncQuery */
+	DCPDPContextCreate,			/* pfnContextCreate */
+	DCPDPContextDestroy,		/* pfnContextDestroy */
+	DCPDPContextConfigure,		/* pfnContextConfigure */
+	DCPDPContextConfigureCheck,	/* pfnContextConfigureCheck */
+	DCPDPBufferAlloc,			/* pfnBufferAlloc */
+	DCPDPBufferAcquire,			/* pfnBufferAcquire */
+	DCPDPBufferRelease,			/* pfnBufferRelease */
+	DCPDPBufferFree,			/* pfnBufferFree */
+	NULL,					/* pfnBufferImport */
+	NULL,					/* pfnBufferMap */
+	NULL,					/* pfnBufferUnmap */
+	DCPDPBufferSystemAcquire,	/* pfnBufferSystemAcquire */
+	DCPDPBufferSystemRelease,	/* pfnBufferSystemRelease */
+
+	DCPDPResetDevice,			/* pfnResetDevice */
+};
+
+static IMG_BOOL DisplayLISRHandler(void *pvData)
+{
+	DCPDP_DEVICE *psDeviceData = (DCPDP_DEVICE *)pvData;
+	IMG_UINT32 ui32InterruptStatus;
+	PVRSRV_ERROR eError;
+
+	ui32InterruptStatus = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTSTAT);
+	if (ui32InterruptStatus & INTS_VBLNK1_MASK)
+	{
+		/* Get the timestamp for this interrupt. */
+		psDeviceData->i64LastVSyncTimeStamp = DC_OSClockns();
+		if (psDeviceData->bVSyncReporting)
+		{
+			eError = DC_OSWorkQueueAddWorkItem(psDeviceData->hVSyncWorkQueue, psDeviceData->hVSyncWorkItem);
+			if (eError != PVRSRV_OK)
+			{
+				DC_OSDebugPrintf(DBGLVL_WARNING,
+						 " - %s: Couldn't queue work item (%s)\n",
+						 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+
+			}
+		}
+
+		if (psDeviceData->psFlipContext)
+		{
+			DCPDP_FLIP_CONTEXT *psContext = psDeviceData->psFlipContext;
+			DCPDP_FLIP_CONFIG *psFlipConfig;
+
+			DC_OSSpinLock(&psContext->sSpinLock);
+
+			psFlipConfig = &psContext->asFlipConfigQueue[psContext->ui32FlipConfigRemoveIndex];
+
+			if (psFlipConfig->eStatus == DCPDP_FLIP_CONFIG_PENDING)
+			{
+				Flip(psFlipConfig->psBuffer);
+			}
+
+			if (psFlipConfig->eStatus != DCPDP_FLIP_CONFIG_INACTIVE)
+			{
+				eError = DC_OSWorkQueueAddWorkItem(psContext->hFlipConfigWorkQueue, psFlipConfig->hWorkItem);
+				if (eError != PVRSRV_OK)
+				{
+					DC_OSDebugPrintf(DBGLVL_WARNING,
+							 " - %s: Couldn't queue work item (%s)\n",
+							 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+				}
+			}
+
+			DC_OSSpinUnlock(&psContext->sSpinLock);
+		}
+		/* Clear the vsync status bit to show that the vsync interrupt has been handled */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_INTCLEAR, (0x1UL << INTCLR_VBLNK1_SHIFT));
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+static PVRSRV_ERROR SetPDPClocks(DCPDP_DEVICE *psDeviceData, IMG_UINT32 uiClkInHz)
+{
+	IMG_CPU_PHYADDR sPLLRegCpuPAddr;
+	IMG_CPU_VIRTADDR pvPLLRegCpuVAddr;
+	IMG_UINT32 ui32PLLRegSize;
+	IMG_UINT32 uiClkInMHz;
+
+	/* Reserve the PLL register region and map the registers in */
+	sPLLRegCpuPAddr = DCPDPGetAddrRangeStart(psDeviceData->psDevicePriv,
+						 DCPDP_ADDRESS_RANGE_PLL);
+	ui32PLLRegSize = DCPDPGetAddrRangeSize(psDeviceData->psDevicePriv,
+					       DCPDP_ADDRESS_RANGE_PLL);
+
+	if (DC_OSRequestAddrRegion(sPLLRegCpuPAddr, ui32PLLRegSize, DRVNAME) == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to reserve PLL registers\n", __func__);
+		return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+	}
+
+	pvPLLRegCpuVAddr = DC_OSMapPhysAddr(sPLLRegCpuPAddr, ui32PLLRegSize);
+	if (pvPLLRegCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to map PLL registers\n", __func__);
+
+		DC_OSReleaseAddrRegion(sPLLRegCpuPAddr, ui32PLLRegSize);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Calculate the clock in MHz */
+	uiClkInMHz = (uiClkInHz + 500000) / 1000000;
+
+	/* Set phase 0, ratio 50:50 and frequency in MHz */
+	DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_CLK0, uiClkInMHz);
+
+	/* setup TCF_CR_PLL_PDP_CLK1TO5 based on the main clock speed */
+	if (uiClkInMHz >= 50)
+	{
+		/* Set clocks 1-5 to be the same frequency as clock 0 */
+		DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_CLK1TO5, 0x0);
+	}
+	else
+	{
+		/* Set clocks 1-5 to be double the frequency of clock 0 */
+		DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_CLK1TO5, 0x3);
+	}
+
+	/* Now initiate reprogramming of the PLLs */
+	DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_DRP_GO, 1);
+	DC_OSDelayus(1000);
+	DC_OSWriteReg32(pvPLLRegCpuVAddr, TCF_PLL_PLL_PDP_DRP_GO, 0);
+
+	/* Unmap and release the PLL address region */
+	DC_OSUnmapPhysAddr(pvPLLRegCpuVAddr, ui32PLLRegSize);
+	DC_OSReleaseAddrRegion(sPLLRegCpuPAddr, ui32PLLRegSize);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+
+static void SetModeRegisters(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_TIMING_DATA *psTimingData = &psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex];
+	IMG_UINT32 ui32Value;
+
+	DC_ASSERT(psDeviceData->psSystemBuffer);
+
+	/* Turn off memory requests */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, 0x00000000);
+
+	/* Disable sync gen */
+	ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL);
+	ui32Value &= ~(SYNCACTIVE_MASK);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, ui32Value);
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+	SetPDPClocks(psDeviceData, psTimingData->ui32ClockFreq);
+#endif
+
+	if (DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL) != 0x0000C010)
+	{
+		/* Buffer request threshold */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL, 0x00001C10);
+	}
+
+	/* Border colour */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_BORDCOL, 0x00005544);
+
+	/* Update control */
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_UPDCTRL, 0x00000000);
+
+	/* Set hsync timings */
+	ui32Value = (psTimingData->ui32HBackPorch << HBPS_SHIFT) | (psTimingData->ui32HTotal << HT_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC1, ui32Value);
+
+	ui32Value = (psTimingData->ui32HActiveStart << HAS_SHIFT) | (psTimingData->ui32HLeftBorder << HLBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC2, ui32Value);
+
+	ui32Value = (psTimingData->ui32HFrontPorch << HFPS_SHIFT) | (psTimingData->ui32HRightBorder << HRBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC3, psTimingData->ui32HFrontPorch << 16 | ui32Value);
+
+	/* Set vsync timings */
+	ui32Value = (psTimingData->ui32VBackPorch << VBPS_SHIFT) | (psTimingData->ui32VTotal << VT_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC1, ui32Value);
+
+	ui32Value = (psTimingData->ui32VActiveStart << VAS_SHIFT) | (psTimingData->ui32VTopBorder << VTBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC2, ui32Value);
+
+	ui32Value = (psTimingData->ui32VFrontPorch << VFPS_SHIFT) | (psTimingData->ui32VBottomBorder << VBBS_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC3, ui32Value);
+
+	/* Horizontal data enable */
+	ui32Value = (psTimingData->ui32HActiveStart << HDES_SHIFT) | (psTimingData->ui32HFrontPorch << HDEF_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_HDECTRL, ui32Value);
+
+	/* Vertical data enable */
+	ui32Value = (psTimingData->ui32VActiveStart << VDES_SHIFT) | (psTimingData->ui32VFrontPorch << VDEF_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VDECTRL, ui32Value);
+
+	/* Vertical event start and vertical fetch start */
+	ui32Value = (psTimingData->ui32VBackPorch << VFETCH_SHIFT) | (psTimingData->ui32VFrontPorch << VEVENT_SHIFT);
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_VEVENT, ui32Value);
+
+	/* Enable sync gen last and set up polarities of sync/blank */
+	ui32Value = 0x1 << SYNCACTIVE_SHIFT |
+		0x1 << FIELDPOL_SHIFT |
+		0x1 << BLNKPOL_SHIFT |
+		0x1 << VSPOL_SHIFT |
+		0x1 << HSPOL_SHIFT;
+	DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, ui32Value);
+
+	/* Turn on memory requests */
+	DCPDPEnableMemoryRequest(psDeviceData, DCPDPGetModuleParameters()->ui32PDPEnabled);
+}
+
+static PVRSRV_ERROR InitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+	DCPDP_BUFFER *psBuffer;
+
+	DC_ASSERT(psDeviceData->psSystemBuffer == NULL);
+
+	psBuffer = DC_OSAllocMem(sizeof(*psBuffer));
+	if (psBuffer == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	DC_OSAtomicWrite(&psBuffer->i32RefCount, 0);
+	psBuffer->psDeviceData		= psDeviceData;
+	psBuffer->ePixelFormat		= psDeviceData->ePixelFormat;
+	psBuffer->ui32Width			= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay;
+	psBuffer->ui32Height		= psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay;
+	psBuffer->ui32ByteStride	= psBuffer->ui32Width * DCPDP_PIXEL_FORMAT_BPP;
+	psBuffer->ui32SizeInBytes	= psDeviceData->ui32BufferSize;
+	psBuffer->ui32SizeInPages	= psBuffer->ui32SizeInBytes >> DC_OSGetPageShift();
+
+	if (BufferAcquireMemory(psDeviceData, psBuffer) == IMG_FALSE)
+	{
+		DC_OSFreeMem(psBuffer);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Make sure we get a buffer with the size that we're expecting (this should be page aligned) */
+	DC_ASSERT((IMG_UINT32) DC_ALIGN(psBuffer->ui32ByteStride * psBuffer->ui32Height,
+	          DC_OSGetPageSize()) <= psDeviceData->ui32BufferSize);
+
+	/* Initialise the system buffer to a nice gradient. */
+	ClearSystemBuffer(psBuffer);
+
+	psDeviceData->psSystemBuffer = psBuffer;
+
+	return PVRSRV_OK;
+}
+
+static void DeInitSystemBuffer(DCPDP_DEVICE *psDeviceData)
+{
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+	{
+		if (DC_OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) != 0)
+		{
+			printk(KERN_INFO "%s: References to psSystemBuffer still exist"
+				   " (%d)", __func__, DC_OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount));
+		}
+		DC_OSAtomicWrite(&psDeviceData->psSystemBuffer->i32RefCount, 0);
+	}
+#else
+	{
+		DC_ASSERT(DC_OSAtomicRead(&psDeviceData->psSystemBuffer->i32RefCount) == 0);
+	}
+#endif
+
+	BufferReleaseMemory(psDeviceData, psDeviceData->psSystemBuffer);
+
+	DC_OSFreeMem(psDeviceData->psSystemBuffer);
+	psDeviceData->psSystemBuffer = NULL;
+}
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+static void GTFCalcFromRefresh(IMG_UINT32 ui32Width, IMG_UINT32 ui32Height,
+			       IMG_UINT32 ui32VRefresh, DCPDP_TIMING_DATA *psTimingData)
+{
+	double fVtFreq, fHzFreq;
+	double fFieldPeriod, fHzPeriod, fVtFreqEst, fDutyCycle, fClockFreq;
+	double fGTF_Mdash, fGTF_Cdash;
+	unsigned long ulVTotal, ulHTotal;
+	unsigned long ulVSyncBP, ulVBP;
+	unsigned long ulHSyncWidth, ulHFrontPorch, ulHBlank;
+	unsigned long ulVSyncMin;
+
+	DC_OSFloatingPointBegin();
+
+	if ((ui32Width * 3UL) == (ui32Height * 4UL))
+	{
+		ulVSyncMin = 4;
+	}
+	else if ((ui32Width * 9UL) == (ui32Height * 16UL))
+	{
+		ulVSyncMin = 5;
+	}
+	else if ((ui32Width * 10UL) == (ui32Height * 16UL))
+	{
+		ulVSyncMin = 6;
+	}
+	else if ((ui32Width * 4UL) == (ui32Height * 5UL))
+	{
+		ulVSyncMin = 7;
+	}
+	else if ((ui32Width * 9UL) == (ui32Height * 15UL))
+	{
+		ulVSyncMin = 7;
+	}
+	else
+	{
+		ulVSyncMin = GTF_VSYNC_WIDTH;
+	}
+
+	fGTF_Mdash = (fGTF_K * fGTF_M) / 256.0;
+	fGTF_Cdash = (((fGTF_C - fGTF_J) * fGTF_K) / 256) + fGTF_J;
+
+	/* Comes in as Hz */
+	fVtFreq = (double)ui32VRefresh;
+
+	/* Refresh rates lower than 0 and higher than 250Hz are nonsense */
+	if (fVtFreq > 250.0)
+	{
+		fVtFreq = 250.0;
+	}
+	else if (fVtFreq <= 0)
+	{
+		fVtFreq = 60.0;
+	}
+
+	/* Calculate Field Vertical Period in uS */
+	fFieldPeriod = 1000000.0f / fVtFreq;
+
+	/* Estimate Hz Period (in uS) & hence lines in (VSync + BP) & lines in Vt BP */
+	fHzPeriod = (fFieldPeriod - fGTF_MIN_VSYNCBP) / (ui32Height + fGTF_MIN_PORCH);
+
+	ulVSyncBP = (unsigned long)(fGTF_MIN_VSYNCBP / fHzPeriod);
+
+	ulVBP = ulVSyncBP - ulVSyncMin;
+	ulVTotal = ui32Height + ulVSyncBP + GTF_MIN_PORCH;
+
+	/* Estimate Vt frequency (in Hz) */
+	fVtFreqEst = 1000000.0f / (fHzPeriod * ulVTotal);
+
+	/* Find Actual Hz Period (in uS) */
+	fHzPeriod = (fHzPeriod * fVtFreqEst) / fVtFreq;
+
+	/* Find actual Vt frequency (in Hz) */
+	fVtFreq = 1000000.0f / (fHzPeriod * ulVTotal);
+
+	/* Find ideal blanking duty cycle from duty cycle equation and therefore
+	   number of pixels in blanking period (to nearest dbl char cell) */
+	fDutyCycle = fGTF_Cdash - ((fGTF_Mdash * fHzPeriod) / 1000.0f);
+
+	ulHBlank = (unsigned long)(((ui32Width * fDutyCycle) / (100.0f - fDutyCycle) / fGTF_CELL_GRAN2) + 0.5);
+	ulHBlank = ulHBlank * GTF_CELL_GRAN2;
+
+	/* Find HTotal & pixel clock(MHz) & Hz Frequency (KHz) */
+	ulHTotal = ui32Width + ulHBlank;
+	fClockFreq = (double)ulHTotal / fHzPeriod;
+	fHzFreq = 1000.0f / fHzPeriod;
+
+	ulHSyncWidth = (unsigned long)(((ulHTotal * fGTF_HSYNC_MARGIN) / (100.0 * fGTF_CELL_GRAN)) + 0.5);
+	ulHSyncWidth *= GTF_CELL_GRAN;
+
+	ulHFrontPorch = (ulHBlank >> 1) - ulHSyncWidth;
+	if (ulHFrontPorch < GTF_MIN_PORCH_PIXELS)
+	{
+		ulHFrontPorch = GTF_MIN_PORCH_PIXELS;
+	}
+
+	psTimingData->ui32HDisplay	= ui32Width;
+	psTimingData->ui32HBackPorch	= ulHSyncWidth;
+	psTimingData->ui32HTotal	= ulHTotal;
+	psTimingData->ui32HActiveStart	= ulHBlank - ulHFrontPorch;
+	psTimingData->ui32HLeftBorder	= psTimingData->ui32HActiveStart;
+	psTimingData->ui32HRightBorder	= psTimingData->ui32HLeftBorder + ui32Width;
+	psTimingData->ui32HFrontPorch	= psTimingData->ui32HRightBorder;
+
+	psTimingData->ui32VDisplay	= ui32Height;
+	psTimingData->ui32VBackPorch	= GTF_VSYNC_WIDTH;
+	psTimingData->ui32VTotal	= ulVTotal;
+	psTimingData->ui32VActiveStart	= ulVSyncBP;
+	psTimingData->ui32VTopBorder	= psTimingData->ui32VActiveStart;
+	psTimingData->ui32VBottomBorder = psTimingData->ui32VTopBorder + ui32Height;
+	psTimingData->ui32VFrontPorch	= psTimingData->ui32VBottomBorder;
+
+	psTimingData->ui32VRefresh	= (IMG_UINT32)fVtFreq;
+
+	/* scale, round and then scale */
+	psTimingData->ui32ClockFreq = ((unsigned long)(fClockFreq * 1000.0)) * 1000UL;
+
+	DC_OSFloatingPointEnd();
+}
+#endif /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+
+PVRSRV_ERROR DCPDPInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+		       const DC_SERVICES_FUNCS *psPVRServicesFuncs,
+		       DCPDP_DEVICE **ppsDeviceData)
+{
+	DCPDP_DEVICE *psDeviceData;
+	IMG_UINT64 ui64BufferCount;
+	PVRSRV_ERROR eError;
+	PHYS_HEAP *psLMAPhysHeap;
+	PPVRSRV_DEVICE_NODE psDevNode;
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+	IMG_UINT32 i;
+#endif
+
+	if (psPVRServicesFuncs == NULL || ppsDeviceData == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Invalid arguments %p %p\n", __func__, psPVRServicesFuncs, ppsDeviceData);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DC_OSSetDrvName(DRVNAME);
+
+	/* Request 0th device node as this module only supports systems with 1 GPU device  */
+	psDevNode = psPVRServicesFuncs->pfnGetDeviceInstance(0);
+	PVR_RETURN_IF_INVALID_PARAM(psDevNode != NULL);
+
+	psDeviceData = DC_OSCallocMem(sizeof(*psDeviceData));
+	if (psDeviceData == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to allocate device data\n", __func__);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psDeviceData->psDevicePriv = psDevicePriv;
+	psDeviceData->psPVRServicesFuncs = psPVRServicesFuncs;
+	psDeviceData->ePixelFormat = DCPDP_PIXEL_FORMAT;
+
+	/* Services manages the card memory so we need to acquire the Local Memory heap and display controller
+	   heap (which is a carve out of the card memory) so we can allocate our own buffers.
+	 */
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapAcquireByUsage((1 << psDevNode->psDevConfig->eDefaultHeap),
+																		 psDevNode,
+																		 &psLMAPhysHeap);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to acquire Local memory heap (%s)\n",
+		                 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeDeviceData;
+	}
+
+	/* Display controller uses local memory offsets instead of CPU Physical mapped Address.
+	 * The local memory heap CPU physical Address is required to calculate the address offset
+	 * used by the display controller.
+	 * Get the CPU base address for the Local memory heap
+	 */
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetCpuPAddr(psLMAPhysHeap,
+										&psDeviceData->sLMACPUPhysBaseAddr);
+	/* Release the acquired heap once the base address is known */
+	psDeviceData->psPVRServicesFuncs->pfnPhysHeapRelease(psLMAPhysHeap);
+
+	if (eError != PVRSRV_OK)
+	{
+
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get Local memory base address (%s)\n",
+		                 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeDeviceData;
+
+	}
+
+	DC_OSDebugPrintf(DBGLVL_DEBUG, " LMA Phys Heap Base: 0x%pX\n",
+	                 psDeviceData->sLMACPUPhysBaseAddr.uiAddr);
+
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapAcquireByUsage(PHYS_HEAP_USAGE_DISPLAY,
+																		 psDevNode,
+																		 &psDeviceData->psPhysHeap);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to acquire heap (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeDeviceData;
+	}
+
+	/* The PDP can only access local memory so make sure the heap is of the correct type */
+	if (psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetType(psDeviceData->psPhysHeap) != PHYS_HEAP_TYPE_LMA)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Heap is of the wrong type\n", __func__);
+		eError = PVRSRV_ERROR_INVALID_HEAP;
+		goto ErrorPhysHeapRelease;
+	}
+
+	/* Get the CPU base address for the display heap */
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetCpuPAddr(psDeviceData->psPhysHeap,
+										&psDeviceData->sDispMemCpuPAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get display memory base address (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	eError = psDeviceData->psPVRServicesFuncs->pfnPhysHeapGetSize(psDeviceData->psPhysHeap,
+									    &psDeviceData->uiDispMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to get the display memory size (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorPhysHeapRelease;
+	}
+
+	/* Reserve and map the PDP registers */
+	psDeviceData->sPDPRegCpuPAddr = DCPDPGetAddrRangeStart(psDeviceData->psDevicePriv,
+							       DCPDP_ADDRESS_RANGE_PDP);
+	psDeviceData->ui32PDPRegSize = DCPDPGetAddrRangeSize(psDeviceData->psDevicePriv,
+							     DCPDP_ADDRESS_RANGE_PDP);
+
+	if (DC_OSRequestAddrRegion(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize, DRVNAME) == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to reserve the PDP registers\n", __func__);
+
+		psDeviceData->sPDPRegCpuPAddr.uiAddr = 0;
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+
+		goto ErrorPhysHeapRelease;
+	}
+
+	psDeviceData->pvPDPRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize);
+	if (psDeviceData->pvPDPRegCpuVAddr == NULL)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to map PDP registers\n", __func__);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+		goto ErrorReleaseMemRegion;
+	}
+
+#if defined(DCPDP_DYNAMIC_GTF_TIMING)
+	psDeviceData->uiTimingDataSize = ARRAY_SIZE(gasVesaModes);
+	psDeviceData->pasTimingData = DC_OSAllocMem(sizeof(DCPDP_TIMING_DATA) * psDeviceData->uiTimingDataSize);
+
+	if (psDeviceData->pasTimingData	== NULL)
+	{
+		goto ErrorPDPOSUnmapPDPRegisters;
+	}
+
+	/* fill array with timing data for VESA standard modes */
+	for (i = 0; i < psDeviceData->uiTimingDataSize; i++)
+	{
+		if (gasVesaModes[i].uiWidth == DCPDPGetModuleParameters()->ui32PDPWidth &&
+		    gasVesaModes[i].uiHeight == DCPDPGetModuleParameters()->ui32PDPHeight)
+		{
+			psDeviceData->uiTimingDataIndex = i;
+		}
+
+		/* Setup timing data */
+		GTFCalcFromRefresh(gasVesaModes[i].uiWidth,
+				   gasVesaModes[i].uiHeight,
+				   60,
+				   &psDeviceData->pasTimingData[i]);
+	}
+
+#else /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+	/* The default PLL setup is for 1280 x 1024 at 60Hz. The default PLL clock
+	   is in flash memory and will be loaded from flash on reset */
+
+	psDeviceData->uiTimingDataSize = 1;
+	psDeviceData->pasTimingData = DC_OSAllocMem(sizeof(DCPDP_TIMING_DATA) * psDeviceData->uiTimingDataSize);
+
+	if (psDeviceData->pasTimingData	== NULL)
+	{
+		goto ErrorPDPOSUnmapPDPRegisters;
+	}
+
+	psDeviceData->uiTimingDataIndex = 0;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay	= 1280;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HTotal		= 1712;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HBackPorch	= 136;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HActiveStart	= 352;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HLeftBorder	= 352;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HFrontPorch	= 1632;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HRightBorder	= 1632;
+
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay	= 1024;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VTotal		= 1059;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VBackPorch	= 7;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VActiveStart	= 34;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VTopBorder	= 34;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VFrontPorch	= 1058;
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VBottomBorder	= 1058;
+
+	psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh	= 60;
+#endif /* defined(DCPDP_DYNAMIC_GTF_TIMING) */
+
+	/* Setup simple buffer allocator */
+	psDeviceData->ui32BufferSize = (IMG_UINT32)DC_ALIGN(psDeviceData->pasTimingData[psDeviceData->uiTimingDataSize - 1].ui32HDisplay *
+							    psDeviceData->pasTimingData[psDeviceData->uiTimingDataSize - 1].ui32VDisplay *
+							    DCPDP_PIXEL_FORMAT_BPP,
+							    DC_OSGetPageSize());
+
+	ui64BufferCount = psDeviceData->uiDispMemSize;
+	ui64BufferCount = DC_OSDiv64(ui64BufferCount, psDeviceData->ui32BufferSize);
+
+	psDeviceData->ui32BufferCount = (IMG_UINT32)ui64BufferCount;
+	if (psDeviceData->ui32BufferCount == 0)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Not enough space for the framebuffer\n", __func__);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorFreeTimingData;
+	}
+	else if (psDeviceData->ui32BufferCount > DCPDP_MAX_BUFFERS)
+	{
+		psDeviceData->ui32BufferCount = DCPDP_MAX_BUFFERS;
+	}
+
+	psDeviceData->ui32BufferUseMask = 0;
+
+	/* Create a system buffer */
+	eError = InitSystemBuffer(psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to initialise the system buffer (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorFreeTimingData;
+	}
+
+	SetModeRegisters(psDeviceData);
+	psDeviceData->bVSyncEnabled = IMG_TRUE;
+	Flip(psDeviceData->psSystemBuffer);
+
+	eError = psDeviceData->psPVRServicesFuncs->pfnDCRegisterDevice(&g_sDCFunctions,
+								       DCPDP_MAX_COMMANDS_INFLIGHT,
+								       psDeviceData,
+								       &psDeviceData->hPVRServicesDevice);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to register the display device (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDestroySystemBuffer;
+	}
+
+	/* Register our handler */
+	eError = DCPDPInstallDeviceLISR(psDeviceData->psDevicePriv,
+					DisplayLISRHandler,
+					psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to install the display device interrupt handler (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDCUnregisterDevice;
+	}
+
+	eError = DC_OSWorkQueueCreate(&psDeviceData->hVSyncWorkQueue, 1);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work queue (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDCUninstallLISR;
+	}
+
+	eError = DC_OSWorkQueueCreateWorkItem(&psDeviceData->hVSyncWorkItem, VSyncProcessor, psDeviceData);
+	if (eError != PVRSRV_OK)
+	{
+		DC_OSDebugPrintf(DBGLVL_ERROR, " - %s: Failed to create a vsync work item (%s)\n",
+				 __func__, psDeviceData->psPVRServicesFuncs->pfnGetErrorString(eError));
+		goto ErrorDCDestroyWorkQueue;
+	}
+
+	/* Print some useful information */
+	DC_OSDebugPrintf(DBGLVL_INFO, " using mode %ux%u@%uHz (phys: %ux%u)\n",
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay,
+			 psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VRefresh,
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32HDisplay * 1000) / DCPDP_DPI * 254) / 10000),
+			 (IMG_UINT32)(((psDeviceData->pasTimingData[psDeviceData->uiTimingDataIndex].ui32VDisplay * 1000) / DCPDP_DPI * 254) / 10000));
+	DC_OSDebugPrintf(DBGLVL_INFO, " register base: 0x%px\n", psDeviceData->sPDPRegCpuPAddr.uiAddr);
+	DC_OSDebugPrintf(DBGLVL_INFO, " memory base: 0x%px\n", psDeviceData->sDispMemCpuPAddr.uiAddr);
+
+	*ppsDeviceData = psDeviceData;
+
+	return PVRSRV_OK;
+
+ErrorDCDestroyWorkQueue:
+	DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+ErrorDCUninstallLISR:
+	DCPDPUninstallDeviceLISR(psDevicePriv);
+
+ErrorDCUnregisterDevice:
+	psDeviceData->psPVRServicesFuncs->pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+
+ErrorDestroySystemBuffer:
+	DeInitSystemBuffer(psDeviceData);
+
+ErrorFreeTimingData:
+	DC_OSFreeMem(psDeviceData->pasTimingData);
+
+ErrorPDPOSUnmapPDPRegisters:
+	DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, psDeviceData->ui32PDPRegSize);
+
+ErrorReleaseMemRegion:
+	DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize);
+
+ErrorPhysHeapRelease:
+	psDeviceData->psPVRServicesFuncs->pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+ErrorFreeDeviceData:
+	DC_OSFreeMem(psDeviceData);
+
+	return eError;
+}
+
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, DCPDP_DEVICE_PRIV **ppsDevicePriv)
+{
+	if (psDeviceData)
+	{
+		DC_OSWorkQueueDestroyWorkItem(psDeviceData->hVSyncWorkItem);
+		DC_OSWorkQueueDestroy(psDeviceData->hVSyncWorkQueue);
+
+		psDeviceData->psPVRServicesFuncs->pfnDCUnregisterDevice(psDeviceData->hPVRServicesDevice);
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		if (PVRSRVGetDriverStatus() != PVRSRV_SERVICES_STATE_OK)
+		{
+			psDeviceData->psFlipContext = NULL;
+		}
+		else
+#else
+		{
+			DC_ASSERT(psDeviceData->psFlipContext == NULL);
+		}
+#endif
+		(void)DCPDPUninstallDeviceLISR(psDeviceData->psDevicePriv);
+
+		DeInitSystemBuffer(psDeviceData);
+
+		/* Reset the register to make sure the PDP doesn't try to access on card memory */
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, 0x00000000);
+
+		DC_OSUnmapPhysAddr(psDeviceData->pvPDPRegCpuVAddr, psDeviceData->ui32PDPRegSize);
+
+		DC_OSReleaseAddrRegion(psDeviceData->sPDPRegCpuPAddr, psDeviceData->ui32PDPRegSize);
+
+		psDeviceData->psPVRServicesFuncs->pfnPhysHeapRelease(psDeviceData->psPhysHeap);
+
+		if (ppsDevicePriv != NULL)
+		{
+			*ppsDevicePriv = psDeviceData->psDevicePriv;
+		}
+
+		DC_OSFreeMem(psDeviceData->pasTimingData);
+		DC_OSFreeMem(psDeviceData);
+	}
+}
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable)
+{
+	if (psDeviceData)
+	{
+		IMG_UINT32 ui32Value;
+
+		ui32Value = DC_OSReadReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL);
+		ui32Value &= ~(STR1STREN_MASK);
+
+		if (bEnable)
+		{
+			ui32Value |= 0x1 << STR1STREN_SHIFT;
+		}
+
+		DC_OSWriteReg32(psDeviceData->pvPDPRegCpuVAddr, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, ui32Value);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..3af6db74f779
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Kbuild.mk
@@ -0,0 +1,59 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-$(CONFIG_X86) += -mno-soft-float
+
+ccflags-y += \
+	-I$(TOP)/services/display/rogue/dc_pdp/include \
+	-I$(TOP)/services/display/include \
+	-I$(TOP)/include/rogue/system/rgx_tc -I$(TOP)/include/system/rgx_tc	\
+	-I$(TOP)/include/rogue/public -I$(TOP)/include/public
+
+dc_pdp-y += \
+	services/display/rogue/dc_pdp/common/dc_pdp_common.o \
+	services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.o	\
+	services/display/common/env/linux/dc_osfuncs.o
+
+ifeq ($(PVR_SYSTEM),rgx_linux_tc)
+ccflags-y += \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc
+
+CFLAGS_services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.o := -DDCPDP_REGISTER_DRIVER
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Linux.mk
new file mode 100644
index 000000000000..5c5a1f234723
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/Linux.mk
@@ -0,0 +1,51 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+ifeq ($(PVR_ARCH),rogue)
+modules := dc_pdp
+
+dc_pdp_type := kernel_module
+dc_pdp_target := dc_pdp.ko
+dc_pdp_makefile := $(THIS_DIR)/Kbuild.mk
+endif
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.c
new file mode 100644
index 000000000000..aaeda41c9ed6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/env/linux/pdp_osfuncs.c
@@ -0,0 +1,682 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+
+#if defined(DCPDP_REGISTER_DRIVER)
+#include "pdp_common.h"
+#include "pdp_regs.h"
+#include "tc_drv.h"
+#endif
+
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#include "dc_pdp.h"
+#include "img_defs.h"
+
+#define DCPDP_WIDTH_MIN			(640)
+#define DCPDP_WIDTH_MAX			(1280)
+#define DCPDP_HEIGHT_MIN		(480)
+#define DCPDP_HEIGHT_MAX		(1024)
+
+#define DCPDP_DEBUGFS_DISPLAY_ENABLED	"display_enabled"
+
+#if defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_WIDTH defined but DCPDP_HEIGHT not defined
+#elif !defined(DCPDP_WIDTH) && defined(DCPDP_HEIGHT)
+#error ERROR: DCPDP_HEIGHT defined but DCPDP_WIDTH not defined
+#elif !defined(DCPDP_WIDTH) && !defined(DCPDP_HEIGHT)
+#define DCPDP_WIDTH			DCPDP_WIDTH_MAX
+#define DCPDP_HEIGHT			DCPDP_HEIGHT_MAX
+#elif (DCPDP_WIDTH > DCPDP_WIDTH_MAX)
+#error ERROR: DCPDP_WIDTH too large (max: 1280)
+#elif (DCPDP_WIDTH < DCPDP_WIDTH_MIN)
+#error ERROR: DCPDP_WIDTH too small (min: 640)
+#elif (DCPDP_HEIGHT > DCPDP_HEIGHT_MAX)
+#error ERROR: DCPDP_HEIGHT too large (max: 1024)
+#elif (DCPDP_HEIGHT < DCPDP_HEIGHT_MIN)
+#error ERROR: DCPDP_HEIGHT too small (max: 480)
+#endif
+
+struct DCPDP_DEVICE_PRIV_TAG
+{
+	struct device		*psDev;
+	DCPDP_DEVICE		*psPDPDevice;
+
+	IMG_HANDLE		hServicesConnection;
+	DC_SERVICES_FUNCS	sServicesFuncs;
+
+	IMG_CPU_PHYADDR		sPDPRegCpuPAddr;
+	IMG_UINT32		ui32PDPRegSize;
+	IMG_CPU_PHYADDR		sPLLRegCpuPAddr;
+	IMG_UINT32		ui32PLLRegSize;
+
+	struct dentry		*psDebugFSEntryDir;
+	struct dentry		*psDisplayEnabledEntry;
+
+#if defined(DCPDP_REGISTER_DRIVER)
+	PFN_LISR		pfnLISR;
+	void			*pvLISRData;
+#else
+	IMG_HANDLE		hLISRData;
+#endif
+};
+
+#if !defined(DCPDP_REGISTER_DRIVER)
+static DCPDP_DEVICE_PRIV *g_psDevicePriv;
+#endif
+
+/* PDP module parameters */
+DCPDP_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32PDPEnabled = 1,
+	.ui32PDPWidth   = DCPDP_WIDTH,
+	.ui32PDPHeight  = DCPDP_HEIGHT
+};
+module_param_named(mem_en, sModuleParams.ui32PDPEnabled, uint, S_IRUGO | S_IWUSR);
+module_param_named(width,  sModuleParams.ui32PDPWidth,   uint, S_IRUGO | S_IWUSR);
+module_param_named(height, sModuleParams.ui32PDPHeight,  uint, S_IRUGO | S_IWUSR);
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void)
+{
+	return &sModuleParams;
+}
+
+IMG_CPU_PHYADDR DCPDPGetAddrRangeStart(DCPDP_DEVICE_PRIV *psDevicePriv,
+				       DCPDP_ADDRESS_RANGE eRange)
+{
+	switch (eRange)
+	{
+		default:
+			DC_ASSERT(!"Unsupported address range");
+			__fallthrough;
+		case DCPDP_ADDRESS_RANGE_PDP:
+			return psDevicePriv->sPDPRegCpuPAddr;
+		case DCPDP_ADDRESS_RANGE_PLL:
+			return psDevicePriv->sPLLRegCpuPAddr;
+	}
+}
+
+IMG_UINT32 DCPDPGetAddrRangeSize(DCPDP_DEVICE_PRIV *psDevicePriv,
+				 DCPDP_ADDRESS_RANGE eRange)
+{
+	switch (eRange)
+	{
+		default:
+			DC_ASSERT(!"Unsupported address range");
+			__fallthrough;
+		case DCPDP_ADDRESS_RANGE_PDP:
+			return psDevicePriv->ui32PDPRegSize;
+		case DCPDP_ADDRESS_RANGE_PLL:
+			return psDevicePriv->ui32PLLRegSize;
+	}
+}
+
+#if defined(DCPDP_REGISTER_DRIVER)
+static void DCPDPLISRHandlerWrapper(void *pvData)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv = pvData;
+
+	if (psDevicePriv->pfnLISR != NULL)
+	{
+		(void) psDevicePriv->pfnLISR(psDevicePriv->pvLISRData);
+	}
+}
+#endif
+
+PVRSRV_ERROR DCPDPInstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv,
+				    PFN_LISR pfnLISR, void *pvData)
+{
+#if defined(DCPDP_REGISTER_DRIVER)
+	struct device *psParentDev = psDevicePriv->psDev->parent;
+	int iErr;
+
+	if (psDevicePriv->pfnLISR != NULL)
+	{
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+
+	psDevicePriv->pfnLISR = pfnLISR;
+	psDevicePriv->pvLISRData = pvData;
+
+	iErr = tc_set_interrupt_handler(psParentDev,
+					TC_INTERRUPT_PDP,
+					DCPDPLISRHandlerWrapper,
+					psDevicePriv);
+	if (iErr)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to install interrupt handler (err=%d)\n",
+		       __func__, iErr);
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	iErr = tc_enable_interrupt(psParentDev, TC_INTERRUPT_PDP);
+	if (iErr)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to enable interrupts (err=%d)\n",
+		       __func__, iErr);
+		tc_set_interrupt_handler(psParentDev, TC_INTERRUPT_PDP, NULL, NULL);
+		psDevicePriv->pfnLISR = NULL;
+		psDevicePriv->pvLISRData = NULL;
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	return PVRSRV_OK;
+#else
+	if (psDevicePriv->hLISRData != NULL)
+	{
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+
+	return psDevicePriv->sServicesFuncs.pfnSysInstallDeviceLISR(psDevicePriv->psDev,
+								    DCPDP_INTERRUPT_ID,
+								    DRVNAME,
+								    pfnLISR,
+								    pvData,
+								    &psDevicePriv->hLISRData);
+#endif
+}
+
+PVRSRV_ERROR DCPDPUninstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+#if defined(DCPDP_REGISTER_DRIVER)
+	struct device *psParentDev = psDevicePriv->psDev->parent;
+	int iErr;
+
+	if (psDevicePriv->pfnLISR == NULL)
+	{
+		return PVRSRV_ERROR_ISR_NOT_INSTALLED;
+	}
+
+	tc_disable_interrupt(psParentDev, TC_INTERRUPT_PDP);
+
+	iErr = tc_set_interrupt_handler(psParentDev, TC_INTERRUPT_PDP, NULL, NULL);
+	if (iErr)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to uninstall interrupt handler (err=%d)\n",
+		       __func__, iErr);
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psDevicePriv->pfnLISR = NULL;
+	psDevicePriv->pvLISRData = NULL;
+
+	return PVRSRV_OK;
+#else
+	PVRSRV_ERROR eError;
+
+	if (psDevicePriv->hLISRData == NULL)
+	{
+		return PVRSRV_ERROR_ISR_NOT_INSTALLED;
+	}
+
+	eError = PVRSRVSystemUninstallDeviceLISR(psDevicePriv->hLISRData);
+	if (eError == PVRSRV_OK)
+	{
+		psDevicePriv->hLISRData = NULL;
+	}
+
+	return eError;
+#endif
+}
+
+static int DisplayEnabledOpen(struct inode *psINode, struct file *psFile)
+{
+	psFile->private_data = psINode->i_private;
+
+	return 0;
+}
+
+static ssize_t DisplayEnabledRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	loff_t uiPosition = *puiPosition;
+	char pszBuffer[] = "N\n";
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	int iErr;
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if (uiPosition >= uiBufferSize || uiCount == 0)
+	{
+		return 0;
+	}
+
+	if (sModuleParams.ui32PDPEnabled)
+	{
+		pszBuffer[0] = 'Y';
+	}
+
+	if (uiCount > uiBufferSize - uiPosition)
+	{
+		uiCount = uiBufferSize - uiPosition;
+	}
+
+	iErr = copy_to_user(psUserBuffer, &pszBuffer[uiPosition], uiCount);
+	if (iErr)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + uiCount;
+
+	return uiCount;
+}
+
+static ssize_t DisplayEnabledWrite(struct file *psFile,
+				   const char __user *psUserBuffer,
+				   size_t uiCount,
+				   loff_t *puiPosition)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv = psFile->private_data;
+	char pszBuffer[3];
+	bool bPDPEnabled;
+	int iErr;
+
+	uiCount = min(uiCount, ARRAY_SIZE(pszBuffer) - 1);
+
+	iErr = copy_from_user(pszBuffer, psUserBuffer, uiCount);
+	if (iErr)
+	{
+		return -EFAULT;
+	}
+
+	pszBuffer[uiCount] = '\0';
+
+	if (strtobool(pszBuffer, &bPDPEnabled) == 0)
+	{
+		sModuleParams.ui32PDPEnabled = bPDPEnabled ? 1 : 0;
+
+		DCPDPEnableMemoryRequest(psDevicePriv->psPDPDevice, bPDPEnabled);
+	}
+
+	return uiCount;
+}
+
+static const struct file_operations gsDisplayEnabledFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = DisplayEnabledOpen,
+	.read = DisplayEnabledRead,
+	.write = DisplayEnabledWrite,
+	.llseek = default_llseek,
+};
+
+static void DCPDPDebugFSInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	psDevicePriv->psDebugFSEntryDir = debugfs_create_dir(DRVNAME, NULL);
+	if (IS_ERR_OR_NULL(psDevicePriv->psDebugFSEntryDir))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs root directory "
+		       "(debugfs entries won't be available)\n", __func__, DRVNAME);
+		psDevicePriv->psDebugFSEntryDir = NULL;
+		return;
+	}
+
+	psDevicePriv->psDisplayEnabledEntry =
+		debugfs_create_file(DCPDP_DEBUGFS_DISPLAY_ENABLED,
+				    S_IFREG | S_IRUGO | S_IWUSR,
+				    psDevicePriv->psDebugFSEntryDir,
+				    psDevicePriv,
+				    &gsDisplayEnabledFileOps);
+	if (IS_ERR_OR_NULL(psDevicePriv->psDisplayEnabledEntry))
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs entry\n",
+		       __func__, DCPDP_DEBUGFS_DISPLAY_ENABLED);
+		psDevicePriv->psDisplayEnabledEntry = NULL;
+		return;
+	}
+}
+
+static void DCPDPDebugFSDeInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	if (psDevicePriv->psDisplayEnabledEntry != NULL)
+	{
+		debugfs_remove(psDevicePriv->psDisplayEnabledEntry);
+		psDevicePriv->psDisplayEnabledEntry = NULL;
+	}
+
+	if (psDevicePriv->psDebugFSEntryDir != NULL)
+	{
+		debugfs_remove(psDevicePriv->psDebugFSEntryDir);
+		psDevicePriv->psDebugFSEntryDir = NULL;
+	}
+}
+
+
+static PVRSRV_ERROR DCPDPServicesInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DC_OSPVRServicesConnectionOpen(&psDevicePriv->hServicesConnection);
+	if (eError != PVRSRV_OK)
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to open connection to PVR Services (%d)\n",
+		       __func__, eError);
+		return eError;
+	}
+
+	eError = DC_OSPVRServicesSetupFuncs(psDevicePriv->hServicesConnection,
+					    &psDevicePriv->sServicesFuncs);
+	if (eError != PVRSRV_OK)
+	{
+		printk(KERN_WARNING DRVNAME " - %s: Failed to setup PVR Services function table (%d)\n",
+		       __func__, eError);
+		goto ErrorServicesConnectionClose;
+	}
+
+	return PVRSRV_OK;
+
+ErrorServicesConnectionClose:
+	DC_OSPVRServicesConnectionClose(psDevicePriv->hServicesConnection);
+	psDevicePriv->hServicesConnection = NULL;
+
+	return eError;
+}
+
+static void DCPDPServicesDeInit(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	DC_OSMemSet(&psDevicePriv->sServicesFuncs, 0,
+		    sizeof(psDevicePriv->sServicesFuncs));
+
+	DC_OSPVRServicesConnectionClose(psDevicePriv->hServicesConnection);
+	psDevicePriv->hServicesConnection = NULL;
+}
+
+static int DCPDPPCIDevicePrivInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+				  struct pci_dev *psPCIDev)
+{
+	IMG_UINT32 ui32RegBaseAddr = DC_OSAddrRangeStart(&psPCIDev->dev,
+							 DCPDP_REG_PCI_BASENUM);
+
+	psDevicePriv->sPDPRegCpuPAddr.uiAddr = ui32RegBaseAddr + DCPDP_PCI_PDP_REG_OFFSET;
+	psDevicePriv->ui32PDPRegSize = DCPDP_PCI_PDP_REG_SIZE;
+
+	psDevicePriv->sPLLRegCpuPAddr.uiAddr = ui32RegBaseAddr + DCPDP_PCI_PLL_REG_OFFSET;
+	psDevicePriv->ui32PLLRegSize = DCPDP_PCI_PLL_REG_SIZE;
+
+	return 0;
+}
+
+static int DCPDPPlatformDevicePrivInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+				       struct platform_device *psPlatDev)
+{
+	struct resource *psRegs;
+
+	psRegs = platform_get_resource_byname(psPlatDev, IORESOURCE_MEM, "pdp-regs");
+	if (psRegs == NULL)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to get PDP registers\n",
+			   __func__);
+		return -ENXIO;
+	}
+
+	psDevicePriv->sPDPRegCpuPAddr.uiAddr =
+		IMG_CAST_TO_CPUPHYADDR_UINT(psRegs->start);
+	psDevicePriv->ui32PDPRegSize = (IMG_UINT32) resource_size(psRegs);
+
+	psRegs = platform_get_resource_byname(psPlatDev, IORESOURCE_MEM, "pll-regs");
+	if (psRegs == NULL)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to get PLL registers\n",
+			   __func__);
+		return -ENXIO;
+	}
+
+	psDevicePriv->sPLLRegCpuPAddr.uiAddr =
+		IMG_CAST_TO_CPUPHYADDR_UINT(psRegs->start);
+	psDevicePriv->ui32PLLRegSize = (IMG_UINT32) resource_size(psRegs);
+
+	return 0;
+}
+
+static DCPDP_DEVICE_PRIV *DCPDPDevicePrivCreate(struct device *psDev)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+	PVRSRV_ERROR eError;
+	int iRet;
+
+	psDevicePriv = DC_OSCallocMem(sizeof(*psDevicePriv));
+	if (psDevicePriv == NULL)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to allocate device private data\n",
+			   __func__);
+		iRet = -ENOMEM;
+		goto ErrorReturn;
+	}
+
+	psDevicePriv->psDev = psDev;
+
+	eError = DCPDPServicesInit(psDevicePriv);
+	if (eError != PVRSRV_OK)
+	{
+		iRet = -ENODEV;
+		goto ErrorFreeDevicePriv;
+	}
+
+	if (dev_is_pci(psDev))
+	{
+		iRet = DCPDPPCIDevicePrivInit(psDevicePriv, to_pci_dev(psDev));
+	}
+	else
+	{
+		iRet = DCPDPPlatformDevicePrivInit(psDevicePriv,
+						   to_platform_device(psDev));
+	}
+
+	if (iRet != 0)
+	{
+		goto ErrorServicesDeInit;
+	}
+
+	eError = DCPDPInit(psDevicePriv,
+			   &psDevicePriv->sServicesFuncs,
+			   &psDevicePriv->psPDPDevice);
+	if (eError != PVRSRV_OK)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to initialise device (%d)\n",
+			   __func__, eError);
+		iRet = -ENODEV;
+		goto ErrorServicesDeInit;
+	}
+
+	DCPDPDebugFSInit(psDevicePriv);
+
+	return psDevicePriv;
+
+ErrorServicesDeInit:
+	DCPDPServicesDeInit(psDevicePriv);
+ErrorFreeDevicePriv:
+	DC_OSFreeMem(psDevicePriv);
+ErrorReturn:
+	return ERR_PTR(iRet);
+}
+
+static void DCPDPDevicePrivDestroy(DCPDP_DEVICE_PRIV *psDevicePriv)
+{
+	DCPDPDebugFSDeInit(psDevicePriv);
+	DCPDPDeInit(psDevicePriv->psPDPDevice, NULL);
+	DCPDPServicesDeInit(psDevicePriv);
+	DC_OSFreeMem(psDevicePriv);
+}
+
+#if defined(DCPDP_REGISTER_DRIVER)
+
+static int DCPDPProbe(struct platform_device *psPlatDev)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+	int iRet;
+
+	iRet = tc_enable(psPlatDev->dev.parent);
+	if (iRet)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to enable device (err=%d)\n",
+			   __func__, iRet);
+		return iRet;
+	}
+
+	psDevicePriv = DCPDPDevicePrivCreate(&psPlatDev->dev);
+	if (IS_ERR(psDevicePriv))
+	{
+		tc_disable(psPlatDev->dev.parent);
+		return PTR_ERR(psDevicePriv);
+	}
+
+	platform_set_drvdata(psPlatDev, psDevicePriv);
+
+	return 0;
+}
+
+static int DCPDPRemove(struct platform_device *psPlatDev)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+
+	psDevicePriv = platform_get_drvdata(psPlatDev);
+	if (WARN_ON(psDevicePriv == NULL))
+	{
+		return -ENODEV;
+	}
+
+	platform_set_drvdata(psPlatDev, NULL);
+
+	DCPDPDevicePrivDestroy(psDevicePriv);
+	tc_disable(psPlatDev->dev.parent);
+
+	return 0;
+}
+
+static void DCPDPShutdown(struct platform_device *psPlatDev)
+{
+	PVR_UNREFERENCED_PARAMETER(psPlatDev);
+}
+
+static struct platform_device_id DCPDPPlatformDeviceIDTable[] =
+{
+	{ .name = APOLLO_DEVICE_NAME_PDP, .driver_data = PDP_VERSION_APOLLO },
+	{ },
+};
+
+static struct platform_driver DCPDPPlatformDriver =
+{
+	.probe		= DCPDPProbe,
+	.remove		= DCPDPRemove,
+	.shutdown	= DCPDPShutdown,
+	.driver		=
+	{
+		.owner  = THIS_MODULE,
+		.name	= DRVNAME,
+	},
+	.id_table	= DCPDPPlatformDeviceIDTable,
+};
+
+module_platform_driver(DCPDPPlatformDriver);
+
+#else /* !defined(DCPDP_REGISTER_DRIVER) */
+
+static int __init dc_pdp_init(void)
+{
+	DCPDP_DEVICE_PRIV *psDevicePriv;
+	struct pci_dev *psPCIDev;
+	int error;
+
+	psPCIDev = pci_get_device(DCPDP_VENDOR_ID_POWERVR, DCPDP_DEVICE_ID_PCI_APOLLO_FPGA, NULL);
+	if (psPCIDev == NULL)
+	{
+		psPCIDev = pci_get_device(DCPDP_VENDOR_ID_POWERVR, DCPDP_DEVICE_ID_PCIE_APOLLO_FPGA, NULL);
+		if (psPCIDev == NULL)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to get PCI device\n", __func__);
+			return -ENODEV;
+		}
+	}
+
+	error = pci_enable_device(psPCIDev);
+	if (error != 0)
+	{
+		printk(KERN_ERR DRVNAME " - %s: Failed to enable PCI device (%d)\n", __func__, error);
+		return -ENODEV;
+	}
+
+	psDevicePriv = DCPDPDevicePrivCreate(&psPCIDev->dev);
+
+	/* To prevent possible problems with system suspend/resume, we don't
+	   keep the device enabled, but rely on the fact that the Rogue driver
+	   will have done a pci_enable_device. */
+	pci_disable_device(psPCIDev);
+
+	if (IS_ERR(psDevicePriv))
+	{
+		return PTR_ERR(psDevicePriv);
+	}
+
+	g_psDevicePriv = psDevicePriv;
+
+	return 0;
+}
+
+static void __exit dc_pdp_deinit(void)
+{
+	if (g_psDevicePriv)
+	{
+		DCPDPDevicePrivDestroy(g_psDevicePriv);
+		g_psDevicePriv = NULL;
+	}
+}
+
+module_init(dc_pdp_init);
+module_exit(dc_pdp_deinit);
+
+#endif /* defined(DCPDP_REGISTER_DRIVER) */
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/include/dc_pdp.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/include/dc_pdp.h
new file mode 100644
index 000000000000..ccfeadafc1d7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/dc_pdp/include/dc_pdp.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DC_PDP_H)
+#define DC_PDP_H
+
+#include "dc_osfuncs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define DRVNAME	"dc_pdp"
+
+#define DCPDP_INTERRUPT_ID		(1)
+
+#include "pdp_regs.h"
+
+/*************************************************************************/ /*!
+ dc_pdp OS functions
+*/ /**************************************************************************/
+
+typedef struct DCPDP_MODULE_PARAMETERS_TAG
+{
+	IMG_UINT32  ui32PDPEnabled;
+	IMG_UINT32  ui32PDPWidth;
+	IMG_UINT32  ui32PDPHeight;
+} DCPDP_MODULE_PARAMETERS;
+
+typedef enum DCPDP_ADDRESS_RANGE_TAG
+{
+	DCPDP_ADDRESS_RANGE_PDP = 0,
+	DCPDP_ADDRESS_RANGE_PLL,
+} DCPDP_ADDRESS_RANGE;
+
+typedef struct DCPDP_DEVICE_PRIV_TAG DCPDP_DEVICE_PRIV;
+
+
+const DCPDP_MODULE_PARAMETERS *DCPDPGetModuleParameters(void);
+
+IMG_CPU_PHYADDR DCPDPGetAddrRangeStart(DCPDP_DEVICE_PRIV *psDevicePriv,
+				       DCPDP_ADDRESS_RANGE eRange);
+IMG_UINT32 DCPDPGetAddrRangeSize(DCPDP_DEVICE_PRIV *psDevicePriv,
+				 DCPDP_ADDRESS_RANGE eRange);
+
+PVRSRV_ERROR DCPDPInstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv,
+				    PFN_LISR pfnLISR, void *pvData);
+PVRSRV_ERROR DCPDPUninstallDeviceLISR(DCPDP_DEVICE_PRIV *psDevicePriv);
+
+/*******************************************************************************
+ * dc_pdp common functions
+ ******************************************************************************/
+
+typedef struct DCPDP_DEVICE_TAG DCPDP_DEVICE;
+
+PVRSRV_ERROR DCPDPInit(DCPDP_DEVICE_PRIV *psDevicePriv,
+		       const DC_SERVICES_FUNCS *psServicesFuncs,
+		       DCPDP_DEVICE **ppsDeviceData);
+void DCPDPDeInit(DCPDP_DEVICE *psDeviceData, DCPDP_DEVICE_PRIV **ppsDevicePriv);
+
+void DCPDPEnableMemoryRequest(DCPDP_DEVICE *psDeviceData, IMG_BOOL bEnable);
+
+#endif /* !defined(DC_PDP_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/edid.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/edid.c
new file mode 100644
index 000000000000..4cdc79de0a9f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/edid.c
@@ -0,0 +1,1673 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+
+#include "edid.h"
+#include "phy.h"
+#include "img_defs.h"
+
+/* Section 2.12.4 of HDMI TX spec */
+const IMG_UINT32 I2C_DIV_FACTOR = 100000;
+const IMG_UINT16 EDID_I2C_MIN_FS_SCL_HIGH_TIME = 600;
+const IMG_UINT16 EDID_I2C_MIN_FS_SCL_LOW_TIME = 1300;
+const IMG_UINT16 EDID_I2C_MIN_SS_SCL_HIGH_TIME = 40000;
+const IMG_UINT16 EDID_I2C_MIN_SS_SCL_LOW_TIME = 47000;
+const IMG_UINT16 EDID_SFR_CLOCK = 2700;
+
+#ifdef HDMI_DEBUG
+
+//#define HDMI_TEST_EDID 1
+
+#ifdef HDMI_TEST_EDID
+IMG_UINT8 gSampleEdid[128] = {0x00,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0x00,
+0x10,0xAC,0x26,0xA0,0x53,0x34,0x43,0x32, // 8
+0x31,0x11,0x01,0x03,0x80,0x34,0x21,0x78, // 16
+0xEE,0x9B,0xB5,0xA9,0x54,0x34,0xAE,0x25, // 24
+0x13,0x50,0x54,0xA5,0x4B,0x00,0x81,0x80, // 32
+0xA9,0x40,0x71,0x4F,0xB3,0x00,0x01,0x01, // 40
+0x01,0x01,0x01,0x01,0x01,0x01,0x28,0x3C, // 48
+0x80,0xA0,0x70,0xB0,0x23,0x40,0x30,0x20, // 56
+0x36,0x00,0x07,0x44,0x21,0x00,0x00,0x1A, // 64
+0x00,0x00,0x00,0xFF,0x00,0x47,0x4D,0x35, // 72
+0x30,0x34,0x37,0x43,0x34,0x32,0x43,0x34, // 80
+0x53,0x20,0x00,0x00,0x00,0xFC,0x00,0x44, // 88
+0x45,0x4C,0x4C,0x32,0x34,0x30,0x37,0x57, // 96
+0x46,0x50,0x48,0x43,0x00,0x00,0x00,0xFD, // 104
+0x00,0x38,0x4C,0x1E,0x53,0x11,0x00,0x0A, // 112
+0x20,0x20,0x20,0x20,0x20,0x20,0x00,0x6F}; // 120
+#endif
+
+#endif
+
+
+#if defined(HDMI_ENABLE_FAST_COUNT)
+/* calculate the fast speed high time counter - round up */
+static IMG_UINT16 EdidCalcCount(IMG_UINT16 sclMinTime)
+{
+	unsigned long tmpSclPeriod = 0;
+	if (((EDID_SFR_CLOCK * sclMinTime) % I2C_DIV_FACTOR) != 0)
+	{
+		tmpSclPeriod = (unsigned long)((EDID_SFR_CLOCK * sclMinTime) + (I2C_DIV_FACTOR - ((EDID_SFR_CLOCK * sclMinTime) % I2C_DIV_FACTOR))) / I2C_DIV_FACTOR;
+	}
+	else
+	{
+		tmpSclPeriod = (unsigned long)(EDID_SFR_CLOCK * sclMinTime) / I2C_DIV_FACTOR;
+	}
+	return (IMG_UINT16)(tmpSclPeriod);
+}
+#endif
+
+static PVRSRV_ERROR EdidFindBaseBlock(IMG_UINT8 * buffer, IMG_UINT32 size, EDID_BASE_BLOCK * pBase)
+{
+    IMG_UINT32 index = 0;
+
+    HDMI_CHECKPOINT;
+
+    /* First find the EDID signature and fill in global structure */
+    for (index = 0; index < size; index++)
+    {
+        if (*((IMG_UINT64*)&buffer[index]) == EDID_SIGNATURE)
+        {
+            HDMI_DEBUG_PRINT("- %s: Found EDID signature at offset %d\n", __func__, index);
+            break;
+        }
+    }
+
+    if (index == size)
+    {
+        HDMI_ERROR_PRINT("- %s: Could not find EDID signature\n", __func__);
+        return PVRSRV_ERROR_NOT_FOUND;
+    }
+
+    /* Copy to global EDID buffer */
+    if (index + EDID_BLOCK_SIZE > size)
+    {
+        HDMI_ERROR_PRINT("- %s: Found signature but it spills over into next block?!?!\n", __func__);
+        return PVRSRV_ERROR_OUT_OF_MEMORY;
+    }
+    memcpy(pBase, &buffer[index], EDID_BLOCK_SIZE);
+
+    HDMI_DEBUG_PRINT("- %s EDID Version %d, Revision ID: %d", __func__, pBase->EdidVersion, pBase->EdidRevision);
+
+#ifdef HDMI_DEBUG
+    {
+        IMG_UINT8* pData = (IMG_UINT8*)pBase;
+        IMG_UINT32 i = 0;
+        HDMI_DEBUG_PRINT("- %s: EDID Data:\n\n", __func__);
+        for (i = 0; i < EDID_BLOCK_SIZE; i+=8)
+        {
+            HDMI_DEBUG_PRINT(" Offset %3d: %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
+                       pData[i], pData[i+1], pData[i+2], pData[i+3], pData[i+4], pData[i+5], pData[i+6], pData[i+7]);
+        }
+    }
+#endif
+
+    return PVRSRV_OK;
+
+}
+
+static PVRSRV_ERROR EdidParseDtd(IMG_UINT8 * pDesc, VIDEO_PARAMS * pVideoParams)
+{
+    IMG_UINT8 index = pVideoParams->mDtdIndex;
+    DTD_RAW * dtd = (DTD_RAW*)pDesc;
+    DTD decodedDtd = {0};
+
+    HDMI_CHECKPOINT;
+
+    if (index >= HDMI_DTD_MAX)
+    {
+        HDMI_ERROR_PRINT("- %s: Ran out of space for DTD\n", __func__);
+        return PVRSRV_ERROR_OUT_OF_MEMORY;
+    }
+
+
+    decodedDtd.mCode = -1;
+    decodedDtd.mPixelRepetitionInput = 0;
+
+    decodedDtd.mPixelClock = dtd->pixelClock; /*  [10000Hz] */
+    if (decodedDtd.mPixelClock < 0x01)
+    { /* 0x0000 is defined as reserved */
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    decodedDtd.mHActive = dtd->hActiveLower | (dtd->hActiveUpper << 8);
+    decodedDtd.mHBlanking = dtd->hBlankingLower | (dtd->hBlankingUpper << 8);
+    decodedDtd.mHFrontPorchWidth = dtd->hFrontPorchLower | (dtd->hFrontPorchUpper << 8);
+    decodedDtd.mHSyncPulseWidth = dtd->hSyncPulseWidthLower | (dtd->hSyncPulseWidthUpper << 8);
+    decodedDtd.mHImageSize = dtd->hImageSizeLower | (dtd->hImageSizeUpper << 8);
+    decodedDtd.mHBorder = dtd->hBorder;
+    decodedDtd.mHBackPorchWidth = decodedDtd.mHBlanking - decodedDtd.mHSyncPulseWidth - decodedDtd.mHFrontPorchWidth;
+
+    decodedDtd.mVActive = dtd->vActiveLower | (dtd->vActiveUpper << 8);
+    decodedDtd.mVBlanking = dtd->vBlankingLower | (dtd->vBlankingUpper << 8);
+    decodedDtd.mVFrontPorchWidth = dtd->vFrontPorchLower | (dtd->vFrontPorchUpper << 8);
+    decodedDtd.mVSyncPulseWidth = dtd->vSyncPulseWidthLower | (dtd->vSyncPulseWidthUpper << 8);
+    decodedDtd.mVImageSize = dtd->vImageSizeLower | (dtd->vImageSizeUpper << 8);
+    decodedDtd.mVBorder = dtd->vBorder;
+    decodedDtd.mVBackPorchWidth = decodedDtd.mVBlanking - decodedDtd.mVSyncPulseWidth - decodedDtd.mVFrontPorchWidth;
+
+    /* no stereo viewing support in HDMI */
+    decodedDtd.mInterlaced = dtd->interlaced;
+    HDMI_DEBUG_PRINT("- %s: sync signal defs: %d", __func__, dtd->syncSignalDefs);
+    if ((dtd->syncSignalDefs >> 3) & 0x01)
+    {
+        decodedDtd.mVSyncPolarity = (dtd->syncSignalDefs >> 1) & 0x01; // BIT 1
+        decodedDtd.mHSyncPolarity = (dtd->syncSignalDefs >> 0) & 0x01; // BIT 0
+    }
+
+    HDMI_DEBUG_PRINT("Decoded DTD %d with pixel clock %dHz, Horz (%d, %d, %d, %d, %d, %d, %d), Vert (%d, %d, %d, %d, %d, %d, %d)\n", index, decodedDtd.mPixelClock*10000,
+        decodedDtd.mHActive, decodedDtd.mHBlanking, decodedDtd.mHBorder, decodedDtd.mHImageSize, decodedDtd.mHFrontPorchWidth, decodedDtd.mHBackPorchWidth, decodedDtd.mHSyncPulseWidth,
+        decodedDtd.mVActive, decodedDtd.mVBlanking, decodedDtd.mVBorder, decodedDtd.mVImageSize, decodedDtd.mVFrontPorchWidth, decodedDtd.mVBackPorchWidth, decodedDtd.mVSyncPulseWidth);
+
+#if defined(HDMI_EXPERIMENTAL_CLOCKS)
+    pVideoParams->mDtdList[index] = decodedDtd;
+    pVideoParams->mDtdIndex++;
+#else
+    if (PhyIsPclkSupported(dtd->pixelClock))
+    {
+        pVideoParams->mDtdList[index] = decodedDtd;
+        pVideoParams->mDtdIndex++;
+    }
+    else
+    {
+        HDMI_DEBUG_PRINT("- %s: Found unsupported pixel clock (%d) in DTD, not adding", __func__, dtd->pixelClock);
+    }
+#endif
+
+    return PVRSRV_OK;
+}
+
+static inline IMG_UINT8 GetDescriptorType(IMG_UINT8 * pDesc)
+{
+    if (pDesc[0] == 0 && pDesc[1] == 0 && pDesc[2] == 0)
+    {
+        return EDID_DESC_DISPLAY;
+    }
+    else
+    {
+        return EDID_DESC_DETAILED_TIMING;
+    }
+}
+
+static PVRSRV_ERROR EdidParseDispDesc(IMG_UINT8 * pDesc, VIDEO_PARAMS * pVideoParams)
+{
+    DISPLAY_DESCRIPTOR * pDispDesc = (DISPLAY_DESCRIPTOR*)pDesc;
+
+    /* Confirm this is a display descriptor */
+    if (pDispDesc->DescTag[0] != 0 || pDispDesc->DescTag[1] != 0 || pDispDesc->DescTag[2] != 0)
+    {
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT(" Display descriptor tag: %x\n", pDispDesc->DispDescTag);
+
+    switch (pDispDesc->DispDescTag)
+    {
+        case DISP_DESC_TAG_PRODUCT_NAME:
+        {
+            IMG_UINT8 monitorName[14] = { 0 }; // Up to 13 characters
+            IMG_UINT8 i = 0;
+
+            for (i = 0; i < 13; i++)
+            {
+                if (pDispDesc->Data[i] != 0x0A) // 0A is terminator
+                {
+                    monitorName[i] = pDispDesc->Data[i];
+                }
+            }
+
+            /* Just print for now */
+            HDMI_DEBUG_PRINT(" Found monitor name descriptor: %s", monitorName);
+            break;
+        }
+        case DISP_DESC_TAG_RANGE_LIMITS:
+        {
+            /* Only fill range limits if offsets are 0 */
+            MONITOR_RANGE_DATA * pData = (MONITOR_RANGE_DATA*)&pDispDesc->Reserved; // This one is unique, no reserved field
+            if (pData->offsetFlags == 0)
+            {
+                HDMI_DEBUG_PRINT("- %s: Found valid monitor range limits, vMin: %x, vMax: %x, hMin: %x, hMax: %x, maxPClock: %x, support flags: %x\n",
+								 __func__, pData->vMinRate, pData->vMaxRate, pData->hMinRate, pData->hMaxRate, pData->maxPixelClock, pData->supportFlags);
+                pVideoParams->mRangeLimits.vMinRate = pData->vMinRate;
+                pVideoParams->mRangeLimits.vMaxRate = pData->vMaxRate;
+                pVideoParams->mRangeLimits.hMinRate = pData->hMinRate;
+                pVideoParams->mRangeLimits.hMaxRate = pData->hMaxRate;
+                pVideoParams->mRangeLimits.maxPixelClock = pData->maxPixelClock;
+                pVideoParams->mRangeLimits.valid = true;
+            }
+            break;
+        }
+        case DISP_DESC_TAG_EST_TIMING_3:
+        case DISP_DESC_TAG_CVT_3:
+        case DISP_DESC_TAG_DCM:
+        case DISP_DESC_TAG_STD_TIMING:
+        case DISP_DESC_TAG_COLOR_POINT_DATA:
+        case DISP_DESC_TAG_DATA_STRING:
+        case DISP_DESC_TAG_SERIAL_NUMBER:
+            /* Don't care */
+            break;
+        default:
+            HDMI_ERROR_PRINT("- %s: Invalid display descriptor tag\n", __func__);
+            return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR EdidParseDescriptor(IMG_UINT8 * pDesc, VIDEO_PARAMS * pVideoParams)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    IMG_UINT8 dtdDesc = 0;
+
+    HDMI_CHECKPOINT;
+
+    dtdDesc = GetDescriptorType(pDesc);
+
+    if (dtdDesc == EDID_DESC_DETAILED_TIMING)
+    {
+        status = EdidParseDtd(pDesc, pVideoParams);
+        if (status != PVRSRV_OK)
+        {
+            HDMI_ERROR_PRINT("- %s: EDID failed to parse descriptor. Status: %d\n", __func__, status);
+            return status;
+        }
+    }
+    else if (dtdDesc == EDID_DESC_DISPLAY)
+    {
+        status = EdidParseDispDesc(pDesc, pVideoParams);
+        if (status != PVRSRV_OK)
+        {
+            HDMI_ERROR_PRINT("- %s: EDID failed to parse descriptor. Status: %d\n", __func__, status);
+            return status;
+        }
+    }
+
+    return status;
+
+}
+
+static IMG_UINT8 EdidGetColorResolution(VIDEO_INPUT_DEFINITION videoInput)
+{
+	IMG_UINT8 colorRes = 0;
+
+	switch (videoInput.digital.colorBitDepth)
+    {
+		case 1:
+            colorRes = 6;
+            break;
+        case 2:
+            colorRes = 8;
+            break;
+        case 3:
+            colorRes = 10;
+            break;
+        case 4:
+            colorRes = 12;
+            break;
+        case 5:
+            colorRes = 14;
+            break;
+        case 6:
+            colorRes = 16;
+            break;
+        default:
+            HDMI_DEBUG_PRINT("- %s: color bit depth undefined!", __func__);
+            break;
+    }
+
+    return colorRes;
+}
+
+static PVRSRV_ERROR EdidParseBaseBlock(EDID_BASE_BLOCK * edidBlock, VIDEO_PARAMS * pVideoParams)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    IMG_UINT8 index = 0;
+    IMG_UINT8 checkSum = 0;
+    IMG_UINT8 * buffer = (IMG_UINT8*)edidBlock;
+
+    HDMI_CHECKPOINT;
+
+    /* Checksum */
+    for (index = 0; index < EDID_BLOCK_SIZE; index++)
+    {
+        checkSum = (IMG_UINT8) (checkSum + buffer[index]);
+    }
+
+    if (checkSum != EDID_CHECKSUM)
+    {
+        HDMI_ERROR_PRINT("- %s: EDID checksum failed, computed checksum: %d\n", __func__, checkSum);
+        status = PVRSRV_ERROR_DC_INVALID_CONFIG;
+        goto EXIT;
+    }
+
+    /* Extract color resolution */
+    HDMI_DEBUG_PRINT("- %s: Video Input definition 0x%x, Feature Support 0x%x", __func__, edidBlock->VideoInputDefinition.value, edidBlock->FeatureSupport.value);
+    if (edidBlock->VideoInputDefinition.digital.signalInterface == 1)
+    {
+        if (edidBlock->VideoInputDefinition.digital.colorBitDepth != 0)
+        {
+            HDMI_DEBUG_PRINT("- %s: Setting color resolution to %d", __func__, EdidGetColorResolution(edidBlock->VideoInputDefinition));
+            pVideoParams->mColorResolution = EdidGetColorResolution(edidBlock->VideoInputDefinition);
+        }
+        /* Otherwise grab from feature support flags */
+        else
+        {
+            /* RGB 4:4:4 must be supported */
+            pVideoParams->mColorResolution = 8;
+        }
+
+#if 0
+        if (edidBlock->VideoInputDefinition.digital.signalInterface == 1)
+#else
+		if (edidBlock->VideoInputDefinition.digital.vidInterfaceSupported == DIGITAL_VIDEO_INTERFACE_HDMI_A ||
+			edidBlock->VideoInputDefinition.digital.vidInterfaceSupported == DIGITAL_VIDEO_INTERFACE_HDMI_B)
+#endif
+		{
+			pVideoParams->mHdmi = 1;
+		}
+    }
+
+    /* First DTD is preferred timing (VESA 1.4) */
+    if (edidBlock->FeatureSupport.fields.preferredTimingIncluded)
+    {
+        HDMI_DEBUG_PRINT("- %s: Found preferred timing descriptor", __func__);
+        status = EdidParseDtd((IMG_UINT8*)&edidBlock->PreferredTimingMode, pVideoParams);
+        if (status != PVRSRV_OK)
+        {
+            HDMI_ERROR_PRINT("- %s: EDID failed to parse preferred timing DTD. Status: %d\n", __func__, status);
+            goto EXIT;
+        }
+
+        pVideoParams->mPreferredTimingIncluded = true;
+    }
+
+    HDMI_CHECKPOINT;
+
+    /* Parse remaining descriptors, could be DTD or display descriptor (first 3 bytes = 0) */
+    status = EdidParseDescriptor(edidBlock->Descriptor2, pVideoParams);
+    CHECK_AND_EXIT(status, EXIT);
+
+    status = EdidParseDescriptor(edidBlock->Descriptor3, pVideoParams);
+    CHECK_AND_EXIT(status, EXIT);
+
+    status = EdidParseDescriptor(edidBlock->Descriptor4, pVideoParams);
+    CHECK_AND_EXIT(status, EXIT);
+
+    HDMI_CHECKPOINT;
+
+EXIT:
+
+    return PVRSRV_OK;
+}
+
+static void EdidParseCeaDataBlock(VIDEO_PARAMS * pVideoParams, IMG_UINT8 * data, IMG_UINT32 * length)
+{
+    IMG_UINT8 tag;
+    IMG_UINT8 nBlocks;
+    IMG_UINT8 i;
+
+    HDMI_CHECKPOINT;
+
+    // Upper 3 bits of first byte are the tag
+    tag = data[0] >> 5;
+    // Bottom 5 bits of first byte are the number of blocks
+    nBlocks = data[0] & 0x1F;
+
+    HDMI_DEBUG_PRINT("- %s: Parsing CEA data block, tag: %d, nBlocks: %d\n", __func__, tag, nBlocks);
+
+    switch (tag)
+    {
+        case EDID_CEA_DATA_BLOCK_VIDEO_TAG:
+            for (i = 1; i < (nBlocks + 1); i++)
+            {
+                HDMI_DEBUG_PRINT("- %s: Native: %d, VIC: %d\n", __func__, data[i] >> 7, data[i] & 0x7F);
+                EdidAddDtd(pVideoParams, data[i] & 0x7F, EDID_REFRESH_RATE);
+                /* If its a native setup, then make it the active */
+                if (data[i] >> 7 == 1 && pVideoParams->mPreferredTimingIncluded == false)
+                {
+                    pVideoParams->mDtdActiveIndex = pVideoParams->mDtdIndex - 1;
+                }
+            }
+            break;
+        case EDID_CEA_DATA_BLOCK_VENDOR_TAG:
+        case EDID_CEA_DATA_BLOCK_AUDIO_TAG:
+        case EDID_CEA_DATA_BLOCK_SPEAKER_TAG:
+        default:
+            // don't care
+            break;
+    }
+
+    *length = nBlocks + 1;
+
+}
+
+static PVRSRV_ERROR EdidParseCeaBlock(IMG_UINT8 * data, VIDEO_PARAMS * pVideoParams)
+{
+    EDID_EXT_CEA * pCeaBlock = (EDID_EXT_CEA*)data;
+    IMG_UINT8 i = 0, j = 0;
+    IMG_UINT32 length; // of each CEA data block
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT("- %s: Parsing CEA extension block, DTD Offset: %d, total size: %d", __func__, pCeaBlock->DtdOffset, sizeof(pCeaBlock->Data));
+
+    /* Only support revision 3 */
+    if (pCeaBlock->Revision != EDID_CEA_REVISION)
+    {
+        HDMI_ERROR_PRINT("- %s: Found unsupported CEA block revision number (%d)\n", __func__, pCeaBlock->Revision);
+        return PVRSRV_ERROR_NOT_FOUND;
+    }
+
+    /* First parse data blocks */
+    for (i = 0; i < (pCeaBlock->DtdOffset + 4); i+= length)
+    {
+        EdidParseCeaDataBlock(pVideoParams, &pCeaBlock->Data[i], &length);
+    }
+
+    /* Now parse DTDs (max = 6)*/
+    for (i = pCeaBlock->DtdOffset - 4, j = 0; i < sizeof(pCeaBlock->Data) && j < 6; i += sizeof(DTD_RAW), j++)
+    {
+        EdidParseDtd(&pCeaBlock->Data[i], pVideoParams);
+    }
+
+    return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR EdidParseExtBlock(IMG_UINT8 * edidBlock, VIDEO_PARAMS * pVideoParams)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    EDID_EXT_BLOCK * pExtBlock = (EDID_EXT_BLOCK*) edidBlock;
+
+    HDMI_CHECKPOINT;
+
+#ifdef HDMI_DEBUG
+    HDMI_DEBUG_PRINT("- %s: Parsing extension block tag %d", __func__, pExtBlock->Tag);
+    {
+        IMG_UINT8* pData = (IMG_UINT8*)edidBlock;
+        IMG_UINT32 i = 0;
+        for (i = 0; i < EDID_BLOCK_SIZE; i+=8)
+        {
+            HDMI_DEBUG_PRINT(" Offset 0x%02x: %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
+                       pData[i], pData[i+1], pData[i+2], pData[i+3], pData[i+4], pData[i+5], pData[i+6], pData[i+7]);
+        }
+    }
+#endif
+
+    /* First byte is tag */
+    switch (pExtBlock->Tag)
+    {
+        case EDID_EXT_TAG_CEA:
+            status = EdidParseCeaBlock(edidBlock, pVideoParams);
+            break;
+        case EDID_EXT_TAG_VTB:
+        case EDID_EXT_TAG_DI:
+        case EDID_EXT_TAG_LS:
+        case EDID_EXT_TAG_DPVL:
+        case EDID_EXT_TAG_BLOCK_MAP:
+        case EDID_EXT_TAG_MFG_DEFINED:
+            status = PVRSRV_ERROR_NOT_SUPPORTED;
+            break;
+        default:
+            HDMI_ERROR_PRINT("- %s: Unknown ext block type %d\n", __func__, pExtBlock->Tag);
+            status = PVRSRV_ERROR_NOT_SUPPORTED;
+            break;
+    }
+
+    return status;
+}
+
+
+static PVRSRV_ERROR EdidI2cReadByte(IMG_CPU_VIRTADDR base, IMG_UINT8 offset, IMG_UINT8 * value)
+{
+    IMG_UINT32 intReg = 0;
+    IMG_UINT32 pollTime = 0;
+
+    #ifdef HDMI_TEST_EDID
+    *value = ((IMG_UINT8*)&gSampleEdid[0])[offset];
+    return PVRSRV_OK;
+    #endif
+
+    /* Write 1 to clear interrupt status */
+    HDMI_WRITE_CORE_REG(base, HDMI_IH_I2CM_STAT0_OFFSET,
+        SET_FIELD(HDMI_IH_I2CM_STAT0_I2CMASTERDONE_START, HDMI_IH_I2CM_STAT0_I2CMASTERDONE_MASK, 1));
+
+    /* Setup EDID base address */
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_SLAVE_OFFSET, EDID_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_SEGADDR_OFFSET, EDID_SEGMENT_ADDRESS);
+
+    // address
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_ADDRESS_OFFSET, offset);
+
+    /* Set operation to normal read */
+    HDMI_WRITE_CORE_REG(base, HDMI_I2CM_OPERATION_OFFSET,
+        SET_FIELD(HDMI_I2CM_OPERATION_RD_START, HDMI_I2CM_OPERATION_RD_MASK, 1));
+
+    // poll on done
+
+    intReg = HDMI_READ_CORE_REG(base, HDMI_IH_I2CM_STAT0_OFFSET);
+    while (!IS_BIT_SET(intReg, HDMI_IH_I2CM_STAT0_I2CMASTERDONE_START)
+        && !IS_BIT_SET(intReg, HDMI_IH_I2CM_STAT0_I2CMASTERERROR_START))
+    {
+        intReg = HDMI_READ_CORE_REG(base, HDMI_IH_I2CM_STAT0_OFFSET);
+        DC_OSDelayus(1 * 100);
+        pollTime += 1;
+        if (pollTime > EDID_I2C_TIMEOUT_MS)
+        {
+            HDMI_ERROR_PRINT("- %s: Timeout polling on I2CMasterDoneBit\n", __func__);
+            return PVRSRV_ERROR_TIMEOUT;
+        }
+    }
+
+    if (IS_BIT_SET(intReg, HDMI_IH_I2CM_STAT0_I2CMASTERERROR_START))
+    {
+        HDMI_ERROR_PRINT("- %s: I2C EDID read failed, Master error bit is set\n", __func__);
+        return PVRSRV_ERROR_TIMEOUT;
+    }
+
+    *value = HDMI_READ_CORE_REG(base, HDMI_I2CM_DATAI_OFFSET) & 0xFF;
+
+    return PVRSRV_OK;
+}
+
+
+/*
+
+Steps for reading EDID information:
+
+1. Initialize I2C DDC interface (clocks, interrupts, etc)
+2. Read EDID block byte-by-byte or with sequential
+read (up to 8 bytes) to limit interrupts
+3. Parse base block:
+    a. Mfg/vendor info
+    b. Preferred timing
+    c. Other descriptors
+4. Parse Extension blocks
+
+*/
+PVRSRV_ERROR EdidRead(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+    IMG_UINT8 bufferOffset = 0;
+    IMG_UINT8 i2cOffset = 0;
+    IMG_UINT8 localBuffer[EDID_BLOCK_SIZE];
+    IMG_UINT8 i = 0;
+    EDID_BASE_BLOCK edidBase;
+
+    HDMI_CHECKPOINT;
+
+    #if defined(HDMI_PDUMP)
+    /* Setup for VIC 16, color res 10 */
+    pvDevice->videoParams.mDtdActiveIndex = pvDevice->videoParams.mDtdIndex;
+    EdidAddDtd(&pvDevice->videoParams, 16, 60000);
+    return PVRSRV_OK;
+    #endif
+
+	/*to incorporate extensions we have to include the following - see VESA E-DDC spec. P 11 */
+
+    /*
+    Each segment pointer contains one byte of addressable space and
+    2 EDID blocks (128 bytes each)
+
+    SegPtr/Address range        Block type
+
+    00h/00h->7Fh                Base EDID structure
+    00h/80h->FFh                Extension block 1 OR block map 1
+    01h/00h->7Fh                Extension block 2
+    01h/80h->FFh                Extension block 3
+    ...
+    40h/00h->7Fh                Block map 2 if 129<=N<=254
+    ...
+    */
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SEGPTR_OFFSET, 0);
+
+    /*
+    Loop through and read byte by byte, could optimize and use ext read
+    */
+    for (bufferOffset = 0; bufferOffset < sizeof(localBuffer); bufferOffset++, i2cOffset++)
+    {
+        // read data
+        status = EdidI2cReadByte(pvDevice->pvHDMIRegCpuVAddr, i2cOffset, &localBuffer[bufferOffset]);
+        if (status != PVRSRV_OK)
+        {
+            return status;
+        }
+    }
+
+    /* Find base block and populate global */
+    status = EdidFindBaseBlock(localBuffer, EDID_BLOCK_SIZE, &edidBase);
+    if (status != PVRSRV_OK)
+    {
+        return status;
+    }
+
+    /* Extract video parameter data from base block (includes DTD parsing) */
+    status = EdidParseBaseBlock(&edidBase, &pvDevice->videoParams);
+    if (status != PVRSRV_OK)
+    {
+        return status;
+    }
+
+    HDMI_DEBUG_PRINT("- %s: Finished parsing base block, extension block count: %d\n", __func__, edidBase.ExtensionBlockCount);
+
+    /* Loop through each extension block */
+    for (i = 0; i < edidBase.ExtensionBlockCount; i++)
+    {
+        PVRSRV_ERROR extstatus = PVRSRV_OK;
+        for (bufferOffset = 0; bufferOffset < sizeof(localBuffer); bufferOffset++)
+        {
+            // read data
+            status = EdidI2cReadByte(pvDevice->pvHDMIRegCpuVAddr, i2cOffset++, &localBuffer[bufferOffset]);
+            if (status != PVRSRV_OK)
+            {
+                return status;
+            }
+        }
+
+        // don't treat error in extension block as fatal, so use separate status var.
+        extstatus = EdidParseExtBlock(localBuffer, &pvDevice->videoParams);
+        if (extstatus != PVRSRV_OK && extstatus != PVRSRV_ERROR_NOT_SUPPORTED)
+        {
+            HDMI_ERROR_PRINT("- %s: Failed to parse extension block %d\n", __func__, i);
+            break;
+        }
+    }
+
+    #if defined(HDMI_FORCE_MODE) && (HDMI_FORCE_MODE != 0)
+    HDMI_DEBUG_PRINT("- %s: Forcing HDMI mode %d", __func__, HDMI_FORCE_MODE);
+    EdidAddDtd(&pvDevice->videoParams, HDMI_FORCE_MODE, EDID_REFRESH_RATE);
+    pvDevice->videoParams.mDtdActiveIndex = pvDevice->videoParams.mDtdIndex - 1;
+    #endif
+
+    return status;
+}
+
+PVRSRV_ERROR EdidInitialize(HDMI_DEVICE * pvDevice)
+{
+    HDMI_CHECKPOINT;
+
+    /* Mask interrupts */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_INT_OFFSET,
+        SET_FIELD(HDMI_I2CM_INT_DONE_MASK_START, HDMI_I2CM_INT_DONE_MASK_MASK, 1) |
+        SET_FIELD(HDMI_I2CM_INT_READ_REQ_MASK_START, HDMI_I2CM_INT_READ_REQ_MASK_MASK, 1));
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_CTLINT_OFFSET,
+        SET_FIELD(HDMI_I2CM_CTLINT_ARB_MASK_START, HDMI_I2CM_CTLINT_ARB_MASK_START, 1) |
+        SET_FIELD(HDMI_I2CM_CTLINT_NACK_MASK_START, HDMI_I2CM_CTLINT_NACK_MASK_START, 1));
+
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_DIV_OFFSET,
+        SET_FIELD(HDMI_I2CM_DIV_FAST_STD_MODE_START, HDMI_I2CM_DIV_FAST_STD_MODE_START, 0));
+
+/* Set clock division and counts */
+
+/* Fast mode setup */
+/*
+    count = EdidCalcCount(EDID_I2C_MIN_FS_SCL_HIGH_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_HCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_HCNT_0_OFFSET, count & 0xFF);
+    count = EdidCalcCount(EDID_I2C_MIN_FS_SCL_LOW_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_LCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_FS_SCL_LCNT_0_OFFSET, count & 0xFF);
+    count = EdidCalcCount(EDID_I2C_MIN_SS_SCL_HIGH_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_HCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_HCNT_0_OFFSET, count & 0xFF);
+    count = EdidCalcCount(EDID_I2C_MIN_SS_SCL_LOW_TIME);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_LCNT_1_OFFSET, count >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_SS_SCL_LCNT_0_OFFSET, count & 0xFF);
+*/
+    /* Re-enable interrupts */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_INT_OFFSET,
+        SET_FIELD(HDMI_I2CM_INT_DONE_MASK_START, HDMI_I2CM_INT_DONE_MASK_MASK, 0));
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_I2CM_CTLINT_OFFSET,
+        SET_FIELD(HDMI_I2CM_CTLINT_ARB_MASK_START, HDMI_I2CM_CTLINT_ARB_MASK_START, 0) |
+        SET_FIELD(HDMI_I2CM_CTLINT_NACK_MASK_START, HDMI_I2CM_CTLINT_NACK_MASK_START, 0));
+
+    return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR EdidAddDtd(VIDEO_PARAMS * pVideoParams, IMG_UINT8 code, IMG_UINT32 refreshRate)
+{
+    DTD * dtd;
+
+    if (pVideoParams->mDtdIndex >= HDMI_DTD_MAX)
+    {
+        HDMI_DEBUG_PRINT("- %s: Hit max DTD entries (%d)", __func__, pVideoParams->mDtdIndex);
+        return PVRSRV_ERROR_OUT_OF_MEMORY;
+    }
+
+    dtd = &pVideoParams->mDtdList[pVideoParams->mDtdIndex];
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT("- %s: Adding DTD with VIC %d and refresh rate %d\n", __func__, code, refreshRate);
+
+	dtd->mCode = code;
+	dtd->mHBorder = DEFAULT_BORDER_WIDTH_PIXELS;
+	dtd->mVBorder = DEFAULT_BORDER_WIDTH_PIXELS;
+	dtd->mPixelRepetitionInput = 0;
+	dtd->mHImageSize = 16;
+	dtd->mVImageSize = 9;
+	dtd->mYcc420 = 0;
+	dtd->mLimitedToYcc420 = 0;
+
+	switch (code)
+	{
+	case 1: /* 640x480p @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		dtd->mHActive = 640;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 160;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 10;
+		dtd->mHSyncPulseWidth = 96;
+		dtd->mVSyncPulseWidth = 2;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0; /* Active low */
+		dtd->mInterlaced = 0; /* not(progressive_nI) */
+		dtd->mPixelClock = (refreshRate == 59940) ? 2517 : 2520; /* pixel clock depends on refresh rate */
+		break;
+	case 2: /* 720x480p @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 3: /* 720x480p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 138;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 62;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 2700 : 2702;
+		break;
+	case 4: /* 1280x720p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 370;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 110;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 7417 : 7425;
+		break;
+	case 5: /* 1920x1080i @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 59940) ? 7417 : 7425;
+		break;
+	case 6: /* 720(1440)x480i @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 7: /* 720(1440)x480i @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 59940) ? 2700 : 2702;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 8: /* 720(1440)x240p @ 59.826/60.054/59.886/60.115Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 9: /* 720(1440)x240p @ 59.826/60.054/59.886/60.115Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = (refreshRate > 60000) ? 22 : 23;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = (refreshRate > 60000) ? 4 : 5;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock
+		= ((refreshRate == 60054) || refreshRate == 59826) ? 2700
+				: 2702; /*  else 60.115/59.886 Hz */
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 10: /* 2880x480i @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 11: /* 2880x480i @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 552;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 76;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 248;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 59940) ? 5400 : 5405;
+		break;
+	case 12: /* 2880x240p @ 59.826/60.054/59.886/60.115Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 13: /* 2880x240p @ 59.826/60.054/59.886/60.115Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 552;
+		dtd->mVBlanking = (refreshRate > 60000) ? 22 : 23;
+		dtd->mHFrontPorchWidth = 76;
+		dtd->mVFrontPorchWidth = (refreshRate > 60000) ? 4 : 5;
+		dtd->mHSyncPulseWidth = 248;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock
+		= ((refreshRate == 60054) || refreshRate == 59826) ? 5400
+				: 5405; /*  else 60.115/59.886 Hz */
+		break;
+	case 14: /* 1440x480p @ 59.94/60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 15: /* 1440x480p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 32;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 5400 : 5405;
+		break;
+	case 16: /* 1920x1080p @ 59.94/60Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 17: /* 720x576p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 18: /* 720x576p @ 50Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 144;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 12;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 64;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 2700;
+		break;
+	case 19: /* 1280x720p @ 50Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 700;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 440;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 20: /* 1920x1080i @ 50Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 7425;
+		break;
+	case 21: /* 720(1440)x576i @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 22: /* 720(1440)x576i @ 50Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 2700;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 23: /* 720(1440)x288p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 24: /* 720(1440)x288p @ 50Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = (refreshRate == 50000) ? 24
+				: ((refreshRate == 49920) ? 25 : 26);
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = (refreshRate == 50000) ? 2
+				: ((refreshRate == 49920) ? 3 : 4);
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 2700;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 25: /* 2880x576i @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 26: /* 2880x576i @ 50Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 576;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 48;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 252;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 5400;
+		break;
+	case 27: /* 2880x288p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 28: /* 2880x288p @ 50Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 576;
+		dtd->mVBlanking = (refreshRate == 50000) ? 24
+				: ((refreshRate == 49920) ? 25 : 26);
+		dtd->mHFrontPorchWidth = 48;
+		dtd->mVFrontPorchWidth = (refreshRate == 50000) ? 2
+				: ((refreshRate == 49920) ? 3 : 4);
+		dtd->mHSyncPulseWidth = 252;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 5400;
+		break;
+	case 29: /* 1440x576p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 30: /* 1440x576p @ 50Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 128;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 5400;
+		break;
+	case 31: /* 1920x1080p @ 50Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 32: /* 1920x1080p @ 23.976/24Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 830;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 638;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 23976) ? 7417 : 7425;
+		break;
+	case 33: /* 1920x1080p @ 25Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 34: /* 1920x1080p @ 29.97/30Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 29970) ? 7417 : 7425;
+		break;
+	case 35: /* 2880x480p @ 60Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 36: /* 2880x480p @ 60Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 552;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 64;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 248;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 59940) ? 10800 : 10810;
+		break;
+	case 37: /* 2880x576p @ 50Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 38: /* 2880x576p @ 50Hz 16:9 */
+		dtd->mHActive = 2880;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 576;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 48;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 256;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 10800;
+		break;
+	case 39: /* 1920x1080i (1250 total) @ 50Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 384;
+		dtd->mVBlanking = 85;
+		dtd->mHFrontPorchWidth = 32;
+		dtd->mVFrontPorchWidth = 23;
+		dtd->mHSyncPulseWidth = 168;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = 1;
+		dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 7200;
+		break;
+	case 40: /* 1920x1080i @ 100Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 14850;
+		break;
+	case 41: /* 1280x720p @ 100Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 700;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 440;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 42: /* 720x576p @ 100Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 43: /* 720x576p @ 100Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 144;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 12;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 64;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 5400;
+		break;
+	case 44: /* 720(1440)x576i @ 100Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 45: /* 720(1440)x576i @ 100Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 5400;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 46: /* 1920x1080i @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 540;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 119880) ? 14835 : 14850;
+		break;
+	case 47: /* 1280x720p @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 370;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 110;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 119880) ? 14835 : 14850;
+		break;
+	case 48: /* 720x480p @ 119.88/120Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 49: /* 720x480p @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 138;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 62;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 119880) ? 5400 : 5405;
+		break;
+	case 50: /* 720(1440)x480i @ 119.88/120Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 51: /* 720(1440)x480i @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 119880) ? 5400 : 5405;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 52: /* 720X576p @ 200Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 53: /* 720X576p @ 200Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 576;
+		dtd->mHBlanking = 144;
+		dtd->mVBlanking = 49;
+		dtd->mHFrontPorchWidth = 12;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 64;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 10800;
+		break;
+	case 54: /* 720(1440)x576i @ 200Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 55: /* 720(1440)x576i @ 200Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 288;
+		dtd->mHBlanking = 288;
+		dtd->mVBlanking = 24;
+		dtd->mHFrontPorchWidth = 24;
+		dtd->mVFrontPorchWidth = 2;
+		dtd->mHSyncPulseWidth = 126;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = 10800;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 56: /* 720x480p @ 239.76/240Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 57: /* 720x480p @ 239.76/240Hz 16:9 */
+		dtd->mHActive = 720;
+		dtd->mVActive = 480;
+		dtd->mHBlanking = 138;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 16;
+		dtd->mVFrontPorchWidth = 9;
+		dtd->mHSyncPulseWidth = 62;
+		dtd->mVSyncPulseWidth = 6;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 239760) ? 10800 : 10810;
+		break;
+	case 58: /* 720(1440)x480i @ 239.76/240Hz 4:3 */
+		dtd->mHImageSize = 4;
+		dtd->mVImageSize = 3;
+		__fallthrough;
+	case 59: /* 720(1440)x480i @ 239.76/240Hz 16:9 */
+		dtd->mHActive = 1440;
+		dtd->mVActive = 240;
+		dtd->mHBlanking = 276;
+		dtd->mVBlanking = 22;
+		dtd->mHFrontPorchWidth = 38;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 124;
+		dtd->mVSyncPulseWidth = 3;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 0;
+		dtd->mInterlaced = 1;
+		dtd->mPixelClock = (refreshRate == 239760) ? 10800 : 10810;
+		dtd->mPixelRepetitionInput = 1;
+		break;
+	case 60: /* 1280x720p @ 23.97/24Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 2020;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 1760;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 23970) ? 5934 : 5940;
+		break;
+	case 61: /* 1280x720p @ 25Hz 16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 2680;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 2420;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 62: /* 1280x720p @ 29.97/30Hz  16:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 2020;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 1760;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 29970) ? 7417: 7425;
+		break;
+	case 63: /* 1920x1080p @ 119.88/120Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = (refreshRate == 119880) ?  29670: 29700;
+		break;
+	case 64: /* 1920x1080p @ 100Hz 16:9 */
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700;
+		break;
+	case 68:
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 700;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 440;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 69: /* 1280x720p @ 60Hz 21:9 */
+		dtd->mHActive = 1280;
+		dtd->mVActive = 720;
+		dtd->mHBlanking = 370;
+		dtd->mVBlanking = 30;
+		dtd->mHFrontPorchWidth = 110;
+		dtd->mVFrontPorchWidth = 5;
+		dtd->mHSyncPulseWidth = 40;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 7425;
+		break;
+	case 75: //1080p
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 720;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 528;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 76: //1080p
+		dtd->mHActive = 1920;
+		dtd->mVActive = 1080;
+		dtd->mHBlanking = 280;
+		dtd->mVBlanking = 45;
+		dtd->mHFrontPorchWidth = 88;
+		dtd->mVFrontPorchWidth = 4;
+		dtd->mHSyncPulseWidth = 44;
+		dtd->mVSyncPulseWidth = 5;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 14850;
+		break;
+	case 93: /* 4k x 2k, 30Hz, HDMI_VIC 3 */
+		dtd->mCode = 93;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1660;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1276;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 94: /* 4k x 2k, 30Hz, HDMI_VIC 2 */
+		dtd->mCode = 94;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1440;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1056;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+
+	case 95: /* 4k x 2k, HDMI_VIC 1 */
+		dtd->mCode = 95;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 560;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 176;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 96: /* 4k x 2k, HDMI_VIC 2 */
+		dtd->mCode = 96;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1440;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1056;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 97: /* 4k x 2k, HDMI_VIC 1 */
+		dtd->mCode = 97;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 560;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 176;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 98: /* 4k x 2k, HDMI_VIC 1 */
+		dtd->mCode = 98;
+		dtd->mHActive = 4096;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1404;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1020;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 103:
+		dtd->mCode = 103;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1660;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1276;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+	case 104:
+		dtd->mCode = 104;
+		dtd->mHActive = 3840;
+		dtd->mVActive = 2160;
+		dtd->mHBlanking = 1440;
+		dtd->mVBlanking = 90;
+		dtd->mHFrontPorchWidth = 1056;
+		dtd->mVFrontPorchWidth = 8;
+		dtd->mHSyncPulseWidth = 88;
+		dtd->mVSyncPulseWidth = 10;
+		dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+		dtd->mInterlaced = 0;
+		dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+		break;
+    case 105:
+        dtd->mCode = 105;
+        dtd->mHActive = 3840;
+        dtd->mVActive = 2160;
+        dtd->mHBlanking = 560;
+        dtd->mVBlanking = 90;
+        dtd->mHFrontPorchWidth = 176;
+        dtd->mVFrontPorchWidth = 8;
+        dtd->mHSyncPulseWidth = 88;
+        dtd->mVSyncPulseWidth = 10;
+        dtd->mHSyncPolarity = dtd->mVSyncPolarity = 1;
+        dtd->mInterlaced = 0;
+        dtd->mPixelClock = 29700; //(refreshRate == 59940) ? 14835 : 14850;
+        break;
+    default:
+        dtd->mCode = 0;
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+#if !defined(HDMI_EXPERIMENTAL_CLOCKS)
+    if (PhyIsPclkSupported(dtd->mPixelClock) == false)
+    {
+        /* Return success but don't add to list */
+        return PVRSRV_OK;
+    }
+#endif
+    dtd->mValid = true;
+
+    pVideoParams->mDtdIndex = (pVideoParams->mDtdIndex + 1) % HDMI_DTD_MAX;
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdcp.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdcp.c
new file mode 100644
index 000000000000..d8702c478949
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdcp.c
@@ -0,0 +1,96 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+#include "hdmi.h"
+#include "hdcp.h"
+
+
+/*
+
+Steps for configuring HDCP:
+
+1. Read product ID1 to determine if HDCP is present
+2. Select DVI or HDMI mode
+3. Set the data enable, Hsync, and Vsync polarity
+4. Set encryption on
+5. Reset HDCP engine
+6. Configure device private keys
+7. Enable the encryption
+8. SHA-1 calculation if connected to a repeater
+
+*/
+PVRSRV_ERROR HdcpConfigure(HDMI_DEVICE * pvDevice)
+{
+
+    HDMI_CHECKPOINT;
+
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdmi.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdmi.c
new file mode 100644
index 000000000000..a09ba0448536
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/hdmi.c
@@ -0,0 +1,622 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#include "hdmi.h"
+#include "edid.h"
+#include "phy.h"
+#include "video.h"
+#include "hdcp.h"
+#include "dc_external.h"
+#include <linux/delay.h>
+#include "plato.h"
+
+static IMG_UINT32 g_uiNumHDMIInstances;   /* Number of instances */
+
+static void PrintDebugInformation(HDMI_DEVICE * pvDevice)
+{
+    HDMI_DEBUG_PRINT("\nDesign ID: %d\nRev ID: %d\nProduct ID: %d\nProduct ID HDCP: %d\n",
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_DESIGN_ID_OFFSET),
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_REVISION_ID_OFFSET),
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_PRODUCT_ID0_OFFSET),
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_PRODUCT_ID1_OFFSET),
+            HDMI_ID_PRODUCT_ID1_HDCP_START, HDMI_ID_PRODUCT_ID1_HDCP_MASK));
+
+    HDMI_DEBUG_PRINT("\nHDCP Present: %d\nHDMI 1.4: %d\nHDMI 2.0: %d\nPHY Type: %d\n",
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG0_ID_OFFSET),
+            HDMI_ID_CONFIG0_ID_HDCP_START, HDMI_ID_CONFIG0_ID_HDCP_MASK),
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG0_ID_OFFSET),
+            HDMI_ID_CONFIG0_ID_HDMI14_START, HDMI_ID_CONFIG0_ID_HDMI14_MASK),
+        GET_FIELD(HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG1_ID_OFFSET),
+            HDMI_ID_CONFIG1_ID_HDMI20_START, HDMI_ID_CONFIG1_ID_HDMI20_MASK),
+        HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_ID_CONFIG2_ID_OFFSET));
+}
+
+#if defined(HDMI_PDUMP)
+
+static bool HdmiPdumpSetup(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+
+    HDMI_CHECKPOINT;
+
+    /* Set to VIC 16 (1920x1080) */
+    #if DC_DEFAULT_VIDEO_FORMAT == VIDEO_FORMAT_1920_1080p
+    EdidAddDtd(&pvDevice->videoParams, 16, 60000);
+    #elif DC_DEFAULT_VIDEO_FORMAT == VIDEO_FORMAT_1280_720p
+    EdidAddDtd(&pvDevice->videoParams, 4, 60000);
+    #endif
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_INVID0_OFFSET, 0x03);
+
+    /* This maps 1:1 to pdump */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVIDCONF_OFFSET, 0x78);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV0_OFFSET, 0x10);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV0_OFFSET, 0x80);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV1_OFFSET, 0x07);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV0_OFFSET, 0x38);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV1_OFFSET, 0x04);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK0_OFFSET, 0x18);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK1_OFFSET, 0x01);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVBLANK_OFFSET, 0x2D);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY0_OFFSET, 0x58);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY1_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINDELAY_OFFSET, 0x04);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH0_OFFSET, 0x2C);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH1_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINWIDTH_OFFSET, 0x05);
+
+    PhyPowerDown(pvDevice);
+
+    /* pixel packing and 10bit color depth */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_PR_CD_OFFSET, 0x50);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_CONF_OFFSET, 0x64);
+
+    /* Enable HPD sense...user guide says this should be the first step */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_PDDQ_START, HDMI_PHY_CONF0_PDDQ_MASK, 1));
+
+    HDMI_CHECKPOINT;
+
+    /* Audio N and CTS value */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N1_OFFSET, 0xE5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N2_OFFSET, 0x0F);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N3_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS1_OFFSET, 0x19);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS2_OFFSET, 0xD5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS3_OFFSET, 0x02);
+
+    /* Write Phy values through I2C, based on MPLL settings in PHY spec (p 193):
+
+    Pixel Clock: 148.5MHz
+    Pixel repetition: no
+    Color depth: 10-bit
+    TMDS clk src: 185.625MHz
+
+    NOTE: THIS IS BASED ON PHY MODEL E302 (see HDMI reference driver: phy.c:3187)
+
+    */
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x214c);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0211);
+    CHECK_AND_EXIT(status, EXIT);
+    status = PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_TXTERM_OFFSET, 0x0004);
+    CHECK_AND_EXIT(status, EXIT);
+
+    /* Similar to step 3.2.2.g (except we're setting ENHPDRXSENSE as well) */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_TXPWRON_START, HDMI_PHY_CONF0_TXPWRON_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_SVSRET_START, HDMI_PHY_CONF0_SVSRET_MASK, 1));
+
+    /* Wait for PHY to lock */
+    status = PhyWaitLock(pvDevice);
+    #if defined(EMULATOR)
+    status = PVRSRV_OK;
+    #endif
+    CHECK_AND_EXIT(status, EXIT);
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CTRLDUR_OFFSET, 0x0D);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLDUR_OFFSET, 0x20);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLSPAC_OFFSET, 0x01);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH0PREAM_OFFSET, 0x0B);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH1PREAM_OFFSET, 0x16);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH2PREAM_OFFSET, 0x21);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_MC_CLKDIS_OFFSET,
+        SET_FIELD(HDMI_MC_CLKDIS_HDCPCLK_DIS_START, HDMI_MC_CLKDIS_HDCPCLK_DIS_MASK, 1));
+    // This should already be set??
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINWIDTH_OFFSET, 0x05);
+
+EXIT:
+    return (status == PVRSRV_OK) ? true : false;
+}
+
+#endif /* HDMI_PDUMP */
+
+static void HdmiInitInterrupts(HDMI_DEVICE * psDeviceData)
+{
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_PHY_STAT0_OFFSET , 0x3e);
+
+    /* Mute all other interrupts */
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_FC_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_FC_STAT1_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_FC_STAT2_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_AS_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_I2CM_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_CEC_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_VP_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_I2CMPHY_STAT0_OFFSET , 0xff);
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_AHBDMAAUD_STAT0_OFFSET , 0xff);
+}
+static void HdmiEnableHPDInterrupt(HDMI_DEVICE * psDeviceData)
+{
+    /* Enable hot plug interrupts */
+    HDMI_CHECKPOINT;
+
+    /* Clear HPD interrupt by writing 1*/
+    if (IS_BIT_SET(HDMI_READ_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET), HDMI_IH_PHY_STAT0_HPD_START))
+    {
+        HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET,
+            SET_FIELD(HDMI_IH_PHY_STAT0_HPD_START, HDMI_IH_PHY_STAT0_HPD_MASK, 1));
+    }
+
+    HdmiInitInterrupts(psDeviceData);
+
+    HDMI_CHECKPOINT;
+
+    /* Power off */
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_TXPWRON_START, HDMI_PHY_CONF0_TXPWRON_MASK, 0) |
+        SET_FIELD(HDMI_PHY_CONF0_PDDQ_START, HDMI_PHY_CONF0_PDDQ_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_SVSRET_START, HDMI_PHY_CONF0_SVSRET_MASK, 1));
+
+    /* Enable hot plug detection */
+    HDMI_READ_MOD_WRITE(psDeviceData->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1));
+
+    /* Now flip the master switch to unmute */
+    HDMI_WRITE_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_OFFSET , SET_FIELD(HDMI_IH_MUTE_ALL_START, HDMI_IH_MUTE_ALL_MASK, 0));
+
+}
+
+static IMG_BOOL HdmiLISRHandler(void *pvData)
+{
+    HDMI_DEVICE* psDevice = (HDMI_DEVICE*)pvData;
+
+    IMG_UINT32 ui32PhyStat0 = 0;
+    IMG_UINT32 ui32IHPhyStat0 = 0;
+    IMG_UINT32 ui32HPDPolarity = 0;
+    IMG_UINT32 ui32Decode = 0;
+    unsigned long ulFlags = 0;
+
+    HDMI_CHECKPOINT;
+
+    DC_OSSpinLockIRQSave(&psDevice->irqLock, ulFlags);
+
+    /* Mute all interrupts */
+    HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_OFFSET, 0x03);
+
+    ui32Decode = HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_DECODE_OFFSET);
+
+    /* Only support interrupts on PHY (eg HPD) */
+    if (!IS_BIT_SET(ui32Decode, HDMI_IH_DECODE_PHY_START))
+    {
+        HDMI_DEBUG_PRINT("- %s: Unknown interrupt generated, decode: %x\n", __func__,
+            HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_DECODE_OFFSET));
+        goto IRQRestore;
+    }
+
+    ui32PhyStat0 = HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_STAT0_OFFSET);
+    ui32HPDPolarity = GET_FIELD(HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_POL0_OFFSET), HDMI_PHY_POL0_HPD_START, HDMI_PHY_POL0_HPD_MASK);
+    ui32IHPhyStat0 = HDMI_READ_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET);
+
+    HDMI_DEBUG_PRINT("- %s: Hdmi interrupt detected PHYStat0: 0x%x, HPD Polarity: 0x%x, IH Decode: 0x%x, IH PhyStat0: 0x%x",
+					 __func__, ui32PhyStat0, ui32HPDPolarity, ui32Decode, ui32IHPhyStat0);
+
+    /* Check if hot-plugging occurred */
+    if (GET_FIELD(ui32PhyStat0, HDMI_PHY_STAT0_HPD_START, HDMI_PHY_STAT0_HPD_MASK) == IMG_TRUE &&
+        GET_FIELD(ui32IHPhyStat0, HDMI_IH_PHY_STAT0_HPD_START, HDMI_IH_PHY_STAT0_HPD_MASK) == IMG_TRUE)
+    {
+        HDMI_DEBUG_PRINT("- %s: Hot plug detected", __func__);
+
+        /* Flip polarity */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_POL0_OFFSET,
+            SET_FIELD(HDMI_PHY_POL0_HPD_START, HDMI_PHY_POL0_HPD_MASK, !ui32HPDPolarity));
+
+        /* Write 1 to clear interrupt */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET, ui32IHPhyStat0);
+
+        /* schedule work for HDMI/PDP init*/
+        if (psDevice->bHPD == IMG_FALSE)
+        {
+            psDevice->bHPD = IMG_TRUE;
+            schedule_delayed_work(&(psDevice->delayedWork), msecs_to_jiffies(10));
+        }
+
+        /* Mute non-HPD interrupts */
+        HdmiInitInterrupts(psDevice);
+
+        /* Now flip the master switch to unmute */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_MUTE_OFFSET, 0);
+
+    }
+    else
+    {
+        HDMI_DEBUG_PRINT("- %s: Cable unplugged\n", __func__);
+
+        /* Flip polarity */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_PHY_POL0_OFFSET,
+            SET_FIELD(HDMI_PHY_POL0_HPD_START, HDMI_PHY_POL0_HPD_MASK, !ui32HPDPolarity));
+
+        /* Write 1 to clear interrupts */
+        HDMI_WRITE_CORE_REG(psDevice->pvHDMIRegCpuVAddr, HDMI_IH_PHY_STAT0_OFFSET, ui32IHPhyStat0);
+
+        /* Unmute and enable HPD interrupt */
+        HdmiEnableHPDInterrupt(psDevice);
+
+        /* Cable was unplugged */
+        psDevice->bHPD = IMG_FALSE;
+    }
+
+IRQRestore:
+
+    DC_OSSpinUnlockIRQRestore(&psDevice->irqLock, ulFlags);
+
+    return IMG_TRUE;
+}
+
+static void HdmiMISRHandler(struct work_struct *work)
+{
+    PVRSRV_ERROR eError;
+    struct delayed_work *delay_work = to_delayed_work(work);
+    HDMI_DEVICE* psDeviceData = container_of(delay_work, HDMI_DEVICE, delayedWork);
+
+    HDMI_CHECKPOINT;
+
+    if (!psDeviceData->bHPD || !psDeviceData->mInitialized)
+    {
+        return;
+    }
+
+#if defined(HDMI_PDUMP)
+    HdmiPdumpSetup(psDeviceData)
+#endif /* HDMI_PDUMP */
+
+    /* Set VGA (640x480p, 60kHz refresh) in DVI mode */
+    psDeviceData->videoParams.mDtdIndex = 0;
+    EdidAddDtd(&psDeviceData->videoParams, 1, 60000);
+
+    /* Step A: Initialize PHY */
+    if ((eError = PhyInitialize(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize PHY\n", __func__);
+        return;
+    }
+
+    /* Step B: Initialize video/frame composer and DTD for VGA/DVI mode */
+    if ((eError = VideoInitialize(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize Video\n", __func__);
+        return;
+    }
+
+    if ((eError = EdidInitialize(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize EDID\n", __func__);
+        return;
+    }
+
+    /* Step C: Read Sinks E-EDID information */
+    if ((eError = EdidRead(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to read EDID\n", __func__);
+        return;
+    }
+
+    /* Step D: Configure video mode */
+    if ((eError = VideoConfigureMode(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to configure video mode\n", __func__);
+        return;
+    }
+
+    /* No need to configure audio on Plato, skip to step F: Configure InfoFrames */
+    if ((eError = VideoConfigureInfoframes(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize PHY\n", __func__);
+        return;
+    }
+
+    /* Step G: Configure HDCP */
+    if ((eError = HdcpConfigure(psDeviceData)) != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to initialize PHY\n", __func__);
+        return;
+    }
+
+#if defined(HDMI_DEBUG)
+    {
+        DTD * dtd = &psDeviceData->videoParams.mDtdList[psDeviceData->videoParams.mDtdActiveIndex];
+        HDMI_DEBUG_PRINT("- %s: Final HDMI timing configuration: \n\nmHdmi: %d\nmEncodingOut: %d\nmColorRes: %d\nmPixelRep: %d\n \
+\nDTD Info: \n\tmCode: %d\n\tmPixelRep: %d\n\tmPixelClock: %d\n\tmInterlaced: %d\n\tmHActive: %d\n\tmHBlanking: %d\n\tmHBorder: %d\n\t\
+mHImageSize: %d\n\tmHFrontPorchWidth: %d\n\tmHSyncPulseWidth: %d\n\tmHsyncPolarity: %d\n\tmVActive: %d\n\tmVBlanking: %d\n\tmVBorder: %d\n\t\
+mVImageSize: %d\n\tmVFrontPorchWidth: %d\n\tmVSyncPulseWidth: %d\n\tmVsyncPolarity: %d\n\n", __func__, psDeviceData->videoParams.mHdmi,
+psDeviceData->videoParams.mEncodingOut, psDeviceData->videoParams.mColorResolution, psDeviceData->videoParams.mPixelRepetitionFactor,
+dtd->mCode, dtd->mPixelRepetitionInput, dtd->mPixelClock, dtd->mInterlaced, dtd->mHActive, dtd->mHBlanking, dtd->mHBorder,
+dtd->mHImageSize, dtd->mHFrontPorchWidth, dtd->mHSyncPulseWidth, dtd->mHSyncPolarity, dtd->mVActive, dtd->mVBlanking, dtd->mVBorder,
+dtd->mVImageSize, dtd->mVFrontPorchWidth, dtd->mVSyncPulseWidth, dtd->mVSyncPolarity);
+    }
+#endif
+
+    if (psDeviceData->pfnPDPInitialize)
+    {
+        eError = psDeviceData->pfnPDPInitialize(&(psDeviceData->videoParams),
+                                                  psDeviceData->ui32Instance);
+    }
+    else
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to get PDP<->Hdmi interface\n", __func__);
+        eError = PVRSRV_ERROR_INIT_FAILURE;
+    }
+
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: PDP initialization failed \n", __func__);
+    }
+
+}
+
+PVRSRV_ERROR HDMIDrvInit(void *pvDevice, HDMI_DEVICE **ppsDeviceData)
+{
+    HDMI_DEVICE *psDeviceData;
+    PVRSRV_ERROR eError;
+
+    if (ppsDeviceData == NULL)
+    {
+        return PVRSRV_ERROR_INVALID_PARAMS;
+    }
+
+    DC_OSSetDrvName(DRVNAME);
+
+    HDMI_CHECKPOINT;
+
+    #if defined(VIRTUAL_PLATFORM)
+    HDMI_ERROR_PRINT("-%s: HDMI not supported on VP\n", __func__);
+    #endif
+
+    psDeviceData = DC_OSCallocMem(sizeof(*psDeviceData));
+    if (psDeviceData == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to allocate device data (size %d)\n", __func__, sizeof(*psDeviceData));
+        eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+        goto FAIL_DEV_ALLOC;
+    }
+
+    psDeviceData->pvDevice = pvDevice;
+    psDeviceData->bHPD = IMG_FALSE;
+    psDeviceData->ui32Instance = g_uiNumHDMIInstances++;
+
+    DC_OSSpinLockCreate(&psDeviceData->irqLock);
+
+    eError = DC_OSPVRServicesConnectionOpen(&psDeviceData->hPVRServicesConnection);
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to open connection to PVR Services (%d)\n", __func__, eError);
+        goto FAIL_CONN_CLOSE;
+    }
+
+    eError = DC_OSPVRServicesSetupFuncs(psDeviceData->hPVRServicesConnection, &psDeviceData->sPVRServicesFuncs);
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to setup PVR Services function table (%d)\n", __func__, eError);
+        goto FAIL_CONN_CLOSE;
+    }
+
+    HDMI_DEBUG_PRINT("- %s: Installing HDMI interrupt handler in services\n", __func__);
+
+    eError = psDeviceData->sPVRServicesFuncs.pfnSysInstallDeviceLISR(pvDevice,
+        PLATO_IRQ_HDMI,
+        DRVNAME,
+        HdmiLISRHandler,
+        psDeviceData,
+        &psDeviceData->hLISRData);
+    if (eError != PVRSRV_OK)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to install system Device LISR (%s) unit %u\n",
+        __func__, psDeviceData->sPVRServicesFuncs.pfnGetErrorString(eError),
+        psDeviceData->ui32Instance);
+        goto FAIL_DETACH_ISR;
+    }
+
+    INIT_DELAYED_WORK(&psDeviceData->delayedWork, HdmiMISRHandler);
+
+    HDMI_DEBUG_PRINT("- %s: Reserving HDMI Core registers\n", __func__);
+
+    /* Reserve and map the HDMI registers */
+    psDeviceData->sHDMIRegCpuPAddr.uiAddr =
+        DC_OSAddrRangeStart(psDeviceData->pvDevice, SYS_PLATO_REG_PCI_BASENUM) + SYS_PLATO_REG_HDMI_OFFSET;
+
+    if (DC_OSRequestAddrRegion(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE, DRVNAME) == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to reserve the HDMI registers\n", __func__);
+        psDeviceData->sHDMIRegCpuPAddr.uiAddr = 0;
+        eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+        goto FAIL_CONN_CLOSE;
+    }
+
+    psDeviceData->pvHDMIRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE);
+    if (psDeviceData->pvHDMIRegCpuVAddr == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to map HDMI registers\n", __func__);
+        eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+        goto RELEASE_MEM_REGION;
+    }
+
+    /* Check for invalid register space */
+    if (HDMI_READ_CORE_REG(psDeviceData->pvHDMIRegCpuVAddr, HDMI_ID_DESIGN_ID_OFFSET) == 0xFFFFFFFF)
+    {
+        HDMI_DEBUG_PRINT("- %s: Invalid HDMI register space", __func__);
+        eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+        goto RELEASE_MEM_REGION;
+    }
+
+    /*
+     * Map Plato TOP registers - needed for pixel clock setup.
+     */
+    psDeviceData->sTopRegCpuPAddr.uiAddr =
+        DC_OSAddrRangeStart(psDeviceData->pvDevice, SYS_PLATO_REG_PCI_BASENUM) + SYS_PLATO_REG_CHIP_LEVEL_OFFSET;
+
+    psDeviceData->pvTopRegCpuVAddr = DC_OSMapPhysAddr(psDeviceData->sTopRegCpuPAddr, SYS_PLATO_REG_CHIP_LEVEL_SIZE);
+    if (psDeviceData->pvTopRegCpuVAddr == NULL)
+    {
+        HDMI_ERROR_PRINT("- %s: Failed to map TOP registers\n", __func__);
+        eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+        goto RELEASE_MEM_REGION;
+    }
+
+    PrintDebugInformation(psDeviceData);
+
+    HdmiEnableHPDInterrupt(psDeviceData);
+
+    *ppsDeviceData = psDeviceData;
+
+    HDMI_CHECKPOINT;
+
+    psDeviceData->mInitialized = IMG_TRUE;
+
+    return PVRSRV_OK;
+
+RELEASE_MEM_REGION:
+    DC_OSReleaseAddrRegion(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE);
+FAIL_DETACH_ISR:
+    /* Uninstall the ISR as we've failed the later stage of initialisation */
+    HDMI_ERROR_PRINT("- %s: Uninstalling Services ISR, Unit %u\n", __func__,
+                     psDeviceData->ui32Instance);
+    g_uiNumHDMIInstances--;
+    if (psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR != NULL)
+    {
+        psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+        DC_OSSpinLockDestroy(&psDeviceData->irqLock);
+    }
+    else
+    {
+        HDMI_ERROR_PRINT("- %s: SysUninstallDeviceLISR handle not present. Power-cycle system to reset\n", __func__);
+    }
+FAIL_CONN_CLOSE:
+    DC_OSFreeMem(psDeviceData);
+
+FAIL_DEV_ALLOC:
+
+    return eError;
+}
+
+void HDMIDrvDeInit(HDMI_DEVICE *psDeviceData)
+{
+    HDMI_CHECKPOINT;
+    if (psDeviceData)
+    {
+
+        HDMI_DEBUG_PRINT(" - %s: DeInit unit %u\n", __func__,
+                         psDeviceData->ui32Instance);
+
+        /* Unregister the LISR - we have to disable and flush the deferred
+         * worker threads here too
+         */
+        if (psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR != NULL)
+        {
+            HDMI_DEBUG_PRINT(" - %s: Uninstall Services LISR unit %u\n",
+                             __func__, psDeviceData->ui32Instance);
+            psDeviceData->sPVRServicesFuncs.pfnSysUninstallDeviceLISR(psDeviceData->hLISRData);
+            HDMI_DEBUG_PRINT(" - %s: Cleanup delayed work queues, unit %u\n",
+                             __func__, psDeviceData->ui32Instance);
+
+            (void) flush_delayed_work(&psDeviceData->delayedWork);
+            (void) cancel_delayed_work_sync(&psDeviceData->delayedWork);
+
+            DC_OSSpinLockDestroy(&psDeviceData->irqLock);
+        }
+
+        DC_OSUnmapPhysAddr(psDeviceData->pvHDMIRegCpuVAddr, SYS_PLATO_REG_HDMI_SIZE);
+        DC_OSReleaseAddrRegion(psDeviceData->sHDMIRegCpuPAddr, SYS_PLATO_REG_HDMI_SIZE);
+        DC_OSPVRServicesConnectionClose(psDeviceData->hPVRServicesConnection);
+        HDMI_CHECKPOINT;
+        DC_OSFreeMem(psDeviceData);
+        g_uiNumHDMIInstances--;
+    }
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/phy.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/phy.c
new file mode 100644
index 000000000000..84988b718451
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/phy.c
@@ -0,0 +1,761 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "phy.h"
+#include <linux/delay.h>
+
+/*
+ * See tables A.1 and B.1 in PHY datasheet
+ */
+#define MinPhyPixelClock    2500
+#define MaxPhyPixelClock    59400
+
+static const struct PhyGenericConfig
+{
+    IMG_UINT32  PixClock;
+    IMG_UINT32  OPMODE_PLLCFG;
+    IMG_UINT32  PLLCURRCTRL;
+    IMG_UINT32  PLLGMPCTRL;
+} gsPhyConfig_8bit[] =
+{
+    // PixelClock.    Reg: 0x06    0x10a   0x15
+    { MinPhyPixelClock,   0x00B3, 0x0018, 0x0000 },
+    {  4950,              0x0072, 0x0028, 0x0001 },
+    {  9450,              0x0051, 0x0038, 0x0002 },
+    { 18562,              0x0040, 0x0038, 0x0003 },
+    { 34850,              0x1A40, 0x0018, 0x0003 },
+    { MaxPhyPixelClock+1, 0x1A40, 0x0018, 0x0003 },    // last entry is invalid and must be > Max allowed
+};
+static const IMG_UINT32 NumPhyConfigs = ARRAY_SIZE(gsPhyConfig_8bit);
+
+#define PHY_POLL(base, offset, mask, error) \
+    { \
+        IMG_UINT16 timeout = 0; \
+        while (!(HDMI_READ_CORE_REG(base, offset) & mask) && \
+            timeout < HDMI_PHY_TIMEOUT_MS) \
+        { \
+            DC_OSDelayus(HDMI_PHY_POLL_INTERVAL_MS * 1000); \
+            timeout += HDMI_PHY_POLL_INTERVAL_MS; \
+        } \
+        if (timeout == HDMI_PHY_TIMEOUT_MS) \
+        { \
+            HDMI_ERROR_PRINT("- %s: PHY timed out polling on 0x%x in register offset 0x%x\n", __func__, mask, offset); \
+            goto error; \
+        } \
+    }
+
+bool PhyIsPclkSupported(IMG_UINT16 pixelClock)
+{
+    return ((pixelClock >= MinPhyPixelClock) && (pixelClock <= MaxPhyPixelClock));
+}
+
+#if defined(READ_HDMI_PHY_REGISTERS)
+static IMG_UINT16 PhyI2CRead(IMG_CPU_VIRTADDR base, IMG_UINT8 addr)
+{
+    IMG_UINT16 data;
+
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_ADDRESS_OFFSET, addr);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_OPERATION_OFFSET,
+        SET_FIELD(HDMI_PHY_I2CM_OPERATION_RD_START, HDMI_PHY_I2CM_OPERATION_RD_MASK, 1));
+    /* Wait for done indication */
+    //PHY_POLL(base, HDMI_IH_I2CMPHY_STAT0_OFFSET, HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_MASK, ERROR);
+    mdelay(10);
+    /* read the data registers */
+    data  = (HDMI_READ_CORE_REG(base, HDMI_PHY_I2CM_DATAI_1_OFFSET) & 0xFF) << 8;
+    data |= (HDMI_READ_CORE_REG(base, HDMI_PHY_I2CM_DATAI_0_OFFSET) & 0xFF);
+
+    return data;
+}
+#endif // READ_HDMI_PHY_REGISTERS
+
+PVRSRV_ERROR PhyI2CWrite(IMG_CPU_VIRTADDR base, IMG_UINT8 addr, IMG_UINT16 data)
+{
+    HDMI_CHECKPOINT;
+
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_ADDRESS_OFFSET, addr);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_DATAO_1_OFFSET, (data >> 8));
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_DATAO_0_OFFSET, data & 0xFF);
+    HDMI_WRITE_CORE_REG(base, HDMI_PHY_I2CM_OPERATION_OFFSET,
+        SET_FIELD(HDMI_PHY_I2CM_OPERATION_WR_START, HDMI_PHY_I2CM_OPERATION_WR_MASK, 1));
+
+    /* Wait for done interrupt */
+    PHY_POLL(base, HDMI_IH_I2CMPHY_STAT0_OFFSET, HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_MASK, ERROR);
+
+    HDMI_WRITE_CORE_REG(base, HDMI_IH_I2CMPHY_STAT0_OFFSET, 0x3);
+
+    return PVRSRV_OK;
+
+ERROR:
+    return PVRSRV_ERROR_TIMEOUT;
+}
+
+/*
+Configure PHY based on pixel clock, color resolution, pixel repetition, and PHY model
+
+NOTE: This assumes PHY model TSMC 28-nm HPM/ 1.8V
+*/
+void PhyConfigureMode(HDMI_DEVICE * pvDevice)
+{
+    IMG_UINT16 pixelClock;
+    IMG_UINT8 colorRes;
+
+    HDMI_CHECKPOINT;
+
+    /* Init slave address */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+
+    pixelClock = pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex].mPixelClock;
+    colorRes = pvDevice->videoParams.mColorResolution;
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET,
+        SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_PDDQ_START, HDMI_PHY_CONF0_PDDQ_MASK, 1));
+
+    //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0000, 0x13); /* PLLPHBYCTRL */
+    //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCLKBISTPHASE_OFFSET, 0x0006);
+    /* RESISTANCE TERM 133Ohm Cfg  */
+    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_TXTERM_OFFSET, 0x0004); /* TXTERM */
+    /* REMOVE CLK TERM */
+    //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKCALCTRL_OFFSET, 0x8000); /* CKCALCTRL */
+
+    HDMI_DEBUG_PRINT(" - %s: pixel Clock: %d, colorResolution: %d\n", __func__, pixelClock, colorRes);
+
+    if ((pixelClock < MinPhyPixelClock) || (pixelClock > MaxPhyPixelClock))
+    {
+        HDMI_ERROR_PRINT("- %s: pixel clock %d out of range [%d,%d]", __func__, pixelClock, MinPhyPixelClock, MaxPhyPixelClock);
+        return;
+    }
+
+    // For 8-bit color formats look up the register values in table based on PHY datasheet.
+    if (colorRes == 8)
+    {
+        IMG_UINT32 CKSYMTXCTRL;
+        IMG_UINT32 VLEVCTRL;
+        IMG_UINT32 ipix;
+        for (ipix  = 0; ipix < NumPhyConfigs-1; ++ipix)
+        {
+            if (gsPhyConfig_8bit[ipix+1].PixClock > pixelClock)
+                break;
+        }
+        if (ipix == NumPhyConfigs-1)
+        {
+            HDMI_ERROR_PRINT("- %s: pixel clock %d > MaxPhyPixelClock %d", __func__, pixelClock, MaxPhyPixelClock);
+            return;
+        }
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, gsPhyConfig_8bit[ipix].OPMODE_PLLCFG);
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   gsPhyConfig_8bit[ipix].PLLCURRCTRL);
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    gsPhyConfig_8bit[ipix].PLLGMPCTRL);
+
+        // Clock Symbol and Transmitter Control Register 0x09
+        if (pixelClock*10 < 29700000)       // 2.97GHz
+            CKSYMTXCTRL = 0x8009;
+        else if (pixelClock*10 < 34000000)  // 3.4GHz
+            CKSYMTXCTRL = 0x8019;
+        else
+            CKSYMTXCTRL = 0x8029;
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, CKSYMTXCTRL);
+
+        // Voltage Level Control Register 0x0E
+        if (pixelClock*10 < 14850000)       // 1.485GHz
+            VLEVCTRL = 0x0232;
+        else if (pixelClock*10 < 22200000)  // 2.22GHz
+            VLEVCTRL = 0x0230;
+        else if (pixelClock*10 < 34000000)  // 3.4GHz
+            VLEVCTRL = 0x0273;
+        else
+            VLEVCTRL = 0x014A;
+        PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, VLEVCTRL);
+    }
+    else    // old code for > 8-bit. Needs to be modified as for 8-bit.
+    switch (pixelClock)
+    {
+        // See dwc_hdmi20_tx_ns_6gbps_tsmc28hpm18_databook.pdf
+		case 2520:
+			switch (colorRes)
+			{
+				case 8:
+					/* PLL/MPLL Cfg */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x00b3);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0018); /* CURRCTRL */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0000); /* GMPCTRL */
+					break;
+				case 10: /* TMDS = 31.50MHz */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2153);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0018);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0000);
+					break;
+				case 12: /* TMDS = 37.80MHz */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x40F3);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0018);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0000);
+					break;
+				case 16: /* TMDS = 50.40MHz */
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x60B2);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0028);
+					PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0001);
+					break;
+				default:
+					HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+					break;
+			}
+			/* PREEMP Cgf 0.00 */
+			//PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, 0x8009); /* CKSYMTXCTRL */
+			/* TX/CK LVL 10 */
+			//PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0251); /* VLEVCTRL */
+			break;
+        case 2700:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x00B3);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0018);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0000);
+	                break;
+	            case 10: /* TMDS = 33.75MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2153);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0018);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0000);
+	                break;
+	            case 12: /* TMDS = 40.50MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40F3);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0018);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0000);
+	                break;
+	            case 16: /* TMDS = 54MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x60B2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            default:
+	                HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+	                break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 5040:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 63MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 75.60MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40A2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 16: /* TMDS = 100.80MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+	            default:
+	                HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+	                break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 5400:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 67.50MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 81MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40A2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 16: /* TMDS = 108MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+	            default:
+	                HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+	                break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 5940:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 74.25MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 89.10MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x40A2);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 16: /* TMDS = 118.80MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        case 7200:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x0072);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 10: /* TMDS = 90MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x2142);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0028);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0001);
+	                break;
+	            case 12: /* TMDS = 108MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+	            case 16: /* TMDS = 144MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET,  0x6071);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,    0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,     0x0002);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x8009, 0x09);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, 0x0251, 0x0E);
+            break;
+        /* 74.25 MHz pixel clock (720p)*/
+        case 7425:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0072);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0028);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0001);
+                    break;
+                case 10: /* TMDS = 92.812MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0002);
+                    break;
+                case 12: /* TMDS = 111.375MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0002);
+                    break;
+                case 16: /* TMDS = 148.5MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6071);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0002);
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, 0x8009);
+            //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0251);
+            break;
+
+        case 10080:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 126MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 12: /* TMDS = 151.20MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 16: /* TMDS = 201.60MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 10100:
+        case 10225:
+        case 10650:
+        case 10800:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                //PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 135MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 12: /* TMDS = 162MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 16: /* TMDS = 216MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 11880:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 148.50MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 12: /* TMDS = 178.20MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4061);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 16: /* TMDS = 237.60MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+
+        case 14400:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+	            case 10: /* TMDS = 180MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x2145);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 12: /* TMDS = 216MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4064);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+	            case 16: /* TMDS = 288MHz */
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0211);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 14625:
+            switch (colorRes)
+            {
+	            case 8:
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009);
+	                PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+	                break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 14850:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+                    break;
+                case 10: /* TMDS = 185.62MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x214C);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0211);
+                    break;
+                case 12: /* TMDS = 222.75MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x4064);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0211);
+                    break;
+                case 16: /* TMDS = 297MHz */
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x6050);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET, 0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET, 0x0003);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET, 0x0273);
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET, 0x8009);
+            break;
+        case 15400:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0051);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0002);
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251);
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        case 24150:
+            switch (colorRes)
+            {
+                case 8:
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_OPMODE_PLLCFG_OFFSET, 0x0040); // section 11.2.7 and table B.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLCURRCTRL_OFFSET,   0x0038); // section 11.2.17 and table B.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_PLLGMPCTRL_OFFSET,    0x0003); // section 11.2.20 and table B.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_VLEVCTRL_OFFSET,      0x0251); // section 11.2.15 and table A.1
+                    PhyI2CWrite(pvDevice->pvHDMIRegCpuVAddr, PHY_CKSYMTXCTRL_OFFSET,   0x8009); // section 11.2.10
+                    break;
+                default:
+                    HDMI_ERROR_PRINT("- %s: Color depth not supported (%d)", __func__, colorRes);
+                    break;
+            }
+            break;
+        default:
+            HDMI_ERROR_PRINT("- %s: \n\n\n*****Unsupported pixel clock ******\n\n\n!", __func__);
+            break;
+    }
+
+    {
+        IMG_UINT32 value = HDMI_READ_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET);
+        value = SET_FIELD(HDMI_PHY_CONF0_SELDATAENPOL_START, HDMI_PHY_CONF0_SELDATAENPOL_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_ENHPDRXSENSE_START, HDMI_PHY_CONF0_ENHPDRXSENSE_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_TXPWRON_START, HDMI_PHY_CONF0_TXPWRON_MASK, 1) |
+        SET_FIELD(HDMI_PHY_CONF0_SVSRET_START, HDMI_PHY_CONF0_SVSRET_MASK, 1);
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_CONF0_OFFSET, value);
+    }
+
+#if defined(READ_HDMI_PHY_REGISTERS)
+    {
+        static IMG_UINT8 addresses[] =
+        {
+            PHY_OPMODE_PLLCFG_OFFSET,
+            PHY_PLLCURRCTRL_OFFSET,
+            PHY_PLLGMPCTRL_OFFSET,
+            PHY_VLEVCTRL_OFFSET,
+            PHY_CKSYMTXCTRL_OFFSET,
+        };
+        IMG_UINT16 data;
+        IMG_UINT32 n;
+        for (n = 0; n < sizeof(addresses); ++n)
+        {
+            data = PhyI2CRead(pvDevice->pvHDMIRegCpuVAddr, addresses[n]);
+            printk(KERN_INFO "HDMI I2 addr 0x%02x = 0x%04x\n", addresses[n], data);
+        }
+    }
+#endif // READ_HDMI_PHY_REGISTERS
+}
+
+void PhyReset(HDMI_DEVICE * pvDevice, IMG_UINT8 value)
+{
+    /* Handle different types of PHY here... */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_MC_PHYRSTZ_OFFSET, value & 0x1);
+    msleep(HDMI_PHY_RESET_TIME_MS);
+}
+
+PVRSRV_ERROR PhyPowerDown(HDMI_DEVICE * pvDevice)
+{
+    /* For HDMI TX 1.4 PHY, power down by placing PHY in reset */
+    PhyReset(pvDevice, 0);
+    PhyReset(pvDevice, 1);
+    PhyReset(pvDevice, 0);
+    PhyReset(pvDevice, 1);
+    return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PhyWaitLock(HDMI_DEVICE * pvDevice)
+{
+    PHY_POLL(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_STAT0_OFFSET, HDMI_PHY_STAT0_TX_PHY_LOCK_MASK, ERROR);
+    return PVRSRV_OK;
+
+ERROR:
+    return PVRSRV_ERROR_TIMEOUT;
+}
+
+PVRSRV_ERROR PhyInitialize(HDMI_DEVICE * pvDevice)
+{
+
+    /* Init slave address */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_PHY_I2CM_SLAVE_OFFSET, HDMI_PHY_SLAVE_ADDRESS);
+
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/video.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/video.c
new file mode 100644
index 000000000000..23fb7f36bed2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/common/video.c
@@ -0,0 +1,868 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+
+#include "video.h"
+#include "phy.h"
+#include "plato_top_regs.h"
+#include <linux/delay.h>
+
+//
+// Pixel clock and core clock are driven by same PLL.
+// For a given pixel clock find a list of core clocks that match.
+// Print these out for the user's information.
+static void FindExactCoreClocks(HDMI_DEVICE* pvDevice, IMG_UINT32 ui32PixelClock)
+{
+    // divide by 3 up to 48, or 4 up to 64 are also valid, but VideoEnablePixelClock doesn't use this.
+    #define VALID_PIXDIV(d) (((d) <= 16) || (((d) <= 32) && !((d) & 1)))
+    const IMG_UINT32 MinPLL =  700000000;  // 700MHz
+    const IMG_UINT32 MaxPLL = 1650000000;  // 1.65GHz
+    const IMG_UINT32 MinCoreClk = 396000000;  // 396MHz
+    const IMG_UINT32 MaxCoreClk = 600000000;  // 600MHz
+    const IMG_UINT32 mindiv = MinPLL / ui32PixelClock;
+    const IMG_UINT32 maxdiv = MaxPLL / ui32PixelClock;
+    #define MaxStoredClocks 10
+    IMG_UINT32 ValidPixelClocks[MaxStoredClocks+1] = { 0 };
+    IMG_UINT32 vpi = 0;
+    IMG_UINT32 pixdiv;
+
+    for (pixdiv = mindiv; pixdiv <= maxdiv; ++pixdiv)
+    {
+        IMG_UINT32 pll;
+        IMG_UINT32 d;
+
+        if (!VALID_PIXDIV(pixdiv))
+            continue;
+        pll = ui32PixelClock * pixdiv;
+        if (pll < MinPLL)
+            continue;
+        if (pll > MaxPLL)
+            break;
+        // find nearest GPU clock to current one for this PLL speed
+        for (d = 1; d <= 4; ++d)
+        {
+            IMG_UINT32 gpuclk = pll / d;
+            if (gpuclk < MinCoreClk)
+                break;
+            if (gpuclk > MaxCoreClk)
+                continue;
+            if (vpi < MaxStoredClocks)
+            {
+                // simple insertion sort
+                IMG_UINT32 i, j;
+                for (i = 0; i < vpi; ++i)
+                {
+                    if (ValidPixelClocks[i] == gpuclk)
+                        break;      // already present
+                    if (gpuclk < ValidPixelClocks[i])
+                    {
+                        for (j = vpi++; j > i; --j)
+                            ValidPixelClocks[j] = ValidPixelClocks[j-1];
+                        ValidPixelClocks[i] = gpuclk;
+                        break;
+                    }
+                }
+                if (i == vpi)
+                    ValidPixelClocks[vpi++] = gpuclk;
+            }
+        }
+    }
+    if (vpi > 0)
+    {
+        IMG_UINT32 i;
+        HDMI_LOG_ALWAYS(" Current core clock %u.%03uMHz: core clocks to achieve exact pixel clock %ukHz:\n",
+                                pvDevice->ui32CoreClockSpeed/1000000, (pvDevice->ui32CoreClockSpeed%1000000)/1000, ui32PixelClock/1000);
+        for (i = 0; i < vpi; ++i)
+            HDMI_LOG_ALWAYS("\t%u\n", ValidPixelClocks[i]);
+    }
+}
+
+
+static int SetPlatoPDPClockDivider(HDMI_DEVICE* pvDevice, IMG_UINT32 pll_clock, IMG_UINT32 pixel_clock)
+{
+    // max can probably go higher if we need to in future. We need about 300MHz for 4Kx2K@30 monitors
+    const IMG_UINT32 MaxPDPClk = (450*1000*1000)+1;
+    IMG_UINT32 best_clk = MaxPDPClk;
+    IMG_UINT32 reg_value = 0;
+    IMG_UINT32 min_clk;
+    IMG_UINT32 d0, d1;
+
+    // we always want PDP core to be >= PLATO_MIN_PDP_CLOCK_SPEED
+    min_clk = (pixel_clock > PLATO_MIN_PDP_CLOCK_SPEED) ? pixel_clock : PLATO_MIN_PDP_CLOCK_SPEED;
+
+    // we have 2x 2-bit dividers for Plato PDP core clock.
+    // start from largest dividers to get to lowest PDP core clock >= pixel clock.
+    for (d0 = 4; d0 >= 1; --d0)
+    {
+        for (d1 = d0; d1 >= 1; --d1)
+        {
+            IMG_UINT32 pdp_clk = pll_clock / (d0*d1);
+            if (pdp_clk >= best_clk)
+                break;	// next divider in loop only makes it larger again
+            if (pdp_clk >= min_clk)
+            {
+                reg_value = ((d0-1) << PLATO_CR_PDPV0_DIV_0_SHIFT) | ((d1-1) << PLATO_CR_PDPV1_DIV_0_SHIFT);
+                best_clk = pdp_clk;
+            }
+        }
+    }
+
+    if (best_clk < MaxPDPClk)
+    {
+        // turn off the clock first
+        OSWriteHWReg32(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_PDP_CLK_CTRL, 0);
+        TOP_REG_POLL(pvDevice->pvTopRegCpuVAddr,   PLATO_TOP_CR_PDP_CLK_CTRL, 0, 0);
+
+        reg_value |= (1 << PLATO_CR_PDPG_GATE_EN_SHIFT);
+        OSWriteHWReg32(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_PDP_CLK_CTRL, reg_value);
+        TOP_REG_POLL(pvDevice->pvTopRegCpuVAddr,   PLATO_TOP_CR_PDP_CLK_CTRL, reg_value, reg_value);
+
+        HDMI_LOG_ALWAYS(" PDP clock speed set to %u\n", best_clk);
+    }
+    else
+        HDMI_LOG_ALWAYS(" ERROR: Cannot set PDP core clock for PLL %u, pixel clock %u\n", pll_clock, pixel_clock);
+
+    return (best_clk < MaxPDPClk);
+}
+
+/*
+ * Pixel Clock setup
+ */
+static void VideoEnablePixelClock(HDMI_DEVICE * pvDevice, IMG_UINT32 ui32PixelClock)
+{
+    IMG_UINT32 ui32Div0;
+    IMG_UINT32 ui32Div1;
+    IMG_UINT32 ui32RegVal;
+    IMG_UINT32 ui32PLLClock = pvDevice->ui32PLLClockSpeed;
+    ui32PixelClock *= 10000;
+
+    /*
+     * Obtain divisor, round to nearest.
+     */
+    ui32Div1 = (ui32PLLClock + ui32PixelClock / 2) / ui32PixelClock;
+
+    if (ui32Div1 <= 32)
+    {
+        if (ui32Div1 < 17)
+        {
+            ui32Div0 = 0;
+            ui32Div1--;
+        }
+        else
+        {
+            ui32Div0 = 1;
+            ui32Div1 /= 2;
+            ui32Div1--;
+        }
+    }
+    else
+    {
+        HDMI_DEBUG_PRINT("- %s: Cannot correctly set up dividers to set pixel clock to requested value! Setting dividers to maximum values ", __func__);
+        ui32Div0 = 1;
+        ui32Div1 = 15;
+    }
+
+    // now we know the actual pixel clock ensure the PDP core clock is fast enough.
+    SetPlatoPDPClockDivider(pvDevice, ui32PLLClock, ui32PLLClock / ((ui32Div1+1) * (ui32Div0+1)));
+
+    // and set the pixel clock after the PDP core clock.
+    ui32RegVal = (PLATO_CR_HDMIG_GATE_EN_MASK | (ui32Div1 << PLATO_CR_HDMIV1_DIV_0_SHIFT) | (ui32Div0 << PLATO_CR_HDMIV0_DIV_0_SHIFT));
+
+    OSWriteHWReg32(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32RegVal);
+    TOP_REG_POLL(pvDevice->pvTopRegCpuVAddr, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32RegVal, ui32RegVal);
+
+    HDMI_DEBUG_PRINT("- %s: Setting dividers %d,%d for requested PIXEL clock %ukHz\n", __func__,
+                                            ui32Div0+1, ui32Div1+1, ui32PixelClock/1000);
+    {
+        // Always print out some log information about clock speeds.
+        // Consider match exact if within 10000Hz (EDID is in units of 10000Hz)
+        IMG_UINT32 diff = ui32PixelClock - ui32PLLClock/((ui32Div0+1)*(ui32Div1+1));
+        if ((diff < 10000) || (diff > 0xFFFFD8F0))
+            HDMI_LOG_ALWAYS(" Exact pixel clock of %ukHz achieved.\n", ui32PixelClock/1000);
+        else
+            HDMI_LOG_ALWAYS(" Requested pixel clock %ukHz, actual pixel clock %ukHz.",
+                                                ui32PixelClock/1000, ui32PLLClock/(1000*(ui32Div0+1)*(ui32Div1+1)));
+    }
+
+    FindExactCoreClocks(pvDevice, ui32PixelClock);
+}
+
+/*
+
+Sets up Video Packetizer registers
+
+*/
+static bool VideoConfigureVP(HDMI_DEVICE * pvDevice)
+{
+    VIDEO_PARAMS * pParams = &pvDevice->videoParams;
+    IMG_UINT8 regValue = 0;
+    IMG_UINT8 outputSelect = 0;
+    IMG_UINT8 remapSize = 0;
+    IMG_UINT8 colorDepth = 0;
+
+    HDMI_CHECKPOINT;
+
+    if (pParams->mEncodingOut == ENCODING_RGB ||
+        pParams->mEncodingOut == ENCODING_YCC444 ||
+        pParams->mEncodingOut == ENCODING_YCC420)
+    {
+        if (pParams->mColorResolution == 0)
+            outputSelect = 3;
+        else if (pParams->mColorResolution == 8)
+        {
+            colorDepth = 4;
+            outputSelect = 3;
+        }
+        else if (pParams->mColorResolution == 10)
+            colorDepth = 5;
+        else if (pParams->mColorResolution == 12)
+            colorDepth = 6;
+        else if (pParams->mColorResolution == 16)
+            colorDepth = 7;
+        else
+        {
+            HDMI_ERROR_PRINT("- %s: Found unsupported color resolution (%d)", __func__, pParams->mColorResolution);
+            return false;
+        }
+    }
+    else if (pParams->mEncodingOut == ENCODING_YCC422)
+    {
+        if ((pParams->mColorResolution == 8)
+                || (pParams->mColorResolution == 0))
+            remapSize = 0;
+        else if (pParams->mColorResolution == 10)
+            remapSize = 1;
+        else if (pParams->mColorResolution == 12)
+            remapSize = 2;
+        else
+        {
+            HDMI_ERROR_PRINT("- %s: Invalid color remap size: %d", __func__, pParams->mColorResolution);
+            return false;
+        }
+        outputSelect = 1;
+    }
+    else
+    {
+        HDMI_ERROR_PRINT("- %s: Invalid output encoding type: %d", __func__, pParams->mEncodingOut);
+        return false;
+    }
+
+    HDMI_DEBUG_PRINT(" - %s: outputSelect: %d, colorDepth: %d\n", __func__, outputSelect, colorDepth);
+
+    regValue = SET_FIELD(HDMI_VP_PR_CD_COLOR_DEPTH_START,
+        HDMI_VP_PR_CD_COLOR_DEPTH_MASK,
+        colorDepth) |
+    SET_FIELD(HDMI_VP_PR_CD_DESIRED_PR_FACTOR_START,
+        HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK,
+        pParams->mPixelRepetitionFactor);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_PR_CD_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_VP_STUFF_IDEFAULT_PHASE_START,
+        HDMI_VP_STUFF_IDEFAULT_PHASE_MASK,
+        pParams->mPixelPackingDefaultPhase);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_STUFF_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_VP_REMAP_YCC422_SIZE_START,
+        HDMI_VP_REMAP_YCC422_SIZE_MASK,
+        remapSize);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_REMAP_OFFSET, regValue);
+
+    /* Grabbed from reference driver */
+    regValue = 0;
+    if (outputSelect == 0)
+    {
+        /* pixel packing */
+        regValue = SET_FIELD(HDMI_VP_CONF_PP_EN_START,
+            HDMI_VP_CONF_PP_EN_MASK,
+            1);
+    }
+    else if (outputSelect == 1)
+    {
+        /* YCC422 */
+        regValue = SET_FIELD(HDMI_VP_CONF_YCC422_EN_START,
+            HDMI_VP_CONF_YCC422_EN_MASK,
+            1);
+    }
+    else if (outputSelect == 2 || outputSelect == 3)
+    {
+        /* Enable bypass */
+        regValue = SET_FIELD(HDMI_VP_CONF_BYPASS_EN_START,
+            HDMI_VP_CONF_BYPASS_EN_MASK,
+            1) |
+	    SET_FIELD(HDMI_VP_CONF_BYPASS_SELECT_START,
+	        HDMI_VP_CONF_BYPASS_SELECT_MASK,
+	        1);
+
+    }
+
+    regValue |= SET_FIELD(HDMI_VP_CONF_OUTPUT_SELECTOR_START,
+        HDMI_VP_CONF_OUTPUT_SELECTOR_MASK,
+        outputSelect);
+
+    #if defined(EMULATOR)
+    regValue |= SET_FIELD(HDMI_VP_CONF_BYPASS_EN_START,
+        HDMI_VP_CONF_BYPASS_EN_MASK,
+        1) |
+    SET_FIELD(HDMI_VP_CONF_BYPASS_SELECT_START,
+        HDMI_VP_CONF_BYPASS_SELECT_MASK,
+        1);
+    #endif
+    /* pixel packing */
+    regValue |= SET_FIELD(HDMI_VP_CONF_PP_EN_START,
+        HDMI_VP_CONF_PP_EN_MASK,
+        1);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_CONF_OFFSET, regValue);
+
+    /* YCC422 and pixel packing stuffing */
+    regValue = SET_FIELD(HDMI_VP_STUFF_IDEFAULT_PHASE_START,
+        HDMI_VP_STUFF_IDEFAULT_PHASE_MASK,
+        pParams->mPixelPackingDefaultPhase) |
+    SET_FIELD(HDMI_VP_STUFF_YCC422_STUFFING_START,
+        HDMI_VP_STUFF_YCC422_STUFFING_MASK,
+        1) |
+    SET_FIELD(HDMI_VP_STUFF_PP_STUFFING_START,
+        HDMI_VP_STUFF_PP_STUFFING_MASK,
+        1);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_VP_STUFF_OFFSET, regValue);
+
+    return true;
+}
+
+/*
+
+Sets up Video Sampler registers
+
+*/
+static bool VideoConfigureVS(HDMI_DEVICE * pvDevice)
+{
+    VIDEO_PARAMS * pParams = &pvDevice->videoParams;
+    IMG_UINT8 mapCode = 0;
+    IMG_UINT8 regValue = 0;
+
+    HDMI_CHECKPOINT;
+
+	if (pParams->mEncodingIn == ENCODING_RGB ||
+        pParams->mEncodingIn == ENCODING_YCC444 ||
+        pParams->mEncodingIn == ENCODING_YCC420)
+	{
+		if (pParams->mColorResolution == 8 || pParams->mColorResolution == 0)
+			mapCode = 1;
+		else if (pParams->mColorResolution == 10)
+			mapCode = 3;
+		else if (pParams->mColorResolution == 12)
+			mapCode = 5;
+		else if (pParams->mColorResolution == 16)
+			mapCode = 7;
+		else
+		{
+			HDMI_ERROR_PRINT("- %s: Invalid color depth\n", __func__);
+			return false;
+		}
+		mapCode += (pParams->mEncodingIn  == ENCODING_YCC444) ? 8 : 0;
+	}
+	else if (pParams->mEncodingIn == ENCODING_YCC422)
+	{
+		/* YCC422 mapping is discontinued - only map 1 is supported */
+		if (pParams->mColorResolution == 12)
+			mapCode = 18;
+		else if (pParams->mColorResolution == 10)
+			mapCode = 20;
+		else if ((pParams->mColorResolution == 8)
+				|| (pParams->mColorResolution == 0))
+			mapCode = 22;
+		else
+		{
+			HDMI_ERROR_PRINT("- %s: Invalid color remap size: %d", __func__, pParams->mColorResolution);
+			return false;
+		}
+	}
+	else
+	{
+		HDMI_ERROR_PRINT("- %s: Invalid input encoding type: %d", __func__, pParams->mEncodingOut);
+		return false;
+    }
+
+    regValue = SET_FIELD(HDMI_TX_INVID0_INTERNAL_DE_GEN_START, HDMI_TX_INVID0_INTERNAL_DE_GEN_MASK, 0);
+    regValue |= SET_FIELD(HDMI_TX_INVID0_VIDEO_MAPPING_START, HDMI_TX_INVID0_VIDEO_MAPPING_MASK, mapCode);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_INVID0_OFFSET, regValue);
+
+    #if !defined(EMULATOR)
+    /* The following is done in the ref driver */
+
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_GYDATA0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_GYDATA1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_RCRDATA0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_RCRDATA1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_BCBDATA0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_BCBDATA1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_TX_INSTUFFING_OFFSET, 0x7);
+
+    #endif
+
+    return true;
+}
+
+static bool VideoConfigureFC(HDMI_DEVICE * pvDevice)
+{
+    IMG_UINT32 regValue = 0;
+    VIDEO_PARAMS * pParams = &pvDevice->videoParams;
+    DTD * pDtd = &pParams->mDtdList[pParams->mDtdActiveIndex];
+
+    /* Input video configuration */
+    regValue = SET_FIELD(HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_START,
+            HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_MASK,
+            pDtd->mVSyncPolarity) |
+        SET_FIELD(HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_START,
+            HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_MASK,
+            pDtd->mHSyncPolarity) |
+        SET_FIELD(HDMI_FC_INVIDCONF_DE_IN_POLARITY_START,
+            HDMI_FC_INVIDCONF_DE_IN_POLARITY_MASK,
+            pParams->mDataEnablePolarity) |
+        SET_FIELD(HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_START,
+            HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_MASK,
+            pDtd->mInterlaced) | // Need to check this out, ref driver sets interlaced for both
+        SET_FIELD(HDMI_FC_INVIDCONF_IN_I_P_START,
+            HDMI_FC_INVIDCONF_IN_I_P_MASK,
+            pDtd->mInterlaced) |
+        SET_FIELD(HDMI_FC_INVIDCONF_DVI_MODEZ_START,
+            HDMI_FC_INVIDCONF_DVI_MODEZ_MASK,
+            pParams->mHdmi);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr,
+        HDMI_FC_INVIDCONF_OFFSET,
+        regValue);
+
+    /*
+    Input HActive Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INHACTIV0_H_IN_ACTIV_START,
+        HDMI_FC_INHACTIV0_H_IN_ACTIV_MASK,
+        pDtd->mHActive & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_START,
+        HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_MASK,
+        pDtd->mHActive >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHACTIV1_OFFSET, regValue);
+
+    /*
+    Input VActive Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INVACTIV0_V_IN_ACTIV_START,
+        HDMI_FC_INVACTIV0_V_IN_ACTIV_MASK,
+        pDtd->mVActive & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_START,
+        HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_MASK,
+        pDtd->mVActive >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVACTIV1_OFFSET, regValue);
+
+    /*
+    Input HBlank Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INHBLANK0_H_IN_BLANK_START,
+        HDMI_FC_INHBLANK0_H_IN_BLANK_MASK,
+        pDtd->mHBlanking & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_START,
+        HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_MASK,
+        pDtd->mHBlanking >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INHBLANK1_OFFSET, regValue);
+
+    /*
+    Input VBlank Pixels
+    */
+    regValue = SET_FIELD(HDMI_FC_INVBLANK_V_IN_BLANK_START,
+        HDMI_FC_INVBLANK_V_IN_BLANK_MASK,
+        pDtd->mVBlanking & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_INVBLANK_OFFSET, regValue);
+
+    /*
+    Input HSync Front Porch (pixel clock cycles from "de" non active edge of the last "de" valid period)
+    */
+    regValue = SET_FIELD(HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_START,
+        HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_MASK,
+        pDtd->mHFrontPorchWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_START,
+        HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_MASK,
+        pDtd->mHFrontPorchWidth >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINDELAY1_OFFSET, regValue);
+
+    /*
+    Input VSync Front porch
+    */
+    regValue = SET_FIELD(HDMI_FC_VSYNCINDELAY_V_IN_DELAY_START,
+        HDMI_FC_VSYNCINDELAY_V_IN_DELAY_MASK,
+        pDtd->mVFrontPorchWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINDELAY_OFFSET, regValue);
+
+    /*
+    Input HSync pulse width
+    */
+    regValue = SET_FIELD(HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_START,
+        HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_MASK,
+        pDtd->mHSyncPulseWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH0_OFFSET, regValue);
+
+    regValue = SET_FIELD(HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_START,
+        HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_MASK,
+        pDtd->mHSyncPulseWidth >> 8);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_HSYNCINWIDTH1_OFFSET, regValue);
+
+    /*
+    Input Vsync pulse width
+    */
+    regValue = SET_FIELD(HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_START,
+        HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_MASK,
+        pDtd->mVSyncPulseWidth & 0xFF);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_VSYNCINWIDTH_OFFSET, regValue);
+
+    return true;
+
+}
+
+
+static void VideoParamsInit(VIDEO_PARAMS * pParams)
+{
+    #if defined(EMULATOR)
+    pParams->mHdmi = 1;
+    pParams->mEncodingOut = ENCODING_RGB;
+    pParams->mEncodingIn = ENCODING_RGB;
+    pParams->mColorResolution = 10;
+    pParams->mPixelRepetitionFactor = 0;
+    pParams->mRgbQuantizationRange = 0;
+    pParams->mPixelPackingDefaultPhase = 0;
+    pParams->mColorimetry = 0;
+    pParams->mScanInfo = 0;
+    pParams->mActiveFormatAspectRatio = 8;
+    pParams->mNonUniformScaling = 0;
+    pParams->mExtColorimetry = ~0;
+    pParams->mItContent = 0;
+    pParams->mEndTopBar = ~0;
+    pParams->mStartBottomBar = ~0;
+    pParams->mEndLeftBar = ~0;
+    pParams->mStartRightBar = ~0;
+    pParams->mCscFilter = 0;
+    pParams->mHdmiVideoFormat = 0;
+    pParams->m3dStructure = 0;
+    pParams->m3dExtData = 0;
+    pParams->mHdmiVic = 0;
+    pParams->mDtdIndex = 0;
+    pParams->mDtdActiveIndex = 0;
+    pParams->mDataEnablePolarity = 1;
+    #else
+    pParams->mEncodingOut = ENCODING_RGB;
+    pParams->mEncodingIn = ENCODING_RGB;
+    #if 1
+    pParams->mHdmi = 0;
+    pParams->mColorResolution = 8;
+    #else
+    pParams->mColorResolution = 10;
+    pParams->mHdmi = 1;
+    #endif
+    pParams->mPixelRepetitionFactor = 0;
+    pParams->mRgbQuantizationRange = 0;
+    pParams->mPixelPackingDefaultPhase = 0;
+    pParams->mColorimetry = 0;
+    pParams->mScanInfo = 0;
+    pParams->mActiveFormatAspectRatio = 8;
+    pParams->mNonUniformScaling = 0;
+    pParams->mExtColorimetry = ~0;
+    pParams->mItContent = 0;
+    pParams->mEndTopBar = ~0;
+    pParams->mStartBottomBar = ~0;
+    pParams->mEndLeftBar = ~0;
+    pParams->mStartRightBar = ~0;
+    pParams->mCscFilter = 0;
+    pParams->mHdmiVideoFormat = 0;
+    pParams->m3dStructure = 0;
+    pParams->m3dExtData = 0;
+    pParams->mHdmiVic = 0;
+    pParams->mDtdIndex = 0;
+    pParams->mDtdActiveIndex = 0;
+    pParams->mDataEnablePolarity = 1;
+    #endif
+}
+
+/*
+
+Steps for Configuring video mode:
+
+1. Power down Tx PHY
+2. Set desired video mode (based on EDID sink information)
+3. Power-on, PLL configuration, and reset Tx PHY
+4. Wait for PHY lock to assert
+
+*/
+PVRSRV_ERROR VideoConfigureMode(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+
+    HDMI_CHECKPOINT;
+
+    HDMI_DEBUG_PRINT(" - %s: Configuring video mode for VIC %d\n", __func__,
+        pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex].mCode);
+
+    VideoEnablePixelClock(pvDevice, pvDevice->videoParams.mDtdList[pvDevice->videoParams.mDtdActiveIndex].mPixelClock);
+
+    /*
+
+    Step D.2: Set desired video mode based on current VideoParams structure
+
+    */
+    if (!VideoConfigureVP(pvDevice))
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    if (!VideoConfigureVS(pvDevice))
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    if (!VideoConfigureFC(pvDevice))
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    /* Place phy in reset */
+    PhyPowerDown(pvDevice);
+
+    /* Audio N and CTS value */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N1_OFFSET, 0xE5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N2_OFFSET, 0x0F);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_N3_OFFSET, 0x00);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS1_OFFSET, 0x19);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS2_OFFSET, 0xD5);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_AUD_CTS3_OFFSET, 0x02);
+
+    /* Setup PHY for intended video mode */
+    PhyConfigureMode(pvDevice);
+
+    /* Wait for PHY lock */
+    status = PhyWaitLock(pvDevice);
+    if (status != PVRSRV_OK)
+    {
+		goto EXIT;
+    }
+
+    /* From pdump */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CTRLDUR_OFFSET, 0x0C);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLDUR_OFFSET, 0x20);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_EXCTRLSPAC_OFFSET, 0x01);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH0PREAM_OFFSET, 0x0B);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH1PREAM_OFFSET, 0x16);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_CH2PREAM_OFFSET, 0x21);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_MC_CLKDIS_OFFSET,
+        SET_FIELD(HDMI_MC_CLKDIS_HDCPCLK_DIS_START, HDMI_MC_CLKDIS_HDCPCLK_DIS_MASK, 1));
+
+EXIT:
+    return status;
+}
+
+/*
+
+See HDMI ref driver packets.c
+
+*/
+PVRSRV_ERROR VideoConfigureInfoframes(HDMI_DEVICE * pvDevice)
+{
+    VIDEO_PARAMS * pVideoParams = &pvDevice->videoParams;
+    DTD * pDtd = &pVideoParams->mDtdList[pVideoParams->mDtdActiveIndex];
+    IMG_UINT32 regValue;
+
+    HDMI_CHECKPOINT;
+
+    #if defined(EMULATOR)
+    return PVRSRV_OK; // This breaks on emu
+    #endif
+
+    /* Only relevant for HDMI */
+    if (pVideoParams->mHdmi == 0)
+    {
+        HDMI_DEBUG_PRINT("- %s: mHdmi not set, not configuring infoframes\n", __func__);
+        return PVRSRV_OK;
+    }
+
+    /* AVI CONF0 setup */
+    regValue = SET_FIELD(HDMI_FC_AVICONF0_RGBYCC_START, HDMI_FC_AVICONF0_RGBYCC_MASK, 0);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF0_SCAN_INFO_START,
+        HDMI_FC_AVICONF0_SCAN_INFO_MASK,
+        pVideoParams->mScanInfo);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET, regValue);
+
+    /* AVI CONF1 setup */
+    regValue = SET_FIELD(HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_START,
+        HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_START,
+        (((pDtd->mHImageSize * 10) % pDtd->mVImageSize) > 5) ? 2 : 1); /* 16:9 or 4:3 */
+    regValue |= SET_FIELD(HDMI_FC_AVICONF1_COLORIMETRY_START,
+        HDMI_FC_AVICONF1_COLORIMETRY_MASK,
+        pVideoParams->mColorimetry);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_START,
+        HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_MASK,
+        pVideoParams->mActiveFormatAspectRatio);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF1_OFFSET, regValue);
+
+    /* Go back and set active format valid bit */
+    regValue = SET_FIELD(HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_START,
+        HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_MASK,
+        1);
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET, regValue);
+
+    /* AVI CONF2 setup */
+    regValue = SET_FIELD(HDMI_FC_AVICONF2_IT_CONTENT_START,
+        HDMI_FC_AVICONF2_IT_CONTENT_MASK,
+        pVideoParams->mItContent);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF2_QUANTIZATION_RANGE_START,
+        HDMI_FC_AVICONF2_QUANTIZATION_RANGE_MASK,
+        pVideoParams->mRgbQuantizationRange);
+    regValue |= SET_FIELD(HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_START,
+        HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_MASK,
+        pVideoParams->mNonUniformScaling);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF2_OFFSET, regValue);
+
+    /* AVI VID setup */
+    if (pDtd->mCode != -1)
+    {
+        regValue = SET_FIELD(HDMI_FC_AVIVID_START, HDMI_FC_AVIVID_MASK, pDtd->mCode);
+    }
+    else
+    {
+        regValue = 0;
+    }
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIVID_OFFSET, regValue);
+
+    /* Set horizontal bars to 0 */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIETB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIETB1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISBB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISBB1_OFFSET, 0);
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET,
+        SET_FIELD(HDMI_FC_AVICONF0_HBAR_VALID_START, HDMI_FC_AVICONF0_HBAR_VALID_MASK, 1));
+
+    /* Set vertical bars to 0 */
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIELB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVIELB1_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISRB0_OFFSET, 0);
+    HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVISRB1_OFFSET, 0);
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_AVICONF0_OFFSET,
+        SET_FIELD(HDMI_FC_AVICONF0_VBAR_VALID_START, HDMI_FC_AVICONF0_VBAR_VALID_MASK, 1));
+
+    /* Set out pixel repetition factor */
+    HDMI_READ_MOD_WRITE(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_PRCONF_OFFSET,
+        SET_FIELD(HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_START,
+            HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK,
+            ((pDtd->mPixelRepetitionInput + 1) * (pVideoParams->mPixelRepetitionFactor + 1) - 1)));
+
+    return PVRSRV_OK;
+}
+
+/*
+
+Initialize routine will setup video mode for VGA DVI mode to
+start EDID communication
+
+*/
+PVRSRV_ERROR VideoInitialize(HDMI_DEVICE * pvDevice)
+{
+    PVRSRV_ERROR status = PVRSRV_OK;
+
+    HDMI_CHECKPOINT;
+
+    VideoParamsInit(&pvDevice->videoParams);
+
+    /* Configure based on video parameters in pvDevice (VGA/DVI on init) */
+    if (VideoConfigureMode(pvDevice) != PVRSRV_OK)
+    {
+        status = PVRSRV_ERROR_INIT_FAILURE;
+        goto EXIT;
+    }
+
+    /* Force video output on init */
+    {
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGTMDS_2_OFFSET, 0x00); /* R */
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGTMDS_1_OFFSET, 0xFF); /* G */
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGTMDS_0_OFFSET, 0x00); /* B */
+
+        HDMI_WRITE_CORE_REG(pvDevice->pvHDMIRegCpuVAddr, HDMI_FC_DBGFORCE_OFFSET,
+            SET_FIELD(HDMI_FC_DBGFORCE_FORCE_VIDEO_START, HDMI_FC_DBGFORCE_FORCE_VIDEO_MASK, 0));
+    }
+
+EXIT:
+
+    return status;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..c0e9b009c62d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Kbuild.mk
@@ -0,0 +1,56 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ccflags-$(CONFIG_X86) += -mno-soft-float
+
+ccflags-y += \
+ -I$(TOP)/services/display/rogue/plato_hdmi/include \
+ -I$(TOP)/services/display/include \
+ -I$(TOP)/kernel/drivers/staging/imgtec/plato	\
+ -I$(TOP)/include/rogue/public -I$(TOP)/include/public
+
+plato_hdmi-y += \
+	services/display/rogue/plato_hdmi/common/hdmi.o \
+	services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.o \
+	services/display/common/env/linux/dc_osfuncs.o \
+	services/display/rogue/plato_hdmi/common/video.o \
+	services/display/rogue/plato_hdmi/common/edid.o \
+	services/display/rogue/plato_hdmi/common/hdcp.o \
+	services/display/rogue/plato_hdmi/common/phy.o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Linux.mk
new file mode 100644
index 000000000000..29f7568c6d87
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/Linux.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+ifneq ($(SERVICES_SC),1)
+
+modules := plato_hdmi
+
+plato_hdmi_type := kernel_module
+plato_hdmi_target := plato_hdmi.ko
+plato_hdmi_makefile := $(THIS_DIR)/Kbuild.mk
+
+endif # SERVICES_SC
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.c b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.c
new file mode 100644
index 000000000000..af25b8fde891
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/env/linux/hdmi_osfuncs.c
@@ -0,0 +1,354 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+#include "dc_osfuncs.h"
+#include "hdmi.h"
+
+/* For MODULE_LICENSE */
+#include "pvrmodule.h"
+
+#define HDMI_DEBUGFS_DISPLAY_ENABLED "display_enabled"
+
+/* Callbacks for PDP<->HDMI interface */
+extern PVRSRV_ERROR PDPInitialize(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance);
+
+/* Helpers for getting DDR/GPU/PLL clock speed */
+extern IMG_UINT32 SysGetPlatoMemClockSpeed(void);
+extern IMG_UINT32 SysGetPlatoCoreClockSpeed(void);
+extern IMG_UINT32 SysGetPlatoPLLClockSpeed(IMG_UINT32 ui32ClockSpeed);
+
+static DLLIST_NODE g_sDeviceDataListHead;    /* List of discovered devices */
+static IMG_UINT32  g_uiNumHDMIDevs;          /* Number of discovered devices */
+
+static struct dentry	*g_psDebugFSEntryDir;/* debugFS top-level directory */
+
+/* HDMI module parameters */
+HDMI_MODULE_PARAMETERS sModuleParams =
+{
+	.ui32HDMIEnabled = 1,
+};
+
+module_param_named(mem_en, sModuleParams.ui32HDMIEnabled, uint, S_IRUGO | S_IWUSR);
+
+static int HdmiDebugFsOpen(struct inode *psINode, struct file *psFile)
+{
+	psFile->private_data = psINode->i_private;
+
+	return 0;
+}
+
+static ssize_t HdmiDebugFsRead(struct file *psFile,
+				  char __user *psUserBuffer,
+				  size_t uiCount,
+				  loff_t *puiPosition)
+{
+	HDMI_DEVICE *psDevice = (HDMI_DEVICE *)psFile->private_data;
+	IMG_BOOL bHDMIEnabled = psDevice->bHDMIEnabled;
+	loff_t uiPosition = *puiPosition;
+	char pszBuffer[] = "N\n";
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	if (uiPosition < 0)
+	{
+		return -EINVAL;
+	}
+	else if ((uiPosition >= (loff_t)uiBufferSize) || (uiCount == 0U))
+	{
+		return 0;
+	}
+
+	if (bHDMIEnabled == IMG_TRUE)
+	{
+		pszBuffer[0] = 'Y';
+	}
+
+	if (uiCount > (uiBufferSize - (size_t)uiPosition))
+	{
+		uiLocCount = uiBufferSize - (size_t)uiPosition;
+	}
+	else
+	{
+		uiLocCount = uiCount;
+	}
+
+	if (copy_to_user(psUserBuffer, &pszBuffer[uiPosition], uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	*puiPosition = uiPosition + (loff_t)uiLocCount;
+
+	return (ssize_t)uiLocCount;
+}
+
+static ssize_t HdmiDebugFsWrite(struct file *psFile,
+				   const char __user *psUserBuffer,
+				   size_t uiCount,
+				   loff_t *puiPosition)
+{
+	HDMI_DEVICE *psDevice = (HDMI_DEVICE *)psFile->private_data;
+	char pszBuffer[3];
+	bool bHDMIEnabled;
+	size_t uiBufferSize = ARRAY_SIZE(pszBuffer);
+	size_t uiLocCount;
+
+	uiLocCount = min(uiCount, (uiBufferSize - 1U));
+
+	if (copy_from_user(pszBuffer, psUserBuffer, uiLocCount) != 0)
+	{
+		return -EFAULT;
+	}
+
+	pszBuffer[uiLocCount] = '\0';
+
+	if (strtobool(pszBuffer, &bHDMIEnabled) == 0)
+	{
+		psDevice->bHDMIEnabled = bHDMIEnabled ? IMG_TRUE : IMG_FALSE;
+	}
+
+	return (ssize_t)uiLocCount;
+}
+
+static const struct file_operations gsDisplayEnabledFileOps =
+{
+	.owner = THIS_MODULE,
+	.open = HdmiDebugFsOpen,
+	.read = HdmiDebugFsRead,
+	.write = HdmiDebugFsWrite,
+	.llseek = default_llseek,
+};
+
+static struct pci_dev * hdmi_getdevice(struct pci_dev *psFromDev)
+{
+	struct pci_dev *psPCIDev;
+
+	psPCIDev = pci_get_device(SYS_RGX_DEV_VENDOR_ID,
+	                          SYS_RGX_DEV_DEVICE_ID,
+	                          psFromDev);
+#if defined(HDMI_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: %x, %x, device %p fromDev %p\n",
+	       __func__,
+	       SYS_RGX_DEV_VENDOR_ID, SYS_RGX_DEV_DEVICE_ID, psPCIDev, psFromDev);
+#endif
+
+	if (psPCIDev != psFromDev)
+	{
+		return psPCIDev;
+	}
+	else
+	{
+#if defined(HDMI_DEBUG)
+		if (psPCIDev != NULL)
+		{
+			printk(KERN_WARNING DRVNAME " - %s: Duplicate entry - terminating scan.\n",
+		         __func__);
+		}
+#endif
+		return NULL;
+	}
+}
+
+static int __init hdmi_init(void)
+{
+	struct pci_dev *psPCIDev, *fromDev = NULL;
+	HDMI_DEVICE *psDeviceData;
+	PVRSRV_ERROR eError;
+	int error;
+	IMG_CHAR aszDevName[4];    /* Location for unit name ("0" .. "N") */
+	static IMG_BOOL bFirstTime = IMG_TRUE;
+
+	#if !defined(HDMI_CONTROLLER)
+	return 0;
+	#endif
+
+	/* Iterate over all PCI instances */
+	dllist_init(&g_sDeviceDataListHead);
+	while ((psPCIDev = hdmi_getdevice(fromDev)) != NULL)
+	{
+		fromDev = psPCIDev;
+#if defined(HDMI_DEBUG)
+		printk(KERN_INFO DRVNAME " - %s: Initializing unit %u, parent %p, dev_data %p\n",
+		        __func__,
+		       g_uiNumHDMIDevs, psPCIDev->dev.parent,
+		       psPCIDev->dev.driver_data);
+#endif
+
+		error = pci_enable_device(psPCIDev);
+		if (error != 0)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to enable PCI device (%d)\n", __func__, error);
+			continue;	/* Continue scan and drop device reference */
+		}
+
+		eError = HDMIDrvInit(&psPCIDev->dev, &psDeviceData);
+		if (eError != PVRSRV_OK)
+		{
+			printk(KERN_ERR DRVNAME " - %s: Failed to initialise device (%d)\n", __func__, eError);
+			continue;	/* Continue scan and drop device reference */
+		}
+		dllist_init(&psDeviceData->sListNode);
+
+		/* To prevent possible problems with system suspend/resume, we don't
+		   keep the device enabled, but rely on the fact that the Rogue driver
+		   will have done a pci_enable_device. */
+		pci_disable_device(psPCIDev);
+
+		/* Create top-level debugFS entries on first iteration only */
+		if (bFirstTime == IMG_TRUE)
+		{
+			bFirstTime = IMG_FALSE;
+
+			g_psDebugFSEntryDir = debugfs_create_dir(DRVNAME, NULL);
+			if (IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s' debugfs root directory "
+				       "(debugfs entries won't be available)\n", __func__, DRVNAME);
+				g_psDebugFSEntryDir = NULL;
+			}
+
+		}
+
+		if (!IS_ERR_OR_NULL(g_psDebugFSEntryDir))
+		{
+			(void) DC_OSSNPrintf(aszDevName, sizeof(aszDevName), "%d", g_uiNumHDMIDevs);
+
+			psDeviceData->psDebugFSEntryDir = debugfs_create_dir(aszDevName, g_psDebugFSEntryDir);
+			if (IS_ERR_OR_NULL(psDeviceData->psDebugFSEntryDir))
+			{
+				printk(KERN_WARNING DRVNAME
+				       " - %s: Failed to create '%s/%s' debugfs root directory "
+				       "(debugfs entries won't be available)\n",
+				       __func__, DRVNAME, aszDevName);
+				psDeviceData->psDebugFSEntryDir = NULL;
+			}
+			psDeviceData->psDisplayEnabledEntry = debugfs_create_file(
+			                         HDMI_DEBUGFS_DISPLAY_ENABLED,
+			                         S_IFREG | S_IRUGO | S_IWUSR,
+			                         psDeviceData->psDebugFSEntryDir,
+			                         psDeviceData,
+			                         &gsDisplayEnabledFileOps);
+			if (IS_ERR_OR_NULL(psDeviceData->psDisplayEnabledEntry))
+			{
+				printk(KERN_WARNING DRVNAME " - %s: Failed to create '%s/%s' debugfs entry\n",
+				       __func__, aszDevName, HDMI_DEBUGFS_DISPLAY_ENABLED);
+				psDeviceData->psDisplayEnabledEntry = NULL;
+			}
+		}
+
+		/* Initialise the bHDMIEnabled value from the modparam setting */
+		psDeviceData->bHDMIEnabled = (sModuleParams.ui32HDMIEnabled == 1U) ? IMG_TRUE : IMG_FALSE;
+
+		psDeviceData->pfnPDPInitialize = PDPInitialize;
+
+		psDeviceData->ui32CoreClockSpeed = SysGetPlatoCoreClockSpeed();
+		psDeviceData->ui32PLLClockSpeed  = SysGetPlatoPLLClockSpeed(psDeviceData->ui32CoreClockSpeed);
+
+		psDeviceData->ui32Instance = g_uiNumHDMIDevs++;
+		dllist_add_to_tail(&g_sDeviceDataListHead, &psDeviceData->sListNode);
+	}
+
+#if defined(HDMI_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: Discovered %u HDMI instances\n",
+	       __func__, g_uiNumHDMIDevs);
+#endif
+
+	return (g_uiNumHDMIDevs > 0U) ? 0 : -ENODEV;
+}
+
+/*
+ Clear all registered devices from the system
+ */
+static void __exit hdmi_deinit(void)
+{
+
+	DLLIST_NODE *psNode, *psNext;
+	HDMI_DEVICE *psDeviceData;
+
+#if defined(HDMI_DEBUG)
+	printk(KERN_INFO DRVNAME " - %s: De-Init Entry %u Devices\n", __func__,
+	       g_uiNumHDMIDevs);
+#endif
+
+	dllist_foreach_node(&g_sDeviceDataListHead, psNode, psNext)
+	{
+		psDeviceData = IMG_CONTAINER_OF(psNode, HDMI_DEVICE, sListNode);
+
+		if (psDeviceData->psDisplayEnabledEntry)
+		{
+			debugfs_remove(psDeviceData->psDisplayEnabledEntry);
+		}
+
+#if defined(HDMI_DEBUG)
+		printk(KERN_INFO DRVNAME " - %s: De-init unit %u\n", __func__,
+		       psDeviceData->ui32Instance);
+#endif
+
+		if (psDeviceData->psDebugFSEntryDir)
+		{
+			debugfs_remove(psDeviceData->psDebugFSEntryDir);
+			psDeviceData->psDebugFSEntryDir = NULL;
+		}
+
+		dllist_remove_node(&psDeviceData->sListNode);
+		HDMIDrvDeInit(psDeviceData);
+		g_uiNumHDMIDevs--;
+	}
+
+	if (g_psDebugFSEntryDir)
+	{
+		debugfs_remove(g_psDebugFSEntryDir);
+		g_psDebugFSEntryDir = NULL;
+	}
+}
+
+module_init(hdmi_init);
+module_exit(hdmi_deinit);
+
+#endif /* defined(__linux__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/edid.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/edid.h
new file mode 100644
index 000000000000..53072673f2bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/edid.h
@@ -0,0 +1,353 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#ifndef DC_EDID_H
+#define DC_EDID_H
+
+#include "hdmi.h"
+
+/*************************************************************************/ /*!
+ EDID Information
+*/ /**************************************************************************/
+#define EDID_SIGNATURE                      0x00ffffffffffff00ull
+#define EDID_BLOCK_SIZE                     128
+#define EDID_CHECKSUM                       0 // per VESA spec, 8 bit checksum of base block
+#define EDID_SLAVE_ADDRESS                  0x50
+#define EDID_SEGMENT_ADDRESS                0x00
+#define EDID_REFRESH_RATE                   60000
+#define EDID_I2C_TIMEOUT_MS					100
+
+/* Extension block tag numbers (sec 2.2.4 of EDID spec) */
+#define EDID_EXT_TAG_CEA                    (0x02)
+#define EDID_EXT_TAG_VTB                    (0x10)
+#define EDID_EXT_TAG_DI                     (0x40)
+#define EDID_EXT_TAG_LS                     (0x50)
+#define EDID_EXT_TAG_DPVL                   (0x60)
+#define EDID_EXT_TAG_BLOCK_MAP              (0xF0)
+#define EDID_EXT_TAG_MFG_DEFINED            (0xFF)
+
+/* CEA Extension defines */
+#define EDID_CEA_REVISION                   (3)
+#define EDID_CEA_DATA_BLOCK_AUDIO_TAG       (1)
+#define EDID_CEA_DATA_BLOCK_VIDEO_TAG       (2)
+#define EDID_CEA_DATA_BLOCK_VENDOR_TAG      (3)
+#define EDID_CEA_DATA_BLOCK_SPEAKER_TAG     (4)
+
+/* Descriptor types */
+#define EDID_DESC_DETAILED_TIMING           0
+#define EDID_DESC_DISPLAY                   1
+
+/* Display descriptor tags */
+enum
+{
+    DISP_DESC_TAG_MFG_START = 0x00,
+    DISP_DESC_TAG_MFG_END = 0x0F,
+    DISP_DESC_TAG_DUMMY,
+    DISP_DESC_TAG_RESERVED_START,
+    DISP_DESC_TAG_RESERVED_END = 0xF6,
+    DISP_DESC_TAG_EST_TIMING_3,
+    DISP_DESC_TAG_CVT_3,
+    DISP_DESC_TAG_DCM,
+    DISP_DESC_TAG_STD_TIMING,
+    DISP_DESC_TAG_COLOR_POINT_DATA,
+    DISP_DESC_TAG_PRODUCT_NAME,
+    DISP_DESC_TAG_RANGE_LIMITS,
+    DISP_DESC_TAG_DATA_STRING,
+    DISP_DESC_TAG_SERIAL_NUMBER,
+};
+
+/* Digital video interface standards.
+ * See chapter 3.6.1 of VESA spec */
+enum
+{
+	DIGITAL_VIDEO_INTERFACE_NULL = 0x0,
+	DIGITAL_VIDEO_INTERFACE_DVI,
+	DIGITAL_VIDEO_INTERFACE_HDMI_A,
+	DIGITAL_VIDEO_INTERFACE_HDMI_B,
+	DIGITAL_VIDEO_INTERFACE_MDDI,
+	DIGITAL_VIDEO_INTERFACE_DISPLAY_PORT,
+};
+
+/* See chapter 3.10.2 of VESA spec */
+typedef struct _DTD_RAW
+{
+    IMG_UINT16                  pixelClock; // 0x0001 - 0xFFFF stored value = pixel clock / 10,000
+
+    IMG_UINT8                   hActiveLower;
+    IMG_UINT8                   hBlankingLower;
+
+    IMG_UINT8                   hBlankingUpper          : 4;
+    IMG_UINT8                   hActiveUpper            : 4;
+
+    IMG_UINT8                   vActiveLower;
+    IMG_UINT8                   vBlankingLower;
+
+    IMG_UINT8                   vBlankingUpper          : 4;
+    IMG_UINT8                   vActiveUpper            : 4;
+
+    IMG_UINT8                   hFrontPorchLower;
+    IMG_UINT8                   hSyncPulseWidthLower;
+
+    // Byte 10
+    IMG_UINT8                   vSyncPulseWidthLower    : 4;
+    IMG_UINT8                   vFrontPorchLower        : 4;
+
+    // Byte 11
+    IMG_UINT8                   vSyncPulseWidthUpper    : 2;
+    IMG_UINT8                   vFrontPorchUpper        : 2;
+    IMG_UINT8                   hSyncPulseWidthUpper    : 2;
+    IMG_UINT8                   hFrontPorchUpper        : 2;
+
+    IMG_UINT8                   hImageSizeLower;
+    IMG_UINT8                   vImageSizeLower;
+    IMG_UINT8                   vImageSizeUpper			: 4;
+    IMG_UINT8                   hImageSizeUpper         : 4;
+    IMG_UINT8                   hBorder;
+    IMG_UINT8                   vBorder;
+
+    // Byte 17
+    IMG_UINT8                   stereoViewing2          : 1;
+    IMG_UINT8                   syncSignalDefs          : 4;
+    IMG_UINT8                   stereoViewing1          : 2;
+    IMG_UINT8                   interlaced              : 1;
+
+} DTD_RAW;
+
+typedef union _ESTABILISHED_TIMING_I
+{
+    IMG_UINT8 value;
+    struct {
+        IMG_UINT8 _800x600_60Hz : 1;
+        IMG_UINT8 _800x600_56Hz : 1;
+        IMG_UINT8 _640x480_75Hz : 1;
+        IMG_UINT8 _640x480_72Hz : 1;
+        IMG_UINT8 _640x480_67Hz : 1;
+        IMG_UINT8 _640x480_60Hz : 1;
+        IMG_UINT8 _720x400_88Hz : 1;
+        IMG_UINT8 _720x400_70Hz : 1;
+    } fields;
+}ESTABILISHED_TIMING_I;
+
+typedef union _STANDARD_TIMING
+{
+    IMG_UINT16 value;
+    struct
+    {
+	    IMG_UINT16 horizontalAddressablePixels : 8;
+        IMG_UINT16 refreshRate : 6;
+        IMG_UINT16 aspectRatio : 2;
+    } fields;
+} STANDARD_TIMING;
+
+typedef union _VIDEO_INPUT_DEFINITION
+{
+    IMG_UINT8 value;
+    struct
+    {
+        IMG_UINT8 serrations : 1; // BIT 0
+        IMG_UINT8 syncTypes : 3; // BIT 3-1
+        IMG_UINT8 videoSetup : 1; // BIT 4
+        IMG_UINT8 signalLevelStandard : 2; // BIT 6-5
+        IMG_UINT8 signalInterface : 1; // BIT 7
+    } analog;
+
+    struct
+    {
+        IMG_UINT8 vidInterfaceSupported : 4; // BIT 3-0
+        IMG_UINT8 colorBitDepth : 3; // BIT 6-4
+        IMG_UINT8 signalInterface : 1; // BIT 7
+    } digital;
+} VIDEO_INPUT_DEFINITION;
+
+typedef union _FEATURE_SUPPORT
+{
+    IMG_UINT8 value;
+    struct
+    {
+	        // Indicates whether Preferred Timing Mode includes native pixel format
+	        // and preferred refresh rate of the display
+        IMG_UINT8 continuousFrequency : 1; // BIT 0
+        IMG_UINT8 preferredTimingIncluded : 1; // BIT 1
+            // BIT 7 at address 14h must be 1 for this layout (signalInterface = digital)
+            // 00 : RBG 4:4:4
+            // 01 : RGB 4:4:4 & YCrCb 4:4:4
+            // 10 : RGB 4:4:4 & YCrCb 4:2:2
+            // 11 : RGB 4:4:4 & YCrCb 4:4:4 & YCrCb 4:2:2
+        IMG_UINT8 sRGBstandard : 1; // BIT 2
+        IMG_UINT8 colorEncodingFormat : 2; // BIT 4 - 3
+        IMG_UINT8 veryLowPowerSupport : 1; // BIT 5
+        IMG_UINT8 suspendSupport : 1; // BIT 6
+        IMG_UINT8 standbySupport : 1; // BIT 7
+    } fields;
+} FEATURE_SUPPORT;
+
+typedef struct _DISPLAY_DESCRIPTOR
+{
+    IMG_UINT8                   DescTag[3];                         // all 0 for display descriptor
+    IMG_UINT8                   DispDescTag;                        // See enum above
+    IMG_UINT8                   Reserved;                           // 00h except for Display Range Limits
+    IMG_UINT8                   Data[13];
+} DISPLAY_DESCRIPTOR;
+
+/* Structures for each type of display descriptor */
+typedef struct _MONITOR_RANGE_DATA
+{
+    IMG_UINT8                   offsetFlags;
+    IMG_UINT8                   vMinRate;
+    IMG_UINT8                   vMaxRate;
+    IMG_UINT8                   hMinRate;
+    IMG_UINT8                   hMaxRate;
+    IMG_UINT8                   maxPixelClock;
+    IMG_UINT8                   supportFlags;
+        // Determines layout of next 7 bytes
+        // 00: Default GTF supported (if BIT 0 in feature support = 1)
+        // 01: Range limits only
+        // 02: Secondary GTF supported
+        // 04: CVT supported
+    union
+    {
+        // Byte 10 = 00h or 01h
+        struct
+        {
+            IMG_UINT8           lineFeed;
+            IMG_UINT8           space[6];
+        };
+        // Byte 10 = 02h or 04h
+        struct
+        {
+            IMG_UINT8           timingData[7];
+        };
+    };
+} MONITOR_RANGE_DATA;
+
+typedef struct _EDID_EXT_BLOCK
+{
+    IMG_UINT8                   Tag;
+    IMG_UINT8                   Revision;
+    IMG_UINT8                   Data[125];
+    IMG_UINT8                   Checksum;
+} EDID_EXT_BLOCK;
+
+typedef struct _EDID_EXT_CEA
+{
+    IMG_UINT8                   Tag;
+    IMG_UINT8                   Revision;
+    IMG_UINT8                   DtdOffset; // Offset within block that has first 18-byte DTD
+    IMG_UINT8                   Flags; // Contains info about YC222, YC444, number of DTDs, etc
+    IMG_UINT8                   Data[123];
+    IMG_UINT8                   Checksum;
+} EDID_EXT_CEA;
+
+
+typedef struct
+{
+    IMG_UINT8                   Header[8];                          //EDID header "00 FF FF FF FF FF FF 00"
+    IMG_UINT16                  ManufactureName;                    //EISA 3-character ID (5 bit compressed ASCII)
+    IMG_UINT16                  ProductCode;                        //Vendor assigned code
+    IMG_UINT32                  SerialNumber;                       //32-bit serial number
+    IMG_UINT8                   WeekOfManufacture;                  //Week number
+    IMG_UINT8                   YearOfManufacture;                  //Year
+    IMG_UINT8                   EdidVersion;                        //EDID Structure Version
+    IMG_UINT8                   EdidRevision;                       //EDID Structure Revision
+    VIDEO_INPUT_DEFINITION      VideoInputDefinition;
+    IMG_UINT8                   MaxHorizontalImageSize;             //cm
+    IMG_UINT8                   MaxVerticalImageSize;               //cm
+    IMG_UINT8                   Gamma;
+    FEATURE_SUPPORT             FeatureSupport;
+    IMG_UINT8                   RedGreenLowBits;                    //Rx1 Rx0 Ry1 Ry0 Gx1 Gx0 Gy1Gy0
+    IMG_UINT8                   BlueWhiteLowBits;                   //Bx1 Bx0 By1 By0 Wx1 Wx0 Wy1 Wy0
+    IMG_UINT8                   RedX;                               //Red-x Bits 9 - 2
+    IMG_UINT8                   RedY;                               //Red-y Bits 9 - 2
+    IMG_UINT8                   GreenX;                             //Green-x Bits 9 - 2
+    IMG_UINT8                   GreenY;                             //Green-y Bits 9 - 2
+    IMG_UINT8                   BlueX;                              //Blue-x Bits 9 - 2
+    IMG_UINT8                   BlueY;                              //Blue-y Bits 9 - 2
+    IMG_UINT8                   WhiteX;                             //White-x Bits 9 - 2
+    IMG_UINT8                   WhiteY;                             //White-x Bits 9 - 2
+    IMG_UINT8                   EstablishedTimings[3];              // Optional (see above)
+    STANDARD_TIMING             StandardTimings[8];                 // Optional
+    /* The next 72 bytes vary per E-EDID Standard (1.4 is below) */
+    DTD_RAW                     PreferredTimingMode;                // Check FEATURE_SUPPORT BIT 1 before using this
+    IMG_UINT8                   Descriptor2[18];                    // Or display descriptor
+    IMG_UINT8                   Descriptor3[18];                    // Or display descriptor
+    IMG_UINT8                   Descriptor4[18];                    // Or display descriptor
+    IMG_UINT8                   ExtensionBlockCount;                //Number of (optional) 128-byte EDID extension blocks to follow
+    IMG_UINT8                   Checksum;
+} EDID_BASE_BLOCK;
+
+/* Function prototypes */
+PVRSRV_ERROR EdidAddDtd(VIDEO_PARAMS * pVideoParams, IMG_UINT8 code, IMG_UINT32 refreshRate);
+PVRSRV_ERROR EdidInitialize(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR EdidRead(HDMI_DEVICE * pvDevice);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdcp.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdcp.h
new file mode 100644
index 000000000000..5770f541ef49
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdcp.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+
+
+PVRSRV_ERROR HdcpConfigure(HDMI_DEVICE * pvDevice);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi.h
new file mode 100644
index 000000000000..cac41483bef2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi.h
@@ -0,0 +1,250 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/ /**************************************************************************/
+
+#ifndef HDMI_H
+#define HDMI_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "hdmi_regs.h"
+#include "pvrsrv.h"
+#include "powervr/imgpixfmts.h"
+#include "dc_osfuncs.h"
+#include "plato.h"
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+
+
+#define DRVNAME	"plato_hdmi"
+#define HDMI_MAX_COMMANDS_INFLIGHT    (2)
+
+typedef PVRSRV_ERROR (*PFN_PDP_INIT)(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance);
+
+/*************************************************************************/ /*!
+ PCI Device Information
+*/ /**************************************************************************/
+
+// See sysinfo.h
+
+/*************************************************************************/ /*!
+ HDMI data structures
+*/ /**************************************************************************/
+
+typedef struct HDMI_MODULE_PARAMETERS_TAG
+{
+	IMG_UINT32	ui32HDMIEnabled;
+} HDMI_MODULE_PARAMETERS;
+
+const HDMI_MODULE_PARAMETERS *HDMIGetModuleParameters(void);
+
+typedef struct HDMI_DEVICE
+{
+	IMG_HANDLE                  hPVRServicesConnection;
+	IMG_HANDLE                  hPVRServicesDevice;
+	IMG_HANDLE                  hLISRData;
+	DC_SERVICES_FUNCS           sPVRServicesFuncs;
+	DC_SPINLOCK                 irqLock;
+	struct delayed_work         delayedWork;
+	void                        *pvDevice;
+
+	IMG_CPU_PHYADDR             sHDMIRegCpuPAddr;
+	IMG_CPU_VIRTADDR            pvHDMIRegCpuVAddr;
+
+	IMG_CPU_PHYADDR             sTopRegCpuPAddr;
+	IMG_CPU_VIRTADDR            pvTopRegCpuVAddr;
+
+	VIDEO_PARAMS                videoParams;
+
+	IMG_BOOL                    mInitialized;
+	IMG_BOOL                    bHPD;
+
+	/* Callback for PDP */
+	PFN_PDP_INIT                pfnPDPInitialize;
+
+	/* Plato core and PLL clock speeds */
+	IMG_UINT32                  ui32CoreClockSpeed;
+	IMG_UINT32                  ui32PLLClockSpeed;
+
+	/* Doubly linked list of device units */
+	DLLIST_NODE                 sListNode;
+	struct dentry               *psDebugFSEntryDir;     /* 'plato_hdmi' entry */
+	struct dentry               *psDisplayEnabledEntry; /* 'display_enabled' entry */
+	IMG_UINT32                  ui32Instance;           /* device instance */
+	IMG_BOOL                    bHDMIEnabled;           /* state of 'display_enabled' */
+} HDMI_DEVICE;
+
+
+/*******************************************************************************
+ * HDMI common functions
+ ******************************************************************************/
+
+PVRSRV_ERROR HDMIDrvInit(void *pvDevice, HDMI_DEVICE **ppsDeviceData);
+void HDMIDrvDeInit(HDMI_DEVICE *psDeviceData);
+
+/*******************************************************************************
+ * Debug/print macros
+ ******************************************************************************/
+#if defined(PLATO_DISPLAY_PDUMP)
+#include "plato_pdump.h"
+
+static void HDMIPdumpReg32(void *pvLinRegBaseAddr, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	OSWriteHWReg32(pvLinRegBaseAddr,  ui32Offset,  ui32Value);
+	plato_pdump_reg32(pvLinRegBaseAddr, ui32Offset, ui32Value, DRVNAME);
+}
+
+#undef DC_OSWriteReg32
+
+#define DC_OSWriteReg32 HDMIPdumpReg32
+
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	plato_pdump_pol(base,reg,val,msk, DRVNAME); \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			HDMI_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#else
+#define polpr(base,reg,val,msk,cnt,intrvl) \
+	do { \
+		IMG_UINT32 polnum; \
+		for (polnum = 0; polnum < cnt; polnum++) \
+		{ \
+			if ((DC_OSReadReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			DC_OSDelayus(intrvl * 1000); \
+		} \
+		if (polnum == cnt) \
+		{ \
+			HDMI_DEBUG_PRINT(" Poll failed for register: 0x%08X", (unsigned int)reg); \
+		} \
+	} while (0)
+#endif // PLATO_DISPLAY_PDUMP
+
+
+#define HDMI_REG_POLL(base,reg,val,msk) polpr(base,reg*4,val,msk,10,10)
+#define TOP_REG_POLL(base,reg,val,msk) polpr(base,reg,val,msk,10,10)
+
+
+#define CHECK_AND_EXIT(status, label) if (status != PVRSRV_OK) goto label;
+#define IS_BIT_SET(value, bit) (value & (1 << bit))
+
+#if !defined(VIRTUAL_PLATFORM)
+#if defined(HDMI_DEBUG)
+	#define HDMI_CHECKPOINT HDMI_DEBUG_PRINT(" CP: - %s, line %d\n", __func__, __LINE__);
+	#define HDMI_DEBUG_PRINT(fmt, ...) \
+		DC_OSDebugPrintf(DBGLVL_INFO, fmt, __VA_ARGS__)
+
+	#define HDMI_WRITE_CORE_REG(base, offset, value) \
+		DC_OSWriteReg32(base, offset * 4, value);
+
+#else
+	#define HDMI_CHECKPOINT
+	#define HDMI_DEBUG_PRINT(fmt, ...)
+	#define HDMI_WRITE_CORE_REG(base, offset, value) DC_OSWriteReg32(base, offset * 4, value);
+#endif
+
+#define HDMI_READ_CORE_REG(base, offset) DC_OSReadReg32(base, offset * 4)
+#define HDMI_READ_MOD_WRITE(base, offset, value) HDMI_WRITE_CORE_REG(base, offset, (HDMI_READ_CORE_REG(base,offset) | value))
+#else /* VIRTUAL_PLATFORM */
+
+#define HDMI_CHECKPOINT HDMI_DEBUG_PRINT(" CP: - %s, line %d\n", __func__, __LINE__);
+#define HDMI_DEBUG_PRINT(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_INFO, fmt, __VA_ARGS__)
+#define HDMI_WRITE_CORE_REG(base, offset, value)
+#define HDMI_READ_CORE_REG(base, offset) (0)
+#define HDMI_READ_MOD_WRITE(base, offset, value)
+
+#endif
+
+#define HDMI_ERROR_PRINT(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_ERROR, fmt, __VA_ARGS__)
+
+#define HDMI_LOG_ALWAYS(fmt, ...) \
+	DC_OSDebugPrintf(DBGLVL_INFO, fmt, __VA_ARGS__)
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi_regs.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi_regs.h
new file mode 100644
index 000000000000..1e9ff9c0e9e7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/hdmi_regs.h
@@ -0,0 +1,587 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HDMI_REGS_H
+#define HDMI_REGS_H
+
+#define SET_FIELD(start, mask, value) \
+    ((value << start) & mask)
+
+#define GET_FIELD(value, start, mask) \
+    ((value & mask) >> start)
+/* Hardware Register definitions
+
+NOTE: These are HDMI core address offsets, SoC level offset will be * 4
+
+*/
+
+/* Identification registers */
+#define HDMI_ID_DESIGN_ID_OFFSET                            (0x0000)
+
+#define HDMI_ID_REVISION_ID_OFFSET                          (0x0001)
+
+#define HDMI_ID_PRODUCT_ID0_OFFSET                          (0x0002)
+
+#define HDMI_ID_PRODUCT_ID1_OFFSET                          (0x0003)
+#define HDMI_ID_PRODUCT_ID1_TX_START                        (0)
+#define HDMI_ID_PRODUCT_ID1_TX_MASK                         (0x1)
+#define HDMI_ID_PRODUCT_ID1_RX_START                        (1)
+#define HDMI_ID_PRODUCT_ID1_RX_MASK                         (0x2)
+#define HDMI_ID_PRODUCT_ID1_HDCP_START                      (6)
+#define HDMI_ID_PRODUCT_ID1_HDCP_MASK                       (0xC0)
+
+#define HDMI_ID_CONFIG0_ID_OFFSET                           (0x0004)
+#define HDMI_ID_CONFIG0_ID_HDCP_START                       (0)
+#define HDMI_ID_CONFIG0_ID_HDCP_MASK                        (0x1)
+#define HDMI_ID_CONFIG0_ID_CEC_START                        (1)
+#define HDMI_ID_CONFIG0_ID_CEC_MASK                         (0x2)
+#define HDMI_ID_CONFIG0_ID_CSC_START                        (2)
+#define HDMI_ID_CONFIG0_ID_CSC_MASK                         (0x4)
+#define HDMI_ID_CONFIG0_ID_HDMI14_START                     (3)
+#define HDMI_ID_CONFIG0_ID_HDMI14_MASK                      (0x8)
+#define HDMI_ID_CONFIG0_ID_AUDI2S_START                     (4)
+#define HDMI_ID_CONFIG0_ID_AUDI2S_MASK                      (0x10)
+#define HDMI_ID_CONFIG0_ID_AUDSPDIF_START                   (5)
+#define HDMI_ID_CONFIG0_ID_AUDSPDIF_MASK                    (0x20)
+#define HDMI_ID_CONFIG0_ID_PREPEN_START                     (7)
+#define HDMI_ID_CONFIG0_ID_PREPEN_MASK                      (0x80)
+
+#define HDMI_ID_CONFIG1_ID_OFFSET                           (0x0005)
+#define HDMI_ID_CONFIG1_ID_CONFAPB_START                    (1)
+#define HDMI_ID_CONFIG1_ID_CONFAPB_MASK                     (0x2)
+#define HDMI_ID_CONFIG1_ID_HDMI20_START                     (5)
+#define HDMI_ID_CONFIG1_ID_HDMI20_MASK                      (0x20)
+#define HDMI_ID_CONFIG1_ID_HDCP22_START                     (6)
+#define HDMI_ID_CONFIG1_ID_HDCP22_MASK                      (0x40)
+
+#define HDMI_ID_CONFIG2_ID_OFFSET                           (0x0006)
+
+#define HDMI_ID_CONFIG3_ID_OFFSET                           (0x0007)
+
+/* Interrupt Registers */
+#define HDMI_IH_FC_STAT0_OFFSET                             (0x0100)
+
+#define HDMI_IH_FC_STAT1_OFFSET                             (0x0101)
+
+#define HDMI_IH_FC_STAT2_OFFSET                             (0x0102)
+
+#define HDMI_IH_AS_STAT0_OFFSET                             (0x0103)
+
+#define HDMI_IH_PHY_STAT0_OFFSET                            (0x0104)
+#define HDMI_IH_PHY_STAT0_HPD_START                         (0)
+#define HDMI_IH_PHY_STAT0_HPD_MASK                          (0x1)
+
+#define HDMI_IH_I2CM_STAT0_OFFSET                           (0x0105)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERERROR_START             (0)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERERROR_MASK              (0x1)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERDONE_START              (1)
+#define HDMI_IH_I2CM_STAT0_I2CMASTERDONE_MASK               (0x2)
+#define HDMI_IH_I2CM_STAT0_SCDC_READREQ_START               (2)
+#define HDMI_IH_I2CM_STAT0_SCDC_READREQ_MASK                (0x4)
+
+#define HDMI_IH_CEC_STAT0_OFFSET                            (0x0106)
+
+#define HDMI_IH_VP_STAT0_OFFSET                             (0x0107)
+
+#define HDMI_IH_I2CMPHY_STAT0_OFFSET                        (0x0108)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYERROR_START            (0)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYERROR_MASK             (0x1)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_START             (1)
+#define HDMI_IH_I2CMPHY_STAT0_I2CMPHYDONE_MASK              (0x2)
+
+#define HDMI_IH_AHBDMAAUD_STAT0_OFFSET                      (0x0109)
+
+#define HDMI_IH_DECODE_OFFSET                               (0x0170)
+#define HDMI_IH_DECODE_PHY_START                            (3)
+#define HDMI_IH_DECODE_PHY_MASK                             (0x8)
+
+#define HDMI_IH_MUTE_FC_STAT0_OFFSET                        (0x0180)
+
+#define HDMI_IH_MUTE_FC_STAT1_OFFSET                        (0x0181)
+
+#define HDMI_IH_MUTE_FC_STAT2_OFFSET                        (0x0182)
+
+#define HDMI_IH_MUTE_AS_STAT0_OFFSET                        (0x0183)
+
+#define HDMI_IH_MUTE_PHY_STAT0_OFFSET                       (0x0184)
+
+#define HDMI_IH_MUTE_I2CM_STAT0_OFFSET                      (0x0185)
+
+#define HDMI_IH_MUTE_CEC_STAT0_OFFSET                       (0x0186)
+
+#define HDMI_IH_MUTE_VP_STAT0_OFFSET                        (0x0187)
+
+#define HDMI_IH_MUTE_I2CMPHY_STAT0_OFFSET                   (0x0188)
+
+#define HDMI_IH_MUTE_AHBDMAAUD_STAT0_OFFSET                 (0x0189)
+
+#define HDMI_IH_MUTE_OFFSET                                 (0x01ff)
+#define HDMI_IH_MUTE_ALL_START                              (0)
+#define HDMI_IH_MUTE_ALL_MASK                               (0x3)
+
+/* Video Sampler registers */
+#define HDMI_TX_INVID0_OFFSET                               (0x0200)
+#define HDMI_TX_INVID0_VIDEO_MAPPING_START                  (0)
+#define HDMI_TX_INVID0_VIDEO_MAPPING_MASK                   (0x1F)
+#define HDMI_TX_INVID0_INTERNAL_DE_GEN_START                (7)
+#define HDMI_TX_INVID0_INTERNAL_DE_GEN_MASK                 (0x80)
+
+#define HDMI_TX_INSTUFFING_OFFSET                           (0x201)
+
+#define HDMI_TX_GYDATA0_OFFSET                              (0x202)
+
+#define HDMI_TX_GYDATA1_OFFSET                              (0x203)
+
+#define HDMI_TX_RCRDATA0_OFFSET                             (0x204)
+
+#define HDMI_TX_RCRDATA1_OFFSET                             (0x205)
+
+#define HDMI_TX_BCBDATA0_OFFSET                             (0x206)
+
+#define HDMI_TX_BCBDATA1_OFFSET                             (0x207)
+
+/* Video Packetizer */
+#define HDMI_VP_STATUS_OFFSET                               (0x0800)
+
+#define HDMI_VP_PR_CD_OFFSET                                (0x0801)
+#define HDMI_VP_PR_CD_DESIRED_PR_FACTOR_START               (0)
+#define HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK                (0xF)
+#define HDMI_VP_PR_CD_COLOR_DEPTH_START                     (4)
+#define HDMI_VP_PR_CD_COLOR_DEPTH_MASK                      (0xF0)
+
+#define HDMI_VP_STUFF_OFFSET                                (0x0802)
+#define HDMI_VP_STUFF_PR_STUFFING_START                     (0)
+#define HDMI_VP_STUFF_PR_STUFFING_MASK                      (0x1)
+#define HDMI_VP_STUFF_PP_STUFFING_START                     (1)
+#define HDMI_VP_STUFF_PP_STUFFING_MASK                      (0x2)
+#define HDMI_VP_STUFF_YCC422_STUFFING_START                 (2)
+#define HDMI_VP_STUFF_YCC422_STUFFING_MASK                  (0x4)
+#define HDMI_VP_STUFF_ICX_GOTO_P0_ST_START                  (3)
+#define HDMI_VP_STUFF_ICX_GOTO_P0_ST_MASK                   (0x8)
+#define HDMI_VP_STUFF_IFIX_PP_TO_LAST_START                 (4)
+#define HDMI_VP_STUFF_IFIX_PP_TO_LAST_MASK                  (0x10)
+#define HDMI_VP_STUFF_IDEFAULT_PHASE_START                  (5)
+#define HDMI_VP_STUFF_IDEFAULT_PHASE_MASK                   (0x20)
+
+#define HDMI_VP_REMAP_OFFSET                                (0x0803)
+#define HDMI_VP_REMAP_YCC422_SIZE_START                     (0)
+#define HDMI_VP_REMAP_YCC422_SIZE_MASK                      (0x3)
+
+#define HDMI_VP_CONF_OFFSET                                 (0x0804)
+#define HDMI_VP_CONF_OUTPUT_SELECTOR_START                  (0)
+#define HDMI_VP_CONF_OUTPUT_SELECTOR_MASK                   (0x3)
+#define HDMI_VP_CONF_BYPASS_SELECT_START                    (2)
+#define HDMI_VP_CONF_BYPASS_SELECT_MASK                     (0x4)
+#define HDMI_VP_CONF_YCC422_EN_START                        (3)
+#define HDMI_VP_CONF_YCC422_EN_MASK                         (0x8)
+#define HDMI_VP_CONF_PR_EN_START                            (4)
+#define HDMI_VP_CONF_PR_EN_MASK                             (0x10)
+#define HDMI_VP_CONF_PP_EN_START                            (5)
+#define HDMI_VP_CONF_PP_EN_MASK                             (0x20)
+#define HDMI_VP_CONF_BYPASS_EN_START                        (6)
+#define HDMI_VP_CONF_BYPASS_EN_MASK                         (0x40)
+
+#define HDMI_VP_MASK_OFFSET                                 (0x0807)
+
+/* Frame Composer */
+#define HDMI_FC_INVIDCONF_OFFSET                            (0x1000)
+#define HDMI_FC_INVIDCONF_IN_I_P_START                      (0)
+#define HDMI_FC_INVIDCONF_IN_I_P_MASK                       (0x1)
+#define HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_START            (1)
+#define HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_MASK             (0x2)
+#define HDMI_FC_INVIDCONF_DVI_MODEZ_START                   (3)
+#define HDMI_FC_INVIDCONF_DVI_MODEZ_MASK                    (0x8)
+#define HDMI_FC_INVIDCONF_DE_IN_POLARITY_START              (4)
+#define HDMI_FC_INVIDCONF_DE_IN_POLARITY_MASK               (0x10)
+#define HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_START           (5)
+#define HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_MASK            (0x20)
+#define HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_START           (6)
+#define HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_MASK            (0x40)
+#define HDMI_FC_INVIDCONF_HDCP_KEEPOUT_START                (7)
+#define HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK                 (0x80)
+
+#define HDMI_FC_INHACTIV0_OFFSET                            (0x1001)
+#define HDMI_FC_INHACTIV0_H_IN_ACTIV_START                  (0)
+#define HDMI_FC_INHACTIV0_H_IN_ACTIV_MASK                   (0xFF)
+
+#define HDMI_FC_INHACTIV1_OFFSET                            (0x1002)
+#define HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_START            (0)
+#define HDMI_FC_INHACTIV1_H_IN_ACTIV_UPPER_MASK             (0x3F)
+
+#define HDMI_FC_INHBLANK0_OFFSET                            (0x1003)
+#define HDMI_FC_INHBLANK0_H_IN_BLANK_START                  (0)
+#define HDMI_FC_INHBLANK0_H_IN_BLANK_MASK                   (0xFF)
+
+#define HDMI_FC_INHBLANK1_OFFSET                            (0x1004)
+#define HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_START            (0)
+#define HDMI_FC_INHBLANK1_H_IN_BLANK_UPPER_MASK             (0x3F)
+
+#define HDMI_FC_INVACTIV0_OFFSET                            (0x1005)
+#define HDMI_FC_INVACTIV0_V_IN_ACTIV_START                  (0)
+#define HDMI_FC_INVACTIV0_V_IN_ACTIV_MASK                   (0xFF)
+
+#define HDMI_FC_INVACTIV1_OFFSET                            (0x1006)
+#define HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_START            (0)
+#define HDMI_FC_INVACTIV1_V_IN_ACTIV_UPPER_MASK             (0x3F)
+
+#define HDMI_FC_INVBLANK_OFFSET                             (0x1007)
+#define HDMI_FC_INVBLANK_V_IN_BLANK_START                   (0)
+#define HDMI_FC_INVBLANK_V_IN_BLANK_MASK                    (0xFF)
+
+#define HDMI_FC_HSYNCINDELAY0_OFFSET                        (0x1008)
+#define HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_START              (0)
+#define HDMI_FC_HSYNCINDELAY0_H_IN_DELAY_MASK               (0xFF)
+
+#define HDMI_FC_HSYNCINDELAY1_OFFSET                        (0x1009)
+#define HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_START        (0)
+#define HDMI_FC_HSYNCINDELAY1_H_IN_DELAY_UPPER_MASK         (0x3F)
+
+#define HDMI_FC_HSYNCINWIDTH0_OFFSET                        (0x100a)
+#define HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_START              (0)
+#define HDMI_FC_HSYNCINWIDTH0_H_IN_WIDTH_MASK               (0xFF)
+
+#define HDMI_FC_HSYNCINWIDTH1_OFFSET                        (0x100b)
+#define HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_START        (0)
+#define HDMI_FC_HSYNCINWIDTH1_H_IN_WIDTH_UPPER_MASK         (0x3)
+
+#define HDMI_FC_VSYNCINDELAY_OFFSET                         (0x100c)
+#define HDMI_FC_VSYNCINDELAY_V_IN_DELAY_START               (0)
+#define HDMI_FC_VSYNCINDELAY_V_IN_DELAY_MASK                (0xFF)
+
+#define HDMI_FC_VSYNCINWIDTH_OFFSET                         (0x100d)
+#define HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_START               (0)
+#define HDMI_FC_VSYNCINWIDTH_V_IN_WIDTH_MASK                (0x3F)
+
+#define HDMI_FC_INFREQ0_OFFSET                              (0x100e)
+
+#define HDMI_FC_INFREQ1_OFFSET                              (0x100f)
+
+#define HDMI_FC_INFREQ2_OFFSET                              (0x1010)
+
+#define HDMI_FC_CTRLDUR_OFFSET                              (0x1011)
+
+#define HDMI_FC_EXCTRLDUR_OFFSET                            (0x1012)
+
+#define HDMI_FC_EXCTRLSPAC_OFFSET                           (0x1013)
+
+#define HDMI_FC_CH0PREAM_OFFSET                             (0x1014)
+
+#define HDMI_FC_CH1PREAM_OFFSET                             (0x1015)
+
+#define HDMI_FC_CH2PREAM_OFFSET                             (0x1016)
+
+#define HDMI_FC_AVICONF3_OFFSET                             (0x1017)
+
+#define HDMI_FC_GCP_OFFSET                                  (0x1018)
+
+#define HDMI_FC_AVICONF0_OFFSET                             (0x1019)
+#define HDMI_FC_AVICONF0_RGBYCC_START                       (0)
+#define HDMI_FC_AVICONF0_RGBYCC_MASK                        (0x3)
+#define HDMI_FC_AVICONF0_VBAR_VALID_START                   (2)
+#define HDMI_FC_AVICONF0_VBAR_VALID_MASK                    (0x4)
+#define HDMI_FC_AVICONF0_HBAR_VALID_START                   (3)
+#define HDMI_FC_AVICONF0_HBAR_VALID_MASK                    (0x8)
+#define HDMI_FC_AVICONF0_SCAN_INFO_START                    (4)
+#define HDMI_FC_AVICONF0_SCAN_INFO_MASK                     (0x30)
+#define HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_START        (6)
+#define HDMI_FC_AVICONF0_ACTIVE_FORMAT_PRESENT_MASK         (0x40)
+#define HDMI_FC_AVICONF0_RGBYCC_2_START                     (7)
+#define HDMI_FC_AVICONF0_RGBYCC_2_MASK                      (0x80)
+
+#define HDMI_FC_AVICONF1_OFFSET                             (0x101a)
+#define HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_START          (0)
+#define HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_MASK           (0xF)
+#define HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_START             (4)
+#define HDMI_FC_AVICONF1_PIC_ASPECT_RATIO_MASK              (0x30)
+#define HDMI_FC_AVICONF1_COLORIMETRY_START                  (6)
+#define HDMI_FC_AVICONF1_COLORIMETRY_MASK                   (0xC0)
+
+#define HDMI_FC_AVICONF2_OFFSET                             (0x101b)
+#define HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_START      (0)
+#define HDMI_FC_AVICONF2_NON_UNIFORM_PIC_SCALING_MASK       (0x3)
+#define HDMI_FC_AVICONF2_QUANTIZATION_RANGE_START           (2)
+#define HDMI_FC_AVICONF2_QUANTIZATION_RANGE_MASK            (0xC)
+#define HDMI_FC_AVICONF2_EXT_COLORIMETRY_START              (4)
+#define HDMI_FC_AVICONF2_EXT_COLORIMETRY_MASK               (0x70)
+#define HDMI_FC_AVICONF2_IT_CONTENT_START                   (7)
+#define HDMI_FC_AVICONF2_IT_CONTENT_MASK                    (0x80)
+
+#define HDMI_FC_AVIVID_OFFSET                               (0x101c)
+#define HDMI_FC_AVIVID_START                                (0)
+#define HDMI_FC_AVIVID_MASK                                 (0xFF)
+
+#define HDMI_FC_AVIETB0_OFFSET                              (0x101d)
+
+#define HDMI_FC_AVIETB1_OFFSET                              (0x101e)
+
+#define HDMI_FC_AVISBB0_OFFSET                              (0x101f)
+
+#define HDMI_FC_AVISBB1_OFFSET                              (0x1020)
+
+#define HDMI_FC_AVIELB0_OFFSET                              (0x1021)
+
+#define HDMI_FC_AVIELB1_OFFSET                              (0x1022)
+
+#define HDMI_FC_AVISRB0_OFFSET                              (0x1023)
+
+#define HDMI_FC_AVISRB1_OFFSET                              (0x1024)
+
+/* A lot of registers related to audio, ACP, ISRC, etc */
+
+#define HDMI_FC_PRCONF_OFFSET                               (0x10e0)
+#define HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_START               (0)
+#define HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK                (0xF)
+#define HDMI_FC_PRCONF_INCOMING_PR_FACTOR_START             (4)
+#define HDMI_FC_PRCONF_INCOMING_PR_FACTOR_MASK              (0xF0)
+
+#define HDMI_FC_SCRAMBLER_CTRL_OFFSET                       (0x10e1)
+
+#define HDMI_FC_MULTISTREAM_CTRL_OFFSET                     (0x10e2)
+
+#define HDMI_FC_PACKET_TX_EN_OFFSET                         (0x10e3)
+
+#define HDMI_FC_DBGFORCE_OFFSET                             (0x1200)
+#define HDMI_FC_DBGFORCE_FORCE_VIDEO_START                  (0)
+#define HDMI_FC_DBGFORCE_FORCE_VIDEO_MASK                   (0x1)
+#define HDMI_FC_DBGFORCE_FORCE_AUDIO_START                  (4)
+#define HDMI_FC_DBGFORCE_FORCE_AUDIO_MASK                   (0x10)
+
+#define HDMI_FC_DBGTMDS_0_OFFSET                            (0x1219)
+
+#define HDMI_FC_DBGTMDS_1_OFFSET                            (0x121A)
+
+#define HDMI_FC_DBGTMDS_2_OFFSET                            (0x121B)
+
+
+/* HDMI Source PHY */
+#define HDMI_PHY_CONF0_OFFSET                               (0x3000)
+#define HDMI_PHY_CONF0_SELDIPIF_START                       (0)
+#define HDMI_PHY_CONF0_SELDIPIF_MASK                        (0x1)
+#define HDMI_PHY_CONF0_SELDATAENPOL_START                   (1)
+#define HDMI_PHY_CONF0_SELDATAENPOL_MASK                    (0x2)
+#define HDMI_PHY_CONF0_ENHPDRXSENSE_START                   (2)
+#define HDMI_PHY_CONF0_ENHPDRXSENSE_MASK                    (0x4)
+#define HDMI_PHY_CONF0_TXPWRON_START                        (3)
+#define HDMI_PHY_CONF0_TXPWRON_MASK                         (0x8)
+#define HDMI_PHY_CONF0_PDDQ_START                           (4)
+#define HDMI_PHY_CONF0_PDDQ_MASK                            (0x10)
+#define HDMI_PHY_CONF0_SVSRET_START                         (5)
+#define HDMI_PHY_CONF0_SVSRET_MASK                          (0x20)
+#define HDMI_PHY_CONF0_ENTMDS_START                         (6)
+#define HDMI_PHY_CONF0_ENTMDS_MASK                          (0x40)
+#define HDMI_PHY_CONF0_PDZ_START                            (7)
+#define HDMI_PHY_CONF0_PDZ_MASK                             (0x80)
+
+#define HDMI_PHY_TST0_OFFSET                                (0x3001)
+
+#define HDMI_PHY_TST1_OFFSET                                (0x3002)
+
+#define HDMI_PHY_TST2_OFFSET                                (0x3003)
+
+#define HDMI_PHY_STAT0_OFFSET                               (0x3004)
+#define HDMI_PHY_STAT0_TX_PHY_LOCK_START                    (0)
+#define HDMI_PHY_STAT0_TX_PHY_LOCK_MASK                     (0x1)
+#define HDMI_PHY_STAT0_HPD_START                            (1)
+#define HDMI_PHY_STAT0_HPD_MASK                             (0x2)
+
+#define HDMI_PHY_INT0_OFFSET                                (0x3005)
+
+#define HDMI_PHY_MASK0_OFFSET                               (0x3006)
+
+#define HDMI_PHY_POL0_OFFSET                                (0x3007)
+#define HDMI_PHY_POL0_HPD_START                             (1)
+#define HDMI_PHY_POL0_HPD_MASK                              (0x2)
+
+/* I2C Master PHY */
+#define HDMI_PHY_I2CM_SLAVE_OFFSET                          (0x3020)
+#define HDMI_PHY_I2CM_SLAVE_SLAVEADDR_START                 (0)
+#define HDMI_PHY_I2CM_SLAVE_SLAVEADDR_MASK                  (0x7F)
+
+#define HDMI_PHY_I2CM_ADDRESS_OFFSET                        (0x3021)
+#define HDMI_PHY_I2CM_ADDRESS_START                         (0)
+#define HDMI_PHY_I2CM_ADDRESS_MASK                          (0xFF)
+
+#define HDMI_PHY_I2CM_DATAO_1_OFFSET                        (0x3022)
+#define HDMI_PHY_I2CM_DATAO_1_START                         (0)
+#define HDMI_PHY_I2CM_DATAO_1_MASK                          (0xFF)
+
+#define HDMI_PHY_I2CM_DATAO_0_OFFSET                        (0x3023)
+#define HDMI_PHY_I2CM_DATAO_0_START                         (0)
+#define HDMI_PHY_I2CM_DATAO_0_MASK                          (0xFF)
+
+#define HDMI_PHY_I2CM_DATAI_1_OFFSET                        (0x3024)
+#define HDMI_PHY_I2CM_DATAI_1_DATAI_START                   (0)
+#define HDMI_PHY_I2CM_DATAI_1_DATAI_MASK                    (0xFF)
+
+#define HDMI_PHY_I2CM_DATAI_0_OFFSET                        (0x3025)
+#define HDMI_PHY_I2CM_DATAI_0_DATAI_START                   (0)
+#define HDMI_PHY_I2CM_DATAI_0_DATAI_MASK                    (0xFF)
+
+#define HDMI_PHY_I2CM_OPERATION_OFFSET                      (0x3026)
+#define HDMI_PHY_I2CM_OPERATION_RD_START                    (0)
+#define HDMI_PHY_I2CM_OPERATION_RD_MASK                     (0x1)
+#define HDMI_PHY_I2CM_OPERATION_WR_START                    (4)
+#define HDMI_PHY_I2CM_OPERATION_WR_MASK                     (0x10)
+// Many more...
+
+/* Audio Sampler */
+#define HDMI_AUDIO_SAMPLER_OFFSET                           (0x3100)
+
+/* Audio packetizer */
+#define HDMI_AUD_N1_OFFSET                                  (0x3200)
+
+#define HDMI_AUD_N2_OFFSET                                  (0x3201)
+
+#define HDMI_AUD_N3_OFFSET                                  (0x3202)
+
+#define HDMI_AUD_CTS1_OFFSET                                (0x3203)
+
+#define HDMI_AUD_CTS2_OFFSET                                (0x3204)
+
+#define HDMI_AUD_CTS3_OFFSET                                (0x3205)
+
+#define HDMI_AUD_INPUTCLKFS_OFFSET                          (0x3206)
+
+/* Generic parallel audio interface */
+#define HDMI_PARALLEL_AUDIO_INTF_OFFFSET                    (0x3500)
+
+/* Audio DMA */
+#define HDMI_AUDIO_DMA_OFFSET                               (0x3600)
+
+/* Main Controller Registers */
+#define HDMI_MC_CLKDIS_OFFSET                               (0x4001)
+#define HDMI_MC_CLKDIS_PIXELCLK_DIS_START                   (0)
+#define HDMI_MC_CLKDIS_PIXELCLK_DIS_MASK                    (0x1)
+#define HDMI_MC_CLKDIS_TMDSCLK_DIS_START                    (1)
+#define HDMI_MC_CLKDIS_TMDSCLK_DIS_MASK                     (0x2)
+#define HDMI_MC_CLKDIS_PREPCLK_DIS_START                    (2)
+#define HDMI_MC_CLKDIS_PREPCLK_DIS_MASK                     (0x4)
+#define HDMI_MC_CLKDIS_AUDCLK_DIS_START                     (3)
+#define HDMI_MC_CLKDIS_AUDCLK_DIS_MASK                      (0x8)
+#define HDMI_MC_CLKDIS_CSCCLK_DIS_START                     (4)
+#define HDMI_MC_CLKDIS_CSCCLK_DIS_MASK                      (0x10)
+#define HDMI_MC_CLKDIS_CECCLK_DIS_START                     (5)
+#define HDMI_MC_CLKDIS_CECCLK_DIS_MASK                      (0x20)
+#define HDMI_MC_CLKDIS_HDCPCLK_DIS_START                    (6)
+#define HDMI_MC_CLKDIS_HDCPCLK_DIS_MASK                     (0x40)
+
+#define HDMI_MC_SWRSTZREQ_OFFSET                            (0x4002)
+#define HDMI_MC_OPCTRL_OFFSET                               (0x4003)
+#define HDMI_MC_FLOWCTRL_OFFSET                             (0x4004)
+
+#define HDMI_MC_PHYRSTZ_OFFSET                              (0x4005)
+#define HDMI_MC_PHYRSTZ_PHYRSTZ_START                       (0)
+#define HDMI_MC_PHYRSTZ_PHYRSTZ_MASK                        (0x1)
+
+#define HDMI_MC_LOCKONCLOCK_OFFSET                          (0x4006)
+#define HDMI_MC_HEACPHY_RST_OFFSET                          (0x4007)
+#define HDMI_MC_LOCKONCLOCK_2_OFFSET                        (0x4008)
+#define HDMI_MC_SWRSTZREQ_2_OFFSET                          (0x4009)
+
+/* Color Space Converter */
+#define HDMI_COLOR_SPACE_CONVERTER_OFFSET                   (0x4100)
+
+/* HDCP Encryption */
+#define HDMI_HDCP_ENCRYPTION_OFFSET                         (0x5000)
+
+/* I2C Master (E-EDID/EDDC) */
+#define HDMI_I2CM_SLAVE_OFFSET                              (0x7e00)
+
+#define HDMI_I2CM_ADDRESS_OFFSET                            (0x7e01)
+
+#define HDMI_I2CM_DATAO_OFFSET                              (0x7e02)
+
+#define HDMI_I2CM_DATAI_OFFSET                              (0x7e03)
+
+#define HDMI_I2CM_OPERATION_OFFSET                          (0x7e04)
+#define HDMI_I2CM_OPERATION_RD_START                        (0)
+#define HDMI_I2CM_OPERATION_RD_MASK                         (0x1)
+#define HDMI_I2CM_OPERATION_RD_EXT_START                    (1)
+#define HDMI_I2CM_OPERATION_RD_EXT_MASK                     (0x2)
+#define HDMI_I2CM_OPERATION_RD_8_START                      (2)
+#define HDMI_I2CM_OPERATION_RD_8_MASK                       (0x4)
+#define HDMI_I2CM_OPERATION_RD_8_EXT_START                  (3)
+#define HDMI_I2CM_OPERATION_RD_8_EXT_MASK                   (0x8)
+#define HDMI_I2CM_OPERATION_WR_START                        (4)
+#define HDMI_I2CM_OPERATION_WR_MASK                         (0x10)
+
+#define HDMI_I2CM_INT_OFFSET                                (0x7e05)
+#define HDMI_I2CM_INT_DONE_MASK_START                       (2)
+#define HDMI_I2CM_INT_DONE_MASK_MASK                        (0x4)
+#define HDMI_I2CM_INT_READ_REQ_MASK_START                   (6)
+#define HDMI_I2CM_INT_READ_REQ_MASK_MASK                    (0x40)
+
+#define HDMI_I2CM_CTLINT_OFFSET                             (0x7e06)
+#define HDMI_I2CM_CTLINT_ARB_MASK_START                     (2)
+#define HDMI_I2CM_CTLINT_ARB_MASK_MASK                      (0x4)
+#define HDMI_I2CM_CTLINT_NACK_MASK_START                    (6)
+#define HDMI_I2CM_CTLINT_NACK_MASK_MASK                     (0x40)
+
+#define HDMI_I2CM_DIV_OFFSET                                (0x7e07)
+#define HDMI_I2CM_DIV_FAST_STD_MODE_START                   (3)
+#define HDMI_I2CM_DIV_FAST_STD_MODE_MASK                    (0x8)
+
+#define HDMI_I2CM_SEGADDR_OFFSET                            (0x7e08)
+
+#define HDMI_I2CM_SOFTRSTZ_OFFSET                           (0x7e09)
+
+#define HDMI_I2CM_SEGPTR_OFFSET                             (0x7e0a)
+
+#define HDMI_I2CM_SS_SCL_HCNT_1_OFFSET                      (0x7e0b)
+
+#define HDMI_I2CM_SS_SCL_HCNT_0_OFFSET                      (0x7e0c)
+
+#define HDMI_I2CM_SS_SCL_LCNT_1_OFFSET                      (0x7e0d)
+
+#define HDMI_I2CM_SS_SCL_LCNT_0_OFFSET                      (0x7e0e)
+
+#define HDMI_I2CM_FS_SCL_HCNT_1_OFFSET                      (0x7e0f)
+
+#define HDMI_I2CM_FS_SCL_HCNT_0_OFFSET                      (0x7e10)
+
+#define HDMI_I2CM_FS_SCL_LCNT_1_OFFSET                      (0x7e11)
+
+#define HDMI_I2CM_FS_SCL_LCNT_0_OFFSET                      (0x7e12)
+
+#define HDMI_I2CM_SDA_HOLD_OFFSET                           (0x7e13)
+
+#define HDMI_I2CM_SCDC_READ_UPDATE_OFFSET                   (0x7e14)
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/phy.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/phy.h
new file mode 100644
index 000000000000..ffd5cf33ced8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/phy.h
@@ -0,0 +1,134 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+#ifndef DC_PHY_H
+#define DC_PHY_H
+
+#include "hdmi.h"
+
+/* PHY Gen2 slave addr */
+#if defined(EMULATOR)
+#define HDMI_PHY_SLAVE_ADDRESS                  0xA0
+#else
+#define HDMI_PHY_SLAVE_ADDRESS                  0x00
+#endif
+
+#if defined(VIRTUAL_PLATFORM)
+#define HDMI_PHY_TIMEOUT_MS                     1
+#define HDMI_PHY_POLL_INTERVAL_MS               1
+/* PHY dependent time for reset */
+#define HDMI_PHY_RESET_TIME_MS                  1
+#else
+/* Timeout for waiting on PHY operations */
+#define HDMI_PHY_TIMEOUT_MS                     200
+#define HDMI_PHY_POLL_INTERVAL_MS               100
+/* PHY dependent time for reset */
+#define HDMI_PHY_RESET_TIME_MS                  10
+#endif
+
+/* PHY I2C register definitions */
+#define PHY_PWRCTRL_OFFSET                      (0x00)
+#define PHY_SERDIVCTRL_OFFSET                   (0x01)
+#define PHY_SERCKCTRL_OFFSET                    (0x02)
+#define PHY_SERCKKILLCTRL_OFFSET                (0x03)
+#define PHY_TXRESCTRL_OFFSET                    (0x04)
+#define PHY_CKCALCTRL_OFFSET                    (0x05)
+#define PHY_OPMODE_PLLCFG_OFFSET                (0x06)
+#define PHY_CLKMEASCTRL_OFFSET                  (0x07)
+#define PHY_TXMEASCTRL_OFFSET                   (0x08)
+#define PHY_CKSYMTXCTRL_OFFSET                  (0x09)
+#define PHY_CMPSEQCTRL_OFFSET                   (0x0A)
+#define PHY_CMPPWRCTRL_OFFSET                   (0x0B)
+#define PHY_CMPMODECTRL_OFFSET                  (0x0C)
+#define PHY_MEASCTRL_OFFSET                     (0x0D)
+#define PHY_VLEVCTRL_OFFSET                     (0x0E)
+#define PHY_D2ACTRL_OFFSET                      (0x0F)
+#define PHY_PLLCURRCTRL_OFFSET                  (0x10)
+#define PHY_PLLDRVANACTRL_OFFSET                (0x11)
+#define PHY_PLLCTRL_OFFSET                      (0x14)
+#define PHY_PLLGMPCTRL_OFFSET                   (0x15)
+#define PHY_PLLMEASCTRL_OFFSET                  (0x16)
+#define PHY_PLLCLKBISTPHASE_OFFSET              (0x17)
+#define PHY_COMPRCAL_OFFSET                     (0x18)
+#define PHY_TXTERM_OFFSET                       (0x19)
+#define PHY_PWRSEQ_PATGENSKIP_OFFSET            (0x1A)
+/* Leaving out BIST regs */
+
+
+bool PhyIsPclkSupported(IMG_UINT16 pixelClock);
+PVRSRV_ERROR PhyI2CWrite(IMG_CPU_VIRTADDR base, IMG_UINT8 addr, IMG_UINT16 data);
+PVRSRV_ERROR PhyInitialize(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR PhyPowerDown(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR PhyWaitLock(HDMI_DEVICE * pvDevice);
+void PhyReset(HDMI_DEVICE * pvDevice, IMG_UINT8 value);
+void PhyConfigureMode(HDMI_DEVICE * pvDevice);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/video.h b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/video.h
new file mode 100644
index 000000000000..7670a92ce3bd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/display/rogue/plato_hdmi/include/video.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Copyright      Portions Copyright (c) Synopsys Ltd. All Rights Reserved
+@License        Synopsys Permissive License
+
+The Synopsys Software Driver and documentation (hereinafter "Software")
+is an unsupported proprietary work of Synopsys, Inc. unless otherwise
+expressly agreed to in writing between  Synopsys and you.
+
+The Software IS NOT an item of Licensed Software or Licensed Product under
+any End User Software License Agreement or Agreement for Licensed Product
+with Synopsys or any supplement thereto.  Permission is hereby granted,
+free of charge, to any person obtaining a copy of this software annotated
+with this license and the Software, to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject
+to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS" BASIS
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+*/ /**************************************************************************/
+#ifndef DC_VIDEO_H
+#define DC_VIDEO_H
+
+#include "hdmi.h"
+
+PVRSRV_ERROR VideoInitialize(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR VideoConfigureMode(HDMI_DEVICE * pvDevice);
+PVRSRV_ERROR VideoConfigureInfoframes(HDMI_DEVICE * pvDevice);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/dma_flags.h b/drivers/gpu/drm/img/img-volcanic/services/include/dma_flags.h
new file mode 100644
index 000000000000..fa6e77fa28c8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/dma_flags.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef DMA_FLAGS_H
+#define DMA_FLAGS_H
+
+/* These should match flags in pvrsrv_dma.h */
+#define DMA_FLAG_MEM_TO_DEV  (1U<<0)
+#define DMA_FLAG_DEV_TO_MEM  (0U<<0)
+
+#define DMA_FLAG_SYNCHRONOUS (1U<<1)
+
+#endif /* DMA_FLAGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_sf.h b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_sf.h
new file mode 100644
index 000000000000..83699a2d9547
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_sf.h
@@ -0,0 +1,272 @@
+/*************************************************************************/ /*!
+@File           htbuffer_sf.h
+@Title          Host Trace Buffer interface string format specifiers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the Host Trace Buffer logging messages. The following
+                list are the messages the host driver prints. Changing anything
+                but the first column or spelling mistakes in the strings will
+                break compatibility with log files created with older/newer
+                driver versions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_SF_H
+#define HTBUFFER_SF_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/******************************************************************************
+ * *DO*NOT* rearrange or delete lines in SFIDLIST or SFGROUPLIST or you
+ *          WILL BREAK host tracing message compatibility with previous
+ *          driver versions. Only add new ones, if so required.
+ *****************************************************************************/
+
+
+/* String used in pvrdebug -h output */
+#define HTB_LOG_GROUPS_STRING_LIST   "ctrl,mmu,sync,main,brg"
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define HTB_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s"
+
+/* Available log groups - Master template
+ *
+ * Group usage is as follows:
+ *    CTRL  - Internal Host Trace information and synchronisation data
+ *    MMU   - MMU page mapping information
+ *    SYNC  - Synchronisation debug
+ *    MAIN  - Data master kicks, etc. tying in with the MAIN group in FWTrace
+ *    DBG   - Temporary debugging group, logs not to be left in the driver
+ *
+ */
+#define HTB_LOG_SFGROUPLIST                               \
+	X( HTB_GROUP_NONE,     NONE  )                        \
+/*     gid,                group flag / apphint name */   \
+	X( HTB_GROUP_CTRL,     CTRL  )                        \
+	X( HTB_GROUP_MMU,      MMU   )                        \
+	X( HTB_GROUP_SYNC,     SYNC  )                        \
+	X( HTB_GROUP_MAIN,     MAIN  )                        \
+	X( HTB_GROUP_BRG,      BRG  )                         \
+/* Debug group HTB_GROUP_DBG must always be last */       \
+	X( HTB_GROUP_DBG,      DBG   )
+
+
+/* Table of String Format specifiers, the group they belong and the number of
+ * arguments each expects. Xmacro styled macros are used to generate what is
+ * needed without requiring hand editing.
+ *
+ * id		: unique id within a group
+ * gid		: group id as defined above
+ * sym name	: symbolic name of enumerations used to identify message strings
+ * string	: Actual string
+ * #args	: number of arguments the string format requires
+ */
+#define HTB_LOG_SFIDLIST \
+/*id,  gid,             sym name,                       string,                           # arguments */ \
+X( 0,  HTB_GROUP_NONE,  HTB_SF_FIRST,                   "You should not use this string", 0, 0) \
+\
+X( 1,  HTB_GROUP_CTRL,  HTB_SF_CTRL_LOGMODE,            "HTB log mode set to %d (1- all PID, 2 - restricted PID)\n", 1, 0) \
+X( 2,  HTB_GROUP_CTRL,  HTB_SF_CTRL_ENABLE_PID,         "HTB enable logging for PID %d\n", 1, 0) \
+X( 3,  HTB_GROUP_CTRL,  HTB_SF_CTRL_ENABLE_GROUP,       "HTB enable logging groups 0x%08x\n", 1, 0) \
+X( 4,  HTB_GROUP_CTRL,  HTB_SF_CTRL_LOG_LEVEL,          "HTB log level set to %d\n", 1, 0) \
+X( 5,  HTB_GROUP_CTRL,  HTB_SF_CTRL_OPMODE,             "HTB operating mode set to %d (1 - droplatest, 2 - drop oldest, 3 - block)\n", 1, 0) \
+X( 6,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_SCALE,       "HTBFWSync OSTS=%08x%08x CRTS=%08x%08x CalcClkSpd=%d\n", 5, 0) \
+X( 7,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_SCALE_RPT,   "FW Sync scale info OSTS=%08x%08x CRTS=%08x%08x CalcClkSpd=%d\n", 5, 0) \
+X( 8,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK,        "FW Sync Partition marker: %d\n", 1, 0) \
+X( 9,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK_RPT,    "FW Sync Partition repeat: %d\n", 1, 0) \
+X( 10, HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK_SCALE,  "Text not used", 6, 0)\
+\
+X( 1,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_TABLE,       "MMU page op table entry page_id=%08x%08x index=%d level=%d val=%08x%08x map=%d\n", 7, 0) \
+X( 2,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_ALLOC,       "MMU allocating DevVAddr from %08x%08x to %08x%08x\n", 4, 0) \
+X( 3,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_FREE,        "MMU freeing DevVAddr from %08x%08x to %08x%08x\n", 4, 0) \
+X( 4,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_MAP,         "MMU mapping DevVAddr %08x%08x to DevPAddr %08x%08x\n", 4, 0) \
+X( 5,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_PMRMAP,      "MMU mapping PMR DevVAddr %08x%08x to DevPAddr %08x%08x\n", 4, 0)  \
+X( 6,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_UNMAP,       "MMU unmapping DevVAddr %08x%08x\n", 2, 0) \
+\
+X( 1,  HTB_GROUP_SYNC,  HTB_SF_SYNC_SERVER_ALLOC,       "Server sync allocation [%08X]\n", 1, 0) \
+X( 2,  HTB_GROUP_SYNC,  HTB_SF_SYNC_SERVER_UNREF,       "Server sync unreferenced [%08X]\n", 1, 0) \
+X( 3,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_CREATE,     "Sync OP create 0x%08x, block count=%d, server syncs=%d, client syncs=%d\n", 4, 0) \
+X( 4,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_TAKE,       "Sync OP take 0x%08x server syncs=%d, client syncs=%d\n", 3, 0) \
+X( 5,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_COMPLETE,   "Sync OP complete 0x%08x\n", 1, 0) \
+X( 6,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_DESTROY,    "Sync OP destroy 0x%08x\n", 1, 0) \
+\
+X( 1,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_TA_DEPRECATED, "Kick TA: FWCtx %08X @ %d\n", 2, 0) \
+X( 2,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_3D_DEPRECATED, "Kick 3D: FWCtx %08X @ %d\n", 2, 0) \
+X( 3,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_CDM_DEPRECATED,"Kick CDM: FWCtx %08X @ %d\n", 2, 0) \
+X( 4,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_RTU,           "Kick RTU: FWCtx %08X @ %d\n", 2, 0) \
+X( 5,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_SHG,           "Kick SHG: FWCtx %08X @ %d\n", 2, 0) \
+X( 6,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_2D_DEPRECATED, "Kick 2D: FWCtx %08X @ %d\n", 2, 0) \
+X( 7,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_UNCOUNTED,     "Kick (uncounted) for all DMs\n", 0, 0) \
+X( 8,  HTB_GROUP_MAIN,  HTB_SF_MAIN_FWCCB_CMD,          "FW CCB Cmd: %d\n", 1, 0) \
+X( 9,  HTB_GROUP_MAIN,  HTB_SF_MAIN_PRE_POWER,          "Pre-power duration @ phase [%d] (0-shutdown,1-startup) RGX: %llu ns SYS: %llu ns\n", 3, 0) \
+X(10,  HTB_GROUP_MAIN,  HTB_SF_MAIN_POST_POWER,         "Post-power duration @ phase [%d] (0-shutdown,1-startup) SYS: %llu ns RGX: %llu ns\n", 3, 0) \
+X(11,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_TA,            "Kick TA: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(12,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_3D,            "Kick 3D: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(13,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_CDM,           "Kick CDM: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(14,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_2D,            "Kick 2D: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(15,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_ERROR,          "Error: (%u) in file: %s:%u\n", 3, 2) \
+X(16,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_MSGLVL_ERROR,   "Log error in file: %s:%u\n", 2, 1) \
+X(17,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_COND_ERROR_T,   "Error: (%u) Conditional is unexpectedly true in file: %s:%u\n", 3, 2) \
+X(18,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_COND_ERROR_F,   "Error: (%u) Conditional is unexpectedly false in file: %s:%u\n", 3, 2) \
+X(19,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_WARNING,        "Warning: (%u) in file: %s @ line: %u\n", 3, 2) \
+X(20,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_MSGLVL_WARN,    "Log warning in file: %s:%u\n", 2, 1) \
+\
+X( 1,  HTB_GROUP_BRG,   HTB_SF_BRG_BRIDGE_CALL,         "Bridge call: start: %010u: bid %03d fid %d\n", 3, 0) \
+X( 2,  HTB_GROUP_BRG,   HTB_SF_BRG_BRIDGE_CALL_ERR,     "Bridge call: start: %010u: bid %03d fid %d error %d\n", 4, 0) \
+\
+X( 1,  HTB_GROUP_DBG,   HTB_SF_DBG_INTPAIR,             "0x%8.8x 0x%8.8x\n", 2, 0) \
+\
+X( 65535, HTB_GROUP_NONE, HTB_SF_LAST,                  "You should not use this string\n", 15, 0)
+
+
+
+/* gid - Group numbers */
+typedef enum _HTB_LOG_SFGROUPS {
+#define X(A,B) A,
+	HTB_LOG_SFGROUPLIST
+#undef X
+} HTB_LOG_SFGROUPS;
+
+
+/* Group flags are stored in an array of elements.
+ * Each of which have a certain number of bits.
+ */
+#define HTB_FLAG_EL_T                   IMG_UINT32
+#define HTB_FLAG_NUM_BITS_IN_EL         (sizeof(HTB_FLAG_EL_T) * 8)
+
+#define HTB_LOG_GROUP_FLAG_GROUP(gid)   ((gid-1) / HTB_FLAG_NUM_BITS_IN_EL)
+#define HTB_LOG_GROUP_FLAG(gid)         (gid ? (0x1 << ((gid-1)%HTB_FLAG_NUM_BITS_IN_EL)) : 0)
+#define HTB_LOG_GROUP_FLAG_NAME(gid)    HTB_LOG_TYPE_ ## gid
+
+/* Group enable flags */
+typedef enum _HTB_LOG_TYPE {
+#define X(a, b) HTB_LOG_GROUP_FLAG_NAME(b) = HTB_LOG_GROUP_FLAG(a),
+	HTB_LOG_SFGROUPLIST
+#undef X
+} HTB_LOG_TYPE;
+
+
+
+/* The symbolic names found in the table above are assigned an ui32 value of
+ * the following format:
+ * 31 30 28 27       20   19  16    15  12      11            0   bits
+ * -   ---   ---- ----     ----      ----        ---- ---- ----
+ *    0-11: id number
+ *   12-15: group id number
+ *   16-19: number of parameters
+ *   20-24: If the format contains a string argument, argument
+ *          index is passed. If there isn't a string argument, 0 is passed
+ *   25-27: unused
+ *   28-30: active: identify SF packet, otherwise regular int32
+ *      31: reserved for signed/unsigned compatibility
+ *
+ * The following macro assigns those values to the enum generated SF ids list.
+ */
+#define HTB_LOG_IDMARKER            (0x70000000)
+#define HTB_LOG_CREATESFID(a,b,e,f)   (((a) | (b << 12) | (e << 16)) | (f << 20) | HTB_LOG_IDMARKER)
+
+#define HTB_LOG_IDMASK              (0xFFF00000)
+#define HTB_LOG_VALIDID(I)          ( ((I) & HTB_LOG_IDMASK) == HTB_LOG_IDMARKER )
+
+typedef enum HTB_LOG_SFids {
+#define X(a, b, c, d, e, f) c = HTB_LOG_CREATESFID(a,b,e,f),
+	HTB_LOG_SFIDLIST
+#undef X
+} HTB_LOG_SFids;
+
+/* Return the group id that the given (enum generated) id belongs to */
+#define HTB_SF_GID(x) (((x)>>12) & 0xf)
+/* Future improvement to support log levels */
+#define HTB_SF_LVL(x) (0)
+/* Returns how many arguments the SF(string format) for the given
+ * (enum generated) id requires.
+ */
+#define HTB_SF_PARAMNUM(x) (((x)>>16) & 0xf)
+/* Returns the index of the argument that contains a string, 1 indexed so 0 means no string argument.
+ */
+#define HTB_SF_STRNUM(x) (((x)>>20) & 0xf)
+/* Returns the id of given enum */
+#define HTB_SF_ID(x) (x & 0xfff)
+
+/* Format of messages is: SF:PID:TID:TIMEPT1:TIMEPT2:[PARn]*
+ */
+#define HTB_LOG_HEADER_SIZE         5
+#define HTB_LOG_MAX_PARAMS          15
+
+/* HTB supports string lengths of up to 23 characters + \0 */
+#define HTB_LOG_STR_ARG_NUM_WORDS         (6)
+#define HTB_LOG_STR_ARG_SIZE              (HTB_LOG_STR_ARG_NUM_WORDS * sizeof(IMG_UINT32))
+
+#if defined(__cplusplus)
+}
+#endif
+
+#if defined(__GNUC__)
+#if GCC_VERSION_AT_LEAST(12, 1)
+#define HTB_FILE_NAME __FILE_NAME__
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+#elif defined(__clang__)
+#if CLANG_VERSION_AT_LEAST(10)
+#define HTB_FILE_NAME __FILE_NAME__
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+
+/* Defines for handling MARK_SCALE special case */
+#define HTB_GID_CTRL 1
+#define HTB_ID_MARK_SCALE 10
+#define HTB_MARK_SCALE_ARG_ARRAY_SIZE 6
+
+/* Defines for extracting args from array for special case MARK_SCALE */
+#define HTB_ARG_SYNCMARK 0
+#define HTB_ARG_OSTS_PT1 1
+#define HTB_ARG_OSTS_PT2 2
+#define HTB_ARG_CRTS_PT1 3
+#define HTB_ARG_CRTS_PT2 4
+#define HTB_ARG_CLKSPD   5
+
+#endif /* HTBUFFER_SF_H */
+/*****************************************************************************
+ End of file (htbuffer_sf.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_types.h b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_types.h
new file mode 100644
index 000000000000..fd061eb428d7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/htbuffer_types.h
@@ -0,0 +1,122 @@
+/*************************************************************************/ /*!
+@File           htbuffer_types.h
+@Title          Host Trace Buffer types.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_TYPES_H
+#define HTBUFFER_TYPES_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+#include "htbuffer_sf.h"
+
+/* The group flags array of ints large enough to store all the group flags */
+#if defined(PVRSRV_ENABLE_HTB)
+#define HTB_FLAG_NUM_EL (((HTB_GROUP_DBG-1) / HTB_FLAG_NUM_BITS_IN_EL) + 1)
+extern IMG_INTERNAL HTB_FLAG_EL_T g_auiHTBGroupEnable[HTB_FLAG_NUM_EL];
+
+#define HTB_GROUP_ENABLED(SF) (g_auiHTBGroupEnable[HTB_LOG_GROUP_FLAG_GROUP(HTB_SF_GID(SF))] & HTB_LOG_GROUP_FLAG(HTB_SF_GID(SF)))
+#else
+#define HTB_GROUP_ENABLED(SF) IMG_FALSE
+#endif
+
+/*************************************************************************/ /*!
+ Host Trace Buffer operation mode
+ Care must be taken if changing this enum to ensure the MapFlags[] array
+ in htbserver.c is kept in-step.
+*/ /**************************************************************************/
+typedef enum
+{
+	/*! Undefined operation mode */
+	HTB_OPMODE_UNDEF = 0,
+
+	/*! Drop latest, intended for continuous logging to a UM daemon.
+	 *  If the daemon does not keep up, the most recent log data
+	 *  will be dropped
+	 */
+	HTB_OPMODE_DROPLATEST,
+
+	/*! Drop oldest, intended for crash logging.
+	 *  Data will be continuously written to a circular buffer.
+	 *  After a crash the buffer will contain events leading up to the crash
+	 */
+	HTB_OPMODE_DROPOLDEST,
+
+	/*! Block write if buffer is full */
+	HTB_OPMODE_BLOCK,
+
+	HTB_OPMODE_LAST = HTB_OPMODE_BLOCK
+} HTB_OPMODE_CTRL;
+
+
+/*************************************************************************/ /*!
+ Host Trace Buffer log mode control
+*/ /**************************************************************************/
+typedef enum
+{
+	/*! Undefined log mode, used if update is not applied */
+	HTB_LOGMODE_UNDEF = 0,
+
+	/*! Log trace messages for all PIDs. */
+	HTB_LOGMODE_ALLPID,
+
+	/*! Log trace messages for specific PIDs only. */
+	HTB_LOGMODE_RESTRICTEDPID,
+
+	HTB_LOGMODE_LAST = HTB_LOGMODE_RESTRICTEDPID
+} HTB_LOGMODE_CTRL;
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_TYPES_H */
+
+/******************************************************************************
+ End of file (htbuffer_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/img_types_check.h b/drivers/gpu/drm/img/img-volcanic/services/include/img_types_check.h
new file mode 100644
index 000000000000..ce73789877f6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/img_types_check.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Performs size checks on some of the IMG types.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_TYPES_CHECK_H
+#define IMG_TYPES_CHECK_H
+
+#ifndef __KERNEL__
+#include <assert.h>
+#endif /* __KERNEL__ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+static_assert(sizeof(IMG_BOOL) == 1, "invalid size of IMG_BOOL");
+static_assert(sizeof(IMG_INT) == 4, "invalid size of IMG_INT");
+static_assert(sizeof(IMG_UINT) == 4, "invalid size of IMG_UINT");
+static_assert(sizeof(PVRSRV_ERROR) == 4, "invalid size of PVRSRV_ERROR");
+
+#endif /* IMG_TYPES_CHECK_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/info_page_client.h b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_client.h
new file mode 100644
index 000000000000..56d394dd99a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_client.h
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose shared memory (i.e. information page) mapped by
+                kernel space driver and user space clients. All info page
+                entries are sizeof(IMG_UINT32) on both 32/64-bit environments.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_CLIENT_H
+#define INFO_PAGE_CLIENT_H
+
+#include "device_connection.h"
+#include "info_page_defs.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#endif
+
+/*************************************************************************/ /*!
+@Function      GetInfoPage
+
+@Description   Return Info Page address
+
+@Input         hDevConnection - Services device connection
+
+@Return        Info Page address
+*/
+/*****************************************************************************/
+static INLINE IMG_PUINT32 GetInfoPage(SHARED_DEV_CONNECTION hDevConnection)
+{
+#if defined(__KERNEL__)
+	PVR_UNREFERENCED_PARAMETER(hDevConnection);
+
+	return (PVRSRVGetPVRSRVData())->pui32InfoPage;
+#else
+	return hDevConnection->pui32InfoPage;
+#endif
+}
+
+/*************************************************************************/ /*!
+@Function      GetInfoPageDebugFlags
+
+@Description   Return Info Page debug flags
+
+@Input         hDevConnection - Services device connection
+
+@Return        Info Page debug flags
+*/
+/*****************************************************************************/
+static INLINE IMG_UINT32 GetInfoPageDebugFlags(SHARED_DEV_CONNECTION hDevConnection)
+{
+	return GetInfoPage(hDevConnection)[DEBUG_FEATURE_FLAGS];
+}
+
+#endif /* INFO_PAGE_CLIENT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/info_page_defs.h b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_defs.h
new file mode 100644
index 000000000000..949afcfd869d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/info_page_defs.h
@@ -0,0 +1,133 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose shared memory (i.e. information page) mapped by
+                kernel space driver and user space clients. All information page
+                entries are sizeof(IMG_UINT32) on both 32/64-bit environments.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_DEFS_H
+#define INFO_PAGE_DEFS_H
+
+/* Info page is divided in "blocks" of size INFO_PAGE_CHUNK_SIZE. Each block
+ * should start with the INFO_PAGE_[NAME]_BLOCK_START macro which takes the
+ * value of previous block (except for the first block which starts from 0).
+ *
+ * Each last value of the block (INFO_PAGE_[NAME]_BLOCK_END) should be unused
+ * within that block since it's a first value of the next block. This value
+ * should be a multiple of INFO_PAGE_CHUNK_SIZE.
+ *
+ * Care must be taken to not go over allowed number of elements in each block
+ * which is marked with the INFO_PAGE_[NAME]_BLOCK_END macro.
+ *
+ * Blocks consist of entries that are defined with the INFO_PAGE_ENTRY() macro.
+ * Each entry must define a unique index within the block and as mentioned
+ * can't go over the INFO_PAGE_[NAME]_BLOCK_END limit.
+ *
+ * Always add blocks to the end of the existing list and update
+ * INFO_PAGE_TOTAL_SIZE after.
+ *
+ * See current usage of the Info Page below for examples.
+ */
+
+#define INFO_PAGE_CHUNK_SIZE                        8
+#define INFO_PAGE_BLOCK_END(start,size)             ((start) + (size) * INFO_PAGE_CHUNK_SIZE)
+#define INFO_PAGE_ENTRY(start,index)                ((start) + (index))
+#define INFO_PAGE_SIZE_IN_BYTES(end)                ((end) * sizeof(IMG_UINT32))
+
+#define INFO_PAGE_CACHEOP_BLOCK_START               0
+#define INFO_PAGE_CACHEOP_BLOCK_END                 INFO_PAGE_BLOCK_END(INFO_PAGE_CACHEOP_BLOCK_START, 1)
+#define INFO_PAGE_HWPERF_BLOCK_START                INFO_PAGE_CACHEOP_BLOCK_END
+#define INFO_PAGE_HWPERF_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_HWPERF_BLOCK_START, 1)
+#define INFO_PAGE_TIMEOUT_BLOCK_START               INFO_PAGE_HWPERF_BLOCK_END
+#define INFO_PAGE_TIMEOUT_BLOCK_END                 INFO_PAGE_BLOCK_END(INFO_PAGE_TIMEOUT_BLOCK_START, 2)
+#define INFO_PAGE_BRIDGE_BLOCK_START                INFO_PAGE_TIMEOUT_BLOCK_END
+#define INFO_PAGE_BRIDGE_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_BRIDGE_BLOCK_START, 1)
+#define INFO_PAGE_DEBUG_BLOCK_START                 INFO_PAGE_BRIDGE_BLOCK_END
+#define INFO_PAGE_DEBUG_BLOCK_END                   INFO_PAGE_BLOCK_END(INFO_PAGE_DEBUG_BLOCK_START, 1)
+#define INFO_PAGE_DEVMEM_BLOCK_START                INFO_PAGE_DEBUG_BLOCK_END
+#define INFO_PAGE_DEVMEM_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_DEVMEM_BLOCK_START, 1)
+
+/* IMPORTANT: Make sure this always uses the last INFO_PAGE_[NAME]_BLOCK_END definition.*/
+#define INFO_PAGE_TOTAL_SIZE                        INFO_PAGE_SIZE_IN_BYTES(INFO_PAGE_DEVMEM_BLOCK_END)
+
+/* CacheOp information page entries */
+
+#define CACHEOP_INFO_UMKMTHRESHLD                   INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 0) /*!< UM=>KM routing threshold in bytes */
+#define CACHEOP_INFO_KMDFTHRESHLD                   INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 1) /*!< KM/DF threshold in bytes */
+#define CACHEOP_INFO_LINESIZE                       INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 2) /*!< CPU data cache line size */
+#define CACHEOP_INFO_PGSIZE                         INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 3) /*!< CPU MMU page size */
+
+/* HWPerf information page entries */
+
+#define HWPERF_FILTER_SERVICES_IDX                  INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 0)
+#define HWPERF_FILTER_EGL_IDX                       INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 1)
+#define HWPERF_FILTER_OPENGLES_IDX                  INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 2)
+#define HWPERF_FILTER_OPENCL_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 3)
+#define HWPERF_FILTER_VULKAN_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 4)
+#define HWPERF_FILTER_OPENGL_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 5)
+
+/* Timeout values */
+
+#define TIMEOUT_INFO_VALUE_RETRIES                  INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 0)
+#define TIMEOUT_INFO_VALUE_TIMEOUT_MS               INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 1)
+#define TIMEOUT_INFO_CONDITION_RETRIES              INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 2)
+#define TIMEOUT_INFO_CONDITION_TIMEOUT_MS           INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 3)
+#define TIMEOUT_INFO_TASK_QUEUE_RETRIES             INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 4)
+#define TIMEOUT_INFO_TASK_QUEUE_FLUSH_TIMEOUT_MS    INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 5)
+
+/* Bridge Info */
+
+#define BRIDGE_INFO_RGX_BRIDGES                     INFO_PAGE_ENTRY(INFO_PAGE_BRIDGE_BLOCK_START, 0)
+#define BRIDGE_INFO_PVR_BRIDGES                     INFO_PAGE_ENTRY(INFO_PAGE_BRIDGE_BLOCK_START, 1)
+
+/* Debug features */
+
+#define DEBUG_FEATURE_FLAGS                         INFO_PAGE_ENTRY(INFO_PAGE_DEBUG_BLOCK_START, 0)
+
+#define DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED    0x1 /* flag - not part of info page */
+#define DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED      0x2 /* flag - not part of info page */
+
+/* Device memory related information */
+
+/* This value is 64-bits wide, next value should have index larger by 2 */
+#define DEVMEM_INFO_PHYS_BUF_MAX_SIZE               INFO_PAGE_ENTRY(INFO_PAGE_DEVMEM_BLOCK_START, 0)
+
+#endif /* INFO_PAGE_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/kerneldisplay.h b/drivers/gpu/drm/img/img-volcanic/services/include/kerneldisplay.h
new file mode 100644
index 000000000000..df6fc23cb676
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/kerneldisplay.h
@@ -0,0 +1,832 @@
+/*************************************************************************/ /*!
+@File
+@Title          Interface between 3rd party display controller (DC) drivers
+                and the Services server module.
+@Description    API between Services and the 3rd party DC driver and vice versa.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(KERNELDISPLAY_H)
+#define KERNELDISPLAY_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include <powervr/mem_types.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "pvrsrv_surface.h"
+#include "dc_external.h"
+
+/*!
+ ******************************************************************************
+ @mainpage Introduction and Document Scope
+This document is a function reference for the 3rd party display
+controller interface. The interface ensures that the 3rd party display driver and
+Services can exchange configurations and buffers and can inform each
+other about occurring events like render completion or vsyncs.
+
+This interface should be used and implemented if the 3rd party display driver
+is not using a native OS interface such as the Linux DRM/KMS API.
+
+To fully understand the document the reader should be aware that
+the Services driver - offering PowerVR core and platform functionality to the
+driver layers (OGLES etc.) above - is split into the "Services Client"
+user mode library and the "Services Server" kernel mode driver that communicate
+via the "Services Bridge". The terms "User-Mode" and "Client" are used as
+synonyms as well as "Kernel-Mode" and "Server".
+
+Please refer to the more comprehensive '3rd Party Display Integration Guide'
+for an architecture overview and an explanation of the data flow between a
+client process, Services and the 3rd party display driver. It also contains
+descriptions about how to make use of the client side interface that is supposed
+to be integrated in some kind of display manager like e.g. PowerVR Graphics DDK WSEGL
+window manager.
+
+The documented functions are split into different parts:
+- Callbacks that need an implementation by the 3rd party display driver and
+that are used by the Services server, some of them optional (kerneldisplay.h)
+- Functions that the Services server module exports and can be used by the
+3rd party display driver. Mainly to register/deregister a new display device
+and to query state information (kerneldisplay.h)
+- Functions that are called by the client process (a display manager e.g. WSEGL)
+to control the DC interaction (dc_client.h)
+- Commonly used structure definitions to exchange data between the modules
+(dc_external.h, pvrsrv_surface.h)
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       GetInfo
+
+@Description    Query the display controller for its information structure.
+
+   Called by client function: #PVRSRVDCGetInfo()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Output         psDisplayInfo           Display info structure
+*/
+/*****************************************************************************/
+typedef void (*GetInfo)(IMG_HANDLE hDeviceData,
+                        DC_DISPLAY_INFO *psDisplayInfo);
+
+/*************************************************************************/ /*!
+@Function       PanelQueryCount
+
+@Description    Query the display controller for how many panels are
+                connected to it.
+
+   Called by client function: #PVRSRVDCPanelQueryCount()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData            Device private data
+
+@Output         pui32NumPanels         Number of panels
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*PanelQueryCount)(IMG_HANDLE hDeviceData,
+                                        IMG_UINT32 *pui32NumPanels);
+
+/*************************************************************************/ /*!
+@Function       PanelQuery
+
+@Description    Query the display controller for information on what panel(s)
+                are connected to it and their properties.
+
+   Called by client function: #PVRSRVDCPanelQuery()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Input          ui32PanelsArraySize     Size of the PanelInfo array
+                                        (i.e. number of panels that
+                                        can be returned)
+
+@Output         pui32NumPanels          Number of panels returned
+
+@Output         pasPanelInfo            Array of formats, allocated beforehand
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*PanelQuery)(IMG_HANDLE hDeviceData,
+                                   IMG_UINT32 ui32PanelsArraySize,
+                                   IMG_UINT32 *pui32NumPanels,
+                                   PVRSRV_PANEL_INFO *pasPanelInfo);
+
+/*************************************************************************/ /*!
+@Function       FormatQuery
+
+@Description    Query the display controller to check if it supports the specified
+                format(s).
+
+   Called by client function: #PVRSRVDCFormatQuery()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Input          ui32NumFormats          Number of formats to check
+                                        (i.e. length of the Format array)
+
+@Input          pasFormat               Array of formats to check
+
+@Output         pui32Supported          For each format, the number of display
+                                        pipes that support that format
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*FormatQuery)(IMG_HANDLE hDeviceData,
+                                    IMG_UINT32 ui32NumFormats,
+                                    PVRSRV_SURFACE_FORMAT *pasFormat,
+                                    IMG_UINT32 *pui32Supported);
+
+/*************************************************************************/ /*!
+@Function       DimQuery
+
+@Description    Query the specified display plane for the display dimensions
+                it supports.
+
+   Called by client function: #PVRSRVDCDimQuery()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Input          ui32NumDims             Number of dimensions to check
+                                        (i.e. length of the Dim array)
+
+@Input          pasDim                  Array of dimensions to check
+
+@Output         pui32Supported          For each dimension, the number of
+                                        display pipes that support that
+                                        dimension
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*DimQuery)(IMG_HANDLE hDeviceData,
+                                 IMG_UINT32 ui32NumDims,
+                                 PVRSRV_SURFACE_DIMS *pasDim,
+                                 IMG_UINT32 *pui32Supported);
+
+/*************************************************************************/ /*!
+@Function       SetBlank
+
+@Description    Enable/disable blanking of the screen.
+
+   Called by client function: #PVRSRVDCSetBlank()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Input          bEnable                 Enable/Disable the blanking
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*SetBlank)(IMG_HANDLE hDeviceData,
+                                 IMG_BOOL bEnable);
+
+/*************************************************************************/ /*!
+@Function       SetVSyncReporting
+
+@Description    Enable VSync reporting. If enabled, the 3rd party display
+                driver is expected to call PVRSRVCheckStatus() after a VSync
+                event occurred. This will signal the Services driver global
+                event object.
+
+   Called by client function: #PVRSRVDCSetVSyncReporting()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Input          bEnable                 Enable/Disable the reporting
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*SetVSyncReporting)(IMG_HANDLE hDeviceData,
+                                          IMG_BOOL bEnable);
+
+/*************************************************************************/ /*!
+@Function       LastVSyncQuery
+
+@Description    Query the time the last vsync happened.
+
+   Called by client function: #PVRSRVDCLastVSyncQuery()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Output         pi64Timestamp           The requested timestamp
+                                        of the system time in ns
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*LastVSyncQuery)(IMG_HANDLE hDeviceData,
+                                       IMG_INT64 *pi64Timestamp);
+
+/*************************************************************************/ /*!
+@Function       ContextCreate
+
+@Description    Create display context.
+                The client application and the display driver have to agree
+                as to what exactly a context represents, Services will just
+                pass it through and tie it to its own concept of a
+                display context.
+                It might contain additional locks, work-queues or other
+                important data that is necessary to drive the interaction
+                with Services. A context is usually associated with one device
+                but that is not a hard requirement.
+
+   Called by client function: #PVRSRVDCDisplayContextCreate()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDeviceData             Device private data
+
+@Output         hDisplayContext         Created display context
+
+@Return         PVRSRV_OK if the context was created
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*ContextCreate)(IMG_HANDLE hDeviceData,
+                                      IMG_HANDLE *hDisplayContext);
+
+/*************************************************************************/ /*!
+@Function       ContextConfigureCheck
+
+@Description    Check to see if a configuration is valid for the display
+                controller. Because of runtime changes the display context
+                might not be able to work with a certain configuration anymore.
+                This function is intended to be called before ContextConfigure
+                so that the application can query details about the current
+                surface config and then do the ContextConfigure call with
+                the updated data.
+                The arrays should be z-sorted, with the farthest plane
+                first and the nearest plane last.
+
+   Called by client function: #PVRSRVDCContextConfigureCheck()
+
+   Implementation of this callback is optional.
+
+@Input          hDisplayContext         Display context
+
+@Input          ui32PipeCount           Number of display pipes to configure
+                                        (length of the input arrays)
+
+@Input          pasSurfAttrib           Array of surface attributes (one for
+                                        each display plane)
+
+@Input          ahBuffers               Array of buffers (one for
+                                        each display plane)
+
+@Return         PVRSRV_OK if the configuration is valid
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*ContextConfigureCheck)(IMG_HANDLE hDisplayContext,
+                                              IMG_UINT32 ui32PipeCount,
+                                              PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+                                              IMG_HANDLE *ahBuffers);
+
+/*************************************************************************/ /*!
+@Function       ContextConfigure
+
+@Description    Configure the display pipeline to display a given buffer.
+                The arrays should be z-sorted, with the farthest plane first
+                and the nearest plane last.
+
+   Called by client function: #PVRSRVDCContextConfigureWithFence()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDisplayContext         Display context
+
+@Input          ui32PipeCount           Number of display pipes to configure
+                                        (i.e. length of the input arrays)
+
+@Input          pasSurfAttrib           Array of surface attributes (one for
+                                        each display plane)
+
+@Input          ahBuffers               Array of buffers (one for
+                                        each display plane)
+
+@Input          ui32DisplayPeriod       The number of VSync periods this
+                                        configuration should be displayed for
+
+@Input          hConfigData             Config handle which gets passed to
+                                        DisplayConfigurationRetired when this
+                                        configuration is retired
+*/
+/*****************************************************************************/
+typedef void (*ContextConfigure)(IMG_HANDLE hDisplayContext,
+                                 IMG_UINT32 ui32PipeCount,
+                                 PVRSRV_SURFACE_CONFIG_INFO *pasSurfAttrib,
+                                 IMG_HANDLE *ahBuffers,
+                                 IMG_UINT32 ui32DisplayPeriod,
+                                 IMG_HANDLE hConfigData);
+
+/*************************************************************************/ /*!
+@Function       ContextDestroy
+
+@Description    Destroy a display context.
+
+   Called by client function: #PVRSRVDCDisplayContextDestroy()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDisplayContext         Display context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+typedef void (*ContextDestroy)(IMG_HANDLE hDisplayContext);
+
+/*************************************************************************/ /*!
+@Function       BufferAlloc
+
+@Description    Allocate a display buffer. This is a request to the display
+                controller to allocate a buffer from memory that is addressable
+                by the display controller.
+
+                Note: The actual allocation of display memory can be deferred
+                until the first call to acquire, but the handle for the buffer
+                still needs to be created and returned to the caller as well
+                as some information about the buffer that's required upfront.
+
+   Called by client function: #PVRSRVDCBufferAlloc()
+
+   Implementation of this callback is mandatory.
+
+@Input          hDisplayContext         Display context this buffer will be
+                                        used on
+
+@Input          psSurfInfo              Attributes of the buffer
+
+@Output         puiLog2PageSize         Pagesize in log2(bytes) of one page
+
+@Output         pui32PageCount          Number of pages in the buffer
+
+@Output         pui32ByteStride         Stride (in bytes) of allocated buffer
+
+@Output         phBuffer                Handle to allocated buffer
+
+@Return         PVRSRV_OK if the buffer was successfully allocated
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferAlloc)(IMG_HANDLE hDisplayContext,
+                                    DC_BUFFER_CREATE_INFO *psSurfInfo,
+                                    IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+                                    IMG_UINT32 *pui32PageCount,
+                                    IMG_UINT32 *pui32ByteStride,
+                                    IMG_HANDLE *phBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferImport
+
+@Description    Import memory allocated from an external source (e.g. Services)
+                to the display controller. The DC checks to see if the import
+                is compatible and potentially sets up HW to map the imported
+                buffer, although this isn't required to happen until the first
+                call to BufferMap.
+
+                Note: Provide this function if the controller
+                can scan out arbitrary memory, allocated for another purpose by
+                Services. To be able to use this buffer (depending on its
+                origin) the display controller probably will need a MMU.
+
+   Called by client function: #PVRSRVDCBufferImport()
+
+   Implementation of this callback is optional.
+
+@Input          hDisplayContext         Display context this buffer will be
+                                        used on
+
+@Input          ui32NumPlanes           Number of planes
+
+@Input          pahImport               Array of handles (one per colour channel)
+
+@Input          psSurfAttrib            Surface attributes of the buffer
+
+@Output         phBuffer                Handle to imported buffer
+
+@Return         PVRSRV_OK if the buffer was successfully imported
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferImport)(IMG_HANDLE hDisplayContext,
+                                     IMG_UINT32 ui32NumPlanes,
+                                     IMG_HANDLE **pahImport,
+                                     DC_BUFFER_IMPORT_INFO *psSurfAttrib,
+                                     IMG_HANDLE *phBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferAcquire
+
+@Description    Acquire the buffer's physical memory pages. If the buffer doesn't
+                have any memory backing yet then this will trigger the 3rd
+                party driver to allocate it.
+
+                Note: The page count isn't passed back in this function as
+                Services has already obtained it during BufferAlloc.
+
+   Called when Services requests buffer access for the first time. Usually part
+   of non display class functions.
+
+   Implementation of this callback is mandatory.
+
+@Input          hBuffer                 Handle to the buffer
+
+@Output         pasDevPAddr             Array of device physical page address
+                                        of this buffer
+
+@Output         ppvLinAddr              CPU virtual address of buffer. This is
+                                        optional but if you have one you must
+                                        return it otherwise return NULL.
+
+@Return         PVRSRV_OK if the buffer was successfully acquired
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferAcquire)(IMG_HANDLE hBuffer,
+                                      IMG_DEV_PHYADDR *pasDevPAddr,
+                                      void **ppvLinAddr);
+
+/*************************************************************************/ /*!
+@Function       BufferRelease
+
+@Description    Undo everything done by BufferAcquire.
+                Will release the buffer's physical memory pages if
+                BufferAcquire allocated them.
+
+   Called when Services finished buffer access.
+
+   Implementation of this callback is mandatory.
+
+@Input          hBuffer                 Handle to the buffer
+*/
+/*****************************************************************************/
+typedef void (*BufferRelease)(IMG_HANDLE hBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferFree
+
+@Description    Release a reference to the device buffer. If this was the last
+                reference the 3rd party driver is entitled to free the backing
+                memory and other related resources.
+
+   Called by client function: #PVRSRVDCBufferFree()
+
+   Implementation of this callback is mandatory.
+
+@Input          hBuffer                 Buffer handle we're releasing
+*/
+/*****************************************************************************/
+typedef void (*BufferFree)(IMG_HANDLE hBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferMap
+
+@Description    Map the buffer into the display controller
+                Note: This function depends on the behaviour of
+                BufferAlloc/BufferAcquire/BufferImport/BufferSystemAcquire
+                and the controller's ability to map in memory.
+                If the controller has no MMU or the above functions already
+                map the buffer this callback does not need an implementation.
+
+   Called by client function: #PVRSRVDCBufferPin()
+
+   Implementation of this callback is optional.
+
+@Input          hBuffer                 Buffer to map
+
+@Return         PVRSRV_OK if the buffer was successfully mapped
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferMap)(IMG_HANDLE hBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferUnmap
+
+@Description    Undo everything done by BufferMap.
+                Usually that means to unmap a buffer from the display controller.
+
+   Called by client function: #PVRSRVDCBufferUnpin()
+
+   Implementation of this callback is optional.
+
+@Input          hBuffer                 Buffer to unmap
+*/
+/*****************************************************************************/
+typedef void (*BufferUnmap)(IMG_HANDLE hBuffer);
+
+
+/*************************************************************************/ /*!
+@Function       BufferSystemAcquire
+
+@Description    DEPRICATED, please use BufferAlloc
+                Acquire the system buffer from the display driver.
+                If the OS should trigger a mode change then it's not allowed to
+                free the previous buffer until Services has released it
+                via BufferSystemRelease.
+
+   Called by client function: #PVRSRVDCSystemBufferAcquire()
+
+   Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+
+@Output         puiLog2PageSize         The physical pagesize in log2(bytes)
+                                        of one page that the buffer is composed of
+
+@Output         pui32PageCount          The number of pages the buffer contains
+
+@Output         pui32ByteStride         Byte stride of the buffer
+
+@Output         phSystemBuffer          Handle to the buffer object
+
+@Return         PVRSRV_OK if the query was successful
+*/
+/*****************************************************************************/
+typedef PVRSRV_ERROR (*BufferSystemAcquire)(IMG_HANDLE hDeviceData,
+                                            IMG_DEVMEM_LOG2ALIGN_T *puiLog2PageSize,
+                                            IMG_UINT32 *pui32PageCount,
+                                            IMG_UINT32 *pui32ByteStride,
+                                            IMG_HANDLE *phSystemBuffer);
+
+/*************************************************************************/ /*!
+@Function       BufferSystemRelease
+
+@Description    DEPRICATED, please use BufferFree
+                Release a display buffer acquired with BufferSystemAcquire.
+                Services calls this after it has no use for the buffer anymore.
+                The buffer must not be destroyed before Services releases it
+                with this call.
+
+   Called by client function: #PVRSRVDCSystemBufferRelease()
+
+   Implementation of this callback is optional.
+
+@Input          hSystemBuffer          Handle to the buffer object
+*/ /**************************************************************************/
+typedef	void (*BufferSystemRelease)(IMG_HANDLE hSystemBuffer);
+
+/*************************************************************************/ /*!
+@Function       ResetDevice
+
+@Description    Reset the device state. This function should perform actions
+                required for getting the device to the working state. This can
+                be for example used during resume operation from a hibernation.
+
+                Implementation of this callback is optional.
+
+@Input          hDeviceData             Device private data
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*ResetDevice)(IMG_HANDLE hDeviceData);
+
+#if defined(INTEGRITY_OS)
+typedef PVRSRV_ERROR (*AcquireKernelMappingData)(IMG_HANDLE hBuffer, IMG_HANDLE *phMapping, void **ppPhysAddr);
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+typedef IMG_HANDLE (*GetPmr)(IMG_HANDLE hBuffer, size_t ulOffset);
+#endif
+#endif
+/*!
+ * Function table for functions to be implemented by the display controller
+ * that will be called from within Services.
+ * The table will be provided to Services with the call to DCRegisterDevice.
+ *
+ * Typedef: ::DC_DEVICE_FUNCTIONS
+ */
+typedef struct _DC_DEVICE_FUNCTIONS_
+{
+	/* Mandatory query functions */
+	GetInfo                 pfnGetInfo; /*!< See #GetInfo */
+	PanelQueryCount         pfnPanelQueryCount; /*!< See #PanelQueryCount */
+	PanelQuery              pfnPanelQuery; /*!< See #PanelQuery */
+	FormatQuery             pfnFormatQuery; /*!< See #FormatQuery */
+	DimQuery                pfnDimQuery; /*!< See #DimQuery */
+
+	/* Optional blank/vsync functions */
+	SetBlank                pfnSetBlank; /*!< See #SetBlank */
+	SetVSyncReporting       pfnSetVSyncReporting; /*!< See #SetVSyncReporting */
+	LastVSyncQuery          pfnLastVSyncQuery; /*!< See #LastVSyncQuery */
+
+	/* Mandatory configure functions */
+	ContextCreate           pfnContextCreate; /*!< See #ContextCreate */
+	ContextDestroy          pfnContextDestroy; /*!< See #ContextDestroy */
+	ContextConfigure        pfnContextConfigure; /*!< See #ContextConfigure */
+
+	/* Optional context functions */
+	ContextConfigureCheck   pfnContextConfigureCheck; /*!< See #ContextConfigureCheck */
+
+	/* Mandatory buffer functions */
+	BufferAlloc             pfnBufferAlloc; /*!< See #BufferAlloc */
+	BufferAcquire           pfnBufferAcquire; /*!< See #BufferAcquire */
+	BufferRelease           pfnBufferRelease; /*!< See #BufferRelease */
+	BufferFree              pfnBufferFree; /*!< See #BufferFree */
+
+	/* Optional - Provide this function if your controller can
+	 * scan out arbitrary memory, allocated for another purpose
+	 * by Services. */
+	BufferImport            pfnBufferImport; /*!< See #BufferImport */
+
+	/* Optional - Provide these functions if your controller
+	 * has an MMU and does not (or cannot) map/unmap buffers at
+	 * alloc/free time */
+	BufferMap               pfnBufferMap; /*!< See #BufferMap */
+	BufferUnmap             pfnBufferUnmap; /*!< See #BufferUnmap */
+
+	/* Optional - DEPRICATED */
+	BufferSystemAcquire     pfnBufferSystemAcquire; /*!< See #BufferSystemAcquire */
+	BufferSystemRelease     pfnBufferSystemRelease; /*!< See #BufferSystemRelease */
+
+#if defined(INTEGRITY_OS)
+	/* The addition of these functions allow dc_server to delegate calls to
+	 * the respective functions on its PMRs towards the DC module
+	 */
+	AcquireKernelMappingData    pfnAcquireKernelMappingData;
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	GetPmr                      pfnGetPmr;
+#endif
+#endif
+
+   /* Optional - Provide this function if your controller should be allowed
+    * to be reset by Services.
+    * This can be used for example during suspend-to-RAM or suspend-to-disk
+    * procedures. */
+	ResetDevice                 pfnResetDevice; /*!< See #ResetDevice */
+} DC_DEVICE_FUNCTIONS;
+
+
+/*
+ * Functions exported by kernel Services for use by 3rd party kernel display
+ * controller device driver
+*/
+
+/*************************************************************************/ /*!
+@Function       DCRegisterDevice
+
+@Description    This needs to be called by the display driver before any further
+                communication with Services.
+                It registers a display controller device with Services. After this
+                registration Services is able to use the display controller
+                and will make use of the callbacks. Services will provide the
+                hDeviceData in the callbacks whenever necessary.
+
+@Input          psFuncTable             Callback function table
+
+@Input          ui32MaxConfigsInFlight  The maximum number of configs that this
+                                        display device can have in-flight. This
+                                        determines the number of possible calls
+                                        to ContextConfigure before Services has to
+                                        wait for DCDisplayConfigurationRetired
+                                        calls.
+
+@Input          hDeviceData             Device private data passed into callbacks
+
+@Output         phSrvHandle             Services handle to pass back into
+                                        UnregisterDCDevice
+
+@Return         PVRSRV_OK if the display controller driver was successfully registered
+*/
+/*****************************************************************************/
+PVRSRV_ERROR DCRegisterDevice(DC_DEVICE_FUNCTIONS *psFuncTable,
+                              IMG_UINT32 ui32MaxConfigsInFlight,
+                              IMG_HANDLE hDeviceData,
+                              IMG_HANDLE *phSrvHandle);
+
+/*************************************************************************/ /*!
+@Function       DCUnregisterDevice
+
+@Description    Unregister a display controller device. Undo everything done with
+                DCRegisterDevice. Services will stop using this device.
+
+@Input          hSrvHandle              Services device handle
+*/
+/*****************************************************************************/
+void DCUnregisterDevice(IMG_HANDLE hSrvHandle);
+
+/*************************************************************************/ /*!
+@Function       DCDisplayConfigurationRetired
+
+@Description    Called when a configuration as been retired due to a new
+                configuration now being active.
+
+@Input          hConfigData             ConfigData that is being retired
+*/
+/*****************************************************************************/
+void DCDisplayConfigurationRetired(IMG_HANDLE hConfigData);
+
+/*************************************************************************/ /*!
+@Function       DCDisplayHasPendingCommand
+
+@Description    Called to check if there are still pending commands in
+                the Software Command Processor queue.
+
+@Input          hConfigData             ConfigData to check for pending
+                                        commands
+
+@Return         IMG_TRUE if there is at least one pending command
+*/
+/*****************************************************************************/
+IMG_BOOL DCDisplayHasPendingCommand(IMG_HANDLE hConfigData);
+
+/*************************************************************************/ /*!
+@Function       DCImportBufferAcquire
+
+@Description    Acquire information about a buffer that was imported with
+                BufferImport. DCImportBufferRelease has to be called after
+                the buffer will not be used anymore.
+
+@Input          hImport                 Import buffer
+
+@Input          uiLog2PageSize          Pagesize in log2(bytes) of the buffer
+
+@Output         pui32PageCount          Size of the buffer in pages
+
+@Output         pasDevPAddr             Array of device physical page address
+                                        of this buffer
+
+@Return         PVRSRV_OK if the import buffer was successfully acquired
+*/
+/*****************************************************************************/
+PVRSRV_ERROR DCImportBufferAcquire(IMG_HANDLE hImport,
+                                   IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+                                   IMG_UINT32 *pui32PageCount,
+                                   IMG_DEV_PHYADDR **pasDevPAddr);
+
+/*************************************************************************/ /*!
+@Function       DCImportBufferRelease
+
+@Description    Release an imported buffer.
+
+@Input          hImport                 Import handle we are releasing
+
+@Input          pasDevPAddr             Import data which was returned from
+                                        DCImportBufferAcquire
+*/
+/*****************************************************************************/
+void DCImportBufferRelease(IMG_HANDLE hImport,
+                           IMG_DEV_PHYADDR *pasDevPAddr);
+
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* KERNELDISPLAY_H */
+
+/******************************************************************************
+ End of file (kerneldisplay.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/km_apphint_defs_common.h b/drivers/gpu/drm/img/img-volcanic/services/include/km_apphint_defs_common.h
new file mode 100644
index 000000000000..ad38c76793d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/km_apphint_defs_common.h
@@ -0,0 +1,308 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef KM_APPHINT_DEFS_COMMON_H
+#define KM_APPHINT_DEFS_COMMON_H
+
+/*
+*******************************************************************************
+ Build variables
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(EnableTrustedDeviceAceConfig,     BOOL,           GPUVIRT_VAL, PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG,  NO_PARAM_TABLE,   ALWAYS   ) \
+X(CleanupThreadPriority,            UINT32,         NEVER,       PVRSRV_APPHINT_CLEANUPTHREADPRIORITY,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(WatchdogThreadPriority,           UINT32,         NEVER,       PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientBufferSize,           UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevmemHistoryBufSizeLog2,         UINT32,         ALWAYS,      PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevmemHistoryMaxEntries,          UINT32,         ALWAYS,      PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES,    NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Module parameters
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(GeneralNon4KHeapPageSize,         UINT32,         ALWAYS,      PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE,      NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableSignatureChecks,            BOOL,           PDUMP,       PVRSRV_APPHINT_ENABLESIGNATURECHECKS,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(SignatureChecksBufSize,           UINT32,         PDUMP,       PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DisableClockGating,               BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLECLOCKGATING,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisableDMOverlap,                 BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLEDMOVERLAP,              NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableRandomContextSwitch,        BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableSoftResetContextSwitch,     BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH,  NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFWContextSwitch,            UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(FWContextSwitchProfile,           UINT32,         VALIDATION,  PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableRDPowerIsland,              UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLERDPOWERISLAND,           NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DriverMode,                       STRING,         ALWAYS,      PVRSRV_APPHINT_DRIVERMODE,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(AutoVzGPUPowerdown,               BOOL,           ALWAYS,      PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(GuestFWHeapStride,                UINT64,         ALWAYS,      PVRSRV_APPHINT_GUESTFWHEAPSTRIDE,             NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FirmwarePerf,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_FIRMWAREPERF,                  NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(HWPerfFWBufSizeInKB,              UINT32,         PDUMP,       PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB,           NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostBufSizeInKB,            UINT32,         VALIDATION,  PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostThreadTimeoutInMS,      UINT32,         VALIDATION,  PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS,   NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(NewFilteringMode,                 BOOL,           VALIDATION,  PVRSRV_APPHINT_NEWFILTERINGMODE,              NO_PARAM_TABLE,   ALWAYS   ) \
+X(TruncateMode,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_TRUNCATEMODE,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(EmuMaxFreq,                       UINT32,         ALWAYS,      PVRSRV_APPHINT_EMUMAXFREQ,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(GPIOValidationMode,               UINT32,         VALIDATION,  PVRSRV_APPHINT_GPIOVALIDATIONMODE,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(RGXBVNC,                          STRING,         ALWAYS,      PVRSRV_APPHINT_RGXBVNC,                       NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FWContextSwitchCrossDM,           UINT32,         ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(ValidateIrq,                      BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATEIRQ,                   NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(TPUTrilinearFracMaskPDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskVDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskCDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskTDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskRDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(HTBufferSizeInKB,                 UINT32,         ALWAYS,      PVRSRV_APPHINT_HTBUFFERSIZE,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(FWTraceBufSizeInDWords,           UINT32,         ALWAYS,      PVRSRV_APPHINT_FWTRACEBUFSIZEINDWORDS,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnablePageFaultDebug,             BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFullSyncTracking,           BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(IgnoreHWReportedBVNC,             BOOL,           ALWAYS,      PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC,          NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(PhysMemTestPasses,                UINT32,         ALWAYS,      PVRSRV_APPHINT_PHYSMEMTESTPASSES,             NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FBCDCVersionOverride,             UINT32,         VALIDATION,  PVRSRV_APPHINT_FBCDCVERSIONOVERRIDE,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TestSLRInterval,                  UINT32,         VALIDATION,  PVRSRV_APPHINT_TESTSLRINTERVAL,               NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnablePollOnChecksumErrorStatus,  UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(RiscvDmiTest,                     BOOL,           VALIDATION,  PVRSRV_APPHINT_RISCVDMITEST,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevMemFWHeapPolicy,               UINT32,         ALWAYS,      PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY,          NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableAPMAll,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_ENABLEAPM,                     NO_PARAM_TABLE,   ALWAYS   ) \
+X(KernelCCBSizeLog2,                UINT32,         VALIDATION,  PVRSRV_APPHINT_KCCB_SIZE_LOG2,                NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(SyncCheckpointPoolMaxLog2,        UINT32,         ALWAYS,      PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(SyncCheckpointPoolInitLog2,       UINT32,         ALWAYS,      PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(PhysHeapMinMemOnConnection,       UINT32,         ALWAYS,      PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION,    NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(RestrictGpuLocalPhysHeapSizeMB,   UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(PhysHeapHybridDefault2CpuLocal,   BOOL,           ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DebugDumpFWTLogType,              UINT32,         ALWAYS,      PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE,           NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableIdleCycleStealing,          UINT32,         VALIDATION,  PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING,       NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FaultDetectionTimeInterval,       UINT32,         VALIDATION,  PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC, NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(ICSTimeIntervalThreshold,         UINT32,         VALIDATION,  PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD,     NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(ICSTestModeOn,                    BOOL,           VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FaultInjection,                   UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Debugfs parameters - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(EnableHTBLogGroup,                UINT32Bitfield, ALWAYS,      PVRSRV_APPHINT_ENABLEHTBLOGGROUP,             htb_loggroup_tbl, ALWAYS   ) \
+X(HTBOperationMode,                 UINT32List,     ALWAYS,      PVRSRV_APPHINT_HTBOPERATIONMODE,              htb_opmode_tbl,   ALWAYS   ) \
+X(EnableFTraceGPU,                  BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEFTRACEGPU,               NO_PARAM_TABLE,   NEVER    ) \
+X(HWPerfClientFilter_Services,      UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_SERVICES,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_EGL,           UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_EGL,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenGLES,      UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGLES,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenCL,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENCL,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_Vulkan,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_VULKAN,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenGL,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGL,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(CacheOpConfig,                    UINT32,         ALWAYS,      PVRSRV_APPHINT_CACHEOPCONFIG,                 NO_PARAM_TABLE,   ALWAYS   ) \
+X(CacheOpUMKMThresholdSize,         UINT32,         ALWAYS,      PVRSRV_APPHINT_CACHEOPUMKMHRESHOLDSIZE,       NO_PARAM_TABLE,   ALWAYS   ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+/* Device Firmware config */\
+X(AssertOnHWRTrigger,               BOOL,           ALWAYS,      APPHNT_BLDVAR_ASSERTONHWRTRIGGER,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(AssertOutOfMemory,                BOOL,           ALWAYS,      PVRSRV_APPHINT_ASSERTOUTOFMEMORY,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(CheckMList,                       BOOL,           ALWAYS,      PVRSRV_APPHINT_CHECKMLIST,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableLogGroup,                   UINT32Bitfield, ALWAYS,      PVRSRV_APPHINT_ENABLELOGGROUP,                fwt_loggroup_tbl, ALWAYS   ) \
+X(FirmwareLogType,                  UINT32List,     ALWAYS,      PVRSRV_APPHINT_FIRMWARELOGTYPE,               fwt_logtype_tbl,  ALWAYS   ) \
+X(HWRDebugDumpLimit,                UINT32,         ALWAYS,      PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(SecondaryOSClockSource,           UINT32List,     ALWAYS,      PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE,        timecorr_clk_tbl, ALWAYS   ) \
+X(HWPerfFWFilter,                   UINT64,         ALWAYS,      PVRSRV_APPHINT_HWPERFFWFILTER,                NO_PARAM_TABLE,   ALWAYS   ) \
+/* Device host config */ \
+X(EnableAPM,                        UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLEAPM,                     NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisableFEDLogging,                BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLEFEDLOGGING,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(ZeroFreelist,                     BOOL,           ALWAYS,      PVRSRV_APPHINT_ZEROFREELIST,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisablePDumpPanic,                BOOL,           PDUMP,       PVRSRV_APPHINT_DISABLEPDUMPPANIC,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFWPoisonOnFree,             BOOL,           DEBUG,       PVRSRV_APPHINT_ENABLEFWPOISONONFREE,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(GPUUnitsPowerChange,              BOOL,           VALIDATION,  PVRSRV_APPHINT_GPUUNITSPOWERCHANGE,           NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostFilter,                 UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFHOSTFILTER,              NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters.
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT_COMMON \
+/* debuginfo device apphint name  modparam name */ \
+X(EnableAPM,                      EnableAPMAll)
+
+/*
+*******************************************************************************
+ * Types used in the APPHINT_LIST_<GROUP> lists must be defined here.
+ * New types require specific handling code to be added
+******************************************************************************/
+#define APPHINT_DATA_TYPE_LIST \
+X(BOOL) \
+X(UINT64) \
+X(UINT32) \
+X(UINT32Bitfield) \
+X(UINT32List) \
+X(STRING)
+
+#define APPHINT_CLASS_LIST \
+X(ALWAYS) \
+X(NEVER) \
+X(DEBUG) \
+X(PDUMP) \
+X(VALIDATION) \
+X(GPUVIRT_VAL)
+
+#define APPHINT_RT_CLASS_LIST \
+X(ALWAYS) \
+X(NEVER)
+
+/*
+*******************************************************************************
+ Visibility control for module parameters
+ These bind build variables to AppHint Visibility Groups.
+******************************************************************************/
+#define APPHINT_ENABLED_CLASS_ALWAYS IMG_TRUE
+#define APPHINT_ENABLED_CLASS_NEVER IMG_FALSE
+#define apphint_modparam_class_ALWAYS(a, b, c) apphint_modparam_enable(a, b, c)
+#if defined(DEBUG)
+	#define APPHINT_ENABLED_CLASS_DEBUG IMG_TRUE
+	#define apphint_modparam_class_DEBUG(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_DEBUG IMG_FALSE
+	#define apphint_modparam_class_DEBUG(a, b, c)
+#endif
+#if defined(PDUMP)
+	#define APPHINT_ENABLED_CLASS_PDUMP IMG_TRUE
+	#define apphint_modparam_class_PDUMP(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_PDUMP IMG_FALSE
+	#define apphint_modparam_class_PDUMP(a, b, c)
+#endif
+	#define APPHINT_ENABLED_CLASS_VALIDATION IMG_FALSE
+	#define apphint_modparam_class_VALIDATION(a, b, c)
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	#define APPHINT_ENABLED_CLASS_GPUVIRT_VAL IMG_TRUE
+	#define apphint_modparam_class_GPUVIRT_VAL(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_GPUVIRT_VAL IMG_FALSE
+	#define apphint_modparam_class_GPUVIRT_VAL(a, b, c)
+#endif
+
+/*
+*******************************************************************************
+ AppHint defaults based on other build parameters
+******************************************************************************/
+#if defined(ASSERTONHWRTRIGGER_DEFAULT_ENABLED)
+	#define APPHNT_BLDVAR_ASSERTONHWRTRIGGER      1
+#else
+	#define APPHNT_BLDVAR_ASSERTONHWRTRIGGER      0
+#endif
+#if defined(DEBUG)
+	#define APPHNT_BLDVAR_DEBUG             1
+	#define APPHNT_BLDVAR_DBGDUMPLIMIT      RGXFWIF_HWR_DEBUG_DUMP_ALL
+#else
+	#define APPHNT_BLDVAR_DEBUG             0
+	#define APPHNT_BLDVAR_DBGDUMPLIMIT      1
+#endif
+#if defined(PDUMP)
+#define APPHNT_BLDVAR_ENABLESIGNATURECHECKS     IMG_TRUE
+#else
+#define APPHNT_BLDVAR_ENABLESIGNATURECHECKS     IMG_FALSE
+#endif
+#if defined(DEBUG)
+#define APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG      IMG_TRUE
+#else
+#define APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG      IMG_FALSE
+#endif
+
+#if defined(DEBUG)
+	#define APPHNT_PHYSMEMTEST_ENABLE             1
+#else
+	#define APPHNT_PHYSMEMTEST_ENABLE             0
+#endif
+
+/* Data types and actions */
+typedef enum {
+	APPHINT_DATA_TYPE_INVALID = 0,
+#define X(a) APPHINT_DATA_TYPE_ ## a,
+	APPHINT_DATA_TYPE_LIST
+#undef X
+	APPHINT_DATA_TYPE_MAX
+} APPHINT_DATA_TYPE;
+
+typedef enum {
+#define X(a) APPHINT_CLASS_ ## a,
+	APPHINT_CLASS_LIST
+#undef X
+	APPHINT_CLASS_MAX
+} APPHINT_CLASS;
+
+typedef enum {
+#define X(a) APPHINT_RT_CLASS_ ## a,
+    APPHINT_RT_CLASS_LIST
+#undef X
+    APPHINT_RT_CLASS_MAX
+} APPHINT_RT_CLASS;
+#endif /* KM_APPHINT_DEFS_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/os_cpu_cache.h b/drivers/gpu/drm/img/img-volcanic/services/include/os_cpu_cache.h
new file mode 100644
index 000000000000..56f92036ff49
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/os_cpu_cache.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS and CPU d-cache maintenance mechanisms
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines for cache management which are visible internally only
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OS_CPU_CACHE_H
+#define OS_CPU_CACHE_H
+
+#include "info_page_defs.h"
+
+#define PVRSRV_CACHE_OP_TIMELINE			0x8 /*!< Request SW_SYNC timeline notification when executed */
+#define PVRSRV_CACHE_OP_FORCE_SYNCHRONOUS	0x10 /*!< Force all batch members to be executed synchronously */
+
+#define CACHEFLUSH_ISA_X86					0x1	/*!< x86/x64 specific UM range-based cache flush */
+#define CACHEFLUSH_ISA_ARM64				0x2	/*!< Aarch64 specific UM range-based cache flush */
+#define CACHEFLUSH_ISA_GENERIC				0x3	/*!< Other ISA's without UM range-based cache flush */
+#ifndef CACHEFLUSH_ISA_TYPE
+	#if defined(__i386__) || defined(__x86_64__)
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_X86
+	#elif defined(__arm64__) || defined(__aarch64__)
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_ARM64
+	#else
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_GENERIC
+	#endif
+#endif
+
+#if (CACHEFLUSH_ISA_TYPE == CACHEFLUSH_ISA_X86) || (CACHEFLUSH_ISA_TYPE == CACHEFLUSH_ISA_ARM64)
+#define CACHEFLUSH_ISA_SUPPORTS_UM_FLUSH		/*!< x86/x86_64/ARM64 supports user-mode d-cache flush */
+#endif
+
+#endif	/* OS_CPU_CACHE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pdump.h b/drivers/gpu/drm/img/img-volcanic/services/include/pdump.h
new file mode 100644
index 000000000000..3ef71846446e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pdump.h
@@ -0,0 +1,238 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef SERVICES_PDUMP_H
+#define SERVICES_PDUMP_H
+
+#include "img_types.h"
+#include "services_km.h"
+
+
+/* A PDump out2.txt script is made up of 3 sections from three buffers:
+ * *
+ *  - Init phase buffer    - holds PDump data written during driver
+ *                            initialisation, non-volatile.
+ *  - Main phase buffer   - holds PDump data written after driver init,
+ *                            volatile.
+ *  - Deinit phase buffer - holds PDump data  needed to shutdown HW/play back,
+ *                            written only during driver initialisation using
+ *                            the DEINIT flag.
+ *
+ * Volatile in this sense means that the buffer is drained and cleared when
+ * the pdump capture application connects and transfers the data to file.
+ *
+ * The PDump sub-system uses the driver state (init/post-init), whether
+ * the pdump capture application is connected or not (capture range set/unset)
+ * and, if pdump connected whether the frame is in the range set, to decide
+ * which of the 3 buffers to write the PDump data. Hence there are several
+ * key time periods in the lifetime of the kernel driver that is enabled
+ * with PDUMP=1 (flag XX labels below time line):
+ *
+ * Events:load              init        pdump       enter          exit         pdump
+ *       driver             done       connects     range          range     disconnects
+ *         |__________________|____________|__________|______________|____________|______ . . .
+ * State:  |   init phase     | no capture | <- capture client connected ->       | no capture
+ *         |                  |            |                                      |
+ *         |__________________|____________|______________________________________|_____ . . .
+ * Flag:   | CT,DI            | NONE,CT,PR | NONE,CT,PR                           | See no
+ *         | Never NONE or PR | Never DI   | Never DI                             |   capture
+ *         |__________________|____________|______________________________________|_____ . . .
+ * Write   | NONE -undef      | -No write  | -No write | -Main buf    | -No write | See no
+ * buffer  | CT -Init buf     | -Main buf  | -Main buf | -Main buf    | -Main buf |   capture
+ *         | PR -undef        | -Init buf  | -undef    | -Init & Main | -undef    |
+ *         | DI -Deinit buf   | -undef     | -undef    | -undef       | -undef    |
+ *         |__________________|____________|___________|______________|___________|_____ . . .
+ *
+ * Note: The time line could repeat if the pdump capture application is
+ * disconnected and reconnected without unloading the driver module.
+ *
+ * The DEINIT (DI) | CONTINUOUS (CT) | PERSISTENT (PR) flags must never
+ * be OR'd together and given to a PDump call since undefined behaviour may
+ * result and produce an invalid PDump which does not play back cleanly.
+ *
+ * The decision on which flag to use comes down to which time period the
+ * client or server driver makes the PDump write call AND the nature/purpose
+ * of the data.
+ *
+ * Note: This is a simplified time line, not all conditions represented.
+ *
+ */
+
+typedef IMG_UINT32 PDUMP_FLAGS_T;
+
+#define PDUMP_FLAGS_NONE            PDUMP_NONE     /*<! Output this entry with no special treatment i.e. output
+                                                          only if in frame range. */
+#define PDUMP_FLAGS_BLKDATA         PDUMP_BLKDATA  /*<! This flag indicates block-mode PDump data to be recorded
+                                                          in Block script stream in addition to Main script stream,
+                                                          if capture mode is set to BLOCKED */
+
+#define PDUMP_FLAGS_DEINIT          0x20000000U    /*<! Output this entry to the de-initialisation section, must
+                                                          only be used by the initialisation code in the Server. */
+
+#define PDUMP_FLAGS_POWER           0x08000000U    /*<! Output this entry even when a power transition is ongoing,
+                                                          as directed by other PDUMP flags. */
+
+#define PDUMP_FLAGS_CONTINUOUS      PDUMP_CONT     /*<! Output this entry always regardless of framed capture range,
+                                                          used by client applications being dumped.
+                                                          During init phase of driver such data carrying this flag
+                                                          will be recorded and present for all PDump client
+                                                          connections.
+                                                          Never combine with the PERSIST flag. */
+
+#define PDUMP_FLAGS_PERSISTENT      PDUMP_PERSIST  /*<! Output this entry always regardless of app and range,
+                                                          used by persistent resources created *after* driver
+                                                          initialisation that must appear in all PDump captures
+                                                          (i.e. current capture regardless of frame range (CONT)
+                                                          and all future PDump captures) for that driver
+                                                          instantiation/session.
+                                                          Effectively this is data that is not forgotten
+                                                          for the second and subsequent PDump client connections.
+                                                          Never combine with the CONTINUOUS flag. */
+
+#define PDUMP_FLAGS_INTERVAL        0x04000000U    /*<! Output this entry even when the capture is on a
+                                                          "no capture interval frame" (see pdump -sr option).
+                                                          Useful for commands that have a resource that was written
+                                                          out in a frame that was captured. For example,
+                                                          used by RGXScheduleCleanupCommand. */
+
+#define PDUMP_FLAGS_DEBUG           0x00010000U    /*<! For internal debugging use */
+
+#define PDUMP_FLAGS_NOHW            0x00000001U    /* For internal use: Skip sending instructions to the hardware
+                                                        when NO_HARDWARE=0 AND PDUMP=1 */
+
+#define PDUMP_FLAGS_FORCESPLIT      0x00000002U	   /* Forces Main and Block script streams to split - Internal
+                                                        flag used in Block mode of PDump */
+
+#define PDUMP_FLAGS_PDUMP_LOCK_HELD 0x00000004U    /* This flags denotes that PDUMP_LOCK is held already so
+                                                        further calls to PDUMP_LOCK  with this flag set will not
+                                                        try to take pdump lock. If PDUMP_LOCK is called without
+                                                        this flag by some other thread then it will try to take
+                                                        PDUMP_LOCK which would make that thread sleep. This flag
+                                                        introduced to enforce the order of pdumping after bridge
+                                                        lock removal */
+
+#define PDUMP_FILEOFFSET_FMTSPEC    "0x%08X"
+typedef IMG_UINT32 PDUMP_FILEOFFSET_T;
+
+/* PDump stream macros*/
+
+/* Parameter stream */
+#define PDUMP_PARAM_INIT_STREAM_NAME       "paramInit"
+#define PDUMP_PARAM_MAIN_STREAM_NAME       "paramMain"
+#define PDUMP_PARAM_DEINIT_STREAM_NAME     "paramDeinit"
+#define PDUMP_PARAM_BLOCK_STREAM_NAME      "paramBlock"
+
+/*** Parameter stream sizes ***/
+
+/* Parameter Init Stream */
+
+/* The streams are made tunable in core.mk, wherein the size of these buffers can be specified.
+ * In case nothing specified in core.mk, then the default size is taken
+ */
+#if defined(PDUMP_PARAM_INIT_STREAM_SIZE)
+	#if (PDUMP_PARAM_INIT_STREAM_SIZE < (700 * 1024))
+		#error PDUMP_PARAM_INIT_STREAM_SIZE must be at least 700 KB
+	#endif
+#endif
+
+/* Parameter Main Stream */
+#if defined(PDUMP_PARAM_MAIN_STREAM_SIZE)
+	#if (PDUMP_PARAM_MAIN_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_PARAM_MAIN_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+/* Parameter Deinit Stream */
+#if defined(PDUMP_PARAM_DEINIT_STREAM_SIZE)
+	#if (PDUMP_PARAM_DEINIT_STREAM_SIZE < (64 * 1024))
+		#error PDUMP_PARAM_DEINIT_STREAM_SIZE must be at least 64 KB
+	#endif
+#endif
+
+/* Parameter Block Stream */
+/* There is no separate parameter Block stream as the Block script stream is
+ * just a filtered Main script stream. Hence it will refer to the Main stream
+ * parameters themselves.
+ */
+
+/*Script stream */
+#define PDUMP_SCRIPT_INIT_STREAM_NAME      "scriptInit"
+#define PDUMP_SCRIPT_MAIN_STREAM_NAME      "scriptMain"
+#define PDUMP_SCRIPT_DEINIT_STREAM_NAME    "scriptDeinit"
+#define PDUMP_SCRIPT_BLOCK_STREAM_NAME     "scriptBlock"
+
+/*** Script stream sizes ***/
+
+/* Script Init Stream */
+#if defined(PDUMP_SCRIPT_INIT_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_INIT_STREAM_SIZE < (256 * 1024))
+		#error PDUMP_SCRIPT_INIT_STREAM_SIZE must be at least 256 KB
+	#endif
+#endif
+
+/* Script Main Stream */
+#if defined(PDUMP_SCRIPT_MAIN_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_MAIN_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_SCRIPT_MAIN_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+/* Script Deinit Stream */
+#if defined(PDUMP_SCRIPT_DEINIT_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_DEINIT_STREAM_SIZE < (64 * 1024))
+		#error PDUMP_SCRIPT_DEINIT_STREAM_SIZE must be at least 64 KB
+	#endif
+#endif
+
+/* Script Block Stream */
+#if defined(PDUMP_SCRIPT_BLOCK_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_BLOCK_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_SCRIPT_BLOCK_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+
+#define PDUMP_PARAM_0_FILE_NAME     "%%0%%.prm"      /*!< Initial Param filename used in PDump capture */
+#define PDUMP_PARAM_N_FILE_NAME     "%%0%%_%02u.prm" /*!< Param filename used when PRM file split */
+#define PDUMP_PARAM_MAX_FILE_NAME   32               /*!< Max Size of parameter name used in out2.txt */
+
+#define PDUMP_IS_CONTINUOUS(flags) ((flags & PDUMP_FLAGS_CONTINUOUS) != 0)
+
+#endif /* SERVICES_PDUMP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/physheap.h b/drivers/gpu/drm/img/img-volcanic/services/include/physheap.h
new file mode 100644
index 000000000000..9cca0dd25286
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/physheap.h
@@ -0,0 +1,564 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physical heap management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the interface for the physical heap management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+#include "opaque_types.h"
+#include "pmr_impl.h"
+#include "physheap_config.h"
+#include "pvrsrv_device.h"
+#include "ra.h"
+
+#ifndef PHYSHEAP_H
+#define PHYSHEAP_H
+
+#define B2KB(x) ((x) >> 10)
+#define B2MB(x) ((x) >> 20)
+
+static inline IMG_UINT64 KB2B(IMG_UINT64 ui64Kilobytes) { return ui64Kilobytes << 10; }
+static inline IMG_UINT64 MB2B(IMG_UINT64 ui64Megabytes) { return ui64Megabytes << 20; }
+
+typedef struct _PHYS_HEAP_ PHYS_HEAP;
+#define INVALID_PHYS_HEAP 0xDEADDEAD
+
+/* PMB (Physical Memory Block) */
+typedef struct _PMB_ PMB;
+
+typedef IMG_UINT32 PHYS_HEAP_POLICY;
+
+/* Heap has default allocation policy and does not require
+ * any additional OS Functionality. Physically contiguous
+ * allocations are required for this physheap.
+ */
+#define PHYS_HEAP_POLICY_DEFAULT (0U)
+
+/*
+ * Heap has allocation strategy that may produce non
+ * physically contiguous allocations, additional OS functionality
+ * is required to map these allocations into the kernel.
+ */
+#define PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG      (1U)
+#define PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK (1U)
+
+struct _CONNECTION_DATA_;
+
+typedef struct _PG_HANDLE_
+{
+	union
+	{
+		void *pvHandle;
+		IMG_UINT64 ui64Handle;
+	}u;
+	/* The allocation order is log2 value of the number of pages to allocate.
+	 * As such this is a correspondingly small value. E.g, for order 4 we
+	 * are talking 2^4 * PAGE_SIZE contiguous allocation.
+	 * DevPxAlloc API does not need to support orders higher than 4.
+	 */
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	IMG_BYTE    uiOrder;    /* Order of the corresponding allocation */
+	IMG_BYTE    uiOSid;     /* OSid to use for allocation arena.
+	                         * Connection-specific. */
+	IMG_BYTE    uiPad1,
+	            uiPad2;     /* Spare */
+#else
+	IMG_BYTE    uiOrder;    /* Order of the corresponding allocation */
+	IMG_BYTE    uiPad1,
+	            uiPad2,
+	            uiPad3;     /* Spare */
+#endif
+} PG_HANDLE;
+
+/*! Pointer to private implementation specific data */
+typedef void *PHEAP_IMPL_DATA;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_DESTROY_DATA
+@Description    Destroy private implementation specific data.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+*/ /**************************************************************************/
+typedef void (*PFN_DESTROY_DATA)(PHEAP_IMPL_DATA);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_DEV_PADDR
+@Description    Get heap device physical address.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_DEV_PHYADDR    Device physical address.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_DEV_PADDR)(PHEAP_IMPL_DATA, IMG_DEV_PHYADDR*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_CPU_PADDR
+@Description    Get heap CPU physical address.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_CPU_PHYADDR    CPU physical address.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_CPU_PADDR)(PHEAP_IMPL_DATA, IMG_CPU_PHYADDR*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_SIZE
+@Description    Get size of heap.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_UINT64         Size of heap.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_SIZE)(PHEAP_IMPL_DATA, IMG_UINT64*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_PAGE_SHIFT
+@Description    Get heap log2 page shift.
+@Return         IMG_UINT32         Log2 page shift
+*/ /**************************************************************************/
+typedef IMG_UINT32 (*PFN_GET_PAGE_SHIFT)(void);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_MEM_STATS
+@Description    Get total and free memory size of the physical heap managed by
+                the PMR Factory.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_UINT64         total Size of heap.
+@Output         IMG_UINT64         free Size available in a heap.
+@Return         none
+*/ /**************************************************************************/
+typedef void (*PFN_GET_MEM_STATS)(PHEAP_IMPL_DATA, IMG_UINT64 *, IMG_UINT64 *);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_HEAP_STATS_STR_ITER
+@Description    Get string of heap memory spans constituting the heap. This
+                function can be iterated on to print sequential lines of the
+                heap data. Iterate until IMG_FALSE is returned
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@InOut          IMG_CHAR           Pointer to string buffer to be populated
+                                   with sequential heap data.
+@Input          IMG_UINT32         Size of the string buffer.
+@InOut          void**             Iter handle.
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_GET_HEAP_SPANS_STR_ITER)(PHEAP_IMPL_DATA, IMG_CHAR *, IMG_UINT32, void **);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_HEAP_DLM_BACKING
+@Description    Extract reference to DLM heap backing the current IMA heap.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@InOut          PHYS_HEAP**        Pointer to DLM backing heap.
+@Return         void
+*/ /**************************************************************************/
+typedef void (*PFN_GET_HEAP_DLM_BACKING)(PHEAP_IMPL_DATA, PHYS_HEAP **);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef PVRSRV_ERROR (*PFN_PAGES_ALLOC_GPV)(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                            PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+                                            IMG_UINT32 ui32OSid, IMG_PID uiPid);
+#endif
+typedef PVRSRV_ERROR (*PFN_PAGES_ALLOC)(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                        PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+                                        IMG_PID uiPid);
+
+typedef void (*PFN_PAGES_FREE)(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle);
+
+typedef PVRSRV_ERROR (*PFN_PAGES_MAP)(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle,
+                                      size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                                      void **pvPtr);
+
+typedef void (*PFN_PAGES_UNMAP)(PHYS_HEAP *psPhysHeap,
+                                PG_HANDLE *psMemHandle, void *pvPtr);
+
+typedef PVRSRV_ERROR (*PFN_PAGES_CLEAN)(PHYS_HEAP *psPhysHeap,
+                                        PG_HANDLE *pshMemHandle,
+                                        IMG_UINT32 uiOffset,
+                                        IMG_UINT32 uiLength);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_CREATE_PMR
+@Description    Create a PMR physical allocation and back with RAM on creation,
+                if required. The RAM page comes either directly from
+                the Phys Heap's associated pool of memory or from an OS API.
+@Input          psPhysHeap         Pointer to Phys Heap.
+@Input          psConnection       Pointer to device connection.
+@Input          uiSize             Allocation size.
+@Input          ui32NumPhysChunks  Physical chunk count.
+@Input          ui32NumVirtChunks  Virtual chunk count.
+@Input          pui32MappingTable  Mapping Table.
+@Input          uiLog2PageSize     Page size.
+@Input          uiFlags            Memalloc flags.
+@Input          pszAnnotation      Annotation.
+@Input          uiPid              Process ID.
+@Output         ppsPMRPtr          Pointer to PMR.
+@Input          ui32PDumpFlag      PDump flags.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CREATE_PMR)(PHYS_HEAP *psPhysHeap,
+									   struct _CONNECTION_DATA_ *psConnection,
+									   IMG_DEVMEM_SIZE_T uiSize,
+									   IMG_UINT32 ui32NumPhysChunks,
+									   IMG_UINT32 ui32NumVirtChunks,
+									   IMG_UINT32 *pui32MappingTable,
+									   IMG_UINT32 uiLog2PageSize,
+									   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+									   const IMG_CHAR *pszAnnotation,
+									   IMG_PID uiPid,
+									   PMR **ppsPMRPtr,
+									   IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_CREATE_PMB
+@Description    Create a PMB physical allocation and back with Card memory
+                on creation, if required. The card memory comes
+                directly from the DLM Phys Heap's associated pool of memory.
+@Input          psPhysHeap         Pointer to Phys Heap to create the PMB on,
+                                   physheap should be DLM type.
+@Input          uiSize             Allocation size.
+@Input          pszAnnotation      Annotation.
+@Output         ppsPMBPtr          Pointer to PMB created.
+@Output         puiBase            Out pointer to RA Base of PMB.
+@Output         puiSize            Out pointer to size of PMB
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CREATE_PMB)(PHYS_HEAP *psPhysHeap,
+                                       IMG_DEVMEM_SIZE_T uiSize,
+                                       const IMG_CHAR *pszAnnotation,
+                                       PMB **ppsPMBPtr,
+                                       RA_BASE_T *puiBase,
+                                       RA_LENGTH_T *puiSize);
+
+/*! Implementation specific function table */
+typedef struct PHEAP_IMPL_FUNCS_TAG
+{
+	PFN_DESTROY_DATA pfnDestroyData;
+	PFN_GET_DEV_PADDR pfnGetDevPAddr;
+	PFN_GET_CPU_PADDR pfnGetCPUPAddr;
+	PFN_GET_SIZE pfnGetSize;
+	PFN_GET_PAGE_SHIFT pfnGetPageShift;
+	PFN_GET_MEM_STATS pfnGetFactoryMemStats;
+	PFN_GET_HEAP_SPANS_STR_ITER pfnGetHeapSpansStringIter;
+	PFN_GET_HEAP_DLM_BACKING pfnGetHeapDLMBacking;
+	PFN_CREATE_PMR pfnCreatePMR;
+	PFN_CREATE_PMB pfnCreatePMB;
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	PFN_PAGES_ALLOC_GPV pfnPagesAllocGPV;
+#endif
+	PFN_PAGES_ALLOC pfnPagesAlloc;
+	PFN_PAGES_FREE pfnPagesFree;
+	PFN_PAGES_MAP pfnPagesMap;
+	PFN_PAGES_UNMAP pfnPagesUnMap;
+	PFN_PAGES_CLEAN pfnPagesClean;
+} PHEAP_IMPL_FUNCS;
+
+/*************************************************************************/ /*!
+@Function       PhysHeapInitDeviceHeaps
+@Description    Registers and acquires physical memory heaps
+@Input          psDeviceNode pointer to device node
+@Input          psDevConfig  pointer to device config
+@Return         PVRSRV_ERROR PVRSRV_OK on success, or a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapInitDeviceHeaps(PPVRSRV_DEVICE_NODE psDeviceNode, PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDeInitDeviceHeaps
+@Description    Releases and unregisters physical memory heaps
+@Input          psDeviceNode pointer to device node
+@Return         PVRSRV_ERROR PVRSRV_OK on success, or a PVRSRV_ error code
+*/ /**************************************************************************/
+void PhysHeapDeInitDeviceHeaps(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateHeapFromConfig
+@Description    Create a new heap. Calls specific heap API depending
+                on heap type.
+@Input          psDevNode    Pointer to device node struct.
+@Input          psConfig     Heap configuration.
+@Output         ppsPhysHeap  Optional pointer to the created heap. Can be NULL
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysHeapCreateHeapFromConfig(PPVRSRV_DEVICE_NODE psDevNode,
+							 PHYS_HEAP_CONFIG *psConfig,
+							 PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreate
+@Description    Create a new heap. Allocated and stored internally.
+                Destroy with PhysHeapDestroy when no longer required.
+@Input          psDevNode    Pointer to device node struct
+@Input          psConfig     Heap configuration.
+@Input          uiPolicy     Phys heap allocation policy.
+@Input          pvImplData   Implementation specific data. Can be NULL.
+@Input          psImplFuncs  Implementation specific function table. Must be
+                             a valid pointer.
+@Output         ppsPhysHeap  Optional pointer to the created heap. Can be NULL
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreate(PPVRSRV_DEVICE_NODE psDevNode,
+							PHYS_HEAP_CONFIG *psConfig,
+							PHYS_HEAP_POLICY uiPolicy,
+							PHEAP_IMPL_DATA pvImplData,
+							PHEAP_IMPL_FUNCS *psImplFuncs,
+							PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDestroyDeviceHeaps
+@Description    Destroys all heaps referenced by a device.
+@Input          psDevNode Pointer to a device node struct.
+@Return         void
+*/ /**************************************************************************/
+void PhysHeapDestroyDeviceHeaps(PPVRSRV_DEVICE_NODE psDevNode);
+
+void PhysHeapDestroy(PHYS_HEAP *psPhysHeap);
+
+PVRSRV_ERROR PhysHeapAcquire(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapAcquireByID
+@Description    Acquire PhysHeap by DevPhysHeap.
+@Input          eDevPhysHeap Device Phys Heap.
+@Input          psDevNode    Pointer to device node struct
+@Output         ppsPhysHeap  PhysHeap if found.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapAcquireByID(PVRSRV_PHYS_HEAP eDevPhysHeap,
+								 PPVRSRV_DEVICE_NODE psDevNode,
+								 PHYS_HEAP **ppsPhysHeap);
+
+void PhysHeapRelease(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetImplData
+@Description    Get physical heap implementation specific data.
+@Input          psPhysHeap   Pointer to physical heap.
+@Input          psConfig     Heap configuration.
+@Return         pvImplData   Implementation specific data. Can be NULL.
+*/ /**************************************************************************/
+PHEAP_IMPL_DATA PhysHeapGetImplData(PHYS_HEAP *psPhysHeap);
+
+PHYS_HEAP_TYPE PhysHeapGetType(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetPolicy
+@Description    Get phys heap allocation policy flags.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         PHYS_HEAP_POLICY Phys heap policy flags.
+*/ /**************************************************************************/
+PHYS_HEAP_POLICY PhysHeapGetPolicy(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetFlags
+@Description    Get phys heap usage flags.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         PHYS_HEAP_USAGE_FLAGS Phys heap usage flags.
+*/ /**************************************************************************/
+PHYS_HEAP_USAGE_FLAGS PhysHeapGetFlags(PHYS_HEAP *psPhysHeap);
+
+IMG_BOOL PhysHeapValidateDefaultHeapExists(PPVRSRV_DEVICE_NODE psDevNode);
+
+#if defined(SUPPORT_STATIC_IPA)
+IMG_UINT32 PhysHeapGetIPAValue(PHYS_HEAP *psPhysHeap);
+
+IMG_UINT32 PhysHeapGetIPAMask(PHYS_HEAP *psPhysHeap);
+
+IMG_UINT32 PhysHeapGetIPAShift(PHYS_HEAP *psPhysHeap);
+#endif
+
+PVRSRV_ERROR PhysHeapGetCpuPAddr(PHYS_HEAP *psPhysHeap,
+									   IMG_CPU_PHYADDR *psCpuPAddr);
+
+
+PVRSRV_ERROR PhysHeapGetSize(PHYS_HEAP *psPhysHeap,
+								   IMG_UINT64 *puiSize);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetMemInfo
+@Description    Get phys heap memory statistics for a given physical heap ID.
+@Input          psDevNode          Pointer to device node struct
+@Input          ui32PhysHeapCount  Physical heap count
+@Input          paePhysHeapID      Physical heap ID
+@Output         paPhysHeapMemStats Buffer that holds the memory statistics
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysHeapGetMemInfo(PPVRSRV_DEVICE_NODE psDevNode,
+                   IMG_UINT32 ui32PhysHeapCount,
+                   PVRSRV_PHYS_HEAP *paePhysHeapID,
+                   PHYS_HEAP_MEM_STATS_PTR paPhysHeapMemStats);
+
+PVRSRV_ERROR PhysHeapGetDevPAddr(PHYS_HEAP *psPhysHeap,
+								 IMG_DEV_PHYADDR *psDevPAddr);
+
+void PhysHeapCpuPAddrToDevPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr);
+
+void PhysHeapDevPAddrToCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr,
+								IMG_DEV_PHYADDR *psDevPAddr);
+
+IMG_CHAR *PhysHeapPDumpMemspaceName(PHYS_HEAP *psPhysHeap);
+
+const IMG_CHAR *PhysHeapName(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreatePMR
+@Description    Function calls an implementation-specific function pointer.
+                See function pointer for details.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreatePMR(PHYS_HEAP *psPhysHeap,
+							   struct _CONNECTION_DATA_ *psConnection,
+							   IMG_DEVMEM_SIZE_T uiSize,
+							   IMG_UINT32 ui32NumPhysChunks,
+							   IMG_UINT32 ui32NumVirtChunks,
+							   IMG_UINT32 *pui32MappingTable,
+							   IMG_UINT32 uiLog2PageSize,
+							   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							   const IMG_CHAR *pszAnnotation,
+							   IMG_PID uiPid,
+							   PMR **ppsPMRPtr,
+							   IMG_UINT32 ui32PDumpFlags,
+							   PVRSRV_MEMALLOCFLAGS_T *uiOutFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreatePMB
+@Description    Function calls an implementation-specific function pointer.
+                See @Ref PFN_CREATE_PMB "PFN_CREATE_PMB" for details.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreatePMB(PHYS_HEAP *psPhysHeap,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               const IMG_CHAR *pszAnnotation,
+                               PMB **ppsPMRPtr,
+                               RA_BASE_T *puiBase,
+                               RA_LENGTH_T *puiSize);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDeviceNode
+@Description    Get pointer to the device node this heap belongs to.
+@Input          psPhysHeap          Pointer to physical heap.
+@Return         PPVRSRV_DEVICE_NODE Pointer to device node.
+*/ /**************************************************************************/
+PPVRSRV_DEVICE_NODE PhysHeapDeviceNode(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapInitByPVRLayer
+@Description    Is phys heap to be initialised in PVR layer?
+@Input          ePhysHeap           phys heap
+@Return         IMG_BOOL            return IMG_TRUE if yes
+*/ /**************************************************************************/
+IMG_BOOL PhysHeapInitByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapUserModeAlloc
+@Description    Is allocation from UM allowed?
+@Input          ePhysHeap           phys heap
+@Return         IMG_BOOL            return IMG_TRUE if yes
+*/ /**************************************************************************/
+IMG_BOOL PhysHeapUserModeAlloc(PVRSRV_PHYS_HEAP ePhysHeap);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+PVRSRV_ERROR PhysHeapPagesAllocGPV(PHYS_HEAP *psPhysHeap,
+                                   size_t uiSize,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_UINT32 ui32OSid, IMG_PID uiPid);
+#endif
+
+PVRSRV_ERROR PhysHeapPagesAlloc(PHYS_HEAP *psPhysHeap,
+                                size_t uiSize,
+                                PG_HANDLE *psMemHandle,
+                                IMG_DEV_PHYADDR *psDevPAddr,
+                                IMG_PID uiPid);
+
+void PhysHeapPagesFree(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle);
+
+PVRSRV_ERROR PhysHeapPagesMap(PHYS_HEAP *psPhysHeap,
+                              PG_HANDLE *pshMemHandle,
+                              size_t uiSize,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              void **pvPtr);
+
+void PhysHeapPagesUnMap(PHYS_HEAP *psPhysHeap,
+                        PG_HANDLE *psMemHandle,
+                        void *pvPtr);
+
+PVRSRV_ERROR PhysHeapPagesClean(PHYS_HEAP *psPhysHeap,
+                                PG_HANDLE *pshMemHandle,
+                                IMG_UINT32 uiOffset,
+                                IMG_UINT32 uiLength);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetPageShift
+@Description    Get phys heap page shift.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         IMG_UINT32   Log2 page shift
+*/ /**************************************************************************/
+IMG_UINT32 PhysHeapGetPageShift(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapFreeMemCheck
+@Description    Check a physheap has the required amount of free memory.
+
+@Input          psPhysHeap          Pointer to physical heap.
+@Input          ui64MinRequiredMem  The minimum free memory for success (bytes).
+@Output         pui64FreeMem        The free memory in the physical heap (bytes).
+
+@Return         PVRSRV_ERROR    If successful PVRSRV_OK else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapFreeMemCheck(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 ui64MinRequiredMem,
+                                  IMG_UINT64 *pui64FreeMem);
+
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+/*************************************************************************/ /*!
+@Function       PhysHeapSpasWithDevice
+@Description    Check to see if the device can "see" the SPAS region of the heap.
+                Or in other words, check if any physheap exists
+                in the same SPAS region as `psFromPhysHeap` and is tied to `psToDevNode`.
+@Input          psFromPhysHeap  The physheap whose SPAS region to match.
+@Input          psToDevNode     The device node the other physheap must be tied to.
+@Return         PVRSRV_ERROR    PVRSRV_OK on success, error code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapSpasWithDevice(PHYS_HEAP* psFromPhysHeap, PPVRSRV_DEVICE_NODE psToDevNode);
+#endif
+
+#endif /* PHYSHEAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/physheap_config.h b/drivers/gpu/drm/img/img-volcanic/services/include/physheap_config.h
new file mode 100644
index 000000000000..53d4c909331f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/physheap_config.h
@@ -0,0 +1,331 @@
+/*************************************************************************/ /*!
+@File           physheap_config.h
+@Title          Physical heap Config API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Physical heap configs are created in the system layer and
+                stored against each device node for use in the Services Server
+                common layer.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSHEAP_CONFIG_H
+#define PHYSHEAP_CONFIG_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "lock_types.h"
+#include "dllist.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+
+typedef IMG_UINT32 PHYS_HEAP_USAGE_FLAGS;
+
+/**
+ * ! IMPORTANT !
+ * If you update the PHYS_HEAP_USAGE_FLAGS definitions, you must update the
+ * g_asPhysHeapUsageFlagStrings structure within physheap.c and the
+ * PHYS_HEAP_USAGE_MASK.
+ */
+#define PHYS_HEAP_USAGE_CPU_LOCAL      (1U <<PVRSRV_PHYS_HEAP_CPU_LOCAL)
+#define PHYS_HEAP_USAGE_GPU_LOCAL      (1U <<PVRSRV_PHYS_HEAP_GPU_LOCAL)
+#define PHYS_HEAP_USAGE_GPU_PRIVATE    (1U <<PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+#define PHYS_HEAP_USAGE_EXTERNAL       (1U <<PVRSRV_PHYS_HEAP_EXTERNAL)
+#define PHYS_HEAP_USAGE_GPU_COHERENT   (1U <<PVRSRV_PHYS_HEAP_GPU_COHERENT)
+#define PHYS_HEAP_USAGE_GPU_SECURE     (1U <<PVRSRV_PHYS_HEAP_GPU_SECURE)
+#define PHYS_HEAP_USAGE_FW_CODE        (1U <<PVRSRV_PHYS_HEAP_FW_CODE)
+#define PHYS_HEAP_USAGE_FW_PRIV_DATA   (1U <<PVRSRV_PHYS_HEAP_FW_PRIV_DATA)
+#define PHYS_HEAP_USAGE_FW_PREMAP_PT   (1U <<PVRSRV_PHYS_HEAP_FW_PREMAP_PT)
+#define PHYS_HEAP_USAGE_FW_PREMAP      (1U <<PVRSRV_PHYS_HEAP_FW_PREMAP0)
+#define PHYS_HEAP_USAGE_WRAP           (1U <<PVRSRV_PHYS_HEAP_WRAP)
+#define PHYS_HEAP_USAGE_DISPLAY        (1U <<PVRSRV_PHYS_HEAP_DISPLAY)
+#define PHYS_HEAP_USAGE_DLM            (1U <<29)
+#define PHYS_HEAP_USAGE_FW_SHARED      (1U <<30)
+#define PHYS_HEAP_USAGE_FW_PRIVATE     (1U <<31)
+
+#define PHYS_HEAP_USAGE_MASK (PHYS_HEAP_USAGE_CPU_LOCAL    | \
+                              PHYS_HEAP_USAGE_GPU_LOCAL    | \
+                              PHYS_HEAP_USAGE_GPU_PRIVATE  | \
+                              PHYS_HEAP_USAGE_EXTERNAL     | \
+                              PHYS_HEAP_USAGE_GPU_COHERENT | \
+                              PHYS_HEAP_USAGE_GPU_SECURE   | \
+                              PHYS_HEAP_USAGE_FW_CODE      | \
+                              PHYS_HEAP_USAGE_FW_PRIV_DATA | \
+                              PHYS_HEAP_USAGE_FW_SHARED    | \
+                              PHYS_HEAP_USAGE_FW_PRIVATE   | \
+                              PHYS_HEAP_USAGE_FW_PREMAP    | \
+                              PHYS_HEAP_USAGE_FW_PREMAP_PT | \
+                              PHYS_HEAP_USAGE_WRAP         | \
+                              PHYS_HEAP_USAGE_DISPLAY      | \
+                              PHYS_HEAP_USAGE_DLM)
+
+#define FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM    PVRSRV_PHYS_HEAP_FW_CODE
+
+typedef void (*CpuPAddrToDevPAddr)(IMG_HANDLE hPrivData,
+                                   IMG_UINT32 ui32NumOfAddr,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_CPU_PHYADDR *psCpuPAddr);
+
+typedef void (*DevPAddrToCpuPAddr)(IMG_HANDLE hPrivData,
+                                   IMG_UINT32 ui32NumOfAddr,
+                                   IMG_CPU_PHYADDR *psCpuPAddr,
+                                   IMG_DEV_PHYADDR *psDevPAddr);
+
+/*! Structure used to hold function pointers used for run-time physical address
+ * translation by Services. Gives flexibility to allow the CPU and GPU to see
+ * the same pool of physical RAM and different physical bus addresses.
+ * Both fields must be valid functions even if the conversion is simple.
+ */
+typedef struct _PHYS_HEAP_FUNCTIONS_
+{
+	/*! Translate CPU physical address to device physical address */
+	CpuPAddrToDevPAddr	pfnCpuPAddrToDevPAddr;
+	/*! Translate device physical address to CPU physical address */
+	DevPAddrToCpuPAddr	pfnDevPAddrToCpuPAddr;
+} PHYS_HEAP_FUNCTIONS;
+
+/*! Structure used to describe the IPA settings for a physical heap.
+ */
+typedef struct _IPA_CONFIG_
+{
+	/*! IPA Policy Default (0-7) */
+	IMG_UINT8             ui8IPAPolicyDefault;
+	/*! IPA Policy Mask (3-bits for this use-case - 0x7) */
+	IMG_UINT8             ui8IPAPolicyMask;
+	/*! IPA Policy mask shift (LSB position - policy value will be << by this amount */
+	IMG_UINT8             ui8IPAPolicyShift;
+	/*! IPA Policy reserved byte for alignment and slop reduction */
+	IMG_UINT8             ui8IPAPolicyReserved;
+} IPA_CONFIG;
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+/* A Shared Physical Address Space (SPAS) region represents a group of physheaps
+ * that have this SPAS property. If 2 physheaps on separate devices are linked with
+ * the same SPAS region, the devices would be able to share one another's memory.
+ *
+ * The region is a dllist of physheaps.
+ * If a heap is in the list, it's in the region. */
+typedef struct _PHYS_HEAP_SPAS_REGION_ {
+	/* The head of the physheap list.
+	 * May be empty. */
+	DLLIST_NODE sListHead;
+	/* Protects the list. */
+	ATOMIC_T ui32Lock;
+} PHYS_HEAP_SPAS_REGION;
+
+#define DECLARE_PHYS_HEAP_SPAS_REGION(name)            \
+PHYS_HEAP_SPAS_REGION name = {                         \
+	.ui32Lock = { 0 },                                 \
+	.sListHead = { &name.sListHead, &name.sListHead }  \
+}
+#endif
+
+typedef struct _PHYS_HEAP_CONFIG_LMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+													 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_CPU_PHYADDR       sStartAddr;           /*!< CPU Physical base address of memory region */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Device physical base address of memory
+													 region as seen from the PoV of the GPU */
+	IMG_UINT64            uiSize;               /*!< Size of memory region in bytes */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+													 psMemFuncs */
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	PHYS_HEAP_SPAS_REGION *psSpasRegion;        /*!< Physheaps with the same SpasRegion are seen
+	                                                 to share the physical address space. */
+#endif
+} PHYS_HEAP_CONFIG_LMA;
+
+typedef struct _PHYS_HEAP_CONFIG_UMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+													 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Optional Device physical base address of memory
+													 region as seen from the PoV of the GPU */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+													 psMemFuncs */
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	PHYS_HEAP_SPAS_REGION *psSpasRegion;        /*!< Physheaps with the same SpasRegion are seen
+	                                                 to share the physical address space. */
+#endif
+} PHYS_HEAP_CONFIG_UMA;
+
+typedef struct _PHYS_HEAP_CONFIG_DLM_
+{
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_UINT32            ui32Log2PMBSize;      /*!< PMB (Physical Memory Block) Log 2 size in bytes
+	                                                 for DLM heap. */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CPU_PHYADDR       sStartAddr;           /*!< CPU Physical base address of memory region */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Device physical base address of memory
+	                                                 region as seen from the PoV of the GPU */
+	IMG_UINT64            uiSize;               /*!< Size of memory region in bytes */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+                                                     psMemFuncs */
+} PHYS_HEAP_CONFIG_DLM;
+
+typedef struct _PHYS_HEAP_CONFIG_IMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+	                                                 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+	                                                 psMemFuncs */
+	IMG_UINT32            ui32PMBStartingMultiple; /*!< Multiple of PMB size defined in DLM heap
+	                                                    to be allocated on creation */
+	IMG_UINT32            uiDLMHeapIdx;         /*!< The index in the array of physheaps to a
+	                                                 DLM heap to import PMBs from. */
+} PHYS_HEAP_CONFIG_IMA;
+
+/*! Structure used to describe a physical Heap supported by a system. A
+ * system layer module can declare multiple physical heaps for different
+ * purposes. At a minimum a system must provide one physical heap tagged for
+ * PHYS_HEAP_USAGE_GPU_LOCAL use.
+ * A heap represents a discrete pool of physical memory and how it is managed,
+ * as well as associating other properties and address translation logic.
+ * The structure fields sStartAddr, sCardBase and uiSize must be given valid
+ * values for LMA and DMA physical heaps types.
+ */
+typedef struct _PHYS_HEAP_CONFIG_
+{
+	PHYS_HEAP_TYPE        eType;                /*!< Class of heap and PMR factory used */
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;       /*!< Supported uses flags, conveys the type of
+	                                                 buffers the physical heap can be used for */
+	IPA_CONFIG            sIPAConfig;           /*!< IPA configuration to be applied to all
+	                                                 requested physical addresses when physically
+	                                                 backed */
+	union {
+		PHYS_HEAP_CONFIG_LMA sLMA;
+		PHYS_HEAP_CONFIG_LMA sDMA;
+		PHYS_HEAP_CONFIG_UMA sUMA;
+		PHYS_HEAP_CONFIG_DLM sDLM;
+		PHYS_HEAP_CONFIG_IMA sIMA;
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+		PHYS_HEAP_CONFIG_UMA sWRAP;
+#endif
+	} uConfig;
+} PHYS_HEAP_CONFIG;
+
+/* Code throughout the driver sometimes relies on the fact that the backing for
+ * LMA and DMA heaps are the same struct, this check is not comprehensive as the structs
+ * could have changed but kept the same overall size, although this is unlikely.
+ */
+static_assert(sizeof(((PHYS_HEAP_CONFIG*)0)->uConfig.sLMA) == sizeof(((PHYS_HEAP_CONFIG*)0)->uConfig.sDMA),
+              "PHYS_HEAP_CONFIG sizeof sLMA != sDMA, Type has potentially changed");
+
+static INLINE IMG_UINT64 PhysHeapConfigGetSize(PHYS_HEAP_CONFIG *psConfig)
+{
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.uiSize;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no size member");
+		return 0;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.uiSize;
+#endif
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.uiSize;
+	case PHYS_HEAP_TYPE_UMA:
+		PVR_ASSERT(!"UMA Config has no size member");
+		return 0;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return 0;
+	}
+}
+
+static INLINE IMG_CPU_PHYADDR PhysHeapConfigGetStartAddr(PHYS_HEAP_CONFIG *psConfig)
+{
+	IMG_CPU_PHYADDR sUnsupportedPhyAddr = {0};
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.sStartAddr;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no StartAddr member");
+		return sUnsupportedPhyAddr;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.sStartAddr;
+#endif
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.sStartAddr;
+	case PHYS_HEAP_TYPE_UMA:
+		PVR_ASSERT(!"UMA Config has no StartAddr member");
+		return sUnsupportedPhyAddr;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return sUnsupportedPhyAddr;
+	}
+}
+
+static INLINE IMG_DEV_PHYADDR PhysHeapConfigGetCardBase(PHYS_HEAP_CONFIG *psConfig)
+{
+	IMG_DEV_PHYADDR sUnsupportedPhyAddr = {0};
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.sCardBase;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no CardBase member");
+		return sUnsupportedPhyAddr;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.sCardBase;
+#endif
+	case PHYS_HEAP_TYPE_UMA:
+		return psConfig->uConfig.sUMA.sCardBase;
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.sCardBase;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return sUnsupportedPhyAddr;
+	}
+}
+
+#define PHYS_HEAP_NAME_SIZE 24
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pvr_bridge.h b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_bridge.h
new file mode 100644
index 000000000000..dafc51dd57f0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_bridge.h
@@ -0,0 +1,443 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the PVR Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BRIDGE_H
+#define PVR_BRIDGE_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "pvrsrv_error.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "common_dc_bridge.h"
+#if defined(SUPPORT_DCPLAT_BRIDGE)
+#include "common_dcplat_bridge.h"
+#endif
+#endif
+#include "common_mm_bridge.h"
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+#include "common_mmplat_bridge.h"
+#endif
+#if defined(SUPPORT_WRAP_EXTMEM)
+#include "common_mmextmem_bridge.h"
+#endif
+#if !defined(EXCLUDE_CMM_BRIDGE)
+#include "common_cmm_bridge.h"
+#endif
+#if defined(__linux__)
+#include "common_dmabuf_bridge.h"
+#endif
+#if defined(PDUMP)
+#include "common_pdump_bridge.h"
+#include "common_pdumpctrl_bridge.h"
+#include "common_pdumpmm_bridge.h"
+#endif
+#include "common_cache_bridge.h"
+#if defined(SUPPORT_DMA_TRANSFER)
+#include "common_dma_bridge.h"
+#endif
+#include "common_srvcore_bridge.h"
+#include "common_sync_bridge.h"
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "common_smm_bridge.h"
+#endif
+#if defined(PVRSRV_ENABLE_HTB)
+#include "common_htbuffer_bridge.h"
+#endif
+#include "common_pvrtl_bridge.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "common_ri_bridge.h"
+#endif
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+#include "common_validation_bridge.h"
+#endif
+
+
+#include "common_devicememhistory_bridge.h"
+#include "common_synctracking_bridge.h"
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#include "common_syncfallback_bridge.h"
+#endif
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+#include "common_di_bridge.h"
+#endif
+
+/*
+ * Bridge Cmd Ids
+ */
+
+
+/* Note: The pattern
+ *   #define PVRSRV_BRIDGE_FEATURE (PVRSRV_BRIDGE_PREVFEATURE + 1)
+ *   #if defined(SUPPORT_FEATURE)
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST (PVRSRV_BRIDGE_PREVFEATURE_DISPATCH_LAST + 1)
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST  (PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST + PVRSRV_BRIDGE_FEATURE_CMD_LAST)
+ *   #else
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST 0
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST  (PVRSRV_BRIDGE_PREVFEATURE_DISPATCH_LAST)
+ *   #endif
+ * is used in the macro definitions below to make PVRSRV_BRIDGE_FEATURE_*
+ * take up no space in the dispatch table if SUPPORT_FEATURE is disabled.
+ *
+ * Note however that a bridge always defines PVRSRV_BRIDGE_FEATURE, even where
+ * the feature is not enabled (each bridge group retains its own ioctl number).
+ */
+
+#define PVRSRV_BRIDGE_FIRST					0UL
+
+/*   0: Default handler */
+#define PVRSRV_BRIDGE_DEFAULT				0UL
+#define PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST 0UL
+#define PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST  (PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST)
+/*   1: CORE functions */
+#define PVRSRV_BRIDGE_SRVCORE				1UL
+#define PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST (PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST+1)
+#define PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST  (PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST + PVRSRV_BRIDGE_SRVCORE_CMD_LAST)
+
+/*   2: SYNC functions */
+#define PVRSRV_BRIDGE_SYNC					2UL
+#define PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST (PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNC_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNC_CMD_LAST)
+
+/* 3,4: Reserved */
+#define PVRSRV_BRIDGE_RESERVED1				3UL
+#define PVRSRV_BRIDGE_RESERVED1_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST)
+
+#define PVRSRV_BRIDGE_RESERVED2				4UL
+#define PVRSRV_BRIDGE_RESERVED2_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RESERVED2_DISPATCH_LAST  (PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST)
+
+/*   5: PDUMP CTRL layer functions */
+#define PVRSRV_BRIDGE_PDUMPCTRL				5UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMPCTRL_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST)
+#endif
+
+/*   6: Memory Management functions */
+#define PVRSRV_BRIDGE_MM					6UL
+#define PVRSRV_BRIDGE_MM_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MM_DISPATCH_LAST  (PVRSRV_BRIDGE_MM_DISPATCH_FIRST + PVRSRV_BRIDGE_MM_CMD_LAST)
+
+/*   7: Non-Linux Memory Management functions */
+#define PVRSRV_BRIDGE_MMPLAT				7UL
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST (PVRSRV_BRIDGE_MM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST + PVRSRV_BRIDGE_MMPLAT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_MM_DISPATCH_LAST)
+#endif
+
+/*   8: Context Memory Management functions */
+#define PVRSRV_BRIDGE_CMM					8UL
+#if !defined(EXCLUDE_CMM_BRIDGE)
+#define PVRSRV_BRIDGE_CMM_DISPATCH_FIRST (PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_CMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CMM_DISPATCH_FIRST + PVRSRV_BRIDGE_CMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_CMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_CMM_DISPATCH_LAST  (PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST)
+#endif
+
+/*   9: PDUMP Memory Management functions */
+#define PVRSRV_BRIDGE_PDUMPMM				9UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST (PVRSRV_BRIDGE_CMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMPMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CMM_DISPATCH_LAST)
+#endif
+
+/*   10: PDUMP functions */
+#define PVRSRV_BRIDGE_PDUMP					10UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMP_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST)
+#endif
+
+/*  11: DMABUF functions */
+#define PVRSRV_BRIDGE_DMABUF				11UL
+#if defined(__linux__)
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST  (PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST + PVRSRV_BRIDGE_DMABUF_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST)
+#endif
+
+/*  12: Display Class functions */
+#define PVRSRV_BRIDGE_DC					12UL
+#if defined(SUPPORT_DISPLAY_CLASS)
+#define PVRSRV_BRIDGE_DC_DISPATCH_FIRST (PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DC_DISPATCH_LAST  (PVRSRV_BRIDGE_DC_DISPATCH_FIRST + PVRSRV_BRIDGE_DC_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DC_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DC_DISPATCH_LAST  (PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST)
+#endif
+
+/*  13: Cache interface functions */
+#define PVRSRV_BRIDGE_CACHE					13UL
+#define PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST (PVRSRV_BRIDGE_DC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_CACHE_DISPATCH_LAST  (PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST + PVRSRV_BRIDGE_CACHE_CMD_LAST)
+
+/*  14: Secure Memory Management functions */
+#define PVRSRV_BRIDGE_SMM					14UL
+#if defined(SUPPORT_SECURE_EXPORT)
+#define PVRSRV_BRIDGE_SMM_DISPATCH_FIRST (PVRSRV_BRIDGE_CACHE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SMM_DISPATCH_LAST  (PVRSRV_BRIDGE_SMM_DISPATCH_FIRST + PVRSRV_BRIDGE_SMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_SMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_SMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CACHE_DISPATCH_LAST)
+#endif
+
+/*  15: Transport Layer interface functions */
+#define PVRSRV_BRIDGE_PVRTL					15UL
+#define PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST (PVRSRV_BRIDGE_SMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST  (PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST + PVRSRV_BRIDGE_PVRTL_CMD_LAST)
+
+/*  16: Resource Information (RI) interface functions */
+#define PVRSRV_BRIDGE_RI					16UL
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#define PVRSRV_BRIDGE_RI_DISPATCH_FIRST (PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RI_DISPATCH_LAST  (PVRSRV_BRIDGE_RI_DISPATCH_FIRST + PVRSRV_BRIDGE_RI_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RI_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RI_DISPATCH_LAST  (PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST)
+#endif
+
+/*  17: Validation interface functions */
+#define PVRSRV_BRIDGE_VALIDATION			17UL
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST (PVRSRV_BRIDGE_RI_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST  (PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST + PVRSRV_BRIDGE_VALIDATION_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST  (PVRSRV_BRIDGE_RI_DISPATCH_LAST)
+#endif
+
+/*  18: TUTILS interface functions */
+#define PVRSRV_BRIDGE_TUTILS				18UL
+#define PVRSRV_BRIDGE_TUTILS_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST  (PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST)
+
+/*  19: DevMem history interface functions */
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY		19UL
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST (PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST  (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST + PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST)
+
+/*  20: Host Trace Buffer interface functions */
+#define PVRSRV_BRIDGE_HTBUFFER				20UL
+#if defined(PVRSRV_ENABLE_HTB)
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST  (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST + PVRSRV_BRIDGE_HTBUFFER_CMD_LAST)
+#else	/* !PVRSRV_ENABLE_HTB */
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST  (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST)
+#endif	/* PVRSRV_ENABLE_HTB */
+
+/*  21: Non-Linux Display functions */
+#define PVRSRV_BRIDGE_DCPLAT				21UL
+#if defined(SUPPORT_DISPLAY_CLASS) && defined(SUPPORT_DCPLAT_BRIDGE)
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST + PVRSRV_BRIDGE_DCPLAT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST)
+#endif
+
+/*  22: Extmem functions */
+#define PVRSRV_BRIDGE_MMEXTMEM				22UL
+#if defined(SUPPORT_WRAP_EXTMEM)
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST (PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST  (PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST + PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST (PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST)
+#endif
+
+/*  23: Sync tracking functions */
+#define PVRSRV_BRIDGE_SYNCTRACKING			23UL
+#define PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST (PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST)
+
+/*  24: Sync fallback functions */
+#define PVRSRV_BRIDGE_SYNCFALLBACK			24UL
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNCFALLBACK_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST)
+#endif
+
+/*  25: Debug Information (DI) interface functions */
+#define PVRSRV_BRIDGE_DI					25UL
+#if defined(SUPPORT_DI_BRG_IMPL)
+#define PVRSRV_BRIDGE_DI_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DI_DISPATCH_LAST  (PVRSRV_BRIDGE_DI_DISPATCH_FIRST + PVRSRV_BRIDGE_DI_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DI_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DI_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST)
+#endif
+
+/*  26: DMA transfer functions */
+
+#define PVRSRV_BRIDGE_DMA			26UL
+#if defined(SUPPORT_DMA_TRANSFER)
+#define PVRSRV_BRIDGE_DMA_DISPATCH_FIRST (PVRSRV_BRIDGE_DI_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DMA_DISPATCH_LAST  (PVRSRV_BRIDGE_DMA_DISPATCH_FIRST + PVRSRV_BRIDGE_DMA_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DMA_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DMA_DISPATCH_LAST  (PVRSRV_BRIDGE_DI_DISPATCH_LAST)
+#endif
+
+/* NB PVRSRV_BRIDGE_LAST below must be the last bridge group defined above (PVRSRV_BRIDGE_FEATURE) */
+#define PVRSRV_BRIDGE_LAST					(PVRSRV_BRIDGE_DMA)
+/* NB PVRSRV_BRIDGE_DISPATCH LAST below must be the last dispatch entry defined above (PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST) */
+#define PVRSRV_BRIDGE_DISPATCH_LAST			(PVRSRV_BRIDGE_DMA_DISPATCH_LAST)
+
+/* bit mask representing the enabled PVR bridges */
+
+static const IMG_UINT32 gui32PVRBridges =
+	  (1U << (PVRSRV_BRIDGE_DEFAULT - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_SRVCORE - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_SYNC - PVRSRV_BRIDGE_FIRST))
+
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_PDUMPCTRL - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_MM - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_MMPLAT - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_CMM)
+	| (1U << (PVRSRV_BRIDGE_CMM - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_PDUMPMM - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_PDUMP - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(__linux__)
+	| (1U << (PVRSRV_BRIDGE_DMABUF - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+	| (1U << (PVRSRV_BRIDGE_DC - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_CACHE - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_SECURE_EXPORT)
+	| (1U << (PVRSRV_BRIDGE_SMM - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_PVRTL - PVRSRV_BRIDGE_FIRST))
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	| (1U << (PVRSRV_BRIDGE_RI - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_DEVICEMEMHISTORY - PVRSRV_BRIDGE_FIRST))
+#if defined(PVRSRV_ENABLE_HTB)
+	| (1U << (PVRSRV_BRIDGE_HTBUFFER - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS) && defined(SUPPORT_DCPLAT_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_DCPLAT - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_WRAP_EXTMEM)
+	| (1U << (PVRSRV_BRIDGE_MMEXTMEM - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_SYNCTRACKING - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	| (1U << (PVRSRV_BRIDGE_SYNCFALLBACK - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DI_BRG_IMPL)
+	| (1U << (PVRSRV_BRIDGE_DI - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DMA_TRANSFER)
+	| (1U << (PVRSRV_BRIDGE_DMA - PVRSRV_BRIDGE_FIRST))
+#endif
+	;
+
+/* bit field representing which PVR bridge groups may optionally not
+ * be present in the server
+ */
+#define PVR_BRIDGES_OPTIONAL \
+	( \
+		(1U << (PVRSRV_BRIDGE_RI - PVRSRV_BRIDGE_FIRST)) \
+	)
+
+/******************************************************************************
+ * Generic bridge structures
+ *****************************************************************************/
+
+
+/******************************************************************************
+ * bridge packaging structure
+ *****************************************************************************/
+typedef struct PVRSRV_BRIDGE_PACKAGE_TAG
+{
+	IMG_UINT32		ui32BridgeID;			/*!< ioctl bridge group */
+	IMG_UINT32		ui32FunctionID;			/*!< ioctl function index */
+	IMG_UINT32		ui32Size;				/*!< size of structure */
+	void __user		*pvParamIn;				/*!< input data buffer */
+	IMG_UINT32		ui32InBufferSize;		/*!< size of input data buffer */
+	void __user		*pvParamOut;			/*!< output data buffer */
+	IMG_UINT32		ui32OutBufferSize;		/*!< size of output data buffer */
+}PVRSRV_BRIDGE_PACKAGE;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_BRIDGE_H */
+
+/******************************************************************************
+ End of file (pvr_bridge.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pvr_dicommon.h b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_dicommon.h
new file mode 100644
index 000000000000..c729dde2fab6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_dicommon.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Debug Information (DI) common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Debug Information (DI) common types and definitions included
+                in both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DICOMMON_H
+#define PVR_DICOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*! Maximum DI entry path length including the null byte. */
+#define DI_IMPL_BRG_PATH_LEN 64
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_DICOMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/pvr_ricommon.h b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_ricommon.h
new file mode 100644
index 000000000000..0521aa151794
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/pvr_ricommon.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Resource Information (RI) common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) common types and definitions included
+                in both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_RICOMMON_H
+#define PVR_RICOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+/*! Maximum text string length including the null byte */
+#define PRVSRVRI_MAX_TEXT_LENGTH	20U
+
+/* PID used to hold PMR allocations which are driver-wide (i.e. have a lifetime
+ * longer than an application process)
+ */
+#define PVR_SYS_ALLOC_PID 1
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_RICOMMON_H */
+/******************************************************************************
+ End of file (pvr_ricommon.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_bridge.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_bridge.h
new file mode 100644
index 000000000000..5168f6f23ea5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_bridge.h
@@ -0,0 +1,250 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the Rogue Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_BRIDGE_H
+#define RGX_BRIDGE_H
+
+#include "pvr_bridge.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define RGXFWINITPARAMS_VERSION   1
+#define RGXFWINITPARAMS_EXTENSION 128
+
+#include "common_rgxta3d_bridge.h"
+#include "common_rgxcmp_bridge.h"
+#if defined(SUPPORT_FASTRENDER_DM)
+#include "common_rgxtq2_bridge.h"
+#endif
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+#include "common_rgxtq_bridge.h"
+#endif
+#if defined(SUPPORT_USC_BREAKPOINT)
+#include "common_rgxbreakpoint_bridge.h"
+#endif
+#include "common_rgxfwdbg_bridge.h"
+#if defined(PDUMP)
+#include "common_rgxpdump_bridge.h"
+#endif
+#include "common_rgxhwperf_bridge.h"
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+#include "common_rgxregconfig_bridge.h"
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "common_rgxkicksync_bridge.h"
+#endif
+#include "common_rgxtimerquery_bridge.h"
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+#include "common_rgxray_bridge.h"
+#endif
+/*
+ * Bridge Cmd Ids
+ */
+
+/* *REMEMBER* to update PVRSRV_BRIDGE_RGX_LAST if you add/remove a bridge
+ * group!
+ * Also you need to ensure all PVRSRV_BRIDGE_RGX_xxx_DISPATCH_FIRST offsets
+ * follow on from the previous bridge group's commands!
+ *
+ * If a bridge group is optional, ensure you *ALWAYS* define its index
+ * (e.g. PVRSRV_BRIDGE_RGXCMP is always 151, even is the feature is not
+ * defined). If an optional bridge group is not defined you must still
+ * define PVRSRV_BRIDGE_RGX_xxx_DISPATCH_FIRST for it with an assigned
+ * value of 0.
+ */
+
+/* The RGX bridge groups start at 128 (PVRSRV_BRIDGE_RGX_FIRST) rather than
+ * follow-on from the other non-device bridge groups (meaning that they then
+ * won't be displaced if other non-device bridge groups are added).
+ */
+
+#define PVRSRV_BRIDGE_RGX_FIRST                  128UL
+
+/* 128: RGX TQ interface functions */
+#define PVRSRV_BRIDGE_RGXTQ                      128UL
+/* The RGXTQ bridge is conditional since the definitions in this header file
+ * support both the rogue and volcanic servers, but the RGXTQ bridge is not
+ * required at all on the Volcanic architecture.
+ */
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST       (PVRSRV_BRIDGE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST        (PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTQ_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST       0
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST        (PVRSRV_BRIDGE_DISPATCH_LAST)
+#endif
+
+/* 129: RGX Compute interface functions */
+#define PVRSRV_BRIDGE_RGXCMP                     129UL
+#define PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST      (PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXCMP_CMD_LAST)
+
+/* 130: RGX TA/3D interface functions */
+#define PVRSRV_BRIDGE_RGXTA3D                    130UL
+#define PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST     (PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST      (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTA3D_CMD_LAST)
+
+/* 131: RGX Breakpoint interface functions */
+#define PVRSRV_BRIDGE_RGXBREAKPOINT                 131UL
+#if defined(SUPPORT_USC_BREAKPOINT)
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST  0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST)
+#endif
+
+/* 132: RGX Debug/Misc interface functions */
+#define PVRSRV_BRIDGE_RGXFWDBG                   132UL
+#define PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST    (PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST)
+
+/* 133: RGX PDump interface functions */
+#define PVRSRV_BRIDGE_RGXPDUMP                   133UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST    (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXPDUMP_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST    0
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST)
+#endif
+
+/* 134: RGX HWPerf interface functions */
+#define PVRSRV_BRIDGE_RGXHWPERF                  134UL
+#define PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST   (PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST)
+
+/* 135: RGX Register Configuration interface functions */
+#define PVRSRV_BRIDGE_RGXREGCONFIG                  135UL
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST   (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST   0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST)
+#endif
+
+/* 136: RGX kicksync interface */
+#define PVRSRV_BRIDGE_RGXKICKSYNC                136UL
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST  (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST   0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST)
+#endif
+/* 137: RGX TQ2 interface */
+#define PVRSRV_BRIDGE_RGXTQ2                     137UL
+#if defined(SUPPORT_FASTRENDER_DM)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST      (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTQ2_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST      (0)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST)
+#endif
+
+/* 138: RGX timer query interface */
+#define PVRSRV_BRIDGE_RGXTIMERQUERY                 138UL
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST)
+
+/* 139: RGX Ray tracing interface */
+#define PVRSRV_BRIDGE_RGXRAY                 139UL
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXRAY_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST  0
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST)
+#endif
+
+#define PVRSRV_BRIDGE_RGX_LAST                   (PVRSRV_BRIDGE_RGXRAY)
+#define PVRSRV_BRIDGE_RGX_DISPATCH_LAST          (PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST)
+
+/* bit mask representing the enabled RGX bridges */
+
+static const IMG_UINT32 gui32RGXBridges =
+	  (1U << (PVRSRV_BRIDGE_RGXTQ - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(RGX_FEATURE_COMPUTE) || defined(__KERNEL__)
+	| (1U << (PVRSRV_BRIDGE_RGXCMP - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXTA3D - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_BREAKPOINT)
+	| (1U << (PVRSRV_BRIDGE_BREAKPOINT - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXFWDBG - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_RGXPDUMP - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_RGXKICKSYNC - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXHWPERF - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_REGCONFIG)
+	| (1U << (PVRSRV_BRIDGE_RGXREGCONFIG - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXKICKSYNC - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_FASTRENDER_DM) || defined(__KERNEL__)
+	| (1U << (PVRSRV_BRIDGE_RGXTQ2 - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+#if defined(SUPPORT_TIMERQUERY)
+	| (1U << (PVRSRV_BRIDGE_RGXTIMERQUERY - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXRAY - PVRSRV_BRIDGE_RGX_FIRST))
+	;
+/* bit field representing which RGX bridge groups may optionally not
+ * be present in the server
+ */
+
+#define RGX_BRIDGES_OPTIONAL \
+	( \
+		0 /* no RGX bridges are currently optional */ \
+	)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_BRIDGE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_fw_info.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_fw_info.h
new file mode 100644
index 000000000000..fb31ac440024
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_fw_info.h
@@ -0,0 +1,144 @@
+/*************************************************************************/ /*!
+@File
+@Title          FW image information
+
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FW_INFO_H)
+#define RGX_FW_INFO_H
+
+#include "img_types.h"
+#include "rgx_common.h"
+
+/*
+ * Firmware binary block unit in bytes.
+ * Raw data stored in FW binary will be aligned to this size.
+ */
+#define FW_BLOCK_SIZE 4096L
+
+typedef enum
+{
+	META_CODE = 0,
+	META_PRIVATE_DATA,
+	META_COREMEM_CODE,
+	META_COREMEM_DATA,
+	MIPS_CODE,
+	MIPS_EXCEPTIONS_CODE,
+	MIPS_BOOT_CODE,
+	MIPS_PRIVATE_DATA,
+	MIPS_BOOT_DATA,
+	MIPS_STACK,
+	RISCV_UNCACHED_CODE,
+	RISCV_CACHED_CODE,
+	RISCV_PRIVATE_DATA,
+	RISCV_COREMEM_CODE,
+	RISCV_COREMEM_DATA,
+} RGX_FW_SECTION_ID;
+
+typedef enum
+{
+	NONE = 0,
+	FW_CODE,
+	FW_DATA,
+	FW_COREMEM_CODE,
+	FW_COREMEM_DATA
+} RGX_FW_SECTION_TYPE;
+
+
+/*
+ * FW binary format with FW info attached:
+ *
+ *          Contents        Offset
+ *     +-----------------+
+ *     |                 |    0
+ *     |                 |
+ *     | Original binary |
+ *     |      file       |
+ *     |   (.ldr/.elf)   |
+ *     |                 |
+ *     |                 |
+ *     +-----------------+
+ *     | FW info header  |  FILE_SIZE - 4K
+ *     +-----------------+
+ *     |                 |
+ *     | FW layout table |
+ *     |                 |
+ *     +-----------------+
+ *                          FILE_SIZE
+ */
+
+#define FW_INFO_VERSION  (2)
+
+/* Firmware is built for open source driver and uses open source version numbering */
+#define FW_INFO_FLAGS_OPEN_SOURCE (1U)
+
+typedef struct
+{
+	/* FW_INFO_VERSION 1 */
+	IMG_UINT32 ui32InfoVersion;      /* FW info version */
+	IMG_UINT32 ui32HeaderLen;        /* Header length */
+	IMG_UINT32 ui32LayoutEntryNum;   /* Number of entries in the layout table */
+	IMG_UINT32 ui32LayoutEntrySize;  /* Size of an entry in the layout table */
+	IMG_UINT64 RGXFW_ALIGN ui64BVNC; /* BVNC */
+	IMG_UINT32 ui32FwPageSize;       /* Page size of processor on which firmware executes */
+	IMG_UINT32 ui32Flags;            /* Compatibility flags */
+
+	/* FW_INFO_VERSION 2 */
+	IMG_UINT16 ui16PVRVersionMajor;  /* DDK major version number */
+	IMG_UINT16 ui16PVRVersionMinor;  /* DDK minor version number */
+	IMG_UINT32 ui32PVRVersionBuild;  /* DDK build number */
+} RGX_FW_INFO_HEADER;
+
+typedef struct
+{
+	RGX_FW_SECTION_ID eId;
+	RGX_FW_SECTION_TYPE eType;
+	IMG_UINT32 ui32BaseAddr;
+	IMG_UINT32 ui32MaxSize;
+	IMG_UINT32 ui32AllocSize;
+	IMG_UINT32 ui32AllocOffset;
+} RGX_FW_LAYOUT_ENTRY;
+
+#endif /* RGX_FW_INFO_H */
+
+/******************************************************************************
+ End of file (rgx_fw_info.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_memallocflags.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_memallocflags.h
new file mode 100644
index 000000000000..e26f42c4f935
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_memallocflags.h
@@ -0,0 +1,58 @@
+/**************************************************************************/ /*!
+@File
+@Title          RGX device specific memory allocation flags
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_MEMALLOCFLAGS_H
+#define RGX_MEMALLOCFLAGS_H
+
+
+/* Include pvrsrv layer header as the flags below are used in the device
+ * field defined in this header inside Services code.
+ * See PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK */
+#include "pvrsrv_memallocflags.h"
+
+
+/* Device specific MMU flags */
+#define PMMETA_PROTECT          (1U << 0)      /*!< Memory that only the PM and Meta can access */
+#define FIRMWARE_CACHED         (1U << 1)      /*!< Memory that is cached in META/MIPS */
+
+
+#endif /* RGX_MEMALLOCFLAGS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_pdump_panics.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_pdump_panics.h
new file mode 100644
index 000000000000..fce2b3efab69
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_pdump_panics.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX PDump panic definitions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX PDump panic definitions header
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_PDUMP_PANICS_H_)
+#define RGX_PDUMP_PANICS_H_
+
+/*! Unique device specific IMG_UINT16 panic IDs to identify the cause of an
+ * RGX PDump panic in a PDump script. */
+typedef enum
+{
+	RGX_PDUMP_PANIC_UNDEFINED = 0,
+
+	/* These panics occur when test parameters and driver configuration
+	 * enable features that require the firmware and host driver to
+	 * communicate. Such features are not supported with off-line playback.
+	 */
+	RGX_PDUMP_PANIC_ZSBUFFER_BACKING         = 101, /*!< Requests ZSBuffer to be backed with physical pages */
+	RGX_PDUMP_PANIC_ZSBUFFER_UNBACKING       = 102, /*!< Requests ZSBuffer to be unbacked */
+	RGX_PDUMP_PANIC_FREELIST_GROW            = 103, /*!< Requests an on-demand freelist grow/shrink */
+	RGX_PDUMP_PANIC_FREELISTS_RECONSTRUCTION = 104, /*!< Requests freelists reconstruction */
+	RGX_PDUMP_PANIC_SPARSEMEM_SWAP           = 105, /*!< Requests sparse remap memory swap feature */
+} RGX_PDUMP_PANIC;
+
+#endif /* RGX_PDUMP_PANICS_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgx_tq_shared.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_tq_shared.h
new file mode 100644
index 000000000000..2db19224c441
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgx_tq_shared.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX transfer queue shared
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared definitions between client and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_TQ_SHARED_H
+#define RGX_TQ_SHARED_H
+
+#define TQ_MAX_PREPARES_PER_SUBMIT		16U
+
+#define TQ_PREP_FLAGS_COMMAND_3D		0x0U
+#define TQ_PREP_FLAGS_COMMAND_2D		0x1U
+#define TQ_PREP_FLAGS_COMMAND_MASK		(0xfU)
+#define TQ_PREP_FLAGS_COMMAND_SHIFT		0
+#define TQ_PREP_FLAGS_PDUMPCONTINUOUS	(1U << 4)
+
+#define TQ_PREP_FLAGS_COMMAND_SET(m) \
+	((TQ_PREP_FLAGS_COMMAND_##m << TQ_PREP_FLAGS_COMMAND_SHIFT) & TQ_PREP_FLAGS_COMMAND_MASK)
+
+#define TQ_PREP_FLAGS_COMMAND_IS(m,n) \
+	(((m & TQ_PREP_FLAGS_COMMAND_MASK) >> TQ_PREP_FLAGS_COMMAND_SHIFT)  == TQ_PREP_FLAGS_COMMAND_##n)
+
+#endif /* RGX_TQ_SHARED_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rgxtransfer_shader.h b/drivers/gpu/drm/img/img-volcanic/services/include/rgxtransfer_shader.h
new file mode 100644
index 000000000000..f261d2d04075
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rgxtransfer_shader.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File           rgxtransfer_shader.h
+@Title          TQ binary shader file info
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header holds info about TQ binary shader file generated
+                by the TQ shader factory. This header is need by shader factory
+                when generating the file; by services KM when reading and
+                loading the file into memory; and by services UM when
+                constructing blits using the shaders.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSHADERHEADER_H)
+#define RGXSHADERHEADER_H
+
+#include "pvrversion.h"
+
+typedef struct _RGX_SHADER_HEADER_
+{
+	IMG_UINT32 ui32Version;
+	IMG_UINT32 ui32NumFragment;
+	IMG_UINT32 ui32SizeFragment;
+	IMG_UINT32 ui32SizeClientMem;
+} RGX_SHADER_HEADER;
+
+/* TQ shaders version is used to check compatibility between the
+   binary TQ shaders file and the DDK. This number should be incremented
+   if a change to the TQ shader factory breaks compatibility. */
+#define RGX_TQ_SHADERS_VERSION 2U
+
+#define RGX_TQ_SHADERS_VERSION_PACK \
+	(((RGX_TQ_SHADERS_VERSION & 0xFFU) << 16) | ((PVRVERSION_MAJ & 0xFFU) << 8) | ((PVRVERSION_MIN & 0xFFU) << 0))
+
+#endif /* RGXSHADERHEADER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rogue/km_apphint_defs.h b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/km_apphint_defs.h
new file mode 100644
index 000000000000..f2ffad0030cd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/km_apphint_defs.h
@@ -0,0 +1,162 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "km_apphint_defs_common.h"
+
+#ifndef KM_APPHINT_DEFS_H
+#define KM_APPHINT_DEFS_H
+
+/* NB: The 'DEVICE' AppHints must be last in this list as they will be
+ * duplicated in the case of a driver supporting multiple devices
+ */
+#define APPHINT_LIST_ALL \
+	APPHINT_LIST_BUILDVAR_COMMON \
+	APPHINT_LIST_BUILDVAR \
+	APPHINT_LIST_MODPARAM_COMMON \
+	APPHINT_LIST_MODPARAM \
+	APPHINT_LIST_DEBUGINFO_COMMON \
+	APPHINT_LIST_DEBUGINFO \
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+	APPHINT_LIST_DEBUGINFO_DEVICE
+
+
+/*
+*******************************************************************************
+ Build variables (rogue-specific)
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR
+
+/*
+*******************************************************************************
+ Module parameters (rogue-specific)
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+X(EnableCDMKillingRandMode,         BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLECDMKILLINGRANDMODE,         NO_PARAM_TABLE,  ALWAYS   ) \
+\
+X(HWPerfDisableCustomCounterFilter, BOOL,           VALIDATION,  PVRSRV_APPHINT_HWPERFDISABLECUSTOMCOUNTERFILTER, NO_PARAM_TABLE,  ALWAYS   ) \
+X(ValidateSOCUSCTimer,              BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATESOCUSCTIMERS,             NO_PARAM_TABLE,  ALWAYS   ) \
+X(ECCRAMErrInj,                     UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+\
+X(TFBCCompressionControlGroup,      UINT32,         VALIDATION,  PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP,      NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlScheme,     UINT32,         VALIDATION,  PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME,     NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlYUVFormat,  BOOL,           VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlLossyMinChannel, BOOL,      VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCVersionDowngrade,             UINT32,         ALWAYS,      PVRSRV_APPHINT_TFBCVERSION,                      NO_PARAM_TABLE,  ALWAYS   ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters (rogue-specific) - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+
+/*
+*******************************************************************************
+ Debugfs parameters (rogue-specific) - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters. Each entry in this table will provide a default value to all
+ devices (i.e. if there is more than one device each device's value will
+ be initialised).
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT \
+/* debuginfo device apphint name  modparam name */
+
+/*
+*******************************************************************************
+
+ Table generated enums
+
+******************************************************************************/
+/* Unique ID for all AppHints */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_ID_ ## a,
+	APPHINT_LIST_ALL
+#undef X
+	APPHINT_ID_MAX
+} APPHINT_ID;
+
+/* ID for build variable Apphints - used for build variable only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_BUILDVAR_ID_ ## a,
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+	APPHINT_BUILDVAR_ID_MAX
+} APPHINT_BUILDVAR_ID;
+
+/* ID for Modparam Apphints - used for modparam only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_MODPARAM_ID_ ## a,
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+	APPHINT_MODPARAM_ID_MAX
+} APPHINT_MODPARAM_ID;
+
+/* ID for Debugfs Apphints - used for debugfs only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+	APPHINT_DEBUGINFO_ID_MAX
+} APPHINT_DEBUGINFO_ID;
+
+/* ID for Debugfs Device Apphints - used for debugfs device only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_DEVICE_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+	APPHINT_DEBUGINFO_DEVICE_ID_MAX
+} APPHINT_DEBUGINFO_DEVICE_ID;
+
+#endif /* KM_APPHINT_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/rogue/rgxapi_km.h b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/rgxapi_km.h
new file mode 100644
index 000000000000..7d7efd0afb5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/rogue/rgxapi_km.h
@@ -0,0 +1,322 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX API Header kernel mode
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported RGX API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXAPI_KM_H
+#define RGXAPI_KM_H
+
+#include "rgx_hwperf.h"
+
+/******************************************************************************
+ * RGX HW Performance Profiling Control API(s)
+ *****************************************************************************/
+
+/*! HWPerf device identification structure */
+typedef struct _RGX_HWPERF_DEVICE_
+{
+	IMG_CHAR pszName[20];	/*!< Helps identify this device uniquely */
+	IMG_HANDLE hDevData;	/*!< Handle for the server */
+
+	struct _RGX_HWPERF_DEVICE_ *psNext;	/*!< Next device if any */
+} RGX_HWPERF_DEVICE;
+
+/*! HWPerf connection structure */
+typedef struct
+{
+	RGX_HWPERF_DEVICE *psHWPerfDevList;	/*!< Pointer to list of devices */
+} RGX_HWPERF_CONNECTION;
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfLazyConnect
+@Description    Obtain a HWPerf connection object to the RGX device(s). The
+                 connections to devices are not actually opened until
+                 HWPerfOpen() is called.
+
+@Output         ppsHWPerfConnection Address of a HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfOpen
+@Description    Opens connection(s) to the RGX device(s). Valid handle to the
+                 connection object has to be provided which means the this
+                 function needs to be preceded by the call to
+                 RGXHWPerfLazyConnect() function.
+
+@Input          psHWPerfConnection HWPerf connection object
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION* psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConnect
+@Description    Obtain a connection object to the RGX HWPerf module. Allocated
+                 connection object(s) reference opened connection(s). Calling
+                 this function is an equivalent of calling RGXHWPerfLazyConnect
+                 and RGXHWPerfOpen. This connect should be used when the caller
+                 will be retrieving event data.
+
+@Output         ppsHWPerfConnection Address of HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfFreeConnection
+@Description    Frees the HWPerf connection object
+
+@Input          psHWPerfConnection Pointer to connection object as returned
+                                    from RGXHWPerfLazyConnect()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfClose
+@Description    Closes all the opened connection(s) to RGX device(s)
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object as
+                                    returned from RGXHWPerfConnect() or
+                                    RGXHWPerfOpen()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisconnect
+@Description    Disconnect from the RGX device
+
+@Input          ppsHWPerfConnection Pointer to HWPerf connection object as
+                                     returned from RGXHWPerfConnect() or
+                                     RGXHWPerfOpen(). Calling this function is
+                                     an equivalent of calling RGXHWPerfClose()
+                                     and RGXHWPerfFreeConnection().
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfControl
+@Description    Enable or disable the generation of RGX HWPerf event packets.
+                 See RGXCtrlHWPerf().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          eStreamId          ID of the HWPerf stream
+@Input          bToggle            Switch to toggle or apply mask.
+@Input          ui64Mask           Mask of events to control.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfGetFilter
+@Description    Reads HWPerf stream filter where stream is identified by the
+                 given stream ID.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ui64Filter   HWPerf filter value
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter
+);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigMuxCounters
+@Description    Enable and configure the performance counter block for one or
+                 more device layout modules.
+                 See RGXHWPerfConfigureAndEnableCustomCounters().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          asBlockConfigs     Address of the array of configuration blocks
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigMuxCounters(
+		RGX_HWPERF_CONNECTION         *psHWPerfConnection,
+		IMG_UINT32                     ui32NumBlocks,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK  *asBlockConfigs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigureAndEnableCustomCounters
+@Description    Enable and configure custom performance counters
+
+@Input          psHWPerfConnection    Pointer to HWPerf connection object
+@Input          ui16CustomBlockID     ID of the custom block to configure
+@Input          ui16NumCustomCounters Number of custom counters
+@Input          pui32CustomCounterIDs Pointer to array containing custom
+                                       counter IDs
+@Return         PVRSRV_ERROR          System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigureAndEnableCustomCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT16             ui16CustomBlockID,
+		IMG_UINT16             ui16NumCustomCounters,
+		IMG_UINT32            *pui32CustomCounterIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisableCounters
+@Description    Disable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from
+                                    the <tt>RGX_HWPERF_CNTBLK_ID</tt>
+                                    enumeration.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfEnableCounters
+@Description    Enable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from the
+                                    <tt>RGX_HWPERF_CNTBLK_ID</tt> enumeration.
+@Return         PVRSRV_ERROR  System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling Retrieval API(s)
+ *
+ * The client must ensure their use of this acquire/release API for a single
+ * connection/stream must not be shared with multiple execution contexts e.g.
+ * between a kernel thread and an ISR handler. It is the client's
+ * responsibility to ensure this API is not interrupted by a high priority
+ * thread/ISR
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfAcquireEvents
+@Description    When there is data available to read this call returns with OK
+                 and the address and length of the data buffer the client can
+                 safely read. This buffer may contain one or more event packets.
+                 When there is no data to read, this call returns with OK and
+                 sets *puiBufLen to 0 on exit.
+                 Clients must pair this call with a RGXHWPerfReleaseEvents()
+                 call.
+                 Data returned in ppBuf will be in the form of a sequence of
+                 HWPerf packets which should be traversed using the pointers,
+                 structures and macros provided by rgx_hwperf.h.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ppBuf        Address of a pointer to a byte buffer. On exit it
+                              contains the address of buffer to read from
+@Output         pui32BufLen  Pointer to an integer. On exit it is the size of
+                              the data to read from the buffer
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfReleaseEvents
+@Description    Called after client has read the event data out of the buffer
+                 retrieved from the Acquire Events call to release resources.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConvertCRTimeStamp
+@Description    Converts the timestamp given by FW events to the common OS
+                 timestamp. The first three inputs are obtained via a CLK_SYNC
+                 event, ui64CRTimeStamp is the CR timestamp from the FW event
+                 to be converted.
+
+@Input          ui32ClkSpeed        Clock speed given by sync event
+@Input          ui64CorrCRTimeStamp CR Timestamp given by sync event
+@Input          ui64CorrOSTimeStamp Correlating OS Timestamp given by sync
+                                     event
+@Input          ui64CRTimeStamp     CR Timestamp to convert
+@Return         IMG_UINT64          Calculated OS Timestamp
+*/ /**************************************************************************/
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp);
+
+#endif /* RGXAPI_KM_H */
+
+/******************************************************************************
+ End of file (rgxapi_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/sync_checkpoint_internal.h b/drivers/gpu/drm/img/img-volcanic/services/include/sync_checkpoint_internal.h
new file mode 100644
index 000000000000..2514ff3b6973
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/sync_checkpoint_internal.h
@@ -0,0 +1,273 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the internal server interface for services
+                synchronisation checkpoints.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_INTERNAL_H
+#define SYNC_CHECKPOINT_INTERNAL_H
+
+#include "img_types.h"
+#include "opaque_types.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "ra.h"
+#include "dllist.h"
+#include "lock.h"
+#include "devicemem.h"
+#include "rgx_fwif_shared.h"
+
+struct SYNC_CHECKPOINT_RECORD;
+
+/*
+	Private structures
+*/
+
+typedef struct _SYNC_CHECKPOINT_CONTEXT_CTL_ _SYNC_CHECKPOINT_CONTEXT_CTL, *_PSYNC_CHECKPOINT_CONTEXT_CTL;
+
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG _SYNC_CHECKPOINT_CONTEXT;
+
+typedef struct _SYNC_CHECKPOINT_BLOCK_
+{
+	ATOMIC_T                  hRefCount;                  /*!< Ref count for this sync block */
+	POS_LOCK                  hLock;
+	_SYNC_CHECKPOINT_CONTEXT  *psContext;                 /*!< Our copy of the services connection */
+	IMG_UINT32                ui32SyncBlockSize;          /*!< Size of the sync checkpoint block */
+	IMG_UINT32                ui32FirmwareAddr;           /*!< Firmware address */
+	DEVMEM_MEMDESC            *hMemDesc;                  /*!< DevMem allocation for block */
+	volatile IMG_UINT32       *pui32LinAddr;              /*!< Server-code CPU mapping */
+	RA_BASE_T                 uiSpanBase;                 /*!< Base of this import (FW DevMem) in the span RA */
+#if defined(PDUMP)
+	DLLIST_NODE               sListNode;                  /*!< List node for the sync chkpt blocks */
+#endif
+} SYNC_CHECKPOINT_BLOCK;
+
+typedef struct SYNC_CHECKPOINT_RECORD* PSYNC_CHECKPOINT_RECORD_HANDLE;
+
+typedef struct SYNC_CHECKPOINT_TAG
+{
+	/* A sync checkpoint is assigned a unique ID, to avoid any confusion should
+	 * the same memory be re-used later for a different checkpoint
+	 */
+	IMG_UINT32                      ui32UID;                /*!< Unique ID assigned to sync checkpoint (to distinguish checkpoints if memory is re-used)*/
+	ATOMIC_T                        hRefCount;              /*!< Ref count for this sync */
+	ATOMIC_T                        hEnqueuedCCBCount;      /*!< Num times sync has been put in CCBs */
+	SYNC_CHECKPOINT_BLOCK           *psSyncCheckpointBlock; /*!< Synchronisation block this checkpoint is allocated on */
+	RA_BASE_T                       uiAllocatedAddr;        /*!< Allocated address of the sync */
+	volatile SYNC_CHECKPOINT_FW_OBJ *psSyncCheckpointFwObj; /*!< CPU view of the data held in the sync block */
+	PRGXFWIF_UFO_ADDR               sCheckpointUFOAddr;     /*!< PRGXFWIF_UFO_ADDR struct used to pass update address to FW */
+	IMG_CHAR                        azName[PVRSRV_SYNC_NAME_LENGTH]; /*!< Name of the checkpoint */
+	PVRSRV_TIMELINE                 hTimeline;              /*!< Timeline on which this sync checkpoint was created */
+	IMG_PID                         uiProcess;              /*!< The Process ID of the process which created this sync checkpoint */
+	PSYNC_CHECKPOINT_RECORD_HANDLE  hRecord;                /*!< Sync record handle */
+	DLLIST_NODE                     sListNode;              /*!< List node for the global sync chkpt list */
+	DLLIST_NODE                     sDeferredFreeListNode;  /*!< List node for the deferred free sync chkpt list */
+	IMG_UINT32                      ui32FWAddr;             /*!< FWAddr stored at sync checkpoint alloc time */
+#if defined(PDUMP)
+	PDUMP_FLAGS_T                   ui32PDumpFlags;         /*!< Pdump Capture mode to be used for POL*/
+#endif
+#if defined(DEBUG)
+    IMG_UINT32                      ui32ValidationCheck;    /*!< Structure validity pattern */
+#endif
+} SYNC_CHECKPOINT;
+
+
+typedef struct _SYNC_CHECKPOINT_SIGNAL_
+{
+	SYNC_CHECKPOINT                asSyncCheckpoint;       /*!< Store sync checkpt for deferred signal */
+	IMG_UINT32                      ui32Status;             /*!< sync checkpt status signal/errored */
+} _SYNC_CHECKPOINT_DEFERRED_SIGNAL;
+
+#define GET_CP_CB_NEXT_IDX(_curridx) (((_curridx) + 1) % SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL)
+#define GET_CP_CB_BASE(_idx)   (IMG_OFFSET_ADDR(psDevNode->pui8DeferredSyncCPSignal, \
+                                                ((_idx) * sizeof(_SYNC_CHECKPOINT_DEFERRED_SIGNAL))))
+
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetFirmwareAddr
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the firmware address of
+
+@Return         The firmware address of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCCBEnqueued
+
+@Description    Increment the CCB enqueued reference count for a
+                synchronisation checkpoint. This indicates how many FW
+                operations (checks/update) have been placed into CCBs for the
+                sync checkpoint.
+                When the FW services these operation, it increments its own
+                reference count. When these two values are equal, we know
+                there are not outstanding FW operating for the checkpoint
+                in any CCB.
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint for which
+                                        to increment the enqueued reference
+                                        count
+
+@Return         None
+
+*/
+/*****************************************************************************/
+void
+SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetEnqueuedCount
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the enqueued count of
+
+@Return         The enqueued count of the sync checkpoint
+                (i.e. the number of FW operations (checks or updates)
+                 currently enqueued in CCBs for the sync checkpoint)
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetReferenceCount
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the reference count of
+
+@Return         The host reference count of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetCreator
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the creating process of
+
+@Return         The process id of the process which created this sync checkpoint.
+
+*/
+/*****************************************************************************/
+IMG_PID
+SyncCheckpointGetCreator(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetId
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the unique Id of
+
+@Return         The unique Id of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetTimeline
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the parent timeline of
+
+@Return         The parent timeline of the sync checkpoint
+
+*/
+/*****************************************************************************/
+PVRSRV_TIMELINE
+SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetRGXFWIFUFOAddr
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the PRGXFWIF_UFO_ADDR of
+
+@Return         The PRGXFWIF_UFO_ADDR of the sync checkpoint, used when
+                providing the update in server kick code.
+
+*/
+/*****************************************************************************/
+PRGXFWIF_UFO_ADDR*
+SyncCheckpointGetRGXFWIFUFOAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+#if !defined(SUPPORT_NATIVE_FENCE_SYNC)
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetAssociatedDevice
+
+@Description    .
+
+@Input          psSyncCheckpointContext Synchronisation Checkpoint context
+                                        to get the device node of
+
+@Return         The PVRSRV_DEVICE_NODE of the device on which the sync
+                checkpoint context was created.
+
+*/
+/*****************************************************************************/
+PPVRSRV_DEVICE_NODE
+SyncCheckpointGetAssociatedDevice(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+#endif /* !defined(SUPPORT_NATIVE_FENCE_SYNC) */
+
+#endif /* SYNC_CHECKPOINT_INTERNAL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/km_apphint_defs.h b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/km_apphint_defs.h
new file mode 100644
index 000000000000..81397088bddb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/km_apphint_defs.h
@@ -0,0 +1,187 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "km_apphint_defs_common.h"
+
+#ifndef KM_APPHINT_DEFS_H
+#define KM_APPHINT_DEFS_H
+
+/* NB: The 'DEVICE' AppHints must be last in this list as they will be
+ * duplicated in the case of a driver supporting multiple devices
+ */
+#define APPHINT_LIST_ALL \
+	APPHINT_LIST_BUILDVAR_COMMON \
+	APPHINT_LIST_BUILDVAR \
+	APPHINT_LIST_MODPARAM_COMMON \
+	APPHINT_LIST_MODPARAM \
+	APPHINT_LIST_DEBUGINFO_COMMON \
+	APPHINT_LIST_DEBUGINFO \
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+	APPHINT_LIST_DEBUGINFO_DEVICE
+
+
+/*
+*******************************************************************************
+ Build variables (volcanic-specific)
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR
+
+/*
+*******************************************************************************
+ Module parameters (volcanic-specific)
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM \
+/* name,                            type,           class,       default,                                         helper,          guest,         */ \
+X(FabricCoherencyOverride,          UINT32,         ALWAYS,      PVRSRV_APPHINT_FABRICCOHERENCYOVERRIDE,          NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(EnableGenericDMKillingRandMode,   BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLEDMKILLINGRANDMODE,          NO_PARAM_TABLE,  ALWAYS          ) \
+X(KillingCtl,                       UINT32,         VALIDATION,  PVRSRV_APPHINT_KILLINGCTL,                       NO_PARAM_TABLE,  ALWAYS          ) \
+X(CDMTDMKillingCtl,                 UINT32,         VALIDATION,  PVRSRV_APPHINT_CDMTDM_KILLINGCTL,                NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValEnableSPUPowerMaskChange,    BOOL,           VALIDATION,  PVRSRV_APPHINT_HWVALENABLESPUPOWERMASKCHANGE,    NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValAvailableSPUMask,            UINT32,         VALIDATION,  PVRSRV_APPHINT_HWVALAVAILABLESPUMASK,            NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValAvailableRACMask,            UINT32,         VALIDATION,  PVRSRV_APPHINT_HWVALAVAILABLERACMASK,            NO_PARAM_TABLE,  ALWAYS          ) \
+X(EnableSPUClockGating,             BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLESPUCLOCKGATING,             NO_PARAM_TABLE,  ALWAYS          ) \
+X(HWValTpuF20BilPrecision,          UINT32,         VALIDATION,  PVRSRV_APPHINT_HWVALTPUF20BILPRECISION,          NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(HWPerfDisableCounterFilter,       BOOL,           VALIDATION,  PVRSRV_APPHINT_HWPERFDISABLECOUNTERFILTER,       NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(ISPSchedulingLatencyMode,         UINT32,         ALWAYS,      PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE,         NO_PARAM_TABLE,  ALWAYS          ) \
+X(ValidateSOCUSCTimer,              BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATESOCUSCTIMERS,             NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(USRMNumRegionsVDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsCDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsDDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsPDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(USRMNumRegionsTDM,                UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(UVBRMNumRegionsVDM,               UINT64,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(UVBRMNumRegionsDDM,               UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(GPUStatePin,                      BOOL,           VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+X(PowerDomainKickInterval,          UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(RCEDisableMask,                   UINT64,         VALIDATION,  PVRSRV_APPHINT_RCEDISABLEMASK,                   NO_PARAM_TABLE,  ALWAYS          ) \
+X(PCGPktDropThresh,                 UINT32,         VALIDATION,  PVRSRV_APPHINT_PCGPKTDROPTHRESH,                 NO_PARAM_TABLE,  ALWAYS          ) \
+X(RaySLCMMUAutoCacheOps,            UINT32,         VALIDATION,  PVRSRV_APPHINT_RAYSLCMMUAUTOCACHEOPS,            NO_PARAM_TABLE,  ALWAYS          ) \
+\
+X(ClkCtrl0,                         UINT64,         VALIDATION,  PVRSRV_APPHINT_CLKCTRL0,                         NO_PARAM_TABLE,  ALWAYS          ) \
+X(ClkCtrl1,                         UINT64,         VALIDATION,  PVRSRV_APPHINT_CLKCTRL1,                         NO_PARAM_TABLE,  ALWAYS          ) \
+X(ClkCtrl2,                         UINT32,         VALIDATION,  PVRSRV_APPHINT_CLKCTRL2,                         NO_PARAM_TABLE,  ALWAYS          ) \
+X(TPUTAGCtrl,                       UINT32,         VALIDATION,  0x10,                                            NO_PARAM_TABLE,  ALWAYS          ) \
+X(TPUMADDCtrl,                      UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS          ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters (volcanic-specific) - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO \
+/* name,                            type,           class,       default,                                         helper,          guest,         */ \
+X(MMUVal_ParityFlipPMR,             STRING,         VALIDATION,  PVRSRV_APPHINT_MMUVAL_PARITYFLIPPMR,             NO_PARAM_TABLE,  ALWAYS          ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters (volcanic-specific) - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE \
+/* name,                            type,           class,       default,                                         helper,          guest,         */ \
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters.
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT \
+/* debuginfo device apphint name  modparam name */
+
+/*
+*******************************************************************************
+
+ Table generated enums
+
+******************************************************************************/
+/* Unique ID for all AppHints */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_ID_ ## a,
+	APPHINT_LIST_ALL
+#undef X
+	APPHINT_ID_MAX
+} APPHINT_ID;
+
+/* ID for build variable Apphints - used for build variable only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_BUILDVAR_ID_ ## a,
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+	APPHINT_BUILDVAR_ID_MAX
+} APPHINT_BUILDVAR_ID;
+
+/* ID for Modparam Apphints - used for modparam only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_MODPARAM_ID_ ## a,
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+	APPHINT_MODPARAM_ID_MAX
+} APPHINT_MODPARAM_ID;
+
+/* ID for Debugfs Apphints - used for debugfs only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+	APPHINT_DEBUGINFO_ID_MAX
+} APPHINT_DEBUGINFO_ID;
+
+/* ID for Debugfs Device Apphints - used for debugfs device only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_DEVICE_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+	APPHINT_DEBUGINFO_DEVICE_ID_MAX
+} APPHINT_DEBUGINFO_DEVICE_ID;
+
+#endif /* KM_APPHINT_DEFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/rgxapi_km.h b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/rgxapi_km.h
new file mode 100644
index 000000000000..487b3796a6a9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/include/volcanic/rgxapi_km.h
@@ -0,0 +1,309 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX API Header kernel mode
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported RGX API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXAPI_KM_H
+#define RGXAPI_KM_H
+
+#include "rgx_hwperf.h"
+
+
+/******************************************************************************
+ * RGX HW Performance Profiling Control API(s)
+ *****************************************************************************/
+
+/*! HWPerf device identification structure */
+typedef struct _RGX_HWPERF_DEVICE_
+{
+	IMG_CHAR pszName[20];	/*!< Helps identify this device uniquely */
+	IMG_HANDLE hDevData;	/*!< Handle for the server */
+
+	struct _RGX_HWPERF_DEVICE_ *psNext;  /*!< Next device if any */
+} RGX_HWPERF_DEVICE;
+
+/*! HWPerf connection structure */
+typedef struct
+{
+	RGX_HWPERF_DEVICE *psHWPerfDevList;  /*!< pointer to list of devices */
+} RGX_HWPERF_CONNECTION;
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfLazyConnect
+@Description    Obtain a HWPerf connection object to the RGX device(s). The
+                connections to devices are not actually opened until
+                HWPerfOpen() is called.
+
+@Output         ppsHWPerfConnection Address of a HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfOpen
+@Description    Opens connection(s) to the RGX device(s). Valid handle to the
+                connection object has to be provided which means the this
+                function needs to be preceded by the call to
+                RGXHWPerfLazyConnect() function.
+
+@Input          psHWPerfConnection HWPerf connection object
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION* psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConnect
+@Description    Obtain a connection object to the RGX HWPerf module. Allocated
+                connection object(s) reference opened connection(s). Calling
+                this function is an equivalent of calling RGXHWPerfLazyConnect
+                and RGXHWPerfOpen. This connect should be used when the caller
+                will be retrieving event data.
+
+@Output         ppsHWPerfConnection Address of HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfFreeConnection
+@Description    Frees the HWPerf connection object
+
+@Input          psHWPerfConnection Pointer to connection object as returned
+                                    from RGXHWPerfLazyConnect()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfClose
+@Description    Closes all the opened connection(s) to RGX device(s)
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object as
+                                    returned from RGXHWPerfConnect() or
+                                    RGXHWPerfOpen()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisconnect
+@Description    Disconnect from the RGX device
+
+@Input          ppsHWPerfConnection Pointer to HWPerf connection object as
+                                     returned from RGXHWPerfConnect() or
+                                     RGXHWPerfOpen(). Calling this function is
+                                     an equivalent of calling RGXHWPerfClose()
+                                     and RGXHWPerfFreeConnection().
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfControl
+@Description    Enable or disable the generation of RGX HWPerf event packets.
+                 See RGXCtrlHWPerf().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          eStreamId          ID of the HWPerf stream
+@Input          bToggle            Switch to toggle or apply mask.
+@Input          ui64Mask           Mask of events to control.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfGetFilter
+@Description    Reads HWPerf stream filter where stream is identified by the
+                 given stream ID.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ui64Filter   HWPerf filter value
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter
+);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigureCounters
+@Description    Enable and configure the performance counter block for one or
+                 more device layout modules.
+                 See RGXConfigHWPerfCounters().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32CtrlWord       One of <tt>RGX_HWPERF_CTRL_NOP</tt>,
+                                    <tt>RGX_HWPERF_CTRL_GEOM_FULLrange</tt>,
+                                    <tt>RGX_HWPERF_CTRL_COMP_FULLRANGE</tt>,
+                                    <tt>RGX_HWPERF_CTRL_TDM_FULLRANGE</tt>
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          asBlockConfigs     Address of the array of configuration blocks
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigureCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32                 ui32CtrlWord,
+		IMG_UINT32                 ui32NumBlocks,
+		RGX_HWPERF_CONFIG_CNTBLK*  asBlockConfigs);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisableCounters
+@Description    Disable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from
+                                    the RGX_HWPERF_CNTBLK_ID enumeration.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfEnableCounters
+@Description    Enable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from the
+                                    <tt>RGX_HWPERF_CNTBLK_ID</tt> enumeration.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling Retrieval API(s)
+ *
+ * The client must ensure their use of this acquire/release API for a single
+ * connection/stream must not be shared with multiple execution contexts e.g.
+ * between a kernel thread and an ISR handler. It is the client's
+ * responsibility to ensure this API is not interrupted by a high priority
+ * thread/ISR
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfAcquireEvents
+@Description    When there is data available to read this call returns with OK
+                 and the address and length of the data buffer the client can
+                 safely read. This buffer may contain one or more event packets.
+                 When there is no data to read, this call returns with OK and
+                 sets *puiBufLen to 0 on exit.
+                 Clients must pair this call with a RGXHWPerfReleaseEvents()
+                 call.
+                 Data returned in ppBuf will be in the form of a sequence of
+                 HWPerf packets which should be traversed using the pointers,
+                 structures and macros provided in rgx_hwperf.h
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ppBuf        Address of a pointer to a byte buffer. On exit it
+                              contains the address of buffer to read from
+@Output         pui32BufLen  Pointer to an integer. On exit it is the size of
+                              the data to read from the buffer
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfReleaseEvents
+@Description    Called after client has read the event data out of the buffer
+                 retrieved from the Acquire Events call to release resources.
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConvertCRTimeStamp
+@Description    Converts the timestamp given by FW events to the common OS
+                 timestamp. The first three inputs are obtained via a CLK_SYNC
+                 event, ui64CRTimeStamp is the CR timestamp from the FW event
+                 to be converted.
+@Input          ui32ClkSpeed        Clock speed given by sync event
+@Input          ui64CorrCRTimeStamp CR Timestamp given by sync event
+@Input          ui64CorrOSTimeStamp Correlating OS Timestamp given by sync
+                                     event
+@Input          ui64CRTimeStamp     CR Timestamp to convert
+@Return         IMG_UINT64          Calculated OS Timestamp
+*/ /**************************************************************************/
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp);
+
+#endif /* RGXAPI_KM_H */
+
+/******************************************************************************
+ End of file (rgxapi_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/cache_km.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/cache_km.c
new file mode 100644
index 000000000000..f9c4ce986e18
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/cache_km.c
@@ -0,0 +1,1646 @@
+/*************************************************************************/ /*!
+@File           cache_km.c
+@Title          CPU d-cache maintenance operations framework
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements server side code for CPU d-cache maintenance taking
+                into account the idiosyncrasies of the various types of CPU
+                d-cache instruction-set architecture (ISA) maintenance
+                mechanisms.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/version.h>
+#include <linux/uaccess.h>
+#include <asm/current.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#endif
+
+#include "pmr.h"
+#include "log2.h"
+#include "device.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "cache_km.h"
+#include "pvr_debug.h"
+#include "lock_types.h"
+#include "allocmem.h"
+#include "process_stats.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+#include "ri_server.h"
+#endif
+#include "devicemem.h"
+#include "pvrsrv_apphint.h"
+#include "pvrsrv_sync_server.h"
+#include "km_apphint_defs.h"
+#include "km_apphint_defs_common.h"
+#include "os_apphint.h"
+#include "di_server.h"
+
+/* This header must always be included last */
+#if defined(__linux__)
+#include "kernel_compatibility.h"
+#endif
+
+/* Top-level file-local build definitions */
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS) && defined(__linux__)
+#define CACHEOP_DEBUG
+#define CACHEOP_STATS_ITEMS_MAX				32
+#define INCR_WRAP(x)						((x+1) >= CACHEOP_STATS_ITEMS_MAX ? 0 : (x+1))
+#define DECR_WRAP(x)						((x-1) < 0 ? (CACHEOP_STATS_ITEMS_MAX-1) : (x-1))
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+/* Refer to CacheOpStatsExecLogHeader() for header item names */
+#define CACHEOP_RI_PRINTF_HEADER			"%-8s %-8s %-10s %-10s %-5s %-16s %-16s %-10s %-10s %-18s"
+#define CACHEOP_RI_PRINTF					"%-8d %-8d %-10s %-10s %-5s 0x%-14llx 0x%-14llx 0x%-8llx 0x%-8llx %-18llu\n"
+#else
+#define CACHEOP_PRINTF_HEADER				"%-8s %-8s %-10s %-10s %-5s %-10s %-10s %-18s"
+#define CACHEOP_PRINTF						"%-8d %-8d %-10s %-10s %-5s 0x%-8llx 0x%-8llx %-18llu\n"
+#endif
+#endif
+
+//#define CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING		/* Force OS page (not cache line) flush granularity */
+#define CACHEOP_PVR_ASSERT(x)							/* Define as PVR_ASSERT(x), enable for swdev & testing */
+#define CACHEOP_DEVMEM_OOR_ERROR_STRING		"cacheop device memory request is out of range"
+#define CACHEOP_MAX_DEBUG_MESSAGE_LEN		160
+
+typedef struct _CACHEOP_WORK_ITEM_
+{
+	PMR *psPMR;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_CACHE_OP uiCacheOp;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	PVRSRV_TIMELINE iTimeline;
+	SYNC_TIMELINE_OBJ sSWTimelineObj;
+	PVRSRV_DEVICE_NODE *psDevNode;
+#if defined(CACHEOP_DEBUG)
+	IMG_UINT64 ui64StartTime;
+	IMG_UINT64 ui64EndTime;
+	IMG_BOOL bKMReq;
+	IMG_PID pid;
+#endif
+} CACHEOP_WORK_ITEM;
+
+typedef struct _CACHEOP_STATS_EXEC_ITEM_
+{
+	IMG_UINT32 ui32DeviceID;
+	IMG_PID pid;
+	PVRSRV_CACHE_OP uiCacheOp;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT64 ui64StartTime;
+	IMG_UINT64 ui64EndTime;
+	IMG_BOOL bKMReq;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+#endif
+} CACHEOP_STATS_EXEC_ITEM;
+
+typedef enum _CACHEOP_CONFIG_
+{
+	CACHEOP_CONFIG_DEFAULT = 0,
+	/* cache flush mechanism types */
+	CACHEOP_CONFIG_URBF    = 4,
+	/* sw-emulated deferred flush mechanism */
+	CACHEOP_CONFIG_KDF     = 8,
+	/* pseudo configuration items */
+	CACHEOP_CONFIG_LAST    = 16,
+	CACHEOP_CONFIG_KLOG    = 16,
+	CACHEOP_CONFIG_ALL     = 31
+} CACHEOP_CONFIG;
+
+typedef struct _CACHEOP_WORK_QUEUE_
+{
+/*
+ * Init. state & primary device node framework
+ * is anchored on.
+ */
+	IMG_BOOL bInit;
+/*
+  MMU page size/shift & d-cache line size
+ */
+	size_t uiPageSize;
+	IMG_UINT32 uiLineSize;
+	IMG_UINT32 uiLineShift;
+	IMG_UINT32 uiPageShift;
+	PMR *psInfoPagePMR;
+	IMG_UINT32 *pui32InfoPage;
+
+#if defined(CACHEOP_DEBUG)
+/*
+  CacheOp statistics
+ */
+	DI_ENTRY *psDIEntry;
+	IMG_HANDLE hStatsExecLock;
+
+	IMG_UINT32 ui32ServerOps;
+	IMG_UINT32 ui32ClientOps;
+	IMG_UINT32 ui32TotalOps;
+	IMG_UINT32 ui32ServerOpUsedUMVA;
+	IMG_UINT32 ui32AvgExecTime;
+	IMG_UINT32 ui32AvgExecTimeRemainder;
+
+	IMG_INT32 i32StatsExecWriteIdx;
+	CACHEOP_STATS_EXEC_ITEM asStatsExecuted[CACHEOP_STATS_ITEMS_MAX];
+#endif
+
+	DI_ENTRY *psConfigTune;
+	IMG_HANDLE hConfigLock;
+	CACHEOP_CONFIG	eConfig;
+	IMG_UINT32		ui32Config;
+	IMG_BOOL		bSupportsUMFlush;
+} CACHEOP_WORK_QUEUE;
+
+/* Top-level CacheOp framework object */
+static CACHEOP_WORK_QUEUE gsCwq;
+
+#define CacheOpConfigSupports(e) ((gsCwq.eConfig & (e)) ? IMG_TRUE : IMG_FALSE)
+
+#if defined(CACHEOP_DEBUG)
+static INLINE void CacheOpStatsExecLogHeader(IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(szBuffer, CACHEOP_MAX_DEBUG_MESSAGE_LEN,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+				CACHEOP_RI_PRINTF_HEADER,
+#else
+				CACHEOP_PRINTF_HEADER,
+#endif
+				"DevID",
+				"Pid",
+				"CacheOp",
+				"Type",
+				"Origin",
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+				"DevVAddr",
+				"DevPAddr",
+#endif
+				"Offset",
+				"Size",
+				"xTime (us)");
+}
+
+static void CacheOpStatsExecLogWrite(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
+{
+	IMG_INT32 i32WriteOffset;
+	IMG_UINT32 ui32ExecTime;
+	printk("log write\n");
+	if (!psCacheOpWorkItem->uiCacheOp)
+	{
+		return;
+	}
+	else if (psCacheOpWorkItem->bKMReq && !CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
+	{
+		/* KM logs spams the history due to frequency, this removes it completely */
+		return;
+	}
+
+	OSLockAcquire(gsCwq.hStatsExecLock);
+
+	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
+	gsCwq.i32StatsExecWriteIdx = INCR_WRAP(gsCwq.i32StatsExecWriteIdx);
+	gsCwq.asStatsExecuted[i32WriteOffset].ui32DeviceID = psCacheOpWorkItem->psDevNode ? psCacheOpWorkItem->psDevNode->sDevId.ui32InternalID : -1;
+	gsCwq.asStatsExecuted[i32WriteOffset].pid = psCacheOpWorkItem->pid;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiSize = psCacheOpWorkItem->uiSize;
+	gsCwq.asStatsExecuted[i32WriteOffset].bKMReq = psCacheOpWorkItem->bKMReq;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiOffset	= psCacheOpWorkItem->uiOffset;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp = psCacheOpWorkItem->uiCacheOp;
+	gsCwq.asStatsExecuted[i32WriteOffset].ui64StartTime = psCacheOpWorkItem->ui64StartTime;
+	gsCwq.asStatsExecuted[i32WriteOffset].ui64EndTime = psCacheOpWorkItem->ui64EndTime;
+
+	CACHEOP_PVR_ASSERT(gsCwq.asStatsExecuted[i32WriteOffset].pid);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	if (gsCwq.bInit && psCacheOpWorkItem->psPMR)
+	{
+		IMG_CPU_PHYADDR sDevPAddr;
+		PVRSRV_ERROR eError, eLockError;
+		IMG_BOOL bValid;
+
+		/* Get more detailed information regarding the sub allocations that
+		   PMR has from RI manager for process that requested the CacheOp */
+		eError = RIDumpProcessListKM(psCacheOpWorkItem->psPMR,
+									 gsCwq.asStatsExecuted[i32WriteOffset].pid,
+									 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+									 &gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* (Re)lock here as some PMR might have not been locked */
+		eLockError = PMRLockSysPhysAddresses(psCacheOpWorkItem->psPMR);
+		PVR_GOTO_IF_ERROR(eLockError, e0);
+
+		eError = PMR_CpuPhysAddr(psCacheOpWorkItem->psPMR,
+								 gsCwq.uiPageShift,
+								 1,
+								 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+								 &sDevPAddr,
+								 &bValid,
+								 CPU_USE);
+
+		eLockError = PMRUnlockSysPhysAddresses(psCacheOpWorkItem->psPMR);
+		PVR_LOG_IF_ERROR(eLockError, "PMRUnlockSysPhysAddresses");
+
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+
+
+		gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr.uiAddr = sDevPAddr.uiAddr;
+	}
+#endif
+
+	/* Calculate the approximate cumulative moving average execution time.
+	 * This calculation is based on standard equation:
+	 *
+	 * CMAnext = (new + count * CMAprev) / (count + 1)
+	 *
+	 * but in simplified form:
+	 *
+	 * CMAnext = CMAprev + (new - CMAprev) / (count + 1)
+	 *
+	 * this gets rid of multiplication and prevents overflow.
+	 *
+	 * Also to increase accuracy that we lose with integer division,
+	 * we hold the moving remainder of the division and add it.
+	 *
+	 * CMAnext = CMAprev + (new - CMAprev + CMRprev) / (count + 1)
+	 *
+	 * Multiple tests proved it to be the best solution for approximating
+	 * CMA using integers.
+	 *
+	 */
+
+	ui32ExecTime =
+		gsCwq.asStatsExecuted[i32WriteOffset].ui64EndTime -
+		gsCwq.asStatsExecuted[i32WriteOffset].ui64StartTime;
+
+	{
+
+	IMG_INT32 i32Div =
+		(IMG_INT32) ui32ExecTime -
+		(IMG_INT32) gsCwq.ui32AvgExecTime +
+		(IMG_INT32) gsCwq.ui32AvgExecTimeRemainder;
+
+	gsCwq.ui32AvgExecTime += i32Div / (IMG_INT32)(gsCwq.ui32TotalOps + 1);
+	gsCwq.ui32AvgExecTimeRemainder = i32Div % (IMG_INT32)(gsCwq.ui32TotalOps + 1);
+
+	gsCwq.ui32TotalOps++;
+
+	}
+
+	if (!gsCwq.asStatsExecuted[i32WriteOffset].bKMReq)
+	{
+		/* This operation queues only UM CacheOp in per-PID process statistics database */
+		PVRSRVStatsUpdateCacheOpStats(
+						gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr,
+						gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr,
+#endif
+						gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+						gsCwq.asStatsExecuted[i32WriteOffset].uiSize,
+						ui32ExecTime,
+						!gsCwq.asStatsExecuted[i32WriteOffset].bKMReq,
+						psCacheOpWorkItem->pid);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+e0:
+#endif
+	OSLockRelease(gsCwq.hStatsExecLock);
+}
+
+static int CacheOpStatsExecLogRead(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	IMG_CHAR *pszFlushType;
+	IMG_CHAR *pszCacheOpType;
+	IMG_CHAR *pszFlushSource;
+	IMG_INT32 i32ReadOffset;
+	IMG_INT32 i32WriteOffset;
+
+	IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN] = {0};
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	OSLockAcquire(gsCwq.hStatsExecLock);
+
+	DIPrintf(psEntry,
+			"Primary CPU d-cache architecture: LSZ: 0x%x, URBF: %s\n",
+			gsCwq.uiLineSize,
+			gsCwq.bSupportsUMFlush ? "Yes" : "No");
+
+	DIPrintf(psEntry,
+			"Configuration: UKT: %d, URBF: %s\n",
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD],
+			gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No");
+
+	DIPrintf(psEntry,
+			"Summary: Total Ops [%d] - Server(using UMVA)/Client [%d(%d)/%d]. Avg execution time [%d]\n",
+			gsCwq.ui32TotalOps, gsCwq.ui32ServerOps, gsCwq.ui32ServerOpUsedUMVA, gsCwq.ui32ClientOps, gsCwq.ui32AvgExecTime);
+
+
+	CacheOpStatsExecLogHeader(szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
+	for (i32ReadOffset = DECR_WRAP(i32WriteOffset);
+		 i32ReadOffset != i32WriteOffset;
+		 i32ReadOffset = DECR_WRAP(i32ReadOffset))
+	{
+		IMG_UINT64 ui64ExecTime =
+			gsCwq.asStatsExecuted[i32ReadOffset].ui64EndTime -
+			gsCwq.asStatsExecuted[i32ReadOffset].ui64StartTime;
+
+		IMG_DEVMEM_SIZE_T ui64NumOfPages =
+			gsCwq.asStatsExecuted[i32ReadOffset].uiSize >> gsCwq.uiPageShift;
+
+
+		if (!gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
+		{
+			break;
+		}
+		if (ui64NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			pszFlushType = "RBF.Fast";
+		}
+		else
+		{
+			pszFlushType = "RBF.Slow";
+		}
+
+		pszFlushSource = gsCwq.asStatsExecuted[i32ReadOffset].bKMReq ? " KM" : " UM";
+
+		switch (gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
+		{
+			case PVRSRV_CACHE_OP_NONE:
+				pszCacheOpType = "None";
+				break;
+			case PVRSRV_CACHE_OP_CLEAN:
+				pszCacheOpType = "Clean";
+				break;
+			case PVRSRV_CACHE_OP_INVALIDATE:
+				pszCacheOpType = "Invalidate";
+				break;
+			case PVRSRV_CACHE_OP_FLUSH:
+				pszCacheOpType = "Flush";
+				break;
+			case PVRSRV_CACHE_OP_TIMELINE:
+				pszCacheOpType = "Timeline";
+				pszFlushType = "      ";
+				break;
+			default:
+				pszCacheOpType = "Unknown";
+				break;
+		}
+
+		DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						CACHEOP_RI_PRINTF,
+#else
+						CACHEOP_PRINTF,
+#endif
+						gsCwq.asStatsExecuted[i32ReadOffset].ui32DeviceID,
+						gsCwq.asStatsExecuted[i32ReadOffset].pid,
+						pszCacheOpType,
+						pszFlushType,
+						pszFlushSource,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						gsCwq.asStatsExecuted[i32ReadOffset].sDevVAddr.uiAddr,
+						gsCwq.asStatsExecuted[i32ReadOffset].sDevPAddr.uiAddr,
+#endif
+						gsCwq.asStatsExecuted[i32ReadOffset].uiOffset,
+						gsCwq.asStatsExecuted[i32ReadOffset].uiSize,
+						ui64ExecTime);
+
+	}
+
+	OSLockRelease(gsCwq.hStatsExecLock);
+
+	return 0;
+}
+#endif /* defined(CACHEOP_DEBUG) */
+
+static INLINE void CacheOpStatsReset(void)
+{
+#if defined(CACHEOP_DEBUG)
+	gsCwq.ui32ServerOps = 0;
+	gsCwq.ui32ClientOps = 0;
+	gsCwq.ui32TotalOps = 0;
+	gsCwq.ui32ServerOpUsedUMVA = 0;
+	gsCwq.ui32AvgExecTime = 0;
+	gsCwq.ui32AvgExecTimeRemainder = 0;
+
+	gsCwq.i32StatsExecWriteIdx = 0;
+
+	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));
+#endif
+}
+
+static void CacheOpConfigUpdate(IMG_UINT32 ui32Config)
+{
+	OSLockAcquire(gsCwq.hConfigLock);
+
+	/* Step 0, set the gsCwq.eConfig bits */
+	if (!(ui32Config & (CACHEOP_CONFIG_LAST - 1)))
+	{
+		gsCwq.eConfig = CACHEOP_CONFIG_KDF;
+		if (gsCwq.bSupportsUMFlush)
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
+		}
+	}
+	else
+	{
+		if (ui32Config & CACHEOP_CONFIG_KDF)
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_KDF;
+		}
+		else
+		{
+			gsCwq.eConfig &= ~CACHEOP_CONFIG_KDF;
+		}
+
+		if (gsCwq.bSupportsUMFlush && (ui32Config & CACHEOP_CONFIG_URBF))
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
+		}
+		else
+		{
+			gsCwq.eConfig &= ~CACHEOP_CONFIG_URBF;
+		}
+	}
+
+	if (ui32Config & CACHEOP_CONFIG_KLOG)
+	{
+		/* Suppress logs from KM caller */
+		gsCwq.eConfig |= CACHEOP_CONFIG_KLOG;
+	}
+	else
+	{
+		gsCwq.eConfig &= ~CACHEOP_CONFIG_KLOG;
+	}
+
+	/* Step 1, set gsCwq.ui32Config based on gsCwq.eConfig */
+	ui32Config = 0;
+
+	if (gsCwq.eConfig & CACHEOP_CONFIG_KDF)
+	{
+		ui32Config |= CACHEOP_CONFIG_KDF;
+	}
+	if (gsCwq.eConfig & CACHEOP_CONFIG_URBF)
+	{
+		ui32Config |= CACHEOP_CONFIG_URBF;
+	}
+	if (gsCwq.eConfig & CACHEOP_CONFIG_KLOG)
+	{
+		ui32Config |= CACHEOP_CONFIG_KLOG;
+	}
+	gsCwq.ui32Config = ui32Config;
+
+
+	/* Step 3, in certain cases where a CacheOp/VA is provided, this threshold determines at what point
+	   the optimisation due to the presence of said VA (i.e. us not having to remap the PMR pages in KM)
+	   is clawed-back because of the overhead of maintaining such large request which might stalls the
+	   user thread; so to hide this latency have these CacheOps executed on deferred CacheOp thread */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] = (IMG_UINT32)(PVR_DIRTY_BYTES_FLUSH_THRESHOLD >> 2);
+
+	/* Step 4, if no UM support, all requests are done in KM so zero these forcing all client requests
+	   to come down into the KM for maintenance */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = 0;
+
+	if (gsCwq.bSupportsUMFlush)
+	{
+		/* With URBF enabled we never go to the kernel */
+		if (gsCwq.eConfig & CACHEOP_CONFIG_URBF)
+		{
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = (IMG_UINT32)~0;
+		}
+	}
+
+	/* Step 5, reset stats. */
+	CacheOpStatsReset();
+
+	OSLockRelease(gsCwq.hConfigLock);
+}
+
+static int CacheOpConfigRead(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	DIPrintf(psEntry, "URBF: %s\n",
+		gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No");
+
+	return 0;
+}
+
+static INLINE PVRSRV_ERROR CacheOpConfigQuery(const PVRSRV_DEVICE_NODE *psDevNode,
+											const void *psPrivate,
+											IMG_UINT32 *pui32Value)
+{
+	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	switch (ui32ID)
+	{
+		case APPHINT_ID_CacheOpConfig:
+			*pui32Value = gsCwq.ui32Config;
+			break;
+
+		case APPHINT_ID_CacheOpUMKMThresholdSize:
+			*pui32Value = gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD];
+			break;
+
+		default:
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR CacheOpConfigSet(const PVRSRV_DEVICE_NODE *psDevNode,
+											const void *psPrivate,
+											IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	switch (ui32ID)
+	{
+		case APPHINT_ID_CacheOpConfig:
+			CacheOpConfigUpdate(ui32Value & CACHEOP_CONFIG_ALL);
+			break;
+
+
+		case APPHINT_ID_CacheOpUMKMThresholdSize:
+		{
+			if (!ui32Value || !gsCwq.bSupportsUMFlush)
+			{
+				/* CPU ISA does not support UM flush, therefore every request goes down into
+				   the KM, silently ignore request to adjust threshold */
+				PVR_ASSERT(! gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD]);
+				break;
+			}
+			else if (ui32Value < gsCwq.uiPageSize)
+			{
+				/* Silently round-up to OS page size */
+				ui32Value = gsCwq.uiPageSize;
+			}
+
+			/* Align to OS page size */
+			ui32Value &= ~(gsCwq.uiPageSize - 1);
+
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = ui32Value;
+
+			break;
+		}
+
+		default:
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR CacheOpTimelineBind(PVRSRV_DEVICE_NODE *psDevNode,
+											   CACHEOP_WORK_ITEM *psCacheOpWorkItem,
+											   PVRSRV_TIMELINE iTimeline)
+{
+	PVRSRV_ERROR eError;
+
+	/* Always default the incoming CacheOp work-item to safe values */
+	SyncClearTimelineObj(&psCacheOpWorkItem->sSWTimelineObj);
+	psCacheOpWorkItem->iTimeline = PVRSRV_NO_TIMELINE;
+	psCacheOpWorkItem->psDevNode = psDevNode;
+	if (iTimeline == PVRSRV_NO_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+
+	psCacheOpWorkItem->iTimeline = iTimeline;
+	eError = SyncSWGetTimelineObj(iTimeline, &psCacheOpWorkItem->sSWTimelineObj);
+	PVR_LOG_IF_ERROR(eError, "SyncSWGetTimelineObj");
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR CacheOpTimelineExec(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
+{
+	PVRSRV_ERROR eError;
+
+	if (psCacheOpWorkItem->iTimeline == PVRSRV_NO_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->sSWTimelineObj.pvTlObj);
+
+	eError = SyncSWTimelineAdvanceKM(psCacheOpWorkItem->psDevNode,
+	                                 &psCacheOpWorkItem->sSWTimelineObj);
+	(void) SyncSWTimelineReleaseKM(&psCacheOpWorkItem->sSWTimelineObj);
+
+	return eError;
+}
+
+static INLINE void CacheOpExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
+										PVRSRV_CACHE_OP uiCacheOp,
+										IMG_BYTE *pbCpuVirtAddr,
+										IMG_CPU_PHYADDR sCpuPhyAddr,
+										IMG_DEVMEM_OFFSET_T uiPgAlignedOffset,
+										IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset,
+										IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset)
+{
+	IMG_BYTE *pbCpuVirtAddrEnd;
+	IMG_BYTE *pbCpuVirtAddrStart;
+	IMG_CPU_PHYADDR sCpuPhyAddrEnd;
+	IMG_CPU_PHYADDR sCpuPhyAddrStart;
+	IMG_DEVMEM_SIZE_T uiRelFlushSize;
+	IMG_DEVMEM_OFFSET_T uiRelFlushOffset;
+	IMG_DEVMEM_SIZE_T uiNextPgAlignedOffset;
+
+	/* These quantities allows us to perform cache operations
+	   at cache-line granularity thereby ensuring we do not
+	   perform more than is necessary */
+	CACHEOP_PVR_ASSERT(uiPgAlignedOffset < uiCLAlignedEndOffset);
+	uiRelFlushSize = (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
+	uiRelFlushOffset = 0;
+
+	if (uiCLAlignedStartOffset > uiPgAlignedOffset)
+	{
+		/* Zero unless initially starting at an in-page offset */
+		uiRelFlushOffset = uiCLAlignedStartOffset - uiPgAlignedOffset;
+		uiRelFlushSize -= uiRelFlushOffset;
+	}
+
+	/* uiRelFlushSize is gsCwq.uiPageSize unless current outstanding CacheOp
+	   size is smaller. The 1st case handles in-page CacheOp range and
+	   the 2nd case handles multiple-page CacheOp range with a last
+	   CacheOp size that is less than gsCwq.uiPageSize */
+	uiNextPgAlignedOffset = uiPgAlignedOffset + (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
+	if (uiNextPgAlignedOffset < uiPgAlignedOffset)
+	{
+		/* uiNextPgAlignedOffset is greater than uiCLAlignedEndOffset
+		   by implication of this wrap-round; this only happens when
+		   uiPgAlignedOffset is the last page aligned offset */
+		uiRelFlushSize = uiRelFlushOffset ?
+				uiCLAlignedEndOffset - uiCLAlignedStartOffset :
+				uiCLAlignedEndOffset - uiPgAlignedOffset;
+	}
+	else
+	{
+		if (uiNextPgAlignedOffset > uiCLAlignedEndOffset)
+		{
+			uiRelFlushSize = uiRelFlushOffset ?
+					uiCLAlignedEndOffset - uiCLAlignedStartOffset :
+					uiCLAlignedEndOffset - uiPgAlignedOffset;
+		}
+	}
+
+	/* More efficient to request cache maintenance operation for full
+	   relative range as opposed to multiple cache-aligned ranges */
+	sCpuPhyAddrStart.uiAddr = sCpuPhyAddr.uiAddr + uiRelFlushOffset;
+	sCpuPhyAddrEnd.uiAddr = sCpuPhyAddrStart.uiAddr + uiRelFlushSize;
+	if (pbCpuVirtAddr)
+	{
+		pbCpuVirtAddrStart = pbCpuVirtAddr + uiRelFlushOffset;
+		pbCpuVirtAddrEnd = pbCpuVirtAddrStart + uiRelFlushSize;
+	}
+	else
+	{
+		/* Some OS/Env layer support functions expect NULL(s) */
+		pbCpuVirtAddrStart = NULL;
+		pbCpuVirtAddrEnd = NULL;
+	}
+
+	/* Perform requested CacheOp on the CPU data cache for successive cache
+	   line worth of bytes up to page or in-page cache-line boundary */
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					__func__, uiCacheOp));
+			break;
+	}
+
+}
+
+static INLINE void CacheOpExecRangeBasedVA(PVRSRV_DEVICE_NODE *psDevNode,
+										 IMG_CPU_VIRTADDR pvAddress,
+										 IMG_DEVMEM_SIZE_T uiSize,
+										 PVRSRV_CACHE_OP uiCacheOp)
+{
+	IMG_CPU_PHYADDR sCpuPhyAddrUnused =
+		{ IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) };
+	IMG_BYTE *pbEnd = (IMG_BYTE*)((uintptr_t)pvAddress + (uintptr_t)uiSize);
+	IMG_BYTE *pbStart = (IMG_BYTE*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiLineSize-1));
+
+	/*
+	  If the start/end address isn't aligned to cache line size, round it up to the
+	  nearest multiple; this ensures that we flush all the cache lines affected by
+	  unaligned start/end addresses.
+	 */
+	pbEnd = (IMG_BYTE *) PVR_ALIGN((uintptr_t)pbEnd, (uintptr_t)gsCwq.uiLineSize);
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					 __func__, uiCacheOp));
+			break;
+	}
+
+}
+
+static INLINE PVRSRV_ERROR CacheOpValidateUMVA(PMR *psPMR,
+											   IMG_CPU_VIRTADDR pvAddress,
+											   IMG_DEVMEM_OFFSET_T uiOffset,
+											   IMG_DEVMEM_SIZE_T uiSize,
+											   PVRSRV_CACHE_OP uiCacheOp,
+											   void **ppvOutAddress)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(__linux__) && !defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *vma;
+#endif
+	void __user *pvAddr;
+
+	IMG_BOOL bReadOnlyInvalidate =
+		(uiCacheOp == PVRSRV_CACHE_OP_INVALIDATE) &&
+		!PVRSRV_CHECK_CPU_WRITEABLE(PMR_Flags(psPMR));
+
+	if (!pvAddress || bReadOnlyInvalidate)
+	{
+		/* As pvAddress is optional, NULL is expected from UM/KM requests */
+		/* Also don't allow invalidates for UMVA of read-only memory */
+		pvAddr = NULL;
+		goto e0;
+	}
+
+
+
+#if !defined(__linux__) || defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	pvAddr = NULL;
+#else
+	/* Validate VA, assume most basic address limit access_ok() check */
+	pvAddr = (void __user *)(uintptr_t)((uintptr_t)pvAddress + uiOffset);
+	if (!access_ok(pvAddr, uiSize))
+	{
+		pvAddr = NULL;
+		if (! mm)
+		{
+			/* Bad KM request, don't silently ignore */
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_CPU_ADDR, e0);
+		}
+	}
+	else if (mm)
+	{
+		mmap_read_lock(mm);
+		vma = find_vma(mm, (unsigned long)(uintptr_t)pvAddr);
+
+		if (!vma ||
+			vma->vm_start > (unsigned long)(uintptr_t)pvAddr ||
+			vma->vm_end < (unsigned long)(uintptr_t)pvAddr + uiSize ||
+			vma->vm_private_data != psPMR)
+		{
+			/*
+			 * Request range is not fully mapped or is not matching the PMR
+			 * Ignore request's VA.
+			 */
+			pvAddr = NULL;
+		}
+		mmap_read_unlock(mm);
+	}
+#endif
+
+e0:
+	*ppvOutAddress = (IMG_CPU_VIRTADDR __force) pvAddr;
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpPMRExec (PMR *psPMR,
+									IMG_CPU_VIRTADDR pvAddress,
+									IMG_DEVMEM_OFFSET_T uiOffset,
+									IMG_DEVMEM_SIZE_T uiSize,
+									PVRSRV_CACHE_OP uiCacheOp,
+									IMG_BOOL bIsRequestValidated)
+
+{
+	IMG_HANDLE hPrivOut = NULL;
+	IMG_BOOL bPMRIsSparse;
+	IMG_UINT32 ui32PageIndex;
+	IMG_UINT32 ui32NumOfPages;
+	size_t uiOutSize;	/* Effectively unused */
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_DEVMEM_SIZE_T uiPgAlignedSize;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedOffset;
+	IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedEndOffset;
+	IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedStartOffset;
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_CPU_PHYADDR asCpuPhyAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_CPU_PHYADDR *psCpuPhyAddr = asCpuPhyAddr;
+	IMG_BOOL bIsPMRInfoValid = IMG_FALSE;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BYTE *pbCpuVirtAddr = NULL;
+	IMG_BOOL *pbValid = abValid;
+	OS_CACHE_OP_ADDR_TYPE eCacheOpAddrType;
+
+	psDevNode = PMR_DeviceNode(psPMR);
+	eCacheOpAddrType = OSCPUCacheOpAddressType(psDevNode);
+
+	if (uiCacheOp == PVRSRV_CACHE_OP_NONE || uiCacheOp == PVRSRV_CACHE_OP_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (! bIsRequestValidated)
+	{
+		/* Need to validate parameters before proceeding */
+		/* Check for size + offset overflow */
+		PVR_LOG_RETURN_IF_FALSE(((uiOffset + uiSize) >= uiSize),
+		                        "Overflow detected on offset + size parameters",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		/* Since size + offset is later aligned to page size check for overflow with alignment */
+		PVR_LOG_RETURN_IF_FALSE((((uiOffset + uiSize) + gsCwq.uiPageSize - 1) >= (uiOffset + uiSize)),
+		                        "Overflow detected on offset + size parameters with applied alignment",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		PVR_LOG_RETURN_IF_FALSE(((uiOffset+uiSize) <= PMR_PhysicalSize(psPMR)),
+		                        CACHEOP_DEVMEM_OOR_ERROR_STRING,
+		                        PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PMRLockSysPhysAddresses");
+	}
+
+	/* Fast track the request if a CPU VA is provided and CPU ISA supports VA only maintenance */
+	eError = CacheOpValidateUMVA(psPMR, pvAddress, uiOffset, uiSize, uiCacheOp, (void**)&pbCpuVirtAddr);
+	if (eError == PVRSRV_OK)
+	{
+		pvAddress = pbCpuVirtAddr;
+
+		if (pvAddress && eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pvAddress, uiSize, uiCacheOp);
+
+			if (!bIsRequestValidated)
+			{
+				eError = PMRUnlockSysPhysAddresses(psPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+			}
+#if defined(CACHEOP_DEBUG)
+			gsCwq.ui32ServerOpUsedUMVA += 1;
+#endif
+			return PVRSRV_OK;
+		}
+		else if (pvAddress)
+		{
+			/* Round down the incoming VA (if any) down to the nearest page aligned VA */
+			pvAddress = (void*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1));
+#if defined(CACHEOP_DEBUG)
+			gsCwq.ui32ServerOpUsedUMVA += 1;
+#endif
+		}
+	}
+	else
+	{
+		/*
+		 * This validation pathway has been added to accommodate any/all requests that might
+		 * cause the kernel to Oops; essentially, KM requests should prevalidate cache maint.
+		 * parameters but if this fails then we would rather fail gracefully than cause the
+		 * kernel to Oops so instead we log the fact that an invalid KM virtual address was
+		 * supplied and what action was taken to mitigate against kernel Oops(ing) if any.
+		 */
+		CACHEOP_PVR_ASSERT(pbCpuVirtAddr == NULL);
+
+		if (eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, using paddress",
+					__func__,
+					pvAddress));
+
+			/* We can still proceed as kernel/cpu uses CPU PA for d-cache maintenance */
+			pvAddress = NULL;
+		}
+		else
+		{
+			/*
+			 * The approach here is to attempt a reacquisition of the PMR kernel VA and see if
+			 * said VA corresponds to the parameter VA, if so fail requested cache maint. op.
+			 * cause this indicates some kind of internal, memory and/or meta-data corruption
+			 * else we reissue the request using this (re)acquired alias PMR kernel VA.
+			 */
+			if (PMR_IsSparse(psPMR))
+			{
+				eError = PMRAcquireSparseKernelMappingData(psPMR,
+														   0,
+														   gsCwq.uiPageSize,
+														   (void **)&pbCpuVirtAddr,
+														   &uiOutSize,
+														   &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", e0);
+			}
+			else
+			{
+				eError = PMRAcquireKernelMappingData(psPMR,
+													 0,
+													 gsCwq.uiPageSize,
+													 (void **)&pbCpuVirtAddr,
+													 &uiOutSize,
+													 &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", e0);
+			}
+
+			/* Here, we only compare these CPU virtual addresses at granularity of the OS page size */
+			if ((uintptr_t)pbCpuVirtAddr == ((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, no alt. so failing request",
+						__func__,
+						pvAddress));
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_GOTO_WITH_ERROR("PMRReleaseKernelMappingData", eError, PVRSRV_ERROR_INVALID_CPU_ADDR, e0);
+			}
+			else if (eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s: Bad vaddress 0x%p in CPU d-cache maint. op, using reacquired vaddress 0x%p",
+						__func__,
+						pvAddress,
+						pbCpuVirtAddr));
+
+				/* Note that this might still fail if there is kernel memory/meta-data corruption;
+				   there is not much we can do here but at the least we will be informed of this
+				   before the kernel Oops(ing) */
+				CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pbCpuVirtAddr, uiSize, uiCacheOp);
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+				eError = PVRSRV_OK;
+				goto e0;
+			}
+			else
+			{
+				/* At this junction, we have exhausted every possible work-around possible but we do
+				   know that VA reacquisition returned another/alias page-aligned VA; so with this
+				   future expectation of PMRAcquireKernelMappingData(), we proceed */
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s: Bad vaddress %p in CPU d-cache maint. op, will use reacquired vaddress",
+						__func__,
+						pvAddress));
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+				/* NULL this to force per-page reacquisition down-stream */
+				pvAddress = NULL;
+			}
+		}
+	}
+
+	/* NULL clobbered var., OK to proceed */
+	pbCpuVirtAddr = NULL;
+	eError = PVRSRV_OK;
+
+	/* Need this for kernel mapping */
+	bPMRIsSparse = PMR_IsSparse(psPMR);
+
+	/* Round the incoming offset down to the nearest cache-line / page aligned-address */
+	uiCLAlignedEndOffset = uiOffset + uiSize;
+	uiCLAlignedEndOffset = PVR_ALIGN(uiCLAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiLineSize);
+	uiCLAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiLineSize-1));
+
+	uiPgAlignedEndOffset = uiCLAlignedEndOffset;
+	uiPgAlignedEndOffset = PVR_ALIGN(uiPgAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize);
+	uiPgAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiPageSize-1));
+	uiPgAlignedSize = uiPgAlignedEndOffset - uiPgAlignedStartOffset;
+
+#if defined(CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING)
+	/* For internal debug if cache-line optimised
+	   flushing is suspected of causing data corruption */
+	uiCLAlignedStartOffset = uiPgAlignedStartOffset;
+	uiCLAlignedEndOffset = uiPgAlignedEndOffset;
+#endif
+
+	/* Type of allocation backing the PMR data */
+	ui32NumOfPages = uiPgAlignedSize >> gsCwq.uiPageShift;
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		/* The pbValid array is allocated first as it is needed in
+		   both physical/virtual cache maintenance methods */
+		pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+		if (! pbValid)
+		{
+			pbValid = abValid;
+		}
+		else if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			psCpuPhyAddr = OSAllocZMem(ui32NumOfPages * sizeof(IMG_CPU_PHYADDR));
+			if (! psCpuPhyAddr)
+			{
+				psCpuPhyAddr = asCpuPhyAddr;
+				OSFreeMem(pbValid);
+				pbValid = abValid;
+			}
+		}
+	}
+
+	/* Lock the PMR while we are obtaining and using phys addrs */
+	PMRLockPMR(psPMR);
+
+	/* We always retrieve PMR data in bulk, up-front if number of pages is within
+	   PMR_MAX_TRANSLATION_STACK_ALLOC limits else we check to ensure that a
+	   dynamic buffer has been allocated to satisfy requests outside limits */
+	if (ui32NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC || pbValid != abValid)
+	{
+		if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			/* Look-up PMR CpuPhyAddr once, if possible */
+			eError = PMR_CpuPhysAddr(psPMR,
+									 gsCwq.uiPageShift,
+									 ui32NumOfPages,
+									 uiPgAlignedStartOffset,
+									 psCpuPhyAddr,
+									 pbValid,
+									 CPU_USE | MAPPING_USE);
+			if (eError == PVRSRV_OK)
+			{
+				bIsPMRInfoValid = IMG_TRUE;
+			}
+		}
+		else
+		{
+			/* Look-up PMR per-page validity once, if possible */
+			eError = PMR_IsOffsetValid(psPMR,
+									   gsCwq.uiPageShift,
+									   ui32NumOfPages,
+									   uiPgAlignedStartOffset,
+									   pbValid);
+			bIsPMRInfoValid = (eError == PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
+		}
+	}
+
+	/* For each (possibly non-contiguous) PMR page(s), carry out the requested cache maint. op. */
+	for (uiPgAlignedOffset = uiPgAlignedStartOffset, ui32PageIndex = 0;
+		 uiPgAlignedOffset < uiPgAlignedEndOffset;
+		 uiPgAlignedOffset += (IMG_DEVMEM_OFFSET_T) gsCwq.uiPageSize, ui32PageIndex += 1)
+	{
+
+		if (! bIsPMRInfoValid)
+		{
+			/* Never cross page boundary without looking up corresponding PMR page physical
+			   address and/or page validity if these were not looked-up, in bulk, up-front */
+			ui32PageIndex = 0;
+			if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+			{
+				eError = PMR_CpuPhysAddr(psPMR,
+										 gsCwq.uiPageShift,
+										 1,
+										 uiPgAlignedOffset,
+										 psCpuPhyAddr,
+										 pbValid,
+										 CPU_USE | MAPPING_USE);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMR_CpuPhysAddr", ErrUnlockPMR);
+			}
+			else
+			{
+				eError = PMR_IsOffsetValid(psPMR,
+										  gsCwq.uiPageShift,
+										  1,
+										  uiPgAlignedOffset,
+										  pbValid);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMR_IsOffsetValid", ErrUnlockPMR);
+			}
+		}
+
+		/* Skip invalid PMR pages (i.e. sparse) */
+		if (pbValid[ui32PageIndex] == IMG_FALSE)
+		{
+			CACHEOP_PVR_ASSERT(bPMRIsSparse);
+			continue;
+		}
+
+		if (pvAddress)
+		{
+			/* The caller has supplied either a KM/UM CpuVA, so use it unconditionally */
+			pbCpuVirtAddr =
+				(void *)(uintptr_t)((uintptr_t)pvAddress + (uintptr_t)(uiPgAlignedOffset-uiPgAlignedStartOffset));
+		}
+		/* Skip CpuVA acquire if CacheOp can be maintained entirely using CpuPA */
+		else if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			if (bPMRIsSparse)
+			{
+				eError =
+					PMRAcquireSparseKernelMappingData(psPMR,
+													  uiPgAlignedOffset,
+													  gsCwq.uiPageSize,
+													  (void **)&pbCpuVirtAddr,
+													  &uiOutSize,
+													  &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", ErrUnlockPMR);
+			}
+			else
+			{
+				eError =
+					PMRAcquireKernelMappingData(psPMR,
+												uiPgAlignedOffset,
+												gsCwq.uiPageSize,
+												(void **)&pbCpuVirtAddr,
+												&uiOutSize,
+												&hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", ErrUnlockPMR);
+			}
+		}
+
+		/* Issue actual cache maintenance for PMR */
+		CacheOpExecRangeBased(psDevNode,
+							uiCacheOp,
+							pbCpuVirtAddr,
+							(eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL) ?
+								psCpuPhyAddr[ui32PageIndex] : psCpuPhyAddr[0],
+							uiPgAlignedOffset,
+							uiCLAlignedStartOffset,
+							uiCLAlignedEndOffset);
+
+		if (! pvAddress)
+		{
+			/* The caller has not supplied either a KM/UM CpuVA, release mapping */
+			if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+			{
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+			}
+		}
+	}
+
+ErrUnlockPMR:
+	PMRUnlockPMR(psPMR);
+e0:
+	if (psCpuPhyAddr != asCpuPhyAddr)
+	{
+		OSFreeMem(psCpuPhyAddr);
+	}
+
+	if (pbValid != abValid)
+	{
+		OSFreeMem(pbValid);
+	}
+
+	if (! bIsRequestValidated)
+	{
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpBatchExecTimeline(PVRSRV_DEVICE_NODE *psDevNode,
+											 PVRSRV_TIMELINE iTimeline)
+{
+	PVRSRV_ERROR eError;
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {NULL};
+
+	eError = CacheOpTimelineBind(psDevNode, &sCacheOpWorkItem, iTimeline);
+	PVR_LOG_RETURN_IF_ERROR(eError, "CacheOpTimelineBind");
+
+	eError = CacheOpTimelineExec(&sCacheOpWorkItem);
+	PVR_LOG_IF_ERROR(eError, "CacheOpTimelineExec");
+
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpBatchExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
+											PMR **ppsPMR,
+											IMG_CPU_VIRTADDR *pvAddress,
+											IMG_DEVMEM_OFFSET_T *puiOffset,
+											IMG_DEVMEM_SIZE_T *puiSize,
+											PVRSRV_CACHE_OP *puiCacheOp,
+											IMG_UINT32 ui32NumCacheOps,
+											PVRSRV_TIMELINE uiTimeline)
+{
+	IMG_UINT32 ui32Idx;
+	IMG_BOOL bBatchHasTimeline;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+#endif
+
+	/* Check if batch has an associated timeline update */
+	bBatchHasTimeline = puiCacheOp[ui32NumCacheOps-1] & PVRSRV_CACHE_OP_TIMELINE;
+	puiCacheOp[ui32NumCacheOps-1] &= ~(PVRSRV_CACHE_OP_TIMELINE);
+
+	for (ui32Idx = 0; ui32Idx < ui32NumCacheOps; ui32Idx++)
+	{
+		/* Fail UM request, don't silently ignore */
+		PVR_GOTO_IF_INVALID_PARAM(puiSize[ui32Idx], eError, e0);
+
+#if defined(CACHEOP_DEBUG)
+		sCacheOpWorkItem.ui64StartTime = OSClockus64();
+#endif
+
+		eError = CacheOpPMRExec(ppsPMR[ui32Idx],
+								pvAddress[ui32Idx],
+								puiOffset[ui32Idx],
+								puiSize[ui32Idx],
+								puiCacheOp[ui32Idx],
+								IMG_FALSE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpExecPMR", e0);
+
+#if defined(CACHEOP_DEBUG)
+		sCacheOpWorkItem.ui64EndTime = OSClockus64();
+
+		sCacheOpWorkItem.psDevNode = psDevNode;
+		sCacheOpWorkItem.psPMR = ppsPMR[ui32Idx];
+		sCacheOpWorkItem.uiSize = puiSize[ui32Idx];
+		sCacheOpWorkItem.uiOffset = puiOffset[ui32Idx];
+		sCacheOpWorkItem.uiCacheOp = puiCacheOp[ui32Idx];
+		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+
+		gsCwq.ui32ServerOps += 1;
+#endif
+	}
+
+e0:
+	if (bBatchHasTimeline)
+	{
+		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline);
+	}
+
+	return eError;
+}
+
+
+PVRSRV_ERROR CacheOpExec (PPVRSRV_DEVICE_NODE psDevNode,
+						  void *pvVirtStart,
+						  void *pvVirtEnd,
+						  IMG_CPU_PHYADDR sCPUPhysStart,
+						  IMG_CPU_PHYADDR sCPUPhysEnd,
+						  PVRSRV_CACHE_OP uiCacheOp)
+{
+#if defined(CACHEOP_DEBUG)
+	IMG_UINT64 ui64StartTime = OSClockus64();
+#endif
+
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					 __func__, uiCacheOp));
+			break;
+	}
+
+#if defined(CACHEOP_DEBUG)
+	if (CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
+	{
+		CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+
+		gsCwq.ui32ServerOps += 1;
+
+		sCacheOpWorkItem.uiOffset = 0;
+		sCacheOpWorkItem.bKMReq = IMG_TRUE;
+		sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+		/* Use information page PMR for logging KM request */
+		sCacheOpWorkItem.psPMR = gsCwq.psInfoPagePMR;
+		sCacheOpWorkItem.psDevNode = psDevNode;
+		sCacheOpWorkItem.ui64StartTime = ui64StartTime;
+		sCacheOpWorkItem.ui64EndTime = OSClockus64();
+		sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+		sCacheOpWorkItem.uiSize = (sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr);
+
+		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR CacheOpValExec(PMR *psPMR,
+						    IMG_UINT64 uiAddress,
+						    IMG_DEVMEM_OFFSET_T uiOffset,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    PVRSRV_CACHE_OP uiCacheOp)
+{
+	PVRSRV_ERROR eError;
+	IMG_CPU_VIRTADDR pvAddress = (IMG_CPU_VIRTADDR)(uintptr_t)uiAddress;
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+
+	sCacheOpWorkItem.ui64StartTime = OSClockus64();
+#endif
+
+	eError = CacheOpPMRExec(psPMR,
+							pvAddress,
+							uiOffset,
+							uiSize,
+							uiCacheOp,
+							IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpPMRExec", e0);
+
+#if defined(CACHEOP_DEBUG)
+	sCacheOpWorkItem.ui64EndTime = OSClockus64();
+
+	sCacheOpWorkItem.psDevNode = PMR_DeviceNode(psPMR);
+	sCacheOpWorkItem.psPMR = psPMR;
+	sCacheOpWorkItem.uiSize = uiSize;
+	sCacheOpWorkItem.uiOffset = uiOffset;
+	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+
+	gsCwq.ui32ServerOps += 1;
+#endif
+
+e0:
+	return eError;
+}
+
+PVRSRV_ERROR CacheOpQueue (CONNECTION_DATA *psConnection,
+						   PVRSRV_DEVICE_NODE *psDevNode,
+						   IMG_UINT32 ui32NumCacheOps,
+						   PMR **ppsPMR,
+						   IMG_UINT64 *puiAddress,
+						   IMG_DEVMEM_OFFSET_T *puiOffset,
+						   IMG_DEVMEM_SIZE_T *puiSize,
+						   PVRSRV_CACHE_OP *puiCacheOp,
+						   IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_TIMELINE uiTimeline = (PVRSRV_TIMELINE)ui32OpTimeline;
+	IMG_CPU_VIRTADDR *pvAddress = (IMG_CPU_VIRTADDR*)(uintptr_t)puiAddress;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!gsCwq.bInit)
+	{
+		PVR_LOG(("CacheOp framework not initialised, failing request"));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+	else if (! ui32NumCacheOps)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	/* Ensure any single timeline CacheOp request is processed immediately */
+	else if (ui32NumCacheOps == 1 && puiCacheOp[0] == PVRSRV_CACHE_OP_TIMELINE)
+	{
+		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline);
+	}
+	/* This is the default entry for all client requests */
+	else
+	{
+		if (!(gsCwq.eConfig & (CACHEOP_CONFIG_LAST-1)))
+		{
+			/* default the configuration before execution */
+			CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
+		}
+
+		eError =
+			CacheOpBatchExecRangeBased(psDevNode,
+									   ppsPMR,
+									   pvAddress,
+									   puiOffset,
+									   puiSize,
+									   puiCacheOp,
+									   ui32NumCacheOps,
+									   uiTimeline);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR CacheOpLog (PMR *psPMR,
+						 IMG_UINT64 puiAddress,
+						 IMG_DEVMEM_OFFSET_T uiOffset,
+						 IMG_DEVMEM_SIZE_T uiSize,
+						 IMG_UINT64 ui64StartTime,
+						 IMG_UINT64 ui64EndTime,
+						 PVRSRV_CACHE_OP uiCacheOp)
+{
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+	PVR_UNREFERENCED_PARAMETER(puiAddress);
+
+	sCacheOpWorkItem.psDevNode = PMR_DeviceNode(psPMR);
+	sCacheOpWorkItem.psPMR = psPMR;
+	sCacheOpWorkItem.uiSize = uiSize;
+	sCacheOpWorkItem.uiOffset = uiOffset;
+	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+
+	sCacheOpWorkItem.ui64StartTime = ui64StartTime;
+	sCacheOpWorkItem.ui64EndTime = ui64EndTime;
+
+	gsCwq.ui32ClientOps += 1;
+
+	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+#else
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiCacheOp);
+	PVR_UNREFERENCED_PARAMETER(puiAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64EndTime);
+#endif
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR CacheOpInit2 (void)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	/* Apphint read/write is not concurrent, so lock protects against this */
+	eError = OSLockCreate((POS_LOCK*)&gsCwq.hConfigLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+
+#if defined(CACHEFLUSH_ISA_SUPPORTS_UM_FLUSH)
+	gsCwq.bSupportsUMFlush = IMG_TRUE;
+#else
+	gsCwq.bSupportsUMFlush = IMG_FALSE;
+#endif
+
+	gsCwq.pui32InfoPage = psPVRSRVData->pui32InfoPage;
+	gsCwq.psInfoPagePMR = psPVRSRVData->psInfoPagePMR;
+
+	/* Normally, platforms should use their default configurations, put exceptions here */
+#if defined(__i386__) || defined(__x86_64__)
+#if !defined(TC_MEMORY_CONFIG)
+	CacheOpConfigUpdate(CACHEOP_CONFIG_URBF | CACHEOP_CONFIG_KDF);
+#else
+	CacheOpConfigUpdate(CACHEOP_CONFIG_KDF);
+#endif
+#else /* defined(__x86__) */
+	CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
+#endif
+
+	/* Initialise the remaining occupants of the CacheOp information page */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_PGSIZE]   = (IMG_UINT32)gsCwq.uiPageSize;
+	gsCwq.pui32InfoPage[CACHEOP_INFO_LINESIZE] = (IMG_UINT32)gsCwq.uiLineSize;
+
+	/* Set before spawning thread */
+	gsCwq.bInit = IMG_TRUE;
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = CacheOpConfigRead};
+		/* Writing the unsigned integer binary encoding of CACHEOP_CONFIG
+		   into this file cycles through avail. configuration(s) */
+		eError = DICreateEntry("cacheop_config", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gsCwq.psConfigTune);
+		PVR_LOG_GOTO_IF_FALSE(gsCwq.psConfigTune, "DICreateEntry", e0);
+	}
+
+	/* Register the CacheOp framework (re)configuration handlers */
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpConfig,
+										CacheOpConfigQuery,
+										CacheOpConfigSet,
+										APPHINT_OF_DRIVER_NO_DEVICE,
+										(void *) APPHINT_ID_CacheOpConfig);
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpUMKMThresholdSize,
+										CacheOpConfigQuery,
+										CacheOpConfigSet,
+										APPHINT_OF_DRIVER_NO_DEVICE,
+										(void *) APPHINT_ID_CacheOpUMKMThresholdSize);
+
+	return PVRSRV_OK;
+e0:
+	CacheOpDeInit2();
+	return eError;
+}
+
+void CacheOpDeInit2 (void)
+{
+	gsCwq.bInit = IMG_FALSE;
+
+	if (gsCwq.hConfigLock)
+	{
+		OSLockDestroy(gsCwq.hConfigLock);
+		gsCwq.hConfigLock = NULL;
+	}
+
+	if (gsCwq.psConfigTune)
+	{
+		DIDestroyEntry(gsCwq.psConfigTune);
+		gsCwq.psConfigTune = NULL;
+	}
+
+	gsCwq.pui32InfoPage = NULL;
+	gsCwq.psInfoPagePMR = NULL;
+}
+
+PVRSRV_ERROR CacheOpInit (void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	gsCwq.uiPageSize = OSGetPageSize();
+	gsCwq.uiPageShift = OSGetPageShift();
+	gsCwq.uiLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+	gsCwq.uiLineShift = ExactLog2(gsCwq.uiLineSize);
+	PVR_LOG_RETURN_IF_FALSE((gsCwq.uiLineSize && gsCwq.uiPageSize && gsCwq.uiPageShift), "", PVRSRV_ERROR_INIT_FAILURE);
+
+#if defined(CACHEOP_DEBUG)
+	/* debugfs file read-out is not concurrent, so lock protects against this */
+	eError = OSLockCreate((POS_LOCK*)&gsCwq.hStatsExecLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	gsCwq.i32StatsExecWriteIdx = 0;
+	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = CacheOpStatsExecLogRead};
+		/* File captures the most recent subset of CacheOp(s) executed */
+		eError = DICreateEntry("cacheop_history", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gsCwq.psDIEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", e0);
+	}
+e0:
+#endif
+	return eError;
+}
+
+void CacheOpDeInit (void)
+{
+#if defined(CACHEOP_DEBUG)
+	if (gsCwq.hStatsExecLock)
+	{
+		OSLockDestroy(gsCwq.hStatsExecLock);
+		gsCwq.hStatsExecLock = NULL;
+	}
+
+	if (gsCwq.psDIEntry)
+	{
+		DIDestroyEntry(gsCwq.psDIEntry);
+		gsCwq.psDIEntry = NULL;
+	}
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/connection_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/connection_server.c
new file mode 100644
index 000000000000..b8d3994bcce1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/connection_server.c
@@ -0,0 +1,515 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Handles connections coming from the client and the management
+                connection based information
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "handle.h"
+#include "pvrsrv.h"
+#include "connection_server.h"
+#include "osconnection_server.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "sync_server.h"
+#include "process_stats.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "tlstream.h"
+#include "rgxhwperf_common.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+/* PID associated with Connection currently being purged by Cleanup thread */
+static IMG_PID gCurrentPurgeConnectionPid;
+
+static PVRSRV_ERROR ConnectionDataDestroy(CONNECTION_DATA *psConnection)
+{
+	PVRSRV_ERROR eError;
+	PROCESS_HANDLE_BASE *psProcessHandleBase;
+	IMG_UINT64 ui64MaxBridgeTime;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+
+	if (psPVRSRVData->bUnload)
+	{
+		/* driver is unloading so do not allow the bridge lock to be released */
+		ui64MaxBridgeTime = 0;
+	}
+	else
+	{
+		ui64MaxBridgeTime = CONNECTION_DEFERRED_CLEANUP_TIMESLICE_NS;
+	}
+
+	PVR_ASSERT(psConnection != NULL);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psConnection, "psConnection");
+
+	/* Close HWPerfClient stream here even though we created it in
+	 * PVRSRVConnectKM(). */
+	if (psConnection->hClientTLStream)
+	{
+		TLStreamClose(psConnection->hClientTLStream);
+		psConnection->hClientTLStream = NULL;
+		PVR_DPF((PVR_DBG_MESSAGE, "Destroyed private stream."));
+	}
+
+	/* Get process handle base to decrement the refcount */
+	psProcessHandleBase = psConnection->psProcessHandleBase;
+
+	if (psProcessHandleBase != NULL)
+	{
+		/* PVRSRVReleaseProcessHandleBase() calls PVRSRVFreeKernelHandles()
+		 * and PVRSRVFreeHandleBase() for the process handle base.
+		 * Releasing kernel handles can never return RETRY error because
+		 * release function for those handles are NOPs and PVRSRVFreeKernelHandles()
+		 * doesn't even call pfnReleaseData() callback.
+		 * Process handles can potentially return RETRY hence additional check
+		 * below. */
+		eError = PVRSRVReleaseProcessHandleBase(psProcessHandleBase,
+		                                        ui64MaxBridgeTime);
+		if (PVRSRVIsRetryError(eError))
+		{
+			return eError;
+		}
+		else
+		{
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVReleaseProcessHandleBase");
+		}
+
+		psConnection->psProcessHandleBase = NULL;
+	}
+
+	/* Free handle base for this connection */
+	if (psConnection->psHandleBase != NULL)
+	{
+		eError = PVRSRVFreeHandleBase(psConnection->psHandleBase, ui64MaxBridgeTime);
+		/*
+		 * If we get PVRSRV_ERROR_RETRY we need to pass this back to the caller
+		 * who will schedule a retry.
+		 * Do not log this as it is an expected exception.
+		 * This can occur if the Firmware is still processing a workload from
+		 * the client when a tear-down request is received.
+		 * Retrying will allow the in-flight work to be completed and the
+		 * tear-down request can be completed when the FW is no longer busy.
+		 */
+		if (PVRSRVIsRetryError(eError))
+		{
+			return eError;
+		}
+		else
+		{
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeHandleBase");
+		}
+
+		psConnection->psHandleBase = NULL;
+	}
+
+	if (psConnection->psSyncConnectionData != NULL)
+	{
+		SyncUnregisterConnection(psConnection->psSyncConnectionData);
+		psConnection->psSyncConnectionData = NULL;
+	}
+
+	if (psConnection->psPDumpConnectionData != NULL)
+	{
+		PDumpUnregisterConnection(psDevNode,
+		                          psConnection->psPDumpConnectionData);
+		psConnection->psPDumpConnectionData = NULL;
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDeviceDisconnect(psDevNode);
+#endif
+
+	/* Call environment specific connection data deinit function */
+	if (psConnection->hOsPrivateData != NULL)
+	{
+		eError = OSConnectionPrivateDataDeInit(psConnection->hOsPrivateData);
+		PVR_LOG_RETURN_IF_ERROR(eError, "OSConnectionPrivateDataDeInit");
+
+		psConnection->hOsPrivateData = NULL;
+	}
+
+	/* Close the PID stats entry as late as possible to catch all frees */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+	if (psConnection->hProcessStats != NULL)
+	{
+		PVRSRVStatsDeregisterProcess(psConnection->hProcessStats);
+		psConnection->hProcessStats = NULL;
+	}
+#endif
+
+	OSFreeMemNoStats(psConnection);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Kick the Firmware to invalidate caches to clear all the zombie PMRs.
+	 * If there are not zombie PMRs or no mappings were freed the kick will not
+	 * be executed.
+	 *
+	 * This is needed:
+	 * - when the process is killed and the connection cleanup has to clean up
+	 *   all dangling handles.
+	 * - there are any outstanding PMRs in the zombie list due to no
+	 *   invalidation being executed before connection destruction
+	 */
+	eError = MMU_CacheInvalidateKick(psDevNode, NULL);
+	PVR_LOG_IF_ERROR(eError, "MMU_CacheInvalidateKick");
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVCommonConnectionConnect(void **ppvPrivData, void *pvOSData)
+{
+	CONNECTION_DATA *psConnection;
+	PVRSRV_ERROR eError;
+	PROCESS_HANDLE_BASE *psProcessHandleBase;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* Allocate connection data area, no stats since process not registered yet */
+	psConnection = OSAllocZMemNoStats(sizeof(*psConnection));
+	PVR_LOG_RETURN_IF_NOMEM(psConnection, "psConnection");
+
+	/* Allocate process statistics as early as possible to catch all allocs */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+	eError = PVRSRVStatsRegisterProcess(&psConnection->hProcessStats);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsRegisterProcess", failure);
+#endif
+
+	/* Call environment specific connection data init function */
+	eError = OSConnectionPrivateDataInit(&psConnection->hOsPrivateData, pvOSData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSConnectionPrivateDataInit", failure);
+
+	/* Must come after OSConnectionPrivateDataInit */
+	psDevNode = OSGetDevNode(psConnection);
+	PVR_LOG_GOTO_IF_NOMEM(psDevNode, eError, failure);
+
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cannot connect to the device during deinit"));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_DEVICE, failure);
+	}
+
+	psConnection->pid = OSGetCurrentClientProcessIDKM();
+	psConnection->vpid = OSGetCurrentVirtualProcessID();
+	psConnection->tid = (IMG_UINT32)OSGetCurrentClientThreadIDKM();
+	OSStringSafeCopy(psConnection->pszProcName, OSGetCurrentClientProcessNameKM(), PVRSRV_CONNECTION_PROCESS_NAME_LEN);
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	eError = PVRSRVInitialiseDMA(psDevNode, psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVInitialiseDMA", failure);
+#endif
+
+	/* Register this connection with the sync core */
+	eError = SyncRegisterConnection(&psConnection->psSyncConnectionData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncRegisterConnection", failure);
+
+	/*
+	 * Register this connection and Sync PDump callback with
+	 * the pdump core. Pass in the Sync connection data.
+	 */
+	eError = PDumpRegisterConnection(psDevNode,
+	                                 psConnection->psSyncConnectionData,
+	                                 SyncConnectionPDumpSyncBlocks,
+	                                 &psConnection->psPDumpConnectionData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpRegisterConnection", failure);
+
+	/* Allocate handle base for this connection */
+	eError = PVRSRVAllocHandleBase(&psConnection->psHandleBase,
+	                               PVRSRV_HANDLE_BASE_TYPE_CONNECTION);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", failure);
+
+	/* get process handle base for the current process (if it doesn't exist it will be allocated) */
+	eError = PVRSRVAcquireProcessHandleBase(&psProcessHandleBase);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireProcessHandleBase", failure);
+
+	/* hConnectionsLock now resides in PVRSRV_DEVICE_NODE */
+	{
+		IMG_BOOL bHostStreamIsNull;
+		PVRSRV_RGXDEV_INFO  *psRgxDevInfo;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		eError = PVRSRVStatsDeviceConnect(psDevNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsDeviceConnect", failure);
+#endif
+
+		OSLockAcquire(psDevNode->hConnectionsLock);
+		dllist_add_to_tail(&psDevNode->sConnections, &psConnection->sConnectionListNode);
+#if defined(DEBUG) || defined(PDUMP)
+		PVR_LOG(("%s connected - (devID = %u)", psConnection->pszProcName,
+		        psDevNode->sDevId.ui32InternalID));
+#endif
+		OSLockRelease(psDevNode->hConnectionsLock);
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+		{
+			psRgxDevInfo = _RGX_DEVICE_INFO_FROM_NODE(psDevNode);
+
+			OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+			bHostStreamIsNull = (IMG_BOOL)(psRgxDevInfo->hHWPerfHostStream == NULL);
+			OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+			if (!bHostStreamIsNull)
+			{
+				if (TLStreamIsOpenForReading(psRgxDevInfo->hHWPerfHostStream))
+				{
+					/* Announce this client connection in the host stream, if event mask is set */
+					RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(psDevNode, psConnection->pid, psConnection->pszProcName);
+				}
+			}
+		}
+	}
+
+	psConnection->psProcessHandleBase = psProcessHandleBase;
+
+	*ppvPrivData = psConnection;
+
+	return PVRSRV_OK;
+
+failure:
+	ConnectionDataDestroy(psConnection);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _CleanupThreadPurgeConnectionData(void *pvConnectionData)
+{
+	PVRSRV_ERROR eErrorConnection, eErrorKernel;
+	CONNECTION_DATA *psConnectionData = pvConnectionData;
+
+	gCurrentPurgeConnectionPid = psConnectionData->pid;
+
+	eErrorConnection = ConnectionDataDestroy(psConnectionData);
+	if (eErrorConnection != PVRSRV_OK)
+	{
+		if (PVRSRVIsRetryError(eErrorConnection))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Failed to purge connection data %p "
+			        "(deferring destruction)", __func__, psConnectionData));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Connection data %p deferred destruction "
+		        "finished", __func__, psConnectionData));
+	}
+
+	/* Check if possible resize the global handle base */
+	eErrorKernel = PVRSRVPurgeHandles(KERNEL_HANDLE_BASE);
+	PVR_LOG_IF_ERROR(eErrorKernel, "PVRSRVPurgeHandles");
+
+	gCurrentPurgeConnectionPid = 0;
+
+	return eErrorConnection;
+}
+
+void PVRSRVCommonConnectionDisconnect(void *pvDataPtr)
+{
+	CONNECTION_DATA *psConnectionData = pvDataPtr;
+	PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnectionData);
+
+	OSLockAcquire(psDevNode->hConnectionsLock);
+	dllist_remove_node(&psConnectionData->sConnectionListNode);
+	OSLockRelease(psDevNode->hConnectionsLock);
+
+	/* Notify the PDump core if the pdump control client is disconnecting */
+	if (psConnectionData->ui32ClientFlags & SRV_FLAGS_PDUMPCTRL)
+	{
+		PDumpDisconnectionNotify(psDevNode);
+	}
+
+	/* Add a HOST_CLIENT_INFO event to match the one on connection */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		IMG_BOOL bHostStreamIsNull;
+		PVRSRV_RGXDEV_INFO  *psRgxDevInfo;
+
+		psRgxDevInfo = _RGX_DEVICE_INFO_FROM_NODE(psDevNode);
+
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+		bHostStreamIsNull = (IMG_BOOL)(psRgxDevInfo->hHWPerfHostStream == NULL);
+		OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+		if (!bHostStreamIsNull)
+		{
+			if (TLStreamIsOpenForReading(psRgxDevInfo->hHWPerfHostStream))
+			{
+				/* Announce this client connection in the host stream, if event mask is set */
+				RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(psDevNode, psConnectionData->pid, psConnectionData->pszProcName);
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Mark remaining resources for driver to free */
+	RIConnectionClosed(psConnectionData);
+#endif
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	PVRSRVDeInitialiseDMA(psDevNode, psConnectionData);
+#endif
+
+#if defined(DEBUG) || defined(PDUMP)
+	PVR_LOG(("%s disconnected - (devID = %u)", psConnectionData->pszProcName,
+	        psDevNode->sDevId.ui32InternalID));
+#endif
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK)
+#endif
+	{
+		/* Defer the release of the connection data */
+		psConnectionData->sCleanupThreadFn.pfnFree = _CleanupThreadPurgeConnectionData;
+		psConnectionData->sCleanupThreadFn.pvData = psConnectionData;
+		/* Some resources in HANDLE_BASE may need FW idle confirmation
+		 * hence setting to TRUE to use the global EO for retries which is
+		 * signalled by the device MISR */
+		psConnectionData->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+		psConnectionData->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_CONNECTION;
+		CLEANUP_THREAD_SET_RETRY_COUNT(&psConnectionData->sCleanupThreadFn,
+		                               CLEANUP_THREAD_RETRY_COUNT_DEFAULT);
+		PVRSRVCleanupThreadAddWork(psDevNode, &psConnectionData->sCleanupThreadFn);
+	}
+}
+
+IMG_PID PVRSRVGetPurgeConnectionPid(void)
+{
+	return gCurrentPurgeConnectionPid;
+}
+
+/* Prefix for debug messages about Active Connections */
+#define DEBUG_DUMP_CONNECTION_FORMAT_STR " P%d-V%d-T%d-%s,"
+#define CONNECTIONS_PREFIX               "Connections Device ID:%u(%d)"
+#define MAX_CONNECTIONS_PREFIX            (29)
+#define MAX_DEBUG_DUMP_CONNECTION_STR_LEN (1+10+10+10+7+PVRSRV_CONNECTION_PROCESS_NAME_LEN)
+#define MAX_DEBUG_DUMP_STRING_LEN         (1+MAX_CONNECTIONS_PREFIX+(3*MAX_DEBUG_DUMP_CONNECTION_STR_LEN))
+
+void PVRSRVConnectionDebugNotify(PVRSRV_DEVICE_NODE *psDevNode,
+								 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile)
+{
+	PDLLIST_NODE pNext, pNode;
+
+	/* We must check for an initialised device before accessing its mutex.
+	 * The mutex is initialised as part of DeviceInitialize() which occurs
+	 * on first access to the device node.
+	 */
+	if ((psDevNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE) &&
+	    (psDevNode->eDevState != PVRSRV_DEVICE_STATE_FROZEN))
+	{
+		PVR_DUMPDEBUG_LOG("Connections: No Devices: No active connections");
+		return;
+	}
+
+	OSLockAcquire(psDevNode->hConnectionsLock);
+	if (dllist_is_empty(&psDevNode->sConnections))
+	{
+		PVR_DUMPDEBUG_LOG(CONNECTIONS_PREFIX " No active connections",
+						  (unsigned char)psDevNode->sDevId.ui32InternalID,
+						  (unsigned char)psDevNode->sDevId.i32KernelDeviceID);
+	}
+	else
+	{
+		IMG_CHAR sActiveConnections[MAX_DEBUG_DUMP_STRING_LEN];
+		IMG_UINT16 i, uiPos = 0;
+		IMG_BOOL bPrinted = IMG_FALSE;
+		size_t uiSize = sizeof(sActiveConnections);
+
+		IMG_CHAR szTmpConBuff[MAX_CONNECTIONS_PREFIX + 1];
+		i = OSSNPrintf(szTmpConBuff,
+					   MAX_CONNECTIONS_PREFIX,
+					   CONNECTIONS_PREFIX,
+					   (unsigned char)psDevNode->sDevId.ui32InternalID,
+					   (unsigned char)psDevNode->sDevId.i32KernelDeviceID);
+		OSStringSafeCopy(sActiveConnections+uiPos, szTmpConBuff, uiSize);
+
+		/* Move the write offset to the end of the current string */
+		uiPos += i;
+		/* Update the amount of remaining space available to copy into */
+		uiSize -= i;
+
+		dllist_foreach_node(&psDevNode->sConnections, pNode, pNext)
+		{
+			CONNECTION_DATA *sData = IMG_CONTAINER_OF(pNode, CONNECTION_DATA, sConnectionListNode);
+
+			IMG_CHAR sTmpBuff[MAX_DEBUG_DUMP_CONNECTION_STR_LEN];
+			i = OSSNPrintf(sTmpBuff, MAX_DEBUG_DUMP_CONNECTION_STR_LEN,
+				DEBUG_DUMP_CONNECTION_FORMAT_STR, sData->pid, sData->vpid, sData->tid, sData->pszProcName);
+			i = MIN(MAX_DEBUG_DUMP_CONNECTION_STR_LEN, i);
+			bPrinted = IMG_FALSE;
+
+			OSStringSafeCopy(sActiveConnections+uiPos, sTmpBuff, uiSize);
+
+			/* Move the write offset to the end of the current string */
+			uiPos += i;
+			/* Update the amount of remaining space available to copy into */
+			uiSize -= i;
+
+			/* If there is not enough space to add another connection to this line, output the line */
+			if (uiSize <= MAX_DEBUG_DUMP_CONNECTION_STR_LEN)
+			{
+				PVR_DUMPDEBUG_LOG("%s", sActiveConnections);
+
+				/*
+				 * Remove the "Connections:" prefix from the buffer.
+				 * Leave the subsequent buffer contents indented by the same
+				 * amount to aid in interpreting the debug output.
+				 */
+				uiPos = sizeof(CONNECTIONS_PREFIX) - 1;
+				/* Reset the amount of space available to copy into */
+				uiSize = MAX_DEBUG_DUMP_STRING_LEN - uiPos;
+				bPrinted = IMG_TRUE;
+			}
+		}
+
+		/* Only print the current line if it hasn't already been printed */
+		if (!bPrinted)
+		{
+			/* Strip off the final comma */
+			sActiveConnections[OSStringNLength(sActiveConnections, MAX_DEBUG_DUMP_STRING_LEN) - 1] = '\0';
+			PVR_DUMPDEBUG_LOG("%s", sActiveConnections);
+		}
+#undef MAX_DEBUG_DUMP_STRING_LEN
+#undef MAX_DEBUG_DUMP_CONNECTIONS_PER_LINE
+	}
+	OSLockRelease(psDevNode->hConnectionsLock);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/debug_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/debug_common.c
new file mode 100644
index 000000000000..4499ed6de993
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/debug_common.c
@@ -0,0 +1,1995 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Creates common debug info entries.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__linux__)
+#include <errno.h>
+#endif /* #if !defined(__linux__) */
+
+#include "debug_common.h"
+#include "pvrsrv.h"
+#include "di_server.h"
+#include "lists.h"
+#include "pvrversion.h"
+#include "rgx_options.h"
+#include "allocmem.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "osfunc.h"
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#include "rgxfwdbg.h"
+#endif
+
+#ifdef SUPPORT_RGX
+#include "rgxdevice.h"
+#include "rgxdebug_common.h"
+#include "rgxinit.h"
+#include "rgxmmudefs_km.h"
+#endif
+
+static DI_ENTRY *gpsVersionDIEntry;
+static DI_ENTRY *gpsStatusDIEntry;
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+static DI_ENTRY *gpsDebugLevelDIEntry;
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+struct DI_VZ_DATA {
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_UINT32 ui32DriverID;
+};
+#endif
+
+static void _DumpDebugDIPrintfWrapper(void *pvDumpDebugFile, const IMG_CHAR *pszFormat, ...)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list ArgList;
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN, "%s\n", pszFormat);
+
+	va_start(ArgList, pszFormat);
+	DIVPrintf(pvDumpDebugFile, szBuffer, ArgList);
+	va_end(ArgList);
+}
+
+/*************************************************************************/ /*!
+ Version DebugFS entry
+*/ /**************************************************************************/
+
+static void *_DebugVersionCompare_AnyVaCb(PVRSRV_DEVICE_NODE *psDevNode,
+                                          va_list va)
+{
+	IMG_UINT64 *pui64CurrentPosition = va_arg(va, IMG_UINT64 *);
+	IMG_UINT64 ui64Position = va_arg(va, IMG_UINT64);
+	IMG_UINT64 ui64CurrentPosition = *pui64CurrentPosition;
+
+	(*pui64CurrentPosition)++;
+
+	return (ui64CurrentPosition == ui64Position) ? psDevNode : NULL;
+}
+
+static void *_VersionDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+
+	if (psPVRSRVData == NULL) {
+		PVR_DPF((PVR_DBG_ERROR, "psPVRSRVData = NULL"));
+		return NULL;
+	}
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+	                                      _DebugVersionCompare_AnyVaCb,
+	                                      &uiCurrentPosition,
+	                                      *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+static void _VersionDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvPriv)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+}
+
+static void *_VersionDINext(OSDI_IMPL_ENTRY *psEntry,void *pvPriv,
+                            IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	(*pui64Pos)++;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+	                                      _DebugVersionCompare_AnyVaCb,
+	                                      &uiCurrentPosition,
+	                                      *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+#define DI_PRINT_VERSION_FMTSPEC \
+		"%s Version: %u.%u @ %u (%s) build options: 0x%08x %s\n"
+#define STR_DEBUG   "debug"
+#define STR_RELEASE "release"
+
+#if defined(DEBUG)
+#define BUILD_OPT_LEN 80
+
+static inline void _AppendOptionStr(IMG_CHAR pszBuildOptions[], const IMG_CHAR* str, OSDI_IMPL_ENTRY *psEntry, IMG_UINT32* pui32BuildOptionLen)
+{
+	IMG_UINT32 ui32BuildOptionLen = *pui32BuildOptionLen;
+	const IMG_UINT32 strLen = OSStringLength(str);
+	const IMG_UINT32 optStrLen = sizeof(IMG_CHAR) * (BUILD_OPT_LEN-1);
+
+	if ((ui32BuildOptionLen + strLen) > optStrLen)
+	{
+		pszBuildOptions[ui32BuildOptionLen] = '\0';
+		DIPrintf(psEntry, "%s\n", pszBuildOptions);
+		ui32BuildOptionLen = 0;
+	}
+	if (strLen < optStrLen)
+	{
+		OSStringSafeCopy(pszBuildOptions+ui32BuildOptionLen, str, strLen);
+		ui32BuildOptionLen += strLen - 1;
+	}
+	*pui32BuildOptionLen = ui32BuildOptionLen;
+}
+#endif /* DEBUG || SUPPORT_VALIDATION */
+
+static int _VersionDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvPriv)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+
+	if (pvPriv == DI_START_TOKEN)
+	{
+		if (psPVRSRVData->sDriverInfo.bIsNoMatch)
+		{
+			const BUILD_INFO *psBuildInfo;
+
+			psBuildInfo = &psPVRSRVData->sDriverInfo.sUMBuildInfo;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+			         "UM Driver",
+			         PVRVERSION_UNPACK_MAJ(psBuildInfo->ui32BuildVersion),
+			         PVRVERSION_UNPACK_MIN(psBuildInfo->ui32BuildVersion),
+			         psBuildInfo->ui32BuildRevision,
+			         (psBuildInfo->ui32BuildType == BUILD_TYPE_DEBUG) ?
+			                 STR_DEBUG : STR_RELEASE,
+			         psBuildInfo->ui32BuildOptions,
+			         PVR_BUILD_DIR);
+
+			psBuildInfo = &psPVRSRVData->sDriverInfo.sKMBuildInfo;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+			         "KM Driver (" PVR_ARCH_NAME ")",
+			         PVRVERSION_UNPACK_MAJ(psBuildInfo->ui32BuildVersion),
+			         PVRVERSION_UNPACK_MIN(psBuildInfo->ui32BuildVersion),
+			         psBuildInfo->ui32BuildRevision,
+			         (psBuildInfo->ui32BuildType == BUILD_TYPE_DEBUG) ?
+			                 STR_DEBUG : STR_RELEASE,
+			         psBuildInfo->ui32BuildOptions,
+			         PVR_BUILD_DIR);
+		}
+		else
+		{
+			/* bIsNoMatch is `false` in one of the following cases:
+			 * - UM & KM version parameters actually match.
+			 * - A comparison between UM & KM has not been made yet, because no
+			 *   client ever connected.
+			 *
+			 * In both cases, available (KM) version info is the best output we
+			 * can provide.
+			 */
+			DIPrintf(psEntry, "Driver Version: %s (%s) (%s) build options: "
+			         "0x%08lx %s\n", PVRVERSION_STRING, PVR_ARCH_NAME,
+			         PVR_BUILD_TYPE, RGX_BUILD_OPTIONS_KM, PVR_BUILD_DIR);
+		}
+	}
+	else if (pvPriv != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *) pvPriv;
+		PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+#ifdef SUPPORT_RGX
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+#if defined(DEBUG)
+		IMG_CHAR pszBuildOptions[BUILD_OPT_LEN];
+		IMG_UINT32 ui32BuildOptionLen = 0;
+		static const char* aszOptions[] = RGX_BUILD_OPTIONS_LIST;
+		int i = 0;
+#endif
+#endif /* SUPPORT_RGX */
+		IMG_BOOL bFwVersionInfoPrinted = IMG_FALSE;
+
+		DIPrintf(psEntry, "\nDevice Name: %s\n", psDevConfig->pszName);
+		DIPrintf(psEntry, "Device ID: %u:%d\n", psDevNode->sDevId.ui32InternalID,
+		                                        psDevNode->sDevId.i32KernelDeviceID);
+
+		if (psDevConfig->pszVersion)
+		{
+			DIPrintf(psEntry, "Device Version: %s\n",
+			          psDevConfig->pszVersion);
+		}
+
+		if (psDevNode->pfnDeviceVersionString)
+		{
+			IMG_CHAR *pszVerStr;
+
+			if (psDevNode->pfnDeviceVersionString(psDevNode,
+			                                      &pszVerStr) == PVRSRV_OK)
+			{
+				DIPrintf(psEntry, "%s\n", pszVerStr);
+
+				OSFreeMem(pszVerStr);
+			}
+		}
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+		{
+#ifdef SUPPORT_RGX
+			/* print device's firmware version info */
+			if (psDevInfo->psRGXFWIfOsInitMemDesc != NULL)
+			{
+				/* psDevInfo->psRGXFWIfOsInitMemDesc should be permanently mapped */
+				if (psDevInfo->psRGXFWIfOsInit != NULL)
+				{
+					RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+				                               INVALIDATE);
+					if (psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated)
+					{
+						const RGXFWIF_COMPCHECKS *psRGXCompChecks =
+						        &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks;
+						IMG_UINT32 ui32DDKVer = psRGXCompChecks->ui32DDKVersion;
+
+						DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+						         "Firmware",
+						         PVRVERSION_UNPACK_MAJ(ui32DDKVer),
+						         PVRVERSION_UNPACK_MIN(ui32DDKVer),
+						         psRGXCompChecks->ui32DDKBuild,
+						         ((psRGXCompChecks->ui32BuildOptions &
+						          OPTIONS_DEBUG_EN) ? STR_DEBUG : STR_RELEASE),
+						         psRGXCompChecks->ui32BuildOptions,
+						         PVR_BUILD_DIR);
+						bFwVersionInfoPrinted = IMG_TRUE;
+
+#if defined(DEBUG)
+						DIPrintf(psEntry, "Firmware Build Options:\n");
+
+						for (i = 0; i < ARRAY_SIZE(aszOptions); i++)
+						{
+							if ((psRGXCompChecks->ui32BuildOptions & 1<<i))
+							{
+								_AppendOptionStr(pszBuildOptions, aszOptions[i], psEntry, &ui32BuildOptionLen);
+							}
+						}
+
+						if (ui32BuildOptionLen != 0)
+						{
+							DIPrintf(psEntry, "%s", pszBuildOptions);
+						}
+						DIPrintf(psEntry, "\n");
+#endif
+					}
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Error acquiring CPU virtual "
+					        "address of FWInitMemDesc", __func__));
+				}
+			}
+#endif /* SUPPORT_RGX */
+		}
+		else
+		{
+#ifdef SUPPORT_RGX
+			RGX_FW_INFO_HEADER	*psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+					 "Firmware",
+					 psFWInfoHeader->ui16PVRVersionMajor,
+					 psFWInfoHeader->ui16PVRVersionMinor,
+					 psFWInfoHeader->ui32PVRVersionBuild,
+					 ((psFWInfoHeader->ui32Flags &
+					 OPTIONS_DEBUG_EN) ? STR_DEBUG : STR_RELEASE),
+					 psFWInfoHeader->ui32Flags,
+					 PVR_BUILD_DIR);
+
+			bFwVersionInfoPrinted = IMG_TRUE;
+#if defined(DEBUG)
+			DIPrintf(psEntry, "Firmware Build Options:\n");
+
+			for (i = 0; i < ARRAY_SIZE(aszOptions); i++)
+			{
+				if ((psFWInfoHeader->ui32Flags & 1<<i))
+				{
+					_AppendOptionStr(pszBuildOptions, aszOptions[i], psEntry, &ui32BuildOptionLen);
+				}
+			}
+
+			if (ui32BuildOptionLen != 0)
+			{
+				DIPrintf(psEntry, "%s", pszBuildOptions);
+			}
+			DIPrintf(psEntry, "\n");
+#endif
+
+#endif /* SUPPORT_RGX */
+		}
+
+		if (!bFwVersionInfoPrinted)
+		{
+			DIPrintf(psEntry, "Firmware Version: Info unavailable %s\n",
+#ifdef NO_HARDWARE
+			         "on NoHW driver"
+#else /* NO_HARDWARE */
+			         "(Is INIT complete?)"
+#endif /* NO_HARDWARE */
+			         );
+		}
+	}
+
+	return 0;
+}
+
+#if defined(SUPPORT_RGX) && defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+
+/*************************************************************************/ /*!
+ Power data DebugFS entry
+*/ /**************************************************************************/
+
+static PVRSRV_ERROR SendPowerCounterCommand(PVRSRV_DEVICE_NODE* psDeviceNode,
+                                            RGXFWIF_COUNTER_DUMP_REQUEST eRequestType,
+                                            IMG_UINT32 *pui32kCCBCommandSlot)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFWIF_KCCB_CMD sCounterDumpCmd;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCounterDumpCmd.eCmdType = RGXFWIF_KCCB_CMD_COUNTER_DUMP;
+	sCounterDumpCmd.uCmdData.sCounterDumpConfigData.eCounterDumpRequest = eRequestType;
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sCounterDumpCmd,
+				PDUMP_FLAGS_CONTINUOUS,
+				pui32kCCBCommandSlot);
+	PVR_LOG_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot");
+
+	return eError;
+}
+
+static int _DebugPowerDataDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	int eError = 0;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "power counter data was requested!"));
+		return -EIO;
+	}
+
+	OSLockAcquire(psDevInfo->hCounterDumpingLock);
+
+	eError = SendPowerCounterCommand(psDeviceNode,
+									 RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+									 &ui32kCCBCommandSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		OSLockRelease(psDevInfo->hCounterDumpingLock);
+		return -EIO;
+	}
+
+	/* Wait for FW complete completion */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo,
+									  ui32kCCBCommandSlot,
+									  PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+		OSLockRelease(psDevInfo->hCounterDumpingLock);
+		return -EIO;
+	}
+
+	/* Read back the buffer */
+	{
+		IMG_UINT32* pui32PowerBuffer;
+		IMG_UINT32 ui32NumOfRegs, ui32SamplePeriod, ui32NumOfCores;
+		IMG_UINT32 i, j;
+
+		if (!psDevInfo->psCounterBufferMemDesc)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Counter buffer not allocated!"));
+			OSLockRelease(psDevInfo->hCounterDumpingLock);
+			return -EINVAL;
+		}
+
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCounterBufferMemDesc,
+										  (void**)&pui32PowerBuffer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "DevmemAcquireCpuVirtAddr");
+			OSLockRelease(psDevInfo->hCounterDumpingLock);
+			return -EIO;
+		}
+
+		RGXFwSharedMemCacheOpExec(pui32PowerBuffer, PAGE_SIZE, PVRSRV_CACHE_OP_INVALIDATE);
+
+		ui32NumOfRegs = *pui32PowerBuffer++;
+		ui32SamplePeriod = *pui32PowerBuffer++;
+		ui32NumOfCores = *pui32PowerBuffer++;
+		PVR_DPF((PVR_DBG_MESSAGE, "Number of power counters: %u.", ui32NumOfRegs));
+
+		if (ui32NumOfCores == 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "No GPU cores enabled!"));
+			eError = -EINVAL;
+		}
+
+		if (ui32NumOfRegs && ui32NumOfCores)
+		{
+			DIPrintf(psEntry, "Power counter data for device\n");
+			DIPrintf(psEntry, "Sample period: 0x%08x\n", ui32SamplePeriod);
+
+			for (i = 0; i < ui32NumOfRegs; i++)
+			{
+				IMG_UINT32 ui32High, ui32Low;
+				IMG_UINT32 ui32RegOffset = *pui32PowerBuffer++;
+				IMG_UINT32 ui32NumOfInstances = *pui32PowerBuffer++;
+
+				PVR_ASSERT(ui32NumOfInstances);
+
+				DIPrintf(psEntry, "0x%08x:", ui32RegOffset);
+
+				for (j = 0; j < ui32NumOfInstances * ui32NumOfCores; j++)
+				{
+					ui32Low = *pui32PowerBuffer++;
+#if defined(RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE_BIT_MASK)
+					if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CATURIX_XTP_TOP_INFRASTRUCTURE))
+					{
+						/* Power counters have 32-bit range */
+						DIPrintf(psEntry, " 0x%08x", ui32Low);
+					}
+					else
+#endif
+					{
+						/* Power counters have 64-bit range */
+						ui32High = *pui32PowerBuffer++;
+
+						DIPrintf(psEntry, " 0x%016" IMG_UINT64_FMTSPECx,
+								 (IMG_UINT64) ui32Low | (IMG_UINT64) ui32High << 32);
+					}
+				}
+
+				DIPrintf(psEntry, "\n");
+			}
+		}
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psCounterBufferMemDesc);
+	}
+
+	OSLockRelease(psDevInfo->hCounterDumpingLock);
+
+	return eError;
+}
+
+static IMG_INT64 PowerDataSet(const IMG_CHAR __user *pcBuffer,
+                              IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                              void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*)pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_COUNTER_DUMP_REQUEST eRequest;
+	IMG_UINT32 ui32kCCBCommandSlot;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "power counter data was requested!"));
+		return -EIO;
+	}
+
+	if (pcBuffer[0] == '1')
+	{
+		eRequest = RGXFWIF_PWR_COUNTER_DUMP_START;
+	}
+	else if (pcBuffer[0] == '0')
+	{
+		eRequest = RGXFWIF_PWR_COUNTER_DUMP_STOP;
+	}
+	else
+	{
+		return -EINVAL;
+	}
+
+	OSLockAcquire(psDevInfo->hCounterDumpingLock);
+
+	SendPowerCounterCommand(psDeviceNode,
+	                        eRequest,
+	                        &ui32kCCBCommandSlot);
+
+	OSLockRelease(psDevInfo->hCounterDumpingLock);
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+#endif /* defined(SUPPORT_RGX) && defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+/*************************************************************************/ /*!
+ Status DebugFS entry
+*/ /**************************************************************************/
+
+static void *_DebugStatusCompare_AnyVaCb(PVRSRV_DEVICE_NODE *psDevNode,
+										 va_list va)
+{
+	IMG_UINT64 *pui64CurrentPosition = va_arg(va, IMG_UINT64 *);
+	IMG_UINT64 ui64Position = va_arg(va, IMG_UINT64);
+	IMG_UINT64 ui64CurrentPosition = *pui64CurrentPosition;
+
+	(*pui64CurrentPosition)++;
+
+	return (ui64CurrentPosition == ui64Position) ? psDevNode : NULL;
+}
+
+static void *_DebugStatusDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =  List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+										  _DebugStatusCompare_AnyVaCb,
+										  &uiCurrentPosition,
+										  *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+static void _DebugStatusDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+}
+
+static void *_DebugStatusDINext(OSDI_IMPL_ENTRY *psEntry,
+								 void *pvData,
+								 IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	(*pui64Pos)++;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =  List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+										  _DebugStatusCompare_AnyVaCb,
+										  &uiCurrentPosition,
+										  *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+#ifdef SUPPORT_RGX
+	if (psDeviceNode && !PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+		if (psDevInfo && psDevInfo->pfnGetGpuUtilStats)
+		{
+			PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+			PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+			if (eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+			{
+				PVRSRV_ERROR eError;
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+				static IMG_BOOL bFirstTime = IMG_TRUE;
+#endif
+
+				OSLockAcquire(psDevInfo->hGpuUtilStatsLock);
+
+				eError = psDevInfo->pfnGetGpuUtilStats(psDeviceNode,
+													   psDebugInfo->hGpuUtilUserDebugFS,
+													   &psDevInfo->sGpuUtilStats);
+
+				OSLockRelease(psDevInfo->hGpuUtilStatsLock);
+
+				if (eError != PVRSRV_OK)
+				{
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+					if (bFirstTime)
+					{
+						bFirstTime = IMG_FALSE;
+#endif	/* defined(EMULATOR) || defined(VIRTUAL_PLATFORM) */
+					PVR_DPF((PVR_DBG_ERROR,
+					        "%s: Failed to get GPU statistics (%s)",
+					        __func__, PVRSRVGetErrorString(eError)));
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+					}
+#endif	/* defined(EMULATOR) || defined(VIRTUAL_PLATFORM) */
+				}
+			}
+		}
+	}
+#endif
+
+	return psDeviceNode;
+}
+
+static int _DebugStatusDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	if (pvData == DI_START_TOKEN)
+	{
+		PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+
+		if (psPVRSRVData != NULL)
+		{
+			switch (psPVRSRVData->eServicesState)
+			{
+				case PVRSRV_SERVICES_STATE_OK:
+					DIPrintf(psEntry, "Driver Status:   OK\n");
+					break;
+				case PVRSRV_SERVICES_STATE_BAD:
+					DIPrintf(psEntry, "Driver Status:   BAD\n");
+					break;
+				case PVRSRV_SERVICES_STATE_UNDEFINED:
+					DIPrintf(psEntry, "Driver Status:   UNDEFINED\n");
+					break;
+				default:
+					DIPrintf(psEntry, "Driver Status:   UNKNOWN (%d)\n",
+					         psPVRSRVData->eServicesState);
+					break;
+			}
+		}
+	}
+	else if (pvData != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+		IMG_CHAR           *pszStatus = "";
+		IMG_CHAR           *pszReason = "";
+		PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus;
+		PVRSRV_DEVICE_HEALTH_REASON eHealthReason;
+
+		DIPrintf(psEntry, "\nDevice ID: %u:%d\n", psDeviceNode->sDevId.ui32InternalID,
+		                                          psDeviceNode->sDevId.i32KernelDeviceID);
+
+		/* Update the health status now if possible... */
+		if (psDeviceNode->pfnUpdateHealthStatus)
+		{
+			psDeviceNode->pfnUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+		}
+		eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+		eHealthReason = OSAtomicRead(&psDeviceNode->eHealthReason);
+
+		switch (eHealthStatus)
+		{
+			case PVRSRV_DEVICE_HEALTH_STATUS_OK:  pszStatus = "OK";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:  pszStatus = "NOT RESPONDING";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:  pszStatus = "DEAD";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:  pszStatus = "FAULT";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:  pszStatus = "UNDEFINED";  break;
+			default:  pszStatus = "UNKNOWN";  break;
+		}
+
+		switch (eHealthReason)
+		{
+			case PVRSRV_DEVICE_HEALTH_REASON_NONE:  pszReason = "";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_ASSERTED:  pszReason = " (Asserted)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING:  pszReason = " (Poll failing)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS:  pszReason = " (Global Event Object timeouts rising)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT:  pszReason = " (KCCB offset invalid)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED:  pszReason = " (KCCB stalled)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_IDLING:  pszReason = " (Idling)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_RESTARTING:  pszReason = " (Restarting)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS:  pszReason = " (Missing interrupts)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR:  pszReason = " (PCI error)";  break;
+			default:  pszReason = " (Unknown reason)";  break;
+		}
+
+		DIPrintf(psEntry, "Firmware Status: %s%s\n", pszStatus, pszReason);
+		if (PVRSRV_ERROR_LIMIT_REACHED)
+		{
+			DIPrintf(psEntry, "Server Errors:   %d+\n", IMG_UINT32_MAX);
+		}
+		else
+		{
+			DIPrintf(psEntry, "Server Errors:   %d\n", PVRSRV_KM_ERRORS);
+		}
+
+
+		/* Write other useful stats to aid the test cycle... */
+		if (psDeviceNode->pvDevice != NULL)
+		{
+#ifdef SUPPORT_RGX
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+			const RGXFWIF_HWRINFOBUF *psHWRInfoBuf;
+			const RGXFWIF_SYSDATA *psFwSysData;
+
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+			psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+			psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+#ifdef PVRSRV_DEBUG_LISR_EXECUTION
+			/* Show the detected #LISR, #MISR scheduled calls */
+			DIPrintf(psEntry, "RGX #LISR: %" IMG_UINT64_FMTSPEC "\n", psDeviceNode->ui64nLISR);
+			DIPrintf(psEntry, "RGX #MISR: %" IMG_UINT64_FMTSPEC "\n", psDeviceNode->ui64nMISR);
+#endif /* PVRSRV_DEBUG_LISR_EXECUTION */
+
+			/* Calculate the number of HWR events in total across all the DMs... */
+			if (psHWRInfoBuf != NULL)
+			{
+				IMG_UINT32 ui32HWREventCount = 0;
+				IMG_UINT32 ui32CRREventCount = 0;
+				IMG_UINT32 ui32DMIndex;
+
+				for (ui32DMIndex = 0; ui32DMIndex < RGXFWIF_DM_MAX; ui32DMIndex++)
+				{
+					ui32HWREventCount += psHWRInfoBuf->aui32HwrDmLockedUpCount[ui32DMIndex];
+					ui32CRREventCount += psHWRInfoBuf->aui32HwrDmOverranCount[ui32DMIndex];
+				}
+
+				DIPrintf(psEntry, "HWR Event Count: %d\n", ui32HWREventCount);
+				DIPrintf(psEntry, "CRR Event Count: %d\n", ui32CRREventCount);
+#ifdef PVRSRV_STALLED_CCB_ACTION
+				/* Write the number of Sync Lockup Recovery (SLR) events... */
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested,
+				                           INVALIDATE);
+				DIPrintf(psEntry, "SLR Event Count: %d\n", psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested);
+#endif /* PVRSRV_STALLED_CCB_ACTION */
+			}
+
+			/* Show error counts */
+			DIPrintf(psEntry, "WGP Error Count: %d\n", psDevInfo->sErrorCounts.ui32WGPErrorCount);
+			DIPrintf(psEntry, "TRP Error Count: %d\n", psDevInfo->sErrorCounts.ui32TRPErrorCount);
+
+			/*
+			 * Guest drivers do not support the following functionality:
+			 *	- Perform actual on-chip fw tracing.
+			 *	- Collect actual on-chip GPU utilization stats.
+			 *	- Perform actual on-chip GPU power/dvfs management.
+			 *	- As a result no more information can be provided.
+			 */
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				if (psFwSysData != NULL)
+				{
+					DIPrintf(psEntry, "FWF Event Count: %d\n", psFwSysData->ui32FWFaults);
+				}
+
+				/* Write the number of APM events... */
+				DIPrintf(psEntry, "APM Event Count: %d\n", psDevInfo->ui32ActivePMReqTotal);
+
+				/* Write the current GPU Utilisation values... */
+				if (psDevInfo->pfnGetGpuUtilStats &&
+					eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+				{
+					PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+					RGXFWIF_GPU_UTIL_STATS *psGpuUtilStats = &psDevInfo->sGpuUtilStats;
+
+					OSLockAcquire(psDevInfo->hGpuUtilStatsLock);
+
+					if ((IMG_UINT32)psGpuUtilStats->ui64GpuStatCumulative)
+					{
+						const IMG_CHAR *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+						IMG_UINT64 util;
+						IMG_UINT32 rem;
+						IMG_UINT32 ui32DriverID;
+						RGXFWIF_DM eDM;
+						IMG_INT    iDM_Util = 0;
+
+						if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+						{
+							apszDmNames[RGXFWIF_DM_TDM] = "2D";
+						}
+
+						util = 100 * psGpuUtilStats->ui64GpuStatActive;
+						util = OSDivide64(util, (IMG_UINT32)psGpuUtilStats->ui64GpuStatCumulative, &rem);
+
+						DIPrintf(psEntry, "GPU Utilisation: %u%%\n", (IMG_UINT32)util);
+
+						DIPrintf(psEntry, "DM Utilisation:");
+
+						FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+						{
+							DIPrintf(psEntry, "  VM%u", ui32DriverID);
+						}
+
+						DIPrintf(psEntry, "\n");
+
+						for (eDM = RGXFWIF_DM_TDM; eDM < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; eDM++,iDM_Util++)
+						{
+							DIPrintf(psEntry, "        %5s: ", apszDmNames[eDM]);
+
+							FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+							{
+								IMG_UINT32 uiDivisor = (IMG_UINT32)psGpuUtilStats->aaui64DMOSStatCumulative[iDM_Util][ui32DriverID];
+
+								if (uiDivisor == 0U)
+								{
+									DIPrintf(psEntry, "   - ");
+									continue;
+								}
+
+								util = 100 * psGpuUtilStats->aaui64DMOSStatActive[iDM_Util][ui32DriverID];
+								util = OSDivide64(util, uiDivisor, &rem);
+
+								DIPrintf(psEntry, "%3u%% ", (IMG_UINT32)util);
+							}
+
+
+							DIPrintf(psEntry, "\n");
+						}
+					}
+					else
+					{
+						DIPrintf(psEntry, "GPU Utilisation: -\n");
+					}
+
+					OSLockRelease(psDevInfo->hGpuUtilStatsLock);
+
+				}
+			}
+#endif /* SUPPORT_RGX */
+		}
+	}
+
+	return 0;
+}
+
+#if defined(DEBUG)
+static IMG_INT64 DebugStatusSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[0] == 'k' || pcBuffer[0] == 'K', -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	psPVRSRVData->eServicesState = PVRSRV_SERVICES_STATE_BAD;
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+#endif
+
+const IMG_CHAR *PVRSRVGetDebugDevStateString(PVRSRV_DEVICE_STATE eDevState)
+{
+	static const char *const _pszDeviceStateStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_STATE_LIST
+	#undef X
+	};
+
+	if (eDevState < 0 || eDevState >= PVRSRV_DEVICE_STATE_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceStateStrings[eDevState];
+}
+
+
+const IMG_CHAR *PVRSRVGetDebugHealthStatusString(PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus)
+{
+	static const char *const _pszDeviceHealthStatusStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_HEALTH_STATUS_LIST
+	#undef X
+	};
+
+	if (eHealthStatus < 0 || eHealthStatus >= PVRSRV_DEVICE_HEALTH_STATUS_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceHealthStatusStrings[eHealthStatus];
+}
+
+
+const IMG_CHAR *PVRSRVGetDebugHealthReasonString(PVRSRV_DEVICE_HEALTH_REASON eHealthReason)
+{
+	static const char *const _pszDeviceHealthReasonStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_HEALTH_REASON_LIST
+	#undef X
+	};
+
+	if (eHealthReason < 0 || eHealthReason >= PVRSRV_DEVICE_HEALTH_REASON_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceHealthReasonStrings[eHealthReason];
+}
+
+/*************************************************************************/ /*!
+ Dump Debug DebugFS entry
+*/ /**************************************************************************/
+
+static int _DebugDumpDebugDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDeviceNode->pvDevice != NULL)
+	{
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX,
+		                   _DumpDebugDIPrintfWrapper, psEntry);
+	}
+
+	return 0;
+}
+
+#ifdef SUPPORT_RGX
+
+/*************************************************************************/ /*!
+ Firmware Trace DebugFS entry
+*/ /**************************************************************************/
+
+static int _DebugFWTraceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDevInfo != NULL)
+	{
+		RGXDumpFirmwareTrace(_DumpDebugDIPrintfWrapper, psEntry, psDevInfo);
+	}
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+ Firmware Translated Page Tables DebugFS entry
+*/ /**************************************************************************/
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+static int _FirmwareMappingsDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32FwVA;
+	IMG_UINT32 ui32FwPageSize;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	psDeviceNode = DIGetPrivData(psEntry);
+
+	if ((psDeviceNode == NULL) ||
+	    (psDeviceNode->pvDevice == NULL) ||
+	    (((PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice)->psKernelMMUCtx == NULL))
+	{
+		/* The Kernel MMU context containing the Firmware mappings is not initialised */
+		return 0;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	DIPrintf(psEntry, "+-----------------+------------------------+------------------------+--------------+\n"
+					  "|    Firmware     |           CPU          |         Device         |      PTE     |\n"
+					  "| Virtual Address |    Physical Address    |    Physical Address    |     Flags    |\n"
+					  "+-----------------+------------------------+------------------------+              +\n");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		DIPrintf(psEntry, "|                                               RI/XI = Read / Execution Inhibit   |\n"
+		                  "|                                               C     = Cache Coherent             |\n"
+		                  "|                                               D     = Dirty Page Table Entry     |\n"
+		                  "|                                               V     = Valid Page Table Entry     |\n"
+		                  "|                                               G     = Global Page Table Entry    |\n"
+		                  "+-----------------+------------------------+------------------------+--------------+\n");
+
+		/* MIPS uses the same page size as the OS */
+		ui32FwPageSize = OSGetPageSize();
+	}
+	else
+#endif
+	{
+		DIPrintf(psEntry, "|                                               P     = Pending Page Table Entry   |\n"
+		                  "|                                               PM    = Parameter Manager Source   |\n"
+		                  "|                                               B     = Bypass SLC                 |\n"
+		                  "|                                               C     = Cache Coherent             |\n"
+		                  "|                                               RW/RO = Device Access Rights       |\n"
+		                  "|                                               V     = Valid Page Table Entry     |\n"
+		                  "+-----------------+------------------------+------------------------+--------------+\n");
+
+		ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+	}
+
+	FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+	{
+		IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) ((RGX_FIRMWARE_RAW_HEAP_BASE +
+		                             (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE)) & UINT_MAX);
+		IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_SIZE & UINT_MAX);
+
+		DIPrintf(psEntry, "|                                       OS ID %u                                    |\n"
+						  "+-----------------+------------------------+------------------------+--------------+\n", ui32DriverID);
+
+		for (ui32FwVA = ui32FwHeapBase;
+		     ui32FwVA < ui32FwHeapEnd;
+		     ui32FwVA += ui32FwPageSize)
+		{
+			PVRSRV_ERROR eError;
+			IMG_UINT64 ui64PTE = 0U;
+			IMG_CPU_PHYADDR sCpuPA = {0U};
+			IMG_DEV_PHYADDR sDevPA = {0U};
+
+			eError = RGXGetFwMapping(psDevInfo, ui32FwVA, &sCpuPA, &sDevPA, &ui64PTE);
+
+			if (eError == PVRSRV_OK)
+			{
+				RGXDocumentFwMapping(psDevInfo, _DumpDebugDIPrintfWrapper, psEntry,
+				                     ui32FwVA, sCpuPA, sDevPA, ui64PTE);
+			}
+			else if (eError != PVRSRV_ERROR_DEVICEMEM_NO_MAPPING)
+			{
+				PVR_LOG_ERROR(eError, "RGXGetFwMapping");
+				return -EIO;
+			}
+		}
+
+		DIPrintf(psEntry, "+-----------------+------------------------+------------------------+--------------+\n");
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+		{
+			break;
+		}
+#endif
+	}
+
+	return 0;
+}
+#endif
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+
+static void *_FirmwareGcovDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo != NULL)
+	{
+		if (psDevInfo->psFirmwareGcovBufferMemDesc != NULL)
+		{
+			void *pvCpuVirtAddr;
+			DevmemAcquireCpuVirtAddr(psDevInfo->psFirmwareGcovBufferMemDesc, &pvCpuVirtAddr);
+			return *pui64Pos ? NULL : pvCpuVirtAddr;
+		}
+	}
+
+	return NULL;
+}
+
+static void _FirmwareGcovDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDevInfo != NULL)
+	{
+		if (psDevInfo->psFirmwareGcovBufferMemDesc != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psFirmwareGcovBufferMemDesc);
+		}
+	}
+}
+
+static void *_FirmwareGcovDINext(OSDI_IMPL_ENTRY *psEntry,
+								  void *pvData,
+								  IMG_UINT64 *pui64Pos)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(pui64Pos);
+	return NULL;
+}
+
+static int _FirmwareGcovDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo != NULL)
+	{
+		DIWrite(psEntry, pvData, psDevInfo->ui32FirmwareGcovSize);
+	}
+	return 0;
+}
+
+#endif /* SUPPORT_FIRMWARE_GCOV */
+
+
+#if  defined(SUPPORT_RISCV_GDB)
+#define RISCV_DMI_SIZE  (8U)
+
+static IMG_INT64 _RiscvDmiRead(IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+
+	ui64Count = MIN(RISCV_DMI_SIZE, ui64Count);
+	memcpy(pcBuffer, &psDebugInfo->ui64RiscvDmi, ui64Count);
+
+	return ui64Count;
+}
+
+static IMG_INT64 _RiscvDmiWrite(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: devinfo is NULL", __func__));
+		return 0;
+	}
+
+	ui64Count -= 1; /* Drop `\0` */
+	ui64Count = MIN(RISCV_DMI_SIZE, ui64Count);
+
+	memcpy(&psDebugInfo->ui64RiscvDmi, pcBuffer, ui64Count);
+
+	RGXRiscvDmiOp(psDevInfo, &psDebugInfo->ui64RiscvDmi);
+
+	return ui64Count;
+}
+#endif
+
+#endif /* SUPPORT_RGX */
+
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+
+/*************************************************************************/ /*!
+ Debug level DebugFS entry
+*/ /**************************************************************************/
+
+static int DebugLevelDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DIPrintf(psEntry, "%u\n", OSDebugLevel());
+
+	return 0;
+}
+
+#ifndef __GNUC__
+static int __builtin_ffsl(long int x)
+{
+	for (size_t i = 0; i < sizeof(x) * 8; i++)
+	{
+		if (x & (1 << i))
+		{
+			return i + 1;
+		}
+	}
+	return 0;
+}
+#endif /* __GNUC__ */
+
+static IMG_INT64 DebugLevelSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const IMG_UINT uiMaxBufferSize = 6;
+	IMG_UINT32 ui32Level;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	if (sscanf(pcBuffer, "%u", &ui32Level) == 0)
+	{
+		return -EINVAL;
+	}
+
+	OSSetDebugLevel(ui32Level & ((1 << __builtin_ffsl(DBGPRIV_LAST)) - 1));
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static int VZPriorityDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+	                    -EINVAL);
+
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ai32DriverPriority[ui32DriverID], INVALIDATE);
+	DIPrintf(psEntry, "%u\n", psRuntimeCfg->ai32DriverPriority[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZPrioritySet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32Priority;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32Priority) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverPriorityKM(NULL, psVZDriverData->psDevNode,
+	                                             psVZDriverData->ui32DriverID, ui32Priority);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZTimeSliceIntervalDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	DIPrintf(psEntry, "%u ms (0: disable)\n", psRuntimeCfg->ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT64 VZTimeSliceIntervalSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+										IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32TSIntervalMs;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32TSIntervalMs) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(NULL, psVZDriverData->psDevNode,
+														  ui32TSIntervalMs);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZTimeSliceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+						-EINVAL);
+
+	DIPrintf(psEntry, "%u (0: auto; 1%% to 100%%)\n", psRuntimeCfg->aui32TSPercentage[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZTimeSliceSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+								IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32TSPercentage;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32TSPercentage) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverTimeSliceKM(NULL, psVZDriverData->psDevNode,
+												  psVZDriverData->ui32DriverID, ui32TSPercentage);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZIsolationGroupDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+	                    -EINVAL);
+
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID], INVALIDATE);
+	DIPrintf(psEntry, "%u\n", psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZIsolationGroupSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                     IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32IsolationGroup;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32IsolationGroup) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverIsolationGroupKM(NULL, psVZDriverData->psDevNode,
+	                                                   psVZDriverData->ui32DriverID, ui32IsolationGroup);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZConnectionCooldownPeriodDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	DIPrintf(psEntry, "%u sec\n", psRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+	return 0;
+}
+
+static IMG_INT64 VZConnectionCooldownPeriodSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+										IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32VzConnectionCooldownPeriodInSec;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32VzConnectionCooldownPeriodInSec) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(NULL, psVZDriverData->psDevNode,
+														  ui32VzConnectionCooldownPeriodInSec);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+#endif
+
+PVRSRV_ERROR DebugCommonInitDriver(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPVRSRVData != NULL);
+
+	/*
+	 * The DebugFS entries are designed to work in a single device system but
+	 * this function will be called multiple times in a multi-device system.
+	 * Return an error in this case.
+	 */
+	if (gpsVersionDIEntry)
+	{
+		return -EEXIST;
+	}
+
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnStart = _VersionDIStart,
+			.pfnStop = _VersionDIStop,
+			.pfnNext = _VersionDINext,
+			.pfnShow = _VersionDIShow
+		};
+
+		eError = DICreateEntry("version", NULL, &sIterator, psPVRSRVData,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsVersionDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnStart = _DebugStatusDIStart,
+			.pfnStop = _DebugStatusDIStop,
+			.pfnNext = _DebugStatusDINext,
+			.pfnShow = _DebugStatusDIShow,
+#if defined(DEBUG)
+			.pfnWrite = DebugStatusSet,
+			//'K' expected + Null terminator
+#endif
+			.ui32WriteLenMax= ((1U)+1U)
+		};
+		eError = DICreateEntry("status", NULL, &sIterator, psPVRSRVData,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsStatusDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = DebugLevelDIShow,
+			.pfnWrite = DebugLevelSet,
+			//Max value of 255(3 char) + Null terminator
+			.ui32WriteLenMax =((3U)+1U)
+		};
+		eError = DICreateEntry("debug_level", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsDebugLevelDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+	return PVRSRV_OK;
+
+return_error_:
+	DebugCommonDeInitDriver();
+
+	return eError;
+}
+
+void DebugCommonDeInitDriver(void)
+{
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+	if (gpsDebugLevelDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsDebugLevelDIEntry);
+	}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+
+	if (gpsStatusDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsStatusDIEntry);
+	}
+
+	if (gpsVersionDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsVersionDIEntry);
+	}
+}
+
+PVRSRV_ERROR DebugCommonInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszDeviceId[sizeof("gpu4294967296")];
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSSNPrintf(pszDeviceId, sizeof(pszDeviceId), "gpu%02d",
+	           psDeviceNode->sDevId.ui32InternalID);
+	eError = DICreateGroup(pszDeviceId, NULL, &psDebugInfo->psGroup);
+	PVR_GOTO_IF_ERROR(eError, return_error_);
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	eError = SORgxGpuUtilStatsRegister(&psDebugInfo->hGpuUtilUserDebugFS);
+	PVR_GOTO_IF_ERROR(eError, return_error_);
+#endif
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = _DebugDumpDebugDIShow};
+		eError = DICreateEntry("debug_dump", psDebugInfo->psGroup, &sIterator,
+		                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+		                       &psDebugInfo->psDumpDebugEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+#ifdef SUPPORT_RGX
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		{
+			DI_ITERATOR_CB sIterator = {.pfnShow = _DebugFWTraceDIShow};
+			eError = DICreateEntry("firmware_trace", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWTraceEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+		{
+			DI_ITERATOR_CB sIterator = {
+				.pfnStart = _FirmwareGcovDIStart,
+				.pfnStop = _FirmwareGcovDIStop,
+				.pfnNext = _FirmwareGcovDINext,
+				.pfnShow = _FirmwareGcovDIShow
+			};
+
+			eError = DICreateEntry("firmware_gcov", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWGCOVEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+#endif /* SUPPORT_FIRMWARE_GCOV */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+		{
+			DI_ITERATOR_CB sIterator = {.pfnShow = _FirmwareMappingsDIShow};
+			eError = DICreateEntry("firmware_mappings", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWMappingsEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+#endif
+
+#if  defined(SUPPORT_RISCV_GDB)
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			DI_ITERATOR_CB sIterator = {
+				.pfnRead = _RiscvDmiRead,
+				.pfnWrite = _RiscvDmiWrite,
+				.ui32WriteLenMax = ((RISCV_DMI_SIZE)+1U)
+			};
+			eError = DICreateEntry("riscv_dmi", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+			                       DI_ENTRY_TYPE_RANDOM_ACCESS, &psDebugInfo->psRiscvDmiDIEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+			psDebugInfo->ui64RiscvDmi = 0ULL;
+		}
+#endif /* SUPPORT_VALIDATION || SUPPORT_RISCV_GDB */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+		{
+			eError = DICreateGroup("vz", psDebugInfo->psGroup, &psDebugInfo->psVZGroup);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+
+			{
+				IMG_UINT32 ui32DriverID;
+
+				DI_ITERATOR_CB sPriorityIterator = {
+					.pfnShow = VZPriorityDIShow,
+					.pfnWrite = VZPrioritySet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sTimeSliceIntervalIterator = {
+					.pfnShow = VZTimeSliceIntervalDIShow,
+					.pfnWrite = VZTimeSliceIntervalSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sTimeSliceIterator = {
+					.pfnShow = VZTimeSliceDIShow,
+					.pfnWrite = VZTimeSliceSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sIsolationGroupIterator = {
+					.pfnShow = VZIsolationGroupDIShow,
+					.pfnWrite = VZIsolationGroupSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sVzConnectionCooldownPeriodIterator = {
+					.pfnShow = VZConnectionCooldownPeriodDIShow,
+					.pfnWrite = VZConnectionCooldownPeriodSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+				{
+					IMG_CHAR szDriverID[4];
+					OSSNPrintf(szDriverID, 4, "%u", ui32DriverID);
+
+					eError = DICreateGroup(szDriverID, psDebugInfo->psVZGroup, &psDebugInfo->apsVZDriverGroups[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					psDebugInfo->apsVZDriverData[ui32DriverID] = OSAllocMem(sizeof(PVRSRV_DEVICE_DEBUG_INFO));
+					PVR_GOTO_IF_NOMEM(psDebugInfo->apsVZDriverData[ui32DriverID], eError, return_error_);
+
+					psDebugInfo->apsVZDriverData[ui32DriverID]->psDevNode = psDeviceNode;
+					psDebugInfo->apsVZDriverData[ui32DriverID]->ui32DriverID = ui32DriverID;
+
+					eError = DICreateEntry("priority", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sPriorityIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					eError = DICreateEntry("time_slice", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sTimeSliceIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+					{
+						eError = DICreateEntry("time_slice_interval", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+											   &sTimeSliceIntervalIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+											   &psDebugInfo->psVZDriverTimeSliceIntervalDIEntry);
+						PVR_GOTO_IF_ERROR(eError, return_error_);
+
+						eError = DICreateEntry("vz_connection_cooldown_period", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+											   &sVzConnectionCooldownPeriodIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+											   &psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry);
+						PVR_GOTO_IF_ERROR(eError, return_error_);
+					}
+
+					eError = DICreateEntry("isolation_group", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sIsolationGroupIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+				}
+			}
+		}
+#endif /* defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1 */
+	}
+
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = _DebugPowerDataDIShow,
+			.pfnWrite = PowerDataSet,
+			//Expects '0' or '1' plus Null terminator
+			.ui32WriteLenMax = ((1U)+1U)
+		};
+		eError = DICreateEntry("power_data", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+		                       DI_ENTRY_TYPE_GENERIC, &psDebugInfo->psPowerDataEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif /* SUPPORT_POWER_SAMPLING_VIA_DEBUGFS */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = PVRSRVPowerStatsPrintElements,
+		};
+		eError = DICreateEntry("power_timing_stats", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+		                       DI_ENTRY_TYPE_GENERIC, &psDebugInfo->psPowerTimingStatsEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif
+#endif /* SUPPORT_RGX */
+
+	return PVRSRV_OK;
+
+return_error_:
+	DebugCommonDeInitDevice(psDeviceNode);
+
+	return eError;
+}
+
+void DebugCommonDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	if (psDebugInfo->psPowerTimingStatsEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psPowerTimingStatsEntry);
+		psDebugInfo->psPowerTimingStatsEntry = NULL;
+	}
+#endif
+
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	if (psDebugInfo->psPowerDataEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psPowerDataEntry);
+		psDebugInfo->psPowerDataEntry = NULL;
+	}
+#endif /* SUPPORT_POWER_SAMPLING_VIA_DEBUGFS */
+
+
+#ifdef SUPPORT_RGX
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			if (psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				if (psDebugInfo->psVZDriverTimeSliceIntervalDIEntry != NULL)
+				{
+					DIDestroyEntry(psDebugInfo->psVZDriverTimeSliceIntervalDIEntry);
+					psDebugInfo->psVZDriverTimeSliceIntervalDIEntry = NULL;
+				}
+
+				if (psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry != NULL)
+				{
+					DIDestroyEntry(psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry);
+					psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry = NULL;
+				}
+			}
+
+			if (psDebugInfo->apsVZDriverData[ui32DriverID] != NULL)
+			{
+				OSFreeMem(psDebugInfo->apsVZDriverData[ui32DriverID]);
+				psDebugInfo->apsVZDriverData[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverGroups[ui32DriverID] != NULL)
+			{
+				DIDestroyGroup(psDebugInfo->apsVZDriverGroups[ui32DriverID]);
+				psDebugInfo->apsVZDriverGroups[ui32DriverID] = NULL;
+			}
+		}
+
+		if (psDebugInfo->psVZGroup != NULL)
+		{
+			DIDestroyGroup(psDebugInfo->psVZGroup);
+			psDebugInfo->psVZGroup = NULL;
+		}
+	}
+#endif
+
+	if (psDebugInfo->psFWTraceEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWTraceEntry);
+		psDebugInfo->psFWTraceEntry = NULL;
+	}
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+	if (psDebugInfo->psFWGCOVEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWGCOVEntry);
+		psDebugInfo->psFWGCOVEntry = NULL;
+	}
+#endif
+
+	if (psDebugInfo->psFWMappingsEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWMappingsEntry);
+		psDebugInfo->psFWMappingsEntry = NULL;
+	}
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR) &&
+		(psDebugInfo->psRiscvDmiDIEntry != NULL))
+	{
+		DIDestroyEntry(psDebugInfo->psRiscvDmiDIEntry);
+		psDebugInfo->psRiscvDmiDIEntry = NULL;
+	}
+#endif
+#endif /* SUPPORT_RGX */
+
+	if (psDebugInfo->psDumpDebugEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psDumpDebugEntry);
+		psDebugInfo->psDumpDebugEntry = NULL;
+	}
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	if (psDebugInfo->hGpuUtilUserDebugFS != NULL)
+	{
+		SORgxGpuUtilStatsUnregister(psDebugInfo->hGpuUtilUserDebugFS);
+		psDebugInfo->hGpuUtilUserDebugFS = NULL;
+	}
+#endif /* defined(SUPPORT_RGX) && !defined(NO_HARDWARE) */
+
+	if (psDebugInfo->psGroup != NULL)
+	{
+		DIDestroyGroup(psDebugInfo->psGroup);
+		psDebugInfo->psGroup = NULL;
+	}
+}
+
+/*
+	Appends flags strings to a null-terminated string buffer
+*/
+void DebugCommonFlagStrings(IMG_CHAR *psDesc,
+							IMG_UINT32 ui32DescSize,
+							const IMG_FLAGS2DESC *psConvTable,
+							IMG_UINT32 ui32TableSize,
+							IMG_UINT32 ui32Flags)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (BITMASK_HAS(ui32Flags, psConvTable[ui32Idx].uiFlag))
+		{
+			OSStringLCat(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_heapcfg.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_heapcfg.c
new file mode 100644
index 000000000000..9b53d026bb5d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_heapcfg.c
@@ -0,0 +1,225 @@
+/*************************************************************************/ /*!
+@File           devicemem_heapcfg.c
+@Title          Device Heap Configuration Helper Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+/* our exported API */
+#include "devicemem_heapcfg.h"
+#include "devicemem_utils.h"
+
+#include "device.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+
+#include "connection_server.h"
+
+static INLINE void _CheckBlueprintHeapAlignment(DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint)
+{
+	IMG_UINT32 ui32OSPageSize = OSGetPageSize();
+
+	/* Any heap length should at least match OS page size at the minimum or
+	 * a multiple of OS page size */
+	if ((psHeapBlueprint->uiHeapLength < DEVMEM_HEAP_MINIMUM_SIZE) ||
+		(psHeapBlueprint->uiHeapLength & (ui32OSPageSize - 1)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC
+		         "("IMG_DEVMEM_SIZE_FMTSPEC")",
+		         __func__,
+		         psHeapBlueprint->pszName,
+		         psHeapBlueprint->uiHeapLength,
+		         psHeapBlueprint->uiHeapLength));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Heap Size should always be at least the DevMem minimum size and a "
+		         "multiple of OS Page Size:%u(0x%x)",
+		         ui32OSPageSize, ui32OSPageSize));
+		PVR_ASSERT(psHeapBlueprint->uiHeapLength >= ui32OSPageSize);
+	}
+
+
+	PVR_ASSERT(psHeapBlueprint->uiReservedRegionLength % DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY == 0);
+}
+
+void HeapCfgBlueprintInit(const IMG_CHAR        *pszName,
+	                      IMG_UINT64             ui64HeapBaseAddr,
+	                      IMG_DEVMEM_SIZE_T      uiHeapLength,
+	                      IMG_DEVMEM_SIZE_T      uiReservedRegionLength,
+	                      IMG_UINT32             ui32Log2DataPageSize,
+	                      IMG_UINT32             uiLog2ImportAlignment,
+	                      PFN_HEAP_INIT          pfnInit,
+	                      PFN_HEAP_DEINIT        pfnDeInit,
+	                      DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint)
+{
+	psHeapBlueprint->pszName                = pszName;
+	psHeapBlueprint->sHeapBaseAddr.uiAddr   = ui64HeapBaseAddr;
+	psHeapBlueprint->uiHeapLength           = uiHeapLength;
+	psHeapBlueprint->uiReservedRegionLength = uiReservedRegionLength;
+	psHeapBlueprint->uiLog2DataPageSize     = ui32Log2DataPageSize;
+	psHeapBlueprint->uiLog2ImportAlignment  = uiLog2ImportAlignment;
+	psHeapBlueprint->pfnInit                = pfnInit;
+	psHeapBlueprint->pfnDeInit              = pfnDeInit;
+
+	_CheckBlueprintHeapAlignment(psHeapBlueprint);
+}
+
+PVRSRV_ERROR
+HeapCfgHeapConfigCount(CONNECTION_DATA * psConnection,
+					   const PVRSRV_DEVICE_NODE *psDeviceNode,
+					   IMG_UINT32 *puiNumHeapConfigsOut)
+{
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	*puiNumHeapConfigsOut = psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapCount(CONNECTION_DATA * psConnection,
+				 const PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_UINT32 uiHeapConfigIndex,
+				 IMG_UINT32 *puiNumHeapsOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	*puiNumHeapsOut = psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapConfigName(CONNECTION_DATA * psConnection,
+					  const PVRSRV_DEVICE_NODE *psDeviceNode,
+					  IMG_UINT32 uiHeapConfigIndex,
+					  IMG_UINT32 uiHeapConfigNameBufSz,
+					  IMG_CHAR *pszHeapConfigNameOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	OSSNPrintf(pszHeapConfigNameOut, uiHeapConfigNameBufSz, "%s", psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].pszName);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgGetCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    PFN_HEAP_INIT *ppfnInit,
+                    PFN_HEAP_DEINIT *ppfnDeinit)
+{
+	DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppfnInit, "ppfnInit");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppfnDeinit, "ppfnDeinit");
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	if (uiHeapIndex >= psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+	}
+
+	psHeapBlueprint = &psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].psHeapBlueprintArray[uiHeapIndex];
+
+	*ppfnInit = psHeapBlueprint->pfnInit;
+	*ppfnDeinit = psHeapBlueprint->pfnDeInit;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapDetails(CONNECTION_DATA * psConnection,
+				   const PVRSRV_DEVICE_NODE *psDeviceNode,
+				   IMG_UINT32 uiHeapConfigIndex,
+				   IMG_UINT32 uiHeapIndex,
+				   IMG_UINT32 uiHeapNameBufSz,
+				   IMG_CHAR *pszHeapNameOut,
+				   IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+				   IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+				   IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+				   IMG_UINT32 *puiLog2DataPageSizeOut,
+				   IMG_UINT32 *puiLog2ImportAlignmentOut)
+{
+	DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	if (uiHeapIndex >= psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+	}
+
+	psHeapBlueprint = &psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].psHeapBlueprintArray[uiHeapIndex];
+
+	OSSNPrintf(pszHeapNameOut, uiHeapNameBufSz, "%s", psHeapBlueprint->pszName);
+	*psDevVAddrBaseOut = psHeapBlueprint->sHeapBaseAddr;
+	*puiHeapLengthOut = psHeapBlueprint->uiHeapLength;
+	*puiReservedRegionLengthOut = psHeapBlueprint->uiReservedRegionLength;
+	*puiLog2DataPageSizeOut = psHeapBlueprint->uiLog2DataPageSize;
+	*puiLog2ImportAlignmentOut = psHeapBlueprint->uiLog2ImportAlignment;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_history_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_history_server.c
new file mode 100644
index 000000000000..f28e96fa55e1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_history_server.c
@@ -0,0 +1,2305 @@
+/*************************************************************************/ /*!
+@File
+@Title          Devicemem history functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Devicemem history functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "allocmem.h"
+#include "img_defs.h"
+#include "pmr.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "pvr_debug.h"
+#include "devicemem_server.h"
+#include "lock.h"
+#include "di_server.h"
+#include "devicemem_history_server.h"
+#include "pdump_km.h"
+
+
+#if (PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES < 5000)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES is too low.
+#elif (PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES > 250000)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES is too high.
+#else
+#define ALLOCATION_LIST_NUM_ENTRIES PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES
+#endif
+
+
+/* data type to hold an allocation index.
+ * we make it 16 bits wide if possible
+ */
+#if ALLOCATION_LIST_NUM_ENTRIES <= 0xFFFF
+typedef uint16_t ALLOC_INDEX_T;
+#else
+typedef uint32_t ALLOC_INDEX_T;
+#endif
+
+/* a record describing a single allocation known to DeviceMemHistory.
+ * this is an element in a doubly linked list of allocations
+ */
+typedef struct _RECORD_ALLOCATION_
+{
+	/* time when this RECORD_ALLOCATION was created/initialised */
+	IMG_UINT64 ui64CreationTime;
+	/* serial number of the PMR relating to this allocation */
+	IMG_UINT64 ui64Serial;
+	/* base DevVAddr of this allocation */
+	IMG_DEV_VIRTADDR sDevVAddr;
+	/* size in bytes of this allocation */
+	IMG_DEVMEM_SIZE_T uiSize;
+	/* Log2 page size of this allocation's GPU pages */
+	IMG_UINT32 ui32Log2PageSize;
+	/* Process ID (PID) this allocation belongs to */
+	IMG_PID uiPID;
+	/* index of previous allocation in the list */
+	ALLOC_INDEX_T ui32Prev;
+	/* index of next allocation in the list */
+	ALLOC_INDEX_T ui32Next;
+	/* annotation/name of this allocation */
+	IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN];
+} RECORD_ALLOCATION;
+
+/* each command in the circular buffer is prefixed with an 8-bit value
+ * denoting the command type
+ */
+typedef enum _COMMAND_TYPE_
+{
+	COMMAND_TYPE_NONE,
+	COMMAND_TYPE_TIMESTAMP,
+	COMMAND_TYPE_MAP_ALL,
+	COMMAND_TYPE_UNMAP_ALL,
+	COMMAND_TYPE_MAP_RANGE,
+	COMMAND_TYPE_UNMAP_RANGE,
+	/* sentinel value */
+	COMMAND_TYPE_COUNT,
+} COMMAND_TYPE;
+
+/* Timestamp command:
+ * This command is inserted into the circular buffer to provide an updated
+ * timestamp.
+ * The nanosecond-accuracy timestamp is packed into a 56-bit integer, in order
+ * for the whole command to fit into 10 bytes.
+ */
+typedef struct __attribute__((__packed__))_COMMAND_TIMESTAMP_
+{
+	IMG_UINT8 aui8TimeNs[7];
+} COMMAND_TIMESTAMP;
+
+/* MAP_ALL command:
+ * This command denotes the allocation at the given index was wholly mapped
+ * in to the GPU MMU
+ */
+typedef struct __attribute__((__packed__))_COMMAND_MAP_ALL_
+{
+	ALLOC_INDEX_T uiAllocIndex;
+} COMMAND_MAP_ALL;
+
+/* UNMAP_ALL command:
+ * This command denotes the allocation at the given index was wholly unmapped
+ * from the GPU MMU
+ * Note: COMMAND_MAP_ALL and COMMAND_UNMAP_ALL commands have the same layout.
+ */
+typedef COMMAND_MAP_ALL COMMAND_UNMAP_ALL;
+
+// This shift allows room for 512GiB virtual memory regions at 4Kb pages.
+#define VM_RANGE_SHIFT 27
+
+/* packing attributes for the MAP_RANGE command */
+#define MAP_RANGE_MAX_START ((1 << VM_RANGE_SHIFT) - 1)
+#define MAP_RANGE_MAX_RANGE ((1 << VM_RANGE_SHIFT) - 1)
+
+/* MAP_RANGE command:
+ * Denotes a range of pages within the given allocation being mapped.
+ * The range is expressed as [Page Index] + [Page Count]
+ * This information is packed into a 72/88-bit struct, in order to make
+ * the command size 9/11 bytes.
+ */
+
+typedef struct __attribute__((__packed__))_COMMAND_MAP_RANGE_
+{
+	IMG_UINT8 aui8Data[7];
+	ALLOC_INDEX_T uiAllocIndex;
+} COMMAND_MAP_RANGE;
+
+/* UNMAP_RANGE command:
+ * Denotes a range of pages within the given allocation being unmapped.
+ * The range is expressed as [Page Index] + [Page Count]
+ * This information is packed into a 72/88-bit struct, in order to make
+ * the command size 9/11 bytes.
+ */
+typedef COMMAND_MAP_RANGE COMMAND_UNMAP_RANGE;
+
+/* wrapper structure for a command */
+typedef struct __attribute__((__packed__))_COMMAND_WRAPPER_
+{
+	IMG_UINT8 ui8Type;
+	union {
+		COMMAND_TIMESTAMP sTimeStamp;
+		COMMAND_MAP_ALL sMapAll;
+		COMMAND_UNMAP_ALL sUnmapAll;
+		COMMAND_MAP_RANGE sMapRange;
+		COMMAND_UNMAP_RANGE sUnmapRange;
+	} u;
+} COMMAND_WRAPPER;
+
+/* target size for the circular buffer of commands */
+#if (PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 < 5)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 is too low.
+#elif (PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 > 18)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 is too high.
+#else
+#define CIRCULAR_BUFFER_SIZE_KB (1 << PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2)
+#endif
+
+
+/* turn the circular buffer target size into a number of commands */
+#define CIRCULAR_BUFFER_NUM_COMMANDS ((CIRCULAR_BUFFER_SIZE_KB * 1024) / sizeof(COMMAND_WRAPPER))
+
+/* index value denoting the end of a list */
+#define END_OF_LIST 0xFFFFFFFF
+#define ALLOC_INDEX_TO_PTR(psDevHData, idx) (&((psDevHData)->sRecords.pasAllocations[idx]))
+#define CHECK_ALLOC_INDEX(idx) (idx < ALLOCATION_LIST_NUM_ENTRIES)
+
+/* wrapper structure for the allocation records and the commands circular buffer */
+typedef struct _RECORDS_
+{
+	RECORD_ALLOCATION *pasAllocations;
+	IMG_UINT32 ui32AllocationsListHead;
+
+	IMG_UINT32 ui32Head;
+	IMG_UINT32 ui32Tail;
+	COMMAND_WRAPPER *pasCircularBuffer;
+	/* Times the CB has wrapped back to start */
+	IMG_UINT64 ui64CBWrapCount;
+	/* Records of CB commands sent */
+	IMG_UINT64 ui64MapAllCount;//Incremented by InsertMapAllCommand()
+	IMG_UINT64 ui64UnMapAllCount;//Incremented by InsertUnmapAllCommand()
+	IMG_UINT64 ui64MapRangeCount;//Incremented by InsertMapRangeCommand()
+	IMG_UINT64 ui64UnMapRangeCount;//Incremented by InsertUnmapRangeCommand()
+	IMG_UINT64 ui64TimeStampCount;//Incremented by InsertTimeStampCommand()
+} RECORDS;
+
+typedef struct _DEVICEMEM_HISTORY_DATA_
+{
+	RECORDS sRecords;
+	POS_LOCK hLock;
+} DEVICEMEM_HISTORY_DATA;
+
+/* Maximum number of device instances supported. This should be DDK global */
+static DEVICEMEM_HISTORY_DATA *gapsDevicememHistoryData[PVRSRV_MAX_DEVICES] = { NULL };
+
+/* DevmemFindDataFromDev
+ *
+ * Return the address of the associated DEVICEMEM_HISTORY_DATA for the given
+ * device. If psDevNode associated unit is out of range we return NULL.
+ */
+static DEVICEMEM_HISTORY_DATA *DevmemFindDataFromDev(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DEVICEMEM_HISTORY_DATA *psDevmemData = NULL;
+
+	IMG_UINT32 uiUnit = psDevNode->sDevId.ui32InternalID;
+
+	PVR_ASSERT(uiUnit < PVRSRV_MAX_DEVICES);
+	if ((uiUnit < PVRSRV_MAX_DEVICES) && (gapsDevicememHistoryData[uiUnit] != NULL))
+	{
+		psDevmemData = gapsDevicememHistoryData[uiUnit];
+	}
+
+	return psDevmemData;
+}
+
+/* gsDevicememHistoryData is static, hLock is NULL unless
+ * EnablePageFaultDebug is set and DevicememHistoryInitKM()
+ * was called.
+ */
+static void DevicememHistoryLock(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	if (psDevHData->hLock)
+	{
+		OSLockAcquire(psDevHData->hLock);
+	}
+}
+
+static void DevicememHistoryUnlock(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	if (psDevHData->hLock)
+	{
+		OSLockRelease(psDevHData->hLock);
+	}
+}
+
+/* given a time stamp, calculate the age in nanoseconds */
+static IMG_UINT64 _CalculateAge(IMG_UINT64 ui64Now,
+						IMG_UINT64 ui64Then,
+						IMG_UINT64 ui64Max)
+{
+	if (ui64Now >= ui64Then)
+	{
+		/* no clock wrap */
+		return ui64Now - ui64Then;
+	}
+	else
+	{
+		/* clock has wrapped */
+		return (ui64Max - ui64Then) + ui64Now + 1;
+	}
+}
+
+/* AcquireCBSlot:
+ * Acquire the next slot in the circular buffer and
+ * move the circular buffer head along by one
+ * Returns a pointer to the acquired slot.
+ */
+static COMMAND_WRAPPER *AcquireCBSlot(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	COMMAND_WRAPPER *psSlot;
+
+	psSlot = &psDevHData->sRecords.pasCircularBuffer[psDevHData->sRecords.ui32Head];
+
+	psDevHData->sRecords.ui32Head =
+		(psDevHData->sRecords.ui32Head + 1)
+				% CIRCULAR_BUFFER_NUM_COMMANDS;
+
+	if (psDevHData->sRecords.ui32Head == 0)
+	{
+		psDevHData->sRecords.ui64CBWrapCount++;
+	}
+
+	return psSlot;
+}
+
+/* TimeStampPack:
+ * Packs the given timestamp value into the COMMAND_TIMESTAMP structure.
+ * This takes a 64-bit nanosecond timestamp and packs it in to a 56-bit
+ * integer in the COMMAND_TIMESTAMP command.
+ */
+static void TimeStampPack(COMMAND_TIMESTAMP *psTimeStamp, IMG_UINT64 ui64Now)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ARRAY_SIZE(psTimeStamp->aui8TimeNs); i++)
+	{
+		psTimeStamp->aui8TimeNs[i] = ui64Now & 0xFF;
+		ui64Now >>= 8;
+	}
+}
+
+/* packing a 64-bit nanosecond into a 7-byte integer loses the
+ * top 8 bits of data. This must be taken into account when
+ * comparing a full timestamp against an unpacked timestamp
+ */
+#define TIME_STAMP_MASK ((1LLU << 56) - 1)
+#define DO_TIME_STAMP_MASK(ns64) (ns64 & TIME_STAMP_MASK)
+
+/* TimeStampUnpack:
+ * Unpack the timestamp value from the given COMMAND_TIMESTAMP command
+ */
+static IMG_UINT64 TimeStampUnpack(COMMAND_TIMESTAMP *psTimeStamp)
+{
+	IMG_UINT64 ui64TimeNs = 0;
+	IMG_UINT32 i;
+
+	for (i = ARRAY_SIZE(psTimeStamp->aui8TimeNs); i > 0; i--)
+	{
+		ui64TimeNs <<= 8;
+		ui64TimeNs |= (IMG_UINT64) psTimeStamp->aui8TimeNs[i - 1];
+	}
+
+	return ui64TimeNs;
+}
+
+#if defined(PDUMP)
+
+static void EmitPDumpAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32AllocationIndex,
+                                RECORD_ALLOCATION *psAlloc)
+{
+	PDUMPCOMMENT(psDeviceNode,
+			"[SrvPFD] Allocation: %u"
+			" Addr: " IMG_DEV_VIRTADDR_FMTSPEC
+			" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+			" Page size: %u"
+			" PID: %u"
+			" Process: %s"
+			" Name: %s",
+			ui32AllocationIndex,
+			psAlloc->sDevVAddr.uiAddr,
+			psAlloc->uiSize,
+			1U << psAlloc->ui32Log2PageSize,
+			psAlloc->uiPID,
+			OSGetCurrentClientProcessNameKM(),
+			psAlloc->szName);
+}
+
+static void EmitPDumpMapUnmapAll(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 COMMAND_TYPE eType,
+                                 IMG_UINT32 ui32AllocationIndex)
+{
+	const IMG_CHAR *pszOpName;
+
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_ALL:
+			pszOpName = "MAP_ALL";
+			break;
+		case COMMAND_TYPE_UNMAP_ALL:
+			pszOpName = "UNMAP_ALL";
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapAll: Invalid type: %u",
+										eType));
+			return;
+
+	}
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "[SrvPFD] Op: %s Allocation: %u",
+	             pszOpName,
+	             ui32AllocationIndex);
+}
+
+static void EmitPDumpMapUnmapRange(PVRSRV_DEVICE_NODE *psDeviceNode,
+					COMMAND_TYPE eType,
+					IMG_UINT32 ui32AllocationIndex,
+					IMG_UINT32 ui32StartPage,
+					IMG_UINT32 ui32Count)
+{
+	const IMG_CHAR *pszOpName;
+
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_RANGE:
+			pszOpName = "MAP_RANGE";
+			break;
+		case COMMAND_TYPE_UNMAP_RANGE:
+			pszOpName = "UNMAP_RANGE";
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapRange: Invalid type: %u",
+										eType));
+			return;
+	}
+
+	PDUMPCOMMENT(psDeviceNode,
+                 "[SrvPFD] Op: %s Allocation: %u Start Page: %u Count: %u",
+                 pszOpName,
+                 ui32AllocationIndex,
+                 ui32StartPage,
+                 ui32Count);
+}
+
+#endif
+
+/* InsertTimeStampCommand:
+ * Insert a timestamp command into the circular buffer.
+ */
+static void InsertTimeStampCommand(IMG_UINT64 ui64Now, PVRSRV_DEVICE_NODE *psDevNode)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDevNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_TIMESTAMP;
+	psDevHData->sRecords.ui64TimeStampCount++;
+	TimeStampPack(&psCommand->u.sTimeStamp, ui64Now);
+}
+
+/* InsertMapAllCommand:
+ * Insert a "MAP_ALL" command for the given allocation into the circular buffer
+ */
+static void InsertMapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32AllocIndex)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_MAP_ALL;
+	psCommand->u.sMapAll.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64MapAllCount++;
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_MAP_ALL, ui32AllocIndex);
+#endif
+}
+
+/* InsertUnmapAllCommand:
+ * Insert a "UNMAP_ALL" command for the given allocation into the circular buffer
+ */
+static void InsertUnmapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32AllocIndex)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_UNMAP_ALL;
+	psCommand->u.sUnmapAll.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64UnMapAllCount++;
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_UNMAP_ALL, ui32AllocIndex);
+#endif
+}
+
+/* MapRangePack:
+ * Pack the given StartPage and Count values into the 75-bit representation
+ * in the MAP_RANGE command.
+ */
+static void MapRangePack(COMMAND_MAP_RANGE *psMapRange,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+
+{
+	IMG_UINT64 ui64Data;
+	IMG_UINT32 i;
+
+	/* we must encode the data into 54 bits:
+	 *   27 bits for the start page index
+	 *   27 bits for the range
+	*/
+
+	PVR_ASSERT(ui32StartPage <= MAP_RANGE_MAX_START);
+	PVR_ASSERT(ui32Count <= MAP_RANGE_MAX_RANGE);
+
+	ui32StartPage &= MAP_RANGE_MAX_START;
+	ui32Count &= MAP_RANGE_MAX_RANGE;
+
+	ui64Data = (((IMG_UINT64) ui32StartPage) << VM_RANGE_SHIFT) | ui32Count;
+
+	for (i = 0; i < ARRAY_SIZE(psMapRange->aui8Data); i++)
+	{
+		psMapRange->aui8Data[i] = ui64Data & 0xFF;
+		ui64Data >>= 8;
+	}
+}
+
+/* MapRangeUnpack:
+ * Unpack the StartPage and Count values from the 75-bit representation
+ * in the MAP_RANGE command.
+ */
+static void MapRangeUnpack(COMMAND_MAP_RANGE *psMapRange,
+						IMG_UINT32 *pui32StartPage,
+						IMG_UINT32 *pui32Count)
+{
+	IMG_UINT64 ui64Data = 0;
+	IMG_UINT32 i;
+
+	for (i = ARRAY_SIZE(psMapRange->aui8Data); i > 0; i--)
+	{
+		ui64Data <<= 8;
+		ui64Data |= (IMG_UINT64) psMapRange->aui8Data[i - 1];
+	}
+
+	*pui32StartPage =  (IMG_UINT32)(ui64Data >> VM_RANGE_SHIFT);
+	*pui32Count = (IMG_UINT32) ui64Data & (MAP_RANGE_MAX_RANGE);
+}
+
+/* InsertMapRangeCommand:
+ * Insert a MAP_RANGE command into the circular buffer with the given
+ * StartPage and Count values.
+ */
+static void InsertMapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32 ui32AllocIndex,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_MAP_RANGE;
+	psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64MapRangeCount++;
+
+	MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapRange(psDeviceNode,
+	                       COMMAND_TYPE_MAP_RANGE,
+	                       ui32AllocIndex,
+	                       ui32StartPage,
+	                       ui32Count);
+#endif
+}
+
+/* InsertUnmapRangeCommand:
+ * Insert a UNMAP_RANGE command into the circular buffer with the given
+ * StartPage and Count values.
+ */
+static void InsertUnmapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32 ui32AllocIndex,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_UNMAP_RANGE;
+	psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64UnMapRangeCount++;
+
+	MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapRange(psDeviceNode,
+	                       COMMAND_TYPE_UNMAP_RANGE,
+	                       ui32AllocIndex,
+	                       ui32StartPage,
+	                       ui32Count);
+#endif
+}
+
+/* InsertAllocationToList:
+ * Helper function for the allocation list.
+ * Inserts the given allocation at the head of the list, whose current head is
+ * pointed to by pui32ListHead
+ */
+static void InsertAllocationToList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
+{
+	RECORD_ALLOCATION *psAlloc;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = IMG_CONTAINER_OF(pui32ListHead, DEVICEMEM_HISTORY_DATA,
+	                              sRecords.ui32AllocationsListHead);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	if (*pui32ListHead == END_OF_LIST)
+	{
+		/* list is currently empty, so just replace it */
+		*pui32ListHead = ui32Alloc;
+		psAlloc->ui32Next = psAlloc->ui32Prev = *pui32ListHead;
+	}
+	else
+	{
+		RECORD_ALLOCATION *psHeadAlloc;
+		RECORD_ALLOCATION *psTailAlloc;
+
+		psHeadAlloc = ALLOC_INDEX_TO_PTR(psDevHData, *pui32ListHead);
+		psTailAlloc = ALLOC_INDEX_TO_PTR(psDevHData, psHeadAlloc->ui32Prev);
+
+		/* make the new alloc point forwards to the previous head */
+		psAlloc->ui32Next = *pui32ListHead;
+		/* make the new alloc point backwards to the previous tail */
+		psAlloc->ui32Prev = psHeadAlloc->ui32Prev;
+
+		/* the head is now our new alloc */
+		*pui32ListHead = ui32Alloc;
+
+		/* the old head now points back to the new head */
+		psHeadAlloc->ui32Prev = *pui32ListHead;
+
+		/* the tail now points forward to the new head */
+		psTailAlloc->ui32Next = ui32Alloc;
+	}
+}
+
+static void InsertAllocationToBusyList(DEVICEMEM_HISTORY_DATA *psDevHData,
+                                       IMG_UINT32 ui32Alloc)
+{
+	InsertAllocationToList(&psDevHData->sRecords.ui32AllocationsListHead, ui32Alloc);
+}
+
+/* RemoveAllocationFromList:
+ * Helper function for the allocation list.
+ * Removes the given allocation from the list, whose head is
+ * pointed to by pui32ListHead
+ */
+static void RemoveAllocationFromList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
+{
+	RECORD_ALLOCATION *psAlloc;
+
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = IMG_CONTAINER_OF(pui32ListHead, DEVICEMEM_HISTORY_DATA,
+	                              sRecords.ui32AllocationsListHead);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	/* if this is the only element in the list then just make the list empty */
+	if ((*pui32ListHead == ui32Alloc) && (psAlloc->ui32Next == ui32Alloc))
+	{
+		*pui32ListHead = END_OF_LIST;
+	}
+	else
+	{
+		RECORD_ALLOCATION *psPrev, *psNext;
+
+		psPrev = ALLOC_INDEX_TO_PTR(psDevHData, psAlloc->ui32Prev);
+		psNext = ALLOC_INDEX_TO_PTR(psDevHData, psAlloc->ui32Next);
+
+		/* remove the allocation from the list */
+		psPrev->ui32Next = psAlloc->ui32Next;
+		psNext->ui32Prev = psAlloc->ui32Prev;
+
+		/* if this allocation is the head then update the head */
+		if (*pui32ListHead == ui32Alloc)
+		{
+			*pui32ListHead = psAlloc->ui32Prev;
+		}
+	}
+}
+
+static void RemoveAllocationFromBusyList(DEVICEMEM_HISTORY_DATA *psDevHData, IMG_UINT32 ui32Alloc)
+{
+	RemoveAllocationFromList(&psDevHData->sRecords.ui32AllocationsListHead, ui32Alloc);
+}
+
+/* TouchBusyAllocation:
+ * Move the given allocation to the head of the list
+ */
+static void TouchBusyAllocation(DEVICEMEM_HISTORY_DATA *psDevHData, IMG_UINT32 ui32Alloc)
+{
+	RemoveAllocationFromBusyList(psDevHData, ui32Alloc);
+	InsertAllocationToBusyList(psDevHData, ui32Alloc);
+}
+
+/* GetOldestBusyAllocation:
+ * Returns the index of the oldest allocation in the MRU list
+ */
+static IMG_UINT32 GetOldestBusyAllocation(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 ui32Alloc;
+	RECORD_ALLOCATION *psAlloc;
+
+	if (psDevHData != NULL)
+	{
+		ui32Alloc = psDevHData->sRecords.ui32AllocationsListHead;
+	}
+	else
+	{
+		ui32Alloc = END_OF_LIST;	/* Default if no psDevHData */
+	}
+
+	if (ui32Alloc == END_OF_LIST)
+	{
+		return END_OF_LIST;
+	}
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	return psAlloc->ui32Prev;
+}
+
+static IMG_UINT32 GetFreeAllocation(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 ui32Alloc;
+
+	ui32Alloc = GetOldestBusyAllocation(psDevHData);
+
+	return ui32Alloc;
+}
+
+
+/* InitialiseAllocation:
+ * Initialise the given allocation structure with the given properties
+ */
+static void InitialiseAllocation(RECORD_ALLOCATION *psAlloc,
+							const IMG_CHAR *pszName,
+							IMG_UINT64 ui64Serial,
+							IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT32 ui32Log2PageSize)
+{
+	OSStringSafeCopy(psAlloc->szName, pszName, sizeof(psAlloc->szName));
+	psAlloc->ui64Serial = ui64Serial;
+	psAlloc->uiPID = uiPID;
+	psAlloc->sDevVAddr = sDevVAddr;
+	psAlloc->uiSize = uiSize;
+	psAlloc->ui32Log2PageSize = ui32Log2PageSize;
+	psAlloc->ui64CreationTime = OSClockns64();
+}
+
+/* CreateAllocation:
+ * Creates a new allocation with the given properties then outputs the
+ * index of the allocation
+ */
+static PVRSRV_ERROR CreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+							const IMG_CHAR *pszName,
+							IMG_UINT64 ui64Serial,
+							IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 *puiAllocationIndex)
+{
+	IMG_UINT32 ui32Alloc;
+	RECORD_ALLOCATION *psAlloc;
+
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	ui32Alloc = GetFreeAllocation(psDevHData);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	InitialiseAllocation(ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc),
+						pszName,
+						ui64Serial,
+						uiPID,
+						sDevVAddr,
+						uiSize,
+						ui32Log2PageSize);
+
+	/* put the newly initialised allocation at the front of the MRU list */
+	TouchBusyAllocation(psDevHData, ui32Alloc);
+
+	*puiAllocationIndex = ui32Alloc;
+
+#if defined(PDUMP)
+	EmitPDumpAllocation(psDeviceNode, ui32Alloc, psAlloc);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/* MatchAllocation:
+ * Tests if the allocation at the given index matches the supplied properties.
+ * Returns IMG_TRUE if it is a match, otherwise IMG_FALSE.
+ */
+static IMG_BOOL MatchAllocation(DEVICEMEM_HISTORY_DATA *psDevHData,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT64 ui64Serial,
+						IMG_DEV_VIRTADDR sDevVAddr,
+						IMG_DEVMEM_SIZE_T uiSize,
+						const IMG_CHAR *pszName,
+						IMG_UINT32 ui32Log2PageSize)
+{
+	RECORD_ALLOCATION *psAlloc;
+
+	if (psDevHData == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocationIndex);
+
+	return (psAlloc->ui64Serial == ui64Serial) &&
+	       (psAlloc->sDevVAddr.uiAddr == sDevVAddr.uiAddr) &&
+	       (psAlloc->uiSize == uiSize) &&
+	       (psAlloc->ui32Log2PageSize == ui32Log2PageSize) &&
+	       (OSStringNCompare(psAlloc->szName, pszName, DEVMEM_ANNOTATION_MAX_LEN) == 0);
+}
+
+/* FindOrCreateAllocation:
+ * Convenience function.
+ * Given a set of allocation properties (serial, DevVAddr, size, name, etc),
+ * this function will look for an existing record of this allocation and
+ * create the allocation if there is no existing record
+ */
+static PVRSRV_ERROR FindOrCreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_UINT32 ui32AllocationIndexHint,
+							IMG_UINT64 ui64Serial,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char *pszName,
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_PID uiPID,
+							IMG_UINT32 *pui32AllocationIndexOut,
+							IMG_BOOL *pbCreated)
+{
+	IMG_UINT32 ui32AllocationIndex;
+	PVRSRV_ERROR eError;
+
+	if (ui32AllocationIndexHint != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE)
+	{
+		IMG_BOOL bHaveAllocation;
+
+		/* first, try to match against the index given by the client.
+		 * if the caller provided a hint but the allocation record is no longer
+		 * there, it must have been purged, so go ahead and create a new allocation
+		 */
+		bHaveAllocation = MatchAllocation(DevmemFindDataFromDev(psDeviceNode),
+								ui32AllocationIndexHint,
+								ui64Serial,
+								sDevVAddr,
+								uiSize,
+								pszName,
+								ui32Log2PageSize);
+		if (bHaveAllocation)
+		{
+			*pbCreated = IMG_FALSE;
+			*pui32AllocationIndexOut = ui32AllocationIndexHint;
+			return PVRSRV_OK;
+		}
+	}
+
+	/* if there is no record of the allocation then we
+	 * create it now
+	 */
+	eError = CreateAllocation(psDeviceNode,
+					pszName,
+					ui64Serial,
+					uiPID,
+					sDevVAddr,
+					uiSize,
+					ui32Log2PageSize,
+					&ui32AllocationIndex);
+
+	if (eError == PVRSRV_OK)
+	{
+		*pui32AllocationIndexOut = ui32AllocationIndex;
+		*pbCreated = IMG_TRUE;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed to create record for allocation %s",
+								__func__,
+								pszName));
+	}
+
+	return eError;
+}
+
+/* GenerateMapUnmapCommandsForSparsePMR:
+ * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the PMR's
+ * current mapping table
+ *
+ * PMR: The PMR whose mapping table to read.
+ * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
+ * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
+ *
+ * This function goes through every page in the PMR's mapping table and looks for
+ * virtually contiguous ranges to record as being mapped or unmapped.
+ */
+static void GenerateMapUnmapCommandsForSparsePMR(PMR *psPMR,
+							IMG_UINT32 ui32AllocIndex,
+							IMG_BOOL bMap)
+{
+	PMR_MAPPING_TABLE *psMappingTable;
+	IMG_UINT32 ui32DonePages = 0;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 i;
+	IMG_BOOL bInARun = IMG_FALSE;
+	IMG_UINT32 ui32CurrentStart = 0;
+	IMG_UINT32 ui32RunCount = 0;
+
+	psMappingTable = PMR_GetMappingTable(psPMR);
+	ui32NumPages = psMappingTable->ui32NumPhysChunks;
+
+	if (ui32NumPages == 0)
+	{
+		/* nothing to do */
+		return;
+	}
+
+	for (i = 0; i < psMappingTable->ui32NumLogicalChunks; i++)
+	{
+		if (psMappingTable->aui32Translation[i] != TRANSLATION_INVALID)
+		{
+			if (!bInARun)
+			{
+				bInARun = IMG_TRUE;
+				ui32CurrentStart = i;
+				ui32RunCount = 1;
+			}
+			else
+			{
+				ui32RunCount++;
+			}
+		}
+
+		if (bInARun)
+		{
+			/* test if we need to end this current run and generate the command,
+			 * either because the next page is not virtually contiguous
+			 * to the current page, we have reached the maximum range,
+			 * or this is the last page in the mapping table
+			 */
+			if ((psMappingTable->aui32Translation[i] == TRANSLATION_INVALID) ||
+				(ui32RunCount == MAP_RANGE_MAX_RANGE) ||
+				(i == (psMappingTable->ui32NumLogicalChunks - 1)))
+			{
+				if (bMap)
+				{
+					InsertMapRangeCommand(PMR_DeviceNode(psPMR),
+					                      ui32AllocIndex,
+					                      ui32CurrentStart,
+					                      ui32RunCount);
+				}
+				else
+				{
+					InsertUnmapRangeCommand(PMR_DeviceNode(psPMR),
+					                        ui32AllocIndex,
+					                        ui32CurrentStart,
+					                        ui32RunCount);
+				}
+
+				ui32DonePages += ui32RunCount;
+
+				if (ui32DonePages == ui32NumPages)
+				{
+					break;
+				}
+
+				bInARun = IMG_FALSE;
+			}
+		}
+	}
+
+}
+
+/* GenerateMapUnmapCommandsForChangeList:
+ * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the
+ * list of page change (page map or page unmap) indices given.
+ *
+ * ui32NumPages: Number of pages which have changed.
+ * pui32PageList: List of indices of the pages which have changed.
+ * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
+ * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
+ *
+ * This function goes through every page in the list and looks for
+ * virtually contiguous ranges to record as being mapped or unmapped.
+ */
+static void GenerateMapUnmapCommandsForChangeList(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_UINT32 ui32NumPages,
+							IMG_UINT32 *pui32PageList,
+							IMG_UINT32 ui32AllocIndex,
+							IMG_BOOL bMap)
+{
+	IMG_UINT32 i;
+	IMG_BOOL bInARun = IMG_FALSE;
+	IMG_UINT32 ui32CurrentStart = 0;
+	IMG_UINT32 ui32RunCount = 0;
+
+	for (i = 0; i < ui32NumPages; i++)
+	{
+		if (!bInARun)
+		{
+			bInARun = IMG_TRUE;
+			ui32CurrentStart = pui32PageList[i];
+		}
+
+		ui32RunCount++;
+
+		 /* we flush if:
+		 * - the next page in the list is not one greater than the current page
+		 * - this is the last page in the list
+		 * - we have reached the maximum range size
+		 */
+		if ((i == (ui32NumPages - 1)) ||
+			((pui32PageList[i] + 1) != pui32PageList[i + 1]) ||
+			(ui32RunCount == MAP_RANGE_MAX_RANGE))
+		{
+			if (bMap)
+			{
+				InsertMapRangeCommand(psDeviceNode,
+									ui32AllocIndex,
+									ui32CurrentStart,
+									ui32RunCount);
+			}
+			else
+			{
+				InsertUnmapRangeCommand(psDeviceNode,
+									ui32AllocIndex,
+									ui32CurrentStart,
+									ui32RunCount);
+			}
+
+			bInARun = IMG_FALSE;
+			ui32RunCount = 0;
+		}
+	}
+}
+
+/* DevicememHistoryMapKM:
+ * Entry point for when an allocation is mapped into the MMU GPU
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistoryMapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_BOOL bSparse = PMR_IsSparse(psPMR);
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	if (!bSparse)
+	{
+		InsertMapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
+	}
+	else
+	{
+		GenerateMapUnmapCommandsForSparsePMR(psPMR,
+								ui32AllocationIndex,
+								IMG_TRUE);
+	}
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+static void VRangeInsertMapUnmapCommands(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_BOOL bMap,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							const IMG_CHAR *pszName)
+{
+	PVR_UNREFERENCED_PARAMETER(pszName);
+
+	while (ui32NumPages > 0)
+	{
+		IMG_UINT32 ui32PagesToAdd;
+
+		ui32PagesToAdd = MIN(ui32NumPages, MAP_RANGE_MAX_RANGE);
+
+		if (ui32StartPage > MAP_RANGE_MAX_START)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Cannot record %s range beginning at page "
+									"%u on allocation %s",
+									bMap ? "map" : "unmap",
+									ui32StartPage,
+									pszName));
+			return;
+		}
+
+		if (bMap)
+		{
+			InsertMapRangeCommand(psDeviceNode,
+								ui32AllocationIndex,
+								ui32StartPage,
+								ui32PagesToAdd);
+		}
+		else
+		{
+			InsertUnmapRangeCommand(psDeviceNode,
+								ui32AllocationIndex,
+								ui32StartPage,
+								ui32PagesToAdd);
+		}
+
+		ui32StartPage += ui32PagesToAdd;
+		ui32NumPages -= ui32PagesToAdd;
+	}
+}
+
+PVRSRV_ERROR DevicememHistoryMapVRangeKM(CONNECTION_DATA *psConnection,
+						PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_DEV_VIRTADDR sBaseDevVAddr,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32NumPages,
+						IMG_DEVMEM_SIZE_T uiAllocSize,
+						const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+						IMG_UINT32 ui32Log2PageSize,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+							ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(psDeviceNode,
+						ui32AllocationIndex,
+						0,
+						sBaseDevVAddr,
+						uiAllocSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	VRangeInsertMapUnmapCommands(psDeviceNode,
+						IMG_TRUE,
+						ui32AllocationIndex,
+						sBaseDevVAddr,
+						ui32StartPage,
+						ui32NumPages,
+						szName);
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+
+}
+
+PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(CONNECTION_DATA *psConnection,
+						PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_DEV_VIRTADDR sBaseDevVAddr,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32NumPages,
+						IMG_DEVMEM_SIZE_T uiAllocSize,
+						const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+						IMG_UINT32 ui32Log2PageSize,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+							ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(psDeviceNode,
+						ui32AllocationIndex,
+						0,
+						sBaseDevVAddr,
+						uiAllocSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	VRangeInsertMapUnmapCommands(psDeviceNode,
+						IMG_FALSE,
+						ui32AllocationIndex,
+						sBaseDevVAddr,
+						ui32StartPage,
+						ui32NumPages,
+						szName);
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+
+
+/* DevicememHistoryUnmapKM:
+ * Entry point for when an allocation is unmapped from the MMU GPU
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistoryUnmapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_BOOL bSparse = PMR_IsSparse(psPMR);
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	if (!bSparse)
+	{
+		InsertUnmapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
+	}
+	else
+	{
+		GenerateMapUnmapCommandsForSparsePMR(psPMR,
+								ui32AllocationIndex,
+								IMG_FALSE);
+	}
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+/* DevicememHistorySparseChangeKM:
+ * Entry point for when a sparse allocation is changed, such that some of the
+ * pages within the sparse allocation are mapped or unmapped.
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocPageCount: Number of pages which have been mapped.
+ * paui32AllocPageIndices: Indices of pages which have been mapped.
+ * ui32FreePageCount: Number of pages which have been unmapped.
+ * paui32FreePageIndices: Indices of pages which have been unmapped.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocPageCount,
+							IMG_UINT32 *paui32AllocPageIndices,
+							IMG_UINT32 ui32FreePageCount,
+							IMG_UINT32 *paui32FreePageIndices,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if (!PMRValidateSize((IMG_UINT64) ui32AllocPageCount << ui32Log2PageSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				 "PMR size exceeds limit #Chunks: %u ChunkSz %"IMG_UINT64_FMTSPECX"",
+				 ui32AllocPageCount,
+				 (IMG_UINT64) 1ULL << ui32Log2PageSize);
+		return PVRSRV_ERROR_PMR_TOO_LARGE;
+	}
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
+							ui32AllocPageCount,
+							paui32AllocPageIndices,
+							ui32AllocationIndex,
+							IMG_TRUE);
+
+	GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
+							ui32FreePageCount,
+							paui32FreePageIndices,
+							ui32AllocationIndex,
+							IMG_FALSE);
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+
+}
+
+/* CircularBufferIterateStart:
+ * Initialise local state for iterating over the circular buffer
+ */
+static void CircularBufferIterateStart(DEVICEMEM_HISTORY_DATA *psHData, IMG_UINT32 *pui32Head, IMG_UINT32 *pui32Iter)
+{
+	*pui32Head = psHData->sRecords.ui32Head;
+
+	if (*pui32Head != 0)
+	{
+		*pui32Iter = *pui32Head - 1;
+	}
+	else
+	{
+		*pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
+	}
+}
+
+/* CircularBufferIteratePrevious:
+ * Iterate to the previous item in the circular buffer.
+ * This is called repeatedly to iterate over the whole circular buffer.
+ */
+static COMMAND_WRAPPER *CircularBufferIteratePrevious(DEVICEMEM_HISTORY_DATA *psHData,
+							IMG_UINT32 ui32Head,
+							IMG_UINT32 *pui32Iter,
+							COMMAND_TYPE *peType,
+							IMG_BOOL *pbLast)
+{
+	IMG_UINT8 *pui8Header;
+	COMMAND_WRAPPER *psOut = NULL;
+
+	psOut = psHData->sRecords.pasCircularBuffer + *pui32Iter;
+
+	pui8Header = (void *) psOut;
+
+	/* Check the command looks valid.
+	 * this condition should never happen, but check for it anyway
+	 * and try to handle it
+	 */
+	if (*pui8Header >= COMMAND_TYPE_COUNT)
+	{
+		/* invalid header detected. Circular buffer corrupted? */
+		PVR_DPF((PVR_DBG_ERROR, "CircularBufferIteratePrevious: "
+							"Invalid header: %u",
+							*pui8Header));
+		*pbLast = IMG_TRUE;
+		return NULL;
+	}
+
+	*peType = *pui8Header;
+
+	if (*pui32Iter != 0)
+	{
+		(*pui32Iter)--;
+	}
+	else
+	{
+		*pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
+	}
+
+
+	/* inform the caller this is the last command if either we have reached
+	 * the head (where we started) or if we have reached an empty command,
+	 * which means we have covered all populated entries
+	 */
+	if ((*pui32Iter == ui32Head) || (*peType == COMMAND_TYPE_NONE))
+	{
+		/* this is the final iteration */
+		*pbLast = IMG_TRUE;
+	}
+
+	return psOut;
+}
+
+/* MapUnmapCommandGetInfo:
+ * Helper function to get the address and mapping information from a MAP_ALL, UNMAP_ALL,
+ * MAP_RANGE or UNMAP_RANGE command
+ */
+static void MapUnmapCommandGetInfo(DEVICEMEM_HISTORY_DATA *psHData,
+					COMMAND_WRAPPER *psCommand,
+					COMMAND_TYPE eType,
+					IMG_DEV_VIRTADDR *psDevVAddrStart,
+					IMG_DEV_VIRTADDR *psDevVAddrEnd,
+					IMG_BOOL *pbMap,
+					IMG_UINT32 *pui32AllocIndex)
+{
+	if ((eType == COMMAND_TYPE_MAP_ALL) || ((eType == COMMAND_TYPE_UNMAP_ALL)))
+	{
+		COMMAND_MAP_ALL *psMapAll = &psCommand->u.sMapAll;
+		RECORD_ALLOCATION *psAlloc;
+
+		*pbMap = (eType == COMMAND_TYPE_MAP_ALL);
+		*pui32AllocIndex = psMapAll->uiAllocIndex;
+
+		psAlloc = ALLOC_INDEX_TO_PTR(psHData, psMapAll->uiAllocIndex);
+
+		*psDevVAddrStart = psAlloc->sDevVAddr;
+		psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr + psAlloc->uiSize - 1;
+	}
+	else if ((eType == COMMAND_TYPE_MAP_RANGE) || ((eType == COMMAND_TYPE_UNMAP_RANGE)))
+	{
+		COMMAND_MAP_RANGE *psMapRange = &psCommand->u.sMapRange;
+		RECORD_ALLOCATION *psAlloc;
+		IMG_UINT32 ui32StartPage, ui32Count;
+
+		*pbMap = (eType == COMMAND_TYPE_MAP_RANGE);
+		*pui32AllocIndex = psMapRange->uiAllocIndex;
+
+		psAlloc = ALLOC_INDEX_TO_PTR(psHData, psMapRange->uiAllocIndex);
+
+		MapRangeUnpack(psMapRange, &ui32StartPage, &ui32Count);
+
+		psDevVAddrStart->uiAddr = psAlloc->sDevVAddr.uiAddr +
+				((1ULL << psAlloc->ui32Log2PageSize) * ui32StartPage);
+
+		psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr +
+				((1ULL << psAlloc->ui32Log2PageSize) * ui32Count) - 1;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid command type: %u",
+								__func__,
+								eType));
+	}
+}
+
+void DevicememHistoryDumpRecordStats(PVRSRV_DEVICE_NODE *psDevNode,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile)
+{
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+	psDevHData = DevmemFindDataFromDev(psDevNode);
+
+	if (psDevHData)
+	{
+		PVR_DUMPDEBUG_LOG("    DevmemHistoryRecordStats -"
+							  " CBWC:%"IMG_UINT64_FMTSPEC
+							  " MAC:%"IMG_UINT64_FMTSPEC
+							  " UMAC:%"IMG_UINT64_FMTSPEC
+							  " MRC:%"IMG_UINT64_FMTSPEC
+							  " UMRC:%"IMG_UINT64_FMTSPEC
+							  " TSC:%"IMG_UINT64_FMTSPEC
+							  " MAX:%"IMG_UINT64_FMTSPEC
+							  " CHD:%u",
+							  psDevHData->sRecords.ui64CBWrapCount,
+							  psDevHData->sRecords.ui64MapAllCount,
+							  psDevHData->sRecords.ui64UnMapAllCount,
+							  psDevHData->sRecords.ui64MapRangeCount,
+							  psDevHData->sRecords.ui64UnMapRangeCount,
+							  psDevHData->sRecords.ui64TimeStampCount,
+							  (IMG_UINT64)CIRCULAR_BUFFER_NUM_COMMANDS,
+							  psDevHData->sRecords.ui32Head);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("    DevmemHistoryRecordStats - None");
+	}
+}
+
+/* DevicememHistoryQuery:
+ * Entry point for rgxdebug to look up addresses relating to a page fault
+ */
+IMG_BOOL DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
+                               DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+                               IMG_UINT32 ui32PageSizeBytes,
+                               IMG_BOOL bMatchAnyAllocInPage)
+{
+	IMG_UINT32 ui32Head, ui32Iter;
+	COMMAND_TYPE eType = COMMAND_TYPE_NONE;
+	COMMAND_WRAPPER *psCommand = NULL;
+	IMG_BOOL bLast = IMG_FALSE;
+	IMG_UINT64 ui64StartTime = OSClockns64();
+	IMG_UINT64 ui64TimeNs = 0;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	/* initialise the results count for the caller */
+	psQueryOut->ui32NumResults = 0;
+	psQueryOut->ui64SearchCount = 0;
+
+	psDevHData = DevmemFindDataFromDev(psQueryIn->psDevNode);
+
+	if (psDevHData == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	/* if the search is constrained to a particular PID then we
+	 * first search the list of allocations to see if this
+	 * PID is known to us
+	 */
+	if (psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY)
+	{
+		IMG_UINT32 ui32Alloc;
+		ui32Alloc = psDevHData->sRecords.ui32AllocationsListHead;
+
+		while (ui32Alloc != END_OF_LIST)
+		{
+			RECORD_ALLOCATION *psAlloc;
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+			if (psAlloc->uiPID == psQueryIn->uiPID)
+			{
+				goto found_pid;
+			}
+
+			if (ui32Alloc == psDevHData->sRecords.ui32AllocationsListHead)
+			{
+				/* gone through whole list */
+				break;
+			}
+		}
+
+		/* PID not found, so we do not have any suitable data for this
+		 * page fault
+		 */
+		goto out_unlock;
+	}
+
+found_pid:
+
+	CircularBufferIterateStart(psDevHData, &ui32Head, &ui32Iter);
+
+	while (!bLast)
+	{
+		psCommand = CircularBufferIteratePrevious(psDevHData, ui32Head, &ui32Iter, &eType, &bLast);
+		if (psCommand == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: CircularBufferIteratePrevious returned NULL psCommand",
+			         __func__));
+			return IMG_FALSE;
+		}
+
+		if (eType == COMMAND_TYPE_TIMESTAMP)
+		{
+			ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
+			continue;
+		}
+
+		if ((eType == COMMAND_TYPE_MAP_ALL) ||
+			(eType == COMMAND_TYPE_UNMAP_ALL) ||
+			(eType == COMMAND_TYPE_MAP_RANGE) ||
+			(eType == COMMAND_TYPE_UNMAP_RANGE))
+		{
+			RECORD_ALLOCATION *psAlloc;
+			IMG_DEV_VIRTADDR sAllocStartAddrOrig, sAllocEndAddrOrig;
+			IMG_DEV_VIRTADDR sAllocStartAddr, sAllocEndAddr;
+			IMG_BOOL bMap;
+			IMG_UINT32 ui32AllocIndex;
+
+			MapUnmapCommandGetInfo(psDevHData,
+			                psCommand,
+							eType,
+							&sAllocStartAddrOrig,
+							&sAllocEndAddrOrig,
+							&bMap,
+							&ui32AllocIndex);
+
+			sAllocStartAddr = sAllocStartAddrOrig;
+			sAllocEndAddr = sAllocEndAddrOrig;
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocIndex);
+
+			/* skip this command if we need to search within
+			 * a particular PID, and this allocation is not from
+			 * that PID
+			 */
+			if ((psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY) &&
+				(psAlloc->uiPID != psQueryIn->uiPID))
+			{
+				continue;
+			}
+
+			/* if the allocation was created after this event, then this
+			 * event must be for an old/removed allocation, so skip it
+			 */
+			if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
+			{
+				continue;
+			}
+
+			psQueryOut->ui64SearchCount++;
+
+			/* if the caller wants us to match any allocation in the
+			 * same page as the allocation then tweak the real start/end
+			 * addresses of the allocation here
+			 */
+			if (bMatchAnyAllocInPage)
+			{
+				sAllocStartAddr.uiAddr = sAllocStartAddr.uiAddr & ~(IMG_UINT64) (ui32PageSizeBytes - 1);
+				sAllocEndAddr.uiAddr = PVR_ALIGN(sAllocEndAddr.uiAddr, (IMG_UINT64)ui32PageSizeBytes);
+			}
+
+			if ((psQueryIn->sDevVAddr.uiAddr >= sAllocStartAddr.uiAddr) &&
+				(psQueryIn->sDevVAddr.uiAddr <  sAllocEndAddr.uiAddr))
+			{
+				DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult = &psQueryOut->sResults[psQueryOut->ui32NumResults];
+
+				OSStringSafeCopy(psResult->szString, psAlloc->szName, sizeof(psResult->szString));
+				psResult->sBaseDevVAddr = psAlloc->sDevVAddr;
+				psResult->uiSize = psAlloc->uiSize;
+				psResult->bMap = bMap;
+				psResult->ui64Age = _CalculateAge(ui64StartTime, ui64TimeNs, TIME_STAMP_MASK);
+				psResult->ui64When = ui64TimeNs;
+				/* write the responsible PID in the placeholder */
+				psResult->sProcessInfo.uiPID = psAlloc->uiPID;
+
+				if ((eType == COMMAND_TYPE_MAP_ALL) || (eType == COMMAND_TYPE_UNMAP_ALL))
+				{
+					psResult->bRange = IMG_FALSE;
+					psResult->bAll = IMG_TRUE;
+				}
+				else
+				{
+					psResult->bRange = IMG_TRUE;
+					MapRangeUnpack(&psCommand->u.sMapRange,
+										&psResult->ui32StartPage,
+										&psResult->ui32PageCount);
+					psResult->bAll = (psResult->ui32PageCount * (1U << psAlloc->ui32Log2PageSize))
+											== psAlloc->uiSize;
+					psResult->sMapStartAddr = sAllocStartAddrOrig;
+					psResult->sMapEndAddr = sAllocEndAddrOrig;
+				}
+
+				psQueryOut->ui32NumResults++;
+
+				if (psQueryOut->ui32NumResults == DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS)
+				{
+					break;
+				}
+			}
+		}
+	}
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return psQueryOut->ui32NumResults > 0;
+}
+
+#if defined(SUPPORT_RGX)
+static void DeviceMemHistoryFmt(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN],
+							IMG_PID uiPID,
+							const IMG_CHAR *pszName,
+							const IMG_CHAR *pszAction,
+							IMG_DEV_VIRTADDR sDevVAddrStart,
+							IMG_DEV_VIRTADDR sDevVAddrEnd,
+							IMG_UINT64 ui64TimeNs)
+{
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+				/* PID NAME MAP/UNMAP MIN-MAX SIZE AbsUS AgeUS*/
+				"%04u %-40s %-10s "
+				IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC " "
+				"0x%08" IMG_UINT64_FMTSPECX " "
+				"%013" IMG_UINT64_FMTSPEC, /* 13 digits is over 2 hours of ns */
+				uiPID,
+				pszName,
+				pszAction,
+				sDevVAddrStart.uiAddr,
+				sDevVAddrEnd.uiAddr,
+				sDevVAddrEnd.uiAddr - sDevVAddrStart.uiAddr + 1,
+				ui64TimeNs);
+}
+
+static void DeviceMemHistoryFmtHeader(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+				"%-4s %-40s %-6s   %10s   %10s   %8s %13s",
+				"PID",
+				"NAME",
+				"ACTION",
+				"ADDR MIN",
+				"ADDR MAX",
+				"SIZE",
+				"ABS NS");
+}
+
+static const char *CommandTypeToString(COMMAND_TYPE eType)
+{
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_ALL:
+			return "MapAll";
+		case COMMAND_TYPE_UNMAP_ALL:
+			return "UnmapAll";
+		case COMMAND_TYPE_MAP_RANGE:
+			return "MapRange";
+		case COMMAND_TYPE_UNMAP_RANGE:
+			return "UnmapRange";
+		case COMMAND_TYPE_TIMESTAMP:
+			return "TimeStamp";
+		default:
+			return "???";
+	}
+}
+
+static void DevicememHistoryPrintAll(OSDI_IMPL_ENTRY *psEntry)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	IMG_UINT32 ui32Iter;
+	IMG_UINT32 ui32Head;
+	IMG_BOOL bLast = IMG_FALSE;
+	IMG_UINT64 ui64TimeNs = 0;
+	IMG_UINT64 ui64StartTime = OSClockns64();
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+
+	DeviceMemHistoryFmtHeader(szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	CircularBufferIterateStart(psDevHData, &ui32Head, &ui32Iter);
+
+	while (!bLast)
+	{
+		COMMAND_WRAPPER *psCommand;
+		COMMAND_TYPE eType = COMMAND_TYPE_NONE;
+
+		psCommand = CircularBufferIteratePrevious(psDevHData, ui32Head, &ui32Iter,
+		                                          &eType, &bLast);
+		if (psCommand == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: CircularBufferIteratePrevious returned NULL psCommand",
+			         __func__));
+			return;
+		}
+
+		if (eType == COMMAND_TYPE_TIMESTAMP)
+		{
+			ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
+			continue;
+		}
+
+
+		if ((eType == COMMAND_TYPE_MAP_ALL) ||
+			(eType == COMMAND_TYPE_UNMAP_ALL) ||
+			(eType == COMMAND_TYPE_MAP_RANGE) ||
+			(eType == COMMAND_TYPE_UNMAP_RANGE))
+		{
+			RECORD_ALLOCATION *psAlloc;
+			IMG_DEV_VIRTADDR sDevVAddrStart, sDevVAddrEnd;
+			IMG_BOOL bMap;
+			IMG_UINT32 ui32AllocIndex;
+
+			MapUnmapCommandGetInfo(psDevHData,
+			                       psCommand,
+			                       eType,
+			                       &sDevVAddrStart,
+			                       &sDevVAddrEnd,
+			                       &bMap,
+			                       &ui32AllocIndex);
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocIndex);
+
+			if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
+			{
+				/* if this event relates to an allocation we
+				 * are no longer tracking then do not print it
+				 */
+				continue;
+			}
+
+			DeviceMemHistoryFmt(szBuffer,
+			                    psAlloc->uiPID,
+			                    psAlloc->szName,
+			                    CommandTypeToString(eType),
+			                    sDevVAddrStart,
+			                    sDevVAddrEnd,
+			                    ui64TimeNs);
+
+			DIPrintf(psEntry, "%s\n", szBuffer);
+		}
+	}
+
+	DIPrintf(psEntry, "\nTimestamp reference: %013" IMG_UINT64_FMTSPEC "\n",
+	         ui64StartTime);
+}
+
+static int DevicememHistoryPrintAllWrapper(OSDI_IMPL_ENTRY *psEntry,
+                                           void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)DIGetPrivData(psEntry);
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	/* Get the backing store associated with the device. If we are
+	 * called before the device has been started (i.e. FW loaded)
+	 * then we haven't yet had this data allocated.
+	 * Return to provide a NULL data stream to the consumer.
+	 */
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+	if (psDevHData == NULL)
+	{
+		return 0;
+	}
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	DevicememHistoryLock(psDevHData);
+	DevicememHistoryPrintAll(psEntry);
+	DevicememHistoryUnlock(psDevHData);
+
+	return 0;
+}
+#endif	/* defined(SUPPORT_RGX) */
+
+static PVRSRV_ERROR CreateRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	psDevHData->sRecords.pasAllocations =
+			OSAllocZMemNoStats(sizeof(RECORD_ALLOCATION) * ALLOCATION_LIST_NUM_ENTRIES);
+
+	PVR_RETURN_IF_NOMEM(psDevHData->sRecords.pasAllocations);
+
+	/* Allocated and initialise the circular buffer with zeros so every
+	 * command is initialised as a command of type COMMAND_TYPE_NONE. */
+	psDevHData->sRecords.pasCircularBuffer =
+			OSAllocZMemNoStats(sizeof(COMMAND_WRAPPER) * CIRCULAR_BUFFER_NUM_COMMANDS);
+
+	if (psDevHData->sRecords.pasCircularBuffer == NULL)
+	{
+		OSFreeMemNoStats(psDevHData->sRecords.pasAllocations);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void DestroyRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	OSFreeMemNoStats(psDevHData->sRecords.pasCircularBuffer);
+	OSFreeMemNoStats(psDevHData->sRecords.pasAllocations);
+}
+
+static void InitialiseRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 i;
+
+	/* initialise the allocations list */
+
+	psDevHData->sRecords.pasAllocations[0].ui32Prev = ALLOCATION_LIST_NUM_ENTRIES - 1;
+	psDevHData->sRecords.pasAllocations[0].ui32Next = 1;
+
+	for (i = 1; i < ALLOCATION_LIST_NUM_ENTRIES; i++)
+	{
+		psDevHData->sRecords.pasAllocations[i].ui32Prev = i - 1;
+		psDevHData->sRecords.pasAllocations[i].ui32Next = i + 1;
+	}
+
+	psDevHData->sRecords.pasAllocations[ALLOCATION_LIST_NUM_ENTRIES - 1].ui32Next = 0;
+
+	psDevHData->sRecords.ui32AllocationsListHead = 0;
+}
+
+static void DevicememHistoryDevDeInitUnit(IMG_UINT32 uiUnit);
+static PVRSRV_ERROR DevicememHistoryDevInitUnit(IMG_UINT32 uiUnit);
+
+PVRSRV_ERROR DevicememHistoryInitKM(void)
+{
+	IMG_UINT32 ui;
+
+	/* Zero-fill the gapsDevicememHistoryData array entries */
+	for (ui = 0; ui < PVRSRV_MAX_DEVICES; ui++)
+	{
+		gapsDevicememHistoryData[ui] = NULL;
+	}
+
+	return PVRSRV_OK;
+}
+
+void DevicememHistoryDeInitKM(void)
+{
+	IMG_UINT32 uiUnit;
+
+	/* Iterate over all potential units and remove their data.
+	 * DI entry is removed by DevicememHistoryDeviceDestroy()
+	 */
+	for (uiUnit = 0; uiUnit < PVRSRV_MAX_DEVICES; uiUnit++)
+	{
+		DevicememHistoryDevDeInitUnit(uiUnit);
+	}
+}
+
+/* Allocate DEVICEMEM_HISTORY_DATA entry for the specified unit */
+static PVRSRV_ERROR DevicememHistoryDevInitUnit(IMG_UINT32 uiUnit)
+{
+	PVRSRV_ERROR eError;
+	DEVICEMEM_HISTORY_DATA *psDevicememHistoryData;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		PVR_LOG_RETURN_IF_FALSE(uiUnit < PVRSRV_MAX_DEVICES, "Invalid Unit",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Valid unit, try and allocate and fill all structure members */
+
+	psDevicememHistoryData = OSAllocZMemNoStats(sizeof(DEVICEMEM_HISTORY_DATA));
+	PVR_RETURN_IF_NOMEM(psDevicememHistoryData);
+
+	eError = OSLockCreate(&psDevicememHistoryData->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", err_lock);
+
+	eError = CreateRecords(psDevicememHistoryData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateRecords", err_allocations);
+
+	InitialiseRecords(psDevicememHistoryData);
+
+	gapsDevicememHistoryData[uiUnit] = psDevicememHistoryData;
+
+	return PVRSRV_OK;
+
+err_allocations:
+	OSLockDestroy(psDevicememHistoryData->hLock);
+	psDevicememHistoryData->hLock = NULL;
+err_lock:
+	OSFreeMemNoStats(psDevicememHistoryData);
+	return eError;
+}
+
+/* Allocate DI entry for specified psDeviceNode */
+PVRSRV_ERROR DevicememHistoryDeviceCreate(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiUnit = psDeviceNode->sDevId.ui32InternalID;
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_DEBUG_INFO *psDevDebugInfo = &psDeviceNode->sDebugInfo;
+	DI_ITERATOR_CB sIterator = {.pfnShow = DevicememHistoryPrintAllWrapper};
+#endif
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Create the DI entry for the device's devicemem_history handle */
+
+
+#if defined(SUPPORT_RGX)
+	eError = DICreateEntry("devicemem_history", psDevDebugInfo->psGroup,
+	                       &sIterator, psDeviceNode,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psDevDebugInfo->psDevMemEntry);
+#endif	/* defined(SUPPORT_RGX) */
+
+	return eError;
+}
+
+/* Allocate the DEVICEMEM_HISTORY_DATA for specified psDeviceNode */
+PVRSRV_ERROR DevicememHistoryDeviceInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiUnit = psDeviceNode->sDevId.ui32InternalID;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = DevicememHistoryDevInitUnit(uiUnit);
+
+	return eError;
+}
+
+static void DevicememHistoryDevDeInitUnit(IMG_UINT32 uiUnit)
+{
+	DEVICEMEM_HISTORY_DATA *psDevicememHistoryData;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return;
+	}
+
+	psDevicememHistoryData = gapsDevicememHistoryData[uiUnit];
+
+	if (psDevicememHistoryData == NULL)
+	{
+		return;
+	}
+
+	DestroyRecords(psDevicememHistoryData);
+
+	if (psDevicememHistoryData->hLock != NULL)
+	{
+		OSLockDestroy(psDevicememHistoryData->hLock);
+		psDevicememHistoryData->hLock = NULL;
+	}
+
+	OSFreeMemNoStats(psDevicememHistoryData);
+	gapsDevicememHistoryData[uiUnit] = NULL;
+}
+
+void DevicememHistoryDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_DEBUG_INFO *psDevDebugInfo = &psDeviceNode->sDebugInfo;
+
+	/* Remove the DI entry associated with this device */
+	DIDestroyEntry(psDevDebugInfo->psDevMemEntry);
+#endif	/* defined(SUPPORT_RGX) */
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_server.c
new file mode 100644
index 000000000000..e26a7ab3b814
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/devicemem_server.c
@@ -0,0 +1,4224 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server-side component of the Device Memory Management.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* our exported API */
+#include "devicemem_server.h"
+#include "devicemem_utils.h"
+#include "devicemem.h"
+
+#include "device.h" /* For device node */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "mmu_common.h"
+#include "pdump_km.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdumpdesc.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+#include "lock.h"
+
+#include "pvrsrv.h" /* for PVRSRVGetPVRSRVData() */
+
+
+#define DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE (1 << 0)
+#define DEVMEMHEAP_REFCOUNT_MIN 1
+#define DEVMEMHEAP_REFCOUNT_MAX IMG_INT32_MAX
+#define DEVMEMRESERVATION_ACQUISITION_MIN 0
+#define DEVMEMRESERVATION_ACQUISITION_MAX IMG_INT32_MAX
+#define DEVMEMRESERVATION_REFCOUNT_MAX IMG_INT32_MAX
+#define DEVMEM_PMR_RSRV_NODE_REFCOUNT_MAX 2
+#define DEVMEMCTX_REFCOUNT_MIN 1
+#define DEVMEMCTX_REFCOUNT_MAX IMG_INT32_MAX
+
+#define DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT 2
+
+struct _DEVMEMINT_CTX_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* MMU common code needs to have a context. There's a one-to-one
+	   correspondence between device memory context and MMU context,
+	   but we have the abstraction here so that we don't need to care
+	   what the MMU does with its context, and the MMU code need not
+	   know about us at all. */
+	MMU_CONTEXT *psMMUContext;
+
+	/* This handle is for devices that require notification when a new
+	   memory context is created and they need to store private data that
+	   is associated with the context. */
+	IMG_HANDLE hPrivData;
+
+	/* The following tracks UM applications that need to be notified of a
+	 * page fault.
+	 * Access to this list is protected by lock defined on a device node:
+	 * PVRSRV_DEVICE_NODE::hPageFaultNotifyLock. */
+	DLLIST_NODE sProcessNotifyListHead;
+	/* The following is a node for the list of registered devmem contexts.
+	 * Access to this list is protected by lock defined on a device node:
+	 * PVRSRV_DEVICE_NODE::hPageFaultNotifyLock. */
+	DLLIST_NODE sPageFaultNotifyListElem;
+
+	/* Device virtual address of a page fault on this context */
+	IMG_DEV_VIRTADDR sFaultAddress;
+
+	/* Bitfield stating which heaps were created on this context. */
+	IMG_UINT64 uiCreatedHeaps;
+
+	/* Context's reference count */
+	ATOMIC_T hRefCount;
+
+	/* General purpose flags */
+	IMG_UINT32 ui32Flags;
+};
+
+struct _DEVMEMINT_CTX_EXPORT_
+{
+	DEVMEMINT_CTX *psDevmemCtx;
+	PMR *psPMR;
+	ATOMIC_T hRefCount;
+	DLLIST_NODE sNode;
+};
+
+struct _DEVMEMINT_HEAP_
+{
+	struct _DEVMEMINT_CTX_ *psDevmemCtx;
+	IMG_DEV_VIRTADDR sBaseAddr;
+	IMG_DEV_VIRTADDR sLastAddr;
+
+	/* Private data for callback functions */
+	IMG_HANDLE hPrivData;
+
+	/* Callback function init */
+	PFN_HEAP_INIT pfnInit;
+
+	/* Callback function deinit */
+	PFN_HEAP_DEINIT pfnDeInit;
+
+	/* Heap's reference count */
+	ATOMIC_T uiRefCount;
+
+	/* Page shift of the heap */
+	IMG_UINT32 uiLog2PageSize;
+
+	/* Copy of the heap index from Device Heap Configuration module */
+	IMG_UINT32 uiHeapIndex;
+};
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+typedef enum _DEVMEM_RESERVATION_TYPE_
+{
+	DEVMEM_RES_TYPE_NONE,
+	DEVMEM_RES_TYPE_STD, /* Devmem Reservation */
+	DEVMEM_RES_TYPE_X, /* DevmemX Reservation */
+	DEVMEM_RES_TYPE_ZOMBIE /* Reservation Freed but still has references */
+} DEVMEM_RESERVATION_TYPE;
+
+/* Forward declare for callback definitions */
+typedef struct _DEVMEM_PMR_RSRV_NODE_ DEVMEM_PMR_RSRV_NODE;
+
+/* Forward declare for Remap functions */
+static PVRSRV_ERROR _RefPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode);
+static void _UnrefAndMaybeDestroyPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode);
+
+/* Callback definition for remaps */
+typedef PVRSRV_ERROR (*PFN_REMAP_PAGE_FN)(PMR *psPMR,
+                                          DLLIST_NODE *psMappingListHead,
+                                          DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                                          IMG_UINT32 ui32LogicalPgOffset);
+/* Callback definition for checking the page is in the mapping */
+typedef IMG_BOOL (*PFN_IS_PAGE_IN_MAP_FN)(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, IMG_UINT32 ui32LogicalPgOffset);
+/* Callback definition for free */
+typedef void (*PFN_FREE_PARENT_STRUCT_FN)(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode);
+
+/* Node linked to PMR for remap.
+ * Also used to control lifetime of struct in which it is embedded.
+ */
+typedef struct _DEVMEM_PMR_RSRV_NODE_
+{
+	/* List node given to the PMR to create the reservation link.*/
+	DLLIST_NODE sLinkNode;
+	/* Type of reservation, determines re-map path, should be
+	 * protected by reservation lock
+	 */
+	DEVMEM_RESERVATION_TYPE eType;
+
+	ATOMIC_T iRefCount;
+	PFN_REMAP_PAGE_FN pfnRemapPage;
+	PFN_IS_PAGE_IN_MAP_FN pfnIsPageInMap;
+	PFN_FREE_PARENT_STRUCT_FN pfnFreeParent;
+
+} DEVMEM_PMR_RSRV_NODE;
+
+/* DevmemX representation of a mapping on the reservation */
+typedef struct _DEVMEMX_MAPPING_
+{
+	/* Link node given to the PMR to create the link for remap */
+	DEVMEM_PMR_RSRV_NODE sPMRRsrvLinkNode;
+	DEVMEMXINT_RESERVATION *psRsrv;
+	PMR *psPMR;
+
+	/* Details of the mapped range */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+	IMG_UINT32 uiPhysPageOffset;
+	IMG_UINT32 uiVirtPageOffset;
+	IMG_UINT32 uiPageCount;
+} DEVMEMX_MAPPING;
+
+typedef struct _DEVMEM_MAPPING_
+{
+	DEVMEM_PMR_RSRV_NODE sPMRRsrvLinkNode;
+	DEVMEMINT_RESERVATION *psRsrv;
+} DEVMEM_MAPPING;
+#endif
+
+/**
+ * Indicates where the reservation will be used.
+ *
+ * Used as locking classes for DevMem reservations to resolve lockdep warnings.
+ */
+typedef enum _RESERVATION_USAGE_
+{
+	RESERVATION_USAGE_SERVER_FW = 0,  // Reservations used by the server / FW (excludes dma_bufs)
+	RESERVATION_USAGE_CLIENT          // Reservations used by client apps (includes dma_bufs)
+} RESERVATION_USAGE;
+
+struct _DEVMEMINT_RESERVATION_
+{
+	struct _DEVMEMINT_HEAP_ *psDevmemHeap;
+	IMG_DEV_VIRTADDR sBase;
+	IMG_DEVMEM_SIZE_T uiLength;
+
+	/* lock used to guard against potential race when freeing reservation, also protects
+	 * parallel operations occurring on the same reservation such as a (un)mapping operation
+	 * from UM and a remap operation from the Linux Kernel
+	 */
+	POS_LOCK hLock;
+	RESERVATION_USAGE eLockClass;
+
+	/* Indicator for ensuring range has not been externally acquired whilst attempting release.
+	 * Protects freeing the reservation while other device resources still have reference such as
+	 * FreeLists and ZSBuffer.
+	 */
+	IMG_INT32 i32DevResAcquisitionCount;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Mapping node representing the mapping on this reservation */
+	DEVMEM_MAPPING *psDevmemMapping;
+#endif
+	/* KM refcount used to protect the lifetime of the reservation itself. This ensures
+	 * synchronisation of use between UM usage and KM usage such as remap paths. We may need
+	 * to keep the reservation alive for these KM paths even once the UM has released it since
+	 * it is used to synchronise operations originating from these KM paths and between UM and KM.
+	 */
+	ATOMIC_T iLifetimeRefCount;
+
+	/* We keep a reference to the single PMR associated with this reservation
+	 * once mapped. When creating the reservation this is null. Used in
+	 * ChangeSparse to validate parameters. We could have a sparse PMR with
+	 * no backing and have it mapped to a reservation.
+	 */
+	PMR *psMappedPMR;
+
+	/* Array of bitfields of size `uiNumPages / MAP_MASK_SHIFT`.
+	 * This array represents the mapping between a PMR (psMappedPMR) and
+	 * the reservation. Each bit represents an index of a physical page - a value
+	 * 1 means the page is mapped and vice versa.
+	 */
+	IMG_UINT8 *pui8Map;
+	#define MAP_MASK_SHIFT 3
+};
+
+/*! Object representing a virtual range reservation and mapping between
+ * the virtual range and a set of PMRs.
+ *
+ * The physical allocations may be mapped entirely or partially to the entire
+ * or partial virtual range. */
+struct _DEVMEMXINT_RESERVATION_
+{
+	/*! Pointer to a device memory heap this reservation is made on. */
+	struct _DEVMEMINT_HEAP_ *psDevmemHeap;
+	/*! Base device virtual address of this reservation. */
+	IMG_DEV_VIRTADDR sBase;
+	/*! Size of this reservation (in bytes). */
+	IMG_DEVMEM_SIZE_T uiLength;
+	/*! Lock for protecting concurrent operations on the mapping. Also protects
+	 * parallel operations occurring on the same reservation such as a (un)mapping operation
+	 * from UM and a remap operation from the Linux Kernel. As such also protects mapping records
+	 * kept for remap operations.
+	 */
+	POS_LOCK hLock;
+
+	/* KM refcount used to protect the lifetime of the reservation itself. This ensures
+	 * synchronisation of use between UM usage and KM usage such as remap paths. We may need
+	 * to keep the reservation alive for these KM paths even once the UM has released it since
+	 * it is used to synchronise operations originating from these KM paths and between UM and KM.
+	 */
+	ATOMIC_T iLifetimeRefCount;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/*! Array of DevmemX Map Ranges of size `uiNumPages`. This array represents how the
+	 *  physical memory is mapped to the virtual range. Each entry in the array
+	 *  represents to one device page which means that one PMR may be spread
+	 *  across many indices and related to many ranges */
+	DEVMEMX_MAPPING *ppsDevmemXMapping[IMG_FLEX_ARRAY_MEMBER];
+#else
+
+	/*! Array of PMRs of size `uiNumPages`. This array represents how the
+	 *  physical memory is mapped to the virtual range. Each entry in the array
+	 *  represents to one device page which means that one PMR may be spread
+	 *  across many indices. */
+	PMR **ppsPMR;
+#endif
+};
+
+struct _DEVMEMINT_PF_NOTIFY_
+{
+	IMG_UINT32  ui32PID;
+	DLLIST_NODE sProcessNotifyListElem;
+};
+
+/** Computes division using log2 of divisor. */
+#define LOG2_DIV(x, log2) ((x) >> (log2))
+
+/** Computes modulo of a power of 2. */
+#define LOG2_MOD(x, log2) ((x) & ((1 << (log2)) - 1))
+
+static INLINE IMG_UINT32 _DevmemReservationPageCount(DEVMEMINT_RESERVATION *psRsrv);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationIsIndexMapped
+@Description    Checks whether a particular index in the reservation has been
+                mapped to a page in psMappedPMR.
+
+@Return         IMG_TRUE if mapped or IMG_FALSE if not.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntReservationIsIndexMapped(DEVMEMINT_RESERVATION *psReservation,
+                                                         IMG_UINT32 ui32Index)
+{
+	IMG_UINT32 ui32MapIndex = LOG2_DIV(ui32Index, MAP_MASK_SHIFT);
+
+	PVR_ASSERT(psReservation != NULL);
+	PVR_ASSERT(ui32Index < _DevmemReservationPageCount(psReservation));
+
+	return BIT_ISSET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationSetMappingIndex
+@Description    Sets an index of the reservation map to indicate a mapped or
+                unmapped PMR page.
+
+@Note           The reservations hLock must be acquired before calling this
+                function.
+
+@Return         None
+*/ /**************************************************************************/
+static void DevmemIntReservationSetMappingIndex(DEVMEMINT_RESERVATION *psReservation,
+                                                IMG_UINT32 ui32Index,
+                                                IMG_BOOL bMap)
+{
+	IMG_UINT32 ui32MapIndex = LOG2_DIV(ui32Index, MAP_MASK_SHIFT);
+
+	PVR_ASSERT(psReservation != NULL);
+	PVR_ASSERT(ui32Index < _DevmemReservationPageCount(psReservation));
+
+	if (bMap)
+	{
+		BIT_SET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+	}
+	else
+	{
+		BIT_UNSET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+	}
+}
+
+static INLINE IMG_UINT32
+_DevmemXReservationPageCount(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	return psRsrv->uiLength >> psRsrv->psDevmemHeap->uiLog2PageSize;
+}
+
+static INLINE IMG_DEV_VIRTADDR
+_DevmemXReservationPageAddress(DEVMEMXINT_RESERVATION *psRsrv, IMG_UINT32 uiVirtPageOffset)
+{
+	IMG_DEV_VIRTADDR sAddr = {
+		.uiAddr = psRsrv->sBase.uiAddr + ((IMG_UINT64)uiVirtPageOffset << psRsrv->psDevmemHeap->uiLog2PageSize)
+	};
+
+	return sAddr;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/****************************************
+ * Map & Remap synchronisation helpers  *
+ ****************************************/
+
+static void
+_AcquireReservationPMRLocks(POS_LOCK hRsrvLock, PMR* psPMR)
+{
+	OSLockAcquire(hRsrvLock);
+	PMRLockPMR(psPMR);
+}
+
+static void
+_AcquireReservationPMRLocksNested(POS_LOCK hRsrvLock, RESERVATION_USAGE eLockClass, PMR* psPMR)
+{
+	OSLockAcquireNested(hRsrvLock, eLockClass);
+	PMRLockPMR(psPMR);
+}
+
+static void
+_ReleaseReservationPMRLocks(POS_LOCK hRsrvLock, PMR* psPMR)
+{
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(hRsrvLock);
+}
+
+static void
+_DeschedAllowRemapToComplete(POS_LOCK hRsrvLock, PMR* psPMR)
+{
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(hRsrvLock);
+	OSReleaseThreadQuanta();
+	OSLockAcquire(hRsrvLock);
+	PMRLockPMR(psPMR);
+}
+
+static void
+_DeschedAllowRemapToCompleteNested(POS_LOCK hRsrvLock, RESERVATION_USAGE eLockClass, PMR* psPMR)
+{
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(hRsrvLock);
+	OSReleaseThreadQuanta();
+	OSLockAcquireNested(hRsrvLock, eLockClass);
+	PMRLockPMR(psPMR);
+}
+
+/*******************
+ * Remap functions *
+ *******************/
+
+static IMG_BOOL
+_CheckMultipleRelatedMappingsExist(DLLIST_NODE *psMappingListHead,
+                                                      IMG_UINT32 ui32LogicalPgOffset)
+{
+	PDLLIST_NODE pNext, pNode;
+	DEVMEM_PMR_RSRV_NODE *psFoundPMRResLinkNode = NULL;
+	IMG_BOOL bMultipleFound = IMG_FALSE;
+
+	dllist_foreach_node(psMappingListHead, pNode, pNext)
+	{
+		DEVMEM_PMR_RSRV_NODE *psPMRResLinkNode = IMG_CONTAINER_OF(pNode, DEVMEM_PMR_RSRV_NODE, sLinkNode);
+
+		if (psPMRResLinkNode->pfnIsPageInMap(psPMRResLinkNode, ui32LogicalPgOffset))
+		{
+			if (psFoundPMRResLinkNode == NULL)
+			{
+				psFoundPMRResLinkNode = psPMRResLinkNode;
+			}
+			else
+			{
+				/* We have found multiple mappings associated with this page,
+				 * we will reject the remap request
+				 */
+				bMultipleFound = IMG_TRUE;
+				break;
+			}
+		}
+	}
+
+	return bMultipleFound;
+}
+
+static PVRSRV_ERROR
+_pfnDevmemRemapPageInPMR(PMR *psPMR,
+                         DLLIST_NODE *psMappingListHead,
+                         DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                         IMG_UINT32 ui32LogicalPgOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MAPPING* psMapping = IMG_CONTAINER_OF(psPMRRsrvNode, DEVMEM_MAPPING, sPMRRsrvLinkNode);
+	DEVMEMINT_RESERVATION *psRsrv = psMapping->psRsrv;
+	IMG_DEV_VIRTADDR sDevVAddr;
+
+	_AcquireReservationPMRLocksNested(psRsrv->hLock,
+	                                  psRsrv->eLockClass,
+	                                  psPMR);
+
+	PVR_ASSERT(psMapping->sPMRRsrvLinkNode.eType != DEVMEM_RES_TYPE_X);
+
+	/* First check if the Mapping is in Zombie state. This means UM has freed it between us
+	 * finding the related mapping on the PMR list and us preparing to action a remap here.
+	 */
+	if (psMapping->sPMRRsrvLinkNode.eType == DEVMEM_RES_TYPE_ZOMBIE)
+	{
+		/* This mapping has been deleted by UM */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	/* Now we have locked the reservation and the PMR, check if multiple mapping entries exist
+	 * on the list related to the page to remap. If there are multiple, we must reject the
+	 * request to avoid inconsistency in event of remap failure.
+	 */
+	if (_CheckMultipleRelatedMappingsExist(psMappingListHead, ui32LogicalPgOffset))
+	{
+		/* This PMR page has multiple associated mappings, reject migrate */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	sDevVAddr.uiAddr = psRsrv->sBase.uiAddr + ((IMG_UINT64)ui32LogicalPgOffset << psRsrv->psDevmemHeap->uiLog2PageSize);
+
+	eError = MMU_RemapPage(psRsrv->psDevmemHeap->psDevmemCtx->psMMUContext,
+	                       psRsrv->uiFlags,
+	                       sDevVAddr,
+	                       psRsrv->psDevmemHeap->uiLog2PageSize,
+	                       psPMR,
+	                       ui32LogicalPgOffset);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Devmem mapped PMR Remap page failed"));
+	}
+
+_Return:
+	_ReleaseReservationPMRLocks(psRsrv->hLock, psPMR);
+	return eError;
+}
+
+static IMG_BOOL
+_pfnDevmemCheckPMRPageInMappingRange(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, IMG_UINT32 ui32LogicalPgOffset)
+{
+	/* Devmem reservations can have 0 or 1 mappings, hence always return true if called for when mapping present.
+	 * We also don't support sparse mappings for movable pages, so the offset will always be valid for the
+	 * mapping should it exist.
+	 */
+	return IMG_TRUE;
+}
+
+static PVRSRV_ERROR
+_pfnDevmemXRemapPageInPMR(PMR *psPMR,
+                          DLLIST_NODE *psMappingListHead,
+                          DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                          IMG_UINT32 ui32LogicalPgOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMX_MAPPING *psDevmemXMapping = IMG_CONTAINER_OF(psPMRRsrvNode,
+	                                                     DEVMEMX_MAPPING,
+	                                                     sPMRRsrvLinkNode);
+	DEVMEMXINT_RESERVATION *psRsrv = psDevmemXMapping->psRsrv;
+	IMG_UINT32 uiVirtPageOffset;
+	IMG_DEV_VIRTADDR sDevVAddr;
+
+	_AcquireReservationPMRLocks(psRsrv->hLock, psPMR);
+
+	PVR_ASSERT(psDevmemXMapping->sPMRRsrvLinkNode.eType != DEVMEM_RES_TYPE_STD);
+
+	/* First check if the Mapping is in Zombie state. This means UM has freed it between us
+	 * finding the related mapping on the PMR list and us preparing to action a remap here.
+	 */
+	if (psDevmemXMapping->sPMRRsrvLinkNode.eType == DEVMEM_RES_TYPE_ZOMBIE)
+	{
+		/* This mapping has been deleted by UM */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	/* Next check if the mapping range is still valid for the PMR page, this could be changed
+	 * by a UM map overwriting part of an old PMR page range if we were already in the process
+	 * of mapping when remap occurred and we checked for the existence of related mappings.
+	 */
+	if (!psPMRRsrvNode->pfnIsPageInMap(psPMRRsrvNode, ui32LogicalPgOffset))
+	{
+		/* This range is no longer related to the page we wish to migrate. */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	/* Finally, check if multiple mapping entries exist on the list related to the page to remap.
+	 * If there are multiple, we must reject the request to avoid inconsistency in event of remap failure.
+	 */
+	if (_CheckMultipleRelatedMappingsExist(psMappingListHead, ui32LogicalPgOffset))
+	{
+		/* This PMR page has multiple associated mappings, reject migrate */
+		eError = PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED;
+		goto _Return;
+	}
+
+	uiVirtPageOffset = psDevmemXMapping->uiVirtPageOffset +
+	                   (ui32LogicalPgOffset - psDevmemXMapping->uiPhysPageOffset);
+
+	sDevVAddr = _DevmemXReservationPageAddress(psDevmemXMapping->psRsrv,
+	                                           uiVirtPageOffset);
+
+	eError =
+		MMU_RemapPage(psDevmemXMapping->psRsrv->psDevmemHeap->psDevmemCtx->psMMUContext,
+					  psDevmemXMapping->uiFlags,
+					  sDevVAddr,
+					  psDevmemXMapping->psRsrv->psDevmemHeap->uiLog2PageSize,
+					  psPMR,
+					  ui32LogicalPgOffset);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "DevmemX mapped PMR Remap page failed"));
+	}
+
+
+_Return:
+	_ReleaseReservationPMRLocks(psRsrv->hLock, psPMR);
+	return eError;
+}
+
+static IMG_BOOL
+_pfnDevmemXCheckPMRPageInMappingRange(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, IMG_UINT32 ui32LogicalPgOffset)
+{
+	DEVMEMX_MAPPING *psDevmemXMapping = IMG_CONTAINER_OF(psPMRRsrvNode,
+                                                         DEVMEMX_MAPPING,
+                                                         sPMRRsrvLinkNode);
+
+	if (ui32LogicalPgOffset >= psDevmemXMapping->uiPhysPageOffset + psDevmemXMapping->uiPageCount ||
+		ui32LogicalPgOffset < psDevmemXMapping->uiPhysPageOffset)
+	{
+		/* This range is not related to the page we wish to migrate. We return IMG_FALSE
+		 * because there is no operation to perform on this mapping.
+		 */
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+
+PVRSRV_ERROR
+DevmemIntRemapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset)
+{
+	DEVMEM_PMR_RSRV_NODE *psFoundPMRResLinkNode = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDLLIST_NODE pNext, pNode;
+
+	/* Lock the PMR whilst obtaining nodes from the list but not performing remap actions on them,
+	 * this allows us to protect lifetimes of the attachments */
+	PMRLockPMR(psPMR);
+
+	/* The PMR caller of this function has no perspective as to if a mapping
+	 * on its list is related to the page it wishes to remap and so we must iterate
+	 * over the list checking for related mappings. If we find a related mapping, pass
+	 * it onto the remap function for processing.
+	 */
+	dllist_foreach_node(psMappingListHead, pNode, pNext)
+	{
+		DEVMEM_PMR_RSRV_NODE *psPMRResLinkNode = IMG_CONTAINER_OF(pNode, DEVMEM_PMR_RSRV_NODE, sLinkNode);
+
+		if (psPMRResLinkNode->pfnIsPageInMap(psPMRResLinkNode, ui32LogicalPgOffset))
+		{
+			psFoundPMRResLinkNode = psPMRResLinkNode;
+			break;
+		}
+	}
+
+	if (psFoundPMRResLinkNode)
+	{
+		eError = _RefPMRRsrvNode(psFoundPMRResLinkNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntRemapPageInPMR.", ErrUnlockPMRExit);
+
+		PMRUnlockPMR(psPMR);
+
+		eError = psFoundPMRResLinkNode->pfnRemapPage(psPMR,
+		                                             psMappingListHead,
+		                                             psFoundPMRResLinkNode,
+		                                             ui32LogicalPgOffset);
+		PVR_GOTO_IF_ERROR(eError, ErrUnrefNodeExit);
+
+		_UnrefAndMaybeDestroyPMRRsrvNode(psFoundPMRResLinkNode);
+	}
+	else
+	{
+		PMRUnlockPMR(psPMR);
+	}
+
+	return eError;
+
+ErrUnlockPMRExit:
+	PMRUnlockPMR(psPMR);
+	return eError;
+
+ErrUnrefNodeExit:
+	_UnrefAndMaybeDestroyPMRRsrvNode(psFoundPMRResLinkNode);
+	return eError;
+}
+
+/**********************************************************
+ * DEVMEM_PMR_RSRV_NODE functions and lifetime management *
+ **********************************************************/
+
+static void _InitialisePMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode,
+                                   DEVMEM_RESERVATION_TYPE eType,
+                                   PFN_REMAP_PAGE_FN pfnRemapPage,
+                                   PFN_IS_PAGE_IN_MAP_FN pfnIsPageInMap,
+                                   PFN_FREE_PARENT_STRUCT_FN pfnFreeParent)
+{
+	/* Automatically start with a ref */
+	OSAtomicWrite(&psPMRRsrvNode->iRefCount, 1);
+
+	psPMRRsrvNode->eType = eType;
+	psPMRRsrvNode->pfnRemapPage = pfnRemapPage;
+	psPMRRsrvNode->pfnIsPageInMap = pfnIsPageInMap;
+	psPMRRsrvNode->pfnFreeParent = pfnFreeParent;
+}
+
+static
+PVRSRV_ERROR _RefPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psPMRRsrvNode->iRefCount, 1,
+	                                        DEVMEM_PMR_RSRV_NODE_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEM_PMR_RSRV_NODE_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the PMRRsrvNode, "
+		"reference count has overflowed.", __func__));
+		return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+void _UnrefAndMaybeDestroyPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	if (OSAtomicDecrement(&psPMRRsrvNode->iRefCount) == 0)
+	{
+		psPMRRsrvNode->pfnFreeParent(psPMRRsrvNode);
+	}
+}
+
+static
+void _LinkPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+	PMRLinkGPUMapping(psPMR, &psPMRRsrvNode->sLinkNode);
+}
+
+static
+void _UnlinkPMRRsrvNode(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode, PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psPMRRsrvNode->eType = DEVMEM_RES_TYPE_ZOMBIE;
+	PMRUnlinkGPUMapping(psPMR, &psPMRRsrvNode->sLinkNode);
+}
+
+/***********************************
+ * Reservation lifetime management *
+ ***********************************/
+
+static PVRSRV_ERROR
+_DevmemReservationRef(DEVMEMINT_RESERVATION *psRsrv)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psRsrv->iLifetimeRefCount, 1,
+	                                        DEVMEMRESERVATION_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMCTX_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the reservation, "
+		"reference count has overflowed.", __func__));
+		return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_DevmemXReservationRef(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psRsrv->iLifetimeRefCount, 1,
+	                                        DEVMEMRESERVATION_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMCTX_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the reservation, "
+		"reference count has overflowed.", __func__));
+		return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+static void
+_DevmemReservationUnrefAndMaybeDestroy(DEVMEMINT_RESERVATION *psRsrv)
+{
+	if (OSAtomicDecrement(&psRsrv->iLifetimeRefCount) == 0)
+	{
+		OSLockDestroy(psRsrv->hLock);
+		OSFreeMem(psRsrv);
+	}
+}
+
+static void
+_DevmemXReservationUnrefAndMaybeDestroy(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	if (OSAtomicDecrement(&psRsrv->iLifetimeRefCount) == 0)
+	{
+		OSLockDestroy(psRsrv->hLock);
+		OSFreeMem(psRsrv);
+	}
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/*****************************
+ * Devmem mapping management *
+ *****************************/
+
+static void _pfnDevmemMappingFreeCallback(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	DEVMEM_MAPPING *psDevmemMapping;
+	PVR_LOG_RETURN_VOID_IF_FALSE(psPMRRsrvNode != NULL, "psPMRRsrvNode");
+
+	psDevmemMapping = IMG_CONTAINER_OF(psPMRRsrvNode, DEVMEM_MAPPING, sPMRRsrvLinkNode);
+
+	_DevmemReservationUnrefAndMaybeDestroy(psDevmemMapping->psRsrv);
+	OSFreeMem(psDevmemMapping);
+}
+
+static DEVMEM_MAPPING* _AllocateDevmemMapping(DEVMEMINT_RESERVATION *psRsrv,
+                                              PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+
+	DEVMEM_MAPPING *psDevmemMapping = OSAllocZMem(sizeof(*psDevmemMapping));
+	if (!psDevmemMapping)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()",
+		"psDevmemMapping",
+		__func__));
+
+		return NULL;
+	}
+
+	eError = _DevmemReservationRef(psRsrv);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psDevmemMapping);
+		return NULL;
+	}
+
+	_InitialisePMRRsrvNode(&psDevmemMapping->sPMRRsrvLinkNode,
+	                       DEVMEM_RES_TYPE_STD,
+	                       _pfnDevmemRemapPageInPMR,
+	                       _pfnDevmemCheckPMRPageInMappingRange,
+	                       _pfnDevmemMappingFreeCallback);
+
+	psDevmemMapping->psRsrv = psRsrv;
+
+	return psDevmemMapping;
+}
+
+/*****************************
+ * DevmemX mapping management *
+ *****************************/
+
+static void _FreePreAllocatedDevmemXMapping(DEVMEMX_MAPPING *psDevmemXMapping)
+{
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevmemXMapping != NULL, "psDevmemXMapping");
+
+	_DevmemXReservationUnrefAndMaybeDestroy(psDevmemXMapping->psRsrv);
+	OSFreeMem(psDevmemXMapping);
+}
+
+static void _pfnDevmemXMappingFreeCallback(DEVMEM_PMR_RSRV_NODE *psPMRRsrvNode)
+{
+	DEVMEMX_MAPPING *psDevmemXMapping;
+	PVR_LOG_RETURN_VOID_IF_FALSE(psPMRRsrvNode != NULL, "psPMRRsrvNode");
+
+	psDevmemXMapping = IMG_CONTAINER_OF(psPMRRsrvNode,
+	                                    DEVMEMX_MAPPING,
+	                                    sPMRRsrvLinkNode);
+
+	/* This pfn is called from either DestroyDevmemXMapping or the Remap path
+	 * dropping its final reference. At this point in both cases the mapping has
+	 * been removed from the list and by doing so de-initialised. Meaning we
+	 * can treat it as though it is in a pre-allocated state.
+	 */
+	_FreePreAllocatedDevmemXMapping(psDevmemXMapping);
+}
+
+static void _DestroyDevmemXMapping(DEVMEMX_MAPPING *psDevmemXMapping)
+{
+	PMRLockHeldAssert(psDevmemXMapping->psPMR);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevmemXMapping != NULL, "psDevmemXMapping");
+
+	_UnlinkPMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode, psDevmemXMapping->psPMR);
+	_UnrefAndMaybeDestroyPMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode);
+}
+
+static DEVMEMX_MAPPING* _PreAllocateDevmemXMapping(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMX_MAPPING *psDevmemXMapping = OSAllocZMem(sizeof(*psDevmemXMapping));
+	if (!psDevmemXMapping)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()",
+		"psDevmemXMapping",
+		__func__));
+
+		return NULL;
+	}
+
+	/* Take ref and set the reservation as we need to fail early */
+	eError = _DevmemXReservationRef(psRsrv);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psDevmemXMapping);
+		return NULL;
+	}
+	psDevmemXMapping->psRsrv = psRsrv;
+
+	_InitialisePMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode,
+	                       DEVMEM_RES_TYPE_X,
+	                       _pfnDevmemXRemapPageInPMR,
+	                       _pfnDevmemXCheckPMRPageInMappingRange,
+	                       _pfnDevmemXMappingFreeCallback);
+
+	return psDevmemXMapping;
+}
+
+static void _InitialiseDevmemXMapping(DEVMEMX_MAPPING *psDevmemXMapping,
+                                      PMR *psPMR,
+                                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                      IMG_UINT32 uiPhysPageOffset,
+                                      IMG_UINT32 uiVirtPageOffset,
+                                      IMG_UINT32 uiPageCount)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psDevmemXMapping->uiFlags = uiFlags;
+	psDevmemXMapping->uiPhysPageOffset = uiPhysPageOffset;
+	psDevmemXMapping->uiVirtPageOffset = uiVirtPageOffset;
+	psDevmemXMapping->uiPageCount = uiPageCount;
+	psDevmemXMapping->psPMR = psPMR;
+
+	_LinkPMRRsrvNode(&psDevmemXMapping->sPMRRsrvLinkNode, psPMR);
+}
+
+static DEVMEMX_MAPPING* _AllocateDevmemXMapping(DEVMEMXINT_RESERVATION *psRsrv,
+                                                PMR *psPMR,
+                                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                                IMG_UINT32 uiPhysPageOffset,
+                                                IMG_UINT32 uiVirtPageOffset,
+                                                IMG_UINT32 uiPageCount)
+{
+	DEVMEMX_MAPPING* psDevmemXMapping;
+
+	PMRLockHeldAssert(psPMR);
+
+	psDevmemXMapping = _PreAllocateDevmemXMapping(psRsrv);
+	if (!psDevmemXMapping)
+	{
+		return NULL;
+	}
+
+	_InitialiseDevmemXMapping(psDevmemXMapping,
+	                          psPMR,
+	                          uiFlags,
+	                          uiPhysPageOffset,
+	                          uiVirtPageOffset,
+	                          uiPageCount);
+
+	return psDevmemXMapping;
+}
+
+/* Return new range */
+static void _SplitDevmemXMapping(DEVMEMX_MAPPING *psSrcDevmemXMapping,
+                                 DEVMEMX_MAPPING *psSplitDevmemXMapping,
+                                 IMG_UINT32 uiNewMappingOffset)
+{
+	IMG_UINT32 uiCorrectedPageCount;
+	IMG_UINT32 uiNewMappingPageCount;
+	IMG_UINT32 uiNewMappingPhysOffset;
+
+	PVR_ASSERT(psSrcDevmemXMapping != NULL);
+	PVR_ASSERT(psSplitDevmemXMapping != NULL);
+
+	uiCorrectedPageCount = uiNewMappingOffset - psSrcDevmemXMapping->uiVirtPageOffset;
+	uiNewMappingPageCount = (psSrcDevmemXMapping->uiVirtPageOffset + psSrcDevmemXMapping->uiPageCount) -
+	                      uiNewMappingOffset;
+	uiNewMappingPhysOffset = psSrcDevmemXMapping->uiPhysPageOffset + uiCorrectedPageCount;
+
+	PVR_LOG_GOTO_IF_FALSE(uiNewMappingOffset > psSrcDevmemXMapping->uiVirtPageOffset,
+	                      "New range offset precedes source mapping.",
+	                      ErrorExit);
+	PVR_LOG_GOTO_IF_FALSE(uiNewMappingOffset <= (psSrcDevmemXMapping->uiVirtPageOffset +
+	                                               psSrcDevmemXMapping->uiPageCount),
+	                      "New range offset exceeds source mapping.",
+	                      ErrorExit);
+
+	_InitialiseDevmemXMapping(psSplitDevmemXMapping,
+	                          psSrcDevmemXMapping->psPMR,
+	                          psSrcDevmemXMapping->uiFlags,
+	                          uiNewMappingPhysOffset,
+	                          uiNewMappingOffset,
+	                          uiNewMappingPageCount);
+
+	psSrcDevmemXMapping->uiPageCount = uiCorrectedPageCount;
+
+	return;
+
+ErrorExit:
+	/* This should never happen and indicates logic error */
+	PVR_DPF((PVR_DBG_FATAL, "Unable to split range for devmemx accounting!"));
+	OSWarnOn(1);
+}
+
+static void _AdjustDevmemXMappingStart(DEVMEMX_MAPPING *psSrcDevmemXMapping,
+                                       IMG_UINT32 uiNewMappingOffsetStart)
+{
+	IMG_UINT32 uiVirtualPagesDiff = (uiNewMappingOffsetStart - psSrcDevmemXMapping->uiVirtPageOffset);
+	IMG_UINT32 uiCorrectedPageCount = psSrcDevmemXMapping->uiPageCount - uiVirtualPagesDiff;
+	IMG_UINT32 uiCorrectedPhysOffset = psSrcDevmemXMapping->uiPhysPageOffset + uiVirtualPagesDiff;
+
+	psSrcDevmemXMapping->uiVirtPageOffset = uiNewMappingOffsetStart;
+	psSrcDevmemXMapping->uiPhysPageOffset = uiCorrectedPhysOffset;
+	psSrcDevmemXMapping->uiPageCount = uiCorrectedPageCount;
+}
+
+static void _AdjustDevmemXMappingEnd(DEVMEMX_MAPPING *psSrcDevmemXMapping,
+                                     IMG_UINT32 uiNewMappingOffsetEnd)
+{
+	IMG_UINT32 uiCorrectedPageCount = uiNewMappingOffsetEnd - psSrcDevmemXMapping->uiVirtPageOffset;
+
+	psSrcDevmemXMapping->uiPageCount = uiCorrectedPageCount;
+}
+
+static void _ReplaceCentreSplitExistingMapping(DEVMEMXINT_RESERVATION *psRsrv,
+                                               IMG_UINT32 uiVirtPageOffset,
+                                               IMG_UINT32 uiPageCount,
+                                               DEVMEMX_MAPPING *psReplaceMapping,
+                                               DEVMEMX_MAPPING *psSplitMappings[2],
+                                               IMG_UINT32 *uiDeferPMRUnlockPhyCount)
+{
+	IMG_UINT32 i;
+	DEVMEMX_MAPPING *psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+	PMR *psCurrentPMR = psCurrentMapping->psPMR;
+
+	/* If the psCurrentMapping references the same PMR we are replacing it with we
+	 * already have the PMR locked above, else we need to lock the PMR we are adjusting.
+	 */
+	if (psReplaceMapping &&
+	    psCurrentPMR == psReplaceMapping->psPMR)
+	{
+		_SplitDevmemXMapping(psCurrentMapping, psSplitMappings[0], uiVirtPageOffset);
+
+		_SplitDevmemXMapping(psSplitMappings[0], psSplitMappings[1], uiVirtPageOffset + uiPageCount);
+
+		_DestroyDevmemXMapping(psSplitMappings[0]);
+
+		*uiDeferPMRUnlockPhyCount = uiPageCount;
+	}
+	else
+	{
+		PMRLockPMR(psCurrentPMR);
+
+		_SplitDevmemXMapping(psCurrentMapping, psSplitMappings[0], uiVirtPageOffset);
+
+		_SplitDevmemXMapping(psSplitMappings[0], psSplitMappings[1], uiVirtPageOffset + uiPageCount);
+
+		_DestroyDevmemXMapping(psSplitMappings[0]);
+
+		PMRUnlockPMR(psCurrentPMR);
+
+		/* Drop references on PMR we have replaced */
+		PMRUnlockSysPhysAddressesN(psCurrentPMR, uiPageCount);
+	}
+
+	/* Update replaced centre region */
+	for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+	{
+		psRsrv->ppsDevmemXMapping[i] = psReplaceMapping;
+	}
+
+	/* Update split after mappings with new entry */
+	for (i = psSplitMappings[1]->uiVirtPageOffset;
+	     i < psSplitMappings[1]->uiVirtPageOffset + psSplitMappings[1]->uiPageCount; i++)
+	{
+		psRsrv->ppsDevmemXMapping[i] = psSplitMappings[1];
+	}
+}
+
+static void _AdjustOrDestroyReplacedMapping(DEVMEMX_MAPPING *psCurrentMapping,
+                                            IMG_UINT32 uiIterStart,
+                                            IMG_UINT32 uiIterEnd)
+{
+	__maybe_unused PMR *psCurrentPMR = psCurrentMapping->psPMR;
+
+	PMRLockHeldAssert(psCurrentPMR);
+
+	/* Adjust or free the map range we just replaced.
+	 *
+	 * 3 Cases:
+	 * New range replaces all existing range
+	 * New range replaces from end of exist range. (This can only happen on first iter)
+	 * New range replaces from start of existing range.
+	 * */
+	if (psCurrentMapping->uiVirtPageOffset == uiIterStart &&
+	    psCurrentMapping->uiVirtPageOffset + psCurrentMapping->uiPageCount == uiIterEnd)
+	{
+		_DestroyDevmemXMapping(psCurrentMapping);
+	}
+	else if (psCurrentMapping->uiVirtPageOffset < uiIterStart)
+	{
+		_AdjustDevmemXMappingEnd(psCurrentMapping, uiIterStart);
+	}
+	else if (uiIterEnd < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount))
+	{
+		_AdjustDevmemXMappingStart(psCurrentMapping, uiIterEnd);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_FATAL, "Unable to perform range record adjustment"));
+	}
+}
+
+static void _ReplaceReservationMappingRecords(DEVMEMXINT_RESERVATION *psRsrv,
+                                             IMG_UINT32 uiVirtPageOffset,
+                                             IMG_UINT32 uiPageCount,
+                                             DEVMEMX_MAPPING *psReplaceMapping,
+                                             IMG_UINT32 *uiDeferPMRUnlockPhyCount)
+{
+	IMG_UINT32 i;
+
+	for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount);)
+	{
+		IMG_UINT32 j = i;
+		IMG_UINT32 uiConsolidate = 0;
+		DEVMEMX_MAPPING *psCurrentMapping = psRsrv->ppsDevmemXMapping[i];
+		PMR *psCurrentPMR = psCurrentMapping->psPMR;
+
+		/* If we have attempted to delete a virtual record from an empty page entry */
+		if (psReplaceMapping == NULL &&
+		    psCurrentMapping == NULL)
+		{
+			i++;
+			continue;
+		}
+		/* If we are adding a new record to the current range reference */
+		else if (psCurrentMapping == NULL)
+		{
+			psRsrv->ppsDevmemXMapping[i] = psReplaceMapping;
+			i++;
+			continue;
+		}
+
+		/* Iterate over records with the same CurrentMapRange and replace them
+		 * in the Map Range array
+		 */
+		do
+		{
+			psRsrv->ppsDevmemXMapping[j] = psReplaceMapping;
+			uiConsolidate++;
+			j++;
+		}
+		while (j < uiVirtPageOffset + uiPageCount &&
+		      psCurrentMapping == psRsrv->ppsDevmemXMapping[j]
+		);
+
+		/* If the psCurrentMapping references the same PMR we are replacing it with we
+		 * already have the PMR locked above, else we need to lock the PMR we are adjusting.
+		 */
+		if (psReplaceMapping &&
+		    psCurrentPMR == psReplaceMapping->psPMR)
+		{
+			_AdjustOrDestroyReplacedMapping(psCurrentMapping, i, j);
+
+			/* We can't UnlockPhyAddrs for this mapping since we have locked it
+			 * in the layer above, defer until we are done with all adjustments
+			 * to this PMR.
+			 */
+			*uiDeferPMRUnlockPhyCount += uiConsolidate;
+		}
+		else
+		{
+			PMRLockPMR(psCurrentPMR);
+			_AdjustOrDestroyReplacedMapping(psCurrentMapping, i, j);
+			PMRUnlockPMR(psCurrentPMR);
+
+			/* Drop references on PMR we have replaced */
+			PMRUnlockSysPhysAddressesN(psCurrentPMR, uiConsolidate);
+		}
+
+		i += uiConsolidate;
+	}
+}
+
+static void _DeleteReservationMappingRecords(DEVMEMXINT_RESERVATION *psRsrv,
+                                             IMG_UINT32 uiVirtPageOffset,
+                                             IMG_UINT32 uiPageCount)
+{
+	__maybe_unused IMG_UINT32 uiDeferUnlockPhyCount = 0;
+	_ReplaceReservationMappingRecords(psRsrv,
+	                                  uiVirtPageOffset,
+	                                  uiPageCount,
+	                                  NULL,
+	                                  &uiDeferUnlockPhyCount);
+
+	/* We don't have a PMR locked when calling this function so there will be
+	 * no UnlockPhys to defer.
+	 */
+	PVR_ASSERT(uiDeferUnlockPhyCount == 0);
+}
+#endif /* defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxAcquire
+@Description    Acquire a reference to the provided device memory context.
+@Return         IMG_TRUE on success, IMG_FALSE on overflow
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntCtxAcquire(DEVMEMINT_CTX *psDevmemCtx)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psDevmemCtx->hRefCount, 1,
+	                                        DEVMEMCTX_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMCTX_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "context, reference count has overflowed.", __func__));
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxRelease
+@Description    Release the reference to the provided device memory context.
+                If this is the last reference which was taken then the
+                memory context will be freed.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void DevmemIntCtxRelease(DEVMEMINT_CTX *psDevmemCtx)
+{
+	if (OSAtomicDecrement(&psDevmemCtx->hRefCount) == 0)
+	{
+		/* The last reference has gone, destroy the context */
+		PVRSRV_DEVICE_NODE *psDevNode = psDevmemCtx->psDevNode;
+		DLLIST_NODE *psNode, *psNodeNext;
+
+		/* Protect removing the node from the list in case it's being accessed
+		 * by DevmemIntPFNotify(). */
+		OSWRLockAcquireWrite(psDevNode->hPageFaultNotifyLock);
+		/* If this context is in the list registered for a debugger, remove
+		 * from that list */
+		if (dllist_node_is_in_list(&psDevmemCtx->sPageFaultNotifyListElem))
+		{
+			dllist_remove_node(&psDevmemCtx->sPageFaultNotifyListElem);
+		}
+		/* It should be safe to release the lock here (as long as
+		 * DevmemIntPFNotify() protects accessing memory contexts as well). */
+		OSWRLockReleaseWrite(psDevNode->hPageFaultNotifyLock);
+
+		/* If there are any PIDs registered for page fault notification.
+		 * Loop through the registered PIDs and free each one */
+		dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+		{
+			DEVMEMINT_PF_NOTIFY *psNotifyNode =
+				IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+			dllist_remove_node(psNode);
+			OSFreeMem(psNotifyNode);
+		}
+
+		if (psDevNode->pfnUnregisterMemoryContext)
+		{
+			psDevNode->pfnUnregisterMemoryContext(psDevmemCtx->hPrivData);
+		}
+		MMU_ContextDestroy(psDevmemCtx->psMMUContext);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Freed memory context %p",
+				 __func__, psDevmemCtx));
+		OSFreeMem(psDevmemCtx);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapAcquire
+@Description    Acquire a reference to the provided device memory heap.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntHeapAcquire(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psDevmemHeap->uiRefCount, 1,
+	                                        DEVMEMHEAP_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMHEAP_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "heap, reference count has overflowed.", __func__));
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapRelease
+@Description    Release the reference to the provided device memory heap.
+                If this is the last reference which was taken then the
+                memory context will be freed.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void DevmemIntHeapRelease(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	IMG_INT32 iOldValue = OSAtomicSubtractUnless(&psDevmemHeap->uiRefCount, 1,
+	                                             DEVMEMHEAP_REFCOUNT_MIN);
+
+	if (iOldValue == DEVMEMHEAP_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "heap, reference count has underflowed.", __func__));
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationAcquire
+@Description    Acquire a reference to the provided device memory reservation.
+                Prevents releasing of the reservation if external device
+                resource components still require it.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+IMG_BOOL DevmemIntReservationAcquire(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+	IMG_BOOL bSuccess = IMG_TRUE;
+
+	OSLockAcquireNested(psDevmemReservation->hLock, psDevmemReservation->eLockClass);
+
+	if (psDevmemReservation->i32DevResAcquisitionCount == DEVMEMRESERVATION_ACQUISITION_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "reservation, acquisition count has overflowed.", __func__));
+
+		bSuccess = IMG_FALSE;
+	}
+	else
+	{
+		psDevmemReservation->i32DevResAcquisitionCount++;
+	}
+
+	OSLockRelease(psDevmemReservation->hLock);
+
+	return bSuccess;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationRelease
+@Description    Release the reference to the provided device memory reservation.
+                Once these references have been released the
+                reservation is allowed to be released from UM.
+@Return         None.
+*/ /**************************************************************************/
+void DevmemIntReservationRelease(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+	OSLockAcquireNested(psDevmemReservation->hLock, psDevmemReservation->eLockClass);
+
+	if (psDevmemReservation->i32DevResAcquisitionCount == DEVMEMRESERVATION_ACQUISITION_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to release the device memory "
+		         "reservation, acquisition count has underflowed.", __func__));
+
+		/* for better debugging */
+		PVR_ASSERT(psDevmemReservation->i32DevResAcquisitionCount == DEVMEMRESERVATION_ACQUISITION_MIN);
+	}
+	else
+	{
+		psDevmemReservation->i32DevResAcquisitionCount--;
+	}
+
+	OSLockRelease(psDevmemReservation->hLock);
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetImportHandle
+@Description    For given exportable memory descriptor returns PMR handle.
+@Return         Memory is exportable - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+                            IMG_HANDLE *phImport)
+{
+	PVRSRV_ERROR eError;
+
+	if ((GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION, e0);
+	}
+
+	/* A new handle means a new import tracking the PMR.
+	 * Hence the source PMR memory layout should be marked fixed
+	 * to make sure the importer view of the memory is the same as
+	 * the exporter throughout its lifetime */
+	PMR_SetLayoutFixed((PMR *)psMemDesc->psImport->hPMR, IMG_TRUE);
+
+	*phImport = psMemDesc->psImport->hPMR;
+	return PVRSRV_OK;
+
+e0:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetHeapHandle
+@Description    For given reservation returns the Heap handle.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetHeapHandle(DEVMEMINT_RESERVATION *psReservation,
+                          IMG_HANDLE *phHeap)
+{
+	if (psReservation == NULL || phHeap == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*phHeap = psReservation->psDevmemHeap;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetContext
+@Description    For given heap returns the context.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetContext(DEVMEMINT_HEAP *psDevmemHeap,
+					   DEVMEMINT_CTX **ppsDevmemCtxPtr)
+{
+	if (psDevmemHeap == NULL || ppsDevmemCtxPtr == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*ppsDevmemCtxPtr = psDevmemHeap->psDevmemCtx;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetPrivData
+@Description    For given context returns the private data handle.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetPrivData(DEVMEMINT_CTX *psDevmemCtx,
+						IMG_HANDLE *phPrivData)
+{
+	if (psDevmemCtx == NULL || phPrivData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*phPrivData = psDevmemCtx->hPrivData;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxCreate
+@Description    Creates and initialises a device memory context.
+@Return         valid Device Memory context handle - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntCtxCreate(CONNECTION_DATA *psConnection,
+                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                   IMG_BOOL bKernelFWMemoryCtx,
+                   DEVMEMINT_CTX **ppsDevmemCtxPtr,
+                   IMG_HANDLE *hPrivData,
+                   IMG_UINT32 *pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtx;
+	IMG_HANDLE hPrivDataInt = NULL;
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = psDeviceNode->pfnGetMMUDeviceAttributes(psDeviceNode,
+	                                                                           bKernelFWMemoryCtx);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s", __func__));
+
+	/* Only allow request for a kernel context that comes from a direct bridge
+	 * (psConnection == NULL). Only the FW/KM Ctx is created over the direct bridge. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(!bKernelFWMemoryCtx || psConnection == NULL,
+	                                "bKernelFWMemoryCtx && psConnection");
+
+	/*
+	 * Ensure that we are safe to perform unaligned accesses on memory
+	 * we mark write-combine, as the compiler might generate
+	 * instructions operating on this memory which require this
+	 * assumption to be true.
+	 */
+	PVR_ASSERT(OSIsWriteCombineUnalignedSafe());
+
+	/* allocate a Devmem context */
+	psDevmemCtx = OSAllocMem(sizeof(*psDevmemCtx));
+	PVR_LOG_GOTO_IF_NOMEM(psDevmemCtx, eError, fail_alloc);
+
+	OSAtomicWrite(&psDevmemCtx->hRefCount, 1);
+	psDevmemCtx->psDevNode = psDeviceNode;
+
+	/* Call down to MMU context creation */
+
+	eError = MMU_ContextCreate(psConnection,
+	                           psDeviceNode,
+	                           &psDevmemCtx->psMMUContext,
+	                           psMMUDevAttrs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_ContextCreate", fail_mmucontext);
+
+	if (psDeviceNode->pfnRegisterMemoryContext)
+	{
+		eError = psDeviceNode->pfnRegisterMemoryContext(psDeviceNode, psDevmemCtx->psMMUContext, &hPrivDataInt);
+		PVR_LOG_GOTO_IF_ERROR(eError, "pfnRegisterMemoryContext", fail_register);
+	}
+
+	/* Store the private data as it is required to unregister the memory context */
+	psDevmemCtx->hPrivData = hPrivDataInt;
+	*hPrivData = hPrivDataInt;
+	*ppsDevmemCtxPtr = psDevmemCtx;
+
+	/* Pass the CPU cache line size through the bridge to the user mode as it can't be queried in user mode.*/
+	*pui32CPUCacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+
+	/* Initialise the PID notify list */
+	dllist_init(&(psDevmemCtx->sProcessNotifyListHead));
+	psDevmemCtx->sPageFaultNotifyListElem.psNextNode = NULL;
+	psDevmemCtx->sPageFaultNotifyListElem.psPrevNode = NULL;
+
+	/* Initialise page fault address */
+	psDevmemCtx->sFaultAddress.uiAddr = 0ULL;
+
+	/* Initialise flags */
+	psDevmemCtx->ui32Flags = 0;
+
+	psDevmemCtx->uiCreatedHeaps = 0;
+
+	return PVRSRV_OK;
+
+fail_register:
+	MMU_ContextDestroy(psDevmemCtx->psMMUContext);
+fail_mmucontext:
+	OSFreeMem(psDevmemCtx);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapCreate
+@Description    Creates and initialises a device memory heap.
+@Return         valid Device Memory heap handle - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntHeapCreate(DEVMEMINT_CTX *psDevmemCtx,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    DEVMEMINT_HEAP **ppsDevmemHeapPtr)
+{
+	DEVMEMINT_HEAP *psDevmemHeap;
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sBlueprintHeapBaseAddr;
+	IMG_DEVMEM_SIZE_T uiBlueprintHeapLength;
+	IMG_DEVMEM_SIZE_T uiBlueprintResRgnLength;
+	IMG_UINT32 ui32BlueprintLog2DataPageSize;
+	IMG_UINT32 ui32BlueprintLog2ImportAlignment;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s", __func__));
+
+	if (!DevmemIntCtxAcquire(psDevmemCtx))
+	{
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+
+	/* Retrieve page size and base addr match the heap blueprint */
+	eError = HeapCfgHeapDetails(NULL,
+	                            psDevmemCtx->psDevNode,
+	                            uiHeapConfigIndex,
+	                            uiHeapIndex,
+	                            0, NULL,
+	                            &sBlueprintHeapBaseAddr,
+	                            &uiBlueprintHeapLength,
+	                            &uiBlueprintResRgnLength,
+	                            &ui32BlueprintLog2DataPageSize,
+	                            &ui32BlueprintLog2ImportAlignment);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get details for HeapConfig:%d HeapIndex:%d.",
+				 __func__, uiHeapConfigIndex, uiHeapIndex));
+		goto ErrorCtxRelease;
+	}
+
+	/* uiHeapConfigIndex and uiHeapIndex are validated in HeapCfgHeapDetails()
+	 * so it should be safe to use here without additional checks. We must assert
+	 * though that the index is less than the number of bits in uiCreatedHeaps
+	 * bitfield (we assume 8 bits in a byte and bitfield width of 64). */
+	PVR_ASSERT(uiHeapIndex < sizeof(psDevmemCtx->uiCreatedHeaps) * 8);
+
+	if (BIT_ISSET(psDevmemCtx->uiCreatedHeaps, uiHeapIndex))
+	{
+		eError = PVRSRV_ERROR_ALREADY_EXISTS;
+		goto ErrorCtxRelease;
+	}
+
+	/* allocate the heap object */
+	psDevmemHeap = OSAllocMem(sizeof(*psDevmemHeap));
+	PVR_LOG_GOTO_IF_NOMEM(psDevmemHeap, eError, ErrorCtxRelease);
+
+	psDevmemHeap->psDevmemCtx = psDevmemCtx;
+	psDevmemHeap->uiLog2PageSize = ui32BlueprintLog2DataPageSize;
+	psDevmemHeap->sBaseAddr = sBlueprintHeapBaseAddr;
+	/* Store the last accessible address as our LastAddr. We can access
+	 * every address between sBlueprintHeapBaseAddr and
+	 * sBlueprintHeapBaseAddr + uiBlueprintHeapLength - 1
+	 */
+	psDevmemHeap->sLastAddr.uiAddr = sBlueprintHeapBaseAddr.uiAddr + uiBlueprintHeapLength - 1;
+	psDevmemHeap->uiHeapIndex = uiHeapIndex;
+
+	OSAtomicWrite(&psDevmemHeap->uiRefCount, 1);
+
+	eError = HeapCfgGetCallbacks(psDevmemCtx->psDevNode,
+	                             uiHeapConfigIndex,
+	                             uiHeapIndex,
+	                             &psDevmemHeap->pfnInit,
+	                             &psDevmemHeap->pfnDeInit);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get callbacks for HeapConfig:%d HeapIndex:%d.",
+				 __func__, uiHeapConfigIndex, uiHeapIndex));
+		goto ErrorFreeDevmemHeap;
+	}
+
+	if (psDevmemHeap->pfnInit != NULL)
+	{
+		eError = psDevmemHeap->pfnInit(psDevmemCtx->psDevNode,
+		                               psDevmemHeap,
+		                               &psDevmemHeap->hPrivData);
+		PVR_GOTO_IF_ERROR(eError, ErrorFreeDevmemHeap);
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: sBaseAddr = %" IMG_UINT64_FMTSPECX ", "
+	        "sLastAddr = %" IMG_UINT64_FMTSPECX, __func__,
+	        psDevmemHeap->sBaseAddr.uiAddr, psDevmemHeap->sLastAddr.uiAddr));
+
+	BIT_SET(psDevmemCtx->uiCreatedHeaps, uiHeapIndex);
+
+	*ppsDevmemHeapPtr = psDevmemHeap;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevmemHeap:
+	OSFreeMem(psDevmemHeap);
+ErrorCtxRelease:
+	DevmemIntCtxRelease(psDevmemCtx);
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR ReserveRangeParamValidation(DEVMEMINT_HEAP *psDevmemHeap,
+                                                       IMG_DEV_VIRTADDR sReservationVAddr,
+                                                       IMG_DEVMEM_SIZE_T uiVirtualSize)
+{
+	IMG_DEV_VIRTADDR sLastReserveAddr;
+	IMG_UINT64 ui64InvalidSizeMask = (1 << psDevmemHeap->uiLog2PageSize) - 1;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap != NULL, "psDevmemHeap");
+
+	sLastReserveAddr.uiAddr = sReservationVAddr.uiAddr + uiVirtualSize - 1;
+
+	/* Check that the requested address is not less than the base address of the heap. */
+	if (sReservationVAddr.uiAddr < psDevmemHeap->sBaseAddr.uiAddr)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"sReservationVAddr ("IMG_DEV_VIRTADDR_FMTSPEC") is invalid! "
+			"Must be greater or equal to "IMG_DEV_VIRTADDR_FMTSPEC,
+			sReservationVAddr.uiAddr,
+			psDevmemHeap->sBaseAddr.uiAddr);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check the allocation size is valid (must be page granular). */
+	if ((uiVirtualSize & ui64InvalidSizeMask) != 0 || uiVirtualSize == 0)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"uiVirtualSize ("IMG_DEVMEM_SIZE_FMTSPEC") is invalid! Must a multiple of %u and greater than 0",
+			uiVirtualSize,
+			1 << psDevmemHeap->uiLog2PageSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiVirtualSize > PMR_MAX_SUPPORTED_SIZE)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"uiVirtualSize must be less than or equal to the max PMR size ("
+			IMG_DEVMEM_SIZE_FMTSPEC")",
+			PMR_MAX_SUPPORTED_SIZE);
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check that requested address + size fits in our heap. */
+	if (sLastReserveAddr.uiAddr > psDevmemHeap->sLastAddr.uiAddr)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"sReservationVAddr ("IMG_DEV_VIRTADDR_FMTSPEC") is invalid! "
+			"Must be lower than "IMG_DEV_VIRTADDR_FMTSPEC,
+			sReservationVAddr.uiAddr,
+			psDevmemHeap->sLastAddr.uiAddr - uiVirtualSize + 1);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemXIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                       IMG_DEV_VIRTADDR sReservationVAddr,
+                       IMG_DEVMEM_SIZE_T uiVirtualSize,
+                       DEVMEMXINT_RESERVATION **ppsRsrv)
+{
+	DEVMEMXINT_RESERVATION *psRsrv;
+	IMG_UINT32 uiNumPages;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(ppsRsrv != NULL);
+
+	eError = ReserveRangeParamValidation(psDevmemHeap,
+	                                     sReservationVAddr,
+	                                     uiVirtualSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "ReserveRangeParamValidation");
+
+
+	if (!DevmemIntHeapAcquire(psDevmemHeap))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, ErrorReturnError);
+	}
+
+	uiNumPages = uiVirtualSize >> psDevmemHeap->uiLog2PageSize;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psRsrv = OSAllocZMem(sizeof(*psRsrv) + sizeof(*psRsrv->ppsDevmemXMapping) * uiNumPages);
+	PVR_LOG_GOTO_IF_NOMEM(psRsrv, eError, ErrorUnreferenceHeap);
+#else
+	psRsrv = OSAllocZMem(sizeof(*psRsrv->ppsPMR) * uiNumPages + sizeof(*psRsrv));
+	PVR_LOG_GOTO_IF_NOMEM(psRsrv, eError, ErrorUnreferenceHeap);
+
+	psRsrv->ppsPMR = IMG_OFFSET_ADDR(psRsrv, sizeof(*psRsrv));
+#endif
+
+	OSAtomicWrite(&psRsrv->iLifetimeRefCount, 1);
+
+	eError = OSLockCreate(&psRsrv->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeReservation);
+
+	psRsrv->sBase = sReservationVAddr;
+	psRsrv->uiLength = uiVirtualSize;
+
+	eError = MMU_Alloc(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                   uiVirtualSize,
+	                   0, /* IMG_UINT32 uiProtFlags */
+	                   0, /* alignment is n/a since we supply devvaddr */
+	                   &sReservationVAddr,
+	                   psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrorDestroyLock);
+
+	/* since we supplied the virt addr, MMU_Alloc shouldn't have
+	   chosen a new one for us */
+	PVR_ASSERT(sReservationVAddr.uiAddr == psRsrv->sBase.uiAddr);
+
+	psRsrv->psDevmemHeap = psDevmemHeap;
+	*ppsRsrv = psRsrv;
+
+	return PVRSRV_OK;
+
+ErrorDestroyLock:
+	OSLockDestroy(psRsrv->hLock);
+ErrorFreeReservation:
+	OSFreeMem(psRsrv);
+ErrorUnreferenceHeap:
+	DevmemIntHeapRelease(psDevmemHeap);
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntUnreserveRange(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	IMG_UINT32 ui32FirstMappedIdx = IMG_UINT32_MAX; // Initialise with invalid value.
+	IMG_UINT32 ui32ContigPageCount = 0;
+	IMG_UINT32 ui32PageIdx;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* No need to lock the mapping here since this is a handle destruction path which can not be
+	 * executed while there are outstanding handle lookups, i.e. other operations are performed
+	 * on the mapping. Bridge and handle framework also make sure this path can also not be executed
+	 * concurrently. */
+
+	for (ui32PageIdx = 0; ui32PageIdx < _DevmemXReservationPageCount(psRsrv); ui32PageIdx++)
+	{
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		if (psRsrv->ppsDevmemXMapping[ui32PageIdx] != NULL)
+#else
+		if (psRsrv->ppsPMR[ui32PageIdx] != NULL)
+#endif
+		{
+			if (ui32ContigPageCount == 0)
+			{
+#if defined(DEBUG)
+				if (ui32FirstMappedIdx == IMG_UINT32_MAX)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Reservation was not unmapped! The reservation will "
+					         "be unmapped before proceeding.",
+					         __func__));
+				}
+#endif
+				ui32FirstMappedIdx = ui32PageIdx;
+			}
+
+			ui32ContigPageCount++;
+		}
+		else
+		{
+			if (ui32ContigPageCount != 0)
+			{
+				eError = DevmemXIntUnmapPages(psRsrv,
+				                              ui32FirstMappedIdx,
+				                              ui32ContigPageCount);
+				PVR_RETURN_IF_ERROR(eError);
+			}
+
+			ui32ContigPageCount = 0;
+		}
+	}
+
+	if (ui32ContigPageCount != 0)
+	{
+		eError = DevmemXIntUnmapPages(psRsrv,
+		                              ui32FirstMappedIdx,
+		                              ui32ContigPageCount);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	MMU_Free(psRsrv->psDevmemHeap->psDevmemCtx->psMMUContext,
+	         psRsrv->sBase,
+	         psRsrv->uiLength,
+	         psRsrv->psDevmemHeap->uiLog2PageSize);
+
+	/* Don't bother with refcount on reservation, as a reservation only ever
+	 * holds one mapping, so we directly decrement the refcount on the heap
+	 * instead.
+	 * Function will print an error if the heap could not be unreferenced. */
+	DevmemIntHeapRelease(psRsrv->psDevmemHeap);
+
+	/* We have partially destroyed the object but we may need to keep the wrapper object
+	 * around since it may be referenced in KM threads where we need the lock to avoid
+	 * race conditions. UM threads will no longer have a handle to this object when this
+	 * bridge call exits.
+	 */
+	_DevmemXReservationUnrefAndMaybeDestroy(psRsrv);
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+DevmemValidateFlags(PMR *psPMR, PVRSRV_MEMALLOCFLAGS_T uiMapFlags)
+{
+	PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (PVRSRV_CHECK_GPU_READABLE(uiMapFlags) && !PVRSRV_CHECK_GPU_READABLE(uiPMRFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not GPU readable.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+	if (PVRSRV_CHECK_GPU_WRITEABLE(uiMapFlags) && !PVRSRV_CHECK_GPU_WRITEABLE(uiPMRFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not GPU writeable.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+	if ((uiMapFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK) !=
+	    (uiPMRFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR's device specific flags don't match mapping flags.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntMapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                   PMR *psPMR,
+                   IMG_UINT32 uiPageCount,
+                   IMG_UINT32 uiPhysPageOffset,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   IMG_UINT32 uiVirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPMRMaxChunkCount = PMR_LogicalSize(psPMR) >> PMR_GetLog2Contiguity(psPMR);
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+	IMG_UINT32 uiLog2PageSize = psDevmemHeap->uiLog2PageSize;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	DEVMEMX_MAPPING *psNewMapping;
+	DEVMEMX_MAPPING *apsPotentialSplitMapping[DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT] = {0};
+	DEVMEMX_MAPPING *psCurrentMapping;
+	IMG_UINT32 uiLockedPMRDeferUnlockPhyCount = 0;
+#endif
+
+	/* Test uiPageCount+uiPhysPageOffset will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiPageCount + (IMG_UINT64)uiPhysPageOffset) <= (IMG_UINT64)IMG_UINT32_MAX, "uiPageCount+uiPhysPageOffset exceeds IMG_UINT32_MAX");
+	/* Test we do not exceed the PMR's maximum physical extent (in pages) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uiPageCount + uiPhysPageOffset) <= uiPMRMaxChunkCount, "uiPageCount+uiPhysPageOffset");
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	/* The range is not valid for the given virtual descriptor */
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+	PVR_LOG_RETURN_IF_FALSE((uiFlags & ~PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK) == 0,
+	                        "invalid flags", PVRSRV_ERROR_INVALID_FLAGS);
+	PVR_LOG_RETURN_IF_FALSE(!PMR_IsSparse(psPMR),
+		                    "PMR is Sparse, devmemx PMRs should be non-sparse", PVRSRV_ERROR_INVALID_FLAGS);
+	PVR_LOG_RETURN_IF_FALSE(!(PMR_Flags(psPMR) & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC),
+		                    "PMR allocation is deferred, devmemx PMRs can not be deferred", PVRSRV_ERROR_INVALID_FLAGS);
+
+	if (uiLog2PageSize > PMR_GetLog2Contiguity(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device heap and PMR have incompatible "
+		        "Log2Contiguity (%u - %u). PMR contiguity must be a multiple "
+		        "of the heap contiguity!", __func__, uiLog2PageSize,
+		        PMR_GetLog2Contiguity(psPMR)));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = DevmemValidateFlags(psPMR, uiFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemValidateFlags");
+
+	OSLockAcquire(psRsrv->hLock);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+
+	if (psCurrentMapping != NULL && /* Does a mapping currently exist in this entry */
+	   /* Does the start of the current mapping precede the requested offset */
+	   psCurrentMapping->uiVirtPageOffset < uiVirtPageOffset &&
+	   /* Does the end of the mapping precede the requested offset + page count */
+	   uiVirtPageOffset + uiPageCount < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount - 1))
+	{
+		apsPotentialSplitMapping[0] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[0] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlockRsrv);
+		}
+
+		apsPotentialSplitMapping[1] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[1] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrFreeFirstSplitMapping);
+		}
+	}
+	/* Lock address `uiPageCount` times. This will also take N references on
+	 * the PMR. */
+	eError = PMRLockSysPhysAddressesN(psPMR, uiPageCount);
+	PVR_GOTO_IF_ERROR(eError, ErrFreeSecondSplitMapping);
+
+	PMRLockPMR(psPMR);
+
+	psNewMapping = _AllocateDevmemXMapping(psRsrv,
+	                                       psPMR,
+	                                       uiFlags,
+	                                       uiPhysPageOffset,
+	                                       uiVirtPageOffset,
+	                                       uiPageCount);
+	if (!psNewMapping)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_AllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlockPMR);
+	}
+
+	do
+	{
+		eError = MMU_MapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+							  uiFlags,
+							  _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+							  psPMR,
+							  uiPhysPageOffset,
+							  uiPageCount,
+							  NULL,
+							  psDevmemHeap->uiLog2PageSize);
+		if (eError == PVRSRV_OK)
+		{
+			break;
+		}
+		else if (eError == PVRSRV_ERROR_RETRY)
+		{
+			/* This path may request this call to retry mapping at a later time, if this happens we
+			 * need to relinquish the synchronisation resources to allow migrate to complete and
+			 * try again
+			 */
+			_DeschedAllowRemapToComplete(psRsrv->hLock, psPMR);
+		}
+		else
+		{
+			/* Unexpected Error code path */
+			goto ErrFreeNewMapping;
+		}
+	} while (eError == PVRSRV_ERROR_RETRY);
+
+	/* If this has overwritten the middle of an existing mapping */
+	if (apsPotentialSplitMapping[0] != NULL)
+	{
+		_ReplaceCentreSplitExistingMapping(psRsrv,
+										   uiVirtPageOffset,
+										   uiPageCount,
+										   psNewMapping,
+										   apsPotentialSplitMapping,
+										   &uiLockedPMRDeferUnlockPhyCount);
+	}
+	else
+	{
+		_ReplaceReservationMappingRecords(psRsrv,
+										  uiVirtPageOffset,
+										  uiPageCount,
+										  psNewMapping,
+										  &uiLockedPMRDeferUnlockPhyCount);
+	}
+	PMRUnlockPMR(psPMR);
+
+	/* Drop any deferred references on PMR we had locked */
+	if (uiLockedPMRDeferUnlockPhyCount != 0)
+	{
+		PMRUnlockSysPhysAddressesN(psPMR, uiLockedPMRDeferUnlockPhyCount);
+	}
+
+#else
+
+	/* Lock address `uiPageCount` times. This will also take N references on
+	 * the PMR. */
+	eError = PMRLockSysPhysAddressesN(psPMR, uiPageCount);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlockRsrv);
+
+	PMRLockPMR(psPMR);
+
+	eError = MMU_MapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                      uiFlags,
+	                      _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                      psPMR,
+	                      uiPhysPageOffset,
+	                      uiPageCount,
+	                      NULL,
+	                      psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlockPMR);
+
+	PMRUnlockPMR(psPMR);
+
+	{
+		IMG_UINT32 i;
+		for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+		{
+			if (psRsrv->ppsPMR[i] != NULL)
+			{
+			{
+				PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddresses(psRsrv->ppsPMR[i]);
+				PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+			}
+			}
+
+			psRsrv->ppsPMR[i] = psPMR;
+		}
+	}
+#endif
+
+	OSLockRelease(psRsrv->hLock);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrFreeNewMapping:
+	_DestroyDevmemXMapping(psNewMapping);
+ErrUnlockPMR:
+	PMRUnlockPMR(psPMR);
+	{
+		PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddressesN(psPMR, uiPageCount);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+ErrFreeSecondSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[1]);
+ErrFreeFirstSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[0]);
+#else
+ErrUnlockPMR:
+	PMRUnlockPMR(psPMR);
+	{
+		PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddressesN(psPMR, uiPageCount);
+		PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+	}
+#endif
+ErrUnlockRsrv:
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntUnmapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                     IMG_UINT32 uiVirtPageOffset,
+                     IMG_UINT32 uiPageCount)
+{
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	DEVMEMX_MAPPING *apsPotentialSplitMapping[DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT] = {0};
+	DEVMEMX_MAPPING *psCurrentMapping;
+#endif
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+
+	OSLockAcquire(psRsrv->hLock);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+	if (psCurrentMapping != NULL &&/* Does a mapping currently exist in this entry */
+	    /* Does the start of the current mapping precede the requested offset */
+	    psCurrentMapping->uiVirtPageOffset < uiVirtPageOffset &&
+	    /* Does the end of the mapping precede the requested offset + page count */
+	    uiVirtPageOffset + uiPageCount < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount - 1))
+	{
+		apsPotentialSplitMapping[0] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[0] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlock);
+		}
+
+		apsPotentialSplitMapping[1] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[1] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrFreeFirstSplitMapping);
+		}
+	}
+#endif
+
+	/* Unmap the pages and mark them invalid in the MMU PTE */
+	eError = MMU_UnmapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                        0,
+	                        _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                        uiPageCount,
+	                        NULL,
+	                        psDevmemHeap->uiLog2PageSize,
+	                        0);
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrFreeSecondSplitMapping);
+
+	/* If this has overwritten the middle of an existing mapping */
+	if (apsPotentialSplitMapping[0] != NULL)
+	{
+		__maybe_unused IMG_UINT32 uiDeferUnlockPhyCount = 0;
+
+		_ReplaceCentreSplitExistingMapping(psRsrv,
+		                                   uiVirtPageOffset,
+		                                   uiPageCount,
+		                                   NULL,
+		                                   apsPotentialSplitMapping,
+		                                   &uiDeferUnlockPhyCount);
+		/* We don't have a PMR locked when calling this function so there will be
+		 * no UnlockPhys to defer.
+		 */
+		PVR_ASSERT(uiDeferUnlockPhyCount == 0);
+	}
+	else
+	{
+		_DeleteReservationMappingRecords(psRsrv,
+										 uiVirtPageOffset,
+										 uiPageCount);
+	}
+#else
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+	{
+		IMG_UINT32 i;
+		for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+		{
+			if (psRsrv->ppsPMR[i] != NULL)
+			{
+				PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddresses(psRsrv->ppsPMR[i]);
+				PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+
+				psRsrv->ppsPMR[i] = NULL;
+			}
+		}
+	}
+#endif
+
+	OSLockRelease(psRsrv->hLock);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrFreeSecondSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[1]);
+ErrFreeFirstSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[0]);
+#endif
+ErrUnlock:
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntMapVRangeToBackingPage(DEVMEMXINT_RESERVATION *psRsrv,
+                                 IMG_UINT32 uiPageCount,
+                                 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                 IMG_UINT32 uiVirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	DEVMEMX_MAPPING *apsPotentialSplitMapping[DEVMEMX_SPLIT_MAPPING_PREALLOC_COUNT] = {0};
+	DEVMEMX_MAPPING *psCurrentMapping;
+#endif
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	/* The range is not valid for the given virtual descriptor */
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+	PVR_LOG_RETURN_IF_FALSE((uiFlags & ~(PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK |
+	                                     PVRSRV_MEMALLOCFLAG_ZERO_BACKING)) == 0,
+	                        "invalid flags", PVRSRV_ERROR_INVALID_FLAGS);
+
+	OSLockAcquire(psRsrv->hLock);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psCurrentMapping = psRsrv->ppsDevmemXMapping[uiVirtPageOffset];
+	if (psCurrentMapping != NULL &&/* Does a mapping currently exist in this entry */
+	    /* Does the start of the current mapping precede the requested offset */
+	    psCurrentMapping->uiVirtPageOffset < uiVirtPageOffset &&
+	    /* Does the end of the mapping precede the requested offset + page count*/
+	    uiVirtPageOffset + uiPageCount < psCurrentMapping->uiVirtPageOffset + (psCurrentMapping->uiPageCount - 1))
+	{
+		apsPotentialSplitMapping[0] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[0] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrUnlock);
+		}
+
+		apsPotentialSplitMapping[1] = _PreAllocateDevmemXMapping(psRsrv);
+		if (apsPotentialSplitMapping[1] == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PreAllocateDevmemXMapping", eError, PVRSRV_ERROR_OUT_OF_MEMORY, ErrFreeFirstSplitMapping);
+		}
+	}
+#endif
+
+	eError = MMUX_MapVRangeToBackingPage(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                                     uiFlags,
+	                                     _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                                     uiPageCount,
+	                                     psDevmemHeap->uiLog2PageSize);
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrFreeSecondSplitMapping);
+
+	/* If this has overwritten the middle of an existing mapping */
+	if (apsPotentialSplitMapping[0] != NULL)
+	{
+		IMG_UINT32 uiDeferUnlockPhyCount = 0;
+
+		_ReplaceCentreSplitExistingMapping(psRsrv,
+										   uiVirtPageOffset,
+										   uiPageCount,
+										   NULL,
+										   apsPotentialSplitMapping,
+										   &uiDeferUnlockPhyCount);
+		/* We don't have a PMR locked when calling this function so there will be
+		 * no UnlockPhys to defer.
+		 */
+		PVR_ASSERT(uiDeferUnlockPhyCount == 0);
+	}
+	else
+	{
+		_DeleteReservationMappingRecords(psRsrv,
+										 uiVirtPageOffset,
+										 uiPageCount);
+	}
+#else
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+
+	{
+		IMG_UINT32 i;
+		for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+		{
+			if (psRsrv->ppsPMR[i] != NULL)
+			{
+				eError = PMRUnlockSysPhysAddresses(psRsrv->ppsPMR[i]);
+				PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+				psRsrv->ppsPMR[i] = NULL;
+			}
+		}
+	}
+#endif
+
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrFreeSecondSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[1]);
+ErrFreeFirstSplitMapping:
+	_FreePreAllocatedDevmemXMapping(apsPotentialSplitMapping[0]);
+#endif
+ErrUnlock:
+	OSLockRelease(psRsrv->hLock);
+	return eError;
+}
+
+static INLINE IMG_UINT32
+_DevmemReservationPageCount(DEVMEMINT_RESERVATION *psRsrv)
+{
+	return psRsrv->uiLength >> psRsrv->psDevmemHeap->uiLog2PageSize;
+}
+
+PVRSRV_ERROR
+DevmemIntMapPMR(DEVMEMINT_RESERVATION *psReservation, PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+	/* number of pages (device pages) that allocation spans */
+	IMG_UINT32 ui32NumDevPages;
+	/* device virtual address of start of allocation */
+	IMG_DEV_VIRTADDR sReservationVAddr;
+	/* and its length */
+	IMG_DEVMEM_SIZE_T uiVirtualSize;
+	IMG_UINT32 uiLog2HeapContiguity = psReservation->psDevmemHeap->uiLog2PageSize;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags = psReservation->uiFlags;
+	IMG_BOOL bIsSparse = IMG_FALSE;
+	void *pvTmpBuf = NULL;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psReservation->psMappedPMR == NULL, "psReservation");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(PMR_LogicalSize(psPMR) == psReservation->uiLength, "psPMR logical size");
+
+	if (uiLog2HeapContiguity > PMR_GetLog2Contiguity(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Device heap and PMR have incompatible contiguity (%u - %u). "
+		         "Heap contiguity must be a multiple of the heap contiguity!",
+		         __func__,
+		         uiLog2HeapContiguity,
+		         PMR_GetLog2Contiguity(psPMR) ));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, ErrorReturnError);
+	}
+
+	eError = DevmemValidateFlags(psPMR, uiMapFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemValidateFlags", ErrorReturnError);
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+
+	uiVirtualSize = psReservation->uiLength;
+
+	ui32NumDevPages = 0xffffffffU & ( ( (uiVirtualSize - 1) >> uiLog2HeapContiguity) + 1);
+	PVR_ASSERT((IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity == uiVirtualSize);
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorReleaseResLock);
+
+	PMRLockPMR(psPMR);
+
+	sReservationVAddr = psReservation->sBase;
+
+	/*Check if the PMR that needs to be mapped is sparse */
+	bIsSparse = PMR_IsSparse(psPMR);
+	if (bIsSparse)
+	{
+		IMG_DEV_PHYADDR *psDevPAddr;
+		IMG_BOOL *pbValid;
+
+		pvTmpBuf = OSAllocMem((sizeof(IMG_DEV_PHYADDR) + sizeof(IMG_BOOL)) *
+		                      ui32NumDevPages);
+		PVR_LOG_GOTO_IF_NOMEM(pvTmpBuf, eError, ErrorUnlockPhysAddr);
+
+		psDevPAddr = IMG_OFFSET_ADDR(pvTmpBuf, 0);
+		pbValid = IMG_OFFSET_ADDR(pvTmpBuf, ui32NumDevPages * sizeof(IMG_DEV_PHYADDR));
+
+		/* N.B. We pass mapping permission flags to MMU_MapPages and let
+		 * it reject the mapping if the permissions on the PMR are not compatible. */
+		eError = MMU_MapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                      uiMapFlags,
+		                      sReservationVAddr,
+		                      psPMR,
+		                      0,
+		                      ui32NumDevPages,
+		                      NULL,
+		                      uiLog2HeapContiguity);
+		PVR_GOTO_IF_ERROR(eError, ErrorFreeSparseTmpBuf);
+
+		/* Determine which entries of the PMR are valid */
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2HeapContiguity,
+		                         ui32NumDevPages,
+		                         0,
+		                         psDevPAddr,
+		                         pbValid,
+		                         DEVICE_USE | MAPPING_USE);
+		/* PVRSRV_ERROR_RETRY with SUPPORT_LINUX_OSPAGE_MIGRATION feature
+		 * enabled will not be returned here even with the use of
+		 * DEVICE_USE | MAPPING_USE because the feature does not
+		 * support migratable sparse PMRs.
+		 */
+		PVR_GOTO_IF_ERROR(eError, ErrorUnmap);
+
+		for (i = 0; i < ui32NumDevPages; i++)
+		{
+			if (DevmemIntReservationIsIndexMapped(psReservation, i))
+			{
+				eError = PMRUnrefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+			}
+
+			if (pbValid[i])
+			{
+				eError = PMRRefPMR(psPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRRefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_TRUE);
+			}
+		}
+
+		OSFreeMem(pvTmpBuf);
+	}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	else
+	{
+		do
+		{
+			eError = MMU_MapPMRFast(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+			                        sReservationVAddr,
+			                        psPMR,
+			                        (IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity,
+			                        uiMapFlags,
+			                        uiLog2HeapContiguity);
+			if (eError == PVRSRV_OK)
+			{
+				break;
+			}
+			else if (eError == PVRSRV_ERROR_RETRY)
+			{
+				/* This path may request this call to retry mapping at a later time, if this happens we
+				 * need to relinquish the synchronisation resources to allow migrate to complete and
+				 * try again
+				 */
+				_DeschedAllowRemapToCompleteNested(psReservation->hLock,
+				                                   psReservation->eLockClass,
+				                                   psPMR);
+			}
+			else
+			{
+				/* Unexpected Error code path */
+				goto ErrorUnlockPhysAddr;
+			}
+		} while (eError == PVRSRV_ERROR_RETRY);
+	}
+#else
+	else
+	{
+		eError = MMU_MapPMRFast(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        sReservationVAddr,
+		                        psPMR,
+		                        (IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity,
+		                        uiMapFlags,
+		                        uiLog2HeapContiguity);
+		PVR_GOTO_IF_ERROR(eError, ErrorUnlockPhysAddr);
+	}
+#endif
+
+	psReservation->psMappedPMR = psPMR;
+
+	/* Increase reservation association count so we know if multiple mappings have been created
+	 * on the PMR, also link the mapping to the PMR if required.
+	 */
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psReservation->psDevmemMapping = _AllocateDevmemMapping(psReservation,
+	                                                        psPMR);
+	PVR_LOG_GOTO_IF_NOMEM(psReservation->psDevmemMapping, eError, ErrorUnsetMappedPMR);
+	_LinkPMRRsrvNode(&psReservation->psDevmemMapping->sPMRRsrvLinkNode, psPMR);
+#else
+	PMRLinkGPUMapping(psPMR);
+#endif
+
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(psReservation->hLock);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrorUnsetMappedPMR:
+	psReservation->psMappedPMR = NULL;
+#endif
+ErrorUnmap:
+	(void) MMU_UnmapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+	                      0,
+	                      sReservationVAddr,
+	                      ui32NumDevPages,
+	                      NULL,
+	                      uiLog2HeapContiguity,
+	                      0);
+ErrorFreeSparseTmpBuf:
+	if (pvTmpBuf)
+	{
+		OSFreeMem(pvTmpBuf);
+	}
+
+ErrorUnlockPhysAddr:
+	PMRUnlockPMR(psPMR);
+	{
+		PVRSRV_ERROR eError1 = PVRSRV_OK;
+		eError1 = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError1, "PMRUnlockSysPhysAddresses");
+	}
+
+ErrorReleaseResLock:
+	OSLockRelease(psReservation->hLock);
+
+ErrorReturnError:
+	PVR_ASSERT (eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntGetReservationData(DEVMEMINT_RESERVATION* psReservation, PMR** ppsPMR, IMG_DEV_VIRTADDR* psDevVAddr)
+{
+	/* Reservation might not have a PMR if a mapping was not yet performed */
+	if (psReservation->psMappedPMR == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevVAddr->uiAddr = psReservation->sBase.uiAddr;
+	*ppsPMR = psReservation->psMappedPMR;
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntReserveRangeAndMapPMR(CONNECTION_DATA *psConnectionData,
+                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                               DEVMEMINT_HEAP *psDevmemHeap,
+                               IMG_DEV_VIRTADDR sReservationVAddr,
+                               IMG_DEVMEM_SIZE_T uiVirtualSize,
+                               PMR *psPMR,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               DEVMEMINT_RESERVATION **ppsReservation)
+{
+	PVRSRV_ERROR eError, eUnreserveError;
+
+	eError = DevmemIntReserveRange(psConnectionData,
+	                               psDeviceNode,
+	                               psDevmemHeap,
+	                               sReservationVAddr,
+	                               uiVirtualSize,
+	                               uiFlags,
+	                               ppsReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorReturnError);
+
+	eError = DevmemIntMapPMR(*ppsReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	eUnreserveError = DevmemIntUnreserveRange(*ppsReservation);
+	*ppsReservation = NULL;
+	PVR_LOG_IF_ERROR(eUnreserveError, "DevmemIntUnreserveRange");
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnmapPMR(DEVMEMINT_RESERVATION *psReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeap = psReservation->psDevmemHeap;
+	/* device virtual address of start of allocation */
+	IMG_DEV_VIRTADDR sReservationVAddr;
+	/* number of pages (device pages) that allocation spans */
+	IMG_UINT32 ui32NumDevPages;
+	IMG_BOOL bIsSparse = IMG_FALSE;
+	IMG_UINT32 i;
+
+	PVR_RETURN_IF_INVALID_PARAM(psReservation->psMappedPMR != NULL);
+
+	ui32NumDevPages = _DevmemReservationPageCount(psReservation);
+	sReservationVAddr = psReservation->sBase;
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+	PMRLockPMR(psReservation->psMappedPMR);
+
+	bIsSparse = PMR_IsSparse(psReservation->psMappedPMR);
+
+	if (bIsSparse)
+	{
+		eError = MMU_UnmapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        0,
+		                        sReservationVAddr,
+		                        ui32NumDevPages,
+		                        NULL,
+		                        psDevmemHeap->uiLog2PageSize,
+		                        0);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+
+		/* We are unmapping the whole PMR */
+		for (i = 0; i < ui32NumDevPages; i++)
+		{
+			if (DevmemIntReservationIsIndexMapped(psReservation, i))
+			{
+				/* All PMRs in the range should be the same, set local PMR
+				 * for Unlocking phys addrs later */
+				eError = PMRUnrefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+			}
+		}
+	}
+	else
+	{
+		eError = MMU_UnmapPMRFast(psDevmemHeap->psDevmemCtx->psMMUContext,
+		                          sReservationVAddr,
+		                          ui32NumDevPages,
+		                          psDevmemHeap->uiLog2PageSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPMRFast", ErrUnlock);
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	_UnlinkPMRRsrvNode(&psReservation->psDevmemMapping->sPMRRsrvLinkNode, psReservation->psMappedPMR);
+	PMRUnlockPMR(psReservation->psMappedPMR);
+
+	_UnrefAndMaybeDestroyPMRRsrvNode(&psReservation->psDevmemMapping->sPMRRsrvLinkNode);
+#else
+	PMRUnlinkGPUMapping(psReservation->psMappedPMR);
+	PMRUnlockPMR(psReservation->psMappedPMR);
+#endif
+
+	eError = PMRUnlockSysPhysAddresses(psReservation->psMappedPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+
+	psReservation->psMappedPMR = NULL;
+
+	OSLockRelease(psReservation->hLock);
+
+	return PVRSRV_OK;
+
+ErrUnlock:
+	PMRUnlockPMR(psReservation->psMappedPMR);
+	OSLockRelease(psReservation->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntReserveRange(CONNECTION_DATA *psConnectionData,
+                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                      DEVMEMINT_HEAP *psDevmemHeap,
+                      IMG_DEV_VIRTADDR sReservationVAddr,
+                      IMG_DEVMEM_SIZE_T uiVirtualSize,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      DEVMEMINT_RESERVATION **ppsReservationPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservation;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT64 ui64MapSize;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_ASSERT(ppsReservationPtr != NULL);
+
+	eError = ReserveRangeParamValidation(psDevmemHeap,
+	                                     sReservationVAddr,
+	                                     uiVirtualSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "ReserveRangeParamValidation");
+
+	if (!DevmemIntHeapAcquire(psDevmemHeap))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW,
+		                    ErrorReturnError);
+	}
+
+	uiNumPages = uiVirtualSize >> psDevmemHeap->uiLog2PageSize;
+
+	/* allocate memory to record the reservation info */
+	ui64MapSize = sizeof(*psReservation->pui8Map) * LOG2_DIV(uiNumPages, MAP_MASK_SHIFT);
+	if (LOG2_MOD(uiNumPages, MAP_MASK_SHIFT) != 0)
+	{
+		ui64MapSize += 1;
+	}
+
+	psReservation = OSAllocZMem(sizeof(*psReservation) + ui64MapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psReservation, eError, ErrorUnreference);
+
+	/* Create lock */
+	eError = OSLockCreate(&psReservation->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeReservation);
+
+	/* Determine where the caller originated from. This is to prevent lockdep
+	 * warnings between reservations that originate from different sides of the
+	 * bridge. */
+	if (psConnectionData == NULL)
+	{
+		psReservation->eLockClass = RESERVATION_USAGE_SERVER_FW;
+	}
+	else
+	{
+		psReservation->eLockClass = RESERVATION_USAGE_CLIENT;
+	}
+
+	/* Initialise external device resource use indicator */
+	psReservation->i32DevResAcquisitionCount = DEVMEMRESERVATION_ACQUISITION_MIN;
+	/* Initialise UM-KM lifetime refcount */
+	OSAtomicWrite(&psReservation->iLifetimeRefCount, 1);
+
+	psReservation->uiFlags = uiFlags;
+	psReservation->sBase = sReservationVAddr;
+	psReservation->uiLength = uiVirtualSize;
+	psReservation->pui8Map = IMG_OFFSET_ADDR(psReservation, sizeof(*psReservation));
+
+	eError = MMU_Alloc(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                   uiVirtualSize,
+	                   0, /* IMG_UINT32 uiProtFlags */
+	                   0, /* alignment is n/a since we supply devvaddr */
+	                   &sReservationVAddr,
+	                   psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrorDestroyLock);
+
+	/* since we supplied the virt addr, MMU_Alloc shouldn't have
+	   chosen a new one for us */
+	PVR_ASSERT(sReservationVAddr.uiAddr == psReservation->sBase.uiAddr);
+
+	psReservation->psDevmemHeap = psDevmemHeap;
+	*ppsReservationPtr = psReservation;
+
+	return PVRSRV_OK;
+
+	/*
+	 *  error exit paths follow
+	 */
+
+ErrorDestroyLock:
+	OSLockDestroy(psReservation->hLock);
+
+ErrorFreeReservation:
+	OSFreeMem(psReservation);
+
+ErrorUnreference:
+	/* if fails there's not much to do (the function will print an error) */
+	DevmemIntHeapRelease(psDevmemHeap);
+
+ErrorReturnError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnreserveRange(DEVMEMINT_RESERVATION *psReservation)
+{
+	IMG_UINT32 i;
+	DEVMEMINT_HEAP *psDevmemHeap = psReservation->psDevmemHeap;
+	PVRSRV_ERROR eError;
+
+	if (psReservation->psMappedPMR != NULL)
+	{
+		/* No warning to be emitted as this is expected behaviour for the
+		 * Devmem interface. */
+		eError = DevmemIntUnmapPMR(psReservation);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevmemIntUnmapPMR");
+	}
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+
+	if (psReservation->i32DevResAcquisitionCount != DEVMEMRESERVATION_ACQUISITION_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but still has existing dev res acquisitions "
+		         "(%d), free existing acquisitions first.", __func__,
+		         psReservation->i32DevResAcquisitionCount));
+
+		OSLockRelease(psReservation->hLock);
+
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	OSLockRelease(psReservation->hLock);
+
+	MMU_Free(psDevmemHeap->psDevmemCtx->psMMUContext,
+	         psReservation->sBase,
+	         psReservation->uiLength,
+	         psDevmemHeap->uiLog2PageSize);
+
+	for (i = 0; i < _DevmemReservationPageCount(psReservation); i++)
+	{
+		if (DevmemIntReservationIsIndexMapped(psReservation, i))
+		{
+			eError = PMRUnrefPMR(psReservation->psMappedPMR);
+			PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+			DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+		}
+	}
+
+	DevmemIntHeapRelease(psDevmemHeap);
+
+	/* We have partially destroyed the object but we may need to keep the wrapper object
+	 * around since it may be referenced in KM threads where we need the lock to avoid
+	 * race conditions. UM threads will no longer have a handle to this object when this
+	 * bridge call exits.
+	 */
+	_DevmemReservationUnrefAndMaybeDestroy(psReservation);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+DevmemIntHeapDestroy(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	if (psDevmemHeap->pfnDeInit != NULL)
+	{
+		psDevmemHeap->pfnDeInit(psDevmemHeap->hPrivData);
+		psDevmemHeap->pfnDeInit = NULL;
+	}
+
+	if (OSAtomicRead(&psDevmemHeap->uiRefCount) != DEVMEMHEAP_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but still has existing references "
+		         "(%d), free existing reservations & mappings first.", __func__,
+		         OSAtomicRead(&psDevmemHeap->uiRefCount)));
+
+		/*
+		 * Try again later when you've freed all the memory
+		 *
+		 * Note:
+		 * We don't expect the application to retry (after all this call would
+		 * succeed if the client had freed all the memory which it should have
+		 * done before calling this function). However, given there should be
+		 * an associated handle, when the handle base is destroyed it will free
+		 * any allocations leaked by the client and then it will retry this call,
+		 * which should then succeed.
+		 */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	PVR_ASSERT(OSAtomicRead(&psDevmemHeap->uiRefCount) == DEVMEMHEAP_REFCOUNT_MIN);
+
+	BIT_UNSET(psDevmemHeap->psDevmemCtx->uiCreatedHeaps, psDevmemHeap->uiHeapIndex);
+
+	DevmemIntCtxRelease(psDevmemHeap->psDevmemCtx);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Freed heap %p", __func__, psDevmemHeap));
+	OSFreeMem(psDevmemHeap);
+
+	return PVRSRV_OK;
+}
+
+IMG_DEV_VIRTADDR
+DevmemIntHeapGetBaseAddr(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	PVR_ASSERT(psDevmemHeap != NULL);
+
+	return psDevmemHeap->sBaseAddr;
+}
+
+static PVRSRV_ERROR
+DevmemIntChangeSparseValidateParams(PMR *psPMR,
+                                    IMG_UINT32 ui32AllocPageCount,
+                                    IMG_UINT32 ui32FreePageCount,
+                                    IMG_UINT32 ui32LogicalChunkCount,
+                                    SPARSE_MEM_RESIZE_FLAGS uiSparseFlags)
+{
+	PMRLockHeldAssert(psPMR);
+
+	/* Ensure a PMR has been mapped to this reservation. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiSparseFlags & SPARSE_RESIZE_BOTH, "uiSparseFlags");
+
+	if (!PMR_IsSparse(psPMR) || PMR_IsMemLayoutFixed(psPMR) ||
+	    PMR_IsClientCpuMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR cannot be changed because one or more of the following"
+		         " were true: !PMR_IsSparse() = %s, PMR_IsMemLayoutFixed() = %s,"
+		         " PMR_IsClientCpuMapped() = %s",
+		         __func__,
+		         !PMR_IsSparse(psPMR) ? "true" : "false",
+		         PMR_IsMemLayoutFixed(psPMR) ? "true" : "false",
+		         PMR_IsClientCpuMapped(psPMR) ? "true" : "false"));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (PMR_IsGpuMultiMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR cannot be changed because PMR_IsGpuMultiMapped() = true",
+		         __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		PVR_LOG_RETURN_IF_INVALID_PARAM(ui32AllocPageCount != 0, "ui32AllocPageCount");
+		PVR_LOG_RETURN_IF_FALSE(ui32AllocPageCount <= ui32LogicalChunkCount,
+		                        "ui32AllocPageCount is invalid",
+		                        PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE);
+	}
+
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		PVR_LOG_RETURN_IF_INVALID_PARAM(ui32FreePageCount != 0, "ui32FreePageCount");
+		PVR_LOG_RETURN_IF_FALSE(ui32FreePageCount <= ui32LogicalChunkCount,
+		                        "ui32FreePageCount is invalid",
+		                        PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+DevmemIntValidateSparsePMRIndices(IMG_UINT32 ui32PMRLogicalChunkCount,
+                                  IMG_UINT32 *paui32LogicalIndices,
+                                  IMG_UINT32 ui32LogicalIndexCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT8 *paui8TrackedIndices;
+	IMG_UINT32 ui32AllocSize;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32PMRLogicalChunkCount != 0);
+	PVR_ASSERT(paui32LogicalIndices != NULL);
+	PVR_ASSERT(ui32LogicalIndexCount != 0 && ui32LogicalIndexCount <= ui32PMRLogicalChunkCount);
+
+	ui32AllocSize = LOG2_DIV(ui32PMRLogicalChunkCount, 3);
+	if (LOG2_MOD(ui32PMRLogicalChunkCount, 3) != 0)
+	{
+		++ui32AllocSize;
+	}
+
+	paui8TrackedIndices = OSAllocZMem(ui32AllocSize);
+	if (paui8TrackedIndices == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	for (i = 0; i < ui32LogicalIndexCount; i++)
+	{
+		IMG_UINT32 ui32LogicalIndex = paui32LogicalIndices[i];
+
+		if (ui32LogicalIndex >= ui32PMRLogicalChunkCount)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Reservation index %u is OOB",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+			break;
+		}
+
+		if (BIT_ISSET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3)))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Duplicate index found: %u",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+
+		BIT_SET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3));
+	}
+
+	OSFreeMem(paui8TrackedIndices);
+	return eError;
+}
+
+static PVRSRV_ERROR
+DevmemIntComputeVirtualIndicesFromLogical(DEVMEMINT_RESERVATION *psReservation,
+                                          IMG_UINT32 ui32LogicalIndexCount,
+                                          IMG_UINT32 *paui32LogicalIndices,
+                                          IMG_UINT32 *pui32VirtualIndexCount,
+                                          IMG_UINT32 **ppaui32VirtualIndices)
+{
+	IMG_UINT32 ui32OrderDiff = PMR_GetLog2Contiguity(psReservation->psMappedPMR) -
+		psReservation->psDevmemHeap->uiLog2PageSize;
+
+	if (ui32OrderDiff == 0)
+	{
+		*pui32VirtualIndexCount = ui32LogicalIndexCount;
+		*ppaui32VirtualIndices = paui32LogicalIndices;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		IMG_UINT32 ui32PagesPerOrder = 1 << ui32OrderDiff;
+		IMG_UINT32 *paui32VirtualIndices;
+		IMG_UINT32 i = 0;
+
+		paui32VirtualIndices = OSAllocMem(*pui32VirtualIndexCount * sizeof(IMG_UINT32));
+		PVR_RETURN_IF_NOMEM(paui32VirtualIndices);
+
+		for (i = 0; i < ui32LogicalIndexCount; i++)
+		{
+			IMG_UINT32 ui32LogicalIndex = paui32LogicalIndices[i];
+			IMG_UINT32 ui32PageOffset;
+
+			for (ui32PageOffset = 0; ui32PageOffset < ui32PagesPerOrder; ui32PageOffset++)
+			{
+				IMG_UINT32 ui32VirtAddr = ui32LogicalIndex * ui32PagesPerOrder + ui32PageOffset;
+				paui32VirtualIndices[i * ui32PagesPerOrder + ui32PageOffset] = ui32VirtAddr;
+			}
+		}
+
+		*ppaui32VirtualIndices = paui32VirtualIndices;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntChangeSparse(IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiSparseFlags,
+                      DEVMEMINT_RESERVATION *psReservation)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	IMG_UINT32 ui32Log2PMRContiguity;
+	IMG_UINT32 ui32Log2HeapContiguity;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+
+	IMG_UINT32 *pai32MapIndices;
+	IMG_UINT32 *pai32UnmapIndices;
+	IMG_UINT32 uiMapPageCount;
+	IMG_UINT32 uiUnmapPageCount;
+
+	IMG_UINT32 ui32LogicalChunkCount;
+	PMR *psPMR;
+
+	OSLockAcquireNested(psReservation->hLock, psReservation->eLockClass);
+
+	uiFlags = psReservation->uiFlags;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psReservation->psMappedPMR != NULL, eError, InvalidPMRErr);
+	psPMR = psReservation->psMappedPMR;
+
+	PMRLockPMR(psPMR);
+
+	ui32LogicalChunkCount = PMR_GetLogicalChunkCount(psPMR);
+	ui32Log2PMRContiguity = PMR_GetLog2Contiguity(psPMR);
+	ui32Log2HeapContiguity = psReservation->psDevmemHeap->uiLog2PageSize;
+
+	eError = DevmemIntChangeSparseValidateParams(psPMR,
+	                                             ui32AllocPageCount,
+	                                             ui32FreePageCount,
+	                                             ui32LogicalChunkCount,
+	                                             uiSparseFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntChangeSparseValidateParams", e0);
+
+	/* This is check is made in DevmemIntMapPMR - no need to do it again in release. */
+	PVR_ASSERT(ui32Log2HeapContiguity <= ui32Log2PMRContiguity);
+
+	pai32MapIndices = pai32AllocIndices;
+	pai32UnmapIndices = pai32FreeIndices;
+	uiMapPageCount = ui32AllocPageCount;
+	uiUnmapPageCount = ui32FreePageCount;
+
+	/* Pre check free indices against reservation given */
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		IMG_UINT32 i;
+
+		eError = DevmemIntValidateSparsePMRIndices(ui32LogicalChunkCount,
+		                                           pai32FreeIndices,
+		                                           ui32FreePageCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntValidateSparsePMRIndices", e0);
+
+		eError = DevmemIntComputeVirtualIndicesFromLogical(psReservation,
+		                                                   ui32FreePageCount,
+		                                                   pai32FreeIndices,
+		                                                   &uiUnmapPageCount,
+		                                                   &pai32UnmapIndices);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntComputeVirtualIndicesFromLogical", e1);
+
+		for (i = 0; i < uiUnmapPageCount; i++)
+		{
+			IMG_UINT32 ui32VirtIndex = pai32UnmapIndices[i];
+			IMG_BOOL bIsMapped;
+
+			bIsMapped = DevmemIntReservationIsIndexMapped(psReservation,
+			                                              ui32VirtIndex);
+			if (!bIsMapped)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Reservation index %u is not mapped into the reservation",
+				         __func__,
+				         ui32VirtIndex));
+
+				PVR_GOTO_WITH_ERROR(eError,
+				                    PVRSRV_ERROR_DEVICEMEM_NO_MAPPING,
+				                    e1);
+			}
+		}
+	}
+
+	/* Pre check alloc indices against reservation given */
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		IMG_UINT32 i;
+
+		eError = DevmemIntValidateSparsePMRIndices(ui32LogicalChunkCount,
+		                                           pai32AllocIndices,
+		                                           ui32AllocPageCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntValidateSparsePMRIndices", e1);
+
+		eError = DevmemIntComputeVirtualIndicesFromLogical(psReservation,
+		                                                   ui32AllocPageCount,
+		                                                   pai32AllocIndices,
+		                                                   &uiMapPageCount,
+		                                                   &pai32MapIndices);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntComputeVirtualIndicesFromLogical", e1);
+
+		for (i = 0; i < uiMapPageCount; i++)
+		{
+			IMG_UINT32 ui32VirtIndex = pai32MapIndices[i];
+			IMG_BOOL bIsMapped;
+
+			bIsMapped = DevmemIntReservationIsIndexMapped(psReservation,
+			                                              ui32VirtIndex);
+			if (bIsMapped)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Reservation index %u is mapped into the reservation",
+				         __func__,
+				         ui32VirtIndex));
+
+				PVR_GOTO_WITH_ERROR(eError,
+				                    PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED,
+				                    e1);
+			}
+		}
+	}
+
+	/* Invalidate the page table entries before freeing the physical pages. */
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		PMR_FLAGS_T uiPMRFlags;
+		IMG_UINT32 i;
+
+		/*Get the flags*/
+		uiPMRFlags = PMR_Flags(psPMR);
+
+		/* Unmap the pages and mark them invalid in the MMU PTE */
+		eError = MMU_UnmapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        uiFlags,
+		                        psReservation->sBase,
+		                        uiUnmapPageCount,
+		                        pai32UnmapIndices,
+		                        ui32Log2HeapContiguity,
+		                        uiPMRFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", e1);
+
+		for (i = 0; i < uiUnmapPageCount; i++)
+		{
+			IMG_UINT32 uiIndex = pai32UnmapIndices[i];
+
+			if (DevmemIntReservationIsIndexMapped(psReservation, uiIndex))
+			{
+				eError = PMRUnrefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation,
+				                                    uiIndex,
+				                                    IMG_FALSE);
+			}
+		}
+	}
+
+	/* Do the PMR specific changes */
+	eError = PMR_ChangeSparseMemUnlocked(psPMR,
+	                                     ui32AllocPageCount,
+	                                     pai32AllocIndices,
+	                                     ui32FreePageCount,
+	                                     pai32FreeIndices,
+	                                     uiSparseFlags);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: Failed to do PMR specific changes.",
+		         __func__));
+		goto e1;
+	}
+
+	/* Wire the pages tables that got allocated */
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		IMG_UINT32 i;
+
+		/* Map the pages and mark them Valid in the MMU PTE */
+		eError = MMU_MapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                      uiFlags,
+		                      psReservation->sBase,
+		                      psPMR,
+		                      0,
+		                      uiMapPageCount,
+		                      pai32MapIndices,
+		                      ui32Log2HeapContiguity);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: Failed to map alloc indices.",
+			         __func__));
+			goto e1;
+		}
+
+		for (i = 0; i < uiMapPageCount; i++)
+		{
+			IMG_UINT32 uiIndex = pai32MapIndices[i];
+
+			if (!DevmemIntReservationIsIndexMapped(psReservation, uiIndex))
+			{
+				PVRSRV_ERROR eError2 = PMRRefPMR(psReservation->psMappedPMR);
+				PVR_LOG_IF_ERROR(eError2, "PMRRefPMR");
+
+				DevmemIntReservationSetMappingIndex(psReservation,
+				                                    uiIndex,
+				                                    IMG_TRUE);
+			}
+		}
+	}
+
+e1:
+	if (pai32MapIndices != pai32AllocIndices)
+	{
+		OSFreeMem(pai32MapIndices);
+	}
+	if (pai32UnmapIndices != pai32FreeIndices)
+	{
+		OSFreeMem(pai32UnmapIndices);
+	}
+e0:
+	PMRUnlockPMR(psPMR);
+InvalidPMRErr:
+	OSLockRelease(psReservation->hLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxDestroy
+@Description    Destroy that created by DevmemIntCtxCreate
+@Input          psDevmemCtx   Device Memory context
+@Return         cannot fail.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntCtxDestroy(DEVMEMINT_CTX *psDevmemCtx)
+{
+	/*
+	   We can't determine if we should be freeing the context here
+	   as a refcount!=1 could be due to either the fact that heap(s)
+	   remain with allocations on them, or that this memory context
+	   has been exported.
+	   As the client couldn't do anything useful with this information
+	   anyway and the fact that the refcount will ensure we only
+	   free the context when _all_ references have been released
+	   don't bother checking and just return OK regardless.
+	   */
+	DevmemIntCtxRelease(psDevmemCtx);
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntGetVDevAddrPageSize
+@Description    Get the page size for a virtual address.
+@Input			psDevNode
+@Input			sDevAddr			Get the page size for this virtual address.
+@Output			puiLog2HeapPageSize	On success returns log2 of the page size.
+@Return         Failure code if the virtual address is outside any heap.
+*/ /**************************************************************************/
+static
+PVRSRV_ERROR DevmemIntGetVDevAddrPageSize(PVRSRV_DEVICE_NODE *psDevNode,
+										  IMG_DEV_VIRTADDR sDevAddr,
+										  IMG_PUINT32 puiLog2HeapPageSize)
+{
+	IMG_UINT32 i, j, uiLog2HeapPageSize = 0;
+	DEVICE_MEMORY_INFO *psDinfo = &psDevNode->sDevMemoryInfo;
+	DEVMEM_HEAP_CONFIG *psConfig = psDinfo->psDeviceMemoryHeapConfigArray;
+
+	IMG_BOOL bFound = IMG_FALSE;
+
+	for (i = 0;
+		 i < psDinfo->uiNumHeapConfigs && !bFound;
+		 i++)
+	{
+		for (j = 0;
+			 j < psConfig[i].uiNumHeaps  && !bFound;
+			 j++)
+		{
+			IMG_DEV_VIRTADDR uiBase =
+					psConfig[i].psHeapBlueprintArray[j].sHeapBaseAddr;
+			IMG_DEVMEM_SIZE_T uiSize =
+					psConfig[i].psHeapBlueprintArray[j].uiHeapLength;
+
+			if ((sDevAddr.uiAddr >= uiBase.uiAddr) &&
+				(sDevAddr.uiAddr < (uiBase.uiAddr + uiSize)))
+			{
+				uiLog2HeapPageSize =
+						psConfig[i].psHeapBlueprintArray[j].uiLog2DataPageSize;
+				bFound = IMG_TRUE;
+			}
+		}
+	}
+
+	if (uiLog2HeapPageSize == 0)
+	{
+		return PVRSRV_ERROR_INVALID_GPU_ADDR;
+	}
+
+	*puiLog2HeapPageSize = uiLog2HeapPageSize;
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntIsVDevAddrValid
+@Description    Checks if a virtual address is valid for access.
+@Input          psConnection
+@Input			psDevNode
+@Input			psDevmemCtx			Device Memory context
+@Input			sDevAddr			Virtual address to check.
+@Return         Failure code if the virtual address is invalid.
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntIsVDevAddrValid(CONNECTION_DATA * psConnection,
+                                      PVRSRV_DEVICE_NODE *psDevNode,
+                                      DEVMEMINT_CTX *psDevMemContext,
+                                      IMG_DEV_VIRTADDR sDevAddr)
+{
+	IMG_UINT32 uiLog2HeapPageSize = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = DevmemIntGetVDevAddrPageSize(psDevNode,
+										  sDevAddr,
+										  &uiLog2HeapPageSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	return MMU_IsVDevAddrValid(psDevMemContext->psMMUContext,
+	                           uiLog2HeapPageSize,
+	                           sDevAddr) ? PVRSRV_OK : PVRSRV_ERROR_INVALID_GPU_ADDR;
+}
+
+PVRSRV_ERROR
+DevmemIntInvalidateFBSCTable(DEVMEMINT_CTX *psDevMemContext,
+                             IMG_UINT64 ui64FBSCEntryMask)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psDevMemContext->psDevNode;
+	MMU_CONTEXT *psMMUContext = psDevMemContext->psMMUContext;
+
+	if (psDevNode->pfnInvalFBSCTable)
+	{
+		return psDevNode->pfnInvalFBSCTable(psDevNode,
+		                                    psMMUContext,
+		                                    ui64FBSCEntryMask);
+	}
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR DevmemIntGetFaultAddress(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDevNode,
+                                      DEVMEMINT_CTX *psDevMemContext,
+                                      IMG_DEV_VIRTADDR *psFaultAddress)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	if ((psDevMemContext->ui32Flags & DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE) == 0)
+	{
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	*psFaultAddress = psDevMemContext->sFaultAddress;
+	psDevMemContext->ui32Flags &= ~DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE;
+
+	return PVRSRV_OK;
+}
+
+static POSWR_LOCK g_hExportCtxListLock;
+static DLLIST_NODE g_sExportCtxList;
+
+PVRSRV_ERROR
+DevmemIntInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	dllist_init(&g_sExportCtxList);
+
+	eError = OSWRLockCreate(&g_hExportCtxListLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntDeInit(void)
+{
+	PVR_ASSERT(dllist_is_empty(&g_sExportCtxList));
+
+	OSWRLockDestroy(g_hExportCtxListLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntExportCtx(DEVMEMINT_CTX *psContext,
+                   PMR *psPMR,
+                   DEVMEMINT_CTX_EXPORT **ppsContextExport)
+{
+	DEVMEMINT_CTX_EXPORT *psCtxExport;
+	PVRSRV_ERROR eError;
+
+	psCtxExport = OSAllocMem(sizeof(DEVMEMINT_CTX_EXPORT));
+	PVR_LOG_RETURN_IF_NOMEM(psCtxExport, "psCtxExport");
+
+	if (!DevmemIntCtxAcquire(psContext))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, ErrorFreeCtxExport);
+	}
+
+	eError = PMRRefPMR(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorIntCtxRelease);
+
+	/* Now that the source PMR is exported, the layout
+	 * can't change as there could be outstanding importers
+	 * This is to make sure both exporter and importers view of
+	 * the memory is same */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+	psCtxExport->psDevmemCtx = psContext;
+	psCtxExport->psPMR = psPMR;
+	OSWRLockAcquireWrite(g_hExportCtxListLock);
+	dllist_add_to_tail(&g_sExportCtxList, &psCtxExport->sNode);
+	OSWRLockReleaseWrite(g_hExportCtxListLock);
+
+	*ppsContextExport = psCtxExport;
+
+	return PVRSRV_OK;
+
+ErrorIntCtxRelease:
+	DevmemIntCtxRelease(psContext);
+ErrorFreeCtxExport:
+	OSFreeMem(psCtxExport);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnexportCtx(DEVMEMINT_CTX_EXPORT *psContextExport)
+{
+	PVRSRV_ERROR eError;
+
+	OSWRLockAcquireWrite(g_hExportCtxListLock);
+	dllist_remove_node(&psContextExport->sNode);
+	OSWRLockReleaseWrite(g_hExportCtxListLock);
+
+	eError = PMRUnrefPMR(psContextExport->psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+	DevmemIntCtxRelease(psContextExport->psDevmemCtx);
+
+	OSFreeMem(psContextExport);
+
+	/* Unable to find exported context, return error */
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntAcquireRemoteCtx(PMR *psPMR,
+                          DEVMEMINT_CTX **ppsContext,
+                          IMG_HANDLE *phPrivData)
+{
+	PDLLIST_NODE psListNode, psListNodeNext;
+	DEVMEMINT_CTX_EXPORT *psCtxExport;
+
+	OSWRLockAcquireRead(g_hExportCtxListLock);
+	/* Find context from list using PMR as key */
+	dllist_foreach_node(&g_sExportCtxList, psListNode, psListNodeNext)
+	{
+		psCtxExport = IMG_CONTAINER_OF(psListNode, DEVMEMINT_CTX_EXPORT, sNode);
+		if (psCtxExport->psPMR == psPMR)
+		{
+			if (!DevmemIntCtxAcquire(psCtxExport->psDevmemCtx))
+			{
+				OSWRLockReleaseRead(g_hExportCtxListLock);
+
+				return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+			}
+			*ppsContext = psCtxExport->psDevmemCtx;
+			*phPrivData = psCtxExport->psDevmemCtx->hPrivData;
+
+			OSWRLockReleaseRead(g_hExportCtxListLock);
+
+			/* PMR should have been already exported to import it
+			 * If a PMR is exported, its immutable and the same is
+			 * checked here */
+			PVR_ASSERT(IMG_TRUE == PMR_IsMemLayoutFixed(psPMR));
+
+			return PVRSRV_OK;
+		}
+	}
+	OSWRLockReleaseRead(g_hExportCtxListLock);
+
+	/* Unable to find exported context, return error */
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed to acquire remote context. Could not retrieve context with given PMR",
+			__func__));
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntRegisterPFNotify
+@Description    Registers a PID to be notified when a page fault occurs on a
+                specific device memory context.
+@Input          psDevmemCtx    The context to be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntRegisterPFNotifyKM(DEVMEMINT_CTX *psDevmemCtx,
+                                         IMG_BOOL      bRegister)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	DLLIST_NODE         *psNode, *psNodeNext;
+	DEVMEMINT_PF_NOTIFY *psNotifyNode;
+	IMG_BOOL            bPresent = IMG_FALSE;
+	PVRSRV_ERROR        eError;
+	IMG_PID             ui32PID;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemCtx, "psDevmemCtx");
+
+	/* We can be certain that the memory context is valid and not freed during
+	 * the call time of this function as it's assured by the handle framework
+	 * (while the handle is looked up it cannot be freed). Therefore we can
+	 * safely retrieve the pointer to the device node and acquire the
+	 * hPageFaultNotifyLock lock.
+	 */
+	psDevNode = psDevmemCtx->psDevNode;
+
+	/* Acquire write lock to avoid resource free while
+	 * sMemoryContextPageFaultNotifyListHead and sProcessNotifyListHead
+	 * are being accessed.
+	 */
+	OSWRLockAcquireWrite(psDevNode->hPageFaultNotifyLock);
+
+	if (bRegister)
+	{
+		/* If this is the first PID in the list, the device memory context
+		 * needs to be registered for notification */
+		if (dllist_is_empty(&psDevmemCtx->sProcessNotifyListHead))
+		{
+			dllist_add_to_tail(&psDevNode->sMemoryContextPageFaultNotifyListHead,
+			                   &psDevmemCtx->sPageFaultNotifyListElem);
+		}
+	}
+
+	/* Obtain current client PID */
+	ui32PID = OSGetCurrentClientProcessIDKM();
+
+	/* Loop through the registered PIDs and check whether this one is
+	 * present */
+	dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+	{
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+
+		if (psNotifyNode->ui32PID == ui32PID)
+		{
+			bPresent = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (bRegister)
+	{
+		if (bPresent)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to register a PID that is already registered",
+			         __func__));
+			eError = PVRSRV_ERROR_PID_ALREADY_REGISTERED;
+			goto err_already_registered;
+		}
+
+		psNotifyNode = OSAllocMem(sizeof(*psNotifyNode));
+		if (psNotifyNode == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unable to allocate memory for the notify list",
+			          __func__));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto err_out_of_mem;
+		}
+		psNotifyNode->ui32PID = ui32PID;
+		/* Write lock is already held */
+		dllist_add_to_tail(&(psDevmemCtx->sProcessNotifyListHead), &(psNotifyNode->sProcessNotifyListElem));
+	}
+	else
+	{
+		if (!bPresent)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to unregister a PID that is not registered",
+			         __func__));
+			eError = PVRSRV_ERROR_PID_NOT_REGISTERED;
+			goto err_not_registered;
+		}
+		/* Write lock is already held */
+		dllist_remove_node(psNode);
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+		OSFreeMem(psNotifyNode);
+
+		/* If the last process in the list is being unregistered, then also
+		 * unregister the device memory context from the notify list. */
+		if (dllist_is_empty(&psDevmemCtx->sProcessNotifyListHead))
+		{
+			dllist_remove_node(&psDevmemCtx->sPageFaultNotifyListElem);
+		}
+	}
+	eError = PVRSRV_OK;
+
+err_already_registered:
+err_out_of_mem:
+err_not_registered:
+	OSWRLockReleaseWrite(psDevNode->hPageFaultNotifyLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntPFNotify
+@Description    Notifies any processes that have registered themselves to be
+                notified when a page fault happens on a specific device memory
+                context.
+@Input          *psDevNode           The device node.
+@Input          ui64FaultedPCAddress The page catalogue address that faulted.
+@Input          sFaultAddress        The address that triggered the fault.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntPFNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                               IMG_UINT64         ui64FaultedPCAddress,
+                               IMG_DEV_VIRTADDR   sFaultAddress)
+{
+	DLLIST_NODE         *psNode, *psNodeNext;
+	DEVMEMINT_PF_NOTIFY *psNotifyNode;
+	PVRSRV_ERROR        eError;
+	DEVMEMINT_CTX       *psDevmemCtx = NULL;
+	IMG_BOOL            bFailed = IMG_FALSE;
+
+	/* Protect access both to sMemoryContextPageFaultNotifyListHead and
+	 * to sProcessNotifyListHead. Those lists must be accessed atomically
+	 * in relation to each other, otherwise we risk accessing context that
+	 * might have already been destroyed. */
+	OSWRLockAcquireRead(psDevNode->hPageFaultNotifyLock);
+	if (dllist_is_empty(&(psDevNode->sMemoryContextPageFaultNotifyListHead)))
+	{
+		OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+		return PVRSRV_OK;
+	}
+
+	dllist_foreach_node(&(psDevNode->sMemoryContextPageFaultNotifyListHead), psNode, psNodeNext)
+	{
+		DEVMEMINT_CTX *psThisContext =
+			IMG_CONTAINER_OF(psNode, DEVMEMINT_CTX, sPageFaultNotifyListElem);
+		IMG_DEV_PHYADDR sPCDevPAddr;
+
+		eError = MMU_AcquireBaseAddr(psThisContext->psMMUContext, &sPCDevPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "MMU_AcquireBaseAddr");
+			OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+			return eError;
+		}
+
+		if (sPCDevPAddr.uiAddr == ui64FaultedPCAddress)
+		{
+			psDevmemCtx = psThisContext;
+			break;
+		}
+	}
+
+	if (psDevmemCtx == NULL)
+	{
+		/* Not found, just return */
+		OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+		return PVRSRV_OK;
+	}
+
+	/*
+	 * Store the first occurrence of a page fault address,
+	 * until that address is consumed by a client.
+	 */
+	if ((psDevmemCtx->ui32Flags & DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE) == 0)
+	{
+		psDevmemCtx->sFaultAddress = sFaultAddress;
+		psDevmemCtx->ui32Flags |= DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE;
+	}
+
+	/* Loop through each registered PID and send a signal to the process */
+	dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+	{
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+
+		eError = OSDebugSignalPID(psNotifyNode->ui32PID);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unable to signal process for PID: %u",
+			         __func__,
+			         psNotifyNode->ui32PID));
+
+			bFailed = IMG_TRUE;
+		}
+	}
+	OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+
+	if (bFailed)
+	{
+		return PVRSRV_ERROR_SIGNAL_FAILED;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+#if defined(PDUMP)
+typedef struct _DEVMEMINT_PDUMP_VALID_REGION_
+{
+	DLLIST_NODE sNode;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+} DEVMEMINT_PDUMP_VALID_REGION;
+
+IMG_UINT32 DevmemIntMMUContextID(DEVMEMINT_CTX *psDevMemContext)
+{
+	IMG_UINT32 ui32MMUContextID;
+	MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32MMUContextID, PDUMP_FLAGS_CONTINUOUS);
+	return ui32MMUContextID;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize,
+                              DLLIST_NODE *psValidRegionsList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2HeapPageSize;
+	IMG_UINT32 uiHeapPageSize;
+	IMG_DEV_VIRTADDR sValidStart, sValidEnd, sCurrent, sEnd, sStartPage;
+
+	/* Get the page size for heap containing the start virtual address. */
+	eError = DevmemIntGetVDevAddrPageSize(psDeviceNode,
+	                                      sDevAddrStart,
+	                                      &uiLog2HeapPageSize);
+	PVR_RETURN_IF_ERROR(eError);
+
+	uiHeapPageSize = 1 << uiLog2HeapPageSize;
+
+	/* Iterate every page in the region to dump... */
+	sValidStart.uiAddr = sValidEnd.uiAddr = 0; /* Start/end of the current region which is valid to read. */
+	sStartPage.uiAddr = sDevAddrStart.uiAddr & ~((IMG_UINT64) uiHeapPageSize - 1); /* Page aligned start of the region to dump. */
+	sCurrent = sStartPage;
+	sEnd.uiAddr = sDevAddrStart.uiAddr + uiSize; /* End of the region to dump. */
+
+	while (IMG_TRUE)
+	{
+		IMG_BOOL bAtEnd = sCurrent.uiAddr >= sEnd.uiAddr;
+		IMG_BOOL bValid = IMG_FALSE;
+
+		if (!bAtEnd)
+		{
+			/* Check if the page starting at the current address is valid for reading. */
+			eError = DevmemIntIsVDevAddrValid(psConnection,
+			                                  psDeviceNode,
+			                                  psDevmemCtx,
+			                                  sCurrent);
+			if (eError == PVRSRV_OK)
+			{
+				/* If the current valid region is empty then set the start
+				 * to the current page. */
+				if (sValidStart.uiAddr == 0)
+				{
+					if (sCurrent.uiAddr == sStartPage.uiAddr)
+					{
+						/* Use the start of the region to dump if it doesn't
+						 * start page aligned. */
+						sValidStart = sDevAddrStart;
+					}
+					else
+					{
+						sValidStart = sCurrent;
+					}
+				}
+				/* Set the end of the valid region. */
+				sValidEnd.uiAddr = sCurrent.uiAddr + uiHeapPageSize;
+				/* Restrict to the region to dump. */
+				if (sValidEnd.uiAddr > sEnd.uiAddr)
+				{
+					sValidEnd = sEnd;
+				}
+				bValid = IMG_TRUE;
+			}
+			/* Move to the next page. */
+			sCurrent.uiAddr += uiHeapPageSize;
+		}
+
+		/* If the current page is invalid or we've reached the end of the region
+		 * to dump then pdump the current valid region. */
+		if (!bValid && sValidEnd.uiAddr > sValidStart.uiAddr)
+		{
+			DEVMEMINT_PDUMP_VALID_REGION *psRegion = OSAllocMem(sizeof(*psRegion));
+			PVR_LOG_GOTO_IF_NOMEM(psRegion, eError, ErrFreeRegions);
+
+			psRegion->sDevVAddr = sValidStart;
+			psRegion->uiSize = sValidEnd.uiAddr - sValidStart.uiAddr;
+
+			dllist_add_to_tail(psValidRegionsList, &psRegion->sNode);
+
+			sValidStart.uiAddr = sValidEnd.uiAddr = 0;
+		}
+
+		if (bAtEnd)
+		{
+			break;
+		}
+	}
+
+	return PVRSRV_OK;
+
+ErrFreeRegions:
+	DevmemIntPDumpFreeValidRegions(psValidRegionsList);
+	return eError;
+}
+
+void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psValidRegionsList)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(psValidRegionsList, psThis, psNext)
+	{
+		DEVMEMINT_PDUMP_VALID_REGION *psRegion =
+		    IMG_CONTAINER_OF(psThis, DEVMEMINT_PDUMP_VALID_REGION, sNode);
+
+		dllist_remove_node(psThis);
+		OSFreeMem(psRegion);
+	}
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+	DLLIST_NODE *psThis, *psNext;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+	                                    &uiPDumpMMUCtx,
+	                                    ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* The following SYSMEM refers to the 'MMU Context', hence it
+	 * should be the MMU context, not the PMR, that says what the PDump
+	 * MemSpace tag is?
+	 * From a PDump P.O.V. it doesn't matter which name space we use as long
+	 * as that MemSpace is used on the 'MMU Context' we're dumping from
+	 */
+
+	dllist_foreach_node(psDevAddrRegions, psThis, psNext)
+	{
+		DEVMEMINT_PDUMP_VALID_REGION *psRegion =
+		    IMG_CONTAINER_OF(psThis, DEVMEMINT_PDUMP_VALID_REGION, sNode);
+
+		eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+		                     psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+		                     uiPDumpMMUCtx,
+		                     psRegion->sDevVAddr,
+		                     psRegion->uiSize,
+		                     pszFilename,
+		                     ui32FileOffset,
+		                     ui32PDumpFlags);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		ui32FileOffset += psRegion->uiSize;
+
+		dllist_remove_node(psThis);
+		OSFreeMem(psRegion);
+	}
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 ui32ArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+	IMG_UINT32 uiLog2HeapPageSize;
+	IMG_UINT32 uiHeapPageSize;
+	IMG_DEV_VIRTADDR sValidStart, sValidEnd, sCurrent, sEnd, sStartPage;
+	IMG_UINT64 ui64PageMask;
+
+	PVR_UNREFERENCED_PARAMETER(ui32ArraySize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+			&uiPDumpMMUCtx,
+			ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/*
+	  Get the page size for heap containing the start virtual address.
+	 */
+	eError = DevmemIntGetVDevAddrPageSize(psDeviceNode,
+										  sDevAddrStart,
+										  &uiLog2HeapPageSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+	uiHeapPageSize = 1 << uiLog2HeapPageSize;
+	ui64PageMask = uiHeapPageSize - 1;
+
+	/*
+	  Iterate every page in the region to dump...
+	 */
+	sValidStart.uiAddr = sValidEnd.uiAddr = 0; /* Start/end of the current region which is valid to read. */
+	sStartPage.uiAddr = sDevAddrStart.uiAddr & ~ui64PageMask; /* Page aligned start of the region to dump. */
+	sCurrent = sStartPage;
+	sEnd.uiAddr = sDevAddrStart.uiAddr + uiSize; /* End of the region to dump. */
+	for (;;)
+	{
+		IMG_BOOL bAtEnd = sCurrent.uiAddr >= sEnd.uiAddr;
+		IMG_BOOL bValid = IMG_FALSE;
+
+		if (!bAtEnd)
+		{
+			/* Check if the page starting at the current address is valid for reading. */
+			eError = DevmemIntIsVDevAddrValid(psConnection,
+											  psDeviceNode,
+											  psDevmemCtx,
+											  sCurrent);
+			if (eError == PVRSRV_OK)
+			{
+				/* If the current valid region is empty then set the start to the current page. */
+				if (sValidStart.uiAddr == 0)
+				{
+					if (sCurrent.uiAddr == sStartPage.uiAddr)
+					{
+						/* Use the start of the region to dump if it doesn't start page aligned. */
+						sValidStart = sDevAddrStart;
+					}
+					else
+					{
+						sValidStart = sCurrent;
+					}
+				}
+				/* Set the end of the valid region. */
+				sValidEnd.uiAddr = sCurrent.uiAddr + uiHeapPageSize;
+				/* Restrict to the region to dump. */
+				if (sValidEnd.uiAddr > sEnd.uiAddr)
+				{
+					sValidEnd = sEnd;
+				}
+				bValid = IMG_TRUE;
+			}
+			/* Move to the next page. */
+			sCurrent.uiAddr += uiHeapPageSize;
+		}
+		/*
+		  If the current page is invalid or we've reached the end of the region to dump then pdump the current valid region.
+		 */
+		if (!bValid && sValidEnd.uiAddr > sValidStart.uiAddr)
+		{
+			IMG_DEVMEM_SIZE_T uiValidSize = sValidEnd.uiAddr - sValidStart.uiAddr;
+			eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+								 psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+								 uiPDumpMMUCtx,
+								 sValidStart,
+								 uiValidSize,
+								 pszFilename,
+								 ui32FileOffset,
+								 ui32PDumpFlags);
+			PVR_ASSERT(eError == PVRSRV_OK);
+
+			ui32FileOffset += uiValidSize;
+
+			sValidStart.uiAddr = sValidEnd.uiAddr = 0;
+		}
+
+		if (bAtEnd)
+		{
+			break;
+		}
+	}
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          DEVMEMINT_CTX *psDevmemCtx,
+                                          IMG_DEV_VIRTADDR sDevAddrStart,
+                                          IMG_DEVMEM_SIZE_T uiSize,
+                                          const IMG_CHAR *pszFilename,
+                                          IMG_UINT32 ui32FileOffset,
+                                          IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+	                                    &uiPDumpMMUCtx,
+	                                    ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* The following SYSMEM refers to the 'MMU Context', hence it
+	 * should be the MMU context, not the PMR, that says what the PDump
+	 * MemSpace tag is?
+	 * From a PDump P.O.V. it doesn't matter which name space we use as long
+	 * as that MemSpace is used on the 'MMU Context' we're dumping from
+	 */
+	eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+	                     psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+	                     uiPDumpMMUCtx,
+	                     sDevAddrStart,
+	                     uiSize,
+	                     pszFilename,
+	                     ui32FileOffset,
+	                     ui32PDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+							  PVRSRV_DEVICE_NODE *psDeviceNode,
+							  DEVMEMINT_CTX *psDevMemContext,
+							  IMG_UINT32 ui32Size,
+							  const IMG_CHAR *pszFileName,
+							  IMG_DEV_VIRTADDR sData,
+							  IMG_UINT32 ui32DataSize,
+							  IMG_UINT32 ui32LogicalWidth,
+							  IMG_UINT32 ui32LogicalHeight,
+							  IMG_UINT32 ui32PhysicalWidth,
+							  IMG_UINT32 ui32PhysicalHeight,
+							  PDUMP_PIXEL_FORMAT ePixFmt,
+							  IMG_MEMLAYOUT eMemLayout,
+							  IMG_FB_COMPRESSION eFBCompression,
+							  const IMG_UINT32 *paui32FBCClearColour,
+							  PDUMP_FBC_SWIZZLE eFBCSwizzle,
+							  IMG_DEV_VIRTADDR sHeader,
+							  IMG_UINT32 ui32HeaderSize,
+							  IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32 ui32ContextID;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	eError = MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32ContextID, ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_AcquirePDumpMMUContext");
+
+	eError = PDumpImageDescriptor(psDeviceNode,
+									ui32ContextID,
+									(IMG_CHAR *)pszFileName,
+									sData,
+									ui32DataSize,
+									ui32LogicalWidth,
+									ui32LogicalHeight,
+									ui32PhysicalWidth,
+									ui32PhysicalHeight,
+									ePixFmt,
+									eMemLayout,
+									eFBCompression,
+									paui32FBCClearColour,
+									eFBCSwizzle,
+									sHeader,
+									ui32HeaderSize,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "PDumpImageDescriptor");
+
+	/* Don't care about return value */
+	(void) MMU_ReleasePDumpMMUContext(psDevMemContext->psMMUContext, ui32PDumpFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+							 PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_CTX *psDevMemContext,
+							 IMG_UINT32 ui32Size,
+							 const IMG_CHAR *pszFileName,
+							 IMG_DEV_VIRTADDR sData,
+							 IMG_UINT32 ui32DataSize,
+							 IMG_UINT32 ui32HeaderType,
+							 IMG_UINT32 ui32ElementType,
+							 IMG_UINT32 ui32ElementCount,
+							 IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32 ui32ContextID;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	if ((ui32HeaderType != IBIN_HEADER_TYPE) &&
+		(ui32HeaderType != DATA_HEADER_TYPE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid header type (%u)",
+		         __func__,
+		         ui32HeaderType));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32ContextID, ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_AcquirePDumpMMUContext");
+
+	eError = PDumpDataDescriptor(psDeviceNode,
+									ui32ContextID,
+									(IMG_CHAR *)pszFileName,
+									sData,
+									ui32DataSize,
+									ui32HeaderType,
+									ui32ElementType,
+									ui32ElementCount,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "PDumpDataDescriptor");
+
+	/* Don't care about return value */
+	(void) MMU_ReleasePDumpMMUContext(psDevMemContext->psMMUContext, ui32PDumpFlags);
+
+	return eError;
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.c
new file mode 100644
index 000000000000..9255e8622412
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.c
@@ -0,0 +1,881 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via pvrdebug.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "allocmem.h"
+#include "hash.h"
+#include "hash_functions.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "lock.h"
+#include "osfunc_common.h"
+#include "osfunc.h" /* for thread */
+#include "tlstream.h"
+#include "dllist.h"
+
+#include "osdi_impl.h"
+#include "di_impl_brg.h"
+#include "di_impl_brg_intern.h"
+#include "pvr_dicommon.h"
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+#include "pvrsrv.h"
+#endif
+
+#define ENTRIES_TABLE_INIT_SIZE 64
+#define STREAM_BUFFER_SIZE 0x4000 /* 16KB */
+#define STREAM_LINE_LENGTH 512
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define WRITER_THREAD_SLEEP_TIMEOUT 0ULL
+#else
+#define WRITER_THREAD_SLEEP_TIMEOUT 28800000000ULL
+#endif
+#define WRITER_THREAD_DESTROY_TIMEOUT 100000ULL
+#define WRITER_THREAD_DESTROY_RETRIES 10U
+
+#define WRITE_RETRY_COUNT 10      /* retry a write to a TL buffer 10 times */
+#define WRITE_RETRY_WAIT_TIME 100 /* wait 10ms between write retries */
+
+typedef enum THREAD_STATE
+{
+	THREAD_STATE_NULL,
+	THREAD_STATE_ALIVE,
+	THREAD_STATE_TERMINATED,
+} THREAD_STATE;
+
+static struct DIIB_IMPL
+{
+	HASH_TABLE *psEntriesTable;    /*!< Table of entries. */
+	POSWR_LOCK psEntriesLock;      /*!< Protects psEntriesTable. */
+	IMG_HANDLE hWriterThread;
+	IMG_HANDLE hWriterEventObject;
+	ATOMIC_T eThreadState;
+
+	DLLIST_NODE sWriterQueue;
+	POS_LOCK psWriterLock;         /*!< Protects sWriterQueue. */
+} *_g_psImpl;
+
+struct DIIB_GROUP
+{
+	const IMG_CHAR *pszName;
+	struct DIIB_GROUP *psParentGroup;
+};
+
+struct DIIB_ENTRY
+{
+	struct DIIB_GROUP *psParentGroup;
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+	DI_ENTRY_TYPE eType;
+	IMG_CHAR pszFullPath[DI_IMPL_BRG_PATH_LEN];
+	void *pvPrivData;
+
+	POS_LOCK hLock; /*!< Protects access to entry's iterator. */
+};
+
+struct DI_CONTEXT_TAG
+{
+	IMG_HANDLE hStream;
+	ATOMIC_T iRefCnt;
+	IMG_BOOL bClientConnected; /*!< Indicated that the client is or is not
+	                                connected to the DI. */
+};
+
+struct DIIB_WORK_ITEM
+{
+	DI_CONTEXT *psContext;
+	DIIB_ENTRY *psEntry;
+	IMG_UINT64 ui64Size;
+	IMG_UINT64 ui64Offset;
+
+	DLLIST_NODE sQueueElement;
+};
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteWithRetires(void *pvNativeHandle, const IMG_CHAR *pszStr,
+                              IMG_UINT uiLen)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iRetry = 0;
+	IMG_UINT32 ui32Flags = TL_FLAG_NO_WRITE_FAILED;
+
+	do
+	{
+		/* Try to write to the buffer but don't inject MOST_RECENT_WRITE_FAILED
+		 * packet in case of failure because we're going to retry. */
+		eError = TLStreamWriteRetFlags(pvNativeHandle, (IMG_UINT8 *) pszStr,
+		                               uiLen, &ui32Flags);
+		if (eError == PVRSRV_ERROR_STREAM_FULL)
+		{
+			// wait to give the client a change to read
+			OSSleepms(WRITE_RETRY_WAIT_TIME);
+		}
+	}
+	while (eError == PVRSRV_ERROR_STREAM_FULL && iRetry++ < WRITE_RETRY_COUNT);
+
+	/* One last try to write to the buffer. In this case upon failure
+	 * a MOST_RECENT_WRITE_FAILED packet will be inject to the buffer to
+	 * indicate data loss. */
+	if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		eError = TLStreamWrite(pvNativeHandle, (IMG_UINT8 *) pszStr, uiLen);
+	}
+
+	PVR_LOG_IF_ERROR(eError, "TLStreamWrite");
+}
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	_WriteWithRetires(pvNativeHandle, pvData, uiSize);
+}
+
+__printf(2, 0)
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	IMG_CHAR pcBuffer[STREAM_LINE_LENGTH];
+	IMG_UINT uiLen = OSVSNPrintf(pcBuffer, sizeof(pcBuffer) - 1, pszFmt, pArgs);
+	pcBuffer[uiLen] = '\0';
+
+	_WriteWithRetires(pvNativeHandle, pcBuffer, uiLen + 1);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	_WriteWithRetires(pvNativeHandle, pszStr, OSStringLength(pszStr) + 1);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvNativeHandle);
+	return IMG_FALSE;
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- entry operations --------------------------------------------------- */
+
+static PVRSRV_ERROR _ContextUnrefAndMaybeDestroy(DI_CONTEXT *psContext)
+{
+	if (OSAtomicDecrement(&psContext->iRefCnt) == 0)
+	{
+		TLStreamClose(psContext->hStream);
+		OSFreeMem(psContext);
+	}
+
+	return PVRSRV_OK;
+}
+
+static IMG_INT64 _ReadGeneric(const DI_CONTEXT *psContext, DIIB_ENTRY *psEntry)
+{
+	IMG_INT64 iRet = 0;
+	IMG_UINT64 ui64Pos = 0;
+	DI_ITERATOR_CB *psIter = &psEntry->sIterCb;
+	OSDI_IMPL_ENTRY *psImplEntry = &psEntry->sImplEntry;
+	PVRSRV_ERROR eError;
+
+	if (psIter->pfnStart != NULL)
+	{
+		/* this is a full sequence of the operation */
+		void *pvData = psIter->pfnStart(psImplEntry, &ui64Pos);
+
+		while (pvData != NULL && psContext->bClientConnected)
+		{
+			iRet = psIter->pfnShow(psImplEntry, pvData);
+			if (iRet < 0)
+			{
+				break;
+			}
+
+			pvData = psIter->pfnNext(psImplEntry, pvData, &ui64Pos);
+		}
+
+		psIter->pfnStop(psImplEntry, pvData);
+	}
+	else if (psIter->pfnShow != NULL)
+	{
+		/* this is a simplified sequence of the operation */
+		iRet = psIter->pfnShow(psImplEntry, NULL);
+	}
+
+	eError = TLStreamMarkEOS(psImplEntry->pvNative, IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamMarkEOS", return_error_);
+
+	return iRet;
+
+return_error_:
+	return -1;
+}
+
+static IMG_INT64 _ReadRndAccess(DIIB_ENTRY *psEntry, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT8 *pui8Buffer;
+	IMG_HANDLE hStream = psEntry->sImplEntry.pvNative;
+
+	if (psEntry->sIterCb.pfnRead == NULL)
+	{
+		return -1;
+	}
+
+	eError = TLStreamReserve(hStream, &pui8Buffer, ui64Count);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamReserve", return_error_);
+
+	psEntry->sIterCb.pfnRead((IMG_CHAR *) pui8Buffer, ui64Count, pui64Pos,
+	                         pvData);
+
+	eError = TLStreamCommit(hStream, ui64Count);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCommit", return_error_);
+
+	eError = TLStreamMarkEOS(psEntry->sImplEntry.pvNative, IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamMarkEOS", return_error_);
+
+	return 0;
+
+return_error_:
+	return -1;
+}
+
+static void _WriterThread(void *pvArg)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	DLLIST_NODE *psNode;
+
+	PVR_UNREFERENCED_PARAMETER(pvArg);
+
+	eError = OSEventObjectOpen(_g_psImpl->hWriterEventObject, &hEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#ifdef PVRSRV_FORCE_UNLOAD_IF_BAD_STATE
+	while (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK &&
+	       OSAtomicRead(&_g_psImpl->eThreadState) == THREAD_STATE_ALIVE)
+#else
+	while (OSAtomicRead(&_g_psImpl->eThreadState) == THREAD_STATE_ALIVE)
+#endif
+	{
+		struct DIIB_WORK_ITEM *psItem = NULL;
+
+		OSLockAcquire(_g_psImpl->psWriterLock);
+		/* Get element from list tail so that we always get the oldest element
+		 * (elements are added to head). */
+		while ((psNode = dllist_get_prev_node(&_g_psImpl->sWriterQueue)) != NULL)
+		{
+			IMG_INT64 i64Ret;
+			DIIB_ENTRY *psEntry;
+			OSDI_IMPL_ENTRY *psImplEntry;
+
+			dllist_remove_node(psNode);
+			OSLockRelease(_g_psImpl->psWriterLock);
+
+			psItem = IMG_CONTAINER_OF(psNode, struct DIIB_WORK_ITEM,
+			                          sQueueElement);
+
+			psEntry = psItem->psEntry;
+			psImplEntry = &psItem->psEntry->sImplEntry;
+
+			/* if client has already disconnected we can just drop this item */
+			if (psItem->psContext->bClientConnected)
+			{
+
+				PVR_ASSERT(psItem->psContext->hStream != NULL);
+
+				psImplEntry->pvNative = psItem->psContext->hStream;
+
+				if (psEntry->eType == DI_ENTRY_TYPE_GENERIC)
+				{
+					i64Ret = _ReadGeneric(psItem->psContext, psEntry);
+					PVR_LOG_IF_FALSE(i64Ret >= 0, "generic access read operation "
+					                 "failed");
+				}
+				else if (psEntry->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+				{
+					IMG_UINT64 ui64Pos = psItem->ui64Offset;
+
+					i64Ret = _ReadRndAccess(psEntry, psItem->ui64Size, &ui64Pos,
+					                        psEntry->pvPrivData);
+					PVR_LOG_IF_FALSE(i64Ret >= 0, "random access read operation "
+					                 "failed");
+				}
+				else
+				{
+					PVR_ASSERT(psEntry->eType == DI_ENTRY_TYPE_GENERIC ||
+					           psEntry->eType == DI_ENTRY_TYPE_RANDOM_ACCESS);
+				}
+
+				psImplEntry->pvNative = NULL;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "client reading entry \"%s\" has "
+				        "disconnected", psEntry->pszFullPath));
+			}
+
+			_ContextUnrefAndMaybeDestroy(psItem->psContext);
+			OSFreeMemNoStats(psItem);
+
+			OSLockAcquire(_g_psImpl->psWriterLock);
+		}
+		OSLockRelease(_g_psImpl->psWriterLock);
+
+		eError = OSEventObjectWaitKernel(hEvent, WRITER_THREAD_SLEEP_TIMEOUT);
+		if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_LOG_ERROR(eError, "OSEventObjectWaitKernel");
+		}
+	}
+
+	OSLockAcquire(_g_psImpl->psWriterLock);
+	/* clear the queue if there are any items pending */
+	while ((psNode = dllist_get_prev_node(&_g_psImpl->sWriterQueue)) != NULL)
+	{
+		struct DIIB_WORK_ITEM *psItem = IMG_CONTAINER_OF(psNode,
+		                                                 struct DIIB_WORK_ITEM,
+		                                                 sQueueElement);
+
+		dllist_remove_node(psNode);
+		_ContextUnrefAndMaybeDestroy(psItem->psContext);
+		OSFreeMem(psItem);
+	}
+	OSLockRelease(_g_psImpl->psWriterLock);
+
+	eError = OSEventObjectClose(hEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_TERMINATED);
+}
+
+/* ----- DI internal API ---------------------------------------------------- */
+
+DIIB_ENTRY *DIImplBrgFind(const IMG_CHAR *pszPath)
+{
+	DIIB_ENTRY *psEntry;
+
+	OSWRLockAcquireRead(_g_psImpl->psEntriesLock);
+	psEntry = (void *) HASH_Retrieve_Extended(_g_psImpl->psEntriesTable,
+	                                          (IMG_CHAR *) pszPath);
+	OSWRLockReleaseRead(_g_psImpl->psEntriesLock);
+
+	return psEntry;
+}
+
+/* ----- DI bridge interface ------------------------------------------------ */
+
+static PVRSRV_ERROR _CreateStream(IMG_CHAR *pszStreamName, IMG_HANDLE *phStream)
+{
+	IMG_UINT32 iRet;
+	IMG_HANDLE hStream;
+	PVRSRV_ERROR eError;
+
+	/* for now only one stream can be created. Should we be able to create
+	 * per context stream? */
+	iRet = OSSNPrintf(pszStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+	                  "di_stream_%x", OSGetCurrentClientProcessIDKM());
+	if (iRet >= PRVSRVTL_MAX_STREAM_NAME_SIZE)
+	{
+		/* this check is superfluous because it can never happen but in case
+		 * someone changes the definition of PRVSRVTL_MAX_STREAM_NAME_SIZE
+		 * handle this case */
+		pszStreamName[0] = '\0';
+		return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+
+	eError = TLStreamCreate(&hStream, pszStreamName, STREAM_BUFFER_SIZE,
+	                        TL_OPMODE_DROP_NEWER, NULL, NULL, NULL, NULL,
+	                        NULL, NULL);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*phStream = hStream;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DICreateContextKM(IMG_CHAR *pszStreamName, DI_CONTEXT **ppsContext)
+{
+	PVRSRV_ERROR eError;
+	DI_CONTEXT *psContext;
+	IMG_HANDLE hStream = NULL;
+	THREAD_STATE eTState;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsContext != NULL, "ppsContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszStreamName != NULL, "pszStreamName");
+
+	psContext = OSAllocMem(sizeof(*psContext));
+	PVR_LOG_GOTO_IF_NOMEM(psContext, eError, return_);
+
+	eError = _CreateStream(pszStreamName, &hStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CreateStream", free_desc_);
+
+	psContext->hStream = hStream;
+	/* indicated to the write thread if the client is still connected and
+	 * waiting for the data */
+	psContext->bClientConnected = IMG_TRUE;
+	OSAtomicWrite(&psContext->iRefCnt, 1);
+
+	eTState = OSAtomicCompareExchange(&_g_psImpl->eThreadState,
+	                                  THREAD_STATE_NULL,
+	                                  THREAD_STATE_ALIVE);
+
+	/* if the thread has not been started yet do it */
+	if (eTState == THREAD_STATE_NULL)
+	{
+		PVR_ASSERT(_g_psImpl->hWriterThread == NULL);
+
+		eError = OSThreadCreate(&_g_psImpl->hWriterThread, "di_writer",
+		                        _WriterThread, NULL, IMG_FALSE, NULL);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreate", free_close_stream_);
+	}
+
+	*ppsContext = psContext;
+
+	return PVRSRV_OK;
+
+free_close_stream_:
+	TLStreamClose(psContext->hStream);
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_TERMINATED);
+free_desc_:
+	OSFreeMem(psContext);
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIDestroyContextKM(DI_CONTEXT *psContext)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+
+	/* pass the information to the write thread that the client has
+	 * disconnected */
+	psContext->bClientConnected = IMG_FALSE;
+
+	return _ContextUnrefAndMaybeDestroy(psContext);
+}
+
+PVRSRV_ERROR DIReadEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT64 ui64Offset, IMG_UINT64 ui64Size)
+{
+	PVRSRV_ERROR eError;
+	struct DIIB_WORK_ITEM *psItem;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszEntryPath != NULL, "pszEntryPath");
+
+	/* 'no stats' to avoid acquiring the process stats locks */
+	psItem = OSAllocMemNoStats(sizeof(*psItem));
+	PVR_LOG_GOTO_IF_NOMEM(psItem, eError, return_);
+
+	psItem->psContext = psContext;
+	psItem->psEntry = DIImplBrgFind(pszEntryPath);
+	PVR_LOG_GOTO_IF_FALSE_VA(psItem->psEntry != NULL, free_item_,
+	                         "entry %s does not exist", pszEntryPath);
+	psItem->ui64Size = ui64Size;
+	psItem->ui64Offset = ui64Offset;
+
+	/* increment ref count on the context so that it doesn't get freed
+	 * before it gets processed by the writer thread. */
+	if (OSAtomicAddUnless(&psContext->iRefCnt, 1, IMG_INT32_MAX) == IMG_INT32_MAX)
+	{
+		/* Job is not scheduled to the writer queue; there are too many waiting. */
+		PVR_LOG_GOTO_WITH_ERROR("OSAtomicAddUnless", eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, overflow_);
+	}
+
+	OSLockAcquire(_g_psImpl->psWriterLock);
+	dllist_add_to_head(&_g_psImpl->sWriterQueue, &psItem->sQueueElement);
+	OSLockRelease(_g_psImpl->psWriterLock);
+
+	eError = OSEventObjectSignal(_g_psImpl->hWriterEventObject);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+
+	return PVRSRV_OK;
+
+free_item_:
+	eError = PVRSRV_ERROR_NOT_FOUND;
+overflow_:
+	OSFreeMemNoStats(psItem);
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIWriteEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT32 ui32ValueSize, const IMG_CHAR *pszValue)
+{
+	DIIB_ENTRY *psEntry;
+	DI_PFN_WRITE pfnEntryPuts;
+	IMG_INT64 i64Length = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszEntryPath != NULL, "pszEntryPath");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszValue != NULL, "pszValue");
+
+	psEntry = DIImplBrgFind(pszEntryPath);
+	PVR_LOG_RETURN_IF_FALSE_VA(psEntry != NULL, PVRSRV_ERROR_NOT_FOUND,
+	                         "entry %s does not exist", pszEntryPath);
+
+	pfnEntryPuts = psEntry->sIterCb.pfnWrite;
+	if (pfnEntryPuts != NULL)
+	{
+		i64Length = pfnEntryPuts(pszValue, ui32ValueSize, (IMG_UINT64*)&i64Length, psEntry->pvPrivData);
+
+		/* To deal with -EINVAL being returned */
+		PVR_LOG_RETURN_IF_INVALID_PARAM(i64Length >= 0, pszValue);
+	}
+	else
+	{
+		PVR_LOG_MSG(PVR_DBG_WARNING, "Unable to write to Entry. Write callback not enabled");
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _listName(uintptr_t k,
+                               uintptr_t v,
+                               void* hStream)
+{
+	PVRSRV_ERROR eError;
+	DIIB_ENTRY *psEntry;
+	IMG_UINT32 ui32Size;
+	IMG_CHAR aszName[DI_IMPL_BRG_PATH_LEN];
+
+	psEntry = (DIIB_ENTRY*) v;
+	PVR_ASSERT(psEntry != NULL);
+	PVR_UNREFERENCED_PARAMETER(k);
+
+	ui32Size = OSSNPrintf(aszName, DI_IMPL_BRG_PATH_LEN, "%s\n", psEntry->pszFullPath);
+	PVR_LOG_IF_FALSE(ui32Size > 5, "ui32Size too small, Error suspected!");
+	eError = TLStreamWrite(hStream, (IMG_UINT8 *)aszName, ui32Size+1);
+
+	return eError;
+}
+
+
+PVRSRV_ERROR DIListAllEntriesKM(DI_CONTEXT *psContext)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+
+	OSWRLockAcquireRead(_g_psImpl->psEntriesLock);
+	eError = HASH_Iterate(_g_psImpl->psEntriesTable, _listName, psContext->hStream);
+	OSWRLockReleaseRead(_g_psImpl->psEntriesLock);
+	PVR_LOG_IF_ERROR(eError, "HASH_Iterate_Extended");
+
+	eError = TLStreamMarkEOS(psContext->hStream, IMG_FALSE);
+	return eError;
+}
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	PVRSRV_ERROR eError;
+
+	_g_psImpl = OSAllocMem(sizeof(*_g_psImpl));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psImpl, eError, return_);
+
+	_g_psImpl->psEntriesTable = HASH_Create_Extended(ENTRIES_TABLE_INIT_SIZE,
+	                                                 DI_IMPL_BRG_PATH_LEN,
+	                                                 HASH_Djb2_Hash,
+	                                                 HASH_Djb2_Compare);
+	PVR_LOG_GOTO_IF_NOMEM(_g_psImpl->psEntriesTable, eError, free_impl_);
+
+	eError = OSWRLockCreate(&_g_psImpl->psEntriesLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSCreateLock", free_table_);
+
+	eError = OSLockCreate(&_g_psImpl->psWriterLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSCreateLock", free_entries_lock_);
+
+	eError = OSEventObjectCreate("DI_WRITER_EO",
+	                             &_g_psImpl->hWriterEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", free_writer_lock_);
+
+	_g_psImpl->hWriterThread = NULL;
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_NULL);
+
+	dllist_init(&_g_psImpl->sWriterQueue);
+
+	return PVRSRV_OK;
+
+free_writer_lock_:
+	OSLockDestroy(_g_psImpl->psWriterLock);
+free_entries_lock_:
+	OSWRLockDestroy(_g_psImpl->psEntriesLock);
+free_table_:
+	HASH_Delete_Extended(_g_psImpl->psEntriesTable, IMG_FALSE);
+free_impl_:
+	OSFreeMem(_g_psImpl);
+	_g_psImpl = NULL;
+return_:
+	return eError;
+}
+
+static void _DeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	THREAD_STATE eTState;
+
+	eTState = OSAtomicCompareExchange(&_g_psImpl->eThreadState,
+	                                  THREAD_STATE_ALIVE,
+	                                  THREAD_STATE_TERMINATED);
+
+	if (eTState == THREAD_STATE_ALIVE)
+	{
+		if (_g_psImpl->hWriterEventObject != NULL)
+		{
+			eError = OSEventObjectSignal(_g_psImpl->hWriterEventObject);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(WRITER_THREAD_DESTROY_TIMEOUT)
+		{
+			eError = OSThreadDestroy(_g_psImpl->hWriterThread);
+			if (eError == PVRSRV_OK)
+			{
+				break;
+			}
+			OSWaitus(WRITER_THREAD_DESTROY_TIMEOUT/WRITER_THREAD_DESTROY_RETRIES);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (_g_psImpl->hWriterEventObject != NULL)
+	{
+		eError = OSEventObjectDestroy(_g_psImpl->hWriterEventObject);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	HASH_Delete_Extended(_g_psImpl->psEntriesTable, IMG_FALSE);
+	OSLockDestroy(_g_psImpl->psWriterLock);
+	OSWRLockDestroy(_g_psImpl->psEntriesLock);
+	OSFreeMem(_g_psImpl);
+	_g_psImpl = NULL;
+}
+
+/* Recursively traverses the ancestors list up to the root group and
+ * appends their names preceded by "/" to the path in reverse order
+ * (root group's name first and psGroup group's name last).
+ * Returns current offset in the path (the current path length without the
+ * NUL character). If there is no more space in the path returns -1
+ * to indicate an error (the path is too long to fit into the buffer). */
+static ssize_t _BuildGroupPath(IMG_CHAR *pszPath, const DIIB_GROUP *psGroup)
+{
+	ssize_t iOff;
+	ssize_t iCopiedCnt;
+	size_t  uFreeCnt;
+
+	PVR_RETURN_IF_FALSE(psGroup != NULL, 0);
+
+	PVR_ASSERT(pszPath != NULL);
+
+	iOff = _BuildGroupPath(pszPath, psGroup->psParentGroup);
+	PVR_RETURN_IF_FALSE(iOff != -1, -1);
+
+	uFreeCnt = DI_IMPL_BRG_PATH_LEN - iOff;
+	PVR_RETURN_IF_FALSE(uFreeCnt > 1, -1);
+
+	pszPath[iOff++] = '/';
+	--uFreeCnt;
+
+	iCopiedCnt = OSStringSafeCopy(&pszPath[iOff], psGroup->pszName, uFreeCnt);
+	PVR_RETURN_IF_FALSE(iCopiedCnt >= 0, -1);
+	iOff += iCopiedCnt;
+
+	return iOff;
+}
+
+static PVRSRV_ERROR _BuildEntryPath(IMG_CHAR *pszPath, const IMG_CHAR *pszName,
+                                    const DIIB_GROUP *psGroup)
+{
+	ssize_t iOff = _BuildGroupPath(pszPath, psGroup);
+	ssize_t iCopiedCnt;
+	size_t  uFreeCnt;
+
+	PVR_RETURN_IF_FALSE(iOff != -1, PVRSRV_ERROR_INVALID_OFFSET);
+
+	uFreeCnt = DI_IMPL_BRG_PATH_LEN - iOff;
+	PVR_RETURN_IF_FALSE(uFreeCnt > 1, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	pszPath[iOff++] = '/';
+	--uFreeCnt;
+
+	iCopiedCnt = OSStringSafeCopy(&pszPath[iOff], pszName, uFreeCnt);
+	PVR_RETURN_IF_FALSE(iCopiedCnt >= 0, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _CreateEntry(const IMG_CHAR *pszName,
+                                 DI_ENTRY_TYPE eType,
+                                 const DI_ITERATOR_CB *psIterCb,
+                                 void *pvPrivData,
+                                 void *pvParentGroup,
+                                 void **pvEntry)
+{
+	DIIB_GROUP *psParentGroup = pvParentGroup;
+	DIIB_ENTRY *psEntry;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvEntry != NULL, "pvEntry");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentGroup != NULL, "pvParentGroup");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, return_);
+	}
+
+	psEntry = OSAllocMem(sizeof(*psEntry));
+	PVR_LOG_GOTO_IF_NOMEM(psEntry, eError, return_);
+
+	eError = OSLockCreate(&psEntry->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", free_entry_);
+
+	psEntry->eType = eType;
+	psEntry->sIterCb = *psIterCb;
+	psEntry->pvPrivData = pvPrivData;
+	psEntry->psParentGroup = psParentGroup;
+	psEntry->pszFullPath[0] = '\0';
+
+	psEntry->sImplEntry.pvPrivData = pvPrivData;
+	psEntry->sImplEntry.pvNative = NULL;
+	psEntry->sImplEntry.psCb = &_g_sEntryCallbacks;
+
+	eError = _BuildEntryPath(psEntry->pszFullPath, pszName,
+	                         psEntry->psParentGroup);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s() failed in _BuildEntryPath() for \"%s\" "
+		        "entry", __func__, pszName));
+		goto destroy_lock_;
+	}
+
+	OSWRLockAcquireWrite(_g_psImpl->psEntriesLock);
+	eError = HASH_Insert_Extended(_g_psImpl->psEntriesTable,
+	                              psEntry->pszFullPath,
+	                              (uintptr_t) psEntry) ?
+	         PVRSRV_OK : PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+	OSWRLockReleaseWrite(_g_psImpl->psEntriesLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HASH_Insert_Extended failed", destroy_lock_);
+
+	*pvEntry = psEntry;
+
+	return PVRSRV_OK;
+
+destroy_lock_:
+	OSLockDestroy(psEntry->hLock);
+free_entry_:
+	OSFreeMem(psEntry);
+return_:
+	return eError;
+}
+
+static void _DestroyEntry(void *pvEntry)
+{
+	DIIB_ENTRY *psEntry = pvEntry;
+	PVR_ASSERT(psEntry != NULL);
+
+	OSWRLockAcquireWrite(_g_psImpl->psEntriesLock);
+	HASH_Remove_Extended(_g_psImpl->psEntriesTable, psEntry->pszFullPath);
+	OSWRLockReleaseWrite(_g_psImpl->psEntriesLock);
+
+	OSLockDestroy(psEntry->hLock);
+	OSFreeMem(psEntry);
+}
+
+static PVRSRV_ERROR _CreateGroup(const IMG_CHAR *pszName,
+                                 void *pvParentGroup,
+                                 void **ppvGroup)
+{
+	DIIB_GROUP *psNewGroup;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvGroup != NULL, "ppvGroup");
+
+	psNewGroup = OSAllocMem(sizeof(*psNewGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psNewGroup, "OSAllocMem");
+
+	psNewGroup->pszName = pszName;
+	psNewGroup->psParentGroup = pvParentGroup;
+
+	*ppvGroup = psNewGroup;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyGroup(void *pvGroup)
+{
+	DIIB_GROUP *psGroup = pvGroup;
+	PVR_ASSERT(psGroup != NULL);
+
+	OSFreeMem(psGroup);
+}
+
+PVRSRV_ERROR PVRDIImplBrgRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateEntry,
+		.pfnDestroyEntry = _DestroyEntry,
+		.pfnCreateGroup = _CreateGroup,
+		.pfnDestroyGroup = _DestroyGroup
+	};
+
+	return DIRegisterImplementation("impl_brg", &sImplCb);
+}
+
+/******************************************************************************
+ End of file (di_impl_brg.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.h b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.h
new file mode 100644
index 000000000000..b8cff25d94ec
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_IMPL_BRG_H
+#define PVR_IMPL_BRG_H
+
+#include "pvrsrv_error.h"
+#include "lock_types.h"
+
+typedef struct DI_CONTEXT_TAG DI_CONTEXT;
+typedef struct DI_ENTRY_DESC DI_ENTRY_DESC;
+
+PVRSRV_ERROR PVRDIImplBrgRegister(void);
+
+/*! @Function DICreateContextKM
+ *
+ * @Description
+ * Creates DI context which among others also creates a TL stream for reading
+ * entries.
+ *
+ * @Output pszStreamName: name of the TL stream created in this context
+ * @Output ppsContext: pointer to the new context
+ *
+ * @Return PVRSRV_ERROR error code
+ *         PVRSRV_OK in case of a success
+ *         PVRSRV_ERROR_INVALID_PARAMS if any of the parameters is invalid
+ *         PVRSRV_ERROR_OUT_OF_MEMORY if any of the memory allocations failed
+ *         error codes returned by TLStreamCreate()
+ */
+PVRSRV_ERROR DICreateContextKM(IMG_CHAR *pszStreamName,
+                               DI_CONTEXT **ppsContext);
+
+/*! @Function DIDestroyContextKM
+ *
+ * @Description
+ * Destroy the DI context and all underlying dependencies.
+ *
+ * @Input psContext: pointer to the context
+ *
+ * @Return PVRSRV_ERROR error code
+ *         PVRSRV_OK in case of a success
+ *         PVRSRV_ERROR_INVALID_PARAMS if invalid context pointer given
+ */
+PVRSRV_ERROR DIDestroyContextKM(DI_CONTEXT *psContext);
+
+PVRSRV_ERROR DIReadEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT64 ui64Offset, IMG_UINT64 ui64Size);
+
+PVRSRV_ERROR DIWriteEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                            IMG_UINT32 ui32ValueSize, const IMG_CHAR *pszValue);
+
+PVRSRV_ERROR DIListAllEntriesKM(DI_CONTEXT *psContext);
+
+
+#endif /* PVR_IMPL_BRG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg_intern.h b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg_intern.h
new file mode 100644
index 000000000000..5e11cac59854
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_impl_brg_intern.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info internal interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_IMPL_BRG_INTERN_H
+#define PVR_IMPL_BRG_INTERN_H
+
+typedef struct DIIB_GROUP DIIB_GROUP;
+typedef struct DIIB_ENTRY DIIB_ENTRY;
+
+/*! @Function DIImplBrgFind
+ *
+ * @Description
+ * Retrieves an entry based on a given path.
+ *
+ * @Input pszPath: Full entry path in form of
+ *                 /rootGroup/.../parentGroup/entryName.
+ *
+ * @Return Returns entry object if exists or NULL otherwise.
+ */
+DIIB_ENTRY *DIImplBrgFind(const IMG_CHAR *pszPath);
+
+#endif /* PVR_IMPL_BRG_INTERN_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/di_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_server.c
new file mode 100644
index 000000000000..47046d7ec3a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/di_server.c
@@ -0,0 +1,800 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Info framework functions and types.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "di_server.h"
+#include "osdi_impl.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "lock.h"
+#include "allocmem.h"
+#include "osfunc.h"
+
+#define ROOT_GROUP_NAME PVR_DRM_NAME
+
+/*! Implementation object. */
+typedef struct DI_IMPL
+{
+	const IMG_CHAR *pszName;       /*<! name of the implementation */
+	OSDI_IMPL_CB sCb;              /*<! implementation callbacks */
+	IMG_BOOL bInitialised;         /*<! set to IMG_TRUE after implementation
+	                                    is initialised */
+
+	DLLIST_NODE sListNode;         /*<! node element of the global list of all
+	                                    implementations */
+} DI_IMPL;
+
+/*! Wrapper object for objects originating from derivative implementations.
+ * This wraps both entries and groups native implementation objects. */
+typedef struct DI_NATIVE_HANDLE
+{
+	void *pvHandle;                /*!< opaque handle to the native object */
+	DI_IMPL *psDiImpl;             /*!< implementation pvHandle is associated
+	                                    with */
+	DLLIST_NODE sListNode;         /*!< node element of native handles list */
+} DI_NATIVE_HANDLE;
+
+/*! Debug Info entry object.
+ *
+ * Depending on the implementation this can be represented differently. For
+ * example for the DebugFS this translates to a file.
+ */
+struct DI_ENTRY
+{
+	const IMG_CHAR *pszName;       /*!< name of the entry */
+	void *pvPrivData;              /*! handle to entry's private data */
+	DI_ENTRY_TYPE eType;           /*! entry type */
+	DI_ITERATOR_CB sIterCb;        /*!< iterator interface for the entry */
+
+	DLLIST_NODE sListNode;         /*!< node element of group's entry list */
+	DLLIST_NODE sNativeHandleList; /*!< list of native handles belonging to this
+	                                    entry */
+};
+
+/*! Debug Info group object.
+ *
+ * Depending on the implementation this can be represented differently. For
+ * example for the DebugFS this translates to a directory.
+ */
+struct DI_GROUP
+{
+	IMG_CHAR *pszName;               /*!< name of the group */
+	const struct DI_GROUP *psParent; /*!< parent groups */
+
+	DLLIST_NODE sListNode;           /*!< node element of group's group list */
+	DLLIST_NODE sGroupList;          /*!< list of groups (children) that belong
+	                                      to this group */
+	DLLIST_NODE sEntryList;          /*!< list of entries (children) that belong
+	                                      to this group */
+	DLLIST_NODE sNativeHandleList;   /*!< list of native handles belonging to
+	                                      this group */
+};
+
+/* List of all registered implementations. */
+static DECLARE_DLLIST(_g_sImpls);
+
+/* Root group for the DI entries and groups. This group is used as a root
+ * group for all other groups and entries if during creation a parent groups
+ * is not given. */
+static DI_GROUP *_g_psRootGroup;
+
+/* Protects access to _g_sImpls and _g_psRootGroup */
+static POS_LOCK _g_hLock;
+
+PVRSRV_ERROR DIInit(void)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__) && defined(__KERNEL__)
+	eError = OSLockCreateNoStats(&_g_hLock);
+#else
+	eError = OSLockCreate(&_g_hLock);
+#endif
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", return_);
+
+	_g_psRootGroup = OSAllocMemNoStats(sizeof(*_g_psRootGroup));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psRootGroup, eError, destroy_lock_);
+
+	_g_psRootGroup->pszName = OSAllocMemNoStats(sizeof(ROOT_GROUP_NAME));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psRootGroup->pszName, eError, cleanup_name_);
+	OSStringSafeCopy(_g_psRootGroup->pszName, ROOT_GROUP_NAME,
+				  sizeof(ROOT_GROUP_NAME));
+
+	dllist_init(&_g_psRootGroup->sListNode);
+	dllist_init(&_g_psRootGroup->sGroupList);
+	dllist_init(&_g_psRootGroup->sEntryList);
+	dllist_init(&_g_psRootGroup->sNativeHandleList);
+
+	return PVRSRV_OK;
+
+cleanup_name_:
+	OSFreeMemNoStats(_g_psRootGroup);
+destroy_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(_g_hLock);
+#else
+	OSLockDestroy(_g_hLock);
+#endif
+return_:
+	return eError;
+}
+
+/* Destroys the whole tree of group and entries for a given group as a root. */
+static void _DeInitGroupRecursively(DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+		DIDestroyEntry(psThisEntry);
+	}
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		_DeInitGroupRecursively(psThisGroup);
+	}
+
+	DIDestroyGroup(psGroup);
+}
+
+void DIDeInit(void)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	OSLockAcquire(_g_hLock);
+
+	if (!dllist_is_empty(&_g_psRootGroup->sGroupList) ||
+	    !dllist_is_empty(&_g_psRootGroup->sEntryList))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: entries or groups still exist during "
+		        "de-initialisation process, destroying all", __func__));
+	}
+
+	_DeInitGroupRecursively(_g_psRootGroup);
+	_g_psRootGroup = NULL;
+
+	/* Remove all of the implementations. */
+	dllist_foreach_node(&_g_sImpls, psThis, psNext)
+	{
+		DI_IMPL *psDiImpl = IMG_CONTAINER_OF(psThis, DI_IMPL, sListNode);
+
+		if (psDiImpl->bInitialised)
+		{
+			psDiImpl->sCb.pfnDeInit();
+			psDiImpl->bInitialised = IMG_FALSE;
+		}
+
+		dllist_remove_node(&psDiImpl->sListNode);
+		OSFreeMem(psDiImpl);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	/* all resources freed so free the lock itself too */
+
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(_g_hLock);
+#else
+	OSLockDestroy(_g_hLock);
+#endif
+}
+
+static IMG_BOOL _ValidateIteratorCb(const DI_ITERATOR_CB *psIterCb,
+                                    DI_ENTRY_TYPE eType)
+{
+	IMG_UINT32 uiFlags = 0;
+
+	if (psIterCb == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	if (eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		uiFlags |= psIterCb->pfnShow != NULL ? BIT(0) : 0;
+		uiFlags |= psIterCb->pfnStart != NULL ? BIT(1) : 0;
+		uiFlags |= psIterCb->pfnStop != NULL ? BIT(2) : 0;
+		uiFlags |= psIterCb->pfnNext != NULL ? BIT(3) : 0;
+
+		/* either only pfnShow or all callbacks need to be set */
+		if (uiFlags != BIT(0) && !BITMASK_HAS(uiFlags, 0x0f))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else if (eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		uiFlags |= psIterCb->pfnRead != NULL ? BIT(0) : 0;
+		uiFlags |= psIterCb->pfnSeek != NULL ? BIT(1) : 0;
+
+		/* either only pfnRead or all callbacks need to be set */
+		if (uiFlags != BIT(0) && !BITMASK_HAS(uiFlags, 0x03))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static PVRSRV_ERROR _CreateNativeEntry(DI_ENTRY *psEntry,
+                                       const DI_NATIVE_HANDLE *psNativeParent)
+{
+	PVRSRV_ERROR eError;
+	DI_IMPL *psImpl = psNativeParent->psDiImpl;
+
+	DI_NATIVE_HANDLE *psNativeEntry = OSAllocMem(sizeof(*psNativeEntry));
+	PVR_LOG_GOTO_IF_NOMEM(psNativeEntry, eError, return_);
+
+	eError = psImpl->sCb.pfnCreateEntry(psEntry->pszName,
+	                                    psEntry->eType,
+	                                    &psEntry->sIterCb,
+	                                    psEntry->pvPrivData,
+	                                    psNativeParent->pvHandle,
+	                                    &psNativeEntry->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateEntry", free_memory_);
+
+	psNativeEntry->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psEntry->sNativeHandleList, &psNativeEntry->sListNode);
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeEntry);
+return_:
+	return eError;
+}
+
+static void _DestroyNativeEntry(DI_NATIVE_HANDLE *psNativeEntry)
+{
+	dllist_remove_node(&psNativeEntry->sListNode);
+	OSFreeMem(psNativeEntry);
+}
+
+PVRSRV_ERROR DICreateEntry(const IMG_CHAR *pszName,
+                           DI_GROUP *psGroup,
+                           const DI_ITERATOR_CB *psIterCb,
+                           void *pvPriv,
+                           DI_ENTRY_TYPE eType,
+                           DI_ENTRY **ppsEntry)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_ENTRY *psEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(_ValidateIteratorCb(psIterCb, eType),
+	                                "psIterCb");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsEntry != NULL, "psEntry");
+
+	psEntry = OSAllocMem(sizeof(*psEntry));
+	PVR_LOG_RETURN_IF_NOMEM(psEntry, "OSAllocMem");
+
+	if (psGroup == NULL)
+	{
+		psGroup = _g_psRootGroup;
+	}
+
+	psEntry->pszName = pszName;
+	psEntry->pvPrivData = pvPriv;
+	psEntry->eType = eType;
+	psEntry->sIterCb = *psIterCb;
+	dllist_init(&psEntry->sNativeHandleList);
+
+	OSLockAcquire(_g_hLock);
+
+	dllist_add_to_tail(&psGroup->sEntryList, &psEntry->sListNode);
+
+	/* Iterate over all of the native handles of parent group to create
+	 * the entry for every registered implementation. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		eError = _CreateNativeEntry(psEntry, psNativeGroup);
+		PVR_GOTO_IF_ERROR(eError, cleanup_);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	*ppsEntry = psEntry;
+
+	return PVRSRV_OK;
+
+cleanup_:
+	OSLockRelease(_g_hLock);
+
+	/* Something went wrong so if there were any native entries created remove
+	 * them from the list, free them and free the DI entry itself. */
+	dllist_foreach_node(&psEntry->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeEntry =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		_DestroyNativeEntry(psNativeEntry);
+	}
+
+	OSFreeMem(psEntry);
+
+	return eError;
+}
+
+void DIDestroyEntry(DI_ENTRY *psEntry)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psEntry != NULL,
+	                             "psEntry invalid in DIDestroyEntry()");
+
+	/* Iterate through all of the native entries of the DI entry, remove
+	 * them from the list and then destroy them. After that, destroy the
+	 * DI entry itself. */
+	dllist_foreach_node(&psEntry->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNative = IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE,
+		                                              sListNode);
+
+		/* The implementation must ensure that entry is not removed if any
+		 * operations are being executed on the entry. If this is the case
+		 * the implementation should block until all of them are finished
+		 * and prevent any further operations.
+		 * This will guarantee proper synchronisation between the DI framework
+		 * and underlying implementations and prevent destruction/access
+		 * races. */
+		psNative->psDiImpl->sCb.pfnDestroyEntry(psNative->pvHandle);
+		dllist_remove_node(&psNative->sListNode);
+		OSFreeMem(psNative);
+	}
+
+	dllist_remove_node(&psEntry->sListNode);
+
+	OSFreeMem(psEntry);
+}
+
+static PVRSRV_ERROR _CreateNativeGroup(DI_GROUP *psGroup,
+                                       const DI_NATIVE_HANDLE *psNativeParent,
+                                       DI_NATIVE_HANDLE **ppsNativeGroup)
+{
+	PVRSRV_ERROR eError;
+	DI_IMPL *psImpl = psNativeParent->psDiImpl;
+
+	DI_NATIVE_HANDLE *psNativeGroup = OSAllocMem(sizeof(*psNativeGroup));
+	PVR_LOG_GOTO_IF_NOMEM(psNativeGroup, eError, return_);
+
+	eError = psImpl->sCb.pfnCreateGroup(psGroup->pszName,
+	                                    psNativeParent->pvHandle,
+	                                    &psNativeGroup->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateGroup", free_memory_);
+
+	psNativeGroup->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psGroup->sNativeHandleList, &psNativeGroup->sListNode);
+
+	*ppsNativeGroup = psNativeGroup;
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeGroup);
+return_:
+	return eError;
+}
+
+static void _DestroyNativeGroup(DI_NATIVE_HANDLE *psNativeEntry)
+{
+	dllist_remove_node(&psNativeEntry->sListNode);
+	OSFreeMem(psNativeEntry);
+}
+
+PVRSRV_ERROR DICreateGroup(const IMG_CHAR *pszName,
+                           DI_GROUP *psParent,
+                           DI_GROUP **ppsGroup)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_GROUP *psGroup;
+	size_t uSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsGroup != NULL, "ppsDiGroup");
+
+	psGroup = OSAllocMem(sizeof(*psGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psGroup, "OSAllocMem");
+
+	if (psParent == NULL)
+	{
+		psParent = _g_psRootGroup;
+	}
+
+	uSize = OSStringLength(pszName) + 1;
+	psGroup->pszName = OSAllocMem(uSize * sizeof(*psGroup->pszName));
+	PVR_LOG_GOTO_IF_NOMEM(psGroup->pszName, eError, cleanup_name_);
+	OSStringSafeCopy(psGroup->pszName, pszName, uSize);
+
+	psGroup->psParent = psParent;
+	dllist_init(&psGroup->sGroupList);
+	dllist_init(&psGroup->sEntryList);
+	dllist_init(&psGroup->sNativeHandleList);
+
+	OSLockAcquire(_g_hLock);
+
+	dllist_add_to_tail(&psParent->sGroupList, &psGroup->sListNode);
+
+	/* Iterate over all of the native handles of parent group to create
+	 * the group for every registered implementation. */
+	dllist_foreach_node(&psParent->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup = NULL, *psNativeParent =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		eError = _CreateNativeGroup(psGroup, psNativeParent, &psNativeGroup);
+		PVR_GOTO_IF_ERROR(eError, cleanup_);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	*ppsGroup = psGroup;
+
+	return PVRSRV_OK;
+
+cleanup_:
+	OSLockRelease(_g_hLock);
+
+	/* Something went wrong so if there were any native groups created remove
+	 * them from the list, free them and free the DI group itself. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		dllist_remove_node(&psNativeGroup->sListNode);
+		OSFreeMem(psNativeGroup);
+	}
+
+	OSFreeMem(psGroup->pszName);
+cleanup_name_:
+	OSFreeMem(psGroup);
+
+	return eError;
+}
+
+void DIDestroyGroup(DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psGroup != NULL,
+	                             "psGroup invalid in DIDestroyGroup()");
+
+	/* Iterate through all of the native groups of the DI group, remove
+	 * them from the list and then destroy them. After that destroy the
+	 * DI group itself. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNative = IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE,
+		                                              sListNode);
+
+		psNative->psDiImpl->sCb.pfnDestroyGroup(psNative->pvHandle);
+		dllist_remove_node(&psNative->sListNode);
+		OSFreeMem(psNative);
+	}
+
+	dllist_remove_node(&psGroup->sListNode);
+
+	if (psGroup == _g_psRootGroup)
+	{
+		OSFreeMemNoStats(psGroup->pszName);
+		OSFreeMemNoStats(psGroup);
+	}
+	else
+	{
+		OSFreeMem(psGroup->pszName);
+		OSFreeMem(psGroup);
+	}
+}
+
+void *DIGetPrivData(const OSDI_IMPL_ENTRY *psEntry)
+{
+	PVR_ASSERT(psEntry != NULL);
+
+	return psEntry->pvPrivData;
+}
+
+void DIWrite(const OSDI_IMPL_ENTRY *psEntry, const void *pvData,
+             IMG_UINT32 uiSize)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnWrite != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnWrite(psEntry->pvNative, pvData, uiSize);
+}
+
+void DIPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt, ...)
+{
+	va_list args;
+
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnVPrintf != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	va_start(args, pszFmt);
+	psEntry->psCb->pfnVPrintf(psEntry->pvNative, pszFmt, args);
+	va_end(args);
+}
+
+void DIVPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt,
+               va_list pArgs)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnVPrintf != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnVPrintf(psEntry->pvNative, pszFmt, pArgs);
+}
+
+void DIPuts(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszStr)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnPuts != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnPuts(psEntry->pvNative, pszStr);
+}
+
+IMG_BOOL DIHasOverflowed(const OSDI_IMPL_ENTRY *psEntry)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnHasOverflowed != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	return psEntry->psCb->pfnHasOverflowed(psEntry->pvNative);
+}
+
+/* ---- OS implementation API ---------------------------------------------- */
+
+static IMG_BOOL _ValidateImplCb(const OSDI_IMPL_CB *psImplCb)
+{
+	PVR_GOTO_IF_FALSE(psImplCb->pfnInit != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDeInit != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnCreateGroup != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDestroyGroup != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnCreateEntry != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDestroyEntry != NULL, failed_);
+
+	return IMG_TRUE;
+
+failed_:
+	return IMG_FALSE;
+}
+
+/* Walks the tree of groups and entries and create all of the native handles
+ * for the given implementation for all of the already existing groups and
+ * entries. */
+static PVRSRV_ERROR _InitNativeHandlesRecursively(DI_IMPL *psImpl,
+                                               DI_GROUP *psGroup,
+                                               DI_NATIVE_HANDLE *psNativeParent)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_NATIVE_HANDLE *psNativeGroup;
+
+	psNativeGroup = OSAllocMem(sizeof(*psNativeGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psNativeGroup, "OSAllocMem");
+
+	eError = psImpl->sCb.pfnCreateGroup(psGroup->pszName,
+	                           psNativeParent ? psNativeParent->pvHandle : NULL,
+	                           &psNativeGroup->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateGroup", free_memory_);
+
+	psNativeGroup->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psGroup->sNativeHandleList,
+	                   &psNativeGroup->sListNode);
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		// and then walk the new group
+		eError = _InitNativeHandlesRecursively(psImpl, psThisGroup,
+		                                       psNativeGroup);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_InitNativeHandlesRecursively");
+	}
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+
+		eError = _CreateNativeEntry(psThisEntry, psNativeGroup);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_CreateNativeEntry");
+	}
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeGroup);
+
+	return eError;
+}
+
+/* Walks the tree of groups and entries and destroys all of the native handles
+ * for the given implementation. */
+static void _DeInitNativeHandlesRecursively(DI_IMPL *psImpl, DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+
+		// free all of the native entries that belong to this implementation
+		dllist_foreach_node(&psThisEntry->sNativeHandleList, psThis, psNext)
+		{
+			DI_NATIVE_HANDLE *psNativeEntry =
+			        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+			if (psNativeEntry->psDiImpl == psImpl)
+			{
+				_DestroyNativeEntry(psNativeEntry);
+				// there can be only one entry on the list for a given
+				// implementation
+				break;
+			}
+		}
+	}
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		// and then walk the new group
+		_DeInitNativeHandlesRecursively(psImpl, psThisGroup);
+	}
+
+	// free all of the native entries that belong to this implementation
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		if (psNativeGroup->psDiImpl == psImpl)
+		{
+			_DestroyNativeGroup(psNativeGroup);
+			// there can be only one entry on the list for a given
+			// implementation
+			break;
+		}
+	}
+}
+
+static PVRSRV_ERROR _InitImpl(DI_IMPL *psImpl)
+{
+	PVRSRV_ERROR eError;
+	// DI_NATIVE_HANDLE *psNativeGroup;
+
+	eError = psImpl->sCb.pfnInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->pfnInit()", return_);
+
+	/* if the implementation is being created after any groups or entries
+	 * have been created we need to walk the current tree and create
+	 * native groups and entries for all of the existing ones */
+	eError = _InitNativeHandlesRecursively(psImpl, _g_psRootGroup, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_InitNativeHandlesRecursively",
+	                      free_native_handles_and_deinit_);
+
+	psImpl->bInitialised = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+free_native_handles_and_deinit_:
+	/* something went wrong so we need to walk the tree and remove all of the
+	 * native entries and groups that we've created before we can destroy
+	 * the implementation */
+	_DeInitNativeHandlesRecursively(psImpl, _g_psRootGroup);
+	psImpl->sCb.pfnDeInit();
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIRegisterImplementation(const IMG_CHAR *pszName,
+                                      const OSDI_IMPL_CB *psImplCb)
+{
+	DI_IMPL *psImpl;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(_ValidateImplCb(psImplCb), "psImplCb");
+	/* if root group does not exist it can mean 2 things:
+	 * - DIInit() was not called so initialisation order is incorrect and needs
+	 *   to be fixed
+	 * - DIInit() failed but if that happens we should never make it here */
+	PVR_ASSERT(_g_psRootGroup != NULL);
+
+	psImpl = OSAllocMem(sizeof(*psImpl));
+	PVR_LOG_RETURN_IF_NOMEM(psImpl, "OSAllocMem");
+
+	psImpl->pszName = pszName;
+	psImpl->sCb = *psImplCb;
+
+	OSLockAcquire(_g_hLock);
+
+	eError = _InitImpl(psImpl);
+	if (eError != PVRSRV_OK)
+	{
+		/* implementation could not be initialised so remove it from the
+		 * list, free the memory and forget about it */
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: could not initialise \"%s\" debug "
+		        "info implementation, discarding", __func__,
+		        psImpl->pszName));
+
+		goto free_impl_;
+	}
+
+	psImpl->bInitialised = IMG_TRUE;
+
+	dllist_add_to_tail(&_g_sImpls, &psImpl->sListNode);
+
+	OSLockRelease(_g_hLock);
+
+	return PVRSRV_OK;
+
+free_impl_:
+	OSLockRelease(_g_hLock);
+
+	OSFreeMem(psImpl);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/dma_km.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/dma_km.c
new file mode 100644
index 000000000000..6e5ae4915847
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/dma_km.c
@@ -0,0 +1,564 @@
+/*************************************************************************/ /*!
+@File           dma_km.c
+@Title          kernel side of dma transfer scheduling
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements server side code for allowing DMA transfers between
+                cpu and device memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/version.h>
+#include <linux/uaccess.h>
+#include <asm/current.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#endif
+
+#include "pmr.h"
+#include "log2.h"
+#include "device.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "dma_km.h"
+#include "pvr_debug.h"
+#include "lock_types.h"
+#include "allocmem.h"
+#include "process_stats.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+#include "ri_server.h"
+#endif
+#include "devicemem.h"
+#include "pvrsrv_apphint.h"
+#include "pvrsrv_sync_server.h"
+#include "km_apphint_defs.h"
+#include "di_server.h"
+#include "dma_flags.h"
+
+/* This header must always be included last */
+#if defined(__linux__)
+#include "kernel_compatibility.h"
+#endif
+
+typedef struct _SERVER_CLEANUP_DATA_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	CONNECTION_DATA *psConnection;
+	IMG_UINT32 uiNumDMA;
+	IMG_UINT32 uiCount;
+	SYNC_TIMELINE_OBJ sTimelineObject;
+	void* pvChan;
+	PMR** ppsPMR;
+} SERVER_CLEANUP_DATA;
+
+#if !defined(NO_HARDWARE)
+static void Cleanup(void* pvCleanupData, IMG_BOOL bAdvanceTimeline)
+{
+	IMG_UINT i;
+	PVRSRV_ERROR eError;
+	SERVER_CLEANUP_DATA* psCleanupData = (SERVER_CLEANUP_DATA*) pvCleanupData;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Server Cleanup thread entry (%p)", pvCleanupData));
+#endif
+
+	for (i=0; i<psCleanupData->uiCount; i++)
+	{
+		eError = PMRUnlockSysPhysAddresses(psCleanupData->ppsPMR[i]);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+	}
+
+	/* Advance timeline */
+	if (psCleanupData->sTimelineObject.pvTlObj && bAdvanceTimeline)
+	{
+		eError = SyncSWTimelineAdvanceKM(psCleanupData->psDevNode, &psCleanupData->sTimelineObject);
+		PVR_LOG_IF_ERROR(eError, "SyncSWTimelineAdvanceKM");
+		eError = SyncSWTimelineReleaseKM(&psCleanupData->sTimelineObject);
+		PVR_LOG_IF_ERROR(eError, "SyncSWTimelineReleaseKM");
+	}
+
+	OSAtomicDecrement(&psCleanupData->psConnection->ui32NumDmaTransfersInFlight);
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Decremented to %d", OSAtomicRead(&psCleanupData->psConnection->ui32NumDmaTransfersInFlight)));
+#endif
+	eError = OSEventObjectSignal(psCleanupData->psConnection->hDmaEventObject);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSEventObjectSignal failed: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+	}
+
+
+	OSFreeMem(psCleanupData->ppsPMR);
+	OSFreeMem(psCleanupData);
+}
+#endif /* !defined(NO_HARDWARE) */
+
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode, CONNECTION_DATA *psConnectionData)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+	PVRSRV_ERROR eError;
+
+	if (!psDevConfig->bHasDma)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pfnSlaveDMAGetChan, "pfnSlaveDMAGetChan");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pfnSlaveDMAFreeChan, "pfnSlaveDMAFreeChan");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pszDmaTxChanName, "pszDmaTxChanName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevConfig->pszDmaRxChanName, "pszDmaRxChanName");
+
+	eError = OSEventObjectCreate("Dma transfer cleanup event object",
+								 &psConnectionData->hDmaEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", dma_init_error1);
+
+	OSLockAcquire(psDeviceNode->hConnectionsLock);
+
+	if (psDeviceNode->ui32RefCountDMA == 0)
+	{
+		psDeviceNode->hDmaTxChan =
+			psDevConfig->pfnSlaveDMAGetChan(psDevConfig,
+											psDevConfig->pszDmaTxChanName);
+		if (!psDeviceNode->hDmaTxChan)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_RESOURCE_UNAVAILABLE, dma_init_error2);
+		}
+		psDeviceNode->hDmaRxChan =
+			psDevConfig->pfnSlaveDMAGetChan(psDevConfig,
+											psDevConfig->pszDmaRxChanName);
+		if (!psDeviceNode->hDmaRxChan)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_RESOURCE_UNAVAILABLE, dma_init_error3);
+		}
+
+		eError = OSLockCreate(&psDeviceNode->hDmaTxLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(hDmaTxLock)", dma_init_error_txlock);
+
+		eError = OSLockCreate(&psDeviceNode->hDmaRxLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(hDmaRxLock)", dma_init_error_rxlock);
+	}
+
+	psDeviceNode->ui32RefCountDMA++;
+
+	OSLockRelease(psDeviceNode->hConnectionsLock);
+
+	OSAtomicWrite(&psConnectionData->ui32NumDmaTransfersInFlight, 0);
+	psConnectionData->bAcceptDmaRequests = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+dma_init_error_rxlock:
+	OSLockDestroy(psDeviceNode->hDmaTxLock);
+dma_init_error_txlock:
+	psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaRxChan);
+dma_init_error3:
+	psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaTxChan);
+dma_init_error2:
+	OSLockRelease(psDeviceNode->hConnectionsLock);
+	OSEventObjectDestroy(psConnectionData->hDmaEventObject);
+dma_init_error1:
+	return eError;
+}
+
+static void WaitForOutstandingDma(CONNECTION_DATA *psConnectionData)
+{
+
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	IMG_UINT32 ui32Tries = 100;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR,
+					"Waiting on %d DMA transfers in flight...", OSAtomicRead(&psConnectionData->ui32NumDmaTransfersInFlight)));
+#endif
+
+	eError = OSEventObjectOpen(psConnectionData->hDmaEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return;
+	}
+
+	while (OSAtomicRead(&psConnectionData->ui32NumDmaTransfersInFlight) != 0)
+	{
+		/*
+		#define DMA_TRANSFER_TIMEOUT_US (5000000ULL)
+
+		This currently doesn't work properly. Wait time is not as requested.
+		Using OSSleepms instead
+
+		OSEventObjectWaitKernel(hEvent, DMA_TRANSFER_TIMEOUT_US);
+		*/
+		OSSleepms(50);
+		if (!ui32Tries)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Timeout while waiting on outstanding DMA transfers!", __func__));
+			break;
+		}
+
+		ui32Tries--;
+	}
+
+	OSEventObjectClose(hEvent);
+}
+
+IMG_EXPORT void
+PVRSRVDeInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode, CONNECTION_DATA *psConnectionData)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+
+	if (!psDevConfig->bHasDma)
+	{
+		return;
+	}
+
+	OSLockAcquire(psDeviceNode->hDmaTxLock);
+	OSLockAcquire(psDeviceNode->hDmaRxLock);
+
+	psConnectionData->bAcceptDmaRequests = IMG_FALSE;
+
+	OSLockRelease(psDeviceNode->hDmaRxLock);
+	OSLockRelease(psDeviceNode->hDmaTxLock);
+
+	WaitForOutstandingDma(psConnectionData);
+
+	OSLockAcquire(psDeviceNode->hConnectionsLock);
+
+	if (psDeviceNode->ui32RefCountDMA == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid ref count (%d)", __func__, psDeviceNode->ui32RefCountDMA));
+		OSLockRelease(psDeviceNode->hConnectionsLock);
+		return;
+	}
+
+	if (--psDeviceNode->ui32RefCountDMA == 0)
+	{
+		if (psDeviceNode->hDmaRxLock != NULL)
+		{
+			OSLockDestroy(psDeviceNode->hDmaRxLock);
+		}
+
+		if (psDeviceNode->hDmaTxLock != NULL)
+		{
+			OSLockDestroy(psDeviceNode->hDmaTxLock);
+		}
+
+		psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaRxChan);
+		psDevConfig->pfnSlaveDMAFreeChan(psDevConfig, psDeviceNode->hDmaTxChan);
+	}
+
+	OSLockRelease(psDeviceNode->hConnectionsLock);
+
+	OSEventObjectDestroy(psConnectionData->hDmaEventObject);
+}
+
+IMG_EXPORT PVRSRV_ERROR
+DmaDeviceParams(CONNECTION_DATA *psConnection,
+				PVRSRV_DEVICE_NODE *psDevNode,
+				IMG_UINT32 *ui32DmaBuffAlign,
+				IMG_UINT32 *ui32DmaTransferMult)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+	*ui32DmaBuffAlign = psDevConfig->ui32DmaAlignment;
+	*ui32DmaTransferMult = psDevConfig->ui32DmaTransferUnit;
+
+	return PVRSRV_OK;
+}
+
+IMG_EXPORT PVRSRV_ERROR
+DmaSparseMappingTable(PMR *psPMR,
+					  IMG_DEVMEM_OFFSET_T uiOffset,
+					  IMG_UINT32 ui32SizeInPages,
+					  IMG_BOOL *pbTable)
+{
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_DEV_PHYADDR *psDevPhyAddr;
+		IMG_BOOL *pbValid;
+
+		psDevPhyAddr = OSAllocZMem(ui32SizeInPages * sizeof(IMG_CPU_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDevPhyAddr, eError, err1);
+
+		pbValid = OSAllocZMem(ui32SizeInPages * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, err2);
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", err3);
+
+		eError = PMR_DevPhysAddr(psPMR,
+								 OSGetPageShift(),
+								 ui32SizeInPages,
+								 uiOffset,
+								 psDevPhyAddr,
+								 pbValid,
+								 CPU_USE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", err3);
+
+		PMRUnlockSysPhysAddresses(psPMR);
+
+		memcpy(pbTable, pbValid, ui32SizeInPages * sizeof(IMG_BOOL));
+
+err3:
+		OSFreeMem(pbValid);
+err2:
+		OSFreeMem(psDevPhyAddr);
+err1:
+		return eError;
+}
+
+IMG_EXPORT PVRSRV_ERROR
+DmaTransfer(CONNECTION_DATA *psConnection,
+		    PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 uiNumDMAs,
+			PMR** ppsPMR,
+			IMG_UINT64 *puiAddress,
+			IMG_DEVMEM_OFFSET_T *puiOffset,
+			IMG_DEVMEM_SIZE_T *puiSize,
+			IMG_UINT32 uiFlags,
+			PVRSRV_TIMELINE iUpdateFenceTimeline)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(NO_HARDWARE)
+	/* On nohw the kernel call just advances the timeline to signal completion */
+
+	SYNC_TIMELINE_OBJ sSwTimeline = {NULL, PVRSRV_NO_TIMELINE};
+
+	if (iUpdateFenceTimeline != PVRSRV_NO_TIMELINE)
+	{
+		eError = SyncSWGetTimelineObj(iUpdateFenceTimeline, &sSwTimeline);
+		PVR_LOG_RETURN_IF_ERROR(eError, "SyncSWGetTimelineObj");
+
+		eError = SyncSWTimelineAdvanceKM(psDevNode, &sSwTimeline);
+		PVR_LOG_RETURN_IF_ERROR(eError, "SyncSWTimelineAdvanceKM");
+
+		eError = SyncSWTimelineReleaseKM(&sSwTimeline);
+		PVR_LOG_RETURN_IF_ERROR(eError, "SyncSWTimelineReleaseKM");
+	}
+
+	return PVRSRV_OK;
+
+#else
+	IMG_DEV_PHYADDR *psDevPhyAddr;
+	IMG_DMA_ADDR *psDmaAddr;
+	IMG_BOOL *pbValid;
+	IMG_UINT32 i;
+	PVRSRV_DEVICE_CONFIG* psDevConfig = psDevNode->psDevConfig;
+	void* pvChan = NULL;
+	SERVER_CLEANUP_DATA* psServerData;
+	void*  pvOSData;
+	POS_LOCK hChanLock = uiFlags & (DMA_FLAG_MEM_TO_DEV) ? psDevNode->hDmaTxLock : psDevNode->hDmaRxLock;
+
+	for (i=0; i<uiNumDMAs; i++)
+	{
+		PMR* psPMR = ppsPMR[i];
+		PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+		IMG_BOOL bIsPrivate = PhysHeapGetFlags(PMR_PhysHeap(psPMR)) & PHYS_HEAP_USAGE_GPU_PRIVATE;
+		IMG_BOOL bWrite = uiFlags & DMA_FLAG_MEM_TO_DEV;
+
+		if (PMR_GetType(psPMR) == PMR_TYPE_OSMEM)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("PMR is type OSMEM, expected device memory.", eError, PVRSRV_ERROR_NOT_SUPPORTED, error_return);
+		}
+
+		if (!bIsPrivate)
+		{
+			if (( bWrite && !PVRSRV_CHECK_CPU_WRITEABLE(uiPMRFlags)) ||
+			    (!bWrite && !PVRSRV_CHECK_CPU_READABLE(uiPMRFlags)))
+			{
+				PVR_LOG_GOTO_WITH_ERROR("Incorrect CPU access parameters on PMR", eError, PVRSRV_ERROR_INVALID_PARAMS, error_return);
+			}
+		}
+		else
+		{
+			/* Treat GPU private PMRs differently since they can't be mapped by the client.
+			   Check access permissions based on GPU flags. */
+			if (( bWrite && !PVRSRV_CHECK_GPU_WRITEABLE(uiPMRFlags)) ||
+			    (!bWrite && !PVRSRV_CHECK_GPU_READABLE(uiPMRFlags))  ||
+			     uiPMRFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT))
+			{
+				PVR_LOG_GOTO_WITH_ERROR("Incorrect GPU access parameters on PMR", eError, PVRSRV_ERROR_INVALID_PARAMS, error_return);
+			}
+		}
+	}
+
+	OSLockAcquire(hChanLock);
+
+	if (!psConnection->bAcceptDmaRequests)
+	{
+		OSLockRelease(hChanLock);
+		return PVRSRV_OK;
+	}
+
+	OSAtomicIncrement(&psConnection->ui32NumDmaTransfersInFlight);
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Incremented to %d", OSAtomicRead(&psConnection->ui32NumDmaTransfersInFlight)));
+#endif
+	psServerData = OSAllocZMem(sizeof(SERVER_CLEANUP_DATA));
+	PVR_LOG_GOTO_IF_NOMEM(psServerData, eError, e0);
+
+	pvChan = uiFlags & (DMA_FLAG_MEM_TO_DEV) ? psDevNode->hDmaTxChan : psDevNode->hDmaRxChan;
+	if (!pvChan)
+	{
+		eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		PVR_LOG_GOTO_IF_ERROR(eError, "Error acquiring DMA channel", e1);
+	}
+
+	if (iUpdateFenceTimeline != PVRSRV_NO_TIMELINE)
+	{
+		eError = SyncSWGetTimelineObj(iUpdateFenceTimeline, &psServerData->sTimelineObject);
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncSWGetTimelineObj", e1);
+	}
+
+	psServerData->uiCount = 0;
+	psServerData->psDevNode = psDevNode;
+	psServerData->psConnection = psConnection;
+	psServerData->pvChan = pvChan;
+	psServerData->ppsPMR = OSAllocZMem(sizeof(PMR*) * uiNumDMAs);
+	PVR_LOG_GOTO_IF_NOMEM(psServerData->ppsPMR, eError, e2);
+
+	eError = OSDmaAllocData(psDevNode, uiNumDMAs, &pvOSData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSDmaAllocData failed", e3);
+
+	for (i=0; i<uiNumDMAs; i++)
+	{
+		IMG_UINT32 ui32SizeInPages;
+		IMG_UINT32 uiOffsetInPage = puiOffset[i] & (OSGetPageSize() - 1);
+		ui32SizeInPages = (puiSize[i] + uiOffsetInPage + OSGetPageSize() - 1) >> OSGetPageShift();
+
+		psDmaAddr = OSAllocZMem(ui32SizeInPages * sizeof(IMG_DMA_ADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDmaAddr, eError, loop_e0);
+
+		psDevPhyAddr = OSAllocZMem(ui32SizeInPages * sizeof(IMG_CPU_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDevPhyAddr, eError, loop_e1);
+
+		pbValid = OSAllocZMem(ui32SizeInPages * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, loop_e2);
+
+		eError = PMRLockSysPhysAddresses(ppsPMR[i]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", loop_e3);
+
+		psServerData->ppsPMR[i] = ppsPMR[i];
+
+		eError = PMR_DevPhysAddr(ppsPMR[i],
+								 OSGetPageShift(),
+								 ui32SizeInPages,
+								 puiOffset[i],
+								 psDevPhyAddr,
+								 pbValid,
+								 CPU_USE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", loop_e4);
+
+		psDevConfig->pfnDevPhysAddr2DmaAddr(psDevConfig,
+											psDmaAddr,
+											psDevPhyAddr,
+											pbValid,
+											ui32SizeInPages,
+											PMR_IsSparse(ppsPMR[i]));
+
+		if (!PMR_IsSparse(ppsPMR[i]))
+		{
+			eError = OSDmaPrepareTransfer(psDevNode,
+										  pvChan,
+										  &psDmaAddr[0], (IMG_UINT64*)puiAddress[i],
+										  puiSize[i], (uiFlags & DMA_FLAG_MEM_TO_DEV), pvOSData,
+										  psServerData, Cleanup, (i == 0));
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSDmaPrepareTransfer", loop_e4);
+			psServerData->uiCount++;
+
+		}
+		else
+		{
+			eError = OSDmaPrepareTransferSparse(psDevNode, pvChan,
+												psDmaAddr, pbValid,
+												(IMG_UINT64*)puiAddress[i], puiSize[i],
+												uiOffsetInPage, ui32SizeInPages,
+												(uiFlags & DMA_FLAG_MEM_TO_DEV),
+												pvOSData, psServerData,
+												Cleanup, (i == 0));
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSDmaPrepareTransferSparse", loop_e4);
+			psServerData->uiCount++;
+		}
+
+		OSFreeMem(pbValid);
+		OSFreeMem(psDevPhyAddr);
+		OSFreeMem(psDmaAddr);
+
+		continue;
+
+loop_e4:
+		PMRUnlockSysPhysAddresses(ppsPMR[i]);
+loop_e3:
+		OSFreeMem(pbValid);
+loop_e2:
+		OSFreeMem(psDevPhyAddr);
+loop_e1:
+		OSFreeMem(psDmaAddr);
+loop_e0:
+		break;
+	}
+
+	if (psServerData->uiCount == uiNumDMAs)
+	{
+		OSDmaSubmitTransfer(psDevNode, pvOSData, pvChan, (uiFlags & DMA_FLAG_SYNCHRONOUS));
+	}
+	else
+	{
+		/* One of the transfers could not be programmed, roll back */
+		OSDmaForceCleanup(psDevNode, pvChan, pvOSData, psServerData, Cleanup);
+	}
+	OSLockRelease(hChanLock);
+	return eError;
+
+e3:
+	OSFreeMem(psServerData->ppsPMR);
+e2:
+	if (iUpdateFenceTimeline != PVRSRV_NO_TIMELINE)
+	{
+		SyncSWTimelineReleaseKM(&psServerData->sTimelineObject);
+	}
+e1:
+	OSFreeMem(psServerData);
+e0:
+	OSLockRelease(hChanLock);
+error_return:
+	return eError;
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/handle.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/handle.c
new file mode 100644
index 000000000000..7ebd38861963
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/handle.c
@@ -0,0 +1,2522 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Handle Manager
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide resource handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+/* See handle.h for a description of the handle API. */
+
+/*
+ * The implementation supports movable handle structures, allowing the address
+ * of a handle structure to change without having to fix up pointers in
+ * any of the handle structures. For example, the linked list mechanism
+ * used to link subhandles together uses handle array indices rather than
+ * pointers to the structures themselves.
+ */
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "handle.h"
+#include "handle_impl.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "connection_server.h"
+
+#define	HANDLE_HASH_TAB_INIT_SIZE 32
+#define HANDLE_PROC_HANDLE_HASH_INIT_SIZE 10
+
+#define	TEST_FLAG(v, f) BITMASK_HAS(v, f)
+#define	TEST_ALLOC_FLAG(psHandleData, f) BITMASK_HAS((psHandleData)->eFlag, f)
+
+
+/* Linked list structure. Used for both the list head and list items */
+typedef struct _HANDLE_LIST_
+{
+	IMG_HANDLE hPrev;
+	IMG_HANDLE hNext;
+	IMG_HANDLE hParent;
+} HANDLE_LIST;
+
+typedef struct _HANDLE_DATA_
+{
+	/* The handle that represents this structure */
+	IMG_HANDLE hHandle;
+
+	/* Handle type */
+	PVRSRV_HANDLE_TYPE eType;
+
+	/* Flags specified when the handle was allocated */
+	PVRSRV_HANDLE_ALLOC_FLAG eFlag;
+
+	/* Pointer to the data that the handle represents */
+	void *pvData;
+
+	/*
+	 * Callback specified at handle allocation time to
+	 * release/destroy/free the data represented by the
+	 * handle when it's reference count reaches 0. This
+	 * should always be NULL for subhandles.
+	 */
+	PFN_HANDLE_RELEASE pfnReleaseData;
+
+	/* List head for subhandles of this handle */
+	HANDLE_LIST sChildren;
+
+	/* List entry for sibling subhandles */
+	HANDLE_LIST sSiblings;
+
+	/* Reference count of lookups made. It helps track which resources are in
+	 * use in concurrent bridge calls. */
+	IMG_INT32 iLookupCount;
+	/* State of a handle. If the handle was already destroyed this is false.
+	 * If this is false and iLookupCount is 0 the pfnReleaseData callback is
+	 * called on the handle. */
+	IMG_BOOL bCanLookup;
+
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	/* Store the handle base used for this handle, so we
+	 * can later access the handle base lock (or check if
+	 * it has been already acquired)
+	 */
+	PVRSRV_HANDLE_BASE *psBase;
+#endif
+
+} HANDLE_DATA;
+
+struct _HANDLE_BASE_
+{
+	/* Pointer to a handle implementations base structure */
+	HANDLE_IMPL_BASE *psImplBase;
+
+	/*
+	 * Pointer to handle hash table.
+	 * The hash table is used to do reverse lookups, converting data
+	 * pointers to handles.
+	 */
+	HASH_TABLE *psHashTab;
+
+	/* Type specific (connection/global/process) Lock handle */
+	POS_LOCK hLock;
+
+	/* Can be connection, process, global */
+	PVRSRV_HANDLE_BASE_TYPE eType;
+};
+
+/*
+ * The key for the handle hash table is an array of three elements, the
+ * pointer to the resource, the resource type and the parent handle (or
+ * NULL if there is no parent). The eHandKey enumeration gives the
+ * array indices of the elements making up the key.
+ */
+enum eHandKey
+{
+	HAND_KEY_DATA = 0,
+	HAND_KEY_TYPE,
+	HAND_KEY_PARENT,
+	HAND_KEY_LEN		/* Must be last item in list */
+};
+
+/* HAND_KEY is the type of the hash table key */
+typedef uintptr_t HAND_KEY[HAND_KEY_LEN];
+
+typedef struct FREE_HANDLE_DATA_TAG
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	PVRSRV_HANDLE_TYPE eHandleFreeType;
+	/* timing data (ns) to release bridge lock upon the deadline */
+	IMG_UINT64 ui64TimeStart;
+	IMG_UINT64 ui64MaxBridgeTime;
+} FREE_HANDLE_DATA;
+
+typedef struct FREE_KERNEL_HANDLE_DATA_TAG
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	HANDLE_DATA *psProcessHandleData;
+	IMG_HANDLE hKernelHandle;
+} FREE_KERNEL_HANDLE_DATA;
+
+/* Stores a pointer to the function table of the handle back-end in use */
+static HANDLE_IMPL_FUNCTAB const *gpsHandleFuncs;
+
+static POS_LOCK gKernelHandleLock;
+static IMG_BOOL gbLockInitialised = IMG_FALSE;
+/* Pointer to process handle base currently being freed */
+static PVRSRV_HANDLE_BASE *g_psProcessHandleBaseBeingFreed;
+/* Lock for the process handle base table */
+static POS_LOCK g_hProcessHandleBaseLock;
+/* Hash table with process handle bases */
+static HASH_TABLE *g_psProcessHandleBaseTable;
+
+void LockHandle(PVRSRV_HANDLE_BASE *psBase)
+{
+	OSLockAcquire(psBase->hLock);
+}
+
+void UnlockHandle(PVRSRV_HANDLE_BASE *psBase)
+{
+	OSLockRelease(psBase->hLock);
+}
+
+/*
+ * Kernel handle base structure. This is used for handles that are not
+ * allocated on behalf of a particular process.
+ */
+PVRSRV_HANDLE_BASE *gpsKernelHandleBase = NULL;
+
+/* Increase the lookup reference count on the given handle.
+ * The handle lock must already be acquired.
+ * Returns: the reference count after the increment
+ */
+static inline IMG_UINT32 HandleGet(HANDLE_DATA *psHandleData)
+{
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	if (!OSLockIsLocked(psHandleData->psBase->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle lock is not locked", __func__));
+		OSDumpStack();
+	}
+#endif
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = %u, iLookupCount %d -> %d",
+	        __func__, psHandleData->bCanLookup, psHandleData->iLookupCount,
+	        psHandleData->iLookupCount + 1));
+#endif /* DEBUG_REFCNT */
+
+	PVR_ASSERT(psHandleData->bCanLookup);
+
+	return ++psHandleData->iLookupCount;
+}
+
+/* Decrease the lookup reference count on the given handle.
+ * The handle lock must already be acquired.
+ * Returns: the reference count after the decrement
+ */
+static inline IMG_UINT32 HandlePut(HANDLE_DATA *psHandleData)
+{
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	if (!OSLockIsLocked(psHandleData->psBase->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle lock is not locked", __func__));
+		OSDumpStack();
+	}
+#endif
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = %u, iLookupCount %d -> %d",
+	        __func__, psHandleData->bCanLookup, psHandleData->iLookupCount,
+	        psHandleData->iLookupCount - 1));
+#endif /* DEBUG_REFCNT */
+
+	/* psHandleData->bCanLookup can be false at this point */
+	PVR_ASSERT(psHandleData->iLookupCount > 0);
+
+	return --psHandleData->iLookupCount;
+}
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+static const IMG_CHAR *HandleTypeToString(PVRSRV_HANDLE_TYPE eType)
+{
+	#define HANDLETYPE(x) \
+			case PVRSRV_HANDLE_TYPE_##x: \
+				return #x;
+	switch (eType)
+	{
+		#include "handle_types.h"
+		#undef HANDLETYPE
+
+		default:
+			return "INVALID";
+	}
+}
+
+static const IMG_CHAR *HandleBaseTypeToString(PVRSRV_HANDLE_BASE_TYPE eType)
+{
+	#define HANDLEBASETYPE(x) \
+			case PVRSRV_HANDLE_BASE_TYPE_##x: \
+				return #x;
+	switch (eType)
+	{
+		HANDLEBASETYPE(CONNECTION);
+		HANDLEBASETYPE(PROCESS);
+		HANDLEBASETYPE(GLOBAL);
+		#undef HANDLEBASETYPE
+
+		default:
+			return "INVALID";
+	}
+}
+#endif
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFree(PVRSRV_HANDLE_BASE *psBase,
+                                                   HANDLE_DATA *psHandleData,
+                                                   IMG_HANDLE hHandle,
+                                                   PVRSRV_HANDLE_TYPE eType);
+
+static PVRSRV_ERROR HandleFreePrivData(PVRSRV_HANDLE_BASE *psBase,
+                                       HANDLE_DATA *psHandleData,
+                                       IMG_HANDLE hHandle,
+                                       PVRSRV_HANDLE_TYPE eType);
+
+static PVRSRV_ERROR HandleFreeDestroy(PVRSRV_HANDLE_BASE *psBase,
+                                      HANDLE_DATA *psHandleData,
+                                      IMG_HANDLE hHandle,
+                                      PVRSRV_HANDLE_TYPE eType);
+
+/*!
+*******************************************************************************
+ @Function      GetHandleData
+ @Description   Get the handle data structure for a given handle
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type or PVRSRV_HANDLE_TYPE_NONE if the handle
+                        type is not to be checked.
+ @Output        ppsHandleData - pointer to a pointer to the handle data struct
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(GetHandleData)
+#endif
+static INLINE
+PVRSRV_ERROR GetHandleData(PVRSRV_HANDLE_BASE *psBase,
+			   HANDLE_DATA **ppsHandleData,
+			   IMG_HANDLE hHandle,
+			   PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError;
+
+	eError = gpsHandleFuncs->pfnGetHandleData(psBase->psImplBase,
+						  hHandle,
+						  (void **)&psHandleData);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/*
+	 * Unless PVRSRV_HANDLE_TYPE_NONE was passed in to this function,
+	 * check handle is of the correct type.
+	 */
+	if (unlikely(eType != PVRSRV_HANDLE_TYPE_NONE && eType != psHandleData->eType))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "GetHandleData: Type mismatch. Lookup request: Handle %p, type: %s (%u) but stored handle is type %s (%u)",
+			 hHandle,
+			 HandleTypeToString(eType),
+			 eType,
+			 HandleTypeToString(psHandleData->eType),
+			 psHandleData->eType));
+		return PVRSRV_ERROR_HANDLE_TYPE_MISMATCH;
+	}
+
+	/* Return the handle structure */
+	*ppsHandleData = psHandleData;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListInit
+ @Description   Initialise a linked list structure embedded in a handle
+                structure.
+ @Input         hHandle - handle containing the linked list structure
+                psList - pointer to linked list structure
+                hParent - parent handle or NULL
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListInit)
+#endif
+static INLINE
+void HandleListInit(IMG_HANDLE hHandle, HANDLE_LIST *psList, IMG_HANDLE hParent)
+{
+	psList->hPrev = hHandle;
+	psList->hNext = hHandle;
+	psList->hParent = hParent;
+}
+
+/*!
+*******************************************************************************
+ @Function      InitParentList
+ @Description   Initialise the children list head in a handle structure.
+                The children are the subhandles of this handle.
+ @Input         psHandleData - pointer to handle data structure
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitParentList)
+#endif
+static INLINE
+void InitParentList(HANDLE_DATA *psHandleData)
+{
+	IMG_HANDLE hParent = psHandleData->hHandle;
+
+	HandleListInit(hParent, &psHandleData->sChildren, hParent);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitChildEntry
+ @Description   Initialise the child list entry in a handle structure. The list
+                entry is used to link together subhandles of a given handle.
+ @Input         psHandleData - pointer to handle data structure
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitChildEntry)
+#endif
+static INLINE
+void InitChildEntry(HANDLE_DATA *psHandleData)
+{
+	HandleListInit(psHandleData->hHandle, &psHandleData->sSiblings, NULL);
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListIsEmpty
+ @Description   Determine whether a given linked list is empty.
+ @Input         hHandle - handle containing the list head
+                psList - pointer to the list head
+ @Return        IMG_TRUE if the list is empty, IMG_FALSE if it isn't.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListIsEmpty)
+#endif
+static INLINE
+IMG_BOOL HandleListIsEmpty(IMG_HANDLE hHandle, HANDLE_LIST *psList) /* Instead of passing in the handle can we not just do (psList->hPrev == psList->hNext) ? IMG_TRUE : IMG_FALSE ??? */
+{
+	IMG_BOOL bIsEmpty;
+
+	bIsEmpty = (IMG_BOOL)(psList->hNext == hHandle);
+
+#ifdef DEBUG
+	{
+		IMG_BOOL bIsEmpty2;
+
+		bIsEmpty2 = (IMG_BOOL)(psList->hPrev == hHandle);
+		PVR_ASSERT(bIsEmpty == bIsEmpty2);
+	}
+#endif
+
+	return bIsEmpty;
+}
+
+#ifdef DEBUG
+/*!
+*******************************************************************************
+ @Function      NoChildren
+ @Description   Determine whether a handle has any subhandles
+ @Input         psHandleData - pointer to handle data structure
+ @Return        IMG_TRUE if the handle has no subhandles, IMG_FALSE if it does.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(NoChildren)
+#endif
+static INLINE
+IMG_BOOL NoChildren(HANDLE_DATA *psHandleData)
+{
+	PVR_ASSERT(psHandleData->sChildren.hParent == psHandleData->hHandle);
+
+	return HandleListIsEmpty(psHandleData->hHandle, &psHandleData->sChildren);
+}
+
+/*!
+*******************************************************************************
+ @Function      NoParent
+ @Description   Determine whether a handle is a subhandle
+ @Input         psHandleData - pointer to handle data structure
+ @Return        IMG_TRUE if the handle is not a subhandle, IMG_FALSE if it is.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(NoParent)
+#endif
+static INLINE
+IMG_BOOL NoParent(HANDLE_DATA *psHandleData)
+{
+	if (HandleListIsEmpty(psHandleData->hHandle, &psHandleData->sSiblings))
+	{
+		PVR_ASSERT(psHandleData->sSiblings.hParent == NULL);
+
+		return IMG_TRUE;
+	}
+
+	PVR_ASSERT(psHandleData->sSiblings.hParent != NULL);
+	return IMG_FALSE;
+}
+#endif /*DEBUG*/
+
+/*!
+*******************************************************************************
+ @Function      ParentHandle
+ @Description   Determine the parent of a handle
+ @Input         psHandleData - pointer to handle data structure
+ @Return        Parent handle, or NULL if the handle is not a subhandle.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(ParentHandle)
+#endif
+static INLINE
+IMG_HANDLE ParentHandle(HANDLE_DATA *psHandleData)
+{
+	return psHandleData->sSiblings.hParent;
+}
+
+/*
+ * GetHandleListFromHandleAndOffset is used to generate either a
+ * pointer to the subhandle list head, or a pointer to the linked list
+ * structure of an item on a subhandle list.
+ * The list head is itself on the list, but is at a different offset
+ * in the handle structure to the linked list structure for items on
+ * the list. The two linked list structures are differentiated by
+ * the third parameter, containing the parent handle. The parent field
+ * in the list head structure references the handle structure that contains
+ * it. For items on the list, the parent field in the linked list structure
+ * references the parent handle, which will be different from the handle
+ * containing the linked list structure.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(GetHandleListFromHandleAndOffset)
+#endif
+static INLINE
+HANDLE_LIST *GetHandleListFromHandleAndOffset(PVRSRV_HANDLE_BASE *psBase,
+					      IMG_HANDLE hEntry,
+					      IMG_HANDLE hParent,
+					      size_t uiParentOffset,
+					      size_t uiEntryOffset)
+{
+	HANDLE_DATA *psHandleData = NULL;
+
+	PVR_ASSERT(psBase != NULL);
+
+	if (GetHandleData(psBase, &psHandleData, hEntry,
+	                  PVRSRV_HANDLE_TYPE_NONE) != PVRSRV_OK)
+	{
+		return NULL;
+	}
+
+	if (hEntry == hParent)
+	{
+		return (HANDLE_LIST *)IMG_OFFSET_ADDR(psHandleData, uiParentOffset);
+	}
+	else
+	{
+		return (HANDLE_LIST *)IMG_OFFSET_ADDR(psHandleData, uiEntryOffset);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListInsertBefore
+ @Description   Insert a handle before a handle currently on the list.
+ @Input         hEntry - handle to be inserted after
+                psEntry - pointer to handle structure to be inserted after
+                uiParentOffset - offset to list head struct in handle structure
+                hNewEntry - handle to be inserted
+                psNewEntry - pointer to handle structure of item to be inserted
+                uiEntryOffset - offset of list item struct in handle structure
+                hParent - parent handle of hNewEntry
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListInsertBefore)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListInsertBefore(PVRSRV_HANDLE_BASE *psBase,
+				    IMG_HANDLE hEntry,
+				    HANDLE_LIST *psEntry,
+				    size_t uiParentOffset,
+				    IMG_HANDLE hNewEntry,
+				    HANDLE_LIST *psNewEntry,
+				    size_t uiEntryOffset,
+				    IMG_HANDLE hParent)
+{
+	HANDLE_LIST *psPrevEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psEntry != NULL, "psEntry");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNewEntry != NULL, "psNewEntry");
+
+	psPrevEntry = GetHandleListFromHandleAndOffset(psBase,
+						       psEntry->hPrev,
+						       hParent,
+						       uiParentOffset,
+						       uiEntryOffset);
+	if (psPrevEntry == NULL)
+	{
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+
+	PVR_ASSERT(psNewEntry->hParent == NULL);
+	PVR_ASSERT(hEntry == psPrevEntry->hNext);
+
+#if defined(DEBUG)
+	{
+		HANDLE_LIST *psParentList;
+
+		psParentList = GetHandleListFromHandleAndOffset(psBase,
+								hParent,
+								hParent,
+								uiParentOffset,
+								uiParentOffset);
+		PVR_ASSERT(psParentList && psParentList->hParent == hParent);
+	}
+#endif /* defined(DEBUG) */
+
+	psNewEntry->hPrev = psEntry->hPrev;
+	psEntry->hPrev = hNewEntry;
+
+	psNewEntry->hNext = hEntry;
+	psPrevEntry->hNext = hNewEntry;
+
+	psNewEntry->hParent = hParent;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      AdoptChild
+ @Description   Assign a subhandle to a handle
+ @Input         psParentData - pointer to handle structure of parent handle
+                psChildData - pointer to handle structure of child subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(AdoptChild)
+#endif
+static INLINE
+PVRSRV_ERROR AdoptChild(PVRSRV_HANDLE_BASE *psBase,
+			HANDLE_DATA *psParentData,
+			HANDLE_DATA *psChildData)
+{
+	IMG_HANDLE hParent = psParentData->sChildren.hParent;
+
+	PVR_ASSERT(hParent == psParentData->hHandle);
+
+	return HandleListInsertBefore(psBase,
+				      hParent,
+				      &psParentData->sChildren,
+				      offsetof(HANDLE_DATA, sChildren),
+				      psChildData->hHandle,
+				      &psChildData->sSiblings,
+				      offsetof(HANDLE_DATA, sSiblings),
+				      hParent);
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListRemove
+ @Description   Remove a handle from a list
+ @Input         hEntry - handle to be removed
+                psEntry - pointer to handle structure of item to be removed
+                uiEntryOffset - offset of list item struct in handle structure
+                uiParentOffset - offset to list head struct in handle structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListRemove)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListRemove(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE hEntry,
+			      HANDLE_LIST *psEntry,
+			      size_t uiEntryOffset,
+			      size_t uiParentOffset)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psEntry != NULL, "psEntry");
+
+	if (!HandleListIsEmpty(hEntry, psEntry))
+	{
+		HANDLE_LIST *psPrev;
+		HANDLE_LIST *psNext;
+
+		psPrev = GetHandleListFromHandleAndOffset(psBase,
+							  psEntry->hPrev,
+							  psEntry->hParent,
+							  uiParentOffset,
+							  uiEntryOffset);
+		if (psPrev == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		psNext = GetHandleListFromHandleAndOffset(psBase,
+							  psEntry->hNext,
+							  psEntry->hParent,
+							  uiParentOffset,
+							  uiEntryOffset);
+		if (psNext == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		/*
+		 * The list head is on the list, and we don't want to
+		 * remove it.
+		 */
+		PVR_ASSERT(psEntry->hParent != NULL);
+
+		psPrev->hNext = psEntry->hNext;
+		psNext->hPrev = psEntry->hPrev;
+
+		HandleListInit(hEntry, psEntry, NULL);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      UnlinkFromParent
+ @Description   Remove a subhandle from its parents list
+ @Input         psHandleData - pointer to handle data structure of child
+                               subhandle.
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(UnlinkFromParent)
+#endif
+static INLINE
+PVRSRV_ERROR UnlinkFromParent(PVRSRV_HANDLE_BASE *psBase,
+			      HANDLE_DATA *psHandleData)
+{
+	return HandleListRemove(psBase,
+				psHandleData->hHandle,
+				&psHandleData->sSiblings,
+				offsetof(HANDLE_DATA, sSiblings),
+				offsetof(HANDLE_DATA, sChildren));
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListIterate
+ @Description   Iterate over the items in a list
+ @Input         psHead - pointer to list head
+                uiParentOffset - offset to list head struct in handle structure
+                uiEntryOffset - offset of list item struct in handle structure
+                pfnIterFunc - function to be called for each handle in the list
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListIterate)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListIterate(PVRSRV_HANDLE_BASE *psBase,
+			       HANDLE_LIST *psHead,
+			       size_t uiParentOffset,
+			       size_t uiEntryOffset,
+			       PVRSRV_ERROR (*pfnIterFunc)(PVRSRV_HANDLE_BASE *, IMG_HANDLE))
+{
+	IMG_HANDLE hHandle = psHead->hNext;
+	IMG_HANDLE hParent = psHead->hParent;
+	IMG_HANDLE hNext;
+
+	PVR_ASSERT(psHead->hParent != NULL);
+
+	/*
+	 * Follow the next chain from the list head until we reach
+	 * the list head again, which signifies the end of the list.
+	 */
+	while (hHandle != hParent)
+	{
+		HANDLE_LIST *psEntry;
+		PVRSRV_ERROR eError;
+
+		psEntry = GetHandleListFromHandleAndOffset(psBase,
+							   hHandle,
+							   hParent,
+							   uiParentOffset,
+							   uiEntryOffset);
+		if (psEntry == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		PVR_ASSERT(psEntry->hParent == psHead->hParent);
+
+		/*
+		 * Get the next index now, in case the list item is
+		 * modified by the iteration function.
+		 */
+		hNext = psEntry->hNext;
+
+		eError = (*pfnIterFunc)(psBase, hHandle);
+		PVR_RETURN_IF_ERROR(eError);
+
+		hHandle = hNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      IterateOverChildren
+ @Description   Iterate over the subhandles of a parent handle
+ @Input         psParentData - pointer to parent handle structure
+                pfnIterFunc - function to be called for each subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(IterateOverChildren)
+#endif
+static INLINE
+PVRSRV_ERROR IterateOverChildren(PVRSRV_HANDLE_BASE *psBase,
+				 HANDLE_DATA *psParentData,
+				 PVRSRV_ERROR (*pfnIterFunc)(PVRSRV_HANDLE_BASE *, IMG_HANDLE))
+{
+	 return HandleListIterate(psBase,
+				  &psParentData->sChildren,
+				  offsetof(HANDLE_DATA, sChildren),
+				  offsetof(HANDLE_DATA, sSiblings),
+				  pfnIterFunc);
+}
+
+/*!
+*******************************************************************************
+ @Function      ParentIfPrivate
+ @Description   Return the parent handle if the handle was allocated with
+                PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE, else return NULL.
+ @Input         psHandleData - pointer to handle data structure
+ @Return        Parent handle or NULL
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(ParentIfPrivate)
+#endif
+static INLINE
+IMG_HANDLE ParentIfPrivate(HANDLE_DATA *psHandleData)
+{
+	return TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE) ?
+			ParentHandle(psHandleData) : NULL;
+}
+
+/*!
+*******************************************************************************
+ @Function      InitKey
+ @Description   Initialise a hash table key for the current process
+ @Input         aKey - pointer to key
+                psBase - pointer to handle base structure
+                pvData - pointer to the resource the handle represents
+                eType - type of resource
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitKey)
+#endif
+static INLINE
+void InitKey(HAND_KEY aKey,
+	     PVRSRV_HANDLE_BASE *psBase,
+	     void *pvData,
+	     PVRSRV_HANDLE_TYPE eType,
+	     IMG_HANDLE hParent)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+
+	aKey[HAND_KEY_DATA] = (uintptr_t)pvData;
+	aKey[HAND_KEY_TYPE] = (uintptr_t)eType;
+	aKey[HAND_KEY_PARENT] = (uintptr_t)hParent;
+}
+
+/*!
+*******************************************************************************
+ @Function      FindHandle
+ @Description   Find handle corresponding to a resource pointer
+ @Input         psBase - pointer to handle base structure
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Return        The handle, or NULL if not found
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(FindHandle)
+#endif
+static INLINE
+IMG_HANDLE FindHandle(PVRSRV_HANDLE_BASE *psBase,
+		      void *pvData,
+		      PVRSRV_HANDLE_TYPE eType,
+		      IMG_HANDLE hParent)
+{
+	HAND_KEY aKey;
+
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+
+	InitKey(aKey, psBase, pvData, eType, hParent);
+
+	return (IMG_HANDLE) HASH_Retrieve_Extended(psBase->psHashTab, aKey);
+}
+
+/*!
+*******************************************************************************
+ @Function      AllocHandle
+ @Description   Allocate a new handle
+ @Input         phHandle - location for new handle
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                hParent - parent handle or NULL
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+static PVRSRV_ERROR AllocHandle(PVRSRV_HANDLE_BASE *psBase,
+				IMG_HANDLE *phHandle,
+				void *pvData,
+				PVRSRV_HANDLE_TYPE eType,
+				PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				IMG_HANDLE hParent,
+				PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	HANDLE_DATA *psNewHandleData;
+	IMG_HANDLE hHandle;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(psBase != NULL && psBase->psHashTab != NULL);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	if (!TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		/* Handle must not already exist */
+		PVR_ASSERT(FindHandle(psBase, pvData, eType, hParent) == NULL);
+	}
+
+	psNewHandleData = OSAllocZMem(sizeof(*psNewHandleData));
+	PVR_LOG_RETURN_IF_NOMEM(psNewHandleData, "OSAllocZMem");
+
+	eError = gpsHandleFuncs->pfnAcquireHandle(psBase->psImplBase, &hHandle,
+	                                          psNewHandleData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnAcquireHandle",
+	                  ErrorFreeHandleData);
+
+	/*
+	 * If a data pointer can be associated with multiple handles, we
+	 * don't put the handle in the hash table, as the data pointer
+	 * may not map to a unique handle
+	 */
+	if (!TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+
+		/* Initialise hash key */
+		InitKey(aKey, psBase, pvData, eType, hParent);
+
+		/* Put the new handle in the hash table */
+		eError = HASH_Insert_Extended(psBase->psHashTab, aKey, (uintptr_t)hHandle) ?
+		        PVRSRV_OK : PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "couldn't add handle to hash table",
+		                  ErrorReleaseHandle);
+	}
+
+	psNewHandleData->hHandle = hHandle;
+	psNewHandleData->eType = eType;
+	psNewHandleData->eFlag = eFlag;
+	psNewHandleData->pvData = pvData;
+	psNewHandleData->pfnReleaseData = pfnReleaseData;
+	psNewHandleData->iLookupCount = 0;
+	psNewHandleData->bCanLookup = IMG_TRUE;
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = true", __func__));
+#endif /* DEBUG_REFCNT */
+
+	InitParentList(psNewHandleData);
+#if defined(DEBUG)
+	PVR_ASSERT(NoChildren(psNewHandleData));
+#endif
+
+	InitChildEntry(psNewHandleData);
+#if defined(DEBUG)
+	PVR_ASSERT(NoParent(psNewHandleData));
+#endif
+
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	psNewHandleData->psBase = psBase;
+#endif
+
+	/* Return the new handle to the client */
+	*phHandle = psNewHandleData->hHandle;
+
+	return PVRSRV_OK;
+
+ErrorReleaseHandle:
+	(void)gpsHandleFuncs->pfnReleaseHandle(psBase->psImplBase, hHandle, NULL);
+
+ErrorFreeHandleData:
+	OSFreeMem(psNewHandleData);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandle
+ @Description   Allocate a handle
+ @Input         psBase - pointer to handle base structure
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase,
+			       IMG_HANDLE *phHandle,
+			       void *pvData,
+			       PVRSRV_HANDLE_TYPE eType,
+			       PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+			       PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVAllocHandleUnlocked(psBase, phHandle, pvData, eType, eFlag, pfnReleaseData);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandleUnlocked
+ @Description   Allocate a handle without acquiring/releasing the handle lock.
+                The function assumes you hold the lock when called.
+ @Input         phHandle - location for new handle
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+			       IMG_HANDLE *phHandle,
+			       void *pvData,
+			       PVRSRV_HANDLE_TYPE eType,
+			       PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+			       PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	*phHandle = NULL;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnReleaseData != NULL, "pfnReleaseData");
+
+	return AllocHandle(psBase, phHandle, pvData, eType, eFlag, NULL, pfnReleaseData);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocSubHandle
+ @Description   Allocate a subhandle
+ @Input         pvData - pointer to resource to be associated with the subhandle
+                eType - the type of resource
+                hParent - parent handle
+ @Output        phHandle - points to new subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData,
+				  PVRSRV_HANDLE_TYPE eType,
+				  PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				  IMG_HANDLE hParent)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVAllocSubHandleUnlocked(psBase, phHandle, pvData, eType, eFlag, hParent);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocSubHandleUnlocked
+ @Description   Allocate a subhandle without acquiring/releasing the handle
+                lock. The function assumes you hold the lock when called.
+ @Input         pvData - pointer to resource to be associated with the subhandle
+                eType - the type of resource
+                hParent - parent handle
+ @Output        phHandle - points to new subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocSubHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData,
+				  PVRSRV_HANDLE_TYPE eType,
+				  PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				  IMG_HANDLE hParent)
+{
+	HANDLE_DATA *psPHandleData = NULL;
+	HANDLE_DATA *psCHandleData = NULL;
+	IMG_HANDLE hParentKey;
+	IMG_HANDLE hHandle;
+	PVRSRV_ERROR eError;
+
+	*phHandle = NULL;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psBase, eError, Exit);
+
+	hParentKey = TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE) ? hParent : NULL;
+
+	/* Lookup the parent handle */
+	eError = GetHandleData(psBase, &psPHandleData, hParent, PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to get parent handle structure",
+	                  Exit);
+
+	eError = AllocHandle(psBase, &hHandle, pvData, eType, eFlag, hParentKey, NULL);
+	PVR_GOTO_IF_ERROR(eError, Exit);
+
+	eError = GetHandleData(psBase, &psCHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+	/* If we were able to allocate the handle then there should be no reason why we
+	 * can't also get it's handle structure. Otherwise something has gone badly wrong.
+	 */
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "Failed to get parent handle structure",
+	                  ExitFreeHandle);
+
+	/*
+	 * Get the parent handle structure again, in case the handle
+	 * structure has moved (depending on the implementation
+	 * of AllocHandle).
+	 */
+	eError = GetHandleData(psBase, &psPHandleData, hParent, PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to get parent handle structure",
+	                  ExitFreeHandle);
+
+	eError = AdoptChild(psBase, psPHandleData, psCHandleData);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "parent handle failed to adopt subhandle",
+	                  ExitFreeHandle);
+
+	*phHandle = hHandle;
+
+	return PVRSRV_OK;
+
+ExitFreeHandle:
+	PVRSRVDestroyHandleUnlocked(psBase, hHandle, eType);
+Exit:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFindHandle
+ @Description   Find handle corresponding to a resource pointer
+ @Input         pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Output        phHandle - points to returned handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE *phHandle,
+			      void *pvData,
+			      PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVFindHandleUnlocked(psBase, phHandle, pvData, eType);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFindHandleUnlocked
+ @Description   Find handle corresponding to a resource pointer without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Output        phHandle - points to the returned handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFindHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE *phHandle,
+			      void *pvData,
+			      PVRSRV_HANDLE_TYPE eType)
+{
+	IMG_HANDLE hHandle;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	/* See if there is a handle for this data pointer */
+	hHandle = FindHandle(psBase, pvData, eType, NULL);
+	if (hHandle == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error finding handle. Type %u",
+			 __func__,
+			 eType));
+
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	*phHandle = hHandle;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupHandle
+ @Description   Lookup the data pointer corresponding to a handle
+ @Input         hHandle - handle from client
+                eType - handle type
+                bRef - If TRUE, a reference will be added on the handle if the
+                       lookup is successful.
+ @Output        ppvData - points to the return data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase,
+				void **ppvData,
+				IMG_HANDLE hHandle,
+				PVRSRV_HANDLE_TYPE eType,
+				IMG_BOOL bRef)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVLookupHandleUnlocked(psBase, ppvData, hHandle, eType, bRef);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupHandleUnlocked
+ @Description   Lookup the data pointer corresponding to a handle without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         hHandle - handle from client
+                eType - handle type
+                bRef - If TRUE, a reference will be added on the handle if the
+                       lookup is successful.
+ @Output        ppvData - points to the returned data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+				void **ppvData,
+				IMG_HANDLE hHandle,
+				PVRSRV_HANDLE_TYPE eType,
+				IMG_BOOL bRef)
+{
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error looking up handle (%s) for base %p of type %s. Handle %p, type %s",
+			 __func__,
+			 PVRSRVGetErrorString(eError),
+			 psBase,
+			 HandleBaseTypeToString(psBase->eType),
+			 (void*) hHandle,
+			 HandleTypeToString(eType)));
+#if defined(DEBUG) || defined(PVRSRV_NEED_PVR_DPF)
+		OSDumpStack();
+#endif
+		return eError;
+	}
+
+	/* If bCanLookup is false it means that a destroy operation was already
+	 * called on this handle; therefore it can no longer be looked up. */
+	if (!psHandleData->bCanLookup)
+	{
+		return PVRSRV_ERROR_HANDLE_NOT_ALLOCATED;
+	}
+
+	if (bRef)
+	{
+		HandleGet(psHandleData);
+	}
+
+	*ppvData = psHandleData->pvData;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupSubHandle
+ @Description   Lookup the data pointer corresponding to a subhandle
+ @Input         hHandle - handle from client
+                eType - handle type
+                hAncestor - ancestor handle
+ @Output        ppvData - points to the returned data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase,
+				   void **ppvData,
+				   IMG_HANDLE hHandle,
+				   PVRSRV_HANDLE_TYPE eType,
+				   IMG_HANDLE hAncestor)
+{
+	HANDLE_DATA *psPHandleData = NULL;
+	HANDLE_DATA *psCHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	LockHandle(psBase);
+
+	eError = GetHandleData(psBase, &psCHandleData, hHandle, eType);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error looking up subhandle (%s). Handle %p, type %u",
+			 __func__,
+			 PVRSRVGetErrorString(eError),
+			 (void*) hHandle,
+			 eType));
+		OSDumpStack();
+		goto ExitUnlock;
+	}
+
+	/* Look for hAncestor among the handle's ancestors */
+	for (psPHandleData = psCHandleData; ParentHandle(psPHandleData) != hAncestor; )
+	{
+		eError = GetHandleData(psBase, &psPHandleData, ParentHandle(psPHandleData), PVRSRV_HANDLE_TYPE_NONE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "GetHandleData");
+			eError = PVRSRV_ERROR_INVALID_SUBHANDLE;
+			goto ExitUnlock;
+		}
+	}
+
+	*ppvData = psCHandleData->pvData;
+
+	eError = PVRSRV_OK;
+
+ExitUnlock:
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseHandle
+ @Description   Release a handle that is no longer needed
+ @Input         hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+void PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase,
+                         IMG_HANDLE hHandle,
+                         PVRSRV_HANDLE_TYPE eType)
+{
+	LockHandle(psBase);
+	PVRSRVReleaseHandleUnlocked(psBase, hHandle, eType);
+	UnlockHandle(psBase);
+}
+
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseHandleUnlocked
+ @Description   Release a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         hHandle - handle from client
+                eType - handle type
+******************************************************************************/
+void PVRSRVReleaseHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                 IMG_HANDLE hHandle,
+                                 PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(psBase != NULL);
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psBase != NULL, "invalid psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error (%s) looking up handle %p of type %s "
+		        "for base %p of type %s.", __func__, PVRSRVGetErrorString(eError),
+		        (void*) hHandle, HandleTypeToString(eType), psBase,
+		        HandleBaseTypeToString(psBase->eType)));
+
+		return;
+	}
+
+	PVR_ASSERT(psHandleData->bCanLookup);
+	PVR_ASSERT(psHandleData->iLookupCount > 0);
+
+	/* If there are still outstanding lookups for this handle or the handle
+	 * has not been destroyed yet, return early */
+	HandlePut(psHandleData);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVPurgeHandles
+ @Description   Purge handles for a given handle base
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPurgeHandles(PVRSRV_HANDLE_BASE *psBase)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	LockHandle(psBase);
+	eError = gpsHandleFuncs->pfnPurgeHandles(psBase->psImplBase);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFreeWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                                          IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleUnrefAndMaybeMarkForFree(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFree(PVRSRV_HANDLE_BASE *psBase,
+                                                   HANDLE_DATA *psHandleData,
+                                                   IMG_HANDLE hHandle,
+                                                   PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	/* If bCanLookup is false it means that the destructor was called more than
+	 * once on this handle. */
+	if (!psHandleData->bCanLookup)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle %p of type %s already freed.",
+		        __func__, psHandleData->hHandle,
+		        HandleTypeToString(psHandleData->eType)));
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	if (psHandleData->iLookupCount > 0)
+	{
+		return PVRSRV_ERROR_OBJECT_STILL_REFERENCED;
+	}
+
+	/* Mark this handle as freed only if it's no longer referenced by any
+	 * lookup. The user space should retry freeing this handle once there are
+	 * no outstanding lookups. */
+	psHandleData->bCanLookup = IMG_FALSE;
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = false, iLookupCount = %d", __func__,
+	        psHandleData->iLookupCount));
+#endif /* DEBUG_REFCNT */
+
+	/* Prepare children for destruction */
+	eError = IterateOverChildren(psBase, psHandleData,
+	                             HandleUnrefAndMaybeMarkForFreeWrapper);
+	if (eError != PVRSRV_OK)
+	{
+		/* If there was a problem destroying any of the child handles,
+		 * the parent handle still needs to be able to be looked-up */
+		psHandleData->bCanLookup = IMG_TRUE;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "HandleUnrefAndMaybeMarkForFreeWrapper");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HandleFreePrivDataWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                              IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleFreePrivData(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleFreePrivData(PVRSRV_HANDLE_BASE *psBase,
+                                       HANDLE_DATA *psHandleData,
+                                       IMG_HANDLE hHandle,
+                                       PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	/* Call the release data callback for each reference on the handle */
+	if (psHandleData->pfnReleaseData != NULL)
+	{
+		eError = psHandleData->pfnReleaseData(psHandleData->pvData);
+		if (eError != PVRSRV_OK)
+		{
+			if (PVRSRVIsRetryError(eError))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Got retry while calling release "
+						"data callback for handle %p of type = %s", __func__,
+						hHandle, HandleTypeToString(psHandleData->eType)));
+			}
+			else
+			{
+				PVR_LOG_ERROR(eError, "pfnReleaseData");
+			}
+
+			return eError;
+		}
+
+		/* we don't need this so make sure it's not called on
+		 * the pvData for the second time
+		 */
+		psHandleData->pfnReleaseData = NULL;
+	}
+
+	/* Free children's data */
+	eError = IterateOverChildren(psBase, psHandleData,
+	                             HandleFreePrivDataWrapper);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IterateOverChildren->HandleFreePrivData");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HandleFreeDestroyWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                             IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleFreeDestroy(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleFreeDestroy(PVRSRV_HANDLE_BASE *psBase,
+                                      HANDLE_DATA *psHandleData,
+                                      IMG_HANDLE hHandle,
+                                      PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psReleasedHandleData;
+	PVRSRV_ERROR eError;
+
+	eError = UnlinkFromParent(psBase, psHandleData);
+	PVR_LOG_RETURN_IF_ERROR(eError, "UnlinkFromParent");
+
+	if (!TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+		IMG_HANDLE hRemovedHandle;
+
+		InitKey(aKey, psBase, psHandleData->pvData, psHandleData->eType,
+		        ParentIfPrivate(psHandleData));
+
+		hRemovedHandle = (IMG_HANDLE) HASH_Remove_Extended(psBase->psHashTab,
+		                                                   aKey);
+
+		PVR_ASSERT(hRemovedHandle != NULL);
+		PVR_ASSERT(hRemovedHandle == psHandleData->hHandle);
+		PVR_UNREFERENCED_PARAMETER(hRemovedHandle);
+	}
+
+	/* Free children */
+	eError = IterateOverChildren(psBase, psHandleData, HandleFreeDestroyWrapper);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IterateOverChildren->HandleFreeDestroy");
+
+	eError = gpsHandleFuncs->pfnReleaseHandle(psBase->psImplBase,
+	                                          psHandleData->hHandle,
+	                                          (void **)&psReleasedHandleData);
+	OSFreeMem(psHandleData);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnReleaseHandle");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DestroyHandle(PVRSRV_HANDLE_BASE *psBase,
+                                  IMG_HANDLE hHandle,
+                                  PVRSRV_HANDLE_TYPE eType,
+                                  IMG_BOOL bReleaseLock)
+{
+	PVRSRV_ERROR eError;
+	HANDLE_DATA *psHandleData = NULL;
+
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	PVR_RETURN_IF_ERROR(eError);
+
+	eError = HandleUnrefAndMaybeMarkForFree(psBase, psHandleData, hHandle, eType);
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (bReleaseLock)
+	{
+		UnlockHandle(psBase);
+	}
+
+	eError = HandleFreePrivData(psBase, psHandleData, hHandle, eType);
+	if (eError != PVRSRV_OK)
+	{
+		if (bReleaseLock)
+		{
+			LockHandle(psBase);
+		}
+
+		/* If the data could not be freed due to a temporary condition the
+		 * handle must be kept alive so that the next destroy call can try again */
+		if (PVRSRVIsRetryError(eError))
+		{
+			psHandleData->bCanLookup = IMG_TRUE;
+		}
+
+		return eError;
+	}
+
+	if (bReleaseLock)
+	{
+		LockHandle(psBase);
+	}
+
+	return HandleFreeDestroy(psBase, psHandleData, hHandle, eType);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandle
+ @Description   Destroys a handle that is no longer needed. Will
+                acquiring the handle lock for duration of the call.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandle(PVRSRV_HANDLE_BASE *psBase,
+                                 IMG_HANDLE hHandle,
+                                 PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = DestroyHandle(psBase, hHandle, eType, IMG_FALSE);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandleUnlocked
+ @Description   Destroys a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                         IMG_HANDLE hHandle,
+                                         PVRSRV_HANDLE_TYPE eType)
+{
+	return DestroyHandle(psBase, hHandle, eType, IMG_FALSE);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandleStagedUnlocked
+ @Description   Destroys a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called. This function, unlike
+                PVRSRVDestroyHandleUnlocked(), releases the handle lock while
+                destroying handle private data. This is done to open the
+                bridge for other bridge calls.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandleStagedUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                               IMG_HANDLE hHandle,
+                                               PVRSRV_HANDLE_TYPE eType)
+{
+	return DestroyHandle(psBase, hHandle, eType, IMG_TRUE);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandleBase
+ @Description   Allocate a handle base structure for a process
+ @Input         eType - handle type
+ @Output        ppsBase - points to handle base structure pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandleBase(PVRSRV_HANDLE_BASE **ppsBase,
+                                   PVRSRV_HANDLE_BASE_TYPE eType)
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(gpsHandleFuncs != NULL, "handle management not initialised",
+	                  PVRSRV_ERROR_NOT_READY);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsBase != NULL, "ppsBase");
+
+	psBase = OSAllocZMem(sizeof(*psBase));
+	PVR_LOG_RETURN_IF_NOMEM(psBase, "psBase");
+
+	eError = OSLockCreate(&psBase->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeHandleBase);
+
+	psBase->eType = eType;
+
+	LockHandle(psBase);
+
+	eError = gpsHandleFuncs->pfnCreateHandleBase(&psBase->psImplBase);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnlock);
+
+	psBase->psHashTab = HASH_Create_Extended(HANDLE_HASH_TAB_INIT_SIZE,
+						 sizeof(HAND_KEY),
+						 HASH_Func_Default,
+						 HASH_Key_Comp_Default);
+	PVR_LOG_GOTO_IF_FALSE(psBase->psHashTab != NULL, "couldn't create data pointer"
+	                  " hash table", ErrorDestroyHandleBase);
+
+	*ppsBase = psBase;
+
+	UnlockHandle(psBase);
+
+	return PVRSRV_OK;
+
+ErrorDestroyHandleBase:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+	(void)gpsHandleFuncs->pfnDestroyHandleBase(psBase->psImplBase);
+
+ErrorUnlock:
+	UnlockHandle(psBase);
+	OSLockDestroy(psBase->hLock);
+
+ErrorFreeHandleBase:
+	OSFreeMem(psBase);
+
+	return eError;
+}
+
+#if defined(DEBUG)
+typedef struct _COUNT_HANDLE_DATA_
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	IMG_UINT32 uiHandleDataCount;
+} COUNT_HANDLE_DATA;
+
+/* Used to count the number of handles that have data associated with them */
+static PVRSRV_ERROR CountHandleDataWrapper(IMG_HANDLE hHandle, void *pvData)
+{
+	COUNT_HANDLE_DATA *psData = (COUNT_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData != NULL, "psData");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->psBase != NULL, "psData->psBase");
+
+	eError = GetHandleData(psData->psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData != NULL)
+	{
+		psData->uiHandleDataCount++;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Print a handle in the handle base. Used with the iterator callback. */
+static PVRSRV_ERROR ListHandlesInBase(IMG_HANDLE hHandle, void *pvData)
+{
+	PVRSRV_HANDLE_BASE *psBase = (PVRSRV_HANDLE_BASE*) pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData != NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "    Handle: %6u, CanLookup: %u, LookupCount: %3u, Type: %s (%u), pvData<%p>",
+		       (IMG_UINT32) (uintptr_t) psHandleData->hHandle, psHandleData->bCanLookup,
+		       psHandleData->iLookupCount, HandleTypeToString(psHandleData->eType),
+		       psHandleData->eType, psHandleData->pvData));
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(DEBUG) */
+
+static INLINE IMG_BOOL _CheckIfMaxTimeExpired(IMG_UINT64 ui64TimeStart, IMG_UINT64 ui64MaxBridgeTime)
+{
+	/* unsigned arithmetic is well defined so this will wrap around correctly */
+	return (IMG_BOOL)((OSClockns64() - ui64TimeStart) >= ui64MaxBridgeTime);
+}
+
+static PVRSRV_ERROR FreeKernelHandlesWrapperIterKernel(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_KERNEL_HANDLE_DATA *psData = (FREE_KERNEL_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psKernelHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	/* Get kernel handle data. */
+	eError = GetHandleData(KERNEL_HANDLE_BASE,
+			    &psKernelHandleData,
+			    hHandle,
+			    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psKernelHandleData->pvData == psData->psProcessHandleData->pvData)
+	{
+		/* This kernel handle belongs to our process handle. */
+		psData->hKernelHandle = hHandle;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FreeKernelHandlesWrapperIterProcess(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_KERNEL_HANDLE_DATA *psData = (FREE_KERNEL_HANDLE_DATA *)pvData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	/* Get process handle data. */
+	eError = GetHandleData(psData->psBase,
+			    &psData->psProcessHandleData,
+			    hHandle,
+			    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psData->psProcessHandleData->eFlag == PVRSRV_HANDLE_ALLOC_FLAG_MULTI
+#if defined(SUPPORT_INSECURE_EXPORT)
+		|| psData->psProcessHandleData->eType == PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT
+#endif
+		)
+	{
+		/* Only multi alloc process handles might be in kernel handle base. */
+		psData->hKernelHandle = NULL;
+		/* Iterate over kernel handles. */
+		eError = gpsHandleFuncs->pfnIterateOverHandles(KERNEL_HANDLE_BASE->psImplBase,
+									&FreeKernelHandlesWrapperIterKernel,
+									(void *)psData);
+		PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "failed to iterate over kernel handles",
+		                  eError);
+
+		if (psData->hKernelHandle)
+		{
+			/* Release kernel handle which belongs to our process handle. */
+			eError = gpsHandleFuncs->pfnReleaseHandle(KERNEL_HANDLE_BASE->psImplBase,
+						psData->hKernelHandle,
+						NULL);
+			PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "couldn't release kernel handle",
+			                  eError);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FreeHandleDataWrapper(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_HANDLE_DATA *psData = (FREE_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData != NULL, "psData");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->psBase != NULL, "psData->psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->eHandleFreeType != PVRSRV_HANDLE_TYPE_NONE,
+	                          "psData->eHandleFreeType");
+
+	eError = GetHandleData(psData->psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData == NULL || psHandleData->eType != psData->eHandleFreeType)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(psHandleData->bCanLookup && psHandleData->iLookupCount == 0);
+
+	if (psHandleData->bCanLookup)
+	{
+		if (psHandleData->pfnReleaseData != NULL)
+		{
+			eError = psHandleData->pfnReleaseData(psHandleData->pvData);
+			if (PVRSRVIsRetryError(eError))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Got retry while calling release "
+				        "data callback for handle %p of type = %s", __func__,
+				        hHandle, HandleTypeToString(psHandleData->eType)));
+
+				return eError;
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected error from pfnReleaseData (%s)",
+						 __func__,
+						 PVRSRVGetErrorString(eError)));
+			}
+		}
+
+		psHandleData->bCanLookup = IMG_FALSE;
+	}
+
+	if (!TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+		IMG_HANDLE hRemovedHandle;
+
+		InitKey(aKey,
+			psData->psBase,
+			psHandleData->pvData,
+			psHandleData->eType,
+			ParentIfPrivate(psHandleData));
+
+		hRemovedHandle = (IMG_HANDLE)HASH_Remove_Extended(psData->psBase->psHashTab, aKey);
+
+		PVR_ASSERT(hRemovedHandle != NULL);
+		PVR_ASSERT(hRemovedHandle == psHandleData->hHandle);
+		PVR_UNREFERENCED_PARAMETER(hRemovedHandle);
+	}
+
+	eError = gpsHandleFuncs->pfnSetHandleData(psData->psBase->psImplBase, hHandle, NULL);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSFreeMem(psHandleData);
+
+	/* If we reach the end of the time slice release we can release the global
+	 * lock, invoke the scheduler and reacquire the lock */
+	if ((psData->ui64MaxBridgeTime != 0) && _CheckIfMaxTimeExpired(psData->ui64TimeStart, psData->ui64MaxBridgeTime))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+			 "%s: Lock timeout (timeout: %" IMG_UINT64_FMTSPEC")",
+			 __func__,
+			 psData->ui64MaxBridgeTime));
+		UnlockHandle(psData->psBase);
+		/* Invoke the scheduler to check if other processes are waiting for the lock */
+		OSReleaseThreadQuanta();
+		LockHandle(psData->psBase);
+		/* Set again lock timeout and reset the counter */
+		psData->ui64TimeStart = OSClockns64();
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Lock acquired again", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/* The Ordered Array of PVRSRV_HANDLE_TYPE Enum Entries.
+ *
+ *   Some handles must be destroyed prior to other handles,
+ *   such relationships are established with respect to handle types.
+ *   Therefore elements of this array have to maintain specific order,
+ *   e.g. the PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET must be placed
+ *   before PVRSRV_HANDLE_TYPE_RGX_FREELIST.
+ *
+ *   If ordering is incorrect driver may fail on the ground of cleanup
+ *   routines. Unfortunately, we can mainly rely on the actual definition of
+ *   the array, there is no explicit information about all relationships
+ *   between handle types. These relationships do not necessarily come from
+ *   bridge-specified handle attributes such as 'sub handle' and 'parent
+ *   handle'. They may come from internal/private ref-counters contained by
+ *   objects referenced by our kernel handles.
+ *
+ *   For example, at the bridge level, PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET
+ *   and PVRSRV_HANDLE_TYPE_RGX_FREELIST have no explicit relationship, meaning
+ *   none of them is a sub-handle for the other.
+ *   However the freelist contains internal ref-count that is decremented by
+ *   the destroy routine for KM_HW_RT_DATASET.
+ *
+ *   BE CAREFUL when adding/deleting/moving handle types.
+ */
+static const PVRSRV_HANDLE_TYPE g_aeOrderedFreeList[] =
+{
+	PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+	PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+	PVRSRV_HANDLE_TYPE_RGX_FW_MEMDESC,
+	PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+	PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+	PVRSRV_HANDLE_TYPE_RGX_MEMORY_BLOCK,
+	PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+	PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RI_HANDLE,
+	PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+	PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+	PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+	PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+	PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_MAPPING,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+	PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+	PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_PAGELIST,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_SECURE_EXPORT,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+	PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+	PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+	PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE,
+	PVRSRV_HANDLE_TYPE_DC_BUFFER,
+	PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT,
+	PVRSRV_HANDLE_TYPE_DC_DEVICE,
+	PVRSRV_HANDLE_TYPE_PVR_HWPERF_SD,
+	PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+	PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+	PVRSRV_HANDLE_TYPE_MM_PLAT_CLEANUP
+};
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFreeKernelHandles
+ @Description   Free kernel handles which belongs to process handles
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFreeKernelHandles(PVRSRV_HANDLE_BASE *psBase)
+{
+	FREE_KERNEL_HANDLE_DATA sHandleData = {NULL};
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	LockHandle(psBase);
+
+	sHandleData.psBase = psBase;
+	/* Iterate over process handles. */
+	eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+								&FreeKernelHandlesWrapperIterProcess,
+								(void *)&sHandleData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnIterateOverHandles", ExitUnlock);
+
+	eError = PVRSRV_OK;
+
+ExitUnlock:
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/* Only called from sync_fallback_server.c */
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+/*!
+*******************************************************************************
+ @Function      PVRSRVRetrieveProcessHandleBase
+ @Description   Returns a pointer to the process handle base for the current
+                process. If the current process is the cleanup thread, then the
+                process handle base for the process currently being cleaned up
+                is returned
+ @Return        Pointer to the process handle base, or NULL if not found.
+******************************************************************************/
+PVRSRV_HANDLE_BASE *PVRSRVRetrieveProcessHandleBase(void)
+{
+	PVRSRV_HANDLE_BASE *psHandleBase = NULL;
+	PROCESS_HANDLE_BASE *psProcHandleBase = NULL;
+	IMG_PID ui32PurgePid = PVRSRVGetPurgeConnectionPid();
+	IMG_PID uiCleanupPid = PVRSRVCleanupThreadGetPid();
+	uintptr_t uiCleanupTid = PVRSRVCleanupThreadGetTid();
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	/* Check to see if we're being called from the cleanup thread... */
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+	    (OSGetCurrentThreadID() == uiCleanupTid) &&
+	    (ui32PurgePid > 0))
+	{
+		/* Check to see if the cleanup thread has already removed the
+		 * process handle base from the HASH table.
+		 */
+		psHandleBase = g_psProcessHandleBaseBeingFreed;
+		/* psHandleBase shouldn't be null, as cleanup thread
+		 * should be removing this from the HASH table before
+		 * we get here, so assert if not.
+		 */
+		PVR_ASSERT(psHandleBase);
+	}
+	else
+	{
+		/* Not being called from the cleanup thread, so return the process
+		 * handle base for the current process.
+		 */
+		uintptr_t uiHashKey;
+
+		uiHashKey = OSAcquireCurrentPPIDResourceRefKM();
+		OSReleasePPIDResourceRefKM(uiHashKey);
+
+		psProcHandleBase = (PROCESS_HANDLE_BASE *)
+		    HASH_Retrieve(g_psProcessHandleBaseTable, uiHashKey);
+	}
+
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	if (psHandleBase == NULL && psProcHandleBase != NULL)
+	{
+		psHandleBase = psProcHandleBase->psHandleBase;
+	}
+	return psHandleBase;
+}
+#endif
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAcquireProcessHandleBase
+ @Description   Increments reference count on the process handle base for the
+                current process and returns pointer to the base. If the handle
+                base does not exist it will be allocated.
+ @Output        ppsBase - pointer to a handle base for the current process
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAcquireProcessHandleBase(PROCESS_HANDLE_BASE **ppsBase)
+{
+	PROCESS_HANDLE_BASE *psBase;
+	PVRSRV_ERROR eError;
+	uintptr_t uiHashKey;
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	/* Acquire the process resource hash key (and take ref) */
+	uiHashKey = OSAcquireCurrentPPIDResourceRefKM();
+	PVR_ASSERT(uiHashKey != 0);
+
+	psBase = (PROCESS_HANDLE_BASE*) HASH_Retrieve(g_psProcessHandleBaseTable, uiHashKey);
+
+	/* In case there is none we are going to allocate one */
+	if (psBase == NULL)
+	{
+		IMG_BOOL bSuccess;
+
+		psBase = OSAllocZMem(sizeof(*psBase));
+		PVR_LOG_GOTO_IF_NOMEM(psBase, eError, ErrorUnlock);
+
+		/* Allocate handle base for this process */
+		eError = PVRSRVAllocHandleBase(&psBase->psHandleBase, PVRSRV_HANDLE_BASE_TYPE_PROCESS);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", ErrorFreeProcessHandleBase);
+
+		/* Insert the handle base into the global hash table */
+		psBase->uiHashKey = uiHashKey;
+		bSuccess = HASH_Insert(g_psProcessHandleBaseTable, uiHashKey, (uintptr_t)psBase);
+		PVR_LOG_GOTO_IF_FALSE(bSuccess, "HASH_Insert failed", ErrorFreeHandleBase);
+	}
+
+	OSAtomicIncrement(&psBase->iRefCount);
+
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	*ppsBase = psBase;
+
+	return PVRSRV_OK;
+
+ErrorFreeHandleBase:
+	eError = PVRSRV_ERROR_INVALID_PARAMS;
+	PVRSRVFreeHandleBase(psBase->psHandleBase, 0);
+ErrorFreeProcessHandleBase:
+	OSFreeMem(psBase);
+ErrorUnlock:
+	OSReleasePPIDResourceRefKM(uiHashKey);
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseProcessHandleBase
+ @Description   Decrements reference count on a process handle base psBase
+                for the current process. If the reference count reaches 0 the
+                process handle base will be freed.
+ @Input         psBase - pointer to a process handle base
+ @Inout         ui64MaxBridgeTime - maximum time a handle destroy operation
+                                    can hold the handle base lock (after that
+                                    time a lock will be release and reacquired
+                                    for another time slice)
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVReleaseProcessHandleBase(PROCESS_HANDLE_BASE *psBase,
+                                            IMG_UINT64 ui64MaxBridgeTime)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iRefCount;
+	uintptr_t uiHashValue = 0;
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	iRefCount = OSAtomicDecrement(&psBase->iRefCount);
+
+	if (iRefCount != 0)
+	{
+		/* Release the process resource hash key (drop ref) */
+		OSReleasePPIDResourceRefKM(psBase->uiHashKey);
+		OSLockRelease(g_hProcessHandleBaseLock);
+		return PVRSRV_OK;
+	}
+
+	/* in case the refcount becomes 0 we can remove the process handle base
+	 * and all related objects */
+
+	uiHashValue = HASH_Remove(g_psProcessHandleBaseTable, psBase->uiHashKey);
+	/* Release the process resource hash key (drop ref) */
+	OSReleasePPIDResourceRefKM(psBase->uiHashKey);
+	psBase->uiHashKey = 0;
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	PVR_LOG_RETURN_IF_FALSE(uiHashValue != 0, "HASH_Remove failed",
+	                        PVRSRV_ERROR_UNABLE_TO_REMOVE_HASH_VALUE);
+
+	eError = PVRSRVFreeKernelHandles(psBase->psHandleBase);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeKernelHandles");
+
+	eError = PVRSRVFreeHandleBase(psBase->psHandleBase, ui64MaxBridgeTime);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeHandleBase");
+
+	OSFreeMem(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFreeHandleBase
+ @Description   Free a handle base structure
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFreeHandleBase(PVRSRV_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime)
+{
+#if defined(DEBUG)
+	COUNT_HANDLE_DATA sCountData = {NULL};
+#endif
+	FREE_HANDLE_DATA sHandleData = {NULL};
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_PID uiCleanupPid = PVRSRVCleanupThreadGetPid();
+	uintptr_t uiCleanupTid = PVRSRVCleanupThreadGetTid();
+	IMG_UINT32 ui32ErrorCount = 0;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	LockHandle(psBase);
+
+	/* If this is a process handle base being freed by the cleanup
+	 * thread, store this in g_psProcessHandleBaseBeingFreed
+	 */
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+	    (OSGetCurrentThreadID() == uiCleanupTid) &&
+	    (psBase->eType == PVRSRV_HANDLE_BASE_TYPE_PROCESS))
+	{
+		g_psProcessHandleBaseBeingFreed = psBase;
+	}
+
+	sHandleData.psBase = psBase;
+	sHandleData.ui64TimeStart = OSClockns64();
+	sHandleData.ui64MaxBridgeTime = ui64MaxBridgeTime;
+
+
+#if defined(DEBUG)
+
+	sCountData.psBase = psBase;
+
+	eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+						       &CountHandleDataWrapper,
+						       (void *)&sCountData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnIterateOverHandles", ExitUnlock);
+
+	if (sCountData.uiHandleDataCount != 0)
+	{
+		IMG_BOOL bList = (IMG_BOOL)(sCountData.uiHandleDataCount < HANDLE_DEBUG_LISTING_MAX_NUM);
+
+		PVR_DPF((PVR_DBG_WARNING,
+			 "%s: %u remaining handles in handle base 0x%p "
+			 "(PVRSRV_HANDLE_BASE_TYPE %u).%s",
+			 __func__,
+			 sCountData.uiHandleDataCount,
+			 psBase,
+			 psBase->eType,
+			 bList ? "": " Skipping details, too many items..."));
+
+		if (bList)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "-------- Listing Handles --------"));
+			(void) gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+			                                             &ListHandlesInBase,
+			                                             psBase);
+			PVR_DPF((PVR_DBG_WARNING, "-------- Done Listing    --------"));
+		}
+	}
+
+#endif /* defined(DEBUG) */
+
+	/*
+	 * As we're freeing handles based on type, make sure all
+	 * handles have actually had their data freed to avoid
+	 * resources being leaked
+	 */
+	for (i = 0; i < ARRAY_SIZE(g_aeOrderedFreeList); i++)
+	{
+		sHandleData.eHandleFreeType = g_aeOrderedFreeList[i];
+
+		/* Make sure all handles have been freed before destroying the handle base */
+		eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+							       &FreeHandleDataWrapper,
+							       (void *)&sHandleData);
+
+		/* On retry error return without destroying handle base. Caller may retry. */
+		if (PVRSRVIsRetryError(eError))
+		{
+			PVR_GOTO_IF_ERROR(eError, ExitUnlock);
+		}
+
+		/* Retry is not possible. Continue freeing remaining handles. */
+		if (eError != PVRSRV_OK)
+		{
+			ui32ErrorCount++;
+		}
+	}
+
+	if (ui32ErrorCount > 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error freeing %d handles.", __func__, ui32ErrorCount));
+	}
+
+	if (psBase->psHashTab != NULL)
+	{
+		HASH_Delete(psBase->psHashTab);
+	}
+
+	eError = gpsHandleFuncs->pfnDestroyHandleBase(psBase->psImplBase);
+	PVR_GOTO_IF_ERROR(eError, ExitUnlock);
+
+	UnlockHandle(psBase);
+	OSLockDestroy(psBase->hLock);
+	OSFreeMem(psBase);
+
+	return eError;
+
+ExitUnlock:
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+		(OSGetCurrentThreadID() == uiCleanupTid))
+	{
+		g_psProcessHandleBaseBeingFreed = NULL;
+	}
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVHandleInit
+ @Description   Initialise handle management
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVHandleInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsKernelHandleBase == NULL);
+	PVR_ASSERT(gpsHandleFuncs == NULL);
+	PVR_ASSERT(g_hProcessHandleBaseLock == NULL);
+	PVR_ASSERT(g_psProcessHandleBaseTable == NULL);
+	PVR_ASSERT(!gbLockInitialised);
+
+	eError = OSLockCreate(&gKernelHandleLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate:1");
+
+	eError = OSLockCreate(&g_hProcessHandleBaseLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate:2", ErrorHandleDeinit);
+
+	gbLockInitialised = IMG_TRUE;
+
+	eError = PVRSRVHandleGetFuncTable(&gpsHandleFuncs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVHandleGetFuncTable", ErrorHandleDeinit);
+
+	eError = PVRSRVAllocHandleBase(&gpsKernelHandleBase,
+	                               PVRSRV_HANDLE_BASE_TYPE_GLOBAL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", ErrorHandleDeinit);
+
+	g_psProcessHandleBaseTable = HASH_Create(HANDLE_PROC_HANDLE_HASH_INIT_SIZE);
+	PVR_LOG_GOTO_IF_NOMEM(g_psProcessHandleBaseTable, eError, ErrorHandleDeinit);
+
+	eError = gpsHandleFuncs->pfnEnableHandlePurging(gpsKernelHandleBase->psImplBase);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnEnableHandlePurging", ErrorHandleDeinit);
+
+	return PVRSRV_OK;
+
+ErrorHandleDeinit:
+	(void) PVRSRVHandleDeInit();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVHandleDeInit
+ @Description   De-initialise handle management
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVHandleDeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (gpsHandleFuncs != NULL)
+	{
+		if (gpsKernelHandleBase != NULL)
+		{
+			eError = PVRSRVFreeHandleBase(gpsKernelHandleBase, 0 /* do not release bridge lock */);
+			if (eError == PVRSRV_OK)
+			{
+				gpsKernelHandleBase = NULL;
+			}
+			else
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVFreeHandleBase");
+			}
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+			gpsHandleFuncs = NULL;
+		}
+	}
+	else
+	{
+		/* If we don't have a handle function table we shouldn't have a handle base either */
+		PVR_ASSERT(gpsKernelHandleBase == NULL);
+	}
+
+	if (g_psProcessHandleBaseTable != NULL)
+	{
+		HASH_Delete(g_psProcessHandleBaseTable);
+		g_psProcessHandleBaseTable = NULL;
+	}
+
+	if (g_hProcessHandleBaseLock != NULL)
+	{
+		OSLockDestroy(g_hProcessHandleBaseLock);
+		g_hProcessHandleBaseLock = NULL;
+	}
+
+	if (gKernelHandleLock != NULL)
+	{
+		OSLockDestroy(gKernelHandleLock);
+		gbLockInitialised = IMG_FALSE;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (handle.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.c
new file mode 100644
index 000000000000..b16592ae5ebb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.c
@@ -0,0 +1,1223 @@
+/*************************************************************************/ /*!
+@File           htb_debug.c
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides kernel side debugFS Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxdevice.h"
+#include "htbserver.h"
+#include "htbuffer_types.h"
+#include "tlstream.h"
+#include "tlclient.h"
+#include "pvrsrv_tlcommon.h"
+#include "di_server.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "htb_debug.h"
+
+// Global data handles for buffer manipulation and processing
+
+typedef struct {
+	IMG_PBYTE	pBuf;		/* Raw data buffer from TL stream */
+	IMG_UINT32	uiBufLen;	/* Amount of data to process from 'pBuf' */
+	IMG_UINT32	uiTotal;	/* Total bytes processed */
+	IMG_UINT32	uiMsgLen;	/* Length of HTB message to be processed */
+	IMG_PBYTE	pCurr;		/* pointer to current message to be decoded */
+	IMG_CHAR	szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];	/* Output string */
+} HTB_Sentinel_t;
+
+typedef struct
+{
+	DI_ENTRY *psDumpHostDiEntry;  /* debug info entry */
+	HTB_Sentinel_t sSentinel;     /* private control structure for HTB DI
+	                                 operations */
+	IMG_HANDLE hStream;           /* stream handle for debugFS use */
+} HTB_DBG_INFO;
+
+static HTB_DBG_INFO g_sHTBData;
+
+// Comment out for extra debug level
+// #define HTB_CHATTY_PRINT(x) PVR_DPF(x)
+#define HTB_CHATTY_PRINT(x)
+
+typedef void (DI_PRINTF)(const OSDI_IMPL_ENTRY *, const IMG_CHAR *, ...);
+
+/******************************************************************************
+ * debugFS display routines
+ *****************************************************************************/
+static int HTBDumpBuffer(DI_PRINTF, OSDI_IMPL_ENTRY *, void *);
+
+static int _DebugHBTraceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	int	retVal;
+
+	PVR_ASSERT(psEntry != NULL);
+
+	/* psEntry should never be NULL */
+	if (psEntry == NULL)
+	{
+		return -1;
+	}
+
+	/* Ensure that we have a valid address to use to dump info from. If NULL we
+	 * return a failure code to terminate the DI read call. pvData is either
+	 * DI_START_TOKEN (for the initial call) or an HTB buffer address for
+	 * subsequent calls [returned from the NEXT function]. */
+	if (pvData == NULL)
+	{
+		return -1;
+	}
+
+	retVal = HTBDumpBuffer(DIPrintf, psEntry, pvData);
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %d", __func__, retVal));
+
+	return retVal;
+}
+
+static IMG_UINT32 idToLogIdx(IMG_UINT32);	/* Forward declaration */
+
+/*
+ * HTB_GetNextMessage
+ *
+ * Get next non-empty message block from the buffer held in pSentinel->pBuf
+ * If we exhaust the data buffer we refill it (after releasing the previous
+ * message(s) [only one non-NULL message, but PAD messages will get released
+ * as we traverse them].
+ *
+ * Input:
+ *	pSentinel		references the already acquired data buffer
+ *
+ * Output:
+ *	pSentinel
+ *		-> uiMsglen updated to the size of the non-NULL message
+ *
+ * Returns:
+ *	Address of first non-NULL message in the buffer (if any)
+ *	NULL if there is no further data available from the stream and the buffer
+ *	contents have been drained.
+ */
+static IMG_PBYTE HTB_GetNextMessage(HTB_Sentinel_t *pSentinel)
+{
+	void	*pNext, *pLast, *pStart, *pData = NULL;
+	void	*pCurrent;		/* Current processing point within buffer */
+	PVRSRVTL_PPACKETHDR	ppHdr;	/* Current packet header */
+	IMG_UINT32	uiHdrType;		/* Packet header type */
+	IMG_UINT32	uiMsgSize;		/* Message size of current packet (bytes) */
+	IMG_BOOL	bUnrecognizedErrorPrinted = IMG_FALSE;
+	IMG_UINT32	ui32Data;
+	IMG_UINT32	ui32LogIdx;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(pSentinel != NULL);
+
+	pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+
+	pStart = pSentinel->pBuf;
+
+	pNext = pStart;
+	pSentinel->uiMsgLen = 0;	// Reset count for this message
+	uiMsgSize = 0;				// nothing processed so far
+	ui32LogIdx = HTB_SF_LAST;	// Loop terminator condition
+
+	do
+	{
+		/*
+		 * If we've drained the buffer we must RELEASE and ACQUIRE some more.
+		 */
+		if (pNext >= pLast)
+		{
+			eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream);
+			if (PVRSRV_OK != eError)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: %s FAILED '%s'", __func__,
+					"TLClientReleaseData", PVRSRVGETERRORSTRING(eError)));
+				return NULL;
+			}
+
+			eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+				g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+			if (PVRSRV_OK != eError)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'", __func__,
+					"TLClientAcquireData", PVRSRVGETERRORSTRING(eError)));
+				return NULL;
+			}
+
+			// Reset our limits - if we've returned an empty buffer we're done.
+			pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+			pStart = pSentinel->pBuf;
+			pNext = pStart;
+
+			if (pStart == NULL || pLast == NULL)
+			{
+				return NULL;
+			}
+		}
+
+		/*
+		 * We should have a header followed by data block(s) in the stream.
+		 */
+
+		pCurrent = pNext;
+		ppHdr = GET_PACKET_HDR(pCurrent);
+
+		if (ppHdr == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unexpected NULL packet in Host Trace buffer",
+			         __func__));
+			pSentinel->uiMsgLen += uiMsgSize;
+			return NULL;		// This should never happen
+		}
+
+		/*
+		 * This should *NEVER* fire. If it does it means we have got some
+		 * dubious packet header back from the HTB stream. In this case
+		 * the sensible thing is to abort processing and return to
+		 * the caller
+		 */
+		uiHdrType = GET_PACKET_TYPE(ppHdr);
+
+		PVR_ASSERT(uiHdrType < PVRSRVTL_PACKETTYPE_LAST &&
+			uiHdrType > PVRSRVTL_PACKETTYPE_UNDEF);
+
+		if (uiHdrType < PVRSRVTL_PACKETTYPE_LAST &&
+			uiHdrType > PVRSRVTL_PACKETTYPE_UNDEF)
+		{
+			/*
+			 * We have a (potentially) valid data header. We should see if
+			 * the associated packet header matches one of our expected
+			 * types.
+			 */
+			pNext = GET_NEXT_PACKET_ADDR(ppHdr);
+
+			PVR_ASSERT(pNext != NULL);
+
+			uiMsgSize = (IMG_UINT32)((size_t)pNext - (size_t)ppHdr);
+
+			pSentinel->uiMsgLen += uiMsgSize;
+
+			pData = GET_PACKET_DATA_PTR(ppHdr);
+
+			/*
+			 * Handle non-DATA packet types. These include PAD fields which
+			 * may have data associated and other types. We simply discard
+			 * these as they have no decodable information within them.
+			 */
+			if (uiHdrType != PVRSRVTL_PACKETTYPE_DATA)
+			{
+				/*
+				 * Now release the current non-data packet and proceed to the
+				 * next entry (if any).
+				 */
+				eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+				    g_sHTBData.hStream, uiMsgSize);
+
+				HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Packet Type %x "
+				                 "Length %u", __func__, uiHdrType, uiMsgSize));
+
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - '%s' message"
+						" size %u", __func__, "TLClientReleaseDataLess",
+						PVRSRVGETERRORSTRING(eError), uiMsgSize));
+				}
+
+				eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+					g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+				if (PVRSRV_OK != eError)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - %s Giving up",
+						__func__, "TLClientAcquireData",
+						PVRSRVGETERRORSTRING(eError)));
+
+					return NULL;
+				}
+				pSentinel->uiMsgLen = 0;
+				// Reset our limits - if we've returned an empty buffer we're done.
+				pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+				pStart = pSentinel->pBuf;
+				pNext = pStart;
+
+				if (pStart == NULL || pLast == NULL)
+				{
+					return NULL;
+				}
+				continue;
+			}
+			if (pData == NULL || pData >= pLast)
+			{
+				continue;
+			}
+			ui32Data = *(IMG_UINT32 *)pData;
+			ui32LogIdx = idToLogIdx(ui32Data);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Unexpected Header @%p value %x",
+				ppHdr, uiHdrType));
+
+			return NULL;
+		}
+
+		/*
+		 * Check if the unrecognized ID is valid and therefore, tracebuf
+		 * needs updating.
+		 */
+		if (HTB_SF_LAST == ui32LogIdx && HTB_LOG_VALIDID(ui32Data)
+			&& IMG_FALSE == bUnrecognizedErrorPrinted)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			    "%s: Unrecognised LOG value '%x' GID %x Params %d ID %x @ '%p'",
+			    __func__, ui32Data, HTB_SF_GID(ui32Data),
+			    HTB_SF_PARAMNUM(ui32Data), ui32Data & 0xfff, pData));
+			bUnrecognizedErrorPrinted = IMG_TRUE;
+		}
+
+	} while (HTB_SF_LAST == ui32LogIdx);
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning data @ %p Log value '%x'",
+	                 __func__, pCurrent, ui32Data));
+
+	return pCurrent;
+}
+
+/*
+ * HTB_GetFirstMessage
+ *
+ * Called from START to obtain the buffer address of the first message within
+ * pSentinel->pBuf. Will ACQUIRE data if the buffer is empty.
+ *
+ * Input:
+ *	pSentinel
+ *	pui64Pos			Offset within the debugFS file
+ *
+ * Output:
+ *	pSentinel->pCurr	Set to reference the first valid non-NULL message within
+ *						the buffer. If no valid message is found set to NULL.
+ *	pSentinel
+ *		->pBuf		if unset on entry
+ *		->uiBufLen	if pBuf unset on entry
+ *
+ * Side-effects:
+ *	HTB TL stream will be updated to bypass any zero-length PAD messages before
+ *	the first non-NULL message (if any).
+ */
+static void HTB_GetFirstMessage(HTB_Sentinel_t *pSentinel, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_ERROR	eError;
+
+	PVR_UNREFERENCED_PARAMETER(pui64Pos);
+
+	if (pSentinel == NULL)
+		return;
+
+	if (pSentinel->pBuf == NULL)
+	{
+		/* Acquire data */
+		pSentinel->uiMsgLen = 0;
+
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		    g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'",
+			    __func__, "TLClientAcquireData", PVRSRVGETERRORSTRING(eError)));
+
+			pSentinel->pBuf = NULL;
+			pSentinel->pCurr = NULL;
+		}
+		else
+		{
+			/*
+			 * If there is no data available we set pSentinel->pCurr to NULL
+			 * and return. This is expected behaviour if we've drained the
+			 * data and nothing else has yet been produced.
+			 */
+			if (pSentinel->uiBufLen == 0 || pSentinel->pBuf == NULL)
+			{
+				HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Empty Buffer @ %p",
+				                 __func__, pSentinel->pBuf));
+
+				pSentinel->pCurr = NULL;
+				return;
+			}
+		}
+	}
+
+	/* Locate next message within buffer. NULL => no more data to process */
+	pSentinel->pCurr = HTB_GetNextMessage(pSentinel);
+}
+
+/*
+ * _DebugHBTraceDIStart:
+ *
+ * Returns the address to use for subsequent 'Show', 'Next', 'Stop' file ops.
+ * Return DI_START_TOKEN for the very first call and allocate a sentinel for
+ * use by the 'Show' routine and its helpers.
+ * This is stored in the psEntry's private hook field.
+ *
+ * We obtain access to the TLstream associated with the HTB. If this doesn't
+ * exist (because no pvrdebug capture trace has been set) we simply return with
+ * a NULL value which will stop the DI traversal.
+ */
+static void *_DebugHBTraceDIStart(OSDI_IMPL_ENTRY *psEntry,
+                                  IMG_UINT64 *pui64Pos)
+{
+	HTB_Sentinel_t	*pSentinel = DIGetPrivData(psEntry);
+	PVRSRV_ERROR	eError;
+	IMG_UINT32		uiTLMode;
+	void			*retVal;
+	IMG_HANDLE		hStream;
+
+	/* The sentinel object should have been allocated during the creation
+	 * of the DI entry. If it's not there it means that something went
+	 * wrong. Return NULL in such case. */
+	if (pSentinel == NULL)
+	{
+		return NULL;
+	}
+
+	/* Check to see if the HTB stream has been configured yet. If not, there is
+	 * nothing to display so we just return NULL to stop the stream access.
+	 */
+	if (!HTBIsConfigured())
+	{
+		return NULL;
+	}
+
+	/* Open the stream in non-blocking mode so that we can determine if there
+	 * is no data to consume. Also disable the producer callback (if any) and
+	 * the open callback so that we do not generate spurious trace data when
+	 * accessing the stream.
+	 */
+	uiTLMode = PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING|
+			   PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK|
+			   PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK;
+
+	/* If two or more processes try to read from this file at the same time
+	 * the TLClientOpenStream() function will handle this by allowing only
+	 * one of them to actually open the stream. The other process will get
+	 * an error stating that the stream is already open. The open function
+	 * is threads safe. */
+	eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE, HTB_STREAM_NAME, uiTLMode,
+	                            &hStream);
+
+	if (eError == PVRSRV_ERROR_ALREADY_OPEN)
+	{
+		/* Stream allows only one reader so return error if it's already
+		 * opened. */
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Stream handle %p already "
+		                 "exists for %s", __func__, g_sHTBData.hStream,
+		                 HTB_STREAM_NAME));
+		return NULL;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		/*
+		 * No stream available so nothing to report
+		 */
+		return NULL;
+	}
+
+	/* There is a window where hStream can be NULL but the stream is already
+	 * opened. This shouldn't matter since the TLClientOpenStream() will make
+	 * sure that only one stream can be opened and only one process can reach
+	 * this place at a time. Also the .stop function will be always called
+	 * after this function returns so there should be no risk of stream
+	 * not being closed. */
+	PVR_ASSERT(g_sHTBData.hStream == NULL);
+	g_sHTBData.hStream = hStream;
+
+	/* We're starting the read operation so ensure we properly zero the
+	 * sentinel object. */
+	memset(pSentinel, 0, sizeof(*pSentinel));
+
+	/*
+	 * Find the first message location within pSentinel->pBuf
+	 * => for DI_START_TOKEN we must issue our first ACQUIRE, also for the
+	 * subsequent re-START calls (if any).
+	 */
+
+	HTB_GetFirstMessage(pSentinel, pui64Pos);
+
+	retVal = *pui64Pos == 0 ? DI_START_TOKEN : pSentinel->pCurr;
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %p, Stream %s @ %p",
+	                 __func__, retVal, HTB_STREAM_NAME, g_sHTBData.hStream));
+
+	return retVal;
+}
+
+/*
+ * _DebugTBTraceDIStop:
+ *
+ * Stop processing data collection and release any previously allocated private
+ * data structure if we have exhausted the previously filled data buffers.
+ */
+static void _DebugHBTraceDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+	IMG_UINT32 uiMsgLen;
+	PVRSRV_ERROR eError;
+
+	if (pSentinel == NULL)
+	{
+		return;
+	}
+
+	uiMsgLen = pSentinel->uiMsgLen;
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: MsgLen = %d", __func__, uiMsgLen));
+
+	/* If we get here the handle should never be NULL because
+	 * _DebugHBTraceDIStart() shouldn't allow that. */
+	if (g_sHTBData.hStream == NULL)
+	{
+		return;
+	}
+
+	if (uiMsgLen != 0)
+	{
+		eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+		                                 g_sHTBData.hStream, uiMsgLen);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - %s, nBytes %u",
+			        __func__, "TLClientReleaseDataLess",
+			        PVRSRVGETERRORSTRING(eError), uiMsgLen));
+		}
+	}
+
+	eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()",
+		        "TLClientCloseStream", PVRSRVGETERRORSTRING(eError),
+		        __func__));
+	}
+
+	g_sHTBData.hStream = NULL;
+}
+
+
+/*
+ * _DebugHBTraceDINext:
+ *
+ * This is where we release any acquired data which has been processed by the
+ * DIShow routine. If we have encountered a DI entry overflow we stop
+ * processing and return NULL. Otherwise we release the message that we
+ * previously processed and simply update our position pointer to the next
+ * valid HTB message (if any)
+ */
+static void *_DebugHBTraceDINext(OSDI_IMPL_ENTRY *psEntry, void *pvPriv,
+                                 IMG_UINT64 *pui64Pos)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+	IMG_UINT64 ui64CurPos;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+	if (pui64Pos)
+	{
+		ui64CurPos = *pui64Pos;
+		*pui64Pos = ui64CurPos + 1;
+	}
+
+	/* Determine if we've had an overflow on the previous 'Show' call. If so
+	 * we leave the previously acquired data in the queue (by releasing 0 bytes)
+	 * and return NULL to end this DI entry iteration.
+	 * If we have not overflowed we simply get the next HTB message and use that
+	 * for our display purposes. */
+
+	if (DIHasOverflowed(psEntry))
+	{
+		(void) TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream,
+		                               0);
+
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: OVERFLOW - returning NULL",
+		                 __func__));
+
+		return NULL;
+	}
+	else
+	{
+		eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+		                                 g_sHTBData.hStream,
+		                                 pSentinel->uiMsgLen);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s' @ %p Length %d",
+			        __func__, "TLClientReleaseDataLess",
+			        PVRSRVGETERRORSTRING(eError), pSentinel->pCurr,
+			        pSentinel->uiMsgLen));
+			PVR_DPF((PVR_DBG_WARNING, "%s: Buffer @ %p..%p", __func__,
+			        pSentinel->pBuf,
+			        (IMG_PBYTE) (pSentinel->pBuf + pSentinel->uiBufLen)));
+
+		}
+
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		                             g_sHTBData.hStream, &pSentinel->pBuf,
+		                             &pSentinel->uiBufLen);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'\nPrev message len %d",
+			        __func__, "TLClientAcquireData",
+			        PVRSRVGETERRORSTRING(eError), pSentinel->uiMsgLen));
+			pSentinel->pBuf = NULL;
+		}
+
+		pSentinel->uiMsgLen = 0; /* We don't (yet) know the message size */
+	}
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %p Msglen %d", __func__,
+	                 pSentinel->pBuf, pSentinel->uiMsgLen));
+
+	if (pSentinel->pBuf == NULL || pSentinel->uiBufLen == 0)
+	{
+		return NULL;
+	}
+
+	pSentinel->pCurr = HTB_GetNextMessage(pSentinel);
+
+	return pSentinel->pCurr;
+}
+
+/******************************************************************************
+ * HTB Dumping routines and definitions
+ *****************************************************************************/
+#define IS_VALID_FMT_STRING(FMT) (strchr(FMT, '%') != NULL)
+#define MAX_STRING_SIZE (128)
+
+typedef enum
+{
+	TRACEBUF_ARG_TYPE_INT,
+	TRACEBUF_ARG_TYPE_ERR,
+	TRACEBUF_ARG_TYPE_NONE
+} TRACEBUF_ARG_TYPE;
+
+/*
+ * Array of all Host Trace log IDs used to convert the tracebuf data
+ */
+typedef struct _HTB_TRACEBUF_LOG_ {
+	HTB_LOG_SFids eSFId;
+	IMG_CHAR      *pszName;
+	IMG_CHAR      *pszFmt;
+	IMG_UINT32    ui32ArgNum;
+	IMG_UINT32    ui32StrArgIdx; /* This is 1 indexed, if 0, there is not a string arg. */
+} HTB_TRACEBUF_LOG;
+
+static const HTB_TRACEBUF_LOG aLogs[] = {
+#define X(a, b, c, d, e, f) {HTB_LOG_CREATESFID(a,b,e,f), #c, d, e, f},
+	HTB_LOG_SFIDLIST
+#undef X
+};
+
+static const IMG_CHAR *aGroups[] = {
+#define X(A,B) #B,
+	HTB_LOG_SFGROUPLIST
+#undef X
+};
+static const IMG_UINT32 uiMax_aGroups = ARRAY_SIZE(aGroups) - 1;
+
+static TRACEBUF_ARG_TYPE ExtractOneArgFmt(IMG_CHAR **, IMG_CHAR *);
+/*
+ * ExtractOneArgFmt
+ *
+ * Scan the input 'printf-like' string *ppszFmt and return the next
+ * value string to be displayed. If there is no '%' format field in the
+ * string we return 'TRACEBUF_ARG_TYPE_NONE' and leave the input string
+ * untouched.
+ *
+ * Input
+ *	ppszFmt          reference to format string to be decoded
+ *	pszOneArgFmt     single field format from *ppszFmt
+ *
+ * Returns
+ *	TRACEBUF_ARG_TYPE_ERR       unrecognised argument
+ *	TRACEBUF_ARG_TYPE_INT       variable is of numeric type
+ *	TRACEBUF_ARG_TYPE_NONE      no variable reference in *ppszFmt
+ *
+ * Side-effect
+ *	*ppszFmt is updated to reference the next part of the format string
+ *	to be scanned
+ */
+static TRACEBUF_ARG_TYPE ExtractOneArgFmt(
+	IMG_CHAR **ppszFmt,
+	IMG_CHAR *pszOneArgFmt)
+{
+	IMG_CHAR          *pszFmt;
+	IMG_CHAR          *psT;
+	IMG_UINT32        ui32Count = MAX_STRING_SIZE;
+	IMG_UINT32        ui32OneArgSize;
+	TRACEBUF_ARG_TYPE eRet = TRACEBUF_ARG_TYPE_ERR;
+
+	if (NULL == ppszFmt)
+		return TRACEBUF_ARG_TYPE_ERR;
+
+	pszFmt = *ppszFmt;
+	if (NULL == pszFmt)
+		return TRACEBUF_ARG_TYPE_ERR;
+
+	/*
+	 * Find the first '%'
+	 * NOTE: we can be passed a simple string to display which will have no
+	 * parameters embedded within it. In this case we simply return
+	 * TRACEBUF_ARG_TYPE_NONE and the string contents will be the full pszFmt
+	 */
+	psT = strchr(pszFmt, '%');
+	if (psT == NULL)
+	{
+		return TRACEBUF_ARG_TYPE_NONE;
+	}
+
+	/* Find next conversion identifier after the initial '%' */
+	while ((*psT++) && (ui32Count-- > 0))
+	{
+		switch (*psT)
+		{
+			case 'd':
+			case 'i':
+			case 'o':
+			case 'u':
+			case 'x':
+			case 'X':
+			{
+				eRet = TRACEBUF_ARG_TYPE_INT;
+				goto _found_arg;
+			}
+			case 's':
+			{
+				eRet = TRACEBUF_ARG_TYPE_ERR;
+				goto _found_arg;
+			}
+		}
+	}
+
+	if ((psT == NULL) || (ui32Count == 0)) return TRACEBUF_ARG_TYPE_ERR;
+
+_found_arg:
+	ui32OneArgSize = psT - pszFmt + 1;
+	OSCachedMemCopy(pszOneArgFmt, pszFmt, ui32OneArgSize);
+	pszOneArgFmt[ui32OneArgSize] = '\0';
+
+	*ppszFmt = psT + 1;
+
+	return eRet;
+}
+
+static IMG_UINT32 idToLogIdx(IMG_UINT32 ui32CheckData)
+{
+	IMG_UINT32	i = 0;
+	for (i = 0; aLogs[i].eSFId != HTB_SF_LAST; i++)
+	{
+		if ( ui32CheckData == aLogs[i].eSFId )
+			return i;
+	}
+	/* Nothing found, return max value */
+	return HTB_SF_LAST;
+}
+
+/*
+ * DecodeHTB
+ *
+ * Decode the data buffer message located at pBuf. This should be a valid
+ * HTB message as we are provided with the start of the buffer. If empty there
+ * is no message to process. We update the uiMsgLen field with the size of the
+ * HTB message that we have processed so that it can be returned to the system
+ * on successful logging of the message to the output file.
+ *
+ *	Input
+ *		pSentinel reference to newly read data and pending completion data
+ *		          from a previous invocation [handle DI entry buffer overflow]
+ *		 -> pBuf         reference to raw data that we are to parse
+ *		 -> uiBufLen     total number of bytes of data available
+ *		 -> pCurr        start of message to decode
+ *
+ *		pvDumpDebugFile     output file
+ *		pfnDumpDebugPrintf  output generating routine
+ *
+ * Output
+ *		pSentinel
+ *		 -> uiMsgLen	length of the decoded message which will be freed to
+ *						the system on successful completion of the DI entry
+ *						update via _DebugHBTraceDINext(),
+ * Return Value
+ *		0				successful decode
+ *		-1				unsuccessful decode
+ */
+static int
+DecodeHTB(HTB_Sentinel_t *pSentinel, OSDI_IMPL_ENTRY *pvDumpDebugFile,
+          DI_PRINTF pfnDumpDebugPrintf)
+{
+	IMG_UINT32	ui32Data, ui32LogIdx, ui32ArgsCur;
+	IMG_CHAR	*pszFmt = NULL;
+	IMG_CHAR	aszOneArgFmt[MAX_STRING_SIZE];
+	IMG_BOOL	bUnrecognizedErrorPrinted = IMG_FALSE;
+
+	size_t	nPrinted;
+
+	void	*pNext, *pLast, *pStart, *pData = NULL;
+	PVRSRVTL_PPACKETHDR	ppHdr;	/* Current packet header */
+	IMG_UINT32	uiHdrType;		/* Packet header type */
+	IMG_UINT32	uiMsgSize;		/* Message size of current packet (bytes) */
+	IMG_BOOL	bPacketsDropped;
+
+	pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+	pStart = pSentinel->pCurr;
+
+	pSentinel->uiMsgLen = 0;	// Reset count for this message
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Buf @ %p..%p, Length = %d",
+	                 __func__, pStart, pLast, pSentinel->uiBufLen));
+
+	/*
+	 * We should have a DATA header with the necessary information following
+	 */
+	ppHdr = GET_PACKET_HDR(pStart);
+
+	if (ppHdr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Unexpected NULL packet in Host Trace buffer", __func__));
+		return -1;
+	}
+
+	uiHdrType = GET_PACKET_TYPE(ppHdr);
+	PVR_ASSERT(uiHdrType == PVRSRVTL_PACKETTYPE_DATA);
+
+	pNext = GET_NEXT_PACKET_ADDR(ppHdr);
+
+	PVR_ASSERT(pNext != NULL);
+
+	uiMsgSize = (IMG_UINT32)((size_t)pNext - (size_t)ppHdr);
+
+	pSentinel->uiMsgLen += uiMsgSize;
+
+	pData = GET_PACKET_DATA_PTR(ppHdr);
+
+	if (pData == NULL || pData >= pLast)
+	{
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: pData = %p, pLast = %p "
+		                 "Returning 0", __func__, pData, pLast));
+		return 0;
+	}
+
+	ui32Data = *(IMG_UINT32 *)pData;
+	ui32LogIdx = idToLogIdx(ui32Data);
+
+	/*
+	 * Check if the unrecognised ID is valid and therefore, tracebuf
+	 * needs updating.
+	 */
+	if (ui32LogIdx == HTB_SF_LAST)
+	{
+		if (HTB_LOG_VALIDID(ui32Data))
+		{
+			if (!bUnrecognizedErrorPrinted)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				    "%s: Unrecognised LOG value '%x' GID %x Params %d ID %x @ '%p'",
+				    __func__, ui32Data, HTB_SF_GID(ui32Data),
+				    HTB_SF_PARAMNUM(ui32Data), ui32Data & 0xfff, pData));
+				bUnrecognizedErrorPrinted = IMG_TRUE;
+			}
+
+			return 0;
+		}
+
+		PVR_DPF((PVR_DBG_ERROR,
+		    "%s: Unrecognised and invalid LOG value detected '%x'",
+		    __func__, ui32Data));
+
+		return -1;
+	}
+
+	/* The string format we are going to display */
+	/*
+	 * The display will show the header (log-ID, group-ID, number of params)
+	 * The maximum parameter list length = 15 (only 4bits used to encode)
+	 * so we need HEADER + 15 * sizeof(UINT32) and the displayed string
+	 * describing the event. We use a buffer in the per-process pSentinel
+	 * structure to hold the data.
+	 */
+	pszFmt = aLogs[ui32LogIdx].pszFmt;
+
+	/* add the message payload size to the running count */
+	ui32ArgsCur = HTB_SF_PARAMNUM(ui32Data);
+
+	/* Determine if we've over-filled the buffer and had to drop packets */
+	bPacketsDropped = CHECK_PACKETS_DROPPED(ppHdr);
+	if (bPacketsDropped ||
+		(uiHdrType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED))
+	{
+		/* Flag this as it is useful to know ... */
+
+		PVR_DUMPDEBUG_LOG("\n<========================== *** PACKETS DROPPED *** ======================>\n");
+	}
+
+	{
+		IMG_UINT32 ui32Timestampns, ui32PID, ui32TID;
+		IMG_UINT64 ui64Timestamp, ui64TimestampSec;
+		IMG_CHAR	*szBuffer = pSentinel->szBuffer;	// Buffer start
+		IMG_CHAR	*pszBuffer = pSentinel->szBuffer;	// Current place in buf
+		size_t		uBufBytesAvailable = sizeof(pSentinel->szBuffer);
+		IMG_UINT32	*pui32Data = (IMG_UINT32 *)pData;
+		IMG_UINT32	ui_aGroupIdx;
+
+		// Get PID field from data stream
+		pui32Data++;
+		ui32PID = *pui32Data;
+		// Get TID field from data stream
+		pui32Data++;
+		ui32TID = *pui32Data;
+		// Get Timestamp part 1 from data stream
+		pui32Data++;
+		ui64Timestamp = (IMG_UINT64) *pui32Data << 32;
+		// Get Timestamp part 2 from data stream
+		pui32Data++;
+		ui64Timestamp |= (IMG_UINT64) *pui32Data;
+		// Move to start of message contents data
+		pui32Data++;
+
+		/*
+		 * We need to snprintf the data to a local in-kernel buffer
+		 * and then PVR_DUMPDEBUG_LOG() that in one shot
+		 */
+		ui_aGroupIdx = MIN(HTB_SF_GID(ui32Data), uiMax_aGroups);
+
+		/* Divide by 1B to get seconds & mod using output var (nanosecond resolution)*/
+		ui64TimestampSec = OSDivide64r64(ui64Timestamp, 1000000000, &ui32Timestampns);
+
+		nPrinted = OSSNPrintf(szBuffer, uBufBytesAvailable, "%010"IMG_UINT64_FMTSPEC".%09u:%-5u-%-5u-%s> ",
+			ui64TimestampSec, ui32Timestampns, ui32PID, ui32TID, aGroups[ui_aGroupIdx]);
+		if (nPrinted >= uBufBytesAvailable)
+		{
+			PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+				" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+				uBufBytesAvailable);
+
+			nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+		}
+
+		PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+		/* Update where our next 'output' point in the buffer is */
+		pszBuffer += nPrinted;
+		uBufBytesAvailable -= nPrinted;
+
+		/*
+		 * Print one argument at a time as this simplifies handling variable
+		 * number of arguments. Special case handling for no arguments.
+		 * This is the case for simple format strings such as
+		 * HTB_SF_MAIN_KICK_UNCOUNTED.
+		 */
+		if (ui32ArgsCur == 0)
+		{
+			if (pszFmt)
+			{
+				const ssize_t iCopiedCnt =
+					OSStringSafeCopy(pszBuffer, pszFmt, uBufBytesAvailable);
+				if (iCopiedCnt < 0)
+				{
+					PVR_DUMPDEBUG_LOG("Buffer overrun - %zu required,"
+					                  " max space %zu\n",
+					                   OSStringLength(pszFmt),
+					                   uBufBytesAvailable);
+
+					/* Ensure we don't overflow buffer */
+					nPrinted = uBufBytesAvailable;
+				}
+				else
+				{
+					nPrinted = iCopiedCnt;
+				}
+				PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+				pszBuffer += nPrinted;
+				/* Don't update the uBufBytesAvailable as we have finished this
+				 * message decode. pszBuffer - szBuffer is the total amount of
+				 * data we have decoded.
+				 */
+			}
+		}
+		else
+		{
+			if (HTB_SF_GID(ui32Data) == HTB_GID_CTRL && HTB_SF_ID(ui32Data) == HTB_ID_MARK_SCALE)
+			{
+				IMG_UINT32 i;
+				IMG_UINT32 ui32ArgArray[HTB_MARK_SCALE_ARG_ARRAY_SIZE];
+				IMG_UINT64 ui64OSTS = 0;
+				IMG_UINT32 ui32OSTSRem = 0;
+				IMG_UINT64 ui64CRTS = 0;
+
+				/* Retrieve 6 args to an array */
+				for (i = 0; i < ARRAY_SIZE(ui32ArgArray); i++)
+				{
+					ui32ArgArray[i] = *pui32Data;
+					pui32Data++;
+					--ui32ArgsCur;
+				}
+
+				ui64OSTS = (IMG_UINT64) ui32ArgArray[HTB_ARG_OSTS_PT1] << 32 | ui32ArgArray[HTB_ARG_OSTS_PT2];
+				ui64CRTS = (IMG_UINT64) ui32ArgArray[HTB_ARG_CRTS_PT1] << 32 | ui32ArgArray[HTB_ARG_CRTS_PT2];
+
+				/* Divide by 1B to get seconds, remainder in nano seconds*/
+				ui64OSTS = OSDivide64r64(ui64OSTS, 1000000000, &ui32OSTSRem);
+
+				nPrinted = OSSNPrintf(pszBuffer,
+						              uBufBytesAvailable,
+						              "HTBFWMkSync Mark=%u OSTS=%010" IMG_UINT64_FMTSPEC ".%09u CRTS=%" IMG_UINT64_FMTSPEC " CalcClkSpd=%u\n",
+						              ui32ArgArray[HTB_ARG_SYNCMARK],
+						              ui64OSTS,
+						              ui32OSTSRem,
+						              ui64CRTS,
+						              ui32ArgArray[HTB_ARG_CLKSPD]);
+
+				if (nPrinted >= uBufBytesAvailable)
+				{
+					PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+						" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+						uBufBytesAvailable);
+					nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+				}
+
+				PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+				pszBuffer += nPrinted;
+				uBufBytesAvailable -= nPrinted;
+			}
+			else
+			{
+				while (IS_VALID_FMT_STRING(pszFmt) && (uBufBytesAvailable > 0))
+				{
+					IMG_UINT32 ui32TmpArg = *pui32Data;
+					TRACEBUF_ARG_TYPE eArgType;
+
+					eArgType = ExtractOneArgFmt(&pszFmt, aszOneArgFmt);
+
+					pui32Data++;
+					ui32ArgsCur--;
+
+					switch (eArgType)
+					{
+						case TRACEBUF_ARG_TYPE_INT:
+							nPrinted = OSSNPrintf(pszBuffer, uBufBytesAvailable,
+								aszOneArgFmt, ui32TmpArg);
+							break;
+
+						case TRACEBUF_ARG_TYPE_NONE:
+							{
+								const ssize_t iCopiedCnt =
+									OSStringSafeCopy(pszBuffer,
+									                 pszFmt,
+									                 uBufBytesAvailable);
+								if (iCopiedCnt < 0)
+								{
+									nPrinted = OSStringLength(pszFmt);
+								}
+								else
+								{
+									nPrinted = iCopiedCnt;
+								}
+							}
+							break;
+
+						default:
+							nPrinted = OSSNPrintf(pszBuffer, uBufBytesAvailable,
+								"Error processing arguments, type not "
+								"recognized (fmt: %s)", aszOneArgFmt);
+							break;
+					}
+					if (nPrinted >= uBufBytesAvailable)
+					{
+						PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+							" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+							uBufBytesAvailable);
+						nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+					}
+					PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+					pszBuffer += nPrinted;
+					uBufBytesAvailable -= nPrinted;
+				}
+				/* Display any remaining text in pszFmt string */
+				if (pszFmt)
+				{
+					const ssize_t iCopiedCnt =
+						OSStringSafeCopy(pszBuffer, pszFmt, uBufBytesAvailable);
+					if (iCopiedCnt < 0)
+					{
+						PVR_DUMPDEBUG_LOG("Buffer overrun - %zu required,"
+						                  " max space %zu\n",
+						                   OSStringLength(pszFmt),
+						                   uBufBytesAvailable);
+
+						/* Ensure we don't overflow buffer */
+						nPrinted = uBufBytesAvailable;
+					}
+					else
+					{
+						nPrinted = iCopiedCnt;
+					}
+					PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+					pszBuffer += nPrinted;
+					/* Don't update the uBufBytesAvailable as we have finished this
+					 * message decode. pszBuffer - szBuffer is the total amount of
+					 * data we have decoded.
+					 */
+				}
+			}
+		}
+
+		/* Update total bytes processed */
+		pSentinel->uiTotal += (pszBuffer - szBuffer);
+	}
+	return 0;
+}
+
+/*
+ * HTBDumpBuffer: Dump the Host Trace Buffer using the TLClient API
+ *
+ * This routine just parses *one* message from the buffer.
+ * The stream will be opened by the Start() routine, closed by the Stop() and
+ * updated for data consumed by this routine once we have DebugPrintf'd it.
+ * We use the new TLReleaseDataLess() routine which enables us to update the
+ * HTB contents with just the amount of data we have successfully processed.
+ * If we need to leave the data available we can call this with a 0 count.
+ * This will happen in the case of a buffer overflow so that we can reprocess
+ * any data which wasn't handled before.
+ *
+ * In case of overflow or an error we return -1 otherwise 0
+ *
+ * Input:
+ *  pfnPrintf           output routine to display data
+ *  psEntry             handle to debug frontend
+ *  pvData              data address to start dumping from
+ *                      (set by Start() / Next())
+ */
+static int HTBDumpBuffer(DI_PRINTF pfnPrintf, OSDI_IMPL_ENTRY *psEntry,
+                         void *pvData)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+
+	PVR_ASSERT(pvData != NULL);
+
+	if (pvData == DI_START_TOKEN)
+	{
+		if (pSentinel->pCurr == NULL)
+		{
+			HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: DI_START_TOKEN, "
+			                 "Empty buffer", __func__));
+			return 0;
+		}
+		PVR_ASSERT(pSentinel->pCurr != NULL);
+
+		/* Display a Header as we have data to process */
+		pfnPrintf(psEntry, "%-20s:%-5s-%-5s-%s  %s\n", "Timestamp", "PID", "TID", "Group>",
+		         "Log Entry");
+	}
+	else
+	{
+		if (pvData != NULL)
+		{
+			PVR_ASSERT(pSentinel->pCurr == pvData);
+		}
+	}
+
+	return DecodeHTB(pSentinel, psEntry, pfnPrintf);
+}
+
+
+/******************************************************************************
+ * External Entry Point routines ...
+ *****************************************************************************/
+/*************************************************************************/ /*!
+ @Function     HTB_CreateDIEntry
+
+ @Description  Create the debugFS entry-point for the host-trace-buffer
+
+ @Returns      eError          internal error code, PVRSRV_OK on success
+
+ */ /*************************************************************************/
+PVRSRV_ERROR HTB_CreateDIEntry_Impl(void)
+{
+	PVRSRV_ERROR eError;
+
+	DI_ITERATOR_CB sIterator = {
+		.pfnStart = _DebugHBTraceDIStart,
+		.pfnStop  = _DebugHBTraceDIStop,
+		.pfnNext  = _DebugHBTraceDINext,
+		.pfnShow  = _DebugHBTraceDIShow,
+	};
+
+	eError = DICreateEntry("host_trace", NULL, &sIterator,
+	                       &g_sHTBData.sSentinel,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &g_sHTBData.psDumpHostDiEntry);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DICreateEntry");
+
+	return PVRSRV_OK;
+}
+
+
+/*************************************************************************/ /*!
+ @Function     HTB_DestroyDIEntry
+
+ @Description  Destroy the debugFS entry-point created by earlier
+               HTB_CreateDIEntry() call.
+*/ /**************************************************************************/
+void HTB_DestroyDIEntry_Impl(void)
+{
+	if (g_sHTBData.psDumpHostDiEntry != NULL)
+	{
+		DIDestroyEntry(g_sHTBData.psDumpHostDiEntry);
+		g_sHTBData.psDumpHostDiEntry = NULL;
+	}
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.h b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.h
new file mode 100644
index 000000000000..cf18bbb2ac5d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/htb_debug.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@File           htb_debug.h
+@Title          Linux debugFS routine setup header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HTB_DEBUG_H
+#define HTB_DEBUG_H
+
+/**************************************************************************/ /*!
+ @Function     HTB_CreateDIEntry
+
+ @Description  Create the debugFS entry-point for the host-trace-buffer
+
+ @Returns      eError          internal error code, PVRSRV_OK on success
+
+ */ /**************************************************************************/
+PVRSRV_ERROR HTB_CreateDIEntry_Impl(void);
+
+/**************************************************************************/ /*!
+ @Function     HTB_DestroyFSEntry
+
+ @Description  Destroy the debugFS entry-point created by earlier
+               HTB_CreateDIEntry() call.
+*/ /**************************************************************************/
+void HTB_DestroyDIEntry_Impl(void);
+
+#if defined(PVRSRV_ENABLE_HTB)
+#define HTB_CreateDIEntry() HTB_CreateDIEntry_Impl()
+#define HTB_DestroyDIEntry() HTB_DestroyDIEntry_Impl()
+#else /* !PVRSRV_ENABLE_HTB */
+#define HTB_CreateDIEntry() PVRSRV_OK
+#define HTB_DestroyDIEntry()
+#endif /* PVRSRV_ENABLE_HTB */
+
+#endif /* HTB_DEBUG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/htbserver.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/htbserver.c
new file mode 100644
index 000000000000..9ed3283f52b2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/htbserver.c
@@ -0,0 +1,998 @@
+/*************************************************************************/ /*!
+@File           htbserver.c
+@Title          Host Trace Buffer server implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "htbserver.h"
+#include "htbuffer_types.h"
+#include "tlstream.h"
+#include "pvrsrv_tlcommon.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+
+/* size of circular buffer controlling the maximum number of concurrent PIDs logged */
+#define HTB_MAX_NUM_PID 8
+
+/* number of times to try rewriting a log entry */
+#define HTB_LOG_RETRY_COUNT 5
+
+#if defined(__linux__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+/*************************************************************************/ /*!
+  Host Trace Buffer control information structure
+*/ /**************************************************************************/
+typedef struct
+{
+	IMG_UINT32 ui32BufferSize;      /*!< Requested buffer size in bytes
+                                         Once set this may not be changed */
+
+	HTB_OPMODE_CTRL eOpMode;        /*!< Control what trace data is dropped if
+                                         the buffer is full.
+                                         Once set this may not be changed */
+
+/*	IMG_UINT32 ui32GroupEnable; */  /*!< Flags word controlling groups to be
+                                         logged */
+
+	IMG_UINT32 ui32LogLevel;        /*!< Log level to control messages logged */
+
+	IMG_UINT32 aui32EnablePID[HTB_MAX_NUM_PID]; /*!< PIDs to enable logging for
+                                                     a specific set of processes */
+
+	IMG_UINT32 ui32PIDCount;        /*!< Current number of PIDs being logged */
+
+	IMG_UINT32 ui32PIDHead;         /*!< Head of the PID circular buffer */
+
+	HTB_LOGMODE_CTRL eLogMode;      /*!< Logging mode control */
+
+	IMG_BOOL bLogDropSignalled;     /*!< Flag indicating if a log message has
+                                         been signalled as dropped */
+
+	/* synchronisation parameters */
+	IMG_UINT64 ui64SyncOSTS;
+	IMG_UINT64 ui64SyncCRTS;
+	IMG_UINT32 ui32SyncCalcClkSpd;
+	IMG_UINT32 ui32SyncMarker;
+
+	IMG_BOOL bInitDone;             /* Set by HTBInit, reset by HTBDeInit */
+
+	POS_SPINLOCK hRepeatMarkerLock;     /*!< Spinlock used in HTBLogKM to protect global variables
+	                                     (ByteCount, OSTS, CRTS ClkSpeed)
+	                                     from becoming inconsistent due to calls from
+	                                     both KM and UM */
+
+	IMG_UINT32 ui32ByteCount; /* Byte count used for triggering repeat sync point */
+	/* static variables containing details of previous sync point */
+	IMG_UINT64 ui64OSTS;
+	IMG_UINT64 ui64CRTS;
+	IMG_UINT32 ui32ClkSpeed;
+
+} HTB_CTRL_INFO;
+
+
+/*************************************************************************/ /*!
+*/ /**************************************************************************/
+static const IMG_UINT32 MapFlags[] =
+{
+	0,                    /* HTB_OPMODE_UNDEF = 0 */
+	TL_OPMODE_DROP_NEWER, /* HTB_OPMODE_DROPLATEST */
+	TL_OPMODE_DROP_OLDEST,/* HTB_OPMODE_DROPOLDEST */
+	TL_OPMODE_BLOCK       /* HTB_OPMODE_BLOCK */
+};
+
+static_assert(0 == HTB_OPMODE_UNDEF,      "Unexpected value for HTB_OPMODE_UNDEF");
+static_assert(1 == HTB_OPMODE_DROPLATEST, "Unexpected value for HTB_OPMODE_DROPLATEST");
+static_assert(2 == HTB_OPMODE_DROPOLDEST, "Unexpected value for HTB_OPMODE_DROPOLDEST");
+static_assert(3 == HTB_OPMODE_BLOCK,      "Unexpected value for HTB_OPMODE_BLOCK");
+
+static_assert(1 == TL_OPMODE_DROP_NEWER,  "Unexpected value for TL_OPMODE_DROP_NEWER");
+static_assert(2 == TL_OPMODE_DROP_OLDEST, "Unexpected value for TL_OPMODE_DROP_OLDEST");
+static_assert(3 == TL_OPMODE_BLOCK,       "Unexpected value for TL_OPMODE_BLOCK");
+
+static const IMG_UINT32 g_ui32TLBaseFlags; //TL_FLAG_NO_SIGNAL_ON_COMMIT
+
+/* Minimum TL buffer size.
+ * Large enough for around 60 worst case messages or 200 average messages
+ */
+#define HTB_TL_BUFFER_SIZE_MIN	(0x10000)
+
+/* Minimum concentration of HTB packets in a TL Stream is 60%
+ * If we just put the HTB header in the TL stream (12 bytes), the TL overhead
+ * is 8 bytes for its own header, so for the smallest possible (and most
+ * inefficient) packet we have 3/5 of the buffer used for actual HTB data.
+ * This shift is used as a guaranteed estimation on when to produce a repeat
+ * packet. By shifting the size of the buffer by 1 we effectively /2 this
+ * under the 60% boundary chance we may have overwritten the marker and thus
+ * guaranteed to always have a marker in the stream */
+#define HTB_MARKER_PREDICTION_THRESHOLD(val) (val >> 1)
+
+static HTB_CTRL_INFO g_sCtrl;
+static IMG_BOOL g_bConfigured = IMG_FALSE;
+static IMG_HANDLE g_hTLStream;
+
+static IMG_HANDLE hHtbDbgReqNotify;
+
+/************************************************************************/ /*!
+ @Function      _LookupFlags
+ @Description   Convert HTBuffer Operation mode to TLStream flags
+
+ @Input         eModeHTBuffer   Operation Mode
+
+ @Return        IMG_UINT32      TLStream FLags
+*/ /**************************************************************************/
+static IMG_UINT32
+_LookupFlags( HTB_OPMODE_CTRL eMode )
+{
+	return (eMode < ARRAY_SIZE(MapFlags)) ? MapFlags[eMode] : 0;
+}
+
+
+/************************************************************************/ /*!
+ @Function      _HTBLogDebugInfo
+ @Description   Debug dump handler used to dump the state of the HTB module.
+                Called for each verbosity level during a debug dump. Function
+                only prints state when called for High verbosity.
+
+ @Input         hDebugRequestHandle See PFN_DBGREQ_NOTIFY
+
+ @Input         ui32VerbLevel       See PFN_DBGREQ_NOTIFY
+
+ @Input         pfnDumpDebugPrintf  See PFN_DBGREQ_NOTIFY
+
+ @Input         pvDumpDebugFile     See PFN_DBGREQ_NOTIFY
+
+*/ /**************************************************************************/
+static void _HTBLogDebugInfo(
+		PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+		IMG_UINT32 ui32VerbLevel,
+		DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+		void *pvDumpDebugFile
+)
+{
+	PVR_UNREFERENCED_PARAMETER(hDebugRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+
+		if (g_bConfigured)
+		{
+			IMG_INT i;
+
+			PVR_DUMPDEBUG_LOG("------[ HTB Log state: On ]------");
+
+			PVR_DUMPDEBUG_LOG("HTB Log mode: %d", g_sCtrl.eLogMode);
+			PVR_DUMPDEBUG_LOG("HTB Log level: %d", g_sCtrl.ui32LogLevel);
+			PVR_DUMPDEBUG_LOG("HTB Buffer Opmode: %d", g_sCtrl.eOpMode);
+
+			for (i=0; i < HTB_FLAG_NUM_EL; i++)
+			{
+				PVR_DUMPDEBUG_LOG("HTB Log group %d: %x", i, g_auiHTBGroupEnable[i]);
+			}
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("------[ HTB Log state: Off ]------");
+		}
+	}
+}
+
+static IMG_UINT32 g_ui32HTBufferSize = HTB_TL_BUFFER_SIZE_MIN;
+
+/*
+ * AppHint access routine forward definitions
+ */
+static PVRSRV_ERROR _HTBSetLogGroup(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32);
+static PVRSRV_ERROR _HTBReadLogGroup(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32 *);
+
+static PVRSRV_ERROR	_HTBSetOpMode(const PVRSRV_DEVICE_NODE *, const void *,
+                                   IMG_UINT32);
+static PVRSRV_ERROR _HTBReadOpMode(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32 *);
+
+static void _OnTLReaderOpenCallback(void *);
+
+/************************************************************************/ /*!
+ @Function      HTBInit
+ @Description   Allocate and initialise the Host Trace Buffer
+                The buffer size may be changed by specifying
+                HTBufferSizeInKB=xxxx
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBInit_Impl(void)
+{
+	void			*pvAppHintState = NULL;
+	IMG_UINT32		ui32AppHintDefault;
+	IMG_UINT32		ui32BufBytes;
+	PVRSRV_ERROR	eError;
+
+	if (g_sCtrl.bInitDone)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "HTBInit: Driver already initialised"));
+		return PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+
+	/*
+	 * Buffer Size can be configured by specifying a value in the AppHint
+	 * This will only take effect at module load time so there is no query
+	 * or setting mechanism available.
+	 */
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HTBufferSizeInKB,
+										NULL,
+										NULL,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableHTBLogGroup,
+	                                    _HTBReadLogGroup,
+	                                    _HTBSetLogGroup,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HTBOperationMode,
+	                                    _HTBReadOpMode,
+	                                    _HTBSetOpMode,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+
+	/*
+	 * Now get whatever values have been configured for our AppHints
+	 */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = HTB_TL_BUFFER_SIZE_MIN / 1024;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+						 &ui32AppHintDefault, &g_ui32HTBufferSize);
+	OSFreeAppHintState(pvAppHintState);
+
+	ui32BufBytes = g_ui32HTBufferSize * 1024;
+
+	/* initialise rest of state */
+	g_sCtrl.ui32BufferSize =
+		(ui32BufBytes < HTB_TL_BUFFER_SIZE_MIN)
+		? HTB_TL_BUFFER_SIZE_MIN
+		: ui32BufBytes;
+	g_sCtrl.eOpMode = HTB_OPMODE_DROPOLDEST;
+	g_sCtrl.ui32LogLevel = 0;
+	g_sCtrl.ui32PIDCount = 0;
+	g_sCtrl.ui32PIDHead = 0;
+	g_sCtrl.eLogMode = HTB_LOGMODE_ALLPID;
+	g_sCtrl.bLogDropSignalled = IMG_FALSE;
+
+	eError = OSSpinLockCreate(&g_sCtrl.hRepeatMarkerLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSSpinLockCreate");
+
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&hHtbDbgReqNotify,
+			 _HTBLogDebugInfo, DEBUG_REQUEST_HTB, NULL);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify");
+
+	g_sCtrl.bInitDone = IMG_TRUE;
+
+	/* Log the current driver parameter setting for the HTBufferSizeInKB.
+	 * We do this here as there is no other infrastructure for obtaining
+	 * the value.
+	 */
+	if (g_ui32HTBufferSize != ui32AppHintDefault)
+	{
+		PVR_LOG(("Increasing HTBufferSize to %uKB", g_ui32HTBufferSize));
+	}
+
+
+	return PVRSRV_OK;
+}
+
+/************************************************************************/ /*!
+ @Function      HTBDeInit
+ @Description   Close the Host Trace Buffer and free all resources. Must
+                perform a no-op if already de-initialised.
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBDeInit_Impl( void )
+{
+	if (!g_sCtrl.bInitDone)
+		return PVRSRV_OK;
+
+	if (hHtbDbgReqNotify)
+	{
+		/* Not much we can do if it fails, driver unloading */
+		(void)PVRSRVUnregisterDriverDbgRequestNotify(hHtbDbgReqNotify);
+		hHtbDbgReqNotify = NULL;
+	}
+
+	if (g_hTLStream)
+	{
+		TLStreamClose( g_hTLStream );
+		g_hTLStream = NULL;
+	}
+
+	if (g_sCtrl.hRepeatMarkerLock != NULL)
+	{
+		OSSpinLockDestroy(g_sCtrl.hRepeatMarkerLock);
+		g_sCtrl.hRepeatMarkerLock = NULL;
+	}
+
+	g_sCtrl.bInitDone = IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+
+/*************************************************************************/ /*!
+ AppHint interface functions
+*/ /**************************************************************************/
+static
+PVRSRV_ERROR _HTBSetLogGroup(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const void *psPrivate,
+                             IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return HTBControlKM(1, &ui32Value, 0, 0,
+	                    HTB_LOGMODE_UNDEF, HTB_OPMODE_UNDEF);
+}
+
+static
+PVRSRV_ERROR _HTBReadLogGroup(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 *pui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value = g_auiHTBGroupEnable[0];
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR _HTBSetOpMode(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                           const void *psPrivate,
+                           IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return HTBControlKM(0, NULL, 0, 0, HTB_LOGMODE_UNDEF, ui32Value);
+}
+
+static
+PVRSRV_ERROR _HTBReadOpMode(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                            const void *psPrivate,
+                            IMG_UINT32 *pui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value = (IMG_UINT32)g_sCtrl.eOpMode;
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_HTB)
+static IMG_BOOL
+_ValidPID( IMG_UINT32 PID )
+{
+	IMG_UINT32 i;
+	for (i = 0; i < g_sCtrl.ui32PIDCount; i++)
+	{
+		if ( g_sCtrl.aui32EnablePID[i] == PID )
+		{
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+#endif	/* PVRSRV_ENABLE_HTB */
+
+/*************************************************************************/ /*!
+ @Function      HTBLogKM
+ @Description   Record a Host Trace Buffer log event
+
+ @Input         PID             The PID of the process the event is associated
+                                with. This is provided as an argument rather
+                                than querying internally so that events associated
+                                with a particular process, but performed by
+                                another can be logged correctly.
+
+ @Input         ui64TimeStamp   The timestamp to be associated with this log event
+
+ @Input         SF              The log event ID
+
+ @Input         ...             Log parameters
+
+ @Return        PVRSRV_OK       Success.
+
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+HTBLogKM(IMG_UINT32 PID,
+		IMG_UINT32 TID,
+		IMG_UINT64 ui64TimeStamp,
+		HTB_LOG_SFids SF,
+		va_list args
+)
+{
+#if defined(PVRSRV_ENABLE_HTB)
+
+	OS_SPINLOCK_FLAGS uiSpinLockFlags;
+	IMG_UINT32 ui32ReturnFlags = 0;
+	IMG_UINT32 ui32CurrentArg = 0;
+
+	/* Local snapshot variables of global counters */
+	IMG_UINT64 ui64OSTSSnap;
+	IMG_UINT64 ui64CRTSSnap;
+	IMG_UINT32 ui32ClkSpeedSnap;
+
+	/* format of messages is: SF:PID:TID:TIMEPT1:TIMEPT2:[PARn]*
+	 * Buffer is on the stack so we don't need a semaphore to guard it
+	 */
+	IMG_UINT32 aui32MessageBuffer[HTB_LOG_HEADER_SIZE+HTB_LOG_MAX_PARAMS];
+	IMG_UINT32 aui32Args[HTB_LOG_MAX_PARAMS + 1] = {0};
+
+	/* Min HTB size is HTB_TL_BUFFER_SIZE_MIN : 10000 bytes and Max message/
+	 * packet size is 4*(HTB_LOG_HEADER_SIZE+HTB_LOG_MAX_PARAMS) = 80 bytes,
+	 * hence with these constraints this design is unlikely to get
+	 * PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED error
+	 */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_ENABLED;
+	IMG_UINT32 ui32RetryCount = HTB_LOG_RETRY_COUNT;
+	IMG_UINT32 * pui32Message = aui32MessageBuffer;
+	IMG_UINT32 ui32NumArgs = HTB_SF_PARAMNUM(SF);
+	IMG_UINT32 ui32StrArg = HTB_SF_STRNUM(SF);
+	IMG_UINT32 ui32CurrentStringArg;
+
+	IMG_UINT32 ui32MessageSize = 4 * (HTB_LOG_HEADER_SIZE+ui32NumArgs);
+
+	if (ui32StrArg > 0)
+	{
+		ui32MessageSize += HTB_LOG_STR_ARG_SIZE - sizeof(IMG_UINT32);
+	}
+
+	PVR_ASSERT(ui32NumArgs <= HTB_LOG_MAX_PARAMS);
+	ui32NumArgs = (ui32NumArgs>HTB_LOG_MAX_PARAMS) ?
+			HTB_LOG_MAX_PARAMS : ui32NumArgs;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32NumArgs == HTB_SF_PARAMNUM(SF), eError, ReturnError);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32NumArgs <= HTB_LOG_MAX_PARAMS, eError, ReturnError);
+
+	/* Needs to be set up here because it's accessed from both `if` blocks below
+	 * and it needs to be pre-populated for both of them (pui32Message case and
+	 * HTB_SF_CTRL_FWSYNC_MARK_SCALE case). */
+
+	while (ui32CurrentArg < ui32NumArgs)
+	{
+		if (ui32StrArg != 0 && ui32CurrentArg == ui32StrArg - 1)
+		{
+			IMG_CHAR* strArg;
+			strArg = va_arg(args, IMG_CHAR*);
+
+			/* if a string is present, it will need more than one UINT32 words. */
+			ui32NumArgs += HTB_LOG_STR_ARG_NUM_WORDS - 1;
+
+			/* Ignore if filename is not supported. */
+			if (strcmp(strArg, "n/a") == 0)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "N/A\n"));
+				return PVRSRV_OK;
+			}
+
+			/* looping through the created string to get the substrings to encode. */
+			for (ui32CurrentStringArg = 0; ui32CurrentStringArg < HTB_LOG_STR_ARG_NUM_WORDS; ui32CurrentStringArg++)
+			{
+				IMG_UINT32 encodedString = 0;
+
+				if (*strArg == '\0')
+				{
+					aui32Args[ui32CurrentArg] = encodedString;
+					ui32CurrentArg++;
+				}
+				else
+				{
+					IMG_UINT32 currentSubstring;
+
+					for (currentSubstring = 0; currentSubstring < sizeof(IMG_UINT32); currentSubstring++)
+					{
+						IMG_UINT32 bitPos = currentSubstring * 8;
+
+						if (*strArg == '\0')
+						{
+							break;
+						}
+						else
+						{
+							encodedString |= (IMG_UINT32) *strArg << bitPos;
+							strArg++;
+						}
+					}
+
+					aui32Args[ui32CurrentArg] = encodedString;
+					ui32CurrentArg++;
+				}
+			}
+		}
+		else
+		{
+			aui32Args[ui32CurrentArg] = va_arg(args, IMG_UINT32);
+			ui32CurrentArg++;
+		}
+	}
+
+	if ( g_hTLStream
+			&& ( 0 == PID || ~0 == PID || HTB_LOGMODE_ALLPID == g_sCtrl.eLogMode || _ValidPID(PID) )
+/*			&& ( g_sCtrl.ui32GroupEnable & (0x1 << HTB_SF_GID(SF)) ) */
+/*			&& ( g_sCtrl.ui32LogLevel >= HTB_SF_LVL(SF) ) */
+			)
+	{
+		*pui32Message++ = SF;
+		*pui32Message++ = PID;
+		*pui32Message++ = TID;
+		*pui32Message++ = ((IMG_UINT32)((ui64TimeStamp>>32)&0xffffffff));
+		*pui32Message++ = ((IMG_UINT32)(ui64TimeStamp&0xffffffff));
+		for (ui32CurrentArg = 0; ui32CurrentArg < ui32NumArgs; ui32CurrentArg++)
+		{
+			pui32Message[ui32CurrentArg] = aui32Args[ui32CurrentArg];
+		}
+
+		eError = TLStreamWriteRetFlags( g_hTLStream, (IMG_UINT8*)aui32MessageBuffer, ui32MessageSize, &ui32ReturnFlags );
+
+		while ( PVRSRV_ERROR_NOT_READY == eError && ui32RetryCount-- )
+		{
+			OSReleaseThreadQuanta();
+			eError = TLStreamWriteRetFlags( g_hTLStream, (IMG_UINT8*)aui32MessageBuffer, ui32MessageSize, &ui32ReturnFlags );
+		}
+
+		if ( PVRSRV_OK == eError )
+		{
+			g_sCtrl.bLogDropSignalled = IMG_FALSE;
+		}
+		else if ( PVRSRV_ERROR_STREAM_FULL != eError || !g_sCtrl.bLogDropSignalled )
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()", "TLStreamWrite", PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		if ( PVRSRV_ERROR_STREAM_FULL == eError )
+		{
+			g_sCtrl.bLogDropSignalled = IMG_TRUE;
+		}
+
+	}
+
+	if (SF == HTB_SF_CTRL_FWSYNC_MARK_SCALE)
+	{
+		OSSpinLockAcquire(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+
+		/* If a marker is being placed reset byte count from last marker */
+		g_sCtrl.ui32ByteCount = 0;
+		g_sCtrl.ui64OSTS = (IMG_UINT64)aui32Args[HTB_ARG_OSTS_PT1] << 32 | aui32Args[HTB_ARG_OSTS_PT2];
+		g_sCtrl.ui64CRTS = (IMG_UINT64)aui32Args[HTB_ARG_CRTS_PT1] << 32 | aui32Args[HTB_ARG_CRTS_PT2];
+		g_sCtrl.ui32ClkSpeed = aui32Args[HTB_ARG_CLKSPD];
+
+		OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+	}
+	else
+	{
+		OSSpinLockAcquire(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+		/* Increase global count */
+		g_sCtrl.ui32ByteCount += ui32MessageSize;
+
+		/* Check if packet has overwritten last marker/rpt &&
+		   If the packet count is over half the size of the buffer */
+		if (ui32ReturnFlags & TL_FLAG_OVERWRITE_DETECTED &&
+				 g_sCtrl.ui32ByteCount > HTB_MARKER_PREDICTION_THRESHOLD(g_sCtrl.ui32BufferSize))
+		{
+			/* Take snapshot of global variables */
+			ui64OSTSSnap = g_sCtrl.ui64OSTS;
+			ui64CRTSSnap = g_sCtrl.ui64CRTS;
+			ui32ClkSpeedSnap = g_sCtrl.ui32ClkSpeed;
+			/* Reset global variable counter */
+			g_sCtrl.ui32ByteCount = 0;
+			OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+
+			/* Produce a repeat marker */
+			HTBSyncPartitionMarkerRepeat(g_sCtrl.ui32SyncMarker, ui64OSTSSnap, ui64CRTSSnap, ui32ClkSpeedSnap);
+		}
+		else
+		{
+			OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+		}
+	}
+
+ReturnError:
+	return eError;
+
+#else
+	/* HTB support is disabled. Just return PVRSRV_OK and do nothing. */
+	PVR_UNREFERENCED_PARAMETER(PID);
+	PVR_UNREFERENCED_PARAMETER(TID);
+	PVR_UNREFERENCED_PARAMETER(ui64TimeStamp);
+	PVR_UNREFERENCED_PARAMETER(SF);
+	PVR_UNREFERENCED_PARAMETER(args);
+	return PVRSRV_OK;
+#endif
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBLog
+ @Description   Record a Host Trace Buffer log event
+ @Input         PID     The PID of the process the event is associated
+                        with. This is provided as an argument rather
+                        than querying internally so that events
+                        associated with a particular process, but
+                        performed by another can be logged correctly.
+ @Input         ui64TimeStamp   The timestamp to be associated with this
+                                log event
+ @Input         SF              The log event ID
+ @Input         ...             Log parameters
+ @Return        PVRSRV_OK       Success.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+HTBLog(IMG_UINT32 PID, IMG_UINT32 TID, IMG_UINT64 ui64TimeStamp,
+		IMG_UINT32 SF, ...)
+{
+	PVRSRV_ERROR eError;
+
+	va_list args;
+	va_start(args, SF);
+	eError = HTBLogKM(PID, TID, ui64TimeStamp, SF, args);
+	va_end(args);
+	return eError;
+}
+
+static void
+_OnTLReaderOpenCallback( void *pvArg )
+{
+	if ( g_hTLStream )
+	{
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+		(void) HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+		              g_sCtrl.ui32SyncMarker,
+		              ((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)),
+		              ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+		              ((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)),
+		              ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+		              g_sCtrl.ui32SyncCalcClkSpd);
+	}
+
+	PVR_UNREFERENCED_PARAMETER(pvArg);
+}
+
+
+/*************************************************************************/ /*!
+ @Function      HTBControlKM
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control the behaviour of the data buffer
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBControlKM_Impl(
+	const IMG_UINT32 ui32NumFlagGroups,
+	const IMG_UINT32 * aui32GroupEnable,
+	const IMG_UINT32 ui32LogLevel,
+	const IMG_UINT32 ui32EnablePID,
+	const HTB_LOGMODE_CTRL eLogMode,
+	const HTB_OPMODE_CTRL eOpMode
+)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32RetryCount = HTB_LOG_RETRY_COUNT;
+	IMG_UINT32 i;
+	IMG_UINT64 ui64Time;
+	OSClockMonotonicns64(&ui64Time);
+
+	if ( !g_bConfigured && ui32NumFlagGroups )
+	{
+		eError = TLStreamCreate(
+				&g_hTLStream,
+				HTB_STREAM_NAME,
+				g_sCtrl.ui32BufferSize,
+				_LookupFlags(HTB_OPMODE_DROPOLDEST) | g_ui32TLBaseFlags,
+				_OnTLReaderOpenCallback, NULL, NULL, NULL, NULL, NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamCreate");
+		g_bConfigured = IMG_TRUE;
+	}
+
+	if (HTB_OPMODE_UNDEF != eOpMode && g_sCtrl.eOpMode != eOpMode)
+	{
+		g_sCtrl.eOpMode = eOpMode;
+		eError = TLStreamReconfigure(g_hTLStream, _LookupFlags(g_sCtrl.eOpMode | g_ui32TLBaseFlags));
+		while ( PVRSRV_ERROR_NOT_READY == eError && ui32RetryCount-- )
+		{
+			OSReleaseThreadQuanta();
+			eError = TLStreamReconfigure(g_hTLStream, _LookupFlags(g_sCtrl.eOpMode | g_ui32TLBaseFlags));
+		}
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamReconfigure");
+	}
+
+	if ( ui32EnablePID )
+	{
+		g_sCtrl.aui32EnablePID[g_sCtrl.ui32PIDHead] = ui32EnablePID;
+		g_sCtrl.ui32PIDHead++;
+		g_sCtrl.ui32PIDHead %= HTB_MAX_NUM_PID;
+		g_sCtrl.ui32PIDCount++;
+		if ( g_sCtrl.ui32PIDCount > HTB_MAX_NUM_PID )
+		{
+			g_sCtrl.ui32PIDCount = HTB_MAX_NUM_PID;
+		}
+	}
+
+	/* HTB_LOGMODE_ALLPID overrides ui32EnablePID */
+	if ( HTB_LOGMODE_ALLPID == eLogMode )
+	{
+		OSCachedMemSet(g_sCtrl.aui32EnablePID, 0, sizeof(g_sCtrl.aui32EnablePID));
+		g_sCtrl.ui32PIDCount = 0;
+		g_sCtrl.ui32PIDHead = 0;
+	}
+	if ( HTB_LOGMODE_UNDEF != eLogMode )
+	{
+		g_sCtrl.eLogMode = eLogMode;
+	}
+
+	if ( ui32NumFlagGroups )
+	{
+		for (i = 0; i < HTB_FLAG_NUM_EL && i < ui32NumFlagGroups; i++)
+		{
+			g_auiHTBGroupEnable[i] = aui32GroupEnable[i];
+		}
+		for (; i < HTB_FLAG_NUM_EL; i++)
+		{
+			g_auiHTBGroupEnable[i] = 0;
+		}
+	}
+
+	if ( ui32LogLevel )
+	{
+		g_sCtrl.ui32LogLevel = ui32LogLevel;
+	}
+
+	/* Dump the current configuration state */
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_OPMODE, g_sCtrl.eOpMode);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_ENABLE_GROUP, g_auiHTBGroupEnable[0]);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_LOG_LEVEL, g_sCtrl.ui32LogLevel);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_LOGMODE, g_sCtrl.eLogMode);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	for (i = 0; i < g_sCtrl.ui32PIDCount; i++)
+	{
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_ENABLE_PID, g_sCtrl.aui32EnablePID[i]);
+		PVR_LOG_IF_ERROR(eError, "HTBLog");
+	}
+	/* Else should never be hit as we set the spd when the power state is updated */
+	if (0 != g_sCtrl.ui32SyncMarker && 0 != g_sCtrl.ui32SyncCalcClkSpd)
+	{
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+				g_sCtrl.ui32SyncMarker,
+				((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+				((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+				g_sCtrl.ui32SyncCalcClkSpd);
+		PVR_LOG_IF_ERROR(eError, "HTBLog");
+	}
+
+	return eError;
+}
+
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarker
+ @Description   Write an HTB sync partition marker to the HTB log
+
+ @Input         ui33Marker      Marker value
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarker_Impl(
+	const IMG_UINT32 ui32Marker
+)
+{
+	g_sCtrl.ui32SyncMarker = ui32Marker;
+	if ( g_hTLStream )
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+
+		/* Else should never be hit as we set the spd when the power state is updated */
+		if (0 != g_sCtrl.ui32SyncCalcClkSpd)
+		{
+			eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+					ui32Marker,
+					((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+					((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+					g_sCtrl.ui32SyncCalcClkSpd);
+			PVR_WARN_IF_ERROR(eError, "HTBLog");
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarkerRepeat
+ @Description   Write a HTB sync partition marker to the HTB log, given
+                the previous values to repeat.
+
+ @Input         ui33Marker      Marker value
+ @Input         ui64SyncOSTS    previous OSTS
+ @Input         ui64SyncCRTS    previous CRTS
+ @Input         ui32ClkSpeed    previous Clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarkerRepeat_Impl(
+	const IMG_UINT32 ui32Marker,
+	const IMG_UINT64 ui64SyncOSTS,
+	const IMG_UINT64 ui64SyncCRTS,
+	const IMG_UINT32 ui32ClkSpeed
+)
+{
+	if ( g_hTLStream )
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+
+		/* Else should never be hit as we set the spd when the power state is updated */
+		if (0 != ui32ClkSpeed)
+		{
+			eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+					ui32Marker,
+					((IMG_UINT32)((ui64SyncOSTS>>32)&0xffffffffU)), ((IMG_UINT32)(ui64SyncOSTS&0xffffffffU)),
+					((IMG_UINT32)((ui64SyncCRTS>>32)&0xffffffffU)), ((IMG_UINT32)(ui64SyncCRTS&0xffffffffU)),
+					ui32ClkSpeed);
+			PVR_WARN_IF_ERROR(eError, "HTBLog");
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncScale
+ @Description   Write FW-Host synchronisation data to the HTB log when clocks
+                change or are re-calibrated
+
+ @Input         bLogValues      IMG_TRUE if value should be immediately written
+                                out to the log
+
+ @Input         ui32OSTS        OS Timestamp
+
+ @Input         ui32CRTS        Rogue timestamp
+
+ @Input         ui32CalcClkSpd  Calculated clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncScale_Impl(
+	const IMG_BOOL bLogValues,
+	const IMG_UINT64 ui64OSTS,
+	const IMG_UINT64 ui64CRTS,
+	const IMG_UINT32 ui32CalcClkSpd
+)
+{
+	g_sCtrl.ui64SyncOSTS = ui64OSTS;
+	g_sCtrl.ui64SyncCRTS = ui64CRTS;
+	g_sCtrl.ui32SyncCalcClkSpd = ui32CalcClkSpd;
+	if (g_hTLStream && bLogValues)
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+				g_sCtrl.ui32SyncMarker,
+				((IMG_UINT32)((ui64OSTS>>32)&0xffffffff)), ((IMG_UINT32)(ui64OSTS&0xffffffff)),
+				((IMG_UINT32)((ui64CRTS>>32)&0xffffffff)), ((IMG_UINT32)(ui64CRTS&0xffffffff)),
+				ui32CalcClkSpd);
+		/*
+		 * Don't spam the log with non-failure cases
+		 */
+		PVR_WARN_IF_ERROR(eError, "HTBLog");
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBLogSimple
+ @Description   Record a Host Trace Buffer log event with implicit PID and
+                Timestamp
+ @Input         SF              The log event ID
+ @Input         ...             Log parameters
+ @Return        PVRSRV_OK       Success.
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBLogSimple_Impl(IMG_UINT32 SF, ...)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64TimeStamp;
+	va_list args;
+	va_start(args, SF);
+	OSClockMonotonicns64(&ui64TimeStamp);
+	eError = HTBLogKM(OSGetCurrentProcessID(), OSGetCurrentThreadID(), ui64TimeStamp,
+			SF, args);
+	va_end(args);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBIsConfigured
+ @Description   Determine if HTB stream has been configured
+
+ @Input         none
+
+ @Return        IMG_FALSE       Stream has not been configured
+                IMG_TRUE        Stream has been configured
+
+*/ /**************************************************************************/
+IMG_BOOL
+HTBIsConfigured(void)
+{
+	return g_bConfigured;
+}
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/info_page_km.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/info_page_km.c
new file mode 100644
index 000000000000..79b03fe7e658
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/info_page_km.c
@@ -0,0 +1,142 @@
+/*************************************************************************/ /*!
+@File           info_page_km.c
+@Title          Kernel/User space shared memory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements general purpose shared memory between kernel driver
+                and user mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "info_page_defs.h"
+#include "info_page.h"
+#include "pvrsrv.h"
+#include "devicemem.h"
+#include "pmr.h"
+
+PVRSRV_ERROR InfoPageCreate(PVRSRV_DATA *psData)
+{
+    const PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+                                              PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+                                              PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+                                              PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL);
+    PVRSRV_ERROR eError;
+
+    PVR_ASSERT(psData != NULL);
+
+    /* Allocate single page of memory for driver information page */
+    eError = DevmemAllocateExportable(psData->psHostMemDeviceNode,
+                                      PVR_ALIGN(INFO_PAGE_TOTAL_SIZE, OSGetPageSize()),
+                                      OSGetPageSize(),
+                                      OSGetPageShift(),
+                                      uiMemFlags,
+                                      "PVRSRVInfoPage",
+                                      &psData->psInfoPageMemDesc);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+    eError =  DevmemAcquireCpuVirtAddr(psData->psInfoPageMemDesc,
+                                       (void **) &psData->pui32InfoPage);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+    /* Look-up the memory descriptor PMR handle */
+    eError = DevmemLocalGetImportHandle(psData->psInfoPageMemDesc,
+                                        (void **) &psData->psInfoPagePMR);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemLocalGetImportHandle", e0);
+
+    eError = OSLockCreate(&psData->hInfoPageLock);
+    PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+    /* Because the memory is allocated read only we need to explicitly set it to
+     * 0. The reason for this is that if the memory is allocated with
+     * ZERO_ON_ALLOC a WRITEABLE attribute is implicitly added to the flags (see
+     * DevmemValidateParams()). */
+    OSCachedMemSet(psData->pui32InfoPage, 0, OSGetPageSize());
+
+    return PVRSRV_OK;
+
+e0:
+    InfoPageDestroy(psData);
+    return eError;
+}
+
+void InfoPageDestroy(PVRSRV_DATA *psData)
+{
+    if (psData->psInfoPageMemDesc)
+    {
+        if (psData->pui32InfoPage != NULL)
+        {
+            DevmemReleaseCpuVirtAddr(psData->psInfoPageMemDesc);
+            psData->pui32InfoPage = NULL;
+        }
+
+        DevmemFree(psData->psInfoPageMemDesc);
+        psData->psInfoPageMemDesc = NULL;
+    }
+
+    if (psData->hInfoPageLock)
+    {
+        OSLockDestroy(psData->hInfoPageLock);
+        psData->hInfoPageLock = NULL;
+    }
+}
+
+PVRSRV_ERROR PVRSRVAcquireInfoPageKM(PMR **ppsPMR)
+{
+    PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+
+    PVR_LOG_RETURN_IF_FALSE(psData->psInfoPageMemDesc != NULL, "invalid MEMDESC"
+                      " handle", PVRSRV_ERROR_INVALID_PARAMS);
+    PVR_LOG_RETURN_IF_FALSE(psData->psInfoPagePMR != NULL, "invalid PMR handle",
+                      PVRSRV_ERROR_INVALID_PARAMS);
+
+    /* Copy the PMR import handle back */
+    *ppsPMR = psData->psInfoPagePMR;
+
+    /* Mark the PMR such that no layout changes can happen
+     * This is a fixed layout created during early stages of
+     * driver loading and shouldn't change later */
+    PMR_SetLayoutFixed(psData->psInfoPagePMR, IMG_TRUE);
+
+    return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVReleaseInfoPageKM(PMR *ppsPMR)
+{
+    /* Nothing to do here as PMR is singleton */
+    PVR_UNREFERENCED_PARAMETER(ppsPMR);
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/lists.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/lists.c
new file mode 100644
index 000000000000..e8e7088a3296
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/lists.c
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linked list shared functions implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implementation of the list iterators for types shared among
+                more than one file in the services code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "lists.h"
+
+/*===================================================================
+  LIST ITERATOR FUNCTIONS USED IN MORE THAN ONE FILE (those used just
+  once are implemented locally).
+  ===================================================================*/
+
+IMPLEMENT_LIST_ANY(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_ANY_2(PVRSRV_DEVICE_NODE, IMG_BOOL, IMG_FALSE)
+IMPLEMENT_LIST_ANY_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK)
+IMPLEMENT_LIST_ANY_VA(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_ANY_VA_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK)
+IMPLEMENT_LIST_FOR_EACH(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_FOR_EACH_VA(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_INSERT_TAIL(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_REMOVE(PVRSRV_DEVICE_NODE)
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/mmu_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/mmu_common.c
new file mode 100644
index 000000000000..c9c5114b2406
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/mmu_common.c
@@ -0,0 +1,5341 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "dllist.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "htbserver.h"
+#include "pvr_ricommon.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+# include "process_stats.h"
+# include "proc_stats.h"
+#endif
+
+#if defined(PDUMP)
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#endif
+
+#include "physmem.h"
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "physmem_lma.h"
+#endif
+
+/*
+Major Interfaces to other modules:
+
+Let's keep this graph up-to-date:
+
+   +-----------+
+   | devicemem |
+   +-----------+
+         |
+   +============+
+   | mmu_common |
+   +============+
+         |
+         +-----------------+
+         |                 |
+    +---------+      +----------+
+    |   pmr   |      |  device  |
+    +---------+      +----------+
+ */
+
+#include "mmu_common.h"
+#include "pmr.h"
+#include "devicemem_server_utils.h"
+
+/* #define MMU_OBJECT_REFCOUNT_DEBUGING 1 */
+#if defined(MMU_OBJECT_REFCOUNT_DEBUGING)
+#define MMU_OBJ_DBG(x)	PVR_DPF(x)
+#else
+#define MMU_OBJ_DBG(x)
+#endif
+
+#define SCRATCH_PAGE 1
+#define DEV_ZERO_PAGE 2
+#if defined(PDUMP)
+#define SCRATCH_PAGE_STR "SCRATCH_PAGE"
+#define DEV_ZERO_PAGE_STR "DEV_ZERO_PAGE"
+#endif
+#define PVR_SCRATCH_PAGE_INIT_VALUE 0
+#define PVR_ZERO_PAGE_INIT_VALUE  0
+
+/*!
+ * Refcounted structure that is shared between the context and
+ * the cleanup thread items.
+ * It is used to keep track of all cleanup items and whether the creating
+ * MMU context has been destroyed and therefore is not allowed to be
+ * accessed any more.
+ *
+ * The cleanup thread is used to defer the freeing of the page tables
+ * because we have to make sure that the MMU cache has been invalidated.
+ * If we don't take care of this the MMU might partially access cached
+ * and uncached tables which might lead to inconsistencies and in the
+ * worst case to MMU pending faults on random memory.
+ */
+typedef struct _MMU_CTX_CLEANUP_DATA_
+{
+	/*! Refcount to know when this structure can be destroyed */
+	ATOMIC_T iRef;
+	/*! Protect items in this structure, especially the refcount */
+	POS_LOCK hCleanupLock;
+	/*! List of all cleanup items currently in flight */
+	DLLIST_NODE sMMUCtxCleanupItemsHead;
+	/*! Was the MMU context destroyed and should not be accessed any more? */
+	IMG_BOOL bMMUContextExists;
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/*! Associated OSid for this context */
+	IMG_UINT32 ui32OSid;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+} MMU_CTX_CLEANUP_DATA;
+
+
+/*!
+ * Structure holding one or more page tables that need to be
+ * freed after the MMU cache has been flushed which is signalled when
+ * the stored sync has a value that is <= the required value.
+ */
+typedef struct _MMU_CLEANUP_ITEM_
+{
+	/*! Cleanup thread data */
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+	/*! List to hold all the MMU_MEMORY_MAPPINGs, i.e. page tables */
+	DLLIST_NODE sMMUMappingHead;
+	/*! Node of the cleanup item list for the context */
+	DLLIST_NODE sMMUCtxCleanupItem;
+	/* Pointer to the cleanup meta data */
+	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData;
+	/* Sync to query if the MMU cache was flushed */
+	PVRSRV_CLIENT_SYNC_PRIM *psSync;
+	/*! The update value of the sync to signal that the cache was flushed */
+	IMG_UINT32 uiRequiredSyncVal;
+	/*! The update value of the power off counter */
+	IMG_UINT32 uiRequiredPowerOffCounter;
+	/*! The device node needed to free the page tables */
+	PVRSRV_DEVICE_NODE *psDevNode;
+} MMU_CLEANUP_ITEM;
+
+/*!
+	All physical allocations and frees are relative to this context, so
+	we would get all the allocations of PCs, PDs, and PTs from the same
+	RA.
+
+	We have one per MMU context in case we have mixed UMA/LMA devices
+	within the same system.
+ */
+typedef struct _MMU_PHYSMEM_CONTEXT_
+{
+	/*! Associated MMU_CONTEXT */
+	struct _MMU_CONTEXT_ *psMMUContext;
+
+	/*! Parent device node */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/*! Refcount so we know when to free up the arena */
+	IMG_UINT32 uiNumAllocations;
+
+	/*! Arena from which physical memory is derived */
+	RA_ARENA *psPhysMemRA;
+	/*! Arena name */
+	IMG_CHAR *pszPhysMemRAName;
+	/*! Size of arena name string */
+	size_t uiPhysMemRANameAllocSize;
+
+	/*! Meta data for deferred cleanup */
+	MMU_CTX_CLEANUP_DATA *psCleanupData;
+	/*! Temporary list of all deferred MMU_MEMORY_MAPPINGs. */
+	DLLIST_NODE sTmpMMUMappingHead;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	IMG_UINT32 ui32OSid;
+	IMG_UINT32 ui32OSidReg;
+	IMG_BOOL   bOSidAxiProt;
+#endif
+
+} MMU_PHYSMEM_CONTEXT;
+
+/*!
+	Mapping structure for MMU memory allocation
+ */
+typedef struct _MMU_MEMORY_MAPPING_
+{
+	/*! Physmem context to allocate from */
+	MMU_PHYSMEM_CONTEXT		*psContext;
+	/*! OS/system Handle for this allocation */
+	PG_HANDLE				sMemHandle;
+	/*! CPU virtual address of this allocation */
+	void					*pvCpuVAddr;
+	/*! Device physical address of this allocation */
+	IMG_DEV_PHYADDR			sDevPAddr;
+	/*! Size of this allocation */
+	size_t					uiSize;
+	/*! Number of current mappings of this allocation */
+	IMG_UINT32				uiCpuVAddrRefCount;
+	/*! Node for the defer free list */
+	DLLIST_NODE				sMMUMappingItem;
+} MMU_MEMORY_MAPPING;
+
+/*!
+	Memory descriptor for MMU objects. There can be more than one memory
+	descriptor per MMU memory allocation.
+ */
+typedef struct _MMU_MEMORY_DESC_
+{
+	/* NB: bValid is set if this descriptor describes physical
+	   memory.  This allows "empty" descriptors to exist, such that we
+	   can allocate them in batches. */
+	/*! Does this MMU object have physical backing */
+	IMG_BOOL				bValid;
+	/*! Device Physical address of physical backing */
+	IMG_DEV_PHYADDR			sDevPAddr;
+	/*! CPU virtual address of physical backing */
+	void					*pvCpuVAddr;
+	/*! Mapping data for this MMU object */
+	MMU_MEMORY_MAPPING		*psMapping;
+	/*! Memdesc offset into the psMapping */
+	IMG_UINT32 uiOffset;
+	/*! Size of the Memdesc */
+	IMG_UINT32 uiSize;
+} MMU_MEMORY_DESC;
+
+/*!
+	MMU levelx structure. This is generic and is used
+	for all levels (PC, PD, PT).
+ */
+typedef struct _MMU_Levelx_INFO_
+{
+	/*! The Number of entries in this level */
+	IMG_UINT32 ui32NumOfEntries;
+
+	/*! Number of times this level has been reference. Note: For Level1 (PTE)
+	    we still take/drop the reference when setting up the page tables rather
+	    then at map/unmap time as this simplifies things */
+	IMG_UINT32 ui32RefCount;
+
+	/*! MemDesc for this level */
+	MMU_MEMORY_DESC sMemDesc;
+
+	/*! Array of infos for the next level. Must be last member in structure */
+	struct _MMU_Levelx_INFO_ *apsNextLevel[IMG_FLEX_ARRAY_MEMBER];
+} MMU_Levelx_INFO;
+
+/*!
+	MMU context structure
+ */
+struct _MMU_CONTEXT_
+{
+	/*! Originating Connection */
+	CONNECTION_DATA *psConnection;
+
+	/*! Device MMU attribute descriptions */
+	MMU_DEVICEATTRIBS *psDevAttrs;
+
+	/*! For allocation and deallocation of the physical memory where
+	    the pagetables live */
+	struct _MMU_PHYSMEM_CONTEXT_ *psPhysMemCtx;
+
+#if defined(PDUMP)
+	/*! PDump context ID (required for PDump commands with virtual addresses) */
+	IMG_UINT32 uiPDumpContextID;
+
+	/*! The refcount of the PDump context ID */
+	IMG_UINT32 ui32PDumpContextIDRefCount;
+#endif
+
+	/*! MMU cache invalidation flags. For Rogue, only one set of flags are
+	 * maintained (in the KernelMMUCtx) and used for all context page table
+	 * updates, while in Volcanic each context maintains its own flags
+	 * for the PTs it contains. This is due to the different memory hierarchy
+	 * designs in use between the architectures, See SLC_VIVT feature. */
+	ATOMIC_T sCacheFlags;
+
+	/*! Lock to ensure exclusive access when manipulating the MMU context or
+	 * reading and using its content
+	 */
+	POS_LOCK hLock;
+
+	/*! Base level info structure. Must be last member in structure */
+	MMU_Levelx_INFO sBaseLevelInfo;
+	/* NO OTHER MEMBERS AFTER THIS STRUCTURE ! */
+};
+
+/*
+ * Only the kernel created (direct bridge) firmware memory context will
+ * have a NULL connection and all other application memory context get a
+ * valid connection object.
+ */
+#define _MMU_IS_FWKM_CTX(_ctx) ((_ctx)->psConnection == NULL)
+
+/*
+ * Used to determine if the MMU Ctx provided is the FWKM memory context
+ * and if it belongs to the VZ Guest.
+ */
+#define _MMU_IS_FWKM_CTX_VZGUEST(_ctx) (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, _ctx->psPhysMemCtx->psDevNode) && _MMU_IS_FWKM_CTX(_ctx))
+
+static const IMG_DEV_PHYADDR gsBadDevPhyAddr = {MMU_BAD_PHYS_ADDR};
+
+#if defined(DEBUG)
+#include "log2.h"
+#endif
+
+
+static PVRSRV_ERROR
+MMU_UnmapPagesUnlocked(MMU_CONTEXT *psMMUContext,
+                       PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                       IMG_DEV_VIRTADDR sDevVAddrBase,
+                       IMG_UINT32 ui32PageCount,
+                       IMG_UINT32 *pai32FreeIndices,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags);
+
+static PVRSRV_ERROR
+MMU_UnmapPMRFastUnlocked(MMU_CONTEXT *psMMUContext,
+                         IMG_DEV_VIRTADDR sDevVAddrBase,
+                         IMG_UINT32 ui32PageCount,
+                         IMG_UINT32 uiLog2PageSize);
+
+/*****************************************************************************
+ *                          Utility functions                                *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _FreeMMUMapping
+
+@Description    Free a given dllist of MMU_MEMORY_MAPPINGs and the page tables
+                they represent.
+
+@Input          psDevNode           Device node
+
+@Input          psTmpMMUMappingHead List of MMU_MEMORY_MAPPINGs to free
+ */
+/*****************************************************************************/
+static void
+_FreeMMUMapping(PVRSRV_DEVICE_NODE *psDevNode,
+                PDLLIST_NODE psTmpMMUMappingHead)
+{
+	PDLLIST_NODE psNode, psNextNode;
+
+	/* Free the current list unconditionally */
+	dllist_foreach_node(psTmpMMUMappingHead,
+	                    psNode,
+	                    psNextNode)
+	{
+		MMU_MEMORY_MAPPING *psMapping = IMG_CONTAINER_OF(psNode,
+		                                                 MMU_MEMORY_MAPPING,
+		                                                 sMMUMappingItem);
+
+		PhysHeapPagesFree(psDevNode->psMMUPhysHeap, &psMapping->sMemHandle);
+		dllist_remove_node(psNode);
+		OSFreeMem(psMapping);
+	}
+}
+
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+/*************************************************************************/ /*!
+@Function       _CleanupThread_FreeMMUMapping
+
+@Description    Function to be executed by the cleanup thread to free
+                MMU_MEMORY_MAPPINGs after the MMU cache has been invalidated.
+
+                This function will request a MMU cache invalidate once and
+                retry to free the MMU_MEMORY_MAPPINGs until the invalidate
+                has been executed.
+
+                If the memory context that created this cleanup item has been
+                destroyed in the meantime this function will directly free the
+                MMU_MEMORY_MAPPINGs without waiting for any MMU cache
+                invalidation.
+
+@Input          pvData           Cleanup data in form of a MMU_CLEANUP_ITEM
+
+@Return         PVRSRV_OK if successful otherwise PVRSRV_ERROR_RETRY
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR
+_CleanupThread_FreeMMUMapping(void* pvData)
+{
+	PVRSRV_ERROR eError;
+	MMU_CLEANUP_ITEM *psCleanup = (MMU_CLEANUP_ITEM *)pvData;
+	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData = psCleanup->psMMUCtxCleanupData;
+	PVRSRV_DEVICE_NODE *psDevNode = psCleanup->psDevNode;
+	IMG_BOOL bFreeNow;
+
+	OSLockAcquire(psMMUCtxCleanupData->hCleanupLock);
+
+	/* Don't attempt to free anything when the context has been destroyed.
+	 * Especially don't access any device specific structures any more!*/
+	if (!psMMUCtxCleanupData->bMMUContextExists)
+	{
+		OSFreeMem(psCleanup);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_OK, e0);
+	}
+
+	/* Has there been a power off since the cleanup item was queued? */
+	bFreeNow = PVRSRVHasCounter32Advanced(psDevNode->uiPowerOffCounter,
+	                                      psCleanup->uiRequiredPowerOffCounter);
+	if (bFreeNow)
+	{
+		goto freeNow;
+	}
+
+	if (psCleanup->psSync == NULL)
+	{
+		IMG_UINT32 uiSyncVal;
+		/* Kick to invalidate the MMU caches and get sync info */
+		eError = psDevNode->pfnMMUCacheInvalidateKick(psDevNode,
+		                                              &uiSyncVal);
+		if (eError != PVRSRV_OK)
+		{
+			OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+			return PVRSRV_ERROR_RETRY;
+		}
+		psCleanup->psSync = psDevNode->psMMUCacheSyncPrim;
+		/* If we have no sync reference we treat this as a simple FREE.
+		 * We cannot retry as there will never be a sync-prim to kick / invalidate.
+		 */
+		if (psCleanup->psSync == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device %u has no MMUCacheSyncPrim",
+			         __func__, psDevNode->sDevId.ui32InternalID));
+			bFreeNow = IMG_TRUE;
+			goto freeNow;
+		}
+	}
+
+	/* Has the invalidate executed (sync is updated when the Firmware performs
+	 * the invalidate)? */
+	bFreeNow = PVRSRVHasCounter32Advanced(OSReadDeviceMem32(psCleanup->psSync->pui32LinAddr),
+	                                      psCleanup->uiRequiredSyncVal);
+
+#if defined(NO_HARDWARE)
+	/* In NOHW the syncs will never be updated so just free the tables */
+	bFreeNow = IMG_TRUE;
+#endif
+	/* If the Invalidate operation is not completed, check if the operation timed out */
+	if (!bFreeNow)
+	{
+		IMG_UINT32 uiTimeStart = psCleanup->sCleanupThreadFn.ui32TimeStart;
+		IMG_UINT32 uiTimeEnd = psCleanup->sCleanupThreadFn.ui32TimeEnd;
+
+		/* If the time left for the completion of invalidate operation is
+		 * within 500ms of time-out, consider the operation as timed out */
+		if ((uiTimeEnd - uiTimeStart - 500) <= (OSClockms() - uiTimeStart))
+		{
+			/* Consider the operation is timed out */
+			bFreeNow = IMG_TRUE;
+		}
+	}
+
+freeNow:
+	/* Free if the invalidate operation completed or the operation itself timed out */
+	if (bFreeNow)
+	{
+		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);
+
+		dllist_remove_node(&psCleanup->sMMUCtxCleanupItem);
+
+		OSFreeMem(psCleanup);
+
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_RETRY;
+	}
+
+e0:
+
+	/* If this cleanup task has been successfully executed we can
+	 * decrease the context cleanup data refcount. Successfully
+	 * means here that the MMU_MEMORY_MAPPINGs have been freed by
+	 * either this cleanup task of when the MMU context has been
+	 * destroyed. */
+	if (eError == PVRSRV_OK)
+	{
+		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+
+		if (OSAtomicDecrement(&psMMUCtxCleanupData->iRef) == 0)
+		{
+			OSLockDestroy(psMMUCtxCleanupData->hCleanupLock);
+			OSFreeMem(psMMUCtxCleanupData);
+		}
+	}
+	else
+	{
+		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+	}
+
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _SetupCleanup_FreeMMUMapping
+
+@Description    Setup a cleanup item for the cleanup thread that will
+                kick off a MMU invalidate request and free the associated
+                MMU_MEMORY_MAPPINGs when the invalidate was successful.
+
+@Input          psPhysMemCtx        The current MMU physmem context
+ */
+/*****************************************************************************/
+static void
+_SetupCleanup_FreeMMUMapping(MMU_PHYSMEM_CONTEXT *psPhysMemCtx)
+{
+
+	MMU_CLEANUP_ITEM *psCleanupItem;
+	MMU_CTX_CLEANUP_DATA *psCleanupData = psPhysMemCtx->psCleanupData;
+	PVRSRV_DEVICE_NODE *psDevNode = psPhysMemCtx->psDevNode;
+
+	if (dllist_is_empty(&psPhysMemCtx->sTmpMMUMappingHead))
+	{
+		goto e0;
+	}
+
+#if defined(PDUMP)
+	/* Free the page tables immediately in case of pdump, which avoids
+	 * changing script files due to the additional invalidation kick */
+	goto e1;
+#endif
+
+	/* Don't defer the freeing if we are currently unloading the driver
+	 * or if the sync has been destroyed */
+	if (PVRSRVGetPVRSRVData()->bUnload ||
+			psDevNode->psMMUCacheSyncPrim == NULL)
+	{
+		goto e1;
+	}
+
+	/* Allocate a cleanup item */
+	psCleanupItem = OSAllocMem(sizeof(*psCleanupItem));
+	if (!psCleanupItem)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to get memory for deferred page table cleanup. "
+				"Freeing tables immediately",
+				__func__));
+		goto e1;
+	}
+
+	/* Set sync to NULL to indicate we did not interact with
+	 * the FW yet. Kicking off an MMU cache invalidate should
+	 * be done in the cleanup thread to not waste time here. */
+	psCleanupItem->psSync = NULL;
+	psCleanupItem->uiRequiredSyncVal = psDevNode->ui32NextMMUInvalidateUpdate;
+	psCleanupItem->uiRequiredPowerOffCounter = psDevNode->uiPowerOffCounterNext;
+	psCleanupItem->psDevNode = psDevNode;
+	psCleanupItem->psMMUCtxCleanupData = psCleanupData;
+
+	OSAtomicIncrement(&psCleanupData->iRef);
+
+	/* Move the page tables to free to the cleanup item */
+	dllist_replace_head(&psPhysMemCtx->sTmpMMUMappingHead,
+	                    &psCleanupItem->sMMUMappingHead);
+
+	/* Add the cleanup item itself to the context list */
+	dllist_add_to_tail(&psCleanupData->sMMUCtxCleanupItemsHead,
+	                   &psCleanupItem->sMMUCtxCleanupItem);
+
+	/* Setup the cleanup thread data and add the work item */
+	psCleanupItem->sCleanupThreadFn.pfnFree = _CleanupThread_FreeMMUMapping;
+	psCleanupItem->sCleanupThreadFn.pvData = psCleanupItem;
+	psCleanupItem->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+	psCleanupItem->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_MMU;
+	CLEANUP_THREAD_SET_RETRY_TIMEOUT(&psCleanupItem->sCleanupThreadFn,
+	                                 CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT);
+
+	PVRSRVCleanupThreadAddWork(psDevNode, &psCleanupItem->sCleanupThreadFn);
+
+	return;
+
+e1:
+	/* Free the page tables now */
+	_FreeMMUMapping(psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+e0:
+	return;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _CalcPCEIdx
+
+@Description    Calculate the page catalogue index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page catalogue index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPCEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPCIndexMask)
+			>> psDevVAddrConfig->uiPCIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _CalcPDEIdx
+
+@Description    Calculate the page directory index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page directory index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPDEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPDIndexMask)
+			>> psDevVAddrConfig->uiPDIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _CalcPTEIdx
+
+@Description    Calculate the page entry index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page entry index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPTEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+	sTmpDevVAddr.uiAddr -= psDevVAddrConfig->uiOffsetInBytes;
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPTIndexMask)
+			>> psDevVAddrConfig->uiPTIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+/*************************************************************************/ /*!
+@Function       _GetParityBit
+
+@Description    Calculate parity bit
+
+@Input          uiSrc           Value to calculate the parity bit from.
+
+@Return         The parity bit
+ */
+/*****************************************************************************/
+static inline IMG_UINT64 _GetParityBit(IMG_UINT64 uiSrc)
+{
+	uiSrc ^= uiSrc >> 32U;
+	uiSrc ^= uiSrc >> 16U;
+	uiSrc ^= uiSrc >> 8U;
+	uiSrc ^= uiSrc >> 4U;
+	uiSrc ^= uiSrc >> 2U;
+	uiSrc ^= uiSrc >> 1U;
+
+	return (uiSrc & 1U);
+}
+
+/*****************************************************************************
+ *         MMU memory allocation/management functions (mem desc)             *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMem_RAImportAlloc
+
+@Description    Imports MMU Px memory into the RA. This is where the
+                actual allocation of physical memory happens.
+
+@Input          hArenaHandle    Handle that was passed in during the
+                                creation of the RA
+
+@Input          uiSize          Size of the memory to import
+
+@Input          uiFlags         Flags that where passed in the allocation.
+
+@Input          uBaseAlignment  Alignment for the base returned, not used
+                                in this context.
+
+@Output         puiBase         The address of where to insert this import
+
+@Output         puiActualSize   The actual size of the import
+
+@Output         phPriv          Handle which will be passed back when
+                                this import is freed
+
+@Return         PVRSRV_OK if import alloc was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _MMU_PhysMem_RAImportAlloc(RA_PERARENA_HANDLE hArenaHandle,
+                                               RA_LENGTH_T uiSize,
+                                               RA_FLAGS_T uiFlags,
+                                               RA_LENGTH_T uBaseAlignment,
+                                               const IMG_CHAR *pszAnnotation,
+                                               RA_IMPORT *psImport)
+{
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx = (MMU_PHYSMEM_CONTEXT *)hArenaHandle;
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psPhysMemCtx->psDevNode;
+	MMU_MEMORY_MAPPING *psMapping;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPid = 0;
+
+	PVR_UNREFERENCED_PARAMETER(pszAnnotation);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	PVR_ASSERT(psDevNode != NULL);
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode, eError, e0);
+
+	psMapping = OSAllocMem(sizeof(MMU_MEMORY_MAPPING));
+	PVR_GOTO_IF_NOMEM(psMapping, eError, e0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	uiPid = psDevNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ?
+	        PVR_SYS_ALLOC_PID : OSGetCurrentClientProcessIDKM();
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/*
+	 * Store the OSid in the PG_HANDLE.uiOSid field for use by the
+	 * pfnDevPxFree() routine.
+	 */
+	psMapping->sMemHandle.uiOSid = psPhysMemCtx->ui32OSid;
+	eError = PhysHeapPagesAllocGPV(psDevNode->psMMUPhysHeap,
+	                               TRUNCATE_64BITS_TO_SIZE_T(uiSize),
+	                               &psMapping->sMemHandle,
+	                               &psMapping->sDevPAddr,
+	                               psPhysMemCtx->ui32OSid,
+	                               uiPid);
+#else
+	eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+	                            TRUNCATE_64BITS_TO_SIZE_T(uiSize),
+	                            &psMapping->sMemHandle,
+	                            &psMapping->sDevPAddr,
+	                            uiPid);
+#endif
+	if (eError != PVRSRV_OK)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		PVRSRVStatsUpdateOOMStat(NULL, psDevNode, PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+					  OSGetCurrentClientProcessIDKM());
+#endif
+		goto e1;
+	}
+
+	psMapping->psContext = psPhysMemCtx;
+	psMapping->uiSize = TRUNCATE_64BITS_TO_SIZE_T(uiSize);
+
+	psMapping->uiCpuVAddrRefCount = 0;
+
+	psImport->hPriv = (RA_PERISPAN_HANDLE) psMapping;
+
+	/* Note: This assumes this memory never gets paged out */
+	psImport->base = (RA_BASE_T)psMapping->sDevPAddr.uiAddr;
+	psImport->uSize = uiSize;
+
+	return PVRSRV_OK;
+
+e1:
+	OSFreeMem(psMapping);
+e0:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMem_RAImportFree
+
+@Description    Imports MMU Px memory into the RA. This is where the
+                actual free of physical memory happens.
+
+@Input          hArenaHandle    Handle that was passed in during the
+                                creation of the RA
+
+@Input          puiBase         The address of where to insert this import
+
+@Output         phPriv          Private data that the import alloc provided
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _MMU_PhysMem_RAImportFree(RA_PERARENA_HANDLE hArenaHandle,
+                                      RA_BASE_T uiBase,
+                                      RA_PERISPAN_HANDLE hPriv)
+{
+	MMU_MEMORY_MAPPING *psMapping = (MMU_MEMORY_MAPPING *)hPriv;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx = (MMU_PHYSMEM_CONTEXT *)hArenaHandle;
+
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+
+	/* Check we have dropped all CPU mappings */
+	PVR_ASSERT(psMapping->uiCpuVAddrRefCount == 0);
+
+	/* Add mapping to defer free list */
+	psMapping->psContext = NULL;
+	dllist_add_to_tail(&psPhysMemCtx->sTmpMMUMappingHead, &psMapping->sMMUMappingItem);
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMemAlloc
+
+@Description    Allocates physical memory for MMU objects
+
+@Input          psPhysMemCtx    Physmem context to do the allocation from
+
+@Output         psMemDesc       Allocation description
+
+@Input          uiBytes         Size of the allocation in bytes
+
+@Input          uiAlignment     Alignment requirement of this allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+
+static PVRSRV_ERROR _MMU_PhysMemAlloc(MMU_PHYSMEM_CONTEXT *psPhysMemCtx,
+                                      MMU_MEMORY_DESC *psMemDesc,
+                                      size_t uiBytes,
+                                      size_t uiAlignment)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiPhysAddr;
+
+	PVR_RETURN_IF_INVALID_PARAM(psMemDesc);
+	PVR_RETURN_IF_INVALID_PARAM(!psMemDesc->bValid);
+
+	eError = RA_Alloc(psPhysMemCtx->psPhysMemRA,
+	                  uiBytes,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0, /* flags */
+	                  uiAlignment,
+	                  "",
+	                  &uiPhysAddr,
+	                  NULL,
+	                  (RA_PERISPAN_HANDLE *)&psMemDesc->psMapping);
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "RA_Alloc");
+
+	psMemDesc->bValid = IMG_TRUE;
+	psMemDesc->pvCpuVAddr = NULL;
+	psMemDesc->sDevPAddr.uiAddr = (IMG_UINT64) uiPhysAddr;
+
+	if (psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
+	{
+		eError = PhysHeapPagesMap(psPhysMemCtx->psDevNode->psMMUPhysHeap,
+		                          &psMemDesc->psMapping->sMemHandle,
+		                          psMemDesc->psMapping->uiSize,
+		                          &psMemDesc->psMapping->sDevPAddr,
+		                          &psMemDesc->psMapping->pvCpuVAddr);
+		if (eError != PVRSRV_OK)
+		{
+			RA_Free(psPhysMemCtx->psPhysMemRA, psMemDesc->sDevPAddr.uiAddr);
+			return eError;
+		}
+	}
+
+	psMemDesc->psMapping->uiCpuVAddrRefCount++;
+	psMemDesc->uiOffset = (psMemDesc->sDevPAddr.uiAddr - psMemDesc->psMapping->sDevPAddr.uiAddr);
+	psMemDesc->pvCpuVAddr = (IMG_UINT8 *)psMemDesc->psMapping->pvCpuVAddr + psMemDesc->uiOffset;
+	psMemDesc->uiSize = uiBytes;
+	PVR_ASSERT(psMemDesc->pvCpuVAddr != NULL);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMemFree
+
+@Description    Allocates physical memory for MMU objects
+
+@Input          psPhysMemCtx    Physmem context to do the free on
+
+@Input          psMemDesc       Allocation description
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _MMU_PhysMemFree(MMU_PHYSMEM_CONTEXT *psPhysMemCtx,
+                             MMU_MEMORY_DESC *psMemDesc)
+{
+	RA_BASE_T uiPhysAddr;
+
+	PVR_ASSERT(psMemDesc->bValid);
+
+	if (--psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
+	{
+		PhysHeapPagesUnMap(psPhysMemCtx->psDevNode->psMMUPhysHeap,
+		                   &psMemDesc->psMapping->sMemHandle,
+		                   psMemDesc->psMapping->pvCpuVAddr);
+	}
+
+	psMemDesc->pvCpuVAddr = NULL;
+
+	uiPhysAddr = psMemDesc->sDevPAddr.uiAddr;
+	RA_Free(psPhysMemCtx->psPhysMemRA, uiPhysAddr);
+
+	psMemDesc->bValid = IMG_FALSE;
+}
+
+
+/*****************************************************************************
+ *              MMU object allocation/management functions                   *
+ *****************************************************************************/
+
+static INLINE PVRSRV_ERROR _MMU_ConvertDevMemFlags(IMG_BOOL bInvalidate,
+                                                   PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                                                   MMU_PROTFLAGS_T *puiMMUProtFlags,
+                                                   MMU_CONTEXT *psMMUContext)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiGPUCacheMode;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Do flag conversion between devmem flags and MMU generic flags */
+	if (bInvalidate == IMG_TRUE)
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_INVALID;
+		return eError; /* OK */
+	}
+
+	/* Convert to valid flags for valid mapping */
+	*puiMMUProtFlags |= ((uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK)
+			>> PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET)
+			<< MMU_PROTFLAGS_DEVICE_OFFSET;
+
+	if (PVRSRV_CHECK_GPU_READABLE(uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+	}
+	if (PVRSRV_CHECK_GPU_WRITEABLE(uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_WRITEABLE;
+	}
+
+	eError = DevmemDeviceCacheMode(uiMappingFlags, &uiGPUCacheMode);
+	PVR_RETURN_IF_ERROR(eError);
+
+	switch (uiGPUCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED:
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC:
+			break;
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHED:
+			*puiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Wrong parameters",
+					__func__));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (DevmemDeviceCacheCoherency(psDevNode, uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_CACHE_COHERENT;
+	}
+
+	if (psDevNode->pfnMMUTweakProtFlags)
+	{
+		psDevNode->pfnMMUTweakProtFlags(psDevNode, psMMUContext->psDevAttrs, uiMappingFlags, puiMMUProtFlags);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _PxMemAlloc
+
+@Description    Allocates physical memory for MMU objects, initialises
+                and PDumps it.
+
+@Input          psMMUContext    MMU context
+
+@Input          uiNumEntries    Number of entries to allocate
+
+@Input          psConfig        MMU Px config
+
+@Input          eMMULevel       MMU level that allocation is for
+
+@Output         psMemDesc       Description of allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _PxMemAlloc(MMU_CONTEXT *psMMUContext,
+                                IMG_UINT32 uiNumEntries,
+                                const MMU_PxE_CONFIG *psConfig,
+                                MMU_LEVEL eMMULevel,
+                                MMU_MEMORY_DESC *psMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+                                IMG_DEV_VIRTADDR* psRunningDevVAddr,
+                                IMG_UINT32 uiLog2DataPageSize,
+#endif
+                                IMG_UINT32 uiLog2Align)
+{
+	PVRSRV_ERROR eError;
+	size_t uiBytes;
+	size_t uiAlign;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	PVR_ASSERT(psConfig->uiBytesPerEntry != 0);
+
+	uiBytes = uiNumEntries * psConfig->uiBytesPerEntry;
+	/* We need here the alignment of the previous level because that is the entry for we generate here */
+	uiAlign = 1 << uiLog2Align;
+
+	/*
+	 * If the hardware specifies an alignment requirement for a page table then
+	 * it also requires that all memory up to the next aligned address is
+	 * zeroed.
+	 *
+	 * Failing to do this can result in uninitialised data outside of the actual
+	 * page table range being read by the MMU and treated as valid, e.g. the
+	 * pending flag.
+	 *
+	 * Typically this will affect 1MiB, 2MiB PT pages which have a size of 16
+	 * and 8 bytes respectively but an alignment requirement of 64 bytes each.
+	 */
+	uiBytes = PVR_ALIGN(uiBytes, uiAlign);
+
+	/* allocate the object */
+	eError = _MMU_PhysMemAlloc(psMMUContext->psPhysMemCtx,
+	                           psMemDesc, uiBytes, uiAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_MMU_PhysMemAlloc", eError, PVRSRV_ERROR_OUT_OF_MEMORY, e0);
+	}
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	/* If parity bit is needed, set it accordingly to avoid parity errors along with page-faults. */
+	if (psConfig->uiParityBitMask && eMMULevel == MMU_LEVEL_1 && psRunningDevVAddr != NULL)
+	{
+		IMG_UINT32 uiParity = (IMG_UINT32)_GetParityBit(psRunningDevVAddr->uiAddr ^ 0ULL);
+
+		/* Only calculate parity for first address in the PT.
+		   This tells us which of the two patterns to use for the rest of the PT. */
+		OSCachedMemCopy(psMemDesc->pvCpuVAddr, psMMUContext->psDevAttrs->pui64PrecomputedAllocParity[uiParity], uiBytes);
+
+		/* Increment running dev virtual address */
+		psRunningDevVAddr->uiAddr += (1 << uiLog2DataPageSize) * uiNumEntries;
+	}
+	else
+#endif
+	{
+		/*
+			Clear the object
+			Note: if any MMUs are cleared with non-zero values then will need a
+			custom clear function
+			Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
+			unlikely
+		*/
+		OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, uiBytes);
+	}
+
+	eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+	                            &psMemDesc->psMapping->sMemHandle,
+	                            psMemDesc->uiOffset,
+	                            psMemDesc->uiSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Alloc MMU object");
+
+	PDumpMMUMalloc(psDevNode,
+	               psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	               eMMULevel,
+	               &psMemDesc->sDevPAddr,
+	               uiBytes,
+	               uiAlign,
+	               psMMUContext->psDevAttrs->eMMUType);
+
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      eMMULevel,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      0,
+	                      uiNumEntries,
+	                      NULL, NULL, 0, /* pdump symbolic info is irrelevant here */
+	                      psConfig->uiBytesPerEntry,
+	                      uiLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      0, /* Unused - Parity bit values are taken directly from PTE memory when PTE is 0-initialised or invalid */
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      psMMUContext->psDevAttrs->eMMUType);
+#else
+	PVR_UNREFERENCED_PARAMETER(eMMULevel);
+#endif
+
+	return PVRSRV_OK;
+e1:
+	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx,
+	                 psMemDesc);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _PxMemFree
+
+@Description    Frees physical memory for MMU objects, de-initialises
+                and PDumps it.
+
+@Input          psMemDesc       Description of allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+
+static void _PxMemFree(MMU_CONTEXT *psMMUContext,
+                       MMU_MEMORY_DESC *psMemDesc, MMU_LEVEL eMMULevel)
+{
+#if defined(MMU_CLEARMEM_ON_FREE)
+	/*
+		Clear the MMU object
+		Note: if any MMUs are cleared with non-zero values then will need a
+		custom clear function
+		Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
+		unlikely
+	 */
+	OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, psMemDesc->ui32Bytes);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Clear MMU object before freeing it");
+#endif
+#endif/* MMU_CLEARMEM_ON_FREE */
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Free MMU object");
+	PDumpMMUFree(psMMUContext->psPhysMemCtx->psDevNode,
+	             psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	             eMMULevel,
+	             &psMemDesc->sDevPAddr,
+	             psMMUContext->psDevAttrs->eMMUType);
+#else
+	PVR_UNREFERENCED_PARAMETER(eMMULevel);
+#endif
+	/* free the PC */
+	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx, psMemDesc);
+}
+
+static INLINE PVRSRV_ERROR _SetupPTE(MMU_CONTEXT *psMMUContext,
+                                     MMU_Levelx_INFO *psLevel,
+                                     IMG_UINT32 uiIndex,
+                                     const MMU_PxE_CONFIG *psConfig,
+                                     const IMG_DEV_PHYADDR *psDevPAddr,
+                                     const IMG_DEV_VIRTADDR *psDevVAddr,
+                                     IMG_BOOL bUnmap,
+#if defined(PDUMP)
+                                     const IMG_CHAR *pszMemspaceName,
+                                     const IMG_CHAR *pszSymbolicAddr,
+                                     IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
+#endif
+                                     IMG_UINT64 uiProtFlags)
+{
+	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;
+	IMG_UINT64 ui64PxE64;
+	IMG_UINT64 uiAddr = psDevPAddr->uiAddr;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_BOOL bParity = psConfig->uiParityBitMask;
+#else
+	IMG_BOOL bParity = psConfig->uiParityBitMask && !bUnmap;
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(bParity);
+#endif
+#endif
+
+	if (psDevNode->pfnValidateOrTweakPhysAddrs)
+	{
+		PVRSRV_ERROR eErr = psDevNode->pfnValidateOrTweakPhysAddrs(psDevNode,
+		                                                           psMMUContext->psDevAttrs,
+		                                                           &uiAddr);
+		/* return if error */
+		PVR_LOG_RETURN_IF_ERROR(eErr, "_SetupPTE");
+	}
+
+	/* Calculate Entry */
+	ui64PxE64 =    uiAddr /* Calculate the offset to that base */
+			>> psConfig->uiAddrLog2Align /* Shift away the useless bits, because the alignment is very coarse and we address by alignment */
+			<< psConfig->uiAddrShift /* Shift back to fit address in the Px entry */
+			& psConfig->uiAddrMask; /* Delete unused bits */
+
+	ui64PxE64 |= uiProtFlags;
+
+	/* Add parity */
+	if (bParity)
+	{
+		ui64PxE64 |= _GetParityBit(psDevVAddr->uiAddr ^ psDevPAddr->uiAddr) << psConfig->uiParityBitShift;
+	}
+
+	/* Set the entry */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+		pui64Px[uiIndex] = ui64PxE64;
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+		/* assert that the result fits into 32 bits before writing
+		   it into the 32-bit array with a cast */
+		PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+		PVR_ASSERT(!bParity);
+
+		pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
+	}
+	else
+	{
+		return PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
+	}
+
+	/* Log modification */
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+	        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+	        uiIndex, MMU_LEVEL_1,
+	        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+	        !bUnmap);
+
+	PVR_UNREFERENCED_PARAMETER(bUnmap); /* when HTBLOGK is disabled */
+
+#if defined(PDUMP)
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      MMU_LEVEL_1,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      uiIndex,
+	                      1,
+	                      pszMemspaceName,
+	                      pszSymbolicAddr,
+	                      uiSymbolicAddrOffset,
+	                      psConfig->uiBytesPerEntry,
+	                      psConfig->uiAddrLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      bParity ? _GetParityBit(psDevVAddr->uiAddr ^ 0) : 0,
+	                      psConfig->uiParityBitShift,
+	                      bParity ? psConfig->uiParityBitMask : 0,
+	                      psMMUContext->psDevAttrs->eMMUType);
+#endif /*PDUMP*/
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _SetupPxE
+
+@Description    Sets up an entry of an MMU object to point to the
+                provided address
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel         Level info for MMU object
+
+@Input          uiIndex         Index into the MMU object to setup
+
+@Input          psConfig        MMU Px config
+
+@Input          eMMULevel       Level of MMU object
+
+@Input          psDevPAddr      Address to setup the MMU object to point to
+
+@Input          pszMemspaceName Name of the PDump memory space that the entry
+                                will point to
+
+@Input          pszSymbolicAddr PDump symbolic address that the entry will
+                                point to
+
+@Input          uiProtFlags     MMU protection flags
+ */
+/*****************************************************************************/
+static void _SetupPxE(MMU_CONTEXT *psMMUContext,
+                      MMU_Levelx_INFO *psLevel,
+                      IMG_UINT32 uiIndex,
+                      const MMU_PxE_CONFIG *psConfig,
+                      MMU_LEVEL eMMULevel,
+                      const IMG_DEV_PHYADDR *psDevPAddr,
+#if defined(PDUMP)
+                      const IMG_CHAR *pszMemspaceName,
+                      const IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
+#endif
+                      MMU_PROTFLAGS_T uiProtFlags,
+                      IMG_UINT32 uiLog2DataPageSize)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;
+
+	IMG_UINT32 (*pfnDerivePxEProt4)(IMG_UINT32);
+	IMG_UINT64 (*pfnDerivePxEProt8)(IMG_UINT32, IMG_UINT32);
+
+	if (!psDevPAddr)
+	{
+		/* Invalidate entry */
+		if (~uiProtFlags & MMU_PROTFLAGS_INVALID)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Error, no physical address specified, but not invalidating entry"));
+			uiProtFlags |= MMU_PROTFLAGS_INVALID;
+		}
+		psDevPAddr = &gsBadDevPhyAddr;
+	}
+	else
+	{
+		if (uiProtFlags & MMU_PROTFLAGS_INVALID)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "A physical address was specified when requesting invalidation of entry"));
+			uiProtFlags |= MMU_PROTFLAGS_INVALID;
+		}
+	}
+
+	switch (eMMULevel)
+	{
+		case MMU_LEVEL_3:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePCEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePCEProt8;
+			break;
+
+		case MMU_LEVEL_2:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePDEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePDEProt8;
+			break;
+
+		case MMU_LEVEL_1:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePTEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePTEProt8;
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid MMU level", __func__));
+			PVR_ASSERT(0);
+			return;
+	}
+
+	/* How big is a PxE in bytes? */
+	/* Filling the actual Px entry with an address */
+	switch (psConfig->uiBytesPerEntry)
+	{
+		case 4:
+		{
+			IMG_UINT32 *pui32Px;
+			IMG_UINT64 ui64PxE64;
+
+			pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+			ui64PxE64 = psDevPAddr->uiAddr               /* Calculate the offset to that base */
+					>> psConfig->uiAddrLog2Align /* Shift away the unnecessary bits of the address */
+					<< psConfig->uiAddrShift     /* Shift back to fit address in the Px entry */
+					& psConfig->uiAddrMask;      /* Delete unused higher bits */
+
+			ui64PxE64 |= (IMG_UINT64)pfnDerivePxEProt4(uiProtFlags);
+			/* assert that the result fits into 32 bits before writing
+			   it into the 32-bit array with a cast */
+			PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+
+			/* We should never invalidate an invalid page */
+			if (uiProtFlags & MMU_PROTFLAGS_INVALID)
+			{
+				PVR_ASSERT(pui32Px[uiIndex] != ui64PxE64);
+			}
+			pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+			        uiIndex, eMMULevel,
+			        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
+			break;
+		}
+		case 8:
+		{
+			IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+			IMG_UINT64 ui64PxE64;
+
+			ui64PxE64 = psDevPAddr->uiAddr             /* Calculate the offset to that base */
+					>> psConfig->uiAddrLog2Align  /* Shift away the unnecessary bits of the address */
+					<< psConfig->uiAddrShift      /* Shift back to fit address in the Px entry */
+					& psConfig->uiAddrMask;       /* Delete unused higher bits */
+			ui64PxE64 |= pfnDerivePxEProt8(uiProtFlags, uiLog2DataPageSize);
+
+			pui64Px[uiIndex] = ui64PxE64;
+
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+			        uiIndex, eMMULevel,
+			        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
+			break;
+		}
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: PxE size not supported (%d) for level %d",
+					__func__, psConfig->uiBytesPerEntry, eMMULevel));
+			PVR_ASSERT(0);
+			return;
+	}
+
+#if defined(PDUMP)
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      eMMULevel,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      uiIndex,
+	                      1,
+	                      pszMemspaceName,
+	                      pszSymbolicAddr,
+	                      uiSymbolicAddrOffset,
+	                      psConfig->uiBytesPerEntry,
+	                      psConfig->uiAddrLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      psMMUContext->psDevAttrs->eMMUType);
+#endif
+
+	psDevNode->pfnMMUCacheInvalidate(psDevNode, psMMUContext,
+	                                 eMMULevel,
+	                                 uiProtFlags & MMU_PROTFLAGS_INVALID);
+
+	return;
+}
+
+/*****************************************************************************
+ *                   MMU host control functions (Level Info)                 *
+ *****************************************************************************/
+
+
+/*************************************************************************/ /*!
+@Function       _MMU_FreeLevel
+
+@Description    Recursively frees the specified range of Px entries. If any
+                level has its last reference dropped then the MMU object
+                memory and the MMU_Levelx_Info will be freed.
+
+				At each level we might be crossing a boundary from one Px to
+				another. The values for auiStartArray should be by used for
+				the first call into each level and the values in auiEndArray
+				should only be used in the last call for each level.
+				In order to determine if this is the first/last call we pass
+				in bFirst and bLast.
+				When one level calls down to the next only if bFirst/bLast is set
+				and it's the first/last iteration of the loop at its level will
+				bFirst/bLast set for the next recursion.
+				This means that each iteration has the knowledge of the previous
+				level which is required.
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel                 Level info on which to free the
+                                        specified range
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          auiEntriesPerPxArray    Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          pui32CurrentLevel       Pointer to a variable which is set to our
+                                        current level
+
+@Input          uiStartIndex            Start index of the range to free
+
+@Input          uiEndIndex              End index of the range to free
+
+@Input			bFirst                  This is the first call for this level
+
+@Input			bLast                   This is the last call for this level
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static IMG_BOOL _MMU_FreeLevel(MMU_CONTEXT *psMMUContext,
+                               MMU_Levelx_INFO *psLevel,
+                               IMG_UINT32 auiStartArray[],
+                               IMG_UINT32 auiEndArray[],
+                               IMG_UINT32 auiEntriesPerPxArray[],
+                               const MMU_PxE_CONFIG *apsConfig[],
+                               MMU_LEVEL aeMMULevel[],
+                               IMG_UINT32 *pui32CurrentLevel,
+                               IMG_UINT32 uiStartIndex,
+                               IMG_UINT32 uiEndIndex,
+                               IMG_BOOL bFirst,
+                               IMG_BOOL bLast,
+                               IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 uiThisLevel = *pui32CurrentLevel;
+	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel];
+	IMG_UINT32 i;
+	IMG_BOOL bFreed = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Call before parameter check for debugging purposes. */
+	MMU_OBJ_DBG((PVR_DBG_ERROR,
+	             "_MMU_FreeLevel: level = %u, range %u - %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], uiStartIndex, uiEndIndex,
+	             (psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX)));
+
+	/* Parameter checks */
+	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);
+
+	if (psLevel == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid MMU level data", __func__));
+		goto ErrReturn;
+	}
+
+	for (i = uiStartIndex; i < uiEndIndex; i++)
+	{
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			MMU_Levelx_INFO *psNextLevel = psLevel->apsNextLevel[i];
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			/* Recurse into the next level */
+			(*pui32CurrentLevel)++;
+			if (_MMU_FreeLevel(psMMUContext, psNextLevel, auiStartArray,
+			                   auiEndArray, auiEntriesPerPxArray,
+			                   apsConfig, aeMMULevel, pui32CurrentLevel,
+			                   uiNextStartIndex, uiNextEndIndex,
+			                   bNextFirst, bNextLast, uiLog2DataPageSize))
+			{
+				/* Un-wire the entry */
+				_SetupPxE(psMMUContext,
+				          psLevel,
+				          i,
+				          psConfig,
+				          aeMMULevel[uiThisLevel],
+				          NULL,
+#if defined(PDUMP)
+				          NULL,	/* Only required for data page */
+				          NULL,	/* Only required for data page */
+				          0,      /* Only required for data page */
+#endif
+				          MMU_PROTFLAGS_INVALID,
+				          uiLog2DataPageSize);
+
+				/* Free table of the level below, pointed to by this table entry.
+				 * We don't destroy the table inside the above _MMU_FreeLevel call because we
+				 * first have to set the table entry of the level above to invalid. */
+				_PxMemFree(psMMUContext, &psNextLevel->sMemDesc, aeMMULevel[*pui32CurrentLevel]);
+				OSFreeMem(psNextLevel);
+
+				/* The level below us is empty, drop the refcount and clear the pointer */
+				psLevel->ui32RefCount--;
+				psLevel->apsNextLevel[i] = NULL;
+
+				/* Check we haven't wrapped around */
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+			}
+			(*pui32CurrentLevel)--;
+		}
+		else
+		{
+			psLevel->ui32RefCount--;
+		}
+
+		/*
+		   Free this level if it is no longer referenced, unless it's the base
+		   level in which case it's part of the MMU context and should be freed
+		   when the MMU context is freed
+		 */
+		if ((psLevel->ui32RefCount == 0) && (psLevel != &psMMUContext->sBaseLevelInfo))
+		{
+			bFreed = IMG_TRUE;
+		}
+	}
+
+	/* Level one flushing is done when we actually write the table entries */
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		PVRSRV_ERROR eError;
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                   &psLevel->sMemDesc.psMapping->sMemHandle,
+		                   uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                   (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+ErrReturn:
+	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_FreeLevel end: level = %u, refcount = %u",
+	             aeMMULevel[uiThisLevel],
+	             bFreed ? 0 : (psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX)));
+
+	return bFreed;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_AllocLevel
+
+@Description    Recursively allocates the specified range of Px entries. If any
+                level has its last reference dropped then the MMU object
+                memory and the MMU_Levelx_Info will be freed.
+
+				At each level we might be crossing a boundary from one Px to
+				another. The values for auiStartArray should be by used for
+				the first call into each level and the values in auiEndArray
+				should only be used in the last call for each level.
+				In order to determine if this is the first/last call we pass
+				in bFirst and bLast.
+				When one level calls down to the next only if bFirst/bLast is set
+				and it's the first/last iteration of the loop at its level will
+				bFirst/bLast set for the next recursion.
+				This means that each iteration has the knowledge of the previous
+				level which is required.
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel                 Level info on which to free the
+                                        specified range
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          auiEntriesPerPxArray    Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          pui32CurrentLevel       Pointer to a variable which is set to our
+                                        current level
+
+@Input          uiStartIndex            Start index of the range to free
+
+@Input          uiEndIndex              End index of the range to free
+
+@Input			bFirst                  This is the first call for this level
+
+@Input			bLast                   This is the last call for this level
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _MMU_AllocLevel(MMU_CONTEXT *psMMUContext,
+                                    MMU_Levelx_INFO *psLevel,
+                                    IMG_UINT32 auiStartArray[],
+                                    IMG_UINT32 auiEndArray[],
+                                    IMG_UINT32 auiEntriesPerPxArray[],
+                                    const MMU_PxE_CONFIG *apsConfig[],
+                                    MMU_LEVEL aeMMULevel[],
+                                    IMG_UINT32 *pui32CurrentLevel,
+                                    IMG_UINT32 uiStartIndex,
+                                    IMG_UINT32 uiEndIndex,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+                                    IMG_DEV_VIRTADDR* psRunningDevVAddr,
+#endif
+                                    IMG_BOOL bFirst,
+                                    IMG_BOOL bLast,
+                                    IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 uiThisLevel = *pui32CurrentLevel; /* Starting with 0 */
+	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel]; /* The table config for the current level */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	IMG_UINT32 uiAllocState = 99; /* Debug info to check what progress was made in the function. Updated during this function. */
+	IMG_UINT32 i;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Parameter check */
+	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);
+
+	MMU_OBJ_DBG((PVR_DBG_ERROR,
+	             "_MMU_AllocLevel: level = %u, range %u - %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], uiStartIndex, uiEndIndex,
+	             psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX));
+
+	/* Go from uiStartIndex to uiEndIndex through the Px */
+	for (i = uiStartIndex;i < uiEndIndex;i++)
+	{
+		/* Only try an allocation if this is not the last level */
+		/*Because a PT allocation is already done while setting the entry in PD */
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If there is already a next Px level existing, do not allocate it */
+			if (!psLevel->apsNextLevel[i])
+			{
+				MMU_Levelx_INFO *psNextLevel;
+				IMG_UINT32 ui32AllocSize;
+				IMG_UINT32 uiNextEntries;
+
+				/* Allocate and setup the next level */
+				uiNextEntries = auiEntriesPerPxArray[uiThisLevel + 1];
+				ui32AllocSize = sizeof(MMU_Levelx_INFO);
+				if (aeMMULevel[uiThisLevel + 1] != MMU_LEVEL_1)
+				{
+					ui32AllocSize += IMG_FLEX_ARRAY_SIZE(sizeof(MMU_Levelx_INFO *), uiNextEntries);
+				}
+				psNextLevel = OSAllocZMem(ui32AllocSize);
+				if (psNextLevel == NULL)
+				{
+					uiAllocState = 0;
+					goto e0;
+				}
+
+				/* Hook in this level for next time */
+				psLevel->apsNextLevel[i] = psNextLevel;
+
+				psNextLevel->ui32NumOfEntries = uiNextEntries;
+				psNextLevel->ui32RefCount = 0;
+				/* Allocate Px memory for a sub level*/
+				eError = _PxMemAlloc(psMMUContext, uiNextEntries, apsConfig[uiThisLevel + 1],
+				                     aeMMULevel[uiThisLevel + 1],
+				                     &psNextLevel->sMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+				                     psRunningDevVAddr,
+				                     uiLog2DataPageSize,
+#endif
+				                     psConfig->uiAddrLog2Align
+);
+				if (eError != PVRSRV_OK)
+				{
+					uiAllocState = 1;
+					goto e0;
+				}
+
+				/* Wire up the entry */
+				_SetupPxE(psMMUContext,
+				          psLevel,
+				          i,
+				          psConfig,
+				          aeMMULevel[uiThisLevel],
+				          &psNextLevel->sMemDesc.sDevPAddr,
+#if defined(PDUMP)
+				          NULL, /* Only required for data page */
+				          NULL, /* Only required for data page */
+				          0,    /* Only required for data page */
+#endif
+				          0,
+				          uiLog2DataPageSize);
+
+				psLevel->ui32RefCount++;
+			}
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+			else
+			{
+				/* The level structure already exists, increment running device virtual address
+				   This is necessary for correct parity bit calculation on further allocated page tables */
+				if (apsConfig[aeMMULevel[MMU_LEVEL_1]]->uiParityBitMask && aeMMULevel[uiThisLevel+1] == MMU_LEVEL_1)
+				{
+					psRunningDevVAddr->uiAddr += psLevel->apsNextLevel[i]->ui32NumOfEntries * (1 << uiLog2DataPageSize);
+				}
+			}
+#endif
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			/* Recurse into the next level */
+			(*pui32CurrentLevel)++;
+			eError = _MMU_AllocLevel(psMMUContext, psLevel->apsNextLevel[i],
+			                         auiStartArray,
+			                         auiEndArray,
+			                         auiEntriesPerPxArray,
+			                         apsConfig,
+			                         aeMMULevel,
+			                         pui32CurrentLevel,
+			                         uiNextStartIndex,
+			                         uiNextEndIndex,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+
+			                         psRunningDevVAddr,
+#endif
+			                         bNextFirst,
+			                         bNextLast,
+			                         uiLog2DataPageSize);
+			(*pui32CurrentLevel)--;
+			if (eError != PVRSRV_OK)
+			{
+				uiAllocState = 3;
+				goto e0;
+			}
+		}
+		else
+		{
+			/* All we need to do for level 1 is bump the refcount */
+			psLevel->ui32RefCount++;
+		}
+
+		if (psLevel->ui32RefCount > psLevel->ui32NumOfEntries)
+		{
+			/* Given how the reference counting is implemented for MMU_LEVEL_2
+			 * and MMU_LEVEL_3 this should never happen for those levels. Only
+			 * in case of MMU_LEVEL_1 this should be possible (e.g. when someone
+			 * takes multiple reservations of the same range). In such case
+			 * return error to prevent reference count to rollover. */
+			uiAllocState = 4;
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, e0);
+		}
+	}
+
+	/* Level one flushing is done when we actually write the table entries */
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, e1);
+	}
+
+	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_AllocLevel end: level = %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], psLevel->ui32RefCount));
+	return PVRSRV_OK;
+
+e0:
+	/* Confirm that we've not come down this route unexpectedly */
+	PVR_ASSERT(uiAllocState!=99);
+	PVR_DPF((PVR_DBG_ERROR, "_MMU_AllocLevel: Error %d allocating Px for level %d in stage %d"
+			,eError, aeMMULevel[uiThisLevel], uiAllocState));
+
+	/* In case of an error process current `i` in special way without
+	 * recursively calling `_MMU_FreeLevel()`. */
+
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		/* If this is not a PT (so it's a PC or PD) it means that an error
+		 * happened either during PT or PD allocation/setup.
+		 *
+		 * - If the error happened during PT allocation/setup it means that
+		 *   we're at the PD level now and we already run below `else` clause
+		 *   for the failing PT and the `for` loop below for the other PTs. This
+		 *   means that all of the PTs that have been referenced during this
+		 *   operation have been dereferenced. So we just need to free that
+		 *   failing PT here and the rest of them in the loop below.
+		 * - If the error happened during PD allocation/setup it means that
+		 *   we're at the PC level now and we already run below `else` clause
+		 *   for the failing PD and the `for` loop below for the other PDs. This
+		 *   means that all of the PTs that have been allocated during this
+		 *   operation for the failing PD have been freed. So we just need to
+		 *   free that failing PD and the rest of them in a recursive manner
+		 *   in the loop below. */
+		if (uiAllocState >= 3)
+		{
+			if (psLevel->apsNextLevel[i] != NULL &&
+			    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+			{
+				psLevel->ui32RefCount--;
+			}
+		}
+		if (uiAllocState >= 2)
+		{
+			if (psLevel->apsNextLevel[i] != NULL &&
+			    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+			{
+				_PxMemFree(psMMUContext, &psLevel->apsNextLevel[i]->sMemDesc,
+				           aeMMULevel[uiThisLevel + 1]);
+			}
+		}
+		if (psLevel->apsNextLevel[i] != NULL &&
+		    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+		{
+			OSFreeMem(psLevel->apsNextLevel[i]);
+			psLevel->apsNextLevel[i] = NULL;
+		}
+	}
+	else
+	{
+		/* This is a PT which means that we just need to dereference it. It's
+		 * going to be freed on the PD level in this error path in the `if`
+		 * clause above. */
+		psLevel->ui32RefCount--;
+	}
+
+	/* Check we haven't wrapped around */
+	PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+
+e1:
+	i--;
+
+	/* The start value of index variable i is not initialised on purpose.
+	 * This clean-up loop deinitialises what was already initialised in
+	 * reverse order, so the i index already has the correct value.
+	 */
+	for (/* i already set */; i >= uiStartIndex && i < uiEndIndex; i--)
+	{
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			(*pui32CurrentLevel)++;
+			if (_MMU_FreeLevel(psMMUContext, psLevel->apsNextLevel[i],
+			                   auiStartArray, auiEndArray,
+			                   auiEntriesPerPxArray, apsConfig,
+			                   aeMMULevel, pui32CurrentLevel,
+			                   uiNextStartIndex, uiNextEndIndex,
+			                   bNextFirst, bNextLast, uiLog2DataPageSize))
+			{
+				_PxMemFree(psMMUContext, &psLevel->apsNextLevel[i]->sMemDesc,
+				           aeMMULevel[uiThisLevel + 1]);
+				OSFreeMem(psLevel->apsNextLevel[i]);
+				psLevel->apsNextLevel[i] = NULL;
+
+				psLevel->ui32RefCount--;
+
+				/* Check we haven't wrapped around */
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+			}
+			(*pui32CurrentLevel)--;
+		}
+		else
+		{
+			/* We should never come down this path, but it's here
+			   for completeness */
+			psLevel->ui32RefCount--;
+
+			/* Check we haven't wrapped around */
+			PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+		}
+	}
+
+	return eError;
+}
+
+/*****************************************************************************
+ *                   MMU page table functions                                *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetLevelData
+
+@Description    Get the all the level data and calculates the indexes for the
+                specified address range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Log2 of the page size to use
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          uiEntriesPerPxArray     Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          ppsMMUDevVAddrConfig    Device virtual address config
+
+@Input			phPriv					Private data of page size config
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static void _MMU_GetLevelData(MMU_CONTEXT *psMMUContext,
+                              IMG_DEV_VIRTADDR sDevVAddrStart,
+                              IMG_DEV_VIRTADDR sDevVAddrEnd,
+                              IMG_UINT32 uiLog2DataPageSize,
+                              IMG_UINT32 auiStartArray[],
+                              IMG_UINT32 auiEndArray[],
+                              IMG_UINT32 auiEntriesPerPx[],
+                              const MMU_PxE_CONFIG *apsConfig[],
+                              MMU_LEVEL aeMMULevel[],
+                              const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+                              IMG_HANDLE *phPriv)
+{
+	const MMU_PxE_CONFIG *psMMUPDEConfig;
+	const MMU_PxE_CONFIG *psMMUPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i = 0;
+
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
+	                                                 &psMMUPDEConfig,
+	                                                 &psMMUPTEConfig,
+	                                                 ppsMMUDevVAddrConfig,
+	                                                 phPriv);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "GetPageSizeConfiguration");
+
+	psDevVAddrConfig = *ppsMMUDevVAddrConfig;
+
+	if (psDevVAddrConfig->uiPCIndexMask != 0)
+	{
+		auiStartArray[i] = _CalcPCEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+		auiEndArray[i] = _CalcPCEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPC;
+		apsConfig[i] = psDevAttrs->psBaseConfig;
+		aeMMULevel[i] = MMU_LEVEL_3;
+		i++;
+	}
+
+	if (psDevVAddrConfig->uiPDIndexMask != 0)
+	{
+		auiStartArray[i] = _CalcPDEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+		auiEndArray[i] = _CalcPDEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPD;
+		if (i == 0)
+		{
+			apsConfig[i] = psDevAttrs->psBaseConfig;
+		}
+		else
+		{
+			apsConfig[i] = psMMUPDEConfig;
+		}
+		aeMMULevel[i] = MMU_LEVEL_2;
+		i++;
+	}
+
+	/*
+		There is always a PTE entry so we have a slightly different behaviour than above.
+		E.g. for 2 MB pages the uiPTIndexMask is 0x0000000000 but still there
+		is a PT with one entry.
+
+	 */
+	auiStartArray[i] = _CalcPTEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+	if (psDevVAddrConfig->uiPTIndexMask !=0)
+	{
+		auiEndArray[i] = _CalcPTEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+	}
+	else
+	{
+		/*
+			If the PTE mask is zero it means there is only 1 PTE and thus, as an
+			an exclusive bound, the end array index is equal to the start index + 1.
+		 */
+
+		auiEndArray[i] = auiStartArray[i] + 1;
+	}
+
+	auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPT;
+
+	if (i == 0)
+	{
+		apsConfig[i] = psDevAttrs->psBaseConfig;
+	}
+	else
+	{
+		apsConfig[i] = psMMUPTEConfig;
+	}
+	aeMMULevel[i] = MMU_LEVEL_1;
+}
+
+static void _MMU_PutLevelData(MMU_CONTEXT *psMMUContext, IMG_HANDLE hPriv)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+}
+
+/*************************************************************************/ /*!
+@Function       _AllocPageTables
+
+@Description    Allocate page tables and any higher level MMU objects required
+                for the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Page size of the data pages
+
+@Return         PVRSRV_OK if the allocation was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR
+_AllocPageTables(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrStart,
+                 IMG_DEV_VIRTADDR sDevVAddrEnd,
+                 IMG_UINT32 uiLog2DataPageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
+	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
+	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
+	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+	IMG_UINT32 ui32CurrentLevel = 0;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_DEV_VIRTADDR sRunningDevVAddrStart;
+	IMG_BOOL bSetParity;
+#endif
+
+	PVR_DPF((PVR_DBG_ALLOC,
+			"_AllocPageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+			sDevVAddrStart.uiAddr,
+			sDevVAddrEnd.uiAddr
+	));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode,
+	             "Allocating page tables for %"IMG_UINT64_FMTSPEC" bytes virtual range: "
+	             IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+	             (IMG_UINT64)sDevVAddrEnd.uiAddr - (IMG_UINT64)sDevVAddrStart.uiAddr,
+	             (IMG_UINT64)sDevVAddrStart.uiAddr,
+	             (IMG_UINT64)sDevVAddrEnd.uiAddr);
+#endif
+
+	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
+	                  (IMG_UINT32) uiLog2DataPageSize, auiStartArray, auiEndArray,
+	                  auiEntriesPerPx, apsConfig, aeMMULevel,
+	                  &psDevVAddrConfig, &hPriv);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	bSetParity = apsConfig[aeMMULevel[MMU_LEVEL_1]]->uiParityBitMask != 0;
+	if (bSetParity)
+	{
+		/* If parity bit needs to be written for PTEs save the first VA of the PT */
+		sRunningDevVAddrStart.uiAddr = sDevVAddrStart.uiAddr & ~((IMG_UINT64)(auiEntriesPerPx[MMU_LEVEL_1] * (1 << uiLog2DataPageSize)) - 1);
+	}
+
+#endif
+
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_ALLOC,
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));
+
+	eError = _MMU_AllocLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
+	                         auiStartArray, auiEndArray, auiEntriesPerPx,
+	                         apsConfig, aeMMULevel, &ui32CurrentLevel,
+	                         auiStartArray[0], auiEndArray[0],
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	                         &sRunningDevVAddrStart,
+#endif
+	                         IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP) && defined(DEBUG)
+	if (bSetParity)
+	{
+		PVR_ASSERT(sRunningDevVAddrStart.uiAddr >= sDevVAddrEnd.uiAddr);
+	}
+#endif
+
+	_MMU_PutLevelData(psMMUContext, hPriv);
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreePageTables
+
+@Description    Free page tables and any higher level MMU objects at are no
+                longer referenced for the specified virtual range.
+                This will fill the temporary free list of the MMU context which
+                needs cleanup after the call.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Page size of the data pages
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _FreePageTables(MMU_CONTEXT *psMMUContext,
+                            IMG_DEV_VIRTADDR sDevVAddrStart,
+                            IMG_DEV_VIRTADDR sDevVAddrEnd,
+                            IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
+	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
+	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
+	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
+	IMG_UINT32 ui32CurrentLevel = 0;
+	IMG_HANDLE hPriv;
+
+	PVR_DPF((PVR_DBG_ALLOC,
+			"_FreePageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+			sDevVAddrStart.uiAddr,
+			sDevVAddrEnd.uiAddr
+	));
+
+	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
+	                  uiLog2DataPageSize, auiStartArray, auiEndArray,
+	                  auiEntriesPerPx, apsConfig, aeMMULevel,
+	                  &psDevVAddrConfig, &hPriv);
+
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_FREE,
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));
+
+	/* ignoring return code, in this case there should be no references
+	 * to the level anymore, and at this stage there is nothing to do with
+	 * the return status */
+	(void) _MMU_FreeLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
+	                      auiStartArray, auiEndArray, auiEntriesPerPx,
+	                      apsConfig, aeMMULevel, &ui32CurrentLevel,
+	                      auiStartArray[0], auiEndArray[0],
+	                      IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);
+
+	_MMU_PutLevelData(psMMUContext, hPriv);
+}
+
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetPTInfo
+
+@Description    Get the PT level information and PT entry index for the specified
+                virtual address
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          psDevVAddr              Device virtual address to get the PTE info
+                                        from.
+
+@Input          psDevVAddrConfig        The current virtual address config obtained
+                                        by another function call before.
+
+@Output         psLevel                 Level info of the PT
+
+@Output         pui32PTEIndex           Index into the PT the address corresponds to
+
+@Return         IMG_TRUE if the operation was successful and IMG_FALSE otherwise
+ */
+/*****************************************************************************/
+static INLINE IMG_BOOL _MMU_GetPTInfo(MMU_CONTEXT                *psMMUContext,
+                                      IMG_DEV_VIRTADDR            sDevVAddr,
+                                      const MMU_DEVVADDR_CONFIG  *psDevVAddrConfig,
+                                      MMU_Levelx_INFO           **ppsLevel,
+                                      IMG_UINT32                 *pui32PTEIndex)
+{
+	MMU_Levelx_INFO *psLocalLevel = NULL;
+	MMU_LEVEL eMMULevel = psMMUContext->psDevAttrs->psBaseConfig->ePxLevel;
+	const MMU_LEVEL eMMUBaseLevel = eMMULevel;
+	IMG_UINT32 uiPCEIndex;
+	IMG_UINT32 uiPDEIndex;
+
+	if ((eMMULevel <= MMU_LEVEL_0) || (eMMULevel >= MMU_LEVEL_LAST))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTEInfo: Invalid MMU level"));
+		PVR_ASSERT(0);
+	}
+
+	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
+	{
+		if (eMMULevel == MMU_LEVEL_3)
+		{
+			/* find the page directory pointed by the PCE */
+			uiPCEIndex = _CalcPCEIdx (sDevVAddr, psDevVAddrConfig,
+			                          IMG_FALSE);
+			psLocalLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiPCEIndex];
+		}
+
+		if (eMMULevel == MMU_LEVEL_2)
+		{
+			/* find the page table pointed by the PDE */
+			uiPDEIndex = _CalcPDEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+			if (psLocalLevel == NULL)
+			{
+				return IMG_FALSE;
+			}
+
+			psLocalLevel = psLocalLevel->apsNextLevel[uiPDEIndex];
+		}
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			/* find PTE index into page table */
+			*pui32PTEIndex = _CalcPTEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLocalLevel == NULL)
+			{
+				if (eMMUBaseLevel == eMMULevel)
+				{
+					/* if the MMU only supports one level return the base level */
+					psLocalLevel = &psMMUContext->sBaseLevelInfo;
+				}
+				else
+				{
+					return IMG_FALSE;
+				}
+			}
+		}
+	}
+
+	*ppsLevel = psLocalLevel;
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetPTConfig
+
+@Description    Get the level config. Call _MMU_PutPTConfig after use!
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiLog2DataPageSize      Log 2 of the page size
+
+@Output         ppsConfig               Config of the PTE
+
+@Output         phPriv                  Private data handle to be passed back
+                                        when the info is put
+
+@Output         ppsDevVAddrConfig       Config of the device virtual addresses
+
+@Return         None
+ */
+/*****************************************************************************/
+static INLINE void _MMU_GetPTConfig(MMU_CONTEXT               *psMMUContext,
+                                    IMG_UINT32                  uiLog2DataPageSize,
+                                    const MMU_PxE_CONFIG      **ppsConfig,
+                                    IMG_HANDLE                 *phPriv,
+                                    const MMU_DEVVADDR_CONFIG **ppsDevVAddrConfig)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+
+	if (psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
+	                                            &psPDEConfig,
+	                                            &psPTEConfig,
+	                                            &psDevVAddrConfig,
+	                                            phPriv) != PVRSRV_OK)
+	{
+		/*
+		   There should be no way we got here unless uiLog2DataPageSize
+		   has changed after the MMU_Alloc call (in which case it's a bug in
+		   the MM code)
+		 */
+		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTConfig: Could not get valid page size config"));
+		PVR_ASSERT(0);
+	}
+
+	*ppsConfig = psPTEConfig;
+	*ppsDevVAddrConfig = psDevVAddrConfig;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PutPTConfig
+
+@Description    Put the level info. Has to be called after _MMU_GetPTConfig to
+                ensure correct refcounting.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          phPriv                  Private data handle created by
+                                        _MMU_GetPTConfig.
+
+@Return         None
+ */
+/*****************************************************************************/
+static INLINE void _MMU_PutPTConfig(MMU_CONTEXT *psMMUContext,
+                                    IMG_HANDLE hPriv)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	if (psDevAttrs->pfnPutPageSizeConfiguration(hPriv) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Could not put page size config",
+				__func__));
+		PVR_ASSERT(0);
+	}
+}
+
+/* scratch / zero pages */
+
+static INLINE PVRSRV_ERROR _MMU_GetDefPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                           IMG_INT uiDefPage,
+                                           PVRSRV_DEF_PAGE **ppsDefPage,
+                                           IMG_CHAR **ppcDefPageName)
+{
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(ppcDefPageName);
+#endif
+
+	switch (uiDefPage)
+	{
+		case SCRATCH_PAGE:
+		{
+			*ppsDefPage = &psDevNode->sScratchPage;
+#if defined(PDUMP)
+			*ppcDefPageName = SCRATCH_PAGE_STR;
+#endif
+			break;
+		}
+		case DEV_ZERO_PAGE:
+		{
+			*ppsDefPage = &psDevNode->sDevZeroPage;
+#if defined(PDUMP)
+			*ppcDefPageName = DEV_ZERO_PAGE_STR;
+#endif
+			break;
+		}
+		default:
+		{
+			/* Invalid pcDefPageName */
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MMU_AllocBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                          IMG_INT uiInitValue,
+                                          IMG_INT uiDefPage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR	sDevPAddr = {0};
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+
+	eError = _MMU_GetDefPage(psDevNode,
+	                         uiDefPage,
+	                         &psDefPage,
+	                         &pcDefPageName);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr != MMU_BAD_PHYS_ADDR)
+	{
+		goto UnlockAndReturn;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Alloc %s page object", pcDefPageName);
+#endif
+
+	/* Allocate the scratch / zero page required for physical backing
+	 * of virtual ranges
+	 */
+	eError = DevPhysMemAlloc(psDevNode,
+	                         (1 << psDefPage->ui32Log2PgSize),
+	                         0,
+	                         uiInitValue,
+	                         IMG_TRUE,
+#if defined(PDUMP)
+	                         psDevNode->psMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                         pcDefPageName,
+	                         &psDefPage->hPdumpPg,
+#endif
+	                         PVR_SYS_ALLOC_PID,
+	                         &psDefPage->sPageHandle,
+	                         &sDevPAddr);
+
+	psDefPage->ui64PgPhysAddr = sDevPAddr.uiAddr;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _MMU_GetBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                        IMG_UINT64 *pui64PgPhysAddr,
+                                        IMG_INT uiDefPage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+
+	eError = _MMU_GetDefPage(psDevNode,
+	                         uiDefPage,
+	                         &psDefPage,
+	                         &pcDefPageName);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr == MMU_BAD_PHYS_ADDR)
+	{
+		eError = PVRSRV_ERROR_NOT_FOUND;
+		PVR_DPF((PVR_DBG_ERROR,"%s: %s (uiDefPage=%d)",
+		         __func__, PVRSRVGetErrorString(eError), uiDefPage));
+		goto UnlockAndReturn;
+	}
+
+	if (pui64PgPhysAddr)
+		*pui64PgPhysAddr = psDefPage->ui64PgPhysAddr;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+
+	return eError;
+}
+
+static void _MMU_FreeBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                 IMG_INT uiDefPage)
+{
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+	PVRSRV_ERROR eErr;
+
+	eErr = _MMU_GetDefPage(psDevNode,
+	                       uiDefPage,
+	                       &psDefPage,
+	                       &pcDefPageName);
+	if (eErr != PVRSRV_OK)
+		return;
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr == MMU_BAD_PHYS_ADDR)
+	{
+		goto UnlockAndReturn;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Free %s page object", pcDefPageName);
+#endif
+
+	DevPhysMemFree(psDevNode,
+#if defined(PDUMP)
+	               psDefPage->hPdumpPg,
+#endif
+	               &psDefPage->sPageHandle);
+
+#if defined(PDUMP)
+	psDefPage->hPdumpPg = NULL;
+#endif
+	psDefPage->ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+}
+
+
+/*****************************************************************************
+ *                     Public interface functions                            *
+ *****************************************************************************/
+
+/*
+	MMU_InitDevice
+*/
+PVRSRV_ERROR MMU_InitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Set the order to 0 */
+	psDevNode->sScratchPage.sPageHandle.uiOrder = 0;
+	psDevNode->sDevZeroPage.sPageHandle.uiOrder = 0;
+
+	/* Set the size of the Scratch and Zero pages to largest page size */
+	if (psDevNode->ui32Non4KPageSizeLog2 != 0)
+	{
+		psDevNode->sScratchPage.ui32Log2PgSize = psDevNode->ui32Non4KPageSizeLog2;
+		psDevNode->sDevZeroPage.ui32Log2PgSize = psDevNode->ui32Non4KPageSizeLog2;
+	}
+	else
+	{
+		psDevNode->sScratchPage.ui32Log2PgSize = OSGetPageShift();
+		psDevNode->sDevZeroPage.ui32Log2PgSize = OSGetPageShift();
+	}
+
+	/* Set the Scratch page phys addr */
+	psDevNode->sScratchPage.ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+	/* Set the Zero page phys addr */
+	psDevNode->sDevZeroPage.ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+	/* The lock can be acquired from MISR (Z-buffer) path */
+	eError = OSLockCreate(&psDevNode->sScratchPage.psPgLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.Scratch", ErrReturnError);
+
+	/* Create the lock for zero page */
+	eError = OSLockCreate(&psDevNode->sDevZeroPage.psPgLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.Zero", ErrFreeScratchPageLock);
+
+#ifdef PDUMP
+	psDevNode->sScratchPage.hPdumpPg = NULL;
+	psDevNode->sDevZeroPage.hPdumpPg = NULL;
+#endif /* PDUMP */
+
+	eError = _MMU_AllocBackingPage(psDevNode,
+	                               PVR_SCRATCH_PAGE_INIT_VALUE,
+	                               SCRATCH_PAGE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_AllocBackingPage.Scratch", ErrFreeZeroPageLock);
+
+	eError = _MMU_AllocBackingPage(psDevNode,
+	                               PVR_ZERO_PAGE_INIT_VALUE,
+	                               DEV_ZERO_PAGE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_AllocBackingPage.Zero", ErrFreeScratchPage);
+
+	return PVRSRV_OK;
+
+ErrFreeScratchPage:
+	_MMU_FreeBackingPage(psDevNode, SCRATCH_PAGE);
+ErrFreeZeroPageLock:
+	OSLockDestroy(psDevNode->sDevZeroPage.psPgLock);
+	psDevNode->sDevZeroPage.psPgLock = NULL;
+ErrFreeScratchPageLock:
+	OSLockDestroy(psDevNode->sScratchPage.psPgLock);
+	psDevNode->sScratchPage.psPgLock = NULL;
+ErrReturnError:
+	return eError;
+}
+
+/*
+	MMU_DeInitDevice
+*/
+void MMU_DeInitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode)
+{
+	if (psDevNode->sScratchPage.psPgLock != NULL)
+	{
+		_MMU_FreeBackingPage(psDevNode, SCRATCH_PAGE);
+
+		OSLockDestroy(psDevNode->sScratchPage.psPgLock);
+		psDevNode->sScratchPage.psPgLock = NULL;
+	}
+
+	if (psDevNode->sDevZeroPage.psPgLock)
+	{
+		_MMU_FreeBackingPage(psDevNode, DEV_ZERO_PAGE);
+
+
+		OSLockDestroy(psDevNode->sDevZeroPage.psPgLock);
+		psDevNode->sDevZeroPage.psPgLock = NULL;
+	}
+}
+
+/*
+	MMU_ContextCreate
+ */
+PVRSRV_ERROR
+MMU_ContextCreate(CONNECTION_DATA *psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  MMU_CONTEXT **ppsMMUContext,
+                  MMU_DEVICEATTRIBS *psDevAttrs)
+{
+	MMU_CONTEXT *psMMUContext;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	const MMU_PxE_CONFIG *psConfig;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx;
+	IMG_UINT32 ui32BaseObjects;
+	IMG_UINT32 ui32Size;
+	IMG_CHAR sBuf[40];
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "MMU context create");
+#endif
+
+	psConfig = psDevAttrs->psBaseConfig;
+	psDevVAddrConfig = psDevAttrs->psTopLevelDevVAddrConfig;
+
+	switch (psDevAttrs->psBaseConfig->ePxLevel)
+	{
+		case MMU_LEVEL_3:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPC;
+			break;
+
+		case MMU_LEVEL_2:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPD;
+			break;
+
+		case MMU_LEVEL_1:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPT;
+			break;
+
+		default:
+			PVR_LOG_GOTO_WITH_ERROR("psDevAttrs->psBaseConfig->ePxLevel", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+	}
+
+	/* Allocate the MMU context with the Level 1 Px info's */
+	ui32Size = sizeof(MMU_CONTEXT) +
+			((ui32BaseObjects - 1) * sizeof(MMU_Levelx_INFO *));
+
+	psMMUContext = OSAllocZMem(ui32Size);
+	PVR_LOG_GOTO_IF_NOMEM(psMMUContext, eError, e0);
+
+#if defined(PDUMP)
+	/* Clear the refcount */
+	psMMUContext->ui32PDumpContextIDRefCount = 0;
+#endif
+	/* Record Device specific attributes in the context for subsequent use */
+	psMMUContext->psDevAttrs = psDevAttrs;
+
+	/*
+	  Allocate physmem context and set it up
+	 */
+	psPhysMemCtx = OSAllocZMem(sizeof(MMU_PHYSMEM_CONTEXT));
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx, eError, e1);
+
+	psMMUContext->psPhysMemCtx = psPhysMemCtx;
+	psMMUContext->psConnection = psConnection;
+
+	psPhysMemCtx->psDevNode = psDevNode;		/* Needed for Direct Bridge case */
+	psPhysMemCtx->psMMUContext = psMMUContext;	/* Back-link to self */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/* Save the app-specific values for external reference via MMU_GetOSids. */
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+	{
+		psPhysMemCtx->ui32OSid     = psConnection->ui32OSid;
+		psPhysMemCtx->ui32OSidReg  = psConnection->ui32OSidReg;
+		psPhysMemCtx->bOSidAxiProt = psConnection->bOSidAxiProtReg;
+	}
+	else
+	{
+		/* Direct Bridge calling sequence e.g. Firmware */
+		psPhysMemCtx->ui32OSid     = 0;
+		psPhysMemCtx->ui32OSidReg  = 0;
+		psPhysMemCtx->bOSidAxiProt = IMG_FALSE;
+	}
+#endif
+
+	OSSNPrintf(sBuf, sizeof(sBuf), "pgtables %p", psPhysMemCtx);
+	psPhysMemCtx->uiPhysMemRANameAllocSize = OSStringLength(sBuf)+1;
+	psPhysMemCtx->pszPhysMemRAName = OSAllocMem(psPhysMemCtx->uiPhysMemRANameAllocSize);
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx->pszPhysMemRAName, eError, e2);
+
+	OSStringSafeCopy(psPhysMemCtx->pszPhysMemRAName, sBuf, psPhysMemCtx->uiPhysMemRANameAllocSize);
+
+	psPhysMemCtx->psPhysMemRA = RA_Create(psPhysMemCtx->pszPhysMemRAName,
+	                                      /* subsequent import */
+	                                      PhysHeapGetPageShift(psDevNode->psMMUPhysHeap),
+	                                      RA_LOCKCLASS_2,
+	                                      _MMU_PhysMem_RAImportAlloc,
+	                                      _MMU_PhysMem_RAImportFree,
+	                                      psPhysMemCtx, /* priv */
+	                                      RA_POLICY_DEFAULT);
+	if (psPhysMemCtx->psPhysMemRA == NULL)
+	{
+		OSFreeMem(psPhysMemCtx->pszPhysMemRAName);
+		psPhysMemCtx->pszPhysMemRAName = NULL;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, e3);
+	}
+
+	/* Setup cleanup meta data to check if a MMU context
+	 * has been destroyed and should not be accessed anymore */
+	psPhysMemCtx->psCleanupData = OSAllocMem(sizeof(*(psPhysMemCtx->psCleanupData)));
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx->psCleanupData, eError, e4);
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/* Record the originating OSid for all allocation / free for this context */
+	psPhysMemCtx->psCleanupData->ui32OSid = psPhysMemCtx->ui32OSid;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+	OSLockCreate(&psPhysMemCtx->psCleanupData->hCleanupLock);
+	psPhysMemCtx->psCleanupData->bMMUContextExists = IMG_TRUE;
+	dllist_init(&psPhysMemCtx->psCleanupData->sMMUCtxCleanupItemsHead);
+	OSAtomicWrite(&psPhysMemCtx->psCleanupData->iRef, 1);
+
+	/* allocate the base level object */
+	/*
+	   Note: Although this is not required by the this file until
+	         the 1st allocation is made, a device specific callback
+	         might request the base object address so we allocate
+	         it up front.
+	         In VZ we only need to allocate the FW/KM ctx on the
+	         Host Driver as all Guest tables are allocated and
+	         pre-mapped into the host FW Memory Context.
+	         Drivers with security support and premapped heaps
+	         rely on the TEE to manage the Firmware mappings.
+	 */
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#else
+	if (!_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+#endif
+	{
+		if (_PxMemAlloc(psMMUContext,
+						ui32BaseObjects,
+						psConfig,
+						psDevAttrs->psBaseConfig->ePxLevel,
+						&psMMUContext->sBaseLevelInfo.sMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+						NULL,
+						0U,
+#endif
+						psDevAttrs->ui32BaseAlign))
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PxMemAlloc", eError, PVRSRV_ERROR_OUT_OF_MEMORY, e5);
+		}
+	}
+
+	dllist_init(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);
+
+	psMMUContext->sBaseLevelInfo.ui32NumOfEntries = ui32BaseObjects;
+	psMMUContext->sBaseLevelInfo.ui32RefCount = 0;
+
+	/* Support cores that need to adjust the base level Px */
+	if (psDevNode->pfnMMUTopLevelPxWorkarounds != NULL)
+	{
+		PVRSRV_ERROR eError;
+		MMU_MEMORY_DESC  *psMemDesc  = &psMMUContext->sBaseLevelInfo.sMemDesc;
+
+		psDevNode->pfnMMUTopLevelPxWorkarounds(psConnection,
+		                                     psDevNode,
+		                                     psMemDesc->sDevPAddr,
+		                                     psMemDesc->pvCpuVAddr);
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psMemDesc->psMapping->sMemHandle,
+		                            psMemDesc->uiOffset,
+		                            psMemDesc->uiSize);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+	eError = OSLockCreate(&psMMUContext->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e6);
+
+	/* return context */
+	*ppsMMUContext = psMMUContext;
+
+	return PVRSRV_OK;
+
+e6:
+	_PxMemFree(psMMUContext, &psMMUContext->sBaseLevelInfo.sMemDesc, psDevAttrs->psBaseConfig->ePxLevel);
+e5:
+	OSFreeMem(psPhysMemCtx->psCleanupData);
+e4:
+	RA_Delete(psPhysMemCtx->psPhysMemRA);
+e3:
+	OSFreeMem(psPhysMemCtx->pszPhysMemRAName);
+e2:
+	OSFreeMem(psPhysMemCtx);
+e1:
+	OSFreeMem(psMMUContext);
+e0:
+	return eError;
+}
+
+/*
+	MMU_ContextDestroy
+ */
+void
+MMU_ContextDestroy (MMU_CONTEXT *psMMUContext)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PDLLIST_NODE psNode, psNextNode;
+
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psMMUContext->psPhysMemCtx->psDevNode;
+	MMU_CTX_CLEANUP_DATA *psCleanupData = psMMUContext->psPhysMemCtx->psCleanupData;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Enter", __func__));
+
+	if (psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK)
+	{
+		/* There should be no way to get here with live pages unless
+		   there is a bug in this module or the MM code */
+		PVR_ASSERT(psMMUContext->sBaseLevelInfo.ui32RefCount == 0);
+	}
+
+	/* Cleanup lock must be acquired before MMUContext lock. Reverse order
+	 * may lead to a deadlock and is reported by lockdep. */
+	OSLockAcquire(psCleanupData->hCleanupLock);
+	OSLockAcquire(psMMUContext->hLock);
+
+	/* Free the top level MMU object - will be put on defer free list.
+	 * This has to be done before the step below that will empty the
+	 * defer-free list. */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#else
+	if (!_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+#endif
+	{
+		_PxMemFree(psMMUContext,
+		           &psMMUContext->sBaseLevelInfo.sMemDesc,
+		           psMMUContext->psDevAttrs->psBaseConfig->ePxLevel);
+	}
+
+	/* Empty the temporary defer-free list of Px */
+	_FreeMMUMapping(psDevNode, &psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);
+	PVR_ASSERT(dllist_is_empty(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead));
+
+	/* Empty the defer free list so the cleanup thread will
+	 * not have to access any MMU context related structures anymore */
+	dllist_foreach_node(&psCleanupData->sMMUCtxCleanupItemsHead,
+	                    psNode,
+	                    psNextNode)
+	{
+		MMU_CLEANUP_ITEM *psCleanup = IMG_CONTAINER_OF(psNode,
+		                                               MMU_CLEANUP_ITEM,
+		                                               sMMUCtxCleanupItem);
+
+		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);
+
+		dllist_remove_node(psNode);
+	}
+	PVR_ASSERT(dllist_is_empty(&psCleanupData->sMMUCtxCleanupItemsHead));
+
+	psCleanupData->bMMUContextExists = IMG_FALSE;
+
+	/* Free physmem context */
+	RA_Delete(psMMUContext->psPhysMemCtx->psPhysMemRA);
+	psMMUContext->psPhysMemCtx->psPhysMemRA = NULL;
+	OSFreeMem(psMMUContext->psPhysMemCtx->pszPhysMemRAName);
+	psMMUContext->psPhysMemCtx->pszPhysMemRAName = NULL;
+
+	OSFreeMem(psMMUContext->psPhysMemCtx);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	OSLockRelease(psCleanupData->hCleanupLock);
+
+	if (OSAtomicDecrement(&psCleanupData->iRef) == 0)
+	{
+		OSLockDestroy(psCleanupData->hCleanupLock);
+		OSFreeMem(psCleanupData);
+	}
+
+	OSLockDestroy(psMMUContext->hLock);
+
+	/* free the context itself. */
+	OSFreeMem(psMMUContext);
+	/*not nulling pointer, copy on stack*/
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Exit", __func__));
+}
+
+/*
+	MMU_Alloc
+ */
+PVRSRV_ERROR
+MMU_Alloc (MMU_CONTEXT *psMMUContext,
+           IMG_DEVMEM_SIZE_T uSize,
+           IMG_UINT32 uiProtFlags,
+           IMG_DEVMEM_SIZE_T uDevVAddrAlignment,
+           IMG_DEV_VIRTADDR *psDevVAddr,
+           IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevVAddrEnd;
+
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	MMU_DEVICEATTRIBS *psDevAttrs;
+	IMG_HANDLE hPriv;
+
+#if !defined(DEBUG)
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uDevVAddrAlignment);
+#endif
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: uSize=" IMG_DEVMEM_SIZE_FMTSPEC
+			", uiProtFlags=0x%x, align="IMG_DEVMEM_ALIGN_FMTSPEC,
+			__func__, uSize, uiProtFlags, uDevVAddrAlignment));
+
+	/* check params */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevVAddr, "psDevVAddr");
+
+	psDevAttrs = psMMUContext->psDevAttrs;
+
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2PageSize,
+	                                                 &psPDEConfig,
+	                                                 &psPTEConfig,
+	                                                 &psDevVAddrConfig,
+	                                                 &hPriv);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnGetPageSizeConfiguration");
+
+	/* size and alignment must be datapage granular */
+	if (((psDevVAddr->uiAddr & psDevVAddrConfig->uiPageOffsetMask) != 0)
+			|| ((uSize & psDevVAddrConfig->uiPageOffsetMask) != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: invalid address or size granularity",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sDevVAddrEnd = *psDevVAddr;
+	sDevVAddrEnd.uiAddr += uSize;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (_MMU_IS_FWKM_CTX(psMMUContext))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver relying on firmware mappings created by the TEE.", __func__));
+	}
+	else
+#endif
+	{
+		eError = _AllocPageTables(psMMUContext, *psDevVAddr, sDevVAddrEnd, uiLog2PageSize);
+	}
+	OSLockRelease(psMMUContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "_AllocPageTables");
+		return PVRSRV_ERROR_MMU_FAILED_TO_ALLOCATE_PAGETABLES;
+	}
+
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_Free
+ */
+void
+MMU_Free (MMU_CONTEXT *psMMUContext,
+          IMG_DEV_VIRTADDR sDevVAddr,
+          IMG_DEVMEM_SIZE_T uiSize,
+          IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_DEV_VIRTADDR sDevVAddrEnd;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx;
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+#endif
+
+
+	PVR_ASSERT(psMMUContext != NULL);
+	PVR_LOG_RETURN_VOID_IF_FALSE(psMMUContext != NULL, "psMMUContext");
+
+	psPhysMemCtx = psMMUContext->psPhysMemCtx;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Freeing DevVAddr " IMG_DEV_VIRTADDR_FMTSPEC,
+			__func__, sDevVAddr.uiAddr));
+
+	/* ensure the address range to free is inside the heap */
+	sDevVAddrEnd = sDevVAddr;
+	sDevVAddrEnd.uiAddr += uiSize;
+
+	/* The Cleanup lock has to be taken before the MMUContext hLock to
+	 * prevent deadlock scenarios. It is necessary only for parts of
+	 * _SetupCleanup_FreeMMUMapping though.*/
+	OSLockAcquire(psPhysMemCtx->psCleanupData->hCleanupLock);
+
+	OSLockAcquire(psMMUContext->hLock);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#endif
+	{
+		_FreePageTables(psMMUContext,
+		                sDevVAddr,
+		                sDevVAddrEnd,
+		                uiLog2DataPageSize);
+	}
+
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+	eError = PVRSRVGetDevicePowerState(psPhysMemCtx->psDevNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		/* Treat unknown power state as ON. */
+		ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+	}
+
+	if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF)
+	{
+		_FreeMMUMapping(psPhysMemCtx->psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+	}
+	else
+	{
+		_SetupCleanup_FreeMMUMapping(psMMUContext->psPhysMemCtx);
+	}
+#else
+	if (!dllist_is_empty(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead))
+	{
+		_FreeMMUMapping(psPhysMemCtx->psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+	}
+#endif
+	OSLockRelease(psMMUContext->hLock);
+
+	OSLockRelease(psMMUContext->psPhysMemCtx->psCleanupData->hCleanupLock);
+}
+
+PVRSRV_ERROR
+MMU_MapPages(MMU_CONTEXT *psMMUContext,
+             PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+             IMG_DEV_VIRTADDR sDevVAddrBase,
+             PMR *psPMR,
+             IMG_UINT32 ui32PhysPgOffset,
+             IMG_UINT32 ui32MapPageCount,
+             IMG_UINT32 *paui32MapIndices,
+             IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hPriv;
+
+	MMU_Levelx_INFO *psLevel = NULL;
+
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+
+	IMG_UINT32 uiPTEIndex = 0;
+	IMG_UINT32 uiPageSize = (1 << uiLog2HeapPageSize);
+	IMG_UINT32 uiLoop = 0;
+	IMG_UINT32 ui32MappedCount = 0;
+	IMG_DEVMEM_OFFSET_T uiPgOffset = 0;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0, uiDefProtFlags=0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *psDevPAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+	IMG_BOOL *pbValid;
+	IMG_BOOL bValid;
+	IMG_BOOL bScratchBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
+	IMG_BOOL bNeedBacking = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+#if defined(PDUMP)
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
+
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Wire up Page Table entries to point to the Data Pages (%"IMG_INT64_FMTSPECd" bytes)",
+	             (IMG_UINT64)(ui32MapPageCount * uiPageSize));
+#endif /*PDUMP*/
+
+	/* Validate the most essential parameters */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext != NULL, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR != NULL, "psPMR");
+
+	psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Allocate memory for page-frame-numbers and validity states,
+	   N.B. assert could be triggered by an illegal uiSizeBytes */
+	if (ui32MapPageCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psDevPAddr = OSAllocMem(ui32MapPageCount * sizeof(IMG_DEV_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDevPAddr, eError, ErrReturnError);
+
+		pbValid = OSAllocMem(ui32MapPageCount * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrFreePAddrMappingArray);
+	}
+	else
+	{
+		psDevPAddr = asDevPAddr;
+		pbValid	= abValid;
+	}
+
+	/* Get the Device physical addresses of the pages we are trying to map
+	 * In the case of non indexed mapping we can get all addresses at once */
+
+	/* DevmemX callers */
+	if (NULL == paui32MapIndices)
+	{
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2HeapPageSize,
+		                         ui32MapPageCount,
+		                         ((IMG_DEVMEM_OFFSET_T) ui32PhysPgOffset << uiLog2HeapPageSize),
+		                         psDevPAddr,
+		                         pbValid,
+		                         DEVICE_USE | MAPPING_USE);
+		/* Since offsets are always used and never indices, we are safe to
+		 * receive retry here as this path won't have performed any mappings
+		 * or overwrites thus far. Retry may be returned due to the combination of
+		 * DEVICE_USE | MAPPING_USE
+		 */
+		PVR_GOTO_IF_ERROR(eError, ErrFreeValidArray);
+	}
+
+	/*Get the Page table level configuration */
+	_MMU_GetPTConfig(psMMUContext,
+	                 (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig,
+	                 &hPriv,
+	                 &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, ErrPutPTConfig);
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE),
+		                                                                  uiLog2HeapPageSize);
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE));
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrPutPTConfig);
+	}
+
+	if (PMR_IsSparse(psPMR))
+	{
+		/* We know there will not be 4G number of PMR's */
+		bScratchBacking = PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(PMR_Flags(psPMR));
+		if (bScratchBacking)
+		{
+			bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(PMR_Flags(psPMR));
+		}
+	}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PMRMarkForDeferFree(psPMR);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	for (uiLoop = 0; uiLoop < ui32MapPageCount; uiLoop++)
+	{
+
+#if defined(PDUMP)
+		IMG_DEVMEM_OFFSET_T uiNextSymName;
+#endif /*PDUMP*/
+
+		/* Devmem Map/Change sparse callers */
+		if (NULL != paui32MapIndices)
+		{
+			uiPgOffset = paui32MapIndices[uiLoop];
+
+			/*Calculate the Device Virtual Address of the page */
+			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr + (uiPgOffset * uiPageSize);
+
+			/* Get the physical address to map */
+			eError = PMR_DevPhysAddr(psPMR,
+			                         uiLog2HeapPageSize,
+			                         1,
+			                         uiPgOffset * uiPageSize,
+			                         &sDevPAddr,
+			                         &bValid,
+			                         DEVICE_USE | MAPPING_USE);
+			/* PVRSRV_ERROR_RETRY with SUPPORT_LINUX_OSPAGE_MIGRATION feature
+			 * enabled will not be returned here even with the use of
+			 * DEVICE_USE | MAPPING_USE because the feature does not
+			 * support migratable sparse PMRs.
+			 */
+			PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+		}
+		else
+		{
+			uiPgOffset = uiLoop + ui32PhysPgOffset;
+			sDevPAddr = psDevPAddr[uiLoop];
+			bValid = pbValid[uiLoop];
+		}
+
+		uiDefProtFlags = uiProtFlags;
+		/*
+			The default value of the entry is invalid so we don't need to mark
+			it as such if the page wasn't valid, we just advance pass that address
+		 */
+		if (bValid || bScratchBacking)
+		{
+			if (!bValid)
+			{
+				if (bZeroBacking)
+				{
+					eError = _MMU_GetBackingPage(psDevNode,
+					                             &sDevPAddr.uiAddr,
+					                             DEV_ZERO_PAGE);
+					PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+					                      ErrUnlockAndUnmapPages);
+
+					/* Ensure the zero back page PTE is read only */
+					uiDefProtFlags = uiProtFlagsReadOnly;
+				}
+				else
+				{
+					eError = _MMU_GetBackingPage(psDevNode,
+					                             &sDevPAddr.uiAddr,
+					                             SCRATCH_PAGE);
+					PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+					                      ErrUnlockAndUnmapPages);
+				}
+			}
+			else
+			{
+				/* check the physical alignment of the memory to map */
+				PVR_ASSERT((sDevPAddr.uiAddr & (uiPageSize-1)) == 0);
+			}
+
+#if defined(DEBUG)
+			{
+				IMG_INT32	i32FeatureVal = 0;
+				IMG_UINT32 ui32BitLength = FloorLog2(sDevPAddr.uiAddr);
+
+				i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+				do {
+					/* i32FeatureVal can be negative for cases where this feature is undefined
+					 * In that situation we need to bail out than go ahead with debug comparison */
+					if (0 > i32FeatureVal)
+						break;
+
+					if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"%s Failed. The physical address bitlength (%u)"
+								" is greater than the chip can handle (%d).",
+								__func__, ui32BitLength, i32FeatureVal));
+
+						PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto ErrUnlockAndUnmapPages;
+					}
+				} while (0);
+			}
+#endif /*DEBUG*/
+
+#if defined(PDUMP)
+			if (bValid)
+			{
+				eError = PMR_PDumpSymbolicAddr(psPMR, uiPgOffset * uiPageSize,
+				                               sizeof(aszMemspaceName), &aszMemspaceName[0],
+				                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
+				                               &uiSymbolicAddrOffset,
+				                               &uiNextSymName);
+				PVR_LOG_IF_ERROR(eError, "PMR_PDumpSymbolicAddr");
+			}
+#endif /*PDUMP*/
+
+			psPrevLevel = psLevel;
+			/* Calculate PT index and get new table descriptor */
+			if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+			                    &psLevel, &uiPTEIndex))
+			{
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+				                    ErrUnlockAndUnmapPages);
+			}
+
+			if (psPrevLevel == psLevel)
+			{
+				/*
+				 * Sparse allocations may have page offsets which
+				 * decrement as well as increment, so make sure we
+				 * update the range we will flush correctly.
+				 */
+				if (uiPTEIndex > uiFlushEnd)
+					uiFlushEnd = uiPTEIndex;
+				else if (uiPTEIndex < uiFlushStart)
+					uiFlushStart = uiPTEIndex;
+			}
+			else
+			{
+				/* Flush if we moved to another psLevel, i.e. page table */
+				if (psPrevLevel != NULL)
+				{
+					eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+					                            &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+					                            uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+					                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+					PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+				}
+
+				uiFlushStart = uiPTEIndex;
+				uiFlushEnd = uiFlushStart;
+			}
+
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_MAP,
+			        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+			        HTBLOG_U64_BITS_HIGH(sDevPAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevPAddr.uiAddr));
+
+			/* Set the PT entry with the specified address and protection flags */
+			eError = _SetupPTE(psMMUContext,
+			                   psLevel,
+			                   uiPTEIndex,
+			                   psConfig,
+			                   &sDevPAddr,
+			                   &sDevVAddr,
+			                   IMG_FALSE,
+#if defined(PDUMP)
+			                   (bValid)?aszMemspaceName:(psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName),
+			                          ((bValid)?aszSymbolicAddress:((bZeroBacking)?DEV_ZERO_PAGE_STR:SCRATCH_PAGE_STR)),
+			                          (bValid)?uiSymbolicAddrOffset:0,
+#endif /*PDUMP*/
+			                    uiDefProtFlags);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockAndUnmapPages);
+
+			if (bValid)
+			{
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+				PVR_DPF ((PVR_DBG_MESSAGE,
+						"%s: devVAddr=" IMG_DEV_VIRTADDR_FMTSPEC ", "
+						"size=" IMG_DEVMEM_OFFSET_FMTSPEC,
+						__func__,
+						sDevVAddr.uiAddr,
+						uiPgOffset * uiPageSize));
+
+				ui32MappedCount++;
+			}
+		}
+
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(pbValid);
+		OSFreeMem(psDevPAddr);
+	}
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Wired up %d Page Table entries (out of %d)", ui32MappedCount, ui32MapPageCount);
+#endif /*PDUMP*/
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	if (PMR_IsSparse(psPMR) && PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiMappingFlags))
+	{
+		bNeedBacking = IMG_TRUE;
+	}
+
+	(void) MMU_UnmapPagesUnlocked(psMMUContext,
+	                              (bNeedBacking) ? uiMappingFlags : 0,
+	                              sDevVAddrBase,
+	                              uiLoop,
+	                              paui32MapIndices,
+	                              uiLog2HeapPageSize,
+	                              uiMappingFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+ErrFreeValidArray:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(pbValid);
+	}
+ErrFreePAddrMappingArray:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+ErrReturnError:
+	return eError;
+}
+
+static PVRSRV_ERROR
+MMU_UnmapPagesUnlocked(MMU_CONTEXT *psMMUContext,
+                       PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                       IMG_DEV_VIRTADDR sDevVAddrBase,
+                       IMG_UINT32 ui32PageCount,
+                       IMG_UINT32 *pai32FreeIndices,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPTEIndex = 0, ui32Loop=0;
+	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+	IMG_HANDLE hPriv;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0, uiMMUReadOnlyProtFlags = 0;
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_DEV_PHYADDR sBackingPgDevPhysAddr;
+	IMG_BOOL bUnmap = IMG_TRUE, bScratchBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+#if defined(PDUMP)
+	const IMG_CHAR *pcBackingPageName = NULL;
+
+	PDUMPCOMMENT(psDevNode,
+	             "Invalidate %d entries in page tables for virtual range: 0x%010"IMG_UINT64_FMTSPECX" to 0x%010"IMG_UINT64_FMTSPECX,
+	             ui32PageCount,
+	             (IMG_UINT64)sDevVAddr.uiAddr,
+	             ((IMG_UINT64)sDevVAddr.uiAddr) + (uiPageSize*ui32PageCount)-1);
+#endif
+
+	PVR_ASSERT(OSLockIsLocked(psMMUContext->hLock));
+
+	bScratchBacking = PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiMemAllocFlags);
+	bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(uiMemAllocFlags);
+
+	if (bZeroBacking)
+	{
+		/* Ensure the zero backing page has been created */
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sBackingPgDevPhysAddr.uiAddr,
+		                             DEV_ZERO_PAGE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_MMU_GetBackingPage (zero)");
+	}
+	else
+	{
+		/* Ensure the scratch backing page has been created */
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sBackingPgDevPhysAddr.uiAddr,
+		                             SCRATCH_PAGE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_MMU_GetBackingPage (scratch)");
+	}
+
+#if defined(PDUMP)
+	if (bScratchBacking)
+	{
+		pcBackingPageName = bZeroBacking ? DEV_ZERO_PAGE_STR : SCRATCH_PAGE_STR;
+	}
+#endif
+
+	bUnmap = (uiMappingFlags)? !bScratchBacking : IMG_TRUE;
+
+	eError = _MMU_ConvertDevMemFlags(bUnmap,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Get PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+	uiMMUReadOnlyProtFlags = (uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE) | MMU_PROTFLAGS_READABLE;
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUReadOnlyProtFlags);
+	}
+	else if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUReadOnlyProtFlags, uiLog2PageSize);
+	}
+
+	/* Unmap page by page */
+	while (ui32Loop < ui32PageCount)
+	{
+		if (NULL != pai32FreeIndices)
+		{
+			/*Calculate the Device Virtual Address of the page */
+			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr +
+					pai32FreeIndices[ui32Loop] * (IMG_UINT64) uiPageSize;
+		}
+
+		psPrevLevel = psLevel;
+		/* Calculate PT index and get new table descriptor */
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND, e0);
+		}
+
+		if (psPrevLevel == psLevel)
+		{
+			/*
+			 * Sparse allocations may have page offsets which
+			 * decrement as well as increment, so make sure we
+			 * update the range we will flush correctly.
+			 */
+			if (uiPTEIndex > uiFlushEnd)
+				uiFlushEnd = uiPTEIndex;
+			else if (uiPTEIndex < uiFlushStart)
+				uiFlushStart = uiPTEIndex;
+		}
+		else
+		{
+			/* Flush if we moved to another psLevel, i.e. page table */
+			if (psPrevLevel != NULL)
+			{
+				PVRSRV_ERROR eError;
+				eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+				                   &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+				                   uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+				                   (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+				PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+			}
+
+			uiFlushStart = uiPTEIndex;
+			uiFlushEnd = uiFlushStart;
+		}
+
+		HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
+		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr));
+
+		/* Set the PT entry to invalid and poison it with a bad address */
+		eError = _SetupPTE(psMMUContext,
+		                   psLevel,
+		                   uiPTEIndex,
+		                   psConfig,
+		                   (bScratchBacking)? &sBackingPgDevPhysAddr : &gsBadDevPhyAddr,
+		                   &sDevVAddr,
+		                   bUnmap,
+#if defined(PDUMP)
+		                   (bScratchBacking)? (psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName): NULL,
+		                   pcBackingPageName,
+		                   0U,
+#endif
+		                   (bZeroBacking)? uiProtFlagsReadOnly: uiProtFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Check we haven't wrapped around */
+		PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+		ui32Loop++;
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		PVRSRV_ERROR eError;
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                   &psLevel->sMemDesc.psMapping->sMemHandle,
+		                   uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                   (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+e0:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+	PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map/unmap page table "
+	         "with error %u", __func__, eError));
+
+	return eError;
+}
+
+/*
+	MMU_UnmapPages
+ */
+PVRSRV_ERROR
+MMU_UnmapPages(MMU_CONTEXT *psMMUContext,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_DEV_VIRTADDR sDevVAddrBase,
+               IMG_UINT32 ui32PageCount,
+               IMG_UINT32 *pai32FreeIndices,
+               IMG_UINT32 uiLog2PageSize,
+               PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = MMU_UnmapPagesUnlocked(psMMUContext,
+	                                uiMappingFlags,
+	                                sDevVAddrBase,
+	                                ui32PageCount,
+	                                pai32FreeIndices,
+	                                uiLog2PageSize,
+	                                uiMemAllocFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+MMUX_MapVRangeToBackingPage(MMU_CONTEXT *psMMUContext,
+                          PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                          IMG_DEV_VIRTADDR sDevVAddrBase,
+                          IMG_UINT32 ui32MapPageCount,
+                          IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError;
+
+	IMG_UINT32 uiPageSize = (1 << uiLog2HeapPageSize);
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+	IMG_UINT32 uiPTEIndex = 0;
+
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+
+	IMG_UINT32 uiLoop = 0;
+
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	IMG_BOOL bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(uiMappingFlags);
+
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+
+	IMG_DEV_PHYADDR sDevPAddr;
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/*Get the Page table level configuration */
+	_MMU_GetPTConfig(psMMUContext,
+	                 (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig,
+	                 &hPriv,
+	                 &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, ErrPutPTConfig);
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE),
+		                                                                  uiLog2HeapPageSize);
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE));
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrPutPTConfig);
+	}
+
+	if (bZeroBacking)
+	{
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sDevPAddr.uiAddr,
+		                             DEV_ZERO_PAGE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+		                      ErrPutPTConfig);
+	}
+	else
+	{
+		eError = _MMU_GetBackingPage(psDevNode,
+									 &sDevPAddr.uiAddr,
+									 SCRATCH_PAGE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+							  ErrPutPTConfig);
+	}
+
+#if defined(DEBUG)
+	{
+		IMG_INT32	i32FeatureVal = 0;
+		IMG_UINT32 ui32BitLength = FloorLog2(sDevPAddr.uiAddr);
+
+		i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+		do {
+			/* i32FeatureVal can be negative for cases where this feature is undefined
+			 * In that situation we need to bail out than go ahead with debug comparison */
+			if (0 > i32FeatureVal)
+				break;
+
+			if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s Failed. The physical address bitlength (%u)"
+						" is greater than the chip can handle (%d).",
+						__func__, ui32BitLength, i32FeatureVal));
+
+				PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto ErrPutPTConfig;
+			}
+		} while (0);
+	}
+#endif /*DEBUG*/
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	for (uiLoop = 0; uiLoop < ui32MapPageCount; uiLoop++)
+	{
+		psPrevLevel = psLevel;
+		/* Calculate PT index and get new table descriptor */
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+			                    ErrUnlockAndUnmapPages);
+		}
+
+		if (psPrevLevel == psLevel)
+		{
+			/*
+			 * Sparse allocations may have page offsets which
+			 * decrement as well as increment, so make sure we
+			 * update the range we will flush correctly.
+			 */
+			if (uiPTEIndex > uiFlushEnd)
+				uiFlushEnd = uiPTEIndex;
+			else if (uiPTEIndex < uiFlushStart)
+				uiFlushStart = uiPTEIndex;
+		}
+		else
+		{
+			/* Flush if we moved to another psLevel, i.e. page table */
+			if (psPrevLevel != NULL)
+			{
+				eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+				                            &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+				                            uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+				                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+				PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+			}
+
+			uiFlushStart = uiPTEIndex;
+			uiFlushEnd = uiFlushStart;
+		}
+
+		HTBLOGK(HTB_SF_MMU_PAGE_OP_MAP,
+		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+		        HTBLOG_U64_BITS_HIGH(sDevPAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevPAddr.uiAddr));
+
+		/* Set the PT entry with the specified address and protection flags */
+		eError = _SetupPTE(psMMUContext,
+		                   psLevel,
+		                   uiPTEIndex,
+		                   psConfig,
+		                   &sDevPAddr,
+		                   &sDevVAddr,
+		                   IMG_FALSE,
+#if defined(PDUMP)
+		                   psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+		                   bZeroBacking ? DEV_ZERO_PAGE_STR : SCRATCH_PAGE_STR,
+		                   0,
+#endif /*PDUMP*/
+		                   bZeroBacking ? uiProtFlagsReadOnly : uiProtFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockAndUnmapPages);
+
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	(void) MMU_UnmapPagesUnlocked(psMMUContext,
+	                              0,
+	                              sDevVAddrBase,
+	                              uiLoop,
+	                              NULL,
+	                              uiLog2HeapPageSize,
+	                              uiMappingFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+	return eError;
+}
+
+PVRSRV_ERROR
+MMU_MapPMRFast(MMU_CONTEXT *psMMUContext,
+               IMG_DEV_VIRTADDR sDevVAddrBase,
+               PMR *psPMR,
+               IMG_DEVMEM_SIZE_T uiSizeBytes,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+
+	IMG_UINT32 i, uiChunkStart, uiLastPTEIndex, uiNumEntriesToWrite;
+	IMG_UINT32 ui32PagesDone=0, uiPTEIndex=0;
+
+	IMG_UINT8 uiAddrLog2Align, uiAddrShift, uiParityShift;
+	IMG_UINT64 uiAddrMask, uiProtFlags;
+	IMG_UINT32 uiBytesPerEntry;
+	IMG_UINT64 uiParityBit = 0;
+	IMG_BOOL bSetParity = IMG_FALSE;
+	IMG_DEV_VIRTADDR sDevVAddrRunning, sDevVAddrBaseCopy = sDevVAddrBase;
+
+	IMG_UINT64* pui64LevelBase;
+	IMG_UINT32* pui32LevelBase;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 uiNumPages = uiSizeBytes >> uiLog2HeapPageSize;
+
+
+#if defined(PVRSRV_ENABLE_HTB)
+	IMG_BOOL bHTBLog =
+	    HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_PMRMAP) ||
+	    HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_TABLE);
+#endif
+
+	IMG_BOOL bValidateOrTweak = psDevNode->pfnValidateOrTweakPhysAddrs ? IMG_TRUE : IMG_FALSE;
+
+#if defined(PDUMP)
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
+	IMG_UINT32 ui32MappedCount = 0;
+	IMG_DEV_VIRTADDR sDevVAddrRunningPdump;
+	PDUMPCOMMENT(psDevNode, "Wire up Page Table entries to point to the Data Pages (%d bytes)", uiNumPages << uiLog2HeapPageSize);
+#endif /*PDUMP*/
+
+	PVR_ASSERT (psMMUContext != NULL);
+	PVR_ASSERT (psPMR != NULL);
+	PVR_ASSERT((IMG_DEVMEM_OFFSET_T)uiNumPages << uiLog2HeapPageSize == uiSizeBytes);
+
+	/* Get general PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, put_mmu_context);
+
+	uiAddrLog2Align = psConfig->uiAddrLog2Align;
+	uiAddrShift = psConfig->uiAddrShift;
+	uiAddrMask = psConfig->uiAddrMask;
+	uiBytesPerEntry = psConfig->uiBytesPerEntry;
+
+	bSetParity = psConfig->uiParityBitMask;
+	uiParityShift = psConfig->uiParityBitShift;
+
+	sDevVAddrRunning.uiAddr = sDevVAddrBase.uiAddr;
+#if defined(PDUMP)
+	sDevVAddrRunningPdump.uiAddr = sDevVAddrBase.uiAddr;
+#endif
+	if (uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+	}
+	else if (uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_MMU_CONFIG_IS_WRONG, put_mmu_context);
+	}
+
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PMRMarkForDeferFree(psPMR);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	do
+	{
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddrBase, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+			                    unlock_mmu_context);
+		}
+
+		pui64LevelBase = (IMG_UINT64*)psLevel->sMemDesc.pvCpuVAddr;
+		pui32LevelBase = (IMG_UINT32*)psLevel->sMemDesc.pvCpuVAddr;
+
+		uiLastPTEIndex = MIN(uiPTEIndex + uiNumPages - ui32PagesDone, psDevVAddrConfig->uiNumEntriesPT);
+		uiNumEntriesToWrite = uiLastPTEIndex - uiPTEIndex;
+
+		for (uiChunkStart = 0; uiChunkStart < uiNumEntriesToWrite; uiChunkStart += PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			IMG_UINT32 uiNumPagesInBlock = MIN(uiNumEntriesToWrite - uiChunkStart, PMR_MAX_TRANSLATION_STACK_ALLOC);
+
+			/* With feature SUPPORT_LINUX_OSPAGE_MIGRATION:
+			 * This call is allowed to fail with retry part way through mapping.
+			 * Since this path is only used in Devicemem for fully contiguous PMR mappings
+			 * we know a PMR is always either fully mapped or not at all and cannot be
+			 * overwritten. On failure due to migrate we can unmap our progress and retry later.
+			 * Retry may be returned due to the combination of DEVICE_USE | MAPPING_USE.
+			 */
+			eError = PMR_DevPhysAddr(psPMR,
+			                         uiLog2HeapPageSize,
+			                         uiNumPagesInBlock,
+			                         (IMG_UINT64) (ui32PagesDone + uiChunkStart) << uiLog2HeapPageSize,
+			                         asDevPAddr,
+			                         abValid,
+			                         DEVICE_USE | MAPPING_USE);
+			PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+
+			if (bValidateOrTweak)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					PVRSRV_ERROR eError = psDevNode->pfnValidateOrTweakPhysAddrs(psDevNode,
+					                                                             psMMUContext->psDevAttrs,
+					                                                             &asDevPAddr[i].uiAddr);
+					PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+				}
+			}
+
+#if defined(DEBUG)
+			{
+				IMG_INT32 i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+
+				if (i32FeatureVal >= 0)
+				{
+					for (i=0; i<uiNumPagesInBlock; i++)
+					{
+						IMG_UINT32 ui32BitLength = FloorLog2(asDevPAddr[i].uiAddr);
+
+						if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+							        "%s Failed. The physical address bitlength (%u)"
+							        " is greater than the chip can handle (%d).",
+							        __func__, ui32BitLength, i32FeatureVal));
+
+							PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+							PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, unlock_mmu_context);
+						}
+					}
+				}
+			}
+#endif /*DEBUG*/
+
+			if (uiBytesPerEntry == 8)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					if (bSetParity)
+					{
+						uiParityBit = _GetParityBit(sDevVAddrRunning.uiAddr ^ asDevPAddr[i].uiAddr);
+						uiParityBit <<= uiParityShift;
+
+						sDevVAddrRunning.uiAddr += (1 << uiLog2HeapPageSize);
+					}
+
+					pui64LevelBase[uiPTEIndex + uiChunkStart + i] =
+						(((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags | uiParityBit;
+				}
+			}
+			else if (uiBytesPerEntry == 4)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+					/* assert that the result fits into 32 bits before writing
+					it into the 32-bit array with a cast */
+					IMG_UINT64 ui64PxE64 =
+					    (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+					PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+					PVR_ASSERT(!bSetParity);
+#endif
+
+					pui32LevelBase[uiPTEIndex + uiChunkStart + i] =
+					    (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+				}
+			}
+
+#if defined(PDUMP)
+			for (i=0; i<uiNumPagesInBlock; i++)
+			{
+				IMG_DEVMEM_OFFSET_T uiNextSymName;
+				IMG_UINT64 uiParityBit = psConfig->uiParityBitMask ? _GetParityBit(sDevVAddrRunningPdump.uiAddr ^ 0ULL) : 0;
+
+				eError = PMR_PDumpSymbolicAddr(psPMR, (ui32PagesDone + uiChunkStart + i) << uiLog2HeapPageSize,
+				                               sizeof(aszMemspaceName), &aszMemspaceName[0],
+				                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
+				                               &uiSymbolicAddrOffset,
+				                               &uiNextSymName);
+				PVR_LOG_IF_ERROR(eError, "PMR_PDumpSymbolicAddr");
+				ui32MappedCount++;
+
+				PDumpMMUDumpPxEntries(psDevNode,
+				                      MMU_LEVEL_1,
+				                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+				                      psLevel->sMemDesc.pvCpuVAddr,
+				                      psLevel->sMemDesc.sDevPAddr,
+				                      uiPTEIndex + uiChunkStart + i,
+				                      1,
+				                      aszMemspaceName,
+				                      aszSymbolicAddress,
+				                      uiSymbolicAddrOffset,
+				                      uiBytesPerEntry,
+				                      uiAddrLog2Align,
+				                      uiAddrShift,
+				                      uiAddrMask,
+				                      psConfig->uiProtMask,
+				                      psConfig->uiValidEnMask,
+				                      0,
+				                      uiParityBit,
+				                      psConfig->uiParityBitShift,
+				                      psConfig->uiParityBitMask,
+				                      psMMUContext->psDevAttrs->eMMUType);
+
+				sDevVAddrRunningPdump.uiAddr += (1 << uiLog2HeapPageSize);
+			}
+#endif /*PDUMP*/
+
+#if defined(PVRSRV_ENABLE_HTB)
+			if (bHTBLog)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					IMG_DEV_VIRTADDR sDevVAddr;
+					IMG_UINT64 ui64PxE64 = (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+					sDevVAddr.uiAddr = sDevVAddrBase.uiAddr + (ui32PagesDone + uiChunkStart + i)  * (1 << uiLog2HeapPageSize);
+
+					if (bSetParity)
+					{
+						ui64PxE64 |= _GetParityBit(sDevVAddr.uiAddr ^ asDevPAddr[i].uiAddr) << uiParityShift;
+					}
+
+					HTBLOGK(HTB_SF_MMU_PAGE_OP_PMRMAP,
+					        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+					        HTBLOG_U64_BITS_HIGH(asDevPAddr[i].uiAddr), HTBLOG_U64_BITS_LOW(asDevPAddr[i].uiAddr));
+
+					HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+					        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+					        uiPTEIndex + uiChunkStart + i, MMU_LEVEL_1,
+					        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+					        IMG_FALSE);
+				}
+			}
+#endif
+		}
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiPTEIndex * uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiNumEntriesToWrite) * uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+
+		sDevVAddrBase.uiAddr += uiNumEntriesToWrite * (1 << uiLog2HeapPageSize);
+		ui32PagesDone += uiNumEntriesToWrite;
+
+	} while (ui32PagesDone < uiNumPages);
+
+	OSLockRelease(psMMUContext->hLock);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Wired up %d Page Table entries (out of %d)", ui32MappedCount, uiNumPages);
+#endif /*PDUMP*/
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return PVRSRV_OK;
+
+unlock_mmu_context:
+	/* Unmap starting from the address passed as an argument. */
+	(void) MMU_UnmapPMRFastUnlocked(psMMUContext,
+	                                sDevVAddrBaseCopy,
+	                                uiNumPages,
+	                                uiLog2HeapPageSize);
+	OSLockRelease(psMMUContext->hLock);
+put_mmu_context:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+MMU_UnmapPMRFastUnlocked(MMU_CONTEXT *psMMUContext,
+                         IMG_DEV_VIRTADDR sDevVAddrBase,
+                         IMG_UINT32 ui32PageCount,
+                         IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiPTEIndex = 0, uiLastPTEIndex = 0, ui32PagesDone=0, i, uiNumEntriesToWrite;
+	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
+	MMU_Levelx_INFO *psLevel = NULL;
+	IMG_HANDLE hPriv;
+	void* pvPTStart;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_UINT64 uiProtFlags = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	IMG_UINT64 uiEntry = 0;
+
+	IMG_UINT64 uiParityBit = 0;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_DEV_VIRTADDR sDevVAddrStartOfTable;
+	IMG_UINT32 uiParityPatternIdx;
+	IMG_BOOL bSetParity = IMG_FALSE;
+	IMG_DEV_VIRTADDR sDevVAddrRunning;
+	IMG_UINT64 ui64BadPhysAddrParity = 0;
+#endif
+	IMG_UINT64 ui64BadPhysAddr = 0;
+
+#if defined(PVRSRV_ENABLE_HTB)
+	IMG_BOOL bLog;
+#endif
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	PVR_ASSERT(OSLockIsLocked(psMMUContext->hLock));
+
+	eError = _MMU_ConvertDevMemFlags(IMG_TRUE,
+	                                 0,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Get PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	bSetParity = psConfig->uiParityBitMask != 0;
+	sDevVAddrRunning.uiAddr = sDevVAddrBase.uiAddr;
+#endif
+
+	/* Callback to get device specific protection flags */
+
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);
+		ui64BadPhysAddr = gsBadDevPhyAddr.uiAddr & ~(psConfig->uiProtMask | psConfig->uiParityBitMask);
+		/* Fill the entry with a bad address but leave space for protection flags and parity bit */
+		uiEntry = ui64BadPhysAddr | uiProtFlags;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		ui64BadPhysAddrParity = _GetParityBit(ui64BadPhysAddr) << psConfig->uiParityBitShift;
+#endif
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		PVR_ASSERT(!bSetParity);
+#endif
+		/* Fill the entry with a bad address but leave space for protection flags */
+		uiEntry = (((IMG_UINT32) gsBadDevPhyAddr.uiAddr) & ~psConfig->uiProtMask) | (IMG_UINT32) uiProtFlags;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: The page table entry byte length is not supported",
+				__func__));
+		goto e0;
+	}
+
+#if defined(PVRSRV_ENABLE_HTB)
+	bLog = HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_UNMAP) ||
+		   HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_TABLE);
+#endif
+
+	do
+	{
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND, e0);
+		}
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		sDevVAddrStartOfTable.uiAddr = sDevVAddr.uiAddr - uiPTEIndex * (1 << uiLog2PageSize);
+		uiParityPatternIdx = _GetParityBit(sDevVAddrStartOfTable.uiAddr);
+#endif
+		pvPTStart = psLevel->sMemDesc.pvCpuVAddr;
+
+		uiLastPTEIndex = MIN(uiPTEIndex + ui32PageCount - ui32PagesDone, psDevVAddrConfig->uiNumEntriesPT);
+		uiNumEntriesToWrite = uiLastPTEIndex - uiPTEIndex;
+
+		if (psConfig->uiBytesPerEntry == 8)
+		{
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+				if (bSetParity)
+				{
+					uiParityBit = psMMUContext->psDevAttrs->pui64PrecomputedAllocParity[uiParityPatternIdx][i] ^ ui64BadPhysAddrParity;
+					sDevVAddrRunning.uiAddr += (1 << uiLog2PageSize);
+				}
+#endif
+				((IMG_UINT64*)pvPTStart)[i] = uiEntry | uiParityBit;
+			}
+		}
+		else
+		{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+			PVR_ASSERT(psConfig->uiBytesPerEntry == 4);
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+			PVR_ASSERT(!bSetParity);
+#endif
+#endif
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+				((IMG_UINT32*)pvPTStart)[i] = (IMG_UINT32) uiEntry;
+			}
+		}
+
+#if defined(PVRSRV_ENABLE_HTB)
+		/* Log separately to avoid polluting the fast path */
+		if (bLog)
+		{
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+				IMG_DEV_VIRTADDR sDevVAddrToLog;
+				sDevVAddrToLog.uiAddr = sDevVAddr.uiAddr + (i - uiPTEIndex) * uiPageSize;
+
+				/* Log modifications */
+				HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
+						HTBLOG_U64_BITS_HIGH(sDevVAddrToLog.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrToLog.uiAddr));
+
+				HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+						HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+						uiPTEIndex, MMU_LEVEL_1,
+						HTBLOG_U64_BITS_HIGH(uiEntry), HTBLOG_U64_BITS_LOW(uiEntry),
+						IMG_FALSE);
+			}
+		}
+#endif
+
+#if defined(PDUMP)
+		PDumpMMUDumpPxEntries(psDevNode,
+		                      MMU_LEVEL_1,
+		                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+		                      psLevel->sMemDesc.pvCpuVAddr,
+		                      psLevel->sMemDesc.sDevPAddr,
+		                      uiPTEIndex,
+		                      uiNumEntriesToWrite,
+		                      NULL,
+		                      NULL,
+		                      0,
+		                      psConfig->uiBytesPerEntry,
+		                      psConfig->uiAddrLog2Align,
+		                      psConfig->uiAddrShift,
+		                      psConfig->uiAddrMask,
+		                      psConfig->uiProtMask,
+		                      psConfig->uiValidEnMask,
+		                      0,
+		                      0, /* Unused - Parity bit values are taken directly from PTE memory when bad phys addr is used */
+		                      0, /* Unused */
+		                      0, /* Unused */
+		                      psMMUContext->psDevAttrs->eMMUType);
+#endif /*PDUMP*/
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+						&psLevel->sMemDesc.psMapping->sMemHandle,
+						uiPTEIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+						(uiNumEntriesToWrite) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+
+		sDevVAddr.uiAddr += uiNumEntriesToWrite * uiPageSize;
+		ui32PagesDone += uiNumEntriesToWrite;
+
+	} while (ui32PagesDone < ui32PageCount);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+e0:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map/unmap page table with error %u",
+	         __func__, eError));
+
+	return eError;
+}
+
+/*
+    MMU_UnmapPMRFast
+ */
+PVRSRV_ERROR
+MMU_UnmapPMRFast(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrBase,
+                 IMG_UINT32 ui32PageCount,
+                 IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = MMU_UnmapPMRFastUnlocked(psMMUContext,
+                                      sDevVAddrBase,
+                                      ui32PageCount,
+                                      uiLog2PageSize);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	return eError;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+PVRSRV_ERROR
+MMU_RemapPage(MMU_CONTEXT *psMMUContext,
+              PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+              IMG_DEV_VIRTADDR sDevVAddr,
+              IMG_UINT32 uiLog2HeapPageSize,
+              PMR *psOriginPMR,
+              IMG_UINT32 ui32LogicalPgOffset)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	const MMU_PxE_CONFIG *psConfig;
+	IMG_HANDLE hPriv;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT32 uiPTEIndex = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+
+	IMG_UINT64 uiProtFlags = 0;
+	IMG_UINT64 ui64PrevPxE64;
+	IMG_UINT32 ui32PrevPxE32;
+
+	IMG_DEV_PHYADDR sTargetDevPAddr;
+	IMG_BOOL bValid = IMG_FALSE;
+
+	/* Validate the most essential parameters */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext != NULL, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psOriginPMR != NULL, "psPMR");
+
+	psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+	_MMU_GetPTConfig(psMMUContext, uiLog2HeapPageSize, &psConfig, &hPriv, &psDevVAddrConfig);
+
+	if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+	                    &psLevel, &uiPTEIndex))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_MMU_GetPTInfo", eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+		                        ErrPutPTConfig);
+	}
+
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Px = psLevel->sMemDesc.pvCpuVAddr;
+		ui32PrevPxE32 = pui32Px[uiPTEIndex];
+
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+	}
+	else if (psConfig->uiBytesPerEntry == 8)
+	{
+		IMG_UINT64 *pui64Px = psLevel->sMemDesc.pvCpuVAddr;
+		ui64PrevPxE64 = pui64Px[uiPTEIndex];
+
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+	}
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = PMR_DevPhysAddr(psOriginPMR,
+	                         uiLog2HeapPageSize,
+	                         1,
+	                         ((IMG_DEVMEM_OFFSET_T) ui32LogicalPgOffset << uiLog2HeapPageSize),
+	                         &sTargetDevPAddr,
+	                         &bValid,
+	                         DEVICE_USE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", ErrUnlockContext);
+
+	if (!bValid)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("PMR_DevPhysAddr",
+		                        eError,
+		                        PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE,
+		                        ErrUnlockContext);
+	}
+
+	/* Set the PT entry with the specified address and protection flags */
+	eError = _SetupPTE(psMMUContext,
+	                   psLevel,
+	                   uiPTEIndex,
+	                   psConfig,
+	                   &sTargetDevPAddr,
+	                   &sDevVAddr,
+	                   IMG_FALSE,
+#if defined(PDUMP)
+	                   /*PDUMP Not supported for remap */
+	                   NULL,
+	                   NULL,
+	                   0,
+#endif /*PDUMP*/
+	                   uiProtFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockContext);
+
+	eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+	                            &psLevel->sMemDesc.psMapping->sMemHandle,
+	                            uiPTEIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+	                            psConfig->uiBytesPerEntry);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapPagesClean", ErrUnlockAndUnmapPages);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs, this queues invalidation, actioning of this
+	 * should be ensured complete by the caller before further GPU
+	 * workloads continue.
+	 */
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	/* Reset old PTE in place */
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Px = psLevel->sMemDesc.pvCpuVAddr;
+		pui32Px[uiPTEIndex] = ui32PrevPxE32;
+	}
+	else if (psConfig->uiBytesPerEntry == 8)
+	{
+		IMG_UINT64 *pui64Px = psLevel->sMemDesc.pvCpuVAddr;
+		pui64Px[uiPTEIndex] = ui64PrevPxE64;
+	}
+
+ErrUnlockContext:
+	OSLockRelease(psMMUContext->hLock);
+
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return eError;
+}
+#endif
+
+/*
+	MMU_AcquireBaseAddr
+ */
+PVRSRV_ERROR
+MMU_AcquireBaseAddr(MMU_CONTEXT *psMMUContext, IMG_DEV_PHYADDR *psPhysAddr)
+{
+	if (!psMMUContext)
+	{
+		psPhysAddr->uiAddr = 0;
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_NOT_SUPPORTED,
+		                        "MMU_AcquireBaseAddr");
+	}
+
+	*psPhysAddr = psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_AcquireCPUBaseAddr
+ */
+PVRSRV_ERROR
+MMU_AcquireCPUBaseAddr(MMU_CONTEXT *psMMUContext, void **ppvCPUVAddr)
+{
+	if (!psMMUContext)
+	{
+		*ppvCPUVAddr = NULL;
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_NOT_SUPPORTED,
+		                        "MMU_AcquireCPUBaseAddr");
+	}
+
+	*ppvCPUVAddr = psMMUContext->sBaseLevelInfo.sMemDesc.pvCpuVAddr;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_ReleaseBaseAddr
+ */
+void
+MMU_ReleaseBaseAddr(MMU_CONTEXT *psMMUContext)
+{
+	PVR_UNREFERENCED_PARAMETER(psMMUContext);
+}
+
+/*
+	MMU_AppendCacheFlags
+*/
+
+void MMU_AppendCacheFlags(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32AppendFlags)
+{
+	PVR_ASSERT(psMMUContext != NULL);
+
+	if (psMMUContext == NULL)
+	{
+		return;
+	}
+
+	OSAtomicOr(&psMMUContext->sCacheFlags, (IMG_INT)ui32AppendFlags);
+}
+
+/*
+	MMU_GetAndResetCacheFlags
+*/
+IMG_UINT32 MMU_GetAndResetCacheFlags(MMU_CONTEXT *psMMUContext)
+{
+	IMG_UINT32 uiFlags;
+
+	PVR_ASSERT(psMMUContext != NULL);
+	if (psMMUContext == NULL)
+	{
+		return 0;
+	}
+
+	uiFlags = (IMG_UINT32) OSAtomicExchange(&psMMUContext->sCacheFlags, 0);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* kick cleanup thread to free all zombie PMRs residing in the device's
+	 * zombie list */
+	if (PMRQueueZombiesForCleanup(psMMUContext->psPhysMemCtx->psDevNode))
+	{
+		BITMASK_SET(uiFlags, RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL | RGXFWIF_MMUCACHEDATA_FLAGS_PT);
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	return uiFlags;
+}
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+/*
+    MMU_GetOSids
+ */
+
+void MMU_GetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 *pui32OSid, IMG_UINT32 *pui32OSidReg, IMG_BOOL *pbOSidAxiProt)
+{
+	*pui32OSid     = psMMUContext->psPhysMemCtx->ui32OSid;
+	*pui32OSidReg  = psMMUContext->psPhysMemCtx->ui32OSidReg;
+	*pbOSidAxiProt = psMMUContext->psPhysMemCtx->bOSidAxiProt;
+}
+
+#endif
+
+static IMG_BOOL _MMUGetPxEFaultLevelData(const MMU_CONTEXT *psMMUContext,
+		                                 IMG_DEV_VIRTADDR *psDevVAddr,
+		                                 const MMU_PxE_CONFIG *psConfig,
+		                                 void *pvCpuVAddr,
+		                                 IMG_UINT32 ui32PxIndex,
+		                                 MMU_LEVEL_DATA *psMMULevelDataOut,
+		                                 IMG_UINT32 *ui32Log2PageSizeOut)
+{
+	static const IMG_CHAR *apszMMUValidStr[4] =  {
+			/*--*/ "not valid",
+			/*-V*/ "valid",
+			/*P-*/ "pending",
+			/*PV*/ "inconsistent (pending and valid)"  };
+	#define _MMU_VALID_STR(_cfg, _entry) \
+		(apszMMUValidStr[((((_entry)&(_cfg->uiPendingEnMask))!=0) << 1)| \
+		                 ((((_entry)&(_cfg->uiValidEnMask))!=0) << 0)])
+	#define _MMU_MASK_VALID_FOR_32BITS(_cfg) \
+		((_cfg->uiPendingEnMask | _cfg->uiValidEnMask) <= 0xFFFFFFFF)
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Ptr = pvCpuVAddr;
+
+		psMMULevelDataOut->ui64Address = pui32Ptr[ui32PxIndex];
+
+		/* Check if we are dealing with a PxE entry where these bits are in the first 32 bits */
+		/* But if we know it is a 4 byte entry, why check this? */
+		if (_MMU_MASK_VALID_FOR_32BITS(psConfig))
+		{
+			psMMULevelDataOut->psDebugStr = _MMU_VALID_STR(psConfig, pui32Ptr[ui32PxIndex] & psConfig->uiProtMask);
+		}
+		else
+		{
+			psMMULevelDataOut->psDebugStr = "";
+			PVR_DPF((PVR_DBG_ERROR, "Invalid %sE masks for 32-bit entry", psConfig->pszPxLevelStr));
+			return IMG_FALSE;
+		}
+
+		if (ui32Log2PageSizeOut != NULL)
+		{
+			if (psDevAttrs->pfnGetPageSizeFromPDE4(pui32Ptr[ui32PxIndex], ui32Log2PageSizeOut) != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from the %sE", psConfig->pszPxLevelStr));
+				return IMG_FALSE;
+			}
+		}
+	}
+	else
+	{
+		IMG_UINT64 *pui64Ptr = pvCpuVAddr;
+
+		psMMULevelDataOut->ui64Address = pui64Ptr[ui32PxIndex];
+		psMMULevelDataOut->psDebugStr  = _MMU_VALID_STR(psConfig, pui64Ptr[ui32PxIndex] & psConfig->uiProtMask);
+
+		if (ui32Log2PageSizeOut != NULL)
+		{
+			/* MMU_VERSION >= 4 */
+			if (psDevAttrs->pfnGetPageSizeFromVirtAddr != NULL)
+			{
+				if (psDevAttrs->pfnGetPageSizeFromVirtAddr(psMMUContext->psPhysMemCtx->psDevNode, *psDevVAddr, ui32Log2PageSizeOut) != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from device VA"));
+					return IMG_FALSE;
+				}
+			}
+			/* MMU_VERSION < 4 */
+			else if (psDevAttrs->pfnGetPageSizeFromPDE8(pui64Ptr[ui32PxIndex], ui32Log2PageSizeOut) != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from the %sE", psConfig->pszPxLevelStr));
+				return IMG_FALSE;
+			}
+		}
+	}
+
+	return IMG_TRUE;
+}
+
+/*
+	MMU_CheckFaultAddress
+ */
+void MMU_CheckFaultAddress(MMU_CONTEXT *psMMUContext,
+                           IMG_DEV_VIRTADDR *psDevVAddr,
+                           MMU_FAULT_DATA *psOutFaultData)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	MMU_LEVEL	eMMULevel = psDevAttrs->psBaseConfig->ePxLevel;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_PxE_CONFIG *psMMUPDEConfig;
+	const MMU_PxE_CONFIG *psMMUPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psMMUDevVAddrConfig;
+	IMG_HANDLE hPriv;
+	MMU_Levelx_INFO *psLevel = NULL;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 uiIndex;
+	IMG_UINT32 ui32PCIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32PDIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32PTIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32Log2PageSize;
+	MMU_FAULT_DATA sMMUFaultData = {0};
+	MMU_LEVEL_DATA *psMMULevelData;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	/*
+		At this point we don't know the page size so assume it's 4K.
+		When we get the PD level (MMU_LEVEL_2) we can check to see
+		if this assumption is correct.
+	 */
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(12,
+	                                                 &psMMUPDEConfig,
+	                                                 &psMMUPTEConfig,
+	                                                 &psMMUDevVAddrConfig,
+	                                                 &hPriv);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("Failed to get the page size info for log2 page sizeof 12"));
+	}
+
+	psLevel = &psMMUContext->sBaseLevelInfo;
+	psConfig = psDevAttrs->psBaseConfig;
+
+	sMMUFaultData.eTopLevel = psDevAttrs->psBaseConfig->ePxLevel;
+	sMMUFaultData.eType = MMU_FAULT_TYPE_NON_PM;
+
+
+	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
+	{
+		if (eMMULevel == MMU_LEVEL_3)
+		{
+			/* Determine the PC index */
+			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexMask;
+			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexShift;
+			ui32PCIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PCIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_3];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PCIndex;
+
+			if (ui32PCIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PCIndex,
+					                        psMMULevelData, NULL);
+
+			psLevel = psLevel->apsNextLevel[ui32PCIndex];
+			if (!psLevel)
+			{
+				break;
+			}
+			psConfig = psMMUPDEConfig;
+			continue; /* continue to the next level */
+		}
+
+
+		if (eMMULevel == MMU_LEVEL_2)
+		{
+			/* Determine the PD index */
+			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexMask;
+			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexShift;
+			ui32PDIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PDIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_2];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PDIndex;
+
+			if (ui32PDIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PDIndex,
+					                        psMMULevelData, &ui32Log2PageSize);
+
+			/*
+				We assumed the page size was 4K, now we have the actual size
+				from the PDE we can confirm if our assumption was correct.
+				Until now it hasn't mattered as the PC and PD are the same
+				regardless of the page size
+			 */
+			if (ui32Log2PageSize != 12)
+			{
+				/* Put the 4K page size data */
+				psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+
+				/* Get the correct size data */
+				eError = psDevAttrs->pfnGetPageSizeConfiguration(ui32Log2PageSize,
+				                                                 &psMMUPDEConfig,
+				                                                 &psMMUPTEConfig,
+				                                                 &psMMUDevVAddrConfig,
+				                                                 &hPriv);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_LOG(("Failed to get the page size info for log2 page sizeof %d", ui32Log2PageSize));
+					break;
+				}
+			}
+
+			psLevel = psLevel->apsNextLevel[ui32PDIndex];
+			if (!psLevel)
+			{
+				break;
+			}
+			psConfig = psMMUPTEConfig;
+			continue; /* continue to the next level */
+		}
+
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			/* Determine the PT index */
+			uiIndex = psDevVAddr->uiAddr & psMMUDevVAddrConfig->uiPTIndexMask;
+			uiIndex = uiIndex >> psMMUDevVAddrConfig->uiPTIndexShift;
+			ui32PTIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PTIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_1];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PTIndex;
+
+			if (ui32PTIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PTIndex,
+					                        psMMULevelData, NULL);
+			break;
+		}
+
+		PVR_LOG(("Unsupported MMU setup: %d", eMMULevel));
+		break;
+	}
+
+	/* Put the page size data back */
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+	OSLockRelease(psMMUContext->hLock);
+
+	*psOutFaultData = sMMUFaultData;
+}
+
+static IMG_UINT64 MMU_GetVDevAddrPTE(MMU_CONTEXT *psMMUContext,
+                                     const MMU_PxE_CONFIG *psConfig,
+                                     const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                                     IMG_DEV_VIRTADDR sDevVAddr,
+                                     IMG_BOOL *pbStatusOut)
+{
+	MMU_Levelx_INFO *psLevel = NULL;
+	IMG_UINT32 uiIndex = 0;
+	IMG_BOOL bStatus = IMG_FALSE;
+	IMG_UINT64 ui64Entry = 0;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	switch (psMMUContext->psDevAttrs->psBaseConfig->ePxLevel)
+	{
+		case MMU_LEVEL_3:
+			uiIndex = _CalcPCEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+			psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];
+			if (psLevel == NULL)
+				break;
+
+			fallthrough;
+		case MMU_LEVEL_2:
+			uiIndex = _CalcPDEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLevel != NULL)
+				psLevel = psLevel->apsNextLevel[uiIndex];
+			else
+				psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];
+
+			if (psLevel == NULL)
+				break;
+
+			fallthrough;
+		case MMU_LEVEL_1:
+			uiIndex = _CalcPTEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLevel == NULL)
+				psLevel = &psMMUContext->sBaseLevelInfo;
+
+			ui64Entry = ((IMG_UINT64 *)psLevel->sMemDesc.pvCpuVAddr)[uiIndex];
+			bStatus = ui64Entry & psConfig->uiValidEnMask;
+
+			break;
+		default:
+			PVR_LOG(("MMU_IsVDevAddrValid: Unsupported MMU setup"));
+			break;
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	*pbStatusOut = bStatus;
+
+	return ui64Entry;
+}
+
+IMG_BOOL MMU_IsVDevAddrValid(MMU_CONTEXT *psMMUContext,
+                             IMG_UINT32 uiLog2PageSize,
+                             IMG_DEV_VIRTADDR sDevVAddr)
+{
+	IMG_BOOL bStatus;
+	const MMU_PxE_CONFIG *psConfig;
+	IMG_HANDLE hPriv;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	_MMU_GetPTConfig(psMMUContext, uiLog2PageSize, &psConfig, &hPriv, &psDevVAddrConfig);
+
+	MMU_GetVDevAddrPTE(psMMUContext,
+	                   psConfig,
+	                   psDevVAddrConfig,
+	                   sDevVAddr,
+	                   &bStatus);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return bStatus;
+}
+
+#if defined(PDUMP)
+/*
+	MMU_ContextDerivePCPDumpSymAddr
+ */
+PVRSRV_ERROR MMU_ContextDerivePCPDumpSymAddr(MMU_CONTEXT *psMMUContext,
+                                             IMG_CHAR *pszPDumpSymbolicNameBuffer,
+                                             size_t uiPDumpSymbolicNameBufferSize)
+{
+	size_t uiCount;
+	IMG_UINT64 ui64PhysAddr;
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	if (!psMMUContext->sBaseLevelInfo.sMemDesc.bValid)
+	{
+		/* We don't have any allocations.  You're not allowed to ask
+		 * for the page catalogue base address until you've made at
+		 * least one allocation.
+		 */
+		return PVRSRV_ERROR_MMU_API_PROTOCOL_ERROR;
+	}
+
+	ui64PhysAddr = (IMG_UINT64)psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr.uiAddr;
+
+	PVR_ASSERT(uiPDumpSymbolicNameBufferSize >= (IMG_UINT32)(21 + OSStringLength(psDevId->pszPDumpDevName)));
+
+	/* Page table Symbolic Name is formed from page table phys addr
+	   prefixed with MMUPT_. */
+	uiCount = OSSNPrintf(pszPDumpSymbolicNameBuffer,
+	                     uiPDumpSymbolicNameBufferSize,
+	                     ":%s:%s%016"IMG_UINT64_FMTSPECX,
+	                     psDevId->pszPDumpDevName,
+	                     psMMUContext->sBaseLevelInfo.sMemDesc.bValid?"MMUPC_":"XXX",
+	                     ui64PhysAddr);
+
+	if (uiCount + 1 > uiPDumpSymbolicNameBufferSize)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_PDumpWritePageCatBase
+ */
+PVRSRV_ERROR
+MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                          const IMG_CHAR *pszSpaceName,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32WordSize,
+                          IMG_UINT32 ui32AlignShift,
+                          IMG_UINT32 ui32Shift,
+                          PDUMP_FLAGS_T uiPdumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszPageCatBaseSymbolicAddr[100];
+	const IMG_CHAR *pszPDumpDevName = psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName;
+
+	eError = MMU_ContextDerivePCPDumpSymAddr(psMMUContext,
+	                                         &aszPageCatBaseSymbolicAddr[0],
+	                                         sizeof(aszPageCatBaseSymbolicAddr));
+	if (eError == PVRSRV_OK)
+	{
+		eError = PDumpWriteSymbAddress(psMMUContext->psPhysMemCtx->psDevNode,
+		                               pszSpaceName,
+		                               uiOffset,
+		                               aszPageCatBaseSymbolicAddr,
+		                               0, /* offset -- Could be non-zero for var. pgsz */
+		                               pszPDumpDevName,
+		                               ui32WordSize,
+		                               ui32AlignShift,
+		                               ui32Shift,
+		                               uiPdumpFlags | PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	return eError;
+}
+
+/*
+	MMU_AcquirePDumpMMUContext
+ */
+PVRSRV_ERROR MMU_AcquirePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                                        IMG_UINT32 *pui32PDumpMMUContextID,
+                                        IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	if (!psMMUContext->ui32PDumpContextIDRefCount)
+	{
+		PDUMP_MMU_ALLOC_MMUCONTEXT(psMMUContext->psPhysMemCtx->psDevNode,
+		                           psDevId->pszPDumpDevName,
+		                           psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr,
+		                           psMMUContext->psDevAttrs->eMMUType,
+		                           &psMMUContext->uiPDumpContextID,
+		                           ui32PDumpFlags);
+	}
+
+	psMMUContext->ui32PDumpContextIDRefCount++;
+	*pui32PDumpMMUContextID = psMMUContext->uiPDumpContextID;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_ReleasePDumpMMUContext
+ */
+PVRSRV_ERROR MMU_ReleasePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+	                                        IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	PVR_ASSERT(psMMUContext->ui32PDumpContextIDRefCount != 0);
+	psMMUContext->ui32PDumpContextIDRefCount--;
+
+	if (psMMUContext->ui32PDumpContextIDRefCount == 0)
+	{
+		PDUMP_MMU_FREE_MMUCONTEXT(psMMUContext->psPhysMemCtx->psDevNode,
+		                          psDevId->pszPDumpDevName,
+		                          psMMUContext->uiPDumpContextID,
+		                          ui32PDumpFlags);
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+PVRSRV_ERROR MMU_CacheInvalidateKick(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                     IMG_UINT32 *puiRequiredSyncValue)
+{
+	IMG_UINT32 uiRequiredSyncValue;
+	PVRSRV_ERROR eError;
+
+	eError = psDeviceNode->pfnMMUCacheInvalidateKick(psDeviceNode, &uiRequiredSyncValue);
+
+	if (puiRequiredSyncValue != NULL)
+	{
+		*puiRequiredSyncValue = uiRequiredSyncValue;
+	}
+
+	return eError;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/******************************************************************************
+ End of file (mmu_common.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_mmu.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_mmu.c
new file mode 100644
index 000000000000..64a5740f52f5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_mmu.c
@@ -0,0 +1,946 @@
+/*************************************************************************/ /*!
+@File
+@Title          MMU PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common PDump (MMU specific) functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if defined(PDUMP)
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pdump_mmu.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+
+#define MAX_PDUMP_MMU_CONTEXTS	(10)
+static IMG_UINT32 guiPDumpMMUContextAvailabilityMask = (1<<MAX_PDUMP_MMU_CONTEXTS)-1;
+
+
+#define MMUPX_FMT(X) ((X<3) ? ((X<2) ?  "MMUPT_\0" : "MMUPD_\0") : "MMUPC_\0")
+#define MIPSMMUPX_FMT(X) ((X<3) ? ((X<2) ?  "MIPSMMUPT_\0" : "MIPSMMUPD_\0") : "MIPSMMUPC_\0")
+
+
+/* Array used to look-up debug strings from MMU_LEVEL */
+static const IMG_CHAR * const apszMMULevelStringLookup[MMU_LEVEL_LAST] =
+{
+	"MMU_LEVEL_0",
+	"PAGE_TABLE",
+	"PAGE_DIRECTORY",
+	"PAGE_CATALOGUE",
+};
+
+static PVRSRV_ERROR
+_ContiguousPDumpBytes(PVRSRV_DEVICE_NODE *psDeviceNode,
+		      const IMG_CHAR *pszSymbolicName,
+		      IMG_UINT32 ui32SymAddrOffset,
+		      IMG_BOOL bFlush,
+		      IMG_UINT32 uiNumBytes,
+		      void *pvBytes,
+		      IMG_UINT32 ui32Flags)
+{
+	static const IMG_CHAR *pvBeyondLastPointer;
+	static const IMG_CHAR *pvBasePointer;
+	static IMG_UINT32 ui32BeyondLastOffset;
+	static IMG_UINT32 ui32BaseOffset;
+	static IMG_UINT32 uiAccumulatedBytes;
+	IMG_UINT32 ui32ParamOutPos;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_AND_FILE_STRING();
+
+	/* Caller has PDUMP_LOCK */
+
+	if (!bFlush && uiAccumulatedBytes > 0)
+	{
+		/* do some tests for contiguity.  If it fails, we flush anyway */
+		if (pvBeyondLastPointer != pvBytes ||
+		    ui32SymAddrOffset != ui32BeyondLastOffset
+		    /* NB: ought to check that symbolic name agrees too, but
+		       we know this always to be the case in the current use-case */
+			)
+		{
+			bFlush = IMG_TRUE;
+		}
+	}
+
+	/* Flush if necessary */
+	if (bFlush && uiAccumulatedBytes > 0)
+	{
+		eErr = PDumpWriteParameter(psDeviceNode,
+					   (IMG_UINT8 *)(uintptr_t)pvBasePointer,
+					   uiAccumulatedBytes, ui32Flags,
+					   &ui32ParamOutPos, pszFileName);
+		if (eErr == PVRSRV_OK)
+		{
+			eErr = PDumpSNPrintf(hScript, ui32MaxLenScript,
+					     "LDB %s:0x%X 0x%X 0x%X %s",
+					     /* dest */
+					     pszSymbolicName,
+					     ui32BaseOffset,
+					     /* size */
+					     uiAccumulatedBytes,
+					     /* file offset */
+					     ui32ParamOutPos,
+					     /* filename */
+					     pszFileName);
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpSNPrintf", ErrOut);
+
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+		else if (eErr != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+		{
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpWriteParameter", ErrOut);
+		}
+		else
+		{
+			/* else Write to parameter file prevented under the flags and
+			 * current state of the driver so skip write to script and error IF.
+			 * this is normal e.g. no in capture range for example.
+			 */
+			eErr = PVRSRV_OK;
+		}
+
+		uiAccumulatedBytes = 0;
+	}
+
+	/* Initialise offsets and pointers if necessary */
+	if (uiAccumulatedBytes == 0)
+	{
+		ui32BaseOffset = ui32BeyondLastOffset = ui32SymAddrOffset;
+		pvBeyondLastPointer = pvBasePointer = (const IMG_CHAR *)pvBytes;
+	}
+
+	/* Accumulate some bytes */
+	ui32BeyondLastOffset += uiNumBytes;
+	pvBeyondLastPointer += uiNumBytes;
+	uiAccumulatedBytes += uiNumBytes;
+
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_AND_FILE_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUMalloc
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+			    const IMG_CHAR *pszPDumpDevName,
+			    MMU_LEVEL eMMULevel,
+			    IMG_DEV_PHYADDR *psDevPAddr,
+			    IMG_UINT32 ui32Size,
+			    IMG_UINT32 ui32Align,
+			    PDUMP_MMU_TYPE eMMUType)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA;
+	IMG_UINT64 ui64SymbolicAddr;
+	IMG_CHAR *pszMMUPX;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_GOTO_IF_INVALID_PARAM(eMMULevel < MMU_LEVEL_LAST, eErr, ErrOut);
+
+	PDUMP_LOCK(ui32Flags);
+
+	/*
+	  Write a comment to the PDump2 script streams indicating the memory allocation
+	*/
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "-- CALLOC :%s:%s Size=0x%08X Alignment=0x%08X 0x0 DevPAddr=0x%08"IMG_UINT64_FMTSPECX,
+			     pszPDumpDevName,
+			     apszMMULevelStringLookup[eMMULevel],
+			     ui32Size,
+			     ui32Align,
+			     psDevPAddr->uiAddr);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	/*
+	  construct the symbolic address
+	*/
+	ui64SymbolicAddr = (IMG_UINT64)psDevPAddr->uiAddr;
+
+	/*
+	  Write to the MMU script stream indicating the memory allocation
+	*/
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(eMMULevel);
+	}
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "CALLOC :%s:%s%016"IMG_UINT64_FMTSPECX" 0x%X 0x%X 0x0",
+			     pszPDumpDevName,
+			     pszMMUPX,
+			     ui64SymbolicAddr,
+			     ui32Size,
+			     ui32Align
+			     /* don't need this sDevPAddr.uiAddr*/);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32Flags);
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUFree
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+			const IMG_CHAR *pszPDumpDevName,
+			  MMU_LEVEL eMMULevel,
+			  IMG_DEV_PHYADDR *psDevPAddr,
+			  PDUMP_MMU_TYPE eMMUType)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT64 ui64SymbolicAddr;
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA;
+	IMG_CHAR *pszMMUPX;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_GOTO_IF_INVALID_PARAM(eMMULevel < MMU_LEVEL_LAST, eErr, ErrOut);
+
+	PDUMP_LOCK(ui32Flags);
+	/*
+	  Write a comment to the PDUMP2 script streams indicating the memory free
+	*/
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "-- FREE :%s:%s",
+			     pszPDumpDevName, apszMMULevelStringLookup[eMMULevel]);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	/*
+	  construct the symbolic address
+	*/
+	ui64SymbolicAddr = (IMG_UINT64)psDevPAddr->uiAddr;
+
+	/*
+	  Write to the MMU script stream indicating the memory free
+	*/
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(eMMULevel);
+	}
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "FREE :%s:%s%016"IMG_UINT64_FMTSPECX,
+			     pszPDumpDevName,
+			     pszMMUPX,
+			     ui64SymbolicAddr);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32Flags);
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/*******************************************************************************************************
+ * Function Name  : PDumpPTBaseObjectToMem64
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write from the baseobject
+ *					for MIPS MMU device type
+********************************************************************************************************/
+PVRSRV_ERROR PDumpPTBaseObjectToMem64(const IMG_CHAR *pszPDumpDevName,
+				      PMR *psPMRDest,
+				      IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+				      IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+				      IMG_UINT32 ui32Flags,
+				      MMU_LEVEL eMMULevel,
+				      IMG_UINT64 ui64PxSymAddr,
+				      IMG_UINT64 ui64PxOffset)
+{
+
+	IMG_CHAR aszMemspaceNameDest[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameDest[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetDest;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameDest;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRDest,
+				     uiLogicalOffsetDest,
+				     PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+				     aszMemspaceNameDest,
+				     PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+				     aszSymbolicNameDest,
+				     &uiPDumpSymbolicOffsetDest,
+				     &uiNextSymNameDest);
+
+	PVR_GOTO_IF_ERROR(eErr, Err);
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+						 "WRW64 :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:%s%016"IMG_UINT64_FMTSPECX":0x%"IMG_UINT64_FMTSPECX,
+						 aszMemspaceNameDest, aszSymbolicNameDest, uiPDumpSymbolicOffsetDest,
+						 pszPDumpDevName, MIPSMMUPX_FMT(eMMULevel), ui64PxSymAddr, ui64PxOffset);
+
+	PVR_GOTO_IF_ERROR(eErr, Err);
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(PMR_DeviceNode(psPMRDest), hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+Err:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUDumpPxEntries
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUDumpPxEntries(PVRSRV_DEVICE_NODE *psDeviceNode,
+				   MMU_LEVEL eMMULevel,
+				   const IMG_CHAR *pszPDumpDevName,
+				   void *pvPxMem,
+				   IMG_DEV_PHYADDR sPxDevPAddr,
+				   IMG_UINT32 uiFirstEntry,
+				   IMG_UINT32 uiNumEntries,
+				   const IMG_CHAR *pszMemspaceName,
+				   const IMG_CHAR *pszSymbolicAddr,
+				   IMG_UINT64 uiSymbolicAddrOffset,
+				   IMG_UINT32 uiBytesPerEntry,
+				   IMG_UINT32 uiLog2Align,
+				   IMG_UINT32 uiAddrShift,
+				   IMG_UINT64 uiAddrMask,
+				   IMG_UINT64 uiPxEProtMask,
+				   IMG_UINT64 uiDataValidEnable,
+				   IMG_UINT32 ui32Flags,
+				   IMG_UINT32 ui32VAParity,
+				   IMG_UINT32 ui32ParityShift,
+				   IMG_UINT64 ui64ParityMask,
+				   PDUMP_MMU_TYPE eMMUType)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT64 ui64PxSymAddr;
+	IMG_UINT64 ui64PxEValueSymAddr;
+	IMG_UINT32 ui32SymAddrOffset = 0;
+	IMG_UINT32 *pui32PxMem;
+	IMG_UINT64 *pui64PxMem;
+	IMG_BOOL   bPxEValid;
+	IMG_UINT32 uiPxEIdx;
+	IMG_INT32  iShiftAmount;
+	IMG_CHAR   *pszWrwSuffix = NULL;
+	void *pvRawBytes = NULL;
+	IMG_CHAR aszPxSymbolicAddr[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_UINT64 ui64PxE64;
+	IMG_UINT64 ui64Protflags64;
+	IMG_CHAR *pszMMUPX;
+
+	PDUMP_GET_SCRIPT_STRING();
+	ui32Flags |= (PDUMP_FLAGS_BLKDATA | PDUMP_FLAGS_CONTINUOUS);
+
+	eErr = PDumpReady();
+	if (eErr != PVRSRV_OK)
+	{
+		/* Mask suspension from caller as this is terminal & logged */
+		eErr = (eErr == PVRSRV_ERROR_PDUMP_NOT_ACTIVE) ? PVRSRV_OK : eErr;
+		goto ErrOut;
+	}
+
+	PVR_GOTO_IF_INVALID_PARAM(pvPxMem, eErr, ErrOut);
+
+	/*
+	  create the symbolic address of the Px
+	*/
+	ui64PxSymAddr = sPxDevPAddr.uiAddr;
+
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(eMMULevel);
+	}
+
+	OSSNPrintf(aszPxSymbolicAddr,
+	           PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	           ":%s:%s%016"IMG_UINT64_FMTSPECX,
+	           pszPDumpDevName,
+	           pszMMUPX,
+	           ui64PxSymAddr);
+
+	PDUMP_LOCK(ui32Flags);
+
+	/*
+	  traverse PxEs, dumping entries
+	*/
+	for (uiPxEIdx = uiFirstEntry;
+	     uiPxEIdx < uiFirstEntry + uiNumEntries;
+	     uiPxEIdx++)
+	{
+		IMG_BOOL bLastEntry = (uiPxEIdx == uiFirstEntry + uiNumEntries - 1);
+		/* Calc the symbolic address offset of the PxE location
+		   This is what we have to add to the table address to get to a certain entry */
+		ui32SymAddrOffset = (uiPxEIdx*uiBytesPerEntry);
+
+		/* Calc the symbolic address of the PxE value and HW protflags */
+		/* just read it here */
+		switch (uiBytesPerEntry)
+		{
+			case 4:
+			{
+				pui32PxMem = pvPxMem;
+				ui64PxE64 = pui32PxMem[uiPxEIdx];
+				pszWrwSuffix = "";
+				pvRawBytes = &pui32PxMem[uiPxEIdx];
+				break;
+			}
+			case 8:
+			{
+				pui64PxMem = pvPxMem;
+				ui64PxE64 = pui64PxMem[uiPxEIdx];
+				pszWrwSuffix = "64";
+				pvRawBytes = &pui64PxMem[uiPxEIdx];
+				break;
+			}
+			default:
+			{
+				PVR_DPF((PVR_DBG_ERROR, "PDumpMMUPxEntries: error"));
+				ui64PxE64 = 0;
+				//!!error
+				break;
+			}
+		}
+
+		ui64PxEValueSymAddr = (ui64PxE64 & uiAddrMask) >> uiAddrShift << uiLog2Align;
+		ui64Protflags64 = ui64PxE64 & uiPxEProtMask;
+		bPxEValid = (ui64Protflags64 & uiDataValidEnable) ? IMG_TRUE : IMG_FALSE;
+
+		if (!bPxEValid)
+		{
+			/* If the entry was "invalid", simply write the actual
+			   value found to the memory location */
+			eErr = _ContiguousPDumpBytes(psDeviceNode,aszPxSymbolicAddr,
+			                             ui32SymAddrOffset, IMG_FALSE,
+			                             uiBytesPerEntry, pvRawBytes,
+			                             ui32Flags);
+			if (bLastEntry && eErr == PVRSRV_OK)
+			{
+				goto done;
+			}
+			else if (eErr == PVRSRV_OK)
+			{
+				continue;
+			}
+			else
+			{
+				goto ErrUnlock;
+			}
+		}
+
+		_ContiguousPDumpBytes(psDeviceNode, aszPxSymbolicAddr,
+		                      ui32SymAddrOffset, IMG_TRUE,
+		                      0, NULL,
+		                      ui32Flags);
+
+		iShiftAmount = (IMG_INT32)(uiLog2Align - uiAddrShift);
+
+		/* First put the symbolic representation of the actual
+		   address of the entry into a pdump internal register */
+		/* MOV seemed cleaner here, since (a) it's 64-bit; (b) the
+		   target is not memory.  However, MOV cannot do the
+		   "reference" of the symbolic address.  Apparently WRW is
+		   correct. */
+
+		if (pszSymbolicAddr == NULL)
+		{
+			pszSymbolicAddr = "none";
+		}
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			if (iShiftAmount == 0)
+			{
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "WRW%s :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:%s:0x%"IMG_UINT64_FMTSPECX" | 0x%"IMG_UINT64_FMTSPECX"\n",
+				                     pszWrwSuffix,
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     pszMMUPX,
+				                     ui64PxSymAddr,
+				                     ui32SymAddrOffset,
+				                     /* src */
+				                     pszMemspaceName,
+				                     pszSymbolicAddr,
+				                     uiSymbolicAddrOffset,
+				                     /* ORing prot flags */
+				                     ui64Protflags64);
+			}
+			else
+			{
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                    "WRW :%s:$1 :%s:%s:0x%"IMG_UINT64_FMTSPECX"\n",
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     /* src */
+				                     pszMemspaceName,
+				                     pszSymbolicAddr,
+				                     uiSymbolicAddrOffset);
+			}
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+			/* OR a parity bit */
+			if (ui64ParityMask)
+			{
+				IMG_UINT32 uiStateWord =
+					((0x1U << PDUMP_SET_PARITY_STATE_WORD_VERSION_SHIFT) & PDUMP_SET_PARITY_STATE_WORD_VERSION_MASK) |
+					((ui32ParityShift << PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_SHIFT) & PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_MASK) |
+					((ui32VAParity << PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_SHIFT) & PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_MASK);
+
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "CMD:SetParity :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:%s:0x%"IMG_UINT64_FMTSPECX" 0x%08X\n",
+				                     /* Dest PTE entry tag */
+				                     pszPDumpDevName,
+				                     pszMMUPX,
+				                     ui64PxSymAddr,
+				                     ui32SymAddrOffset,
+				                     /* Source tag to calc PA parity from */
+				                     pszMemspaceName,
+				                     pszSymbolicAddr,
+				                     uiSymbolicAddrOffset,
+				                     /* State word */
+				                     uiStateWord);
+
+				PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+				PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+			}
+		}
+		else
+		{
+			if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+			{
+				pszMMUPX = MIPSMMUPX_FMT(eMMULevel - 1);
+			}
+			else
+			{
+				pszMMUPX = MMUPX_FMT(eMMULevel - 1);
+			}
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "WRW :%s:$1 :%s:%s%016"IMG_UINT64_FMTSPECX":0x0",
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src */
+			                     pszPDumpDevName,
+			                     pszMMUPX,
+			                     ui64PxEValueSymAddr);
+			if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+			{
+				pszMMUPX = MIPSMMUPX_FMT(eMMULevel);
+			}
+			else
+			{
+				pszMMUPX = MMUPX_FMT(eMMULevel);
+			}
+
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+
+
+		/* Now shift it to the right place, if necessary: */
+		/* Now shift that value down, by the "Align shift"
+		   amount, to get it into units (ought to assert that
+		   we get an integer - i.e. we don't shift any bits
+		   off the bottom, don't know how to do PDUMP
+		   assertions yet) and then back up by the right
+		   amount to get it into the position of the field.
+		   This is optimised into a single shift right by the
+		   difference between the two. */
+		if (iShiftAmount > 0)
+		{
+			/* Page X Address is specified in units larger
+			   than the position in the PxE would suggest. */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "SHR :%s:$1 :%s:$1 0x%X",
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src A */
+			                     pszPDumpDevName,
+			                     /* src B */
+			                     iShiftAmount);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+		else if (iShiftAmount < 0)
+		{
+			/* Page X Address is specified in units smaller
+			   than the position in the PxE would suggest. */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "SHL :%s:$1 :%s:$1 0x%X",
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src A */
+			                     pszPDumpDevName,
+			                     /* src B */
+			                     -iShiftAmount);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			if (iShiftAmount != 0)
+			{
+				/* Now we can "or" in the protection flags */
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "OR :%s:$1 :%s:$1 0x%"IMG_UINT64_FMTSPECX,
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     /* src A */
+				                     pszPDumpDevName,
+				                     /* src B */
+				                     ui64Protflags64);
+				PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+				PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+				eErr = PDumpSNPrintf(hScript,
+				                     ui32MaxLen,
+				                     "WRW%s :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:$1",
+				                     pszWrwSuffix,
+				                     /* dest */
+				                     pszPDumpDevName,
+				                     pszMMUPX,
+				                     ui64PxSymAddr,
+				                     ui32SymAddrOffset,
+				                     /* src */
+				                     pszPDumpDevName);
+				PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+				PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+			}
+		}
+		else
+		{
+			/* Now we can "or" in the protection flags */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "OR :%s:$1 :%s:$1 0x%"IMG_UINT64_FMTSPECX,
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     /* src A */
+			                     pszPDumpDevName,
+			                     /* src B */
+			                     ui64Protflags64);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+			/* Finally, we write the register into the actual PxE */
+			eErr = PDumpSNPrintf(hScript,
+			                     ui32MaxLen,
+			                     "WRW%s :%s:%s%016"IMG_UINT64_FMTSPECX":0x%08X :%s:$1",
+			                     pszWrwSuffix,
+			                     /* dest */
+			                     pszPDumpDevName,
+			                     pszMMUPX,
+			                     ui64PxSymAddr,
+			                     ui32SymAddrOffset,
+			                     /* src */
+			                     pszPDumpDevName);
+			PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+			PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		}
+	}
+
+done:
+	/* flush out any partly accumulated stuff for LDB */
+	_ContiguousPDumpBytes(psDeviceNode, aszPxSymbolicAddr,
+	                      ui32SymAddrOffset, IMG_TRUE,
+	                      0, NULL,
+	                      ui32Flags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32Flags);
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : _PdumpAllocMMUContext
+ * Inputs         : pui32MMUContextID
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : pdump util to allocate MMU contexts
+**************************************************************************/
+static PVRSRV_ERROR _PdumpAllocMMUContext(IMG_UINT32 *pui32MMUContextID)
+{
+	IMG_UINT32 i;
+
+	/* there are MAX_PDUMP_MMU_CONTEXTS contexts available, find one */
+	for (i=0; i<MAX_PDUMP_MMU_CONTEXTS; i++)
+	{
+		if ((guiPDumpMMUContextAvailabilityMask & (1U << i)))
+		{
+			/* mark in use */
+			guiPDumpMMUContextAvailabilityMask &= ~(1U << i);
+			*pui32MMUContextID = i;
+			return PVRSRV_OK;
+		}
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "_PdumpAllocMMUContext: no free MMU context ids"));
+
+	return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+}
+
+/**************************************************************************
+ * Function Name  : _PdumpFreeMMUContext
+ * Inputs         : ui32MMUContextID
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : pdump util to free MMU contexts
+**************************************************************************/
+static PVRSRV_ERROR _PdumpFreeMMUContext(IMG_UINT32 ui32MMUContextID)
+{
+	if (ui32MMUContextID < MAX_PDUMP_MMU_CONTEXTS)
+	{
+		/* free the id */
+		PVR_ASSERT (!(guiPDumpMMUContextAvailabilityMask & (1U << ui32MMUContextID)));
+		guiPDumpMMUContextAvailabilityMask |= (1U << ui32MMUContextID);
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "_PdumpFreeMMUContext: MMU context ids invalid"));
+
+	return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUAllocMMUContext
+ * Inputs         :
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Alloc MMU Context
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUAllocMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+				const IMG_CHAR *pszPDumpMemSpaceName,
+				     IMG_DEV_PHYADDR sPCDevPAddr,
+				     PDUMP_MMU_TYPE eMMUType,
+				     IMG_UINT32 *pui32MMUContextID,
+				     IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT64 ui64PCSymAddr;
+	IMG_CHAR *pszMMUPX;
+	IMG_UINT32 ui32MMUContextID;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = _PdumpAllocMMUContext(&ui32MMUContextID);
+	if (eErr != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eErr, "_PdumpAllocMMUContext");
+		PVR_DBG_BREAK;
+		goto ErrOut;
+	}
+
+	/*
+	  create the symbolic address of the PC
+	*/
+	ui64PCSymAddr = sPCDevPAddr.uiAddr;
+
+	if (eMMUType == PDUMP_MMU_TYPE_MIPS_MICROAPTIV)
+	{
+		pszMMUPX = MIPSMMUPX_FMT(1);
+		/* Giving it a mock value until the Pdump player implements
+		   the support for the MIPS microAptiv MMU*/
+		eMMUType = PDUMP_MMU_TYPE_VARPAGE_40BIT;
+	}
+	else
+	{
+		pszMMUPX = MMUPX_FMT(3);
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "MMU :%s:v%d %d :%s:%s%016"IMG_UINT64_FMTSPECX,
+			     /* mmu context */
+			     pszPDumpMemSpaceName,
+			     ui32MMUContextID,
+			     /* mmu type */
+			     eMMUType,
+			     /* PC base address */
+			     pszPDumpMemSpaceName,
+			     pszMMUPX,
+			     ui64PCSymAddr);
+	if (eErr != PVRSRV_OK)
+	{
+		PVR_DBG_BREAK;
+		goto ErrOut;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	/* return the MMU Context ID */
+	*pui32MMUContextID = ui32MMUContextID;
+
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+/**************************************************************************
+ * Function Name  : PDumpMMUFreeMMUContext
+ * Inputs         :
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Free MMU Context
+**************************************************************************/
+PVRSRV_ERROR PDumpMMUFreeMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+				    const IMG_CHAR *pszPDumpMemSpaceName,
+				    IMG_UINT32 ui32MMUContextID,
+				    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	PDUMP_GET_SCRIPT_STRING();
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "-- Clear MMU Context for memory space %s", pszPDumpMemSpaceName);
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode,hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+	                     ui32MaxLen,
+	                     "MMU :%s:v%d",
+	                     pszPDumpMemSpaceName,
+	                     ui32MMUContextID);
+
+	PVR_GOTO_IF_ERROR(eErr, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = _PdumpFreeMMUContext(ui32MMUContextID);
+	PVR_LOG_GOTO_IF_ERROR(eErr, "_PdumpFreeMMUContext", ErrUnlock);
+
+ErrUnlock:
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+PVRSRV_ERROR
+PDumpMMUSAB(PVRSRV_DEVICE_NODE *psDeviceNode,
+	    const IMG_CHAR *pszPDumpMemNamespace,
+	    IMG_UINT32 uiPDumpMMUCtx,
+	    IMG_DEV_VIRTADDR sDevAddrStart,
+	    IMG_DEVMEM_SIZE_T uiSize,
+	    const IMG_CHAR *pszFilename,
+	    IMG_UINT32 uiFileOffset,
+	    IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	// "SAB :%s:v%x:0x%010llX 0x%08X 0x%08X %s.bin",
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpReady();
+	if (eErr != PVRSRV_OK)
+	{
+		/* Mask suspension from caller as this is terminal & logged */
+		eErr = (eErr == PVRSRV_ERROR_PDUMP_NOT_ACTIVE) ? PVRSRV_OK : eErr;
+		goto ErrOut;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			     ui32MaxLen,
+			     "SAB :%s:v%x:" IMG_DEV_VIRTADDR_FMTSPEC " "
+			     IMG_DEVMEM_SIZE_FMTSPEC " "
+			     "0x%x %s.bin\n",
+			     pszPDumpMemNamespace,
+			     uiPDumpMMUCtx,
+			     sDevAddrStart.uiAddr,
+			     uiSize,
+			     uiFileOffset,
+			     pszFilename);
+	PVR_GOTO_IF_ERROR(eErr, ErrOut);
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+ErrOut:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eErr;
+}
+
+#endif /* #if defined(PDUMP) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_physmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_physmem.c
new file mode 100644
index 000000000000..e25b1f40e069
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_physmem.c
@@ -0,0 +1,742 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common PDump (PMR specific) functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /***************************************************************************/
+
+#if defined(PDUMP)
+
+#if defined(__linux__)
+#include <linux/ctype.h>
+#else
+#include <ctype.h>
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "pdump_physmem.h"
+#include "pdump_km.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+
+#include "pvrsrv.h"
+
+/* #define MAX_PDUMP_MMU_CONTEXTS	(10) */
+/* static IMG_UINT32 guiPDumpMMUContextAvailabilityMask = (1<<MAX_PDUMP_MMU_CONTEXTS)-1; */
+
+
+struct _PDUMP_PHYSMEM_INFO_T_
+{
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH];
+	IMG_UINT64 ui64Size;
+	IMG_UINT32 ui32Align;
+	IMG_UINT32 ui32SerialNum;
+};
+
+static IMG_BOOL _IsAllowedSym(IMG_CHAR sym)
+{
+	/* Numbers, Characters or '_' are allowed */
+	if (isalnum(sym) || sym == '_')
+		return IMG_TRUE;
+	else
+		return IMG_FALSE;
+}
+
+static IMG_BOOL _IsLowerCaseSym(IMG_CHAR sym)
+{
+	if (sym >= 'a' && sym <= 'z')
+		return IMG_TRUE;
+	else
+		return IMG_FALSE;
+}
+
+void PDumpMakeStringValid(IMG_CHAR *pszString,
+                          IMG_UINT32 ui32StrLen)
+{
+	IMG_UINT32 i;
+
+	if (pszString)
+	{
+		for (i = 0; i < ui32StrLen; i++)
+		{
+			if (_IsAllowedSym(pszString[i]))
+			{
+				if (_IsLowerCaseSym(pszString[i]))
+					pszString[i] = pszString[i]-32;
+				else
+					pszString[i] = pszString[i];
+			}
+			else
+			{
+				pszString[i] = '_';
+			}
+		}
+	}
+}
+
+/**************************************************************************
+ * Function Name  : PDumpGetSymbolicAddr
+ * Inputs         :
+ * Outputs        :
+ * Returns        : PVRSRV_ERROR
+ * Description    :
+ **************************************************************************/
+PVRSRV_ERROR PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                                  IMG_CHAR **ppszSymbolicAddress)
+{
+	PDUMP_PHYSMEM_INFO_T *psPDumpAllocationInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(hPhysmemPDumpHandle);
+
+	psPDumpAllocationInfo = (PDUMP_PHYSMEM_INFO_T *)hPhysmemPDumpHandle;
+	*ppszSymbolicAddress = psPDumpAllocationInfo->aszSymbolicAddress;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpMalloc
+@Description    Builds and writes an allocation command to pdump output. Whilst
+                writing the thread is locked.
+
+@Input          psDeviceNode        A pointer to a device node.
+@Input          pszDevSpace         Device space string.
+@Input          pszSymbolicAddress  Name of the allocation.
+@Input          ui64Size            String size.
+@Input          uiAlign             Command alignment.
+@Input          bInitialise         Should the command initialise the allocation.
+@Input          ui8InitValue        The value memory is initialised to.
+@Input          phHandlePtr         PDump allocation handle.
+@Input          ui32PDumpFlags      PDump allocation flags.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         const IMG_CHAR *pszDevSpace,
+                         const IMG_CHAR *pszSymbolicAddress,
+                         IMG_UINT64 ui64Size,
+                         IMG_DEVMEM_ALIGN_T uiAlign,
+                         IMG_BOOL bInitialise,
+                         IMG_UINT8 ui8InitValue,
+                         IMG_HANDLE *phHandlePtr,
+                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	eError = PDumpMallocUnlocked(psDeviceNode,
+	                             pszDevSpace,
+	                             pszSymbolicAddress,
+	                             ui64Size,
+	                             uiAlign,
+	                             bInitialise,
+	                             ui8InitValue,
+	                             phHandlePtr,
+	                             ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpMallocUnlocked
+@Description    Builds and writes an allocation command to pdump output. Whilst
+                writing the thread remains unlocked.
+
+@Input          psDeviceNode        A pointer to a device node.
+@Input          pszDevSpace         Device space string.
+@Input          pszSymbolicAddress  Name of the allocation.
+@Input          ui64Size            String size.
+@Input          uiAlign             Command alignment.
+@Input          bInitialise         Should the command initialise the allocation.
+@Input          ui8InitValue        The value memory is initialised to.
+@Input          phHandlePtr         PDump allocation handle.
+@Input          ui32PDumpFlags      PDump allocation flags.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 const IMG_CHAR *pszDevSpace,
+                                 const IMG_CHAR *pszSymbolicAddress,
+                                 IMG_UINT64 ui64Size,
+                                 IMG_DEVMEM_ALIGN_T uiAlign,
+                                 IMG_BOOL bInitialise,
+                                 IMG_UINT8 ui8InitValue,
+                                 IMG_HANDLE *phHandlePtr,
+                                 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDUMP_PHYSMEM_INFO_T *psPDumpAllocationInfo;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psPDumpAllocationInfo = OSAllocMem(sizeof*psPDumpAllocationInfo);
+	PVR_ASSERT(psPDumpAllocationInfo != NULL);
+
+	/*
+	 * PDUMP_CONT and PDUMP_PERSIST flag can't set together.
+	 */
+	if (ui32PDumpFlags == PDUMP_NONE)
+	{
+		/*
+			Set continuous flag because there is no way of knowing beforehand which
+			allocation is needed for playback of the captured range.
+		*/
+		ui32PDumpFlags |= PDUMP_FLAGS_CONTINUOUS;
+	}
+
+	ui32PDumpFlags |= PDUMP_FLAGS_BLKDATA;
+
+	/*
+		construct the symbolic address
+	 */
+
+	OSSNPrintf(psPDumpAllocationInfo->aszSymbolicAddress,
+	           sizeof(psPDumpAllocationInfo->aszSymbolicAddress),
+	           ":%s:%s",
+	           pszDevSpace,
+	           pszSymbolicAddress);
+
+	/*
+		Write to the MMU script stream indicating the memory allocation
+	*/
+	if (bInitialise)
+	{
+		eError = PDumpSNPrintf(hScript, ui32MaxLen, "CALLOC %s 0x%"IMG_UINT64_FMTSPECX" 0x%"IMG_UINT64_FMTSPECX" 0x%X\n",
+		                          psPDumpAllocationInfo->aszSymbolicAddress,
+		                          ui64Size,
+		                          uiAlign,
+		                          ui8InitValue);
+	}
+	else
+	{
+		eError = PDumpSNPrintf(hScript, ui32MaxLen, "MALLOC %s 0x%"IMG_UINT64_FMTSPECX" 0x%"IMG_UINT64_FMTSPECX"\n",
+		                          psPDumpAllocationInfo->aszSymbolicAddress,
+		                          ui64Size,
+		                          uiAlign);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psPDumpAllocationInfo);
+		goto _return;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	psPDumpAllocationInfo->ui64Size = ui64Size;
+	psPDumpAllocationInfo->ui32Align = TRUNCATE_64BITS_TO_32BITS(uiAlign);
+
+	*phHandlePtr = (IMG_HANDLE)psPDumpAllocationInfo;
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpFree
+@Description    Writes a FREE command for an allocation handle to the pdump out2
+                stream. When writing to the output stream the thread is locked.
+
+@Input          psDeviceNode                A pointer to a device node.
+@Input          hPDumpAllocationInfoHandle  A PDump allocation handle.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_LOCK(PDUMP_FLAGS_NONE);
+	eError = PDumpFreeUnlocked(psDeviceNode, hPDumpAllocationInfoHandle);
+	PDUMP_UNLOCK(PDUMP_FLAGS_NONE);
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PDumpFreeUnlocked
+@Description    Writes a FREE command for an allocation handle to the pdump
+                out2 stream. When writing to the output stream the thread
+                remains unlocked.
+
+@Input          psDeviceNode                A pointer to a device node.
+@Input          hPDumpAllocationInfoHandle  A PDump allocation handle.
+
+@Return         This function returns a PVRSRV_ERROR. PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDUMP_PHYSMEM_INFO_T *psPDumpAllocationInfo;
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psPDumpAllocationInfo = (PDUMP_PHYSMEM_INFO_T *)hPDumpAllocationInfoHandle;
+
+	/*
+		Write to the MMU script stream indicating the memory free
+	 */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "FREE %s\n",
+	                          psPDumpAllocationInfo->aszSymbolicAddress);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	OSFreeMem(psPDumpAllocationInfo);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW32(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT32 ui32Value,
+              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                       ui32MaxLen,
+	                       "WRW :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                       PMR_VALUE32_FMTSPEC,
+	                       pszDevSpace,
+	                       pszSymbolicName,
+	                       uiOffset,
+	                       ui32Value);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW32InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "WRW :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " %s",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          pszInternalVar);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRRDW32MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "RDW %s :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC,
+	                          pszInternalVar,
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW64(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT64 ui64Value,
+              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                       ui32MaxLen,
+	                       "WRW64 :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                       PMR_VALUE64_FMTSPEC,
+	                       pszDevSpace,
+	                       pszSymbolicName,
+	                       uiOffset,
+	                       ui64Value);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRWRW64InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "WRW64 :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " %s",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          pszInternalVar);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRRDW64MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "RDW64 %s :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC,
+	                          pszInternalVar,
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRLDB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "LDB :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          IMG_DEVMEM_SIZE_FMTSPEC " "
+	                          PDUMP_FILEOFFSET_FMTSPEC " %s\n",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          uiSize,
+	                          uiFileOffset,
+	                          pszFilename);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR PDumpPMRSAB(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         const IMG_CHAR *pszDevSpace,
+                         const IMG_CHAR *pszSymbolicName,
+                         IMG_DEVMEM_OFFSET_T uiOffset,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         const IMG_CHAR *pszFileName,
+                         IMG_UINT32 uiFileOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiPDumpFlags;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	uiPDumpFlags = 0;
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "SAB :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          IMG_DEVMEM_SIZE_FMTSPEC " "
+	                          "0x%08X %s.bin\n",
+	                          pszDevSpace,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          uiSize,
+	                          uiFileOffset,
+	                          pszFileName);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRPOL(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMemspaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_UINT32 ui32Value,
+            IMG_UINT32 ui32Mask,
+            PDUMP_POLL_OPERATOR eOperator,
+            IMG_UINT32 uiCount,
+            IMG_UINT32 uiDelay,
+            PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "POL :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          "0x%08X 0x%08X %d %d %d\n",
+	                          pszMemspaceName,
+	                          pszSymbolicName,
+	                          uiOffset,
+	                          ui32Value,
+	                          ui32Mask,
+	                          eOperator,
+	                          uiCount,
+	                          uiDelay);
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpPMRCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMemspaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDUMP_FLAGS_T uiPDumpFlags = 0;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "CBP :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " "
+	                          IMG_DEVMEM_OFFSET_FMTSPEC " " IMG_DEVMEM_SIZE_FMTSPEC " " IMG_DEVMEM_SIZE_FMTSPEC "\n",
+	                          pszMemspaceName,
+	                          pszSymbolicName,
+	                          uiReadOffset,
+	                          uiWriteOffset,
+	                          uiPacketSize,
+	                          uiBufferSize);
+
+	PVR_GOTO_IF_ERROR(eError, _return);
+
+	PDUMP_LOCK(uiPDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+_return:
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/* Checking that the request is for the PDump-bound device
+ * should be done before the following function is called
+ */
+PVRSRV_ERROR
+PDumpWriteParameterBlob(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiNumBytes,
+                        PDUMP_FLAGS_T uiPDumpFlags,
+                        IMG_CHAR *pszFilenameOut,
+                        size_t uiFilenameBufSz,
+                        PDUMP_FILEOFFSET_T *puiOffsetOut)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS;
+	PVR_UNREFERENCED_PARAMETER(uiFilenameBufSz);
+
+	PVR_ASSERT(uiNumBytes > 0);
+
+	eError = PDumpReady();
+	if (eError != PVRSRV_OK)
+	{
+		/* Mask suspension from caller as this is terminal & logged */
+		eError = (eError == PVRSRV_ERROR_PDUMP_NOT_ACTIVE) ?
+				PVRSRV_ERROR_PDUMP_NOT_ALLOWED :
+				eError;
+		return eError;
+	}
+
+	PVR_ASSERT(uiFilenameBufSz <= PDUMP_PARAM_MAX_FILE_NAME);
+
+	PDUMP_LOCK(uiPDumpFlags);
+
+	eError = PDumpWriteParameter(psDeviceNode, pcBuffer, uiNumBytes,
+	                             uiPDumpFlags, puiOffsetOut, pszFilenameOut);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+	if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_PDUMP_NOT_ALLOWED))
+	{
+		PVR_LOG_RETURN_IF_ERROR(eError, "PDumpWriteParameter");
+	}
+	/* else Write to parameter file Ok or Prevented under the flags or
+	 * current state of the driver so skip further writes and let caller know.
+	 */
+	return eError;
+}
+
+#endif /* PDUMP */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_server.c
new file mode 100644
index 000000000000..4b42e9c207a0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pdump_server.c
@@ -0,0 +1,5740 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common Server PDump functions layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(PDUMP)
+
+#if defined(__linux__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "pvrversion.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+#include "pdump_physmem.h"
+#include "hash.h"
+#include "connection_server.h"
+#include "services_km.h"
+#include <powervr/buffer_attribs.h>
+#include "os_apphint.h"
+
+/* pdump headers */
+#include "tlstream.h"
+#include "pdump_km.h"
+
+#include "pdumpdesc.h"
+#include "rgxpdump.h"
+
+/* Allow temporary buffer size override */
+#if !defined(PDUMP_TEMP_BUFFER_SIZE)
+#define PDUMP_TEMP_BUFFER_SIZE (64 * 1024U)
+#endif
+
+#define	PTR_PLUS(t, p, x) ((t)(((IMG_CHAR *)(p)) + (x)))
+#define	VPTR_PLUS(p, x) PTR_PLUS(void *, p, x)
+#define	VPTR_INC(p, x) ((p) = VPTR_PLUS(p, x))
+#define MAX_PDUMP_MMU_CONTEXTS	(32)
+
+#define PRM_FILE_SIZE_MAX	0x7FDFFFFFU /*!< Default maximum file size to split output files, 2GB-2MB as fwrite limits it to 2GB-1 on 32bit systems */
+
+#define MAX_PDUMP_WRITE_RETRIES	200	/*!< Max number of retries to dump pdump data in to respective buffers */
+
+/* 'Magic' cookie used in this file only, where no psDeviceNode is available
+ * but writing to the PDump log should be permitted
+ */
+#define PDUMP_MAGIC_COOKIE 0x9E0FF
+
+static ATOMIC_T		g_sConnectionCount;
+
+/*
+ * Structure to store some essential attributes of a PDump stream buffer.
+ */
+typedef struct
+{
+	IMG_CHAR*  pszName;                     /*!< Name of the PDump TL Stream buffer */
+	IMG_HANDLE hTL;                         /*!< Handle of created TL stream buffer */
+	IMG_UINT32 ui32BufferSize;              /*!< The size of the buffer in bytes */
+	IMG_UINT32 ui32BufferFullRetries;       /*!< The number of times the buffer got full */
+	IMG_UINT32 ui32BufferFullAborts;        /*!< The number of times we failed to write data */
+	IMG_UINT32 ui32HighestRetriesWatermark; /*!< Max number of retries try to dump pdump data */
+	IMG_UINT32 ui32MaxAllowedWriteSize;     /*!< Max allowed write packet size */
+} PDUMP_STREAM;
+
+typedef struct
+{
+	PDUMP_STREAM sInitStream;   /*!< Driver initialisation PDump stream */
+	PDUMP_STREAM sMainStream;   /*!< App framed PDump stream */
+	PDUMP_STREAM sDeinitStream; /*!< Driver/HW de-initialisation PDump stream */
+	PDUMP_STREAM sBlockStream;  /*!< Block mode PDump block data stream - currently its script only */
+} PDUMP_CHANNEL;
+
+typedef struct
+{
+	PDUMP_CHANNEL sCh;         /*!< Channel handles */
+	IMG_UINT32    ui32FileIdx; /*!< File index gets incremented on script out file split */
+} PDUMP_SCRIPT;
+
+typedef struct
+{
+	IMG_UINT32    ui32Init;    /*!< Count of bytes written to the init phase stream */
+	IMG_UINT32    ui32Main;    /*!< Count of bytes written to the main stream */
+	IMG_UINT32    ui32Deinit;  /*!< Count of bytes written to the deinit stream */
+	IMG_UINT32    ui32Block;   /*!< Count of bytes written to the block stream */
+} PDUMP_CHANNEL_WOFFSETS;
+
+typedef struct
+{
+	PDUMP_CHANNEL          sCh;             /*!< Channel handles */
+	PDUMP_CHANNEL_WOFFSETS sWOff;           /*!< Channel file write offsets */
+	IMG_UINT32             ui32FileIdx;     /*!< File index used when file size limit reached and a new file is started, parameter channel only */
+	IMG_UINT32             ui32MaxFileSize; /*!< Maximum file size for parameter files */
+
+	PDUMP_FILEOFFSET_T     uiZeroPageOffset; /*!< Offset of the zero page in the parameter file */
+	size_t                 uiZeroPageSize;   /*!< Size of the zero page in the parameter file */
+	IMG_CHAR               szZeroPageFilename[PDUMP_PARAM_MAX_FILE_NAME]; /*< PRM file name where the zero page was pdumped */
+} PDUMP_PARAMETERS;
+
+/* PDump global connection count - used to determine when/if the last
+ * connection (from a PDump generating app) has been closed.
+ * This is used to key the AUTO_TERMINATED behaviour if enabled.
+ */
+static ATOMIC_T gPDumpNumConnex;
+
+/* PDump lock to keep pdump write atomic.
+ * Which will protect g_PDumpScript & g_PDumpParameters pdump
+ * specific shared variable.
+ */
+static POS_LOCK g_hPDumpWriteLock;
+
+static PDUMP_SCRIPT     g_PDumpScript    = { {
+		{	PDUMP_SCRIPT_INIT_STREAM_NAME,   NULL,
+			PDUMP_SCRIPT_INIT_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_SCRIPT_MAIN_STREAM_NAME,   NULL,
+			PDUMP_SCRIPT_MAIN_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_SCRIPT_DEINIT_STREAM_NAME, NULL,
+			PDUMP_SCRIPT_DEINIT_STREAM_SIZE, 0, 0, 0 },
+		{	PDUMP_SCRIPT_BLOCK_STREAM_NAME,   NULL,
+			PDUMP_SCRIPT_BLOCK_STREAM_SIZE,   0, 0, 0 },
+		}, 0 };
+static PDUMP_PARAMETERS g_PDumpParameters = { {
+		{	PDUMP_PARAM_INIT_STREAM_NAME,   NULL,
+			PDUMP_PARAM_INIT_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_PARAM_MAIN_STREAM_NAME,   NULL,
+			PDUMP_PARAM_MAIN_STREAM_SIZE,   0, 0, 0 },
+		{	PDUMP_PARAM_DEINIT_STREAM_NAME, NULL,
+			PDUMP_PARAM_DEINIT_STREAM_SIZE, 0, 0, 0 },
+		{	PDUMP_PARAM_BLOCK_STREAM_NAME,   NULL,
+			PDUMP_PARAM_BLOCK_STREAM_SIZE,   0, 0, 0 },
+		}, {0, 0, 0, 0}, 0, PRM_FILE_SIZE_MAX};
+
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+/* counter increments each time debug write is called */
+ATOMIC_T g_sEveryLineCounter;
+#endif
+
+// #define PDUMP_DEBUG_TRANSITION
+#if defined(PDUMP_DEBUG_TRANSITION)
+# define DEBUG_OUTFILES_COMMENT(dev, fmt, ...) (void)PDumpCommentWithFlags(dev, PDUMP_FLAGS_CONTINUOUS, fmt, __VA_ARGS__)
+#else
+# define DEBUG_OUTFILES_COMMENT(dev, fmt, ...)
+#endif
+
+#if defined(PDUMP_DEBUG) || defined(REFCOUNT_DEBUG)
+# define PDUMP_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+# define PDUMP_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+/* Prototype for the test/debug state dump routine used in debugging */
+#if defined(PDUMP_TRACE_STATE)
+void PDumpCommonDumpState(void);
+#endif
+
+
+/*****************************************************************************/
+/* PDump Control Module Definitions                                          */
+/*****************************************************************************/
+
+/*
+ * struct _PDUMP_CAPTURE_RANGE_ is interpreted differently in different modes of PDump
+ *
+ * Non-Block mode:
+ *    ui32Start     - Start frame number of range
+ *    ui32End       - End frame number of range
+ *    ui32Interval  - Frame sample rate interval
+ *
+ * Block mode:
+ *    ui32Start     - If set to '0', first PDump-block will of minimal (i.e. PDUMP_BLOCKLEN_MIN)
+ *                    length, else all blocks will be of block-length provided
+ *
+ *    ui32End       - By default this is set to PDUMP_FRAME_MAX so that Blocked PDump
+ *                    will be captured indefinitely till stopped externally. On force capture
+ *                    stop, this will be set to (ui32CurrentFrame + 1) to stop capture from
+ *                    next frame onwards
+ *
+ *    ui32Interval  - This will be interpreted as PDump block-length provided
+ **/
+typedef struct _PDUMP_CAPTURE_RANGE_
+{
+	IMG_UINT32 ui32Start;
+	IMG_UINT32 ui32End;
+	IMG_UINT32 ui32Interval;
+} PDUMP_CAPTURE_RANGE;
+
+/* PDump Block mode specific controls */
+typedef struct _PDUMP_BLOCK_CTRL_
+{
+	IMG_UINT32 ui32BlockLength;       /*!< PDump block length in term of number of frames per block */
+	IMG_UINT32 ui32CurrentBlock;      /*!< Current block number */
+} PDUMP_BLOCK_CTRL;
+
+/*! PDump common module State Machine states */
+typedef enum _PDUMP_SM_
+{
+	PDUMP_SM_UNINITIALISED,           /*!< Starting state */
+	PDUMP_SM_INITIALISING,            /*!< Module is initialising */
+	PDUMP_SM_READY,                   /*!< Module is initialised and ready */
+	PDUMP_SM_READY_CLIENT_CONNECTED,  /*!< Module is ready and capture client connected */
+	PDUMP_SM_FORCED_SUSPENDED,        /*!< Module forced error, PDumping suspended, this is to force driver reload before next capture */
+	PDUMP_SM_ERROR_SUSPENDED,         /*!< Module fatal error, PDumping suspended semi-final state */
+	PDUMP_SM_DEINITIALISED            /*!< Final state */
+} PDUMP_SM;
+
+/*! PDump control flags */
+#define FLAG_IS_DRIVER_IN_INIT_PHASE 0x1  /*! Control flag that keeps track of State of driver initialisation phase */
+#define FLAG_IS_IN_CAPTURE_RANGE     0x2  /*! Control flag that keeps track of Current capture status, is current frame in range */
+#define FLAG_IS_IN_CAPTURE_INTERVAL  0x4  /*! Control flag that keeps track of Current capture status, is current frame in an interval where no capture takes place. */
+#define FLAG_IS_AUTO_TERMINATED      0x8  /*! Control flag that indicates app has auto-terminated. */
+
+#define CHECK_PDUMP_CONTROL_FLAG(PDUMP_CONTROL_FLAG) BITMASK_HAS(g_PDumpCtrl.ui32Flags, PDUMP_CONTROL_FLAG)
+#define SET_PDUMP_CONTROL_FLAG(PDUMP_CONTROL_FLAG)   BITMASK_SET(g_PDumpCtrl.ui32Flags, PDUMP_CONTROL_FLAG)
+#define UNSET_PDUMP_CONTROL_FLAG(PDUMP_CONTROL_FLAG) BITMASK_UNSET(g_PDumpCtrl.ui32Flags, PDUMP_CONTROL_FLAG)
+
+/* No direct access to members from outside the control module - please */
+typedef struct _PDUMP_CTRL_STATE_
+{
+	PDUMP_SM            eServiceState;      /*!< State of the pdump_common module */
+	IMG_UINT32          ui32Flags;
+
+	IMG_UINT32          ui32DefaultCapMode; /*!< Capture mode of the dump */
+	PDUMP_CAPTURE_RANGE sCaptureRange;      /*|< The capture range for capture mode 'framed' */
+	IMG_UINT32          ui32CurrentFrame;   /*!< Current frame number */
+
+	PDUMP_BLOCK_CTRL    sBlockCtrl;         /*!< Pdump block mode ctrl data */
+
+	POS_LOCK            hLock;              /*!< Exclusive lock to this structure */
+	IMG_PID             InPowerTransitionPID;/*!< pid of thread requesting power transition */
+	IMG_UINT32          ui32TimeoutFrequency;/*!< Timer frequency for checking process existence */
+} PDUMP_CTRL_STATE;
+
+static PDUMP_CTRL_STATE g_PDumpCtrl =
+{
+	PDUMP_SM_UNINITIALISED,
+
+	FLAG_IS_DRIVER_IN_INIT_PHASE,
+
+	PDUMP_CAPMODE_UNSET,
+	{
+		PDUMP_FRAME_UNSET,
+		PDUMP_FRAME_UNSET,
+		0
+	},
+	0,
+
+	{
+		0,
+		PDUMP_BLOCKNUM_INVALID,
+	},
+
+	NULL,
+	0,
+	0
+};
+
+static IMG_HANDLE g_PDumpTimerID;
+
+static void PDumpAssertWriteLockHeld(void);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+
+/*************************************************************************/ /*!
+ @Function		PDumpCreateIncVarNameStr
+ @Description	When 64 bit register access is split between two 32 bit
+	accesses, it needs two PDump Internal variables to store register value.
+	This function creates the string for the second PDump Internal variable
+	for example if Passed Variable name is :SYSMEM:$1 this function will
+	generate the string :SYSMEM:$2
+
+ @Input	pszInternalVar	String for PDump internal variable in use
+
+ @Return IMG_CHAR*  String for second PDump internal variable to be used
+*/ /**************************************************************************/
+static INLINE IMG_CHAR* PDumpCreateIncVarNameStr(const IMG_CHAR* pszInternalVar)
+{
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32Size = (IMG_UINT32)OSStringLength(pszInternalVar);
+	if (ui32Size == 0)
+	{
+		return NULL;
+	}
+
+	ui32Size++;
+	pszPDumpVarName = (IMG_CHAR*)OSAllocMem((ui32Size) * sizeof(IMG_CHAR));
+	if (pszPDumpVarName == NULL)
+	{
+		return NULL;
+	}
+
+	OSStringSafeCopy(pszPDumpVarName, pszInternalVar, ui32Size);
+	/* Increase the number on the second variable */
+	pszPDumpVarName[ui32Size-2] += 1;
+	return pszPDumpVarName;
+}
+
+/*************************************************************************/ /*!
+ @Function		PDumpFreeIncVarNameStr
+ @Description	Free the string created by function PDumpCreateIncVarNameStr
+
+ @Input	pszPDumpVarName String to free
+
+ @Return	void
+*/ /**************************************************************************/
+static INLINE void PDumpFreeIncVarNameStr(IMG_CHAR* pszPDumpVarName)
+{
+	if (pszPDumpVarName != NULL)
+	{
+		OSFreeMem(pszPDumpVarName);
+	}
+}
+#endif
+
+static PVRSRV_ERROR PDumpCtrlInit(void)
+{
+	g_PDumpCtrl.eServiceState = PDUMP_SM_INITIALISING;
+
+	/* Create lock for PDUMP_CTRL_STATE struct, which is shared between pdump client
+	   and PDumping app. This lock will help us serialize calls from pdump client
+	   and PDumping app */
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&g_PDumpCtrl.hLock), "OSLockCreate");
+
+	return PVRSRV_OK;
+}
+
+static void PDumpCtrlDeInit(void)
+{
+	if (g_PDumpCtrl.hLock)
+	{
+		OSLockDestroy(g_PDumpCtrl.hLock);
+		g_PDumpCtrl.hLock = NULL;
+	}
+}
+
+static INLINE void PDumpCtrlLockAcquire(void)
+{
+	OSLockAcquire(g_PDumpCtrl.hLock);
+}
+
+static INLINE void PDumpCtrlLockRelease(void)
+{
+	OSLockRelease(g_PDumpCtrl.hLock);
+}
+
+static INLINE PDUMP_SM PDumpCtrlGetModuleState(void)
+{
+	return g_PDumpCtrl.eServiceState;
+}
+
+PVRSRV_ERROR PDumpValidateUMFlags(PDUMP_FLAGS_T uiFlags)
+{
+	/* If these flags are or'd together, they are invalid */
+	return ((uiFlags & (~(PDUMP_BLKDATA | PDUMP_CONT))) != 0) ? PVRSRV_ERROR_INVALID_PARAMS : PVRSRV_OK;
+}
+
+PVRSRV_ERROR PDumpReady(void)
+{
+	switch (PDumpCtrlGetModuleState())
+	{
+	case PDUMP_SM_READY:
+	case PDUMP_SM_READY_CLIENT_CONNECTED:
+		return PVRSRV_OK;
+
+	case PDUMP_SM_FORCED_SUSPENDED:
+	case PDUMP_SM_ERROR_SUSPENDED:
+		return PVRSRV_ERROR_PDUMP_NOT_ACTIVE;
+
+	case PDUMP_SM_UNINITIALISED:
+	case PDUMP_SM_INITIALISING:
+	case PDUMP_SM_DEINITIALISED:
+		return PVRSRV_ERROR_PDUMP_NOT_AVAILABLE;
+
+	default:
+		/* Bad state */
+		PVR_ASSERT(1);
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+}
+
+
+/******************************************************************************
+	NOTE:
+	The following PDumpCtrl*** functions require the PDUMP_CTRL_STATE lock be
+	acquired BEFORE they are called. This is because the PDUMP_CTRL_STATE data
+	is shared between the PDumping App and the PDump client, hence an exclusive
+	access is required. The lock can be acquired and released by using the
+	PDumpCtrlLockAcquire & PDumpCtrlLockRelease functions respectively.
+******************************************************************************/
+
+static void PDumpCtrlUpdateCaptureStatus(void)
+{
+	if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_FRAMED)
+	{
+		if ((g_PDumpCtrl.ui32CurrentFrame >= g_PDumpCtrl.sCaptureRange.ui32Start) &&
+			(g_PDumpCtrl.ui32CurrentFrame <= g_PDumpCtrl.sCaptureRange.ui32End))
+		{
+			if (((g_PDumpCtrl.ui32CurrentFrame - g_PDumpCtrl.sCaptureRange.ui32Start) % g_PDumpCtrl.sCaptureRange.ui32Interval) == 0)
+			{
+				SET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+				UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+			}
+			else
+			{
+				UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+				SET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+			}
+		}
+		else
+		{
+			UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+			UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+		}
+	}
+	else if ((g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_CONTINUOUS) || (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_BLOCKED))
+	{
+		SET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+	}
+	else if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_UNSET)
+	{
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+	}
+	else
+	{
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE);
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL);
+		PVR_DPF((PVR_DBG_ERROR, "PDumpCtrlUpdateCaptureStatus: Unexpected capture mode (%x)", g_PDumpCtrl.ui32DefaultCapMode));
+	}
+
+}
+
+static INLINE IMG_UINT32 PDumpCtrlCapModIsBlocked(void)
+{
+	return (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_BLOCKED);
+}
+
+static INLINE IMG_UINT32 PDumpCtrlMinimalFirstBlock(void)
+{
+	/* If ui32Start is set to zero, first block length will be set to minimum
+	 * (i.e. PDUMP_BLOCKLEN_MIN), else it will be of same length as that of
+	 * rest of the blocks (i.e. ui32BlockLength)
+	 *
+	 * Having shorter first block reduces playback time of final capture.
+	 * */
+
+	return (PDumpCtrlCapModIsBlocked() && (g_PDumpCtrl.sCaptureRange.ui32Start == 0));
+}
+
+static void PDumpCtrlSetBlock(IMG_UINT32 ui32BlockNum)
+{
+	g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock = PDumpCtrlCapModIsBlocked()? ui32BlockNum : PDUMP_BLOCKNUM_INVALID;
+}
+
+static INLINE IMG_UINT32 PDumpCtrlGetBlock(void)
+{
+	return PDumpCtrlCapModIsBlocked()? g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock : PDUMP_BLOCKNUM_INVALID;
+}
+
+static PVRSRV_ERROR PDumpCtrlForcedStop(void)
+{
+	/* In block-mode on forced stop request, capture will be stopped after (current_frame + 1)th frame number.
+	 * This ensures that DumpAfterRender always be called on last frame before exiting the PDump capturing
+	 * */
+	g_PDumpCtrl.sCaptureRange.ui32End = g_PDumpCtrl.ui32CurrentFrame + 1;
+
+	return PVRSRV_OK;
+}
+
+static INLINE IMG_BOOL PDumpCtrlIsCaptureForceStopped(void)
+{
+	return (PDumpCtrlCapModIsBlocked() && (g_PDumpCtrl.ui32CurrentFrame > g_PDumpCtrl.sCaptureRange.ui32End));
+}
+
+static void PDumpCtrlSetCurrentFrame(IMG_UINT32 ui32Frame)
+{
+	g_PDumpCtrl.ui32CurrentFrame = ui32Frame;
+
+	PDumpCtrlUpdateCaptureStatus();
+
+	/* Force PDump module to suspend PDumping on forced capture stop */
+	if ((PDumpCtrlGetModuleState() != PDUMP_SM_FORCED_SUSPENDED) && PDumpCtrlIsCaptureForceStopped())
+	{
+		PVR_LOG(("PDump forced capture stop received. Suspend PDumping to force driver reload before next capture."));
+		g_PDumpCtrl.eServiceState = PDUMP_SM_FORCED_SUSPENDED;
+	}
+#if defined(PDUMP_TRACE_STATE)
+	PDumpCommonDumpState();
+#endif
+}
+
+static void PDumpCtrlSetDefaultCaptureParams(IMG_UINT32 ui32Mode, IMG_UINT32 ui32Start, IMG_UINT32 ui32End, IMG_UINT32 ui32Interval, IMG_UINT32 ui32AutoTermTimeout)
+{
+	/* Set the capture range to that supplied by the PDump client tool
+	 */
+	g_PDumpCtrl.ui32DefaultCapMode = ui32Mode;
+	g_PDumpCtrl.sCaptureRange.ui32Start = ui32Start;
+	g_PDumpCtrl.sCaptureRange.ui32End = ui32End;
+	g_PDumpCtrl.sCaptureRange.ui32Interval = ui32Interval;
+
+	/* Disable / Enable AUTO_TERMINATE behaviour if AutoTermTimeout is set */
+	if (ui32AutoTermTimeout == 0U)
+	{
+		if (g_PDumpTimerID != NULL)
+		{
+			PVRSRV_ERROR eError;
+
+			eError = OSDisableTimer(g_PDumpTimerID);
+			PVR_LOG_IF_ERROR(eError, "OSDisableTimer");
+
+			/* Now destroy it too */
+			eError = OSRemoveTimer(g_PDumpTimerID);
+			PVR_LOG_IF_ERROR(eError, "OSRemoveTimer");
+
+			g_PDumpTimerID = NULL;
+		}
+		g_PDumpCtrl.ui32TimeoutFrequency = 0U;
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED);
+	}
+
+	/* Set pdump block mode ctrl variables */
+	g_PDumpCtrl.sBlockCtrl.ui32BlockLength = (ui32Mode == PDUMP_CAPMODE_BLOCKED)? ui32Interval : 0; /* ui32Interval is interpreted as block length */
+	g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock = PDUMP_BLOCKNUM_INVALID;
+
+	/* Change module state to record capture client connected */
+	if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_UNSET)
+		g_PDumpCtrl.eServiceState = PDUMP_SM_READY;
+	else
+		g_PDumpCtrl.eServiceState = PDUMP_SM_READY_CLIENT_CONNECTED;
+
+	/* Reset the current frame on reset of the capture range, the helps to
+	 * avoid inter-pdump start frame issues when the driver is not reloaded.
+	 * No need to call PDumpCtrlUpdateCaptureStatus() direct as the set
+	 * current frame call will.
+	 */
+	PDumpCtrlSetCurrentFrame(0);
+
+}
+
+static IMG_UINT32 PDumpCtrlGetCurrentFrame(void)
+{
+	return g_PDumpCtrl.ui32CurrentFrame;
+}
+
+static INLINE IMG_BOOL PDumpCtrlCaptureOn(void)
+{
+	return ((g_PDumpCtrl.eServiceState == PDUMP_SM_READY_CLIENT_CONNECTED) &&
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE)) ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE IMG_BOOL PDumpCtrlCaptureInInterval(void)
+{
+	return ((g_PDumpCtrl.eServiceState == PDUMP_SM_READY_CLIENT_CONNECTED) &&
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL)) ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE IMG_BOOL PDumpCtrlCaptureRangePast(void)
+{
+	return (g_PDumpCtrl.ui32CurrentFrame > g_PDumpCtrl.sCaptureRange.ui32End);
+}
+
+static IMG_BOOL PDumpCtrlIsLastCaptureFrame(void)
+{
+	if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_FRAMED)
+	{
+		/* Is the next capture frame within the range end limit? */
+		if ((g_PDumpCtrl.ui32CurrentFrame + g_PDumpCtrl.sCaptureRange.ui32Interval) > g_PDumpCtrl.sCaptureRange.ui32End)
+		{
+			return IMG_TRUE;
+		}
+	}
+	else if (g_PDumpCtrl.ui32DefaultCapMode == PDUMP_CAPMODE_BLOCKED)
+	{
+		if (g_PDumpCtrl.ui32CurrentFrame == g_PDumpCtrl.sCaptureRange.ui32End)
+		{
+			return IMG_TRUE;
+		}
+	}
+	/* Return false for all other conditions: framed mode but not last frame,
+	 * continuous mode; unset mode.
+	 */
+	return IMG_FALSE;
+}
+
+static INLINE IMG_BOOL PDumpCtrlInitPhaseComplete(void)
+{
+	return !CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE);
+}
+
+static INLINE void PDumpCtrlSetInitPhaseComplete(IMG_BOOL bIsComplete)
+{
+	PDUMP_HERE_VAR;
+
+	if (bIsComplete)
+	{
+		UNSET_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE);
+		PDUMP_HEREA(102);
+	}
+	else
+	{
+		SET_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE);
+		PDUMP_HEREA(103);
+	}
+}
+
+static INLINE void PDumpCtrlPowerTransitionStart(void)
+{
+	g_PDumpCtrl.InPowerTransitionPID = OSGetCurrentProcessID();
+}
+
+static INLINE void PDumpCtrlPowerTransitionEnd(void)
+{
+	g_PDumpCtrl.InPowerTransitionPID = 0;
+}
+
+static INLINE IMG_PID PDumpCtrlInPowerTransitionPID(void)
+{
+	return g_PDumpCtrl.InPowerTransitionPID;
+}
+
+static INLINE IMG_BOOL PDumpCtrlInPowerTransition(void)
+{
+	IMG_BOOL bPDumpInPowerTransition = IMG_FALSE;
+	if (PDumpCtrlInPowerTransitionPID())
+	{
+		bPDumpInPowerTransition = IMG_TRUE;
+	}
+	return bPDumpInPowerTransition;
+}
+
+static PVRSRV_ERROR PDumpCtrlGetState(IMG_UINT64 *ui64State)
+{
+	PDUMP_SM eState;
+
+	*ui64State = 0;
+
+	if (PDumpCtrlCaptureOn())
+	{
+		*ui64State |= PDUMP_STATE_CAPTURE_FRAME;
+	}
+
+	if (PDumpCtrlCaptureInInterval())
+	{
+		*ui64State |= PDUMP_STATE_CAPTURE_IN_INTERVAL;
+	}
+
+	eState = PDumpCtrlGetModuleState();
+
+	if (eState == PDUMP_SM_READY_CLIENT_CONNECTED)
+	{
+		*ui64State |= PDUMP_STATE_CONNECTED;
+	}
+
+	if (eState == PDUMP_SM_ERROR_SUSPENDED)
+	{
+		*ui64State |= PDUMP_STATE_SUSPENDED;
+	}
+
+	if (CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED))
+	{
+		*ui64State |= PDUMP_STATE_APP_TERMINATED;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR PDumpSetAutoTerminate(IMG_UINT32 ui32TimeoutFrequency)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bEnable = (ui32TimeoutFrequency != 0U) ? IMG_TRUE : IMG_FALSE;
+
+	eError = PDumpReady();
+	PVR_LOG_RETURN_IF_ERROR(eError, "PDumpReady");
+
+	g_PDumpCtrl.ui32TimeoutFrequency = ui32TimeoutFrequency;
+
+	if (bEnable)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: ENABLING Auto Termination - Timeout %u",
+		         __func__, ui32TimeoutFrequency));
+		PDUMP_REFCOUNT_PRINT(("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                      &gPDumpNumConnex, OSAtomicRead(&gPDumpNumConnex)));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: DISABLING Auto Termination",
+		         __func__));
+		PDUMP_REFCOUNT_PRINT(("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                      &gPDumpNumConnex,
+		                      OSAtomicRead(&gPDumpNumConnex)));
+	}
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+	End of PDumpCtrl*** functions
+******************************************************************************/
+
+/*
+	Wrapper functions which need to be exposed in pdump_km.h for use in other
+	pdump_*** modules safely. These functions call the specific PDumpCtrl layer
+	function after acquiring the PDUMP_CTRL_STATE lock, hence making the calls
+	from other modules hassle free by avoiding the acquire/release CtrlLock
+	calls.
+*/
+
+static INLINE void PDumpModuleTransitionState(PDUMP_SM eNewState)
+{
+	PDumpCtrlLockAcquire();
+	g_PDumpCtrl.eServiceState = eNewState;
+	PDumpCtrlLockRelease();
+}
+
+void PDumpPowerTransitionStart(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PDumpCtrlLockAcquire();
+		PDumpCtrlPowerTransitionStart();
+		PDumpCtrlLockRelease();
+	}
+}
+
+void PDumpPowerTransitionEnd(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PDumpCtrlLockAcquire();
+		PDumpCtrlPowerTransitionEnd();
+		PDumpCtrlLockRelease();
+	}
+}
+
+static PVRSRV_ERROR PDumpGetCurrentBlockKM(IMG_UINT32 *pui32BlockNum)
+{
+	PDumpCtrlLockAcquire();
+	*pui32BlockNum = PDumpCtrlGetBlock();
+	PDumpCtrlLockRelease();
+
+	return PVRSRV_OK;
+}
+
+static IMG_BOOL PDumpIsClientConnected(void)
+{
+	IMG_BOOL bPDumpClientConnected;
+
+	PDumpCtrlLockAcquire();
+	bPDumpClientConnected = (PDumpCtrlGetModuleState() == PDUMP_SM_READY_CLIENT_CONNECTED);
+	PDumpCtrlLockRelease();
+
+	return bPDumpClientConnected;
+}
+
+/* Prototype write allowed for exposure in PDumpCheckFlagsWrite */
+static IMG_BOOL PDumpWriteAllowed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32Flags, IMG_UINT32* ui32ExitHere);
+
+IMG_BOOL PDumpCheckFlagsWrite(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_UINT32 ui32Flags)
+{
+	return PDumpWriteAllowed(psDeviceNode, ui32Flags, NULL);
+}
+
+/*****************************************************************************/
+/* PDump Common Write Layer just above common Transport Layer                */
+/*****************************************************************************/
+
+
+/*!
+ * \name	_PDumpOSGetStreamOffset
+ */
+static IMG_BOOL _PDumpSetSplitMarker(IMG_HANDLE hStream, IMG_BOOL bRemoveOld)
+{
+	PVRSRV_ERROR eError;
+	/* We have to indicate the reader that we wish to split. Insert an EOS packet in the TL stream */
+	eError = TLStreamMarkEOS(hStream, bRemoveOld);
+
+	/* If unsuccessful, return false */
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "TLStreamMarkEOS");
+
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+IMG_BOOL PDumpIsDevicePermitted(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if ((void*)psDeviceNode == (void*)PDUMP_MAGIC_COOKIE)
+	{
+		/* Always permit PDumping if passed 'magic' cookie */
+		return IMG_TRUE;
+	}
+
+	if (psDeviceNode)
+	{
+		if (psDeviceNode->eDevState < PVRSRV_DEVICE_STATE_CREATED)
+		{
+			PVR_DPF((PVR_DBG_FATAL,"%s: PDump output requested for Device %d "
+			         "before device created. Not permitted - please fix driver.",
+			         __func__, psDeviceNode->sDevId.ui32InternalID));
+			return IMG_FALSE;
+		}
+		if ((psDeviceNode->sDevId.ui32InternalID > PVRSRV_MAX_DEVICES) ||
+		    ((psPVRSRVData->ui32PDumpBoundDevice < PVRSRV_MAX_DEVICES) &&
+		     (psDeviceNode->sDevId.ui32InternalID != psPVRSRVData->ui32PDumpBoundDevice)))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else
+	{
+		/* Assert if provided with a NULL psDeviceNode */
+		OSDumpStack();
+		PVR_ASSERT(psDeviceNode);
+		return IMG_FALSE;
+	}
+	return IMG_TRUE;
+}
+
+/*
+	Checks in this method were seeded from the original PDumpWriteILock()
+	and DBGDrivWriteCM() and have grown since to ensure PDump output
+	matches legacy output.
+	Note: the order of the checks in this method is important as some
+	writes have multiple pdump flags set!
+ */
+static IMG_BOOL PDumpWriteAllowed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32Flags, IMG_UINT32* ui32ExitHere)
+{
+	PDUMP_HERE_VAR;
+
+	/* No writes if for a different device than the PDump-bound device
+	 *  NB. psDeviceNode may be NULL if called during initialisation
+	 */
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PDUMP_HERE(5);
+		goto returnFalse;
+	}
+
+	/* PDUMP_FLAGS_CONTINUOUS and PDUMP_FLAGS_PERSISTENT can't come together. */
+	PVR_ASSERT(IMG_FALSE == ((ui32Flags & PDUMP_FLAGS_CONTINUOUS) &&
+		                     (ui32Flags & PDUMP_FLAGS_PERSISTENT)));
+
+	/* Lock down the PDUMP_CTRL_STATE struct before calling the following
+	   PDumpCtrl*** functions. This is to avoid updates to the Control data
+	   while we are reading from it */
+	PDumpCtrlLockAcquire();
+
+	/* No writes if in framed mode and range pasted */
+	if (PDumpCtrlCaptureRangePast())
+	{
+		PDUMP_HERE(10);
+		goto unlockAndReturnFalse;
+	}
+
+	/* No writes while PDump is not ready or is suspended */
+	if (PDumpReady() != PVRSRV_OK)
+	{
+		PDUMP_HERE(11);
+		goto unlockAndReturnFalse;
+	}
+
+	/* Prevent PDumping during a power transition */
+	if (PDumpCtrlInPowerTransition())
+	{	/* except when it's flagged */
+		if (ui32Flags & PDUMP_FLAGS_POWER)
+		{
+			PDUMP_HERE(20);
+			goto unlockAndReturnTrue;
+		}
+		else if (PDumpCtrlInPowerTransitionPID() == OSGetCurrentProcessID())
+		{
+			PDUMP_HERE(16);
+			goto unlockAndReturnFalse;
+		}
+	}
+
+	/* Always allow dumping in init phase and when persistent flagged */
+	if (ui32Flags & PDUMP_FLAGS_PERSISTENT)
+	{
+		PDUMP_HERE(12);
+		goto unlockAndReturnTrue;
+	}
+	if (!PDumpCtrlInitPhaseComplete())
+	{
+		PDUMP_HERE(15);
+		goto unlockAndReturnTrue;
+	}
+
+	/* The following checks are made when the driver has completed initialisation */
+	/* No last/deinit statements allowed when not in initialisation phase */
+	else /* init phase over */
+	{
+		if (ui32Flags & PDUMP_FLAGS_DEINIT)
+		{
+			PVR_ASSERT(0);
+			PDUMP_HERE(17);
+			goto unlockAndReturnFalse;
+		}
+	}
+
+	/* If PDump client connected allow continuous flagged writes */
+	if (PDUMP_IS_CONTINUOUS(ui32Flags))
+	{
+		if (PDumpCtrlGetModuleState() != PDUMP_SM_READY_CLIENT_CONNECTED) /* Is client connected? */
+		{
+			PDUMP_HERE(13);
+			goto unlockAndReturnFalse;
+		}
+		PDUMP_HERE(14);
+		goto unlockAndReturnTrue;
+	}
+
+	/* If in a capture interval but a write is still required.
+	 * Force write out if FLAGS_INTERVAL has been set and we are in
+	 * a capture interval */
+	if (ui32Flags & PDUMP_FLAGS_INTERVAL)
+	{
+		if (PDumpCtrlCaptureInInterval()){
+			PDUMP_HERE(21);
+			goto unlockAndReturnTrue;
+		}
+	}
+
+	/*
+		If no flags are provided then it is FRAMED output and the frame
+		range must be checked matching expected behaviour.
+	 */
+	if (!PDumpCtrlCaptureOn())
+	{
+		PDUMP_HERE(18);
+		goto unlockAndReturnFalse;
+	}
+
+	PDUMP_HERE(19);
+
+unlockAndReturnTrue:
+	/* Allow the write to take place */
+
+	PDumpCtrlLockRelease();
+	return IMG_TRUE;
+
+unlockAndReturnFalse:
+	PDumpCtrlLockRelease();
+returnFalse:
+	if (ui32ExitHere != NULL)
+	{
+		*ui32ExitHere = here;
+	}
+	return IMG_FALSE;
+}
+
+
+/*************************************************************************/ /*!
+ @Function		PDumpWriteToBuffer
+ @Description	Write the supplied data to the PDump stream buffer and attempt
+                to handle any buffer full conditions to ensure all the data
+                requested to be written, is.
+
+ @Input			psDeviceNode The device the PDump pertains to
+ @Input			psStream	The address of the PDump stream buffer to write to
+ @Input			pui8Data    Pointer to the data to be written
+ @Input			ui32BCount	Number of bytes to write
+ @Input			ui32Flags	PDump statement flags.
+
+ @Return		IMG_UINT32  Actual number of bytes written, may be less than
+                            ui32BCount when buffer full condition could not
+                            be avoided.
+*/ /**************************************************************************/
+static IMG_UINT32 PDumpWriteToBuffer(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     PDUMP_STREAM* psStream,
+                                     IMG_UINT8 *pui8Data,
+                                     IMG_UINT32 ui32BCount,
+                                     IMG_UINT32 ui32Flags)
+{
+	IMG_UINT32	ui32BytesToBeWritten;
+	IMG_UINT32	ui32Off = 0;
+	IMG_BYTE *pbyDataBuffer;
+	IMG_UINT32 ui32BytesAvailable = 0;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiRetries = 0;
+
+	/* Check PDump stream validity */
+	if (psStream->hTL == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PDumpWriteToBuffer: PDump stream '%s' is invalid", psStream->pszName));
+		return 0;
+	}
+
+	/* This API always called by holding pdump write lock
+	 * to ensure atomic pdump write happened and
+	 * if not holding pdump lock then assert.
+	 */
+	PDumpAssertWriteLockHeld();
+
+	/* No need to check size of data to write as this is asserted
+	 * higher up in the call stack as 1KB and 16KB for each channel
+	 * respectively. */
+
+	while (ui32BCount > 0)
+	{
+		ui32BytesToBeWritten = MIN ( ui32BCount, psStream->ui32MaxAllowedWriteSize );
+
+		eError = TLStreamReserve2(psStream->hTL, &pbyDataBuffer, ui32BytesToBeWritten, 0, &ui32BytesAvailable, NULL);
+		if (eError == PVRSRV_ERROR_STREAM_FULL)
+		{
+			psStream->ui32BufferFullRetries++;
+
+			/*! Retry write2 only if available bytes is at least 1024 or more. */
+			if (ui32BytesAvailable >= 0x400)
+			{
+				ui32BytesToBeWritten = ui32BytesAvailable;
+				PVR_DPF((PVR_DBG_WARNING, "PDumpWriteToBuffer: TL buffer '%s' retrying write2=%u out of %u", psStream->pszName, ui32BytesToBeWritten, ui32BCount));
+				eError = TLStreamReserve(psStream->hTL, &pbyDataBuffer, ui32BytesToBeWritten);
+				/*! Not expected to get PVRSRV_ERROR_STREAM_FULL error and other error may get */
+				PVR_ASSERT(eError != PVRSRV_ERROR_STREAM_FULL);
+			}
+			else
+			{
+				uiRetries++;
+				PVR_DPF((PVR_DBG_WARNING, "PDumpWriteToBuffer: TL buffer '%s' full, rq=%u, av=%u, retrying write", psStream->pszName, ui32BCount, ui32BytesAvailable));
+
+				/* Check if we are out of retries , if so then print warning */
+				if (uiRetries >= MAX_PDUMP_WRITE_RETRIES)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+					         "PDumpWriteToBuffer: PDump writes blocked to dump %d bytes, %s TLBuffers full for %d seconds, check system",
+					         ui32BCount,
+					         psStream->pszName,
+					         ((200 * uiRetries)/1000)));
+
+					if (uiRetries > psStream->ui32HighestRetriesWatermark)
+					{
+						psStream->ui32HighestRetriesWatermark = uiRetries;
+					}
+
+					psStream->ui32BufferFullAborts++;
+					uiRetries = 0;
+
+					/* As uiRetries exceed max write retries that means,
+					 * something went wrong in system and thus suspend pdump.
+					 */
+					PDumpModuleTransitionState(PDUMP_SM_ERROR_SUSPENDED);
+					return 0;
+				}
+
+				OSSleepms(100);
+				continue;
+			}
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+			PVR_ASSERT(pbyDataBuffer != NULL);
+
+			OSDeviceMemCopy((void*)pbyDataBuffer, pui8Data + ui32Off, ui32BytesToBeWritten);
+
+			eError = TLStreamCommit(psStream->hTL, ui32BytesToBeWritten);
+			if (PVRSRV_OK != eError)
+			{
+				return 0;
+			}
+
+			if (uiRetries > psStream->ui32HighestRetriesWatermark)
+			{
+				psStream->ui32HighestRetriesWatermark = uiRetries;
+			}
+
+			uiRetries = 0;
+			ui32Off += ui32BytesToBeWritten;
+			ui32BCount -= ui32BytesToBeWritten;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToBuffer: TLStreamReserve2(%s) unrecoverable error %s", psStream->pszName, PVRSRVGETERRORSTRING(eError)));
+			/* Fatal -suspend PDump to prevent flooding kernel log buffer */
+			PVR_LOG(("Unrecoverable error, PDump suspended!"));
+
+			PDumpModuleTransitionState(PDUMP_SM_ERROR_SUSPENDED);
+			return 0;
+		}
+
+		/*
+		   if the capture range is unset
+		   (which is detected via PDumpWriteAllowed())
+		*/
+
+		if (!PDumpWriteAllowed(psDeviceNode, ui32Flags, NULL))
+		{
+			psStream->ui32BufferFullAborts++;
+			break;
+		}
+	}
+
+	return ui32Off;
+}
+
+/*************************************************************************/ /*!
+ @Function      PDumpWriteToChannel
+ @Description   Write the supplied data to the PDump channel specified obeying
+                flags to write to the necessary channel buffers.
+
+ @Input         psDeviceNode The device the PDump pertains to
+ @Input         psChannel   Address of the script or parameter channel object
+ @Input/Output  psWOff      Address of the channel write offsets object to
+                            update on successful writing
+ @Input         pui8Data    Pointer to the data to be written
+ @Input         ui32Size    Number of bytes to write
+ @Input         ui32Flags   PDump statement flags, they may be clear (no flags)
+                            or persistent flagged and they determine how the
+                            which implies framed data, continuous flagged, data
+                            is output. On the first test app run after driver
+                            load, the Display Controller dumps a resource that
+                            is persistent and this needs writing to both the
+                            init (persistent) and main (continuous) channel
+                            buffers to ensure the data is dumped in subsequent
+                            test runs without reloading the driver.
+                            In subsequent runs the PDump client 'freezes' the
+                            init buffer so that only one dump of persistent
+                            data for the "extended init phase" is captured to
+                            the init buffer.
+ @Return        IMG_BOOL    True when the data has been consumed, false otherwise
+*/ /**************************************************************************/
+static IMG_BOOL PDumpWriteToChannel(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    PDUMP_CHANNEL* psChannel,
+                                    PDUMP_CHANNEL_WOFFSETS* psWOff,
+                                    IMG_UINT8* pui8Data,
+                                    IMG_UINT32 ui32Size,
+                                    IMG_UINT32 ui32Flags)
+{
+	IMG_UINT32 ui32BytesWritten = 0;
+	PDUMP_HERE_VAR;
+
+	PDUMP_HERE(210);
+
+	/* At this point, PDumpWriteAllowed() has returned TRUE (or called from
+	 * PDumpParameterChannelZeroedPageBlock() during driver init) we know the
+	 * write must proceed because:
+	 * - pdump is not suspended and
+	 * - there is not an ongoing power transition or POWER override flag is set or
+	 * - in driver init phase with ANY flag set or
+	 * - post init with the pdump client connected and
+	 * -   - PERSIST flag is present, xor
+	 * -   - the CONTINUOUS flag is present, xor
+	 * -   - in capture frame range
+	 */
+	PDumpAssertWriteLockHeld();
+
+	/* Dump data to deinit buffer when flagged as deinit */
+	if (ui32Flags & PDUMP_FLAGS_DEINIT)
+	{
+		PDUMP_HERE(211);
+		ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+		                                      &psChannel->sDeinitStream,
+		                                      pui8Data, ui32Size, ui32Flags);
+		if (ui32BytesWritten != ui32Size)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: DEINIT Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+			PDUMP_HERE(212);
+			return IMG_FALSE;
+		}
+
+		if (psWOff)
+		{
+			psWOff->ui32Deinit += ui32Size;
+		}
+
+	}
+	else
+	{
+		IMG_BOOL bDumpedToInitAlready = IMG_FALSE;
+		IMG_BOOL bMainStreamData = IMG_FALSE;
+		PDUMP_STREAM*  psStream = NULL;
+		IMG_UINT32* pui32Offset = NULL;
+
+		/* Always append persistent data to init phase so it's available on
+		 * subsequent app runs, but also to the main stream if client connected */
+		if (ui32Flags & PDUMP_FLAGS_PERSISTENT)
+		{
+			PDUMP_HERE(213);
+			ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+			                                      &psChannel->sInitStream,
+			                                      pui8Data, ui32Size, ui32Flags);
+			if (ui32BytesWritten != ui32Size)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: PERSIST Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+				PDUMP_HERE(214);
+				return IMG_FALSE;
+			}
+
+			bDumpedToInitAlready = IMG_TRUE;
+			if (psWOff)
+			{
+				psWOff->ui32Init += ui32Size;
+			}
+
+			/* Don't write continuous data if client not connected */
+			if (PDumpCtrlGetModuleState() != PDUMP_SM_READY_CLIENT_CONNECTED)
+			{
+				return IMG_TRUE;
+			}
+		}
+
+		/* Prepare to write the data to the main stream for
+		 * persistent, continuous or framed data. Override and use init
+		 * stream if driver still in init phase and we have not written
+		 * to it yet.*/
+		PDumpCtrlLockAcquire();
+		if (!PDumpCtrlInitPhaseComplete() && !bDumpedToInitAlready)
+		{
+			PDUMP_HERE(215);
+			psStream = &psChannel->sInitStream;
+			if (psWOff)
+			{
+				pui32Offset = &psWOff->ui32Init;
+			}
+		}
+		else
+		{
+			PDUMP_HERE(216);
+			psStream = &psChannel->sMainStream;
+			if (psWOff)
+			{
+				pui32Offset = &psWOff->ui32Main;
+			}
+			bMainStreamData = IMG_TRUE;
+
+		}
+		PDumpCtrlLockRelease();
+
+		if (PDumpCtrlCapModIsBlocked() && bMainStreamData && !psWOff)
+		{
+			/* if PDUMP_FLAGS_BLKDATA flag is set in Blocked mode, Make copy of Main script stream data to Block script stream as well */
+			if (ui32Flags & PDUMP_FLAGS_BLKDATA)
+			{
+				PDUMP_HERE(217);
+				ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+				                                      &psChannel->sBlockStream,
+				                                      pui8Data, ui32Size, ui32Flags);
+				if (ui32BytesWritten != ui32Size)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: BLOCK Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+					PDUMP_HERE(218);
+					return IMG_FALSE;
+				}
+			}
+		}
+
+		/* Write the data to the stream */
+		ui32BytesWritten = PDumpWriteToBuffer(psDeviceNode,
+		                                      psStream, pui8Data,
+		                                      ui32Size, ui32Flags);
+		if (ui32BytesWritten != ui32Size)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDumpWriteToChannel: MAIN Written length (%d) does not match data length (%d), PDump incomplete!", ui32BytesWritten, ui32Size));
+			PDUMP_HERE(219);
+			return IMG_FALSE;
+		}
+
+		if (pui32Offset)
+		{
+			*pui32Offset += ui32BytesWritten;
+		}
+	}
+
+	return IMG_TRUE;
+}
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+
+static IMG_UINT32 _GenerateChecksum(void *pvData, size_t uiSize)
+{
+	IMG_UINT32 ui32Sum = 0;
+	IMG_UINT32 *pui32Data = pvData;
+	IMG_UINT8 *pui8Data = pvData;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32LeftOver;
+
+	for (i = 0; i < uiSize / sizeof(IMG_UINT32); i++)
+	{
+		ui32Sum += pui32Data[i];
+	}
+
+	ui32LeftOver = uiSize % sizeof(IMG_UINT32);
+
+	while (ui32LeftOver)
+	{
+		ui32Sum += pui8Data[uiSize - ui32LeftOver];
+		ui32LeftOver--;
+	}
+
+	return ui32Sum;
+}
+
+#endif
+
+PVRSRV_ERROR PDumpWriteParameter(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT8 *pui8Data,
+                                 IMG_UINT32 ui32Size,
+                                 IMG_UINT32 ui32Flags,
+                                 IMG_UINT32* pui32FileOffset,
+                                 IMG_CHAR* aszFilenameStr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bPDumpCtrlInitPhaseComplete = IMG_FALSE;
+	IMG_UINT32 here = 0;
+	IMG_INT32 iCount;
+
+	PDumpAssertWriteLockHeld();
+
+	PVR_ASSERT(pui8Data && (ui32Size!=0));
+	PVR_ASSERT(pui32FileOffset && aszFilenameStr);
+
+	PDUMP_HERE(1);
+
+	/* Check if write can proceed? */
+	if (!PDumpWriteAllowed(psDeviceNode, ui32Flags, &here))
+	{
+		/* Abort write for the above reason but indicate what happened to
+		 * caller to avoid disrupting the driver, caller should treat it as OK
+		 * but skip any related PDump writes to the script file. */
+		return PVRSRV_ERROR_PDUMP_NOT_ALLOWED;
+	}
+
+	PDUMP_HERE(2);
+
+	PDumpCtrlLockAcquire();
+	bPDumpCtrlInitPhaseComplete = PDumpCtrlInitPhaseComplete();
+	PDumpCtrlLockRelease();
+
+	if (!bPDumpCtrlInitPhaseComplete || (ui32Flags & PDUMP_FLAGS_PERSISTENT))
+	{
+		PDUMP_HERE(3);
+
+		/* Init phase stream not expected to get above the file size max */
+		PVR_ASSERT(g_PDumpParameters.sWOff.ui32Init < g_PDumpParameters.ui32MaxFileSize);
+
+		/* Return the file write offset at which the parameter data was dumped */
+		*pui32FileOffset = g_PDumpParameters.sWOff.ui32Init;
+	}
+	else
+	{
+		PDUMP_HERE(4);
+
+		/* Do we need to signal the PDump client that a split is required? */
+		if (g_PDumpParameters.sWOff.ui32Main + ui32Size > g_PDumpParameters.ui32MaxFileSize)
+		{
+			PDUMP_HERE(5);
+			_PDumpSetSplitMarker(g_PDumpParameters.sCh.sMainStream.hTL, IMG_FALSE);
+			g_PDumpParameters.ui32FileIdx++;
+			g_PDumpParameters.sWOff.ui32Main = 0;
+		}
+
+		/* Return the file write offset at which the parameter data was dumped */
+		*pui32FileOffset = g_PDumpParameters.sWOff.ui32Main;
+	}
+
+	/* Create the parameter file name, based on index, to be used in the script */
+	if (g_PDumpParameters.ui32FileIdx == 0)
+	{
+		iCount = OSSNPrintf(aszFilenameStr, PDUMP_PARAM_MAX_FILE_NAME, PDUMP_PARAM_0_FILE_NAME);
+	}
+	else
+	{
+		PDUMP_HERE(6);
+		iCount = OSSNPrintf(aszFilenameStr, PDUMP_PARAM_MAX_FILE_NAME, PDUMP_PARAM_N_FILE_NAME, g_PDumpParameters.ui32FileIdx);
+	}
+
+	PVR_LOG_GOTO_IF_FALSE(((iCount != -1) && (iCount < PDUMP_PARAM_MAX_FILE_NAME)), "OSSNPrintf", errExit);
+
+	/* Write the parameter data to the parameter channel */
+	eError = PVRSRV_ERROR_PDUMP_BUFFER_FULL;
+	if (!PDumpWriteToChannel(psDeviceNode, &g_PDumpParameters.sCh,
+	                         &g_PDumpParameters.sWOff, pui8Data,
+	                         ui32Size, ui32Flags))
+	{
+		PDUMP_HERE(7);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpWrite", errExit);
+	}
+#if defined(PDUMP_DEBUG_OUTFILES)
+	else
+	{
+		IMG_UINT32 ui32Checksum;
+		PDUMP_GET_SCRIPT_STRING();
+
+		ui32Checksum = _GenerateChecksum(pui8Data, ui32Size);
+
+		/* CHK CHKSUM SIZE PRMOFFSET PRMFILE */
+		eError = PDumpSNPrintf(hScript, ui32MaxLen, "-- CHK 0x%08X 0x%08X 0x%08X %s",
+									ui32Checksum,
+									ui32Size,
+									*pui32FileOffset,
+									aszFilenameStr);
+		PVR_GOTO_IF_ERROR(eError, errExit);
+
+		PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+		PDUMP_RELEASE_SCRIPT_STRING();
+	}
+#endif
+
+	return PVRSRV_OK;
+
+errExit:
+	return eError;
+}
+
+
+IMG_BOOL PDumpWriteScript(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          IMG_HANDLE hString, IMG_UINT32 ui32Flags)
+{
+	PDUMP_HERE_VAR;
+
+	PVR_ASSERT(hString);
+
+	PDumpAssertWriteLockHeld();
+
+	PDUMP_HERE(201);
+
+#if defined(DEBUG)
+	/* Since buffer sizes and buffer writing/reading are a balancing act to
+	 * avoid buffer full errors, check here our assumption on the maximum write size.
+	 */
+	{
+		IMG_UINT32 ui32Size = (IMG_UINT32) OSStringLength((const IMG_CHAR *)hString);
+		if (ui32Size > 0x400) // 1KB
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PDUMP large script write %u bytes", ui32Size));
+			OSDumpStack();
+		}
+	}
+#endif
+
+	if (!PDumpWriteAllowed(psDeviceNode, ui32Flags, NULL))
+	{
+		/* Abort write for the above reasons but indicated it was OK to
+		 * caller to avoid disrupting the driver */
+		return IMG_TRUE;
+	}
+
+	if (PDumpCtrlCapModIsBlocked())
+	{
+		if (ui32Flags & PDUMP_FLAGS_FORCESPLIT)
+		{
+			IMG_UINT32 ui32CurrentBlock;
+
+			PDumpGetCurrentBlockKM(&ui32CurrentBlock);
+			/* Keep Main stream script output files belongs to first and last block only */
+			if (ui32CurrentBlock == 1)
+			{
+				/* To keep first(0th) block, do not remove old script file while
+				 * splitting to second(1st) block (i.e. bRemoveOld=IMG_FALSE).
+				 * */
+				_PDumpSetSplitMarker(g_PDumpScript.sCh.sMainStream.hTL, IMG_FALSE);
+			}
+			else
+			{
+				/* Previous block's Main script output file will be removed
+				 * before splitting to next
+				 * */
+				_PDumpSetSplitMarker(g_PDumpScript.sCh.sMainStream.hTL, IMG_TRUE);
+			}
+
+			/* Split Block stream output file
+			 *
+			 * We are keeping block script output files from all PDump blocks.
+			 * */
+			_PDumpSetSplitMarker(g_PDumpScript.sCh.sBlockStream.hTL, IMG_FALSE);
+			g_PDumpScript.ui32FileIdx++;
+		}
+	}
+
+	return PDumpWriteToChannel(psDeviceNode, &g_PDumpScript.sCh, NULL,
+	                          (IMG_UINT8*) hString,
+	                          (IMG_UINT32) OSStringLength((IMG_CHAR*) hString),
+	                          ui32Flags);
+}
+
+
+/*****************************************************************************/
+
+
+struct _PDUMP_CONNECTION_DATA_ {
+	ATOMIC_T                  sRefCount;
+	POS_LOCK                  hLock;                       /*!< Protects access to sListHead. */
+	DLLIST_NODE               sListHead;
+	IMG_UINT32                ui32LastSetFrameNumber;
+	PDUMP_TRANSITION_EVENT    eLastEvent;                  /*!< Last processed transition event */
+	PDUMP_TRANSITION_EVENT    eFailedEvent;                /*!< Failed transition event to retry */
+	PFN_PDUMP_SYNCBLOCKS      pfnPDumpSyncBlocks;          /*!< Callback to PDump sync blocks */
+	void                      *hSyncPrivData;              /*!< Sync private data */
+};
+
+static PDUMP_CONNECTION_DATA * _PDumpConnectionAcquire(PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psPDumpConnectionData->sRefCount);
+
+	PDUMP_REFCOUNT_PRINT("%s: PDump connection %p, refcount = %d", __func__,
+	                     psPDumpConnectionData, iRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+
+	return psPDumpConnectionData;
+}
+
+static void _PDumpConnectionRelease(PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psPDumpConnectionData->sRefCount);
+	if (iRefCount == 0)
+	{
+		OSLockDestroy(psPDumpConnectionData->hLock);
+		PVR_ASSERT(dllist_is_empty(&psPDumpConnectionData->sListHead));
+		OSFreeMem(psPDumpConnectionData);
+	}
+
+	PDUMP_REFCOUNT_PRINT("%s: PDump connection %p, refcount = %d", __func__,
+	                     psPDumpConnectionData, iRefCount);
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInitStreams
+ * Outputs        : None
+ * Returns        :
+ * Description    : Create the PDump streams
+******************************************************************************/
+static PVRSRV_ERROR PDumpInitStreams(PDUMP_CHANNEL* psParam, PDUMP_CHANNEL* psScript)
+{
+
+	PVRSRV_ERROR   eError;
+	TL_STREAM_INFO sTLStreamInfo;
+
+	/* TL - Create the streams */
+
+	/**************************** Parameter stream ***************************/
+
+	/* Parameter - Init */
+	eError = TLStreamCreate(&psParam->sInitStream.hTL,
+				psParam->sInitStream.pszName, psParam->sInitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ParamInit", end);
+
+	TLStreamInfo(psParam->sInitStream.hTL, &sTLStreamInfo);
+	psParam->sInitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Parameter - Main */
+	eError = TLStreamCreate(&psParam->sMainStream.hTL,
+				psParam->sMainStream.pszName, psParam->sMainStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER ,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ParamMain", param_main_failed);
+
+	TLStreamInfo(psParam->sMainStream.hTL, &sTLStreamInfo);
+	psParam->sMainStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Parameter - Deinit */
+	eError = TLStreamCreate(&psParam->sDeinitStream.hTL,
+				psParam->sDeinitStream.pszName,	psParam->sDeinitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ParamDeinit", param_deinit_failed);
+
+	TLStreamInfo(psParam->sDeinitStream.hTL, &sTLStreamInfo);
+	psParam->sDeinitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Parameter - Block */
+	/* As in current implementation Block script stream is just a filtered
+	 * Main script stream using PDUMP_FLAGS_BLKDATA flag, no separate
+	 * Parameter stream is needed. Block script will be referring to the
+	 * same Parameters as that of Main script stream.
+	 */
+
+	/***************************** Script streams ****************************/
+
+	/* Script - Init */
+	eError = TLStreamCreate(&psScript->sInitStream.hTL,
+				psScript->sInitStream.pszName, psScript->sInitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptInit", script_init_failed);
+
+	TLStreamInfo(psScript->sInitStream.hTL, &sTLStreamInfo);
+	psScript->sInitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Script - Main */
+	eError = TLStreamCreate(&psScript->sMainStream.hTL,
+				psScript->sMainStream.pszName, psScript->sMainStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptMain", script_main_failed);
+
+	TLStreamInfo(psScript->sMainStream.hTL, &sTLStreamInfo);
+	psScript->sMainStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Script - Deinit */
+	eError = TLStreamCreate(&psScript->sDeinitStream.hTL,
+				psScript->sDeinitStream.pszName, psScript->sDeinitStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER | TL_FLAG_PERMANENT_NO_WRAP,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptDeinit", script_deinit_failed);
+
+	TLStreamInfo(psScript->sDeinitStream.hTL, &sTLStreamInfo);
+	psScript->sDeinitStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	/* Script - Block */
+	eError = TLStreamCreate(&psScript->sBlockStream.hTL,
+				psScript->sBlockStream.pszName, psScript->sBlockStream.ui32BufferSize,
+				TL_OPMODE_DROP_NEWER,
+				NULL, NULL,
+				NULL, NULL,
+				NULL, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate ScriptBlock", script_block_failed);
+
+	TLStreamInfo(psScript->sBlockStream.hTL, &sTLStreamInfo);
+	psScript->sBlockStream.ui32MaxAllowedWriteSize = sTLStreamInfo.maxTLpacketSize;
+
+	return PVRSRV_OK;
+
+script_block_failed:
+	TLStreamClose(psScript->sDeinitStream.hTL);
+
+script_deinit_failed:
+	TLStreamClose(psScript->sMainStream.hTL);
+
+script_main_failed:
+	TLStreamClose(psScript->sInitStream.hTL);
+
+script_init_failed:
+	TLStreamClose(psParam->sDeinitStream.hTL);
+
+param_deinit_failed:
+	TLStreamClose(psParam->sMainStream.hTL);
+
+param_main_failed:
+	TLStreamClose(psParam->sInitStream.hTL);
+
+end:
+	return eError;
+}
+/******************************************************************************
+ * Function Name  : PDumpDeInitStreams
+ * Inputs         : psParam, psScript
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Deinitialises the PDump streams
+******************************************************************************/
+static void PDumpDeInitStreams(PDUMP_CHANNEL* psParam, PDUMP_CHANNEL* psScript)
+{
+	/* Script streams */
+	TLStreamClose(psScript->sDeinitStream.hTL);
+	TLStreamClose(psScript->sMainStream.hTL);
+	TLStreamClose(psScript->sInitStream.hTL);
+	TLStreamClose(psScript->sBlockStream.hTL);
+
+	/* Parameter streams */
+	TLStreamClose(psParam->sDeinitStream.hTL);
+	TLStreamClose(psParam->sMainStream.hTL);
+	TLStreamClose(psParam->sInitStream.hTL);
+
+}
+
+/******************************************************************************
+ * Function Name  : PDumpParameterChannelZeroedPageBlock
+ * Inputs         : psDeviceNode
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Set up the zero page block in the parameter stream
+******************************************************************************/
+static PVRSRV_ERROR PDumpParameterChannelZeroedPageBlock(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT8 aui8Zero[32] = { 0 };
+	size_t uiBytesToWrite;
+	PVRSRV_ERROR eError;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE;
+	IMG_UINT32 ui32GeneralNon4KHeapPageSize;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, GeneralNon4KHeapPageSize,
+			&ui32AppHintDefault, &ui32GeneralNon4KHeapPageSize);
+	OSFreeAppHintState(pvAppHintState);
+
+	/* ZeroPageSize can't be smaller than page size */
+	g_PDumpParameters.uiZeroPageSize = MAX(ui32GeneralNon4KHeapPageSize, OSGetPageSize());
+
+	/* ensure the zero page size of a multiple of the zero source on the stack */
+	PVR_ASSERT(g_PDumpParameters.uiZeroPageSize % sizeof(aui8Zero) == 0);
+
+	/* the first write gets the parameter file name and stream offset,
+	 * then subsequent writes do not need to know this as the data is
+	 * contiguous in the stream
+	 */
+	PDUMP_LOCK(0);
+	eError = PDumpWriteParameter(psDeviceNode, aui8Zero,
+							sizeof(aui8Zero),
+							0,
+							&g_PDumpParameters.uiZeroPageOffset,
+							g_PDumpParameters.szZeroPageFilename);
+
+	/* Also treat PVRSRV_ERROR_PDUMP_NOT_ALLOWED as an error in this case
+	 * as it should never happen since all writes during driver Init are
+	 * allowed.
+	*/
+	PVR_GOTO_IF_ERROR(eError, err_write);
+
+	uiBytesToWrite = g_PDumpParameters.uiZeroPageSize - sizeof(aui8Zero);
+
+	while (uiBytesToWrite)
+	{
+		IMG_BOOL bOK;
+
+		bOK = PDumpWriteToChannel(psDeviceNode,
+								  &g_PDumpParameters.sCh,
+								  &g_PDumpParameters.sWOff,
+								  aui8Zero,
+								  sizeof(aui8Zero), 0);
+
+		if (!bOK)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PDUMP_BUFFER_FULL, err_write);
+		}
+
+		uiBytesToWrite -= sizeof(aui8Zero);
+	}
+
+err_write:
+	PDUMP_UNLOCK(0);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to initialise parameter stream zero block"));
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpGetParameterZeroPageInfo
+ * Inputs         : None
+ * Outputs        : puiZeroPageOffset: set to the offset of the zero page
+ *                : puiZeroPageSize: set to the size of the zero page
+ *                : ppszZeroPageFilename: set to a pointer to the PRM file name
+ *                :                       containing the zero page
+ * Returns        : None
+ * Description    : Get information about the zero page
+******************************************************************************/
+void PDumpGetParameterZeroPageInfo(PDUMP_FILEOFFSET_T *puiZeroPageOffset,
+					size_t *puiZeroPageSize,
+					const IMG_CHAR **ppszZeroPageFilename)
+{
+		*puiZeroPageOffset = g_PDumpParameters.uiZeroPageOffset;
+		*puiZeroPageSize = g_PDumpParameters.uiZeroPageSize;
+		*ppszZeroPageFilename = g_PDumpParameters.szZeroPageFilename;
+}
+
+
+PVRSRV_ERROR PDumpInitCommon(void)
+{
+	PVRSRV_ERROR eError;
+	PDUMP_HERE_VAR;
+
+	PDUMP_HEREA(2010);
+
+	/* Initialised with default initial value */
+	OSAtomicWrite(&g_sConnectionCount, 0);
+#if defined(PDUMP_DEBUG_OUTFILES)
+	OSAtomicWrite(&g_sEveryLineCounter, 1);
+#endif
+
+	eError = OSLockCreate(&g_hPDumpWriteLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", errRet);
+
+	/* Initialise PDump control module in common layer, also sets
+	 * state to PDUMP_SM_INITIALISING.
+	 */
+	eError = PDumpCtrlInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCtrlInit", errRetLock);
+
+	/* Call environment specific PDump initialisation Part 2*/
+	eError = PDumpInitStreams(&g_PDumpParameters.sCh, &g_PDumpScript.sCh);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpInitStreams", errRetCtrl);
+
+	/* PDump now ready for write calls */
+	PDumpModuleTransitionState(PDUMP_SM_READY);
+
+	PDUMP_HEREA(2011);
+
+	/* Test PDump initialised and ready by logging driver details */
+	eError = PDumpCommentWithFlags((PVRSRV_DEVICE_NODE*)PDUMP_MAGIC_COOKIE,
+	                               PDUMP_FLAGS_CONTINUOUS,
+	                               "Driver Product Version: %s - %s (%s)",
+	                               PVRVERSION_STRING, PVR_BUILD_DIR, PVR_BUILD_TYPE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCommentWithFlags", errRetState);
+
+	eError = PDumpCommentWithFlags((PVRSRV_DEVICE_NODE*)PDUMP_MAGIC_COOKIE,
+	                               PDUMP_FLAGS_CONTINUOUS,
+	                               "Start of Init Phase");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCommentWithFlags", errRetState);
+
+	eError = PDumpParameterChannelZeroedPageBlock((PVRSRV_DEVICE_NODE*)PDUMP_MAGIC_COOKIE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpParameterChannelZeroedPageBlock", errRetState);
+
+	PDUMP_HEREA(2012);
+ret:
+	return eError;
+
+
+errRetState:
+	PDumpModuleTransitionState(PDUMP_SM_UNINITIALISED);
+	PDumpDeInitStreams(&g_PDumpParameters.sCh, &g_PDumpScript.sCh);
+errRetCtrl:
+	PDumpCtrlDeInit();
+errRetLock:
+	OSLockDestroy(g_hPDumpWriteLock);
+	PDUMP_HEREA(2013);
+errRet:
+	goto ret;
+}
+void PDumpDeInitCommon(void)
+{
+	PDUMP_HERE_VAR;
+
+	PDUMP_HEREA(2020);
+
+	/* Suspend PDump as we want PDumpWriteAllowed to deliberately fail during PDump deinit */
+	PDumpModuleTransitionState(PDUMP_SM_DEINITIALISED);
+
+	/*Call environment specific PDump Deinitialisation */
+	PDumpDeInitStreams(&g_PDumpParameters.sCh, &g_PDumpScript.sCh);
+
+	/* DeInit the PDUMP_CTRL_STATE data */
+	PDumpCtrlDeInit();
+
+	/* take down the global PDump lock */
+	OSLockDestroy(g_hPDumpWriteLock);
+}
+
+void PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32PDumpBoundDevice = PVRSRVGetPVRSRVData()->ui32PDumpBoundDevice;
+
+	/* Stop the init phase for the PDump-bound device only */
+	if (psDeviceNode->sDevId.ui32InternalID == ui32PDumpBoundDevice)
+	{
+		/* output this comment to indicate init phase ending OSs */
+		PDUMPCOMMENT(psDeviceNode, "Stop Init Phase");
+
+		PDumpCtrlLockAcquire();
+		PDumpCtrlSetInitPhaseComplete(IMG_TRUE);
+		PDumpCtrlLockRelease();
+	}
+}
+
+PVRSRV_ERROR PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame)
+{
+	PDumpCtrlLockAcquire();
+	*pbIsLastCaptureFrame = PDumpCtrlIsLastCaptureFrame();
+	PDumpCtrlLockRelease();
+
+	return PVRSRV_OK;
+}
+
+
+
+typedef struct _PDUMP_Transition_DATA_
+{
+	PFN_PDUMP_TRANSITION        pfnCallback;
+	void                        *hPrivData;
+	void                        *pvDevice;
+	PDUMP_CONNECTION_DATA       *psPDumpConnectionData;
+	DLLIST_NODE                 sNode;
+} PDUMP_Transition_DATA;
+
+PVRSRV_ERROR PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+											  PFN_PDUMP_TRANSITION pfnCallback,
+											  void *hPrivData,
+											  void *pvDevice,
+											  void **ppvHandle)
+{
+	PDUMP_Transition_DATA *psData;
+	PVRSRV_ERROR eError;
+
+	psData = OSAllocMem(sizeof(*psData));
+	PVR_GOTO_IF_NOMEM(psData, eError, fail_alloc);
+
+	/* Setup the callback and add it to the list for this process */
+	psData->pfnCallback = pfnCallback;
+	psData->hPrivData = hPrivData;
+	psData->pvDevice = pvDevice;
+
+	OSLockAcquire(psPDumpConnectionData->hLock);
+	dllist_add_to_head(&psPDumpConnectionData->sListHead, &psData->sNode);
+	OSLockRelease(psPDumpConnectionData->hLock);
+
+	/* Take a reference on the connection so it doesn't get freed too early */
+	psData->psPDumpConnectionData =_PDumpConnectionAcquire(psPDumpConnectionData);
+	*ppvHandle = psData;
+
+	return PVRSRV_OK;
+
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void PDumpUnregisterTransitionCallback(void *pvHandle)
+{
+	PDUMP_Transition_DATA *psData = pvHandle;
+
+	OSLockAcquire(psData->psPDumpConnectionData->hLock);
+	dllist_remove_node(&psData->sNode);
+	OSLockRelease(psData->psPDumpConnectionData->hLock);
+	_PDumpConnectionRelease(psData->psPDumpConnectionData);
+	OSFreeMem(psData);
+}
+
+typedef struct _PDUMP_Transition_DATA_FENCE_SYNC_
+{
+	PFN_PDUMP_TRANSITION_FENCE_SYNC         pfnCallback;
+	void                                    *hPrivData;
+} PDUMP_Transition_DATA_FENCE_SYNC;
+
+PVRSRV_ERROR PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+							  PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback, void **ppvHandle)
+{
+	PDUMP_Transition_DATA_FENCE_SYNC *psData;
+	PVRSRV_ERROR eError;
+
+	psData = OSAllocMem(sizeof(*psData));
+	PVR_GOTO_IF_NOMEM(psData, eError, fail_alloc_exit);
+
+	/* Setup the callback and add it to the list for this process */
+	psData->pfnCallback = pfnCallback;
+	psData->hPrivData = hPrivData;
+
+	*ppvHandle = psData;
+	return PVRSRV_OK;
+
+fail_alloc_exit:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle)
+{
+	PDUMP_Transition_DATA_FENCE_SYNC *psData = pvHandle;
+
+	OSFreeMem(psData);
+}
+
+static PVRSRV_ERROR _PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                 PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+	                                 PDUMP_TRANSITION_EVENT eEvent,
+	                                 IMG_UINT32 ui32PDumpFlags)
+{
+	DLLIST_NODE *psNode, *psNext;
+	PVRSRV_ERROR eError;
+
+	/* Only call the callbacks if we've really got new event */
+	if ((eEvent != psPDumpConnectionData->eLastEvent) && (eEvent != PDUMP_TRANSITION_EVENT_NONE))
+	{
+		OSLockAcquire(psPDumpConnectionData->hLock);
+
+		dllist_foreach_node(&psPDumpConnectionData->sListHead, psNode, psNext)
+		{
+			PDUMP_Transition_DATA *psData =
+				IMG_CONTAINER_OF(psNode, PDUMP_Transition_DATA, sNode);
+
+			eError = psData->pfnCallback(psData->hPrivData, psData->pvDevice, eEvent, ui32PDumpFlags);
+
+			if (eError != PVRSRV_OK)
+			{
+				OSLockRelease(psPDumpConnectionData->hLock);
+				psPDumpConnectionData->eFailedEvent = eEvent; /* Save failed event to retry */
+				return eError;
+			}
+		}
+		OSLockRelease(psPDumpConnectionData->hLock);
+
+		/* PDump sync blocks:
+		 *
+		 * Client sync prims are managed in blocks.
+		 *
+		 * sync-blocks gets re-dumped each time we enter into capture range or
+		 * enter into new PDump block. Ensure that live-FW thread and app-thread
+		 * are synchronised before this.
+		 *
+		 * At playback time, script-thread and sim-FW threads needs to be
+		 * synchronised before re-loading sync-blocks.
+		 * */
+		psPDumpConnectionData->pfnPDumpSyncBlocks(psDeviceNode, psPDumpConnectionData->hSyncPrivData, eEvent);
+
+		if (psDeviceNode->hTransition)
+		{
+			PDUMP_Transition_DATA_FENCE_SYNC *psData = (PDUMP_Transition_DATA_FENCE_SYNC*)psDeviceNode->hTransition;
+			psData->pfnCallback(psData->hPrivData, eEvent);
+		}
+
+		psPDumpConnectionData->eLastEvent = eEvent;
+		psPDumpConnectionData->eFailedEvent = PDUMP_TRANSITION_EVENT_NONE; /* Clear failed event on success */
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _PDumpBlockTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                          PDUMP_TRANSITION_EVENT eEvent,
+                                          IMG_UINT32 ui32PDumpFlags)
+{
+
+	/* Need to follow following sequence for Block transition:
+	 *
+	 * (1) _PDumpTransition with BLOCK_FINISHED event for current block
+	 * (2) Split MAIN and Block script files
+	 * (3) _PDumpTransition with BLOCK_STARTED event for new block
+	 *
+	 * */
+
+	PVRSRV_ERROR        eError;
+	IMG_UINT32          ui32CurrentBlock;
+	IMG_UINT32          ui32Flags = (PDUMP_FLAGS_BLKDATA | PDUMP_FLAGS_CONTINUOUS); /* Internal Block mode specific PDump flags */
+
+	PDumpGetCurrentBlockKM(&ui32CurrentBlock);
+
+	if (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_FINISHED)
+	{
+		/* (1) Current block has finished */
+		eError = _PDumpTransition(psDeviceNode,
+			                      psPDumpConnectionData,
+			                      PDUMP_TRANSITION_EVENT_BLOCK_FINISHED,
+			                      ui32PDumpFlags);
+		PVR_RETURN_IF_ERROR(eError);
+
+		(void) PDumpCommentWithFlags(psDeviceNode, ui32Flags,
+		                             "}PDUMP_BLOCK_END_0x%08X",
+		                             ui32CurrentBlock - 1); /* Add pdump-block end marker */
+
+		/* (2) Split MAIN and BLOCK script out files on current pdump-block end */
+		ui32Flags |= PDUMP_FLAGS_FORCESPLIT;
+
+		(void) PDumpCommentWithFlags(psDeviceNode, ui32Flags,
+		                             "PDUMP_BLOCK_START_0x%08X{",
+		                             ui32CurrentBlock); /* Add pdump-block start marker */
+	}
+
+	/* (3) New block has started */
+	return _PDumpTransition(psDeviceNode,
+		                    psPDumpConnectionData,
+		                    PDUMP_TRANSITION_EVENT_BLOCK_STARTED,
+		                    ui32PDumpFlags);
+}
+
+
+PVRSRV_ERROR PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                             PDUMP_TRANSITION_EVENT eEvent,
+                             IMG_UINT32 ui32PDumpFlags)
+{
+	if ((eEvent == PDUMP_TRANSITION_EVENT_BLOCK_FINISHED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		/* Block mode transition events */
+		PVR_ASSERT(PDumpCtrlCapModIsBlocked());
+		return _PDumpBlockTransition(psDeviceNode, psPDumpConnectionData, eEvent, ui32PDumpFlags);
+	}
+	else
+	{
+		/* Non-block mode transition events */
+		return _PDumpTransition(psDeviceNode, psPDumpConnectionData, eEvent, ui32PDumpFlags);
+	}
+}
+
+static PVRSRV_ERROR PDumpIsCaptureFrame(IMG_BOOL *bInCaptureRange)
+{
+	IMG_UINT64 ui64State = 0;
+	PVRSRV_ERROR eError;
+
+	eError = PDumpCtrlGetState(&ui64State);
+
+	*bInCaptureRange = (ui64State & PDUMP_STATE_CAPTURE_FRAME) ? IMG_TRUE : IMG_FALSE;
+
+	return eError;
+}
+
+PVRSRV_ERROR PDumpGetStateKM(IMG_UINT64 *ui64State)
+{
+	PVRSRV_ERROR eError;
+
+	PDumpCtrlLockAcquire();
+	eError = PDumpCtrlGetState(ui64State);
+	PDumpCtrlLockRelease();
+
+	return eError;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpUpdateBlockCtrlStatus
+ * Inputs         : ui32Frame - frame number
+ * Outputs        : None
+ * Returns        : IMG_TRUE if Block transition is required, else IMG_FALSE
+ * Description    : Updates Block Ctrl status and checks if block transition
+ *                  is required or not
+******************************************************************************/
+static INLINE IMG_BOOL PDumpUpdateBlockCtrlStatus(IMG_UINT32 ui32Frame)
+{
+	IMG_BOOL bForceBlockTransition;
+
+	/* Default length of first block will be PDUMP_BLOCKLEN_MIN.
+	 * User can force it to be same as block length provided (i.e. ui32BlockLength)
+	 * through pdump client.
+	 *
+	 * Here is how blocks will be created.
+	 *
+	 * Assume,
+	 * ui32BlockLength = 20
+	 * PDUMP_BLOCKLEN_MIN = 10
+	 *
+	 * Then different pdump blocks will have following number of frames in it:
+	 *
+	 * if(!PDumpCtrlMinimalFirstBlock())
+	 * {
+	 *		//pdump -b<block len>
+	 *		block 0 -> 00...09        -->minimal first block
+	 *		block 1 -> 10...29
+	 *		block 2 -> 30...49
+	 *		block 3 -> 50...69
+	 *		...
+	 * }
+	 * else
+	 * {
+	 *		//pdump -bf<block len>
+	 *		block 0 -> 00...19
+	 *		block 1 -> 20...39
+	 *		block 2 -> 40...59
+	 *		block 3 -> 60...79
+	 *		...
+	 * }
+	 *
+	 * */
+
+	if (PDumpCtrlMinimalFirstBlock())
+	{
+		bForceBlockTransition = ((ui32Frame >= PDUMP_BLOCKLEN_MIN) && !((ui32Frame - PDUMP_BLOCKLEN_MIN) % g_PDumpCtrl.sBlockCtrl.ui32BlockLength)) || (ui32Frame == 0);
+	}
+	else
+	{
+		bForceBlockTransition = !(ui32Frame % g_PDumpCtrl.sBlockCtrl.ui32BlockLength);
+	}
+
+	if (bForceBlockTransition) /* Entering in new pdump-block */
+	{
+		/* Update block number
+		 *
+		 * Logic below is to maintain block number and frame number mappings
+		 * in case of some applications where setFrame(0) gets called twice
+		 * at the start.
+		 * */
+		PDumpCtrlLockAcquire();
+		PDumpCtrlSetBlock((ui32Frame == 0)? 0 : (PDumpCtrlGetBlock() + 1));
+		PDumpCtrlLockRelease();
+
+		if (ui32Frame > 0) /* Do not do transition on first frame itself */
+		{
+			return IMG_TRUE; /* Transition */
+		}
+	}
+	return IMG_FALSE; /* No transition */
+}
+
+PVRSRV_ERROR PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* If call is not for the pdump-bound device, return immediately
+	 * taking no action.
+	 */
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		return PVRSRV_OK;
+	}
+
+	if (!PDumpCtrlCapModIsBlocked())
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: This call is valid only in Block mode of PDump i.e. pdump -b<block-len>", __func__));
+		return PVRSRV_ERROR_PDUMP_NOT_ALLOWED;
+	}
+
+	(void) PDumpCommentWithFlags(psDeviceNode,
+	                             PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_BLKDATA,
+	                             "PDdump forced STOP capture request received at frame %u",
+	                             g_PDumpCtrl.ui32CurrentFrame);
+
+	PDumpCtrlLockAcquire();
+	eError = PDumpCtrlForcedStop();
+	PDumpCtrlLockRelease();
+
+	return eError;
+}
+
+static PVRSRV_ERROR _PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32Frame)
+{
+	PDUMP_CONNECTION_DATA *psPDumpConnectionData = psConnection->psPDumpConnectionData;
+	PDUMP_TRANSITION_EVENT eTransitionEvent = PDUMP_TRANSITION_EVENT_NONE;
+	IMG_BOOL bWasInCaptureRange = IMG_FALSE;
+	IMG_BOOL bIsInCaptureRange = IMG_FALSE;
+	PVRSRV_ERROR eError;
+
+	/*
+		Note:
+		As we can't test to see if the new frame will be in capture range
+		before we set the frame number and we don't want to roll back the
+		frame number if we fail then we have to save the "transient" data
+		which decides if we're entering or exiting capture range along
+		with a failure boolean so we know what's required on a retry
+	*/
+	if (psPDumpConnectionData->ui32LastSetFrameNumber != ui32Frame)
+	{
+		(void) PDumpCommentWithFlags(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                             "Set pdump frame %u", ui32Frame);
+
+		/*
+			The boolean values below decide if the PDump transition
+			should trigger because of the current context setting the
+			frame number, hence the functions below should execute
+			atomically and do not give a chance to some other context
+			to transition
+		*/
+		PDumpCtrlLockAcquire();
+
+		PDumpIsCaptureFrame(&bWasInCaptureRange);
+		PDumpCtrlSetCurrentFrame(ui32Frame);
+		PDumpIsCaptureFrame(&bIsInCaptureRange);
+
+		PDumpCtrlLockRelease();
+
+		psPDumpConnectionData->ui32LastSetFrameNumber = ui32Frame;
+
+		/* Check for any transition event only if client is connected */
+		if (PDumpIsClientConnected())
+		{
+			if (!bWasInCaptureRange && bIsInCaptureRange)
+			{
+				eTransitionEvent = PDUMP_TRANSITION_EVENT_RANGE_ENTERED;
+			}
+			else if (bWasInCaptureRange && !bIsInCaptureRange)
+			{
+				eTransitionEvent = PDUMP_TRANSITION_EVENT_RANGE_EXITED;
+			}
+
+			if (PDumpCtrlCapModIsBlocked())
+			{
+				/* Update block ctrl status and check for block transition */
+				if (PDumpUpdateBlockCtrlStatus(ui32Frame))
+				{
+					PVR_ASSERT(eTransitionEvent == PDUMP_TRANSITION_EVENT_NONE); /* Something went wrong, can't handle two events at same time */
+					eTransitionEvent = PDUMP_TRANSITION_EVENT_BLOCK_FINISHED;
+				}
+			}
+		}
+	}
+	else if (psPDumpConnectionData->eFailedEvent != PDUMP_TRANSITION_EVENT_NONE)
+	{
+		/* Load the Transition data so we can try again */
+		eTransitionEvent = psPDumpConnectionData->eFailedEvent;
+	}
+	else
+	{
+		/* New frame is the same as the last frame set and the last
+		 * transition succeeded, no need to perform another transition.
+		 */
+		return PVRSRV_OK;
+	}
+
+	if (eTransitionEvent != PDUMP_TRANSITION_EVENT_NONE)
+	{
+		DEBUG_OUTFILES_COMMENT(psDeviceNode, "PDump transition event(%u)-begin frame %u (post)", eTransitionEvent, ui32Frame);
+		eError = PDumpTransition(psDeviceNode, psPDumpConnectionData, eTransitionEvent, PDUMP_FLAGS_NONE);
+		DEBUG_OUTFILES_COMMENT(psDeviceNode, "PDump transition event(%u)-complete frame %u (post)", eTransitionEvent, ui32Frame);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32Frame)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* If call is not for the pdump-bound device, return immediately
+	 * taking no action.
+	 */
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(PDUMP_TRACE_STATE)
+	PVR_DPF((PVR_DBG_WARNING, "PDumpSetFrameKM: ui32Frame( %d )", ui32Frame));
+#endif
+
+	DEBUG_OUTFILES_COMMENT(psDeviceNode, "(pre) Set pdump frame %u", ui32Frame);
+
+	eError = _PDumpSetFrameKM(psConnection, psDeviceNode, ui32Frame);
+	if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+	{
+		PVR_LOG_ERROR(eError, "_PDumpSetFrameKM");
+	}
+
+	DEBUG_OUTFILES_COMMENT(psDeviceNode, "(post) Set pdump frame %u", ui32Frame);
+
+	return eError;
+}
+
+PVRSRV_ERROR PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                             IMG_UINT32* pui32Frame)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/*
+		It may be safe to avoid acquiring this lock here as all the other calls
+		which read/modify current frame will wait on the PDump Control bridge
+		lock first. Also, in no way as of now, does the PDumping app modify the
+		current frame through a call which acquires the global bridge lock.
+		Still, as a legacy we acquire and then read.
+	*/
+	PDumpCtrlLockAcquire();
+
+	*pui32Frame = PDumpCtrlGetCurrentFrame();
+
+	PDumpCtrlLockRelease();
+	return eError;
+}
+
+PVRSRV_ERROR PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_UINT32 ui32Mode,
+                                            IMG_UINT32 ui32Start,
+                                            IMG_UINT32 ui32End,
+                                            IMG_UINT32 ui32Interval,
+                                            IMG_UINT32 ui32MaxParamFileSize,
+                                            IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* NB. We choose not to check that the device is the pdump-bound
+	 * device here, as this particular bridge call is made only from the pdump
+	 * tool itself (which may only connect to the bound device).
+	 */
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	eError = PDumpReady();
+	PVR_LOG_RETURN_IF_ERROR(eError, "PDumpReady");
+
+	/* Validate parameters */
+	if ((ui32End < ui32Start) || (ui32Mode > PDUMP_CAPMODE_MAX))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (ui32Mode == PDUMP_CAPMODE_BLOCKED)
+	{
+		if ((ui32Interval < PDUMP_BLOCKLEN_MIN) || (ui32Interval > PDUMP_BLOCKLEN_MAX))
+		{
+			/* Force client to set ui32Interval (i.e. block length) in valid range */
+			eError = PVRSRV_ERROR_PDUMP_INVALID_BLOCKLEN;
+		}
+
+		if (ui32End != PDUMP_FRAME_MAX)
+		{
+			/* Force client to set ui32End to PDUMP_FRAME_MAX */
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	else if ((ui32Mode != PDUMP_CAPMODE_UNSET) && (ui32Interval < 1))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "PDumpSetDefaultCaptureParamsKM");
+
+	/*
+	   Acquire PDUMP_CTRL_STATE struct lock before modifications as a
+	   PDumping app may be reading the state data for some checks
+	*/
+	PDumpCtrlLockAcquire();
+	PDumpCtrlSetDefaultCaptureParams(ui32Mode, ui32Start, ui32End, ui32Interval, ui32AutoTermTimeout);
+	PDumpSetAutoTerminate(ui32AutoTermTimeout);
+	PDumpCtrlLockRelease();
+
+	if (ui32MaxParamFileSize == 0)
+	{
+		g_PDumpParameters.ui32MaxFileSize = PRM_FILE_SIZE_MAX;
+	}
+	else
+	{
+		g_PDumpParameters.ui32MaxFileSize = ui32MaxParamFileSize;
+	}
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpReg32
+ * Inputs         : pszPDumpDevName, Register offset, and value to write
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+******************************************************************************/
+PVRSRV_ERROR PDumpReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_CHAR	*pszPDumpRegName,
+						IMG_UINT32	ui32Reg,
+						IMG_UINT32	ui32Data,
+						IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X 0x%08X", pszPDumpRegName, ui32Reg, ui32Data);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpReg64
+ * Inputs         : pszPDumpDevName, Register offset, and value to write
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+******************************************************************************/
+PVRSRV_ERROR PDumpReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_CHAR	*pszPDumpRegName,
+						IMG_UINT32	ui32Reg,
+						IMG_UINT64	ui64Data,
+						IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Data >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Data);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X 0x%08X",
+					pszPDumpRegName, ui32Reg, ui32LowerValue);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X 0x%08X",
+					pszPDumpRegName, ui32Reg + 4, ui32UpperValue);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:0x%08X 0x%010" IMG_UINT64_FMTSPECX, pszPDumpRegName, ui32Reg, ui64Data);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToReg64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+ *                  from a register label
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32RegDst,
+                                  IMG_UINT32 ui32RegSrc,
+                                  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:0x%08X :%s:0x%08X", pszPDumpRegName, ui32RegDst, pszPDumpRegName, ui32RegSrc);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToMem32
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write
+ *                  from a register label
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToMem32(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:0x%08X",aszMemspaceName, aszSymbolicName,
+							uiPDumpSymbolicOffset, pszPDumpRegName, ui32Reg);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToMem64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write
+ *                  from a register label
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToMem64(IMG_CHAR *pszPDumpRegName,
+								  IMG_UINT32 ui32Reg,
+								  PMR *psPMR,
+								  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+								  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:0x%08X",aszMemspaceName, aszSymbolicName,
+							uiPDumpSymbolicOffset, pszPDumpRegName, ui32Reg);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpPhysHandleToInternalVar64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents an internal var
+                    write using a PDump pages handle
+******************************************************************************/
+PVRSRV_ERROR PDumpPhysHandleToInternalVar64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_CHAR *pszInternalVar,
+                                            IMG_HANDLE hPdumpPages,
+                                            IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR *pszSymbolicName;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpGetSymbolicAddr(hPdumpPages,
+	                            &pszSymbolicName);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+	                        "WRW %s %s:0x%llX",
+	                        pszInternalVar, pszSymbolicName, 0llu);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s 0x%X", pszPDumpVarName, 0);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#endif
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpMemLabelToInternalVar64
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents an internal var
+ *                  write using a memory label
+******************************************************************************/
+PVRSRV_ERROR PDumpMemLabelToInternalVar64(IMG_CHAR *pszInternalVar,
+                                          PMR *psPMR,
+                                          IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                          IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:%s:0x%"IMG_UINT64_FMTSPECX, pszInternalVar,
+							aszMemspaceName, aszSymbolicName, uiPDumpSymbolicOffset);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:%s:0x%"IMG_UINT64_FMTSPECX, pszPDumpVarName,
+							aszMemspaceName, aszSymbolicName, uiPDumpSymbolicOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "SHR %s %s 0x20", pszPDumpVarName, pszPDumpVarName);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#endif
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalVarToMemLabel
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory label
+ *                  write using an internal var
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalVarToMemLabel(PMR *psPMR,
+                                        IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMR);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMR,
+	                             uiLogicalOffset,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceName,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicName,
+	                             &uiPDumpSymbolicOffset,
+	                             &uiNextSymName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:%s:0x%"IMG_UINT64_FMTSPECX" %s",
+							aszMemspaceName, aszSymbolicName, uiPDumpSymbolicOffset, pszInternalVar);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s 0x%X", pszPDumpVarName, 0);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#endif
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteRegORValueOp
+
+ @Description
+
+ Emits the PDump commands for the logical OR operation
+ Var <- Var OR Value
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarORValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const IMG_CHAR *pszInternalVariable,
+                                    const IMG_UINT64 ui64Value,
+                                    const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Value >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Value);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			"OR %s %s 0x%X",
+#else
+			"OR %s %s 0x%"IMG_UINT64_FMTSPECX,
+#endif
+			pszInternalVariable,
+			pszInternalVariable,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			ui32LowerValue
+#else
+			ui64Value
+#endif
+			);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVariable);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"OR %s %s 0x%X",
+			pszPDumpVarName,
+			pszPDumpVarName,
+			ui32UpperValue);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+#endif
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteVarORVarOp
+
+ @Description
+
+ Emits the PDump commands for the logical OR operation
+ Var <- Var OR Var2
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarORVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const IMG_CHAR *pszInternalVar,
+                                  const IMG_CHAR *pszInternalVar2,
+                                  const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"OR %s %s %s",
+			pszInternalVar,
+			pszInternalVar,
+			pszInternalVar2);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteVarANDVarOp
+
+ @Description
+
+ Emits the PDump commands for the logical AND operation
+ Var <- Var AND Var2
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarANDVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const IMG_CHAR *pszInternalVar,
+                                   const IMG_CHAR *pszInternalVar2,
+                                   const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"AND %s %s %s",
+			pszInternalVar,
+			pszInternalVar,
+			pszInternalVar2);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpRegLabelToInternalVar
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which writes a register label into
+ *                  an internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpRegLabelToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR *pszPDumpRegName,
+                                        IMG_UINT32 ui32Reg,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:0x%08X", pszInternalVar, pszPDumpRegName, ui32Reg);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW %s :%s:0x%08X", pszPDumpVarName, pszPDumpRegName, ui32Reg + 4);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+#endif
+
+	PDUMP_UNLOCK(ui32Flags);
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalVarToReg32
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+ *                  from an internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalVarToReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X %s", pszPDumpRegName, ui32Reg, pszInternalVar);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalVarToReg64
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a register write
+ *                  from an internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalVarToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+	PDUMP_GET_SCRIPT_STRING()
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X %s", pszPDumpRegName, ui32Reg, pszInternalVar);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW :%s:0x%08X %s", pszPDumpRegName, ui32Reg + 4, pszPDumpVarName);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "WRW64 :%s:0x%08X %s", pszPDumpRegName, ui32Reg, pszInternalVar);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+
+
+/******************************************************************************
+ * Function Name  : PDumpMemLabelToMem32
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write from
+ *                  a memory label
+******************************************************************************/
+PVRSRV_ERROR PDumpMemLabelToMem32(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceNameSource[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameSource[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_CHAR aszMemspaceNameDest[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameDest[PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetSource;
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetDest;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameSource;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameDest;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMRSource);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRSource,
+	                             uiLogicalOffsetSource,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameSource,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameSource,
+	                             &uiPDumpSymbolicOffsetSource,
+	                             &uiNextSymNameSource);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRDest,
+	                             uiLogicalOffsetDest,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameDest,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameDest,
+	                             &uiPDumpSymbolicOffsetDest,
+	                             &uiNextSymNameDest);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+	                     "WRW :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:%s:0x%"IMG_UINT64_FMTSPECX,
+	                     aszMemspaceNameDest, aszSymbolicNameDest,
+	                     uiPDumpSymbolicOffsetDest, aszMemspaceNameSource,
+	                     aszSymbolicNameSource, uiPDumpSymbolicOffsetSource);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpMemLabelToMem64
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which represents a memory write from
+ *                  a memory label
+******************************************************************************/
+PVRSRV_ERROR PDumpMemLabelToMem64(PMR *psPMRSource,
+								  PMR *psPMRDest,
+								  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+								  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+								  IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	IMG_CHAR aszMemspaceNameSource[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameSource[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_CHAR aszMemspaceNameDest[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicNameDest[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetSource;
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffsetDest;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameSource;
+	IMG_DEVMEM_OFFSET_T uiNextSymNameDest;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	psDeviceNode = PMR_DeviceNode(psPMRSource);
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRSource,
+	                             uiLogicalOffsetSource,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameSource,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameSource,
+	                             &uiPDumpSymbolicOffsetSource,
+	                             &uiNextSymNameSource);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PMR_PDumpSymbolicAddr(psPMRDest,
+	                             uiLogicalOffsetDest,
+	                             PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH,
+	                             aszMemspaceNameDest,
+	                             PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH,
+	                             aszSymbolicNameDest,
+	                             &uiPDumpSymbolicOffsetDest,
+	                             &uiNextSymNameDest);
+
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+	                     "WRW64 :%s:%s:0x%"IMG_UINT64_FMTSPECX" :%s:%s:0x%"IMG_UINT64_FMTSPECX,
+	                     aszMemspaceNameDest, aszSymbolicNameDest,
+	                     uiPDumpSymbolicOffsetDest, aszMemspaceNameSource,
+	                     aszSymbolicNameSource, uiPDumpSymbolicOffsetSource);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteVarSHRValueOp
+
+ @Description
+
+ Emits the PDump commands for the logical SHR operation
+ Var <-  Var SHR Value
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarSHRValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Value >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Value);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			"SHR %s %s 0x%X",
+#else
+			"SHR %s %s 0x%"IMG_UINT64_FMTSPECX,
+#endif
+			pszInternalVariable,
+			pszInternalVariable,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			ui32LowerValue
+#else
+			ui64Value
+#endif
+			);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVariable);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"SHR %s %s 0x%X",
+			pszPDumpVarName,
+			pszPDumpVarName,
+			ui32UpperValue);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+#endif
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpWriteRegANDValueOp
+
+ @Description
+
+ Emits the PDump commands for the logical AND operation
+ Var <-  Var AND Value
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PDumpWriteVarANDValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64Value >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64Value);
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			"AND %s %s 0x%X",
+#else
+			"AND %s %s 0x%"IMG_UINT64_FMTSPECX,
+#endif
+			pszInternalVariable,
+			pszInternalVariable,
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+			ui32LowerValue
+#else
+			ui64Value
+#endif
+			);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVariable);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"AND %s %s 0x%X",
+			pszPDumpVarName,
+			pszPDumpVarName,
+			ui32UpperValue);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32PDumpFlags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+#endif
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpSAW
+ * Inputs         : pszDevSpaceName -- device space from which to output
+ *                  ui32Offset -- offset value from register base
+ *                  ui32NumSaveBytes -- number of bytes to output
+ *                  pszOutfileName -- name of file to output to
+ *                  ui32OutfileOffsetByte -- offset into output file to write
+ *                  uiPDumpFlags -- flags to pass to PDumpOSWriteScript
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Dumps the contents of a register bank into a file
+ *                  NB: ui32NumSaveBytes must be divisible by 4
+******************************************************************************/
+PVRSRV_ERROR PDumpSAW(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR           *pszDevSpaceName,
+                      IMG_UINT32         ui32HPOffsetBytes,
+                      IMG_UINT32         ui32NumSaveBytes,
+                      IMG_CHAR           *pszOutfileName,
+                      IMG_UINT32         ui32OutfileOffsetByte,
+                      PDUMP_FLAGS_T      uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PDUMP_GET_SCRIPT_STRING()
+
+	PVR_DPF((PVR_DBG_ERROR, "PDumpSAW"));
+
+	eError = PDumpSNPrintf(hScript,
+	                          ui32MaxLen,
+	                          "SAW :%s:0x%x 0x%x 0x%x %s\n",
+	                          pszDevSpaceName,
+	                          ui32HPOffsetBytes,
+	                          ui32NumSaveBytes / (IMG_UINT32)sizeof(IMG_UINT32),
+	                          ui32OutfileOffsetByte,
+	                          pszOutfileName);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDumpSAW PDumpSNPrintf failed: eError=%u", eError));
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eError;
+	}
+
+	PDUMP_LOCK(uiPDumpFlags);
+	if (! PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDumpSAW PDumpWriteScript failed!"));
+	}
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpRegPolKM
+ * Inputs         : Description of what this register read is trying to do
+ *					pszPDumpDevName
+ *					Register offset
+ *					expected value
+ *					mask for that value
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents a register read
+ *					with the expected value
+******************************************************************************/
+PVRSRV_ERROR PDumpRegPolKM(PVRSRV_DEVICE_NODE	*psDeviceNode,
+						   IMG_CHAR				*pszPDumpRegName,
+						   IMG_UINT32			ui32RegAddr,
+						   IMG_UINT32			ui32RegValue,
+						   IMG_UINT32			ui32Mask,
+						   IMG_UINT32			ui32Flags,
+						   PDUMP_POLL_OPERATOR	eOperator)
+{
+	/* Timings correct for Linux and XP */
+	/* Timings should be passed in */
+	#define POLL_DELAY			1000U
+	#define POLL_COUNT_LONG		(2000000000U / POLL_DELAY)
+	#define POLL_COUNT_SHORT	(1000000U / POLL_DELAY)
+
+	PVRSRV_ERROR eErr;
+	IMG_UINT32	ui32PollCount;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	ui32PollCount = POLL_COUNT_LONG;
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "POL :%s:0x%08X 0x%08X 0x%08X %d %u %d",
+							pszPDumpRegName, ui32RegAddr, ui32RegValue,
+							ui32Mask, eOperator, ui32PollCount, POLL_DELAY);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return PVRSRV_OK;
+}
+
+/*!
+ * \name	PDumpOSVerifyLineEnding
+ */
+static void _PDumpVerifyLineEnding(IMG_HANDLE hBuffer, IMG_UINT32 ui32BufferSizeMax)
+{
+	IMG_UINT32 ui32Count;
+	IMG_CHAR* pszBuf = hBuffer;
+
+	/* strlen */
+	ui32Count = OSStringNLength(pszBuf, ui32BufferSizeMax);
+
+	/* Put \n sequence at the end if it isn't already there */
+	if ((ui32Count >= 1) && (pszBuf[ui32Count-1] != '\n') && (ui32Count<ui32BufferSizeMax))
+	{
+		pszBuf[ui32Count] = '\n';
+		ui32Count++;
+		pszBuf[ui32Count] = '\0';
+	}
+}
+
+
+/* Never call direct, needs caller to hold OS Lock.
+ * Use PDumpCommentWithFlags() from within the server.
+ * Clients call this via the bridge and PDumpCommentKM().
+ */
+static PVRSRV_ERROR _PDumpWriteComment(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_CHAR *pszComment,
+                                       IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_DEBUG_OUTFILES)
+	IMG_CHAR pszTemp[PVRSRV_PDUMP_MAX_COMMENT_SIZE+80];
+	IMG_INT32 iCount;
+#endif
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_ASSERT(pszComment != NULL);
+
+	if (OSStringNLength(pszComment, ui32MaxLen) == 0)
+	{
+		/* PDumpOSVerifyLineEnding silently fails if pszComment is too short to
+		   actually hold the line endings that it's trying to enforce, so
+		   short circuit it and force safety */
+		pszComment = "\n";
+	}
+	else
+	{
+		/* Put line ending sequence at the end if it isn't already there */
+		_PDumpVerifyLineEnding(pszComment, ui32MaxLen);
+	}
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+	/* Prefix comment with PID and line number */
+	iCount = OSSNPrintf(pszTemp, PVRSRV_PDUMP_MAX_COMMENT_SIZE+80, "%u %u:%lu %s: %s",
+		OSAtomicRead(&g_sEveryLineCounter),
+		OSGetCurrentClientProcessIDKM(),
+		(unsigned long)OSGetCurrentClientThreadIDKM(),
+		OSGetCurrentClientProcessNameKM(),
+		pszComment);
+	if ((iCount < 0) || (iCount >= (PVRSRV_PDUMP_MAX_COMMENT_SIZE+80)))
+	{
+		eErr = PVRSRV_ERROR_PDUMP_BUF_OVERFLOW;
+	}
+
+	/* Append the comment to the script stream */
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "-- %s",
+		pszTemp);
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "-- %s",
+		pszComment);
+#endif
+	if ((eErr != PVRSRV_OK) &&
+		(eErr != PVRSRV_ERROR_PDUMP_BUF_OVERFLOW))
+	{
+		PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpSNPrintf", ErrUnlock);
+	}
+
+	if (!PDumpWriteScript(psDeviceNode, hScript, ui32Flags))
+	{
+		if (PDUMP_IS_CONTINUOUS(ui32Flags))
+		{
+			eErr = PVRSRV_ERROR_PDUMP_BUFFER_FULL;
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpWriteScript", ErrUnlock);
+		}
+		else
+		{
+			eErr = PVRSRV_ERROR_CMD_NOT_PROCESSED;
+			PVR_LOG_GOTO_IF_ERROR(eErr, "PDumpWriteScript", ErrUnlock);
+		}
+	}
+
+ErrUnlock:
+	PDUMP_RELEASE_SCRIPT_STRING()
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentKM
+ * Inputs         : ui32CommentSize, pszComment, ui32Flags
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Dumps a pre-formatted comment, primarily called from the
+ *                : bridge.
+******************************************************************************/
+PVRSRV_ERROR PDumpCommentKM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT32 ui32CommentSize,
+                            IMG_CHAR *pszComment,
+                            IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32CommentSize); /* Generated bridge code appends null char to pszComment. */
+
+	PDUMP_LOCK(ui32Flags);
+
+	eErr = _PDumpWriteComment(psDeviceNode, pszComment, ui32Flags);
+
+	PDUMP_UNLOCK(ui32Flags);
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlagsNoLockVA
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : args      - pre-started va_list args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comment, caller need to acquire pdump lock
+ *                  explicitly before calling this function
+******************************************************************************/
+static PVRSRV_ERROR PDumpCommentWithFlagsNoLockVA(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32 ui32Flags,
+                                           const IMG_CHAR * pszFormat, va_list args)
+{
+	IMG_INT32 iCount;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	PDUMP_GET_MSG_STRING();
+
+	/* Construct the string */
+	iCount = OSVSNPrintf(pszMsg, ui32MaxLen, pszFormat, args);
+	PVR_LOG_GOTO_IF_FALSE(((iCount != -1) && (iCount < ui32MaxLen)), "OSVSNPrintf", exit);
+
+	eErr = _PDumpWriteComment(psDeviceNode, pszMsg, ui32Flags);
+
+exit:
+	PDUMP_RELEASE_MSG_STRING();
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlagsNoLock
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : ... - args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comment, caller need to acquire pdump lock
+ *                  explicitly before calling this function.
+******************************************************************************/
+__printf(3, 4)
+static PVRSRV_ERROR PDumpCommentWithFlagsNoLock(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_UINT32 ui32Flags,
+                                         IMG_CHAR *pszFormat, ...)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	va_list args;
+
+	va_start(args, pszFormat);
+	PDumpCommentWithFlagsNoLockVA(psDeviceNode, ui32Flags, pszFormat, args);
+	va_end(args);
+
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlags
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : ... - args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comments
+******************************************************************************/
+PVRSRV_ERROR PDumpCommentWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_UINT32 ui32Flags,
+                                   IMG_CHAR * pszFormat, ...)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	va_list args;
+
+	va_start(args, pszFormat);
+	PDumpCommentWithFlagsVA(psDeviceNode, ui32Flags, pszFormat, args);
+	va_end(args);
+
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCommentWithFlagsVA
+ * Inputs         : ui32Flags - PDump flags
+ *				  : pszFormat - format string for comment
+ *				  : args      - pre-started va_list args for format string
+ * Outputs        : None
+ * Returns        : None
+ * Description    : PDumps a comments
+******************************************************************************/
+PVRSRV_ERROR PDumpCommentWithFlagsVA(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32Flags,
+                                     const IMG_CHAR * pszFormat, va_list args)
+{
+	IMG_INT32 iCount;
+	PVRSRV_ERROR eErr = PVRSRV_ERROR_INVALID_PARAMS;
+	PDUMP_GET_MSG_STRING();
+
+	/* Construct the string */
+	iCount = OSVSNPrintf(pszMsg, ui32MaxLen, pszFormat, args);
+	PVR_LOG_GOTO_IF_FALSE(((iCount != -1) && (iCount < ui32MaxLen)), "OSVSNPrintf", exit);
+
+	PDUMP_LOCK(ui32Flags);
+	eErr = _PDumpWriteComment(psDeviceNode, pszMsg, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+exit:
+	PDUMP_RELEASE_MSG_STRING();
+	return eErr;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpCOMCommand
+ * Inputs         : ui32PDumpFlags - PDump flags
+ *			: pszPdumpStr - string for COM command
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : PDumps a COM command
+******************************************************************************/
+PVRSRV_ERROR PDumpCOMCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32PDumpFlags,
+                             const IMG_CHAR * pszPdumpStr)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "COM %s\n", pszPdumpStr);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ * Function Name  : PDumpPanic
+ * Inputs         : ui32PanicNo - Unique number for panic condition
+ *				  : pszPanicMsg - Panic reason message limited to ~90 chars
+ *				  : pszPPFunc   - Function name string where panic occurred
+ *				  : ui32PPline  - Source line number where panic occurred
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : PDumps a panic assertion. Used when the host driver
+ *                : detects a condition that will lead to an invalid PDump
+ *                : script that cannot be played back off-line.
+ */ /*************************************************************************/
+PVRSRV_ERROR PDumpPanic(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32      ui32PanicNo,
+						IMG_CHAR*       pszPanicMsg,
+						const IMG_CHAR* pszPPFunc,
+						IMG_UINT32      ui32PPline)
+{
+	PVRSRV_ERROR   eError = PVRSRV_OK;
+	PDUMP_FLAGS_T  uiPDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+	PDUMP_GET_SCRIPT_STRING();
+
+	/* Log the panic condition to the live kern.log in both REL and DEB mode
+	 * to aid user PDump troubleshooting. */
+	PVR_LOG(("PDUMP PANIC %08x: %s", ui32PanicNo, pszPanicMsg));
+	PVR_DPF((PVR_DBG_MESSAGE, "PDUMP PANIC start %s:%d", pszPPFunc, ui32PPline));
+
+	/* Check the supplied panic reason string is within length limits */
+	PVR_ASSERT(OSStringLength(pszPanicMsg)+sizeof("PANIC   ") < PVRSRV_PDUMP_MAX_COMMENT_SIZE-1);
+
+	/* Obtain lock to keep the multi-line
+	 * panic statement together in a single atomic write */
+	PDUMP_BLKSTART(uiPDumpFlags);
+
+
+	/* Write -- Panic start (Function:line) */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "-- Panic start (%s:%d)", pszPPFunc, ui32PPline);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpSNPrintf", e1);
+	(void)PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+	/* Write COM messages */
+	eError = PDumpCOMCommand(psDeviceNode, uiPDumpFlags,
+				  "**** Script invalid and not compatible with off-line playback. ****");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCOMCommand", e1);
+
+	eError = PDumpCOMCommand(psDeviceNode, uiPDumpFlags,
+				  "**** Check test parameters and driver configuration, stop imminent. ****");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpCOMCommand", e1);
+
+	/* Write PANIC no msg command */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "PANIC %08x %s", ui32PanicNo, pszPanicMsg);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpSNPrintf", e1);
+	(void)PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+	/* Write -- Panic end */
+	eError = PDumpSNPrintf(hScript, ui32MaxLen, "-- Panic end");
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpSNPrintf", e1);
+	(void)PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+e1:
+	PDUMP_BLKEND(uiPDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ * Function Name  : PDumpCaptureError
+ * Inputs         : ui32ErrorNo - Unique number for panic condition
+ *                : pszErrorMsg - Panic reason message limited to ~90 chars
+ *                : pszPPFunc   - Function name string where panic occurred
+ *                : ui32PPline  - Source line number where panic occurred
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : PDumps an error string to the script file to interrupt
+ *                : play back to inform user of a fatal issue that occurred
+ *                : during PDump capture.
+ */ /*************************************************************************/
+PVRSRV_ERROR PDumpCaptureError(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               PVRSRV_ERROR       ui32ErrorNo,
+                               IMG_CHAR*          pszErrorMsg,
+                               const IMG_CHAR     *pszPPFunc,
+                               IMG_UINT32         ui32PPline)
+{
+	IMG_CHAR*       pszFormatStr = "DRIVER_ERROR: %3d: %s";
+	PDUMP_FLAGS_T   uiPDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	/* Need to return an error using this macro */
+	PDUMP_GET_SCRIPT_STRING();
+
+	/* Check the supplied panic reason string is within length limits */
+	PVR_ASSERT(OSStringLength(pszErrorMsg)+sizeof(pszFormatStr) < PVRSRV_PDUMP_MAX_COMMENT_SIZE-1);
+
+	/* Write driver error message to the script file */
+	(void) PDumpSNPrintf(hScript, ui32MaxLen, pszFormatStr, ui32ErrorNo, pszErrorMsg);
+
+	/* Obtain lock to keep the multi-line
+	 * panic statement together in a single atomic write */
+	PDUMP_LOCK(uiPDumpFlags);
+	(void) PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	PDUMP_UNLOCK(uiPDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpImageDescriptor
+
+ @Description
+
+ Dumps an OutputImage command and its associated header info.
+
+ @Input    psDeviceNode			: device
+ @Input    ui32MMUContextID		: MMU context
+ @Input    pszSABFileName		: filename string
+
+ @Return   PVRSRV_ERROR			:
+
+******************************************************************************/
+PVRSRV_ERROR PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32MMUContextID,
+									IMG_CHAR *pszSABFileName,
+									IMG_DEV_VIRTADDR sData,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_DEV_VIRTADDR sHeader,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32PDumpFlags)
+{
+#if !defined(SUPPORT_RGX)
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#else
+	PVRSRV_ERROR  eErr = PVRSRV_OK;
+	IMG_CHAR      *pszPDumpDevName = psDeviceNode->sDevId.pszPDumpDevName;
+	IMG_BYTE      abyPDumpDesc[IMAGE_HEADER_SIZE];
+	IMG_UINT32    ui32ParamOutPos, ui32SABOffset = 0;
+	IMG_BOOL      bRawImageData = IMG_FALSE;
+
+	PDUMP_GET_SCRIPT_AND_FILE_STRING();
+
+	if (pszSABFileName == NULL)
+	{
+		eErr = PVRSRV_ERROR_INVALID_PARAMS;
+		goto error_release_script;
+	}
+
+	/* Writing image descriptor to persistent buffer is not permitted */
+	if (ui32PDumpFlags & PDUMP_FLAGS_PERSISTENT)
+	{
+		goto error_release_script;
+	}
+
+	/* Prepare OutputImage descriptor header */
+	eErr = RGXPDumpPrepareOutputImageDescriptorHdr(psDeviceNode,
+									ui32HeaderSize,
+									ui32DataSize,
+									ui32LogicalWidth,
+									ui32LogicalHeight,
+									ui32PhysicalWidth,
+									ui32PhysicalHeight,
+									ePixFmt,
+									eMemLayout,
+									eFBCompression,
+									paui32FBCClearColour,
+									eFBCSwizzle,
+									&(abyPDumpDesc[0]));
+	PVR_LOG_GOTO_IF_ERROR(eErr, "RGXPDumpPrepareOutputImageDescriptorHdr", error_release_script);
+
+	PDUMP_LOCK(ui32PDumpFlags);
+
+	PDumpCommentWithFlagsNoLock(psDeviceNode, ui32PDumpFlags, "Dump Image descriptor");
+
+	bRawImageData =
+		 (ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_YUV8
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV12
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV8
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV8
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV10
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV10
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_VY0UY1_8888
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_UY0VY1_8888
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_Y0UY1V_8888
+	   || ePixFmt == PVRSRV_PDUMP_PIXEL_FORMAT_Y0VY1U_8888);
+
+
+	if (bRawImageData)
+	{
+		IMG_UINT32 ui32ElementType;
+		IMG_UINT32 ui32ElementCount;
+
+		PDumpCommentWithFlagsNoLock(psDeviceNode, ui32PDumpFlags,
+		                            "YUV data. Switching from OutputImage to SAB. Width=0x%08X Height=0x%08X",
+		                            ui32LogicalWidth, ui32LogicalHeight);
+
+		PDUMP_UNLOCK(ui32PDumpFlags);
+
+		PDUMP_RELEASE_SCRIPT_AND_FILE_STRING();
+
+		ui32ElementType = 0;
+		ui32ElementCount = 0;
+
+		/* Switch to CMD:OutputData with IBIN header. */
+		return PDumpDataDescriptor(psDeviceNode,
+								   ui32MMUContextID,
+								   pszSABFileName,
+								   sData,
+								   ui32DataSize,
+								   IBIN_HEADER_TYPE,
+								   ui32ElementType,
+								   ui32ElementCount,
+								   ui32PDumpFlags);
+	}
+
+	/* Write OutputImage descriptor header to parameter file */
+	eErr = PDumpWriteParameter(psDeviceNode,
+							   abyPDumpDesc,
+							   IMAGE_HEADER_SIZE,
+							   ui32PDumpFlags,
+							   &ui32ParamOutPos,
+							   pszFileName);
+	if (eErr != PVRSRV_OK)
+	{
+		if (eErr != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+		{
+			PDUMP_ERROR(psDeviceNode, eErr,
+			            "Failed to write device allocation to parameter file");
+			PVR_LOG_ERROR(eErr, "PDumpWriteParameter");
+		}
+		else
+		{
+			/*
+			 * Write to parameter file prevented under the flags and
+			 * current state of the driver so skip write to script and return.
+			 */
+			eErr = PVRSRV_OK;
+		}
+		goto error;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"MALLOC :%s:BINHEADER 0x%08X 0x%08X\n",
+							pszPDumpDevName,
+							IMAGE_HEADER_SIZE,
+							IMAGE_HEADER_SIZE);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"LDB :%s:BINHEADER:0x00 0x%08x 0x%08x %s\n",
+							pszPDumpDevName,
+							IMAGE_HEADER_SIZE,
+							ui32ParamOutPos,
+							pszFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:BINHEADER:0x00 0x%08X 0x00000000 %s.bin\n",
+							pszPDumpDevName,
+							IMAGE_HEADER_SIZE,
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	ui32SABOffset += IMAGE_HEADER_SIZE;
+
+	/*
+	 * Write out the header section if image is FB compressed
+	 */
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		eErr = PDumpSNPrintf(hScript,
+								ui32MaxLenScript,
+								"SAB :%s:v%x:0x%010"IMG_UINT64_FMTSPECX" 0x%08X 0x%08X %s.bin\n",
+								pszPDumpDevName,
+								ui32MMUContextID,
+								(IMG_UINT64)sHeader.uiAddr,
+								ui32HeaderSize,
+								ui32SABOffset,
+								pszSABFileName);
+		PVR_GOTO_IF_ERROR(eErr, error);
+		PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+		ui32SABOffset += ui32HeaderSize;
+	}
+
+	/*
+	 * Now dump out the actual data associated with the surface
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:v%x:0x%010"IMG_UINT64_FMTSPECX" 0x%08X 0x%08X %s.bin\n",
+							pszPDumpDevName,
+							ui32MMUContextID,
+							(IMG_UINT64)sData.uiAddr,
+							ui32DataSize,
+							ui32SABOffset,
+							pszSABFileName);
+
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	/*
+	 * The OutputImage command is required to trigger processing of the output
+	 * data
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"CMD:OutputImage %s.bin\n",
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"FREE :%s:BINHEADER\n",
+							pszPDumpDevName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+error:
+	PDUMP_UNLOCK(ui32PDumpFlags);
+error_release_script:
+	PDUMP_RELEASE_SCRIPT_AND_FILE_STRING()
+	return eErr;
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpDataDescriptor
+
+ @Description
+
+ Dumps an OutputData command and its associated header info.
+
+ @Input    psDeviceNode         : device
+ @Input    ui32MMUContextID     : MMU context
+ @Input    pszSABFileName       : filename string
+ @Input    sData                : GPU virtual address of data
+ @Input    ui32HeaderType       : Header type
+ @Input    ui32DataSize         : Data size
+ @Input    ui32ElementType      : Element type being dumped
+ @Input    ui32ElementCount     : Number of elements to be dumped
+ @Input    ui32PDumpFlags       : PDump flags
+
+ @Return   PVRSRV_ERROR         :
+
+******************************************************************************/
+PVRSRV_ERROR PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32MMUContextID,
+									IMG_CHAR *pszSABFileName,
+									IMG_DEV_VIRTADDR sData,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_UINT32 ui32PDumpFlags)
+{
+#if !defined(SUPPORT_RGX)
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#else
+	PVRSRV_ERROR   eErr = PVRSRV_OK;
+	IMG_CHAR       *pszPDumpDevName = psDeviceNode->sDevId.pszPDumpDevName;
+	IMG_BYTE       abyPDumpDesc[DATA_HEADER_SIZE];
+	IMG_UINT32     ui32ParamOutPos, ui32SABOffset = 0;
+	IMG_UINT32     ui32HeaderSize;
+
+	PDUMP_GET_SCRIPT_AND_FILE_STRING();
+
+	PVR_GOTO_IF_INVALID_PARAM(pszSABFileName, eErr, error_release_script);
+
+	if (ui32HeaderType == DATA_HEADER_TYPE)
+	{
+		ui32HeaderSize = DATA_HEADER_SIZE;
+	}
+	else if (ui32HeaderType == IBIN_HEADER_TYPE)
+	{
+		ui32HeaderSize = IBIN_HEADER_SIZE;
+	}
+	else
+	{
+		PVR_GOTO_WITH_ERROR(eErr, PVRSRV_ERROR_INVALID_PARAMS, error_release_script);
+	}
+
+	/* Writing data descriptor to persistent buffer is not permitted */
+	if (ui32PDumpFlags & PDUMP_FLAGS_PERSISTENT)
+	{
+		goto error_release_script;
+	}
+
+	/* Prepare OutputData descriptor header */
+	eErr = RGXPDumpPrepareOutputDataDescriptorHdr(psDeviceNode,
+									ui32HeaderType,
+									ui32DataSize,
+									ui32ElementType,
+									ui32ElementCount,
+									&(abyPDumpDesc[0]));
+	PVR_LOG_GOTO_IF_ERROR(eErr, "RGXPDumpPrepareOutputDataDescriptorHdr", error_release_script);
+
+	PDUMP_LOCK(ui32PDumpFlags);
+
+	PDumpCommentWithFlagsNoLock(psDeviceNode, ui32PDumpFlags, "Dump Data descriptor");
+
+	/* Write OutputImage command header to parameter file */
+	eErr = PDumpWriteParameter(psDeviceNode,
+							   abyPDumpDesc,
+							   ui32HeaderSize,
+							   ui32PDumpFlags,
+							   &ui32ParamOutPos,
+							   pszFileName);
+	if (eErr != PVRSRV_OK)
+	{
+		if (eErr != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+		{
+			PDUMP_ERROR(psDeviceNode, eErr,
+			            "Failed to write device allocation to parameter file");
+			PVR_LOG_ERROR(eErr, "PDumpWriteParameter");
+		}
+		else
+		{
+			/*
+			 * Write to parameter file prevented under the flags and
+			 * current state of the driver so skip write to script and return.
+			 */
+			eErr = PVRSRV_OK;
+		}
+		goto error;
+	}
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"MALLOC :%s:BINHEADER 0x%08X 0x%08X\n",
+							pszPDumpDevName,
+							ui32HeaderSize,
+							ui32HeaderSize);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"LDB :%s:BINHEADER:0x00 0x%08x 0x%08x %s\n",
+							pszPDumpDevName,
+							ui32HeaderSize,
+							ui32ParamOutPos,
+							pszFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:BINHEADER:0x00 0x%08X 0x00000000 %s.bin\n",
+							pszPDumpDevName,
+							ui32HeaderSize,
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	ui32SABOffset += ui32HeaderSize;
+
+	/*
+	 * Now dump out the actual data associated
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"SAB :%s:v%x:0x%010"IMG_UINT64_FMTSPECX" 0x%08X 0x%08X %s.bin\n",
+							pszPDumpDevName,
+							ui32MMUContextID,
+							(IMG_UINT64)sData.uiAddr,
+							ui32DataSize,
+							ui32SABOffset,
+							pszSABFileName);
+
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	/*
+	 * The OutputData command is required to trigger processing of the output
+	 * data
+	 */
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"CMD:OutputData %s.bin\n",
+							pszSABFileName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLenScript,
+							"FREE :%s:BINHEADER\n",
+							pszPDumpDevName);
+	PVR_GOTO_IF_ERROR(eErr, error);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+
+error:
+	PDUMP_UNLOCK(ui32PDumpFlags);
+error_release_script:
+	PDUMP_RELEASE_SCRIPT_AND_FILE_STRING()
+	return eErr;
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	PDumpReadRegKM
+
+ @Description
+
+ Dumps a read from a device register to a file
+
+ @Input    psConnection			: connection info
+ @Input    pszFileName
+ @Input    ui32FileOffset
+ @Input    ui32Address
+ @Input    ui32Size
+ @Input    ui32PDumpFlags
+
+ @Return   PVRSRV_ERROR			:
+
+******************************************************************************/
+PVRSRV_ERROR PDumpReadRegKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            IMG_CHAR *pszFileName,
+                            IMG_UINT32 ui32FileOffset,
+                            IMG_UINT32 ui32Address,
+                            IMG_UINT32 ui32Size,
+                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	eErr = PDumpSNPrintf(hScript,
+			ui32MaxLen,
+			"SAB :%s:0x%08X 0x%08X %s",
+			pszPDumpRegName,
+			ui32Address,
+			ui32FileOffset,
+			pszFileName);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegRead32ToInternalVar
+ * Outputs        : None
+ * Returns        : PVRSRV_ERROR
+ * Description    : Create a PDUMP string, which reads register into an
+ *                  internal variable
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead32ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_CHAR *pszPDumpRegName,
+							IMG_UINT32 ui32Reg,
+							IMG_CHAR *pszInternalVar,
+							IMG_UINT32 ui32Flags)
+
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript,
+							ui32MaxLen,
+							"RDW %s :%s:0x%08X",
+							pszInternalVar,
+							pszPDumpRegName,
+							ui32Reg);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ @name		PDumpRegRead32
+ @brief		Dump 32-bit register read to script
+ @param		pszPDumpDevName - pdump device name
+ @param		ui32RegOffset - register offset
+ @param		ui32Flags - pdump flags
+ @return	Error
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead32(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_CHAR *pszPDumpRegName,
+							const IMG_UINT32 ui32RegOffset,
+							IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW :%s:0x%X",
+							pszPDumpRegName,
+							ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ @name      PDumpRegRead64ToInternalVar
+ @brief     Read 64-bit register into an internal variable
+ @param     pszPDumpDevName - pdump device name
+ @param     ui32RegOffset - register offset
+ @param     ui32Flags - pdump flags
+ @return    Error
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead64ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            IMG_CHAR *pszInternalVar,
+                            const IMG_UINT32 ui32RegOffset,
+                            IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	IMG_CHAR *pszPDumpVarName;
+#endif
+	PDUMP_GET_SCRIPT_STRING();
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW %s :%s:0x%X",
+	                     pszInternalVar,
+	                     pszPDumpRegName,
+	                     ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	pszPDumpVarName = PDumpCreateIncVarNameStr(pszInternalVar);
+	if (pszPDumpVarName == NULL)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW %s :%s:0x%X",
+	                        pszPDumpVarName,
+	                        pszPDumpRegName,
+	                        ui32RegOffset + 4);
+
+	PDumpFreeIncVarNameStr(pszPDumpVarName);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW64 %s :%s:0x%X",
+	                        pszInternalVar,
+	                        pszPDumpRegName,
+	                        ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ @name		PDumpRegRead64
+ @brief		Dump 64-bit register read to script
+ @param		pszPDumpDevName - pdump device name
+ @param		ui32RegOffset - register offset
+ @param		ui32Flags - pdump flags
+ @return	Error
+******************************************************************************/
+PVRSRV_ERROR PDumpRegRead64(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_CHAR *pszPDumpRegName,
+							const IMG_UINT32 ui32RegOffset,
+							IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+#if defined(PDUMP_SPLIT_64BIT_REGISTER_ACCESS)
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW :%s:0x%X",
+							pszPDumpRegName, ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		return eErr;
+	}
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW :%s:0x%X",
+							pszPDumpRegName, ui32RegOffset + 4);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING()
+		PDUMP_UNLOCK(ui32Flags);
+		return eErr;
+	}
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#else
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "RDW64 :%s:0x%X",
+							pszPDumpRegName,
+							ui32RegOffset);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+#endif
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ FUNCTION	: PDumpWriteShiftedMaskedValue
+
+ PURPOSE	: Emits the PDump commands for writing a masked shifted address
+              into another location
+
+ PARAMETERS	: PDump symbolic name and offset of target word
+              PDump symbolic name and offset of source address
+              right shift amount
+              left shift amount
+              mask
+
+ RETURNS	: None
+******************************************************************************/
+PVRSRV_ERROR
+PDumpWriteShiftedMaskedValue(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const IMG_CHAR *pszDestRegspaceName,
+                             const IMG_CHAR *pszDestSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiDestOffset,
+                             const IMG_CHAR *pszRefRegspaceName,
+                             const IMG_CHAR *pszRefSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiRefOffset,
+                             IMG_UINT32 uiSHRAmount,
+                             IMG_UINT32 uiSHLAmount,
+                             IMG_UINT32 uiMask,
+                             IMG_DEVMEM_SIZE_T uiWordSize,
+                             IMG_UINT32 uiPDumpFlags)
+{
+	PVRSRV_ERROR         eError;
+
+	/* Suffix of WRW command in PDump (i.e. WRW or WRW64) */
+	const IMG_CHAR       *pszWrwSuffix;
+
+	/* Internal PDump register used for interim calculation */
+	const IMG_CHAR       *pszPDumpIntRegSpace;
+	IMG_UINT32           uiPDumpIntRegNum;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	if ((uiWordSize != 4) && (uiWordSize != 8))
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	pszWrwSuffix = (uiWordSize == 8) ? "64" : "";
+
+	/* Should really "Acquire" a pdump register here */
+	pszPDumpIntRegSpace = pszDestRegspaceName;
+	uiPDumpIntRegNum = 1;
+
+	PDUMP_LOCK(uiPDumpFlags);
+	eError = PDumpSNPrintf(hScript,
+	               ui32MaxLen,
+	               /* Should this be "MOV" instead? */
+	               "WRW :%s:$%d :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC "\n",
+	               /* dest */
+	               pszPDumpIntRegSpace,
+	               uiPDumpIntRegNum,
+	               /* src */
+	               pszRefRegspaceName,
+	               pszRefSymbolicName,
+	               uiRefOffset);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+	if (uiSHRAmount > 0)
+	{
+		eError = PDumpSNPrintf(hScript,
+		               ui32MaxLen,
+		               "SHR :%s:$%d :%s:$%d 0x%X\n",
+		               /* dest */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src A */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src B */
+		               uiSHRAmount);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+	if (uiSHLAmount > 0)
+	{
+		eError = PDumpSNPrintf(hScript,
+		               ui32MaxLen,
+		               "SHL :%s:$%d :%s:$%d 0x%X\n",
+		               /* dest */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src A */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src B */
+		               uiSHLAmount);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+	if (uiMask != (1ULL << (8*uiWordSize))-1)
+	{
+		eError = PDumpSNPrintf(hScript,
+		               ui32MaxLen,
+		               "AND :%s:$%d :%s:$%d 0x%X\n",
+		               /* dest */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src A */
+		               pszPDumpIntRegSpace,
+		               uiPDumpIntRegNum,
+		               /* src B */
+		               uiMask);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+	eError = PDumpSNPrintf(hScript,
+	               ui32MaxLen,
+	               "WRW%s :%s:%s:" IMG_DEVMEM_OFFSET_FMTSPEC " :%s:$%d\n",
+	               pszWrwSuffix,
+	               /* dest */
+	               pszDestRegspaceName,
+	               pszDestSymbolicName,
+	               uiDestOffset,
+	               /* src */
+	               pszPDumpIntRegSpace,
+	               uiPDumpIntRegNum);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+	PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+ErrUnlock:
+	PDUMP_UNLOCK(uiPDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PDumpWriteSymbAddress(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      const IMG_CHAR *pszDestSpaceName,
+                      IMG_DEVMEM_OFFSET_T uiDestOffset,
+                      const IMG_CHAR *pszRefSymbolicName,
+                      IMG_DEVMEM_OFFSET_T uiRefOffset,
+                      const IMG_CHAR *pszPDumpDevName,
+                      IMG_UINT32 ui32WordSize,
+                      IMG_UINT32 ui32AlignShift,
+                      IMG_UINT32 ui32Shift,
+                      IMG_UINT32 uiPDumpFlags)
+{
+	const IMG_CHAR       *pszWrwSuffix = "";
+	PVRSRV_ERROR         eError = PVRSRV_OK;
+
+	PDUMP_GET_SCRIPT_STRING();
+
+	if (ui32WordSize == 8)
+	{
+		pszWrwSuffix = "64";
+	}
+
+	PDUMP_LOCK(uiPDumpFlags);
+
+	if (ui32AlignShift != ui32Shift)
+	{
+		/* Write physical address into a variable */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "WRW%s :%s:$1 %s:" IMG_DEVMEM_OFFSET_FMTSPEC "\n",
+				       pszWrwSuffix,
+				       /* dest */
+				       pszPDumpDevName,
+				       /* src */
+				       pszRefSymbolicName,
+				       uiRefOffset);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+		/* apply address alignment */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "SHR :%s:$1 :%s:$1 0x%X",
+				       /* dest */
+				       pszPDumpDevName,
+				       /* src A */
+				       pszPDumpDevName,
+				       /* src B */
+				       ui32AlignShift);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+		/* apply address shift */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "SHL :%s:$1 :%s:$1 0x%X",
+				       /* dest */
+				       pszPDumpDevName,
+				       /* src A */
+				       pszPDumpDevName,
+				       /* src B */
+				       ui32Shift);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+
+
+		/* write result to register */
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "WRW%s :%s:0x%08X :%s:$1",
+				       pszWrwSuffix,
+				       pszDestSpaceName,
+				       (IMG_UINT32)uiDestOffset,
+				       pszPDumpDevName);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+	else
+	{
+		eError = PDumpSNPrintf(hScript,
+				       ui32MaxLen,
+				       "WRW%s :%s:" IMG_DEVMEM_OFFSET_FMTSPEC " %s:" IMG_DEVMEM_OFFSET_FMTSPEC "\n",
+				       pszWrwSuffix,
+				       /* dest */
+				       pszDestSpaceName,
+				       uiDestOffset,
+				       /* src */
+				       pszRefSymbolicName,
+				       uiRefOffset);
+		PVR_GOTO_IF_ERROR(eError, symbAddress_error);
+		PDumpWriteScript(psDeviceNode, hScript, uiPDumpFlags);
+	}
+
+symbAddress_error:
+	PDUMP_UNLOCK(uiPDumpFlags);
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return eError;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpIDLWithFlags
+ * Inputs         : Idle time in clocks
+ * Outputs        : None
+ * Returns        : Error
+ * Description    : Dump IDL command to script
+******************************************************************************/
+PVRSRV_ERROR PDumpIDLWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Clocks, IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "IDL %u", ui32Clocks);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpIDL
+ * Inputs         : Idle time in clocks
+ * Outputs        : None
+ * Returns        : Error
+ * Description    : Dump IDL command to script
+******************************************************************************/
+PVRSRV_ERROR PDumpIDL(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32Clocks)
+{
+	return PDumpIDLWithFlags(psDeviceNode, ui32Clocks, PDUMP_FLAGS_CONTINUOUS);
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegBasedCBP
+ * Inputs         : pszPDumpRegName, ui32RegOffset, ui32WPosVal, ui32PacketSize
+ *                  ui32BufferSize, ui32Flags
+ * Outputs        : None
+ * Returns        : Error
+ * Description    : Dump CBP command to script
+******************************************************************************/
+PVRSRV_ERROR PDumpRegBasedCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+							  IMG_CHAR		*pszPDumpRegName,
+							  IMG_UINT32	ui32RegOffset,
+							  IMG_UINT32	ui32WPosVal,
+							  IMG_UINT32	ui32PacketSize,
+							  IMG_UINT32	ui32BufferSize,
+							  IMG_UINT32	ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	eErr = PDumpSNPrintf(hScript,
+			 ui32MaxLen,
+			 "CBP :%s:0x%08X 0x%08X 0x%08X 0x%08X",
+			 pszPDumpRegName,
+			 ui32RegOffset,
+			 ui32WPosVal,
+			 ui32PacketSize,
+			 ui32BufferSize);
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PDumpTRG(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR *pszMemSpace,
+                      IMG_UINT32 ui32MMUCtxID,
+                      IMG_UINT32 ui32RegionID,
+                      IMG_BOOL bEnable,
+                      IMG_UINT64 ui64VAddr,
+                      IMG_UINT64 ui64LenBytes,
+                      IMG_UINT32 ui32XStride,
+                      IMG_UINT32 ui32Flags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING();
+
+	if (bEnable)
+	{
+		eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+				     "TRG :%s:v%u %u 0x%08"IMG_UINT64_FMTSPECX" 0x%08"IMG_UINT64_FMTSPECX" %u",
+				     pszMemSpace, ui32MMUCtxID, ui32RegionID,
+				     ui64VAddr, ui64LenBytes, ui32XStride);
+	}
+	else
+	{
+		eErr = PDumpSNPrintf(hScript, ui32MaxLen,
+				     "TRG :%s:v%u %u",
+				     pszMemSpace, ui32MMUCtxID, ui32RegionID);
+
+	}
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32Flags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32Flags);
+	PDUMP_UNLOCK(ui32Flags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpConnectionNotify
+ * Description    : Called by the srvcore to tell PDump core that the
+ *                  PDump capture and control client has connected
+******************************************************************************/
+void PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(TL_BUFFER_STATS)
+	PVRSRV_ERROR		eErr;
+#endif
+
+	OSAtomicIncrement(&g_sConnectionCount);
+
+	/* Reset the parameter file attributes */
+	g_PDumpParameters.sWOff.ui32Main = g_PDumpParameters.sWOff.ui32Init;
+	g_PDumpParameters.ui32FileIdx = 0;
+
+	/* Reset the script file attributes */
+	g_PDumpScript.ui32FileIdx = 0;
+
+	/* The Main script & parameter buffers should be empty after the previous
+	 * PDump capture if it completed correctly.
+	 * When PDump client is not connected, writes are prevented to Main
+	 * buffers in PDumpWriteAllowed() since no capture range, no client,
+	 * no writes to Main buffers for continuous flagged and regular writes.
+	 */
+	if (!TLStreamOutOfData(g_PDumpParameters.sCh.sMainStream.hTL)) /* !empty */
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDump Main parameter buffer not empty, capture will be corrupt!"));
+	}
+	if (!TLStreamOutOfData(g_PDumpScript.sCh.sMainStream.hTL)) /* !empty */
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PDump Main script buffer not empty, capture will be corrupt!"));
+	}
+
+#if defined(TL_BUFFER_STATS)
+	eErr = TLStreamResetProducerByteCount(g_PDumpParameters.sCh.sMainStream.hTL, g_PDumpParameters.sWOff.ui32Init);
+	PVR_LOG_IF_ERROR(eErr, "TLStreamResetByteCount Parameter Main");
+
+	eErr = TLStreamResetProducerByteCount(g_PDumpScript.sCh.sMainStream.hTL, 0);
+	PVR_LOG_IF_ERROR(eErr, "TLStreamResetByteCount Script Main");
+#endif
+
+	if (psDeviceNode->pfnPDumpInitDevice)
+	{
+		/* Reset pdump according to connected device */
+		psDeviceNode->pfnPDumpInitDevice(psDeviceNode);
+	}
+}
+
+/******************************************************************************
+ * Function Name  : PDumpDisconnectionNotify
+ * Description    : Called by the connection_server to tell PDump core that
+ *                  the PDump capture and control client has disconnected
+******************************************************************************/
+void PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eErr;
+
+	if (PDumpCtrlCaptureOn())
+	{
+		PVR_LOG(("pdump killed, capture files may be invalid or incomplete!"));
+
+		/* Disable capture in server, in case PDump client was killed and did
+		 * not get a chance to reset the capture parameters.
+		 * Will set module state back to READY.
+		 */
+		eErr = PDumpSetDefaultCaptureParamsKM(NULL, psDeviceNode, PDUMP_CAPMODE_UNSET,
+						      PDUMP_FRAME_UNSET, PDUMP_FRAME_UNSET, 0, 0, 0);
+		PVR_LOG_IF_ERROR(eErr, "PDumpSetDefaultCaptureParamsKM");
+	}
+}
+
+/******************************************************************************
+ * Function Name  : PDumpRegCondStr
+ * Inputs         : Description of what this register read is trying to do
+ *					pszPDumpDevName
+ *					Register offset
+ *					expected value
+ *					mask for that value
+ * Outputs        : PDump conditional test for use with 'IF' and 'DOW'
+ * Returns        : None
+ * Description    : Create a PDUMP conditional test. The string is allocated
+ *					on the heap and should be freed by the caller on success.
+******************************************************************************/
+PVRSRV_ERROR PDumpRegCondStr(IMG_CHAR            **ppszPDumpCond,
+                             IMG_CHAR            *pszPDumpRegName,
+                             IMG_UINT32          ui32RegAddr,
+                             IMG_UINT32          ui32RegValue,
+                             IMG_UINT32          ui32Mask,
+                             IMG_UINT32          ui32Flags,
+                             PDUMP_POLL_OPERATOR eOperator)
+{
+	IMG_UINT32	ui32PollCount;
+
+	PDUMP_GET_MSG_STRING();
+
+	ui32PollCount = POLL_COUNT_SHORT;
+
+	if (0 == OSSNPrintf(pszMsg, ui32MaxLen, ":%s:0x%08X 0x%08X 0x%08X %d %u %d",
+						pszPDumpRegName, ui32RegAddr, ui32RegValue,
+						ui32Mask, eOperator, ui32PollCount, POLL_DELAY))
+	{
+		PDUMP_RELEASE_MSG_STRING()
+		return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+
+	*ppszPDumpCond = pszMsg;
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpInternalValCondStr
+ * Inputs         : Description of what this register read is trying to do
+ *					pszPDumpDevName
+ *					Internal variable
+ *					expected value
+ *					mask for that value
+ * Outputs        : PDump conditional test for use with 'IF' and 'DOW'
+ * Returns        : None
+ * Description    : Create a PDUMP conditional test. The string is allocated
+ *					on the heap and should be freed by the caller on success.
+******************************************************************************/
+PVRSRV_ERROR PDumpInternalValCondStr(IMG_CHAR            **ppszPDumpCond,
+                                     IMG_CHAR            *pszInternalVar,
+                                     IMG_UINT32          ui32RegValue,
+                                     IMG_UINT32          ui32Mask,
+                                     IMG_UINT32          ui32Flags,
+                                     PDUMP_POLL_OPERATOR eOperator)
+{
+	IMG_UINT32	ui32PollCount;
+
+	PDUMP_GET_MSG_STRING();
+
+	ui32PollCount = POLL_COUNT_SHORT;
+
+	if (0 == OSSNPrintf(pszMsg, ui32MaxLen, "%s 0x%08X 0x%08X %d %u %d",
+						pszInternalVar, ui32RegValue,
+						ui32Mask, eOperator, ui32PollCount, POLL_DELAY))
+	{
+		PDUMP_RELEASE_MSG_STRING()
+		return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+
+	*ppszPDumpCond = pszMsg;
+
+	return PVRSRV_OK;
+}
+
+
+/******************************************************************************
+ * Function Name  : PDumpIfKM
+ * Inputs         : pszPDumpCond - string for condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents IF command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpIfKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "IF %s\n", pszPDumpCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpElseKM
+ * Inputs         : pszPDumpCond - string for condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents ELSE command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpElseKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "ELSE %s\n", pszPDumpCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpFiKM
+ * Inputs         : pszPDumpCond - string for condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents FI command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpFiKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "FI %s\n", pszPDumpCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpStartDoLoopKM
+ * Inputs         : None
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents SDO command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpStartDoLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "SDO");
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ * Function Name  : PDumpEndDoWhileLoopKM
+ * Inputs         : pszPDumpWhileCond - string for loop condition
+ * Outputs        : None
+ * Returns        : None
+ * Description    : Create a PDUMP string which represents DOW command
+					with condition.
+******************************************************************************/
+PVRSRV_ERROR PDumpEndDoWhileLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_CHAR *pszPDumpWhileCond,
+                                   IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eErr;
+	PDUMP_GET_SCRIPT_STRING()
+
+	eErr = PDumpSNPrintf(hScript, ui32MaxLen, "DOW %s\n", pszPDumpWhileCond);
+
+	if (eErr != PVRSRV_OK)
+	{
+		PDUMP_RELEASE_SCRIPT_STRING();
+		return eErr;
+	}
+
+	PDUMP_LOCK(ui32PDumpFlags);
+	PDumpWriteScript(psDeviceNode, hScript, ui32PDumpFlags);
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	PDUMP_RELEASE_SCRIPT_STRING();
+
+	return PVRSRV_OK;
+}
+
+
+void PDumpLock(void)
+{
+	OSLockAcquire(g_hPDumpWriteLock);
+}
+void PDumpUnlock(void)
+{
+	OSLockRelease(g_hPDumpWriteLock);
+}
+static void PDumpAssertWriteLockHeld(void)
+{
+	/* It is expected to be g_hPDumpWriteLock is locked at this point. */
+	PVR_ASSERT(OSLockIsLocked(g_hPDumpWriteLock));
+}
+
+#if defined(PDUMP_TRACE_STATE)
+void PDumpCommonDumpState(void)
+{
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.hTL (In, Mn, De, Bk) ( %p, %p, %p, %p )",
+			g_PDumpScript.sCh.sInitStream.hTL, g_PDumpScript.sCh.sMainStream.hTL, g_PDumpScript.sCh.sDeinitStream.hTL, g_PDumpScript.sCh.sBlockStream.hTL));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.ui32BufferFullRetries (In, Mn, De, Bk) ( %5d, %5d, %5d, %5d )",
+			g_PDumpScript.sCh.sInitStream.ui32BufferFullRetries,
+			g_PDumpScript.sCh.sMainStream.ui32BufferFullRetries,
+			g_PDumpScript.sCh.sDeinitStream.ui32BufferFullRetries,
+			g_PDumpScript.sCh.sBlockStream.ui32BufferFullRetries));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.ui32BufferFullAborts (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+				g_PDumpScript.sCh.sInitStream.ui32BufferFullAborts,
+				g_PDumpScript.sCh.sMainStream.ui32BufferFullAborts,
+				g_PDumpScript.sCh.sDeinitStream.ui32BufferFullAborts,
+				g_PDumpScript.sCh.sBlockStream.ui32BufferFullAborts));
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.sCh.*.ui32HighestRetriesWatermark (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+				g_PDumpScript.sCh.sInitStream.ui32HighestRetriesWatermark,
+				g_PDumpScript.sCh.sMainStream.ui32HighestRetriesWatermark,
+				g_PDumpScript.sCh.sDeinitStream.ui32HighestRetriesWatermark,
+			    g_PDumpScript.sCh.sBlockStream.ui32HighestRetriesWatermark));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpScript.ui32FileIdx( %d )", g_PDumpScript.ui32FileIdx));
+
+
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.hTL (In, Mn, De, Bk) ( %p, %p, %p, %p )",
+			g_PDumpParameters.sCh.sInitStream.hTL, g_PDumpParameters.sCh.sMainStream.hTL, g_PDumpParameters.sCh.sDeinitStream.hTL, g_PDumpParameters.sCh.sBlockStream.hTL));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.ui32BufferFullRetries (In, Mn, De, Bk) ( %5d, %5d, %5d, %5d )",
+			g_PDumpParameters.sCh.sInitStream.ui32BufferFullRetries,
+			g_PDumpParameters.sCh.sMainStream.ui32BufferFullRetries,
+			g_PDumpParameters.sCh.sDeinitStream.ui32BufferFullRetries,
+			g_PDumpParameters.sCh.sBlockStream.ui32BufferFullRetries));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.ui32BufferFullAborts (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+			g_PDumpParameters.sCh.sInitStream.ui32BufferFullAborts,
+			g_PDumpParameters.sCh.sMainStream.ui32BufferFullAborts,
+			g_PDumpParameters.sCh.sDeinitStream.ui32BufferFullAborts,
+			g_PDumpParameters.sCh.sBlockStream.ui32BufferFullAborts));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sCh.*.ui32HighestRetriesWatermark (In, Mn, De, Bk)  ( %5d, %5d, %5d, %5d )",
+				g_PDumpParameters.sCh.sInitStream.ui32HighestRetriesWatermark,
+				g_PDumpParameters.sCh.sMainStream.ui32HighestRetriesWatermark,
+				g_PDumpParameters.sCh.sDeinitStream.ui32HighestRetriesWatermark,
+				g_PDumpParameters.sCh.sBlockStream.ui32HighestRetriesWatermark));
+
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.sWOff.* (In, Mn, De, Bk) ( %d, %d, %d, %d )",
+			g_PDumpParameters.sWOff.ui32Init, g_PDumpParameters.sWOff.ui32Main, g_PDumpParameters.sWOff.ui32Deinit, g_PDumpParameters.sWOff.ui32Block));
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpParameters.ui32FileIdx( %d )", g_PDumpParameters.ui32FileIdx));
+
+	PVR_LOG(("--- PDUMP COMMON: g_PDumpCtrl( %p ) eServiceState( %d ), IsDriverInInitPhase( %s ) ui32Flags( %x )",
+			&g_PDumpCtrl, g_PDumpCtrl.eServiceState, CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_DRIVER_IN_INIT_PHASE) ? "yes" : "no", g_PDumpCtrl.ui32Flags));
+	PVR_LOG(("--- PDUMP COMMON: ui32DefaultCapMode( %d ) ui32CurrentFrame( %d )",
+			g_PDumpCtrl.ui32DefaultCapMode, g_PDumpCtrl.ui32CurrentFrame));
+	PVR_LOG(("--- PDUMP COMMON: sCaptureRange.ui32Start( %x ) sCaptureRange.ui32End( %x ) sCaptureRange.ui32Interval( %u )",
+			g_PDumpCtrl.sCaptureRange.ui32Start, g_PDumpCtrl.sCaptureRange.ui32End, g_PDumpCtrl.sCaptureRange.ui32Interval));
+	PVR_LOG(("--- PDUMP COMMON: IsInCaptureRange( %s ) IsInCaptureInterval( %s ) InPowerTransition( %d )",
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_RANGE) ? "yes" : "no",
+			CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_IN_CAPTURE_INTERVAL) ? "yes" : "no",
+			PDumpCtrlInPowerTransition()));
+	PVR_LOG(("--- PDUMP COMMON: sBlockCtrl.ui32BlockLength( %d ), sBlockCtrl.ui32CurrentBlock( %d )",
+			g_PDumpCtrl.sBlockCtrl.ui32BlockLength, g_PDumpCtrl.sBlockCtrl.ui32CurrentBlock));
+}
+#endif /* defined(PDUMP_TRACE_STATE) || defined(PVR_TESTING_UTILS) */
+
+static void PDumpStartTimer(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     void *hSyncPrivData,
+                                     PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                                     PDUMP_CONNECTION_DATA **ppsPDumpConnectionData)
+{
+	PDUMP_CONNECTION_DATA *psPDumpConnectionData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(ppsPDumpConnectionData != NULL);
+	PVR_ASSERT(pfnPDumpSyncBlocks != NULL);
+	PVR_ASSERT(hSyncPrivData != NULL);
+
+	psPDumpConnectionData = OSAllocMem(sizeof(*psPDumpConnectionData));
+	PVR_GOTO_IF_NOMEM(psPDumpConnectionData, eError, fail_alloc);
+
+	eError = OSLockCreate(&psPDumpConnectionData->hLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lockcreate);
+
+	dllist_init(&psPDumpConnectionData->sListHead);
+	OSAtomicWrite(&psPDumpConnectionData->sRefCount, 1);
+	psPDumpConnectionData->ui32LastSetFrameNumber = PDUMP_FRAME_UNSET;
+	psPDumpConnectionData->eLastEvent = PDUMP_TRANSITION_EVENT_NONE;
+	psPDumpConnectionData->eFailedEvent = PDUMP_TRANSITION_EVENT_NONE;
+
+	/*
+	 * Although we don't take a ref count here, handle base destruction
+	 * will ensure that any resource that might trigger us to do a Transition
+	 * will have been freed before the sync blocks which are keeping the sync
+	 * connection data alive.
+	 */
+	psPDumpConnectionData->hSyncPrivData = hSyncPrivData;
+	psPDumpConnectionData->pfnPDumpSyncBlocks = pfnPDumpSyncBlocks;
+
+	*ppsPDumpConnectionData = psPDumpConnectionData;
+
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		IMG_INT iRefCount;
+
+		/* Add this new reference to the global count of active connections */
+		iRefCount = OSAtomicIncrement(&gPDumpNumConnex);
+		PDUMP_REFCOUNT_PRINT("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                     &gPDumpNumConnex, iRefCount);
+
+		if (((iRefCount > 1) &&
+			(g_PDumpCtrl.ui32TimeoutFrequency != 0U)) &&
+			(g_PDumpTimerID == NULL))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: Starting Timeout Chain now, refcnt = %d",
+			         __func__, iRefCount));
+			PDumpStartTimer(psDeviceNode);
+		}
+	}
+
+	return PVRSRV_OK;
+
+fail_lockcreate:
+	OSFreeMem(psPDumpConnectionData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	IMG_INT iRefCount;
+
+	_PDumpConnectionRelease(psPDumpConnectionData);
+	if (PDumpIsDevicePermitted(psDeviceNode))
+	{
+		/* Remove this connection from the global count */
+		iRefCount = OSAtomicDecrement(&gPDumpNumConnex);
+		PDUMP_REFCOUNT_PRINT("%s: gPDumpNumConnex (%p) = %d", __func__,
+		                     &gPDumpNumConnex, iRefCount);
+		PVR_ASSERT(iRefCount >= 0);
+	}
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+
+/*!
+ * \name	PDumpSNPrintf
+ */
+PVRSRV_ERROR PDumpSNPrintf(IMG_HANDLE hBuf, IMG_UINT32 ui32ScriptSizeMax, IMG_CHAR* pszFormat, ...)
+{
+	IMG_CHAR* pszBuf = hBuf;
+	IMG_INT32 n;
+	va_list	vaArgs;
+
+	va_start(vaArgs, pszFormat);
+
+	n = OSVSNPrintf(pszBuf, ui32ScriptSizeMax, pszFormat, vaArgs);
+
+	va_end(vaArgs);
+
+	if (n>=(IMG_INT32)ui32ScriptSizeMax || n==-1)	/* glibc >= 2.1 or glibc 2.0 */
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Buffer overflow detected, pdump output may be incomplete."));
+
+		return PVRSRV_ERROR_PDUMP_BUF_OVERFLOW;
+	}
+
+#if defined(PDUMP_DEBUG_OUTFILES)
+	OSAtomicIncrement(&g_sEveryLineCounter);
+#endif
+
+	/* Put line ending sequence at the end if it isn't already there */
+	_PDumpVerifyLineEnding(pszBuf, ui32ScriptSizeMax);
+
+	return PVRSRV_OK;
+}
+
+void PDumpTimerCB(void *pvData);
+void PDumpTimerCB(void *pvData)
+{
+	IMG_INT iRefCount;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (CHECK_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Already flagged as TERMINATED",
+		         __func__));
+	}
+
+	/* Simply check to see if the global connection count indicates that all
+	 * subsequent applications requiring PDump logging have terminated.
+	 * In a quiescent state we will have a singleton pdump utility still
+	 * connected.
+	 */
+	if ((iRefCount = OSAtomicRead(&gPDumpNumConnex)) == 1)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: No connections active (%d), flagging as AUTO_TERMINATED",
+		         __func__, iRefCount));
+		SET_PDUMP_CONTROL_FLAG(FLAG_IS_AUTO_TERMINATED);
+	}
+}
+
+/*
+ * FunctionName  PDumpStartTimer
+ * Description   Start an OSTimer chain running to scan for active connections
+ *               being present in the connections associated with the given
+ *               psDeviceNode. Only started if we have AutoTerminate flagged
+ *               in the internal PDump state.
+ * Inputs        psDeviceNode   associated device node to scan for connections
+ * Returns       nothing
+ */
+static void PDumpStartTimer(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (!PDumpIsDevicePermitted(psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: DeviceID %u not valid", __func__,
+		         psDeviceNode->sDevId.ui32InternalID));
+		return;
+	}
+
+	if (g_PDumpCtrl.ui32TimeoutFrequency == 0U)
+	{
+		return;
+	}
+
+	if (g_PDumpTimerID != NULL)
+	{
+		eError = OSDisableTimer(g_PDumpTimerID);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSDisableTimer");
+		eError = OSRemoveTimer(g_PDumpTimerID);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSRemoveTimer");
+	}
+
+	g_PDumpTimerID = OSAddTimer(PDumpTimerCB, NULL, g_PDumpCtrl.ui32TimeoutFrequency * 1000U);
+
+	if (g_PDumpTimerID != NULL)
+	{
+		eError = OSEnableTimer(g_PDumpTimerID);
+
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEnableTimer");
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Timer %p now active.", __func__,
+		         g_PDumpTimerID));
+	}
+}
+
+#endif /* defined(PDUMP) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physheap.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physheap.c
new file mode 100644
index 000000000000..c5c603024b54
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physheap.c
@@ -0,0 +1,2331 @@
+/*************************************************************************/ /*!
+@File           physheap.c
+@Title          Physical heap management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Management functions for the physical heap(s). A heap contains
+                all the information required by services when using memory from
+                that heap (such as CPU <> Device physical address translation).
+                A system must register one heap but can have more then one which
+                is why a heap must register with a (system) unique ID.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "physheap.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "physmem.h"
+#include "physmem_hostmem.h"
+#include "physmem_lma.h"
+#include "physmem_dlm.h"
+#include "physmem_ima.h"
+#include "physmem_osmem.h"
+#include "debug_common.h"
+
+struct _PHYS_HEAP_
+{
+	/*! The type of this heap */
+	PHYS_HEAP_TYPE			eType;
+
+	/*! The allocation policy for this heap */
+	PHYS_HEAP_POLICY uiPolicy;
+
+	/* Config flags */
+	PHYS_HEAP_USAGE_FLAGS		ui32UsageFlags;
+
+	/* OOM Detection state */
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	ATOMIC_T					sOOMDetected;
+#endif
+
+	/*! Pointer to device node struct */
+	PPVRSRV_DEVICE_NODE         psDevNode;
+	/*! PDump name of this physical memory heap */
+	IMG_CHAR					*pszPDumpMemspaceName;
+	/*! Physheap name of this physical memory heap */
+	IMG_CHAR					aszName[PHYS_HEAP_NAME_SIZE];
+	/*! Private data for the translate routines */
+	IMG_HANDLE					hPrivData;
+	/*! Function callbacks */
+	PHYS_HEAP_FUNCTIONS			*psMemFuncs;
+
+	/*! Refcount */
+	IMG_UINT32					ui32RefCount;
+
+	/*! Implementation specific */
+	PHEAP_IMPL_DATA             pvImplData;
+	PHEAP_IMPL_FUNCS            *psImplFuncs;
+
+	/*! Pointer to next physical heap */
+	struct _PHYS_HEAP_		*psNext;
+
+#if defined(SUPPORT_STATIC_IPA)
+	/*! IPA Policy value from Heap Config */
+	IMG_UINT32                  ui32IPAPolicyValue;
+
+	/*! IPA Clear Mask value from Heap Config */
+	IMG_UINT32                  ui32IPAClearMask;
+
+	/*! IPA Bit Shift value from Heap Config */
+	IMG_UINT32                  ui32IPAShift;
+#endif /* defined(SUPPORT_STATIC_IPA) */
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	/* Pointer to the Spas region the physheap is part of
+	 * or NULL. */
+	PHYS_HEAP_SPAS_REGION      *psSpasRegion;
+	/* The Spas regions list of physheaps.
+	 * Each physheap is only part of at most 1 Spas region. */
+	DLLIST_NODE                 sSpasSibling;
+#endif
+};
+
+#if defined(REFCOUNT_DEBUG)
+#define PHYSHEAP_REFCOUNT_PRINT(fmt, ...)	\
+	PVRSRVDebugPrintf(PVR_DBG_WARNING,	\
+			  __FILE__,		\
+			  __LINE__,		\
+			  fmt,			\
+			  __VA_ARGS__)
+#else
+#define PHYSHEAP_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+#define IsOOMError(err) ((err == PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES) | \
+						 (err == PVRSRV_ERROR_OUT_OF_MEMORY) | \
+						 (err == PVRSRV_ERROR_PMR_TOO_LARGE))
+
+typedef enum _PVR_LAYER_HEAP_ACTION_
+{
+	PVR_LAYER_HEAP_ACTION_IGNORE,          /* skip heap during heap init */
+	PVR_LAYER_HEAP_ACTION_INSTANTIATE,     /* instantiate heap but don't acquire */
+	PVR_LAYER_HEAP_ACTION_INITIALISE       /* instantiate and acquire */
+
+} PVR_LAYER_HEAP_ACTION;
+
+typedef struct PHYS_HEAP_PROPERTIES_TAG
+{
+	PVRSRV_PHYS_HEAP eFallbackHeap;
+	PVR_LAYER_HEAP_ACTION ePVRLayerAction;
+	IMG_BOOL bUserModeAlloc;
+} PHYS_HEAP_PROPERTIES;
+
+/* NOTE: Table entries and order must match enum PVRSRV_PHYS_HEAP to ensure
+ * correct operation of PhysHeapCreatePMR().
+ */
+static PHYS_HEAP_PROPERTIES gasHeapProperties[PVRSRV_PHYS_HEAP_LAST] =
+{
+	/* eFallbackHeap,               ePVRLayerAction,                   bUserModeAlloc */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* DEFAULT */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* CPU_LOCAL */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_LOCAL */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_PRIVATE */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_MAIN */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_FALSE }, /* EXTERNAL */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_FALSE }, /* GPU_COHERENT */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_SECURE */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_CONFIG */
+    {  PVRSRV_PHYS_HEAP_FW_MAIN,    PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_CODE */
+    {  PVRSRV_PHYS_HEAP_FW_MAIN,    PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PRIV_DATA */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP_PT */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP0, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP0 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP1, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP1 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP2, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP2 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP3, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP3 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP4, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP4 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP5, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP5 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP6, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP6 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP7, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP7 */
+    {  PVRSRV_PHYS_HEAP_WRAP,       PVR_LAYER_HEAP_ACTION_INSTANTIATE, IMG_FALSE }, /* WRAP */
+    {  PVRSRV_PHYS_HEAP_DISPLAY,    PVR_LAYER_HEAP_ACTION_INSTANTIATE, IMG_FALSE }, /* DISPLAY */
+};
+
+static_assert((ARRAY_SIZE(gasHeapProperties) == PVRSRV_PHYS_HEAP_LAST),
+	"Size or order of gasHeapProperties entries incorrect for PVRSRV_PHYS_HEAP enum");
+
+static IMG_BOOL PhysHeapCreatedByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+static IMG_BOOL PhysHeapAcquiredByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+
+/**
+ * ! IMPORTANT !
+ * Do not change this string array unless the usage flag definitions in
+ * physheap_config.h have changed.
+ *
+ * NOTE: Use DebugCommonFlagStrings or GetPhysHeapUsageString to get
+ * usage flags string.
+ */
+static const IMG_FLAGS2DESC g_asPhysHeapUsageFlagStrings[] =
+{
+	{PHYS_HEAP_USAGE_CPU_LOCAL,		"CPU_LOCAL"},
+	{PHYS_HEAP_USAGE_GPU_LOCAL,		"GPU_LOCAL"},
+	{PHYS_HEAP_USAGE_GPU_PRIVATE,	"GPU_PRIVATE"},
+	{PHYS_HEAP_USAGE_EXTERNAL,		"EXTERNAL"},
+	{PHYS_HEAP_USAGE_GPU_COHERENT,	"GPU_COHERENT"},
+	{PHYS_HEAP_USAGE_GPU_SECURE,	"GPU_SECURE"},
+	{PHYS_HEAP_USAGE_FW_SHARED,		"FW_SHARED"},
+	{PHYS_HEAP_USAGE_FW_PRIVATE,	"FW_PRIVATE"},
+	{PHYS_HEAP_USAGE_FW_CODE,		"FW_CODE"},
+	{PHYS_HEAP_USAGE_FW_PRIV_DATA,	"FW_PRIV_DATA"},
+	{PHYS_HEAP_USAGE_FW_PREMAP_PT,	"FW_PREMAP_PT"},
+	{PHYS_HEAP_USAGE_FW_PREMAP,		"FW_PREMAP"},
+	{PHYS_HEAP_USAGE_WRAP,			"WRAP"},
+	{PHYS_HEAP_USAGE_DISPLAY,		"DISPLAY"},
+	{PHYS_HEAP_USAGE_DLM,			"DLM"}
+};
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCheckValidUsageFlags
+@Description    Checks if any bits were set outside of the valid ones within
+                PHYS_HEAP_USAGE_FLAGS.
+
+@Input          ui32PhysHeapUsage  The value of the usage flag.
+
+@Return         True or False depending on whether there were only valid bits set.
+*/ /**************************************************************************/
+static inline IMG_BOOL PhysHeapCheckValidUsageFlags(PHYS_HEAP_USAGE_FLAGS ui32PhysHeapUsage)
+{
+	return !(ui32PhysHeapUsage & ~PHYS_HEAP_USAGE_MASK);
+}
+
+/*************************************************************************/ /*!
+@Function       GetPhysHeapUsageString
+@Description    This function is used to create a comma separated string of all
+                usage flags passed in as a bitfield.
+
+@Input          ui32UsageFlags     The bitfield of usage flags.
+@Input          ui32Size           The size of the memory pointed to by
+                                   pszUsageString.
+@Output         pszUsageString     A pointer to memory where the created string
+                                   will be stored.
+
+@Return         If successful PVRSRV_OK, else a PVRSRV_ERROR.
+*/ /**************************************************************************/
+static PVRSRV_ERROR GetPhysHeapUsageString(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags,
+                                          IMG_UINT32 ui32Size,
+                                          IMG_CHAR *const pszUsageString)
+{
+	IMG_UINT32 i;
+	IMG_BOOL bFirst = IMG_TRUE;
+	size_t uiSize = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszUsageString != NULL, "pszUsageString");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui32Size > 0, "ui32Size");
+
+	/* Initialise the string to be null terminated at the beginning */
+	pszUsageString[0] = '\0';
+
+	if (ui32UsageFlags == 0)
+	{
+		const ssize_t iCopiedCnt = OSStringSafeCopy(pszUsageString, "NONE", (size_t)ui32Size);
+		PVR_LOG_RETURN_IF_FALSE((iCopiedCnt >= 0), "OSStringSafeCopy", PVRSRV_ERROR_OUT_OF_MEMORY);
+
+		return PVRSRV_OK;
+	}
+
+	/* Process from left to right. */
+	for (i = (sizeof(PHYS_HEAP_USAGE_FLAGS) * BITS_PER_BYTE - 1); i > 0; i--)
+	{
+		IMG_UINT32 ui32Flag = BIT(i);
+
+		if (BITMASK_HAS(ui32UsageFlags, ui32Flag))
+		{
+			IMG_CHAR pszString[32] = "\0";
+
+			if (PhysHeapCheckValidUsageFlags(ui32Flag))
+			{
+				DebugCommonFlagStrings(pszString,
+				                   sizeof(pszString),
+				                   g_asPhysHeapUsageFlagStrings,
+				                   ARRAY_SIZE(g_asPhysHeapUsageFlagStrings),
+				                   ui32Flag);
+			}
+			else
+			{
+				uiSize = OSStringLCat(pszString,
+				                      "INVALID",
+				                      sizeof(pszString));
+				PVR_LOG_RETURN_IF_FALSE((uiSize < sizeof(pszString)), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+			}
+
+			if (!bFirst)
+			{
+				uiSize = OSStringLCat(pszUsageString,
+				                      ", ",
+				                      (size_t)ui32Size);
+				PVR_LOG_RETURN_IF_FALSE((uiSize < ui32Size), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+			}
+			else
+			{
+				bFirst = IMG_FALSE;
+			}
+
+			uiSize = OSStringLCat(pszUsageString,
+			                      pszString,
+			                      (size_t)ui32Size);
+			PVR_LOG_RETURN_IF_FALSE((uiSize < ui32Size), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapPrintHeapProperties
+@Description    This function is used to print properties
+                of the specified physheap.
+
+@Input          psPhysHeap          The physheap to create the string from.
+@Input          pfnDumpDebugPrintf  The specified print function that should be
+                                    used to dump any debug information
+                                    (see PVRSRVDebugRequest).
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+
+@Return         If successful PVRSRV_OK, else a PVRSRV_ERROR.
+*/ /**************************************************************************/
+static PVRSRV_ERROR PhysHeapPrintHeapProperties(PHYS_HEAP *psPhysHeap,
+                                                IMG_BOOL bDefaultHeap,
+                                                DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                                void *pvDumpDebugFile)
+{
+	static const IMG_CHAR *const pszTypeStrings[] = {
+	#define X(_name) #_name,
+		PHYS_HEAP_TYPE_LIST
+	#undef X
+	};
+
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	IMG_CHAR pszUsageString[127] = "\0";
+	PVRSRV_ERROR eError;
+
+	if (psPhysHeap->eType >= ARRAY_SIZE(pszTypeStrings))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PhysHeap at address %p eType is not a PHYS_HEAP_TYPE",
+		         psPhysHeap));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAPINFO, failure);
+	}
+
+	psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+	                                               &ui64TotalSize,
+	                                               &ui64FreeSize);
+
+	eError = GetPhysHeapUsageString(psPhysHeap->ui32UsageFlags,
+	                                sizeof(pszUsageString),
+	                                pszUsageString);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GetPhysHeapUsageString", failure);
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC ", Spas Region: %p"
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap) (psPhysHeap->psSpasRegion)
+#else
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC "%s"
+#define PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap) ""
+#endif
+
+	if ((psPhysHeap->eType == PHYS_HEAP_TYPE_LMA) ||
+	    (psPhysHeap->eType == PHYS_HEAP_TYPE_DLM))
+	{
+		IMG_CPU_PHYADDR sCPUPAddr;
+		IMG_DEV_PHYADDR sGPUPAddr;
+
+		PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetCPUPAddr != NULL);
+		PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetDevPAddr != NULL);
+
+		eError = psPhysHeap->psImplFuncs->pfnGetCPUPAddr(psPhysHeap->pvImplData,
+		                                                 &sCPUPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "pfnGetCPUPAddr");
+			sCPUPAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(IMG_UINT64_MAX);
+		}
+
+		eError = psPhysHeap->psImplFuncs->pfnGetDevPAddr(psPhysHeap->pvImplData,
+		                                                 &sGPUPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "pfnGetDevPAddr");
+			sGPUPAddr.uiAddr = IMG_UINT64_MAX;
+		}
+
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "CPU PA Base: " CPUPHYADDR_UINT_FMTSPEC", "
+		                  "GPU PA Base: 0x%08"IMG_UINT64_FMTSPECx", "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B"
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC,
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  CPUPHYADDR_FMTARG(sCPUPAddr.uiAddr),
+		                  sGPUPAddr.uiAddr,
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize,
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap));
+	}
+	else if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+	{
+		IMG_CHAR pszSpanString[128] = "\0";
+		void *pvIterHandle = NULL;
+
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B Spans:",
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize);
+
+		while (psPhysHeap->psImplFuncs->pfnGetHeapSpansStringIter(psPhysHeap->pvImplData,
+		                                                                   pszSpanString,
+		                                                                   sizeof(pszSpanString),
+		                                                                   &pvIterHandle))
+		{
+			PVR_DUMPDEBUG_LOG("%s", pszSpanString);
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B"
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC,
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize,
+		                  PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG(psPhysHeap));
+	}
+
+#undef PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTSPEC
+#undef PHYS_HEAP_DISPLAY_PROPS_SPAS_GROUP_FMTARG
+
+	return PVRSRV_OK;
+
+failure:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDebugRequest
+@Description    This function is used to output debug information for a given
+                device's PhysHeaps.
+@Input          pfnDbgRequestHandle Data required by this function that is
+                                    passed through the RegisterDeviceDbgRequestNotify
+                                    function.
+@Input          ui32VerbLevel       The maximum verbosity of the debug request.
+@Input          pfnDumpDebugPrintf  The specified print function that should be
+                                    used to dump any debug information
+                                    (see PVRSRVDebugRequest).
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+@Return         void
+*/ /**************************************************************************/
+static void PhysHeapDebugRequest(PVRSRV_DBGREQ_HANDLE pfnDbgRequestHandle,
+                                  IMG_UINT32 ui32VerbLevel,
+                                  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                  void *pvDumpDebugFile)
+{
+	PPVRSRV_DEVICE_NODE psDeviceNode = (PPVRSRV_DEVICE_NODE)pfnDbgRequestHandle;
+	PHYS_HEAP *psPhysHeap;
+
+	PVR_UNREFERENCED_PARAMETER(ui32VerbLevel);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDeviceNode != NULL,
+	                             "Phys Heap debug request failed. psDeviceNode was NULL");
+
+	PVR_DUMPDEBUG_LOG("------[ Device ID: %d - Phys Heaps ]------",
+	                  psDeviceNode->sDevId.i32KernelDeviceID);
+
+	for (psPhysHeap = psDeviceNode->psPhysHeapList; psPhysHeap != NULL; psPhysHeap = psPhysHeap->psNext)
+	{
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_BOOL bDefaultHeap = psPhysHeap == psDeviceNode->apsPhysHeap[psDeviceNode->psDevConfig->eDefaultHeap];
+
+		eError = PhysHeapPrintHeapProperties(psPhysHeap,
+		                                     bDefaultHeap,
+		                                     pfnDumpDebugPrintf,
+		                                     pvDumpDebugFile);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "PhysHeapCreateProperties");
+			continue;
+		}
+	}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	OSLockAcquire(psDeviceNode->hPMRZombieListLock);
+	PVR_DUMPDEBUG_LOG("PMR Zombie Count: %u, PMR Zombie Count In Cleanup: %u",
+	                  psDeviceNode->uiPMRZombieCount,
+	                  psDeviceNode->uiPMRZombieCountInCleanup);
+	OSLockRelease(psDeviceNode->hPMRZombieListLock);
+#endif
+	PVR_DUMPDEBUG_LOG("PMR Live Count: %d", PMRGetLiveCount());
+}
+
+/*************************************************************************/ /*!
+@Function       HeapCfgUsedByPVRLayer
+@Description    Checks if a physheap config must be handled by the PVR Layer
+@Input          psConfig PhysHeapConfig
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+static IMG_BOOL HeapCfgUsedByPVRLayer(PHYS_HEAP_CONFIG *psConfig)
+{
+	PVRSRV_PHYS_HEAP eHeap;
+	IMG_BOOL bPVRHeap = IMG_FALSE;
+
+	/* Heaps are triaged for initialisation by either
+	 * the PVR Layer or the device-specific heap handler. */
+	for (eHeap = PVRSRV_PHYS_HEAP_DEFAULT;
+		 eHeap < PVRSRV_PHYS_HEAP_LAST;
+		 eHeap++)
+	{
+		if ((BIT_ISSET(psConfig->ui32UsageFlags, eHeap) &&
+		     PhysHeapCreatedByPVRLayer(eHeap)))
+		{
+			bPVRHeap = IMG_TRUE;
+			break;
+		}
+	}
+
+	return bPVRHeap;
+}
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateDLMIMAHeapsFromConfig
+@Description    Create new heaps for a device from DLM and IMA configs.
+                This function will both create and construct the link
+                between them.
+@Input          psDevNode      Pointer to device node struct
+@Input          pasConfigs     Pointer to array of Heap configurations.
+@Input          ui32NumConfigs Number of configurations in array.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PhysHeapCreateDLMIMAHeapsFromConfig(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PHYS_HEAP_CONFIG *pasConfigs,
+                                    IMG_UINT32 ui32NumConfigs)
+{
+	/* The DLM heaps must be created before IMA heaps and the config order is not well-defined.
+	 * So for each DLM heap create it and then create the IMA heaps associated. */
+
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiDLMIdx, uiIMAIdx;
+	PHYS_HEAP_POLICY uiIMAPolicy = OSIsMapPhysNonContigSupported() ? PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG : PHYS_HEAP_POLICY_DEFAULT;
+	PHYS_HEAP *psDLMHeap;
+
+	/* Iterate and create the DLM heaps */
+	for (uiDLMIdx = 0; uiDLMIdx < ui32NumConfigs; uiDLMIdx++)
+	{
+		if (pasConfigs[uiDLMIdx].eType == PHYS_HEAP_TYPE_DLM)
+		{
+			IMG_UINT32 uiHeapCount = 0;
+			eError = PhysmemCreateHeapDLM(psDevNode,
+			                               PHYS_HEAP_POLICY_DEFAULT,
+			                               &pasConfigs[uiDLMIdx],
+			                               pasConfigs[uiDLMIdx].uConfig.sDLM.pszHeapName,
+			                               &psDLMHeap);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeapDLM");
+
+			/* Then iterate and create the IMA heaps linked to this DLM heap */
+			for (uiIMAIdx = 0; uiIMAIdx < ui32NumConfigs; uiIMAIdx++)
+			{
+				if (pasConfigs[uiIMAIdx].eType == PHYS_HEAP_TYPE_IMA &&
+				    pasConfigs[uiIMAIdx].uConfig.sIMA.uiDLMHeapIdx == uiDLMIdx)
+				{
+					PhysmemCreateHeapIMA(psDevNode,
+					                     uiIMAPolicy,
+					                     &pasConfigs[uiIMAIdx],
+					                     pasConfigs[uiIMAIdx].uConfig.sIMA.pszHeapName,
+					                     psDLMHeap,
+					                     pasConfigs[uiDLMIdx].uConfig.sDLM.ui32Log2PMBSize,
+					                     NULL);
+					PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeapIMA");
+					uiHeapCount++;
+				}
+			}
+
+			if (uiHeapCount == 0)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "DLM phys heap config %d: No connected IMA heaps. Phys heap will go unused.", uiDLMIdx));
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+PhysHeapDestroyDLMIMAHeaps(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PHYS_HEAP *psNode = psDevNode->psPhysHeapList;
+
+	/* We need to ensure IMA heaps are destroyed before DLM so that
+	 * we don't cause RA leaks by freeing the DLM first.
+	 */
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+
+		if (psTmp->eType == PHYS_HEAP_TYPE_IMA)
+		{
+			PhysHeapDestroy(psTmp);
+		}
+	}
+
+	/* Reset the loop */
+	psNode = psDevNode->psPhysHeapList;
+
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+
+		if (psTmp->eType == PHYS_HEAP_TYPE_DLM)
+		{
+			PhysHeapDestroy(psTmp);
+		}
+	}
+}
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateDeviceHeapsFromConfigs
+@Description    Create new heaps for a device from configs.
+@Input          psDevNode      Pointer to device node struct
+@Input          pasConfigs     Pointer to array of Heap configurations.
+@Input          ui32NumConfigs Number of configurations in array.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PhysHeapCreateDeviceHeapsFromConfigs(PPVRSRV_DEVICE_NODE psDevNode,
+                                     PHYS_HEAP_CONFIG *pasConfigs,
+                                     IMG_UINT32 ui32NumConfigs)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	psDevNode->psPhysHeapList = NULL;
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+	/* DLM/IMA heaps must be constructed in a specific order */
+	eError = PhysHeapCreateDLMIMAHeapsFromConfig(psDevNode,
+	                                             pasConfigs,
+	                                             ui32NumConfigs);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateDLMIMAHeapsFromConfig");
+#endif
+
+
+	for (i = 0; i < ui32NumConfigs; i++)
+	{
+		/* A PhysHeapConfig can have multiple usage flags. If any flag in a
+		 * heap's set points to a heap type that is handled by the PVR Layer
+		 * then we assume that a single heap is shared between multiple
+		 * allocators and it is safe to instantiate it here. If the heap
+		 * is not marked to be initialised by the PVR Layer, leave it
+		 * to the device specific handler.
+		 * DLM Heaps have usage flags but don't have element in the
+		 * fallback table, this check will prevent instantiate them twice.
+		 */
+		if (HeapCfgUsedByPVRLayer(&pasConfigs[i]))
+		{
+			eError = PhysHeapCreateHeapFromConfig(psDevNode, &pasConfigs[i], NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+		}
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST)
+	/* For a temporary device node there will never be a debug dump
+	 * request targeting it */
+	if (psDevNode->hDebugTable != NULL)
+#endif
+	{
+		eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hPhysHeapDbgReqNotify,
+		                                              psDevNode,
+		                                              PhysHeapDebugRequest,
+		                                              DEBUG_REQUEST_SYS,
+		                                              psDevNode);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify");
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PhysHeapCreateHeapFromConfig(PVRSRV_DEVICE_NODE *psDevNode,
+							 PHYS_HEAP_CONFIG *psConfig,
+							 PHYS_HEAP **ppsPhysHeap)
+{
+	PVRSRV_ERROR eResult;
+
+	if (psConfig->eType == PHYS_HEAP_TYPE_UMA
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+		|| psConfig->eType == PHYS_HEAP_TYPE_WRAP
+#endif
+		)
+	{
+		eResult = PhysmemCreateHeapOSMEM(psDevNode,
+		                                 PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG,
+		                                 psConfig,
+		                                 ppsPhysHeap);
+	}
+	else if ((psConfig->eType == PHYS_HEAP_TYPE_LMA) ||
+	         (psConfig->eType == PHYS_HEAP_TYPE_DMA))
+	{
+		PHYS_HEAP_POLICY uiHeapPolicy;
+
+		if (psDevNode->pfnPhysHeapGetLMAPolicy != NULL)
+		{
+			uiHeapPolicy = psDevNode->pfnPhysHeapGetLMAPolicy(psConfig->ui32UsageFlags, psDevNode);
+		}
+		else
+		{
+			uiHeapPolicy = OSIsMapPhysNonContigSupported() ?
+			               PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG :
+			               PHYS_HEAP_POLICY_DEFAULT;
+		}
+
+		eResult = PhysmemCreateHeapLMA(psDevNode,
+		                               uiHeapPolicy,
+		                               psConfig,
+		                               (psConfig->eType == PHYS_HEAP_TYPE_LMA) ?
+		                                "GPU LMA (Sys)" :
+		                                "GPU LMA DMA (Sys)",
+		                               ppsPhysHeap);
+	}
+	else if ((psConfig->eType == PHYS_HEAP_TYPE_DLM) ||
+	         (psConfig->eType == PHYS_HEAP_TYPE_IMA))
+	{
+		/* These heaps have already been instantiated in
+		 * PhysHeapCreateDLMIMAHeapsFromConfig
+		 */
+		eResult = PVRSRV_OK;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s Invalid phys heap type: %d",
+				 __func__, psConfig->eType));
+		eResult = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eResult;
+}
+
+#define PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE (IMG_UINT64_C(0x100000) * IMG_UINT64_C(32)) /* 32MB */
+
+static PVRSRV_ERROR PVRSRVValidatePhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_UINT32 ui32FlagsAccumulate = 0;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_FALSE(psDevConfig->ui32PhysHeapCount > 0,
+							"Device config must specify at least one phys heap config.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		/* Flags that may be used by multiple heaps. */
+		const PHYS_HEAP_USAGE_FLAGS uiDuplicateFlags = PHYS_HEAP_USAGE_DLM;
+		PHYS_HEAP_CONFIG *psHeapConf = &psDevConfig->pasPhysHeaps[i];
+
+		PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->ui32UsageFlags != 0,
+		                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+		                           "Phys heap config %d: must specify usage flags.", i);
+
+		PVR_LOG_RETURN_IF_FALSE_VA(((ui32FlagsAccumulate & ~uiDuplicateFlags) & psHeapConf->ui32UsageFlags) == 0,
+		                             PVRSRV_ERROR_PHYSHEAP_CONFIG,
+		                             "Phys heap config %d: duplicate usage flags.", i);
+
+		ui32FlagsAccumulate |= psHeapConf->ui32UsageFlags;
+
+		if (BITMASK_ANY((1U << psDevConfig->eDefaultHeap), PHYS_HEAP_USAGE_MASK) &&
+		    BITMASK_ANY((1U << psDevConfig->eDefaultHeap), psHeapConf->ui32UsageFlags))
+		{
+			switch (psHeapConf->eType)
+			{
+#if defined(__KERNEL__)
+				case PHYS_HEAP_TYPE_DMA:
+#endif
+				case PHYS_HEAP_TYPE_LMA:
+				{
+					if (psHeapConf->uConfig.sLMA.uiSize < PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE)
+					{
+						/* Output message if default heap is LMA and smaller than recommended minimum. */
+						PVR_DPF((PVR_DBG_ERROR, "%s: Size of default heap is 0x%" IMG_UINT64_FMTSPECX
+						         " (recommended minimum heap size is 0x%" IMG_UINT64_FMTSPECX ")",
+						         __func__, psHeapConf->uConfig.sLMA.uiSize,
+						         PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE));
+					}
+					break;
+				}
+				case PHYS_HEAP_TYPE_IMA:
+				{
+					/* Check if chained DLM has more than default LMA size.
+					 * This is not perfect as other connected IMAs could reserve all the memory. */
+					IMG_UINT64 uiSize = psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].uConfig.sDLM.uiSize;
+
+					if (uiSize < PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Default heap has access to 0x%" IMG_UINT64_FMTSPECX
+						         " (recommended minimum heap size is 0x%" IMG_UINT64_FMTSPECX ")",
+						         __func__, uiSize,
+						         PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE));
+					}
+					break;
+				}
+				case PHYS_HEAP_TYPE_DLM:
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Phys heap config %d: "
+					         "A DLM heap cannot be the default heap.",
+					         __func__, i));
+					return PVRSRV_ERROR_PHYSHEAP_CONFIG;
+				}
+				default:
+					break;
+			}
+		}
+
+#if !defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+		if ((psHeapConf->eType == PHYS_HEAP_TYPE_IMA) || (psHeapConf->eType == PHYS_HEAP_TYPE_DLM))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Phys heap config %d: "
+			         "Dynamic phys heaps are not supported! "
+			         "Please enable PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS",
+			         __func__, i));
+			return PVRSRV_ERROR_PHYSHEAP_CONFIG;
+		}
+#else /* PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS */
+		if (psHeapConf->eType == PHYS_HEAP_TYPE_DLM)
+		{
+			PVR_LOG_RETURN_IF_FALSE_VA((psHeapConf->ui32UsageFlags & PHYS_HEAP_USAGE_DLM) == PHYS_HEAP_USAGE_DLM,
+			                            PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                            "Phys heap config %d: DLM heap must specify the DLM usage flag.", i);
+
+			/* If a DLM heap has less space than a PMB, it cannot export a single PMB. The PMB size should be lowered. */
+			PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->uConfig.sDLM.uiSize >= psHeapConf->uConfig.sDLM.ui32Log2PMBSize,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: Size of DLM heap is 0x%" IMG_UINT64_FMTSPECX
+			                           " but the PMB size is 0x%" IMG_UINT64_FMTSPECX
+			                           ". The total size must be greater than or equal to the PMB size.",
+			                           i, psHeapConf->uConfig.sDLM.uiSize,
+			                           IMG_UINT64_C(1) << psHeapConf->uConfig.sDLM.ui32Log2PMBSize);
+		}
+
+		/* IMA heaps must point to a DLM heap. */
+		if (psHeapConf->eType == PHYS_HEAP_TYPE_IMA)
+		{
+			PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->uConfig.sIMA.uiDLMHeapIdx < psDevConfig->ui32PhysHeapCount,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: IMA heap is trying to link to a DLM heap out of bounds. "
+			                           "Requested Heap Index: %d, Heap Array Count: %d",
+			                           i, psHeapConf->uConfig.sIMA.uiDLMHeapIdx, psDevConfig->ui32PhysHeapCount);
+			PVR_LOG_RETURN_IF_FALSE_VA(psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].eType == PHYS_HEAP_TYPE_DLM,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: IMA heap is trying to link to a NON-DLM heap type. "
+			                           "Requested Heap Idx: %d, Heap Type: %d",
+			                           i, psHeapConf->uConfig.sIMA.uiDLMHeapIdx,
+			                           psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].eType);
+		}
+#endif /* PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS */
+
+	}
+
+
+	if (psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_GPU_LOCAL)
+	{
+		PVR_LOG_RETURN_IF_FALSE(((ui32FlagsAccumulate & PHYS_HEAP_USAGE_GPU_LOCAL) != 0) ,
+							"Device config must specify GPU local phys heap config.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+	else if (psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_CPU_LOCAL)
+	{
+		PVR_LOG_RETURN_IF_FALSE(((ui32FlagsAccumulate & PHYS_HEAP_USAGE_CPU_LOCAL) != 0) ,
+						"Device config must specify CPU local phys heap config.",
+						PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+static void DestroyGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       CreateGpuVirtValArenas
+@Description    Create virtualization validation arenas
+@Input          psDeviceNode The device node
+@Return         PVRSRV_ERROR PVRSRV_OK on success
+*/ /**************************************************************************/
+static PVRSRV_ERROR CreateGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* aui64OSidMin and aui64OSidMax are what we program into HW registers.
+	   The values are different from base/size of arenas. */
+	IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS];
+	IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS];
+	PHYS_HEAP_CONFIG *psGPULocalHeap = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig, PHYS_HEAP_USAGE_GPU_LOCAL);
+	IMG_DEV_PHYADDR sGPULocalCardBase = PhysHeapConfigGetCardBase(psGPULocalHeap);
+	IMG_UINT64 uiGPULocalSize = PhysHeapConfigGetSize(psGPULocalHeap);
+	PHYS_HEAP_CONFIG *psDisplayHeap = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig, PHYS_HEAP_USAGE_DISPLAY);
+
+	IMG_UINT64 uPrivateRABase;
+	IMG_UINT64 uPrivateRASize;
+	IMG_UINT64 uSharedRABase;
+	IMG_UINT64 uSharedRASize;
+	IMG_UINT64 uSharedRegionBase;
+	IMG_UINT64 uSharedRegionSize;
+	IMG_UINT32 i;
+
+	//PVR_LOG_RETURN_IF_FALSE((psDeviceNode->psOSSharedArena == NULL), "Validation RAs already created.", PVRSRV_OK);
+
+	/* Shared region is fixed size, the remaining space is divided amongst OSes */
+	uPrivateRASize = uiGPULocalSize - PVR_ALIGN(GPUVIRT_SIZEOF_SHARED, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+	uPrivateRASize /= GPUVIRT_VALIDATION_NUM_OS;
+	uPrivateRASize = uPrivateRASize & ~((IMG_UINT64)OSGetPageSize() - 1ULL); /* Align, round down */
+	uSharedRASize = uiGPULocalSize - uPrivateRASize * GPUVIRT_VALIDATION_NUM_OS;
+
+	PVR_LOG(("GPUVIRT_VALIDATION: GPU_LOCAL heap base (base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ")"
+			 " split into %u private regions of size 0x%" IMG_UINT64_FMTSPECX " and one shared region of size 0x%" IMG_UINT64_FMTSPECX " ",
+			 sGPULocalCardBase.uiAddr, uiGPULocalSize, GPUVIRT_VALIDATION_NUM_OS, uPrivateRASize, uSharedRASize));
+
+	/* If a display heap config exists, include the display heap in the non-secure regions */
+	if (psDisplayHeap)
+	{
+		IMG_DEV_PHYADDR sGPUDisplayCardBase = PhysHeapConfigGetCardBase(psDisplayHeap);
+		IMG_UINT64 uiGPUDisplaySize = PhysHeapConfigGetSize(psDisplayHeap);
+
+		PVR_LOG(("GPUVIRT_VALIDATION: DISPLAY heap (base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ") merged into shared region.",
+		         sGPUDisplayCardBase.uiAddr,
+		         uiGPUDisplaySize));
+
+		if (sGPUDisplayCardBase.uiAddr > sGPULocalCardBase.uiAddr)
+		{
+			/*
+			 * ---------------------------------------------------------------------------------------------------
+			 * |        .        .        .     GPU LOCAL HEAP       .        .        .        |  DISPLAY HEAP  |
+			 * ---------------------------------------------------------------------------------------------------
+			 *  \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \_______________/
+			 *   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7   SHARED      DISPLAY
+			 *                                                                            \____SHARED REGION____/
+			 *  OSID 0 has full access over everything
+			 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+			 */
+			if ((sGPULocalCardBase.uiAddr + uiGPULocalSize) != sGPUDisplayCardBase.uiAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: GPU Local heap and display heap must be adjacent in physical memory.", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			uPrivateRABase = sGPULocalCardBase.uiAddr;
+			uSharedRABase = sGPULocalCardBase.uiAddr + uiGPULocalSize - uSharedRASize;
+			uSharedRegionBase = uSharedRABase;
+		}
+		else
+		{
+			/*
+			 * ---------------------------------------------------------------------------------------------------
+			 * |  DISPLAY HEAP   |        .        .        .     GPU LOCAL HEAP       .        .        .       |
+			 * ---------------------------------------------------------------------------------------------------
+			 *  \_______________/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/
+			 *        DISPLAY      SHARED   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7
+			 *   \____SHARED REGION____/
+			 *
+			 *  OSID 0 has full access over everything
+			 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+			 */
+			if ((sGPUDisplayCardBase.uiAddr + uiGPUDisplaySize) != sGPULocalCardBase.uiAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: GPU Local heap and display heap must be adjacent in physical memory.", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			uSharedRABase = sGPULocalCardBase.uiAddr;
+			uPrivateRABase = sGPULocalCardBase.uiAddr + uSharedRASize;
+			uSharedRegionBase = sGPUDisplayCardBase.uiAddr;
+		}
+
+		uSharedRegionSize = uSharedRASize + uiGPUDisplaySize;
+	}
+	else
+	{
+		/*
+		 * ----------------------------------------------------------------------------------
+		 * |        .        .        .     GPU LOCAL HEAP       .        .        .        |
+		 * ----------------------------------------------------------------------------------
+		 *  \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/
+		 *   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7   SHARED
+		 *
+		 *  OSID 0 has full access over everything
+		 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+		 */
+		uPrivateRABase = sGPULocalCardBase.uiAddr;
+		uSharedRABase = uPrivateRABase + uPrivateRASize * GPUVIRT_VALIDATION_NUM_OS;
+		uSharedRASize = uiGPULocalSize - (uSharedRABase - uPrivateRABase);
+		uSharedRegionSize = uSharedRASize;
+		uSharedRegionBase = uSharedRABase;
+	}
+
+	if (uPrivateRASize < GPUVIRT_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Per-OSID private memory regions are too small (current: 0x%" IMG_UINT64_FMTSPECX ""
+								" required: 0x%" IMG_UINT64_FMTSPECX "). Increase GPU Local heap size.",
+								__func__, uPrivateRASize, GPUVIRT_MIN_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uSharedRASize < GPUVIRT_SIZEOF_SHARED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Memory region shared between all OSIDs is too small (current: 0x%" IMG_UINT64_FMTSPECX ""
+								" required: 0x%" IMG_UINT64_FMTSPECX "). Increase GPU Local heap size.",
+								__func__, uSharedRASize, GPUVIRT_SIZEOF_SHARED));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	FOREACH_VALIDATION_OSID(i)
+	{
+		IMG_CHAR aszOSRAName[RA_MAX_NAME_LENGTH];
+
+		PVR_LOG(("GPUVIRT_VALIDATION: create arena Private OSID: %u, base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ".", i, uPrivateRABase, uPrivateRASize));
+
+		OSSNPrintf(aszOSRAName, RA_MAX_NAME_LENGTH, "GPUVIRT_OS%u", i);
+
+		psDeviceNode->psOSidSubArena[i] = RA_Create_With_Span(aszOSRAName,
+		                                                      OSGetPageShift(),
+		                                                      0,
+		                                                      uPrivateRABase,
+		                                                      uPrivateRASize,
+		                                                      RA_POLICY_DEFAULT);
+		PVR_GOTO_IF_FALSE(psDeviceNode->psOSidSubArena[i] != NULL, FreeArenas);
+		aui64OSidMin[GPUVIRT_VAL_REGION_SECURE][i] = uPrivateRABase;
+
+		if (i == 0)
+		{
+			/* Host OSID0 has access to all regions */
+			aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i] = uiGPULocalSize - 1ULL;
+		}
+		else
+		{
+			/* Guest OSIDs are limited to their private range */
+			aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i] = uPrivateRABase + uPrivateRASize - 1ULL;
+		}
+
+		/* uSharedRegionSize includes display heap */
+		aui64OSidMin[GPUVIRT_VAL_REGION_SHARED][i] = uSharedRegionBase;
+		aui64OSidMax[GPUVIRT_VAL_REGION_SHARED][i] = uSharedRegionBase + uSharedRegionSize - 1ULL;
+
+		uPrivateRABase += uPrivateRASize;
+	}
+
+	PVR_LOG(("GPUVIRT_VALIDATION: create arena Shared, base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ".", uSharedRABase, uSharedRASize));
+
+	/* uSharedRASize does not include display heap */
+	psDeviceNode->psOSSharedArena = RA_Create_With_Span("GPUVIRT_SHARED",
+	                                                    OSGetPageShift(),
+	                                                    0,
+	                                                    uSharedRABase,
+	                                                    uSharedRASize,
+	                                                    RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_FALSE(psDeviceNode->psOSSharedArena != NULL, FreeArenas);
+
+	FOREACH_VALIDATION_OSID(i)
+	{
+		PVR_LOG(("GPUVIRT_VALIDATION: HW regions OSID %u: Private Region = [0x%" IMG_UINT64_FMTSPECX "..0x%" IMG_UINT64_FMTSPECX "]"
+														" Shared Region = [0x%" IMG_UINT64_FMTSPECX "..0x%" IMG_UINT64_FMTSPECX "]",
+				 i,
+				 aui64OSidMin[GPUVIRT_VAL_REGION_SECURE][i],
+				 aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i],
+				 aui64OSidMin[GPUVIRT_VAL_REGION_SHARED][i],
+				 aui64OSidMax[GPUVIRT_VAL_REGION_SHARED][i]));
+	}
+
+	if (psDeviceNode->psDevConfig->pfnSysInitFirewall != NULL)
+	{
+		psDeviceNode->psDevConfig->pfnSysInitFirewall(psDeviceNode->psDevConfig->hSysData, aui64OSidMin, aui64OSidMax);
+	}
+
+	return PVRSRV_OK;
+
+FreeArenas:
+	DestroyGpuVirtValArenas(psDeviceNode);
+
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+}
+
+/*
+ * Counter-part to CreateGpuVirtValArenas.
+ */
+static void DestroyGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32	uiCounter = 0;
+
+	FOREACH_VALIDATION_OSID(uiCounter)
+	{
+		if (uiCounter == RGXFW_HOST_DRIVER_ID)
+		{
+			/*
+			 * NOTE: We overload psOSidSubArena[0] into the psLocalMemArena so we must
+			 * not free it here as it gets cleared later.
+			 */
+			continue;
+		}
+
+		if (psDeviceNode->psOSidSubArena[uiCounter] == NULL)
+		{
+			continue;
+		}
+		RA_Delete(psDeviceNode->psOSidSubArena[uiCounter]);
+	}
+
+	if (psDeviceNode->psOSSharedArena != NULL)
+	{
+		RA_Delete(psDeviceNode->psOSSharedArena);
+	}
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       PhysHeapMMUPxSetup
+@Description    Setup MMU Px allocation function pointers.
+@Input          psDeviceNode Pointer to device node struct
+@Return         PVRSRV_ERROR PVRSRV_OK on success.
+*/ /**************************************************************************/
+static PVRSRV_ERROR PhysHeapMMUPxSetup(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PHYS_HEAP_TYPE eHeapType;
+	PVRSRV_ERROR eError;
+
+	eError = PhysHeapAcquireByID(psDeviceNode->psDevConfig->eDefaultHeap,
+	                                      psDeviceNode, &psDeviceNode->psMMUPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorDeinit);
+
+	eHeapType = PhysHeapGetType(psDeviceNode->psMMUPhysHeap);
+
+	if (eHeapType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: GPU physical heap uses OS System memory (UMA)", __func__));
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		PVR_DPF((PVR_DBG_ERROR, "%s: Virtualisation Validation builds are currently only"
+								 " supported on systems with local memory (LMA).", __func__));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto ErrorDeinit;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: GPU physical heap uses local memory managed by the driver (LMA)", __func__));
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		eError = CreateGpuVirtValArenas(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "CreateGpuVirtValArenas", ErrorDeinit);
+#endif
+	}
+
+	return PVRSRV_OK;
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapMMUPxDeInit
+@Description    Deinit after PhysHeapMMUPxSetup.
+@Input          psDeviceNode Pointer to device node struct
+*/ /**************************************************************************/
+static void PhysHeapMMUPxDeInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/* Remove local LMA subarenas */
+	DestroyGpuVirtValArenas(psDeviceNode);
+#endif	/* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+	if (psDeviceNode->psMMUPhysHeap != NULL)
+	{
+		PhysHeapRelease(psDeviceNode->psMMUPhysHeap);
+		psDeviceNode->psMMUPhysHeap = NULL;
+	}
+}
+
+PVRSRV_ERROR PhysHeapInitDeviceHeaps(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP ePhysHeap;
+
+	eError = OSLockCreate(&psDeviceNode->hPhysHeapLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	eError = PVRSRVValidatePhysHeapConfig(psDevConfig);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVValidatePhysHeapConfig");
+
+	eError = PhysHeapCreateDeviceHeapsFromConfigs(psDeviceNode,
+	                                              psDevConfig->pasPhysHeaps,
+	                                              psDevConfig->ui32PhysHeapCount);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreateDeviceHeapsFromConfigs", ErrorDeinit);
+
+	/* Must loop from the 2nd heap to the last */
+	PVR_ASSERT(PVRSRV_PHYS_HEAP_DEFAULT == 0);
+	for (ePhysHeap = (PVRSRV_PHYS_HEAP)(PVRSRV_PHYS_HEAP_DEFAULT+1); ePhysHeap < PVRSRV_PHYS_HEAP_LAST; ePhysHeap++)
+	{
+		if (PhysHeapAcquiredByPVRLayer(ePhysHeap))
+		{
+			eError = PhysHeapAcquireByID(ePhysHeap, psDeviceNode, &psDeviceNode->apsPhysHeap[ePhysHeap]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorDeinit);
+		}
+	}
+
+	if (PhysHeapValidateDefaultHeapExists(psDeviceNode))
+	{
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapValidateDefaultHeapExists", ErrorDeinit);
+	}
+
+	eError = PhysHeapMMUPxSetup(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapMMUPxSetup", ErrorDeinit);
+
+	return PVRSRV_OK;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+
+	return eError;
+}
+
+void PhysHeapDeInitDeviceHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_PHYS_HEAP ePhysHeapIdx;
+	IMG_UINT32 i;
+
+	PhysHeapMMUPxDeInit(psDeviceNode);
+
+	/* Release heaps */
+	for (ePhysHeapIdx = PVRSRV_PHYS_HEAP_DEFAULT;
+		 ePhysHeapIdx < ARRAY_SIZE(psDeviceNode->apsPhysHeap);
+		 ePhysHeapIdx++)
+	{
+		if (psDeviceNode->apsPhysHeap[ePhysHeapIdx])
+		{
+			PhysHeapRelease(psDeviceNode->apsPhysHeap[ePhysHeapIdx]);
+		}
+	}
+
+	FOREACH_SUPPORTED_DRIVER(i)
+	{
+		if (psDeviceNode->apsFWPremapPhysHeap[i])
+		{
+			PhysHeapDestroy(psDeviceNode->apsFWPremapPhysHeap[i]);
+			psDeviceNode->apsFWPremapPhysHeap[i] = NULL;
+		}
+	}
+
+	PhysHeapDestroyDeviceHeaps(psDeviceNode);
+
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+}
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+static void PhysHeapSpasRegionLock(PHYS_HEAP_SPAS_REGION *psSpasRegion)
+{
+	while (OSAtomicCompareExchange(&psSpasRegion->ui32Lock, 0, 1) != 0);
+	OSMemoryBarrier(NULL);
+}
+static void PhysHeapSpasRegionUnlock(PHYS_HEAP_SPAS_REGION *psSpasRegion)
+{
+	OSMemoryBarrier(NULL);
+	OSAtomicWrite(&psSpasRegion->ui32Lock, 0);
+}
+
+/* Insert the physheap into a given SPAS. */
+static void PhysHeapSpasInsert(PHYS_HEAP* psPhysHeap, PHYS_HEAP_SPAS_REGION *psSpasRegion)
+{
+	if (psSpasRegion == NULL)
+		return; /* Not in a SPAS region. */
+	/* A physheap may only be a part of a single SPAS region. */
+	PVR_ASSERT(psPhysHeap->psSpasRegion == NULL);
+
+	psPhysHeap->psSpasRegion = psSpasRegion;
+
+	PhysHeapSpasRegionLock(psSpasRegion);
+	dllist_add_to_head(&psSpasRegion->sListHead, &psPhysHeap->sSpasSibling);
+	PhysHeapSpasRegionUnlock(psSpasRegion);
+}
+
+/* Remove the PhysHeap from its SPAS region if it is in one. */
+static void PhysHeapSpasRemove(PHYS_HEAP *psPhysHeap)
+{
+	PHYS_HEAP_SPAS_REGION *psSpasRegion = psPhysHeap->psSpasRegion;
+
+	if (psSpasRegion == NULL)
+		return; /* Not in a region. */
+
+	PhysHeapSpasRegionLock(psSpasRegion);
+	psPhysHeap->psSpasRegion = NULL;
+	dllist_remove_node(&psPhysHeap->sSpasSibling);
+	PhysHeapSpasRegionUnlock(psSpasRegion);
+
+	psPhysHeap->psSpasRegion = NULL;
+}
+
+PVRSRV_ERROR PhysHeapSpasWithDevice(PHYS_HEAP* psFromPhysHeap,
+                                    PPVRSRV_DEVICE_NODE psToDevNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_FOUND;
+	PDLLIST_NODE psNode, psNextNode;
+
+	if (psFromPhysHeap->psSpasRegion == NULL)
+		return eError;
+
+	PhysHeapSpasRegionLock(psFromPhysHeap->psSpasRegion);
+	dllist_foreach_node(&psFromPhysHeap->psSpasRegion->sListHead, psNode, psNextNode)
+	{
+		PHYS_HEAP *psPhysHeapCursor = IMG_CONTAINER_OF(psNode, PHYS_HEAP, sSpasSibling);
+		if (psPhysHeapCursor->psDevNode == psToDevNode)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_OK, exit_unlock_);
+		}
+	}
+exit_unlock_:
+	PhysHeapSpasRegionUnlock(psFromPhysHeap->psSpasRegion);
+	return eError;
+}
+
+static IMG_UINT64 PhysHeapSpasDeviceBitmap(PHYS_HEAP* psPhysHeap)
+{
+	PDLLIST_NODE psNode, psNextNode;
+	IMG_UINT64 uiDevices = 0;
+
+	static_assert((sizeof(IMG_UINT64) * 8) >= PVRSRV_MAX_DEVICES, "PhysHeapSpasDeviceBitmap() requires updating");
+
+	if (psPhysHeap->psSpasRegion == NULL)
+		return uiDevices;
+
+	PhysHeapSpasRegionLock(psPhysHeap->psSpasRegion);
+	dllist_foreach_node(&psPhysHeap->psSpasRegion->sListHead, psNode, psNextNode)
+	{
+		PHYS_HEAP *psPhysHeapCursor = IMG_CONTAINER_OF(psNode, PHYS_HEAP, sSpasSibling);
+		uiDevices |= IMG_UINT64_C(1) << psPhysHeapCursor->psDevNode->sDevId.ui32InternalID;
+	}
+	PhysHeapSpasRegionUnlock(psPhysHeap->psSpasRegion);
+	return uiDevices;
+}
+#endif
+
+PVRSRV_ERROR PhysHeapCreate(PPVRSRV_DEVICE_NODE psDevNode,
+							  PHYS_HEAP_CONFIG *psConfig,
+							  PHYS_HEAP_POLICY uiPolicy,
+							  PHEAP_IMPL_DATA pvImplData,
+							  PHEAP_IMPL_FUNCS *psImplFuncs,
+							  PHYS_HEAP **ppsPhysHeap)
+{
+	PHYS_HEAP *psNew;
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	if (psConfig->eType == PHYS_HEAP_TYPE_UNKNOWN)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psImplFuncs != NULL, "psImplFuncs");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psImplFuncs->pfnCreatePMR != NULL ||
+	                                psImplFuncs->pfnCreatePMB,
+	                                "psImplFuncs->pfnCreatePMR || psImplFuncs->pfnCreatePMB");
+
+	psNew = OSAllocMem(sizeof(PHYS_HEAP));
+	PVR_RETURN_IF_NOMEM(psNew);
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	psNew->psSpasRegion = NULL;
+#endif
+
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		psNew->psMemFuncs = psConfig->uConfig.sLMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sLMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sLMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sLMA.pszHeapName) ? psConfig->uConfig.sLMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+#if defined(PVRSRV_ENABLE_XD_MEM)
+		PhysHeapSpasInsert(psNew, psConfig->uConfig.sLMA.psSpasRegion);
+#endif
+		break;
+	case PHYS_HEAP_TYPE_IMA:
+		psNew->psMemFuncs = psConfig->uConfig.sIMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sIMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sIMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sIMA.pszHeapName) ? psConfig->uConfig.sIMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_DMA:
+		psNew->psMemFuncs = psConfig->uConfig.sDMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sDMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sDMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sDMA.pszHeapName) ? psConfig->uConfig.sDMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_UMA:
+		psNew->psMemFuncs = psConfig->uConfig.sUMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sUMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sUMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sUMA.pszHeapName) ? psConfig->uConfig.sUMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+#if defined(PVRSRV_ENABLE_XD_MEM)
+		PhysHeapSpasInsert(psNew, psConfig->uConfig.sUMA.psSpasRegion);
+#endif
+		break;
+	case PHYS_HEAP_TYPE_DLM:
+		psNew->psMemFuncs = psConfig->uConfig.sDLM.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sDLM.hPrivData;
+		psNew->pszPDumpMemspaceName = "None";
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sDLM.pszHeapName) ? psConfig->uConfig.sDLM.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+	case PHYS_HEAP_TYPE_WRAP:
+		psNew->psMemFuncs = psConfig->uConfig.sWRAP.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sWRAP.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sWRAP.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sWRAP.pszHeapName) ? psConfig->uConfig.sWRAP.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+#endif
+	default:
+		PVR_LOG_ERROR(PVRSRV_ERROR_NOT_IMPLEMENTED, "psConfig->eType not implemented");
+	}
+
+	psNew->eType = psConfig->eType;
+	psNew->ui32UsageFlags = psConfig->ui32UsageFlags;
+	psNew->uiPolicy = uiPolicy;
+	psNew->ui32RefCount = 0;
+	psNew->psDevNode = psDevNode;
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	OSAtomicWrite(&psNew->sOOMDetected, IMG_FALSE);
+#endif
+	psNew->pvImplData = pvImplData;
+	psNew->psImplFuncs = psImplFuncs;
+
+#if defined(SUPPORT_STATIC_IPA)
+	{
+		IMG_UINT8  ui8Val;
+
+		/* Ensure we do not cause an address fault by accessing beyond
+		 * the end of the psConfig->sIPAConfig structure.
+		 */
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyDefault;
+		psNew->ui32IPAPolicyValue = (IMG_UINT32)ui8Val;
+
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyMask;
+		psNew->ui32IPAClearMask = (IMG_UINT32)ui8Val;
+
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyShift;
+		psNew->ui32IPAShift = (IMG_UINT32)ui8Val;
+		PVR_LOG_VA(PVR_DBG_MESSAGE, "%s: Physheap <%p> ['%s'] Config @ <%p> IPA = [0x%x, 0x%x, 0x%x]",
+		           __func__, psNew, psNew->aszName,
+		           psConfig, psNew->ui32IPAPolicyValue,
+		           psNew->ui32IPAClearMask, psNew->ui32IPAShift);
+	}
+#endif
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psNew;
+	}
+
+	psNew->psNext = psDevNode->psPhysHeapList;
+	psDevNode->psPhysHeapList = psNew;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNew);
+}
+
+void PhysHeapDestroyDeviceHeaps(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psNode;
+
+	if (psDevNode->hPhysHeapDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hPhysHeapDbgReqNotify);
+	}
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+	/* DLM/IMA heaps must be destroyed in a specific order */
+	PhysHeapDestroyDLMIMAHeaps(psDevNode);
+#endif
+
+	psNode = psDevNode->psPhysHeapList;
+
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+		PhysHeapDestroy(psTmp);
+	}
+}
+
+void PhysHeapDestroy(PHYS_HEAP *psPhysHeap)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PPVRSRV_DEVICE_NODE psDevNode = psPhysHeap->psDevNode;
+
+	PVR_DPF_ENTERED1(psPhysHeap);
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK)
+#endif
+	{
+		PVR_ASSERT(psPhysHeap->ui32RefCount == 0);
+	}
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	PhysHeapSpasRemove(psPhysHeap);
+#endif
+
+	if (psDevNode->psPhysHeapList == psPhysHeap)
+	{
+		psDevNode->psPhysHeapList = psPhysHeap->psNext;
+	}
+	else
+	{
+		PHYS_HEAP *psTmp = psDevNode->psPhysHeapList;
+
+		while (psTmp->psNext != psPhysHeap)
+		{
+			psTmp = psTmp->psNext;
+		}
+		psTmp->psNext = psPhysHeap->psNext;
+	}
+
+	if (psImplFuncs->pfnDestroyData != NULL)
+	{
+		psImplFuncs->pfnDestroyData(psPhysHeap->pvImplData);
+	}
+
+	OSFreeMem(psPhysHeap);
+
+	PVR_DPF_RETURN;
+}
+
+static void _PhysHeapCountUserModeHeaps(PPVRSRV_DEVICE_NODE psDevNode,
+										PHYS_HEAP_USAGE_FLAGS ui32UsageFlags)
+{
+	PVRSRV_PHYS_HEAP eHeap;
+
+	for (eHeap = PVRSRV_PHYS_HEAP_DEFAULT;
+		 eHeap <= PVRSRV_PHYS_HEAP_LAST;
+		 eHeap++)
+	{
+		if (BIT_ISSET(ui32UsageFlags, eHeap) &&
+			PhysHeapUserModeAlloc(eHeap))
+		{
+			psDevNode->ui32UserAllocHeapCount++;
+			break;
+		}
+	}
+}
+
+PVRSRV_ERROR PhysHeapAcquire(PHYS_HEAP *psPhysHeap)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+
+	psPhysHeap->ui32RefCount++;
+
+	/* When acquiring a heap for the 1st time, perform a check and
+	 * calculate the total number of user accessible physical heaps */
+	if (psPhysHeap->ui32RefCount == 1)
+	{
+		_PhysHeapCountUserModeHeaps(psPhysHeap->psDevNode,
+									psPhysHeap->ui32UsageFlags);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PHYS_HEAP * _PhysHeapFindHeapOrFallback(PVRSRV_PHYS_HEAP ePhysHeap,
+								   PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psPhysHeapNode = psDevNode->psPhysHeapList;
+	PVRSRV_PHYS_HEAP eFallback;
+
+	/* Swap the default heap alias for the system's real default PhysHeap */
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	/* Check cache of PhysHeaps to see if it has been resolved before */
+	if (psDevNode->apsPhysHeap[ePhysHeap] != NULL)
+	{
+		return psDevNode->apsPhysHeap[ePhysHeap];
+	}
+
+	/* Cache not ready, carry out search with fallback */
+	while (psPhysHeapNode)
+	{
+		if (BIT_ISSET(psPhysHeapNode->ui32UsageFlags, ePhysHeap))
+		{
+			return psPhysHeapNode;
+		}
+
+		psPhysHeapNode = psPhysHeapNode->psNext;
+	}
+
+	/* Find fallback PhysHeap */
+	eFallback = gasHeapProperties[ePhysHeap].eFallbackHeap;
+	if (ePhysHeap == eFallback)
+	{
+		return NULL;
+	}
+	else
+	{
+		return _PhysHeapFindHeapOrFallback(eFallback, psDevNode);
+	}
+}
+
+/*
+ * Acquire heap, no fallback, no recursion: single loop acquisition
+ */
+static PHYS_HEAP* _PhysHeapFindRealHeapNoFallback(PVRSRV_PHYS_HEAP ePhysHeap,
+                                          PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psPhysHeapNode = psDevNode->psPhysHeapList;
+
+	/* Swap the default heap alias for the system's real default PhysHeap */
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	/* Check cache of PhysHeaps to see if it has been resolved before.
+	 * The physheap must initialised to access its cache. */
+	if (psDevNode->apsPhysHeap[ePhysHeap] != NULL &&
+	    BIT_ISSET(psDevNode->apsPhysHeap[ePhysHeap]->ui32UsageFlags, ePhysHeap))
+	{
+		return psDevNode->apsPhysHeap[ePhysHeap];
+	}
+
+	/* Cache not ready, carry out search for real PhysHeap, no fallback */
+	while (psPhysHeapNode)
+	{
+		if (BIT_ISSET(psPhysHeapNode->ui32UsageFlags, ePhysHeap))
+		{
+			return psPhysHeapNode;
+		}
+
+		psPhysHeapNode = psPhysHeapNode->psNext;
+	}
+	return NULL;
+}
+
+PVRSRV_ERROR PhysHeapAcquireByID(PVRSRV_PHYS_HEAP eDevPhysHeap,
+								 PPVRSRV_DEVICE_NODE psDevNode,
+								 PHYS_HEAP **ppsPhysHeap)
+{
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(eDevPhysHeap < PVRSRV_PHYS_HEAP_LAST, "eDevPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	PVR_DPF_ENTERED1(ui32Flags);
+
+	OSLockAcquire(psDevNode->hPhysHeapLock);
+
+	psPhysHeap = _PhysHeapFindHeapOrFallback(eDevPhysHeap, psDevNode);
+
+	if (psPhysHeap != NULL)
+	{
+		psPhysHeap->ui32RefCount++;
+		PHYSHEAP_REFCOUNT_PRINT("%s: Heap %p, refcount = %d",
+								__func__, psPhysHeap, psPhysHeap->ui32RefCount);
+
+		/* When acquiring a heap for the 1st time, perform a check and
+		 * calculate the total number of user accessible physical heaps */
+		if (psPhysHeap->ui32RefCount == 1)
+		{
+			_PhysHeapCountUserModeHeaps(psDevNode, BIT(eDevPhysHeap));
+		}
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_PHYSHEAP_ID_INVALID;
+	}
+
+	OSLockRelease(psDevNode->hPhysHeapLock);
+
+	*ppsPhysHeap = psPhysHeap;
+	PVR_DPF_RETURN_RC1(eError, *ppsPhysHeap);
+}
+
+void PhysHeapRelease(PHYS_HEAP *psPhysHeap)
+{
+	PVR_DPF_ENTERED1(psPhysHeap);
+
+	OSLockAcquire(psPhysHeap->psDevNode->hPhysHeapLock);
+	psPhysHeap->ui32RefCount--;
+	PHYSHEAP_REFCOUNT_PRINT("%s: Heap %p, refcount = %d",
+							__func__, psPhysHeap, psPhysHeap->ui32RefCount);
+	OSLockRelease(psPhysHeap->psDevNode->hPhysHeapLock);
+
+	PVR_DPF_RETURN;
+}
+
+PHEAP_IMPL_DATA PhysHeapGetImplData(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->pvImplData;
+}
+
+PHYS_HEAP_TYPE PhysHeapGetType(PHYS_HEAP *psPhysHeap)
+{
+	PVR_ASSERT(psPhysHeap->eType != PHYS_HEAP_TYPE_UNKNOWN);
+	return psPhysHeap->eType;
+}
+
+PHYS_HEAP_POLICY PhysHeapGetPolicy(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->uiPolicy;
+}
+
+PHYS_HEAP_USAGE_FLAGS PhysHeapGetFlags(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32UsageFlags;
+}
+
+const IMG_CHAR *PhysHeapName(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->aszName;
+}
+
+IMG_BOOL PhysHeapValidateDefaultHeapExists(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_PHYS_HEAP eDefaultHeap = psDevNode->psDevConfig->eDefaultHeap;
+
+	return ((psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_DEFAULT] != NULL) &&
+			((psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_DEFAULT] ==
+			  psDevNode->apsPhysHeap[eDefaultHeap])));
+}
+
+#if defined(SUPPORT_STATIC_IPA)
+IMG_UINT32 PhysHeapGetIPAValue(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAPolicyValue;
+}
+
+IMG_UINT32 PhysHeapGetIPAMask(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAClearMask;
+}
+
+IMG_UINT32 PhysHeapGetIPAShift(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAShift;
+}
+#endif
+
+/*
+ * This function will set the psDevPAddr to whatever the system layer
+ * has set it for the referenced region.
+ * It will not fail if the psDevPAddr is invalid.
+ */
+PVRSRV_ERROR PhysHeapGetDevPAddr(PHYS_HEAP *psPhysHeap,
+									   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetDevPAddr != NULL)
+	{
+		eResult = psImplFuncs->pfnGetDevPAddr(psPhysHeap->pvImplData,
+											  psDevPAddr);
+	}
+
+	return eResult;
+}
+
+/*
+ * This function will set the psCpuPAddr to whatever the system layer
+ * has set it for the referenced region.
+ * It will not fail if the psCpuPAddr is invalid.
+ */
+PVRSRV_ERROR PhysHeapGetCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetCPUPAddr != NULL)
+	{
+		eResult = psImplFuncs->pfnGetCPUPAddr(psPhysHeap->pvImplData,
+											  psCpuPAddr);
+	}
+
+	return eResult;
+}
+
+PVRSRV_ERROR PhysHeapGetSize(PHYS_HEAP *psPhysHeap,
+								   IMG_UINT64 *puiSize)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetSize != NULL)
+	{
+		eResult = psImplFuncs->pfnGetSize(psPhysHeap->pvImplData,
+										  puiSize);
+	}
+
+	return eResult;
+}
+
+/* Used to add the total and free sizes of an IMA heap and the backing
+ * DLM heap
+ */
+static void PhysHeapIMAGetMemInfo(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 *puiTotalSize,
+                                  IMG_UINT64 *puiFreeSize)
+{
+	PHYS_HEAP *psDLMHeap;
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	IMG_UINT64 ui64DLMTotalSize;
+	IMG_UINT64 ui64DLMFreeSize;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(
+		psPhysHeap->eType == PHYS_HEAP_TYPE_IMA,
+		"Physheap type not IMA");
+
+	/* Obtain the DLM heap chained to this IMA heap */
+	PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetHeapDLMBacking);
+	psPhysHeap->psImplFuncs->pfnGetHeapDLMBacking(psPhysHeap->pvImplData,
+	                                              &psDLMHeap);
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDLMHeap != NULL, "pfnGetHeapDLMBacking");
+
+	/* Obtain the memstats for the current IMA heap */
+	PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetFactoryMemStats);
+	psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+	                                               &ui64TotalSize,
+	                                               &ui64FreeSize);
+
+	/* Obtain the memstats for the chained DLM heap */
+	PVR_ASSERT(psDLMHeap->psImplFuncs->pfnGetFactoryMemStats);
+	psDLMHeap->psImplFuncs->pfnGetFactoryMemStats(psDLMHeap->pvImplData,
+	                                              &ui64DLMTotalSize,
+	                                              &ui64DLMFreeSize);
+
+	/* Total size is just the DLM heap size as the IMA heap imports a
+	 * subset of that total size.
+	 * Free size is a combination of the two because DLM heap may
+	 * have spare PMBs to provide and the IMA heap may have space
+	 * in an imported PMB.
+	 */
+	*puiTotalSize = ui64DLMTotalSize;
+	*puiFreeSize = ui64FreeSize + ui64DLMFreeSize;
+}
+
+PVRSRV_ERROR
+PhysHeapGetMemInfo(PVRSRV_DEVICE_NODE *psDevNode,
+                   IMG_UINT32 ui32PhysHeapCount,
+                   PVRSRV_PHYS_HEAP *paePhysHeapID,
+                   PHYS_HEAP_MEM_STATS_PTR paPhysHeapMemStats)
+{
+	IMG_UINT32 i = 0;
+	PHYS_HEAP *psPhysHeap;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui32PhysHeapCount <= MAX_USER_MODE_ALLOC_PHYS_HEAPS, "ui32PhysHeapCount invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(paePhysHeapID != NULL, "paePhysHeapID invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(paPhysHeapMemStats != NULL, "paPhysHeapMemStats invalid");
+
+	for (i = 0; i < ui32PhysHeapCount; i++)
+	{
+		if (paePhysHeapID[i] >= PVRSRV_PHYS_HEAP_LAST)
+		{
+			return PVRSRV_ERROR_PHYSHEAP_ID_INVALID;
+		}
+
+		psPhysHeap = _PhysHeapFindRealHeapNoFallback(paePhysHeapID[i], psDevNode);
+
+		paPhysHeapMemStats[i].ui32PhysHeapFlags = 0;
+		paPhysHeapMemStats[i].ui64DevicesInSPAS = 0;
+
+		if (psPhysHeap && PhysHeapUserModeAlloc(paePhysHeapID[i])
+				&& psPhysHeap->psImplFuncs->pfnGetFactoryMemStats)
+		{
+			if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+			{
+				PhysHeapIMAGetMemInfo(psPhysHeap,
+				                      &paPhysHeapMemStats[i].ui64TotalSize,
+				                      &paPhysHeapMemStats[i].ui64FreeSize);
+			}
+			else
+			{
+				psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+				                                               &paPhysHeapMemStats[i].ui64TotalSize,
+				                                               &paPhysHeapMemStats[i].ui64FreeSize);
+			}
+
+			if (paePhysHeapID[i] == psDevNode->psDevConfig->eDefaultHeap)
+			{
+				paPhysHeapMemStats[i].ui32PhysHeapFlags |= PVRSRV_PHYS_HEAP_FLAGS_IS_DEFAULT;
+			}
+#if defined(PVRSRV_ENABLE_XD_MEM)
+			paPhysHeapMemStats[i].ui64DevicesInSPAS = PhysHeapSpasDeviceBitmap(psPhysHeap);
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+			paPhysHeapMemStats[i].ePhysHeapType = psPhysHeap->eType;
+		}
+		else
+		{
+			paPhysHeapMemStats[i].ui64TotalSize = 0;
+			paPhysHeapMemStats[i].ui64FreeSize = 0;
+			paPhysHeapMemStats[i].ePhysHeapType = PHYS_HEAP_TYPE_UNKNOWN;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+void PhysHeapCpuPAddrToDevPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	psPhysHeap->psMemFuncs->pfnCpuPAddrToDevPAddr(psPhysHeap->hPrivData,
+												 ui32NumOfAddr,
+												 psDevPAddr,
+												 psCpuPAddr);
+}
+
+void PhysHeapDevPAddrToCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr,
+								IMG_DEV_PHYADDR *psDevPAddr)
+{
+	psPhysHeap->psMemFuncs->pfnDevPAddrToCpuPAddr(psPhysHeap->hPrivData,
+												 ui32NumOfAddr,
+												 psCpuPAddr,
+												 psDevPAddr);
+}
+
+IMG_CHAR *PhysHeapPDumpMemspaceName(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->pszPDumpMemspaceName;
+}
+
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+static inline void _LogOOMDetection(IMG_BOOL isOOMDetected, PHYS_HEAP *psPhysHeap, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	IMG_BOOL bExistingVal = OSAtomicExchange(&psPhysHeap->sOOMDetected, isOOMDetected);
+	PVRSRV_PHYS_HEAP ePhysIdx = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	if (bExistingVal != isOOMDetected)
+	{
+		PVR_LOG(("Device: %d Physheap: %s OOM: %s",
+				(psPhysHeap->psDevNode->sDevId.ui32InternalID),
+				g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel,
+				(isOOMDetected) ? "Detected" : "Resolved"));
+	}
+}
+#endif
+
+PVRSRV_ERROR PhysHeapCreatePMR(PHYS_HEAP *psPhysHeap,
+							   struct _CONNECTION_DATA_ *psConnection,
+							   IMG_DEVMEM_SIZE_T uiSize,
+							   IMG_UINT32 ui32NumPhysChunks,
+							   IMG_UINT32 ui32NumVirtChunks,
+							   IMG_UINT32 *pui32MappingTable,
+							   IMG_UINT32 uiLog2PageSize,
+							   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							   const IMG_CHAR *pszAnnotation,
+							   IMG_PID uiPid,
+							   PMR **ppsPMRPtr,
+							   IMG_UINT32 ui32PDumpFlags,
+							   PVRSRV_MEMALLOCFLAGS_T *puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	IMG_UINT64 uiFreeBytes;
+	PVRSRV_PHYS_HEAP eDemotionPhysIdx;
+	PVRSRV_MEMALLOCFLAGS_T uiDemotionFlags = uiFlags;
+	PVRSRV_PHYS_HEAP ePhysIdx = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+	PHYS_HEAP *psDemotionHeap = NULL;
+#endif
+	eError = psImplFuncs->pfnCreatePMR(psPhysHeap,
+									 psConnection,
+									 uiSize,
+									 ui32NumPhysChunks,
+									 ui32NumVirtChunks,
+									 pui32MappingTable,
+									 uiLog2PageSize,
+									 uiFlags,
+									 pszAnnotation,
+									 uiPid,
+									 ppsPMRPtr,
+									 ui32PDumpFlags);
+
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	/* Check for OOM error, return if otherwise */
+	_LogOOMDetection(((IsOOMError(eError)) ? IMG_TRUE : IMG_FALSE), psPhysHeap, uiFlags);
+	if (eError == PVRSRV_OK)
+	{
+		if (puiOutFlags)
+		{
+			*puiOutFlags = uiFlags;
+		}
+		return eError;
+	}
+	PVR_LOG_RETURN_IF_FALSE((IsOOMError(eError)), "Failed to allocate PMR", eError);
+
+	/* Skip logic and return if mandate flag is set */
+	if (PVRSRV_CHECK_MANDATED_PHYSHEAP(uiFlags))
+	{
+		return eError;
+	}
+
+	/* Demotion only occurs on CPU_LOCAL,GPU_LOCAL,GPU_PRIVATE */
+	if (ePhysIdx > PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+	{
+		return eError;
+	}
+
+	eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	for (eDemotionPhysIdx = (PVRSRV_PHYS_HEAP)(ePhysIdx-1); eDemotionPhysIdx != PVRSRV_PHYS_HEAP_DEFAULT; eDemotionPhysIdx--)
+	{
+		PVRSRV_CHANGE_PHYS_HEAP_HINT(eDemotionPhysIdx, uiDemotionFlags);
+		PVR_LOG_IF_FALSE_VA(PVR_DBG_MESSAGE, (ePhysIdx-eDemotionPhysIdx < 2), "Demoted from %s to CPU_LOCAL. "
+						"Expect Performance to be affected!", g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel);
+		psDemotionHeap = _PhysHeapFindRealHeapNoFallback(eDemotionPhysIdx, psPhysHeap->psDevNode);
+
+		/* Either no alternative available, or allocation already failed on selected heap */
+		if (psDemotionHeap == NULL || psPhysHeap == psDemotionHeap)
+		{
+			continue;
+		}
+
+		if (PhysHeapFreeMemCheck(psDemotionHeap, uiSize, &uiFreeBytes) != PVRSRV_OK)
+		{
+			_LogOOMDetection(IMG_TRUE, psDemotionHeap, uiDemotionFlags);
+			continue;
+		}
+
+		psImplFuncs = psDemotionHeap->psImplFuncs;
+		eError = psImplFuncs->pfnCreatePMR(psDemotionHeap,
+						 psConnection,
+						 uiSize,
+						 ui32NumPhysChunks,
+						 ui32NumVirtChunks,
+						 pui32MappingTable,
+						 uiLog2PageSize,
+						 uiDemotionFlags,
+						 pszAnnotation,
+						 uiPid,
+						 ppsPMRPtr,
+						 ui32PDumpFlags);
+		_LogOOMDetection(((IsOOMError(eError)) ? IMG_TRUE : IMG_FALSE), psDemotionHeap, uiDemotionFlags);
+
+		if (eError == PVRSRV_OK)
+		{
+			if (puiOutFlags)
+			{
+				*puiOutFlags = uiDemotionFlags;
+			}
+			break;
+		}
+	}
+	if (eError == PVRSRV_OK)
+	{
+		/* Success demotion worked error Ok - emit warning. */
+		PVR_LOG_VA(PVR_DBG_WARNING, "PhysHeap(%s) failed to allocate PMR. Demoted to %s" ,
+						g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel,
+						g_asPhysHeapUsageFlagStrings[eDemotionPhysIdx-1].pszLabel);
+	}
+	else
+	{
+		/* Unable to create PMR (Heap not found or CreatePMR failed) - emit Error */
+		PVR_LOG_VA(PVR_DBG_ERROR, "Error raised %s : Unable to %s." ,
+		                          PVRSRVGETERRORSTRING(eError),
+		                          (psDemotionHeap == NULL) ?  "find heaps for demotion" :
+		                                                      "allocate PMR via Demotion heap");
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		{
+			PPVRSRV_DEVICE_NODE psDevNode = PhysHeapDeviceNode(psPhysHeap);
+			OSLockAcquire(psDevNode->hPMRZombieListLock);
+			PVR_LOG_VA(PVR_DBG_ERROR, "PMR Zombie Count: %u, PMR Zombie Count In Cleanup: %u",
+			                          psDevNode->uiPMRZombieCount,
+			                          psDevNode->uiPMRZombieCountInCleanup);
+			OSLockRelease(psDevNode->hPMRZombieListLock);
+		}
+#endif
+	}
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PhysHeapCreatePMB(PHYS_HEAP *psPhysHeap,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               const IMG_CHAR *pszAnnotation,
+                               PMB **ppsPMRPtr,
+                               RA_BASE_T *puiBase,
+                               RA_LENGTH_T *puiSize)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		psPhysHeap->eType == PHYS_HEAP_TYPE_DLM,
+		"Physheap type not DLM",
+		PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT(psImplFuncs->pfnCreatePMB);
+
+	return psImplFuncs->pfnCreatePMB(psPhysHeap,
+	                                 uiSize,
+	                                 pszAnnotation,
+	                                 ppsPMRPtr,
+	                                 puiBase,
+	                                 puiSize);
+}
+
+PPVRSRV_DEVICE_NODE PhysHeapDeviceNode(PHYS_HEAP *psPhysHeap)
+{
+	PVR_ASSERT(psPhysHeap != NULL);
+
+	return psPhysHeap->psDevNode;
+}
+
+static IMG_BOOL PhysHeapCreatedByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return (gasHeapProperties[ePhysHeap].ePVRLayerAction != PVR_LAYER_HEAP_ACTION_IGNORE);
+}
+
+static IMG_BOOL PhysHeapAcquiredByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return (gasHeapProperties[ePhysHeap].ePVRLayerAction == PVR_LAYER_HEAP_ACTION_INITIALISE);
+}
+
+IMG_BOOL PhysHeapUserModeAlloc(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return gasHeapProperties[ePhysHeap].bUserModeAlloc;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+PVRSRV_ERROR PhysHeapPagesAllocGPV(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_UINT32 ui32OSid, IMG_PID uiPid)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesAllocGPV != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesAllocGPV(psPhysHeap,
+		                                        uiSize, psMemHandle, psDevPAddr, ui32OSid, uiPid);
+	}
+
+	return eResult;
+}
+#endif
+
+PVRSRV_ERROR PhysHeapPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+								PG_HANDLE *psMemHandle,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_PID uiPid)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesAlloc != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesAlloc(psPhysHeap,
+		                                       uiSize, psMemHandle, psDevPAddr, uiPid);
+	}
+
+	return eResult;
+}
+
+void PhysHeapPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_ASSERT(psImplFuncs->pfnPagesFree != NULL);
+
+	if (psImplFuncs->pfnPagesFree != NULL)
+	{
+		psImplFuncs->pfnPagesFree(psPhysHeap,
+		                          psMemHandle);
+	}
+}
+
+PVRSRV_ERROR PhysHeapPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle, size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+							  void **pvPtr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesMap != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesMap(psPhysHeap,
+		                                   pshMemHandle, uiSize, psDevPAddr, pvPtr);
+	}
+
+	return eResult;
+}
+
+void PhysHeapPagesUnMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_ASSERT(psImplFuncs->pfnPagesUnMap != NULL);
+
+	if (psImplFuncs->pfnPagesUnMap != NULL)
+	{
+		psImplFuncs->pfnPagesUnMap(psPhysHeap,
+		                           psMemHandle, pvPtr);
+	}
+}
+
+PVRSRV_ERROR PhysHeapPagesClean(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle,
+							  IMG_UINT32 uiOffset,
+							  IMG_UINT32 uiLength)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesClean != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesClean(psPhysHeap,
+		                                     pshMemHandle, uiOffset, uiLength);
+	}
+
+	return eResult;
+}
+
+IMG_UINT32 PhysHeapGetPageShift(PHYS_HEAP *psPhysHeap)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	IMG_UINT32 ui32PageShift = 0;
+
+	PVR_ASSERT(psImplFuncs->pfnGetPageShift != NULL);
+
+	if (psImplFuncs->pfnGetPageShift != NULL)
+	{
+		ui32PageShift = psImplFuncs->pfnGetPageShift();
+	}
+
+	return ui32PageShift;
+}
+
+PVRSRV_ERROR PhysHeapFreeMemCheck(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 ui64MinRequiredMem,
+                                  IMG_UINT64 *pui64FreeMem)
+{
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui64FreeMem != NULL, "pui64FreeMem");
+
+
+	if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+	{
+		PhysHeapIMAGetMemInfo(psPhysHeap->pvImplData,
+		                            &ui64TotalSize,
+		                            &ui64FreeSize);
+	}
+	else
+	{
+		psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+		                                               &ui64TotalSize,
+		                                               &ui64FreeSize);
+	}
+
+	*pui64FreeMem = ui64FreeSize;
+	if (ui64MinRequiredMem >= *pui64FreeMem)
+	{
+		eError = PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (physheap.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem.c
new file mode 100644
index 000000000000..ec05b472cc70
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem.c
@@ -0,0 +1,864 @@
+/*************************************************************************/ /*!
+@File           physmem.c
+@Title          Physmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common entry point for creation of RAM backed PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "device.h"
+#include "physmem.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "pdump_physmem.h"
+#include "pdump_km.h"
+#include "pvr_ricommon.h"
+#include "allocmem.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#if defined(DEBUG)
+static IMG_UINT32 PMRAllocFail;
+
+#if defined(__linux__)
+#include <linux/moduleparam.h>
+
+module_param(PMRAllocFail, uint, 0644);
+MODULE_PARM_DESC(PMRAllocFail, "When number of PMR allocs reaches "
+				 "this value, it will fail (default value is 0 which "
+				 "means that alloc function will behave normally).");
+#endif /* defined(__linux__) */
+#endif /* defined(DEBUG) */
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#include "proc_stats.h"
+#endif
+
+/** Computes division using log2 of divisor. */
+#define LOG2_DIV(x, log2) ((x) >> (log2))
+
+/** Computes modulo of a power of 2. */
+#define LOG2_MOD(x, log2) ((x) & ((1 << (log2)) - 1))
+
+PVRSRV_ERROR DevPhysMemAlloc(PVRSRV_DEVICE_NODE	*psDevNode,
+                             IMG_UINT32 ui32MemSize,
+                             IMG_UINT32 ui32Log2Align,
+                             const IMG_UINT8 u8Value,
+                             IMG_BOOL bInitPage,
+#if defined(PDUMP)
+                             const IMG_CHAR *pszDevSpace,
+                             const IMG_CHAR *pszSymbolicAddress,
+                             IMG_HANDLE *phHandlePtr,
+#endif
+                             IMG_PID uiPid,
+                             IMG_HANDLE hMemHandle,
+                             IMG_DEV_PHYADDR *psDevPhysAddr)
+{
+	void *pvCpuVAddr;
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	IMG_CHAR szFilenameOut[PDUMP_PARAM_MAX_FILE_NAME];
+	PDUMP_FILEOFFSET_T uiOffsetOut;
+	IMG_UINT32 ui32PageSize;
+	IMG_UINT32 ui32PDumpMemSize = ui32MemSize;
+	PVRSRV_ERROR ePDumpError;
+#endif
+	PG_HANDLE *psMemHandle;
+	IMG_UINT64 uiMask;
+	IMG_DEV_PHYADDR sDevPhysAddr_int;
+
+	psMemHandle = hMemHandle;
+
+	/* Allocate the pages */
+	eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+	                            TRUNCATE_64BITS_TO_SIZE_T(ui32MemSize),
+	                            psMemHandle,
+	                            &sDevPhysAddr_int,
+	                            uiPid);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnDevPxAlloc:1");
+
+	/* Check to see if the page allocator returned pages with our desired
+	 * alignment, which is not unlikely
+	 */
+	uiMask = IMG_PAGE2BYTES32(ui32Log2Align) - 1;
+	if (ui32Log2Align && (sDevPhysAddr_int.uiAddr & uiMask))
+	{
+		/* use over allocation instead */
+		PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+
+		ui32MemSize += (IMG_UINT32) uiMask;
+		eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+		                            TRUNCATE_64BITS_TO_SIZE_T(ui32MemSize),
+		                            psMemHandle,
+		                            &sDevPhysAddr_int,
+		                            uiPid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "pfnDevPxAlloc:2");
+
+		sDevPhysAddr_int.uiAddr += uiMask;
+		sDevPhysAddr_int.uiAddr &= ~uiMask;
+	}
+	*psDevPhysAddr = sDevPhysAddr_int;
+
+#if defined(PDUMP)
+	ui32PageSize = ui32Log2Align? IMG_PAGE2BYTES32(ui32Log2Align) : OSGetPageSize();
+	eError = PDumpMalloc(psDevNode,
+	                     pszDevSpace,
+	                     pszSymbolicAddress,
+	                     ui32PDumpMemSize,
+	                     ui32PageSize,
+	                     IMG_FALSE,
+	                     0,
+	                     phHandlePtr,
+	                     PDUMP_NONE);
+	if (PVRSRV_OK != eError)
+	{
+		PDUMPCOMMENT(psDevNode, "Allocating pages failed");
+		*phHandlePtr = NULL;
+	}
+	ePDumpError = eError;
+#endif
+
+	if (bInitPage)
+	{
+		/*Map the page to the CPU VA space */
+		eError = PhysHeapPagesMap(psDevNode->psMMUPhysHeap,
+		                          psMemHandle,
+		                          ui32MemSize,
+		                          &sDevPhysAddr_int,
+		                          &pvCpuVAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_LOG_ERROR(eError, "DevPxMap");
+			PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+			return eError;
+		}
+
+		/*Fill the memory with given content */
+		OSDeviceMemSet(pvCpuVAddr, u8Value, ui32MemSize);
+
+		/*Map the page to the CPU VA space */
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            psMemHandle,
+		                            0,
+		                            ui32MemSize);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_LOG_ERROR(eError, "DevPxClean");
+			PhysHeapPagesUnMap(psDevNode->psMMUPhysHeap, psMemHandle, pvCpuVAddr);
+			PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+			return eError;
+		}
+
+#if defined(PDUMP)
+		if (ePDumpError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+		{
+			/* PDumping of the page contents can be done in two ways
+			 * 1. Store the single byte init value to the .prm file
+			 *    and load the same value to the entire dummy page buffer
+			 *    This method requires lot of LDB's inserted into the out2.txt
+			 *
+			 * 2. Store the entire contents of the buffer to the .prm file
+			 *    and load them back.
+			 *    This only needs a single LDB instruction in the .prm file
+			 *    and chosen this method
+			 *    size of .prm file might go up but that's not huge at least
+			 *    for this allocation
+			 */
+			/* Write the buffer contents to the prm file */
+			eError = PDumpWriteParameterBlob(psDevNode,
+											 pvCpuVAddr,
+											 ui32PDumpMemSize,
+											 PDUMP_FLAGS_CONTINUOUS,
+											 szFilenameOut,
+											 sizeof(szFilenameOut),
+											 &uiOffsetOut);
+			if (PVRSRV_OK == eError)
+			{
+				/* Load the buffer back to the allocated memory when playing the pdump */
+				eError = PDumpPMRLDB(psDevNode,
+									 pszDevSpace,
+									 pszSymbolicAddress,
+									 0,
+									 ui32PDumpMemSize,
+									 szFilenameOut,
+									 uiOffsetOut,
+									 PDUMP_FLAGS_CONTINUOUS);
+				if (PVRSRV_OK != eError)
+				{
+					PDUMP_ERROR(psDevNode, eError, "Failed to write LDB statement to script file");
+					PVR_LOG_ERROR(eError, "PDumpPMRLDB");
+				}
+			}
+			else if (eError != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+			{
+				PDUMP_ERROR(psDevNode, eError, "Failed to write device allocation to parameter file");
+				PVR_LOG_ERROR(eError, "PDumpWriteParameterBlob");
+			}
+			else
+			{
+				/* Else write to parameter file prevented under the flags and
+				 * current state of the driver so skip write to script and error IF.
+				 * This is expected e.g., if not in the capture range.
+				 */
+				eError = PVRSRV_OK;
+			}
+		}
+#endif
+
+		/* Unmap the page */
+		PhysHeapPagesUnMap(psDevNode->psMMUPhysHeap,
+		                   psMemHandle,
+		                   pvCpuVAddr);
+	}
+
+	return PVRSRV_OK;
+}
+
+void DevPhysMemFree(PVRSRV_DEVICE_NODE *psDevNode,
+#if defined(PDUMP)
+							IMG_HANDLE hPDUMPMemHandle,
+#endif
+							IMG_HANDLE	hMemHandle)
+{
+	PG_HANDLE *psMemHandle;
+
+	psMemHandle = hMemHandle;
+	PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+#if defined(PDUMP)
+	if (NULL != hPDUMPMemHandle)
+	{
+		PDumpFree(psDevNode, hPDUMPMemHandle);
+	}
+#endif
+
+}
+
+PVRSRV_ERROR PhysMemValidateMappingTable(IMG_UINT32 ui32TotalNumVirtChunks,
+                                         IMG_UINT32 ui32IndexCount,
+                                         const IMG_UINT32 *pui32MappingTable)
+{
+	IMG_UINT8 *paui8TrackedIndices;
+	IMG_UINT32 ui32BytesToTrackIndicies;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Allocate memory for a bitmask to track indices.
+	 * We allocate 'n' bytes with 1 bit representing each index, to allow
+	 * us to check for any repeated entries in pui32MappingTable.
+	 */
+	ui32BytesToTrackIndicies = LOG2_DIV(ui32TotalNumVirtChunks, 3);
+	if (LOG2_MOD(ui32TotalNumVirtChunks, 3) != 0)
+	{
+		++ui32BytesToTrackIndicies;
+	}
+	paui8TrackedIndices = OSAllocZMem(ui32BytesToTrackIndicies);
+	if (paui8TrackedIndices == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	for (i = 0; i < ui32IndexCount; i++)
+	{
+		IMG_UINT32 ui32LogicalIndex = pui32MappingTable[i];
+
+		/* Check that index is within the bounds of the allocation */
+		if (ui32LogicalIndex >= ui32TotalNumVirtChunks)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Index %u is OOB",
+			         __func__,
+			         ui32LogicalIndex));
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+
+		/* Check that index is not repeated */
+		if (BIT_ISSET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3)))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Duplicate index found: %u",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+		BIT_SET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3));
+	}
+
+	OSFreeMem(paui8TrackedIndices);
+
+	return eError;
+}
+
+/* Checks the input parameters and adjusts them if possible and necessary */
+PVRSRV_ERROR PhysMemValidateParams(PVRSRV_DEVICE_NODE *psDevNode,
+                                   IMG_UINT32 ui32NumPhysChunks,
+                                   IMG_UINT32 ui32NumVirtChunks,
+                                   IMG_UINT32 *pui32MappingTable,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   IMG_PID uiPid,
+                                   IMG_UINT32 *puiLog2AllocPageSize,
+                                   IMG_DEVMEM_SIZE_T *puiSize)
+{
+	IMG_UINT32 uiLog2AllocPageSize = *puiLog2AllocPageSize;
+	IMG_UINT32 ui32PageSize = IMG_PAGE2BYTES32(uiLog2AllocPageSize);
+	IMG_DEVMEM_SIZE_T uiSize = *puiSize;
+	/* Sparse if we have different number of virtual and physical chunks plus
+	 * in general all allocations with more than one virtual chunk */
+	IMG_BOOL bIsSparse = (ui32NumVirtChunks != ui32NumPhysChunks ||
+			ui32NumVirtChunks > 1) ? IMG_TRUE : IMG_FALSE;
+
+	/* Sparse allocations must be backed immediately as the requested
+	 * pui32MappingTable is not retained in any structure if not immediately
+	 * actioned on allocation.
+	 */
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags) && bIsSparse)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid to specify ON_DEMAND for a sparse allocation: 0x%" IMG_UINT64_FMTSPECX, __func__, uiFlags));
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags) && uiPid == PVR_SYS_ALLOC_PID)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot move system allocated resources PID = PVR_SYS_ALLOC_PID",
+		         __func__));
+
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags) &&
+	    PVRSRV_CHECK_OS_LINUX_DENY_MOVE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot move denied movable allocation",
+		         __func__));
+
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif
+
+	if (ui32NumVirtChunks == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of virtual chunks cannot be 0",
+		         __func__));
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Protect against invalid page sizes */
+	if ((ui32PageSize & psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask) == 0)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR, "Page size of %u is invalid", ui32PageSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Range check of the alloc size */
+	if (!PMRValidateSize(uiSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				   "PMR size exceeds limit #Chunks: %u ChunkSz 0x%08X",
+				   ui32NumVirtChunks,
+				   IMG_PAGE2BYTES32(uiLog2AllocPageSize));
+		return PVRSRV_ERROR_PMR_TOO_LARGE;
+	}
+
+	/* Fail if requesting coherency on one side but uncached on the other */
+	if (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) &&
+	    (PVRSRV_CHECK_GPU_UNCACHED(uiFlags) || PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for CPU coherency but specifying GPU uncached "
+				"Please use GPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+	    (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for GPU coherency but specifying CPU uncached "
+				"Please use CPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags) && PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Zero on Alloc and Poison on Alloc are mutually exclusive.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (bIsSparse)
+	{
+		/* For sparse we need correct parameters like a suitable page size....  */
+		if (OSGetPageShift() > uiLog2AllocPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid log2-contiguity for sparse allocation. "
+					"Requested %u, required minimum %zd",
+					__func__,
+					uiLog2AllocPageSize,
+					OSGetPageShift() ));
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		if (IMG_PAGES2BYTES64(ui32NumVirtChunks, uiLog2AllocPageSize) != uiSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Total alloc size (%#" IMG_UINT64_FMTSPECx ") "
+					 "is not equal to virtual chunks * chunk size "
+					 "(%#" IMG_UINT64_FMTSPECx ")",
+					 __func__,
+					 uiSize,
+					 IMG_PAGES2BYTES64(ui32NumVirtChunks, uiLog2AllocPageSize)));
+
+			return PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE;
+		}
+
+		if (ui32NumPhysChunks > ui32NumVirtChunks)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Number of physical chunks (%u) must not be greater "
+					"than number of virtual chunks (%u)",
+					__func__,
+					ui32NumPhysChunks,
+					ui32NumVirtChunks));
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	else
+	{
+		/*
+		 * Silently round up alignment/pagesize if request was less that PAGE_SHIFT
+		 * because it would never be harmful for memory to be _more_ contiguous that
+		 * was desired.
+		 */
+		uiLog2AllocPageSize = OSGetPageShift() > uiLog2AllocPageSize ?
+				OSGetPageShift() : uiLog2AllocPageSize;
+
+		/* Same for total size */
+		uiSize = PVR_ALIGN(uiSize, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+	}
+
+	if ((uiSize & (IMG_PAGE2BYTES32(uiLog2AllocPageSize) - 1)) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Total size (%#" IMG_UINT64_FMTSPECx ") "
+		        "must be a multiple of the requested contiguity (%u)",
+		        __func__,
+		        uiSize,
+		        IMG_PAGE2BYTES32(uiLog2AllocPageSize)));
+		return PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE;
+	}
+
+	/* Parameter validation - Mapping table entries */
+	{
+		PVRSRV_ERROR eErr = PhysMemValidateMappingTable(ui32NumVirtChunks,
+		                                                ui32NumPhysChunks,
+		                                                pui32MappingTable);
+		PVR_RETURN_IF_ERROR(eErr);
+	}
+
+	*puiLog2AllocPageSize = uiLog2AllocPageSize;
+	*puiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _DevPhysHeapFromFlags(PVRSRV_MEMALLOCFLAGS_T uiFlags,
+										  PVRSRV_PHYS_HEAP *peDevPhysHeap,
+										  PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_PHYS_HEAP eHeap = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	switch (eHeap)
+	{
+		case PVRSRV_PHYS_HEAP_FW_PREMAP0:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP1:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP2:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP3:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP4:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP5:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP6:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP7:
+		{
+			/* keep heap (with check) */
+			PVR_RETURN_IF_INVALID_PARAM(!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode));
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_LAST:
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+		default:
+		{
+			break;
+		}
+	}
+
+	*peDevPhysHeap = eHeap;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void _PromoteToCpuCached(PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)) == 0)
+	{
+		/* We don't need to upgrade if we don't map into the CPU */
+		return;
+	}
+
+	/* Clear the existing CPU cache flags */
+	*puiFlags &= ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK);
+
+	/* Add CPU cached flags */
+	*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT;
+}
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR_direct(CONNECTION_DATA *psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2AllocPageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMRPtr,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	PVRSRV_PHYS_HEAP ePhysHeapIdx;
+	PVRSRV_MEMALLOCFLAGS_T uiPMRFlags = uiFlags;
+	uiPid = (psConnection != NULL) ? OSGetCurrentClientProcessIDKM() : uiPid;
+
+	/* This is where we would expect to validate the uiAnnotationLength parameter
+	   (to confirm it is sufficient to store the string in pszAnnotation plus a
+	   terminating NULL). However, we do not make reference to this value when
+	   we copy the string in PMRCreatePMR() - instead there we use strlcpy()
+	   to copy at most chars and ensure whatever is copied is null-terminated.
+	   The parameter is only used by the generated bridge code.
+	 */
+	PVR_UNREFERENCED_PARAMETER(uiAnnotationLength);
+
+	if (PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY)
+	{
+		_PromoteToCpuCached(&uiPMRFlags);
+	}
+
+	eError = PhysMemValidateParams(psDevNode,
+	                               ui32NumPhysChunks,
+	                               ui32NumVirtChunks,
+	                               pui32MappingTable,
+	                               uiFlags,
+	                               uiPid,
+	                               &uiLog2AllocPageSize,
+	                               &uiSize);
+	PVR_RETURN_IF_ERROR(eError);
+
+#if defined(SUPPORT_STATIC_IPA)
+#if !defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	/* Do not permit IPA PMR allocation flags to be passed through to the
+	 * new PMR.
+	 */
+	uiPMRFlags &= ~PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+#endif
+#endif
+
+#if defined(PDUMP)
+	eError = PDumpValidateUMFlags(ui32PDumpFlags);
+	PVR_RETURN_IF_ERROR(eError);
+#endif
+
+	for (i = 0; i < ui32NumPhysChunks; i++)
+	{
+		PVR_LOG_RETURN_IF_FALSE(pui32MappingTable[i] < ui32NumVirtChunks,
+		                        "Mapping table value exceeds ui32NumVirtChunks",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = _DevPhysHeapFromFlags(uiFlags, &ePhysHeapIdx, psDevNode);
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeapIdx = psDevNode->psDevConfig->eDefaultHeap;
+		PVRSRV_CHANGE_PHYS_HEAP_HINT(ePhysHeapIdx, uiPMRFlags);
+	}
+
+	if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_GPU_LOCAL)
+	{
+		if ((uiFlags & PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK) == 0)
+		{
+			ePhysHeapIdx = PVRSRV_PHYS_HEAP_GPU_PRIVATE;
+			PVRSRV_SET_PHYS_HEAP_HINT(GPU_PRIVATE, uiPMRFlags);
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Consider explicit use of GPU_PRIVATE for PMR %s."
+			        " Implicit conversion to GPU PRIVATE performed",
+			        __func__, pszAnnotation));
+		}
+		else if (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+				 PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig))
+		{
+			ePhysHeapIdx = PVRSRV_PHYS_HEAP_GPU_COHERENT;
+			PVRSRV_SET_PHYS_HEAP_HINT(GPU_COHERENT, uiPMRFlags);
+		}
+	}
+	else if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+	{
+		if (uiFlags & PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid flags for PMR %s!"
+			        " Client requested GPU_PRIVATE physical heap with CPU access flags.",
+			        __func__, pszAnnotation));
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+	}
+
+	if (NULL == psDevNode->apsPhysHeap[ePhysHeapIdx])
+	{
+		/* In case a heap hasn't been acquired for this type, return invalid heap error */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Requested allocation on device node (%p) from "
+		        "an invalid heap (HeapIndex=%d)",
+		        __func__, psDevNode, ePhysHeapIdx));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+#if defined(DEBUG)
+	if (PMRAllocFail > 0)
+	{
+		static IMG_UINT32 ui32AllocCount = 1;
+
+		if (ui32AllocCount < PMRAllocFail)
+		{
+			ui32AllocCount++;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s failed on %d allocation.",
+			         __func__, ui32AllocCount));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+#endif /* defined(DEBUG) */
+
+	/* If the driver is in an 'init' state all of the allocated memory
+	 * should be attributed to the driver (PID 1) rather than to the
+	 * process those allocations are made under. Same applies to the memory
+	 * allocated for the Firmware. */
+	if (psDevNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+	    PVRSRV_CHECK_FW_MAIN(uiFlags))
+	{
+		uiPid = PVR_SYS_ALLOC_PID;
+	}
+
+	/* ePhysHeapIdx and PhysHeap hint in uiPMRFlags match and provide the
+	 * intended PhysHeap to use at this point, but systems vary so the next
+	 * call may fallback (apsPhysHeap[]) or demote (OutOfMem) and not be the
+	 * heap that was intended, e.g. GPU_PRIVATE index may fallback to GPU_LOCAL,
+	 * GPU_LOCAL may demote to CPU_LOCAL.
+	 * On output uiPMRFlags show the PhysHeap finally used.
+	 */
+	eError = PhysHeapCreatePMR(psDevNode->apsPhysHeap[ePhysHeapIdx],
+							   psConnection,
+							   uiSize,
+							   ui32NumPhysChunks,
+							   ui32NumVirtChunks,
+							   pui32MappingTable,
+							   uiLog2AllocPageSize,
+							   uiPMRFlags,
+							   pszAnnotation,
+							   uiPid,
+							   ppsPMRPtr,
+							   ui32PDumpFlags,
+							   &uiPMRFlags);
+
+	if (puiPMRFlags != NULL)
+	{
+		*puiPMRFlags = uiPMRFlags;
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	if (eError != PVRSRV_OK)
+	{
+		PVRSRVStatsUpdateOOMStat(psConnection,
+		                          psDevNode,
+		                          PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+		                          OSGetCurrentClientProcessIDKM());
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR(CONNECTION_DATA *psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2AllocPageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMRPtr,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags)
+{
+	PVRSRV_PHYS_HEAP ePhysHeap = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ePhysHeap < PVRSRV_PHYS_HEAP_LAST, "uiFlags");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiAnnotationLength != 0, "uiAnnotationLength");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	if (!PhysHeapUserModeAlloc(ePhysHeap))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid phys heap hint: %d.", __func__, ePhysHeap));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PhysmemNewRamBackedPMR_direct(psConnection,
+	                                     psDevNode,
+	                                     uiSize,
+	                                     ui32NumPhysChunks,
+	                                     ui32NumVirtChunks,
+	                                     pui32MappingTable,
+	                                     uiLog2AllocPageSize,
+	                                     uiFlags,
+	                                     uiAnnotationLength,
+	                                     pszAnnotation,
+	                                     uiPid,
+	                                     ppsPMRPtr,
+	                                     ui32PDumpFlags,
+	                                     puiPMRFlags);
+}
+
+PVRSRV_ERROR
+PVRSRVGetDefaultPhysicalHeapKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDevNode,
+                               PVRSRV_PHYS_HEAP *peHeap)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	*peHeap = psDevNode->psDevConfig->eDefaultHeap;
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfoKM(CONNECTION_DATA *psConnection,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           IMG_UINT32 ui32PhysHeapCount,
+                           PVRSRV_PHYS_HEAP *paePhysHeapID,
+                           PHYS_HEAP_MEM_STATS_V1 *paPhysHeapMemStats)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysHeapCount);
+	PVR_UNREFERENCED_PARAMETER(paePhysHeapID);
+	PVR_UNREFERENCED_PARAMETER(paPhysHeapMemStats);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfo2KM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDevNode,
+                            IMG_UINT32 ui32PhysHeapCount,
+                            PVRSRV_PHYS_HEAP *paePhysHeapID,
+                            PHYS_HEAP_MEM_STATS_V2 *paPhysHeapMemStats)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	return PhysHeapGetMemInfo(psDevNode,
+	                          ui32PhysHeapCount,
+	                          paePhysHeapID,
+	                          paPhysHeapMemStats);
+}
+
+/* 'Wrapper' function to call PMRImportPMR(), which first checks the PMR is
+ * for the current device. This avoids the need to do this in pmr.c, which
+ * would then need PVRSRV_DEVICE_NODE (defining this type in pmr.h causes a
+ * typedef redefinition issue).
+ */
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR
+PhysmemImportPMR(CONNECTION_DATA *psConnection,
+             PVRSRV_DEVICE_NODE *psDevNode,
+             PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (PMRGetExportDeviceNode(psPMRExport) != psDevNode)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR invalid for this device", __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	return PMRImportPMR(psPMRExport,
+	                    uiPassword,
+	                    uiSize,
+	                    uiLog2Contig,
+	                    ppsPMR);
+}
+#endif /* if defined(SUPPORT_INSECURE_EXPORT) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_cpumap_history.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_cpumap_history.c
new file mode 100644
index 000000000000..6975926a45d8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_cpumap_history.c
@@ -0,0 +1,455 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR Mapping History for OS managed memory
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This module allows for an OS independent method of capture of
+                mapping history for OS memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/mem_types.h>
+#include "di_common.h"
+#include "di_server.h"
+#include "pvr_notifier.h"
+#include "physmem_cpumap_history.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#define MAPPING_HISTORY_CB_NUM_ENTRIES 10000
+#if defined(DEBUG)
+#define MAX_MAPPING_ANNOT_STR PVR_ANNOTATION_MAX_LEN
+#else
+#define MAX_MAPPING_ANNOT_STR 32
+#endif
+
+typedef struct _MAPPING_RECORD_ MAPPING_RECORD; /* Forward declaration */
+typedef void (*PFN_MAPPING_RECORD_STRING)(MAPPING_RECORD *psRecord,
+                                          IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN]);
+
+typedef struct _MAP_DATA_
+{
+	IMG_CHAR aszAnnotation[MAX_MAPPING_ANNOT_STR];
+	IMG_PID uiPID;
+	IMG_CPU_VIRTADDR pvAddress;
+	IMG_CPU_PHYADDR sCpuPhyAddr;
+	IMG_UINT32 ui32CPUCacheFlags;
+	size_t uiMapOffset; /* Mapping offset used when we don't map the whole PMR */
+	IMG_UINT32 ui32PageCount; /* # pages mapped */
+} MAP_DATA;
+
+typedef struct _UNMAP_DATA_
+{
+	IMG_CPU_VIRTADDR pvAddress;
+	IMG_CPU_PHYADDR sCpuPhyAddr;
+	IMG_UINT32 ui32CPUCacheFlags;
+	IMG_UINT32 ui32PageCount;
+} UNMAP_DATA;
+
+struct _MAPPING_RECORD_
+{
+	enum MAPPING_OP
+	{
+		UNSET = 0,
+		MAP,
+		UNMAP,
+	} etype;
+	union
+	{
+		MAP_DATA sMapData;
+		UNMAP_DATA sUnmapData;
+	} u;
+	PFN_MAPPING_RECORD_STRING pfnRecordString;
+};
+
+typedef struct _RECORDS_
+{
+	/* CB of mapping records that will be overwritten by newer entries */
+	MAPPING_RECORD *pasMapRecordsCB;
+	/* Current head of CB, used to get next slot, newest record +1 */
+	IMG_UINT32 ui32Head;
+	/* Current tail of CB, oldest record */
+	IMG_UINT32 ui32Tail;
+	/* Have we overwritten any records */
+	IMG_BOOL bOverwrite;
+} RECORDS;
+
+typedef struct _PHYSMEM_CPUMAP_HISTORY_DATA_
+{
+	DI_ENTRY *psDIEntry;
+	RECORDS sRecords;
+	POS_LOCK hLock;
+	IMG_HANDLE hDbgNotifier;
+
+} PHYSMEM_CPUMAP_HISTORY_DATA;
+
+typedef struct _MAPPING_HISTORY_ITERATOR_
+{
+	IMG_UINT32 ui32Start;
+	IMG_UINT32 ui32Finish;
+	IMG_UINT32 ui32Current;
+} MAPPING_HISTORY_ITERATOR;
+
+static PHYSMEM_CPUMAP_HISTORY_DATA gsMappingHistoryData;
+
+static PVRSRV_ERROR CreateMappingRecords(void)
+{
+	gsMappingHistoryData.sRecords.pasMapRecordsCB =
+	    OSAllocMem(sizeof(MAPPING_RECORD) * MAPPING_HISTORY_CB_NUM_ENTRIES);
+	PVR_RETURN_IF_NOMEM(gsMappingHistoryData.sRecords.pasMapRecordsCB);
+
+	return PVRSRV_OK;
+}
+
+static void InitMappingRecordCB(void)
+{
+	gsMappingHistoryData.sRecords.ui32Head = 0;
+	gsMappingHistoryData.sRecords.ui32Tail = 0;
+	gsMappingHistoryData.sRecords.bOverwrite = IMG_FALSE;
+}
+
+static void DestroyMappingRecords(void)
+{
+	OSFreeMem(gsMappingHistoryData.sRecords.pasMapRecordsCB);
+}
+
+static void MappingHistoryLock(void)
+{
+	if (gsMappingHistoryData.hLock)
+	{
+		OSLockAcquire(gsMappingHistoryData.hLock);
+	}
+}
+
+static void MappingHistoryUnlock(void)
+{
+	if (gsMappingHistoryData.hLock)
+	{
+		OSLockRelease(gsMappingHistoryData.hLock);
+	}
+}
+
+static MAPPING_HISTORY_ITERATOR CreateMappingHistoryIterator(void)
+{
+	MAPPING_HISTORY_ITERATOR sIter;
+
+	sIter.ui32Start = gsMappingHistoryData.sRecords.ui32Tail;
+	sIter.ui32Current = gsMappingHistoryData.sRecords.ui32Tail;
+	sIter.ui32Finish = gsMappingHistoryData.sRecords.ui32Head;
+
+	return sIter;
+}
+
+static IMG_BOOL MappingHistoryHasRecords(void)
+{
+	if ((gsMappingHistoryData.sRecords.ui32Head ==
+	     gsMappingHistoryData.sRecords.ui32Tail) &&
+	    !gsMappingHistoryData.sRecords.bOverwrite)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static IMG_BOOL MappingHistoryIteratorNext(MAPPING_HISTORY_ITERATOR *psIter)
+{
+	psIter->ui32Current = (psIter->ui32Current + 1) % MAPPING_HISTORY_CB_NUM_ENTRIES;
+
+	if (psIter->ui32Current == psIter->ui32Finish)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+#define MAPPING_ITER_TO_PTR(psIter) (&(gsMappingHistoryData.sRecords.pasMapRecordsCB[psIter.ui32Current]))
+
+static MAPPING_RECORD* AcquireMappingHistoryCBSlot(void)
+{
+	MAPPING_RECORD *psSlot;
+
+	psSlot = &gsMappingHistoryData.sRecords.pasMapRecordsCB[gsMappingHistoryData.sRecords.ui32Head];
+
+	gsMappingHistoryData.sRecords.ui32Head =
+	    (gsMappingHistoryData.sRecords.ui32Head + 1)
+	    % MAPPING_HISTORY_CB_NUM_ENTRIES;
+
+	if (!gsMappingHistoryData.sRecords.bOverwrite)
+	{
+		if (gsMappingHistoryData.sRecords.ui32Head == gsMappingHistoryData.sRecords.ui32Tail)
+		{
+			gsMappingHistoryData.sRecords.bOverwrite = IMG_TRUE;
+		}
+	}
+	else
+	{
+		gsMappingHistoryData.sRecords.ui32Tail = gsMappingHistoryData.sRecords.ui32Head;
+	}
+
+	return psSlot;
+}
+
+static IMG_CHAR* CacheFlagsToStr(IMG_UINT32 ui32CPUCacheFlags)
+{
+	switch (ui32CPUCacheFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			return "UC";
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			return "WC";
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+			return "CA";
+
+		default:
+			return "UN";
+	}
+}
+
+static void MapRecordString(MAPPING_RECORD *psRecord,
+                            IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(*pszBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+	           "%-32s %-8u 0x%p "CPUPHYADDR_UINT_FMTSPEC" %-5u (%-2s) %-12lu %-8u",
+	           psRecord->u.sMapData.aszAnnotation,
+	           psRecord->u.sMapData.uiPID,
+	           psRecord->u.sMapData.pvAddress,
+	           CPUPHYADDR_FMTARG(psRecord->u.sMapData.sCpuPhyAddr.uiAddr),
+	           psRecord->u.sMapData.ui32CPUCacheFlags,
+	           CacheFlagsToStr(psRecord->u.sMapData.ui32CPUCacheFlags),
+	           (unsigned long) psRecord->u.sMapData.uiMapOffset,
+	           psRecord->u.sMapData.ui32PageCount);
+}
+
+static void UnMapRecordString(MAPPING_RECORD *psRecord,
+                              IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(*pszBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+	           "%-41s 0x%p "CPUPHYADDR_UINT_FMTSPEC" %-5u (%-2s) %-12s %-8u",
+	           "UnMap", /* PADDING */
+	           psRecord->u.sUnmapData.pvAddress,
+	           CPUPHYADDR_FMTARG(psRecord->u.sUnmapData.sCpuPhyAddr.uiAddr),
+	           psRecord->u.sUnmapData.ui32CPUCacheFlags,
+	           CacheFlagsToStr(psRecord->u.sUnmapData.ui32CPUCacheFlags),
+	           "", /* PADDING */
+	           psRecord->u.sUnmapData.ui32PageCount);
+}
+
+static void MappingHistoryGetHeaderString(IMG_CHAR (*pszBuffer)[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(*pszBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+	           "%-32s %-8s %-18s %-18s %-9s %-8s %-8s",
+	           "PMRAnnotation",
+	           "PID",
+	           "CpuVirtAddr",
+	           "CpuPhyAddr",
+	           "CacheFlags",
+	           "PMRMapOffset",
+	           "ui32PageCount");
+}
+
+static void MappingHistoryPrintAll(OSDI_IMPL_ENTRY *psEntry)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	MAPPING_HISTORY_ITERATOR sIter = CreateMappingHistoryIterator();
+
+	if (!MappingHistoryHasRecords())
+	{
+		DIPrintf(psEntry, "No Records...\n");
+		return;
+	}
+
+	MappingHistoryGetHeaderString(&szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	do
+	{
+		MAPPING_RECORD *psRecord = MAPPING_ITER_TO_PTR(sIter);
+		psRecord->pfnRecordString(psRecord, &szBuffer);
+		DIPrintf(psEntry, "%s\n", szBuffer);
+
+	}
+	while (MappingHistoryIteratorNext(&sIter));
+}
+
+static int MappingHistoryPrintAllWrapper(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	MappingHistoryLock();
+	MappingHistoryPrintAll(psEntry);
+	MappingHistoryUnlock();
+
+	return 0;
+}
+
+static void MappingHistoryDebugDump(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                    IMG_UINT32 ui32VerbLevel,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile)
+{
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MAX))
+	{
+		MAPPING_HISTORY_ITERATOR sIter = CreateMappingHistoryIterator();
+		IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+
+		PVR_UNREFERENCED_PARAMETER(hDebugRequestHandle);
+
+		PVR_DUMPDEBUG_LOG("------[ Physmem CPU Map History ]------");
+
+		if (!MappingHistoryHasRecords())
+		{
+			PVR_DUMPDEBUG_LOG("No Records...");
+			return;
+		}
+
+		MappingHistoryGetHeaderString(&szBuffer);
+		PVR_DUMPDEBUG_LOG("%s", szBuffer);
+
+		do
+		{
+			MAPPING_RECORD *psRecord = MAPPING_ITER_TO_PTR(sIter);
+			psRecord->pfnRecordString(psRecord, &szBuffer);
+			PVR_DUMPDEBUG_LOG("%s", szBuffer);
+		}
+		while (MappingHistoryIteratorNext(&sIter));
+	}
+}
+
+PVRSRV_ERROR CPUMappingHistoryInit(void)
+{
+	PVRSRV_ERROR eError;
+	DI_ITERATOR_CB sIterator =
+	{ .pfnShow = MappingHistoryPrintAllWrapper };
+
+	eError = OSLockCreate(&gsMappingHistoryData.hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", error);
+
+	eError = CreateMappingRecords();
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateMappingRecords", error);
+
+	InitMappingRecordCB();
+
+	eError = DICreateEntry("cpumap_history", NULL, &sIterator, NULL,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &gsMappingHistoryData.psDIEntry);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", error);
+
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&gsMappingHistoryData.hDbgNotifier,
+	                                              MappingHistoryDebugDump,
+	                                              DEBUG_REQUEST_SRV,
+	                                              NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterDriverDbgRequestNotify", error);
+
+	return PVRSRV_OK;
+
+error:
+	CPUMappingHistoryDeInit();
+	return eError;
+}
+
+void CPUMappingHistoryDeInit(void)
+{
+	if (gsMappingHistoryData.hDbgNotifier != NULL)
+	{
+		PVRSRVUnregisterDriverDbgRequestNotify(gsMappingHistoryData.hDbgNotifier);
+		gsMappingHistoryData.hDbgNotifier = NULL;
+	}
+
+	if (gsMappingHistoryData.psDIEntry != NULL)
+	{
+		DIDestroyEntry(gsMappingHistoryData.psDIEntry);
+		gsMappingHistoryData.psDIEntry = NULL;
+	}
+
+	if (gsMappingHistoryData.sRecords.pasMapRecordsCB)
+	{
+		DestroyMappingRecords();
+		gsMappingHistoryData.sRecords.pasMapRecordsCB = NULL;
+	}
+
+	if (gsMappingHistoryData.hLock != NULL)
+	{
+		OSLockDestroy(gsMappingHistoryData.hLock);
+		gsMappingHistoryData.hLock = NULL;
+	}
+}
+
+void InsertMappingRecord(const IMG_CHAR *pszAnnotation,
+                         IMG_PID uiPID,
+                         IMG_CPU_VIRTADDR pvAddress,
+                         IMG_CPU_PHYADDR sCpuPhyAddr,
+                         IMG_UINT32 ui32CPUCacheFlags,
+                         size_t uiMapOffset,
+                         IMG_UINT32 ui32PageCount)
+{
+	MAPPING_RECORD *psRecord;
+
+	psRecord = AcquireMappingHistoryCBSlot();
+	psRecord->etype = MAP;
+	psRecord->pfnRecordString = MapRecordString;
+
+	OSStringSafeCopy(psRecord->u.sMapData.aszAnnotation, pszAnnotation, MAX_MAPPING_ANNOT_STR);
+	psRecord->u.sMapData.uiPID = uiPID;
+	psRecord->u.sMapData.pvAddress = pvAddress;
+	psRecord->u.sMapData.ui32CPUCacheFlags = ui32CPUCacheFlags;
+	psRecord->u.sMapData.sCpuPhyAddr = sCpuPhyAddr;
+	psRecord->u.sMapData.uiMapOffset = uiMapOffset;
+	psRecord->u.sMapData.ui32PageCount = ui32PageCount;
+}
+
+void InsertUnMappingRecord(IMG_CPU_VIRTADDR pvAddress,
+                           IMG_CPU_PHYADDR sCpuPhyAddr,
+                           IMG_UINT32 ui32CPUCacheFlags,
+                           IMG_UINT32 ui32PageCount)
+{
+	MAPPING_RECORD *psRecord;
+
+	psRecord = AcquireMappingHistoryCBSlot();
+	psRecord->etype = UNMAP;
+	psRecord->pfnRecordString = UnMapRecordString;
+
+	psRecord->u.sUnmapData.pvAddress = pvAddress;
+	psRecord->u.sUnmapData.ui32CPUCacheFlags = ui32CPUCacheFlags;
+	psRecord->u.sUnmapData.sCpuPhyAddr = sCpuPhyAddr;
+	psRecord->u.sUnmapData.ui32PageCount = ui32PageCount;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_dlm.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_dlm.c
new file mode 100644
index 000000000000..07a92b2973df
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_dlm.c
@@ -0,0 +1,341 @@
+/*************************************************************************/ /*!
+@File           physmem_dlm.c
+@Title          Dedicated Local Memory allocator for Physical Memory Blocks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for Dedicated local memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+#include "physmem_dlm.h"
+
+typedef struct PHYSMEM_DLM_DATA_TAG {
+	RA_ARENA           *psRA;
+	IMG_CPU_PHYADDR    sStartAddr;
+	IMG_DEV_PHYADDR    sCardBase;
+	IMG_UINT64         uiSize;
+	IMG_UINT32         uiLog2PMBSize;
+} PHYSMEM_DLM_DATA;
+
+/* PMB (Physical Memory Block) */
+struct _PMB_
+{
+	RA_ARENA *pArena;
+	RA_BASE_T uiBase;
+	RA_LENGTH_T uiSize;
+	const IMG_CHAR *pszAnnotation;
+};
+
+/* PMBCreatePMB
+ *
+ * Creates a new PMB used to represent a block of memory
+ * obtained from a DLM heap.
+ */
+static
+PVRSRV_ERROR PMBCreatePMB(RA_ARENA *pArena,
+                          RA_LENGTH_T uiSize,
+                          RA_LENGTH_T uiAlignment,
+                          const IMG_CHAR *pszAnnotation,
+                          PMB **ppsPMB)
+{
+	PVRSRV_ERROR eError;
+	PMB* psPMB = OSAllocMem(sizeof(*psPMB));
+	PVR_LOG_GOTO_IF_NOMEM(psPMB, eError, error_Return);
+
+	eError = RA_Alloc(pArena,
+	                  uiSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                         /* No flags */
+	                  uiAlignment,
+	                  pszAnnotation,
+	                  &psPMB->uiBase,
+	                  &psPMB->uiSize,
+	                  NULL);                     /* No private handle */
+	PVR_LOG_GOTO_IF_ERROR(eError, "RA_Alloc", error_FreePMB);
+
+	psPMB->pArena = pArena;
+	psPMB->pszAnnotation = pszAnnotation;
+
+	*ppsPMB = psPMB;
+
+	return PVRSRV_OK;
+
+error_FreePMB:
+	OSFreeMem(psPMB);
+error_Return:
+	return eError;
+}
+
+void
+PMBDestroy(PMB *psPMB)
+{
+	PVR_LOG_RETURN_VOID_IF_FALSE(psPMB, "psPMB NULL");
+
+	RA_Free(psPMB->pArena, psPMB->uiBase);
+	OSFreeMem(psPMB);
+}
+
+
+const IMG_CHAR *
+PMBGetAnnotation(PMB *psPMB)
+{
+	if (psPMB == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"psPMB in %s",__func__));
+		return "";
+	}
+
+	return psPMB->pszAnnotation;
+}
+
+/* DLM */
+
+static void PFNGetLocalRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                                 IMG_UINT64 *pui64TotalSize,
+                                 IMG_UINT64 *pui64FreeSize)
+{
+	RA_USAGE_STATS sRAUsageStats;
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_VOID_IF_FALSE(pvImplData, "pvImplData NULL");
+
+	RA_Get_Usage_Stats(psDLMData->psRA, &sRAUsageStats);
+
+	*pui64TotalSize = sRAUsageStats.ui64TotalArenaSize;
+	*pui64FreeSize = sRAUsageStats.ui64FreeArenaSize;
+}
+
+/*
+* This function will set the psDevPAddr to whatever the system layer
+* has set it for the referenced heap.
+* It will not fail if the psDevPAddr is invalid.
+*/
+static PVRSRV_ERROR
+PFNGetDevPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvImplData != NULL, "pvImplData");
+
+	*psDevPAddr = psDLMData->sCardBase;
+
+	return PVRSRV_OK;
+}
+
+/*
+* This function will set the psCpuPAddr to whatever the system layer
+* has set it for the referenced heap.
+* It will not fail if the psCpuPAddr is invalid.
+*/
+static PVRSRV_ERROR
+PFNGetCPUPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvImplData != NULL, "pvImplData");
+
+	*psCpuPAddr = psDLMData->sStartAddr;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PFNGetSize(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *puiSize)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvImplData != NULL, "pvImplData");
+
+	*puiSize = psDLMData->uiSize;
+
+	return PVRSRV_OK;
+}
+
+static IMG_UINT32
+PFNGetPageShift(void)
+{
+	return PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT;
+}
+
+static PVRSRV_ERROR
+CreateArenas(PHYSMEM_DLM_DATA *psDLMData, IMG_CHAR *pszLabel, PHYS_HEAP_POLICY uiPolicy)
+{
+	psDLMData->psRA = RA_Create_With_Span(pszLabel,
+	                             OSGetPageShift(),
+	                             psDLMData->sStartAddr.uiAddr,
+	                             psDLMData->sCardBase.uiAddr,
+	                             psDLMData->uiSize,
+	                             RA_POLICY_DEFAULT);
+	PVR_LOG_RETURN_IF_NOMEM(psDLMData->psRA, "RA_Create_With_Span");
+
+	return PVRSRV_OK;
+}
+
+static void
+DestroyArenas(PHYSMEM_DLM_DATA *psDLMData)
+{
+	/* Remove RAs and RA names for dedicated local memory */
+	if (psDLMData->psRA)
+	{
+		RA_Delete(psDLMData->psRA);
+		psDLMData->psRA = NULL;
+	}
+}
+
+static void
+PFNDestroyImplData(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_DLM_DATA *psDLMData = (PHYSMEM_DLM_DATA*)pvImplData;
+	PVR_LOG_RETURN_VOID_IF_FALSE(pvImplData, "pvImplData NULL");
+
+	DestroyArenas(pvImplData);
+
+	OSFreeMem(psDLMData);
+}
+
+static PVRSRV_ERROR
+PFNPhysmemNewLocalRamBackedPMB(PHYS_HEAP *psPhysHeap,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            const IMG_CHAR *pszAnnotation,
+                            PMB **ppPMBPtr,
+                            RA_BASE_T *puiBase,
+                            RA_LENGTH_T *puiSize)
+{
+	PHYSMEM_DLM_DATA *psDLMData;
+	PVRSRV_ERROR eError;
+	PMB* pPMB;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppPMBPtr != NULL, "ppPMBPtr");
+
+	/* Check size is aligned to page size */
+	if (uiSize & ((1 << PFNGetPageShift()) - 1ULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: uiSize %" IMG_UINT64_FMTSPEC " is not aligned to page size: %u",
+		         __func__,
+		         uiSize,
+		         1 << PFNGetPageShift()));
+		return PVRSRV_ERROR_PMB_NOT_PAGE_MULTIPLE;
+	}
+
+	PVR_ASSERT(PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_DLM);
+
+	psDLMData = (PHYSMEM_DLM_DATA*)PhysHeapGetImplData(psPhysHeap);
+
+	eError = PMBCreatePMB(psDLMData->psRA, uiSize, IMG_UINT64_C(1) << psDLMData->uiLog2PMBSize, pszAnnotation, &pPMB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMBCreatePMB", error_Return);
+
+	*ppPMBPtr = pPMB;
+	*puiBase = pPMB->uiBase;
+	*puiSize = pPMB->uiSize;
+
+	return PVRSRV_OK;
+
+error_Return:
+	return eError;
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = &PFNDestroyImplData,
+	.pfnGetDevPAddr = &PFNGetDevPAddr,
+	.pfnGetCPUPAddr = &PFNGetCPUPAddr,
+	.pfnGetSize = &PFNGetSize,
+	.pfnGetPageShift = &PFNGetPageShift,
+	.pfnGetFactoryMemStats = &PFNGetLocalRamMemStats,
+	.pfnCreatePMB = &PFNPhysmemNewLocalRamBackedPMB,
+};
+
+PVRSRV_ERROR
+PhysmemCreateHeapDLM(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap)
+{
+	PHYSMEM_DLM_DATA *psDLMData;
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psConfig != NULL, "psConfig");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszLabel != NULL, "pszLabel");
+
+	PVR_ASSERT(psConfig->eType == PHYS_HEAP_TYPE_DLM);
+
+	psDLMData = OSAllocMem(sizeof(*psDLMData));
+	PVR_LOG_RETURN_IF_NOMEM(psDLMData, "OSAllocMem");
+
+	psDLMData->sStartAddr = PhysHeapConfigGetStartAddr(psConfig);
+	psDLMData->sCardBase = PhysHeapConfigGetCardBase(psConfig);
+	psDLMData->uiSize = PhysHeapConfigGetSize(psConfig);
+	psDLMData->uiLog2PMBSize = psConfig->uConfig.sDLM.ui32Log2PMBSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDLMData->uiLog2PMBSize >= OSGetPageShift(), "ui32Log2PMBSize must be greater than or equal to OSPageSize");
+
+	eError = PhysHeapCreate(psDevNode,
+							psConfig,
+							uiPolicy,
+							(PHEAP_IMPL_DATA)psDLMData,
+							&_sPHEAPImplFuncs,
+							&psPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreate", error_FreeDlmData);
+
+	eError = CreateArenas(psDLMData, pszLabel, uiPolicy);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateArenas", error_FreePhysHeap);
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psPhysHeap;
+	}
+
+	return PVRSRV_OK;
+
+error_FreePhysHeap:
+	PhysHeapDestroy(psPhysHeap);
+error_FreeDlmData:
+	OSFreeMem(psDLMData);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_extmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_extmem.c
new file mode 100644
index 000000000000..d365524f9a9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_extmem.c
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File           physmem_extmem.c
+@Title          Physmem - External Memory Import Support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common entry point for Importing Externally allocated Memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv.h"
+#include "physmem_extmem.h"
+
+
+#if defined(LMA)
+#pragma message "WARNING! You are compiling SUPPORT_WRAP_EXTMEM on a platform with LMA. " \
+	"Make sure the memory you wrap is actually accessible by the GPU!"
+#endif
+
+PVRSRV_ERROR
+PhysmemWrapExtMem(CONNECTION_DATA * psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT64 pvCpuVAddr,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  PMR **ppsPMRPtr)
+{
+	return PhysmemWrapExtMemOS(psConnection,
+	                           psDevNode,
+	                           uiSize,
+	                           (IMG_CPU_VIRTADDR)(uintptr_t)pvCpuVAddr,
+	                           uiFlags,
+	                           ppsPMRPtr);
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_hostmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_hostmem.c
new file mode 100644
index 000000000000..6c87632bbea5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_hostmem.c
@@ -0,0 +1,213 @@
+/*************************************************************************/ /*!
+@File           physmem_hostmem.c
+@Title          Host memory device node functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Functions relevant to device memory allocations made from host
+                mem device node.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "physmem_hostmem.h"
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "pvrsrv_device.h"
+#include "physheap.h"
+#include "physmem_osmem.h"
+
+static void HostMemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void HostMemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr);
+
+/* heap callbacks for host driver's device's heap */
+static PHYS_HEAP_FUNCTIONS gsHostMemDevPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	HostMemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	HostMemDevPAddrToCpuPAddr,
+};
+
+static PVRSRV_DEVICE_CONFIG gsHostMemDevConfig[];
+
+/* heap configuration for host driver's device */
+static PHYS_HEAP_CONFIG gsPhysHeapConfigHostMemDevice[] =
+{
+	{
+		.eType = PHYS_HEAP_TYPE_UMA,
+		.ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL,
+		.uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM",
+		.uConfig.sUMA.psMemFuncs = &gsHostMemDevPhysHeapFuncs,
+		.uConfig.sUMA.pszHeapName = "uma_cpu_local",
+		.uConfig.sUMA.hPrivData = (IMG_HANDLE)&gsHostMemDevConfig[0]
+	}
+};
+
+/* device configuration for host driver's device */
+static PVRSRV_DEVICE_CONFIG gsHostMemDevConfig[] =
+{
+	{
+		.pszName = "HostMemDevice",
+		.eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE,
+		.pasPhysHeaps = &gsPhysHeapConfigHostMemDevice[0],
+		.ui32PhysHeapCount = ARRAY_SIZE(gsPhysHeapConfigHostMemDevice),
+	}
+};
+
+static void HostMemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void HostMemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+}
+
+PVRSRV_ERROR HostMemDeviceCreate(PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = &gsHostMemDevConfig[0];
+
+	/* Assert ensures HostMemory device isn't already created and
+	 * that data is initialised */
+	PVR_ASSERT(*ppsDeviceNode == NULL);
+
+	/* for now, we only know a single heap (UMA) config for host device */
+	PVR_ASSERT(psDevConfig->ui32PhysHeapCount == 1 &&
+				psDevConfig->pasPhysHeaps[0].eType == PHYS_HEAP_TYPE_UMA);
+
+	/* N.B.- In case of any failures in this function, we just return error to
+	   the caller, as clean-up is taken care by _HostMemDeviceDestroy function */
+
+	psDeviceNode = OSAllocZMem(sizeof(*psDeviceNode) + sizeof(*psDeviceNode->psMMUDevAttrs));
+	PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "OSAllocZMem");
+
+	/* early save return pointer to aid clean-up */
+	*ppsDeviceNode = psDeviceNode;
+
+	psDeviceNode->sDevId.ui32InternalID = PVRSRV_HOST_DEVICE_ID;
+	psDeviceNode->psDevConfig = psDevConfig;
+	psDeviceNode->psPhysHeapList = NULL;
+
+	eError = OSLockCreate(&psDeviceNode->hPhysHeapLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", LockCreateErr);
+
+	eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+										  &psDevConfig->pasPhysHeaps[0],
+										  NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig", PhysHeapCreateErr);
+
+	/* Only CPU local heap is valid on host-mem DevNode, so enable minimal callbacks */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_CPU_LOCAL,
+								 psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire", AcquirePhysHeapErr);
+
+	psDeviceNode->psMMUDevAttrs = (MMU_DEVICEATTRIBS*)(psDeviceNode + 1);
+	psDeviceNode->psMMUDevAttrs->ui32ValidPageSizeMask = OSGetPageSize();
+
+	dllist_init(&psDeviceNode->sCleanupThreadWorkList);
+
+	return PVRSRV_OK;
+
+AcquirePhysHeapErr:
+	PhysHeapDestroyDeviceHeaps(psDeviceNode);
+PhysHeapCreateErr:
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+LockCreateErr:
+	OSFreeMem(psDeviceNode);
+
+	return eError;
+}
+
+void HostMemDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (!psDeviceNode)
+	{
+		return;
+	}
+	else
+	{
+		if (psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL])
+		{
+			PhysHeapRelease(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL]);
+		}
+
+		PhysHeapDestroyDeviceHeaps(psDeviceNode);
+	}
+
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+
+	OSFreeMem(psDeviceNode);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ima.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ima.c
new file mode 100644
index 000000000000..87595e1c8391
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ima.c
@@ -0,0 +1,655 @@
+/*************************************************************************/ /*!
+@File           physmem_ima.c
+@Title          Import memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of memory management. This module is responsible for
+                implementing the function callbacks for local card memory when
+                used under a shared heap system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "device.h"
+#include "osfunc.h"
+#include "physmem_ima.h"
+#include "physmem_dlm.h"
+#include "physmem_ramem.h"
+
+typedef struct PHYSMEM_IMA_DATA_TAG {
+	PHYS_HEAP          *psPhysHeap;
+	RA_ARENA           *psRA;
+	PHYS_HEAP          *pDLMHeap;
+	IMG_UINT32          uiLog2PMBSize;
+	IMG_UINT32          uiReservedPMBs;
+	PMB               **ppsReservedPMBs;
+} PHYSMEM_IMA_DATA;
+
+static PVRSRV_ERROR
+IMAGetDevPAddr(PHEAP_IMPL_DATA pvImplData,
+               IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_ARENA_ITERATOR *psRAIter = RA_IteratorAcquire(psIMAData->psRA, IMG_FALSE);
+	RA_ITERATOR_DATA sData = {0};
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_NOMEM(psRAIter, "RA_IteratorAcquire");
+
+	if (!RA_IteratorNext(psRAIter, &sData))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("RA_IteratorNext",
+		                        eError,
+		                        PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR,
+		                        err_free_iter);
+	}
+
+	psDevPAddr->uiAddr = sData.uiAddr;
+
+err_free_iter:
+	RA_IteratorRelease(psRAIter);
+	return eError;
+}
+
+static PVRSRV_ERROR
+IMAGetCPUPAddr(PHEAP_IMPL_DATA pvImplData,
+               IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	IMG_DEV_PHYADDR sDevPAddr;
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	PVRSRV_ERROR eError = IMAGetDevPAddr(pvImplData, &sDevPAddr);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IMAGetDevPAddr");
+
+	PhysHeapDevPAddrToCpuPAddr(
+		psIMAData->pDLMHeap,
+		1,
+		psCpuPAddr,
+		&sDevPAddr
+	);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+IMAGetSize(PHEAP_IMPL_DATA pvImplData,
+           IMG_UINT64 *puiSize)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_USAGE_STATS sRAUsageStats;
+
+	RA_Get_Usage_Stats(psIMAData->psRA, &sRAUsageStats);
+
+	*puiSize = sRAUsageStats.ui64TotalArenaSize;
+
+	return PVRSRV_OK;
+}
+
+static void IMAPhysmemGetRAMemRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                                          IMG_UINT64 *pui64TotalSize,
+                                          IMG_UINT64 *pui64FreeSize)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_USAGE_STATS sRAUsageStats;
+
+	RA_Get_Usage_Stats(psIMAData->psRA, &sRAUsageStats);
+
+	*pui64TotalSize = sRAUsageStats.ui64TotalArenaSize;
+	*pui64FreeSize = sRAUsageStats.ui64FreeArenaSize;
+}
+
+/*
+ * We iterate on this function multiple times using the IterHandle.
+ * First iteration will create the IterHandle.
+ * We return IMG_TRUE when there are iterations remaining.
+ * We return IMG_FALSE when there are no iterations remaining, this
+ * will also free the IterHandle.
+ * Callers must always iterate until false to ensure IterHandle is
+ * freed.
+ * */
+static IMG_BOOL IMAGetHeapSpansStringIter(PHEAP_IMPL_DATA pvImplData,
+                                          IMG_CHAR *ppszStrBuf,
+                                          IMG_UINT32 uiStrBufSize,
+                                          void **ppvIterHandle)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	RA_ITERATOR_DATA sData = {0};
+	PVRSRV_ERROR eError;
+	IMG_BOOL bIter = IMG_FALSE;
+
+	/* If we haven't been given an IterHandle */
+	if (!*ppvIterHandle)
+	{
+		*ppvIterHandle = RA_IteratorAcquire(psIMAData->psRA, IMG_TRUE);
+		PVR_LOG_GOTO_IF_NOMEM(*ppvIterHandle, eError, return_false);
+	}
+
+	bIter = RA_IteratorNextSpan(*ppvIterHandle, &sData);
+	if (bIter)
+	{
+		IMG_DEV_PHYADDR sRangeBase = {sData.uiAddr};
+		IMG_CHAR aszCpuAddr[18] = "Non-Addressable"; /* length of CPUPHYADDR_UINT_FMTSPEC */
+		IMG_UINT64 uiRangeSize = sData.uiSize;
+		IMG_INT32 iCount;
+
+		/* Cannot get the CPU addr for private heaps. Display "Non-Addressable" instead. */
+		if ((PhysHeapGetFlags(psIMAData->psPhysHeap) & PHYS_HEAP_USAGE_GPU_PRIVATE) != PHYS_HEAP_USAGE_GPU_PRIVATE)
+		{
+			IMG_CPU_PHYADDR sCPURangeBase;
+			PhysHeapDevPAddrToCpuPAddr(psIMAData->pDLMHeap,
+			                           1,
+			                           &sCPURangeBase,
+			                           &sRangeBase);
+			OSSNPrintf(aszCpuAddr, ARRAY_SIZE(aszCpuAddr), CPUPHYADDR_UINT_FMTSPEC, CPUPHYADDR_FMTARG(sCPURangeBase.uiAddr));
+		}
+
+		iCount = OSSNPrintf(ppszStrBuf,
+							uiStrBufSize,
+							"                            " /* padding */
+							"CPU PA Base: %s, "
+							"GPU PA Base: 0x%08"IMG_UINT64_FMTSPECx", "
+							"Size: %"IMG_UINT64_FMTSPEC"B",
+							aszCpuAddr,
+							sRangeBase.uiAddr,
+							uiRangeSize);
+		if (!(0 < iCount && iCount < (IMG_INT32)uiStrBufSize))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "OSSNPrintf in %s(), "
+			                        "Heap Span print may be corrupt!", __func__));
+		}
+		return IMG_TRUE;
+	}
+	/* else end iteration and free the iter handle */
+
+	RA_IteratorRelease(*ppvIterHandle);
+
+return_false:
+	return IMG_FALSE;
+}
+
+static void
+IMAGetHeapDLMBacking(PHEAP_IMPL_DATA pvImplData,
+                     PHYS_HEAP **psDLMPhysHeap)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+	*psDLMPhysHeap = psIMAData->pDLMHeap;
+}
+
+/* When uiSize > DLM PMB size, allocates a single physically contiguous Huge PMB to satisfy the import.
+ * A huge PMB is a PMB with a size larger than the DLM PMB size (1 << psIMAData->uiLog2PMBSize).
+ * The requested import size will be rounded up to a multiple of the DLM PMB size. */
+static PVRSRV_ERROR IMAImportDLMAllocHuge(RA_PERARENA_HANDLE hArenaHandle,
+                                          RA_LENGTH_T uiSize,
+                                          RA_FLAGS_T uiFlags,
+                                          RA_LENGTH_T uBaseAlignment,
+                                          const IMG_CHAR *pszAnnotation,
+                                          RA_IMPORT *psImport)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*) hArenaHandle;
+	PHYS_HEAP *psPhysHeap = psIMAData->pDLMHeap;
+	PMB *pPMB;
+	IMG_UINT64 uiPMBSizeBytes;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+	PVR_ASSERT(psIMAData->uiLog2PMBSize != 0);
+	/* Round up the allocation size to the nearest multiple of the
+	 * DLM PMB size (1 << psIMAData->uiLog2PMBSize).
+	 * If this is greater than DLM PMB size, a huge PMB will be imported. */
+	uiPMBSizeBytes = PVR_ALIGN(uiSize, IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize);
+
+	eError = PhysHeapCreatePMB(psPhysHeap,
+	                           uiPMBSizeBytes,
+	                           pszAnnotation,
+	                           &pPMB,
+	                           &psImport->base,
+	                           &psImport->uSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreatePMB");
+
+	psImport->hPriv = pPMB;
+
+	return PVRSRV_OK;
+}
+
+/* When uiSize <= DLM PMB size, this returns a single PMB, otherwise
+ * it requests multiple PMBs from the connected DLM heap.
+ * The PMBs will be RA_Free'd once there are no busy segments inside of the span.
+ * This will trigger the normal imp_free callback.
+ * This strategy relies on RA_POLICY_ALLOC_ALLOW_NONCONTIG
+ * and that all spans in the IMA heap are all the same size of 1 PMB, otherwise
+ * we might add a span that is not used, invalidating the invariants of the RA. */
+static PVRSRV_ERROR IMAImportDLMAllocMulti(RA_PERARENA_HANDLE hArenaHandle,
+                                           RA_LENGTH_T uiSize,
+                                           RA_FLAGS_T uiFlags,
+                                           RA_LENGTH_T uBaseAlignment,
+                                           const IMG_CHAR *pszAnnotation,
+                                           IMG_UINT32 *puiImportsCount,
+                                           RA_IMPORT **ppsImports)
+{
+	PVRSRV_ERROR eError;
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*) hArenaHandle;
+	PHYS_HEAP *psPhysHeap = psIMAData->pDLMHeap;
+
+	IMG_UINT32 i;
+	IMG_UINT32 uiPMBCount;
+
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+	PVR_ASSERT(psIMAData->uiLog2PMBSize != 0);
+
+	/* Round up the allocation size to the nearest multiple of the
+	 * DLM PMB size (1 << psIMAData->uiLog2PMBSize). */
+	uiSize = PVR_ALIGN(uiSize, IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize);
+	/* Get the number of PMBs required to make the allocation. */
+	uiPMBCount = uiSize >> psIMAData->uiLog2PMBSize;
+
+	/* Try use the provided ppsImports array instead of allocating a new array
+	 * if the array is large enough to hold all the new PMBs. */
+	if (uiPMBCount > *puiImportsCount)
+	{
+		*ppsImports = OSAllocMem(uiPMBCount * sizeof(**ppsImports));
+		PVR_LOG_RETURN_IF_NOMEM(ppsImports, "ppsImports");
+	}
+	*puiImportsCount = uiPMBCount;
+
+	/* Create the PMB that will be become the spans. */
+	for (i = 0; i < uiPMBCount; i++)
+	{
+		RA_IMPORT *psImport = &(*ppsImports)[i];
+		eError = PhysHeapCreatePMB(psPhysHeap,
+		                           IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize,
+		                           pszAnnotation,
+		                           (PMB**) &psImport->hPriv,
+		                           &psImport->base,
+		                           &psImport->uSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreatePMB", err_FreePMBs);
+	}
+
+	return PVRSRV_OK;
+
+/* Free i PMBs from the array. */
+err_FreePMBs:
+	uiPMBCount = i;
+	for (i = 0; i < uiPMBCount; i++)
+	{
+		PMBDestroy((*ppsImports)[i].hPriv);
+	}
+	return eError;
+}
+
+static void IMAImportDLMFree(RA_PERARENA_HANDLE hArenaHandle,
+                             RA_BASE_T uiBase,
+                             RA_PERISPAN_HANDLE hPriv)
+{
+	PMB *pPMB = (PMB*) hPriv;
+	PVR_ASSERT(pPMB != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(hArenaHandle);
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+
+	PMBDestroy(pPMB);
+}
+
+static void
+FreeReservedMemory(PHYSMEM_IMA_DATA *psIMAData)
+{
+	IMG_UINT32 i;
+	if (psIMAData->ppsReservedPMBs)
+	{
+		for (i = 0; i < psIMAData->uiReservedPMBs; i++)
+		{
+			/* The PMBs will be RA_Free'd by RA_Delete,
+			 * so it doesn't need to be done manually */
+			PMBDestroy(psIMAData->ppsReservedPMBs[i]);
+		}
+		OSFreeMem(psIMAData->ppsReservedPMBs);
+	}
+}
+
+static PVRSRV_ERROR
+AllocateReservedMemory(PHYSMEM_IMA_DATA *psIMAData,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       IMG_UINT32 uiReservedPMBs)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bSuccess;
+	IMG_UINT32 i;
+
+	IMG_UINT64 uiPMBSize = IMG_UINT64_C(1) << psIMAData->uiLog2PMBSize;
+	IMG_UINT32 uiPMBCount = uiReservedPMBs;
+
+	RA_BASE_T uiBase;
+	RA_LENGTH_T uiReserveActSize;
+
+	if (uiReservedPMBs == 0)
+	{
+		psIMAData->uiReservedPMBs = 0;
+		psIMAData->ppsReservedPMBs = NULL;
+		return PVRSRV_OK;
+	}
+
+	/* If we do not support non-contiguous, we can allocate a single huge PMB.
+	 * We cannot do this for non-contiguous as IMAImportDLMAllocMulti requires all spans in the IMA-RA
+	 * to be exactly the same: 1 << psIMAData->uiLog2PMBSize. */
+	if ((uiPolicy & PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	{
+		uiPMBSize *= uiPMBCount;
+		uiPMBCount = 1;
+	}
+
+	/* Count the number of PMBs that have been successfully created. */
+	psIMAData->uiReservedPMBs = 0;
+	psIMAData->ppsReservedPMBs = OSAllocMem(uiPMBCount * sizeof(*psIMAData->ppsReservedPMBs));
+	PVR_LOG_RETURN_IF_NOMEM(psIMAData->ppsReservedPMBs, "OSAllocMem");
+
+	for (i = 0; i < uiPMBCount; i++)
+	{
+		eError = PhysHeapCreatePMB(psIMAData->pDLMHeap,
+		                           uiPMBSize,
+		                           "PMB Reserved",
+		                           &psIMAData->ppsReservedPMBs[i],
+		                           &uiBase,
+		                           &uiReserveActSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreatePMB", err_FreeReserved);
+
+		psIMAData->uiReservedPMBs++;
+
+		bSuccess = RA_Add(psIMAData->psRA,
+		                  uiBase,
+		                  uiReserveActSize,
+		                  0,
+		                  (RA_PERISPAN_HANDLE) psIMAData->ppsReservedPMBs[i]);
+		PVR_LOG_GOTO_IF_FALSE(bSuccess, "RA_Add", err_AddFailed);
+	}
+
+	return PVRSRV_OK;
+
+err_AddFailed:
+    eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+err_FreeReserved:
+	FreeReservedMemory(psIMAData);
+	return eError;
+}
+
+static PVRSRV_ERROR
+CreateIMAArena(PHYSMEM_IMA_DATA *psIMAData,
+               IMG_CHAR *pszLabel,
+               PHYS_HEAP_POLICY uiPolicy,
+               IMG_UINT32 uiReservedPMBs)
+{
+	/* In practice an IMA heap only differs from LMA in the fact it can import more memory
+	 * when it has expended its current extent. */
+
+	/* If non contiguous mapping is available then we should allow that for an IMA heap.*/
+	IMG_UINT32 ui32RAPolicy =
+	    ((uiPolicy & PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	    ? RA_POLICY_ALLOC_ALLOW_NONCONTIG : RA_POLICY_DEFAULT;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psIMAData != NULL);
+
+	if ((ui32RAPolicy & RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == RA_POLICY_ALLOC_ALLOW_NONCONTIG)
+	{
+		psIMAData->psRA = RA_CreateMulti(pszLabel,
+		                                 OSGetPageShift(),
+		                                 RA_LOCKCLASS_1,
+		                                 IMAImportDLMAllocMulti,
+		                                 IMAImportDLMFree,
+		                                 psIMAData,
+		                                 ui32RAPolicy);
+	}
+	else
+	{
+		psIMAData->psRA = RA_Create(pszLabel,
+		                            OSGetPageShift(),
+		                            RA_LOCKCLASS_1,
+		                            IMAImportDLMAllocHuge,
+		                            IMAImportDLMFree,
+		                            psIMAData,
+		                            ui32RAPolicy);
+	}
+	PVR_LOG_RETURN_IF_NOMEM(psIMAData->psRA, "RA_Create");
+
+	eError = AllocateReservedMemory(psIMAData, uiPolicy, uiReservedPMBs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "AllocateReservedMemory", err_ra_free);
+
+	return PVRSRV_OK;
+
+err_ra_free:
+	RA_Delete(psIMAData->psRA);
+	psIMAData->psRA = NULL;
+	return eError;
+}
+
+static void
+DestroyIMAArena(PHYSMEM_IMA_DATA *psIMAData)
+{
+	PVR_ASSERT(psIMAData != NULL);
+
+	/* Locked imports should be implicitly freed has a part of
+	 * the RA_Delete assuming there are no allocations remaining on
+	 * the import
+	 */
+	FreeReservedMemory(psIMAData);
+
+	/* Remove RAs and RA names for local card memory */
+	if (psIMAData->psRA)
+	{
+		RA_Delete(psIMAData->psRA);
+		psIMAData->psRA = NULL;
+	}
+}
+
+static void
+IMADestroyImplData(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)pvImplData;
+
+	DestroyIMAArena(pvImplData);
+
+	OSFreeMem(psIMAData);
+}
+
+static PVRSRV_ERROR
+PhysmemGetArenaIMA(PHYS_HEAP *psPhysHeap,
+                   RA_ARENA **ppsArena)
+{
+	PHYSMEM_IMA_DATA *psIMAData = (PHYSMEM_IMA_DATA*)PhysHeapGetImplData(psPhysHeap);
+
+	PVR_LOG_RETURN_IF_FALSE(psIMAData != NULL, "psIMAData", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	*ppsArena = psIMAData->psRA;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+IMAPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap,
+                       size_t uiSize,
+                       PG_HANDLE *psMemHandle,
+                       IMG_DEV_PHYADDR *psDevPAddr,
+                       IMG_PID uiPid)
+{
+	PVRSRV_ERROR eError;
+
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+
+	eError = PhysmemGetArenaIMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLocalMem");
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = (1 << ui32Log2NumPages) * OSGetPageSize();
+
+	eError = RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                    psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "LocalDoPhyContigPagesAlloc");
+
+	return eError;
+}
+
+static void
+IMAPhyContigPagesFree(PHYS_HEAP *psPhysHeap,
+                      PG_HANDLE *psMemHandle)
+{
+	RA_ARENA	*pArena;
+
+	PhysmemGetArenaIMA(psPhysHeap, &pArena);
+
+	RAMemDoPhyContigPagesFree(pArena,
+	                          psMemHandle);
+}
+
+static PVRSRV_ERROR
+IMAPhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                               CONNECTION_DATA *psConnection,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               IMG_UINT32 ui32NumPhysChunks,
+                               IMG_UINT32 ui32NumVirtChunks,
+                               IMG_UINT32 *pui32MappingTable,
+                               IMG_UINT32 uiLog2AllocPageSize,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               const IMG_CHAR *pszAnnotation,
+                               IMG_PID uiPid,
+                               PMR **ppsPMRPtr,
+                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	RA_ARENA *pArena;
+
+	eError = PhysmemGetArenaIMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaIMA");
+
+	eError = PhysmemNewRAMemRamBackedPMR(psPhysHeap,
+	                                     pArena,
+	                                     psConnection,
+	                                     uiSize,
+	                                     ui32NumPhysChunks,
+	                                     ui32NumVirtChunks,
+	                                     pui32MappingTable,
+	                                     uiLog2AllocPageSize,
+	                                     uiFlags,
+	                                     pszAnnotation,
+	                                     uiPid,
+	                                     ppsPMRPtr,
+	                                     ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewLocalRamBackedPMR");
+
+	return PVRSRV_OK;
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncsIMA =
+{
+	.pfnDestroyData = &IMADestroyImplData,
+	.pfnGetDevPAddr = &IMAGetDevPAddr,
+	.pfnGetCPUPAddr = &IMAGetCPUPAddr,
+	.pfnGetSize = &IMAGetSize,
+	.pfnGetPageShift = &RAMemGetPageShift,
+	.pfnGetFactoryMemStats = &IMAPhysmemGetRAMemRamMemStats,
+	.pfnGetHeapSpansStringIter = &IMAGetHeapSpansStringIter,
+	.pfnGetHeapDLMBacking = &IMAGetHeapDLMBacking,
+	.pfnCreatePMR = &IMAPhysmemNewRAMemRamBackedPMR,
+	.pfnPagesAlloc = &IMAPhyContigPagesAlloc,
+	.pfnPagesFree = &IMAPhyContigPagesFree,
+	.pfnPagesMap = &RAMemPhyContigPagesMap,
+	.pfnPagesUnMap = &RAMemPhyContigPagesUnmap,
+	.pfnPagesClean = &RAMemPhyContigPagesClean,
+};
+
+PVRSRV_ERROR
+PhysmemCreateHeapIMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP *psDLMHeap,
+                     IMG_UINT32 uiLog2PMBSize,
+                     PHYS_HEAP **ppsPhysHeap)
+{
+	PHYSMEM_IMA_DATA *psIMAData;
+	PHYS_HEAP *psPhysHeap;
+	IMG_UINT32 uiPMBStartingMultiple = psConfig->uConfig.sIMA.ui32PMBStartingMultiple;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psConfig != NULL, "psConfig");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszLabel != NULL, "pszLabel");
+
+	PVR_ASSERT(psConfig->eType == PHYS_HEAP_TYPE_IMA);
+
+	psIMAData = OSAllocMem(sizeof(*psIMAData));
+	PVR_LOG_RETURN_IF_NOMEM(psIMAData, "OSAllocMem");
+
+	eError = PhysHeapCreate(psDevNode,
+							psConfig,
+							uiPolicy,
+							(PHEAP_IMPL_DATA)psIMAData,
+							&_sPHEAPImplFuncsIMA,
+							&psPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreate", err_free_ima_data);
+
+	psIMAData->psPhysHeap = psPhysHeap;
+	psIMAData->pDLMHeap = psDLMHeap;
+	psIMAData->uiLog2PMBSize = uiLog2PMBSize;
+
+	eError = CreateIMAArena(psIMAData,
+	                        pszLabel,
+	                        uiPolicy,
+	                        uiPMBStartingMultiple);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateIMAArena", err_free_physheap);
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psPhysHeap;
+	}
+
+	return PVRSRV_OK;
+
+err_free_physheap:
+	PhysHeapDestroy(psPhysHeap);
+err_free_ima_data:
+	OSFreeMem(psIMAData);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_lma.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_lma.c
new file mode 100644
index 000000000000..3075166b5366
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_lma.c
@@ -0,0 +1,523 @@
+/*************************************************************************/ /*!
+@File           physmem_lma.c
+@Title          Local card memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for local card memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "physmem_lma.h"
+#include "pvr_debug.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "device.h"
+#include "osfunc.h"
+#include "physmem_ramem.h"
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "rgxutils.h"
+#endif
+
+typedef struct PHYSMEM_LMA_DATA_TAG {
+	RA_ARENA           *psRA;
+	IMG_CPU_PHYADDR    sStartAddr;
+	IMG_DEV_PHYADDR    sCardBase;
+	IMG_UINT64         uiSize;
+} PHYSMEM_LMA_DATA;
+
+/*
+ * This function will set the psDevPAddr to whatever the system layer
+ * has set it for the referenced heap.
+ * It will not fail if the psDevPAddr is invalid.
+ */
+static PVRSRV_ERROR
+_GetDevPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*psDevPAddr = psLMAData->sCardBase;
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This function will set the psCpuPAddr to whatever the system layer
+ * has set it for the referenced heap.
+ * It will not fail if the psCpuPAddr is invalid.
+ */
+static PVRSRV_ERROR
+_GetCPUPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*psCpuPAddr = psLMAData->sStartAddr;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_GetSize(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *puiSize)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*puiSize = psLMAData->uiSize;
+
+	return PVRSRV_OK;
+}
+
+static void PhysmemGetRAMemRamMemStats(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *pui64TotalSize,
+		 IMG_UINT64 *pui64FreeSize)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+	RA_USAGE_STATS sRAUsageStats;
+
+	RA_Get_Usage_Stats(psLMAData->psRA, &sRAUsageStats);
+
+	*pui64TotalSize = sRAUsageStats.ui64TotalArenaSize;
+	*pui64FreeSize = sRAUsageStats.ui64FreeArenaSize;
+}
+
+static PVRSRV_ERROR
+PhysmemGetArenaLMA(PHYS_HEAP *psPhysHeap,
+				   RA_ARENA **ppsArena)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)PhysHeapGetImplData(psPhysHeap);
+
+	PVR_LOG_RETURN_IF_FALSE(psLMAData != NULL, "psLMAData", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	*ppsArena = psLMAData->psRA;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_CreateArenas(PHEAP_IMPL_DATA pvImplData, IMG_CHAR *pszLabel, PHYS_HEAP_POLICY uiPolicy)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	IMG_UINT32 ui32RAPolicy =
+	    ((uiPolicy & PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	    ? RA_POLICY_ALLOC_ALLOW_NONCONTIG : RA_POLICY_DEFAULT;
+
+	psLMAData->psRA = RA_Create_With_Span(pszLabel,
+	                             OSGetPageShift(),
+	                             psLMAData->sStartAddr.uiAddr,
+	                             psLMAData->sCardBase.uiAddr,
+	                             psLMAData->uiSize,
+	                             ui32RAPolicy);
+	PVR_LOG_RETURN_IF_NOMEM(psLMAData->psRA, "RA_Create_With_Span");
+
+	return PVRSRV_OK;
+}
+
+static void
+_DestroyArenas(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	/* Remove RAs and RA names for local card memory */
+	if (psLMAData->psRA)
+	{
+		RA_Delete(psLMAData->psRA);
+		psLMAData->psRA = NULL;
+	}
+}
+
+static void
+_DestroyImplData(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	_DestroyArenas(pvImplData);
+
+	OSFreeMem(psLMAData);
+}
+
+struct _PHYS_HEAP_ITERATOR_ {
+	PHYS_HEAP *psPhysHeap;
+	RA_ARENA_ITERATOR *psRAIter;
+
+	IMG_UINT64 uiTotalSize;
+	IMG_UINT64 uiInUseSize;
+};
+
+PVRSRV_ERROR LMA_HeapIteratorCreate(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_PHYS_HEAP ePhysHeap,
+                                    PHYS_HEAP_ITERATOR **ppsIter)
+{
+	PVRSRV_ERROR eError;
+	PHYSMEM_LMA_DATA *psLMAData;
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	PHYS_HEAP *psPhysHeap = NULL;
+	RA_USAGE_STATS sStats;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsIter != NULL, "ppsIter");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	eError = PhysHeapAcquireByID(ePhysHeap, psDevNode, &psPhysHeap);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquireByID");
+
+	PVR_LOG_GOTO_IF_FALSE(PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA,
+	                      "PhysHeap must be of LMA type", release_heap);
+
+	psLMAData = (PHYSMEM_LMA_DATA *) PhysHeapGetImplData(psPhysHeap);
+
+	psHeapIter = OSAllocMem(sizeof(*psHeapIter));
+	PVR_LOG_GOTO_IF_NOMEM(psHeapIter, eError, release_heap);
+
+	psHeapIter->psPhysHeap = psPhysHeap;
+	psHeapIter->psRAIter = RA_IteratorAcquire(psLMAData->psRA, IMG_FALSE);
+	PVR_LOG_GOTO_IF_NOMEM(psHeapIter->psRAIter, eError, free_heap_iter);
+
+	/* get heap usage */
+	RA_Get_Usage_Stats(psLMAData->psRA, &sStats);
+
+	psHeapIter->uiTotalSize = sStats.ui64TotalArenaSize;
+	psHeapIter->uiInUseSize = sStats.ui64TotalArenaSize - sStats.ui64FreeArenaSize;
+
+	*ppsIter = psHeapIter;
+
+	return PVRSRV_OK;
+
+free_heap_iter:
+	OSFreeMem(psHeapIter);
+
+release_heap:
+	PhysHeapRelease(psPhysHeap);
+
+	return eError;
+}
+
+void LMA_HeapIteratorDestroy(PHYS_HEAP_ITERATOR *psIter)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psHeapIter != NULL, "psHeapIter is NULL");
+
+	PhysHeapRelease(psHeapIter->psPhysHeap);
+	RA_IteratorRelease(psHeapIter->psRAIter);
+	OSFreeMem(psHeapIter);
+}
+
+PVRSRV_ERROR LMA_HeapIteratorReset(PHYS_HEAP_ITERATOR *psIter)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeapIter != NULL, "ppsIter");
+
+	RA_IteratorReset(psHeapIter->psRAIter);
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL LMA_HeapIteratorNext(PHYS_HEAP_ITERATOR *psIter,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              IMG_UINT64 *puiSize)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+	RA_ITERATOR_DATA sData = {0};
+
+	if (psHeapIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "psHeapIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	if (!RA_IteratorNext(psHeapIter->psRAIter, &sData))
+	{
+		return IMG_FALSE;
+	}
+
+	PVR_ASSERT(sData.uiSize != 0);
+
+	psDevPAddr->uiAddr = sData.uiAddr;
+	*puiSize = sData.uiSize;
+
+	return IMG_TRUE;
+}
+
+PVRSRV_ERROR LMA_HeapIteratorGetHeapStats(PHYS_HEAP_ITERATOR *psIter,
+                                          IMG_UINT64 *puiTotalSize,
+                                          IMG_UINT64 *puiInUseSize)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeapIter != NULL, "psHeapIter");
+
+	*puiTotalSize = psHeapIter->uiTotalSize;
+	*puiInUseSize = psHeapIter->uiInUseSize;
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+static PVRSRV_ERROR
+LMA_PhyContigPagesAllocGPV(PHYS_HEAP *psPhysHeap,
+                           size_t uiSize,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_UINT32 ui32OSid,
+                           IMG_PID uiPid)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	PVR_ASSERT(ui32OSid < GPUVIRT_VALIDATION_NUM_OS);
+	if (ui32OSid >= GPUVIRT_VALIDATION_NUM_OS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Arena index %u defaulting to 0",
+		        __func__, ui32OSid));
+		ui32OSid = 0;
+	}
+
+	pArena = psDevNode->psOSidSubArena[ui32OSid];
+
+	if (psMemHandle->uiOSid != ui32OSid)
+	{
+		PVR_LOG(("%s: Unexpected OSid value %u - expecting %u", __func__,
+		        psMemHandle->uiOSid, ui32OSid));
+	}
+
+	psMemHandle->uiOSid = ui32OSid;		/* For Free() use */
+
+	eError =  RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                     psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "RAMemDoPhyContigPagesAlloc");
+
+	return eError;
+}
+#endif
+
+static PVRSRV_ERROR
+LMA_PhyContigPagesAlloc(PHYS_HEAP *psPhysHeap,
+                         size_t uiSize,
+                         PG_HANDLE *psMemHandle,
+                         IMG_DEV_PHYADDR *psDevPAddr,
+                         IMG_PID uiPid)
+{
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	IMG_UINT32 ui32OSid = 0;
+	return LMA_PhyContigPagesAllocGPV(psPhysHeap, uiSize, psMemHandle, psDevPAddr,
+									  ui32OSid, uiPid);
+#else
+	PVRSRV_ERROR eError;
+
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+
+	eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLMA");
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	eError = RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                    psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "RAMemDoPhyContigPagesAlloc");
+
+	return eError;
+#endif
+}
+
+static void
+LMA_PhyContigPagesFree(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle)
+{
+	RA_ARENA	*pArena;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	RA_BASE_T uiCardAddr = (RA_BASE_T) psMemHandle->u.ui64Handle;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	IMG_UINT32	ui32OSid = psMemHandle->uiOSid;
+
+	/*
+	 * The Arena ID is set by the originating allocation, and maintained via
+	 * the call stacks into this function. We have a limited range of IDs
+	 * and if the passed value falls outside this we simply treat it as a
+	 * 'global' arena ID of 0. This is where all default OS-specific allocations
+	 * are created.
+	 */
+	PVR_ASSERT(ui32OSid < GPUVIRT_VALIDATION_NUM_OS);
+	if (ui32OSid >= GPUVIRT_VALIDATION_NUM_OS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Arena index %u PhysAddr 0x%"
+		         IMG_UINT64_FMTSPECx " Reverting to Arena 0", __func__,
+		         ui32OSid, uiCardAddr));
+		/*
+		 * No way of determining what we're trying to free so default to the
+		 * global default arena index 0.
+		 */
+		ui32OSid = 0;
+	}
+
+	pArena = psDevNode->psOSidSubArena[ui32OSid];
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: (GPU Virtualisation) Freeing 0x%"
+	        IMG_UINT64_FMTSPECx ", Arena %u", __func__,
+	        uiCardAddr, ui32OSid));
+
+#else
+	PVRSRV_ERROR eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysmemGetArenaLMA");
+#endif
+
+	RAMemDoPhyContigPagesFree(pArena, psMemHandle);
+}
+
+static PVRSRV_ERROR
+LMAPhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                               CONNECTION_DATA *psConnection,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               IMG_UINT32 ui32NumPhysChunks,
+                               IMG_UINT32 ui32NumVirtChunks,
+                               IMG_UINT32 *pui32MappingTable,
+                               IMG_UINT32 uiLog2AllocPageSize,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               const IMG_CHAR *pszAnnotation,
+                               IMG_PID uiPid,
+                               PMR **ppsPMRPtr,
+                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	RA_ARENA *pArena;
+
+	eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLMA");
+
+	eError = PhysmemNewRAMemRamBackedPMR(psPhysHeap,
+	                                     pArena,
+	                                     psConnection,
+	                                     uiSize,
+	                                     ui32NumPhysChunks,
+	                                     ui32NumVirtChunks,
+	                                     pui32MappingTable,
+	                                     uiLog2AllocPageSize,
+	                                     uiFlags,
+	                                     pszAnnotation,
+	                                     uiPid,
+	                                     ppsPMRPtr,
+	                                     ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewLocalRamBackedPMR");
+
+	return PVRSRV_OK;
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = &_DestroyImplData,
+	.pfnGetDevPAddr = &_GetDevPAddr,
+	.pfnGetCPUPAddr = &_GetCPUPAddr,
+	.pfnGetSize = &_GetSize,
+	.pfnGetPageShift = &RAMemGetPageShift,
+	.pfnGetFactoryMemStats = &PhysmemGetRAMemRamMemStats,
+	.pfnCreatePMR = &LMAPhysmemNewRAMemRamBackedPMR,
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	.pfnPagesAllocGPV = &LMA_PhyContigPagesAllocGPV,
+#endif
+	.pfnPagesAlloc = &LMA_PhyContigPagesAlloc,
+	.pfnPagesFree = &LMA_PhyContigPagesFree,
+	.pfnPagesMap = &RAMemPhyContigPagesMap,
+	.pfnPagesUnMap = &RAMemPhyContigPagesUnmap,
+	.pfnPagesClean = &RAMemPhyContigPagesClean,
+};
+
+PVRSRV_ERROR
+PhysmemCreateHeapLMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap)
+{
+	PHYSMEM_LMA_DATA *psLMAData;
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszLabel != NULL, "pszLabel");
+
+	PVR_ASSERT(psConfig->eType == PHYS_HEAP_TYPE_LMA ||
+	           psConfig->eType == PHYS_HEAP_TYPE_DMA);
+
+	psLMAData = OSAllocMem(sizeof(*psLMAData));
+	PVR_LOG_RETURN_IF_NOMEM(psLMAData, "OSAllocMem");
+
+	psLMAData->sStartAddr = PhysHeapConfigGetStartAddr(psConfig);
+	psLMAData->sCardBase = PhysHeapConfigGetCardBase(psConfig);
+	psLMAData->uiSize = PhysHeapConfigGetSize(psConfig);
+
+	eError = PhysHeapCreate(psDevNode,
+							psConfig,
+							uiPolicy,
+							(PHEAP_IMPL_DATA)psLMAData,
+							&_sPHEAPImplFuncs,
+							&psPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreate", err_free_lma_data);
+
+	eError = _CreateArenas(psLMAData, pszLabel, uiPolicy);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CreateArenas", err_free_physheap);
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psPhysHeap;
+	}
+
+	return PVRSRV_OK;
+
+err_free_physheap:
+	PhysHeapDestroy(psPhysHeap);
+err_free_lma_data:
+	OSFreeMem(psLMAData);
+	return eError;
+
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_osmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_osmem.c
new file mode 100644
index 000000000000..cef1ea620821
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_osmem.c
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File           physmem_osmem.c
+@Title          OS Memory PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                OS memory PMR factory API that must be defined so that the
+                common & device layer code in the Services Server can allocate
+                new PMRs back with pages from the OS page allocator. Applicable
+                for UMA based platforms, such platforms must implement this API
+                in the OS Porting layer, in the "env" directory for that
+                system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/* services/server/include/ */
+#include "physheap.h"
+#include "osfunc.h"
+#include "physmem_osmem.h"
+
+
+static IMG_UINT32 PhysHeapOSGetPageShift(void)
+{
+	return (IMG_UINT32)OSGetPageShift();
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = NULL,
+	.pfnGetFactoryMemStats = PhysmemGetOSRamMemStats,
+	.pfnCreatePMR = PhysmemNewOSRamBackedPMR,
+	.pfnPagesAlloc = &OSPhyContigPagesAlloc,
+	.pfnPagesFree = &OSPhyContigPagesFree,
+	.pfnPagesMap = &OSPhyContigPagesMap,
+	.pfnPagesUnMap = &OSPhyContigPagesUnmap,
+	.pfnPagesClean = &OSPhyContigPagesClean,
+	.pfnGetPageShift = &PhysHeapOSGetPageShift,
+};
+
+
+PVRSRV_ERROR
+PhysmemCreateHeapOSMEM(PVRSRV_DEVICE_NODE *psDevNode,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       PHYS_HEAP_CONFIG *psConfig,
+                       PHYS_HEAP **ppsPhysHeap)
+{
+	return PhysHeapCreate(psDevNode,
+	                      psConfig,
+	                      uiPolicy,
+	                      NULL,
+	                      &_sPHEAPImplFuncs,
+	                      ppsPhysHeap);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ramem.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ramem.c
new file mode 100644
index 000000000000..6597042770d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/physmem_ramem.c
@@ -0,0 +1,2609 @@
+/*************************************************************************/ /*!
+@File           physmem_ramem.c
+@Title          Resource allocator managed PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                RA managed memory PMR factory API that is shared between local
+                physheap implementations (LMA & IMA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "physmem_ramem.h"
+#include "physheap.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "connection_server.h"
+#include "device.h"
+#include "devicemem_server_utils.h"
+#include "osfunc.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "rgx_pdump_panics.h"
+#include "pdump_km.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "rgxutils.h"
+#endif
+
+#if defined(INTEGRITY_OS)
+#include "mm.h"
+#include "integrity_memobject.h"
+#endif
+
+#include "physmem_dlm.h"
+
+
+#if defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+#define PHYSHEAP_DPF_LVL PVR_DBG_ERROR
+#else
+#define PHYSHEAP_DPF_LVL PVR_DBG_WARNING
+#endif
+
+/* Common Physheap Callback implementations */
+
+IMG_UINT32
+RAMemGetPageShift(void)
+{
+	return PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT;
+}
+
+PVRSRV_ERROR
+RAMemDoPhyContigPagesAlloc(RA_ARENA *pArena,
+                           size_t uiSize,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_PID uiPid)
+{
+	RA_BASE_T uiCardAddr = 0;
+	RA_LENGTH_T uiActualSize;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2NumPages;
+
+#if defined(DEBUG)
+	static IMG_UINT32 ui32MaxLog2NumPagesHistory = 0;
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* Firmware heaps on VZ are usually LMA, and several 64kb allocations need to
+	 * be made on Guest drivers: FwGuardPage, FwSysInit, FwConnectionCtl, FwOsInit.
+	 * Increase the maximum number to avoid driver warnings. */
+	IMG_UINT32	ui32MaxLog2NumPages = 7;	/* 128 pages => 512KB */
+#else
+	IMG_UINT32	ui32MaxLog2NumPages = 4;	/*  16 pages =>  64KB */
+#endif
+	ui32MaxLog2NumPages = MAX(ui32MaxLog2NumPages, OSGetOrder(1 << psDevNode->ui32Non4KPageSizeLog2));
+#else /* defined(DEBUG) */
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+#endif /* defined(DEBUG) */
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	eError = RA_Alloc(pArena,
+	                  uiSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                         /* No flags */
+	                  uiSize,
+	                  "RAMemPhyContigPagesAlloc",
+	                  &uiCardAddr,
+	                  &uiActualSize,
+	                  NULL);                     /* No private handle */
+
+	if (eError != PVRSRV_OK)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(pArena, &sRAStats);
+
+		PVR_DPF((PVR_DBG_ERROR,
+				"Failed to Allocate size = 0x"IMG_SIZE_FMTSPECX", align = 0x"
+				IMG_SIZE_FMTSPECX" Arena Free Space 0x%"IMG_UINT64_FMTSPECX,
+				uiSize, uiSize,	sRAStats.ui64FreeArenaSize));
+		return eError;
+	}
+
+	PVR_ASSERT(uiSize == uiActualSize);
+
+	psMemHandle->u.ui64Handle = uiCardAddr;
+	psDevPAddr->uiAddr = (IMG_UINT64) uiCardAddr;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+	                                    uiSize,
+	                                    uiCardAddr,
+	                                    uiPid);
+#else
+	{
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = psDevPAddr->uiAddr;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+									 NULL,
+									 sCpuPAddr,
+									 uiSize,
+									 uiPid
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#else /* PVRSRV_ENABLE_PROCESS_STATS */
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: (GPU Virtualisation) Allocated 0x" IMG_SIZE_FMTSPECX " at 0x%"
+	        IMG_UINT64_FMTSPECX ", Arena ID %u",
+	        __func__, uiSize, psDevPAddr->uiAddr, psMemHandle->uiOSid));
+#endif
+
+#if defined(DEBUG)
+	if (ui32Log2NumPages > ui32MaxLog2NumPages && ui32Log2NumPages > ui32MaxLog2NumPagesHistory)
+	{
+		PVR_ASSERT((ui32Log2NumPages <= ui32MaxLog2NumPages));
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ui32MaxLog2NumPages = %u, increasing to %u", __func__,
+		        ui32MaxLog2NumPages, ui32Log2NumPages ));
+		ui32MaxLog2NumPagesHistory = ui32Log2NumPages;
+	}
+#endif /* defined(DEBUG) */
+	psMemHandle->uiOrder = ui32Log2NumPages;
+
+	return eError;
+}
+
+void
+RAMemDoPhyContigPagesFree(RA_ARENA *pArena,
+                          PG_HANDLE *psMemHandle)
+{
+	RA_BASE_T uiCardAddr = (RA_BASE_T) psMemHandle->u.ui64Handle;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+	                                      (IMG_UINT64)uiCardAddr);
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+									(IMG_UINT64)uiCardAddr,
+									OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	RA_Free(pArena, uiCardAddr);
+	psMemHandle->uiOrder = 0;
+}
+
+PVRSRV_ERROR
+RAMemPhyContigPagesMap(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle,
+                       size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                       void **pvPtr)
+{
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_UINT32 ui32NumPages = (1 << psMemHandle->uiOrder);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+
+	PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPAddr, psDevPAddr);
+	*pvPtr = OSMapPhysToLin(sCpuPAddr,
+							ui32NumPages * OSGetPageSize(),
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+	PVR_RETURN_IF_NOMEM(*pvPtr);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+	                            ui32NumPages * OSGetPageSize(),
+	                            OSGetCurrentClientProcessIDKM());
+#else
+	{
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+									 *pvPtr,
+									 sCpuPAddr,
+									 ui32NumPages * OSGetPageSize(),
+									 OSGetCurrentClientProcessIDKM()
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#endif
+	return PVRSRV_OK;
+}
+
+void
+RAMemPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap,
+                         PG_HANDLE *psMemHandle,
+                         void *pvPtr)
+{
+	IMG_UINT32 ui32NumPages = (1 << psMemHandle->uiOrder);
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+		                            ui32NumPages * OSGetPageSize(),
+		                            OSGetCurrentClientProcessIDKM());
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+	                                (IMG_UINT64)(uintptr_t)pvPtr,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	OSUnMapPhysToLin(pvPtr, ui32NumPages * OSGetPageSize());
+}
+
+PVRSRV_ERROR
+RAMemPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                         PG_HANDLE *psMemHandle,
+                         IMG_UINT32 uiOffset,
+                         IMG_UINT32 uiLength)
+{
+	/* No need to flush because we map as uncached */
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+	PVR_UNREFERENCED_PARAMETER(psMemHandle);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiLength);
+
+	return PVRSRV_OK;
+}
+
+/* Local memory allocation routines */
+
+/* Assert that the conversions between the RA base type and the device
+ * physical address are safe.
+ */
+static_assert(sizeof(IMG_DEV_PHYADDR) == sizeof(RA_BASE_T),
+              "Size IMG_DEV_PHYADDR != RA_BASE_T");
+
+/* Since 0x0 is a valid DevPAddr, we rely on max 64-bit value to be an invalid
+ * page address */
+#define INVALID_PAGE_ADDR ~((IMG_UINT64)0x0)
+#define ZERO_PAGE_VALUE 0
+
+typedef struct _PMR_KERNEL_MAP_HANDLE_ {
+	void *vma;
+	void *pvKernelAddress;
+	/* uiSize has 2 uses:
+	 * In Physically contiguous case it is used to track size of the mapping
+	 * for free.
+	 * In Physically sparse case it is used to determine free path to use, single page
+	 * sparse mapping or multi page
+	 */
+	size_t uiSize;
+} PMR_KERNEL_MAPPING;
+
+typedef struct _PMR_LMALLOCARRAY_DATA_ {
+
+#define FLAG_ZERO              (0U)
+#define FLAG_POISON_ON_FREE    (1U)
+#define FLAG_POISON_ON_ALLOC   (2U)
+#define FLAG_ONDEMAND          (3U)
+#define FLAG_SPARSE            (4U)
+#define FLAG_PHYS_CONTIG       (5U)
+#define FLAG_ZOMBIE            (6U)
+
+	IMG_PID uiPid;
+
+	/*
+	 * N.B Chunks referenced in this struct commonly are
+	 * to OS page sized. But in reality it is dependent on
+	 * the uiLog2ChunkSize.
+	 * Chunks will always be one 1 << uiLog2ChunkSize in size.
+	 * */
+
+	/*
+	 * The number of chunks currently allocated in the PMR.
+	 */
+	IMG_INT32 iNumChunksAllocated;
+
+	/*
+	 * Total number of (Virtual) chunks supported by this PMR.
+	 */
+	IMG_UINT32 uiTotalNumChunks;
+
+	/* The number of chunks to next be allocated for the PMR.
+	 * This will initially be the number allocated at first alloc
+	 * but may be changed in later calls to change sparse.
+	 * It represents the number of chunks to next be allocated.
+	 * This is used to store this value because we have the ability to
+	 * defer allocation.
+	 */
+	IMG_UINT32 uiChunksToAlloc;
+
+	/*
+	 * Log2 representation of the chunksize.
+	 */
+	IMG_UINT32 uiLog2ChunkSize;
+
+	/* Physical heap and arena pointers for this allocation */
+	PHYS_HEAP* psPhysHeap;
+	RA_ARENA* psArena;
+	PVRSRV_MEMALLOCFLAGS_T uiAllocFlags;
+
+	/*
+	   Connection data for this requests' originating process. NULL for
+	   direct-bridge originating calls
+	 */
+	CONNECTION_DATA *psConnection;
+
+	/*
+	 * Allocation flags related to the pages:
+	 * Zero              - Should we Zero memory on alloc
+	 * Poison on free    - Should we Poison the memory on free.
+	 * Poison on alloc   - Should we Poison the memory on alloc.
+	 * On demand         - Is the allocation on Demand i.e Do we defer allocation to time of use.
+	 * Sparse            - Is the PMR sparse.
+	 * Phys Contig       - Is the alloc Physically contiguous
+	 * Zombie            - Is zombie
+	 * */
+	IMG_UINT32 ui32Flags;
+
+	RA_BASE_ARRAY_T aBaseArray; /* Array of RA Bases */
+
+} PMR_LMALLOCARRAY_DATA;
+
+static PVRSRV_ERROR
+_FreeLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+             IMG_UINT32 *pui32FreeIndices,
+             IMG_UINT32 ui32FreeChunkCount);
+
+static PVRSRV_ERROR _MapPhysicalContigAlloc(PHYS_HEAP *psPhysHeap,
+                                            RA_BASE_ARRAY_T paBaseArray,
+                                            size_t uiSize,
+                                            PMR_FLAGS_T ulFlags,
+                                            PMR_KERNEL_MAPPING *psMapping)
+{
+	IMG_UINT32 ui32CPUCacheFlags;
+	PVRSRV_ERROR eError;
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+	sDevPAddr.uiAddr = RA_BASE_STRIP_GHOST_BIT(*paBaseArray);
+
+	eError = DevmemCPUCacheMode(ulFlags, &ui32CPUCacheFlags);
+	PVR_RETURN_IF_ERROR(eError);
+
+	PhysHeapDevPAddrToCpuPAddr(psPhysHeap,
+	                           1,
+	                           &sCpuPAddr,
+	                           &sDevPAddr);
+
+	psMapping->pvKernelAddress = OSMapPhysToLin(sCpuPAddr, uiSize, ui32CPUCacheFlags);
+	PVR_LOG_RETURN_IF_FALSE(psMapping->pvKernelAddress,
+	                        "OSMapPhyToLin: out of VM Mem",
+	                        PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING);
+	psMapping->vma = NULL;
+	psMapping->uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MapPhysicalSparseAlloc(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                                            RA_BASE_ARRAY_T paBaseArray,
+                                            size_t uiSize,
+                                            PMR_KERNEL_MAPPING *psMapping)
+{
+	IMG_UINT32 uiChunkCount = uiSize >> psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_CPU_PHYADDR uiPages[PMR_MAX_TRANSLATION_STACK_ALLOC], *puiPages;
+	PVRSRV_ERROR eError;
+	size_t uiPageShift = OSGetPageShift();
+	IMG_UINT32 uiOSPagesPerChunkShift = psLMAllocArrayData->uiLog2ChunkSize - uiPageShift;
+	IMG_UINT32 uiOSPageCount = uiChunkCount << uiOSPagesPerChunkShift;
+
+	if (uiOSPageCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPages = OSAllocZMem(sizeof(IMG_CPU_PHYADDR) * uiOSPageCount);
+		PVR_RETURN_IF_NOMEM(puiPages);
+	}
+	else
+	{
+		puiPages = &uiPages[0];
+	}
+
+	if (uiOSPagesPerChunkShift == 0)
+	{
+		IMG_UINT32 i;
+		PhysHeapDevPAddrToCpuPAddr(psLMAllocArrayData->psPhysHeap,
+								   uiChunkCount,
+								   puiPages,
+								   (IMG_DEV_PHYADDR *)paBaseArray);
+
+		/* If the ghost bit is present then the addrs returned will be off by 1
+		 * Strip the ghost bit to correct to real page aligned addresses.
+		 * */
+		for (i = 0; i < uiChunkCount; i++)
+		{
+			puiPages[i].uiAddr = RA_BASE_STRIP_GHOST_BIT(puiPages[i].uiAddr);
+		}
+	}
+	else
+	{
+		IMG_UINT32 i = 0, j = 0, index = 0;
+		for (i = 0; i < uiChunkCount; i++)
+		{
+			IMG_UINT32 ui32OSPagesPerDeviceChunk = (1 << uiOSPagesPerChunkShift);
+			IMG_DEV_PHYADDR uiDevAddr;
+			uiDevAddr.uiAddr = RA_BASE_STRIP_GHOST_BIT(paBaseArray[i]);
+			for (j = 0; j < ui32OSPagesPerDeviceChunk; j++)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psLMAllocArrayData->psPhysHeap,
+										   1,
+										   &puiPages[index],
+										   &uiDevAddr);
+				uiDevAddr.uiAddr += (1ULL << uiPageShift);
+				index++;
+			}
+		}
+	}
+
+	eError = OSMapPhysArrayToLin(puiPages,
+	                             uiOSPageCount,
+	                             &psMapping->pvKernelAddress,
+	                             &psMapping->vma);
+	if (eError == PVRSRV_OK)
+	{
+		psMapping->uiSize = uiSize;
+	}
+
+	if (puiPages != &uiPages[0])
+	{
+		OSFreeMem(puiPages);
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR _MapPMRKernel(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                                  RA_BASE_ARRAY_T paBaseArray,
+                                  size_t uiSize,
+                                  PMR_FLAGS_T ulFlags,
+                                  PMR_KERNEL_MAPPING *psMapping)
+{
+	PVRSRV_ERROR eError;
+	PHYS_HEAP *psPhysHeap = psLMAllocArrayData->psPhysHeap;
+	if (!BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* Physically Contig */
+		if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			eError = _MapPhysicalContigAlloc(psPhysHeap,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 ulFlags,
+			                                 psMapping);
+		}
+		/* Physically Sparse */
+		else
+		{
+			eError = _MapPhysicalSparseAlloc(psLMAllocArrayData,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 psMapping);
+		}
+	}
+	else
+	{
+		/* Sparse Alloc Single Chunk */
+		if (uiSize == IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize))
+		{
+			eError = _MapPhysicalContigAlloc(psPhysHeap,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 ulFlags,
+			                                 psMapping);
+		}
+		/* Sparse Alloc Multi Chunk */
+		else
+		{
+			eError = _MapPhysicalSparseAlloc(psLMAllocArrayData,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 psMapping);
+		}
+	}
+
+	return eError;
+}
+
+static void _UnMapPhysicalContigAlloc(PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	OSUnMapPhysToLin(psKernelMapping->pvKernelAddress, psKernelMapping->uiSize);
+}
+
+static void _UnMapPhysicalSparseAlloc(PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	OSUnMapPhysArrayToLin(psKernelMapping->pvKernelAddress,
+	                   psKernelMapping->vma);
+}
+
+static void _UnMapPMRKernel(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                            PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	if (!BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* Physically Contig */
+		if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			_UnMapPhysicalContigAlloc(psKernelMapping);
+		}
+		/* Physically Sparse */
+		else
+		{
+			_UnMapPhysicalSparseAlloc(psKernelMapping);
+		}
+	}
+	else
+	{
+		/* Sparse Alloc Single Chunk */
+		if (psKernelMapping->uiSize == IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize))
+		{
+			_UnMapPhysicalContigAlloc(psKernelMapping);
+		}
+		/* Sparse Alloc Multi Chunk */
+		else
+		{
+			_UnMapPhysicalSparseAlloc(psKernelMapping);
+		}
+	}
+}
+
+static PVRSRV_ERROR
+_PhysPgMemSet(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+              RA_BASE_ARRAY_T paBaseArray,
+              size_t uiSize,
+              IMG_BYTE ui8SetValue)
+{
+	PVRSRV_ERROR eError;
+	PMR_KERNEL_MAPPING sKernelMapping;
+
+	eError = _MapPMRKernel(psLMAllocArrayData,
+	                       paBaseArray,
+	                       uiSize,
+	                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+	                       &sKernelMapping);
+	PVR_GOTO_IF_ERROR(eError, map_failed);
+
+	OSCachedMemSetWMB(sKernelMapping.pvKernelAddress, ui8SetValue, uiSize);
+
+	_UnMapPMRKernel(psLMAllocArrayData, &sKernelMapping);
+
+	return PVRSRV_OK;
+
+map_failed:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to poison/zero allocation"));
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPageArray(PMR_SIZE_T uiSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 uiLog2AllocPageSize,
+                  IMG_UINT32 ui32Flags,
+                  PHYS_HEAP* psPhysHeap,
+                  RA_ARENA* pArena,
+                  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+                  IMG_PID uiPid,
+                  PMR_LMALLOCARRAY_DATA **ppsPageArrayDataPtr,
+                  CONNECTION_DATA *psConnection)
+{
+	PMR_LMALLOCARRAY_DATA *psPageArrayData = NULL;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiNumPages;
+
+	PVR_ASSERT(!BIT_ISSET(ui32Flags, FLAG_ZERO) || !BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC));
+	PVR_ASSERT(OSGetPageShift() <= uiLog2AllocPageSize);
+
+	/* Use of cast below is justified by the assertion that follows to
+	prove that no significant bits have been truncated */
+	uiNumPages = (IMG_UINT32)(((uiSize - 1) >> uiLog2AllocPageSize) + 1);
+	PVR_ASSERT(((PMR_SIZE_T)uiNumPages << uiLog2AllocPageSize) == uiSize);
+
+	psPageArrayData = OSAllocMem(sizeof(PMR_LMALLOCARRAY_DATA) + IMG_FLEX_ARRAY_SIZE(sizeof(RA_BASE_T), uiNumPages));
+	PVR_GOTO_IF_NOMEM(psPageArrayData, eError, errorOnAllocArray);
+
+	if (BIT_ISSET(ui32Flags, FLAG_SPARSE))
+	{
+		/* Since no pages are allocated yet, initialise page addresses to INVALID_PAGE_ADDR */
+		OSCachedMemSet(psPageArrayData->aBaseArray,
+					   0xFF,
+					   sizeof(RA_BASE_T) *
+					   uiNumPages);
+	}
+	else
+	{
+		/* Base pointers have been allocated for the full PMR in case we require a non
+		 * physically contiguous backing for the virtually contiguous allocation but the most
+		 * common case will be contiguous and so only require the first Base to be present
+		 */
+		psPageArrayData->aBaseArray[0] = INVALID_BASE_ADDR;
+	}
+
+	psPageArrayData->uiTotalNumChunks = uiNumPages;
+	psPageArrayData->uiChunksToAlloc = BIT_ISSET(ui32Flags, FLAG_SPARSE) ? ui32NumPhysChunks : uiNumPages;
+	psPageArrayData->uiLog2ChunkSize = uiLog2AllocPageSize;
+
+	psPageArrayData->psConnection = psConnection;
+	psPageArrayData->uiPid = uiPid;
+	psPageArrayData->iNumChunksAllocated = 0;
+	psPageArrayData->ui32Flags = ui32Flags;
+	psPageArrayData->psPhysHeap = psPhysHeap;
+	psPageArrayData->psArena = pArena;
+	psPageArrayData->uiAllocFlags = uiAllocFlags;
+
+	*ppsPageArrayDataPtr = psPageArrayData;
+
+	return PVRSRV_OK;
+
+/*
+  error exit path follows:
+*/
+
+errorOnAllocArray:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPagesContig(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiChunksToAlloc, uiLog2ChunkSize);
+	IMG_BOOL bPhysContig;
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+
+	eError = RA_AllocMulti(psPageArrayData->psArena,
+	                  ui64PhysSize,
+	                  uiLog2ChunkSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                       /* No flags */
+	                  "LMA_Page_Alloc",
+	                  psPageArrayData->aBaseArray,
+	                  psPageArrayData->uiTotalNumChunks,
+	                  &bPhysContig);
+
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"Contig: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long)ui64PhysSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+	if (bPhysContig)
+	{
+		BIT_SET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+			psPageArrayData->aBaseArray[0]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoison);
+	}
+
+	if (BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       ZERO_PAGE_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnZero);
+	}
+
+	psPageArrayData->iNumChunksAllocated += psPageArrayData->uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, ui64PhysSize, psPageArrayData->uiPid);
+#else
+	if (bPhysContig)
+	{
+		IMG_CPU_PHYADDR sLocalCpuPAddr;
+		sLocalCpuPAddr.uiAddr = (IMG_UINT64) psPageArrayData->aBaseArray[0];
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+								 NULL,
+								 sLocalCpuPAddr,
+								 IMG_PAGES2BYTES64(psPageArrayData->uiTotalNumChunks, uiLog2ChunkSize),
+								 psPageArrayData->uiPid
+								 DEBUG_MEMSTATS_VALUES);
+	}
+	else
+	{
+		IMG_UINT32 i, j;
+		IMG_CPU_PHYADDR sLocalCpuPAddr;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks;)
+		{
+			IMG_UINT32 ui32AllocSizeInChunks = 1;
+
+			for (j = i;
+			     j + 1 != psPageArrayData->uiTotalNumChunks &&
+			     RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+			     j++)
+			{
+				ui32AllocSizeInChunks++;
+			}
+
+			sLocalCpuPAddr.uiAddr = (IMG_UINT64) psPageArrayData->aBaseArray[i];
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 IMG_PAGES2BYTES64(ui32AllocSizeInChunks, uiLog2ChunkSize),
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+
+			i += ui32AllocSizeInChunks;
+		}
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnZero:
+errorOnPoison:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMulti(psPageArrayData->psArena,
+	              psPageArrayData->aBaseArray,
+	              psPageArrayData->uiTotalNumChunks);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+ * Fully allocated variant of sparse allocation does not take in as argument an
+ * array of indices. It is used in cases where the amount of chunks to allocate is
+ * the same as the total the PMR can represent. I.E when we want to fully populate
+ * a sparse PMR.
+ */
+static PVRSRV_ERROR
+_AllocLMPagesSparseFull(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiChunksToAlloc, uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+
+	eError = RA_AllocMultiSparse(psPageArrayData->psArena,
+	                  uiLog2ChunkSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                       /* No flags */
+	                  "LMA_Page_Alloc",
+	                  psPageArrayData->aBaseArray,
+	                  psPageArrayData->uiTotalNumChunks,
+	                  NULL, /* No indices given meaning allocate full base array using chunk count below */
+	                  psPageArrayData->uiChunksToAlloc);
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"SparseFull: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long)ui64PhysSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+		"(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+		psPageArrayData->aBaseArray[0]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoison);
+	}
+
+	if (BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       ZERO_PAGE_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnZero);
+	}
+
+	psPageArrayData->iNumChunksAllocated += psPageArrayData->uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, ui64PhysSize, psPageArrayData->uiPid);
+#else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			IMG_CPU_PHYADDR sLocalCpuPAddr;
+			sLocalCpuPAddr.uiAddr =
+				(IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[i]);
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 1 << uiLog2ChunkSize,
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+		}
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnZero:
+errorOnPoison:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMulti(psPageArrayData->psArena,
+	              psPageArrayData->aBaseArray,
+	              psPageArrayData->uiTotalNumChunks);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPagesSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData, IMG_UINT32 *pui32MapTable)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT32 uiChunksToAlloc = psPageArrayData->uiChunksToAlloc;
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+	if (!pui32MapTable)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("pui32MapTable", eError, PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY, errorOnRAAlloc);
+	}
+
+#if defined(DEBUG)
+	/*
+	 * This block performs validation of the mapping table input in the following ways:
+	 * Check that each index in the mapping table does not exceed the number of the chunks
+	 * the whole PMR supports.
+	 * Check that each index given by the mapping table is not already allocated.
+	 * Check that there are no duplicated indices given in the mapping table.
+	 */
+	{
+		IMG_UINT32 i;
+		IMG_BOOL bIssueDetected = IMG_FALSE;
+		PVRSRV_ERROR eMapCheckError;
+
+		for (i = 0; i < uiChunksToAlloc; i++)
+		{
+			if (pui32MapTable[i] >= psPageArrayData->uiTotalNumChunks)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Page alloc request Index out of bounds for PMR @0x%p",
+						__func__,
+						psPageArrayData));
+				eMapCheckError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+
+			if (!RA_BASE_IS_INVALID(psPageArrayData->aBaseArray[pui32MapTable[i]]))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Mapping already exists Index %u Mapping index %u",
+						__func__,
+						i,
+						pui32MapTable[i]));
+				eMapCheckError = PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+
+			if (RA_BASE_IS_SPARSE_PREP(psPageArrayData->aBaseArray[pui32MapTable[i]]))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Mapping already exists in mapping table given Index %u Mapping index %u",
+						__func__,
+						i,
+						pui32MapTable[i]));
+				eMapCheckError = PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+			else
+			{
+				/* Set the To Prep value so we can detect duplicated map indices */
+				psPageArrayData->aBaseArray[pui32MapTable[i]] = RA_BASE_SPARSE_PREP_ALLOC_ADDR;
+			}
+		}
+		/* Unwind the Alloc Prep Values */
+		if (bIssueDetected)
+		{
+			/* We don't want to affect the index of the issue seen
+			 * as it could be a valid mapping. If it is a duplicated
+			 * mapping in the given table then we will clean-up the
+			 * previous instance anyway.
+			 */
+			IMG_UINT32 uiUnwind = i;
+
+			for (i = 0; i < uiUnwind; i++)
+			{
+				psPageArrayData->aBaseArray[pui32MapTable[i]] = INVALID_BASE_ADDR;
+			}
+
+			PVR_GOTO_WITH_ERROR(eError, eMapCheckError, errorOnRAAlloc);
+		}
+	}
+#endif
+
+	eError = RA_AllocMultiSparse(psPageArrayData->psArena,
+	                              psPageArrayData->uiLog2ChunkSize,
+	                              RA_NO_IMPORT_MULTIPLIER,
+	                              0,
+	                              "LMA_Page_Alloc",
+	                              psPageArrayData->aBaseArray,
+	                              psPageArrayData->uiTotalNumChunks,
+	                              pui32MapTable,
+	                              uiChunksToAlloc);
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"Sparse: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long) uiChunksToAlloc << uiLog2ChunkSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+	        psPageArrayData->aBaseArray[pui32MapTable[0]]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC) || BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		IMG_UINT32 i, ui32Index = 0;
+		for (i = 0; i < uiChunksToAlloc; i++)
+		{
+			ui32Index = pui32MapTable[i];
+
+			eError = _PhysPgMemSet(psPageArrayData,
+			                       &psPageArrayData->aBaseArray[ui32Index],
+			                       ui64ChunkSize,
+			                       BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC) ? PVRSRV_POISON_ON_ALLOC_VALUE :
+			                                                                    ZERO_PAGE_VALUE);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoisonZero);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+	                            uiChunksToAlloc << uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiChunksToAlloc; i++)
+		{
+			IMG_UINT32 ui32Index = pui32MapTable[i];
+			IMG_CPU_PHYADDR sLocalCpuPAddr;
+			sLocalCpuPAddr.uiAddr =
+				(IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[ui32Index]);
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 ui64ChunkSize,
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+		}
+	}
+#endif
+#endif
+
+	psPageArrayData->iNumChunksAllocated += uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnPoisonZero:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMultiSparse(psPageArrayData->psArena,
+	                    psPageArrayData->aBaseArray,
+	                    psPageArrayData->uiTotalNumChunks,
+	                    psPageArrayData->uiLog2ChunkSize,
+	                    pui32MapTable,
+	                    &uiChunksToAlloc);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR
+_AllocLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData, IMG_UINT32 *pui32MapTable)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(NULL != psPageArrayData);
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	if (psPageArrayData->uiTotalNumChunks <
+			(psPageArrayData->iNumChunksAllocated + psPageArrayData->uiChunksToAlloc))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Pages requested to allocate don't fit PMR alloc Size. "
+				"Allocated: %u + Requested: %u > Total Allowed: %u",
+				psPageArrayData->iNumChunksAllocated,
+				psPageArrayData->uiChunksToAlloc,
+				psPageArrayData->uiTotalNumChunks));
+		return PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+	}
+
+	/* If we have a non-backed sparse PMR then we can just return */
+	if (psPageArrayData->uiChunksToAlloc == 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+							"%s: Non-Backed Sparse PMR Created: %p.",
+							__func__,
+							psPageArrayData));
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	{
+		IMG_UINT32 ui32OSid=0;
+		PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPageArrayData->psPhysHeap);
+
+		/* Obtain the OSid specific data from our connection handle */
+		if (psPageArrayData->psConnection != NULL)
+		{
+			ui32OSid = psPageArrayData->psConnection->ui32OSid;
+		}
+
+		/* Replace the given RA Arena with OS Arena */
+		if (PVRSRV_CHECK_SHARED_BUFFER(psPageArrayData->uiAllocFlags))
+		{
+			psPageArrayData->psArena = psDevNode->psOSSharedArena;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "(GPU Virtualization Validation): Giving from shared mem"));
+		}
+		else
+		{
+			psPageArrayData->psArena = psDevNode->psOSidSubArena[ui32OSid];
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "(GPU Virtualization Validation): Giving from OS slot %d",
+					 ui32OSid));
+		}
+	}
+#endif
+
+	/*
+	 * 3 cases:
+	 * Sparse allocation populating the whole PMR.
+	 * [**********]
+	 * Sparse allocation partially populating the PMR at given indices.
+	 * [*** *** **]
+	 * Contiguous allocation.
+	 * [**********]
+	 *
+	 * Note: Separate cases are required for 1 and 3 due to memstats tracking.
+	 * In Contiguous case we can track the block as a single memstat record as we know
+	 * we will also free in that size record.
+	 * Sparse allocations require a memstat record per chunk as they can be arbitrarily
+	 * free'd.
+	 */
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		if (psPageArrayData->uiTotalNumChunks == psPageArrayData->uiChunksToAlloc &&
+		    !pui32MapTable)
+		{
+			eError = _AllocLMPagesSparseFull(psPageArrayData);
+		}
+		else
+		{
+			eError = _AllocLMPagesSparse(psPageArrayData, pui32MapTable);
+		}
+	}
+	else
+	{
+		eError = _AllocLMPagesContig(psPageArrayData);
+	}
+
+	return eError;
+}
+
+static void
+_FreeLMPageArray(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"physmem_lma.c: freed local memory array structure for PMR @0x%p",
+			psPageArrayData));
+
+	OSFreeMem(psPageArrayData);
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesContig(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	RA_ARENA *pArena = psPageArrayData->psArena;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiTotalNumChunks, psPageArrayData->uiLog2ChunkSize);
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated != 0);
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated ==
+	           psPageArrayData->uiTotalNumChunks);
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_POISON_ON_FREE))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+							   psPageArrayData->aBaseArray,
+							   ui64PhysSize,
+							   PVRSRV_POISON_ON_FREE_VALUE);
+		PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            ui64PhysSize,
+	                            psPageArrayData->uiPid);
+#else
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		PVRSRVStatsRemoveMemAllocRecord(uiStat,
+		                                (IMG_UINT64) psPageArrayData->aBaseArray[0],
+		                                psPageArrayData->uiPid);
+	}
+	else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+			{
+				PVRSRVStatsRemoveMemAllocRecord(uiStat,
+				                                (IMG_UINT64) psPageArrayData->aBaseArray[i],
+				                                psPageArrayData->uiPid);
+			}
+		}
+	}
+#endif
+#endif
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		eError = RA_FreeMulti(pArena,
+							  psPageArrayData->aBaseArray,
+							  1);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+	}
+	else
+	{
+		eError = RA_FreeMulti(pArena,
+							  psPageArrayData->aBaseArray,
+							  psPageArrayData->iNumChunksAllocated);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+	}
+
+	psPageArrayData->iNumChunksAllocated = 0;
+
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: freed %"IMG_UINT64_FMTSPEC" local memory for PMR @0x%p",
+			__func__,
+			ui64PhysSize,
+			psPageArrayData));
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesRemainingSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(psPageArrayData->uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+	IMG_BOOL bPoisonOnFree = (BIT_ISSET(ui32Flags, FLAG_POISON_ON_FREE));
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            psPageArrayData->iNumChunksAllocated << psPageArrayData->uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#endif
+
+	for (i = 0; i < psPageArrayData->uiTotalNumChunks;)
+	{
+		if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+		{
+			IMG_UINT32 j;
+			IMG_UINT32 ui32AccumulatedChunks = 1;
+
+			for (j = i;
+				 j + 1 != psPageArrayData->uiTotalNumChunks &&
+				 RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+				 j++)
+			{
+				ui32AccumulatedChunks++;
+			}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+			for (j = i; j < (i + ui32AccumulatedChunks); j++)
+			{
+				PVRSRVStatsRemoveMemAllocRecord(uiStat,
+				                                RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[j]),
+				                                psPageArrayData->uiPid);
+				if (bPoisonOnFree)
+#else
+			for (j = i; j < (i + ui32AccumulatedChunks) && bPoisonOnFree; j++)
+			{
+#endif
+				{
+					eError = _PhysPgMemSet(psPageArrayData,
+										   &psPageArrayData->aBaseArray[j],
+										   ui64ChunkSize,
+										   PVRSRV_POISON_ON_FREE_VALUE);
+					PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+				}
+			}
+
+			eError = RA_FreeMulti(psPageArrayData->psArena,
+			                       &psPageArrayData->aBaseArray[i],
+			                       ui32AccumulatedChunks);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+
+			psPageArrayData->iNumChunksAllocated -= ui32AccumulatedChunks;
+			i += ui32AccumulatedChunks;
+		}
+		else if (RA_BASE_IS_INVALID(psPageArrayData->aBaseArray[i]))
+		{
+			i++;
+		}
+	}
+
+	/* We have freed all allocations in the previous loop */
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+                   IMG_UINT32 *pui32FreeIndices,
+                   IMG_UINT32 ui32FreeChunkCount)
+{
+	RA_ARENA *pArena = psPageArrayData->psArena;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+	IMG_UINT32 uiActualFreeCount = ui32FreeChunkCount;
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated != 0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < ui32FreeChunkCount; i++)
+		{
+			IMG_UINT32 ui32Index = pui32FreeIndices[i];
+
+			PVRSRVStatsRemoveMemAllocRecord(uiStat,
+			                                (IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(
+			                                psPageArrayData->aBaseArray[ui32Index]),
+			                                psPageArrayData->uiPid);
+		}
+	}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_FREE))
+	{
+		IMG_UINT32 i, ui32Index = 0;
+		for (i = 0; i < ui32FreeChunkCount; i++)
+		{
+			ui32Index = pui32FreeIndices[i];
+
+			eError = _PhysPgMemSet(psPageArrayData,
+								   &psPageArrayData->aBaseArray[ui32Index],
+								   ui64ChunkSize,
+								   PVRSRV_POISON_ON_FREE_VALUE);
+			PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+		}
+	}
+
+	eError = RA_FreeMultiSparse(pArena,
+	                             psPageArrayData->aBaseArray,
+	                             psPageArrayData->uiTotalNumChunks,
+	                             uiLog2ChunkSize,
+	                             pui32FreeIndices,
+	                             &uiActualFreeCount);
+	psPageArrayData->iNumChunksAllocated -= uiActualFreeCount;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            uiActualFreeCount << psPageArrayData->uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#endif
+	if (eError == PVRSRV_ERROR_RA_FREE_INVALID_CHUNK)
+	{
+		/* Log the RA error but convert it to PMR level to match the interface,
+		 * this is important because other PMR factories may not use the RA but
+		 * still return error, returning a PMR based error
+		 * keeps the interface agnostic to implementation behaviour.
+		 */
+		PVR_LOG_IF_ERROR(eError, "RA_FreeMultiSparse");
+		return PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMultiSparse");
+
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: freed %" IMG_UINT64_FMTSPEC " local memory for PMR @0x%p",
+			__func__,
+			(uiActualFreeCount * ui64ChunkSize),
+			psPageArrayData));
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRFreeZombiePagesRAMem(PMR_IMPL_ZOMBIEPAGES pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psZombiePageArray = pvPriv;
+
+	eError = _FreeLMPages(psZombiePageArray, NULL, 0);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	_FreeLMPageArray(psZombiePageArray);
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+
+/* Allocates a new PMR_LMALLOCARRAY_DATA object and fills it with
+ * pages to be extracted from psSrcPageArrayData.
+ */
+static PVRSRV_ERROR
+_ExtractPages(PMR_LMALLOCARRAY_DATA *psSrcPageArrayData,
+			  IMG_UINT32 *pai32ExtractIndices,
+			  IMG_UINT32 ui32ExtractPageCount,
+			  PMR_LMALLOCARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ExtractPageCountSaved;
+	PMR_LMALLOCARRAY_DATA* psDstPageArrayData;
+
+	/* Alloc PMR_LMALLOCARRAY_DATA for the extracted pages */
+	eError = _AllocLMPageArray(ui32ExtractPageCount << psSrcPageArrayData->uiLog2ChunkSize,
+	                           ui32ExtractPageCount,
+	                           psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->ui32Flags,
+	                           psSrcPageArrayData->psPhysHeap,
+	                           psSrcPageArrayData->psArena,
+	                           psSrcPageArrayData->uiAllocFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData,
+	                           psSrcPageArrayData->psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_AllocLMPageArray", alloc_error);
+
+	ui32ExtractPageCountSaved = ui32ExtractPageCount;
+	/* Transfer pages from source base array to newly allocated page array */
+	eError = RA_TransferMultiSparseIndices(psSrcPageArrayData->psArena,
+	                                       psSrcPageArrayData->aBaseArray,
+	                                       psSrcPageArrayData->uiTotalNumChunks,
+	                                       psDstPageArrayData->aBaseArray,
+	                                       psDstPageArrayData->uiTotalNumChunks,
+	                                       psSrcPageArrayData->uiLog2ChunkSize,
+	                                       pai32ExtractIndices,
+	                                       &ui32ExtractPageCountSaved);
+	PVR_LOG_GOTO_IF_FALSE((eError == PVRSRV_OK) && (ui32ExtractPageCountSaved == ui32ExtractPageCount),
+	                      "RA_TransferMultiSparseIndices failed",
+	                      transfer_error);
+
+
+	/* Update page counts */
+	psSrcPageArrayData->iNumChunksAllocated -= ui32ExtractPageCount;
+	psDstPageArrayData->iNumChunksAllocated += ui32ExtractPageCount;
+
+	*psOutPageArrayData = psDstPageArrayData;
+
+	return PVRSRV_OK;
+transfer_error:
+	_FreeLMPageArray(psDstPageArrayData);
+alloc_error:
+	return eError;
+}
+
+/* Extracts all allocated pages referenced psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted
+ * pages information.
+ */
+static PVRSRV_ERROR
+_ExtractAllPages(PMR_LMALLOCARRAY_DATA *psSrcPageArrayData,
+				 PMR_LMALLOCARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA* psDstPageArrayData;
+	IMG_UINT32 ui32IdxSrc, ui32IdxDst;
+
+	if (psSrcPageArrayData->iNumChunksAllocated == 0)
+	{
+		/* Do nothing if psSrcPageArrayData contains no allocated pages */
+		return PVRSRV_OK;
+	}
+
+	/* Alloc PMR_LMALLOCARRAY_DATA for the extracted pages */
+	eError = _AllocLMPageArray(psSrcPageArrayData->iNumChunksAllocated << psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->iNumChunksAllocated,
+	                           psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->ui32Flags,
+	                           psSrcPageArrayData->psPhysHeap,
+	                           psSrcPageArrayData->psArena,
+	                           psSrcPageArrayData->uiAllocFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData,
+	                           psSrcPageArrayData->psConnection);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "_AllocLMPageArray failed in %s", __func__);
+
+	/* Now do the transfer */
+	ui32IdxDst=0;
+	for (ui32IdxSrc=0; ((ui32IdxDst<psSrcPageArrayData->iNumChunksAllocated) &&
+	                    (psDstPageArrayData->iNumChunksAllocated<psSrcPageArrayData->iNumChunksAllocated)); ui32IdxSrc++)
+	{
+		if (psSrcPageArrayData->aBaseArray[ui32IdxSrc] != INVALID_BASE_ADDR)
+		{
+			psDstPageArrayData->aBaseArray[ui32IdxDst++] = psSrcPageArrayData->aBaseArray[ui32IdxSrc];
+			psSrcPageArrayData->aBaseArray[ui32IdxSrc] = INVALID_BASE_ADDR;
+			psDstPageArrayData->iNumChunksAllocated++;
+		}
+	}
+
+	/* Update src page count */
+	psSrcPageArrayData->iNumChunksAllocated = 0;
+
+	*psOutPageArrayData = psDstPageArrayData;
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_FreeLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+             IMG_UINT32 *pui32FreeIndices,
+             IMG_UINT32 ui32FreeChunkCount)
+{
+	PVRSRV_ERROR eError;
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		if (!pui32FreeIndices)
+		{
+			eError =  _FreeLMPagesRemainingSparse(psPageArrayData);
+		}
+		else
+		{
+			eError = _FreeLMPagesSparse(psPageArrayData, pui32FreeIndices, ui32FreeChunkCount);
+		}
+	}
+	else
+	{
+		eError = _FreeLMPagesContig(psPageArrayData);
+	}
+
+	return eError;
+}
+
+/*
+ *
+ * Implementation of PMR callback functions
+ *
+ */
+
+/* destructor func is called after last reference disappears, but
+   before PMR itself is freed. */
+static void
+PMRFinalizeLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+
+	/* We can't free pages until now. */
+	if (psLMAllocArrayData->iNumChunksAllocated != 0)
+	{
+		eError = _FreeLMPages(psLMAllocArrayData, NULL, 0);
+		PVR_LOG_IF_ERROR(eError, "_FreeLMPages");
+		PVR_ASSERT (eError == PVRSRV_OK);
+	}
+
+	_FreeLMPageArray(psLMAllocArrayData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static inline void _TransferToMemZombieRecord_LmaPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+                                                       RA_BASE_T uiBase)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = {
+		.uiAddr = RA_BASE_STRIP_GHOST_BIT(uiBase)
+	};
+
+	PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+	                                  PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES,
+	                                  sCPUPhysAddr.uiAddr,
+	                                  psPageArrayData->uiPid
+	                                  DEBUG_MEMSTATS_VALUES);
+}
+#endif
+
+static PVRSRV_ERROR PMRZombifyLocalMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+	PMR_LMALLOCARRAY_DATA *psPageArrayData = pvPriv;
+
+	BIT_SET(psPageArrayData->ui32Flags, FLAG_ZOMBIE);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_PID uiPid = psPageArrayData->uiPid;
+		IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+		IMG_UINT64 uiSize = BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE) ?
+			(IMG_UINT64) psPageArrayData->iNumChunksAllocated << uiLog2ChunkSize :
+			(IMG_UINT64) psPageArrayData->uiTotalNumChunks << uiLog2ChunkSize;
+
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, uiSize, uiPid);
+		PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES, uiSize, uiPid);
+	}
+#else /* !defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* _FreeLMPagesRemainingSparse path */
+
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+			{
+				IMG_UINT32 j;
+				IMG_UINT32 ui32AccumulatedChunks = 1;
+
+				for (j = i;
+				     j + 1 != psPageArrayData->uiTotalNumChunks &&
+				         RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+				     j++)
+				{
+					ui32AccumulatedChunks++;
+				}
+
+				for (j = i; j < (i + ui32AccumulatedChunks); j++)
+				{
+					_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+					                                    psPageArrayData->aBaseArray[j]);
+				}
+			}
+		}
+	}
+	else
+	{
+		/* _FreeLMPagesContig path */
+
+		if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+			                                    psPageArrayData->aBaseArray[0]);
+		}
+		else
+		{
+			IMG_UINT32 i;
+
+			for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+			{
+				if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+				{
+					_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+					                                    psPageArrayData->aBaseArray[i]);
+				}
+			}
+
+		}
+	}
+#endif /* !defined(PVRSRV_ENABLE_MEMORY_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* callback function for locking the system physical page addresses.
+   As we are LMA there is nothing to do as we control physical memory. */
+static PVRSRV_ERROR
+PMRLockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData;
+
+	psLMAllocArrayData = pvPriv;
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_ONDEMAND))
+	{
+		/* Allocate Memory for deferred allocation */
+		eError = _AllocLMPages(psLMAllocArrayData, NULL);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+                                  PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_LMALLOCARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+	*ppvZombiePages = NULL;
+#endif
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_ONDEMAND))
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (psLMAllocArrayData->iNumChunksAllocated == 0)
+		{
+			*ppvZombiePages = NULL;
+			return PVRSRV_OK;
+		}
+
+		eError = _ExtractAllPages(psLMAllocArrayData,
+		                          &psExtractedPagesPageArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractAllPages", e0);
+
+		if (psExtractedPagesPageArray)
+		{
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyLocalMem(psExtractedPagesPageArray, NULL);
+			PVR_WARN_IF_ERROR(eError, "PMRZombifyLocalMem");
+		}
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		/* Free Memory for deferred allocation */
+		eError = _FreeLMPages(psLMAllocArrayData, NULL, 0);
+		PVR_RETURN_IF_ERROR(eError);
+#endif
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+e0:
+#endif
+	PVR_ASSERT(eError == PVRSRV_OK);
+	return eError;
+}
+
+/* N.B. It is assumed that PMRLockSysPhysAddressesLocalMem() is called _before_ this function! */
+static PVRSRV_ERROR
+PMRSysPhysAddrLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+					   IMG_UINT32 ui32Log2PageSize,
+					   IMG_UINT32 ui32NumOfPages,
+					   IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					   IMG_UINT64 ui64IPAPolicyValue,
+					   IMG_UINT64 ui64IPAClearMask,
+#endif
+					   IMG_BOOL *pbValid,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	IMG_UINT32 idx;
+	IMG_UINT32 uiLog2AllocSize;
+	IMG_UINT64 uiAllocIndex;
+	IMG_DEVMEM_OFFSET_T uiInAllocOffset;
+	IMG_UINT32 uiNumAllocs = psLMAllocArrayData->uiTotalNumChunks;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (psLMAllocArrayData->uiLog2ChunkSize < ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "for incompatible contiguity %u!",
+		         __func__,
+		         ui32Log2PageSize));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	PVR_ASSERT(psLMAllocArrayData->uiLog2ChunkSize != 0);
+	PVR_ASSERT(ui32Log2PageSize >= RA_BASE_FLAGS_LOG2);
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		for (idx=0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				psDevPAddr[idx].uiAddr = psLMAllocArrayData->aBaseArray[0] + puiOffset[idx];
+#if defined(SUPPORT_STATIC_IPA)
+				/* Modify the physical address with the associated IPA values */
+				psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+				psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+			}
+		}
+	}
+	else
+	{
+		uiLog2AllocSize = psLMAllocArrayData->uiLog2ChunkSize;
+
+		for (idx=0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				uiAllocIndex = puiOffset[idx] >> uiLog2AllocSize;
+				uiInAllocOffset = puiOffset[idx] - (uiAllocIndex << uiLog2AllocSize);
+
+				PVR_LOG_RETURN_IF_FALSE(uiAllocIndex < uiNumAllocs,
+										"puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+				PVR_ASSERT(uiInAllocOffset < (1ULL << uiLog2AllocSize));
+
+				/* The base may or may not be a ghost base, but we don't care,
+				 * we just need the real representation of the base.
+				 */
+				psDevPAddr[idx].uiAddr = RA_BASE_STRIP_GHOST_BIT(
+					psLMAllocArrayData->aBaseArray[uiAllocIndex]) + uiInAllocOffset;
+#if defined(SUPPORT_STATIC_IPA)
+				/* Modify the physical address with the associated IPA values */
+				psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+				psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+								 size_t uiOffset,
+								 size_t uiSize,
+								 void **ppvKernelAddressOut,
+								 IMG_HANDLE *phHandleOut,
+								 PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	PMR_KERNEL_MAPPING *psKernelMapping;
+	RA_BASE_T *paBaseArray;
+	IMG_UINT32 ui32ChunkIndex = 0;
+	size_t uiOffsetMask = uiOffset;
+
+	IMG_UINT32 uiLog2ChunkSize = psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT64 ui64PhysSize;
+
+	PVR_ASSERT(psLMAllocArrayData);
+	PVR_ASSERT(ppvKernelAddressOut);
+	PVR_ASSERT(phHandleOut);
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		IMG_UINT32 i;
+		/* Locate the desired physical chunk to map in */
+		ui32ChunkIndex = uiOffset >> psLMAllocArrayData->uiLog2ChunkSize;
+
+		if (OSIsMapPhysNonContigSupported())
+		{
+			/* If a size hasn't been supplied assume we are mapping a single page */
+			IMG_UINT32 uiNumChunksToMap;
+
+			/* This is to support OSMapPMR originated parameters */
+			if (uiOffset == 0 && uiSize == 0)
+			{
+				uiNumChunksToMap = psLMAllocArrayData->iNumChunksAllocated;
+			}
+			else
+			{
+				uiNumChunksToMap = uiSize >> psLMAllocArrayData->uiLog2ChunkSize;
+			}
+
+			/* Check we are attempting to map at least a chunk in size */
+			if (uiNumChunksToMap < 1)
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_INVALID_PARAMS, "uiNumChunksToMap < 1");
+			}
+
+			/* Check contiguous region doesn't exceed size of PMR */
+			if (ui32ChunkIndex + (uiNumChunksToMap - 1) > psLMAllocArrayData->uiTotalNumChunks)
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_INVALID_PARAMS,
+				                        "Mapping range exceeds total num chunks in PMR");
+			}
+
+			/* Check the virtually contiguous region given is physically backed */
+			for (i = ui32ChunkIndex; i < ui32ChunkIndex + uiNumChunksToMap; i++)
+			{
+				if (RA_BASE_IS_INVALID(psLMAllocArrayData->aBaseArray[i]))
+				{
+					PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY, "Sparse contiguity check");
+				}
+			}
+			/* Size of virtually contiguous sparse alloc */
+			ui64PhysSize = IMG_PAGES2BYTES64(uiNumChunksToMap, psLMAllocArrayData->uiLog2ChunkSize);
+		}
+		else
+		{
+			size_t uiStart = uiOffset;
+			size_t uiEnd = uiOffset + uiSize - 1;
+			size_t uiChunkMask = ~(IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize) - 1);
+
+			/* We can still map if only one chunk is required */
+			if ((uiStart & uiChunkMask) != (uiEnd & uiChunkMask))
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY, "Sparse contiguity check");
+			}
+			/* Map a single chunk */
+			ui64PhysSize = ui64ChunkSize;
+		}
+
+		paBaseArray = &psLMAllocArrayData->aBaseArray[ui32ChunkIndex];
+
+		/* Offset mask to be used for address offsets within a chunk */
+		uiOffsetMask = (1U << psLMAllocArrayData->uiLog2ChunkSize) - 1;
+	}
+	else
+	{
+		paBaseArray = psLMAllocArrayData->aBaseArray;
+		ui64PhysSize = IMG_PAGES2BYTES64(psLMAllocArrayData->uiTotalNumChunks, uiLog2ChunkSize);
+	}
+
+	PVR_ASSERT(ui32ChunkIndex < psLMAllocArrayData->uiTotalNumChunks);
+
+	psKernelMapping = OSAllocMem(sizeof(*psKernelMapping));
+	PVR_RETURN_IF_NOMEM(psKernelMapping);
+
+	eError = _MapPMRKernel(psLMAllocArrayData,
+	                       paBaseArray,
+	                       ui64PhysSize,
+	                       ulFlags,
+	                       psKernelMapping);
+	if (eError == PVRSRV_OK)
+	{
+		/* uiOffset & uiOffsetMask is used to get the kernel addr within the page */
+		*ppvKernelAddressOut = ((IMG_CHAR *) psKernelMapping->pvKernelAddress) + (uiOffset & uiOffsetMask);
+		*phHandleOut = psKernelMapping;
+	}
+	else
+	{
+		OSFreeMem(psKernelMapping);
+		PVR_LOG_ERROR(eError, "_MapPMRKernel");
+	}
+
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+                                                IMG_HANDLE hHandle)
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = (PMR_LMALLOCARRAY_DATA *) pvPriv;
+	PMR_KERNEL_MAPPING *psKernelMapping = (PMR_KERNEL_MAPPING *) hHandle;
+
+	PVR_ASSERT(psLMAllocArrayData);
+	PVR_ASSERT(psKernelMapping);
+
+	_UnMapPMRKernel(psLMAllocArrayData,
+	                psKernelMapping);
+
+	OSFreeMem(psKernelMapping);
+}
+
+static PVRSRV_ERROR
+CopyBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+				  IMG_DEVMEM_OFFSET_T uiOffset,
+				  IMG_UINT8 *pcBuffer,
+				  size_t uiBufSz,
+				  size_t *puiNumBytes,
+				  void (*pfnCopyBytes)(IMG_UINT8 *pcBuffer,
+									   IMG_UINT8 *pcPMR,
+									   size_t uiSize))
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	size_t uiBytesCopied;
+	size_t uiBytesToCopy;
+	size_t uiBytesCopyableFromAlloc;
+	PMR_KERNEL_MAPPING sMapping;
+	IMG_UINT8 *pcKernelPointer = NULL;
+	size_t uiBufferOffset;
+	IMG_UINT64 uiAllocIndex;
+	IMG_DEVMEM_OFFSET_T uiInAllocOffset;
+	IMG_UINT32 uiLog2ChunkSize = psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT64 ui64PhysSize;
+	PVRSRV_ERROR eError;
+
+	uiBytesCopied = 0;
+	uiBytesToCopy = uiBufSz;
+	uiBufferOffset = 0;
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		while (uiBytesToCopy > 0)
+		{
+			/* we have to map one alloc in at a time */
+			PVR_ASSERT(psLMAllocArrayData->uiLog2ChunkSize != 0);
+			uiAllocIndex = uiOffset >> psLMAllocArrayData->uiLog2ChunkSize;
+			uiInAllocOffset = uiOffset - (uiAllocIndex << psLMAllocArrayData->uiLog2ChunkSize);
+			uiBytesCopyableFromAlloc = uiBytesToCopy;
+			if (uiBytesCopyableFromAlloc + uiInAllocOffset > (1ULL << psLMAllocArrayData->uiLog2ChunkSize))
+			{
+				uiBytesCopyableFromAlloc = TRUNCATE_64BITS_TO_SIZE_T((1ULL << psLMAllocArrayData->uiLog2ChunkSize)-uiInAllocOffset);
+			}
+			/* Mapping a single chunk at a time */
+			ui64PhysSize = ui64ChunkSize;
+
+			PVR_ASSERT(uiBytesCopyableFromAlloc != 0);
+			PVR_ASSERT(uiAllocIndex < psLMAllocArrayData->uiTotalNumChunks);
+			PVR_ASSERT(uiInAllocOffset < (1ULL << uiLog2ChunkSize));
+
+			eError = _MapPMRKernel(psLMAllocArrayData,
+			                       &psLMAllocArrayData->aBaseArray[uiAllocIndex],
+			                       ui64PhysSize,
+			                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+			                       &sMapping);
+			PVR_GOTO_IF_ERROR(eError, e0);
+			pcKernelPointer = sMapping.pvKernelAddress;
+			pfnCopyBytes(&pcBuffer[uiBufferOffset], &pcKernelPointer[uiInAllocOffset], uiBytesCopyableFromAlloc);
+
+			_UnMapPMRKernel(psLMAllocArrayData,
+			                &sMapping);
+
+			uiBufferOffset += uiBytesCopyableFromAlloc;
+			uiBytesToCopy -= uiBytesCopyableFromAlloc;
+			uiOffset += uiBytesCopyableFromAlloc;
+			uiBytesCopied += uiBytesCopyableFromAlloc;
+		}
+	}
+	else
+	{
+		ui64PhysSize = IMG_PAGES2BYTES64(psLMAllocArrayData->uiTotalNumChunks, uiLog2ChunkSize);
+		PVR_ASSERT((uiOffset + uiBufSz) <= ui64PhysSize);
+		PVR_ASSERT(ui64ChunkSize != 0);
+		eError = _MapPMRKernel(psLMAllocArrayData,
+		                       psLMAllocArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+		                       &sMapping);
+		PVR_GOTO_IF_ERROR(eError, e0);
+		pcKernelPointer = sMapping.pvKernelAddress;
+		pfnCopyBytes(pcBuffer, &pcKernelPointer[uiOffset], uiBufSz);
+
+		_UnMapPMRKernel(psLMAllocArrayData,
+		                &sMapping);
+
+		uiBytesCopied = uiBufSz;
+	}
+	*puiNumBytes = uiBytesCopied;
+	return PVRSRV_OK;
+e0:
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+static void ReadLocalMem(IMG_UINT8 *pcBuffer,
+						 IMG_UINT8 *pcPMR,
+						 size_t uiSize)
+{
+	/* the memory is mapped as WC (and also aligned to page size) so we can
+	 * safely call "Cached" memcpy */
+	OSCachedMemCopy(pcBuffer, pcPMR, uiSize);
+}
+
+static PVRSRV_ERROR
+PMRReadBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+				  IMG_DEVMEM_OFFSET_T uiOffset,
+				  IMG_UINT8 *pcBuffer,
+				  size_t uiBufSz,
+				  size_t *puiNumBytes)
+{
+	return CopyBytesLocalMem(pvPriv,
+							 uiOffset,
+							 pcBuffer,
+							 uiBufSz,
+							 puiNumBytes,
+							 ReadLocalMem);
+}
+
+static void WriteLocalMem(IMG_UINT8 *pcBuffer,
+						  IMG_UINT8 *pcPMR,
+						  size_t uiSize)
+{
+	/* the memory is mapped as WC (and also aligned to page size) so we can
+	 * safely call "Cached" memcpy but need to issue a write memory barrier
+	 * to flush the write buffers after */
+	OSCachedMemCopyWMB(pcPMR, pcBuffer, uiSize);
+}
+
+static PVRSRV_ERROR
+PMRWriteBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+					  IMG_DEVMEM_OFFSET_T uiOffset,
+					  IMG_UINT8 *pcBuffer,
+					  size_t uiBufSz,
+					  size_t *puiNumBytes)
+{
+	return CopyBytesLocalMem(pvPriv,
+							 uiOffset,
+							 pcBuffer,
+							 uiBufSz,
+							 puiNumBytes,
+							 WriteLocalMem);
+}
+
+/*************************************************************************/ /*!
+@Function       PMRChangeSparseMemLocalMem
+@Description    This function Changes the sparse mapping by allocating and
+                freeing of pages. It also changes the GPU maps accordingly.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PMRChangeSparseMemLocalMem(PMR_IMPL_PRIVDATA pPriv,
+                           const PMR *psPMR,
+                           IMG_UINT32 ui32AllocPageCount,
+                           IMG_UINT32 *pai32AllocIndices,
+                           IMG_UINT32 ui32FreePageCount,
+                           IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+                           PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+                           IMG_UINT32 uiFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+	IMG_UINT32 ui32AdtnlAllocPages = 0;
+	IMG_UINT32 ui32AdtnlFreePages = 0;
+	IMG_UINT32 ui32CommonRequstCount = 0;
+	IMG_UINT32 ui32Loop = 0;
+	IMG_UINT32 ui32Index = 0;
+	IMG_UINT32 uiAllocpgidx;
+	IMG_UINT32 uiFreepgidx;
+
+	PMR_LMALLOCARRAY_DATA *psPMRPageArrayData = (PMR_LMALLOCARRAY_DATA *)pPriv;
+	IMG_UINT32 uiLog2ChunkSize = psPMRPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+
+#if defined(DEBUG)
+	IMG_BOOL bPoisonFail = IMG_FALSE;
+	IMG_BOOL bZeroFail = IMG_FALSE;
+#endif
+
+	/* Fetch the Page table array represented by the PMR */
+	RA_BASE_T *paBaseArray = psPMRPageArrayData->aBaseArray;
+	PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappingTable(psPMR);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+	/* The incoming request is classified into two operations independent of
+	 * each other: alloc & free chunks.
+	 * These operations can be combined with two mapping operations as well
+	 * which are GPU & CPU space mappings.
+	 *
+	 * From the alloc and free chunk requests, the net amount of chunks to be
+	 * allocated or freed is computed. Chunks that were requested to be freed
+	 * will be reused to fulfil alloc requests.
+	 *
+	 * The order of operations is:
+	 * 1. Allocate new Chunks.
+	 * 2. Move the free chunks from free request to alloc positions.
+	 * 3. Free the rest of the chunks not used for alloc
+	 *
+	 * Alloc parameters are validated at the time of allocation
+	 * and any error will be handled then. */
+
+	if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH))
+	{
+		ui32CommonRequstCount = (ui32AllocPageCount > ui32FreePageCount) ?
+				ui32FreePageCount : ui32AllocPageCount;
+
+		PDUMP_PANIC(PMR_DeviceNode(psPMR), SPARSEMEM_SWAP, "Request to swap alloc & free chunks not supported");
+	}
+
+	if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC))
+	{
+		ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequstCount;
+	}
+	else
+	{
+		ui32AllocPageCount = 0;
+	}
+
+	if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE))
+	{
+		ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequstCount;
+	}
+	else
+	{
+		ui32FreePageCount = 0;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(
+	    (ui32CommonRequstCount | ui32AdtnlAllocPages | ui32AdtnlFreePages) != 0,
+	    "Invalid combination of parameters: ui32CommonRequstCount,"
+	    " ui32AdtnlAllocPages and ui32AdtnlFreePages.",
+	    PVRSRV_ERROR_INVALID_PARAMS
+	);
+
+	{
+		/* Validate the free page indices */
+		if (ui32FreePageCount)
+		{
+			if (pai32FreeIndices != NULL)
+			{
+				for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++)
+				{
+					uiFreepgidx = pai32FreeIndices[ui32Loop];
+
+					if (uiFreepgidx >= psPMRPageArrayData->uiTotalNumChunks)
+					{
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+					}
+
+					if (RA_BASE_IS_INVALID(paBaseArray[uiFreepgidx]))
+					{
+						PVR_LOG_GOTO_WITH_ERROR("paBaseArray[uiFreepgidx]", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Given non-zero free count but missing indices array",
+				         __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+		}
+
+		/* The following block of code verifies any issues with common alloc chunk indices */
+		for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++)
+		{
+			uiAllocpgidx = pai32AllocIndices[ui32Loop];
+			if (uiAllocpgidx >= psPMRPageArrayData->uiTotalNumChunks)
+			{
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+			}
+
+			if ((!RA_BASE_IS_INVALID(paBaseArray[uiAllocpgidx])) ||
+					(psPMRMapTable->aui32Translation[uiAllocpgidx] != TRANSLATION_INVALID))
+			{
+				PVR_LOG_GOTO_WITH_ERROR("Trying to allocate already allocated page again", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+			}
+		}
+
+		ui32Loop = 0;
+
+		/* Allocate new chunks */
+		if (0 != ui32AdtnlAllocPages)
+		{
+			/* Say how many chunks to allocate */
+			psPMRPageArrayData->uiChunksToAlloc = ui32AdtnlAllocPages;
+
+			eError = _AllocLMPages(psPMRPageArrayData, pai32AllocIndices);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_AllocLMPages", e0);
+
+			/* Mark the corresponding chunks of translation table as valid */
+			for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++)
+			{
+				psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop];
+			}
+
+			psPMRMapTable->ui32NumPhysChunks += ui32AdtnlAllocPages;
+		}
+
+		ui32Index = ui32Loop;
+		ui32Loop = 0;
+
+		/* Move the corresponding free chunks to alloc request */
+		eError = RA_SwapSparseMem(psPMRPageArrayData->psArena,
+		                           paBaseArray,
+		                           psPMRPageArrayData->uiTotalNumChunks,
+		                           psPMRPageArrayData->uiLog2ChunkSize,
+		                           &pai32AllocIndices[ui32Index],
+		                           &pai32FreeIndices[ui32Loop],
+		                           ui32CommonRequstCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RA_SwapSparseMem", unwind_alloc);
+
+		for (ui32Loop = 0; ui32Loop < ui32CommonRequstCount; ui32Loop++, ui32Index++)
+		{
+			uiAllocpgidx = pai32AllocIndices[ui32Index];
+			uiFreepgidx  = pai32FreeIndices[ui32Loop];
+
+			psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID;
+			psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
+
+			/* Be sure to honour the attributes associated with the allocation
+			 * such as zeroing, poisoning etc. */
+			if (BIT_ISSET(psPMRPageArrayData->ui32Flags, FLAG_POISON_ON_ALLOC))
+			{
+				eError = _PhysPgMemSet(psPMRPageArrayData,
+				                       &psPMRPageArrayData->aBaseArray[uiAllocpgidx],
+				                       ui64ChunkSize,
+				                       PVRSRV_POISON_ON_ALLOC_VALUE);
+
+				/* Consider this as a soft failure and go ahead but log error to kernel log */
+				if (eError != PVRSRV_OK)
+				{
+#if defined(DEBUG)
+					bPoisonFail = IMG_TRUE;
+#endif
+				}
+			}
+
+			if (BIT_ISSET(psPMRPageArrayData->ui32Flags, FLAG_ZERO))
+			{
+				eError = _PhysPgMemSet(psPMRPageArrayData,
+									   &psPMRPageArrayData->aBaseArray[uiAllocpgidx],
+									   ui64ChunkSize,
+									   ZERO_PAGE_VALUE);
+				/* Consider this as a soft failure and go ahead but log error to kernel log */
+				if (eError != PVRSRV_OK)
+				{
+#if defined(DEBUG)
+					/* Don't think we need to zero any chunks further */
+					bZeroFail = IMG_TRUE;
+#endif
+				}
+			}
+		}
+
+		/* Free or zombie the additional free chunks */
+		if (0 != ui32AdtnlFreePages)
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			PMR_LMALLOCARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+			eError = _ExtractPages(psPMRPageArrayData, &pai32FreeIndices[ui32Loop], ui32AdtnlFreePages, &psExtractedPagesPageArray);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractPages", e0);
+
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyLocalMem(psExtractedPagesPageArray, NULL);
+			PVR_LOG_IF_ERROR(eError, "PMRZombifyLocalMem");
+
+			*ppvZombiePages = psExtractedPagesPageArray;
+#else
+			eError = _FreeLMPages(psPMRPageArrayData, &pai32FreeIndices[ui32Loop], ui32AdtnlFreePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_FreeLMPages", e0);
+#endif /* SUPPORT_PMR_PAGES_DEFERRED_FREE */
+			ui32Index = ui32Loop;
+			ui32Loop = 0;
+
+			while (ui32Loop++ < ui32AdtnlFreePages)
+			{
+				/* Set the corresponding mapping table entry to invalid address */
+				psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Index++]] = TRANSLATION_INVALID;
+			}
+
+			psPMRMapTable->ui32NumPhysChunks -= ui32AdtnlFreePages;
+		}
+	}
+
+#if defined(DEBUG)
+	if (IMG_TRUE == bPoisonFail)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error in poisoning the chunk", __func__));
+	}
+
+	if (IMG_TRUE == bZeroFail)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error in zeroing the chunk", __func__));
+	}
+#endif
+
+	return PVRSRV_OK;
+
+unwind_alloc:
+	_FreeLMPages(psPMRPageArrayData, pai32AllocIndices, ui32Index);
+
+	for (ui32Loop = 0; ui32Loop < ui32Index; ui32Loop++)
+	{
+		psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = TRANSLATION_INVALID;
+	}
+
+e0:
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRLMAFuncTab = {
+	.pfnLockPhysAddresses = &PMRLockSysPhysAddressesLocalMem,
+	.pfnUnlockPhysAddresses = &PMRUnlockSysPhysAddressesLocalMem,
+	.pfnDevPhysAddr = &PMRSysPhysAddrLocalMem,
+	.pfnAcquireKernelMappingData = &PMRAcquireKernelMappingDataLocalMem,
+	.pfnReleaseKernelMappingData = &PMRReleaseKernelMappingDataLocalMem,
+	.pfnReadBytes = &PMRReadBytesLocalMem,
+	.pfnWriteBytes = &PMRWriteBytesLocalMem,
+	.pfnChangeSparseMem = &PMRChangeSparseMemLocalMem,
+	.pfnMMap = NULL,
+	.pfnFinalize = &PMRFinalizeLocalMem,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	.pfnFreeZombiePages = &PMRFreeZombiePagesRAMem,
+#endif
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = &PMRZombifyLocalMem,
+#endif
+};
+
+PVRSRV_ERROR
+PhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                            RA_ARENA *pArena,
+                            CONNECTION_DATA *psConnection,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_UINT32 ui32NumPhysChunks,
+                            IMG_UINT32 ui32NumVirtChunks,
+                            IMG_UINT32 *pui32MappingTable,
+                            IMG_UINT32 uiLog2AllocPageSize,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                            const IMG_CHAR *pszAnnotation,
+                            IMG_PID uiPid,
+                            PMR **ppsPMRPtr,
+                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	PMR *psPMR = NULL;
+	PMR_LMALLOCARRAY_DATA *psPrivData = NULL;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_UINT32 ui32LMAllocFlags = 0;
+
+	/* This path is checking for the type of PMR to create, if sparse we
+	 * have to perform additional validation as we can only map sparse ranges
+	 * if the os functionality to do so is present. We can also only map virtually
+	 * contiguous sparse regions. Non backed gaps in a range cannot be mapped.
+	 */
+	if (ui32NumPhysChunks != ui32NumVirtChunks || ui32NumVirtChunks > 1)
+	{
+		if (PVRSRV_CHECK_KERNEL_CPU_MAPPABLE(uiFlags) &&
+		   !OSIsMapPhysNonContigSupported())
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: LMA kernel mapping functions not available "
+					"for physically discontiguous memory.",
+					__func__));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, errorOnParam);
+		}
+		BIT_SET(ui32LMAllocFlags, FLAG_SPARSE);
+	}
+
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_ONDEMAND);
+	}
+
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_ZERO);
+	}
+
+	if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_POISON_ON_ALLOC);
+	}
+
+#if defined(DEBUG)
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_POISON_ON_FREE);
+	}
+#endif
+
+	/* Create Array structure that holds the physical pages */
+	eError = _AllocLMPageArray(uiSize,
+	                           ui32NumPhysChunks,
+	                           uiLog2AllocPageSize,
+	                           ui32LMAllocFlags,
+	                           psPhysHeap,
+	                           pArena,
+	                           uiFlags,
+	                           uiPid,
+	                           &psPrivData,
+	                           psConnection);
+	PVR_GOTO_IF_ERROR(eError, errorOnAllocPageArray);
+
+	if (!BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND))
+	{
+		/* Allocate the physical pages */
+		eError = _AllocLMPages(psPrivData, pui32MappingTable);
+		PVR_GOTO_IF_ERROR(eError, errorOnAllocPages);
+	}
+
+	/* In this instance, we simply pass flags straight through.
+
+	   Generically, uiFlags can include things that control the PMR
+	   factory, but we don't need any such thing (at the time of
+	   writing!), and our caller specifies all PMR flags so we don't
+	   need to meddle with what was given to us.
+	*/
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+	/* check no significant bits were lost in cast due to different
+	   bit widths for flags */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND))
+	{
+		PDUMPCOMMENT(PhysHeapDeviceNode(psPhysHeap), "Deferred Allocation PMR (LMA)");
+	}
+
+	eError = PMRCreatePMR(psPhysHeap,
+						  uiSize,
+						  ui32NumPhysChunks,
+						  ui32NumVirtChunks,
+						  pui32MappingTable,
+						  uiLog2AllocPageSize,
+						  uiPMRFlags,
+						  pszAnnotation,
+						  &_sPMRLMAFuncTab,
+						  psPrivData,
+						  PMR_TYPE_LMA,
+						  &psPMR,
+						  ui32PDumpFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRCreatePMR", errorOnCreate);
+
+	*ppsPMRPtr = psPMR;
+	return PVRSRV_OK;
+
+errorOnCreate:
+	if (!BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND) && psPrivData->iNumChunksAllocated)
+	{
+		eError2 = _FreeLMPages(psPrivData, NULL, 0);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+errorOnAllocPages:
+	_FreeLMPageArray(psPrivData);
+
+errorOnAllocPageArray:
+errorOnParam:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pmr.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pmr.c
new file mode 100644
index 000000000000..588b076a4ed5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pmr.c
@@ -0,0 +1,5468 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                the "PMR" abstraction.  A PMR (Physical Memory Resource)
+                represents some unit of physical memory which is
+                allocated/freed/mapped/unmapped as an indivisible unit
+                (higher software levels provide an abstraction above that
+                to deal with dividing this down into smaller manageable units).
+                Importantly, this module knows nothing of virtual memory, or
+                of MMUs etc., with one excusable exception.  We have the
+                concept of a "page size", which really means nothing in
+                physical memory, but represents a "contiguity quantum" such
+                that the higher level modules which map this memory are able
+                to verify that it matches the needs of the page size for the
+                virtual realm into which it is being mapped.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "devicemem_server_utils.h"
+#include "devicemem_server.h"
+
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "pmr_impl.h"
+#include "pmr_os.h"
+#include "pmr_env.h"
+#include "pvrsrv.h"
+
+#include "allocmem.h"
+#include "lock.h"
+#include "uniq_key_splay_tree.h"
+
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "secure_export.h"
+#include "ossecure_export.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+/* ourselves */
+#include "pmr.h"
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#include "proc_stats.h"
+#endif
+
+#include "pdump_km.h"
+
+#define PMR_FLAG_INTERNAL_SPARSE_ALLOC     (1 << 0)
+#define PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE (1 << 1)
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PMR_FLAG_INTERNAL_DEFER_FREE       (1 << 2)
+#define PMR_FLAG_INTERNAL_IS_ZOMBIE        (1 << 3)
+
+/* Indicates PMR should be destroyed immediately and not deferred. */
+#define PMR_NO_ZOMBIE_FENCE IMG_UINT64_MAX
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Memalloc flags can be converted into pmr, ra or psplay flags.
+ * Ensure flags types are same size.
+ */
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(PMR_FLAGS_T),
+			  "Mismatch memalloc and pmr flags type size.");
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(RA_FLAGS_T),
+			  "Mismatch memalloc and ra flags type size.");
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(IMG_PSPLAY_FLAGS_T),
+			  "Mismatch memalloc and psplay flags type size.");
+
+/* A "context" for the physical memory block resource allocator.
+ *
+ * Context is probably the wrong word.
+ *
+ * There is almost certainly only one of these, ever, in the system.
+ * But, let's keep the notion of a context anyway, "just-in-case".
+ */
+static struct _PMR_CTX_
+{
+	/* For debugging, and PDump, etc., let's issue a forever incrementing
+	 * serial number to each allocation.
+	 */
+	IMG_UINT64 uiNextSerialNum;
+
+	/* For security, we only allow a PMR to be mapped if the caller knows
+	 * its key. We can pseudo-randomly generate keys
+	 */
+	IMG_UINT64 uiNextKey;
+
+	/* For debugging only, I guess: Number of live PMRs */
+	ATOMIC_T uiNumLivePMRs;
+
+	/* Lock for this structure */
+	POS_LOCK hLock;
+
+	/* In order to seed the uiNextKey, we enforce initialisation at driver
+	 * load time. Also, we can debug check at driver unload that the PMR
+	 * count is zero.
+	 */
+	IMG_BOOL bModuleInitialised;
+} _gsSingletonPMRContext = { 1, 0, {0}, NULL, IMG_FALSE };
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+typedef enum _PMR_ZOMBIE_TYPE_ {
+	PMR_ZOMBIE_TYPE_PMR,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_ZOMBIE_TYPE_PAGES,
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	PMR_ZOMBIE_TYPE_DEVICE_IMPORT,
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+} PMR_ZOMBIE_TYPE;
+
+typedef struct _PMR_HEADER_
+{
+	/* List node used to put the header on the zombie list
+	 * (psDevNode->sPMRZombieList). */
+	DLLIST_NODE sZombieNode;
+
+	PMR_ZOMBIE_TYPE eZombieType;
+} PMR_HEADER;
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/*
+ * A structure describing zombie pages.
+ */
+typedef struct _PMR_ZOMBIE_PAGES_
+{
+	PMR_HEADER sHeader;
+	PMR_IMPL_ZOMBIEPAGES pvFactoryPages;
+	PFN_FREE_ZOMBIE_PAGES_FN pfnFactoryFreeZombies;
+} PMR_ZOMBIE_PAGES;
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+typedef enum _PMR_STATE_
+{
+	PMR_STATE_INIT,
+	PMR_STATE_ACTIVE,
+	PMR_STATE_PAGES_IN_MIGRATE
+} PMR_STATE;
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+typedef struct _PMR_DEVICE_IMPORT_
+{
+	PMR_HEADER sHeader;             /* psDevNode zombie queue list node. */
+	DLLIST_NODE sNext;              /* PMR::sXDeviceImports list node. */
+	PVRSRV_DEVICE_NODE *psDevNode;  /* Device this import is representing. */
+	PMR *psParent;                  /* PMR the import belongs to. */
+} PMR_DEVICE_IMPORT;
+#endif
+
+/* A PMR. One per physical allocation. May be "shared".
+ *
+ * "shared" is ambiguous. We need to be careful with terminology.
+ * There are two ways in which a PMR may be "shared" and we need to be sure
+ * that we are clear which we mean.
+ *
+ * i)   multiple small allocations living together inside one PMR.
+ *
+ * ii)  one single allocation filling a PMR but mapped into multiple memory
+ *      contexts.
+ *
+ * This is more important further up the stack - at this level, all we care is
+ * that the PMR is being referenced multiple times.
+ */
+struct _PMR_
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* A Common header structure shared between PMR and PMR-like PMR_ZOMBIE_PAGES object */
+	PMR_HEADER sHeader;
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	/* This object is strictly refcounted. References include:
+	 * - mapping
+	 * - live handles (to this object)
+	 * - live export handles
+	 * (thus it is normal for allocated and exported memory to have a refcount of 3)
+	 * The object is destroyed when and only when the refcount reaches 0
+	 */
+
+	/* Physical address translation (device <> cpu) is done on a per device
+	 * basis which means we need the physical heap info
+	 */
+	PHYS_HEAP *psPhysHeap;
+
+	/* Reference count of the PMR. */
+	IMG_UINT32 uiRefCount;
+	/* Lock protecting reference counting (uiRefCount). */
+	POS_SPINLOCK hRefCountLock;
+
+	/* Client CPU mapping count - this is the number of times the PMR has been
+	 * mapped by the client to the CPU. It is used to determine when it
+	 * is safe to permit modification of a sparse allocation's layout.
+	 * Note that the process of mapping also increments uiRefCount
+	 * independently (as that is used to determine when a PMR may safely
+	 * be destroyed).
+	 */
+	ATOMIC_T iClientCpuMapCount;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+	/* Kernel CPU mapping count - number of times the PMR has been
+	 * mapped into the kernel. Used to signal if migrate is allowed
+	 * to occur.
+	 */
+	ATOMIC_T iKernelCpuMapCount;
+	/*
+	 * GPU mappings associated with PMR.
+	 * Must be protected with PMR lock.
+	 */
+	DLLIST_NODE sGpuMappingListHead;
+
+	/*
+	 * Current state of PMR
+	 * Must be protected by PMR lock.
+	 * Used to describe the current state of the PMR and determine which interactions
+	 * are possible. Pages could be in progress of migration during which physical
+	 * addresses cannot be requested.
+	 */
+	PMR_STATE eState;
+#endif
+
+	/* Count of how many reservations refer to this
+	 * PMR as a part of a GPU mapping. Must be protected
+	 * by PMR lock.
+	 */
+	IMG_INT32 iAssociatedResCount;
+
+	/* Lock count - this is the number of times PMRLockSysPhysAddresses()
+	 * has been called, less the number of PMRUnlockSysPhysAddresses()
+	 * calls. This is arguably here for debug reasons only, as the refcount
+	 * is already incremented as a matter of course.
+	 * Really, this just allows us to trap protocol errors: i.e. calling
+	 * PMRSysPhysAddr(), without a lock, or calling
+	 * PMRUnlockSysPhysAddresses() too many or too few times.
+	 */
+	ATOMIC_T iLockCount;
+
+	/* Lock for this structure */
+	POS_LOCK hLock;
+	/* Protects: `uiInternalFlags` & `uiDevImportBitmap` */
+	POS_SPINLOCK hBitmapLock;
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+	/* See PMR_ImportedDevicesMask()
+	 * Protected by hBitmapLock. */
+	IMG_UINT64 uiDevImportBitmap;
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	/* List of PMR_DEVICE_IMPORT's */
+	DLLIST_NODE sXDeviceImports;
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	/* Incrementing serial number to each allocation. */
+	IMG_UINT64 uiSerialNum;
+
+	/* For security, we only allow a PMR to be mapped if the caller knows
+	 * its key. We can pseudo-randomly generate keys
+	 */
+	PMR_PASSWORD_T uiKey;
+
+	/* Callbacks for per-flavour functions */
+	const PMR_IMPL_FUNCTAB *psFuncTab;
+
+	/* Data associated with the "subtype" */
+	PMR_IMPL_PRIVDATA pvFlavourData;
+
+	/* Environment specific data */
+	DECLARE_PMR_ENV_DATA(sEnvData)
+
+	/* What kind of PMR do we have? */
+	PMR_IMPL_TYPE eFlavour;
+
+	/* And for pdump */
+	const IMG_CHAR *pszPDumpDefaultMemspaceName;
+
+	/* Allocation annotation */
+	IMG_CHAR szAnnotation[DEVMEM_ANNOTATION_MAX_LEN];
+
+#if defined(PDUMP)
+
+	IMG_HANDLE hPDumpAllocHandle;
+
+	IMG_UINT32 uiNumPDumpBlocks;
+#endif
+
+	/* Logical size of allocation. "logical", because a PMR can represent
+	 * memory that will never physically exist.  This is the amount of
+	 * virtual space that the PMR would consume when it's mapped into a
+	 * virtual allocation.
+	 */
+	PMR_SIZE_T uiLogicalSize;
+
+	/* Mapping table for the allocation.
+	 * PMR's can be sparse in which case not all the "logic" addresses in
+	 * it are valid. We need to know which addresses are and aren't valid
+	 * when mapping or reading the PMR.
+	 * The mapping table translates "logical" offsets into physical offsets
+	 * which is what we always pass to the PMR factory (so it doesn't have
+	 * to be concerned about sparseness issues)
+	 */
+	PMR_MAPPING_TABLE *psMappingTable;
+
+	/* Minimum Physical Contiguity Guarantee.  Might be called "page size",
+	 * but that would be incorrect, as page size is something meaningful
+	 * only in virtual realm. This contiguity guarantee provides an
+	 * inequality that can be verified/asserted/whatever to ensure that
+	 * this PMR conforms to the page size requirement of the place the PMR
+	 * gets mapped. (May be used to select an appropriate heap in variable
+	 * page size systems)
+	 *
+	 * The absolutely necessary condition is this:
+	 *
+	 *    device MMU page size <= actual physical contiguity.
+	 *
+	 * We go one step further in order to be able to provide an early
+	 * warning / early compatibility check and say this:
+	 *
+	 *     device MMU page size <=
+	 *         2**(uiLog2ContiguityGuarantee) <=
+	 *             actual physical contiguity.
+	 *
+	 * In this way, it is possible to make the page table reservation
+	 * in the device MMU without even knowing the granularity of the
+	 * physical memory (i.e. useful for being able to allocate virtual
+	 * before physical)
+	 */
+	PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee;
+
+	/* Flags. We store a copy of the "PMR flags" (usually a subset of the
+	 * flags given at allocation time) and return them to any caller of
+	 * PMR_Flags(). The intention of these flags is that the ones stored
+	 * here are used to represent permissions, such that no one is able
+	 * to map a PMR in a mode in which they are not allowed, e.g.,
+	 * writeable for a read-only PMR, etc.
+	 */
+	PMR_FLAGS_T uiFlags;
+
+	/* Various flags informing about PMR's state:
+	 *
+	 * SPARSE_ALLOC:
+	 *   indicates whether this PMR has been allocated as sparse.
+	 *   The condition for this variable to be set at allocation time is:
+	 *   (numVirtChunks != numPhysChunks) || (numVirtChunks > 1)
+	 *
+	 * NO_LAYOUT_CHANGE:
+	 *   Flag that conveys mutability of the PMR:
+	 *   - set:   indicates the PMR is immutable (no more memory changes)
+	 *   - unset: means the memory layout associated with the PMR is mutable
+	 *
+	 *   A PMR is always mutable by default but is marked immutable on the
+	 *   first export for the rest of its life.
+	 *
+	 *   Also, any PMRs that track the same memory through imports are
+	 *   marked immutable as well.
+	 *
+	 * DEFER_FREE:
+	 *   If present the PMR is marked to be freed by the CleanupThread.
+	 *
+	 * IS_ZOMBIE:
+	 *   Indicates if the PMR is in the zombie state (marked for free in the
+	 *   CleanupThread). */
+	IMG_UINT32 uiInternalFlags;
+
+	/* Do we really need this?
+	 * For now we'll keep it, until we know we don't.
+	 * NB: this is not the "memory context" in client terms - this is
+	 * _purely_ the "PMR" context, of which there is almost certainly only
+	 * ever one per system as a whole, but we'll keep the concept anyway,
+	 * just-in-case.
+	 */
+	struct _PMR_CTX_ *psContext;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Stored handle to PMR RI entry */
+	void		*hRIHandle;
+#endif
+};
+
+/* Do we need a struct for the export handle?
+ * I'll use one for now, but if nothing goes in it, we'll lose it
+ */
+struct _PMR_EXPORT_
+{
+	struct _PMR_ *psPMR;
+};
+
+struct _PMR_PAGELIST_
+{
+	struct _PMR_ *psReferencePMR;
+};
+
+static INLINE IMG_UINT32
+_GetRef(const PMR *const psPMR)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT32 uiRefCount;
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+	uiRefCount = psPMR->uiRefCount;
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return uiRefCount;
+}
+
+#if defined(PDUMP)
+static INLINE IMG_BOOL _IsHostDevicePMR(const PMR *const psPMR)
+{
+	const PVRSRV_DEVICE_NODE *psDevNode = PVRSRVGetPVRSRVData()->psHostMemDeviceNode;
+	return psPMR->psPhysHeap == psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL];
+}
+
+static void
+PDumpPMRFreePMR(PMR *psPMR,
+                IMG_DEVMEM_SIZE_T uiSize,
+                IMG_DEVMEM_ALIGN_T uiBlockSize,
+                IMG_UINT32 uiLog2Contiguity,
+                IMG_HANDLE hPDumpAllocationInfoHandle);
+
+static void
+PDumpPMRMallocPMR(PMR *psPMR,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT32 ui32ChunkSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 ui32NumLogicalChunks,
+                  IMG_UINT32 *puiMappingTable,
+                  IMG_UINT32 uiLog2Contiguity,
+                  IMG_BOOL bInitialise,
+                  IMG_UINT8 ui8InitValue,
+                  IMG_HANDLE *phPDumpAllocInfoOut,
+                  IMG_UINT32 ui32PDumpFlags);
+
+static void
+PDumpPMRChangeSparsePMR(PMR *psPMR,
+                        IMG_UINT32 uiBlockSize,
+                        IMG_UINT32 ui32AllocPageCount,
+                        IMG_UINT32 *pai32AllocIndices,
+                        IMG_UINT32 ui32FreePageCount,
+                        IMG_UINT32 *pai32FreeIndices,
+                        IMG_BOOL bInitialise,
+                        IMG_UINT8 ui8InitValue,
+                        IMG_HANDLE *phPDumpAllocInfoOut);
+#endif /* defined PDUMP */
+
+IMG_INT32 PMRGetLiveCount(void)
+{
+	return OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs);
+}
+
+PPVRSRV_DEVICE_NODE PMRGetExportDeviceNode(PMR_EXPORT *psExportPMR)
+{
+	PPVRSRV_DEVICE_NODE psReturnedDeviceNode = NULL;
+
+	PVR_ASSERT(psExportPMR != NULL);
+	if (psExportPMR)
+	{
+		PVR_ASSERT(psExportPMR->psPMR != NULL);
+		if (psExportPMR->psPMR)
+		{
+			IMG_UINT32 uiRefCount = _GetRef(psExportPMR->psPMR);
+			if (uiRefCount > 0)
+			{
+				psReturnedDeviceNode = PMR_DeviceNode(psExportPMR->psPMR);
+			}
+#ifdef PVRSRV_NEED_PVR_ASSERT
+			else
+			{
+				PVR_ASSERT(IMG_FALSE);
+			}
+#endif
+		}
+	}
+
+	return psReturnedDeviceNode;
+}
+
+static PVRSRV_ERROR
+_PMRCreate(PMR_SIZE_T uiLogicalSize,
+           IMG_UINT32 ui32NumPhysChunks,
+           IMG_UINT32 ui32NumLogicalChunks,
+           IMG_UINT32 *pui32MappingTable,
+           PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+           PMR_FLAGS_T uiFlags,
+           PMR **ppsPMR)
+{
+	void *pvPMRLinAddr;
+	PMR *psPMR;
+	PMR_MAPPING_TABLE *psMappingTable;
+	struct _PMR_CTX_ *psContext;
+	IMG_UINT32 i, ui32Temp = 0;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bSparse = IMG_FALSE;
+	PMR_SIZE_T uiChunkSize;
+
+	psContext = &_gsSingletonPMRContext;
+
+	/* Do we have a sparse allocation? */
+	if ( (ui32NumLogicalChunks != ui32NumPhysChunks) ||
+			(ui32NumLogicalChunks > 1) )
+	{
+		bSparse = IMG_TRUE;
+		uiChunkSize = 1ULL << uiLog2ContiguityGuarantee;
+	}
+	else
+	{
+		uiChunkSize = uiLogicalSize;
+	}
+
+	/* Extra checks required for sparse PMRs */
+	if (bSparse)
+	{
+		/* Check the logical size and chunk information agree with each other */
+		if (uiLogicalSize != (uiChunkSize * ui32NumLogicalChunks))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Bad mapping size (uiLogicalSize = 0x%llx, uiChunkSize = 0x%llx, ui32NumLogicalChunks = %d)",
+					__func__, (unsigned long long)uiLogicalSize, (unsigned long long)uiChunkSize, ui32NumLogicalChunks));
+			return PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+		}
+	}
+
+	pvPMRLinAddr = OSAllocMem(sizeof(*psPMR) + sizeof(*psMappingTable) + IMG_FLEX_ARRAY_SIZE(sizeof(IMG_UINT32), ui32NumLogicalChunks));
+	PVR_RETURN_IF_NOMEM(pvPMRLinAddr);
+
+	psPMR = (PMR *) pvPMRLinAddr;
+	psMappingTable = IMG_OFFSET_ADDR(pvPMRLinAddr, sizeof(*psPMR));
+
+	/* Setup the mapping table */
+	psMappingTable->uiChunkSize = uiChunkSize;
+	psMappingTable->ui32NumLogicalChunks = ui32NumLogicalChunks;
+	psMappingTable->ui32NumPhysChunks = ui32NumPhysChunks;
+	OSCachedMemSet(&psMappingTable->aui32Translation[0], 0xFF, sizeof(psMappingTable->aui32Translation[0])*
+	               ui32NumLogicalChunks);
+	for (i=0; i<ui32NumPhysChunks; i++)
+	{
+		ui32Temp = pui32MappingTable[i];
+		PVR_ASSERT(ui32Temp < ui32NumLogicalChunks);
+		psMappingTable->aui32Translation[ui32Temp] = ui32Temp;
+	}
+
+	eError = OSLockCreate(&psPMR->hLock);
+	PVR_GOTO_IF_ERROR(eError, ErrFreePMR);
+
+	eError = OSSpinLockCreate(&psPMR->hBitmapLock);
+	PVR_GOTO_IF_ERROR(eError, ErrFreePMRLock);
+
+	eError = OSSpinLockCreate(&psPMR->hRefCountLock);
+	PVR_GOTO_IF_ERROR(eError, ErrFreeBitmapLock);
+
+	/* Setup the PMR */
+	psPMR->uiRefCount = 0;
+	OSAtomicWrite(&psPMR->iClientCpuMapCount, 0);
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	OSAtomicWrite(&psPMR->iKernelCpuMapCount, 0);
+	dllist_init(&psPMR->sGpuMappingListHead);
+#endif
+	psPMR->iAssociatedResCount = 0;
+
+	/* If allocation is not made on demand, it will be backed now and
+	 * backing will not be removed until the PMR is destroyed, therefore
+	 * we can initialise the iLockCount to 1 rather than 0.
+	 */
+	OSAtomicWrite(&psPMR->iLockCount, (PVRSRV_CHECK_ON_DEMAND(uiFlags) ? 0 : 1));
+
+	psPMR->psContext = psContext;
+	psPMR->uiLogicalSize = uiLogicalSize;
+	psPMR->uiLog2ContiguityGuarantee = uiLog2ContiguityGuarantee;
+	psPMR->uiFlags = uiFlags;
+	psPMR->psMappingTable = psMappingTable;
+	psPMR->uiInternalFlags = bSparse ? PMR_FLAG_INTERNAL_SPARSE_ALLOC : 0;
+	psPMR->szAnnotation[0] = '\0';
+	PMR_ENV_INITIALIZE(psPMR, sEnvData);
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+	psPMR->uiDevImportBitmap = 0;
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	psPMR->sHeader.eZombieType = PMR_ZOMBIE_TYPE_PMR;
+	dllist_init(&psPMR->sHeader.sZombieNode);
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	dllist_init(&psPMR->sXDeviceImports);
+#endif /* defined(SUPPORT_DEVICE_IMPORT_DEFERRED_FREE) */
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	psPMR->hRIHandle = NULL;
+#endif
+	OSLockAcquire(psContext->hLock);
+	psPMR->uiKey = psContext->uiNextKey;
+	psPMR->uiSerialNum = psContext->uiNextSerialNum;
+	psContext->uiNextKey = (0x80200003 * psContext->uiNextKey)
+								^ (0xf00f0081 * (uintptr_t)pvPMRLinAddr);
+	psContext->uiNextSerialNum++;
+	*ppsPMR = psPMR;
+	OSLockRelease(psContext->hLock);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+
+	/* Increment live PMR count */
+	OSAtomicIncrement(&psContext->uiNumLivePMRs);
+
+	return PVRSRV_OK;
+
+ErrFreeBitmapLock:
+	OSSpinLockDestroy(psPMR->hBitmapLock);
+ErrFreePMRLock:
+	OSLockDestroy(psPMR->hLock);
+ErrFreePMR:
+	OSFreeMem(psPMR);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_Ref(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+
+	if (psPMR->uiRefCount == 0)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Ref Count == 0 PMR: @0x%p Annot: %s",
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+	else if (psPMR->uiRefCount >= IMG_UINT32_MAX - uiRefCount)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Ref Count >= IMG_UINT32_MAX PMR: @0x%p "
+		                        "Annot: %s RefCount: %u",
+		                        psPMR,
+		                        psPMR->szAnnotation,
+		                        uiRefCount));
+		OSWarnOn(1);
+
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+
+	psPMR->uiRefCount += uiRefCount;
+
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_Unref(PMR *psPMR, IMG_INT32 uiRefCount, IMG_UINT32 *pui32RefCount)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+
+	if (psPMR->uiRefCount == 0)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Unref Count = 0 PMR: @0x%p Annot: %s "
+		                        "RefCount: %u",
+		                        psPMR,
+		                        psPMR->szAnnotation,
+		                        uiRefCount));
+		OSWarnOn(1);
+
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+
+	psPMR->uiRefCount -= uiRefCount;
+
+	if (pui32RefCount != NULL)
+	{
+		*pui32RefCount = psPMR->uiRefCount;
+	}
+
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+#if defined(DEBUG)
+void PMRLockHeldAssert(const PMR *psPMR)
+{
+	OSLockHeldAssert(psPMR->hLock);
+}
+#endif
+
+void
+PMRLockPMR(const PMR *psPMR)
+{
+	OSLockAcquire(psPMR->hLock);	/* Uses same lock as PhysAddresses */
+}
+
+void
+PMRUnlockPMR(const PMR *psPMR)
+{
+	OSLockRelease(psPMR->hLock);	/* Uses same lock as PhysAddresses */
+}
+
+static INLINE void _IntFlagSet(PMR *psPMR, const IMG_UINT32 uiValue)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_SET(psPMR->uiInternalFlags, uiValue);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+
+static INLINE void _IntFlagClr(PMR *psPMR, const IMG_UINT32 uiValue)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_UNSET(psPMR->uiInternalFlags, uiValue);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+
+static INLINE IMG_BOOL _IntFlagIsSet(const PMR *psPMR, const IMG_UINT32 uiValue)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_BOOL bIsSet;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	bIsSet = BITMASK_HAS(psPMR->uiInternalFlags, uiValue);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return bIsSet;
+}
+
+static INLINE void
+_FactoryLock(const PMR_IMPL_FUNCTAB *psFuncTable)
+{
+	if (psFuncTable->pfnGetPMRFactoryLock != NULL)
+	{
+		psFuncTable->pfnGetPMRFactoryLock();
+	}
+}
+
+static INLINE void
+_FactoryUnlock(const PMR_IMPL_FUNCTAB *psFuncTable)
+{
+	if (psFuncTable->pfnReleasePMRFactoryLock != NULL)
+	{
+		psFuncTable->pfnReleasePMRFactoryLock();
+	}
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/* Protects:
+ * - `psDevNode->sPMRZombieList`
+ * - `uiPMRZombieCount`
+ * - `uiPMRZombieCountInCleanup`
+ *
+ * and all `PMR_ZOMBIE_CLEANUP_ITEM::sZombieList` where
+ * `PMR_ZOMBIE_CLEANUP_ITEM::psDevNode == psDevNode` */
+static INLINE void
+_ZombieListLock(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OSLockAcquire(psDevNode->hPMRZombieListLock);
+}
+
+static INLINE void
+_ZombieListUnlock(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OSLockRelease(psDevNode->hPMRZombieListLock);
+}
+
+static IMG_BOOL _IsDeviceOnAndOperating(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+
+	eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		/* Treat unknown power state as ON. */
+		ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+	}
+
+	/* The device does not accept zombies when its power is OFF as
+	 * the cache invalidation comes as a given. */
+	return !( ePowerState == PVRSRV_DEV_POWER_STATE_OFF
+	       || psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+static IMG_UINT64
+_DeviceImportBitmapGet(const PMR *psPMR)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_UINT64 uiDevImportBitmap;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	uiDevImportBitmap = psPMR->uiDevImportBitmap;
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return uiDevImportBitmap;
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(PVRSRV_ENABLE_XD_MEM) && !defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static void
+_DeviceImportBitmapSet(PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_SET(psPMR->uiDevImportBitmap, IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) && !defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static void
+_DeviceImportBitmapClr(PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	BITMASK_UNSET(psPMR->uiDevImportBitmap, IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+}
+
+static IMG_BOOL
+_DeviceImportBitmapIsSet(const PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_BOOL bIsSet;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	bIsSet = BITMASK_HAS(psPMR->uiDevImportBitmap,
+	                     IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return bIsSet;
+}
+
+static IMG_BOOL
+/* Atomically, return if the `psDevNode` is set in the bitmap and then set it. */
+_DeviceImportBitmapFetchAndSet(PMR *psPMR, const PPVRSRV_DEVICE_NODE psDevNode)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	IMG_BOOL bIsSet;
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	bIsSet = BITMASK_HAS(psPMR->uiDevImportBitmap,
+	                     IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	BITMASK_SET(psPMR->uiDevImportBitmap,
+	            IMG_UINT64_C(1) << psDevNode->sDevId.ui32InternalID);
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+
+	return bIsSet;
+}
+
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static PVRSRV_ERROR
+_DeviceImportRegister(PMR *psPMR, PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_DEVICE_IMPORT *psImport;
+
+	PVR_ASSERT(psPMR);
+	PVR_ASSERT(psDevNode);
+	PVR_ASSERT(PMR_DeviceNode(psPMR) != psDevNode);
+
+	/* Explicitly reject:
+	 * - PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC
+	 * - !PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE
+	 * as XD PMRs don't have support for
+	 * SUPPORT_PMR_PAGES_DEFERRED_FREE. */
+	if (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ||
+	    !_IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE))
+	{
+		eError = PVRSRV_ERROR_PMR_NOT_PERMITTED;
+		PVR_LOG_ERROR(eError,
+		              "PVRSRV_CHECK_ON_DEMAND || !PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE");
+		return eError;
+	}
+
+	/* Check if the device is already imported */
+	if (_DeviceImportBitmapFetchAndSet(psPMR, psDevNode))
+	{
+		return PVRSRV_OK;
+	}
+
+	psImport = OSAllocMem(sizeof(*psImport));
+	PVR_LOG_RETURN_IF_NOMEM(psImport, "PMR_DEVICE_IMPORT");
+
+	psImport->psParent = psPMR;
+	psImport->psDevNode = psDevNode;
+	dllist_init(&psImport->sHeader.sZombieNode);
+	psImport->sHeader.eZombieType = PMR_ZOMBIE_TYPE_DEVICE_IMPORT;
+
+	PMRLockPMR(psPMR);
+	dllist_add_to_tail(&psPMR->sXDeviceImports, &psImport->sNext);
+	PMRUnlockPMR(psPMR);
+
+	return eError;
+}
+
+static void
+_DeviceImportFreeImportZombie(PMR_DEVICE_IMPORT *psImport)
+{
+	PVR_ASSERT(_DeviceImportBitmapIsSet(psImport->psParent, psImport->psDevNode));
+	_DeviceImportBitmapClr(psImport->psParent, psImport->psDevNode);
+
+	PMRLockPMR(psImport->psParent);
+	dllist_remove_node(&psImport->sNext);
+	PMRUnlockPMR(psImport->psParent);
+
+	OSFreeMem(psImport);
+}
+
+static IMG_BOOL
+_DeviceImportEnqueueZombie(PMR_DEVICE_IMPORT *psImport)
+{
+	PVR_ASSERT(_DeviceImportBitmapIsSet(psImport->psParent, psImport->psDevNode));
+
+	if (!_IsDeviceOnAndOperating(psImport->psDevNode))
+	{
+		_DeviceImportFreeImportZombie(psImport);
+		return IMG_FALSE;
+	}
+
+	_ZombieListLock(psImport->psDevNode);
+	dllist_add_to_tail(&psImport->psDevNode->sPMRZombieList,
+	                   &psImport->sHeader.sZombieNode);
+	psImport->psDevNode->uiPMRZombieCount++;
+	_ZombieListUnlock(psImport->psDevNode);
+
+	return IMG_TRUE;
+}
+
+static void
+_DeviceImportsReviveZombies(PMR *psPMR)
+{
+	PDLLIST_NODE psNode, psNext;
+	PMR_DEVICE_IMPORT *psImport;
+
+	dllist_foreach_node(&psPMR->sXDeviceImports, psNode, psNext)
+	{
+		psImport = IMG_CONTAINER_OF(psNode, PMR_DEVICE_IMPORT, sNext);
+		_ZombieListLock(psImport->psDevNode);
+		if (!dllist_is_empty(&psImport->sHeader.sZombieNode))
+		{
+			dllist_remove_node(&psImport->sHeader.sZombieNode);
+			psImport->psDevNode->uiPMRZombieCount--;
+		}
+		_ZombieListUnlock(psImport->psDevNode);
+	}
+}
+
+static IMG_BOOL
+_DeviceImportsEnqueueZombies(PMR *psPMR)
+{
+	PDLLIST_NODE psNode, psNext;
+	PMR_DEVICE_IMPORT *psImport;
+	IMG_BOOL bEnqueued = IMG_FALSE;
+
+	PMRLockPMR(psPMR);
+
+	dllist_foreach_node(&psPMR->sXDeviceImports, psNode, psNext)
+	{
+		psImport = IMG_CONTAINER_OF(psNode, PMR_DEVICE_IMPORT, sNext);
+		bEnqueued |= _DeviceImportEnqueueZombie(psImport);
+	}
+
+	PMRUnlockPMR(psPMR);
+
+	return bEnqueued;
+}
+
+static void
+_DeviceImportsUnregisterAll(PMR *psPMR)
+{
+	OS_SPINLOCK_FLAGS uiLockingFlags;
+	PDLLIST_NODE psNode, psNext;
+
+	PMRLockPMR(psPMR);
+	dllist_foreach_node(&psPMR->sXDeviceImports, psNode, psNext)
+	{
+		PMR_DEVICE_IMPORT *psImport = IMG_CONTAINER_OF(psNode, PMR_DEVICE_IMPORT, sNext);
+		PVR_ASSERT(_DeviceImportBitmapIsSet(psPMR, psImport->psDevNode));
+		OSFreeMem(psImport);
+	}
+	dllist_init(&psPMR->sXDeviceImports);
+
+	OSSpinLockAcquire(psPMR->hBitmapLock, uiLockingFlags);
+	psPMR->uiDevImportBitmap = 0;
+	OSSpinLockRelease(psPMR->hBitmapLock, uiLockingFlags);
+	PMRUnlockPMR(psPMR);
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+static void
+_PMRDestroy(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	PMR_ENV_FINALIZE(psPMR, sEnvData);
+
+	if (psPMR->psFuncTab->pfnFinalize != NULL)
+	{
+		psPMR->psFuncTab->pfnFinalize(psPMR->pvFlavourData);
+	}
+
+#if defined(PDUMP)
+	/* if allocation is done on the host node don't include it in the PDUMP */
+	if (!_IsHostDevicePMR(psPMR))
+	{
+		PDumpPMRFreePMR(psPMR,
+		                psPMR->uiLogicalSize,
+		                IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee),
+		                psPMR->uiLog2ContiguityGuarantee,
+		                psPMR->hPDumpAllocHandle);
+	}
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	/* This PMR is about to be destroyed, update its mmap stats record (if present)
+	 * to avoid dangling pointer. Additionally, this is required because mmap stats
+	 * are identified by PMRs and a new PMR down the line "might" get the same address
+	 * as the one we're about to free and we'd like 2 different entries in mmaps
+	 * stats for such cases */
+	MMapStatsRemovePMR(psPMR);
+#endif
+
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	/* If not backed on demand, iLockCount should be 1 otherwise it should be 0 */
+	PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1));
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Delete RI entry */
+	if (psPMR->hRIHandle)
+	{
+		PVRSRV_ERROR eError = RIDeletePMREntryKM(psPMR->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "RIDeletePMREntryKM");
+		/* continue destroying the PMR */
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	_DeviceImportsUnregisterAll(psPMR);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	/* Decrement live PMR count. Probably only of interest for debugging */
+	PVR_ASSERT(OSAtomicRead(&psPMR->psContext->uiNumLivePMRs) > 0);
+	OSAtomicDecrement(&psPMR->psContext->uiNumLivePMRs);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Detect programming errors here, either a reference on the PMR
+	 * has not been taken by a user or the mapping records added
+	 * to the list head have not been destroyed correctly.
+	 */
+	PVR_ASSERT(dllist_is_empty(&psPMR->sGpuMappingListHead));
+#endif
+
+	OSSpinLockDestroy(psPMR->hRefCountLock);
+	OSSpinLockDestroy(psPMR->hBitmapLock);
+	OSLockDestroy(psPMR->hLock);
+	OSFreeMem(psPMR);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static INLINE PMR_ZOMBIE_TYPE
+PMR_GetZombieTypeFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return psPMRHeader->eZombieType;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static INLINE PMR_ZOMBIE_PAGES*
+PMR_GetZombiePagesFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR_ZOMBIE_PAGES, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+static INLINE PMR_DEVICE_IMPORT*
+PMR_GetDeviceImportFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR_DEVICE_IMPORT, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+static INLINE PMR*
+PMR_GetPMRFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_UnrefAndMaybeDestroy(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	const PMR_IMPL_FUNCTAB *psFuncTable;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_BOOL bQueuedDeviceImports = IMG_FALSE;
+#endif
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	psFuncTable = psPMR->psFuncTab;
+
+	_FactoryLock(psFuncTable);
+
+	eError = _Unref(psPMR, uiRefCount, &uiRefCount);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_Unref", ErrFactoryUnlock);
+
+	if (uiRefCount > 0)
+	{
+		/* PMR is still referenced so just return */
+		_FactoryUnlock(psFuncTable);
+		return PVRSRV_OK;
+	}
+
+#if !defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Don't defer PMR destruction in NoHW and PDUMP drivers. */
+	_PMRDestroy(psPMR);
+#else /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+	psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+
+	/* PMRs that are not marked for deferred free can be freed right away.
+	 * Those are the PMRs that have not been mapped to the device.
+	 * All PMRs that have been mapped to the device need to go through
+	 * the defer free path unless the power is OFF for the PMR's device
+	 * and for all of the device imports. If power is OFF
+	 * the cache invalidation comes as a given. */
+	if (!_IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_DEFER_FREE))
+	{
+		_PMRDestroy(psPMR);
+		goto exit_;
+	}
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	bQueuedDeviceImports = _DeviceImportsEnqueueZombies(psPMR);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	if (!bQueuedDeviceImports
+	    && !_IsDeviceOnAndOperating(psDevNode)
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	    && !PVRSRV_CHECK_OS_LINUX_MOVABLE(psPMR->uiFlags)
+#endif
+	    )
+	{
+		_PMRDestroy(psPMR);
+	}
+	else
+	{
+		/* Defer freeing the PMR until the Firmware invalidates the caches. */
+		_ZombieListLock(psDevNode);
+
+		_IntFlagSet(psPMR, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+
+		dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMR->sHeader.sZombieNode);
+		psDevNode->uiPMRZombieCount++;
+
+		/* PMR pages are accounted by the driver/process stats. Those stats
+		 * are available on page level hence they need to be adjusted by
+		 * the factories. This is done by the pfnZombify callback.
+		 * Operation needs to be done while holding hPMRZombieListLock
+		 * to prevent CleanupThread from freeing pages while memory stats
+		 * accounting is ongoing. */
+		if (psPMR->psFuncTab->pfnZombify != NULL)
+		{
+			eError = psPMR->psFuncTab->pfnZombify(psPMR->pvFlavourData, psPMR);
+			PVR_LOG_IF_ERROR(eError, "pfnZombify");
+		}
+
+		_ZombieListUnlock(psDevNode);
+	}
+exit_:
+#endif /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	_FactoryUnlock(psFuncTable);
+
+	return PVRSRV_OK;
+
+ErrFactoryUnlock:
+	_FactoryUnlock(psFuncTable);
+
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+typedef struct _PMR_ZOMBIE_CLEANUP_ITEM_
+{
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+	DLLIST_NODE sZombieList;                     /*!< See _ZombieListLock */
+	PPVRSRV_DEVICE_NODE psDevNode;
+	PVRSRV_CLIENT_SYNC_PRIM *psSync;
+	IMG_UINT32 uiRequiredSyncValue;
+	IMG_UINT32 uiRequiredPowerOffCounter;
+} PMR_ZOMBIE_CLEANUP_ITEM;
+
+static INLINE IMG_BOOL
+_CanNotFreeZombies(const PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem)
+{
+	const PVRSRV_DEVICE_NODE *psDevNode = psCleanupItem->psDevNode;
+
+	/* For a zombie PMR to be eligible to be freed either the GPU MMU caches
+	 * need to be flushed (the Firmware updates the sync) or the GPU power needs
+	 * to be off. */
+	return !PVRSRVHasCounter32Advanced(OSReadDeviceMem32(psCleanupItem->psSync->pui32LinAddr),
+	                                   psCleanupItem->uiRequiredSyncValue) &&
+	       !PVRSRVHasCounter32Advanced(psDevNode->uiPowerOffCounter,
+	                                   psCleanupItem->uiRequiredPowerOffCounter);
+}
+
+static PVRSRV_ERROR _PmrZombieCleanup(void *pvData)
+{
+	PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem = pvData;
+	DLLIST_NODE *psNode;
+	DLLIST_NODE sRetryHead;
+	IMG_UINT32 uiRetryCount = 0;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (_CanNotFreeZombies(psCleanupItem))
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+	dllist_init(&sRetryHead);
+
+	do
+	{
+		/* hPMRZombieListLock will prevent removing a node while the list is
+		 * processed. If the lock is already acquired by other process which
+		 * intends to remove an item from the list it'll assure the list
+		 * consistency.
+		 * If this thread acquires the lock first it's possible that another
+		 * thread might be holding PMR factory lock. */
+
+		_ZombieListLock(psCleanupItem->psDevNode);
+		psNode = dllist_get_next_node(&psCleanupItem->sZombieList);
+		_ZombieListUnlock(psCleanupItem->psDevNode);
+
+		if (psNode == NULL)
+		{
+			continue;
+		}
+
+		switch (PMR_GetZombieTypeFromNode(psNode))
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_PAGES:
+			{
+				PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psNode);
+				eError = psZombiePages->pfnFactoryFreeZombies(psZombiePages->pvFactoryPages);
+				_ZombieListLock(psCleanupItem->psDevNode);
+				dllist_remove_node(psNode);
+				psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+				_ZombieListUnlock(psCleanupItem->psDevNode);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Cannot free zombie pages! Skipping object %p", psZombiePages));
+					dllist_add_to_tail(&sRetryHead, psNode);
+					uiRetryCount++;
+				}
+				else
+				{
+					OSFreeMem(psZombiePages);
+				}
+				break;
+			}
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_DEVICE_IMPORT:
+			{
+				PMR_DEVICE_IMPORT *psImport = PMR_GetDeviceImportFromNode(psNode);
+				_ZombieListLock(psCleanupItem->psDevNode);
+				dllist_remove_node(psNode);
+				psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+				_ZombieListUnlock(psCleanupItem->psDevNode);
+				_DeviceImportFreeImportZombie(psImport);
+				break;
+			}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+			case PMR_ZOMBIE_TYPE_PMR:
+			{
+				PMR* psPMR = PMR_GetPMRFromNode(psNode);
+				const PMR_IMPL_FUNCTAB *psFuncTable = psPMR->psFuncTab;
+
+				_FactoryLock(psFuncTable);
+				_ZombieListLock(psCleanupItem->psDevNode);
+				/* It is possible that the element might have been removed so
+				 * we have to check if the PMR is still a zombie.
+				 * It's also possible that the PMR has been revived
+				 * (PMRReviveZombieAndRef()), mapped, unmapped and zombified
+				 * again while the lock was not held.
+				 * Considering above only immediately free the PMR if the
+				 * PMR is still a part of this cleanup item. */
+				if (psNode == dllist_get_next_node(&psCleanupItem->sZombieList))
+				{
+					dllist_remove_node(psNode);
+					psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+					/* The PMR cannot be freed if other devices are
+					 * still waiting for the cache flush. */
+					if (_DeviceImportBitmapGet(psPMR) != 0)
+					{
+						/* Request it to be retried and continue
+						 * to the next zombie item. */
+						dllist_add_to_tail(&sRetryHead, psNode);
+						uiRetryCount++;
+						_ZombieListUnlock(psCleanupItem->psDevNode);
+						_FactoryUnlock(psFuncTable);
+						continue;
+					}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+					/* Unlock here to avoid locking dependency with the power lock.
+					 * It's okay to do it here since the factory lock is the one
+					 * that needs to be held during PMR destruction. */
+					_ZombieListUnlock(psCleanupItem->psDevNode);
+					_PMRDestroy(psPMR);
+				}
+				else
+				{
+					_ZombieListUnlock(psCleanupItem->psDevNode);
+				}
+				_FactoryUnlock(psFuncTable);
+				break;
+			}
+		}
+	} while (psNode != NULL);
+
+	if (uiRetryCount)
+	{
+		eError = PVRSRV_ERROR_RETRY;
+		_ZombieListLock(psCleanupItem->psDevNode);
+		/* Add the retry items back to this cleanup item for when the
+		 * cleanup item is retried. Oldest items will reside at the head of
+		 * the list. The cleanup item will be placed at the back of the cleanup
+		 * queue to process other dependencies first. */
+		dllist_insert_list_at_head(&psCleanupItem->sZombieList, &sRetryHead);
+		psCleanupItem->psDevNode->uiPMRZombieCountInCleanup += uiRetryCount;
+		_ZombieListUnlock(psCleanupItem->psDevNode);
+	}
+	else
+	{
+		OSFreeMem(psCleanupItem);
+	}
+
+	return eError;
+}
+
+IMG_BOOL PMRQueueZombiesForCleanup(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem;
+
+	/* Don't defer the freeing if we are currently unloading the driver
+	 * or if the sync has been destroyed */
+	if (psPVRSRVData->bUnload || psDevNode->psMMUCacheSyncPrim == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	_ZombieListLock(psDevNode);
+
+	if (dllist_is_empty(&psDevNode->sPMRZombieList))
+	{
+		_ZombieListUnlock(psDevNode);
+		return IMG_FALSE;
+	}
+
+	psCleanupItem = OSAllocMem(sizeof(*psCleanupItem));
+	if (psCleanupItem == NULL)
+	{
+		_ZombieListUnlock(psDevNode);
+		return IMG_FALSE;
+	}
+
+	psCleanupItem->sCleanupThreadFn.pfnFree = _PmrZombieCleanup;
+	psCleanupItem->sCleanupThreadFn.pvData = psCleanupItem;
+	psCleanupItem->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+	psCleanupItem->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_PMR;
+	CLEANUP_THREAD_SET_RETRY_TIMEOUT(&psCleanupItem->sCleanupThreadFn,
+	                                 CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT);
+
+	psCleanupItem->psDevNode = psDevNode;
+	psCleanupItem->psSync = psDevNode->psMMUCacheSyncPrim;
+	psCleanupItem->uiRequiredSyncValue = psDevNode->ui32NextMMUInvalidateUpdate;
+	psCleanupItem->uiRequiredPowerOffCounter = psDevNode->uiPowerOffCounterNext;
+
+	/* This moves the zombie list to the cleanup item. */
+	dllist_replace_head(&psDevNode->sPMRZombieList, &psCleanupItem->sZombieList);
+	psDevNode->uiPMRZombieCountInCleanup += psDevNode->uiPMRZombieCount;
+	psDevNode->uiPMRZombieCount = 0;
+
+	OSLockRelease(psDevNode->hPMRZombieListLock);
+
+	PVRSRVCleanupThreadAddWork(psDevNode, &psCleanupItem->sCleanupThreadFn);
+
+	return IMG_TRUE;
+}
+
+void
+PMRReviveZombieAndRef(PMR *psPMR)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	DLLIST_NODE *psThis, *psNext;
+	IMG_BOOL bIsOnZombieList = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	psDeviceNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+
+	/* If this was on a list then it's brought back to life. */
+	_ZombieListLock(psDeviceNode);
+
+	/* Need to reference this PMR since it was about to be destroyed and its
+	 * reference count must be 0 (can't use _Ref() due to the warning). */
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+	psPMR->uiRefCount++;
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+#if  defined(DEBUG)
+	PVR_LOG(("%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+#endif
+
+	/* If we got to this point the PMR must be on a list. If it's not
+	 * it should mean a race of some sort. */
+	PVR_ASSERT(!dllist_is_empty(&psPMR->sHeader.sZombieNode));
+
+	/* For the sake of correct accounting check if the PMR is in the zombie
+	 * list or in the cleanup item. */
+	dllist_foreach_node(&psDeviceNode->sPMRZombieList, psThis, psNext)
+	{
+		if (psThis == &psPMR->sHeader.sZombieNode)
+		{
+			bIsOnZombieList = IMG_TRUE;
+			break;
+		}
+	}
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+	if (_DeviceImportBitmapGet(psPMR) != 0) {
+		PMRLockPMR(psPMR);
+		_DeviceImportsReviveZombies(psPMR);
+		PMRUnlockPMR(psPMR);
+	}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+	/* Revive the PMR (remove it from the zombie list) and therefore
+	 * prevent it's destruction. */
+	dllist_remove_node(&psPMR->sHeader.sZombieNode);
+	_IntFlagClr(psPMR, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+
+	if (bIsOnZombieList)
+	{
+		psDeviceNode->uiPMRZombieCount--;
+	}
+	else
+	{
+		psDeviceNode->uiPMRZombieCountInCleanup--;
+	}
+
+	_ZombieListUnlock(psDeviceNode);
+}
+
+void
+PMRMarkForDeferFree(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	if (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags))
+	{
+		/* If PMR pages are allocated on demand the freeing is handled
+		 * by `SUPPORT_PMR_PAGES_DEFERRED_FREE` path in
+		 * `PMRUnlockSysPhysAddressesNested()`. */
+		return;
+	}
+
+	_IntFlagSet(psPMR, PMR_FLAG_INTERNAL_DEFER_FREE);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static INLINE IMG_BOOL _PMRIsSparse(const PMR *psPMR)
+{
+	return _IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_SPARSE_ALLOC);
+}
+
+PVRSRV_ERROR
+PMRCreatePMR(PHYS_HEAP *psPhysHeap,
+             PMR_SIZE_T uiLogicalSize,
+             IMG_UINT32 ui32NumPhysChunks,
+             IMG_UINT32 ui32NumLogicalChunks,
+             IMG_UINT32 *pui32MappingTable,
+             PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+             PMR_FLAGS_T uiFlags,
+             const IMG_CHAR *pszAnnotation,
+             const PMR_IMPL_FUNCTAB *psFuncTab,
+             PMR_IMPL_PRIVDATA pvPrivData,
+             PMR_IMPL_TYPE eType,
+             PMR **ppsPMRPtr,
+             IMG_UINT32 ui32PDumpFlags)
+{
+	PMR *psPMR = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+
+	eError = _PMRCreate(uiLogicalSize,
+	                    ui32NumPhysChunks,
+	                    ui32NumLogicalChunks,
+	                    pui32MappingTable,
+	                    uiLog2ContiguityGuarantee,
+	                    uiFlags,
+	                    &psPMR);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	psPMR->psPhysHeap = psPhysHeap;
+	psPMR->psFuncTab = psFuncTab;
+	psPMR->pszPDumpDefaultMemspaceName = PhysHeapPDumpMemspaceName(psPhysHeap);
+	psPMR->pvFlavourData = pvPrivData;
+	psPMR->eFlavour = eType;
+	psPMR->uiRefCount = 1;
+
+	OSStringSafeCopy(psPMR->szAnnotation, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PDUMP)
+	/* if allocation was done on the host node don't include it in the PDUMP */
+	if (!_IsHostDevicePMR(psPMR))
+	{
+		PMR_FLAGS_T uiFlags = psPMR->uiFlags;
+		IMG_BOOL bInitialise = IMG_FALSE;
+		IMG_UINT8 ui8InitValue = 0;
+
+		if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+		{
+			bInitialise = IMG_TRUE;
+		}
+		else if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+		{
+			ui8InitValue = (IMG_UINT8)PVRSRV_POISON_ON_ALLOC_VALUE;
+			bInitialise = IMG_TRUE;
+		}
+
+		PDumpPMRMallocPMR(psPMR,
+		                  uiLogicalSize,
+		                  IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee),
+		                  ui32NumPhysChunks,
+		                  ui32NumLogicalChunks,
+		                  pui32MappingTable,
+		                  uiLog2ContiguityGuarantee,
+		                  bInitialise,
+		                  ui8InitValue,
+		                  &psPMR->hPDumpAllocHandle,
+		                  ui32PDumpFlags);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	psPMR->eState = PMR_STATE_ACTIVE;
+#endif
+
+	*ppsPMRPtr = psPMR;
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRLockSysPhysAddresses(PMR *psPMR)
+{
+	return PMRLockSysPhysAddressesNested(psPMR, 1, 0);
+}
+
+PVRSRV_ERROR
+PMRLockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount)
+{
+	return PMRLockSysPhysAddressesNested(psPMR, uiLockCount, 0);
+}
+
+PVRSRV_ERROR PMRLockSysPhysAddressesNested(PMR *psPMR,
+                                        IMG_UINT32 uiLockCount,
+                                        IMG_UINT32 ui32NestingLevel)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiCallbackValue;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	/* Note: taking this lock is not required to protect the PMR reference
+	 * count, because the PMR reference count is atomic. Rather, taking
+	 * the lock here guarantees that no caller will exit this function
+	 * without the underlying physical addresses being locked.
+	 */
+	OSLockAcquireNested(psPMR->hLock, ui32NestingLevel);
+	/* We also count the locks as references, so that the PMR is not freed
+	 * while someone is using a physical address.
+	 * "lock" here simply means incrementing the refcount. It means the
+	 * refcount is multipurpose, but that's okay. We only have to promise
+	 * that physical addresses are valid after this point, and remain valid
+	 * until the corresponding PMRUnlockSysPhysAddressesOSMem()
+	 */
+	eError = _Ref(psPMR, uiLockCount);
+	if (eError != PVRSRV_OK)
+	{
+		OSLockRelease(psPMR->hLock);
+		return eError;
+	}
+
+	/* Also count locks separately from other types of references, to
+	 * allow for debug assertions
+	 */
+
+	/* Only call callback if lock count transitions from 0 to 1 (or 1 to 2 if not
+	 * backed on demand) */
+	uiCallbackValue = uiLockCount + (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1);
+
+	if (OSAtomicAdd(&psPMR->iLockCount, uiLockCount) == uiCallbackValue)
+	{
+		if (psPMR->psFuncTab->pfnLockPhysAddresses != NULL)
+		{
+			/* must always have lock and unlock in pairs! */
+			PVR_ASSERT(psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL);
+
+			eError = psPMR->psFuncTab->pfnLockPhysAddresses(psPMR->pvFlavourData);
+
+			PVR_GOTO_IF_ERROR(eError, e1);
+		}
+	}
+	OSLockRelease(psPMR->hLock);
+
+	return PVRSRV_OK;
+
+e1:
+	OSAtomicSubtract(&psPMR->iLockCount, uiLockCount);
+	PVR_ASSERT(_GetRef(psPMR) != 0);
+	OSLockRelease(psPMR->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	(void) _Unref(psPMR, uiLockCount, NULL);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddresses(PMR *psPMR)
+{
+	return PMRUnlockSysPhysAddressesNested(psPMR, 1, 2);
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount)
+{
+	return PMRUnlockSysPhysAddressesNested(psPMR, uiLockCount, 2);
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesNested(PMR *psPMR,
+                             IMG_UINT32 uiLockCount,
+                             IMG_UINT32 ui32NestingLevel)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_IMPL_ZOMBIEPAGES pvZombiePages = NULL;
+	PMR_ZOMBIE_PAGES* psPMRZombiePages = NULL;
+#endif
+
+	PVR_ASSERT(psPMR != NULL);
+
+	/* Acquiring the lock here, as well as during the Lock operation ensures
+	 * the lock count hitting zero and the unlocking of the phys addresses is
+	 * an atomic operation
+	 */
+	OSLockAcquireNested(psPMR->hLock, ui32NestingLevel);
+
+	PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) >= uiLockCount);
+
+	if (OSAtomicSubtract(&psPMR->iLockCount, uiLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1))
+	{
+		if (psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL)
+		{
+			PVR_ASSERT(psPMR->psFuncTab->pfnLockPhysAddresses != NULL);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			eError = psPMR->psFuncTab->pfnUnlockPhysAddresses(psPMR->pvFlavourData,
+			                                                  &pvZombiePages);
+#else
+			eError = psPMR->psFuncTab->pfnUnlockPhysAddresses(psPMR->pvFlavourData);
+#endif
+			PVR_LOG_IF_ERROR(eError, "pfnUnlockPhysAddresses");
+		}
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		psPMRZombiePages = OSAllocZMem(sizeof(PMR_ZOMBIE_PAGES));
+		PVR_GOTO_IF_NOMEM(psPMRZombiePages, eError, ErrRelockPhysAddresses);
+	}
+#endif
+
+	OSLockRelease(psPMR->hLock);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+		PVRSRV_DEVICE_NODE *psDevNode;
+
+		psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+		eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+		if (eError != PVRSRV_OK)
+		{
+			/* Treat unknown power state as ON. */
+			ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+		}
+
+		if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF ||
+			psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+		{
+			/* Free preallocated psPMRZombiePages as these won't be used*/
+			OSFreeMem(psPMRZombiePages);
+
+			eError = psPMR->psFuncTab->pfnFreeZombiePages(pvZombiePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "Error when trying to free zombies immediately.",
+			                      ErrReturn);
+		}
+		else
+		{
+			PVR_ASSERT(psPMRZombiePages != NULL);
+			psPMRZombiePages->sHeader.eZombieType = PMR_ZOMBIE_TYPE_PAGES;
+			psPMRZombiePages->pfnFactoryFreeZombies = psPMR->psFuncTab->pfnFreeZombiePages;
+			psPMRZombiePages->pvFactoryPages = pvZombiePages;
+
+			_ZombieListLock(psDevNode);
+			dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMRZombiePages->sHeader.sZombieNode);
+			psDevNode->uiPMRZombieCount++;
+			_ZombieListUnlock(psDevNode);
+		}
+	}
+	else
+	{
+		OSFreeMem(psPMRZombiePages);
+	}
+#endif
+
+	/* We also count the locks as references, so that the PMR is not
+	 * freed while someone is using a physical address.
+	 */
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	(void) _UnrefAndMaybeDestroy(psPMR, uiLockCount);
+
+	return eError;
+#else
+	return _UnrefAndMaybeDestroy(psPMR, uiLockCount);
+#endif
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+ErrRelockPhysAddresses:
+	{
+		IMG_UINT32 uiCallbackValue =
+		    uiLockCount + (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1);
+
+		if (OSAtomicAdd(&psPMR->iLockCount, uiLockCount) == uiCallbackValue)
+		{
+			if (psPMR->psFuncTab->pfnLockPhysAddresses != NULL)
+			{
+				eError = psPMR->psFuncTab->pfnLockPhysAddresses(psPMR->pvFlavourData);
+				PVR_LOG_IF_ERROR(eError, "pfnLockPhysAddresses");
+			}
+		}
+	}
+
+	OSLockRelease(psPMR->hLock);
+
+ErrReturn:
+	return eError;
+#endif
+}
+
+PVRSRV_ERROR
+PMRMakeLocalImportHandle(PMR *psPMR,
+                         PMR **ppsPMR)
+{
+	PVRSRV_ERROR eError = PMRRefPMR(psPMR);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppsPMR = psPMR;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRUnmakeLocalImportHandle(PMR *psPMR)
+{
+	return PMRUnrefPMR(psPMR);
+}
+
+/*
+	Note:
+	We pass back the PMR as it was passed in as a different handle type
+	(DEVMEM_MEM_IMPORT) and it allows us to change the import structure
+	type if we should need to embed any meta data in it.
+ */
+PVRSRV_ERROR
+PMRLocalImportPMR(PMR *psPMR,
+                  PMR **ppsPMR,
+                  IMG_DEVMEM_SIZE_T *puiSize,
+                  IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PVRSRV_ERROR eError = _Ref(psPMR, 1);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Return the PMR */
+	*ppsPMR = psPMR;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiAlign = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+	return PVRSRV_OK;
+}
+
+inline IMG_UINT64
+PMRInternalGetUID(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiSerialNum;
+}
+
+PVRSRV_ERROR
+PMRGetUID(PMR *psPMR,
+          IMG_UINT64 *pui64UID)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	*pui64UID = PMRInternalGetUID(psPMR);
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR
+PMRExportPMR(PMR *psPMR,
+             PMR_EXPORT **ppsPMRExportPtr,
+             PMR_SIZE_T *puiSize,
+             PMR_LOG2ALIGN_T *puiLog2Contig,
+             PMR_PASSWORD_T *puiPassword)
+{
+	IMG_UINT64 uiPassword;
+	PMR_EXPORT *psPMRExport;
+	PVRSRV_ERROR eError;
+
+	uiPassword = psPMR->uiKey;
+
+	psPMRExport = OSAllocMem(sizeof(*psPMRExport));
+	PVR_RETURN_IF_NOMEM(psPMRExport);
+
+	psPMRExport->psPMR = psPMR;
+
+	eError = _Ref(psPMR, 1);
+	PVR_GOTO_IF_ERROR(eError, ErrFreePMRExport);
+
+	/* The layout of a PMR can't change once exported
+	 * to make sure the importers view of the memory is
+	 * the same as exporter. */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	*ppsPMRExportPtr = psPMRExport;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiLog2Contig = psPMR->uiLog2ContiguityGuarantee;
+	*puiPassword = uiPassword;
+
+	return PVRSRV_OK;
+
+ErrFreePMRExport:
+	OSFreeMem(psPMRExport);
+
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMRUnexportPMR(PMR_EXPORT *psPMRExport)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPMRExport != NULL);
+	PVR_ASSERT(psPMRExport->psPMR != NULL);
+	PVR_ASSERT(psPMRExport->psPMR->uiRefCount > 0);
+
+	eError = _UnrefAndMaybeDestroy(psPMRExport->psPMR, 1);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_UnrefAndMaybeDestroy");
+
+	OSFreeMem(psPMRExport);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PMRImportPMR(PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR)
+{
+	PMR *psPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPMRExport->psPMR->uiRefCount > 0);
+
+	psPMR = psPMRExport->psPMR;
+
+	PVR_ASSERT(PMR_IsMemLayoutFixed(psPMR));
+
+	if (psPMR->uiKey != uiPassword)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"PMRImport: Import failed, password specified does not match the export"));
+		return PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR;
+	}
+
+	if (psPMR->uiLogicalSize != uiSize || psPMR->uiLog2ContiguityGuarantee != uiLog2Contig)
+	{
+		return PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES;
+	}
+
+	eError = _Ref(psPMR, 1);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppsPMR = psPMR;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRUnimportPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+
+#endif /* if defined(SUPPORT_INSECURE_EXPORT) */
+
+#if defined(SUPPORT_SECURE_EXPORT)
+PVRSRV_ERROR PMRSecureUnexportPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+
+static PVRSRV_ERROR _ReleaseSecurePMR(void *psExport)
+{
+	return PMRSecureUnexportPMR(psExport);
+}
+
+PVRSRV_ERROR PMRSecureExportPMR(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+                                PMR *psPMR,
+                                IMG_SECURE_TYPE *phSecure,
+                                PMR **ppsPMR,
+                                CONNECTION_DATA **ppsSecureConnection)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ppsSecureConnection);
+
+	/* We are acquiring reference to PMR here because OSSecureExport
+	 * releases bridge lock and PMR lock for a moment and we don't want PMR
+	 * to be removed by other thread in the meantime. */
+	eError = _Ref(psPMR, 1);
+	PVR_RETURN_IF_ERROR(eError);
+
+	eError = OSSecureExport("secure_pmr",
+	                        _ReleaseSecurePMR,
+	                        (void *) psPMR,
+	                        phSecure);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	*ppsPMR = psPMR;
+
+	/* Mark the PMR immutable once exported
+	 * This allows the importers and exporter to have
+	 * the same view of the memory */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	return PVRSRV_OK;
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	(void) _UnrefAndMaybeDestroy(psPMR, 1);
+	return eError;
+}
+
+PVRSRV_ERROR PMRSecureImportPMR(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE *psDevNode,
+                                IMG_SECURE_TYPE hSecure,
+                                PMR **ppsPMR,
+                                IMG_DEVMEM_SIZE_T *puiSize,
+                                IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMR;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = OSSecureImport(hSecure, (void **) &psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrReturnError);
+
+	PVR_LOG_RETURN_IF_FALSE(PhysHeapDeviceNode(psPMR->psPhysHeap) == psDevNode,
+	                        "PMR invalid for this device",
+	                        PVRSRV_ERROR_PMR_NOT_PERMITTED);
+
+	eError = _Ref(psPMR, 1);
+	PVR_GOTO_IF_ERROR(eError, ErrReturnError);
+
+	/* The PMR should be immutable once exported
+	 * This allows the importers and exporter to have
+	 * the same view of the memory */
+	PVR_ASSERT(PMR_IsMemLayoutFixed(psPMR));
+
+	/* Return the PMR */
+	*ppsPMR = psPMR;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiAlign = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+
+	return PVRSRV_OK;
+
+ErrReturnError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PMRSecureUnimportPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR
+PMRStoreRIHandle(PMR *psPMR,
+                 void *hRIHandle)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	psPMR->hRIHandle = hRIHandle;
+	return PVRSRV_OK;
+}
+#endif
+
+static PVRSRV_ERROR
+_PMRAcquireKernelMappingData(PMR *psPMR,
+                             size_t uiLogicalOffset,
+                             size_t uiSize,
+                             void **ppvKernelAddressOut,
+                             size_t *puiLengthOut,
+                             IMG_HANDLE *phPrivOut,
+                             IMG_BOOL bMapSparse)
+{
+	PVRSRV_ERROR eError;
+	void *pvKernelAddress;
+	IMG_HANDLE hPriv;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	if (_PMRIsSparse(psPMR) && !bMapSparse)
+	{
+		/* Mapping of sparse allocations must be signalled. */
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	/* Acquire/Release functions must be overridden in pairs */
+	if (psPMR->psFuncTab->pfnAcquireKernelMappingData == NULL)
+	{
+		PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData == NULL);
+
+		/* If PMR implementation does not supply this pair of
+		 * functions, it means they do not permit the PMR to be mapped
+		 * into kernel memory at all
+		 */
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PERMITTED, e0);
+	}
+	PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL);
+
+	eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+	                                                       uiLogicalOffset,
+	                                                       uiSize,
+	                                                       &pvKernelAddress,
+	                                                       &hPriv,
+	                                                       psPMR->uiFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	*ppvKernelAddressOut = pvKernelAddress;
+	if (uiSize == 0)
+	{
+		/* Zero size means map in the whole PMR ... */
+		*puiLengthOut = (size_t)psPMR->uiLogicalSize;
+	}
+	else if (uiSize > IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee))
+	{
+		/* ... map in the requested pages ... */
+		*puiLengthOut = uiSize;
+	}
+	else
+	{
+		/* ... otherwise we just map in one page */
+		*puiLengthOut = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+	}
+	*phPrivOut = hPriv;
+
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRAcquireKernelMappingData(PMR *psPMR,
+                            size_t uiLogicalOffset,
+                            size_t uiSize,
+                            void **ppvKernelAddressOut,
+                            size_t *puiLengthOut,
+                            IMG_HANDLE *phPrivOut)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountIncr(psPMR);
+
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psPMR)))
+	{
+		do
+		{
+			eError = _PMRAcquireKernelMappingData(psPMR,
+			                                      uiLogicalOffset,
+			                                      uiSize,
+			                                      ppvKernelAddressOut,
+			                                      puiLengthOut,
+			                                      phPrivOut,
+			                                      IMG_FALSE);
+		}
+		while (eError == PVRSRV_ERROR_RETRY);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PMRAcquireKernelMappingData", error_fail_decr);
+	}
+	else
+#endif
+	{
+		eError = _PMRAcquireKernelMappingData(psPMR,
+		                                      uiLogicalOffset,
+		                                      uiSize,
+		                                      ppvKernelAddressOut,
+		                                      puiLengthOut,
+		                                      phPrivOut,
+		                                      IMG_FALSE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PMRAcquireKernelMappingData", error_fail_decr);
+	}
+
+	return eError;
+
+error_fail_decr:
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountDecr(psPMR);
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRAcquireSparseKernelMappingData(PMR *psPMR,
+                                  size_t uiLogicalOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  size_t *puiLengthOut,
+                                  IMG_HANDLE *phPrivOut)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountIncr(psPMR);
+#endif
+
+	eError =  _PMRAcquireKernelMappingData(psPMR,
+	                                    uiLogicalOffset,
+	                                    uiSize,
+	                                    ppvKernelAddressOut,
+	                                    puiLengthOut,
+	                                    phPrivOut,
+	                                    IMG_TRUE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_PMRAcquireKernelMappingData", error_fail_decr);
+
+	return eError;
+
+error_fail_decr:
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountDecr(psPMR);
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRReleaseKernelMappingData(PMR *psPMR,
+                            IMG_HANDLE hPriv)
+{
+	PVR_ASSERT (psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL);
+	PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL);
+
+
+	psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+	                                              hPriv);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRKernelCpuMapCountDecr(psPMR);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	_PMRLogicalOffsetToPhysicalOffset
+
+	Translate between the "logical" offset which the upper levels
+	provide and the physical offset which is what the PMR
+	factories works on.
+
+	As well as returning the physical offset we return the number of
+	bytes remaining till the next chunk and if this chunk is valid.
+
+	For multi-page operations, upper layers communicate their
+	Log2PageSize else argument is redundant (set to zero).
+ */
+
+static PVRSRV_ERROR
+_PMRLogicalOffsetToPhysicalOffset(const PMR *psPMR,
+                                  IMG_UINT32 ui32Log2PageSize,
+                                  IMG_UINT32 ui32NumOfPages,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_DEVMEM_OFFSET_T *puiPhysicalOffset,
+                                  IMG_UINT32 *pui32BytesRemain,
+                                  IMG_BOOL *bValid)
+{
+	PMR_MAPPING_TABLE *psMappingTable = psPMR->psMappingTable;
+	IMG_DEVMEM_OFFSET_T uiPageSize = 1ULL << ui32Log2PageSize;
+	IMG_DEVMEM_OFFSET_T uiOffset = uiLogicalOffset;
+	IMG_UINT64 ui64ChunkIndex;
+	IMG_UINT32 ui32Remain;
+	IMG_UINT32 idx;
+
+	/* Must be translating at least a page */
+	PVR_ASSERT(ui32NumOfPages > 0);
+
+	if (psMappingTable->ui32NumPhysChunks == psMappingTable->ui32NumLogicalChunks)
+	{
+		/* Fast path the common case, as logical and physical offsets are
+		 * equal we assume the ui32NumOfPages span is also valid */
+		*pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiOffset);
+		puiPhysicalOffset[0] = uiOffset;
+		bValid[0] = IMG_TRUE;
+
+		if (ui32NumOfPages > 1)
+		{
+			/* initial offset may not be page aligned, round down */
+			uiOffset &= ~(uiPageSize-1);
+			for (idx = 1; idx < ui32NumOfPages; idx++)
+			{
+				uiOffset += uiPageSize;
+				puiPhysicalOffset[idx] = uiOffset;
+				bValid[idx] = IMG_TRUE;
+			}
+		}
+	}
+	else
+	{
+		for (idx = 0; idx < ui32NumOfPages; idx++)
+		{
+			IMG_UINT32 uiTranslation;
+			const IMG_UINT32 uiChunkSize = psMappingTable->uiChunkSize;
+
+			ui64ChunkIndex = OSDivide64r64(
+					uiOffset,
+					TRUNCATE_64BITS_TO_32BITS(uiChunkSize),
+					&ui32Remain);
+
+			/* In some cases ui32NumOfPages can come from the user space which
+			 * means that the uiOffset could go out-of-bounds when the number
+			 * of pages is invalid. */
+			if (ui64ChunkIndex >= psMappingTable->ui32NumLogicalChunks)
+			{
+				return PVRSRV_ERROR_BAD_MAPPING;
+			}
+
+			uiTranslation = psMappingTable->aui32Translation[ui64ChunkIndex];
+
+			if (uiTranslation == TRANSLATION_INVALID)
+			{
+				bValid[idx] = IMG_FALSE;
+				/* explicitly set to an invalid value */
+				puiPhysicalOffset[idx] = IMG_UINT64_C(0xffffffffffffffff);
+			}
+			else
+			{
+				bValid[idx] = IMG_TRUE;
+				puiPhysicalOffset[idx] = uiTranslation * uiChunkSize + ui32Remain;
+			}
+
+			if (idx == 0)
+			{
+				*pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(uiChunkSize - ui32Remain);
+				/* initial offset may not be page aligned, round down */
+				uiOffset &= ~(uiPageSize-1);
+			}
+
+			uiOffset += uiPageSize;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_PMR_ReadBytesPhysical(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                       IMG_UINT8 *pcBuffer,
+                       size_t uiBufSz,
+                       size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError;
+
+	if (psPMR->psFuncTab->pfnReadBytes != NULL)
+	{
+		/* defer to callback if present */
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		eError = psPMR->psFuncTab->pfnReadBytes(psPMR->pvFlavourData,
+		                                        uiPhysicalOffset,
+		                                        pcBuffer,
+		                                        uiBufSz,
+		                                        puiNumBytes);
+		PMRUnlockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+	}
+	else if (psPMR->psFuncTab->pfnAcquireKernelMappingData)
+	{
+		/* "default" handler for reading bytes */
+
+		IMG_HANDLE hKernelMappingHandle;
+		IMG_UINT8 *pcKernelAddress;
+
+		eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+		                                                       (size_t) uiPhysicalOffset,
+		                                                       uiBufSz,
+		                                                       (void **)&pcKernelAddress,
+		                                                       &hKernelMappingHandle,
+		                                                       psPMR->uiFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Use the conservative 'DeviceMemCopy' here because we can't
+		 * know if this PMR will be mapped cached.
+		 */
+
+		OSDeviceMemCopy(&pcBuffer[0], pcKernelAddress, uiBufSz);
+		*puiNumBytes = uiBufSz;
+
+		psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+		                                              hKernelMappingHandle);
+	}
+	else
+	{
+		OSPanic();
+#ifndef __CHECKER__ /* exclude for Smatch */
+		PVR_LOG_GOTO_WITH_ERROR("psPMR->psFuncTab", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+#endif
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*puiNumBytes = 0;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMR_ReadBytes(PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT8 *pcBuffer,
+              size_t uiBufSz,
+              size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	size_t uiBytesCopied = 0;
+
+	/* Check for integer overflow as uiLogicalOffset might come from the client */
+	if (uiLogicalOffset + uiBufSz < uiLogicalOffset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize)
+	{
+		uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset);
+	}
+	PVR_ASSERT(uiBufSz > 0);
+	PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize);
+
+	/* PMR implementations can override this. If they don't, a "default"
+	 * handler uses kernel virtual mappings.  If the kernel can't
+	 * provide a kernel virtual mapping, this function fails.
+	 */
+	PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL ||
+	           psPMR->psFuncTab->pfnReadBytes != NULL);
+
+	while (uiBytesCopied != uiBufSz)
+	{
+		IMG_UINT32 ui32Remain;
+		size_t uiBytesToCopy;
+		size_t uiRead;
+		IMG_BOOL bValid;
+
+		eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+		                                           0,
+		                                           1,
+		                                           uiLogicalOffset,
+		                                           &uiPhysicalOffset,
+		                                           &ui32Remain,
+		                                           &bValid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+		/* Copy till either then end of the chunk or end
+		 * of the buffer
+		 */
+		uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain);
+
+		if (bValid)
+		{
+			/* Read the data from the PMR */
+			eError = _PMR_ReadBytesPhysical(psPMR,
+			                                uiPhysicalOffset,
+			                                &pcBuffer[uiBytesCopied],
+			                                uiBytesToCopy,
+			                                &uiRead);
+			if ((eError != PVRSRV_OK) || (uiRead != uiBytesToCopy))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to read chunk (eError = %s, uiRead = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")",
+						__func__,
+						PVRSRVGetErrorString(eError),
+						uiRead,
+						uiBytesToCopy));
+				/* Bail out as soon as we hit an error */
+				break;
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Invalid phys offset at logical offset (" IMG_DEVMEM_OFFSET_FMTSPEC ") logical size (" IMG_DEVMEM_OFFSET_FMTSPEC ")",
+					__func__,
+					uiLogicalOffset,
+					psPMR->uiLogicalSize));
+			/* Fill invalid chunks with 0 */
+			OSCachedMemSet(&pcBuffer[uiBytesCopied], 0, uiBytesToCopy);
+			uiRead = uiBytesToCopy;
+			eError = PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR;
+		}
+		uiLogicalOffset += uiRead;
+		uiBytesCopied += uiRead;
+	}
+
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+static PVRSRV_ERROR
+_PMR_WriteBytesPhysical(PMR *psPMR,
+                        IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiBufSz,
+                        size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError;
+
+	if (psPMR->psFuncTab->pfnWriteBytes != NULL)
+	{
+		/* defer to callback if present */
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		eError = psPMR->psFuncTab->pfnWriteBytes(psPMR->pvFlavourData,
+		                                         uiPhysicalOffset,
+		                                         pcBuffer,
+		                                         uiBufSz,
+		                                         puiNumBytes);
+		PMRUnlockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+	}
+	else if (psPMR->psFuncTab->pfnAcquireKernelMappingData)
+	{
+		/* "default" handler for reading bytes */
+
+		IMG_HANDLE hKernelMappingHandle;
+		IMG_UINT8 *pcKernelAddress;
+
+		eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+		                                                       (size_t) uiPhysicalOffset,
+		                                                       uiBufSz,
+		                                                       (void **)&pcKernelAddress,
+		                                                       &hKernelMappingHandle,
+		                                                       psPMR->uiFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Use the conservative 'DeviceMemCopy' here because we can't know
+		 * if this PMR will be mapped cached.
+		 */
+
+		OSDeviceMemCopy(pcKernelAddress, &pcBuffer[0], uiBufSz);
+		*puiNumBytes = uiBufSz;
+
+		psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+		                                              hKernelMappingHandle);
+	}
+	else
+	{
+		/* The write callback is optional as it's only required by the
+		 * debug tools
+		 */
+		OSPanic();
+#ifndef __CHECKER__ /* exclude for Smatch */
+		PVR_LOG_GOTO_WITH_ERROR("psPMR->psFuncTab", eError, PVRSRV_ERROR_PMR_NOT_PERMITTED, e0);
+#endif
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*puiNumBytes = 0;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMR_WriteBytes(PMR *psPMR,
+               IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+               IMG_UINT8 *pcBuffer,
+               size_t uiBufSz,
+               size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	size_t uiBytesCopied = 0;
+
+	/* Check for integer overflow as uiLogicalOffset might come from the client */
+	if (uiLogicalOffset + uiBufSz < uiLogicalOffset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize)
+	{
+		uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset);
+	}
+	PVR_ASSERT(uiBufSz > 0);
+	PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize);
+
+	/* PMR implementations can override this. If they don't, a "default"
+	 * handler uses kernel virtual mappings. If the kernel can't provide
+	 * a kernel virtual mapping, this function fails.
+	 */
+	PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL ||
+	           psPMR->psFuncTab->pfnWriteBytes != NULL);
+
+	while (uiBytesCopied != uiBufSz)
+	{
+		IMG_UINT32 ui32Remain;
+		size_t uiBytesToCopy;
+		size_t uiWrite;
+		IMG_BOOL bValid;
+
+		eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+		                                           0,
+		                                           1,
+		                                           uiLogicalOffset,
+		                                           &uiPhysicalOffset,
+		                                           &ui32Remain,
+		                                           &bValid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+		/* Copy till either then end of the chunk or end of the buffer
+		 */
+		uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain);
+
+		if (bValid)
+		{
+			/* Write the data to the PMR */
+			eError = _PMR_WriteBytesPhysical(psPMR,
+			                                 uiPhysicalOffset,
+			                                 &pcBuffer[uiBytesCopied],
+			                                 uiBytesToCopy,
+			                                 &uiWrite);
+			if ((eError != PVRSRV_OK) || (uiWrite != uiBytesToCopy))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to read chunk (eError = %s, uiWrite = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")",
+						__func__,
+						PVRSRVGetErrorString(eError),
+						uiWrite,
+						uiBytesToCopy));
+				/* Bail out as soon as we hit an error */
+				break;
+			}
+		}
+		else
+		{
+			/* Ignore writes to invalid pages */
+			uiWrite = uiBytesToCopy;
+		}
+		uiLogicalOffset += uiWrite;
+		uiBytesCopied += uiWrite;
+	}
+
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRMMapPMR(PMR *psPMR, PMR_MMAP_DATA pOSMMapData, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	/* if writeable mapping is requested on non-writeable PMR then fail */
+	PVR_RETURN_IF_FALSE(PVRSRV_CHECK_CPU_WRITEABLE(psPMR->uiFlags) ||
+	                    !PVRSRV_CHECK_CPU_WRITEABLE(uiFlags),
+	                    PVRSRV_ERROR_PMR_NOT_PERMITTED);
+
+	PVR_LOG_RETURN_IF_TRUE(PMR_PhysicalSize(psPMR) == 0, "PVRSRV_ERROR_BAD_MAPPING can not map PMR of 0 physical size", PVRSRV_ERROR_BAD_MAPPING);
+
+	if (psPMR->psFuncTab->pfnMMap)
+	{
+		return psPMR->psFuncTab->pfnMMap(psPMR->pvFlavourData, psPMR, pOSMMapData);
+	}
+
+	return OSMMapPMRGeneric(psPMR, pOSMMapData);
+}
+
+PVRSRV_ERROR
+PMRRefPMR(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _Ref(psPMR, 1);
+}
+
+PVRSRV_ERROR
+PMRUnrefPMR(PMR *psPMR)
+{
+	return _UnrefAndMaybeDestroy(psPMR, 1);
+}
+
+PVRSRV_ERROR
+PMRRefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _Ref(psPMR, uiRefCount);
+}
+
+PVRSRV_ERROR
+PMRUnrefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _UnrefAndMaybeDestroy(psPMR, uiRefCount);
+}
+
+#define PMR_MAPCOUNT_MIN 0
+#define PMR_MAPCOUNT_MAX IMG_INT32_MAX
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+PVRSRV_ERROR PMRTryRefPMR(PMR *psPMR)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	OSSpinLockAcquire(psPMR->hRefCountLock, uiFlags);
+
+	if (psPMR->uiRefCount == 0 || psPMR->uiRefCount == IMG_UINT32_MAX)
+	{
+		OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	psPMR->uiRefCount++;
+
+	OSSpinLockRelease(psPMR->hRefCountLock, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+void
+PMRKernelCpuMapCountIncr(PMR *psPMR)
+{
+	if (OSAtomicAddUnless(&psPMR->iKernelCpuMapCount, 1, PMR_MAPCOUNT_MAX) == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iKernelCpuMapCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+void
+PMRKernelCpuMapCountDecr(PMR *psPMR)
+{
+	if (OSAtomicSubtractUnless(&psPMR->iKernelCpuMapCount, 1, PMR_MAPCOUNT_MIN) == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iKernelCpuMapCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        (IMG_INT32) OSAtomicRead(&psPMR->iKernelCpuMapCount),
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+IMG_BOOL
+PMR_IsKernelCpuMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return (OSAtomicRead(&psPMR->iKernelCpuMapCount) > 0);
+}
+
+#endif /* #if defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+void
+PMRClientCpuMapCountIncr(PMR *psPMR)
+{
+	if (OSAtomicAddUnless(&psPMR->iClientCpuMapCount, 1, PMR_MAPCOUNT_MAX) == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iClientCpuMapCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+void
+PMRClientCpuMapCountDecr(PMR *psPMR)
+{
+	if (OSAtomicSubtractUnless(&psPMR->iClientCpuMapCount, 1, PMR_MAPCOUNT_MIN) == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iClientCpuMapCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        (IMG_INT32) OSAtomicRead(&psPMR->iClientCpuMapCount),
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+IMG_BOOL
+PMR_IsClientCpuMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return (OSAtomicRead(&psPMR->iClientCpuMapCount) > 0);
+}
+
+static void
+PMRGpuResCountIncr(PMR *psPMR)
+{
+	if (psPMR->iAssociatedResCount == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iAssociatedResCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+		return;
+	}
+
+	psPMR->iAssociatedResCount++;
+}
+
+static void
+PMRGpuResCountDecr(PMR *psPMR)
+{
+	if (psPMR->iAssociatedResCount == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iAssociatedResCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        psPMR->iAssociatedResCount,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+		return;
+	}
+
+	psPMR->iAssociatedResCount--;
+}
+
+IMG_BOOL
+PMR_IsGpuMultiMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->iAssociatedResCount > 1;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRLinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode)
+#else
+void
+PMRLinkGPUMapping(PMR *psPMR)
+#endif
+{
+	PMRLockHeldAssert(psPMR);
+
+	PMRGpuResCountIncr(psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	dllist_add_to_head(&psPMR->sGpuMappingListHead, psMappingNode);
+#endif
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRUnlinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode)
+#else
+void
+PMRUnlinkGPUMapping(PMR *psPMR)
+#endif
+
+{
+	PMRLockHeldAssert(psPMR);
+
+	PMRGpuResCountDecr(psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	dllist_remove_node(psMappingNode);
+#endif
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRNotifyMigrateInProgress(PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psPMR->eState = PMR_STATE_PAGES_IN_MIGRATE;
+}
+
+void
+PMRNotifyMigrateComplete(PMR *psPMR)
+{
+	PMRLockHeldAssert(psPMR);
+
+	psPMR->eState = PMR_STATE_ACTIVE;
+}
+
+PVRSRV_ERROR
+PMRRemapGPUPMR(PMR *psPMR, IMG_UINT32 ui32LogicalPgOffset)
+{
+	if (psPMR->eState != PMR_STATE_PAGES_IN_MIGRATE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Remap requested on PMR not in migrate state.", __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	return DevmemIntRemapPageInPMR(psPMR, &psPMR->sGpuMappingListHead, ui32LogicalPgOffset);
+}
+#endif
+
+PVRSRV_DEVICE_NODE *
+PMR_DeviceNode(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return PhysHeapDeviceNode(psPMR->psPhysHeap);
+}
+
+PMR_FLAGS_T
+PMR_Flags(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiFlags;
+}
+
+IMG_BOOL
+PMR_IsSparse(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return _PMRIsSparse(psPMR);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+IMG_BOOL
+PMR_IsZombie(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return _IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Function that alters the mutability property
+ * of the PMR
+ * Setting it to TRUE makes sure the PMR memory layout
+ * can't be changed through future calls */
+void
+PMR_SetLayoutFixed(PMR *psPMR, IMG_BOOL bFlag)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	if (bFlag)
+	{
+		_IntFlagSet(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+	}
+	else
+	{
+		_IntFlagClr(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+	}
+}
+
+IMG_BOOL PMR_IsMemLayoutFixed(PMR *psPMR)
+{
+
+	PVR_ASSERT(psPMR != NULL);
+
+	return _IntFlagIsSet(psPMR, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+}
+
+IMG_DEVMEM_SIZE_T
+PMR_PhysicalSize(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	/* iLockCount will be > 0 for any backed PMR (backed on demand or not) */
+	if (OSAtomicRead(&psPMR->iLockCount) > 0)
+	{
+		if (_PMRIsSparse(psPMR))
+		{
+			return psPMR->psMappingTable->uiChunkSize * psPMR->psMappingTable->ui32NumPhysChunks;
+		}
+		else
+		{
+			return psPMR->uiLogicalSize;
+		}
+	}
+
+	return 0;
+}
+
+PVRSRV_ERROR
+PMR_IsOffsetValid(const PMR *psPMR,
+                  IMG_UINT32 ui32Log2PageSize,
+                  IMG_UINT32 ui32NumOfPages,
+                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                  IMG_BOOL *pbValid)
+{
+	IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 aui32BytesRemain[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset;
+	IMG_UINT32 *pui32BytesRemain = aui32BytesRemain;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psPMR != NULL);
+	PVR_ASSERT(psPMR->uiLogicalSize >= uiLogicalOffset);
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T));
+		PVR_GOTO_IF_NOMEM(puiPhysicalOffset, eError, e0);
+
+		pui32BytesRemain = OSAllocMem(ui32NumOfPages * sizeof(IMG_UINT32));
+		PVR_GOTO_IF_NOMEM(pui32BytesRemain, eError, e0);
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           ui32Log2PageSize,
+	                                           ui32NumOfPages,
+	                                           uiLogicalOffset,
+	                                           puiPhysicalOffset,
+	                                           pui32BytesRemain,
+	                                           pbValid);
+	PVR_LOG_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+e0:
+	if (puiPhysicalOffset != auiPhysicalOffset && puiPhysicalOffset != NULL)
+	{
+		OSFreeMem(puiPhysicalOffset);
+	}
+
+	if (pui32BytesRemain != aui32BytesRemain && pui32BytesRemain != NULL)
+	{
+		OSFreeMem(pui32BytesRemain);
+	}
+
+	return eError;
+}
+
+PHYS_HEAP *
+PMR_PhysHeap(const PMR *psPMR)
+{
+	return psPMR->psPhysHeap;
+}
+
+PMR_MAPPING_TABLE *
+PMR_GetMappingTable(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->psMappingTable;
+
+}
+
+IMG_UINT32
+PMR_GetLog2Contiguity(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->uiLog2ContiguityGuarantee;
+}
+
+IMG_DEVMEM_SIZE_T
+PMR_LogicalSize(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiLogicalSize;
+}
+
+IMG_UINT32 PMR_GetLogicalChunkCount(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->psMappingTable->ui32NumLogicalChunks;
+}
+
+const IMG_CHAR *
+PMR_GetAnnotation(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->szAnnotation;
+}
+
+PMR_IMPL_TYPE
+PMR_GetType(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->eFlavour;
+}
+
+IMG_CHAR *
+PMR_GetTypeStr(const PMR *psPMR)
+{
+	static IMG_CHAR *pszFlavour[] = {
+#define X(type) #type
+		PMR_IMPL_TYPES
+#undef X
+	};
+
+	if (psPMR->eFlavour >= PMR_TYPE_LAST)
+	{
+		return "INVALID";
+	}
+
+	return pszFlavour[psPMR->eFlavour];
+}
+
+IMG_INT32
+PMR_GetRefCount(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return _GetRef(psPMR);
+}
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRGetIPAPolicy(PMR *psPMR,
+                IMG_UINT8 *pui8IPAPolicy)
+{
+	IMG_UINT32 ui32FlagsIPAPolicy;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui8IPAPolicy != NULL, "pui8IPAPolicy");
+	ui32FlagsIPAPolicy = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+
+	*pui8IPAPolicy = (IMG_UINT8)ui32FlagsIPAPolicy;
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING) */
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRGetIPAInfo(PMR *psPMR, IMG_UINT32 *pui32IPAPolicy, IMG_UINT32 *pui32IPAShift,
+              IMG_UINT32 *pui32IPAMask, IMG_UINT32 *pui32IPAFlagsValue)
+{
+	IMG_UINT32 ui32IPAFlagsPolicyValue;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAPolicy != NULL, "pui32IPAPolicy");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAShift != NULL, "pui32IPAShift");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAMask != NULL, "pui32IPAMask");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAFlagsValue != NULL, "pui32IPAFlagsValue");
+
+	/* Get the underlying heap-provided default IPA settings (if any) */
+	*pui32IPAShift = PhysHeapGetIPAShift(psPMR->psPhysHeap);
+	*pui32IPAPolicy = PhysHeapGetIPAValue(psPMR->psPhysHeap);
+	*pui32IPAMask = PhysHeapGetIPAMask(psPMR->psPhysHeap);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: Shift, Policy, Mask for Heap %p = %d, %d, %d", __func__,
+	        psPMR->psPhysHeap, *pui32IPAShift, *pui32IPAPolicy, *pui32IPAMask));
+
+	/* Query the current PMR flags settings for current IPA policy */
+	ui32IPAFlagsPolicyValue = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+	*pui32IPAFlagsValue = ui32IPAFlagsPolicyValue;
+	return PVRSRV_OK;
+}
+#endif	/* PVRSRV_INTERNAL_IPA_FEATURE_TESTING */
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRModifyIPAPolicy(PMR *psPMR,
+                   IMG_UINT8 ui8NewIPAPolicy)
+{
+	IMG_UINT64 ui64IPAFlagsPolicyValue;
+	IMG_UINT32 ui32IPAPolicyMaskValue;
+
+	ui32IPAPolicyMaskValue = (IMG_UINT32)(PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK >> PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET);
+
+	/* ui8NewIPAPolicy must be between 0 .. uiMaskValue inclusive */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui8NewIPAPolicy <= ui32IPAPolicyMaskValue,
+	                                "ui8NewIPAPolicy");
+
+	/* Set the appropriate policy bits in the PMR */
+	ui64IPAFlagsPolicyValue = ((IMG_UINT64)(ui8NewIPAPolicy) << PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET) &
+	                           PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+
+	psPMR->uiFlags &= ~PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+	psPMR->uiFlags |= ui64IPAFlagsPolicyValue;
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: ui32IPAPolicy Mask = 0x%x, Value = 0x%x",
+	         __func__, ui32IPAPolicyMaskValue, (IMG_UINT32)ui8NewIPAPolicy));
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: uiFlags = 0x%016" IMG_UINT64_FMTSPECx ", ui64IPAFlags = 0x%016" IMG_UINT64_FMTSPECx, __func__, psPMR->uiFlags, ui64IPAFlagsPolicyValue));
+
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING) */
+
+/* must have called PMRLockSysPhysAddresses() before calling this! */
+PVRSRV_ERROR
+PMR_DevPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEV_PHYADDR *psDevAddrPtr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage)
+{
+	IMG_UINT32 ui32Remain;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset;
+#if defined(SUPPORT_STATIC_IPA)
+	IMG_UINT32 ui32FlagsIPAPolicy;      /* Local value for the IPA policy */
+	IMG_UINT32 ui32IPAHeapShift;        /* Phys-heap bit-shift value */
+	IMG_UINT32 ui32IPAHeapPolicyValue;  /* Phys-heap default policy value */
+	IMG_UINT32 ui32IPAHeapClearMask;    /* Phys-heap ClearMask bitmask */
+	IMG_UINT64 ui64IPAPolicy;           /* IPAPolicy value to be applied to physical address(es) */
+	IMG_UINT64 ui64IPAClearMask;        /* IPAClearMask to be applied to physical address(es) */
+#endif
+
+	PVR_ASSERT(psPMR != NULL);
+	PVR_ASSERT(ui32NumOfPages > 0);
+	PVR_ASSERT(psPMR->psFuncTab->pfnDevPhysAddr != NULL);
+
+	if (OSAtomicRead(&psPMR->iLockCount) <= 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"Attempt to obtain DevPhysAddr of non-backed PMR"));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T));
+		PVR_RETURN_IF_NOMEM(puiPhysicalOffset);
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           ui32Log2PageSize,
+	                                           ui32NumOfPages,
+	                                           uiLogicalOffset,
+	                                           puiPhysicalOffset,
+	                                           &ui32Remain,
+	                                           pbValid);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset", FreeOffsetArray);
+
+#if defined(SUPPORT_STATIC_IPA)
+	/* Need to determine the values to pass into the pfnDevPhysAddr
+	 * for Intermediate Physical Address settings associated with
+	 * this PMR.
+	 * If the ui32FlagsIPAPolicy value is non-zero, the value will be used in
+	 * preference to the default value specified in the physheap config.
+	 * Whichever value is used the associated physheap configuration bit shift and
+	 * mask values are passed to the pfnDevPhysAddr PMR factory function to modify
+	 * the returned address(es).
+	 */
+
+	PVR_ASSERT(psPMR->psPhysHeap != NULL);
+
+	ui32IPAHeapShift = PhysHeapGetIPAShift(psPMR->psPhysHeap);
+	ui32IPAHeapPolicyValue = PhysHeapGetIPAValue(psPMR->psPhysHeap);
+	ui32IPAHeapClearMask = PhysHeapGetIPAMask(psPMR->psPhysHeap);
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	ui32FlagsIPAPolicy = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+	if (ui32FlagsIPAPolicy == 0U)
+	{
+		ui32FlagsIPAPolicy = ui32IPAHeapPolicyValue;
+	}
+#else
+	ui32FlagsIPAPolicy = ui32IPAHeapPolicyValue;	/* Use heap default values*/
+#endif	/* PVRSRV_INTERNAL_IPA_FEATURE_TESTING */
+	/* To handle the 'disabling' of IPAPolicy setting for some callers we
+	 * check to see if the uiPMRUsage is set to DEVICE_USE.
+	 * If so, we simply use the calculated shifts and policy values determined
+	 * above. If disabled BIT_ISSET(uiPMRUsage, CPU_USE) we pass 0 values to the PMR
+	 * factory which will result in no IPA modification being made to the
+	 * phys_heap physical addresses.
+	 */
+	if (unlikely(BITMASK_HAS(uiPMRUsage, CPU_USE)))
+	{
+		ui32IPAHeapClearMask = 0U;
+		ui32FlagsIPAPolicy = 0U;
+	}
+	ui64IPAPolicy = (IMG_UINT64)ui32FlagsIPAPolicy << ui32IPAHeapShift;
+	ui64IPAClearMask = (IMG_UINT64)ui32IPAHeapClearMask << ui32IPAHeapShift;
+#endif	/* SUPPORT_STATIC_IPA */
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* PMRs marked with migrate ability need to wait for migrate to complete
+	 * whilst in the CPU mapping path. PMR lock must be held in this path to
+	 * synchronise obtaining phys addrs.
+	 */
+	if (BITMASK_HAS(uiPMRUsage, MAPPING_USE) && BITMASK_HAS(uiPMRUsage, CPU_USE))
+	{
+		PMRLockHeldAssert(psPMR);
+
+		/* We must wait and retry until we are allowed to obtain phys addrs, no timeout
+		 * because we need this to complete and migrate is guaranteed to finish eventually
+		 */
+		while (1)
+		{
+			if (psPMR->eState == PMR_STATE_ACTIVE)
+			{
+				break;
+			}
+			/* Allow another thread to take the lock, this should allow migration to continue
+			 * and complete before we take physical addresses for use
+			 */
+			PMRUnlockPMR(psPMR);
+			OSReleaseThreadQuanta();
+			PMRLockPMR(psPMR);
+		}
+
+		eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData,
+												  ui32Log2PageSize,
+												  ui32NumOfPages,
+												  puiPhysicalOffset,
+#if defined(SUPPORT_STATIC_IPA)
+												  ui64IPAPolicy,
+												  ui64IPAClearMask,
+#endif
+												  pbValid,
+												  psDevAddrPtr);
+	}
+	/* PMRs marked with migrate ability may need to retry an attempt at Device mappings
+	 * for a migrate to complete whilst in the mapping path. This is because we hold higher
+	 * synchronisation methods in device mapping paths that also need to release for the
+	 * migrate to complete.
+	 */
+	else if (BITMASK_HAS(uiPMRUsage, MAPPING_USE) && BITMASK_HAS(uiPMRUsage, DEVICE_USE))
+	{
+		PMRLockHeldAssert(psPMR);
+
+		/* If PMR is in migrate state then ask the requester to retry */
+		if (psPMR->eState != PMR_STATE_ACTIVE)
+		{
+			eError = PVRSRV_ERROR_RETRY;
+			goto FreeOffsetArray;
+		}
+
+		eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData,
+												  ui32Log2PageSize,
+												  ui32NumOfPages,
+												  puiPhysicalOffset,
+#if defined(SUPPORT_STATIC_IPA)
+												  ui64IPAPolicy,
+												  ui64IPAClearMask,
+#endif
+												  pbValid,
+												  psDevAddrPtr);
+	}
+	else
+	/* All other paths that are not used in GPU mapping will never return
+	 * PVRSRV_ERROR_RETRY.
+	 */
+#endif
+	{
+		/* Sparse PMR may not always have the first page valid */
+		eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData,
+												  ui32Log2PageSize,
+												  ui32NumOfPages,
+												  puiPhysicalOffset,
+#if defined(SUPPORT_STATIC_IPA)
+												  ui64IPAPolicy,
+												  ui64IPAClearMask,
+#endif
+												  pbValid,
+												  psDevAddrPtr);
+		PVR_GOTO_IF_ERROR(eError, FreeOffsetArray);
+	}
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+	/* Currently excluded from the default build because of performance
+	 * concerns.
+	 * We do not need this part in all systems because the GPU has the same
+	 * address view of system RAM as the CPU.
+	 * Alternatively this could be implemented as part of the PMR-factories
+	 * directly */
+	if (PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_UMA ||
+		PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_DMA)
+	{
+		IMG_UINT32 i;
+		IMG_DEV_PHYADDR sDevPAddrCorrected;
+
+		/* Copy the translated addresses to the correct array */
+		for (i = 0; i < ui32NumOfPages; i++)
+		{
+			PhysHeapCpuPAddrToDevPAddr(psPMR->psPhysHeap,
+									   1,
+									   &sDevPAddrCorrected,
+									   (IMG_CPU_PHYADDR *) &psDevAddrPtr[i]);
+			psDevAddrPtr[i].uiAddr = sDevPAddrCorrected.uiAddr;
+		}
+	}
+#endif
+
+
+FreeOffsetArray:
+	if (puiPhysicalOffset != auiPhysicalOffset)
+	{
+		OSFreeMem(puiPhysicalOffset);
+	}
+
+	return eError;
+}
+
+/* must have called PMRLockSysPhysAddresses() before calling this! */
+PVRSRV_ERROR
+PMR_CpuPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_CPU_PHYADDR *psCpuAddrPtr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage)
+{
+	IMG_UINT32 idx;
+	PVRSRV_ERROR eError;
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *psDevPAddr = asDevPAddr;
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+		PVR_GOTO_IF_NOMEM(psDevPAddr, eError, e0);
+	}
+
+	eError = PMR_DevPhysAddr(psPMR, ui32Log2PageSize, ui32NumOfPages,
+	                         uiLogicalOffset, psDevPAddr, pbValid, uiPMRUsage);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	if (_PMRIsSparse(psPMR))
+	{
+		/* Loop over each page.
+		 * If Dev addr valid, populate the CPU addr from the Dev addr
+		 */
+		for (idx = 0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPMR->psPhysHeap, 1, &psCpuAddrPtr[idx], &psDevPAddr[idx]);
+			}
+		}
+	}
+	else
+	{
+		/* In this case all addrs will be valid, so we can block translate */
+		PhysHeapDevPAddrToCpuPAddr(psPMR->psPhysHeap, ui32NumOfPages, psCpuAddrPtr, psDevPAddr);
+	}
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+
+	return PVRSRV_OK;
+e1:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PMR_ChangeSparseMem(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PMRLockPMR(psPMR);
+
+	eError = PMR_ChangeSparseMemUnlocked(psPMR,
+	                                     ui32AllocPageCount,
+	                                     pai32AllocIndices,
+	                                     ui32FreePageCount,
+	                                     pai32FreeIndices,
+	                                     uiSparseFlags);
+
+	PMRUnlockPMR(psPMR);
+
+	return eError;
+}
+
+PVRSRV_ERROR PMR_ChangeSparseMemUnlocked(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_IMPL_ZOMBIEPAGES pvZombiePages = NULL;
+	PMR_ZOMBIE_PAGES* psPMRZombiePages = NULL;
+#endif
+
+	if (PMR_IsMemLayoutFixed(psPMR) || PMR_IsClientCpuMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This PMR layout cannot be changed - PMR_IsMemLayoutFixed()=%c, _PMR_IsClientCpuMapped()=%c",
+				__func__,
+				PMR_IsMemLayoutFixed(psPMR) ? 'Y' : 'n',
+				PMR_IsClientCpuMapped(psPMR) ? 'Y' : 'n'));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (NULL == psPMR->psFuncTab->pfnChangeSparseMem)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This type of sparse PMR cannot be changed.",
+				__func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		/* Speculatively preallocate in order to simplify error handling later */
+		psPMRZombiePages = OSAllocZMem(sizeof(PMR_ZOMBIE_PAGES));
+		PVR_GOTO_IF_NOMEM(psPMRZombiePages, eError, e0);
+	}
+#endif
+
+	eError = psPMR->psFuncTab->pfnChangeSparseMem(psPMR->pvFlavourData,
+	                                              psPMR,
+	                                              ui32AllocPageCount,
+	                                              pai32AllocIndices,
+	                                              ui32FreePageCount,
+	                                              pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	                                              &pvZombiePages,
+#endif
+	                                              uiSparseFlags);
+	if (eError != PVRSRV_OK)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		if (eError == PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES)
+		{
+			PVRSRVStatsUpdateOOMStat(NULL,
+									  PMR_DeviceNode(psPMR),
+									  PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+									  OSGetCurrentClientProcessIDKM());
+		}
+#endif
+		goto e1;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+		PVRSRV_DEVICE_NODE *psDevNode;
+
+		psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+		eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+		if (eError != PVRSRV_OK)
+		{
+			/* Treat unknown power state as ON. */
+			ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+		}
+
+		if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF ||
+			psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+		{
+			/* Free preallocated psPMRZombiePages as these won't be used*/
+			OSFreeMem(psPMRZombiePages);
+
+			eError = psPMR->psFuncTab->pfnFreeZombiePages(pvZombiePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "Error when trying to free zombies immediately.", e0);
+		}
+		else
+		{
+			PVR_ASSERT(psPMRZombiePages != NULL);
+			psPMRZombiePages->sHeader.eZombieType = PMR_ZOMBIE_TYPE_PAGES;
+			psPMRZombiePages->pfnFactoryFreeZombies = psPMR->psFuncTab->pfnFreeZombiePages;
+			psPMRZombiePages->pvFactoryPages = pvZombiePages;
+
+			_ZombieListLock(psDevNode);
+			dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMRZombiePages->sHeader.sZombieNode);
+			psDevNode->uiPMRZombieCount++;
+			_ZombieListUnlock(psDevNode);
+		}
+	}
+	else
+	{
+		/* Free psPMRZombiePages as change sparse has not produced zombie pages */
+		OSFreeMem(psPMRZombiePages);
+	}
+#endif
+
+#if defined(PDUMP)
+	{
+		IMG_BOOL bInitialise = IMG_FALSE;
+		IMG_UINT8 ui8InitValue = 0;
+
+		if (PVRSRV_CHECK_ZERO_ON_ALLOC(PMR_Flags(psPMR)))
+		{
+			bInitialise = IMG_TRUE;
+		}
+		else if (PVRSRV_CHECK_POISON_ON_ALLOC(PMR_Flags(psPMR)))
+		{
+			ui8InitValue = (IMG_UINT8)PVRSRV_POISON_ON_ALLOC_VALUE;
+			bInitialise = IMG_TRUE;
+		}
+
+		PDumpPMRChangeSparsePMR(psPMR,
+		                        IMG_PAGE2BYTES32(psPMR->uiLog2ContiguityGuarantee),
+		                        ui32AllocPageCount,
+		                        pai32AllocIndices,
+		                        ui32FreePageCount,
+		                        pai32FreeIndices,
+		                        bInitialise,
+		                        ui8InitValue,
+		                        &psPMR->hPDumpAllocHandle);
+	}
+
+#endif
+
+	return PVRSRV_OK;
+e1:
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		OSFreeMem(psPMRZombiePages);
+	}
+e0:
+#endif
+	return eError;
+}
+
+#if defined(PDUMP)
+
+static PVRSRV_ERROR
+_PMR_PDumpSymbolicAddrPhysical(const PMR *psPMR,
+                               IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                               IMG_UINT32 ui32MemspaceNameLen,
+                               IMG_CHAR *pszMemspaceName,
+                               IMG_UINT32 ui32SymbolicAddrLen,
+                               IMG_CHAR *pszSymbolicAddr,
+                               IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                               IMG_DEVMEM_OFFSET_T *puiNextSymName)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (PVRSRV_CHECK_PHYS_HEAP(FW_CODE, psPMR->uiFlags) ||
+		PVRSRV_CHECK_PHYS_HEAP(FW_PRIV_DATA, psPMR->uiFlags) ||
+	    PVRSRV_CHECK_PHYS_HEAP(GPU_SECURE, psPMR->uiFlags))
+	{
+		OSSNPrintf(pszMemspaceName, ui32MemspaceNameLen, PMR_MEMSPACE_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+	else
+#endif
+	if (DevmemDeviceCacheCoherency(psDevNode, psPMR->uiFlags))
+	{
+		OSSNPrintf(pszMemspaceName,
+		           ui32MemspaceNameLen,
+		           PMR_MEMSPACE_CACHE_COHERENT_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+	else
+	{
+		OSSNPrintf(pszMemspaceName, ui32MemspaceNameLen, PMR_MEMSPACE_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+
+	OSSNPrintf(pszSymbolicAddr,
+	           ui32SymbolicAddrLen,
+	           PMR_SYMBOLICADDR_FMTSPEC,
+	           PMR_IsSparse(psPMR) ? PMR_SPARSE_PREFIX : PMR_DEFAULT_PREFIX,
+	           psPMR->uiSerialNum,
+	           uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR),
+	           psPMR->szAnnotation);
+
+	if (pszSymbolicAddr)
+	{
+		PDumpMakeStringValid(pszSymbolicAddr, OSStringLength(pszSymbolicAddr));
+	}
+
+
+	*puiNewOffset = uiPhysicalOffset & (IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR))-1);
+	*puiNextSymName = (IMG_DEVMEM_OFFSET_T) (((uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR))+1)
+			<< PMR_GetLog2Contiguity(psPMR));
+
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32MemspaceNameLen,
+                      IMG_CHAR *pszMemspaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                      IMG_DEVMEM_OFFSET_T *puiNextSymName
+)
+{
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	IMG_UINT32 ui32Remain;
+	IMG_BOOL bValid;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiLogicalOffset < psPMR->uiLogicalSize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           0,
+	                                           1,
+	                                           uiLogicalOffset,
+	                                           &uiPhysicalOffset,
+	                                           &ui32Remain,
+	                                           &bValid);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+	if (!bValid)
+	{
+		/* For sparse allocations, for a given logical address, there
+		 * may not be a physical memory backing, the virtual range can
+		 * still be valid.
+		 */
+		uiPhysicalOffset = uiLogicalOffset;
+	}
+
+	return _PMR_PDumpSymbolicAddrPhysical(psPMR,
+	                                      uiPhysicalOffset,
+	                                      ui32MemspaceNameLen,
+	                                      pszMemspaceName,
+	                                      ui32SymbolicAddrLen,
+	                                      pszSymbolicAddr,
+	                                      puiNewOffset,
+	                                      puiNextSymName);
+}
+
+/*!
+ * @brief Writes a WRW command to the script2 buffer, representing a
+ *        dword write to a physical allocation. Size is always
+ *        sizeof(IMG_UINT32).
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - offset
+ * @param ui32Value - value to write
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                       IMG_UINT32 ui32Value,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + sizeof(ui32Value) <= psPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+			<= uiPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the PMR */
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW32(PMR_DeviceNode(psPMR),
+	                       aszMemspaceName,
+	                       aszSymbolicName,
+	                       uiPDumpSymbolicOffset,
+	                       ui32Value,
+	                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a RDW followed by a WRW command to the pdump script to perform
+ *        an effective copy from memory to memory. Memory copied is of size
+ *        sizeof(IMG_UINT32)
+ *
+ * @param psDstPMR - PMR object representing allocation of destination
+ * @param uiDstLogicalOffset - destination offset
+ * @param psSrcPMR - PMR object representing allocation of source
+ * @param uiSrcLogicalOffset - source offset
+ * @param pszTmpVar - pdump temporary variable used during the copy
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem32(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	const IMG_UINT32 uiDstPMRPageSize = 1 << psDstPMR->uiLog2ContiguityGuarantee;
+	const IMG_UINT32 uiSrcPMRPageSize = 1 << psSrcPMR->uiLog2ContiguityGuarantee;
+
+	PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiSrcLogicalOffset & (uiSrcPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= uiSrcPMRPageSize));
+
+	PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiDstLogicalOffset & (uiDstPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= uiDstPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the source PMR */
+	eError = PMR_PDumpSymbolicAddr(psSrcPMR,
+	                               uiSrcLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Issue PDump read command */
+	eError = PDumpPMRRDW32MemToInternalVar(PMR_DeviceNode(psSrcPMR),
+	                                       pszTmpVar,
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+
+	eError = PMRLockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Get the symbolic address of the destination PMR */
+	eError = PMR_PDumpSymbolicAddr(psDstPMR,
+	                               uiDstLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW32InternalVarToMem(PMR_DeviceNode(psDstPMR),
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       pszTmpVar,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	eError = PMRUnlockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a WRW64 command to the script2 buffer, representing a
+ *        dword write to a physical allocation. Size is always
+ *        sizeof(IMG_UINT64).
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - offset
+ * @param ui64Value - value to write
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                       IMG_UINT64 ui64Value,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PMR_SIZE_T ui64PMRPageSize = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + sizeof(ui64Value) <= psPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (ui64PMRPageSize-1)) + sizeof(ui64Value))
+			<= ui64PMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the PMR */
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW64(PMR_DeviceNode(psPMR),
+	                       aszMemspaceName,
+	                       aszSymbolicName,
+	                       uiPDumpSymbolicOffset,
+	                       ui64Value,
+	                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a RDW64 followed by a WRW64 command to the pdump script to
+ *        perform an effective copy from memory to memory. Memory copied is of
+ *        size sizeof(IMG_UINT32)
+ *
+ * @param psDstPMR - PMR object representing allocation of destination
+ * @param uiDstLogicalOffset - destination offset
+ * @param psSrcPMR - PMR object representing allocation of source
+ * @param uiSrcLogicalOffset - source offset
+ * @param pszTmpVar - pdump temporary variable used during the copy
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem64(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	const PMR_SIZE_T ui64DstPMRPageSize = IMG_PAGE2BYTES64(psDstPMR->uiLog2ContiguityGuarantee);
+	const PMR_SIZE_T ui64SrcPMRPageSize = IMG_PAGE2BYTES64(psSrcPMR->uiLog2ContiguityGuarantee);
+
+	PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiSrcLogicalOffset & (ui64SrcPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= ui64SrcPMRPageSize));
+
+	PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiDstLogicalOffset & (ui64DstPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= ui64DstPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the source PMR */
+	eError = PMR_PDumpSymbolicAddr(psSrcPMR,
+	                               uiSrcLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Issue PDump read command */
+	eError = PDumpPMRRDW64MemToInternalVar(PMR_DeviceNode(psSrcPMR),
+	                                       pszTmpVar,
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+
+	eError = PMRLockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Get the symbolic address of the destination PMR */
+	eError = PMR_PDumpSymbolicAddr(psDstPMR,
+	                               uiDstLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW64InternalVarToMem(PMR_DeviceNode(psDstPMR),
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       pszTmpVar,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	eError = PMRUnlockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief PDumps the contents of the given allocation.
+ * If bZero is IMG_TRUE then the zero page in the parameter stream is used
+ * as the source of data, rather than the allocation's actual backing.
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - Offset to write at
+ * @param uiSize - Number of bytes to write
+ * @param uiPDumpFlags - PDump flags
+ * @param bZero - Use the PDump zero page as the source
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOutOffset;
+	IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName = 0;
+	const IMG_CHAR *pszParamStreamFileName;
+	PDUMP_FILEOFFSET_T uiParamStreamFileOffset;
+
+	/* required when !bZero */
+#define PMR_MAX_PDUMP_BUFSZ (1<<21)
+	IMG_CHAR aszParamStreamFilename[PDUMP_PARAM_MAX_FILE_NAME];
+	IMG_UINT8 *pcBuffer = NULL;
+	size_t ui64BufSz;
+	IMG_BOOL bValid;
+	IMG_DEVMEM_SIZE_T uiSizeRemain = uiSize;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(psDevNode))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Forcibly initialise the name to a 'NULL' 0-length string */
+	aszParamStreamFilename[0] = '\0';
+
+	PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize);
+
+	/* Check if pdump client is connected */
+	if (!PDumpCheckFlagsWrite(psDevNode,
+		                      PDUMP_FLAGS_CONTINUOUS))
+	{
+		/* Dumping of memory in Pdump buffer will be rejected for no client connected case.
+		 * So return early and save reading of data from PMR. */
+		return PVRSRV_OK;
+	}
+
+	/* Get the correct PDump stream file name */
+	if (bZero)
+	{
+		PDumpCommentWithFlags(psDevNode,
+		                      uiPDumpFlags,
+		                      "Zeroing allocation (" IMG_DEVMEM_SIZE_FMTSPEC " bytes)",
+		                      uiSize);
+
+		/* get the zero page information. it is constant for this function */
+		PDumpGetParameterZeroPageInfo(&uiParamStreamFileOffset,
+		                              &ui64BufSz,
+		                              &pszParamStreamFileName);
+	}
+	else
+	{
+
+		ui64BufSz = IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR));
+		PVR_ASSERT(IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR)) <= PMR_MAX_PDUMP_BUFSZ);
+
+		pcBuffer = OSAllocMem(ui64BufSz);
+
+		PVR_LOG_RETURN_IF_NOMEM(pcBuffer, "OSAllocMem");
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		pszParamStreamFileName = aszParamStreamFilename;
+	}
+
+	/* Loop over all touched symbolic addresses of the PMR and
+	 * emit LDBs to load the contents. */
+	while (uiCurrentOffset < (uiLogicalOffset + uiSize))
+	{
+		/* Get the correct symbolic name for the current offset */
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiCurrentOffset,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOutOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		PVR_ASSERT((uiNextSymName - uiCurrentOffset) <= ui64BufSz);
+
+		PMR_IsOffsetValid(psPMR,
+		                  0,
+		                  1,
+		                  uiCurrentOffset,
+		                  &bValid);
+
+		/* Either just LDB the zeros or read from the PMR and store that
+		 * in the pdump stream */
+		if (bValid)
+		{
+			size_t uiNumBytes;
+			IMG_BOOL bOk2Write = IMG_TRUE;
+
+			if (bZero)
+			{
+				uiNumBytes = MIN(uiSizeRemain, uiNextSymName - uiCurrentOffset);
+			}
+			else
+			{
+				IMG_DEVMEM_OFFSET_T uiReadOffset;
+				uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ?
+						uiLogicalOffset + uiSize - uiCurrentOffset :
+						uiNextSymName - uiCurrentOffset);
+
+				eError = PMR_ReadBytes(psPMR,
+				                       uiCurrentOffset,
+				                       pcBuffer,
+				                       uiReadOffset,
+				                       &uiNumBytes);
+				PVR_ASSERT(eError == PVRSRV_OK);
+
+				eError = PDumpWriteParameterBlob(psDevNode,
+				                          pcBuffer,
+				                          uiNumBytes,
+				                          uiPDumpFlags,
+				                          &aszParamStreamFilename[0],
+				                          sizeof(aszParamStreamFilename),
+				                          &uiParamStreamFileOffset);
+
+				if (eError == PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+				{
+					/* Write to parameter file prevented under the flags and
+					 * current state of the driver so skip further writes.
+					 */
+					eError = PVRSRV_OK;
+					bOk2Write = IMG_FALSE;	/* Do *NOT* write anything */
+				}
+				else if (eError != PVRSRV_OK)
+				{
+					PDUMP_ERROR(psDevNode,
+					            eError, "Failed to write PMR memory to parameter file");
+					bOk2Write = IMG_FALSE;	/* Do *NOT* write anything */
+				}
+			}
+
+			if (bOk2Write)
+			{
+				/* Emit the LDB command to the current symbolic address */
+				eError = PDumpPMRLDB(psDevNode,
+				                     aszMemspaceName,
+				                     aszSymbolicName,
+				                     uiOutOffset,
+				                     uiNumBytes,
+				                     pszParamStreamFileName,
+				                     uiParamStreamFileOffset,
+				                     uiPDumpFlags);
+			}
+			uiSizeRemain = uiSizeRemain - uiNumBytes;
+		}
+		uiCurrentOffset = uiNextSymName;
+	}
+
+	if (!bZero)
+	{
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		OSFreeMem(pcBuffer);
+	}
+
+	return PVRSRV_OK;
+}
+
+
+
+PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOutOffset;
+	IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName = 0;
+	IMG_UINT32 uiCurrentFileOffset = uiFileOffset;
+
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize);
+
+	while (uiCurrentOffset < (uiLogicalOffset + uiSize))
+	{
+		IMG_DEVMEM_OFFSET_T uiReadOffset;
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiCurrentOffset,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOutOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		PVR_ASSERT(uiNextSymName <= psPMR->uiLogicalSize);
+
+		uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ?
+				uiLogicalOffset + uiSize - uiCurrentOffset :
+				uiNextSymName - uiCurrentOffset);
+
+		eError = PDumpPMRSAB(PMR_DeviceNode(psPMR),
+		                     aszMemspaceName,
+		                     aszSymbolicName,
+		                     uiOutOffset,
+		                     uiReadOffset,
+		                     pszFilename,
+		                     uiCurrentFileOffset);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		uiCurrentFileOffset += uiNextSymName - uiCurrentOffset;
+		uiCurrentOffset = uiNextSymName;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+			<= uiPMRPageSize));
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+#define _MEMPOLL_DELAY		(1000)
+#define _MEMPOLL_COUNT		(2000000000 / _MEMPOLL_DELAY)
+
+	eError = PDumpPMRPOL(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     ui32Value,
+	                     ui32Mask,
+	                     eOperator,
+	                     _MEMPOLL_COUNT,
+	                     _MEMPOLL_DELAY,
+	                     uiPDumpFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+	           < uiPMRPageSize));
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = PDumpPMRPOL(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     ui32Value,
+	                     ui32Mask,
+	                     eOperator,
+	                     1,
+	                     1,
+	                     uiPDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiReadOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	eError = PDumpPMRCBP(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     uiWriteOffset,
+	                     uiPacketSize,
+	                     uiBufferSize);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void
+PDumpPMRChangeSparsePMR(PMR *psPMR,
+                        IMG_UINT32 uiBlockSize,
+                        IMG_UINT32 ui32AllocPageCount,
+                        IMG_UINT32 *pai32AllocIndices,
+                        IMG_UINT32 ui32FreePageCount,
+                        IMG_UINT32 *pai32FreeIndices,
+                        IMG_BOOL bInitialise,
+                        IMG_UINT8 ui8InitValue,
+                        IMG_HANDLE *phPDumpAllocInfoOut)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE *phPDumpAllocInfo = (IMG_HANDLE*) psPMR->hPDumpAllocHandle;
+
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 i, uiIndex;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	/* Remove pages from the PMR */
+	for (i = 0; i < ui32FreePageCount; i++)
+	{
+		uiIndex = pai32FreeIndices[i];
+
+		eError = PDumpFree(psDevNode,
+		                   phPDumpAllocInfo[uiIndex]);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		phPDumpAllocInfo[uiIndex] = NULL;
+	}
+
+	/* Add new pages to the PMR */
+	for (i = 0; i < ui32AllocPageCount; i++)
+	{
+		uiIndex = pai32AllocIndices[i];
+
+		PVR_ASSERT(phPDumpAllocInfo[uiIndex] == NULL);
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiIndex * uiBlockSize,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpMalloc(psDevNode,
+		                     aszMemspaceName,
+		                     aszSymbolicName,
+		                     uiBlockSize,
+		                     uiBlockSize,
+		                     bInitialise,
+		                     ui8InitValue,
+		                     &phPDumpAllocInfo[uiIndex],
+		                     PDUMP_NONE);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* (IMG_HANDLE) <- (IMG_HANDLE*) */
+	*phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo;
+}
+
+static void
+PDumpPMRFreePMR(PMR *psPMR,
+                IMG_DEVMEM_SIZE_T uiSize,
+                IMG_DEVMEM_ALIGN_T uiBlockSize,
+                IMG_UINT32 uiLog2Contiguity,
+                IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	/* (IMG_HANDLE*) <- (IMG_HANDLE) */
+	IMG_HANDLE *ahPDumpAllocHandleArray = (IMG_HANDLE*) hPDumpAllocationInfoHandle;
+
+	PDUMP_LOCK(PDUMP_FLAGS_NONE);
+
+	for (i = 0; i < psPMR->uiNumPDumpBlocks; i++)
+	{
+		if (ahPDumpAllocHandleArray[i] != NULL)
+		{
+			eError = PDumpFreeUnlocked(PMR_DeviceNode(psPMR),
+			                           ahPDumpAllocHandleArray[i]);
+			PVR_ASSERT(eError == PVRSRV_OK);
+			ahPDumpAllocHandleArray[i] = NULL;
+		}
+	}
+
+	PDUMP_UNLOCK(PDUMP_FLAGS_NONE);
+	OSFreeMem(ahPDumpAllocHandleArray);
+}
+
+static void
+PDumpPMRMallocPMR(PMR *psPMR,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT32 ui32ChunkSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 ui32NumLogicalChunks,
+                  IMG_UINT32 *puiMappingTable,
+                  IMG_UINT32 uiLog2Contiguity,
+                  IMG_BOOL bInitialise,
+                  IMG_UINT8 ui8InitValue,
+                  IMG_HANDLE *phPDumpAllocInfoOut,
+                  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE *phPDumpAllocInfo;
+
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiNumPhysBlocks;
+	IMG_UINT32 uiNumVirtBlocks;
+	IMG_UINT32 i, uiIndex;
+
+	if (PMR_IsSparse(psPMR))
+	{
+		uiNumPhysBlocks = (ui32ChunkSize * ui32NumPhysChunks) >> uiLog2Contiguity;
+		/* Make sure we did not cut off anything */
+		PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == (ui32ChunkSize * ui32NumPhysChunks));
+	}
+	else
+	{
+		uiNumPhysBlocks = uiSize >> uiLog2Contiguity;
+		/* Make sure we did not cut off anything */
+		PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == uiSize);
+	}
+
+	uiNumVirtBlocks = uiSize >> uiLog2Contiguity;
+	PVR_ASSERT(uiNumVirtBlocks << uiLog2Contiguity == uiSize);
+
+	psPMR->uiNumPDumpBlocks = uiNumVirtBlocks;
+
+	phPDumpAllocInfo = (IMG_HANDLE*) OSAllocZMem(uiNumVirtBlocks * sizeof(IMG_HANDLE));
+
+	PDUMP_LOCK(ui32PDumpFlags);
+
+	for (i = 0; i < uiNumPhysBlocks; i++)
+	{
+		uiIndex = PMR_IsSparse(psPMR) ? puiMappingTable[i] : i;
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiIndex * ui32ChunkSize,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpMallocUnlocked(PMR_DeviceNode(psPMR),
+		                             aszMemspaceName,
+		                             aszSymbolicName,
+		                             ui32ChunkSize,
+		                             ui32ChunkSize,
+		                             bInitialise,
+		                             ui8InitValue,
+		                             &phPDumpAllocInfo[uiIndex],
+		                             ui32PDumpFlags);
+		PVR_LOG_RETURN_VOID_IF_FALSE((eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE),
+		                             "PDumpPMRMalloc PDump capture bound to other device");
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	/* (IMG_HANDLE) <- (IMG_HANDLE*) */
+	*phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo;
+
+}
+#endif	/* PDUMP */
+
+
+void *PMRGetPrivateData(const PMR *psPMR,
+                        const PMR_IMPL_FUNCTAB *psFuncTab)
+{
+	return (psFuncTab == psPMR->psFuncTab) ? psPMR->pvFlavourData : NULL;
+}
+
+#define PMR_PM_WORD_SIZE 4
+
+PVRSRV_ERROR
+PMRWritePMPageList(/* Target PMR, offset, and length */
+		PMR *psPageListPMR,
+		IMG_DEVMEM_OFFSET_T uiTableOffset,
+		IMG_DEVMEM_SIZE_T  uiTableLength,
+		/* Referenced PMR, and "page" granularity */
+		PMR *psReferencePMR,
+		IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+		PMR_PAGELIST **ppsPageList)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEVMEM_SIZE_T uiWordSize;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	PMR_FLAGS_T uiFlags = psPageListPMR->uiFlags;
+	PMR_PAGELIST *psPageList;
+#if defined(PDUMP)
+	IMG_CHAR aszTableEntryMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszTableEntrySymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiTableEntryPDumpOffset;
+	IMG_CHAR aszPageMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszPageSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPagePDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+#endif
+#if !defined(NO_HARDWARE)
+	PMR_SIZE_T ui64PageListPageSize = IMG_PAGE2BYTES64(psPageListPMR->uiLog2ContiguityGuarantee);
+	IMG_UINT64 uiPageListPMRPage = 0;
+	IMG_UINT64 uiPrevPageListPMRPage = 0;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+	IMG_UINT32 *pui32DataPtr = NULL;
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *pasDevAddrPtr;
+	IMG_BOOL *pbPageIsValid;
+#endif
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPageListPMR->psPhysHeap);
+	IMG_BOOL bCPUCacheSnoop =
+		(PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		 psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY);
+
+	uiWordSize = PMR_PM_WORD_SIZE;
+
+	/* check we're being asked to write the same number of 4-byte units as there are pages */
+	uiNumPages = (IMG_UINT32)(psReferencePMR->uiLogicalSize >> uiLog2PageSize);
+
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psReferencePMR->uiLogicalSize)
+	{
+		/* Strictly speaking, it's possible to provoke this error in two ways:
+			(i) if it's not a whole multiple of the page size; or
+			(ii) if there are more than 4 billion pages.
+			The latter is unlikely. :) but the check is required in order to justify the cast.
+		 */
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, return_error);
+	}
+	uiWordSize = (IMG_UINT32)uiTableLength / uiNumPages;
+	if (uiNumPages * uiWordSize != uiTableLength)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, return_error);
+	}
+
+	/* Check for integer overflow */
+	PVR_GOTO_IF_INVALID_PARAM(uiTableOffset + uiTableLength > uiTableOffset, eError, return_error);
+	/* Check we're not being asked to write off the end of the PMR */
+	PVR_GOTO_IF_INVALID_PARAM(uiTableOffset + uiTableLength <= psPageListPMR->uiLogicalSize, eError, return_error);
+
+	/* the PMR into which we are writing must not be user CPU mappable: */
+	if (PVRSRV_CHECK_CPU_READABLE(uiFlags) || PVRSRV_CHECK_CPU_WRITEABLE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Masked flags = 0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         (PMR_FLAGS_T)(uiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE | PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE))));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Page list PMR allows CPU mapping (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	/* the PMR into which we are writing must not be user CPU cacheable: */
+	if (!bCPUCacheSnoop &&
+		(PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags) ||
+		 PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+		 PVRSRV_CHECK_CPU_CACHED(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Masked flags = 0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         (PMR_FLAGS_T)(uiFlags &  PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Page list PMR allows CPU caching (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	if (_PMRIsSparse(psPageListPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPageListPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	if (_PMRIsSparse(psReferencePMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psReferencePMR", eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	psPageList = OSAllocMem(sizeof(PMR_PAGELIST));
+	PVR_LOG_GOTO_IF_NOMEM(psPageList, eError, return_error);
+
+	psPageList->psReferencePMR = psReferencePMR;
+
+	/* Need to lock down the physical addresses of the reference PMR */
+	/* N.B.  This also checks that the requested "contiguity" is achievable */
+	eError = PMRLockSysPhysAddresses(psReferencePMR);
+	PVR_GOTO_IF_ERROR(eError, free_page_list);
+
+#if !defined(NO_HARDWARE)
+	if (uiNumPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		pasDevAddrPtr = OSAllocMem(uiNumPages * sizeof(IMG_DEV_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(pasDevAddrPtr, eError, unlock_phys_addrs);
+
+		pbPageIsValid = OSAllocMem(uiNumPages * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbPageIsValid, eError, free_devaddr_array);
+	}
+	else
+	{
+		pasDevAddrPtr = asDevPAddr;
+		pbPageIsValid = abValid;
+	}
+
+	eError = PMR_DevPhysAddr(psReferencePMR, uiLog2PageSize, uiNumPages, 0,
+	                         pasDevAddrPtr, pbPageIsValid, DEVICE_USE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", free_valid_array);
+#endif
+
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		IMG_DEVMEM_OFFSET_T uiPMROffset = uiTableOffset + (uiWordSize * uiPageIndex);
+
+#if defined(PDUMP)
+		eError = PMR_PDumpSymbolicAddr(psPageListPMR,
+		                               uiPMROffset,
+		                               sizeof(aszTableEntryMemspaceName),
+		                               &aszTableEntryMemspaceName[0],
+		                               sizeof(aszTableEntrySymbolicName),
+		                               &aszTableEntrySymbolicName[0],
+		                               &uiTableEntryPDumpOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PMR_PDumpSymbolicAddr(psReferencePMR,
+		                               IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                               sizeof(aszPageMemspaceName),
+		                               &aszPageMemspaceName[0],
+		                               sizeof(aszPageSymbolicName),
+		                               &aszPageSymbolicName[0],
+		                               &uiPagePDumpOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpWriteShiftedMaskedValue(PMR_DeviceNode(psReferencePMR),
+		                                      /* destination */
+		                                      aszTableEntryMemspaceName,
+		                                      aszTableEntrySymbolicName,
+		                                      uiTableEntryPDumpOffset,
+		                                      /* source */
+		                                      aszPageMemspaceName,
+		                                      aszPageSymbolicName,
+		                                      uiPagePDumpOffset,
+		                                      /* shift right */
+		                                      uiLog2PageSize,
+		                                      /* shift left */
+		                                      0,
+		                                      /* mask */
+		                                      0xffffffff,
+		                                      /* word size */
+		                                      uiWordSize,
+		                                      /* flags */
+		                                      PDUMP_FLAGS_CONTINUOUS);
+		PVR_ASSERT(eError == PVRSRV_OK);
+#else
+		PVR_UNREFERENCED_PARAMETER(uiPMROffset);
+#endif
+
+#if !defined(NO_HARDWARE)
+
+		/*
+			We check for sparse PMR's at function entry, but as we can,
+			check that every page is valid
+		 */
+		PVR_ASSERT(pbPageIsValid[uiPageIndex]);
+		PVR_ASSERT(pasDevAddrPtr[uiPageIndex].uiAddr != 0);
+		PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000LL) == 0);
+
+		uiPageListPMRPage = uiPMROffset >> psReferencePMR->uiLog2ContiguityGuarantee;
+
+		if ((pui32DataPtr == NULL) || (uiPageListPMRPage != uiPrevPageListPMRPage))
+		{
+			size_t uiMappingOffset = uiPMROffset & (~(ui64PageListPageSize - 1));
+			size_t uiMappedSize;
+
+			/* If we already had a page list mapped, we need to unmap it... */
+			if (pui32DataPtr != NULL)
+			{
+				PMRReleaseKernelMappingData(psPageListPMR, hPrivData);
+			}
+
+			eError = PMRAcquireKernelMappingData(psPageListPMR,
+			                                     uiMappingOffset,
+			                                     ui64PageListPageSize,
+			                                     &pvKernAddr,
+			                                     &uiMappedSize,
+			                                     &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Error mapping page list PMR page (%" IMG_UINT64_FMTSPEC ") into kernel (%d)",
+						uiPageListPMRPage, eError));
+				goto free_valid_array;
+			}
+
+			uiPrevPageListPMRPage = uiPageListPMRPage;
+			PVR_ASSERT(uiMappedSize >= ui64PageListPageSize);
+			PVR_ASSERT(pvKernAddr != NULL);
+
+			pui32DataPtr = IMG_OFFSET_ADDR(pvKernAddr, (uiPMROffset & (ui64PageListPageSize - 1)));
+		}
+
+		PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000LL) == 0);
+
+		/* Write the physical page index into the page list PMR */
+		*pui32DataPtr++ = TRUNCATE_64BITS_TO_32BITS(pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize);
+
+		/* Last page so unmap */
+		if (uiPageIndex == (uiNumPages - 1))
+		{
+			PMRReleaseKernelMappingData(psPageListPMR, hPrivData);
+		}
+#endif
+	}
+
+	OSWriteMemoryBarrier(NULL);
+
+#if !defined(NO_HARDWARE)
+	if (pasDevAddrPtr != asDevPAddr)
+	{
+		OSFreeMem(pbPageIsValid);
+		OSFreeMem(pasDevAddrPtr);
+	}
+#endif
+	*ppsPageList = psPageList;
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+#if !defined(NO_HARDWARE)
+
+free_valid_array:
+	if (pbPageIsValid != abValid)
+	{
+		OSFreeMem(pbPageIsValid);
+	}
+
+free_devaddr_array:
+	if (pasDevAddrPtr != asDevPAddr)
+	{
+		OSFreeMem(pasDevAddrPtr);
+	}
+
+unlock_phys_addrs:
+	PMRUnlockSysPhysAddresses(psReferencePMR);
+#endif
+
+free_page_list:
+	OSFreeMem(psPageList);
+
+return_error:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMRUnwritePMPageList(PMR_PAGELIST *psPageList)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PMRUnlockSysPhysAddresses(psPageList->psReferencePMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+	OSFreeMem(psPageList);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRZeroingPMR(PMR *psPMR,
+              IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize)
+{
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	PMR_SIZE_T ui64PageSize = IMG_PAGE2BYTES64(uiLog2PageSize);
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	size_t uiMappedSize;
+
+	PVR_ASSERT(psPMR);
+
+	/* Calculate number of pages in this PMR */
+	uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize);
+
+	/* Verify the logical Size is a multiple or the physical page size */
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR is not a multiple of %" IMG_UINT64_FMTSPEC,
+		         __func__,
+		         ui64PageSize));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, MultiPage_Error);
+	}
+
+	if (_PMRIsSparse(psPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, Sparse_Error);
+	}
+
+	/* Scan through all pages of the PMR */
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		/* map the physical page (for a given PMR offset) into kernel space */
+		eError = PMRAcquireKernelMappingData(psPMR,
+		                                     IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                                     ui64PageSize,
+		                                     &pvKernAddr,
+		                                     &uiMappedSize,
+		                                     &hPrivData);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", AcquireKernelMapping_Error);
+
+		/* ensure the mapped page size is the same as the physical page size */
+		if (uiMappedSize != ui64PageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Physical Page size = 0x%08" IMG_UINT64_FMTSPECx ", Size of Mapping = 0x%016" IMG_UINT64_FMTSPECx,
+			         __func__,
+			         ui64PageSize,
+			         (IMG_UINT64)uiMappedSize));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, MappingSize_Error);
+		}
+
+		/* Use the conservative 'DeviceMemSet' here because we can't know
+		 * if this PMR will be mapped cached.
+		 */
+		OSDeviceMemSet(pvKernAddr, 0, ui64PageSize);
+
+		/* release mapping */
+		PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: Zeroing PMR %p done (num pages %u, page size %" IMG_UINT64_FMTSPEC ")",
+	         __func__,
+	         psPMR,
+	         uiNumPages,
+	         ui64PageSize));
+
+	return PVRSRV_OK;
+
+
+	/* Error handling */
+
+MappingSize_Error:
+	PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+AcquireKernelMapping_Error:
+Sparse_Error:
+MultiPage_Error:
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRDumpPageList(PMR *psPMR,
+                IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize)
+{
+	IMG_DEV_PHYADDR sDevAddrPtr;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	IMG_BOOL bPageIsValid;
+	IMG_UINT32 ui32Col = 16;
+	IMG_UINT32 ui32SizePerCol = 11;
+	IMG_UINT32 ui32ByteCount = 0;
+	IMG_CHAR pszBuffer[16 /* ui32Col */ * 11 /* ui32SizePerCol */ + 1];
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Get number of pages */
+	uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize);
+
+	/* Verify the logical Size is a multiple or the physical page size */
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not a multiple of %" IMG_UINT64_FMTSPEC,
+		        __func__, (IMG_UINT64) (1ULL << uiLog2PageSize)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, MultiPage_Error);
+	}
+
+	if (_PMRIsSparse(psPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, Sparse_Error);
+	}
+
+	PVR_LOG(("    PMR %p, Number of pages %u, Log2PageSize %d", psPMR, uiNumPages, uiLog2PageSize));
+
+	/* Print the address of the physical pages */
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		/* Get Device physical Address */
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2PageSize,
+		                         1,
+		                         IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                         &sDevAddrPtr,
+		                         &bPageIsValid,
+		                         DEVICE_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PMR %p failed to get DevPhysAddr with error %u",
+					__func__,
+					psPMR,
+					eError));
+			goto DevPhysAddr_Error;
+		}
+
+		ui32ByteCount += OSSNPrintf(pszBuffer + ui32ByteCount, ui32SizePerCol + 1, "%08x ", (IMG_UINT32)(sDevAddrPtr.uiAddr >> uiLog2PageSize));
+		PVR_ASSERT(ui32ByteCount < ui32Col * ui32SizePerCol);
+
+		if (uiPageIndex % ui32Col == ui32Col-1)
+		{
+			PVR_LOG(("      Phys Page: %s", pszBuffer));
+			ui32ByteCount = 0;
+		}
+	}
+	if (ui32ByteCount > 0)
+	{
+		PVR_LOG(("      Phys Page: %s", pszBuffer));
+	}
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+DevPhysAddr_Error:
+Sparse_Error:
+MultiPage_Error:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+DEFINE_PMR_ENV_GET_DATA(psPMR, sEnvData)
+
+PVRSRV_ERROR
+PMRInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	/* Singleton PMR context already initialised */
+	if (_gsSingletonPMRContext.bModuleInitialised)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+	eError = OSLockCreate(&_gsSingletonPMRContext.hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", out);
+
+	_gsSingletonPMRContext.uiNextSerialNum = 1;
+
+	_gsSingletonPMRContext.uiNextKey = 0x8300f001 * (uintptr_t)&_gsSingletonPMRContext;
+
+	_gsSingletonPMRContext.bModuleInitialised = IMG_TRUE;
+
+	OSAtomicWrite(&_gsSingletonPMRContext.uiNumLivePMRs, 0);
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	eError = MMapStatsInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMapStatsInit", destroy_context_lock);
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+destroy_context_lock:
+	OSLockDestroy(_gsSingletonPMRContext.hLock);
+	_gsSingletonPMRContext.hLock = NULL;
+#endif
+out:
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRDeInit(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		goto out;
+	}
+
+	/* Singleton PMR context is not initialised */
+	if (!_gsSingletonPMRContext.bModuleInitialised)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	MMapStatsDeInit();
+#endif
+
+	if (OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error: %d live PMRs remain",
+				__func__,
+				OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs)));
+		PVR_DPF((PVR_DBG_ERROR, "%s: This is an unrecoverable error; a subsequent crash is inevitable",
+				__func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+	if (_gsSingletonPMRContext.hLock != NULL)
+	{
+		OSLockDestroy(_gsSingletonPMRContext.hLock);
+	}
+
+	_gsSingletonPMRContext.bModuleInitialised = IMG_FALSE;
+
+out:
+	PVR_ASSERT(eError == PVRSRV_OK);
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+PVRSRV_ERROR
+PMRInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDeviceNode->hPMRZombieListLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	dllist_init(&psDeviceNode->sPMRZombieList);
+	psDeviceNode->uiPMRZombieCount = 0;
+	psDeviceNode->uiPMRZombieCountInCleanup = 0;
+
+	return PVRSRV_OK;
+}
+
+void
+PMRFreeZombies(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	DECLARE_DLLIST(sZombieList);
+	DLLIST_NODE *psThis, *psNext;
+	IMG_INT32 uiZombieCount;
+
+	_ZombieListLock(psDeviceNode);
+	/* Move the zombie list to a local copy. The original list will become
+	 * an empty list. This will allow us to process the list without holding
+	 * the list lock. */
+	dllist_replace_head(&psDeviceNode->sPMRZombieList, &sZombieList);
+	uiZombieCount = psDeviceNode->uiPMRZombieCount;
+	psDeviceNode->uiPMRZombieCount = 0;
+	_ZombieListUnlock(psDeviceNode);
+
+	dllist_foreach_node(&sZombieList, psThis, psNext)
+	{
+		dllist_remove_node(psThis);
+		switch (PMR_GetZombieTypeFromNode(psThis))
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_PAGES:
+			{
+				PVRSRV_ERROR eError;
+				PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psThis);
+
+				eError = psZombiePages->pfnFactoryFreeZombies(psZombiePages->pvFactoryPages);
+				if (eError != PVRSRV_OK)
+				{
+					/* In case of failure to free zombie pages, remove it from
+					* the sZombieList and add back to the original list. */
+					_ZombieListLock(psDeviceNode);
+					dllist_add_to_tail(&psDeviceNode->sPMRZombieList, psThis);
+					psDeviceNode->uiPMRZombieCount++;
+					_ZombieListUnlock(psDeviceNode);
+
+					PVR_DPF((PVR_DBG_ERROR, "Cannot free zombie pages!"));
+					continue;
+				}
+
+				OSFreeMem(psZombiePages);
+				break;
+			}
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_DEVICE_IMPORT:
+			{
+				PMR_DEVICE_IMPORT *psImport = PMR_GetDeviceImportFromNode(psThis);
+				_DeviceImportFreeImportZombie(psImport);
+				break;
+			}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+			case PMR_ZOMBIE_TYPE_PMR:
+			{
+				PMR *psPMR = PMR_GetPMRFromNode(psThis);
+				const PMR_IMPL_FUNCTAB *psFuncTable = psPMR->psFuncTab;
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+				/* The PMR cannot be freed as other devices are
+				 * still waiting for the cache flush. */
+				PMRLockPMR(psPMR);
+				if (_DeviceImportBitmapGet(psPMR) != 0)
+				{
+					PDLLIST_NODE psNodeImport;
+					PMR_DEVICE_IMPORT *psImport;
+					/* Transfer the ownership to a different
+					 * device queue that has not been processed yet.
+					 * There will be a PMR_DEVICE_IMPORT on the same
+					 * queue, however, this doesn't have any knock on affects as
+					 * it will be freed before the PMR is reached again. */
+					psNodeImport = dllist_get_next_node(&psPMR->sXDeviceImports);
+					PVR_ASSERT(psNodeImport);
+					psImport = IMG_CONTAINER_OF(psNodeImport, PMR_DEVICE_IMPORT, sNext);
+					_ZombieListLock(psImport->psDevNode);
+					dllist_add_to_tail(&psImport->psDevNode->sPMRZombieList, psThis);
+					psImport->psDevNode->uiPMRZombieCount++;
+					_ZombieListUnlock(psImport->psDevNode);
+					PMRUnlockPMR(psPMR);
+					break;
+				}
+				PMRUnlockPMR(psPMR);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+				_FactoryLock(psFuncTable);
+				_PMRDestroy(psPMR);
+				_FactoryUnlock(psFuncTable);
+				break;
+			}
+		}
+		uiZombieCount--;
+	}
+
+	PVR_ASSERT(uiZombieCount == 0);
+}
+
+void
+PMRDumpZombies(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	_ZombieListLock(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR, "Items in zombie list: %u",
+	        psDeviceNode->uiPMRZombieCount));
+
+	dllist_foreach_node(&psDeviceNode->sPMRZombieList, psThis, psNext)
+	{
+		switch (PMR_GetZombieTypeFromNode(psThis))
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_PAGES:
+			{
+				PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psThis);
+				PVR_DPF((PVR_DBG_ERROR, "Zombie Pages = %p", psZombiePages));
+				break;
+			}
+#endif
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+			case PMR_ZOMBIE_TYPE_DEVICE_IMPORT:
+			{
+				PMR_DEVICE_IMPORT* psImport = PMR_GetDeviceImportFromNode(psThis);
+				PVR_DPF((PVR_DBG_ERROR, "Device Import = %p, DevID = %u, PMR = %px (%s)",
+				         psImport,
+				         psImport->psDevNode->sDevId.ui32InternalID,
+				         psImport->psParent,
+				         PMR_GetAnnotation(psImport->psParent)));
+				break;
+			}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+
+			case PMR_ZOMBIE_TYPE_PMR:
+			{
+				PMR *psPMR = PMR_GetPMRFromNode(psThis);
+				PVR_DPF((PVR_DBG_ERROR, "PMR = %px, Flavour = %s, Annotation: %s",
+						psPMR, PMR_GetTypeStr(psPMR), PMR_GetAnnotation(psPMR)));
+				break;
+			}
+		}
+	}
+
+	_ZombieListUnlock(psDeviceNode);
+}
+
+void
+PMRDeInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PMRFreeZombies(psDeviceNode);
+
+	OSLockDestroy(psDeviceNode->hPMRZombieListLock);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+IMG_UINT64
+PMR_ImportedDevicesMask(const PMR* psPMR)
+{
+	return _DeviceImportBitmapGet(psPMR) & ((IMG_UINT64_C(1) << PVRSRV_MAX_DEVICES) - 1);
+}
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+PVRSRV_ERROR
+PMR_RegisterDeviceImport(PMR* psPMR, PPVRSRV_DEVICE_NODE psDevNode)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVR_ASSERT(!PMR_IsZombie(psPMR));
+#endif /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	if (PMR_DeviceNode(psPMR) != psDevNode)
+	{
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE)
+		PVRSRV_ERROR eError = _DeviceImportRegister(psPMR, psDevNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_DeviceImportRegister");
+#else
+		/* `_DeviceImportRegister` already sets the bitmap.
+		 * This is still needs to be set without device import zombie support
+		 * for debugging information, i.e. the RI. */
+		_DeviceImportBitmapSet(psPMR, psDevNode);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) */
+	}
+	/* else: We explicitly don't add the PMR's dev node to the list because
+	 *       this bitmask lets us know if the PMR is cross device. It's not
+	 *       an error to register with the original dev node, as the user is
+	 *       declaring "The PMR is using `psDevNode`", not that it's a new
+	 *       devnode. */
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/power.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/power.c
new file mode 100644
index 000000000000..43647aa9c78c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/power.c
@@ -0,0 +1,1525 @@
+/*************************************************************************/ /*!
+@File           power.c
+@Title          Power management functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for power management functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if !defined(__linux__)
+#include <errno.h>
+#endif /* #if !defined(__linux__) */
+
+#include "pdump_km.h"
+#include "allocmem.h"
+#include "osfunc.h"
+
+#include "lock.h"
+#include "pvrsrv.h"
+#include "power.h"
+#include "pvr_debug.h"
+#include "htbserver.h"
+#include "di_server.h"
+
+#define OS_POWERLOCK_TIMEOUT_MS		(5000UL)
+#define OS_POWERLOCK_TIMEOUT_US		(OS_POWERLOCK_TIMEOUT_MS * 1000UL)
+#define OS_POWERLOCK_RETRY_COUNT	(50)
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+typedef struct _EXTRA_POWER_STATS_
+{
+	IMG_UINT64	ui64PreClockSpeedChangeDuration;
+	IMG_UINT64	ui64BetweenPreEndingAndPostStartingDuration;
+	IMG_UINT64	ui64PostClockSpeedChangeDuration;
+} EXTRA_POWER_STATS;
+
+/* For the power timing stats we need 16 variables to store all the
+ * combinations of forced/not forced, power-on/power-off, pre-power/post-power
+ * and device/system statistics
+ */
+#define NUM_POWER_STATS        (16)
+#define NUM_EXTRA_POWER_STATS	10
+
+typedef struct PVRSRV_POWER_STATS_TAG
+{
+	EXTRA_POWER_STATS               asClockSpeedChanges[NUM_EXTRA_POWER_STATS];
+	IMG_UINT64                      ui64PreClockSpeedChangeMark;
+	IMG_UINT64                      ui64FirmwareIdleDuration;
+	IMG_UINT32                      aui32PowerTimingStats[NUM_POWER_STATS];
+	IMG_UINT32                      ui32ClockSpeedIndexStart;
+	IMG_UINT32                      ui32ClockSpeedIndexEnd;
+	IMG_UINT32                      ui32FirmwareStartTimestamp;
+} PVRSRV_POWER_STATS;
+#endif
+
+struct _PVRSRV_POWER_DEV_TAG_
+{
+	PFN_PRE_POWER					pfnDevicePrePower;
+	PFN_POST_POWER					pfnDevicePostPower;
+	PFN_SYS_PRE_POWER				pfnSystemPrePower;
+	PFN_SYS_POST_POWER				pfnSystemPostPower;
+	PFN_PRE_CLOCKSPEED_CHANGE		pfnPreClockSpeedChange;
+	PFN_POST_CLOCKSPEED_CHANGE		pfnPostClockSpeedChange;
+	PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest;
+	PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest;
+	PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC	pfnForcedIdleCancelRequestAsync;
+	PFN_GPU_UNITS_POWER_CHANGE		pfnGPUUnitsPowerChange;
+	IMG_HANDLE						hSysData;
+	IMG_HANDLE						psDevNode;
+	PVRSRV_DEV_POWER_STATE			eDefaultPowerState;
+	ATOMIC_T						eCurrentPowerState;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_POWER_STATS				sPowerStats;
+#endif
+	/* Counts number of device idle references that have been taken,
+	 * must be protected by power lock.
+	 */
+	IMG_INT32                       iIdleReqInFlight;
+};
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+/*
+ * Power statistics related definitions
+ */
+
+/* For the mean time, use an exponentially weighted moving average with a
+ * 1/4 weighting for the new measurement.
+ */
+#define MEAN_TIME(A, B)     ( ((3*(A))/4) + ((1 * (B))/4) )
+
+#define UPDATE_TIME(time, newtime) \
+	((time) > 0 ? MEAN_TIME((time), (newtime)) : (newtime))
+
+/* Enum to be used as input to GET_POWER_STAT_INDEX */
+typedef enum
+{
+	DEVICE     = 0,
+	SYSTEM     = 1,
+	POST_POWER = 0,
+	PRE_POWER  = 2,
+	POWER_OFF  = 0,
+	POWER_ON   = 4,
+	NOT_FORCED = 0,
+	FORCED     = 8,
+} PVRSRV_POWER_STAT_TYPE;
+
+/* Macro used to access one of the power timing statistics inside an array */
+#define GET_POWER_STAT_INDEX(forced,powon,prepow,system) \
+	((forced) + (powon) + (prepow) + (system))
+
+void PVRSRVSetFirmwareStartTime(PVRSRV_POWER_DEV *psPowerDevice,
+						  IMG_UINT32 ui32Time)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->ui32FirmwareStartTimestamp =
+		UPDATE_TIME(psPowerStats->ui32FirmwareStartTimestamp,
+					ui32Time);
+}
+
+void PVRSRVSetFirmwareHandshakeIdleTime(PVRSRV_POWER_DEV *psPowerDevice,
+								  IMG_UINT64 ui64Duration)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->ui64FirmwareIdleDuration =
+		UPDATE_TIME(psPowerStats->ui64FirmwareIdleDuration,
+					ui64Duration);
+}
+
+static void _InsertPowerTimeStatistic(PVRSRV_POWER_DEV *psPowerDevice,
+									  IMG_UINT64 ui64SysStartTime, IMG_UINT64 ui64SysEndTime,
+									  IMG_UINT64 ui64DevStartTime, IMG_UINT64 ui64DevEndTime,
+									  IMG_BOOL bForced, IMG_BOOL bPowerOn, IMG_BOOL bPrePower)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+	IMG_UINT32 *pui32Stat;
+	IMG_UINT64 ui64DeviceDiff = ui64DevEndTime - ui64DevStartTime;
+	IMG_UINT64 ui64SystemDiff = ui64SysEndTime - ui64SysStartTime;
+	IMG_UINT32 ui32Index;
+
+	if (bPrePower)
+	{
+		HTBLOGK(HTB_SF_MAIN_PRE_POWER, bPowerOn, ui64DeviceDiff, ui64SystemDiff);
+	}
+	else
+	{
+		HTBLOGK(HTB_SF_MAIN_POST_POWER, bPowerOn, ui64SystemDiff, ui64DeviceDiff);
+	}
+
+	ui32Index = GET_POWER_STAT_INDEX(bForced ? FORCED : NOT_FORCED,
+	                                 bPowerOn ? POWER_ON : POWER_OFF,
+	                                 bPrePower ? PRE_POWER : POST_POWER,
+	                                 DEVICE);
+	pui32Stat = &psPowerStats->aui32PowerTimingStats[ui32Index];
+	*pui32Stat = UPDATE_TIME(*pui32Stat, ui64DeviceDiff);
+
+	ui32Index = GET_POWER_STAT_INDEX(bForced ? FORCED : NOT_FORCED,
+	                                 bPowerOn ? POWER_ON : POWER_OFF,
+	                                 bPrePower ? PRE_POWER : POST_POWER,
+	                                 SYSTEM);
+	pui32Stat = &psPowerStats->aui32PowerTimingStats[ui32Index];
+	*pui32Stat = UPDATE_TIME(*pui32Stat, ui64SystemDiff);
+}
+
+static void _InsertPowerTimeStatisticExtraPre(PVRSRV_POWER_DEV *psPowerDevice,
+											  IMG_UINT64 ui64StartTimer,
+											  IMG_UINT64 ui64Stoptimer)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64PreClockSpeedChangeDuration =
+		ui64Stoptimer - ui64StartTimer;
+
+	psPowerStats->ui64PreClockSpeedChangeMark = OSClockus();
+}
+
+static void _InsertPowerTimeStatisticExtraPost(PVRSRV_POWER_DEV *psPowerDevice,
+											   IMG_UINT64 ui64StartTimer,
+											   IMG_UINT64 ui64StopTimer)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+	IMG_UINT64 ui64Duration = ui64StartTimer - psPowerStats->ui64PreClockSpeedChangeMark;
+
+	PVR_ASSERT(psPowerStats->ui64PreClockSpeedChangeMark > 0);
+
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64BetweenPreEndingAndPostStartingDuration = ui64Duration;
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64PostClockSpeedChangeDuration = ui64StopTimer - ui64StartTimer;
+
+	psPowerStats->ui32ClockSpeedIndexEnd = (psPowerStats->ui32ClockSpeedIndexEnd + 1) % NUM_EXTRA_POWER_STATS;
+
+	if (psPowerStats->ui32ClockSpeedIndexEnd == psPowerStats->ui32ClockSpeedIndexStart)
+	{
+		psPowerStats->ui32ClockSpeedIndexStart = (psPowerStats->ui32ClockSpeedIndexStart + 1) % NUM_EXTRA_POWER_STATS;
+	}
+
+	psPowerStats->ui64PreClockSpeedChangeMark = 0;
+}
+
+static INLINE void _PowerStatsPrintGroup(IMG_UINT32 *pui32Stats,
+										 OSDI_IMPL_ENTRY *psEntry,
+										 PVRSRV_POWER_STAT_TYPE eForced,
+										 PVRSRV_POWER_STAT_TYPE ePowerOn)
+{
+	IMG_UINT32 ui32Index;
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, PRE_POWER, DEVICE);
+	DIPrintf(psEntry, "  Pre-Device:  %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, PRE_POWER, SYSTEM);
+	DIPrintf(psEntry, "  Pre-System:  %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, POST_POWER, SYSTEM);
+	DIPrintf(psEntry, "  Post-System: %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, POST_POWER, DEVICE);
+	DIPrintf(psEntry, "  Post-Device: %9u\n", pui32Stats[ui32Index]);
+}
+
+int PVRSRVPowerStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	PVRSRV_POWER_STATS *psPowerStats;
+	IMG_UINT32 *pui32Stats;
+	IMG_UINT32 ui32Idx;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psPowerDevice == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "reading power timing stats!"));
+		return -EIO;
+	}
+
+	psPowerStats = &psPowerDevice->sPowerStats;
+
+	pui32Stats = &psPowerStats->aui32PowerTimingStats[0];
+
+	DIPrintf(psEntry, "Forced Power-on Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, FORCED, POWER_ON);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Forced Power-off Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, FORCED, POWER_OFF);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Not Forced Power-on Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, NOT_FORCED, POWER_ON);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Not Forced Power-off Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, NOT_FORCED, POWER_OFF);
+	DIPrintf(psEntry, "\n");
+
+
+	DIPrintf(psEntry, "FW bootup time (timer ticks): %u\n", psPowerStats->ui32FirmwareStartTimestamp);
+	DIPrintf(psEntry, "Host Acknowledge Time for FW Idle Signal (timer ticks): %u\n", (IMG_UINT32)(psPowerStats->ui64FirmwareIdleDuration));
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Last %d Clock Speed Change Timers (nanoseconds):\n", NUM_EXTRA_POWER_STATS);
+	DIPrintf(psEntry, "Prepare DVFS\tDVFS Change\tPost DVFS\n");
+
+	for (ui32Idx = psPowerStats->ui32ClockSpeedIndexStart;
+		 ui32Idx != psPowerStats->ui32ClockSpeedIndexEnd;
+		 ui32Idx = (ui32Idx + 1) % NUM_EXTRA_POWER_STATS)
+	{
+		DIPrintf(psEntry, "%12" IMG_UINT64_FMTSPEC "\t%11" IMG_UINT64_FMTSPEC "\t%9" IMG_UINT64_FMTSPEC "\n",
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64PreClockSpeedChangeDuration,
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64BetweenPreEndingAndPostStartingDuration,
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64PostClockSpeedChangeDuration);
+	}
+
+	return 0;
+}
+
+#else /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+static void _InsertPowerTimeStatistic(PVRSRV_POWER_DEV *psPowerDevice,
+									  IMG_UINT64 ui64SysStartTime, IMG_UINT64 ui64SysEndTime,
+									  IMG_UINT64 ui64DevStartTime, IMG_UINT64 ui64DevEndTime,
+									  IMG_BOOL bForced, IMG_BOOL bPowerOn, IMG_BOOL bPrePower)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64SysStartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64SysEndTime);
+	PVR_UNREFERENCED_PARAMETER(ui64DevStartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64DevEndTime);
+	PVR_UNREFERENCED_PARAMETER(bForced);
+	PVR_UNREFERENCED_PARAMETER(bPowerOn);
+	PVR_UNREFERENCED_PARAMETER(bPrePower);
+}
+
+static void _InsertPowerTimeStatisticExtraPre(PVRSRV_POWER_DEV *psPowerDevice,
+											  IMG_UINT64 ui64StartTimer,
+											  IMG_UINT64 ui64Stoptimer)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTimer);
+	PVR_UNREFERENCED_PARAMETER(ui64Stoptimer);
+}
+
+static void _InsertPowerTimeStatisticExtraPost(PVRSRV_POWER_DEV *psPowerDevice,
+											   IMG_UINT64 ui64StartTimer,
+											   IMG_UINT64 ui64StopTimer)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTimer);
+	PVR_UNREFERENCED_PARAMETER(ui64StopTimer);
+}
+#endif
+
+const char *PVRSRVSysPowerStateToString(PVRSRV_SYS_POWER_STATE eState)
+{
+	switch (eState) {
+#define X(name, _) \
+		case PVRSRV_SYS_POWER_STATE_##name: \
+			return #name;
+		_PVRSRV_SYS_POWER_STATES
+#undef X
+		default:
+			return "unknown";
+	}
+}
+
+const char *PVRSRVDevPowerStateToString(PVRSRV_DEV_POWER_STATE eState)
+{
+	switch (eState) {
+		case PVRSRV_DEV_POWER_STATE_DEFAULT:
+			return "DEFAULT";
+		case PVRSRV_DEV_POWER_STATE_OFF:
+			return "OFF";
+		case PVRSRV_DEV_POWER_STATE_ON:
+			return "ON";
+		default:
+			return "unknown";
+	}
+}
+
+/*!
+  Typedef for a pointer to a function that will be called for re-acquiring
+  device powerlock after releasing it temporarily for some timeout period
+  in function PVRSRVDeviceIdleRequestKM
+ */
+#if defined(DEBUG)
+typedef PVRSRV_ERROR (*PFN_POWER_LOCK_ACQUIRE) (PPVRSRV_DEVICE_NODE psDevNode,
+                                                const char *pszFile, const unsigned int ui32LineNum);
+#else
+typedef PVRSRV_ERROR (*PFN_POWER_LOCK_ACQUIRE) (PPVRSRV_DEVICE_NODE psDevNode);
+#endif
+
+static inline IMG_UINT64 PVRSRVProcessStatsGetTimeNs(void)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	return OSClockns64();
+#else
+	return 0;
+#endif
+}
+
+static inline IMG_UINT64 PVRSRVProcessStatsGetTimeUs(void)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	return OSClockus();
+#else
+	return 0;
+#endif
+}
+
+/*!
+******************************************************************************
+
+ @Function	_IsSystemStatePowered
+
+ @Description	Tests whether a given system state represents powered-up.
+
+ @Input		eSystemPowerState : a system power state
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+static IMG_BOOL _IsSystemStatePowered(PVRSRV_SYS_POWER_STATE eSystemPowerState)
+{
+	return (eSystemPowerState == PVRSRV_SYS_POWER_STATE_ON);
+}
+
+/* We don't expect PID=0 to acquire device power-lock */
+#define PWR_LOCK_OWNER_PID_CLR_VAL 0
+
+PVRSRV_ERROR PVRSRVPowerLockInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDeviceNode->hPowerLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	return PVRSRV_OK;
+}
+
+void PVRSRVPowerLockDeInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	OSLockDestroy(psDeviceNode->hPowerLock);
+}
+
+IMG_BOOL PVRSRVPwrLockIsLockedByMe(PCPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	return OSLockIsLocked(psDeviceNode->hPowerLock) &&
+	       OSGetCurrentClientProcessIDKM() == psDeviceNode->uiPwrLockOwnerPID;
+}
+
+
+#if defined(DEBUG)
+static void _PVRSRVPowerLockOwner(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const char *pszFile, const unsigned int ui32LineNum)
+{
+	psDeviceNode->sPowerLockOwner.pszFile = pszFile;
+	psDeviceNode->sPowerLockOwner.ui32LineNum = ui32LineNum;
+	psDeviceNode->sPowerLockOwner.ui64Timestamp = OSClockus64();
+}
+#endif
+
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                   const char *pszFile, const unsigned int ui32LineNum)
+#else
+PVRSRV_ERROR PVRSRVPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+#endif
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+#if defined(DEBUG)
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+#endif
+
+	return PVRSRV_OK;
+}
+
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerTryLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      const char *pszFile, const unsigned int ui32LineNum)
+#else
+PVRSRV_ERROR PVRSRVPowerTryLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+#endif
+{
+	if (!(OSTryLockAcquire(psDeviceNode->hPowerLock)))
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+#if defined(DEBUG)
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVPowerTryLockWaitForTimeout(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Take power lock, retry if it's in use in another task. */
+	LOOP_UNTIL_TIMEOUT_US(OS_POWERLOCK_TIMEOUT_US)
+	{
+		eError = PVRSRVPowerTryLock(psDeviceNode);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			return eError;
+		}
+
+		OSSleepms(OS_POWERLOCK_TIMEOUT_MS / OS_POWERLOCK_RETRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError == PVRSRV_ERROR_RETRY)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function     _PVRSRVForcedPowerLock
+
+ @Description  Obtain the mutex for power transitions regardless of system
+               power state
+
+ @Return       Always returns PVRSRV_OK. Function prototype required same as
+               PFN_POWER_LOCK_ACQUIRE
+
+******************************************************************************/
+#if defined(DEBUG)
+#define _PVRSRVForcedPowerLock(DEV_NODE)	\
+	_PVRSRVForcedPowerLock_Debug(DEV_NODE, __FILE__, __LINE__)
+
+static PVRSRV_ERROR _PVRSRVForcedPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                 const char *pszFile, const unsigned int ui32LineNum)
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+
+	return PVRSRV_OK;
+}
+#else
+static PVRSRV_ERROR _PVRSRVForcedPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+}
+#endif
+
+void PVRSRVPowerUnlock(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDeviceNode));
+
+	/* Reset uiPwrLockOwnerPID before releasing lock */
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	OSLockRelease(psDeviceNode->hPowerLock);
+}
+
+IMG_BOOL PVRSRVDeviceIsDefaultStateOFF(PVRSRV_POWER_DEV *psPowerDevice)
+{
+	return (psPowerDevice->eDefaultPowerState == PVRSRV_DEV_POWER_STATE_OFF);
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceDefaultPowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+					PVRSRV_DEV_POWER_STATE eNewPowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psPowerDevice->eDefaultPowerState = eNewPowerState;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceCurrentPowerState(PVRSRV_POWER_DEV *psPowerDevice,
+					PVRSRV_DEV_POWER_STATE eNewPowerState)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_DEV_POWER_STATE eOldPowerState;
+#endif
+
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if !defined(SUPPORT_PMR_DEFERRED_FREE)
+	OSAtomicWrite(&psPowerDevice->eCurrentPowerState, eNewPowerState);
+#else
+	eOldPowerState = OSAtomicExchange(&psPowerDevice->eCurrentPowerState,
+	                                  eNewPowerState);
+
+	psDeviceNode = psPowerDevice->psDevNode;
+	PVR_ASSERT(psDeviceNode);
+
+	if (eNewPowerState == PVRSRV_DEV_POWER_STATE_OFF &&
+	    eNewPowerState != eOldPowerState)
+	{
+		psDeviceNode->uiPowerOffCounter = psDeviceNode->uiPowerOffCounterNext;
+
+		/* It's not really important to know if any zombies were queued. */
+		(void) PMRQueueZombiesForCleanup(psDeviceNode);
+
+		psDeviceNode->uiPowerOffCounterNext++;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ @Input       pfnPowerLockAcquire  : Function to re-acquire power-lock in-case
+                                     it was necessary to release it.
+*/
+static PVRSRV_ERROR _PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF    pfnIsDefaultStateOff,
+					IMG_BOOL                            bDeviceOffPermitted,
+					PFN_POWER_LOCK_ACQUIRE              pfnPowerLockAcquire)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+	PVRSRV_ERROR eError;
+
+	/* if pfnIsDefaultStateOff not provided or pfnIsDefaultStateOff(psPowerDev)
+	 * is true (which means that the default state is OFF) then force idle. */
+	if ((psPowerDev && psPowerDev->pfnForcedIdleRequest) &&
+	    (pfnIsDefaultStateOff == NULL || pfnIsDefaultStateOff(psPowerDev)))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = psPowerDev->pfnForcedIdleRequest(psPowerDev->psDevNode,
+			                                          bDeviceOffPermitted);
+			if (eError == PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+			{
+				PVRSRV_ERROR eErrPwrLockAcq;
+				/* FW denied idle request */
+				PVRSRVPowerUnlock(psDeviceNode);
+
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+
+#if defined(DEBUG)
+				eErrPwrLockAcq = pfnPowerLockAcquire(psDeviceNode, __FILE__, __LINE__);
+#else
+				eErrPwrLockAcq = pfnPowerLockAcquire(psDeviceNode);
+#endif
+				if (eErrPwrLockAcq != PVRSRV_OK)
+				{
+					/* We only understand PVRSRV_ERROR_RETRY, so assert on others.
+					 * Moreover, we've ended-up releasing the power-lock which was
+					 * originally "held" by caller before calling this function -
+					 * since this needs vigilant handling at call-site, we pass
+					 * back an explicit error, for caller(s) to "avoid" calling
+					 * PVRSRVPowerUnlock */
+					PVR_ASSERT(eErrPwrLockAcq == PVRSRV_ERROR_RETRY);
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to re-acquire power-lock "
+					         "(%s) after releasing it for a time-out",
+							 __func__, PVRSRVGetErrorString(eErrPwrLockAcq)));
+					return PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED;
+				}
+			}
+			else
+			{
+				/* idle request successful or some other error occurred, return */
+				break;
+			}
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*
+ * Wrapper function helps limiting calling complexity of supplying additional
+ * PFN_POWER_LOCK_ACQUIRE argument (required by _PVRSRVDeviceIdleRequestKM)
+ */
+inline PVRSRV_ERROR PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF      pfnIsDefaultStateOff,
+					IMG_BOOL                              bDeviceOffPermitted)
+{
+	return _PVRSRVDeviceIdleRequestKM(psDeviceNode,
+	                                  pfnIsDefaultStateOff,
+	                                  bDeviceOffPermitted,
+#if defined(DEBUG)
+	                                  PVRSRVPowerLock_Debug
+#else
+	                                  PVRSRVPowerLock
+#endif
+	                                  );
+}
+
+PVRSRV_ERROR PVRSRVDeviceIdleCancelRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+
+	if (psPowerDev && psPowerDev->pfnForcedIdleCancelRequest)
+	{
+		return psPowerDev->pfnForcedIdleCancelRequest(psPowerDev->psDevNode);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedGetKM(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	if ((++psPowerDev->iIdleReqInFlight) > 1)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return PVRSRV_OK;
+	}
+
+	eError = _PVRSRVDeviceIdleRequestKM(psDeviceNode,
+									  &PVRSRVDeviceIsDefaultStateOFF,
+									  IMG_TRUE,
+#if defined(DEBUG)
+									  PVRSRVPowerLock_Debug
+#else
+									  PVRSRVPowerLock
+#endif
+									  );
+	if (eError != PVRSRV_OK)
+	{
+		psPowerDev->iIdleReqInFlight--;
+	}
+
+	if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedPutAsyncKM(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPowerLock", ErrorOut);
+
+	if (psPowerDev &&
+	    (--psPowerDev->iIdleReqInFlight) == 0 &&
+	    psPowerDev->pfnForcedIdleCancelRequestAsync)
+	{
+		eError = psPowerDev->pfnForcedIdleCancelRequestAsync(psPowerDev->psDevNode);
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+ErrorOut:
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceSystemPrePowerStateKM
+
+ @Description
+
+ Perform device and system-specific processing required before a power
+ transition
+
+ @Input		psPowerDevice : Power device
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static
+PVRSRV_ERROR PVRSRVDeviceSystemPrePowerStateKM(PVRSRV_POWER_DEV			*psPowerDevice,
+											   PVRSRV_DEV_POWER_STATE	eNewPowerState,
+											   PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+	IMG_UINT64 ui64SysTimer1 = 0;
+	IMG_UINT64 ui64SysTimer2 = 0;
+	IMG_UINT64 ui64DevTimer1 = 0;
+	IMG_UINT64 ui64DevTimer2 = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(eNewPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	if (psPowerDevice->pfnDevicePrePower != NULL)
+	{
+		ui64DevTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		/* Call the device's power callback. */
+		eError = psPowerDevice->pfnDevicePrePower(psPowerDevice->psDevNode,
+												  eNewPowerState,
+												  eCurrentPowerState,
+												  ePwrFlags);
+
+		ui64DevTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	/* Do any required system-layer processing. */
+	if (psPowerDevice->pfnSystemPrePower != NULL)
+	{
+		ui64SysTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		eError = psPowerDevice->pfnSystemPrePower(psPowerDevice->hSysData,
+												  (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													PVRSRV_SYS_POWER_STATE_ON :
+													PVRSRV_SYS_POWER_STATE_OFF,
+												  (eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													PVRSRV_SYS_POWER_STATE_ON :
+													PVRSRV_SYS_POWER_STATE_OFF,
+												  ePwrFlags);
+
+		ui64SysTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_GOTO_IF_ERROR(eError, ErrRestorePowerState);
+	}
+
+	_InsertPowerTimeStatistic(psPowerDevice, ui64SysTimer1, ui64SysTimer2,
+							 ui64DevTimer1, ui64DevTimer2,
+							 BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED),
+							 eNewPowerState == PVRSRV_DEV_POWER_STATE_ON,
+							 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+ErrRestorePowerState:
+	/* In a situation where pfnDevicePrePower() succeeded but pfnSystemPrePower()
+	 * failed we need to restore the device's power state from before the current
+	 * request. Otherwise it will result in an inconsistency between the device's
+	 * actual state and what the driver thinks the state is. */
+	{
+		PVRSRV_ERROR eError2 = PVRSRV_OK;
+
+		if (psPowerDevice->pfnDevicePrePower != NULL)
+		{
+			/* Call the device's power callback. */
+			eError2 = psPowerDevice->pfnDevicePrePower(psPowerDevice->psDevNode,
+			                                           eCurrentPowerState,
+			                                           eNewPowerState,
+			                                           ePwrFlags);
+			PVR_LOG_IF_ERROR(eError2, "pfnDevicePrePower");
+		}
+		if (eError2 == PVRSRV_OK && psPowerDevice->pfnDevicePostPower != NULL)
+		{
+			/* Call the device's power callback. */
+			eError2 = psPowerDevice->pfnDevicePostPower(psPowerDevice->psDevNode,
+			                                            eCurrentPowerState,
+			                                            eNewPowerState,
+			                                            ePwrFlags);
+			PVR_LOG_IF_ERROR(eError2, "pfnDevicePostPower");
+		}
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceSystemPostPowerStateKM
+
+ @Description
+
+ Perform device and system-specific processing required after a power
+ transition
+
+ @Input		psPowerDevice : Power device
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static
+PVRSRV_ERROR PVRSRVDeviceSystemPostPowerStateKM(PVRSRV_POWER_DEV		*psPowerDevice,
+												PVRSRV_DEV_POWER_STATE	eNewPowerState,
+												PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+	IMG_UINT64 ui64SysTimer1 = 0;
+	IMG_UINT64 ui64SysTimer2 = 0;
+	IMG_UINT64 ui64DevTimer1 = 0;
+	IMG_UINT64 ui64DevTimer2 = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(eNewPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	/* Do any required system-layer processing. */
+	if (psPowerDevice->pfnSystemPostPower != NULL)
+	{
+		ui64SysTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		eError = psPowerDevice->pfnSystemPostPower(psPowerDevice->hSysData,
+												   (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													 PVRSRV_SYS_POWER_STATE_ON :
+													 PVRSRV_SYS_POWER_STATE_OFF,
+												   (eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													 PVRSRV_SYS_POWER_STATE_ON :
+													 PVRSRV_SYS_POWER_STATE_OFF,
+												   ePwrFlags);
+
+		ui64SysTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psPowerDevice->pfnDevicePostPower != NULL)
+	{
+		ui64DevTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		/* Call the device's power callback. */
+		eError = psPowerDevice->pfnDevicePostPower(psPowerDevice->psDevNode,
+												   eNewPowerState,
+												   eCurrentPowerState,
+												   ePwrFlags);
+
+		ui64DevTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	_InsertPowerTimeStatistic(psPowerDevice, ui64SysTimer1, ui64SysTimer2,
+							 ui64DevTimer1, ui64DevTimer2,
+							 BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED),
+							 eNewPowerState == PVRSRV_DEV_POWER_STATE_ON,
+							 IMG_FALSE);
+
+	PVRSRVSetDeviceCurrentPowerState(psPowerDevice, eNewPowerState);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVSetDevicePowerStateKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE eNewPowerState,
+										 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_ERROR	eError;
+	PVRSRV_DATA*    psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (!psPowerDevice)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (eNewPowerState == PVRSRV_DEV_POWER_STATE_DEFAULT)
+	{
+		eNewPowerState = psPowerDevice->eDefaultPowerState;
+	}
+
+	/* Call power function if the state change or if this is an OS request. */
+	if (OSAtomicRead(&psPowerDevice->eCurrentPowerState) != eNewPowerState ||
+	    BITMASK_ANY(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ | PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ))
+	{
+		eError = PVRSRVDeviceSystemPrePowerStateKM(psPowerDevice,
+												   eNewPowerState,
+												   ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		eError = PVRSRVDeviceSystemPostPowerStateKM(psPowerDevice,
+													eNewPowerState,
+													ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		psDeviceNode->eCurrentSysPowerState = eNewPowerState;
+
+		/* Signal Device Watchdog Thread about power mode change. */
+		if (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans++;
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			if (psPVRSRVData->ui32DevicesWatchdogTimeout == DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT)
+#endif
+			{
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+		}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+		else if (eNewPowerState == PVRSRV_DEV_POWER_STATE_OFF)
+		{
+			/* signal watchdog thread and give it a chance to switch to
+			 * longer / infinite wait time */
+			eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	return PVRSRV_OK;
+
+ErrorExit:
+
+	if (eError == PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Transition to %d was denied, Flags=0x%08x",
+				 __func__, eNewPowerState, ePwrFlags));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Transition to %d FAILED (%s)",
+				 __func__, eNewPowerState, PVRSRVGetErrorString(eError)));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState,
+											 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT uiStage = 0;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+#endif
+
+	PVRSRV_DATA*    psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_POWER_DEV *psPowerDevice;
+	PVRSRV_DEV_POWER_STATE eNewDevicePowerState;
+	PFN_SYS_DEV_IS_DEFAULT_STATE_OFF pfnIsDefaultStateOff;
+
+	/* Require a proper power state */
+	if (eNewSysPowerState == PVRSRV_SYS_POWER_STATE_Unspecified)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		/* Power device is not initialised. */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device not initialised", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Prevent simultaneous SetPowerStateKM calls */
+	_PVRSRVForcedPowerLock(psDeviceNode);
+	psPowerDevice = psDeviceNode->psPowerDev;
+
+	eNewDevicePowerState = _IsSystemStatePowered(eNewSysPowerState)
+	    ? psPowerDevice->eDefaultPowerState : PVRSRV_DEV_POWER_STATE_OFF;
+
+	/* If setting devices to default state, force idle all devices whose default state is off */
+	pfnIsDefaultStateOff =
+	  (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_OFF) ? PVRSRVDeviceIsDefaultStateOFF : NULL;
+
+	/* No power transition requested, so do nothing */
+	if (eNewSysPowerState == psDeviceNode->eCurrentSysPowerState)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return PVRSRV_OK;
+	}
+
+	/* If the device is already off don't send the idle request. */
+	if ((psDeviceNode->eCurrentSysPowerState != PVRSRV_SYS_POWER_STATE_OFF) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		&& (eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+#endif
+		)
+	{
+		eError = _PVRSRVDeviceIdleRequestKM(psDeviceNode, pfnIsDefaultStateOff,
+#if defined(DEBUG)
+		                                    IMG_TRUE, _PVRSRVForcedPowerLock_Debug
+#else
+		                                    IMG_TRUE, _PVRSRVForcedPowerLock
+#endif
+		                                    );
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PVRSRVDeviceIdleRequestKM");
+			uiStage = 1;
+			goto ErrorExit;
+		}
+	}
+
+	/* Call power function if the state change or if this is an OS request. */
+	if (OSAtomicRead(&psPowerDevice->eCurrentPowerState) != eNewDevicePowerState ||
+	    BITMASK_ANY(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ | PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ))
+	{
+		eError = PVRSRVDeviceSystemPrePowerStateKM(psPowerDevice,
+												   eNewDevicePowerState,
+												   ePwrFlags);
+		uiStage = 2;
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		eError = PVRSRVDeviceSystemPostPowerStateKM(psPowerDevice,
+													eNewDevicePowerState,
+													ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		/* Signal Device Watchdog Thread about power mode change. */
+		if (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans++;
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			if (psPVRSRVData->ui32DevicesWatchdogTimeout == DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT)
+#endif
+			{
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+		}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+		else if (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_OFF)
+		{
+			/* signal watchdog thread and give it a chance to switch to
+			 * longer / infinite wait time */
+			eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	psDeviceNode->eCurrentSysPowerState = eNewSysPowerState;
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Transition from %s to %s FAILED (%s) at stage "
+	         "%u. Dumping debug info.", __func__,
+	         PVRSRVSysPowerStateToString(psDeviceNode->eCurrentSysPowerState),
+	         PVRSRVSysPowerStateToString(eNewSysPowerState),
+	         PVRSRVGetErrorString(eError), uiStage));
+
+	PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVSetSystemPowerState(PVRSRV_DEVICE_CONFIG *psDevConfig,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDevNode = psDevConfig->psDevNode;
+	PVRSRV_SYS_POWER_STATE eCurrentSysPowerState;
+
+	PVRSRVGetSystemPowerState(psDevNode, &eCurrentSysPowerState);
+
+	/* no power transition requested, so do nothing */
+	if (eNewSysPowerState == eCurrentSysPowerState)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (psDevConfig->pfnPrePowerState != NULL)
+	{
+		eError = psDevConfig->pfnPrePowerState(psDevConfig->hSysData,
+												  eNewSysPowerState,
+												  eCurrentSysPowerState,
+												  PVRSRV_POWER_FLAGS_FORCED);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psDevConfig->pfnPostPowerState != NULL)
+	{
+		eError = psDevConfig->pfnPostPowerState(psDevConfig->hSysData,
+												   eNewSysPowerState,
+												   eCurrentSysPowerState,
+												   PVRSRV_POWER_FLAGS_FORCED);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psDevNode != NULL)
+	{
+		psDevNode->eCurrentSysPowerState = eNewSysPowerState;
+	}
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVSetPowerCallbacks(PPVRSRV_DEVICE_NODE				psDeviceNode,
+							 PVRSRV_POWER_DEV					*psPowerDevice,
+							 PFN_PRE_POWER						pfnDevicePrePower,
+							 PFN_POST_POWER						pfnDevicePostPower,
+							 PFN_SYS_PRE_POWER					pfnSystemPrePower,
+							 PFN_SYS_POST_POWER					pfnSystemPostPower,
+							 PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync)
+{
+	if (psPowerDevice != NULL)
+	{
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+		{
+			psPowerDevice->pfnSystemPrePower = NULL;
+			psPowerDevice->pfnSystemPostPower = NULL;
+		}
+		else
+		{
+			psPowerDevice->pfnSystemPrePower = pfnSystemPrePower;
+			psPowerDevice->pfnSystemPostPower = pfnSystemPostPower;
+		}
+
+		psPowerDevice->pfnDevicePrePower = pfnDevicePrePower;
+		psPowerDevice->pfnDevicePostPower = pfnDevicePostPower;
+		psPowerDevice->pfnForcedIdleRequest = pfnForcedIdleRequest;
+		psPowerDevice->pfnForcedIdleCancelRequest = pfnForcedIdleCancelRequest;
+		psPowerDevice->pfnForcedIdleCancelRequestAsync = pfnForcedIdleCancelRequestAsync;
+	}
+}
+
+PVRSRV_ERROR PVRSRVRegisterPowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode,
+									   PFN_PRE_POWER				pfnDevicePrePower,
+									   PFN_POST_POWER				pfnDevicePostPower,
+									   PFN_SYS_PRE_POWER			pfnSystemPrePower,
+									   PFN_SYS_POST_POWER			pfnSystemPostPower,
+									   PFN_PRE_CLOCKSPEED_CHANGE	pfnPreClockSpeedChange,
+									   PFN_POST_CLOCKSPEED_CHANGE	pfnPostClockSpeedChange,
+									   PFN_FORCED_IDLE_REQUEST	pfnForcedIdleRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync,
+									   PFN_GPU_UNITS_POWER_CHANGE	pfnGPUUnitsPowerChange,
+									   IMG_HANDLE					psDevNode,
+									   PVRSRV_DEV_POWER_STATE		eCurrentPowerState,
+									   PVRSRV_DEV_POWER_STATE		eDefaultPowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	PVR_ASSERT(!psDeviceNode->psPowerDev);
+
+	PVR_ASSERT(eCurrentPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+	PVR_ASSERT(eDefaultPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	psPowerDevice = OSAllocMem(sizeof(PVRSRV_POWER_DEV));
+	PVR_LOG_RETURN_IF_NOMEM(psPowerDevice, "psPowerDevice");
+
+	/* setup device for power manager */
+	PVRSRVSetPowerCallbacks(psDeviceNode,
+							psPowerDevice,
+							pfnDevicePrePower,
+							pfnDevicePostPower,
+							pfnSystemPrePower,
+							pfnSystemPostPower,
+							pfnForcedIdleRequest,
+							pfnForcedIdleCancelRequest,
+							pfnForcedIdleCancelRequestAsync);
+
+	psPowerDevice->pfnPreClockSpeedChange = pfnPreClockSpeedChange;
+	psPowerDevice->pfnPostClockSpeedChange = pfnPostClockSpeedChange;
+	psPowerDevice->pfnGPUUnitsPowerChange = pfnGPUUnitsPowerChange;
+	psPowerDevice->hSysData = psDeviceNode->psDevConfig->hSysData;
+	psPowerDevice->psDevNode = psDevNode;
+	PVRSRVSetDeviceCurrentPowerState(psPowerDevice, eCurrentPowerState);
+	psPowerDevice->eDefaultPowerState = eDefaultPowerState;
+	psPowerDevice->iIdleReqInFlight = 0;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	OSCachedMemSet(&psPowerDevice->sPowerStats, 0, sizeof(psPowerDevice->sPowerStats));
+#endif
+
+	psDeviceNode->psPowerDev = psPowerDevice;
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVRemovePowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	if (psDeviceNode->psPowerDev)
+	{
+		/* RGX device/firmware should now be powered down */
+		PVR_ASSERT(PVRSRVIsDevicePowered(psDeviceNode) == IMG_FALSE);
+		OSFreeMem(psDeviceNode->psPowerDev);
+		psDeviceNode->psPowerDev = NULL;
+	}
+}
+
+PVRSRV_ERROR PVRSRVGetDevicePowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+									   PPVRSRV_DEV_POWER_STATE pePowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	*pePowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL PVRSRVIsDevicePowered(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_DEV_POWER_STATE ePowerState;
+
+	if (PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState) != PVRSRV_OK)
+	{
+		return IMG_FALSE;
+	}
+
+	return (ePowerState == PVRSRV_DEV_POWER_STATE_ON);
+}
+
+PVRSRV_ERROR PVRSRVGetSystemPowerState(PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                   PPVRSRV_SYS_POWER_STATE peCurrentSysPowerState)
+{
+	if (psDeviceNode != NULL)
+	{
+		if (psDeviceNode->psDevConfig->pfnGpuDomainPower)
+		{
+			*peCurrentSysPowerState = psDeviceNode->psDevConfig->pfnGpuDomainPower(psDeviceNode);
+		}
+		else
+		{
+			*peCurrentSysPowerState = psDeviceNode->eCurrentSysPowerState;
+		}
+	}
+	else
+	{
+		/* assume power is off if no device node */
+		*peCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_OFF;
+		return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL PVRSRVIsSystemPowered(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_SYS_POWER_STATE eSysPowerState;
+
+	if (PVRSRVGetSystemPowerState(psDeviceNode, &eSysPowerState) != PVRSRV_OK)
+	{
+		return IMG_FALSE;
+	}
+
+	return (eSysPowerState == PVRSRV_SYS_POWER_STATE_ON);
+}
+
+PVRSRV_ERROR
+PVRSRVDevicePreClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                IMG_BOOL            bIdleDevice,
+                                void*               pvInfo)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	IMG_UINT64 ui64StartTimer, ui64StopTimer;
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+
+	PVR_UNREFERENCED_PARAMETER(pvInfo);
+
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	ui64StartTimer = PVRSRVProcessStatsGetTimeUs();
+
+	/* This mutex is released in PVRSRVDevicePostClockSpeedChange. */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	if ((eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) && bIdleDevice)
+	{
+		/* We can change the clock speed if the device is either IDLE or OFF */
+		eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_TRUE);
+
+		if (eError != PVRSRV_OK)
+		{
+			/* FW Can signal denied when busy with SPM or other work it can not idle */
+			if (eError != PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Error (%s) from %s()", __func__,
+				         PVRSRVGETERRORSTRING(eError), "PVRSRVDeviceIdleRequestKM"));
+			}
+			if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+			return eError;
+		}
+	}
+
+	eError = psPowerDevice->pfnPreClockSpeedChange(psPowerDevice->psDevNode,
+	                                               eCurrentPowerState);
+
+	ui64StopTimer = PVRSRVProcessStatsGetTimeUs();
+
+	_InsertPowerTimeStatisticExtraPre(psPowerDevice, ui64StartTimer, ui64StopTimer);
+
+	return eError;
+}
+
+void
+PVRSRVDevicePostClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                 IMG_BOOL            bIdleDevice,
+                                 void*               pvInfo)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	IMG_UINT64 ui64StartTimer, ui64StopTimer;
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+
+	PVR_UNREFERENCED_PARAMETER(pvInfo);
+
+	if (psPowerDevice == NULL)
+	{
+		return;
+	}
+
+	ui64StartTimer = PVRSRVProcessStatsGetTimeUs();
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	eError = psPowerDevice->pfnPostClockSpeedChange(psPowerDevice->psDevNode,
+													eCurrentPowerState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p failed (%s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+	}
+
+	if ((eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) && bIdleDevice)
+	{
+		eError = PVRSRVDeviceIdleCancelRequestKM(psDeviceNode);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceIdleCancelRequestKM");
+	}
+
+	/* This mutex was acquired in PVRSRVDevicePreClockSpeedChange. */
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	OSAtomicIncrement(&psDeviceNode->iNumClockSpeedChanges);
+
+	ui64StopTimer = PVRSRVProcessStatsGetTimeUs();
+
+	_InsertPowerTimeStatisticExtraPost(psPowerDevice, ui64StartTimer, ui64StopTimer);
+}
+
+PVRSRV_ERROR PVRSRVDeviceGPUUnitsPowerChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                             IMG_UINT32 ui32NewValue)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_POWER_DEV	*psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice)
+	{
+		PVRSRV_DEV_POWER_STATE eDevicePowerState;
+
+		eError = PVRSRVPowerLock(psDeviceNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+		eDevicePowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+		if (eDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			/* Device must be idle to change GPU unit(s) power state */
+			eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_FALSE);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+				if (eError == PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+				{
+					goto ErrorExit;
+				}
+				goto ErrorUnlockAndExit;
+			}
+		}
+
+		if (psPowerDevice->pfnGPUUnitsPowerChange != NULL)
+		{
+			PVRSRV_ERROR eError2 = psPowerDevice->pfnGPUUnitsPowerChange(psPowerDevice->psDevNode, ui32NewValue);
+
+			if (eError2 != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Device %p failed (%s)",
+				         __func__, psDeviceNode,
+				         PVRSRVGetErrorString(eError2)));
+			}
+		}
+
+		if (eDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			eError = PVRSRVDeviceIdleCancelRequestKM(psDeviceNode);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDeviceIdleCancelRequestKM", ErrorUnlockAndExit);
+		}
+
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return eError;
+
+ErrorUnlockAndExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+ErrorExit:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (power.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/process_stats.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/process_stats.c
new file mode 100644
index 000000000000..77441c578089
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/process_stats.c
@@ -0,0 +1,3492 @@
+/*************************************************************************/ /*!
+@File
+@Title          Process based statistics
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Manages a collection of statistics based around a process
+                and referenced via OS agnostic methods.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "lock.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "process_stats.h"
+#include "ri_server.h"
+#include "hash.h"
+#include "connection_server.h"
+#include "pvrsrv.h"
+#include "proc_stats.h"
+#include "pvr_ricommon.h"
+#include "di_server.h"
+#include "dllist.h"
+#if defined(__linux__)
+#include "trace_events.h"
+#endif
+
+#if defined(SUPPORT_LINUX_FDINFO) && defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "dkp_impl.h"
+
+#define PROC_STATS_FDINFO_ENABLED
+#endif
+
+/* Enabled OS Statistics entries: DEBUGFS on Linux, undefined for other OSs */
+#if defined(__linux__) && ( \
+	defined(PVRSRV_ENABLE_PERPID_STATS) || \
+	defined(PVRSRV_ENABLE_CACHEOP_STATS) || \
+	defined(PVRSRV_ENABLE_MEMORY_STATS) || \
+	defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) )
+#define ENABLE_DEBUGFS_PIDS
+#endif
+
+/* Enable GPU memory accounting tracepoint */
+#if defined(__linux__) && ( \
+	defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) )
+#define ENABLE_GPU_MEM_TRACEPOINT
+#endif
+
+/*
+ * Maximum history of process statistics that will be kept.
+ */
+#define MAX_DEAD_LIST_PROCESSES  (10)
+
+/*
+ * Maximum size of a process name to be displayed in process_stats.
+ */
+#define MAX_PROC_NAME_LEN  (16)
+
+/*
+ * Definition of all the strings used to format process based statistics.
+ */
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+/* Array of Process stat type defined using the X-Macro */
+#define X(stat_type, stat_str, drm_str) stat_str,
+static const IMG_CHAR *const pszProcessStatType[PVRSRV_PROCESS_STAT_TYPE_COUNT] = { PVRSRV_PROCESS_STAT_KEY };
+#undef X
+#define X(stat_type, stat_str) stat_str,
+static const IMG_CHAR *const pszDeviceStatType[PVRSRV_DEVICE_STAT_TYPE_COUNT] = { PVRSRV_DEVICE_STAT_KEY };
+#undef X
+#endif
+
+/* Array of Driver stat type defined using the X-Macro */
+#define X(stat_type, stat_str) stat_str,
+static const IMG_CHAR *const pszDriverStatType[PVRSRV_DRIVER_STAT_TYPE_COUNT] = { PVRSRV_DRIVER_STAT_KEY };
+#undef X
+
+/* structure used in hash table to track statistic entries */
+typedef struct {
+	size_t	   uiSizeInBytes;
+	IMG_PID	   uiPid;
+} _PVR_STATS_TRACKING_HASH_ENTRY;
+
+/* Function used internally to decrement tracked per-process statistic entries */
+static void _StatsDecrMemTrackedStat(_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry,
+                                     PVRSRV_MEM_ALLOC_TYPE eAllocType);
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+int RawProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+#endif
+int GlobalStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/* Note: all of the accesses to the global stats should be protected
+ * by the gsGlobalStats.hGlobalStatsLock lock. This means all of the
+ * invocations of macros *_GLOBAL_STAT_VALUE. */
+
+/* Macros for fetching stat values */
+#define GET_STAT_VALUE(ptr,var) (ptr)->i64StatValue[(var)]
+#define GET_GLOBAL_STAT_VALUE(idx) gsGlobalStats.ui64StatValue[idx]
+
+#define GET_GPUMEM_GLOBAL_STAT_VALUE() \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT)
+
+#define GET_GPUMEM_PERPID_STAT_VALUE(ptr) \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT)
+/*
+ * Macros for updating stat values.
+ */
+#define UPDATE_MAX_VALUE(a,b)					do { if ((b) > (a)) {(a) = (b);} } while (0)
+#define INCREASE_STAT_VALUE(ptr,var,val)		do { (ptr)->i64StatValue[(var)] += (IMG_INT64)(val); if ((ptr)->i64StatValue[(var)] > (ptr)->i64StatValue[(var##_MAX)]) {(ptr)->i64StatValue[(var##_MAX)] = (ptr)->i64StatValue[(var)];} } while (0)
+#define INCREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { (var).ui64StatValue[(idx)] += (IMG_UINT64)(val); if ((var).ui64StatValue[(idx)] > (var).ui64StatValue[(idx##_MAX)]) {(var).ui64StatValue[(idx##_MAX)] = (var).ui64StatValue[(idx)];} } while (0)
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+/* Allow stats to go negative */
+#define DECREASE_STAT_VALUE(ptr,var,val)		do { (ptr)->i64StatValue[(var)] -= (val); } while (0)
+#define DECREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { (var).ui64StatValue[(idx)] -= (val); } while (0)
+#else
+#define DECREASE_STAT_VALUE(ptr,var,val)		do { if ((ptr)->i64StatValue[(var)] >= (val)) { (ptr)->i64StatValue[(var)] -= (IMG_INT64)(val); } else { (ptr)->i64StatValue[(var)] = 0; } } while (0)
+#define DECREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { if ((var).ui64StatValue[(idx)] >= (val)) { (var).ui64StatValue[(idx)] -= (IMG_UINT64)(val); } else { (var).ui64StatValue[(idx)] = 0; } } while (0)
+#endif
+#define MAX_CACHEOP_STAT 16
+#define INCREMENT_CACHEOP_STAT_IDX_WRAP(x) ((x+1) >= MAX_CACHEOP_STAT ? 0 : (x+1))
+#define DECREMENT_CACHEOP_STAT_IDX_WRAP(x) ((x-1) < 0 ? (MAX_CACHEOP_STAT-1) : (x-1))
+
+/*
+ * Track the search of one process when PVRSRV_DEBUG_LINUX_MEMORY_STATS
+ * is enabled.
+ */
+typedef enum _PVRSRV_PROC_SEARCH_STATE_
+{
+	PVRSRV_PROC_NOTFOUND,
+	PVRSRV_PROC_FOUND,
+	PVRSRV_PROC_RESURRECTED,
+} PVRSRV_PROC_SEARCH_STATE;
+
+/*
+ * Structures for holding statistics...
+ */
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+typedef struct _PVRSRV_MEM_ALLOC_REC_
+{
+	PVRSRV_MEM_ALLOC_TYPE           eAllocType;
+	void*                           pvCpuVAddr;
+	IMG_CPU_PHYADDR	                sCpuPAddr;
+	size_t                          uiBytes;
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	void*                           pvAllocdFromFile;
+	IMG_UINT32                      ui32AllocdFromLine;
+#endif
+} PVRSRV_MEM_ALLOC_REC;
+
+typedef struct PVRSRV_MEM_ALLOC_PRINT_DATA_TAG
+{
+	OSDI_IMPL_ENTRY *psEntry;
+	IMG_PID	        pid;
+	IMG_UINT32      ui32NumEntries;
+} PVRSRV_MEM_ALLOC_PRINT_DATA;
+#endif
+
+typedef struct _PVRSRV_PROCESS_STATS_ {
+
+	/* Linked list pointers */
+	DLLIST_NODE                    sNode;
+
+	/* Create per process lock that need to be held
+	 * to edit of its members */
+	POS_LOCK                       hLock;
+
+	/* OS level process ID */
+	IMG_PID	                       pid;
+	IMG_CHAR                       processName[MAX_PROC_NAME_LEN];
+	IMG_UINT32                     ui32RefCount;
+
+	/* Process memory stats */
+	IMG_INT64                      i64StatValue[PVRSRV_PROCESS_STAT_TYPE_COUNT];
+	IMG_UINT32                     ui32StatAllocFlags;
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	struct _CACHEOP_STRUCT_ {
+		PVRSRV_CACHE_OP        uiCacheOp;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+		IMG_DEV_VIRTADDR       sDevVAddr;
+		IMG_DEV_PHYADDR        sDevPAddr;
+#endif
+		IMG_DEVMEM_SIZE_T      uiOffset;
+		IMG_DEVMEM_SIZE_T      uiSize;
+		IMG_UINT64             ui64ExecuteTime;
+		IMG_BOOL               bUserModeFlush;
+		IMG_BOOL               bIsFence;
+		IMG_PID                ownerPid;
+	}                              asCacheOp[MAX_CACHEOP_STAT];
+	IMG_INT32                      uiCacheOpWriteIndex;
+#endif
+
+	/* Other statistics structures */
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	HASH_TABLE* psMemoryRecords;
+#endif
+	/* Device stats */
+	IMG_UINT32                     ui32DevCount;
+	IMG_INT32                      ai32DevStats[][PVRSRV_DEVICE_STAT_TYPE_COUNT];
+} PVRSRV_PROCESS_STATS;
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+static PVRDKF_DKP_HANDLE ghDKPHandle;
+#endif
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+
+typedef struct _PVRSRV_OS_STAT_ENTRY_
+{
+	DI_GROUP *psStatsDIGroup;
+	DI_ENTRY *psProcessStatsDIEntry;
+	DI_ENTRY *psMemStatsDIEntry;
+	DI_ENTRY *psRIMemStatsDIEntry;
+	DI_ENTRY *psCacheOpStatsDIEntry;
+} PVRSRV_OS_STAT_ENTRY;
+
+static PVRSRV_OS_STAT_ENTRY gsLiveStatEntries;
+static PVRSRV_OS_STAT_ENTRY gsRetiredStatEntries;
+
+int GenericStatsPrintElementsLive(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+int GenericStatsPrintElementsRetired(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*
+ * Functions for printing the information stored...
+ */
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+void ProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                               PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+void MemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                           PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+void RIMemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                             PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void CacheOpStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                               PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+typedef void (PVRSRV_STATS_PRINT_ELEMENTS)(OSDI_IMPL_ENTRY *psEntry,
+                                           PVRSRV_PROCESS_STATS *psProcessStats);
+
+typedef enum
+{
+	PVRSRV_STAT_TYPE_PROCESS,
+	PVRSRV_STAT_TYPE_MEMORY,
+	PVRSRV_STAT_TYPE_RIMEMORY,
+	PVRSRV_STAT_TYPE_CACHEOP,
+	PVRSRV_STAT_TYPE_LAST
+} PVRSRV_STAT_TYPE;
+
+#define SEPARATOR_STR_LEN 166
+
+typedef struct _PVRSRV_STAT_PV_DATA_ {
+
+	PVRSRV_STAT_TYPE eStatType;
+	PVRSRV_STATS_PRINT_ELEMENTS* pfnStatsPrintElements;
+	IMG_CHAR szLiveStatsHeaderStr[SEPARATOR_STR_LEN + 1];
+	IMG_CHAR szRetiredStatsHeaderStr[SEPARATOR_STR_LEN + 1];
+
+} PVRSRV_STAT_PV_DATA;
+
+static PVRSRV_STAT_PV_DATA g_StatPvDataArr[] = {
+						{ PVRSRV_STAT_TYPE_PROCESS,  NULL, " Process"               , " Process"               },
+						{ PVRSRV_STAT_TYPE_MEMORY,   NULL, " Memory Allocation"     , " Memory Allocation"     },
+						{ PVRSRV_STAT_TYPE_RIMEMORY, NULL, " Resource Allocation"   , " Resource Allocation"   },
+						{ PVRSRV_STAT_TYPE_CACHEOP,  NULL, " Cache Maintenance Ops" , " Cache Maintenance Ops" }
+					      };
+
+#define GET_STAT_ENTRY_ID(STAT_TYPE) &g_StatPvDataArr[(STAT_TYPE)]
+
+/* Generic header strings */
+static const IMG_CHAR g_szLiveHeaderStr[]    = " Statistics for LIVE Processes ";
+static const IMG_CHAR g_szRetiredHeaderStr[] = " Statistics for RETIRED Processes ";
+
+/* Separator string used for separating stats for different PIDs */
+static IMG_CHAR g_szSeparatorStr[SEPARATOR_STR_LEN + 1] = "";
+
+static inline void
+_prepareStatsHeaderString(IMG_CHAR *pszStatsSpecificStr, const IMG_CHAR* pszGenericHeaderStr)
+{
+	IMG_UINT32 ui32NumSeparators;
+	IMG_CHAR szStatsHeaderFooterStr[75];
+
+	/* Prepare text content of the header in a local string */
+	OSStringSafeCopy(szStatsHeaderFooterStr, pszStatsSpecificStr, ARRAY_SIZE(szStatsHeaderFooterStr));
+	OSStringLCat(szStatsHeaderFooterStr, pszGenericHeaderStr, ARRAY_SIZE(szStatsHeaderFooterStr));
+
+	/* Write all '-' characters to the header string */
+	memset(pszStatsSpecificStr, '-', SEPARATOR_STR_LEN);
+	pszStatsSpecificStr[SEPARATOR_STR_LEN] = '\0';
+
+	/* Find the spot for text content in the header string */
+	ui32NumSeparators = (SEPARATOR_STR_LEN - OSStringLength(szStatsHeaderFooterStr)) >> 1;
+
+	/* Finally write the text content */
+	OSSNPrintf(pszStatsSpecificStr + ui32NumSeparators,
+		   OSStringLength(szStatsHeaderFooterStr),
+		   "%s", szStatsHeaderFooterStr);
+
+	/* Overwrite the '\0' character added by OSSNPrintf() */
+	if (OSStringLength(szStatsHeaderFooterStr) > 0)
+	{
+		pszStatsSpecificStr[ui32NumSeparators + OSStringLength(szStatsHeaderFooterStr) - 1] = ' ';
+	}
+}
+
+static inline void
+_prepareSeparatorStrings(void)
+{
+	IMG_UINT32 i;
+
+	/* Prepare header strings for each stat type */
+	for (i = 0; i < PVRSRV_STAT_TYPE_LAST; ++i)
+	{
+		_prepareStatsHeaderString(g_StatPvDataArr[i].szLiveStatsHeaderStr, g_szLiveHeaderStr);
+		_prepareStatsHeaderString(g_StatPvDataArr[i].szRetiredStatsHeaderStr, g_szRetiredHeaderStr);
+	}
+
+	/* Prepare separator string to separate stats for different PIDs */
+	memset(g_szSeparatorStr, '-', SEPARATOR_STR_LEN);
+	g_szSeparatorStr[SEPARATOR_STR_LEN] = '\0';
+}
+
+static inline void
+_prepareStatsPrivateData(void)
+{
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_PROCESS].pfnStatsPrintElements = ProcessStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_MEMORY].pfnStatsPrintElements = MemStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_RIMEMORY].pfnStatsPrintElements = RIMemStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_CACHEOP].pfnStatsPrintElements = CacheOpStatsPrintElements;
+#endif
+
+	_prepareSeparatorStrings();
+}
+
+#endif
+
+/*
+ * Global Boolean to flag when the statistics are ready to monitor
+ * memory allocations.
+ */
+static IMG_BOOL bProcessStatsInitialised = IMG_FALSE;
+
+/*
+ * Linked lists for process stats. Live stats are for processes which are still running
+ * and the dead list holds those that have exited.
+ */
+static DLLIST_NODE gsLiveList;
+static DLLIST_NODE gsDeadList;
+
+static POS_LOCK g_psLinkedListLock;
+/* Lockdep feature in the kernel cannot differentiate between different instances of same lock type.
+ * This allows it to group all such instances of the same lock type under one class
+ * The consequence of this is that, if lock acquisition is nested on different instances, it generates
+ * a false warning message about the possible occurrence of deadlock due to recursive lock acquisition.
+ * Hence we create the following sub classes to explicitly appraise Lockdep of such safe lock nesting */
+#define PROCESS_LOCK_SUBCLASS_CURRENT	1
+#if defined(ENABLE_DEBUGFS_PIDS)
+/*
+ * Pointer to OS folder to hold PID folders.
+ */
+static DI_GROUP *psProcStatsDIGroup;
+#endif
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+static DI_ENTRY *psProcStatsDIEntry;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+/* Global driver PID stats registration handle */
+static IMG_HANDLE g_hDriverProcessStats;
+#endif
+
+/* Global driver-data folders */
+typedef struct _GLOBAL_STATS_
+{
+	IMG_UINT64 ui64StatValue[PVRSRV_DRIVER_STAT_TYPE_COUNT];
+	POS_LOCK   hGlobalStatsLock;
+} GLOBAL_STATS;
+
+static DI_ENTRY *psGlobalMemDIEntry;
+static GLOBAL_STATS gsGlobalStats;
+
+#define HASH_INITIAL_SIZE 5
+/* A hash table used to store the size of any vmalloc'd allocation
+ * against its address (not needed for kmallocs as we can use ksize()) */
+static HASH_TABLE* gpsSizeTrackingHashTable;
+static POS_LOCK	 gpsSizeTrackingHashTableLock;
+
+static PVRSRV_ERROR _RegisterProcess(IMG_HANDLE *phProcessStats, IMG_PID ownerPid);
+
+static void _DestroyProcessStat(PVRSRV_PROCESS_STATS* psProcessStats);
+
+static void _DecreaseProcStatValueUnlocked(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                                           PVRSRV_PROCESS_STATS* psProcessStats,
+                                           IMG_UINT64 uiBytes);
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStatsInLiveList
+@Description    Searches the Live Process List for a statistics structure that
+                matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStatsInLiveList(IMG_PID pid)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats;
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+
+		if (psProcessStats->pid == pid)
+		{
+			return psProcessStats;
+		}
+	}
+	return NULL;
+} /* _FindProcessStatsInLiveList */
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStatsInDeadList
+@Description    Searches the Dead Process List for a statistics structure that
+                matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStatsInDeadList(IMG_PID pid)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats;
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+
+		if (psProcessStats->pid == pid)
+		{
+			return psProcessStats;
+		}
+	}
+	return NULL;
+} /* _FindProcessStatsInDeadList */
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStats
+@Description    Searches the Live and Dead Process Lists for a statistics
+                structure that matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStats(IMG_PID pid)
+{
+	PVRSRV_PROCESS_STATS* psProcessStats = _FindProcessStatsInLiveList(pid);
+
+	if (psProcessStats == NULL)
+	{
+		psProcessStats = _FindProcessStatsInDeadList(pid);
+	}
+
+	return psProcessStats;
+} /* _FindProcessStats */
+
+/*************************************************************************/ /*!
+@Function       _CompressMemoryUsage
+@Description    Reduces memory usage by deleting old statistics data.
+                This function requires that the list lock is not held!
+*/ /**************************************************************************/
+static void
+_CompressMemoryUsage(void)
+{
+	IMG_INT32 i32ItemsRemaining;
+	DLLIST_NODE *psNode, *psNext;
+	DLLIST_NODE sToBeFreedHead;
+
+	/*
+	 * We hold the lock whilst checking the list, but we'll release it
+	 * before freeing memory (as that will require the lock too)!
+	 */
+	OSLockAcquire(g_psLinkedListLock);
+
+	/* Check that the dead list is not bigger than the max size... */
+	i32ItemsRemaining = MAX_DEAD_LIST_PROCESSES;
+
+	dllist_init(&sToBeFreedHead);
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		i32ItemsRemaining--;
+		if (i32ItemsRemaining < 0)
+		{
+			/* This is the last allowed process, cut the linked list here! */
+			dllist_remove_node(psNode);
+			dllist_add_to_tail(&sToBeFreedHead, psNode);
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	dllist_foreach_node(&sToBeFreedHead, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS *psProcessStatsToBeFreed;
+		psProcessStatsToBeFreed = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		RIDeleteEntriesForPID(psProcessStatsToBeFreed->pid);
+#endif
+		_DestroyProcessStat(psProcessStatsToBeFreed);
+	}
+} /* _CompressMemoryUsage */
+
+/* These functions move the process stats from the live to the dead list.
+ * _MoveProcessToDeadList moves the entry in the global lists and
+ * it needs to be protected by g_psLinkedListLock.
+ * _MoveProcessToDeadList performs the OS calls and it
+ * shouldn't be used under g_psLinkedListLock because this could generate a
+ * lockdep warning. */
+static void
+_MoveProcessToDeadList(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	/* Take the element out of the live list and append to the dead list... */
+	PVR_ASSERT(psProcessStats != NULL);
+	dllist_remove_node(&psProcessStats->sNode);
+	dllist_add_to_head(&gsDeadList, &psProcessStats->sNode);
+} /* _MoveProcessToDeadList */
+
+/* These functions move the process stats from the dead to the live list.
+ * _MoveProcessToLiveList moves the entry in the global lists and
+ * it needs to be protected by g_psLinkedListLock.
+ * _MoveProcessToLiveList performs the OS calls and it
+ * shouldn't be used under g_psLinkedListLock because this could generate a
+ * lockdep warning. */
+static void
+_MoveProcessToLiveList(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	/* Take the element out of the live list and append to the dead list... */
+	PVR_ASSERT(psProcessStats != NULL);
+	dllist_remove_node(&psProcessStats->sNode);
+	dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+} /* _MoveProcessToLiveList */
+
+static PVRSRV_ERROR
+_AllocateProcessStats(PVRSRV_PROCESS_STATS **ppsProcessStats, IMG_PID ownerPid)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PROCESS_STATS *psProcessStats;
+	PVRSRV_DATA	*psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DevCount = 0;
+
+	if (psPVRSRVData != NULL)
+	{
+		ui32DevCount = psPVRSRVData->ui32RegisteredDevices;
+	}
+
+	psProcessStats = OSAllocZMemNoStats(sizeof(PVRSRV_PROCESS_STATS) +
+	                                    ui32DevCount * PVRSRV_DEVICE_STAT_TYPE_COUNT * sizeof(IMG_INT32));
+	PVR_RETURN_IF_NOMEM(psProcessStats);
+
+	psProcessStats->pid             = ownerPid;
+	psProcessStats->ui32RefCount    = 1;
+	psProcessStats->ui32DevCount    = ui32DevCount;
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	psProcessStats->psMemoryRecords = HASH_Create(HASH_INITIAL_SIZE);
+	PVR_GOTO_IF_NOMEM(psProcessStats->psMemoryRecords, eError, free_process_stats);
+#endif
+
+	eError = OSLockCreateNoStats(&psProcessStats->hLock);
+	PVR_GOTO_IF_ERROR(eError, destroy_mem_recs);
+
+	*ppsProcessStats = psProcessStats;
+	return PVRSRV_OK;
+
+destroy_mem_recs:
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	HASH_Delete(psProcessStats->psMemoryRecords);
+free_process_stats:
+#endif
+	OSFreeMemNoStats(psProcessStats);
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+}
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static PVRSRV_ERROR _FreeMemStatsEntry(uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	PVRSRV_MEM_ALLOC_REC *psRecord = (PVRSRV_MEM_ALLOC_REC *)(uintptr_t)v;
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	PVR_DPF((PVR_DBG_WARNING, "Mem Stats Record not freed: 0x%" IMG_UINT64_FMTSPECx " %p, size="IMG_SIZE_FMTSPEC", %s:%d",
+			 (IMG_UINT64)(k), psRecord, psRecord->uiBytes,
+			 (IMG_CHAR*)psRecord->pvAllocdFromFile, psRecord->ui32AllocdFromLine));
+#else
+	PVR_UNREFERENCED_PARAMETER(k);
+#endif
+	OSFreeMemNoStats(psRecord);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _DestroyProcessStat
+@Description    Frees memory and resources held by a process statistic.
+@Input          psProcessStats  Process stats to destroy.
+*/ /**************************************************************************/
+static void
+_DestroyProcessStat(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	PVR_ASSERT(psProcessStats != NULL);
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+	/* Free the memory statistics... */
+	HASH_Iterate(psProcessStats->psMemoryRecords, (HASH_pfnCallback)_FreeMemStatsEntry, NULL);
+	HASH_Delete(psProcessStats->psMemoryRecords);
+
+	OSLockRelease(psProcessStats->hLock);
+#endif
+
+	/*Destroy the lock */
+	OSLockDestroyNoStats(psProcessStats->hLock);
+
+	/* Free the memory... */
+	OSFreeMemNoStats(psProcessStats);
+} /* _DestroyProcessStat */
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+static inline void
+_createStatsFiles(PVRSRV_OS_STAT_ENTRY* psStatsEntries,
+                  DI_PFN_SHOW pfnStatsShow)
+{
+	PVRSRV_ERROR eError;
+	DI_ITERATOR_CB sIterator = {.pfnShow = pfnStatsShow};
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	eError = DICreateEntry("process_stats", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_PROCESS),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psProcessStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (1)");
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	eError = DICreateEntry("cache_ops_exec", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_CACHEOP),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psCacheOpStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (2)");
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	eError = DICreateEntry("mem_area", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_MEMORY),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psMemStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (3)");
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = DICreateEntry("gpu_mem_area", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_RIMEMORY),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psRIMemStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (4)");
+#endif
+}
+
+static inline void
+_createStatisticsEntries(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DICreateGroup("proc_stats", NULL, &psProcStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (1)");
+	eError = DICreateGroup("live_pids_stats", psProcStatsDIGroup,
+                           &gsLiveStatEntries.psStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (2)");
+	eError = DICreateGroup("retired_pids_stats", psProcStatsDIGroup,
+                           &gsRetiredStatEntries.psStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (3)");
+
+	_createStatsFiles(&gsLiveStatEntries, GenericStatsPrintElementsLive);
+	_createStatsFiles(&gsRetiredStatEntries, GenericStatsPrintElementsRetired);
+
+	_prepareStatsPrivateData();
+}
+
+static inline void
+_removeStatsFiles(PVRSRV_OS_STAT_ENTRY* psStatsEntries)
+{
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	DIDestroyEntry(psStatsEntries->psProcessStatsDIEntry);
+	psStatsEntries->psProcessStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	DIDestroyEntry(psStatsEntries->psCacheOpStatsDIEntry);
+    psStatsEntries->psCacheOpStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	DIDestroyEntry(psStatsEntries->psMemStatsDIEntry);
+	psStatsEntries->psMemStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	DIDestroyEntry(psStatsEntries->psRIMemStatsDIEntry);
+	psStatsEntries->psRIMemStatsDIEntry = NULL;
+#endif
+}
+
+static inline void
+_removeStatisticsEntries(void)
+{
+	_removeStatsFiles(&gsLiveStatEntries);
+	_removeStatsFiles(&gsRetiredStatEntries);
+
+	DIDestroyGroup(gsLiveStatEntries.psStatsDIGroup);
+	gsLiveStatEntries.psStatsDIGroup = NULL;
+	DIDestroyGroup(gsRetiredStatEntries.psStatsDIGroup);
+	gsRetiredStatEntries.psStatsDIGroup = NULL;
+	DIDestroyGroup(psProcStatsDIGroup);
+	psProcStatsDIGroup = NULL;
+}
+#endif
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+
+#define GENERATE_DKP_PRINT(_handle, _key, _value)                   \
+	PVRDKPOutput((_handle),                                         \
+	             "%s:%-*s%"IMG_UINT64_FMTSPEC"\n",                  \
+	             (_key),                                            \
+	             (int)(50 - sizeof((_key))),                        \
+	              "",                                               \
+	             (IMG_UINT64)(_value))
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#define GENERATE_PROCESS_STAT_FDINFO(_dkp_handle, _stat_type, _key, _process_stats) \
+	do { \
+		IMG_UINT32 ui32Value = MAX(0, (_process_stats)->i64StatValue[(_stat_type)]); \
+		if (((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) || \
+			((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES)) \
+		{ \
+			ui32Value = RITotalAllocProcessUnlocked((_process_stats)->pid, \
+			                                        ((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) ? PHYS_HEAP_TYPE_LMA : PHYS_HEAP_TYPE_UMA); \
+		} \
+		GENERATE_DKP_PRINT((_dkp_handle), (_key), ui32Value); \
+	} while (0)
+#else
+#define GENERATE_PROCESS_STAT_FDINFO(_dkp_handle, _stat_type, _key, _process_stats) \
+	GENERATE_DKP_PRINT((_dkp_handle), (_key), MAX(0, (_process_stats)->i64StatValue[(_stat_type)]));
+#endif
+
+static void _ProcessStatsDKPShow(PVRSRV_DEVICE_NODE *psDevNode,
+                                 int pid,
+                                 IMG_HANDLE hPrivData)
+{
+	PVRSRV_PROCESS_STATS *psProcessStats = _FindProcessStatsInLiveList((IMG_PID)pid);
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	if (psProcessStats == NULL)
+	{
+		/* Just return if the specified process is no longer in the process
+		 * stats list.
+		 */
+		return;
+	}
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-memory-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_MEMORY
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-shared-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_SHARED
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-total-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_TOTAL
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-resident-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_RESIDENT
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-purgeable-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_PURGEABLE
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-active-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_ACTIVE
+	#undef X
+}
+
+#undef GENERATE_PROCESS_STAT_FDINFO
+#undef GENERATE_DKP_PRINT
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsInitialise
+@Description    Entry point for initialising the statistics module.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVStatsInitialise(void)
+{
+	PVRSRV_ERROR error;
+
+	PVR_ASSERT(g_psLinkedListLock == NULL);
+	PVR_ASSERT(gpsSizeTrackingHashTable == NULL);
+	PVR_ASSERT(bProcessStatsInitialised == IMG_FALSE);
+
+	/* We need a lock to protect the linked lists... */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&g_psLinkedListLock);
+#else
+	error = OSLockCreate(&g_psLinkedListLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, return_);
+
+	/* We also need a lock to protect the hash table used for size tracking. */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&gpsSizeTrackingHashTableLock);
+#else
+	error = OSLockCreate(&gpsSizeTrackingHashTableLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, destroy_linked_list_lock_);
+
+	/* We also need a lock to protect the GlobalStat counters */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&gsGlobalStats.hGlobalStatsLock);
+#else
+	error = OSLockCreate(&gsGlobalStats.hGlobalStatsLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, destroy_hashtable_lock_);
+
+	/* Flag that we are ready to start monitoring memory allocations. */
+
+	gpsSizeTrackingHashTable = HASH_Create(HASH_INITIAL_SIZE);
+	PVR_GOTO_IF_NOMEM(gpsSizeTrackingHashTable, error, destroy_stats_lock_);
+
+	dllist_init(&gsLiveList);
+	dllist_init(&gsDeadList);
+
+	bProcessStatsInitialised = IMG_TRUE;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Register our 'system' PID to hold driver-wide alloc stats */
+	_RegisterProcess(&g_hDriverProcessStats, PVR_SYS_ALLOC_PID);
+#endif
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+	_createStatisticsEntries();
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = RawProcessStatsPrintElements};
+		error = DICreateEntry("memtrack_stats", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &psProcStatsDIEntry);
+		PVR_LOG_IF_ERROR(error, "DICreateEntry (1)");
+	}
+#endif
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = GlobalStatsPrintElements};
+		error = DICreateEntry("driver_stats", NULL, &sIterator, NULL,
+		                      DI_ENTRY_TYPE_GENERIC, &psGlobalMemDIEntry);
+		PVR_LOG_IF_ERROR(error, "DICreateEntry (3)");
+	}
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+	/* We might error here, but drm usage keys related to memory are optional.
+	 * We'll notify that the DKP registration has failed but continue as if nothing
+	 * happened. */
+	{
+		error = PVRSRVRegisterDKP(NULL,
+		                          "proc-stats",
+		                          &_ProcessStatsDKPShow,
+		                          DKP_CONNECTION_FLAG_SERVICES,
+		                          &ghDKPHandle);
+		if (error != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Failed to register process stats DKP", __func__));
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+destroy_stats_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(gsGlobalStats.hGlobalStatsLock);
+#else
+	OSLockDestroy(gsGlobalStats.hGlobalStatsLock);
+#endif
+	gsGlobalStats.hGlobalStatsLock = NULL;
+destroy_hashtable_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(gpsSizeTrackingHashTableLock);
+#else
+	OSLockDestroy(gpsSizeTrackingHashTableLock);
+#endif
+	gpsSizeTrackingHashTableLock = NULL;
+destroy_linked_list_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(g_psLinkedListLock);
+#else
+	OSLockDestroy(g_psLinkedListLock);
+#endif
+	g_psLinkedListLock = NULL;
+return_:
+	return error;
+
+}
+
+static PVRSRV_ERROR _DumpAllVMallocEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+#if defined(PVRSRV_NEED_PVR_DPF) || defined(DOXYGEN)
+	_PVR_STATS_TRACKING_HASH_ENTRY *psNewTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)(uintptr_t)v;
+	IMG_UINT64 uiCpuVAddr = (IMG_UINT64)k;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: " IMG_SIZE_FMTSPEC " bytes @ 0x%" IMG_UINT64_FMTSPECx " (PID %u)", __func__,
+	         psNewTrackingHashEntry->uiSizeInBytes,
+	         uiCpuVAddr,
+	         psNewTrackingHashEntry->uiPid));
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsDestroy
+@Description    Method for destroying the statistics module data.
+*/ /**************************************************************************/
+void
+PVRSRVStatsDestroy(void)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_ASSERT(bProcessStatsInitialised);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Deregister our 'system' PID which holds driver-wide alloc stats */
+	PVRSRVStatsDeregisterProcess(g_hDriverProcessStats);
+#endif
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+	if (ghDKPHandle != NULL)
+	{
+		PVRSRV_ERROR eError = PVRSRVUnRegisterDKP(NULL, ghDKPHandle);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to unregister process stats DKP", __func__));
+		}
+	}
+#endif
+
+	/* Stop monitoring memory allocations... */
+	bProcessStatsInitialised = IMG_FALSE;
+
+	/* Destroy the locks... */
+	if (g_psLinkedListLock != NULL)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(g_psLinkedListLock);
+#else
+		OSLockDestroy(g_psLinkedListLock);
+#endif
+		g_psLinkedListLock = NULL;
+	}
+
+	/* Free the live and dead lists... */
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		dllist_remove_node(&psProcessStats->sNode);
+		_DestroyProcessStat(psProcessStats);
+	}
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		dllist_remove_node(&psProcessStats->sNode);
+		_DestroyProcessStat(psProcessStats);
+	}
+
+	if (gpsSizeTrackingHashTable != NULL)
+	{
+		/* Dump all remaining entries in HASH table (list any remaining vmallocs) */
+		HASH_Iterate(gpsSizeTrackingHashTable, (HASH_pfnCallback)_DumpAllVMallocEntries, NULL);
+		HASH_Delete(gpsSizeTrackingHashTable);
+	}
+	if (gpsSizeTrackingHashTableLock != NULL)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(gpsSizeTrackingHashTableLock);
+#else
+		OSLockDestroy(gpsSizeTrackingHashTableLock);
+#endif
+		gpsSizeTrackingHashTableLock = NULL;
+	}
+
+	if (NULL != gsGlobalStats.hGlobalStatsLock)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(gsGlobalStats.hGlobalStatsLock);
+#else
+		OSLockDestroy(gsGlobalStats.hGlobalStatsLock);
+#endif
+		gsGlobalStats.hGlobalStatsLock = NULL;
+	}
+
+}
+
+void
+PVRSRVStatsDestroyDI(void)
+{
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+	if (psProcStatsDIEntry != NULL)
+	{
+		DIDestroyEntry(psProcStatsDIEntry);
+		psProcStatsDIEntry = NULL;
+	}
+#endif
+
+	/* Destroy the global data entry */
+	if (psGlobalMemDIEntry != NULL)
+	{
+		DIDestroyEntry(psGlobalMemDIEntry);
+		psGlobalMemDIEntry = NULL;
+	}
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+	_removeStatisticsEntries();
+#endif
+}
+
+static void _decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize;
+#endif
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	ui64InitialSize = GET_GPUMEM_GLOBAL_STAT_VALUE();
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_KMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			break;
+#endif
+
+		default:
+			PVR_ASSERT(0);
+			break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_GLOBAL_STAT_VALUE();
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemGlobal(0, ui64Size);
+		}
+	}
+#endif
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+}
+
+static void _increase_global_stat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize;
+#endif
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	ui64InitialSize = GET_GPUMEM_GLOBAL_STAT_VALUE();
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_KMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			break;
+#endif
+
+		default:
+			PVR_ASSERT(0);
+			break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_GLOBAL_STAT_VALUE();
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemGlobal(0, ui64Size);
+		}
+	}
+#endif
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+}
+
+static PVRSRV_ERROR
+_RegisterProcess(IMG_HANDLE *phProcessStats, IMG_PID ownerPid)
+{
+	PVRSRV_PROCESS_STATS*	psProcessStats=NULL;
+	PVRSRV_ERROR			eError;
+
+	PVR_ASSERT(phProcessStats != NULL);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Register process PID %d [%s]",
+			__func__, ownerPid, (ownerPid == PVR_SYS_ALLOC_PID)
+			? "system" : OSGetCurrentClientProcessNameKM()));
+
+	/* Check the PID has not already moved to the dead list... */
+	OSLockAcquire(g_psLinkedListLock);
+	psProcessStats = _FindProcessStatsInDeadList(ownerPid);
+	if (psProcessStats != NULL)
+	{
+		/* Move it back onto the live list! */
+		_MoveProcessToLiveList(psProcessStats);
+	}
+	else
+	{
+		/* Check the PID is not already registered in the live list... */
+		psProcessStats = _FindProcessStatsInLiveList(ownerPid);
+	}
+
+	/* If the PID is on the live list then just increment the ref count and return... */
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		psProcessStats->ui32RefCount++;
+
+		OSLockRelease(psProcessStats->hLock);
+		OSLockRelease(g_psLinkedListLock);
+
+		*phProcessStats = psProcessStats;
+
+		return PVRSRV_OK;
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	/* Allocate a new node structure and initialise it... */
+	eError = _AllocateProcessStats(&psProcessStats, ownerPid);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	OSStringSafeCopy(psProcessStats->processName, OSGetCurrentProcessName(), MAX_PROC_NAME_LEN);
+
+	/* Add it to the live list... */
+	OSLockAcquire(g_psLinkedListLock);
+	dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+	OSLockRelease(g_psLinkedListLock);
+
+	/* Done */
+	*phProcessStats = (IMG_HANDLE) psProcessStats;
+
+	return PVRSRV_OK;
+
+e0:
+	*phProcessStats = (IMG_HANDLE) NULL;
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+} /* _RegisterProcess */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsRegisterProcess
+@Description    Register a process into the list statistics list.
+@Output         phProcessStats  Handle to the process to be used to deregister.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVStatsRegisterProcess(IMG_HANDLE* phProcessStats)
+{
+	return _RegisterProcess(phProcessStats, OSGetCurrentClientProcessIDKM());
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsDeregisterProcess
+@Input          hProcessStats  Handle to the process returned when registered.
+@Description    Method for destroying the statistics module data.
+*/ /**************************************************************************/
+void
+PVRSRVStatsDeregisterProcess(IMG_HANDLE hProcessStats)
+{
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Deregister process entered PID %d [%s]",
+			__func__, OSGetCurrentClientProcessIDKM(),
+			OSGetCurrentProcessName()));
+
+	if (hProcessStats != (IMG_HANDLE) NULL)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = (PVRSRV_PROCESS_STATS*) hProcessStats;
+
+		/* Lower the reference count, if zero then move it to the dead list */
+		OSLockAcquire(g_psLinkedListLock);
+		if (psProcessStats->ui32RefCount > 0)
+		{
+			OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+			psProcessStats->ui32RefCount--;
+
+#if !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+			if (psProcessStats->ui32RefCount == 0)
+			{
+				OSLockRelease(psProcessStats->hLock);
+				_MoveProcessToDeadList(psProcessStats);
+			}else
+#endif
+			{
+				OSLockRelease(psProcessStats->hLock);
+			}
+		}
+		OSLockRelease(g_psLinkedListLock);
+
+		/* Check if the dead list needs to be reduced */
+		_CompressMemoryUsage();
+	}
+} /* PVRSRVStatsDeregisterProcess */
+
+PVRSRV_ERROR PVRSRVStatsDeviceConnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+	IMG_PID ownerPid = OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS*	psProcessStats;
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStatsInLiveList(ownerPid);
+
+	if (psProcessStats != NULL)
+	{
+		if ((ui32DevID < psProcessStats->ui32DevCount) ||
+		    (ui32DevID == 0 && psProcessStats->ui32DevCount == 0))
+		{
+			psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]++;
+			UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_MAX_CONNECTIONS],
+			                 psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device index %d is greater than device count %d for PID %d.",
+			         __func__, ui32DevID, psProcessStats->ui32DevCount, ownerPid));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Process %d not found.",
+				 __func__, ownerPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVStatsDeviceDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+	IMG_PID	currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA* psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_PID currentPid = OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS*	psProcessStats;
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+	}
+
+	if (psProcessStats != NULL)
+	{
+		if ((ui32DevID < psProcessStats->ui32DevCount) ||
+		    (ui32DevID == 0 && psProcessStats->ui32DevCount == 0))
+		{
+			psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]--;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device index %d is greater than device count %d for PID %d.",
+					 __func__, ui32DevID, psProcessStats->ui32DevCount, currentPid));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Process %d not found.",
+				 __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+/* Assumes PVRSRV_PROCESS_STATS lock (psProcessStats->hLock) has already been taken */
+static void
+_IncreaseProcStatValueUnlocked(PVRSRV_PROCESS_STATS* psProcessStats,
+                               PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                               size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+#endif
+
+	/* Update the memory watermarks... */
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_KMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMALLOC, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		default:
+		{
+			PVR_ASSERT(0);
+		}
+		break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemPerProcess(0, psProcessStats->pid,
+			                                 ui64Size);
+		}
+	}
+#endif
+}
+
+/* Assumes PVRSRV_PROCESS_STATS lock (psProcessStats->hLock) has already been taken */
+static void
+_DecreaseProcStatValueUnlocked(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                               PVRSRV_PROCESS_STATS* psProcessStats,
+                               IMG_UINT64 uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_KMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMALLOC, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMALLOC] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		default:
+		{
+			PVR_ASSERT(0);
+		}
+		break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemPerProcess(0, psProcessStats->pid, ui64Size);
+		}
+	}
+#endif
+}
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static INLINE IMG_BOOL
+_CheckKeyPhysAddr(PVRSRV_MEM_ALLOC_TYPE eAllocType)
+{
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+#endif
+			return IMG_TRUE;
+		default:
+			return IMG_FALSE;
+	}
+}
+#endif
+
+void
+PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+							 void *pvCpuVAddr,
+							 IMG_CPU_PHYADDR sCpuPAddr,
+							 size_t uiBytes,
+							 IMG_PID currentPid
+							 DEBUG_MEMSTATS_PARAMS)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_MEM_ALLOC_REC*  psRecord = NULL;
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+	__maybe_unused PVRSRV_PROC_SEARCH_STATE eProcSearch = PVRSRV_PROC_FOUND;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/*
+	 * To prevent a recursive loop, we make the memory allocations for our
+	 * memstat records via OSAllocMemNoStats(), which does not try to
+	 * create a memstat record entry.
+	 */
+
+	/* Allocate the memory record... */
+	psRecord = OSAllocZMemNoStats(sizeof(PVRSRV_MEM_ALLOC_REC));
+	if (psRecord == NULL)
+	{
+		return;
+	}
+
+	psRecord->eAllocType       = eAllocType;
+	psRecord->pvCpuVAddr       = pvCpuVAddr;
+	psRecord->sCpuPAddr.uiAddr = sCpuPAddr.uiAddr;
+	psRecord->uiBytes          = uiBytes;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	psRecord->pvAllocdFromFile = pvAllocFromFile;
+	psRecord->ui32AllocdFromLine = ui32AllocFromLine;
+#endif
+
+	_increase_global_stat(eAllocType, uiBytes);
+	/* Lock while we find the correct process... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+			if (!psProcessStats)
+			{
+				psProcessStats = _FindProcessStatsInDeadList(currentPid);
+				eProcSearch = PVRSRV_PROC_RESURRECTED;
+			}
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		if (!psProcessStats)
+		{
+			psProcessStats = _FindProcessStatsInDeadList(currentPid);
+			eProcSearch = PVRSRV_PROC_RESURRECTED;
+		}
+	}
+
+	if (psProcessStats == NULL)
+	{
+		eProcSearch = PVRSRV_PROC_NOTFOUND;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat increment called for 'unknown' process PID(%d)",
+				 __func__, currentPid));
+
+		if (_AllocateProcessStats(&psProcessStats, currentPid) != PVRSRV_OK)
+		{
+			OSLockRelease(g_psLinkedListLock);
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s UNABLE TO CREATE process_stats entry for pid %d [%s] (" IMG_SIZE_FMTSPEC " bytes)",
+			        __func__, currentPid, OSGetCurrentProcessName(), uiBytes));
+			goto free_record;
+		}
+
+		/* Add it to the live list... */
+		dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+
+		OSLockRelease(g_psLinkedListLock);
+
+#else /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+		OSLockRelease(g_psLinkedListLock);
+		goto free_record;
+#endif /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+	}
+	else
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		if (eProcSearch == PVRSRV_PROC_RESURRECTED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat incremented on 'dead' process PID(%d)",
+				 __func__, currentPid));
+			/* Move process from dead list to live list */
+			_MoveProcessToLiveList(psProcessStats);
+		}
+#endif
+		OSLockRelease(g_psLinkedListLock);
+	}
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_UINT64 ui64Key;
+
+		if (_CheckKeyPhysAddr(eAllocType))
+		{
+			ui64Key = psRecord->sCpuPAddr.uiAddr;
+		}
+		else
+		{
+			ui64Key = (IMG_UINT64)psRecord->pvCpuVAddr;
+		}
+
+		/* Insert the memory record... */
+		if (!HASH_Insert(psProcessStats->psMemoryRecords, ui64Key, (uintptr_t)psRecord))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s UNABLE TO CREATE mem stats record for pid %d [%s] (" IMG_SIZE_FMTSPEC " bytes)",
+					 __func__, currentPid, OSGetCurrentProcessName(), uiBytes));
+		}
+	}
+#endif
+
+	/* Update the memory watermarks... */
+	_IncreaseProcStatValueUnlocked(psProcessStats,
+	                               eAllocType,
+	                               uiBytes);
+
+	OSLockRelease(psProcessStats->hLock);
+
+	return;
+
+free_record:
+	_decrease_global_stat(eAllocType, uiBytes);
+	if (psRecord != NULL)
+	{
+		OSFreeMemNoStats(psRecord);
+	}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	PVR_UNREFERENCED_PARAMETER(eAllocType);
+	PVR_UNREFERENCED_PARAMETER(pvCpuVAddr);
+	PVR_UNREFERENCED_PARAMETER(sCpuPAddr);
+	PVR_UNREFERENCED_PARAMETER(uiBytes);
+	PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+} /* PVRSRVStatsAddMemAllocRecord */
+
+void
+PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								IMG_UINT64 ui64Key,
+								IMG_PID currentPid)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+	PVRSRV_MEM_ALLOC_REC*  psRecord		  = NULL;
+	IMG_BOOL			   bFound	      = IMG_FALSE;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/* Lock while we find the correct process and remove this record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+		OSLockRelease(psProcessStats->hLock);
+		bFound = psRecord != NULL;
+	}
+
+	/* If not found, we need to do a full search in case it was allocated to a different PID... */
+	if (!bFound)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStatsAlreadyChecked = psProcessStats;
+		DLLIST_NODE *psNode, *psNext;
+
+		/* Search all live lists first... */
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			if (psProcessStats != psProcessStatsAlreadyChecked)
+			{
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+				psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+				OSLockRelease(psProcessStats->hLock);
+				bFound = psRecord != NULL;
+			}
+
+			if (bFound)
+			{
+				break;
+			}
+		}
+
+		/* If not found, then search all dead lists next... */
+		if (!bFound)
+		{
+			dllist_foreach_node(&gsDeadList, psNode, psNext)
+			{
+				psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+				if (psProcessStats != psProcessStatsAlreadyChecked)
+				{
+					OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+					psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+					OSLockRelease(psProcessStats->hLock);
+					bFound = psRecord != NULL;
+				}
+
+				if (bFound)
+				{
+					break;
+				}
+			}
+		}
+	}
+
+	/* Update the watermark and remove this record... */
+	if (bFound)
+	{
+		_decrease_global_stat(eAllocType, psRecord->uiBytes);
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               psRecord->uiBytes);
+
+		OSLockRelease(psProcessStats->hLock);
+		OSLockRelease(g_psLinkedListLock);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		/* If all stats are now zero, remove the entry for this thread */
+		if (psProcessStats->ui32StatAllocFlags == 0)
+		{
+			OSLockAcquire(g_psLinkedListLock);
+			_MoveProcessToDeadList(psProcessStats);
+			OSLockRelease(g_psLinkedListLock);
+
+			/* Check if the dead list needs to be reduced */
+			_CompressMemoryUsage();
+		}
+#endif
+		/*
+		 * Free the record outside the lock so we don't deadlock and so we
+		 * reduce the time the lock is held.
+		 */
+		OSFreeMemNoStats(psRecord);
+	}
+	else
+	{
+		OSLockRelease(g_psLinkedListLock);
+	}
+
+#else
+	PVR_UNREFERENCED_PARAMETER(eAllocType);
+	PVR_UNREFERENCED_PARAMETER(ui64Key);
+	PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif
+} /* PVRSRVStatsRemoveMemAllocRecord */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void
+PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eCurrentType,
+                                  PVRSRV_MEM_ALLOC_TYPE eTargetType,
+                                  IMG_UINT64 ui64Key,
+                                  IMG_PID currentPid
+                                  DEBUG_MEMSTATS_PARAMS)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+	PVRSRV_MEM_ALLOC_REC*  psRecord		  = NULL;
+	IMG_BOOL			   bFound	      = IMG_FALSE;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	if (_CheckKeyPhysAddr(eCurrentType) != _CheckKeyPhysAddr(eTargetType))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Key type used for current type must match target type, "
+		                        "Record transfer failed!"));
+		return;
+	}
+
+	/* Lock while we find the correct process and remove this record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+		OSLockRelease(psProcessStats->hLock);
+		bFound = psRecord != NULL;
+	}
+
+	/* If not found, we need to do a full search in case it was allocated to a different PID... */
+	if (!bFound)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStatsAlreadyChecked = psProcessStats;
+		DLLIST_NODE *psNode, *psNext;
+
+		/* Search all live lists first... */
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			if (psProcessStats != psProcessStatsAlreadyChecked)
+			{
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+				psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+				OSLockRelease(psProcessStats->hLock);
+				bFound = psRecord != NULL;
+			}
+
+			if (bFound)
+			{
+				break;
+			}
+		}
+	}
+
+	if (bFound)
+	{
+		/* Update the current type watermark by reducing */
+		_decrease_global_stat(eCurrentType, psRecord->uiBytes);
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		_DecreaseProcStatValueUnlocked(eCurrentType,
+		                               psProcessStats,
+		                               psRecord->uiBytes);
+
+		/* Change the type of the record */
+		psRecord->eAllocType = eTargetType;
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+		/* Adjust where the record transfer originated */
+		psRecord->pvAllocdFromFile = pvAllocFromFile;
+		psRecord->ui32AllocdFromLine = ui32AllocFromLine;
+#endif
+
+		/* Update the target watermark by increasing */
+		_IncreaseProcStatValueUnlocked(psProcessStats,
+		                               eTargetType,
+		                               psRecord->uiBytes);
+		OSLockRelease(psProcessStats->hLock);
+
+		_increase_global_stat(eTargetType, psRecord->uiBytes);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+#else
+PVR_UNREFERENCED_PARAMETER(eCurrentType);
+PVR_UNREFERENCED_PARAMETER(eTargetType);
+PVR_UNREFERENCED_PARAMETER(ui64Key);
+PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+void
+PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									size_t uiBytes,
+									IMG_UINT64 uiCpuVAddr,
+									IMG_PID uiPid)
+{
+	IMG_BOOL bRes = IMG_FALSE;
+	_PVR_STATS_TRACKING_HASH_ENTRY *psNewTrackingHashEntry = NULL;
+
+	if (!bProcessStatsInitialised || (gpsSizeTrackingHashTable == NULL))
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/* Alloc untracked memory for the new hash table entry */
+	psNewTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)OSAllocMemNoStats(sizeof(*psNewTrackingHashEntry));
+	if (psNewTrackingHashEntry == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "*** %s : @ line %d Failed to alloc memory for psNewTrackingHashEntry!",
+				 __func__, __LINE__));
+		return;
+	}
+
+	/* Fill-in the size of the allocation and PID of the allocating process */
+	psNewTrackingHashEntry->uiSizeInBytes = uiBytes;
+	psNewTrackingHashEntry->uiPid = uiPid;
+	OSLockAcquire(gpsSizeTrackingHashTableLock);
+	/* Insert address of the new struct into the hash table */
+	bRes = HASH_Insert(gpsSizeTrackingHashTable, uiCpuVAddr, (uintptr_t)psNewTrackingHashEntry);
+	OSLockRelease(gpsSizeTrackingHashTableLock);
+	if (bRes)
+	{
+		PVRSRVStatsIncrMemAllocStat(eAllocType, uiBytes, uiPid);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "*** %s : @ line %d HASH_Insert() failed!",
+				 __func__, __LINE__));
+		/* Free the memory allocated for psNewTrackingHashEntry, as we
+		 * failed to insert it into the Hash table.
+		 */
+		OSFreeMemNoStats(psNewTrackingHashEntry);
+	}
+}
+
+void
+PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                            size_t uiBytes,
+                            IMG_PID currentPid)
+
+{
+	IMG_PID				  currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		  psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS* psProcessStats = NULL;
+	__maybe_unused PVRSRV_PROC_SEARCH_STATE eProcSearch = PVRSRV_PROC_FOUND;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	_increase_global_stat(eAllocType, uiBytes);
+	OSLockAcquire(g_psLinkedListLock);
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+			if (!psProcessStats)
+			{
+				psProcessStats = _FindProcessStatsInDeadList(currentPid);
+				eProcSearch = PVRSRV_PROC_RESURRECTED;
+			}
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		if (!psProcessStats)
+		{
+			psProcessStats = _FindProcessStatsInDeadList(currentPid);
+			eProcSearch = PVRSRV_PROC_RESURRECTED;
+		}
+	}
+
+	if (psProcessStats == NULL)
+	{
+		eProcSearch = PVRSRV_PROC_NOTFOUND;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat increment called for 'unknown' process PID(%d)",
+				 __func__, currentPid));
+
+		if (bProcessStatsInitialised)
+		{
+			if (_AllocateProcessStats(&psProcessStats, currentPid) != PVRSRV_OK)
+			{
+				OSLockRelease(g_psLinkedListLock);
+				return;
+			}
+			/* Add it to the live list... */
+			dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+		}
+#else
+		OSLockRelease(g_psLinkedListLock);
+#endif /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+
+	}
+
+	if (psProcessStats != NULL)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		if (eProcSearch == PVRSRV_PROC_RESURRECTED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "%s: Process stat incremented on 'dead' process PID(%d)",
+					 __func__, currentPid));
+
+			/* Move process from dead list to live list */
+			_MoveProcessToLiveList(psProcessStats);
+		}
+#endif
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Release the list lock as soon as we acquire the process lock,
+		 * this ensures if the process is in deadlist the entry cannot be
+		 * deleted or modified
+		 */
+		OSLockRelease(g_psLinkedListLock);
+
+		/* Update the memory watermarks... */
+		_IncreaseProcStatValueUnlocked(psProcessStats,
+		                               eAllocType,
+		                               uiBytes);
+
+		OSLockRelease(psProcessStats->hLock);
+	}
+}
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+int RawProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_PROCESS_STATS *psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	DIPrintf(psEntry,
+	         "%s,%s,%s,%s,%s,%s,%s\n",
+	         "PID",
+	         "MemoryUsageKMalloc",           // PVRSRV_PROCESS_STAT_TYPE_KMALLOC
+	         "MemoryUsageAllocPTMemoryUMA",  // PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA
+	         "MemoryUsageAllocPTMemoryLMA",  // PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA
+	         "MemoryUsageAllocGPUMemLMA",    // PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES
+	         "MemoryUsageAllocGPUMemUMA",    // PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES
+	         "MemoryUsageDmaBufImport");     // PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+		{
+			DIPrintf(psEntry,
+			         "%d,%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd","
+			         "%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd","
+			         "%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd"\n",
+			         psProcessStats->pid,
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT]);
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+} /* RawProcessStatsPrintElements */
+#endif
+
+void
+PVRSRVStatsDecrMemKAllocStat(size_t uiBytes,
+                             IMG_PID decrPID)
+{
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE_KMALLOC, uiBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(decrPID);
+
+	if (psProcessStats != NULL)
+	{
+		/* Decrement the kmalloc memory stat... */
+		DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+		DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+static void
+_StatsDecrMemTrackedStat(_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry,
+                        PVRSRV_MEM_ALLOC_TYPE eAllocType)
+{
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(eAllocType, psTrackingHashEntry->uiSizeInBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(psTrackingHashEntry->uiPid);
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Decrement the memory stat... */
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               psTrackingHashEntry->uiSizeInBytes);
+		OSLockRelease(psProcessStats->hLock);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+void
+PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									  IMG_UINT64 uiCpuVAddr)
+{
+	_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry = NULL;
+
+	if (!bProcessStatsInitialised || (gpsSizeTrackingHashTable == NULL))
+	{
+		return;
+	}
+
+	OSLockAcquire(gpsSizeTrackingHashTableLock);
+	psTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)HASH_Remove(gpsSizeTrackingHashTable, uiCpuVAddr);
+	OSLockRelease(gpsSizeTrackingHashTableLock);
+	if (psTrackingHashEntry)
+	{
+		_StatsDecrMemTrackedStat(psTrackingHashEntry, eAllocType);
+		OSFreeMemNoStats(psTrackingHashEntry);
+	}
+}
+
+void
+PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+							size_t uiBytes,
+							IMG_PID currentPid)
+{
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(eAllocType, uiBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Release the list lock as soon as we acquire the process lock,
+		 * this ensures if the process is in deadlist the entry cannot be
+		 * deleted or modified
+		 */
+		OSLockRelease(g_psLinkedListLock);
+		/* Update the memory watermarks... */
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               uiBytes);
+		OSLockRelease(psProcessStats->hLock);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		/* If all stats are now zero, remove the entry for this thread */
+		if (psProcessStats->ui32StatAllocFlags == 0)
+		{
+			OSLockAcquire(g_psLinkedListLock);
+			_MoveProcessToDeadList(psProcessStats);
+			OSLockRelease(g_psLinkedListLock);
+
+			/* Check if the dead list needs to be reduced */
+			_CompressMemoryUsage();
+		}
+#endif
+	}else{
+		OSLockRelease(g_psLinkedListLock);
+	}
+}
+
+/* For now we do not want to expose the global stats API
+ * so we wrap it into this specific function for pooled pages.
+ * As soon as we need to modify the global stats directly somewhere else
+ * we want to replace these functions with more general ones.
+ */
+void
+PVRSRVStatsIncrMemAllocPoolStat(size_t uiBytes)
+{
+	_increase_global_stat(PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES, uiBytes);
+}
+
+void
+PVRSRVStatsDecrMemAllocPoolStat(size_t uiBytes)
+{
+	_decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES, uiBytes);
+}
+
+PVRSRV_ERROR
+PVRSRVStatsUpdateOOMStat(CONNECTION_DATA *psConnection,
+						  PVRSRV_DEVICE_NODE *psDeviceNode,
+						  IMG_UINT32 ui32OOMStatType,
+						  IMG_PID pidOwner)
+{
+	PVRSRV_DEVICE_STAT_TYPE eOOMStatType = (PVRSRV_DEVICE_STAT_TYPE) ui32OOMStatType;
+	IMG_PID	pidCurrent = pidOwner;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32OOMStatType >= PVRSRV_DEVICE_STAT_TYPE_COUNT)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(pidCurrent);
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[psDeviceNode->sDevId.ui32InternalID][eOOMStatType]++;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PVRSRVStatsUpdateOOMStat: Process not found for Pid=%d", pidCurrent));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+} /* PVRSRVStatsUpdateOOMStat */
+
+void
+PVRSRVStatsUpdateRenderContextStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32TotalNumPartialRenders,
+									IMG_UINT32 ui32TotalNumOutOfMemory,
+									IMG_UINT32 ui32NumTAStores,
+									IMG_UINT32 ui32Num3DStores,
+									IMG_UINT32 ui32NumCDMStores,
+									IMG_UINT32 ui32NumTDMStores,
+									IMG_UINT32 ui32NumRayStores,
+									IMG_PID pidOwner)
+{
+	IMG_PID	pidCurrent = pidOwner;
+
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(pidCurrent);
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_PRS]       += ui32TotalNumPartialRenders;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_OOMS]      += ui32TotalNumOutOfMemory;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_TA_STORES] += ui32NumTAStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_3D_STORES] += ui32Num3DStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_CDM_STORES]+= ui32NumCDMStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_TDM_STORES]+= ui32NumTDMStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_RAY_STORES]+= ui32NumRayStores;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PVRSRVStatsUpdateRenderContextStats: Process not found for Pid=%d", pidCurrent));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateRenderContextStats */
+
+void
+PVRSRVStatsUpdateZSBufferStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+							   IMG_UINT32 ui32NumReqByApp,
+							   IMG_UINT32 ui32NumReqByFW,
+							   IMG_PID owner)
+{
+	IMG_PID				  currentPid = (owner==0)?OSGetCurrentClientProcessIDKM():owner;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_APP] += ui32NumReqByApp;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_FW] += ui32NumReqByFW;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Process not found for Pid=%d", __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateZSBufferStats */
+
+void
+PVRSRVStatsUpdateFreelistStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+							   IMG_UINT32 ui32NumGrowReqByApp,
+							   IMG_UINT32 ui32NumGrowReqByFW,
+							   IMG_UINT32 ui32InitFLPages,
+							   IMG_UINT32 ui32NumHighPages,
+							   IMG_PID ownerPid)
+{
+	IMG_PID				  currentPid = (ownerPid!=0)?ownerPid:OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_APP] += ui32NumGrowReqByApp;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_FW] += ui32NumGrowReqByFW;
+
+		UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_PAGES_INIT],
+						 (IMG_INT32) ui32InitFLPages);
+
+		UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_MAX_PAGES],
+						 (IMG_INT32) ui32NumHighPages);
+		OSLockRelease(psProcessStats->hLock);
+
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Process not found for Pid=%d", __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateFreelistStats */
+
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+
+int
+GenericStatsPrintElementsLive(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_STAT_PV_DATA *psStatType = DIGetPrivData(psEntry);
+	PVRSRV_PROCESS_STATS* psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	PVR_ASSERT(psStatType->pfnStatsPrintElements != NULL);
+
+	DIPrintf(psEntry, "%s\n", psStatType->szLiveStatsHeaderStr);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (dllist_is_empty(&gsLiveList))
+	{
+		DIPrintf(psEntry, "No Stats to display\n%s\n", g_szSeparatorStr);
+	}
+	else
+	{
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			psStatType->pfnStatsPrintElements(psEntry, psProcessStats);
+			DIPrintf(psEntry, "%s\n", g_szSeparatorStr);
+		}
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+}
+
+int
+GenericStatsPrintElementsRetired(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_STAT_PV_DATA *psStatType = DIGetPrivData(psEntry);
+	PVRSRV_PROCESS_STATS* psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	PVR_ASSERT(psStatType->pfnStatsPrintElements != NULL);
+
+	DIPrintf(psEntry, "%s\n", psStatType->szRetiredStatsHeaderStr);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (dllist_is_empty(&gsDeadList))
+	{
+		DIPrintf(psEntry, "No Stats to display\n%s\n", g_szSeparatorStr);
+	}
+	else
+	{
+		dllist_foreach_node(&gsDeadList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			psStatType->pfnStatsPrintElements(psEntry, psProcessStats);
+			DIPrintf(psEntry, "%s\n", g_szSeparatorStr);
+		}
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+/*************************************************************************/ /*!
+@Function       ProcessStatsPrintElements
+@Description    Prints all elements for this process statistic record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+ProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                          PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_UINT32 ui32StatNumber;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RILockAcquireKM();
+#endif
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+	DIPrintf(psEntry, "PID %u - %s\n", psProcessStats->pid, psProcessStats->processName);
+
+	/* Print device stats table PVRSRV_DEVICE_STAT_TYPE */
+	if (psProcessStats->ui32DevCount > 0)
+	{
+		IMG_UINT32 i;
+
+		for (ui32StatNumber = 0;
+			 ui32StatNumber < ARRAY_SIZE(pszDeviceStatType);
+			 ui32StatNumber++)
+		{
+			if (OSStringNCompare(pszDeviceStatType[ui32StatNumber], "", 1) != 0)
+			{
+				DIPrintf(psEntry, "%-34s",
+						 pszDeviceStatType[ui32StatNumber]);
+
+				for (i = 0; i < psProcessStats->ui32DevCount; i++)
+				{
+					if (i == 0)
+					{
+						DIPrintf(psEntry, "%10d",
+								 psProcessStats->ai32DevStats[i][ui32StatNumber]);
+					}
+					else
+					{
+						DIPrintf(psEntry, ",%d",
+								 psProcessStats->ai32DevStats[i][ui32StatNumber]);
+					}
+				}
+			}
+
+			DIPrintf(psEntry, "\n");
+		}
+	}
+
+	/* Print process memory stats table PVRSRV_PROCESS_STAT_TYPE */
+	for (ui32StatNumber = 0;
+	     ui32StatNumber < ARRAY_SIZE(pszProcessStatType);
+	     ui32StatNumber++)
+	{
+		if (OSStringNCompare(pszProcessStatType[ui32StatNumber], "", 1) != 0)
+		{
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+			if ((ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) ||
+			    (ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES))
+			{
+				/* get the stat from RI */
+				IMG_INT32 ui32Total = RITotalAllocProcessUnlocked(psProcessStats->pid,
+									    (ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES)
+									    ? PHYS_HEAP_TYPE_LMA : PHYS_HEAP_TYPE_UMA);
+
+				DIPrintf(psEntry, "%-34s%10d %8dK\n",
+						 pszProcessStatType[ui32StatNumber], ui32Total, ui32Total>>10);
+			}
+			else
+#endif
+			{
+				if (ui32StatNumber >= PVRSRV_PROCESS_STAT_TYPE_KMALLOC &&
+					ui32StatNumber <= PVRSRV_PROCESS_STAT_TYPE_TOTAL_MAX)
+				{
+					DIPrintf(psEntry, "%-34s%10"IMG_INT64_FMTSPECd" %8"IMG_INT64_FMTSPECd"K\n",
+					         pszProcessStatType[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber] >> 10);
+				}
+				else
+				{
+					DIPrintf(psEntry, "%-34s%10"IMG_INT64_FMTSPECd"\n",
+					         pszProcessStatType[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber]);
+				}
+			}
+		}
+	}
+
+	OSLockRelease(psProcessStats->hLock);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RILockReleaseKM();
+#endif
+} /* ProcessStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void
+PVRSRVStatsUpdateCacheOpStats(PVRSRV_CACHE_OP uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEV_PHYADDR sDevPAddr,
+#endif
+							IMG_DEVMEM_SIZE_T uiOffset,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT64 ui64ExecuteTime,
+							IMG_BOOL bUserModeFlush,
+							IMG_PID ownerPid)
+{
+	IMG_PID				  currentPid = (ownerPid!=0)?ownerPid:OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+
+	if (psProcessStats != NULL)
+	{
+		IMG_INT32 Idx;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		/* Look-up next buffer write index */
+		Idx = psProcessStats->uiCacheOpWriteIndex;
+		psProcessStats->uiCacheOpWriteIndex = INCREMENT_CACHEOP_STAT_IDX_WRAP(Idx);
+
+		/* Store all CacheOp meta-data */
+		psProcessStats->asCacheOp[Idx].uiCacheOp = uiCacheOp;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+		psProcessStats->asCacheOp[Idx].sDevVAddr = sDevVAddr;
+		psProcessStats->asCacheOp[Idx].sDevPAddr = sDevPAddr;
+#endif
+		psProcessStats->asCacheOp[Idx].uiOffset = uiOffset;
+		psProcessStats->asCacheOp[Idx].uiSize = uiSize;
+		psProcessStats->asCacheOp[Idx].bUserModeFlush = bUserModeFlush;
+		psProcessStats->asCacheOp[Idx].ui64ExecuteTime = ui64ExecuteTime;
+
+		OSLockRelease(psProcessStats->hLock);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateCacheOpStats */
+
+/*************************************************************************/ /*!
+@Function       CacheOpStatsPrintElements
+@Description    Prints all elements for this process statistic CacheOp record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+CacheOpStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                          PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_CHAR  *pszCacheOpType, *pszFlushType, *pszFlushMode;
+	IMG_INT32 i32WriteIdx, i32ReadIdx;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	#define CACHEOP_RI_PRINTF_HEADER \
+		"%-10s %-10s %-5s %-16s %-16s %-10s %-10s %-12s\n"
+	#define CACHEOP_RI_PRINTF		\
+		"%-10s %-10s %-5s 0x%-14llx 0x%-14llx 0x%-8llx 0x%-8llx %-12llu\n"
+#else
+	#define CACHEOP_PRINTF_HEADER	\
+		"%-10s %-10s %-5s %-10s %-10s %-12s\n"
+	#define CACHEOP_PRINTF			\
+		"%-10s %-10s %-5s 0x%-8llx 0x%-8llx %-12llu\n"
+#endif
+
+	DIPrintf(psEntry, "PID %u - %s\n", psProcessStats->pid, psProcessStats->processName);
+
+	/* File header info */
+	DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+					CACHEOP_RI_PRINTF_HEADER,
+#else
+					CACHEOP_PRINTF_HEADER,
+#endif
+					"CacheOp",
+					"Type",
+					"Mode",
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+					"DevVAddr",
+					"DevPAddr",
+#endif
+					"Offset",
+					"Size",
+					"Time (us)");
+
+	/* Take a snapshot of write index, read backwards in buffer
+	   and wrap round at boundary */
+	i32WriteIdx = psProcessStats->uiCacheOpWriteIndex;
+	for (i32ReadIdx = DECREMENT_CACHEOP_STAT_IDX_WRAP(i32WriteIdx);
+		 i32ReadIdx != i32WriteIdx;
+		 i32ReadIdx = DECREMENT_CACHEOP_STAT_IDX_WRAP(i32ReadIdx))
+	{
+		IMG_UINT64 ui64ExecuteTime = psProcessStats->asCacheOp[i32ReadIdx].ui64ExecuteTime;
+		IMG_DEVMEM_SIZE_T ui64NumOfPages = psProcessStats->asCacheOp[i32ReadIdx].uiSize >> OSGetPageShift();
+
+		if (ui64NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			pszFlushType = "RBF.Fast";
+		}
+		else
+		{
+			pszFlushType = "RBF.Slow";
+		}
+
+		if (psProcessStats->asCacheOp[i32ReadIdx].bUserModeFlush)
+		{
+			pszFlushMode = "UM";
+		}
+		else
+		{
+			pszFlushMode = "KM";
+		}
+
+		switch (psProcessStats->asCacheOp[i32ReadIdx].uiCacheOp)
+		{
+			case PVRSRV_CACHE_OP_NONE:
+				pszCacheOpType = "None";
+				break;
+			case PVRSRV_CACHE_OP_CLEAN:
+				pszCacheOpType = "Clean";
+				break;
+			case PVRSRV_CACHE_OP_INVALIDATE:
+				pszCacheOpType = "Invalidate";
+				break;
+			case PVRSRV_CACHE_OP_FLUSH:
+				pszCacheOpType = "Flush";
+				break;
+			default:
+				pszCacheOpType = "Unknown";
+				break;
+		}
+
+		DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							CACHEOP_RI_PRINTF,
+#else
+							CACHEOP_PRINTF,
+#endif
+							pszCacheOpType,
+							pszFlushType,
+							pszFlushMode,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							psProcessStats->asCacheOp[i32ReadIdx].sDevVAddr.uiAddr,
+							psProcessStats->asCacheOp[i32ReadIdx].sDevPAddr.uiAddr,
+#endif
+							psProcessStats->asCacheOp[i32ReadIdx].uiOffset,
+							psProcessStats->asCacheOp[i32ReadIdx].uiSize,
+							ui64ExecuteTime);
+		}
+
+} /* CacheOpStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static PVRSRV_ERROR _PrintMemStatsEntry(uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	IMG_UINT32	ui32VAddrFields = sizeof(void*)/sizeof(IMG_UINT32);
+	IMG_UINT32	ui32PAddrFields = sizeof(IMG_CPU_PHYADDR)/sizeof(IMG_UINT32);
+	IMG_UINT32 ui32ItemNumber;
+	PVRSRV_MEM_ALLOC_REC *psRecord = (PVRSRV_MEM_ALLOC_REC *)(uintptr_t)v;
+	PVRSRV_MEM_ALLOC_PRINT_DATA *psPrintData = (PVRSRV_MEM_ALLOC_PRINT_DATA *)pvPriv;
+	OSDI_IMPL_ENTRY *psEntry = psPrintData->psEntry;
+
+	if (psRecord != NULL)
+	{
+		IMG_BOOL bPrintStat = IMG_TRUE;
+
+		DIPrintf(psEntry, "%-5d  ", psPrintData->pid);
+
+		switch (psRecord->eAllocType)
+		{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:             DIPrintf(psEntry, "KMALLOC             "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:             DIPrintf(psEntry, "VMALLOC             "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:  DIPrintf(psEntry, "ALLOC_PAGES_PT_LMA  "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:  DIPrintf(psEntry, "ALLOC_PAGES_PT_UMA  "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:      DIPrintf(psEntry, "IOREMAP_PT_LMA      "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:         DIPrintf(psEntry, "VMAP_PT_UMA         "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:     DIPrintf(psEntry, "ALLOC_LMA_PAGES     "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:    DIPrintf(psEntry, "ZOMBIE_LMA_PAGES    "); break;
+#endif
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:     DIPrintf(psEntry, "ALLOC_UMA_PAGES     "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:    DIPrintf(psEntry, "ZOMBIE_UMA_PAGES    "); break;
+#endif
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:   DIPrintf(psEntry, "MAP_UMA_LMA_PAGES   "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:      DIPrintf(psEntry, "DMA_BUF_IMPORT      "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:      DIPrintf(psEntry, "DMA_BUF_ZOMBIE      "); break;
+#endif
+		default:                                        DIPrintf(psEntry, "INVALID             "); break;
+		}
+
+		if (bPrintStat)
+		{
+			for (ui32ItemNumber = 0; ui32ItemNumber < ui32VAddrFields; ui32ItemNumber++)
+			{
+				DIPrintf(psEntry, "%08x", *(((IMG_UINT32*) &psRecord->pvCpuVAddr) + ui32VAddrFields - ui32ItemNumber - 1));
+			}
+			DIPrintf(psEntry, "  ");
+
+			for (ui32ItemNumber = 0; ui32ItemNumber < ui32PAddrFields; ui32ItemNumber++)
+			{
+				DIPrintf(psEntry, "%08x", *(((IMG_UINT32*) &psRecord->sCpuPAddr.uiAddr) + ui32PAddrFields - ui32ItemNumber - 1));
+			}
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+			DIPrintf(psEntry, "  " IMG_SIZE_FMTSPEC, psRecord->uiBytes);
+
+			DIPrintf(psEntry, "  %s", (IMG_CHAR*) psRecord->pvAllocdFromFile);
+
+			DIPrintf(psEntry, "  %d\n", psRecord->ui32AllocdFromLine);
+#else
+			DIPrintf(psEntry, "  " IMG_SIZE_FMTSPEC "\n", psRecord->uiBytes);
+#endif
+		}
+
+		psPrintData->ui32NumEntries++;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       MemStatsPrintElements
+@Description    Prints all elements for the memory statistic record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+MemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                      PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_UINT32	ui32VAddrFields = sizeof(void*)/sizeof(IMG_UINT32);
+	IMG_UINT32	ui32PAddrFields = sizeof(IMG_CPU_PHYADDR)/sizeof(IMG_UINT32);
+	IMG_UINT32 ui32ItemNumber;
+	PVRSRV_MEM_ALLOC_PRINT_DATA sPrintData;
+
+	sPrintData.psEntry = psEntry;
+	sPrintData.pid = psProcessStats->pid;
+	sPrintData.ui32NumEntries = 0;
+
+	/* Write the header... */
+	DIPrintf(psEntry, "PID    ");
+
+	DIPrintf(psEntry, "Type                VAddress");
+	for (ui32ItemNumber = 1;  ui32ItemNumber < ui32VAddrFields;  ui32ItemNumber++)
+	{
+		DIPrintf(psEntry, "        ");
+	}
+
+	DIPrintf(psEntry, "  PAddress");
+	for (ui32ItemNumber = 1;  ui32ItemNumber < ui32PAddrFields;  ui32ItemNumber++)
+	{
+		DIPrintf(psEntry, "        ");
+	}
+
+	DIPrintf(psEntry, "  Size(bytes)\n");
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+	HASH_Iterate(psProcessStats->psMemoryRecords, (HASH_pfnCallback)_PrintMemStatsEntry, &sPrintData);
+	OSLockRelease(psProcessStats->hLock);
+
+	if (sPrintData.ui32NumEntries == 0)
+	{
+		DIPrintf(psEntry, "%-5d\n", psProcessStats->pid);
+	}
+} /* MemStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+/*************************************************************************/ /*!
+@Function       RIMemStatsPrintElements
+@Description    Prints all elements for the RI Memory record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void RIMemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                             PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_CHAR   *pszStatFmtText  = NULL;
+	IMG_HANDLE *pRIHandle       = NULL;
+
+	/* Acquire RI lock */
+	RILockAcquireKM();
+
+	/*
+	 * Loop through the RI system to get each line of text.
+	 */
+	while (RIGetListEntryKM(psProcessStats->pid,
+							&pRIHandle,
+							&pszStatFmtText))
+	{
+		DIPrintf(psEntry, "%s", pszStatFmtText);
+	}
+
+	/* Release RI lock */
+	RILockReleaseKM();
+
+} /* RIMemStatsPrintElements */
+#endif
+
+#endif
+
+int GlobalStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	IMG_UINT32 ui32StatNumber;
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+	for (ui32StatNumber = 0;
+	     ui32StatNumber < ARRAY_SIZE(pszDriverStatType);
+	     ui32StatNumber++)
+	{
+		if (OSStringNCompare(pszDriverStatType[ui32StatNumber], "", 1) != 0)
+		{
+			DIPrintf(psEntry, "%-34s%12" IMG_UINT64_FMTSPEC "\n",
+				    pszDriverStatType[ui32StatNumber],
+				    GET_GLOBAL_STAT_VALUE(ui32StatNumber));
+		}
+	}
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVFindProcessMemStats
+@Description    Using the provided PID find memory stats for that process.
+                Memstats will be provided for live/connected processes only.
+                Memstat values provided by this API relate only to the physical
+                memory allocated by the process and does not relate to any of
+                the mapped or imported memory.
+@Input          pid                 Process to search for.
+@Input          ArraySize           Size of the array where memstat
+                                    records will be stored
+@Input          bAllProcessStats    Flag to denote if stats for
+                                    individual process are requested
+                                    stats for all processes are
+                                    requested
+@Input          MemoryStats         Handle to the memory where memstats
+                                    are stored.
+@Output         Memory statistics records for the requested pid.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVFindProcessMemStats(IMG_PID pid,
+                                       IMG_UINT32 ui32ArrSize,
+                                       IMG_BOOL bAllProcessStats,
+                                       IMG_UINT64 *pui64MemoryStats)
+{
+	IMG_INT i;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui64MemoryStats, "pui64MemoryStats");
+
+	if (bAllProcessStats)
+	{
+		PVR_LOG_RETURN_IF_FALSE(ui32ArrSize == PVRSRV_DRIVER_STAT_TYPE_COUNT,
+			"MemStats array size is incorrect",
+			PVRSRV_ERROR_INVALID_PARAMS);
+
+		OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+		for (i = 0; i < ui32ArrSize; i++)
+		{
+			pui64MemoryStats[i] = GET_GLOBAL_STAT_VALUE(i);
+		}
+
+		OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrSize == PVRSRV_PROCESS_STAT_TYPE_COUNT,
+		"MemStats array size is incorrect",
+		PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	/* Search for the given PID in the Live List */
+	psProcessStats = _FindProcessStatsInLiveList(pid);
+
+	if (psProcessStats == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Process %d not found. This process may not be live anymore.", (IMG_INT)pid));
+		OSLockRelease(g_psLinkedListLock);
+
+		return PVRSRV_ERROR_PROCESS_NOT_FOUND;
+	}
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+	for (i = 0; i < ui32ArrSize; i++)
+	{
+		pui64MemoryStats[i] = psProcessStats->i64StatValue[i];
+	}
+	OSLockRelease(psProcessStats->hLock);
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+
+} /* PVRSRVFindProcessMemStats */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetProcessMemUsage
+@Description    Calculate allocated kernel and graphics memory for all live or
+                connected processes. Memstat values provided by this API relate
+                only to the physical memory allocated by the process and does
+                not relate to any of the mapped or imported memory.
+@Output         pui64TotalMem                   Total memory usage for all live
+                                                PIDs connected to the driver.
+@Output         pui32NumberOfLivePids           Number of live pids currently
+                                                connected to the server.
+@Output         ppsPerProcessMemUsageData       Handle to an array of
+                                                PVRSRV_PER_PROCESS_MEM_USAGE,
+                                                number of elements defined by
+                                                pui32NumberOfLivePids.
+@Return         PVRSRV_OK                       Success
+                PVRSRV_ERROR_PROCESS_NOT_FOUND  No live processes.
+                PVRSRV_ERROR_OUT_OF_MEMORY      Failed to allocate memory for
+                                                ppsPerProcessMemUsageData.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVGetProcessMemUsage(IMG_UINT64 *pui64TotalMem,
+									  IMG_UINT32 *pui32NumberOfLivePids,
+									  PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsageData)
+{
+	IMG_UINT32 ui32NumberOfLivePids = 0;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_PROCESS_NOT_FOUND;
+	PVRSRV_PER_PROCESS_MEM_USAGE* psPerProcessMemUsageData = NULL;
+	DLLIST_NODE *psNode, *psNext;
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+	*pui64TotalMem = GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_KMALLOC) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_VMALLOC) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA);
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		ui32NumberOfLivePids++;
+	}
+
+	if (ui32NumberOfLivePids > 0)
+	{
+		/* Use OSAllocZMemNoStats to prevent deadlock. */
+		psPerProcessMemUsageData = OSAllocZMemNoStats(ui32NumberOfLivePids * sizeof(*psPerProcessMemUsageData));
+
+		if (psPerProcessMemUsageData)
+		{
+			PVRSRV_PROCESS_STATS* psProcessStats = NULL;
+			IMG_UINT32 ui32Counter = 0;
+
+			dllist_foreach_node(&gsLiveList, psNode, psNext)
+			{
+				psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+				psPerProcessMemUsageData[ui32Counter].ui32Pid = (IMG_UINT32)psProcessStats->pid;
+
+				psPerProcessMemUsageData[ui32Counter].ui64KernelMemUsage =
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMALLOC];
+
+				psPerProcessMemUsageData[ui32Counter].ui64GraphicsMemUsage =
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES];
+
+				OSLockRelease(psProcessStats->hLock);
+				ui32Counter++;
+			}
+			eError = PVRSRV_OK;
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+	*pui32NumberOfLivePids = ui32NumberOfLivePids;
+	*ppsPerProcessMemUsageData = psPerProcessMemUsageData;
+
+	return eError;
+
+} /* PVRSRVGetProcessMemUsage */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvr_notifier.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvr_notifier.c
new file mode 100644
index 000000000000..fd9653a11fb4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvr_notifier.c
@@ -0,0 +1,671 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR notifier interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "allocmem.h"
+#include "dllist.h"
+
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrversion.h"
+#include "connection_server.h"
+
+#include "osfunc.h"
+#include "sofunc_pvr.h"
+
+#define PVR_DUMP_DRIVER_INFO(x, y)														\
+	PVR_DUMPDEBUG_LOG("%s info: %d.%d @ %8d (%s) build options: 0x%08x",				\
+					   (x),																\
+					   PVRVERSION_UNPACK_MAJ((y).ui32BuildVersion),						\
+					   PVRVERSION_UNPACK_MIN((y).ui32BuildVersion),						\
+					   (y).ui32BuildRevision,											\
+					   (BUILD_TYPE_DEBUG == (y).ui32BuildType) ? "debug":"release",		\
+					   (y).ui32BuildOptions);
+
+#if !defined(WINDOW_SYSTEM)
+#define WINDOW_SYSTEM "Unknown"
+#endif
+
+#define IS_DECLARED(x) (x[0] != '\0')
+
+/*************************************************************************/ /*!
+Command Complete Notifier Interface
+*/ /**************************************************************************/
+
+typedef struct PVRSRV_CMDCOMP_NOTIFY_TAG
+{
+	PVRSRV_CMDCOMP_HANDLE	hCmdCompHandle;
+	PFN_CMDCOMP_NOTIFY		pfnCmdCompleteNotify;
+	DLLIST_NODE				sListNode;
+} PVRSRV_CMDCOMP_NOTIFY;
+
+/* Head of the list of callbacks called when command complete happens */
+static DLLIST_NODE g_sCmdCompNotifyHead;
+static POSWR_LOCK g_hCmdCompNotifyLock;
+
+PVRSRV_ERROR
+PVRSRVCmdCompleteInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSWRLockCreate(&g_hCmdCompNotifyLock);
+	PVR_RETURN_IF_ERROR(eError);
+
+	dllist_init(&g_sCmdCompNotifyHead);
+
+	return PVRSRV_OK;
+}
+
+void
+PVRSRVCmdCompleteDeinit(void)
+{
+	/* Check that all notify function have been unregistered */
+	if (!dllist_is_empty(&g_sCmdCompNotifyHead))
+	{
+		PDLLIST_NODE psNode;
+
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Command complete notify list is not empty!", __func__));
+
+		/* Clean up any stragglers */
+		psNode = dllist_get_next_node(&g_sCmdCompNotifyHead);
+		while (psNode)
+		{
+			PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+			dllist_remove_node(psNode);
+
+			psNotify = IMG_CONTAINER_OF(psNode, PVRSRV_CMDCOMP_NOTIFY, sListNode);
+			OSFreeMem(psNotify);
+
+			psNode = dllist_get_next_node(&g_sCmdCompNotifyHead);
+		}
+	}
+
+	if (g_hCmdCompNotifyLock)
+	{
+		OSWRLockDestroy(g_hCmdCompNotifyLock);
+	}
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterCmdCompleteNotify(IMG_HANDLE *phNotify,
+								PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify,
+								PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phNotify, "phNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnCmdCompleteNotify, "pfnCmdCompleteNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hCmdCompHandle, "hCmdCompHandle");
+
+	psNotify = OSAllocMem(sizeof(*psNotify));
+	PVR_LOG_RETURN_IF_NOMEM(psNotify, "psNotify");
+
+	/* Set-up the notify data */
+	psNotify->hCmdCompHandle = hCmdCompHandle;
+	psNotify->pfnCmdCompleteNotify = pfnCmdCompleteNotify;
+
+	/* Add it to the list of Notify functions */
+	OSWRLockAcquireWrite(g_hCmdCompNotifyLock);
+	dllist_add_to_tail(&g_sCmdCompNotifyHead, &psNotify->sListNode);
+	OSWRLockReleaseWrite(g_hCmdCompNotifyLock);
+
+	*phNotify = psNotify;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterCmdCompleteNotify(IMG_HANDLE hNotify)
+{
+	PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+	psNotify = (PVRSRV_CMDCOMP_NOTIFY *) hNotify;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNotify, "hNotify");
+
+	OSWRLockAcquireWrite(g_hCmdCompNotifyLock);
+	dllist_remove_node(&psNotify->sListNode);
+	OSWRLockReleaseWrite(g_hCmdCompNotifyLock);
+
+	OSFreeMem(psNotify);
+
+	return PVRSRV_OK;
+}
+
+void
+PVRSRVNotifyCommandCompletion(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle)
+{
+#if !defined(NO_HARDWARE)
+	DLLIST_NODE *psNode, *psNext;
+
+	/* Call notify callbacks to check if blocked work items can now proceed */
+	OSWRLockAcquireRead(g_hCmdCompNotifyLock);
+	dllist_foreach_node(&g_sCmdCompNotifyHead, psNode, psNext)
+	{
+		PVRSRV_CMDCOMP_NOTIFY *psNotify =
+			IMG_CONTAINER_OF(psNode, PVRSRV_CMDCOMP_NOTIFY, sListNode);
+
+		if (hCmdCompCallerHandle != psNotify->hCmdCompHandle)
+		{
+			psNotify->pfnCmdCompleteNotify(psNotify->hCmdCompHandle);
+		}
+	}
+	OSWRLockReleaseRead(g_hCmdCompNotifyLock);
+#endif
+}
+
+inline void
+PVRSRVSignalDriverWideEO(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectSignal(psPVRSRVData->hGlobalEventObject);
+	}
+	/* Cleanup Thread could be waiting on Cleanup event object,
+	 * signal it as well to ensure work is processed
+	 */
+	if (psPVRSRVData->hCleanupEventObject && (OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued) != 0))
+	{
+		OSEventObjectSignal(psPVRSRVData->hCleanupEventObject);
+	}
+}
+
+inline void
+PVRSRVCheckStatus(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle)
+{
+	PVRSRVNotifyCommandCompletion(hCmdCompCallerHandle);
+	PVRSRVSignalDriverWideEO();
+}
+
+/*************************************************************************/ /*!
+Debug Notifier Interface
+*/ /**************************************************************************/
+
+/* Lockdep sees both locks as the same class due to same struct used thus warns
+ * about a possible deadlock (false positive),
+ * using nested api we can supply separate Class'
+ * */
+#define DN_LOCKCLASS_DRIVER 0
+#define DN_LOCKCLASS_DEVICE 1
+
+typedef struct DEBUG_REQUEST_ENTRY_TAG
+{
+	IMG_UINT32      ui32RequesterID;
+	DLLIST_NODE     sListHead;
+} DEBUG_REQUEST_ENTRY;
+
+typedef struct DEBUG_REQUEST_TABLE_TAG
+{
+	POSWR_LOCK              hLock;
+	DEBUG_REQUEST_ENTRY     asEntry[IMG_FLEX_ARRAY_MEMBER];
+} DEBUG_REQUEST_TABLE;
+
+typedef struct DEBUG_REQUEST_NOTIFY_TAG
+{
+	IMG_HANDLE              hDebugTable;
+	PVRSRV_DBGREQ_HANDLE    hDbgRequestHandle;
+	PFN_DBGREQ_NOTIFY       pfnDbgRequestNotify;
+	IMG_UINT32              ui32RequesterID;
+	DLLIST_NODE             sListNode;
+} DEBUG_REQUEST_NOTIFY;
+
+static DEBUG_REQUEST_TABLE *g_psDriverDebugTable;
+
+static const IMG_UINT32 g_aui32DebugOrderTable[] = {
+	DEBUG_REQUEST_SRV,
+	DEBUG_REQUEST_RGX,
+	DEBUG_REQUEST_SYS,
+	DEBUG_REQUEST_APPHINT,
+	DEBUG_REQUEST_HTB,
+	DEBUG_REQUEST_DC,
+	DEBUG_REQUEST_SYNCCHECKPOINT,
+	DEBUG_REQUEST_SYNCTRACKING,
+	DEBUG_REQUEST_ANDROIDSYNC,
+	DEBUG_REQUEST_FALLBACKSYNC,
+	DEBUG_REQUEST_LINUXFENCE
+};
+static const IMG_UINT32 g_ui32DebugOrderTableReqCount = ARRAY_SIZE(g_aui32DebugOrderTable);
+
+static PVRSRV_ERROR
+_RegisterDebugTableI(DEBUG_REQUEST_TABLE **ppsDebugTable)
+{
+	DEBUG_REQUEST_TABLE *psDebugTable;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	if (*ppsDebugTable)
+	{
+		return PVRSRV_ERROR_DBGTABLE_ALREADY_REGISTERED;
+	}
+
+	psDebugTable = OSAllocMem(sizeof(DEBUG_REQUEST_TABLE) +
+	                          IMG_FLEX_ARRAY_SIZE(sizeof(DEBUG_REQUEST_ENTRY), g_ui32DebugOrderTableReqCount));
+	PVR_RETURN_IF_NOMEM(psDebugTable);
+
+	eError = OSWRLockCreate(&psDebugTable->hLock);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeDebugTable);
+
+	/* Init the list heads */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		psDebugTable->asEntry[i].ui32RequesterID = g_aui32DebugOrderTable[i];
+		dllist_init(&psDebugTable->asEntry[i].sListHead);
+	}
+
+	*ppsDebugTable = psDebugTable;
+
+	return PVRSRV_OK;
+
+ErrorFreeDebugTable:
+	OSFreeMem(psDebugTable);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgTable(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	return _RegisterDebugTableI((DEBUG_REQUEST_TABLE**)&psDevNode->hDebugTable);
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgTable(void)
+{
+	return _RegisterDebugTableI(&g_psDriverDebugTable);
+}
+
+static void _UnregisterDbgTableI(DEBUG_REQUEST_TABLE **ppsDebugTable)
+{
+	DEBUG_REQUEST_TABLE *psDebugTable;
+	IMG_UINT32 i;
+
+	PVR_ASSERT(*ppsDebugTable);
+	psDebugTable = *ppsDebugTable;
+	*ppsDebugTable = NULL;
+
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		if (!dllist_is_empty(&psDebugTable->asEntry[i].sListHead))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Found registered callback(s) on %d",
+					 __func__, i));
+		}
+	}
+
+	OSWRLockDestroy(psDebugTable->hLock);
+	psDebugTable->hLock = NULL;
+
+	OSFreeMem(psDebugTable);
+}
+
+void
+PVRSRVUnregisterDeviceDbgTable(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	_UnregisterDbgTableI((DEBUG_REQUEST_TABLE**)&psDevNode->hDebugTable);
+	PVR_ASSERT(!psDevNode->hDebugTable);
+}
+
+void
+PVRSRVUnregisterDriverDbgTable(void)
+{
+	_UnregisterDbgTableI(&g_psDriverDebugTable);
+	PVR_ASSERT(!g_psDriverDebugTable);
+}
+
+static PVRSRV_ERROR
+_RegisterDbgRequestNotifyI(IMG_HANDLE *phNotify,
+		                   DEBUG_REQUEST_TABLE *psDebugTable,
+		                   PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+		                   IMG_UINT32 ui32RequesterID,
+		                   PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	DEBUG_REQUEST_NOTIFY *psNotify;
+	PDLLIST_NODE psHead = NULL;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phNotify, "phNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDebugTable, "psDebugTable");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnDbgRequestNotify, "pfnDbRequestNotify");
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psNotify = OSAllocMemNoStats(sizeof(*psNotify));
+	PVR_LOG_RETURN_IF_NOMEM(psNotify, "psNotify");
+
+	/* Set-up the notify data */
+	psNotify->hDebugTable = psDebugTable;
+	psNotify->hDbgRequestHandle = hDbgRequestHandle;
+	psNotify->pfnDbgRequestNotify = pfnDbgRequestNotify;
+	psNotify->ui32RequesterID = ui32RequesterID;
+
+	/* Lock down all the lists */
+	OSWRLockAcquireWrite(psDebugTable->hLock);
+
+	/* Find which list to add it to */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		if (psDebugTable->asEntry[i].ui32RequesterID == ui32RequesterID)
+		{
+			psHead = &psDebugTable->asEntry[i].sListHead;
+		}
+	}
+
+	/* Failed to find debug requester */
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psHead, eError, ErrorReleaseLock);
+
+	/* Add it to the list of Notify functions */
+	dllist_add_to_tail(psHead, &psNotify->sListNode);
+
+	/* Unlock the lists */
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+
+	*phNotify = psNotify;
+
+	return PVRSRV_OK;
+
+ErrorReleaseLock:
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+	OSFreeMem(psNotify);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgRequestNotify(IMG_HANDLE *phNotify,
+	                                 PVRSRV_DEVICE_NODE *psDevNode,
+	                                 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	                                 IMG_UINT32 ui32RequesterID,
+	                                 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode, "psDevNode");
+	if (!psDevNode->hDebugTable)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: psDevNode->hDebugTable not yet initialised!",
+				 __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	return _RegisterDbgRequestNotifyI(phNotify,
+			                          (DEBUG_REQUEST_TABLE *)psDevNode->hDebugTable,
+			                          pfnDbgRequestNotify,
+			                          ui32RequesterID,
+			                          hDbgRequestHandle);
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgRequestNotify(IMG_HANDLE *phNotify,
+	                                 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	                                 IMG_UINT32 ui32RequesterID,
+	                                 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	if (!g_psDriverDebugTable)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: g_psDriverDebugTable not yet initialised!",
+				 __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	return _RegisterDbgRequestNotifyI(phNotify,
+			                          g_psDriverDebugTable,
+			                          pfnDbgRequestNotify,
+			                          ui32RequesterID,
+			                          hDbgRequestHandle);
+}
+
+PVRSRV_ERROR
+SOPvrDbgRequestNotifyRegister(IMG_HANDLE *phNotify,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+							  IMG_UINT32 ui32RequesterID,
+							  PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	return PVRSRVRegisterDeviceDbgRequestNotify(phNotify,
+			psDevNode,
+			pfnDbgRequestNotify,
+			ui32RequesterID,
+			hDbgRequestHandle);
+}
+
+static PVRSRV_ERROR
+_UnregisterDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	DEBUG_REQUEST_NOTIFY *psNotify = (DEBUG_REQUEST_NOTIFY *) hNotify;
+	DEBUG_REQUEST_TABLE *psDebugTable;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNotify, "psNotify");
+
+	psDebugTable = (DEBUG_REQUEST_TABLE *) psNotify->hDebugTable;
+
+	OSWRLockAcquireWrite(psDebugTable->hLock);
+	dllist_remove_node(&psNotify->sListNode);
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+
+	OSFreeMemNoStats(psNotify);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterDeviceDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterDriverDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+PVRSRV_ERROR
+SOPvrDbgRequestNotifyUnregister(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+void
+PVRSRVDebugRequest(PVRSRV_DEVICE_NODE *psDevNode,
+				   IMG_UINT32 ui32VerbLevel,
+				   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				   void *pvDumpDebugFile)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	DEBUG_REQUEST_TABLE *psDebugTable =
+		(DEBUG_REQUEST_TABLE *) psDevNode->hDebugTable;
+	DEBUG_REQUEST_TABLE *psDriverDebugTable =
+		(DEBUG_REQUEST_TABLE *) g_psDriverDebugTable;
+	static const IMG_CHAR *apszVerbosityTable[] = { "Low", "Medium", "High" };
+	const IMG_CHAR *szVerbosityLevel;
+	const IMG_CHAR *Bit32 = "32 Bit", *Bit64 = "64 Bit";
+	IMG_UINT32 i;
+
+	static_assert(ARRAY_SIZE(apszVerbosityTable) == DEBUG_REQUEST_VERBOSITY_MAX+1,
+	              "Incorrect number of verbosity levels");
+
+	PVR_ASSERT(psDebugTable);
+	PVR_ASSERT(psDriverDebugTable);
+
+	if (ui32VerbLevel < ARRAY_SIZE(apszVerbosityTable))
+	{
+		szVerbosityLevel = apszVerbosityTable[ui32VerbLevel];
+	}
+	else
+	{
+		szVerbosityLevel = "unknown";
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Invalid verbosity level received", __func__));
+	}
+
+	PVR_DUMPDEBUG_LOG("------------[ PVR DBG: START (%s) ]------------",
+	                  szVerbosityLevel);
+
+#if defined(RGX_IRQ_HYPERV_HANDLER)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+#endif
+	{
+		OSDumpVersionInfo(pfnDumpDebugPrintf, pvDumpDebugFile);
+	}
+
+	PVR_DUMPDEBUG_LOG("DDK info: %s (%s) %s",
+	                  PVRVERSION_STRING, PVR_BUILD_TYPE, PVR_BUILD_DIR);
+
+	PVR_DUMPDEBUG_LOG("Time now: %" IMG_UINT64_FMTSPEC "us",
+	                  OSClockus64());
+
+	switch (psPVRSRVData->eServicesState)
+	{
+		case PVRSRV_SERVICES_STATE_OK:
+			PVR_DUMPDEBUG_LOG("Services State: OK");
+			break;
+		case PVRSRV_SERVICES_STATE_BAD:
+			PVR_DUMPDEBUG_LOG("Services State: BAD");
+			break;
+		case PVRSRV_SERVICES_STATE_UNDEFINED:
+			PVR_DUMPDEBUG_LOG("Services State: UNDEFINED");
+			break;
+		default:
+			PVR_DUMPDEBUG_LOG("Services State: UNKNOWN (%d)",
+			                  psPVRSRVData->eServicesState);
+			break;
+	}
+
+	PVR_DUMPDEBUG_LOG("Server Errors: %d",
+	          PVRSRV_KM_ERRORS);
+
+	PVRSRVConnectionDebugNotify(psDevNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+
+	PVR_DUMPDEBUG_LOG("------[ Driver Info ]------");
+
+	PVR_DUMPDEBUG_LOG("Comparison of UM/KM components: %s",
+	                 (psPVRSRVData->sDriverInfo.bIsNoMatch) ? "MISMATCH" : "MATCHING");
+
+	if (psPVRSRVData->sDriverInfo.ui8KMBitArch)
+	{
+		PVR_DUMPDEBUG_LOG("KM Arch: %s",(psPVRSRVData->sDriverInfo.ui8KMBitArch & BUILD_ARCH_64BIT) ? Bit64 : Bit32);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("KM Arch is undefined");
+	}
+
+	PVR_DUMPDEBUG_LOG("Driver Mode: %s",
+	                  PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDevNode) ? "Native" : (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDevNode)) ? "Host":"Guest");
+
+	if (psPVRSRVData->sDriverInfo.ui8UMSupportedArch)
+	{
+		if ((psPVRSRVData->sDriverInfo.ui8UMSupportedArch & BUILD_ARCH_BOTH) ==
+			BUILD_ARCH_BOTH)
+		{
+			PVR_DUMPDEBUG_LOG("UM Connected Clients Arch: %s and %s", Bit64, Bit32);
+
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("UM Connected Clients: %s",
+			                 (psPVRSRVData->sDriverInfo.ui8UMSupportedArch & BUILD_ARCH_64BIT) ? Bit64 : Bit32);
+		}
+	}
+
+	PVR_DUMP_DRIVER_INFO("UM", psPVRSRVData->sDriverInfo.sUMBuildInfo);
+	PVR_DUMP_DRIVER_INFO("KM", psPVRSRVData->sDriverInfo.sKMBuildInfo);
+
+	PVR_DUMPDEBUG_LOG("Window system: %s", (IS_DECLARED(WINDOW_SYSTEM)) ? (WINDOW_SYSTEM) : "Not declared");
+
+	PVR_DUMPDEBUG_LOG("Power lock status: %s", OSLockIsLocked(psDevNode->hPowerLock) ? "Locked" : "Free");
+#if defined(DEBUG)
+	if (OSLockIsLocked(psDevNode->hPowerLock))
+	{
+		/* Ensure this info is logged before the power lock is taken, in case
+		 * it's already in use. */
+		PVR_DUMPDEBUG_LOG("Power lock owner: PID = %u at timestamp %" IMG_UINT64_FMTSPEC " (%s:%u)",
+		                  psDevNode->uiPwrLockOwnerPID, psDevNode->sPowerLockOwner.ui64Timestamp,
+		                  psDevNode->sPowerLockOwner.pszFile, psDevNode->sPowerLockOwner.ui32LineNum);
+	}
+#endif
+
+	/* Driver debug table */
+	OSWRLockAcquireReadNested(psDriverDebugTable->hLock, DN_LOCKCLASS_DRIVER);
+	/* Device debug table*/
+	OSWRLockAcquireReadNested(psDebugTable->hLock, DN_LOCKCLASS_DEVICE);
+
+	/* For each requester in Driver and Device table */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		DLLIST_NODE *psNode;
+		DLLIST_NODE *psNext;
+
+		/* For each notifier on this requestor */
+		dllist_foreach_node(&psDriverDebugTable->asEntry[i].sListHead, psNode, psNext)
+		{
+			DEBUG_REQUEST_NOTIFY *psNotify =
+				IMG_CONTAINER_OF(psNode, DEBUG_REQUEST_NOTIFY, sListNode);
+			psNotify->pfnDbgRequestNotify(psNotify->hDbgRequestHandle, ui32VerbLevel,
+				                          pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+
+		/* For each notifier on this requestor */
+		dllist_foreach_node(&psDebugTable->asEntry[i].sListHead, psNode, psNext)
+		{
+			DEBUG_REQUEST_NOTIFY *psNotify =
+				IMG_CONTAINER_OF(psNode, DEBUG_REQUEST_NOTIFY, sListNode);
+			psNotify->pfnDbgRequestNotify(psNotify->hDbgRequestHandle, ui32VerbLevel,
+							pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+	}
+
+	OSWRLockReleaseRead(psDebugTable->hLock);
+	OSWRLockReleaseRead(psDriverDebugTable->hLock);
+
+	PVR_DUMPDEBUG_LOG("------------[ PVR DBG: END ]------------");
+
+	if (!pfnDumpDebugPrintf)
+	{
+		/* Only notify OS of an issue if the caller requested it,
+		 * using a NULL pointer in pfnDumpDebugPrintf. */
+		OSWarnOn(IMG_TRUE);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv.c
new file mode 100644
index 000000000000..934593ff3ae5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv.c
@@ -0,0 +1,3871 @@
+/*************************************************************************/ /*!
+@File
+@Title          core services functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for core services functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#define CLEANUP_TYPE_STRINGS
+
+#include "img_defs.h"
+#include "rgxdebug_common.h"
+#include "handle.h"
+
+#include "pmr.h"
+#include "connection_server.h"
+#include "osconnection_server.h"
+#include "pdump_km.h"
+#include "ra.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "srvcore.h"
+#include "services_km.h"
+#include "pvrsrv_device.h"
+#include "pvr_debug.h"
+#include "debug_common.h"
+#include "pvr_notifier.h"
+#include "sync.h"
+#include "sync_server.h"
+#include "sync_checkpoint.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint_init.h"
+#include "devicemem.h"
+#include "cache_km.h"
+#include "info_page.h"
+#include "info_page_defs.h"
+#include "pvrsrv_bridge_init.h"
+#include "devicemem_server.h"
+#include "km_apphint_defs.h"
+#include "di_server.h"
+#include "di_impl_brg.h"
+#include "htb_debug.h"
+#include "dma_km.h"
+#include "pmr.h"
+
+#include "log2.h"
+
+#include "lists.h"
+#include "dllist.h"
+#include "syscommon.h"
+#include "sysvalidation.h"
+#include "hash_functions.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+#include "physmem_hostmem.h"
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+#include "physmem_cpumap_history.h"
+#endif
+
+#include "tlintern.h"
+#include "htbserver.h"
+#include "rgxmulticore.h"
+
+
+//#define MULTI_DEVICE_BRINGUP
+
+#if defined(MULTI_DEVICE_BRINGUP)
+#define MULTI_DEVICE_BRINGUP_DPF(msg, ...) PVR_DPF((PVR_DBG_MESSAGE, msg, __VA_ARGS__))
+#else
+#define MULTI_DEVICE_BRINGUP_DPF(msg, ...)
+#endif
+
+#if defined(SUPPORT_RGX)
+#include "rgxinit.h"
+#include "rgxhwperf.h"
+#include "rgxfwutils.h"
+#include "rgx_bridge_init.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#include "vz_vmm_pvz.h"
+
+#include "devicemem_history_server.h"
+
+#if defined(SUPPORT_LINUX_DVFS)
+#include "pvr_dvfs_device.h"
+#elif defined(SUPPORT_PDVFS)
+#include "pvr_dvfs_common.h"
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+
+#include "rgx_options.h"
+#include "srvinit.h"
+#include "rgxutils.h"
+
+#include "os_apphint.h"
+#include "pvrsrv_apphint.h"
+
+#include "pvrsrv_tlstreams.h"
+#include "tlstream.h"
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+#include "physmem_test.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#if defined(__linux__)
+#include "km_apphint.h"
+#endif /* defined(__linux__) */
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define INFINITE_SLEEP_TIMEOUT 0ULL
+#endif
+
+/*! Wait 100ms before retrying deferred clean-up again */
+#define CLEANUP_THREAD_WAIT_RETRY_TIMEOUT 100000ULL
+
+/*! Wait 8hrs when no deferred clean-up required. Allows a poll several times
+ * a day to check for any missed clean-up. */
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT INFINITE_SLEEP_TIMEOUT
+#else
+#define CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT 28800000000ULL
+#endif
+
+/*! When unloading try a few times to free everything remaining on the list */
+#define CLEANUP_THREAD_UNLOAD_RETRY 4
+
+#define PVRSRV_TL_CTRL_STREAM_SIZE 4096
+
+static PVRSRV_DATA	*gpsPVRSRVData;
+static IMG_UINT32 g_ui32InitFlags;
+
+/* mark which parts of Services were initialised */
+#define		INIT_DATA_ENABLE_PDUMPINIT	0x1U
+
+#define CLEANUP_QUEUE_TOTAL_DPF "QUEUED: %1.5d "
+#define CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE sizeof(CLEANUP_QUEUE_TOTAL_DPF)+1
+
+#define CLEANUP_STRING_SUMMARY_MAX_LEN (CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE+(PVRSRV_CLEANUP_TYPE_LAST * CLEANUP_TYPE_ITEM_DPF_MAX_SIZE))
+
+static IMG_CHAR *_ConcatCleanupString(IMG_CHAR *cleanupString)
+{
+	IMG_UINT32 uiLoop;
+	IMG_UINT32 strSize = CLEANUP_STRING_SUMMARY_MAX_LEN;
+
+	PVRSRV_DATA *psPVRSRVData =  PVRSRVGetPVRSRVData();
+
+	OSSNPrintf(cleanupString, CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE, CLEANUP_QUEUE_TOTAL_DPF,
+				OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued));
+
+	PVR_ASSERT(PVRSRV_CLEANUP_TYPE_CONNECTION == 1);
+
+	for (uiLoop=PVRSRV_CLEANUP_TYPE_CONNECTION; (strSize > CLEANUP_TYPE_ITEM_DPF_MAX_SIZE) && (uiLoop < PVRSRV_CLEANUP_TYPE_LAST); uiLoop++)
+	{
+		IMG_CHAR acTempQueued[CLEANUP_TYPE_ITEM_DPF_MAX_SIZE];
+		OSSNPrintf(acTempQueued, CLEANUP_TYPE_ITEM_DPF_MAX_SIZE, CLEANUP_TYPE_ITEM_DPF,
+				PVRSRVGetCleanupName(uiLoop),
+				OSAtomicRead(&psPVRSRVData->i32CleanupItemTypes[uiLoop]));
+		OSStringLCat(cleanupString, acTempQueued, strSize);
+	}
+
+	return cleanupString;
+}
+
+/* Callback to dump info of cleanup thread in debug_dump */
+static void CleanupThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                                  void *pvDumpDebugFile)
+{
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_DUMPDEBUG_LOG("    Number of deferred cleanup items: %s", _ConcatCleanupString(acCleanupString));
+
+	PVR_DUMPDEBUG_LOG("    Number of deferred cleanup items dropped after "
+					  "retry limit reached : %d",
+					  OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsNotCompleted));
+}
+
+static void _CleanupThreadDecrementStats(PVRSRV_DATA *psPVRSRVData,
+                                         PVRSRV_CLEANUP_TYPE eCleanupType)
+{
+#if defined(DEBUG)
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+#endif
+
+	PVR_ASSERT(OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued) >= 0);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "BEFORE REMOVING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+	OSAtomicDecrement(&psPVRSRVData->i32NumCleanupItemsQueued);
+
+	if ((eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) || ((eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item passed: %d", eCleanupType));
+	}
+	else
+	{
+		PVR_ASSERT(OSAtomicRead(&psPVRSRVData->i32CleanupItemTypes[eCleanupType]) >= 0);
+		OSAtomicDecrement(&psPVRSRVData->i32CleanupItemTypes[eCleanupType]);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "AFTER REMOVING ----- %s", _ConcatCleanupString(acCleanupString)));
+}
+
+#if defined(DEBUG)
+static void _CleanupThreadWorkListDump(PVRSRV_DATA *psPVRSRVData)
+{
+	DLLIST_NODE *psNode;
+	DLLIST_NODE *psNextNode;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+	char pszCleanupLog[128];
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* Iterate over all devices. */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		if (dllist_is_empty(&psDeviceNode->sCleanupThreadWorkList))
+		{
+			OSSNPrintf(pszCleanupLog, 128, "Dev_%p: CLEAN", psDeviceNode);
+			PVR_LOG(("%s", pszCleanupLog));
+			continue;
+		}
+
+		OSSNPrintf(pszCleanupLog, 128, "Dev_%p: TASKS", psDeviceNode);
+		PVR_LOG(("%s", pszCleanupLog));
+
+		/* Iterate over all cleanup items. */
+		dllist_foreach_node(&psDeviceNode->sCleanupThreadWorkList, psNode, psNextNode)
+		{
+			PVRSRV_CLEANUP_THREAD_WORK *psData;
+
+			psData = IMG_CONTAINER_OF(psNode, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+
+			PVR_ASSERT(psData != NULL);
+
+			if ((psData->eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) ||
+				((psData->eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item found: %d.", psData->eCleanupType));
+				continue;
+			}
+
+			OSSNPrintf(pszCleanupLog, 128, "+ %p: type %u, depends-HW %s (%s:%d)",
+					   psData,
+					   psData->eCleanupType,
+					   (psData->bDependsOnHW) ? "Yes" : " No",
+					   psData->pszFun, psData->ui32LineNum);
+			PVR_LOG(("%s", pszCleanupLog));
+		}
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+}
+#endif
+
+/* Add work to the cleanup thread work list.
+ * The work item will be executed by the cleanup thread
+ */
+#if defined(DEBUG)
+void PVRSRVCleanupThreadAddWork_Debug(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_CLEANUP_THREAD_WORK *psData,
+	const char *pszFun, const unsigned int ui32LineNum)
+#else
+void PVRSRVCleanupThreadAddWork_Int(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_CLEANUP_THREAD_WORK *psData)
+#endif
+{
+#if defined(DEBUG)
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+#endif
+
+	PVRSRV_DATA *psPVRSRVData;
+	PVRSRV_ERROR eError;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(psData != NULL);
+
+	if ((psData->eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) ||
+		((psData->eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item passed: %d.", psData->eCleanupType));
+	}
+
+	psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "AFTER QUEUEING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+
+	/* Cleanup thread work item added during device destructing:
+	   1. bDependsOnHW: Item may use device resources. Not safe to clean.
+	   2. !bDependsOnHW: Clean item immediately to prevent memory leak. */
+	if (psPVRSRVData->bUnload
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		|| psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK
+#endif
+		|| (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING && !psData->bDependsOnHW))
+	{
+		CLEANUP_THREAD_FN pfnFree = psData->pfnFree;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Cleanup thread has already quit: doing work immediately"));
+
+		eError = pfnFree(psData->pvData);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "BEFORE ADDING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to free resource "
+						"(callback " IMG_PFN_FMTSPEC "). "
+						"Immediate free will not be retried.",
+						pfnFree));
+		}
+	}
+	else
+	{
+		/*
+		 * Access psData before putting it in the work list.
+		 * Cleanup thread will free psData after it is done with the cleanup.
+		 * We should consider psData potentially freed after it was put into the worklist.
+		 */
+
+		OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsQueued);
+		OSAtomicIncrement(&psPVRSRVData->i32CleanupItemTypes[psData->eCleanupType]);
+#if defined(DEBUG)
+		psData->pszFun = pszFun;
+		psData->ui32LineNum = ui32LineNum;
+#endif
+
+		/* add this work item to the list */
+		OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+		OSAtomicIncrement(&psDeviceNode->i32NumCleanupItems);
+		dllist_add_to_tail(&psDeviceNode->sCleanupThreadWorkList, &psData->sNode);
+		OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+		/* signal the cleanup thread to ensure this item gets processed */
+		eError = OSEventObjectSignal(psPVRSRVData->hCleanupEventObject);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "AFTER ADDING ----- %s", _ConcatCleanupString(acCleanupString)));
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+	}
+}
+
+void PVRSRVCleanupThreadWaitForDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(DEBUG)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+#endif
+	IMG_INT32 i32NumCleanupItems;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+
+	if (gpsPVRSRVData->hCleanupThread == NULL)
+	{
+		return;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+	{
+		PVRSRV_ERROR eError;
+
+		if (gpsPVRSRVData->hCleanupEventObject)
+		{
+			eError = OSEventObjectSignal(gpsPVRSRVData->hCleanupEventObject);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+
+		i32NumCleanupItems = OSAtomicRead(&psDeviceNode->i32NumCleanupItems);
+
+		if (i32NumCleanupItems == 0)
+		{
+#if defined(DEBUG)
+			OS_SPINLOCK_FLAGS uiFlags;
+
+			OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+			PVR_LOG_IF_FALSE(dllist_is_empty(&psDeviceNode->sCleanupThreadWorkList),
+							 "Cleanup thread work list is not empty");
+
+			OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+#endif
+			break;
+		}
+
+		OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US / OS_THREAD_DESTROY_RETRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (i32NumCleanupItems != 0)
+	{
+		PVR_LOG(("Failed to flush device cleanup queue. %d items remain.", i32NumCleanupItems));
+	}
+
+#if defined(DEBUG)
+	_CleanupThreadWorkListDump(psPVRSRVData);
+#endif
+}
+
+static INLINE DLLIST_NODE *_CleanupThreadWorkListLast(PVRSRV_DATA *psPVRSRVData)
+{
+	DLLIST_NODE *psNode = NULL;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* We treat the Host device node as the last node in the list. */
+	psNode = dllist_get_prev_node(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList);
+	if (psNode != NULL)
+	{
+		OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+		return psNode;
+	}
+
+	/* Iterate over all devices and find the last node of the last device. */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		DLLIST_NODE *psCurrNode = dllist_get_prev_node(&psDeviceNode->sCleanupThreadWorkList);
+		if (psCurrNode != NULL)
+		{
+			psNode = psCurrNode;
+		}
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psNode;
+}
+
+/* Pop an item from the head of the cleanup thread work lists.
+ *
+ * This pops an item in a round robin manner:
+ *
+ * 1. It starts from the first device and moves to next device if no cleanup
+ *    item found.
+ * 2. If no items were found in the regular devices bound lists it moves to the
+ *    Host device list.
+ * 3. In case there was nothing in the Host device items list but there was
+ *    something in the devices lists, it returns first found device bound item
+ *
+ * If `*ppsDeviceNode` is not NULL the lookup of the next cleanup item will
+ * start from the next devices after `*ppsDeviceNode`. This makes sure that
+ * we always pop a first item from each device and move to the next device.
+ * We prevent "starvation" of some devices this way.
+ */
+static INLINE DLLIST_NODE *_CleanupThreadWorkListPop(PVRSRV_DATA *psPVRSRVData,
+                                                     PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL, *psFirstDevice = NULL;
+	DLLIST_NODE *psNode = NULL, *psFirstNode = NULL;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* always need to start from the beginning of the list in case some of the
+	 * devices were already destroyed */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		/* not safe to run cleanup during deinit */
+		if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+		{
+			continue;
+		}
+
+		psNode = dllist_get_next_node(&psDeviceNode->sCleanupThreadWorkList);
+
+		/* remember first non-NULL node and device in case no items are found
+		 * on later devices */
+		if (psNode != NULL && psFirstNode == NULL)
+		{
+			psFirstNode = psNode;
+			psFirstDevice = psDeviceNode;
+		}
+
+		/* to ensure that further device don't get starved skip the last device
+		 * until all devices are processed and then move to non-device bound
+		 * items */
+		if (*ppsDeviceNode != NULL) {
+			if (psDeviceNode == *ppsDeviceNode)
+			{
+				*ppsDeviceNode = NULL;
+			}
+
+			/* in case this is the last device and next iteration exits the
+			 * loop */
+			psNode = NULL;
+
+			continue;
+		}
+
+		if (psNode != NULL)
+		{
+			*ppsDeviceNode = psDeviceNode;
+			dllist_remove_node(psNode);
+
+			break;
+		}
+	}
+
+	/* if no item found in the regular devices check also the Host device */
+	if (psNode == NULL)
+	{
+		psNode = dllist_get_next_node(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList);
+		if (psNode != NULL)
+		{
+			dllist_remove_node(psNode);
+		}
+		*ppsDeviceNode = psPVRSRVData->psHostMemDeviceNode;
+	}
+
+	/* if no item found for the Host device check if there was a cleanup item
+	 * in one of the previous devices, this starts processing items from the
+	 * beginning without one extra empty call to "wrap around" */
+	if (psNode == NULL && psFirstNode != NULL)
+	{
+		psNode = psFirstNode;
+		*ppsDeviceNode = psFirstDevice;
+		dllist_remove_node(psNode);
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psNode;
+}
+
+/* Process the cleanup thread work list */
+static IMG_BOOL _CleanupThreadProcessWorkList(PVRSRV_DATA *psPVRSRVData,
+                                              IMG_BOOL *pbUseHWTimeout)
+{
+	DLLIST_NODE *psNodeIter, *psNodeLast;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bNeedRetry = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_CLEANUP_TYPE eCleanupType;
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+
+	/* Reset HWTimeout Flag */
+	*pbUseHWTimeout = IMG_FALSE;
+
+	psNodeLast = _CleanupThreadWorkListLast(psPVRSRVData);
+	if (psNodeLast == NULL)
+	{
+		/* no elements to clean up */
+		return IMG_FALSE;
+	}
+
+	/* any callback functions which return error will be
+	 * moved to the back of the list, and additional items can be added
+	 * to the list at any time so we ensure we only iterate from the
+	 * head of the list to the current tail (since the tail may always
+	 * be changing)
+	 */
+	do
+	{
+		PVRSRV_CLEANUP_THREAD_WORK *psData;
+		CLEANUP_THREAD_FN pfnFree;
+		IMG_BOOL bRetry = IMG_FALSE;
+
+		psNodeIter = _CleanupThreadWorkListPop(psPVRSRVData, &psDeviceNode);
+		if (psNodeIter == NULL)
+		{
+			return IMG_FALSE;
+		}
+
+		psData = IMG_CONTAINER_OF(psNodeIter, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+
+		/* get the function pointer address here so we have access to it
+		 * in order to report the error in case of failure, without having
+		 * to depend on psData not having been freed
+		 */
+		pfnFree = psData->pfnFree;
+		eCleanupType = psData->eCleanupType;
+		eError = pfnFree(psData->pvData);
+
+		if (eError != PVRSRV_OK)
+		{
+			/* Move to back of the list, if this item's
+			 * retry count hasn't hit zero.
+			 */
+			if (CLEANUP_THREAD_IS_RETRY_TIMEOUT(psData))
+			{
+				if (CLEANUP_THREAD_RETRY_TIMEOUT_NOT_REACHED(psData))
+				{
+					bNeedRetry = IMG_TRUE;
+					bRetry = IMG_TRUE;
+					/* If any items require retry and are HW dependent
+					 * use the HW timeout
+					 */
+					if (psData->bDependsOnHW)
+					{
+						*pbUseHWTimeout = psData->bDependsOnHW;
+					}
+				}
+			}
+			else
+			{
+				if (psData->ui32RetryCount > 0)
+				{
+					psData->ui32RetryCount--;
+					bNeedRetry = IMG_TRUE;
+					bRetry = IMG_TRUE;
+					/* If any items require retry and are HW dependent
+					 * use the HW timeout
+					 */
+					if (psData->bDependsOnHW)
+					{
+						*pbUseHWTimeout = psData->bDependsOnHW;
+					}
+				}
+			}
+
+			/* If the work depends on HW then we should add it to the back of the list,
+			 * the cleanup thread will sleep for longer if required and the next MISR
+			 * from the device will wake the task again in which it might be ready.
+			 */
+			if (bRetry || psData->bDependsOnHW)
+			{
+				/* If any items on the work list depend on HW
+				 * and didn't get cleaned up.
+				 */
+				OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+				dllist_add_to_tail(&psDeviceNode->sCleanupThreadWorkList, psNodeIter);
+				OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to free resource (callback " IMG_PFN_FMTSPEC "). "
+				        "Retry limit reached", pfnFree));
+				OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsNotCompleted);
+				OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+				/* Dropping item */
+				_CleanupThreadDecrementStats(psPVRSRVData, eCleanupType);
+			}
+		}
+		else
+		{
+			/* Ok returned */
+			OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+			_CleanupThreadDecrementStats(psPVRSRVData, eCleanupType);
+		}
+	} while (psNodeIter != NULL && psNodeIter != psNodeLast);
+
+	return bNeedRetry;
+}
+
+// #define CLEANUP_DPFL PVR_DBG_WARNING
+#define CLEANUP_DPFL    PVR_DBG_MESSAGE
+
+/* Create/initialise data required by the cleanup thread,
+ * before the cleanup thread is started
+ */
+static PVRSRV_ERROR _CleanupThreadPrepare(PVRSRV_DATA *psPVRSRVData)
+{
+	PVRSRV_ERROR eError;
+
+	/* Create the clean up event object */
+
+	eError = OSEventObjectCreate("PVRSRV_CLEANUP_EVENTOBJECT", &gpsPVRSRVData->hCleanupEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Exit);
+
+	/* initialise the mutex and linked list required for the cleanup thread work list */
+
+	eError = OSSpinLockCreate(&psPVRSRVData->hCleanupThreadWorkListLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", Exit);
+
+Exit:
+	return eError;
+}
+
+static void CleanupThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_BOOL     bRetryWorkList = IMG_FALSE;
+	IMG_BOOL     bUseHWTimeout = IMG_FALSE;
+	IMG_HANDLE	 hOSEvent;
+	PVRSRV_ERROR eRc;
+	IMG_UINT32 uiUnloadRetry = 0;
+	DLLIST_NODE *psNodeIter, *psNodeLast;
+
+	/* Store the process id (pid) of the clean-up thread */
+	psPVRSRVData->cleanupThreadPid = OSGetCurrentProcessID();
+	psPVRSRVData->cleanupThreadTid = OSGetCurrentThreadID();
+
+	PVR_DPF((CLEANUP_DPFL, "CleanupThread: thread starting... "));
+
+	/* Open an event on the clean up event object so we can listen on it,
+	 * abort the clean up thread and driver if this fails.
+	 */
+	eRc = OSEventObjectOpen(psPVRSRVData->hCleanupEventObject, &hOSEvent);
+	PVR_ASSERT(eRc == PVRSRV_OK);
+
+	/* While the driver is in a good state and is not being unloaded
+	 * try to free any deferred items when signalled
+	 */
+	while (psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK)
+	{
+		IMG_UINT64 ui64Timeoutus;
+		if (psPVRSRVData->bUnload)
+		{
+			if (dllist_is_empty(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList) ||
+			    uiUnloadRetry > CLEANUP_THREAD_UNLOAD_RETRY)
+			{
+				break;
+			}
+			uiUnloadRetry++;
+		}
+
+		/* Wait until signalled for deferred clean up OR wait for a
+		 * short period if the previous deferred clean up was not able
+		 * to release all the resources before trying again.
+		 * Bridge lock re-acquired on our behalf before the wait call returns.
+		 */
+
+		if (bRetryWorkList && bUseHWTimeout)
+		{
+			/* If item depends on HW we are
+			 * waiting for GPU work to finish, so
+			 * use MAX_HW_TIME_US as timeout (this
+			 * will be set appropriately when
+			 * running on systems with emulated
+			 * hardware, etc).
+			 */
+			ui64Timeoutus = MAX_HW_TIME_US;
+		}
+		else
+		{
+			/* Use the default retry timeout. */
+			ui64Timeoutus = CLEANUP_THREAD_WAIT_RETRY_TIMEOUT;
+		}
+
+		eRc = OSEventObjectWaitKernel(hOSEvent,
+				bRetryWorkList ?
+				ui64Timeoutus :
+				CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT);
+		if (eRc == PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_DPF((CLEANUP_DPFL, "CleanupThread: wait timeout"));
+		}
+		else if (eRc == PVRSRV_OK)
+		{
+			PVR_DPF((CLEANUP_DPFL, "CleanupThread: wait OK, signal received"));
+		}
+		else
+		{
+			PVR_LOG_ERROR(eRc, "OSEventObjectWaitKernel");
+		}
+
+		bRetryWorkList = _CleanupThreadProcessWorkList(psPVRSRVData,
+		                                               &bUseHWTimeout);
+	}
+
+	psNodeLast = _CleanupThreadWorkListLast(psPVRSRVData);
+	if (psNodeLast != NULL)
+	{
+		do
+		{
+			PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+			PVRSRV_CLEANUP_THREAD_WORK *psData;
+			psNodeIter = _CleanupThreadWorkListPop(psPVRSRVData, &psDeviceNode);
+			if (psNodeIter == NULL)
+			{
+				break;
+			}
+
+			psData = IMG_CONTAINER_OF(psNodeIter, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+			OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsNotCompleted);
+			OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+			/* Dropping item */
+			_CleanupThreadDecrementStats(psPVRSRVData, psData->eCleanupType);
+		}
+		while (psNodeIter != NULL && psNodeIter != psNodeLast);
+
+		PVR_DPF((PVR_DBG_ERROR, "Cleanup Thread Failed to free %d resources", OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsNotCompleted)));
+	}
+
+	OSSpinLockDestroy(psPVRSRVData->hCleanupThreadWorkListLock);
+
+	eRc = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eRc, "OSEventObjectClose");
+
+	PVR_DPF((CLEANUP_DPFL, "CleanupThread: thread ending... "));
+}
+
+IMG_PID PVRSRVCleanupThreadGetPid(void)
+{
+	return gpsPVRSRVData->cleanupThreadPid;
+}
+
+uintptr_t PVRSRVCleanupThreadGetTid(void)
+{
+	return gpsPVRSRVData->cleanupThreadTid;
+}
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*
+ * Firmware is unresponsive.
+ * The Host will initiate a recovery process during which the
+ * Firmware and GPU are reset and returned to a working state.
+ */
+static PVRSRV_ERROR HandleFwHostSideRecovery(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32CtxIdx = 0U;
+	IMG_UINT32 ui32Nodes  = 0U;
+
+	OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+	/* Get the number of nodes in a linked list */
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		++ui32Nodes;
+	}
+
+	/* Any client contexts active at the moment? */
+	if (ui32Nodes > 0U)
+	{
+		/* Free the active context buffer previously allocated */
+		if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+			psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+		}
+
+		/* Setup allocations to store the active contexts */
+		eError = RGXSetupFwAllocation(psDevInfo,
+				RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+				(ui32Nodes + 1) * sizeof(RGXFWIF_ACTIVE_CONTEXT_BUF_DATA),
+				"FwSysActiveContextBufData",
+				&psDevInfo->psRGXFWIfActiveContextBufDesc,
+				(void *)  &psDevInfo->psRGXFWIfSysInit->sActiveContextBufBase.ui32Addr,
+				(void **) &psDevInfo->psRGXFWIfActiveContextBuf,
+				RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation",Error);
+
+		/* List of contexts to be rekicked by FW powering up the device */
+		dllist_foreach_node_backwards(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+		{
+			psDevInfo->psRGXFWIfActiveContextBuf[ui32CtxIdx].psContext =
+					RGXGetFWCommonContextAddrFromServerCommonCtx(psDevInfo, psNode);
+			++ui32CtxIdx;
+		}
+		/* Null context as the terminator marker */
+		psDevInfo->psRGXFWIfActiveContextBuf[ui32CtxIdx].psContext.ui32Addr = 0;
+	}
+	OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+
+	/* Host can't expect a response on power-down request as FW is in BAD state */
+	eError = PVRSRVSetDeviceCurrentPowerState(psDeviceNode->psPowerDev, PVRSRV_DEV_POWER_STATE_OFF);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVSetDeviceCurrentPowerState OFF", Error);
+
+	/* Flag to be set to notify FW while recovering from crash */
+	psDevInfo->psRGXFWIfSysInit->bFwHostRecoveryMode = IMG_TRUE;
+
+	/* Flush here because we have setup a fw alloc addr in the structure earlier */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit, FLUSH);
+
+	/* Power-on the device resetting GPU & FW */
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+			PVRSRV_DEV_POWER_STATE_ON,
+			PVRSRV_POWER_FLAGS_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVSetDevicePowerStateKM ON", Error);
+	OSLockRelease(psDeviceNode->hPowerLock);
+
+Error:
+	return eError;
+}
+#endif
+
+
+void PVRSRVDeviceSetState(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_STATE eNewDevState)
+{
+	if (eNewDevState == psDeviceNode->eDevState)
+	{
+		return;
+	}
+
+	switch (eNewDevState)
+	{
+		case PVRSRV_DEVICE_STATE_PCI_ERROR:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetDebugDevStateString(eNewDevState)));
+			psDeviceNode->eDevState = eNewDevState;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+			PMRFreeZombies(psDeviceNode);
+#endif
+			break;
+		}
+		case PVRSRV_DEVICE_STATE_CREATING:
+		case PVRSRV_DEVICE_STATE_CREATED:
+		case PVRSRV_DEVICE_STATE_ACTIVE:
+		case PVRSRV_DEVICE_STATE_FROZEN:
+		case PVRSRV_DEVICE_STATE_DEINIT:
+		case PVRSRV_DEVICE_STATE_DESTRUCTING:
+		case PVRSRV_DEVICE_STATE_BAD:
+		{
+			/* PCI_ERROR is a terminal state. Reload driver to recover. */
+			if (eNewDevState != PVRSRV_DEVICE_STATE_PCI_ERROR)
+			{
+				psDeviceNode->eDevState = eNewDevState;
+			}
+			break;
+		}
+		case PVRSRV_DEVICE_STATE_UNDEFINED:
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unknown state (%d)", __func__, eNewDevState));
+			break;
+		}
+	}
+}
+
+static void DevicesWatchdogThread_ForEachVaCb(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  va_list va)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+#endif
+	PVRSRV_DEVICE_HEALTH_STATUS *pePreviousHealthStatus, eHealthStatus;
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_DEBUG_DUMP_STATUS eDebugDumpState;
+	IMG_BOOL bCheckAfterTimePassed;
+
+	pePreviousHealthStatus = va_arg(va, PVRSRV_DEVICE_HEALTH_STATUS *);
+	/* IMG_BOOL (i.e. bool) is converted to int during default argument promotion
+	 * in variadic argument list. Thus, fetch an int to get IMG_BOOL */
+	bCheckAfterTimePassed = (IMG_BOOL) va_arg(va, int);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		return;
+	}
+
+	/* Block here if needed */
+	PVRSRVBlockIfFrozen(psDeviceNode);
+
+	if (psDeviceNode->pfnUpdateHealthStatus != NULL)
+	{
+		eError = psDeviceNode->pfnUpdateHealthStatus(psDeviceNode, bCheckAfterTimePassed);
+		PVR_WARN_IF_ERROR(eError, "pfnUpdateHealthStatus");
+	}
+	eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+	if (eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		if (eHealthStatus != *pePreviousHealthStatus)
+		{
+#if defined(SUPPORT_RGX)
+			if (!(psDevInfo->ui32DeviceFlags &
+				  RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN))
+#else
+			/* In this case we don't have an RGX device */
+			if (eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED)
+#endif
+			{
+				PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: "
+						 "Device status not OK!!!"));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX,
+								   NULL, NULL);
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+				HandleFwHostSideRecovery(psDeviceNode);
+#endif
+			}
+		}
+	}
+
+	*pePreviousHealthStatus = eHealthStatus;
+
+	/* Have we received request from FW to capture debug dump(could be due to HWR) */
+	eDebugDumpState = (PVRSRV_DEVICE_DEBUG_DUMP_STATUS)OSAtomicCompareExchange(
+						&psDeviceNode->eDebugDumpRequested,
+						PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE,
+						PVRSRV_DEVICE_DEBUG_DUMP_NONE);
+	if (PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE == eDebugDumpState)
+	{
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	}
+
+}
+
+#if defined(SUPPORT_RGX)
+static void HWPerfPeriodicHostEventsThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+
+	eError = OSEventObjectOpen(psPVRSRVData->hHWPerfHostPeriodicEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload && !psPVRSRVData->bHWPerfHostThreadStop)
+#else
+	while (!psPVRSRVData->bUnload && !psPVRSRVData->bHWPerfHostThreadStop)
+#endif
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode;
+		IMG_BOOL bInfiniteSleep = IMG_TRUE;
+
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)psPVRSRVData->ui32HWPerfHostThreadTimeout * 1000);
+		if (eError == PVRSRV_OK && (psPVRSRVData->bUnload || psPVRSRVData->bHWPerfHostThreadStop))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "HWPerfPeriodicHostEventsThread: Shutdown event received."));
+			break;
+		}
+
+		for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+		     psDeviceNode != NULL;
+		     psDeviceNode = psDeviceNode->psNext)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+			/* If the psDevInfo or hHWPerfHostStream are NULL it most
+			 * likely means that this device or stream has not been
+			 * initialised yet, so just skip */
+			if (psDevInfo == NULL || psDevInfo->hHWPerfHostStream == NULL)
+			{
+				continue;
+			}
+
+			/* Check if the HWPerf host stream is open for reading before writing
+			 * a packet, this covers cases where the event filter is not zeroed
+			 * before a reader disconnects. */
+			if (TLStreamIsOpenForReading(psDevInfo->hHWPerfHostStream))
+			{
+				/* As long as any of the streams is opened don't go into
+				 * indefinite sleep. */
+				bInfiniteSleep = IMG_FALSE;
+#if defined(SUPPORT_RGX)
+				RGXSRV_HWPERF_HOST_INFO(psDevInfo, RGX_HWPERF_INFO_EV_MEM64_USAGE);
+#endif
+			}
+		}
+
+		if (bInfiniteSleep)
+		{
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			psPVRSRVData->ui32HWPerfHostThreadTimeout = INFINITE_SLEEP_TIMEOUT;
+#else
+			/* Use an 8 hour timeout if indefinite sleep is not supported. */
+			psPVRSRVData->ui32HWPerfHostThreadTimeout = 60 * 60 * 8 * 1000;
+#endif
+		}
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+#endif
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+typedef enum
+{
+	DWT_ST_INIT,
+	DWT_ST_SLEEP_POWERON,
+	DWT_ST_SLEEP_POWEROFF,
+	DWT_ST_SLEEP_DEFERRED,
+	DWT_ST_FINAL
+} DWT_STATE;
+
+typedef enum
+{
+	DWT_SIG_POWERON,
+	DWT_SIG_POWEROFF,
+	DWT_SIG_TIMEOUT,
+	DWT_SIG_UNLOAD,
+	DWT_SIG_ERROR
+} DWT_SIGNAL;
+
+static inline IMG_BOOL _DwtIsPowerOn(PVRSRV_DATA *psPVRSRVData)
+{
+	return List_PVRSRV_DEVICE_NODE_IMG_BOOL_Any(psPVRSRVData->psDeviceNodeList,
+	                                            PVRSRVIsDevicePowered);
+}
+
+static inline void _DwtCheckHealthStatus(PVRSRV_DATA *psPVRSRVData,
+                                         PVRSRV_DEVICE_HEALTH_STATUS *peStatus,
+                                         IMG_BOOL bTimeOut)
+{
+	List_PVRSRV_DEVICE_NODE_ForEach_va(psPVRSRVData->psDeviceNodeList,
+	                                   DevicesWatchdogThread_ForEachVaCb,
+	                                   peStatus,
+	                                   bTimeOut);
+}
+
+static DWT_SIGNAL _DwtWait(PVRSRV_DATA *psPVRSRVData, IMG_HANDLE hOSEvent,
+                           IMG_UINT32 ui32Timeout)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64) ui32Timeout * 1000);
+
+
+	if (eError == PVRSRV_OK)
+	{
+		if (psPVRSRVData->bUnload)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Shutdown event"
+			        " received."));
+			return DWT_SIG_UNLOAD;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power state "
+			        "change event received."));
+
+			if (_DwtIsPowerOn(psPVRSRVData))
+			{
+				return DWT_SIG_POWERON;
+			}
+			else
+			{
+				return DWT_SIG_POWEROFF;
+			}
+		}
+	}
+	else if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		return DWT_SIG_TIMEOUT;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: Error (%d) when"
+	        " waiting for event!", eError));
+	return DWT_SIG_ERROR;
+}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+
+static void DevicesWatchdogThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	PVRSRV_DEVICE_HEALTH_STATUS ePreviousHealthStatus = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	DWT_STATE eState = DWT_ST_INIT;
+	const IMG_UINT32 ui32OnTimeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	const IMG_UINT32 ui32OffTimeout = INFINITE_SLEEP_TIMEOUT;
+#else
+	IMG_UINT32 ui32Timeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	/* Flag used to defer the sleep timeout change by 1 loop iteration.
+	 * This helps to ensure at least two health checks are performed before a long sleep.
+	 */
+	IMG_BOOL bDoDeferredTimeoutChange = IMG_FALSE;
+#endif
+
+	PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power off sleep time: %d.",
+			DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT));
+
+	/* Open an event on the devices watchdog event object so we can listen on it
+	   and abort the devices watchdog thread. */
+	eError = OSEventObjectOpen(psPVRSRVData->hDevicesWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+	/* Loop continuously checking the device status every few seconds. */
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload)
+#else
+	while (!psPVRSRVData->bUnload)
+#endif
+	{
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+		switch (eState)
+		{
+			case DWT_ST_INIT:
+			{
+				if (_DwtIsPowerOn(psPVRSRVData))
+				{
+					eState = DWT_ST_SLEEP_POWERON;
+				}
+				else
+				{
+					eState = DWT_ST_SLEEP_POWEROFF;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_POWERON:
+			{
+				DWT_SIGNAL eSignal = _DwtWait(psPVRSRVData, hOSEvent,
+				                                    ui32OnTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_POWEROFF:
+						eState = DWT_ST_SLEEP_DEFERRED;
+						break;
+					case DWT_SIG_TIMEOUT:
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_TRUE);
+						/* self-transition */
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_POWEROFF:
+			{
+				DWT_SIGNAL eSignal = _DwtWait(psPVRSRVData, hOSEvent,
+				                                    ui32OffTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						eState = DWT_ST_SLEEP_POWERON;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_POWEROFF:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_TIMEOUT:
+						/* self-transition */
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_TRUE);
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_DEFERRED:
+			{
+				DWT_SIGNAL eSignal =_DwtWait(psPVRSRVData, hOSEvent,
+				                                   ui32OnTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						eState = DWT_ST_SLEEP_POWERON;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_POWEROFF:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_TIMEOUT:
+						eState = DWT_ST_SLEEP_POWEROFF;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_FINAL:
+				/* the loop should terminate on next spin if this state is
+				 * reached so nothing to do here. */
+				break;
+		}
+
+#else /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+		IMG_BOOL bPwrIsOn = IMG_FALSE;
+		IMG_BOOL bTimeOut = IMG_FALSE;
+
+		/* Wait time between polls (done at the start of the loop to allow devices
+		   to initialise) or for the event signal (shutdown or power on). */
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+
+		if (eError == PVRSRV_OK)
+		{
+			if (psPVRSRVData->bUnload)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Shutdown event received."));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power state change event received."));
+			}
+		}
+		else if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			/* If timeout do nothing otherwise print warning message. */
+			PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: "
+					"Error (%d) when waiting for event!", eError));
+		}
+		else
+		{
+			bTimeOut = IMG_TRUE;
+		}
+
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		List_PVRSRV_DEVICE_NODE_ForEach_va(psPVRSRVData->psDeviceNodeList,
+		                                   DevicesWatchdogThread_ForEachVaCb,
+		                                   &ePreviousHealthStatus,
+		                                   bTimeOut);
+		bPwrIsOn = List_PVRSRV_DEVICE_NODE_IMG_BOOL_Any(psPVRSRVData->psDeviceNodeList,
+														PVRSRVIsDevicePowered);
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+		if (bPwrIsOn || psPVRSRVData->ui32DevicesWatchdogPwrTrans)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans = 0;
+			ui32Timeout = psPVRSRVData->ui32DevicesWatchdogTimeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+			bDoDeferredTimeoutChange = IMG_FALSE;
+		}
+		else
+		{
+			/* First, check if the previous loop iteration signalled a need to change the timeout period */
+			if (bDoDeferredTimeoutChange)
+			{
+				ui32Timeout = psPVRSRVData->ui32DevicesWatchdogTimeout = DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT;
+				bDoDeferredTimeoutChange = IMG_FALSE;
+			}
+			else
+			{
+				/* Signal that we need to change the sleep timeout in the next loop iteration
+				 * to allow the device health check code a further iteration at the current
+				 * sleep timeout in order to determine bad health (e.g. stalled cCCB) by
+				 * comparing past and current state snapshots */
+				bDoDeferredTimeoutChange = IMG_TRUE;
+			}
+		}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+#if defined(SUPPORT_AUTOVZ)
+static void AutoVzWatchdogThread_ForEachCb(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (psDeviceNode->pfnUpdateAutoVzWatchdog != NULL)
+	{
+		psDeviceNode->pfnUpdateAutoVzWatchdog(psDeviceNode);
+	}
+}
+
+static void AutoVzWatchdogThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Timeout = PVR_AUTOVZ_WDG_PERIOD_MS / 3;
+
+	/* Open an event on the devices watchdog event object so we can listen on it
+	   and abort the devices watchdog thread. */
+	eError = OSEventObjectOpen(psPVRSRVData->hAutoVzWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload)
+#else
+	while (!psPVRSRVData->bUnload)
+#endif
+	{
+		/* Wait time between polls (done at the start of the loop to allow devices
+		   to initialise) or for the event signal (shutdown or power on). */
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+
+		List_PVRSRV_DEVICE_NODE_ForEach(psPVRSRVData->psDeviceNodeList,
+		                                AutoVzWatchdogThread_ForEachCb);
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_DATA *PVRSRVGetPVRSRVData(void)
+{
+	return gpsPVRSRVData;
+}
+
+static PVRSRV_ERROR InitialiseInfoPageTimeouts(PVRSRV_DATA *psPVRSRVData)
+{
+	if (NULL == psPVRSRVData)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_VALUE_RETRIES] = WAIT_TRY_COUNT;
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_VALUE_TIMEOUT_MS] =
+		((MAX_HW_TIME_US / 10000) + 1000);
+		/* TIMEOUT_INFO_VALUE_TIMEOUT_MS resolves to...
+			vp       : 2000  + 1000
+			emu      : 2000  + 1000
+			rgx_nohw : 50    + 1000
+			plato    : 30000 + 1000 (VIRTUAL_PLATFORM or EMULATOR)
+			           50    + 1000 (otherwise)
+		*/
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_CONDITION_RETRIES] = 5;
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_CONDITION_TIMEOUT_MS] =
+		((MAX_HW_TIME_US / 10000) + 100);
+		/* TIMEOUT_INFO_CONDITION_TIMEOUT_MS resolves to...
+			vp       : 2000  + 100
+			emu      : 2000  + 100
+			rgx_nohw : 50    + 100
+			plato    : 30000 + 100 (VIRTUAL_PLATFORM or EMULATOR)
+			           50    + 100 (otherwise)
+		*/
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_TASK_QUEUE_RETRIES] = 10;
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_TASK_QUEUE_FLUSH_TIMEOUT_MS] =
+		MAX_HW_TIME_US / 1000U;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR PopulateInfoPageBridges(PVRSRV_DATA *psPVRSRVData)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psPVRSRVData);
+
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_PVR_BRIDGES] = gui32PVRBridges;
+
+#if defined(SUPPORT_RGX)
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_RGX_BRIDGES] = gui32RGXBridges;
+#else
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_RGX_BRIDGES] = 0;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static void _ThreadsDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDbgRequestHandle,
+                                       IMG_UINT32 ui32VerbLevel,
+                                       DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                       void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(hDbgRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+		PVR_DUMPDEBUG_LOG("------[ Server Thread Summary ]------");
+		OSThreadDumpInfo(pfnDumpDebugPrintf, pvDumpDebugFile);
+	}
+}
+
+#define PVRVZMODE_STR_SIZE_MAX (10U)
+#define PVRVZMODE_BUFFER_SIZE (((PVRSRV_MAX_DEVICES)*(PVRVZMODE_STR_SIZE_MAX))+1)
+
+static const IMG_CHAR *const asModeStrings[] =
+{
+	"native",
+	"host",
+	"guest",
+	"default"
+};
+
+static void _InitDefaultVzDriverMode(PVRSRV_DATA *psPVRSRVData, void *pvAppHintState)
+{
+	char *pszMode, pszModeAppHint[PVRVZMODE_BUFFER_SIZE];
+	static const IMG_CHAR *pszDefault = "";
+	PVRSRV_DRIVER_MODE aeInitModes[PVRSRV_MAX_DEVICES];
+	IMG_BOOL bRet;
+	IMG_UINT ui32Dev;
+
+	for (ui32Dev = 0; ui32Dev < PVRSRV_MAX_DEVICES; ui32Dev++)
+	{
+		aeInitModes[ui32Dev] = DRIVER_MODE_DEFAULT;
+	}
+
+	pszMode = pszModeAppHint;
+	pszModeAppHint[0] = '\0';
+
+	bRet = (IMG_BOOL) OSGetAppHintSTRING(APPHINT_NO_DEVICE,
+						pvAppHintState,
+						DriverMode,
+						pszDefault,
+						pszModeAppHint,
+						PVRVZMODE_BUFFER_SIZE);
+	PVR_GOTO_IF_FALSE(bRet, uninit_default);
+
+	ui32Dev = 0;
+
+	while (*pszMode && (ui32Dev < PVRSRV_MAX_DEVICES))
+	{
+		PVRSRV_DRIVER_MODE eMode;
+
+		while (*pszMode == ',')
+		{
+			/* skip commas */
+			pszMode++;
+		}
+
+		for (eMode = DRIVER_MODE_NATIVE; eMode <= DRIVER_MODE_DEFAULT; eMode++)
+		{
+			/* if no matching strings found, device's mode remains DEFAULT */
+			if (OSStringNCompare(pszMode,
+								 asModeStrings[eMode],
+								 OSStringLength(asModeStrings[eMode])) == 0)
+			{
+				aeInitModes[ui32Dev] = eMode;
+			}
+		}
+
+		while ((*pszMode != '\0') && (*pszMode != ','))
+		{
+			/* advance until the next command or string end */
+			pszMode++;
+		}
+
+		ui32Dev++;
+	}
+
+uninit_default:
+	for (ui32Dev = 0; ui32Dev < PVRSRV_MAX_DEVICES; ui32Dev++)
+	{
+		psPVRSRVData->aeModuleParamDriverMode[ui32Dev] = aeInitModes[ui32Dev];
+	}
+}
+
+PVRSRV_ERROR
+PVRSRVCommonDriverInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVRSRV_DATA	*psPVRSRVData = NULL;
+
+	IMG_UINT32 ui32AppHintCleanupThreadPriority;
+	IMG_UINT32 ui32AppHintWatchdogThreadPriority;
+	IMG_BOOL bEnablePageFaultDebug;
+	IMG_BOOL bEnableFullSyncTracking;
+
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+
+	/*
+	 * As this function performs one time driver initialisation, use the
+	 * Services global device-independent data to determine whether or not
+	 * this function has already been called.
+	 */
+	if (gpsPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Driver already initialised", __func__));
+		return PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+
+#if defined(SUPPORT_RGX)
+	RGXBridgeDriverInit();
+#endif
+
+	eError = DIInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	eError = PVRSRVStatsInitialise();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	eError = PVRDIImplBrgRegister();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	eError = HTB_CreateDIEntry();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/*
+	 * Allocate the device-independent data. Do NOT track this allocation.
+	 */
+	psPVRSRVData = OSAllocZMemNoStats(sizeof(*gpsPVRSRVData));
+	PVR_GOTO_IF_NOMEM(psPVRSRVData, eError, Error);
+
+	/* Now it is set up, point gpsPVRSRVData to the actual data */
+	gpsPVRSRVData = psPVRSRVData;
+
+	eError = OSWRLockCreate(&gpsPVRSRVData->hDeviceNodeListLock);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Register the driver context debug table */
+	eError = PVRSRVRegisterDriverDbgTable();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Register the Server Thread Debug notifier */
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&gpsPVRSRVData->hThreadsDbgReqNotify,
+		                                          _ThreadsDebugRequestNotify,
+		                                          DEBUG_REQUEST_SRV,
+		                                          NULL);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/*
+	 * Initialise the server bridges
+	 */
+	eError = ServerBridgeInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = DevmemIntInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = DebugCommonInitDriver();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = BridgeDispatcherInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Init any OS specific's */
+	eError = OSInitEnvData();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Early init. server cache maintenance */
+	eError = CacheOpInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RIInitKM();
+#endif
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG;
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnablePageFaultDebug,
+			&bAppHintDefault, &bEnablePageFaultDebug);
+	OSFreeAppHintState(pvAppHintState);
+
+	if (bEnablePageFaultDebug)
+	{
+		eError = DevicememHistoryInitKM();
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevicememHistoryInitKM", Error);
+	}
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	eError = CPUMappingHistoryInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	eError = PMRInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = DCInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	/* Initialise overall system state */
+	gpsPVRSRVData->eServicesState = PVRSRV_SERVICES_STATE_OK;
+
+	/* Create an event object */
+	eError = OSEventObjectCreate("PVRSRV_GLOBAL_EVENTOBJECT", &gpsPVRSRVData->hGlobalEventObject);
+	PVR_GOTO_IF_ERROR(eError, Error);
+	gpsPVRSRVData->ui32GEOConsecutiveTimeouts = 0;
+
+	eError = PVRSRVCmdCompleteInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = PVRSRVHandleInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = PVRSRV_APPHINT_CLEANUPTHREADPRIORITY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, CleanupThreadPriority,
+	                     &ui32AppHintDefault, &ui32AppHintCleanupThreadPriority);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, WatchdogThreadPriority,
+	                     &ui32AppHintDefault, &ui32AppHintWatchdogThreadPriority);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnableFullSyncTracking,
+			&bAppHintDefault, &bEnableFullSyncTracking);
+
+	_InitDefaultVzDriverMode(gpsPVRSRVData, pvAppHintState);
+
+
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	eError = HostMemDeviceCreate(&gpsPVRSRVData->psHostMemDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = _CleanupThreadPrepare(gpsPVRSRVData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CleanupThreadPrepare", Error);
+
+	/* Create a thread which is used to do the deferred cleanup */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hCleanupThread,
+	                                "pvr_defer_free",
+	                                CleanupThread,
+	                                CleanupThreadDumpInfo,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                ui32AppHintCleanupThreadPriority);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:1", Error);
+
+	/* Create the devices watchdog event object */
+	eError = OSEventObjectCreate("PVRSRV_DEVICESWATCHDOG_EVENTOBJECT", &gpsPVRSRVData->hDevicesWatchdogEvObj);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Error);
+
+	/* Create a thread which is used to detect fatal errors */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hDevicesWatchdogThread,
+	                                "pvr_device_wdg",
+	                                DevicesWatchdogThread,
+	                                NULL,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                ui32AppHintWatchdogThreadPriority);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:2", Error);
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Create the devices watchdog event object */
+	eError = OSEventObjectCreate("PVRSRV_AUTOVZ_WATCHDOG_EVENTOBJECT", &gpsPVRSRVData->hAutoVzWatchdogEvObj);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Error);
+
+	/* Create a thread that maintains the FW-KM connection by regularly updating the virtualization watchdog */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hAutoVzWatchdogThread,
+	                                "pvr_autovz_wdg",
+	                                AutoVzWatchdogThread,
+	                                NULL,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                OS_THREAD_HIGHEST_PRIORITY);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:3", Error);
+#endif /* SUPPORT_AUTOVZ */
+
+#if defined(SUPPORT_RGX)
+	eError = OSLockCreate(&gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", Error);
+#endif
+
+	/* Initialise the Transport Layer */
+	eError = TLInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Initialise pdump */
+	eError = PDUMPINIT();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	g_ui32InitFlags |= INIT_DATA_ENABLE_PDUMPINIT;
+
+	/* Initialise TL control stream */
+	eError = TLStreamCreate(&psPVRSRVData->hTLCtrlStream,
+	                        PVRSRV_TL_CTRL_STREAM,
+	                        PVRSRV_TL_CTRL_STREAM_SIZE,
+	                        TL_OPMODE_DROP_OLDEST,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+                            NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "TLStreamCreate");
+		psPVRSRVData->hTLCtrlStream = NULL;
+	}
+
+	eError = InfoPageCreate(psPVRSRVData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InfoPageCreate", Error);
+
+
+	/* Initialise the Timeout Info */
+	eError = InitialiseInfoPageTimeouts(psPVRSRVData);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = PopulateInfoPageBridges(psPVRSRVData);
+
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	if (bEnableFullSyncTracking)
+	{
+		psPVRSRVData->pui32InfoPage[DEBUG_FEATURE_FLAGS] |= DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED;
+	}
+	if (bEnablePageFaultDebug)
+	{
+		psPVRSRVData->pui32InfoPage[DEBUG_FEATURE_FLAGS] |= DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED;
+	}
+
+	{
+		IMG_UINT64 *pui64InfoPage;
+
+		pui64InfoPage = IMG_OFFSET_ADDR_DW(psPVRSRVData->pui32InfoPage,
+		                                   DEVMEM_INFO_PHYS_BUF_MAX_SIZE);
+		*pui64InfoPage = PMR_MAX_SUPPORTED_SIZE;
+	}
+
+	/* Initialise the Host Trace Buffer */
+	eError = HTBInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(SUPPORT_RGX)
+	RGXHWPerfClientInitAppHintCallbacks();
+#endif
+
+	eError = OSLockCreate(&psPVRSRVData->hClientStreamTableLock);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	psPVRSRVData->psClientStreamTable = HASH_Create_Extended(16, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+	                                                         HASH_Djb2_Hash, HASH_Djb2_Compare);
+	PVR_GOTO_IF_NOMEM(psPVRSRVData->psClientStreamTable, eError, Error);
+
+	/* Late init. client cache maintenance via info. page */
+	eError = CacheOpInit2();
+	PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpInit2", Error);
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	eError = SyncFbRegisterSyncFunctions();
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncFbRegisterSyncFunctions", Error);
+#endif
+
+#if defined(PDUMP)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	/* If initialising the device on first connection, we will
+	 * bind PDump capture to the first device we connect to later.
+	 */
+	psPVRSRVData->ui32PDumpBoundDevice = PVRSRV_MAX_DEVICES;
+#else
+	/* If not initialising the device on first connection, bind PDump
+	 * capture to device 0. This is because we need to capture PDump
+	 * during device initialisation but only want to capture PDump for
+	 * a single device (by default, device 0).
+	 */
+	psPVRSRVData->ui32PDumpBoundDevice = 0;
+#endif
+#endif
+
+	return 0;
+
+Error:
+	PVRSRVCommonDriverDeInit();
+	return eError;
+}
+
+void
+PVRSRVCommonDriverDeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bEnablePageFaultDebug = IMG_FALSE;
+
+	if (gpsPVRSRVData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: missing device-independent data",
+				 __func__));
+		return;
+	}
+
+	if (gpsPVRSRVData->psClientStreamTable != NULL)
+	{
+		HASH_Delete(gpsPVRSRVData->psClientStreamTable);
+	}
+
+	if (gpsPVRSRVData->hClientStreamTableLock != NULL)
+	{
+		OSLockDestroy(gpsPVRSRVData->hClientStreamTableLock);
+	}
+
+	if (gpsPVRSRVData->pui32InfoPage != NULL)
+	{
+		bEnablePageFaultDebug = GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED;
+	}
+
+	gpsPVRSRVData->bUnload = IMG_TRUE;
+
+#if defined(SUPPORT_RGX)
+	PVRSRVDestroyHWPerfHostThread();
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock)
+	{
+		OSLockDestroy(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+		gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock = NULL;
+	}
+#endif
+
+	if (gpsPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectSignal(gpsPVRSRVData->hGlobalEventObject);
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Stop and cleanup the devices watchdog thread */
+	if (gpsPVRSRVData->hAutoVzWatchdogThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hAutoVzWatchdogEvObj)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hAutoVzWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hAutoVzWatchdogThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hAutoVzWatchdogThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hAutoVzWatchdogEvObj)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hAutoVzWatchdogEvObj);
+		gpsPVRSRVData->hAutoVzWatchdogEvObj = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+#endif /* SUPPORT_AUTOVZ */
+
+	/* Stop and cleanup the devices watchdog thread */
+	if (gpsPVRSRVData->hDevicesWatchdogThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hDevicesWatchdogEvObj)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hDevicesWatchdogThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hDevicesWatchdogThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hDevicesWatchdogEvObj)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hDevicesWatchdogEvObj);
+		gpsPVRSRVData->hDevicesWatchdogEvObj = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	/* Stop and cleanup the deferred clean up thread, event object and
+	 * deferred context list.
+	 */
+	if (gpsPVRSRVData->hCleanupThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hCleanupEventObject)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hCleanupEventObject);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hCleanupThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hCleanupThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hCleanupEventObject)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hCleanupEventObject);
+		gpsPVRSRVData->hCleanupEventObject = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	/* Tear down the HTB before PVRSRVHandleDeInit() removes its TL handle */
+	/* HTB De-init happens in device de-registration currently */
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	/* Tear down CacheOp framework information page first */
+	CacheOpDeInit2();
+
+	/* Clean up information page */
+	InfoPageDestroy(gpsPVRSRVData);
+
+	/* Close the TL control plane stream. */
+	if (gpsPVRSRVData->hTLCtrlStream != NULL)
+	{
+		TLStreamClose(gpsPVRSRVData->hTLCtrlStream);
+	}
+
+	/* deinitialise pdump */
+	if ((g_ui32InitFlags & INIT_DATA_ENABLE_PDUMPINIT) > 0)
+	{
+		PDUMPDEINIT();
+	}
+
+	/* Clean up Transport Layer resources that remain */
+	TLDeInit();
+
+	HostMemDeviceDestroy(gpsPVRSRVData->psHostMemDeviceNode);
+	gpsPVRSRVData->psHostMemDeviceNode = NULL;
+
+	eError = PVRSRVHandleDeInit();
+	PVR_LOG_IF_ERROR(eError, "PVRSRVHandleDeInit");
+
+	/* destroy event object */
+	if (gpsPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectDestroy(gpsPVRSRVData->hGlobalEventObject);
+		gpsPVRSRVData->hGlobalEventObject = NULL;
+	}
+
+	PVRSRVCmdCompleteDeinit();
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = DCDeInit();
+	PVR_LOG_IF_ERROR(eError, "DCDeInit");
+#endif
+
+	eError = PMRDeInit();
+	PVR_LOG_IF_ERROR(eError, "PMRDeInit");
+
+	BridgeDispatcherDeinit();
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	CPUMappingHistoryDeInit();
+#endif
+
+	if (bEnablePageFaultDebug)
+	{
+		/* Clear all allocated history tracking data */
+		DevicememHistoryDeInitKM();
+	}
+
+	CacheOpDeInit();
+
+	OSDeInitEnvData();
+
+	(void) DevmemIntDeInit();
+
+	ServerBridgeDeInit();
+
+	HTB_DestroyDIEntry();
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDestroyDI();		/* Stage 1 freeing */
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+	DebugCommonDeInitDriver();
+
+	DIDeInit();
+
+	if (gpsPVRSRVData->hThreadsDbgReqNotify)
+	{
+		PVRSRVUnregisterDriverDbgRequestNotify(gpsPVRSRVData->hThreadsDbgReqNotify);
+	}
+
+	PVRSRVUnregisterDriverDbgTable();
+
+	OSWRLockDestroy(gpsPVRSRVData->hDeviceNodeListLock);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDestroy();		/* Stage 2 freeing */
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RIDeInitKM();
+#endif
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+	OSFreeMemNoStats(gpsPVRSRVData);		/* Not trackable */
+
+	gpsPVRSRVData = NULL;
+}
+
+static void _SysDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	/* Only dump info once */
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hDebugRequestHandle;
+
+	PVR_UNREFERENCED_PARAMETER(ui32VerbLevel);
+
+	PVR_DUMPDEBUG_LOG("------[ System Summary Device ID:%d ]------", psDeviceNode->sDevId.ui32InternalID);
+
+	switch (psDeviceNode->eCurrentSysPowerState)
+	{
+		case PVRSRV_SYS_POWER_STATE_OFF:
+			PVR_DUMPDEBUG_LOG("Device System Power State: OFF");
+			break;
+		case PVRSRV_SYS_POWER_STATE_ON:
+			PVR_DUMPDEBUG_LOG("Device System Power State: ON");
+			break;
+		default:
+			PVR_DUMPDEBUG_LOG("Device System Power State: UNKNOWN (%d)",
+							   psDeviceNode->eCurrentSysPowerState);
+			break;
+	}
+
+	PVR_DUMPDEBUG_LOG("MaxHWTOut: %dus, WtTryCt: %d, WDGTOut(on,off): (%dms,%dms)",
+	                  MAX_HW_TIME_US, WAIT_TRY_COUNT, DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT, DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT);
+
+	SysDebugInfo(psDeviceNode->psDevConfig, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+PHYS_HEAP_CONFIG* PVRSRVFindPhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										   PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		if (BITMASK_HAS(psDevConfig->pasPhysHeaps[i].ui32UsageFlags, ui32Flags))
+		{
+			return &psDevConfig->pasPhysHeaps[i];
+		}
+	}
+
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVAcquireInternalID(IMG_UINT32 *pui32InternalID)
+{
+	IMG_UINT32 ui32InternalID = 0;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	for (ui32InternalID = 0;
+		 ui32InternalID < PVRSRV_MAX_DEVICES;
+		 ui32InternalID++)
+	{
+		if (PVRSRVGetDeviceInstance(ui32InternalID) == NULL)
+		{
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (bFound)
+	{
+		*pui32InternalID = ui32InternalID;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		return PVRSRV_ERROR_NO_FREE_DEVICEIDS_AVAILABLE;
+	}
+}
+
+PVRSRV_ERROR PVRSRVCommonDeviceCreate(void *pvOSDevice,
+											 IMG_INT32 i32KernelDeviceID,
+											 PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_DATA				*psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR			eError;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig;
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+	IMG_UINT32				ui32InternalID;
+
+	void *pvAppHintState    = NULL;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_HANDLE				hProcessStats;
+#endif
+	IMG_BOOL				bAppHintDefault;
+	IMG_BOOL				bEnablePageFaultDebug = IMG_FALSE;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL				bAutoVzGPUPowerdown   = IMG_FALSE;
+#endif
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceCreate: DevId %d", i32KernelDeviceID);
+
+	/* Determine if we've got EnablePageFaultDebug set or not */
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnablePageFaultDebug,
+			&bAppHintDefault, &bEnablePageFaultDebug);
+
+#if defined(SUPPORT_AUTOVZ)
+	bAppHintDefault = PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, AutoVzGPUPowerdown,
+			&bAppHintDefault, &bAutoVzGPUPowerdown);
+#endif
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	psDeviceNode = OSAllocZMemNoStats(sizeof(*psDeviceNode));
+	PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "psDeviceNode");
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Allocate process statistics */
+	eError = PVRSRVStatsRegisterProcess(&hProcessStats);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsRegisterProcess", ErrorFreeDeviceNode);
+#endif
+
+	dllist_init(&psDeviceNode->sCleanupThreadWorkList);
+	OSAtomicWrite(&psDeviceNode->i32NumCleanupItems, 0);
+
+	/* Record setting of EnablePageFaultDebug in device-node */
+	psDeviceNode->bEnablePFDebug = bEnablePageFaultDebug;
+#if defined(SUPPORT_AUTOVZ)
+	psDeviceNode->bAutoVzAllowGPUPowerdown = bAutoVzGPUPowerdown;
+#endif
+	psDeviceNode->sDevId.i32KernelDeviceID = i32KernelDeviceID;
+	eError = PVRSRVAcquireInternalID(&ui32InternalID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireInternalID", ErrorDeregisterStats);
+
+	eError = SysDevInit(pvOSDevice, &psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysDevInit", ErrorDeregisterStats);
+
+	/* Lock down the InternalID for this device by saving the value into
+	 * its device node. This is done after SysDevInit to allow the system
+	 * layer to identify the current device's the internal ID the same way. */
+	psDeviceNode->sDevId.ui32InternalID = ui32InternalID;
+
+	PVR_ASSERT(psDevConfig);
+	PVR_ASSERT(psDevConfig->pvOSDevice == pvOSDevice);
+	PVR_ASSERT(!psDevConfig->psDevNode);
+
+	if ((psDevConfig->eDefaultHeap != PVRSRV_PHYS_HEAP_GPU_LOCAL) &&
+	    (psDevConfig->eDefaultHeap != PVRSRV_PHYS_HEAP_CPU_LOCAL))
+	{
+		PVR_LOG_MSG(PVR_DBG_ERROR, "DEFAULT Heap is invalid, "
+		                           "it must be GPU_LOCAL or CPU_LOCAL");
+		PVR_LOG_GOTO_IF_ERROR(eError, "SysDevInit", ErrorDeregisterStats);
+	}
+	PVR_DPF((PVR_DBG_MESSAGE, "Device PhysHeap Default: %s",
+		(psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_CPU_LOCAL)
+		? "CPU_LOCAL" : "GPU_LOCAL"));
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_CREATING);
+
+	psDeviceNode->psDevConfig = psDevConfig;
+	psDevConfig->psDevNode = psDeviceNode;
+
+	if (psDevConfig->pfnGpuDomainPower)
+	{
+		psDeviceNode->eCurrentSysPowerState = psDevConfig->pfnGpuDomainPower(psDeviceNode);
+	}
+	else
+	{
+		/* If the System Layer doesn't provide a function to query the power state
+		 * of the system hardware, assume the system starting state below. */
+
+#if defined(SUPPORT_AUTOVZ)
+		psDeviceNode->eCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_ON;
+#else
+		psDeviceNode->eCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_OFF;
+#endif
+	}
+
+
+	/* Next update value will be 0xFFFFFFF7 since sync prim starts with 0xFFFFFFF6.
+	 * Has to be set before call to PMRInitDevice(). */
+	psDeviceNode->ui32NextMMUInvalidateUpdate = 0xFFFFFFF7U;
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+	psDeviceNode->uiPowerOffCounter = 0;
+	psDeviceNode->uiPowerOffCounterNext = 1;
+#endif
+
+	eError = PVRSRVRegisterDeviceDbgTable(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorPvzConnectionDeInit);
+
+	eError = PVRSRVPowerLockInit(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnregisterDbgTable);
+
+	eError = PhysHeapInitDeviceHeaps(psDeviceNode, psDevConfig);
+	PVR_GOTO_IF_ERROR(eError, ErrorPowerLockDeInit);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	eError = PMRInitDevice(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorPhysHeapDeInitDeviceHeaps);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(SUPPORT_RGX)
+	/* Requirements:
+	 *  registered GPU and FW local heaps */
+	/*  debug table */
+	eError = RGXRegisterDevice(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXRegisterDevice");
+		eError = PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+		goto ErrorPMRDeInitDevice;
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	if (PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		IMG_UINT32 ui32AppHintDefault = 0;
+		IMG_UINT32 ui32AppHintPhysMemTestPasses;
+
+		/* Read AppHint - Configurable memory test pass count */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, PhysMemTestPasses,
+				&ui32AppHintDefault, &ui32AppHintPhysMemTestPasses);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		if (ui32AppHintPhysMemTestPasses > 0)
+		{
+			eError = PhysMemTest(psDeviceNode, psDevConfig, ui32AppHintPhysMemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysMemTest", ErrorDeInitRgx);
+		}
+	}
+#endif
+#endif
+
+	/* Inform the device layer PhysHeaps are now initialised so that device
+	 * specific heaps can be obtained along with carrying out any Vz setup. */
+	if (psDeviceNode->pfnPhysMemDeviceHeapsInit != NULL)
+	{
+		eError = psDeviceNode->pfnPhysMemDeviceHeapsInit(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorPMRDeInitDevice);
+	}
+
+	/* Carry out initialisation of a dedicated FW MMU data, if the FW CPU has
+	 * an MMU separate to the GPU MMU e.g. MIPS based FW. */
+	if (psDeviceNode->pfnFwMMUInit != NULL)
+	{
+		eError = psDeviceNode->pfnFwMMUInit(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorFwMMUDeinit);
+	}
+
+	eError = SyncServerInit(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorDeInitRgx);
+
+	eError = SyncCheckpointInit(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncCheckpointInit", ErrorSyncCheckpointInit);
+
+	/*
+	 * This is registered before doing device specific initialisation to ensure
+	 * generic device information is dumped first during a debug request.
+	 */
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDeviceNode->hDbgReqNotify,
+		                                          psDeviceNode,
+		                                          _SysDebugRequestNotify,
+		                                          DEBUG_REQUEST_SYS,
+		                                          psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify", ErrorRegDbgReqNotify);
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	eError = InitDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InitDVFS", ErrorDVFSInitFail);
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+	eError = InitPDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InitPDVFS", ErrorPDVFSInitFail);
+#endif
+
+	OSAtomicWrite(&psDeviceNode->iNumClockSpeedChanges, 0);
+
+	OSWRLockCreate(&psDeviceNode->hPageFaultNotifyLock);
+	if (psDeviceNode->hPageFaultNotifyLock == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for PF notify list",
+		        __func__));
+		goto ErrorPageFaultLockFailCreate;
+	}
+
+	dllist_init(&psDeviceNode->sMemoryContextPageFaultNotifyListHead);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Registered device %p", psDeviceNode));
+	PVR_DPF((PVR_DBG_MESSAGE, "Register bank address = 0x%08lx",
+			 (unsigned long)psDevConfig->sRegsCpuPBase.uiAddr));
+	PVR_DPF((PVR_DBG_MESSAGE, "IRQ = %d", psDevConfig->ui32IRQ));
+
+/* SUPPORT_ALT_REGBASE is defined for rogue cores only */
+#if defined(SUPPORT_RGX) && defined(SUPPORT_ALT_REGBASE)
+	{
+		IMG_DEV_PHYADDR sRegsGpuPBase;
+
+		PhysHeapCpuPAddrToDevPAddr(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+		                           1,
+		                           &sRegsGpuPBase,
+		                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+
+		PVR_LOG(("%s: Using alternate Register bank GPU address: 0x%08lx (orig: 0x%08lx)", __func__,
+		         (unsigned long)psDevConfig->sAltRegsGpuPBase.uiAddr,
+		         (unsigned long)sRegsGpuPBase.uiAddr));
+	}
+#endif
+
+	eError = DebugCommonInitDevice(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DebugCommonInitDevice",
+	                      ErrorDestroyMemoryContextPageFaultNotifyListLock);
+
+	/* Create the devicemem_history hook for the device. We need to
+	 * have the debug-info instantiated before calling this.
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		eError = DevicememHistoryDeviceCreate(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevicememHistoryDeviceCreate", ErrorDebugCommonDeInitDevice);
+	}
+
+#if defined(__linux__)
+	/* Register the device specific AppHints so individual AppHints can be
+	 * configured before the FW is initialised. This must be called after
+	 * DebugCommonInitDevice() above as it depends on the created gpuXX/apphint
+	 * DI Group.
+	 */
+	{
+		int iError = pvr_apphint_device_register(psDeviceNode);
+		PVR_LOG_IF_FALSE(iError == 0, "pvr_apphint_device_register() failed");
+	}
+#endif /* defined(__linux__) */
+
+#if defined(SUPPORT_RGX)
+	RGXHWPerfInitAppHintCallbacks(psDeviceNode);
+#endif
+
+	/* Finally insert the device into the dev-list and set it as active */
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_InsertTail(&psPVRSRVData->psDeviceNodeList,
+									   psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices++;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+	/* Initialise the paravirtualised connection */
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		PvzConnectionInit(psDevConfig);
+		PVR_GOTO_IF_ERROR(eError, ErrorSysDevDeInit);
+	}
+
+	BIT_SET(psDevConfig->psDevNode->ui32VmState, RGXFW_HOST_DRIVER_ID);
+
+	*ppsDeviceNode = psDeviceNode;
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	/* Register the DVFS device now the device node is present in the dev-list */
+	eError = RegisterDVFSDevice(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RegisterDVFSDevice", ErrorRegisterDVFSDeviceFail);
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+#endif
+
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_CREATED);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+ErrorRegisterDVFSDeviceFail:
+	/* Remove the device from the list */
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_Remove(psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices--;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+#if defined(__linux__)
+	pvr_apphint_device_unregister(psDeviceNode);
+#endif /* defined(__linux__) */
+
+	/* Remove the devicemem_history hook if we created it */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		DevicememHistoryDeviceDestroy(psDeviceNode);
+	}
+#endif
+
+ErrorDebugCommonDeInitDevice:
+	DebugCommonDeInitDevice(psDeviceNode);
+
+ErrorDestroyMemoryContextPageFaultNotifyListLock:
+	OSWRLockDestroy(psDeviceNode->hPageFaultNotifyLock);
+	psDeviceNode->hPageFaultNotifyLock = NULL;
+
+ErrorPageFaultLockFailCreate:
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+ErrorDVFSInitFail:
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+ErrorPDVFSInitFail:
+#endif
+
+	if (psDeviceNode->hDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceNode->hDbgReqNotify);
+	}
+
+ErrorRegDbgReqNotify:
+	SyncCheckpointDeinit(psDeviceNode);
+
+ErrorSyncCheckpointInit:
+	SyncServerDeinit(psDeviceNode);
+
+ErrorDeInitRgx:
+#if defined(SUPPORT_RGX)
+	DevDeInitRGX(psDeviceNode);
+#endif
+ErrorFwMMUDeinit:
+ErrorPMRDeInitDevice:
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PMRDeInitDevice(psDeviceNode);
+ErrorPhysHeapDeInitDeviceHeaps:
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+ErrorPowerLockDeInit:
+	PVRSRVPowerLockDeInit(psDeviceNode);
+ErrorUnregisterDbgTable:
+	PVRSRVUnregisterDeviceDbgTable(psDeviceNode);
+ErrorPvzConnectionDeInit:
+	psDevConfig->psDevNode = NULL;
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		PvzConnectionDeInit(psDevConfig);
+	}
+ErrorSysDevDeInit:
+	SysDevDeInit(psDevConfig);
+ErrorDeregisterStats:
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+ErrorFreeDeviceNode:
+#endif
+	OSFreeMemNoStats(psDeviceNode);
+
+	return eError;
+}
+
+#if defined(SUPPORT_RGX)
+static PVRSRV_ERROR _SetDeviceFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL bValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXSetDeviceFlags((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                            ui32Flag, bValue);
+
+	return eResult;
+}
+
+static PVRSRV_ERROR _ReadDeviceFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL *pbValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+	IMG_UINT32 ui32State;
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXGetDeviceFlags((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                            &ui32State);
+
+	if (PVRSRV_OK == eResult)
+	{
+		*pbValue = (ui32State & ui32Flag)? IMG_TRUE: IMG_FALSE;
+	}
+
+	return eResult;
+}
+static PVRSRV_ERROR _SetStateFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                  const void *psPrivate, IMG_BOOL bValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXStateFlagCtrl((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                           ui32Flag, NULL, bValue);
+
+	return eResult;
+}
+
+static PVRSRV_ERROR _ReadStateFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL *pbValue)
+{
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+	IMG_UINT32 ui32State;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevice->pvDevice;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags,
+	                           INVALIDATE);
+	ui32State = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+
+	if (pbValue)
+	{
+		*pbValue = (ui32State & ui32Flag)? IMG_TRUE: IMG_FALSE;
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+PVRSRV_ERROR PVRSRVCommonDeviceInitialise(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_BOOL bInitSuccessful = IMG_FALSE;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_HANDLE hProcessStats;
+#endif
+	PVRSRV_ERROR eError;
+
+	PDUMPCOMMENT(psDeviceNode, "Common Device Initialisation");
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceInitialise: DevId %d", psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_CREATED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device already initialised", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Allocate an OSEventObject to use in the Freeze / Thaw transitioning to
+	 * allow LBIST requests to be serviced. This object is held for the life of
+	 * this device-node.
+	 */
+	eError = OSEventObjectCreate("PVRSRV_DEVICE_THREAD_EVENT_OBJECT",
+	                             &psDeviceNode->hDeviceThreadEvObj);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSEventObjectCreate");
+
+	eError = OSEventObjectOpen(psDeviceNode->hDeviceThreadEvObj,
+	                           &psDeviceNode->hDeviceFreezeThaw);
+
+	if (PVRSRV_OK != eError)
+	{
+		OSEventObjectDestroy(psDeviceNode->hDeviceThreadEvObj);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "OSEventObjectOpen");
+	}
+
+	/* Initial zero-set for the number of Frozen threads */
+	OSAtomicWrite(&psDeviceNode->iFreezeCount, 0);
+	OSAtomicWrite(&psDeviceNode->iTotalFreezes, 0);
+
+	/* Initial zero-set for number of active threads on this device */
+	OSAtomicWrite(&psDeviceNode->iThreadsActive, 0);
+
+	/* Allocate devmem_history backing store for the device if we have
+	 * EnablePageFaultDebug set
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		eError = DevicememHistoryDeviceInit(psDeviceNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevicememHistoryDeviceInit");
+	}
+
+#if defined(PDUMP)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	{
+		PVRSRV_DATA *psSRVData = PVRSRVGetPVRSRVData();
+
+		/* If first connection, bind this and future PDump clients to use this device */
+		if (psSRVData->ui32PDumpBoundDevice == PVRSRV_MAX_DEVICES)
+		{
+			psSRVData->ui32PDumpBoundDevice = psDeviceNode->sDevId.ui32InternalID;
+		}
+	}
+#endif
+#endif
+
+	/* Initialise Connection_Data access mechanism */
+	dllist_init(&psDeviceNode->sConnections);
+	eError = OSLockCreate(&psDeviceNode->hConnectionsLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	/* Allocate process statistics */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	eError = PVRSRVStatsRegisterProcess(&hProcessStats);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVStatsRegisterProcess");
+#endif
+
+	eError = MMU_InitDevice(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_InitDevice");
+
+#if defined(SUPPORT_RGX)
+	eError = RGXInit(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInit", Exit);
+#endif
+
+	bInitSuccessful = IMG_TRUE;
+
+#if defined(SUPPORT_RGX)
+Exit:
+#endif
+	eError = PVRSRVDeviceFinalise(psDeviceNode, bInitSuccessful);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceFinalise");
+
+#if defined(SUPPORT_RGX)
+	if (bInitSuccessful && (eError == PVRSRV_OK))
+	{
+		RGXInitMultiCoreInfo(psDeviceNode);
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableClockGating,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  APPHINT_OF_DRIVER_NO_DEVICE,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_DISABLE_CLKGATING_EN));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableDMOverlap,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  APPHINT_OF_DRIVER_NO_DEVICE,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_DISABLE_DM_OVERLAP));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_AssertOnHWRTrigger,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_AssertOutOfMemory,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_CheckMList,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_CHECK_MLIST_EN));
+		}
+
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableFEDLogging,
+		                                  _ReadDeviceFlag, _SetDeviceFlag,
+		                                  psDeviceNode,
+		                                  (void*)((uintptr_t)RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN));
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_ZeroFreelist,
+		                                  _ReadDeviceFlag, _SetDeviceFlag,
+		                                  psDeviceNode,
+		                                  (void*)((uintptr_t)RGXKM_DEVICE_STATE_ZERO_FREELIST));
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisablePDumpPanic,
+		                                  RGXQueryPdumpPanicDisable, RGXSetPdumpPanicDisable,
+		                                  psDeviceNode,
+		                                  NULL);
+	}
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+#endif
+
+	return eError;
+}
+
+void PVRSRVCommonDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DATA				*psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR			eError;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL				bForceUnload = IMG_FALSE;
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+	if ((PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK) ||
+		(eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_DEAD) ||
+		(eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+	{
+		bForceUnload = IMG_TRUE;
+	}
+#endif
+
+	/* Remove DI hook for the devicemem_history for this device (if any).
+	 * The associated devicemem_history buffers are freed by the final
+	 * call to DevicememHistoryDeInitKM() as they are used asynchronously
+	 * by other parts of the DDK.
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		DevicememHistoryDeviceDestroy(psDeviceNode);
+	}
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceDestroy: DevId %d", psDeviceNode->sDevId.i32KernelDeviceID);
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_DEINIT);
+
+	PVRSRVCleanupThreadWaitForDevice(psDeviceNode);
+
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_Remove(psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices--;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_DESTRUCTING);
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	UnregisterDVFSDevice(psDeviceNode);
+#endif
+
+#if defined(__linux__)
+	pvr_apphint_device_unregister(psDeviceNode);
+#endif /* defined(__linux__) */
+
+	DebugCommonDeInitDevice(psDeviceNode);
+
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	SyncFbDeregisterDevice(psDeviceNode);
+#endif
+	/* Counter part to what gets done in PVRSRVDeviceFinalise */
+	if (psDeviceNode->hSyncCheckpointContext)
+	{
+		SyncCheckpointContextDestroy(psDeviceNode->hSyncCheckpointContext);
+		psDeviceNode->hSyncCheckpointContext = NULL;
+	}
+	if (psDeviceNode->hSyncPrimContext)
+	{
+		if (psDeviceNode->psMMUCacheSyncPrim)
+		{
+			PVRSRV_CLIENT_SYNC_PRIM *psSync = psDeviceNode->psMMUCacheSyncPrim;
+
+			/* Ensure there are no pending MMU Cache Ops in progress before freeing this sync. */
+			eError = PVRSRVPollForValueKM(psDeviceNode,
+			                              psSync->pui32LinAddr,
+			                              psDeviceNode->ui32NextMMUInvalidateUpdate-1,
+			                              0xFFFFFFFF,
+			                              POLL_FLAG_LOG_ERROR,
+			                              NULL);
+			PVR_LOG_IF_ERROR(eError, "PVRSRVPollForValueKM");
+
+			/* Important to set the device node pointer to NULL
+			 * before we free the sync-prim to make sure we don't
+			 * defer the freeing of the sync-prim's page tables itself.
+			 * The sync is used to defer the MMU page table
+			 * freeing. */
+			psDeviceNode->psMMUCacheSyncPrim = NULL;
+
+			/* Free general purpose sync primitive */
+			SyncPrimFree(psSync);
+		}
+
+		SyncPrimContextDestroy(psDeviceNode->hSyncPrimContext);
+		psDeviceNode->hSyncPrimContext = NULL;
+	}
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError == PVRSRV_OK)
+	{
+		IMG_BOOL bHasPowerLock = IMG_TRUE;
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		/*
+		 * Firmware probably not responding if bForceUnload is set, but we still want to unload the
+		 * driver.
+		 */
+		if (!bForceUnload)
+#endif
+		{
+			/* Force idle device */
+			eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_TRUE);
+			PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+			if (eError == PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				bHasPowerLock = IMG_FALSE;
+			}
+		}
+
+		if (bHasPowerLock)
+		{
+			/* Power down the device if necessary */
+			eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+												 PVRSRV_DEV_POWER_STATE_OFF,
+												 PVRSRV_POWER_FLAGS_FORCED);
+			PVRSRVPowerUnlock(psDeviceNode);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVSetDevicePowerStateKM");
+
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+		}
+	}
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	DeinitDVFS(psDeviceNode);
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+	DeinitPDVFS(psDeviceNode);
+#endif
+
+	if (psDeviceNode->hDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceNode->hDbgReqNotify);
+	}
+
+	SyncCheckpointDeinit(psDeviceNode);
+
+	SyncServerDeinit(psDeviceNode);
+
+	MMU_DeInitDevice(psDeviceNode);
+
+#if defined(SUPPORT_RGX)
+	DevDeInitRGX(psDeviceNode);
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* must be called before PhysHeapDeInitDeviceHeaps() */
+	PMRDeInitDevice(psDeviceNode);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+	PVRSRVPowerLockDeInit(psDeviceNode);
+
+	PVRSRVUnregisterDeviceDbgTable(psDeviceNode);
+
+	if (psDeviceNode->hPageFaultNotifyLock != NULL)
+	{
+		OSWRLockDestroy(psDeviceNode->hPageFaultNotifyLock);
+	}
+
+	/* Release the Connection-Data lock as late as possible. */
+	if (psDeviceNode->hConnectionsLock)
+	{
+		OSLockDestroy(psDeviceNode->hConnectionsLock);
+	}
+
+	/* Release the hDeviceThreadEvObj as late as possible. */
+	if (psDeviceNode->hDeviceThreadEvObj)
+	{
+		OSEventObjectClose(psDeviceNode->hDeviceFreezeThaw);
+
+		OSEventObjectDestroy(psDeviceNode->hDeviceThreadEvObj);
+	}
+
+	PVR_ASSERT(OSAtomicRead(&psDeviceNode->iFreezeCount) == 0);
+
+	psDeviceNode->psDevConfig->psDevNode = NULL;
+
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		PvzConnectionDeInit(psDeviceNode->psDevConfig);
+	}
+	SysDevDeInit(psDeviceNode->psDevConfig);
+
+	OSFreeMemNoStats(psDeviceNode);
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceFinalise
+@Description  Performs the final parts of device initialisation.
+@Input        psDeviceNode            Device node of the device to finish
+                                      initialising
+@Input        bInitSuccessful         Whether or not device specific
+                                      initialisation was successful
+@Return       PVRSRV_ERROR     PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceFinalise(PVRSRV_DEVICE_NODE *psDeviceNode,
+											   IMG_BOOL bInitSuccessful)
+{
+	PVRSRV_ERROR eError;
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	if (bInitSuccessful)
+	{
+		eError = SyncCheckpointContextCreate(psDeviceNode,
+											 &psDeviceNode->hSyncCheckpointContext);
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncCheckpointContextCreate", ErrorExit);
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+		eError = SyncFbRegisterDevice(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+#endif
+		eError = SyncPrimContextCreate(psDeviceNode,
+									   &psDeviceNode->hSyncPrimContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "SyncPrimContextCreate");
+			SyncCheckpointContextDestroy(psDeviceNode->hSyncCheckpointContext);
+			goto ErrorExit;
+		}
+
+		/* Allocate MMU cache invalidate sync */
+		eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
+							   &psDeviceNode->psMMUCacheSyncPrim,
+							   "pvrsrv dev MMU cache");
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncPrimAlloc", ErrorExit);
+
+		/* Set the sync prim value to a much higher value near the
+		 * wrapping range. This is so any wrapping bugs would be
+		 * seen early in the driver start-up.
+		 */
+		SyncPrimSet(psDeviceNode->psMMUCacheSyncPrim, 0xFFFFFFF6UL);
+
+		eError = PVRSRVPowerLock(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPowerLock", ErrorExit);
+
+		/*
+		 * Always ensure a single power on command appears in the pdump. This
+		 * should be the only power related call outside of PDUMPPOWCMDSTART
+		 * and PDUMPPOWCMDEND.
+		 */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_FORCED);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set device %p power state to 'on' (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			PVRSRVPowerUnlock(psDeviceNode);
+			goto ErrorExit;
+		}
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+		eError = ValidateFWOnLoad(psDeviceNode->pvDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "ValidateFWOnLoad");
+			PVRSRVPowerUnlock(psDeviceNode);
+			return eError;
+		}
+#endif
+
+		/*
+		 * Guest driver must do a runtime compatibility check against the
+		 * data provided by the Firmware.
+		 */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Failed compatibility check for device %p (%s)",
+						 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVPowerUnlock(psDeviceNode);
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				goto ErrorExit;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+
+		/* Force the device to idle if its default power state is off */
+		eError = PVRSRVDeviceIdleRequestKM(psDeviceNode,
+										   &PVRSRVDeviceIsDefaultStateOFF,
+										   IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+			if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+			goto ErrorExit;
+		}
+
+		/* Place device into its default power state. */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_DEFAULT,
+											 PVRSRV_POWER_FLAGS_FORCED);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set device %p into its default power state (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			goto ErrorExit;
+		}
+
+		PVRSRVPowerUnlock(psDeviceNode);
+
+		/*
+		 * If PDUMP is enabled and RGX device is supported, then initialise the
+		 * performance counters that can be further modified in PDUMP. Then,
+		 * before ending the init phase of the pdump, drain the commands put in
+		 * the kCCB during the init phase.
+		 */
+#if defined(SUPPORT_RGX)
+#if defined(PDUMP)
+		{
+			eError = RGXInitHWPerfCounters(psDeviceNode);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitHWPerfCounters", ErrorExit);
+
+			eError = RGXPdumpDrainKCCB(psDevInfo,
+									   psDevInfo->psKernelCCBCtlLocal->ui32WriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", ErrorExit);
+		}
+#endif
+#endif /* defined(SUPPORT_RGX) */
+		/* Now that the device(s) are fully initialised set them as active */
+		PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_ACTIVE);
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		/* Initialisation failed so set the device(s) into a bad state */
+		PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_BAD);
+		eError = PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	/* Give PDump control a chance to end the init phase, depends on OS */
+	PDUMPENDINITPHASE(psDeviceNode);
+	return eError;
+
+ErrorExit:
+	/* Initialisation failed so set the device(s) into a bad state */
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_BAD);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* Only check devices which specify a compatibility check callback */
+	if (psDeviceNode->pfnInitDeviceCompatCheck)
+		return psDeviceNode->pfnInitDeviceCompatCheck(psDeviceNode);
+	else
+		return PVRSRV_OK;
+}
+
+/*
+	PollForValueKM
+*/
+static
+PVRSRV_ERROR PollForValueKM (volatile IMG_UINT32 __iomem           *pui32LinMemAddr,
+										  IMG_UINT32                ui32Value,
+										  IMG_UINT32                ui32Mask,
+										  IMG_UINT32                ui32Timeoutus,
+										  IMG_UINT32                ui32PollPeriodus,
+										  POLL_FLAGS                ePollFlags,
+										  PFN_INVALIDATE_CACHEFUNC  pfnFwInvalidate)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(pui32LinMemAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(ui32Timeoutus);
+	PVR_UNREFERENCED_PARAMETER(ui32PollPeriodus);
+	PVR_UNREFERENCED_PARAMETER(ePollFlags);
+	return PVRSRV_OK;
+#else
+	IMG_UINT32 ui32ActualValue = 0xFFFFFFFFU; /* Initialiser only required to prevent incorrect warning */
+
+	LOOP_UNTIL_TIMEOUT_US(ui32Timeoutus)
+	{
+		if (pfnFwInvalidate)
+		{
+			pfnFwInvalidate((const volatile void __force *)pui32LinMemAddr,
+			                sizeof(*pui32LinMemAddr),
+			                PVRSRV_CACHE_OP_INVALIDATE);
+		}
+
+		ui32ActualValue = OSReadHWReg32((void __iomem *)pui32LinMemAddr, 0) & ui32Mask;
+
+		if (ui32ActualValue == ui32Value)
+		{
+			return PVRSRV_OK;
+		}
+
+		if (gpsPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+#define ONE_MSEC_IN_USECS	(1000U)
+
+		if (ui32PollPeriodus <= ONE_MSEC_IN_USECS)
+		{
+			OSWaitus(ui32PollPeriodus);
+		}
+		else
+		{
+			OSSleepms(ui32PollPeriodus / ONE_MSEC_IN_USECS);
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (BITMASK_HAS(ePollFlags, POLL_FLAG_LOG_ERROR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PollForValueKM: Timeout. Expected 0x%x but found 0x%x (mask 0x%x).",
+		         ui32Value, ui32ActualValue, ui32Mask));
+	}
+
+	return PVRSRV_ERROR_TIMEOUT;
+#endif /* NO_HARDWARE */
+}
+
+
+/*
+	PVRSRVPollForValueKM
+*/
+PVRSRV_ERROR PVRSRVPollForValueKM (PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem   *pui32LinMemAddr,
+												IMG_UINT32                     ui32Value,
+												IMG_UINT32                     ui32Mask,
+												POLL_FLAGS                     ePollFlags,
+												PFN_INVALIDATE_CACHEFUNC       pfnFwInvalidate)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PollForValueKM(pui32LinMemAddr, ui32Value, ui32Mask,
+						  MAX_HW_TIME_US,
+						  MAX_HW_TIME_US/WAIT_TRY_COUNT,
+						  ePollFlags,
+						  pfnFwInvalidate);
+	if (eError != PVRSRV_OK && BITMASK_HAS(ePollFlags, POLL_FLAG_DEBUG_DUMP))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		                        __func__, PVRSRVGetErrorString(eError),
+								pui32LinMemAddr, ui32Value));
+		PVRSRVDebugRequest(psDevNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVWaitForValueKM(volatile IMG_UINT32 __iomem   *pui32LinMemAddr,
+                     IMG_UINT32                    ui32Value,
+                     IMG_UINT32                    ui32Mask,
+                     PFN_INVALIDATE_CACHEFUNC      pfnFwInvalidate)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(pui32LinMemAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	return PVRSRV_OK;
+#else
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eErrorWait;
+	IMG_UINT32 ui32ActualValue;
+
+	eError = OSEventObjectOpen(psPVRSRVData->hGlobalEventObject, &hOSEvent);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectOpen", EventObjectOpenError);
+
+	eError = PVRSRV_ERROR_TIMEOUT; /* Initialiser for following loop */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (pfnFwInvalidate)
+		{
+			pfnFwInvalidate((const volatile void __force *)pui32LinMemAddr,
+			                sizeof(*pui32LinMemAddr),
+			                PVRSRV_CACHE_OP_INVALIDATE);
+		}
+
+		ui32ActualValue = (OSReadDeviceMem32(pui32LinMemAddr) & ui32Mask);
+
+		if (ui32ActualValue == ui32Value)
+		{
+			/* Expected value has been found */
+			eError = PVRSRV_OK;
+			break;
+		}
+		else if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			/* Services in bad state, don't wait any more */
+			eError = PVRSRV_ERROR_NOT_READY;
+			break;
+		}
+		else
+		{
+			/* wait for event and retry */
+			eErrorWait = OSEventObjectWait(hOSEvent);
+			if (eErrorWait != PVRSRV_OK  &&  eErrorWait != PVRSRV_ERROR_TIMEOUT)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Failed with error %d. Found value 0x%x but was expected "
+				         "to be 0x%x (Mask 0x%08x). Retrying",
+						 __func__,
+						 eErrorWait,
+						 ui32ActualValue,
+						 ui32Value,
+						 ui32Mask));
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	OSEventObjectClose(hOSEvent);
+
+	/* One last check in case the object wait ended after the loop timeout... */
+	if (eError != PVRSRV_OK &&
+	    (OSReadDeviceMem32(pui32LinMemAddr) & ui32Mask) == ui32Value)
+	{
+		eError = PVRSRV_OK;
+	}
+
+	/* Provide event timeout information to aid the Device Watchdog Thread... */
+	if (eError == PVRSRV_OK)
+	{
+		psPVRSRVData->ui32GEOConsecutiveTimeouts = 0;
+	}
+	else if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		psPVRSRVData->ui32GEOConsecutiveTimeouts++;
+	}
+
+EventObjectOpenError:
+
+	return eError;
+
+#endif /* NO_HARDWARE */
+}
+
+int PVRSRVGetDriverStatus(void)
+{
+	return PVRSRVGetPVRSRVData()->eServicesState;
+}
+
+/*
+	PVRSRVSystemHasCacheSnooping
+*/
+IMG_BOOL PVRSRVSystemHasCacheSnooping(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if ((psDevConfig->eCacheSnoopingMode != PVRSRV_DEVICE_SNOOP_NONE) &&
+		(psDevConfig->eCacheSnoopingMode != PVRSRV_DEVICE_SNOOP_EMULATED))
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemSnoopingIsEmulated(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_EMULATED)
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemSnoopingOfCPUCache(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if ((psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_CPU_ONLY) ||
+		(psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_CROSS))
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemHasNonMappableLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	return psDevConfig->bHasNonMappableLocalMemory;
+}
+
+/*
+	PVRSRVSystemWaitCycles
+*/
+void PVRSRVSystemWaitCycles(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT32 ui32Cycles)
+{
+	/* Delay in us */
+	IMG_UINT32 ui32Delayus = 1;
+
+	/* obtain the device freq */
+	if (psDevConfig->pfnClockFreqGet != NULL)
+	{
+		IMG_UINT32 ui32DeviceFreq;
+
+		ui32DeviceFreq = psDevConfig->pfnClockFreqGet(psDevConfig->hSysData);
+
+		ui32Delayus = (ui32Cycles*1000000)/ui32DeviceFreq;
+
+		if (ui32Delayus == 0)
+		{
+			ui32Delayus = 1;
+		}
+	}
+
+	OSWaitus(ui32Delayus);
+}
+
+static void *
+PVRSRVSystemInstallDeviceLISR_Match_AnyVaCb(PVRSRV_DEVICE_NODE *psDeviceNode,
+											va_list va)
+{
+	void *pvOSDevice = va_arg(va, void *);
+
+	if (psDeviceNode->psDevConfig->pvOSDevice == pvOSDevice)
+	{
+		return psDeviceNode;
+	}
+
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVSystemInstallDeviceLISR(void *pvOSDevice,
+										   IMG_UINT32 ui32IRQ,
+										   const IMG_CHAR *pszName,
+										   PFN_LISR pfnLISR,
+										   void *pvData,
+										   IMG_HANDLE *phLISRData)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =
+		List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+									   &PVRSRVSystemInstallDeviceLISR_Match_AnyVaCb,
+									   pvOSDevice);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (!psDeviceNode)
+	{
+		/* Device can't be found in the list so it isn't in the system */
+		PVR_DPF((PVR_DBG_ERROR, "%s: device %p with irq %d is not present",
+				 __func__, pvOSDevice, ui32IRQ));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	return SysInstallDeviceLISR(psDeviceNode->psDevConfig->hSysData, ui32IRQ,
+								pszName, pfnLISR, pvData, phLISRData);
+}
+
+PVRSRV_ERROR PVRSRVSystemUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return SysUninstallDeviceLISR(hLISRData);
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION) && defined(EMULATOR)
+/* functions only used on rogue, but header defining them is common */
+void SetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState)
+{
+	SysSetAxiProtOSid(hSysData, ui32OSid, bState);
+}
+
+void SetTrustedDeviceAceEnabled(IMG_HANDLE hSysData)
+{
+	SysSetTrustedDeviceAceEnabled(hSysData);
+}
+#endif
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR PVRSRVCreateHWPerfHostThread(IMG_UINT32 ui32Timeout)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!ui32Timeout)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	OSLockAcquire(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+
+	/* Create only once */
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread == NULL)
+	{
+		/* Create the HWPerf event object */
+		eError = OSEventObjectCreate("PVRSRV_HWPERFHOSTPERIODIC_EVENTOBJECT", &gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectCreate");
+
+		if (eError == PVRSRV_OK)
+		{
+			gpsPVRSRVData->bHWPerfHostThreadStop = IMG_FALSE;
+			gpsPVRSRVData->ui32HWPerfHostThreadTimeout = ui32Timeout;
+			/* Create a thread which is used to periodically emit host stream packets */
+			eError = OSThreadCreate(&gpsPVRSRVData->hHWPerfHostPeriodicThread,
+				"pvr_hwperf_host",
+				HWPerfPeriodicHostEventsThread,
+				NULL, IMG_TRUE, gpsPVRSRVData);
+			PVR_LOG_IF_ERROR(eError, "OSThreadCreate");
+		}
+	}
+	/* If the thread has already been created then just update the timeout and wake up thread */
+	else
+	{
+		gpsPVRSRVData->ui32HWPerfHostThreadTimeout = ui32Timeout;
+		eError = OSEventObjectSignal(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+	}
+
+	OSLockRelease(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDestroyHWPerfHostThread(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	OSLockAcquire(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+
+	/* Stop and cleanup the HWPerf periodic thread */
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread)
+	{
+		if (gpsPVRSRVData->hHWPerfHostPeriodicEvObj)
+		{
+			gpsPVRSRVData->bHWPerfHostThreadStop = IMG_TRUE;
+			eError = OSEventObjectSignal(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			eError = OSThreadDestroy(gpsPVRSRVData->hHWPerfHostPeriodicThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hHWPerfHostPeriodicThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+
+		if (gpsPVRSRVData->hHWPerfHostPeriodicEvObj)
+		{
+			eError = OSEventObjectDestroy(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+			gpsPVRSRVData->hHWPerfHostPeriodicEvObj = NULL;
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+		}
+	}
+
+	OSLockRelease(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	return eError;
+}
+#endif
+
+/*
+ * Scan the list of known devices until we find the specific instance or
+ * exhaust the list
+ */
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstance(IMG_UINT32 uiInstance)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	if (uiInstance >= gpsPVRSRVData->ui32RegisteredDevices)
+	{
+		return NULL;
+	}
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (psDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     psDevNode != NULL; psDevNode = psDevNode->psNext)
+	{
+		if (uiInstance == psDevNode->sDevId.ui32InternalID)
+		{
+			break;
+		}
+	}
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	return psDevNode;
+}
+
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstanceByKernelDevID(IMG_INT32 i32OSInstance)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (psDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     psDevNode != NULL; psDevNode = psDevNode->psNext)
+	{
+		if (i32OSInstance == psDevNode->sDevId.i32KernelDeviceID)
+		{
+			MULTI_DEVICE_BRINGUP_DPF("%s: Found DevId %d. Retrieving node.", __func__, i32OSInstance);
+			break;
+		}
+		else
+		{
+			MULTI_DEVICE_BRINGUP_DPF("%s: Searching for DevId %d: Id %d not matching", __func__, i32OSInstance, psDevNode->sDevId.i32KernelDeviceID);
+		}
+	}
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	if (psDevNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: DevId %d not found.", __func__, i32OSInstance));
+	}
+	return psDevNode;
+}
+
+#define _FROZEN 1		/* Device is already frozen */
+#define _NOT_FROZEN 0	/* Device is not frozen */
+
+/* Freeze the specified device if not already frozen */
+PVRSRV_ERROR PVRSRVDeviceFreeze(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL     bHasPowerLock = IMG_FALSE;
+
+	/* Verify that given argument *IS* a recognised device node */
+	PVRSRV_DEVICE_NODE *lpsDevNode;
+
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (lpsDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     lpsDevNode != NULL && lpsDevNode != psDevNode;
+	     lpsDevNode = lpsDevNode->psNext)
+		;
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	PVR_LOG_RETURN_IF_FALSE((lpsDevNode == psDevNode), "Device node not known",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Mark the device as 'frozen' if not already marked */
+	if (OSAtomicCompareExchange(&psDevNode->eFrozen, _NOT_FROZEN,
+	                            _FROZEN) == _FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p already frozen", __func__,
+		        psDevNode));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure there are no other threads active within this device. */
+	while (OSAtomicRead(&psDevNode->iThreadsActive) > 0)
+	{
+		OSReleaseThreadQuanta();	/* Let other threads execute */
+	}
+
+	/* Attempt to idle the GPU without power-down. If this fails, we try with
+	 * a potential power-down.
+	 */
+	eError = PVRSRVPowerLock(psDevNode);
+	if (eError != PVRSRV_OK)
+	{
+		/* Device is powered down, we can continue
+		 * as there are no in-flight requests in the GPU.
+		 */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unable to Idle Device (%p) [%u/%d]"
+		        " Device powered down",
+		        __func__, psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		bHasPowerLock = IMG_TRUE;
+
+		eError = PVRSRVDeviceIdleRequestKM(psDevNode,
+		                                   &PVRSRVDeviceIsDefaultStateOFF,
+		                                   IMG_TRUE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Failed to Idle. Need to remove the pending _FROZEN state. */
+		(void) OSAtomicExchange(&psDevNode->eFrozen, _NOT_FROZEN);
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unable to Idle Device (%p) [%u/%d]",
+		        __func__, psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+
+		if (bHasPowerLock)
+		{
+			PVRSRVPowerUnlock(psDevNode);
+		}
+		return eError;
+	}
+
+	/* Now change the device-state to not be ACTIVE until we unfreeze */
+	if (psDevNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected Device State %u for device %p",
+		        __func__, psDevNode->eDevState, psDevNode));
+
+		/* Reset device to UNFROZEN */
+		(void) OSAtomicExchange(&psDevNode->eFrozen, _NOT_FROZEN);
+
+		if (bHasPowerLock)
+		{
+			(void) PVRSRVDeviceIdleCancelRequestKM(psDevNode);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+		else
+		{
+			(void) PVRSRVSetDeviceSystemPowerState(psDevNode,
+	                                               PVRSRV_SYS_POWER_STATE_ON,
+	                                               PVRSRV_POWER_FLAGS_NONE);
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_FROZEN);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Device (%p) [%u/%d] FROZEN", __func__,
+	        psDevNode,
+	        psDevNode->sDevId.ui32InternalID,
+	        psDevNode->sDevId.i32KernelDeviceID));
+
+	OSAtomicIncrement(&psDevNode->iTotalFreezes);
+
+	if (bHasPowerLock)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Unfreeze / Thaw the specified device if frozen */
+PVRSRV_ERROR PVRSRVDeviceThaw(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *lpsDevNode;
+	IMG_BOOL           bHasPowerLock = IMG_FALSE;
+
+	/* Verify that the given argument *IS* a recognised device node */
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (lpsDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     lpsDevNode != NULL && lpsDevNode != psDevNode;
+	     lpsDevNode = lpsDevNode->psNext)
+		;
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	PVR_LOG_RETURN_IF_FALSE((lpsDevNode == psDevNode), "Device node not known",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Unfreeze the device */
+	if (OSAtomicCompareExchange(&psDevNode->eFrozen, _FROZEN,
+	                            _NOT_FROZEN) == _NOT_FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device (%p) [%u/%d] already unfrozen",
+		        __func__,
+		        psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDevNode->eDevState != PVRSRV_DEVICE_STATE_FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Unexpected Device state %u for device (%p) [%u/%d]",
+		        __func__, psDevNode->eDevState, psDevNode,
+		        psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+	}
+	else
+	{
+		PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_ACTIVE);
+
+		PVR_DPF((PVR_DBG_VERBOSE, "%s: Device (%p) [%u/%d] UNFROZEN", __func__,
+		        psDevNode,
+		        psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+	}
+
+	/* Now unblock the device by clearing any ForcedIdle state and/or
+	 * powering-on the device if needed.
+	 */
+	eError = PVRSRVPowerLock(psDevNode);
+
+	bHasPowerLock = (eError == PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
+
+	if (bHasPowerLock)
+	{
+		eError = PVRSRVDeviceIdleCancelRequestKM(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Could not cancel Idle state: Device (%p) [%u/%d] '%s'",
+			        __func__,
+			        psDevNode,
+			        psDevNode->sDevId.ui32InternalID,
+			        psDevNode->sDevId.i32KernelDeviceID,
+			        PVRSRVGetErrorString(eError)));
+		}
+
+		PVRSRVPowerUnlock(psDevNode);
+	}
+	else
+	{
+		/* Force device back into POWER_STATE_ON as it must already be OFF. */
+		(void) PVRSRVSetDeviceSystemPowerState(psDevNode,
+	                                           PVRSRV_SYS_POWER_STATE_ON,
+	                                           PVRSRV_POWER_FLAGS_NONE);
+	}
+
+	/* Unblock any waiting threads */
+	while (OSAtomicRead(&psDevNode->iFreezeCount) > 0)
+	{
+		eError = OSEventObjectSignal(psDevNode->hDeviceThreadEvObj);
+		if (OSAtomicRead(&psDevNode->iFreezeCount) > 0)
+		{
+			/* Sleep for 1ms to allow waiter to receive signal */
+			OSSleepms(1U);
+		}
+	}
+
+	/* Ensure that any blocked queues get rescheduled if we've woken up a
+	 * waiter.
+	 */
+	PVRSRVCheckStatus(NULL);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_DRIVER_MODE PVRSRVGetVzModeByDevNum(IMG_UINT32 ui32DevNum)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DRIVER_MODE eRetMode = DRIVER_MODE_NATIVE;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+
+	/* Iterate over all devices. */
+	for (psDevNode = psPVRSRVData->psDeviceNodeList;
+		 psDevNode != NULL;
+		 psDevNode = psDevNode->psNext)
+	{
+		if (psDevNode->sDevId.ui32InternalID == ui32DevNum)
+		{
+			eRetMode = psDevNode->psDevConfig->eDriverMode;
+			break;
+		}
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	return eRetMode;
+}
+
+/*****************************************************************************
+ End of file (pvrsrv.c)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_bridge_init.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_bridge_init.c
new file mode 100644
index 000000000000..13beb3f5c454
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_bridge_init.c
@@ -0,0 +1,379 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_bridge_init.h"
+#include "srvcore.h"
+
+/* These will go when full bridge gen comes in */
+#if defined(PDUMP)
+PVRSRV_ERROR InitPDUMPCTRLBridge(void);
+void DeinitPDUMPCTRLBridge(void);
+PVRSRV_ERROR InitPDUMPBridge(void);
+void DeinitPDUMPBridge(void);
+PVRSRV_ERROR InitRGXPDUMPBridge(void);
+void DeinitRGXPDUMPBridge(void);
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+PVRSRV_ERROR InitDCBridge(void);
+void DeinitDCBridge(void);
+#endif
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+#endif
+PVRSRV_ERROR InitPDUMPMMBridge(void);
+void DeinitPDUMPMMBridge(void);
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+#if defined(SUPPORT_DMA_TRANSFER)
+PVRSRV_ERROR InitDMABridge(void);
+void DeinitDMABridge(void);
+#endif
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+PVRSRV_ERROR InitRGXTQBridge(void);
+void DeinitRGXTQBridge(void);
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void);
+void DeinitRGXBREAKPOINTBridge(void);
+#endif
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+#endif
+#endif /* SUPPORT_RGX */
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+#if defined(SUPPORT_SECURE_EXPORT)
+PVRSRV_ERROR InitSMMBridge(void);
+void DeinitSMMBridge(void);
+#endif
+#if defined(PVRSRV_ENABLE_HTB)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+#endif
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+#endif
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+PVRSRV_ERROR InitVALIDATIONBridge(void);
+void DeinitVALIDATIONBridge(void);
+#endif
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+#if defined(SUPPORT_WRAP_EXTMEM)
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+#endif
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void);
+void DeinitSYNCFALLBACKBridge(void);
+#endif
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+#if defined(SUPPORT_DI_BRG_IMPL)
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+#endif
+
+PVRSRV_ERROR
+ServerBridgeInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	BridgeDispatchTableStartOffsetsInit();
+
+	eError = InitSRVCOREBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSRVCOREBridge");
+
+	eError = InitSYNCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCBridge");
+
+#if defined(PDUMP)
+	eError = InitPDUMPCTRLBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPCTRLBridge");
+#endif
+
+	eError = InitMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitMMBridge");
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+	eError = InitCMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitCMMBridge");
+#endif
+
+#if defined(PDUMP)
+	eError = InitPDUMPMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPMMBridge");
+
+	eError = InitPDUMPBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPBridge");
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = InitDCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDCBridge");
+#endif
+
+	eError = InitCACHEBridge();
+	PVR_LOG_IF_ERROR(eError, "InitCACHEBridge");
+
+#if defined(SUPPORT_SECURE_EXPORT)
+	eError = InitSMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSMMBridge");
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+	eError = InitHTBUFFERBridge();
+	PVR_LOG_IF_ERROR(eError, "InitHTBUFFERBridge");
+#endif
+
+	eError = InitPVRTLBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPVRTLBridge");
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = InitRIBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRIBridge");
+#endif
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+	eError = InitVALIDATIONBridge();
+	PVR_LOG_IF_ERROR(eError, "InitVALIDATIONBridge");
+#endif
+
+
+	eError = InitDEVICEMEMHISTORYBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDEVICEMEMHISTORYBridge");
+
+	eError = InitSYNCTRACKINGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCTRACKINGBridge");
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	eError = InitDMABridge();
+	PVR_LOG_IF_ERROR(eError, "InitDMABridge");
+#endif
+
+#if defined(SUPPORT_RGX)
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+	eError = InitRGXTQBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTQBridge");
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+	eError = InitRGXTA3DBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTA3DBridge");
+
+	#if defined(SUPPORT_USC_BREAKPOINT)
+	eError = InitRGXBREAKPOINTBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXBREAKPOINTBridge");
+#endif
+
+	eError = InitRGXFWDBGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXFWDBGBridge");
+
+#if defined(PDUMP)
+	eError = InitRGXPDUMPBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXPDUMPBridge");
+#endif
+
+	eError = InitRGXHWPERFBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXHWPERFBridge");
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+	eError = InitRGXREGCONFIGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXREGCONFIGBridge");
+#endif
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	eError = InitRGXKICKSYNCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXKICKSYNCBridge");
+#endif
+
+	eError = InitRGXTIMERQUERYBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTIMERQUERYBridge");
+
+#endif /* SUPPORT_RGX */
+
+#if defined(SUPPORT_WRAP_EXTMEM)
+	eError = InitMMEXTMEMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitMMEXTMEMBridge");
+#endif
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	eError = InitSYNCFALLBACKBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCFALLBACKBridge");
+#endif
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	eError = InitDIBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDIBridge");
+#endif
+
+	eError = OSPlatformBridgeInit();
+	PVR_LOG_IF_ERROR(eError, "OSPlatformBridgeInit");
+
+	return eError;
+}
+
+void ServerBridgeDeInit(void)
+{
+	OSPlatformBridgeDeInit();
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	DeinitDIBridge();
+#endif
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	DeinitSYNCFALLBACKBridge();
+#endif
+
+#if defined(SUPPORT_WRAP_EXTMEM)
+	DeinitMMEXTMEMBridge();
+#endif
+
+	DeinitSRVCOREBridge();
+
+	DeinitSYNCBridge();
+
+#if defined(PDUMP)
+	DeinitPDUMPCTRLBridge();
+#endif
+
+	DeinitMMBridge();
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+	DeinitCMMBridge();
+#endif
+
+#if defined(PDUMP)
+	DeinitPDUMPMMBridge();
+
+	DeinitPDUMPBridge();
+#endif
+
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	DeinitDCBridge();
+#endif
+
+	DeinitCACHEBridge();
+
+#if defined(SUPPORT_SECURE_EXPORT)
+	DeinitSMMBridge();
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+	DeinitHTBUFFERBridge();
+#endif
+
+	DeinitPVRTLBridge();
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+	DeinitVALIDATIONBridge();
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	DeinitRIBridge();
+#endif
+
+	DeinitDEVICEMEMHISTORYBridge();
+
+	DeinitSYNCTRACKINGBridge();
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	DeinitDMABridge();
+#endif
+
+#if defined(SUPPORT_RGX)
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+	DeinitRGXTQBridge();
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+	DeinitRGXTA3DBridge();
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+	DeinitRGXBREAKPOINTBridge();
+#endif
+
+	DeinitRGXFWDBGBridge();
+
+#if defined(PDUMP)
+	DeinitRGXPDUMPBridge();
+#endif
+
+	DeinitRGXHWPERFBridge();
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+	DeinitRGXREGCONFIGBridge();
+#endif
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DeinitRGXKICKSYNCBridge();
+#endif
+
+	DeinitRGXTIMERQUERYBridge();
+#endif /* SUPPORT_RGX */
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_pool.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_pool.c
new file mode 100644
index 000000000000..9fa95ea98756
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/pvrsrv_pool.c
@@ -0,0 +1,260 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services pool implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a generic pool implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "lock.h"
+#include "dllist.h"
+#include "allocmem.h"
+
+struct _PVRSRV_POOL_
+{
+	POS_LOCK hLock;
+	/* total max number of permitted entries in the pool */
+	IMG_UINT uiMaxEntries;
+	/* currently number of pool entries created. these may be in the pool
+	 * or in-use
+	 */
+	IMG_UINT uiNumBusy;
+	/* number of not-in-use entries currently free in the pool */
+	IMG_UINT uiNumFree;
+
+	DLLIST_NODE sFreeList;
+
+	const IMG_CHAR *pszName;
+
+	PVRSRV_POOL_ALLOC_FUNC *pfnAlloc;
+	PVRSRV_POOL_FREE_FUNC *pfnFree;
+	void *pvPrivData;
+};
+
+typedef struct _PVRSRV_POOL_ENTRY_
+{
+	DLLIST_NODE sNode;
+	void *pvData;
+} PVRSRV_POOL_ENTRY;
+
+PVRSRV_ERROR PVRSRVPoolCreate(PVRSRV_POOL_ALLOC_FUNC *pfnAlloc,
+					PVRSRV_POOL_FREE_FUNC *pfnFree,
+					IMG_UINT32 ui32MaxEntries,
+					const IMG_CHAR *pszName,
+					void *pvPrivData,
+					PVRSRV_POOL **ppsPool)
+{
+	PVRSRV_POOL *psPool;
+	PVRSRV_ERROR eError;
+
+	psPool = OSAllocMem(sizeof(PVRSRV_POOL));
+	PVR_GOTO_IF_NOMEM(psPool, eError, err_alloc);
+
+	eError = OSLockCreate(&psPool->hLock);
+
+	PVR_GOTO_IF_ERROR(eError, err_lock_create);
+
+	psPool->uiMaxEntries = ui32MaxEntries;
+	psPool->uiNumBusy = 0;
+	psPool->uiNumFree = 0;
+	psPool->pfnAlloc = pfnAlloc;
+	psPool->pfnFree = pfnFree;
+	psPool->pvPrivData = pvPrivData;
+	psPool->pszName = pszName;
+
+	dllist_init(&psPool->sFreeList);
+
+	*ppsPool = psPool;
+
+	return PVRSRV_OK;
+
+err_lock_create:
+	OSFreeMem(psPool);
+err_alloc:
+	return eError;
+}
+
+static PVRSRV_ERROR _DestroyPoolEntry(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_ENTRY *psEntry)
+{
+	psPool->pfnFree(psPool->pvPrivData, psEntry->pvData);
+	OSFreeMem(psEntry);
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVPoolDestroy(PVRSRV_POOL *psPool)
+{
+	if (psPool->uiNumBusy != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Attempt to destroy pool %s "
+						"with %u entries still in use",
+						__func__,
+						psPool->pszName,
+						psPool->uiNumBusy));
+		return;
+	}
+
+	OSLockDestroy(psPool->hLock);
+
+	if (psPool->uiNumFree)
+	{
+		PVRSRV_POOL_ENTRY *psEntry;
+		DLLIST_NODE *psChosenNode;
+
+		psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+
+		while (psChosenNode)
+		{
+			dllist_remove_node(psChosenNode);
+
+			psEntry = IMG_CONTAINER_OF(psChosenNode, PVRSRV_POOL_ENTRY, sNode);
+			_DestroyPoolEntry(psPool, psEntry);
+
+			psPool->uiNumFree--;
+
+			psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+		}
+
+		PVR_ASSERT(psPool->uiNumFree == 0);
+	}
+
+	OSFreeMem(psPool);
+}
+
+static PVRSRV_ERROR _CreateNewPoolEntry(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_ENTRY **ppsEntry)
+{
+	PVRSRV_POOL_ENTRY *psNewEntry;
+	PVRSRV_ERROR eError;
+
+	psNewEntry = OSAllocMem(sizeof(PVRSRV_POOL_ENTRY));
+	PVR_GOTO_IF_NOMEM(psNewEntry, eError, err_allocmem);
+
+	dllist_init(&psNewEntry->sNode);
+
+	eError = psPool->pfnAlloc(psPool->pvPrivData, &psNewEntry->pvData);
+
+	PVR_GOTO_IF_ERROR(eError, err_pfn_alloc);
+
+	*ppsEntry = psNewEntry;
+
+	return PVRSRV_OK;
+
+err_pfn_alloc:
+	OSFreeMem(psNewEntry);
+err_allocmem:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVPoolGet(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_TOKEN *hToken,
+					void **ppvDataOut)
+{
+	PVRSRV_POOL_ENTRY *psEntry;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psChosenNode;
+
+	OSLockAcquire(psPool->hLock);
+
+	psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+	if (unlikely(psChosenNode == NULL))
+	{
+		/* no available elements in the pool. try to create one */
+
+		eError = _CreateNewPoolEntry(psPool, &psEntry);
+
+		PVR_GOTO_IF_ERROR(eError, out_unlock);
+	}
+	else
+	{
+		dllist_remove_node(psChosenNode);
+
+		psEntry = IMG_CONTAINER_OF(psChosenNode, PVRSRV_POOL_ENTRY, sNode);
+
+		psPool->uiNumFree--;
+	}
+
+#if defined(DEBUG)
+	/* Don't poison the IN buffer as that is copied from client and would be
+	 * waste of cycles.
+	 */
+	OSCachedMemSet(((IMG_PBYTE)psEntry->pvData)+PVRSRV_MAX_BRIDGE_IN_SIZE,
+			PVRSRV_POISON_ON_ALLOC_VALUE, PVRSRV_MAX_BRIDGE_OUT_SIZE);
+#endif
+
+	psPool->uiNumBusy++;
+	*hToken = psEntry;
+	*ppvDataOut = psEntry->pvData;
+
+out_unlock:
+	OSLockRelease(psPool->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVPoolPut(PVRSRV_POOL *psPool, PVRSRV_POOL_TOKEN hToken)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_POOL_ENTRY *psEntry = hToken;
+
+	PVR_ASSERT(psPool->uiNumBusy > 0);
+
+	OSLockAcquire(psPool->hLock);
+
+	/* put this entry in the pool if the pool has space,
+	 * otherwise free it
+	 */
+	if (psPool->uiNumFree < psPool->uiMaxEntries)
+	{
+		dllist_add_to_tail(&psPool->sFreeList, &psEntry->sNode);
+		psPool->uiNumFree++;
+	}
+	else
+	{
+		eError = _DestroyPoolEntry(psPool, psEntry);
+	}
+
+	psPool->uiNumBusy--;
+
+	OSLockRelease(psPool->hLock);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/ri_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/ri_server.c
new file mode 100644
index 000000000000..370e41dd4553
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/ri_server.c
@@ -0,0 +1,2638 @@
+/*************************************************************************/ /*!
+@File           ri_server.c
+@Title          Resource Information (RI) server implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) server functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#include "img_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+
+#include "srvkm.h"
+#include "lock.h"
+
+/* services/include */
+#include "pvr_ricommon.h"
+
+/* services/server/include/ */
+#include "ri_server.h"
+
+/* services/include/shared/ */
+#include "hash.h"
+/* services/shared/include/ */
+#include "dllist.h"
+
+#include "pmr.h"
+#include "physheap.h"
+
+/* include/device.h */
+#include "device.h"
+
+#if !defined(RI_UNIT_TEST)
+#include "pvrsrv.h"
+#endif
+
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+#define USE_RI_LOCK		1
+
+/*
+ * Initial size use for Hash table. (Used to index the RI list entries).
+ */
+#define _RI_INITIAL_HASH_TABLE_SIZE	64
+
+/*
+ * If this define is set to 1, details of the linked lists (addresses,
+ * prev/next ptrs, etc) are also output when function RIDumpList() is called.
+ */
+#define _DUMP_LINKEDLIST_INFO		0
+
+
+typedef IMG_UINT64 _RI_BASE_T;
+
+
+/* No +1 in SIZE macros since sizeof includes \0 byte in size */
+
+#define RI_PROC_BUF_SIZE    16
+#define RI_ANNO_BUF_SIZE    80
+#define RI_ANNO_FRMT_SIZE (sizeof(RI_ANNO_FRMT))
+
+#define RI_DEV_ID_BUF_SIZE  4
+
+#define RI_MEMDESC_SUM_FRMT     "PID:%d %s MEMDESCs Alloc'd:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K) + "\
+                                                  "Imported:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K) = "\
+                                                     "Total:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K)\n"
+#define RI_MEMDESC_SUM_BUF_SIZE (sizeof(RI_MEMDESC_SUM_FRMT)+5+RI_PROC_BUF_SIZE+30+60)
+
+
+#define RI_PMR_SUM_FRMT     "PID:%d %s PMRs Alloc'd:0x%010" IMG_UINT64_FMTSPECx ", %" IMG_UINT64_FMTSPEC "K  "\
+                                        "[Physical: 0x%010" IMG_UINT64_FMTSPECx ", %" IMG_UINT64_FMTSPEC "K]\n"
+#define RI_PMR_SUM_BUF_SIZE (sizeof(RI_PMR_SUM_FRMT)+(20+40))
+
+#define RI_FREED_BY_DRIVER "{Freed by KM}"
+
+#define RI_PMR_ENTRY_IMPORTED_FRMT_BEGIN "{Imported to DEV:"
+#define RI_PMR_ENTRY_IMPORTED_FRMT_END   "\b}"
+#define RI_PMR_ENTRY_IMPORTED_DEV_FRMT "%2u,"
+#define RI_PMR_ENTRY_IMPORTED_DEV_SIZE 3
+#define RI_PMR_ENTRY_IMPORTED_BUF_SIZE (sizeof(RI_PMR_ENTRY_IMPORTED_FRMT_BEGIN)+sizeof(RI_PMR_ENTRY_IMPORTED_FRMT_END) + PVRSRV_MAX_DEVICES * RI_PMR_ENTRY_IMPORTED_DEV_SIZE)
+
+#define RI_PMR_ENTRY_FRMT      "%%sPID:%%-5d DEV:%%s <%%p>\t%%-%ds\t%%-%ds\t0x%%010" IMG_UINT64_FMTSPECx "\t[0x%%010" IMG_UINT64_FMTSPECx "]\t%%s%%s%%c"
+#define RI_PMR_ENTRY_BUF_SIZE  (sizeof(RI_PMR_ENTRY_FRMT)+(3+5+RI_DEV_ID_BUF_SIZE+16+(PVR_ANNOTATION_MAX_LEN/2)+PHYS_HEAP_NAME_SIZE+10+10)+ sizeof(RI_FREED_BY_DRIVER))
+#define RI_PMR_ENTRY_FRMT_SIZE (sizeof(RI_PMR_ENTRY_FRMT))
+
+/* Use %5d rather than %d so the output aligns in server/kernel.log, debugFS sees extra spaces */
+#define RI_MEMDESC_ENTRY_PROC_FRMT        "[%5d:%s]"
+#define RI_MEMDESC_ENTRY_PROC_BUF_SIZE    (sizeof(RI_MEMDESC_ENTRY_PROC_FRMT)+5+16)
+
+#define RI_SYS_ALLOC_IMPORT_FRMT      "{Import from PID %d}"
+#define RI_SYS_ALLOC_IMPORT_FRMT_SIZE (sizeof(RI_SYS_ALLOC_IMPORT_FRMT)+5)
+static IMG_CHAR g_szSysAllocImport[RI_SYS_ALLOC_IMPORT_FRMT_SIZE];
+
+#define RI_MEMDESC_ENTRY_IMPORT_FRMT     "{Import from PID %d}"
+#define RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE (sizeof(RI_MEMDESC_ENTRY_IMPORT_FRMT)+5)
+
+#define RI_MEMDESC_ENTRY_FRMT      "%%sPID:%%-5d DEV:%%s 0x%%010" IMG_UINT64_FMTSPECx "\t%%-%ds %%s\t0x%%010" IMG_UINT64_FMTSPECx "\t<%%p> %%s%%s %%s%%c"
+#define RI_MEMDESC_ENTRY_BUF_SIZE  (sizeof(RI_MEMDESC_ENTRY_FRMT)+(3+5+RI_DEV_ID_BUF_SIZE+10+PVR_ANNOTATION_MAX_LEN+RI_MEMDESC_ENTRY_PROC_BUF_SIZE+16+\
+                                               RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE+RI_SYS_ALLOC_IMPORT_FRMT_SIZE)) + sizeof(RI_FREED_BY_DRIVER)
+#define RI_MEMDESC_ENTRY_FRMT_SIZE (sizeof(RI_MEMDESC_ENTRY_FRMT))
+
+
+#define RI_FRMT_SIZE_MAX (MAX(RI_MEMDESC_ENTRY_BUF_SIZE,\
+                              MAX(RI_PMR_ENTRY_BUF_SIZE,\
+                                  MAX(RI_MEMDESC_SUM_BUF_SIZE,\
+                                      RI_PMR_SUM_BUF_SIZE))))
+
+typedef struct _RI_PMR_INFO_
+{
+	uintptr_t uiAddr;
+	PHYS_HEAP *psHeap;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	size_t uiLogicalSize;
+	size_t uiPhysicalSize;
+	IMG_CHAR *pszAnnotation;
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	IMG_UINT64 uiXDevices;
+#endif
+} RI_PMR_INFO;
+
+
+#define RI_DEVID_MASK						0xFF /* Highest value is PVRSRV_HOST_DEVICE_ID[255] */
+
+/* Flags for Memdescs(RI_SUBLIST) */
+#define RI_IMPORT_FLAG						(8)  /* Indicates the PMR is an import */
+#define RI_SUBALLOC_FLAG					(9)  /* Indicates the PMR is sub-allocatable */
+
+/* Flags for PMRs(RI_LIST) */
+#define RI_SYSALLOC_PMR_FLAG				(10) /* Indicates the PMR belongs to the 'system' process */
+#define RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG	(11) /* Indicates size counted prior, to prevent double reads */
+
+/* Shared flags */
+#define RI_RACC_FLAG						(12) /* Freed by Driver after disconnect */
+#define RI_VALID_FLAG						(13) /* Valid on creation, Only invalid after delete */
+#define RI_HAS_PMR_INFO						(14) /* Entry has RI_PMR_INFO object rather than reference to a PMR */
+
+#define GET_DEVICE_ID(entry)			((entry)->ui16Flags & RI_DEVID_MASK)
+#define SET_DEVICE_ID(entry,id)			BITMASK_SET((entry)->ui16Flags, (id & RI_DEVID_MASK))
+#define IS_HOST_DEVICE(entry)			(((entry)->ui16Flags & RI_DEVID_MASK) == PVRSRV_HOST_DEVICE_ID)
+
+#define IS_IMPORT(entry)				BIT_ISSET((entry)->ui16Flags, RI_IMPORT_FLAG)
+#define IS_SUBALLOC(entry)				BIT_ISSET((entry)->ui16Flags, RI_SUBALLOC_FLAG)
+
+#define IS_SYSPMR(entry)				BIT_ISSET((entry)->ui16Flags, RI_SYSALLOC_PMR_FLAG)
+#define IS_COUNTED_BY_DEBUGFS(entry)	BIT_ISSET((entry)->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG)
+
+#define IS_RACC(entry)					BIT_ISSET((entry)->ui16Flags, RI_RACC_FLAG)
+#define IS_VALID(entry)					BIT_ISSET((entry)->ui16Flags, RI_VALID_FLAG)
+#define HAS_PMR_INFO(entry)				BIT_ISSET((entry)->ui16Flags, RI_HAS_PMR_INFO)
+
+/* Structure used to make linked sublist of memory allocations (MEMDESC) */
+struct _RI_SUBLIST_ENTRY_
+{
+	DLLIST_NODE				sListNode;
+	DLLIST_NODE				sProcListNode; /* Sublist entries that share the same PID.
+	                                          IMPORTANT: This list does not contain a head node! */
+	CONNECTION_DATA			*psConnection;
+	struct _RI_LIST_ENTRY_	*psRI;
+	IMG_CHAR				*pszTextB; /* Annotation+(NUL)+ProcName+(NUL) */
+	IMG_DEV_VIRTADDR		sVAddr;
+	IMG_UINT64				ui64Offset;
+	IMG_UINT64				ui64Size;
+	IMG_PID					pid;
+	IMG_UINT16				ui16TextBLength; /* Total length of Annotation+(NUL)+ProcName+(NUL) */
+	IMG_UINT16				ui16Flags; /* Refer to above */
+};
+
+/*
+ * Structure used to make linked list of PMRs. Sublists of allocations
+ * (MEMDESCs) made from these PMRs are chained off these entries.
+ */
+struct _RI_LIST_ENTRY_
+{
+	DLLIST_NODE				sListNode;
+	DLLIST_NODE				sSubListFirst;
+	union {
+		PMR *psPMR;
+		RI_PMR_INFO *psPmrInfo;
+	} pmr_info;
+	IMG_PID					pid;
+	IMG_UINT16				ui16SubListCount;
+	IMG_UINT16				ui16Flags; /* Refer to above */
+};
+
+#define GET_KEY(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (void*)(entry)->pmr_info.psPmrInfo \
+			: (void*)(entry)->pmr_info.psPMR \
+	)
+
+#define GET_ADDR(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (void*)(entry)->pmr_info.psPmrInfo->uiAddr \
+			: (void*)(entry)->pmr_info.psPMR \
+	)
+#define GET_HEAP(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->psHeap \
+			: PMR_PhysHeap((entry)->pmr_info.psPMR) \
+	)
+#define GET_DEVNODE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->psDeviceNode \
+			: (PVRSRV_DEVICE_NODE *) PMR_DeviceNode((entry)->pmr_info.psPMR) \
+	)
+#define GET_LOGICAL_SIZE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiLogicalSize \
+			: PMR_LogicalSize((entry)->pmr_info.psPMR) \
+	)
+#define GET_PHYSICAL_SIZE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiPhysicalSize \
+			: PMR_PhysicalSize((entry)->pmr_info.psPMR) \
+	)
+#define GET_NAME(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->pszAnnotation \
+			: PMR_GetAnnotation((entry)->pmr_info.psPMR) \
+	)
+#define GET_XDEVICES(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiXDevices \
+			: PMR_ImportedDevicesMask((entry)->pmr_info.psPMR) \
+	)
+
+/* pszTextB = <Memdesc Annotation>,0,<Proc Name>,0 */
+/* Retrieve the string pointer to the ProcName */
+#define GET_PROC(entry) ((IMG_CHAR*) (strchr((entry)->pszTextB, '\0') + 1))
+
+typedef struct _RI_LIST_ENTRY_ RI_LIST_ENTRY;
+typedef struct _RI_SUBLIST_ENTRY_ RI_SUBLIST_ENTRY;
+
+static HASH_TABLE	*g_pPMR2RIListHashTable;
+static HASH_TABLE	*g_PID2RISublistHashTable;
+
+static POS_LOCK		g_hRILock;
+
+/* Linked list of PMR allocations made against the PVR_SYS_ALLOC_PID and lock
+ * to prevent concurrent access to it.
+ */
+static POS_LOCK		g_hSysAllocPidListLock;
+static DLLIST_NODE	g_sSysAllocPidListHead;
+
+/*
+ * Flag used to indicate if RILock should be destroyed when final PMR entry is
+ * deleted, i.e. if RIDeInitKM() has already been called before that point but
+ * the handle manager has deferred deletion of RI entries.
+ */
+static IMG_BOOL		bRIDeInitDeferred = IMG_FALSE;
+
+/*
+ * Used as head of linked-list of PMR RI entries - this is useful when we wish
+ * to iterate all PMR list entries (when we don't have a PMR ref)
+ */
+static DLLIST_NODE	g_sClientsListHead;
+
+/* Function used to produce string containing info for MEMDESC RI entries (used for both debugfs and kernel log output) */
+static void _GenerateMEMDESCEntryString(RI_SUBLIST_ENTRY *psRISubEntry, IMG_BOOL bDebugFs, IMG_UINT16 ui16MaxStrLen, IMG_CHAR *pszEntryString);
+/* Function used to produce string containing info for PMR RI entries (used for both debugfs and kernel log output) */
+static void _GeneratePMREntryString(RI_LIST_ENTRY *psRIEntry, IMG_BOOL bDebugFs, IMG_UINT16 ui16MaxStrLen, IMG_CHAR *pszEntryString);
+
+static PVRSRV_ERROR _DumpAllEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DeleteAllEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DeleteAllProcEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DumpList(PMR *psPMR, IMG_PID pid);
+#define _RIOutput(x) PVR_LOG(x)
+
+static IMG_UINT32
+_ProcHashFunc(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+static IMG_UINT32
+_ProcHashFunc(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_UINT32 *p = (IMG_UINT32 *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = *p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+static IMG_BOOL
+_ProcHashComp(size_t uKeySize, void *pKey1, void *pKey2);
+
+static IMG_BOOL
+_ProcHashComp(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_UINT32 *p1 = (IMG_UINT32 *)pKey1;
+	IMG_UINT32 *p2 = (IMG_UINT32 *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static void _RILock(void)
+{
+#if (USE_RI_LOCK == 1)
+	OSLockAcquire(g_hRILock);
+#endif
+}
+
+static void _RIUnlock(void)
+{
+#if (USE_RI_LOCK == 1)
+	OSLockRelease(g_hRILock);
+#endif
+}
+
+/* This value maintains a count of the number of PMRs attributed to the
+ * PVR_SYS_ALLOC_PID. Access to this value is protected by g_hRILock, so it
+ * does not need to be an ATOMIC_T.
+ */
+static IMG_UINT32 g_ui32SysAllocPMRCount;
+
+static IMG_BOOL RICheckListHandle(RI_HANDLE hRIHandle)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *) hRIHandle, *psTableEntry;
+	void *pPMRHashKey;
+	IMG_BOOL bValid = IMG_FALSE;
+
+	if ((GET_DEVICE_ID(psRIEntry) > PVRSRV_MAX_DEVICES &&
+	     !IS_HOST_DEVICE(psRIEntry)) ||
+	    !g_pPMR2RIListHashTable)
+	{
+		//Shortcut check
+		return IMG_FALSE;
+	}
+
+	pPMRHashKey = GET_KEY(psRIEntry);
+
+	_RILock();
+
+	/* Look-up psPMR in Hash Table */
+	psTableEntry = (RI_LIST_ENTRY *) HASH_Retrieve_Extended(g_pPMR2RIListHashTable,
+	                                                        &pPMRHashKey);
+	if (psTableEntry != NULL)
+	{
+		bValid = IS_VALID(psTableEntry);
+	}
+
+	_RIUnlock();
+
+	return bValid;
+}
+
+static IMG_BOOL RICheckSubListHandle(RI_HANDLE hRIHandle)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry, *psTableEntry;
+	uintptr_t hashData;
+	IMG_PID pid;
+	IMG_BOOL bValid = IMG_FALSE;
+
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if ((GET_DEVICE_ID(psRISubEntry) > PVRSRV_MAX_DEVICES) || !g_PID2RISublistHashTable)
+	{
+		//Shortcut check
+		return bValid;
+	}
+	pid = psRISubEntry->pid;
+
+	_RILock();
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (hashData)
+	{
+		psTableEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+		bValid = IS_VALID(psTableEntry);
+	}
+
+	/* Release RI Lock */
+	_RIUnlock();
+	return bValid;
+}
+
+PVRSRV_ERROR RIInitKM(void)
+{
+	IMG_INT iCharsWritten;
+	PVRSRV_ERROR eError;
+
+	bRIDeInitDeferred = IMG_FALSE;
+
+	iCharsWritten = OSSNPrintf(g_szSysAllocImport,
+	            RI_SYS_ALLOC_IMPORT_FRMT_SIZE,
+	            RI_SYS_ALLOC_IMPORT_FRMT,
+	            PVR_SYS_ALLOC_PID);
+	PVR_LOG_IF_FALSE((iCharsWritten>0 && iCharsWritten<(IMG_INT32)RI_SYS_ALLOC_IMPORT_FRMT_SIZE),
+			"OSSNPrintf failed to initialise g_szSysAllocImport");
+
+	eError = OSLockCreate(&g_hSysAllocPidListLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: OSLockCreate (g_hSysAllocPidListLock) failed (returned %d)",
+		         __func__,
+		         eError));
+	}
+	dllist_init(&(g_sSysAllocPidListHead));
+#if (USE_RI_LOCK == 1)
+	eError = OSLockCreate(&g_hRILock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: OSLockCreate (g_hRILock) failed (returned %d)",
+		         __func__,
+		         eError));
+	}
+#endif
+	return eError;
+}
+
+void RIDeInitKM(void)
+{
+#if (USE_RI_LOCK == 1)
+	if (g_pPMR2RIListHashTable != NULL && HASH_Count(g_pPMR2RIListHashTable) > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: called with %d entries remaining - deferring OSLockDestroy()",
+		         __func__,
+		         HASH_Count(g_pPMR2RIListHashTable)));
+		bRIDeInitDeferred = IMG_TRUE;
+	}
+	else
+	{
+		OSLockDestroy(g_hRILock);
+		OSLockDestroy(g_hSysAllocPidListLock);
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RILockAcquireKM
+
+ @Description
+            Acquires the RI Lock (which protects the integrity of the RI
+            linked lists). Caller will be suspended until lock is acquired.
+
+ @Return	None
+
+******************************************************************************/
+void RILockAcquireKM(void)
+{
+	_RILock();
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RILockReleaseKM
+
+ @Description
+            Releases the RI Lock (which protects the integrity of the RI
+            linked lists).
+
+ @Return	None
+
+******************************************************************************/
+void RILockReleaseKM(void)
+{
+	_RIUnlock();
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWritePMREntryWithOwnerKM
+
+ @Description
+            Writes a new Resource Information list entry.
+            The new entry will be inserted at the head of the list of
+            PMR RI entries and assigned the values provided.
+
+ @input     psPMR - Reference (handle) to the PMR to which this reference relates
+
+ @input     ui32Owner - PID of the process which owns the allocation. This
+                        may not be the current process (e.g. a request to
+                        grow a buffer may happen in the context of a kernel
+                        thread, or we may import further resource for a
+                        suballocation made from the FW heap which can then
+                        also be utilized by other processes)
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWritePMREntryWithOwnerKM(PMR *psPMR,
+                                        IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR *pPMRHashKey = psPMR;
+	RI_LIST_ENTRY *psRIEntry;
+	uintptr_t hashData;
+
+	/* if Hash table has not been created, create it now */
+	if (!g_pPMR2RIListHashTable)
+	{
+		g_pPMR2RIListHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(PMR*), HASH_Func_Default, HASH_Key_Comp_Default);
+		g_PID2RISublistHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(IMG_PID), _ProcHashFunc, _ProcHashComp);
+	}
+	PVR_LOG_RETURN_IF_NOMEM(g_pPMR2RIListHashTable, "HASH_Create_Extended");
+	PVR_LOG_RETURN_IF_NOMEM(g_PID2RISublistHashTable, "HASH_Create_Extended");
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	psRIEntry = (RI_LIST_ENTRY *)hashData;
+	if (psRIEntry == NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)PMR_DeviceNode(psPMR);
+
+		/*
+		 * If failed to find a matching existing entry, create a new one
+		 */
+		psRIEntry = (RI_LIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_LIST_ENTRY));
+		PVR_LOG_GOTO_IF_NOMEM(psRIEntry, eError, exit_);
+		{
+			PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+			IMG_UINT32 ui32RICount = HASH_Count(g_pPMR2RIListHashTable);
+
+			/*
+			 * Add new RI Entry
+			 */
+			if (ui32RICount == 0)
+			{
+				/* Initialise PMR entry linked-list head */
+				dllist_init(&g_sClientsListHead);
+			}
+			else if (ui32RICount == IMG_UINT32_MAX)
+			{
+				PVR_LOG_GOTO_WITH_ERROR("ui32RICount", eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, exit_);
+			}
+
+			dllist_init (&(psRIEntry->sListNode));
+			dllist_init (&(psRIEntry->sSubListFirst));
+			BIT_SET(psRIEntry->ui16Flags, RI_VALID_FLAG);
+
+			/* Check if this PMR should be accounted for under the
+			 * PVR_SYS_ALLOC_PID debugFS entry. This should happen if
+			 * we are in the driver init phase, the flags indicate
+			 * this is a FW Main allocation (made from FW heap)
+			 * or the owner PID is PVR_SYS_ALLOC_PID.
+			 * Also record host dev node allocs on the system PID.
+			 */
+			if (psDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+				PVRSRV_CHECK_FW_MAIN(uiPMRFlags) ||
+				ui32Owner == PVR_SYS_ALLOC_PID ||
+				psDeviceNode == PVRSRVGetPVRSRVData()->psHostMemDeviceNode)
+			{
+				BIT_SET(psRIEntry->ui16Flags, RI_SYSALLOC_PMR_FLAG);
+				psRIEntry->pid = PVR_SYS_ALLOC_PID;
+				OSLockAcquire(g_hSysAllocPidListLock);
+				/* Add this psRIEntry to the list of entries for PVR_SYS_ALLOC_PID */
+				dllist_add_to_tail(&g_sSysAllocPidListHead, (PDLLIST_NODE)&(psRIEntry->sListNode));
+				OSLockRelease(g_hSysAllocPidListLock);
+				g_ui32SysAllocPMRCount++;
+			}
+			else
+			{
+				psRIEntry->pid = ui32Owner;
+				dllist_add_to_tail(&g_sClientsListHead, (PDLLIST_NODE)&(psRIEntry->sListNode));
+			}
+		}
+
+		psRIEntry->pmr_info.psPMR = psPMR;
+		SET_DEVICE_ID(psRIEntry, psDeviceNode->sDevId.ui32InternalID);
+
+		/* Create index entry in Hash Table */
+		HASH_Insert_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey, (uintptr_t)psRIEntry);
+
+		/* Store phRIHandle in PMR structure, so it can delete the associated RI entry when it destroys the PMR */
+		PMRStoreRIHandle(psPMR, psRIEntry);
+	}
+exit_:
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWritePMREntryKM
+
+ @Description
+            Writes a new Resource Information list entry.
+            The new entry will be inserted at the head of the list of
+            PMR RI entries and assigned the values provided.
+
+ @input     psPMR - Reference (handle) to the PMR to which this reference relates
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWritePMREntryKM(PMR *psPMR)
+{
+	return RIWritePMREntryWithOwnerKM(psPMR,
+	                                  OSGetCurrentClientProcessIDKM());
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWriteMEMDESCEntryKM
+
+ @Description
+            Writes a new Resource Information sublist entry.
+            The new entry will be inserted at the head of the sublist of
+            the indicated PMR list entry, and assigned the values provided.
+
+ @input     psConnection - Reference to the Services connection
+ @input     psDeviceNode - Reference to the device node
+ @input     psPMR - Reference (handle) to the PMR to which this MEMDESC RI entry relates
+ @input     ui32TextBSize - Length of string provided in psz8TextB parameter
+ @input     psz8TextB - String describing this secondary reference (may be null)
+ @input     ui64Offset - Offset from the start of the PMR at which this allocation begins
+ @input     ui64Size - Size of this allocation
+ @input     uiFlags - Flags indicating if this is a sub-allocation or an import
+ @output    phRIHandle - Handle to the created RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWriteMEMDESCEntryKM(void* psConnection,
+                                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   PMR *psPMR,
+                                   IMG_UINT32 ui32TextBSize,
+                                   const IMG_CHAR *psz8TextB,
+                                   IMG_UINT64 ui64Offset,
+                                   IMG_UINT64 ui64Size,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   RI_HANDLE *phRIHandle)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry;
+	RI_LIST_ENTRY *psRIEntry;
+	PMR *pPMRHashKey = psPMR;
+	uintptr_t hashData;
+	IMG_PID	pid;
+
+	IMG_CHAR szProcName[RI_PROC_BUF_SIZE];
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	/* Check Hash tables have been created (meaning at least one PMR has been defined) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(g_pPMR2RIListHashTable, "g_pPMR2RIListHashTable");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(g_PID2RISublistHashTable, "g_PID2RISublistHashTable");
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phRIHandle, "phRIHandle");
+
+	/* Only allow request for a firmware context that comes from a direct bridge
+	 * (psConnection == NULL). */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(!(psConnection != NULL && PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags)),
+	                                "psConnection for firmware context");
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	*phRIHandle = NULL;
+
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	psRIEntry = (RI_LIST_ENTRY *)hashData;
+	if (!psRIEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		PVR_DPF((PVR_DBG_ERROR, "RI Entry not found for PMR: '%s'", PMR_GetAnnotation(psPMR)));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_SUBLIST_ENTRY));
+	if (!psRISubEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		/* Error - no memory to allocate for new RI sublist entry */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	else
+	{
+		psRISubEntry->psRI = psRIEntry;
+		dllist_add_to_head(&(psRIEntry->sSubListFirst), &(psRISubEntry->sListNode));
+
+		/* Increment number of entries in sublist */
+		psRIEntry->ui16SubListCount++;
+		BIT_SET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+	}
+
+	if (PVRSRV_CHECK_RI_IMPORT(uiFlags))
+	{
+		BIT_SET(psRISubEntry->ui16Flags, RI_IMPORT_FLAG);
+	}
+	else
+	{
+		BIT_UNSET(psRISubEntry->ui16Flags, RI_IMPORT_FLAG);
+	}
+
+	if (PVRSRV_CHECK_RI_SUBALLOC(uiFlags))
+	{
+		BIT_SET(psRISubEntry->ui16Flags, RI_SUBALLOC_FLAG);
+	}
+	else
+	{
+		BIT_UNSET(psRISubEntry->ui16Flags, RI_SUBALLOC_FLAG);
+	}
+
+	/* If allocation is made during device or driver initialisation,
+	 * track the MEMDESC entry under PVR_SYS_ALLOC_PID, otherwise use
+	 * the current PID. */
+	{
+		PVRSRV_DEVICE_NODE *psRIDeviceNode = GET_DEVNODE(psRISubEntry->psRI);
+		IMG_INT iRet;
+
+		/* HostMemDevice doesn't update eDevState hence there's no need to test
+		 * for it. */
+		if (psRIDeviceNode == PVRSRVGetPVRSRVData()->psHostMemDeviceNode)
+		{
+			/* Imports on HostMemDev should be attributed to the importing
+			 * process. This way if an import is not freed before disconnect
+			 * it will be outlined in the gpu_mem_area for the given process.
+			 * Otherwise attribute the rest of the records to PVR_SYS_ALLOC_PID. */
+			if (IS_IMPORT(psRISubEntry))
+			{
+				psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+				iRet = OSStringSafeCopy(szProcName, OSGetCurrentClientProcessNameKM(),
+				                        RI_PROC_BUF_SIZE);
+			}
+			else
+			{
+				psRISubEntry->pid = PVR_SYS_ALLOC_PID;
+				iRet = OSStringSafeCopy(szProcName, "SysProc", RI_PROC_BUF_SIZE);
+
+				if (psRISubEntry->pid != psRISubEntry->psRI->pid)
+				{
+					PVR_LOG(("%s(1): current PID = %u, RI entry PID = %u, RI sub-entry PID = %u",
+					         __func__, OSGetCurrentClientProcessIDKM(), psRISubEntry->psRI->pid,
+					         psRISubEntry->pid));
+				}
+			}
+		}
+		else
+		{
+			/* All allocations done during device initialisation or belonging
+			 * to the Firmware should be attributed to PVR_SYS_ALLOC_PID. */
+			if (psRIDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+			    PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags))
+			{
+				psRISubEntry->pid = PVR_SYS_ALLOC_PID;
+				iRet = OSStringSafeCopy(szProcName, "SysProc", RI_PROC_BUF_SIZE);
+
+				if (psRISubEntry->pid != psRISubEntry->psRI->pid)
+				{
+					PVR_LOG(("%s(2): current PID = %u, RI entry PID = %u, RI sub-entry PID = %u",
+					         __func__, OSGetCurrentClientProcessIDKM(), psRISubEntry->psRI->pid,
+					         psRISubEntry->pid));
+				}
+			}
+			else
+			{
+				psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+				iRet = OSStringSafeCopy(szProcName, OSGetCurrentClientProcessNameKM(),
+				                        RI_PROC_BUF_SIZE);
+			}
+		}
+		if (iRet < 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: process name has been truncated from '%s' to '%s'",
+			         __func__, OSGetCurrentClientProcessNameKM(), szProcName));
+			psRISubEntry->ui16TextBLength = RI_PROC_BUF_SIZE - 1;
+		}
+		else
+		{
+			psRISubEntry->ui16TextBLength = iRet;
+		}
+
+		if (psz8TextB == NULL)
+		{
+			psz8TextB = "";
+			ui32TextBSize = 0;
+		}
+		if (ui32TextBSize > (RI_ANNO_BUF_SIZE))
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: TextBSize too long (%u). Text will be truncated "
+					 "to %d characters", __func__,
+					 ui32TextBSize, RI_ANNO_BUF_SIZE));
+			ui32TextBSize = RI_ANNO_BUF_SIZE;
+		}
+
+		/* copy TextB field data plus separator char and terminator */
+		psRISubEntry->ui16TextBLength += ui32TextBSize + 2;
+		psRISubEntry->pszTextB = OSAllocZMemNoStats(psRISubEntry->ui16TextBLength);
+		if (!psRISubEntry->pszTextB)
+		{
+			OSFreeMemNoStats(psRISubEntry);
+			_RIUnlock();
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		/* TextB is formatted as Annotation+NullTerm+ProcName so we can still
+		 * print annotation without proc name.
+		 * If any of the strings is too long it will be truncated. */
+		(void) OSStringSafeCopy(psRISubEntry->pszTextB, psz8TextB, ui32TextBSize + 1);
+		(void) OSStringSafeCopy(psRISubEntry->pszTextB + ui32TextBSize + 1, szProcName,
+		                        psRISubEntry->ui16TextBLength - ui32TextBSize - 1);
+
+		psRISubEntry->ui64Offset = ui64Offset;
+		psRISubEntry->ui64Size = ui64Size;
+		psRISubEntry->psConnection = psConnection;
+		dllist_init (&(psRISubEntry->sProcListNode));
+	}
+
+	/*
+	 *	Now insert this MEMDESC into the proc list
+	 */
+	/* look-up pid in Hash Table */
+	pid = psRISubEntry->pid;
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (!hashData)
+	{
+		/*
+		 * No allocations for this pid yet
+		 */
+		if (!HASH_Insert_Extended(g_PID2RISublistHashTable,
+		                          (void *) &pid,
+		                          (uintptr_t) &psRISubEntry->sProcListNode))
+		{
+			dllist_remove_node(&psRISubEntry->sListNode);
+			psRIEntry->ui16SubListCount--;
+			OSFreeMemNoStats(psRISubEntry->pszTextB);
+			OSFreeMemNoStats(psRISubEntry);
+
+			_RIUnlock();
+			return PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED;
+		}
+	}
+	else
+	{
+		/*
+		 * Insert allocation into pid allocations linked list
+		 */
+		PDLLIST_NODE currentNode = (PDLLIST_NODE)hashData;
+
+		/*
+		 * Insert new entry
+		 */
+		dllist_add_to_tail(currentNode, (PDLLIST_NODE)&(psRISubEntry->sProcListNode));
+	}
+
+	*phRIHandle = (RI_HANDLE)psRISubEntry;
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWriteProcListEntryKM
+
+ @Description
+            Write a new entry in the process list directly. We have to do this
+            because there might be no, multiple or changing PMR handles.
+
+            In the common case we have a PMR that will be added to the PMR list
+            and one or several MemDescs that are associated to it in a sub-list.
+            Additionally these MemDescs will be inserted in the per-process list.
+
+            There might be special descriptors from e.g. new user APIs that
+            are associated with no or multiple PMRs and not just one.
+            These can be now added to the per-process list (as RI_SUBLIST_ENTRY)
+            directly with this function and won't be listed in the PMR list (RIEntry)
+            because there might be no PMR.
+
+            To remove entries from the per-process list, just use
+            RIDeleteMEMDESCEntryKM().
+
+ @input     psConnection - Reference to the Services connection
+ @input     psDeviceNode - Reference to the device node
+ @input     psz8TextB - String describing this secondary reference (may be null)
+ @input     ui64Size - Size of this allocation
+ @input     ui64DevVAddr - Virtual address of this entry
+ @output    phRIHandle - Handle to the created RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWriteProcListEntryKM(void* psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32TextBSize,
+                                    const IMG_CHAR *psz8TextB,
+                                    IMG_UINT64 ui64Size,
+                                    IMG_UINT64 ui64DevVAddr,
+                                    RI_HANDLE *phRIHandle)
+{
+	uintptr_t hashData = 0;
+	IMG_PID		pid;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_INT iRet;
+
+	if (!g_pPMR2RIListHashTable)
+	{
+		g_pPMR2RIListHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(PMR*), HASH_Func_Default, HASH_Key_Comp_Default);
+		g_PID2RISublistHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(IMG_PID), _ProcHashFunc, _ProcHashComp);
+
+		if (!g_pPMR2RIListHashTable || !g_PID2RISublistHashTable)
+		{
+			/* Error - no memory to allocate for Hash table(s) */
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	*phRIHandle = NULL;
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_SUBLIST_ENTRY));
+	if (!psRISubEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		/* Error - no memory to allocate for new RI sublist entry */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	BIT_SET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+
+	psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+	SET_DEVICE_ID(psRISubEntry, psDeviceNode->sDevId.ui32InternalID);
+
+	if (psz8TextB == NULL)
+	{
+		psz8TextB = "";
+		ui32TextBSize = 0;
+	}
+	if (ui32TextBSize > (RI_ANNO_BUF_SIZE))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: TextBSize too long (%u). Text will be truncated "
+				 "to %d characters", __func__,
+				 ui32TextBSize, RI_ANNO_BUF_SIZE));
+		ui32TextBSize = RI_ANNO_BUF_SIZE;
+	}
+
+	/* copy TextB field data */
+	psRISubEntry->ui16TextBLength = ui32TextBSize + RI_PROC_BUF_SIZE;
+	psRISubEntry->pszTextB = OSAllocZMemNoStats(psRISubEntry->ui16TextBLength);
+	if (!psRISubEntry->pszTextB)
+	{
+		OSFreeMemNoStats(psRISubEntry);
+		_RIUnlock();
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	/* TextB is formatted as Annotation+NullTerm+ProcName so we can still
+	 * print annotation without proc name.
+	 */
+	(void) OSStringSafeCopy(psRISubEntry->pszTextB, psz8TextB, ui32TextBSize + 1);
+	iRet = OSStringSafeCopy(psRISubEntry->pszTextB + ui32TextBSize + 1,
+	                        OSGetCurrentClientProcessNameKM(),
+	                        psRISubEntry->ui16TextBLength - ui32TextBSize - 1);
+	if (iRet < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: process name has been truncated from '%s' to '%s'",
+		         __func__, OSGetCurrentClientProcessNameKM(),
+		         psRISubEntry->pszTextB + ui32TextBSize + 1));
+	}
+	psRISubEntry->ui64Size = ui64Size;
+	psRISubEntry->sVAddr.uiAddr = ui64DevVAddr;
+	psRISubEntry->psConnection = psConnection;
+	dllist_init (&(psRISubEntry->sProcListNode));
+
+	/*
+	 *	Now insert this MEMDESC into the proc list
+	 */
+	/* look-up pid in Hash Table */
+	pid = psRISubEntry->pid;
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (!hashData)
+	{
+		/*
+		 * No allocations for this pid yet
+		 */
+		if (!HASH_Insert_Extended(g_PID2RISublistHashTable,
+		                          (void *) &pid,
+		                          (uintptr_t) &psRISubEntry->sProcListNode))
+		{
+			OSFreeMemNoStats(psRISubEntry->pszTextB);
+			OSFreeMemNoStats(psRISubEntry);
+			_RIUnlock();
+			return PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED;
+		}
+	}
+	else
+	{
+		/*
+		 * Insert allocation into pid allocations linked list
+		 */
+		PDLLIST_NODE currentNode = (PDLLIST_NODE)hashData;
+
+		/*
+		 * Insert new entry
+		 */
+		dllist_add_to_tail(currentNode, (PDLLIST_NODE)&(psRISubEntry->sProcListNode));
+	}
+	*phRIHandle = (RI_HANDLE)psRISubEntry;
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIUpdateMEMDESCAddrKM
+
+ @Description
+            Update a Resource Information entry.
+
+ @input     hRIHandle - Handle of object whose reference info is to be updated
+ @input     sVAddr - New address for the RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIUpdateMEMDESCAddrKM(RI_HANDLE hRIHandle,
+								   IMG_DEV_VIRTADDR sVAddr)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hRIHandle, "hRIHandle");
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if (!IS_VALID(psRISubEntry))
+	{
+		/* Pointer does not point to valid structure */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	psRISubEntry->sVAddr.uiAddr = sVAddr.uiAddr;
+
+	/* Release RI lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RIDeletePMREntryUnlocked(RI_LIST_ENTRY *psRIEntry)
+{
+	PMR			*pPMRHashKey;
+	uintptr_t hashValue = 0;
+
+	/* Remove the HASH table index entry */
+	pPMRHashKey = GET_KEY(psRIEntry);
+	hashValue = HASH_Remove_Extended(g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "RI");
+
+	BIT_UNSET(psRIEntry->ui16Flags, RI_VALID_FLAG);
+
+	/* Remove PMR entry from linked-list of PMR entries */
+	dllist_remove_node((PDLLIST_NODE)&(psRIEntry->sListNode));
+
+	if (IS_SYSPMR(psRIEntry))
+	{
+		g_ui32SysAllocPMRCount--;
+	}
+
+	if (IS_RACC(psRIEntry))
+	{
+		OSFreeMemNoStats(psRIEntry->pmr_info.psPmrInfo);
+	}
+
+	/* Now, free the memory used to store the RI entry */
+	OSFreeMemNoStats(psRIEntry);
+	psRIEntry = NULL;
+
+	/* If the hash table is now empty we can delete the RI hash table */
+	if (HASH_Count(g_pPMR2RIListHashTable) == 0)
+	{
+		HASH_Delete(g_pPMR2RIListHashTable);
+		g_pPMR2RIListHashTable = NULL;
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RICreateAndSetPmrInfo(RI_LIST_ENTRY *const psRIEntry)
+{
+	uintptr_t uiEntry;
+	void* pPMRHashKey;
+	const IMG_CHAR *const pszAnnotation = PMR_GetAnnotation(psRIEntry->pmr_info.psPMR);
+	const IMG_UINT32 uiLength = OSStringNLength(pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN) + 1;
+
+	RI_PMR_INFO *psPmrInfo = OSAllocZMemNoStats(sizeof(*psPmrInfo) + uiLength);
+	PVR_LOG_RETURN_IF_NOMEM(psPmrInfo, "OSAllocZMemNoStats");
+
+	psPmrInfo->uiAddr = (uintptr_t) psRIEntry->pmr_info.psPMR;
+	psPmrInfo->psHeap = PMR_PhysHeap(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->psDeviceNode = (PVRSRV_DEVICE_NODE *) PMR_DeviceNode(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->uiLogicalSize = PMR_LogicalSize(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->uiPhysicalSize = PMR_PhysicalSize(psRIEntry->pmr_info.psPMR);
+
+	psPmrInfo->pszAnnotation = IMG_OFFSET_ADDR(psPmrInfo, sizeof(*psPmrInfo));
+	OSStringSafeCopy(psPmrInfo->pszAnnotation, pszAnnotation, uiLength);
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	psPmrInfo->uiXDevices = PMR_ImportedDevicesMask(psRIEntry->pmr_info.psPMR);
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+	/* Swap the key from the PMR address to the psPmrInfo's address. */
+
+	pPMRHashKey = GET_KEY(psRIEntry);
+	uiEntry = HASH_Remove_Extended(g_pPMR2RIListHashTable, (void*) &pPMRHashKey);
+	/* Should be removing itself to change the key. */
+	PVR_ASSERT(((void*) uiEntry) == ((void*) psRIEntry));
+
+	psRIEntry->pmr_info.psPmrInfo = psPmrInfo;
+	BIT_SET(psRIEntry->ui16Flags, RI_HAS_PMR_INFO);
+
+	pPMRHashKey = GET_KEY(psRIEntry);
+	HASH_Insert_Extended(g_pPMR2RIListHashTable, (void*) &pPMRHashKey, (uintptr_t) psRIEntry);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeletePMREntryKM
+
+ @Description
+            Delete a Resource Information entry.
+
+ @input     hRIHandle - Handle of object whose reference info is to be deleted
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeletePMREntryKM(RI_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_LIST_ENTRY *psRIEntry;
+
+	psRIEntry = (RI_LIST_ENTRY *) hRIHandle;
+
+	if (!RICheckListHandle(hRIHandle))
+	{
+		/* Pointer does not point to valid structure. */
+		PVR_LOG_GOTO_WITH_ERROR("RICheckListHandle", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrReturn);
+	}
+	else if (IS_RACC(psRIEntry))
+	{
+		/* Keep this entry so that it can be inspected as a memory leak in the
+		 * gpu_mem_area stats. */
+		return PVRSRV_OK;
+	}
+
+	if (psRIEntry->ui16SubListCount != 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%p not deleted. RIEntry(%s) still has %u allocation(s)",
+		         psRIEntry, GET_NAME(psRIEntry), psRIEntry->ui16SubListCount));
+
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP,
+		                    ErrCreateAndAssignPmrInfo);
+	}
+
+	_RILock();
+
+	eError = RIDeletePMREntryUnlocked(psRIEntry);
+
+	_RIUnlock();
+
+	/* If deInit has been deferred, we can now destroy the RI Lock */
+	if (bRIDeInitDeferred && (g_pPMR2RIListHashTable == NULL || HASH_Count(g_pPMR2RIListHashTable) == 0))
+	{
+		OSLockDestroy(g_hRILock);
+	}
+
+	return PVRSRV_OK;
+
+ErrCreateAndAssignPmrInfo:
+	{
+		PVRSRV_ERROR eError2;
+		_RILock();
+		eError2 = _RICreateAndSetPmrInfo(psRIEntry);
+		_RIUnlock();
+		PVR_LOG_RETURN_IF_ERROR(eError2, "_RICreateAndSetPmrInfo");
+	}
+ErrReturn:
+	return eError;
+}
+
+static PVRSRV_ERROR RIDeleteMemdescEntryUnlocked(RI_SUBLIST_ENTRY *psRISubEntry)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	uintptr_t hashData;
+	uintptr_t hashValue = 0;
+	IMG_PID pid;
+
+	/* For entries which do have a parent PMR remove the node from the sublist */
+	if (psRISubEntry->psRI)
+	{
+		psRIEntry = (RI_LIST_ENTRY *)psRISubEntry->psRI;
+
+		/* Now, remove entry from the sublist */
+		dllist_remove_node(&(psRISubEntry->sListNode));
+	}
+
+	BIT_UNSET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+
+	/* Remove the entry from the proc allocations linked list */
+	pid = psRISubEntry->pid;
+	/* If this is the only allocation for this pid, just remove it from the hash table */
+	if (dllist_get_next_node(&(psRISubEntry->sProcListNode)) == NULL)
+	{
+		hashValue = HASH_Remove_Extended(g_PID2RISublistHashTable, (void *)&pid);
+		PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "PID");
+		/* Delete the hash table if there are now no entries */
+		if (HASH_Count(g_PID2RISublistHashTable) == 0)
+		{
+			HASH_Delete(g_PID2RISublistHashTable);
+			g_PID2RISublistHashTable = NULL;
+		}
+	}
+	else
+	{
+		hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+		if ((PDLLIST_NODE)hashData == &(psRISubEntry->sProcListNode))
+		{
+			hashValue = HASH_Remove_Extended(g_PID2RISublistHashTable, (void *)&pid);
+			PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "PID");
+			HASH_Insert_Extended (g_PID2RISublistHashTable, (void *)&pid, (uintptr_t)dllist_get_next_node(&(psRISubEntry->sProcListNode)));
+		}
+	}
+	dllist_remove_node(&(psRISubEntry->sProcListNode));
+
+	/* Now, free the memory used to store the sublist entry */
+	OSFreeMemNoStats(psRISubEntry->pszTextB);
+	OSFreeMemNoStats(psRISubEntry);
+	psRISubEntry = NULL;
+
+	/*
+	 * Decrement number of entries in sublist if this MemDesc had a parent entry.
+	 */
+	if (psRIEntry)
+	{
+		psRIEntry->ui16SubListCount--;
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeleteMEMDESCEntryKM
+
+ @Description
+            Delete a Resource Information entry.
+            Entry can be from RIEntry list or ProcList.
+
+ @input     hRIHandle - Handle of object whose reference info is to be deleted
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeleteMEMDESCEntryKM(RI_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_SUBLIST_ENTRY *psRISubEntry;
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if (!RICheckSubListHandle(hRIHandle))
+	{
+		/* Pointer does not point to valid structure */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (IS_RACC(psRISubEntry))
+	{
+		return PVRSRV_OK;
+	}
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = RIDeleteMemdescEntryUnlocked(psRISubEntry);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	/*
+	 * Make the handle NULL once MEMDESC RI entry is deleted
+	 */
+	hRIHandle = NULL;
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeleteListKM
+
+ @Description
+            Delete all Resource Information entries and free associated
+            memory.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeleteListKM(void)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+
+	_RILock();
+
+	if (g_pPMR2RIListHashTable)
+	{
+		eResult = HASH_Iterate(g_pPMR2RIListHashTable, (HASH_pfnCallback)_DeleteAllEntries, NULL);
+		if (eResult == PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+		{
+			/*
+			 * PVRSRV_ERROR_RESOURCE_UNAVAILABLE is used to stop the Hash iterator when
+			 * the hash table gets deleted as a result of deleting the final PMR entry,
+			 * so this is not a real error condition...
+			 */
+			eResult = PVRSRV_OK;
+		}
+	}
+
+	/* After the run through the RIHashTable that holds the PMR entries there might be
+	 * still entries left in the per-process hash table because they were added with
+	 * RIWriteProcListEntryKM() and have no PMR parent associated.
+	 */
+	if (g_PID2RISublistHashTable)
+	{
+		eResult = HASH_Iterate(g_PID2RISublistHashTable, (HASH_pfnCallback) _DeleteAllProcEntries, NULL);
+		if (eResult == PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+		{
+			/*
+			 * PVRSRV_ERROR_RESOURCE_UNAVAILABLE is used to stop the Hash iterator when
+			 * the hash table gets deleted as a result of deleting the final PMR entry,
+			 * so this is not a real error condition...
+			 */
+			eResult = PVRSRV_OK;
+		}
+	}
+
+	_RIUnlock();
+
+	return eResult;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpListKM
+
+ @Description
+            Dumps out the contents of the RI List entry for the
+            specified PMR, and all MEMDESC allocation entries
+            in the associated sub linked list.
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @input     psPMR - PMR for which RI entry details are to be output
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpListKM(PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpList(psPMR, 0);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIGetListEntryKM
+
+ @Description
+            Returns pointer to a formatted string with details of the specified
+            list entry. If no entry exists (e.g. it may have been deleted
+            since the previous call), NULL is returned.
+
+ @input     pid - pid for which RI entry details are to be output
+ @input     ppHandle - handle to the entry, if NULL, the first entry will be
+                     returned.
+ @output    pszEntryString - string to be output for the entry
+ @output    hEntry - hEntry will be returned pointing to the next entry
+                     (or NULL if there is no next entry)
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+IMG_BOOL RIGetListEntryKM(IMG_PID pid,
+						  IMG_HANDLE **ppHandle,
+						  IMG_CHAR **ppszEntryString)
+{
+	RI_SUBLIST_ENTRY  *psRISubEntry = NULL;
+	RI_LIST_ENTRY  *psRIEntry = NULL;
+	uintptr_t     hashData = 0;
+	IMG_PID       hashKey  = pid;
+
+	static IMG_CHAR acStringBuffer[RI_FRMT_SIZE_MAX];
+
+	static IMG_UINT64 ui64TotalMemdescAlloc;
+	static IMG_UINT64 ui64TotalImport;
+	static IMG_UINT64 ui64TotalPMRAlloc;
+	static IMG_UINT64 ui64TotalPMRBacked;
+	static enum {
+		RI_GET_STATE_MEMDESCS_LIST_START,
+		RI_GET_STATE_MEMDESCS_SUMMARY,
+		RI_GET_STATE_PMR_LIST,
+		RI_GET_STATE_PMR_SUMMARY,
+		RI_GET_STATE_END,
+		RI_GET_STATE_LAST
+	} g_bNextGetState = RI_GET_STATE_MEMDESCS_LIST_START;
+
+	static DLLIST_NODE *psNode;
+	static DLLIST_NODE *psSysAllocNode;
+	static IMG_CHAR szProcName[RI_PROC_BUF_SIZE];
+	static IMG_UINT32 ui32ProcessedSysAllocPMRCount;
+
+	acStringBuffer[0] = '\0';
+
+	switch (g_bNextGetState)
+	{
+	case RI_GET_STATE_MEMDESCS_LIST_START:
+		/* look-up pid in Hash Table, to obtain first entry for pid */
+		hashData = HASH_Retrieve_Extended(g_PID2RISublistHashTable, (void *)&hashKey);
+		if (hashData)
+		{
+			if (*ppHandle)
+			{
+				psRISubEntry = (RI_SUBLIST_ENTRY *)*ppHandle;
+				if (!IS_VALID(psRISubEntry))
+				{
+					psRISubEntry = NULL;
+				}
+			}
+			else
+			{
+				psRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+				if (!IS_VALID(psRISubEntry))
+				{
+					psRISubEntry = NULL;
+				}
+			}
+		}
+
+		if (psRISubEntry)
+		{
+			PDLLIST_NODE psNextProcListNode = dllist_get_next_node(&psRISubEntry->sProcListNode);
+
+			if (IS_IMPORT(psRISubEntry))
+			{
+				ui64TotalImport += psRISubEntry->ui64Size;
+			}
+			else
+			{
+				ui64TotalMemdescAlloc += psRISubEntry->ui64Size;
+			}
+
+			_GenerateMEMDESCEntryString(psRISubEntry,
+										IMG_TRUE,
+										RI_MEMDESC_ENTRY_BUF_SIZE,
+										acStringBuffer);
+
+			if (szProcName[0] == '\0')
+			{
+				OSStringSafeCopy(szProcName, (pid == PVR_SYS_ALLOC_PID) ?
+						PVRSRV_MODNAME : GET_PROC(psRISubEntry), RI_PROC_BUF_SIZE);
+			}
+
+
+			*ppszEntryString = acStringBuffer;
+			*ppHandle        = (IMG_HANDLE)IMG_CONTAINER_OF(psNextProcListNode, RI_SUBLIST_ENTRY, sProcListNode);
+
+			if (psNextProcListNode == NULL ||
+				psNextProcListNode == (PDLLIST_NODE)hashData)
+			{
+				g_bNextGetState = RI_GET_STATE_MEMDESCS_SUMMARY;
+			}
+			/* else continue to list MEMDESCs */
+		}
+		else
+		{
+			if (ui64TotalMemdescAlloc == 0)
+			{
+				acStringBuffer[0] = '\0';
+				*ppszEntryString = acStringBuffer;
+				g_bNextGetState = RI_GET_STATE_MEMDESCS_SUMMARY;
+			}
+			/* else continue to list MEMDESCs */
+		}
+		break;
+
+	case RI_GET_STATE_MEMDESCS_SUMMARY:
+		OSSNPrintf(acStringBuffer,
+		           RI_MEMDESC_SUM_BUF_SIZE,
+		           RI_MEMDESC_SUM_FRMT,
+		           pid,
+		           szProcName,
+		           ui64TotalMemdescAlloc,
+		           ui64TotalMemdescAlloc >> 10,
+		           ui64TotalImport,
+		           ui64TotalImport >> 10,
+		           (ui64TotalMemdescAlloc + ui64TotalImport),
+		           (ui64TotalMemdescAlloc + ui64TotalImport) >> 10);
+
+		*ppszEntryString = acStringBuffer;
+		ui64TotalMemdescAlloc = 0;
+		ui64TotalImport = 0;
+		szProcName[0] = '\0';
+
+		g_bNextGetState = RI_GET_STATE_PMR_LIST;
+		break;
+
+	case RI_GET_STATE_PMR_LIST:
+		if (pid == PVR_SYS_ALLOC_PID)
+		{
+			OSLockAcquire(g_hSysAllocPidListLock);
+			acStringBuffer[0] = '\0';
+			if (!psSysAllocNode)
+			{
+				psSysAllocNode = &g_sSysAllocPidListHead;
+				ui32ProcessedSysAllocPMRCount = 0;
+			}
+			psSysAllocNode = dllist_get_next_node(psSysAllocNode);
+
+			if (szProcName[0] == '\0')
+			{
+				OSStringSafeCopy(szProcName, PVRSRV_MODNAME, RI_PROC_BUF_SIZE);
+			}
+			if (psSysAllocNode != NULL && psSysAllocNode != &g_sSysAllocPidListHead)
+			{
+				psRIEntry = IMG_CONTAINER_OF((PDLLIST_NODE)psSysAllocNode, RI_LIST_ENTRY, sListNode);
+				_GeneratePMREntryString(psRIEntry,
+										IMG_TRUE,
+										RI_PMR_ENTRY_BUF_SIZE,
+										acStringBuffer);
+				ui64TotalPMRAlloc += GET_LOGICAL_SIZE(psRIEntry);
+				ui64TotalPMRBacked += GET_PHYSICAL_SIZE(psRIEntry);
+
+				ui32ProcessedSysAllocPMRCount++;
+				if (ui32ProcessedSysAllocPMRCount > g_ui32SysAllocPMRCount+1)
+				{
+					g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+				}
+				/* else continue to list PMRs */
+			}
+			else
+			{
+				g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+			}
+			*ppszEntryString = (IMG_CHAR *)acStringBuffer;
+			OSLockRelease(g_hSysAllocPidListLock);
+		}
+		else
+		{
+			IMG_BOOL bPMRToDisplay = IMG_FALSE;
+
+			/* Iterate through the 'touched' PMRs and display details */
+			if (!psNode)
+			{
+				psNode = dllist_get_next_node(&g_sClientsListHead);
+			}
+			else
+			{
+				psNode = dllist_get_next_node(psNode);
+			}
+
+			while ((psNode != NULL && psNode != &g_sClientsListHead) &&
+					!bPMRToDisplay)
+			{
+				psRIEntry =	IMG_CONTAINER_OF(psNode, RI_LIST_ENTRY, sListNode);
+				if (psRIEntry->pid == pid)
+				{
+					/* This PMR was 'touched', so display details and unflag it*/
+					_GeneratePMREntryString(psRIEntry,
+											IMG_TRUE,
+											RI_PMR_ENTRY_BUF_SIZE,
+											acStringBuffer);
+					ui64TotalPMRAlloc += GET_LOGICAL_SIZE(psRIEntry);
+					ui64TotalPMRBacked += GET_PHYSICAL_SIZE(psRIEntry);
+
+					/* Remember the name of the process for 1 PMR for the summary */
+					if (szProcName[0] == '\0')
+					{
+						psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)), RI_SUBLIST_ENTRY, sListNode);
+						OSStringSafeCopy(szProcName, GET_PROC(psRISubEntry), RI_PROC_BUF_SIZE);
+					}
+					bPMRToDisplay = IMG_TRUE;
+				}
+				else
+				{
+					psNode = dllist_get_next_node(psNode);
+				}
+			}
+
+			if (psNode == NULL || (psNode == &g_sClientsListHead))
+			{
+				g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+			}
+			/* else continue listing PMRs */
+		}
+		break;
+
+	case RI_GET_STATE_PMR_SUMMARY:
+		OSSNPrintf(acStringBuffer,
+		           RI_PMR_SUM_BUF_SIZE,
+		           RI_PMR_SUM_FRMT,
+		           pid,
+		           szProcName,
+		           ui64TotalPMRAlloc,
+		           ui64TotalPMRAlloc >> 10,
+		           ui64TotalPMRBacked,
+		           ui64TotalPMRBacked >> 10);
+
+		*ppszEntryString = acStringBuffer;
+		ui64TotalPMRAlloc = 0;
+		ui64TotalPMRBacked = 0;
+		szProcName[0] = '\0';
+		psSysAllocNode = NULL;
+
+		g_bNextGetState = RI_GET_STATE_END;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "%s: Bad %d)",__func__, g_bNextGetState));
+
+		__fallthrough;
+	case RI_GET_STATE_END:
+		/* Reset state ready for the next gpu_mem_area file to display */
+		*ppszEntryString = NULL;
+		*ppHandle        = NULL;
+		psNode = NULL;
+		szProcName[0] = '\0';
+
+		g_bNextGetState = RI_GET_STATE_MEMDESCS_LIST_START;
+		return IMG_FALSE;
+		break;
+	}
+
+	return IMG_TRUE;
+}
+
+/* Function used to produce string containing info for MEMDESC RI entries (used for both debugfs and kernel log output) */
+static void _GenerateMEMDESCEntryString(RI_SUBLIST_ENTRY *psRISubEntry,
+                                        IMG_BOOL bDebugFs,
+                                        IMG_UINT16 ui16MaxStrLen,
+                                        IMG_CHAR *pszEntryString)
+{
+	IMG_CHAR szProc[RI_MEMDESC_ENTRY_PROC_BUF_SIZE];
+	IMG_CHAR szImport[RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE];
+	IMG_CHAR szEntryFormat[RI_MEMDESC_ENTRY_FRMT_SIZE];
+	const IMG_CHAR *pszAnnotationText;
+	IMG_PID uiRIPid = 0;
+	PMR* psRIPMR = NULL;
+	IMG_BOOL bSysPMR = IMG_FALSE;
+	IMG_BOOL bHostDevice = IS_HOST_DEVICE(psRISubEntry);
+	IMG_CHAR szDeviceID[RI_DEV_ID_BUF_SIZE];
+
+	if (psRISubEntry->psRI != NULL)
+	{
+		uiRIPid = psRISubEntry->psRI->pid;
+		psRIPMR = (PMR*)GET_ADDR(psRISubEntry->psRI);
+		bSysPMR = IS_SYSPMR(psRISubEntry->psRI);
+	}
+
+	OSSNPrintf(szEntryFormat,
+			RI_MEMDESC_ENTRY_FRMT_SIZE,
+			RI_MEMDESC_ENTRY_FRMT,
+			DEVMEM_ANNOTATION_MAX_LEN);
+
+	if (!bDebugFs)
+	{
+		/* we don't include process ID info for debugfs output */
+		OSSNPrintf(szProc,
+				RI_MEMDESC_ENTRY_PROC_BUF_SIZE,
+				RI_MEMDESC_ENTRY_PROC_FRMT,
+				psRISubEntry->pid,
+				GET_PROC(psRISubEntry));
+	}
+
+	if (!bHostDevice)
+	{
+		OSSNPrintf(szDeviceID,
+				   sizeof(szDeviceID),
+				   "%-3d",
+				   GET_DEVICE_ID(psRISubEntry));
+	}
+
+	if (IS_IMPORT(psRISubEntry) && psRIPMR)
+	{
+		OSSNPrintf((IMG_CHAR *)&szImport,
+		           RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE,
+		           RI_MEMDESC_ENTRY_IMPORT_FRMT,
+		           uiRIPid);
+		/* Set pszAnnotationText to that of the 'parent' PMR RI entry */
+		pszAnnotationText = GET_NAME(psRISubEntry->psRI);
+	}
+	else if (!IS_SUBALLOC(psRISubEntry) && psRIPMR)
+	{
+		/* Set pszAnnotationText to that of the 'parent' PMR RI entry */
+		pszAnnotationText = GET_NAME(psRISubEntry->psRI);
+	}
+	else
+	{
+		/* Set pszAnnotationText to that of the MEMDESC RI entry */
+		pszAnnotationText = psRISubEntry->pszTextB;
+	}
+
+	/* Don't print memdescs if they are local imports
+	 * (i.e. imported PMRs allocated by this process)
+	 */
+	if (bDebugFs &&
+		((psRISubEntry->sVAddr.uiAddr + psRISubEntry->ui64Offset) == 0) &&
+		(IS_IMPORT(psRISubEntry) && ((psRISubEntry->pid == uiRIPid)
+									 || (psRISubEntry->pid == PVR_SYS_ALLOC_PID))))
+	{
+		/* Don't print this entry */
+		pszEntryString[0] = '\0';
+	}
+	else
+	{
+		OSSNPrintf(pszEntryString,
+				   ui16MaxStrLen,
+				   szEntryFormat,
+				   (bDebugFs ? "" : "   "),
+				   psRISubEntry->pid,
+				   (bHostDevice ? "-  " : szDeviceID),
+				   (psRISubEntry->sVAddr.uiAddr + psRISubEntry->ui64Offset),
+				   pszAnnotationText,
+				   (bDebugFs ? "" : (char *)szProc),
+				   psRISubEntry->ui64Size,
+				   psRIPMR,
+				   (IS_IMPORT(psRISubEntry) ? (char *)&szImport : ""),
+				   (!IS_IMPORT(psRISubEntry) &&
+				    (bSysPMR) &&
+				    (psRISubEntry->pid != PVR_SYS_ALLOC_PID)) ? g_szSysAllocImport : "",
+				   (IS_RACC(psRISubEntry) ? RI_FREED_BY_DRIVER : ""),
+				   (bDebugFs ? '\n' : ' '));
+	}
+}
+
+/* Function used to produce string containing info for PMR RI entries (used for debugfs and kernel log output) */
+static void _GeneratePMREntryString(RI_LIST_ENTRY *psRIEntry,
+                                    IMG_BOOL bDebugFs,
+                                    IMG_UINT16 ui16MaxStrLen,
+                                    IMG_CHAR *pszEntryString)
+{
+	const IMG_CHAR*   pszAnnotationText;
+	const IMG_CHAR*   pszHeapText;
+
+	IMG_CHAR          szEntryFormat[RI_PMR_ENTRY_FRMT_SIZE];
+	IMG_BOOL          bHostDevice = IS_HOST_DEVICE(psRIEntry);
+	IMG_CHAR          szDeviceID[RI_DEV_ID_BUF_SIZE];
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	IMG_CHAR          szXDevices[RI_PMR_ENTRY_IMPORTED_BUF_SIZE];
+#else
+	IMG_CHAR          szXDevices[] = "";
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+	OSSNPrintf(szEntryFormat,
+			RI_PMR_ENTRY_FRMT_SIZE,
+			RI_PMR_ENTRY_FRMT,
+			(DEVMEM_ANNOTATION_MAX_LEN/2),
+			PHYS_HEAP_NAME_SIZE);
+
+	/* Set pszAnnotationText to that PMR RI entry */
+	pszAnnotationText = GET_NAME(psRIEntry);
+
+	/* Acquire PhysHeap Name to that PMR RI entry */
+	pszHeapText = PhysHeapName(GET_HEAP(psRIEntry));
+
+	if (!bHostDevice)
+	{
+		OSSNPrintf(szDeviceID,
+				   sizeof(szDeviceID),
+				   "%-3d",
+				   GET_DEVICE_ID(psRIEntry));
+	}
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	if (GET_XDEVICES(psRIEntry) != 0)
+	{
+		IMG_UINT64 uiXDevices = GET_XDEVICES(psRIEntry);
+		IMG_UINT8 i;
+		IMG_CHAR *pszHead = szXDevices;
+
+		pszHead += OSSNPrintf(pszHead,
+		                        sizeof(szXDevices) - (pszHead - szXDevices),
+		                        RI_PMR_ENTRY_IMPORTED_FRMT_BEGIN);
+		for (i = 0; i < PVRSRV_MAX_DEVICES; i++)
+		{
+			if (uiXDevices & (IMG_UINT64_C(1) << i))
+			{
+				pszHead += OSSNPrintf(pszHead,
+				                        sizeof(szXDevices) - (pszHead - szXDevices),
+				                        RI_PMR_ENTRY_IMPORTED_DEV_FRMT,
+				                        i);
+			}
+		}
+		pszHead += OSSNPrintf(pszHead,
+		                        sizeof(szXDevices) - (pszHead - szXDevices),
+		                        RI_PMR_ENTRY_IMPORTED_FRMT_END);
+	} else {
+		szXDevices[0] = '\0';
+	}
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+	OSSNPrintf(pszEntryString,
+	           ui16MaxStrLen,
+	           szEntryFormat,
+	           (bDebugFs ? "" : "   "),
+	           psRIEntry->pid,
+	           (bHostDevice ? "-  " : szDeviceID),
+	           GET_ADDR(psRIEntry),
+	           pszAnnotationText,
+	           pszHeapText,
+	           GET_LOGICAL_SIZE(psRIEntry),
+	           GET_PHYSICAL_SIZE(psRIEntry),
+	           (IS_RACC(psRIEntry) ? RI_FREED_BY_DRIVER : ""),
+			   szXDevices,
+	           (bDebugFs ? '\n' : ' '));
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_DumpList
+
+ @Description
+            Dumps out RI List entries according to parameters passed.
+
+ @input     psPMR - If not NULL, function will output the RI entries for
+                   the specified PMR only
+ @input     pid - If non-zero, the function will only output MEMDESC RI
+                  entries made by the process with ID pid.
+                  If zero, all MEMDESC RI entries will be output.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR _DumpList(PMR *psPMR, IMG_PID pid)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_UINT16 ui16SubEntriesParsed = 0;
+	uintptr_t hashData = 0;
+	IMG_PID hashKey;
+	PMR *pPMRHashKey = psPMR;
+	IMG_BOOL bDisplayedThisPMR = IMG_FALSE;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		if (pid != 0)
+		{
+			/* look-up pid in Hash Table */
+			hashKey = pid;
+			hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&hashKey);
+			if (hashData)
+			{
+				psRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+				if (psRISubEntry)
+				{
+					psRIEntry = psRISubEntry->psRI;
+				}
+			}
+		}
+		else
+		{
+			/* Look-up psPMR in Hash Table */
+			hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+			psRIEntry = (RI_LIST_ENTRY *)hashData;
+		}
+		if (!psRIEntry)
+		{
+			/* No entry found in hash table */
+			return PVRSRV_ERROR_NOT_FOUND;
+		}
+		while (psRIEntry)
+		{
+			bDisplayedThisPMR = IMG_FALSE;
+			/* Output details for RI entry */
+			if (!pid)
+			{
+				_RIOutput (("%s <%p> suballocs:%d size:0x%010" IMG_UINT64_FMTSPECx,
+				            GET_NAME(psRIEntry),
+				            GET_ADDR(psRIEntry),
+				            (IMG_UINT)psRIEntry->ui16SubListCount,
+				            GET_LOGICAL_SIZE(psRIEntry)));
+				bDisplayedThisPMR = IMG_TRUE;
+			}
+			ui16SubEntriesParsed = 0;
+			if (psRIEntry->ui16SubListCount)
+			{
+#if _DUMP_LINKEDLIST_INFO
+				_RIOutput (("RI LIST: {sSubListFirst.psNextNode:0x%p}\n",
+				            psRIEntry->sSubListFirst.psNextNode));
+#endif /* _DUMP_LINKEDLIST_INFO */
+				if (!pid)
+				{
+					psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)),
+					                                RI_SUBLIST_ENTRY, sListNode);
+				}
+				/* Traverse RI sublist and output details for each entry */
+				while (psRISubEntry)
+				{
+					if (psRIEntry)
+					{
+						if ((ui16SubEntriesParsed >= psRIEntry->ui16SubListCount))
+						{
+							break;
+						}
+						if (!bDisplayedThisPMR)
+						{
+							_RIOutput (("%s <%p> suballocs:%d size:0x%010" IMG_UINT64_FMTSPECx,
+								    GET_NAME(psRIEntry),
+								    GET_ADDR(psRIEntry),
+								    (IMG_UINT)psRIEntry->ui16SubListCount,
+								    GET_LOGICAL_SIZE(psRIEntry)));
+							bDisplayedThisPMR = IMG_TRUE;
+						}
+					}
+#if _DUMP_LINKEDLIST_INFO
+					_RIOutput (("RI LIST:    [this subentry:0x%p]\n",psRISubEntry));
+					_RIOutput (("RI LIST:     psRI:0x%p\n",psRISubEntry->psRI));
+#endif /* _DUMP_LINKEDLIST_INFO */
+
+					{
+						IMG_CHAR szEntryString[RI_MEMDESC_ENTRY_BUF_SIZE];
+
+						_GenerateMEMDESCEntryString(psRISubEntry,
+						                            IMG_FALSE,
+						                            RI_MEMDESC_ENTRY_BUF_SIZE,
+						                            szEntryString);
+						_RIOutput (("%s",szEntryString));
+					}
+
+					if (pid)
+					{
+						if ((dllist_get_next_node(&(psRISubEntry->sProcListNode)) == NULL) ||
+							(dllist_get_next_node(&(psRISubEntry->sProcListNode)) == (PDLLIST_NODE)hashData))
+						{
+							psRISubEntry = NULL;
+						}
+						else
+						{
+							psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sProcListNode)),
+							                                RI_SUBLIST_ENTRY, sProcListNode);
+							if (psRISubEntry)
+							{
+								if (psRIEntry != psRISubEntry->psRI)
+								{
+									/*
+									 * The next MEMDESC in the process linked list is in a different PMR
+									 */
+									psRIEntry = psRISubEntry->psRI;
+									bDisplayedThisPMR = IMG_FALSE;
+								}
+							}
+						}
+					}
+					else
+					{
+						ui16SubEntriesParsed++;
+						psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sListNode)),
+						                                RI_SUBLIST_ENTRY, sListNode);
+					}
+				}
+			}
+			if (!pid && psRIEntry)
+			{
+				if (ui16SubEntriesParsed != psRIEntry->ui16SubListCount)
+				{
+					/*
+					 * Output error message as sublist does not contain the
+					 * number of entries indicated by sublist count
+					 */
+					_RIOutput (("RI ERROR: RI sublist contains %d entries, not %d entries\n",
+					            ui16SubEntriesParsed, psRIEntry->ui16SubListCount));
+				}
+				else if (psRIEntry->ui16SubListCount && !dllist_get_next_node(&(psRIEntry->sSubListFirst)))
+				{
+					/*
+					 * Output error message as sublist is empty but sublist count
+					 * is not zero
+					 */
+					_RIOutput (("RI ERROR: ui16SubListCount=%d for empty RI sublist\n",
+					            psRIEntry->ui16SubListCount));
+				}
+			}
+			psRIEntry = NULL;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpAllKM
+
+ @Description
+            Dumps out the contents of all RI List entries (i.e. for all
+            MEMDESC allocations for each PMR).
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpAllKM(void)
+{
+	if (g_pPMR2RIListHashTable)
+	{
+		return HASH_Iterate(g_pPMR2RIListHashTable, (HASH_pfnCallback)_DumpAllEntries, NULL);
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpProcessKM
+
+ @Description
+            Dumps out the contents of all MEMDESC RI List entries (for every
+            PMR) which have been allocate by the specified process only.
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpProcessKM(IMG_PID pid)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 dummyPMR;
+
+	if (!g_PID2RISublistHashTable)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpList((PMR *)&dummyPMR, pid);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_TotalAllocsForProcess
+
+ @Description
+            Totals all PMR physical backing for given process.
+
+ @input     pid - ID of process.
+
+ @input     ePhysHeapType - type of Physical Heap for which to total allocs
+
+ @Return	Size of all physical backing for PID's PMRs allocated from the
+            specified heap type (in bytes).
+
+******************************************************************************/
+static IMG_INT32 _TotalAllocsForProcess(const IMG_PID pid, const PHYS_HEAP_TYPE ePhysHeapType)
+{
+	IMG_INT32 i32TotalPhysical = 0;
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		if (pid == PVR_SYS_ALLOC_PID)
+		{
+			DLLIST_NODE *psSysAllocNode = NULL;
+
+			OSLockAcquire(g_hSysAllocPidListLock);
+
+			for (psSysAllocNode = dllist_get_next_node(&g_sSysAllocPidListHead);
+			     psSysAllocNode != NULL && psSysAllocNode != &g_sSysAllocPidListHead;
+			     psSysAllocNode = dllist_get_next_node(psSysAllocNode))
+			{
+				const RI_LIST_ENTRY *const psRIEntry =
+					IMG_CONTAINER_OF((PDLLIST_NODE)psSysAllocNode, RI_LIST_ENTRY, sListNode);
+
+				/* Exclude RACC entries from the stats. This memory should have
+				 * been already freed during connection destruction so they should
+				 * not affect figure showing memory usage. Also RACC entries existence
+				 * is to show unfreed memory in `gpu_mem_area`, not to affect other
+				 * process stats. Finally if the RI entry exists only due to sub-entries
+				 * referencing it but the underlying PMR has been freed don't include
+				 * it in the total stats. */
+				if (PhysHeapGetType(GET_HEAP(psRIEntry)) == ePhysHeapType &&
+				    !IS_RACC(psRIEntry) && !HAS_PMR_INFO(psRIEntry))
+				{
+					IMG_UINT64 ui64PhysicalSize = GET_PHYSICAL_SIZE(psRIEntry);
+
+					if (((IMG_UINT64)i32TotalPhysical + ui64PhysicalSize > IMG_INT32_MAX))
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: i32TotalPhysical exceeding size for i32",
+						         __func__));
+					}
+					i32TotalPhysical += (IMG_INT32)(ui64PhysicalSize & 0x00000000ffffffff);
+				}
+			}
+
+			OSLockRelease(g_hSysAllocPidListLock);
+		}
+		else
+		{
+			RI_LIST_ENTRY *psRIEntry = NULL;
+			RI_SUBLIST_ENTRY *psInitialRISubEntry = NULL, *psRISubEntry = NULL;
+			uintptr_t hashData = 0;
+
+			if (pid != 0)
+			{
+				/* look-up pid in Hash Table */
+				IMG_PID hashKey = pid;
+				hashData = HASH_Retrieve_Extended(g_PID2RISublistHashTable, (void *)&hashKey);
+				if (hashData)
+				{
+					psInitialRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psInitialRISubEntry != NULL)
+					{
+						psRISubEntry = psInitialRISubEntry;
+						psRIEntry = psInitialRISubEntry->psRI;
+					}
+				}
+			}
+
+			while (psRISubEntry != NULL && psRIEntry != NULL)
+			{
+				DLLIST_NODE *psNextNode;
+
+				if (!IS_IMPORT(psRISubEntry) &&
+				    !IS_RACC(psRISubEntry) &&
+					!IS_COUNTED_BY_DEBUGFS(psRIEntry) &&
+				    !IS_SYSPMR(psRIEntry) &&
+				    (PhysHeapGetType(GET_HEAP(psRIEntry)) == ePhysHeapType))
+				{
+					IMG_UINT64 ui64PhysicalSize = GET_PHYSICAL_SIZE(psRIEntry);
+
+					if (((IMG_UINT64)i32TotalPhysical + ui64PhysicalSize > IMG_INT32_MAX))
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: i32TotalPhysical exceeding size for i32",
+						         __func__));
+					}
+
+					i32TotalPhysical += (IMG_INT32)(ui64PhysicalSize & 0x00000000ffffffff);
+					BIT_SET(psRIEntry->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG);
+				}
+
+				psNextNode = dllist_get_next_node(&(psRISubEntry->sProcListNode));
+				if (psNextNode == NULL || psNextNode == (PDLLIST_NODE)hashData)
+				{
+					psRISubEntry = NULL;
+					psRIEntry = NULL;
+				}
+				else
+				{
+					psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psRISubEntry)
+					{
+						psRIEntry = psRISubEntry->psRI;
+					}
+				}
+			}
+
+			psRISubEntry = psInitialRISubEntry;
+			if (psRISubEntry)
+			{
+				psRIEntry = psRISubEntry->psRI;
+			}
+
+			while (psRISubEntry != NULL && psRIEntry != NULL)
+			{
+				const DLLIST_NODE *const psNextNode =
+				    dllist_get_next_node(&(psRISubEntry->sProcListNode));
+
+				BIT_UNSET(psRIEntry->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG);
+
+				if (psNextNode == NULL || psNextNode == (PDLLIST_NODE)hashData)
+				{
+					psRISubEntry = NULL;
+					psRIEntry = NULL;
+				}
+				else
+				{
+					psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psRISubEntry)
+					{
+						psRIEntry = psRISubEntry->psRI;
+					}
+				}
+			}
+		}
+	}
+
+	return i32TotalPhysical;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RITotalAllocProcessKM
+
+ @Description
+            Returns the total of allocated GPU memory (backing for PMRs)
+            which has been allocated from the specific heap by the specified
+            process only.
+
+ @Return	Amount of physical backing allocated (in bytes)
+
+******************************************************************************/
+IMG_INT32 RITotalAllocProcessUnlocked(IMG_PID pid, PHYS_HEAP_TYPE ePhysHeapType)
+{
+	IMG_INT32 i32BackingTotal = 0;
+
+	if (g_PID2RISublistHashTable)
+	{
+		i32BackingTotal = _TotalAllocsForProcess(pid, ePhysHeapType);
+	}
+	return i32BackingTotal;
+}
+
+#if defined(DEBUG)
+/*!
+*******************************************************************************
+
+ @Function	_DumpProcessList
+
+ @Description
+            Dumps out RI List entries according to parameters passed.
+
+ @input     psPMR - If not NULL, function will output the RI entries for
+                   the specified PMR only
+ @input     pid - If non-zero, the function will only output MEMDESC RI
+                  entries made by the process with ID pid.
+                  If zero, all MEMDESC RI entries will be output.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR _DumpProcessList(PMR *psPMR,
+									 IMG_PID pid,
+									 IMG_UINT64 ui64Offset,
+									 IMG_DEV_VIRTADDR *psDevVAddr)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_UINT16 ui16SubEntriesParsed = 0;
+	uintptr_t hashData = 0;
+	PMR *pPMRHashKey = psPMR;
+
+	psDevVAddr->uiAddr = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		PVR_ASSERT(psPMR && pid);
+
+		/* Look-up psPMR in Hash Table */
+		hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+		psRIEntry = (RI_LIST_ENTRY *)hashData;
+
+		if (!psRIEntry)
+		{
+			/* No entry found in hash table */
+			return PVRSRV_ERROR_NOT_FOUND;
+		}
+
+		if (psRIEntry->ui16SubListCount)
+		{
+			psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)),
+											RI_SUBLIST_ENTRY, sListNode);
+
+			/* Traverse RI sublist and output details for each entry */
+			while (psRISubEntry && (ui16SubEntriesParsed < psRIEntry->ui16SubListCount))
+			{
+				if (pid == psRISubEntry->pid)
+				{
+					IMG_UINT64 ui64StartOffset = psRISubEntry->ui64Offset;
+					IMG_UINT64 ui64EndOffset = psRISubEntry->ui64Offset + psRISubEntry->ui64Size;
+
+					if (ui64Offset >= ui64StartOffset && ui64Offset < ui64EndOffset)
+					{
+						psDevVAddr->uiAddr = psRISubEntry->sVAddr.uiAddr;
+						return PVRSRV_OK;
+					}
+				}
+
+				ui16SubEntriesParsed++;
+				psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sListNode)),
+												RI_SUBLIST_ENTRY, sListNode);
+			}
+		}
+	}
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpProcessListKM
+
+ @Description
+            Dumps out selected contents of all MEMDESC RI List entries (for a
+            PMR) which have been allocate by the specified process only.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpProcessListKM(PMR *psPMR,
+								 IMG_PID pid,
+								 IMG_UINT64 ui64Offset,
+								 IMG_DEV_VIRTADDR *psDevVAddr)
+{
+	PVRSRV_ERROR eError;
+
+	if (!g_PID2RISublistHashTable)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpProcessList(psPMR,
+							  pid,
+							  ui64Offset,
+							  psDevVAddr);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+#endif
+
+static PVRSRV_ERROR _MarkRACCEntries(uintptr_t k, uintptr_t v, void *psConnection)
+{
+	/* This is not a head node. See sProcListNode. */
+	DLLIST_NODE *psListStart = (DLLIST_NODE *) v;
+	DLLIST_NODE *psNextNode = psListStart;
+
+	PVR_UNREFERENCED_PARAMETER(k);
+
+	do
+	{
+		RI_SUBLIST_ENTRY *psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY,
+		                                                  sProcListNode);
+
+		if (psRISubEntry->psConnection == psConnection)
+		{
+			RI_LIST_ENTRY *psRIEntry = psRISubEntry->psRI;
+
+			BIT_SET(psRISubEntry->ui16Flags, RI_RACC_FLAG);
+
+			/* RI sub-entry may not have an RI entry associated with it. If that's the
+			 * case just skip processing it. */
+			if (psRIEntry != NULL)
+			{
+				if (!IS_RACC(psRIEntry) &&
+				    !HAS_PMR_INFO(psRIEntry) &&
+				    /* Mark as RACC only if the entry doesn't belong to the system process.
+				     * System process allocations are alive for the whole driver lifetime
+					 * hence they will always exist at a connection closed for every process
+					 * that references them. */
+				    psRIEntry->pid != PVR_SYS_ALLOC_PID)
+				{
+					PVRSRV_ERROR eError;
+
+					eError = _RICreateAndSetPmrInfo(psRIEntry);
+					PVR_LOG_RETURN_IF_ERROR(eError, "_RICreateAndSetPmrInfo");
+
+					BIT_SET(psRIEntry->ui16Flags, RI_RACC_FLAG);
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "RIEntry(%s) is already RACC", GET_NAME(psRIEntry)));
+				}
+			}
+		}
+
+		psNextNode = dllist_get_next_node(psNextNode);
+	} while (psNextNode != NULL && psNextNode != psListStart);
+
+	return PVRSRV_OK;
+}
+
+void RIConnectionClosed(void* psConnection)
+{
+	PVRSRV_ERROR eError;
+	if (g_PID2RISublistHashTable)
+	{
+		_RILock();
+		eError = HASH_Iterate(g_PID2RISublistHashTable, (HASH_pfnCallback)_MarkRACCEntries, psConnection);
+		PVR_LOG_IF_FALSE(eError == PVRSRV_OK, "_MarkRACCEntries");
+		_RIUnlock();
+	}
+}
+
+static PVRSRV_ERROR DeleteRACCEntry(RI_SUBLIST_ENTRY *psRISubEntry)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_LIST_ENTRY *psRIEntry = psRISubEntry->psRI;
+
+	PVR_LOG_IF_FALSE_VA(PVR_DBG_WARNING, IS_RACC(psRISubEntry), "Non-RACC entry(%s)",
+	                    psRISubEntry->pszTextB);
+
+	eError = RIDeleteMemdescEntryUnlocked(psRISubEntry);
+	PVR_LOG_IF_ERROR(eError, "RIDeleteMemdescEntryUnlocked");
+
+	/* RI sub-entry may not have an RI entry associated with it. If that's the
+	 * case just skip processing it. */
+	if (psRIEntry != NULL)
+	{
+		if (psRIEntry->ui16SubListCount)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%p: More than 1 sublist present! (%s)",
+			         psRIEntry, __func__));
+		}
+		else
+		{
+			eError = RIDeletePMREntryUnlocked(psRIEntry);
+			PVR_LOG_IF_ERROR(eError, "RIDeletePMREntryUnlocked");
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RIDeleteEntriesForPID(IMG_PID pid)
+{
+	/* This is not a head node. See sProcListNode. */
+	DLLIST_NODE *psListStart;
+	DLLIST_NODE *psNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_PID hashKey  = pid;
+
+	_RILock();
+
+	PVR_LOG_GOTO_IF_FALSE(g_PID2RISublistHashTable != NULL, "g_PID2RISublistHashTable is NULL", ErrHashError);
+
+	psListStart = (DLLIST_NODE *) HASH_Retrieve_Extended(g_PID2RISublistHashTable,
+	                                                    (void *) &hashKey);
+	PVR_GOTO_IF_FALSE(psListStart != NULL, ExitUnlockAndReturn);
+	psNode = psListStart;
+
+	do
+	{
+		DLLIST_NODE *psNext = dllist_get_next_node(psNode);
+		RI_SUBLIST_ENTRY *psRISubEntry = IMG_CONTAINER_OF(psNode,
+		                                                  RI_SUBLIST_ENTRY,
+		                                                  sProcListNode);
+
+		eError = DeleteRACCEntry(psRISubEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DeleteRACCEntry", ExitUnlockAndReturn);
+
+		psNode = psNext;
+	} while (psNode != NULL && psNode != psListStart);
+
+	goto ExitUnlockAndReturn;
+
+ErrHashError:
+    eError = PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE;
+
+ExitUnlockAndReturn:
+	_RIUnlock();
+
+	return eError;
+}
+
+static PVRSRV_ERROR _DumpAllEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *)v;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	return RIDumpListKM(GET_ADDR(psRIEntry));
+}
+
+static PVRSRV_ERROR _DeleteAllEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *)v;
+	RI_SUBLIST_ENTRY *psRISubEntry;
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	while ((eResult == PVRSRV_OK) && (psRIEntry->ui16SubListCount > 0))
+	{
+		psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)), RI_SUBLIST_ENTRY, sListNode);
+		eResult = RIDeleteMEMDESCEntryKM((RI_HANDLE)psRISubEntry);
+	}
+	if (eResult == PVRSRV_OK)
+	{
+		eResult = RIDeletePMREntryKM((RI_HANDLE)psRIEntry);
+		/*
+		 * If we've deleted the Hash table, return
+		 * an error to stop the iterator...
+		 */
+		if (!g_pPMR2RIListHashTable)
+		{
+			eResult = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		}
+	}
+	return eResult;
+}
+
+static PVRSRV_ERROR _DeleteAllProcEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry = (RI_SUBLIST_ENTRY *)v;
+	PVRSRV_ERROR eResult;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	eResult = RIDeleteMEMDESCEntryKM((RI_HANDLE) psRISubEntry);
+	if (eResult == PVRSRV_OK && !g_PID2RISublistHashTable)
+	{
+		/*
+		 * If we've deleted the Hash table, return
+		 * an error to stop the iterator...
+		 */
+		eResult = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	return eResult;
+}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/srvcore.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/srvcore.c
new file mode 100644
index 000000000000..203c89adda15
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/srvcore.c
@@ -0,0 +1,1621 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements core PVRSRV API, server side
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types_check.h"
+#include "pvr_debug.h"
+#include "ra.h"
+#include "pvr_bridge.h"
+#include "connection_server.h"
+#include "device.h"
+#include "htbserver.h"
+
+#include "pdump_km.h"
+
+#include "srvkm.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "log2.h"
+
+#include "srvcore.h"
+#include "pvrsrv.h"
+#include "power.h"
+
+#include "os_apphint.h"
+
+#if defined(SUPPORT_RGX)
+#include "rgxdevice.h"
+#include "rgxinit.h"
+#include "rgx_compat_bvnc.h"
+#endif
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+#include "lock.h"
+#include "osfunc.h"
+#include "device_connection.h"
+#include "process_stats.h"
+#include "pvrsrv_pool.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "physmem_lma.h"
+#include "services_km.h"
+#endif
+
+#include "pvrsrv_tlstreams.h"
+#include "tlstream.h"
+
+#if defined(PVRSRV_MISSING_NO_SPEC_IMPL)
+#pragma message ("There is no implementation of OSConfineArrayIndexNoSpeculation() - see osfunc.h")
+#endif
+
+/* For the purpose of maintainability, it is intended that this file should not
+ * contain any OS specific #ifdefs. Please find a way to add e.g.
+ * an osfunc.c abstraction or override the entire function in question within
+ * env,*,pvr_bridge_k.c
+ */
+
+PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY g_BridgeDispatchTable[BRIDGE_DISPATCH_TABLE_ENTRY_COUNT] = { {.pfFunction = DummyBW,} ,};
+
+#define PVR_DISPATCH_OFFSET_FIRST_FUNC			0
+#define PVR_DISPATCH_OFFSET_LAST_FUNC			1
+#define PVR_DISPATCH_OFFSET_ARRAY_MAX			2
+
+#define PVRSRV_CLIENT_TL_STREAM_SIZE_DEFAULT PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE
+
+static IMG_UINT16 g_BridgeDispatchTableStartOffsets[BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT][PVR_DISPATCH_OFFSET_ARRAY_MAX];
+
+
+#define PVRSRV_MAX_POOLED_BRIDGE_BUFFERS 8	/*!< Initial number of pooled bridge buffers */
+
+static PVRSRV_POOL *g_psBridgeBufferPool;	/*! Pool of bridge buffers */
+
+
+#if defined(DEBUG_BRIDGE_KM)
+/* a lock used for protecting bridge call timing calculations
+ * for calls which do not acquire a lock
+ */
+static POS_LOCK g_hStatsLock;
+PVRSRV_BRIDGE_GLOBAL_STATS g_BridgeGlobalStats;
+
+void BridgeGlobalStatsLock(void)
+{
+	OSLockAcquire(g_hStatsLock);
+}
+
+void BridgeGlobalStatsUnlock(void)
+{
+	OSLockRelease(g_hStatsLock);
+}
+#endif
+
+void BridgeDispatchTableStartOffsetsInit(void)
+{
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEFAULT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEFAULT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SRVCORE][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SRVCORE][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED1][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RESERVED1_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED1][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED2][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RESERVED2_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED2][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RESERVED2_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPCTRL][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPCTRL][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMPLAT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMPLAT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_CMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_CMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMP][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMP][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMABUF][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMABUF][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CACHE][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CACHE][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_CACHE_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PVRTL][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PVRTL][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RI][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RI_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RI][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RI_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_VALIDATION][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_VALIDATION][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_TUTILS][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_TUTILS_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_TUTILS][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEVICEMEMHISTORY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEVICEMEMHISTORY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_HTBUFFER][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_HTBUFFER][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DCPLAT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DCPLAT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMEXTMEM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMEXTMEM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCTRACKING][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCTRACKING][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCFALLBACK][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCFALLBACK][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DI][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DI_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DI][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DI_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMA][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DMA_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMA][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DMA_DISPATCH_LAST;
+#if defined(SUPPORT_RGX)
+	/* Need a gap here to start next entry at element 128 */
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXCMP][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXCMP][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTA3D][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTA3D][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXBREAKPOINT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXBREAKPOINT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXFWDBG][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXFWDBG][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXPDUMP][PVR_DISPATCH_OFFSET_FIRST_FUNC]= PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXPDUMP][PVR_DISPATCH_OFFSET_LAST_FUNC]= PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXHWPERF][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXHWPERF][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXREGCONFIG][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXREGCONFIG][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXKICKSYNC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXKICKSYNC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ2][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ2][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTIMERQUERY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTIMERQUERY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXRAY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXRAY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST;
+#endif
+}
+
+#if defined(DEBUG_BRIDGE_KM)
+
+#if defined(INTEGRITY_OS)
+void PVRSRVPrintBridgeStats(void)
+{
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Remainder;
+
+	BridgeGlobalStatsLock();
+
+	printf("Total Bridge call count = %u\n"
+		   "Total number of bytes copied via copy_from_user = %u\n"
+		   "Total number of bytes copied via copy_to_user = %u\n"
+		   "Total number of bytes copied via copy_*_user = %u\n\n"
+		   "%3s: %-60s | %-48s | %10s | %20s | %20s | %20s | %20s\n",
+		   g_BridgeGlobalStats.ui32IOCTLCount,
+		   g_BridgeGlobalStats.ui32TotalCopyFromUserBytes,
+		   g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		   g_BridgeGlobalStats.ui32TotalCopyFromUserBytes + g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		   "#",
+		   "Bridge Name",
+		   "Wrapper Function",
+		   "Call Count",
+		   "copy_from_user (B)",
+		   "copy_to_user (B)",
+		   "Total Time (us)",
+		   "Max Time (us)");
+
+	/* Is the item asked for (starts at 0) a valid table index? */
+	for ( ui32Index=0; ui32Index < BRIDGE_DISPATCH_TABLE_ENTRY_COUNT; ui32Index++ )
+	{
+		PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psEntry = &g_BridgeDispatchTable[ui32Index];
+		printf("%3d: %-60s   %-48s   %-10u   %-20u   %-20u   %-20llu   %-20llu\n",
+			   (IMG_UINT32)(((size_t)psEntry-(size_t)g_BridgeDispatchTable)/sizeof(*g_BridgeDispatchTable)),
+			   psEntry->pszIOCName,
+			   (psEntry->pfFunction != NULL) ? psEntry->pszFunctionName : "(null)",
+			   psEntry->ui32CallCount,
+			   psEntry->ui32CopyFromUserTotalBytes,
+			   psEntry->ui32CopyToUserTotalBytes,
+			   (unsigned long long) OSDivide64r64(psEntry->ui64TotalTimeNS, 1000, &ui32Remainder),
+			   (unsigned long long) OSDivide64r64(psEntry->ui64MaxTimeNS, 1000, &ui32Remainder));
+
+
+	}
+
+	BridgeGlobalStatsUnlock();
+}
+#endif
+
+PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+					IMG_UINT32 ui32DispatchTableEntry,
+					void *pvDest,
+					void __user *pvSrc,
+					IMG_UINT32 ui32Size)
+{
+	BridgeGlobalStatsLock();
+	g_BridgeDispatchTable[ui32DispatchTableEntry].ui32CopyFromUserTotalBytes+=ui32Size;
+	g_BridgeGlobalStats.ui32TotalCopyFromUserBytes+=ui32Size;
+	BridgeGlobalStatsUnlock();
+
+	return OSBridgeCopyFromUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+				  IMG_UINT32 ui32DispatchTableEntry,
+				  void __user *pvDest,
+				  void *pvSrc,
+				  IMG_UINT32 ui32Size)
+{
+	BridgeGlobalStatsLock();
+	g_BridgeDispatchTable[ui32DispatchTableEntry].ui32CopyToUserTotalBytes+=ui32Size;
+	g_BridgeGlobalStats.ui32TotalCopyToUserBytes+=ui32Size;
+	BridgeGlobalStatsUnlock();
+
+	return OSBridgeCopyToUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+#endif
+
+/**************************************************************************/ /*!
+@Function       DeviceDefaultPhysHeapFreeMemCheck
+
+@Description    Check if the required amount of free space is available in the
+                Default PhysHeap for a connection to be made.
+
+@Input          psDeviceNode    The device the connection is being
+                                made on.
+@Input          ui32MinMemInMBs The minimum memory required to be
+                                available in the Default PhysHeap.
+
+@Return         PVRSRV_OK if successful else a PVRSRV_ERROR.
+*/ /***************************************************************************/
+static PVRSRV_ERROR DeviceDefaultPhysHeapFreeMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                      IMG_UINT32 ui32MinMemInMBs)
+{
+	PHYS_HEAP *psDefaultHeap = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL, "psDeviceNode");
+
+	psDefaultHeap = psDeviceNode->apsPhysHeap[psDeviceNode->psDevConfig->eDefaultHeap];
+	if (psDefaultHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's default PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psDefaultHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psDefaultHeap,
+		                              MB2B(ui32MinMemInMBs),
+		                              &ui64FreePhysHeapMem);
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Default PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"MB "
+				"Minimum required: %uMB",
+				B2MB(ui64FreePhysHeapMem),
+				ui32MinMemInMBs));
+		}
+	}
+
+	return eError;
+}
+
+/**************************************************************************/ /*!
+@Function       CheckConnectionPhysHeapMem
+
+@Description    Check if there is enough memory in the PhysHeaps to allow a
+                connection to be made.
+
+@Input          psConnection    The connection being made.
+
+@Return         PVRSRV_OK if successful else a PVRSRV_ERROR.
+*/ /***************************************************************************/
+static PVRSRV_ERROR CheckConnectionPhysHeapMem(CONNECTION_DATA *psConnection)
+{
+	IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION;
+	IMG_UINT32 ui32AppHintPhysHeapMinMemOnConnection = 0;
+	void *pvAppHintState = NULL;
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_INVALID_PARAM(psConnection);
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, PhysHeapMinMemOnConnection,
+		&ui32AppHintDefault, &ui32AppHintPhysHeapMinMemOnConnection);
+	OSFreeAppHintState(pvAppHintState);
+
+	psDeviceNode = OSGetDevNode(psConnection);
+
+	if (ui32AppHintPhysHeapMinMemOnConnection != 0)
+	{
+		eError = DeviceDefaultPhysHeapFreeMemCheck(psDeviceNode,
+		                                           ui32AppHintPhysHeapMinMemOnConnection);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DeviceDefaultPhysHeapFreeMemCheck");
+
+		if (psDeviceNode->pfnCheckForSufficientFWPhysMem != NULL
+		    && RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION > 0)
+		{
+			eError = psDeviceNode->pfnCheckForSufficientFWPhysMem(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "pfnCheckForSufficientFWPhysMem");
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVConnectKM(CONNECTION_DATA *psConnection,
+				PVRSRV_DEVICE_NODE * psDeviceNode,
+				IMG_UINT32 ui32Flags,
+				IMG_UINT32 ui32ClientBuildOptions,
+				IMG_UINT32 ui32ClientDDKVersion,
+				IMG_UINT32 ui32ClientDDKBuild,
+				IMG_UINT8  *pui8KernelArch,
+				IMG_UINT32 *pui32CapabilityFlags,
+				IMG_UINT64 *ui64PackedBvnc)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32ServerBuildOptions;
+	IMG_UINT32			ui32DDKVersion, ui32DDKBuild;
+	PVRSRV_DATA			*psSRVData = NULL;
+	IMG_UINT64			ui64ProcessVASpaceSize = OSGetCurrentProcessVASpaceSize();
+	static IMG_BOOL		bIsFirstConnection=IMG_FALSE;
+#if defined(SUPPORT_RGX)
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#endif
+
+	/* Check the minimum free PhysHeap memory is available before allowing
+	 * the connection to succeed */
+	eError = CheckConnectionPhysHeapMem(psConnection);
+	PVR_RETURN_IF_ERROR(eError);
+
+#if defined(SUPPORT_RGX)
+	/* Gather BVNC information to output to UM */
+
+	*ui64PackedBvnc = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+	                       psDevInfo->sDevFeatureCfg.ui32V,
+	                       psDevInfo->sDevFeatureCfg.ui32N,
+	                       psDevInfo->sDevFeatureCfg.ui32C);
+#else
+	*ui64PackedBvnc = 0;
+#endif /* defined(SUPPORT_RGX)*/
+
+	/* Clear the flags */
+	*pui32CapabilityFlags = 0;
+
+	psSRVData = PVRSRVGetPVRSRVData();
+
+	psConnection->ui32ClientFlags = ui32Flags;
+
+	/*Set flags to pass back to the client showing which cache coherency is available.*/
+	/* Is the system snooping of caches emulated in software? */
+	if (PVRSRVSystemSnoopingIsEmulated(psDeviceNode->psDevConfig))
+	{
+		*pui32CapabilityFlags |= PVRSRV_CACHE_COHERENT_EMULATE_FLAG;
+	}
+	else
+	{
+		/*Set flags to pass back to the client showing which cache coherency is available.*/
+		/*Is the system CPU cache coherent?*/
+		if (PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig))
+		{
+			*pui32CapabilityFlags |= PVRSRV_CACHE_COHERENT_DEVICE_FLAG;
+		}
+	}
+
+	/* Has the system device non-mappable local memory?*/
+	if (PVRSRVSystemHasNonMappableLocalMemory(psDeviceNode->psDevConfig))
+	{
+		*pui32CapabilityFlags |= PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG;
+	}
+
+	/* Is system using FBCDC v31? */
+	if (psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode))
+	{
+		*pui32CapabilityFlags |= PVRSRV_FBCDC_V3_1_USED;
+	}
+
+	/* Set flags to indicate shared-virtual-memory (SVM) allocation availability */
+	if (! psDeviceNode->ui64GeneralSVMHeapTopVA || ! ui64ProcessVASpaceSize)
+	{
+		*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED;
+	}
+	else
+	{
+		if (ui64ProcessVASpaceSize <= psDeviceNode->ui64GeneralSVMHeapTopVA)
+		{
+			*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED;
+		}
+		else
+		{
+			/* This can happen when processor has more virtual address bits
+			   than device (i.e. alloc is not always guaranteed to succeed) */
+			*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL;
+		}
+	}
+
+	/* Is the system DMA capable? */
+	if (psDeviceNode->psDevConfig->bHasDma)
+	{
+		*pui32CapabilityFlags |= PVRSRV_SYSTEM_DMA_USED;
+	}
+
+#if defined(SUPPORT_RGX) && defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	/* For GPUs with lossy TFBC support, is system using lossy control group 1? */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT))
+	{
+		if (psDeviceNode->pfnGetTFBCLossyGroup(psDeviceNode) == 1)
+		{
+			*pui32CapabilityFlags |= PVRSRV_TFBC_LOSSY_GROUP_1;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+{
+	IMG_UINT32 ui32OSid = 0, ui32OSidReg = 0;
+	IMG_BOOL   bOSidAxiProtReg = IMG_FALSE;
+
+	ui32OSid    = (ui32Flags & SRV_VIRTVAL_FLAG_OSID_MASK)    >> (VIRTVAL_FLAG_OSID_SHIFT);
+	ui32OSidReg = (ui32Flags & SRV_VIRTVAL_FLAG_OSIDREG_MASK) >> (VIRTVAL_FLAG_OSIDREG_SHIFT);
+
+#if defined(EMULATOR)
+{
+    /* AXI_ACELITE is only supported on rogue cores - volcanic cores all support full ACE
+     * and don't want to compile the code below (RGX_FEATURE_AXI_ACELITE_BIT_MASK is not
+     * defined for volcanic cores).
+     */
+
+     PVRSRV_RGXDEV_INFO *psDevInfo;
+     psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#else
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACE))
+#endif
+	{
+		IMG_UINT32 ui32OSidAxiProtReg = 0, ui32OSidAxiProtTD = 0;
+
+		ui32OSidAxiProtReg = (ui32Flags & SRV_VIRTVAL_FLAG_AXIPREG_MASK) >> (VIRTVAL_FLAG_AXIPREG_SHIFT);
+		ui32OSidAxiProtTD  = (ui32Flags & SRV_VIRTVAL_FLAG_AXIPTD_MASK)  >> (VIRTVAL_FLAG_AXIPTD_SHIFT);
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"[AxiProt & Virt]: Setting bOSidAxiProt of Emulator's Trusted Device for Catbase %d to %s",
+				ui32OSidReg,
+				(ui32OSidAxiProtTD == 1)?"TRUE":"FALSE"));
+
+		bOSidAxiProtReg = ui32OSidAxiProtReg == 1;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"[AxiProt & Virt]: Setting bOSidAxiProt of FW's Register for Catbase %d to %s",
+				ui32OSidReg,
+				bOSidAxiProtReg?"TRUE":"FALSE"));
+
+		SetAxiProtOSid(psDeviceNode->psDevConfig->hSysData, ui32OSidReg, ui32OSidAxiProtTD);
+	}
+}
+#endif /* defined(EMULATOR) */
+
+	/* We now know the OSid, OSidReg and bOSidAxiProtReg setting for this
+	 * connection. We can access these from wherever we have a connection
+	 * reference and do not need to traverse an arbitrary linked-list to
+	 * obtain them. The settings are process-specific.
+	 */
+	psConnection->ui32OSid = ui32OSid;
+	psConnection->ui32OSidReg = ui32OSidReg;
+	psConnection->bOSidAxiProtReg = bOSidAxiProtReg;
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "[GPU Virtualization Validation]: OSIDs: %d, %d",
+	         ui32OSid,
+	         ui32OSidReg));
+}
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Only enabled if enabled in the UM */
+	if (!(ui32ClientBuildOptions & RGX_BUILD_OPTIONS_KM & OPTIONS_WORKLOAD_ESTIMATION_EN))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Workload Estimation disabled. Not enabled in UM",
+				__func__));
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/* Only enabled if enabled in the UM */
+	if (!(ui32ClientBuildOptions & RGX_BUILD_OPTIONS_KM & OPTIONS_PDVFS_EN))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Proactive DVFS disabled. Not enabled in UM",
+		         __func__));
+	}
+#endif
+
+	ui32DDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+	ui32DDKBuild = PVRVERSION_BUILD;
+
+	if (ui32Flags & SRV_FLAGS_CLIENT_64BIT_COMPAT)
+	{
+		psSRVData->sDriverInfo.ui8UMSupportedArch |= BUILD_ARCH_64BIT;
+	}
+	else
+	{
+		psSRVData->sDriverInfo.ui8UMSupportedArch |= BUILD_ARCH_32BIT;
+	}
+
+	if (IMG_FALSE == bIsFirstConnection)
+	{
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildOptions = (RGX_BUILD_OPTIONS_KM);
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildOptions = ui32ClientBuildOptions;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildVersion = ui32DDKVersion;
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildVersion = ui32ClientDDKVersion;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildRevision = ui32DDKBuild;
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildRevision = ui32ClientDDKBuild;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildType =
+				((RGX_BUILD_OPTIONS_KM) & OPTIONS_DEBUG_EN) ? BUILD_TYPE_DEBUG : BUILD_TYPE_RELEASE;
+
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildType =
+				(ui32ClientBuildOptions & OPTIONS_DEBUG_EN) ? BUILD_TYPE_DEBUG : BUILD_TYPE_RELEASE;
+
+		if (sizeof(void *) == POINTER_SIZE_64BIT)
+		{
+			psSRVData->sDriverInfo.ui8KMBitArch |= BUILD_ARCH_64BIT;
+		}
+		else
+		{
+			psSRVData->sDriverInfo.ui8KMBitArch |= BUILD_ARCH_32BIT;
+		}
+	}
+
+	/* Masking out every option that is not kernel specific*/
+	ui32ClientBuildOptions &= RGX_BUILD_OPTIONS_MASK_KM;
+
+	/*
+	 * Validate the build options
+	 */
+	ui32ServerBuildOptions = (RGX_BUILD_OPTIONS_KM);
+	if (ui32ServerBuildOptions != ui32ClientBuildOptions)
+	{
+		IMG_UINT32			ui32ServerBuildOptionsMismatch = ui32ServerBuildOptions ^ ui32ClientBuildOptions;
+		IMG_UINT32			ui32ClientBuildOptionsMismatch = ui32ServerBuildOptionsMismatch;
+
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask the debug flag option out as we do support combinations of debug vs release in um & km*/
+		ui32ServerBuildOptionsMismatch &= KM_OPTIONS_STRICT;
+		ui32ClientBuildOptionsMismatch &= UM_OPTIONS_STRICT;
+#endif
+		if ( (ui32ClientBuildOptions & ui32ClientBuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) %s: Mismatch in client-side and KM driver build options; "
+				"extra options present in client-side driver: (0x%x). Please check rgx_options.h",
+				__func__,
+				ui32ClientBuildOptions & ui32ClientBuildOptionsMismatch));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH, chk_exit);
+		}
+
+		if ( (ui32ServerBuildOptions & ui32ServerBuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) %s: Mismatch in client-side and KM driver build options; "
+				"extra options present in KM driver: (0x%x). Please check rgx_options.h",
+				__func__,
+				ui32ServerBuildOptions & ui32ServerBuildOptionsMismatch ));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH, chk_exit);
+		}
+		if (IMG_FALSE == bIsFirstConnection)
+		{
+			PVR_LOG(("%s: COMPAT_TEST: Client-side (0x%04x) (%s) and KM driver (0x%04x) (%s) build options differ.",
+																			__func__,
+																			ui32ClientBuildOptions,
+																			(psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildType)?"release":"debug",
+																			ui32ServerBuildOptions,
+																			(psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildType)?"release":"debug"));
+		}else{
+			PVR_DPF((PVR_DBG_WARNING, "%s: COMPAT_TEST: Client-side (0x%04x) and KM driver (0x%04x) build options differ.",
+																		__func__,
+																		ui32ClientBuildOptions,
+																		ui32ServerBuildOptions));
+
+		}
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: Client-side and KM driver build options match. [ OK ]", __func__));
+	}
+
+	/*
+	 * Validate DDK version
+	 */
+	if (ui32ClientDDKVersion != ui32DDKVersion)
+	{
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+		PVR_LOG(("(FAIL) %s: Incompatible driver DDK version (%u.%u) / client DDK version (%u.%u).",
+				__func__,
+				PVRVERSION_MAJ, PVRVERSION_MIN,
+				PVRVERSION_UNPACK_MAJ(ui32ClientDDKVersion),
+				PVRVERSION_UNPACK_MIN(ui32ClientDDKVersion)));
+		PVR_DBG_BREAK;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DDK_VERSION_MISMATCH, chk_exit);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: driver DDK version (%u.%u) and client DDK version (%u.%u) match. [ OK ]",
+				__func__,
+				PVRVERSION_MAJ, PVRVERSION_MIN, PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	/* Create stream for every connection except for the special clients
+	 * that don't need it e.g.: recipients of HWPerf data. */
+	if (!(psConnection->ui32ClientFlags & SRV_NO_HWPERF_CLIENT_STREAM))
+	{
+		IMG_CHAR acStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+		OSSNPrintf(acStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+		           PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC,
+		           psDeviceNode->sDevId.i32KernelDeviceID,
+		           psConnection->pid);
+
+		eError = TLStreamCreate(&psConnection->hClientTLStream,
+		                        acStreamName,
+		                        PVRSRV_CLIENT_TL_STREAM_SIZE_DEFAULT,
+		                        TL_OPMODE_DROP_NEWER |
+		                        TL_FLAG_ALLOCATE_ON_FIRST_OPEN,
+		                        NULL, NULL, NULL, NULL, NULL, NULL);
+		if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_ALREADY_EXISTS)
+		{
+			PVR_LOG_ERROR(eError, "TLStreamCreate");
+			psConnection->hClientTLStream = NULL;
+		}
+		else if (eError == PVRSRV_OK)
+		{
+			/* Set "tlctrl" stream as a notification channel. This channel is
+			 * is used to notify recipients about stream open/close (by writer)
+			 * actions (and possibly other actions in the future). */
+			eError = TLStreamSetNotifStream(psConnection->hClientTLStream,
+			                                psSRVData->hTLCtrlStream);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "TLStreamSetNotifStream");
+				TLStreamClose(psConnection->hClientTLStream);
+				psConnection->hClientTLStream = NULL;
+			}
+		}
+
+		/* Reset error status. We don't want to propagate any errors from here. */
+		eError = PVRSRV_OK;
+		PVR_DPF((PVR_DBG_MESSAGE, "Created stream \"%s\".", acStreamName));
+	}
+
+	/*
+	 * Validate DDK build
+	 */
+	if (ui32ClientDDKBuild != ui32DDKBuild)
+	{
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+		PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch in driver DDK revision (%d) / client DDK revision (%d).",
+				__func__, ui32DDKBuild, ui32ClientDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		PVR_DBG_BREAK;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DDK_BUILD_MISMATCH, chk_exit);
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: driver DDK revision (%d) and client DDK revision (%d) match. [ OK ]",
+				__func__, ui32DDKBuild, ui32ClientDDKBuild));
+	}
+
+#if defined(PDUMP)
+	/* Success so far so is it the PDump client that is connecting? */
+	if (ui32Flags & SRV_FLAGS_PDUMPCTRL)
+	{
+		if (psDeviceNode->sDevId.ui32InternalID == psSRVData->ui32PDumpBoundDevice)
+		{
+			PDumpConnectionNotify(psDeviceNode);
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+			PVR_DPF((PVR_DBG_ERROR, "%s: PDump requested for device %u but only permitted for device %u",
+					__func__, psDeviceNode->sDevId.ui32InternalID, psSRVData->ui32PDumpBoundDevice));
+			goto chk_exit;
+		}
+	}
+	else
+	{
+		/* Warn if the app is connecting to a device PDump won't be able to capture */
+		if (psDeviceNode->sDevId.ui32InternalID != psSRVData->ui32PDumpBoundDevice)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: NB. App running on device %d won't be captured by PDump (must be on device %u)",
+					__func__, psDeviceNode->sDevId.ui32InternalID, psSRVData->ui32PDumpBoundDevice));
+		}
+	}
+#endif
+
+	PVR_ASSERT(pui8KernelArch != NULL);
+
+	if (psSRVData->sDriverInfo.ui8KMBitArch & BUILD_ARCH_64BIT)
+	{
+		*pui8KernelArch = 64;
+	}
+	else
+	{
+		*pui8KernelArch = 32;
+	}
+
+	bIsFirstConnection = IMG_TRUE;
+
+#if defined(DEBUG_BRIDGE_KM)
+	{
+		int ii;
+
+		/* dump dispatch table offset lookup table */
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: g_BridgeDispatchTableStartOffsets[0-%lu] entries:", __func__, BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT - 1));
+		for (ii=0; ii < BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT; ii++)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "g_BridgeDispatchTableStartOffsets[%d]: %u", ii, g_BridgeDispatchTableStartOffsets[ii][PVR_DISPATCH_OFFSET_FIRST_FUNC]));
+		}
+	}
+#endif
+
+#if defined(PDUMP)
+	if (!(ui32Flags & SRV_FLAGS_PDUMPCTRL))
+	{
+		IMG_UINT64 ui64PDumpState = 0;
+
+		PDumpGetStateKM(&ui64PDumpState);
+		if (ui64PDumpState & PDUMP_STATE_CONNECTED)
+		{
+			*pui32CapabilityFlags |= PVRSRV_PDUMP_IS_RECORDING;
+		}
+	}
+#endif
+
+chk_exit:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVDisconnectKM(void)
+{
+#if defined(INTEGRITY_OS) && defined(DEBUG_BRIDGE_KM)
+	PVRSRVPrintBridgeStats();
+#endif
+	/* just return OK, per-process data is cleaned up by resmgr */
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       PVRSRVAcquireGlobalEventObjectKM
+@Description    Acquire the global event object.
+@Output         phGlobalEventObject    On success, points to the global event
+                                       object handle
+@Return         PVRSRV_ERROR           PVRSRV_OK on success or an error
+                                       otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR
+PVRSRVAcquireGlobalEventObjectKM(IMG_HANDLE *phGlobalEventObject)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	*phGlobalEventObject = psPVRSRVData->hGlobalEventObject;
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       PVRSRVReleaseGlobalEventObjectKM
+@Description    Release the global event object.
+@Output         hGlobalEventObject    Global event object handle
+@Return         PVRSRV_ERROR          PVRSRV_OK on success or an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR
+PVRSRVReleaseGlobalEventObjectKM(IMG_HANDLE hGlobalEventObject)
+{
+	PVR_ASSERT(PVRSRVGetPVRSRVData()->hGlobalEventObject == hGlobalEventObject);
+
+	return PVRSRV_OK;
+}
+
+static void _DumpDebugUMReqPrintWrapper(void *pvPriv, const IMG_CHAR *pszFmt, ...)
+{
+	va_list pvArgs;
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+	va_start(pvArgs, pszFmt);
+	PVRSRVReleasePrintfVArgs(pszFmt, pvArgs);
+	va_end(pvArgs);
+}
+
+/*
+	PVRSRVDumpDebugInfoKM
+*/
+PVRSRV_ERROR
+PVRSRVDumpDebugInfoKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDeviceNode,
+					  IMG_UINT32 ui32VerbLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32VerbLevel > DEBUG_REQUEST_VERBOSITY_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	PVR_LOG(("User requested PVR debug info"));
+
+	PVRSRVDebugRequest(psDeviceNode, ui32VerbLevel, _DumpDebugUMReqPrintWrapper, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVGetDevClockSpeedKM
+*/
+PVRSRV_ERROR
+PVRSRVGetDevClockSpeedKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_PUINT32  pui32RGXClockSpeed)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVR_ASSERT(psDeviceNode->pfnDeviceClockSpeed != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = psDeviceNode->pfnDeviceClockSpeed(psDeviceNode, pui32RGXClockSpeed);
+	PVR_WARN_IF_ERROR(eError, "pfnDeviceClockSpeed");
+
+	return eError;
+}
+
+
+/*
+	PVRSRVHWOpTimeoutKM
+*/
+PVRSRV_ERROR
+PVRSRVHWOpTimeoutKM(CONNECTION_DATA *psConnection,
+					PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(PVRSRV_RESET_ON_HWTIMEOUT)
+	PVR_LOG(("User requested OS reset"));
+	OSPanic();
+#endif
+	PVR_LOG(("HW operation timeout, dump server info"));
+	PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	return PVRSRV_OK;
+}
+
+
+IMG_INT
+DummyBW(IMG_UINT32 ui32DispatchTableEntry,
+		IMG_UINT8 *psBridgeIn,
+		IMG_UINT8 *psBridgeOut,
+		CONNECTION_DATA *psConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(psBridgeIn);
+	PVR_UNREFERENCED_PARAMETER(psBridgeOut);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(DEBUG_BRIDGE_KM)
+	PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE ERROR: ui32DispatchTableEntry %u (%s) mapped to "
+			 "Dummy Wrapper (probably not what you want!)",
+			 __func__, ui32DispatchTableEntry, g_BridgeDispatchTable[ui32DispatchTableEntry].pszIOCName));
+#else
+	PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE ERROR: ui32DispatchTableEntry %u mapped to "
+			 "Dummy Wrapper (probably not what you want!)",
+			 __func__, ui32DispatchTableEntry));
+#endif
+	return PVRSRV_ERROR_BRIDGE_ENOTTY;
+}
+
+PVRSRV_ERROR PVRSRVAlignmentCheckKM(CONNECTION_DATA *psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32AlignChecksSize,
+                                    IMG_UINT32 aui32AlignChecks[])
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if !defined(NO_HARDWARE)
+
+	PVR_ASSERT(psDeviceNode->pfnAlignmentCheck != NULL);
+	return psDeviceNode->pfnAlignmentCheck(psDeviceNode, ui32AlignChecksSize,
+	                                       aui32AlignChecks);
+
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32AlignChecksSize);
+	PVR_UNREFERENCED_PARAMETER(aui32AlignChecks);
+
+	return PVRSRV_OK;
+
+#endif /* !defined(NO_HARDWARE) */
+
+}
+
+PVRSRV_ERROR PVRSRVGetDeviceStatusKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 *pui32DeviceStatus)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* First try to update the status. */
+	if (psDeviceNode->pfnUpdateHealthStatus != NULL)
+	{
+		PVRSRV_ERROR eError = psDeviceNode->pfnUpdateHealthStatus(psDeviceNode,
+		                                                          IMG_FALSE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "PVRSRVGetDeviceStatusKM: Failed to "
+					 "check for device status (%d)", eError));
+
+			/* Return unknown status and error because we don't know what
+			 * happened and if the status is valid. */
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_UNKNOWN;
+			return eError;
+		}
+	}
+
+	switch (OSAtomicRead(&psDeviceNode->eHealthStatus))
+	{
+		case PVRSRV_DEVICE_HEALTH_STATUS_OK:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_OK;
+			return PVRSRV_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_NOT_RESPONDING;
+			return PVRSRV_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:
+		case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:
+		case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_DEVICE_ERROR;
+			return PVRSRV_OK;
+		default:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_UNKNOWN;
+			return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+}
+
+PVRSRV_ERROR PVRSRVGetMultiCoreInfoKM(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32CapsSize,
+                                      IMG_UINT32 *pui32NumCores,
+                                      IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32CapsSize > 0)
+	{
+		/* Clear the buffer to ensure no uninitialised data is returned to UM
+		 * if the pfn call below does not write to the whole array, or is null.
+		 */
+		memset(pui64Caps, 0x00, (ui32CapsSize * sizeof(IMG_UINT64)));
+	}
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		eError = psDeviceNode->pfnGetMultiCoreInfo(psDeviceNode, ui32CapsSize, pui32NumCores, pui64Caps);
+	}
+	return eError;
+}
+
+
+/*!
+ * *****************************************************************************
+ * @brief A wrapper for removing entries in the g_BridgeDispatchTable array.
+ *		  All this does is zero the entry to allow for a full table re-population
+ *		  later.
+ *
+ * @param ui32BridgeGroup
+ * @param ui32Index
+ *
+ * @return
+ ********************************************************************************/
+void
+UnsetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup, IMG_UINT32 ui32Index)
+{
+	ui32Index += g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+
+	g_BridgeDispatchTable[ui32Index].pfFunction = NULL;
+	g_BridgeDispatchTable[ui32Index].hBridgeLock = NULL;
+#if defined(DEBUG_BRIDGE_KM)
+	g_BridgeDispatchTable[ui32Index].pszIOCName = NULL;
+	g_BridgeDispatchTable[ui32Index].pszFunctionName = NULL;
+	g_BridgeDispatchTable[ui32Index].pszBridgeLockName = NULL;
+	g_BridgeDispatchTable[ui32Index].ui32CallCount = 0;
+	g_BridgeDispatchTable[ui32Index].ui32CopyFromUserTotalBytes = 0;
+	g_BridgeDispatchTable[ui32Index].ui64TotalTimeNS = 0;
+	g_BridgeDispatchTable[ui32Index].ui64MaxTimeNS = 0;
+#endif
+}
+
+/*!
+ * *****************************************************************************
+ * @brief A wrapper for filling in the g_BridgeDispatchTable array that does
+ *		  error checking.
+ *
+ * @param ui32Index
+ * @param pszIOCName
+ * @param pfFunction
+ * @param pszFunctionName
+ * @param hBridgeLock
+ * @param pszBridgeLockName
+ * @param ui32InBufferSize
+ * @param ui32OutBufferSize
+ *
+ * @return
+ ********************************************************************************/
+void
+_SetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+					   IMG_UINT32 ui32Index,
+					   const IMG_CHAR *pszIOCName,
+					   BridgeWrapperFunction pfFunction,
+					   const IMG_CHAR *pszFunctionName,
+					   POS_LOCK hBridgeLock,
+					   const IMG_CHAR *pszBridgeLockName,
+					   IMG_UINT32 ui32InBufferSize,
+					   IMG_UINT32 ui32OutBufferSize)
+{
+	static IMG_UINT32 ui32PrevIndex = IMG_UINT32_MAX;		/* -1 */
+
+#if !defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE) && !defined(DEBUG_BRIDGE_KM)
+	PVR_UNREFERENCED_PARAMETER(pszFunctionName);
+	PVR_UNREFERENCED_PARAMETER(pszBridgeLockName);
+#endif
+
+	ui32Index += g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+	/* Enable this to dump out the dispatch table entries */
+	PVR_DPF((PVR_DBG_WARNING, "%s: g_BridgeDispatchTableStartOffsets[%d]=%d", __func__, ui32BridgeGroup, g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC]));
+	PVR_DPF((PVR_DBG_WARNING, "%s: %d %s %s %s", __func__, ui32Index, pszIOCName, pszFunctionName, pszBridgeLockName));
+#endif
+
+	/* Any gaps are sub-optimal in-terms of memory usage, but we are mainly
+	 * interested in spotting any large gap of wasted memory that could be
+	 * accidentally introduced.
+	 *
+	 * This will currently flag up any gaps > 5 entries.
+	 *
+	 * NOTE: This shouldn't be debug only since switching from debug->release
+	 * etc is likely to modify the available ioctls and thus be a point where
+	 * mistakes are exposed. This isn't run at a performance critical time.
+	 */
+	if ((ui32PrevIndex != IMG_UINT32_MAX) &&
+		((ui32Index >= ui32PrevIndex + DISPATCH_TABLE_GAP_THRESHOLD) ||
+		 (ui32Index <= ui32PrevIndex)))
+	{
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: There is a gap in the dispatch table between indices %u (%s) and %u (%s)",
+				 __func__, ui32PrevIndex, g_BridgeDispatchTable[ui32PrevIndex].pszIOCName,
+				 ui32Index, pszIOCName));
+#else
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: There is a gap in the dispatch table between indices %u and %u (%s)",
+				 __func__, (IMG_UINT)ui32PrevIndex, (IMG_UINT)ui32Index, pszIOCName));
+#endif
+	}
+
+	if (ui32Index >= BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Index %u (%s) out of range",
+				 __func__, (IMG_UINT)ui32Index, pszIOCName));
+
+#if defined(DEBUG_BRIDGE_KM)
+		PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE_DISPATCH_TABLE_ENTRY_COUNT = %lu",
+				 __func__, BRIDGE_DISPATCH_TABLE_ENTRY_COUNT));
+#if defined(SUPPORT_RGX)
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST));
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGX_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGX_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGX_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGX_LAST));
+#endif
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST));
+#endif
+
+		OSPanic();
+	}
+
+	/* Panic if the previous entry has been overwritten as this is not allowed!
+	 * NOTE: This shouldn't be debug only since switching from debug->release
+	 * etc is likely to modify the available ioctls and thus be a point where
+	 * mistakes are exposed. This isn't run at a performance critical time.
+	 */
+	if (g_BridgeDispatchTable[ui32Index].pfFunction)
+	{
+		if (g_BridgeDispatchTable[ui32Index].pfFunction != pfFunction)
+		{
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+			PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Adding dispatch table entry for %s clobbers an existing entry for %s (current pfn=<%p>, new pfn=<%p>)",
+				 __func__, pszIOCName, g_BridgeDispatchTable[ui32Index].pszIOCName,
+				 (void*)g_BridgeDispatchTable[ui32Index].pfFunction, (void*)pfFunction));
+#else
+			PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Adding dispatch table entry for %s clobbers an existing entry (index=%u). (current pfn=<%p>, new pfn=<%p>)",
+				 __func__, pszIOCName, ui32Index,
+				 (void*)g_BridgeDispatchTable[ui32Index].pfFunction, (void*)pfFunction));
+			PVR_DPF((PVR_DBG_WARNING, "NOTE: Enabling DEBUG_BRIDGE_KM_DISPATCH_TABLE may help debug this issue."));
+#endif
+			OSPanic();
+		}
+	}
+	else
+	{
+		g_BridgeDispatchTable[ui32Index].pfFunction = pfFunction;
+		g_BridgeDispatchTable[ui32Index].hBridgeLock = hBridgeLock;
+		g_BridgeDispatchTable[ui32Index].ui32InBufferSize = ui32InBufferSize;
+		g_BridgeDispatchTable[ui32Index].ui32OutBufferSize = ui32OutBufferSize;
+#if defined(DEBUG_BRIDGE_KM)
+		g_BridgeDispatchTable[ui32Index].pszIOCName = pszIOCName;
+		g_BridgeDispatchTable[ui32Index].pszFunctionName = pszFunctionName;
+		g_BridgeDispatchTable[ui32Index].pszBridgeLockName = pszBridgeLockName;
+		g_BridgeDispatchTable[ui32Index].ui32CallCount = 0;
+		g_BridgeDispatchTable[ui32Index].ui32CopyFromUserTotalBytes = 0;
+		g_BridgeDispatchTable[ui32Index].ui64TotalTimeNS = 0;
+		g_BridgeDispatchTable[ui32Index].ui64MaxTimeNS = 0;
+#endif
+	}
+
+	ui32PrevIndex = ui32Index;
+}
+
+static PVRSRV_ERROR _BridgeBufferAlloc(void *pvPrivData, void **pvOut)
+{
+	PVR_UNREFERENCED_PARAMETER(pvPrivData);
+
+	*pvOut = OSAllocZMem(PVRSRV_MAX_BRIDGE_IN_SIZE +
+	                     PVRSRV_MAX_BRIDGE_OUT_SIZE);
+	PVR_RETURN_IF_NOMEM(*pvOut);
+
+	return PVRSRV_OK;
+}
+
+static void _BridgeBufferFree(void *pvPrivData, void *pvFreeData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvPrivData);
+
+	OSFreeMem(pvFreeData);
+}
+
+PVRSRV_ERROR BridgeDispatcherInit(void)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(DEBUG_BRIDGE_KM)
+	eError = OSLockCreate(&g_hStatsLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", errorLockCreateFailed);
+#endif
+
+	eError = PVRSRVPoolCreate(_BridgeBufferAlloc,
+	                          _BridgeBufferFree,
+	                          PVRSRV_MAX_POOLED_BRIDGE_BUFFERS,
+	                          "Bridge buffer pool",
+	                          NULL,
+	                          &g_psBridgeBufferPool);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPoolCreate", erroPoolCreateFailed);
+
+	return PVRSRV_OK;
+
+erroPoolCreateFailed:
+#if defined(DEBUG_BRIDGE_KM)
+	OSLockDestroy(g_hStatsLock);
+	g_hStatsLock = NULL;
+errorLockCreateFailed:
+#endif
+	return eError;
+}
+
+void BridgeDispatcherDeinit(void)
+{
+	if (g_psBridgeBufferPool)
+	{
+		PVRSRVPoolDestroy(g_psBridgeBufferPool);
+		g_psBridgeBufferPool = NULL;
+	}
+
+#if defined(DEBUG_BRIDGE_KM)
+	if (g_hStatsLock)
+	{
+		OSLockDestroy(g_hStatsLock);
+		g_hStatsLock = NULL;
+	}
+#endif
+}
+
+PVRSRV_ERROR BridgedDispatchKM(CONNECTION_DATA * psConnection,
+                          PVRSRV_BRIDGE_PACKAGE   * psBridgePackageKM)
+{
+
+	void       * psBridgeIn=NULL;
+	void       * psBridgeOut=NULL;
+	BridgeWrapperFunction pfBridgeHandler;
+	IMG_UINT32   ui32DispatchTableEntry, ui32GroupBoundary;
+	PVRSRV_ERROR err = PVRSRV_OK;
+#if !defined(INTEGRITY_OS)
+	PVRSRV_POOL_TOKEN hBridgeBufferPoolToken = NULL;
+#endif
+	IMG_UINT32 ui32Timestamp = OSClockus();
+#if defined(DEBUG_BRIDGE_KM)
+	IMG_UINT64	ui64TimeStart;
+	IMG_UINT64	ui64TimeEnd;
+	IMG_UINT64	ui64TimeDiff;
+#endif
+	IMG_UINT32	ui32DispatchTableIndex, ui32DispatchTableEntryIndex;
+
+#if defined(DEBUG_BRIDGE_KM_STOP_AT_DISPATCH)
+	PVR_DBG_BREAK;
+#endif
+
+#if !defined(PVRSRV_ENABLE_HTB)
+	PVR_UNREFERENCED_PARAMETER(ui32Timestamp);
+#endif
+
+	if (psBridgePackageKM->ui32BridgeID >= BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Out of range dispatch table group ID: %d",
+		        __func__, psBridgePackageKM->ui32BridgeID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+	ui32DispatchTableIndex = OSConfineArrayIndexNoSpeculation(psBridgePackageKM->ui32BridgeID, BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT);
+
+	ui32DispatchTableEntry = g_BridgeDispatchTableStartOffsets[ui32DispatchTableIndex][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+	ui32GroupBoundary = g_BridgeDispatchTableStartOffsets[ui32DispatchTableIndex][PVR_DISPATCH_OFFSET_LAST_FUNC];
+
+	/* bridge function is not implemented in this build */
+	if (0 == ui32DispatchTableEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Dispatch table entry=%d, boundary = %d, (bridge module %d, function %d)",
+		         __func__,
+		         ui32DispatchTableEntry,
+		         ui32GroupBoundary,
+		         psBridgePackageKM->ui32BridgeID,
+		         psBridgePackageKM->ui32FunctionID));
+		/* this points to DummyBW() which returns PVRSRV_ERROR_ENOTTY */
+		err = g_BridgeDispatchTable[ui32DispatchTableEntry].pfFunction(ui32DispatchTableEntry,
+				  psBridgeIn,
+				  psBridgeOut,
+				  psConnection);
+		goto return_error;
+	}
+	if ((ui32DispatchTableEntry + psBridgePackageKM->ui32FunctionID) > ui32GroupBoundary)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Dispatch table entry=%d, boundary = %d, (bridge module %d, function %d)",
+		         __func__,
+		         ui32DispatchTableEntry,
+		         ui32GroupBoundary,
+		         psBridgePackageKM->ui32BridgeID,
+		         psBridgePackageKM->ui32FunctionID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+	ui32DispatchTableEntry += psBridgePackageKM->ui32FunctionID;
+	ui32DispatchTableEntryIndex = OSConfineArrayIndexNoSpeculation(ui32DispatchTableEntry, ui32GroupBoundary+1);
+	if (BRIDGE_DISPATCH_TABLE_ENTRY_COUNT <= ui32DispatchTableEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dispatch table entry=%d, entry count = %lu,"
+		        " (bridge module %d, function %d)", __func__,
+		        ui32DispatchTableEntry, BRIDGE_DISPATCH_TABLE_ENTRY_COUNT,
+		        psBridgePackageKM->ui32BridgeID,
+		        psBridgePackageKM->ui32FunctionID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+#if defined(DEBUG_BRIDGE_KM)
+	BridgeGlobalStatsLock();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s idx:%d mod:%d, func:%d",
+			__func__, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].pszIOCName,
+			ui32DispatchTableEntryIndex, psBridgePackageKM->ui32BridgeID, psBridgePackageKM->ui32FunctionID));
+
+	g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32CallCount++;
+	g_BridgeGlobalStats.ui32IOCTLCount++;
+	BridgeGlobalStatsUnlock();
+#endif
+
+	if (psBridgePackageKM->ui32InBufferSize != g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32InBufferSize ||
+	    psBridgePackageKM->ui32OutBufferSize != g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32OutBufferSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Bridge buffer sizes mismatch! "
+		        "In: User(%u), Kernel(%u) - Out: User(%u), Kernel(%u)",
+		        __func__,
+		        psBridgePackageKM->ui32InBufferSize, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32InBufferSize,
+		        psBridgePackageKM->ui32OutBufferSize, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32OutBufferSize));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+	if (g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock != NULL)
+	{
+		OSLockAcquire(g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock);
+	}
+#if !defined(INTEGRITY_OS)
+	/* try to acquire a bridge buffer from the pool */
+
+	err = PVRSRVPoolGet(g_psBridgeBufferPool,
+			&hBridgeBufferPoolToken,
+			&psBridgeIn);
+	PVR_LOG_GOTO_IF_ERROR(err, "PVRSRVPoolGet", unlock_and_return_error);
+
+	psBridgeOut = ((IMG_BYTE *) psBridgeIn) + PVRSRV_MAX_BRIDGE_IN_SIZE;
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+	ui64TimeStart = OSClockns64();
+#endif
+
+#if !defined(INTEGRITY_OS)
+	if ((CopyFromUserWrapper (psConnection,
+							  ui32DispatchTableEntryIndex,
+							  psBridgeIn,
+							  psBridgePackageKM->pvParamIn,
+							  psBridgePackageKM->ui32InBufferSize) != PVRSRV_OK)
+#if defined(__QNXNTO__)
+/* For Neutrino, the output bridge buffer acts as an input as well */
+					|| (CopyFromUserWrapper(psConnection,
+											ui32DispatchTableEntryIndex,
+											psBridgeOut,
+											(void *)((uintptr_t)psBridgePackageKM->pvParamIn + psBridgePackageKM->ui32InBufferSize),
+											psBridgePackageKM->ui32OutBufferSize) != PVRSRV_OK)
+#endif
+		) /* end of if-condition */
+	{
+		PVR_LOG_GOTO_WITH_ERROR("CopyFromUserWrapper", err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+	}
+#else
+	psBridgeIn = psBridgePackageKM->pvParamIn;
+	psBridgeOut = psBridgePackageKM->pvParamOut;
+#endif
+
+	pfBridgeHandler =
+		(BridgeWrapperFunction)g_BridgeDispatchTable[ui32DispatchTableEntryIndex].pfFunction;
+
+	if (pfBridgeHandler == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui32DispatchTableEntry = %d is not a registered function!",
+				 __func__, ui32DispatchTableEntry));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+	}
+
+	/* pfBridgeHandler functions do not fail and return an IMG_INT.
+	 * The value returned is either 0 or PVRSRV_OK (0).
+	 * In the event this changes an error may be +ve or -ve,
+	 * so try to return something consistent here.
+	 */
+	if (0 != pfBridgeHandler(ui32DispatchTableEntryIndex,
+						  psBridgeIn,
+						  psBridgeOut,
+						  psConnection)
+		)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("pfBridgeHandler", err, PVRSRV_ERROR_BRIDGE_EPERM, unlock_and_return_error);
+	}
+
+	/*
+	   This should always be true as a.t.m. all bridge calls have to
+	   return an error message, but this could change so we do this
+	   check to be safe.
+	*/
+#if !defined(INTEGRITY_OS)
+	if (psBridgePackageKM->ui32OutBufferSize > 0)
+	{
+		if (CopyToUserWrapper (psConnection,
+						ui32DispatchTableEntryIndex,
+						psBridgePackageKM->pvParamOut,
+						psBridgeOut,
+						psBridgePackageKM->ui32OutBufferSize) != PVRSRV_OK)
+		{
+			PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+		}
+	}
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+	ui64TimeEnd = OSClockns64();
+
+	ui64TimeDiff = ui64TimeEnd - ui64TimeStart;
+
+	BridgeGlobalStatsLock();
+
+	g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64TotalTimeNS += ui64TimeDiff;
+
+	if (ui64TimeDiff > g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64MaxTimeNS)
+	{
+		g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64MaxTimeNS = ui64TimeDiff;
+	}
+
+	BridgeGlobalStatsUnlock();
+#endif
+
+unlock_and_return_error:
+
+	if (g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock != NULL)
+	{
+		OSLockRelease(g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock);
+	}
+
+#if !defined(INTEGRITY_OS)
+	if (hBridgeBufferPoolToken != NULL)
+	{
+		err = PVRSRVPoolPut(g_psBridgeBufferPool,
+				hBridgeBufferPoolToken);
+		PVR_LOG_IF_ERROR(err, "PVRSRVPoolPut");
+	}
+#endif
+
+return_error:
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: returning (err = %d)", __func__, err));
+	}
+	/* ignore transport layer bridge to avoid HTB flooding */
+	if (psBridgePackageKM->ui32BridgeID != PVRSRV_BRIDGE_PVRTL)
+	{
+		if (err)
+		{
+			HTBLOGK(HTB_SF_BRG_BRIDGE_CALL_ERR, ui32Timestamp,
+			        psBridgePackageKM->ui32BridgeID,
+			        psBridgePackageKM->ui32FunctionID, err);
+		}
+		else
+		{
+			HTBLOGK(HTB_SF_BRG_BRIDGE_CALL, ui32Timestamp,
+			        psBridgePackageKM->ui32BridgeID,
+			        psBridgePackageKM->ui32FunctionID);
+		}
+	}
+
+	return err;
+}
+
+PVRSRV_ERROR PVRSRVFindProcessMemStatsKM(IMG_PID pid, IMG_UINT32 ui32ArrSize, IMG_BOOL bAllProcessStats, IMG_UINT64 *pui64MemStatArray)
+{
+#if !defined(__QNXNTO__)
+	return PVRSRVFindProcessMemStats(pid,
+					ui32ArrSize,
+					bAllProcessStats,
+					pui64MemStatArray);
+#else
+	PVR_DPF((PVR_DBG_ERROR, "This functionality is not yet implemented for this platform"));
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+
+}
+
+void PVRSRVBlockIfFrozen(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Short-circuit if we're not marked as frozen */
+	if (OSAtomicRead(&psDevNode->eFrozen) == 0)
+	{
+		return;
+	}
+
+	while ((psDevNode->eDevState == PVRSRV_DEVICE_STATE_FROZEN) &&
+	       ((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_TIMEOUT)))
+	{
+		IMG_HANDLE hEvent;
+
+		eError = OSEventObjectOpen(psDevNode->hDeviceThreadEvObj, &hEvent);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Failed to open event object (%d)", __func__,
+			        eError));
+
+			/* Continue with the loop by resetting to PVRSRV_OK. This makes it
+			 * a busy wait (with a 10ms delay) if we cannot grab an EvObj
+			 * reference.
+			 */
+			eError = PVRSRV_OK;
+			OSSleepms(10U);		/* Allow failure (OOM etc.) to resolve */
+			continue;
+		}
+
+		if (hEvent != NULL)
+		{
+			/* Register that we've got an interest in the device */
+			(void) OSAtomicIncrement(&psDevNode->iFreezeCount);
+
+			eError = OSEventObjectWait(hEvent);
+
+			OSEventObjectClose(hEvent);
+
+			(void) OSAtomicDecrement(&psDevNode->iFreezeCount);
+		}
+	}
+
+	/* We expect to exit the above loop only when the device is no longer
+	 * FROZEN. If we are still marked as frozen that is an unexpected error
+	 * so log it.
+	 */
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_FROZEN)
+	{
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectWait");
+		if (eError != PVRSRV_OK)
+		{
+			OSDumpStack();
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_checkpoint.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_checkpoint.c
new file mode 100644
index 000000000000..a72ec7dc1809
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_checkpoint.c
@@ -0,0 +1,3454 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation checkpoint interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server side code for services synchronisation interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "dllist.h"
+#include "sync.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+#include "sync_checkpoint_init.h"
+#include "lock.h"
+#include "log2.h"
+#include "pvrsrv.h"
+#include "pdump_km.h"
+#include "info_page.h"
+#include "os_apphint.h"
+#include "rgxfwutils.h"
+
+#include "pvrsrv_sync_km.h"
+#include "rgxhwperf.h"
+
+
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/* Enable this to turn on debug relating to the creation and
+   resolution of contexts */
+#define ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG 0
+
+/* Enable this to turn on debug relating to the creation and
+   resolution of fences */
+#define ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint
+   allocation and freeing */
+#define ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint
+   enqueuing and signalling */
+#define ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint pool */
+#define ENABLE_SYNC_CHECKPOINT_POOL_DEBUG 0
+
+/* Enable this to turn on debug relating to sync checkpoint UFO
+   lookup */
+#define ENABLE_SYNC_CHECKPOINT_UFO_DEBUG 0
+
+/* Enable this to turn on sync checkpoint deferred cleanup debug
+ * (for syncs we have been told to free but which have some
+ * outstanding FW operations remaining (enqueued in CCBs)
+ */
+#define ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG 0
+
+#else
+
+#define ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_POOL_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_UFO_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG 0
+
+#endif
+
+/* Maximum number of deferred sync checkpoint signal/error received for atomic context */
+#define SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL 500
+
+/* Set the size of the sync checkpoint pool (not used if 0).
+ * A pool will be maintained for each sync checkpoint context.
+ * SYNC_CHECKPOINT_POOL_LIMIT must be a power of 2 (POT),
+ * as the pool wrap mask is calculated using it.
+ */
+#define SYNC_CHECKPOINT_POOL_LIMIT	1024
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT & (SYNC_CHECKPOINT_POOL_LIMIT - 1)) != 0
+#error "SYNC_CHECKPOINT_POOL_LIMIT must be power of 2."
+#endif
+
+/*
+	This defines the maximum amount of synchronisation memory
+	that can be allocated per sync checkpoint context.
+	In reality this number is meaningless as we would run out
+	of synchronisation memory before we reach this limit, but
+	we need to provide a size to the span RA.
+ */
+#define MAX_SYNC_CHECKPOINT_MEM  (4 * 1024 * 1024)
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+/* Flags used to indicate state of pool */
+#define SYNC_CHECKPOINT_POOL_FULL  (1)
+#define SYNC_CHECKPOINT_POOL_VALID (1 << 7)
+#define CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags & SYNC_CHECKPOINT_POOL_FULL
+#define CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags & SYNC_CHECKPOINT_POOL_VALID
+#define SET_CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags |= SYNC_CHECKPOINT_POOL_FULL
+#define SET_CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags |= SYNC_CHECKPOINT_POOL_VALID
+#define CLEAR_CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags &= ~SYNC_CHECKPOINT_POOL_FULL
+#define CLEAR_CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags &= ~SYNC_CHECKPOINT_POOL_VALID
+#endif
+struct _SYNC_CHECKPOINT_CONTEXT_CTL_
+{
+	SHARED_DEV_CONNECTION					psDeviceNode;
+	/*
+	 *  Used as head of linked-list of sync checkpoints for which
+	 *  SyncCheckpointFree() has been called, but have outstanding
+	 *  FW operations (enqueued in CCBs)
+	 *  This list will be check whenever a SyncCheckpointFree() is
+	 *  called, and when SyncCheckpointContextDestroy() is called.
+	 */
+	DLLIST_NODE								sDeferredCleanupListHead;
+	/* Lock to protect the deferred cleanup list */
+	POS_SPINLOCK							hDeferredCleanupListLock;
+
+	/* Counters to provide stats for number of checkpoints used at any one time */
+	IMG_UINT32                              ui32CurrentInUseSyncCheckpoints;
+	IMG_UINT32                              ui32MaxInUseSyncCheckpoints;
+	IMG_UINT32                              ui32CurrentInUseMirroredSyncCPs;
+	IMG_UINT32                              ui32MaxInUseMirroredSyncCPs;
+	/* Lock to protect the checkpoint stats */
+	POS_SPINLOCK							hSyncCheckpointStatsLock;
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	IMG_UINT32								ui32SyncCheckpointPoolSize;    /*! Allocated size of the pool */
+	IMG_UINT32								ui32SyncCheckpointPoolCount;   /*! Number of checkpoints currently in pool */
+	IMG_UINT32								ui32SyncCheckpointPoolWp;      /*! Pool write pointer */
+	IMG_UINT32								ui32SyncCheckpointPoolRp;      /*! Pool read pointer */
+#if defined(PDUMP)
+	SYNC_CHECKPOINT_BLOCK					*psSyncCheckpointPoolBlock;    /*! Block that all pool checkpoints come from */
+	IMG_BOOL								bAllocateFromCheckpointPool;   /*! PDumps only use the pool once it has allocated all its checkpoints */
+#endif
+	POS_SPINLOCK							hSyncCheckpointPoolLock;       /*! Lock to protect access to pool control data */
+	IMG_UINT8								ui8PoolStateFlags;             /*! Flags to indicate state of pool */
+	/*! Array of SYNC_CHECKPOINTs. Must be last member in structure */
+	SYNC_CHECKPOINT *apsSyncCheckpointPool[IMG_FLEX_ARRAY_MEMBER];   /*! The allocated checkpoint pool */
+#endif
+}; /*_SYNC_CHECKPOINT_CONTEXT_CTL is already typedef-ed in sync_checkpoint_internal.h */
+
+struct SYNC_CHECKPOINT_CONTEXT_TAG
+{
+#if defined(PDUMP)
+	DLLIST_NODE						sSyncCheckpointBlockListHead;          /*!< List head for the sync chkpt blocks in this context*/
+	DLLIST_NODE						sListNode;				               /*!< List node for the sync chkpt context list*/
+	POS_LOCK						hSyncCheckpointBlockListLock;          /*!< sync chkpt blocks list lock*/
+#endif
+	RA_ARENA						*psSpanRA;                             /*!< RA used for span management of SubAllocRA */
+	RA_ARENA						*psSubAllocRA;                         /*!< RA context */
+	_PSYNC_CHECKPOINT_CONTEXT_CTL	psContextCtl;
+	ATOMIC_T						hCheckpointCount;                      /*!< Checkpoint count for this context */
+	ATOMIC_T						hRefCount;                             /*!< Ref count for this context */
+}; /*_SYNC_CHECKPOINT_CONTEXT is already typedef-ed in sync_checkpoint_internal.h */
+
+/* this is the max number of sync checkpoint records we will search or dump
+ * at any time.
+ */
+#define SYNC_CHECKPOINT_RECORD_LIMIT 20000
+
+#define DECREMENT_WITH_WRAP(value, sz) ((value) ? ((value) - 1) : ((sz) - 1))
+
+struct SYNC_CHECKPOINT_RECORD
+{
+	SYNC_CHECKPOINT_BLOCK	*psSyncCheckpointBlock;	/*!< handle to SYNC_CHECKPOINT_BLOCK */
+	IMG_UINT32				ui32SyncOffset;			/*!< offset to sync in block */
+	IMG_UINT32				ui32FwBlockAddr;
+	IMG_PID					uiPID;
+	IMG_UINT32				ui32UID;
+	IMG_UINT64				ui64OSTime;
+	DLLIST_NODE				sNode;
+	IMG_CHAR				szClassName[PVRSRV_SYNC_NAME_LENGTH];
+	PSYNC_CHECKPOINT		pSyncCheckpt;
+};
+
+static PFN_SYNC_CHECKPOINT_STRUCT *g_psSyncCheckpointPfnStruct = NULL;
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static SYNC_CHECKPOINT *_GetCheckpointFromPool(_SYNC_CHECKPOINT_CONTEXT *psContext);
+static IMG_BOOL _PutCheckpointInPool(SYNC_CHECKPOINT *psSyncCheckpoint);
+static IMG_UINT32 _CleanCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext);
+#endif
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+static IMG_UINT32 gui32NumSyncCheckpointContexts = 0;
+#endif
+
+/* Defined values to indicate status of sync checkpoint, which is
+ * stored in the memory of the structure */
+#define SYNC_CHECKPOINT_PATTERN_IN_USE 0x1a1aa
+#define SYNC_CHECKPOINT_PATTERN_IN_POOL 0x2b2bb
+#define SYNC_CHECKPOINT_PATTERN_FREED 0x3c3cc
+
+#if defined(SUPPORT_RGX)
+static inline void RGXSRVHWPerfSyncCheckpointUFOIsSignalled(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               SYNC_CHECKPOINT *psSyncCheckpointInt, IMG_UINT32 ui32FenceSyncFlags)
+{
+	if (RGXHWPerfHostIsEventEnabled(psDevInfo, RGX_HWPERF_HOST_UFO)
+	    && !(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		RGX_HWPERF_UFO_EV eEv;
+		RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+		if (psSyncCheckpointInt)
+		{
+			IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+			if ((ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ||
+				(ui32State == PVRSRV_SYNC_CHECKPOINT_ERRORED))
+			{
+				sSyncData.sCheckSuccess.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+				sSyncData.sCheckSuccess.ui32Value = ui32State;
+				eEv = RGX_HWPERF_UFO_EV_CHECK_SUCCESS;
+			}
+			else
+			{
+				sSyncData.sCheckFail.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+				sSyncData.sCheckFail.ui32Value = ui32State;
+				sSyncData.sCheckFail.ui32Required = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				eEv = RGX_HWPERF_UFO_EV_CHECK_FAIL;
+			}
+			RGXHWPerfHostPostUfoEvent(psDevInfo, eEv, &sSyncData,
+			    (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+		}
+	}
+}
+
+static inline void RGXSRVHWPerfSyncCheckpointUFOUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               SYNC_CHECKPOINT *psSyncCheckpointInt, IMG_UINT32 ui32FenceSyncFlags)
+{
+	if (RGXHWPerfHostIsEventEnabled(psDevInfo, RGX_HWPERF_HOST_UFO)
+	    && !(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+		if (psSyncCheckpointInt)
+		{
+			sSyncData.sUpdate.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+			sSyncData.sUpdate.ui32OldValue = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+			sSyncData.sUpdate.ui32NewValue = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+			RGXHWPerfHostPostUfoEvent(psDevInfo, RGX_HWPERF_UFO_EV_UPDATE, &sSyncData,
+			    (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+		}
+	}
+}
+#endif
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordAdd(PSYNC_CHECKPOINT_RECORD_HANDLE *phRecord,
+	                    SYNC_CHECKPOINT_BLOCK *hSyncCheckpointBlock,
+	                    IMG_UINT32 ui32FwBlockAddr,
+	                    IMG_UINT32 ui32SyncOffset,
+	                    IMG_UINT32 ui32UID,
+	                    IMG_UINT32 ui32ClassNameSize,
+	                    const IMG_CHAR *pszClassName, PSYNC_CHECKPOINT pSyncCheckpt);
+static PVRSRV_ERROR
+_SyncCheckpointRecordRemove(PSYNC_CHECKPOINT_RECORD_HANDLE hRecord);
+static void _SyncCheckpointState(PDLLIST_NODE psNode,
+                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile);
+static void _SyncCheckpointDebugRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                        IMG_UINT32 ui32VerbLevel,
+                                        DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                        void *pvDumpDebugFile);
+static PVRSRV_ERROR _SyncCheckpointRecordListInit(PVRSRV_DEVICE_NODE *psDevNode);
+static void _SyncCheckpointRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+#if defined(PDUMP)
+static void
+MISRHandler_PdumpDeferredSyncSignalPoster(void *pvData);
+static PVRSRV_ERROR _SyncCheckpointAllocPDump(PVRSRV_DEVICE_NODE *psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint);
+static PVRSRV_ERROR _SyncCheckpointUpdatePDump(PPVRSRV_DEVICE_NODE psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint, IMG_UINT32 ui32Status, IMG_UINT32 ui32FenceSyncFlags);
+static PVRSRV_ERROR _SyncCheckpointPDumpTransition(void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+#endif
+
+/* Unique incremental ID assigned to sync checkpoints when allocated */
+static IMG_UINT32 g_SyncCheckpointUID;
+
+static void _CheckDeferredCleanupList(_SYNC_CHECKPOINT_CONTEXT *psContext);
+
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContextInt = (_SYNC_CHECKPOINT_CONTEXT *) psContext;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContextInt->psContextCtl;
+	IMG_UINT32 ui32RefCt = OSAtomicRead(&psContextInt->hRefCount);
+
+	if (ui32RefCt == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_INVALID_CONTEXT,
+		              "SyncCheckpointContextUnref context already freed");
+	}
+	else if (OSAtomicDecrement(&psContextInt->hRefCount) == 0)
+	{
+#if defined(PDUMP)
+		PPVRSRV_DEVICE_NODE psDeviceNode = psCtxCtl->psDeviceNode;
+#endif
+
+		/* SyncCheckpointContextDestroy only when no longer referenced */
+		OSSpinLockDestroy(psCtxCtl->hDeferredCleanupListLock);
+		psCtxCtl->hDeferredCleanupListLock = NULL;
+
+		OSSpinLockDestroy(psCtxCtl->hSyncCheckpointStatsLock);
+		psCtxCtl->hSyncCheckpointStatsLock = NULL;
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+		if (psCtxCtl->ui32SyncCheckpointPoolCount)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			        "%s called for context<%p> with %d sync checkpoints still"
+			        " in the pool",
+			        __func__,
+			        (void *) psContext,
+			        psCtxCtl->ui32SyncCheckpointPoolCount));
+		}
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+		OSSpinLockDestroy(psCtxCtl->hSyncCheckpointPoolLock);
+		psCtxCtl->hSyncCheckpointPoolLock = NULL;
+#endif
+		OSFreeMem(psContextInt->psContextCtl);
+		RA_Delete(psContextInt->psSpanRA);
+		RA_Delete(psContextInt->psSubAllocRA);
+
+#if defined(PDUMP)
+		PVR_ASSERT(dllist_is_empty(&psContext->sSyncCheckpointBlockListHead));
+
+		OSLockAcquire(psDeviceNode->hSyncCheckpointContextListLock);
+		dllist_remove_node(&psContext->sListNode);
+		OSLockRelease(psDeviceNode->hSyncCheckpointContextListLock);
+
+		OSLockDestroy(psContext->hSyncCheckpointBlockListLock);
+#endif
+
+		OSFreeMem(psContext);
+	}
+}
+
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContextInt = (_SYNC_CHECKPOINT_CONTEXT *)psContext;
+	IMG_UINT32 ui32RefCt = OSAtomicRead(&psContextInt->hRefCount);
+
+	if (ui32RefCt == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_INVALID_CONTEXT,
+		              "SyncCheckpointContextRef context use after free");
+	}
+	else
+	{
+		OSAtomicIncrement(&psContextInt->hRefCount);
+	}
+}
+
+/*
+	Internal interfaces for management of synchronisation block memory
+ */
+static PVRSRV_ERROR
+_AllocSyncCheckpointBlock(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                          SYNC_CHECKPOINT_BLOCK    **ppsSyncBlock)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	SYNC_CHECKPOINT_BLOCK *psSyncBlk;
+	PVRSRV_ERROR eError;
+
+	psSyncBlk = OSAllocMem(sizeof(*psSyncBlk));
+	PVR_LOG_GOTO_IF_NOMEM(psSyncBlk, eError, fail_alloc);
+
+	psSyncBlk->psContext = psContext;
+
+	/* Allocate sync checkpoint block */
+	psDevNode = psContext->psContextCtl->psDeviceNode;
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psDevNode, eError, fail_alloc_ufo_block);
+
+	eError = psDevNode->pfnAllocUFOBlock(psDevNode,
+	                                     SYNC_CHECKPOINT_POOL_LIMIT * sizeof(SYNC_CHECKPOINT_FW_OBJ),
+	                                     &psSyncBlk->hMemDesc,
+	                                     &psSyncBlk->ui32FirmwareAddr,
+	                                     &psSyncBlk->ui32SyncBlockSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnAllocUFOBlock", fail_alloc_ufo_block);
+
+	eError = DevmemAcquireCpuVirtAddr(psSyncBlk->hMemDesc,
+	                                  (void **) &psSyncBlk->pui32LinAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail_devmem_acquire);
+
+	OSAtomicWrite(&psSyncBlk->hRefCount, 1);
+
+	OSLockCreate(&psSyncBlk->hLock);
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Allocated Sync Checkpoint UFO block (FirmwareVAddr = 0x%08x)",
+	                      psSyncBlk->ui32FirmwareAddr);
+#if defined(PDUMP)
+	OSLockAcquire(psContext->hSyncCheckpointBlockListLock);
+	dllist_add_to_tail(&psContext->sSyncCheckpointBlockListHead, &psSyncBlk->sListNode);
+	OSLockRelease(psContext->hSyncCheckpointBlockListLock);
+#endif
+
+	*ppsSyncBlock = psSyncBlk;
+	return PVRSRV_OK;
+
+fail_devmem_acquire:
+	psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->hMemDesc);
+fail_alloc_ufo_block:
+	OSFreeMem(psSyncBlk);
+fail_alloc:
+	return eError;
+}
+
+static void
+_FreeSyncCheckpointBlock(SYNC_CHECKPOINT_BLOCK *psSyncBlk)
+{
+	OSLockAcquire(psSyncBlk->hLock);
+	if (0 == OSAtomicDecrement(&psSyncBlk->hRefCount))
+	{
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncBlk->psContext;
+		PVRSRV_DEVICE_NODE *psDevNode = psContext->psContextCtl->psDeviceNode;
+
+#if defined(PDUMP)
+		OSLockAcquire(psSyncBlk->psContext->hSyncCheckpointBlockListLock);
+		dllist_remove_node(&psSyncBlk->sListNode);
+		OSLockRelease(psSyncBlk->psContext->hSyncCheckpointBlockListLock);
+#endif
+		DevmemReleaseCpuVirtAddr(psSyncBlk->hMemDesc);
+		psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->hMemDesc);
+		OSLockRelease(psSyncBlk->hLock);
+		OSLockDestroy(psSyncBlk->hLock);
+		psSyncBlk->hLock = NULL;
+		OSFreeMem(psSyncBlk);
+	}
+	else
+	{
+		OSLockRelease(psSyncBlk->hLock);
+	}
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointBlockImport(RA_PERARENA_HANDLE hArena,
+                           RA_LENGTH_T uSize,
+                           RA_FLAGS_T uFlags,
+                           RA_LENGTH_T uBaseAlignment,
+                           const IMG_CHAR *pszAnnotation,
+                           RA_IMPORT *psImport)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = hArena;
+	SYNC_CHECKPOINT_BLOCK *psSyncBlock = NULL;
+	RA_LENGTH_T uiSpanSize;
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(uFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM((hArena != NULL), "hArena");
+
+	/* Check we've not be called with an unexpected size */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uSize == sizeof(SYNC_CHECKPOINT_FW_OBJ)), "uSize");
+
+	/*
+		Ensure the sync checkpoint context doesn't go away while we have
+		sync blocks attached to it.
+	 */
+	SyncCheckpointContextRef((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+	/* Allocate the block of memory */
+	eError = _AllocSyncCheckpointBlock(psContext, &psSyncBlock);
+	PVR_GOTO_IF_ERROR(eError, fail_syncblockalloc);
+
+	/* Allocate a span for it */
+	eError = RA_Alloc(psContext->psSpanRA,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  pszAnnotation,
+	                  &psSyncBlock->uiSpanBase,
+	                  &uiSpanSize,
+	                  NULL);
+	PVR_GOTO_IF_ERROR(eError, fail_spanalloc);
+
+	/*
+		There is no reason the span RA should return an allocation larger
+		then we request
+	 */
+	PVR_LOG_IF_FALSE((uiSpanSize == psSyncBlock->ui32SyncBlockSize),
+	                 "uiSpanSize invalid");
+
+	psImport->base = psSyncBlock->uiSpanBase;
+	psImport->uSize = psSyncBlock->ui32SyncBlockSize;
+	psImport->hPriv = psSyncBlock;
+	return PVRSRV_OK;
+
+fail_spanalloc:
+	_FreeSyncCheckpointBlock(psSyncBlock);
+fail_syncblockalloc:
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+	return eError;
+}
+
+static void
+_SyncCheckpointBlockUnimport(RA_PERARENA_HANDLE hArena,
+                             RA_BASE_T uiBase,
+                             RA_PERISPAN_HANDLE hImport)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = hArena;
+	SYNC_CHECKPOINT_BLOCK   *psSyncBlock = hImport;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE((psContext != NULL), "hArena invalid");
+	PVR_LOG_RETURN_VOID_IF_FALSE((psSyncBlock != NULL), "hImport invalid");
+	PVR_LOG_RETURN_VOID_IF_FALSE((uiBase == psSyncBlock->uiSpanBase), "uiBase invalid");
+
+	/* Free the span this import is using */
+	RA_Free(psContext->psSpanRA, uiBase);
+
+	/* Free the sync checkpoint block */
+	_FreeSyncCheckpointBlock(psSyncBlock);
+
+	/*	Drop our reference to the sync checkpoint context */
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+}
+
+static INLINE IMG_UINT32 _SyncCheckpointGetOffset(SYNC_CHECKPOINT *psSyncInt)
+{
+	IMG_UINT64 ui64Temp;
+
+	ui64Temp = (IMG_UINT64)psSyncInt->uiAllocatedAddr -
+	           (IMG_UINT64)psSyncInt->psSyncCheckpointBlock->uiSpanBase;
+	PVR_ASSERT(ui64Temp<IMG_UINT32_MAX);
+	return (IMG_UINT32)ui64Temp;
+}
+
+/* Used by SyncCheckpointContextCreate() below */
+static INLINE IMG_UINT32 _Log2(IMG_UINT32 ui32Align)
+{
+	PVR_ASSERT(IsPower2(ui32Align));
+	return ExactLog2(ui32Align);
+}
+
+/*
+	External interfaces
+ */
+
+PVRSRV_ERROR
+SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns)
+{
+	PVR_ASSERT(psSyncCheckpointPfns != NULL);
+
+	if (g_psSyncCheckpointPfnStruct)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but already initialised", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	g_psSyncCheckpointPfnStruct = psSyncCheckpointPfns;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+SyncCheckpointResolveFence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PVRSRV_FENCE hFence, IMG_UINT32 *pui32NumSyncCheckpoints,
+                           PSYNC_CHECKPOINT **papsSyncCheckpoints,
+                           IMG_UINT64 *pui64FenceUID,
+                           PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+#if defined(PDUMP)
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	_PSYNC_CHECKPOINT_CONTEXT_CTL	psContextCtl;
+#endif
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceResolve))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_pfnFenceResolve is NULL");
+		return eError;
+	}
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	psContextCtl = psSyncCheckpointContext->psContextCtl;
+
+	PVR_DPF((PVR_DBG_WARNING,
+	        "%s: Checkpoint %p DeviceID = [%u/%d]\n", __func__,
+	        psSyncCheckpointContext,
+	        psContextCtl->psDeviceNode->sDevId.ui32InternalID,
+	        psContextCtl->psDeviceNode->sDevId.i32KernelDeviceID));
+#endif
+
+	if (papsSyncCheckpoints)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceResolve(
+		                           psSyncCheckpointContext,
+		                           hFence,
+		                           pui32NumSyncCheckpoints,
+		                           papsSyncCheckpoints,
+		                           pui64FenceUID);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceResolve");
+
+#if defined(PDUMP)
+	if (*papsSyncCheckpoints)
+	{
+		for (i = 0; i < *pui32NumSyncCheckpoints; i++)
+		{
+			psSyncCheckpoint = (SYNC_CHECKPOINT *)(*papsSyncCheckpoints)[i];
+			psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	if (*pui32NumSyncCheckpoints > MAX_SYNC_CHECKPOINTS_PER_FENCE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: g_psSyncCheckpointPfnStruct->pfnFenceResolve() returned too many checkpoints (%u > MAX_SYNC_CHECKPOINTS_PER_FENCE=%u)",
+				__func__, *pui32NumSyncCheckpoints, MAX_SYNC_CHECKPOINTS_PER_FENCE));
+
+		/* Free resources after error */
+		if (*papsSyncCheckpoints)
+		{
+			for (i = 0; i < *pui32NumSyncCheckpoints; i++)
+			{
+				SyncCheckpointDropRef((*papsSyncCheckpoints)[i]);
+			}
+
+			SyncCheckpointFreeCheckpointListMem(*papsSyncCheckpoints);
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	{
+		IMG_UINT32 ii;
+
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: g_psSyncCheckpointPfnStruct->pfnFenceResolve() for fence %d returned the following %d checkpoints:",
+				__func__,
+				hFence,
+				*pui32NumSyncCheckpoints));
+
+		for (ii=0; ii<*pui32NumSyncCheckpoints; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(*papsSyncCheckpoints +  ii);
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s:   *papsSyncCheckpoints[%d]:<%p>",
+					__func__,
+					ii,
+					(void*)psNextCheckpoint));
+		}
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointResolveExportFence(PVRSRV_FENCE hExportFence,
+                                 PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                 PSYNC_CHECKPOINT *ppsSyncCheckpoint,
+                                 PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnExportFenceResolve))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+		        __func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "pfnExportFenceResolve is NULL");
+		return eError;
+	}
+
+	if (ppsSyncCheckpoint)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnExportFenceResolve(
+		                           hExportFence,
+		                           psSyncCheckpointContext,
+		                           ppsSyncCheckpoint);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnExportFenceResolve");
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s: g_psSyncCheckpointPfnStruct->pfnExportFenceResolve() for fence %d returned the following checkpoint:",
+		        __func__,
+		        hExportFence));
+
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s:   psSyncCheckpoint:<%p>",
+		         __func__,
+		         (void*)*ppsSyncCheckpoint));
+	}
+#endif
+
+#if defined(PDUMP)
+	if (*ppsSyncCheckpoint)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT *)(*ppsSyncCheckpoint);
+		psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointCreateFence(PVRSRV_DEVICE_NODE *psDevNode,
+                          const IMG_CHAR *pszFenceName,
+                          PVRSRV_TIMELINE hTimeline,
+                          PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                          PVRSRV_FENCE *phNewFence,
+                          IMG_UINT64 *puiUpdateFenceUID,
+                          void **ppvFenceFinaliseData,
+                          PSYNC_CHECKPOINT *psNewSyncCheckpoint,
+                          void **ppvTimelineUpdateSyncPrim,
+                          IMG_UINT32 *pui32TimelineUpdateValue,
+                          PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceCreate))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceCreate is NULL");
+	}
+	else
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceCreate(
+		                          psDevNode,
+		                          pszFenceName,
+		                          hTimeline,
+		                          psSyncCheckpointContext,
+		                          phNewFence,
+		                          puiUpdateFenceUID,
+		                          ppvFenceFinaliseData,
+		                          psNewSyncCheckpoint,
+		                          ppvTimelineUpdateSyncPrim,
+		                          pui32TimelineUpdateValue);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s failed to create new fence<%p> for timeline<%d> using "
+					"sync checkpoint context<%p>, psNewSyncCheckpoint=<%p>, eError=%s",
+					__func__,
+					(void*)phNewFence,
+					hTimeline,
+					(void*)psSyncCheckpointContext,
+					(void*)psNewSyncCheckpoint,
+					PVRSRVGetErrorString(eError)));
+		}
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s [%u/%d] created new fence<%d> for timeline<%d> "
+					"using sync checkpoint context<%p>, "
+					"new sync_checkpoint=<%p>",
+					__func__,
+					psDevNode->sDevId.ui32InternalID,
+					psDevNode->sDevId.i32KernelDeviceID,
+					*phNewFence,
+					hTimeline,
+					(void*)psSyncCheckpointContext,
+					(void*)*psNewSyncCheckpoint));
+		}
+#endif
+
+#if defined(PDUMP)
+		if (eError == PVRSRV_OK)
+		{
+			SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT*)(*psNewSyncCheckpoint);
+			if (psSyncCheckpoint)
+			{
+				psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+			}
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointRollbackFenceData(PVRSRV_FENCE hFence, void *pvFinaliseData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceDataRollback)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceDataRollback is NULL");
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: called to rollback fence data <%p>",
+				__func__,
+				pvFinaliseData));
+#endif
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceDataRollback(
+		            hFence, pvFinaliseData);
+		PVR_LOG_IF_ERROR(eError,
+		                 "g_psSyncCheckpointPfnStruct->pfnFenceDataRollback returned error");
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointFinaliseFence(PPVRSRV_DEVICE_NODE psDevNode,
+                            PVRSRV_FENCE hFence,
+                            void *pvFinaliseData,
+                            PSYNC_CHECKPOINT psSyncCheckpoint,
+                            const IMG_CHAR *pszName)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceFinalise)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Warning (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED) (this is permitted)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: called to finalise fence <%d>",
+				__func__,
+				hFence));
+#endif
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceFinalise(hFence, pvFinaliseData);
+		PVR_LOG_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceFinalise returned error");
+
+		RGXSRV_HWPERF_ALLOC_FENCE(psDevNode, OSGetCurrentClientProcessIDKM(), hFence,
+		                          SyncCheckpointGetFirmwareAddr(psSyncCheckpoint),
+		                          pszName, OSStringLength(pszName));
+	}
+	return eError;
+}
+
+void
+SyncCheckpointFreeCheckpointListMem(void *pvCheckpointListMem)
+{
+	if (g_psSyncCheckpointPfnStruct->pfnFreeCheckpointListMem)
+	{
+		g_psSyncCheckpointPfnStruct->pfnFreeCheckpointListMem(pvCheckpointListMem);
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointNoHWUpdateTimelines(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines is NULL");
+	}
+	else
+	{
+		g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines(pvPrivateData);
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointNoHWSignalExportFence(PVRSRV_FENCE iExportFenceToSignal)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence is NULL");
+	}
+	else
+	{
+		g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence(iExportFenceToSignal);
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointDumpInfoOnStalledUFOs(IMG_UINT32 ui32NumUFOs, IMG_UINT32 *pui32Vaddrs, IMG_UINT32 *pui32NumSyncOwnedUFOs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_FALSE((pui32NumSyncOwnedUFOs != NULL), "pui32NumSyncOwnedUFOs invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs)
+	{
+		*pui32NumSyncOwnedUFOs = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs is NULL");
+	}
+	else
+	{
+		*pui32NumSyncOwnedUFOs = g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs(ui32NumUFOs, pui32Vaddrs);
+		PVR_LOG(("%d sync checkpoint%s owned by %s in stalled context",
+		         *pui32NumSyncOwnedUFOs, *pui32NumSyncOwnedUFOs==1 ? "" : "s",
+		         g_psSyncCheckpointPfnStruct->pszImplName));
+	}
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocSyncCheckpoint(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                     SYNC_CHECKPOINT **ppsSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psNewSyncCheckpoint = NULL;
+	PVRSRV_ERROR eError;
+
+	/* Allocate sync checkpoint */
+	psNewSyncCheckpoint = OSAllocMem(sizeof(*psNewSyncCheckpoint));
+	PVR_LOG_RETURN_IF_NOMEM(psNewSyncCheckpoint, "OSAllocMem"); /* Sets OOM error code */
+
+	eError = RA_Alloc(psContext->psSubAllocRA,
+                      sizeof(*psNewSyncCheckpoint->psSyncCheckpointFwObj),
+                      RA_NO_IMPORT_MULTIPLIER,
+                      0,
+                      sizeof(IMG_UINT32),
+                      NULL,
+                      &psNewSyncCheckpoint->uiAllocatedAddr,
+                      NULL,
+                      (RA_PERISPAN_HANDLE *) &psNewSyncCheckpoint->psSyncCheckpointBlock);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RA_Alloc", fail_ra_alloc);
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+	        "%s CALLED RA_Alloc(), psSubAllocRA=<%p>, ui32SpanAddr=0x%llx",
+	        __func__,
+	        (void*)psContext->psSubAllocRA,
+	        psNewSyncCheckpoint->uiAllocatedAddr));
+#endif
+	psNewSyncCheckpoint->psSyncCheckpointFwObj =
+	        (volatile SYNC_CHECKPOINT_FW_OBJ*)(void *)(psNewSyncCheckpoint->psSyncCheckpointBlock->pui32LinAddr +
+	                (_SyncCheckpointGetOffset(psNewSyncCheckpoint)/sizeof(IMG_UINT32)));
+	psNewSyncCheckpoint->ui32FWAddr = psNewSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+	                                  _SyncCheckpointGetOffset(psNewSyncCheckpoint) + 1;
+	OSAtomicIncrement(&psNewSyncCheckpoint->psSyncCheckpointBlock->psContext->hCheckpointCount);
+
+#if defined(DEBUG)
+	psNewSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_USE;
+#endif
+	psNewSyncCheckpoint->sListNode.psPrevNode = NULL;
+	psNewSyncCheckpoint->sListNode.psNextNode = NULL;
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s called to allocate new sync checkpoint<%p> for context<%p>",
+	         __func__, (void*)psNewSyncCheckpoint, (void*)psContext));
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s                    psSyncCheckpointFwObj<%p>",
+	         __func__, (void*)psNewSyncCheckpoint->psSyncCheckpointFwObj));
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s                    psSyncCheckpoint FwAddr=0x%x",
+	         __func__, SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psNewSyncCheckpoint)));
+#endif
+	*ppsSyncCheckpoint = psNewSyncCheckpoint;
+	return PVRSRV_OK;
+
+fail_ra_alloc:
+	OSFreeMem(psNewSyncCheckpoint);
+	return eError;
+}
+
+/* Poisons and frees the checkpoint
+ * Decrements context refcount. */
+static void _FreeSyncCheckpoint(SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+
+	psSyncCheckpoint->sCheckpointUFOAddr.ui32Addr = 0;
+	psSyncCheckpoint->psSyncCheckpointFwObj = NULL;
+#if defined(DEBUG)
+	psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_FREED;
+#endif
+
+	RA_Free(psSyncCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA,
+	        psSyncCheckpoint->uiAllocatedAddr);
+	psSyncCheckpoint->psSyncCheckpointBlock = NULL;
+
+	OSFreeMem(psSyncCheckpoint);
+
+	OSAtomicDecrement(&psContext->hCheckpointCount);
+}
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static PVRSRV_ERROR
+_PrepopulateSyncCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                               IMG_UINT32 ui32InitPoolSize)
+{
+	IMG_UINT32 ui32SyncCheckpoint;
+	SYNC_CHECKPOINT *psNewSyncCheckpoint;
+	PVRSRV_ERROR eError;
+
+	/* Allocate sync checkpoints and place in the pool */
+	for (ui32SyncCheckpoint=0; ui32SyncCheckpoint<ui32InitPoolSize; ui32SyncCheckpoint++)
+	{
+		eError = _AllocSyncCheckpoint(psContext, &psNewSyncCheckpoint);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_AllocSyncCheckpoint");
+
+#if defined(PDUMP)
+		if (ui32SyncCheckpoint == 0)
+		{
+			psContext->psContextCtl->psSyncCheckpointPoolBlock = psNewSyncCheckpoint->psSyncCheckpointBlock;
+		}
+#endif
+
+		if (!_PutCheckpointInPool(psNewSyncCheckpoint))
+		{
+			_FreeSyncCheckpoint(psNewSyncCheckpoint);
+		}
+	}
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,"%s: Sync checkpoint pool [%d/%d]",
+	         __func__, psContext->psContextCtl->ui32SyncCheckpointPoolCount,
+	         psContext->psContextCtl->ui32SyncCheckpointPoolSize));
+#endif
+	return PVRSRV_OK;
+}
+#endif /* if (SYNC_CHECKPOINT_POOL_LIMIT > 0) */
+
+PVRSRV_ERROR
+SyncCheckpointContextCreate(PPVRSRV_DEVICE_NODE psDevNode,
+                            PSYNC_CHECKPOINT_CONTEXT *ppsSyncCheckpointContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = NULL;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *psContextCtl = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR azTempName[PVRSRV_SYNC_NAME_LENGTH] = {0};
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if !defined(PDUMP)
+	void *pvAppHintState = NULL;
+	const IMG_UINT32 ui32DefaultMaxPoolLog2Size = PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2;
+	const IMG_UINT32 ui32DefaultInitPoolLog2Size = PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2;
+#endif
+	IMG_UINT32 ui32MaxPoolLog2Size;
+	IMG_UINT32 ui32InitPoolLog2Size;
+	IMG_UINT32 ui32InitPoolSize = 0;
+#endif
+	IMG_UINT32 ui32MaxPoolSize = 0;
+
+	PVR_LOG_RETURN_IF_FALSE((ppsSyncCheckpointContext != NULL),
+	                  "ppsSyncCheckpointContext invalid",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if defined(PDUMP)
+	/*
+	   Pdumps use the maximum pool size to limit checkpoint address reuse and start with
+	   a minimum pool to avoid extra work allocating checkpoints which might not be used.
+	   Since during a pdump capture the pool is setup once during the pdump init phase
+	   and not emptied / de-initialised between captures we need to map at least one
+	   checkpoint at creation time, so that the pdump records the CPU mapping of the
+	   pool. Not doing this would lead to errors on subsequent pdump captures because
+	   only the first pdump would contain the mapping.
+	*/
+	ui32MaxPoolLog2Size = 10;
+	ui32InitPoolLog2Size = 0;
+	ui32InitPoolSize = 1;
+#else
+	/* Read AppHints to determine the size of the sync checkpoint pool, if specified */
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, SyncCheckpointPoolMaxLog2,
+	                     &ui32DefaultMaxPoolLog2Size, &ui32MaxPoolLog2Size);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, SyncCheckpointPoolInitLog2,
+	                     &ui32DefaultInitPoolLog2Size, &ui32InitPoolLog2Size);
+	OSFreeAppHintState(pvAppHintState);
+#endif
+
+	if (ui32MaxPoolLog2Size > 0)
+	{
+		ui32MaxPoolSize = 1 << ui32MaxPoolLog2Size;
+		if (ui32MaxPoolSize > SYNC_CHECKPOINT_POOL_LIMIT)
+		{
+			ui32MaxPoolSize = SYNC_CHECKPOINT_POOL_LIMIT;
+		}
+	}
+#endif
+
+	psContext = OSAllocMem(sizeof(*psContext));
+	PVR_LOG_GOTO_IF_NOMEM(psContext, eError, fail_alloc); /* Sets OOM error code */
+
+	/* psContextCtl includes allocation for the sync checkpoint pool) */
+	psContextCtl = OSAllocMem(sizeof(*psContextCtl) + IMG_FLEX_ARRAY_SIZE(sizeof(SYNC_CHECKPOINT*), ui32MaxPoolSize));
+	PVR_LOG_GOTO_IF_NOMEM(psContextCtl, eError, fail_alloc2); /* Sets OOM error code */
+
+	eError = OSSpinLockCreate(&psContextCtl->hDeferredCleanupListLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate:1", fail_create_deferred_cleanup_lock);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	eError = OSSpinLockCreate(&psContextCtl->hSyncCheckpointPoolLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate:2", fail_create_pool_lock);
+#endif
+
+	dllist_init(&psContextCtl->sDeferredCleanupListHead);
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	psContextCtl->ui32SyncCheckpointPoolCount = 0;
+	psContextCtl->ui32SyncCheckpointPoolWp = 0;
+	psContextCtl->ui32SyncCheckpointPoolRp = 0;
+#if defined(PDUMP)
+	psContextCtl->psSyncCheckpointPoolBlock = NULL;
+	psContextCtl->bAllocateFromCheckpointPool = IMG_FALSE;
+#endif
+	psContextCtl->ui8PoolStateFlags = SYNC_CHECKPOINT_POOL_VALID;
+#endif
+	psContextCtl->psDeviceNode = (SHARED_DEV_CONNECTION)psDevNode;
+
+	/*
+		Create the RA for sub-allocations of the sync checkpoints
+
+		Note:
+		The import size doesn't matter here as the server will pass
+		back the blocksize when it does the import which overrides
+		what we specify here.
+	 */
+	OSSNPrintf(azTempName, PVRSRV_SYNC_NAME_LENGTH,"Sync Prim RA-%p", psContext);
+	psContext->psSubAllocRA = RA_Create(azTempName,
+	                                    /* Params for imports */
+	                                    _Log2(sizeof(IMG_UINT32)),
+	                                    RA_LOCKCLASS_3,
+	                                    _SyncCheckpointBlockImport,
+	                                    _SyncCheckpointBlockUnimport,
+	                                    psContext,
+	                                    RA_POLICY_DEFAULT);
+	PVR_LOG_GOTO_IF_NOMEM(psContext->psSubAllocRA, eError, fail_suballoc);
+
+	/*
+		Create the span-management RA
+
+		The RA requires that we work with linear spans. For our use
+		here we don't require this behaviour as we're always working
+		within offsets of blocks (imports). However, we need to keep
+		the RA happy so we create the "span" management RA which
+		ensures that all are imports are added to the RA in a linear
+		fashion
+	 */
+	OSSNPrintf(azTempName, PVRSRV_SYNC_NAME_LENGTH, "Sync Prim span RA-%p", psContext);
+	psContext->psSpanRA = RA_Create(azTempName,
+	                                /* Params for imports */
+	                                0,
+	                                RA_LOCKCLASS_1,
+	                                NULL,
+	                                NULL,
+	                                NULL,
+	                                RA_POLICY_DEFAULT);
+	PVR_LOG_GOTO_IF_NOMEM(psContext->psSpanRA, eError, fail_span);
+
+	if (!RA_Add(psContext->psSpanRA, 0, MAX_SYNC_CHECKPOINT_MEM, 0, NULL))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		PVR_LOG_ERROR(eError, "SyncCheckpointContextCreate call to RA_Add(span) failed");
+		goto fail_span_add;
+	}
+
+	OSAtomicWrite(&psContext->hRefCount, 1);
+	OSAtomicWrite(&psContext->hCheckpointCount, 0);
+
+	psContext->psContextCtl = psContextCtl;
+
+	*ppsSyncCheckpointContext = (PSYNC_CHECKPOINT_CONTEXT)psContext;
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s: created psSyncCheckpointContext=<%p> (%d contexts exist)",
+			__func__,
+			(void*)*ppsSyncCheckpointContext,
+			++gui32NumSyncCheckpointContexts));
+#endif
+
+#if defined(PDUMP)
+	dllist_init(&psContext->sSyncCheckpointBlockListHead);
+
+	eError = OSLockCreate(&psContext->hSyncCheckpointBlockListLock);
+	PVR_GOTO_IF_ERROR(eError, fail_span_add);
+
+	OSLockAcquire(psDevNode->hSyncCheckpointContextListLock);
+	dllist_add_to_tail(&psDevNode->sSyncCheckpointContextListHead, &psContext->sListNode);
+	OSLockRelease(psDevNode->hSyncCheckpointContextListLock);
+#endif
+
+	psContextCtl->ui32CurrentInUseSyncCheckpoints = 0;
+	psContextCtl->ui32MaxInUseSyncCheckpoints = 0;
+	psContextCtl->ui32CurrentInUseMirroredSyncCPs = 0;
+	psContextCtl->ui32MaxInUseMirroredSyncCPs = 0;
+	eError = OSSpinLockCreate(&psContextCtl->hSyncCheckpointStatsLock);
+	PVR_GOTO_IF_ERROR(eError, fail_span_stat);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	/* Pre-populate the sync checkpoint pool, if specified */
+	psContextCtl->ui32SyncCheckpointPoolSize = ui32MaxPoolSize;
+
+	/* Ensure ui32MaxPoolSize is a POT and does not exceed SYNC_CHECKPOINT_POOL_LIMIT,
+	 * and ui32InitPoolSize does not exceed ui32MaxPoolSize.
+	 */
+	if (psContextCtl->ui32SyncCheckpointPoolSize > SYNC_CHECKPOINT_POOL_LIMIT)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: AppHint SyncCheckpointPoolMaxLog2(%d) would exceed "
+		                          "SYNC_CHECKPOINT_POOL_LIMIT(%d) - limiting to %d",
+		                          __func__, ui32MaxPoolLog2Size,
+		                          SYNC_CHECKPOINT_POOL_LIMIT, SYNC_CHECKPOINT_POOL_LIMIT));
+		psContextCtl->ui32SyncCheckpointPoolSize = SYNC_CHECKPOINT_POOL_LIMIT;
+	}
+
+	if (ui32InitPoolLog2Size > 0)
+	{
+		ui32InitPoolSize = 1 << ui32InitPoolLog2Size;
+	}
+	if (ui32InitPoolSize > psContextCtl->ui32SyncCheckpointPoolSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: AppHint SyncCheckpointPoolInitLog2(%d) would exceed "
+		                          "ui32SyncCheckpointPoolSize(%d) - limiting to %d",
+		                          __func__, ui32InitPoolLog2Size,
+		                          psContextCtl->ui32SyncCheckpointPoolSize,
+		                          psContextCtl->ui32SyncCheckpointPoolSize));
+		ui32InitPoolSize = psContextCtl->ui32SyncCheckpointPoolSize;
+	}
+
+	if (ui32InitPoolSize > 0)
+	{
+		eError = _PrepopulateSyncCheckpointPool(psContext, ui32InitPoolSize);
+		PVR_LOG_RETURN_IF_ERROR_VA(eError, "_PrepopulateSyncCheckpointPool(%d)", ui32InitPoolSize);
+	}
+#endif
+
+	return PVRSRV_OK;
+fail_span_stat:
+#if defined(PDUMP)
+	OSLockDestroy(psContext->hSyncCheckpointBlockListLock);
+	psContext->hSyncCheckpointBlockListLock = NULL;
+#endif
+fail_span_add:
+	RA_Delete(psContext->psSpanRA);
+fail_span:
+	RA_Delete(psContext->psSubAllocRA);
+fail_suballoc:
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	OSSpinLockDestroy(psContextCtl->hSyncCheckpointPoolLock);
+	psContextCtl->hSyncCheckpointPoolLock = NULL;
+fail_create_pool_lock:
+#endif
+	OSSpinLockDestroy(psContextCtl->hDeferredCleanupListLock);
+	psContextCtl->hDeferredCleanupListLock = NULL;
+fail_create_deferred_cleanup_lock:
+	OSFreeMem(psContextCtl);
+fail_alloc2:
+	OSFreeMem(psContext);
+fail_alloc:
+	return eError;
+}
+
+PVRSRV_ERROR SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_INT iRf = 0;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncCheckpointContext != NULL),
+	                  "psSyncCheckpointContext invalid",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevNode = (PVRSRV_DEVICE_NODE *)psContext->psContextCtl->psDeviceNode;
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s: destroying psSyncCheckpointContext=<%p> (now have %d contexts)",
+			__func__,
+			(void*)psSyncCheckpointContext,
+			--gui32NumSyncCheckpointContexts));
+#endif
+
+	_CheckDeferredCleanupList(psContext);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	if (psContext->psContextCtl->ui32SyncCheckpointPoolCount > 0)
+	{
+		IMG_UINT32 ui32NumFreedFromPool = _CleanCheckpointPool(psContext);
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s freed %d sync checkpoints that were still in the pool for context<%p>",
+				__func__,
+				ui32NumFreedFromPool,
+				(void*)psContext));
+#else
+		PVR_UNREFERENCED_PARAMETER(ui32NumFreedFromPool);
+#endif
+	}
+#endif
+
+	iRf = OSAtomicRead(&psContext->hCheckpointCount);
+
+	if (iRf != 0)
+	{
+		OS_SPINLOCK_FLAGS uiFlags;
+
+		/* Note, this is not a permanent error as the caller may retry later */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s <%p> attempted with active references (iRf=%d), "
+				"may be the result of a race",
+				__func__,
+				(void*)psContext,
+				iRf));
+
+		eError = PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT;
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		{
+			DLLIST_NODE *psNode, *psNext;
+
+			dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+			{
+				SYNC_CHECKPOINT *psSyncCheckpoint = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+				bool bDeferredFree = dllist_node_is_in_list(&psSyncCheckpoint->sDeferredFreeListNode);
+#if defined(DEBUG)
+				IMG_UINT32 ui32State = psSyncCheckpoint->psSyncCheckpointFwObj->ui32State;
+#endif
+
+				/* Line below avoids build error in release builds (where PVR_DPF is not defined) */
+				PVR_UNREFERENCED_PARAMETER(bDeferredFree);
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s syncCheckpoint<%p> ID=%d, %s, refs=%d, state=%s, fwaddr=%#08x, enqCount:%d, FWCount:%d %s",
+						__func__,
+						(void*)psSyncCheckpoint,
+						psSyncCheckpoint->ui32UID,
+						psSyncCheckpoint->azName,
+						OSAtomicRead(&psSyncCheckpoint->hRefCount),
+						ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED ?
+								"PVRSRV_SYNC_CHECKPOINT_SIGNALLED" :
+								ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE ?
+										"PVRSRV_SYNC_CHECKPOINT_ACTIVE" : "PVRSRV_SYNC_CHECKPOINT_ERRORED",
+						psSyncCheckpoint->ui32FWAddr,
+						OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+						psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+						bDeferredFree ? "(deferred free)" : ""));
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+				gui32NumSyncCheckpointContexts++;
+#endif
+			}
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+	else
+	{
+		SyncCheckpointContextUnref(psSyncCheckpointContext);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                    PVRSRV_TIMELINE hTimeline,
+                    PVRSRV_FENCE hFence,
+                    const IMG_CHAR *pszCheckpointName,
+                    PSYNC_CHECKPOINT *ppsSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psNewSyncCheckpoint = NULL;
+	_SYNC_CHECKPOINT_CONTEXT *psSyncContextInt = (_SYNC_CHECKPOINT_CONTEXT*)psSyncContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncContext != NULL), "psSyncContext invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((ppsSyncCheckpoint != NULL), "ppsSyncCheckpoint invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevNode = (PVRSRV_DEVICE_NODE *)psSyncContextInt->psContextCtl->psDeviceNode;
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+	PVR_DPF((PVR_DBG_WARNING, "%s Entry, Getting checkpoint from pool",
+			 __func__));
+#endif
+	psNewSyncCheckpoint = _GetCheckpointFromPool(psSyncContextInt);
+	if (!psNewSyncCheckpoint)
+	{
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s     checkpoint pool empty - will have to allocate",
+				 __func__));
+#endif
+	}
+#endif
+	/* If pool is empty (or not defined) alloc the new sync checkpoint */
+	if (!psNewSyncCheckpoint)
+	{
+		eError = _AllocSyncCheckpoint(psSyncContextInt, &psNewSyncCheckpoint);
+		PVR_LOG_GOTO_IF_NOMEM(psNewSyncCheckpoint, eError, fail_alloc); /* Sets OOM error code */
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0) && defined(PDUMP)
+		/* If the pool is not yet enabled and we allocated all the block, then the pool is ready to use. */
+		if (!psSyncContextInt->psContextCtl->bAllocateFromCheckpointPool)
+		{
+			RA_USAGE_STATS  sRAUsageStats;
+
+			RA_Get_Usage_Stats(psSyncContextInt->psSubAllocRA, &sRAUsageStats);
+
+			if (sRAUsageStats.ui64FreeArenaSize < sizeof(SYNC_CHECKPOINT_FW_OBJ))
+			{
+				psSyncContextInt->psContextCtl->bAllocateFromCheckpointPool = IMG_TRUE;
+			}
+		}
+#endif
+	}
+
+	OSSpinLockAcquire(psSyncContextInt->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+	if (++psSyncContextInt->psContextCtl->ui32CurrentInUseSyncCheckpoints > psSyncContextInt->psContextCtl->ui32MaxInUseSyncCheckpoints)
+	{
+		psSyncContextInt->psContextCtl->ui32MaxInUseSyncCheckpoints = psSyncContextInt->psContextCtl->ui32CurrentInUseSyncCheckpoints;
+	}
+	if ((hTimeline == SYNC_CHECKPOINT_MIRRORED_CHECKPOINT) &&
+	    ++psSyncContextInt->psContextCtl->ui32CurrentInUseMirroredSyncCPs > psSyncContextInt->psContextCtl->ui32MaxInUseMirroredSyncCPs)
+	{
+		psSyncContextInt->psContextCtl->ui32MaxInUseMirroredSyncCPs = psSyncContextInt->psContextCtl->ui32CurrentInUseMirroredSyncCPs;
+	}
+	OSSpinLockRelease(psSyncContextInt->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+	psNewSyncCheckpoint->hTimeline = hTimeline;
+	OSAtomicWrite(&psNewSyncCheckpoint->hRefCount, 1);
+	OSAtomicWrite(&psNewSyncCheckpoint->hEnqueuedCCBCount, 0);
+	psNewSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount = 0;
+	psNewSyncCheckpoint->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_ACTIVE;
+	psNewSyncCheckpoint->uiProcess = OSGetCurrentClientProcessIDKM();
+	OSCachedMemSet(&psNewSyncCheckpoint->sDeferredFreeListNode, 0, sizeof(psNewSyncCheckpoint->sDeferredFreeListNode));
+
+	if (pszCheckpointName)
+	{
+		/* Copy over the checkpoint name annotation */
+		OSStringSafeCopy(psNewSyncCheckpoint->azName, pszCheckpointName, PVRSRV_SYNC_NAME_LENGTH);
+	}
+	else
+	{
+		/* No sync checkpoint name annotation */
+		psNewSyncCheckpoint->azName[0] = '\0';
+	}
+
+	/* Store sync checkpoint FW address in PRGXFWIF_UFO_ADDR struct */
+	psNewSyncCheckpoint->sCheckpointUFOAddr.ui32Addr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psNewSyncCheckpoint);
+
+	/* Assign unique ID to this sync checkpoint */
+	psNewSyncCheckpoint->ui32UID = g_SyncCheckpointUID++;
+
+#if defined(PDUMP)
+	/* Flushing deferred fence signals to pdump */
+	MISRHandler_PdumpDeferredSyncSignalPoster(psDevNode);
+
+	_SyncCheckpointAllocPDump(psDevNode, psNewSyncCheckpoint);
+#endif
+
+	RGXSRV_HWPERF_ALLOC_SYNC_CP(psDevNode, psNewSyncCheckpoint->hTimeline,
+	                            OSGetCurrentClientProcessIDKM(),
+	                            hFence,
+	                            psNewSyncCheckpoint->ui32FWAddr,
+	                            psNewSyncCheckpoint->azName,
+	                            OSStringLength(psNewSyncCheckpoint->azName));
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		IMG_CHAR szChkptName[PVRSRV_SYNC_NAME_LENGTH];
+
+		if (pszCheckpointName)
+		{
+			/* Copy the checkpoint name annotation into a fixed-size array */
+			OSStringSafeCopy(szChkptName, pszCheckpointName, PVRSRV_SYNC_NAME_LENGTH);
+		}
+		else
+		{
+			/* No checkpoint name annotation */
+			szChkptName[0] = 0;
+		}
+		/* record this sync */
+		eError = _SyncCheckpointRecordAdd(&psNewSyncCheckpoint->hRecord,
+		                                 psNewSyncCheckpoint->psSyncCheckpointBlock,
+		                                 psNewSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr,
+		                                 _SyncCheckpointGetOffset(psNewSyncCheckpoint),
+		                                 psNewSyncCheckpoint->ui32UID,
+		                                 OSStringNLength(szChkptName, PVRSRV_SYNC_NAME_LENGTH),
+		                                 szChkptName, (PSYNC_CHECKPOINT)psNewSyncCheckpoint);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync checkpoint record \"%s\" (%s)",
+					__func__,
+					szChkptName,
+					PVRSRVGetErrorString(eError)));
+			psNewSyncCheckpoint->hRecord = NULL;
+			/* note the error but continue without affecting driver operation */
+		}
+	}
+
+	/* Add the sync checkpoint to the device list */
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_add_to_head(&psDevNode->sSyncCheckpointSyncsList,
+	                   &psNewSyncCheckpoint->sListNode);
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+	*ppsSyncCheckpoint = (PSYNC_CHECKPOINT)psNewSyncCheckpoint;
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Exit(Ok), psNewSyncCheckpoint->ui32UID=%d <%p>",
+			__func__,
+			psNewSyncCheckpoint->ui32UID,
+			(void*)psNewSyncCheckpoint));
+#endif
+	return PVRSRV_OK;
+
+fail_alloc:
+	return eError;
+}
+
+static void SyncCheckpointUnref(SYNC_CHECKPOINT *psSyncCheckpointInt)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psContext = psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+	psDevNode = (PVRSRV_DEVICE_NODE *)psContext->psContextCtl->psDeviceNode;
+
+	/*
+	 * Without this reference, the context may be destroyed as soon
+	 * as _FreeSyncCheckpoint is called, but the context is still
+	 * needed when _CheckDeferredCleanupList is called at the end
+	 * of this function.
+	 */
+	SyncCheckpointContextRef((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+#if defined(DEBUG)
+	PVR_ASSERT(psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE);
+#endif
+	if (!OSAtomicRead(&psSyncCheckpointInt->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncCheckpointUnref sync checkpoint already freed"));
+	}
+	else if (0 == OSAtomicDecrement(&psSyncCheckpointInt->hRefCount))
+	{
+		/* If the firmware has serviced all enqueued references to the sync checkpoint, free it */
+		if (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount ==
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)))
+		{
+			PVRSRV_ERROR eError;
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s No outstanding FW ops and hRef is zero, deleting SyncCheckpoint..",
+					__func__));
+#endif
+
+			if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+				&& psSyncCheckpointInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = _SyncCheckpointRecordRemove(psSyncCheckpointInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "_SyncCheckpointRecordRemove");
+			}
+
+			/* Remove the sync checkpoint from the global list */
+			OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+			dllist_remove_node(&psSyncCheckpointInt->sListNode);
+			OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+			RGXSRV_HWPERF_FREE(psDevNode, SYNC_CP, psSyncCheckpointInt->ui32FWAddr);
+
+			OSSpinLockAcquire(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+			psContext->psContextCtl->ui32CurrentInUseSyncCheckpoints--;
+			if (psSyncCheckpointInt->hTimeline == SYNC_CHECKPOINT_MIRRORED_CHECKPOINT)
+			{
+				psContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs--;
+			}
+			OSSpinLockRelease(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s attempting to return sync checkpoint to the pool",
+					__func__));
+#endif
+			if (!_PutCheckpointInPool(psSyncCheckpointInt))
+#endif
+			{
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s pool is full, so just free it",
+						__func__));
+#endif
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s CALLING RA_Free(psSyncCheckpoint(ID:%d)<%p>), psSubAllocRA=<%p>, ui32AllocatedAddr=0x%llx",
+						__func__,
+						psSyncCheckpointInt->ui32UID,
+						(void*)psSyncCheckpointInt,
+						(void*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext->psSubAllocRA,
+						psSyncCheckpointInt->uiAllocatedAddr));
+#endif
+				_FreeSyncCheckpoint(psSyncCheckpointInt);
+			}
+		}
+		else
+		{
+			OS_SPINLOCK_FLAGS uiFlags;
+#if ((ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s Outstanding FW ops hEnqueuedCCBCount=%d != FwObj->ui32FwRefCount=%d "
+					"- DEFERRING CLEANUP psSyncCheckpoint(ID:%d)<%p>",
+					__func__,
+					OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+					psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount,
+					psSyncCheckpointInt->ui32UID,
+					(void*)psSyncCheckpointInt));
+#endif
+			/* Add the sync checkpoint to the deferred free list */
+			OSSpinLockAcquire(psContext->psContextCtl->hDeferredCleanupListLock, uiFlags);
+			dllist_add_to_tail(&psContext->psContextCtl->sDeferredCleanupListHead,
+			                   &psSyncCheckpointInt->sDeferredFreeListNode);
+			OSSpinLockRelease(psContext->psContextCtl->hDeferredCleanupListLock, uiFlags);
+		}
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s psSyncCheckpoint(ID:%d)<%p>, hRefCount decremented to %d",
+				__func__,
+				psSyncCheckpointInt->ui32UID,
+				(void*)psSyncCheckpointInt,
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount))));
+#endif
+	}
+
+	/* See if any sync checkpoints in the deferred cleanup list can be freed */
+	_CheckDeferredCleanupList(psContext);
+
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+}
+
+void SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Entry,  psSyncCheckpoint(ID:%d)<%p>, hRefCount=%d, psSyncCheckpoint->ui32ValidationCheck=0x%x",
+			__func__,
+			psSyncCheckpointInt->ui32UID,
+			(void*)psSyncCheckpoint,
+			(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount)),
+			psSyncCheckpointInt->ui32ValidationCheck));
+#else
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Entry,  psSyncCheckpoint(ID:%d)<%p>, hRefCount=%d",
+			__func__,
+			psSyncCheckpointInt->ui32UID,
+			(void*)psSyncCheckpoint,
+			(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount))));
+#endif
+#endif
+	SyncCheckpointUnref(psSyncCheckpointInt);
+}
+
+void
+SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+			RGXSRVHWPerfSyncCheckpointUFOUpdate(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+
+#if defined(SUPPORT_RGX) && defined(PDUMP)
+			_SyncCheckpointUpdatePDump(psContext->psContextCtl->psDeviceNode, psSyncCheckpointInt, PVRSRV_SYNC_CHECKPOINT_SIGNALLED, ui32FenceSyncFlags);
+#endif
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s asked to set PVRSRV_SYNC_CHECKPOINT_SIGNALLED(%d) for (psSyncCheckpointInt->ui32UID=%d), "
+					 "when value is already %d",
+					 __func__,
+					 PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+					 psSyncCheckpointInt->ui32UID,
+					 ui32State));
+		}
+	}
+}
+
+void
+SyncCheckpointSignalNoHW(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+			RGXSRVHWPerfSyncCheckpointUFOUpdate(psDevInfo, psSyncCheckpointInt, PVRSRV_FENCE_FLAG_NONE);
+#endif
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+		}
+		else
+		{
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s asked to set PVRSRV_SYNC_CHECKPOINT_SIGNALLED(%d) for (psSyncCheckpointInt->ui32UID=%d), "
+					"when value is already %d",
+					__func__,
+					PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+					psSyncCheckpointInt->ui32UID,
+					ui32State));
+#endif
+		}
+	}
+}
+
+void
+SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+			if (!(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+			{
+				RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+				sSyncData.sUpdate.ui32FWAddr = SyncCheckpointGetFirmwareAddr(psSyncCheckpoint);
+				sSyncData.sUpdate.ui32OldValue = ui32State;
+				sSyncData.sUpdate.ui32NewValue = PVRSRV_SYNC_CHECKPOINT_ERRORED;
+
+				RGXSRV_HWPERF_UFO(psDevInfo, RGX_HWPERF_UFO_EV_UPDATE, &sSyncData,
+				                  (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+			}
+#endif
+
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_ERRORED;
+
+#if defined(SUPPORT_RGX) && defined(PDUMP)
+			_SyncCheckpointUpdatePDump(psContext->psContextCtl->psDeviceNode, psSyncCheckpointInt, PVRSRV_SYNC_CHECKPOINT_ERRORED, ui32FenceSyncFlags);
+#endif
+		}
+	}
+}
+
+IMG_BOOL SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+#if defined(SUPPORT_RGX)
+		PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+		RGXSRVHWPerfSyncCheckpointUFOIsSignalled(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+
+		switch (ui32State)
+		{
+			case PVRSRV_SYNC_CHECKPOINT_UNDEF:
+			case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			{
+				break;
+			}
+			case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			{
+				bRet = IMG_TRUE;
+				break;
+			}
+			default:
+			{
+
+				PVRSRV_DEVICE_NODE *psDeviceNode = psContext->psContextCtl->psDeviceNode;
+
+#if defined(SUPPORT_RGX)
+				RGXUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+#endif
+				if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: PCI error - state=0x%x, ID=%d, %s, fwaddr=%#08x",
+							 __func__, ui32State, psSyncCheckpoint->ui32UID,
+							 psSyncCheckpoint->azName, psSyncCheckpoint->ui32FWAddr));
+					bRet = IMG_TRUE;
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: unknown state (0x%x) - ID=%d, %s, fwaddr=%#08x",
+							 __func__, ui32State, psSyncCheckpoint->ui32UID,
+							 psSyncCheckpoint->azName, psSyncCheckpoint->ui32FWAddr));
+				}
+				break;
+			}
+
+		}
+
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s called for psSyncCheckpoint<%p>, returning %d",
+				__func__,
+				(void*)psSyncCheckpoint,
+				bRet));
+#endif
+	}
+	return bRet;
+}
+
+IMG_BOOL
+SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+#if defined(SUPPORT_RGX)
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+		PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+		RGXSRVHWPerfSyncCheckpointUFOIsSignalled(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+		bRet = (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_ERRORED);
+
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s called for psSyncCheckpoint<%p>, returning %d",
+				__func__,
+				(void*)psSyncCheckpoint,
+				bRet));
+#endif
+	}
+	return bRet;
+}
+
+const IMG_CHAR *
+SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid", "Null");
+
+	switch (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State)
+	{
+		case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			return "Signalled";
+		case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			return "Active";
+		case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			return "Errored";
+		case PVRSRV_SYNC_CHECKPOINT_UNDEF:
+			return "Undefined";
+		default:
+			return "Unknown";
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	PVRSRV_ERROR eRet = PVRSRV_OK;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncCheckpoint, "psSyncCheckpoint");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount),
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount)+1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	OSAtomicIncrement(&psSyncCheckpointInt->hRefCount);
+
+	return eRet;
+}
+
+PVRSRV_ERROR
+SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	PVRSRV_ERROR eRet = PVRSRV_OK;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncCheckpoint, "psSyncCheckpoint");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount),
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount)-1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	SyncCheckpointUnref(psSyncCheckpointInt);
+
+	return eRet;
+}
+
+void
+SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psSyncCheckpointInt != NULL, "psSyncCheckpoint");
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(psSyncCheckpointInt);
+#else /* !defined(NO_HARDWARE) */
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			(void*)psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+			OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)+1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	OSAtomicIncrement(&psSyncCheckpointInt->hEnqueuedCCBCount);
+#endif
+}
+
+PRGXFWIF_UFO_ADDR*
+SyncCheckpointGetRGXFWIFUFOAddr(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if defined(DEBUG)
+	if (psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE)
+#endif
+	{
+		return &psSyncCheckpointInt->sCheckpointUFOAddr;
+	}
+#if defined(DEBUG)
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called for psSyncCheckpoint<%p>, but ui32ValidationCheck=0x%x",
+				__func__,
+				(void*)psSyncCheckpointInt,
+				psSyncCheckpointInt->ui32ValidationCheck));
+	}
+#endif
+
+invalid_chkpt:
+	return NULL;
+}
+
+IMG_UINT32
+SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	IMG_UINT32 ui32Ret = 0;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if defined(DEBUG)
+	if (psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE)
+#endif
+	{
+		ui32Ret = psSyncCheckpointInt->ui32FWAddr;
+	}
+#if defined(DEBUG)
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called for psSyncCheckpoint<%p>, but ui32ValidationCheck=0x%x",
+				__func__,
+				(void*)psSyncCheckpointInt,
+				psSyncCheckpointInt->ui32ValidationCheck));
+	}
+#endif
+
+invalid_chkpt:
+	return ui32Ret;
+}
+
+IMG_UINT32
+SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	IMG_UINT32 ui32Ret = 0U;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s returning ID for sync checkpoint<%p>",
+			__func__,
+			(void*)psSyncCheckpointInt));
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s (validationCheck=0x%x)",
+			__func__,
+			psSyncCheckpointInt->ui32ValidationCheck));
+#endif
+#endif
+	ui32Ret = psSyncCheckpointInt->ui32UID;
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s (ui32UID=0x%x)",
+			__func__,
+			psSyncCheckpointInt->ui32UID));
+#endif
+
+invalid_chkpt:
+	return ui32Ret;
+}
+
+PVRSRV_TIMELINE
+SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+	return psSyncCheckpointInt->hTimeline;
+
+invalid_chkpt:
+	return 0;
+}
+
+
+IMG_UINT32
+SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount);
+}
+
+IMG_UINT32
+SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return OSAtomicRead(&psSyncCheckpointInt->hRefCount);
+}
+
+IMG_PID
+SyncCheckpointGetCreator(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return psSyncCheckpointInt->uiProcess;
+}
+
+IMG_UINT32 SyncCheckpointStateFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt;
+	PDLLIST_NODE psNode, psNext;
+	IMG_UINT32 ui32State = 0;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+	{
+		psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+		if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+		{
+			ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+			break;
+		}
+	}
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	return ui32State;
+}
+
+void SyncCheckpointErrorFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt;
+	PDLLIST_NODE psNode, psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s called to error UFO with ui32FWAddr=%d",
+			__func__,
+			ui32FwAddr));
+#endif
+
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+	{
+		psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+		if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+		{
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s calling SyncCheckpointError for sync checkpoint <%p>",
+					__func__,
+					(void*)psSyncCheckpointInt));
+#endif
+			/* Mark as errored */
+			SyncCheckpointError((PSYNC_CHECKPOINT)psSyncCheckpointInt, PVRSRV_FENCE_FLAG_NONE);
+			break;
+		}
+	}
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+}
+
+void SyncCheckpointRollbackFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr)
+{
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s called to rollback UFO with ui32FWAddr=0x%x",
+			__func__,
+			ui32FwAddr));
+#endif
+#if !defined(NO_HARDWARE)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpointInt = NULL;
+		PDLLIST_NODE psNode = NULL, psNext = NULL;
+		OS_SPINLOCK_FLAGS uiFlags;
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+		{
+			psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+			if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+			{
+#if ((ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)) || (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s called for psSyncCheckpointInt<%p> %d->%d",
+				        __func__,
+				        (void *) psSyncCheckpointInt,
+				        OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+				        OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount) - 1));
+#endif
+				OSAtomicDecrement(&psSyncCheckpointInt->hEnqueuedCCBCount);
+				break;
+			}
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+#endif
+}
+
+PVRSRV_ERROR SyncCheckpointRollbackExportFence(PVRSRV_FENCE hExportFence)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnExportFenceRollback))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+		        __func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "pfnExportFenceRollback is NULL");
+		return eError;
+	}
+
+	if (hExportFence != PVRSRV_NO_FENCE)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnExportFenceRollback(hExportFence);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnExportFenceRollback");
+
+	return eError;
+}
+
+static void _SyncCheckpointState(PDLLIST_NODE psNode,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	SYNC_CHECKPOINT *psSyncCheckpoint = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+
+	if (psSyncCheckpoint->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+	{
+		PVR_DUMPDEBUG_LOG("\t- ID = %d, FWAddr = 0x%08x, r%d:e%d:f%d: %s",
+		                   psSyncCheckpoint->ui32UID,
+		                   psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+		                   _SyncCheckpointGetOffset(psSyncCheckpoint),
+		                   OSAtomicRead(&psSyncCheckpoint->hRefCount),
+		                   OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+		                   psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+		                   psSyncCheckpoint->azName);
+	}
+}
+
+static void _SyncCheckpointDebugRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	DLLIST_NODE *psNode, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psDevNode->hSyncCheckpointContext == NULL) return;
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		PVR_DUMPDEBUG_LOG("------[ Active Sync Checkpoints ]------");
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+		PVR_DUMPDEBUG_LOG("(SyncCP Counts: InUse:%d Max:%d)",
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32CurrentInUseSyncCheckpoints,
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32MaxInUseSyncCheckpoints);
+		PVR_DUMPDEBUG_LOG("(SyncCP Mirrored Counts: InUse:%d Max:%d)",
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs,
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32MaxInUseMirroredSyncCPs);
+		OSSpinLockRelease(psDevNode->hSyncCheckpointContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+		{
+			_SyncCheckpointState(psNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointInit(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+#endif
+
+	eError = OSSpinLockCreate(&psDevNode->hSyncCheckpointListLock);
+	PVR_RETURN_IF_ERROR(eError);
+
+	dllist_init(&psDevNode->sSyncCheckpointSyncsList);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncCheckpointNotify,
+		                                          psDevNode,
+		                                          _SyncCheckpointDebugRequest,
+		                                          DEBUG_REQUEST_SYNCCHECKPOINT,
+		                                          (PVRSRV_DBGREQ_HANDLE)psDevNode);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		_SyncCheckpointRecordListInit(psDevNode);
+	}
+
+#if defined(PDUMP)
+	eError = OSSpinLockCreate(&psDevInfo->hSyncCheckpointSignalSpinLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+		goto e1;
+	}
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointSignalLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hSyncCheckpointSignalLock = NULL;
+		goto e2;
+	}
+
+	psDevNode->pui8DeferredSyncCPSignal = OSAllocMem(SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL
+	                                                      * sizeof(_SYNC_CHECKPOINT_DEFERRED_SIGNAL));
+	PVR_GOTO_IF_NOMEM(psDevNode->pui8DeferredSyncCPSignal, eError, e3);
+
+	psDevNode->ui16SyncCPWriteIdx = 0;
+	psDevNode->ui16SyncCPReadIdx = 0;
+
+	eError = OSInstallMISR(&psDevNode->pvSyncCPMISR,
+					MISRHandler_PdumpDeferredSyncSignalPoster,
+					psDevNode,
+					"RGX_PdumpDeferredSyncSignalPoster");
+	PVR_GOTO_IF_ERROR(eError, e4);
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointContextListLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hSyncCheckpointContextListLock = NULL;
+		goto e5;
+	}
+
+
+	dllist_init(&psDevNode->sSyncCheckpointContextListHead);
+
+	eError = PDumpRegisterTransitionCallbackFenceSync(psDevNode,
+								_SyncCheckpointPDumpTransition,
+								&psDevNode->hTransition);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hTransition = NULL;
+		goto e6;
+	}
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PDUMP)
+e6:
+	OSLockDestroy(psDevNode->hSyncCheckpointContextListLock);
+	psDevNode->hSyncCheckpointContextListLock = NULL;
+e5:
+	(void) OSUninstallMISR(psDevNode->pvSyncCPMISR);
+	psDevNode->pvSyncCPMISR = NULL;
+e4:
+	if (psDevNode->pui8DeferredSyncCPSignal)
+	{
+		OSFreeMem(psDevNode->pui8DeferredSyncCPSignal);
+		psDevNode->pui8DeferredSyncCPSignal = NULL;
+	}
+e3:
+	OSLockDestroy(psDevNode->hSyncCheckpointSignalLock);
+	psDevNode->hSyncCheckpointSignalLock = NULL;
+e2:
+	OSSpinLockDestroy(psDevInfo->hSyncCheckpointSignalSpinLock);
+	psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+e1:
+	_SyncCheckpointRecordListDeinit(psDevNode);
+#endif
+e0:
+	OSSpinLockDestroy(psDevNode->hSyncCheckpointListLock);
+	psDevNode->hSyncCheckpointListLock = NULL;
+
+	return eError;
+}
+
+void SyncCheckpointDeinit(PPVRSRV_DEVICE_NODE psDevNode)
+{
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+	PDumpUnregisterTransitionCallbackFenceSync(psDevNode->hTransition);
+	psDevNode->hTransition = NULL;
+
+	if (psDevNode->hSyncCheckpointContextListLock)
+	{
+		OSLockDestroy(psDevNode->hSyncCheckpointContextListLock);
+		psDevNode->hSyncCheckpointContextListLock = NULL;
+	}
+
+	if (psDevNode->pvSyncCPMISR)
+	{
+		(void) OSUninstallMISR(psDevNode->pvSyncCPMISR);
+		psDevNode->pvSyncCPMISR = NULL;
+	}
+
+	if (psDevNode->pui8DeferredSyncCPSignal)
+	{
+		OSFreeMem(psDevNode->pui8DeferredSyncCPSignal);
+		psDevNode->pui8DeferredSyncCPSignal = NULL;
+	}
+	if (psDevNode->hSyncCheckpointSignalLock)
+	{
+		OSLockDestroy(psDevNode->hSyncCheckpointSignalLock);
+		psDevNode->hSyncCheckpointSignalLock = NULL;
+	}
+	if (psDevInfo->hSyncCheckpointSignalSpinLock)
+	{
+		OSSpinLockDestroy(psDevInfo->hSyncCheckpointSignalSpinLock);
+		psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+	}
+#endif
+
+	PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncCheckpointNotify);
+	psDevNode->hSyncCheckpointNotify = NULL;
+	OSSpinLockDestroy(psDevNode->hSyncCheckpointListLock);
+	psDevNode->hSyncCheckpointListLock = NULL;
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		_SyncCheckpointRecordListDeinit(psDevNode);
+	}
+}
+
+void SyncCheckpointRecordLookup(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr,
+                                IMG_CHAR * pszSyncInfo, size_t len)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (!pszSyncInfo)
+	{
+		return;
+	}
+
+	pszSyncInfo[0] = '\0';
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+	{
+		struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+		if ((psSyncCheckpointRec->ui32FwBlockAddr + psSyncCheckpointRec->ui32SyncOffset + 1) == ui32FwAddr)
+		{
+			SYNC_CHECKPOINT_BLOCK *psSyncCheckpointBlock = psSyncCheckpointRec->psSyncCheckpointBlock;
+			if (psSyncCheckpointBlock && psSyncCheckpointBlock->pui32LinAddr)
+			{
+				void *pSyncCheckpointAddr = IMG_OFFSET_ADDR(psSyncCheckpointBlock->pui32LinAddr,
+													psSyncCheckpointRec->ui32SyncOffset);
+				OSSNPrintf(pszSyncInfo, len, "%s Checkpoint:%05u (%s)",
+				           (*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ?
+				                   "SIGNALLED" :
+				                   ((*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_ERRORED) ?
+				                           "ERRORED" : "ACTIVE"),
+				                           psSyncCheckpointRec->uiPID,
+				                           psSyncCheckpointRec->szClassName);
+			}
+			else
+			{
+				OSSNPrintf(pszSyncInfo, len, "Checkpoint:%05u (%s)",
+				           psSyncCheckpointRec->uiPID,
+				           psSyncCheckpointRec->szClassName);
+			}
+
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	if (!bFound && (psDevNode->ui32SyncCheckpointRecordCountHighWatermark == SYNC_CHECKPOINT_RECORD_LIMIT))
+	{
+		OSSNPrintf(pszSyncInfo, len, "(Record may be lost)");
+	}
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordAdd(
+			PSYNC_CHECKPOINT_RECORD_HANDLE * phRecord,
+			SYNC_CHECKPOINT_BLOCK *hSyncCheckpointBlock,
+			IMG_UINT32 ui32FwBlockAddr,
+			IMG_UINT32 ui32SyncOffset,
+			IMG_UINT32 ui32UID,
+			IMG_UINT32 ui32ClassNameSize,
+			const IMG_CHAR *pszClassName, PSYNC_CHECKPOINT pSyncCheckpt)
+{
+	struct SYNC_CHECKPOINT_RECORD * psSyncRec;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)hSyncCheckpointBlock->psContext;
+	PVRSRV_DEVICE_NODE *psDevNode = psContext->psContextCtl->psDeviceNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_INVALID_PARAM(phRecord);
+
+	*phRecord = NULL;
+
+	psSyncRec = OSAllocMem(sizeof(*psSyncRec));
+	PVR_LOG_GOTO_IF_NOMEM(psSyncRec, eError, fail_alloc); /* Sets OOM error code */
+
+	psSyncRec->psSyncCheckpointBlock = hSyncCheckpointBlock;
+	psSyncRec->ui32SyncOffset = ui32SyncOffset;
+	psSyncRec->ui32FwBlockAddr = ui32FwBlockAddr;
+	psSyncRec->ui64OSTime = OSClockns64();
+	psSyncRec->uiPID = OSGetCurrentProcessID();
+	psSyncRec->ui32UID = ui32UID;
+	psSyncRec->pSyncCheckpt = pSyncCheckpt;
+	if (pszClassName)
+	{
+		if (ui32ClassNameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			ui32ClassNameSize = PVRSRV_SYNC_NAME_LENGTH;
+		/* Copy over the class name annotation */
+		OSStringSafeCopy(psSyncRec->szClassName, pszClassName, ui32ClassNameSize);
+	}
+	else
+	{
+		/* No class name annotation */
+		psSyncRec->szClassName[0] = 0;
+	}
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	if (psDevNode->ui32SyncCheckpointRecordCount < SYNC_CHECKPOINT_RECORD_LIMIT)
+	{
+		dllist_add_to_head(&psDevNode->sSyncCheckpointRecordList, &psSyncRec->sNode);
+		psDevNode->ui32SyncCheckpointRecordCount++;
+
+		if (psDevNode->ui32SyncCheckpointRecordCount > psDevNode->ui32SyncCheckpointRecordCountHighWatermark)
+		{
+			psDevNode->ui32SyncCheckpointRecordCountHighWatermark = psDevNode->ui32SyncCheckpointRecordCount;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync checkpoint record \"%s\". %u records already exist.",
+				__func__,
+				pszClassName,
+				psDevNode->ui32SyncCheckpointRecordCount));
+		OSFreeMem(psSyncRec);
+		psSyncRec = NULL;
+		eError = PVRSRV_ERROR_TOOMANYBUFFERS;
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	*phRecord = (PSYNC_CHECKPOINT_RECORD_HANDLE)psSyncRec;
+
+fail_alloc:
+	return eError;
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordRemove(PSYNC_CHECKPOINT_RECORD_HANDLE hRecord)
+{
+	struct SYNC_CHECKPOINT_RECORD **ppFreedSync;
+	struct SYNC_CHECKPOINT_RECORD *pSync = (struct SYNC_CHECKPOINT_RECORD*)hRecord;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = pSync->psSyncCheckpointBlock->psContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_RETURN_IF_INVALID_PARAM(hRecord);
+
+	psDevNode = psContext->psContextCtl->psDeviceNode;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+
+	dllist_remove_node(&pSync->sNode);
+
+	if (psDevNode->uiSyncCheckpointRecordFreeIdx >= PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: psDevNode->uiSyncCheckpointRecordFreeIdx out of range",
+				__func__));
+		psDevNode->uiSyncCheckpointRecordFreeIdx = 0;
+	}
+	ppFreedSync = &psDevNode->apsSyncCheckpointRecordsFreed[psDevNode->uiSyncCheckpointRecordFreeIdx];
+	psDevNode->uiSyncCheckpointRecordFreeIdx =
+			(psDevNode->uiSyncCheckpointRecordFreeIdx + 1) % PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN;
+
+	if (*ppFreedSync)
+	{
+		OSFreeMem(*ppFreedSync);
+	}
+	pSync->psSyncCheckpointBlock = NULL;
+	pSync->ui64OSTime = OSClockns64();
+	*ppFreedSync = pSync;
+
+	psDevNode->ui32SyncCheckpointRecordCount--;
+
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	return PVRSRV_OK;
+}
+
+#define NS_IN_S (1000000000UL)
+static void _SyncCheckpointRecordPrint(struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec,
+                                       IMG_UINT64 ui64TimeNow,
+                                       DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                       void *pvDumpDebugFile)
+{
+	SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT *)psSyncCheckpointRec->pSyncCheckpt;
+	SYNC_CHECKPOINT_BLOCK *psSyncCheckpointBlock = psSyncCheckpointRec->psSyncCheckpointBlock;
+	IMG_UINT64 ui64DeltaS;
+	IMG_UINT32 ui32DeltaF;
+	IMG_UINT64 ui64Delta = ui64TimeNow - psSyncCheckpointRec->ui64OSTime;
+	ui64DeltaS = OSDivide64(ui64Delta, NS_IN_S, &ui32DeltaF);
+
+	if (psSyncCheckpointBlock && psSyncCheckpointBlock->pui32LinAddr)
+	{
+		void *pSyncCheckpointAddr;
+		pSyncCheckpointAddr = IMG_OFFSET_ADDR(psSyncCheckpointBlock->pui32LinAddr,
+											psSyncCheckpointRec->ui32SyncOffset);
+
+		PVR_DUMPDEBUG_LOG("\t%05u %05" IMG_UINT64_FMTSPEC ".%09u %010u "
+		                  "FWAddr=0x%08x (r%d:e%d:f%d) State=%s (%s)",
+		                  psSyncCheckpointRec->uiPID,
+		                  ui64DeltaS, ui32DeltaF, psSyncCheckpointRec->ui32UID,
+		                  (psSyncCheckpointRec->ui32FwBlockAddr+psSyncCheckpointRec->ui32SyncOffset),
+		                  OSAtomicRead(&psSyncCheckpoint->hRefCount),
+	                      OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+	                      psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+		                  (*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ?
+		                          "SIGNALLED" :
+		                          ((*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_ERRORED) ?
+		                                  "ERRORED" : "ACTIVE"),
+		                                  psSyncCheckpointRec->szClassName);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("\t%05u %05" IMG_UINT64_FMTSPEC ".%09u %010u FWAddr=0x%08x State=<null_ptr> (%s)",
+		                  psSyncCheckpointRec->uiPID,
+		                  ui64DeltaS, ui32DeltaF, psSyncCheckpointRec->ui32UID,
+		                  (psSyncCheckpointRec->ui32FwBlockAddr+psSyncCheckpointRec->ui32SyncOffset),
+		                  psSyncCheckpointRec->szClassName
+		);
+	}
+}
+
+static void _SyncCheckpointRecordRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                         IMG_UINT32 ui32VerbLevel,
+                                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                         void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	IMG_UINT64 ui64TimeNowS;
+	IMG_UINT32 ui32TimeNowF;
+	IMG_UINT64 ui64TimeNow = OSClockns64();
+	DLLIST_NODE *psNode, *psNext;
+
+	ui64TimeNowS = OSDivide64(ui64TimeNow, NS_IN_S, &ui32TimeNowF);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+
+		OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+
+		PVR_DUMPDEBUG_LOG("Dumping allocated sync checkpoints. Allocated: %u High watermark: %u (time ref %05" IMG_UINT64_FMTSPEC ".%09u)",
+		                  psDevNode->ui32SyncCheckpointRecordCount,
+		                  psDevNode->ui32SyncCheckpointRecordCountHighWatermark,
+		                  ui64TimeNowS,
+		                  ui32TimeNowF);
+		if (psDevNode->ui32SyncCheckpointRecordCountHighWatermark == SYNC_CHECKPOINT_RECORD_LIMIT)
+		{
+			PVR_DUMPDEBUG_LOG("Warning: Record limit (%u) was reached. Some sync checkpoints may not have been recorded in the debug information.",
+			                  SYNC_CHECKPOINT_RECORD_LIMIT);
+		}
+		PVR_DUMPDEBUG_LOG("\t%-5s %-15s %-10s %-17s %-14s (%s)",
+		                  "PID", "Time Delta (s)", "UID", "Address", "State", "Annotation");
+
+		dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+		{
+			struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec =
+					IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+			_SyncCheckpointRecordPrint(psSyncCheckpointRec, ui64TimeNow,
+			                           pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+
+		PVR_DUMPDEBUG_LOG("Dumping all recently freed sync checkpoints @ %05" IMG_UINT64_FMTSPEC ".%09u",
+		                  ui64TimeNowS,
+		                  ui32TimeNowF);
+		PVR_DUMPDEBUG_LOG("\t%-5s %-15s %-10s %-17s %-14s (%s)",
+		                  "PID", "Time Delta (s)", "UID", "Address", "State", "Annotation");
+		for (i = DECREMENT_WITH_WRAP(psDevNode->uiSyncCheckpointRecordFreeIdx, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN);
+				i != psDevNode->uiSyncCheckpointRecordFreeIdx;
+				i = DECREMENT_WITH_WRAP(i, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN))
+		{
+			if (psDevNode->apsSyncCheckpointRecordsFreed[i])
+			{
+				_SyncCheckpointRecordPrint(psDevNode->apsSyncCheckpointRecordsFreed[i],
+				                           ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+			else
+			{
+				break;
+			}
+		}
+		OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+	}
+}
+#undef NS_IN_S
+static PVRSRV_ERROR _SyncCheckpointRecordListInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointRecordLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lock_create);
+	dllist_init(&psDevNode->sSyncCheckpointRecordList);
+
+	psDevNode->ui32SyncCheckpointRecordCount = 0;
+	psDevNode->ui32SyncCheckpointRecordCountHighWatermark = 0;
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncCheckpointRecordNotify,
+	                                              psDevNode,
+	                                              _SyncCheckpointRecordRequest,
+	                                              DEBUG_REQUEST_SYNCCHECKPOINT,
+	                                              (PVRSRV_DBGREQ_HANDLE)psDevNode);
+	PVR_GOTO_IF_ERROR(eError, fail_dbg_register);
+
+	return PVRSRV_OK;
+
+fail_dbg_register:
+	OSLockDestroy(psDevNode->hSyncCheckpointRecordLock);
+fail_lock_create:
+	return eError;
+}
+
+static void _SyncCheckpointRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DLLIST_NODE *psNode, *psNext;
+	int i;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+	{
+		struct SYNC_CHECKPOINT_RECORD *pSyncCheckpointRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+
+		dllist_remove_node(psNode);
+		OSFreeMem(pSyncCheckpointRec);
+	}
+
+	for (i = 0; i < PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN; i++)
+	{
+		if (psDevNode->apsSyncCheckpointRecordsFreed[i])
+		{
+			OSFreeMem(psDevNode->apsSyncCheckpointRecordsFreed[i]);
+			psDevNode->apsSyncCheckpointRecordsFreed[i] = NULL;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	if (psDevNode->hSyncCheckpointRecordNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncCheckpointRecordNotify);
+	}
+	OSLockDestroy(psDevNode->hSyncCheckpointRecordLock);
+}
+
+#if defined(PDUMP)
+
+static PVRSRV_ERROR
+_SyncCheckpointAllocPDump(PVRSRV_DEVICE_NODE *psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Allocated Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+	                      psSyncCheckpoint->azName,
+	                      psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+	                      psSyncCheckpoint->sCheckpointUFOAddr.ui32Addr);
+
+	DevmemPDumpLoadMemValue32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+	                          _SyncCheckpointGetOffset(psSyncCheckpoint),
+	                          PVRSRV_SYNC_CHECKPOINT_ACTIVE,
+	                          PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointUpdatePDump(PPVRSRV_DEVICE_NODE psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint, IMG_UINT32 ui32Status, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bSleepAllowed = (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	 */
+	if (bSleepAllowed)
+	{
+		if (ui32Status == PVRSRV_SYNC_CHECKPOINT_ERRORED)
+		{
+			PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+					"Errored Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+					psSyncCheckpoint->azName,
+					psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+					(psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+					_SyncCheckpointGetOffset(psSyncCheckpoint)));
+		}
+		else
+		{
+			PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+					"Signalled Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+					psSyncCheckpoint->azName,
+					psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+					(psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+					_SyncCheckpointGetOffset(psSyncCheckpoint)));
+		}
+
+		DevmemPDumpLoadMemValue32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+		                          _SyncCheckpointGetOffset(psSyncCheckpoint),
+		                          ui32Status,
+		                          PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		_SYNC_CHECKPOINT_DEFERRED_SIGNAL *psSyncData;
+		OS_SPINLOCK_FLAGS uiFlags;
+		IMG_UINT16 ui16NewWriteIdx;
+
+		OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+
+		ui16NewWriteIdx = GET_CP_CB_NEXT_IDX(psDevNode->ui16SyncCPWriteIdx);
+		if (ui16NewWriteIdx == psDevNode->ui16SyncCPReadIdx)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR Deferred SyncCheckpointSignal CB is full)",
+				__func__));
+		}
+		else
+		{
+			psSyncData = GET_CP_CB_BASE(psDevNode->ui16SyncCPWriteIdx);
+			psSyncData->asSyncCheckpoint = *psSyncCheckpoint;
+			psSyncData->ui32Status = ui32Status;
+			psDevNode->ui16SyncCPWriteIdx = ui16NewWriteIdx;
+		}
+
+		OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+
+		OSScheduleMISR(psDevNode->pvSyncCPMISR);
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+MISRHandler_PdumpDeferredSyncSignalPoster(void *pvData)
+{
+	PPVRSRV_DEVICE_NODE psDevNode = (PPVRSRV_DEVICE_NODE) pvData;
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT16 ui16ReadIdx, ui16WriteIdx;
+	_SYNC_CHECKPOINT_DEFERRED_SIGNAL *psSyncData;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointSignalLock);
+
+	OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+	/* Snapshot current write and read offset of CB */
+	ui16WriteIdx = psDevNode->ui16SyncCPWriteIdx;
+	ui16ReadIdx = psDevNode->ui16SyncCPReadIdx;
+
+	OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+	/* CB is empty */
+	if (ui16WriteIdx == ui16ReadIdx)
+	{
+		OSLockRelease(psDevNode->hSyncCheckpointSignalLock);
+		return;
+	}
+	do
+	{
+		/* Read item in the CB and flush it to pdump */
+		psSyncData = GET_CP_CB_BASE(ui16ReadIdx);
+		_SyncCheckpointUpdatePDump(psDevNode, &psSyncData->asSyncCheckpoint, psSyncData->ui32Status, PVRSRV_FENCE_FLAG_NONE);
+		ui16ReadIdx = GET_CP_CB_NEXT_IDX(psDevNode->ui16SyncCPReadIdx);
+		/* Increment read offset in CB as one item is flushed to pdump */
+		OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+		psDevNode->ui16SyncCPReadIdx = ui16ReadIdx;
+		OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+		/* Call to this function will flush all the items present in CB
+		 * when this function is called i.e. use snapshot of WriteOffset
+		 * taken at the beginning in this function and iterate till Write != Read */
+	} while (ui16WriteIdx != ui16ReadIdx);
+
+	OSLockRelease(psDevNode->hSyncCheckpointSignalLock);
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR PVRSRVSyncCheckpointSignalledPDumpPolKM(PVRSRV_FENCE hFence)
+{
+	PVRSRV_ERROR eError;
+	PSYNC_CHECKPOINT *apsCheckpoints = NULL;
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+	IMG_UINT32 i, uiNumCheckpoints = 0;
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+
+	if (hFence != PVRSRV_NO_FENCE)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnSyncFenceGetCheckpoints(hFence, &uiNumCheckpoints, &apsCheckpoints);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_pfnFenceGetCheckpoints");
+
+	if (uiNumCheckpoints)
+	{
+		/* Flushing deferred fence signals to pdump */
+		psSyncCheckpoint = (SYNC_CHECKPOINT *)apsCheckpoints[0];
+		psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+		MISRHandler_PdumpDeferredSyncSignalPoster(psContext->psContextCtl->psDeviceNode);
+	}
+
+	for (i=0; i < uiNumCheckpoints; i++)
+	{
+		psSyncCheckpoint = (SYNC_CHECKPOINT *)apsCheckpoints[i];
+		if (psSyncCheckpoint->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED)
+		{
+			psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+			PDUMPCOMMENTWITHFLAGS(psContext->psContextCtl->psDeviceNode,
+							psSyncCheckpoint->ui32PDumpFlags,
+							"Wait for Fence %s (ID:%d)",
+							psSyncCheckpoint->azName,
+							psSyncCheckpoint->ui32UID);
+
+			eError = DevmemPDumpDevmemPol32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+								_SyncCheckpointGetOffset(psSyncCheckpoint),
+								PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+								0xFFFFFFFF,
+								PDUMP_POLL_OPERATOR_EQUAL,
+								psSyncCheckpoint->ui32PDumpFlags);
+			PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+		}
+	}
+
+
+
+	/* Free the memory that was allocated for the sync checkpoint list returned */
+	if (apsCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsCheckpoints);
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* #if defined(PDUMP) */
+
+static PVRSRV_ERROR
+_SyncCheckpointPDumpTransition(void *pvData, PDUMP_TRANSITION_EVENT eEvent)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+	DLLIST_NODE *psNode, *psNext;
+	DLLIST_NODE *psNode1, *psNext1;
+	PPVRSRV_DEVICE_NODE psDevNode = (PPVRSRV_DEVICE_NODE) pvData;
+
+	if ((eEvent == PDUMP_TRANSITION_EVENT_RANGE_ENTERED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		OSLockAcquire(psDevNode->hSyncCheckpointContextListLock);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointContextListHead, psNode, psNext)
+		{
+			psContext = IMG_CONTAINER_OF(psNode, _SYNC_CHECKPOINT_CONTEXT, sListNode);
+
+			OSLockAcquire(psContext->hSyncCheckpointBlockListLock);
+			dllist_foreach_node(&psContext->sSyncCheckpointBlockListHead, psNode1, psNext1)
+			{
+				SYNC_CHECKPOINT_BLOCK *psSyncBlk =
+					IMG_CONTAINER_OF(psNode1, SYNC_CHECKPOINT_BLOCK, sListNode);
+				DevmemPDumpLoadMem(psSyncBlk->hMemDesc,
+							   0,
+							   psSyncBlk->ui32SyncBlockSize,
+							   PDUMP_FLAGS_CONTINUOUS);
+			}
+			OSLockRelease(psContext->hSyncCheckpointBlockListLock);
+		}
+		OSLockRelease(psDevNode->hSyncCheckpointContextListLock);
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+static void _CheckDeferredCleanupList(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE*)psCtxCtl->psDeviceNode;
+	DECLARE_DLLIST(sCleanupList);
+	DLLIST_NODE *psNode, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_ERROR eError;
+
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called", __func__));
+#endif
+
+	/* Check the deferred cleanup list and free any sync checkpoints we can */
+	OSSpinLockAcquire(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+
+	if (dllist_is_empty(&psCtxCtl->sDeferredCleanupListHead))
+	{
+		OSSpinLockRelease(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING, "%s: Defer free list is empty", __func__));
+#endif
+		/* if list is empty then we have nothing to do here */
+		return;
+	}
+
+	dllist_foreach_node(&psCtxCtl->sDeferredCleanupListHead, psNode, psNext)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpointInt =
+				IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sDeferredFreeListNode);
+
+		if (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount ==
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)))
+		{
+			if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+				&& psSyncCheckpointInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = _SyncCheckpointRecordRemove(psSyncCheckpointInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "_SyncCheckpointRecordRemove");
+			}
+
+			/* Move the sync checkpoint from the deferred free list to local list */
+			dllist_remove_node(&psSyncCheckpointInt->sDeferredFreeListNode);
+			/* It's not an ideal solution to traverse list of checkpoints-to-free
+			 * twice but it allows us to avoid holding the lock for too long */
+			dllist_add_to_tail(&sCleanupList, &psSyncCheckpointInt->sDeferredFreeListNode);
+		}
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s psSyncCheckpoint '%s'' (ID:%d)<%p>), "
+			        "still pending (enq=%d,FWRef=%d)", __func__,
+			        psSyncCheckpointInt->azName, psSyncCheckpointInt->ui32UID,
+			        (void*)psSyncCheckpointInt,
+			        (IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)),
+			        psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+		}
+#endif
+	}
+
+	OSSpinLockRelease(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+
+	dllist_foreach_node(&sCleanupList, psNode, psNext) {
+		SYNC_CHECKPOINT *psSyncCheckpointInt =
+				IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sDeferredFreeListNode);
+
+		/* Remove the sync checkpoint from the global list */
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_remove_node(&psSyncCheckpointInt->sListNode);
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+		RGXSRV_HWPERF_FREE(psDevNode, SYNC_CP, psSyncCheckpointInt->ui32FWAddr);
+
+		/* Unref the checkpoint in use */
+		OSSpinLockAcquire(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+		psContext->psContextCtl->ui32CurrentInUseSyncCheckpoints--;
+		if (psSyncCheckpointInt->hTimeline == SYNC_CHECKPOINT_MIRRORED_CHECKPOINT)
+		{
+			psContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs--;
+		}
+		OSSpinLockRelease(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s attempting to return sync(ID:%d),%p> to pool",
+		        __func__,
+		        psSyncCheckpointInt->ui32UID,
+		        (void *) psSyncCheckpointInt));
+#endif
+		if (!_PutCheckpointInPool(psSyncCheckpointInt))
+#endif
+		{
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING, "%s pool is full, so just free it",
+			        __func__));
+#endif
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+		}
+		else
+		{
+#endif
+			PVR_DPF((PVR_DBG_WARNING,
+			       "%s psSyncCheckpoint '%s'' (ID:%d)<%p>), still pending (enq=%d,FWRef=%d)",
+			        __func__,
+			        psSyncCheckpointInt->azName,
+			        psSyncCheckpointInt->ui32UID,
+			        (void*)psSyncCheckpointInt,
+			        (IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)),
+			        psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+			_FreeSyncCheckpoint(psSyncCheckpointInt);
+		}
+	}
+}
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static SYNC_CHECKPOINT *_GetCheckpointFromPool(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psCtxCtl->ui32SyncCheckpointPoolSize == 0)
+	{
+		goto pool_not_used;
+	}
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* Check if we can allocate from the pool */
+	if (CHECKPOINT_POOL_VALID(psCtxCtl) &&
+#if defined(PDUMP)
+	    psCtxCtl->bAllocateFromCheckpointPool &&
+#endif
+	    (psCtxCtl->ui32SyncCheckpointPoolCount > 0))
+	{
+		/* Get the next sync checkpoint from the pool */
+		psSyncCheckpoint = psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolRp];
+		psCtxCtl->ui32SyncCheckpointPoolRp =
+		        (psCtxCtl->ui32SyncCheckpointPoolRp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount--;
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+#if defined(DEBUG)
+		psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_USE;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s checkpoint(old ID:%d)<-POOL(%d/%d), psContext=<%p>, "
+		        "poolRp=%d, poolWp=%d",
+		        __func__,
+		        psSyncCheckpoint->ui32UID,
+		        psCtxCtl->ui32SyncCheckpointPoolCount,
+		         psCtxCtl->ui32SyncCheckpointPoolSize,
+		        (void *) psContext,
+		        psCtxCtl->ui32SyncCheckpointPoolRp,
+		        psCtxCtl->ui32SyncCheckpointPoolWp));
+#endif
+	}
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+pool_not_used:
+	return psSyncCheckpoint;
+}
+
+static IMG_BOOL _PutCheckpointInPool(SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	IMG_BOOL bReturnedToPool = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psCtxCtl->ui32SyncCheckpointPoolSize == 0)
+	{
+		return IMG_FALSE;
+	}
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* Check if pool has space */
+	if (CHECKPOINT_POOL_VALID(psCtxCtl) &&
+#if defined(PDUMP)
+			(psSyncCheckpoint->psSyncCheckpointBlock == psCtxCtl->psSyncCheckpointPoolBlock) &&
+#endif
+			!(CHECKPOINT_POOL_FULL(psCtxCtl)))
+	{
+		/* Put the sync checkpoint into the next write slot in the pool */
+		psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolWp] = psSyncCheckpoint;
+		psCtxCtl->ui32SyncCheckpointPoolWp =
+		        (psCtxCtl->ui32SyncCheckpointPoolWp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount++;
+		/* Update if the checkpoint that was just added filled up the pool */
+		if (psCtxCtl->ui32SyncCheckpointPoolWp == psCtxCtl->ui32SyncCheckpointPoolRp)
+		{
+			SET_CHECKPOINT_POOL_FULL(psCtxCtl);
+#if defined(PDUMP)
+			/* The pool is now fully allocated and ready to use. */
+			psCtxCtl->bAllocateFromCheckpointPool = IMG_TRUE;
+#endif
+		}
+		bReturnedToPool = IMG_TRUE;
+		psSyncCheckpoint->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_UNDEF;
+#if defined(DEBUG)
+		psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_POOL;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s checkpoint(ID:%d)->POOL(%d/%d), poolRp=%d, poolWp=%d",
+		        __func__,
+		        psSyncCheckpoint->ui32UID,
+		        psCtxCtl->ui32SyncCheckpointPoolCount,
+		        psCtxCtl->ui32SyncCheckpointPoolSize,
+		        psCtxCtl->ui32SyncCheckpointPoolRp,
+		        psCtxCtl->ui32SyncCheckpointPoolWp));
+#endif
+	}
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	return bReturnedToPool;
+}
+
+static IMG_UINT32 _CleanCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	SYNC_CHECKPOINT *psCheckpoint = NULL;
+	DECLARE_DLLIST(sCleanupList);
+	DLLIST_NODE *psThis, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT32 ui32ItemsFreed = 0, ui32NullScpCount = 0;
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* While the pool still contains sync checkpoints, free them */
+	while (CHECKPOINT_POOL_VALID(psCtxCtl) && psCtxCtl->ui32SyncCheckpointPoolCount > 0)
+	{
+		/* Get the sync checkpoint from the next read slot in the pool */
+		psCheckpoint = psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolRp];
+		psCtxCtl->ui32SyncCheckpointPoolRp =
+		        (psCtxCtl->ui32SyncCheckpointPoolRp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount--;
+
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+
+		if (psCheckpoint)
+		{
+			PVR_ASSERT(!dllist_node_is_in_list(&psCheckpoint->sListNode));
+			/* before checkpoints are added to the pool they are removed
+			 * from the list so it's safe to use sListNode here */
+			dllist_add_to_head(&sCleanupList, &psCheckpoint->sListNode);
+		}
+		else
+		{
+			ui32NullScpCount++;
+		}
+	}
+
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* go through the local list and free all of the sync checkpoints */
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s psContext=<%p>, ui8PoolStateFlags=0x%x, "
+	        "uiSyncCheckpointPoolCount=%d", __func__, (void *) psContext,
+	        psCtxCtl->ui8PoolStateFlags, psCtxCtl->ui32SyncCheckpointPoolCount));
+
+	if (ui32NullScpCount > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s pool contained %u NULL entries", __func__,
+		        ui32NullScpCount));
+	}
+#endif
+
+	dllist_foreach_node(&sCleanupList, psThis, psNext)
+	{
+		psCheckpoint = IMG_CONTAINER_OF(psThis, SYNC_CHECKPOINT, sListNode);
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+#if defined(DEBUG)
+		if (psCheckpoint->ui32ValidationCheck != SYNC_CHECKPOINT_PATTERN_IN_POOL)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s pool contains invalid entry "
+			        "(ui32ValidationCheck=0x%x)", __func__,
+			        psCheckpoint->ui32ValidationCheck));
+		}
+#endif
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint(ID:%d)",
+		        __func__, psCheckpoint->ui32UID));
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->ui32ValidationCheck=0x%x",
+		        __func__, psCheckpoint->ui32ValidationCheck));
+#endif
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->uiAllocatedAddr=0x%llx",
+		        __func__, psCheckpoint->uiAllocatedAddr));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock->psContext=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock->psContext));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA));
+
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s CALLING RA_Free(psSyncCheckpoint(ID:%d)<%p>), "
+		        "psSubAllocRA=<%p>, ui32SpanAddr=0x%llx",
+		        __func__,
+		        psCheckpoint->ui32UID,
+		        (void *) psCheckpoint,
+		        (void *) psCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA,
+		        psCheckpoint->uiAllocatedAddr));
+#endif
+
+		dllist_remove_node(psThis);
+
+		_FreeSyncCheckpoint(psCheckpoint);
+		ui32ItemsFreed++;
+	}
+
+	return ui32ItemsFreed;
+}
+#endif /* (SYNC_CHECKPOINT_POOL_LIMIT > 0) */
+
+IMG_BOOL SyncCheckpointCommonDeviceIDs(PSYNC_CHECKPOINT_CONTEXT psContext,
+									   IMG_HANDLE hDevRef)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDevRef;
+
+	if (unlikely((hDevRef == NULL) || (psContext == NULL)))
+	{
+		return IMG_TRUE;
+	}
+
+	return (psContext->psContextCtl->psDeviceNode->sDevId.i32KernelDeviceID ==
+	        psDevNode->sDevId.i32KernelDeviceID) ? IMG_TRUE : IMG_FALSE;
+}
+
+PVRSRV_ERROR SyncCheckpointGetCounters(PPVRSRV_DEVICE_NODE psDevNode,
+                                       IMG_UINT32 *puiInUse,
+                                       IMG_UINT32 *puiMax,
+                                       IMG_UINT32 *puiXDInUse,
+                                       IMG_UINT32 *puiXDMax)
+{
+	PSYNC_CHECKPOINT_CONTEXT psSyncContext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_LOG_RETURN_IF_FALSE((psDevNode != NULL), "psDevNode invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_RETURN_IF_FALSE((psDevNode->hSyncCheckpointContext != NULL), PVRSRV_OK);
+
+	psSyncContext = (PSYNC_CHECKPOINT_CONTEXT)psDevNode->hSyncCheckpointContext;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncContext != NULL), "psSyncContext invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiInUse != NULL), "puiInUse invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiMax != NULL), "puiMax invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiXDInUse != NULL), "puiXDInUse invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((puiXDMax != NULL), "puiXDMax invalid",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSSpinLockAcquire(psSyncContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+	*puiInUse = psSyncContext->psContextCtl->ui32CurrentInUseSyncCheckpoints;
+	*puiMax = psSyncContext->psContextCtl->ui32MaxInUseSyncCheckpoints;
+	*puiXDInUse = psSyncContext->psContextCtl->ui32CurrentInUseMirroredSyncCPs;
+	*puiXDMax = psSyncContext->psContextCtl->ui32MaxInUseMirroredSyncCPs;
+	OSSpinLockRelease(psSyncContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_fallback_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_fallback_server.c
new file mode 100644
index 000000000000..47abd65c83ab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_fallback_server.c
@@ -0,0 +1,2959 @@
+/*************************************************************************/ /*!
+@File           sync_fallback_server.c
+@Title          Fallback implementation of server fence sync interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    The server implementation of software fallback synchronisation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(DEBUG)
+#define PVR_DPF_FUNCTION_TRACE_ON 1
+#endif
+
+#include "pvr_debug.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_sync_km.h"
+#include "pvrsrv_sync_server.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint_internal.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "handle.h"
+#include "pvrsrv.h"
+#include "hash.h"
+#include "rgxhwperf.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+
+
+#include "ossecure_export.h"
+
+/* Refcounting debug.
+ * Define SYNC_FB_REF_DEBUG to print out a reference count log. */
+// #define SYNC_FB_REF_DEBUG 1
+
+#if defined(SYNC_FB_REF_DEBUG)
+#define PRINT_REF(type, opchr, pRef, ptr, name, info, num) \
+	PVR_LOG(("        %s REF(%c) -> %d - %6s: %-5u - %-30s (0x%p)", type, opchr, pRef, info, num, name, ptr))
+#else
+#define PRINT_REF(type, opchr, pRef, ptr, name, info, num)
+#endif
+
+#define REF_SET(type, pRef, val, ptr, name, info, num) OSAtomicWrite(pRef, val); PRINT_REF(type, '=', val, ptr, name, info, num)
+#define REF_INC(type, pRef,      ptr, name, info, num) OSAtomicIncrement(pRef); PRINT_REF(type, '+', OSAtomicRead(pRef), ptr, name, info, num)
+#define REF_DEC(type, pRef,      ptr, name, info, num) OSAtomicDecrement(pRef); PRINT_REF(type, '-', OSAtomicRead(pRef), ptr, name, info, num)
+
+/* Timelines */
+#define TL_REF_SET(pRef, val, ptr) REF_SET("TL", pRef, val, ptr, ptr->pszName, "UID", (IMG_UINT64) ptr->iUID)
+#define TL_REF_INC(pRef, ptr)      REF_INC("TL", pRef,      ptr, ptr->pszName, "UID", (IMG_UINT64) ptr->iUID)
+#define TL_REF_DEC(pRef, ptr)      REF_DEC("TL", pRef,      ptr, ptr->pszName, "UID", (IMG_UINT64) ptr->iUID)
+
+/* Fences */
+#define FENCE_REF_SET(pRef, val, ptr) REF_SET("FE", pRef, val, ptr, ptr->pszName, "#Syncs", ptr->uiNumSyncs)
+#define FENCE_REF_INC(pRef, ptr)      REF_INC("FE", pRef,      ptr, ptr->pszName, "#Syncs", ptr->uiNumSyncs)
+#define FENCE_REF_DEC(pRef, ptr)      REF_DEC("FE", pRef,      ptr, ptr->pszName, "#Syncs", ptr->uiNumSyncs)
+
+/* SyncPt */
+#define PT_REF_SET(pRef, val, ptr) REF_SET("PT", pRef, val, ptr, ptr->psTl->pszName, "SeqNum" ,ptr->uiSeqNum)
+#define PT_REF_INC(pRef, ptr)      REF_INC("PT", pRef,      ptr, ptr->psTl->pszName, "SeqNum" ,ptr->uiSeqNum)
+#define PT_REF_DEC(pRef, ptr)      REF_DEC("PT", pRef,      ptr, ptr->psTl->pszName, "SeqNum" ,ptr->uiSeqNum)
+
+
+/* Simple prints for error and warning */
+#define ERR(msg, ...) PVR_DPF((PVR_DBG_ERROR, \
+                          "%s: " msg, \
+                          __func__, \
+                          ##__VA_ARGS__));
+
+#define WRN(msg, ...) PVR_DPF((PVR_DBG_WARNING, \
+                          "%s: " msg, \
+                          __func__, \
+                          ##__VA_ARGS__));
+
+// #define SYNC_FB_DEBUG 1
+#if defined(SYNC_FB_DEBUG)
+#define DBG(...) PVR_LOG(__VA_ARGS__)
+#else
+#define DBG(...)
+#endif
+
+/* Functions for timelines */
+typedef IMG_BOOL (*PFN_SYNC_PT_HAS_SIGNALLED)(PVRSRV_SYNC_PT *psSyncPt);
+typedef void (*PFN_SYNC_FREE) (IMG_HANDLE hAttachedSync);
+
+
+/* The states a SyncPt can be in */
+typedef enum
+{
+	PVRSRV_SYNC_NOT_SIGNALLED,     /*!< sync pt has not yet signalled */
+	PVRSRV_SYNC_SIGNALLED,         /*!< sync pt has signalled */
+	PVRSRV_SYNC_ERRORED            /*!< sync pt has errored*/
+} PVRSRV_SYNC_STATE;
+
+typedef enum
+{
+	PVRSRV_SYNC_HANDLE_UNKNOWN,
+	PVRSRV_SYNC_HANDLE_PVR,
+	PVRSRV_SYNC_HANDLE_SW
+} PVRSRV_SYNC_HANDLE_TYPE;
+
+typedef struct _PVRSRV_SYNC_SIGNAL_CB_
+{
+	DLLIST_NODE sCallbackNode;
+	IMG_HANDLE  hAttachedSync;
+	IMG_HANDLE  hPrivData;
+	PVRSRV_ERROR (*pfnSignal)(IMG_HANDLE hAttachedSync,
+	                          PVRSRV_SYNC_STATE eState);
+	PFN_SYNC_FREE pfnSyncFree;
+
+} PVRSRV_SYNC_SIGNAL_CB;
+
+struct _PVRSRV_SYNC_PT_
+{
+	/* The timeline this sync pt is associated with */
+	PVRSRV_TIMELINE_SERVER	*psTl;
+	IMG_UINT32				uiSeqNum;
+	/* Refcount */
+	ATOMIC_T				iRef;
+	/* Timeline list. Take TL lock! */
+	DLLIST_NODE				sTlSyncList;
+	/* Timeline active list. Take TL lock! */
+	DLLIST_NODE				sTlSyncActiveList;
+
+	/* List of callbacks to signal attached syncs.
+	 *
+	 * THE FIRST ITEM OF THIS LIST DEFINES THE FLAVOUR OF THE SYNC PT
+	 * AND MUST BE CREATED TOGETHER WITH THE SYNC PT!
+	 * Usually this is done when creating a fence.
+	 * E.g. if PVR has been asked to create a fence we would
+	 * create a sync pt for it with an attached sync checkpoint.
+	 *
+	 * In case someone waits for this sync pt who is not able
+	 * to access the first item, a new foreign sync
+	 * needs to be attached that can be read by the waiter.
+	 * This might be the case if a second device is involved that cannot
+	 * access sync checkpoints of another device or a device that needs
+	 * to wait for a different sync type that it is not able to read
+	 * e.g. a SW sync */
+	DLLIST_NODE				sSignalCallbacks;
+	/* Can have a PVRSRV_SYNC_STATE */
+	ATOMIC_T				iStatus;
+	/* PID of the sync pt creator, used for cleanup-unblocking */
+	IMG_UINT32				uiPID;
+};
+
+/* Definition representing an attached SW sync pt.
+ * This is the counterpart to the SYNC_CHECKPOINTS for syncs that get
+ * signalled by the CPU. */
+typedef struct _PVRSRV_SYNC_PT_SW_
+{
+	 IMG_BOOL bSignalled;
+} PVRSRV_SYNC_PT_SW;
+
+/*! Possible states for a PVRSRV_FENCE */
+typedef enum
+{
+    PVRSRV_FENCE_NOT_SIGNALLED,             /*!< fence has not yet signalled (not all components have signalled) */
+    PVRSRV_FENCE_SIGNALLED                  /*!< fence has signalled (all components have signalled/errored) */
+} PVRSRV_FENCE_STATE;
+
+struct _PVRSRV_FENCE_SERVER_
+{
+	IMG_UINT32			uiNumSyncs;
+	PVRSRV_SYNC_PT		**apsFenceSyncList;
+	ATOMIC_T			iRef;
+	/* Only written to when waiter checks if fence is met */
+	ATOMIC_T			iStatus;
+	IMG_INT64			iUID;
+	IMG_CHAR			pszName[SYNC_FB_FENCE_MAX_LENGTH];
+	PVRSRV_DEVICE_NODE	*psDevNode;
+	DLLIST_NODE			sFenceListNode;
+};
+
+struct _PVRSRV_FENCE_EXPORT_
+{
+	PVRSRV_FENCE_SERVER *psFence;
+};
+
+typedef struct _PVRSRV_TIMELINE_OPS_
+{
+	/* Supposed to be called when someone queries the TL
+	 * to update its active generic syncs */
+	PFN_SYNC_PT_HAS_SIGNALLED pfnSyncPtHasSignalled;
+} PVRSRV_TIMELINE_OPS;
+
+struct _PVRSRV_TIMELINE_SERVER_
+{
+	/* Never take the fence lock after this one */
+	POS_LOCK			hTlLock;
+	/* Timeline list. Contains all sync pts of the timeline that
+	 * were not destroyed. Signalled or unsignalled. Take TL lock! */
+	DLLIST_NODE			sSyncList;
+	/* Timeline active list. Contains all sync pts of the timeline
+	 * that were not signalled yet.
+	 * Before removing node, check if it's still in list. Take TL lock! */
+	DLLIST_NODE			sSyncActiveList;
+	IMG_CHAR			pszName[SYNC_FB_TIMELINE_MAX_LENGTH];
+	ATOMIC_T			iRef;
+	PVRSRV_TIMELINE_OPS sTlOps;
+	DLLIST_NODE			sTlList;
+	/* This ID helps to order the sync pts in a fence when merging */
+	IMG_INT64			iUID;
+	/* The sequence number of the last sync pt created */
+	ATOMIC_T			iSeqNum;
+	/* The sequence number of the latest signalled sync pt */
+	ATOMIC_T			iLastSignalledSeqNum;
+	/* The PID of the process which created this timeline */
+	IMG_PID				uiPID;
+};
+
+typedef struct _SYNC_FB_CONTEXT_DEVICE_LIST_
+{
+	DLLIST_NODE sDeviceListNode;
+	IMG_HANDLE hDBGNotify;
+	PVRSRV_DEVICE_NODE *psDevice;
+} SYNC_FB_CONTEXT_DEVICE_LIST;
+
+typedef struct _SYNC_FB_CONTEXT_
+{
+	IMG_HANDLE hSyncEventObject;
+	IMG_HANDLE hCMDNotify;
+	DLLIST_NODE sDeviceList;
+	DLLIST_NODE sTlList;
+	POS_LOCK hFbContextLock;
+	DLLIST_NODE sFenceList; /* protected by hFbContextLock */
+	HASH_TABLE *sCheckpointHashTable; /* protected by hFbContextLock */
+	PFN_SYNC_CHECKPOINT_STRUCT sSyncCheckpointReg;
+} SYNC_FB_CONTEXT;
+
+/* GLOBALS */
+static SYNC_FB_CONTEXT gsSyncFbContext;
+
+/* Declarations */
+static void _SyncFbTimelineAcquire(PVRSRV_TIMELINE_SERVER *psTl);
+static void _SyncFbFenceAcquire(PVRSRV_FENCE_SERVER *psFence);
+static PVRSRV_ERROR _SyncFbSyncPtSignalAttached(PVRSRV_SYNC_PT *psSyncPt,
+                                        PVRSRV_SYNC_STATE eSignal);
+static PVRSRV_ERROR SyncFbFenceRollbackPVR(PVRSRV_FENCE iFence, void *pvFenceData);
+static PVRSRV_ERROR _SyncFbSyncPtSignalPVR(IMG_HANDLE hSync,
+                                           PVRSRV_SYNC_STATE eState);
+static PVRSRV_ERROR _SyncFbSyncPtSignalSW(IMG_HANDLE hSync,
+                                          PVRSRV_SYNC_STATE eState);
+static IMG_BOOL _SyncFbFenceSyncsHaveSignalled(PVRSRV_FENCE_SERVER *psFence);
+static IMG_BOOL _SyncFbSyncPtHasSignalled(PVRSRV_SYNC_PT *psSyncPt);
+static IMG_BOOL _SyncFbSyncPtHasSignalledPVR(PVRSRV_SYNC_PT *psSyncPt);
+static IMG_BOOL _SyncFbSyncPtHasSignalledSW(PVRSRV_SYNC_PT *psSyncPt);
+static IMG_BOOL _SyncFbFenceAddPt(PVRSRV_FENCE_SERVER *psFence,
+                                  IMG_UINT32 *i,
+                                  PVRSRV_SYNC_PT *psSyncPt);
+static PVRSRV_ERROR _SyncFbSWTimelineFenceCreate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                 PVRSRV_TIMELINE_SERVER *psTl,
+                                                 const IMG_CHAR *pszFenceName,
+                                                 PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                                 IMG_UINT64 *pui64SyncPtIdx);
+static PVRSRV_ERROR _SyncSWTimelineAdvanceSigErr(PVRSRV_TIMELINE_SERVER *psTl,
+                                                 PVRSRV_SYNC_STATE eState,
+                                                 IMG_UINT64 *pui64SyncPtIdx);
+static void _SyncSWTimelineCheckForUnsignalledPts(PVRSRV_TIMELINE_SERVER *psTl);
+
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         GENERIC FUNCTIONS                                 */
+/*                                                                           */
+/*****************************************************************************/
+
+
+/* Add a fence to the global fence list */
+static inline void _SyncFbFenceListAdd(PVRSRV_FENCE_SERVER *psFence)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_add_to_tail(&gsSyncFbContext.sFenceList, &psFence->sFenceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Remove a fence from the global fence list */
+static inline void _SyncFbFenceListDel(PVRSRV_FENCE_SERVER *psFence)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_remove_node(&psFence->sFenceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Add a timeline to the global timeline list */
+static inline void _SyncFbFTimelineListAdd(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_add_to_tail(&gsSyncFbContext.sTlList, &psTl->sTlList);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Remove a timeline from the global timeline list */
+static inline void _SyncFbTimelineListDel(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_remove_node(&psTl->sTlList);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+}
+
+/* Signal the sync event object to wake up waiters */
+static PVRSRV_ERROR _SyncFbSignalEO(void)
+{
+	PVRSRV_ERROR eError;
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(gsSyncFbContext.hSyncEventObject != NULL);
+
+	eError = OSEventObjectSignal(gsSyncFbContext.hSyncEventObject);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Retrieve the process handle base for the calling PID to look up sync objects */
+static PVRSRV_ERROR _SyncFbGetProcHandleBase(PVRSRV_HANDLE_BASE **ppsHandleBase)
+{
+	*ppsHandleBase = PVRSRVRetrieveProcessHandleBase();
+
+	if (*ppsHandleBase == NULL)
+	{
+		ERR("Failed to retrieve process handle base");
+		return PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Look up a handle in the process handle base of the calling PID */
+static PVRSRV_ERROR _SyncFbLookupProcHandle(IMG_HANDLE hHandle,
+                                            PVRSRV_HANDLE_TYPE eType,
+                                            IMG_BOOL bRefHandle,
+                                            void **ppvData,
+                                            PVRSRV_HANDLE_BASE **ppsBase)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHandleBase;
+
+	eError = _SyncFbGetProcHandleBase(&psHandleBase);
+
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	DBG(("%s: Handle Base: %p", __func__, psHandleBase));
+
+	eError = PVRSRVLookupHandle(psHandleBase,
+	                            ppvData,
+	                            hHandle,
+	                            eType,
+	                            bRefHandle);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*ppsBase = psHandleBase;
+
+	return PVRSRV_OK;
+
+e1:
+	return eError;
+}
+
+/* Destroy a handle in case a resource has not been registered with
+ * the resource manager */
+static PVRSRV_ERROR _SyncFbDestroyHandle(IMG_HANDLE hHandle,
+                                         PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHandleBase;
+
+	eError = _SyncFbGetProcHandleBase(&psHandleBase);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	eError = PVRSRVDestroyHandle(psHandleBase, hHandle, eType);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDestroyHandle", e1);
+
+	return PVRSRV_OK;
+
+e1:
+	return eError;
+}
+
+/* Currently unused */
+/*
+static PVRSRV_ERROR _SyncFbFindProcHandle(void *pvData,
+                                          PVRSRV_HANDLE_TYPE eType,
+                                          IMG_HANDLE *phHandle,
+                                          PVRSRV_HANDLE_BASE **ppsBase)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHandleBase;
+
+	PVR_DPF_ENTERED;
+
+	eError = _SyncFbGetProcHandleBase(&psHandleBase);
+
+	PVR_GOTO_IF_ERROR(eError, eExit);
+
+	eError = PVRSRVFindHandle(psHandleBase,
+	                          phHandle,
+	                          pvData,
+							  eType);
+	PVR_GOTO_IF_ERROR(eError, eExit);
+
+	*ppsBase = psHandleBase;
+
+	PVR_DPF_RETURN_OK;
+
+eExit:
+	PVR_DPF_RETURN_RC(eError);
+}
+*/
+
+/* Returns the type of a sync point determined by its registered
+ * signalling callback. Type can be e.g. a PVR sync point containing
+ * sync checkpoints or a software sync point*/
+static PVRSRV_SYNC_HANDLE_TYPE _SyncFbSyncPtHandleType(PVRSRV_SYNC_SIGNAL_CB *psCb)
+{
+	if (psCb == NULL)
+		return PVRSRV_SYNC_HANDLE_UNKNOWN;
+
+	if (psCb->pfnSignal == &_SyncFbSyncPtSignalPVR)
+		return PVRSRV_SYNC_HANDLE_PVR;
+
+	if (psCb->pfnSignal == &_SyncFbSyncPtSignalSW)
+		return PVRSRV_SYNC_HANDLE_SW;
+
+	return PVRSRV_SYNC_HANDLE_UNKNOWN;
+}
+
+static PVRSRV_SYNC_HANDLE_TYPE _SyncFbTimelineHandleType(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	if (psTl == NULL)
+		return PVRSRV_SYNC_HANDLE_UNKNOWN;
+
+	if (psTl->sTlOps.pfnSyncPtHasSignalled == &_SyncFbSyncPtHasSignalledPVR)
+		return PVRSRV_SYNC_HANDLE_PVR;
+
+	if (psTl->sTlOps.pfnSyncPtHasSignalled == &_SyncFbSyncPtHasSignalledSW)
+		return PVRSRV_SYNC_HANDLE_SW;
+
+	return PVRSRV_SYNC_HANDLE_UNKNOWN;
+}
+
+/* Print info about a sync point to the debug dump log */
+static void _SyncFbDebugRequestPrintSyncPt(PVRSRV_SYNC_PT *psSyncPt,
+                                           IMG_BOOL bPrintTl,
+                                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                           void *pvDumpDebugFile)
+{
+	PDLLIST_NODE psCBNode, psNextCBNode;
+	PVRSRV_SYNC_SIGNAL_CB *psCb;
+
+	if (bPrintTl)
+	{
+		PVR_DUMPDEBUG_LOG(" - SyncPt: SeqNum: %u, Refs: %d, Timeline: %-9s <%#"IMG_UINT64_FMTSPECx">, %-9s - <0x%p>",
+						  psSyncPt->uiSeqNum,
+						  OSAtomicRead(&psSyncPt->iRef),
+						  psSyncPt->psTl->pszName,
+						  psSyncPt->psTl->iUID,
+						  OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_SIGNALLED ? "Signalled" :
+							OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_ERRORED ? "Errored" : "Active",
+						  psSyncPt);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG(" - SyncPt: SeqNum: %u, Refs: %d, %-9s - <0x%p>",
+						  psSyncPt->uiSeqNum,
+						  OSAtomicRead(&psSyncPt->iRef),
+						  OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_SIGNALLED ? "Signalled" :
+							OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_ERRORED ? "Errored" : "Active",
+						  psSyncPt);
+	}
+
+	/* ... all attached syncs to that sync point*/
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks,
+	                    psCBNode,
+	                    psNextCBNode)
+	{
+		psCb = IMG_CONTAINER_OF(psCBNode,
+		                        PVRSRV_SYNC_SIGNAL_CB,
+		                        sCallbackNode);
+
+		switch (_SyncFbSyncPtHandleType(psCb))
+		{
+			case PVRSRV_SYNC_HANDLE_PVR:
+			{
+				PSYNC_CHECKPOINT pCP = psCb->hAttachedSync;
+				PVR_DUMPDEBUG_LOG("    - CbType: PVR-Checkpoint, ID: %u, FWAddr: %#08x, Enq: %d, Ref: %d, %-9s - <0x%p>",
+				                  SyncCheckpointGetId(pCP),
+				                  SyncCheckpointGetFirmwareAddr(pCP),
+				                  SyncCheckpointGetEnqueuedCount(pCP),
+				                  SyncCheckpointGetReferenceCount(pCP),
+				                  SyncCheckpointGetStateString(pCP),
+				                  pCP);
+				break;
+			}
+			case PVRSRV_SYNC_HANDLE_SW:
+			{
+				PVRSRV_SYNC_PT_SW *psSWPt = psCb->hAttachedSync;
+				PVR_DUMPDEBUG_LOG("    - CbType: SW-Syncpoint, %-9s - <0x%p>",
+				                  psSWPt->bSignalled ? "Signalled" : "Active",
+				                  psSWPt);
+				break;
+			}
+			case PVRSRV_SYNC_HANDLE_UNKNOWN:
+			default:
+				PVR_DUMPDEBUG_LOG("    - CbType: Unknown - <0x%p>",
+				                  psCb->hAttachedSync);
+		}
+	}
+}
+
+/* Function registered with the debug dump mechanism. Prints out all timelines
+ * with pending syncs. */
+static void _SyncFbDebugRequest(IMG_HANDLE hDebugRequestHandle,
+                                IMG_UINT32 ui32VerbLevel,
+                                DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(hDebugRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+
+		PDLLIST_NODE psTlNode, psNextTlNode;
+		PVRSRV_TIMELINE_SERVER *psTl;
+
+		PDLLIST_NODE psFenceNode, psNextFenceNode;
+		PVRSRV_FENCE_SERVER *psFence;
+
+		PDLLIST_NODE psPtNode, psNextPtNode;
+		PVRSRV_SYNC_PT *psSyncPt;
+
+		OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+		PVR_DUMPDEBUG_LOG("------[ Fallback Fence Sync: timelines ]------");
+
+		/* Iterate over all timelines */
+		dllist_foreach_node(&gsSyncFbContext.sTlList, psTlNode, psNextTlNode)
+		{
+
+			psTl = IMG_CONTAINER_OF(psTlNode,
+			                        PVRSRV_TIMELINE_SERVER,
+			                        sTlList);
+
+			OSLockAcquire(psTl->hTlLock);
+
+			PVR_DUMPDEBUG_LOG("Timeline: %s, SeqNum: %d/%d - <%#"IMG_UINT64_FMTSPECx">",
+			                  psTl->pszName,
+			                  OSAtomicRead(&psTl->iLastSignalledSeqNum),
+			                  OSAtomicRead(&psTl->iSeqNum),
+			                  psTl->iUID);
+
+			/* ... all active sync points in the timeline */
+			dllist_foreach_node(&psTl->sSyncActiveList, psPtNode, psNextPtNode)
+			{
+
+				psSyncPt = IMG_CONTAINER_OF(psPtNode,
+				                            PVRSRV_SYNC_PT,
+				                            sTlSyncActiveList);
+
+				_SyncFbDebugRequestPrintSyncPt(psSyncPt,
+				                               IMG_FALSE,
+				                               pfnDumpDebugPrintf,
+				                               pvDumpDebugFile);
+
+			}
+			OSLockRelease(psTl->hTlLock);
+		}
+
+		PVR_DUMPDEBUG_LOG("------[ Fallback Fence Sync: fences ]------");
+
+		/* Iterate over all fences */
+		dllist_foreach_node(&gsSyncFbContext.sFenceList,
+							psFenceNode,
+							psNextFenceNode)
+		{
+			psFence = IMG_CONTAINER_OF(psFenceNode,
+			                           PVRSRV_FENCE_SERVER,
+			                           sFenceListNode);
+
+			PVR_DUMPDEBUG_LOG("Fence: %s, %-9s - <%#"IMG_UINT64_FMTSPECx">",
+			                  psFence->pszName,
+			                  _SyncFbFenceSyncsHaveSignalled(psFence) ?
+			                      "Signalled" : "Active",
+				               psFence->iUID);
+
+			/* ... all sync points in the fence */
+			for (i = 0; i < psFence->uiNumSyncs; i++)
+			{
+				_SyncFbDebugRequestPrintSyncPt(psFence->apsFenceSyncList[i],
+                                               IMG_TRUE,
+				                               pfnDumpDebugPrintf,
+				                               pvDumpDebugFile);
+			}
+		}
+
+		OSLockRelease(gsSyncFbContext.hFbContextLock);
+	}
+
+}
+
+/* Notify callback that is called as part of the RGX MISR e.g. after FW
+ * signalled the host that work completed. */
+static void _SyncFbTimelineUpdate_NotifyCMD(void *psSyncFbContext)
+{
+	PVRSRV_TIMELINE_SERVER *psTl;
+	PVRSRV_SYNC_PT *psSyncPt;
+	PDLLIST_NODE psTlList = &gsSyncFbContext.sTlList;
+	PDLLIST_NODE psCurrentTl, psNextTl;
+	PDLLIST_NODE psCurrentPt, psNextPt;
+	IMG_BOOL bSignalled = IMG_FALSE, bSignal = IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psSyncFbContext);
+
+	PVR_DPF_ENTERED;
+
+	/* Outer loop over all timelines */
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_foreach_node(psTlList, psCurrentTl, psNextTl)
+	{
+		psTl = IMG_CONTAINER_OF(psCurrentTl,
+		                        PVRSRV_TIMELINE_SERVER,
+		                        sTlList);
+
+		/* Inner loop over all SyncPts in the timeline.
+		 * Check & Update all active SyncPts */
+		OSLockAcquire(psTl->hTlLock);
+		dllist_foreach_node(&psTl->sSyncActiveList, psCurrentPt, psNextPt)
+		{
+			psSyncPt = IMG_CONTAINER_OF(psCurrentPt,
+			                            PVRSRV_SYNC_PT,
+			                            sTlSyncActiveList);
+
+			/* If the SyncPt has been signalled we have to
+			 * update all attached syncs */
+			bSignalled = psTl->sTlOps.pfnSyncPtHasSignalled(psSyncPt);
+			if (bSignalled)
+			{
+				/* Wake up waiters after releasing the locks */
+				bSignal = IMG_TRUE;
+
+				/* Remove the SyncPt from the active list of the timeline. */
+				dllist_remove_node(psCurrentPt);
+			}
+			else
+			{
+				/* No need to check further points on this timeline because
+				 * this sync pt will be signalled first */
+				break;
+			}
+
+		}/* End inner loop */
+		OSLockRelease(psTl->hTlLock);
+
+	} /* End outer loop */
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	if (bSignal)
+	{
+		PVR_LOG_IF_ERROR(_SyncFbSignalEO(), "_SyncFbSignalEO");
+	}
+
+	PVR_DPF_RETURN;
+}
+
+static IMG_UINT32
+_SyncCheckpointFWAddrHash(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_UINT32 *p = (IMG_UINT32 *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = *p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+static IMG_BOOL
+_SyncCheckpointFWAddrCompare(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_UINT32 *p1 = (IMG_UINT32 *)pKey1;
+	IMG_UINT32 *p2 = (IMG_UINT32 *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+#if defined(PDUMP)
+static PVRSRV_ERROR SyncFbFenceGetCheckpoints(PVRSRV_FENCE hFence, IMG_UINT32 *puiNumCheckpoints,
+		                                PSYNC_CHECKPOINT **papsCheckpoints)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psFence;
+	PVRSRV_HANDLE_BASE *psHBase;
+	PSYNC_CHECKPOINT *apsCheckpoints;
+	PSYNC_CHECKPOINT psCheckpoint;
+	PVRSRV_SYNC_SIGNAL_CB *psSyncCB;
+	PVRSRV_SYNC_PT *psSyncPt;
+	PDLLIST_NODE psNode;
+	IMG_UINT32 i, uiNumCheckpoints = 0;
+
+	if (hFence == PVRSRV_NO_FENCE)
+	{
+		*puiNumCheckpoints = 0;
+		eError = PVRSRV_OK;
+		goto e0;
+	}
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) hFence,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                                 IMG_TRUE,
+	                                 (void**)&psFence,
+	                                 &psHBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	apsCheckpoints = OSAllocMem(sizeof(*apsCheckpoints) * psFence->uiNumSyncs);
+	PVR_LOG_GOTO_IF_NOMEM(apsCheckpoints, eError, e1);
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+	/* Increase refcount to make sure fence is not destroyed while waiting */
+	_SyncFbFenceAcquire(psFence);
+
+	/* Go through all syncs and add them to the list */
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		psSyncPt = psFence->apsFenceSyncList[i];
+
+		psNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psSyncCB = IMG_CONTAINER_OF(psNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+
+		if (_SyncFbSyncPtHandleType(psSyncCB) == PVRSRV_SYNC_HANDLE_PVR)
+		{
+			psCheckpoint = (PSYNC_CHECKPOINT) psSyncCB->hAttachedSync;
+			apsCheckpoints[uiNumCheckpoints++] = psCheckpoint;
+		}
+	}
+
+	SyncFbFenceRelease(psFence);
+
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	*puiNumCheckpoints = uiNumCheckpoints;
+	*papsCheckpoints = apsCheckpoints;
+
+e1:
+	PVRSRVReleaseHandle(psHBase,
+	                    (IMG_HANDLE) (uintptr_t) hFence,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+e0:
+	return eError;
+}
+#endif
+
+PVRSRV_ERROR SyncFbRegisterSyncFunctions(void)
+{
+	/* Initialise struct and register with sync_checkpoint.c */
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceResolve = &SyncFbFenceResolvePVR;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceCreate = &SyncFbFenceCreatePVR;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceDataRollback = &SyncFbFenceRollbackPVR;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFenceFinalise = NULL; /* no fence finalise function required */
+	gsSyncFbContext.sSyncCheckpointReg.pfnNoHWUpdateTimelines = &_SyncFbTimelineUpdate_NotifyCMD;
+	gsSyncFbContext.sSyncCheckpointReg.pfnFreeCheckpointListMem = OSFreeMem;
+	gsSyncFbContext.sSyncCheckpointReg.pfnDumpInfoOnStalledUFOs = &SyncFbDumpInfoOnStalledUFOs;
+	OSStringSafeCopy(gsSyncFbContext.sSyncCheckpointReg.pszImplName, "SyncFb", SYNC_CHECKPOINT_IMPL_MAX_STRLEN);
+#if defined(PDUMP)
+	gsSyncFbContext.sSyncCheckpointReg.pfnSyncFenceGetCheckpoints = &SyncFbFenceGetCheckpoints;
+#endif
+
+	return SyncCheckpointRegisterFunctions(&gsSyncFbContext.sSyncCheckpointReg);
+}
+
+PVRSRV_ERROR SyncFbRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	SYNC_FB_CONTEXT_DEVICE_LIST *psNewDeviceEntry;
+
+	PVR_DPF_ENTERED;
+
+	/* Initialise the sync fallback context */
+	if (gsSyncFbContext.hSyncEventObject == NULL)
+	{
+		eError = OSEventObjectCreate("Sync event object",
+		                             &gsSyncFbContext.hSyncEventObject);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		dllist_init(&gsSyncFbContext.sTlList);
+		dllist_init(&gsSyncFbContext.sFenceList);
+		dllist_init(&gsSyncFbContext.sDeviceList);
+		gsSyncFbContext.sCheckpointHashTable = HASH_Create_Extended(64, sizeof(IMG_UINT32), _SyncCheckpointFWAddrHash, _SyncCheckpointFWAddrCompare);
+
+		eError = OSLockCreate(&gsSyncFbContext.hFbContextLock);
+		PVR_GOTO_IF_ERROR(eError, e2);
+
+		eError = PVRSRVRegisterCmdCompleteNotify(&gsSyncFbContext.hCMDNotify,
+		                                         &_SyncFbTimelineUpdate_NotifyCMD,
+		                                         &gsSyncFbContext);
+		PVR_GOTO_IF_ERROR(eError, e3);
+	}
+
+	psNewDeviceEntry = OSAllocMem(sizeof(*psNewDeviceEntry));
+	PVR_GOTO_IF_NOMEM(psNewDeviceEntry, eError, e4);
+
+	psNewDeviceEntry->psDevice = psDeviceNode;
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_add_to_tail(&gsSyncFbContext.sDeviceList, &psNewDeviceEntry->sDeviceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psNewDeviceEntry->hDBGNotify,
+	                                              psDeviceNode,
+	                                              _SyncFbDebugRequest,
+	                                              DEBUG_REQUEST_FALLBACKSYNC,
+	                                              NULL);
+	PVR_GOTO_IF_ERROR(eError, e5);
+
+	PVR_DPF_RETURN_RC(eError);
+
+
+e5:
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	dllist_remove_node(&psNewDeviceEntry->sDeviceListNode);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+	OSFreeMem(psNewDeviceEntry);
+e4:
+	PVRSRVUnregisterCmdCompleteNotify(gsSyncFbContext.hCMDNotify);
+e3:
+	OSLockDestroy(gsSyncFbContext.hFbContextLock);
+e2:
+	HASH_Delete_Extended(gsSyncFbContext.sCheckpointHashTable, IMG_TRUE);
+	OSEventObjectDestroy(gsSyncFbContext.hSyncEventObject);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbDeregisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_FB_CONTEXT_DEVICE_LIST *psDeviceEntry;
+	PDLLIST_NODE psNode, psNext;
+
+	PVR_DPF_ENTERED;
+
+	/* Return if Init was never called */
+	if (gsSyncFbContext.hSyncEventObject == NULL)
+		goto e1;
+
+	/* Check device list for the given device and remove it */
+	dllist_foreach_node(&gsSyncFbContext.sDeviceList, psNode, psNext)
+	{
+		psDeviceEntry = IMG_CONTAINER_OF(psNode,
+		                                 SYNC_FB_CONTEXT_DEVICE_LIST,
+		                                 sDeviceListNode);
+
+		if (psDeviceEntry->psDevice == psDeviceNode)
+		{
+			PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceEntry->hDBGNotify);
+
+			OSLockAcquire(gsSyncFbContext.hFbContextLock);
+			dllist_remove_node(psNode);
+			OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+			OSFreeMem(psDeviceEntry);
+			break;
+		}
+	}
+
+	/* If there are still devices registered with us don't deinit module */
+	if (!dllist_is_empty(&gsSyncFbContext.sDeviceList))
+	{
+		goto e1;
+	}
+
+	PVRSRVUnregisterCmdCompleteNotify(gsSyncFbContext.hCMDNotify);
+
+	HASH_Delete_Extended(gsSyncFbContext.sCheckpointHashTable, IMG_TRUE);
+	eError = OSEventObjectDestroy(gsSyncFbContext.hSyncEventObject);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+	gsSyncFbContext.hSyncEventObject = NULL;
+
+	OSLockDestroy(gsSyncFbContext.hFbContextLock);
+
+e1:
+	return eError;
+}
+
+/* HOLD TL LOCK!
+ * Creates a new sync point on a timeline */
+static PVRSRV_ERROR _SyncFbSyncPtCreate(PVRSRV_SYNC_PT **ppsSyncPt,
+                                       PVRSRV_TIMELINE_SERVER *psTl,
+                                       IMG_UINT32 uiSeqNumber)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_SYNC_PT *psNewSyncPt;
+
+	PVR_DPF_ENTERED;
+
+	psNewSyncPt = OSAllocMem(sizeof(*psNewSyncPt));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSyncPt, eError, e1);
+
+	psNewSyncPt->psTl = psTl;
+	OSAtomicWrite(&psNewSyncPt->iStatus, PVRSRV_SYNC_NOT_SIGNALLED);
+
+	psNewSyncPt->uiSeqNum = uiSeqNumber;
+	psNewSyncPt->uiPID = OSGetCurrentClientProcessIDKM();
+	PT_REF_SET(&psNewSyncPt->iRef, 1,psNewSyncPt);
+
+	dllist_init(&psNewSyncPt->sTlSyncList);
+	dllist_init(&psNewSyncPt->sTlSyncActiveList);
+	dllist_init(&psNewSyncPt->sSignalCallbacks);
+
+	/* Increment Tl ref due to new checkpoint*/
+	_SyncFbTimelineAcquire(psTl);
+
+	dllist_add_to_tail(&psTl->sSyncList, &psNewSyncPt->sTlSyncList);
+	dllist_add_to_tail(&psTl->sSyncActiveList, &psNewSyncPt->sTlSyncActiveList);
+
+	*ppsSyncPt = psNewSyncPt;
+
+	PVR_DPF_RETURN_OK;
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Increment sync point refcount */
+static void _SyncFbSyncPtAcquire(PVRSRV_SYNC_PT *psSyncPt)
+{
+	PT_REF_INC(&psSyncPt->iRef, psSyncPt);
+}
+
+/* Release and maybe destroy sync point if refcount is 0 */
+static PVRSRV_ERROR _SyncFbSyncPtRelease(PVRSRV_SYNC_PT *psSyncPt,
+                                         IMG_BOOL bError)
+{
+	PVRSRV_ERROR eError;
+	PDLLIST_NODE psNode, psNext;
+	PVRSRV_SYNC_SIGNAL_CB *psSyncCB;
+	IMG_INT iRef;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	iRef = PT_REF_DEC(&psSyncPt->iRef, psSyncPt);
+	if (iRef != 0)
+	{
+		eError = PVRSRV_OK;
+		goto e1;
+	}
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	/* Remove all checkpoints from the ufo lookup table.*/
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks, psNode, psNext)
+	{
+		psSyncCB = IMG_CONTAINER_OF(psNode,
+		                            PVRSRV_SYNC_SIGNAL_CB,
+		                            sCallbackNode);
+
+		if (_SyncFbSyncPtHandleType(psSyncCB) == PVRSRV_SYNC_HANDLE_PVR)
+		{
+			IMG_UINT32 ui32FwAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT) psSyncCB->hAttachedSync);
+
+			DBG(("%s: Removing key <%#08x>", __func__, ui32FwAddr));
+			HASH_Remove_Extended(gsSyncFbContext.sCheckpointHashTable, &ui32FwAddr);
+		}
+	}
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	OSLockAcquire(psSyncPt->psTl->hTlLock);
+	if (dllist_node_is_in_list(&psSyncPt->sTlSyncActiveList))
+		dllist_remove_node(&psSyncPt->sTlSyncActiveList);
+
+	dllist_remove_node(&psSyncPt->sTlSyncList);
+
+	if (bError)
+	{
+		_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_ERRORED);
+	}
+	OSLockRelease(psSyncPt->psTl->hTlLock);
+
+	/* Remove all attached nodes and signal them.*/
+	while (!dllist_is_empty(&psSyncPt->sSignalCallbacks))
+	{
+		psNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psSyncCB = IMG_CONTAINER_OF(psNode,
+		                            PVRSRV_SYNC_SIGNAL_CB,
+		                            sCallbackNode);
+
+		psSyncCB->pfnSyncFree(psSyncCB->hAttachedSync);
+		dllist_remove_node(&psSyncCB->sCallbackNode);
+		OSFreeMem(psSyncCB);
+	}
+
+	eError = SyncFbTimelineRelease(psSyncPt->psTl);
+	PVR_LOG_IF_ERROR(eError, "SyncFbTimelineRelease");
+
+	OSFreeMem(psSyncPt);
+
+	PVR_DPF_RETURN_OK;
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* HOLD TL LOCK!
+ * Mark all attached syncs of a sync point with the state eSignal */
+static PVRSRV_ERROR _SyncFbSyncPtSignalAttached(PVRSRV_SYNC_PT *psSyncPt,
+                                                PVRSRV_SYNC_STATE eSignal)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK, eRet;
+	PDLLIST_NODE psCurrentCB, psNextCB;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	if (dllist_is_empty(&psSyncPt->sSignalCallbacks))
+	{
+		ERR("Sync pt has no attached syncs. Make sure to attach one "
+		    "when creating a new sync pt to define its flavour");
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e1;
+	}
+
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks, psCurrentCB, psNextCB)
+	{
+		psCB = IMG_CONTAINER_OF(psCurrentCB,
+		                        PVRSRV_SYNC_SIGNAL_CB,
+		                        sCallbackNode);
+		eRet = psCB->pfnSignal(psCB->hAttachedSync, eSignal);
+		if (eRet != PVRSRV_OK)
+		{
+			ERR("Failed to signal an attached sync, system might block!");
+			eError = eRet;
+			/* Don't jump to exit but try to signal remaining syncs */
+		}
+	}
+
+e1:
+	PVR_DPF_RETURN_RC1(eError, psSyncPt);
+}
+
+/* HOLD TL LOCK!
+ * Mark all a sync point with the state eSignal */
+static PVRSRV_ERROR _SyncFbSyncPtSignal(PVRSRV_SYNC_PT *psSyncPt, PVRSRV_SYNC_STATE eSignal)
+{
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	OSAtomicWrite(&psSyncPt->iStatus, eSignal);
+
+	if (psSyncPt->uiSeqNum >
+	    OSAtomicRead(&psSyncPt->psTl->iLastSignalledSeqNum))
+	{
+		OSAtomicWrite(&psSyncPt->psTl->iLastSignalledSeqNum,
+		              psSyncPt->uiSeqNum);
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/* Check whether all syncs in a fence were signalled */
+static IMG_BOOL _SyncFbFenceSyncsHaveSignalled(PVRSRV_FENCE_SERVER *psFence)
+{
+	IMG_UINT32 i;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		if (OSAtomicRead(&psFence->apsFenceSyncList[i]->iStatus) ==
+			PVRSRV_SYNC_NOT_SIGNALLED)
+		{
+			PVR_DPF_RETURN_RC1(IMG_FALSE, psFence);
+		}
+	}
+
+	OSAtomicWrite(&psFence->iStatus,
+	              PVRSRV_FENCE_SIGNALLED);
+
+	PVR_DPF_RETURN_RC1(IMG_TRUE, psFence);
+}
+
+/* Increment timeline refcount */
+static void _SyncFbTimelineAcquire(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	TL_REF_INC(&psTl->iRef, psTl);
+}
+
+PVRSRV_ERROR SyncFbTimelineRelease(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	IMG_INT iRef;
+
+	PVR_DPF_ENTERED1(psTl);
+
+	iRef = TL_REF_DEC(&psTl->iRef, psTl);
+	if (iRef != 0)
+	{
+		PVR_DPF_RETURN_OK;
+	}
+
+	if (_SyncFbTimelineHandleType(psTl) == PVRSRV_SYNC_HANDLE_SW)
+	{
+		_SyncSWTimelineCheckForUnsignalledPts(psTl);
+	}
+
+	_SyncFbTimelineListDel(psTl);
+
+	OSLockDestroy(psTl->hTlLock);
+
+#if defined(DEBUG)
+	psTl->sTlOps.pfnSyncPtHasSignalled = NULL;
+	psTl->hTlLock = NULL;
+#endif
+
+	OSFreeMem(psTl);
+
+	PVR_DPF_RETURN_OK;
+}
+
+/* Increment fence refcount */
+static void _SyncFbFenceAcquire(PVRSRV_FENCE_SERVER *psFence)
+{
+	FENCE_REF_INC(&psFence->iRef, psFence);
+}
+
+PVRSRV_ERROR SyncFbFenceRelease(PVRSRV_FENCE_SERVER *psFence)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK, eRet;
+	IMG_INT iRef;
+	IMG_UINT32 i;
+	IMG_BOOL bError = IMG_FALSE;
+	IMG_BOOL bCleanup = IMG_FALSE;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	/* If cleanup thread, force cleanup of SW fences, otherwise
+	 * simply decrement psFence->iRef
+	 */
+	if (OSGetCurrentClientProcessIDKM() == PVRSRVGetPVRSRVData()->cleanupThreadPid)
+	{
+		IMG_UINT32 uiSyncPt;
+
+		/* Force erroring of any SW sync pts this fence may contain */
+		for (uiSyncPt=0; uiSyncPt < psFence->uiNumSyncs; uiSyncPt++)
+		{
+			PVRSRV_SYNC_PT *psSyncPt = psFence->apsFenceSyncList[uiSyncPt];
+
+			/* If this is a SW sync point from a timeline which was created
+			 * by the process being cleaned-up, then error it
+			 */
+			if ((_SyncFbTimelineHandleType(psSyncPt->psTl) == PVRSRV_SYNC_HANDLE_SW) &&
+			    (PVRSRVGetPurgeConnectionPid() == psSyncPt->psTl->uiPID))
+			{
+				OSLockAcquire(psSyncPt->psTl->hTlLock);
+				_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_ERRORED);
+				OSLockRelease(psSyncPt->psTl->hTlLock);
+			}
+		}
+	}
+
+	iRef = FENCE_REF_DEC(&psFence->iRef, psFence);
+	if (iRef != 0)
+	{
+		goto e1;
+	}
+
+	PDUMPCOMMENTWITHFLAGS(psFence->psDevNode, 0,
+	                      "Destroy Fence %s (ID:%"IMG_UINT64_FMTSPEC")",
+	                      psFence->pszName,
+	                      psFence->iUID);
+
+	if (OSGetCurrentClientProcessIDKM() ==
+			PVRSRVGetPVRSRVData()->cleanupThreadPid)
+	{
+		bCleanup = IMG_TRUE;
+	}
+
+	_SyncFbFenceListDel(psFence);
+
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		PVRSRV_SYNC_PT *psSyncPt = psFence->apsFenceSyncList[i];
+
+		if (bCleanup &&
+				_SyncFbTimelineHandleType(psSyncPt->psTl) == PVRSRV_SYNC_HANDLE_SW)
+		{
+			bError = IMG_TRUE;
+		}
+
+		eRet = _SyncFbSyncPtRelease(psSyncPt,
+		                            bError);
+		if (eRet != PVRSRV_OK)
+		{
+			ERR("Error when releasing SyncPt, this might leak memory")
+			eError = eRet;
+			/* Try to continue and release the other sync pts, return error */
+		}
+	}
+
+#if defined(DEBUG)
+	{
+		for (i = 0; i < psFence->uiNumSyncs; i++)
+		{
+			psFence->apsFenceSyncList[i] = NULL;
+		}
+		psFence->uiNumSyncs = 0;
+	}
+#endif
+
+	OSFreeMem(psFence->apsFenceSyncList);
+	OSFreeMem(psFence);
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbFenceDup(PVRSRV_FENCE_SERVER *psInFence,
+                            PVRSRV_FENCE_SERVER **ppsOutFence)
+{
+	PVR_DPF_ENTERED1(psInFence);
+
+	FENCE_REF_INC(&psInFence->iRef, psInFence);
+
+	PDUMPCOMMENTWITHFLAGS(psInFence->psDevNode, 0,
+	                      "Dup Fence %s (ID:%"IMG_UINT64_FMTSPEC").",
+	                      psInFence->pszName,
+	                      psInFence->iUID);
+
+	*ppsOutFence = psInFence;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, *ppsOutFence);
+}
+
+static IMG_BOOL _SyncFbSyncPtHasSignalled(PVRSRV_SYNC_PT *psSyncPt)
+{
+	return psSyncPt->psTl->sTlOps.pfnSyncPtHasSignalled(psSyncPt);
+}
+
+static IMG_BOOL _SyncFbFenceAddPt(PVRSRV_FENCE_SERVER *psFence,
+                                  IMG_UINT32 *i,
+                                  PVRSRV_SYNC_PT *psSyncPt)
+{
+	/*
+	 * If the fence is signalled there is no need to add it to the fence.
+	 * One exception is PDUMP drivers where we need to make sure we
+	 * set up proper synchronisation in the pdump stream.
+	 */
+#if !defined(PDUMP)
+	if (_SyncFbSyncPtHasSignalled(psSyncPt)) return IMG_FALSE;
+#endif
+	_SyncFbSyncPtAcquire(psSyncPt);
+	psFence->apsFenceSyncList[*i] = psSyncPt;
+	(*i)++;
+	return IMG_TRUE;
+}
+
+PVRSRV_ERROR SyncFbFenceMerge(PVRSRV_FENCE_SERVER *psInFence1,
+                              PVRSRV_FENCE_SERVER *psInFence2,
+                              IMG_UINT32 uiFenceNameSize,
+                              const IMG_CHAR *pszFenceName,
+                              PVRSRV_FENCE_SERVER **ppsOutFence)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	IMG_UINT32 i, i1, i2;
+	IMG_UINT32 uiFenceSyncListSize;
+
+	PVR_DPF_ENTERED;
+
+	psNewFence = OSAllocMem(sizeof(*psNewFence));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence, eError, e1);
+
+	psNewFence->psDevNode = psInFence1->psDevNode;
+
+	uiFenceSyncListSize = sizeof(*(psNewFence->apsFenceSyncList)) *
+			(psInFence1->uiNumSyncs + psInFence2->uiNumSyncs);
+
+	psNewFence->apsFenceSyncList = OSAllocMem(uiFenceSyncListSize);
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence->apsFenceSyncList, eError, e2);
+
+	if (uiFenceNameSize == 1)
+	{
+		OSSNPrintf(psNewFence->pszName,
+		           SYNC_FB_FENCE_MAX_LENGTH,
+		           "Fence-Merged");
+	}
+	else
+	{
+		if (pszFenceName)
+		{
+			OSStringSafeCopy(psNewFence->pszName,
+						  pszFenceName,
+						  SYNC_FB_FENCE_MAX_LENGTH);
+		}
+		else
+		{
+			psNewFence->pszName[0] = '\0';
+		}
+	}
+
+	/* Add sync pts from input fence 1 & 2
+	 * - no duplicates in one timeline
+	 * - sync pts in one fence are ordered by timeline UID
+	 *
+	 * */
+	for (i = 0, i1 = 0, i2 = 0;
+	     i1 < psInFence1->uiNumSyncs && i2 < psInFence2->uiNumSyncs;)
+	{
+		PVRSRV_SYNC_PT *psSyncPt1 = psInFence1->apsFenceSyncList[i1];
+		PVRSRV_SYNC_PT *psSyncPt2 = psInFence2->apsFenceSyncList[i2];
+
+		/* Adding sync pts in order of their timeline UID, smaller ID first */
+		if (psSyncPt1->psTl->iUID <
+            psSyncPt2->psTl->iUID)
+		{
+			_SyncFbFenceAddPt(psNewFence, &i, psSyncPt1);
+			i1++;
+		}
+		else if (psSyncPt1->psTl->iUID >
+		         psSyncPt2->psTl->iUID)
+		{
+			_SyncFbFenceAddPt(psNewFence, &i, psSyncPt2);
+			i2++;
+		}
+		/* In case the timeline UID is the same just add the point that is
+		 * later on that timeline. */
+		else
+		{
+			/* --> Some C magic to find out if 'a' is a point later in the
+			 * timeline than 'b', wrap around is taken into account:
+			 *			(a - b <= ((IMG_INT)(~0U>>1)) ) */
+			if ( psSyncPt1->uiSeqNum - psSyncPt2->uiSeqNum <=
+			    ((IMG_INT)(~0U>>1)) )
+			{
+				_SyncFbFenceAddPt(psNewFence, &i, psSyncPt1);
+			}
+			else
+			{
+				_SyncFbFenceAddPt(psNewFence, &i, psSyncPt2);
+			}
+
+			i1++;
+			i2++;
+		}
+	}
+
+	/* Add the remaining syncs pts to the fence. At this point we only enter
+	 * either the first or the second loop because one fence has
+	 * more sync pts than the other.
+	 */
+	for (; i1 < psInFence1->uiNumSyncs; i1++)
+	{
+		_SyncFbFenceAddPt(psNewFence, &i, psInFence1->apsFenceSyncList[i1]);
+	}
+
+	for (; i2 < psInFence2->uiNumSyncs; i2++)
+	{
+		_SyncFbFenceAddPt(psNewFence, &i, psInFence2->apsFenceSyncList[i2]);
+	}
+
+	/* Fill remaining fields */
+	psNewFence->uiNumSyncs = i;
+	psNewFence->iUID = (IMG_INT64)(uintptr_t) psNewFence;
+	FENCE_REF_SET(&psNewFence->iRef, 1, psNewFence);
+
+	OSAtomicWrite(&psNewFence->iStatus, PVRSRV_SYNC_NOT_SIGNALLED);
+
+	_SyncFbFenceListAdd(psNewFence);
+
+	PDUMPCOMMENTWITHFLAGS(psInFence1->psDevNode, 0,
+	                      "Merge Fence1 %s (ID:%"IMG_UINT64_FMTSPEC"), Fence2 %s (ID:%"IMG_UINT64_FMTSPEC") "
+	                      "to Fence %s (ID:%"IMG_UINT64_FMTSPEC")",
+	                      psInFence1->pszName,
+	                      psInFence1->iUID,
+	                      psInFence2->pszName,
+	                      psInFence2->iUID,
+	                      psNewFence->pszName,
+	                      psNewFence->iUID);
+
+	*ppsOutFence = psNewFence;
+
+	PVR_DPF_RETURN_RC1(eError, *ppsOutFence);
+
+e2:
+	OSFreeMem(psNewFence);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+PVRSRV_ERROR SyncFbFenceWait(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 ui32TimeoutInMs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_HANDLE hOSEvent;
+	IMG_UINT32 t1 = 0, t2 = 0;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	/* Increase refcount to make sure fence is not destroyed while waiting */
+	_SyncFbFenceAcquire(psFence);
+
+	if (OSAtomicRead(&psFence->iStatus) == PVRSRV_FENCE_NOT_SIGNALLED)
+	{
+		PVRSRV_ERROR eErrorClose;
+
+		/* If the status of the fence is not signalled it could mean that
+		 * there are actually syncs still pending or that we have not
+		 * checked yet whether the syncs were met, therefore do the
+		 * check now and return in case they are. If they are not, go
+		 * to sleep and wait. */
+
+		if (_SyncFbFenceSyncsHaveSignalled(psFence))
+		{
+			goto e1;
+		}
+		else if (ui32TimeoutInMs == 0)
+		{
+			eError = PVRSRV_ERROR_TIMEOUT;
+			goto e1;
+		}
+
+		eError = OSEventObjectOpen(gsSyncFbContext.hSyncEventObject,
+		                           &hOSEvent);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		while (!_SyncFbFenceSyncsHaveSignalled(psFence) && ui32TimeoutInMs)
+		{
+			t1 = OSClockms();
+			/* Wait for EO to be signalled */
+			eError = OSEventObjectWaitTimeout(hOSEvent,
+			                                  ui32TimeoutInMs * 1000);
+			t2 = OSClockms();
+
+			if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_TIMEOUT)
+			{
+				break;
+			}
+
+
+			/* Reduce timeout by the time we have just waited */
+			if (ui32TimeoutInMs < (t2-t1))
+			{
+				ui32TimeoutInMs = 0;
+			}
+			else
+			{
+				ui32TimeoutInMs -= (t2-t1);
+			}
+		}
+
+		eErrorClose = OSEventObjectClose(hOSEvent);
+		if (eErrorClose != PVRSRV_OK)
+		{
+			ERR("Unable to close Event Object");
+
+			/* Do not overwrite previous error
+			 * if it was something else than PVRSRV_OK */
+			if (eError == PVRSRV_OK)
+			{
+				eError = eErrorClose;
+			}
+		}
+	}
+e1:
+
+	SyncFbFenceRelease(psFence);
+
+	PVR_DPF_RETURN_RC1(eError, psFence);
+}
+
+PVRSRV_ERROR SyncFbFenceDump(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiLine,
+                             IMG_UINT32 uiFileNameLength,
+                             const IMG_CHAR *pszFile,
+                             IMG_UINT32 uiModuleLength,
+                             const IMG_CHAR *pszModule,
+                             IMG_UINT32 uiDescLength,
+                             const IMG_CHAR *pszDesc)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+
+	PVR_UNREFERENCED_PARAMETER(uiFileNameLength);
+	PVR_UNREFERENCED_PARAMETER(uiModuleLength);
+	PVR_UNREFERENCED_PARAMETER(uiDescLength);
+
+	PVR_DPF_ENTERED1(psFence);
+
+	PVR_LOG(("  Fence dump request from:"));
+#if defined(DEBUG)
+	PVR_LOG(("    %s (%s:%u)", pszModule, pszFile, uiLine));
+#else
+	PVR_LOG(("    %s (location only available in debug build)", pszModule));
+	PVR_UNREFERENCED_PARAMETER(pszFile);
+	PVR_UNREFERENCED_PARAMETER(uiLine);
+#endif
+	PVR_LOG(("  Desc: %s", pszDesc));
+	PVR_LOG(("---------------- FENCE ----------------"));
+	PVR_LOG(("%s (UID: %"IMG_UINT64_FMTSPEC")", psFence->pszName, psFence->iUID));
+
+	PVR_LOG(("  Signalled: %s",
+	        _SyncFbFenceSyncsHaveSignalled(psFence)?"Yes":"No"));
+	PVR_LOG(("  Ref: %d", OSAtomicRead(&psFence->iRef) ));
+
+	PVR_LOG(("  Sync Points:"));
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		PVRSRV_SYNC_PT *psSP = psFence->apsFenceSyncList[i];
+		PVR_LOG(("    Point %u)", i));
+		PVR_LOG(("      On timeline:     %s (UID: %"IMG_UINT64_FMTSPEC")",
+		         psSP->psTl->pszName, psSP->psTl->iUID));
+		PVR_LOG(("      Sequence number: %u", psSP->uiSeqNum));
+		PVR_LOG(("      Signalled:       %s",
+		        psSP->psTl->sTlOps.pfnSyncPtHasSignalled(psSP)? "Yes":"No"));
+		PVR_LOG(("      Ref:             %d", OSAtomicRead(&psSP->iRef)));
+	}
+	PVR_LOG(("----------------------------------------"));
+
+	PVR_DPF_RETURN_RC1(eError, psFence);
+}
+
+static PVRSRV_ERROR _SyncFbTimelineCreate(PFN_SYNC_PT_HAS_SIGNALLED pfnHasPtSignalled,
+                                          IMG_UINT32 uiTimelineNameSize,
+                                          const IMG_CHAR *pszTimelineName,
+                                          PVRSRV_TIMELINE_SERVER **ppsTimeline)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_TIMELINE_SERVER *psNewTl;
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ppsTimeline, eError, e1);
+
+	psNewTl = OSAllocMem(sizeof(*psNewTl));
+	PVR_LOG_GOTO_IF_NOMEM(psNewTl, eError, e2);
+
+	eError = OSLockCreate(&psNewTl->hTlLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e3);
+
+	if (uiTimelineNameSize == 1)
+	{
+		OSSNPrintf(psNewTl->pszName,
+		           SYNC_FB_TIMELINE_MAX_LENGTH,
+		           "TL-%s-%d",
+		           OSGetCurrentClientProcessNameKM(),
+		           OSGetCurrentClientProcessIDKM());
+	}
+	else
+	{
+		if (pszTimelineName)
+		{
+			OSStringSafeCopy((IMG_CHAR*) psNewTl->pszName,
+			              pszTimelineName,
+			              SYNC_FB_TIMELINE_MAX_LENGTH);
+		}
+		else
+		{
+			psNewTl->pszName[0] = '\0';
+		}
+	}
+
+	dllist_init(&psNewTl->sSyncList);
+	dllist_init(&psNewTl->sSyncActiveList);
+	dllist_init(&psNewTl->sTlList);
+
+	psNewTl->sTlOps.pfnSyncPtHasSignalled = pfnHasPtSignalled;
+	psNewTl->iUID = (IMG_INT64)(uintptr_t) psNewTl; /* Not unique throughout the driver lifetime */
+	OSAtomicWrite(&psNewTl->iSeqNum, 0);
+	OSAtomicWrite(&psNewTl->iLastSignalledSeqNum, 0);
+	psNewTl->uiPID = OSGetCurrentClientProcessIDKM();
+
+	/* Set initial refcount value */
+	TL_REF_SET(&psNewTl->iRef, 1, psNewTl);
+
+	_SyncFbFTimelineListAdd(psNewTl);
+
+	*ppsTimeline = psNewTl;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNewTl);
+
+e3:
+	OSFreeMem(psNewTl);
+e2:
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         PVR SPECIFIC FUNCTIONS                            */
+/*                                                                           */
+/*****************************************************************************/
+
+/* Free a PVR sync point with its sync checkpoint */
+static void _SyncFbSyncPtFreePVR(IMG_HANDLE hSync)
+{
+	PVR_DPF_ENTERED1(hSync);
+
+	SyncCheckpointFree((PSYNC_CHECKPOINT) hSync);
+
+	PVR_DPF_RETURN;
+}
+
+
+/* Mark a sync checkpoint with the given state.
+ * MAKE SURE TO WAKE UP FW AFTER CALLING THIS */
+static PVRSRV_ERROR _SyncFbSyncPtSignalPVR(IMG_HANDLE hSync,
+                                           PVRSRV_SYNC_STATE eState)
+{
+	PSYNC_CHECKPOINT psSyncCheck = (PSYNC_CHECKPOINT) hSync;
+
+	PVR_DPF_ENTERED1(hSync);
+
+	if (!SyncCheckpointIsSignalled(psSyncCheck, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		switch (eState)
+		{
+			case PVRSRV_SYNC_SIGNALLED:
+				SyncCheckpointSignal(psSyncCheck, PVRSRV_FENCE_FLAG_NONE);
+				break;
+			case PVRSRV_SYNC_ERRORED:
+				SyncCheckpointError(psSyncCheck, PVRSRV_FENCE_FLAG_NONE);
+				break;
+			default:
+				ERR("Passed unknown sync state, "
+						"please use a valid one for signalling.");
+				return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, hSync);
+}
+
+/* Check whether the native sync of the SyncPt has signalled.
+ *
+ * HOLD TL LOCK!
+ */
+static IMG_BOOL _SyncFbSyncPtHasSignalledPVR(PVRSRV_SYNC_PT *psSyncPt)
+{
+	PDLLIST_NODE psCBNode;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+	PSYNC_CHECKPOINT psSyncCheck;
+	IMG_BOOL bRet = IMG_FALSE;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	/* If the SyncPt is not signalled yet,
+	 * check whether the first attached sync has.
+	 *
+	 * Change SyncPt state to signalled or errored if yes.
+	 * Also notify other attached syncs.
+	 */
+	if (OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_NOT_SIGNALLED)
+	{
+		/* List must have at least the device sync attached if we are called */
+		PVR_ASSERT(!dllist_is_empty(&psSyncPt->sSignalCallbacks));
+
+		/* Retrieve the first sync checkpoint of that sync pt */
+		psCBNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psCB = IMG_CONTAINER_OF(psCBNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+		psSyncCheck = (PSYNC_CHECKPOINT) psCB->hAttachedSync;
+
+		if (SyncCheckpointIsSignalled(psSyncCheck, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		{
+			_SyncFbSyncPtSignal(psSyncPt, PVRSRV_SYNC_SIGNALLED);
+
+			/* Signal all other attached syncs */
+			PVR_LOG_IF_ERROR(_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_SIGNALLED),
+			                 "_SyncFbSyncPtSignalAttached");
+
+			bRet = IMG_TRUE;
+		}
+
+		PVR_DPF_RETURN_RC1(bRet, psSyncPt);
+	}
+	else
+	{
+		PVR_DPF_RETURN_RC1(IMG_TRUE, psSyncPt);
+	}
+}
+
+PVRSRV_ERROR SyncFbTimelineCreatePVR(IMG_UINT32 uiTimelineNameSize,
+                                     const IMG_CHAR *pszTimelineName,
+                                     PVRSRV_TIMELINE_SERVER **ppsTimeline)
+{
+	return _SyncFbTimelineCreate(&_SyncFbSyncPtHasSignalledPVR,
+	                             uiTimelineNameSize,
+	                             pszTimelineName,
+	                             ppsTimeline);
+}
+
+PVRSRV_ERROR SyncFbFenceCreatePVR(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const IMG_CHAR *pszName,
+                                  PVRSRV_TIMELINE iTl,
+                                  PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                  PVRSRV_FENCE *piOutFence,
+                                  IMG_UINT64 *puiFenceUID,
+                                  void **ppvFenceFinaliseData,
+                                  PSYNC_CHECKPOINT *ppsOutCheckpoint,
+                                  void **ppvTimelineUpdateSync,
+                                  IMG_UINT32 *puiTimelineUpdateValue)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	PVRSRV_SYNC_PT *psNewSyncPt = NULL;
+	PVRSRV_SYNC_SIGNAL_CB *psNewSyncSignalCB;
+	PVRSRV_HANDLE_BASE	*psHandleBase;
+	PVRSRV_TIMELINE_SERVER *psTl;
+	IMG_HANDLE hOutFence;
+	IMG_UINT32 ui32FwAddr;
+
+	PVR_UNREFERENCED_PARAMETER(ppvTimelineUpdateSync);
+	PVR_UNREFERENCED_PARAMETER(puiTimelineUpdateValue);
+
+	PVR_DPF_ENTERED;
+
+	/* The fallback implementation does not need to finalise
+	 * the fence, so set the ppvFenceFinaliseData to NULL
+	 * (if provided)
+	 */
+	if (ppvFenceFinaliseData != NULL )
+	{
+		*ppvFenceFinaliseData = NULL;
+	}
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(pszName, eError, e0);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(piOutFence, eError, e0);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ppsOutCheckpoint, eError, e0);
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) iTl,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+	                                 IMG_TRUE,
+	                                 (void**) &psTl,
+	                                 &psHandleBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (unlikely(_SyncFbTimelineHandleType(psTl) != PVRSRV_SYNC_HANDLE_PVR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_SyncFbTimelineHandleType", eError, PVRSRV_ERROR_INVALID_PARAMS, e1);
+	}
+
+	/* Allocate:
+	 *		Fence
+	 *		Sync Signal CB
+	 *		SyncPt List
+	 *		Sync Checkpoint
+	 *		SyncPt
+	 *		Handle
+	 *	Setup
+	 */
+	psNewFence = OSAllocMem(sizeof(*psNewFence));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence, eError, e2);
+
+	psNewSyncSignalCB = OSAllocMem(sizeof(*psNewSyncSignalCB));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSyncSignalCB, eError, e3);
+
+	psNewFence->apsFenceSyncList = OSAllocMem(sizeof(*(psNewFence->apsFenceSyncList)));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence->apsFenceSyncList, eError, e4);
+
+	psNewFence->psDevNode = psDeviceNode;
+
+	/* Lock down TL until new point is fully created and inserted */
+	OSLockAcquire(psTl->hTlLock);
+
+	eError = _SyncFbSyncPtCreate(&psNewSyncPt,
+	                             psTl,
+	                             OSAtomicIncrement(&psTl->iSeqNum));
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		OSLockRelease(psTl->hTlLock);
+		ERR("Cannot allocate SyncPt.");
+		goto e5;
+	}
+
+	eError = SyncCheckpointAlloc(psSyncCheckpointContext,
+	                             iTl,
+	                             PVRSRV_NO_FENCE,
+	                             pszName,
+	                             ppsOutCheckpoint);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		OSLockRelease(psTl->hTlLock);
+		ERR("Cannot allocate SyncCheckpoint.");
+		goto e6;
+	}
+
+	ui32FwAddr = SyncCheckpointGetFirmwareAddr(*ppsOutCheckpoint);
+
+	/* Init Sync Signal CB */
+	psNewSyncSignalCB->hAttachedSync = (IMG_HANDLE) *ppsOutCheckpoint;
+	psNewSyncSignalCB->pfnSignal = &_SyncFbSyncPtSignalPVR;
+	psNewSyncSignalCB->pfnSyncFree = &_SyncFbSyncPtFreePVR;
+	psNewSyncSignalCB->hPrivData = (IMG_HANDLE) psSyncCheckpointContext;
+
+	dllist_add_to_tail(&psNewSyncPt->sSignalCallbacks,
+	                   &psNewSyncSignalCB->sCallbackNode);
+
+	OSLockRelease(psTl->hTlLock);
+
+	DBG(("%s: Inserting key <%p> = %p", __func__, *ppsOutCheckpoint, psNewSyncPt));
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+	HASH_Insert_Extended(gsSyncFbContext.sCheckpointHashTable, (void *)&ui32FwAddr, (uintptr_t) psNewSyncPt);
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	/* Init Fence */
+	OSStringSafeCopy(psNewFence->pszName,
+	              pszName,
+	              SYNC_FB_FENCE_MAX_LENGTH);
+
+	psNewFence->apsFenceSyncList[0] = psNewSyncPt;
+	psNewFence->uiNumSyncs = 1;
+	FENCE_REF_SET(&psNewFence->iRef, 1, psNewFence);
+	OSAtomicWrite(&psNewFence->iStatus, PVRSRV_SYNC_NOT_SIGNALLED);
+	psNewFence->iUID = (IMG_INT64)(uintptr_t) psNewFence; /* Not unique throughout the driver lifetime */
+
+	eError = PVRSRVAllocHandle(psHandleBase,
+	                           &hOutFence,
+	                           (void*) psNewFence,
+	                           PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                           PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+	                           (PFN_HANDLE_RELEASE) &SyncFbFenceRelease);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandle", e7);
+
+	_SyncFbFenceListAdd(psNewFence);
+
+	PDUMPCOMMENTWITHFLAGS(psNewFence->psDevNode, 0,
+	                      "Allocated PVR Fence %s (ID:%"IMG_UINT64_FMTSPEC") with Checkpoint (ID:%d) "
+	                      "on Timeline %s (ID:%"IMG_UINT64_FMTSPEC")",
+	                      psNewFence->pszName,
+	                      psNewFence->iUID,
+	                      SyncCheckpointGetId(psNewSyncSignalCB->hAttachedSync),
+	                      psTl->pszName,
+	                      psTl->iUID);
+
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iTl,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+
+	*puiFenceUID = psNewFence->iUID;
+	*piOutFence = (PVRSRV_FENCE) (uintptr_t) hOutFence;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNewFence);
+
+e7:
+	SyncCheckpointFree(*ppsOutCheckpoint);
+e6:
+	_SyncFbSyncPtRelease(psNewSyncPt, IMG_FALSE);
+e5:
+	OSFreeMem(psNewFence->apsFenceSyncList);
+e4:
+	OSFreeMem(psNewSyncSignalCB);
+e3:
+	OSFreeMem(psNewFence);
+e2:
+e1:
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iTl,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+e0:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Resolve caller has to free the sync checkpoints and free the
+ * array that holds the pointers. */
+PVRSRV_ERROR SyncFbFenceResolvePVR(PSYNC_CHECKPOINT_CONTEXT psContext,
+                                   PVRSRV_FENCE iFence,
+                                   IMG_UINT32 *puiNumCheckpoints,
+                                   PSYNC_CHECKPOINT **papsCheckpoints,
+                                   IMG_UINT64 *pui64FenceUID)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psFence;
+	PVRSRV_HANDLE_BASE *psHBase;
+	PSYNC_CHECKPOINT *apsCheckpoints;
+	PSYNC_CHECKPOINT psCheckpoint;
+	PVRSRV_SYNC_SIGNAL_CB *psSyncCB, *psNewSyncCB;
+	PVRSRV_SYNC_PT *psSyncPt;
+	PDLLIST_NODE psNode;
+	IMG_UINT32 i, uiNumCheckpoints = 0;
+
+	PVR_DPF_ENTERED;
+
+	if (iFence == PVRSRV_NO_FENCE)
+	{
+		*puiNumCheckpoints = 0;
+		eError = PVRSRV_OK;
+		goto e0;
+	}
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) iFence,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                                 IMG_TRUE,
+	                                 (void**)&psFence,
+	                                 &psHBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	apsCheckpoints = OSAllocMem(sizeof(*apsCheckpoints) * psFence->uiNumSyncs);
+	PVR_LOG_GOTO_IF_NOMEM(apsCheckpoints, eError, e1);
+
+	/* Go through all syncs and add them to the list */
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		psNewSyncCB = NULL;
+		psSyncPt = psFence->apsFenceSyncList[i];
+
+		/* Don't skip signalled fences on PDUMP to make sure we set up proper
+		   synchronisation in the pdump stream. */
+#if !defined(PDUMP)
+		if (_SyncFbSyncPtHasSignalled(psSyncPt))
+		{
+			continue;
+		}
+#endif
+
+		OSLockAcquire(gsSyncFbContext.hFbContextLock);
+		OSLockAcquire(psSyncPt->psTl->hTlLock);
+		psNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psSyncCB = IMG_CONTAINER_OF(psNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+
+		/* If we have a sync checkpoint AND
+		 * it uses the same context as the given one,
+		 * just add the checkpoint to the resolve list.*/
+		if ((_SyncFbSyncPtHandleType(psSyncCB) == PVRSRV_SYNC_HANDLE_PVR) &&
+		    (psContext == (PSYNC_CHECKPOINT_CONTEXT) psSyncCB->hPrivData))
+		{
+			psCheckpoint = (PSYNC_CHECKPOINT) psSyncCB->hAttachedSync;
+		}
+		/* Else create a new sync checkpoint in the given context */
+		else
+		{
+			IMG_UINT32 ui32FwAddr;
+
+			eError = SyncCheckpointAlloc(psContext,
+			                             SYNC_CHECKPOINT_FOREIGN_CHECKPOINT,
+			                             iFence,
+			                             psFence->pszName,
+			                             &psCheckpoint);
+			if (eError != PVRSRV_OK)
+			{
+				OSLockRelease(psSyncPt->psTl->hTlLock);
+				OSLockRelease(gsSyncFbContext.hFbContextLock);
+				goto e2;
+			}
+
+			psNewSyncCB = OSAllocMem(sizeof(*psNewSyncCB));
+			if (psNewSyncCB == NULL)
+			{
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				OSLockRelease(psSyncPt->psTl->hTlLock);
+				OSLockRelease(gsSyncFbContext.hFbContextLock);
+				goto e3;
+			}
+
+			psNewSyncCB->hAttachedSync = (IMG_HANDLE) psCheckpoint;
+			psNewSyncCB->hPrivData = (IMG_HANDLE) psContext;
+			psNewSyncCB->pfnSignal = &_SyncFbSyncPtSignalPVR;
+			psNewSyncCB->pfnSyncFree = &_SyncFbSyncPtFreePVR;
+			dllist_add_to_tail(&psFence->apsFenceSyncList[i]->sSignalCallbacks,
+			                   &psNewSyncCB->sCallbackNode);
+
+			/* Insert the checkpoint into the firmware address lookup table,
+			 * in case this checkpoint gets errored by the host driver. */
+			ui32FwAddr = SyncCheckpointGetFirmwareAddr(psCheckpoint);
+
+			DBG(("%s: Inserting key (fwAddr0x%x)<%p> = %p", __func__, ui32FwAddr, psCheckpoint, psSyncPt));
+			HASH_Insert_Extended(gsSyncFbContext.sCheckpointHashTable, (void *)&ui32FwAddr, (uintptr_t) psSyncPt);
+
+			/* If the existing sync pt has already been signalled, then signal
+			 * this new sync too */
+			if (_SyncFbSyncPtHasSignalled(psFence->apsFenceSyncList[i]))
+			{
+				_SyncFbSyncPtSignalPVR(psNewSyncCB->hAttachedSync, PVRSRV_SYNC_SIGNALLED);
+			}
+		}
+		OSLockRelease(psSyncPt->psTl->hTlLock);
+		OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+		/* Take a reference, resolve caller is responsible
+		 * to drop it after use */
+		eError = SyncCheckpointTakeRef(psCheckpoint);
+		PVR_GOTO_IF_ERROR(eError, e4);
+
+		apsCheckpoints[uiNumCheckpoints++] = psCheckpoint;
+	}
+
+	*pui64FenceUID = psFence->iUID;
+	*puiNumCheckpoints = uiNumCheckpoints;
+	*papsCheckpoints = apsCheckpoints;
+
+	PVRSRVReleaseHandle(psHBase,
+	                    (IMG_HANDLE) (uintptr_t) iFence,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+
+	PVR_DPF_RETURN_OK;
+
+e4:
+	if (psNewSyncCB)
+		OSFreeMem(psNewSyncCB);
+e3:
+	SyncCheckpointFree(psCheckpoint);
+e2:
+	for (; i > 0; i--)
+	{
+		SyncCheckpointDropRef(apsCheckpoints[i-1]);
+		SyncCheckpointFree(apsCheckpoints[i-1]);
+	}
+
+	OSFreeMem(apsCheckpoints);
+e1:
+	PVRSRVReleaseHandle(psHBase,
+	                    (IMG_HANDLE) (uintptr_t) iFence,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+e0:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* In case something went wrong after FenceCreate we can roll back (destroy)
+ * the fence in the server */
+static PVRSRV_ERROR SyncFbFenceRollbackPVR(PVRSRV_FENCE iFence, void *pvFenceData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+	PVR_UNREFERENCED_PARAMETER(pvFenceData);
+
+	PVR_GOTO_IF_INVALID_PARAM(iFence != PVRSRV_NO_FENCE, eError, e1);
+
+	eError = _SyncFbDestroyHandle((IMG_HANDLE) (uintptr_t) iFence,
+	                              PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	PVR_DPF_RETURN_OK;
+
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Dump debug info on syncs relating to any of the FWAddrs in the
+ * given array. This is called when rgx_ccb.c determines we have a
+ * stalled CCB, so this debug will aid in debugging which sync(s)
+ * have failed to signal.
+ */
+IMG_UINT32 SyncFbDumpInfoOnStalledUFOs(IMG_UINT32 nr_ufos, IMG_UINT32 *vaddrs)
+{
+	IMG_UINT32 ui32NumFallbackUfos = 0;
+	PDLLIST_NODE psFenceNode, psNextFenceNode;
+	PVRSRV_FENCE_SERVER *psFence;
+	IMG_UINT32 *pui32NextFWAddr = vaddrs;
+	IMG_UINT32 ui32CurrentUfo;
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+	for (ui32CurrentUfo=0; ui32CurrentUfo<nr_ufos; ui32CurrentUfo++)
+	{
+		if (pui32NextFWAddr)
+		{
+			/* Iterate over all fences */
+			dllist_foreach_node(&gsSyncFbContext.sFenceList,
+								psFenceNode,
+								psNextFenceNode)
+			{
+				IMG_UINT32 i;
+				IMG_BOOL bFenceDumped = IMG_FALSE;
+				IMG_UINT32			  ui32SyncPtBitmask = 0;
+				IMG_UINT32			  ui32SyncCheckpointFWAddr = 0;
+				PVRSRV_SYNC_PT		  *psSyncPt = NULL;
+
+				psFence = IMG_CONTAINER_OF(psFenceNode,
+										   PVRSRV_FENCE_SERVER,
+										   sFenceListNode);
+
+				/* ... all sync points in the fence */
+				for (i = 0; i < psFence->uiNumSyncs; i++)
+				{
+					PDLLIST_NODE psCBNode, psNextCBNode;
+
+					psSyncPt = psFence->apsFenceSyncList[i];
+
+					dllist_foreach_node(&psSyncPt->sSignalCallbacks,
+										psCBNode,
+										psNextCBNode)
+					{
+						PVRSRV_SYNC_SIGNAL_CB *psCb = IMG_CONTAINER_OF(psCBNode,
+						                                               PVRSRV_SYNC_SIGNAL_CB,
+						                                               sCallbackNode);
+
+						switch (_SyncFbSyncPtHandleType(psCb))
+						{
+							case PVRSRV_SYNC_HANDLE_PVR:
+							{
+								ui32SyncCheckpointFWAddr = SyncCheckpointGetFirmwareAddr(psCb->hAttachedSync);
+								ui32SyncPtBitmask |= 1;
+								break;
+							}
+							case PVRSRV_SYNC_HANDLE_SW:
+							{
+								ui32SyncPtBitmask |= 2;
+								break;
+							}
+							default:
+								break;
+						}
+					}
+				}
+
+				if ((ui32SyncPtBitmask == 0x3) &&
+					(ui32SyncCheckpointFWAddr == *pui32NextFWAddr))
+				{
+					/* Print fence info (if not already done so) */
+					if (!bFenceDumped)
+					{
+						PVR_LOG(("Fence: %s, ID: %"IMG_UINT64_FMTSPEC", %s - (0x%p)",
+								  psFence->pszName,
+								  psFence->iUID,
+								  _SyncFbFenceSyncsHaveSignalled(psFence) ?
+										  "Signalled" : "Pending  ",
+								  psFence));
+						bFenceDumped = IMG_TRUE;
+					}
+					_SyncFbDebugRequestPrintSyncPt(psSyncPt,
+					                               IMG_TRUE,
+												   NULL,
+												   NULL);
+					ui32NumFallbackUfos++;
+				}
+			}
+			pui32NextFWAddr++;
+		}
+	}
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	return ui32NumFallbackUfos;
+}
+
+static inline PVRSRV_SYNC_STATE _SyncFbCheckpointToSyncState(PVRSRV_SYNC_CHECKPOINT_STATE eState)
+{
+	switch (eState)
+	{
+		case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			return PVRSRV_SYNC_SIGNALLED;
+		case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			return PVRSRV_SYNC_ERRORED;
+		case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			return PVRSRV_SYNC_NOT_SIGNALLED;
+		default:
+			PVR_ASSERT("Unknown sync checkpoint state");
+			return PVRSRV_SYNC_ERRORED;
+	}
+}
+
+IMG_BOOL SyncFbCheckpointHasSignalled(IMG_UINT32 ui32FwAddr, IMG_UINT32 ui32Value)
+{
+	PVRSRV_SYNC_PT *psSyncPt;
+	PVRSRV_SYNC_STATE eSignal;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+	PDLLIST_NODE psCurrentCB, psNextCB;
+
+	PVR_DPF_ENTERED1(ui32FwAddr);
+
+	OSLockAcquire(gsSyncFbContext.hFbContextLock);
+
+	DBG(("%s: Looking up key (fwAddr=0x%x)", __func__, ui32FwAddr));
+	psSyncPt = (PVRSRV_SYNC_PT *) HASH_Retrieve_Extended(gsSyncFbContext.sCheckpointHashTable, (void *) &ui32FwAddr);
+	DBG(("%s: Found sync pt <%p>", __func__, psSyncPt));
+
+	if (!psSyncPt)
+	{
+		/* Sync Fallback does not know about this UFO, skip processing it. */
+		goto err_release;
+	}
+
+	eSignal = _SyncFbCheckpointToSyncState(ui32Value);
+	_SyncFbSyncPtSignal(psSyncPt, eSignal);
+
+	/*
+	 * Signal all attachments except for the one representing the signalled
+	 * checkpoint.
+	 */
+
+	if (dllist_is_empty(&psSyncPt->sSignalCallbacks))
+	{
+		ERR("Sync pt has no attached syncs. Make sure to attach one "
+		    "when creating a new sync pt to define its flavour");
+	}
+
+	dllist_foreach_node(&psSyncPt->sSignalCallbacks, psCurrentCB, psNextCB)
+	{
+		psCB = IMG_CONTAINER_OF(psCurrentCB,
+		                        PVRSRV_SYNC_SIGNAL_CB,
+		                        sCallbackNode);
+
+		if (_SyncFbSyncPtHandleType(psCB) == PVRSRV_SYNC_HANDLE_PVR &&
+		    SyncCheckpointGetFirmwareAddr(psCB->hAttachedSync) == ui32FwAddr)
+		{
+			continue;
+		}
+
+		if (psCB->pfnSignal(psCB->hAttachedSync, eSignal) != PVRSRV_OK)
+		{
+			ERR("Failed to signal an attached sync, system might block!");
+			/* Don't jump to exit but try to signal remaining syncs */
+		}
+	}
+
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+
+	PVR_DPF_RETURN_RC(IMG_TRUE);
+
+err_release:
+	OSLockRelease(gsSyncFbContext.hFbContextLock);
+	PVR_DPF_RETURN_RC(IMG_FALSE);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         SW SPECIFIC FUNCTIONS                            */
+/*                                                                           */
+/*****************************************************************************/
+
+/* Free a SW sync point with its sync checkpoint */
+static void _SyncFbSyncPtFreeSW(IMG_HANDLE hSync)
+{
+	PVR_DPF_ENTERED1(hSync);
+
+	OSFreeMem(hSync);
+
+	PVR_DPF_RETURN;
+}
+
+static IMG_BOOL _SyncFbSyncPtHasSignalledSW(PVRSRV_SYNC_PT *psSyncPt)
+{
+	PDLLIST_NODE psCBNode;
+	PVRSRV_SYNC_SIGNAL_CB *psCB;
+	PVRSRV_SYNC_PT_SW *psSWSyncPt;
+	IMG_BOOL bRet = IMG_FALSE;
+
+	PVR_DPF_ENTERED1(psSyncPt);
+
+	/* If the SyncPt has not been signalled yet,
+	 * check whether the first attached sync has.
+	 *
+	 * Change SyncPt state to signalled or errored if yes.
+	 * Also notify other attached syncs.
+	 */
+	if (OSAtomicRead(&psSyncPt->iStatus) == PVRSRV_SYNC_NOT_SIGNALLED)
+	{
+		/* List must have at least the device sync attached if we are called */
+		PVR_ASSERT(!dllist_is_empty(&psSyncPt->sSignalCallbacks));
+
+		/* Retrieve the first sync checkpoint of that sync pt */
+		psCBNode = dllist_get_next_node(&psSyncPt->sSignalCallbacks);
+		psCB = IMG_CONTAINER_OF(psCBNode, PVRSRV_SYNC_SIGNAL_CB, sCallbackNode);
+		psSWSyncPt = (PVRSRV_SYNC_PT_SW*) psCB->hAttachedSync;
+
+		if (psSWSyncPt->bSignalled)
+		{
+			_SyncFbSyncPtSignal(psSyncPt, PVRSRV_SYNC_SIGNALLED);
+
+			/* Signal all other attached syncs */
+			PVR_LOG_IF_ERROR(_SyncFbSyncPtSignalAttached(psSyncPt, PVRSRV_SYNC_SIGNALLED),
+			                 "_SyncFbSyncPtSignalAttached");
+
+			bRet = IMG_TRUE;
+		}
+
+		PVR_DPF_RETURN_RC1(bRet, psSyncPt);
+	}
+	else
+	{
+		PVR_DPF_RETURN_RC1(IMG_TRUE, psSyncPt);
+	}
+}
+
+/* Mark an attached sw sync pt with the given state.
+ * MAKE SURE TO WAKE UP FW AFTER CALLING THIS (if enqueued)*/
+static PVRSRV_ERROR _SyncFbSyncPtSignalSW(IMG_HANDLE hSync,
+                                          PVRSRV_SYNC_STATE eState)
+{
+	PVRSRV_SYNC_PT_SW *psSWSyncPt = (PVRSRV_SYNC_PT_SW*) hSync;
+
+	PVR_DPF_ENTERED1(hSync);
+
+	if (!psSWSyncPt->bSignalled)
+	{
+		switch (eState)
+		{
+			case PVRSRV_SYNC_SIGNALLED:
+			case PVRSRV_SYNC_ERRORED:
+				psSWSyncPt->bSignalled = IMG_TRUE;
+				break;
+			default:
+				ERR("Passed unknown sync state (%d), "
+						"please use a valid one for signalling.",
+						eState);
+				return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, hSync);
+}
+
+PVRSRV_ERROR SyncFbTimelineCreateSW(IMG_UINT32 uiTimelineNameSize,
+                                    const IMG_CHAR *pszTimelineName,
+                                    PVRSRV_TIMELINE_SERVER **ppsTimeline)
+{
+	return _SyncFbTimelineCreate(&_SyncFbSyncPtHasSignalledSW,
+	                             uiTimelineNameSize,
+	                             pszTimelineName,
+	                             ppsTimeline);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                       SOFTWARE_TIMELINE FUNCTIONS                         */
+/*                                                                           */
+/*****************************************************************************/
+static PVRSRV_ERROR _SyncFbSWTimelineFenceCreate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          PVRSRV_TIMELINE_SERVER *psTl,
+                                          const IMG_CHAR *pszFenceName,
+                                          PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                          IMG_UINT64 *pui64SyncPtIdx)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	PVRSRV_SYNC_PT *psNewSyncPt = NULL;
+	PVRSRV_SYNC_PT_SW *psNewSWSyncPt;
+	PVRSRV_SYNC_SIGNAL_CB *psNewSyncSignalCB;
+	IMG_INT iNextSeqNum;
+
+	if (_SyncFbTimelineHandleType(psTl) != PVRSRV_SYNC_HANDLE_SW)
+	{
+		ERR("Passed timeline is not a SW timeline.");
+		eError = PVRSRV_ERROR_NOT_SW_TIMELINE;
+		goto e1;
+	}
+
+	/* Allocate:
+	 *		Fence
+	 *		Sync Signal CB
+	 *		SyncPt List
+	 *		SW Sync
+	 *		SyncPt
+	 *		Handle
+	 *	Setup
+	 */
+	psNewFence = OSAllocMem(sizeof(*psNewFence));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence, eError, e1);
+
+	psNewFence->psDevNode = psDeviceNode;
+
+	psNewSyncSignalCB = OSAllocMem(sizeof(*psNewSyncSignalCB));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSyncSignalCB, eError, e2);
+
+	psNewFence->apsFenceSyncList = OSAllocMem(sizeof(*(psNewFence->apsFenceSyncList)));
+	PVR_LOG_GOTO_IF_NOMEM(psNewFence->apsFenceSyncList, eError, e3);
+
+	psNewSWSyncPt = OSAllocMem(sizeof(*psNewSWSyncPt));
+	PVR_LOG_GOTO_IF_NOMEM(psNewSWSyncPt, eError, e4);
+
+	/* Lock down TL until new point is fully created and inserted */
+	OSLockAcquire(psTl->hTlLock);
+
+	/* Sample our next sync pt value - we won't actually increment
+	 * iSeqNum for this SW timeline until we know the fence has been
+	 * successfully created.
+	 */
+	iNextSeqNum = OSAtomicRead(&psTl->iSeqNum) + 1;
+
+	eError = _SyncFbSyncPtCreate(&psNewSyncPt, psTl, iNextSeqNum);
+	if (eError != PVRSRV_OK)
+	{
+		OSLockRelease(psTl->hTlLock);
+		goto e5;
+	}
+
+	if (OSAtomicRead(&psTl->iLastSignalledSeqNum) < psNewSyncPt->uiSeqNum)
+	{
+		psNewSWSyncPt->bSignalled = IMG_FALSE;
+	}
+	else
+	{
+		psNewSWSyncPt->bSignalled = IMG_TRUE;
+		OSAtomicWrite(&psNewSyncPt->iStatus, PVRSRV_SYNC_SIGNALLED);
+	}
+
+	/* Init Sync Signal CB */
+	psNewSyncSignalCB->hAttachedSync = (IMG_HANDLE) psNewSWSyncPt;
+	psNewSyncSignalCB->pfnSignal = &_SyncFbSyncPtSignalSW;
+	psNewSyncSignalCB->pfnSyncFree = &_SyncFbSyncPtFreeSW;
+	psNewSyncSignalCB->hPrivData = NULL;
+
+	dllist_add_to_tail(&psNewSyncPt->sSignalCallbacks,
+					   &psNewSyncSignalCB->sCallbackNode);
+
+	/* Now that the fence has been created, increment iSeqNum */
+	OSAtomicIncrement(&psTl->iSeqNum);
+
+	OSLockRelease(psTl->hTlLock);
+
+	if (pszFenceName)
+	{
+		/* Init Fence */
+		OSStringSafeCopy(psNewFence->pszName,
+		              pszFenceName,
+		              SYNC_FB_FENCE_MAX_LENGTH);
+	}
+	else
+	{
+		psNewFence->pszName[0] = '\0';
+	}
+
+	psNewFence->apsFenceSyncList[0] = psNewSyncPt;
+	psNewFence->uiNumSyncs = 1;
+	FENCE_REF_SET(&psNewFence->iRef, 1, psNewFence);
+	OSAtomicWrite(&psNewFence->iStatus, PVRSRV_FENCE_NOT_SIGNALLED);
+	psNewFence->iUID = (IMG_INT64)(uintptr_t) psNewFence;
+
+	_SyncFbFenceListAdd(psNewFence);
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+						  "Allocated SW Fence %s (ID:%"IMG_UINT64_FMTSPEC") with sequence number %u "
+						  "on Timeline %s (ID:%"IMG_UINT64_FMTSPEC")",
+						  psNewFence->pszName,
+						  psNewFence->iUID,
+						  psNewSyncPt->uiSeqNum,
+						  psTl->pszName,
+						  psTl->iUID);
+
+	*ppsOutputFence = psNewFence;
+	if (pui64SyncPtIdx != NULL)
+	{
+		*pui64SyncPtIdx = psNewSyncPt->uiSeqNum;
+	}
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNewFence);
+
+e5:
+	OSFreeMem(psNewSWSyncPt);
+e4:
+	OSFreeMem(psNewFence->apsFenceSyncList);
+e3:
+	OSFreeMem(psNewSyncSignalCB);
+e2:
+	OSFreeMem(psNewFence);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/* Kernel mode function (SyncFb implementation) to create fence on a SW timeline */
+PVRSRV_ERROR SyncFbSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           PVRSRV_TIMELINE iSWTimeline,
+                                           const IMG_CHAR *pszFenceName,
+                                           PVRSRV_FENCE *piOutputFence,
+                                           IMG_UINT64 *pui64SyncPtIdx)
+{
+
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psNewFence;
+	PVRSRV_HANDLE_BASE	*psHandleBase;
+	PVRSRV_TIMELINE_SERVER *psTl;
+	IMG_HANDLE hOutFence;
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(piOutputFence, eError, e0);
+
+	/* Lookup up the ST Timeline (and take a reference on it while
+	 * we are creating the new sync pt and fence)
+	 */
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE) (uintptr_t) iSWTimeline,
+									 PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+									 IMG_TRUE,
+									 (void**) &psTl,
+									 &psHandleBase);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	eError = _SyncFbSWTimelineFenceCreate(psDeviceNode, psTl,
+	                                      pszFenceName,
+	                                      &psNewFence,
+	                                      pui64SyncPtIdx);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	eError = PVRSRVAllocHandle(psHandleBase,
+							   &hOutFence,
+							   (void*) psNewFence,
+							   PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+							   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							   (PFN_HANDLE_RELEASE) &SyncFbFenceRelease);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	/* Drop the reference we took on the timeline earlier */
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iSWTimeline,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+
+	*piOutputFence = (PVRSRV_FENCE) (uintptr_t) hOutFence;
+
+	return PVRSRV_OK;
+
+e2:
+	/* Release the fence we created, as we failed to
+	 * allocate a handle for it */
+	SyncFbFenceRelease(psNewFence);
+
+e1:
+	/* Drop the reference we took on the timeline earlier */
+	PVRSRVReleaseHandle(psHandleBase,
+	                    (IMG_HANDLE) (uintptr_t) iSWTimeline,
+	                    PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER);
+e0:
+	PVR_DPF_RETURN_RC(eError);
+
+}
+
+/* Client (bridge) interface to the SyncSWTimelineFenceCreateKM() function */
+PVRSRV_ERROR SyncFbFenceCreateSW(CONNECTION_DATA *psConnection,
+                                 PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 PVRSRV_TIMELINE_SERVER *psTimeline,
+                                 IMG_UINT32 uiFenceNameSize,
+                                 const IMG_CHAR *pszFenceName,
+                                 PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                 IMG_UINT64 *pui64SyncPtIdx)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(uiFenceNameSize);
+
+	eError =  _SyncFbSWTimelineFenceCreate(psDeviceNode,
+	                                       psTimeline,
+	                                       pszFenceName,
+	                                       ppsOutputFence,
+	                                       pui64SyncPtIdx);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _SyncSWTimelineAdvanceSigErr(PVRSRV_TIMELINE_SERVER *psTl,
+                                                 PVRSRV_SYNC_STATE eState,
+                                                 IMG_UINT64 *pui64SyncPtIdx)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PDLLIST_NODE psPtNode, psNextNode;
+	PVRSRV_SYNC_PT *psSyncPt;
+	IMG_INT32 uiTlSeqNum;
+
+	PVR_DPF_ENTERED1(psTl);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psTl, eError, errExit);
+
+	if (_SyncFbTimelineHandleType(psTl) != PVRSRV_SYNC_HANDLE_SW)
+	{
+		ERR("Passed timeline is not a SW timeline.");
+		eError = PVRSRV_ERROR_NOT_SW_TIMELINE;
+		goto errExit;
+	}
+
+	OSLockAcquire(psTl->hTlLock);
+
+	/* Don't allow incrementing of SW timeline beyond its last created pt */
+	if (OSAtomicRead(&psTl->iLastSignalledSeqNum) == OSAtomicRead(&psTl->iSeqNum))
+	{
+		DBG(("%s: !! TL<%p> (%d->%d/%d) !!", __func__, (void*)psTl, OSAtomicRead(&psTl->iLastSignalledSeqNum), OSAtomicRead(&psTl->iLastSignalledSeqNum)+1, OSAtomicRead(&psTl->iSeqNum)));
+		WRN("Attempt to advance SW timeline beyond last created point.");
+		eError = PVRSRV_ERROR_SW_TIMELINE_AT_LATEST_POINT;
+		goto errUnlockAndExit;
+	}
+
+	uiTlSeqNum = OSAtomicIncrement(&psTl->iLastSignalledSeqNum);
+
+	/* Go through list of active sync pts and
+	 * signal the points that are met now */
+	dllist_foreach_node(&psTl->sSyncActiveList, psPtNode, psNextNode)
+	{
+		psSyncPt = IMG_CONTAINER_OF(psPtNode,
+		                            PVRSRV_SYNC_PT,
+		                            sTlSyncActiveList);
+		if (psSyncPt->uiSeqNum <= uiTlSeqNum)
+		{
+			_SyncFbSyncPtSignal(psSyncPt, eState);
+
+			/* Signal all other attached syncs */
+			PVR_LOG_IF_ERROR(_SyncFbSyncPtSignalAttached(psSyncPt, eState),
+			                 "_SyncFbSyncPtSignalAttached");
+
+			dllist_remove_node(psPtNode);
+		}
+	}
+
+	OSLockRelease(psTl->hTlLock);
+
+	if (pui64SyncPtIdx != NULL)
+	{
+		*pui64SyncPtIdx = uiTlSeqNum;
+	}
+
+	/* A completed SW operation may un-block the GPU */
+	PVRSRVCheckStatus(NULL);
+
+	eError = _SyncFbSignalEO();
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbSignalEO", errExit);
+
+	PVR_DPF_RETURN_OK;
+
+errUnlockAndExit:
+	OSLockRelease(psTl->hTlLock);
+errExit:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static void _SyncSWTimelineCheckForUnsignalledPts(PVRSRV_TIMELINE_SERVER *psTl)
+{
+	IMG_UINT32 ui32TlSeqNum = OSAtomicRead(&psTl->iSeqNum);
+	IMG_UINT32 ui32TlLastSigSeqNum = OSAtomicRead(&psTl->iLastSignalledSeqNum);
+
+	while (ui32TlLastSigSeqNum < ui32TlSeqNum)
+	{
+		ui32TlLastSigSeqNum++;
+		PVR_DPF((PVR_DBG_WARNING,"%s: Found unsignalled SW timeline <%p> '%s' sync pt (%d/%d)",__func__, psTl, psTl->pszName, ui32TlLastSigSeqNum, ui32TlSeqNum));
+		_SyncSWTimelineAdvanceSigErr(psTl, PVRSRV_SYNC_ERRORED, NULL);
+	}
+}
+
+/* kernel mode function to advance a SW timeline */
+PVRSRV_ERROR SyncFbSWTimelineAdvanceKM(void *pvSWTimelineObj, IMG_UINT64 *pui64SyncPtIdx)
+{
+	return _SyncSWTimelineAdvanceSigErr(pvSWTimelineObj,
+	                                    PVRSRV_SYNC_SIGNALLED,
+	                                    pui64SyncPtIdx);
+}
+
+/* Client (bridge) interface to the SyncSWTimelineAdvanceKM() function */
+PVRSRV_ERROR SyncFbTimelineAdvanceSW(PVRSRV_TIMELINE_SERVER *psTimeline,
+                                     IMG_UINT64 *pui64SyncPtIdx)
+{
+	return _SyncSWTimelineAdvanceSigErr(psTimeline,
+	                                    PVRSRV_SYNC_SIGNALLED,
+	                                    pui64SyncPtIdx);
+}
+
+PVRSRV_ERROR SyncFbFenceReleaseKM(void *pvFenceObj)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED1(pvFenceObj);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(pvFenceObj, eError, e0);
+
+	eError = SyncFbFenceRelease((PVRSRV_FENCE_SERVER*) pvFenceObj);
+
+e0:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbSWGetTimelineObj(PVRSRV_TIMELINE iSWTimeline,
+                                    void **ppvSWTimelineObj)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHB;
+
+	PVR_DPF_ENTERED1(iSWTimeline);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(iSWTimeline != PVRSRV_NO_TIMELINE, eError, err_out);
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE)(uintptr_t) iSWTimeline,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+	                                 IMG_FALSE,
+	                                 ppvSWTimelineObj,
+	                                 &psHB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbLookupProcHandle", err_out);
+
+	_SyncFbTimelineAcquire((PVRSRV_TIMELINE_SERVER*) *ppvSWTimelineObj);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+PVRSRV_ERROR SyncFbGetFenceObj(PVRSRV_FENCE iFence,
+                               void **ppvFenceObj)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_HANDLE_BASE *psHB;
+
+	PVR_DPF_ENTERED1(iFence);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(iFence != PVRSRV_NO_FENCE, eError, err_out);
+
+	eError = _SyncFbLookupProcHandle((IMG_HANDLE)(uintptr_t) iFence,
+	                                 PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	                                 IMG_FALSE,
+	                                 ppvFenceObj,
+	                                 &psHB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbLookupProcHandle", err_out);
+
+	_SyncFbFenceAcquire((PVRSRV_FENCE_SERVER*) *ppvFenceObj);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbDumpFenceKM(void *pvSWFenceObj,
+	                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	                           void *pvDumpDebugFile)
+{
+	PVRSRV_FENCE_SERVER *psFence = (PVRSRV_FENCE_SERVER *) pvSWFenceObj;
+	IMG_UINT32 i;
+
+	PVR_DUMPDEBUG_LOG("Fence: %s, %-9s - <%#"IMG_UINT64_FMTSPECx">",
+	                  psFence->pszName,
+	                  _SyncFbFenceSyncsHaveSignalled(psFence) ?
+	                      "Signalled" : "Active",
+	                  psFence->iUID);
+
+	/* ... all sync points in the fence */
+	for (i = 0; i < psFence->uiNumSyncs; i++)
+	{
+		_SyncFbDebugRequestPrintSyncPt(psFence->apsFenceSyncList[i],
+		                               IMG_TRUE,
+		                               pfnDumpDebugPrintf,
+		                               pvDumpDebugFile);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SyncFbSWDumpTimelineKM(void *pvSWTimelineObj,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile)
+{
+	PVRSRV_TIMELINE_SERVER *psTl = (PVRSRV_TIMELINE_SERVER *) pvSWTimelineObj;
+
+	OSLockAcquire(psTl->hTlLock);
+
+	PVR_DUMPDEBUG_LOG("TL: %s, SeqNum: %d/%d - <%#"IMG_UINT64_FMTSPECx">",
+	                  psTl->pszName,
+	                  OSAtomicRead(&psTl->iLastSignalledSeqNum),
+	                  OSAtomicRead(&psTl->iSeqNum),
+	                  psTl->iUID);
+
+	OSLockRelease(psTl->hTlLock);
+
+	return PVRSRV_OK;
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                       IMPORT/EXPORT FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+static PVRSRV_ERROR _SyncFbFenceExport(PVRSRV_FENCE_SERVER *psFence,
+                                       PVRSRV_FENCE_EXPORT **ppsExport)
+{
+	PVRSRV_FENCE_EXPORT *psExport;
+	PVRSRV_ERROR eError;
+
+	psExport = OSAllocMem(sizeof(*psExport));
+	PVR_LOG_GOTO_IF_NOMEM(psExport, eError, err_out);
+
+	_SyncFbFenceAcquire(psFence);
+
+	psExport->psFence = psFence;
+	*ppsExport = psExport;
+
+	eError = PVRSRV_OK;
+err_out:
+	return eError;
+}
+
+static PVRSRV_ERROR _SyncFbFenceExportDestroy(PVRSRV_FENCE_EXPORT *psExport)
+{
+	PVRSRV_ERROR eError;
+
+	eError = SyncFbFenceRelease(psExport->psFence);
+	PVR_LOG_IF_ERROR(eError, "SyncFbFenceRelease");
+
+	OSFreeMem(psExport);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _SyncFbFenceImport(PVRSRV_FENCE_EXPORT *psImport,
+                                       PVRSRV_FENCE_SERVER **psFenceOut)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_SERVER *psFence;
+
+	psFence = psImport->psFence;
+	_SyncFbFenceAcquire(psFence);
+
+	*psFenceOut = psFence;
+
+	eError = PVRSRV_OK;
+	return eError;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR SyncFbFenceExportInsecure(PVRSRV_FENCE_SERVER *psFence,
+                                       PVRSRV_FENCE_EXPORT **ppsExport)
+{
+	return _SyncFbFenceExport(psFence, ppsExport);
+}
+
+PVRSRV_ERROR SyncFbFenceExportDestroyInsecure(PVRSRV_FENCE_EXPORT *psExport)
+{
+	return _SyncFbFenceExportDestroy(psExport);
+}
+
+PVRSRV_ERROR SyncFbFenceImportInsecure(CONNECTION_DATA *psConnection,
+                                       PVRSRV_DEVICE_NODE *psDevice,
+                                       PVRSRV_FENCE_EXPORT *psImport,
+                                       PVRSRV_FENCE_SERVER **psFenceOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	return _SyncFbFenceImport(psImport, psFenceOut);
+}
+#endif /* defined(SUPPORT_INSECURE_EXPORT) */
+
+PVRSRV_ERROR SyncFbFenceExportDestroySecure(PVRSRV_FENCE_EXPORT *psExport)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED1(psExport);
+
+	eError = _SyncFbFenceExportDestroy(psExport);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _SyncFbReleaseSecureExport(void *pvExport)
+{
+	return SyncFbFenceExportDestroySecure(pvExport);
+}
+
+PVRSRV_ERROR SyncFbFenceExportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE * psDevNode,
+                                     PVRSRV_FENCE_SERVER *psFence,
+                                     IMG_SECURE_TYPE *phSecure,
+                                     PVRSRV_FENCE_EXPORT **ppsExport,
+                                     CONNECTION_DATA **ppsSecureConnection)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_EXPORT *psExport;
+
+	PVR_DPF_ENTERED1(psFence);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ppsSecureConnection);
+
+	eError = _SyncFbFenceExport(psFence, &psExport);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_SyncFbFenceExport", err_out);
+
+	/* Transform it into a secure export */
+	eError = OSSecureExport("fallback_fence",
+	                        _SyncFbReleaseSecureExport,
+	                        (void *) psExport,
+	                        phSecure);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSecureExport", err_export);
+
+	*ppsExport = psExport;
+	PVR_DPF_RETURN_OK;
+err_export:
+	_SyncFbFenceExportDestroy(psExport);
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR SyncFbFenceImportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDevice,
+                                     IMG_SECURE_TYPE hSecure,
+                                     PVRSRV_FENCE_SERVER **ppsFence)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE_EXPORT *psImport;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	PVR_DPF_ENTERED1(hSecure);
+
+	eError = OSSecureImport(hSecure, (void **) &psImport);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSecureImport", err_out);
+
+	eError = _SyncFbFenceImport(psImport, ppsFence);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                            TESTING FUNCTIONS                              */
+/*                                                                           */
+/*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_server.c
new file mode 100644
index 000000000000..1746d946aca1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/sync_server.c
@@ -0,0 +1,1230 @@
+/*************************************************************************/ /*!
+@File           sync_server.c
+@Title          Server side synchronisation functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side functions that for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "sync_server.h"
+#include "allocmem.h"
+#include "device.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pdump.h"
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "pdump_km.h"
+#include "sync.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+#include "htbserver.h"
+#include "rgxhwperf.h"
+#include "info_page.h"
+
+#include "sync_checkpoint_internal.h"
+#include "sync_checkpoint.h"
+
+/* Include this to obtain MAX_SYNC_CHECKPOINTS_PER_FENCE */
+#include "sync_checkpoint_external.h"
+
+/* Include this to obtain PVRSRV_MAX_DEV_VARS */
+#include "pvrsrv_devvar.h"
+
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "ossecure_export.h"
+#endif
+
+/* Set this to enable debug relating to the construction and maintenance of the sync address list */
+#define SYNC_ADDR_LIST_DEBUG 0
+
+/* Set maximum number of FWAddrs that can be accommodated in a SYNC_ADDR_LIST.
+ * This should allow for PVRSRV_MAX_DEV_VARS dev vars plus
+ * MAX_SYNC_CHECKPOINTS_PER_FENCE sync checkpoints for check fences.
+ * The same SYNC_ADDR_LIST is also used to hold UFOs for updates. While this
+ * may need to accommodate the additional sync prim update returned by Native
+ * sync implementation (used for timeline debug), the size calculated from
+ * PVRSRV_MAX_DEV_VARS+MAX_SYNC_CHECKPOINTS_PER_FENCE should be ample.
+ */
+#define PVRSRV_MAX_SYNC_ADDR_LIST_SIZE (PVRSRV_MAX_DEV_VARS+MAX_SYNC_CHECKPOINTS_PER_FENCE)
+/* Check that helper functions will not be preparing longer lists of
+ * UFOs than the FW can handle.
+ */
+static_assert(PVRSRV_MAX_SYNC_ADDR_LIST_SIZE <= RGXFWIF_CCB_CMD_MAX_UFOS,
+              "PVRSRV_MAX_SYNC_ADDR_LIST_SIZE > RGXFWIF_CCB_CMD_MAX_UFOS.");
+
+/* Max number of syncs allowed in a sync prim op */
+#define SYNC_PRIM_OP_MAX_SYNCS 1024
+
+struct _SYNC_PRIMITIVE_BLOCK_
+{
+	PVRSRV_DEVICE_NODE	*psDevNode;
+	DEVMEM_MEMDESC		*psMemDesc;
+	IMG_UINT32			*pui32LinAddr;
+	IMG_UINT32			ui32BlockSize;		/*!< Size of the Sync Primitive Block */
+	ATOMIC_T			sRefCount;
+	DLLIST_NODE			sConnectionNode;
+	SYNC_CONNECTION_DATA *psSyncConnectionData;	/*!< Link back to the sync connection data if there is one */
+	PRGXFWIF_UFO_ADDR		uiFWAddr;	/*!< The firmware address of the sync prim block */
+};
+
+struct _SYNC_CONNECTION_DATA_
+{
+	DLLIST_NODE	sListHead;  /*!< list of sync block associated with / created against this connection */
+	ATOMIC_T	sRefCount;  /*!< number of references to this object */
+	POS_LOCK	hLock;      /*!< lock protecting the list of sync blocks */
+};
+
+#define DECREMENT_WITH_WRAP(value, sz) ((value) ? ((value) - 1) : ((sz) - 1))
+
+/* this is the max number of syncs we will search or dump
+ * at any time.
+ */
+#define SYNC_RECORD_LIMIT 20000
+
+enum SYNC_RECORD_TYPE
+{
+	SYNC_RECORD_TYPE_UNKNOWN = 0,
+	SYNC_RECORD_TYPE_CLIENT,
+	SYNC_RECORD_TYPE_SERVER,
+};
+
+struct SYNC_RECORD
+{
+	PVRSRV_DEVICE_NODE		*psDevNode;
+	SYNC_PRIMITIVE_BLOCK	*psServerSyncPrimBlock;	/*!< handle to _SYNC_PRIMITIVE_BLOCK_ */
+	IMG_UINT32				ui32SyncOffset;			/*!< offset to sync in block */
+	IMG_UINT32				ui32FwBlockAddr;
+	IMG_PID					uiPID;
+	IMG_UINT64				ui64OSTime;
+	enum SYNC_RECORD_TYPE	eRecordType;
+	DLLIST_NODE				sNode;
+	IMG_CHAR				szClassName[PVRSRV_SYNC_NAME_LENGTH];
+};
+
+#if defined(SYNC_DEBUG) || defined(REFCOUNT_DEBUG)
+#define SYNC_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define SYNC_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+#if defined(SYNC_DEBUG)
+#define SYNC_UPDATES_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define SYNC_UPDATES_PRINT(fmt, ...)
+#endif
+
+/*!
+*****************************************************************************
+ @Function      : SyncPrimitiveBlockToFWAddr
+
+ @Description   : Given a pointer to a sync primitive block and an offset,
+                  returns the firmware address of the sync.
+
+ @Input           psSyncPrimBlock : Sync primitive block which contains the sync
+ @Input           ui32Offset      : Offset of sync within the sync primitive block
+ @Output          psAddrOut       : Absolute FW address of the sync is written out through
+                                    this pointer
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncPrimitiveBlockToFWAddr(SYNC_PRIMITIVE_BLOCK *psSyncPrimBlock,
+							IMG_UINT32 ui32Offset,
+						PRGXFWIF_UFO_ADDR *psAddrOut)
+{
+	/* check offset is legal */
+	if (unlikely((ui32Offset >= psSyncPrimBlock->ui32BlockSize) ||
+		(ui32Offset % sizeof(IMG_UINT32))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncPrimitiveBlockToFWAddr: parameters check failed"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAddrOut->ui32Addr = psSyncPrimBlock->uiFWAddr.ui32Addr + ui32Offset;
+	return PVRSRV_OK;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListGrow
+
+ @Description   : Grow the SYNC_ADDR_LIST so it can accommodate the given
+                  number of syncs, up to a maximum of PVRSRV_MAX_SYNC_PRIMS.
+
+ @Input           psList       : The SYNC_ADDR_LIST to grow
+ @Input           ui32NumSyncs : The number of sync addresses to be able to hold
+ @Return :        PVRSRV_OK on success
+*****************************************************************************/
+
+static PVRSRV_ERROR SyncAddrListGrow(SYNC_ADDR_LIST *psList, IMG_UINT32 ui32NumSyncs)
+{
+	if (unlikely(ui32NumSyncs > PVRSRV_MAX_SYNC_ADDR_LIST_SIZE))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui32NumSyncs=%u > PVRSRV_MAX_SYNC_ADDR_LIST_SIZE=%u", __func__, ui32NumSyncs, PVRSRV_MAX_SYNC_ADDR_LIST_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s:     Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	if (ui32NumSyncs > psList->ui32NumSyncs)
+	{
+		if (psList->pasFWAddrs == NULL)
+		{
+			psList->pasFWAddrs = OSAllocMem(sizeof(PRGXFWIF_UFO_ADDR) * PVRSRV_MAX_SYNC_ADDR_LIST_SIZE);
+			PVR_RETURN_IF_NOMEM(psList->pasFWAddrs);
+		}
+
+		psList->ui32NumSyncs = ui32NumSyncs;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s:     Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	return PVRSRV_OK;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListInit
+
+ @Description   : Initialise a SYNC_ADDR_LIST structure ready for use
+
+ @Input           psList        : The SYNC_ADDR_LIST structure to initialise
+ @Return        : None
+*****************************************************************************/
+
+void
+SyncAddrListInit(SYNC_ADDR_LIST *psList)
+{
+	psList->ui32NumSyncs = 0;
+	psList->pasFWAddrs   = NULL;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListDeinit
+
+ @Description   : Frees any resources associated with the given SYNC_ADDR_LIST
+
+ @Input           psList        : The SYNC_ADDR_LIST structure to deinitialise
+ @Return        : None
+*****************************************************************************/
+
+void
+SyncAddrListDeinit(SYNC_ADDR_LIST *psList)
+{
+	if (psList->pasFWAddrs != NULL)
+	{
+		OSFreeMem(psList->pasFWAddrs);
+	}
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListPopulate
+
+ @Description   : Populate the given SYNC_ADDR_LIST with the FW addresses
+                  of the syncs given by the SYNC_PRIMITIVE_BLOCKs and sync offsets
+
+ @Input           ui32NumSyncs    : The number of syncs being passed in
+ @Input           apsSyncPrimBlock: Array of pointers to SYNC_PRIMITIVE_BLOCK structures
+                                    in which the syncs are based
+ @Input           paui32SyncOffset: Array of offsets within each of the sync primitive blocks
+                                    where the syncs are located
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncAddrListPopulate(SYNC_ADDR_LIST *psList,
+						IMG_UINT32 ui32NumSyncs,
+						SYNC_PRIMITIVE_BLOCK **apsSyncPrimBlock,
+						IMG_UINT32 *paui32SyncOffset)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	if (ui32NumSyncs > psList->ui32NumSyncs)
+	{
+		eError = SyncAddrListGrow(psList, ui32NumSyncs);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	psList->ui32NumSyncs = ui32NumSyncs;
+
+	for (i = 0; i < ui32NumSyncs; i++)
+	{
+		eError = SyncPrimitiveBlockToFWAddr(apsSyncPrimBlock[i],
+								paui32SyncOffset[i],
+								&psList->pasFWAddrs[i]);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+SyncAddrListAppendSyncPrim(SYNC_ADDR_LIST          *psList,
+						   PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32FwAddr = 0;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs));
+#endif
+	/* Ensure there's room in psList for the additional sync prim update */
+	eError = SyncAddrListGrow(psList, psList->ui32NumSyncs + 1);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	SyncPrimGetFirmwareAddr(psSyncPrim, &ui32FwAddr);
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Appending sync prim <%p> UFO addr (0x%x) to psList[->pasFWAddrss[%d]", __func__, (void*)psSyncPrim, ui32FwAddr, psList->ui32NumSyncs-1));
+#endif
+	psList->pasFWAddrs[psList->ui32NumSyncs-1].ui32Addr = ui32FwAddr;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	{
+		IMG_UINT32 iii;
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: psList->ui32NumSyncs=%d", __func__, psList->ui32NumSyncs));
+		for (iii=0; iii<psList->ui32NumSyncs; iii++)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: psList->pasFWAddrs[%d].ui32Addr=0x%x", __func__, iii, psList->pasFWAddrs[iii].ui32Addr));
+		}
+	}
+#endif
+e0:
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d", __func__, (void*)psList, psList->ui32NumSyncs));
+#endif
+	return eError;
+}
+
+
+static PVRSRV_ERROR
+_AppendCheckpoints(SYNC_ADDR_LIST *psList,
+				   IMG_UINT32 ui32NumCheckpoints,
+				   PSYNC_CHECKPOINT *apsSyncCheckpoint,
+				   IMG_BOOL bDeRefCheckpoints)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32SyncCheckpointIndex;
+	IMG_UINT32 ui32RollbackSize = psList->ui32NumSyncs;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+#endif
+	/* Ensure there's room in psList for the sync checkpoints */
+	eError = SyncAddrListGrow(psList, psList->ui32NumSyncs + ui32NumCheckpoints);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: * * * * ERROR * * * * Trying to SyncAddrListGrow(psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+		goto e0;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: (ui32NumCheckpoints=%d) (psList->ui32NumSyncs is now %d) array already contains %d FWAddrs:", __func__, ui32NumCheckpoints, psList->ui32NumSyncs, ui32RollbackSize));
+	if (ui32RollbackSize > 0)
+	{
+		{
+			IMG_UINT32 kk;
+			for (kk=0; kk<ui32RollbackSize; kk++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:    <%p>psList->pasFWAddrs[%d].ui32Addr = %u(0x%x)", __func__,
+						 (void*)&psList->pasFWAddrs[kk], kk,
+						 psList->pasFWAddrs[kk].ui32Addr, psList->pasFWAddrs[kk].ui32Addr));
+			}
+		}
+	}
+	PVR_DPF((PVR_DBG_ERROR, "%s: apsSyncCheckpoint=<%p>, apsSyncCheckpoint[0] = <%p>", __func__, (void*)apsSyncCheckpoint, (void*)apsSyncCheckpoint[0]));
+#endif
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		psList->pasFWAddrs[ui32RollbackSize + ui32SyncCheckpointIndex].ui32Addr = SyncCheckpointGetFirmwareAddr(apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s:  SyncCheckpointCCBEnqueued(<%p>)", __func__, (void*)apsSyncCheckpoint[ui32SyncCheckpointIndex]));
+		PVR_DPF((PVR_DBG_ERROR, "%s:                           ID:%d", __func__, SyncCheckpointGetId((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex])));
+#endif
+		SyncCheckpointCCBEnqueued((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		if (bDeRefCheckpoints)
+		{
+			/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+			SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		}
+	}
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	if (psList->ui32NumSyncs > 0)
+	{
+		IMG_UINT32 kk;
+		for (kk=0; kk<psList->ui32NumSyncs; kk++)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s:    <%p>psList->pasFWAddrs[%d].ui32Addr = %u(0x%x)", __func__,
+			         (void*)&psList->pasFWAddrs[kk], kk,
+			         psList->pasFWAddrs[kk].ui32Addr, psList->pasFWAddrs[kk].ui32Addr));
+		}
+	}
+#endif
+	return eError;
+
+e0:
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		if (bDeRefCheckpoints)
+		{
+			/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+			SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		}
+	}
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+#endif
+	return eError;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListAppendCheckpoints
+
+ @Description   : Append the FW addresses of the sync checkpoints given in
+                  the PSYNC_CHECKPOINTs array to the given SYNC_ADDR_LIST
+
+ @Input           ui32NumSyncCheckpoints : The number of sync checkpoints
+                                           being passed in
+ @Input           apsSyncCheckpoint : Array of PSYNC_CHECKPOINTs whose details
+                                      are to be appended to the SYNC_ADDR_LIST
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+PVRSRV_ERROR
+SyncAddrListAppendCheckpoints(SYNC_ADDR_LIST *psList,
+								IMG_UINT32 ui32NumCheckpoints,
+								PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	return _AppendCheckpoints(psList, ui32NumCheckpoints, apsSyncCheckpoint, IMG_FALSE);
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListAppendAndDeRefCheckpoints
+
+ @Description   : Append the FW addresses of the sync checkpoints given in
+                  the PSYNC_CHECKPOINTs array to the given SYNC_ADDR_LIST.
+                  A reference is dropped for each of the checkpoints.
+
+ @Input           ui32NumSyncCheckpoints : The number of sync checkpoints
+                                           being passed in
+ @Input           apsSyncCheckpoint : Array of PSYNC_CHECKPOINTs whose details
+                                      are to be appended to the SYNC_ADDR_LIST
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+PVRSRV_ERROR
+SyncAddrListAppendAndDeRefCheckpoints(SYNC_ADDR_LIST *psList,
+									  IMG_UINT32 ui32NumCheckpoints,
+									  PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	return _AppendCheckpoints(psList, ui32NumCheckpoints, apsSyncCheckpoint, IMG_TRUE);
+}
+
+void
+SyncAddrListDeRefCheckpoints(IMG_UINT32 ui32NumCheckpoints,
+							 PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	IMG_UINT32 ui32SyncCheckpointIndex;
+
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+		SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+	}
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListRollbackCheckpoints
+
+ @Description   : Rollback the enqueued count of each sync checkpoint in
+                  the given SYNC_ADDR_LIST. This needs to be done in the
+                  event of the kick call failing, so that the reference
+                  taken on each sync checkpoint on the firmware's behalf
+                  is dropped.
+
+ @Input           psList        : The SYNC_ADDR_LIST structure containing
+                                  sync checkpoints to be rolled back
+
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncAddrListRollbackCheckpoints(PVRSRV_DEVICE_NODE *psDevNode, SYNC_ADDR_LIST *psList)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32SyncIndex;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: called (psList=<%p>)", __func__, (void*)psList));
+#endif
+	if (psList)
+	{
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: psList->ui32NumSyncs=%d", __func__, psList->ui32NumSyncs));
+#endif
+		for (ui32SyncIndex=0; ui32SyncIndex<psList->ui32NumSyncs; ui32SyncIndex++)
+		{
+			if (psList->pasFWAddrs[ui32SyncIndex].ui32Addr & 0x1)
+			{
+				SyncCheckpointRollbackFromUFO(psDevNode, psList->pasFWAddrs[ui32SyncIndex].ui32Addr);
+			}
+		}
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncRecordAddKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDevNode,
+					  SYNC_RECORD_HANDLE *phRecord,
+					  SYNC_PRIMITIVE_BLOCK *hServerSyncPrimBlock,
+					  IMG_UINT32 ui32FwBlockAddr,
+					  IMG_UINT32 ui32SyncOffset,
+					  IMG_BOOL bServerSync,
+					  IMG_UINT32 ui32ClassNameSize,
+					  const IMG_CHAR *pszClassName)
+{
+	struct SYNC_RECORD * psSyncRec;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	RGXSRV_HWPERF_ALLOC(psDevNode, SYNC,
+	                    ui32FwBlockAddr + ui32SyncOffset,
+	                    pszClassName,
+	                    ui32ClassNameSize);
+
+	PVR_RETURN_IF_INVALID_PARAM(phRecord);
+
+	*phRecord = NULL;
+
+	psSyncRec = OSAllocMem(sizeof(*psSyncRec));
+	PVR_GOTO_IF_NOMEM(psSyncRec, eError, fail_alloc);
+
+	psSyncRec->psDevNode = psDevNode;
+	psSyncRec->psServerSyncPrimBlock = hServerSyncPrimBlock;
+	psSyncRec->ui32SyncOffset = ui32SyncOffset;
+	psSyncRec->ui32FwBlockAddr = ui32FwBlockAddr;
+	psSyncRec->ui64OSTime = OSClockns64();
+	psSyncRec->uiPID = OSGetCurrentProcessID();
+	psSyncRec->eRecordType = bServerSync? SYNC_RECORD_TYPE_SERVER: SYNC_RECORD_TYPE_CLIENT;
+
+	if (pszClassName)
+	{
+		if (ui32ClassNameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			ui32ClassNameSize = PVRSRV_SYNC_NAME_LENGTH;
+		/* Copy over the class name annotation */
+		OSStringSafeCopy(psSyncRec->szClassName, pszClassName, ui32ClassNameSize);
+	}
+	else
+	{
+		/* No class name annotation */
+		psSyncRec->szClassName[0] = 0;
+	}
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	if (psDevNode->ui32SyncServerRecordCount < SYNC_RECORD_LIMIT)
+	{
+		dllist_add_to_head(&psDevNode->sSyncServerRecordList, &psSyncRec->sNode);
+		psDevNode->ui32SyncServerRecordCount++;
+
+		if (psDevNode->ui32SyncServerRecordCount > psDevNode->ui32SyncServerRecordCountHighWatermark)
+		{
+			psDevNode->ui32SyncServerRecordCountHighWatermark = psDevNode->ui32SyncServerRecordCount;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync record \"%s\". %u records already exist.",
+											__func__,
+											pszClassName,
+											psDevNode->ui32SyncServerRecordCount));
+		OSFreeMem(psSyncRec);
+		psSyncRec = NULL;
+		eError = PVRSRV_ERROR_TOOMANYBUFFERS;
+	}
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	*phRecord = (SYNC_RECORD_HANDLE)psSyncRec;
+
+fail_alloc:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncRecordRemoveByHandleKM(
+			SYNC_RECORD_HANDLE hRecord)
+{
+	struct SYNC_RECORD **ppFreedSync;
+	struct SYNC_RECORD *pSync = (struct SYNC_RECORD*)hRecord;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_RETURN_IF_INVALID_PARAM(hRecord);
+
+	psDevNode = pSync->psDevNode;
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+
+	RGXSRV_HWPERF_FREE(psDevNode, SYNC, pSync->ui32FwBlockAddr + pSync->ui32SyncOffset);
+
+	dllist_remove_node(&pSync->sNode);
+
+	if (psDevNode->uiSyncServerRecordFreeIdx >= PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: freed sync record index out of range",
+				 __func__));
+		psDevNode->uiSyncServerRecordFreeIdx = 0;
+	}
+	ppFreedSync = &psDevNode->apsSyncServerRecordsFreed[psDevNode->uiSyncServerRecordFreeIdx];
+	psDevNode->uiSyncServerRecordFreeIdx =
+		(psDevNode->uiSyncServerRecordFreeIdx + 1) % PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN;
+
+	if (*ppFreedSync)
+	{
+		OSFreeMem(*ppFreedSync);
+	}
+	pSync->psServerSyncPrimBlock = NULL;
+	pSync->ui64OSTime = OSClockns64();
+	*ppFreedSync = pSync;
+
+	psDevNode->ui32SyncServerRecordCount--;
+
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncAllocEventKM(CONNECTION_DATA *psConnection,
+			PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_BOOL bServerSync,
+			IMG_UINT32 ui32FWAddr,
+			IMG_UINT32 ui32ClassNameSize,
+			const IMG_CHAR *pszClassName)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	RGXSRV_HWPERF_ALLOC(psDevNode, SYNC, ui32FWAddr, pszClassName, ui32ClassNameSize);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncFreeEventKM(CONNECTION_DATA *psConnection,
+			PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 ui32FWAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	RGXSRV_HWPERF_FREE(psDevNode, SYNC, ui32FWAddr);
+
+	return PVRSRV_OK;
+}
+
+static
+void _SyncConnectionRef(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psSyncConnectionData->sRefCount);
+
+	SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+						__func__, psSyncConnectionData, iRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+static
+void _SyncConnectionUnref(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psSyncConnectionData->sRefCount);
+	if (iRefCount == 0)
+	{
+		SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+		                    __func__, psSyncConnectionData, iRefCount);
+
+		PVR_ASSERT(dllist_is_empty(&psSyncConnectionData->sListHead));
+		OSLockDestroy(psSyncConnectionData->hLock);
+		OSFreeMem(psSyncConnectionData);
+	}
+	else
+	{
+		SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+		                    __func__, psSyncConnectionData, iRefCount);
+		PVR_ASSERT(iRefCount > 0);
+	}
+}
+
+static
+void _SyncConnectionAddBlock(CONNECTION_DATA *psConnection, SYNC_PRIMITIVE_BLOCK *psBlock)
+{
+	if (psConnection)
+	{
+		SYNC_CONNECTION_DATA *psSyncConnectionData = psConnection->psSyncConnectionData;
+
+		/*
+			Make sure the connection doesn't go away. It doesn't matter that we will release
+			the lock between as the refcount and list don't have to be atomic w.r.t. to each other
+		*/
+		_SyncConnectionRef(psSyncConnectionData);
+
+		OSLockAcquire(psSyncConnectionData->hLock);
+		dllist_add_to_head(&psSyncConnectionData->sListHead, &psBlock->sConnectionNode);
+		OSLockRelease(psSyncConnectionData->hLock);
+		psBlock->psSyncConnectionData = psSyncConnectionData;
+	}
+	else
+	{
+		psBlock->psSyncConnectionData = NULL;
+	}
+}
+
+static
+void _SyncConnectionRemoveBlock(SYNC_PRIMITIVE_BLOCK *psBlock)
+{
+	SYNC_CONNECTION_DATA *psSyncConnectionData = psBlock->psSyncConnectionData;
+
+	if (psBlock->psSyncConnectionData)
+	{
+		OSLockAcquire(psSyncConnectionData->hLock);
+		dllist_remove_node(&psBlock->sConnectionNode);
+		OSLockRelease(psSyncConnectionData->hLock);
+
+		_SyncConnectionUnref(psBlock->psSyncConnectionData);
+	}
+}
+
+static inline
+void _DoPrimBlockFree(SYNC_PRIMITIVE_BLOCK *psSyncBlk)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psSyncBlk->psDevNode;
+
+	SYNC_REFCOUNT_PRINT("%s: Sync block %p, refcount = %d (remove)",
+	                    __func__, psSyncBlk, OSAtomicRead(&psSyncBlk->sRefCount));
+
+	PVR_ASSERT(OSAtomicRead(&psSyncBlk->sRefCount) == 1);
+
+	_SyncConnectionRemoveBlock(psSyncBlk);
+	DevmemReleaseCpuVirtAddr(psSyncBlk->psMemDesc);
+	psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->psMemDesc);
+	OSFreeMem(psSyncBlk);
+}
+
+PVRSRV_ERROR
+PVRSRVAllocSyncPrimitiveBlockKM(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+								SYNC_PRIMITIVE_BLOCK **ppsSyncBlk,
+								IMG_UINT32 *puiSyncPrimVAddr,
+								IMG_UINT32 *puiSyncPrimBlockSize,
+								PMR        **ppsSyncPMR)
+{
+	SYNC_PRIMITIVE_BLOCK *psNewSyncBlk;
+	PVRSRV_ERROR eError;
+
+	psNewSyncBlk = OSAllocMem(sizeof(SYNC_PRIMITIVE_BLOCK));
+	PVR_GOTO_IF_NOMEM(psNewSyncBlk, eError, e0);
+
+	psNewSyncBlk->psDevNode = psDevNode;
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS, "Allocate UFO block");
+
+	eError = psDevNode->pfnAllocUFOBlock(psDevNode,
+										 sizeof(IMG_UINT32),
+										 &psNewSyncBlk->psMemDesc,
+										 &psNewSyncBlk->uiFWAddr.ui32Addr,
+										 &psNewSyncBlk->ui32BlockSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = psNewSyncBlk->uiFWAddr.ui32Addr;
+
+	eError = DevmemAcquireCpuVirtAddr(psNewSyncBlk->psMemDesc,
+									  (void **) &psNewSyncBlk->pui32LinAddr);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	eError = DevmemLocalGetImportHandle(psNewSyncBlk->psMemDesc, (void **) ppsSyncPMR);
+
+	PVR_GOTO_IF_ERROR(eError, e3);
+
+	OSAtomicWrite(&psNewSyncBlk->sRefCount, 1);
+
+	/* If there is a connection pointer then add the new block onto it's list */
+	_SyncConnectionAddBlock(psConnection, psNewSyncBlk);
+
+	*ppsSyncBlk = psNewSyncBlk;
+	*puiSyncPrimBlockSize = psNewSyncBlk->ui32BlockSize;
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+						  "Allocated UFO block (FirmwareVAddr = 0x%08x)",
+						  *puiSyncPrimVAddr);
+
+	return PVRSRV_OK;
+
+e3:
+	DevmemReleaseCpuVirtAddr(psNewSyncBlk->psMemDesc);
+e2:
+	psDevNode->pfnFreeUFOBlock(psDevNode, psNewSyncBlk->psMemDesc);
+e1:
+	OSFreeMem(psNewSyncBlk);
+e0:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVFreeSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk)
+{
+
+	/* This function is an alternative to the above without reference counting.
+	 * With the removal of sync prim ops for server syncs we no longer have to
+	 * reference count prim blocks as the reference will never be incremented /
+	 * decremented by a prim op */
+	_DoPrimBlockFree(psSyncBlk);
+	return PVRSRV_OK;
+}
+
+static INLINE IMG_BOOL _CheckSyncIndex(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+                                       IMG_UINT32 ui32Index)
+{
+	if (psSyncBlk->ui32BlockSize == 0)
+	{
+		return IMG_FALSE;
+	}
+
+	return (ui32Index < psSyncBlk->ui32BlockSize / sizeof(IMG_UINT32));
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimSetKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+                    IMG_UINT32 ui32Index,
+                    IMG_UINT32 ui32Value)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncBlk != NULL, "psSyncBlk");
+
+	if (_CheckSyncIndex(psSyncBlk, ui32Index))
+	{
+		psSyncBlk->pui32LinAddr[ui32Index] = ui32Value;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVSyncPrimSetKM: Index %u out of range for "
+		         "0x%08X byte sync block (value 0x%08X)",
+		         ui32Index,
+		         psSyncBlk->ui32BlockSize,
+		         ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	*/
+	DevmemPDumpLoadMemValue32(psSyncBlk->psMemDesc,
+					   ui32Offset,
+					   ui32Value,
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset)
+{
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	*/
+	DevmemPDumpLoadMem(psSyncBlk->psMemDesc,
+					   ui32Offset,
+					   sizeof(IMG_UINT32),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	DevmemPDumpDevmemPol32(psSyncBlk->psMemDesc,
+						   ui32Offset,
+						   ui32Value,
+						   ui32Mask,
+						   eOperator,
+						   ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize)
+{
+	DevmemPDumpCBP(psSyncBlk->psMemDesc,
+				   ui32Offset,
+				   uiWriteOffset,
+				   uiPacketSize,
+				   uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif
+
+/* SyncRegisterConnection */
+PVRSRV_ERROR SyncRegisterConnection(SYNC_CONNECTION_DATA **ppsSyncConnectionData)
+{
+	SYNC_CONNECTION_DATA *psSyncConnectionData;
+	PVRSRV_ERROR eError;
+
+	psSyncConnectionData = OSAllocMem(sizeof(SYNC_CONNECTION_DATA));
+	if (psSyncConnectionData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+
+	eError = OSLockCreate(&psSyncConnectionData->hLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lockcreate);
+	dllist_init(&psSyncConnectionData->sListHead);
+	OSAtomicWrite(&psSyncConnectionData->sRefCount, 1);
+
+	*ppsSyncConnectionData = psSyncConnectionData;
+	return PVRSRV_OK;
+
+fail_lockcreate:
+	OSFreeMem(psSyncConnectionData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* SyncUnregisterConnection */
+void SyncUnregisterConnection(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	_SyncConnectionUnref(psSyncConnectionData);
+}
+
+void SyncConnectionPDumpSyncBlocks(PVRSRV_DEVICE_NODE *psDevNode, void *hSyncPrivData, PDUMP_TRANSITION_EVENT eEvent)
+{
+	if ((eEvent == PDUMP_TRANSITION_EVENT_RANGE_ENTERED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		SYNC_CONNECTION_DATA *psSyncConnectionData = hSyncPrivData;
+		DLLIST_NODE *psNode, *psNext;
+
+		OSLockAcquire(psSyncConnectionData->hLock);
+
+		PDUMPCOMMENT(psDevNode, "Dump client Sync Prim state");
+		dllist_foreach_node(&psSyncConnectionData->sListHead, psNode, psNext)
+		{
+			SYNC_PRIMITIVE_BLOCK *psSyncBlock =
+				IMG_CONTAINER_OF(psNode, SYNC_PRIMITIVE_BLOCK, sConnectionNode);
+
+			DevmemPDumpLoadMem(psSyncBlock->psMemDesc,
+					0,
+					psSyncBlock->ui32BlockSize,
+					PDUMP_FLAGS_CONTINUOUS);
+		}
+
+		OSLockRelease(psSyncConnectionData->hLock);
+	}
+}
+
+void SyncRecordLookup(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 ui32FwAddr,
+					  IMG_CHAR * pszSyncInfo, size_t len)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_INT iEnd;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (!pszSyncInfo)
+	{
+		return;
+	}
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	pszSyncInfo[0] = '\0';
+
+	dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+	{
+		struct SYNC_RECORD *psSyncRec =
+			IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+		if ((psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset) == ui32FwAddr
+			&& SYNC_RECORD_TYPE_UNKNOWN != psSyncRec->eRecordType
+			&& psSyncRec->psServerSyncPrimBlock
+			&& psSyncRec->psServerSyncPrimBlock->pui32LinAddr
+			)
+		{
+			IMG_UINT32 *pui32SyncAddr;
+			pui32SyncAddr = psSyncRec->psServerSyncPrimBlock->pui32LinAddr
+				+ (psSyncRec->ui32SyncOffset/sizeof(IMG_UINT32));
+			iEnd = OSSNPrintf(pszSyncInfo, len, "Cur=0x%08x %s:%05u (%s)",
+				*pui32SyncAddr,
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				psSyncRec->szClassName
+				);
+			if (iEnd >= 0 && iEnd < len)
+			{
+				pszSyncInfo[iEnd] = '\0';
+			}
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	if (!bFound && (psDevNode->ui32SyncServerRecordCountHighWatermark == SYNC_RECORD_LIMIT))
+	{
+		OSSNPrintf(pszSyncInfo, len, "(Record may be lost)");
+	}
+}
+
+#define NS_IN_S (1000000000UL)
+static void _SyncRecordPrint(struct SYNC_RECORD *psSyncRec,
+					IMG_UINT64 ui64TimeNow,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	SYNC_PRIMITIVE_BLOCK *psSyncBlock = psSyncRec->psServerSyncPrimBlock;
+
+	if (SYNC_RECORD_TYPE_UNKNOWN != psSyncRec->eRecordType)
+	{
+		IMG_UINT64 ui64DeltaS;
+		IMG_UINT32 ui32DeltaF;
+		IMG_UINT64 ui64Delta = ui64TimeNow - psSyncRec->ui64OSTime;
+		ui64DeltaS = OSDivide64(ui64Delta, NS_IN_S, &ui32DeltaF);
+
+		if (psSyncBlock && psSyncBlock->pui32LinAddr)
+		{
+			IMG_UINT32 *pui32SyncAddr;
+			pui32SyncAddr = psSyncBlock->pui32LinAddr
+				+ (psSyncRec->ui32SyncOffset/sizeof(IMG_UINT32));
+
+			PVR_DUMPDEBUG_LOG("\t%s %05u %05" IMG_UINT64_FMTSPEC ".%09u FWAddr=0x%08x Val=0x%08x (%s)",
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				ui64DeltaS, ui32DeltaF,
+				(psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset),
+				*pui32SyncAddr,
+				psSyncRec->szClassName
+				);
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("\t%s %05u %05" IMG_UINT64_FMTSPEC ".%09u FWAddr=0x%08x Val=<null_ptr> (%s)",
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				ui64DeltaS, ui32DeltaF,
+				(psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset),
+				psSyncRec->szClassName
+				);
+		}
+	}
+}
+
+static void _SyncRecordRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	IMG_UINT64 ui64TimeNowS;
+	IMG_UINT32 ui32TimeNowF;
+	IMG_UINT64 ui64TimeNow = OSClockns64();
+	DLLIST_NODE *psNode, *psNext;
+
+	ui64TimeNowS = OSDivide64(ui64TimeNow, NS_IN_S, &ui32TimeNowF);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+		OSLockAcquire(psDevNode->hSyncServerRecordLock);
+
+		PVR_DUMPDEBUG_LOG("Dumping all allocated syncs. Allocated: %u High watermark: %u @ %05" IMG_UINT64_FMTSPEC ".%09u",
+										psDevNode->ui32SyncServerRecordCount,
+										psDevNode->ui32SyncServerRecordCountHighWatermark,
+										ui64TimeNowS,
+										ui32TimeNowF);
+		if (psDevNode->ui32SyncServerRecordCountHighWatermark == SYNC_RECORD_LIMIT)
+		{
+			PVR_DUMPDEBUG_LOG("Warning: Record limit (%u) was reached. Some sync checkpoints may not have been recorded in the debug information.",
+                                                                                                                SYNC_RECORD_LIMIT);
+		}
+
+		PVR_DUMPDEBUG_LOG("\t%-6s %-5s %-15s %-17s %-14s (%s)",
+					"Type", "PID", "Time Delta (s)", "Address", "Value", "Annotation");
+
+		dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+		{
+			struct SYNC_RECORD *psSyncRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+			_SyncRecordPrint(psSyncRec, ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+
+		PVR_DUMPDEBUG_LOG("Dumping all recently freed syncs @ %05" IMG_UINT64_FMTSPEC ".%09u",
+						  ui64TimeNowS, ui32TimeNowF);
+		PVR_DUMPDEBUG_LOG("\t%-6s %-5s %-15s %-17s %-14s (%s)",
+					"Type", "PID", "Time Delta (s)", "Address", "Value", "Annotation");
+		for (i = DECREMENT_WITH_WRAP(psDevNode->uiSyncServerRecordFreeIdx, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN);
+			 i != psDevNode->uiSyncServerRecordFreeIdx;
+			 i = DECREMENT_WITH_WRAP(i, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN))
+		{
+			if (psDevNode->apsSyncServerRecordsFreed[i])
+			{
+				_SyncRecordPrint(psDevNode->apsSyncServerRecordsFreed[i],
+								 ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+			else
+			{
+				break;
+			}
+		}
+
+		OSLockRelease(psDevNode->hSyncServerRecordLock);
+	}
+}
+#undef NS_IN_S
+
+static PVRSRV_ERROR SyncRecordListInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	psDevNode->ui32SyncServerRecordCount = 0;
+	psDevNode->ui32SyncServerRecordCountHighWatermark = 0;
+
+	eError = OSLockCreate(&psDevNode->hSyncServerRecordLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lock_create);
+	dllist_init(&psDevNode->sSyncServerRecordList);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncServerRecordNotify,
+		                                          psDevNode,
+		                                          _SyncRecordRequest,
+		                                          DEBUG_REQUEST_SYNCTRACKING,
+		                                          psDevNode);
+
+	PVR_GOTO_IF_ERROR(eError, fail_dbg_register);
+
+	return PVRSRV_OK;
+
+fail_dbg_register:
+	OSLockDestroy(psDevNode->hSyncServerRecordLock);
+fail_lock_create:
+	return eError;
+}
+
+static void SyncRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DLLIST_NODE *psNode, *psNext;
+	int i;
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+	{
+		struct SYNC_RECORD *pSyncRec =
+			IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+
+		dllist_remove_node(psNode);
+		OSFreeMem(pSyncRec);
+	}
+
+	for (i = 0; i < PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN; i++)
+	{
+		if (psDevNode->apsSyncServerRecordsFreed[i])
+		{
+			OSFreeMem(psDevNode->apsSyncServerRecordsFreed[i]);
+			psDevNode->apsSyncServerRecordsFreed[i] = NULL;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	if (psDevNode->hSyncServerRecordNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncServerRecordNotify);
+	}
+	OSLockDestroy(psDevNode->hSyncServerRecordLock);
+}
+
+PVRSRV_ERROR SyncServerInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		eError = SyncRecordListInit(psDevNode);
+		PVR_GOTO_IF_ERROR(eError, fail_record_list);
+	}
+
+	return PVRSRV_OK;
+
+fail_record_list:
+	return eError;
+}
+
+void SyncServerDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		SyncRecordListDeinit(psDevNode);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/tlintern.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlintern.c
new file mode 100644
index 000000000000..3bd6b0af1718
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlintern.c
@@ -0,0 +1,442 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer functions available to driver components in
+                the driver.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "allocmem.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "devicemem.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "tlintern.h"
+
+/*
+ * Make functions
+ */
+PTL_STREAM_DESC
+TLMakeStreamDesc(PTL_SNODE f1, IMG_UINT32 f2, IMG_HANDLE f3)
+{
+	PTL_STREAM_DESC ps = OSAllocZMem(sizeof(TL_STREAM_DESC));
+	if (ps == NULL)
+	{
+		return NULL;
+	}
+	ps->psNode = f1;
+	ps->ui32Flags = f2;
+	ps->hReadEvent = f3;
+	ps->uiRefCount = 1;
+
+	if (f2 & PVRSRV_STREAM_FLAG_READ_LIMIT)
+	{
+		ps->ui32ReadLimit = f1->psStream->ui32Write;
+	}
+	return ps;
+}
+
+PTL_SNODE
+TLMakeSNode(IMG_HANDLE f2, TL_STREAM *f3, TL_STREAM_DESC *f4)
+{
+	PTL_SNODE ps = OSAllocZMem(sizeof(TL_SNODE));
+	if (ps == NULL)
+	{
+		return NULL;
+	}
+	ps->hReadEventObj = f2;
+	ps->psStream = f3;
+	ps->psRDesc = f4;
+	f3->psNode = ps;
+	return ps;
+}
+
+/*
+ * Transport Layer Global top variables and functions
+ */
+static TL_GLOBAL_DATA sTLGlobalData;
+
+TL_GLOBAL_DATA *TLGGD(void) /* TLGetGlobalData() */
+{
+	return &sTLGlobalData;
+}
+
+/* TLInit must only be called once at driver initialisation.
+ * An assert is provided to check this condition on debug builds.
+ */
+PVRSRV_ERROR
+TLInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(sTLGlobalData.hTLGDLock == NULL && sTLGlobalData.hTLEventObj == NULL);
+
+	/* Allocate a lock for TL global data, to be used while updating the TL data.
+	 * This is for making TL global data multi-thread safe */
+	eError = OSLockCreate(&sTLGlobalData.hTLGDLock);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	/* Allocate the event object used to signal global TL events such as
+	 * a new stream created */
+	eError = OSEventObjectCreate("TLGlobalEventObj", &sTLGlobalData.hTLEventObj);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	PVR_DPF_RETURN_OK;
+
+/* Don't allow the driver to start up on error */
+e1:
+	OSLockDestroy (sTLGlobalData.hTLGDLock);
+	sTLGlobalData.hTLGDLock = NULL;
+e0:
+	PVR_DPF_RETURN_RC (eError);
+}
+
+static void RemoveAndFreeStreamNode(PTL_SNODE psRemove)
+{
+	TL_GLOBAL_DATA*  psGD = TLGGD();
+	PTL_SNODE*       last;
+	PTL_SNODE        psn;
+	PVRSRV_ERROR     eError;
+
+	PVR_DPF_ENTERED;
+
+	/* Unlink the stream node from the master list */
+	PVR_ASSERT(psGD->psHead);
+	last = &psGD->psHead;
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn == psRemove)
+		{
+			/* Other calling code may have freed and zeroed the pointers */
+			if (psn->psRDesc)
+			{
+				OSFreeMem(psn->psRDesc);
+				psn->psRDesc = NULL;
+			}
+			if (psn->psStream)
+			{
+				OSFreeMem(psn->psStream);
+				psn->psStream = NULL;
+			}
+			*last = psn->psNext;
+			break;
+		}
+		last = &psn->psNext;
+	}
+
+	/* Release the event list object owned by the stream node */
+	if (psRemove->hReadEventObj)
+	{
+		eError = OSEventObjectDestroy(psRemove->hReadEventObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+		psRemove->hReadEventObj = NULL;
+	}
+
+	/* Release the memory of the stream node */
+	OSFreeMem(psRemove);
+
+	PVR_DPF_RETURN;
+}
+
+static void FreeGlobalData(void)
+{
+	PTL_SNODE psCurrent = sTLGlobalData.psHead;
+	PTL_SNODE psNext;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	/* Clean up the SNODE list */
+	if (psCurrent)
+	{
+		while (psCurrent)
+		{
+			psNext = psCurrent->psNext;
+
+			/* Other calling code may have freed and zeroed the pointers */
+			if (psCurrent->psRDesc)
+			{
+				OSFreeMem(psCurrent->psRDesc);
+				psCurrent->psRDesc = NULL;
+			}
+			if (psCurrent->psStream)
+			{
+				OSFreeMem(psCurrent->psStream);
+				psCurrent->psStream = NULL;
+			}
+
+			/* Release the event list object owned by the stream node */
+			if (psCurrent->hReadEventObj)
+			{
+				eError = OSEventObjectDestroy(psCurrent->hReadEventObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+				psCurrent->hReadEventObj = NULL;
+			}
+
+			OSFreeMem(psCurrent);
+			psCurrent = psNext;
+		}
+
+		sTLGlobalData.psHead = NULL;
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void
+TLDeInit(void)
+{
+	PVR_DPF_ENTERED;
+
+	if (sTLGlobalData.uiClientCnt)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TLDeInit transport layer but %d client streams are still connected", sTLGlobalData.uiClientCnt));
+		sTLGlobalData.uiClientCnt = 0;
+	}
+
+	FreeGlobalData();
+
+	/* Clean up the TL global event object */
+	if (sTLGlobalData.hTLEventObj)
+	{
+		OSEventObjectDestroy(sTLGlobalData.hTLEventObj);
+		sTLGlobalData.hTLEventObj = NULL;
+	}
+
+	/* Destroy the TL global data lock */
+	if (sTLGlobalData.hTLGDLock)
+	{
+		OSLockDestroy (sTLGlobalData.hTLGDLock);
+		sTLGlobalData.hTLGDLock = NULL;
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void TLAddStreamNode(PTL_SNODE psAdd)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psAdd);
+	psAdd->psNext = TLGGD()->psHead;
+	TLGGD()->psHead = psAdd;
+
+	PVR_DPF_RETURN;
+}
+
+PTL_SNODE TLFindStreamNodeByName(const IMG_CHAR *pszName)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(pszName);
+
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn->psStream && OSStringNCompare(psn->psStream->szName, pszName, PRVSRVTL_MAX_STREAM_NAME_SIZE)==0)
+		{
+			PVR_DPF_RETURN_VAL(psn);
+		}
+	}
+
+	PVR_DPF_RETURN_VAL(NULL);
+}
+
+PTL_SNODE TLFindStreamNodeByDesc(PTL_STREAM_DESC psDesc)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDesc);
+
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn->psRDesc == psDesc || psn->psWDesc == psDesc)
+		{
+			PVR_DPF_RETURN_VAL(psn);
+		}
+	}
+	PVR_DPF_RETURN_VAL(NULL);
+}
+
+IMG_UINT32 TLDiscoverStreamNodes(const IMG_CHAR *pszNamePattern,
+                          IMG_CHAR aaszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+                          IMG_UINT32 ui32Max)
+{
+	TL_GLOBAL_DATA *psGD = TLGGD();
+	PTL_SNODE psn;
+	IMG_UINT32 ui32Count = 0;
+	size_t uiLen;
+
+	PVR_ASSERT(pszNamePattern);
+
+	if ((uiLen = OSStringLength(pszNamePattern)) == 0)
+		return 0;
+
+	for (psn = psGD->psHead; psn; psn = psn->psNext)
+	{
+		if (OSStringNCompare(pszNamePattern, psn->psStream->szName, uiLen) != 0)
+			continue;
+
+		/* If aaszStreams is NULL we only count how many string match
+		 * the given pattern. If it's a valid pointer we also return
+		 * the names. */
+		if (aaszStreams != NULL)
+		{
+			if (ui32Count >= ui32Max)
+				break;
+
+			/* all of names are shorter than MAX and null terminated */
+			OSStringSafeCopy(aaszStreams[ui32Count], psn->psStream->szName,
+			              PRVSRVTL_MAX_STREAM_NAME_SIZE);
+		}
+
+		ui32Count++;
+	}
+
+	return ui32Count;
+}
+
+PTL_SNODE TLFindAndGetStreamNodeByDesc(PTL_STREAM_DESC psDesc)
+{
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	psn = TLFindStreamNodeByDesc(psDesc);
+	if (psn == NULL)
+		PVR_DPF_RETURN_VAL(NULL);
+
+	PVR_ASSERT(psDesc == psn->psWDesc);
+
+	psn->uiWRefCount++;
+	psDesc->uiRefCount++;
+
+	PVR_DPF_RETURN_VAL(psn);
+}
+
+void TLReturnStreamNode(PTL_SNODE psNode)
+{
+	psNode->uiWRefCount--;
+	psNode->psWDesc->uiRefCount--;
+
+	PVR_ASSERT(psNode->uiWRefCount > 0);
+	PVR_ASSERT(psNode->psWDesc->uiRefCount > 0);
+}
+
+IMG_BOOL TLTryRemoveStreamAndFreeStreamNode(PTL_SNODE psRemove)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRemove);
+
+	/* If there is a client connected to this stream, defer stream's deletion */
+	if (psRemove->psRDesc != NULL || psRemove->psWDesc != NULL)
+	{
+		PVR_DPF_RETURN_VAL(IMG_FALSE);
+	}
+
+	/* Remove stream from TL_GLOBAL_DATA's list and free stream node */
+	psRemove->psStream = NULL;
+	RemoveAndFreeStreamNode(psRemove);
+
+	PVR_DPF_RETURN_VAL(IMG_TRUE);
+}
+
+IMG_BOOL TLUnrefDescAndTryFreeStreamNode(PTL_SNODE psNodeToRemove,
+                                          PTL_STREAM_DESC psSD)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psNodeToRemove);
+	PVR_ASSERT(psSD);
+
+	/* Decrement reference count. For descriptor obtained by reader it must
+	 * reach 0 (only single reader allowed) and for descriptors obtained by
+	 * writers it must reach value greater or equal to 0 (multiple writers
+	 * model). */
+	psSD->uiRefCount--;
+
+	if (psSD == psNodeToRemove->psRDesc)
+	{
+		PVR_ASSERT(0 == psSD->uiRefCount);
+		/* Remove stream descriptor (i.e. stream reader context) */
+		psNodeToRemove->psRDesc = NULL;
+	}
+	else if (psSD == psNodeToRemove->psWDesc)
+	{
+		PVR_ASSERT(0 <= psSD->uiRefCount);
+
+		psNodeToRemove->uiWRefCount--;
+
+		/* Remove stream descriptor if reference == 0 */
+		if (0 == psSD->uiRefCount)
+		{
+			psNodeToRemove->psWDesc = NULL;
+		}
+	}
+
+	/* Do not Free Stream Node if there is a write reference (a producer
+	 * context) to the stream */
+	if (NULL != psNodeToRemove->psRDesc || NULL != psNodeToRemove->psWDesc ||
+	    0 != psNodeToRemove->uiWRefCount)
+	{
+		PVR_DPF_RETURN_VAL(IMG_FALSE);
+	}
+
+	/* Make stream pointer NULL to prevent it from being destroyed in
+	 * RemoveAndFreeStreamNode. Cleanup of stream should be done by the
+	 * calling context */
+	psNodeToRemove->psStream = NULL;
+	RemoveAndFreeStreamNode(psNodeToRemove);
+
+	PVR_DPF_RETURN_VAL(IMG_TRUE);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/tlserver.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlserver.c
new file mode 100644
index 000000000000..5c8359271508
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlserver.c
@@ -0,0 +1,755 @@
+/*************************************************************************/ /*!
+@File
+@Title          KM server Transport Layer implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main bridge APIs for Transport Layer client functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+
+/*#define PVR_DPF_FUNCTION_TRACE_ON 1*/
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "connection_server.h"
+#include "allocmem.h"
+#include "devicemem.h"
+
+#include "tlintern.h"
+#include "tlstream.h"
+#include "tlserver.h"
+
+#include "pvrsrv_tlstreams.h"
+#define NO_STREAM_WAIT_PERIOD_US 2000000ULL
+#define NO_DATA_WAIT_PERIOD_US    500000ULL
+#define NO_ACQUIRE               0xffffffffU
+
+
+/*
+ * Transport Layer Client API Kernel-Mode bridge implementation
+ */
+PVRSRV_ERROR
+TLServerOpenStreamKM(const IMG_CHAR*   pszName,
+                     IMG_UINT32        ui32Mode,
+                     PTL_STREAM_DESC*  ppsSD,
+                     PMR**             ppsTLPMR)
+{
+	PVRSRV_ERROR	eError = PVRSRV_OK;
+	PVRSRV_ERROR	eErrorEO = PVRSRV_OK;
+	PTL_SNODE		psNode;
+	PTL_STREAM		psStream;
+	TL_STREAM_DESC *psNewSD = NULL;
+	IMG_HANDLE		hEvent;
+	IMG_BOOL		bIsWriteOnly = ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	                               IMG_TRUE : IMG_FALSE;
+	IMG_BOOL		bResetOnOpen = ui32Mode & PVRSRV_STREAM_FLAG_RESET_ON_OPEN ?
+	                               IMG_TRUE : IMG_FALSE;
+	IMG_BOOL		bNoOpenCB    = ui32Mode & PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK ?
+	                               IMG_TRUE : IMG_FALSE;
+	PTL_GLOBAL_DATA psGD = TLGGD();
+
+#if defined(PVR_DPF_FUNCTION_TRACE_ON)
+	PVR_DPF((PVR_DBG_CALLTRACE, "--> %s:%d entered (%s, %x)", __func__, __LINE__, pszName, ui32Mode));
+#endif
+
+	PVR_ASSERT(pszName);
+
+	/* Acquire TL_GLOBAL_DATA lock here, as if the following TLFindStreamNodeByName
+	 * returns NON NULL PTL_SNODE, we try updating the global data client count and
+	 * PTL_SNODE's psRDesc and we want to make sure the TL_SNODE is valid (eg. has
+	 * not been deleted) while we are updating it
+	 */
+	OSLockAcquire (psGD->hTLGDLock);
+
+	psNode = TLFindStreamNodeByName(pszName);
+	if ((psNode == NULL) && (ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WAIT))
+	{	/* Blocking code to wait for stream to be created if it does not exist */
+		eError = OSEventObjectOpen(psGD->hTLEventObj, &hEvent);
+		PVR_LOG_GOTO_IF_ERROR (eError, "OSEventObjectOpen", e0);
+
+		do
+		{
+			if ((psNode = TLFindStreamNodeByName(pszName)) == NULL)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "Stream %s does not exist, waiting...", pszName));
+
+				/* Release TL_GLOBAL_DATA lock before sleeping */
+				OSLockRelease (psGD->hTLGDLock);
+
+				/* Will exit OK or with timeout, both cases safe to ignore */
+				eErrorEO = OSEventObjectWaitTimeout(hEvent, NO_STREAM_WAIT_PERIOD_US);
+
+				/* Acquire lock after waking up */
+				OSLockAcquire (psGD->hTLGDLock);
+			}
+		}
+		while ((psNode == NULL) && (eErrorEO == PVRSRV_OK));
+
+		eError = OSEventObjectClose(hEvent);
+		PVR_LOG_GOTO_IF_ERROR (eError, "OSEventObjectClose", e0);
+	}
+
+	/* Make sure we have found a stream node after wait/search */
+	if (psNode == NULL)
+	{
+		/* Did we exit the wait with timeout, inform caller */
+		if (eErrorEO == PVRSRV_ERROR_TIMEOUT)
+		{
+			eError = eErrorEO;
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_NOT_FOUND;
+			PVR_DPF((PVR_DBG_ERROR, "Stream \"%s\" does not exist", pszName));
+		}
+		goto e0;
+	}
+
+	psStream = psNode->psStream;
+
+	/* Allocate memory for the stream. The memory will be allocated with the
+	 * first call. */
+	eError = TLAllocSharedMemIfNull(psStream);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to allocate memory for stream"
+				" \"%s\"", pszName));
+		goto e0;
+	}
+
+	if (bIsWriteOnly)
+	{
+
+		/* If psWDesc == NULL it means that this is the first attempt
+		 * to open stream for write. If yes create the descriptor or increment
+		 * reference count otherwise. */
+		if (psNode->psWDesc == NULL)
+		{
+			psNewSD = TLMakeStreamDesc(psNode, ui32Mode, NULL);
+			psNode->psWDesc = psNewSD;
+		}
+		else
+		{
+			psNewSD = psNode->psWDesc;
+			psNode->psWDesc->uiRefCount++;
+		}
+
+		PVR_LOG_GOTO_IF_NOMEM(psNewSD, eError, e0);
+
+		psNode->uiWRefCount++;
+	}
+	else
+	{
+		/* Only one reader per stream supported */
+		if (psNode->psRDesc != NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Cannot open \"%s\" stream, stream already"
+			        " opened", pszName));
+			eError = PVRSRV_ERROR_ALREADY_OPEN;
+			goto e0;
+		}
+
+		/* Create an event handle for this client to wait on when no data in
+		 * stream buffer. */
+		eError = OSEventObjectOpen(psNode->hReadEventObj, &hEvent);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "OSEventObjectOpen");
+			eError = PVRSRV_ERROR_UNABLE_TO_CREATE_EVENT;
+			goto e0;
+		}
+
+		psNewSD = TLMakeStreamDesc(psNode, ui32Mode, hEvent);
+		psNode->psRDesc = psNewSD;
+
+		if (!psNewSD)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Not possible to make a new stream descriptor"));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e1;
+		}
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+		        "TLServerOpenStreamKM evList=%p, evObj=%p",
+		        psNode->hReadEventObj,
+		        psNode->psRDesc->hReadEvent));
+	}
+
+	/* Copy the import handle back to the user mode API to enable access to
+	 * the stream buffer from user-mode process. */
+	eError = DevmemLocalGetImportHandle(TLStreamGetBufferPointer(psStream),
+	                                    (void**) ppsTLPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemLocalGetImportHandle", e2);
+
+	psGD->uiClientCnt++;
+
+	/* Global data updated. Now release global lock */
+	OSLockRelease (psGD->hTLGDLock);
+
+	*ppsSD = psNewSD;
+
+	if (bResetOnOpen)
+	{
+		TLStreamReset(psStream);
+	}
+
+	/* This callback is executed only on reader open. There are some actions
+	 * executed on reader open that don't make much sense for writers e.g.
+	 * injection on time synchronisation packet into the stream. */
+	if (!bIsWriteOnly && psStream->pfOnReaderOpenCallback != NULL && !bNoOpenCB)
+	{
+		psStream->pfOnReaderOpenCallback(psStream->pvOnReaderOpenUserData);
+	}
+
+	/* psNode->uiWRefCount is set to '1' on stream create so the first open
+	 * is '2'. */
+	if (bIsWriteOnly && psStream->psNotifStream != NULL &&
+	    psNode->uiWRefCount == 2)
+	{
+		TLStreamMarkStreamOpen(psStream);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Stream %s opened for %s", __func__, pszName,
+	        ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ? "write" : "read"));
+
+	PVR_DPF_RETURN_OK;
+
+e2:
+	OSFreeMem(psNewSD);
+e1:
+	if (!bIsWriteOnly)
+		OSEventObjectClose(hEvent);
+e0:
+	OSLockRelease (psGD->hTLGDLock);
+	PVR_DPF_RETURN_RC (eError);
+}
+
+PVRSRV_ERROR
+TLServerCloseStreamKM(PTL_STREAM_DESC psSD)
+{
+	PVRSRV_ERROR    eError = PVRSRV_OK;
+	PTL_GLOBAL_DATA psGD = TLGGD();
+	PTL_SNODE		psNode;
+	PTL_STREAM	psStream;
+	IMG_BOOL	bDestroyStream;
+	IMG_BOOL	bIsWriteOnly = psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	                           IMG_TRUE : IMG_FALSE;
+	IMG_BOOL	bNoOpenCB    = psSD->ui32Flags & PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK ?
+	                           IMG_TRUE : IMG_FALSE;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since the descriptor is valid, the stream should not have been made NULL */
+	PVR_ASSERT (psNode->psStream);
+
+	/* Save the stream's reference in-case its destruction is required after this
+	 * client is removed */
+	psStream = psNode->psStream;
+
+	/* Acquire TL_GLOBAL_DATA lock as the following TLRemoveDescAndTryFreeStreamNode
+	 * call will update the TL_SNODE's descriptor value */
+	OSLockAcquire (psGD->hTLGDLock);
+
+	/* Close event handle because event object list might be destroyed in
+	 * TLUnrefDescAndTryFreeStreamNode(). */
+	if (!bIsWriteOnly)
+	{
+		/* Reset the read position on close if the stream requires it. */
+		TLStreamResetReadPos(psStream);
+
+		/* Close and free the event handle resource used by this descriptor */
+		eError = OSEventObjectClose(psSD->hReadEvent);
+		if (eError != PVRSRV_OK)
+		{
+			/* Log error but continue as it seems best */
+			PVR_LOG_ERROR(eError, "OSEventObjectClose");
+			eError = PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+		}
+	}
+	else if (psNode->uiWRefCount == 2 && psStream->psNotifStream != NULL)
+	{
+		/* psNode->uiWRefCount is set to '1' on stream create so the last close
+		 * before destruction is '2'. */
+		TLStreamMarkStreamClose(psStream);
+	}
+
+	/* Remove descriptor from stream object/list */
+	bDestroyStream = TLUnrefDescAndTryFreeStreamNode (psNode, psSD);
+
+	/* Check the counter is sensible after input data validated. */
+	PVR_ASSERT(psGD->uiClientCnt > 0);
+	psGD->uiClientCnt--;
+
+	/* This callback is executed only on reader close. */
+	if (!bIsWriteOnly && psStream->pfOnReaderCloseCallback != NULL && !bNoOpenCB)
+	{
+		psStream->pfOnReaderCloseCallback(psStream->pvOnReaderCloseUserData);
+	}
+
+	OSLockRelease (psGD->hTLGDLock);
+
+	/* Destroy the stream if its TL_SNODE was removed from TL_GLOBAL_DATA */
+	if (bDestroyStream)
+	{
+		TLStreamDestroy (psStream);
+		psStream = NULL;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Stream closed", __func__));
+
+	/* Free the descriptor if ref count reaches 0. */
+	if (psSD->uiRefCount == 0)
+	{
+		/* Free the stream descriptor object */
+		OSFreeMem(psSD);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerReserveStreamKM(PTL_STREAM_DESC psSD,
+                        IMG_UINT32* ui32BufferOffset,
+                        IMG_UINT32 ui32Size,
+                        IMG_UINT32 ui32SizeMin,
+                        IMG_UINT32* pui32Available)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+	IMG_UINT8* pui8Buffer = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire the global lock. We have to be sure that no one modifies
+	 * the list while we are looking for our stream. */
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	/* The TL writers that currently land here are at a very low to none risk
+	 * to breach max TL packet size constraint (even if there is no reader
+	 * connected to the TL stream and hence eventually will cause the TL stream
+	 * to be full). Hence no need to know the status of TL stream reader
+	 * connection.
+	 */
+	eError = TLStreamReserve2(psNode->psStream, &pui8Buffer, ui32Size,
+				  ui32SizeMin, pui32Available, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Failed to reserve %u (%u, %u) bytes in the stream, error %s.",
+				ui32Size, ui32SizeMin, *pui32Available, PVRSRVGETERRORSTRING(eError)));
+	}
+	else if (pui8Buffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Not enough space in the stream."));
+		eError = PVRSRV_ERROR_STREAM_FULL;
+	}
+	else
+	{
+		*ui32BufferOffset = pui8Buffer - psNode->psStream->pbyBuffer;
+		PVR_ASSERT(*ui32BufferOffset < psNode->psStream->ui32Size);
+	}
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerCommitStreamKM(PTL_STREAM_DESC psSD,
+                       IMG_UINT32 ui32Size)
+{
+	TL_GLOBAL_DATA*	psGD = TLGGD();
+	PTL_SNODE psNode;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire the global lock. We have to be sure that no one modifies
+	 * the list while we are looking for our stream. */
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	eError = TLStreamCommit(psNode->psStream, ui32Size);
+	PVR_LOG_IF_ERROR(eError, "TLStreamCommit");
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerDiscoverStreamsKM(const IMG_CHAR *pszNamePattern,
+                          IMG_UINT32 ui32Size,
+                          IMG_CHAR *pszStreams,
+                          IMG_UINT32 *pui32NumFound)
+{
+	PTL_SNODE psNode = NULL;
+	IMG_CHAR (*paszStreams)[PRVSRVTL_MAX_STREAM_NAME_SIZE] =
+			(IMG_CHAR (*)[PRVSRVTL_MAX_STREAM_NAME_SIZE]) (void *)pszStreams;
+
+	if (*pszNamePattern == '\0')
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (ui32Size % PRVSRVTL_MAX_STREAM_NAME_SIZE != 0)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	/* Quick exit if there are no streams */
+	if (TLGGD()->psHead == NULL)
+	{
+		*pui32NumFound = 0;
+		return PVRSRV_OK;
+	}
+
+	OSLockAcquire(TLGGD()->hTLGDLock);
+
+	*pui32NumFound = TLDiscoverStreamNodes(pszNamePattern, paszStreams,
+	                                  ui32Size / PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	/* Find "tlctrl" stream and reset it */
+	psNode = TLFindStreamNodeByName(PVRSRV_TL_CTRL_STREAM);
+	if (psNode != NULL)
+		TLStreamReset(psNode->psStream);
+
+	OSLockRelease(TLGGD()->hTLGDLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+TLServerAcquireDataKM(PTL_STREAM_DESC psSD,
+                      IMG_UINT32*     puiReadOffset,
+                      IMG_UINT32*     puiReadLen)
+{
+	PVRSRV_ERROR    eError = PVRSRV_OK;
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	IMG_UINT32      uiTmpOffset;
+	IMG_UINT32      uiTmpLen = 0;
+	PTL_SNODE       psNode;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	TL_COUNTER_INC(psSD->ui32AcquireCount);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* If we are here, the stream will never be made NULL until this context itself
+	 * calls TLRemoveDescAndTryFreeStreamNode(). This is because the producer will
+	 * fail to make the stream NULL (by calling TLTryRemoveStreamAndFreeStreamNode)
+	 * when a valid stream descriptor is present (i.e. a client is connected).
+	 * Hence, no checks for stream being NON NULL are required after this. */
+	PVR_ASSERT (psNode->psStream);
+
+	psSD->ui32ReadLen = 0;	/* Handle NULL read returns */
+
+	do
+	{
+		uiTmpLen = TLStreamAcquireReadPos(psNode->psStream, psSD->ui32Flags & PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK, &uiTmpOffset);
+
+		/* Check we have not already exceeded read limit with just offset
+		 * regardless of data length to ensure the client sees the RC */
+		if (psSD->ui32Flags & PVRSRV_STREAM_FLAG_READ_LIMIT)
+		{
+			/* Check to see if we are reading beyond the read limit */
+			if (uiTmpOffset >= psSD->ui32ReadLimit)
+			{
+				PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_READLIMIT_REACHED);
+			}
+		}
+
+		if (uiTmpLen > 0)
+		{ /* Data found */
+
+			/* Check we have not already exceeded read limit offset+len */
+			if (psSD->ui32Flags & PVRSRV_STREAM_FLAG_READ_LIMIT)
+			{
+				/* Adjust the read length if it goes beyond the read limit
+				 * limit always guaranteed to be on packet */
+				if ((uiTmpOffset + uiTmpLen) >= psSD->ui32ReadLimit)
+				{
+					uiTmpLen = psSD->ui32ReadLimit - uiTmpOffset;
+				}
+			}
+
+			*puiReadOffset = uiTmpOffset;
+			*puiReadLen = uiTmpLen;
+			psSD->ui32ReadLen = uiTmpLen;	/* Save the original data length in the stream desc */
+			PVR_DPF_RETURN_OK;
+		}
+		else if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING))
+		{ /* No data found blocking */
+
+			/* Instead of doing a complete sleep for `NO_DATA_WAIT_PERIOD_US` us, we sleep in chunks
+			 * of 168 ms. In a "deferred" signal scenario from writer, this gives us a chance to
+			 * wake-up (timeout) early and continue reading in-case some data is available */
+			IMG_UINT64 ui64WaitInChunksUs = MIN(NO_DATA_WAIT_PERIOD_US, 168000ULL);
+			IMG_BOOL bDataFound = IMG_FALSE;
+
+			TL_COUNTER_INC(psSD->ui32NoDataSleep);
+
+			LOOP_UNTIL_TIMEOUT_US(NO_DATA_WAIT_PERIOD_US)
+			{
+				eError = OSEventObjectWaitTimeout(psSD->hReadEvent, ui64WaitInChunksUs);
+				if (eError == PVRSRV_OK)
+				{
+					bDataFound = IMG_TRUE;
+					TL_COUNTER_INC(psSD->ui32Signalled);
+					break;
+				}
+				else if (eError == PVRSRV_ERROR_TIMEOUT)
+				{
+					if (TLStreamOutOfData(psNode->psStream))
+					{
+						/* Return on timeout if stream empty, else let while exit and return data */
+						continue;
+					}
+					else
+					{
+						bDataFound = IMG_TRUE;
+						TL_COUNTER_INC(psSD->ui32TimeoutData);
+						PVR_DPF((PVR_DBG_MESSAGE, "%s: Data found at timeout. Current BuffUt = %u",
+												 __func__, TLStreamGetUT(psNode->psStream)));
+						break;
+					}
+				}
+				else
+				{ /* Some other system error with event objects */
+					PVR_DPF_RETURN_RC(eError);
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (bDataFound)
+			{
+				continue;
+			}
+			else
+			{
+				TL_COUNTER_INC(psSD->ui32TimeoutEmpty);
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+		}
+		else
+		{ /* No data non-blocking */
+			TL_COUNTER_INC(psSD->ui32NoData);
+
+			/* When no-data in non-blocking mode, uiReadOffset should be set to NO_ACQUIRE
+			 * signifying there's no need of Release call */
+			*puiReadOffset = NO_ACQUIRE;
+			*puiReadLen = 0;
+			PVR_DPF_RETURN_OK;
+		}
+	}
+	while (1);
+}
+
+PVRSRV_ERROR
+TLServerReleaseDataKM(PTL_STREAM_DESC psSD,
+                      IMG_UINT32      uiReadOffset,
+                      IMG_UINT32      uiReadLen)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+
+	PVR_DPF_ENTERED;
+
+	/* Unreferenced in release builds */
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+
+	PVR_ASSERT(psSD);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	if ((uiReadLen % PVRSRVTL_PACKET_ALIGNMENT != 0))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "TLReleaseDataKM uiReadOffset=%d, uiReadLen=%d", uiReadOffset, uiReadLen));
+
+	/* Move read position on to free up space in stream buffer */
+	PVR_DPF_RETURN_RC(TLStreamAdvanceReadPos(psNode->psStream, uiReadLen, psSD->ui32ReadLen));
+}
+
+PVRSRV_ERROR
+TLServerWriteDataKM(PTL_STREAM_DESC psSD,
+                    IMG_UINT32 ui32Size,
+                    IMG_BYTE* pui8Data)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	eError = TLStreamWrite(psNode->psStream, pui8Data, ui32Size);
+	/* propagate error up but don't print anything here */
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ End of file (tlserver.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/tlstream.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlstream.c
new file mode 100644
index 000000000000..29cc42a802fe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/tlstream.c
@@ -0,0 +1,1686 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer API implementation.
+                These functions are provided to driver components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "allocmem.h"
+#include "devicemem.h"
+#include "pvrsrv_error.h"
+#include "sysinfo.h"
+#include "osfunc.h"
+#include "log2.h"
+
+#include "tlintern.h"
+#include "tlstream.h"
+
+#include "pvrsrv.h"
+
+#if !defined(EVENT_OBJECT_TIMEOUT_US)
+#error EVENT_OBJECT_TIMEOUT_US should be defined sysinfo.h
+#endif
+
+#define READ_PENDING_TIMEOUT_US 100000ULL
+
+/*! Compute maximum TL packet size for this stream. Max packet size will be
+ * minimum of PVRSRVTL_MAX_PACKET_SIZE and (BufferSize / 2.5). This computation
+ * is required to avoid a corner case that was observed when TL buffer size is
+ * smaller than twice of TL max packet size and read, write index are positioned
+ * in such a way that the TL packet (write packet + padding packet) size is may
+ * be bigger than the buffer size itself.
+ */
+#define GET_TL_MAX_PACKET_SIZE( bufSize ) PVRSRVTL_ALIGN( MIN( PVRSRVTL_MAX_PACKET_SIZE, ( 2 * bufSize ) / 5 ) )
+
+/* Given the state of the buffer it returns a number of bytes that the client
+ * can use for a successful allocation. */
+static INLINE IMG_UINT32 suggestAllocSize(IMG_UINT32 ui32LRead,
+                                          IMG_UINT32 ui32LWrite,
+                                          IMG_UINT32 ui32CBSize,
+                                          IMG_UINT32 ui32ReqSizeMin,
+                                          IMG_UINT32 ui32MaxPacketSize)
+{
+	IMG_UINT32 ui32AvSpace = 0;
+
+	/* This could be written in fewer lines using the ? operator but it
+		would not be kind to potential readers of this source at all. */
+	if (ui32LRead > ui32LWrite)                          /* Buffer WRAPPED */
+	{
+		if ((ui32LRead - ui32LWrite) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32LRead - ui32LWrite - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+	}
+	else                                                  /* Normal, no wrap */
+	{
+		if ((ui32CBSize - ui32LWrite) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32CBSize - ui32LWrite - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+		else if ((ui32LRead - 0) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32LRead - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+	}
+	/* The max size of a TL packet currently is UINT16. adjust accordingly */
+	return MIN(ui32AvSpace, ui32MaxPacketSize);
+}
+
+/* Returns bytes left in the buffer. Negative if there is not any.
+ * two 8b aligned values are reserved, one for the write failed buffer flag
+ * and one to be able to distinguish the buffer full state to the buffer
+ * empty state.
+ * Always returns free space -8 even when the "write failed" packet may be
+ * already in the stream before this write. */
+static INLINE IMG_INT
+circbufSpaceLeft(IMG_UINT32 ui32Read, IMG_UINT32 ui32Write, IMG_UINT32 ui32size)
+{
+	/* We need to reserve 8b (one packet) in the buffer to be able to tell empty
+	 * buffers from full buffers and one more for packet write fail packet */
+	if (ui32Read > ui32Write)
+	{
+		return (IMG_INT)ui32Read - (IMG_INT)ui32Write - (IMG_INT)BUFFER_RESERVED_SPACE;
+	}
+	else
+	{
+		return (IMG_INT)ui32size - ((IMG_INT)ui32Write - (IMG_INT)ui32Read) - (IMG_INT)BUFFER_RESERVED_SPACE;
+	}
+}
+
+IMG_UINT32 TLStreamGetUT(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	IMG_UINT32 ui32LRead = psStream->ui32Read, ui32LWrite = psStream->ui32Write;
+
+	if (ui32LWrite >= ui32LRead)
+	{
+		return (ui32LWrite-ui32LRead);
+	}
+	else
+	{
+		return (psStream->ui32Size-ui32LRead+ui32LWrite);
+	}
+}
+
+PVRSRV_ERROR TLAllocSharedMemIfNull(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	PVRSRV_ERROR eError;
+
+	/* CPU Local memory used as these buffers are not accessed by the device.
+	 * CPU Uncached write combine memory used to improve write performance,
+	 * memory barrier added in TLStreamCommit to ensure data written to memory
+	 * before CB write point is updated before consumption by the reader.
+	 */
+	IMG_CHAR pszBufferLabel[PRVSRVTL_MAX_STREAM_NAME_SIZE + 20];
+	PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                                    PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                                    PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                                    PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                                    PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL) |  /* TL for now is only used by host driver, so cpulocal mem suffices */
+	                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+	/* Exit if memory has already been allocated. */
+	if (psStream->pbyBuffer != NULL)
+		return PVRSRV_OK;
+
+	OSSNPrintf(pszBufferLabel, sizeof(pszBufferLabel), "TLStreamBuf-%s",
+	           psStream->szName);
+
+	/* Use HostMemDeviceNode instead of psStream->psDevNode to benefit from faster
+	 * accesses to CPU local memory. When the framework to access CPU_LOCAL device
+	 * memory from GPU is fixed, we'll switch back to use psStream->psDevNode for
+	 * TL buffers */
+	eError = DevmemAllocateExportable((IMG_HANDLE)PVRSRVGetPVRSRVData()->psHostMemDeviceNode,
+	                                  (IMG_DEVMEM_SIZE_T) psStream->ui32Size,
+	                                  (IMG_DEVMEM_ALIGN_T) OSGetPageSize(),
+	                                  ExactLog2(OSGetPageSize()),
+	                                  uiMemFlags,
+	                                  pszBufferLabel,
+	                                  &psStream->psStreamMemDesc);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+	eError = DevmemAcquireCpuVirtAddr(psStream->psStreamMemDesc,
+	                                  (void**) &psStream->pbyBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", e1);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFree(psStream->psStreamMemDesc);
+e0:
+	return eError;
+}
+
+void TLFreeSharedMem(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+
+	if (psStream->pbyBuffer != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psStream->psStreamMemDesc);
+		psStream->pbyBuffer = NULL;
+	}
+	if (psStream->psStreamMemDesc != NULL)
+	{
+		DevmemFree(psStream->psStreamMemDesc);
+		psStream->psStreamMemDesc = NULL;
+	}
+}
+
+/* Special space left routine for TL_FLAG_PERMANENT_NO_WRAP streams */
+static INLINE IMG_UINT
+bufSpaceLeft(IMG_UINT32 ui32Read, IMG_UINT32 ui32Write, IMG_UINT32 ui32size)
+{
+	 /* buffers from full buffers and one more for packet write fail packet */
+	PVR_ASSERT(ui32Read<=ui32Write);
+	return ui32size - ui32Write;
+}
+
+/*******************************************************************************
+ * TL Server public API implementation.
+ ******************************************************************************/
+PVRSRV_ERROR
+TLStreamCreate(IMG_HANDLE *phStream,
+               const IMG_CHAR *szStreamName,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32StreamFlags,
+               TL_STREAM_ONREADEROPENCB pfOnReaderOpenCB,
+               void *pvOnReaderOpenUD,
+               TL_STREAM_ONREADERCLOSECB pfOnReaderCloseCB,
+               void *pvOnReaderCloseUD,
+               TL_STREAM_SOURCECB pfProducerCB,
+               void *pvProducerUD)
+{
+	PTL_STREAM    psTmp;
+	PVRSRV_ERROR  eError;
+	IMG_HANDLE    hEventList;
+	PTL_SNODE     psn;
+	TL_OPMODE     eOpMode;
+
+	PVR_DPF_ENTERED;
+	/* Parameter checks: non NULL handler required */
+	if (NULL == phStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	if (szStreamName == NULL || *szStreamName == '\0' ||
+	    OSStringLength(szStreamName) >= PRVSRVTL_MAX_STREAM_NAME_SIZE)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eOpMode = ui32StreamFlags & TL_OPMODE_MASK;
+	if (( eOpMode <= TL_OPMODE_UNDEF ) || ( eOpMode >= TL_OPMODE_LAST ))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OpMode for TL stream is invalid"));
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Acquire TL_GLOBAL_DATA lock here because, if the following TLFindStreamNodeByName()
+	 * returns NULL, a new TL_SNODE will be added to TL_GLOBAL_DATA's TL_SNODE list */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Check if there already exists a stream with this name. */
+	psn = TLFindStreamNodeByName( szStreamName );
+	if (NULL != psn)
+	{
+		eError = PVRSRV_ERROR_ALREADY_EXISTS;
+		goto e0;
+	}
+
+	/* Allocate stream structure container (stream struct) for the new stream */
+	psTmp = OSAllocZMem(sizeof(TL_STREAM));
+	if (NULL == psTmp)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	OSStringSafeCopy(psTmp->szName, szStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	if (ui32StreamFlags & TL_FLAG_FORCE_FLUSH)
+	{
+		psTmp->bWaitForEmptyOnDestroy = IMG_TRUE;
+	}
+
+	psTmp->bNoSignalOnCommit = (ui32StreamFlags&TL_FLAG_NO_SIGNAL_ON_COMMIT) ? IMG_TRUE : IMG_FALSE;
+	psTmp->bNoWrapPermanent = (ui32StreamFlags&TL_FLAG_PERMANENT_NO_WRAP) ? IMG_TRUE : IMG_FALSE;
+
+	psTmp->eOpMode = eOpMode;
+	if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+	{
+		/* Only allow drop properties to be mixed with no-wrap type streams
+		 * since space does not become available when reads take place hence
+		 * no point blocking.
+		 */
+		if (psTmp->bNoWrapPermanent)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e1;
+		}
+	}
+
+	/* Additional synchronisation object required for some streams e.g. blocking */
+	eError = OSEventObjectCreate(NULL, &psTmp->hProducerEventObj);
+	PVR_GOTO_IF_ERROR(eError, e1);
+	/* Create an event handle for this kind of stream */
+	eError = OSEventObjectOpen(psTmp->hProducerEventObj, &psTmp->hProducerEvent);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	psTmp->pfOnReaderOpenCallback = pfOnReaderOpenCB;
+	psTmp->pvOnReaderOpenUserData = pvOnReaderOpenUD;
+	psTmp->pfOnReaderCloseCallback = pfOnReaderCloseCB;
+	psTmp->pvOnReaderCloseUserData = pvOnReaderCloseUD;
+	/* Remember producer supplied CB and data for later */
+	psTmp->pfProducerCallback = (void(*)(void))pfProducerCB;
+	psTmp->pvProducerUserData = pvProducerUD;
+
+	psTmp->psNotifStream = NULL;
+
+	/* Round the requested bytes to a multiple of array elements' size, eg round 3 to 4 */
+	psTmp->ui32Size = PVRSRVTL_ALIGN(ui32Size);
+
+	/* Signalling from TLStreamCommit is deferred until buffer is slightly (~12%) filled */
+	psTmp->ui32ThresholdUsageForSignal = psTmp->ui32Size >> 3;
+	psTmp->ui32MaxPacketSize = GET_TL_MAX_PACKET_SIZE(psTmp->ui32Size);
+	psTmp->ui32Read = 0;
+	psTmp->ui32Write = 0;
+	psTmp->ui32Pending = NOTHING_PENDING;
+	psTmp->bReadPending = IMG_FALSE;
+	psTmp->bSignalPending = IMG_FALSE;
+
+#if defined(TL_BUFFER_STATS)
+	OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 0);
+	/* Setting MAX possible value for "minimum" time to full,
+	 * helps in the logic which calculates this time */
+	psTmp->ui32MinTimeToFullInUs = IMG_UINT32_MAX;
+#endif
+
+	/* Memory will be allocated on first connect to the stream */
+	if (!(ui32StreamFlags & TL_FLAG_ALLOCATE_ON_FIRST_OPEN))
+	{
+		/* Allocate memory for the circular buffer and export it to user space. */
+		eError = TLAllocSharedMemIfNull(psTmp);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TLAllocSharedMem", e3);
+	}
+
+	/* Synchronisation object to synchronise with user side data transfers. */
+	eError = OSEventObjectCreate(psTmp->szName, &hEventList);
+	PVR_GOTO_IF_ERROR(eError, e4);
+
+	eError = OSLockCreate (&psTmp->hStreamWLock);
+	PVR_GOTO_IF_ERROR(eError, e5);
+
+	eError = OSLockCreate (&psTmp->hReadLock);
+	PVR_GOTO_IF_ERROR(eError, e6);
+
+	/* Now remember the stream in the global TL structures */
+	psn = TLMakeSNode(hEventList, (TL_STREAM *)psTmp, NULL);
+	PVR_GOTO_IF_NOMEM(psn, eError, e7);
+
+	/* Stream node created, now reset the write reference count to 1
+	 * (i.e. this context's reference) */
+	psn->uiWRefCount = 1;
+
+	TLAddStreamNode(psn);
+
+	/* Release TL_GLOBAL_DATA lock as the new TL_SNODE is now added to the list */
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	/* Best effort signal, client wait timeout will ultimately let it find the
+	 * new stream if this fails, acceptable to avoid clean-up as it is tricky
+	 * at this point */
+	(void) OSEventObjectSignal(TLGGD()->hTLEventObj);
+
+	/* Pass the newly created stream handle back to caller */
+	*phStream = (IMG_HANDLE)psTmp;
+	PVR_DPF_RETURN_OK;
+
+e7:
+	OSLockDestroy(psTmp->hReadLock);
+e6:
+	OSLockDestroy(psTmp->hStreamWLock);
+e5:
+	OSEventObjectDestroy(hEventList);
+e4:
+	TLFreeSharedMem(psTmp);
+e3:
+	OSEventObjectClose(psTmp->hProducerEvent);
+e2:
+	OSEventObjectDestroy(psTmp->hProducerEventObj);
+e1:
+	OSFreeMem(psTmp);
+e0:
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+void TLStreamReset(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	IMG_HANDLE hEventWaitForWriterToComplete;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psStream != NULL);
+
+	OSLockAcquire(psStream->hStreamWLock);
+
+	while (psStream->ui32Pending != NOTHING_PENDING)
+	{
+		/* We're in the middle of a write so we cannot reset the stream.
+		 * We are going to wait until the data is committed. Release lock while
+		 * we're here. */
+		OSLockRelease(psStream->hStreamWLock);
+
+		eError = OSEventObjectOpen(psStream->psNode->hReadEventObj,
+		                           &hEventWaitForWriterToComplete);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+		/* Event when psStream->bNoSignalOnCommit is set we can still use
+		 * the timeout capability of event object API (time in us). */
+		eError = OSEventObjectWaitTimeout(hEventWaitForWriterToComplete, 100);
+		if (eError != PVRSRV_ERROR_TIMEOUT && eError != PVRSRV_OK)
+		{
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectWaitTimeout", TimeoutError);
+		}
+
+		eError = OSEventObjectClose(hEventWaitForWriterToComplete);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectClose");
+
+		OSLockAcquire(psStream->hStreamWLock);
+
+		/* Either timeout occurred or the stream has been signalled.
+		 * If former we have to check if the data was committed and if latter
+		 * if the stream hasn't been re-reserved. Either way we have to go
+		 * back to the condition.
+		 * If the stream has been released we'll exit with the lock held so
+		 * we can finally go and reset the stream. */
+	}
+
+	psStream->ui32Read = 0;
+	psStream->ui32Write = 0;
+	/* we know that ui32Pending already has correct value (no need to set) */
+
+	OSLockRelease(psStream->hStreamWLock);
+
+	return;
+
+TimeoutError:
+	eError = OSEventObjectClose(hEventWaitForWriterToComplete);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+PVRSRV_ERROR
+TLStreamSetNotifStream(IMG_HANDLE hStream, IMG_HANDLE hNotifStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+
+	if (hStream == NULL || hNotifStream == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psStream->psNotifStream = (PTL_STREAM) hNotifStream;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+TLStreamReconfigure(
+		IMG_HANDLE hStream,
+		IMG_UINT32 ui32StreamFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PTL_STREAM psTmp;
+	TL_OPMODE eOpMode;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eOpMode = ui32StreamFlags & TL_OPMODE_MASK;
+	if (( eOpMode <= TL_OPMODE_UNDEF ) || ( eOpMode >= TL_OPMODE_LAST ))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OpMode for TL stream is invalid"));
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Prevent the TL Stream buffer from being written to
+	 * while its mode is being reconfigured
+	 */
+	OSLockAcquire (psTmp->hStreamWLock);
+	if (NOTHING_PENDING != psTmp->ui32Pending)
+	{
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_READY);
+	}
+	psTmp->ui32Pending = 0;
+	OSLockRelease (psTmp->hStreamWLock);
+
+	psTmp->eOpMode = eOpMode;
+	if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+	{
+		/* Only allow drop properties to be mixed with no-wrap type streams
+		 * since space does not become available when reads take place hence
+		 * no point blocking.
+		 */
+		if (psTmp->bNoWrapPermanent)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e1;
+		}
+	}
+
+	OSLockAcquire (psTmp->hStreamWLock);
+	psTmp->ui32Pending = NOTHING_PENDING;
+	OSLockRelease (psTmp->hStreamWLock);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLStreamOpen(IMG_HANDLE     *phStream,
+             const IMG_CHAR *szStreamName)
+{
+	PTL_SNODE psTmpSNode;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == phStream || NULL == szStreamName)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Acquire the TL_GLOBAL_DATA lock first to ensure,
+	 * the TL_STREAM while returned and being modified,
+	 * is not deleted by some other context */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Search for a stream node with a matching stream name */
+	psTmpSNode = TLFindStreamNodeByName(szStreamName);
+
+	if (NULL == psTmpSNode)
+	{
+		OSLockRelease (TLGGD()->hTLGDLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_FOUND);
+	}
+
+	if (psTmpSNode->psStream->psNotifStream != NULL &&
+	    psTmpSNode->uiWRefCount == 1)
+	{
+		TLStreamMarkStreamOpen(psTmpSNode->psStream);
+	}
+
+	/* The TL_SNODE->uiWRefCount governs the presence of this node in the
+	 * TL_GLOBAL_DATA list i.e. when uiWRefCount falls to zero we try removing
+	 * this node from the TL_GLOBAL_DATA list. Hence, is protected using the
+	 * TL_GLOBAL_DATA lock and not TL_STREAM lock */
+	psTmpSNode->uiWRefCount++;
+
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	/* Return the stream handle to the caller */
+	*phStream = (IMG_HANDLE)psTmpSNode->psStream;
+
+	PVR_DPF_RETURN_VAL(PVRSRV_OK);
+}
+
+void
+TLStreamClose(IMG_HANDLE hStream)
+{
+	PTL_STREAM	psTmp;
+	IMG_BOOL	bDestroyStream;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "TLStreamClose failed as NULL stream handler passed, nothing done."));
+		PVR_DPF_RETURN;
+	}
+
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Acquire TL_GLOBAL_DATA lock for updating the reference count as this will be required
+	 * in-case this TL_STREAM node is to be deleted */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Decrement write reference counter of the stream */
+	psTmp->psNode->uiWRefCount--;
+
+	if (0 != psTmp->psNode->uiWRefCount)
+	{
+		/* The stream is still being used in other context(s) do not destroy
+		 * anything */
+
+		/* uiWRefCount == 1 means that stream was closed for write. Next
+		 * close is pairing TLStreamCreate(). Send notification to indicate
+		 * that no writer are connected to the stream any more. */
+		if (psTmp->psNotifStream != NULL && psTmp->psNode->uiWRefCount == 1)
+		{
+			TLStreamMarkStreamClose(psTmp);
+		}
+
+		OSLockRelease (TLGGD()->hTLGDLock);
+		PVR_DPF_RETURN;
+	}
+	else
+	{
+		/* Now we try removing this TL_STREAM from TL_GLOBAL_DATA */
+
+		if (psTmp->bWaitForEmptyOnDestroy)
+		{
+			/* We won't require the TL_STREAM lock to be acquired here for accessing its read
+			 * and write offsets. REASON: We are here because there is no producer context
+			 * referencing this TL_STREAM, hence its ui32Write offset won't be changed now.
+			 * Also, the update of ui32Read offset is not protected by locks */
+			while (psTmp->ui32Read != psTmp->ui32Write)
+			{
+				/* Release lock before sleeping */
+				OSLockRelease (TLGGD()->hTLGDLock);
+
+				OSEventObjectWaitTimeout(psTmp->hProducerEvent, EVENT_OBJECT_TIMEOUT_US);
+
+				OSLockAcquire (TLGGD()->hTLGDLock);
+
+				/* Ensure destruction of stream is still required */
+				if (0 != psTmp->psNode->uiWRefCount)
+				{
+					OSLockRelease (TLGGD()->hTLGDLock);
+					PVR_DPF_RETURN;
+				}
+			}
+		}
+
+		/* Try removing the stream from TL_GLOBAL_DATA */
+		bDestroyStream = TLTryRemoveStreamAndFreeStreamNode (psTmp->psNode);
+
+		OSLockRelease (TLGGD()->hTLGDLock);
+
+		if (bDestroyStream)
+		{
+			/* Destroy the stream if it was removed from TL_GLOBAL_DATA */
+			TLStreamDestroy (psTmp);
+			psTmp = NULL;
+		}
+		PVR_DPF_RETURN;
+	}
+}
+
+/*
+ * DoTLSetPacketHeader
+ *
+ * Ensure that whenever we update a Header we always add the RESERVED field
+ */
+static inline void DoTLSetPacketHeader(PVRSRVTL_PPACKETHDR, IMG_UINT32);
+static inline void
+DoTLSetPacketHeader(PVRSRVTL_PPACKETHDR pHdr,
+				IMG_UINT32 ui32Val)
+{
+	PVR_ASSERT(((size_t)pHdr & (size_t)(PVRSRVTL_PACKET_ALIGNMENT - 1)) == 0);
+
+	/* Check that this is a correctly aligned packet header. */
+	if (((size_t)pHdr & (size_t)(PVRSRVTL_PACKET_ALIGNMENT - 1)) != 0)
+	{
+		/* Should return an error because the header is misaligned */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Misaligned header @ %p", __func__, pHdr));
+		pHdr->uiTypeSize = ui32Val;
+	}
+	else
+	{
+		pHdr->uiTypeSize = ui32Val;
+		pHdr->uiReserved = PVRSRVTL_PACKETHDR_RESERVED;
+	}
+}
+
+static PVRSRV_ERROR
+DoTLStreamReserve(IMG_HANDLE hStream,
+				IMG_UINT8 **ppui8Data,
+				IMG_UINT32 ui32ReqSize,
+				IMG_UINT32 ui32ReqSizeMin,
+				PVRSRVTL_PACKETTYPE ePacketType,
+				IMG_UINT32* pui32AvSpace,
+				IMG_UINT32* pui32Flags)
+{
+	PTL_STREAM psTmp;
+	IMG_UINT32 *pui32Buf, ui32LRead, ui32LWrite, ui32LPending, lReqSizeAligned, lReqSizeActual, ui32CreateFreeSpace;
+	IMG_UINT32 ui32InputFlags = 0;
+	IMG_INT pad, iFreeSpace;
+	IMG_UINT8 *pui8IncrRead = NULL;
+	PVRSRVTL_PPACKETHDR pHdr;
+
+	PVR_DPF_ENTERED;
+	if (pui32AvSpace) *pui32AvSpace = 0;
+	if (pui32Flags)
+	{
+		ui32InputFlags = *pui32Flags;
+		*pui32Flags = 0;
+	}
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Assert used as the packet type parameter is currently only provided
+	 * by the TL APIs, not the calling client */
+	PVR_ASSERT((PVRSRVTL_PACKETTYPE_UNDEF < ePacketType) && (PVRSRVTL_PACKETTYPE_LAST >= ePacketType));
+
+	/* The buffer is only used in "rounded" (aligned) chunks */
+	lReqSizeAligned = PVRSRVTL_ALIGN(ui32ReqSize);
+
+	/* Lock the stream before reading it's pending value, because if pending is set
+	 * to NOTHING_PENDING, we update the pending value such that subsequent calls to
+	 * this function from other context(s) fail with PVRSRV_ERROR_NOT_READY */
+	OSLockAcquire (psTmp->hStreamWLock);
+
+#if defined(TL_BUFFER_STATS)
+	/* If writing into an empty buffer, start recording time-to-full */
+	if (psTmp->ui32Read == psTmp->ui32Write)
+	{
+		OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 1);
+		psTmp->ui32TimeStart = OSClockus();
+	}
+
+	if (ui32ReqSize > psTmp->ui32MaxReserveWatermark)
+	{
+		psTmp->ui32MaxReserveWatermark = ui32ReqSize;
+	}
+#endif
+
+	/* Get a local copy of the stream buffer parameters */
+	ui32LRead  = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+	ui32LPending = psTmp->ui32Pending;
+
+	/* Multiple pending reserves are not supported. */
+	if (NOTHING_PENDING != ui32LPending)
+	{
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_READY);
+	}
+
+	if (psTmp->ui32MaxPacketSize < lReqSizeAligned)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Requested Size: %u > TL Max Packet size: %u", lReqSizeAligned, psTmp->ui32MaxPacketSize));
+		psTmp->ui32Pending = NOTHING_PENDING;
+		if (pui32AvSpace)
+		{
+			*pui32AvSpace = suggestAllocSize(ui32LRead, ui32LWrite, psTmp->ui32Size, ui32ReqSizeMin, psTmp->ui32MaxPacketSize);
+			if (*pui32AvSpace == 0 && psTmp->eOpMode == TL_OPMODE_DROP_OLDEST)
+			{
+				*pui32AvSpace = psTmp->ui32MaxPacketSize;
+				PVR_DPF((PVR_DBG_MESSAGE, "Opmode is Drop_Oldest, so Available Space changed to: %u", *pui32AvSpace));
+			}
+		}
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED);
+	}
+
+	/* Prevent other threads from entering this region before we are done
+	 * updating the pending value and write offset (in case of padding). This
+	 * is not exactly a lock but a signal for other contexts that there is a
+	 * TLStreamCommit operation pending on this stream */
+	psTmp->ui32Pending = 0;
+
+	OSLockRelease (psTmp->hStreamWLock);
+
+	/* If there is enough contiguous space following the current Write
+	 * position then no padding is required */
+	if (  psTmp->ui32Size
+		< ui32LWrite + lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR) )
+	{
+		pad = psTmp->ui32Size - ui32LWrite;
+	}
+	else
+	{
+		pad = 0;
+	}
+
+	lReqSizeActual = lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR) + pad;
+	if (psTmp->bNoWrapPermanent)
+	{
+		iFreeSpace = bufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+	else
+	{
+		iFreeSpace = circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+
+	if (iFreeSpace < (IMG_INT) lReqSizeActual)
+	{
+		/* If this is a blocking reserve and there is not enough space then wait. */
+		if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+		{
+			/* Stream create should stop us entering here when
+			 * psTmp->bNoWrapPermanent is true as it does not make sense to
+			 * block on permanent data streams. */
+			PVR_ASSERT(psTmp->bNoWrapPermanent == IMG_FALSE);
+			while ( ( circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size)
+				 <(IMG_INT) lReqSizeActual ) )
+			{
+				/* The TL bridge is lockless now, so changing to OSEventObjectWait() */
+				OSEventObjectWait(psTmp->hProducerEvent);
+				// update local copies.
+				ui32LRead  = psTmp->ui32Read;
+				ui32LWrite = psTmp->ui32Write;
+			}
+		}
+		/* Data overwriting, also insert PACKETS_DROPPED flag into existing packet */
+		else if (psTmp->eOpMode == TL_OPMODE_DROP_OLDEST)
+		{
+			OSLockAcquire(psTmp->hReadLock);
+
+			while (psTmp->bReadPending)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "Waiting for the pending read operation to complete."));
+				OSLockRelease(psTmp->hReadLock);
+#if defined(TL_BUFFER_STATS)
+				TL_COUNTER_INC(psTmp->ui32CntWriteWaits);
+#endif
+				(void) OSEventObjectWaitTimeout(psTmp->hProducerEvent, READ_PENDING_TIMEOUT_US);
+				OSLockAcquire(psTmp->hReadLock);
+			}
+
+#if defined(TL_BUFFER_STATS)
+			TL_COUNTER_INC(psTmp->ui32CntWriteSuccesses);
+#endif
+			ui32LRead = psTmp->ui32Read;
+
+			if ( circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size)
+			     < (IMG_INT) lReqSizeActual )
+			{
+				ui32CreateFreeSpace = 5 * (psTmp->ui32Size / 100);
+				if (ui32CreateFreeSpace < lReqSizeActual)
+				{
+					ui32CreateFreeSpace = lReqSizeActual;
+				}
+
+				while (ui32CreateFreeSpace > (IMG_UINT32)circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size))
+				{
+					pui8IncrRead = &psTmp->pbyBuffer[ui32LRead];
+					ui32LRead += (sizeof(PVRSRVTL_PACKETHDR) + PVRSRVTL_ALIGN( GET_PACKET_DATA_LEN(pui8IncrRead) ));
+
+					/* Check if buffer needs to wrap */
+					if (ui32LRead >= psTmp->ui32Size)
+					{
+						ui32LRead = 0;
+					}
+				}
+				psTmp->ui32Read = ui32LRead;
+				pui8IncrRead = &psTmp->pbyBuffer[psTmp->ui32Read];
+
+				pHdr = GET_PACKET_HDR(pui8IncrRead);
+				DoTLSetPacketHeader(pHdr, SET_PACKETS_DROPPED(pHdr));
+			}
+			/* else fall through as there is enough space now to write the data */
+
+			OSLockRelease(psTmp->hReadLock);
+			/* If we accepted a flag var set the OVERWRITE bit*/
+			if (pui32Flags) *pui32Flags |= TL_FLAG_OVERWRITE_DETECTED;
+		}
+		/* No data overwriting, insert write_failed flag and return */
+		else if (psTmp->eOpMode == TL_OPMODE_DROP_NEWER)
+		{
+			/* Caller should not try to use ppui8Data,
+			 * NULLify to give user a chance of avoiding memory corruption */
+			*ppui8Data = NULL;
+
+			/* This flag should not be inserted two consecutive times, so
+			 * check the last ui32 in case it was a packet drop packet. */
+			pui32Buf = ui32LWrite
+					  ?
+					    (void *)&psTmp->pbyBuffer[ui32LWrite - sizeof(PVRSRVTL_PACKETHDR)]
+					   : // Previous four bytes are not guaranteed to be a packet header...
+					    (void *)&psTmp->pbyBuffer[psTmp->ui32Size - PVRSRVTL_PACKET_ALIGNMENT];
+
+			pHdr = GET_PACKET_HDR(pui32Buf);
+			if ( PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED
+				 !=
+				 GET_PACKET_TYPE( pHdr ) && (ui32InputFlags & TL_FLAG_NO_WRITE_FAILED) == 0)
+			{
+				/* Insert size-stamped packet header */
+				pui32Buf = (void *)&psTmp->pbyBuffer[ui32LWrite];
+				pHdr = GET_PACKET_HDR(pui32Buf);
+				DoTLSetPacketHeader(pHdr, PVRSRVTL_SET_PACKET_WRITE_FAILED);
+				ui32LWrite += sizeof(PVRSRVTL_PACKETHDR);
+				ui32LWrite %= psTmp->ui32Size;
+				iFreeSpace -= sizeof(PVRSRVTL_PACKETHDR);
+			}
+
+			OSLockAcquire (psTmp->hStreamWLock);
+			psTmp->ui32Write = ui32LWrite;
+			psTmp->ui32Pending = NOTHING_PENDING;
+			OSLockRelease (psTmp->hStreamWLock);
+
+			if (pui32AvSpace)
+			{
+				*pui32AvSpace = suggestAllocSize(ui32LRead, ui32LWrite, psTmp->ui32Size, ui32ReqSizeMin, psTmp->ui32MaxPacketSize);
+			}
+
+			/* Inform call of permanent stream misuse, no space left,
+			 * the size of the stream will need to be increased. */
+			if (psTmp->bNoWrapPermanent)
+			{
+				PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE);
+			}
+
+			PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_FULL);
+		}
+	}
+
+	/* The easy case: buffer has enough space to hold the requested packet (data + header) */
+
+	/* Should we treat the buffer as non-circular buffer? */
+	if (psTmp->bNoWrapPermanent)
+	{
+		iFreeSpace = bufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+	else
+	{
+		iFreeSpace = circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+
+	if (iFreeSpace >= (IMG_INT) lReqSizeActual)
+	{
+		if (pad)
+		{
+			/* Inserting padding packet. */
+			pui32Buf = (void *)&psTmp->pbyBuffer[ui32LWrite];
+			pHdr = GET_PACKET_HDR(pui32Buf);
+			DoTLSetPacketHeader(pHdr,
+				PVRSRVTL_SET_PACKET_PADDING(pad-sizeof(PVRSRVTL_PACKETHDR)));
+
+			/* CAUTION: the used pad value should always result in a properly
+			 *          aligned ui32LWrite pointer, which in this case is 0 */
+			ui32LWrite = (ui32LWrite + pad) % psTmp->ui32Size;
+			/* Detect unaligned pad value */
+			PVR_ASSERT(ui32LWrite == 0);
+		}
+		/* Insert size-stamped packet header */
+		pui32Buf = (void *) &psTmp->pbyBuffer[ui32LWrite];
+
+		pHdr = GET_PACKET_HDR(pui32Buf);
+		DoTLSetPacketHeader(pHdr,
+			PVRSRVTL_SET_PACKET_HDR(ui32ReqSize, ePacketType));
+
+		/* return the next position in the buffer to the user */
+		*ppui8Data = &psTmp->pbyBuffer[ ui32LWrite+sizeof(PVRSRVTL_PACKETHDR) ];
+
+		/* update pending offset: size stamp + data */
+		ui32LPending = lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR);
+	}
+	else
+	{
+		OSLockAcquire (psTmp->hStreamWLock);
+		psTmp->ui32Pending = NOTHING_PENDING;
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire stream lock for updating stream parameters */
+	OSLockAcquire (psTmp->hStreamWLock);
+	psTmp->ui32Write = ui32LWrite;
+	psTmp->ui32Pending = ui32LPending;
+	OSLockRelease (psTmp->hStreamWLock);
+
+#if defined(TL_BUFFER_STATS)
+	TL_COUNTER_INC(psTmp->ui32CntNumWriteSuccess);
+#endif
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamReserve(IMG_HANDLE hStream,
+				IMG_UINT8 **ppui8Data,
+				IMG_UINT32 ui32Size)
+{
+	return DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32Size, PVRSRVTL_PACKETTYPE_DATA, NULL, NULL);
+}
+
+PVRSRV_ERROR
+TLStreamReserve2(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size,
+                IMG_UINT32 ui32SizeMin,
+                IMG_UINT32* pui32Available,
+                IMG_BOOL* pbIsReaderConnected)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32SizeMin, PVRSRVTL_PACKETTYPE_DATA, pui32Available, NULL);
+	if (eError != PVRSRV_OK && pbIsReaderConnected != NULL)
+	{
+		*pbIsReaderConnected = TLStreamIsOpenForReading(hStream);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+TLStreamReserveReturnFlags(IMG_HANDLE hStream,
+						   IMG_UINT8  **ppui8Data,
+						   IMG_UINT32 ui32Size,
+						   IMG_UINT32* pui32Flags)
+{
+	return DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32Size, PVRSRVTL_PACKETTYPE_DATA, NULL, pui32Flags);
+}
+
+PVRSRV_ERROR
+TLStreamCommit(IMG_HANDLE hStream, IMG_UINT32 ui32ReqSize)
+{
+	PTL_STREAM psTmp;
+	IMG_UINT32 ui32LRead, ui32OldWrite, ui32LWrite, ui32LPending;
+	PVRSRV_ERROR eError;
+
+#if defined(TL_BUFFER_STATS)
+	IMG_UINT32 ui32UnreadBytes;
+#endif
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Get a local copy of the stream buffer parameters */
+	ui32LRead = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+	ui32LPending = psTmp->ui32Pending;
+
+	ui32OldWrite = ui32LWrite;
+
+	// Space in buffer is aligned
+	ui32ReqSize = PVRSRVTL_ALIGN(ui32ReqSize) + sizeof(PVRSRVTL_PACKETHDR);
+
+	/* Check pending reserver and ReqSize + packet header size. */
+	if ((ui32LPending == NOTHING_PENDING) || (ui32ReqSize > ui32LPending))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_MISUSE);
+	}
+
+	/* Update pointer to written data. */
+	ui32LWrite = (ui32LWrite + ui32ReqSize) % psTmp->ui32Size;
+
+	/* and reset LPending to 0 since data are now submitted */
+	ui32LPending = NOTHING_PENDING;
+
+#if defined(TL_BUFFER_STATS)
+	/* Calculate new number of bytes unread */
+	if (ui32LWrite > ui32LRead)
+	{
+		ui32UnreadBytes = (ui32LWrite-ui32LRead);
+	}
+	else if (ui32LWrite < ui32LRead)
+	{
+		ui32UnreadBytes = (psTmp->ui32Size-ui32LRead+ui32LWrite);
+	}
+	else
+	{ /* else equal, ignore */
+		ui32UnreadBytes = 0;
+	}
+
+	/* Calculate high water mark for debug purposes */
+	if (ui32UnreadBytes > psTmp->ui32BufferUt)
+	{
+		psTmp->ui32BufferUt = ui32UnreadBytes;
+	}
+#endif
+
+	/* Memory barrier required to ensure prior data written by writer is
+	 * flushed from WC buffer to main memory. */
+	OSWriteMemoryBarrier(NULL);
+
+	/* Acquire stream lock to ensure other context(s) (if any)
+	 * wait on the lock (in DoTLStreamReserve) for consistent values
+	 * of write offset and pending value */
+	OSLockAcquire (psTmp->hStreamWLock);
+
+	/* Update stream buffer parameters to match local copies */
+	psTmp->ui32Write = ui32LWrite;
+	psTmp->ui32Pending = ui32LPending;
+
+	/* Ensure write pointer is flushed */
+	OSWriteMemoryBarrier(&psTmp->ui32Write);
+
+	TL_COUNTER_ADD(psTmp->ui32ProducerByteCount, ui32ReqSize);
+	TL_COUNTER_INC(psTmp->ui32NumCommits);
+
+#if defined(TL_BUFFER_STATS)
+	/* IF there has been no-reader since first reserve on an empty-buffer,
+	 * AND current utilisation is considerably high (90%), calculate the
+	 * time taken to fill up the buffer */
+	if ((OSAtomicRead(&psTmp->bNoReaderSinceFirstReserve) == 1) &&
+	    (TLStreamGetUT(psTmp) >= 90 * psTmp->ui32Size/100))
+	{
+		IMG_UINT32 ui32TimeToFullInUs = OSClockus() - psTmp->ui32TimeStart;
+		if (psTmp->ui32MinTimeToFullInUs > ui32TimeToFullInUs)
+		{
+			psTmp->ui32MinTimeToFullInUs = ui32TimeToFullInUs;
+		}
+		/* Following write ensures ui32MinTimeToFullInUs doesn't lose its
+		 * real (expected) value in case there is no reader until next Commit call */
+		OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 0);
+	}
+#endif
+
+	if (!psTmp->bNoSignalOnCommit)
+	{
+		/* If we have transitioned from an empty buffer to a non-empty buffer, we
+		 * must signal possibly waiting consumer. BUT, let the signal be "deferred"
+		 * until buffer is at least 'ui32ThresholdUsageForSignal' bytes full. This
+		 * avoids a race between OSEventObjectSignal and OSEventObjectWaitTimeout
+		 * (in TLServerAcquireDataKM), where a "signal" might happen before "wait",
+		 * resulting into signal being lost and stream-reader waiting even though
+		 * buffer is no-more empty */
+		if (ui32OldWrite == ui32LRead)
+		{
+			psTmp->bSignalPending = IMG_TRUE;
+		}
+
+		if (psTmp->bSignalPending && (TLStreamGetUT(psTmp) >= psTmp->ui32ThresholdUsageForSignal))
+		{
+			TL_COUNTER_INC(psTmp->ui32SignalsSent);
+			psTmp->bSignalPending = IMG_FALSE;
+
+			/* Signal consumers that may be waiting */
+			eError = OSEventObjectSignal(psTmp->psNode->hReadEventObj);
+			if (eError != PVRSRV_OK)
+			{
+				OSLockRelease (psTmp->hStreamWLock);
+				PVR_DPF_RETURN_RC(eError);
+			}
+		}
+		else
+		{
+			TL_COUNTER_INC(psTmp->ui32SignalNotSent);
+		}
+	}
+	OSLockRelease (psTmp->hStreamWLock);
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamWrite(IMG_HANDLE hStream, IMG_UINT8 *pui8Src, IMG_UINT32 ui32Size)
+{
+	IMG_BYTE *pbyDest = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = TLStreamReserve(hStream, &pbyDest, ui32Size);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	else
+	{
+		OSDeviceMemCopy((void*)pbyDest, (void*)pui8Src, ui32Size);
+		eError = TLStreamCommit(hStream, ui32Size);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF_RETURN_RC(eError);
+		}
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamWriteRetFlags(IMG_HANDLE hStream, IMG_UINT8 *pui8Src, IMG_UINT32 ui32Size, IMG_UINT32 *pui32Flags){
+	IMG_BYTE *pbyDest = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = TLStreamReserveReturnFlags(hStream, &pbyDest, ui32Size, pui32Flags);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	else
+	{
+		OSDeviceMemCopy((void*)pbyDest, (void*)pui8Src, ui32Size);
+		eError = TLStreamCommit(hStream, ui32Size);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF_RETURN_RC(eError);
+		}
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+void TLStreamInfo(IMG_HANDLE hStream, PTL_STREAM_INFO psInfo)
+{
+	IMG_DEVMEM_SIZE_T actual_req_size;
+	IMG_DEVMEM_ALIGN_T align = 4; /* Low fake value so the real value can be obtained */
+
+	actual_req_size = 2;
+	/* ignore error as OSGetPageShift() should always return correct value */
+	(void) DevmemExportalignAdjustSizeAndAlign(OSGetPageShift(), &actual_req_size, &align);
+
+	psInfo->headerSize = sizeof(PVRSRVTL_PACKETHDR);
+	psInfo->minReservationSize = sizeof(IMG_UINT32);
+	psInfo->pageSize = (IMG_UINT32)(actual_req_size);
+	psInfo->pageAlign = (IMG_UINT32)(align);
+	psInfo->maxTLpacketSize = ((PTL_STREAM)hStream)->ui32MaxPacketSize;
+}
+
+PVRSRV_ERROR
+TLStreamMarkEOS(IMG_HANDLE psStream, IMG_BOOL bRemoveOld)
+{
+	PTL_STREAM   psTmp;
+	PVRSRV_ERROR eError;
+	IMG_UINT8*   pData;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psTmp = (PTL_STREAM)psStream;
+
+	/* Do not support EOS packets on permanent stream buffers at present,
+	 * EOS is best used with streams where data is consumed. */
+	if (psTmp->bNoWrapPermanent)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_MISUSE);
+	}
+
+	if (bRemoveOld)
+	{
+		eError = DoTLStreamReserve(psStream, &pData, 0, 0, PVRSRVTL_PACKETTYPE_MARKER_EOS_REMOVEOLD, NULL, NULL);
+	}
+	else
+	{
+		eError = DoTLStreamReserve(psStream, &pData, 0, 0, PVRSRVTL_PACKETTYPE_MARKER_EOS, NULL, NULL);
+	}
+
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	PVR_DPF_RETURN_RC(TLStreamCommit(psStream, 0));
+}
+
+
+static PVRSRV_ERROR
+_TLStreamMarkOC(IMG_HANDLE hStream, PVRSRVTL_PACKETTYPE ePacketType)
+{
+	PVRSRV_ERROR eError;
+	PTL_STREAM psStream = hStream;
+	IMG_UINT32 ui32Size;
+	IMG_UINT8 *pData;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	if (NULL == psStream->psNotifStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_NOTIF_STREAM);
+	}
+
+	ui32Size = OSStringLength(psStream->szName) + 1;
+
+	eError = DoTLStreamReserve(psStream->psNotifStream, &pData, ui32Size,
+	                           ui32Size, ePacketType, NULL, NULL);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	OSDeviceMemCopy(pData, psStream->szName, ui32Size);
+
+	PVR_DPF_RETURN_RC(TLStreamCommit(psStream->psNotifStream, ui32Size));
+}
+
+PVRSRV_ERROR
+TLStreamMarkStreamOpen(IMG_HANDLE psStream)
+{
+	return _TLStreamMarkOC(psStream, PVRSRVTL_PACKETTYPE_STREAM_OPEN_FOR_WRITE);
+}
+
+PVRSRV_ERROR
+TLStreamMarkStreamClose(IMG_HANDLE psStream)
+{
+	return _TLStreamMarkOC(psStream, PVRSRVTL_PACKETTYPE_STREAM_CLOSE_FOR_WRITE);
+}
+
+PVRSRV_ERROR
+TLStreamSync(IMG_HANDLE psStream)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PTL_STREAM   psTmp;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)psStream;
+
+	/* If read client exists and has opened stream in blocking mode,
+	 * signal when data is available to read. */
+	if (psTmp->psNode->psRDesc &&
+		 (!(psTmp->psNode->psRDesc->ui32Flags & PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING)) &&
+			psTmp->ui32Read != psTmp->ui32Write)
+	{
+		TL_COUNTER_INC(psTmp->ui32ManSyncs);
+		eError = OSEventObjectSignal(psTmp->psNode->hReadEventObj);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+IMG_BOOL
+TLStreamIsOpenForReading(IMG_HANDLE hStream)
+{
+	PTL_STREAM psTmp;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+
+	PVR_DPF_RETURN_VAL(psTmp->psNode->psRDesc != NULL);
+}
+
+IMG_BOOL
+TLStreamOutOfData(IMG_HANDLE hStream)
+{
+	PTL_STREAM psTmp;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+
+	/* If both pointers are equal then the buffer is empty */
+	PVR_DPF_RETURN_VAL(psTmp->ui32Read == psTmp->ui32Write);
+}
+
+
+PVRSRV_ERROR
+TLStreamResetProducerByteCount(IMG_HANDLE hStream, IMG_UINT32 ui32Value)
+{
+	PTL_STREAM   psTmp;
+	IMG_UINT32   ui32LRead, ui32LWrite;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+	ui32LRead = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+
+	if (ui32LRead != ui32LWrite)
+	{
+		eErr = PVRSRV_ERROR_STREAM_MISUSE;
+	}
+#if defined(TL_BUFFER_STATS)
+	psTmp->ui32ProducerByteCount = ui32Value;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+#endif
+	PVR_DPF_RETURN_RC(eErr);
+}
+/*
+ * Internal stream APIs to server part of Transport Layer, declared in
+ * header tlintern.h. Direct pointers to stream objects are used here as
+ * these functions are internal.
+ */
+IMG_UINT32
+TLStreamAcquireReadPos(PTL_STREAM psStream,
+                       IMG_BOOL bDisableCallback,
+                       IMG_UINT32* puiReadOffset)
+{
+	IMG_UINT32 uiReadLen = 0;
+	IMG_UINT32 ui32LRead, ui32LWrite;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+	PVR_ASSERT(puiReadOffset);
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		if (!OSTryLockAcquire(psStream->hReadLock))
+		{
+			/*
+			 * This is a normal event when the system is under load.
+			 * An example of how to produce this is to run testrunner /
+			 * regression/ddk_test_seq2_host_fw_mem.conf with HTB / pvrhtbd
+			 * configured as
+			 *
+			 * # pvrdebug -log trace -loggroups main,pow,debug \
+			 * -hostloggroups main,ctrl,sync,brg -hostlogtype dropoldest
+			 *
+			 * # pvrhtbd -hostloggroups main,ctrl,sync,brg
+			 *
+			 * We will see a small number of these collisions but as this is
+			 * an expected calling path, and an expected return code, we drop
+			 * the severity to just be a debug MESSAGE instead of WARNING
+			 */
+			PVR_DPF((PVR_DBG_MESSAGE,
+			    "%s: Read lock on stream '%s' is acquired by some writer, "
+			    "hence reader failed to acquire read lock.", __func__,
+			    psStream->szName));
+#if defined(TL_BUFFER_STATS)
+			TL_COUNTER_INC(psStream->ui32CntReadFails);
+#endif
+			PVR_DPF_RETURN_VAL(0);
+		}
+	}
+
+#if defined(TL_BUFFER_STATS)
+	TL_COUNTER_INC(psStream->ui32CntReadSuccesses);
+#endif
+
+	/* Grab a local copy */
+	ui32LRead = psStream->ui32Read;
+	ui32LWrite = psStream->ui32Write;
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		psStream->bReadPending = IMG_TRUE;
+		OSLockRelease(psStream->hReadLock);
+	}
+
+	/* No data available and CB defined - try and get data */
+	if ((ui32LRead == ui32LWrite) && psStream->pfProducerCallback && !bDisableCallback)
+	{
+		PVRSRV_ERROR eRc;
+		IMG_UINT32   ui32Resp = 0;
+
+		eRc = ((TL_STREAM_SOURCECB)psStream->pfProducerCallback)(psStream, TL_SOURCECB_OP_CLIENT_EOS,
+				&ui32Resp, psStream->pvProducerUserData);
+		PVR_LOG_IF_ERROR(eRc, "TLStream->pfProducerCallback");
+
+		ui32LWrite = psStream->ui32Write;
+	}
+
+	/* No data available... */
+	if (ui32LRead == ui32LWrite)
+	{
+		if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+		{
+			psStream->bReadPending = IMG_FALSE;
+		}
+		PVR_DPF_RETURN_VAL(0);
+	}
+
+#if defined(TL_BUFFER_STATS)
+	/* The moment reader knows it will see a non-zero data, it marks its presence in writer's eyes */
+	OSAtomicWrite (&psStream->bNoReaderSinceFirstReserve, 0);
+#endif
+
+	/* Data is available to read... */
+	*puiReadOffset = ui32LRead;
+
+	/*PVR_DPF((PVR_DBG_VERBOSE,
+	 *		"TLStreamAcquireReadPos Start before: Write:%d, Read:%d, size:%d",
+	 *		ui32LWrite, ui32LRead, psStream->ui32Size));
+	 */
+
+	if (ui32LRead > ui32LWrite)
+	{	/* CB has wrapped around. */
+		PVR_ASSERT(!psStream->bNoWrapPermanent);
+		/* Return the first contiguous piece of memory, ie [ReadLen,EndOfBuffer]
+		 * and let a subsequent AcquireReadPos read the rest of the Buffer */
+		/*PVR_DPF((PVR_DBG_VERBOSE, "TLStreamAcquireReadPos buffer has wrapped"));*/
+		uiReadLen = psStream->ui32Size - ui32LRead;
+		TL_COUNTER_INC(psStream->ui32AcquireRead2);
+	}
+	else
+	{	/* CB has not wrapped */
+		uiReadLen = ui32LWrite - ui32LRead;
+		TL_COUNTER_INC(psStream->ui32AcquireRead1);
+	}
+
+	PVR_DPF_RETURN_VAL(uiReadLen);
+}
+
+PVRSRV_ERROR
+TLStreamAdvanceReadPos(PTL_STREAM psStream,
+                       IMG_UINT32 uiReadLen,
+                       IMG_UINT32 uiOrigReadLen)
+{
+	IMG_UINT32 uiNewReadPos;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	/*
+	 * This API does not use Read lock as 'bReadPending' is sufficient
+	 * to keep Read index safe by preventing a write from updating the
+	 * index and 'bReadPending' itself is safe as it can only be modified
+	 * by readers and there can be only one reader in action at a time.
+	 */
+
+	/* Update the read offset by the length provided in a circular manner.
+	 * Assuming the update to be atomic hence, avoiding use of locks
+	 */
+	uiNewReadPos = (psStream->ui32Read + uiReadLen) % psStream->ui32Size;
+
+	/* Must validate length is on a packet boundary, for
+	 * TLReleaseDataLess calls.
+	 */
+	if (uiReadLen != uiOrigReadLen) /* buffer not empty */
+	{
+		PVRSRVTL_PPACKETHDR psHdr = GET_PACKET_HDR(psStream->pbyBuffer+uiNewReadPos);
+		PVRSRVTL_PACKETTYPE eType = GET_PACKET_TYPE(psHdr);
+
+		if ((psHdr->uiReserved != PVRSRVTL_PACKETHDR_RESERVED) ||
+			(eType == PVRSRVTL_PACKETTYPE_UNDEF) ||
+			(eType >= PVRSRVTL_PACKETTYPE_LAST))
+		{
+			PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_ALIGNMENT);
+		}
+		/* else OK, on a packet boundary */
+	}
+	/* else no check needed */
+
+	psStream->ui32Read = uiNewReadPos;
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		psStream->bReadPending = IMG_FALSE;
+	}
+
+	/* notify reserves that may be pending */
+	/* The producer event object is used to signal the StreamReserve if the TL
+	 * Buffer is in blocking mode and is full.
+	 * Previously this event was only signalled if the buffer was created in
+	 * blocking mode. Since the buffer mode can now change dynamically the event
+	 * is signalled every time to avoid any potential race where the signal is
+	 * required, but not produced.
+	 */
+	{
+		PVRSRV_ERROR eError;
+		eError = OSEventObjectSignal(psStream->hProducerEventObj);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "Error in TLStreamAdvanceReadPos: OSEventObjectSignal returned:%u",
+					 eError));
+			/* We've failed to notify the producer event. This means there may
+			 * be a delay in generating more data to be consumed until the next
+			 * Write() generating action occurs.
+			 */
+		}
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE,
+			 "TLStreamAdvanceReadPos Read now at: %d",
+			psStream->ui32Read));
+	PVR_DPF_RETURN_OK;
+}
+
+void
+TLStreamResetReadPos(PTL_STREAM psStream)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	if (psStream->bNoWrapPermanent)
+	{
+
+		/* Update the read offset by the length provided in a circular manner.
+		 * Assuming the update to be atomic hence, avoiding use of locks */
+		psStream->ui32Read = 0;
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+				 "TLStreamResetReadPos Read now at: %d",
+					psStream->ui32Read));
+	}
+	else
+	{
+		/* else for other stream types this is a no-op */
+		PVR_DPF((PVR_DBG_VERBOSE,
+				"No need to reset read position of circular tlstream"));
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void
+TLStreamDestroy (PTL_STREAM psStream)
+{
+	PVR_ASSERT (psStream);
+
+	OSLockDestroy (psStream->hStreamWLock);
+	OSLockDestroy (psStream->hReadLock);
+
+	OSEventObjectClose(psStream->hProducerEvent);
+	OSEventObjectDestroy(psStream->hProducerEventObj);
+
+	TLFreeSharedMem(psStream);
+	OSFreeMem(psStream);
+}
+
+DEVMEM_MEMDESC*
+TLStreamGetBufferPointer(PTL_STREAM psStream)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	PVR_DPF_RETURN_VAL(psStream->psStreamMemDesc);
+}
+
+/*
+ * Determine the maximum transfer size which will fit into the specified
+ * L2 consumer stream.
+ * This is a point-in-time snapshot which compares the given requested transfer
+ * size and returns the minimum of that vs the available buffer space within
+ * the L2 stream. This is the largest amount of data that can be successfully
+ * copied in the stream without truncation occurring.
+ * Worst-case scenario is that we copy fewer bytes than the maximum
+ * actually available, but we will never copy too much data.
+ */
+IMG_UINT32
+TLStreamGetMaxTransfer(IMG_UINT32 uiXferSize, IMG_HANDLE hConsumerStream)
+{
+	PTL_STREAM  psConsumer = (PTL_STREAM)hConsumerStream;
+	IMG_UINT32  ui32MaxTransfer = 0U;
+
+	IMG_UINT32  ui32Xfer;
+
+	/* Local copies */
+	IMG_UINT32	ui32Read = psConsumer->ui32Read;
+	IMG_UINT32	ui32Write = psConsumer->ui32Write;
+
+	if (ui32Write >= ui32Read)
+	{
+		/* Can transfer Write .. End-of-buffer + Read bytes at start */
+		ui32Xfer = psConsumer->ui32Size - ui32Write + ui32Read;
+	}
+	else
+	{
+		/* Can transfer Read - Write bytes maximum */
+		ui32Xfer = ui32Read - ui32Write;
+	}
+
+	PVR_ASSERT(ui32Xfer <= psConsumer->ui32Size);
+
+	ui32MaxTransfer = MIN(ui32Xfer, uiXferSize);
+
+	return ui32MaxTransfer;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_client.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_client.c
new file mode 100644
index 000000000000..c989cea789ea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_client.c
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File			vmm_pvz_client.c
+@Title          VM manager client para-virtualization
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides VMM client para-virtualization APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#include "vmm_impl.h"
+#include "vz_vmm_pvz.h"
+#include "vmm_pvz_client.h"
+
+
+static inline void
+PvzClientLockAcquire(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockAcquire(psPVRSRVData->hPvzConnectionLock);
+}
+
+static inline void
+PvzClientLockRelease(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockRelease(psPVRSRVData->hPvzConnectionLock);
+}
+
+/*
+ * ===========================================================
+ *  The following client para-virtualization (pvz) functions
+ *  are exclusively called by guests to initiate a pvz call
+ *  to the host via hypervisor (guest -> vm manager -> host)
+ * ===========================================================
+ */
+
+PVRSRV_ERROR
+PvzClientMapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_PHYADDR sDevPAddr;
+	VMM_PVZ_CONNECTION *psVmmPvz;
+	PHYS_HEAP *psFwPhysHeap = psDevConfig->psDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+
+	eError = PhysHeapGetDevPAddr(psFwPhysHeap, &sDevPAddr);
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+	{
+		IMG_DEV_PHYADDR sDevPAddrTranslated;
+
+		/* If required, perform a software translation between CPU and Device physical addresses. */
+		PhysHeapCpuPAddrToDevPAddr(psFwPhysHeap, 1, &sDevPAddrTranslated, (IMG_CPU_PHYADDR *)&sDevPAddr);
+		sDevPAddr.uiAddr = sDevPAddrTranslated.uiAddr;
+	}
+#endif
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapGetDevPAddr");
+	PVR_LOG_RETURN_IF_FALSE((sDevPAddr.uiAddr != 0), "PhysHeapGetDevPAddr", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psVmmPvz = PvzConnectionAcquire();
+	PvzClientLockAcquire();
+
+	eError = psVmmPvz->sClientFuncTab.pfnMapDevPhysHeap(RGX_FIRMWARE_RAW_HEAP_SIZE, sDevPAddr.uiAddr);
+
+	PvzClientLockRelease();
+	PvzConnectionRelease(psVmmPvz);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzClientUnmapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	VMM_PVZ_CONNECTION *psVmmPvz = PvzConnectionAcquire();
+	PVR_ASSERT(psVmmPvz);
+
+	PvzClientLockAcquire();
+
+	PVR_ASSERT(psVmmPvz->sClientFuncTab.pfnUnmapDevPhysHeap);
+
+	eError = psVmmPvz->sClientFuncTab.pfnUnmapDevPhysHeap();
+
+	PvzClientLockRelease();
+	PvzConnectionRelease(psVmmPvz);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (vmm_pvz_client.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_server.c
new file mode 100644
index 000000000000..7e67d397a908
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vmm_pvz_server.c
@@ -0,0 +1,251 @@
+/*************************************************************************/ /*!
+@File			vmm_pvz_server.c
+@Title          VM manager server para-virtualization handlers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides VMM server para-virtz handler APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxfwutils.h"
+
+#include "vz_vm.h"
+#include "vmm_impl.h"
+#include "vz_vmm_pvz.h"
+#include "vmm_pvz_server.h"
+
+static inline void
+PvzServerLockAcquire(void)
+{
+#if !defined(FPGA)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockAcquire(psPVRSRVData->hPvzConnectionLock);
+#endif
+}
+
+static inline void
+PvzServerLockRelease(void)
+{
+#if !defined(FPGA)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockRelease(psPVRSRVData->hPvzConnectionLock);
+#endif
+}
+
+#define VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID) do {							\
+	if ((ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED) ||								\
+		(ui32DriverID < RGXFW_GUEST_DRIVER_ID_START))								\
+	{																				\
+		PVR_DPF((PVR_DBG_ERROR,														\
+					"%s: Invalid OSID %u. Supported Guest OSID range: %u - %u",		\
+					__func__,														\
+					ui32DriverID,													\
+					RGXFW_GUEST_DRIVER_ID_START,									\
+					RGX_NUM_DRIVERS_SUPPORTED-1));									\
+		return PVRSRV_ERROR_INVALID_PARAMS;											\
+	}																				\
+	if (PVRSRVGetDeviceInstance(ui32DevID) == NULL)									\
+	{																				\
+		PVR_DPF((PVR_DBG_ERROR,														\
+					"%s: Invalid Device ID %u.",									\
+					__func__,														\
+					ui32DevID));													\
+		return PVRSRV_ERROR_INVALID_PARAMS;											\
+	}																				\
+} while (false);
+
+/*
+ * ===========================================================
+ *  The following server para-virtualization (pvz) functions
+ *  are exclusively called by the VM manager (hypervisor) on
+ *  behalf of guests to complete guest pvz calls
+ *  (guest -> vm manager -> host)
+ * ===========================================================
+ */
+
+PVRSRV_ERROR
+PvzServerMapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						IMG_UINT32 ui32DevID,
+						IMG_UINT64 ui64Size,
+						IMG_UINT64 ui64PAddr)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && !defined(FPGA)
+		/*
+		 * Reject hypercall if called on a system configured at build time to
+		 * preallocate the Guest's firmware heaps from static carveout memory.
+		 */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Host PVZ config: Does not match with Guest PVZ config."
+		         " Host preallocates the Guest's FW physheap from static memory carveouts at startup.", __func__));
+		return PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+#else
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+
+	PvzServerLockAcquire();
+
+#if defined(SUPPORT_RGX)
+	if (IsVmOnline(ui32DriverID, ui32DevID))
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstance(ui32DevID);
+		IMG_DEV_PHYADDR sDevPAddr = {ui64PAddr};
+		IMG_UINT32 sync;
+
+		eError = RGXFwRawHeapAllocMap(psDeviceNode, ui32DriverID, sDevPAddr, ui64Size);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", e0);
+
+		/* Invalidate MMU cache in preparation for a kick from this Guest */
+		eError = psDeviceNode->pfnMMUCacheInvalidateKick(psDeviceNode, &sync);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMUCacheInvalidateKick", e0);
+
+		/* Everything is ready for the firmware to start interacting with this OS */
+		eError = RGXFWSetFwOsState(psDeviceNode->pvDevice, ui32DriverID, RGXFWIF_OS_ONLINE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Paravirtualized request to map Guest%u's Firmware heap"
+								" rejected: Guest not allowed to run.", __func__, ui32DriverID));
+		eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+	}
+e0:
+#endif /* defined(SUPPORT_RGX) */
+	PvzServerLockRelease();
+
+	return eError;
+#endif
+}
+
+PVRSRV_ERROR
+PvzServerUnmapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						  IMG_UINT32 ui32DevID)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && !defined(FPGA)
+		/*
+		 * Reject hypercall if called on a system configured at built time to
+		 * preallocate the Guest's firmware heaps from static carveout memory.
+		 */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Host PVZ config: Does not match with Guest PVZ config."
+		         " Host preallocates the Guest's FW physheap from static memory carveouts at startup.", __func__));
+		return PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+#else
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+
+	PvzServerLockAcquire();
+
+#if defined(SUPPORT_RGX)
+	if (IsVmOnline(ui32DriverID, ui32DevID))
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+		/* Order firmware to offload this OS' data and stop accepting commands from it */
+		eError = RGXFWSetFwOsState(psDeviceNode->pvDevice, ui32DriverID, RGXFWIF_OS_OFFLINE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXFWSetFwOsState", e0);
+
+		/* it is now safe to remove the Guest's memory mappings */
+		RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+	}
+e0:
+#endif
+
+	PvzServerLockRelease();
+
+	return eError;
+#endif
+}
+
+/*
+ * ============================================================
+ *  The following server para-virtualization (pvz) functions
+ *  are exclusively called by the VM manager (hypervisor) to
+ *  pass side band information to the host (vm manager -> host)
+ * ============================================================
+ */
+
+PVRSRV_ERROR
+PvzServerOnVmOnline(IMG_UINT32 ui32DriverID,
+					IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzOnVmOnline(ui32DriverID, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzServerOnVmOffline(IMG_UINT32 ui32DriverID,
+					 IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzOnVmOffline(ui32DriverID, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzServerVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+					  IMG_UINT32 ui32ParamValue,
+					  IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(RGXFW_GUEST_DRIVER_ID_START, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzVMMConfigure(eVMMParamType, ui32ParamValue, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (vmm_pvz_server.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_pvz.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_pvz.c
new file mode 100644
index 000000000000..bb7c8f6426c6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_pvz.c
@@ -0,0 +1,201 @@
+/*************************************************************************/ /*!
+@File           vz_vmm_pvz.c
+@Title          VM manager para-virtualization APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    VM manager para-virtualization management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "vz_vmm_pvz.h"
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR
+PvzConnectionValidate(void)
+{
+	VMM_PVZ_CONNECTION *psVmmPvz;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/*
+	 * Acquire the underlying VM manager PVZ connection & validate it.
+	 */
+	psVmmPvz = PvzConnectionAcquire();
+	if (psVmmPvz == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Unable to acquire PVZ connection", __func__));
+		eError = PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+		goto e0;
+	}
+
+	/* Log which PVZ setup type is being used by driver */
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/*
+	 *  Static PVZ bootstrap setup
+	 *
+	 *  This setup uses carve-out memory, has no hypercall mechanism & does not support
+	 *  out-of-order initialisation of host/guest VMs/drivers. The host driver has all
+	 *  the information needed to initialize all Drivers firmware state when it's loaded
+	 *  and its PVZ layer must mark all guest Drivers as being online as part of its PVZ
+	 *  initialisation. Having no out-of-order initialisation support, the guest driver
+	 *  can only submit a workload to the device after the host driver has completely
+	 *  initialized the firmware, the VZ hypervisor/VM setup must guarantee this.
+	 */
+	PVR_LOG(("Using static PVZ bootstrap setup"));
+#else
+	/*
+	 *  Dynamic PVZ bootstrap setup
+	 *
+	 *  This setup uses guest memory, has PVZ hypercall mechanism & supports out-of-order
+	 *  initialisation of host/guest VMs/drivers. The host driver initializes only its
+	 *  own Driver-0 firmware state when its loaded and each guest driver will use its PVZ
+	 *  interface to hypercall to the host driver to both synchronise its initialisation
+	 *  so it does not submit any workload to the firmware before the host driver has
+	 *  had a chance to initialize the firmware and to also initialize its own Driver-x
+	 *  firmware state.
+	 */
+	PVR_LOG(("Using dynamic PVZ bootstrap setup"));
+#endif
+
+	PvzConnectionRelease(psVmmPvz);
+e0:
+	return eError;
+}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+PVRSRV_ERROR PvzConnectionInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+
+#if (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_DPF((PVR_DBG_ERROR, "This kernel driver does not support virtualization. Please rebuild with RGX_NUM_DRIVERS_SUPPORTED > 1"));
+	eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	goto e0;
+#else
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if ((psPVRSRVData->hPvzConnection != NULL) &&
+		(psPVRSRVData->hPvzConnectionLock != NULL))
+	{
+		eError = PVRSRV_OK;
+		PVR_DPF((PVR_DBG_MESSAGE, "PVzConnection already initialised."));
+		goto e0;
+	}
+
+	/* Create para-virtualization connection lock */
+	eError = OSLockCreate(&psPVRSRVData->hPvzConnectionLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	/* Create VM manager para-virtualization connection */
+	eError = VMMCreatePvzConnection((VMM_PVZ_CONNECTION **)&psPVRSRVData->hPvzConnection, psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		OSLockDestroy(psPVRSRVData->hPvzConnectionLock);
+		psPVRSRVData->hPvzConnectionLock = NULL;
+
+		PVR_LOG_ERROR(eError, "VMMCreatePvzConnection");
+		goto e0;
+	}
+
+	/* Ensure pvz connection is configured correctly */
+	eError = PvzConnectionValidate();
+	PVR_LOG_RETURN_IF_ERROR(eError, "PvzConnectionValidate");
+#endif
+e0:
+	return eError;
+}
+
+void PvzConnectionDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDN;
+	IMG_BOOL bCanDestroyPvzData = IMG_TRUE;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	for (psDN = psPVRSRVData->psDeviceNodeList; psDN != NULL; psDN = psDN->psNext)
+	{
+		if ((psDN->psDevConfig != psDevConfig) &&
+			(!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDN)))
+		{
+			/* if any other virtual devices are present keep the pvz data */
+			bCanDestroyPvzData = IMG_FALSE;
+			break;
+		}
+	}
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (bCanDestroyPvzData)
+	{
+		if ((psPVRSRVData->hPvzConnection == NULL) &&
+			(psPVRSRVData->hPvzConnectionLock == NULL))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "PVzConnection already deinitialised."));
+			return;
+		}
+
+		VMMDestroyPvzConnection(psPVRSRVData->hPvzConnection, psDevConfig);
+		psPVRSRVData->hPvzConnection = NULL;
+
+		OSLockDestroy(psPVRSRVData->hPvzConnectionLock);
+		psPVRSRVData->hPvzConnectionLock = NULL;
+	}
+}
+
+VMM_PVZ_CONNECTION* PvzConnectionAcquire(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVR_ASSERT(psPVRSRVData->hPvzConnection != NULL);
+	return psPVRSRVData->hPvzConnection;
+}
+
+void PvzConnectionRelease(VMM_PVZ_CONNECTION *psParaVz)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	/* Nothing to do, just validate the pointer we're passed back */
+	PVR_ASSERT(psParaVz == psPVRSRVData->hPvzConnection);
+}
+
+/******************************************************************************
+ End of file (vz_vmm_pvz.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_vm.c b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_vm.c
new file mode 100644
index 000000000000..bc737f5acd81
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/common/vz_vmm_vm.c
@@ -0,0 +1,294 @@
+/*************************************************************************/ /*!
+@File			vz_vmm_vm.c
+@Title          System virtualization VM support APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System virtualization VM support functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv.h"
+#include "pvrsrv_error.h"
+#include "vz_vm.h"
+#include "rgxfwutils.h"
+#include "rgxfwdbg.h"
+
+bool IsVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		return false;
+	}
+	else
+	{
+		return BIT_ISSET(psDevNode->ui32VmState, ui32DriverID);
+	}
+}
+
+PVRSRV_ERROR PvzOnVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVRSRV_ERROR       eError          = PVRSRV_ERROR_INVALID_PARAMS;
+#else
+	PVRSRV_ERROR       eError          = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+	else if (BIT_ISSET(psDevNode->ui32VmState, ui32DriverID))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: DriverID %u on Device %u is already enabled.",
+				 __func__, ui32DriverID, ui32DevID));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_CREATED)
+	{
+		/* Firmware not initialized yet, do it here */
+		eError = PVRSRVCommonDeviceInitialise(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: failed to initialize firmware (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+	}
+
+	eError = RGXFWHealthCheckCmd(psDevNode->pvDevice);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	BIT_SET(psDevNode->ui32VmState, ui32DriverID);
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* Everything is ready for the firmware to start interacting with this OS */
+	eError = RGXFWSetFwOsState(psDevNode->pvDevice, ui32DriverID, RGXFWIF_OS_ONLINE);
+#endif
+
+e0:
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PvzOnVmOffline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVRSRV_ERROR       eError          = PVRSRV_ERROR_INVALID_PARAMS;
+#else
+	PVRSRV_ERROR      eError          = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+	else if (!BIT_ISSET(psDevNode->ui32VmState, ui32DriverID))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: DriverID %u on Device %u is already disabled.",
+				 __func__, ui32DriverID, ui32DevID));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	psDevInfo = psDevNode->pvDevice;
+	if (psDevInfo == NULL)
+	{
+		eError = PVRSRV_ERROR_INVALID_DEVINFO;
+		goto e0;
+	}
+
+	eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+	if (eError == PVRSRV_OK)
+	{
+		BIT_UNSET(psDevNode->ui32VmState, ui32DriverID);
+	}
+
+e0:
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PvzVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+							 IMG_UINT32 ui32ParamValue,
+							 IMG_UINT32 ui32DevID)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+
+	psDevInfo = psDevNode->pvDevice;
+	if (psDevInfo == NULL)
+	{
+		eError = PVRSRV_ERROR_INVALID_DEVINFO;
+		goto e0;
+	}
+
+	switch (eVMMParamType)
+	{
+		case VMM_CONF_PRIO_DRV0:
+		case VMM_CONF_PRIO_DRV1:
+		case VMM_CONF_PRIO_DRV2:
+		case VMM_CONF_PRIO_DRV3:
+		case VMM_CONF_PRIO_DRV4:
+		case VMM_CONF_PRIO_DRV5:
+		case VMM_CONF_PRIO_DRV6:
+		case VMM_CONF_PRIO_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType;
+			IMG_UINT32 ui32Prio = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverPriorityKM(NULL, psDevNode, ui32DriverID, ui32Prio);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_HCS_DEADLINE:
+		{
+			IMG_UINT32 ui32HCSDeadline = ui32ParamValue;
+			eError = RGXFWSetHCSDeadline(psDevInfo, ui32HCSDeadline);
+			break;
+		}
+		case VMM_CONF_ISOLATION_GROUP_DRV0:
+		case VMM_CONF_ISOLATION_GROUP_DRV1:
+		case VMM_CONF_ISOLATION_GROUP_DRV2:
+		case VMM_CONF_ISOLATION_GROUP_DRV3:
+		case VMM_CONF_ISOLATION_GROUP_DRV4:
+		case VMM_CONF_ISOLATION_GROUP_DRV5:
+		case VMM_CONF_ISOLATION_GROUP_DRV6:
+		case VMM_CONF_ISOLATION_GROUP_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType - VMM_CONF_ISOLATION_GROUP_DRV0;
+			IMG_UINT32 ui32IsolationGroup = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverIsolationGroupKM(NULL, psDevNode, ui32DriverID, ui32IsolationGroup);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_TIME_SLICE_DRV0:
+		case VMM_CONF_TIME_SLICE_DRV1:
+		case VMM_CONF_TIME_SLICE_DRV2:
+		case VMM_CONF_TIME_SLICE_DRV3:
+		case VMM_CONF_TIME_SLICE_DRV4:
+		case VMM_CONF_TIME_SLICE_DRV5:
+		case VMM_CONF_TIME_SLICE_DRV6:
+		case VMM_CONF_TIME_SLICE_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType - VMM_CONF_TIME_SLICE_DRV0;
+			IMG_UINT32 ui32TSPercentage = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverTimeSliceKM(NULL, psDevNode, ui32DriverID, ui32TSPercentage);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_TIME_SLICE_INTERVAL:
+		{
+			IMG_UINT32 ui32TSIntervalMs = ui32ParamValue;
+
+			eError = PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(NULL, psDevNode, ui32TSIntervalMs);
+			break;
+		}
+		case VMM_CONF_VZ_CONNECTION_COOLDOWN_PERIOD:
+		{
+			IMG_UINT32 ui32VzConnectionCooldownPeriodInSec = ui32ParamValue;
+			eError = RGXFWSetVzConnectionCooldownPeriod(psDevInfo, ui32VzConnectionCooldownPeriodInSec);
+			break;
+		}
+		default:
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+e0:
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(eVMMParamType);
+	PVR_UNREFERENCED_PARAMETER(ui32ParamValue);
+	PVR_UNREFERENCED_PARAMETER(ui32DevID);
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+}
+
+/******************************************************************************
+ End of file (vz_vmm_vm.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.c
new file mode 100644
index 000000000000..1de4f08dd2b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.c
@@ -0,0 +1,166 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR device dependent bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements device dependent PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_bridge_init.h"
+#include "rgxdevice.h"
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+PVRSRV_ERROR InitRGXRAYBridge(void);
+void DeinitRGXRAYBridge(void);
+#endif
+
+/* Reference counts for device-conditional
+ * bridges. This ensures that bridges remain
+ * valid while there are still devices using
+ * them.
+ */
+static ATOMIC_T i32RGXCMPBridgeRefCt;
+static ATOMIC_T i32RGXTQ2BridgeRefCt;
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+static ATOMIC_T i32RGXRayBridgeRefCt;
+#endif
+static IMG_BOOL bAtomicsInitialised = IMG_FALSE;
+
+void RGXBridgeDriverInit(void)
+{
+	if (!bAtomicsInitialised)
+	{
+		bAtomicsInitialised = IMG_TRUE;
+		OSAtomicWrite(&i32RGXCMPBridgeRefCt, 0);
+		OSAtomicWrite(&i32RGXTQ2BridgeRefCt, 0);
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+		OSAtomicWrite(&i32RGXRayBridgeRefCt, 0);
+#endif
+	}
+}
+
+PVRSRV_ERROR RGXRegisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	if (!bAtomicsInitialised)
+	{
+		eError = PVRSRV_ERROR_NOT_INITIALISED;
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXBridgeRefCts");
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		if (OSAtomicIncrement(&i32RGXCMPBridgeRefCt) == 1)
+		{
+			eError = InitRGXCMPBridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXCMPBridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXCMPBridge");
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		if (OSAtomicIncrement(&i32RGXTQ2BridgeRefCt) == 1)
+		{
+			eError = InitRGXTQ2Bridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXTQ2BridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXTQ2Bridge");
+		}
+	}
+
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 0)
+	{
+		if (OSAtomicIncrement(&i32RGXRayBridgeRefCt) == 1)
+		{
+			eError = InitRGXRAYBridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXRayBridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXRAYBridge");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+void RGXUnregisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		if (OSAtomicDecrement(&i32RGXCMPBridgeRefCt) == 0)
+		{
+			DeinitRGXCMPBridge();
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		if (OSAtomicDecrement(&i32RGXTQ2BridgeRefCt) == 0)
+		{
+			DeinitRGXTQ2Bridge();
+		}
+	}
+
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 0)
+	{
+		if (OSAtomicDecrement(&i32RGXRayBridgeRefCt) == 0)
+		{
+			DeinitRGXRAYBridge();
+		}
+	}
+#endif
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.h
new file mode 100644
index 000000000000..8622848b51b1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_bridge_init.h
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR device dependent bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements device dependent PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_BRIDGE_INIT_H)
+#define RGX_BRIDGE_INIT_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "device.h"
+#include "rgxdevice.h"
+
+void RGXBridgeDriverInit(void);
+PVRSRV_ERROR RGXRegisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo);
+void RGXUnregisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#endif /* RGX_BRIDGE_INIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_heaps_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_heaps_server.h
new file mode 100644
index 000000000000..a1dd31113fa3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgx_heaps_server.h
@@ -0,0 +1,86 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap (server) definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if !defined RGX_HEAPS_SERVER_H
+#define RGX_HEAPS_SERVER_H
+
+#include "img_types.h"
+#include "rgx_heaps.h"
+
+/*
+ *  Supported log2 page size values for RGX_GENERAL_NON_4K_HEAP_ID
+ */
+#define RGX_HEAP_PAGE_SHIFTS_DEF \
+	X(4KB, 12U) \
+	X(16KB, 14U) \
+	X(64KB, 16U) \
+	X(256KB, 18U) \
+	X(1MB, 20U) \
+	X(2MB, 21U)
+
+typedef enum RGX_HEAP_PAGE_SHIFTS_TAG
+{
+#define X(_name, _shift) RGX_HEAP_ ## _name ## _PAGE_SHIFT = _shift,
+	RGX_HEAP_PAGE_SHIFTS_DEF
+#undef X
+} RGX_HEAP_PAGE_SHIFTS;
+
+/* Base and size alignment 2MB */
+#define RGX_HEAP_BASE_SIZE_ALIGN 0x200000UL
+#define RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT 0x8000UL
+
+/*************************************************************************/ /*!
+@Function       RGXGetValidHeapPageSizeMask
+@Description    Returns a bitmask indicating all supported virtual heap page sizes.
+
+@Return         IMG_UINT32  A 32-bit mask with enabled bits indicating valid
+                            page sizes.
+*/ /**************************************************************************/
+static inline IMG_UINT32 RGXGetValidHeapPageSizeMask(void)
+{
+	/* Generates a bit mask with the values of RGX_HEAP_PAGE_SHIFTS_DEF.
+	 * 0 is required for the first shift to properly bitwise OR. */
+#define X(_name, _shift) | (1 << _shift)
+	return 0 RGX_HEAP_PAGE_SHIFTS_DEF;
+#undef X
+}
+
+#endif /* RGX_HEAPS_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.c
new file mode 100644
index 000000000000..902cbe0318d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.c
@@ -0,0 +1,292 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Breakpoint routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Breakpoint routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxbreakpoint.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxmem.h"
+#include "device.h"
+#include "sync_internal.h"
+#include "pdump_km.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR PVRSRVRGXSetBreakpointKM(CONNECTION_DATA    * psConnection,
+                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                      IMG_HANDLE           hMemCtxPrivData,
+                                      RGXFWIF_DM           eFWDataMaster,
+                                      IMG_UINT64           ui64TempSpillingAddr,
+                                      IMG_UINT32           ui32BPAddr,
+                                      IMG_UINT32           ui32HandlerAddr,
+                                      IMG_UINT32           ui32DataMaster)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet)
+	{
+		eError = PVRSRV_ERROR_BP_ALREADY_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPAddr = ui32BPAddr;
+	sBPCmd.uCmdData.sBPData.ui32HandlerAddr = ui32HandlerAddr;
+	sBPCmd.uCmdData.sBPData.ui32BPDM = ui32DataMaster;
+	sBPCmd.uCmdData.sBPData.ui64SpillAddr = ui64TempSpillingAddr;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_WRITE | RGXFWIF_BPDATA_FLAGS_ENABLE;
+	sBPCmd.uCmdData.sBPData.eDM = eFWDataMaster;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          eFWDataMaster,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+	psDevInfo->eBPDM = eFWDataMaster;
+	psDevInfo->bBPSet = IMG_TRUE;
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXClearBreakpointKM(CONNECTION_DATA    * psConnection,
+                                        PVRSRV_DEVICE_NODE * psDeviceNode,
+                                        IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPAddr = 0;
+	sBPCmd.uCmdData.sBPData.ui32HandlerAddr = 0;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_WRITE | RGXFWIF_BPDATA_FLAGS_CTL;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+	psDevInfo->bBPSet = IMG_FALSE;
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXEnableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                         PVRSRV_DEVICE_NODE * psDeviceNode,
+                                         IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet == IMG_FALSE)
+	{
+		eError = PVRSRV_ERROR_BP_NOT_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_CTL | RGXFWIF_BPDATA_FLAGS_ENABLE;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDisableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet == IMG_FALSE)
+	{
+		eError = PVRSRV_ERROR_BP_NOT_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_CTL;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXOverallocateBPRegistersKM(CONNECTION_DATA    * psConnection,
+                                                PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                IMG_UINT32           ui32TempRegs,
+                                                IMG_UINT32           ui32SharedRegs)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_REGS;
+	sBPCmd.uCmdData.sBPData.ui32TempRegs = ui32TempRegs;
+	sBPCmd.uCmdData.sBPData.ui32SharedRegs = ui32SharedRegs;
+	sBPCmd.uCmdData.sBPData.psFWMemContext.ui32Addr = 0U;
+	sBPCmd.uCmdData.sBPData.eDM = RGXFWIF_DM_GP;
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxbreakpoint.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.h
new file mode 100644
index 000000000000..2470de250914
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbreakpoint.h
@@ -0,0 +1,142 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX breakpoint functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX breakpoint functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXBREAKPOINT_H)
+#define RGXBREAKPOINT_H
+
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetBreakpointKM
+
+ @Description
+	Server-side implementation of RGXSetBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input eDataMaster - Data Master to schedule command for
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32BPAddr - Address of breakpoint
+ @Input ui32HandlerAddr - Address of breakpoint handler
+ @Input ui32BPCtl - Breakpoint controls
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetBreakpointKM(CONNECTION_DATA    * psConnection,
+                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                      IMG_HANDLE           hMemCtxPrivData,
+                                      RGXFWIF_DM           eFWDataMaster,
+                                      IMG_UINT64           ui64TempSpillingAddr,
+                                      IMG_UINT32           ui32BPAddr,
+                                      IMG_UINT32           ui32HandlerAddr,
+                                      IMG_UINT32           ui32DataMaster);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXClearBreakpointKM
+
+ @Description
+	Server-side implementation of RGXClearBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXClearBreakpointKM(CONNECTION_DATA    * psConnection,
+                                        PVRSRV_DEVICE_NODE * psDeviceNode,
+                                        IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXEnableBreakpointKM
+
+ @Description
+	Server-side implementation of RGXEnableBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXEnableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                         PVRSRV_DEVICE_NODE * psDeviceNode,
+                                         IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDisableBreakpointKM
+
+ @Description
+	Server-side implementation of RGXDisableBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDisableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXOverallocateBPRegistersKM
+
+ @Description
+	Server-side implementation of RGXOverallocateBPRegisters
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui32TempRegs - Number of temporary registers to overallocate
+ @Input ui32SharedRegs - Number of shared registers to overallocate
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXOverallocateBPRegistersKM(CONNECTION_DATA    * psConnection,
+                                                PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                IMG_UINT32           ui32TempRegs,
+                                                IMG_UINT32           ui32SharedRegs);
+#endif /* RGXBREAKPOINT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.c
new file mode 100644
index 000000000000..9f82ff1260bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.c
@@ -0,0 +1,960 @@
+/*************************************************************************/ /*!
+@File
+@Title          BVNC handling specific routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Functions used for BVNC related work
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxbvnc.h"
+#define RGXBVNC_C
+#include "rgx_bvnc_table_km.h"
+#undef RGXBVNC_C
+#include "os_apphint.h"
+#include "pvrsrv.h"
+#include "pdump_km.h"
+#include "rgx_compat_bvnc.h"
+#include "allocmem.h"
+
+#define RGX_FEATURE_TRUE_VALUE_TYPE_UINT16 (RGX_FEATURE_VALUE_TYPE_UINT16 >> RGX_FEATURE_TYPE_BIT_SHIFT)
+#define RGX_FEATURE_TRUE_VALUE_TYPE_UINT32 (RGX_FEATURE_VALUE_TYPE_UINT32 >> RGX_FEATURE_TYPE_BIT_SHIFT)
+#define RGXBVNC_BUFFER_SIZE (((PVRSRV_MAX_DEVICES)*(RGX_BVNC_STR_SIZE_MAX))+1)
+
+/* This function searches the given array for a given search value */
+static IMG_UINT64* _RGXSearchBVNCTable( IMG_UINT64 *pui64Array,
+								IMG_UINT uiEnd,
+								IMG_UINT64 ui64SearchValue,
+								IMG_UINT uiColCount)
+{
+	IMG_UINT uiStart = 0, index;
+	IMG_UINT64 value, *pui64Ptr = NULL;
+
+	while (uiStart < uiEnd)
+	{
+		index = (uiStart + uiEnd)/2;
+		pui64Ptr = pui64Array + (index * uiColCount);
+		value = *(pui64Ptr);
+
+		if (value == ui64SearchValue)
+		{
+			return pui64Ptr;
+		}
+
+		if (value > ui64SearchValue)
+		{
+			uiEnd = index;
+		}else
+		{
+			uiStart = index + 1;
+		}
+	}
+	return NULL;
+}
+#define RGX_SEARCH_BVNC_TABLE(t, b) (_RGXSearchBVNCTable((IMG_UINT64*)(t), \
+                                ARRAY_SIZE(t), (b), \
+                                sizeof((t)[0])/sizeof(IMG_UINT64)) )
+
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@brief		This function reads the (P)BVNC core_ID register and extracts
+			the BVNC configuration. Supports the old scheme and the newer
+			PBVNC scheme.
+@param		psDeviceNode - Device Node pointer
+@param		ui32CoreNum  - Core/bank number (0 for single core)
+@param		pB           - Address of branch value (output)
+@param		pV           - Address of version value (output)
+@param		pN           - Address of number of clusters/scalable shading units value (output)
+@param		pC           - Address of configuration value (output)
+@return		BVNC encoded in 64-bit value, 16-bits per field
+*/ /**************************************************************************/
+static
+IMG_UINT64 _RGXReadBVNCFromReg(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32CoreNum,
+							   IMG_UINT32 *pB, IMG_UINT32 *pV, IMG_UINT32 *pN, IMG_UINT32 *pC)
+{
+	IMG_UINT64 ui64BVNC;
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_CORE_ID__PBVNC)
+	/* Core ID reading code for Rogue */
+
+	/* Read the BVNC, in to new way first, if B not set, use old scheme */
+	ui64BVNC = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID__PBVNC + (ui32CoreNum << 16));
+
+	if (GET_B(ui64BVNC))
+	{
+		B = GET_PBVNC_B(ui64BVNC);
+		V = GET_PBVNC_V(ui64BVNC);
+		N = GET_PBVNC_N(ui64BVNC);
+		C = GET_PBVNC_C(ui64BVNC);
+	}
+	else
+	{
+		IMG_UINT64 ui32CoreID, ui32CoreRev;
+		ui32CoreRev = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_REVISION + (ui32CoreNum << 16));
+		ui32CoreID = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID + (ui32CoreNum << 16));
+		B = (ui32CoreRev & ~RGX_CR_CORE_REVISION_MAJOR_CLRMSK) >>
+												RGX_CR_CORE_REVISION_MAJOR_SHIFT;
+		V = (ui32CoreRev & ~RGX_CR_CORE_REVISION_MINOR_CLRMSK) >>
+												RGX_CR_CORE_REVISION_MINOR_SHIFT;
+		N = (ui32CoreID & ~RGX_CR_CORE_ID_CONFIG_N_CLRMSK) >>
+												RGX_CR_CORE_ID_CONFIG_N_SHIFT;
+		C = (ui32CoreID & ~RGX_CR_CORE_ID_CONFIG_C_CLRMSK) >>
+												RGX_CR_CORE_ID_CONFIG_C_SHIFT;
+		ui64BVNC = rgx_bvnc_pack(B, V, N, C);
+	}
+#else
+	/* Core ID reading code for Volcanic */
+
+	ui64BVNC = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID + (ui32CoreNum << 16));
+
+	B = (ui64BVNC & ~RGX_CR_CORE_ID_BRANCH_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_BRANCH_ID_SHIFT;
+	V = (ui64BVNC & ~RGX_CR_CORE_ID_VERSION_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_VERSION_ID_SHIFT;
+	N = (ui64BVNC & ~RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_CLRMSK) >>
+											RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_SHIFT;
+	C = (ui64BVNC & ~RGX_CR_CORE_ID_CONFIG_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_CONFIG_ID_SHIFT;
+#endif
+
+	*pB = B; *pV = V; *pN = N; *pC = C;
+	return ui64BVNC;
+}
+#endif
+
+#if defined(DEBUG) || defined(SUPPORT_PERFORMANCE_RUN)
+
+#define PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, szShortName, Feature)															\
+	if ( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] != RGX_FEATURE_VALUE_DISABLED )			\
+		{ PVR_LOG(("%s %d", szShortName, psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX])); }		\
+	else																\
+		{ PVR_LOG(("%s N/A", szShortName)); }
+
+static void _RGXBvncDumpParsedConfig(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT64 ui64Mask = 0, ui32IdOrNameIdx = 1;
+
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NC:       ", NUM_CLUSTERS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "CSF:      ", CDM_CONTROL_STREAM_FORMAT);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "FBCDCA:   ", FBCDC_ARCHITECTURE);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "META:     ", META);
+#if defined(RGX_FEATURE_META_COREMEM_BANKS_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MCMB:     ", META_COREMEM_BANKS);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MCMS:     ", META_COREMEM_SIZE);
+#if defined(RGX_FEATURE_META_DMA_CHANNEL_COUNT_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MDMACnt:  ", META_DMA_CHANNEL_COUNT);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NIIP:     ", NUM_ISP_IPP_PIPES);
+#if defined(RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NIPS:     ", NUM_ISP_PER_SPU);
+#endif
+#if defined(RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "PPS:      ", PBE_PER_SPU);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NSPU:     ", NUM_SPU);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "PBW:      ", PHYS_BUS_WIDTH);
+#if defined(RGX_FEATURE_SCALABLE_TE_ARCH_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "STEArch:  ", SCALABLE_TE_ARCH);
+#endif
+#if defined(RGX_FEATURE_SCALABLE_VCE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SVCEA:    ", SCALABLE_VCE);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SLCBanks: ", SLC_BANKS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SLCCLS:   ", SLC_CACHE_LINE_SIZE_BITS);
+	PVR_LOG(("SLCSize:   %d",  psDevInfo->sDevFeatureCfg.ui32SLCSizeInBytes));
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "VASB:     ", VIRTUAL_ADDRESS_SPACE_BITS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NOSIDS:   ", NUM_OSIDS);
+
+#if defined(FEATURE_NO_VALUES_NAMES_MAX_IDX)
+	/* Dump the features with no values */
+	ui64Mask = psDevInfo->sDevFeatureCfg.ui64Features;
+	while (ui64Mask)
+	{
+		if (ui64Mask & 0x01)
+		{
+			if (ui32IdOrNameIdx <= FEATURE_NO_VALUES_NAMES_MAX_IDX)
+			{
+				PVR_LOG(("%s", gaszFeaturesNoValuesNames[ui32IdOrNameIdx - 1]));
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				         "Feature with Mask doesn't exist: 0x%016" IMG_UINT64_FMTSPECx,
+				         ((IMG_UINT64)1 << (ui32IdOrNameIdx - 1))));
+			}
+		}
+		ui64Mask >>= 1;
+		ui32IdOrNameIdx++;
+	}
+#endif
+
+#if defined(ERNSBRNS_IDS_MAX_IDX)
+	/* Dump the ERN and BRN flags for this core */
+	ui64Mask = psDevInfo->sDevFeatureCfg.ui64ErnsBrns;
+	ui32IdOrNameIdx = 1;
+
+	while (ui64Mask)
+	{
+		if (ui64Mask & 0x1)
+		{
+			if (ui32IdOrNameIdx <= ERNSBRNS_IDS_MAX_IDX)
+			{
+				PVR_LOG(("ERN/BRN : %d", gaui64ErnsBrnsIDs[ui32IdOrNameIdx - 1]));
+			}
+			else
+			{
+				PVR_LOG(("Unknown ErnBrn bit: 0x%0" IMG_UINT64_FMTSPECx, ((IMG_UINT64)1 << (ui32IdOrNameIdx - 1))));
+			}
+		}
+		ui64Mask >>= 1;
+		ui32IdOrNameIdx++;
+	}
+#endif
+
+#if !defined(ERNSBRNS_IDS_MAX_IDX) && !defined(FEATURE_NO_VALUES_NAMES_MAX_IDX)
+	PVR_UNREFERENCED_PARAMETER(ui64Mask);
+	PVR_UNREFERENCED_PARAMETER(ui32IdOrNameIdx);
+#endif
+
+}
+#endif
+
+static PVRSRV_ERROR _RGXBvncParseFeatureValues(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                               IMG_UINT64 *pui64Cfg)
+{
+	IMG_UINT32 ui32Index;
+
+	/* Read the feature values for the runtime BVNC */
+	for (ui32Index = 0; ui32Index < RGX_FEATURE_WITH_VALUES_MAX_IDX; ui32Index++)
+	{
+		IMG_UINT16 ui16BitPosition = aui16FeaturesWithValuesBitPositions[ui32Index];
+		IMG_UINT64 ui64PackedValues = pui64Cfg[2 + ui16BitPosition / 64];
+		IMG_UINT16 ui16ValueIndex = (ui64PackedValues & aui64FeaturesWithValuesBitMasks[ui32Index]) >> (ui16BitPosition % 64);
+
+		if (ui16ValueIndex >= gaFeaturesValuesMaxIndexes[ui32Index])
+		{
+			/* This case should never be reached */
+			psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] = RGX_FEATURE_VALUE_INVALID;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Feature with index (%d) decoded wrong value index (%d)", __func__, ui32Index, ui16ValueIndex));
+			PVR_ASSERT(ui16ValueIndex < gaFeaturesValuesMaxIndexes[ui32Index]);
+			return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+
+		switch (ui16BitPosition >> RGX_FEATURE_TYPE_BIT_SHIFT)
+		{
+			case RGX_FEATURE_TRUE_VALUE_TYPE_UINT16:
+			{
+				IMG_UINT16 *pui16FeatureValues = (IMG_UINT16*)gaFeaturesValues[ui32Index];
+				if (pui16FeatureValues[ui16ValueIndex] == (IMG_UINT16)RGX_FEATURE_VALUE_DISABLED)
+				{
+					psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+						RGX_FEATURE_VALUE_DISABLED;
+				}
+				else
+				{
+					psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+						pui16FeatureValues[ui16ValueIndex];
+				}
+				break;
+			}
+			case RGX_FEATURE_TRUE_VALUE_TYPE_UINT32:
+				psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+					((IMG_UINT32*)gaFeaturesValues[ui32Index])[ui16ValueIndex];
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Feature with index %d has invalid feature type",
+				         __func__,
+				         ui32Index));
+				return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+	}
+
+
+#if defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX)
+	/* Code path for Volcanic */
+
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = RGXFWIF_DM_CDM+1;
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_RAY+1);
+	}
+#if defined(SUPPORT_AGP)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM2+1);
+#if defined(SUPPORT_AGP4)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM4+1);
+#endif
+#endif
+
+	/* Get the max number of dusts in the core */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS))
+	{
+		RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) == 1)
+		{
+			/* per SPU power island */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = MAX(1, (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) / 2));
+		}
+		else if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) >= 2)
+		{
+			/* per Cluster power island */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS);
+		}
+		else
+		{
+			/* All volcanic cores support power islanding */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Power island feature version not found!", __func__));
+			PVR_ASSERT(0);
+			return PVRSRV_ERROR_FEATURE_DISABLED;
+		}
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+			RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+		{
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RT_RAC_PER_SPU))
+			{
+				psDevInfo->sDevFeatureCfg.ui32MAXRACCount = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU);
+			}
+			else
+			{
+				psDevInfo->sDevFeatureCfg.ui32MAXRACCount = 1;
+			}
+		}
+	}
+	else
+	{
+		/* This case should never be reached as all cores have clusters */
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of clusters feature value missing!", __func__));
+		PVR_ASSERT(0);
+		return PVRSRV_ERROR_FEATURE_DISABLED;
+	}
+#else /* defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX) */
+	/* Code path for Rogue */
+
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = RGXFWIF_DM_CDM+1;
+#if defined(SUPPORT_AGP)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM2+1);
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_META_IDX] = RGX_FEATURE_VALUE_DISABLED;
+	}
+
+	/* Get the max number of dusts in the core */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS))
+	{
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = MAX(1, (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) / 2));
+	}
+	else
+	{
+		/* This case should never be reached as all cores have clusters */
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of clusters feature value missing!", __func__));
+		PVR_ASSERT(0);
+		return PVRSRV_ERROR_FEATURE_DISABLED;
+	}
+#endif /* defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX) */
+
+	/* Transform the META coremem size info in bytes */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_META_COREMEM_SIZE_IDX] *= 1024;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RGXBvncAcquireAppHint(IMG_CHAR *pszBVNC, const IMG_UINT32 ui32RGXDevCount)
+{
+	const IMG_CHAR *pszAppHintDefault = PVRSRV_APPHINT_RGXBVNC;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32BVNCCount = 0;
+	IMG_BOOL bRet;
+	IMG_CHAR *pszBVNCAppHint;
+	IMG_CHAR *pszCurrentBVNC;
+	pszBVNCAppHint = (IMG_CHAR *)OSAllocMem(RGXBVNC_BUFFER_SIZE);
+	if (pszBVNCAppHint == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pszBVNCAppHint[0] = '\0';
+
+	pszCurrentBVNC = pszBVNCAppHint;
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bRet = (IMG_BOOL)OSGetAppHintSTRING(APPHINT_NO_DEVICE,
+						pvAppHintState,
+						RGXBVNC,
+						pszAppHintDefault,
+						pszBVNCAppHint,
+						RGXBVNC_BUFFER_SIZE
+	);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	if (!bRet || (pszBVNCAppHint[0] == '\0'))
+	{
+		OSFreeMem(pszBVNCAppHint);
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC module param list: %s",__func__, pszBVNCAppHint));
+
+	while (*pszCurrentBVNC != '\0')
+	{
+		IMG_CHAR *pszNext = pszCurrentBVNC;
+
+		if (ui32BVNCCount >= PVRSRV_MAX_DEVICES)
+		{
+			break;
+		}
+
+		while (1)
+		{
+			if (*pszNext == ',')
+			{
+				pszNext[0] = '\0';
+				pszNext++;
+				break;
+			} else if (*pszNext == '\0')
+			{
+				break;
+			}
+			pszNext++;
+		}
+
+		if (ui32BVNCCount == ui32RGXDevCount)
+		{
+			OSStringSafeCopy(pszBVNC, pszCurrentBVNC, RGX_BVNC_STR_SIZE_MAX);
+			OSFreeMem(pszBVNCAppHint);
+			return PVRSRV_OK;
+		}
+
+		ui32BVNCCount++;
+		pszCurrentBVNC = pszNext;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Given module parameters list is shorter than "
+	"number of actual devices", __func__));
+
+	/* If only one BVNC parameter is specified, the same is applied for all RGX
+	 * devices detected */
+	if (1 == ui32BVNCCount)
+	{
+		OSStringSafeCopy(pszBVNC, pszBVNCAppHint, RGX_BVNC_STR_SIZE_MAX);
+	}
+
+	OSFreeMem(pszBVNCAppHint);
+
+	return PVRSRV_OK;
+}
+
+/* Function that parses the BVNC List passed as module parameter */
+static PVRSRV_ERROR _RGXBvncParseList(IMG_UINT32 *pB,
+									  IMG_UINT32 *pV,
+									  IMG_UINT32 *pN,
+									  IMG_UINT32 *pC,
+									  const IMG_UINT32 ui32RGXDevCount)
+{
+	IMG_CHAR aszBVNCString[RGX_BVNC_STR_SIZE_MAX];
+	IMG_CHAR *pcTemp, *pcNext;
+	PVRSRV_ERROR eError;
+
+	aszBVNCString[0] = '\0';
+
+	/* 4 components of a BVNC string is B, V, N & C */
+#define RGX_BVNC_INFO_PARAMS (4)
+
+	eError = _RGXBvncAcquireAppHint(aszBVNCString, ui32RGXDevCount);
+
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	if ('\0' == aszBVNCString[0])
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+
+	/* Parse the given RGX_BVNC string */
+	pcTemp = &aszBVNCString[0];
+	pcNext = strchr(pcTemp, '.');
+	if (pcNext == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+
+	*pcNext = '\0';
+	if (OSStringToUINT32(pcTemp, 0, pB) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	/* remove any 'p' from the V string, as this will
+	 * cause the call to OSStringToUINT32 to fail
+	 */
+	pcNext = strchr(pcTemp, 'p');
+	if (pcNext)
+	{
+		/* found one- - changing to '\0' */
+		*pcNext = '\0';
+		/* Move to next '.' */
+		pcNext++;
+	}
+	else
+	{
+		/* none found, so find next '.' and change to '\0' */
+		pcNext = strchr(pcTemp, '.');
+		if (pcNext == NULL)
+		{
+			return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+		*pcNext = '\0';
+	}
+	if (OSStringToUINT32(pcTemp, 0, pV) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	pcNext = strchr(pcTemp, '.');
+	if (pcNext == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	*pcNext = '\0';
+	if (OSStringToUINT32(pcTemp, 0, pN) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	if (OSStringToUINT32(pcTemp, 0, pC) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	PVR_LOG(("BVNC module parameter honoured: %d.%d.%d.%d", *pB, *pV, *pN, *pC));
+
+	return PVRSRV_OK;
+}
+
+#if !defined(NO_HARDWARE)
+/*
+ * This function obtains the SLCSize from the physical device for GPUs which provide
+ * this information. If the GPU does not provide support we return a value of 0 which will
+ * result in the BVNC supplied definition being used to provide the SLCSize.
+ * Must only be called from driver-live with hardware powered-on.
+ */
+static IMG_UINT32 _RGXBvncReadSLCSize(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT64 ui64SLCSize = 0ULL;
+
+#if defined(RGX_CR_SLC_SIZE_IN_KB)
+	/* Rogue hardware */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_SIZE_CONFIGURABLE))
+	{
+		ui64SLCSize = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SLC_SIZE_IN_KB);
+		if (ui64SLCSize == 0ULL)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Unexpected 0 SLC size. Using default", __func__));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: RGX_CR_SIZE_IN_KB = %u", __func__,
+			        (IMG_UINT32) ui64SLCSize));
+		}
+	}
+#else
+	/* Volcanic hardware */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_SIZE_ADJUSTMENT))
+	{
+		ui64SLCSize = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SLC_STATUS2);
+		ui64SLCSize &= ~RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_CLRMSK;
+		ui64SLCSize >>= RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_SHIFT;
+
+		if (ui64SLCSize == 0ULL)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Unexpected 0 SLC size. Using default", __func__));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: SLC_SIZE_IN_KILOBYTES = %u", __func__,
+			        (IMG_UINT32) ui64SLCSize));
+		}
+	}
+#endif
+
+	return (IMG_UINT32)ui64SLCSize * 1024U;
+}
+#endif /* !defined(NO_HARDWARE) */
+
+/* This function detects the Rogue variant and configures the essential
+ * config info associated with such a device.
+ * The config info includes features, errata, etc
+ */
+PVRSRV_ERROR RGXBvncInitialiseConfiguration(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	static IMG_UINT32 ui32RGXDevCnt = 0;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT64 ui64BVNC=0;
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+	IMG_UINT64 *pui64Cfg = NULL;
+	IMG_UINT32 ui32Cores = 1U;
+	IMG_UINT32 ui32SLCSize = 0;
+
+	/* Check for load time RGX BVNC parameter */
+	eError = _RGXBvncParseList(&B,&V,&N,&C, ui32RGXDevCnt);
+	if (PVRSRV_OK == eError)
+	{
+		PVR_LOG(("Read BVNC " RGX_BVNC_STR_FMTSPEC
+				" from driver load parameter", B, V, N, C));
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+		PVR_LOG_IF_FALSE((pui64Cfg != NULL), "Driver parameter BVNC configuration not found!");
+	}
+
+	{
+		void *pvAppHintState = NULL;
+		const IMG_BOOL bAppHintDefault = PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC;
+
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							IgnoreHWReportedBVNC,
+							&bAppHintDefault,
+							&psDevInfo->bIgnoreHWReportedBVNC);
+		OSFreeAppHintState(pvAppHintState);
+	}
+
+#if !defined(NO_HARDWARE)
+
+	/* Try to detect the RGX BVNC from the HW device */
+	if ((NULL == pui64Cfg) && !psDevInfo->bIgnoreHWReportedBVNC)
+	{
+		IMG_BOOL bPowerDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+		}
+
+		/* Read the BVNC from HW */
+		_RGXReadBVNCFromReg(psDeviceNode, 0 /*core0*/, &B, &V, &N, &C);
+
+		PVR_LOG(("Read BVNC " RGX_BVNC_STR_FMTSPEC
+				" from HW device registers", B, V, N, C));
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Read the number of cores in the system for newer BVNC (Branch ID > 20) */
+			if (B > 20)
+			{
+				ui32Cores = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM);
+			}
+		}
+
+		/* Obtain the SLC size from the device */
+		ui32SLCSize = _RGXBvncReadSLCSize(psDeviceNode);
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: SLC Size reported as %u", __func__, ui32SLCSize));
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		if (ui64BVNC != 0)
+		{
+			pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+			PVR_LOG_IF_FALSE((pui64Cfg != NULL), "HW device BVNC configuration not found!");
+		}
+		else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/*
+			 * On host OS we should not get here as CORE_ID should not be zero, so flag an error.
+			 * On older cores, guest OS only has CORE_ID if defined(RGX_FEATURE_COREID_PER_OS)
+			 */
+			PVR_LOG_ERROR(PVRSRV_ERROR_DEVICE_REGISTER_FAILED, "CORE_ID register returns zero. Unknown BVNC");
+		}
+	}
+#endif
+
+#if defined(RGX_BVNC_KM_B) && defined(RGX_BVNC_KM_N) && defined(RGX_BVNC_KM_C)
+	if (NULL == pui64Cfg)
+	{
+		IMG_CHAR acVStr[5] = RGX_BVNC_KM_V_ST;
+
+		/* We reach here if the HW is not present,
+		 * or we are running in a guest OS with no COREID_PER_OS feature,
+		 * or HW is unstable during register read giving invalid values,
+		 * or runtime detection has been disabled - fall back to compile time BVNC
+		 */
+		B = RGX_BVNC_KM_B;
+		N = RGX_BVNC_KM_N;
+		C = RGX_BVNC_KM_C;
+
+		/* Clear any 'p' that may have been in RGX_BVNC_KM_V_ST,
+		 * as OSStringToUINT32() will otherwise return an error.
+		 */
+		if (acVStr[strlen(acVStr)-1] == 'p')
+		{
+			acVStr[strlen(acVStr)-1] = '\0';
+		}
+
+		if (OSStringToUINT32(&acVStr[0], 0, &V) != PVRSRV_OK)
+		{
+			V = 0;
+		}
+		PVR_LOG(("Reverting to compile time BVNC %s", RGX_BVNC_KM));
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+		PVR_LOG_IF_FALSE((pui64Cfg != NULL), "Compile time BVNC configuration not found!");
+	}
+#endif /* defined(RGX_BVNC) */
+
+	/* Have we failed to identify the BVNC to use? */
+	if (NULL == pui64Cfg)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: BVNC Detection and feature lookup failed. "
+		    "Unsupported BVNC: 0x%016" IMG_UINT64_FMTSPECx, __func__, ui64BVNC));
+		return PVRSRV_ERROR_BVNC_UNSUPPORTED;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC Feature found config: 0x%016"
+	    IMG_UINT64_FMTSPECx " 0x%016" IMG_UINT64_FMTSPECx " 0x%016"
+	    IMG_UINT64_FMTSPECx " 0x%016" IMG_UINT64_FMTSPECx "\n", __func__,
+	    pui64Cfg[0], pui64Cfg[1], pui64Cfg[2], pui64Cfg[3]));
+
+	/* Parsing feature config depends on available features on the core
+	 * hence this parsing should always follow the above feature assignment */
+	psDevInfo->sDevFeatureCfg.ui64Features = pui64Cfg[1];
+	eError = _RGXBvncParseFeatureValues(psDevInfo, pui64Cfg);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Add 'V' to the packed BVNC value to get the BVNC ERN and BRN config. */
+	ui64BVNC = BVNC_PACK(B,V,N,C);
+	pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaErnsBrns, ui64BVNC);
+	if (NULL == pui64Cfg)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: BVNC ERN/BRN lookup failed. "
+		    "Unsupported BVNC: 0x%016" IMG_UINT64_FMTSPECx, __func__, ui64BVNC));
+		psDevInfo->sDevFeatureCfg.ui64ErnsBrns = 0;
+		return PVRSRV_ERROR_BVNC_UNSUPPORTED;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC ERN/BRN Cfg: 0x%016" IMG_UINT64_FMTSPECx
+	    " 0x%016" IMG_UINT64_FMTSPECx, __func__, *pui64Cfg, pui64Cfg[1]));
+	psDevInfo->sDevFeatureCfg.ui64ErnsBrns = pui64Cfg[1];
+
+	psDevInfo->sDevFeatureCfg.ui32B = B;
+	psDevInfo->sDevFeatureCfg.ui32V = V;
+	psDevInfo->sDevFeatureCfg.ui32N = N;
+	psDevInfo->sDevFeatureCfg.ui32C = C;
+
+
+	/*
+	 * Store the SLCSize in the device info field. If 0 it means the device uses the BVNC
+	 * values so grab them here as we've already populated the internal structures.
+	 */
+	if (ui32SLCSize == 0U)
+	{
+		ui32SLCSize = RGX_GET_FEATURE_VALUE(psDevInfo, SLC_SIZE_IN_KILOBYTES) * 1024U;
+
+		/* Verify that we have a valid value returned from the BVNC */
+		PVR_ASSERT(ui32SLCSize != 0U);
+	}
+	psDevInfo->sDevFeatureCfg.ui32SLCSizeInBytes = ui32SLCSize;
+
+	/* Message to confirm configuration look up was a success */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT) &&
+		!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(NO_HARDWARE)
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32Cores);
+			PVR_LOG(("RGX Device registered with BVNC " RGX_BVNC_STR_FMTSPEC,
+					B, V, N, C));
+		}
+#else
+		{
+			PVR_LOG(("RGX Device registered BVNC " RGX_BVNC_STR_FMTSPEC
+					" with %u %s in the system", B ,V ,N ,C, ui32Cores ,
+					((ui32Cores == 1U)?"core":"cores")));
+		}
+#endif
+	}
+	else
+	{
+		PVR_LOG(("RGX Device registered with BVNC " RGX_BVNC_STR_FMTSPEC,
+					B, V, N, C));
+	}
+
+	ui32RGXDevCnt++;
+
+#if defined(DEBUG) || defined(SUPPORT_PERFORMANCE_RUN)
+	_RGXBvncDumpParsedConfig(psDeviceNode);
+#endif
+	return PVRSRV_OK;
+}
+
+/*
+ * This function checks if a particular feature is available on the given rgx device */
+IMG_BOOL RGXBvncCheckFeatureSupported(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64FeatureMask)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->sDevFeatureCfg.ui64Features & ui64FeatureMask)
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+/*
+ * This function returns the value of a feature on the given rgx device */
+IMG_INT32 RGXBvncGetSupportedFeatureValue(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_FEATURE_WITH_VALUE_INDEX eFeatureIndex)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	if (eFeatureIndex >= RGX_FEATURE_WITH_VALUES_MAX_IDX)
+	{
+		return -1;
+	}
+
+	if (psDevInfo->sDevFeatureCfg.ui32FeaturesValues[eFeatureIndex] == RGX_FEATURE_VALUE_DISABLED)
+	{
+		return -1;
+	}
+
+	return psDevInfo->sDevFeatureCfg.ui32FeaturesValues[eFeatureIndex];
+}
+
+/**************************************************************************/ /*!
+@Function       RGXVerifyBVNC
+@Description    Checks that the device's BVNC registers have the correct values.
+@Input          psDeviceNode		Device node
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+#define NUM_RGX_CORE_IDS    8
+PVRSRV_ERROR RGXVerifyBVNC(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT64 ui64MatchBVNC;
+#if !defined(NO_HARDWARE)
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+#endif
+	IMG_UINT32 i;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	/* The device info */
+	psDevInfo = psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "PDUMP VERIFY CORE_ID registers for all OSIDs\n");
+
+	/* construct the value to match against */
+	if ((ui64GivenBVNC | ui64CoreIdMask) == 0) /* both zero means use configured DDK value */
+	{
+		ui64MatchBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+									psDevInfo->sDevFeatureCfg.ui32V,
+									psDevInfo->sDevFeatureCfg.ui32N,
+									psDevInfo->sDevFeatureCfg.ui32C);
+	}
+	else
+	{
+		/* use the value in CORE_ID for any zero elements in the BVNC */
+#if !defined(NO_HARDWARE)
+		IMG_UINT64 ui64BVNC = _RGXReadBVNCFromReg(psDeviceNode, 0, &B, &V, &N, &C);
+		ui64MatchBVNC = (ui64GivenBVNC & ~ui64CoreIdMask) | (ui64BVNC & ui64CoreIdMask);
+#else
+		ui64MatchBVNC = 0;
+#endif
+	}
+	PVR_LOG(("matchBVNC %d.%d.%d.%d",
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_B) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_V) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_N) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_C) & 0xffff)));
+
+	/* read in all the CORE_ID registers */
+	for (i = 0; i < NUM_RGX_CORE_IDS; ++i)
+	{
+#if !defined(NO_HARDWARE)
+		IMG_UINT64 ui64BVNC = _RGXReadBVNCFromReg(psDeviceNode, i, &B, &V, &N, &C);
+		PVR_LOG(("CORE_ID%d returned %d.%d.%d.%d", i, B, V, N, C));
+
+		if (ui64BVNC != ui64MatchBVNC)
+		{
+			eError = PVRSRV_ERROR_BVNC_MISMATCH;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid CORE_ID%d %d.%d.%d.%d, Expected %d.%d.%d.%d", __func__, i,
+					B, V, N, C,
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_B) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_V) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_N) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_C) & 0xffff)));
+			break;
+		}
+#endif
+
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.h
new file mode 100644
index 000000000000..64c418bc4edb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxbvnc.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          BVNC handling specific header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the BVNC related work
+                (see hwdefs/km/rgx_bvnc_table_km.h and
+                hwdefs/km/rgx_bvnc_defs_km.h
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXBVNC_H)
+#define RGXBVNC_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "rgxdevice.h"
+
+/*************************************************************************/ /*!
+@brief		This function detects the Rogue variant and configures the
+			essential config info associated with such a device.
+			The config info includes features, errata, etc
+@param		psDeviceNode - Device Node pointer
+@return		PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXBvncInitialiseConfiguration(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@brief		This function checks if a particular feature is available on
+			the given rgx device
+@param		psDeviceNode - Device Node pointer
+@param		ui64FeatureMask - feature to be checked
+@return		true if feature is supported, false otherwise
+*/ /**************************************************************************/
+IMG_BOOL RGXBvncCheckFeatureSupported(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64FeatureMask);
+
+/*************************************************************************/ /*!
+@brief		This function returns the value of a feature on the given
+			rgx device
+@param		psDeviceNode - Device Node pointer
+@param		ui64FeatureMask - feature for which to return the value
+@return		the value for the specified feature
+*/ /**************************************************************************/
+IMG_INT32 RGXBvncGetSupportedFeatureValue(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_FEATURE_WITH_VALUE_INDEX eFeatureIndex);
+
+/*************************************************************************/ /*!
+@brief		This function validates that the BVNC values in CORE_ID regs are
+			consistent and correct.
+@param		psDeviceNode - Device Node pointer
+@param		GivenBVNC - BVNC to be verified against as supplied by caller
+@param		CoreIdMask - mask of components to pull from CORE_ID register
+@return		success or fail
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXVerifyBVNC(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask);
+
+#endif /* RGXBVNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.c
new file mode 100644
index 000000000000..8c45398a97bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.c
@@ -0,0 +1,2863 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX CCB routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX CCB routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgx_memallocflags.h"
+#include "devicemem_pdump.h"
+#include "dllist.h"
+#if defined(__linux__)
+#include "trace_events.h"
+#endif
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "rgxutils.h"
+#include "info_page.h"
+#include "rgxtimerquery.h"
+
+#if defined(PVRSRV_FORCE_FLUSH_CCCB_ON_KICK)
+#include "cache_km.h"
+#endif
+
+/*
+ *  Uncomment PVRSRV_ENABLE_CCCB_UTILISATION_INFO define for verbose
+ *  info and statistics regarding CCB usage.
+ */
+//#define PVRSRV_ENABLE_CCCB_UTILISATION_INFO
+
+/* Default threshold (as a percentage) for the PVRSRV_ENABLE_CCCB_UTILISATION_INFO feature. */
+#define PVRSRV_ENABLE_CCCB_UTILISATION_INFO_THRESHOLD  (90)
+
+/*
+ * Defines the number of fence updates to record so that future fences in the
+ * CCB. Can be checked to see if they are already known to be satisfied.
+ */
+#define RGX_CCCB_FENCE_UPDATE_LIST_SIZE  (32)
+
+#define RGX_UFO_PTR_ADDR(ufoptr) \
+	(((ufoptr)->puiAddrUFO.ui32Addr) & 0xFFFFFFFC)
+
+#define GET_CCB_SPACE(WOff, ROff, CCBSize) \
+	((((ROff) - (WOff)) + ((CCBSize) - 1)) & ((CCBSize) - 1))
+
+#define UPDATE_CCB_OFFSET(Off, PacketSize, CCBSize) \
+	(Off) = (((Off) + (PacketSize)) & ((CCBSize) - 1))
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_THRESHOLD 0x1
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED 0x2
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_FULL_CCB 0x4
+
+typedef struct _RGX_CLIENT_CCB_UTILISATION_
+{
+	/* the threshold in bytes.
+	 * when the CCB utilisation hits the threshold then we will print
+	 * a warning message.
+	 */
+	IMG_UINT32 ui32ThresholdBytes;
+	/* Maximum cCCB usage at some point in time */
+	IMG_UINT32 ui32HighWaterMark;
+	/* keep track of the warnings already printed.
+	 * bit mask of PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_xyz
+	 */
+	IMG_UINT32 ui32Warnings;
+	/* Keep track how many times CCB was full.
+	 * Counters are reset after every grow.
+	 */
+	IMG_UINT32 ui32CCBFull;
+	IMG_UINT32 ui32CCBAcquired;
+} RGX_CLIENT_CCB_UTILISATION;
+
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+struct _RGX_CLIENT_CCB_ {
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;				/*!< CPU mapping of the CCB control structure used by the fw */
+	void						*pvClientCCB;					/*!< CPU mapping of the CCB */
+	DEVMEM_MEMDESC				*psClientCCBMemDesc;			/*!< MemDesc for the CCB */
+	DEVMEM_MEMDESC				*psClientCCBCtrlMemDesc;		/*!< MemDesc for the CCB control */
+	IMG_UINT32					ui32HostWriteOffset;			/*!< CCB write offset from the driver side */
+	IMG_UINT32					ui32LastPDumpWriteOffset;		/*!< CCB write offset from the last time we submitted a command in capture range */
+	IMG_UINT32					ui32LastROff;					/*!< Last CCB Read offset to help detect any CCB wedge */
+	IMG_UINT32					ui32LastWOff;					/*!< Last CCB Write offset to help detect any CCB wedge */
+	IMG_UINT32					ui32ByteCount;					/*!< Count of the number of bytes written to CCCB */
+	IMG_UINT32					ui32LastByteCount;				/*!< Last value of ui32ByteCount to help detect any CCB wedge */
+	IMG_UINT32					ui32Size;						/*!< Size of the CCB */
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	POS_LOCK					hCCBGrowLock;					/*!< Prevents CCB Grow while DumpCCB() is called and vice versa */
+	IMG_UINT32					ui32VirtualAllocSize;			/*!< Virtual size of the CCB */
+	IMG_UINT32					ui32ChunkSize;					/*!< CCB Sparse allocation chunk size */
+	IMG_PUINT32					pui32MappingTable;				/*!< Mapping table for sparse allocation of the CCB */
+#endif
+	DLLIST_NODE					sNode;							/*!< Node used to store this CCB on the per connection list */
+	PDUMP_CONNECTION_DATA		*psPDumpConnectionData;			/*!< Pointer to the per connection data in which we reside */
+	void						*hTransition;					/*!< Handle for Transition callback */
+	IMG_CHAR					szName[MAX_CLIENT_CCB_NAME];	/*!< Name of this client CCB */
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;			/*!< Parent server common context that this CCB belongs to */
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	RGX_CCB_REQUESTOR_TYPE		eRGXCCBRequestor;
+	RGX_CLIENT_CCB_UTILISATION	sUtilisation;					/*!< CCB utilisation data */
+#endif
+#if defined(DEBUG)
+	IMG_UINT32					ui32UpdateEntries;				/*!< Number of Fence Updates in asFenceUpdateList */
+	RGXFWIF_UFO					asFenceUpdateList[RGX_CCCB_FENCE_UPDATE_LIST_SIZE]; /*!< List of recent updates written in this CCB */
+#endif
+	IMG_UINT32					ui32CCBFlags;                   /*!< Bitmask for various flags relating to CCB. Bit defines in rgxccb.h */
+};
+
+/* Forms a table, with array of strings for each requestor type (listed in RGX_CCB_REQUESTORS X macro), to be used for
+   DevMemAllocation comments and PDump comments. Each tuple in the table consists of 3 strings:
+	{ "FwClientCCB:" <requestor_name>, "FwClientCCBControl:" <requestor_name>, <requestor_name> },
+   The first string being used as comment when allocating ClientCCB for the given requestor, the second for CCBControl
+   structure, and the 3rd one for use in PDUMP comments. The number of tuples in the table must adhere to the following
+   build assert. */
+const IMG_CHAR *const aszCCBRequestors[][3] =
+{
+#define REQUESTOR_STRING(prefix,req) #prefix ":" #req
+#define FORM_REQUESTOR_TUPLE(req) { REQUESTOR_STRING(FwClientCCB,req), REQUESTOR_STRING(FwClientCCBControl,req), #req },
+	RGX_CCB_REQUESTORS(FORM_REQUESTOR_TUPLE)
+#undef FORM_REQUESTOR_TUPLE
+};
+
+PVRSRV_ERROR RGXCCBPDumpDrainCCB(RGX_CLIENT_CCB *psClientCCB,
+						IMG_UINT32 ui32PDumpFlags)
+{
+
+	IMG_UINT32 ui32PollOffset;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+#endif
+
+	/* Command to a finished CCB stream and FW is drained to empty
+	 * out remaining commands until R==W.
+	 */
+	ui32PollOffset = psClientCCB->ui32LastPDumpWriteOffset;
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+						  ui32PDumpFlags,
+						  "cCCB(%s@%p): Draining CCB rgxfw_roff == woff (%d)",
+						  psClientCCB->szName,
+						  psClientCCB,
+						  ui32PollOffset);
+
+	return DevmemPDumpDevmemPol32(psClientCCB->psClientCCBCtrlMemDesc,
+									offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+									ui32PollOffset,
+									0xffffffff,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXCCBPDumpSyncCCB
+
+ PURPOSE	: Synchronise Client CCBs from both live and playback contexts.
+               Waits for live-FW to empty live-CCB.
+               Waits for sim-FW to empty sim-CCB by adding POL
+
+ PARAMETERS	: psClientCCB		- The client CCB
+              ui32PDumpFlags    - PDump flags
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXCCBPDumpSyncCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Wait for the live FW to catch up/empty CCB. This is done by returning
+	 * retry which will get pushed back out to Services client where it
+	 * waits on the event object and then resubmits the command.
+	 */
+	if (psClientCCB->psClientCCBCtrl->ui32ReadOffset != psClientCCB->ui32HostWriteOffset)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Wait for the sim FW to catch up/empty sim CCB.
+	 * We drain whenever capture range is entered, even if no commands
+	 * have been issued on this CCB when out of capture range. We have to
+	 * wait for commands that might have been issued in the last capture
+	 * range to finish so the connection's sync block snapshot dumped after
+	 * all the PDumpTransition callbacks have been execute doesn't clobber
+	 * syncs which the sim FW is currently working on.
+	 *
+	 * Although this is sub-optimal for play-back - while out of capture
+	 * range for every continuous operation we synchronise the sim
+	 * play-back processing the script and the sim FW, there is no easy
+	 * solution. Not all modules that work with syncs register a
+	 * PDumpTransition callback and thus we have no way of knowing if we
+	 * can skip this sim CCB drain and sync block dump or not.
+	 */
+
+	eError = RGXCCBPDumpDrainCCB(psClientCCB, ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "RGXCCBPDumpDrainCCB");
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Live CCB and simulation CCB now empty, FW idle on CCB in both
+	 * contexts.
+	 */
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXCCBPDumpFastForwardCCB
+
+ PURPOSE	: Fast-forward sim-CCB and live-CCB offsets to live app-thread
+              values.
+               This helps to skip any commands submitted when out of capture
+               range and start with first command in capture range in both
+               live and playback contexts. In case of Block mode, this helps
+               to playback any intermediate PDump block directly after first
+               block.
+
+
+ PARAMETERS	: psClientCCB		- The client CCB
+			  ui32PDumpFlags    - PDump flags
+
+ RETURNS	: void
+******************************************************************************/
+static void RGXCCBPDumpFastForwardCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32PDumpFlags)
+{
+	volatile RGXFWIF_CCCB_CTL *psCCBCtl = psClientCCB->psClientCCBCtrl;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+#endif
+
+	/* Make sure that we have synced live-FW and live-App threads */
+	PVR_ASSERT(psCCBCtl->ui32ReadOffset == psClientCCB->ui32HostWriteOffset);
+
+	psCCBCtl->ui32ReadOffset = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32DepOffset = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32WriteOffset = psClientCCB->ui32HostWriteOffset;
+
+	psCCBCtl->ui32ReadOffset2 = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32ReadOffset3 = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32ReadOffset4 = psClientCCB->ui32HostWriteOffset;
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+			ui32PDumpFlags,
+			"cCCB(%s@%p): Fast-forward from %d to %d",
+			psClientCCB->szName,
+			psClientCCB,
+			psClientCCB->ui32LastPDumpWriteOffset,
+			psClientCCB->ui32HostWriteOffset);
+
+	DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+			0,
+			sizeof(RGXFWIF_CCCB_CTL),
+			ui32PDumpFlags);
+
+	/* Although we've entered capture range for this process connection
+	 * we might not do any work on this CCB so update the
+	 * ui32LastPDumpWriteOffset to reflect where we got to for next
+	 * time so we start the drain from where we got to last time.
+	 */
+	psClientCCB->ui32LastPDumpWriteOffset = psClientCCB->ui32HostWriteOffset;
+
+}
+
+static PVRSRV_ERROR _RGXCCBPDumpTransition(void *pvData, void *pvDevice, PDUMP_TRANSITION_EVENT eEvent, IMG_UINT32 ui32PDumpFlags)
+{
+	RGX_CLIENT_CCB *psClientCCB = (RGX_CLIENT_CCB *) pvData;
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) pvDevice;
+#else
+	PVR_UNREFERENCED_PARAMETER(pvDevice);
+#endif
+
+	/* Block mode:
+	 * Here is block structure at transition (ui32BlockLength=N frames):
+	 *
+	 * ...
+	 * ...
+	 * PDUMP_BLOCK_START_0x0000000x{
+	 *   <Fast-forward sim-CCCB>
+	 *   <Re-dump SyncBlocks>
+	 *   ...
+	 *   ...
+	 *   ... (N frames data)
+	 *   ...
+	 *   ...
+	 *   <(1) Drain sim-KCCB>                     ''|
+	 *   <(2) Sync live and sim CCCB>               |
+	 * }PDUMP_BLOCK_END_0x0000000x                  | <- BlockTransition Steps
+	 *   <(3) Split MAIN and BLOCK stream script>   |
+	 * PDUMP_BLOCK_START_0x0000000y{                |
+	 *   <(4) Fast-forward sim-CCCB>                |
+	 *   <(5) Re-dump SyncBlocks>                 ,,|
+	 *   ...
+	 *   ...
+	 *   ... (N frames data)
+	 *   ...
+	 *   ...
+	 *   <Drain sim-KCCB>
+	 *   <Sync live and sim CCCB>
+	 * }PDUMP_BLOCK_END_0x0000000y
+	 * ...
+	 * ...
+	 *
+	 * Steps (3) and (5) are done in pdump_server.c
+	 * */
+	switch (eEvent)
+	{
+		case PDUMP_TRANSITION_EVENT_RANGE_ENTERED:
+			{
+				/* We're about to transition into capture range and we've submitted
+				 * new commands since the last time we entered capture range so drain
+				 * the live CCB and simulation (sim) CCB as required, i.e. leave CCB
+				 * idle in both live and sim contexts.
+				 * This requires the host driver to ensure the live FW & the sim FW
+				 * have both emptied out the remaining commands until R==W (CCB empty).
+				 */
+
+				eError = RGXCCBPDumpSyncCCB(psClientCCB, ui32PDumpFlags);
+				PVR_RETURN_IF_ERROR(eError);
+
+				if (psClientCCB->ui32LastPDumpWriteOffset != psClientCCB->ui32HostWriteOffset)
+				{
+					/* If new commands have been written when out of capture range in
+					 * the live CCB then we need to fast forward the sim CCBCtl
+					 * offsets past uncaptured commands. This is done by PDUMPing
+					 * the CCBCtl memory to align sim values with the live CCBCtl
+					 * values. Both live and sim FWs can start with the 1st command
+					 * which is in the new capture range.
+					 */
+					RGXCCBPDumpFastForwardCCB(psClientCCB, ui32PDumpFlags);
+				}
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_RANGE_APPEND:
+			{
+				/* Capture range already entered. Skip dump sync and fast forward.
+				 */
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_RANGE_EXITED:
+			{
+				/* Nothing to do */
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_BLOCK_FINISHED:
+			{
+				/* (1) Drain KCCB from current block before starting new:
+				 *
+				 * At playback, this will ensure that sim-FW drains all commands in KCCB
+				 * belongs to current block before 'jumping' to any future commands (from
+				 * next block). This will synchronise script-thread and sim-FW thread KCCBs
+				 * at end of each pdump block.
+				 *
+				 * This will additionally force redump of KCCBCtl structure at start of next/new block.
+				 * */
+
+#if defined(PDUMP)
+				eError = RGXPdumpDrainKCCB(psDevInfo, psDevInfo->psKernelCCBCtl->ui32WriteOffset);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXPdumpDrainKCCB");
+#endif
+
+				/* (2) Synchronise Client CCBs from live and playback contexts before starting new block:
+				 *
+				 * This operation will,
+				 * a. Force synchronisation between app-thread and live-FW thread (i.e. Wait
+				 *    for live-FW to empty live Client CCB).
+				 *
+				 * b. Next, it will dump poll command to drain Client CCB at end of every
+				 *    pdump block. At playback time this will synchronise sim-FW and
+				 *    script-thread Client CCBs at end of each block.
+				 *
+				 * This is to ensure that all commands in CCB from current block are processed
+				 * before moving on to future commands.
+				 * */
+
+				eError = RGXCCBPDumpSyncCCB(psClientCCB, ui32PDumpFlags);
+				PVR_RETURN_IF_ERROR(eError);
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_BLOCK_STARTED:
+			{
+				/* (4) Fast-forward CCB write offsets to current live values:
+				 *
+				 * We have already synchronised live-FW and app-thread above at end of each
+				 * block (in Step 2a above), now fast-forward Client CCBCtl write offsets to that of
+				 * current app-thread values at start of every block. This will allow us to
+				 * skip any intermediate pdump blocks and start with last (or any next) block
+				 * immediately after first pdump block.
+				 * */
+
+				RGXCCBPDumpFastForwardCCB(psClientCCB, ui32PDumpFlags);
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_NONE:
+			/* Invalid event for transition */
+		default:
+			{
+				/* Unknown Transition event */
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+	}
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+
+static INLINE void _RGXInitCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	psClientCCB->sUtilisation.ui32HighWaterMark = 0; /* initialize ui32HighWaterMark level to zero */
+	psClientCCB->sUtilisation.ui32ThresholdBytes = (psClientCCB->ui32Size *
+							PVRSRV_ENABLE_CCCB_UTILISATION_INFO_THRESHOLD)	/ 100;
+	psClientCCB->sUtilisation.ui32Warnings = 0;
+	psClientCCB->sUtilisation.ui32CCBAcquired = 0;
+	psClientCCB->sUtilisation.ui32CCBFull = 0;
+}
+
+static INLINE void _RGXCCBUtilisationEvent(RGX_CLIENT_CCB *psClientCCB,
+						IMG_UINT32 ui32WarningType,
+						IMG_UINT32 ui32CmdSize)
+{
+	/* in VERBOSE mode we will print a message for each different
+	 * event type as they happen.
+	 */
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	if (!(psClientCCB->sUtilisation.ui32Warnings & ui32WarningType))
+	{
+		if (ui32WarningType == PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED)
+		{
+			PVR_LOG(("Failed to acquire CCB space for %u byte command:", ui32CmdSize));
+		}
+
+		PVR_LOG(("%s: Client CCB (%s) watermark (%u) hit %d%% of its allocation size (%u)",
+									__func__,
+									psClientCCB->szName,
+									psClientCCB->sUtilisation.ui32HighWaterMark,
+									psClientCCB->sUtilisation.ui32HighWaterMark * 100 / psClientCCB->ui32Size,
+									psClientCCB->ui32Size));
+
+		/* record that we have issued a warning of this type */
+		psClientCCB->sUtilisation.ui32Warnings |= ui32WarningType;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psClientCCB);
+	PVR_UNREFERENCED_PARAMETER(ui32WarningType);
+	PVR_UNREFERENCED_PARAMETER(ui32CmdSize);
+#endif
+}
+
+/* Check the current CCB utilisation. Print a one-time warning message if it is above the
+ * specified threshold
+ */
+static INLINE void _RGXCheckCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	/* Print a warning message if the cCCB watermark is above the threshold value */
+	if (psClientCCB->sUtilisation.ui32HighWaterMark >= psClientCCB->sUtilisation.ui32ThresholdBytes)
+	{
+		_RGXCCBUtilisationEvent(psClientCCB,
+					PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_THRESHOLD,
+					0);
+	}
+}
+
+/* Update the cCCB high watermark level if necessary */
+static void _RGXUpdateCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	IMG_UINT32 ui32FreeSpace, ui32MemCurrentUsage;
+
+	RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+	                           INVALIDATE);
+
+	ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+									  psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+									  psClientCCB->ui32Size);
+	ui32MemCurrentUsage = psClientCCB->ui32Size - ui32FreeSpace;
+
+	if (ui32MemCurrentUsage > psClientCCB->sUtilisation.ui32HighWaterMark)
+	{
+		psClientCCB->sUtilisation.ui32HighWaterMark = ui32MemCurrentUsage;
+
+		/* The high water mark has increased. Check if it is above the
+		 * threshold so we can print a warning if necessary.
+		 */
+		_RGXCheckCCBUtilisation(psClientCCB);
+	}
+}
+
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+PVRSRV_ERROR RGXCreateCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+						  IMG_UINT32			ui32CCBSizeLog2,
+						  IMG_UINT32			ui32CCBMaxSizeLog2,
+						  IMG_UINT32			ui32ContextFlags,
+						  CONNECTION_DATA		*psConnectionData,
+						  RGX_CCB_REQUESTOR_TYPE		eRGXCCBRequestor,
+						  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+						  RGX_CLIENT_CCB		**ppsClientCCB,
+						  DEVMEM_MEMDESC		**ppsClientCCBMemDesc,
+						  DEVMEM_MEMDESC		**ppsClientCCBCtrlMemDesc)
+{
+	PVRSRV_ERROR	eError = PVRSRV_OK;
+	PVRSRV_MEMALLOCFLAGS_T	uiClientCCBMemAllocFlags, uiClientCCBCtlMemAllocFlags;
+	IMG_UINT32		ui32FWMainLog2PageSize = DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap);
+	IMG_UINT32		ui32ChunkSize = (1U << ui32FWMainLog2PageSize);
+	IMG_UINT32		ui32AllocSize = MAX((1U << ui32CCBSizeLog2), ui32ChunkSize);
+	IMG_UINT32		ui32MinAllocSize = MAX((1U << MIN_SAFE_CCB_SIZE_LOG2), ui32ChunkSize);
+	RGX_CLIENT_CCB	*psClientCCB;
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	IMG_UINT32		ui32NumChunks = ui32AllocSize / ui32ChunkSize;
+	IMG_UINT32		ui32VirtualAllocSize = (1U << ui32CCBMaxSizeLog2);
+	IMG_UINT32		ui32NumVirtChunks = ui32VirtualAllocSize / ui32ChunkSize;
+	IMG_UINT32		i;
+
+	/* For the allocation request to be valid, at least one page is required.
+	 * This is relevant on systems where the page size is greater than the client CCB size. */
+	ui32NumVirtChunks = MAX(1, ui32NumVirtChunks);
+	PVR_ASSERT((ui32ChunkSize >= (1U << PAGE_SHIFT)));
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32CCBMaxSizeLog2);
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+	/* All client CCBs should be at-least of the "minimum" size and not to exceed "maximum" */
+	if ((ui32CCBSizeLog2 < MIN_SAFE_CCB_SIZE_LOG2) ||
+		(ui32CCBSizeLog2 > MAX_SAFE_CCB_SIZE_LOG2))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: %s CCB size is invalid (%d). Should be from %d to %d",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         ui32CCBSizeLog2, MIN_SAFE_CCB_SIZE_LOG2, MAX_SAFE_CCB_SIZE_LOG2));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if ((ui32CCBMaxSizeLog2 < ui32CCBSizeLog2) ||
+		(ui32CCBMaxSizeLog2 > MAX_SAFE_CCB_SIZE_LOG2))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: %s CCB maximum size is invalid (%d). Should be from %d to %d",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         ui32CCBMaxSizeLog2, ui32CCBSizeLog2, MAX_SAFE_CCB_SIZE_LOG2));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+
+	psClientCCB = OSAllocMem(sizeof(*psClientCCB));
+	if (psClientCCB == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+	psClientCCB->psServerCommonContext = psServerCommonContext;
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	psClientCCB->ui32VirtualAllocSize = 0;
+	psClientCCB->pui32MappingTable = NULL;
+	psClientCCB->ui32ChunkSize = ui32ChunkSize;
+#endif
+
+	uiClientCCBMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+								PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+								PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+								PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+#if defined(DEBUG)
+								PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC |
+#endif
+								PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+								PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	uiClientCCBCtlMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+								PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+								PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+								PVRSRV_MEMALLOCFLAG_CPU_UNCACHED |
+								PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+								PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+								PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	/* If connection data indicates Sync Lockup Recovery (SLR) should be disabled,
+	 * or if the caller has set ui32ContextFlags to disable SLR for this context,
+	 * indicate this in psClientCCB->ui32CCBFlags.
+	 */
+	if ((psConnectionData->ui32ClientFlags & SRV_FLAGS_CLIENT_SLR_DISABLED) ||
+	    (ui32ContextFlags & RGX_CONTEXT_FLAG_DISABLESLR))
+	{
+		BIT_SET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGXFW cCCB");
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if (BITMASK_HAS(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN))
+	{
+		PHYS_HEAP *psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+		PHYS_HEAP_POLICY uiHeapPolicy = PhysHeapGetPolicy(psPhysHeap);
+
+		psClientCCB->ui32VirtualAllocSize = ui32VirtualAllocSize;
+
+		if (uiHeapPolicy != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+		{
+			psClientCCB->pui32MappingTable = NULL;
+			/*
+			 * On LMA sparse memory can't be mapped to kernel without support for non physically
+			 * sparse allocations.
+			 * To work around this whole ccb memory is allocated at once as contiguous.
+			 */
+			eError = DevmemFwAllocate(psDevInfo,
+									ui32VirtualAllocSize,
+									uiClientCCBMemAllocFlags,
+									aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+									&psClientCCB->psClientCCBMemDesc);
+		}
+		else
+		{
+			/*
+			 * Growing CCB is doubling the size. Last grow would require only ui32NumVirtChunks/2 new chunks
+			 * because another ui32NumVirtChunks/2 is already allocated.
+			 * Sometimes initial chunk count would be higher (when CCB size is equal to CCB maximum size) so MAX is needed.
+			 */
+			psClientCCB->pui32MappingTable = OSAllocMem(MAX(ui32NumChunks, ui32NumVirtChunks/2) * sizeof(IMG_UINT32));
+			if (psClientCCB->pui32MappingTable == NULL)
+			{
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_mtable;
+			}
+
+			for (i = 0; i < ui32NumChunks; i++)
+			{
+				psClientCCB->pui32MappingTable[i] = i;
+			}
+
+			eError = DevmemFwAllocateSparse(psDevInfo,
+											ui32VirtualAllocSize,
+											ui32NumChunks,
+											ui32NumVirtChunks,
+											psClientCCB->pui32MappingTable,
+											uiClientCCBMemAllocFlags,
+											aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+											&psClientCCB->psClientCCBMemDesc);
+		}
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+		psClientCCB->pui32MappingTable = NULL;
+		psClientCCB->ui32VirtualAllocSize = 0;
+	}
+
+	if (!BITMASK_HAS(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN) ||
+	    (eError != PVRSRV_OK))
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+	{
+		/* Allocate ui32AllocSize, or the next best POT allocation */
+		do
+		{
+			eError = DevmemFwAllocate(psDevInfo,
+									ui32AllocSize,
+									uiClientCCBMemAllocFlags,
+									aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+									&psClientCCB->psClientCCBMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				/* Failed to allocate - ensure CCB grow is disabled from
+				 * now on for this device.
+				 */
+				BITMASK_UNSET(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+
+				/* Failed to allocate, try next POT down */
+				ui32AllocSize >>= 1;
+			}
+		} while ((eError != PVRSRV_OK) && (ui32AllocSize > ui32MinAllocSize));
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate RGX client CCB (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_alloc_ccb;
+	}
+
+	OSSNPrintf(psClientCCB->szName, MAX_CLIENT_CCB_NAME, "%s-P%lu-T%lu-%s",
+									aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+									(unsigned long) OSGetCurrentClientProcessIDKM(),
+									(unsigned long) OSGetCurrentClientThreadIDKM(),
+									OSGetCurrentClientProcessNameKM());
+
+	if (ui32AllocSize < (1U << ui32CCBSizeLog2))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unable to allocate %d bytes for RGX client CCB (%s) but allocated %d bytes",
+		         __func__,
+		         (1U << ui32CCBSizeLog2),
+		         psClientCCB->szName,
+		         ui32AllocSize));
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									  &psClientCCB->pvClientCCB);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map RGX client CCB (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_map_ccb;
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGXFW cCCB control");
+	eError = DevmemFwAllocate(psDevInfo,
+										sizeof(RGXFWIF_CCCB_CTL),
+										uiClientCCBCtlMemAllocFlags,
+										aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_CONTROL_STRING],
+										&psClientCCB->psClientCCBCtrlMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate RGX client CCB control (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_alloc_ccbctrl;
+	}
+
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc,
+									  (void **) &psClientCCB->psClientCCBCtrl);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map RGX client CCB control (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_map_ccbctrl;
+	}
+
+	/* psClientCCBCtrlMemDesc was zero alloc'd so no need to initialise offsets. */
+	psClientCCB->psClientCCBCtrl->ui32WrapMask = ui32AllocSize - 1;
+
+	/* Flush the whole struct since other parts are implicitly init (zero'd) */
+	RGXFwSharedMemCacheOpPtr(psClientCCB->psClientCCBCtrl,
+	                         FLUSH);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "cCCB control");
+	DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCCB_CTL),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	psClientCCB->ui32HostWriteOffset = 0;
+	psClientCCB->ui32LastPDumpWriteOffset = 0;
+	psClientCCB->ui32Size = ui32AllocSize;
+	psClientCCB->ui32LastROff = ui32AllocSize - 1;
+	psClientCCB->ui32ByteCount = 0;
+	psClientCCB->ui32LastByteCount = 0;
+	BIT_UNSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	eError = OSLockCreate(&psClientCCB->hCCBGrowLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create hCCBGrowLock (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_create_ccbgrow_lock;
+	}
+#endif
+#if defined(DEBUG)
+	psClientCCB->ui32UpdateEntries = 0;
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXInitCCBUtilisation(psClientCCB);
+	psClientCCB->eRGXCCBRequestor = eRGXCCBRequestor;
+#endif
+	eError = PDumpRegisterTransitionCallback(psConnectionData->psPDumpConnectionData,
+											  _RGXCCBPDumpTransition,
+											  psClientCCB,
+											  psDevInfo,
+											  &psClientCCB->hTransition);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_pdumpreg;
+	}
+
+	/*
+	 * Note:
+	 * Save the PDump specific structure, which is ref counted unlike
+	 * the connection data, to ensure it's not freed too early
+	 */
+	psClientCCB->psPDumpConnectionData = psConnectionData->psPDumpConnectionData;
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "New RGXFW cCCB(%s@%p) created",
+				 psClientCCB->szName,
+				 psClientCCB);
+
+	*ppsClientCCB = psClientCCB;
+	*ppsClientCCBMemDesc = psClientCCB->psClientCCBMemDesc;
+	*ppsClientCCBCtrlMemDesc = psClientCCB->psClientCCBCtrlMemDesc;
+	return PVRSRV_OK;
+
+fail_pdumpreg:
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockDestroy(psClientCCB->hCCBGrowLock);
+fail_create_ccbgrow_lock:
+#endif
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc);
+fail_map_ccbctrl:
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBCtrlMemDesc);
+fail_alloc_ccbctrl:
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+fail_map_ccb:
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBMemDesc);
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+fail_alloc_ccb:
+	if (psClientCCB->pui32MappingTable)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+	}
+fail_alloc_mtable:
+#else
+fail_alloc_ccb:
+#endif
+	OSFreeMem(psClientCCB);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void RGXDestroyCCB(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_CLIENT_CCB *psClientCCB)
+{
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	if (psClientCCB->sUtilisation.ui32CCBFull)
+	{
+		PVR_LOG(("CCBUtilisationInfo: GPU %s command buffer was full %d times out of %d. "
+				"This is not an error but the application may not run optimally.",
+				aszCCBRequestors[psClientCCB->eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+				psClientCCB->sUtilisation.ui32CCBFull,
+				psClientCCB->sUtilisation.ui32CCBAcquired));
+	}
+#endif
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockDestroy(psClientCCB->hCCBGrowLock);
+#endif
+	PDumpUnregisterTransitionCallback(psClientCCB->hTransition);
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBCtrlMemDesc);
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBMemDesc);
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if (psClientCCB->pui32MappingTable)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+	}
+#endif
+	OSFreeMem(psClientCCB);
+}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+static PVRSRV_ERROR _RGXCCBMemChangeSparse(RGX_CLIENT_CCB *psClientCCB,
+										  IMG_UINT32 ui32AllocPageCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32	 i;
+
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+
+	for (i = 0; i < ui32AllocPageCount; i++)
+	{
+		psClientCCB->pui32MappingTable[i] = ui32AllocPageCount + i;
+	}
+
+	/* Double the CCB size (CCB must be POT) by adding ui32AllocPageCount new pages */
+	eError = DeviceMemChangeSparse(psClientCCB->psClientCCBMemDesc,
+									ui32AllocPageCount,
+									psClientCCB->pui32MappingTable,
+									0,
+									NULL,
+									SPARSE_RESIZE_ALLOC);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to grow RGX client CCB (%s)",
+				PVRSRVGetErrorString(eError)));
+
+		if (DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									&psClientCCB->pvClientCCB) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to reacquire CCB mapping"));
+			psClientCCB->pvClientCCB = NULL;
+		}
+
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									&psClientCCB->pvClientCCB);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to map RGX client CCB (%s)",
+				PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+PVRSRV_ERROR RGXCheckSpaceCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32CmdSize)
+{
+	IMG_UINT32 ui32FreeSpace;
+
+	/* Check that the CCB can hold this command + padding */
+	if ((ui32CmdSize + PADDING_COMMAND_SIZE + 1) > psClientCCB->ui32Size)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Command size (%d bytes) too big for CCB"
+		        " (%d bytes)", ui32CmdSize, psClientCCB->ui32Size));
+		return PVRSRV_ERROR_CMD_TOO_BIG;
+	}
+
+	/*
+		Check we don't overflow the end of the buffer and make sure we have
+		enough space for the padding command. If we don't have enough space
+		(including the minimum amount for the padding command) we need to make
+		sure we insert a padding command now and wrap before adding the main
+		command.
+	*/
+
+	/* Invalidate read offset */
+	RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+	                           INVALIDATE);
+	if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) <= psClientCCB->ui32Size)
+	{
+		ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+		                              psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+		                              psClientCCB->ui32Size);
+
+		/* Don't allow all the space to be used */
+		if (ui32FreeSpace > ui32CmdSize)
+		{
+			return PVRSRV_OK;
+		}
+
+		goto e_retry;
+	}
+	else
+	{
+		IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+		ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+		                              psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+		                              psClientCCB->ui32Size);
+
+		/* Check there is space for both the command and the padding command */
+		if (ui32FreeSpace > ui32Remain + ui32CmdSize)
+		{
+			return PVRSRV_OK;
+		}
+
+		goto e_retry;
+	}
+
+e_retry:
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXCCBUtilisationEvent(psClientCCB,
+	            PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_FULL_CCB,
+	            ui32CmdSize);
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+	return PVRSRV_ERROR_RETRY;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireCCB
+
+ PURPOSE	: Obtains access to write some commands to a CCB
+
+ PARAMETERS	: psClientCCB		- The client CCB
+			  ui32CmdSize		- How much space is required
+			  ppvBufferSpace	- Pointer to space in the buffer
+			  ui32PDumpFlags - Should this be PDump continuous?
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXAcquireCCB(RGX_CLIENT_CCB *psClientCCB,
+										IMG_UINT32		ui32CmdSize,
+										void			**ppvBufferSpace,
+										IMG_UINT32		ui32PDumpFlags)
+{
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	IMG_UINT32	ui32RetryCount = 2;
+#endif
+
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_BOOL	bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, ui32PDumpFlags);
+	IMG_BOOL	bPDumpFlagsContinuous = PDUMP_IS_CONTINUOUS(ui32PDumpFlags);
+
+	/*
+		PDumpSetFrame will detect as we Transition into capture range for
+		frame based data but if we are PDumping continuous data then we
+		need to inform the PDump layer ourselves
+
+		First check is to confirm we are in continuous mode
+		Second check is to confirm the pdump client is connected and ready.
+		Third check is to confirm we are not in capture range.
+	*/
+	if (bPDumpFlagsContinuous &&
+		bPDumpEnabled &&
+		!PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_NONE))
+	{
+		PDUMP_TRANSITION_EVENT eTransitionEvent = BIT_ISSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN) ?
+			PDUMP_TRANSITION_EVENT_RANGE_APPEND : PDUMP_TRANSITION_EVENT_RANGE_ENTERED;
+
+		eError = PDumpTransition(psDeviceNode,
+		                         psClientCCB->psPDumpConnectionData,
+		                         eTransitionEvent,
+		                         ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	/* Check that the CCB can hold this command + padding */
+	if ((ui32CmdSize + PADDING_COMMAND_SIZE + 1) > psClientCCB->ui32Size)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Command size (%d bytes) too big for CCB (%d bytes)",
+								ui32CmdSize, psClientCCB->ui32Size));
+		return PVRSRV_ERROR_CMD_TOO_BIG;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	while (ui32RetryCount--)
+#endif
+	{
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+		psClientCCB->sUtilisation.ui32CCBAcquired++;
+#endif
+
+		/*
+			Check we don't overflow the end of the buffer and make sure we have
+			enough space for the padding command. We don't have enough space (including the
+			minimum amount for the padding command) we will need to make sure we insert a
+			padding command now and wrap before adding the main command.
+		*/
+
+		if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) <= psClientCCB->ui32Size)
+		{
+			/* The command can fit without wrapping... */
+			IMG_UINT32 ui32FreeSpace;
+
+#if defined(PDUMP)
+			/* Wait for sufficient CCB space to become available */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+								  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+								  ui32CmdSize, psClientCCB->ui32HostWriteOffset,
+								  psClientCCB->szName);
+			DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+						offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+						psClientCCB->ui32HostWriteOffset,
+						ui32CmdSize,
+						psClientCCB->ui32Size);
+#endif
+
+			RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+			                           INVALIDATE);
+			ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+										psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+										psClientCCB->ui32Size);
+
+			/* Can command fit? */
+			if (ui32FreeSpace > ui32CmdSize)
+			{
+				*ppvBufferSpace = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+				return PVRSRV_OK;
+			}
+			/* There is not enough free space in CCB. */
+			goto e_retry;
+		}
+		else
+		{
+			/*
+				We're at the end of the buffer without enough contiguous space.
+				The command cannot fit without wrapping, we need to insert a
+				padding command and wrap. We need to do this in one go otherwise
+				we would be leaving unflushed commands and forcing the client to
+				deal with flushing the padding command but not the command they
+				wanted to write. Therefore we either do all or nothing.
+			*/
+			RGXFWIF_CCB_CMD_HEADER *psHeader;
+			IMG_UINT32 ui32FreeSpace;
+			IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+			RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+			                           INVALIDATE);
+			/* Check this is a growable CCB */
+			if (psClientCCB->ui32VirtualAllocSize > 0)
+			{
+				PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+
+				ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+											psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+											psClientCCB->ui32Size);
+				/*
+				 * Check if CCB should grow or be wrapped.
+				 * Wrap CCB if there is no need for grow (CCB is half empty) or CCB can't grow,
+				 * and when is free space for command and padding.
+				 */
+				if (((ui32FreeSpace > psClientCCB->ui32Size/2) || (psClientCCB->ui32Size == psClientCCB->ui32VirtualAllocSize)) &&
+					(ui32FreeSpace > ui32Remain + ui32CmdSize))
+				{
+					/* Wrap CCB */
+					psHeader = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+					psHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PADDING;
+					psHeader->ui32CmdSize = ui32Remain - sizeof(RGXFWIF_CCB_CMD_HEADER);
+
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags,
+										  "cCCB(%p): Padding cmd %d", psClientCCB, psHeader->ui32CmdSize);
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+										psClientCCB->ui32HostWriteOffset,
+										ui32Remain,
+										ui32PDumpFlags);
+					}
+#endif
+
+					*ppvBufferSpace = psClientCCB->pvClientCCB;
+					return PVRSRV_OK;
+				}
+				else if ((psClientCCB->ui32Size < psClientCCB->ui32VirtualAllocSize) &&
+				         (psClientCCB->ui32HostWriteOffset >= psClientCCB->psClientCCBCtrl->ui32ReadOffset))
+				{
+					/* Grow CCB */
+					PHYS_HEAP *psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+					PHYS_HEAP_POLICY uiHeapPolicy = PhysHeapGetPolicy(psPhysHeap);
+					PVRSRV_ERROR eErr = PVRSRV_OK;
+
+					/* Something went wrong if we are here a second time */
+					PVR_ASSERT(ui32RetryCount != 0);
+					OSLockAcquire(psClientCCB->hCCBGrowLock);
+
+					/*
+					 * On LMA sparse memory can't be mapped to kernel without support for non physically
+					 * sparse allocations.
+					 * To work around this whole ccb memory was allocated at once as contiguous.
+					 * In such case below sparse change is not needed because memory is already allocated.
+					 */
+					if (uiHeapPolicy == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+					{
+						IMG_UINT32 ui32AllocChunkCount = psClientCCB->ui32Size / psClientCCB->ui32ChunkSize;
+
+						eErr = _RGXCCBMemChangeSparse(psClientCCB, ui32AllocChunkCount);
+					}
+
+					/* Setup new CCB size */
+					if (eErr == PVRSRV_OK)
+					{
+						psClientCCB->ui32Size += psClientCCB->ui32Size;
+					}
+					else
+					{
+						PVR_LOG(("%s: Client CCB (%s) grow failed (%s)", __func__, psClientCCB->szName, PVRSRVGetErrorString(eErr)));
+						OSLockRelease(psClientCCB->hCCBGrowLock);
+						goto e_retry;
+					}
+
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "cCCB update for grow");
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+											offsetof(RGXFWIF_CCCB_CTL, ui32WrapMask),
+											sizeof(psClientCCB->psClientCCBCtrl->ui32WrapMask),
+											ui32PDumpFlags);
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+											offsetof(RGX_CLIENT_CCB, ui32Size),
+											sizeof(psClientCCB->ui32Size),
+											ui32PDumpFlags);
+					}
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+					PVR_LOG(("%s: Client CCB (%s) grew to %u", __func__, psClientCCB->szName, psClientCCB->ui32Size));
+					/* Reset counters */
+					_RGXInitCCBUtilisation(psClientCCB);
+#endif
+
+					/* CCB doubled the size so retry now. */
+					OSLockRelease(psClientCCB->hCCBGrowLock);
+				}
+				else
+				{
+					/* CCB can't grow anymore and can't be wrapped */
+#if defined(PDUMP)
+					/* Wait for sufficient CCB space to become available */
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+										  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+										  ui32Remain, psClientCCB->ui32HostWriteOffset,
+										  psClientCCB->szName);
+					DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+								offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+								psClientCCB->ui32HostWriteOffset,
+								ui32Remain,
+								psClientCCB->ui32Size);
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+										  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+										  ui32CmdSize, 0 /*ui32HostWriteOffset after wrap */,
+										  psClientCCB->szName);
+					DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+								offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+								0 /*ui32HostWriteOffset after wrap */,
+								ui32CmdSize,
+								psClientCCB->ui32Size);
+					/* CCB has now space for our command so try wrapping again. Retry now. */
+#else /* defined(PDUMP) */
+					goto e_retry;
+#endif /* defined(PDUMP) */
+				}
+			}
+			else
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+			{
+#if defined(PDUMP)
+				/* Wait for sufficient CCB space to become available */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+									  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+									  ui32Remain, psClientCCB->ui32HostWriteOffset,
+									  psClientCCB->szName);
+				DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+							offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+							psClientCCB->ui32HostWriteOffset,
+							ui32Remain,
+							psClientCCB->ui32Size);
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+									  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+									  ui32CmdSize, 0 /*ui32HostWriteOffset after wrap */,
+									  psClientCCB->szName);
+				DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+							offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+							0 /*ui32HostWriteOffset after wrap */,
+							ui32CmdSize,
+							psClientCCB->ui32Size);
+#endif
+				RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+				                           INVALIDATE);
+				ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+											psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+											psClientCCB->ui32Size);
+
+				if (ui32FreeSpace > ui32Remain + ui32CmdSize)
+				{
+					psHeader = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+					psHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PADDING;
+					psHeader->ui32CmdSize = ui32Remain - sizeof(RGXFWIF_CCB_CMD_HEADER);
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "cCCB(%p): Padding cmd %d", psClientCCB, psHeader->ui32CmdSize);
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+										psClientCCB->ui32HostWriteOffset,
+										ui32Remain,
+										ui32PDumpFlags);
+					}
+#endif
+
+					*ppvBufferSpace = psClientCCB->pvClientCCB;
+					return PVRSRV_OK;
+				}
+
+				goto e_retry;
+			}
+		}
+	}
+e_retry:
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	psClientCCB->sUtilisation.ui32CCBFull++;
+	_RGXCCBUtilisationEvent(psClientCCB,
+				PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED,
+				ui32CmdSize);
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+	return PVRSRV_ERROR_RETRY;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXReleaseCCB
+
+ PURPOSE	: Release a CCB that we have been writing to.
+
+ PARAMETERS	: psDevData			- device data
+			  psCCB				- the CCB
+
+ RETURNS	: None
+******************************************************************************/
+void RGXReleaseCCB(RGX_CLIENT_CCB *psClientCCB,
+								IMG_UINT32		ui32CmdSize,
+								IMG_UINT32		ui32PDumpFlags)
+{
+	IMG_UINT32 ui32NewWriteOffset;
+
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_BOOL	bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, ui32PDumpFlags);
+	IMG_BOOL	bPDumpFlagsContinuous = PDUMP_IS_CONTINUOUS(ui32PDumpFlags);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockAcquire(psClientCCB->hCCBGrowLock);
+#endif
+	/*
+	 * If a padding command was needed then we should now move ui32HostWriteOffset
+	 * forward. The command has already be dumped (if bPDumpEnabled).
+	 */
+	if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) > psClientCCB->ui32Size)
+	{
+		IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+		UPDATE_CCB_OFFSET(psClientCCB->ui32HostWriteOffset,
+						  ui32Remain,
+						  psClientCCB->ui32Size);
+		psClientCCB->ui32ByteCount += ui32Remain;
+	}
+
+#if defined(PDUMP)
+	/* Dump the CCB data */
+	if (bPDumpEnabled)
+	{
+		DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+						   psClientCCB->ui32HostWriteOffset,
+						   ui32CmdSize,
+						   ui32PDumpFlags);
+	}
+#endif
+
+	/*
+	 * Check if there any fences being written that will already be
+	 * satisfied by the last written update command in this CCB. At the
+	 * same time we can ASSERT that all sync addresses are not NULL.
+	 */
+#if defined(DEBUG)
+	{
+		void *pvBufferStart = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+		void *pvBufferEnd   = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset + ui32CmdSize);
+		IMG_BOOL  bMessagePrinted  = IMG_FALSE;
+
+		/* Walk through the commands in this section of CCB being released... */
+		while (pvBufferStart < pvBufferEnd)
+		{
+			RGXFWIF_CCB_CMD_HEADER *psCmdHeader = pvBufferStart;
+
+			if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_UPDATE)
+			{
+				/* If an UPDATE then record the values in case an adjacent fence uses it. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				psClientCCB->ui32UpdateEntries = 0;
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+					if (psClientCCB->ui32UpdateEntries < RGX_CCCB_FENCE_UPDATE_LIST_SIZE)
+					{
+						psClientCCB->asFenceUpdateList[psClientCCB->ui32UpdateEntries++] = *psUFOPtr++;
+					}
+				}
+			}
+			else if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE)
+			{
+				/* If a FENCE then check the values against the last UPDATE issued. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+
+					if (bMessagePrinted == IMG_FALSE)
+					{
+						RGXFWIF_UFO *psUpdatePtr = psClientCCB->asFenceUpdateList;
+						IMG_UINT32  ui32UpdateIndex;
+
+						for (ui32UpdateIndex = 0; ui32UpdateIndex < psClientCCB->ui32UpdateEntries; ui32UpdateIndex++)
+						{
+							if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+							{
+								if (RGX_UFO_PTR_ADDR(psUFOPtr) == RGX_UFO_PTR_ADDR(psUpdatePtr))
+								{
+									PVR_DPF((PVR_DBG_MESSAGE, "Redundant sync checkpoint check found in cCCB(%p) - 0x%x -> 0x%x",
+											psClientCCB, RGX_UFO_PTR_ADDR(psUFOPtr), psUFOPtr->ui32Value));
+									bMessagePrinted = IMG_TRUE;
+									break;
+								}
+							}
+							else
+							{
+								if (psUFOPtr->puiAddrUFO.ui32Addr == psUpdatePtr->puiAddrUFO.ui32Addr  &&
+									psUFOPtr->ui32Value == psUpdatePtr->ui32Value)
+								{
+									PVR_DPF((PVR_DBG_MESSAGE, "Redundant fence check found in cCCB(%p) - 0x%x -> 0x%x",
+											psClientCCB, psUFOPtr->puiAddrUFO.ui32Addr, psUFOPtr->ui32Value));
+									bMessagePrinted = IMG_TRUE;
+									break;
+								}
+							}
+							psUpdatePtr++;
+						}
+					}
+
+					psUFOPtr++;
+				}
+			}
+			else if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR)
+			{
+				/* For all other UFO ops check the UFO address is not NULL. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+					psUFOPtr++;
+				}
+			}
+
+			/* Move to the next command in this section of CCB being released... */
+			pvBufferStart = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER) + psCmdHeader->ui32CmdSize);
+		}
+	}
+#endif /* REDUNDANT_SYNCS_DEBUG */
+
+
+#if defined(PVRSRV_FORCE_FLUSH_CCCB_ON_KICK)
+	{
+		DEVMEM_MEMDESC* psClientCCBMemDesc = psClientCCB->psClientCCBMemDesc;
+		void *pvClientCCBAddr = psClientCCB->pvClientCCB;
+		PMR *psClientCCBMemDescPMR = NULL;
+		IMG_DEVMEM_OFFSET_T uiPMROffset;
+
+		DevmemGetPMRData(psClientCCBMemDesc,
+			             (IMG_HANDLE*)&psClientCCBMemDescPMR,
+			             &uiPMROffset);
+
+		CacheOpValExec(psClientCCBMemDescPMR,
+					   (IMG_UINT64)(uintptr_t) pvClientCCBAddr,
+					   uiPMROffset,
+					   psClientCCBMemDesc->uiAllocSize,
+					   PVRSRV_CACHE_OP_FLUSH);
+
+	}
+#endif
+
+	ui32NewWriteOffset = psClientCCB->ui32HostWriteOffset;
+	/*
+	 * Update the CCB write offset.
+	 */
+	UPDATE_CCB_OFFSET(ui32NewWriteOffset, ui32CmdSize,
+	                  psClientCCB->ui32Size);
+	psClientCCB->ui32ByteCount += ui32CmdSize;
+
+	/* Flush the CCB data */
+	RGXFwSharedMemFlushCCB(psClientCCB->pvClientCCB,
+	                       psClientCCB->ui32HostWriteOffset,
+	                       ui32NewWriteOffset,
+	                       psClientCCB->psClientCCBCtrl->ui32WrapMask + 1);
+
+	psClientCCB->ui32HostWriteOffset = ui32NewWriteOffset;
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXUpdateCCBUtilisation(psClientCCB);
+#endif
+	/*
+		PDumpSetFrame will detect as we Transition out of capture range for
+		frame based data but if we are PDumping continuous data then we
+		need to inform the PDump layer ourselves
+
+		First check is to confirm we are in continuous mode
+		Second check is to confirm the pdump client is connected and ready.
+		Third check is to confirm we are not in capture range.
+	*/
+#if defined(PDUMP)
+	if (bPDumpFlagsContinuous &&
+		bPDumpEnabled &&
+		!PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_NONE))
+	{
+		PVRSRV_ERROR eError;
+
+		/* Only Transitioning into capture range can cause an error */
+		eError = PDumpTransition(psDeviceNode,
+		                         psClientCCB->psPDumpConnectionData,
+		                         PDUMP_TRANSITION_EVENT_RANGE_EXITED,
+		                         ui32PDumpFlags);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	if (bPDumpEnabled)
+	{
+		/* Update the PDump write offset to show we PDumped this command */
+		psClientCCB->ui32LastPDumpWriteOffset = psClientCCB->ui32HostWriteOffset;
+	}
+#endif
+
+#if defined(NO_HARDWARE)
+	/*
+		The firmware is not running, it cannot update these; we do here instead.
+	*/
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32DepOffset = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset2 = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset3 = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset4 = psClientCCB->ui32HostWriteOffset;
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psClientCCB->hCCBGrowLock);
+#endif
+}
+
+IMG_UINT32 RGXGetHostWriteOffsetCCB(RGX_CLIENT_CCB *psClientCCB)
+{
+	return psClientCCB->ui32HostWriteOffset;
+}
+
+IMG_UINT32 RGXGetWrapMaskCCB(RGX_CLIENT_CCB *psClientCCB)
+{
+	return psClientCCB->ui32Size-1;
+}
+
+void RGXSetCCBFlags(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32		ui32Flags)
+{
+	if ((ui32Flags & RGX_CONTEXT_FLAG_DISABLESLR))
+	{
+		BIT_SET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+	else
+	{
+		BIT_UNSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+}
+
+void RGXCmdHelperInitCmdCCB_CommandSize(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                        IMG_UINT64 ui64FBSCEntryMask,
+                                        IMG_UINT32 ui32ClientFenceCount,
+                                        IMG_UINT32 ui32ClientUpdateCount,
+                                        IMG_UINT32 ui32CmdSize,
+                                        PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                                        PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                                        PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                                        RGX_CCB_CMD_HELPER_DATA *psCmdHelperData)
+{
+	IMG_UINT32 ui32FenceCmdSize = 0;
+	IMG_UINT32 ui32UpdateCmdSize = 0;
+
+	/* Init the generated data members */
+	psCmdHelperData->ui32FBSCInvalCmdSize = 0;
+	psCmdHelperData->ui64FBSCEntryMask = 0;
+	psCmdHelperData->ui32PreTimeStampCmdSize = 0;
+	psCmdHelperData->ui32PostTimeStampCmdSize = 0;
+	psCmdHelperData->ui32RMWUFOCmdSize = 0;
+
+	/* Total FBSC invalidate command size (header plus command data) */
+	if (ui64FBSCEntryMask != 0)
+	{
+		psCmdHelperData->ui32FBSCInvalCmdSize =
+			        RGX_CCB_FWALLOC_ALIGN(sizeof(psCmdHelperData->ui64FBSCEntryMask) +
+			                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+		psCmdHelperData->ui64FBSCEntryMask = ui64FBSCEntryMask;
+	}
+
+	/* total DM command size (header plus command data) */
+
+	psCmdHelperData->ui32DMCmdSize =
+	        RGX_CCB_FWALLOC_ALIGN(ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+	if (ui32ClientFenceCount != 0)
+	{
+		ui32FenceCmdSize =
+		        RGX_CCB_FWALLOC_ALIGN(ui32ClientFenceCount * sizeof(RGXFWIF_UFO) +
+		                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+	}
+
+	if (ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateCmdSize =
+		        RGX_CCB_FWALLOC_ALIGN(ui32ClientUpdateCount * sizeof(RGXFWIF_UFO) +
+		                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+	}
+
+	if (ppPreAddr && (ppPreAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32PreTimeStampCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER)
+			+ PVR_ALIGN(sizeof(RGXFWIF_DEV_VIRTADDR), RGXFWIF_FWALLOC_ALIGN);
+	}
+
+	if (ppPostAddr && (ppPostAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32PostTimeStampCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER)
+			+ PVR_ALIGN(sizeof(RGXFWIF_DEV_VIRTADDR), RGXFWIF_FWALLOC_ALIGN);
+	}
+
+	if (ppRMWUFOAddr && (ppRMWUFOAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32RMWUFOCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_UFO);
+	}
+
+	psCmdHelperData->ui32TotalSize =
+		ui32FenceCmdSize +
+		psCmdHelperData->ui32FBSCInvalCmdSize +
+		psCmdHelperData->ui32DMCmdSize +
+		ui32UpdateCmdSize +
+		psCmdHelperData->ui32PreTimeStampCmdSize +
+		psCmdHelperData->ui32PostTimeStampCmdSize +
+		psCmdHelperData->ui32RMWUFOCmdSize;
+
+	psCmdHelperData->ui32DMCmdOffset =
+		ui32FenceCmdSize +
+		psCmdHelperData->ui32PreTimeStampCmdSize +
+		psCmdHelperData->ui32FBSCInvalCmdSize;
+}
+
+/*
+	Work out how much space this command will require
+*/
+void RGXCmdHelperInitCmdCCB_OtherData(PVRSRV_RGXDEV_INFO        *psDevInfo,
+                                      RGX_CLIENT_CCB            *psClientCCB,
+                                      IMG_UINT32                ui32ClientFenceCount,
+                                      PRGXFWIF_UFO_ADDR         *pauiFenceUFOAddress,
+                                      IMG_UINT32                *paui32FenceValue,
+                                      IMG_UINT32                ui32ClientUpdateCount,
+                                      PRGXFWIF_UFO_ADDR         *pauiUpdateUFOAddress,
+                                      IMG_UINT32                *paui32UpdateValue,
+                                      IMG_UINT32                ui32CmdSize,
+                                      IMG_PBYTE                 pui8DMCmd,
+                                      PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                                      PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                                      PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                                      RGXFWIF_CCB_CMD_TYPE      eType,
+                                      IMG_UINT32                ui32ExtJobRef,
+                                      IMG_UINT32                ui32IntJobRef,
+                                      IMG_UINT32                ui32PDumpFlags,
+                                      RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                                      IMG_CHAR                  *pszCommandName,
+                                      IMG_BOOL                  bCCBStateOpen,
+                                      RGX_CCB_CMD_HELPER_DATA   *psCmdHelperData)
+{
+	/* Job reference values */
+	psCmdHelperData->ui32ExtJobRef = ui32ExtJobRef;
+	psCmdHelperData->ui32IntJobRef = ui32IntJobRef;
+
+	/* Save the data we require in the submit call */
+	psCmdHelperData->psClientCCB = psClientCCB;
+#if defined(PDUMP)
+	psCmdHelperData->ui32PDumpFlags = ui32PDumpFlags;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+	psCmdHelperData->pszCommandName = pszCommandName;
+	if (bCCBStateOpen)
+	{
+		BIT_SET(psCmdHelperData->psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+	}
+	else
+	{
+		BIT_UNSET(psCmdHelperData->psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+	}
+
+	/* Client sync data */
+	psCmdHelperData->ui32ClientFenceCount = ui32ClientFenceCount;
+	psCmdHelperData->pauiFenceUFOAddress = pauiFenceUFOAddress;
+	psCmdHelperData->paui32FenceValue = paui32FenceValue;
+	psCmdHelperData->ui32ClientUpdateCount = ui32ClientUpdateCount;
+	psCmdHelperData->pauiUpdateUFOAddress = pauiUpdateUFOAddress;
+	psCmdHelperData->paui32UpdateValue = paui32UpdateValue;
+
+	/* Command data */
+	psCmdHelperData->ui32CmdSize = ui32CmdSize;
+	psCmdHelperData->pui8DMCmd = pui8DMCmd;
+	psCmdHelperData->eType = eType;
+
+	if (ppPreAddr)
+	{
+		psCmdHelperData->pPreTimestampAddr = *ppPreAddr;
+	}
+
+	if (ppPostAddr)
+	{
+		psCmdHelperData->pPostTimestampAddr = *ppPostAddr;
+	}
+
+	if (ppRMWUFOAddr)
+	{
+		psCmdHelperData->pRMWUFOAddr = *ppRMWUFOAddr;
+	}
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+			"%s Command Server Init on FWCtx %08x", pszCommandName,
+			FWCommonContextGetFWAddress(psClientCCB->psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Workload Data added */
+		psCmdHelperData->psWorkEstKickData = psWorkEstKickData;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(psWorkEstKickData);
+#endif
+}
+
+/*
+	Work out how much space this command will require
+*/
+void RGXCmdHelperInitCmdCCB(PVRSRV_RGXDEV_INFO		*psDevInfo,
+                            RGX_CLIENT_CCB            *psClientCCB,
+                            IMG_UINT64                ui64FBSCEntryMask,
+                            IMG_UINT32                ui32ClientFenceCount,
+                            PRGXFWIF_UFO_ADDR         *pauiFenceUFOAddress,
+                            IMG_UINT32                *paui32FenceValue,
+                            IMG_UINT32                ui32ClientUpdateCount,
+                            PRGXFWIF_UFO_ADDR         *pauiUpdateUFOAddress,
+                            IMG_UINT32                *paui32UpdateValue,
+                            IMG_UINT32                ui32CmdSize,
+                            IMG_PBYTE                 pui8DMCmd,
+                            PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                            PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                            PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                            RGXFWIF_CCB_CMD_TYPE      eType,
+                            IMG_UINT32                ui32ExtJobRef,
+                            IMG_UINT32                ui32IntJobRef,
+                            IMG_UINT32                ui32PDumpFlags,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                            IMG_CHAR                  *pszCommandName,
+                            IMG_BOOL                  bCCBStateOpen,
+                            RGX_CCB_CMD_HELPER_DATA   *psCmdHelperData)
+{
+	RGXCmdHelperInitCmdCCB_CommandSize(psDevInfo,
+	                                 ui64FBSCEntryMask,
+	                                 ui32ClientFenceCount,
+	                                 ui32ClientUpdateCount,
+	                                 ui32CmdSize,
+	                                 ppPreAddr,
+	                                 ppPostAddr,
+	                                 ppRMWUFOAddr,
+	                                 psCmdHelperData);
+
+	RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+	                                 psClientCCB,
+	                                 ui32ClientFenceCount,
+	                                 pauiFenceUFOAddress,
+	                                 paui32FenceValue,
+	                                 ui32ClientUpdateCount,
+	                                 pauiUpdateUFOAddress,
+	                                 paui32UpdateValue,
+	                                 ui32CmdSize,
+	                                 pui8DMCmd,
+	                                 ppPreAddr,
+	                                 ppPostAddr,
+	                                 ppRMWUFOAddr,
+	                                 eType,
+	                                 ui32ExtJobRef,
+	                                 ui32IntJobRef,
+	                                 ui32PDumpFlags,
+	                                 psWorkEstKickData,
+	                                 pszCommandName,
+	                                 bCCBStateOpen,
+	                                 psCmdHelperData);
+}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*************************************************************************/ /*!
+@Function       RGXIsValidWorkloadEstCCBCommand
+
+@Description    Checks if command type can be used for workload estimation
+
+@Input          eType       Command type to check
+
+
+@Return        IMG_BOOL
+******************************************************************************/
+FORCE_INLINE IMG_BOOL RGXIsValidWorkloadEstCCBCommand(RGXFWIF_CCB_CMD_TYPE eType)
+{
+	switch (eType)
+	{
+			case RGXFWIF_CCB_CMD_TYPE_GEOM:
+			case RGXFWIF_CCB_CMD_TYPE_3D:
+			case RGXFWIF_CCB_CMD_TYPE_CDM:
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+			case RGXFWIF_CCB_CMD_TYPE_RAY:
+#endif
+			case RGXFWIF_CCB_CMD_TYPE_TQ_TDM:
+				return IMG_TRUE;
+			default:
+				PVR_ASSERT(IMG_FALSE);
+				return IMG_FALSE;
+	}
+}
+#endif
+
+static inline void RGXWriteCmdHeader(void *pvCCB, IMG_UINT32 eCmdType, IMG_UINT32 ui32TotalSize,
+									 IMG_UINT32 ui32ExtJobRef, IMG_UINT32 ui32IntJobRef,
+									 RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData)
+{
+	RGXFWIF_CCB_CMD_HEADER sCmdHeader;
+
+	sCmdHeader.eCmdType = eCmdType;
+	sCmdHeader.ui32CmdSize = ui32TotalSize - sizeof(RGXFWIF_CCB_CMD_HEADER);
+	sCmdHeader.ui32ExtJobRef = ui32ExtJobRef;
+	sCmdHeader.ui32IntJobRef = ui32IntJobRef;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (psWorkEstKickData != NULL &&
+		RGXIsValidWorkloadEstCCBCommand(eCmdType))
+	{
+		sCmdHeader.sWorkEstKickData = *psWorkEstKickData;
+	}
+	else
+	{
+		sCmdHeader.sWorkEstKickData.ui16ReturnDataIndex = 0;
+		sCmdHeader.sWorkEstKickData.ui64Deadline = 0;
+		sCmdHeader.sWorkEstKickData.ui32CyclesPrediction = 0;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psWorkEstKickData);
+#endif
+
+	OSCachedMemCopy(pvCCB, &sCmdHeader, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+}
+
+/*
+	Reserve space in the CCB and fill in the command and client sync data
+*/
+PVRSRV_ERROR RGXCmdHelperAcquireCmdCCB(IMG_UINT32 ui32CmdCount,
+									   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+	void *pvStartPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(asCmdHelperData->psClientCCB->psServerCommonContext);
+	RGXFWIF_UFO asUFOs[RGXFWIF_CCB_CMD_MAX_UFOS];
+
+	/*
+		Check the number of fences & updates are valid.
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = &asCmdHelperData[i];
+
+		if (psCmdHelperData->ui32ClientFenceCount > RGXFWIF_CCB_CMD_MAX_UFOS ||
+		    psCmdHelperData->ui32ClientUpdateCount > RGXFWIF_CCB_CMD_MAX_UFOS)
+		{
+			return PVRSRV_ERROR_TOO_MANY_SYNCS;
+		}
+	}
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	ui32AllocSize = RGXCmdHelperGetCommandSize(ui32CmdCount, asCmdHelperData);
+
+#if defined(PDUMP)
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		if ((asCmdHelperData[0].ui32PDumpFlags ^ asCmdHelperData[i].ui32PDumpFlags) & PDUMP_FLAGS_CONTINUOUS)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PDump continuous is not consistent (%s != %s) for command %d",
+					 __func__,
+					 PDUMP_IS_CONTINUOUS(asCmdHelperData[0].ui32PDumpFlags)?"IMG_TRUE":"IMG_FALSE",
+					 PDUMP_IS_CONTINUOUS(asCmdHelperData[i].ui32PDumpFlags)?"IMG_TRUE":"IMG_FALSE",
+					 ui32CmdCount));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	/*
+		Acquire space in the CCB for all the command(s).
+	*/
+	eError = RGXAcquireCCB(asCmdHelperData[0].psClientCCB,
+						   ui32AllocSize,
+						   &pvStartPtr,
+						   asCmdHelperData[0].ui32PDumpFlags);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		return eError;
+	}
+
+	/*
+		For each command fill in the fence, DM, and update command
+
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = & asCmdHelperData[i];
+		void *pvCmdPtr;
+#if defined(PDUMP)
+		IMG_UINT32 ui32CtxAddr = FWCommonContextGetFWAddress(asCmdHelperData->psClientCCB->psServerCommonContext).ui32Addr;
+		IMG_UINT32 ui32CcbWoff = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(asCmdHelperData->psClientCCB->psServerCommonContext));
+#endif
+
+		if (psCmdHelperData->ui32ClientFenceCount+psCmdHelperData->ui32ClientUpdateCount != 0)
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "Start of %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+
+		pvCmdPtr = pvStartPtr;
+
+		/*
+			Create the fence command.
+		*/
+		if (psCmdHelperData->ui32ClientFenceCount > 0)
+		{
+			IMG_UINT k, uiNextValueIndex;
+			IMG_UINT32 ui32FenceCmdSize =
+				RGX_CCB_FWALLOC_ALIGN(psCmdHelperData->ui32ClientFenceCount * sizeof(RGXFWIF_UFO) +
+									  sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_FENCE,
+							  ui32FenceCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			/* Fill in the client fences */
+			uiNextValueIndex = 0;
+			for (k = 0; k < psCmdHelperData->ui32ClientFenceCount; k++)
+			{
+				PVR_ASSERT(k < RGXFWIF_CCB_CMD_MAX_UFOS);
+
+				asUFOs[k].puiAddrUFO = psCmdHelperData->pauiFenceUFOAddress[k];
+
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(psCmdHelperData->pauiFenceUFOAddress[k].ui32Addr))
+				{
+					asUFOs[k].ui32Value = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				}
+				else
+				{
+					/* Only increment uiNextValueIndex for non sync checkpoints
+					 * (as paui32FenceValue only contains values for sync prims)
+					 */
+					asUFOs[k].ui32Value = psCmdHelperData->paui32FenceValue[uiNextValueIndex++];
+				}
+
+#if defined(SYNC_COMMAND_DEBUG)
+				PVR_DPF((PVR_DBG_ERROR, "%s client sync fence - 0x%x -> 0x%x",
+						psCmdHelperData->psClientCCB->szName, asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value));
+#endif
+				PDUMPCOMMENT(psDevInfo->psDeviceNode,
+							 ".. %s client sync fence - 0x%x -> 0x%x",
+							 psCmdHelperData->psClientCCB->szName,
+							 asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value);
+
+
+			}
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&asUFOs, psCmdHelperData->ui32ClientFenceCount * sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, ui32FenceCmdSize);
+		}
+
+		/*
+			Create the FBSC invalidate command.
+		*/
+		if (psCmdHelperData->ui32FBSCInvalCmdSize)
+		{
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE,
+							  psCmdHelperData->ui32FBSCInvalCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->ui64FBSCEntryMask,
+							sizeof(psCmdHelperData->ui64FBSCEntryMask));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32FBSCInvalCmdSize);
+		}
+
+		/*
+		  Create the pre DM timestamp commands. Pre and Post timestamp commands are supposed to
+		  sandwich the DM cmd. The padding code with the CCB wrap upsets the FW if we don't have
+		  the task type bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+		*/
+		if (psCmdHelperData->ui32PreTimeStampCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP,
+							  psCmdHelperData->ui32PreTimeStampCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->pPreTimestampAddr.ui32Addr,
+							sizeof(psCmdHelperData->pPreTimestampAddr.ui32Addr));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32PreTimeStampCmdSize);
+
+		}
+
+		/*
+			Create the DM command
+		*/
+		if (psCmdHelperData->ui32DMCmdSize)
+		{
+			PVR_ASSERT(psCmdHelperData->ui32DMCmdSize == sizeof(RGXFWIF_CCB_CMD_HEADER) + psCmdHelperData->ui32CmdSize);
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  psCmdHelperData->eType,
+							  psCmdHelperData->ui32DMCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)) ? NULL : psCmdHelperData->psWorkEstKickData);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							psCmdHelperData->pui8DMCmd,
+							psCmdHelperData->ui32CmdSize);
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32DMCmdSize);
+		}
+
+
+		if (psCmdHelperData->ui32PostTimeStampCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP,
+							  psCmdHelperData->ui32PostTimeStampCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->pPostTimestampAddr.ui32Addr,
+							sizeof(psCmdHelperData->pPostTimestampAddr.ui32Addr));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32PostTimeStampCmdSize);
+
+		}
+
+
+		if (psCmdHelperData->ui32RMWUFOCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE,
+							  psCmdHelperData->ui32RMWUFOCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			asUFOs[0].puiAddrUFO = psCmdHelperData->pRMWUFOAddr;
+			asUFOs[0].ui32Value = 0;
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							asUFOs, sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32RMWUFOCmdSize);
+		}
+
+		/*
+			Create the update command.
+		*/
+		if (psCmdHelperData->ui32ClientUpdateCount > 0)
+		{
+			IMG_UINT k, uiNextValueIndex;
+			IMG_UINT32 ui32UpdateCmdSize =
+				RGX_CCB_FWALLOC_ALIGN(psCmdHelperData->ui32ClientUpdateCount * sizeof(RGXFWIF_UFO) +
+									  sizeof(RGXFWIF_CCB_CMD_HEADER));
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_UPDATE,
+							  ui32UpdateCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			/* Fill in the client updates */
+			uiNextValueIndex = 0;
+			for (k = 0; k < psCmdHelperData->ui32ClientUpdateCount; k++)
+			{
+				PVR_ASSERT(k < RGXFWIF_CCB_CMD_MAX_UFOS);
+
+				asUFOs[k].puiAddrUFO = psCmdHelperData->pauiUpdateUFOAddress[k];
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(psCmdHelperData->pauiUpdateUFOAddress[k].ui32Addr))
+				{
+					asUFOs[k].ui32Value = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				}
+				else
+				{
+					/* Only increment uiNextValueIndex for non sync checkpoints
+					 * (as paui32UpdateValue only contains values for sync prims)
+					 */
+					asUFOs[k].ui32Value = psCmdHelperData->paui32UpdateValue[uiNextValueIndex++];
+				}
+
+#if defined(SYNC_COMMAND_DEBUG)
+				PVR_DPF((PVR_DBG_ERROR, "%s client sync update - 0x%x -> 0x%x",
+						psCmdHelperData->psClientCCB->szName, asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value));
+#endif
+				PDUMPCOMMENT(psDevInfo->psDeviceNode,
+							 ".. %s client sync update - 0x%x -> 0x%x",
+							 psCmdHelperData->psClientCCB->szName,
+							 asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value);
+
+			}
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&asUFOs, psCmdHelperData->ui32ClientUpdateCount * sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, ui32UpdateCmdSize);
+		}
+
+		/* Set the start pointer for the next iteration around the loop */
+		pvStartPtr = IMG_OFFSET_ADDR(pvStartPtr, psCmdHelperData->ui32TotalSize);
+
+		if (psCmdHelperData->ui32ClientFenceCount+psCmdHelperData->ui32ClientUpdateCount != 0)
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "End of %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+		else
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "No %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	Fill in the server syncs data and release the CCB space
+*/
+void RGXCmdHelperReleaseCmdCCB(IMG_UINT32 ui32CmdCount,
+							   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+							   const IMG_CHAR *pcszDMName,
+							   IMG_UINT32 ui32CtxAddr)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	IMG_BOOL bTraceChecks = trace_rogue_are_fence_checks_traced();
+	IMG_BOOL bTraceUpdates = trace_rogue_are_fence_updates_traced();
+#endif
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	ui32AllocSize = RGXCmdHelperGetCommandSize(ui32CmdCount, asCmdHelperData);
+	/*
+		For each command write PDump comments and emit FTrace events for fence
+		checks and updates if they exist.
+	*/
+	for (i=0;i<ui32CmdCount;i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = &asCmdHelperData[i];
+#if defined(PDUMP) || (defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS))
+		PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psCmdHelperData->psClientCCB->psServerCommonContext);
+#endif
+
+#if (!defined(__linux__) || !defined(PDUMP))
+		PVR_UNREFERENCED_PARAMETER(psCmdHelperData);
+#endif
+
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		if (bTraceChecks)
+		{
+			trace_rogue_fence_checks(psCmdHelperData->pszCommandName,
+									 pcszDMName,
+									 psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+									 ui32CtxAddr,
+									 psCmdHelperData->psClientCCB->ui32HostWriteOffset + ui32AllocSize,
+									 psCmdHelperData->ui32ClientFenceCount,
+									 psCmdHelperData->pauiFenceUFOAddress,
+									 psCmdHelperData->paui32FenceValue);
+		}
+		if (bTraceUpdates)
+		{
+			trace_rogue_fence_updates(psCmdHelperData->pszCommandName,
+									  pcszDMName,
+									  psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+									  ui32CtxAddr,
+									  psCmdHelperData->psClientCCB->ui32HostWriteOffset + ui32AllocSize,
+									  psCmdHelperData->ui32ClientUpdateCount,
+									  psCmdHelperData->pauiUpdateUFOAddress,
+									  psCmdHelperData->paui32UpdateValue);
+		}
+#else
+	PVR_UNREFERENCED_PARAMETER(pcszDMName);
+	PVR_UNREFERENCED_PARAMETER(ui32CtxAddr);
+#endif
+
+		/*
+			All the commands have been filled in so release the CCB space.
+			The FW still won't run this command until we kick it
+		*/
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+				psCmdHelperData->ui32PDumpFlags,
+				"%s Command Server Release on FWCtx %08x",
+				psCmdHelperData->pszCommandName, ui32CtxAddr);
+	}
+
+	RGXReleaseCCB(asCmdHelperData[0].psClientCCB,
+				  ui32AllocSize,
+				  asCmdHelperData[0].ui32PDumpFlags);
+
+	BIT_UNSET(asCmdHelperData[0].psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+}
+
+IMG_UINT32 RGXCmdHelperGetCommandSize(IMG_UINT32              ui32CmdCount,
+                                      RGX_CCB_CMD_HELPER_DATA *asCmdHelperData)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		ui32AllocSize += asCmdHelperData[i].ui32TotalSize;
+	}
+
+	return ui32AllocSize;
+}
+
+/* Work out how much of an offset there is to a specific command. */
+IMG_UINT32 RGXCmdHelperGetCommandOffset(RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+                                        IMG_UINT32              ui32Cmdindex)
+{
+	return RGXCmdHelperGetCommandSize(ui32Cmdindex, asCmdHelperData);
+}
+
+/* Returns the offset of the data master command from a write offset */
+IMG_UINT32 RGXCmdHelperGetDMCommandHeaderOffset(RGX_CCB_CMD_HELPER_DATA *psCmdHelperData)
+{
+	return psCmdHelperData->ui32DMCmdOffset;
+}
+
+static const char *_CCBCmdTypename(RGXFWIF_CCB_CMD_TYPE cmdType)
+{
+	switch (cmdType)
+	{
+		case RGXFWIF_CCB_CMD_TYPE_GEOM: return "TA";
+		case RGXFWIF_CCB_CMD_TYPE_3D: return "3D";
+		case RGXFWIF_CCB_CMD_TYPE_3D_PR: return "3D_PR";
+		case RGXFWIF_CCB_CMD_TYPE_CDM: return "CDM";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_3D: return "TQ_3D";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_2D: return "TQ_2D";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_TDM: return "TQ_TDM";
+		case RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE: return "FBSC_INVALIDATE";
+		case RGXFWIF_CCB_CMD_TYPE_NULL: return "NULL";
+		case RGXFWIF_CCB_CMD_TYPE_FENCE: return "FENCE";
+		case RGXFWIF_CCB_CMD_TYPE_UPDATE: return "UPDATE";
+		case RGXFWIF_CCB_CMD_TYPE_FENCE_PR: return "FENCE_PR";
+		case RGXFWIF_CCB_CMD_TYPE_PRIORITY: return "PRIORITY";
+		case RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP: return "PRE_TIMESTAMP";
+		case RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE: return "RMW_UPDATE";
+		case RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP: return "POST_TIMESTAMP";
+		case RGXFWIF_CCB_CMD_TYPE_PADDING: return "PADDING";
+
+		default:
+			PVR_ASSERT(IMG_FALSE);
+		break;
+	}
+
+	return "INVALID";
+}
+
+PVRSRV_ERROR CheckForStalledCCB(PVRSRV_DEVICE_NODE *psDevNode, RGX_CLIENT_CCB *psCurrentClientCCB, RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;
+	IMG_UINT32					ui32SampledRdOff, ui32SampledDpOff, ui32SampledWrOff, ui32WrapMask;
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+
+	if (psCurrentClientCCB == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "CheckForStalledCCB: CCCB is NULL"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	/* If CCB grow is enabled, take the lock while sampling offsets
+	 * (to guard against a grow happening mid-sample)
+	 */
+	OSLockAcquire(psCurrentClientCCB->hCCBGrowLock);
+#endif
+	/* NB. use psCurrentClientCCB->ui32Size as basis for wrap mask (rather than psClientCCBCtrl->ui32WrapMask)
+	 * as if CCB grow happens, psCurrentClientCCB->ui32Size will have been updated but
+	 * psClientCCBCtrl->ui32WrapMask is only updated once the firmware sees the CCB has grown.
+	 * If we use the wrong value, we might incorrectly determine that the offsets are invalid.
+	 */
+	ui32WrapMask = RGXGetWrapMaskCCB(psCurrentClientCCB);
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	ui32SampledRdOff = psClientCCBCtrl->ui32ReadOffset;
+	ui32SampledDpOff = psClientCCBCtrl->ui32DepOffset;
+	ui32SampledWrOff = psCurrentClientCCB->ui32HostWriteOffset;
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+
+	if (ui32SampledRdOff > ui32WrapMask ||
+		ui32SampledDpOff > ui32WrapMask ||
+		ui32SampledWrOff > ui32WrapMask)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "CheckForStalledCCB: CCCB has invalid offset (ROFF=%d DOFF=%d WOFF=%d)",
+				ui32SampledRdOff, ui32SampledDpOff, ui32SampledWrOff));
+		return PVRSRV_ERROR_INVALID_OFFSET;
+	}
+
+	if (ui32SampledRdOff != ui32SampledWrOff &&
+				psCurrentClientCCB->ui32LastROff != psCurrentClientCCB->ui32LastWOff &&
+				ui32SampledRdOff == psCurrentClientCCB->ui32LastROff &&
+				(psCurrentClientCCB->ui32ByteCount - psCurrentClientCCB->ui32LastByteCount) < psCurrentClientCCB->ui32Size)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+
+		/* Only log a stalled CCB if GPU is idle (any state other than POW_ON is considered idle).
+		 * Guest drivers do not initialize psRGXFWIfFwSysData, so they assume FW internal state is ON. */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+		                           INVALIDATE);
+		if (((psDevInfo->psRGXFWIfFwSysData == NULL) || (psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_ON)) &&
+			(psDevInfo->ui32SLRHoldoffCounter == 0))
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Don't log this by default unless debugging since a higher up
+			 * function will log the stalled condition. Helps avoid double
+			 * messages in the log.
+			 */
+			PVR_DPF((PVR_DBG_ERROR, "%s (%s): CCCB has not progressed (ROFF=%d DOFF=%d WOFF=%d) for \"%s\"",
+					__func__, pszStalledAction, ui32SampledRdOff,
+					ui32SampledDpOff, ui32SampledWrOff,
+					psCurrentClientCCB->szName));
+			eError = PVRSRV_ERROR_CCCB_STALLED;
+
+			{
+				void				*pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+				RGXFWIF_CCB_CMD_HEADER	*psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledRdOff);
+				PVRSRV_RGXDEV_INFO		*psDevInfo = FWCommonContextGetRGXDevInfo(psCurrentClientCCB->psServerCommonContext);
+
+				/* Special case - if readOffset is on a PADDING packet, CCB has wrapped.
+				 * In this case, skip over the PADDING packet.
+				 */
+				if (psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_PADDING)
+				{
+					psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff,
+					                                             ((ui32SampledRdOff +
+					                                               psCommandHeader->ui32CmdSize +
+					                                               sizeof(RGXFWIF_CCB_CMD_HEADER))
+					                                              & psCurrentClientCCB->psClientCCBCtrl->ui32WrapMask));
+				}
+
+				/* Only try to recover a 'stalled' context (ie one waiting on a fence), as some work (eg compute) could
+				 * take a long time to complete, during which time the CCB ptrs would not advance.
+				 */
+				if (((psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE) ||
+				     (psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR)) &&
+				    (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff)))
+				{
+					/* Acquire the cCCB recovery lock */
+					OSLockAcquire(psDevInfo->hCCBRecoveryLock);
+
+					if (!psDevInfo->pvEarliestStalledClientCCB)
+					{
+						psDevInfo->pvEarliestStalledClientCCB = (void*)psCurrentClientCCB;
+						psDevInfo->ui32OldestSubmissionOrdinal = psCommandHeader->ui32IntJobRef;
+					}
+					else
+					{
+						/* Check if this fence cmd header has an older submission stamp than the one we are currently considering unblocking
+						 * (account for submission stamp wrap by checking diff is less than 0x80000000) - if it is older, then this becomes
+						 * our preferred fence to be unblocked/
+						 */
+						if ((psCommandHeader->ui32IntJobRef < psDevInfo->ui32OldestSubmissionOrdinal) &&
+						    ((psDevInfo->ui32OldestSubmissionOrdinal - psCommandHeader->ui32IntJobRef) < 0x8000000))
+						{
+							psDevInfo->pvEarliestStalledClientCCB = (void*)psCurrentClientCCB;
+							psDevInfo->ui32OldestSubmissionOrdinal = psCommandHeader->ui32IntJobRef;
+						}
+					}
+
+					/* Release the cCCB recovery lock */
+					OSLockRelease(psDevInfo->hCCBRecoveryLock);
+				}
+			}
+		}
+	}
+
+	psCurrentClientCCB->ui32LastROff = ui32SampledRdOff;
+	psCurrentClientCCB->ui32LastWOff = ui32SampledWrOff;
+	psCurrentClientCCB->ui32LastByteCount = psCurrentClientCCB->ui32ByteCount;
+
+	return eError;
+}
+
+void DumpCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+			PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+			RGX_CLIENT_CCB *psCurrentClientCCB,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	volatile RGXFWIF_CCCB_CTL *psClientCCBCtrl;
+	void *pvClientCCBBuff;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32DepOffset;
+	IMG_UINT32 ui32EndOffset;
+	IMG_UINT32 ui32WrapMask;
+	IMG_CHAR * pszState = "Ready";
+
+	/* Ensure hCCBGrowLock is acquired before reading
+	 * psCurrentClientCCB->pvClientCCB as a CCB grow
+	 * could remap the virtual addresses.
+	 */
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockAcquire(psCurrentClientCCB->hCCBGrowLock);
+#endif
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+	ui32EndOffset = psCurrentClientCCB->ui32HostWriteOffset;
+	OSMemoryBarrier(NULL);
+	ui32Offset = psClientCCBCtrl->ui32ReadOffset;
+	ui32DepOffset = psClientCCBCtrl->ui32DepOffset;
+	/* NB. Use psCurrentClientCCB->ui32Size as basis for wrap mask (rather
+	 * than psClientCCBCtrl->ui32WrapMask) as if CCB grow happened,
+	 * psCurrentClientCCB->ui32Size will have been updated but
+	 * psClientCCBCtrl->ui32WrapMask is only updated once the firmware
+	 * sees the CCB has grown. If we use the wrong value, ui32NextOffset
+	 * can end up being wrapped prematurely and pointing to garbage.
+	 */
+	ui32WrapMask = RGXGetWrapMaskCCB(psCurrentClientCCB);
+
+	PVR_DUMPDEBUG_LOG("FWCtx 0x%08X (%s)", sFWCommonContext.ui32Addr, psCurrentClientCCB->szName);
+	if (ui32Offset == ui32EndOffset)
+	{
+		PVR_DUMPDEBUG_LOG("  `--<Empty>");
+	}
+
+	if (ui32Offset > ui32WrapMask)
+	{
+		PVR_DUMPDEBUG_LOG("  `--<Invalid offset>");
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+		OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+		return;
+	}
+
+	while (ui32Offset != ui32EndOffset)
+	{
+		RGXFWIF_CCB_CMD_HEADER *psCmdHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32Offset);
+		IMG_UINT32 ui32NextOffset = (ui32Offset + psCmdHeader->ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER)) & ui32WrapMask;
+		IMG_BOOL bLastCommand = (ui32NextOffset == ui32EndOffset)? IMG_TRUE: IMG_FALSE;
+		IMG_BOOL bLastUFO;
+		#define CCB_SYNC_INFO_LEN 80
+		IMG_CHAR pszSyncInfo[CCB_SYNC_INFO_LEN];
+		IMG_UINT32 ui32NoOfUpdates, i;
+		RGXFWIF_UFO *psUFOPtr;
+
+		ui32NoOfUpdates = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+		psUFOPtr = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32Offset + sizeof(RGXFWIF_CCB_CMD_HEADER));
+		pszSyncInfo[0] = '\0';
+
+		if (ui32Offset == ui32DepOffset)
+		{
+			pszState = "Waiting";
+		}
+
+		PVR_DUMPDEBUG_LOG("  %s--%s %s @ %u Int=%u Ext=%u",
+			bLastCommand? "`": "|",
+			pszState, _CCBCmdTypename(psCmdHeader->eCmdType),
+			ui32Offset, psCmdHeader->ui32IntJobRef, psCmdHeader->ui32ExtJobRef
+			);
+
+		/* switch on type and write checks and updates */
+		switch (psCmdHeader->eCmdType)
+		{
+			case RGXFWIF_CCB_CMD_TYPE_UPDATE:
+			case RGXFWIF_CCB_CMD_TYPE_FENCE:
+			case RGXFWIF_CCB_CMD_TYPE_FENCE_PR:
+			{
+				for (i = 0; i < ui32NoOfUpdates; i++, psUFOPtr++)
+				{
+					bLastUFO = (ui32NoOfUpdates-1 == i)? IMG_TRUE: IMG_FALSE;
+
+					if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+					{
+						if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+						{
+							SyncCheckpointRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+						else
+						{
+							SyncRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+					}
+
+					PVR_DUMPDEBUG_LOG("  %s  %s--Addr:0x%08x Val=0x%08x %s",
+						bLastCommand? " ": "|",
+						bLastUFO? "`": "|",
+						psUFOPtr->puiAddrUFO.ui32Addr, psUFOPtr->ui32Value,
+						pszSyncInfo
+						);
+				}
+				break;
+			}
+			case RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE:
+			{
+				for (i = 0; i < ui32NoOfUpdates; i++, psUFOPtr++)
+				{
+					bLastUFO = (ui32NoOfUpdates-1 == i)? IMG_TRUE: IMG_FALSE;
+
+					if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+					{
+						if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+						{
+							SyncCheckpointRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+						else
+						{
+							SyncRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+					}
+
+					PVR_DUMPDEBUG_LOG("  %s  %s--Addr:0x%08x Val++ %s",
+						bLastCommand? " ": "|",
+						bLastUFO? "`": "|",
+						psUFOPtr->puiAddrUFO.ui32Addr,
+						pszSyncInfo
+						);
+				}
+				break;
+			}
+			default:
+				break;
+		}
+
+		/* Check the command size is valid, otherwise if corruption was present this loop might hang... */
+		if ((ui32Offset == ui32NextOffset) ||
+		    ((ui32Offset + psCmdHeader->ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER)) > ui32WrapMask+1))
+		{
+			PVR_DUMPDEBUG_LOG("  `--Invalid CCB offset!");
+			break;
+		}
+
+		ui32Offset = ui32NextOffset;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+}
+
+void DumpFirstCCBCmd(PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+				RGX_CLIENT_CCB *psCurrentClientCCB,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;
+	void					*pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+	IMG_UINT32					ui32SampledWrOff = psCurrentClientCCB->ui32HostWriteOffset;
+	IMG_UINT32					ui32SampledRdOff;
+	IMG_UINT32					ui32SampledDepOff;
+
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	ui32SampledRdOff = psClientCCBCtrl->ui32ReadOffset;
+	ui32SampledDepOff = psClientCCBCtrl->ui32DepOffset;
+
+	if ((ui32SampledRdOff == ui32SampledDepOff) &&
+		(ui32SampledRdOff != ui32SampledWrOff))
+	{
+		volatile RGXFWIF_CCB_CMD_HEADER *psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledRdOff);
+		RGXFWIF_CCB_CMD_TYPE	eCommandType = psCommandHeader->eCmdType;
+		volatile void				*pvPtr = psCommandHeader;
+
+		/* CCB is stalled on a fence... */
+		if ((eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE) || (eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR))
+		{
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+			PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psCurrentClientCCB->psServerCommonContext);
+			IMG_UINT32 ui32Val;
+#endif
+			RGXFWIF_UFO *psUFOPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader));
+			IMG_UINT32 jj;
+
+			/* Display details of the fence object on which the context is pending */
+			PVR_DUMPDEBUG_LOG("FWCtx 0x%08X @ %d (%s) pending on %s:",
+							   sFWCommonContext.ui32Addr,
+							   ui32SampledRdOff,
+							   psCurrentClientCCB->szName,
+							   _CCBCmdTypename(eCommandType));
+			for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+			{
+#if !defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+				PVR_DUMPDEBUG_LOG("  Addr:0x%08x  Value=0x%08x",psUFOPtr[jj].puiAddrUFO.ui32Addr, psUFOPtr[jj].ui32Value);
+#else
+				ui32Val = 0;
+				RGXReadFWModuleAddr(psDevInfo, psUFOPtr[jj].puiAddrUFO.ui32Addr, &ui32Val);
+				PVR_DUMPDEBUG_LOG("  Addr:0x%08x Value(Host)=0x%08x Value(FW)=0x%08x",
+				                   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+				                   psUFOPtr[jj].ui32Value, ui32Val);
+#endif
+			}
+
+			/* Advance psCommandHeader past the FENCE to the next command header (this will be the TA/3D command that is fenced) */
+			pvPtr = IMG_OFFSET_ADDR(psUFOPtr, psCommandHeader->ui32CmdSize);
+			psCommandHeader = pvPtr;
+			if (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff))
+			{
+				PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X fenced command is of type %s",sFWCommonContext.ui32Addr, _CCBCmdTypename(psCommandHeader->eCmdType));
+				/* Advance psCommandHeader past the TA/3D to the next command header (this will possibly be an UPDATE) */
+				pvPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader) + psCommandHeader->ui32CmdSize);
+				psCommandHeader = pvPtr;
+				/* If the next command is an update, display details of that so we can see what would then become unblocked */
+				if (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff))
+				{
+					eCommandType = psCommandHeader->eCmdType;
+
+					if (eCommandType == RGXFWIF_CCB_CMD_TYPE_UPDATE)
+					{
+						psUFOPtr = IMG_OFFSET_ADDR(psCommandHeader, sizeof(*psCommandHeader));
+						PVR_DUMPDEBUG_LOG(" preventing %s:",_CCBCmdTypename(eCommandType));
+						for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+						{
+#if !defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+							PVR_DUMPDEBUG_LOG("  Addr:0x%08x  Value=0x%08x",psUFOPtr[jj].puiAddrUFO.ui32Addr, psUFOPtr[jj].ui32Value);
+#else
+							ui32Val = 0;
+							RGXReadFWModuleAddr(psDevInfo, psUFOPtr[jj].puiAddrUFO.ui32Addr, &ui32Val);
+							PVR_DUMPDEBUG_LOG("  Addr:0x%08x Value(Host)=0x%08x Value(FW)=0x%08x",
+							                   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							                   psUFOPtr[jj].ui32Value,
+							                   ui32Val);
+#endif
+						}
+					}
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X has no further commands",sFWCommonContext.ui32Addr);
+				}
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X has no further commands",sFWCommonContext.ui32Addr);
+			}
+		}
+	}
+}
+
+void DumpStalledContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_CLIENT_CCB *psStalledClientCCB;
+
+	PVR_ASSERT(psDevInfo);
+
+	psStalledClientCCB = (RGX_CLIENT_CCB *)psDevInfo->pvEarliestStalledClientCCB;
+
+	if (psStalledClientCCB)
+	{
+		volatile RGXFWIF_CCCB_CTL *psClientCCBCtrl;
+		IMG_UINT32 ui32SampledDepOffset;
+		void                 *pvPtr;
+		RGXFWIF_CCB_CMD_HEADER    *psCommandHeader;
+		RGXFWIF_CCB_CMD_TYPE      eCommandType;
+
+		RGXFwSharedMemCacheOpValue(psStalledClientCCB->psClientCCBCtrl->ui32DepOffset,
+		                           INVALIDATE);
+		psClientCCBCtrl = psStalledClientCCB->psClientCCBCtrl;
+		ui32SampledDepOffset = psClientCCBCtrl->ui32DepOffset;
+		/* No need to invalidate CCCB as FW doesn't write to it, only read. */
+		pvPtr = IMG_OFFSET_ADDR(psStalledClientCCB->pvClientCCB, ui32SampledDepOffset);
+		psCommandHeader = pvPtr;
+		eCommandType = psCommandHeader->eCmdType;
+
+		if ((eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE) || (eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR))
+		{
+			RGXFWIF_UFO *psUFOPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader));
+			IMG_UINT32 jj;
+			IMG_UINT32 ui32NumUnsignalledUFOs = 0;
+			IMG_UINT32 ui32UnsignalledUFOVaddrs[PVRSRV_MAX_SYNCS];
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+			                         INVALIDATE);
+			if (!psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.aszCCBName[0])
+			{
+				OSClockMonotonicns64(&psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui64Timestamp);
+				psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui32NumUFOs = (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO));
+				psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui32FWCtxAddr = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr;
+				OSStringSafeCopy(psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.aszCCBName,
+				              psStalledClientCCB->szName,
+				              MAX_CLIENT_CCB_NAME);
+			}
+			else
+			{
+				OSClockMonotonicns64(&psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui64Timestamp);
+				psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui32NumUFOs = (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO));
+				psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui32FWCtxAddr = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr;
+				OSStringSafeCopy(psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].aszCCBName,
+				              psStalledClientCCB->szName,
+				              MAX_CLIENT_CCB_NAME);
+				psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp = (psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp + 1) % PVR_SLR_LOG_ENTRIES;
+			}
+			psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested++;
+			/* flush write buffers for psRGXFWIfFwOsData */
+			OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp]);
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+			                         FLUSH);
+#endif
+			PVR_LOG(("Fence found on context 0x%x '%s' @ %d has %d UFOs",
+			         FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr,
+			         psStalledClientCCB->szName, ui32SampledDepOffset,
+			         (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO))));
+
+			for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+			{
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT((RGXFWIF_UFO *)&psUFOPtr[jj]))
+				{
+					IMG_UINT32 ui32ReadValue = SyncCheckpointStateFromUFO(psDevInfo->psDeviceNode,
+					                                           psUFOPtr[jj].puiAddrUFO.ui32Addr);
+					PVR_LOG(("  %d/%d FWAddr 0x%x requires 0x%x (currently 0x%x)", jj+1,
+							   (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO)),
+							   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							   psUFOPtr[jj].ui32Value,
+							   ui32ReadValue));
+					/* If fence is unmet, dump debug info on it */
+					if (ui32ReadValue != psUFOPtr[jj].ui32Value)
+					{
+						/* Add to our list to pass to pvr_sync */
+						ui32UnsignalledUFOVaddrs[ui32NumUnsignalledUFOs] = psUFOPtr[jj].puiAddrUFO.ui32Addr;
+						ui32NumUnsignalledUFOs++;
+					}
+				}
+				else
+				{
+					PVR_LOG(("  %d/%d FWAddr 0x%x requires 0x%x", jj+1,
+							   (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO)),
+							   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							   psUFOPtr[jj].ui32Value));
+				}
+			}
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_FALLBACK_FENCE_SYNC)
+			if (ui32NumUnsignalledUFOs > 0)
+			{
+				IMG_UINT32 ui32NumSyncsOwned;
+				PVRSRV_ERROR eErr = SyncCheckpointDumpInfoOnStalledUFOs(ui32NumUnsignalledUFOs, &ui32UnsignalledUFOVaddrs[0], &ui32NumSyncsOwned);
+
+				PVR_LOG_IF_ERROR(eErr, "SyncCheckpointDumpInfoOnStalledUFOs() call failed.");
+			}
+#endif
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+			if (BIT_ISSET(psStalledClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED))
+			{
+				PRGXFWIF_FWCOMMONCONTEXT psContext = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext);
+
+				PVR_LOG(("SLR disabled for FWCtx 0x%08X", psContext.ui32Addr));
+			}
+			else
+			{
+				if (ui32NumUnsignalledUFOs > 0)
+				{
+					RGXFWIF_KCCB_CMD sSignalFencesCmd;
+
+					sSignalFencesCmd.eCmdType = RGXFWIF_KCCB_CMD_FORCE_UPDATE;
+					sSignalFencesCmd.ui32KCCBFlags = 0;
+					sSignalFencesCmd.uCmdData.sForceUpdateData.psContext = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext);
+					sSignalFencesCmd.uCmdData.sForceUpdateData.ui32CCBFenceOffset = ui32SampledDepOffset;
+
+					PVR_LOG(("Forced update command issued for FWCtx 0x%08X", sSignalFencesCmd.uCmdData.sForceUpdateData.psContext.ui32Addr));
+
+					RGXScheduleCommand(FWCommonContextGetRGXDevInfo(psStalledClientCCB->psServerCommonContext),
+					                   RGXFWIF_DM_GP,
+					                   &sSignalFencesCmd,
+					                   PDUMP_FLAGS_CONTINUOUS);
+				}
+			}
+#endif
+		}
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+}
+
+/******************************************************************************
+ End of file (rgxccb.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.h
new file mode 100644
index 000000000000..e9edd9a37bc3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxccb.h
@@ -0,0 +1,357 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Circular Command Buffer functionality.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Circular Command Buffer functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXCCB_H)
+#define RGXCCB_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "rgxdefs_km.h"
+#include "pvr_notifier.h"
+
+#define MAX_CLIENT_CCB_NAME	30
+#define SYNC_FLAG_MASK_ALL  IMG_UINT32_MAX
+
+/*
+ * This size is to be used when a client CCB is found to consume very
+ * negligible space (e.g. a few hundred bytes to few KBs - less than a page).
+ * In such a case, instead of allocating CCB of size of only a few KBs, we
+ * allocate at-least this much to be future risk-free.
+ */
+#define MIN_SAFE_CCB_SIZE_LOG2         13  /* 8K (2 Pages) */
+#define MAX_SAFE_CCB_SIZE_LOG2         18  /* 256K (64 Pages) */
+
+#define RGX_TQ3D_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D
+static_assert(RGX_TQ3D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TQ3D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ3D CCB size is invalid");
+#define RGX_TQ3D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ3D
+static_assert(RGX_TQ3D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D
+	&& RGX_TQ3D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ3D max CCB size is invalid");
+
+#define RGX_TQ2D_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D
+static_assert(RGX_TQ2D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TQ2D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ2D CCB size is invalid");
+#define RGX_TQ2D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ2D
+static_assert(RGX_TQ2D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D &&
+	RGX_TQ2D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ2D max CCB size is invalid");
+
+#define RGX_CDM_CCB_SIZE_LOG2          PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM
+static_assert(RGX_CDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_CDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "CDM CCB size is invalid");
+#define RGX_CDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_CDM
+static_assert(RGX_CDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM &&
+	RGX_CDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "CDM max CCB size is invalid");
+
+#define RGX_TA_CCB_SIZE_LOG2           PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA
+static_assert(RGX_TA_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TA_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TA CCB size is invalid");
+#define RGX_TA_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TA
+static_assert(RGX_TA_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA &&
+	RGX_TA_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TA max CCB size is invalid");
+
+#define RGX_3D_CCB_SIZE_LOG2           PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D
+static_assert(RGX_3D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_3D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "3D CCB size is invalid");
+#define RGX_3D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_3D
+static_assert(RGX_3D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D &&
+	RGX_3D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "3D max CCB size is invalid");
+
+#define RGX_KICKSYNC_CCB_SIZE_LOG2     PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC
+static_assert(RGX_KICKSYNC_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_KICKSYNC_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "KickSync CCB size is invalid");
+#define RGX_KICKSYNC_CCB_MAX_SIZE_LOG2	PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_KICKSYNC
+static_assert(RGX_KICKSYNC_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC &&
+	RGX_KICKSYNC_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "KickSync max CCB size is invalid");
+
+#define RGX_TDM_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM
+static_assert(RGX_TDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TDM CCB size is invalid");
+#define RGX_TDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TDM
+static_assert(RGX_TDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM &&
+	RGX_TDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TDM max CCB size is invalid");
+
+#define RGX_RDM_CCB_SIZE_LOG2          PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM
+static_assert(RGX_RDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_RDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "RDM CCB size is invalid");
+#define RGX_RDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_RDM
+static_assert(RGX_RDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM &&
+	RGX_RDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "RDM max CCB size is invalid");
+
+typedef struct _RGX_CLIENT_CCB_ RGX_CLIENT_CCB;
+
+/*
+	This structure is declared here as it's allocated on the heap by
+	the callers
+*/
+
+typedef struct _RGX_CCB_CMD_HELPER_DATA_ {
+	/* Data setup at command init time */
+	RGX_CLIENT_CCB			*psClientCCB;
+	IMG_CHAR				*pszCommandName;
+	IMG_UINT32				ui32PDumpFlags;
+
+	IMG_UINT32				ui32ClientFenceCount;
+	PRGXFWIF_UFO_ADDR		*pauiFenceUFOAddress;
+	IMG_UINT32				*paui32FenceValue;
+	IMG_UINT32				ui32ClientUpdateCount;
+	PRGXFWIF_UFO_ADDR		*pauiUpdateUFOAddress;
+	IMG_UINT32				*paui32UpdateValue;
+	RGXFWIF_CCB_CMD_TYPE	eType;
+	IMG_UINT32				ui32CmdSize;
+	IMG_UINT8				*pui8DMCmd;
+	IMG_UINT32				ui32FBSCInvalCmdSize;
+	IMG_UINT32				ui32DMCmdSize;
+	IMG_UINT32				ui32TotalSize;
+	IMG_UINT32				ui32DMCmdOffset;
+
+	/* data for FBSC invalidate command */
+	IMG_UINT64				ui64FBSCEntryMask;
+
+	/* timestamp commands */
+	PRGXFWIF_TIMESTAMP_ADDR pPreTimestampAddr;
+	IMG_UINT32              ui32PreTimeStampCmdSize;
+	PRGXFWIF_TIMESTAMP_ADDR pPostTimestampAddr;
+	IMG_UINT32              ui32PostTimeStampCmdSize;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+	IMG_UINT32              ui32RMWUFOCmdSize;
+
+	/* Job reference fields */
+	IMG_UINT32				ui32ExtJobRef;
+	IMG_UINT32				ui32IntJobRef;
+
+	/* FW Memdesc for Workload information */
+	RGXFWIF_WORKEST_KICK_DATA	*psWorkEstKickData;
+
+} RGX_CCB_CMD_HELPER_DATA;
+
+#define PADDING_COMMAND_SIZE	(sizeof(RGXFWIF_CCB_CMD_HEADER))
+
+
+#define RGX_CCB_REQUESTORS(TYPE) \
+	/* for debugging purposes */ TYPE(UNDEF)	\
+	TYPE(TA)	\
+	TYPE(3D)	\
+	TYPE(CDM)	\
+	TYPE(SH)	\
+	TYPE(RS)	\
+	TYPE(TQ_3D)	\
+	TYPE(TQ_2D)	\
+	TYPE(TQ_TDM)	\
+	TYPE(KICKSYNC)	\
+	TYPE(RAY)	\
+
+/* Forms an enum constant for each type present in RGX_CCB_REQUESTORS list. The enum is mainly used as
+   an index to the aszCCBRequestors table defined in rgxccb.c. The total number of enums must adhere
+   to the following build assert.
+*/
+typedef enum _RGX_CCB_REQUESTOR_TYPE_
+{
+#define CONSTRUCT_ENUM(req) REQ_TYPE_##req,
+	RGX_CCB_REQUESTORS (CONSTRUCT_ENUM)
+#undef CONSTRUCT_ENUM
+
+	/* should always be at the end */
+	REQ_TYPE_TOTAL_COUNT,
+} RGX_CCB_REQUESTOR_TYPE;
+
+/* Tuple describing the columns of the following table */
+typedef enum _RGX_CCB_REQUESTOR_TUPLE_
+{
+	REQ_RGX_FW_CLIENT_CCB_STRING,          /* Index to comment to be dumped in DevMemAllocs when allocating FirmwareClientCCB for this requestor */
+	REQ_RGX_FW_CLIENT_CCB_CONTROL_STRING,  /* Index to comment to be dumped in DevMemAllocs when allocating FirmwareClientCCBControl for this requestor */
+	REQ_PDUMP_COMMENT,                     /* Index to comment to be dumped in PDUMPs */
+
+	/* should always be at the end */
+	REQ_TUPLE_CARDINALITY,
+} RGX_CCB_REQUESTOR_TUPLE;
+
+/* Unpack U8 values from U32. */
+#define U32toU8_Unpack1(U32Packed) (U32Packed & 0xFF)
+#define U32toU8_Unpack2(U32Packed) ((U32Packed>>8) & 0xFF)
+#define U32toU8_Unpack3(U32Packed) ((U32Packed>>16) & 0xFF)
+#define U32toU8_Unpack4(U32Packed) ((U32Packed>>24) & 0xFF)
+
+/* Defines for bit meanings within the ui32CCBFlags member of struct _RGX_CLIENT_CCB_
+ *
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *   31                             10
+ *    |                             ||
+ *    ------------------------------XX
+ *  Bit   Meaning
+ *    0 = If set, CCB is still open and commands will be appended to it
+ *    1 = If set, do not perform Sync Lockup Recovery (SLR) for this CCB
+ */
+#define CCB_FLAGS_CCB_STATE_OPEN (0)  /*!< This bit is set to indicate CCB is in the 'Open' state. */
+#define CCB_FLAGS_SLR_DISABLED   (1)  /*!< This bit is set to disable Sync Lockup Recovery (SLR) for this CCB. */
+
+
+/*	Table containing an array of strings for each requestor type in the list of RGX_CCB_REQUESTORS. In addition to its use in
+	this module (rgxccb.c), this table is also used to access string to be dumped in PDUMP comments, hence, marking it extern for
+	use in other modules.
+*/
+extern const IMG_CHAR *const aszCCBRequestors[][REQ_TUPLE_CARDINALITY];
+
+PVRSRV_ERROR RGXCCBPDumpDrainCCB(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR RGXCreateCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+						  IMG_UINT32			ui32CCBSizeLog2,
+						  IMG_UINT32			ui32CCBMaxSizeLog2,
+						  IMG_UINT32			ui32ContextFlags,
+						  CONNECTION_DATA		*psConnectionData,
+						  RGX_CCB_REQUESTOR_TYPE	eCCBRequestor,
+						  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+						  RGX_CLIENT_CCB		**ppsClientCCB,
+						  DEVMEM_MEMDESC		**ppsClientCCBMemDesc,
+						  DEVMEM_MEMDESC		**ppsClientCCBCtlMemDesc);
+
+void RGXDestroyCCB(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_CLIENT_CCB *psClientCCB);
+
+PVRSRV_ERROR RGXCheckSpaceCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32CmdSize);
+
+PVRSRV_ERROR RGXAcquireCCB(RGX_CLIENT_CCB *psClientCCB,
+						   IMG_UINT32		ui32CmdSize,
+						   void				**ppvBufferSpace,
+						   IMG_UINT32		ui32PDumpFlags);
+
+void RGXReleaseCCB(RGX_CLIENT_CCB *psClientCCB,
+								IMG_UINT32		ui32CmdSize,
+								IMG_UINT32		ui32PDumpFlags);
+
+IMG_UINT32 RGXGetHostWriteOffsetCCB(RGX_CLIENT_CCB *psClientCCB);
+IMG_UINT32 RGXGetWrapMaskCCB(RGX_CLIENT_CCB *psClientCCB);
+
+void RGXSetCCBFlags(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32		ui32Flags);
+
+void RGXCmdHelperInitCmdCCB_CommandSize(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT64 ui64FBSCEntryMask,
+                                        IMG_UINT32 ui32ClientFenceCount,
+                                        IMG_UINT32 ui32ClientUpdateCount,
+                                        IMG_UINT32 ui32CmdSize,
+                                        PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                                        PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                                        PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                                        RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void RGXCmdHelperInitCmdCCB_OtherData(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      RGX_CLIENT_CCB *psClientCCB,
+                                      IMG_UINT32 ui32ClientFenceCount,
+                                      PRGXFWIF_UFO_ADDR *pauiFenceUFOAddress,
+                                      IMG_UINT32 *paui32FenceValue,
+                                      IMG_UINT32 ui32ClientUpdateCount,
+                                      PRGXFWIF_UFO_ADDR *pauiUpdateUFOAddress,
+                                      IMG_UINT32 *paui32UpdateValue,
+                                      IMG_UINT32 ui32CmdSize,
+                                      IMG_PBYTE pui8DMCmd,
+                                      PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                                      PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                                      PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                                      RGXFWIF_CCB_CMD_TYPE eType,
+                                      IMG_UINT32 ui32ExtJobRef,
+                                      IMG_UINT32 ui32IntJobRef,
+                                      IMG_UINT32 ui32PDumpFlags,
+                                      RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                                      IMG_CHAR *pszCommandName,
+                                      IMG_BOOL bCCBStateOpen,
+                                      RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void RGXCmdHelperInitCmdCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+							RGX_CLIENT_CCB          *psClientCCB,
+                            IMG_UINT64              ui64FBSCEntryMask,
+                            IMG_UINT32              ui32ClientFenceCount,
+                            PRGXFWIF_UFO_ADDR       *pauiFenceUFOAddress,
+                            IMG_UINT32              *paui32FenceValue,
+                            IMG_UINT32              ui32ClientUpdateCount,
+                            PRGXFWIF_UFO_ADDR       *pauiUpdateUFOAddress,
+                            IMG_UINT32              *paui32UpdateValue,
+                            IMG_UINT32              ui32CmdSize,
+                            IMG_UINT8               *pui8DMCmd,
+                            PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                            PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                            PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                            RGXFWIF_CCB_CMD_TYPE    eType,
+                            IMG_UINT32              ui32ExtJobRef,
+                            IMG_UINT32              ui32IntJobRef,
+                            IMG_UINT32              ui32PDumpFlags,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                            IMG_CHAR                *pszCommandName,
+                            IMG_BOOL                bCCBStateOpen,
+                            RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+PVRSRV_ERROR RGXCmdHelperAcquireCmdCCB(IMG_UINT32 ui32CmdCount,
+									   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData);
+
+void RGXCmdHelperReleaseCmdCCB(IMG_UINT32 ui32CmdCount,
+							   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+							   const IMG_CHAR *pcszDMName,
+							   IMG_UINT32 ui32CtxAddr);
+
+IMG_UINT32 RGXCmdHelperGetCommandSize(IMG_UINT32 ui32CmdCount,
+								   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData);
+
+IMG_UINT32 RGXCmdHelperGetCommandOffset(RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+                                        IMG_UINT32              ui32Cmdindex);
+
+IMG_UINT32 RGXCmdHelperGetDMCommandHeaderOffset(RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void DumpFirstCCBCmd(PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+				RGX_CLIENT_CCB  *psCurrentClientCCB,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile);
+
+void DumpCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+			PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+			RGX_CLIENT_CCB *psCurrentClientCCB,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile);
+
+PVRSRV_ERROR CheckForStalledCCB(PVRSRV_DEVICE_NODE *psDevNode, RGX_CLIENT_CCB  *psCurrentClientCCB, RGX_KICK_TYPE_DM eKickTypeDM);
+
+void DumpStalledContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif /* RGXCCB_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.c
new file mode 100644
index 000000000000..65ff0cf76d53
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.c
@@ -0,0 +1,1872 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Compute routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Compute routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvkm.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxcompute.h"
+#include "rgx_bvnc_defs_km.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "htbserver.h"
+#include "rgxshader.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_memallocflags.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_CMP_UFO_DUMP	0
+
+//#define CMP_CHECKPOINT_DEBUG 1
+
+#if defined(CMP_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+struct _RGX_SERVER_COMPUTE_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	DEVMEM_MEMDESC				*psFWComputeContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC				*psFWComputeContextStateMemDesc;
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListFence;
+	SYNC_ADDR_LIST				sSyncAddrListUpdate;
+	POS_LOCK					hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+PVRSRV_ERROR PVRSRVRGXCreateComputeContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE			*psDeviceNode,
+											 IMG_INT32					i32Priority,
+											 IMG_UINT32					ui32FrameworkCommandSize,
+											 IMG_PBYTE					pbyFrameworkCommand,
+											 IMG_HANDLE					hMemCtxPrivData,
+											 IMG_UINT32					ui32StaticComputeContextStateSize,
+											 IMG_PBYTE					pStaticComputeContextState,
+											 IMG_UINT32					ui32PackedCCBSizeU88,
+											 IMG_UINT32					ui32ContextFlags,
+											 IMG_UINT64					ui64RobustnessAddress,
+											 IMG_UINT32					ui32MaxDeadlineMS,
+											 RGX_SERVER_COMPUTE_CONTEXT	**ppsComputeContext)
+{
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext;
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+	RGXFWIF_FWCOMPUTECONTEXT	*psFWComputeContext;
+	IMG_UINT32					ui32CCBAllocSizeLog2, ui32CCBMaxAllocSizeLog2;
+
+	/* Prepare cleanup struct */
+	*ppsComputeContext = NULL;
+
+	psComputeContext = OSAllocZMem(sizeof(*psComputeContext));
+	if (psComputeContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW compute context, this has the CDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWCOMPUTECONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwComputeContext",
+			&psComputeContext->psFWComputeContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwcomputecontext;
+	}
+
+	eError = OSLockCreate(&psComputeContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	psComputeContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware compute context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_COMPUTECTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwComputeContextState",
+							  &psComputeContext->psFWComputeContextStateMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate firmware GPU context suspend state (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_contextsuspendalloc;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitCompute(psDevInfo, &psComputeContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psComputeContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psComputeContext->psFWFrameworkMemDesc,
+				pbyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psComputeContext->psFWFrameworkMemDesc;
+	}
+
+	ui32CCBAllocSizeLog2 = U32toU8_Unpack1(ui32PackedCCBSizeU88);
+	ui32CCBMaxAllocSizeLog2 = U32toU8_Unpack2(ui32PackedCCBSizeU88);
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_CDM,
+									 RGXFWIF_DM_CDM,
+									 hMemCtxPrivData,
+									 psComputeContext->psFWComputeContextMemDesc,
+									 offsetof(RGXFWIF_FWCOMPUTECONTEXT, sCDMContext),
+									 psFWMemContextMemDesc,
+									 psComputeContext->psFWComputeContextStateMemDesc,
+									 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_CDM_CCB_SIZE_LOG2,
+									 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_CDM_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 i32Priority,
+									 ui32MaxDeadlineMS,
+									 ui64RobustnessAddress,
+									 &sInfo,
+									 &psComputeContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc,
+			(void **)&psFWComputeContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	OSDeviceMemCopy(&psFWComputeContext->sStaticComputeContextState, pStaticComputeContextState, ui32StaticComputeContextStateSize);
+	DevmemPDumpLoadMem(psComputeContext->psFWComputeContextMemDesc, 0, sizeof(RGXFWIF_FWCOMPUTECONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpValue(psFWComputeContext->sStaticComputeContextState, FLUSH);
+	DevmemReleaseCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psComputeContext->psBufferSyncContext =
+			pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+					"rogue-cdm");
+	if (IS_ERR(psComputeContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to create buffer_sync context (err=%ld)",
+				__func__, PTR_ERR(psComputeContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psComputeContext->sSyncAddrListFence);
+	SyncAddrListInit(&psComputeContext->sSyncAddrListUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hComputeCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sComputeCtxtListHead), &(psComputeContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hComputeCtxListLock);
+	}
+
+	*ppsComputeContext = psComputeContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	FWCommonContextFree(psComputeContext->psServerCommonContext);
+fail_contextalloc:
+fail_frameworkcopy:
+	if (psComputeContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextStateMemDesc);
+fail_contextsuspendalloc:
+	OSLockDestroy(psComputeContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextMemDesc);
+fail_fwcomputecontext:
+	OSFreeMem(psComputeContext);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyComputeContextKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext)
+{
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psComputeContext->psDeviceNode,
+											  psComputeContext->psServerCommonContext,
+											  RGXFWIF_DM_CDM,
+											  PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psComputeContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* remove after RGXFWRequestCommonContextCleanUp() because we might return
+	 * RETRY and don't want to be calling this twice */
+	if (psComputeContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psComputeContext->psBufferSyncContext);
+		psComputeContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_FWCOMPUTECONTEXT	*psFWComputeContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc,
+				(void **)&psFWComputeContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware compute context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+		RGXFwSharedMemCacheOpValue(psFWComputeContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+		ui32WorkEstCCBSubmitted = psFWComputeContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psComputeContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psComputeContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+	/* ... it has so we can free its resources */
+
+	OSWRLockAcquireWrite(psDevInfo->hComputeCtxListLock);
+	dllist_remove_node(&(psComputeContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hComputeCtxListLock);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		WorkEstDeInitCompute(psDevInfo, &psComputeContext->sWorkEstData);
+	}
+#endif
+
+	SyncAddrListDeinit(&psComputeContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psComputeContext->sSyncAddrListUpdate);
+
+	FWCommonContextFree(psComputeContext->psServerCommonContext);
+	psComputeContext->psServerCommonContext = NULL;
+
+	if (psComputeContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWFrameworkMemDesc);
+	}
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextStateMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextMemDesc);
+
+	OSLockDestroy(psComputeContext->hLock);
+	OSFreeMem(psComputeContext);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXKickCDMKM(RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext,
+								IMG_UINT32					ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK		**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								IMG_UINT32					*paui32ClientUpdateValue,
+								PVRSRV_FENCE				iCheckFence,
+								PVRSRV_TIMELINE				iUpdateTimeline,
+								PVRSRV_FENCE				*piUpdateFence,
+								IMG_CHAR					pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								PVRSRV_FENCE				iExportFenceToSignal,
+								IMG_UINT32					ui32CmdSize,
+								IMG_PBYTE					pui8DMCmd,
+								IMG_UINT32					ui32PDumpFlags,
+								IMG_UINT32					ui32ExtJobRef,
+								IMG_UINT32					ui32SyncPMRCount,
+								IMG_UINT32					*paui32SyncPMRFlags,
+								PMR							**ppsSyncPMRs,
+								IMG_UINT32					ui32NumWorkgroups,
+								IMG_UINT32					ui32NumWorkitems,
+								IMG_UINT64					ui64DeadlineInus,
+								IMG_PUINT32					pui32IntJobRef)
+{
+	RGXFWIF_KCCB_CMD		sCmpKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	PVRSRV_ERROR			eError;
+	IMG_UINT32				ui32CDMCmdOffset = 0;
+	PVRSRV_RGXDEV_INFO      *psDevInfo = FWCommonContextGetRGXDevInfo(psComputeContext->psServerCommonContext);
+	RGX_CLIENT_CCB          *psClientCCB = FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext);
+	IMG_UINT32              ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32				ui32FWCtx;
+	IMG_BOOL				bCCBStateOpen = IMG_FALSE;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataCompute = {0};
+	IMG_UINT32 ui32CDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32CDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32CDMCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT64 ui64FBSCEntryMask = 0;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64 uiCheckFenceUID = 0;
+	IMG_UINT64 uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT psExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	CMD_COMMON *psComputeCmdCmn = IMG_OFFSET_ADDR(pui8DMCmd, 0);
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				"iUpdateTimeline=%d but piUpdateFence is NULL - PVRSRV_ERROR_INVALID_PARAMS",
+				__func__, iUpdateTimeline));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (pui32IntJobRef)
+	{
+		*pui32IntJobRef = ui32IntJobRef;
+	}
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	ui32IntClientUpdateCount = ui32ClientUpdateCount;
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListUpdate,
+									ui32ClientUpdateCount,
+									pauiClientUpdateUFODevVarBlock,
+									paui32ClientUpdateSyncOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount)
+	{
+		pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+	if (ui32SyncPMRCount != 0)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling "
+		          "pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psComputeContext->psBufferSyncContext,
+		    psComputeContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%s)",
+				        __func__, PVRSRVGetErrorString(eError)));
+			}
+
+			goto fail_resolve_input_fence;
+		}
+
+		/* Append buffer sync fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints "
+			          "to CDM Fence (&psTransferContext->sSyncAddrListFence=<%p>, "
+			          "pauiIntFenceUFOAddress=<%p>)...", __func__,
+			          ui32BufferFenceSyncCheckpointCount,
+			          (void *) &psComputeContext->sSyncAddrListFence ,
+			          (void *) pauiIntFenceUFOAddress));
+
+			SyncAddrListAppendAndDeRefCheckpoints(&psComputeContext->sSyncAddrListFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (pauiIntFenceUFOAddress == NULL)
+			{
+				pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		/* Append the update (from output fence) */
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+			                              1, &psBufferUpdateSyncCheckpoint);
+			if (pauiIntUpdateUFOAddress == NULL)
+			{
+				pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+		}
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+		PVR_UNREFERENCED_PARAMETER(ppsSyncPMRs);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers",
+		        __func__, ui32SyncPMRCount));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_populate_sync_addr_list;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										iCheckFence,
+										&ui32FenceSyncCheckpointCount,
+										&apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (%s)", __func__, PVRSRVGetErrorString(eError)));
+		goto fail_free_buffer_sync_data;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, iCheckFence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, ii, (void*)psNextCheckpoint, SyncCheckpointGetFirmwareAddr(psNextCheckpoint)));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psComputeContext->psDeviceNode,
+		                                   pszUpdateFenceName,
+										   iUpdateTimeline,
+										   psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s)", __func__, PVRSRVGetErrorString(eError)));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: iExportFenceToSignal=%d", __func__, iExportFenceToSignal));
+		/* Resolve the iExportFenceToSignal (if required) */
+		if (iExportFenceToSignal != PVRSRV_NO_FENCE)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+			eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+			                                          psComputeContext->psDeviceNode->hSyncCheckpointContext,
+			                                          &psExportFenceSyncCheckpoint,
+			                                          ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), psExportFenceSyncCheckpoint));
+				goto fail_resolve_export_fence;
+			}
+
+			/* Check that the export fence was not also included as part of the
+			 * check fence (which is an error and would lead to a stalled kick).
+			 */
+			if (ui32FenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Checking export fence is not part of check fence...", __func__));
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   ui32FenceSyncCheckpointCount=%d",
+						   __func__, ui32FenceSyncCheckpointCount));
+				if (ui32FenceSyncCheckpointCount > 0)
+				{
+					IMG_UINT32 iii;
+
+					for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+						if (apsFenceSyncCheckpoints[iii] == psExportFenceSyncCheckpoint)
+						{
+							CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, psExportFenceSyncCheckpoint));
+							eError = PVRSRV_ERROR_INVALID_PARAMS;
+							PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+							goto fail_check_fence_includes_export_fence;
+						}
+					}
+				}
+			}
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			if (paui32IntUpdateValue)
+			{
+				OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+			}
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the compute context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the compute context update list */
+			SyncAddrListAppendSyncPrim(&psComputeContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Compute CDM Fence (&psComputeContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psComputeContext->sSyncAddrListFence));
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			IMG_UINT32 iii;
+
+			for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+			}
+		}
+#endif
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32IntClientUpdateCount > 0)
+	{
+		IMG_UINT32 iii;
+		IMG_UINT32 *pui32Tmp = (IMG_UINT32*)paui32IntUpdateValue;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Dumping %d update values (paui32IntUpdateValue=<%p>)...", __func__, ui32IntClientUpdateCount, (void*)paui32IntUpdateValue));
+		for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: paui32IntUpdateValue[%d] = <%p>", __func__, iii, (void*)pui32Tmp));
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: *paui32IntUpdateValue[%d] = 0x%x", __func__, iii, *pui32Tmp));
+			pui32Tmp++;
+		}
+	}
+#endif
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+
+	if (psExportFenceSyncCheckpoint)
+	{
+		/* Append the update (from export fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psExportFenceSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psExportFenceSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_CMP_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping Compute (CDM) fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Compute (CDM) fence syncs (&psComputeContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psComputeContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Compute (CDM) update syncs (&psComputeContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psComputeContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+	/*
+	 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+	 * in other words, take the value and set it to zero afterwards.
+	 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+	 * as it must be ready at the time of context activation.
+	 */
+	{
+		eError = RGXExtractFBSCEntryMaskFromMMUContext(psComputeContext->psDeviceNode,
+													   FWCommonContextGetServerMMUCtx(psComputeContext->psServerCommonContext),
+													   &ui64FBSCEntryMask);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%s)", PVRSRVGetErrorString(eError)));
+			goto fail_cmdinvalfbsc;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psComputeContext->psDeviceNode))
+	{
+		sWorkloadCharacteristics.sCompute.ui32NumberOfWorkgroups = ui32NumWorkgroups;
+		sWorkloadCharacteristics.sCompute.ui32NumberOfWorkitems  = ui32NumWorkitems;
+
+		/* Prepare workload estimation */
+		WorkEstPrepare(psComputeContext->psDeviceNode->pvDevice,
+				&psComputeContext->sWorkEstData,
+				&psComputeContext->sWorkEstData.uWorkloadMatchingData.sCompute.sDataCDM,
+				RGXFWIF_CCB_CMD_TYPE_CDM,
+				&sWorkloadCharacteristics,
+				ui64DeadlineInus,
+				&sWorkloadKickDataCompute);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32NumWorkgroups);
+	PVR_UNREFERENCED_PARAMETER(ui32NumWorkitems);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psComputeContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       ui64FBSCEntryMask,
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+	                       &pPreAddr,
+	                       &pPostAddr,
+	                       &pRMWUFOAddr,
+	                       RGXFWIF_CCB_CMD_TYPE_CDM,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	                       &sWorkloadKickDataCompute,
+#else
+	                       NULL,
+#endif
+	                       "Compute",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+
+	/*
+		We should reserve space in the kernel CCB here and fill in the command
+		directly.
+		This is so if there isn't space in the kernel CCB we can return with
+		retry back to services client before we take any operations
+	*/
+
+	/*
+		We might only be kicking for flush out a padding packet so only submit
+		the command if the create was successful
+	*/
+
+	/*
+	  All the required resources are ready at this point, we can't fail so
+	  take the required server sync operations and commit all the resources
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	ui32CDMCmdOffset = RGXGetHostWriteOffsetCCB(psClientCCB);
+	RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "CDM", FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+	{
+		/* The following is used to determine the offset of the command header containing
+		   the workload estimation data so that can be accessed when the KCCB is read */
+		ui32CDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(asCmdHelperData);
+
+		ui32CDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32CDMCmdOffset < ui32CDMCmdOffsetWrapCheck)
+		{
+			ui32CDMWorkloadDataRO = ui32CDMCmdOffset;
+		}
+		else
+		{
+			ui32CDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/* Construct the kernel compute CCB command. */
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sCmpKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+	{
+		/* Store the offset to the CCCB command header so that it can be referenced
+		 * when the KCCB command reaches the FW */
+		sCmpKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32CDMWorkloadDataRO + ui32CDMCmdHeaderOffset;
+	}
+#endif
+
+	ui32FWCtx = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr;
+
+	if (psComputeCmdCmn)
+	{
+		HTBLOGK(HTB_SF_MAIN_KICK_CDM,
+				sCmpKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui32CDMCmdOffset,
+				psComputeCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef);
+	}
+
+	RGXSRV_HWPERF_ENQ(psComputeContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_CDM,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	/*
+	 * Submit the compute command to the firmware.
+	 */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandWithoutPowerLock(psComputeContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_CDM,
+									&sCmpKCCBCmd,
+									ui32PDumpFlags);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_schedulecmd;
+	}
+	else
+	{
+		PVRGpuTraceEnqueueEvent(psComputeContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_CDM);
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__,
+		           (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+	if (psExportFenceSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)psExportFenceSyncCheckpoint, SyncCheckpointGetId(psExportFenceSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(psExportFenceSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psExportFenceSyncCheckpoint);
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal=%d)", __func__, iExportFenceToSignal));
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psComputeContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pszUpdateFenceName);
+	}
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_schedulecmd:
+fail_acquirepowerlock:
+fail_cmdacquire:
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+fail_cmdinvalfbsc:
+#endif
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+fail_check_fence_includes_export_fence:
+fail_resolve_export_fence:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+
+fail_free_buffer_sync_data:
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+
+fail_resolve_input_fence:
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendCancelCmdKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext, IMG_UINT32 ui32FirstIntJobRefToCancel, IMG_UINT32 ui32LastIntJobRefToCancel)
+{
+	RGXFWIF_KCCB_CMD sCancelWorkKCCBCmd;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	sCancelWorkKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_CANCEL_WORK;
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.ui32FirstIntJobRefToCancel = ui32FirstIntJobRefToCancel;
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.ui32LastIntJobRefToCancel = ui32LastIntJobRefToCancel;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psComputeContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_CDM,
+		                            &sCancelWorkKCCBCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+
+		/* Iterate if we hit a PVRSRV_ERROR_KERNEL_CCB_FULL error */
+		if ((eError != PVRSRV_ERROR_RETRY) &&
+		    (eError != PVRSRV_ERROR_KERNEL_CCB_FULL))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Sending compute cancel command for context <%p>. Work with IntJobRef below ui32FirstValidIntJobRef=0x%x might be discarded.",
+	    psComputeContext->psServerCommonContext, ui32LastIntJobRefToCancel + 1));
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXFlushComputeDataKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext)
+{
+	RGXFWIF_KCCB_CMD sFlushCmd;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psComputeContext->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Submit Compute flush");
+#endif
+	sFlushCmd.eCmdType = RGXFWIF_KCCB_CMD_SLCFLUSHINVAL;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.bInval = IMG_FALSE;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.bDMContext = IMG_TRUE;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+									RGXFWIF_DM_CDM,
+									&sFlushCmd,
+									PDUMP_FLAGS_CONTINUOUS,
+									&ui32kCCBCommandSlot);
+		/* Iterate if we hit a PVRSRV_ERROR_KERNEL_CCB_FULL error */
+		if ((eError != PVRSRV_ERROR_RETRY) &&
+		    (eError != PVRSRV_ERROR_KERNEL_CCB_FULL))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If we hit a temporary KCCB exhaustion, return a RETRY to caller */
+		if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: Returning RETRY to caller", __func__));
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to schedule SLC flush command (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+		}
+	}
+	else
+	{
+		/* Wait for the SLC flush to complete */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Compute flush aborted (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+			goto error_exit;
+		}
+
+		if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] &
+		                  RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+	}
+
+error_exit:
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(RGX_SERVER_COMPUTE_CONTEXT  *psComputeContext)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+	IMG_UINT32 ui32ControlStreamFormat =
+		RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, CDM_CONTROL_STREAM_FORMAT) ?
+		RGX_GET_FEATURE_VALUE(psDevInfo, CDM_CONTROL_STREAM_FORMAT) : 0;
+
+	if (ui32ControlStreamFormat >= 2U && ui32ControlStreamFormat < 5U)
+	{
+
+		RGXFWIF_KCCB_CMD  sKCCBCmd;
+		PVRSRV_ERROR      eError;
+
+		OSLockAcquire(psComputeContext->hLock);
+
+		/* Schedule the firmware command */
+		sKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE;
+		sKCCBCmd.uCmdData.sWriteOffsetUpdateData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = RGXScheduleCommand(psComputeContext->psDeviceNode->pvDevice,
+										RGXFWIF_DM_CDM,
+										&sKCCBCmd,
+										PDUMP_FLAGS_NONE);
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to schedule the FW command %d (%s)",
+					__func__,
+					eError,
+					PVRSRVGetErrorString(eError)));
+		}
+
+		OSLockRelease(psComputeContext->hLock);
+		return eError;
+	}
+	else
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                  PVRSRV_DEVICE_NODE * psDeviceNode,
+												  RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = ContextSetPriority(psComputeContext->psServerCommonContext,
+								psConnection,
+								psComputeContext->psDeviceNode->pvDevice,
+								i32Priority,
+								RGXFWIF_DM_CDM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the compute context (%s)", __func__, PVRSRVGetErrorString(eError)));
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+/*
+ * PVRSRVRGXSetComputeContextPropertyKM
+ */
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPropertyKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                                  RGX_CONTEXT_PROPERTY eContextProperty,
+                                                  IMG_UINT64 ui64Input,
+                                                  IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psComputeContext->hLock);
+			eError = FWCommonContextSetFlags(psComputeContext->psServerCommonContext,
+			                                 ui32ContextFlags);
+			OSLockRelease(psComputeContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpComputeCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hComputeCtxListLock);
+	dllist_foreach_node(&psDevInfo->sComputeCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMPUTE_CONTEXT *psCurrentServerComputeCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMPUTE_CONTEXT, sListNode);
+		DumpFWCommonContextInfo(psCurrentServerComputeCtx->psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hComputeCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientComputeCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 ui32ContextBitMask = 0;
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hComputeCtxListLock);
+	dllist_foreach_node(&psDevInfo->sComputeCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMPUTE_CONTEXT *psCurrentServerComputeCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMPUTE_CONTEXT, sListNode);
+
+		if (CheckStalledClientCommonContext(psCurrentServerComputeCtx->psServerCommonContext, RGX_KICK_TYPE_DM_CDM)
+			== PVRSRV_ERROR_CCCB_STALLED)
+		{
+			ui32ContextBitMask |= RGX_KICK_TYPE_DM_CDM;
+		}
+	}
+	OSWRLockReleaseRead(psDevInfo->hComputeCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * PVRSRVRGXGetLastDeviceErrorKM
+ */
+PVRSRV_ERROR PVRSRVRGXGetLastDeviceErrorKM(CONNECTION_DATA    *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32         *ui32Error)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	*ui32Error = psDevInfo->eLastDeviceError;
+	psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_NONE;
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXKickTimestampQueryKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTimestampQueryKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                           PVRSRV_FENCE iCheckFence,
+										   PVRSRV_TIMELINE iUpdateTimeline,
+										   PVRSRV_FENCE *piUpdateFence,
+                                           IMG_CHAR pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                           IMG_UINT32 ui32CmdSize,
+                                           IMG_PBYTE pui8DMCmd,
+                                           IMG_UINT32 ui32ExtJobRef)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psComputeContext->psServerCommonContext);
+	RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext);
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	IMG_UINT32 ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32 ui32PDumpFlags = 0;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64 uiCheckFenceUID = 0;
+	IMG_UINT64 uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *paui32ClientUpdateValue = NULL;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+	PVRSRV_ERROR eError;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListUpdate,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount)
+	{
+		pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psComputeContext->psDeviceNode->hSyncCheckpointContext,
+	                                    iCheckFence,
+	                                    &ui32FenceSyncCheckpointCount,
+	                                    &apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (%s)", __func__, PVRSRVGetErrorString(eError)));
+		goto err_populate_sync_addr_list;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, ii, (void*)psNextCheckpoint, SyncCheckpointGetFirmwareAddr(psNextCheckpoint)));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psComputeContext->psDeviceNode,
+		                                   pszUpdateFenceName,
+										   iUpdateTimeline,
+										   psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s)", __func__, PVRSRVGetErrorString(eError)));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			if (paui32IntUpdateValue)
+			{
+				OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+			}
+
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the compute context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the compute context update list */
+			SyncAddrListAppendSyncPrim(&psComputeContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Compute CDM Fence (&psComputeContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psComputeContext->sSyncAddrListFence));
+
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+	}
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       0, /* empty ui64FBSCEntryMask */
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       PDUMP_FLAGS_NONE,
+	                       NULL,
+	                       "VkTimestamp",
+	                       IMG_FALSE, /* bCCBStateOpen */
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXCmdHelperAcquireCmdCCB", fail_cmdacquire);
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "CDM",
+	                          FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr);
+
+	/* Construct the kernel compute CCB command. */
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sCmpKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/*
+	 * Submit the RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP
+	 * command to the firmware.
+	 */
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandWithoutPowerLock(psComputeContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_CDM,
+		                            &sCmpKCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_schedulecmd;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__, (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif /* defined(NO_HARDWARE) */
+
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psComputeContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pszUpdateFenceName);
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned
+	 * by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_schedulecmd:
+fail_acquirepowerlock:
+fail_cmdacquire:
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXCDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXCDMReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxcompute.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.h
new file mode 100644
index 000000000000..b93c42efc044
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxcompute.h
@@ -0,0 +1,226 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX compute functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX compute functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXCOMPUTE_H)
+#define RGXCOMPUTE_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+
+
+typedef struct _RGX_SERVER_COMPUTE_CONTEXT_ RGX_SERVER_COMPUTE_CONTEXT;
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXCreateComputeContextKM
+@Description    Creates an RGX device context for submitting commands to CDM.
+@Input          psConnection          Device connection
+@Input          psDeviceNode          Services-managed device
+@Input          i32Priority           Scheduling priority for commands
+                                      on this context
+@Input          ui32FrameworkCommandSize
+                                      Framework command size
+@Input          pabyFrameworkCommand  Pointer to framework command
+@Input          hMemCtxPrivData       Private data
+@Input          ui32StaticComputeContextStateSize
+                                      Size of fixed compute context state
+@Input          pStaticComputeContextState
+                                      Compute context state
+@Input          ui32PackedCCBSizeU88  Packed CCB size. The first byte contains
+                                      the log2 CCB size and the second byte
+                                      the log2 maximum CCB size.
+@Input          ui32ContextFlags      Flags with context properties
+@Input          ui64RobustnessAddress Address for FW to signal a context reset
+@Input          ui32MaxDeadlineMS     Max deadline limit in MS that the
+                                      workload can run
+@Output         ppsComputeContext     Cleanup data
+@Return         PVRSRV_ERROR          Returns PVRSRV_OK or an error.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateComputeContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE			*psDeviceNode,
+											 IMG_INT32					i32Priority,
+											 IMG_UINT32					ui32FrameworkCommandSize,
+											 IMG_PBYTE					pabyFrameworkCommand,
+											 IMG_HANDLE					hMemCtxPrivData,
+											 IMG_UINT32					ui32StaticComputeContextStateSize,
+											 IMG_PBYTE					pStaticComputeContextState,
+											 IMG_UINT32					ui32PackedCCBSizeU88,
+											 IMG_UINT32					ui32ContextFlags,
+											 IMG_UINT64					ui64RobustnessAddress,
+											 IMG_UINT32					ui32MaxDeadlineMS,
+											 RGX_SERVER_COMPUTE_CONTEXT	**ppsComputeContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDestroyComputeContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyComputeContext
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyComputeContextKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXKickCDMKM
+
+ @Description
+	Server-side implementation of RGXKickCDM
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickCDMKM(RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext,
+								IMG_UINT32					ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK		**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								IMG_UINT32					*paui32ClientUpdateValue,
+								PVRSRV_FENCE				iCheckFence,
+								PVRSRV_TIMELINE				iUpdateTimeline,
+								PVRSRV_FENCE				*piUpdateFence,
+								IMG_CHAR					pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								PVRSRV_FENCE				iExportFenceToSignal,
+								IMG_UINT32					ui32CmdSize,
+								IMG_PBYTE					pui8DMCmd,
+								IMG_UINT32					ui32PDumpFlags,
+								IMG_UINT32					ui32ExtJobRef,
+								IMG_UINT32					ui32SyncPMRCount,
+								IMG_UINT32					*paui32SyncPMRFlags,
+								PMR							**ppsSyncPMRs,
+								IMG_UINT32					ui32NumWorkgroups,
+								IMG_UINT32					ui32NumWorkitems,
+								IMG_UINT64					ui64DeadlineInus,
+								IMG_PUINT32					pui32IntJobRef);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXFlushComputeDataKM
+
+ @Description
+	Server-side implementation of RGXFlushComputeData
+
+ @Input psComputeContext - Compute context to flush
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFlushComputeDataKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSendCancelCmd
+
+ @Description
+	Server-side implementation of RGXSendCancelCmd
+
+ @Input psComputeContext - Compute context to cancel work on
+ @Input ui32LastIntJobRefToCancel - Last IntJobRef to cancel
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSendCancelCmdKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                      IMG_UINT32 ui32FirstIntJobRefToCancel,
+                                      IMG_UINT32 ui32LastIntJobRefToCancel);
+
+
+/*!
+*******************************************************************************
+
+ @Function	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM
+ @Description   Server-side implementation of RGXNotifyComputeWriteOffsetUpdate
+
+ @Input         psComputeContext - Compute context to flush
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPriorityKM(CONNECTION_DATA *psConnection,
+												  PVRSRV_DEVICE_NODE *psDeviceNode,
+												  RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPropertyKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  RGX_CONTEXT_PROPERTY eContextProperty,
+												  IMG_UINT64 ui64Input,
+												  IMG_UINT64 *pui64Output);
+
+PVRSRV_ERROR PVRSRVRGXGetLastDeviceErrorKM(CONNECTION_DATA    *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32         *ui32Error);
+
+PVRSRV_ERROR PVRSRVRGXKickTimestampQueryKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                           PVRSRV_FENCE iCheckFence,
+										   PVRSRV_TIMELINE iUpdateTimeline,
+										   PVRSRV_FENCE *piUpdateFence,
+                                           IMG_CHAR pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                           IMG_UINT32 ui32CmdSize,
+                                           IMG_PBYTE pui8DMCmd,
+                                           IMG_UINT32 ui32ExtJobRef);
+
+/* Debug - Dump debug info of compute contexts on this device */
+void DumpComputeCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client compute contexts are stalled */
+IMG_UINT32 CheckForStalledClientComputeCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR PVRSRVRGXCDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+PVRSRV_ERROR PVRSRVRGXCDMReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+#endif /* RGXCOMPUTE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.c
new file mode 100644
index 000000000000..fca648b26f56
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.c
@@ -0,0 +1,3236 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_sf.h"
+#include "rgxfw_log_helper.h"
+#include "fwtrace_string.h"
+#include "rgxmmudefs_km.h"
+#include "rgxinit.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "rgxfwimageutils.h"
+#include "fwload.h"
+#include "rgx_options.h"
+#include "devicemem_history_server.h"
+#include "debug_common.h"
+#include "info_page.h"
+#include "osfunc.h"
+#if defined(SUPPORT_SOC_TIMER)
+#include "rgxtimecorr.h"
+#endif
+
+#define MAX_FW_DESCRIPTION_LENGTH	(600U)
+
+#define PVR_DUMP_FIRMWARE_INFO(x)														\
+	PVR_DUMPDEBUG_LOG("FW info: %d.%d @ %8d (%s) build options: 0x%08x",				\
+						PVRVERSION_UNPACK_MAJ((x).ui32DDKVersion),						\
+						PVRVERSION_UNPACK_MIN((x).ui32DDKVersion),						\
+						(x).ui32DDKBuild,												\
+						((x).ui32BuildOptions & OPTIONS_DEBUG_EN) ? "debug":"release",	\
+						(x).ui32BuildOptions);
+
+#define PVR_DUMP_FIRMWARE_INFO_HDR(x)												\
+	PVR_DUMPDEBUG_LOG("FW info: %d.%d @ %8d (%s) build options: 0x%08x",		\
+						(x).ui16PVRVersionMajor,								\
+						(x).ui16PVRVersionMinor,								\
+						(x).ui32PVRVersionBuild,								\
+						((x).ui32Flags & OPTIONS_DEBUG_EN) ? "debug":"release",	\
+						(x).ui32Flags);
+
+typedef struct {
+	IMG_UINT16     ui16Mask;
+	const IMG_CHAR *pszStr;
+} RGXFWT_DEBUG_INFO_MSKSTR; /* pair of bit mask and debug info message string */
+
+/*
+ *  Array of all the Firmware Trace log IDs used to convert the trace data.
+ */
+typedef struct _TRACEBUF_LOG_ {
+	RGXFW_LOG_SFids	eSFId;
+	const IMG_CHAR	*pszName;
+	const IMG_CHAR	*pszFmt;
+	IMG_UINT32		ui32ArgNum;
+} TRACEBUF_LOG;
+
+static const TRACEBUF_LOG aLogDefinitions[] =
+{
+#define X(a, b, c, d, e) {RGXFW_LOG_CREATESFID(a,b,e), #c, d, e},
+	RGXFW_LOG_SFIDLIST
+#undef X
+};
+
+static const IMG_FLAGS2DESC asCswOpts2Description[] =
+{
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST, " Fast CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM, " Medium CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW, " Slow CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY, " No Delay CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_MODE_RAND, " Random Csw enabled;"},
+	{RGXFWIF_INICFG_CTXSWITCH_SRESET_EN, " SoftReset;"},
+};
+
+static const IMG_FLAGS2DESC asMisc2Description[] =
+{
+	{RGXFWIF_INICFG_POW_RASCALDUST, " Power Rascal/Dust;"},
+	{RGXFWIF_INICFG_SPU_CLOCK_GATE, " SPU Clock Gating (requires Power Rascal/Dust);"},
+	{RGXFWIF_INICFG_HWPERF_EN, " HwPerf EN;"},
+	{RGXFWIF_INICFG_FBCDC_V3_1_EN, " FBCDCv3.1;"},
+	{RGXFWIF_INICFG_CHECK_MLIST_EN, " Check MList;"},
+	{RGXFWIF_INICFG_DISABLE_CLKGATING_EN, " ClockGating Off;"},
+	{RGXFWIF_INICFG_REGCONFIG_EN, " Register Config;"},
+	{RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY, " Assert on OOM;"},
+	{RGXFWIF_INICFG_HWP_DISABLE_FILTER, " HWP Filter Off;"},
+	{RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN, " CDM Random kill;"},
+	{RGXFWIF_INICFG_DISABLE_DM_OVERLAP, " DM Overlap Off;"},
+	{RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER, " Assert on HWR;"},
+	{RGXFWIF_INICFG_INJECT_ICS_FAULT, " Inject ICS Fault;"},
+	{RGXFWIF_INICFG_VALIDATE_IRQ, " Validate IRQ;"},
+	{RGXFWIF_INICFG_DISABLE_PDP_EN, " PDUMP Panic off;"},
+	{RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN, " SPU Pow mask change on;"},
+	{RGXFWIF_INICFG_WORKEST, " Workload Estim;"},
+	{RGXFWIF_INICFG_PDVFS, " PDVFS;"},
+	{RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP, " ISP v1 scheduling;"},
+	{RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP, " ISP v2 scheduling;"},
+	{RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER, " Validate SOC&USC timers;"},
+};
+
+static const IMG_FLAGS2DESC asFwOsCfg2Description[] =
+{
+	{RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN, " TDM;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN, " GEOM;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN, " 3D;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN, " CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN, " RDM;"},
+#endif
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM, " LowPrio TDM;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM, " LowPrio GEOM;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D, " LowPrio 3D;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM, " LowPrio CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM, " LowPrio RDM;"},
+#endif
+#if defined(SUPPORT_ICS)
+	{RGXFWIF_INICFG_OS_ICS_TDM_EN, " TDM;"},
+	{RGXFWIF_INICFG_OS_ICS_GEOM_EN, " GEOM;"},
+	{RGXFWIF_INICFG_OS_ICS_3D_EN, " 3D;"},
+	{RGXFWIF_INICFG_OS_ICS_CDM_EN, " CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_ICS_RDM_EN, " RDM;"},
+#endif
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_LONG, " Long FDTI profile;"},
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_MEDIUM, " Medium FDTI profile;"},
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_SHORT, " Short FDTI profile;"},
+#endif
+};
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+const IMG_CHAR * const gapszMipsPermissionPTFlags[4] =
+{
+	"    ",
+	"XI  ",
+	"RI  ",
+	"RIXI"
+};
+
+const IMG_CHAR * const gapszMipsCoherencyPTFlags[8] =
+{
+	"C",
+	"C",
+	" ",
+	"C",
+	"C",
+	"C",
+	"C",
+	" "
+};
+
+const IMG_CHAR * const gapszMipsDirtyGlobalValidPTFlags[8] =
+{
+	"   ",
+	"  G",
+	" V ",
+	" VG",
+	"D  ",
+	"D G",
+	"DV ",
+	"DVG"
+};
+
+#if !defined(NO_HARDWARE)
+/* Translation of MIPS exception encoding */
+typedef struct _MIPS_EXCEPTION_ENCODING_
+{
+	const IMG_CHAR *const pszStr;	/* Error type */
+	const IMG_BOOL bIsFatal;	/* Error is fatal or non-fatal */
+} MIPS_EXCEPTION_ENCODING;
+
+static const MIPS_EXCEPTION_ENCODING apsMIPSExcCodes[] =
+{
+	{"Interrupt", IMG_FALSE},
+	{"TLB modified exception", IMG_FALSE},
+	{"TLB exception (load/instruction fetch)", IMG_FALSE},
+	{"TLB exception (store)", IMG_FALSE},
+	{"Address error exception (load/instruction fetch)", IMG_TRUE},
+	{"Address error exception (store)", IMG_TRUE},
+	{"Bus error exception (instruction fetch)", IMG_TRUE},
+	{"Bus error exception (load/store)", IMG_TRUE},
+	{"Syscall exception", IMG_FALSE},
+	{"Breakpoint exception (FW assert)", IMG_FALSE},
+	{"Reserved instruction exception", IMG_TRUE},
+	{"Coprocessor Unusable exception", IMG_FALSE},
+	{"Arithmetic Overflow exception", IMG_FALSE},
+	{"Trap exception", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"Implementation-Specific Exception 1 (COP2)", IMG_FALSE},
+	{"CorExtend Unusable", IMG_FALSE},
+	{"Coprocessor 2 exceptions", IMG_FALSE},
+	{"TLB Read-Inhibit", IMG_TRUE},
+	{"TLB Execute-Inhibit", IMG_TRUE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"Reference to WatchHi/WatchLo address", IMG_FALSE},
+	{"Machine check", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"DSP Module State Disabled exception", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	/* Can only happen in MIPS debug mode */
+	{"Parity error", IMG_FALSE},
+	{NULL, IMG_FALSE}
+};
+
+static IMG_CHAR const *_GetMIPSExcString(IMG_UINT32 ui32ExcCode)
+{
+	if (ui32ExcCode >= sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Only %lu exceptions available in MIPS, %u is not a valid exception code",
+		         (unsigned long)sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING), ui32ExcCode));
+		return NULL;
+	}
+
+	return apsMIPSExcCodes[ui32ExcCode].pszStr;
+}
+#endif
+
+typedef struct _RGXMIPSFW_C0_DEBUG_TBL_ENTRY_
+{
+    IMG_UINT32 ui32Mask;
+    const IMG_CHAR * pszExplanation;
+} RGXMIPSFW_C0_DEBUG_TBL_ENTRY;
+
+#if !defined(NO_HARDWARE)
+static const RGXMIPSFW_C0_DEBUG_TBL_ENTRY sMIPS_C0_DebugTable[] =
+{
+    { RGXMIPSFW_C0_DEBUG_DSS,      "Debug single-step exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DBP,      "Debug software breakpoint exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBL,     "Debug data break exception occurred on a load" },
+    { RGXMIPSFW_C0_DEBUG_DDBS,     "Debug data break exception occurred on a store" },
+    { RGXMIPSFW_C0_DEBUG_DIB,      "Debug instruction break exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DINT,     "Debug interrupt exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DIBIMPR,  "Imprecise debug instruction break exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBLIMPR, "Imprecise debug data break load exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBSIMPR, "Imprecise debug data break store exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_IEXI,     "Imprecise error exception inhibit controls exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DBUSEP,   "Data access Bus Error exception pending" },
+    { RGXMIPSFW_C0_DEBUG_CACHEEP,  "Imprecise Cache Error pending" },
+    { RGXMIPSFW_C0_DEBUG_MCHECKP,  "Imprecise Machine Check exception pending" },
+    { RGXMIPSFW_C0_DEBUG_IBUSEP,   "Instruction fetch Bus Error exception pending" },
+    { (IMG_UINT32)RGXMIPSFW_C0_DEBUG_DBD,      "Debug exception occurred in branch delay slot" }
+};
+#endif
+#endif
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)*/
+
+#define NARGS_MASK ~(0xF<<16)
+static IMG_BOOL _FirmwareTraceIntegrityCheck(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile)
+{
+	const TRACEBUF_LOG *psLogDef = &aLogDefinitions[0];
+	IMG_BOOL bIntegrityOk = IMG_TRUE;
+
+	/*
+	 * For every log ID, check the format string and number of arguments is valid.
+	 */
+	while (psLogDef->eSFId != RGXFW_SF_LAST)
+	{
+		const TRACEBUF_LOG *psLogDef2;
+		const IMG_CHAR *pszString;
+		IMG_UINT32 ui32Count;
+
+		/*
+		 * Check the number of arguments matches the number of '%' in the string and
+		 * check that no string uses %s which is not supported as it requires a
+		 * pointer to memory that is not going to be valid.
+		 */
+		pszString = psLogDef->pszFmt;
+		ui32Count = 0;
+
+		while (*pszString != '\0')
+		{
+			if (*pszString++ == '%')
+			{
+				ui32Count++;
+				if (*pszString == 's')
+				{
+					bIntegrityOk = IMG_FALSE;
+					PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has an unsupported type not recognized (fmt: %%%c). Please fix.",
+									  psLogDef->pszName, *pszString);
+				}
+				else if (*pszString == '%')
+				{
+					/* Double % is a printable % sign and not a format string... */
+					ui32Count--;
+					pszString++;
+				}
+			}
+		}
+
+		if (ui32Count != psLogDef->ui32ArgNum)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has %d arguments but only %d are specified. Please fix.",
+			                  psLogDef->pszName, ui32Count, psLogDef->ui32ArgNum);
+		}
+
+		/* RGXDumpFirmwareTrace() has a hardcoded limit of supporting up to 20 arguments... */
+		if (ui32Count > 20)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has %d arguments but a maximum of 20 are supported. Please fix.",
+			                  psLogDef->pszName, ui32Count);
+		}
+
+		/* Check the id number is unique (don't take into account the number of arguments) */
+		ui32Count = 0;
+		psLogDef2 = &aLogDefinitions[0];
+
+		while (psLogDef2->eSFId != RGXFW_SF_LAST)
+		{
+			if ((psLogDef->eSFId & NARGS_MASK) == (psLogDef2->eSFId & NARGS_MASK))
+			{
+				ui32Count++;
+			}
+			psLogDef2++;
+		}
+
+		if (ui32Count != 1)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s id %x is not unique, there are %d more. Please fix.",
+			                  psLogDef->pszName, psLogDef->eSFId, ui32Count - 1);
+		}
+
+		/* Move to the next log ID... */
+		psLogDef++;
+	}
+
+	return bIntegrityOk;
+}
+
+void RGXDumpFirmwareTrace(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO  *psDevInfo)
+{
+	RGXFWIF_TRACEBUF  *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	static IMG_BOOL   bIntegrityCheckPassed = IMG_FALSE;
+
+	/* Check that the firmware trace is correctly defined... */
+	if (!bIntegrityCheckPassed)
+	{
+		bIntegrityCheckPassed = _FirmwareTraceIntegrityCheck(pfnDumpDebugPrintf, pvDumpDebugFile);
+		if (!bIntegrityCheckPassed)
+		{
+			return;
+		}
+	}
+
+	/* Dump FW trace information... */
+	if (psRGXFWIfTraceBufCtl != NULL)
+	{
+		IMG_UINT32  tid;
+
+		PVR_DUMPDEBUG_LOG("Device ID: %u", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+
+		/* Print the log type settings... */
+		if (psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+		{
+			PVR_DUMPDEBUG_LOG("Debug log type: %s ( " RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC ")",
+							  ((psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)?("trace"):("tbi")),
+							  RGXFWIF_LOG_ENABLED_GROUPS_LIST(psRGXFWIfTraceBufCtl->ui32LogType)
+							  );
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("Debug log type: none");
+		}
+
+		/* Print the decoded log for each thread... */
+		for (tid = 0;  tid < RGXFW_THREAD_NUM;  tid++)
+		{
+			RGXDumpFirmwareTraceDecoded(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXPrepareExtraDebugInfo
+
+ @Description
+
+ Prepares debug info string by decoding ui16DebugInfo value passed
+
+ @Input pszBuffer	 - pointer to debug info string buffer
+
+ @Return   void
+
+******************************************************************************/
+static void RGXPrepareExtraDebugInfo(IMG_CHAR *pszBuffer, IMG_UINT32 ui32BufferSize, IMG_UINT16 ui16DebugInfo)
+{
+	const RGXFWT_DEBUG_INFO_MSKSTR aDebugInfoMskStr[] =
+	{
+#define X(a, b) {a, b},
+		RGXFWT_DEBUG_INFO_MSKSTRLIST
+#undef X
+	};
+
+	IMG_UINT32 ui32NumFields = sizeof(aDebugInfoMskStr)/sizeof(RGXFWT_DEBUG_INFO_MSKSTR);
+	IMG_UINT32 i;
+	IMG_BOOL   bHasExtraDebugInfo = IMG_FALSE;
+
+	/* Add prepend string */
+	OSStringSafeCopy(pszBuffer, RGXFWT_DEBUG_INFO_STR_PREPEND, ui32BufferSize);
+
+	/* Add debug info strings */
+	for (i = 0; i < ui32NumFields; i++)
+	{
+		if (ui16DebugInfo & aDebugInfoMskStr[i].ui16Mask)
+		{
+			if (bHasExtraDebugInfo)
+			{
+				OSStringLCat(pszBuffer, ", ", ui32BufferSize); /* Add comma separator */
+			}
+			OSStringLCat(pszBuffer, aDebugInfoMskStr[i].pszStr, ui32BufferSize);
+			bHasExtraDebugInfo = IMG_TRUE;
+		}
+	}
+
+	/* Add append string */
+	OSStringLCat(pszBuffer, RGXFWT_DEBUG_INFO_STR_APPEND, ui32BufferSize);
+}
+
+#define PVR_MAX_DEBUG_PARTIAL_LINES (40U)
+#define PVR_DUMPDEBUG_LOG_LINES(fmt, ...) \
+	if (!bPrintAllLines) { \
+		OSSNPrintf(&pszLineBuffer[ui32LastLineIdx * PVR_MAX_DEBUG_MESSAGE_LEN], PVR_MAX_DEBUG_MESSAGE_LEN, (fmt), ##__VA_ARGS__); \
+		ui32LineCount++; \
+		ui32LastLineIdx = ui32LineCount % PVR_MAX_DEBUG_PARTIAL_LINES; \
+	} else { \
+		PVR_UNREFERENCED_PARAMETER(pszLineBuffer); \
+		PVR_UNREFERENCED_PARAMETER(ui32LineCount); \
+		PVR_UNREFERENCED_PARAMETER(ui32LastLineIdx); \
+		PVR_DUMPDEBUG_LOG((fmt), ##__VA_ARGS__); \
+	}
+
+static void RGXDumpFirmwareTraceLines(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID,
+				bool bPrintAllLines)
+{
+	volatile IMG_UINT32  *pui32FWWrapCount;
+	volatile IMG_UINT32  *pui32FWTracePtr;
+	IMG_UINT32           *pui32TraceBuf;
+	IMG_UINT32           *pui32LocalTraceBuf = NULL;
+	IMG_UINT32           ui32HostWrapCount;
+	IMG_UINT32           ui32HostTracePtr;
+	IMG_UINT32           ui32Count           = 0;
+	IMG_UINT32           ui32LineCount       = 0;
+	IMG_UINT32           ui32LastLineIdx     = 0;
+	IMG_CHAR             *pszLineBuffer      = NULL;
+	IMG_UINT32           ui32TraceBufSizeInDWords;
+
+	RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID], INVALIDATE);
+	RGXFwSharedMemCacheOpExec(psDevInfo->apui32TraceBuffer[ui32TID],
+	                          psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	pui32FWWrapCount = &(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].ui32WrapCount);
+	pui32FWTracePtr  = &(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].ui32TracePointer);
+	pui32TraceBuf    = psDevInfo->apui32TraceBuffer[ui32TID];
+	ui32HostWrapCount = *pui32FWWrapCount;
+	ui32HostTracePtr  = *pui32FWTracePtr;
+
+	if (pui32TraceBuf == NULL)
+	{
+		/* trace buffer not yet allocated */
+		return;
+	}
+
+	if (!bPrintAllLines)
+	{
+		pszLineBuffer = OSAllocMem(PVR_MAX_DEBUG_MESSAGE_LEN * PVR_MAX_DEBUG_PARTIAL_LINES);
+		PVR_LOG_RETURN_VOID_IF_FALSE(pszLineBuffer != NULL, "pszLineBuffer alloc failed");
+	}
+
+	ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+	{
+		PVR_DUMPDEBUG_LOG_LINES("WARNING: Trace pointer (%d) greater than buffer size (%d).",
+								ui32HostTracePtr, ui32TraceBufSizeInDWords);
+		ui32HostTracePtr %= ui32TraceBufSizeInDWords;
+	}
+
+	/*
+	 *  Allocate a local copy of the trace buffer which will contain a static non-changing
+	 *  snapshot view of the buffer. This removes the issue of a fast GPU wrapping and
+	 *  overwriting the tail data of the buffer.
+	 */
+	pui32LocalTraceBuf = OSAllocMem(ui32TraceBufSizeInDWords * sizeof(IMG_UINT32));
+	if (pui32LocalTraceBuf != NULL)
+	{
+		memcpy(pui32LocalTraceBuf, pui32TraceBuf, ui32TraceBufSizeInDWords * sizeof(IMG_UINT32));
+		ui32HostTracePtr = *pui32FWTracePtr;
+		pui32TraceBuf = pui32LocalTraceBuf;
+	}
+
+	while (ui32Count < ui32TraceBufSizeInDWords)
+	{
+		IMG_UINT32  ui32Data, ui32DataToId;
+
+		/* Find the first valid log ID, skipping whitespace... */
+		do
+		{
+			IMG_UINT32 ui32ValidatedHostTracePtr;
+			ui32ValidatedHostTracePtr = OSConfineArrayIndexNoSpeculation(ui32HostTracePtr,
+			                                                             ui32TraceBufSizeInDWords);
+			ui32Data     = pui32TraceBuf[ui32ValidatedHostTracePtr];
+			ui32DataToId = idToStringID(ui32Data, SFs);
+
+			/* If an unrecognized id is found it may be inconsistent data or a firmware trace error. */
+			if (ui32DataToId == RGXFW_SF_LAST  &&  RGXFW_LOG_VALIDID(ui32Data))
+			{
+				PVR_DUMPDEBUG_LOG_LINES("WARNING: Unrecognized id (%x). From here on the trace might be wrong!", ui32Data);
+			}
+
+			/* Update the trace pointer... */
+			ui32HostTracePtr++;
+			if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+			{
+				ui32HostTracePtr = 0;
+				ui32HostWrapCount++;
+			}
+			ui32Count++;
+		} while ((RGXFW_SF_LAST == ui32DataToId)  &&
+				 ui32Count < ui32TraceBufSizeInDWords);
+
+		if (ui32Count < ui32TraceBufSizeInDWords)
+		{
+			IMG_CHAR   szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN] = "%" IMG_UINT64_FMTSPEC ":T%u-%s> ";
+			IMG_CHAR   szDebugInfoBuffer[RGXFWT_DEBUG_INFO_STR_MAXLEN] = "";
+			IMG_UINT64 ui64Timestamp;
+			IMG_UINT16 ui16DebugInfo;
+
+			/* If we hit the ASSERT message then this is the end of the log... */
+			if (ui32Data == RGXFW_SF_MAIN_ASSERT_FAILED)
+			{
+				PVR_DUMPDEBUG_LOG_LINES("ASSERTION %.*s failed at %.*s:%u",
+				                        RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.szInfo,
+				                        RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.szPath,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.ui32LineNum);
+				break;
+			}
+
+			ui64Timestamp = (IMG_UINT64)(pui32TraceBuf[(ui32HostTracePtr + 0) % ui32TraceBufSizeInDWords]) << 32 |
+							(IMG_UINT64)(pui32TraceBuf[(ui32HostTracePtr + 1) % ui32TraceBufSizeInDWords]);
+
+			ui16DebugInfo = (IMG_UINT16) ((ui64Timestamp & ~RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK) >> RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT);
+			ui64Timestamp = (ui64Timestamp & ~RGXFWT_TIMESTAMP_TIME_CLRMSK) >> RGXFWT_TIMESTAMP_TIME_SHIFT;
+
+			/*
+			 * Print the trace string and provide up to 20 arguments which
+			 * printf function will be able to use. We have already checked
+			 * that no string uses more than this.
+			 */
+			OSStringLCat(szBuffer, SFs[ui32DataToId].psName, PVR_MAX_DEBUG_MESSAGE_LEN);
+
+			/* Check and append any extra debug info available */
+			if (ui16DebugInfo)
+			{
+				/* Prepare debug info string */
+				RGXPrepareExtraDebugInfo(szDebugInfoBuffer, RGXFWT_DEBUG_INFO_STR_MAXLEN, ui16DebugInfo);
+
+				/* Append debug info string */
+				OSStringLCat(szBuffer, szDebugInfoBuffer, PVR_MAX_DEBUG_MESSAGE_LEN);
+			}
+
+			PVR_DUMPDEBUG_LOG_LINES(szBuffer, ui64Timestamp, ui32TID, groups[RGXFW_SF_GID(ui32Data)],
+			                        pui32TraceBuf[(ui32HostTracePtr +  2) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  3) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  4) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  5) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  6) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  7) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  8) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  9) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 10) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 11) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 12) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 13) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 14) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 15) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 16) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 17) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 18) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 19) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 20) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 21) % ui32TraceBufSizeInDWords]);
+
+			/* Update the trace pointer... */
+			ui32HostTracePtr = ui32HostTracePtr + 2 + RGXFW_SF_PARAMNUM(ui32Data);
+			if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+			{
+				ui32HostTracePtr = ui32HostTracePtr % ui32TraceBufSizeInDWords;
+				ui32HostWrapCount++;
+			}
+			ui32Count = (ui32Count + 2 + RGXFW_SF_PARAMNUM(ui32Data));
+
+			/* Has the FW trace buffer overtaken the host pointer during the last line printed??? */
+			if ((pui32LocalTraceBuf == NULL)  &&
+			    ((*pui32FWWrapCount > ui32HostWrapCount) ||
+			     ((*pui32FWWrapCount == ui32HostWrapCount) && (*pui32FWTracePtr > ui32HostTracePtr))))
+			{
+				/* Move forward to the oldest entry again... */
+				PVR_DUMPDEBUG_LOG_LINES(". . .");
+				ui32HostWrapCount = *pui32FWWrapCount;
+				ui32HostTracePtr  = *pui32FWTracePtr;
+			}
+		}
+	}
+
+	/* Free the local copy of the trace buffer if it was allocated... */
+	if (pui32LocalTraceBuf != NULL)
+	{
+		OSFreeMem(pui32LocalTraceBuf);
+	}
+
+	if (!bPrintAllLines)
+	{
+		IMG_UINT32 ui32FirstLineIdx;
+
+		if (ui32LineCount > PVR_MAX_DEBUG_PARTIAL_LINES)
+		{
+			ui32FirstLineIdx = ui32LastLineIdx;
+			ui32LineCount = PVR_MAX_DEBUG_PARTIAL_LINES;
+		}
+		else
+		{
+			ui32FirstLineIdx = 0;
+		}
+
+		for (ui32Count = 0; ui32Count < ui32LineCount; ui32Count++)
+		{
+			PVR_DUMPDEBUG_LOG("%s", &pszLineBuffer[((ui32FirstLineIdx + ui32Count) % PVR_MAX_DEBUG_PARTIAL_LINES) * PVR_MAX_DEBUG_MESSAGE_LEN]);
+		}
+
+		OSFreeMem(pszLineBuffer);
+	}
+}
+
+void RGXDumpFirmwareTraceDecoded(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	RGXDumpFirmwareTraceLines(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+	                          psRGXFWIfTraceBufCtl, ui32TID, true);
+}
+
+void RGXDumpFirmwareTracePartial(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	RGXDumpFirmwareTraceLines(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+	                          psRGXFWIfTraceBufCtl, ui32TID, false);
+}
+
+void RGXDumpFirmwareTraceBinary(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	IMG_UINT32	i;
+	IMG_BOOL	bPrevLineWasZero = IMG_FALSE;
+	IMG_BOOL	bLineIsAllZeros = IMG_FALSE;
+	IMG_UINT32	ui32CountLines = 0;
+	IMG_UINT32	*pui32TraceBuffer;
+	IMG_CHAR *pszLine;
+
+	RGXFwSharedMemCacheOpExec(psDevInfo->apui32TraceBuffer[ui32TID],
+	                          psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+	pui32TraceBuffer = psDevInfo->apui32TraceBuffer[ui32TID];
+
+/* Max number of DWords to be printed per line, in debug dump binary output */
+#define PVR_DD_FW_TRACEBUF_LINESIZE 30U
+	/* each element in the line is 8 characters plus a space.  The '+ 1' is because of the final trailing '\0'. */
+	pszLine = OSAllocMem(9 * PVR_DD_FW_TRACEBUF_LINESIZE + 1);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Out of mem allocating line string (size: %d)",
+				__func__,
+				9 * PVR_DD_FW_TRACEBUF_LINESIZE + 1));
+		return;
+	}
+
+	for (i = 0; i < psDevInfo->ui32TraceBufSizeInDWords; i += PVR_DD_FW_TRACEBUF_LINESIZE)
+	{
+		IMG_UINT32 k = 0;
+		IMG_UINT32 ui32Line = 0x0;
+		IMG_UINT32 ui32LineOffset = i*sizeof(IMG_UINT32);
+		IMG_CHAR   *pszBuf = pszLine;
+
+		for (k = 0; k < PVR_DD_FW_TRACEBUF_LINESIZE; k++)
+		{
+			if ((i + k) >= psDevInfo->ui32TraceBufSizeInDWords)
+			{
+				/* Stop reading when the index goes beyond trace buffer size. This condition is
+				 * hit during printing the last line in DD when ui32TraceBufSizeInDWords is not
+				 * a multiple of PVR_DD_FW_TRACEBUF_LINESIZE */
+				break;
+			}
+
+			ui32Line |= pui32TraceBuffer[i + k];
+
+			/* prepare the line to print it. The '+1' is because of the trailing '\0' added */
+			OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32TraceBuffer[i + k]);
+			pszBuf += 9; /* write over the '\0' */
+		}
+
+		bLineIsAllZeros = (ui32Line == 0x0);
+
+		if (bLineIsAllZeros)
+		{
+			if (bPrevLineWasZero)
+			{
+				ui32CountLines++;
+			}
+			else
+			{
+				bPrevLineWasZero = IMG_TRUE;
+				ui32CountLines = 1;
+				PVR_DUMPDEBUG_LOG("FWT[%08x]: 00000000 ... 00000000", ui32LineOffset);
+			}
+		}
+		else
+		{
+			if (bPrevLineWasZero  &&  ui32CountLines > 1)
+			{
+				PVR_DUMPDEBUG_LOG("FWT[...]: %d lines were all zero", ui32CountLines);
+			}
+			bPrevLineWasZero = IMG_FALSE;
+
+			PVR_DUMPDEBUG_LOG("FWT[%08x]:%s", ui32LineOffset, pszLine);
+		}
+	}
+
+	if (bPrevLineWasZero)
+	{
+		PVR_DUMPDEBUG_LOG("FWT[END]: %d lines were all zero", ui32CountLines);
+	}
+
+	OSFreeMem(pszLine);
+}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+void RGXDocumentFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				const IMG_UINT32 ui32FwVA,
+				const IMG_CPU_PHYADDR sCpuPA,
+				const IMG_DEV_PHYADDR sDevPA,
+				const IMG_UINT64 ui64PTE)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		PVR_DUMPDEBUG_LOG("|    0x%08X   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "%s%s%s   |",
+						  ui32FwVA,
+						  (IMG_UINT64) sCpuPA.uiAddr,
+						  sDevPA.uiAddr,
+						  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(ui64PTE)],
+						  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(ui64PTE)],
+						  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(ui64PTE)]);
+	}
+	else
+#endif
+	{
+		const char *pszSLCBypass =
+#if defined(RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN)
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN) ? "B" : " ";
+#else
+						  " ";
+#endif
+
+		/* META and RISCV use a subset of the GPU's virtual address space */
+		PVR_DUMPDEBUG_LOG("|    0x%08X   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "%s%s%s%s%s%s   |",
+						  ui32FwVA,
+						  (IMG_UINT64) sCpuPA.uiAddr,
+						  sDevPA.uiAddr,
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN)   ? "P" : " ",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_PM_SRC_EN)          ? "PM" : "  ",
+						  pszSLCBypass,
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_CC_EN)              ? "C" : " ",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_READ_ONLY_EN)       ? "RO" : "RW",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_VALID_EN)           ? "V" : " ");
+	}
+}
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+#if !defined(NO_HARDWARE)
+static PVRSRV_ERROR
+RGXPollMetaRegThroughSP(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32RegOffset,
+                        IMG_UINT32 ui32PollValue, IMG_UINT32 ui32Mask)
+{
+	IMG_UINT32 ui32RegValue, ui32NumPolls = 0;
+	PVRSRV_ERROR eError;
+
+	do
+	{
+		eError = RGXReadFWModuleAddr(psDevInfo, ui32RegOffset, &ui32RegValue);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	} while (((ui32RegValue & ui32Mask) != ui32PollValue) && (ui32NumPolls++ < 1000));
+
+	return ((ui32RegValue & ui32Mask) == ui32PollValue) ? PVRSRV_OK : PVRSRV_ERROR_RETRY;
+}
+
+PVRSRV_ERROR
+RGXReadMetaCoreReg(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32RegAddr, IMG_UINT32 *pui32RegVal)
+{
+	PVRSRV_ERROR eError;
+
+	/* Core Read Ready? */
+	eError = RGXPollMetaRegThroughSP(psDevInfo,
+	                                 META_CR_TXUXXRXRQ_OFFSET,
+	                                 META_CR_TXUXXRXRQ_DREADY_BIT,
+									 META_CR_TXUXXRXRQ_DREADY_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXPollMetaRegThroughSP");
+
+	/* Set the reg we are interested in reading */
+	eError = RGXWriteFWModuleAddr(psDevInfo, META_CR_TXUXXRXRQ_OFFSET,
+	                        ui32RegAddr | META_CR_TXUXXRXRQ_RDnWR_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXWriteFWModuleAddr");
+
+	/* Core Read Done? */
+	eError = RGXPollMetaRegThroughSP(psDevInfo,
+	                                 META_CR_TXUXXRXRQ_OFFSET,
+	                                 META_CR_TXUXXRXRQ_DREADY_BIT,
+									 META_CR_TXUXXRXRQ_DREADY_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXPollMetaRegThroughSP");
+
+	/* Read the value */
+	return RGXReadFWModuleAddr(psDevInfo, META_CR_TXUXXRXDT_OFFSET, pui32RegVal);
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR _ValidateWithFWModule(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo,
+						RGXFWIF_DEV_VIRTADDR *psFWAddr,
+						void *pvHostCodeAddr,
+						IMG_UINT32 ui32MaxLen,
+						const IMG_CHAR *pszDesc,
+						IMG_UINT32 ui32StartOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32Value = 0;
+	IMG_UINT32 ui32FWCodeDevVAAddr = psFWAddr->ui32Addr + ui32StartOffset;
+	IMG_UINT32 *pui32FWCode = (IMG_UINT32*) IMG_OFFSET_ADDR(pvHostCodeAddr,ui32StartOffset);
+	IMG_UINT32 i;
+
+#if defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	ui32MaxLen -= ui32StartOffset;
+	ui32MaxLen /= sizeof(IMG_UINT32); /* Byte -> 32 bit words */
+
+	for (i = 0; i < ui32MaxLen; i++)
+	{
+		eError = RGXReadFWModuleAddr(psDevInfo, ui32FWCodeDevVAAddr, &ui32Value);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+#if defined(EMULATOR)
+		if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+#endif
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "0x%x: CPU 0x%08x, FW 0x%08x", i * 4, pui32FWCode[i], ui32Value));
+
+			if (pui32FWCode[i] != ui32Value)
+			{
+				PVR_DUMPDEBUG_LOG("%s: Mismatch while validating %s at offset 0x%x: CPU 0x%08x (%p), FW 0x%08x (%x)",
+					 __func__, pszDesc,
+					 (i * 4) + ui32StartOffset, pui32FWCode[i], pui32FWCode, ui32Value, ui32FWCodeDevVAAddr);
+				return PVRSRV_ERROR_FW_IMAGE_MISMATCH;
+			}
+		}
+
+		ui32FWCodeDevVAAddr += 4;
+	}
+
+	PVR_DUMPDEBUG_LOG("Match between Host and Firmware view of the %s", pszDesc);
+	return PVRSRV_OK;
+}
+#endif
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXValidateFWImage(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if !defined(SUPPORT_TRUSTED_DEVICE)
+	PVRSRV_ERROR eError;
+	IMG_UINT32 *pui32HostFWCode = NULL, *pui32HostFWCoremem = NULL;
+	OS_FW_IMAGE *psRGXFW = NULL;
+	const IMG_BYTE *pbRGXFirmware = NULL;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_UINT32 *pui32CodeMemoryPointer;
+#endif
+	RGXFWIF_DEV_VIRTADDR sFWAddr;
+	IMG_UINT32 ui32StartOffset = 0;
+
+#if defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		PVR_DUMPDEBUG_LOG("Validation of RISC-V FW code is disabled on emulator");
+		return PVRSRV_OK;
+	}
+#endif
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: RGX registers not mapped yet!", __func__));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	/* Load FW from system for code verification */
+	pui32HostFWCode = OSAllocZMem(psDevInfo->ui32FWCodeSizeInBytes);
+	if (pui32HostFWCode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed in allocating memory for FW code. "
+				"So skipping FW code verification",
+				__func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Coremem is not present on all GPU cores, so size can be zero */
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		pui32HostFWCoremem = OSAllocZMem(psDevInfo->ui32FWCorememCodeSizeInBytes);
+		if (pui32HostFWCoremem == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed in allocating memory for FW core code. "
+					"So skipping FW code verification",
+					__func__));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto freeHostFWCode;
+		}
+	}
+
+	/* Load FW image */
+	eError = RGXLoadAndGetFWData(psDevInfo->psDeviceNode, &psRGXFW);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to load FW image file (%s).",
+		         __func__, PVRSRVGetErrorString(eError)));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto cleanup_initfw;
+	}
+
+	pbRGXFirmware = (const IMG_BYTE *)OSFirmwareData(psRGXFW);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		eError = ProcessLDRCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+						(void*) pui32HostFWCode, NULL,
+						(void*) pui32HostFWCoremem, NULL, NULL);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = ProcessELFCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+		                                 pui32HostFWCode, NULL,
+		                                 NULL, NULL);
+	}
+#endif
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = ProcessELFCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+		                                 pui32HostFWCode, NULL,
+		                                 pui32HostFWCoremem, NULL);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed in parsing FW image file.", __func__));
+		goto cleanup_initfw;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, (void **)&pui32CodeMemoryPointer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Error in acquiring MIPS FW code memory area (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+
+		RGXFwSharedMemCacheOpExec(pui32CodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, PVRSRV_CACHE_OP_INVALIDATE);
+
+		if (OSMemCmp(pui32HostFWCode, pui32CodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes) == 0)
+		{
+			PVR_DUMPDEBUG_LOG("Match between Host and MIPS views of the FW code" );
+		}
+		else
+		{
+			IMG_UINT32 ui32Count = 10; /* Show only the first 10 mismatches */
+			IMG_UINT32 ui32Offset;
+
+			PVR_DUMPDEBUG_LOG("Mismatch between Host and MIPS views of the FW code");
+			for (ui32Offset = 0; (ui32Offset*4 < psDevInfo->ui32FWCodeSizeInBytes) || (ui32Count == 0); ui32Offset++)
+			{
+				if (pui32HostFWCode[ui32Offset] != pui32CodeMemoryPointer[ui32Offset])
+				{
+					PVR_DUMPDEBUG_LOG("At %d bytes, code should be 0x%x but it is instead 0x%x",
+					   ui32Offset*4, pui32HostFWCode[ui32Offset], pui32CodeMemoryPointer[ui32Offset]);
+					ui32Count--;
+				}
+			}
+		}
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+	else
+#endif
+	{
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			/* starting checking after BOOT LOADER config */
+			sFWAddr.ui32Addr = RGXFW_BOOTLDR_META_ADDR;
+
+			ui32StartOffset = RGXFW_MAX_BOOTLDR_OFFSET;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			/* Use bootloader code remap which is always configured before the FW is started */
+			if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+			{
+				sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE;
+			}
+			else
+#endif
+			{
+				sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+			}
+		}
+
+		eError = _ValidateWithFWModule(pfnDumpDebugPrintf, pvDumpDebugFile,
+						psDevInfo, &sFWAddr,
+						pui32HostFWCode, psDevInfo->ui32FWCodeSizeInBytes,
+						"FW code", ui32StartOffset);
+		if (eError != PVRSRV_OK)
+		{
+			goto cleanup_initfw;
+		}
+
+		/* Coremem is not present on all GPU cores, so may not be alloc'd */
+		if (pui32HostFWCoremem != NULL) // && psDevInfo->ui32FWCorememCodeSizeInBytes
+		{
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+			{
+				sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, META_COREMEM_CODE);
+			}
+			else
+			{
+				sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, RISCV_COREMEM_CODE);
+
+				/* Core must be halted while issuing abstract commands */
+				eError = RGXRiscvHalt(psDevInfo);
+				PVR_GOTO_IF_ERROR(eError, cleanup_initfw);
+			}
+
+			eError = _ValidateWithFWModule(pfnDumpDebugPrintf, pvDumpDebugFile,
+							psDevInfo, &sFWAddr,
+							pui32HostFWCoremem, psDevInfo->ui32FWCorememCodeSizeInBytes,
+							"FW coremem code", 0);
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+			{
+				eError = RGXRiscvResume(psDevInfo);
+				PVR_GOTO_IF_ERROR(eError, cleanup_initfw);
+			}
+		}
+	}
+
+cleanup_initfw:
+	if (psRGXFW)
+	{
+		OSUnloadFirmware(psRGXFW);
+	}
+
+	if (pui32HostFWCoremem)
+	{
+		OSFreeMem(pui32HostFWCoremem);
+	}
+freeHostFWCode:
+	if (pui32HostFWCode)
+	{
+		OSFreeMem(pui32HostFWCode);
+	}
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+PVRSRV_ERROR ValidateFWOnLoad(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_TRUSTED_DEVICE)
+	IMG_PBYTE pbCodeMemoryPointer;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR sFWAddr;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, (void **)&pbCodeMemoryPointer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	RGXFwSharedMemCacheOpExec(pbCodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, PVRSRV_CACHE_OP_INVALIDATE);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		sFWAddr.ui32Addr = RGXFW_BOOTLDR_META_ADDR;
+	}
+	else
+	{
+		PVR_ASSERT(RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR));
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+		{
+			sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE;
+		}
+		else
+#endif
+		{
+			sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+		}
+	};
+
+	eError = _ValidateWithFWModule(NULL, NULL, psDevInfo, &sFWAddr, pbCodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, "FW code", 0);
+	if (eError != PVRSRV_OK)
+	{
+		goto releaseFWCodeMapping;
+	}
+
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, (void **)&pbCodeMemoryPointer);
+		if (eError != PVRSRV_OK)
+		{
+			goto releaseFWCoreCodeMapping;
+		}
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, META_COREMEM_CODE);
+		}
+		else
+		{
+			PVR_ASSERT(RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR));
+			sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, RISCV_COREMEM_CODE);
+		}
+
+		eError = _ValidateWithFWModule(NULL, NULL, psDevInfo, &sFWAddr, pbCodeMemoryPointer,
+						psDevInfo->ui32FWCorememCodeSizeInBytes, "FW coremem code", 0);
+	}
+
+releaseFWCoreCodeMapping:
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+releaseFWCodeMapping:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+#endif
+}
+#endif
+
+static const IMG_CHAR* _RGXGetDebugDevPowerStateString(PVRSRV_DEV_POWER_STATE ePowerState)
+{
+	switch (ePowerState)
+	{
+		case PVRSRV_DEV_POWER_STATE_DEFAULT: return "DEFAULT";
+		case PVRSRV_DEV_POWER_STATE_OFF: return "OFF";
+		case PVRSRV_DEV_POWER_STATE_ON: return "ON";
+		default: return "UNKNOWN";
+	}
+}
+
+/*
+	Writes flags strings to an uninitialised buffer.
+*/
+static void _GetFwSysFlagsDescription(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, IMG_UINT32 ui32RawFlags)
+{
+	const IMG_CHAR szCswLabel[] = "Ctx switch options:";
+	size_t uLabelLen = sizeof(szCswLabel) - 1;
+	const size_t uiBytesPerDesc = (ui32DescSize - uLabelLen) / 2U - 1U;
+
+	OSStringSafeCopy(psDesc, szCswLabel, ui32DescSize);
+
+	DebugCommonFlagStrings(psDesc, uiBytesPerDesc + uLabelLen, asCswOpts2Description, ARRAY_SIZE(asCswOpts2Description), ui32RawFlags);
+	DebugCommonFlagStrings(psDesc, ui32DescSize, asMisc2Description, ARRAY_SIZE(asMisc2Description), ui32RawFlags);
+}
+
+static void _GetFwOsFlagsDescription(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, IMG_UINT32 ui32RawFlags)
+{
+	const IMG_CHAR szCswLabel[] = "Ctx switch:";
+	size_t uLabelLen = sizeof(szCswLabel) - 1;
+	const size_t uiBytesPerDesc = (ui32DescSize - uLabelLen) / 2U - 1U;
+
+	OSStringSafeCopy(psDesc, szCswLabel, ui32DescSize);
+
+	DebugCommonFlagStrings(psDesc, uiBytesPerDesc + uLabelLen, asFwOsCfg2Description, ARRAY_SIZE(asFwOsCfg2Description), ui32RawFlags);
+}
+
+
+typedef enum _DEVICEMEM_HISTORY_QUERY_INDEX_
+{
+	DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING,
+	DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED,
+	DEVICEMEM_HISTORY_QUERY_INDEX_NEXT,
+	DEVICEMEM_HISTORY_QUERY_INDEX_COUNT,
+} DEVICEMEM_HISTORY_QUERY_INDEX;
+
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintDevicememHistoryQueryResult
+
+ @Description
+
+ Print details of a single result from a DevicememHistory query
+
+ @Input pfnDumpDebugPrintf       - Debug printf function
+ @Input pvDumpDebugFile          - Optional file identifier to be passed to the
+                                   'printf' function if required
+ @Input psFaultProcessInfo       - The process info derived from the page fault
+ @Input psResult                 - The DevicememHistory result to be printed
+ @Input ui32Index                - The index of the result
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintDevicememHistoryQueryResult(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						RGXMEM_PROCESS_INFO *psFaultProcessInfo,
+						DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult,
+						IMG_UINT32 ui32Index,
+						const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+	RGXConvertOSTimestampToSAndNS(psResult->ui64When,
+							&ui64Seconds,
+							&ui64Nanoseconds);
+
+	if (psFaultProcessInfo->uiPID != RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		PVR_DUMPDEBUG_LOG("%s    [%u] Name: %s Base address: " IMG_DEV_VIRTADDR_FMTSPEC
+					" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+					" Operation: %s Modified: %" IMG_UINT64_FMTSPEC
+					" us ago (OS time %" IMG_UINT64_FMTSPEC
+					".%09" IMG_UINT64_FMTSPEC " s)",
+						pszIndent,
+						ui32Index,
+						psResult->szString,
+						psResult->sBaseDevVAddr.uiAddr,
+						psResult->uiSize,
+						psResult->bMap ? "Map": "Unmap",
+						OSDivide64r64(psResult->ui64Age, 1000, &ui32Remainder),
+						ui64Seconds,
+						ui64Nanoseconds);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s    [%u] Name: %s Base address: " IMG_DEV_VIRTADDR_FMTSPEC
+					" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+					" Operation: %s Modified: %" IMG_UINT64_FMTSPEC
+					" us ago (OS time %" IMG_UINT64_FMTSPEC
+					".%09" IMG_UINT64_FMTSPEC
+					") PID: %u (%s)",
+						pszIndent,
+						ui32Index,
+						psResult->szString,
+						psResult->sBaseDevVAddr.uiAddr,
+						psResult->uiSize,
+						psResult->bMap ? "Map": "Unmap",
+						OSDivide64r64(psResult->ui64Age, 1000, &ui32Remainder),
+						ui64Seconds,
+						ui64Nanoseconds,
+						psResult->sProcessInfo.uiPID,
+						psResult->sProcessInfo.szProcessName);
+	}
+
+	if (!psResult->bRange)
+	{
+		PVR_DUMPDEBUG_LOG("%s        Whole allocation was %s", pszIndent, psResult->bMap ? "mapped": "unmapped");
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s        Pages %u to %u (" IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC ") %s%s",
+										pszIndent,
+										psResult->ui32StartPage,
+										psResult->ui32StartPage + psResult->ui32PageCount - 1,
+										psResult->sMapStartAddr.uiAddr,
+										psResult->sMapEndAddr.uiAddr,
+										psResult->bAll ? "(whole allocation) " : "",
+										psResult->bMap ? "mapped": "unmapped");
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintDevicememHistoryQueryOut
+
+ @Description
+
+ Print details of all the results from a DevicememHistory query
+
+ @Input pfnDumpDebugPrintf       - Debug printf function
+ @Input pvDumpDebugFile          - Optional file identifier to be passed to the
+                                   'printf' function if required
+ @Input psFaultProcessInfo       - The process info derived from the page fault
+ @Input psQueryOut               - Storage for the query results
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintDevicememHistoryQueryOut(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						RGXMEM_PROCESS_INFO *psFaultProcessInfo,
+						DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+						const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 i;
+
+	if (psQueryOut->ui32NumResults == 0)
+	{
+		PVR_DUMPDEBUG_LOG("%s    No results", pszIndent);
+	}
+	else
+	{
+		for (i = 0; i < psQueryOut->ui32NumResults; i++)
+		{
+			_PrintDevicememHistoryQueryResult(pfnDumpDebugPrintf, pvDumpDebugFile,
+									psFaultProcessInfo,
+									&psQueryOut->sResults[i],
+									i,
+									pszIndent);
+		}
+	}
+}
+
+/* table of HW page size values and the equivalent */
+static const unsigned int aui32HWPageSizeTable[][2] =
+{
+	{ 0, PVRSRV_4K_PAGE_SIZE },
+	{ 1, PVRSRV_16K_PAGE_SIZE },
+	{ 2, PVRSRV_64K_PAGE_SIZE },
+	{ 3, PVRSRV_256K_PAGE_SIZE },
+	{ 4, PVRSRV_1M_PAGE_SIZE },
+	{ 5, PVRSRV_2M_PAGE_SIZE }
+};
+
+/*!
+*******************************************************************************
+
+ @Function	_PageSizeHWToBytes
+
+ @Description
+
+ Convert a HW page size value to its size in bytes
+
+ @Input ui32PageSizeHW     - The HW page size value
+
+ @Return   IMG_UINT32      The page size in bytes
+
+******************************************************************************/
+static IMG_UINT32 _PageSizeHWToBytes(IMG_UINT32 ui32PageSizeHW)
+{
+	if (ui32PageSizeHW > 5)
+	{
+		/* This is invalid, so return a default value as we cannot ASSERT in this code! */
+		return PVRSRV_4K_PAGE_SIZE;
+	}
+
+	return aui32HWPageSizeTable[ui32PageSizeHW][1];
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetDevicememHistoryData
+
+ @Description
+
+ Get the DevicememHistory results for the given PID and faulting device virtual address.
+ The function will query DevicememHistory for information about the faulting page, as well
+ as the page before and after.
+
+ @Input psDeviceNode       - The device which this allocation search should be made on
+ @Input uiPID              - The process ID to search for allocations belonging to
+ @Input sFaultDevVAddr     - The device address to search for allocations at/before/after
+ @Input asQueryOut         - Storage for the query results
+ @Input ui32PageSizeBytes  - Faulted page size in bytes
+
+ @Return IMG_BOOL          - IMG_TRUE if any results were found for this page fault
+
+******************************************************************************/
+static IMG_BOOL _GetDevicememHistoryData(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sFaultDevVAddr,
+							DEVICEMEM_HISTORY_QUERY_OUT asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_COUNT],
+							IMG_UINT32 ui32PageSizeBytes)
+{
+	DEVICEMEM_HISTORY_QUERY_IN sQueryIn;
+	IMG_BOOL bAnyHits = IMG_FALSE;
+
+	/* if the page fault originated in the firmware then the allocation may
+	 * appear to belong to any PID, because FW allocations are attributed
+	 * to the client process creating the allocation, so instruct the
+	 * devicemem_history query to search all available PIDs
+	 */
+	if (uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		sQueryIn.uiPID = DEVICEMEM_HISTORY_PID_ANY;
+	}
+	else
+	{
+		sQueryIn.uiPID = uiPID;
+	}
+
+	sQueryIn.psDevNode = psDeviceNode;
+	/* Query the DevicememHistory for all allocations in the previous page... */
+	sQueryIn.sDevVAddr.uiAddr = (sFaultDevVAddr.uiAddr & ~(IMG_UINT64)(ui32PageSizeBytes - 1)) - ui32PageSizeBytes;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING],
+	                          ui32PageSizeBytes, IMG_TRUE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+
+	/* Query the DevicememHistory for any record at the exact address... */
+	sQueryIn.sDevVAddr = sFaultDevVAddr;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED],
+	                          ui32PageSizeBytes, IMG_FALSE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+	else
+	{
+		/* If not matched then try matching any record in the faulting page... */
+		if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED],
+		                          ui32PageSizeBytes, IMG_TRUE))
+		{
+			bAnyHits = IMG_TRUE;
+		}
+	}
+
+	/* Query the DevicememHistory for all allocations in the next page... */
+	sQueryIn.sDevVAddr.uiAddr = (sFaultDevVAddr.uiAddr & ~(IMG_UINT64)(ui32PageSizeBytes - 1)) + ui32PageSizeBytes;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_NEXT],
+	                          ui32PageSizeBytes, IMG_TRUE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+
+	return bAnyHits;
+}
+
+/* stored data about one page fault */
+typedef struct _FAULT_INFO_
+{
+	/* the process info of the memory context that page faulted */
+	RGXMEM_PROCESS_INFO sProcessInfo;
+	IMG_DEV_VIRTADDR sFaultDevVAddr;
+	MMU_FAULT_DATA   sMMUFaultData;
+	DEVICEMEM_HISTORY_QUERY_OUT asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_COUNT];
+	/* the CR timer value at the time of the fault, recorded by the FW.
+	 * used to differentiate different page faults
+	 */
+	IMG_UINT64 ui64CRTimer;
+	/* time when this FAULT_INFO entry was added. used for timing
+	 * reference against the map/unmap information
+	 */
+	IMG_UINT64 ui64When;
+	IMG_UINT32 ui32FaultInfoFlags;
+} FAULT_INFO;
+
+/* history list of page faults.
+ * Keeps the first `n` page faults and the last `n` page faults, like the FW
+ * HWR log
+ */
+typedef struct _FAULT_INFO_LOG_
+{
+	IMG_UINT32 ui32Head;
+	/* the number of faults in this log need not correspond exactly to
+	 * the HWINFO number of the FW, as the FW HWINFO log may contain
+	 * non-page fault HWRs
+	 */
+	FAULT_INFO asFaults[RGXFWIF_HWINFO_MAX];
+} FAULT_INFO_LOG;
+
+#define FAULT_INFO_PROC_INFO   (0x1U)
+#define FAULT_INFO_DEVMEM_HIST (0x2U)
+
+static FAULT_INFO_LOG gsFaultInfoLog = { 0 };
+
+static void _FillAppForFWFaults(PVRSRV_RGXDEV_INFO *psDevInfo,
+							FAULT_INFO *psInfo)
+{
+	IMG_UINT32 i, j;
+
+	for (i = 0; i < DEVICEMEM_HISTORY_QUERY_INDEX_COUNT; i++)
+	{
+		for (j = 0; j < DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS; j++)
+		{
+			IMG_BOOL bFound;
+
+			RGXMEM_PROCESS_INFO *psProcInfo = &psInfo->asQueryOut[i].sResults[j].sProcessInfo;
+			bFound = RGXPCPIDToProcessInfo(psDevInfo,
+								psProcInfo->uiPID,
+								psProcInfo);
+			if (!bFound)
+			{
+				OSStringSafeCopy(psProcInfo->szProcessName,
+								"(unknown)",
+								sizeof(psProcInfo->szProcessName));
+			}
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintFaultInfo
+
+ @Description
+
+ Print all the details of a page fault from a FAULT_INFO structure
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psInfo               - The page fault occurrence to print
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintFaultInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					PVRSRV_DEVICE_NODE *psDevNode,
+					void *pvDumpDebugFile,
+					FAULT_INFO *psInfo,
+					const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+	RGXConvertOSTimestampToSAndNS(psInfo->ui64When, &ui64Seconds, &ui64Nanoseconds);
+
+	if (BITMASK_HAS(psInfo->ui32FaultInfoFlags, FAULT_INFO_PROC_INFO))
+	{
+		IMG_PID uiPID = (psInfo->sProcessInfo.uiPID == RGXMEM_SERVER_PID_FIRMWARE || psInfo->sProcessInfo.uiPID == RGXMEM_SERVER_PID_PM) ?
+							0 : psInfo->sProcessInfo.uiPID;
+
+		PVR_DUMPDEBUG_LOG("%sDevice memory history for page fault address " IMG_DEV_VIRTADDR_FMTSPEC
+							", CRTimer: 0x%016" IMG_UINT64_FMTSPECX
+							", PID: %u (%s, unregistered: %u) OS time: "
+							"%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+					pszIndent,
+					psInfo->sFaultDevVAddr.uiAddr,
+					psInfo->ui64CRTimer,
+					uiPID,
+					psInfo->sProcessInfo.szProcessName,
+					psInfo->sProcessInfo.bUnregistered,
+					ui64Seconds,
+					ui64Nanoseconds);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%sCould not find PID for device memory history on PC of the fault", pszIndent);
+	}
+
+	if (BITMASK_HAS(psInfo->ui32FaultInfoFlags, FAULT_INFO_DEVMEM_HIST))
+	{
+		for (i = DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING; i < DEVICEMEM_HISTORY_QUERY_INDEX_COUNT; i++)
+		{
+			const IMG_CHAR *pszWhich = NULL;
+
+			switch (i)
+			{
+				case DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING:
+					pszWhich = "Preceding page";
+					break;
+				case DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED:
+					pszWhich = "Faulted page";
+					break;
+				case DEVICEMEM_HISTORY_QUERY_INDEX_NEXT:
+					pszWhich = "Next page";
+					break;
+			}
+
+			PVR_DUMPDEBUG_LOG("%s  %s:", pszIndent, pszWhich);
+			_PrintDevicememHistoryQueryOut(pfnDumpDebugPrintf, pvDumpDebugFile,
+								&psInfo->sProcessInfo,
+								&psInfo->asQueryOut[i],
+								pszIndent);
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s  No matching Devmem History for fault address", pszIndent);
+		DevicememHistoryDumpRecordStats(psDevNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+		PVR_DUMPDEBUG_LOG("%s  Records Searched -"
+		                  " PP:%"IMG_UINT64_FMTSPEC
+		                  " FP:%"IMG_UINT64_FMTSPEC
+		                  " NP:%"IMG_UINT64_FMTSPEC,
+		                  pszIndent,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING].ui64SearchCount,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED].ui64SearchCount,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_NEXT].ui64SearchCount);
+	}
+}
+
+static void _RecordFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					FAULT_INFO *psInfo,
+					IMG_DEV_VIRTADDR sFaultDevVAddr,
+					IMG_DEV_PHYADDR sPCDevPAddr,
+					IMG_UINT64 ui64CRTimer,
+					IMG_UINT32 ui32PageSizeBytes)
+{
+	IMG_BOOL bFound = IMG_FALSE, bIsPMFault = IMG_FALSE;
+	RGXMEM_PROCESS_INFO sProcessInfo;
+
+	psInfo->ui32FaultInfoFlags = 0;
+	psInfo->sFaultDevVAddr = sFaultDevVAddr;
+	psInfo->ui64CRTimer = ui64CRTimer;
+	psInfo->ui64When = OSClockns64();
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		/* Check if this is PM fault */
+		if (psInfo->sMMUFaultData.eType == MMU_FAULT_TYPE_PM)
+		{
+			bIsPMFault = IMG_TRUE;
+			bFound = IMG_TRUE;
+			sProcessInfo.uiPID = RGXMEM_SERVER_PID_PM;
+			OSStringSafeCopy(sProcessInfo.szProcessName, "PM", sizeof(sProcessInfo.szProcessName));
+			sProcessInfo.szProcessName[sizeof(sProcessInfo.szProcessName) - 1] = '\0';
+			sProcessInfo.bUnregistered = IMG_FALSE;
+		}
+		else
+		{
+			/* look up the process details for the faulting page catalogue */
+			bFound = RGXPCAddrToProcessInfo(psDevInfo, sPCDevPAddr, &sProcessInfo);
+		}
+
+		if (bFound)
+		{
+			IMG_BOOL bHits;
+
+			psInfo->ui32FaultInfoFlags = FAULT_INFO_PROC_INFO;
+			psInfo->sProcessInfo = sProcessInfo;
+
+			if (bIsPMFault)
+			{
+				bHits = IMG_TRUE;
+			}
+			else
+			{
+				/* get any DevicememHistory data for the faulting address */
+				bHits = _GetDevicememHistoryData(psDevInfo->psDeviceNode,
+								 sProcessInfo.uiPID,
+								 sFaultDevVAddr,
+								 psInfo->asQueryOut,
+								 ui32PageSizeBytes);
+
+				if (bHits)
+				{
+					psInfo->ui32FaultInfoFlags |= FAULT_INFO_DEVMEM_HIST;
+
+					/* if the page fault was caused by the firmware then get information about
+					 * which client application created the related allocations.
+					 *
+					 * Fill in the process info data for each query result.
+					 */
+
+					if (sProcessInfo.uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+					{
+						_FillAppForFWFaults(psDevInfo, psInfo);
+					}
+				}
+			}
+		}
+	}
+}
+
+void RGXDumpFaultAddressHostView(MMU_FAULT_DATA *psFaultData,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const IMG_CHAR* pszIndent)
+{
+	MMU_LEVEL eTopLevel;
+	const IMG_CHAR szPageLevel[][4] = {"", "PTE", "PDE", "PCE" };
+	const IMG_CHAR szPageError[][3] = {"", "PT",  "PD",  "PC"  };
+
+	eTopLevel = psFaultData->eTopLevel;
+
+	if (psFaultData->eType == MMU_FAULT_TYPE_UNKNOWN)
+	{
+		PVR_DUMPDEBUG_LOG("%sNo live host MMU data available", pszIndent);
+		return;
+	}
+	else if (psFaultData->eType == MMU_FAULT_TYPE_PM)
+	{
+		PVR_DUMPDEBUG_LOG("%sPM faulted at PC address = 0x%016" IMG_UINT64_FMTSPECx, pszIndent, psFaultData->sLevelData[MMU_LEVEL_0].ui64Address);
+	}
+	else
+	{
+		MMU_LEVEL eCurrLevel;
+		PVR_ASSERT(eTopLevel < MMU_LEVEL_LAST);
+
+		for (eCurrLevel = eTopLevel; eCurrLevel > MMU_LEVEL_0; eCurrLevel--)
+		{
+			MMU_LEVEL_DATA *psMMULevelData = &psFaultData->sLevelData[eCurrLevel];
+			if (psMMULevelData->ui64Address)
+			{
+				if (psMMULevelData->uiBytesPerEntry == 4)
+				{
+					PVR_DUMPDEBUG_LOG("%s%s for index %d = 0x%08x and is %s",
+								pszIndent,
+								szPageLevel[eCurrLevel],
+								psMMULevelData->ui32Index,
+								(IMG_UINT) psMMULevelData->ui64Address,
+								psMMULevelData->psDebugStr);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("%s%s for index %d = 0x%016" IMG_UINT64_FMTSPECx " and is %s",
+								pszIndent,
+								szPageLevel[eCurrLevel],
+								psMMULevelData->ui32Index,
+								psMMULevelData->ui64Address,
+								psMMULevelData->psDebugStr);
+				}
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("%s%s index (%d) out of bounds (%d)",
+							pszIndent,
+							szPageError[eCurrLevel],
+							psMMULevelData->ui32Index,
+							psMMULevelData->ui32NumOfEntries);
+				break;
+			}
+		}
+	}
+
+}
+
+void RGXDumpFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile,
+					  const RGX_HWRINFO *psHWRInfo,
+                      IMG_UINT32 ui32ReadIndex,
+                      IMG_DEV_VIRTADDR *psFaultDevVAddr,
+                      IMG_DEV_PHYADDR *psPCDevPAddr,
+                      bool bPMFault,
+                      IMG_UINT32 ui32PageSize)
+{
+	FAULT_INFO *psInfo;
+
+	OSLockAcquire(psDevInfo->hDebugFaultInfoLock);
+
+	/* Find the matching Fault Info for this HWRInfo */
+	psInfo = &gsFaultInfoLog.asFaults[ui32ReadIndex];
+
+	/* if they do not match, we need to update the psInfo */
+	if ((psInfo->ui64CRTimer != psHWRInfo->ui64CRTimer) ||
+		(psInfo->sFaultDevVAddr.uiAddr != psFaultDevVAddr->uiAddr))
+	{
+		MMU_FAULT_DATA *psFaultData = &psInfo->sMMUFaultData;
+
+		psFaultData->eType = MMU_FAULT_TYPE_UNKNOWN;
+
+		if (bPMFault)
+		{
+			/* PM fault and we dump PC details only */
+			psFaultData->eTopLevel = MMU_LEVEL_0;
+			psFaultData->eType     = MMU_FAULT_TYPE_PM;
+			psFaultData->sLevelData[MMU_LEVEL_0].ui64Address = psPCDevPAddr->uiAddr;
+		}
+		else
+		{
+			RGXCheckFaultAddress(psDevInfo, psFaultDevVAddr, psPCDevPAddr, psFaultData);
+		}
+
+		_RecordFaultInfo(psDevInfo, psInfo,
+					*psFaultDevVAddr, *psPCDevPAddr, psHWRInfo->ui64CRTimer,
+					_PageSizeHWToBytes(ui32PageSize));
+
+	}
+
+	RGXDumpFaultAddressHostView(&psInfo->sMMUFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_NORMAL_INDENT);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		_PrintFaultInfo(pfnDumpDebugPrintf, psDevInfo->psDeviceNode, pvDumpDebugFile, psInfo, DD_NORMAL_INDENT);
+	}
+
+	OSLockRelease(psDevInfo->hDebugFaultInfoLock);
+}
+
+void RGXConvertOSTimestampToSAndNS(IMG_UINT64 ui64OSTimer,
+							IMG_UINT64 *pui64Seconds,
+							IMG_UINT64 *pui64Nanoseconds)
+{
+	IMG_UINT32 ui32Remainder;
+
+	*pui64Seconds = OSDivide64r64(ui64OSTimer, 1000000000, &ui32Remainder);
+	*pui64Nanoseconds = ui64OSTimer - (*pui64Seconds * 1000000000ULL);
+}
+
+#if !defined(NO_HARDWARE)
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static PVRSRV_ERROR _RGXMipsExtraDebug(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_MIPS_STATE *psMIPSState)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	IMG_UINT32 ui32RegRead;
+	IMG_UINT32 eError = PVRSRV_OK;
+	IMG_UINT32 volatile *pui32SyncFlag;
+
+	/* Acquire the NMI operations lock */
+	OSLockAcquire(psDevInfo->hNMILock);
+
+	/* Make sure the synchronisation flag is set to 0 */
+	pui32SyncFlag = &psDevInfo->psRGXFWIfSysInit->sMIPSState.ui32Sync;
+	*pui32SyncFlag = 0;
+
+	/* Readback performed as a part of memory barrier */
+	OSWriteMemoryBarrier(pui32SyncFlag);
+	RGXFwSharedMemCacheOpPtr(pui32SyncFlag,
+	                         FLUSH);
+
+
+	/* Enable NMI issuing in the MIPS wrapper */
+	OSWriteHWReg64(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_EN);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_ENABLE);
+
+	/* Check the MIPS is not in error state already (e.g. it is booting or an NMI has already been requested) */
+	ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_EXCEPTION_STATUS);
+	if ((ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN) || (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN))
+	{
+
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Issue NMI */
+	OSWriteHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_EVENT,
+				   RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_EN);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_EVENT);
+
+
+	/* Wait for NMI Taken to be asserted */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+									RGX_CR_MIPS_EXCEPTION_STATUS);
+		if (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if ((ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN) == 0)
+	{
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Allow the firmware to proceed */
+	*pui32SyncFlag = 1;
+
+	/* Readback performed as a part of memory barrier */
+	OSWriteMemoryBarrier(pui32SyncFlag);
+	RGXFwSharedMemCacheOpPtr(pui32SyncFlag,
+	                         FLUSH);
+
+
+	/* Wait for the FW to have finished the NMI routine */
+	ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+								RGX_CR_MIPS_EXCEPTION_STATUS);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+									RGX_CR_MIPS_EXCEPTION_STATUS);
+		if (!(ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	if (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN)
+	{
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Copy state */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->sMIPSState,
+	                           INVALIDATE);
+	OSDeviceMemCopy(psMIPSState, &psDevInfo->psRGXFWIfSysInit->sMIPSState, sizeof(*psMIPSState));
+
+	--(psMIPSState->ui32ErrorEPC);
+	--(psMIPSState->ui32EPC);
+
+	/* Disable NMI issuing in the MIPS wrapper */
+	OSWriteHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE,
+				   0);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_ENABLE);
+
+fail:
+	/* Release the NMI operations lock */
+	OSLockRelease(psDevInfo->hNMILock);
+	return eError;
+}
+
+/* Print decoded information from cause register */
+static void _RGXMipsDumpCauseDecode(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile,
+                                    IMG_UINT32 ui32Cause,
+                                    IMG_UINT32 ui32ErrorState)
+{
+#define INDENT "    "
+	const IMG_UINT32 ui32ExcCode = RGXMIPSFW_C0_CAUSE_EXCCODE(ui32Cause);
+	const IMG_CHAR * const pszException = _GetMIPSExcString(ui32ExcCode);
+
+	if (ui32ErrorState != 0 &&
+	    pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Cause exception: %s", pszException);
+	}
+
+	if (ui32Cause & RGXMIPSFW_C0_CAUSE_FDCIPENDING)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "FDC interrupt pending");
+	}
+
+	if (!(ui32Cause & RGXMIPSFW_C0_CAUSE_IV))
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Interrupt uses general interrupt vector");
+	}
+
+	if (ui32Cause & RGXMIPSFW_C0_CAUSE_PCIPENDING)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Performance Counter Interrupt pending");
+	}
+
+	/* Unusable Coproc exception */
+	if (ui32ExcCode == 11)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Unusable Coprocessor: %d", RGXMIPSFW_C0_CAUSE_UNUSABLE_UNIT(ui32Cause));
+	}
+
+#undef INDENT
+}
+
+static IMG_BOOL _IsFWCodeException(IMG_UINT32 ui32ExcCode)
+{
+	if (ui32ExcCode >= sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Only %lu exceptions available in MIPS, %u is not a valid exception code",
+		         (unsigned long)sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING), ui32ExcCode));
+		return IMG_FALSE;
+	}
+
+	return apsMIPSExcCodes[ui32ExcCode].bIsFatal;
+}
+
+static void _RGXMipsDumpDebugDecode(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32Debug,
+					IMG_UINT32 ui32DEPC)
+{
+	const IMG_CHAR *pszDException = NULL;
+	IMG_UINT32 i;
+#define INDENT "    "
+
+	if (!(ui32Debug & RGXMIPSFW_C0_DEBUG_DM))
+	{
+		return;
+	}
+
+	PVR_DUMPDEBUG_LOG("DEBUG                        :");
+
+	pszDException = _GetMIPSExcString(RGXMIPSFW_C0_DEBUG_EXCCODE(ui32Debug));
+
+	if (pszDException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Debug exception: %s", pszDException);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sMIPS_C0_DebugTable); ++i)
+	{
+		const RGXMIPSFW_C0_DEBUG_TBL_ENTRY * const psDebugEntry = &sMIPS_C0_DebugTable[i];
+
+		if (ui32Debug & psDebugEntry->ui32Mask)
+		{
+			PVR_DUMPDEBUG_LOG(INDENT "%s", psDebugEntry->pszExplanation);
+		}
+	}
+#undef INDENT
+	PVR_DUMPDEBUG_LOG("DEPC                    :0x%08X", ui32DEPC);
+}
+
+static inline void _GetMipsTLBPARanges(const RGX_MIPS_TLB_ENTRY *psTLBEntry,
+                                       const RGX_MIPS_REMAP_ENTRY *psRemapEntry0,
+                                       const RGX_MIPS_REMAP_ENTRY *psRemapEntry1,
+                                       IMG_UINT64 *pui64PA0Start,
+                                       IMG_UINT64 *pui64PA0End,
+                                       IMG_UINT64 *pui64PA1Start,
+                                       IMG_UINT64 *pui64PA1End)
+{
+	IMG_BOOL bUseRemapOutput = (psRemapEntry0 != NULL && psRemapEntry1 != NULL) ? IMG_TRUE : IMG_FALSE;
+	IMG_UINT64 ui64PageSize = RGXMIPSFW_TLB_GET_PAGE_SIZE(psTLBEntry->ui32TLBPageMask);
+
+	if ((psTLBEntry->ui32TLBLo0 & RGXMIPSFW_TLB_VALID) == 0)
+	{
+		/* Dummy values to fail the range checks later */
+		*pui64PA0Start = -1ULL;
+		*pui64PA0End   = -1ULL;
+	}
+	else if (bUseRemapOutput)
+	{
+		*pui64PA0Start = (IMG_UINT64)psRemapEntry0->ui32RemapAddrOut << 12;
+		*pui64PA0End   = *pui64PA0Start + ui64PageSize - 1;
+	}
+	else
+	{
+		*pui64PA0Start = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo0);
+		*pui64PA0End   = *pui64PA0Start + ui64PageSize - 1;
+	}
+
+	if ((psTLBEntry->ui32TLBLo1 & RGXMIPSFW_TLB_VALID) == 0)
+	{
+		/* Dummy values to fail the range checks later */
+		*pui64PA1Start = -1ULL;
+		*pui64PA1End   = -1ULL;
+	}
+	else if (bUseRemapOutput)
+	{
+		*pui64PA1Start = (IMG_UINT64)psRemapEntry1->ui32RemapAddrOut << 12;
+		*pui64PA1End   = *pui64PA1Start + ui64PageSize - 1;
+	}
+	else
+	{
+		*pui64PA1Start = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo1);
+		*pui64PA1End   = *pui64PA1Start + ui64PageSize - 1;
+	}
+}
+
+static void _CheckMipsTLBDuplicatePAs(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                      void *pvDumpDebugFile,
+                                      const RGX_MIPS_TLB_ENTRY *psTLB,
+                                      const RGX_MIPS_REMAP_ENTRY *psRemap)
+{
+	IMG_UINT64 ui64PA0StartI, ui64PA1StartI, ui64PA0StartJ, ui64PA1StartJ;
+	IMG_UINT64 ui64PA0EndI,   ui64PA1EndI,   ui64PA0EndJ,   ui64PA1EndJ;
+	IMG_UINT32 i, j;
+
+#define RANGES_OVERLAP(start0,end0,start1,end1)  ((start0) < (end1) && (start1) < (end0))
+
+	for (i = 0; i < RGXMIPSFW_NUMBER_OF_TLB_ENTRIES; i++)
+	{
+		_GetMipsTLBPARanges(&psTLB[i],
+		                    psRemap ? &psRemap[i] : NULL,
+		                    psRemap ? &psRemap[i + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+		                    &ui64PA0StartI, &ui64PA0EndI,
+		                    &ui64PA1StartI, &ui64PA1EndI);
+
+		for (j = i + 1; j < RGXMIPSFW_NUMBER_OF_TLB_ENTRIES; j++)
+		{
+			_GetMipsTLBPARanges(&psTLB[j],
+			                    psRemap ? &psRemap[j] : NULL,
+			                    psRemap ? &psRemap[j + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+			                    &ui64PA0StartJ, &ui64PA0EndJ,
+			                    &ui64PA1StartJ, &ui64PA1EndJ);
+
+			if (RANGES_OVERLAP(ui64PA0StartI, ui64PA0EndI, ui64PA0StartJ, ui64PA0EndJ) ||
+			    RANGES_OVERLAP(ui64PA0StartI, ui64PA0EndI, ui64PA1StartJ, ui64PA1EndJ) ||
+			    RANGES_OVERLAP(ui64PA1StartI, ui64PA1EndI, ui64PA0StartJ, ui64PA0EndJ) ||
+			    RANGES_OVERLAP(ui64PA1StartI, ui64PA1EndI, ui64PA1StartJ, ui64PA1EndJ)  )
+			{
+				PVR_DUMPDEBUG_LOG("Overlap between TLB entry %u and %u", i , j);
+			}
+		}
+	}
+}
+
+static inline IMG_UINT32 _GetMIPSRemapRegionSize(IMG_UINT32 ui32RegionSizeEncoding)
+{
+    return 1U << ((ui32RegionSizeEncoding + 1U) << 1U);
+}
+
+static inline void _RGXMipsDumpTLBEntry(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                        void *pvDumpDebugFile,
+                                        const RGX_MIPS_TLB_ENTRY *psTLBEntry,
+                                        const RGX_MIPS_REMAP_ENTRY *psRemapEntry0,
+                                        const RGX_MIPS_REMAP_ENTRY *psRemapEntry1,
+                                        IMG_UINT32 ui32Index)
+{
+	IMG_BOOL bDumpRemapEntries = (psRemapEntry0 != NULL && psRemapEntry1 != NULL) ? IMG_TRUE : IMG_FALSE;
+	IMG_UINT64 ui64PA0 = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo0);
+	IMG_UINT64 ui64PA1 = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo1);
+	IMG_UINT64 ui64Remap0AddrOut = 0, ui64Remap1AddrOut = 0;
+	IMG_UINT32 ui32Remap0AddrIn = 0, ui32Remap1AddrIn = 0;
+
+	if (bDumpRemapEntries)
+	{
+		/* RemapAddrIn is always 4k aligned and on 32 bit */
+		ui32Remap0AddrIn = psRemapEntry0->ui32RemapAddrIn << 12;
+		ui32Remap1AddrIn = psRemapEntry1->ui32RemapAddrIn << 12;
+
+		/* RemapAddrOut is always 4k aligned and on 32 or 36 bit */
+		ui64Remap0AddrOut = (IMG_UINT64)psRemapEntry0->ui32RemapAddrOut << 12;
+		ui64Remap1AddrOut = (IMG_UINT64)psRemapEntry1->ui32RemapAddrOut << 12;
+
+		/* If TLB and remap entries match, then merge them else, print them separately */
+		if ((IMG_UINT32)ui64PA0 == ui32Remap0AddrIn &&
+		    (IMG_UINT32)ui64PA1 == ui32Remap1AddrIn)
+		{
+			ui64PA0 = ui64Remap0AddrOut;
+			ui64PA1 = ui64Remap1AddrOut;
+			bDumpRemapEntries = IMG_FALSE;
+		}
+	}
+
+	PVR_DUMPDEBUG_LOG("%2u) VA 0x%08X (%3uk) -> PA0 0x%08" IMG_UINT64_FMTSPECx " %s%s%s, "
+	                                           "PA1 0x%08" IMG_UINT64_FMTSPECx " %s%s%s",
+	                  ui32Index,
+	                  psTLBEntry->ui32TLBHi,
+	                  RGXMIPSFW_TLB_GET_PAGE_SIZE(psTLBEntry->ui32TLBPageMask),
+	                  ui64PA0,
+	                  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(psTLBEntry->ui32TLBLo0)],
+	                  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(psTLBEntry->ui32TLBLo0)],
+	                  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(psTLBEntry->ui32TLBLo0)],
+	                  ui64PA1,
+	                  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(psTLBEntry->ui32TLBLo1)],
+	                  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(psTLBEntry->ui32TLBLo1)],
+	                  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(psTLBEntry->ui32TLBLo1)]);
+
+	if (bDumpRemapEntries)
+	{
+		PVR_DUMPDEBUG_LOG("    Remap %2u : IN 0x%08X (%3uk) => OUT 0x%08" IMG_UINT64_FMTSPECx,
+		                  ui32Index,
+		                  ui32Remap0AddrIn,
+		                  _GetMIPSRemapRegionSize(psRemapEntry0->ui32RemapRegionSize),
+		                  ui64Remap0AddrOut);
+
+		PVR_DUMPDEBUG_LOG("    Remap %2u : IN 0x%08X (%3uk) => OUT 0x%08" IMG_UINT64_FMTSPECx,
+		                  ui32Index + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES,
+		                  ui32Remap1AddrIn,
+		                  _GetMIPSRemapRegionSize(psRemapEntry1->ui32RemapRegionSize),
+		                  ui64Remap1AddrOut);
+	}
+}
+
+static RGX_MIPS_REMAP_ENTRY RGXDecodeMIPSRemap(IMG_UINT64 ui64RemapReg)
+{
+	RGX_MIPS_REMAP_ENTRY sRemapInfo;
+
+	sRemapInfo.ui32RemapAddrIn =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_SHIFT;
+
+	sRemapInfo.ui32RemapAddrOut =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_SHIFT;
+
+	sRemapInfo.ui32RemapRegionSize =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_SHIFT;
+
+	return sRemapInfo;
+}
+
+static void RGXDumpMIPSState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							 void *pvDumpDebugFile,
+							 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	RGX_MIPS_STATE sMIPSState = {0};
+	PVRSRV_ERROR eError;
+
+	eError = _RGXMipsExtraDebug(psDevInfo, &sMIPSState);
+	PVR_DUMPDEBUG_LOG("---- [ MIPS internal state ] ----");
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DUMPDEBUG_LOG("MIPS extra debug not available");
+	}
+	else
+	{
+		DDLOGVAL32("PC", sMIPSState.ui32ErrorEPC);
+		DDLOGVAL32("STATUS_REGISTER", sMIPSState.ui32StatusRegister);
+		DDLOGVAL32("CAUSE_REGISTER", sMIPSState.ui32CauseRegister);
+		_RGXMipsDumpCauseDecode(pfnDumpDebugPrintf, pvDumpDebugFile,
+								sMIPSState.ui32CauseRegister, sMIPSState.ui32ErrorState);
+		DDLOGVAL32("BAD_REGISTER", sMIPSState.ui32BadRegister);
+		DDLOGVAL32("EPC", sMIPSState.ui32EPC);
+		DDLOGVAL32("SP", sMIPSState.ui32SP);
+		DDLOGVAL32("BAD_INSTRUCTION", sMIPSState.ui32BadInstr);
+		_RGXMipsDumpDebugDecode(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+								sMIPSState.ui32Debug, sMIPSState.ui32DEPC);
+
+		{
+			IMG_UINT32 ui32Idx;
+			RGX_MIPS_REMAP_ENTRY *psMipsRemaps = NULL;
+
+			IMG_BOOL bCheckBRN63553WA =
+				RGX_IS_BRN_SUPPORTED(psDevInfo, 63553) &&
+				(OSReadHWReg32(pvRegsBaseKM, RGX_CR_MIPS_ADDR_REMAP5_CONFIG1) == (0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN));
+
+			IMG_BOOL bUseRemapRanges = RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32;
+
+			if (bUseRemapRanges)
+			{
+				psMipsRemaps = OSAllocMem(sizeof(RGX_MIPS_REMAP_ENTRY) * RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES);
+				PVR_LOG_RETURN_VOID_IF_FALSE(psMipsRemaps != NULL, "psMipsRemaps alloc failed.");
+			}
+
+			PVR_DUMPDEBUG_LOG("TLB                           :");
+
+			for (ui32Idx = 0; ui32Idx < ARRAY_SIZE(sMIPSState.asTLB); ui32Idx++)
+			{
+				if (bUseRemapRanges)
+				{
+					psMipsRemaps[ui32Idx] =
+							RGXDecodeMIPSRemap(sMIPSState.aui64Remap[ui32Idx]);
+
+					psMipsRemaps[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] =
+							RGXDecodeMIPSRemap(sMIPSState.aui64Remap[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES]);
+				}
+
+				_RGXMipsDumpTLBEntry(pfnDumpDebugPrintf,
+								     pvDumpDebugFile,
+								     &sMIPSState.asTLB[ui32Idx],
+								     (bUseRemapRanges) ? &psMipsRemaps[ui32Idx] : NULL,
+								     (bUseRemapRanges) ? &psMipsRemaps[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+								     ui32Idx);
+
+				if (bCheckBRN63553WA)
+				{
+					const RGX_MIPS_TLB_ENTRY *psTLBEntry = &sMIPSState.asTLB[ui32Idx];
+
+					#define BRN63553_TLB_IS_NUL(X)  (((X) & RGXMIPSFW_TLB_VALID) && (RGXMIPSFW_TLB_GET_PA(X) == 0x0))
+
+					if (BRN63553_TLB_IS_NUL(psTLBEntry->ui32TLBLo0) || BRN63553_TLB_IS_NUL(psTLBEntry->ui32TLBLo1))
+					{
+						PVR_DUMPDEBUG_LOG("BRN63553 WA present with a valid TLB entry mapping address 0x0.");
+					}
+				}
+			}
+
+			/* This implicitly also checks for overlaps between memory and regbank addresses */
+			_CheckMipsTLBDuplicatePAs(pfnDumpDebugPrintf,
+									  pvDumpDebugFile,
+									  sMIPSState.asTLB,
+									  bUseRemapRanges ? psMipsRemaps : NULL);
+
+			if (bUseRemapRanges)
+			{
+				/* Dump unmapped address if it was dumped in FW, otherwise it will be 0 */
+				if (sMIPSState.ui32UnmappedAddress)
+				{
+					PVR_DUMPDEBUG_LOG("Remap unmapped address => 0x%08X",
+									  sMIPSState.ui32UnmappedAddress);
+				}
+			}
+
+			if (psMipsRemaps != NULL)
+			{
+				OSFreeMem(psMipsRemaps);
+			}
+		}
+
+		/* Check FW code corruption in case of known errors */
+		if (_IsFWCodeException(RGXMIPSFW_C0_CAUSE_EXCCODE(sMIPSState.ui32CauseRegister)))
+		{
+			eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+			}
+		}
+	}
+	PVR_DUMPDEBUG_LOG("--------------------------------");
+}
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+static void _RGXDumpMetaSPExtraDebugInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+/* List of extra META Slave Port debug registers */
+#define RGX_META_SP_EXTRA_DEBUG \
+			X(RGX_CR_META_SP_MSLVCTRL0) \
+			X(RGX_CR_META_SP_MSLVCTRL1) \
+			X(RGX_CR_META_SP_MSLVDATAX) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL)
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+/* Order in these two initialisers and the one above must match */
+#define RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_EQ1_AND_MRUA_ACCESSES \
+			X(RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA)
+
+#define RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_GT1_AND_MRUA_ACCESSES \
+			X(RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA)
+#endif
+
+	IMG_UINT32 ui32Idx;
+	IMG_UINT32 ui32RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+	const IMG_UINT32* pui32DebugRegAddr;
+	const IMG_UINT32 aui32DebugRegAddr[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG
+#undef X
+		};
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	const IMG_UINT32 aui32DebugRegAddrUAHSV1[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_EQ1_AND_MRUA_ACCESSES
+#undef X
+		};
+
+	const IMG_UINT32 aui32DebugRegAddrUAHSGT1[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_GT1_AND_MRUA_ACCESSES
+#undef X
+		};
+#endif
+
+	const IMG_CHAR* apszDebugRegName[] = {
+#define X(A) #A,
+	RGX_META_SP_EXTRA_DEBUG
+#undef X
+	};
+
+	PVR_DUMPDEBUG_LOG("META Slave Port extra debug:");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* array of register offset values depends on feature. But don't augment names in apszDebugRegName */
+	PVR_ASSERT(sizeof(aui32DebugRegAddrUAHSGT1) == sizeof(aui32DebugRegAddr));
+	PVR_ASSERT(sizeof(aui32DebugRegAddrUAHSV1) == sizeof(aui32DebugRegAddr));
+	pui32DebugRegAddr = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+						((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ? (aui32DebugRegAddrUAHSGT1) : (aui32DebugRegAddrUAHSV1)) : aui32DebugRegAddr;
+#else
+	pui32DebugRegAddr = aui32DebugRegAddr;
+#endif
+
+	/* dump set of Slave Port debug registers */
+	for (ui32Idx = 0; ui32Idx < sizeof(aui32DebugRegAddr)/sizeof(IMG_UINT32); ui32Idx++)
+	{
+		const IMG_CHAR* pszRegName = apszDebugRegName[ui32Idx];
+
+		ui32RegAddr = pui32DebugRegAddr[ui32Idx];
+		ui32RegVal = OSReadUncheckedHWReg32(RGX_GET_RISCV_REGS_BASE(psDevInfo), ui32RegAddr);
+		PVR_DUMPDEBUG_LOG("  * %s: 0x%8.8X", pszRegName, ui32RegVal);
+	}
+}
+#endif
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+PVRSRV_ERROR RGXDumpRGXRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(NO_HARDWARE)
+	PVR_DUMPDEBUG_LOG("------[ RGX registers ]------");
+	PVR_DUMPDEBUG_LOG("(Not supported for NO_HARDWARE builds)");
+
+	return PVRSRV_OK;
+#else /* !defined(NO_HARDWARE) */
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32   ui32Meta = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? RGX_GET_FEATURE_VALUE(psDevInfo, META) : 0;
+#endif
+	IMG_BOOL     bFirmwarePerf;
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_DUMPDEBUG_LOG("------[ RGX registers ]------");
+	PVR_DUMPDEBUG_LOG("RGX Register Base Address (Linear):   0x%p", psDevInfo->pvRegsBaseKM);
+	PVR_DUMPDEBUG_LOG("RGX Register Base Address (Physical): 0x%08lX", (unsigned long)psDevInfo->sRegsPhysBase.uiAddr);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Linear): N/A in REE ");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Linear):   0x%p",
+							psDevInfo->pvSecureRegsBaseKM);
+#endif
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Physical): 0x%08lX",
+							(unsigned long)psDevInfo->sRegsPhysBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+#endif
+
+	/* Check if firmware perf was set at Init time */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->eFirmwarePerf,
+	                           INVALIDATE);
+	bFirmwarePerf = (psDevInfo->psRGXFWIfSysInit->eFirmwarePerf != FW_PERF_CONF_NONE);
+
+	RGXDumpCoreRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (ui32Meta)
+	{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		IMG_UINT32 ui32MSlvCtrl1Reg = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+				((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ?
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA :
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA) :
+				RGX_CR_META_SP_MSLVCTRL1;
+
+		/* Forcing bit 6 of MslvCtrl1 to 0 to avoid internal reg read going through the core */
+		OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32MSlvCtrl1Reg, 0x0);
+#else
+		/* Forcing bit 6 of MslvCtrl1 to 0 to avoid internal reg read going through the core */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL1, 0x0);
+#endif
+
+		DDLOG32(META_SP_MSLVIRQSTATUS);
+	}
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		RGXDumpMulticoreRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+	}
+
+	RGXDumpClkRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	DDLOG32(EVENT_STATUS);
+	DDLOG64(TIMER);
+
+	RGXDumpMMURegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	RGXDumpDMRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	DDLOG32(MTS_INTCTX);
+	DDLOG32(MTS_BGCTX);
+	DDLOG32(MTS_BGCTX_COUNTED_SCHEDULE);
+	DDLOG32(MTS_SCHEDULE);
+	DDLOG32(MTS_GPU_INT_STATUS);
+
+	RGXDumpSLCRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	RGXDumpMiscRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	PVR_DUMPDEBUG_LOG("FW Core Registers not available to REE");
+#else
+	if (ui32Meta)
+	{
+		IMG_BOOL bIsT0Enabled = IMG_FALSE, bIsFWFaulted = IMG_FALSE;
+		IMG_UINT32 ui32RegVal;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		IMG_UINT32 ui32MSlvIrqStatusReg = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+				((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ?
+					RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA :
+					RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA) :
+				RGX_CR_META_SP_MSLVIRQSTATUS;
+
+		PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, "META_SP_MSLVIRQSTATUS", OSReadUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32MSlvIrqStatusReg));
+#endif
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0ENABLE_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXENABLE", ui32RegVal);
+		if (ui32RegVal & META_CR_TXENABLE_ENABLE_BIT)
+		{
+			bIsT0Enabled = IMG_TRUE;
+		}
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0STATUS_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXSTATUS", ui32RegVal);
+
+		/* check for FW fault */
+		if (((ui32RegVal >> 20) & 0x3) == 0x2)
+		{
+			bIsFWFaulted = IMG_TRUE;
+		}
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0DEFR_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXDEFR", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_PC, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 PC", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_PCX, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 PCX", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_SP, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 SP", ui32RegVal);
+
+		if ((ui32Meta == MTP218) || (ui32Meta == MTP219))
+		{
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1ENABLE_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXENABLE", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1STATUS_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXSTATUS", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1DEFR_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXDEFR", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_PC, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 PC", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_PCX, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 PCX", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_SP, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 SP", ui32RegVal);
+		}
+
+		if (bFirmwarePerf)
+		{
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_PERF_COUNT0, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("META_CR_PERF_COUNT0", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_PERF_COUNT1, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("META_CR_PERF_COUNT1", ui32RegVal);
+		}
+
+		if (bIsT0Enabled & bIsFWFaulted)
+		{
+			eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+			}
+		}
+		else if (bIsFWFaulted)
+		{
+			PVR_DUMPDEBUG_LOG("Skipping FW code memory corruption checking as META is disabled");
+		}
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		DDLOG32(MIPS_ADDR_REMAP1_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP1_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP2_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP2_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP3_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP3_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP4_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP4_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP5_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP5_CONFIG2);
+		DDLOG64(MIPS_WRAPPER_CONFIG);
+		DDLOG32(MIPS_EXCEPTION_STATUS);
+
+		RGXDumpMIPSState(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+	}
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = RGXDumpRISCVState(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION) */
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		DDLOGVAL32("TFBC_VERSION", RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION));
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		DDLOGVAL32("TFBC_COMPRESSION_CONTROL", psDevInfo->psRGXFWIfSysInit->ui32TFBCCompressionControl);
+	}
+#endif
+
+	return eError;
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+_METASPError:
+	PVR_DUMPDEBUG_LOG("Dump Slave Port debug information");
+	_RGXDumpMetaSPExtraDebugInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	return eError;
+#endif
+#endif /* defined(NO_HARDWARE) */
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDebugRequestProcess
+
+ @Description
+
+ This function will print out the debug for the specified level of verbosity
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+ @Input ui32VerbLevel       - Verbosity level
+
+ @Return   void
+
+******************************************************************************/
+static
+void RGXDebugRequestProcess(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32VerbLevel)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_DEV_POWER_STATE  ePowerState;
+#if defined(SUPPORT_SOC_TIMER)
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#endif
+	IMG_BOOL                bRGXPoweredON;
+	RGXFWIF_TRACEBUF        *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	const RGXFWIF_OSDATA    *psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+	IMG_BOOL                bPwrLockAlreadyHeld;
+	IMG_BOOL                bPwrLockAcquired = IMG_TRUE;
+	IMG_UINT8               ui8FwOsCount;
+
+	bPwrLockAlreadyHeld = PVRSRVPwrLockIsLockedByMe(psDeviceNode);
+	if (!bPwrLockAlreadyHeld)
+	{
+		/* Only acquire the power-lock if not already held by the calling context */
+		eError = PVRSRVPowerTryLockWaitForTimeout(psDeviceNode);
+		if (eError == PVRSRV_ERROR_TIMEOUT)
+		{
+			bPwrLockAcquired = IMG_FALSE;
+#if defined(DEBUG)
+			PVR_DPF((PVR_DBG_WARNING, "Power lock owner: PID = %u at timestamp %" IMG_UINT64_FMTSPEC " (%s:%u)",
+		             psDeviceNode->uiPwrLockOwnerPID, psDeviceNode->sPowerLockOwner.ui64Timestamp,
+		             psDeviceNode->sPowerLockOwner.pszFile, psDeviceNode->sPowerLockOwner.ui32LineNum));
+#endif
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire lock (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+#if !defined(DEBUG_OVERRIDE_PWRLOCK)
+			/* Bail out if another thread has the power lock and may be
+			 * powering down the device. */
+			return;
+#endif
+		}
+		else if (eError != PVRSRV_OK)
+		{
+			return;
+		}
+	}
+	/* This should satisfy all accesses below */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                           INVALIDATE);
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error retrieving RGX power state. No debug info dumped.",
+				__func__));
+		goto Exit;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	{
+		PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						   1, RGX_NUM_DRIVERS_SUPPORTED);
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Device ID:%d Start ]------", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+	bRGXPoweredON = (ePowerState == PVRSRV_DEV_POWER_STATE_ON);
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Info ]------");
+	PVR_DUMPDEBUG_LOG("Device Node (Info): %p (%p)", psDevInfo->psDeviceNode, psDevInfo);
+	DevicememHistoryDumpRecordStats(psDevInfo->psDeviceNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+	PVR_DUMPDEBUG_LOG("RGX BVNC: %d.%d.%d.%d (%s)", psDevInfo->sDevFeatureCfg.ui32B,
+											   psDevInfo->sDevFeatureCfg.ui32V,
+											   psDevInfo->sDevFeatureCfg.ui32N,
+											   psDevInfo->sDevFeatureCfg.ui32C,
+											   PVR_ARCH_NAME);
+	PVR_DUMPDEBUG_LOG("RGX Device State: %s", PVRSRVGetDebugDevStateString(psDeviceNode->eDevState));
+	PVR_DUMPDEBUG_LOG("RGX Power State: %s", _RGXGetDebugDevPowerStateString(ePowerState));
+
+	PVR_DUMPDEBUG_LOG("RGX Health Status: %s, Reason: %s",PVRSRVGetDebugHealthStatusString(OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus)),
+						PVRSRVGetDebugHealthReasonString(OSAtomicRead(&psDevInfo->psDeviceNode->eHealthReason)));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		if (psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated)
+		{
+			PVR_DUMP_FIRMWARE_INFO(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks);
+			ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount);
+			}
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("FW info: UNINITIALIZED");
+		}
+	}
+	else
+	{
+		PVR_DUMP_FIRMWARE_INFO_HDR(psDevInfo->sFWInfoHeader);
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode) && BITMASK_HAS(psDevInfo->sFWInfoHeader.ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psDevInfo->sFWInfoHeader.ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount);
+			}
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TILE_REGION_PROTECTION))
+	{
+#if defined(SUPPORT_TRP)
+		PVR_DUMPDEBUG_LOG("TRP: HW support - Yes; SW enabled");
+#else
+		PVR_DUMPDEBUG_LOG("TRP: HW support - Yes; SW disabled");
+#endif
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("TRP: HW support - No");
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, WORKGROUP_PROTECTION))
+	{
+#if defined(SUPPORT_WGP)
+		PVR_DUMPDEBUG_LOG("WGP: HW support - Yes; SW enabled");
+#else
+		PVR_DUMPDEBUG_LOG("WGP: HW support - Yes; SW disabled");
+#endif
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("WGP: HW support - No");
+	}
+
+#if defined(SUPPORT_SOC_TIMER)
+	if (psDevConfig && psDevConfig->pfnSoCTimerRead)
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*)psDevConfig->hDevData;
+		RGX_TIMING_INFORMATION *psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+		IMG_UINT32 ui32Remainder;
+		IMG_UINT64 ui64CurrentSoCTime = psDevConfig->pfnSoCTimerRead(psDevConfig->hSysData);
+		IMG_UINT64 ui64CurrentSoCTimeInNS =
+			OSDivide64r64(ui64CurrentSoCTime * SECONDS_TO_MICROSECONDS, RGXFWIF_CONVERT_TO_KHZ(psRGXTimingInfo->ui32SOCClockSpeed), &ui32Remainder);
+		IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+		RGXConvertOSTimestampToSAndNS(ui64CurrentSoCTimeInNS, &ui64Seconds, &ui64Nanoseconds);
+		PVR_DUMPDEBUG_LOG("SoC timer counter: 0x%" IMG_UINT64_FMTSPECx, ui64CurrentSoCTime);
+		PVR_DUMPDEBUG_LOG("SoC timer: %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC " seconds",
+				ui64Seconds, ui64Nanoseconds);
+	}
+#endif
+
+	RGXDumpRGXDebugSummary(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, bRGXPoweredON);
+
+	/* Dump out the kernel CCB. */
+	{
+		const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		const RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+
+		if (psKCCBCtl != NULL)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Kernel CCB WO:0x%X RO:0x%X",
+							  psKCCBCtlLocal->ui32WriteOffset,
+							  psKCCBCtl->ui32ReadOffset);
+		}
+	}
+
+	/* Dump out the firmware CCB. */
+	{
+		const RGXFWIF_CCB_CTL *psFCCBCtl = psDevInfo->psFirmwareCCBCtl;
+		const RGXFWIF_CCB_CTL *psFCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psFirmwareCCBCtl, INVALIDATE);
+
+		if (psFCCBCtl != NULL)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Firmware CCB WO:0x%X RO:0x%X",
+							   psFCCBCtl->ui32WriteOffset,
+							   psFCCBCtlLocal->ui32ReadOffset);
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Dump out the Workload estimation CCB. */
+		const RGXFWIF_CCB_CTL *psWorkEstCCBCtl = psDevInfo->psWorkEstFirmwareCCBCtl;
+		const RGXFWIF_CCB_CTL *psWorkEstCCBCtlLocal = psDevInfo->psWorkEstFirmwareCCBCtlLocal;
+
+		if (psWorkEstCCBCtl != NULL)
+		{
+			RGXFwSharedMemCacheOpPtr(psWorkEstCCBCtl, INVALIDATE);
+			PVR_DUMPDEBUG_LOG("RGX WorkEst CCB WO:0x%X RO:0x%X",
+							  psWorkEstCCBCtl->ui32WriteOffset,
+							  psWorkEstCCBCtlLocal->ui32ReadOffset);
+		}
+	}
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psFwOsData,
+	                         INVALIDATE);
+
+	if (psFwOsData != NULL)
+	{
+		/* Dump the KCCB commands executed */
+		PVR_DUMPDEBUG_LOG("RGX Kernel CCB commands executed = %d",
+						  psFwOsData->ui32KCCBCmdsExecuted);
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+		/* Dump the number of times we have performed a forced UFO update,
+		 * and (if non-zero) the timestamp of the most recent occurrence/
+		 */
+		PVR_DUMPDEBUG_LOG("RGX SLR: Forced UFO updates requested = %d",
+						  psFwOsData->ui32ForcedUpdatesRequested);
+		if (psFwOsData->ui32ForcedUpdatesRequested > 0)
+		{
+			IMG_UINT8 ui8Idx;
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			if (psFwOsData->ui64LastForcedUpdateTime > 0ULL)
+			{
+				RGXConvertOSTimestampToSAndNS(psFwOsData->ui64LastForcedUpdateTime, &ui64Seconds, &ui64Nanoseconds);
+				PVR_DUMPDEBUG_LOG("RGX SLR: (most recent forced update was around %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ")",
+								  ui64Seconds, ui64Nanoseconds);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("RGX SLR: (unable to force update as fence contained no sync checkpoints)");
+			}
+			/* Dump SLR log */
+			if (psFwOsData->sSLRLogFirst.aszCCBName[0])
+			{
+				RGXConvertOSTimestampToSAndNS(psFwOsData->sSLRLogFirst.ui64Timestamp, &ui64Seconds, &ui64Nanoseconds);
+				PVR_DUMPDEBUG_LOG("RGX SLR:{%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC
+								  "} Fence found on context 0x%x [%u/%d] '%.*s' has %d UFOs",
+								  ui64Seconds, ui64Nanoseconds,
+								  psFwOsData->sSLRLogFirst.ui32FWCtxAddr,
+								  psDeviceNode->sDevId.ui32InternalID,
+								  psDeviceNode->sDevId.i32KernelDeviceID,
+								  PVR_SLR_LOG_STRLEN, psFwOsData->sSLRLogFirst.aszCCBName,
+								  psFwOsData->sSLRLogFirst.ui32NumUFOs);
+			}
+			for (ui8Idx=0; ui8Idx<PVR_SLR_LOG_ENTRIES;ui8Idx++)
+			{
+				if (psFwOsData->sSLRLog[ui8Idx].aszCCBName[0])
+				{
+					RGXConvertOSTimestampToSAndNS(psFwOsData->sSLRLog[ui8Idx].ui64Timestamp, &ui64Seconds, &ui64Nanoseconds);
+					PVR_DUMPDEBUG_LOG("RGX SLR:[%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC
+									  "] Fence found on context 0x%x '%.*s' has %d UFOs",
+									  ui64Seconds, ui64Nanoseconds,
+									  psFwOsData->sSLRLog[ui8Idx].ui32FWCtxAddr,
+									  PVR_SLR_LOG_STRLEN, psFwOsData->sSLRLog[ui8Idx].aszCCBName,
+									  psFwOsData->sSLRLog[ui8Idx].ui32NumUFOs);
+				}
+			}
+		}
+#else
+		PVR_DUMPDEBUG_LOG("RGX SLR: Disabled");
+#endif
+
+		/* Dump the error counts */
+		PVR_DUMPDEBUG_LOG("RGX Errors: WGP:%d, TRP:%d",
+						  psDevInfo->sErrorCounts.ui32WGPErrorCount,
+						  psDevInfo->sErrorCounts.ui32TRPErrorCount);
+
+		/* Dump the IRQ info for threads or OS IDs */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* only Host has access to registers containing IRQ counters */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+#endif
+		{
+			IMG_UINT32 ui32idx;
+
+			for_each_irq_cnt(ui32idx)
+			{
+				IMG_UINT32 ui32IrqCnt;
+
+				get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+				if (ui32IrqCnt)
+				{
+					PVR_DUMPDEBUG_LOG(MSG_IRQ_CNT_TYPE "%u: FW IRQ count = %u", ui32idx, ui32IrqCnt);
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+					if (ui32idx == RGXFW_HOST_DRIVER_ID)
+#endif
+					{
+						PVR_DUMPDEBUG_LOG("Last sampled IRQ count in LISR = %u", psDevInfo->aui32SampleIRQCount[ui32idx]);
+					}
+				}
+			}
+		}
+	}
+
+	/* Dump the FW Sys config flags on the Host */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+		IMG_CHAR sFwSysFlagsDescription[MAX_FW_DESCRIPTION_LENGTH];
+
+		if (!psFwSysData)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Fw Sys Data is not mapped into CPU space", __func__));
+			goto Exit;
+		}
+
+		RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags,
+		                           INVALIDATE);
+
+		_GetFwSysFlagsDescription(sFwSysFlagsDescription, MAX_FW_DESCRIPTION_LENGTH, psFwSysData->ui32ConfigFlags);
+		PVR_DUMPDEBUG_LOG("FW System config flags = 0x%08X (%s)", psFwSysData->ui32ConfigFlags, sFwSysFlagsDescription);
+	}
+
+	/* Dump the FW OS config flags */
+	{
+		IMG_CHAR sFwOsFlagsDescription[MAX_FW_DESCRIPTION_LENGTH];
+
+		if (!psFwOsData)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Fw Os Data is not mapped into CPU space", __func__));
+			goto Exit;
+		}
+
+		_GetFwOsFlagsDescription(sFwOsFlagsDescription, MAX_FW_DESCRIPTION_LENGTH, psFwOsData->ui32FwOsConfigFlags);
+		PVR_DUMPDEBUG_LOG("FW OS config flags = 0x%08X (%s)", psFwOsData->ui32FwOsConfigFlags, sFwOsFlagsDescription);
+	}
+
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDumpRGXRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: RGXDumpRGXRegisters failed (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG(" (!) %s. No registers dumped", PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? "Guest Mode of operation" : "RGX power is down");
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX FW Trace Info ]------");
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_INT tid;
+		/* Dump FW trace information */
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			RGX_FWT_LOGTYPE eFWTLogType = psDevInfo->eDebugDumpFWTLogType;
+
+			if (eFWTLogType == RGX_FWT_LOGTYPE_NONE)
+			{
+				PVR_DUMPDEBUG_LOG("Firmware trace printing disabled.");
+			}
+			else
+			{
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+
+				for (tid = 0; tid < RGXFW_THREAD_NUM; tid++)
+				{
+					IMG_UINT32 *pui32TraceBuffer;
+
+					if (psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+					{
+						PVR_DUMPDEBUG_LOG("Debug log type: %s ( " RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC ")",
+										  ((psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)?("trace"):("tbi")),
+										  RGXFWIF_LOG_ENABLED_GROUPS_LIST(psRGXFWIfTraceBufCtl->ui32LogType)
+										  );
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("Debug log type: none");
+					}
+
+					pui32TraceBuffer = psDevInfo->apui32TraceBuffer[tid];
+
+					/* Skip if trace buffer is not allocated */
+					if (pui32TraceBuffer == NULL)
+					{
+						PVR_DUMPDEBUG_LOG("RGX FW thread %d: Trace buffer not yet allocated",tid);
+						continue;
+					}
+
+					RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[tid].ui32TracePointer, INVALIDATE);
+					PVR_DUMPDEBUG_LOG("------[ RGX FW thread %d trace START ]------", tid);
+					PVR_DUMPDEBUG_LOG("FWT[traceptr]: %X", psRGXFWIfTraceBufCtl->sTraceBuf[tid].ui32TracePointer);
+					PVR_DUMPDEBUG_LOG("FWT[tracebufsize]: %X", psDevInfo->ui32TraceBufSizeInDWords);
+
+					if (eFWTLogType == RGX_FWT_LOGTYPE_BINARY)
+					{
+						RGXDumpFirmwareTraceBinary(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+					else if (eFWTLogType == RGX_FWT_LOGTYPE_DECODED)
+					{
+						RGXDumpFirmwareTraceDecoded(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+					else if (eFWTLogType == RGX_FWT_LOGTYPE_PARTIAL)
+					{
+						RGXDumpFirmwareTracePartial(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+
+					PVR_DUMPDEBUG_LOG("------[ RGX FW thread %d trace END ]------", tid);
+				}
+			}
+		}
+
+		{
+			if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+			{
+				PVR_DUMPDEBUG_LOG("------[ Full CCB Status ]------");
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("------[ FWCtxs Next CMD ]------");
+			}
+
+			RGXDumpAllContextInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Device ID:%d End ]------", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+Exit:
+	if (!bPwrLockAlreadyHeld && bPwrLockAcquired)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+}
+
+/*!
+ ******************************************************************************
+
+ @Function	RGXDebugRequestNotify
+
+ @Description Dump the debug data for RGX
+
+ ******************************************************************************/
+static void RGXDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDbgRequestHandle,
+		IMG_UINT32 ui32VerbLevel,
+		DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+		void *pvDumpDebugFile)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = hDbgRequestHandle;
+
+	/* Only action the request if we've fully init'ed */
+	if (psDevInfo->bDevInit2Done)
+	{
+		RGXDebugRequestProcess(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, ui32VerbLevel);
+	}
+}
+
+PVRSRV_ERROR RGXDebugInit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return PVRSRVRegisterDeviceDbgRequestNotify(&psDevInfo->hDbgReqNotify,
+		                                        psDevInfo->psDeviceNode,
+		                                        RGXDebugRequestNotify,
+		                                        DEBUG_REQUEST_RGX,
+		                                        psDevInfo);
+}
+
+PVRSRV_ERROR RGXDebugDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (psDevInfo->hDbgReqNotify)
+	{
+		return PVRSRVUnregisterDeviceDbgRequestNotify(psDevInfo->hDbgReqNotify);
+	}
+
+	/* No notifier registered */
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxdebug_common.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.h
new file mode 100644
index 000000000000..f7707b87fa9f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxdebug_common.h
@@ -0,0 +1,433 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX debug header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEBUG_COMMON_H)
+#define RGXDEBUG_COMMON_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "rgxdevice.h"
+#include "rgxfwmemctx.h"
+
+#define DD_NORMAL_INDENT   "    "
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+extern const IMG_CHAR * const gapszMipsPermissionPTFlags[4];
+extern const IMG_CHAR * const gapszMipsCoherencyPTFlags[8];
+extern const IMG_CHAR * const gapszMipsDirtyGlobalValidPTFlags[8];
+#endif
+
+/**
+ * Debug utility macro for printing FW IRQ count and Last sampled IRQ count in
+ * LISR for each RGX FW thread.
+ * Macro takes pointer to PVRSRV_RGXDEV_INFO as input.
+ */
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+#define for_each_irq_cnt(ui32idx)   FOREACH_SUPPORTED_DRIVER(ui32idx)
+
+#define get_irq_cnt_val(ui32Dest, ui32idx, psRgxDevInfo) \
+	do { \
+		extern const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS]; \
+		ui32Dest = PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo) ? 0 : OSReadHWReg32((psRgxDevInfo)->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[ui32idx]); \
+	} while (false)
+
+#define MSG_IRQ_CNT_TYPE "OS"
+
+#else
+
+#define for_each_irq_cnt(ui32idx) \
+	for (ui32idx = 0; ui32idx < RGXFW_THREAD_NUM; ui32idx++)
+
+#define get_irq_cnt_val(ui32Dest, ui32idx, psRgxDevInfo) \
+	do { \
+		RGXFwSharedMemCacheOpValue(psRgxDevInfo->psRGXFWIfFwOsData->aui32InterruptCount[ui32idx], \
+		                           INVALIDATE); \
+		ui32Dest = (psRgxDevInfo)->psRGXFWIfFwOsData->aui32InterruptCount[ui32idx]; \
+	} while (false)
+#define MSG_IRQ_CNT_TYPE "Thread"
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+static inline void RGXDEBUG_PRINT_IRQ_COUNT(PVRSRV_RGXDEV_INFO* psRgxDevInfo)
+{
+#if defined(PVRSRV_NEED_PVR_DPF) && defined(DEBUG)
+	IMG_UINT32 ui32idx;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		IMG_UINT32 ui32IrqCnt;
+
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psRgxDevInfo);
+
+		PVR_DPF((DBGPRIV_VERBOSE, MSG_IRQ_CNT_TYPE
+		         " %u FW IRQ count = %u", ui32idx, ui32IrqCnt));
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		if (ui32idx == RGXFW_HOST_DRIVER_ID)
+#endif
+		{
+			PVR_DPF((DBGPRIV_VERBOSE, "Last sampled IRQ count in LISR = %u",
+			        (psRgxDevInfo)->aui32SampleIRQCount[ui32idx]));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psRgxDevInfo);
+#endif /* PVRSRV_NEED_PVR_DPF */
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpFirmwareTrace
+
+ @Description Dumps the decoded version of the firmware trace buffer.
+
+ Dump useful debugging info
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+
+ @Return   void
+
+******************************************************************************/
+void RGXDumpFirmwareTrace(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO  *psDevInfo);
+
+void RGXDumpFirmwareTraceBinary(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+void RGXDumpFirmwareTracePartial(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+void RGXDumpFirmwareTraceDecoded(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+
+/* Helper macros to emit data */
+#define REG32_FMTSPEC   "%-30s: 0x%08X"
+#define REG64_FMTSPEC   "%-30s: 0x%016" IMG_UINT64_FMTSPECX
+#define DDLOG32(R)      PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, #R, OSReadHWReg32(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOG64(R)      PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadHWReg64(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOGUNCHECKED64(R)      PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadUncheckedHWReg64(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOG32_DPX(R)  PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, #R, OSReadHWReg32(pvRegsBaseKM, DPX_CR_##R));
+#define DDLOG64_DPX(R)  PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadHWReg64(pvRegsBaseKM, DPX_CR_##R));
+#define DDLOGVAL32(S,V) PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, S, V);
+
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpRGXRegisters
+
+ @Description
+
+ Dumps an extensive list of RGX registers required for debugging
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+
+ @Return PVRSRV_ERROR         PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXDumpRGXRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#if !defined(NO_HARDWARE)
+/*!
+*******************************************************************************
+
+ @Function	RGXReadMetaCoreReg
+
+ @Description  Read a META core register's value
+
+ @Input psDevInfo      RGX device info
+ @Input ui32RegAddr    Register address to read from
+ @Output pui32RegVal   Pointer to the resulting register value
+
+ @Return PVRSRV_ERROR  PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXReadMetaCoreReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT32 ui32RegAddr,
+                                IMG_UINT32 *pui32RegVal);
+
+/*!
+*******************************************************************************
+
+ @Function  RGXValidateFWImage
+
+ @Description  Validate the currently running firmware
+               against the firmware image
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo	         - RGX device info
+
+ @Return PVRSRV_ERROR  PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXValidateFWImage(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+/*!
+*******************************************************************************
+
+ @Function     ValidateFWOnLoad
+
+ @Description  Compare the Firmware image as seen from the CPU point of view
+               against the same memory area as seen from the firmware point
+               of view after first power up.
+
+ @Input        psDevInfo - Device Info
+
+ @Return       PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR ValidateFWOnLoad(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpRGXDebugSummary
+
+ @Description
+
+ Dump a summary in human readable form with the RGX state
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo	         - RGX device info
+ @Input bRGXPoweredON        - IMG_TRUE if RGX device is on
+
+ @Return   void
+
+******************************************************************************/
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON);
+
+/*!
+*******************************************************************************
+
+ @Function RGXDebugInit
+
+ @Description
+
+ Setup debug requests, calls into PVRSRVRegisterDeviceDbgRequestNotify
+
+ @Input          psDevInfo            RGX device info
+ @Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+
+******************************************************************************/
+PVRSRV_ERROR RGXDebugInit(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function RGXDebugDeinit
+
+ @Description
+
+ Remove debug requests, calls into PVRSRVUnregisterDeviceDbgRequestNotify
+
+ @Output         phNotify             Points to debug notifier handle
+ @Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+
+******************************************************************************/
+PVRSRV_ERROR RGXDebugDeinit(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetFwMapping
+
+ @Description    Retrieve any of the CPU Physical Address, Device Physical
+                Address or the raw value of the page table entry associated
+                with the firmware virtual address given.
+
+ @Input          psDevInfo           Pointer to device info
+ @Input          pfnDumpDebugPrintf  The debug printf function
+ @Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+ @Input          ui32FwVA            The Fw VA that needs decoding
+ @Output         psCpuPA             Pointer to the resulting CPU PA
+ @Output         psDevPA             Pointer to the resulting Dev PA
+ @Output         pui64PTE            Pointer to the raw Page Table Entry value
+
+ @Return         void
+
+******************************************************************************/
+void RGXDocumentFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          const IMG_UINT32 ui32FwVA,
+                          const IMG_CPU_PHYADDR sCpuPA,
+                          const IMG_DEV_PHYADDR sDevPA,
+                          const IMG_UINT64 ui64PTE);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXConvertOSTimestampToSAndNS
+
+ @Description    Convert the OS time to seconds and nanoseconds
+
+ @Input          ui64OSTimer       OS time to convert
+ @Output         pui64Seconds      Pointer to the resulting seconds
+ @Output         pui64Nanoseconds  Pointer to the resulting nanoseconds
+
+ @Return         void
+
+******************************************************************************/
+void RGXConvertOSTimestampToSAndNS(IMG_UINT64 ui64OSTimer,
+							IMG_UINT64 *pui64Seconds,
+							IMG_UINT64 *pui64Nanoseconds);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDumpAllContextInfo
+
+ @Description    Dump debug info of all contexts on a device
+
+ @Input          psDevInfo           Pointer to device info
+ @Input          pfnDumpDebugPrintf  The debug printf function
+ @Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+ @Input          ui32VerbLevel       Verbosity level
+
+ @Return         void
+
+******************************************************************************/
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpFaultAddressHostView
+
+ @Description
+
+ Dump FW HWR fault status in human readable form.
+
+ @Input ui32Index            - Index of global Fault info
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Return   void
+
+******************************************************************************/
+void RGXDumpFaultAddressHostView(MMU_FAULT_DATA *psFaultData,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const IMG_CHAR* pszIndent);
+
+void RGXDumpFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGX_HWRINFO *psHWRInfo,
+					IMG_UINT32 ui32ReadIndex,
+					IMG_DEV_VIRTADDR *psFaultDevVAddr,
+					IMG_DEV_PHYADDR *psPCDevPAddr,
+					bool bPMFault,
+					IMG_UINT32 ui32PageSize);
+
+#endif /* RGXDEBUG_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.c
new file mode 100644
index 000000000000..62a33fb9995d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.c
@@ -0,0 +1,810 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware common context utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware common context utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxfwcmnctx.h"
+#include "rgxfwutils.h"
+#include "devicemem_pdump.h"
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#include "rogue_trace_events.h"
+#endif
+
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#include "linux/cred.h"
+#include "linux/pid.h"
+#endif
+/*
+ * Maximum length of time a DM can run for before the DM will be marked
+ * as out-of-time. CDM has an increased value due to longer running kernels.
+ *
+ * These deadlines are increased on FPGA, EMU and VP due to the slower
+ * execution time of these platforms. PDUMPS are also included since they
+ * are often run on EMU, FPGA or in CSim.
+ */
+#if defined(FPGA) || defined(EMULATOR) || defined(VIRTUAL_PLATFORM) || defined(PDUMP)
+#define RGXFWIF_MAX_WORKLOAD_DEADLINE_MS     (480000)
+#define RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS (10800000)
+#else
+#define RGXFWIF_MAX_WORKLOAD_DEADLINE_MS     (40000)
+#define RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS (600000)
+#endif
+#define RGXFWIF_MAX_RDM_WORKLOAD_DEADLINE_MS (36000000)
+
+struct _RGX_SERVER_COMMON_CONTEXT_ {
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	DEVMEM_MEMDESC *psFWCommonContextMemDesc;
+	PRGXFWIF_FWCOMMONCONTEXT sFWCommonContextFWAddr;
+	SERVER_MMU_CONTEXT *psServerMMUContext;
+	DEVMEM_MEMDESC *psFWMemContextMemDesc;
+	DEVMEM_MEMDESC *psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC *psContextStateMemDesc;
+	RGX_CLIENT_CCB *psClientCCB;
+	DEVMEM_MEMDESC *psClientCCBMemDesc;
+	DEVMEM_MEMDESC *psClientCCBCtrlMemDesc;
+	IMG_BOOL bCommonContextMemProvided;
+	IMG_UINT32 ui32ContextID;
+	DLLIST_NODE sListNode;
+	RGX_CONTEXT_RESET_REASON eLastResetReason;
+	IMG_UINT32 ui32LastResetJobRef;
+	IMG_INT32 i32Priority;
+	RGX_CCB_REQUESTOR_TYPE eRequestor;
+};
+
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+static IMG_UINT32 _GetUID(IMG_PID pid)
+{
+	struct task_struct *psTask;
+	struct pid *psPid;
+
+	psPid = find_get_pid((pid_t)pid);
+	if (!psPid)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to lookup PID %u.",
+		                        __func__, pid));
+		return 0;
+	}
+
+	psTask = get_pid_task(psPid, PIDTYPE_PID);
+	if (!psTask)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get pid task for PID %u.",
+		                        __func__, pid));
+	}
+	put_pid(psPid);
+
+	return psTask ? from_kuid(&init_user_ns, psTask->cred->uid) : 0;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _CheckPriority
+@Description    Check if priority is allowed for requestor type
+@Input          psDevInfo    pointer to DevInfo struct
+@Input          i32Priority Requested priority
+@Input          eRequestor   Requestor type specifying data master
+@Return         PVRSRV_ERROR PVRSRV_OK on success
+*/ /**************************************************************************/
+static PVRSRV_ERROR _CheckPriority(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_INT32 i32Priority,
+								   RGX_CCB_REQUESTOR_TYPE eRequestor)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Only contexts from a single PID allowed with real time priority (highest priority) */
+	if (i32Priority == RGX_CTX_PRIORITY_REALTIME)
+	{
+		DLLIST_NODE *psNode, *psNext;
+
+		OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+
+		dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+		{
+			RGX_SERVER_COMMON_CONTEXT *psThisContext =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+			if (psThisContext->i32Priority == RGX_CTX_PRIORITY_REALTIME &&
+				psThisContext->eRequestor == eRequestor &&
+				RGXGetPIDFromServerMMUContext(psThisContext->psServerMMUContext) != OSGetCurrentClientProcessIDKM())
+			{
+				PVR_LOG(("Only one process can have contexts with real time priority"));
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				break;
+			}
+		}
+
+		OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
+									 PVRSRV_DEVICE_NODE *psDeviceNode,
+									 RGX_CCB_REQUESTOR_TYPE eRGXCCBRequestor,
+									 RGXFWIF_DM eDM,
+									 SERVER_MMU_CONTEXT *psServerMMUContext,
+									 DEVMEM_MEMDESC *psAllocatedMemDesc,
+									 IMG_UINT32 ui32AllocatedOffset,
+									 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+									 DEVMEM_MEMDESC *psContextStateMemDesc,
+									 IMG_UINT32 ui32CCBAllocSizeLog2,
+									 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+									 IMG_UINT32 ui32ContextFlags,
+									 IMG_INT32 i32Priority,
+									 IMG_UINT32 ui32MaxDeadlineMS,
+									 IMG_UINT64 ui64RobustnessAddress,
+									 RGX_COMMON_CONTEXT_INFO *psInfo,
+									 RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_COMMON_CONTEXT *psServerCommonContext;
+	RGXFWIF_FWCOMMONCONTEXT sFWCommonContext = {{0}};
+	IMG_UINT32 ui32FWCommonContextOffset;
+	IMG_UINT8 *pui8Ptr;
+	PVRSRV_ERROR eError;
+
+	/*
+	 * Allocate all the resources that are required
+	 */
+	psServerCommonContext = OSAllocMem(sizeof(*psServerCommonContext));
+	if (psServerCommonContext == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+
+	psServerCommonContext->psDevInfo = psDevInfo;
+	psServerCommonContext->psServerMMUContext = psServerMMUContext;
+
+	if (psAllocatedMemDesc)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+					 "Using existing MemDesc for Rogue firmware %s context (offset = %d)",
+					 aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+					 ui32AllocatedOffset);
+		ui32FWCommonContextOffset = ui32AllocatedOffset;
+		psServerCommonContext->psFWCommonContextMemDesc = psAllocatedMemDesc;
+		psServerCommonContext->bCommonContextMemProvided = IMG_TRUE;
+	}
+	else
+	{
+		/* Allocate device memory for the firmware context */
+		PDUMPCOMMENT(psDeviceNode,
+					 "Allocate Rogue firmware %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
+		eError = DevmemFwAllocate(psDevInfo,
+								sizeof(sFWCommonContext),
+								RGX_FWCOMCTX_ALLOCFLAGS,
+								"FwContext",
+								&psServerCommonContext->psFWCommonContextMemDesc);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to allocate firmware %s context (%s)",
+			         __func__,
+			         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+			         PVRSRVGetErrorString(eError)));
+			goto fail_contextalloc;
+		}
+		ui32FWCommonContextOffset = 0;
+		psServerCommonContext->bCommonContextMemProvided = IMG_FALSE;
+	}
+
+	/* Record this context so we can refer to it if the FW needs to tell us it was reset. */
+	psServerCommonContext->eLastResetReason    = RGX_CONTEXT_RESET_REASON_NONE;
+	psServerCommonContext->ui32LastResetJobRef = 0;
+	psServerCommonContext->ui32ContextID       = psDevInfo->ui32CommonCtxtCurrentID++;
+
+	/*
+	 * Temporarily map the firmware context to the kernel and initialise it
+	 */
+	eError = DevmemAcquireCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc,
+	                                  (void **)&pui8Ptr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map firmware %s context to CPU (%s)",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         PVRSRVGetErrorString(eError)));
+		goto fail_cpuvirtacquire;
+	}
+
+	/* Allocate the client CCB */
+	eError = RGXCreateCCB(psDevInfo,
+						  ui32CCBAllocSizeLog2,
+						  ui32CCBMaxAllocSizeLog2,
+						  ui32ContextFlags,
+						  psConnection,
+						  eRGXCCBRequestor,
+						  psServerCommonContext,
+						  &psServerCommonContext->psClientCCB,
+						  &psServerCommonContext->psClientCCBMemDesc,
+						  &psServerCommonContext->psClientCCBCtrlMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: failed to create CCB for %s context (%s)",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         PVRSRVGetErrorString(eError)));
+		goto fail_allocateccb;
+	}
+
+	sFWCommonContext.eDM = eDM;
+	BITMASK_SET(sFWCommonContext.ui32CompatFlags, RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT);
+
+	/* Set the firmware CCB device addresses in the firmware common context */
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psCCB,
+						  psServerCommonContext->psClientCCBMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail_cccbfwaddr);
+
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psCCBCtl,
+						  psServerCommonContext->psClientCCBCtrlMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail_cccbctrlfwaddr);
+
+#if defined(RGX_FEATURE_META_DMA_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		RGXSetMetaDMAAddress(&sFWCommonContext.sCCBMetaDMAAddr,
+							 psServerCommonContext->psClientCCBMemDesc,
+							 &sFWCommonContext.psCCB,
+							 0);
+	}
+#endif
+
+	/* Set the memory context device address */
+	psServerCommonContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psFWMemContext,
+						  psFWMemContextMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", fail_fwmemctxfwaddr);
+
+	/* Set the framework register updates address */
+	psServerCommonContext->psFWFrameworkMemDesc = psInfo->psFWFrameworkMemDesc;
+	if (psInfo->psFWFrameworkMemDesc != NULL)
+	{
+		eError = RGXSetFirmwareAddress(&sFWCommonContext.psRFCmd,
+				psInfo->psFWFrameworkMemDesc,
+				0, RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:4", fail_fwframeworkfwaddr);
+	}
+	else
+	{
+		/* This should never be touched in this contexts without a framework
+		 * memdesc, but ensure it is zero so we see crashes if it is.
+		 */
+		sFWCommonContext.psRFCmd.ui32Addr = 0;
+	}
+
+	eError = _CheckPriority(psDevInfo, i32Priority, eRGXCCBRequestor);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CheckPriority", fail_checkpriority);
+
+	psServerCommonContext->i32Priority = i32Priority;
+	psServerCommonContext->eRequestor = eRGXCCBRequestor;
+
+	sFWCommonContext.i32Priority = i32Priority;
+	sFWCommonContext.ui32PrioritySeqNum = 0;
+
+	if (eDM == RGXFWIF_DM_CDM)
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS);
+	}
+	else if (eDM == RGXFWIF_DM_RAY)
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_RDM_WORKLOAD_DEADLINE_MS);
+	}
+	else
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_WORKLOAD_DEADLINE_MS);
+	}
+
+	sFWCommonContext.ui64RobustnessAddress = ui64RobustnessAddress;
+
+	/* Store a references to Server Common Context and PID for notifications back from the FW. */
+	sFWCommonContext.ui32ServerCommonContextID = psServerCommonContext->ui32ContextID;
+	sFWCommonContext.ui32PID                   = OSGetCurrentClientProcessIDKM();
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	sFWCommonContext.ui32UID                   = _GetUID(sFWCommonContext.ui32PID);
+#endif
+	OSStringSafeCopy(sFWCommonContext.szProcName, psConnection->pszProcName, RGXFW_PROCESS_NAME_LEN);
+
+	/* Set the firmware GPU context state buffer */
+	psServerCommonContext->psContextStateMemDesc = psContextStateMemDesc;
+	if (psContextStateMemDesc)
+	{
+		eError = RGXSetFirmwareAddress(&sFWCommonContext.psContextState,
+							  psContextStateMemDesc,
+							  0,
+							  RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:5", fail_ctxstatefwaddr);
+	}
+
+	OSCachedMemCopy(IMG_OFFSET_ADDR(pui8Ptr, ui32FWCommonContextOffset), &sFWCommonContext, sizeof(sFWCommonContext));
+	RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pui8Ptr, ui32FWCommonContextOffset),
+	                          sizeof(sFWCommonContext),
+	                          PVRSRV_CACHE_OP_FLUSH);
+
+	/*
+	 * Dump the created context
+	 */
+	PDUMPCOMMENT(psDeviceNode,
+				 "Dump %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
+	DevmemPDumpLoadMem(psServerCommonContext->psFWCommonContextMemDesc,
+					   ui32FWCommonContextOffset,
+					   sizeof(sFWCommonContext),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* We've finished the setup so release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
+
+	/* Map this allocation into the FW */
+	eError = RGXSetFirmwareAddress(&psServerCommonContext->sFWCommonContextFWAddr,
+						  psServerCommonContext->psFWCommonContextMemDesc,
+						  ui32FWCommonContextOffset,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:6", fail_fwcommonctxfwaddr);
+
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	{
+		IMG_UINT32 ui32FWAddr;
+		switch (eDM) {
+			case RGXFWIF_DM_GEOM:
+				ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
+						psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, sTAContext));
+				break;
+			case RGXFWIF_DM_3D:
+				ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
+						psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, s3DContext));
+				break;
+			default:
+				ui32FWAddr = psServerCommonContext->sFWCommonContextFWAddr.ui32Addr;
+				break;
+		}
+
+		trace_rogue_create_fw_context(OSGetCurrentClientProcessNameKM(),
+									  aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+									  psDeviceNode->sDevId.ui32InternalID,
+									  ui32FWAddr);
+	}
+#endif
+
+	/*Add the node to the list when finalised */
+	OSWRLockAcquireWrite(psDevInfo->hCommonCtxtListLock);
+	dllist_add_to_tail(&(psDevInfo->sCommonCtxtListHead), &(psServerCommonContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hCommonCtxtListLock);
+
+	*ppsServerCommonContext = psServerCommonContext;
+	return PVRSRV_OK;
+
+fail_fwcommonctxfwaddr:
+	if (psContextStateMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psContextStateMemDesc);
+	}
+fail_ctxstatefwaddr:
+fail_checkpriority:
+	if (psInfo->psFWFrameworkMemDesc != NULL)
+	{
+		RGXUnsetFirmwareAddress(psInfo->psFWFrameworkMemDesc);
+	}
+fail_fwframeworkfwaddr:
+	RGXUnsetFirmwareAddress(psFWMemContextMemDesc);
+fail_fwmemctxfwaddr:
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
+fail_cccbctrlfwaddr:
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
+fail_cccbfwaddr:
+	RGXDestroyCCB(psDevInfo, psServerCommonContext->psClientCCB);
+fail_allocateccb:
+	DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
+fail_cpuvirtacquire:
+	if (!psServerCommonContext->bCommonContextMemProvided)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psServerCommonContext->psFWCommonContextMemDesc);
+		psServerCommonContext->psFWCommonContextMemDesc = NULL;
+	}
+fail_contextalloc:
+	OSFreeMem(psServerCommonContext);
+fail_alloc:
+	return eError;
+}
+
+void FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+
+	OSWRLockAcquireWrite(psServerCommonContext->psDevInfo->hCommonCtxtListLock);
+	/* Remove the context from the list of all contexts. */
+	dllist_remove_node(&psServerCommonContext->sListNode);
+	OSWRLockReleaseWrite(psServerCommonContext->psDevInfo->hCommonCtxtListLock);
+
+	/*
+		Unmap the context itself and then all its resources
+	*/
+
+	/* Unmap the FW common context */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psFWCommonContextMemDesc);
+	/* Umap context state buffer (if there was one) */
+	if (psServerCommonContext->psContextStateMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psServerCommonContext->psContextStateMemDesc);
+	}
+	/* Unmap the framework buffer */
+	if (psServerCommonContext->psFWFrameworkMemDesc != NULL)
+	{
+		RGXUnsetFirmwareAddress(psServerCommonContext->psFWFrameworkMemDesc);
+	}
+	/* Unmap client CCB and CCB control */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
+	/* Unmap the memory context */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psFWMemContextMemDesc);
+
+	/* Destroy the client CCB */
+	RGXDestroyCCB(psServerCommonContext->psDevInfo, psServerCommonContext->psClientCCB);
+
+
+	/* Free the FW common context (if there was one) */
+	if (!psServerCommonContext->bCommonContextMemProvided)
+	{
+		DevmemFwUnmapAndFree(psServerCommonContext->psDevInfo,
+						psServerCommonContext->psFWCommonContextMemDesc);
+		psServerCommonContext->psFWCommonContextMemDesc = NULL;
+	}
+	/* Free the hosts representation of the common context */
+	OSFreeMem(psServerCommonContext);
+}
+
+PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->sFWCommonContextFWAddr;
+}
+
+RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psClientCCB;
+}
+
+SERVER_MMU_CONTEXT *FWCommonContextGetServerMMUCtx(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psServerMMUContext;
+}
+
+RGX_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                                               IMG_UINT32 *pui32LastResetJobRef)
+{
+	RGX_CONTEXT_RESET_REASON eLastResetReason;
+
+	PVR_ASSERT(psServerCommonContext != NULL);
+	PVR_ASSERT(pui32LastResetJobRef != NULL);
+
+	/* Take the most recent reason & job ref and reset for next time... */
+	eLastResetReason      = psServerCommonContext->eLastResetReason;
+	*pui32LastResetJobRef = psServerCommonContext->ui32LastResetJobRef;
+	psServerCommonContext->eLastResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+	psServerCommonContext->ui32LastResetJobRef = 0;
+
+	if (eLastResetReason == RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "A Hard Context Switch was triggered on the GPU to ensure Quality of Service."));
+	}
+
+	return eLastResetReason;
+}
+
+PVRSRV_RGXDEV_INFO* FWCommonContextGetRGXDevInfo(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psDevInfo;
+}
+
+PVRSRV_ERROR RGXGetFWCommonContextAddrFromServerMMUCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													   SERVER_MMU_CONTEXT *psServerMMUContext,
+													   PRGXFWIF_FWCOMMONCONTEXT *psFWCommonContextFWAddr)
+{
+	DLLIST_NODE *psNode, *psNext;
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMMON_CONTEXT *psThisContext =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+		if (psThisContext->psServerMMUContext == psServerMMUContext)
+		{
+			psFWCommonContextFWAddr->ui32Addr = psThisContext->sFWCommonContextFWAddr.ui32Addr;
+			return PVRSRV_OK;
+		}
+	}
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+PRGXFWIF_FWCOMMONCONTEXT RGXGetFWCommonContextAddrFromServerCommonCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													                  DLLIST_NODE *psNode)
+{
+	RGX_SERVER_COMMON_CONTEXT *psThisContext =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+	return FWCommonContextGetFWAddress(psThisContext);
+}
+
+PVRSRV_ERROR FWCommonContextSetFlags(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                     IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (BITMASK_ANY(ui32ContextFlags, ~RGX_CONTEXT_FLAGS_WRITEABLE_MASK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Context flag(s) invalid or not writeable (%d)",
+		         __func__, ui32ContextFlags));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else
+	{
+		RGXSetCCBFlags(psServerCommonContext->psClientCCB,
+		               ui32ContextFlags);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
+								CONNECTION_DATA *psConnection,
+								PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_INT32 i32Priority,
+								RGXFWIF_DM eDM)
+{
+	IMG_UINT32				ui32CmdSize;
+	IMG_UINT8				*pui8CmdPtr;
+	RGXFWIF_KCCB_CMD		sPriorityCmd = { 0 };
+	RGXFWIF_CCB_CMD_HEADER	*psCmdHeader;
+	RGXFWIF_CMD_PRIORITY	*psCmd;
+	PVRSRV_ERROR			eError;
+	RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psContext);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = _CheckPriority(psDevInfo, i32Priority, psContext->eRequestor);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CheckPriority", fail_checkpriority);
+
+	/*
+		Get space for command
+	*/
+	ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_CMD_PRIORITY));
+
+	eError = RGXAcquireCCB(psClientCCB,
+						   ui32CmdSize,
+						   (void **) &pui8CmdPtr,
+						   PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire space for client CCB", __func__));
+		}
+		goto fail_ccbacquire;
+	}
+
+	/*
+		Write the command header and command
+	*/
+	psCmdHeader = IMG_OFFSET_ADDR(pui8CmdPtr, 0);
+	psCmdHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PRIORITY;
+	psCmdHeader->ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CMD_PRIORITY));
+	pui8CmdPtr += sizeof(*psCmdHeader);
+
+	psCmd = IMG_OFFSET_ADDR(pui8CmdPtr, 0);
+	psCmd->i32Priority = i32Priority;
+	pui8CmdPtr += sizeof(*psCmd);
+
+	/*
+		We should reserve space in the kernel CCB here and fill in the command
+		directly.
+		This is so if there isn't space in the kernel CCB we can return with
+		retry back to services client before we take any operations
+	*/
+
+	/*
+		Submit the command
+	*/
+	RGXReleaseCCB(psClientCCB,
+				  ui32CmdSize,
+				  PDUMP_FLAGS_CONTINUOUS);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release space in client CCB", __func__));
+		return eError;
+	}
+
+	/* Construct the priority command. */
+	sPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sPriorityCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psContext);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	sPriorityCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									eDM,
+									&sPriorityCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit set priority command with error (%u)",
+				__func__,
+				eError));
+		goto fail_cmdacquire;
+	}
+
+	psContext->i32Priority = i32Priority;
+
+	return PVRSRV_OK;
+
+fail_ccbacquire:
+fail_checkpriority:
+fail_cmdacquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext, RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	if (psCurrentServerCommonContext == NULL)
+	{
+		/* the context has already been freed so there is nothing to do here */
+		return PVRSRV_OK;
+	}
+
+	return CheckForStalledCCB(psCurrentServerCommonContext->psDevInfo->psDeviceNode,
+	                          psCurrentServerCommonContext->psClientCCB,
+	                          eKickTypeDM);
+}
+
+void DumpFWCommonContextInfo(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
+                             DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             IMG_UINT32 ui32VerbLevel)
+{
+	if (psCurrentServerCommonContext == NULL)
+	{
+		/* the context has already been freed so there is nothing to do here */
+		return;
+	}
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+		/* If high verbosity requested, dump whole CCB */
+		DumpCCB(psCurrentServerCommonContext->psDevInfo,
+		        psCurrentServerCommonContext->sFWCommonContextFWAddr,
+		        psCurrentServerCommonContext->psClientCCB,
+		        pfnDumpDebugPrintf,
+		        pvDumpDebugFile);
+	}
+	else
+	{
+		/* Otherwise, only dump first command in the CCB */
+		DumpFirstCCBCmd(psCurrentServerCommonContext->sFWCommonContextFWAddr,
+		                      psCurrentServerCommonContext->psClientCCB,
+		                      pfnDumpDebugPrintf,
+		                      pvDumpDebugFile);
+	}
+}
+
+void FWCommonContextListSetLastResetReason(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                           IMG_UINT32 *pui32ErrorPid,
+                                           const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_SERVER_COMMON_CONTEXT *psServerCommonContext = NULL;
+	IMG_UINT32 ui32ErrorPid = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMMON_CONTEXT *psThisContext =
+		    IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+		/* If the notification applies to all contexts update reset info
+		* for all contexts, otherwise only do so for the appropriate ID.
+		*/
+		if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS)
+		{
+			/* Notification applies to all contexts */
+			psThisContext->eLastResetReason    = psCmdContextResetNotification->eResetReason;
+			psThisContext->ui32LastResetJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+		}
+		else
+		{
+			/* Notification applies to one context only */
+			if (psThisContext->ui32ContextID == psCmdContextResetNotification->ui32ServerCommonContextID)
+			{
+				psServerCommonContext = psThisContext;
+				psServerCommonContext->eLastResetReason    = psCmdContextResetNotification->eResetReason;
+				psServerCommonContext->ui32LastResetJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+				ui32ErrorPid = RGXGetPIDFromServerMMUContext(psServerCommonContext->psServerMMUContext);
+				break;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+
+	if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: All contexts reset (Reason=%d, JobRef=0x%08x)",
+				__func__,
+				(IMG_UINT32)(psCmdContextResetNotification->eResetReason),
+				psCmdContextResetNotification->ui32ResetJobRef));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Context 0x%p reset (ID=0x%08x, Reason=%d, JobRef=0x%08x)",
+				__func__,
+				psServerCommonContext,
+				psCmdContextResetNotification->ui32ServerCommonContextID,
+				(IMG_UINT32)(psCmdContextResetNotification->eResetReason),
+				psCmdContextResetNotification->ui32ResetJobRef));
+	}
+
+	if (pui32ErrorPid)
+	{
+		*pui32ErrorPid = ui32ErrorPid;
+	}
+}
+
+/******************************************************************************
+ End of file (rgxfwcmnctx.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.h
new file mode 100644
index 000000000000..eba2fac705da
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwcmnctx.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware common context utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware common context utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWCMNCTX_H
+#define RGXFWCMNCTX_H
+
+#include "connection_server.h"
+#include "device.h"
+#include "rgxccb.h"
+#include "rgx_common.h"
+#include "devicemem_typedefs.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+
+/*************************************************************************/ /*!
+@Function       FWCommonContextAllocate
+
+@Description    Allocate a FW common context. This allocates the HW memory
+                for the context, the CCB and wires it all together.
+
+@Input          psConnection            Connection this context is being created on
+@Input          psDeviceNode            Device node to create the FW context on
+                                        (must be RGX device node)
+@Input          eRGXCCBRequestor        RGX_CCB_REQUESTOR_TYPE enum constant which
+                                        represents the requestor of this FWCC
+@Input          eDM                     Data Master type
+@Input          psServerMMUContext      Server MMU memory context.
+@Input          psAllocatedMemDesc      Pointer to pre-allocated MemDesc to use
+                                        as the FW context or NULL if this function
+                                        should allocate it
+@Input          ui32AllocatedOffset     Offset into pre-allocate MemDesc to use
+                                        as the FW context. If psAllocatedMemDesc
+                                        is NULL then this parameter is ignored
+@Input          psFWMemContextMemDesc   MemDesc of the FW memory context this
+                                        common context resides on
+@Input          psContextStateMemDesc   FW context state (context switch) MemDesc
+@Input          ui32CCBAllocSizeLog2    Size of the CCB for this context
+@Input          ui32CCBMaxAllocSizeLog2 Maximum size to which CCB can grow for this context
+@Input          ui32ContextFlags        Flags which specify properties of the context
+@Input          i32Priority             Priority of the context
+@Input          ui32MaxDeadlineMS       Max deadline limit in MS that the workload can run
+@Input          ui64RobustnessAddress   Address for FW to signal a context reset
+@Input          psInfo                  Structure that contains extra info
+                                        required for the creation of the context
+                                        (elements might change from core to core)
+@Return         PVRSRV_OK if the context was successfully created
+*/ /**************************************************************************/
+PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
+									 PVRSRV_DEVICE_NODE *psDeviceNode,
+									 RGX_CCB_REQUESTOR_TYPE eRGXCCBRequestor,
+									 RGXFWIF_DM eDM,
+									 SERVER_MMU_CONTEXT *psServerMMUContext,
+									 DEVMEM_MEMDESC *psAllocatedMemDesc,
+									 IMG_UINT32 ui32AllocatedOffset,
+									 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+									 DEVMEM_MEMDESC *psContextStateMemDesc,
+									 IMG_UINT32 ui32CCBAllocSizeLog2,
+									 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+									 IMG_UINT32 ui32ContextFlags,
+									 IMG_INT32 i32Priority,
+									 IMG_UINT32 ui32MaxDeadlineMS,
+									 IMG_UINT64 ui64RobustnessAddress,
+									 RGX_COMMON_CONTEXT_INFO *psInfo,
+									 RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext);
+
+
+void FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+SERVER_MMU_CONTEXT *FWCommonContextGetServerMMUCtx(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+RGX_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                                           IMG_UINT32 *pui32LastResetJobRef);
+
+PVRSRV_RGXDEV_INFO* FWCommonContextGetRGXDevInfo(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+PVRSRV_ERROR RGXGetFWCommonContextAddrFromServerMMUCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													   SERVER_MMU_CONTEXT *psServerMMUContext,
+													   PRGXFWIF_FWCOMMONCONTEXT *psFWCommonContextFWAddr);
+
+PRGXFWIF_FWCOMMONCONTEXT RGXGetFWCommonContextAddrFromServerCommonCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													                  DLLIST_NODE *psNode);
+
+PVRSRV_ERROR FWCommonContextSetFlags(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                     IMG_UINT32 ui32ContextFlags);
+
+PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
+								CONNECTION_DATA *psConnection,
+								PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_INT32 i32Priority,
+								RGXFWIF_DM eDM);
+
+PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext, RGX_KICK_TYPE_DM eKickTypeDM);
+
+void DumpFWCommonContextInfo(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
+                             DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             IMG_UINT32 ui32VerbLevel);
+
+void FWCommonContextListSetLastResetReason(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                           IMG_UINT32 *pui32ErrorPid,
+                                           const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification);
+
+#endif /* RGXFWCMNCTX_H */
+/******************************************************************************
+ End of file (rgxfwcmnctx.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.c
new file mode 100644
index 000000000000..2271adf3ea9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.c
@@ -0,0 +1,666 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debugging and miscellaneous functions server implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel services functions for debugging and other
+                miscellaneous functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvr_debug.h"
+#include "rgxfwdbg.h"
+#include "rgxfwutils.h"
+#include "rgxta3d.h"
+#include "pdump_km.h"
+#include "mmu_common.h"
+#include "devicemem_server.h"
+#include "osfunc.h"
+#include "vmm_pvz_server.h"
+#include "vz_vm.h"
+#if defined(PDUMP)
+#include "devicemem_pdump.h"
+#endif
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugQueryFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 *pui32RGXFWLogType)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!psDeviceNode || !pui32RGXFWLogType)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+	psDevInfo = psDeviceNode->pvDevice;
+
+	if (!psDevInfo || !psDevInfo->psRGXFWIfTraceBufCtl)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+	*pui32RGXFWLogType = psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType;
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetFWLogKM(
+	const CONNECTION_DATA * psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32RGXFWLogType)
+{
+	RGXFWIF_KCCB_CMD sLogTypeUpdateCmd;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevInfo;
+	IMG_UINT32 ui32OldRGXFWLogTpe;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo = psDeviceNode->pvDevice;
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+	ui32OldRGXFWLogTpe = psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType;
+
+	/* check log type is valid */
+	if (ui32RGXFWLogType & ~RGXFWIF_LOG_TYPE_MASK)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquire(psDevInfo->hRGXFWIfBufInitLock);
+
+	/* set the new log type and ensure the new log type is written to memory
+	 * before requesting the FW to read it
+	 */
+	psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = ui32RGXFWLogType;
+	OSMemoryBarrier(&psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+
+	/* Allocate firmware trace buffer resource(s) if not already done */
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo, RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	/* Check if LogType is TBI then allocate resource on demand and copy
+	 * SFs to it
+	 */
+	else if (RGXTBIBufferIsInitRequired(psDevInfo))
+	{
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+	}
+
+	/* TBI buffer address will be 0 if not initialised */
+	sLogTypeUpdateCmd.uCmdData.sTBIBuffer = psDevInfo->sRGXFWIfTBIBuffer;
+#else
+	sLogTypeUpdateCmd.uCmdData.sTBIBuffer.ui32Addr = 0;
+#endif
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate resource on-demand. Reverting to old value",
+		         __func__));
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = ui32OldRGXFWLogTpe;
+		OSMemoryBarrier(&psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType);
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+
+		OSLockRelease(psDevInfo->hRGXFWIfBufInitLock);
+
+		return eError;
+	}
+
+	OSLockRelease(psDevInfo->hRGXFWIfBufInitLock);
+
+	eError = PVRSRVPowerLock((PPVRSRV_DEVICE_NODE) psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire power lock (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState((PPVRSRV_DEVICE_NODE) psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of logType value */
+		sLogTypeUpdateCmd.eCmdType = RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sLogTypeUpdateCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock( (PPVRSRV_DEVICE_NODE) psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the LogType value to be updated in FW */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetHCSDeadlineKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32HCSDeadlineMS)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWSetHCSDeadline(psDevInfo, ui32HCSDeadlineMS);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugMapGuestHeapKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32DriverID,
+	IMG_UINT64 ui64GuestHeapBase)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID = psDeviceNode->sDevId.ui32InternalID;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		if (ui64GuestHeapBase == IMG_UINT64_MAX)
+		{
+			/* unmap heap and set DriverID to offline */
+			eError = PvzServerUnmapDevPhysHeap(ui32DriverID, ui32DeviceID);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap()");
+			eError = PvzServerOnVmOffline(ui32DriverID, ui32DeviceID);
+		}
+		else
+		{
+			/* set DriverID online if necessary and map firmware heap */
+			if (!IsVmOnline(ui32DriverID, ui32DeviceID))
+			{
+				eError = PvzServerOnVmOnline(ui32DriverID, ui32DeviceID);
+				PVR_LOG_RETURN_IF_ERROR(eError, "PvzServerOnVmOnline()");
+			}
+
+			eError = PvzServerMapDevPhysHeap(ui32DriverID, ui32DeviceID, RGX_FIRMWARE_RAW_HEAP_SIZE, ui64GuestHeapBase);
+		}
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		PVR_DPF((PVR_DBG_ERROR, " %s: Driver must run in Host mode to support Guest Mapping operations\n", __func__));
+	}
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32DriverID);
+	PVR_UNREFERENCED_PARAMETER(ui64GuestHeapBase);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32TSIntervalMs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzTSIntervalCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	sVzTSIntervalCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs = ui32TSIntervalMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the timeslice interval inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32TSIntervalMs),
+							  ui32TSIntervalMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzTSIntervalCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32TSPercentage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sVzTimeSliceCmd = { 0 };
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_INT32 ui32TSPercentageMax = 0;
+	IMG_UINT32 ui32DriverIDLoop;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_NOT_INITIALISED);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, PVRSRV_ERROR_NOT_INITIALISED);
+
+	/*
+	 * Each time slice is a number between 0 -> 100.
+	 * Use '0' to disable time slice based CSW for the driver.
+	 */
+	 /* Check if the sum exceeds PVRSRV_VZ_TIME_SLICE_MAX */
+	if (ui32TSPercentage)
+	{
+		PVR_RETURN_IF_FALSE(ui32TSPercentage <= PVRSRV_VZ_TIME_SLICE_MAX, PVRSRV_ERROR_INVALID_PARAMS);
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverIDLoop)
+		{
+			if (ui32DriverID != ui32DriverIDLoop)
+			{
+				ui32TSPercentageMax += psRuntimeCfg->aui32TSPercentage[ui32DriverIDLoop];
+			}
+			else
+			{
+				ui32TSPercentageMax += ui32TSPercentage;
+			}
+
+			PVR_RETURN_IF_FALSE(ui32TSPercentageMax <= PVRSRV_VZ_TIME_SLICE_MAX, PVRSRV_ERROR_INVALID_PARAMS);
+		}
+	}
+
+	sVzTimeSliceCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE;
+	psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] = ui32TSPercentage;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the timeslice of DriverID %u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, aui32TSPercentage) + (ui32DriverID * sizeof(ui32TSPercentage)),
+							  ui32TSPercentage,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzTimeSliceCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverPriorityKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_INT32   i32DriverPriority)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzPriorityCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((i32DriverPriority & ~RGXFW_VZ_PRIORITY_MASK) != 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sVzPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE;
+	psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID] = i32DriverPriority;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the priority of DriverID %u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ai32DriverPriority) + (ui32DriverID * sizeof(i32DriverPriority)),
+							  i32DriverPriority,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzPriorityCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverIsolationGroupKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32DriverIsolationGroup)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzIsolationGroupCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sVzIsolationGroupCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE;
+	psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = ui32DriverIsolationGroup;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the isolation group of DriverID%u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, aui32DriverIsolationGroup) + (ui32DriverID * sizeof(ui32DriverIsolationGroup)),
+							  ui32DriverIsolationGroup,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+				RGXFWIF_DM_GP,
+				&sVzIsolationGroupCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetOSNewOnlineStateKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32OSNewState)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_OS_STATE_CHANGE eState;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eState = (ui32OSNewState) ? (RGXFWIF_OS_ONLINE) : (RGXFWIF_OS_OFFLINE);
+	return RGXFWSetFwOsState(psDevInfo, ui32DriverID, eState);
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32DriverID);
+	PVR_UNREFERENCED_PARAMETER(ui32OSNewState);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPHRConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWConfigPHR(psDevInfo,
+	                      ui32PHRMode);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugWdgConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWConfigWdg(psDevInfo,
+	                      ui32WdgPeriodUs);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugDumpFreelistPageListKM(
+	CONNECTION_DATA * psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (dllist_is_empty(&psDevInfo->sFreeListHead))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG(("---------------[ Begin Freelist Page List Dump ]------------------"));
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		RGXDumpFreeListPageList(psFreeList);
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	PVR_LOG(("----------------[ End Freelist Page List Dump ]-------------------"));
+
+	return PVRSRV_OK;
+
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInjectFaultKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWInjectFault(psDevInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOffKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+#if defined(SUPPORT_AUTOVZ)
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	return PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+					       PVRSRV_SYS_POWER_STATE_OFF,
+					       PVRSRV_POWER_FLAGS_NONE);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOnKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+					       PVRSRV_SYS_POWER_STATE_ON,
+					       PVRSRV_POWER_FLAGS_NONE);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWSetVzConnectionCooldownPeriod(psDevInfo, ui32VzConnectionCooldownPeriodInSec);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.h
new file mode 100644
index 000000000000..59c48ed5f4f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwdbg.h
@@ -0,0 +1,160 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debugging and miscellaneous functions server interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel services functions for debugging and other
+                miscellaneous functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXFWDBG_H)
+#define RGXFWDBG_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "device.h"
+#include "pmr.h"
+
+#include "connection_server.h"
+
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInitFWImageKM(
+	PMR *psFWImgDestPMR,
+	PMR *psFWImgSrcPMR,
+	IMG_UINT64 ui64FWImgLen,
+	PMR *psFWImgSigPMR,
+	IMG_UINT64 ui64FWSigLen);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugQueryFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 *pui32RGXFWLogType);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32RGXFWLogType);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetHCSDeadlineKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32HCSDeadlineMS);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32TSPercentage);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32TSIntervalMs);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverPriorityKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_INT32   i32DriverPriority);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverIsolationGroupKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32DriverIsolationGroup);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetOSNewOnlineStateKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32OSNewState);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugMapGuestHeapKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32DriverID,
+	IMG_UINT64 ui64GuestHeapBase);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPHRConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32PHRMode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugWdgConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32WdgPeriodUs);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugDumpFreelistPageListKM(
+	CONNECTION_DATA * psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInjectFaultKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOffKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOnKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32VzConnectionCooldownPeriodInSec);
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.c
new file mode 100644
index 000000000000..afc69f7228f8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.c
@@ -0,0 +1,1165 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Firmware image utilities used at init time
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services Firmware image utilities used at init time
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxfwimageutils.h"
+#include "pvrversion.h"
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+#include "rgxfwutils.h"
+#endif
+
+/************************************************************************
+* FW layout information
+************************************************************************/
+#define MAX_NUM_ENTRIES (8)
+static RGX_FW_LAYOUT_ENTRY asRGXFWLayoutTable[MAX_NUM_ENTRIES];
+static IMG_UINT32 ui32LayoutEntryNum;
+
+
+static RGX_FW_LAYOUT_ENTRY* GetTableEntry(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		if (asRGXFWLayoutTable[i].eId == eId)
+		{
+			return &asRGXFWLayoutTable[i];
+		}
+	}
+
+	RGXErrorLog(hPrivate, "%s: id %u not found, returning entry 0\n",
+	            __func__, eId);
+
+	return &asRGXFWLayoutTable[0];
+}
+
+/*!
+*******************************************************************************
+
+ @Function      FindMMUSegment
+
+ @Description   Given a 32 bit FW address attempt to find the corresponding
+                pointer to FW allocation
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         ui32OffsetIn             : 32 bit FW address
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem code
+ @Input         uiHostAddrOut            : CPU pointer equivalent to ui32OffsetIn
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR FindMMUSegment(const void *hPrivate,
+                                   IMG_UINT32 ui32OffsetIn,
+                                   void *pvHostFWCodeAddr,
+                                   void *pvHostFWDataAddr,
+                                   void *pvHostFWCorememCodeAddr,
+                                   void *pvHostFWCorememDataAddr,
+                                   void **uiHostAddrOut)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		if ((ui32OffsetIn >= asRGXFWLayoutTable[i].ui32BaseAddr) &&
+		    (ui32OffsetIn < (asRGXFWLayoutTable[i].ui32BaseAddr + asRGXFWLayoutTable[i].ui32AllocSize)))
+		{
+			switch (asRGXFWLayoutTable[i].eType)
+			{
+				case FW_CODE:
+					*uiHostAddrOut = pvHostFWCodeAddr;
+					break;
+
+				case FW_DATA:
+					if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+					{
+						*uiHostAddrOut = RGXCalculateHostFWDataAddress(hPrivate, pvHostFWDataAddr);
+					}
+					else
+					{
+						*uiHostAddrOut = pvHostFWDataAddr;
+					}
+					break;
+
+				case FW_COREMEM_CODE:
+					*uiHostAddrOut = pvHostFWCorememCodeAddr;
+					break;
+
+				case FW_COREMEM_DATA:
+					*uiHostAddrOut = pvHostFWCorememDataAddr;
+					break;
+
+				default:
+					return PVRSRV_ERROR_INIT_FAILURE;
+			}
+
+			goto found;
+		}
+	}
+
+	return PVRSRV_ERROR_INIT_FAILURE;
+
+found:
+	if (*uiHostAddrOut == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Add offset to pointer to FW allocation now that allocation is found */
+
+	/* Direct Mem write to mapped memory */
+	ui32OffsetIn -= asRGXFWLayoutTable[i].ui32BaseAddr;
+	ui32OffsetIn += asRGXFWLayoutTable[i].ui32AllocOffset;
+
+	*(IMG_UINT8 **)uiHostAddrOut += ui32OffsetIn;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureSegID
+
+ @Description   Configures a single segment of the Segment MMU
+                (base, limit and out_addr)
+
+ @Input         hPrivate        : Implementation specific data
+ @Input         ui64SegOutAddr  : Segment output base address (40 bit devVaddr)
+ @Input         ui32SegBase     : Segment input base address (32 bit FW address)
+ @Input         ui32SegLimit    : Segment size
+ @Input         ui32SegID       : Segment ID
+ @Input         pszName         : Segment name
+ @Input         ppui32BootConf  : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureSegID(const void *hPrivate,
+                                IMG_UINT64 ui64SegOutAddr,
+                                IMG_UINT32 ui32SegBase,
+                                IMG_UINT32 ui32SegLimit,
+                                IMG_UINT32 ui32SegID,
+                                IMG_UINT32 **ppui32BootConf)
+{
+	IMG_UINT32 *pui32BootConf = *ppui32BootConf;
+	IMG_UINT32 ui32SegOutAddr0 = ui64SegOutAddr & 0x00000000FFFFFFFFUL;
+	IMG_UINT32 ui32SegOutAddr1 = (ui64SegOutAddr >> 32) & 0x00000000FFFFFFFFUL;
+
+	/* META segments have a minimum size */
+	IMG_UINT32 ui32LimitOff = (ui32SegLimit < RGXFW_SEGMMU_ALIGN) ?
+	                          RGXFW_SEGMMU_ALIGN : ui32SegLimit;
+	/* the limit is an offset, therefore off = size - 1 */
+	ui32LimitOff -= 1;
+
+	RGXCommentLog(hPrivate,
+	              "* Seg%d: meta_addr = 0x%08x, devv_addr = 0x%" IMG_UINT64_FMTSPECx ", limit = 0x%x",
+	              ui32SegID,
+	              ui32SegBase,
+	              ui64SegOutAddr,
+	              ui32LimitOff);
+
+	ui32SegBase |= RGXFW_SEGMMU_ALLTHRS_WRITEABLE;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_BASE(ui32SegID);
+	*pui32BootConf++ = ui32SegBase;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_LIMIT(ui32SegID);
+	*pui32BootConf++ = ui32LimitOff;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_OUTA0(ui32SegID);
+	*pui32BootConf++ = ui32SegOutAddr0;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_OUTA1(ui32SegID);
+	*pui32BootConf++ = ui32SegOutAddr1;
+
+	*ppui32BootConf = pui32BootConf;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureSegMMU
+
+ @Description   Configures META's Segment MMU
+
+ @Input         hPrivate             : Implementation specific data
+ @Input         psFWCodeDevVAddrBase : FW code base device virtual address
+ @Input         psFWDataDevVAddrBase : FW data base device virtual address
+ @Input         ppui32BootConf       : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureSegMMU(const void       *hPrivate,
+                                 IMG_DEV_VIRTADDR *psFWCodeDevVAddrBase,
+                                 IMG_DEV_VIRTADDR *psFWDataDevVAddrBase,
+                                 IMG_UINT32       **ppui32BootConf)
+{
+	IMG_UINT64 ui64SegOutAddrTop;
+	IMG_UINT32 i;
+
+	PVR_UNREFERENCED_PARAMETER(psFWCodeDevVAddrBase);
+
+	/* Configure Segment MMU */
+	RGXCommentLog(hPrivate, "********** FW configure Segment MMU **********");
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SLC_VIVT))
+	{
+		ui64SegOutAddrTop = RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(MMU_CONTEXT_MAPPING_FWPRIV);
+	}
+	else
+#endif
+	{
+		ui64SegOutAddrTop = RGXFW_SEGMMU_OUTADDR_TOP_SLC(MMU_CONTEXT_MAPPING_FWPRIV, RGXFW_SEGMMU_META_BIFDM_ID);
+	}
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		/*
+		 * FW code is using the bootloader segment which is already configured on boot.
+		 * FW coremem code and data don't use the segment MMU.
+		 * Only the FW data segment needs to be configured.
+		 */
+
+		if (asRGXFWLayoutTable[i].eType == FW_DATA)
+		{
+			IMG_UINT64 ui64SegOutAddr;
+			IMG_UINT32 ui32SegId = RGXFW_SEGMMU_DATA_ID;
+
+			ui64SegOutAddr = (psFWDataDevVAddrBase->uiAddr | ui64SegOutAddrTop) +
+			                  asRGXFWLayoutTable[i].ui32AllocOffset;
+
+			RGXFWConfigureSegID(hPrivate,
+			                    ui64SegOutAddr,
+			                    asRGXFWLayoutTable[i].ui32BaseAddr,
+			                    asRGXFWLayoutTable[i].ui32AllocSize,
+			                    ui32SegId,
+			                    ppui32BootConf); /*write the sequence to the bootldr */
+
+			break;
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureMetaCaches
+
+ @Description   Configure and enable the Meta instruction and data caches
+
+ @Input         hPrivate          : Implementation specific data
+ @Input         ui32NumThreads    : Number of FW threads in use
+ @Input         ppui32BootConf    : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureMetaCaches(const void *hPrivate,
+                                     IMG_UINT32 ui32NumThreads,
+                                     IMG_UINT32 **ppui32BootConf)
+{
+	IMG_UINT32 *pui32BootConf = *ppui32BootConf;
+	IMG_UINT32 ui32DCacheT0, ui32ICacheT0;
+	IMG_UINT32 ui32DCacheT1, ui32ICacheT1;
+	IMG_UINT32 ui32DCacheT2, ui32ICacheT2;
+	IMG_UINT32 ui32DCacheT3, ui32ICacheT3;
+
+#define META_CR_MMCU_LOCAL_EBCTRL                        (0x04830600)
+#define META_CR_MMCU_LOCAL_EBCTRL_ICWIN                  (0x3 << 14)
+#define META_CR_MMCU_LOCAL_EBCTRL_DCWIN                  (0x3 << 6)
+#define META_CR_SYSC_DCPART(n)                           (0x04830200 + (n)*0x8)
+#define META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE         (0x1 << 31)
+#define META_CR_SYSC_ICPART(n)                           (0x04830220 + (n)*0x8)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF  (0x8 << 16)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE       (0xF)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE       (0x7)
+#define META_CR_MMCU_DCACHE_CTRL                         (0x04830018)
+#define META_CR_MMCU_ICACHE_CTRL                         (0x04830020)
+#define META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN           (0x1)
+
+	RGXCommentLog(hPrivate, "********** Meta caches configuration *********");
+
+	/* Initialise I/Dcache settings */
+	ui32DCacheT0 = ui32DCacheT1 = (IMG_UINT32)META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE;
+	ui32DCacheT2 = ui32DCacheT3 = (IMG_UINT32)META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE;
+	ui32ICacheT0 = ui32ICacheT1 = ui32ICacheT2 = ui32ICacheT3 = 0;
+
+	if (ui32NumThreads == 1)
+	{
+		ui32DCacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE;
+		ui32ICacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE;
+	}
+	else
+	{
+		ui32DCacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE;
+		ui32ICacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE;
+
+		ui32DCacheT1 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE |
+		                META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF;
+		ui32ICacheT1 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE |
+		                META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF;
+	}
+
+	/* Local region MMU enhanced bypass: WIN-3 mode for code and data caches */
+	*pui32BootConf++ = META_CR_MMCU_LOCAL_EBCTRL;
+	*pui32BootConf++ = META_CR_MMCU_LOCAL_EBCTRL_ICWIN |
+	                   META_CR_MMCU_LOCAL_EBCTRL_DCWIN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_LOCAL_EBCTRL,
+	              META_CR_MMCU_LOCAL_EBCTRL_ICWIN | META_CR_MMCU_LOCAL_EBCTRL_DCWIN);
+
+	/* Data cache partitioning thread 0 to 3 */
+	*pui32BootConf++ = META_CR_SYSC_DCPART(0);
+	*pui32BootConf++ = ui32DCacheT0;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(1);
+	*pui32BootConf++ = ui32DCacheT1;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(2);
+	*pui32BootConf++ = ui32DCacheT2;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(3);
+	*pui32BootConf++ = ui32DCacheT3;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(0), ui32DCacheT0);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(1), ui32DCacheT1);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(2), ui32DCacheT2);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(3), ui32DCacheT3);
+
+	/* Enable data cache hits */
+	*pui32BootConf++ = META_CR_MMCU_DCACHE_CTRL;
+	*pui32BootConf++ = META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_DCACHE_CTRL,
+	              META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN);
+
+	/* Instruction cache partitioning thread 0 to 3 */
+	*pui32BootConf++ = META_CR_SYSC_ICPART(0);
+	*pui32BootConf++ = ui32ICacheT0;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(1);
+	*pui32BootConf++ = ui32ICacheT1;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(2);
+	*pui32BootConf++ = ui32ICacheT2;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(3);
+	*pui32BootConf++ = ui32ICacheT3;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(0), ui32ICacheT0);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(1), ui32ICacheT1);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(2), ui32ICacheT2);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(3), ui32ICacheT3);
+
+	/* Enable instruction cache hits */
+	*pui32BootConf++ = META_CR_MMCU_ICACHE_CTRL;
+	*pui32BootConf++ = META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_ICACHE_CTRL,
+	              META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN);
+
+	*pui32BootConf++ = 0x040000C0;
+	*pui32BootConf++ = 0;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x", 0x040000C0, 0);
+
+	*ppui32BootConf = pui32BootConf;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessLDRCommandStream
+
+ @Description   Process the output of the Meta toolchain in the .LDR format
+                copying code and data sections into their final location and
+                passing some information to the Meta bootloader
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbLDR                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+ @Input         ppui32BootConf           : Pointer to bootloader data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessLDRCommandStream(const void *hPrivate,
+                                     const IMG_BYTE* pbLDR,
+                                     void* pvHostFWCodeAddr,
+                                     void* pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr,
+                                     IMG_UINT32 **ppui32BootConf)
+{
+	RGX_META_LDR_BLOCK_HDR *psHeader = (RGX_META_LDR_BLOCK_HDR *) pbLDR;
+	RGX_META_LDR_L1_DATA_BLK *psL1Data =
+	    (RGX_META_LDR_L1_DATA_BLK*) ((IMG_UINT8 *) pbLDR + psHeader->ui32SLData);
+
+	IMG_UINT32 *pui32BootConf  = ppui32BootConf ? *ppui32BootConf : NULL;
+	IMG_UINT32 ui32CorememSize = RGXGetFWCorememSize(hPrivate);
+
+	RGXCommentLog(hPrivate, "**********************************************");
+	RGXCommentLog(hPrivate, "************** Begin LDR Parsing *************");
+	RGXCommentLog(hPrivate, "**********************************************");
+
+	while (psL1Data != NULL)
+	{
+		if (RGX_META_LDR_BLK_IS_COMMENT(psL1Data->ui16Cmd))
+		{
+			/* Don't process comment blocks */
+			goto NextBlock;
+		}
+
+		switch (psL1Data->ui16Cmd & RGX_META_LDR_CMD_MASK)
+		{
+			case RGX_META_LDR_CMD_LOADMEM:
+			{
+				RGX_META_LDR_L2_DATA_BLK *psL2Block =
+				    (RGX_META_LDR_L2_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->aui32CmdData[1]);
+				IMG_UINT32 ui32Offset = psL1Data->aui32CmdData[0];
+				IMG_UINT32 ui32DataSize = psL2Block->ui16Length - 6 /* L2 Tag length and checksum */;
+				void *pvWriteAddr;
+				PVRSRV_ERROR eError;
+
+				if (!RGX_META_IS_COREMEM_CODE(ui32Offset, ui32CorememSize) &&
+				    !RGX_META_IS_COREMEM_DATA(ui32Offset, ui32CorememSize))
+				{
+					/* Global range is aliased to local range */
+					ui32Offset &= ~META_MEM_GLOBAL_RANGE_BIT;
+				}
+
+				eError = FindMMUSegment(hPrivate,
+				                        ui32Offset,
+				                        pvHostFWCodeAddr,
+				                        pvHostFWDataAddr,
+				                        pvHostFWCorememCodeAddr,
+				                        pvHostFWCorememDataAddr,
+				                        &pvWriteAddr);
+
+				if (eError != PVRSRV_OK)
+				{
+					RGXErrorLog(hPrivate,
+					            "ProcessLDRCommandStream: Addr 0x%x (size: %d) not found in any segment",
+					            ui32Offset, ui32DataSize);
+					return eError;
+				}
+
+				/* Write to FW allocation only if available */
+				if (pvWriteAddr)
+				{
+					RGXMemCopy(hPrivate,
+					           pvWriteAddr,
+					           psL2Block->aui32BlockData,
+					           ui32DataSize);
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+					RGXFwSharedMemCacheOpExec(pvWriteAddr, ui32DataSize, PVRSRV_CACHE_OP_FLUSH);
+#endif
+				}
+
+				break;
+			}
+			case RGX_META_LDR_CMD_LOADCORE:
+			case RGX_META_LDR_CMD_LOADMMREG:
+			{
+				return PVRSRV_ERROR_INIT_FAILURE;
+			}
+			case RGX_META_LDR_CMD_START_THREADS:
+			{
+				/* Don't process this block */
+				break;
+			}
+			case RGX_META_LDR_CMD_ZEROMEM:
+			{
+				IMG_UINT32 ui32Offset = psL1Data->aui32CmdData[0];
+				IMG_UINT32 ui32ByteCount = psL1Data->aui32CmdData[1];
+				void *pvWriteAddr;
+				PVRSRV_ERROR  eError;
+
+				if (RGX_META_IS_COREMEM_DATA(ui32Offset, ui32CorememSize))
+				{
+					/* cannot zero coremem directly */
+					break;
+				}
+
+				/* Global range is aliased to local range */
+				ui32Offset &= ~META_MEM_GLOBAL_RANGE_BIT;
+
+				eError = FindMMUSegment(hPrivate,
+				                        ui32Offset,
+				                        pvHostFWCodeAddr,
+				                        pvHostFWDataAddr,
+				                        pvHostFWCorememCodeAddr,
+				                        pvHostFWCorememDataAddr,
+				                        &pvWriteAddr);
+
+				if (eError != PVRSRV_OK)
+				{
+					RGXErrorLog(hPrivate,
+					            "ProcessLDRCommandStream: Addr 0x%x (size: %d) not found in any segment",
+					            ui32Offset, ui32ByteCount);
+					return eError;
+				}
+
+				/* Write to FW allocation only if available */
+				if (pvWriteAddr)
+				{
+					RGXMemSet(hPrivate, pvWriteAddr, 0, ui32ByteCount);
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+					RGXFwSharedMemCacheOpExec(pvWriteAddr, ui32ByteCount, PVRSRV_CACHE_OP_FLUSH);
+#endif
+				}
+
+				break;
+			}
+			case RGX_META_LDR_CMD_CONFIG:
+			{
+				RGX_META_LDR_L2_DATA_BLK *psL2Block =
+				    (RGX_META_LDR_L2_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->aui32CmdData[0]);
+				RGX_META_LDR_CFG_BLK *psConfigCommand = (RGX_META_LDR_CFG_BLK*) psL2Block->aui32BlockData;
+				IMG_UINT32 ui32L2BlockSize = psL2Block->ui16Length - 6 /* L2 Tag length and checksum */;
+				IMG_UINT32 ui32CurrBlockSize = 0;
+
+				while (ui32L2BlockSize)
+				{
+					switch (psConfigCommand->ui32Type)
+					{
+						case RGX_META_LDR_CFG_PAUSE:
+						case RGX_META_LDR_CFG_READ:
+						{
+							ui32CurrBlockSize = 8;
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+						case RGX_META_LDR_CFG_WRITE:
+						{
+							IMG_UINT32 ui32RegisterOffset = psConfigCommand->aui32BlockData[0];
+							IMG_UINT32 ui32RegisterValue  = psConfigCommand->aui32BlockData[1];
+
+							/* Only write to bootloader if we got a valid
+							 * pointer to the FW code allocation
+							 */
+							if (pui32BootConf)
+							{
+								/* Do register write */
+								*pui32BootConf++ = ui32RegisterOffset;
+								*pui32BootConf++ = ui32RegisterValue;
+							}
+
+							RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+							              ui32RegisterOffset, ui32RegisterValue);
+
+							ui32CurrBlockSize = 12;
+							break;
+						}
+						case RGX_META_LDR_CFG_MEMSET:
+						case RGX_META_LDR_CFG_MEMCHECK:
+						{
+							ui32CurrBlockSize = 20;
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+						default:
+						{
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+					}
+					ui32L2BlockSize -= ui32CurrBlockSize;
+					psConfigCommand = (RGX_META_LDR_CFG_BLK*) (((IMG_UINT8*) psConfigCommand) + ui32CurrBlockSize);
+				}
+
+				break;
+			}
+			default:
+			{
+				return PVRSRV_ERROR_INIT_FAILURE;
+			}
+		}
+
+NextBlock:
+
+		if (psL1Data->ui32Next == 0xFFFFFFFF)
+		{
+			psL1Data = NULL;
+		}
+		else
+		{
+			psL1Data = (RGX_META_LDR_L1_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->ui32Next);
+		}
+	}
+
+	if (pui32BootConf)
+	{
+		*ppui32BootConf = pui32BootConf;
+	}
+
+	RGXCommentLog(hPrivate, "**********************************************");
+	RGXCommentLog(hPrivate, "************** End Loader Parsing ************");
+	RGXCommentLog(hPrivate, "**********************************************");
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessELFCommandStream
+
+ @Description   Process a file in .ELF format copying code and data sections
+                into their final location
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbELF                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessELFCommandStream(const void *hPrivate,
+                                     const IMG_BYTE *pbELF,
+                                     void *pvHostFWCodeAddr,
+                                     void *pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr)
+{
+	IMG_UINT32 ui32Entry;
+	IMG_ELF_HDR *psHeader = (IMG_ELF_HDR *)pbELF;
+	IMG_ELF_PROGRAM_HDR *psProgramHeader =
+	    (IMG_ELF_PROGRAM_HDR *)(pbELF + psHeader->ui32Ephoff);
+	PVRSRV_ERROR eError;
+
+	for (ui32Entry = 0; ui32Entry < psHeader->ui32Ephnum; ui32Entry++, psProgramHeader++)
+	{
+		void *pvWriteAddr;
+
+		/* Only consider loadable entries in the ELF segment table */
+		if (psProgramHeader->ui32Ptype != ELF_PT_LOAD) continue;
+
+		eError = FindMMUSegment(hPrivate,
+		                        psProgramHeader->ui32Pvaddr,
+		                        pvHostFWCodeAddr,
+		                        pvHostFWDataAddr,
+		                        pvHostFWCorememCodeAddr,
+		                        pvHostFWCorememDataAddr,
+		                        &pvWriteAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate,
+			            "%s: Addr 0x%x (size: %d) not found in any segment",__func__,
+			            psProgramHeader->ui32Pvaddr,
+			            psProgramHeader->ui32Pfilesz);
+			return eError;
+		}
+
+		/* Write to FW allocation only if available */
+		if (pvWriteAddr)
+		{
+			RGXMemCopy(hPrivate,
+			           pvWriteAddr,
+			           (IMG_PBYTE)(pbELF + psProgramHeader->ui32Poffset),
+			           psProgramHeader->ui32Pfilesz);
+
+			RGXMemSet(hPrivate,
+			          (IMG_PBYTE)pvWriteAddr + psProgramHeader->ui32Pfilesz,
+			          0,
+			          psProgramHeader->ui32Pmemsz - psProgramHeader->ui32Pfilesz);
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+			RGXFwSharedMemCacheOpExec(pvWriteAddr, psProgramHeader->ui32Pmemsz, PVRSRV_CACHE_OP_FLUSH);
+#endif
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_UINT32 RGXGetFWImageSectionOffset(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32AllocOffset;
+}
+
+IMG_UINT32 RGXGetFWImageSectionMaxSize(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32MaxSize;
+}
+
+IMG_UINT32 RGXGetFWImageSectionAllocSize(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32AllocSize;
+}
+
+IMG_UINT32 RGXGetFWImageSectionAddress(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32BaseAddr;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion1(const void *hPrivate,
+                                         const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	/* Applicable to any FW_INFO_VERSION */
+	if (psInfoHeader->ui32LayoutEntrySize != sizeof(RGX_FW_LAYOUT_ENTRY))
+	{
+		RGXErrorLog(hPrivate, "%s: FW layout entry sizes mismatch (expected: %u, found: %u)",
+		            __func__,
+		            (IMG_UINT32) sizeof(RGX_FW_LAYOUT_ENTRY),
+		            psInfoHeader->ui32LayoutEntrySize);
+	}
+
+	/* Applicable to any FW_INFO_VERSION */
+	if (psInfoHeader->ui32LayoutEntryNum > MAX_NUM_ENTRIES)
+	{
+		RGXErrorLog(hPrivate, "%s: Not enough storage for the FW layout table (max: %u entries, found: %u)",
+		            __func__,
+		            MAX_NUM_ENTRIES,
+		            psInfoHeader->ui32LayoutEntryNum);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Applicable to any FW_INFO_VERSION */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		if (psInfoHeader->ui32FwPageSize != RGXGetOSPageSize(hPrivate))
+		{
+			RGXErrorLog(hPrivate, "%s: FW page size mismatch (expected: %u, found: %u)",
+			            __func__,
+			            (IMG_UINT32) RGXGetOSPageSize(hPrivate),
+			            psInfoHeader->ui32FwPageSize);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	if (psInfoHeader->ui32InfoVersion != FW_INFO_VERSION)
+	{
+		/* Not an error because RGX_FW_INFO_HEADER is now versioned. It can grow
+		 * incrementally and it must be backwards compatible.
+		 */
+		RGXCommentLog(hPrivate, "%s: FW info version mismatch (expected: %u, found: %u)",
+		              __func__,
+		              (IMG_UINT32) FW_INFO_VERSION,
+		              psInfoHeader->ui32InfoVersion);
+		goto exit_version1_validation;
+	}
+
+	if (psInfoHeader->ui32HeaderLen != sizeof(RGX_FW_INFO_HEADER))
+	{
+		RGXErrorLog(hPrivate, "%s: FW info header sizes mismatch (expected: %u, found: %u)",
+		            __func__,
+		            (IMG_UINT32) sizeof(RGX_FW_INFO_HEADER),
+		            psInfoHeader->ui32HeaderLen);
+	}
+
+exit_version1_validation:
+	return PVRSRV_OK;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion2(const void *hPrivate,
+                                         const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	if (psInfoHeader->ui16PVRVersionMajor != PVRVERSION_MAJ ||
+	    psInfoHeader->ui16PVRVersionMinor != PVRVERSION_MIN)
+	{
+		RGXErrorLog(hPrivate, "%s: KM and FW version mismatch (expected: %u.%u, found: %u.%u)",
+		            __func__,
+		            PVRVERSION_MAJ,
+		            PVRVERSION_MIN,
+		            psInfoHeader->ui16PVRVersionMajor,
+		            psInfoHeader->ui16PVRVersionMinor);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion(const void *hPrivate,
+                                        const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	PVRSRV_ERROR eError;
+
+	switch (psInfoHeader->ui32InfoVersion)
+	{
+		default:
+			__fallthrough;
+		case 2:
+			eError = RGXValidateFWHeaderVersion2(hPrivate, psInfoHeader);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			__fallthrough;
+		case 1:
+			eError = RGXValidateFWHeaderVersion1(hPrivate, psInfoHeader);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			break;
+		case 0:
+			RGXErrorLog(hPrivate, "%s: invalid FW_INFO_VERSION", __func__);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetFWImageAllocSize(const void *hPrivate,
+                                    const IMG_BYTE    *pbRGXFirmware,
+                                    const IMG_UINT32  ui32RGXFirmwareSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWDataAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememDataAllocSize,
+                                    RGX_FW_INFO_HEADER *psFWInfoHeader)
+{
+	RGX_FW_INFO_HEADER *psInfoHeader;
+	const IMG_BYTE *pbRGXFirmwareInfo;
+	const IMG_BYTE *pbRGXFirmwareLayout;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	if (pbRGXFirmware == NULL || ui32RGXFirmwareSize == 0 || ui32RGXFirmwareSize <= FW_BLOCK_SIZE)
+	{
+		RGXErrorLog(hPrivate, "%s: Invalid FW binary at %p, size %u",
+		            __func__, pbRGXFirmware, ui32RGXFirmwareSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/*
+	 * Acquire pointer to the FW info header within the FW image.
+	 * The format of the header in the FW image might not be the one expected
+	 * by the driver, but the driver should still be able to correctly read
+	 * the information below, as long as new/incompatible elements are added
+	 * at the end of the header (they will be ignored by the driver).
+	 */
+
+	pbRGXFirmwareInfo = pbRGXFirmware + ui32RGXFirmwareSize - FW_BLOCK_SIZE;
+	psInfoHeader = (RGX_FW_INFO_HEADER*)pbRGXFirmwareInfo;
+
+	eError = RGXValidateFWHeaderVersion(hPrivate, psInfoHeader);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	ui32LayoutEntryNum = psInfoHeader->ui32LayoutEntryNum;
+
+
+	/*
+	 * Copy FW layout table from FW image to local array.
+	 * One entry is copied at a time and the copy is limited to what the driver
+	 * expects to find in it. Assuming that new/incompatible elements
+	 * are added at the end of each entry, the loop below adapts the table
+	 * in the FW image into the format expected by the driver.
+	 */
+
+	pbRGXFirmwareLayout = pbRGXFirmwareInfo + psInfoHeader->ui32HeaderLen;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		RGX_FW_LAYOUT_ENTRY *psOutEntry = &asRGXFWLayoutTable[i];
+
+		RGX_FW_LAYOUT_ENTRY *psInEntry = (RGX_FW_LAYOUT_ENTRY*)
+			(pbRGXFirmwareLayout + i * psInfoHeader->ui32LayoutEntrySize);
+
+		RGXMemCopy(hPrivate,
+		           (void*)psOutEntry,
+		           (void*)psInEntry,
+		           sizeof(RGX_FW_LAYOUT_ENTRY));
+	}
+
+
+	/* Calculate how much memory the FW needs for its code and data segments */
+
+	*puiFWCodeAllocSize = 0;
+	*puiFWDataAllocSize = 0;
+	*puiFWCorememCodeAllocSize = 0;
+	*puiFWCorememDataAllocSize = 0;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		switch (asRGXFWLayoutTable[i].eType)
+		{
+			case FW_CODE:
+				*puiFWCodeAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_DATA:
+				*puiFWDataAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_COREMEM_CODE:
+				*puiFWCorememCodeAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_COREMEM_DATA:
+				*puiFWCorememDataAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			default:
+				RGXErrorLog(hPrivate, "%s: Unknown FW section type %u\n",
+				            __func__, asRGXFWLayoutTable[i].eType);
+				break;
+		}
+	}
+
+	*psFWInfoHeader = *psInfoHeader;
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXProcessFWImage(const void *hPrivate,
+                               const IMG_BYTE *pbRGXFirmware,
+                               void *pvFWCode,
+                               void *pvFWData,
+                               void *pvFWCorememCode,
+                               void *pvFWCorememData,
+                               PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMIPS = IMG_FALSE;
+	IMG_BOOL bRISCV = RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR);
+	IMG_BOOL bMETA;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	bMIPS = (IMG_BOOL)RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS);
+#endif
+	bMETA = (IMG_BOOL)(!bMIPS && !bRISCV);
+
+	if (bMETA)
+	{
+		IMG_UINT32 *pui32BootConf = NULL;
+		/* Skip bootloader configuration if a pointer to the FW code
+		 * allocation is not available
+		 */
+		if (pvFWCode)
+		{
+			/* This variable points to the bootloader code which is mostly
+			 * a sequence of <register address,register value> pairs
+			 */
+			pui32BootConf = ((IMG_UINT32*) pvFWCode) + RGXFW_BOOTLDR_CONF_OFFSET;
+
+			/* Slave port and JTAG accesses are privileged */
+			*pui32BootConf++ = META_CR_SYSC_JTAG_THREAD;
+			*pui32BootConf++ = META_CR_SYSC_JTAG_THREAD_PRIV_EN;
+
+			RGXFWConfigureSegMMU(hPrivate,
+			                     &puFWParams->sMeta.sFWCodeDevVAddr,
+			                     &puFWParams->sMeta.sFWDataDevVAddr,
+			                     &pui32BootConf);
+		}
+
+		/* Process FW image data stream */
+		eError = ProcessLDRCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 pvFWCorememCode,
+		                                 pvFWCorememData,
+		                                 &pui32BootConf);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		/* Skip bootloader configuration if a pointer to the FW code
+		 * allocation is not available
+		 */
+		if (pvFWCode)
+		{
+			IMG_UINT32 ui32NumThreads   = puFWParams->sMeta.ui32NumThreads;
+
+			if ((ui32NumThreads == 0) || (ui32NumThreads > 2))
+			{
+				RGXErrorLog(hPrivate,
+				            "ProcessFWImage: Wrong Meta threads configuration, using one thread only");
+
+				ui32NumThreads = 1;
+			}
+
+			RGXFWConfigureMetaCaches(hPrivate,
+			                         ui32NumThreads,
+			                         &pui32BootConf);
+
+			/* Signal the end of the conf sequence */
+			*pui32BootConf++ = 0x0;
+			*pui32BootConf++ = 0x0;
+
+			if (puFWParams->sMeta.uiFWCorememCodeSize && (puFWParams->sMeta.sFWCorememCodeFWAddr.ui32Addr != 0))
+			{
+				*pui32BootConf++ = puFWParams->sMeta.sFWCorememCodeFWAddr.ui32Addr;
+				*pui32BootConf++ = puFWParams->sMeta.uiFWCorememCodeSize;
+			}
+			else
+			{
+				*pui32BootConf++ = 0;
+				*pui32BootConf++ = 0;
+			}
+
+#if defined(RGX_FEATURE_META_DMA_BIT_MASK)
+			if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_DMA))
+			{
+				*pui32BootConf++ = (IMG_UINT32) (puFWParams->sMeta.sFWCorememCodeDevVAddr.uiAddr >> 32);
+				*pui32BootConf++ = (IMG_UINT32) puFWParams->sMeta.sFWCorememCodeDevVAddr.uiAddr;
+			}
+			else
+#endif
+			{
+				*pui32BootConf++ = 0;
+				*pui32BootConf++ = 0;
+			}
+		}
+	}
+#if defined(RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES)
+	else if (bMIPS)
+	{
+		/* Process FW image data stream */
+		eError = ProcessELFCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 NULL,
+		                                 NULL);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		if (pvFWData)
+		{
+			RGXMIPSFW_BOOT_DATA *psBootData = (RGXMIPSFW_BOOT_DATA*)
+				/* To get a pointer to the bootloader configuration data start from a pointer to the FW image... */
+				IMG_OFFSET_ADDR(pvFWData,
+				/* ... jump to the boot/NMI data page... */
+				(RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA)
+				/* ... and then jump to the bootloader data offset within the page */
+				+ RGXMIPSFW_BOOTLDR_CONF_OFFSET));
+
+			/* Rogue Registers physical address */
+			psBootData->ui64RegBase = puFWParams->sMips.sGPURegAddr.uiAddr;
+
+			/* MIPS Page Table physical address */
+			psBootData->ui32PTLog2PageSize = puFWParams->sMips.ui32FWPageTableLog2PageSize;
+			psBootData->ui32PTNumPages     = puFWParams->sMips.ui32FWPageTableNumPages;
+			psBootData->aui64PTPhyAddr[0U] = puFWParams->sMips.asFWPageTableAddr[0U].uiAddr;
+			psBootData->aui64PTPhyAddr[1U] = puFWParams->sMips.asFWPageTableAddr[1U].uiAddr;
+			psBootData->aui64PTPhyAddr[2U] = puFWParams->sMips.asFWPageTableAddr[2U].uiAddr;
+			psBootData->aui64PTPhyAddr[3U] = puFWParams->sMips.asFWPageTableAddr[3U].uiAddr;
+
+			/* MIPS Stack Pointer Physical Address */
+			psBootData->ui64StackPhyAddr = puFWParams->sMips.sFWStackAddr.uiAddr;
+
+			/* Reserved for future use */
+			psBootData->ui32Reserved1 = 0;
+			psBootData->ui32Reserved2 = 0;
+		}
+	}
+#endif /* #if defined(RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES) */
+	else
+	{
+		/* Process FW image data stream */
+		eError = ProcessELFCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 pvFWCorememCode,
+		                                 pvFWCorememData);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		if (pvFWData)
+		{
+			RGXRISCVFW_BOOT_DATA *psBootData = (RGXRISCVFW_BOOT_DATA*)
+				IMG_OFFSET_ADDR(pvFWData, RGXRISCVFW_BOOTLDR_CONF_OFFSET);
+
+			psBootData->ui64CorememCodeDevVAddr = puFWParams->sRISCV.sFWCorememCodeDevVAddr.uiAddr;
+			psBootData->ui32CorememCodeFWAddr   = puFWParams->sRISCV.sFWCorememCodeFWAddr.ui32Addr;
+			psBootData->ui32CorememCodeSize     = puFWParams->sRISCV.uiFWCorememCodeSize;
+
+			psBootData->ui64CorememDataDevVAddr = puFWParams->sRISCV.sFWCorememDataDevVAddr.uiAddr;
+			psBootData->ui32CorememDataFWAddr   = puFWParams->sRISCV.sFWCorememDataFWAddr.ui32Addr;
+			psBootData->ui32CorememDataSize     = puFWParams->sRISCV.uiFWCorememDataSize;
+		}
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.h
new file mode 100644
index 000000000000..b4cedb8d3271
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwimageutils.h
@@ -0,0 +1,224 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for Services Firmware image utilities used at init time
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for Services Firmware image utilities used at init time
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWIMAGEUTILS_H
+#define RGXFWIMAGEUTILS_H
+
+/* The routines declared here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when DRM security is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary.
+ */
+#include "rgxlayer.h"
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionOffset
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return offset of a Firmware section, relative to the beginning
+               of the code or data allocation (depending on the section id)
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionOffset(const void *hPrivate,
+                                      RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionMaxSize
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return maximum size (not allocation size) of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionMaxSize(const void *hPrivate,
+                                       RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionAllocSize
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return allocation size of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionAllocSize(const void *hPrivate,
+                                         RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionAddress
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return base address of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionAddress(const void *hPrivate,
+                                       RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageAllocSize
+
+ @Description  Return size of Firmware code/data/coremem code allocations
+
+ @Input        hPrivate            : Implementation specific data
+ @Input        pbRGXFirmware       : Pointer to FW binary
+ @Input        ui32RGXFirmwareSize : FW binary size
+ @Output       puiFWCodeAllocSize  : Code size
+ @Output       puiFWDataAllocSize  : Data size
+ @Output       puiFWCorememCodeAllocSize : Coremem code size (0 if N/A)
+ @Output       puiFWCorememDataAllocSize : Coremem data size (0 if N/A)
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXGetFWImageAllocSize(const void *hPrivate,
+                                    const IMG_BYTE    *pbRGXFirmware,
+                                    const IMG_UINT32  ui32RGXFirmwareSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWDataAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememDataAllocSize,
+                                    RGX_FW_INFO_HEADER *psFWInfoHeader);
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessLDRCommandStream
+
+ @Description   Process the output of the Meta toolchain in the .LDR format
+                copying code and data sections into their final location and
+                passing some information to the Meta bootloader
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbLDR                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+ @Input         ppui32BootConf           : Pointer to bootloader data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessLDRCommandStream(const void *hPrivate,
+                                     const IMG_BYTE* pbLDR,
+                                     void* pvHostFWCodeAddr,
+                                     void* pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr,
+                                     IMG_UINT32 **ppui32BootConf);
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessELFCommandStream
+
+ @Description   Process a file in .ELF format copying code and data sections
+                into their final location
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbELF                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessELFCommandStream(const void *hPrivate,
+                                     const IMG_BYTE *pbELF,
+                                     void *pvHostFWCodeAddr,
+                                     void *pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXProcessFWImage
+
+ @Description  Process the Firmware binary blob copying code and data
+               sections into their final location and passing some
+               information to the Firmware bootloader.
+               If a pointer to the final memory location for FW code or data
+               is not valid (NULL) then the relative section will not be
+               processed.
+
+ @Input        hPrivate        : Implementation specific data
+ @Input        pbRGXFirmware   : Pointer to FW blob
+ @Input        pvFWCode        : Pointer to FW code
+ @Input        pvFWData        : Pointer to FW data
+ @Input        pvFWCorememCode : Pointer to FW coremem code
+ @Input        pvFWCorememData : Pointer to FW coremem data
+ @Input        puFWParams      : Parameters used by the FW at boot time
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessFWImage(const void *hPrivate,
+                               const IMG_BYTE *pbRGXFirmware,
+                               void *pvFWCode,
+                               void *pvFWData,
+                               void *pvFWCorememCode,
+                               void *pvFWCorememData,
+                               PVRSRV_FW_BOOT_PARAMS *puFWParams);
+
+#endif /* RGXFWIMAGEUTILS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwmemctx.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwmemctx.h
new file mode 100644
index 000000000000..49f37b1c27aa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwmemctx.h
@@ -0,0 +1,156 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware Memctx routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for operations on FWKM Shared memory context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWMEMCTX_H
+#define RGXFWMEMCTX_H
+
+#include "device.h"
+#include "rgx_memallocflags.h"
+#include "pvrsrv.h"
+#include "cache_ops.h"
+#include "cache_km.h"
+#include "pvr_debug.h"
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+/*
+ * RGXFwSharedMemCPUCacheMode()
+ * We upgrade allocations on ARM64 Linux when CPU Cache snooping is enabled.
+ * This is because of the Linux direct mapping causing interference due to PIPT
+ * cache. All allocations are normally UCWC but snooping can return a bad value from the
+ * direct mapping as it is cached. Upgrade our allocations to cached to prevent bad cached
+ * values but in turn we require flushing.
+ */
+static INLINE void RGXFwSharedMemCPUCacheMode(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)) == 0)
+	{
+		/* We don't need to upgrade if we don't map into the CPU */
+		return;
+	}
+
+	/* Clear the existing CPU cache flags */
+	*puiFlags &= ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK);
+
+	if (PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig))
+	{
+		*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT;
+	}
+	else
+	{
+		*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+	}
+}
+
+#define RGXFwSharedMemCheckSnoopMode(psDeviceConfig) PVR_ASSERT(PVRSRVSystemSnoopingOfCPUCache(psDeviceConfig))
+
+/*
+ * FWSharedMemCacheOpExec()
+ * This is the CPU data-cache maintenance interface for FW shared allocations.
+ * We have to be very careful that the VAs supplied to this function are
+ * sensible as to not cause a kernel oops. Given that this should only be
+ * used for allocations used for the FW this should be guaranteed.
+ */
+static INLINE PVRSRV_ERROR RGXFwSharedMemCacheOpExec(const volatile void *pvVirtStart,
+                                                     IMG_UINT64 uiSize,
+                                                     PVRSRV_CACHE_OP uiCacheOp)
+{
+	IMG_UINT64 uiEndAddr = (IMG_UINT64) pvVirtStart + uiSize;
+	IMG_CPU_PHYADDR uiUnusedPhysAddr = {.uiAddr = 0};
+
+	if (!pvVirtStart || uiSize == 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return CacheOpExec(NULL,
+					   (void*) pvVirtStart,
+					   (void*) uiEndAddr,
+					   uiUnusedPhysAddr,
+					   uiUnusedPhysAddr,
+					   uiCacheOp);
+}
+
+#define RGXFwSharedMemCacheOpValue(value, cacheop) (RGXFwSharedMemCacheOpExec(&value, sizeof(value), PVRSRV_CACHE_OP_##cacheop))
+#define RGXFwSharedMemCacheOpPtr(ptr, cacheop) (RGXFwSharedMemCacheOpExec(ptr, sizeof(*ptr), PVRSRV_CACHE_OP_##cacheop))
+#define RGXFwSharedMemCacheOpExecPfn RGXFwSharedMemCacheOpExec
+
+static INLINE void RGXFwSharedMemFlushCCB(void *pvCCBVirtAddr,
+                                          IMG_UINT64 uiStart,
+                                          IMG_UINT64 uiFinish,
+                                          IMG_UINT64 uiLimit)
+{
+	if (uiFinish >= uiStart)
+	{
+		/* Flush the CCB data */
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, uiStart),
+		                          uiFinish - uiStart,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+	else
+	{
+		/* CCCB wrapped around - flush the pre and post wrap boundary separately */
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, uiStart),
+		                          uiLimit - uiStart,
+		                          PVRSRV_CACHE_OP_FLUSH);
+
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, 0),
+		                          uiFinish,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+}
+#else
+#define RGXFwSharedMemCPUCacheMode(...)
+#define RGXFwSharedMemCheckSnoopMode(...)
+/* NULL value required for function callbacks */
+#define RGXFwSharedMemCacheOpExec(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpValue(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpPtr(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpExecPfn NULL
+#define RGXFwSharedMemFlushCCB(...)
+#endif
+
+
+#endif /* RGXFWMEMCTX_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.c
new file mode 100644
index 000000000000..555691010517
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.c
@@ -0,0 +1,1074 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware RISC-V utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware RISC-V utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGX_GET_DMI_REG(psDevInfo, value) \
+	((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? \
+	 RGX_CR_FWCORE_DMI_##value##__HOST_SECURITY_GEQ4 : RGX_CR_FWCORE_DMI_##value)
+#else
+#define RGX_GET_DMI_REG(psDevInfo, value) RGX_CR_FWCORE_DMI_##value
+#endif
+
+/*
+ * RGXRiscvHalt
+ */
+PVRSRV_ERROR RGXRiscvHalt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_DMCONTROL_Reg = RGX_GET_DMI_REG(psDevInfo, DMCONTROL);
+	__maybe_unused IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Halt RISC-V FW");
+
+	/* Send halt request (no need to select one or more harts on this RISC-V core) */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN |
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until hart is halted */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DMSTATUS_Reg,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	/* Clear halt request */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send halt request (no need to select one or more harts on this RISC-V core) */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN |
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+
+	/* Wait until hart is halted */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_DMSTATUS_Reg/sizeof(IMG_UINT32),
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Hart not halted (0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Clear halt request */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXRiscvIsHalted
+ */
+IMG_BOOL RGXRiscvIsHalted(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	/* Assume the core is always halted in nohw */
+	return IMG_TRUE;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+	IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+	return (OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg) &
+	        RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN) != 0U;
+#endif
+}
+
+/*
+ * RGXRiscvResume
+ */
+PVRSRV_ERROR RGXRiscvResume(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_DMCONTROL_Reg = RGX_GET_DMI_REG(psDevInfo, DMCONTROL);
+	__maybe_unused IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Resume RISC-V FW");
+
+	/* Send resume request (no need to select one or more harts on this RISC-V core) */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN |
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until hart is resumed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DMSTATUS_Reg,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	/* Clear resume request */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send resume request (no need to select one or more harts on this RISC-V core) */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN |
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+
+	/* Wait until hart is resumed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_DMSTATUS_Reg/sizeof(IMG_UINT32),
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Hart not resumed (0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Clear resume request */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvCheckAbstractCmdError
+
+@Description    Check for RISC-V abstract command errors and clear them
+
+@Input          psDevInfo    Pointer to GPU device info
+
+@Return         RGXRISCVFW_ABSTRACT_CMD_ERR
+******************************************************************************/
+static RGXRISCVFW_ABSTRACT_CMD_ERR RGXRiscvCheckAbstractCmdError(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXRISCVFW_ABSTRACT_CMD_ERR eCmdErr;
+
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	eCmdErr = RISCV_ABSTRACT_CMD_NO_ERROR;
+
+	/* Check error status */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            RISCV_ABSTRACT_CMD_NO_ERROR << RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT,
+	            ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	void __iomem *pvRegsBaseKM = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Check error status */
+	eCmdErr = (OSReadUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_ABSTRACTCS_Reg)
+	          & ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK)
+	          >> RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT;
+
+	if (eCmdErr != RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RISC-V FW abstract command error %u", eCmdErr));
+
+		/* Clear the error (note CMDERR field is write-1-to-clear) */
+		OSWriteUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_ABSTRACTCS_Reg,
+		               ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK);
+	}
+#endif
+
+	return eCmdErr;
+}
+
+/*
+ * RGXRiscReadReg
+ */
+PVRSRV_ERROR RGXRiscvReadReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading HW registers is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send abstract register read command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_READ |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	               ui32RegAddr);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckAbstractCmdError(psDevInfo) == RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		/* Read register value */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+
+/*
+ * RGXRiscvPollReg
+ */
+PVRSRV_ERROR RGXRiscvPollReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V register 0x%x (expected 0x%08x)",
+	                      ui32RegAddr, ui32Value);
+
+	/* Send abstract register read command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_READ |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	           ui32RegAddr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckAbstractCmdError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling HW registers is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+/*
+ * RGXRiscvWriteReg
+ */
+PVRSRV_ERROR RGXRiscvWriteReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32RegAddr,
+                              IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V register 0x%x (value 0x%08x)",
+	                      ui32RegAddr, ui32Value);
+
+	/* Prepare data to be written to register */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract register write command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_WRITE |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	           ui32RegAddr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare data to be written to register */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg, ui32Value);
+
+	/* Send abstract register write command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_WRITE |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	               ui32RegAddr);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvCheckSysBusError
+
+@Description    Check for RISC-V system bus errors and clear them
+
+@Input          psDevInfo    Pointer to GPU device info
+
+@Return         RGXRISCVFW_SYSBUS_ERR
+******************************************************************************/
+static __maybe_unused RGXRISCVFW_SYSBUS_ERR RGXRiscvCheckSysBusError(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXRISCVFW_SYSBUS_ERR eSBError;
+
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	eSBError = RISCV_SYSBUS_NO_ERROR;
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            RISCV_SYSBUS_NO_ERROR << RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT,
+	            ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	void __iomem *pvRegsBaseKM = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	eSBError = (OSReadUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_SBCS_Reg)
+	         & ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK)
+	         >> RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT;
+
+	if (eSBError != RISCV_SYSBUS_NO_ERROR)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RISC-V FW system bus error %u", eSBError));
+
+		/* Clear the error (note SBERROR field is write-1-to-clear) */
+		OSWriteUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_SBCS_Reg,
+		               ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK);
+	}
+#endif
+
+	return eSBError;
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadAbstractMem
+
+@Description    Read a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvReadAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading memory is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare read address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA1_Reg, ui32Addr);
+
+	/* Send abstract memory read command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_READ |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckAbstractCmdError(psDevInfo) == RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		/* Read memory value */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollAbstractMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvPollAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V address 0x%x (expected 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Prepare read address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA1_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract memory read command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_READ |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckAbstractCmdError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling memory is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadSysBusMem
+
+@Description    Read a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvReadSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading memory is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_SBCS_Reg,
+	               (RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT) |
+	               RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN);
+
+	/* Perform read */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBADDRESS0_Reg, ui32Addr);
+
+	/* Wait until system bus is idle */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_SBCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: System Bus did not go idle in time (sbcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckSysBusError(psDevInfo) == RISCV_SYSBUS_NO_ERROR)
+	{
+		/* Read value from debug system bus */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollSysBusMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvPollSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBDATA0_Reg = RGX_GET_DMI_REG(psDevInfo, SBDATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V address 0x%x (expected 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBCS_Reg,
+	           (RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT) |
+	           RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Perform read */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_SBADDRESS0_Reg,
+	           ui32Addr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until system bus is idle */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckSysBusError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBDATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling memory is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+#if !defined(EMULATOR)
+/*
+ * RGXRiscvReadMem
+ */
+PVRSRV_ERROR RGXRiscvReadMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 *pui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvReadAbstractMem(psDevInfo, ui32Addr, pui32Value);
+	}
+
+	return RGXRiscvReadSysBusMem(psDevInfo, ui32Addr, pui32Value);
+}
+#endif /* !defined(EMULATOR) */
+
+/*
+ * RGXRiscvPollMem
+ */
+PVRSRV_ERROR RGXRiscvPollMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 ui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvPollAbstractMem(psDevInfo, ui32Addr, ui32Value);
+	}
+
+	return RGXRiscvPollSysBusMem(psDevInfo, ui32Addr, ui32Value);
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteAbstractMem
+
+@Description    Write a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvWriteAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V address 0x%x (value 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Prepare write address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA1_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write data */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract register write command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_WRITE |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare write address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA1_Reg, ui32Addr);
+
+	/* Prepare write data */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg, ui32Value);
+
+	/* Send abstract memory write command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_WRITE |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteSysBusMem
+
+@Description    Write a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvWriteSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBDATA0_Reg = RGX_GET_DMI_REG(psDevInfo, SBDATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V address 0x%x (value 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBCS_Reg,
+	           RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBADDRESS0_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write data and initiate write */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBDATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until system bus is idle */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Configure system bus for 32 bit accesses */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_SBCS_Reg,
+	               RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT);
+
+	/* Prepare write address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBADDRESS0_Reg, ui32Addr);
+
+	/* Prepare write data and initiate write */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBDATA0_Reg, ui32Value);
+
+	/* Wait until system bus is idle */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_SBCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: System Bus did not go idle in time (sbcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXRiscvWriteMem
+ */
+PVRSRV_ERROR RGXRiscvWriteMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32Addr,
+                              IMG_UINT32 ui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvWriteAbstractMem(psDevInfo, ui32Addr, ui32Value);
+	}
+
+	return RGXRiscvWriteSysBusMem(psDevInfo, ui32Addr, ui32Value);
+}
+#endif /* !defined(EMULATOR) */
+
+/*
+ * RGXRiscvDmiOp
+ */
+PVRSRV_ERROR RGXRiscvDmiOp(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT64 *pui64DMI)
+{
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(pui64DMI);
+
+	/* Accessing DM registers is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define DMI_BASE     ((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4 : RGX_CR_FWCORE_DMI_RESERVED00)
+#else
+#define DMI_BASE     RGX_CR_FWCORE_DMI_RESERVED00
+#endif
+#define DMI_STRIDE  (RGX_CR_FWCORE_DMI_RESERVED01 - RGX_CR_FWCORE_DMI_RESERVED00)
+#define DMI_REG(r)  ((DMI_BASE) + (DMI_STRIDE) * (r))
+
+#define DMI_OP_SHIFT            0U
+#define DMI_OP_MASK             0x3ULL
+#define DMI_DATA_SHIFT          2U
+#define DMI_DATA_MASK           0x3FFFFFFFCULL
+#define DMI_ADDRESS_SHIFT       34U
+#define DMI_ADDRESS_MASK        0xFC00000000ULL
+
+#define DMI_OP_NOP	            0U
+#define DMI_OP_READ	            1U
+#define DMI_OP_WRITE	        2U
+#define DMI_OP_RESERVED	        3U
+
+#define DMI_OP_STATUS_SUCCESS	0U
+#define DMI_OP_STATUS_RESERVED	1U
+#define DMI_OP_STATUS_FAILED	2U
+#define DMI_OP_STATUS_BUSY	    3U
+
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64Op, ui64Address, ui64Data;
+
+	ui64Op      = (*pui64DMI & DMI_OP_MASK) >> DMI_OP_SHIFT;
+	ui64Address = (*pui64DMI & DMI_ADDRESS_MASK) >> DMI_ADDRESS_SHIFT;
+	ui64Data    = (*pui64DMI & DMI_DATA_MASK) >> DMI_DATA_SHIFT;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		ui64Op = DMI_OP_STATUS_FAILED;
+		goto dmiop_update;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to retrieve RGX power state (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		ui64Op = DMI_OP_STATUS_FAILED;
+		goto dmiop_release_lock;
+	}
+
+	if (ePowerState == PVRSRV_DEV_POWER_STATE_ON)
+	{
+		void __iomem *pvRegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+		switch (ui64Op)
+		{
+			case DMI_OP_NOP:
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			case DMI_OP_WRITE:
+				OSWriteUncheckedHWReg32(pvRegsBase,
+						DMI_REG(ui64Address),
+						(IMG_UINT32)ui64Data);
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			case DMI_OP_READ:
+				ui64Data = (IMG_UINT64)OSReadUncheckedHWReg32(pvRegsBase,
+						DMI_REG(ui64Address));
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR, "%s: unknown op %u", __func__, (IMG_UINT32)ui64Op));
+				ui64Op = DMI_OP_STATUS_FAILED;
+				break;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Accessing RISC-V Debug Module is not "
+					"possible while the GPU is powered off", __func__));
+
+		ui64Op = DMI_OP_STATUS_FAILED;
+	}
+
+dmiop_release_lock:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+dmiop_update:
+	*pui64DMI = (ui64Op << DMI_OP_SHIFT) |
+		(ui64Address << DMI_ADDRESS_SHIFT) |
+		(ui64Data << DMI_DATA_SHIFT);
+
+	return eError;
+#endif
+}
+
+/******************************************************************************
+ End of file (rgxfwriscv.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.h
new file mode 100644
index 000000000000..7821c167fa2a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwriscv.h
@@ -0,0 +1,217 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware RISC-V utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware RISC-V utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWRISCV_H
+#define RGXFWRISCV_H
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGXRISCVFW_GET_REMAP_SECURE(r)          (RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4 + ((r) * 8U))
+#define RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE    (RGXRISCVFW_GET_REMAP_SECURE(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_REMAP_SECURE    (RGXRISCVFW_GET_REMAP_SECURE(RGXRISCVFW_BOOTLDR_DATA_REGION))
+#define RGX_GET_RISCV_REGS_BASE(psDevInfo) \
+	((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? \
+	 (psDevInfo)->pvSecureRegsBaseKM : (psDevInfo)->pvRegsBaseKM)
+#else
+#define RGX_GET_RISCV_REGS_BASE(psDevInfo) ((psDevInfo)->pvRegsBaseKM)
+#endif
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvHalt
+
+@Description    Halt the RISC-V FW core (required for certain operations
+                done through Debug Module)
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvHalt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvIsHalted
+
+@Description    Check if the RISC-V FW is halted
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         IMG_BOOL
+******************************************************************************/
+IMG_BOOL RGXRiscvIsHalted(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvResume
+
+@Description    Resume the RISC-V FW core
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvResume(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadReg
+
+@Description    Read a value from the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvReadReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollReg
+
+@Description    Poll for a value from the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvPollReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteReg
+
+@Description    Write a value to the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvWriteReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32RegAddr,
+                              IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvPollMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 ui32Value);
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadMem
+
+@Description    Read a value at the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvReadMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteMem
+
+@Description    Write a value to the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvWriteMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32Addr,
+                              IMG_UINT32 ui32Value);
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvDmiOp
+
+@Description    Acquire the powerlock and perform an operation on the RISC-V
+                Debug Module Interface, but only if the GPU is powered on.
+
+@Input          psDevInfo       Pointer to device info
+@InOut          pui64DMI        Encoding of a request for the RISC-V Debug
+                                Module with same format as the 'dmi' register
+                                from the RISC-V debug specification (v0.13+).
+                                On return, this is updated with the result of
+                                the request, encoded the same way.
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvDmiOp(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT64 *pui64DMI);
+
+#endif /* RGXFWRISCV_H */
+/******************************************************************************
+ End of file (rgxfwriscv.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwtrace_strings.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwtrace_strings.c
new file mode 100644
index 000000000000..d95050871994
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwtrace_strings.c
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File           rgxfwtrace_strings.c
+@Title          RGX Firmware trace strings
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgx_fwif_sf.h"
+#include "fwtrace_string.h"
+
+/* The tuple pairs that will be generated using XMacros will be stored here.
+ * This macro definition must match the definition of SFids in rgx_fwif_sf.h
+ */
+const RGXKM_STID_FMT SFs[]= {
+#define X(a, b, c, d, e) { RGXFW_LOG_CREATESFID(a,b,e), d },
+	RGXFW_LOG_SFIDLIST
+#undef X
+};
+
+const IMG_UINT32 g_ui32SFsCount = ARRAY_SIZE(SFs);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwutils.h
new file mode 100644
index 000000000000..6752bc95fa18
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxfwutils.h
@@ -0,0 +1,1298 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWUTILS_H
+#define RGXFWUTILS_H
+
+#include "rgx_memallocflags.h"
+#include "log2.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "devicemem.h"
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "connection_server.h"
+#include "rgxta3d.h"
+#include "devicemem_utils.h"
+#include "rgxmem.h"
+#include "rgxfwmemctx.h"
+#include "rgxinit_apphints.h"
+
+#define RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT   "FwRawDriverID%d" /*!< RGX Raw Firmware Heap identifier */
+
+static INLINE PVRSRV_ERROR _SelectDevMemHeap(PVRSRV_RGXDEV_INFO *psDevInfo,
+											 PVRSRV_MEMALLOCFLAGS_T *puiFlags,
+											 DEVMEM_HEAP **ppsFwHeap)
+{
+	PVRSRV_PHYS_HEAP ePhysHeap = (PVRSRV_PHYS_HEAP)PVRSRV_GET_PHYS_HEAP_HINT(*puiFlags);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (ePhysHeap)
+	{
+		case PVRSRV_PHYS_HEAP_FW_CODE:
+		case PVRSRV_PHYS_HEAP_FW_PRIV_DATA:
+		case PVRSRV_PHYS_HEAP_FW_MAIN:
+		{
+			*ppsFwHeap = psDevInfo->psFirmwareMainHeap;
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_FW_CONFIG:
+		{
+			*ppsFwHeap = psDevInfo->psFirmwareConfigHeap;
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_FW_PREMAP0:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP1:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP2:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP3:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP4:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP5:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP6:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP7:
+		{
+			IMG_UINT32 ui32DriverID = ePhysHeap - PVRSRV_PHYS_HEAP_FW_PREMAP0;
+
+			PVR_LOG_RETURN_IF_INVALID_PARAM(ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED, "ui32DriverID");
+			*ppsFwHeap = psDevInfo->psPremappedFwRawHeap[ui32DriverID];
+			break;
+		}
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid phys heap", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			break;
+		}
+	}
+
+	return eError;
+}
+
+/*
+ * Firmware-only allocation (which are initialised by the host) must be aligned to the SLC cache line size.
+ * This is because firmware-only allocations are GPU_CACHE_INCOHERENT and this causes problems
+ * if two allocations share the same cache line; e.g. the initialisation of the second allocation won't
+ * make it into the SLC cache because it has been already loaded when accessing the content of the first allocation.
+ */
+static INLINE PVRSRV_ERROR DevmemFwAllocate(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_DEVMEM_SIZE_T uiSize,
+											PVRSRV_MEMALLOCFLAGS_T uiFlags,
+											const IMG_CHAR *pszText,
+											DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_DEVMEM_ALIGN_T uiAlign;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) && (pszText[0] == 'F') && (pszText[1] == 'w'));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	if (psFwHeap == psDevInfo->psFirmwareConfigHeap)
+	{
+		/*
+		 * All structures allocated from the Firmware Config sub-heap must start at the same pre-determined
+		 * offsets, regardless of the system's page size (e.g. 4k,16k,64k). The alignment requirement is
+		 * satisfied for virtual addresses during the mapping stage. Physical allocations do not take
+		 * alignment into consideration.
+		 * VZ drivers usually preallocate and pre-map the entire Firmware heap range. Any allocations from
+		 * this heap are physical allocations only, having their device VAs derived from their PAs. This makes
+		 * it impossible to fulfil alignment requirements.
+		 * To work around this limitation, allocation sizes are rounded to the nearest multiple of 64kb,
+		 * regardless of the actual size of object.
+		 */
+		uiAlign = RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY;
+
+		uiSize = PVR_ALIGN(uiSize, RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY);
+	}
+	else
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			/* Aligning FW based allocations for MIPS based rogue cores at cache line boundary(16 bytes) instead
+			 * of SLC(64 bytes) to have more compact memory with less waste and hopefully save some TLB misses.
+			 * MIPS CPU cores alignment.
+			 */
+			uiAlign = RGXMIPSFW_MICROAPTIVEAP_CACHELINE_SIZE;
+		}
+		else
+#endif
+		{
+			/* Non-MIPS CPU cores alignment */
+			uiAlign = (GET_ROGUE_CACHE_LINE_SIZE(RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS)));
+		}
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+	eError = DevmemAllocateAndMap(psFwHeap,
+				uiSize,
+				uiAlign,
+				uiFlags,
+				pszText,
+				ppsMemDescPtr,
+				&sTmpDevVAddr);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static INLINE PVRSRV_ERROR DevmemFwAllocateExportable(PVRSRV_DEVICE_NODE *psDeviceNode,
+													  IMG_DEVMEM_SIZE_T uiSize,
+													  IMG_DEVMEM_ALIGN_T uiAlign,
+													  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+													  const IMG_CHAR *pszText,
+													  DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_UINT32 ui32HeapLog2PageSize;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) &&
+			(pszText[0] == 'F') && (pszText[1] == 'w') &&
+			(pszText[2] == 'E') && (pszText[3] == 'x'));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS cores */
+		ui32HeapLog2PageSize = ExactLog2(uiAlign);
+	}
+	else
+#endif
+	{
+		/* Meta and RiscV cores */
+		ui32HeapLog2PageSize = DevmemGetHeapLog2PageSize(psFwHeap);
+	}
+
+	eError = DevmemAllocateExportable(psDeviceNode,
+									  uiSize,
+									  uiAlign,
+									  ui32HeapLog2PageSize,
+									  uiFlags,
+									  pszText,
+									  ppsMemDescPtr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW DevmemAllocateExportable failed (%u)", eError));
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	/*
+		We need to map it so the heap for this allocation
+		is set
+	*/
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+							   psFwHeap,
+							   &sTmpDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsMemDescPtr);
+		PVR_DPF((PVR_DBG_ERROR, "FW DevmemMapToDevice failed (%u)", eError));
+	}
+
+	PVR_DPF_RETURN_RC1(eError, *ppsMemDescPtr);
+}
+
+static INLINE PVRSRV_ERROR DevmemFwAllocateSparse(PVRSRV_RGXDEV_INFO *psDevInfo,
+												IMG_DEVMEM_SIZE_T uiSize,
+												IMG_UINT32 ui32NumPhysChunks,
+												IMG_UINT32 ui32NumVirtChunks,
+												IMG_UINT32 *pui32MappingTable,
+												PVRSRV_MEMALLOCFLAGS_T uiFlags,
+												const IMG_CHAR *pszText,
+												DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_UINT32 ui32Align;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) && (pszText[0] == 'F') && (pszText[1] == 'w'));
+	ui32Align = GET_ROGUE_CACHE_LINE_SIZE(RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+	eError = DevmemAllocateSparse(psDevInfo->psDeviceNode,
+								uiSize,
+								ui32NumPhysChunks,
+								ui32NumVirtChunks,
+								pui32MappingTable,
+								ui32Align,
+								DevmemGetHeapLog2PageSize(psFwHeap),
+								uiFlags | PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING,
+								pszText,
+								ppsMemDescPtr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	/*
+		We need to map it so the heap for this allocation
+		is set
+	*/
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+				   psFwHeap,
+				   &sTmpDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsMemDescPtr);
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+static INLINE void DevmemFwUnmapAndFree(PVRSRV_RGXDEV_INFO *psDevInfo,
+								DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_DPF_ENTERED1(psMemDesc);
+
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+
+	DevmemReleaseDevVirtAddr(psMemDesc);
+	DevmemFree(psMemDesc);
+
+	PVR_DPF_RETURN;
+}
+
+/*
+ * This function returns the value of the hardware register RGX_CR_TIMER
+ * which is a timer counting in ticks.
+ */
+
+static INLINE IMG_UINT64 RGXReadHWTimerReg(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 ui64Time = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TIMER);
+
+	/*
+	*  In order to avoid having to issue three 32-bit reads to detect the
+	*  lower 32-bits wrapping, the MSB of the low 32-bit word is duplicated
+	*  in the MSB of the high 32-bit word. If the wrap happens, we just read
+	*  the register again (it will not wrap again so soon).
+	*/
+	if ((ui64Time ^ (ui64Time << 32)) & ~RGX_CR_TIMER_BIT31_CLRMSK)
+	{
+		ui64Time = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TIMER);
+	}
+
+	return (ui64Time & ~RGX_CR_TIMER_VALUE_CLRMSK) >> RGX_CR_TIMER_VALUE_SHIFT;
+}
+
+/*
+ * This FW Common Context is only mapped into kernel for initialisation and cleanup purposes.
+ * Otherwise this allocation is only used by the FW.
+ * Therefore the GPU cache doesn't need coherency, and write-combine will
+ * suffice on the CPU side (WC buffer will be flushed at the first kick)
+ */
+#define RGX_FWCOMCTX_ALLOCFLAGS      (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                      PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)| \
+                                      PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                      PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                      PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+#define RGX_FWCODEDATA_ALLOCFLAGS    (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                      PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | \
+                                      PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)
+
+#define RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                         PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                         PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+#define RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CONFIG))
+
+#define RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+/* Firmware memory that is not accessible by the CPU. */
+#define RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                             PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                             PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC)
+
+/* Firmware shared memory that is supposed to be read-only to the CPU.
+ * In reality it isn't due to ZERO_ON_ALLOC which enforces CPU_WRITEABLE
+ * flag on the allocations. */
+#define RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC)
+
+/* data content being kept from previous boot cycles from physical memory must not be cleared during allocation */
+#define RGX_AUTOVZ_KEEP_FW_DATA_MASK(bKeepMem) ((bKeepMem) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL))
+
+/******************************************************************************
+ * RGXSetFirmwareAddress Flags
+ *****************************************************************************/
+#define RFW_FWADDR_FLAG_NONE		(0)			/*!< Void flag */
+#define RFW_FWADDR_NOREF_FLAG		(1U << 0)	/*!< It is safe to immediately release the reference to the pointer,
+												  otherwise RGXUnsetFirmwareAddress() must be call when finished. */
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferIsInitRequired
+
+@Description    Returns true if the firmware trace buffer is not allocated and
+                might be required by the firmware soon. Trace buffer allocated
+                on-demand to reduce RAM footprint on systems not needing
+                firmware trace.
+
+@Input          psDevInfo RGX device info
+
+@Return         IMG_BOOL  Whether on-demand allocation(s) is/are needed or not
+*/ /**************************************************************************/
+FORCE_INLINE IMG_BOOL RGXTraceBufferIsInitRequired(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	RGXFwSharedMemCacheOpValue(psTraceBufCtl->ui32LogType, INVALIDATE);
+
+	/* The firmware expects a trace buffer only when:
+	 *	- Logtype is "trace" AND
+	 *	- at least one LogGroup is configured
+	 *	- the Driver Mode is not Guest
+	 */
+	if ((psDevInfo->psRGXFWIfTraceBufferMemDesc[0] == NULL)
+		&& (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)
+		&& (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+		&& !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo, PVRSRV_MEMALLOCFLAGS_T uiAllocFlags);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferIsInitRequired
+
+@Description    Returns true if the firmware tbi buffer is not allocated and
+                might be required by the firmware soon. TBI buffer allocated
+                on-demand to reduce RAM footprint on systems not needing
+                tbi.
+
+@Input          psDevInfo RGX device info
+
+@Return         IMG_BOOL  Whether on-demand allocation(s) is/are needed or not
+*/ /**************************************************************************/
+FORCE_INLINE IMG_BOOL RGXTBIBufferIsInitRequired(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	RGXFwSharedMemCacheOpValue(psTraceBufCtl->ui32LogType, INVALIDATE);
+
+	/* The firmware expects a tbi buffer only when:
+	 *	- Logtype is "tbi"
+	 */
+	if ((psDevInfo->psRGXFWIfTBIBufferMemDesc == NULL)
+		 && (psTraceBufCtl->ui32LogType & ~RGXFWIF_LOG_TYPE_TRACE)
+		 && (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                              RGX_INIT_APPHINTS        *psApphints,
+                              IMG_UINT32               ui32ConfigFlags,
+                              IMG_UINT32               ui32ConfigFlagsExt,
+                              IMG_UINT32               ui32FwOsCfgFlags);
+
+
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO   *psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags);
+
+/*************************************************************************/ /*!
+@Function       RGXSetFirmwareAddress
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          ppDest          Address of the pointer to set
+@Input          psSrc           MemDesc describing the pointer
+@Input          ui32Flags       Any combination of RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+								   DEVMEM_MEMDESC		*psSrc,
+								   IMG_UINT32			uiOffset,
+								   IMG_UINT32			ui32Flags);
+
+
+/*************************************************************************/ /*!
+@Function       RGXSetMetaDMAAddress
+
+@Description    Fills a Firmware structure used to setup the Meta DMA with two
+                pointers to the same data, one on 40 bit and one on 32 bit
+                (pointer in the FW memory space).
+
+@Input          ppDest          Address of the structure to set
+@Input          psSrcMemDesc    MemDesc describing the pointer
+@Input          psSrcFWDevVAddr Firmware memory space pointer
+
+@Return         void
+*/ /**************************************************************************/
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset);
+
+
+/*************************************************************************/ /*!
+@Function       RGXUnsetFirmwareAddress
+
+@Description    Unsets a pointer in a firmware data structure
+
+@Input          psSrc           MemDesc describing the pointer
+
+@Return         void
+*/ /**************************************************************************/
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc);
+
+/*!
+*******************************************************************************
+@Function       RGXScheduleProcessQueuesKM
+
+@Description    Software command complete handler
+                (sends uncounted kicks for all the DMs through the MISR)
+
+@Input          hCmdCompHandle  RGX device node
+
+@Return         None
+******************************************************************************/
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle);
+
+
+/*!
+*******************************************************************************
+
+@Function       RGXInstallProcessQueuesMISR
+
+@Description    Installs the MISR to handle Process Queues operations
+
+@Input          phMISR          Pointer to the MISR handler
+@Input          psDeviceNode    RGX Device node
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll);
+
+/*************************************************************************/ /*!
+@Function       RGXSendCommandWithPowLockAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM without honouring
+                pending cache operations but taking the power lock.
+
+@Input          psDevInfo       Device Info
+@Input          psKCCBCmd       The cmd to send.
+@Input          ui32PDumpFlags  Pdump flags
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32			*pui32CmdKCCBSlot);
+
+#define RGXSendCommandWithPowLock(psDevInfo, psKCCBCmd, ui32PDumpFlags) \
+  RGXSendCommandWithPowLockAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, NULL)
+
+/*************************************************************************/ /*!
+@Function       RGXSendCommandAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM without honouring
+                pending cache operations or the power lock.
+                The function flushes any deferred KCCB commands first.
+
+@Input          psDevInfo       Device Info
+@Input          psKCCBCmd       The cmd to send.
+@Input          uiPdumpFlags    PDump flags.
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+										  RGXFWIF_KCCB_CMD   *psKCCBCmd,
+										  PDUMP_FLAGS_T      uiPdumpFlags,
+										  IMG_UINT32         *pui32CmdKCCBSlot);
+
+#define RGXSendCommand(psDevInfo, psKCCBCmd, ui32PDumpFlags) \
+  RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, NULL)
+
+/*************************************************************************/ /*!
+@Function       _RGXScheduleCommandAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM and kicks the firmware but
+                first schedules any commands which have to happen before
+                handle
+
+@Input          psDevInfo          Device Info
+@Input          eDM                To which DM the cmd is sent.
+@Input          psKCCBCmd          The cmd to send.
+@Input          ui32PDumpFlags     PDump flags
+@Input          bCallerHasPwrLock  Caller already has power lock
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											   RGXFWIF_DM         eKCCBType,
+											   RGXFWIF_KCCB_CMD   *psKCCBCmd,
+											   IMG_UINT32         ui32PDumpFlags,
+											   IMG_UINT32         *pui32CmdKCCBSlot,
+											   IMG_BOOL           bCallerHasPwrLock);
+
+#define RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot) \
+	_RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot, IMG_FALSE)
+
+#define RGXScheduleCommand(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags) \
+  RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, NULL)
+
+#define RGXScheduleCommandWithoutPowerLock(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags) \
+	_RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, NULL, IMG_TRUE)
+
+/*************************************************************************/ /*!
+@Function       RGXWaitForKCCBSlotUpdate
+
+@Description    Waits until the required kCCB slot value is updated by the FW
+                (signifies command completion). Additionally, dumps a relevant
+                PDump poll command.
+
+@Input          psDevInfo       Device Info
+@Input          ui32SlotNum     The kCCB slot number to wait for an update on
+@Input          ui32PDumpFlags
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXFrameworkCopyCommand
+
+@Description    Copy framework command into FW addressable buffer
+
+@param          psDeviceNode
+@param          psFWFrameworkMemDesc
+@param          pbyGPUFRegisterList
+@param          ui32FrameworkRegisterSize
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+										   DEVMEM_MEMDESC   *psFWFrameworkMemDesc,
+										   IMG_PBYTE        pbyGPUFRegisterList,
+										   IMG_UINT32       ui32FrameworkRegisterSize);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXFrameworkCreateKM
+
+@Description    Create FW addressable buffer for framework
+
+@param          psDeviceNode
+@param          ppsFWFrameworkMemDesc
+@param          ui32FrameworkRegisterSize
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE * psDeviceNode,
+										DEVMEM_MEMDESC     ** ppsFWFrameworkMemDesc,
+										IMG_UINT32         ui32FrameworkRegisterSize);
+
+/*************************************************************************/ /*!
+@Function       RGXPollForGPCommandCompletion
+
+@Description    Polls for completion of a submitted GP command. Poll is done
+                on a value matching a masked read from the address.
+
+@Input          psDevNode       Pointer to device node struct
+@Input          pui32LinMemAddr CPU linear address to poll
+@Input          ui32Value       Required value
+@Input          ui32Mask        Mask
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE *psDevNode,
+									volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+									IMG_UINT32                   ui32Value,
+									IMG_UINT32                   ui32Mask);
+
+/*************************************************************************/ /*!
+@Function       RGXStateFlagCtrl
+
+@Description    Set and return FW internal state flags.
+
+@Input          psDevInfo       Device Info
+@Input          ui32Config      AppHint config flags
+@Output         pui32State      Current AppHint state flag configuration
+@Input          bSetNotClear    Set or clear the provided config flags
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32State,
+				IMG_BOOL bSetNotClear);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestCommonContextCleanUp
+
+@Description    Schedules a FW common context cleanup. The firmware doesn't
+                block waiting for the resource to become idle but rather
+                notifies the host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psServerCommonContext context to be cleaned up
+@Input          eDM             Data master, to which the cleanup command should
+                                be sent
+@Input          ui32PDumpFlags  PDump continuous flag
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestHWRTDataCleanUp
+
+@Description    Schedules a FW HWRTData memory cleanup. The firmware doesn't
+                block waiting for the resource to become idle but rather
+                notifies the host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psHWRTData      firmware address of the HWRTData for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+										 PRGXFWIF_HWRTDATA psHWRTData);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestFreeListCleanUp
+
+@Description    Schedules a FW FreeList cleanup. The firmware doesn't block
+                waiting for the resource to become idle but rather notifies the
+                host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psFWFreeList    firmware address of the FreeList for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDeviceNode,
+										 PRGXFWIF_FREELIST psFWFreeList);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestZSBufferCleanUp
+
+@Description    Schedules a FW ZS Buffer cleanup. The firmware doesn't block
+                waiting for the resource to become idle but rather notifies the
+                host that the resource is busy.
+
+@Input          psDevInfo       pointer to device node
+@Input          psFWZSBuffer    firmware address of the ZS Buffer for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer);
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetHCSDeadline
+
+@Description    Requests the Firmware to set a new Hard Context Switch timeout
+                deadline. Context switches that surpass that deadline cause the
+                system to kill the currently running workloads.
+
+@Input          psDeviceNode    pointer to device node
+@Input          ui32HCSDeadlineMs  The deadline in milliseconds.
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs);
+
+/*!
+*******************************************************************************
+@Function       RGXFWHealthCheckCmdInt
+
+@Description    Ping the firmware to check if it is responsive.
+
+@Input          psDevInfo          pointer to device info
+@Input          bCallerHasPwrLock  Caller already has power lock
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock);
+
+#define RGXFWHealthCheckCmd(psDevInfo)	\
+	RGXFWHealthCheckCmdInt(psDevInfo, IMG_FALSE)
+
+#define RGXFWHealthCheckCmdWithoutPowerLock(psDevInfo)	\
+	RGXFWHealthCheckCmdInt(psDevInfo, IMG_TRUE)
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetFwOsState
+
+@Description    Requests the Firmware to change the guest OS Online states.
+                This should be initiated by the VMM when a guest VM comes
+                online or goes offline. If offline, the FW offloads any current
+                resource from that DriverID. The request is repeated until the
+                FW has had time to free all the resources or has waited for
+                workloads to finish.
+
+@Input          psDevInfo       pointer to device info
+@Input          ui32DriverID    The driver whose state is being altered
+@Input          eOSOnlineState  The new state (Online or Offline)
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32DriverID,
+								RGXFWIF_OS_STATE_CHANGE eOSOnlineState);
+
+#if defined(SUPPORT_AUTOVZ)
+/*!
+*******************************************************************************
+@Function       RGXUpdateAutoVzWdgToken
+
+@Description    If the driver-firmware connection is active, read the
+                firmware's watchdog token and copy its value back into the OS
+                token. This indicates to the firmware that this driver is alive
+                and responsive.
+
+@Input          psDevInfo       pointer to device info
+******************************************************************************/
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXDisconnectAllGuests
+
+@Description    Send requests to FW to disconnect all guest connections.
+
+@Input          psDeviceNode    pointer to device node
+******************************************************************************/
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode);
+#endif
+
+/*!
+*******************************************************************************
+@Function       RGXFWConfigPHR
+
+@Description    Configure the Periodic Hardware Reset functionality
+
+@Input          psDevInfo       pointer to device info
+@Input          ui32PHRMode     desired PHR mode
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode);
+
+/*!
+*******************************************************************************
+@Function       RGXFWConfigWdg
+
+@Description    Configure the Safety watchdog trigger period
+
+@Input          psDevInfo        pointer to device info
+@Input          ui32WdgPeriodUs  requested period in microseconds
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32WdgPeriod);
+
+/*!
+*******************************************************************************
+@Function       RGXCheckFirmwareCCB
+
+@Description    Processes all commands that are found in the Firmware CCB.
+
+@Input          psDevInfo       pointer to device
+
+@Return         None
+******************************************************************************/
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXCheckForStalledClientContexts
+
+@Description    Checks all client contexts, for the device with device info
+                provided, to see if any are waiting for a fence to signal and
+                optionally force signalling of the fence for the context which
+                has been waiting the longest.
+                This function is called by RGXUpdateHealthStatus() and also
+                may be invoked from other trigger points.
+
+@Input          psDevInfo       pointer to device info
+@Input          bIgnorePrevious If IMG_TRUE, any stalled contexts will be
+                                indicated immediately, rather than only
+                                checking against any previous stalled contexts
+
+@Return         None
+******************************************************************************/
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious);
+
+/*!
+*******************************************************************************
+@Function       RGXUpdateHealthStatus
+
+@Description    Tests a number of conditions which might indicate a fatal error
+                has occurred in the firmware. The result is stored in the
+                device node eHealthStatus.
+
+@Input         psDevNode        Pointer to device node structure.
+@Input         bCheckAfterTimePassed  When TRUE, the function will also test
+                                for firmware queues and polls not changing
+                                since the previous test.
+
+                                Note: if not enough time has passed since the
+                                last call, false positives may occur.
+
+@Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed);
+
+#if defined(SUPPORT_AUTOVZ)
+/*!
+*******************************************************************************
+@Function       RGXUpdateAutoVzWatchdog
+
+@Description    Updates AutoVz watchdog that maintains the fw-driver connection
+
+@Input         psDevNode        Pointer to device node structure.
+******************************************************************************/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode);
+#endif /* SUPPORT_AUTOVZ */
+
+/*!
+*******************************************************************************
+@Function       AttachKickResourcesCleanupCtls
+
+@Description    Attaches the cleanup structures to a kick command so that
+                submission reference counting can be performed when the
+                firmware processes the command
+
+@Output         apsCleanupCtl   Array of CleanupCtl structure pointers to populate.
+@Output         pui32NumCleanupCtl  Number of CleanupCtl structure pointers written out.
+@Input          eDM             Which data master is the subject of the command.
+@Input          bKick           TRUE if the client originally wanted to kick this DM.
+@Input          psRTDataCleanup Optional RTData cleanup associated with the command.
+@Input          psZBuffer       Optional ZSBuffer associated with the command.
+
+@Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer);
+
+/*!
+*******************************************************************************
+@Function       RGXResetHWRLogs
+
+@Description    Resets the HWR Logs buffer
+                (the hardware recovery count is not reset)
+
+@Input          psDevNode       Pointer to the device
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*!
+*******************************************************************************
+@Function       RGXGetPhyAddr
+
+@Description    Get the physical address of a PMR at an offset within it
+
+@Input          psPMR           PMR of the allocation
+@Input          ui32LogicalOffset  Logical offset
+
+@Output         psPhyAddr       Physical address of the allocation
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid);
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+@Function       RGXPdumpDrainKCCB
+
+@Description    Wait for the firmware to execute all the commands in the kCCB
+
+@Input          psDevInfo       Pointer to the device
+@Input          ui32WriteOffset Woff we have to POL for the Roff to be equal to
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+							   IMG_UINT32 ui32WriteOffset);
+#endif /* PDUMP */
+
+/*!
+*******************************************************************************
+@Function       RGXFwRawHeapAllocMap
+
+@Description    Register and maps to device, a raw firmware physheap
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize);
+
+/*!
+*******************************************************************************
+@Function       RGXFwRawHeapUnmapFree
+
+@Description    Unregister and unmap from device, a raw firmware physheap
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID);
+
+/*!
+*******************************************************************************
+@Function       RGXReadFWModuleAddr
+
+@Description    Read a value at the given address in META or RISCV memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in META or RISCV memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                 IMG_UINT32 ui32Addr,
+                                 IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXWriteFWModuleAddr
+
+@Description    Write a value to the given address in META or RISC memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32 ui32MemAddr,
+                                  IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXGetFwMapping
+
+@Description    Retrieve any of the CPU Physical Address, Device Physical
+                Address or the raw value of the page table entry associated
+                with the firmware virtual address given.
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32FwVA        The Fw VA that needs decoding
+@Output         psCpuPA         Pointer to the resulting CPU PA
+@Output         psDevPA         Pointer to the resulting Dev PA
+@Output         pui64RawPTE     Pointer to the raw Page Table Entry value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                    IMG_UINT32 ui32FwVA,
+                                    IMG_CPU_PHYADDR *psCpuPA,
+                                    IMG_DEV_PHYADDR *psDevPA,
+                                    IMG_UINT64 *pui64RawPTE);
+
+/*!
+*******************************************************************************
+@Function       RGXFWInjectFault
+
+@Description    Injecting firmware fault to validate recovery through Host
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetVzConnectionCooldownPeriod
+
+@Description    Set Vz connection cooldown period
+
+@Input          psDevInfo                              pointer to device info
+@Input          ui32VzConnectionCooldownPeriodInSec    Cooldown period in secs
+******************************************************************************/
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(SUPPORT_AUTOVZ_HW_REGS) && !defined(SUPPORT_AUTOVZ)
+#error "VZ build configuration error: use of OS scratch registers supported only in AutoVz drivers."
+#endif
+
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+/* AutoVz with hw support */
+#define KM_GET_FW_CONNECTION(psDevInfo)				OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH3)
+#define KM_GET_OS_CONNECTION(psDevInfo)				OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH2)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH2, RGXFW_CONNECTION_OS_##val)
+
+#define KM_GET_FW_ALIVE_TOKEN(psDevInfo)			OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH1)
+#define KM_GET_OS_ALIVE_TOKEN(psDevInfo)			OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH0)
+#define KM_SET_OS_ALIVE_TOKEN(val, psDevInfo)		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH0, val)
+
+#define KM_ALIVE_TOKEN_CACHEOP(Target, CacheOp)
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)
+
+#else
+
+#if defined(SUPPORT_AUTOVZ)
+#define KM_GET_FW_ALIVE_TOKEN(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->ui32AliveFwToken)
+#define KM_GET_OS_ALIVE_TOKEN(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->ui32AliveOsToken)
+#define KM_SET_OS_ALIVE_TOKEN(val, psDevInfo)		do { \
+                                                    OSWriteDeviceMem32WithWMB((volatile IMG_UINT32 *) &psDevInfo->psRGXFWIfConnectionCtl->ui32AliveOsToken, val); \
+                                                    KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH); \
+                                                    } while (0)
+
+#define KM_ALIVE_TOKEN_CACHEOP(Target, CacheOp)     RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfConnectionCtl->ui32Alive##Target##Token, \
+													                           CacheOp);
+#endif /* defined(SUPPORT_AUTOVZ) */
+
+#if !defined(NO_HARDWARE) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* static, dynamic and AutoVz DDKs using shared memory */
+#define KM_GET_FW_CONNECTION(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->eConnectionFwState)
+#define KM_GET_OS_CONNECTION(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->eConnectionOsState)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)	do { \
+                                                OSWriteDeviceMem32WithWMB((void*)&psDevInfo->psRGXFWIfConnectionCtl->eConnectionOsState, RGXFW_CONNECTION_OS_##val); \
+                                                KM_CONNECTION_CACHEOP(Os, FLUSH); \
+                                                } while (0)
+
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)  RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfConnectionCtl->eConnection##Target##State, \
+												                           CacheOp);
+#else
+/* nohw & native */
+#define KM_GET_FW_CONNECTION(psDevInfo)			(RGXFW_CONNECTION_FW_ACTIVE)
+#define KM_GET_OS_CONNECTION(psDevInfo)			(RGXFW_CONNECTION_OS_ACTIVE)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED == 1) */
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+#define RGX_FIRST_RAW_HEAP_DRIVER_ID		RGXFW_HOST_DRIVER_ID
+#else
+#define RGX_FIRST_RAW_HEAP_DRIVER_ID		RGXFW_GUEST_DRIVER_ID_START
+#endif
+
+#define KM_OS_CONNECTION_IS(val, psDevInfo)		(KM_GET_OS_CONNECTION(psDevInfo) == RGXFW_CONNECTION_OS_##val)
+#define KM_FW_CONNECTION_IS(val, psDevInfo)		(KM_GET_FW_CONNECTION(psDevInfo) == RGXFW_CONNECTION_FW_##val)
+
+#endif /* RGXFWUTILS_H */
+/******************************************************************************
+ End of file (rgxfwutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf.h
new file mode 100644
index 000000000000..06d3c3ae0487
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf.h
@@ -0,0 +1,109 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX HWPerf functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHWPERF_H_
+#define RGXHWPERF_H_
+
+#include "rgxhwperf_common.h"
+
+/******************************************************************************
+ * RGX HW Performance Profiling API(s)
+ *****************************************************************************/
+
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+PVRSRV_ERROR PVRSRVRGXConfigMuxHWPerfCountersKM(
+	CONNECTION_DATA               *psConnection,
+	PVRSRV_DEVICE_NODE            *psDeviceNode,
+	IMG_UINT32                     ui32ArrayLen,
+	RGX_HWPERF_CONFIG_MUX_CNTBLK  *psBlockConfigs);
+
+
+PVRSRV_ERROR PVRSRVRGXConfigCustomCountersKM(
+	CONNECTION_DATA    * psConnection,
+	PVRSRV_DEVICE_NODE * psDeviceNode,
+	IMG_UINT16           ui16CustomBlockID,
+	IMG_UINT16           ui16NumCustomCounters,
+	IMG_UINT32         * pui32CustomCounterIDs);
+#endif
+
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+	CONNECTION_DATA       * psConnection,
+	PVRSRV_DEVICE_NODE    * psDeviceNode,
+	IMG_UINT32            ui32CtrlWord,
+	IMG_UINT32            ui32ArrayLen,
+	RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs);
+
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(CONNECTION_DATA *psConnection,
+                                                       PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                       const IMG_UINT32 ui32BlockID,
+                                                       RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                     RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                     IMG_UINT32 ui32BlockID,
+                                                     RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters);
+#endif
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLength,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs);
+
+typedef struct HWPERF_STREAM_DESC_TAG HWPERF_STREAM_DESC;
+
+PVRSRV_ERROR PVRSRVRGXOpenHWPerfClientStreamKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               HWPERF_STREAM_DESC **ppsSD);
+PVRSRV_ERROR PVRSRVRGXCloseHWPerfClientStreamKM(HWPERF_STREAM_DESC *psSD);
+PVRSRV_ERROR PVRSRVRGXWriteHWPerfClientEventKM(HWPERF_STREAM_DESC *psSD,
+                                               IMG_UINT32 uiSize,
+                                               IMG_BYTE *puiData);
+
+#endif /* RGXHWPERF_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.c
new file mode 100644
index 000000000000..b440560f2141
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.c
@@ -0,0 +1,4318 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+#include "tlserver.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+#include "ospvr_gputrace.h"
+#endif
+
+#include "info_page_defs.h"
+
+/* This is defined by default to enable producer callbacks.
+ * Clients of the TL interface can disable the use of the callback
+ * with PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK. */
+#define SUPPORT_TL_PRODUCER_CALLBACK 1
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+/* Defines size of buffers returned from acquire/release calls */
+#define FW_STREAM_BUFFER_SIZE (0x80000)
+#define HOST_STREAM_BUFFER_SIZE (0x20000)
+
+/* Must be at least as large as two tl packets of maximum size */
+static_assert(HOST_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "HOST_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+static_assert(FW_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "FW_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+
+IMG_INTERNAL /*static inline*/ IMG_UINT32 RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **);
+static IMG_UINT64 RGXHWPerfFwSetEventFilterNoLock(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                                  IMG_UINT64 uiFilter);
+
+static inline IMG_UINT32
+RGXHWPerfGetPackets(IMG_UINT32  ui32BytesExp,
+                    IMG_UINT32  ui32AllowedSize,
+                    RGX_PHWPERF_V2_PACKET_HDR psCurPkt )
+{
+	IMG_UINT32 sizeSum = 0;
+	RGXFwSharedMemCacheOpValue(psCurPkt->ui32Size, INVALIDATE);
+
+	/* Traverse the array to find how many packets will fit in the available space. */
+	while ( sizeSum + RGX_HWPERF_GET_SIZE(psCurPkt) <= ui32BytesExp &&
+			sizeSum + RGX_HWPERF_GET_SIZE(psCurPkt) < ui32AllowedSize )
+	{
+		sizeSum += RGX_HWPERF_GET_SIZE(psCurPkt);
+		psCurPkt = RGX_HWPERF_GET_NEXT_PACKET(psCurPkt);
+		RGXFwSharedMemCacheOpValue(psCurPkt->ui32Size, INVALIDATE);
+	}
+
+	return sizeSum;
+}
+
+static inline void
+RGXSuspendHWPerfL2DataCopy(PVRSRV_RGXDEV_INFO* psDeviceInfo,
+                           RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                           IMG_BOOL bIsReaderConnected)
+{
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	if (!bIsReaderConnected)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s : HWPerf FW events enabled but L2 host buffer "
+		        "for stream %u is full and no reader is currently connected, suspending "
+		        "event collection. Connect a reader or restart driver to avoid event loss.",
+		        __func__, eL2StreamId));
+		psDeviceInfo->bSuspendHWPerfL2DataCopy[eL2StreamId] = IMG_TRUE;
+	}
+}
+
+static IMG_UINT32 RGXHWPerfCopyData(PVRSRV_RGXDEV_INFO *psDeviceInfo,
+                                    IMG_BYTE *pbSrcBuffer,
+                                    RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                    IMG_UINT32 uiBytesToCopy)
+{
+	IMG_BYTE *pbDestBuffer;
+	IMG_UINT32 uiBytesCopied = 0;
+	IMG_UINT32 uiFreeSpace;
+	IMG_UINT32 uiBytesToCopyMin = RGX_HWPERF_GET_SIZE(RGX_HWPERF_GET_PACKET(pbSrcBuffer));
+	IMG_BOOL bIsReaderConnected;
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hHWPerfDestStream;
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	hHWPerfDestStream = psDeviceInfo->hHWPerfStream[eL2StreamId];
+
+	PVR_DPF_ENTERED;
+
+	/* Try submitting all data in one TL packet. */
+	eError = TLStreamReserve2(hHWPerfDestStream, &pbDestBuffer, uiBytesToCopy, uiBytesToCopyMin,
+	                          &uiFreeSpace, &bIsReaderConnected);
+	if (eError == PVRSRV_OK)
+	{
+		RGXFwSharedMemCacheOpExec(pbSrcBuffer, uiBytesToCopy, PVRSRV_CACHE_OP_INVALIDATE);
+		OSDeviceMemCopy(pbDestBuffer, pbSrcBuffer, (size_t) uiBytesToCopy);
+
+		eError = TLStreamCommit(hHWPerfDestStream, uiBytesToCopy);
+		PVR_LOG_GOTO_IF_ERROR_VA(eError, ErrReturn, "TLStreamCommit() failed with error %d, "
+		                         "unable to copy packet from L1 to L2 buffer", eError);
+
+		/* Data were successfully written */
+		uiBytesCopied = (size_t) uiBytesToCopy;
+	}
+	else if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		/* There was not enough space for all data, copy as much as possible */
+		IMG_UINT32 uiSizeSum = RGXHWPerfGetPackets(uiBytesToCopy, uiFreeSpace,
+		                                           RGX_HWPERF_GET_PACKET(pbSrcBuffer));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Unable to reserve space (%d) in host buffer on first attempt, "
+		         "remaining free space: %d", uiBytesToCopy, uiFreeSpace));
+
+		if (uiSizeSum != 0)
+		{
+			eError = TLStreamReserve(hHWPerfDestStream, &pbDestBuffer, uiSizeSum);
+
+			if (eError == PVRSRV_OK)
+			{
+				RGXFwSharedMemCacheOpExec(pbSrcBuffer, uiSizeSum, PVRSRV_CACHE_OP_INVALIDATE);
+				OSDeviceMemCopy(pbDestBuffer, pbSrcBuffer, (size_t) uiSizeSum);
+
+				eError = TLStreamCommit(hHWPerfDestStream, uiSizeSum);
+				PVR_LOG_GOTO_IF_ERROR_VA(eError, ErrReturn, "TLStreamCommit() failed with error "
+				                         "%d, unable to copy packet from L1 to L2 buffer", eError);
+
+				/* uiSizeSum bytes of hwperf packets have been successfully written */
+				uiBytesCopied = uiSizeSum;
+			}
+			else if (eError == PVRSRV_ERROR_STREAM_FULL)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "Cannot write HWPerf packet into host buffer, check data "
+				         "in case of packet loss, remaining free space: %d", uiFreeSpace));
+				RGXSuspendHWPerfL2DataCopy(psDeviceInfo, eL2StreamId, bIsReaderConnected);
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "Cannot find space in host buffer, check data in case of "
+			         "packet loss, remaining free space: %d", uiFreeSpace));
+			RGXSuspendHWPerfL2DataCopy(psDeviceInfo, eL2StreamId, bIsReaderConnected);
+		}
+	}
+
+	/* Some other error occurred. Full error handled by caller, we returning the copied bytes count
+	 * to caller */
+	if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_STREAM_FULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "HWPerf enabled: Unexpected Error (%d) while copying FW buffer "
+		         "to destination buffer.", eError));
+	}
+
+ErrReturn:
+	/* Return the remaining packets left to be transported. */
+	PVR_DPF_RETURN_VAL(uiBytesCopied);
+}
+
+/*
+  RGXHWPerfGetMaxTransfer
+ */
+static IMG_UINT32 RGXHWPerfGetMaxTransfer(PVRSRV_RGXDEV_INFO *psDeviceInfo,
+                                          IMG_UINT32 ui32BytesExp,
+                                          IMG_UINT32 uiL2StreamCopyMask)
+{
+	IMG_UINT32 uiMaxXfer = ui32BytesExp;
+	IMG_UINT32 eL2StreamId;
+
+	for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+	{
+		if (BIT_ISSET(uiL2StreamCopyMask, eL2StreamId))
+		{
+			IMG_UINT32 uiMaxXferSize = TLStreamGetMaxTransfer(
+			                             ui32BytesExp,
+			                             psDeviceInfo->hHWPerfStream[eL2StreamId]);
+
+			if (uiMaxXferSize < uiMaxXfer)
+			{
+				/* New minimum size found, save it for later */
+#if defined(DEBUG)
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s(dev %u, len %u, mask %u) New/Old = [0x%x/0x%x]",
+				         __func__, eL2StreamId, ui32BytesExp,
+				         uiL2StreamCopyMask, uiMaxXferSize, uiMaxXfer));
+#endif
+
+				uiMaxXfer = uiMaxXferSize;
+			}
+#if defined(DEBUG)
+			else
+			{
+				PVR_DPF((PVR_DBG_VERBOSE,
+				         "%s(dev %u, len %u, mask %u) New/Old = [0x%x/0x%x]",
+				         __func__, eL2StreamId, ui32BytesExp,
+				         uiL2StreamCopyMask, uiMaxXferSize, uiMaxXfer));
+			}
+#endif
+		}
+	}
+
+	return uiMaxXfer;
+}
+
+/*
+	RGXHWPerfCopyDataL1toL2
+ */
+static IMG_UINT32 RGXHWPerfCopyDataL1toL2(PVRSRV_RGXDEV_INFO* psDeviceInfo,
+                                          IMG_BYTE   *pbFwBuffer,
+                                          IMG_UINT32 uiL1DataSize,
+                                          IMG_UINT32 uiL2StreamCopyMask)
+{
+	IMG_UINT32 eL2StreamId, ui32L2AvailableSpace, uiHWPerfBytesCopied = 0;
+
+	/* HWPERF_MISR_FUNC_DEBUG enables debug code for investigating HWPerf issues */
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	static IMG_UINT32 gui32Ordinal = IMG_UINT32_MAX;
+#endif
+
+	/* Invalidate initial packet header, type/size cast via RGX_HWPEF_GET_PACKET */
+	RGXFwSharedMemCacheOpPtr(RGX_HWPERF_GET_PACKET(pbFwBuffer), INVALIDATE);
+
+	PVR_DPF_ENTERED;
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	PVR_DPF((PVR_DBG_VERBOSE, "EVENTS to copy from 0x%p length:%05d mask:0x%2x",
+	         pbFwBuffer, uiL1DataSize, uiL2StreamCopyMask));
+#endif
+
+	/* Determine the maximum space available in all consumer (L2) streams.
+	 * This limits the amount of data that will be copied if we have multiple
+	 * L2 consumers registered. In this case we only transfer the amount of
+	 * data that can fit into all available L2 consumer streams and so we will
+	 * exert back-pressure onto the L1 buffer whenever one of the L2 consumer
+	 * streams gets filled.
+	 * This effectively lock-steps the L2 consumer streams together.
+	 */
+	ui32L2AvailableSpace = RGXHWPerfGetMaxTransfer(psDeviceInfo, uiL1DataSize, uiL2StreamCopyMask);
+
+	for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+	{
+		if (BIT_ISSET(uiL2StreamCopyMask, eL2StreamId))
+		{
+			IMG_UINT32 uiBytesCopied, uiPacketDataSize = ui32L2AvailableSpace;
+			IMG_UINT32 uiL2MaxPacketSize = psDeviceInfo->ui32L2BufMaxPacketSize[eL2StreamId];
+
+			/* Adjust the copied data size if the maximum packet size is smaller
+			 * than the available space or if the L1 data we want to read can't
+			 * fit into the available space. */
+			if (ui32L2AvailableSpace > uiL2MaxPacketSize || ui32L2AvailableSpace < uiL1DataSize)
+			{
+				uiPacketDataSize = RGXHWPerfGetPackets(ui32L2AvailableSpace, uiL2MaxPacketSize,
+				                                       RGX_HWPERF_GET_PACKET(pbFwBuffer));
+
+				if (uiPacketDataSize == 0)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Failed to write data into host buffer "
+					        "(%u) as packet is too big (%u) and can't fit into available "
+					        "space (%u)", eL2StreamId, uiL1DataSize,
+					        MIN(ui32L2AvailableSpace, uiL2MaxPacketSize)));
+
+					continue;
+				}
+			}
+
+			uiBytesCopied = RGXHWPerfCopyData(psDeviceInfo, pbFwBuffer, eL2StreamId,
+			                                  uiPacketDataSize);
+
+			uiHWPerfBytesCopied = MAX(uiBytesCopied, uiHWPerfBytesCopied);
+		}
+	}
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	if (uiHWPerfBytesCopied > 0)
+	{
+		/* Check the incoming buffer of data has not lost any packets */
+		IMG_BYTE *pbFwBufferIter = pbFwBuffer;
+		IMG_BYTE *pbFwBufferEnd = pbFwBuffer + uiHWPerfBytesCopied;
+
+		do
+		{
+			RGX_HWPERF_V2_PACKET_HDR *asCurPos = RGX_HWPERF_GET_PACKET(pbFwBufferIter);
+			IMG_UINT32 ui32CurOrdinal;
+			/* Invalidate HDR pointed to by asCurPos as we use both ordinal for detecting
+			 * lost packets and size for iteration.
+			 */
+			RGXFwSharedMemCacheOpPtr(asCurPos, INVALIDATE);
+			ui32CurOrdinal = asCurPos->ui32Ordinal;
+			if (gui32Ordinal != IMG_UINT32_MAX)
+			{
+				if ((gui32Ordinal+1) != ui32CurOrdinal)
+				{
+					if (gui32Ordinal < ui32CurOrdinal)
+					{
+						PVR_DPF((PVR_DBG_WARNING,
+								"HWPerf [%p] packets lost (%u packets) between ordinal %u...%u",
+								pbFwBufferIter,
+								ui32CurOrdinal - gui32Ordinal - 1,
+								gui32Ordinal,
+								ui32CurOrdinal));
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_WARNING,
+								"HWPerf [%p] packet ordinal out of sequence last: %u, current: %u",
+								pbFwBufferIter,
+								gui32Ordinal,
+								ui32CurOrdinal));
+					}
+				}
+			}
+			gui32Ordinal = asCurPos->ui32Ordinal;
+			pbFwBufferIter += RGX_HWPERF_GET_SIZE(asCurPos);
+		} while (pbFwBufferIter < pbFwBufferEnd);
+	}
+#endif
+
+	/* Return the remaining packets left to be transported. */
+	PVR_DPF_RETURN_VAL(uiHWPerfBytesCopied);
+}
+
+
+static INLINE IMG_UINT32 RGXHWPerfAdvanceRIdx(
+		const IMG_UINT32 ui32BufSize,
+		const IMG_UINT32 ui32Pos,
+		const IMG_UINT32 ui32Size)
+{
+	return ( ui32Pos + ui32Size < ui32BufSize ? ui32Pos + ui32Size : 0 );
+}
+
+
+/*
+	RGXHWPerfDataStore
+
+	This function copies HWPerf data from L1 buffer to all L2 streams.
+	The number of copied data is always the maximum read number of packets.
+	In case where one of the stream is not able to accept the same amount of
+	data as other streams it will suffer from gaps in the data.
+
+	To avoid these gaps, we will exert back-pressure on the L1 buffer by
+	pre-calculating the maximum amount of data that can be copied to all
+	L2 streams (currently a maximum of 2).
+ */
+static IMG_UINT32 RGXHWPerfDataStore(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFWIF_SYSDATA	*psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	IMG_BYTE*				psHwPerfInfo = psDevInfo->psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32SrcRIdx, ui32SrcWIdx, ui32SrcWrapCount;
+	IMG_UINT32				ui32BytesExp = 0, ui32BytesCopied = 0, ui32BytesCopiedSum = 0;
+	IMG_UINT32				uiStreamCopyMask = 0;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	IMG_UINT32				ui32BytesExpSum = 0;
+#endif
+
+	/* It's unlikely that we're ever going to have more than 32 consumers
+	 * for the Firmware L1 buffer but check just to be safe. */
+	static_assert(RGX_HWPERF_L2_STREAM_LAST <= sizeof(uiStreamCopyMask) * 8,
+	              "RGX_HWPERF_L2_STREAM_LAST cannot be greater than 32.")
+
+	PVR_DPF_ENTERED;
+
+	/* Caller should check this member is valid before calling */
+	{
+		RGX_HWPERF_L2_STREAM_ID eL2StreamId;
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+		IMG_UINT32 uiNotNullCount = 0;
+#endif
+		for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+		{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+			if (psDevInfo->hHWPerfStream[eL2StreamId] != NULL)
+			{
+				uiNotNullCount++;
+			}
+#endif
+			if (!psDevInfo->bSuspendHWPerfL2DataCopy[eL2StreamId])
+			{
+				BIT_SET(uiStreamCopyMask, eL2StreamId);
+			}
+		}
+
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+		/* At least one stream must exist. */
+		PVR_ASSERT(uiNotNullCount > 0);
+#endif
+
+		/* Only proceed if any of the streams are not suspended.
+		 * Build bit field representing each L2 stream's suspend status. This
+		 * will be passed down the stack for copy function to determine if the
+		 * data should or should not be copied. */
+		if (uiStreamCopyMask == 0)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s : Copying data to all L2 host buffers for FW events is "
+			        "suspended. Start at least one of the HWPerf consumers or restart the driver "
+			        "if HWPerf FW events are needed", __func__));
+
+			PVR_DPF_RETURN_VAL(0);
+		}
+	}
+
+	/* Invalidate partial region of struct */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl,
+	                           INVALIDATE);
+
+	/* Get a copy of the current
+	 *   read (first packet to read)
+	 *   write (empty location for the next write to be inserted)
+	 *   WrapCount (size in bytes of the buffer at or past end)
+	 * indexes of the FW buffer */
+	ui32SrcRIdx = psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx;
+	ui32SrcWIdx = psFwSysData->sHWPerfCtrl.ui32HWPerfWIdx;
+	OSMemoryBarrier(NULL);
+	ui32SrcWrapCount = psFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount;
+
+#if defined(HWPERF_MISR_FUNC_DEBUG) || defined(EMULATOR)
+	{
+		IMG_UINT32  ui32SrcBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+
+		if (ui32SrcRIdx >= ui32SrcBufSize || ui32SrcWIdx >= ui32SrcBufSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s : Invalid read/write offsets found! srcRIdx:%u srcWIdx:%u srcBufSize:%u",
+			        __func__, ui32SrcRIdx, ui32SrcWIdx, ui32SrcBufSize));
+
+			PVR_DPF_RETURN_VAL(ui32BytesCopiedSum);
+		}
+	}
+#endif
+
+	/* Is there any data in the buffer not yet retrieved? */
+	if ( ui32SrcRIdx != ui32SrcWIdx )
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfDataStore EVENTS found srcRIdx:%d srcWIdx: %d", ui32SrcRIdx, ui32SrcWIdx));
+
+		/* Is the write position higher than the read position? */
+		if ( ui32SrcWIdx > ui32SrcRIdx )
+		{
+			/* Yes, buffer has not wrapped */
+			ui32BytesExp = ui32SrcWIdx - ui32SrcRIdx;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+			ui32BytesExpSum += ui32BytesExp;
+#endif
+			ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+			                                          psHwPerfInfo + ui32SrcRIdx,
+			                                          ui32BytesExp,
+			                                          uiStreamCopyMask);
+
+			/* Advance the read index and the free bytes counter by the number
+			 * of bytes transported. Items will be left in buffer if not all data
+			 * could be transported. Exit to allow buffer to drain. */
+			OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                          RGXHWPerfAdvanceRIdx(psDevInfo->ui32RGXFWIfHWPerfBufSize,
+			                                               ui32SrcRIdx,
+			                                               ui32BytesCopied));
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                           FLUSH);
+
+			ui32BytesCopiedSum += ui32BytesCopied;
+		}
+		/* No, buffer has wrapped and write position is behind read position */
+		else
+		{
+			/* Byte count equal to
+			 *     number of bytes from read position to the end of the buffer,
+			 *   + data in the extra space in the end of the buffer. */
+			ui32BytesExp = ui32SrcWrapCount - ui32SrcRIdx;
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+			ui32BytesExpSum += ui32BytesExp;
+#endif
+			ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+			                                          psHwPerfInfo + ui32SrcRIdx,
+			                                          ui32BytesExp,
+			                                          uiStreamCopyMask);
+
+			/* Advance read index as before and Update the local copy of the
+			 * read index as it might be used in the last if branch*/
+			ui32SrcRIdx = RGXHWPerfAdvanceRIdx(
+					psDevInfo->ui32RGXFWIfHWPerfBufSize, ui32SrcRIdx,
+					ui32BytesCopied);
+
+			/* Update Wrap Count */
+			if ( ui32SrcRIdx == 0)
+			{
+				OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount,
+				                          psDevInfo->ui32RGXFWIfHWPerfBufSize);
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount,
+				                           FLUSH);
+			}
+			OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx, ui32SrcRIdx);
+
+			ui32BytesCopiedSum += ui32BytesCopied;
+
+			/* If all the data in the end of the array was copied, try copying
+			 * wrapped data in the beginning of the array, assuming there is
+			 * any and the RIdx was wrapped. */
+			if (   (ui32BytesCopied == ui32BytesExp)
+					&& (ui32SrcWIdx > 0)
+					&& (ui32SrcRIdx == 0) )
+			{
+				ui32BytesExp = ui32SrcWIdx;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+				ui32BytesExpSum += ui32BytesExp;
+#endif
+				ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+				                                          psHwPerfInfo,
+				                                          ui32BytesExp,
+				                                          uiStreamCopyMask);
+				/* Advance the FW buffer read position. */
+				psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = RGXHWPerfAdvanceRIdx(
+						psDevInfo->ui32RGXFWIfHWPerfBufSize, ui32SrcRIdx,
+						ui32BytesCopied);
+
+				ui32BytesCopiedSum += ui32BytesCopied;
+			}
+			/* This flush covers both writes above */
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                           FLUSH);
+		}
+#ifdef HWPERF_MISR_FUNC_DEBUG
+		if (ui32BytesCopiedSum != ui32BytesExpSum)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfDataStore: FW L1 RIdx:%u. Not all bytes copied to L2: %u bytes out of %u expected", psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx, ui32BytesCopiedSum, ui32BytesExpSum));
+		}
+#endif
+
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_VERBOSE, "RGXHWPerfDataStore NO EVENTS to transport"));
+	}
+
+	PVR_DPF_RETURN_VAL(ui32BytesCopiedSum);
+}
+
+/* Function called from MISR to copy data from L1 buffer to L2 streams. */
+PVRSRV_ERROR RGXHWPerfDataStoreCB(PVRSRV_DEVICE_NODE *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+	IMG_UINT32          ui32BytesCopied;
+
+	PVR_ASSERT(psDevInfo);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevInfo, PVRSRV_OK);
+
+	PVR_DPF_ENTERED;
+
+	psRgxDevInfo = psDevInfo->pvDevice;
+
+	/* Store FW event data if the destination buffer exists.*/
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->uiHWPerfStreamCount > 0)
+	{
+		ui32BytesCopied = RGXHWPerfDataStore(psRgxDevInfo);
+		if ( ui32BytesCopied )
+		{
+			/* It's possible that the HWPerf stream doesn't exist yet. It's
+			 * possible that only FTrace L2 stream has been created so far. */
+			if (psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] != NULL)
+			{
+				/* Signal consumers that packets may be available to read when
+				 * running from a HW kick, not when called by client APP thread
+				 * via the transport layer CB as this can lead to stream
+				 * corruption. */
+				eError = TLStreamSync(psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF]);
+				PVR_LOG_IF_ERROR(eError, "TLStreamSync");
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfDataStoreCB: Zero bytes copied"));
+			RGXDEBUG_PRINT_IRQ_COUNT(psRgxDevInfo);
+		}
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+/* Currently supported by default */
+#if !defined(NO_HARDWARE) && defined(SUPPORT_TL_PRODUCER_CALLBACK)
+static PVRSRV_ERROR RGXHWPerfTLCB(IMG_HANDLE hStream,
+                                  IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)pvUser;
+
+	PVR_UNREFERENCED_PARAMETER(hStream);
+	PVR_UNREFERENCED_PARAMETER(ui32Resp);
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	switch (ui32ReqOp)
+	{
+		case TL_SOURCECB_OP_CLIENT_EOS:
+			/* Keep HWPerf resource init check and use of
+			 * resources atomic, they may not be freed during use
+			 */
+
+			/* This solution is for avoiding a deadlock situation where -
+			 * in DoTLStreamReserve(), writer has acquired HWPerfLock and
+			 * ReadLock and is waiting on ReadPending (which will be reset
+			 * by reader), And
+			 * the reader after setting ReadPending in TLStreamAcquireReadPos(),
+			 * is waiting for HWPerfLock in RGXHWPerfTLCB().
+			 * So here in RGXHWPerfTLCB(), if HWPerfLock is already acquired we
+			 * will return to the reader without waiting to acquire HWPerfLock.
+			 */
+			if (!OSTryLockAcquire(psRgxDevInfo->hHWPerfLock))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "hHWPerfLock is already acquired, a write "
+						"operation might already be in process"));
+				return PVRSRV_OK;
+			}
+
+			if (psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] != NULL)
+			{
+				(void) RGXHWPerfDataStore(psRgxDevInfo);
+			}
+			OSLockRelease(psRgxDevInfo->hHWPerfLock);
+			break;
+
+		default:
+			break;
+	}
+
+	return eError;
+}
+#endif
+
+
+static void RGXHWPerfL1BufferDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	if (psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc)
+	{
+		if (psRgxDevInfo->psRGXFWIfHWPerfBuf != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+			psRgxDevInfo->psRGXFWIfHWPerfBuf = NULL;
+		}
+		DevmemFwUnmapAndFree(psRgxDevInfo, psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+		psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc = NULL;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInit
+
+@Description    Called during driver init for initialization of HWPerf module
+				in the Rogue device driver. This function keeps allocated
+				only the minimal necessary resources, which are required for
+				functioning of HWPerf server module.
+
+@Input          psRgxDevInfo	RGX Device Info
+
+@Return			PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PVR_DPF_ENTERED;
+
+	/* expecting a valid device info */
+	PVR_RETURN_IF_INVALID_PARAM(psRgxDevInfo != NULL);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_OK);
+
+	/* Create a lock for HWPerf server module used for serializing, L1 to L2
+	 * copy calls (e.g. in case of TL producer callback) and L1, L2 resource
+	 * allocation */
+	eError = OSLockCreate(&psRgxDevInfo->hHWPerfLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	/* Initialise only non-zero fields since psRgxDevInfo is zeroed
+	 * on allocation. */
+	for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+	{
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[i] = IMG_TRUE;
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfIsInitRequired
+
+@Description    Returns true if the HWperf firmware buffer (L1 buffer) and host
+                driver TL buffer (L2 buffer) are not already allocated. Caller
+                must possess hHWPerfLock lock before calling this
+                function so the state tested is not inconsistent.
+
+@Input          psRgxDevInfo RGX Device Info, on which init requirement is
+                checked.
+
+@Return         IMG_BOOL	Whether initialization (allocation) is required
+ */ /**************************************************************************/
+static INLINE IMG_BOOL RGXHWPerfIsInitRequired(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                               RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hHWPerfLock));
+
+#if !defined(NO_HARDWARE)
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Both L1 and L2 buffers are required (for HWPerf functioning) on driver
+	 * built for actual hardware (TC, EMU, etc.)
+	 */
+	return psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc == NULL ||
+	       psRgxDevInfo->hHWPerfStream[eL2StreamId] == NULL;
+#else
+	/* On a NO-HW driver L2 is not allocated. So, no point in checking its
+	 * allocation */
+	return psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc == NULL;
+#endif
+}
+#if !defined(NO_HARDWARE)
+static void _HWPerfFWOnReaderOpenCB(void *pvArg)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	PVRSRV_DEVICE_NODE* psDevNode = (PVRSRV_DEVICE_NODE*) psRgxDevInfo->psDeviceNode;
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	IMG_UINT32 ui32kCCBCommandSlot;
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDevNode);
+
+	/* Clear any previously suspended state for bSuspendHWPerfL2DataCopy as we
+	 * now have a reader attached so the data will be delivered upstream. */
+	if (psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF])
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Resuming HWPerf FW event collection.",
+		        __func__));
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF] = IMG_FALSE;
+	}
+
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG;
+	sKccbCmd.uCmdData.sHWPerfCtrl.eOpCode = RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV;
+	sKccbCmd.uCmdData.sHWPerfCtrl.ui64Mask = 0;
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevNode->pvDevice,
+											  RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to generate feature packet in "
+				"firmware (error = %d)", __func__, eError));
+		return;
+	}
+
+	eError = RGXWaitForKCCBSlotUpdate(psRgxDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+}
+
+static void _HWPerfFWOnReaderCloseCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF] = IMG_TRUE;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInitOnDemandL1Buffer
+
+@Description    This function allocates the HWperf firmware buffer (L1 buffer)
+                if HWPerf is enabled at driver load time. Otherwise, this
+                buffer is allocated on-demand as and when required. Caller must
+                possess hHWPerfLock lock before calling this function so the
+                state tested is not inconsistent if called outside of
+                initialisation.
+
+@Input          psRgxDevInfo RGX Device Info, on which init is done
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInitOnDemandL1Buffer(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_ERROR eError;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_DPF_ENTERED;
+
+	/* This function might be called more than once due to initialisation of
+	 * multiple consumers. Make sure that L1 is only ever initialised once. */
+	if (psRgxDevInfo->psRGXFWIfHWPerfBuf != NULL)
+	{
+		PVR_DPF_RETURN_OK;
+	}
+
+	/* Create the L1 HWPerf buffer on demand, read-only for the CPU
+	 * (except for the zero/poison operations) */
+	uiMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)
+						| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+						| PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE
+						| PVRSRV_MEMALLOCFLAG_GPU_UNCACHED
+						| PVRSRV_MEMALLOCFLAG_CPU_READABLE
+						| PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC
+						| PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+#if defined(PDUMP) /* Helps show where the packet data ends */
+						| PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC
+#else /* Helps show corruption issues in driver-live */
+						| PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC
+#endif
+						| PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC
+						| PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	/* Allocate HWPerf FW L1 buffer */
+	eError = DevmemFwAllocate(psRgxDevInfo,
+	                          /* Pad it enough to hold the biggest variable sized packet. */
+	                          psRgxDevInfo->ui32RGXFWIfHWPerfBufSize+RGX_HWPERF_MAX_PACKET_SIZE,
+	                          uiMemAllocFlags,
+	                          "FwHWPerfBuffer",
+	                          &psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate kernel fw hwperf buffer (%u)",
+				__func__, eError));
+		goto ErrReturn;
+	}
+
+	/* Expecting the RuntimeCfg structure is mapped into CPU virtual memory.
+	 * Also, make sure the FW address is not already set */
+	PVR_ASSERT(psRgxDevInfo->psRGXFWIfRuntimeCfg && psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr == 0x0);
+
+	/* Meta cached flag removed from this allocation as it was found
+	 * FW performance was better without it. */
+	eError = RGXSetFirmwareAddress(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf,
+	                      psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                      0, RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrDeInitL1Buffer);
+
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+	RGXSetMetaDMAAddress(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfDMABuf,
+	                     psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                     &psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf,
+	                     0);
+#endif
+
+	/* flush write buffers for psRgxDevInfo->psRGXFWIfRuntimeCfg */
+	OSWriteMemoryBarrier(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr);
+	RGXFwSharedMemCacheOpValue(psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr, FLUSH);
+
+	eError = DevmemAcquireCpuVirtAddr(psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                                  (void**)&psRgxDevInfo->psRGXFWIfHWPerfBuf);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire kernel hwperf buffer (%u)",
+				__func__, eError));
+		goto ErrDeInitL1Buffer;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf buffer size in bytes: L1: %d",
+	         psRgxDevInfo->ui32RGXFWIfHWPerfBufSize));
+
+	PVR_DPF_RETURN_OK;
+
+ErrDeInitL1Buffer:
+	/* L1 buffer initialisation failures */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+ErrReturn:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInitOnDemandL2Stream
+
+@Description    This function allocates the HWperf firmware buffer (L1 buffer)
+                and host driver TL buffer (L2 buffer) if HWPerf is enabled at
+                driver load time. Otherwise, these buffers are allocated
+                on-demand as and when required. Caller must possess hHWPerfLock
+                lock before calling this function so the state tested is not
+                inconsistent if called outside of driver initialisation.
+
+@Input          psRgxDevInfo RGX Device Info, on which init is done
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInitOnDemandL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                           RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if !defined(NO_HARDWARE)
+	IMG_HANDLE hStream;
+	TL_STREAM_INFO sTLStreamInfo;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	PVR_DPF_ENTERED;
+
+#if !defined(NO_HARDWARE)
+	if (eL2StreamId == RGX_HWPERF_L2_STREAM_HWPERF)
+	{
+		/* On NO-HW driver, there is no MISR installed to copy data from L1 to L2. Hence,
+		 * L2 buffer is not allocated */
+		IMG_CHAR pszHWPerfStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 4];
+			/* + 4 is used to allow names up to "hwperf_fw_999", which is enough */
+
+		/* form the HWPerf stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfStreamName, sizeof(pszHWPerfStreamName), "%s%d",
+					   PVRSRV_TL_HWPERF_RGX_FW_STREAM,
+					   psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		eError = TLStreamCreate(&hStream,
+								pszHWPerfStreamName,
+								psRgxDevInfo->ui32RGXL2HWPerfBufSize,
+								TL_OPMODE_DROP_NEWER | TL_FLAG_NO_SIGNAL_ON_COMMIT,
+								_HWPerfFWOnReaderOpenCB, psRgxDevInfo,
+								_HWPerfFWOnReaderCloseCB, psRgxDevInfo,
+#if !defined(SUPPORT_TL_PRODUCER_CALLBACK)
+								NULL, NULL
+#else
+								/* Not enabled by default */
+								RGXHWPerfTLCB, psRgxDevInfo
+#endif
+								);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate", ErrClearStream);
+
+		eError = TLStreamSetNotifStream(hStream,
+										PVRSRVGetPVRSRVData()->hTLCtrlStream);
+		/* we can still discover host stream so leave it as is and just log error */
+		PVR_LOG_IF_ERROR(eError, "TLStreamSetNotifStream");
+
+		/* send the event here because host stream is implicitly opened for write
+		 * in TLStreamCreate and TLStreamOpen is never called (so the event is
+		 * never emitted) */
+		TLStreamMarkStreamOpen(hStream);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf buffer size in bytes: L1: %d  L2: %d",
+			psRgxDevInfo->ui32RGXFWIfHWPerfBufSize,
+			psRgxDevInfo->ui32RGXL2HWPerfBufSize));
+
+		psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] = hStream;
+		psRgxDevInfo->uiHWPerfStreamCount++;
+		PVR_ASSERT(psRgxDevInfo->uiHWPerfStreamCount <= RGX_HWPERF_L2_STREAM_LAST);
+	}
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+	else if (eL2StreamId == RGX_HWPERF_L2_STREAM_FTRACE)
+	{
+		eError = PVRGpuTraceInitStream(psRgxDevInfo);
+		PVR_LOG_IF_ERROR(eError, "PVRGpuTraceInitStream");
+	}
+#endif
+
+	TLStreamInfo(psRgxDevInfo->hHWPerfStream[eL2StreamId], &sTLStreamInfo);
+	psRgxDevInfo->ui32L2BufMaxPacketSize[eL2StreamId] = sTLStreamInfo.maxTLpacketSize;
+#else
+	psRgxDevInfo->hHWPerfStream[eL2StreamId] = NULL;
+#endif /* !defined(NO_HARDWARE) */
+
+	PVR_DPF_RETURN_OK;
+
+#if !defined(NO_HARDWARE)
+ErrClearStream: /* L2 buffer initialisation failures */
+	psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] = NULL;
+#endif
+	/* L1 buffer initialisation failures */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+void RGXHWPerfDeinitL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                             RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	IMG_HANDLE hStream;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+
+	hStream = psRgxDevInfo->hHWPerfStream[eL2StreamId];
+
+	/* Clean up the L2 buffer stream object if allocated */
+	if (hStream)
+	{
+		psRgxDevInfo->hHWPerfStream[eL2StreamId] = NULL;
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[eL2StreamId] = IMG_TRUE;
+		psRgxDevInfo->uiHWPerfStreamCount--;
+		PVR_ASSERT(psRgxDevInfo->uiHWPerfStreamCount < RGX_HWPERF_L2_STREAM_LAST);
+
+		/* send the event here because host stream is implicitly opened for
+		 * write in TLStreamCreate and TLStreamClose is never called (so the
+		 * event is never emitted) */
+		TLStreamMarkStreamClose(hStream);
+		TLStreamClose(hStream);
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void RGXHWPerfDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	/* Cleanup L1 buffer resources */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+
+	/* Cleanup the HWPerf server module lock resource */
+	if (psRgxDevInfo->hHWPerfLock)
+	{
+		OSLockDestroy(psRgxDevInfo->hHWPerfLock);
+		psRgxDevInfo->hHWPerfLock = NULL;
+	}
+}
+
+
+/******************************************************************************
+ * RGX HW Performance Profiling Server API(s)
+ *****************************************************************************/
+
+static PVRSRV_ERROR RGXHWPerfCtrlFwBuffer(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                          IMG_BOOL bToggle,
+                                          IMG_UINT64 ui64Mask)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevice = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_UINT64 ui64MaskValue = ui64Mask;
+	IMG_UINT64 ui64OldMaskValue;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Modify mask to include the default bit settings if it is non-zero */
+	if (!bToggle && (ui64Mask != 0ULL))
+	{
+		ui64MaskValue = ui64Mask | RGX_HWPERF_EVENT_MASK_DEFAULT;
+	}
+
+	/* If this method is being used whether to enable or disable
+	 * then the hwperf buffers (host and FW) are likely to be needed
+	 * eventually so create them, also helps unit testing. Buffers
+	 * allocated on demand to reduce RAM foot print on systems not
+	 * needing HWPerf resources.
+	 * Obtain lock first, test and init if required. */
+	OSLockAcquire(psDevice->hHWPerfLock);
+
+	if (!psDevice->bFirmwareInitialised)
+	{
+		/* No other initialisation can be done at this point until the FW is
+		 * initialised so unlock, log and return Ok so the caller knows
+		 * the filter was set. */
+		(void) RGXHWPerfFwSetEventFilterNoLock(psDevice, eL2StreamId, ui64MaskValue);
+		OSLockRelease(psDevice->hHWPerfLock);
+		goto done_;
+	}
+
+	if (RGXHWPerfIsInitRequired(psDevice, eL2StreamId))
+	{
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Initialisation of on-demand HWPerfFW "
+					"resources failed", __func__));
+			goto unlock_and_return;
+		}
+
+		/* if this fails it also cleans up L1 buffer */
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevice, eL2StreamId);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Initialisation of on-demand HWPerfFW "
+					"resources failed", __func__));
+			goto unlock_and_return;
+		}
+	}
+
+	/* Unlock here as no further HWPerf resources are used below that would be
+	 * affected if freed by another thread */
+	OSLockRelease(psDevice->hHWPerfLock);
+
+	/* Return if the filter is the same */
+	if (!bToggle && psDevice->ui64HWPerfFilter[eL2StreamId] == ui64MaskValue)
+	{
+		goto done_;
+	}
+
+	ui64OldMaskValue = psDevice->ui64HWPerfFilter[eL2StreamId];
+	ui64MaskValue = RGXHWPerfFwSetEventFilter(psDevice, eL2StreamId, bToggle
+	                                          ? psDevice->ui64HWPerfFilter[eL2StreamId] ^ ui64MaskValue
+	                                          : ui64MaskValue);
+
+	/* Prepare command parameters ... */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG;
+	sKccbCmd.uCmdData.sHWPerfCtrl.eOpCode = bToggle ? RGXFWIF_HWPERF_CTRL_TOGGLE : RGXFWIF_HWPERF_CTRL_SET;
+	sKccbCmd.uCmdData.sHWPerfCtrl.ui64Mask = ui64MaskValue;
+
+	/* Ask the FW to carry out the HWPerf configuration command */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set new HWPerfFW filter in "
+				"firmware (error = %d)", __func__, eError));
+		goto restore_mask_;
+	}
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", restore_mask_);
+
+done_:
+	return PVRSRV_OK;
+
+unlock_and_return:
+	OSLockRelease(psDevice->hHWPerfLock);
+
+	return eError;
+
+restore_mask_:
+	(void) RGXHWPerfFwSetEventFilter(psDevice, eL2StreamId, ui64OldMaskValue);
+
+	return eError;
+}
+
+#define HWPERF_HOST_MAX_DEFERRED_PACKETS 800
+
+static PVRSRV_ERROR RGXHWPerfCtrlHostBuffer(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_BOOL bToggle,
+                                            IMG_UINT32 ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevice = psDeviceNode->pvDevice;
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32 ui32OldFilter = psDevice->ui32HWPerfHostFilter;
+#endif
+
+	if (psDevice->hHWPerfHostStream == NULL)
+	{
+		eError = RGXHWPerfHostInitOnDemandResources(psDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Initialisation of on-demand HWPerfHost resources failed",
+					 __func__));
+			return eError;
+		}
+	}
+
+	OSLockAcquire(psDevice->hLockHWPerfHostStream);
+	psDevice->ui32HWPerfHostFilter = bToggle ?
+			psDevice->ui32HWPerfHostFilter ^ ui32Mask : ui32Mask;
+
+	// Deferred creation of host periodic events thread
+	if (psDevice->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HOST_INFO))
+	{
+		eError = PVRSRVCreateHWPerfHostThread(PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVCreateHWPerfHostThread");
+	}
+	else
+	{
+		eError = PVRSRVDestroyHWPerfHostThread();
+		PVR_LOG_IF_ERROR(eError, "PVRSRVDestroyHWPerfHostThread");
+	}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	// Log deferred events stats if filter changed from non-zero to zero
+	if ((ui32OldFilter != 0) && (psDevice->ui32HWPerfHostFilter == 0))
+	{
+		PVR_LOG(("HWPerfHost deferred events buffer high-watermark / size: (%u / %u)",
+				psDevice->ui32DEHighWatermark, HWPERF_HOST_MAX_DEFERRED_PACKETS));
+
+		PVR_LOG(("HWPerfHost deferred event retries: WaitForAtomicCtxPktHighWatermark(%u) "
+				"WaitForRightOrdPktHighWatermark(%u)",
+				psDevice->ui32WaitForAtomicCtxPktHighWatermark,
+				psDevice->ui32WaitForRightOrdPktHighWatermark));
+	}
+#endif
+
+	OSLockRelease(psDevice->hLockHWPerfHostStream);
+
+	if (bToggle)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfHost events (%x) have been TOGGLED",
+				ui32Mask));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfHost mask has been SET to (%x)",
+				ui32Mask));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXHWPerfCtrlClientBuffer(IMG_BOOL bToggle,
+                                              IMG_UINT32 ui32InfoPageIdx,
+                                              IMG_UINT32 ui32Mask)
+{
+	PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+
+	PVR_LOG_RETURN_IF_FALSE(ui32InfoPageIdx >= INFO_PAGE_HWPERF_BLOCK_START &&
+	                        ui32InfoPageIdx < INFO_PAGE_HWPERF_BLOCK_END,
+	                        "invalid info page index", PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquire(psData->hInfoPageLock);
+	psData->pui32InfoPage[ui32InfoPageIdx] = bToggle ?
+			psData->pui32InfoPage[ui32InfoPageIdx] ^ ui32Mask : ui32Mask;
+	OSLockRelease(psData->hInfoPageLock);
+
+	if (bToggle)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfClient (%u) events (%x) have been TOGGLED",
+				ui32InfoPageIdx, ui32Mask));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfClient (%u) mask has been SET to (%x)",
+				ui32InfoPageIdx, ui32Mask));
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(CONNECTION_DATA    *psConnection,
+                                                  PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                  RGX_HWPERF_BVNC    *psBVNC)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR        eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDeviceNode), "psDeviceNode invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = psDeviceNode->pvDevice;
+	eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, psBVNC);
+
+	return eError;
+}
+
+/*
+	AppHint interfaces
+ */
+static
+PVRSRV_ERROR RGXHWPerfSetFwFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT64 ui64Value)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	eError = RGXHWPerfCtrlFwBuffer(psDeviceNode, RGX_HWPERF_L2_STREAM_HWPERF,
+	                               IMG_FALSE, ui64Value);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Failed to set HWPerf firmware filter for device (%u)",
+		        psDeviceNode->sDevId.ui32InternalID));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfReadFwFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT64 *pui64Value)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pui64Value = psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF];
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfSetHostFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const void *psPrivate,
+                                    IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	eError = RGXHWPerfCtrlHostBuffer(psDeviceNode, IMG_FALSE, ui32Value);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Failed to set HWPerf firmware filter for device (%u)",
+		        psDeviceNode->sDevId.ui32InternalID));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfReadHostFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const void *psPrivate,
+                                     IMG_UINT32 *pui32Value)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32HWPerfHostFilter;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _ReadClientFilter(const PVRSRV_DEVICE_NODE *psDevice,
+                                      const void *psPrivData,
+                                      IMG_UINT32 *pui32Value)
+{
+	PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32Idx = (IMG_UINT32) (uintptr_t) psPrivData;
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	OSLockAcquire(psData->hInfoPageLock);
+	*pui32Value = psData->pui32InfoPage[ui32Idx];
+	OSLockRelease(psData->hInfoPageLock);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _WriteClientFilter(const PVRSRV_DEVICE_NODE *psDevice,
+                                       const void *psPrivData,
+                                       IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 ui32Idx = (IMG_UINT32) (uintptr_t) psPrivData;
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	return RGXHWPerfCtrlClientBuffer(IMG_FALSE, ui32Idx, ui32Value);
+}
+
+void RGXHWPerfInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersUINT64(APPHINT_ID_HWPerfFWFilter,
+	                                    RGXHWPerfReadFwFilter,
+	                                    RGXHWPerfSetFwFilter,
+	                                    psDeviceNode,
+	                                    NULL);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfHostFilter,
+	                                    RGXHWPerfReadHostFilter,
+	                                    RGXHWPerfSetHostFilter,
+	                                    psDeviceNode,
+	                                    NULL);
+}
+
+void RGXHWPerfClientInitAppHintCallbacks(void)
+{
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_Services,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_SERVICES_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_EGL,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_EGL_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenGLES,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_OPENGLES_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenCL,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_OPENCL_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_Vulkan,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_VULKAN_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenGL,
+		                                _ReadClientFilter,
+		                                _WriteClientFilter,
+		                                APPHINT_OF_DRIVER_NO_DEVICE,
+		                                (void *) HWPERF_FILTER_OPENGL_IDX);
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCountersKM(CONNECTION_DATA *psConnection,
+                                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                    const IMG_UINT32 ui32BlockID,
+                                                    RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psConfiguredCounters != NULL, "psConfiguredCounters is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXGetHWPerfCtl");
+
+	eError = PVRSRVRGXGetConfiguredHWPerfCounters(psDeviceNode,
+	                                              psHWPerfCtl,
+	                                              ui32BlockID,
+	                                              psConfiguredCounters);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRGXGetConfiguredHWPerfCounters");
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocksKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               const IMG_UINT32 ui32ArrayLen,
+                                               IMG_UINT32 *pui32BlockCount,
+                                               IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	IMG_UINT32 *pui32BlockIDs = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(pui32BlockCount != NULL, "pui32BlockCount is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXGetHWPerfCtl");
+
+	if (pui32EnabledBlockIDs != NULL)
+	{
+		pui32BlockIDs = OSAllocMem(sizeof(IMG_UINT32) * ui32ArrayLen);
+		if (pui32BlockIDs == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("OSAllocMem", eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+		}
+	}
+
+	eError = PVRSRVRGXGetEnabledHWPerfBlocks(psDeviceNode,
+	                                         psHWPerfCtl,
+	                                         ui32ArrayLen,
+	                                         pui32BlockCount,
+	                                         pui32BlockIDs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRGXGetEnabledHWPerfBlocks", Error);
+
+	if (pui32EnabledBlockIDs != NULL)
+	{
+		IMG_UINT32 i;
+		if (*pui32BlockCount > ui32ArrayLen)
+		{
+			*pui32BlockCount = 0;
+			PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+			PVR_LOG_GOTO_WITH_ERROR(__func__, eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+		}
+		else if (*pui32BlockCount < ui32ArrayLen)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen greater than the number of enabled blocks."));
+		}
+
+		for (i = 0; i < *pui32BlockCount; i++)
+		{
+			pui32EnabledBlockIDs[i] = pui32BlockIDs[i];
+		}
+	}
+
+Error:
+	if (pui32BlockIDs != NULL)
+	{
+		OSFreeMem(pui32BlockIDs);
+	}
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+static INLINE IMG_UINT32 _RGXHWPerfFixBufferSize(IMG_UINT32 ui32BufSizeKB)
+{
+	if (ui32BufSizeKB > HWPERF_HOST_TL_STREAM_SIZE_MAX)
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "RGXHWPerfHostInit: HWPerf Host buffer size "
+		         "value (%u) too big, using maximum (%u)",
+		         ui32BufSizeKB, HWPERF_HOST_TL_STREAM_SIZE_MAX));
+		return HWPERF_HOST_TL_STREAM_SIZE_MAX<<10;
+	}
+	else if (ui32BufSizeKB >= HWPERF_HOST_TL_STREAM_SIZE_MIN)
+	{
+		return ui32BufSizeKB<<10;
+	}
+	else if (ui32BufSizeKB > 0)
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "RGXHWPerfHostInit: HWPerf Host buffer size "
+		         "value (%u) too small, using minimum (%u)",
+		         ui32BufSizeKB, HWPERF_HOST_TL_STREAM_SIZE_MIN));
+		return HWPERF_HOST_TL_STREAM_SIZE_MIN<<10;
+	}
+	else
+	{
+		/* 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. */
+		return HWPERF_HOST_TL_STREAM_SIZE_DEFAULT<<10;
+	}
+}
+
+/******************************************************************************
+ * RGX HW Performance Host Stream API
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfHostInit
+
+@Description    Called during driver init for initialisation of HWPerfHost
+                stream in the Rogue device driver. This function keeps allocated
+                only the minimal necessary resources, which are required for
+                functioning of HWPerf server module.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfHostInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32BufSizeKB)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_INVALID_PARAM(psRgxDevInfo != NULL);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_OK);
+
+	eError = OSLockCreate(&psRgxDevInfo->hLockHWPerfHostStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", error);
+
+	psRgxDevInfo->hHWPerfHostStream = NULL;
+	psRgxDevInfo->ui32HWPerfHostFilter = 0; /* disable all events */
+	psRgxDevInfo->ui32HWPerfHostNextOrdinal = 1;
+	psRgxDevInfo->ui32HWPerfHostBufSize = _RGXHWPerfFixBufferSize(ui32BufSizeKB);
+	psRgxDevInfo->pvHostHWPerfMISR = NULL;
+	psRgxDevInfo->pui8DeferredEvents = NULL;
+	/* First packet has ordinal=1, so LastOrdinal=0 will ensure ordering logic
+	 * is maintained */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = 0;
+	psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+
+error:
+	return eError;
+}
+
+#define RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE \
+	((IMG_UINT32)(offsetof(RGX_HWPERF_HOST_CLIENT_INFO_DATA, uDetail) + \
+		sizeof(((RGX_HWPERF_HOST_CLIENT_INFO_DETAIL*)0)->sProcName.ui32Count)))
+
+static void _HWPerfHostOnConnectCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psDevice;
+	PVRSRV_ERROR eError;
+
+	RGXSRV_HWPERF_CLK_SYNC(pvArg);
+
+	psDevice = (PVRSRV_RGXDEV_INFO*) pvArg;
+
+	/* Handle the case where we may be being called as part of a multi-device
+	 * initialisation sequence. If the bDevInit2Done flag is not yet set we can
+	 * perform no action for this device. Simply return.
+	 */
+	if (!psDevice->bDevInit2Done)
+	{
+		return;
+	}
+
+	/* Handle the case where the RGX_HWPERF_HOST_INFO bit is set in the event filter
+	 * before the host stream is opened for reading by a HWPerf client.
+	 * Which can result in the host periodic thread sleeping for a long duration as TLStreamIsOpenForReading may return false. */
+	if (psDevice->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HOST_INFO))
+	{
+		eError = PVRSRVCreateHWPerfHostThread(PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVCreateHWPerfHostThread");
+	}
+
+	RGXSRV_HWPERF_DEVICE_INFO_FEATURES(psDevice);
+
+	if (RGXHWPerfHostIsEventEnabled(psDevice, RGX_HWPERF_HOST_CLIENT_INFO))
+	{
+		// GCC throws -Werror=frame-larger-than error if the frame size is > 1024 bytes,
+		// so use a heap allocation - is there an alternate solution?
+		IMG_BYTE *pbPktPayload = (IMG_BYTE*)OSAllocMem(RGX_HWPERF_MAX_PAYLOAD_SIZE);
+
+		if (pbPktPayload)
+		{
+			RGX_HWPERF_HOST_CLIENT_INFO_DATA *psHostClientInfo;
+			RGX_HWPERF_HOST_CLIENT_PROC_NAME *psProcName;
+			IMG_UINT32 ui32TotalPayloadSize, ui32NameLen, ui32ProcNamePktSize;
+			DLLIST_NODE *pNode, *pNext;
+
+			psHostClientInfo = IMG_OFFSET_ADDR(pbPktPayload,0);
+			psHostClientInfo->eType = RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME;
+			psHostClientInfo->uDetail.sProcName.ui32Count = 0U;
+			psProcName = psHostClientInfo->uDetail.sProcName.asProcNames;
+			ui32TotalPayloadSize = RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE;
+
+			OSLockAcquire(psDevice->psDeviceNode->hConnectionsLock);
+
+			// Announce current client connections to the reader
+			dllist_foreach_node(&psDevice->psDeviceNode->sConnections, pNode, pNext)
+			{
+				CONNECTION_DATA *psData = IMG_CONTAINER_OF(pNode, CONNECTION_DATA, sConnectionListNode);
+
+				ui32NameLen = OSStringLength(psData->pszProcName) + 1U;
+				ui32ProcNamePktSize = RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen);
+
+				// Unlikely case where we have too much data to fit into a single hwperf packet
+				if (ui32ProcNamePktSize + ui32TotalPayloadSize > RGX_HWPERF_MAX_PAYLOAD_SIZE)
+				{
+					RGXHWPerfHostPostRaw(psDevice, RGX_HWPERF_HOST_CLIENT_INFO, pbPktPayload, ui32TotalPayloadSize);
+
+					psHostClientInfo->uDetail.sProcName.ui32Count = 0U;
+					psProcName = psHostClientInfo->uDetail.sProcName.asProcNames;
+					ui32TotalPayloadSize = RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE;
+				}
+
+				// Setup packet data
+				psHostClientInfo->uDetail.sProcName.ui32Count++;
+				psProcName->uiClientPID = psData->pid;
+				psProcName->ui32Length = ui32NameLen;
+				(void)OSCachedMemCopy(psProcName->acName, psData->pszProcName, ui32NameLen);
+
+				psProcName = (RGX_HWPERF_HOST_CLIENT_PROC_NAME*)IMG_OFFSET_ADDR(psProcName, ui32ProcNamePktSize);
+				ui32TotalPayloadSize += ui32ProcNamePktSize;
+			}
+
+			OSLockRelease(psDevice->psDeviceNode->hConnectionsLock);
+			RGXHWPerfHostPostRaw(psDevice, RGX_HWPERF_HOST_CLIENT_INFO, pbPktPayload, ui32TotalPayloadSize);
+			OSFreeMem(pbPktPayload);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OUT OF MEMORY. Could not allocate memory for RGX_HWPERF_HOST_CLIENT_INFO_DATA packet.", __func__));
+		}
+	}
+}
+
+/* Avoiding a holder struct using fields below, as a struct gets along padding,
+ * packing, and other compiler dependencies, and we want a continuous stream of
+ * bytes for (header+data) for use in TLStreamWrite. See
+ * _HWPerfHostDeferredEventsEmitter().
+ *
+ * A deferred (UFO) packet is represented in memory as:
+ *     - IMG_BOOL                 --> Indicates whether a packet write is
+ *                                    "complete" by atomic context or not.
+ *     - RGX_HWPERF_V2_PACKET_HDR --.
+ *                                  |--> Fed together to TLStreamWrite for
+ *                                  |    deferred packet to be written to
+ *                                  |    HWPerfHost buffer
+ *     - RGX_HWPERF_HOST_UFO_DATA---`
+ *
+ * PS: Currently only UFO events are supported in deferred list */
+#define HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE (sizeof(IMG_BOOL) +\
+		sizeof(RGX_HWPERF_V2_PACKET_HDR) +\
+		sizeof(RGX_HWPERF_HOST_UFO_DATA))
+
+static void RGX_MISRHandler_HWPerfPostDeferredHostEvents(void *pvData);
+static void _HWPerfHostDeferredEventsEmitter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                             IMG_UINT32 ui32MaxOrdinal);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfHostInitOnDemandResources
+
+@Description    This function allocates the HWPerfHost buffer if HWPerf is
+                enabled at driver load time. Otherwise, these buffers are
+                allocated on-demand as and when required.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfHostInitOnDemandResources(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError;
+	/* 4 makes space up to "hwperf_host_999" streams */
+	IMG_CHAR pszHWPerfHostStreamName[sizeof(PVRSRV_TL_HWPERF_HOST_SERVER_STREAM) + 4];
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psRgxDevInfo->hHWPerfHostStream != NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf host stream already initialised"));
+		return PVRSRV_OK;
+	}
+
+	/* form the HWPerf host stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszHWPerfHostStreamName, sizeof(pszHWPerfHostStreamName), "%s%d",
+	               PVRSRV_TL_HWPERF_HOST_SERVER_STREAM,
+	               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to form HWPerf host stream name for device %d",
+				__func__,
+				psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = TLStreamCreate(&psRgxDevInfo->hHWPerfHostStream,
+	                        pszHWPerfHostStreamName, psRgxDevInfo->ui32HWPerfHostBufSize,
+	                        TL_OPMODE_DROP_NEWER,
+	                        _HWPerfHostOnConnectCB, psRgxDevInfo,
+	                        NULL, NULL, NULL, NULL);
+	PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamCreate");
+
+	eError = TLStreamSetNotifStream(psRgxDevInfo->hHWPerfHostStream,
+	                                PVRSRVGetPVRSRVData()->hTLCtrlStream);
+	/* we can still discover host stream so leave it as is and just log error */
+	PVR_LOG_IF_ERROR(eError, "TLStreamSetNotifStream");
+
+	/* send the event here because host stream is implicitly opened for write
+	 * in TLStreamCreate and TLStreamOpen is never called (so the event is
+	 * never emitted) */
+	eError = TLStreamMarkStreamOpen(psRgxDevInfo->hHWPerfHostStream);
+	PVR_LOG_IF_ERROR(eError, "TLStreamMarkStreamOpen");
+
+	/* HWPerfHost deferred events specific initialization */
+	eError = OSInstallMISR(&psRgxDevInfo->pvHostHWPerfMISR,
+	                       RGX_MISRHandler_HWPerfPostDeferredHostEvents,
+	                       psRgxDevInfo,
+	                       "RGX_HWPerfDeferredEventPoster");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR", err_install_misr);
+
+	eError = OSSpinLockCreate(&psRgxDevInfo->hHWPerfHostSpinLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate", err_spinlock_create);
+
+	psRgxDevInfo->pui8DeferredEvents = OSAllocMem(HWPERF_HOST_MAX_DEFERRED_PACKETS
+	                                              * HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE);
+	if (NULL == psRgxDevInfo->pui8DeferredEvents)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OUT OF MEMORY. Could not allocate memory for "
+				"HWPerfHost deferred events array", __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_alloc_deferred_events;
+	}
+	psRgxDevInfo->ui16DEReadIdx = 0;
+	psRgxDevInfo->ui16DEWriteIdx = 0;
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	psRgxDevInfo->ui32DEHighWatermark = 0;
+	psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark = 0;
+	psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark = 0;
+#endif
+
+	PVR_DPF((DBGPRIV_MESSAGE, "HWPerf Host buffer size is %uKB",
+			psRgxDevInfo->ui32HWPerfHostBufSize));
+
+	return PVRSRV_OK;
+
+err_alloc_deferred_events:
+	OSSpinLockDestroy(psRgxDevInfo->hHWPerfHostSpinLock);
+	psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+
+err_spinlock_create:
+	(void) OSUninstallMISR(psRgxDevInfo->pvHostHWPerfMISR);
+	psRgxDevInfo->pvHostHWPerfMISR = NULL;
+
+err_install_misr:
+	TLStreamMarkStreamClose(psRgxDevInfo->hHWPerfHostStream);
+	TLStreamClose(psRgxDevInfo->hHWPerfHostStream);
+	psRgxDevInfo->hHWPerfHostStream = NULL;
+
+	return eError;
+}
+
+void RGXHWPerfHostDeInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVR_ASSERT (psRgxDevInfo);
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+
+	if (psRgxDevInfo->pui8DeferredEvents)
+	{
+		OSFreeMem(psRgxDevInfo->pui8DeferredEvents);
+		psRgxDevInfo->pui8DeferredEvents = NULL;
+	}
+
+	if (psRgxDevInfo->hHWPerfHostSpinLock)
+	{
+		OSSpinLockDestroy(psRgxDevInfo->hHWPerfHostSpinLock);
+		psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+	}
+
+	if (psRgxDevInfo->pvHostHWPerfMISR)
+	{
+		(void) OSUninstallMISR(psRgxDevInfo->pvHostHWPerfMISR);
+		psRgxDevInfo->pvHostHWPerfMISR = NULL;
+	}
+
+	if (psRgxDevInfo->hHWPerfHostStream)
+	{
+		/* send the event here because host stream is implicitly opened for
+		 * write in TLStreamCreate and TLStreamClose is never called (so the
+		 * event is never emitted) */
+		TLStreamMarkStreamClose(psRgxDevInfo->hHWPerfHostStream);
+		TLStreamClose(psRgxDevInfo->hHWPerfHostStream);
+		psRgxDevInfo->hHWPerfHostStream = NULL;
+	}
+
+	if (psRgxDevInfo->hLockHWPerfHostStream)
+	{
+		OSLockDestroy(psRgxDevInfo->hLockHWPerfHostStream);
+		psRgxDevInfo->hLockHWPerfHostStream = NULL;
+	}
+}
+
+static IMG_UINT64 RGXHWPerfFwSetEventFilterNoLock(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                                  IMG_UINT64 uiFilter)
+{
+	IMG_UINT64 uiTmpFilter = 0;
+	IMG_UINT32 i;
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Set filter for the given L2 stream. */
+	psRgxDevInfo->ui64HWPerfFilter[eL2StreamId] = uiFilter;
+
+	/* Compute compound filter from all existing L2 streams' filters. */
+	for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+	{
+		uiTmpFilter |= psRgxDevInfo->ui64HWPerfFilter[i];
+	}
+
+	psRgxDevInfo->ui64HWPerfFwFilter = uiTmpFilter;
+
+#if !defined(NO_HARDWARE)
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerfFW mask has been SET to 0x%" IMG_UINT64_FMTSPECx
+	         " (stream %u value SET to 0x%" IMG_UINT64_FMTSPECx ")",
+	         psRgxDevInfo->ui64HWPerfFwFilter, eL2StreamId,
+	         psRgxDevInfo->ui64HWPerfFilter[eL2StreamId]));
+#endif
+
+	return uiTmpFilter;
+}
+
+IMG_UINT64 RGXHWPerfFwSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                     RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                     IMG_UINT64 uiFilter)
+{
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	uiFilter = RGXHWPerfFwSetEventFilterNoLock(psRgxDevInfo, eL2StreamId, uiFilter);
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+	return uiFilter;
+}
+
+inline void RGXHWPerfHostSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Filter)
+{
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+	psRgxDevInfo->ui32HWPerfHostFilter = ui32Filter;
+}
+
+inline IMG_BOOL RGXHWPerfHostIsEventEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo, RGX_HWPERF_HOST_EVENT_TYPE eEvent)
+{
+	PVR_ASSERT(psRgxDevInfo);
+	return (psRgxDevInfo->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(eEvent)) ? IMG_TRUE : IMG_FALSE;
+}
+
+#define MAX_RETRY_COUNT 80
+static inline void _PostFunctionPrologue(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_UINT32 ui32CurrentOrdinal)
+{
+	IMG_UINT32 ui32Retry = MAX_RETRY_COUNT;
+
+	PVR_ASSERT(psRgxDevInfo->hLockHWPerfHostStream != NULL);
+	PVR_ASSERT(psRgxDevInfo->hHWPerfHostStream != NULL);
+
+	OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+
+	/* First, flush pending events (if any) */
+	_HWPerfHostDeferredEventsEmitter(psRgxDevInfo, ui32CurrentOrdinal);
+
+	while ((ui32CurrentOrdinal != psRgxDevInfo->ui32HWPerfHostLastOrdinal + 1)
+		   && (--ui32Retry != 0))
+	{
+		/* Release lock and give a chance to a waiting context to emit the
+		 * expected packet */
+		OSLockRelease (psRgxDevInfo->hLockHWPerfHostStream);
+		OSSleepms(100);
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+	}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	if ((ui32Retry == 0) && !(psRgxDevInfo->bWarnedPktOrdinalBroke))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Will warn only once! Potential packet(s) lost after ordinal"
+				 " %u (Current ordinal = %u)",
+				 __func__,
+				 psRgxDevInfo->ui32HWPerfHostLastOrdinal, ui32CurrentOrdinal));
+		psRgxDevInfo->bWarnedPktOrdinalBroke = IMG_TRUE;
+	}
+
+	if (psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark < (MAX_RETRY_COUNT - ui32Retry))
+	{
+		psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark = MAX_RETRY_COUNT - ui32Retry;
+	}
+#endif
+}
+
+static inline void _PostFunctionEpilogue(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_UINT32 ui32CurrentOrdinal)
+{
+	/* update last ordinal emitted */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = ui32CurrentOrdinal;
+
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hLockHWPerfHostStream));
+	OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+}
+
+static inline IMG_UINT8 *_ReserveHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Size)
+{
+	IMG_UINT8 *pui8Dest;
+
+	PVRSRV_ERROR eError = TLStreamReserve(psRgxDevInfo->hHWPerfHostStream,
+	                                      &pui8Dest, ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not reserve space in %s buffer"
+				" (%d). Dropping packet.",
+				__func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+		return NULL;
+	}
+	PVR_ASSERT(pui8Dest != NULL);
+
+	return pui8Dest;
+}
+
+static inline void _CommitHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError = TLStreamCommit(psRgxDevInfo->hHWPerfHostStream,
+	                                     ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not commit data to %s"
+				" (%d)", __func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+	}
+}
+
+/* Returns IMG_TRUE if packet write passes, IMG_FALSE otherwise */
+static inline IMG_BOOL _WriteHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                          RGX_HWPERF_V2_PACKET_HDR *psHeader)
+{
+	PVRSRV_ERROR eError = TLStreamWrite(psRgxDevInfo->hHWPerfHostStream,
+	                                    IMG_OFFSET_ADDR(psHeader, 0), psHeader->ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not write packet in %s buffer"
+				" (%d). Dropping packet.",
+				__func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+	}
+
+	/* Regardless of whether write passed/failed, we consider it "written" */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = psHeader->ui32Ordinal;
+
+	return (eError == PVRSRV_OK);
+}
+
+/* Helper macros for deferred events operations */
+#define GET_DE_NEXT_IDX(_curridx) ((_curridx + 1) % HWPERF_HOST_MAX_DEFERRED_PACKETS)
+#define GET_DE_EVENT_BASE(_idx)   (IMG_OFFSET_ADDR(psRgxDevInfo->pui8DeferredEvents, \
+		(_idx) * HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE))
+
+#define GET_DE_EVENT_WRITE_STATUS(_base) ((IMG_BOOL*)((void *)(_base)))
+#define GET_DE_EVENT_DATA(_base)         (IMG_OFFSET_ADDR((_base), sizeof(IMG_BOOL)))
+
+/* Emits HWPerfHost event packets present in the deferred list stopping when one
+ * of the following cases is hit:
+ * case 1: Packet ordering breaks i.e. a packet found doesn't meet ordering
+ *         criteria (ordinal == last_ordinal + 1)
+ *
+ * case 2: A packet with ordinal > ui32MaxOrdinal is found
+ *
+ * case 3: Deferred list's (read == write) i.e. no more deferred packets.
+ *
+ * NOTE: Caller must possess the hLockHWPerfHostStream lock before calling
+ *       this function.*/
+static void _HWPerfHostDeferredEventsEmitter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                             IMG_UINT32 ui32MaxOrdinal)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader;
+	IMG_UINT32 ui32Retry;
+	IMG_UINT8  *pui8DeferredEvent;
+	IMG_BOOL   *pbPacketWritten;
+	IMG_BOOL   bWritePassed;
+
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hLockHWPerfHostStream));
+
+	while (psRgxDevInfo->ui16DEReadIdx != psRgxDevInfo->ui16DEWriteIdx)
+	{
+		pui8DeferredEvent = GET_DE_EVENT_BASE(psRgxDevInfo->ui16DEReadIdx);
+		pbPacketWritten   = GET_DE_EVENT_WRITE_STATUS(pui8DeferredEvent);
+		psHeader          = (RGX_HWPERF_V2_PACKET_HDR*) GET_DE_EVENT_DATA(pui8DeferredEvent);
+
+		for (ui32Retry = MAX_RETRY_COUNT; !(*pbPacketWritten) && (ui32Retry != 0); ui32Retry--)
+		{
+			/* Packet not yet written, re-check after a while. Wait for a short period as
+			 * atomic contexts are generally expected to finish fast */
+			OSWaitus(10);
+		}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+		if ((ui32Retry == 0) && !(psRgxDevInfo->bWarnedAtomicCtxPktLost))
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: Will warn only once. Dropping a deferred packet as atomic context"
+					 " took too long to write it",
+					 __func__));
+			psRgxDevInfo->bWarnedAtomicCtxPktLost = IMG_TRUE;
+		}
+
+		if (psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark < (MAX_RETRY_COUNT - ui32Retry))
+		{
+			psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark = MAX_RETRY_COUNT - ui32Retry;
+		}
+#endif
+
+		if (*pbPacketWritten)
+		{
+			if ((psHeader->ui32Ordinal > ui32MaxOrdinal) ||
+					(psHeader->ui32Ordinal != (psRgxDevInfo->ui32HWPerfHostLastOrdinal + 1)))
+			{
+				/* Leave remaining events to be emitted by next call to this function */
+				break;
+			}
+			bWritePassed = _WriteHWPerfStream(psRgxDevInfo, psHeader);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Atomic context packet lost!", __func__));
+			bWritePassed = IMG_FALSE;
+		}
+
+		/* Move on to next packet */
+		psRgxDevInfo->ui16DEReadIdx = GET_DE_NEXT_IDX(psRgxDevInfo->ui16DEReadIdx);
+
+		if (!bWritePassed // if write failed
+				&& ui32MaxOrdinal == IMG_UINT32_MAX // and we are from MISR
+				&& psRgxDevInfo->ui16DEReadIdx != psRgxDevInfo->ui16DEWriteIdx) // and there are more events
+		{
+			/* Stop emitting here and re-schedule MISR */
+			OSScheduleMISR(psRgxDevInfo->pvHostHWPerfMISR);
+			break;
+		}
+	}
+}
+
+static void RGX_MISRHandler_HWPerfPostDeferredHostEvents(void *pvData)
+{
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo = pvData;
+
+	OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+
+	/* Since we're called from MISR, there is no upper cap of ordinal to be emitted.
+	 * Send IMG_UINT32_MAX to signify all possible packets. */
+	_HWPerfHostDeferredEventsEmitter(psRgxDevInfo, IMG_UINT32_MAX);
+
+	OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+static inline void _UpdateDEBufferHighWatermark(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	IMG_UINT32 ui32DEWatermark;
+	IMG_UINT16 ui16LRead = psRgxDevInfo->ui16DEReadIdx;
+	IMG_UINT16 ui16LWrite = psRgxDevInfo->ui16DEWriteIdx;
+
+	if (ui16LWrite >= ui16LRead)
+	{
+		ui32DEWatermark = ui16LWrite - ui16LRead;
+	}
+	else
+	{
+		ui32DEWatermark = (HWPERF_HOST_MAX_DEFERRED_PACKETS - ui16LRead) + (ui16LWrite);
+	}
+
+	if (ui32DEWatermark > psRgxDevInfo->ui32DEHighWatermark)
+	{
+		psRgxDevInfo->ui32DEHighWatermark = ui32DEWatermark;
+	}
+}
+#endif
+
+/* @Description Gets the data/members that concerns the accuracy of a packet in HWPerfHost
+                buffer. Since the data returned by this function is required in both, an
+                atomic as well as a process/sleepable context, it is protected under spinlock
+
+   @Output      pui32Ordinal Pointer to ordinal number assigned to this packet
+   @Output      pui64Timestamp Timestamp value for this packet
+   @Output      ppui8Dest If the current context cannot sleep, pointer to a place in
+                          deferred events buffer where the packet data should be written.
+                          Don't care, otherwise.
+ */
+static void _GetHWPerfHostPacketSpecifics(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                          IMG_UINT32 *pui32Ordinal,
+                                          IMG_UINT64 *pui64Timestamp,
+                                          IMG_UINT8 **ppui8Dest,
+                                          IMG_BOOL    bSleepAllowed)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* Spin lock is required to avoid getting scheduled out by a higher priority
+	 * context while we're getting header specific details and packet place in
+	 * HWPerf buffer (when in atomic context) for ourselves */
+	OSSpinLockAcquire(psRgxDevInfo->hHWPerfHostSpinLock, uiFlags);
+
+	*pui32Ordinal = psRgxDevInfo->ui32HWPerfHostNextOrdinal++;
+	/* For HWPerf we always use monotonic clock. */
+	(void) OSClockMonotonicus64(pui64Timestamp);
+
+	if (!bSleepAllowed)
+	{
+		/* We're in an atomic context. So return the next position available in
+		 * deferred events buffer */
+		IMG_UINT16 ui16NewWriteIdx;
+		IMG_BOOL *pbPacketWritten;
+
+		PVR_ASSERT(ppui8Dest != NULL);
+
+		ui16NewWriteIdx = GET_DE_NEXT_IDX(psRgxDevInfo->ui16DEWriteIdx);
+		if (ui16NewWriteIdx == psRgxDevInfo->ui16DEReadIdx)
+		{
+			/* This shouldn't happen. HWPERF_HOST_MAX_DEFERRED_PACKETS should be
+			 * big enough to avoid any such scenario */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+			/* PVR_LOG/printk isn't recommended in atomic context. Perhaps we'll do
+			 * this debug output here when trace_printk support is added to DDK */
+//			PVR_LOG(("%s: No more space in deferred events buffer (%u/%u) W=%u,R=%u",
+//                     __func__, psRgxDevInfo->ui32DEHighWatermark,
+//					 HWPERF_HOST_MAX_DEFERRED_PACKETS, psRgxDevInfo->ui16DEWriteIdx,
+//					 psRgxDevInfo->ui16DEReadIdx));
+#endif
+			*ppui8Dest = NULL;
+		}
+		else
+		{
+			/* Return the position where deferred event would be written */
+			*ppui8Dest = GET_DE_EVENT_BASE(psRgxDevInfo->ui16DEWriteIdx);
+
+			/* Make sure packet write "state" is "write-pending" _before_ moving write
+			 * pointer forward */
+			pbPacketWritten = GET_DE_EVENT_WRITE_STATUS(*ppui8Dest);
+			*pbPacketWritten = IMG_FALSE;
+
+			psRgxDevInfo->ui16DEWriteIdx = ui16NewWriteIdx;
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+			_UpdateDEBufferHighWatermark(psRgxDevInfo);
+#endif
+		}
+	}
+
+	OSSpinLockRelease(psRgxDevInfo->hHWPerfHostSpinLock, uiFlags);
+}
+
+static inline void _SetupHostPacketHeader(IMG_UINT8 *pui8Dest,
+                                          RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+                                          IMG_UINT32 ui32Size,
+                                          IMG_UINT32 ui32Ordinal,
+                                          IMG_UINT64 ui64Timestamp)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader = (RGX_HWPERF_V2_PACKET_HDR *) ((void *)pui8Dest);
+
+	PVR_ASSERT(ui32Size<=RGX_HWPERF_MAX_PACKET_SIZE);
+
+	psHeader->ui32Ordinal = ui32Ordinal;
+	psHeader->ui64Timestamp = ui64Timestamp;
+	psHeader->ui32Sig = HWPERF_PACKET_V2B_SIG;
+	psHeader->eTypeId = RGX_HWPERF_MAKE_TYPEID(RGX_HWPERF_STREAM_ID1_HOST,
+	                                           eEvType, 0, 0, 0);
+	psHeader->ui32Size = ui32Size;
+}
+
+static inline void _SetupHostEnqPacketData(IMG_UINT8 *pui8Dest,
+                                           RGX_HWPERF_KICK_TYPE eEnqType,
+                                           IMG_UINT32 ui32Pid,
+                                           IMG_UINT32 ui32FWDMContext,
+                                           IMG_UINT32 ui32ExtJobRef,
+                                           IMG_UINT32 ui32IntJobRef,
+                                           PVRSRV_FENCE hCheckFence,
+                                           PVRSRV_FENCE hUpdateFence,
+                                           PVRSRV_TIMELINE hUpdateTimeline,
+                                           IMG_UINT64 ui64CheckFenceUID,
+                                           IMG_UINT64 ui64UpdateFenceUID,
+                                           IMG_UINT64 ui64DeadlineInus,
+                                           IMG_UINT32 ui32CycleEstimate)
+{
+	RGX_HWPERF_HOST_ENQ_DATA *psData = (RGX_HWPERF_HOST_ENQ_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->ui32EnqType = eEnqType;
+	psData->ui32PID = ui32Pid;
+	psData->ui32ExtJobRef = ui32ExtJobRef;
+	psData->ui32IntJobRef = ui32IntJobRef;
+	psData->ui32DMContext = ui32FWDMContext;
+	psData->hCheckFence = hCheckFence;
+	psData->hUpdateFence = hUpdateFence;
+	psData->hUpdateTimeline = hUpdateTimeline;
+	psData->ui64CheckFence_UID = ui64CheckFenceUID;
+	psData->ui64UpdateFence_UID = ui64UpdateFenceUID;
+	psData->ui64DeadlineInus = ui64DeadlineInus;
+	psData->ui32CycleEstimate = ui32CycleEstimate;
+}
+
+void RGXHWPerfHostPostRaw(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+						  RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+						  IMG_BYTE *pbPayload,
+						  IMG_UINT32 ui32PayloadSize)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32PktSize;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	PVR_ASSERT(ui32PayloadSize <= RGX_HWPERF_MAX_PAYLOAD_SIZE);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32PktSize = RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32PayloadSize);
+	pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32PktSize);
+
+	if (pui8Dest == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, eEvType, ui32PktSize, ui32Ordinal, ui64Timestamp);
+	OSDeviceMemCopy((IMG_UINT8*)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR)), pbPayload, ui32PayloadSize);
+	_CommitHWPerfStream(psRgxDevInfo, ui32PktSize);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+void RGXHWPerfHostPostEnqEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_KICK_TYPE eEnqType,
+                               IMG_UINT32 ui32Pid,
+                               IMG_UINT32 ui32FWDMContext,
+                               IMG_UINT32 ui32ExtJobRef,
+                               IMG_UINT32 ui32IntJobRef,
+                               PVRSRV_FENCE hCheckFence,
+                               PVRSRV_FENCE hUpdateFence,
+                               PVRSRV_TIMELINE hUpdateTimeline,
+                               IMG_UINT64 ui64CheckFenceUID,
+                               IMG_UINT64 ui64UpdateFenceUID,
+                               IMG_UINT64 ui64DeadlineInus,
+                               IMG_UINT32 ui32CycleEstimate )
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_ENQ_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_ENQ, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostEnqPacketData(pui8Dest,
+	                        eEnqType,
+	                        ui32Pid,
+	                        ui32FWDMContext,
+	                        ui32ExtJobRef,
+	                        ui32IntJobRef,
+	                        hCheckFence,
+	                        hUpdateFence,
+	                        hUpdateTimeline,
+	                        ui64CheckFenceUID,
+	                        ui64UpdateFenceUID,
+	                        ui64DeadlineInus,
+	                        ui32CycleEstimate);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _CalculateHostUfoPacketSize(RGX_HWPERF_UFO_EV eUfoType)
+{
+	IMG_UINT32 ui32Size =
+			(IMG_UINT32) offsetof(RGX_HWPERF_HOST_UFO_DATA, aui32StreamData);
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData;
+
+	switch (eUfoType)
+	{
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			ui32Size += sizeof(puData->sCheckSuccess);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			ui32Size += sizeof(puData->sCheckFail);
+			break;
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			ui32Size += sizeof(puData->sUpdate);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostUfoEvent: Invalid UFO"
+					" event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostUfoPacketData(IMG_UINT8 *pui8Dest,
+                                           RGX_HWPERF_UFO_EV eUfoType,
+                                           RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData)
+{
+	RGX_HWPERF_HOST_UFO_DATA *psData = (RGX_HWPERF_HOST_UFO_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData = (RGX_HWPERF_UFO_DATA_ELEMENT *)
+					 psData->aui32StreamData;
+
+	psData->eEvType = eUfoType;
+	/* HWPerfHost always emits 1 UFO at a time, since each UFO has 1-to-1 mapping
+	 * with an underlying DevNode, and each DevNode has a dedicated HWPerf buffer */
+	psData->ui32StreamInfo = RGX_HWPERF_MAKE_UFOPKTINFO(1,
+	                                                    offsetof(RGX_HWPERF_HOST_UFO_DATA, aui32StreamData));
+
+	switch (eUfoType)
+	{
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			puData->sCheckSuccess.ui32FWAddr =
+					psUFOData->sCheckSuccess.ui32FWAddr;
+			puData->sCheckSuccess.ui32Value =
+					psUFOData->sCheckSuccess.ui32Value;
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			puData->sCheckFail.ui32FWAddr =
+					psUFOData->sCheckFail.ui32FWAddr;
+			puData->sCheckFail.ui32Value =
+					psUFOData->sCheckFail.ui32Value;
+			puData->sCheckFail.ui32Required =
+					psUFOData->sCheckFail.ui32Required;
+			break;
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			puData->sUpdate.ui32FWAddr =
+					psUFOData->sUpdate.ui32FWAddr;
+			puData->sUpdate.ui32OldValue =
+					psUFOData->sUpdate.ui32OldValue;
+			puData->sUpdate.ui32NewValue =
+					psUFOData->sUpdate.ui32NewValue;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostUfoEvent: Invalid UFO"
+					" event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+void RGXHWPerfHostPostUfoEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_UFO_EV eUfoType,
+                               RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData,
+                               const IMG_BOOL bSleepAllowed)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = _CalculateHostUfoPacketSize(eUfoType);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_BOOL   *pbPacketWritten = NULL;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              &pui8Dest, bSleepAllowed);
+
+	if (bSleepAllowed)
+	{
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+		{
+			goto cleanup;
+		}
+	}
+	else
+	{
+		if (pui8Dest == NULL)
+		{
+			// Give-up if we couldn't get a place in deferred events buffer
+			goto cleanup;
+		}
+		pbPacketWritten = GET_DE_EVENT_WRITE_STATUS(pui8Dest);
+		pui8Dest = GET_DE_EVENT_DATA(pui8Dest);
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_UFO, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostUfoPacketData(pui8Dest, eUfoType, psUFOData);
+
+	if (bSleepAllowed)
+	{
+		_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+	}
+	else
+	{
+		*pbPacketWritten = IMG_TRUE;
+		OSScheduleMISR(psRgxDevInfo->pvHostHWPerfMISR);
+	}
+
+cleanup:
+	if (bSleepAllowed)
+	{
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+	}
+}
+
+#define UNKNOWN_SYNC_NAME "UnknownSync"
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH==PVRSRV_SYNC_NAME_LENGTH, "Sync class name max does not match Fence Sync name max");
+
+static inline IMG_UINT32 _FixNameAndCalculateHostAllocPacketSize(
+		RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+		const IMG_CHAR **ppsName,
+		IMG_UINT32 *ui32NameSize)
+{
+	RGX_HWPERF_HOST_ALLOC_DATA *psData;
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_ALLOC_DATA, uAllocDetail);
+
+	if (*ppsName != NULL && *ui32NameSize > 0)
+	{
+		/* if string longer than maximum cut it (leave space for '\0') */
+		if ((*ui32NameSize +1U) >= PVRSRV_SYNC_NAME_LENGTH)
+			*ui32NameSize = PVRSRV_SYNC_NAME_LENGTH;
+		else
+			*ui32NameSize += 1U;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfHostPostAllocEvent: Invalid"
+				" resource name given."));
+		*ppsName = UNKNOWN_SYNC_NAME;
+		*ui32NameSize = sizeof(UNKNOWN_SYNC_NAME);
+	}
+
+	switch (eAllocType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			ui32Size += sizeof(psData->uAllocDetail.sSyncAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			ui32Size += sizeof(psData->uAllocDetail.sFenceAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW:
+			ui32Size += sizeof(psData->uAllocDetail.sSWFenceAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			ui32Size += sizeof(psData->uAllocDetail.sSyncCheckPointAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostAllocEvent: Invalid alloc event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostAllocPacketData(IMG_UINT8 *pui8Dest,
+                                             RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                             RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail,
+                                             const IMG_CHAR *psName,
+                                             IMG_UINT32 ui32NameSize)
+{
+	RGX_HWPERF_HOST_ALLOC_DATA *psData = (RGX_HWPERF_HOST_ALLOC_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	IMG_CHAR *acName = NULL;
+
+	psData->ui32AllocType = eAllocType;
+
+	switch (eAllocType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			psData->uAllocDetail.sSyncAlloc.ui32FWAddr = puAllocDetail->sSyncAlloc.ui32FWAddr;
+			acName = psData->uAllocDetail.sSyncAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uAllocDetail.sFenceAlloc.uiPID = puAllocDetail->sFenceAlloc.uiPID;
+			psData->uAllocDetail.sFenceAlloc.hFence = puAllocDetail->sFenceAlloc.hFence;
+			psData->uAllocDetail.sFenceAlloc.ui32CheckPt_FWAddr = puAllocDetail->sFenceAlloc.ui32CheckPt_FWAddr;
+			acName = psData->uAllocDetail.sFenceAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW:
+			psData->uAllocDetail.sSWFenceAlloc.uiPID = puAllocDetail->sSWFenceAlloc.uiPID;
+			psData->uAllocDetail.sSWFenceAlloc.hSWFence = puAllocDetail->sSWFenceAlloc.hSWFence;
+			psData->uAllocDetail.sSWFenceAlloc.hSWTimeline = puAllocDetail->sSWFenceAlloc.hSWTimeline;
+			psData->uAllocDetail.sSWFenceAlloc.ui64SyncPtIndex = puAllocDetail->sSWFenceAlloc.ui64SyncPtIndex;
+			acName = psData->uAllocDetail.sSWFenceAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			psData->uAllocDetail.sSyncCheckPointAlloc.ui32CheckPt_FWAddr = puAllocDetail->sSyncCheckPointAlloc.ui32CheckPt_FWAddr;
+			psData->uAllocDetail.sSyncCheckPointAlloc.hTimeline = puAllocDetail->sSyncCheckPointAlloc.hTimeline;
+			psData->uAllocDetail.sSyncCheckPointAlloc.uiPID = puAllocDetail->sSyncCheckPointAlloc.uiPID;
+			psData->uAllocDetail.sSyncCheckPointAlloc.hFence = puAllocDetail->sSyncCheckPointAlloc.hFence;
+			acName = psData->uAllocDetail.sSyncCheckPointAlloc.acName;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostAllocEvent: Invalid alloc event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+
+
+	if (acName != NULL)
+	{
+		if (ui32NameSize)
+		{
+			OSStringSafeCopy(acName, psName, ui32NameSize);
+		}
+		else
+		{
+			/* In case no name was given make sure we don't access random
+			 * memory */
+			acName[0] = '\0';
+		}
+	}
+}
+
+void RGXHWPerfHostPostAllocEvent(PVRSRV_RGXDEV_INFO* psRgxDevInfo,
+                                 RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                 const IMG_CHAR *psName,
+                                 IMG_UINT32 ui32NameSize,
+                                 RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT32 ui32Size = _FixNameAndCalculateHostAllocPacketSize(eAllocType,
+	                                                              &psName,
+	                                                              &ui32NameSize);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_ALLOC, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+
+	_SetupHostAllocPacketData(pui8Dest,
+	                          eAllocType,
+	                          puAllocDetail,
+	                          psName,
+	                          ui32NameSize);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostFreePacketData(IMG_UINT8 *pui8Dest,
+                                            RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                            IMG_UINT64 ui64UID,
+                                            IMG_UINT32 ui32FWAddr)
+{
+	RGX_HWPERF_HOST_FREE_DATA *psData = (RGX_HWPERF_HOST_FREE_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->ui32FreeType = eFreeType;
+
+	switch (eFreeType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			psData->uFreeDetail.sSyncFree.ui32FWAddr = ui32FWAddr;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uFreeDetail.sFenceDestroy.ui64Fence_UID = ui64UID;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			psData->uFreeDetail.sSyncCheckPointFree.ui32CheckPt_FWAddr = ui32FWAddr;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostFreeEvent: Invalid free event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+}
+
+void RGXHWPerfHostPostFreeEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                IMG_UINT64 ui64UID,
+                                IMG_UINT32 ui32PID,
+                                IMG_UINT32 ui32FWAddr)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_FREE_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	PVR_UNREFERENCED_PARAMETER(ui32PID);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_FREE, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostFreePacketData(pui8Dest,
+	                         eFreeType,
+	                         ui64UID,
+	                         ui32FWAddr);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _FixNameAndCalculateHostModifyPacketSize(
+		RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+		const IMG_CHAR **ppsName,
+		IMG_UINT32 *ui32NameSize)
+{
+	RGX_HWPERF_HOST_MODIFY_DATA *psData;
+	RGX_HWPERF_HOST_MODIFY_DETAIL *puData;
+	IMG_UINT32 ui32Size = sizeof(psData->ui32ModifyType);
+
+	if (*ppsName != NULL && *ui32NameSize > 0)
+	{
+		/* first strip the terminator */
+		if ((*ppsName)[*ui32NameSize - 1] == '\0')
+			*ui32NameSize -= 1;
+		/* if string longer than maximum cut it (leave space for '\0') */
+		if (*ui32NameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			*ui32NameSize = PVRSRV_SYNC_NAME_LENGTH - 1;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfHostPostModifyEvent: Invalid"
+				" resource name given."));
+		*ppsName = UNKNOWN_SYNC_NAME;
+		*ui32NameSize = sizeof(UNKNOWN_SYNC_NAME) - 1;
+	}
+
+	switch (eModifyType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			ui32Size += sizeof(puData->sFenceMerge) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize + 1; /* +1 for '\0' */
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostModifyEvent: Invalid modify event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostModifyPacketData(IMG_UINT8 *pui8Dest,
+                                              RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                              IMG_UINT64 ui64NewUID,
+                                              IMG_UINT64 ui64UID1,
+                                              IMG_UINT64 ui64UID2,
+                                              const IMG_CHAR *psName,
+                                              IMG_UINT32 ui32NameSize)
+{
+	RGX_HWPERF_HOST_MODIFY_DATA *psData = (RGX_HWPERF_HOST_MODIFY_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	IMG_CHAR *acName = NULL;
+
+	psData->ui32ModifyType = eModifyType;
+
+	switch (eModifyType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uModifyDetail.sFenceMerge.ui64NewFence_UID = ui64NewUID;
+			psData->uModifyDetail.sFenceMerge.ui64InFence1_UID = ui64UID1;
+			psData->uModifyDetail.sFenceMerge.ui64InFence2_UID = ui64UID2;
+			acName = psData->uModifyDetail.sFenceMerge.acName;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostModifyEvent: Invalid modify event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+
+	if (acName != NULL)
+	{
+		if (ui32NameSize)
+		{
+			OSStringSafeCopy(acName, psName, ui32NameSize);
+		}
+		else
+		{
+			/* In case no name was given make sure we don't access random
+			 * memory */
+			acName[0] = '\0';
+		}
+	}
+}
+
+void RGXHWPerfHostPostModifyEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                  RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                  IMG_UINT64 ui64NewUID,
+                                  IMG_UINT64 ui64UID1,
+                                  IMG_UINT64 ui64UID2,
+                                  const IMG_CHAR *psName,
+                                  IMG_UINT32 ui32NameSize)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT32 ui32Size = _FixNameAndCalculateHostModifyPacketSize(eModifyType,
+	                                                               &psName,
+	                                                               &ui32NameSize);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_MODIFY, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostModifyPacketData(pui8Dest,
+	                           eModifyType,
+	                           ui64NewUID,
+	                           ui64UID1,
+	                           ui64UID2,
+	                           psName,
+	                           ui32NameSize);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostClkSyncPacketData(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT8 *pui8Dest)
+{
+	RGX_HWPERF_HOST_CLK_SYNC_DATA *psData = (RGX_HWPERF_HOST_CLK_SYNC_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psRgxDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32CurrIdx;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32CurrIdx = RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount);
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32CurrIdx], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32CurrIdx];
+
+	psData->ui64CRTimestamp = psTimeCorr->ui64CRTimeStamp;
+	psData->ui64OSSecondaryTimeStamp = psTimeCorr->ui64OSTimeStamp;
+	psData->ui64OSMonoTimestamp = psTimeCorr->ui64OSMonoTimeStamp;
+	psData->ui32ClockSpeed = psTimeCorr->ui32CoreClockSpeed;
+}
+
+void RGXHWPerfHostPostClkSyncEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size =
+			RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_CLK_SYNC_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	/* if the buffer for time correlation data is not yet available (possibly
+	 * device not initialised yet) skip this event */
+	if (psRgxDevInfo->psRGXFWIfGpuUtilFW == NULL)
+	{
+		return;
+	}
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_CLK_SYNC, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostClkSyncPacketData(psRgxDevInfo, pui8Dest);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostDeviceInfoPacketData(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_DEV_INFO_EV eEvType,
+                                                  RGX_HWPERF_HOST_DEV_INFO_DETAIL *puPacketData,
+                                                  IMG_UINT8 *pui8Dest)
+{
+	RGX_HWPERF_HOST_DEV_INFO_DATA *psData = (RGX_HWPERF_HOST_DEV_INFO_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->eEvType = eEvType;
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_DEV_INFO_EV_HEALTH:
+			if (puPacketData != NULL)
+			{
+				psData->uDevInfoDetail.sDeviceStatus.eDeviceHealthStatus =
+					puPacketData->sDeviceStatus.eDeviceHealthStatus;
+				psData->uDevInfoDetail.sDeviceStatus.eDeviceHealthReason =
+					puPacketData->sDeviceStatus.eDeviceHealthReason;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: puPacketData is invalid."));
+			}
+			break;
+		case RGX_HWPERF_DEV_INFO_EV_FEATURES:
+			{
+				PVRSRV_ERROR eError;
+				eError = RGXServerFeatureFlagsToHWPerfFlags(psRgxDevInfo,
+				                                            &psData->uDevInfoDetail.sBVNC);
+				PVR_LOG_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags");
+				psData->uDevInfoDetail.sBVNC.ui32BvncKmFeatureFlags |=
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+					RGX_HWPERF_FEATURE_ROGUE_FLAG;
+#elif defined(RGX_FEATURE_HWPERF_VOLCANIC)
+					RGX_HWPERF_FEATURE_VOLCANIC_FLAG;
+#else
+					0x0;
+#endif
+			}
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+static inline IMG_UINT32 _CalculateHostDeviceInfoPacketSize(RGX_HWPERF_DEV_INFO_EV eEvType)
+{
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_DEV_INFO_DATA, uDevInfoDetail);
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_DEV_INFO_EV_HEALTH:
+			ui32Size += sizeof(((RGX_HWPERF_HOST_DEV_INFO_DATA*)0)->uDevInfoDetail.sDeviceStatus);
+			break;
+		case RGX_HWPERF_DEV_INFO_EV_FEATURES:
+			ui32Size += sizeof(((RGX_HWPERF_HOST_DEV_INFO_DATA*)0)->uDevInfoDetail.sBVNC);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+void RGXHWPerfHostPostDeviceInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 RGX_HWPERF_DEV_INFO_EV eEvType,
+                                 RGX_HWPERF_HOST_DEV_INFO_DETAIL *puData)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Size;
+
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->hHWPerfHostStream != (IMG_HANDLE) NULL)
+	{
+		_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+		ui32Size = _CalculateHostDeviceInfoPacketSize(eEvType);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) != NULL)
+		{
+			_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_DEV_INFO, ui32Size, ui32Ordinal, ui64Timestamp);
+			_SetupHostDeviceInfoPacketData(psRgxDevInfo, eEvType, puData, pui8Dest);
+			_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+		}
+
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+}
+
+static inline void _SetupHostInfoPacketData(RGX_HWPERF_INFO_EV eEvType,
+												  IMG_UINT64 ui64TotalMemoryUsage,
+												  IMG_UINT32 ui32LivePids,
+												  PVRSRV_PER_PROCESS_MEM_USAGE *psPerProcessMemUsage,
+												  IMG_UINT8 *pui8Dest)
+{
+	IMG_INT i;
+	RGX_HWPERF_HOST_INFO_DATA *psData = (RGX_HWPERF_HOST_INFO_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->eEvType = eEvType;
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_INFO_EV_MEM64_USAGE:
+			psData->uInfoDetail.sMemUsageStats.ui64TotalMemoryUsage = ui64TotalMemoryUsage;
+
+			if (psPerProcessMemUsage)
+			{
+				for (i = 0; i < ui32LivePids; ++i)
+				{
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui32Pid = psPerProcessMemUsage[i].ui32Pid;
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui64KernelMemUsage = psPerProcessMemUsage[i].ui64KernelMemUsage;
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui64GraphicsMemUsage = psPerProcessMemUsage[i].ui64GraphicsMemUsage;
+				}
+			}
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+static inline IMG_UINT32 _CalculateHostInfoPacketSize(RGX_HWPERF_INFO_EV eEvType,
+															IMG_UINT64 *pui64TotalMemoryUsage,
+															IMG_UINT32 *pui32LivePids,
+															PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsage)
+{
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_INFO_DATA, uInfoDetail);
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_INFO_EV_MEM64_USAGE:
+#if !defined(__QNXNTO__)
+			if (PVRSRVGetProcessMemUsage(pui64TotalMemoryUsage, pui32LivePids, ppsPerProcessMemUsage) == PVRSRV_OK)
+			{
+				ui32Size += offsetof(RGX_HWPERF_HOST_INFO_DETAIL, sMemUsageStats.sPerProcessUsage)
+					+ ((*pui32LivePids) * sizeof(struct _RGX_HWPERF_HOST_INFO_PER_PROC_USAGE_));
+			}
+#else
+			PVR_DPF((PVR_DBG_ERROR, "This functionality is not yet implemented for this platform"));
+#endif
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+void RGXHWPerfHostPostInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_INFO_EV eEvType)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT64 ui64TotalMemoryUsage = 0;
+	PVRSRV_PER_PROCESS_MEM_USAGE *psPerProcessMemUsage = NULL;
+	IMG_UINT32 ui32LivePids = 0;
+
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->hHWPerfHostStream != (IMG_HANDLE) NULL)
+	{
+		_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+		ui32Size = _CalculateHostInfoPacketSize(eEvType, &ui64TotalMemoryUsage, &ui32LivePids, &psPerProcessMemUsage);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) != NULL)
+		{
+			_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_INFO, ui32Size, ui32Ordinal, ui64Timestamp);
+			_SetupHostInfoPacketData(eEvType, ui64TotalMemoryUsage, ui32LivePids, psPerProcessMemUsage, pui8Dest);
+			_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+		}
+
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+
+		if (psPerProcessMemUsage)
+			OSFreeMemNoStats(psPerProcessMemUsage); // psPerProcessMemUsage was allocated with OSAllocZMemNoStats
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+}
+
+static inline IMG_UINT32
+_CalculateHostFenceWaitPacketSize(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eWaitType)
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *psSizeCalculator;
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA, uDetail);
+
+	switch (eWaitType)
+	{
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN:
+			ui32Size += sizeof(psSizeCalculator->uDetail.sBegin);
+			break;
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END:
+			ui32Size += sizeof(psSizeCalculator->uDetail.sEnd);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid wait event type (%u)", __func__,
+			         eWaitType));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void
+_SetupHostFenceWaitPacketData(IMG_UINT8 *pui8Dest,
+                              RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eWaitType,
+                              IMG_PID uiPID,
+                              PVRSRV_FENCE hFence,
+                              IMG_UINT32 ui32Data)
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *psData = (RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *)
+	                IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->eType = eWaitType;
+	psData->uiPID = uiPID;
+	psData->hFence = hFence;
+
+	switch (eWaitType)
+	{
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN:
+			psData->uDetail.sBegin.ui32TimeoutInMs = ui32Data;
+			break;
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END:
+			psData->uDetail.sEnd.eResult =
+			    (RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT) ui32Data;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid fence-wait event type", __func__));
+			PVR_ASSERT(IMG_FALSE);
+	}
+}
+
+void RGXHWPerfHostPostFenceWait(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType,
+								IMG_PID uiPID,
+								PVRSRV_FENCE hFence,
+								IMG_UINT32 ui32Data)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32Size = _CalculateHostFenceWaitPacketSize(eType);
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_SYNC_FENCE_WAIT,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+	_SetupHostFenceWaitPacketData(pui8Dest, eType, uiPID, hFence, ui32Data);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _CalculateHostSWTimelineAdvPacketSize(void)
+{
+	IMG_UINT32 ui32Size = sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA);
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void
+_SetupHostSWTimelineAdvPacketData(IMG_UINT8 *pui8Dest,
+                                  IMG_PID uiPID,
+								  PVRSRV_TIMELINE hSWTimeline,
+								  IMG_UINT64 ui64SyncPtIndex)
+
+{
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA *psData = (RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA *)
+	                IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->uiPID = uiPID;
+	psData->hTimeline = hSWTimeline;
+	psData->ui64SyncPtIndex = ui64SyncPtIndex;
+}
+
+void RGXHWPerfHostPostSWTimelineAdv(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                    IMG_PID uiPID,
+                                    PVRSRV_TIMELINE hSWTimeline,
+                                    IMG_UINT64 ui64SyncPtIndex)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32Size = _CalculateHostSWTimelineAdvPacketSize();
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+	_SetupHostSWTimelineAdvPacketData(pui8Dest, uiPID, hSWTimeline, ui64SyncPtIndex);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+
+}
+
+void RGXHWPerfHostPostClientInfoProcName(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_PID uiPID,
+									     const IMG_CHAR *psName)
+{
+	RGX_HWPERF_HOST_CLIENT_INFO_DATA* psPkt;
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32NameLen;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32NameLen = OSStringLength(psName) + 1U;
+	ui32Size = RGX_HWPERF_MAKE_SIZE_VARIABLE(RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE
+		+ RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen));
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_CLIENT_INFO,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+
+	psPkt = (RGX_HWPERF_HOST_CLIENT_INFO_DATA*)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psPkt->eType = RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME;
+	psPkt->uDetail.sProcName.ui32Count = 1U;
+	psPkt->uDetail.sProcName.asProcNames[0].uiClientPID = uiPID;
+	psPkt->uDetail.sProcName.asProcNames[0].ui32Length = ui32NameLen;
+	(void)OSCachedMemCopy(psPkt->uDetail.sProcName.asProcNames[0].acName, psName, ui32NameLen);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGX_KM_HWPERF_DEVDATA *psDevData;
+	RGX_HWPERF_DEVICE *psNewHWPerfDevice;
+	RGX_HWPERF_CONNECTION* psHWPerfConnection;
+	IMG_BOOL bFWActive = IMG_FALSE;
+
+	/* avoid uninitialised data */
+	PVR_ASSERT(*ppsHWPerfConnection == NULL);
+	PVR_ASSERT(psPVRSRVData);
+
+	/* Allocate connection object */
+	psHWPerfConnection = OSAllocZMem(sizeof(*psHWPerfConnection));
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	/* early save the return pointer to aid clean-up if failure occurs */
+	*ppsHWPerfConnection = psHWPerfConnection;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	while (psDeviceNode)
+	{
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_NOT_IMPLEMENTED;
+		}
+
+		if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: HWPerf: Device not currently active. ID:%u",
+					 __func__,
+					 psDeviceNode->sDevId.i32KernelDeviceID));
+			psDeviceNode = psDeviceNode->psNext;
+			continue;
+		}
+		/* Create a list node to be attached to connection object's list */
+		psNewHWPerfDevice = OSAllocMem(sizeof(*psNewHWPerfDevice));
+		if (!psNewHWPerfDevice)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		/* Insert node at head of the list */
+		psNewHWPerfDevice->psNext = psHWPerfConnection->psHWPerfDevList;
+		psHWPerfConnection->psHWPerfDevList = psNewHWPerfDevice;
+
+		/* create a device data object for kernel server */
+		psDevData = OSAllocZMem(sizeof(*psDevData));
+		psNewHWPerfDevice->hDevData = (IMG_HANDLE)psDevData;
+		if (!psDevData)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		if (OSSNPrintf(psNewHWPerfDevice->pszName, sizeof(psNewHWPerfDevice->pszName),
+			           "hwperf_device_%d", psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf device name for device %d",
+					__func__,
+					psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		psDevData->psRgxDevNode = psDeviceNode;
+		psDevData->psRgxDevInfo = psDeviceNode->pvDevice;
+
+		psDeviceNode = psDeviceNode->psNext;
+
+		/* At least one device is active */
+		bFWActive = IMG_TRUE;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (!bFWActive)
+	{
+		return PVRSRV_ERROR_NOT_READY;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION *psHWPerfConnection)
+{
+	RGX_KM_HWPERF_DEVDATA *psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo;
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszHWPerfFwStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 5];
+	IMG_CHAR pszHWPerfHostStreamName[sizeof(PVRSRV_TL_HWPERF_HOST_SERVER_STREAM) + 5];
+	IMG_UINT32 ui32BufSize;
+
+	/* Disable producer callback by default for the Kernel API. */
+	IMG_UINT32 ui32StreamFlags = PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING |
+			PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK;
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+		psRgxDevInfo = psDevData->psRgxDevInfo;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* In the case where the AppHint has not been set we need to
+		 * initialise the HWPerf resources here. Allocated on-demand
+		 * to reduce RAM foot print on systems not needing HWPerf.
+		 */
+		OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+		if (RGXHWPerfIsInitRequired(psRgxDevInfo, RGX_HWPERF_L2_STREAM_HWPERF))
+		{
+			eError = RGXHWPerfInitOnDemandL1Buffer(psRgxDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfFW resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hHWPerfLock);
+				return eError;
+			}
+
+			/* if this fails it also cleans up L1 buffer */
+			eError = RGXHWPerfInitOnDemandL2Stream(psRgxDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfFW resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hHWPerfLock);
+				return eError;
+			}
+		}
+		OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+		if (psRgxDevInfo->hHWPerfHostStream == NULL)
+		{
+			eError = RGXHWPerfHostInitOnDemandResources(psRgxDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfHost resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+				return eError;
+			}
+		}
+		OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+		/* form the HWPerf stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfFwStreamName, sizeof(pszHWPerfFwStreamName), "%s%d",
+		               PVRSRV_TL_HWPERF_RGX_FW_STREAM,
+		               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+		/* Open the RGX TL stream for reading in this session */
+		eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+		                            pszHWPerfFwStreamName,
+		                            ui32StreamFlags,
+		                            &psDevData->hSD[RGX_HWPERF_STREAM_ID0_FW]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientOpenStream(RGX_HWPerf)");
+
+		/* form the HWPerf host stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfHostStreamName, sizeof(pszHWPerfHostStreamName), "%s%d",
+		               PVRSRV_TL_HWPERF_HOST_SERVER_STREAM,
+		               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf host stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Open the host TL stream for reading in this session */
+		eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+		                            pszHWPerfHostStreamName,
+		                            PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING,
+		                            &psDevData->hSD[RGX_HWPERF_STREAM_ID1_HOST]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientOpenStream(Host_HWPerf)");
+
+		/* Allocate a large enough buffer for use during the entire session to
+		 * avoid the need to resize in the Acquire call as this might be in an ISR
+		 * Choose size that can contain at least one packet.
+		 */
+		/* Allocate buffer for FW Stream */
+		ui32BufSize = FW_STREAM_BUFFER_SIZE;
+		psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW] = OSAllocMem(ui32BufSize);
+		if (psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW] == NULL)
+		{
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		psDevData->pHwpBufEnd[RGX_HWPERF_STREAM_ID0_FW] = psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW]+ui32BufSize;
+
+		/* Allocate buffer for Host Stream */
+		ui32BufSize = HOST_STREAM_BUFFER_SIZE;
+		psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST] = OSAllocMem(ui32BufSize);
+		if (psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST] == NULL)
+		{
+			OSFreeMem(psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW]);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		psDevData->pHwpBufEnd[RGX_HWPERF_STREAM_ID1_HOST] = psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST]+ui32BufSize;
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXHWPerfLazyConnect(ppsHWPerfConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfLazyConnect", e0);
+
+	eError = RGXHWPerfOpen(*ppsHWPerfConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfOpen", e1);
+
+	return PVRSRV_OK;
+
+e1: /* HWPerfOpen might have opened some, and then failed */
+	RGXHWPerfClose(*ppsHWPerfConnection);
+e0: /* LazyConnect might have allocated some resources and then failed,
+	 * make sure they are cleaned up */
+	RGXHWPerfFreeConnection(ppsHWPerfConnection);
+	return eError;
+}
+
+/*
+    PVRSRVRGXControlHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXControlHWPerfBlocksKM(
+		CONNECTION_DATA             * psConnection,
+		PVRSRV_DEVICE_NODE          * psDeviceNode,
+		IMG_BOOL                      bEnable,
+		IMG_UINT32                    ui32ArrayLen,
+		IMG_UINT16                  * psBlockIDs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_ASSERT(psDeviceNode);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBlockIDs != NULL, "psBlockIDs");
+	PVR_LOG_RETURN_IF_INVALID_PARAM((ui32ArrayLen>0) && (ui32ArrayLen <= RGXFWIF_HWPERF_CTRL_BLKS_MAX), "ui32ArrayLen");
+
+	psDevice = psDeviceNode->pvDevice;
+
+	/* Fill in the command structure with the parameters needed
+	 */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCtrlBlks.bEnable = bEnable;
+	sKccbCmd.uCmdData.sHWPerfCtrlBlks.ui32NumBlocks = ui32ArrayLen;
+
+	OSDeviceMemCopy(sKccbCmd.uCmdData.sHWPerfCtrlBlks.aeBlockIDs, psBlockIDs, sizeof(IMG_UINT16) * ui32ArrayLen);
+
+
+	/* Ask the FW to carry out the HWPerf configuration command
+	 */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot");
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+
+	if (bEnable)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks have been ENABLED", ui32ArrayLen));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks have been DISABLED", ui32ArrayLen));
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/*
+	PVRSRVRGXCtrlHWPerfKM
+ */
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		RGX_HWPERF_STREAM_ID     eStreamId,
+		IMG_BOOL                 bToggle,
+		IMG_UINT64               ui64Mask)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_ASSERT(psDeviceNode);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_DPF_ENTERED;
+
+	if (eStreamId == RGX_HWPERF_STREAM_ID0_FW)
+	{
+		return RGXHWPerfCtrlFwBuffer(psDeviceNode, RGX_HWPERF_L2_STREAM_HWPERF,
+		                             bToggle, ui64Mask);
+	}
+	else if (eStreamId == RGX_HWPERF_STREAM_ID1_HOST)
+	{
+		return RGXHWPerfCtrlHostBuffer(psDeviceNode, bToggle, (IMG_UINT32) ui64Mask);
+	}
+	else if (eStreamId == RGX_HWPERF_STREAM_ID2_CLIENT)
+	{
+		IMG_UINT32 ui32Index = (IMG_UINT32) (ui64Mask >> 32);
+		IMG_UINT32 ui32Mask = (IMG_UINT32) ui64Mask;
+
+		return RGXHWPerfCtrlClientBuffer(bToggle, ui32Index, ui32Mask);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXCtrlHWPerfKM: Unknown stream id."));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfFW(
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		RGX_HWPERF_L2_STREAM_ID  eL2StreamId,
+		IMG_UINT64               ui64Mask,
+		HWPERF_FILTER_OPERATION  eMaskOp)
+{
+	IMG_UINT64 uiTmpFilter;
+
+	PVR_DPF_ENTERED;
+	PVR_ASSERT(psDeviceNode);
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	uiTmpFilter =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui64HWPerfFilter[eL2StreamId];
+
+	switch (eMaskOp)
+	{
+		case HWPERF_FILTER_OPERATION_SET:
+			uiTmpFilter = ui64Mask;
+			break;
+		case HWPERF_FILTER_OPERATION_BIT_CLR:
+			uiTmpFilter &= ~ui64Mask;
+			break;
+		case HWPERF_FILTER_OPERATION_BIT_OR:
+			uiTmpFilter |= ui64Mask;
+			break;
+	}
+
+	PVR_DPF_RETURN_RC(RGXHWPerfCtrlFwBuffer(psDeviceNode, eL2StreamId, IMG_FALSE,
+	                  uiTmpFilter));
+}
+
+/*
+	PVRSRVRGXGetHWPerfTimeStampKM
+ */
+PVRSRV_ERROR PVRSRVRGXGetHWPerfTimeStampKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		IMG_UINT64              *pui64TimeStamp)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	*pui64TimeStamp = RGXTimeCorrGetClockus64(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE* psHWPerfDev;
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXCtrlHWPerfKM(NULL, psDevData->psRgxDevNode, eStreamId, bToggle, ui64Mask);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR RGXHWPerfToggleCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs,
+		IMG_BOOL     bToggle,
+		const char* szFunctionString);
+
+IMG_INTERNAL PVRSRV_ERROR RGXHWPerfToggleCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs,
+		IMG_BOOL     bToggle,
+		const char* szFunctionString)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE*     psHWPerfDev;
+
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !aeBlockIDs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXControlHWPerfBlocksKM(NULL,
+		                                        psDevData->psRgxDevNode,
+		                                        bToggle,
+		                                        ui32NumBlocks,
+		                                        aeBlockIDs);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, szFunctionString);
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32            ui32NumBlocks,
+		IMG_UINT16*           aeBlockIDs)
+{
+	return RGXHWPerfToggleCounters(psHWPerfConnection,
+			                        ui32NumBlocks,
+			                        aeBlockIDs,
+			                        IMG_FALSE,
+			                        __func__);
+}
+
+
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32            ui32NumBlocks,
+		IMG_UINT16*           aeBlockIDs)
+{
+	return RGXHWPerfToggleCounters(psHWPerfConnection,
+			                        ui32NumBlocks,
+			                        aeBlockIDs,
+			                        IMG_TRUE,
+			                        __func__);
+}
+
+
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen)
+{
+	PVRSRV_ERROR			eError;
+	RGX_KM_HWPERF_DEVDATA*	psDevData = (RGX_KM_HWPERF_DEVDATA*)hDevData;
+	IMG_PBYTE				pDataDest;
+	IMG_UINT32			ui32TlPackets = 0;
+	IMG_PBYTE			pBufferEnd;
+	PVRSRVTL_PPACKETHDR psHDRptr;
+	PVRSRVTL_PACKETTYPE ui16TlType;
+
+	/* Reset the output arguments in case we discover an error */
+	*ppBuf = NULL;
+	*pui32BufLen = 0;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psDevData || eStreamId >= RGX_HWPERF_MAX_STREAM_ID)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psDevData->pTlBuf[eStreamId] == NULL)
+	{
+		/* Acquire some data to read from the HWPerf TL stream */
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		                             psDevData->hSD[eStreamId],
+		                             &psDevData->pTlBuf[eStreamId],
+		                             &psDevData->ui32AcqDataLen[eStreamId]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientAcquireData");
+
+		psDevData->pTlBufPos[eStreamId] = psDevData->pTlBuf[eStreamId];
+	}
+
+	/* TL indicates no data exists so return OK and zero. */
+	if ((psDevData->pTlBufPos[eStreamId] == NULL) || (psDevData->ui32AcqDataLen[eStreamId] == 0))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Process each TL packet in the data buffer we have acquired */
+	pBufferEnd = psDevData->pTlBuf[eStreamId]+psDevData->ui32AcqDataLen[eStreamId];
+	pDataDest = psDevData->pHwpBuf[eStreamId];
+	psHDRptr = GET_PACKET_HDR(psDevData->pTlBufPos[eStreamId]);
+	psDevData->pTlBufRead[eStreamId] = psDevData->pTlBufPos[eStreamId];
+	while (psHDRptr < (PVRSRVTL_PPACKETHDR)((void *)pBufferEnd))
+	{
+		ui16TlType = GET_PACKET_TYPE(psHDRptr);
+		if (ui16TlType == PVRSRVTL_PACKETTYPE_DATA)
+		{
+			IMG_UINT16 ui16DataLen = GET_PACKET_DATA_LEN(psHDRptr);
+			if (0 == ui16DataLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfAcquireEvents: ZERO Data in TL data packet: %p", psHDRptr));
+			}
+			else
+			{
+				/* Check next packet does not fill buffer */
+				if (pDataDest + ui16DataLen > psDevData->pHwpBufEnd[eStreamId])
+				{
+					break;
+				}
+
+				/* For valid data copy it into the client buffer and move
+				 * the write position on */
+				OSDeviceMemCopy(pDataDest, GET_PACKET_DATA_PTR(psHDRptr), ui16DataLen);
+				pDataDest += ui16DataLen;
+			}
+		}
+		else if (ui16TlType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfAcquireEvents: Indication that the transport buffer was full"));
+		}
+		else
+		{
+			/* else Ignore padding packet type and others */
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfAcquireEvents: Ignoring TL packet, type %d", ui16TlType ));
+		}
+
+		/* Update loop variable to the next packet and increment counts */
+		psHDRptr = GET_NEXT_PACKET_ADDR(psHDRptr);
+		/* Updated to keep track of the next packet to be read. */
+		psDevData->pTlBufRead[eStreamId] = (IMG_PBYTE) ((void *)psHDRptr);
+		ui32TlPackets++;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "RGXHWPerfAcquireEvents: TL Packets processed %03d", ui32TlPackets));
+
+	psDevData->bRelease[eStreamId] = IMG_FALSE;
+	if (psHDRptr >= (PVRSRVTL_PPACKETHDR)((void *)pBufferEnd))
+	{
+		psDevData->bRelease[eStreamId] = IMG_TRUE;
+	}
+
+	/* Update output arguments with client buffer details and true length */
+	*ppBuf = psDevData->pHwpBuf[eStreamId];
+	*pui32BufLen = pDataDest - psDevData->pHwpBuf[eStreamId];
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId)
+{
+	PVRSRV_ERROR			eError = PVRSRV_OK;
+	RGX_KM_HWPERF_DEVDATA*	psDevData = (RGX_KM_HWPERF_DEVDATA*)hDevData;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psDevData || eStreamId >= RGX_HWPERF_MAX_STREAM_ID)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psDevData->bRelease[eStreamId])
+	{
+		/* Inform the TL that we are done with reading the data. */
+		eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, psDevData->hSD[eStreamId]);
+		psDevData->ui32AcqDataLen[eStreamId] = 0;
+		psDevData->pTlBuf[eStreamId] = NULL;
+	}
+	else
+	{
+		psDevData->pTlBufPos[eStreamId] = psDevData->pTlBufRead[eStreamId];
+	}
+	return eError;
+}
+
+
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo =
+			hDevData ? ((RGX_KM_HWPERF_DEVDATA*) hDevData)->psRgxDevInfo : NULL;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psRgxDevInfo)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid pointer to the RGX device",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	/* No need to take hHWPerfLock here since we are only reading data
+	 * from always existing integers to return to debugfs which is an
+	 * atomic operation.
+	 */
+	switch (eStreamId) {
+		case RGX_HWPERF_STREAM_ID0_FW:
+			*ui64Filter = psRgxDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF];
+			break;
+		case RGX_HWPERF_STREAM_ID1_HOST:
+			*ui64Filter = psRgxDevInfo->ui32HWPerfHostFilter;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid stream ID",
+					__func__));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	RGX_HWPERF_DEVICE *psHWPerfDev, *psHWPerfNextDev;
+	RGX_HWPERF_CONNECTION *psHWPerfConnection = *ppsHWPerfConnection;
+
+	/* if connection object itself is NULL, nothing to free */
+	if (psHWPerfConnection == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	psHWPerfNextDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfNextDev)
+	{
+		psHWPerfDev = psHWPerfNextDev;
+		psHWPerfNextDev = psHWPerfNextDev->psNext;
+
+		/* Free the session memory */
+		if (psHWPerfDev->hDevData)
+			OSFreeMem(psHWPerfDev->hDevData);
+		OSFreeMem(psHWPerfDev);
+	}
+	OSFreeMem(psHWPerfConnection);
+	*ppsHWPerfConnection = NULL;
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection)
+{
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	IMG_UINT uiStreamId;
+	PVRSRV_ERROR eError;
+
+	/* Check session connection is not zero */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		for (uiStreamId = 0; uiStreamId < RGX_HWPERF_MAX_STREAM_ID; uiStreamId++)
+		{
+			/* If the TL buffer exists they have not called ReleaseData
+			 * before disconnecting so clean it up */
+			if (psDevData->pTlBuf[uiStreamId])
+			{
+				/* TLClientReleaseData call and null out the buffer fields
+				 * and length */
+				eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, psDevData->hSD[uiStreamId]);
+				psDevData->ui32AcqDataLen[uiStreamId] = 0;
+				psDevData->pTlBuf[uiStreamId] = NULL;
+				PVR_LOG_IF_ERROR(eError, "TLClientReleaseData");
+				/* Packets may be lost if release was not required */
+				if (!psDevData->bRelease[uiStreamId])
+				{
+					PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfClose: Events in buffer waiting to be read, remaining events may be lost."));
+				}
+			}
+
+			/* Close the TL stream, ignore the error if it occurs as we
+			 * are disconnecting */
+			if (psDevData->hSD[uiStreamId])
+			{
+				eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+				                             psDevData->hSD[uiStreamId]);
+				PVR_LOG_IF_ERROR(eError, "TLClientCloseStream");
+				psDevData->hSD[uiStreamId] = NULL;
+			}
+
+			/* Free the client buffer used in session */
+			if (psDevData->pHwpBuf[uiStreamId])
+			{
+				OSFreeMem(psDevData->pHwpBuf[uiStreamId]);
+				psDevData->pHwpBuf[uiStreamId] = NULL;
+			}
+		}
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXHWPerfClose(*ppsHWPerfConnection);
+	PVR_LOG_IF_ERROR(eError, "RGXHWPerfClose");
+
+	eError = RGXHWPerfFreeConnection(ppsHWPerfConnection);
+	PVR_LOG_IF_ERROR(eError, "RGXHWPerfFreeConnection");
+
+	return eError;
+}
+
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp)
+{
+	IMG_UINT64 ui64CRDeltaToOSDeltaKNs;
+	IMG_UINT64 ui64EventOSTimestamp, deltaRgxTimer, delta_ns;
+
+	if (!(ui64CRTimeStamp) || !(ui32ClkSpeed) || !(ui64CorrCRTimeStamp) || !(ui64CorrOSTimeStamp))
+	{
+		return 0;
+	}
+
+	ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(ui32ClkSpeed);
+
+	/* RGX CR timer ticks delta */
+	deltaRgxTimer = ui64CRTimeStamp - ui64CorrCRTimeStamp;
+	/* RGX time delta in nanoseconds */
+	delta_ns = RGXFWIF_GET_DELTA_OSTIME_NS(deltaRgxTimer, ui64CRDeltaToOSDeltaKNs);
+	/* Calculate OS time of HWPerf event */
+	ui64EventOSTimestamp = ui64CorrOSTimeStamp + delta_ns;
+
+	return ui64EventOSTimestamp;
+}
+
+#define CLIENT_STREAM_REFCOUNT_MIN 0
+#define CLIENT_STREAM_REFCOUNT_MAX IMG_INT32_MAX
+
+struct HWPERF_STREAM_DESC_TAG
+{
+    TL_STREAM_DESC *psSD;
+    PMR *psPMR;
+    POS_LOCK hLock;
+    ATOMIC_T iRefCount;
+    IMG_UINT32 uiOrdinal;
+};
+
+PVRSRV_ERROR PVRSRVRGXOpenHWPerfClientStreamKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               HWPERF_STREAM_DESC **ppsSD)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	HWPERF_STREAM_DESC *psHwpSD;
+	TL_STREAM_DESC *psTlSD;
+	PMR *psPMR;
+	PVRSRV_ERROR eError;
+	IMG_CHAR acStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+
+	OSSNPrintf(acStreamName, sizeof(acStreamName),
+	           PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC,
+	           psDeviceNode->sDevId.i32KernelDeviceID,
+	           psConnection->pid);
+
+	OSLockAcquire(psPVRSRVData->hClientStreamTableLock);
+
+	psHwpSD = (void *) HASH_Retrieve_Extended(psPVRSRVData->psClientStreamTable,
+	                                          (void *) acStreamName);
+	if (psHwpSD != NULL)
+	{
+		IMG_INT32 iOldRefCount = OSAtomicAddUnless(&psHwpSD->iRefCount, 1,
+		                                           CLIENT_STREAM_REFCOUNT_MAX);
+		if (iOldRefCount == CLIENT_STREAM_REFCOUNT_MAX)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("HWPerf stream reference count overflow",
+			                        eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW,
+			                        ErrUnlockAndReturnError);
+		}
+
+		*ppsSD = psHwpSD;
+
+		OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+		return PVRSRV_OK;
+	}
+
+	psHwpSD = OSAllocMem(sizeof(*psHwpSD));
+	PVR_LOG_GOTO_IF_NOMEM(psHwpSD, eError, ErrUnlockAndReturnError);
+
+	eError = OSLockCreate(&psHwpSD->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLServerOpenStreamKM", ErrFreeSD);
+
+	eError = TLServerOpenStreamKM(acStreamName, PVRSRV_STREAM_FLAG_OPEN_WO,
+	                              &psTlSD, &psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLServerOpenStreamKM", ErrFreeLock);
+
+	psHwpSD->psSD = psTlSD;
+	psHwpSD->psPMR = psPMR;
+	psHwpSD->uiOrdinal = IMG_UINT32_MAX;
+	OSAtomicWrite(&psHwpSD->iRefCount, CLIENT_STREAM_REFCOUNT_MIN + 1);
+
+	if (!HASH_Insert_Extended(psPVRSRVData->psClientStreamTable, (void *) acStreamName,
+	                          (uintptr_t) psHwpSD))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("HASH_Insert", eError, PVRSRV_ERROR_OUT_OF_MEMORY,
+		                        ErrCloseStream);
+	}
+
+	OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+	*ppsSD = psHwpSD;
+
+	return PVRSRV_OK;
+
+ErrCloseStream:
+	TLServerCloseStreamKM(psTlSD);
+ErrFreeLock:
+	OSLockDestroy(psHwpSD->hLock);
+ErrFreeSD:
+	OSFreeMem(psHwpSD);
+ErrUnlockAndReturnError:
+	OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+	return eError;
+}
+
+/* This function is called from the handle framework's handle destruction
+ * path which means there should not be any lookup outstanding, i.e. the handle
+ * should not be used at the given moment. */
+PVRSRV_ERROR PVRSRVRGXCloseHWPerfClientStreamKM(HWPERF_STREAM_DESC *psSD)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_INT iOldRefCount;
+	PVRSRV_ERROR eError;
+	void *psElement;
+
+	PVR_ASSERT(psSD != NULL);
+
+	iOldRefCount = OSAtomicSubtractUnless(&psSD->iRefCount, 1, CLIENT_STREAM_REFCOUNT_MIN);
+	if (iOldRefCount == CLIENT_STREAM_REFCOUNT_MIN)
+	{
+		/* If the old value is 0 return error. This means that something went
+		 * wrong and the reference count was already 0 and resources were
+		 * freed. */
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+	else if (iOldRefCount != (CLIENT_STREAM_REFCOUNT_MIN + 1))
+	{
+		/* If the old value is not 1 then return ok, the stream is still used
+		 * by someone. If it's 1, then progress with the resources destruction
+		 * since the current value is 0. */
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(iOldRefCount > 0);
+
+	OSLockAcquire(psPVRSRVData->hClientStreamTableLock);
+
+	psElement = (void *) HASH_Remove_Extended(psPVRSRVData->psClientStreamTable,
+	                                          psSD->psSD->psNode->psStream->szName);
+	PVR_ASSERT(psElement != NULL);
+
+	OSLockRelease(psPVRSRVData->hClientStreamTableLock);
+
+	eError = TLServerCloseStreamKM(psSD->psSD);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLServerCloseStreamKM", ErrReturnError);
+
+	OSLockDestroy(psSD->hLock);
+
+	OSFreeMem(psSD);
+
+	return PVRSRV_OK;
+
+ErrReturnError:
+	return eError;
+}
+
+/* This function is called from the handle framework's lookup path which means
+ * there should not be a valid pointer to the looked up resource, i.e. no
+ * destructor is allowed to run on the resource. */
+PVRSRV_ERROR PVRSRVRGXWriteHWPerfClientEventKM(HWPERF_STREAM_DESC *psSD,
+                                               IMG_UINT32 uiSize,
+                                               IMG_BYTE *puiData)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader = (RGX_HWPERF_V2_PACKET_HDR *) ((void *)puiData);
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSD != NULL);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiSize > sizeof(*psHeader), "uiSize");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeader->ui32Size == uiSize, "uiSize");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeader->ui32Sig == HWPERF_PACKET_V2B_SIG, "ui32Sig");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeader->ui32Ordinal == 0, "ui32Ordinal");
+
+	OSLockAcquire(psSD->hLock);
+
+	psHeader->ui32Ordinal = ++psSD->uiOrdinal;
+
+	eError = TLServerWriteDataKM(psSD->psSD, uiSize, puiData);
+
+	OSLockRelease(psSD->hLock);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf_common.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.h
new file mode 100644
index 000000000000..6f1b932f3322
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxhwperf_common.h
@@ -0,0 +1,655 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX HWPerf functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHWPERF_COMMON_H_
+#define RGXHWPERF_COMMON_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#include "device.h"
+#include "connection_server.h"
+#include "rgxdevice.h"
+#include "rgx_hwperf.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(PVR_ARCH_VOLCANIC)
+#include "rgx_hwperf_table.h"
+#endif
+#include "cache_ops.h"
+#include "rgxfwmemctx.h"
+
+/* HWPerf host buffer size constraints in KBs */
+#define HWPERF_HOST_TL_STREAM_SIZE_DEFAULT PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB
+#define HWPERF_HOST_TL_STREAM_SIZE_MIN     (32U)
+#define HWPERF_HOST_TL_STREAM_SIZE_MAX     (3072U)
+
+/* Operations on HWPerf filter. */
+typedef enum HWPERF_FILTER_OPERATION_TAG
+{
+	HWPERF_FILTER_OPERATION_SET,
+	HWPERF_FILTER_OPERATION_BIT_CLR,
+	HWPERF_FILTER_OPERATION_BIT_OR,
+} HWPERF_FILTER_OPERATION;
+
+/******************************************************************************
+ * RGX HW Performance decode Bvnc Features for HWPerf
+ *****************************************************************************/
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+		                                        RGX_HWPERF_BVNC    *psBVNC);
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(CONNECTION_DATA    *psConnection,
+                                                  PVRSRV_DEVICE_NODE *psDeviceNode,
+											      RGX_HWPERF_BVNC    *psBVNC);
+
+/******************************************************************************
+ * RGX HW Performance Data Transport Routines
+ *****************************************************************************/
+
+PVRSRV_ERROR RGXHWPerfDataStoreCB(PVRSRV_DEVICE_NODE* psDevInfo);
+
+PVRSRV_ERROR RGXHWPerfInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+void RGXHWPerfDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+
+PVRSRV_ERROR RGXHWPerfInitOnDemandL1Buffer(PVRSRV_RGXDEV_INFO* psRgxDevInfo);
+PVRSRV_ERROR RGXHWPerfInitOnDemandL2Stream(PVRSRV_RGXDEV_INFO* psRgxDevInfo,
+                                           RGX_HWPERF_L2_STREAM_ID eL2StreamId);
+void RGXHWPerfDeinitL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                             RGX_HWPERF_L2_STREAM_ID eL2StreamId);
+void RGXHWPerfInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+void RGXHWPerfClientInitAppHintCallbacks(void);
+
+static INLINE PVRSRV_ERROR RGXAcquireHWPerfCtlCPUAddr(PVRSRV_DEVICE_NODE *psDevNode,
+                                                      RGXFWIF_HWPERF_CTL **ppsHWPerfCtl)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+	                                  (void**)ppsHWPerfCtl);
+	RGXFwSharedMemCacheOpPtr(ppsHWPerfCtl, INVALIDATE);
+
+	return eError;
+}
+
+static INLINE void RGXReleaseHWPerfCtlCPUAddr(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevNode != NULL, "psDevNode is invalid");
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevInfo != NULL, "psDevInfo invalid");
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCountersKM(CONNECTION_DATA *psConnection,
+                                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                    const IMG_UINT32 ui32BlockID,
+                                                    RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocksKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               const IMG_UINT32 ui32ArrayLen,
+                                               IMG_UINT32 *pui32BlockCount,
+                                               IMG_UINT32 *pui32EnabledBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling API(s)
+ *****************************************************************************/
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfKM(
+	CONNECTION_DATA      * psConnection,
+	PVRSRV_DEVICE_NODE   * psDeviceNode,
+	 RGX_HWPERF_STREAM_ID  eStreamId,
+	IMG_BOOL               bToggle,
+	IMG_UINT64             ui64Mask);
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfFW(
+	PVRSRV_DEVICE_NODE      *psDeviceNode,
+	RGX_HWPERF_L2_STREAM_ID  eL2StreamId,
+	IMG_UINT64               ui64Mask,
+	HWPERF_FILTER_OPERATION  eMaskOp);
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfTimeStampKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		IMG_UINT64              *pui64TimeStamp);
+
+PVRSRV_ERROR PVRSRVRGXControlHWPerfBlocksKM(
+	CONNECTION_DATA       * psConnection,
+	PVRSRV_DEVICE_NODE    * psDeviceNode,
+	IMG_BOOL              bEnable,
+	IMG_UINT32            ui32ArrayLen,
+	IMG_UINT16            * psBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Host Stream API
+ *****************************************************************************/
+
+static inline RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS
+RGXHWPerfConvDeviceHealthStatus(PVRSRV_DEVICE_HEALTH_STATUS eDeviceHealthStatus)
+{
+	switch (eDeviceHealthStatus)
+	{
+		case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:         return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED;
+		case PVRSRV_DEVICE_HEALTH_STATUS_OK:                return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:    return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+		case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:              return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_DEAD;
+		case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:             return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_FAULT;
+		default:                                            return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED;
+	}
+}
+
+static inline RGX_HWPERF_HOST_DEVICE_HEALTH_REASON
+RGXHWPerfConvDeviceHealthReason(PVRSRV_DEVICE_HEALTH_REASON eDeviceHealthReason)
+{
+	switch (eDeviceHealthReason)
+	{
+		case PVRSRV_DEVICE_HEALTH_REASON_NONE:                  return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_NONE;
+		case PVRSRV_DEVICE_HEALTH_REASON_ASSERTED:              return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_ASSERTED;
+		case PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING:          return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_POLL_FAILING;
+		case PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS:              return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_TIMEOUTS;
+		case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT:         return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+		case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED:         return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+		case PVRSRV_DEVICE_HEALTH_REASON_IDLING:                return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_IDLING;
+		case PVRSRV_DEVICE_HEALTH_REASON_RESTARTING:            return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_RESTARTING;
+		case PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS:    return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+		case PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR:             return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_PCI_ERROR;
+		default:                                                return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_UNDEFINED;
+	}
+}
+
+PVRSRV_ERROR RGXHWPerfHostInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32BufSizeKB);
+PVRSRV_ERROR RGXHWPerfHostInitOnDemandResources(PVRSRV_RGXDEV_INFO* psRgxDevInfo);
+void RGXHWPerfHostDeInit(PVRSRV_RGXDEV_INFO	*psRgxDevInfo);
+
+IMG_UINT64 RGXHWPerfFwSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                     RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                     IMG_UINT64 uiFilter);
+
+void RGXHWPerfHostSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 IMG_UINT32 ui32Filter);
+
+void RGXHWPerfHostPostRaw(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+						  RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+						  IMG_BYTE *pbPayload,
+						  IMG_UINT32 ui32PayloadSize);
+
+void RGXHWPerfHostPostEnqEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_KICK_TYPE eEnqType,
+                               IMG_UINT32 ui32Pid,
+                               IMG_UINT32 ui32FWDMContext,
+                               IMG_UINT32 ui32ExtJobRef,
+                               IMG_UINT32 ui32IntJobRef,
+                               PVRSRV_FENCE hCheckFence,
+                               PVRSRV_FENCE hUpdateFence,
+                               PVRSRV_TIMELINE hUpdateTimeline,
+                               IMG_UINT64 ui64CheckFenceUID,
+                               IMG_UINT64 ui64UpdateFenceUID,
+                               IMG_UINT64 ui64DeadlineInus,
+                               IMG_UINT32 ui32CycleEstimate);
+
+void RGXHWPerfHostPostAllocEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                 const IMG_CHAR *psName,
+                                 IMG_UINT32 ui32NameSize,
+                                 RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail);
+
+void RGXHWPerfHostPostFreeEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                IMG_UINT64 ui64UID,
+                                IMG_UINT32 ui32PID,
+                                IMG_UINT32 ui32FWAddr);
+
+void RGXHWPerfHostPostModifyEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                  RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                  IMG_UINT64 ui64NewUID,
+                                  IMG_UINT64 ui64UID1,
+                                  IMG_UINT64 ui64UID2,
+                                  const IMG_CHAR *psName,
+                                  IMG_UINT32 ui32NameSize);
+
+void RGXHWPerfHostPostUfoEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_UFO_EV eUfoType,
+                               RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData,
+							   const IMG_BOOL bSleepAllowed);
+
+void RGXHWPerfHostPostClkSyncEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+
+void RGXHWPerfHostPostDeviceInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_DEV_INFO_EV eEvType,
+								 RGX_HWPERF_HOST_DEV_INFO_DETAIL *psData);
+
+void RGXHWPerfHostPostInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_INFO_EV eEvType);
+
+void RGXHWPerfHostPostFenceWait(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType,
+								IMG_PID uiPID,
+								PVRSRV_FENCE hFence,
+								IMG_UINT32 ui32Data);
+
+void RGXHWPerfHostPostSWTimelineAdv(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                    IMG_PID uiPID,
+									PVRSRV_TIMELINE hSWTimeline,
+									IMG_UINT64 ui64SyncPtIndex);
+
+void RGXHWPerfHostPostClientInfoProcName(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_PID uiPID,
+									     const IMG_CHAR *psName);
+
+IMG_BOOL RGXHWPerfHostIsEventEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo, RGX_HWPERF_HOST_EVENT_TYPE eEvent);
+
+#if defined(PVR_ARCH_VOLCANIC)
+IMG_INTERNAL /*static inline*/ IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel);
+
+/*!
+*******************************************************************************
+ @Function    RGXHWPerfMaxDefinedBlks
+
+ @Description Return the number of valid block-IDs for the given device node
+
+ @Input       (PVRSRV_RGXDEV_INFO *)   pvDevice    device-node to query
+
+ @Returns     (IMG_UINT32)             Number of block-IDs (RGX_CNTBLK_ID)
+                                       valid for this device.
+******************************************************************************/
+IMG_INTERNAL IMG_UINT32
+RGXHWPerfMaxDefinedBlks(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#define _RGX_HWPERF_HOST_FILTER(CTX, EV) \
+		(((PVRSRV_RGXDEV_INFO *)CTX->psDeviceNode->pvDevice)->ui32HWPerfHostFilter \
+		& RGX_HWPERF_EVENT_MASK_VALUE(EV))
+
+#define _RGX_DEVICE_INFO_FROM_CTX(CTX) \
+		((PVRSRV_RGXDEV_INFO *)CTX->psDeviceNode->pvDevice)
+
+#define _RGX_DEVICE_INFO_FROM_NODE(DEVNODE) \
+		((PVRSRV_RGXDEV_INFO *)DEVNODE->pvDevice)
+
+/* Deadline and cycle estimate is not supported for all ENQ events */
+#define NO_DEADLINE 0
+#define NO_CYCEST   0
+
+
+#if defined(SUPPORT_RGX)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param C      Kick context
+ * @param P      Pid of kicking process
+ * @param X      Related FW context
+ * @param E      External job reference
+ * @param I      Job ID
+ * @param K      Kick type
+ * @param CF     Check fence handle
+ * @param UF     Update fence handle
+ * @param UT     Update timeline (on which above UF was created) handle
+ * @param CHKUID Check fence UID
+ * @param UPDUID Update fence UID
+ * @param D      Deadline
+ * @param CE     Cycle estimate
+ */
+#define RGXSRV_HWPERF_ENQ(C, P, X, E, I, K, CF, UF, UT, CHKUID, UPDUID, D, CE) \
+		do { \
+			if (_RGX_HWPERF_HOST_FILTER(C, RGX_HWPERF_HOST_ENQ)) \
+			{ \
+				RGXHWPerfHostPostEnqEvent(_RGX_DEVICE_INFO_FROM_CTX(C), \
+				                          (K), (P), (X), (E), (I), \
+				                          (CF), (UF), (UT), \
+				                          (CHKUID), (UPDUID), (D), (CE)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I Device Info pointer
+ * @param T Host UFO event type
+ * @param D Pointer to UFO data
+ * @param S Is sleeping allowed?
+ */
+#define RGXSRV_HWPERF_UFO(I, T, D, S) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_UFO)) \
+			{ \
+				RGXHWPerfHostPostUfoEvent((I), (T), (D), (S)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device node pointer
+ * @param T Host ALLOC event type
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC(D, T, FWADDR, N, Z) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSyncAlloc.ui32FWAddr = (FWADDR); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+											(N), (Z), &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param PID ID of allocating process
+ * @param FENCE PVRSRV_FENCE object
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_FENCE(D, PID, FENCE, FWADDR, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sFenceAlloc.uiPID = (PID); \
+				uAllocDetail.sFenceAlloc.hFence = (FENCE); \
+				uAllocDetail.sFenceAlloc.ui32CheckPt_FWAddr = (FWADDR); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * @param D Device Node pointer
+ * @param TL PVRSRV_TIMELINE on which CP is allocated
+ * @param PID Allocating process ID of this TL/FENCE
+ * @param FENCE PVRSRV_FENCE as passed to SyncCheckpointResolveFence OR PVRSRV_NO_FENCE
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_SYNC_CP(D, TL, PID, FENCE, FWADDR, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSyncCheckPointAlloc.ui32CheckPt_FWAddr = (FWADDR); \
+				uAllocDetail.sSyncCheckPointAlloc.hTimeline = (TL); \
+				uAllocDetail.sSyncCheckPointAlloc.uiPID = (PID); \
+				uAllocDetail.sSyncCheckPointAlloc.hFence = (FENCE); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * @param D Device Node pointer
+ * @param PID ID of allocating process
+ * @param SW_FENCE PVRSRV_FENCE object
+ * @param SW_TL PVRSRV_TIMELINE on which SW_FENCE is allocated
+ * @param SPI Sync point index on the SW_TL on which this SW_FENCE is allocated
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_SW_FENCE(D, PID, SW_FENCE, SW_TL, SPI, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSWFenceAlloc.uiPID = (PID); \
+				uAllocDetail.sSWFenceAlloc.hSWFence = (SW_FENCE); \
+				uAllocDetail.sSWFenceAlloc.hSWTimeline = (SW_TL); \
+				uAllocDetail.sSWFenceAlloc.ui64SyncPtIndex = (SPI); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param FWADDR sync firmware address
+ */
+#define RGXSRV_HWPERF_FREE(D, T, FWADDR) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_FREE)) \
+			{ \
+				RGXHWPerfHostPostFreeEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                           RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                           (0), (0), (FWADDR)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param UID ID of input object
+ * @param PID ID of allocating process
+ * @param FWADDR sync firmware address
+ */
+#define RGXSRV_HWPERF_FREE_FENCE_SYNC(D, T, UID, PID, FWADDR) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_FREE)) \
+			{ \
+				RGXHWPerfHostPostFreeEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                           RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                           (UID), (PID), (FWADDR)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param NEWUID ID of output object
+ * @param UID1 ID of first input object
+ * @param UID2 ID of second input object
+ * @param N string containing new object's name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_MODIFY_FENCE_SYNC(D, T, NEWUID, UID1, UID2, N, Z) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_MODIFY)) \
+			{ \
+				RGXHWPerfHostPostModifyEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                             RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                             (NEWUID), (UID1), (UID2), N, Z); \
+			} \
+		} while (0)
+
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I Device info pointer
+ */
+#define RGXSRV_HWPERF_CLK_SYNC(I) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_CLK_SYNC)) \
+			{ \
+				RGXHWPerfHostPostClkSyncEvent((I)); \
+			} \
+		} while (0)
+
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts a device info health event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ * @param H      Health status enum
+ * @param R      Health reason enum
+ */
+#define RGXSRV_HWPERF_DEVICE_INFO_HEALTH(I, H, R) \
+	do { \
+		if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_DEV_INFO)) \
+		{ \
+			RGX_HWPERF_HOST_DEV_INFO_DETAIL uDevDetail; \
+			uDevDetail.sDeviceStatus.eDeviceHealthStatus = RGXHWPerfConvDeviceHealthStatus(H); \
+			uDevDetail.sDeviceStatus.eDeviceHealthReason = RGXHWPerfConvDeviceHealthReason(R); \
+			RGXHWPerfHostPostDeviceInfo((I), RGX_HWPERF_DEV_INFO_EV_HEALTH, &uDevDetail); \
+		} \
+	} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts a device info features event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ */
+#define RGXSRV_HWPERF_DEVICE_INFO_FEATURES(I) \
+	do { \
+		if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_DEV_INFO)) \
+		{ \
+			RGXHWPerfHostPostDeviceInfo((I), RGX_HWPERF_DEV_INFO_EV_FEATURES, NULL); \
+		} \
+	} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ * @param T      Event type
+ */
+#define RGXSRV_HWPERF_HOST_INFO(I, T) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_INFO)) \
+	{ \
+		RGXHWPerfHostPostInfo((I), (T)); \
+	} \
+} while (0)
+
+/**
+ * @param I      Device info pointer
+ * @param T      Wait Event type
+ * @param PID    Process ID that the following fence belongs to
+ * @param F      Fence handle
+ * @param D      Data for this wait event type
+ */
+#define RGXSRV_HWPERF_SYNC_FENCE_WAIT(I, T, PID, F, D) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_SYNC_FENCE_WAIT)) \
+	{ \
+		RGXHWPerfHostPostFenceWait(I, RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_##T, \
+		                           (PID), (F), (D)); \
+	} \
+} while (0)
+
+/**
+ * @param I      Device info pointer
+ * @param PID    Process ID that the following timeline belongs to
+ * @param F      SW-timeline handle
+ * @param SPI    Sync-pt index where this SW-timeline has reached
+ */
+#define RGXSRV_HWPERF_SYNC_SW_TL_ADV(I, PID, SW_TL, SPI)\
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE)) \
+	{ \
+		RGXHWPerfHostPostSWTimelineAdv((I), (PID), (SW_TL), (SPI)); \
+	} \
+} while (0)
+
+/**
+ * @param D      Device Node pointer
+ * @param PID    Process ID that the following timeline belongs to
+ * @param N      Null terminated string containing the process name
+ */
+#define RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(D, PID, N) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_CLIENT_INFO)) \
+	{ \
+		RGXHWPerfHostPostClientInfoProcName(_RGX_DEVICE_INFO_FROM_NODE(D), (PID), (N)); \
+	} \
+} while (0)
+
+#else
+
+#define RGXSRV_HWPERF_ENQ(C, P, X, E, I, K, CF, UF, UT, CHKUID, UPDUID, D, CE)
+#define RGXSRV_HWPERF_UFO(I, T, D, S)
+#define RGXSRV_HWPERF_ALLOC(D, T, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_FENCE(D, PID, FENCE, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_SYNC_CP(D, TL, PID, FENCE, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_SW_FENCE(D, PID, SW_FENCE, SW_TL, SPI, N, Z)
+#define RGXSRV_HWPERF_FREE(D, T, FWADDR)
+#define RGXSRV_HWPERF_FREE_FENCE_SYNC(D, T, UID, PID, FWADDR)
+#define RGXSRV_HWPERF_MODIFY_FENCE_SYNC(D, T, NEWUID, UID1, UID2, N, Z)
+#define RGXSRV_HWPERF_CLK_SYNC(I)
+#define RGXSRV_HWPERF_DEVICE_INFO_HEALTH(I, H, R)
+#define RGXSRV_HWPERF_DEVICE_INFO_FEATURES(I)
+#define RGXSRV_HWPERF_HOST_INFO(I, T)
+#define RGXSRV_HWPERF_SYNC_FENCE_WAIT(I, T, PID, F, D)
+#define RGXSRV_HWPERF_SYNC_SW_TL_ADV(I, PID, SW_TL, SPI)
+#define RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(D, PID, N)
+
+#endif
+
+#endif /* RGXHWPERF_COMMON_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit.h
new file mode 100644
index 000000000000..807f6a0b6fce
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit.h
@@ -0,0 +1,313 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX initialisation header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXINIT_H)
+#define RGXINIT_H
+
+#include "connection_server.h"
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_bridge.h"
+#include "fwload.h"
+#include "rgxinit_apphints.h"
+
+#if defined(__linux__)
+#define OS_FW_VERIFY_FUNCTION OSVerifyFirmware
+#else
+#define OS_FW_VERIFY_FUNCTION NULL
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXInitDevPart2
+
+ @Description
+
+ Second part of server-side RGX initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInitDevPart2 (PVRSRV_DEVICE_NODE	*psDeviceNode,
+							  RGX_INIT_APPHINTS		*psApphints);
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    ui32FWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    ui32FWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen);
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXInitFirmware
+
+ @Description
+
+ Server-side RGX firmware initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags);
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXLoadAndGetFWData
+
+ @Description
+
+ Load FW and return pointer to FW data.
+
+ @Input psDeviceNode - device node
+
+ @Input ppsRGXFW - fw pointer
+
+ @Return PVRSRV_ERROR - PVRSRV_OK on success
+                        PVRSRV_ERROR_NOT_READY if filesystem is not ready
+                        PVRSRV_ERROR_NOT_FOUND if no suitable FW image found
+                        PVRSRV_ERROR_OUT_OF_MEMORY if unable to alloc memory for FW image
+                        PVRSRV_ERROR_NOT_AUTHENTICATED if FW image failed verification
+
+******************************************************************************/
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW);
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function	RGXInitHWPerfCounters
+
+ @Description
+
+ Initialisation of the performance counters
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE	*psDeviceNode);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXRegisterDevice
+
+ @Description
+
+ Registers the device with the system
+
+ @Input:   psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevBVNCString
+
+ @Description
+
+ Returns the Device BVNC string. It will allocate and fill it first, if necessary.
+
+ @Input:   psDevInfo - device info (must not be null)
+
+ @Return   IMG_PCHAR - pointer to BVNC string
+
+******************************************************************************/
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function	DevDeInitRGX
+
+ @Description
+
+ Reset and deinitialise Chip
+
+ @Input psDeviceNode - device info. structure
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+
+#if !defined(NO_HARDWARE)
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsRegister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Initialise data used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument). This function must be called only once for each
+               different user/handle.
+
+ @Input        phGpuUtilUser - Pointer to handle used to identify a user of
+                               RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser);
+
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsUnregister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Free data previously used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument).
+
+ @Input        hGpuUtilUser - Handle used to identify a user of
+                              RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*!
+************************************************************************************
+ @Function		RGXSystemGetFabricCoherency
+
+ @Description	Get the system fabric coherency for the device by reading default
+				configuration from device register, subject to AppHint overrides.
+
+ @Input			sRegsCpuPBase		: Device register CPU physical address base
+				ui32RegsSize		: Device register size
+				peDevFabricType		: Device memory bus fabric type
+				peCacheSnoopingMode : Fabric coherency override
+
+ @Return		PVRSRV_ERROR
+************************************************************************************/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDeviceConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode);
+#endif
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXInitCreateFWKernelMemoryContext
+
+ @Description   Called to perform initialisation during firmware kernel context
+                creation.
+
+ @Input         psDeviceNode  device node
+ ******************************************************************************/
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXDeInitDestroyFWKernelMemoryContext
+
+ @Description   Called to perform deinitialisation during firmware kernel
+                context destruction.
+
+ @Input         psDeviceNode  device node
+ ******************************************************************************/
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXHeapDerivePageSize
+
+ @Description   Ensure the desire page size is suitable for the RGX hardware
+
+ @Input         uiLog2PageSize target page log2 size
+
+ @Return        IMG_UINT32 valid page log2 size
+ ******************************************************************************/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXGetNon4KHeapPageShift
+
+ @Description   Retrieves the log2 page size of the General non 4k heap
+
+ @Output        pui32Log2Non4KPgShift page shift
+ ******************************************************************************/
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate,
+                                     IMG_UINT32 *pui32Log2Non4KPgShift);
+
+
+#endif /* RGXINIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit_apphints.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit_apphints.h
new file mode 100644
index 000000000000..f3bea6280a2e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxinit_apphints.h
@@ -0,0 +1,120 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services device initialisation settings
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device initialisation settings
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXINIT_APPHINTS_H)
+#define RGXINIT_APPHINTS_H
+
+#include "img_defs.h"
+
+#include "rgx_fwif_km.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+
+/*
+ * Container for all the apphints used by this module
+ */
+typedef struct _RGX_INIT_APPHINTS_
+{
+	IMG_BOOL   bEnableSignatureChecks;
+	IMG_UINT32 ui32SignatureChecksBufSize;
+
+	IMG_BOOL   bAssertOnOutOfMem;
+	IMG_BOOL   bAssertOnHWRTrigger;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32TFBCVersion;
+	IMG_UINT32 ui32TFBCCompressionControlGroup;
+	IMG_UINT32 ui32TFBCCompressionControlScheme;
+	IMG_BOOL   bTFBCCompressionControlYUVFormat;
+	IMG_BOOL   bTFBCCompressionControlLossyMinChannel;
+#endif
+	IMG_BOOL   bCheckMlist;
+	IMG_BOOL   bDisableClockGating;
+	IMG_BOOL   bDisableDMOverlap;
+	IMG_BOOL   bDisablePDP;
+	IMG_BOOL   bEnableDMKillRand;
+	IMG_BOOL   bEnableRandomCsw;
+	IMG_BOOL   bEnableSoftResetCsw;
+	IMG_BOOL   bHWPerfDisableCounterFilter;
+	IMG_UINT32 ui32DeviceFlags;
+	IMG_UINT32 ui32FilterFlags;
+	IMG_UINT32 ui32EnableFWContextSwitch;
+	IMG_UINT32 ui32FWContextSwitchProfile;
+	IMG_UINT32 ui32HWPerfFWBufSize;
+	IMG_UINT32 ui32HWPerfHostBufSize;
+	IMG_UINT32 ui32HWPerfFilter0;
+	IMG_UINT32 ui32HWPerfFilter1;
+	IMG_UINT32 ui32HWPerfHostFilter;
+	IMG_UINT32 ui32SecondaryOSClockSource;
+	IMG_UINT32 ui32HWRDebugDumpLimit;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32KCCBSizeLog2;
+#if defined(PVR_ARCH_VOLCANIC)
+	IMG_UINT32 ui32ISPSchedulingLatencyMode;
+#endif
+	FW_PERF_CONF eFirmwarePerf;
+	RGX_ACTIVEPM_CONF eRGXActivePMConf;
+	RGX_RD_POWER_ISLAND_CONF eRGXRDPowerIslandConf;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32 ui32AvailablePowUnitsMask;
+	IMG_UINT32 ui32AvailableRACMask;
+	IMG_BOOL bSPUClockGating;
+#endif
+	IMG_BOOL   bEnableTrustedDeviceAceConfig;
+	IMG_UINT32 ui32FWContextSwitchCrossDM;
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	IMG_UINT32 ui32PhysMemTestPasses;
+#endif
+	RGX_FWT_LOGTYPE eDebugDumpFWTLogType;
+#if defined(SUPPORT_ICS)
+	IMG_UINT32 ui32EnableIdleCycleStealing;
+	IMG_UINT32 ui32FDTI;
+	IMG_UINT32 ui32ICSThreshold;
+	IMG_BOOL   bTestModeOn;
+#endif
+} RGX_INIT_APPHINTS;
+
+#endif /* RGXINIT_APPHINTS_H */
+
+/******************************************************************************
+ End of file (rgxinit_apphints.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.c
new file mode 100644
index 000000000000..b7ab800400c8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.c
@@ -0,0 +1,799 @@
+/*************************************************************************/ /*!
+@File           rgxkicksync.c
+@Title          Server side of the sync only kick API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxkicksync.h"
+
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "allocmem.h"
+#include "sync.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_KICKSYNC_UFO_DUMP	0
+
+//#define KICKSYNC_CHECKPOINT_DEBUG 1
+
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+
+struct _RGX_SERVER_KICKSYNC_CONTEXT_
+{
+	PVRSRV_DEVICE_NODE        * psDeviceNode;
+	RGX_SERVER_COMMON_CONTEXT * psServerCommonContext;
+	DLLIST_NODE                 sListNode;
+	SYNC_ADDR_LIST              sSyncAddrListFence;
+	SYNC_ADDR_LIST              sSyncAddrListUpdate;
+	POS_LOCK                    hLock;
+};
+
+
+PVRSRV_ERROR PVRSRVRGXCreateKickSyncContextKM(CONNECTION_DATA              *psConnection,
+                                              PVRSRV_DEVICE_NODE           *psDeviceNode,
+                                              IMG_HANDLE                    hMemCtxPrivData,
+                                              IMG_UINT32                    ui32PackedCCBSizeU88,
+                                              IMG_UINT32                    ui32ContextFlags,
+                                              RGX_SERVER_KICKSYNC_CONTEXT **ppsKickSyncContext)
+{
+	PVRSRV_RGXDEV_INFO          * psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC              * psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext;
+	RGX_COMMON_CONTEXT_INFO      sInfo;
+	PVRSRV_ERROR                 eError;
+	IMG_UINT32                   ui32CCBAllocSizeLog2, ui32CCBMaxAllocSizeLog2;
+
+	memset(&sInfo, 0, sizeof(sInfo));
+
+	/* Prepare cleanup struct */
+	* ppsKickSyncContext = NULL;
+	psKickSyncContext = OSAllocZMem(sizeof(*psKickSyncContext));
+	if (psKickSyncContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psKickSyncContext->hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+									__func__,
+									PVRSRVGetErrorString(eError)));
+		goto err_lockcreate;
+	}
+
+	psKickSyncContext->psDeviceNode = psDeviceNode;
+
+	ui32CCBAllocSizeLog2 = U32toU8_Unpack1(ui32PackedCCBSizeU88);
+	ui32CCBMaxAllocSizeLog2 = U32toU8_Unpack2(ui32PackedCCBSizeU88);
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_KICKSYNC,
+									 RGXFWIF_DM_GP,
+									 hMemCtxPrivData,
+									 NULL,
+									 0,
+									 psFWMemContextMemDesc,
+									 NULL,
+									 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_KICKSYNC_CCB_SIZE_LOG2,
+									 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_KICKSYNC_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 0, /* priority */
+									 0, /* max deadline MS */
+									 0, /* robustness address */
+									 & sInfo,
+									 & psKickSyncContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	OSWRLockAcquireWrite(psDevInfo->hKickSyncCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sKickSyncCtxtListHead), &(psKickSyncContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hKickSyncCtxListLock);
+
+	SyncAddrListInit(&psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListInit(&psKickSyncContext->sSyncAddrListUpdate);
+
+	* ppsKickSyncContext = psKickSyncContext;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+	OSLockDestroy(psKickSyncContext->hLock);
+err_lockcreate:
+	OSFreeMem(psKickSyncContext);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXDestroyKickSyncContextKM(RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = psKickSyncContext->psDeviceNode->pvDevice;
+	PVRSRV_ERROR         eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psKickSyncContext->psDeviceNode,
+	                                          psKickSyncContext->psServerCommonContext,
+	                                          RGXFWIF_DM_GP,
+	                                          PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psKickSyncContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+
+	OSWRLockAcquireWrite(psDevInfo->hKickSyncCtxListLock);
+	dllist_remove_node(&(psKickSyncContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hKickSyncCtxListLock);
+
+	FWCommonContextFree(psKickSyncContext->psServerCommonContext);
+	psKickSyncContext->psServerCommonContext = NULL;
+
+	SyncAddrListDeinit(&psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psKickSyncContext->sSyncAddrListUpdate);
+
+	OSLockDestroy(psKickSyncContext->hLock);
+
+	OSFreeMem(psKickSyncContext);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetKickSyncContextPropertyKM(RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContext,
+                                                   RGX_CONTEXT_PROPERTY eContextProperty,
+                                                   IMG_UINT64 ui64Input,
+                                                   IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psKickSyncContext->hLock);
+			eError = FWCommonContextSetFlags(psKickSyncContext->psServerCommonContext,
+			                                 ui32ContextFlags);
+
+			OSLockRelease(psKickSyncContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpKickSyncCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hKickSyncCtxListLock);
+	dllist_foreach_node(&psDevInfo->sKickSyncCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_KICKSYNC_CONTEXT *psCurrentServerKickSyncCtx =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_KICKSYNC_CONTEXT, sListNode);
+
+		if (NULL != psCurrentServerKickSyncCtx->psServerCommonContext)
+		{
+			DumpFWCommonContextInfo(psCurrentServerKickSyncCtx->psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+	OSWRLockReleaseRead(psDevInfo->hKickSyncCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientKickSyncCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hKickSyncCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sKickSyncCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_KICKSYNC_CONTEXT *psCurrentServerKickSyncCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_KICKSYNC_CONTEXT, sListNode);
+
+		if (NULL != psCurrentServerKickSyncCtx->psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerKickSyncCtx->psServerCommonContext, RGX_KICK_TYPE_DM_GP) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_GP;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hKickSyncCtxListLock);
+	return ui32ContextBitMask;
+}
+
+PVRSRV_ERROR PVRSRVRGXKickSyncKM(RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext,
+                                 IMG_UINT32                    ui32ClientUpdateCount,
+                                 SYNC_PRIMITIVE_BLOCK       ** pauiClientUpdateUFODevVarBlock,
+                                 IMG_UINT32                  * paui32ClientUpdateOffset,
+                                 IMG_UINT32                  * paui32ClientUpdateValue,
+                                 PVRSRV_FENCE                  iCheckFence,
+                                 PVRSRV_TIMELINE               iUpdateTimeline,
+                                 PVRSRV_FENCE                * piUpdateFence,
+                                 IMG_CHAR                      szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                 IMG_UINT32                    ui32ExtJobRef)
+{
+	RGXFWIF_KCCB_CMD         sKickSyncKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA  asCmdHelperData[1];
+	PVRSRV_ERROR             eError;
+	PVRSRV_ERROR             eError2;
+	IMG_BOOL                 bCCBStateOpen = IMG_FALSE;
+	PRGXFWIF_UFO_ADDR        *pauiClientFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR        *pauiClientUpdateUFOAddress = NULL;
+	IMG_UINT32               ui32ClientFenceCount = 0;
+	IMG_UINT32               *paui32ClientFenceValue = NULL;
+	PVRSRV_FENCE             iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT32               ui32FWCtx = FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext).ui32Addr;
+	PVRSRV_RGXDEV_INFO       *psDevInfo = FWCommonContextGetRGXDevInfo(psKickSyncContext->psServerCommonContext);
+	RGX_CLIENT_CCB           *psClientCCB = FWCommonContextGetClientCCB(psKickSyncContext->psServerCommonContext);
+	IMG_UINT32               ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have dev var updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	OSLockAcquire(psKickSyncContext->hLock);
+	eError = SyncAddrListPopulate(&psKickSyncContext->sSyncAddrListUpdate,
+							ui32ClientUpdateCount,
+							pauiClientUpdateUFODevVarBlock,
+							paui32ClientUpdateOffset);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_syncaddrlist;
+	}
+
+	if (ui32ClientUpdateCount > 0)
+	{
+		pauiClientUpdateUFOAddress = psKickSyncContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	/* Ensure the string is null-terminated (Required for safety) */
+	szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	/* This will never be true if called from the bridge since piUpdateFence will always be valid */
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto out_unlock;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), "
+			   "psKickSyncContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+			   __func__, iCheckFence,
+			   (void*)psKickSyncContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psKickSyncContext->psDeviceNode->hSyncCheckpointContext,
+	                                    iCheckFence,
+	                                    &ui32FenceSyncCheckpointCount,
+	                                    &apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID,
+	                                    PDUMP_FLAGS_NONE);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_resolve_fence;
+	}
+
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: calling SyncCheckpointCreateFence (iUpdateTimeline=%d)...",
+				   __func__, iUpdateTimeline));
+		eError = SyncCheckpointCreateFence(psKickSyncContext->psDeviceNode,
+		                                   szUpdateFenceName,
+		                                   iUpdateTimeline,
+		                                   psKickSyncContext->psDeviceNode->hSyncCheckpointContext,
+		                                   &iUpdateFence,
+		                                   &uiUpdateFenceUID,
+		                                   &pvUpdateFenceFinaliseData,
+		                                   &psUpdateSyncCheckpoint,
+		                                   (void*)&psFenceTimelineUpdateSync,
+		                                   &ui32FenceTimelineUpdateValue,
+		                                   PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%d)",
+					   __func__, eError));
+			goto fail_create_output_fence;
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: ...returned from SyncCheckpointCreateFence "
+				   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+				   "ui32FenceTimelineUpdateValue=%u)",
+				   __func__, iUpdateFence, psFenceTimelineUpdateSync,
+				   ui32FenceTimelineUpdateValue));
+
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*paui32ClientUpdateValue) * (ui32ClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32ClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32ClientUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32ClientUpdateCount);
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32ClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32ClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32ClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x",
+							   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now append the timeline sync prim addr to the kicksync context update list */
+			SyncAddrListAppendSyncPrim(&psKickSyncContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+		}
+	}
+
+	/* Reset number of fence syncs in kicksync context fence list to 0 */
+	SyncAddrListPopulate(&psKickSyncContext->sSyncAddrListFence,
+	                     0, NULL, NULL);
+
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		/* Append the checks (from input fence) */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Append %d sync checkpoints to KickSync Fence "
+				   "(&psKickSyncContext->sSyncAddrListFence=<%p>)...",
+				   __func__, ui32FenceSyncCheckpointCount,
+				   (void*)&psKickSyncContext->sSyncAddrListFence));
+		SyncAddrListAppendCheckpoints(&psKickSyncContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+
+		pauiClientFenceUFOAddress = psKickSyncContext->sSyncAddrListFence.pasFWAddrs;
+
+		ui32ClientFenceCount += ui32FenceSyncCheckpointCount;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiClientFenceUFOAddress;
+
+				for (iii=0; iii<ui32ClientFenceCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientFenceUFOAddress[%d](<%p>) = 0x%x",
+							   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+	}
+
+	if (psUpdateSyncCheckpoint)
+	{
+		PVRSRV_ERROR eErr;
+
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Append 1 sync checkpoint to KickSync Update "
+				   "(&psKickSyncContext->sSyncAddrListUpdate=<%p>)...",
+				   __func__, (void*)&psKickSyncContext->sSyncAddrListUpdate));
+		eErr = SyncAddrListAppendCheckpoints(&psKickSyncContext->sSyncAddrListUpdate,
+											 1,
+											 &psUpdateSyncCheckpoint);
+		if (eErr != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:  ...done. SyncAddrListAppendCheckpoints() returned error (%d)",
+					   __func__, eErr));
+		}
+		else
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:  ...done.", __func__));
+		}
+		if (!pauiClientUpdateUFOAddress)
+		{
+			pauiClientUpdateUFOAddress = psKickSyncContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32ClientUpdateCount++;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiClientUpdateUFOAddress;
+
+			for (iii=0; iii<ui32ClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: pauiClientUpdateUFOAddress[%d](<%p>) = 0x%x",
+						   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+
+#if (ENABLE_KICKSYNC_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping KICKSYNC fence/updates syncs...",
+				 __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiClientFenceUFOAddress;
+			IMG_UINT32 *pui32TmpIntFenceValue = paui32ClientFenceValue;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiClientUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32ClientUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Prepared %d KickSync fence syncs "
+					 "(&psKickSyncContext->sSyncAddrListFence=<%p>, "
+					 "pauiClientFenceUFOAddress=<%p>):",
+					 __func__, ui32ClientFenceCount,
+					 (void*)&psKickSyncContext->sSyncAddrListFence,
+					 (void*)pauiClientFenceUFOAddress));
+			for (ii=0; ii<ui32ClientFenceCount; ii++)
+			{
+				if (psTmpIntFenceUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, "
+							 "CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+							 __func__, ii + 1, ui32ClientFenceCount,
+							 (void*)psTmpIntFenceUFOAddress,
+							 psTmpIntFenceUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+							 __func__, ii + 1, ui32ClientFenceCount,
+							 (void*)psTmpIntFenceUFOAddress,
+							 psTmpIntFenceUFOAddress->ui32Addr,
+							 *pui32TmpIntFenceValue,
+							 *pui32TmpIntFenceValue));
+					pui32TmpIntFenceValue++;
+				}
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Prepared %d KickSync update syncs "
+					 "(&psKickSyncContext->sSyncAddrListUpdate=<%p>, "
+					 "pauiClientUpdateUFOAddress=<%p>):",
+					 __func__, ui32ClientUpdateCount,
+					 (void*)&psKickSyncContext->sSyncAddrListUpdate,
+					 (void*)pauiClientUpdateUFOAddress));
+			for (ii=0; ii<ui32ClientUpdateCount; ii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:  Line %d, psTmpIntUpdateUFOAddress=<%p>",
+						   __func__, __LINE__,
+						   (void*)psTmpIntUpdateUFOAddress));
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:  Line %d, pui32TmpIntUpdateValue=<%p>",
+						   __func__, __LINE__,
+						   (void*)pui32TmpIntUpdateValue));
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, "
+							 "UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+							 __func__, ii + 1, ui32ClientUpdateCount,
+							 (void*)psTmpIntUpdateUFOAddress,
+							 psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d",
+							 __func__, ii + 1, ui32ClientUpdateCount,
+							 (void*)psTmpIntUpdateUFOAddress,
+							 psTmpIntUpdateUFOAddress->ui32Addr,
+							 *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       0, /* empty ui64FBSCEntryMask */
+	                       ui32ClientFenceCount,
+	                       pauiClientFenceUFOAddress,
+	                       paui32ClientFenceValue,
+	                       ui32ClientUpdateCount,
+	                       pauiClientUpdateUFOAddress,
+	                       paui32ClientUpdateValue,
+	                       0,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       RGXFWIF_CCB_CMD_TYPE_NULL,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       PDUMP_FLAGS_NONE,
+	                       NULL,
+	                       "KickSync",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+
+	/*
+	 *  We should reserve space in the kernel CCB here and fill in the command
+	 *  directly.
+	 *  This is so if there isn't space in the kernel CCB we can return with
+	 *  retry back to services client before we take any operations
+	 */
+
+	/*
+	 * All the required resources are ready at this point, we can't fail so
+	 * take the required server sync operations and commit all the resources
+	 */
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	RGXCmdHelperReleaseCmdCCB(1,
+							  asCmdHelperData,
+							  "KickSync",
+							  FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext).ui32Addr);
+
+	/* Construct the kernel kicksync CCB command. */
+	sKickSyncKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext);
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+	/*
+	 * Submit the kicksync command to the firmware.
+	 */
+	RGXSRV_HWPERF_ENQ(psKickSyncContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_SYNC,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError2 = RGXScheduleCommandWithoutPowerLock(psKickSyncContext->psDeviceNode->pvDevice,
+		                             RGXFWIF_DM_GP,
+		                             & sKickSyncKCCBCmd,
+		                             PDUMP_FLAGS_NONE);
+		if (eError2 != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	PVRGpuTraceEnqueueEvent(psKickSyncContext->psDeviceNode,
+	                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+	                        RGX_HWPERF_KICK_TYPE2_SYNC);
+
+	if (eError2 != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXKickSync failed to schedule kernel CCB command. (0x%x)",
+		         eError));
+		eError = eError2;
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier call
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdacquire;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim<%p> to %d",
+				   __func__, (void*)psFenceTimelineUpdateSync,
+				   ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	*piUpdateFence = iUpdateFence;
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psKickSyncContext->psDeviceNode, iUpdateFence,
+									pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, szUpdateFenceName);
+	}
+
+	OSLockRelease(psKickSyncContext->hLock);
+	return PVRSRV_OK;
+
+fail_acquirepowerlock:
+fail_cmdacquire:
+	SyncAddrListRollbackCheckpoints(psKickSyncContext->psDeviceNode, &psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psKickSyncContext->psDeviceNode, &psKickSyncContext->sSyncAddrListUpdate);
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+
+	/* Free memory allocated to hold update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+	}
+fail_alloc_update_values_mem:
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free memory allocated to hold the resolved fence's checkpoints */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+fail_resolve_fence:
+fail_syncaddrlist:
+out_unlock:
+	OSLockRelease(psKickSyncContext->hLock);
+	return eError;
+}
+
+/**************************************************************************//**
+ End of file (rgxkicksync.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.h
new file mode 100644
index 000000000000..57b49a03da5c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxkicksync.h
@@ -0,0 +1,128 @@
+/*************************************************************************/ /*!
+@File           rgxkicksync.h
+@Title          Server side of the sync only kick API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXKICKSYNC_H)
+#define RGXKICKSYNC_H
+
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+#include "sync_server.h"
+#include "rgxdevice.h"
+
+
+typedef struct _RGX_SERVER_KICKSYNC_CONTEXT_ RGX_SERVER_KICKSYNC_CONTEXT;
+
+/**************************************************************************/ /*!
+@Function       DumpKickSyncCtxtsInfo
+@Description    Function that dumps debug info of kick sync ctxs on this device
+@Return         none
+*/ /**************************************************************************/
+void
+DumpKickSyncCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile,
+                      IMG_UINT32 ui32VerbLevel);
+
+/**************************************************************************/ /*!
+@Function       CheckForStalledClientKickSyncCtxt
+@Description    Function that checks if a kick sync client is stalled
+@Return         RGX_KICK_TYPE_DM_GP on stalled context. Otherwise, 0
+*/ /**************************************************************************/
+IMG_UINT32 CheckForStalledClientKickSyncCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXCreateKickSyncContextKM
+@Description    Server-side implementation of RGXCreateKicksyncContext
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXCreateKickSyncContextKM(CONNECTION_DATA              * psConnection,
+                                 PVRSRV_DEVICE_NODE           * psDeviceNode,
+                                 IMG_HANDLE                     hMemCtxPrivData,
+                                 IMG_UINT32                     ui32PackedCCBSizeU88,
+                                 IMG_UINT32                     ui32ContextFlags,
+                                 RGX_SERVER_KICKSYNC_CONTEXT ** ppsKicksyncContext);
+
+
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXDestroyKickSyncContextKM
+@Description    Server-side implementation of RGXDestroyKicksyncContext
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXDestroyKickSyncContextKM(RGX_SERVER_KICKSYNC_CONTEXT * psKicksyncContext);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXSetKickSyncContextPropertyKM
+@Description    Server-side implementation of RGXSetKickSyncContextProperty
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+ */ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetKickSyncContextPropertyKM(RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContext,
+                                                   RGX_CONTEXT_PROPERTY eContextProperty,
+                                                   IMG_UINT64 ui64Input,
+                                                   IMG_UINT64 *pui64Output);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXKickSyncKM
+@Description    Kicks a sync only command
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXKickSyncKM(RGX_SERVER_KICKSYNC_CONTEXT * psKicksyncContext,
+                    IMG_UINT32                    ui32ClientUpdateCount,
+                    SYNC_PRIMITIVE_BLOCK       ** pauiClientUpdateUFODevVarBlock,
+                    IMG_UINT32                  * paui32ClientUpdateDevVarOffset,
+                    IMG_UINT32                  * paui32ClientUpdateValue,
+                    PVRSRV_FENCE                  iCheckFence,
+                    PVRSRV_TIMELINE               iUpdateTimeline,
+                    PVRSRV_FENCE                * piUpdateFence,
+                    IMG_CHAR                      szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+
+                    IMG_UINT32                    ui32ExtJobRef);
+
+#endif /* RGXKICKSYNC_H */
+
+/**************************************************************************//**
+ End of file (rgxkicksync.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer.h
new file mode 100644
index 000000000000..3141db5c4488
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer.h
@@ -0,0 +1,920 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declaration of an interface layer used to abstract code that
+                can be compiled outside of the DDK, potentially in a
+                completely different OS.
+                All the headers included by this file must also be copied to
+                the alternative source tree.
+                All the functions declared here must have a DDK implementation
+                inside the DDK source tree (e.g. rgxlayer_impl.h/.c) and
+                another different implementation in case they are used outside
+                of the DDK.
+                All of the functions accept as a first parameter a
+                "const void *hPrivate" argument. It should be used to pass
+                around any implementation specific data required.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXLAYER_H)
+#define RGXLAYER_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "img_elf.h"
+#include "pvrsrv_error.h" /* includes pvrsrv_errors.h */
+#include "rgx_firmware_boot.h"
+#include "rgx_bvnc_defs_km.h"
+#include "rgx_fw_info.h"
+#include "rgx_common.h"
+#include "rgx_meta.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgx_mips.h"
+#endif
+#include "rgx_riscv.h"
+
+#include "rgxdefs_km.h"
+/* includes:
+ * rgx_cr_defs_km.h,
+ * RGX_BVNC_CORE_KM_HEADER (rgxcore_km_B.V.N.C.h),
+ * RGX_BNC_CONFIG_KM_HEADER (rgxconfig_km_B.V.N.C.h)
+ */
+
+/*
+ * Specific fields for RGX_CR_IDLE must not be polled in pdumps
+ * (technical reasons)
+ */
+#define CR_IDLE_UNSELECTED_MASK ((~RGX_CR_SLC_IDLE_ACE_CONVERTERS_CLRMSK) |	\
+								 (~RGX_CR_SLC_IDLE_OWDB_CLRMSK) |			\
+								 (RGX_CR_SLC_IDLE_FBCDC_ARB_EN))
+
+
+/*!
+*******************************************************************************
+
+ @Function       RGXMemCopy
+
+ @Description    MemCopy implementation
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pvDst      : Pointer to the destination
+ @Input          pvSrc      : Pointer to the source location
+ @Input          uiSize     : The amount of memory to copy in bytes
+
+ @Return         void
+
+******************************************************************************/
+void RGXMemCopy(const void *hPrivate,
+                void *pvDst,
+                void *pvSrc,
+                size_t uiSize);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXMemSet
+
+ @Description    MemSet implementation
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pvDst      : Pointer to the start of the memory region
+ @Input          ui8Value   : The value to be written
+ @Input          uiSize     : The number of bytes to be set to ui8Value
+
+ @Return         void
+
+******************************************************************************/
+void RGXMemSet(const void *hPrivate,
+               void *pvDst,
+               IMG_UINT8 ui8Value,
+               size_t uiSize);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXCommentLog
+
+ @Description    Generic log function used for debugging or other purposes
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pszString  : Message to be printed
+ @Input          ...        : Variadic arguments
+
+ @Return         void
+
+******************************************************************************/
+__printf(2, 3)
+void RGXCommentLog(const void *hPrivate,
+                   const IMG_CHAR *pszString,
+                   ...);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXErrorLog
+
+ @Description    Generic error log function used for debugging or other purposes
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pszString  : Message to be printed
+ @Input          ...        : Variadic arguments
+
+ @Return         void
+
+******************************************************************************/
+__printf(2, 3)
+void RGXErrorLog(const void *hPrivate,
+                 const IMG_CHAR *pszString,
+                 ...);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function       RGXGetOSPageSize
+
+ @Description    Return Page size used on OS
+
+ @Input          hPrivate   : Implementation specific data
+
+ @Return         IMG_UINT32
+
+******************************************************************************/
+
+IMG_UINT32 RGXGetOSPageSize(const void *hPrivate);
+#endif
+
+/* This is used to check if a specific feature is enabled.
+ * Should be used instead of calling RGXDeviceHasFeature.  */
+#define RGX_DEVICE_HAS_FEATURE(hPrivate, Feature) \
+			RGXDeviceHasFeature(hPrivate, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if a specific feature with value is enabled.
+ * Should be used instead of calling RGXDeviceGetFeatureValue.  */
+#define RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, Feature) \
+			(RGXDeviceGetFeatureValue(hPrivate, RGX_FEATURE_##Feature##_IDX) >= 0)
+
+/* This is used to get the value of a specific feature from hPrivate.
+ * Should be used instead of calling RGXDeviceGetFeatureValue.  */
+#define RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, Feature) \
+			RGXDeviceGetFeatureValue(hPrivate, RGX_FEATURE_##Feature##_IDX)
+
+
+/* This is used to check if a specific ERN is enabled from hPrivate.
+ * Should be used instead of calling RGXDeviceHasErnBrn.  */
+#define RGX_DEVICE_HAS_ERN(hPrivate, ERN) \
+			RGXDeviceHasErnBrn(hPrivate, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if a specific BRN is enabled from hPrivate.
+ * Should be used instead of calling RGXDeviceHasErnBrn.  */
+#define RGX_DEVICE_HAS_BRN(hPrivate, BRN) \
+			RGXDeviceHasErnBrn(hPrivate, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+#define CLK_CTRL_FORCE_ON(X, Module) \
+			X = (((X) & RGX_CR_##Module##_CLRMSK) | RGX_CR_##Module##_ON)
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceGetFeatureValue
+
+ @Description    Checks if a device has a particular feature with values
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64Feature  : Feature with values to check
+
+ @Return         Value >= 0 if the given feature is available, -1 otherwise
+
+******************************************************************************/
+IMG_INT32 RGXDeviceGetFeatureValue(const void *hPrivate, IMG_UINT64 ui64Feature);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceHasFeature
+
+ @Description    Checks if a device has a particular feature
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64Feature  : Feature to check
+
+ @Return         IMG_TRUE if the given feature is available, IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDeviceHasFeature(const void *hPrivate, IMG_UINT64 ui64Feature);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceHasErnBrn
+
+ @Description    Checks if a device has a particular errata
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64ErnsBrns : Flags to check
+
+ @Return         IMG_TRUE if the given errata is available, IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDeviceHasErnBrn(const void *hPrivate, IMG_UINT64 ui64ErnsBrns);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetFWCorememSize
+
+ @Description    Get the FW coremem size
+
+ @Input          hPrivate   : Implementation specific data
+
+ @Return         FW coremem size
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWCorememSize(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXWriteReg32/64
+
+ @Description   Write a value to a 32/64 bit RGX register
+
+ @Input         hPrivate         : Implementation specific data
+ @Input         ui32RegAddr      : Register offset inside the register bank
+ @Input         ui32/64RegValue  : New register value
+
+ @Return        void
+
+******************************************************************************/
+void RGXWriteReg32(const void *hPrivate,
+                   IMG_UINT32 ui32RegAddr,
+                   IMG_UINT32 ui32RegValue);
+
+void RGXWriteReg64(const void *hPrivate,
+                   IMG_UINT32 ui32RegAddr,
+                   IMG_UINT64 ui64RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadReg32/64
+
+ @Description    Read a 32/64 bit RGX register
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui32RegAddr  : Register offset inside the register bank
+
+ @Return         Register value
+
+******************************************************************************/
+IMG_UINT32 RGXReadReg32(const void *hPrivate,
+                        IMG_UINT32 ui32RegAddr);
+
+IMG_UINT64 RGXReadReg64(const void *hPrivate,
+                        IMG_UINT32 ui32RegAddr);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadModifyWriteReg32
+
+ @Description    Read-modify-write a 32 bit RGX register
+
+ @Input          hPrivate     : Implementation specific data.
+ @Input          ui32RegAddr  : Register offset inside the register bank.
+ @Input          ui32RegValue : New register value.
+ @Input          ui32RegMask  : Keep the bits set in the mask.
+
+ @Return         Always returns PVRSRV_OK
+
+******************************************************************************/
+IMG_UINT32 RGXReadModifyWriteReg64(const void *hPrivate,
+                                   IMG_UINT32 ui32RegAddr,
+                                   IMG_UINT64 ui64RegValue,
+                                   IMG_UINT64 ui64RegKeepMask);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXPollReg32/64
+
+ @Description    Poll on a 32/64 bit RGX register until some bits are set/unset
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          ui32/64RegValue  : Value expected from the register
+ @Input          ui32/64RegMask   : Only the bits set in this mask will be
+                                    checked against uiRegValue
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV_ERROR_TIMEOUT if the poll takes too long
+
+******************************************************************************/
+PVRSRV_ERROR RGXPollReg32(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT32 ui32RegValue,
+                          IMG_UINT32 ui32RegMask);
+
+PVRSRV_ERROR RGXPollReg64(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64RegValue,
+                          IMG_UINT64 ui64RegMask);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXWriteMetaRegThroughSP
+
+ @Description    Write a register value using the META slave port
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          ui32RegValue     : Value written to the register
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadMetaRegThroughSP
+
+ @Description    Read a register value using the META slave port
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          *ui32RegValue    : Value read from the register
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXSetPoweredState
+
+ @Description    Declare if the device is powered or not
+
+ @Input          hPrivate    : Implementation specific data
+ @Input          bPowered    : true is powered, false otherwise
+
+ @Return         void
+
+******************************************************************************/
+void RGXSetPoweredState(const void *hPrivate, IMG_BOOL bPowered);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXWaitCycles
+
+ @Description    Wait for a number of GPU cycles and/or microseconds
+
+ @Input          hPrivate    : Implementation specific data
+ @Input          ui32Cycles  : Number of GPU cycles to wait for in pdumps,
+                               it can also be used when running driver-live
+                               if desired (ignoring the next parameter)
+ @Input          ui32WaitUs  : Number of microseconds to wait for when running
+                               driver-live
+
+ @Return         void
+
+******************************************************************************/
+void RGXWaitCycles(const void *hPrivate,
+                   IMG_UINT32 ui32Cycles,
+                   IMG_UINT32 ui32WaitUs);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireKernelMMUPC
+
+ @Description     Acquire the Kernel MMU Page Catalogue device physical address
+
+ @Input           hPrivate  : Implementation specific data
+ @Input           psPCAddr  : Returned page catalog address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireKernelMMUPC(const void *hPrivate, IMG_DEV_PHYADDR *psPCAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXWriteKernelMMUPC32/64
+
+ @Description     Write the Kernel MMU Page Catalogue to the 32/64 bit
+                  RGX register passed as argument.
+                  In a driver-live scenario without PDump these functions
+                  are the same as RGXWriteReg32/64 and they don't need
+                  to be reimplemented.
+
+ @Input           hPrivate        : Implementation specific data
+ @Input           ui32PCReg       : Register offset inside the register bank
+ @Input           ui32AlignShift  : PC register alignshift
+ @Input           ui32Shift       : PC register shift
+ @Input           ui32/64PCVal    : Page catalog value (aligned and shifted)
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+void RGXWriteKernelMMUPC64(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT64 ui64PCVal);
+#endif
+
+void RGXWriteKernelMMUPC32(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT32 ui32PCVal);
+#else /* defined(PDUMP) */
+
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGXWriteKernelMMUPC64(priv, pcreg, alignshift, shift, pcval) \
+	RGXWriteReg64(priv, pcreg, pcval)
+#endif
+
+#define RGXWriteKernelMMUPC32(priv, pcreg, alignshift, shift, pcval) \
+	RGXWriteReg32(priv, pcreg, pcval)
+
+#endif /* defined(PDUMP) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireGPURegsAddr
+
+ @Description     Acquire the GPU registers base device physical address
+
+ @Input           hPrivate       : Implementation specific data
+ @Input           psGPURegsAddr  : Returned GPU registers base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireGPURegsAddr(const void *hPrivate, IMG_DEV_PHYADDR *psGPURegsAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXMIPSWrapperConfig
+
+ @Description     Write GPU register bank transaction ID and MIPS boot mode
+                  to the MIPS wrapper config register (passed as argument).
+                  In a driver-live scenario without PDump this is the same as
+                  RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate          : Implementation specific data
+ @Input           ui32RegAddr       : Register offset inside the register bank
+ @Input           ui64GPURegsAddr   : GPU registers base address
+ @Input           ui32GPURegsAlign  : Register bank transactions alignment
+ @Input           ui32BootMode      : Mips BOOT ISA mode
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXMIPSWrapperConfig(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64GPURegsAddr,
+                          IMG_UINT32 ui32GPURegsAlign,
+                          IMG_UINT32 ui32BootMode);
+#else
+#define RGXMIPSWrapperConfig(priv, regaddr, gpuregsaddr, gpuregsalign, bootmode) \
+	RGXWriteReg64(priv, regaddr, ((gpuregsaddr) >> (gpuregsalign)) | (bootmode))
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS bootloader
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootRemapAddr  : Base address of the remapped bootloader
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psBootRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXBootRemapConfig
+
+ @Description     Configure the bootloader remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXBootRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXBootRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireCodeRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS code
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psCodeRemapAddr  : Base address of the remapped code
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireCodeRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psCodeRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXCodeRemapConfig
+
+ @Description     Configure the code remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXCodeRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXCodeRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireDataRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS data
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psDataRemapAddr  : Base address of the remapped data
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireDataRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psDataRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXDataRemapConfig
+
+ @Description     Configure the data remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXDataRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXDataRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireTrampolineRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS data
+                  accessed through remap region
+
+ @Input           hPrivate             : Implementation specific data
+ @Output          psTrampolineRemapAddr: Base address of the remapped data
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireTrampolineRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psTrampolineRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXTrampolineRemapConfig
+
+ @Description     Configure the trampoline remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXTrampolineRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXTrampolineRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+/*!
+*******************************************************************************
+
+ @Function        RGXDoFWSlaveBoot
+
+ @Description     Returns whether or not a FW Slave Boot is required
+                  while powering on
+
+ @Input           hPrivate       : Implementation specific data
+
+ @Return          IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXFabricCoherencyTest
+
+ @Description    Performs fabric coherency test
+
+ @Input          hPrivate         : Implementation specific data
+
+ @Return         PVRSRV_OK if the test succeeds,
+                 PVRSRV_ERROR_INIT_FAILURE if the test fails at some point
+
+******************************************************************************/
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDeviceSLCBanks
+
+ @Description    Returns the number of SLC banks used by the device
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Number of SLC banks
+
+******************************************************************************/
+IMG_UINT32 RGXGetDeviceSLCBanks(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDeviceCacheLineSize
+
+ @Description    Returns the device cache line size
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Cache line size
+
+******************************************************************************/
+IMG_UINT32 RGXGetDeviceCacheLineSize(const void *hPrivate);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDevicePhysBusWidth
+
+ @Description    Returns the device physical bus width
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Physical bus width
+
+******************************************************************************/
+IMG_UINT32 RGXGetDevicePhysBusWidth(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDevicePA0IsValid
+
+ @Description    Returns true if the device physical address 0x0 is a valid
+                 address and can be accessed by the GPU.
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         IMG_TRUE if device physical address 0x0 is a valid address,
+                 IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDevicePA0IsValid(const void *hPrivate);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootCodeAddr
+
+ @Description     Acquire the device virtual address of the RISCV boot code
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootCodeAddr   : Boot code base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootCodeAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootCodeAddr);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXCalculateHostFWDataAddress
+
+ @Description   Calculates the base host address of the RISCV firmware data
+
+ @Input         hPrivate         : Implementation specific data
+ @Input         pvHostFWDataAddr : Initial host address of the firmware data
+
+ @Return:       Base host address of the RISCV firmware data
+
+******************************************************************************/
+void *RGXCalculateHostFWDataAddress(const void *hPrivate, void *pvHostFWDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootDataAddr
+
+ @Description     Acquire the device virtual address of the RISCV boot data
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootDataAddr   : Boot data base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootDataAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXDeviceAckIrq
+
+ @Description   Checks the implementation specific IRQ status register,
+                clearing it if necessary and returning the IRQ status.
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return:       IRQ status
+
+******************************************************************************/
+IMG_BOOL RGXDeviceAckIrq(const void *hPrivate);
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*!
+*******************************************************************************
+
+@Function       RGXMMUInitRangeValue
+
+@Description    Returns the appropriate config value for each MMU range
+
+@Input          ui32MMURange   : Hardware MMU range to be initialised
+
+@Return         64-bit register value
+
+******************************************************************************/
+IMG_UINT64 RGXMMUInitRangeValue(IMG_UINT32 ui32MMURange);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGXLAYER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl.h
new file mode 100644
index 000000000000..b46efee8c71c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXLAYER_IMPL_H)
+#define RGXLAYER_IMPL_H
+
+#include "rgxlayer.h"
+#include "device_connection.h"
+
+typedef struct _RGX_LAYER_PARAMS_
+{
+	void *psDevInfo;
+	void *psDevConfig;
+#if defined(PDUMP)
+	IMG_UINT32 ui32PdumpFlags;
+#endif
+
+	IMG_DEV_PHYADDR sPCAddr;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEV_PHYADDR sGPURegAddr;
+	IMG_DEV_PHYADDR sBootRemapAddr;
+	IMG_DEV_PHYADDR sCodeRemapAddr;
+	IMG_DEV_PHYADDR sDataRemapAddr;
+	IMG_DEV_PHYADDR sTrampolineRemapAddr;
+	IMG_BOOL bDevicePA0IsValid;
+#endif
+} RGX_LAYER_PARAMS;
+
+#endif /* RGXLAYER_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl_common.c
new file mode 100644
index 000000000000..7eab6fde4dcd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxlayer_impl_common.c
@@ -0,0 +1,914 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwimageutils.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr) \
+	(((ui32RegAddr) < RGX_HOST_SECURE_REGBANK_OFFSET) ? \
+	 ((psDevInfo)->pvRegsBaseKM) : ((psDevInfo)->pvSecureRegsBaseKM))
+#else
+#define RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr) ((psDevInfo)->pvRegsBaseKM)
+#endif
+
+void RGXMemCopy(const void *hPrivate,
+                void *pvDst,
+                void *pvSrc,
+                size_t uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	OSDeviceMemCopy(pvDst, pvSrc, uiSize);
+}
+
+void RGXMemSet(const void *hPrivate,
+               void *pvDst,
+               IMG_UINT8 ui8Value,
+               size_t uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	OSDeviceMemSet(pvDst, ui8Value, uiSize);
+}
+
+void RGXCommentLog(const void *hPrivate,
+                   const IMG_CHAR *pszString,
+                   ...)
+{
+#if defined(PDUMP)
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	va_list argList;
+	va_start(argList, pszString);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	PDumpCommentWithFlagsVA(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS, pszString, argList);
+	va_end(argList);
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	PVR_UNREFERENCED_PARAMETER(pszString);
+#endif
+}
+
+void RGXErrorLog(const void *hPrivate,
+                 const IMG_CHAR *pszString,
+                 ...)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list argList;
+
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+
+	va_start(argList, pszString);
+	vsnprintf(szBuffer, sizeof(szBuffer), pszString, argList);
+	va_end(argList);
+
+	PVR_DPF((PVR_DBG_ERROR, "%s", szBuffer));
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+IMG_UINT32 RGXGetOSPageSize(const void *hPrivate)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	return OSGetPageSize();
+}
+#endif
+
+IMG_INT32 RGXDeviceGetFeatureValue(const void *hPrivate, IMG_UINT64 ui64Feature)
+{
+	IMG_INT32 i32Ret = -1;
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	if ((psDeviceNode->pfnGetDeviceFeatureValue))
+	{
+		i32Ret = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode, ui64Feature);
+	}
+
+	return i32Ret;
+}
+
+IMG_BOOL RGXDeviceHasFeature(const void *hPrivate, IMG_UINT64 ui64Feature)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->sDevFeatureCfg.ui64Features & ui64Feature) != 0;
+}
+
+IMG_UINT32 RGXGetFWCorememSize(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32CorememSize = 0;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		ui32CorememSize = RGX_GET_FEATURE_VALUE(psDevInfo, META_COREMEM_SIZE);
+	}
+
+	return ui32CorememSize;
+}
+
+void RGXWriteReg32(const void *hPrivate, IMG_UINT32 ui32RegAddr, IMG_UINT32 ui32RegValue)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		OSWriteUncheckedHWReg32(pvRegsBase, ui32RegAddr, ui32RegValue);
+	}
+
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	           ui32RegAddr, ui32RegValue, psParams->ui32PdumpFlags);
+}
+
+void RGXWriteReg64(const void *hPrivate, IMG_UINT32 ui32RegAddr, IMG_UINT64 ui64RegValue)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		OSWriteUncheckedHWReg64(pvRegsBase, ui32RegAddr, ui64RegValue);
+	}
+
+	PDUMPREG64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	           ui32RegAddr, ui64RegValue, psParams->ui32PdumpFlags);
+}
+
+IMG_UINT32 RGXReadReg32(const void *hPrivate, IMG_UINT32 ui32RegAddr)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+	IMG_UINT32 ui32RegValue;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW)
+	{
+		ui32RegValue = IMG_UINT32_MAX;
+	}
+	else
+#endif
+	{
+		ui32RegValue = OSReadUncheckedHWReg32(pvRegsBase, ui32RegAddr);
+	}
+
+	PDUMPREGREAD32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	               ui32RegAddr, psParams->ui32PdumpFlags);
+
+	return ui32RegValue;
+}
+
+IMG_UINT64 RGXReadReg64(const void *hPrivate, IMG_UINT32 ui32RegAddr)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+	IMG_UINT64 ui64RegValue;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW)
+	{
+		ui64RegValue = IMG_UINT64_MAX;
+	}
+	else
+#endif
+	{
+		ui64RegValue = OSReadUncheckedHWReg64(pvRegsBase, ui32RegAddr);
+	}
+
+	PDUMPREGREAD64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	               ui32RegAddr, PDUMP_FLAGS_CONTINUOUS);
+
+	return ui64RegValue;
+}
+
+IMG_UINT32 RGXReadModifyWriteReg64(const void *hPrivate,
+                                   IMG_UINT32 ui32RegAddr,
+                                   IMG_UINT64 uiRegValueNew,
+                                   IMG_UINT64 uiRegKeepMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+#if defined(PDUMP)
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+	/* only use the new values for bits we update according to the keep mask */
+	uiRegValueNew &= ~uiRegKeepMask;
+
+#if defined(PDUMP)
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store register offset to temp PDump variable */
+	PDumpRegRead64ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                            ":SYSMEM:$1", ui32RegAddr, ui32PDumpFlags);
+
+	/* Keep the bits set in the mask */
+	PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                        uiRegKeepMask, ui32PDumpFlags);
+
+	/* OR the new values */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                       uiRegValueNew, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                        ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+
+	{
+		IMG_UINT64 uiRegValue = OSReadUncheckedHWReg64(pvRegsBase, ui32RegAddr);
+		uiRegValue &= uiRegKeepMask;
+		OSWriteUncheckedHWReg64(pvRegsBase, ui32RegAddr, uiRegValue | uiRegValueNew);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollReg32(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT32 ui32RegValue,
+                          IMG_UINT32 ui32RegMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr),
+		                         ui32RegValue,
+		                         ui32RegMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg32: Poll for Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+	}
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr,
+	            ui32RegValue,
+	            ui32RegMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollReg64(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64RegValue,
+                          IMG_UINT64 ui64RegMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	/* Split lower and upper words */
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64RegValue >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64RegValue);
+	IMG_UINT32 ui32UpperMask = (IMG_UINT32) (ui64RegMask >> 32);
+	IMG_UINT32 ui32LowerMask = (IMG_UINT32) (ui64RegMask);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr + 4),
+		                         ui32UpperValue,
+		                         ui32UpperMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg64: Poll for upper part of Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr),
+		                         ui32LowerValue,
+		                         ui32LowerMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg64: Poll for lower part of Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+	}
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr + 4,
+	            ui32UpperValue,
+	            ui32UpperMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr,
+	            ui32LowerValue,
+	            ui32LowerMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+}
+
+void RGXSetPoweredState(const void *hPrivate, IMG_BOOL bPowered)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	psDevInfo->bRGXPowered = bPowered;
+}
+
+void RGXWaitCycles(const void *hPrivate, IMG_UINT32 ui32Cycles, IMG_UINT32 ui32TimeUs)
+{
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+	OSWaitus(ui32TimeUs);
+	PDUMPIDLWITHFLAGS(psDevInfo->psDeviceNode, ui32Cycles, PDUMP_FLAGS_CONTINUOUS);
+}
+
+void RGXAcquireKernelMMUPC(const void *hPrivate, IMG_DEV_PHYADDR *psPCAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psPCAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sPCAddr;
+}
+
+#if defined(PDUMP)
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+void RGXWriteKernelMMUPC64(const void *hPrivate,
+		IMG_UINT32 ui32PCReg,
+		IMG_UINT32 ui32PCRegAlignShift,
+		IMG_UINT32 ui32PCRegShift,
+		IMG_UINT64 ui64PCVal)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write the cat-base address */
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, ui32PCReg, ui64PCVal);
+
+	/* Pdump catbase address */
+	MMU_PDumpWritePageCatBase(psDevInfo->psKernelMMUCtx,
+			RGX_PDUMPREG_NAME,
+			ui32PCReg,
+			8,
+			ui32PCRegAlignShift,
+			ui32PCRegShift,
+			PDUMP_FLAGS_CONTINUOUS);
+}
+#endif
+
+void RGXWriteKernelMMUPC32(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT32 ui32PCVal)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32PCReg);
+
+	/* Write the cat-base address */
+	OSWriteUncheckedHWReg32(pvRegsBase, ui32PCReg, ui32PCVal);
+
+	/* Pdump catbase address */
+	MMU_PDumpWritePageCatBase(psDevInfo->psKernelMMUCtx,
+	                          RGX_PDUMPREG_NAME,
+	                          ui32PCReg,
+	                          4,
+	                          ui32PCRegAlignShift,
+	                          ui32PCRegShift,
+	                          PDUMP_FLAGS_CONTINUOUS);
+}
+#endif /* defined(PDUMP) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXAcquireGPURegsAddr(const void *hPrivate, IMG_DEV_PHYADDR *psGPURegsAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psGPURegsAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sGPURegAddr;
+}
+
+#if defined(PDUMP)
+void RGXMIPSWrapperConfig(const void *hPrivate,
+		IMG_UINT32 ui32RegAddr,
+		IMG_UINT64 ui64GPURegsAddr,
+		IMG_UINT32 ui32GPURegsAlign,
+		IMG_UINT32 ui32BootMode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
+			ui32RegAddr,
+			(ui64GPURegsAddr >> ui32GPURegsAlign) | ui32BootMode);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store register offset to temp PDump variable */
+	PDumpRegLabelToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                           ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	/* Align register transactions identifier */
+	PDumpWriteVarSHRValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                        ui32GPURegsAlign, ui32PDumpFlags);
+
+	/* Enable micromips instruction encoding */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                       ui32BootMode, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                        ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+}
+#endif
+
+void RGXAcquireBootRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psBootRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psBootRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sBootRemapAddr;
+}
+
+void RGXAcquireCodeRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psCodeRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psCodeRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sCodeRemapAddr;
+}
+
+void RGXAcquireDataRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psDataRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psDataRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sDataRemapAddr;
+}
+
+void RGXAcquireTrampolineRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psTrampolineRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psTrampolineRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sTrampolineRemapAddr;
+}
+
+#if defined(PDUMP)
+static inline
+void RGXWriteRemapConfig2Reg(void __iomem *pvRegs,
+		PMR *psPMR,
+		IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+		IMG_UINT32 ui32RegAddr,
+		IMG_UINT64 ui64PhyAddr,
+		IMG_UINT64 ui64PhyMask,
+		IMG_UINT64 ui64Settings)
+{
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_ASSERT(psPMR != NULL);
+	psDevNode = PMR_DeviceNode(psPMR);
+
+	OSWriteHWReg64(pvRegs, ui32RegAddr, (ui64PhyAddr & ui64PhyMask) | ui64Settings);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store memory offset to temp PDump variable */
+	PDumpMemLabelToInternalVar64(":SYSMEM:$1", psPMR,
+	                             uiLogicalOffset, ui32PDumpFlags);
+
+	/* Keep only the relevant bits of the output physical address */
+	PDumpWriteVarANDValueOp(psDevNode, ":SYSMEM:$1", ui64PhyMask, ui32PDumpFlags);
+
+	/* Extra settings for this remapped region */
+	PDumpWriteVarORValueOp(psDevNode, ":SYSMEM:$1", ui64Settings, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevNode, RGX_PDUMPREG_NAME, ui32RegAddr,
+	                        ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+}
+
+void RGXBootRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32BootRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWCodeMemDesc->uiOffset + ui32BootRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXCodeRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32CodeRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWCodeMemDesc->uiOffset + ui32CodeRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXDataRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32DataRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWDataMemDesc->uiOffset + ui32DataRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXTrampolineRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* write the register for real, without PDump */
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store the memory address in a PDump variable */
+	PDumpPhysHandleToInternalVar64(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			psDevInfo->psTrampoline->hPdumpPages,
+			ui32PDumpFlags);
+
+	/* Keep only the relevant bits of the input physical address */
+	PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			~RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_CLRMSK,
+			ui32PDumpFlags);
+
+	/* Enable bit */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+			ui32PDumpFlags);
+
+	/* Do the PDump register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode,
+			RGX_PDUMPREG_NAME,
+			ui32Config1RegAddr,
+			":SYSMEM:$1",
+			ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+
+	/* this can be written directly */
+	RGXWriteReg64(hPrivate,
+			ui32Config2RegAddr,
+			(ui64Config2PhyAddr & ui64Config2PhyMask) | ui64Config2Settings);
+}
+#endif /* defined(PDUMP) */
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+IMG_BOOL RGXDeviceHasErnBrn(const void *hPrivate, IMG_UINT64 ui64ErnsBrns)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->sDevFeatureCfg.ui64ErnsBrns & ui64ErnsBrns) != 0;
+}
+
+IMG_UINT32 RGXGetDeviceSLCBanks(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_BANKS))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, SLC_BANKS);
+}
+
+IMG_UINT32 RGXGetDeviceCacheLineSize(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_CACHE_LINE_SIZE_BITS))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+IMG_UINT32 RGXGetDevicePhysBusWidth(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PHYS_BUS_WIDTH))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH);
+}
+
+IMG_BOOL RGXDevicePA0IsValid(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return psDevInfo->sLayerParams.bDevicePA0IsValid;
+}
+#endif
+
+void RGXAcquireBootCodeAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootCodeAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	*psBootCodeAddr = psDevInfo->sFWCodeDevVAddrBase;
+}
+
+void RGXAcquireBootDataAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootDataAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	*psBootDataAddr = psDevInfo->sFWDataDevVAddrBase;
+}
+
+void *RGXCalculateHostFWDataAddress(const void *hPrivate, void *pvHostFWDataAddr)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT8 *ui8HostFWDataAddr = (IMG_UINT8*)pvHostFWDataAddr;
+	IMG_UINT32 ui32Offset = 0U;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		ui32Offset =
+			PVR_ALIGN(RGXGetFWImageSectionAllocSize(hPrivate, RISCV_UNCACHED_CODE),
+			          RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN) +
+			PVR_ALIGN(RGXGetFWImageSectionAllocSize(hPrivate, RISCV_CACHED_CODE),
+			          RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN);
+	}
+
+	ui8HostFWDataAddr -= ui32Offset;
+	return (void*)ui8HostFWDataAddr;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+
+	return pvHostFWDataAddr;
+#endif
+}
+
+IMG_BOOL RGXDeviceAckIrq(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->pfnRGXAckIrq != NULL) ?
+			psDevInfo->pfnRGXAckIrq(psDevInfo) : IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.c
new file mode 100644
index 000000000000..4df36ef5d1f0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.c
@@ -0,0 +1,973 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX memory context management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX memory context management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_server_utils.h"
+#include "devicemem_pdump.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_km.h"
+#include "rgxfwutils.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "sync_internal.h"
+#include "rgx_memallocflags.h"
+#include "rgx_bvnc_defs_km.h"
+#include "info_page.h"
+
+# include "rgxmmudefs_km.h"
+
+#if defined(PDUMP)
+#include "sync.h"
+#endif
+
+struct SERVER_MMU_CONTEXT_TAG
+{
+	DEVMEM_MEMDESC *psFWMemContextMemDesc;
+	PRGXFWIF_FWMEMCONTEXT sFWMemContextDevVirtAddr;
+	MMU_CONTEXT *psMMUContext;
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_UINT64 ui64FBSCEntryMask;
+	DLLIST_NODE sNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+}; /* SERVER_MMU_CONTEXT is typedef-ed in rgxmem.h */
+
+PVRSRV_ERROR RGXInvalidateFBSCTable(PVRSRV_DEVICE_NODE *psDeviceNode,
+									MMU_CONTEXT *psMMUContext,
+									IMG_UINT64 ui64FBSCEntryMask)
+{
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSWRLockAcquireRead(psDevInfo->hMemoryCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psIter = IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+		if (psIter->psMMUContext == psMMUContext)
+		{
+			psServerMMUContext = psIter;
+			break;
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hMemoryCtxListLock);
+
+	if (! psServerMMUContext)
+	{
+		return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+	}
+
+	/* Accumulate the FBSC invalidate request */
+	psServerMMUContext->ui64FBSCEntryMask |= ui64FBSCEntryMask;
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXExtractFBSCEntryMaskFromMMUContext
+ *
+ */
+PVRSRV_ERROR RGXExtractFBSCEntryMaskFromMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+												   SERVER_MMU_CONTEXT *psServerMMUContext,
+												   IMG_UINT64 *pui64FBSCEntryMask)
+{
+	if (!psServerMMUContext)
+	{
+		return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+	}
+
+	*pui64FBSCEntryMask = psServerMMUContext->ui64FBSCEntryMask;
+	psServerMMUContext->ui64FBSCEntryMask = 0;
+
+	return PVRSRV_OK;
+}
+
+void RGXMMUCacheInvalidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   MMU_CONTEXT *psMMUContext,
+						   MMU_LEVEL eMMULevel,
+						   IMG_BOOL bUnmap)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT32 ui32NewCacheFlags;
+
+	PVR_UNREFERENCED_PARAMETER(bUnmap);
+
+	switch (eMMULevel)
+	{
+		case MMU_LEVEL_3:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PC;
+
+			break;
+		case MMU_LEVEL_2:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PD;
+
+			break;
+		case MMU_LEVEL_1:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PT;
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+			if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT)))
+#endif
+			{
+				ui32NewCacheFlags |= RGXFWIF_MMUCACHEDATA_FLAGS_TLB;
+			}
+
+			break;
+		default:
+			ui32NewCacheFlags = 0;
+			PVR_ASSERT(0);
+
+			break;
+	}
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	/* For VIVT devices only accumulate the flags on the Firmware MMU context
+	 * since the Firmware/HW invalidates caches on every kick automatically. */
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT) ||
+	    psMMUContext == psDevInfo->psKernelMMUCtx)
+#endif
+	{
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32NewCacheFlags);
+	}
+}
+
+static
+PVRSRV_ERROR _PrepareAndSubmitCacheCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           RGXFWIF_DM eDM,
+                                           IMG_UINT32 ui32CacheFlags,
+                                           IMG_BOOL bInterrupt,
+                                           IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sFlushCmd;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	*pui32MMUInvalidateUpdate = psDeviceNode->ui32NextMMUInvalidateUpdate++;
+
+	/* Setup cmd and add the device nodes sync object */
+	sFlushCmd.eCmdType = RGXFWIF_KCCB_CMD_MMUCACHE;
+	sFlushCmd.uCmdData.sMMUCacheData.ui32MMUCacheSyncUpdateValue = *pui32MMUInvalidateUpdate;
+	SyncPrimGetFirmwareAddr(psDeviceNode->psMMUCacheSyncPrim,
+	                        &sFlushCmd.uCmdData.sMMUCacheData.sMMUCacheSync.ui32Addr);
+
+	/*
+	 *  Indicate if the firmware should signal command completion to the host.
+	 *  The sync update will always happen. This flag requests that the FW pass
+	 *  back the result of the KCCB command and interrupt the host. The KCCB
+	 *  response is not used but the interrupt signal to the host is as a way
+	 *  to know the sync update may have happened.
+	 */
+	if (bInterrupt)
+	{
+		ui32CacheFlags |= RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT;
+	}
+
+	sFlushCmd.uCmdData.sMMUCacheData.ui32CacheFlags = ui32CacheFlags;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Submit MMU flush and invalidate (flags = 0x%08x)",
+	                      ui32CacheFlags);
+#endif
+
+	/* Schedule MMU cache command */
+	eError = RGXSendCommand(psDevInfo,
+							&sFlushCmd,
+							PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule MMU cache command to "
+		         "DM=%d with error (%u)",
+		         __func__, eDM, eError));
+		psDeviceNode->ui32NextMMUInvalidateUpdate--;
+
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32CacheFlags);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXMMUCacheInvalidateKick(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32FWCacheFlags;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+		goto RGXMMUCacheInvalidateKick_exit;
+	}
+
+	/*
+	 * Atomically clear flags to ensure we never accidentally read state
+	 * inconsistently or overwrite valid cache flags with 0.
+	 */
+	ui32FWCacheFlags = MMU_GetAndResetCacheFlags(psDevInfo->psKernelMMUCtx);
+	if (ui32FWCacheFlags == 0)
+	{
+		/* Nothing to do if no cache ops pending */
+		eError = PVRSRV_OK;
+		goto _PowerUnlockAndReturnErr;
+	}
+
+	/* Ensure device is powered up before sending cache command */
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32FWCacheFlags);
+		goto _PowerUnlockAndReturnErr;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = _PrepareAndSubmitCacheCommand(psDeviceNode, RGXFWIF_DM_GP, ui32FWCacheFlags,
+											   IMG_TRUE, pui32MMUInvalidateUpdate);
+		if (!PVRSRVIsRetryError(eError))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_LOG_IF_ERROR(eError, "_PrepareAndSubmitCacheCommand");
+
+_PowerUnlockAndReturnErr:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+RGXMMUCacheInvalidateKick_exit:
+	return eError;
+}
+
+PVRSRV_ERROR RGXPreKickCacheCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+									RGXFWIF_DM eDM,
+									IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32FWCacheFlags;
+
+	/* Caller should ensure that power lock is held before calling this function */
+	PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+
+	/*
+	 * Atomically clear flags to ensure we never accidentally read state
+	 * inconsistently or overwrite valid cache flags with 0.
+	 */
+	ui32FWCacheFlags = MMU_GetAndResetCacheFlags(psDevInfo->psKernelMMUCtx);
+	if (ui32FWCacheFlags == 0)
+	{
+		/* Nothing to do if no cache ops pending */
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = _PrepareAndSubmitCacheCommand(psDeviceNode, eDM, ui32FWCacheFlags,
+		                                       IMG_FALSE, pui32MMUInvalidateUpdate);
+		if (!PVRSRVIsRetryError(eError))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "_PrepareAndSubmitCacheCommand");
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+/*
+ * RGXMapBRN71422TargetPhysicalAddress
+ *
+ * Only used on early Volcanic cores.
+ * Set-up a special MMU tree mapping with a single page that eventually points
+ * to RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR. This is supplied by the
+ * customer (in rgxdefs_km.h) and hence this WA is off by default.
+ *
+ * PC entries are 32b, with the last 4 bits being 0 except for the LSB bit that should be 1 (Valid). Addr is 4KB aligned.
+ * PD entries are 64b, with addr in bits 39:5 and everything else 0 except for LSB bit that is Valid. Addr is byte aligned?
+ * PT entries are 64b, with phy addr in bits 39:12 and everything else 0 except for LSB bit that is Valid. Addr is 4KB aligned.
+ * So, we can construct the page tables in a single page like this:
+ *   0x00 : PCE (PCE index 0)
+ *   0x04 : 0x0
+ *   0x08 : PDEa (PDE index 1)
+ *   0x0C : PDEb
+ *   0x10 : PTEa (PTE index 2)
+ *   0x14 : PTEb
+ *
+ * With the PCE and the PDE pointing to this same page.
+ * The VA address that we are mapping is therefore:
+ *  VA = PCE_idx*PCE_size + PDE_idx*PDE_size + PTE_idx*PTE_size =
+ *     =      0 * 1GB     +      1 * 2MB     +      2 * 4KB     =
+ *     =        0         +    0x20_0000     +      0x2000      =
+ *     = 0x00_0020_2000
+ */
+void RGXMapBRN71422TargetPhysicalAddress(CONNECTION_DATA *psConnection,
+                                         PVRSRV_DEVICE_NODE *psDevNode,
+                                         IMG_DEV_PHYADDR sPhysAddrL1Px,
+                                         void *pxL1PxCpuVAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32       *pui32Px    = pxL1PxCpuVAddr;
+	IMG_UINT64       *pui64Px    = pxL1PxCpuVAddr;
+	IMG_UINT64       ui64Entry;
+
+
+	/* Only setup the BRN71422 workaround if this is the FW memory
+	 * context and BRN present.
+	 */
+	if ((psConnection != NULL) || !RGX_IS_BRN_SUPPORTED(psDevInfo, 71422))
+	{
+		return;
+	}
+
+	/* Setup dummy mapping to fast constant time physical address. */
+	/* PCE points to PC */
+	ui64Entry = sPhysAddrL1Px.uiAddr;
+	ui64Entry = ui64Entry >> RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT;
+	ui64Entry = ui64Entry << RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PC_DATA_VALID_EN;
+	pui32Px[0] = (IMG_UINT32) ui64Entry;
+
+	/* PDE points to PC */
+	ui64Entry = sPhysAddrL1Px.uiAddr;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PD_DATA_VALID_EN;
+	pui64Px[1] = ui64Entry;
+
+	/* PTE points to PAddr */
+	ui64Entry = RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PT_DATA_VALID_EN;
+	pui64Px[2] = ui64Entry;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Mapping the BRN71422 workaround to target physical address 0x%" IMG_UINT64_FMTSPECx ".",
+	         __func__, RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR));
+
+}
+#endif
+
+/* Common header, only needed for architectures with MIPS FW CPU */
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXMMUTweakProtFlags(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                  MMU_DEVICEATTRIBS *psDevAttrs,
+		                  PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+		                  MMU_PROTFLAGS_T *puiMMUProtFlags)
+{
+	/* If we are allocating on the MMU of the firmware processor, the
+	 * cached/uncached attributes must depend on the FIRMWARE_CACHED
+	 * allocation flag.
+	 */
+	if (psDevAttrs == psDevNode->psFirmwareMMUDevAttrs)
+	{
+		if (uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED))
+		{
+			*puiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
+		}
+		else
+		{
+			*puiMMUProtFlags &= ~MMU_PROTFLAGS_CACHED;
+
+		}
+		*puiMMUProtFlags &= ~MMU_PROTFLAGS_CACHE_COHERENT;
+	}
+}
+#endif
+
+
+/* page fault debug is the only current use case for needing to find process info
+ * after that process device memory context has been destroyed
+ */
+
+typedef struct _UNREGISTERED_MEMORY_CONTEXT_
+{
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_DEV_PHYADDR sPCDevPAddr;
+} UNREGISTERED_MEMORY_CONTEXT;
+
+/* must be a power of two */
+#define UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE (1 << 3)
+
+static UNREGISTERED_MEMORY_CONTEXT gasUnregisteredMemCtxs[UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE];
+static IMG_UINT32 gui32UnregisteredMemCtxsHead;
+
+/* record a device memory context being unregistered.
+ * the list of unregistered contexts can be used to find the PID and process name
+ * belonging to a memory context which has been destroyed
+ */
+static void _RecordUnregisteredMemoryContext(PVRSRV_RGXDEV_INFO *psDevInfo, SERVER_MMU_CONTEXT *psServerMMUContext)
+{
+	UNREGISTERED_MEMORY_CONTEXT *psRecord;
+
+	OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+	psRecord = &gasUnregisteredMemCtxs[gui32UnregisteredMemCtxsHead];
+
+	gui32UnregisteredMemCtxsHead = (gui32UnregisteredMemCtxsHead + 1)
+					& (UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1);
+
+	OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+
+	psRecord->uiPID = psServerMMUContext->uiPID;
+	if (MMU_AcquireBaseAddr(psServerMMUContext->psMMUContext, &psRecord->sPCDevPAddr) != PVRSRV_OK)
+	{
+		PVR_LOG(("_RecordUnregisteredMemoryContext: Failed to get PC address for memory context"));
+	}
+	OSStringSafeCopy(psRecord->szProcessName, psServerMMUContext->szProcessName, sizeof(psRecord->szProcessName));
+}
+
+
+void RGXUnregisterMemoryContext(IMG_HANDLE hPrivData)
+{
+	SERVER_MMU_CONTEXT *psServerMMUContext = hPrivData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psServerMMUContext->psDevInfo;
+
+#if defined(PDUMP)
+	{
+		RGXFWIF_DEV_VIRTADDR sFWAddr;
+
+		RGXSetFirmwareAddress(&sFWAddr,
+		                      psServerMMUContext->psFWMemContextMemDesc,
+		                      0,
+		                      RFW_FWADDR_NOREF_FLAG);
+
+		/*
+		 * MMU cache commands (always dumped) might have a pointer to this FW
+		 * memory context, wait until the FW has caught-up to the latest command.
+		 */
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+		             "Ensure FW has executed all MMU invalidations on FW memory "
+		             "context 0x%x before freeing it", sFWAddr.ui32Addr);
+		SyncPrimPDumpPol(psDevInfo->psDeviceNode->psMMUCacheSyncPrim,
+		                 psDevInfo->psDeviceNode->ui32NextMMUInvalidateUpdate - 1,
+		                 0xFFFFFFFF,
+		                 PDUMP_POLL_OPERATOR_GREATEREQUAL,
+		                 PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+	OSWRLockAcquireWrite(psDevInfo->hMemoryCtxListLock);
+	dllist_remove_node(&psServerMMUContext->sNode);
+	OSWRLockReleaseWrite(psDevInfo->hMemoryCtxListLock);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		_RecordUnregisteredMemoryContext(psDevInfo, psServerMMUContext);
+	}
+
+	/*
+	 * Release the page catalogue address acquired in RGXRegisterMemoryContext().
+	 */
+	MMU_ReleaseBaseAddr(NULL);
+
+	/*
+	 * Free the firmware memory context.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free FW memory context");
+	DevmemFwUnmapAndFree(psDevInfo, psServerMMUContext->psFWMemContextMemDesc);
+
+	OSFreeMem(psServerMMUContext);
+}
+
+/*
+ * RGXRegisterMemoryContext
+ */
+PVRSRV_ERROR RGXRegisterMemoryContext(PVRSRV_DEVICE_NODE	*psDeviceNode,
+									  MMU_CONTEXT			*psMMUContext,
+									  IMG_HANDLE			*hPrivData)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_FWMEMCONTEXT	*psFWMemContext;
+	DEVMEM_MEMDESC			*psFWMemContextMemDesc;
+	SERVER_MMU_CONTEXT *psServerMMUContext;
+
+	if (psDevInfo->psKernelMMUCtx == NULL)
+	{
+		/*
+		 * This must be the creation of the Kernel memory context. Take a copy
+		 * of the MMU context for use when programming the BIF.
+		 */
+		psDevInfo->psKernelMMUCtx = psMMUContext;
+	}
+	else
+	{
+		psServerMMUContext = OSAllocMem(sizeof(*psServerMMUContext));
+		if (psServerMMUContext == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_alloc_server_ctx;
+		}
+
+		psServerMMUContext->psDevInfo = psDevInfo;
+		psServerMMUContext->ui64FBSCEntryMask = 0;
+		psServerMMUContext->sFWMemContextDevVirtAddr.ui32Addr = 0;
+
+		/*
+			Allocate device memory for the firmware memory context for the new
+			application.
+		*/
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGX firmware memory context");
+		eError = DevmemFwAllocate(psDevInfo,
+								sizeof(*psFWMemContext),
+								RGX_FWCOMCTX_ALLOCFLAGS,
+								"FwMemoryContext",
+								&psFWMemContextMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to allocate firmware memory context (%u)",
+			         __func__,
+			         eError));
+			goto fail_alloc_fw_ctx;
+		}
+
+		/*
+			Temporarily map the firmware memory context to the kernel.
+		*/
+		eError = DevmemAcquireCpuVirtAddr(psFWMemContextMemDesc,
+										  (void **)&psFWMemContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map firmware memory context (%u)",
+			         __func__,
+			         eError));
+			goto fail_acquire_cpu_addr;
+		}
+
+		/*
+		 * Write the new memory context's page catalogue into the firmware memory
+		 * context for the client.
+		 */
+		eError = MMU_AcquireBaseAddr(psMMUContext, &psFWMemContext->sPCDevPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to acquire Page Catalogue address (%u)",
+			         __func__,
+			         eError));
+			DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+			goto fail_acquire_base_addr;
+		}
+
+		/*
+		 * Set default values for the rest of the structure.
+		 */
+		psFWMemContext->uiPageCatBaseRegSet = RGXFW_BIF_INVALID_PCSET;
+		psFWMemContext->uiBreakpointAddr = 0;
+		psFWMemContext->uiBPHandlerAddr = 0;
+		psFWMemContext->uiBreakpointCtl = 0;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+{
+		IMG_UINT32 ui32OSid = 0, ui32OSidReg = 0;
+		IMG_BOOL   bOSidAxiProt;
+
+		MMU_GetOSids(psMMUContext, &ui32OSid, &ui32OSidReg, &bOSidAxiProt);
+
+		psFWMemContext->ui32OSid     = ui32OSidReg;
+		psFWMemContext->bOSidAxiProt = bOSidAxiProt;
+}
+#endif
+
+#if defined(PDUMP)
+		{
+			IMG_CHAR			aszName[PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH];
+			IMG_DEVMEM_OFFSET_T uiOffset = 0;
+
+			/*
+			 * Dump the Mem context allocation
+			 */
+			DevmemPDumpLoadMem(psFWMemContextMemDesc, 0, sizeof(*psFWMemContext), PDUMP_FLAGS_CONTINUOUS);
+
+
+			/*
+			 * Obtain a symbolic addr of the mem context structure
+			 */
+			eError = DevmemPDumpPageCatBaseToSAddr(psFWMemContextMemDesc,
+												   &uiOffset,
+												   aszName,
+												   PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Failed to generate a Dump Page Catalogue address (%u)",
+				         __func__,
+				         eError));
+				DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+				goto fail_pdump_cat_base_addr;
+			}
+
+			/*
+			 * Dump the Page Cat tag in the mem context (symbolic address)
+			 */
+			eError = MMU_PDumpWritePageCatBase(psMMUContext,
+												aszName,
+												uiOffset,
+												8, /* 64-bit register write */
+												0,
+												0,
+												0);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Failed to acquire Page Catalogue address (%u)",
+				         __func__,
+				         eError));
+				DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+				goto fail_pdump_cat_base;
+			}
+		}
+#endif
+
+		/*
+		 * Release kernel address acquired above.
+		 */
+		RGXFwSharedMemCacheOpPtr(psFWMemContext, FLUSH);
+		DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+
+		/*
+		 * Store the process information for this device memory context
+		 * for use with the host page-fault analysis.
+		 */
+		psServerMMUContext->uiPID = OSGetCurrentClientProcessIDKM();
+		psServerMMUContext->psMMUContext = psMMUContext;
+		psServerMMUContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
+		OSStringSafeCopy(psServerMMUContext->szProcessName,
+		              OSGetCurrentClientProcessNameKM(),
+		              sizeof(psServerMMUContext->szProcessName));
+
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "New memory context: Process Name: %s PID: %u (0x%08X)",
+		                      psServerMMUContext->szProcessName,
+		                      psServerMMUContext->uiPID,
+		                      psServerMMUContext->uiPID);
+
+		OSWRLockAcquireWrite(psDevInfo->hMemoryCtxListLock);
+		dllist_add_to_tail(&psDevInfo->sMemoryContextList, &psServerMMUContext->sNode);
+		OSWRLockReleaseWrite(psDevInfo->hMemoryCtxListLock);
+
+		*hPrivData = psServerMMUContext;
+	}
+
+	return PVRSRV_OK;
+
+#if defined(PDUMP)
+fail_pdump_cat_base:
+fail_pdump_cat_base_addr:
+	MMU_ReleaseBaseAddr(NULL);
+#endif
+fail_acquire_base_addr:
+	/* Done before jumping to the fail point as the release is done before exit */
+fail_acquire_cpu_addr:
+	DevmemFwUnmapAndFree(psDevInfo, psServerMMUContext->psFWMemContextMemDesc);
+fail_alloc_fw_ctx:
+	OSFreeMem(psServerMMUContext);
+fail_alloc_server_ctx:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+DEVMEM_MEMDESC *RGXGetFWMemDescFromMemoryContextHandle(IMG_HANDLE hPriv)
+{
+	SERVER_MMU_CONTEXT *psMMUContext = (SERVER_MMU_CONTEXT *) hPriv;
+
+	return psMMUContext->psFWMemContextMemDesc;
+}
+
+void RGXSetFWMemContextDevVirtAddr(SERVER_MMU_CONTEXT *psServerMMUContext,
+						RGXFWIF_DEV_VIRTADDR	sFWMemContextAddr)
+{
+	psServerMMUContext->sFWMemContextDevVirtAddr.ui32Addr = sFWMemContextAddr.ui32Addr;
+}
+
+void RGXCheckFaultAddress(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_DEV_VIRTADDR *psDevVAddr,
+				IMG_DEV_PHYADDR *psDevPAddr,
+				MMU_FAULT_DATA *psOutFaultData)
+{
+	IMG_DEV_PHYADDR sPCDevPAddr;
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hMemoryCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psServerMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+
+		if (MMU_AcquireBaseAddr(psServerMMUContext->psMMUContext, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for memory context"));
+			continue;
+		}
+
+		if (psDevPAddr->uiAddr == sPCDevPAddr.uiAddr)
+		{
+			MMU_CheckFaultAddress(psServerMMUContext->psMMUContext, psDevVAddr, psOutFaultData);
+			goto out_unlock;
+		}
+	}
+
+	/* Lastly check for fault in the kernel allocated memory */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		if (MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for kernel memory context"));
+		}
+
+		if (psDevPAddr->uiAddr == sPCDevPAddr.uiAddr)
+		{
+			MMU_CheckFaultAddress(psDevInfo->psKernelMMUCtx, psDevVAddr, psOutFaultData);
+		}
+	}
+
+out_unlock:
+	OSWRLockReleaseRead(psDevInfo->hMemoryCtxListLock);
+}
+
+/* given the physical address of a page catalogue, searches for a corresponding
+ * MMU context and if found, provides the caller details of the process.
+ * Returns IMG_TRUE if a process is found.
+ */
+IMG_BOOL RGXPCAddrToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_PHYADDR sPCAddress,
+								RGXMEM_PROCESS_INFO *psInfo)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+
+	/* check if the input PC addr corresponds to an active memory context */
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psThisMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+		IMG_DEV_PHYADDR sPCDevPAddr;
+
+		if (MMU_AcquireBaseAddr(psThisMMUContext->psMMUContext, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for memory context"));
+			continue;
+		}
+
+		if (sPCAddress.uiAddr == sPCDevPAddr.uiAddr)
+		{
+			psServerMMUContext = psThisMMUContext;
+			break;
+		}
+	}
+
+	if (psServerMMUContext != NULL)
+	{
+		psInfo->uiPID = psServerMMUContext->uiPID;
+		OSStringSafeCopy(psInfo->szProcessName, psServerMMUContext->szProcessName, sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+	/* else check if the input PC addr corresponds to the firmware */
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_DEV_PHYADDR sKernelPCDevPAddr;
+		PVRSRV_ERROR eError;
+
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sKernelPCDevPAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for kernel memory context"));
+		}
+		else
+		{
+			if (sPCAddress.uiAddr == sKernelPCDevPAddr.uiAddr)
+			{
+				psInfo->uiPID = RGXMEM_SERVER_PID_FIRMWARE;
+				OSStringSafeCopy(psInfo->szProcessName, "Firmware", sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_FALSE;
+				bRet = IMG_TRUE;
+			}
+		}
+	}
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(bRet == IMG_FALSE))
+	{
+		/* no active memory context found with the given PC address.
+		 * Check the list of most recently freed memory contexts.
+		 */
+		const IMG_UINT32 ui32Mask = UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1;
+		IMG_UINT32 i, j;
+
+		OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+		/* iterate through the list of unregistered memory contexts
+		 * from newest (one before the head) to the oldest (the current head)
+		 */
+		for (i = (gui32UnregisteredMemCtxsHead - 1) & ui32Mask, j = 0;
+		     j < UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE;
+		     i = (i - 1) & ui32Mask, j++)
+		{
+			UNREGISTERED_MEMORY_CONTEXT *psRecord = &gasUnregisteredMemCtxs[i];
+
+			if (psRecord->sPCDevPAddr.uiAddr == sPCAddress.uiAddr)
+			{
+				psInfo->uiPID = psRecord->uiPID;
+				OSStringSafeCopy(psInfo->szProcessName, psRecord->szProcessName, sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_TRUE;
+				bRet = IMG_TRUE;
+				break;
+			}
+		}
+
+		OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	return bRet;
+}
+
+IMG_BOOL RGXPCPIDToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_PID uiPID,
+								RGXMEM_PROCESS_INFO *psInfo)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+
+	/* check if the input PID corresponds to an active memory context */
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psThisMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+
+		if (psThisMMUContext->uiPID == uiPID)
+		{
+			psServerMMUContext = psThisMMUContext;
+			break;
+		}
+	}
+
+	if (psServerMMUContext != NULL)
+	{
+		psInfo->uiPID = psServerMMUContext->uiPID;
+		OSStringSafeCopy(psInfo->szProcessName, psServerMMUContext->szProcessName, sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+	/* else check if the input PID corresponds to the firmware */
+	else if (uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		psInfo->uiPID = RGXMEM_SERVER_PID_FIRMWARE;
+		OSStringSafeCopy(psInfo->szProcessName, "Firmware", sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(bRet == IMG_FALSE))
+	{
+		/* if the PID didn't correspond to an active context or the
+		 * FW address then see if it matches a recently unregistered context
+		 */
+		const IMG_UINT32 ui32Mask = UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1;
+		IMG_UINT32 i, j;
+
+		OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+		for (i = (gui32UnregisteredMemCtxsHead - 1) & ui32Mask, j = 0;
+		     j < UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE;
+		     i = (i - 1) & ui32Mask, j++)
+		{
+			UNREGISTERED_MEMORY_CONTEXT *psRecord = &gasUnregisteredMemCtxs[i];
+
+			if (psRecord->uiPID == uiPID)
+			{
+				psInfo->uiPID = psRecord->uiPID;
+				OSStringSafeCopy(psInfo->szProcessName, psRecord->szProcessName, sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_TRUE;
+				bRet = IMG_TRUE;
+				break;
+			}
+		}
+
+		OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	return bRet;
+}
+
+IMG_PID RGXGetPIDFromServerMMUContext(SERVER_MMU_CONTEXT *psServerMMUContext)
+{
+	if (psServerMMUContext)
+	{
+		return psServerMMUContext->uiPID;
+	}
+	return 0;
+}
+
+/******************************************************************************
+ End of file (rgxmem.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.h
new file mode 100644
index 000000000000..2e959fc79eda
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmem.h
@@ -0,0 +1,157 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX memory context management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for RGX memory context management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXMEM_H)
+#define RGXMEM_H
+
+#include "pvrsrv_error.h"
+#include "device.h"
+#include "mmu_common.h"
+#include "rgxdevice.h"
+
+#define RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME 16
+
+/* this PID denotes the firmware */
+#define RGXMEM_SERVER_PID_FIRMWARE 0xFFFFFFFF
+
+/* this PID denotes the PM */
+#define RGXMEM_SERVER_PID_PM 0xEFFFFFFF
+
+typedef struct _RGXMEM_PROCESS_INFO_
+{
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_BOOL bUnregistered;
+} RGXMEM_PROCESS_INFO;
+
+typedef struct SERVER_MMU_CONTEXT_TAG SERVER_MMU_CONTEXT;
+
+IMG_DEV_PHYADDR GetPC(MMU_CONTEXT * psContext);
+
+void RGXSetFWMemContextDevVirtAddr(SERVER_MMU_CONTEXT *psServerMMUContext,
+			RGXFWIF_DEV_VIRTADDR	sFWMemContextAddr);
+
+void RGXMMUSyncPrimAlloc(PVRSRV_DEVICE_NODE *psDevNode);
+void RGXMMUSyncPrimFree(void);
+
+PVRSRV_ERROR RGXInvalidateFBSCTable(PVRSRV_DEVICE_NODE *psDeviceNode,
+									MMU_CONTEXT *psMMUContext,
+									IMG_UINT64 ui64FBSCEntryMask);
+
+PVRSRV_ERROR RGXExtractFBSCEntryMaskFromMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+												   SERVER_MMU_CONTEXT *psServerMMUContext,
+												   IMG_UINT64 *pui64FBSCEntryMask);
+
+void RGXMMUCacheInvalidate(PVRSRV_DEVICE_NODE *psDevNode,
+						   MMU_CONTEXT *psMMUContext,
+						   MMU_LEVEL eMMULevel,
+						   IMG_BOOL bUnmap);
+
+/*************************************************************************/ /*!
+@Function       RGXMMUCacheInvalidateKick
+
+@Description    Sends a flush command to a particular DM but first takes
+                the power lock.
+
+@Input          psDevNode   Device Node pointer
+@Input          pui32NextMMUInvalidateUpdate
+
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXMMUCacheInvalidateKick(PVRSRV_DEVICE_NODE *psDevNode,
+                                       IMG_UINT32 *pui32NextMMUInvalidateUpdate);
+
+/*************************************************************************/ /*!
+@Function       RGXPreKickCacheCommand
+
+@Description    Sends a cache flush command to a particular DM without
+                honouring the power lock. It's the caller's responsibility
+                to ensure power lock is held before calling this function.
+
+@Input          psDevInfo   Device Info
+@Input          eDM			To which DM the cmd is sent.
+@Input          pui32MMUInvalidateUpdate
+
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPreKickCacheCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+									RGXFWIF_DM eDM,
+									IMG_UINT32 *pui32MMUInvalidateUpdate);
+
+/* Needed for Volcanic architectures with BRN71422 */
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+void RGXMapBRN71422TargetPhysicalAddress(struct _CONNECTION_DATA_ *psConnection,
+		                                 struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                                 IMG_DEV_PHYADDR sPhysAddrL1Px,
+		                                 void *pxL1PxCpuVAddr);
+#endif
+
+/* Needed for Rogue architecture where a MIPS FW CPU is used */
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXMMUTweakProtFlags(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                  MMU_DEVICEATTRIBS *psDevAttrs,
+		                  PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+		                  MMU_PROTFLAGS_T *uiMMUProtFlags);
+#endif
+
+void RGXUnregisterMemoryContext(IMG_HANDLE hPrivData);
+PVRSRV_ERROR RGXRegisterMemoryContext(PVRSRV_DEVICE_NODE	*psDevNode,
+									  MMU_CONTEXT			*psMMUContext,
+									  IMG_HANDLE			*hPrivData);
+
+DEVMEM_MEMDESC *RGXGetFWMemDescFromMemoryContextHandle(IMG_HANDLE hPriv);
+
+void RGXCheckFaultAddress(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_DEV_VIRTADDR *psDevVAddr,
+				IMG_DEV_PHYADDR *psDevPAddr,
+				MMU_FAULT_DATA *psOutFaultData);
+
+IMG_BOOL RGXPCAddrToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_PHYADDR sPCAddress,
+								RGXMEM_PROCESS_INFO *psInfo);
+
+IMG_BOOL RGXPCPIDToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_PID uiPID,
+                               RGXMEM_PROCESS_INFO *psInfo);
+
+IMG_PID RGXGetPIDFromServerMMUContext(SERVER_MMU_CONTEXT *psServerMMUContext);
+
+#endif /* RGXMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.c
new file mode 100644
index 000000000000..29049ad8c827
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.c
@@ -0,0 +1,1421 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+#include "rgxmmuinit.h"
+#include "rgxmmudefs_km.h"
+
+#include "rgxdevice.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "mmu_common.h"
+#include "pdump_mmu.h"
+
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "rgx_memallocflags.h"
+#include "rgx_heaps_server.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+
+/* useful macros */
+/* units represented in a bitfield */
+#define UNITS_IN_BITFIELD(Mask, Shift)	((Mask >> Shift) + 1)
+
+/*
+ * Bits of PT, PD and PC not involving addresses
+ */
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/* protection bits for MMU_VERSION <= 3 */
+#define RGX_MMUCTRL_PTE_PROTMASK	(RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN | \
+		~RGX_MMUCTRL_PT_DATA_AXCACHE_CLRMSK | \
+		RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PT_DATA_PM_SRC_EN | \
+		RGX_MMUCTRL_PT_DATA_CC_EN | \
+		RGX_MMUCTRL_PT_DATA_READ_ONLY_EN | \
+		RGX_MMUCTRL_PT_DATA_VALID_EN)
+#else
+#define RGX_MMUCTRL_PTE_PROTMASK	(RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN | \
+		RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PT_DATA_PM_SRC_EN | \
+		RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN | \
+		RGX_MMUCTRL_PT_DATA_CC_EN | \
+		RGX_MMUCTRL_PT_DATA_READ_ONLY_EN | \
+		RGX_MMUCTRL_PT_DATA_VALID_EN)
+#endif
+
+#define RGX_MMUCTRL_PDE_PROTMASK	(RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN | \
+		~RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK | \
+		RGX_MMUCTRL_PD_DATA_VALID_EN)
+
+#define RGX_MMUCTRL_PCE_PROTMASK	(RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PC_DATA_VALID_EN)
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*
+ * protection bits for MMU_VERSION >= 4
+ * MMU4 has no PENDING or PAGE_SIZE fields in PxE
+ */
+#define RGX_MMU4CTRL_PTE_PROTMASK	(RGX_MMUCTRL_PTE_PROTMASK & ~RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN)
+
+#define RGX_MMU4CTRL_PDE_PROTMASK	(RGX_MMUCTRL_PD_DATA_VALID_EN)
+
+#define RGX_MMU4CTRL_PCE_PROTMASK	(RGX_MMUCTRL_PC_DATA_VALID_EN)
+#endif
+
+
+static MMU_PxE_CONFIG sRGXMMUPCEConfig;
+static MMU_DEVVADDR_CONFIG sRGXMMUTopLevelDevVAddrConfig;
+
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_4KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_4KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_4KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig4KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_16KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_16KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_16KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig16KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_64KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_64KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_64KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig64KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_256KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_256KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig256KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 1MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_1MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_1MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_1MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig1MB;
+
+
+/*
+ *
+ *  Configuration for heaps with 2MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_2MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_2MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_2MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig2MB;
+
+
+/* Forward declaration of protection bits derivation functions, for
+   the following structure */
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags);
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/* protection bits derivation functions for MMUv4 */
+static IMG_UINT64 RGXMMU4DerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromVirtAddr(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												   IMG_DEV_VIRTADDR sDevVAddr,
+												   IMG_UINT32 *pui32Log2PageSize);
+#endif
+
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+		const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+		const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+		const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+		IMG_HANDLE *phPriv);
+
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv);
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+
+static MMU_DEVICEATTRIBS sRGXMMUDeviceAttributes;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+/* This is a basis for a pattern of parity bit values for consecutive VAs.
+   For each PT with 512 entries we'd get either this pattern or its reverse. */
+static const IMG_UINT8 ui8ParityPTPattern[512] = {
+    0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,
+    1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0,
+    0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+    1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
+    0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1,
+    0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1,
+    1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,
+    1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1,
+    0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1,
+    1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0,
+    0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0,
+    1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0,
+    0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+    1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
+    0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1,
+    1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1};
+#endif
+
+PVRSRV_ERROR RGXMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK) || defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	IMG_BOOL bHaveParity = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MH_PARITY);
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	IMG_BOOL bHaveMMU4 = (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4);
+#define RGX_GET_MMUCTRL_PROTMASK(entry) \
+	(bHaveMMU4 ? RGX_MMU4CTRL_##entry##_PROTMASK : RGX_MMUCTRL_##entry##_PROTMASK)
+#else
+#define RGX_GET_MMUCTRL_PROTMASK(entry) RGX_MMUCTRL_##entry##_PROTMASK
+#endif
+
+	/* Setup of Px Entries:
+	 *
+	 *
+	 * PAGE TABLE (8 Byte):
+	 *
+	 * | 62              | 61...40         | 39...12 (varies) | 11...6          | 5             | 4      | 3               | 2               | 1         | 0     |
+	 * | PM/Meta protect | VP Page (39:18) | Physical Page    | VP Page (17:12) | Entry Pending | PM src | SLC Bypass Ctrl | Cache Coherency | Read Only | Valid |
+	 *
+	 *
+	 * PAGE DIRECTORY (8 Byte):
+	 *
+	 *  | 40            | 39...5  (varies)        | 4          | 3...1     | 0     |
+	 *  | Entry Pending | Page Table base address | (reserved) | Page Size | Valid |
+	 *
+	 *
+	 * PAGE CATALOGUE (4 Byte):
+	 *
+	 *  | 31...4                      | 3...2      | 1             | 0     |
+	 *  | Page Directory base address | (reserved) | Entry Pending | Valid |
+	 *
+	 */
+
+
+	/* Example how to get the PD address from a PC entry.
+	 * The procedure is the same for PD and PT entries to retrieve PT and Page addresses:
+	 *
+	 * 1) sRGXMMUPCEConfig.uiAddrMask applied to PC entry with '&':
+	 *  | 31...4   | 3...2      | 1             | 0     |
+	 *  | PD Addr  | 0          | 0             | 0     |
+	 *
+	 * 2) sRGXMMUPCEConfig.uiAddrShift applied with '>>':
+	 *  | 27...0   |
+	 *  | PD Addr  |
+	 *
+	 * 3) sRGXMMUPCEConfig.uiAddrLog2Align applied with '<<':
+	 *  | 39...0   |
+	 *  | PD Addr  |
+	 *
+	 */
+
+
+	sRGXMMUDeviceAttributes.pszMMUPxPDumpMemSpaceName =
+			PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+
+	/*
+	 * Setup sRGXMMUPCEConfig
+	 */
+	sRGXMMUPCEConfig.ePxLevel = MMU_LEVEL_3;
+	sRGXMMUPCEConfig.pszPxLevelStr = "PC";
+	sRGXMMUPCEConfig.uiBytesPerEntry = 4;     /* 32 bit entries */
+
+	sRGXMMUPCEConfig.uiAddrMask = 0xfffffff0; /* Mask to get significant address bits of PC entry i.e. the address of the PD */
+	sRGXMMUPCEConfig.uiAddrShift = 4;         /* Shift this many bits to get PD address */
+	sRGXMMUPCEConfig.uiAddrLog2Align = 12;    /* Alignment of PD physical addresses. */
+
+	sRGXMMUPCEConfig.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PCE); /* Mask to get the status bits */
+
+	sRGXMMUPCEConfig.uiProtShift = 0;                       /* Shift this many bits to get the status bits */
+
+	sRGXMMUPCEConfig.uiValidEnMask = RGX_MMUCTRL_PC_DATA_VALID_EN;     /* Mask to get entry valid bit of the PC */
+	sRGXMMUPCEConfig.uiValidEnShift = RGX_MMUCTRL_PC_DATA_VALID_SHIFT; /* Shift this many bits to get entry valid bit */
+
+	/*
+	 *  Setup sRGXMMUTopLevelDevVAddrConfig
+	 */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK; /* Mask to get PC index applied to a 40 bit virt. device address */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;  /* Shift a 40 bit virt. device address by this amount to get the PC index */
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask,
+			sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift));
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK; /* Mask to get PD index applied to a 40 bit virt. device address */
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;  /* Shift a 40 bit virt. device address by this amount to get the PD index */
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask,
+			sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift));
+
+	/*
+	 *
+	 *  Configuration for heaps with 4kB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20   12   11        0
+	 *      |         \ /       \ /       \ /           \
+	 *      |....PCE...|...PDE...|...PTE...|....VAddr...|
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_4KBDP
+	 */
+	sRGXMMUPDEConfig_4KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_4KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_4KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_4KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_4KBDP.uiAddrShift = 12;
+	sRGXMMUPDEConfig_4KBDP.uiAddrLog2Align = 12;
+
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_4KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_4KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_4KBDP
+	 */
+	sRGXMMUPTEConfig_4KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_4KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_4KBDP.uiAddrMask = IMG_UINT64_C(0xfffffff000);
+	sRGXMMUPTEConfig_4KBDP.uiAddrShift = 12;
+	sRGXMMUPTEConfig_4KBDP.uiAddrLog2Align = 12; /* Alignment of the physical addresses of the pages NOT PTs */
+
+	sRGXMMUPTEConfig_4KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_4KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_4KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_4KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_4KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_4KBDP
+	 */
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask = ~RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift = RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000000fff);
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig4KB
+	 */
+	gsPageSizeConfig4KB.psPDEConfig = &sRGXMMUPDEConfig_4KBDP;
+	gsPageSizeConfig4KB.psPTEConfig = &sRGXMMUPTEConfig_4KBDP;
+	gsPageSizeConfig4KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_4KBDP;
+	gsPageSizeConfig4KB.uiRefCount = 0;
+	gsPageSizeConfig4KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 16kB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20 14   13          0
+	 *      |         \ /       \ /     \ /             \
+	 *      |....PCE...|...PDE...|..PTE..|.....VAddr....|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_16KBDP
+	 */
+	sRGXMMUPDEConfig_16KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_16KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_16KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_16KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_16KBDP.uiAddrShift = 10;
+	sRGXMMUPDEConfig_16KBDP.uiAddrLog2Align = 10;
+
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_16KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_16KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_16KBDP
+	 */
+	sRGXMMUPTEConfig_16KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_16KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_16KBDP.uiAddrMask = IMG_UINT64_C(0xffffffc000);
+	sRGXMMUPTEConfig_16KBDP.uiAddrShift = 14;
+	sRGXMMUPTEConfig_16KBDP.uiAddrLog2Align = 14;
+
+	sRGXMMUPTEConfig_16KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_16KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_16KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_16KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_16KBDP
+	 */
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001fc000);
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift = 14;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift));
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000003fff);
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig16KB
+	 */
+	gsPageSizeConfig16KB.psPDEConfig = &sRGXMMUPDEConfig_16KBDP;
+	gsPageSizeConfig16KB.psPTEConfig = &sRGXMMUPTEConfig_16KBDP;
+	gsPageSizeConfig16KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_16KBDP;
+	gsPageSizeConfig16KB.uiRefCount = 0;
+	gsPageSizeConfig16KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 64kB Data-Page size
+	 *
+	 * Bit  39      30   29   21  20 16 15             0
+	 *      |         \ /       \ /   \ /               \
+	 *      |....PCE...|...PDE...|.PTE.|.....VAddr......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_64KBDP
+	 */
+	sRGXMMUPDEConfig_64KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_64KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_64KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_64KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_64KBDP.uiAddrShift = 8;
+	sRGXMMUPDEConfig_64KBDP.uiAddrLog2Align = 8;
+
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_64KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_64KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_64KBDP
+	 */
+	sRGXMMUPTEConfig_64KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_64KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_64KBDP.uiAddrMask = IMG_UINT64_C(0xffffff0000);
+	sRGXMMUPTEConfig_64KBDP.uiAddrShift = 16;
+	sRGXMMUPTEConfig_64KBDP.uiAddrLog2Align = 16;
+
+	sRGXMMUPTEConfig_64KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_64KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_64KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_64KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_64KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_64KBDP
+	 */
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001f0000);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift = 16;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000000ffff);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig64KB
+	 */
+	gsPageSizeConfig64KB.psPDEConfig = &sRGXMMUPDEConfig_64KBDP;
+	gsPageSizeConfig64KB.psPTEConfig = &sRGXMMUPTEConfig_64KBDP;
+	gsPageSizeConfig64KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_64KBDP;
+	gsPageSizeConfig64KB.uiRefCount = 0;
+	gsPageSizeConfig64KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 256kB Data-Page size
+	 *
+	 * Bit  39      30   29   21 20 18 17              0
+	 *      |         \ /       \|   |/                 \
+	 *      |....PCE...|...PDE...|PTE|.......VAddr......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_256KBDP
+	 */
+	sRGXMMUPDEConfig_256KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_256KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_256KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_256KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_256KBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_256KBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_256KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_256KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP
+	 */
+	sRGXMMUPTEConfig_256KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_256KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_256KBDP.uiAddrMask = IMG_UINT64_C(0xfffffc0000);
+	sRGXMMUPTEConfig_256KBDP.uiAddrShift = 18;
+	sRGXMMUPTEConfig_256KBDP.uiAddrLog2Align = 18;
+
+	sRGXMMUPTEConfig_256KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_256KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_256KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_256KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_256KBDP
+	 */
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001c0000);
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift = 18;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000003ffff);
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig256KB
+	 */
+	gsPageSizeConfig256KB.psPDEConfig = &sRGXMMUPDEConfig_256KBDP;
+	gsPageSizeConfig256KB.psPTEConfig = &sRGXMMUPTEConfig_256KBDP;
+	gsPageSizeConfig256KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_256KBDP;
+	gsPageSizeConfig256KB.uiRefCount = 0;
+	gsPageSizeConfig256KB.uiMaxRefCount = 0;
+
+	/*
+	 *
+	 *  Configuration for heaps with 1MB Data-Page size
+	 *
+	 * Bit  39      30   29   21 20  19               0
+	 *      |         \ /       \ | /                  \
+	 *      |....PCE...|...PDE...|.|........VAddr.......|
+	 *                           PTE
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_1MBDP
+	 */
+	sRGXMMUPDEConfig_1MBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_1MBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_1MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_1MBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	/*
+	 * The hardware requires that PT tables need be 1<<6 = 64 byte aligned even
+	 * if they contain fewer entries.
+	 */
+	sRGXMMUPDEConfig_1MBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_1MBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_1MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_1MBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_1MBDP
+	 */
+	sRGXMMUPTEConfig_1MBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_1MBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_1MBDP.uiAddrMask = IMG_UINT64_C(0xfffff00000);
+	sRGXMMUPTEConfig_1MBDP.uiAddrShift = 20;
+	sRGXMMUPTEConfig_1MBDP.uiAddrLog2Align = 20;
+
+	sRGXMMUPTEConfig_1MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_1MBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_1MBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_1MBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_1MBDP
+	 */
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask = IMG_UINT64_C(0x0000100000);
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift = 20;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetMask = IMG_UINT64_C(0x00000fffff);
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig1MB
+	 */
+	gsPageSizeConfig1MB.psPDEConfig = &sRGXMMUPDEConfig_1MBDP;
+	gsPageSizeConfig1MB.psPTEConfig = &sRGXMMUPTEConfig_1MBDP;
+	gsPageSizeConfig1MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_1MBDP;
+	gsPageSizeConfig1MB.uiRefCount = 0;
+	gsPageSizeConfig1MB.uiMaxRefCount = 0;
+
+	/*
+	 *
+	 *  Configuration for heaps with 2MB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20                0
+	 *      |         \ /       \ /                   \
+	 *      |....PCE...|...PDE...|.........VAddr.......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_2MBDP
+	 */
+	sRGXMMUPDEConfig_2MBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_2MBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_2MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_2MBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	/*
+	 * The hardware requires that PT tables need be 1<<6 = 64 byte aligned even
+	 * if they contain fewer entries.
+	 */
+	sRGXMMUPDEConfig_2MBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_2MBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_2MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_2MBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_2MBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_2MBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUPTEConfig_2MBDP
+	 */
+	sRGXMMUPTEConfig_2MBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_2MBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_2MBDP.uiAddrMask = IMG_UINT64_C(0xffffe00000);
+	sRGXMMUPTEConfig_2MBDP.uiAddrShift = 21;
+	sRGXMMUPTEConfig_2MBDP.uiAddrLog2Align = 21;
+
+	sRGXMMUPTEConfig_2MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_2MBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_2MBDP
+	 */
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask = IMG_UINT64_C(0x0000000000);
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift = 21;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetMask = IMG_UINT64_C(0x00001fffff);
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig2MB
+	 */
+	gsPageSizeConfig2MB.psPDEConfig = &sRGXMMUPDEConfig_2MBDP;
+	gsPageSizeConfig2MB.psPTEConfig = &sRGXMMUPTEConfig_2MBDP;
+	gsPageSizeConfig2MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_2MBDP;
+	gsPageSizeConfig2MB.uiRefCount = 0;
+	gsPageSizeConfig2MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUDeviceAttributes
+	 */
+	sRGXMMUDeviceAttributes.eMMUType = PDUMP_MMU_TYPE_VARPAGE_40BIT;
+	sRGXMMUDeviceAttributes.ui32BaseAlign = RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT;
+	sRGXMMUDeviceAttributes.psBaseConfig = &sRGXMMUPCEConfig;
+	sRGXMMUDeviceAttributes.psTopLevelDevVAddrConfig = &sRGXMMUTopLevelDevVAddrConfig;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	{
+		IMG_UINT32 i;
+
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT   <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT  <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT  <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT   <= 512);
+		PVR_ASSERT(sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT   <= 512);
+
+		PVR_ASSERT(sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry   == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry  == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry  == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry   == 8);
+		PVR_ASSERT(sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry   == 8);
+
+		sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[0] =
+		    OSAllocZMem(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT * sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry);
+		sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[1] =
+		    OSAllocZMem(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT * sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry);
+
+		/* Generate two precomputed pages in host memory for zero-initialised PTs with parity bits set. */
+		for (i=0; i<sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT; i++)
+		{
+			sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[0][i] =
+			    ((IMG_UINT64)(ui8ParityPTPattern[i]))         << RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+			sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[1][i] =
+			    ((IMG_UINT64)(ui8ParityPTPattern[i] ? 0 : 1)) << RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+		}
+	}
+#endif
+
+	/* Functions for deriving page table/dir/cat protection bits */
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt8 = RGXDerivePCEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt4 = RGXDerivePCEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXDerivePDEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt4 = RGXDerivePDEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt8 = RGXDerivePTEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt4 = RGXDerivePTEProt4;
+
+	/* Functions for establishing configurations for PDE/PTE/DEVVADDR
+	   on per-heap basis */
+	sRGXMMUDeviceAttributes.pfnGetPageSizeConfiguration = RGXGetPageSizeConfigCB;
+	sRGXMMUDeviceAttributes.pfnPutPageSizeConfiguration = RGXPutPageSizeConfigCB;
+
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE4 = RGXGetPageSizeFromPDE4;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXGetPageSizeFromPDE8;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromVirtAddr = NULL;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (bHaveMMU4)
+	{
+		/* override some of these functions for MMU4 as page size is not stored in PD entries */
+		sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXMMU4DerivePDEProt8;
+		sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXMMU4GetPageSizeFromPDE8;
+		sRGXMMUDeviceAttributes.pfnGetPageSizeFromVirtAddr = RGXMMU4GetPageSizeFromVirtAddr;
+	}
+#endif
+
+	sRGXMMUDeviceAttributes.ui32ValidPageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	psDeviceNode->psMMUDevAttrs = &sRGXMMUDeviceAttributes;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	psDeviceNode->pfnMMUGetContextID = NULL;
+#endif
+
+	psDeviceNode->psMMUDevAttrs = NULL;
+
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_MESSAGE, "Variable Page Size Heap Stats:"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 4K page heaps: %d",
+			gsPageSizeConfig4KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 4K page heaps (should be 0): %d",
+			gsPageSizeConfig4KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 16K page heaps: %d",
+			gsPageSizeConfig16KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 16K page heaps (should be 0): %d",
+			gsPageSizeConfig16KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 64K page heaps: %d",
+			gsPageSizeConfig64KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 64K page heaps (should be 0): %d",
+			gsPageSizeConfig64KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 256K page heaps: %d",
+			gsPageSizeConfig256KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 256K page heaps (should be 0): %d",
+			gsPageSizeConfig256KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 1M page heaps: %d",
+			gsPageSizeConfig1MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 1M page heaps (should be 0): %d",
+			gsPageSizeConfig1MB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 2M page heaps: %d",
+			gsPageSizeConfig2MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 2M page heaps (should be 0): %d",
+			gsPageSizeConfig2MB.uiRefCount));
+#endif
+	if (gsPageSizeConfig4KB.uiRefCount > 0 ||
+			gsPageSizeConfig16KB.uiRefCount > 0 ||
+			gsPageSizeConfig64KB.uiRefCount > 0 ||
+			gsPageSizeConfig256KB.uiRefCount > 0 ||
+			gsPageSizeConfig1MB.uiRefCount > 0 ||
+			gsPageSizeConfig2MB.uiRefCount > 0
+	)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXMMUInit_Unregister: Unbalanced MMU API Usage (Internal error)"));
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt4
+@Description    calculate the PCE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags)
+{
+	return (uiProtFlags & MMU_PROTFLAGS_INVALID)?0:RGX_MMUCTRL_PC_DATA_VALID_EN;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt8
+@Description    calculate the PCE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "8-byte PCE not supported on this device"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt4
+@Description    derive the PDE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PDE not supported on this device"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize The log2 of the required page size.
+                E.g, for 4KiB pages, this parameter must be 12.
+                For 2MiB pages, it must be set to 21.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ret_value = 0; /* 0 means invalid */
+
+	if (!(uiProtFlags & MMU_PROTFLAGS_INVALID)) /* if not invalid */
+	{
+		switch (uiLog2DataPageSize)
+		{
+		case RGX_HEAP_4KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB;
+			break;
+		case RGX_HEAP_16KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB;
+			break;
+		case RGX_HEAP_64KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB;
+			break;
+		case RGX_HEAP_256KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB;
+			break;
+		case RGX_HEAP_1MB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB;
+			break;
+		case RGX_HEAP_2MB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid parameter log2_page_size. Expected {12, 14, 16, 18, 20, 21}. Got [%u]",
+					__func__, uiLog2DataPageSize));
+		}
+	}
+	return ret_value;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt4
+@Description    calculate the PTE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PTE not supported on this device"));
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt8
+@Description    calculate the PTE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ui64MMUFlags=0;
+
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	if (((MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE) & uiProtFlags) == (MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE))
+	{
+		/* read/write */
+	}
+	else if (MMU_PROTFLAGS_READABLE & uiProtFlags)
+	{
+		/* read only */
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_READ_ONLY_EN;
+	}
+	else if (MMU_PROTFLAGS_WRITEABLE & uiProtFlags)
+	{
+		/* write only */
+		PVR_DPF((PVR_DBG_WARNING, "RGXDerivePTEProt8: write-only is not possible on this device"));
+	}
+	else if ((MMU_PROTFLAGS_INVALID & uiProtFlags) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt8: neither read nor write specified..."));
+	}
+
+	/* cache coherency */
+	if (MMU_PROTFLAGS_CACHE_COHERENT & uiProtFlags)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_CC_EN;
+	}
+
+#if defined(RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN)
+	/* cache setup */
+	if ((MMU_PROTFLAGS_CACHED & uiProtFlags) == 0)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN;
+	}
+#endif
+
+	if ((uiProtFlags & MMU_PROTFLAGS_INVALID) == 0)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_VALID_EN;
+	}
+
+	if (MMU_PROTFLAGS_DEVICE(PMMETA_PROTECT) & uiProtFlags)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN;
+	}
+
+#if defined(RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC)
+	/**
+	 * Always enable caching on the fabric level cache irrespective of type of
+	 * cache coherent interconnect and memory cache attributes.
+	 * This needs to be updated, if selective caching policy needs to be
+	 * implemented based on cache attributes requested by caller and based on
+	 * cache coherent interconnect.
+	 */
+	ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC;
+#endif
+
+	return ui64MMUFlags;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXGetPageSizeConfig
+@Description    Set up configuration for variable sized data pages.
+                RGXPutPageSizeConfigCB has to be called to ensure correct
+                refcounting.
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+		const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+		const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+		const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+		IMG_HANDLE *phPriv)
+{
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGX_HEAP_4KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	case RGX_HEAP_16KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig16KB;
+		break;
+	case RGX_HEAP_64KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGX_HEAP_256KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig256KB;
+		break;
+	case RGX_HEAP_1MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig1MB;
+		break;
+	case RGX_HEAP_2MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig2MB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXGetPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				uiLog2DataPageSize));
+		*phPriv = NULL;
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Refer caller's pointers to the data */
+	*ppsMMUPDEConfig = psPageSizeConfig->psPDEConfig;
+	*ppsMMUPTEConfig = psPageSizeConfig->psPTEConfig;
+	*ppsMMUDevVAddrConfig = psPageSizeConfig->psDevVAddrConfig;
+
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	/* Increment ref-count - not that we're allocating anything here
+       (I'm using static structs), but one day we might, so we want
+       the Get/Put code to be balanced properly */
+	psPageSizeConfig->uiRefCount++;
+
+	/* This is purely for debug statistics */
+	psPageSizeConfig->uiMaxRefCount = MAX(psPageSizeConfig->uiMaxRefCount,
+			psPageSizeConfig->uiRefCount);
+#endif
+
+	*phPriv = (IMG_HANDLE)(uintptr_t)uiLog2DataPageSize;
+	PVR_ASSERT (uiLog2DataPageSize == (IMG_UINT32)(uintptr_t)*phPriv);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPutPageSizeConfig
+@Description    Tells this code that the mmu module is done with the
+                configurations set in RGXGetPageSizeConfig.  This can
+                be a no-op.
+                Called after RGXGetPageSizeConfigCB.
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv)
+{
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+	IMG_UINT32 uiLog2DataPageSize;
+
+	uiLog2DataPageSize = (IMG_UINT32)(uintptr_t) hPriv;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGX_HEAP_4KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	case RGX_HEAP_16KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig16KB;
+		break;
+	case RGX_HEAP_64KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGX_HEAP_256KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig256KB;
+		break;
+	case RGX_HEAP_1MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig1MB;
+		break;
+	case RGX_HEAP_2MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig2MB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXPutPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				uiLog2DataPageSize));
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Ref-count here is not especially useful, but it's an extra
+	   check that the API is being used correctly */
+	psPageSizeConfig->uiRefCount--;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PDE not supported on this device"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	IMG_UINT64 ui64PageSizeBits = ui64PDE & (~RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK);
+
+	switch (ui64PageSizeBits)
+	{
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB:
+		*pui32Log2PageSize = RGX_HEAP_4KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB:
+		*pui32Log2PageSize = RGX_HEAP_16KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB:
+		*pui32Log2PageSize = RGX_HEAP_64KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB:
+		*pui32Log2PageSize = RGX_HEAP_256KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB:
+		*pui32Log2PageSize = RGX_HEAP_1MB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB:
+		*pui32Log2PageSize = RGX_HEAP_2MB_PAGE_SHIFT;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXGetPageSizeFromPDE8: Invalid page size bitfield %" IMG_UINT64_FMTSPECx " in PDE",
+				ui64PageSizeBits));
+
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*************************************************************************/ /*!
+@Function       RGXMMU4DerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize: ignored as MMU4 doesn't put page size in PD entries.
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXMMU4DerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ret_value = 0; /* 0 means invalid */
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	if (!(uiProtFlags & MMU_PROTFLAGS_INVALID)) /* if not invalid */
+	{
+		/*  page size in range config registers. Bits in PD entries are reserved */
+		ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	}
+	return ret_value;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXMMU4GetPageSizeFromPDE8
+@Description    The upper layers should be such that this function is never called
+                as pages size are not stored in PD entries for MMU4.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+
+	PVR_ASSERT(0 && "RGXMMU4GetPageSizeFromPDE8 called in error. MMU4 does not store page sizes in PDT.");
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXMMU4GetPageSizeFromVirtAddr
+@Description    Get page size by walking through range config registers
+                looking for a match against the virtual address.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromVirtAddr(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												   IMG_DEV_VIRTADDR sDevVAddr,
+												   IMG_UINT32 *pui32Log2PageSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+
+	/* assume default of 4KB page size */
+	*pui32Log2PageSize = 12;
+
+	/* Loop through the range registers looking for the given target address */
+	for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+	{
+		IMG_UINT64 ui64RangeVal = psDevInfo->aui64MMUPageSizeRangeValue[i];
+
+		if (ui64RangeVal != 0)
+		{
+			/* end addr in register is inclusive in the range so add 1 to move it over the end */
+			IMG_UINT64 ui64Base = ((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT;
+			IMG_UINT64 ui64End  = (((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_CLRMSK)
+												>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_SHIFT) + 1)
+												<< RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT;
+
+			if ((sDevVAddr.uiAddr >= ui64Base) && (sDevVAddr.uiAddr < ui64End))
+			{
+				IMG_UINT32 ui32PageSizeField = (IMG_UINT32)((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_CLRMSK)
+														>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT);
+				if (ui32PageSizeField < 5)
+				{
+					*pui32Log2PageSize = (ui32PageSizeField << 1) + 12;   /* 12 (4K), 14 (16K), 16 (64K), 18 (256K), 20 (1MB) */
+				}
+				else if (ui32PageSizeField == 5)
+				{
+					*pui32Log2PageSize = 21;    /* 2MB */
+				}
+				else
+				{
+					eError = PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+				}
+				break;
+			}
+		}
+	}
+
+	return eError;
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.h
new file mode 100644
index 000000000000..b70ddad0cb64
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmmuinit.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* NB: this file is not to be included arbitrarily.  It exists solely
+   for the linkage between rgxinit.c and rgxmmuinit.c, the former
+   being otherwise cluttered by the contents of the latter */
+
+#ifndef SRVKM_RGXMMUINIT_H
+#define SRVKM_RGXMMUINIT_H
+
+#include "device.h"
+#include "img_types.h"
+#include "mmu_common.h"
+#include "img_defs.h"
+
+PVRSRV_ERROR RGXMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR RGXMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* #ifndef SRVKM_RGXMMUINIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmulticore.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmulticore.h
new file mode 100644
index 000000000000..b45a20ab5e14
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxmulticore.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.h
+@Title          Functions related to multicore devices
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose memory shared between kernel driver and user
+                mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXMULTICORE_H
+#define RGXMULTICORE_H
+
+#include "pvrsrv_error.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode);
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* RGXMULTICORE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump.h
new file mode 100644
index 000000000000..365269af59ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump.h
@@ -0,0 +1,213 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX pdump Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX pdump functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxdevice.h"
+#include "device.h"
+#include "devicemem.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpSignatureBufferKM
+
+ @Description
+
+ Dumps TA and 3D signature and checksum buffers
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_UINT32 ui32PDumpFlags);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpCRCSignatureCheckKM
+
+ @Description
+
+ Poll on FBC/FBDC end-to-end signature status
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpCRCSignatureCheckKM(CONNECTION_DATA * psConnection,
+                                            PVRSRV_DEVICE_NODE * psDeviceNode,
+                                            IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpValCheckPreCommandKM
+
+ @Description
+
+ Poll on various GPU status/signature status for validation, before
+ sending the GPU command.
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpValCheckPreCommandKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpValCheckPostCommandKM
+
+ @Description
+
+ Poll on various GPU status/signature status for validation, after
+ sending the GPU command.
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpValCheckPostCommandKM(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE * psDeviceNode,
+                                              IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVPDumpTraceBufferKM
+
+ @Description
+
+ Dumps TA and 3D signature and checksum buffers
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPDumpTraceBufferKM(CONNECTION_DATA * psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXPDumpPrepareOutputImageDescriptorHdr
+
+ @Description
+
+ Dumps the header for an OutputImage command
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXPDumpPrepareOutputImageDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_PBYTE abyPDumpDesc);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXPDumpPrepareOutputDataDescriptorHdr
+
+ @Description
+
+ Dumps the header for an OutputData command
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXPDumpPrepareOutputDataDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_PBYTE pbyPDumpDataHdr);
+
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVPDumpSignatureBufferKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVPDumpTraceBufferKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVPDumpTraceBufferKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+/******************************************************************************
+ End of file (rgxpdump.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.c
new file mode 100644
index 000000000000..fb1303c5f565
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.c
@@ -0,0 +1,121 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX PDump Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX common PDump functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxpdump_common.h"
+#include "devicemem_utils.h"
+#include "devicemem_server.h"
+
+PVRSRV_ERROR PDumpGetValidRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 DEVMEM_MEMDESC *psMemDesc,
+                                 IMG_UINT32 uiSize,
+                                 DLLIST_NODE *psRegionList)
+{
+	IMG_DEV_VIRTADDR sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset;
+
+	return DevmemIntPDumpGetValidRegions(
+		GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+		psDeviceNode,
+		psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+		sDevAddrStart,
+		uiSize,
+		psRegionList
+	);
+}
+
+void PDumpSaveToFileVirtual(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            DEVMEM_MEMDESC *psMemDesc,
+                            DLLIST_NODE *psRegionList,
+                            const IMG_CHAR *pszFileName,
+                            IMG_UINT32 uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DevmemIntPDumpSaveFromRegionListToFileVirtual(
+		GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+		psDeviceNode,
+		psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+		psRegionList,
+		pszFileName,
+		0,
+		uiPDumpFlags
+	);
+	PVR_LOG_IF_ERROR(eError, "DevmemIntPDumpSaveFromRegionListToFileVirtual");
+
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+}
+
+void PDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      DEVMEM_MEMDESC *psMemDesc,
+                                      IMG_DEVMEM_OFFSET_T uiOffset,
+                                      IMG_UINT32 uiSize,
+                                      const IMG_CHAR *pszFileName,
+                                      IMG_UINT32 uiFileOffset,
+                                      IMG_UINT32 uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset + uiOffset;
+
+	eError = DevmemIntPDumpSaveToFileVirtualNoValidate(
+		psDeviceNode,
+		psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+		sDevAddrStart,
+		uiSize,
+		pszFileName,
+		uiFileOffset,
+		uiPDumpFlags
+	);
+	PVR_LOG_IF_ERROR(eError, "DevmemIntPDumpSaveToFileVirtualNoValidate");
+
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.h
new file mode 100644
index 000000000000..8df9389a13ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdump_common.h
@@ -0,0 +1,106 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX PDump Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX common PDump functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXPDUMP_COMMON_H
+#define RGXPDUMP_COMMON_H
+
+#if defined(PDUMP)
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "devicemem_typedefs.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       PDumpGetValidRegion
+@Description    Checks valid regions within a given DEVMEM_MEMDESC and returns
+                a list containing region descriptors.
+@Input          psDeviceNode Pointer to the device node.
+@Input          psMemDesc    Pointer to the memory descriptor.
+@Input          uiSize       Size of the allocation.
+@Output         psRegionList List of valid regions.
+@Return         PVRSRV_ERROR PVRSRV_OK on success or error code on failure.
+*/ /**************************************************************************/
+PVRSRV_ERROR PDumpGetValidRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 DEVMEM_MEMDESC *psMemDesc,
+                                 IMG_UINT32 uiSize,
+                                 DLLIST_NODE *psRegionList);
+
+/*************************************************************************/ /*!
+@Function       PDumpSaveToFileVirtual
+@Description    Emits SAB PDump command to file.
+@Input          psDeviceNode Pointer to the device node.
+@Input          psMemDesc    Pointer to the memory descriptor.
+@Input          psRegionList List of valid regions.
+@Input          pszFileName  Output file name.
+@Input          uiPDumpFlags PDump flags.
+*/ /**************************************************************************/
+void PDumpSaveToFileVirtual(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            DEVMEM_MEMDESC *psMemDesc,
+                            DLLIST_NODE *psRegionList,
+                            const IMG_CHAR *pszFileName,
+                            IMG_UINT32 uiPDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PDumpSaveToFileVirtualNoValidate
+@Description    Emits SAB PDump command to file.
+@Input          psDeviceNode Pointer to the device node.
+@Input          psMemDesc    Pointer to the memory descriptor.
+@Input          uiOffset     Offset within memory region.
+@Input          uiSize       Size of memory region.
+@Input          pszFileName  Output file name.
+@Input          uiFileOffset Offset within file.
+@Input          uiPDumpFlags PDump flags.
+*/ /**************************************************************************/
+void PDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      DEVMEM_MEMDESC *psMemDesc,
+                                      IMG_DEVMEM_OFFSET_T uiOffset,
+                                      IMG_UINT32 uiSize,
+                                      const IMG_CHAR *pszFileName,
+                                      IMG_UINT32 uiFileOffset,
+                                      IMG_UINT32 uiPDumpFlags);
+
+#endif /* PDUMP */
+
+#endif /* RGXPDUMP_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.c
new file mode 100644
index 000000000000..e96091ec8b23
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.c
@@ -0,0 +1,146 @@
+/*************************************************************************/ /*!
+@File           rgxpdvfs.c
+@Title          RGX Proactive DVFS Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode Proactive DVFS Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxpdvfs.h"
+#include "rgxfwutils.h"
+#include "rgxpower.h"
+#include "rgx_options.h"
+#include "rgxtimecorr.h"
+
+#define USEC_TO_MSEC 1000
+
+PVRSRV_ERROR PDVFSLimitMaxFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MaxOPPPoint)
+{
+	RGXFWIF_KCCB_CMD		sGPCCBCmd;
+	PVRSRV_ERROR			eError;
+	IMG_UINT32				ui32CmdKCCBSlot;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	/* send feedback */
+	sGPCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ;
+	sGPCCBCmd.uCmdData.sPDVFSMaxFreqData.ui32MaxOPPPoint = ui32MaxOPPPoint;
+
+	/* Submit command to the firmware.  */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+		                                      &sGPCCBCmd,
+		                                      PDUMP_FLAGS_CONTINUOUS,
+		                                      &ui32CmdKCCBSlot);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR PDVFSLimitMinFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MinOPPPoint)
+{
+	RGXFWIF_KCCB_CMD		sGPCCBCmd;
+	PVRSRV_ERROR			eError;
+	IMG_UINT32				ui32CmdKCCBSlot;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	/* send feedback */
+	sGPCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MIN_FREQ;
+	sGPCCBCmd.uCmdData.sPDVFSMinFreqData.ui32MinOPPPoint = ui32MinOPPPoint;
+
+	/* Submit command to the firmware.  */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+		                                      &sGPCCBCmd,
+		                                      PDUMP_FLAGS_CONTINUOUS,
+		                                      &ui32CmdKCCBSlot);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+/*************************************************************************/ /*!
+@Function       RGXPDVFSCheckCoreClkRateChange
+@Description    Checks if core clock rate has changed since the last snap-shot.
+@Input          psDevInfo    A pointer to PVRSRV_RGXDEV_INFO.
+@Return         None.
+*/ /**************************************************************************/
+void RGXPDVFSCheckCoreClkRateChange(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 ui32CoreClkRate = *psDevInfo->pui32RGXFWIFCoreClkRate;
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return;
+	}
+
+	if (ui32CoreClkRate != 0 && psDevInfo->ui32CoreClkRateSnapshot != ui32CoreClkRate)
+	{
+		psDevInfo->ui32CoreClkRateSnapshot = ui32CoreClkRate;
+		RGX_PROCESS_CORE_CLK_RATE_CHANGE(psDevInfo, ui32CoreClkRate);
+	}
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.h
new file mode 100644
index 000000000000..f2bec1184077
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpdvfs.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           rgxpdvfs.h
+@Title          RGX Proactive DVFS Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the kernel mode Proactive DVFS Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXPDVFS_H
+#define RGXPDVFS_H
+
+#include "img_types.h"
+#include "rgxdevice.h"
+#include "rgx_options.h"
+#include "pvrsrv.h"
+
+
+static inline IMG_BOOL _PDVFSEnabled(void)
+{
+	PVRSRV_DATA *psSRVData = PVRSRVGetPVRSRVData();
+
+	if (psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildOptions &
+	    psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildOptions &
+	    OPTIONS_PDVFS_EN)
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR PDVFSLimitMaxFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MaxOPPPoint);
+
+PVRSRV_ERROR PDVFSLimitMinFrequency(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32MinOPPPoint);
+
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+void RGXPDVFSCheckCoreClkRateChange(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#endif /* RGXPDVFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.c
new file mode 100644
index 000000000000..7f77edefe558
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.c
@@ -0,0 +1,1853 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific power routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "rgxpower.h"
+#include "rgxinit.h"
+#include "rgx_fwif_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "rgxdebug_common.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxtimecorr.h"
+#include "devicemem_utils.h"
+#include "htbserver.h"
+#include "rgxstartstop.h"
+#include "rgxfwimageutils.h"
+#include "sync.h"
+#include "rgxdefs_km.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+#include "pvr_dvfs_device.h"
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_FREQ)
+#include "pvr_gpufreq.h"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_FREQ) */
+
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+#include "rgxpdvfs.h"
+#endif
+
+static PVRSRV_ERROR RGXFWNotifyHostTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_KCCB_CMD sCmd;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CmdKCCBSlot;
+
+	/* Send the Timeout notification to the FW */
+	sCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_HOST_TIMEOUT;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	return eError;
+}
+
+static void _RGXUpdateGPUUtilStats(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_GPU_UTIL_FW *psUtilFW;
+	IMG_UINT64 ui64LastPeriod;
+	IMG_UINT64 ui64LastState;
+	IMG_UINT64 ui64LastReducedState;
+	IMG_UINT64 ui64LastTime;
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT64 ui64DMOSStatsCounter;
+
+	psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+	ui64TimeNow = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDevInfo->psDeviceNode));
+
+	/* Update counters to account for the time since the last update */
+	ui64LastState  = RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord);
+	ui64LastTime   = RGXFWIF_GPU_UTIL_GET_TIME(psUtilFW->ui64GpuLastWord);
+	ui64LastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64LastTime);
+	psUtilFW->aui64GpuStatsCounters[ui64LastState] += ui64LastPeriod;
+
+	/* Update state and time of the latest update */
+	psUtilFW->ui64GpuLastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(ui64TimeNow, ui64LastState);
+
+	/* convert last period into the same units as used by fw */
+	ui64TimeNow  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+
+	FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+	{
+		RGXFWIF_GPU_STATS *psStats = &psUtilFW->sStats[ui32DriverID];
+		RGXFWIF_DM eDM;
+
+		for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+		{
+			ui64LastState  = (IMG_UINT64)RGXFWIF_GPU_UTIL_GET_STATE32(psStats->aui32DMOSLastWord[eDM]);
+			ui64LastTime   = (IMG_UINT64)RGXFWIF_GPU_UTIL_GET_TIME32(psStats->aui32DMOSLastWord[eDM]) +
+			                 ((IMG_UINT64)psStats->aui32DMOSLastWordWrap[eDM] << 32);
+			ui64LastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64LastTime);
+			/* for states statistics per DM per driver we only care about the time in Active state,
+			so we "combine" other states (Idle and Blocked) together */
+			ui64LastReducedState = (ui64LastState == RGXFWIF_GPU_UTIL_STATE_ACTIVE) ?
+			                       RGXFWIF_GPU_UTIL_STATE_ACTIVE : RGXFWIF_GPU_UTIL_STATE_INACTIVE;
+			ui64DMOSStatsCounter = (IMG_UINT64)psStats->aaui32DMOSStatsCounters[eDM][ui64LastReducedState] + ui64LastPeriod;
+			psStats->aaui32DMOSStatsCounters[eDM][ui64LastReducedState] = (IMG_UINT32)(ui64DMOSStatsCounter & IMG_UINT32_MAX);
+			if (ui64DMOSStatsCounter > IMG_UINT32_MAX)
+			{
+				psStats->aaui32DMOSCountersWrap[eDM][ui64LastReducedState] += (IMG_UINT32)(ui64DMOSStatsCounter >> 32);
+			}
+
+			/* Update state and time of the latest update */
+			psStats->aui32DMOSLastWord[eDM] = RGXFWIF_GPU_UTIL_MAKE_WORD32((ui64TimeNow & (IMG_UINT64)IMG_UINT32_MAX), ui64LastState);
+			if (ui64TimeNow > IMG_UINT32_MAX)
+			{
+				if (psStats->aui32DMOSLastWordWrap[eDM] != (IMG_UINT32)(ui64TimeNow >> 32))
+				{
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64TimeNow >> 32);
+				}
+			}
+		}
+	}
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+
+	OSLockRelease(psDevInfo->hGPUUtilLock);
+}
+
+static INLINE PVRSRV_ERROR RGXDoStop(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDeviceNode->psDevConfig->pfnTDRGXStop == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPrePowerState: TDRGXStop not implemented!"));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	psDevInfo->bRGXPowered = IMG_FALSE;
+	eError = psDeviceNode->psDevConfig->pfnTDRGXStop(psDeviceNode->psDevConfig->hSysData);
+#else
+	eError = RGXStop(&psDevInfo->sLayerParams);
+#endif
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSendPowerOffKick
+@Description    Send a KCCB kick to power off the GPU FW. This function will wait
+                for completion of the command before exiting.
+
+@Input          psDeviceNode    The device node struct associated with the GPU.
+@Input          bForce          A boolean indicating if the power off command
+                                should be forced.
+
+@Return         Failure code if the virtual address is invalid.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXSendPowerOffKick(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_BOOL bForce)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_KCCB_CMD sPowCmd;
+	IMG_UINT32 ui32CmdKCCBSlot;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Send the Power off request to the FW */
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_OFF_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.bForced = bForce;
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+		         __func__));
+		return eError;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send Power off request",
+		         __func__));
+		return eError;
+	}
+
+	/* Wait for the firmware to complete processing. It cannot use PVRSRVWaitForValueKM as it relies
+	 * on the EventObject which is signalled in this MISR */
+	return RGXPollForGPCommandCompletion(psDeviceNode,
+	                                     psDevInfo->psPowSyncPrim->pui32LinAddr,
+	                                     0x1, 0xFFFFFFFF);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXFinalisePowerOff
+@Description    Finalises the GPU power transition to off.
+
+@Input          psDeviceNode    The device node struct associated with the GPU.
+
+@Return         Result code indicating the success or reason for failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXFinalisePowerOff(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_SYNC_IRQ)
+	IMG_UINT32 ui32idx;
+#endif
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if !defined(NO_HARDWARE)
+	/* Driver takes the VZ Fw-KM connection down, preventing the
+		* firmware from submitting further interrupts */
+	KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+	KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+#if defined(SUPPORT_SYNC_IRQ)
+	/* Wait for the pending IRQ handlers to complete. */
+	OSSyncIRQ(psDeviceNode->psDevConfig->ui32IRQ);
+#else
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	ui32idx = RGXFW_HOST_DRIVER_ID;
+#else
+	for_each_irq_cnt(ui32idx)
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+	{
+		IMG_UINT32 ui32IrqCnt;
+
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+
+		/* Wait for the pending FW processor to host interrupts to come back. */
+		eError = PVRSRVPollForValueKM(psDeviceNode,
+		                              (IMG_UINT32 __iomem *)&psDevInfo->aui32SampleIRQCount[ui32idx],
+		                              ui32IrqCnt,
+		                              0xffffffff,
+		                              POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+		                              NULL);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Wait for pending interrupts failed (DevID %u)." MSG_IRQ_CNT_TYPE " %u Host: %u, FW: %u",
+			         __func__,
+			         psDeviceNode->sDevId.ui32InternalID,
+			         ui32idx,
+			         psDevInfo->aui32SampleIRQCount[ui32idx],
+			         ui32IrqCnt));
+
+			RGX_WaitForInterruptsTimeout(psDevInfo);
+#if !defined(RGX_FW_IRQ_OS_COUNTERS)
+			break;
+#endif
+		}
+	}
+#endif /* SUPPORT_SYNC_IRQ */
+#endif /* NO_HARDWARE */
+
+	/* Update GPU frequency and timer correlation related data */
+	RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_POWER);
+
+	/* Update GPU state counters */
+	_RGXUpdateGPUUtilStats(psDevInfo);
+
+#if defined(SUPPORT_LINUX_DVFS)
+	eError = SuspendDVFS(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to suspend DVFS", __func__));
+		return eError;
+	}
+#endif
+
+	eError = RGXDoStop(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		/* Power down failures are treated as successful since the power was removed but logged. */
+		PVR_DPF((PVR_DBG_WARNING, "%s: RGXDoStop failed (%s)",
+		         __func__, PVRSRVGetErrorString(eError)));
+		psDevInfo->ui32ActivePMReqNonIdle++;
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPrePowerState
+@Description    Initial step for setting power state, to be followed by
+                RGXPostPowerState.
+
+@Input          psDeviceNode        The device node struct associated with the GPU.
+@Input          eNewPowerState      The power state the GPU is to transition to.
+@Input          eCurrentPowerState  The current power state of the GPU.
+@Input          ePwrFlags           Flags indicating the behaviour of the transition.
+
+@Return         Result code indicating the success or reason for failure.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPrePowerState(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              PVRSRV_DEV_POWER_STATE eNewPowerState,
+                              PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL              bDeviceOk;
+#endif
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psDevInfo != NULL);
+
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if ((eNewPowerState == eCurrentPowerState) ||
+	    (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON))
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	bDeviceOk = (OSAtomicRead(&psDeviceNode->eHealthStatus) == PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	if (bDeviceOk)
+#endif
+	{
+		IMG_BOOL bForce = IMG_FALSE;
+
+		if (BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED))
+		{
+			bForce = IMG_TRUE;
+		}
+
+		eError = RGXSendPowerOffKick(psDeviceNode, bForce);
+		if (eError == PVRSRV_ERROR_TIMEOUT)
+		{
+			/* timeout waiting for the FW to ack the request: return timeout */
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: Timeout waiting for powoff ack from the FW",
+			         __func__));
+			return eError;
+		}
+		else if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error waiting for powoff ack from the FW (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+			return PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE;
+		}
+	}
+
+	/* Check the Power state after the answer */
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState, INVALIDATE);
+	if ((psFwSysData->ePowState != RGXFWIF_POW_OFF)
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	    && (bDeviceOk)
+#endif
+		)
+	{
+		if (BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED))
+		{	/* It is an error for a forced request to be denied */
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failure to power off during a forced power off. FW: %d",
+			         __func__, psFwSysData->ePowState));
+		}
+
+		/* the sync was updated but the pow state isn't off -> the FW denied the transition */
+		return PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
+	}
+
+	return RGXFinalisePowerOff(psDeviceNode);
+}
+
+#if defined(SUPPORT_AUTOVZ)
+static PVRSRV_ERROR _RGXWaitForGuestsToDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TIMEOUT;
+	IMG_UINT32 ui32FwTimeout = (20 * SECONDS_TO_MICROSECONDS);
+
+	LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+	{
+		IMG_UINT32 ui32DriverID;
+		IMG_BOOL bGuestOnline = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+		                           INVALIDATE);
+
+		for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+			 ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED; ui32DriverID++)
+		{
+			RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+					psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+			if ((eGuestState == RGXFW_CONNECTION_FW_ACTIVE) ||
+				(eGuestState == RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD) ||
+				(eGuestState == RGXFW_CONNECTION_FW_FORCED_OFFLOAD))
+			{
+				bGuestOnline = IMG_TRUE;
+				PVR_DPF((PVR_DBG_WARNING, "%s: Guest OS %u still online.", __func__, ui32DriverID));
+			}
+		}
+
+		if (!bGuestOnline)
+		{
+			/* Allow Guests to finish reading Connection state registers before disconnecting. */
+			OSSleepms(100);
+
+			PVR_DPF((PVR_DBG_WARNING, "%s: All Guest connections are down. "
+									  "Host can power down the GPU.", __func__));
+			eError = PVRSRV_OK;
+			break;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Waiting for Guests to disconnect "
+									  "before powering down GPU.", __func__));
+
+			if (PVRSRVPwrLockIsLockedByMe(psDeviceNode))
+			{
+				/* Don't wait with the power lock held as this prevents the vz
+				 * watchdog thread from keeping the fw-km connection alive. */
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+		}
+
+		OSSleepms(10);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (!PVRSRVPwrLockIsLockedByMe(psDeviceNode))
+	{
+		/* Take back power lock after waiting for Guests */
+		eError = PVRSRVPowerLock(psDeviceNode);
+	}
+
+	return eError;
+}
+#endif /* defined(SUPPORT_AUTOVZ) */
+
+/*
+	RGXVzPrePowerState
+*/
+PVRSRV_ERROR RGXVzPrePowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                                PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                                PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", eError);
+
+	if (eNewPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering down */
+#if defined(SUPPORT_AUTOVZ)
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode) && (!psDeviceNode->bAutoVzFwIsUp || psDeviceNode->bAutoVzAllowGPUPowerdown))
+		{
+			if (psDeviceNode->bAutoVzFwIsUp)
+			{
+				/* bAutoVzAllowGPUPowerdown must be TRUE here and
+				 * bAutoVzFwIsUp=TRUE indicates that this is a powerdown event
+				 * so send requests to the FW to disconnect all guest connections
+				 * before GPU is powered down. */
+				eError = RGXDisconnectAllGuests(psDeviceNode);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXDisconnectAllGuests");
+			}
+
+			/* The Host must ensure all Guest drivers have disconnected from the GPU before powering it down.
+			 * Guest drivers regularly access hardware registers during runtime. If an attempt is made to
+			 * access a GPU register while the GPU is down, the SoC might lock up. */
+			eError = _RGXWaitForGuestsToDisconnect(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "_RGXWaitForGuestsToDisconnect");
+
+			if (psDeviceNode->bAutoVzAllowGPUPowerdown)
+			{
+				psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+			}
+
+			/* Temporarily restore all power callbacks used by the driver to fully power down the GPU.
+			 * Under AutoVz, power transitions requests (e.g. on driver deinitialisation and unloading)
+			 * are generally ignored and the GPU power state is unaffected. Special power requests like
+			 * those triggered by Suspend/Resume calls must reinstate the callbacks when needed. */
+			PVRSRVSetPowerCallbacks(psDeviceNode, psDeviceNode->psPowerDev,
+									&RGXVzPrePowerState, &RGXVzPostPowerState,
+									psDeviceNode->psDevConfig->pfnPrePowerState,
+									psDeviceNode->psDevConfig->pfnPostPowerState,
+									&RGXForcedIdleRequest, &RGXCancelForcedIdleRequest,
+									&RGXCancelForcedIdleRequestAsync);
+		}
+		else
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+			KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+			KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+			if (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+				KM_OS_CONNECTION_IS(ACTIVE, psDevInfo))
+			{
+				PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, 0, RGXFWIF_OS_OFFLINE);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+			}
+		}
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering down: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+	}
+	else if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering up */
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering up: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+
+	}
+
+	if (!(PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp)))
+	{
+		/* call regular device power function */
+		eError = RGXPrePowerState(psDeviceNode, eNewPowerState, eCurrentPowerState, ePwrFlags);
+	}
+
+	return eError;
+}
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+static PVRSRV_ERROR RGXVzWaitFirmwareReady(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	if (!KM_FW_CONNECTION_IS(READY, psDevInfo))
+	{
+		PVR_LOG(("%s: Firmware Connection is not in Ready state. Waiting for Firmware ...", __func__));
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(RGX_VZ_CONNECTION_TIMEOUT_US)
+	{
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+		{
+			PVR_LOG(("%s: Firmware Connection is Ready. Initialisation proceeding.", __func__));
+			break;
+		}
+		else
+		{
+			OSSleepms(10);
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	if (!KM_FW_CONNECTION_IS(READY, psDevInfo))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Timed out waiting for the Firmware to enter Ready state.", __func__));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXVzPostPowerState
+*/
+PVRSRV_ERROR RGXVzPostPowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                                 PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                                 PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", eError);
+
+	if (!(PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp)))
+	{
+		if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+		{
+			KM_SET_OS_CONNECTION(READY, psDevInfo);
+			KM_CONNECTION_CACHEOP(Os, FLUSH);
+		}
+		/* call regular device power function */
+		eError = RGXPostPowerState(psDeviceNode, eNewPowerState, eCurrentPowerState, ePwrFlags);
+	}
+	else
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	if (eNewPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering down */
+		if (psDeviceNode->bAutoVzFwIsUp)
+		{
+			PVR_LOG(("%s: AutoVz Fw active, power not changed", __func__));
+			return eError;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering down: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+
+#if !defined(SUPPORT_AUTOVZ_HW_REGS)
+		/* The connection states must be reset on a GPU power cycle. If the states are kept
+		 * in hardware scratch registers, they will be cleared on power down. When using shared
+		 * memory the connection data must be explicitly cleared by the driver. */
+		OSCachedMemSetWMB(psDevInfo->psRGXFWIfConnectionCtl, 0, sizeof(RGXFWIF_CONNECTION_CTL));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfConnectionCtl, FLUSH);
+#endif /* defined(SUPPORT_AUTOVZ) && !defined(SUPPORT_AUTOVZ_HW_REGS) */
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+		{
+#if defined(SUPPORT_AUTOVZ)
+			/* AutoVz Guests attempting to suspend have updated their connections earlier in RGXVzPrePowerState.
+			 * Skip this redundant register write, as the Host could have powered down the GPU by now. */
+			if (psDeviceNode->bAutoVzFwIsUp)
+#endif
+			{
+				/* Take the VZ connection down to prevent firmware from submitting further interrupts */
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+			/* Power transition callbacks were not executed, update RGXPowered flag here */
+			psDevInfo->bRGXPowered = IMG_FALSE;
+		}
+	}
+	else if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering up */
+		IMG_UINT32 ui32FwTimeout = (3 * SECONDS_TO_MICROSECONDS);
+		volatile IMG_BOOL *pbUpdatedFlag;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                           INVALIDATE);
+		pbUpdatedFlag = &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated;
+
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering up: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Guests don't execute the power transition callbacks, so update their RGXPowered flag here */
+			psDevInfo->bRGXPowered = IMG_TRUE;
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+			/* Guest drivers expect the firmware to have set its end of the
+			 * connection to Ready state by now. */
+			eError = RGXVzWaitFirmwareReady(psDevInfo);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXVzWaitFirmwareReady()");
+#endif /* RGX_VZ_STATIC_CARVEOUT_FW_HEAPS */
+
+			OSWriteDeviceMem32WithWMB(pbUpdatedFlag, IMG_FALSE);
+
+			/* Kick an initial dummy command to make the firmware initialise all
+			 * its internal guest OS data structures and compatibility information.
+			 * Use the lower-level RGXSendCommand() for the job, to make
+			 * sure only 1 KCCB command is issued to the firmware.
+			 * The default RGXFWHealthCheckCmd() prefaces each HealthCheck command with
+			 * a pre-kick cache command which can interfere with the FW-KM init handshake. */
+			{
+				RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+				sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+				KM_SET_OS_CONNECTION(READY, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+				eError = RGXSendCommand(psDevInfo, &sCmpKCCBCmd, PDUMP_FLAGS_CONTINUOUS);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXSendCommand()");
+			}
+		}
+		else
+		{
+#if defined(SUPPORT_AUTOVZ)
+			/* Disable power callbacks that should not be run on virtualised drivers after the GPU
+			 * is fully initialised: system layer pre/post functions and driver idle requests.
+			 * The original device RGX Pre/Post functions are called from this Vz wrapper. */
+			PVRSRVSetPowerCallbacks(psDeviceNode, psDeviceNode->psPowerDev,
+									&RGXVzPrePowerState, &RGXVzPostPowerState,
+									NULL, NULL, NULL, NULL, NULL);
+
+			/* AutoVz Host driver reconnecting to running Firmware */
+			if (psDeviceNode->bAutoVzFwIsUp)
+			{
+				/* Firmware already running, send a KCCB command to establish the new connection */
+				RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+				sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+				eError = RGXVzWaitFirmwareReady(psDevInfo);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXVzWaitFirmwareReady()");
+
+				KM_SET_OS_CONNECTION(READY, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+				eError = RGXSendCommand(psDevInfo, &sCmpKCCBCmd, PDUMP_FLAGS_CONTINUOUS);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXSendCommand()");
+			}
+			else
+			{
+				/* During first-time boot the flag is set here, while subsequent reboots will already
+				 * have set it earlier in RGXInit. Set to true from this point on. */
+				psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+			}
+#endif
+		}
+
+		/* Wait for the firmware to accept and enable the connection with this OS by setting its state to Active */
+		LOOP_UNTIL_TIMEOUT_US(RGX_VZ_CONNECTION_TIMEOUT_US)
+		{
+			KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+			if (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo))
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Connection is Active. Initialisation proceeding.", __func__));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Connection is not in Active state. Waiting for Firmware ...", __func__));
+				OSSleepms(10);
+			}
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		if (!KM_FW_CONNECTION_IS(ACTIVE, psDevInfo))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Timed out waiting for the Firmware to enter Active state.", __func__));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* poll on the Firmware supplying the compatibility data */
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				if (*pbUpdatedFlag)
+				{
+					break;
+				}
+				OSSleepms(10);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			PVR_LOG_RETURN_IF_FALSE(*pbUpdatedFlag, "Firmware does not respond with compatibility data.", PVRSRV_ERROR_TIMEOUT);
+		}
+
+		KM_SET_OS_CONNECTION(ACTIVE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(TRACK_FW_BOOT)
+static INLINE void RGXCheckFWBootStage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	FW_BOOT_STAGE eStage;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		/* Boot stage temporarily stored to the register below */
+		eStage = OSReadHWReg32(psDevInfo->pvRegsBaseKM,
+		                       RGX_FW_BOOT_STAGE_REGISTER);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		IMG_BYTE *pbBootData;
+
+		if (PVRSRV_OK != DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+		                                          (void**)&pbBootData))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Could not acquire pointer to FW boot stage", __func__));
+			eStage = FW_BOOT_STAGE_NOT_AVAILABLE;
+		}
+		else
+		{
+			pbBootData += RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+
+			eStage = *(FW_BOOT_STAGE*)&pbBootData[RGXMIPSFW_BOOT_STAGE_OFFSET];
+
+			if (eStage == FW_BOOT_STAGE_TLB_INIT_FAILURE)
+			{
+				RGXMIPSFW_BOOT_DATA *psBootData =
+					(RGXMIPSFW_BOOT_DATA*) (pbBootData + RGXMIPSFW_BOOTLDR_CONF_OFFSET);
+
+				PVR_LOG(("MIPS TLB could not be initialised. Boot data info:"
+						 " num PT pages %u, log2 PT page size %u, PT page addresses"
+						 " %"IMG_UINT64_FMTSPECx " %"IMG_UINT64_FMTSPECx
+						 " %"IMG_UINT64_FMTSPECx " %"IMG_UINT64_FMTSPECx,
+						 psBootData->ui32PTNumPages,
+						 psBootData->ui32PTLog2PageSize,
+						 psBootData->aui64PTPhyAddr[0U],
+						 psBootData->aui64PTPhyAddr[1U],
+						 psBootData->aui64PTPhyAddr[2U],
+						 psBootData->aui64PTPhyAddr[3U]));
+			}
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		}
+	}
+#endif
+	else
+	{
+		eStage = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SCRATCH14);
+	}
+
+	PVR_LOG(("%s: FW reached boot stage %i/%i.",
+	         __func__, eStage, FW_BOOT_INIT_DONE));
+}
+#endif
+
+static INLINE PVRSRV_ERROR RGXDoStart(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+
+	if (psDevConfig->pfnTDRGXStart == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPostPowerState: TDRGXStart not implemented!"));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	eError = psDevConfig->pfnTDRGXStart(psDevConfig->hSysData);
+
+	if (eError == PVRSRV_OK)
+	{
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+#else
+	eError = RGXStart(&psDevInfo->sLayerParams);
+#endif
+
+	return eError;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+/*
+ * To validate the MTS unit we do the following:
+ *  - Immediately after firmware loading for each OSID
+ *    - Write the OSid to a memory location shared with FW
+ *    - Kick the register of that OSid
+ *         (Uncounted, DM 0)
+ *    - FW clears the memory location if OSid matches
+ *    - Host checks that memory location is cleared
+ *
+ *  See firmware/rgxfw_bg.c
+ */
+static PVRSRV_ERROR RGXVirtualisationPowerupSidebandTest(PVRSRV_DEVICE_NODE	 *psDeviceNode,
+														 RGXFWIF_SYSINIT *psFwSysInit,
+														 PVRSRV_RGXDEV_INFO	 *psDevInfo)
+{
+	IMG_UINT32 ui32ScheduleRegister;
+	IMG_UINT32 ui32OSid;
+	IMG_UINT32 ui32KickType;
+	IMG_UINT32 ui32OsRegBanksMapped = (psDeviceNode->psDevConfig->ui32RegsSize / RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+
+	/* Nothing to do if the device does not support GPU_VIRTUALISATION */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, GPU_VIRTUALISATION))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Testing per-os kick registers:"));
+
+	ui32OsRegBanksMapped = MIN(ui32OsRegBanksMapped, GPUVIRT_VALIDATION_NUM_OS);
+
+	if (ui32OsRegBanksMapped != RGXFW_MAX_NUM_OSIDS)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "The register bank mapped into kernel VA does not cover all OS' registers:"));
+		PVR_DPF((PVR_DBG_WARNING, "Maximum OS count = %d / Per-os register banks mapped = %d", RGXFW_MAX_NUM_OSIDS, ui32OsRegBanksMapped));
+		PVR_DPF((PVR_DBG_WARNING, "Only first %d MTS registers will be tested", ui32OsRegBanksMapped));
+	}
+
+	ui32KickType = RGX_CR_MTS_SCHEDULE_DM_DM0 | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED;
+
+	for (ui32OSid = 0; ui32OSid < ui32OsRegBanksMapped; ui32OSid++)
+	{
+		/* set Test field */
+		psFwSysInit->ui32OSKickTest = (ui32OSid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT;
+
+#if defined(PDUMP)
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, ui32OSKickTest),
+						   sizeof(psFwSysInit->ui32OSKickTest),
+						   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+		/* Force a read-back to memory to avoid posted writes on certain buses */
+		OSWriteMemoryBarrier(&psFwSysInit->ui32OSKickTest);
+		RGXFwSharedMemCacheOpValue(psFwSysInit->ui32OSKickTest, FLUSH);
+
+		/* kick register */
+		ui32ScheduleRegister = RGX_CR_MTS_SCHEDULE + (ui32OSid * RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+		PVR_DPF((PVR_DBG_MESSAGE, "  Testing OS: %u, Kick Reg: %X",
+				 ui32OSid,
+				 ui32ScheduleRegister));
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32ScheduleRegister, ui32KickType);
+
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS, "VZ sideband test, kicking MTS register %u", ui32OSid);
+
+		PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+				ui32ScheduleRegister, ui32KickType, PDUMP_FLAGS_CONTINUOUS);
+
+		DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							   offsetof(RGXFWIF_SYSINIT, ui32OSKickTest),
+							   0,
+							   0xFFFFFFFF,
+							   PDUMP_POLL_OPERATOR_EQUAL,
+							   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if !defined(NO_HARDWARE)
+		OSMemoryBarrier((IMG_BYTE*) psDevInfo->pvRegsBaseKM + ui32ScheduleRegister);
+
+		/* Wait test enable bit to be unset */
+		if (PVRSRVPollForValueKM(psDeviceNode,
+								 (volatile IMG_UINT32 __iomem *)&psFwSysInit->ui32OSKickTest,
+								 0,
+								 RGXFWIF_KICK_TEST_ENABLED_BIT,
+								 POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+								 RGXFwSharedMemCacheOpExecPfn) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Testing OS %u kick register failed: firmware did not clear test location (contents: 0x%X)",
+					 ui32OSid,
+					 psFwSysInit->ui32OSKickTest));
+
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Check that the value is what we expect */
+		if (psFwSysInit->ui32OSKickTest != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Testing OS %u kick register failed: firmware wrote 0x%X to test location",
+					 ui32OSid,
+					 psFwSysInit->ui32OSKickTest));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "    PASS"));
+#endif
+	}
+
+	PVR_LOG(("MTS passed sideband tests"));
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION_MTS) */
+
+
+/*
+	RGXPostPowerState
+*/
+PVRSRV_ERROR RGXPostPowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                               PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                               PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                               PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(ePwrFlags);
+
+	if ((eNewPowerState == eCurrentPowerState) ||
+	    (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGXPostPowerState: Current state: %d, New state: %d",
+		             eCurrentPowerState, eNewPowerState);
+
+		return PVRSRV_OK;
+	}
+
+	/* Update timer correlation related data */
+	RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_POWER);
+
+	/* Update GPU state counters */
+	_RGXUpdateGPUUtilStats(psDevInfo);
+
+	eError = RGXDoStart(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDoStart", fail);
+
+	OSMemoryBarrier(NULL);
+
+
+	/*
+		* Check whether the FW has started by polling on bFirmwareStarted flag
+		*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->bFirmwareStarted,
+	                           INVALIDATE);
+	if (PVRSRVPollForValueKM(psDeviceNode,
+	                         (IMG_UINT32 __iomem *)&psDevInfo->psRGXFWIfSysInit->bFirmwareStarted,
+	                         IMG_TRUE,
+	                         0xFFFFFFFF,
+	                         POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+	                         RGXFwSharedMemCacheOpExecPfn) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPostPowerState: Polling for 'FW started' flag failed."));
+		eError = PVRSRV_ERROR_TIMEOUT;
+
+#if defined(TRACK_FW_BOOT)
+		RGXCheckFWBootStage(psDevInfo);
+#endif
+
+		/*
+		* When bFirmwareStarted fails some info may be gained by doing the following
+		* debug dump but unfortunately it could be potentially dangerous if the reason
+		* for not booting is the GPU power is not ON. However, if we have reached this
+		* point the System Layer has returned without errors, we assume the GPU power
+		* is indeed ON.
+		*/
+		RGXDumpRGXDebugSummary(NULL, NULL, psDeviceNode->pvDevice, IMG_TRUE);
+		RGXDumpRGXRegisters(NULL, NULL, psDeviceNode->pvDevice);
+
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPollForValueKM(bFirmwareStarted)", fail);
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS, "Wait for the Firmware to start.");
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfSysInitMemDesc,
+	                                offsetof(RGXFWIF_SYSINIT, bFirmwareStarted),
+	                                IMG_TRUE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXPostPowerState: problem pdumping POL for psRGXFWIfSysInitMemDesc (%d)",
+		         eError));
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", fail);
+	}
+
+#endif /* defined(PDUMP) */
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	eError = RGXVirtualisationPowerupSidebandTest(psDeviceNode, psDevInfo->psRGXFWIfSysInit, psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXVirtualisationPowerupSidebandTest", fail);
+#endif
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp,
+	                           INVALIDATE);
+	PVRSRVSetFirmwareStartTime(psDeviceNode->psPowerDev,
+	                           psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp,
+	                           FLUSH);
+#endif
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32MarkerVal,
+	                           INVALIDATE);
+	HTBSyncPartitionMarker(psDevInfo->psRGXFWIfSysInit->ui32MarkerVal);
+
+#if defined(SUPPORT_LINUX_DVFS)
+	eError = ResumeDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "ResumeDVFS", fail);
+#endif
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXPostPowerState: Current state: %d, New state: %d",
+	             eCurrentPowerState, eNewPowerState);
+
+	return eError;
+
+fail:
+	psDevInfo->bRGXPowered = IMG_FALSE;
+
+	return eError;
+}
+
+/*
+	RGXPreClockSpeedChange
+*/
+PVRSRV_ERROR RGXPreClockSpeedChange(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                    PVRSRV_DEV_POWER_STATE	eCurrentPowerState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	const PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused const RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "RGXPreClockSpeedChange: RGX clock speed was %uHz",
+			psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF) &&
+	    (psFwSysData->ePowState != RGXFWIF_POW_OFF))
+	{
+		/* Update GPU frequency and timer correlation related data */
+		RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+	}
+
+	return eError;
+}
+
+/*
+	RGXPostClockSpeedChange
+*/
+PVRSRV_ERROR RGXPostClockSpeedChange(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                     PVRSRV_DEV_POWER_STATE	eCurrentPowerState)
+{
+	const PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32NewClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	/* Update runtime configuration with the new value */
+	OSWriteDeviceMem32WithWMB(&psDevInfo->psRGXFWIfRuntimeCfg->ui32CoreClockSpeed,
+	                          ui32NewClockSpeed);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32CoreClockSpeed, FLUSH);
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+	if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF) &&
+	    (psFwSysData->ePowState != RGXFWIF_POW_OFF))
+	{
+		RGXFWIF_KCCB_CMD sCOREClkSpeedChangeCmd;
+		IMG_UINT32 ui32CmdKCCBSlot;
+
+		RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+
+		sCOREClkSpeedChangeCmd.eCmdType = RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE;
+		sCOREClkSpeedChangeCmd.uCmdData.sCoreClkSpeedChangeData.ui32NewClockSpeed = ui32NewClockSpeed;
+
+		PDUMPCOMMENT(psDeviceNode, "Scheduling CORE clock speed change command");
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+		                                      &sCOREClkSpeedChangeCmd,
+		                                      PDUMP_FLAGS_NONE,
+		                                      &ui32CmdKCCBSlot);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Scheduling CORE clock speed change command failed");
+			PVR_DPF((PVR_DBG_ERROR, "RGXPostClockSpeedChange: Scheduling KCCB command failed. Error:%u", eError));
+			return eError;
+		}
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_FREQ)
+		GpuTraceFrequency(psDeviceNode->sDevId.ui32InternalID,
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed);
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_FREQ) */
+
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXPostClockSpeedChange: RGX clock speed changed to %uHz",
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	RGXPowUnitsChange
+
+ @Description Change power units state
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32PowUnits : On Rogue: Number of DUSTs to make transition to.
+                          On Volcanic: Mask containing power state of SPUs.
+                          Each bit corresponds to an SPU and value must be non-zero.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPowUnitsChange(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32PowUnits)
+
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError;
+	RGXFWIF_KCCB_CMD	sPowUnitsChange;
+	IMG_UINT32			ui32AvailablePowUnits;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AvailablePowUnits = psDevInfo->ui32AvailablePowUnitsMask;
+
+	/**
+	 * Validate the input. At-least one PU must be powered on and all requested
+	 * PU's must be a subset of full PU mask.
+	 */
+	if ((ui32PowUnits == 0) || (ui32PowUnits & ~ui32AvailablePowUnits))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid Power Units mask requested (0x%X). Value should be non-zero and sub-set of 0x%X mask",
+				__func__,
+				ui32PowUnits,
+				ui32AvailablePowUnits));
+		return PVRSRV_ERROR_INVALID_SPU_MASK;
+	}
+#else
+	ui32AvailablePowUnits = psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount;
+
+	if (ui32PowUnits > ui32AvailablePowUnits)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid number of DUSTs (%u) while expecting value within <0,%u>",
+				__func__,
+				ui32PowUnits,
+				ui32AvailablePowUnits));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+
+	psRuntimeCfg->ui32PowUnitsState = ui32PowUnits;
+	OSWriteMemoryBarrier(&psRuntimeCfg->ui32PowUnitsState);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ui32PowUnitsState, FLUSH);
+
+#if !defined(NO_HARDWARE)
+	{
+		const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+		RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+		                           INVALIDATE);
+
+		if (psFwSysData->ePowState == RGXFWIF_POW_OFF)
+		{
+			return PVRSRV_OK;
+		}
+
+		if (psFwSysData->ePowState != RGXFWIF_POW_FORCED_IDLE)
+		{
+			eError = PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Powered units state can not be changed when not IDLE",
+					 __func__));
+			return eError;
+		}
+	}
+#endif
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+				__func__));
+		return eError;
+	}
+
+	sPowUnitsChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowUnitsChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_NUM_UNITS_CHANGE;
+	sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32PowUnits = ui32PowUnits;
+#if defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX)
+	sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32RACUnits = 0;
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, POWER_ISLAND_VERSION) >= 3)
+	{
+		sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32RACUnits =
+			(1U << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+	}
+#endif
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "Scheduling command to change power units state to 0x%X",
+	             ui32PowUnits);
+	eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+	                                      &sPowUnitsChange,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "Scheduling command to change power units state. Error:%u",
+		             eError);
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Scheduling KCCB to change power units state. Error:%u",
+				 __func__, eError));
+		return eError;
+	}
+
+	/* Wait for the firmware to answer. */
+	eError = RGXPollForGPCommandCompletion(psDeviceNode,
+	                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+								  0x1, 0xFFFFFFFF);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Timeout waiting for idle request", __func__));
+		return eError;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "%s: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             __func__, psDevInfo->psPowSyncPrim->pui32LinAddr,
+	             RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ @Function	RGXAPMLatencyChange
+*/
+PVRSRV_ERROR RGXAPMLatencyChange(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 IMG_UINT32			ui32ActivePMLatencyms,
+                                 IMG_BOOL			bActivePMLatencyPersistant)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError;
+	RGXFWIF_RUNTIME_CFG	*psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	PVRSRV_DEV_POWER_STATE	ePowerState;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	/* Update runtime configuration with the new values and ensure the
+	 * new APM latency is written to memory before requesting the FW to
+	 * read it
+	 */
+	psRuntimeCfg->ui32ActivePMLatencyms = ui32ActivePMLatencyms;
+	psRuntimeCfg->bActivePMLatencyPersistant = bActivePMLatencyPersistant;
+	OSWriteMemoryBarrier(&psRuntimeCfg->bActivePMLatencyPersistant);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ui32ActivePMLatencyms, FLUSH);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->bActivePMLatencyPersistant, FLUSH);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF)
+	{
+		/* Power is off, APM latency will be read on next firmware boot */
+		return PVRSRV_OK;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire power lock (%u)",
+		         __func__, eError));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		RGXFWIF_KCCB_CMD	sActivePMLatencyChange;
+		sActivePMLatencyChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
+		sActivePMLatencyChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_APM_LATENCY_CHANGE;
+
+		PDUMPCOMMENT(psDeviceNode,
+		             "Scheduling command to change APM latency to %u",
+		             ui32ActivePMLatencyms);
+		eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+		                                      &sActivePMLatencyChange,
+		                                      PDUMP_FLAGS_NONE,
+		                                      &ui32CmdKCCBSlot);
+
+		if (eError != PVRSRV_OK)
+		{
+			PDUMPCOMMENT(psDeviceNode,
+			             "Scheduling command to change APM latency failed. Error:%u",
+			             eError);
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Scheduling KCCB to change APM latency failed. Error:%u",
+			         __func__, eError));
+			goto ErrorExit;
+		}
+	}
+
+ErrorExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	return eError;
+}
+
+/*
+	RGXActivePowerRequest
+*/
+PVRSRV_ERROR RGXActivePowerRequest(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+
+	psDevInfo->ui32ActivePMReqTotal++;
+
+	/* Powerlock to avoid further requests from racing with the FW hand-shake
+	 * from now on (previous kicks to this point are detected by the FW)
+	 * PVRSRVPowerLock is replaced with PVRSRVPowerTryLock to avoid
+	 * potential dead lock between PDumpWriteLock and PowerLock
+	 * during 'DriverLive + PDUMP=1 + EnableAPM=1'.
+	 */
+	eError = PVRSRVPowerTryLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			PVR_LOG_ERROR(eError, "PVRSRVPowerTryLock");
+		}
+		else
+		{
+			psDevInfo->ui32ActivePMReqRetry++;
+		}
+		goto _RGXActivePowerRequest_PowerLock_failed;
+	}
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	/* Check again for IDLE once we have the power lock */
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		PVRSRVSetFirmwareHandshakeIdleTime(psDeviceNode->psPowerDev,
+		                                   RGXReadHWTimerReg(psDevInfo)-psFwSysData->ui64StartIdleTime);
+#endif
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+		                                     PVRSRV_DEV_POWER_STATE_OFF,
+		                                     PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError == PVRSRV_OK)
+		{
+			psDevInfo->ui32ActivePMReqOk++;
+		}
+		else if (eError == PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+		{
+			psDevInfo->ui32ActivePMReqDenied++;
+		}
+	}
+	else
+	{
+		psDevInfo->ui32ActivePMReqNonIdle++;
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_RGXActivePowerRequest_PowerLock_failed:
+
+	return eError;
+}
+/*
+	RGXForcedIdleRequest
+*/
+
+#define RGX_FORCED_IDLE_RETRY_COUNT 10
+
+PVRSRV_ERROR RGXForcedIdleRequest(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_BOOL bDeviceOffPermitted)
+{
+	PVRSRV_RGXDEV_INFO    *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD      sPowCmd;
+	PVRSRV_ERROR          eError;
+	IMG_UINT32            ui32RetryCount = 0;
+	IMG_UINT32            ui32CmdKCCBSlot;
+#if !defined(NO_HARDWARE)
+	const RGXFWIF_SYSDATA *psFwSysData;
+#endif
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if !defined(NO_HARDWARE)
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	/* Firmware already forced idle */
+	if (psFwSysData->ePowState == RGXFWIF_POW_FORCED_IDLE)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Firmware is not powered. Sometimes this is permitted, for instance we were forcing idle to power down. */
+	if (psFwSysData->ePowState == RGXFWIF_POW_OFF)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Firmware is powered OFF (bDeviceOffPermitted = %s)",
+				 bDeviceOffPermitted ? "Yes" : "No"));
+		return (bDeviceOffPermitted) ? PVRSRV_OK : PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
+	}
+#endif
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+				__func__));
+		return eError;
+	}
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_FORCE_IDLE;
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Sending forced idle command");
+
+	/* Send one forced IDLE command to GP */
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send idle request", __func__));
+		return eError;
+	}
+
+	/* Wait for GPU to finish current workload */
+	do {
+		eError = RGXPollForGPCommandCompletion(psDeviceNode,
+		                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+									  0x1, 0xFFFFFFFF);
+		if ((eError == PVRSRV_OK) || (ui32RetryCount == RGX_FORCED_IDLE_RETRY_COUNT))
+		{
+			break;
+		}
+		ui32RetryCount++;
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Request timeout. Retry %d of %d",
+				 __func__, ui32RetryCount, RGX_FORCED_IDLE_RETRY_COUNT));
+	} while (IMG_TRUE);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXFWNotifyHostTimeout(psDevInfo);
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Idle request failed. Firmware potentially left in forced idle state",
+				 __func__));
+		return eError;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* Check the firmware state for idleness */
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+	if (psFwSysData->ePowState != RGXFWIF_POW_FORCED_IDLE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "FW power state (%u) is not RGXFWIF_POW_FORCED_IDLE (%u)",
+				 psFwSysData->ePowState, RGXFWIF_POW_FORCED_IDLE));
+		return PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RGXSendCancelForceIdleCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                   IMG_UINT32         *ui32CmdKCCBSlot)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD	sPowCmd;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	                      "SyncPrimSet: Failed to set Power sync prim",
+	                      ErrorExit);
+
+	/* Send the IDLE request to the FW */
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_CANCEL_FORCED_IDLE;
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Sending cancel forced idle command");
+
+	/* Send cancel forced IDLE command to GP */
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      ui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGXCancelForcedIdleRequest: Failed to send cancel IDLE request for DM%d",
+		             RGXFWIF_DM_GP);
+		goto ErrorExit;
+	}
+
+ErrorExit:
+	return eError;
+}
+/*
+	RGXCancelForcedIdleRequest
+*/
+PVRSRV_ERROR RGXCancelForcedIdleRequest(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	eError = _RGXSendCancelForceIdleCommand(psDeviceNode, &ui32CmdKCCBSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	                      "_RGXSendCancelForceIdleCommand",
+	                      ErrorExit);
+
+	/* Wait for the firmware to answer. */
+	eError = RGXPollForGPCommandCompletion(psDeviceNode,
+	                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+								  1, 0xFFFFFFFF);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	                      "RGXPollForGPCommandCompletion: Timeout waiting for cancel idle request",
+	                      ErrorExit);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXCancelForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+	return eError;
+
+ErrorExit:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Firmware potentially left in forced idle state err: %u",
+	         __func__,
+	         eError));
+	return eError;
+}
+
+PVRSRV_ERROR RGXCancelForcedIdleRequestAsync(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = _RGXSendCancelForceIdleCommand(psDeviceNode, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Firmware potentially left in forced idle state err: %u",
+		         __func__,
+		         eError));
+	}
+
+	return eError;
+}
+
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+/*************************************************************************/ /*!
+@Function       PDVFSProcessCoreClkChangeRequest
+@Description    Processes a core clock rate change request.
+@Input          psDevInfo            A pointer to PVRSRV_RGXDEV_INFO.
+@Input          ui32CoreClockRate    New core clock rate.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeRequest(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = &psDevConfig->sDVFS.sDVFSDeviceCfg;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = ((RGX_DATA*)(psDevConfig->hDevData))->psRGXTimingInfo;
+	IMG_UINT32 ui32CoreClockRateCurrent = psRGXTimingInfo->ui32CoreClockSpeed;
+	const IMG_OPP *psOpp = NULL;
+	IMG_UINT32 ui32Index;
+	PVRSRV_ERROR eError;
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Core clock rate = %u", ui32CoreClockRate));
+
+	/* Find the matching OPP (Exact). */
+	for (ui32Index = 0; ui32Index < psDVFSDeviceCfg->ui32OPPTableSize; ui32Index++)
+	{
+		if (ui32CoreClockRate == psDVFSDeviceCfg->pasOPPTable[ui32Index].ui32Freq)
+		{
+			psOpp = &psDVFSDeviceCfg->pasOPPTable[ui32Index];
+			break;
+		}
+	}
+
+	if (! psOpp)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Frequency not present in OPP table - %u", ui32CoreClockRate));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = PVRSRVDevicePreClockSpeedChange(psDevInfo->psDeviceNode, psDVFSDeviceCfg->bIdleReq, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVDevicePreClockSpeedChange failed"));
+		return eError;
+	}
+
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+
+	/* Increasing frequency, change voltage first */
+	if (ui32CoreClockRate > ui32CoreClockRateCurrent)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDevConfig->hSysData, psOpp->ui32Volt);
+	}
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDevConfig->hSysData, ui32CoreClockRate);
+
+	/* Decreasing frequency, change frequency first */
+	if (ui32CoreClockRate < ui32CoreClockRateCurrent)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDevConfig->hSysData, psOpp->ui32Volt);
+	}
+
+	PVRSRVDevicePostClockSpeedChange(psDevInfo->psDeviceNode, psDVFSDeviceCfg->bIdleReq, NULL);
+
+	return PVRSRV_OK;
+}
+#else
+/*************************************************************************/ /*!
+@Function       PDVFSProcessCoreClkChangeNotification
+@Description    Processes a core clock rate change notification.
+@Input          psDevInfo            A pointer to PVRSRV_RGXDEV_INFO.
+@Input          ui32CoreClockRate    New core clock rate.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeNotification(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = ((RGX_DATA*)(psDevConfig->hDevData))->psRGXTimingInfo;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire lock (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDevInfo->psDeviceNode, &ePowerState);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire power state (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+		return eError;
+	}
+
+	/* Guest drivers do not initialize psRGXFWIfFwSysData */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	if ((ePowerState != PVRSRV_DEV_POWER_STATE_OFF)
+	    && ((psDevInfo->psRGXFWIfFwSysData == NULL) || (psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)))
+	{
+		/* Update GPU frequency and timer correlation related data */
+		RGXTimeCorrEnd(psDevInfo->psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+		psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+		RGXTimeCorrBegin(psDevInfo->psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+	}
+	else
+	{
+		psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	return PVRSRV_OK;
+}
+#endif
+#endif /* SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY */
+
+
+
+/******************************************************************************
+ End of file (rgxpower.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.h
new file mode 100644
index 000000000000..206f44db6cba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxpower.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX power header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX power
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXPOWER_H)
+#define RGXPOWER_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "servicesext.h"
+#include "rgxdevice.h"
+
+
+/*!
+******************************************************************************
+
+ @Function	RGXPrePowerState
+
+ @Description
+
+ does necessary preparation before power state transition
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPrePowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+							  PVRSRV_DEV_POWER_STATE	eNewPowerState,
+							  PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+							  PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPostPowerState
+
+ @Description
+
+ does necessary preparation after power state transition
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPostPowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+							   PVRSRV_DEV_POWER_STATE	eNewPowerState,
+							   PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+							   PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXVzPrePowerState
+
+ @Description
+
+ does necessary preparation before power state transition on a vz driver
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXVzPrePowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+								PVRSRV_DEV_POWER_STATE	eNewPowerState,
+								PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+								PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXVzPostPowerState
+
+ @Description
+
+ does necessary preparation after power state transition on a vz driver
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXVzPostPowerState(PPVRSRV_DEVICE_NODE	psDeviceNode,
+								 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+								 PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+								 PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPreClockSpeedChange
+
+ @Description
+
+	Does processing required before an RGX clock speed change.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eCurrentPowerState : Power state of the device
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPreClockSpeedChange(PPVRSRV_DEVICE_NODE		psDeviceNode,
+									PVRSRV_DEV_POWER_STATE	eCurrentPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPostClockSpeedChange
+
+ @Description
+
+	Does processing required after an RGX clock speed change.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eCurrentPowerState : Power state of the device
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPostClockSpeedChange(PPVRSRV_DEVICE_NODE	psDeviceNode,
+									 PVRSRV_DEV_POWER_STATE	eCurrentPowerState);
+
+
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+/*!
+******************************************************************************
+
+ @Function	RGXProcessCoreClkChangeRequest
+
+ @Input	   psDevInfo : RGX Device Info
+ @Input	   ui32CoreClockRate : New clock frequency to send to system layer.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeRequest(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate);
+#define RGX_PROCESS_CORE_CLK_RATE_CHANGE(devinfo, clk)  RGXProcessCoreClkChangeRequest(devinfo, clk)
+
+#else
+/*!
+******************************************************************************
+
+ @Function	RGXProcessCoreClkChangeNotification
+
+ @Input	   psDevInfo : RGX Device Info
+ @Input	   ui32CoreClockRate : New clock frequency.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeNotification(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate);
+#define RGX_PROCESS_CORE_CLK_RATE_CHANGE(devinfo, clk)  RGXProcessCoreClkChangeNotification(devinfo, clk)
+#endif
+#endif /* SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY */
+
+/*!
+******************************************************************************
+
+ @Function	RGXPowUnitsChange
+
+ @Description Change power units state
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32PowUnits : On Rogue: Number of DUSTs to make transition to.
+                          On Volcanic: Mask containing power state of SPUs.
+                          Each bit corresponds to an SPU and value must be non-zero.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPowUnitsChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                               IMG_UINT32 ui32PowUnits);
+
+/*!
+******************************************************************************
+
+ @Function	RGXAPMLatencyChange
+
+ @Description
+
+	Changes the wait duration used before firmware indicates IDLE.
+	Reducing this value will cause the firmware to shut off faster and
+	more often but may increase bubbles in GPU scheduling due to the added
+	power management activity. If bPersistent is NOT set, APM latency will
+	return back to system default on power up.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32ActivePMLatencyms : Number of milliseconds to wait
+ @Input	   bActivePMLatencyPersistant : Set to ensure new value is not reset
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXAPMLatencyChange(PPVRSRV_DEVICE_NODE	psDeviceNode,
+				IMG_UINT32				ui32ActivePMLatencyms,
+				IMG_BOOL				bActivePMLatencyPersistant);
+
+/*!
+******************************************************************************
+
+ @Function	RGXActivePowerRequest
+
+ @Description Initiate a handshake with the FW to power off the GPU
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXActivePowerRequest(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	RGXForcedIdleRequest
+
+ @Description Initiate a handshake with the FW to idle the GPU
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Input    bDeviceOffPermitted : Set to indicate device state being off is not
+                                 erroneous.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXForcedIdleRequest(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  IMG_BOOL bDeviceOffPermitted);
+
+/*!
+******************************************************************************
+
+ @Function	RGXCancelForcedIdleRequest
+
+ @Description Send a request to cancel idle to the firmware.
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXCancelForcedIdleRequest(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	RGXCancelForcedIdleRequestAsync
+
+ @Description Send a request to cancel idle to the firmware.
+              Does not wait for response from FW.
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXCancelForcedIdleRequestAsync(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+
+#endif /* RGXPOWER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.c
new file mode 100644
index 000000000000..c2f681c70f12
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.c
@@ -0,0 +1,331 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Register configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Regconfig routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxregconfig.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "device.h"
+#include "sync_internal.h"
+#include "pdump_km.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR PVRSRVRGXSetRegConfigTypeKM(CONNECTION_DATA * psDevConnection,
+                                         PVRSRV_DEVICE_NODE	 *psDeviceNode,
+                                         IMG_UINT8           ui8RegCfgType)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR          eError      = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO    *psDevInfo  = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG        *psRegCfg   = &psDevInfo->sRegConfig;
+	RGXFWIF_REG_CFG_TYPE  eRegCfgType = (RGXFWIF_REG_CFG_TYPE) ui8RegCfgType;
+
+	PVR_UNREFERENCED_PARAMETER(psDevConnection);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (eRegCfgType < psRegCfg->eRegCfgTypeToPush)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Register configuration requested (%d) is not valid since it has to be at least %d."
+			 " Configurations of different types need to go in order",
+			 __func__,
+			 eRegCfgType,
+			 psRegCfg->eRegCfgTypeToPush));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_INVALID_TYPE;
+	}
+
+	psRegCfg->eRegCfgTypeToPush = eRegCfgType;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui8RegCfgType);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXAddRegConfigKM(CONNECTION_DATA * psConnection,
+                                     PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                     IMG_UINT32		ui32RegAddr,
+                                     IMG_UINT64		ui64RegValue,
+                                     IMG_UINT64		ui64RegMask)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (psRegCfg->bEnabled)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Cannot add record whilst register configuration active.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_ENABLED;
+	}
+	if (psRegCfg->ui32NumRegRecords == RGXFWIF_REG_CFG_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Register configuration full.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_FULL;
+	}
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Addr = (IMG_UINT64) ui32RegAddr;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Value = ui64RegValue;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Mask = ui64RegMask;
+	sRegCfgCmd.uCmdData.sRegConfigData.eRegConfigType = psRegCfg->eRegCfgTypeToPush;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_ADD;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->ui32NumRegRecords++;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(ui64RegValue);
+	PVR_UNREFERENCED_PARAMETER(ui64RegMask);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXClearRegConfigKM(CONNECTION_DATA * psConnection,
+                                       PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (psRegCfg->bEnabled)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Attempt to clear register configuration whilst active.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_ENABLED;
+	}
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_CLEAR;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->ui32NumRegRecords = 0;
+	psRegCfg->eRegCfgTypeToPush = RGXFWIF_REG_CFG_TYPE_PWR_ON;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXEnableRegConfigKM(CONNECTION_DATA * psConnection,
+                                        PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_ENABLE;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->bEnabled = IMG_TRUE;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXDisableRegConfigKM(CONNECTION_DATA * psConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_DISABLE;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->bEnabled = IMG_FALSE;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+/******************************************************************************
+ End of file (rgxregconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.h
new file mode 100644
index 000000000000..b0921d98cb14
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxregconfig.h
@@ -0,0 +1,130 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX register configuration functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX register configuration functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXREGCONFIG_H)
+#define RGXREGCONFIG_H
+
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetRegConfigTypeKM
+
+ @Description
+	Server-side implementation of RGXSetRegConfig
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui8RegPowerIsland - Reg configuration
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetRegConfigTypeKM(CONNECTION_DATA * psDevConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                         IMG_UINT8 ui8RegPowerIsland);
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetRegConfigKM
+
+ @Description
+	Server-side implementation of RGXSetRegConfig
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui64RegAddr - Register address
+ @Input ui64RegValue - Reg value
+ @Input ui64RegMask - Reg mask
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+
+PVRSRV_ERROR PVRSRVRGXAddRegConfigKM(CONNECTION_DATA * psConnection,
+                                     PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                     IMG_UINT32	ui64RegAddr,
+                                     IMG_UINT64	ui64RegValue,
+                                     IMG_UINT64	ui64RegMask);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXClearRegConfigKM
+
+ @Description
+	Server-side implementation of RGXClearRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXClearRegConfigKM(CONNECTION_DATA * psConnection,
+                                       PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXEnableRegConfigKM
+
+ @Description
+	Server-side implementation of RGXEnableRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXEnableRegConfigKM(CONNECTION_DATA * psConnection,
+                                        PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDisableRegConfigKM
+
+ @Description
+	Server-side implementation of RGXDisableRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDisableRegConfigKM(CONNECTION_DATA * psConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+#endif /* RGXREGCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.c
new file mode 100644
index 000000000000..8eace4b16c81
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.c
@@ -0,0 +1,321 @@
+/*************************************************************************/ /*!
+@File           rgxshader.c
+@Title          TQ Shader Load
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shader code and info are shared for all context on the device.
+                If allocation doesn't already exist, read shader data from file
+                and allocate PMR memory. PMR memory is not deallocated until
+                device deinit.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxshader.h"
+#include "osfunc_common.h"
+#include "rgxdevice.h"
+#include "pdump_km.h"
+#include "physmem.h"
+#include "ri_server.h"
+#include "pvr_ricommon.h"
+#include "pvrsrv.h"
+
+static void
+RGXShaderReadHeader(OS_FW_IMAGE *psShaderFW, RGX_SHADER_HEADER *psHeader)
+{
+	const void * pvData;
+
+	pvData = OSFirmwareData(psShaderFW);
+
+	OSDeviceMemCopy(psHeader, pvData, sizeof(RGX_SHADER_HEADER));
+}
+
+static size_t
+RGXShaderCLIMemSize(OS_FW_IMAGE *psShaderFW)
+{
+	RGX_SHADER_HEADER sHeader;
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	return sHeader.ui32SizeClientMem;
+}
+
+static size_t
+RGXShaderUSCMemSize(OS_FW_IMAGE *psShaderFW)
+{
+	RGX_SHADER_HEADER sHeader;
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	return sHeader.ui32SizeFragment;
+}
+
+static void *
+RGXShaderCLIMem(OS_FW_IMAGE *psShaderFW)
+{
+	return (void*)OSFirmwareData(psShaderFW);
+}
+
+static void *
+RGXShaderUSCMem(OS_FW_IMAGE *psShaderFW)
+{
+	IMG_PBYTE pui8Data;
+
+	pui8Data = (IMG_PBYTE)OSFirmwareData(psShaderFW);
+
+	pui8Data += RGXShaderCLIMemSize(psShaderFW);
+
+	return (void*) pui8Data;
+}
+
+#define RGX_SHADER_FILENAME_MAX_SIZE   ((sizeof(RGX_SH_FILENAME)+	\
+										 RGX_BVNC_STR_SIZE_MAX))
+
+static void
+_GetShaderFileName(PVRSRV_DEVICE_NODE * psDeviceNode,
+				   IMG_CHAR           * pszShaderFilenameStr,
+				   IMG_CHAR           * pszShaderpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSSNPrintf(pszShaderFilenameStr, RGX_SHADER_FILENAME_MAX_SIZE,
+			   RGX_SH_FILENAME "." RGX_BVNC_STR_FMTSPEC,
+			   psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			   psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C);
+
+	OSSNPrintf(pszShaderpFilenameStr, RGX_SHADER_FILENAME_MAX_SIZE,
+			   RGX_SH_FILENAME "." RGX_BVNC_STRP_FMTSPEC,
+			   psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			   psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C);
+}
+
+PVRSRV_ERROR
+PVRSRVTQLoadShaders(PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_FW_IMAGE        *psShaderFW;
+	RGX_SHADER_HEADER   sHeader;
+	IMG_UINT32          ui32MappingTable = 0;
+	IMG_UINT32          ui32NumPages;
+	IMG_CHAR            aszShaderFilenameStr[RGX_SHADER_FILENAME_MAX_SIZE];
+	IMG_CHAR            aszShaderpFilenameStr[RGX_SHADER_FILENAME_MAX_SIZE];
+	const IMG_CHAR      *pszShaderFilenameStr = aszShaderFilenameStr;
+	size_t              uiNumBytes;
+	PVRSRV_DEVICE_NODE *psHostDevNode = PVRSRVGetPVRSRVData()->psHostMemDeviceNode;
+	size_t              uiLog2PageSize = OSGetPageShift();
+	IMG_DEVMEM_SIZE_T   uiTQUSCMemSize;
+	PVRSRV_ERROR        eError;
+
+	_GetShaderFileName(psDeviceNode, aszShaderFilenameStr, aszShaderpFilenameStr);
+
+	eError = OSLoadFirmware(psDeviceNode, aszShaderFilenameStr, NULL, &psShaderFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		eError = OSLoadFirmware(psDeviceNode, aszShaderpFilenameStr,
+		                        NULL, &psShaderFW);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to load shader binary file %s (%s)",
+			         __func__,
+			         aszShaderpFilenameStr,
+			         PVRSRVGetErrorString(eError)));
+			eError = PVRSRV_ERROR_UNABLE_TO_FIND_RESOURCE;
+			goto failed_init;
+		}
+
+		pszShaderFilenameStr = aszShaderpFilenameStr;
+	}
+
+	PVR_LOG(("Shader binary image '%s' loaded", pszShaderFilenameStr));
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	if (sHeader.ui32Version != RGX_TQ_SHADERS_VERSION_PACK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: unsupported TQ shaders version: %d != %d",
+				 __func__, sHeader.ui32Version, RGX_TQ_SHADERS_VERSION_PACK));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto failed_firmware;
+	}
+
+	ui32NumPages = (sHeader.ui32SizeFragment / IMG_PAGE2BYTES32(uiLog2PageSize)) + 1;
+
+	uiTQUSCMemSize = ui32NumPages * IMG_PAGE2BYTES32(uiLog2PageSize);
+
+	if (uiTQUSCMemSize > RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TQ shaders out of space ("IMG_DEVMEM_SIZE_FMTSPEC" > 0x%X)",
+				 __func__, uiTQUSCMemSize, RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto failed_firmware;
+	}
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate TDM USC PMR Block (Pages %08X)", ui32NumPages);
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiTQUSCMemSize,
+									1,
+									1,
+									&ui32MappingTable,
+									uiLog2PageSize,
+									PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT
+									| PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER,
+									sizeof("tquscpmr"),
+									"tquscpmr",
+									PVR_SYS_ALLOC_PID,
+									(PMR**)&psDevInfo->hTQUSCSharedMem,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PhysmemNewRamBackedPMR (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_firmware;
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(psDevInfo->hTQUSCSharedMem, PVR_SYS_ALLOC_PID);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RIWritePMREntryWithOwnerKM (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+#endif
+
+	eError = PMR_WriteBytes(psDevInfo->hTQUSCSharedMem, 0, RGXShaderUSCMem(psShaderFW), RGXShaderUSCMemSize(psShaderFW), &uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PMR_WriteBytes (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+
+	PDUMPCOMMENT(psDeviceNode, "Load TDM USC PMR Block (Size "IMG_SIZE_FMTSPECX")", RGXShaderUSCMemSize(psShaderFW));
+	PMRPDumpLoadMem(psDevInfo->hTQUSCSharedMem, 0, RGXShaderUSCMemSize(psShaderFW), PDUMP_FLAGS_CONTINUOUS, false);
+
+	ui32NumPages = (sHeader.ui32SizeClientMem / IMG_PAGE2BYTES32(uiLog2PageSize)) + 1;
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate TDM Client PMR Block (Pages %08X)", ui32NumPages);
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psHostDevNode,
+									(IMG_DEVMEM_SIZE_T)ui32NumPages * IMG_PAGE2BYTES32(uiLog2PageSize),
+									1,
+									1,
+									&ui32MappingTable,
+									uiLog2PageSize,
+									PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+									| PVRSRV_MEMALLOCFLAG_CPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT
+									| PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER
+									| PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL),
+									sizeof("tqclipmr"),
+									"tqclipmr",
+									PVR_SYS_ALLOC_PID,
+									(PMR**)&psDevInfo->hTQCLISharedMem,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PhysmemNewRamBackedPMR (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(psDevInfo->hTQCLISharedMem, PVR_SYS_ALLOC_PID);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RIWritePMREntryWithOwnerKM (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_clipmr;
+	}
+#endif
+
+	eError = PMR_WriteBytes(psDevInfo->hTQCLISharedMem, 0, RGXShaderCLIMem(psShaderFW), RGXShaderCLIMemSize(psShaderFW), &uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PMR_WriteBytes (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_clipmr;
+	}
+
+	OSUnloadFirmware(psShaderFW);
+
+	PVR_ASSERT(psDevInfo->hTQUSCSharedMem != NULL);
+	PVR_ASSERT(psDevInfo->hTQCLISharedMem != NULL);
+
+	return PVRSRV_OK;
+
+failed_clipmr:
+	(void) PMRUnrefPMR(psDevInfo->hTQCLISharedMem);
+failed_uscpmr:
+	(void) PMRUnrefPMR(psDevInfo->hTQUSCSharedMem);
+failed_firmware:
+	OSUnloadFirmware(psShaderFW);
+failed_init:
+	return eError;
+}
+
+void
+PVRSRVTQAcquireShaders(PVRSRV_DEVICE_NODE  * psDeviceNode,
+					   PMR                ** ppsCLIPMRMem)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_ASSERT(psDevInfo->hTQCLISharedMem != NULL);
+
+	*ppsCLIPMRMem = psDevInfo->hTQCLISharedMem;
+}
+
+void PVRSRVTQUnloadShaders(PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	(void) PMRUnrefPMR(psDevInfo->hTQUSCSharedMem);
+	(void) PMRUnrefPMR(psDevInfo->hTQCLISharedMem);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.h
new file mode 100644
index 000000000000..4ba50fc3fc86
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxshader.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File           rgxshader.h
+@Title          TQ Shader Load
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shader code and info are shared for all context on the device.
+                If allocation doesn't already exist, read shader data from file
+                and allocate PMR memory. PMR memory is not deallocated until
+                device deinit.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSHADER_H)
+#define RGXSHADER_H
+
+#include "fwload.h"
+#include "rgxtransfer_shader.h"
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQLoadShaders
+@Description    If PMR is not allocated, reads shader binary data from file
+                and allocates new PMR memory.
+@Input          psDeviceNode Device node
+@Return         PVRSRV_ERROR Returns PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVTQLoadShaders(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQAcquireShaders
+@Description    Get handle to ready allocated shader PMR memory
+@Input          psDeviceNode Device node
+@Output         ppsCLIPMRMem Shader data used by CPU client side.
+*/ /**************************************************************************/
+void
+PVRSRVTQAcquireShaders(PVRSRV_DEVICE_NODE  *psDeviceNode,
+                       PMR                **ppsCLIPMRMem);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQUnLoadShaders
+@Description    Unref PMR memory.
+@Input          psDeviceNode Device node
+*/ /**************************************************************************/
+void PVRSRVTQUnloadShaders(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* RGXSHADER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxstartstop.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxstartstop.h
new file mode 100644
index 000000000000..178afe2849a0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxstartstop.h
@@ -0,0 +1,84 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX start/stop header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX start/stop functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSTARTSTOP_H)
+#define RGXSTARTSTOP_H
+
+/* The routines declared here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when DRM security is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary.
+ */
+#include "rgxlayer.h"
+
+/*!
+*******************************************************************************
+
+ @Function      RGXStart
+
+ @Description   Perform GPU reset and initialisation
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXStart(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXStop
+
+ @Description   Stop Rogue in preparation for power down
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXStop(const void *hPrivate);
+
+#endif /* RGXSTARTSTOP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.c
new file mode 100644
index 000000000000..cec0597d0301
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.c
@@ -0,0 +1,184 @@
+/*************************************************************************/ /*!
+@File           rgxsyncutils.c
+@Title          RGX Sync Utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Sync helper functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxsyncutils.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "allocmem.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static
+void _DebugSyncValues(IMG_UINT32 *pui32UpdateValues,
+					  IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 iii;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (iii = 0; iii < ui32Count; iii++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+#else
+#define CHKPT_DBG(X)
+#endif
+
+
+PVRSRV_ERROR RGXSyncAppendTimelineUpdate(IMG_UINT32 ui32FenceTimelineUpdateValue,
+										 SYNC_ADDR_LIST	*psSyncList,
+										 SYNC_ADDR_LIST	*psPRSyncList,
+										 PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync,
+										 RGX_SYNC_DATA *psSyncData,
+										 IMG_BOOL bKick3D)
+{
+	IMG_UINT32 *pui32TimelineUpdateWOff = NULL;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+
+	IMG_UINT32 ui32ClientUpdateValueCount = psSyncData->ui32ClientUpdateValueCount;
+
+	/* Space for original client updates, and the one new update */
+	size_t uiUpdateSize = sizeof(*pui32IntAllocatedUpdateValues) * (ui32ClientUpdateValueCount + 1);
+
+	if (!bKick3D)
+	{
+		/* Additional space for one PR update, only the newest one */
+		uiUpdateSize += sizeof(*pui32IntAllocatedUpdateValues) * 1;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+		   "%s: About to allocate memory to hold updates in pui32IntAllocatedUpdateValues(<%p>)",
+		   __func__,
+		   (void*)pui32IntAllocatedUpdateValues));
+
+	/* Allocate memory to hold the list of update values (including our timeline update) */
+	pui32IntAllocatedUpdateValues = OSAllocMem(uiUpdateSize);
+	if (!pui32IntAllocatedUpdateValues)
+	{
+		/* Failed to allocate memory */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xcc, uiUpdateSize);
+	pui32TimelineUpdateWOff = pui32IntAllocatedUpdateValues;
+
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: Copying %d %s update values into pui32IntAllocatedUpdateValues(<%p>)",
+			   __func__,
+			   ui32ClientUpdateValueCount,
+			   bKick3D ? "TA/3D" : "TA/PR",
+			   (void*)pui32IntAllocatedUpdateValues));
+		/* Copy the update values into the new memory, then append our timeline update value */
+		OSCachedMemCopy(pui32TimelineUpdateWOff, psSyncData->paui32ClientUpdateValue, ui32ClientUpdateValueCount * sizeof(*psSyncData->paui32ClientUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(pui32TimelineUpdateWOff, ui32ClientUpdateValueCount);
+#endif
+
+		pui32TimelineUpdateWOff += ui32ClientUpdateValueCount;
+	}
+
+	/* Now set the additional update value and append the timeline sync prim addr to either the
+	 * render context 3D (or TA) update list
+	 */
+	CHKPT_DBG((PVR_DBG_ERROR,
+		   "%s: Appending the additional update value (0x%x) to psRenderContext->sSyncAddrList%sUpdate...",
+		   __func__,
+		   ui32FenceTimelineUpdateValue,
+		   bKick3D ? "TA/3D" : "TA/PR"));
+
+	/* Append the TA/3D update */
+	{
+		*pui32TimelineUpdateWOff++ = ui32FenceTimelineUpdateValue;
+		psSyncData->ui32ClientUpdateValueCount++;
+		psSyncData->ui32ClientUpdateCount++;
+		SyncAddrListAppendSyncPrim(psSyncList, psFenceTimelineUpdateSync);
+
+		if (!psSyncData->pauiClientUpdateUFOAddress)
+		{
+			psSyncData->pauiClientUpdateUFOAddress = psSyncList->pasFWAddrs;
+		}
+		/* Update paui32ClientUpdateValue to point to our new list of update values */
+		psSyncData->paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(pui32IntAllocatedUpdateValues, psSyncData->ui32ClientUpdateValueCount);
+#endif
+	}
+
+	if (!bKick3D)
+	{
+		/* Use the sSyncAddrList3DUpdate for PR (as it doesn't have one of its own) */
+		*pui32TimelineUpdateWOff++ = ui32FenceTimelineUpdateValue;
+		psSyncData->ui32ClientPRUpdateValueCount = 1;
+		psSyncData->ui32ClientPRUpdateCount = 1;
+		SyncAddrListAppendSyncPrim(psPRSyncList, psFenceTimelineUpdateSync);
+
+		if (!psSyncData->pauiClientPRUpdateUFOAddress)
+		{
+			psSyncData->pauiClientPRUpdateUFOAddress = psPRSyncList->pasFWAddrs;
+		}
+		/* Update paui32ClientPRUpdateValue to point to our new list of update values */
+		psSyncData->paui32ClientPRUpdateValue = &pui32IntAllocatedUpdateValues[psSyncData->ui32ClientUpdateValueCount];
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(psSyncData->paui32ClientPRUpdateValue, psSyncData->ui32ClientPRUpdateValueCount);
+#endif
+	}
+
+	/* Do not free the old psSyncData->ui32ClientUpdateValueCount,
+	 * as it was constant data passed through the bridge down to PVRSRVRGXKickTA3DKM() */
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.h
new file mode 100644
index 000000000000..bfc46937f185
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxsyncutils.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           rgxsyncutils.h
+@Title          RGX Sync Utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Sync helper functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXSYNCUTILS_H
+#define RGXSYNCUTILS_H
+
+#include "rgxdevice.h"
+#include "sync_server.h"
+#include "rgxdebug_common.h"
+#include "rgx_fwif_km.h"
+
+typedef struct _RGX_SYNC_DATA_
+{
+	PRGXFWIF_UFO_ADDR *pauiClientUpdateUFOAddress;
+	IMG_UINT32 *paui32ClientUpdateValue;
+	IMG_UINT32 ui32ClientUpdateValueCount;
+	IMG_UINT32 ui32ClientUpdateCount;
+
+	PRGXFWIF_UFO_ADDR *pauiClientPRUpdateUFOAddress;
+	IMG_UINT32 *paui32ClientPRUpdateValue;
+	IMG_UINT32 ui32ClientPRUpdateValueCount;
+	IMG_UINT32 ui32ClientPRUpdateCount;
+} RGX_SYNC_DATA;
+
+PVRSRV_ERROR RGXSyncAppendTimelineUpdate(IMG_UINT32 ui32FenceTimelineUpdateValue,
+										 SYNC_ADDR_LIST	*psSyncList,
+										 SYNC_ADDR_LIST	*psPRSyncList,
+										 PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync,
+										 RGX_SYNC_DATA *psSyncData,
+										 IMG_BOOL bKick3D);
+
+#endif /* RGXSYNCUTILS_H */
+
+/******************************************************************************
+ End of file (rgxsyncutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.c
new file mode 100644
index 000000000000..dd9d05ee18b2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.c
@@ -0,0 +1,1486 @@
+/*************************************************************************/ /*!
+@File           rgxtdmtransfer.c
+@Title          Device specific TDM transfer queue routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxtdmtransfer.h"
+#include "rgx_tq_shared.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_memallocflags.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxshader.h"
+
+#include "pdump_km.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TDM_UFO_DUMP	0
+
+//#define TDM_CHECKPOINT_DEBUG 1
+
+#if defined(TDM_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+typedef struct {
+	RGX_SERVER_COMMON_CONTEXT * psServerCommonContext;
+	IMG_INT32                   i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_TDM_DATA;
+
+
+struct _RGX_SERVER_TQ_TDM_CONTEXT_ {
+	PVRSRV_DEVICE_NODE      *psDeviceNode;
+	DEVMEM_MEMDESC          *psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC          *psFWTransferContextMemDesc;
+	IMG_UINT32              ui32Flags;
+	RGX_SERVER_TQ_TDM_DATA  sTDMData;
+	DLLIST_NODE             sListNode;
+	SYNC_ADDR_LIST          sSyncAddrListFence;
+	SYNC_ADDR_LIST          sSyncAddrListUpdate;
+	POS_LOCK		hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA       sWorkEstData;
+#endif
+};
+
+static PVRSRV_ERROR _CreateTDMTransferContext(
+	CONNECTION_DATA         * psConnection,
+	PVRSRV_DEVICE_NODE      * psDeviceNode,
+	DEVMEM_MEMDESC          * psAllocatedMemDesc,
+	IMG_UINT32                ui32AllocatedOffset,
+	SERVER_MMU_CONTEXT      * psServerMMUContext,
+	DEVMEM_MEMDESC          * psFWMemContextMemDesc,
+	IMG_INT32                 i32Priority,
+	IMG_INT32                 ui32MaxDeadlineMS,
+	RGX_COMMON_CONTEXT_INFO * psInfo,
+	RGX_SERVER_TQ_TDM_DATA  * psTDMData,
+	IMG_UINT32                ui32CCBAllocSizeLog2,
+	IMG_UINT32                ui32CCBMaxAllocSizeLog2,
+	IMG_UINT32                ui32ContextFlags,
+	IMG_UINT64                ui64RobustnessAddress)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psTDMData->psBufferSyncContext =
+			pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+			                               "rogue-tdm");
+	if (IS_ERR(psTDMData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to create buffer_sync context (err=%ld)",
+				__func__, PTR_ERR(psTDMData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = FWCommonContextAllocate(
+			psConnection,
+			psDeviceNode,
+			REQ_TYPE_TQ_TDM,
+			RGXFWIF_DM_TDM,
+			psServerMMUContext,
+			psAllocatedMemDesc,
+			ui32AllocatedOffset,
+			psFWMemContextMemDesc,
+			NULL,
+			ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TDM_CCB_SIZE_LOG2,
+			ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TDM_CCB_MAX_SIZE_LOG2,
+			ui32ContextFlags,
+			i32Priority,
+			ui32MaxDeadlineMS,
+			ui64RobustnessAddress,
+			psInfo,
+			&psTDMData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	psTDMData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(psTDMData->psBufferSyncContext);
+	psTDMData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+static PVRSRV_ERROR _DestroyTDMTransferContext(
+	RGX_SERVER_TQ_TDM_DATA  * psTDMData,
+	PVRSRV_DEVICE_NODE      * psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(
+		psDeviceNode,
+		psTDMData->psServerCommonContext,
+		RGXFWIF_DM_TDM,
+		PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free it's resources */
+	FWCommonContextFree(psTDMData->psServerCommonContext);
+	psTDMData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(psTDMData->psBufferSyncContext);
+	psTDMData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+
+/*
+ * PVRSRVCreateTransferContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXTDMCreateTransferContextKM(
+	CONNECTION_DATA            * psConnection,
+	PVRSRV_DEVICE_NODE         * psDeviceNode,
+	IMG_INT32                    i32Priority,
+	IMG_UINT32                   ui32FrameworkCommandSize,
+	IMG_PBYTE                    pabyFrameworkCommand,
+	IMG_HANDLE                   hMemCtxPrivData,
+	IMG_UINT32                   ui32PackedCCBSizeU88,
+	IMG_UINT32                   ui32ContextFlags,
+	IMG_UINT64                   ui64RobustnessAddress,
+	IMG_UINT32                   ui32MaxDeadlineMS,
+	RGX_SERVER_TQ_TDM_CONTEXT ** ppsTransferContext)
+{
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext;
+
+	DEVMEM_MEMDESC          * psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_RGXDEV_INFO      * psDevInfo = psDeviceNode->pvDevice;
+	RGX_COMMON_CONTEXT_INFO   sInfo = {NULL};
+	PVRSRV_ERROR              eError = PVRSRV_OK;
+
+	/* Allocate the server side structure */
+	*ppsTransferContext = NULL;
+	psTransferContext = OSAllocZMem(sizeof(*psTransferContext));
+	if (psTransferContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW transfer context, this has the TDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWTDMCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwTransferContext",
+			&psTransferContext->psFWTransferContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwtransfercontext;
+	}
+
+	eError = OSLockCreate(&psTransferContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_lockcreate;
+	}
+
+	psTransferContext->psDeviceNode = psDeviceNode;
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psTransferContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psTransferContext->psFWFrameworkMemDesc,
+				pabyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psTransferContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _CreateTDMTransferContext(psConnection,
+	                                   psDeviceNode,
+	                                   psTransferContext->psFWTransferContextMemDesc,
+	                                   offsetof(RGXFWIF_FWTDMCONTEXT, sTDMContext),
+	                                   hMemCtxPrivData,
+	                                   psFWMemContextMemDesc,
+	                                   i32Priority,
+	                                   ui32MaxDeadlineMS,
+	                                   &sInfo,
+	                                   &psTransferContext->sTDMData,
+									   U32toU8_Unpack1(ui32PackedCCBSizeU88),
+									   U32toU8_Unpack2(ui32PackedCCBSizeU88),
+	                                   ui32ContextFlags,
+	                                   ui64RobustnessAddress);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tdmtransfercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTDM(psDevInfo, &psTransferContext->sWorkEstData);
+	}
+#endif
+
+	SyncAddrListInit(&psTransferContext->sSyncAddrListFence);
+	SyncAddrListInit(&psTransferContext->sSyncAddrListUpdate);
+
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTDMCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+	*ppsTransferContext = psTransferContext;
+
+	return PVRSRV_OK;
+
+fail_tdmtransfercontext:
+fail_frameworkcopy:
+	if (psTransferContext->psFWFrameworkMemDesc != NULL)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	OSLockDestroy(psTransferContext->hLock);
+fail_lockcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+fail_fwtransfercontext:
+	OSFreeMem(psTransferContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*ppsTransferContext = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMDestroyTransferContextKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		RGXFWIF_FWTDMCONTEXT	*psFWTransferContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psTransferContext->psFWTransferContextMemDesc,
+										  (void **)&psFWTransferContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to map firmware transfer context (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+		RGXFwSharedMemCacheOpValue(psFWTransferContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+		ui32WorkEstCCBSubmitted = psFWTransferContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psTransferContext->psFWTransferContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psTransferContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					 __func__, ui32WorkEstCCBSubmitted,
+					 psTransferContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_remove_node(&(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+
+
+	eError = _DestroyTDMTransferContext(&psTransferContext->sTDMData,
+	                                    psTransferContext->psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_destroyTDM;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstDeInitTDM(psDevInfo, &psTransferContext->sWorkEstData);
+	}
+#endif
+
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+
+	SyncAddrListDeinit(&psTransferContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psTransferContext->sSyncAddrListUpdate);
+
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+
+	OSLockDestroy(psTransferContext->hLock);
+
+	OSFreeMem(psTransferContext);
+
+	return PVRSRV_OK;
+
+fail_destroyTDM:
+
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTDMCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+ * PVRSRVSubmitTQ3DKickKM
+ */
+/* Old bridge call for backwards compatibility. */
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransferKM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * paui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32                  ui32TDMCharacteristic1,
+	IMG_UINT32                  ui32TDMCharacteristic2,
+	IMG_UINT64                  ui64DeadlineInus)
+{
+	return PVRSRVRGXTDMSubmitTransfer3KM(psTransferContext,
+										 ui32PDumpFlags,
+										 ui32ClientUpdateCount,
+										 pauiClientUpdateUFODevVarBlock,
+										 paui32ClientUpdateSyncOffset,
+										 paui32ClientUpdateValue,
+										 iCheckFence,
+										 iUpdateTimeline,
+										 piUpdateFence,
+										 szUpdateFenceName,
+										 PVRSRV_NO_FENCE,
+										 ui32FWCommandSize,
+										 pui8FWCommand,
+										 ui32ExtJobRef,
+										 ui32SyncPMRCount,
+										 paui32SyncPMRFlags,
+										 ppsSyncPMRs,
+										 ui32TDMCharacteristic1,
+										 ui32TDMCharacteristic2,
+										 ui64DeadlineInus);
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransfer3KM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	PVRSRV_FENCE                iExportFenceToSignal,
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * paui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32                  ui32TDMCharacteristic1,
+	IMG_UINT32                  ui32TDMCharacteristic2,
+	IMG_UINT64                  ui64DeadlineInus)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	RGX_CCB_CMD_HELPER_DATA *psCmdHelper;
+	PRGXFWIF_UFO_ADDR * pauiIntFenceUFOAddress   = NULL;
+	PRGXFWIF_UFO_ADDR * pauiIntUpdateUFOAddress  = NULL;
+	IMG_UINT32          ui32IntClientFenceCount  = 0;
+	IMG_UINT32        * paui32IntUpdateValue     = paui32ClientUpdateValue;
+	IMG_UINT32          ui32IntClientUpdateCount = ui32ClientUpdateCount;
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE iUpdateFence = PVRSRV_NO_FENCE;
+	PVRSRV_RGXDEV_INFO  *psDevInfo = FWCommonContextGetRGXDevInfo(psTransferContext->sTDMData.psServerCommonContext);
+	RGX_CLIENT_CCB      *psClientCCB = FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext);
+	IMG_UINT32          ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+
+	IMG_UINT64          ui64FBSCEntryMask = 0;
+
+	IMG_BOOL bCCBStateOpen;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32 ui32CmdOffset = 0;
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTransfer = {0};
+	IMG_UINT32 ui32TDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32TDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32TDMCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+#endif
+
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT psExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(SUPPORT_WORKLOAD_ESTIMATION)
+	PVR_UNREFERENCED_PARAMETER(ui32TDMCharacteristic1);
+	PVR_UNREFERENCED_PARAMETER(ui32TDMCharacteristic2);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	if (ui32SyncPMRCount != 0)
+	{
+		if (!ppsSyncPMRs)
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* We can't allocate the required amount of stack space on all consumer architectures */
+	psCmdHelper = OSAllocMem(sizeof(RGX_CCB_CMD_HELPER_DATA));
+	if (psCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_allochelper;
+	}
+
+
+	/*
+		Init the command helper commands for all the prepares
+	*/
+	{
+		IMG_CHAR *pszCommandName;
+		RGXFWIF_CCB_CMD_TYPE eType;
+#if defined(SUPPORT_BUFFER_SYNC)
+		struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+
+		pszCommandName = "TQ-TDM";
+
+		if (ui32FWCommandSize == 0)
+		{
+			/* A NULL CMD for TDM is used to append updates to a non finished
+			 * FW command. bCCBStateOpen is used in case capture range is
+			 * entered on this command, to not drain CCB up to the Roff for this
+			 * command, but the finished command prior to this.
+			 */
+			bCCBStateOpen = IMG_TRUE;
+			eType = RGXFWIF_CCB_CMD_TYPE_NULL;
+		}
+		else
+		{
+			bCCBStateOpen = IMG_FALSE;
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_TDM;
+		}
+
+#if defined(SUPPORT_BUFFER_SYNC)
+		psBufferSyncContext = psTransferContext->sTDMData.psBufferSyncContext;
+#endif
+
+		eError = SyncAddrListPopulate(&psTransferContext->sSyncAddrListFence,
+		                              0,
+		                              NULL,
+		                              NULL);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_populate_sync_addr_list;
+		}
+
+		eError = SyncAddrListPopulate(&psTransferContext->sSyncAddrListUpdate,
+		                              ui32ClientUpdateCount,
+		                              pauiClientUpdateUFODevVarBlock,
+		                              paui32ClientUpdateSyncOffset);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_populate_sync_addr_list;
+		}
+		paui32IntUpdateValue    = paui32ClientUpdateValue;
+		pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+
+
+		if (ui32SyncPMRCount)
+		{
+#if defined(SUPPORT_BUFFER_SYNC)
+			int err;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling pvr_buffer_sync_resolve_and_create_fences", __func__));
+			err = pvr_buffer_sync_resolve_and_create_fences(psBufferSyncContext,
+			                                                psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                                ui32SyncPMRCount,
+			                                                ppsSyncPMRs,
+			                                                paui32SyncPMRFlags,
+			                                                &ui32BufferFenceSyncCheckpointCount,
+			                                                &apsBufferFenceSyncCheckpoints,
+			                                                &psBufferUpdateSyncCheckpoint,
+			                                                &psBufferSyncData);
+			if (err)
+			{
+				switch (err)
+				{
+					case -EINTR:
+						eError = PVRSRV_ERROR_RETRY;
+						break;
+					case -ENOMEM:
+						eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+						break;
+					default:
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						break;
+				}
+
+				if (eError != PVRSRV_ERROR_RETRY)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   pvr_buffer_sync_resolve_and_create_fences failed (%s)", __func__, PVRSRVGetErrorString(eError)));
+				}
+				goto fail_resolve_input_fence;
+			}
+
+			/* Append buffer sync fences */
+			if (ui32BufferFenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints to TQ Fence (&psTransferContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>)...", __func__, ui32BufferFenceSyncCheckpointCount, (void*)&psTransferContext->sSyncAddrListFence , (void*)pauiIntFenceUFOAddress));
+				SyncAddrListAppendAndDeRefCheckpoints(&psTransferContext->sSyncAddrListFence,
+				                                      ui32BufferFenceSyncCheckpointCount,
+				                                      apsBufferFenceSyncCheckpoints);
+				if (!pauiIntFenceUFOAddress)
+				{
+					pauiIntFenceUFOAddress = psTransferContext->sSyncAddrListFence.pasFWAddrs;
+				}
+				ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+			}
+
+			if (psBufferUpdateSyncCheckpoint)
+			{
+				/* Append the update (from output fence) */
+				SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+				                              1,
+				                              &psBufferUpdateSyncCheckpoint);
+				if (!pauiIntUpdateUFOAddress)
+				{
+					pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+				}
+				ui32IntClientUpdateCount++;
+			}
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+			PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+			PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers", __func__, ui32SyncPMRCount));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto fail_populate_sync_addr_list;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+		}
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(psTransferContext->psDeviceNode->hSyncCheckpointContext,
+		                                    iCheckFence,
+		                                    &ui32FenceSyncCheckpointCount,
+		                                    &apsFenceSyncCheckpoints,
+		                                    &uiCheckFenceUID,
+		                                    ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_resolve_input_fence;
+		}
+#if defined(TDM_CHECKPOINT_DEBUG)
+		{
+			IMG_UINT32 ii;
+			for (ii=0; ii<32; ii++)
+			{
+				PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints +  ii);
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint)); //psFenceSyncCheckpoints[ii]));
+			}
+		}
+#endif
+		/* Create the output fence (if required) */
+		if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+		{
+			eError = SyncCheckpointCreateFence(psTransferContext->psDeviceNode,
+			                                   szUpdateFenceName,
+			                                   iUpdateTimeline,
+			                                   psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                   &iUpdateFence,
+			                                   &uiUpdateFenceUID,
+			                                   &pvUpdateFenceFinaliseData,
+			                                   &psUpdateSyncCheckpoint,
+			                                   (void*)&psFenceTimelineUpdateSync,
+			                                   &ui32FenceTimelineUpdateValue,
+			                                   ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				goto fail_create_output_fence;
+			}
+
+			/* Resolve the iExportFenceToSignal (if required) */
+			if (iExportFenceToSignal != PVRSRV_NO_FENCE)
+			{
+				IMG_UINT32 iii;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+				eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+														  psTransferContext->psDeviceNode->hSyncCheckpointContext,
+														  &psExportFenceSyncCheckpoint,
+														  ui32PDumpFlags);
+				if (eError != PVRSRV_OK)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), psExportFenceSyncCheckpoint));
+					goto fail_resolve_export_fence;
+				}
+
+				/* Check that the export fence was not also included as part of the
+				 * check fence (which is an error and would lead to a stalled kick).
+				 */
+				for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+				{
+					if (apsFenceSyncCheckpoints[iii] == psExportFenceSyncCheckpoint)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, psExportFenceSyncCheckpoint));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+						goto fail_check_fence_includes_export_fence;
+					}
+				}
+			}
+
+			/* Append the sync prim update for the timeline (if required) */
+			if (psFenceTimelineUpdateSync)
+			{
+				IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+				/* Allocate memory to hold the list of update values (including our timeline update) */
+				pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				if (!pui32IntAllocatedUpdateValues)
+				{
+					/* Failed to allocate memory */
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto fail_alloc_update_values_mem;
+				}
+				OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				/* Copy the update values into the new memory, then append our timeline update value */
+				if (paui32IntUpdateValue)
+				{
+					OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+				}
+				/* Now set the additional update value */
+				pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+				*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+				ui32IntClientUpdateCount++;
+#if defined(TDM_CHECKPOINT_DEBUG)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Now append the timeline sync prim addr to the transfer context update list */
+				SyncAddrListAppendSyncPrim(&psTransferContext->sSyncAddrListUpdate,
+				                           psFenceTimelineUpdateSync);
+#if defined(TDM_CHECKPOINT_DEBUG)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+				paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+			}
+		}
+
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			/* Append the checks (from input fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to TQ Fence (&psTransferContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psTransferContext->sSyncAddrListFence));
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntFenceUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListFence,
+										  ui32FenceSyncCheckpointCount,
+										  apsFenceSyncCheckpoints);
+			if (!pauiIntFenceUFOAddress)
+			{
+				pauiIntFenceUFOAddress = psTransferContext->sSyncAddrListFence.pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+		if (psUpdateSyncCheckpoint)
+		{
+			/* Append the update (from output fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to TQ Update (&psTransferContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)&psTransferContext->sSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+			                              1,
+			                              &psUpdateSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		if (psExportFenceSyncCheckpoint)
+		{
+			/* Append the update (from export fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Transfer CDM Update (&psTransferContext->sSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)&psTransferContext->sSyncAddrListUpdate , (void*)psExportFenceSyncCheckpoint));
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+										  1,
+										  &psExportFenceSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+#if (ENABLE_TDM_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping TDM fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TDM fence syncs (&psTransferContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psTransferContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TDM update syncs (&psTransferContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psTransferContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+		RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psTransferContext->psDeviceNode->pvDevice,
+		                          &pPreAddr,
+		                          &pPostAddr,
+		                          &pRMWUFOAddr);
+
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+		/*
+		 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+		 * in other words, take the value and set it to zero afterwards.
+		 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+		 * as it must be ready at the time of context activation.
+		 */
+		{
+			eError = RGXExtractFBSCEntryMaskFromMMUContext(psTransferContext->psDeviceNode,
+														   FWCommonContextGetServerMMUCtx(psTransferContext->sTDMData.psServerCommonContext),
+														   &ui64FBSCEntryMask);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%d)", eError));
+				goto fail_invalfbsc;
+			}
+		}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			sWorkloadCharacteristics.sTransfer.ui32Characteristic1 = ui32TDMCharacteristic1;
+			sWorkloadCharacteristics.sTransfer.ui32Characteristic2 = ui32TDMCharacteristic2;
+
+			/* Prepare workload estimation */
+			WorkEstPrepare(psDeviceNode->pvDevice,
+					&psTransferContext->sWorkEstData,
+					&psTransferContext->sWorkEstData.uWorkloadMatchingData.sTransfer.sDataTDM,
+					eType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTransfer);
+		}
+#endif
+
+		/*
+			Create the command helper data for this command
+		*/
+		RGXCmdHelperInitCmdCCB(psDevInfo,
+		                       psClientCCB,
+		                       ui64FBSCEntryMask,
+		                       ui32IntClientFenceCount,
+		                       pauiIntFenceUFOAddress,
+		                       NULL,
+		                       ui32IntClientUpdateCount,
+		                       pauiIntUpdateUFOAddress,
+		                       paui32IntUpdateValue,
+		                       ui32FWCommandSize,
+		                       pui8FWCommand,
+		                       &pPreAddr,
+		                       &pPostAddr,
+		                       &pRMWUFOAddr,
+		                       eType,
+		                       ui32ExtJobRef,
+		                       ui32IntJobRef,
+		                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                       &sWorkloadKickDataTransfer,
+#else /* SUPPORT_WORKLOAD_ESTIMATION */
+		                       NULL,
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+		                       pszCommandName,
+		                       bCCBStateOpen,
+		                       psCmdHelper);
+
+	}
+
+	/*
+		Acquire space for all the commands in one go
+	*/
+
+	eError = RGXCmdHelperAcquireCmdCCB(1, psCmdHelper);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcmdacquire;
+	}
+
+
+	/*
+		We should acquire the kernel CCB(s) space here as the schedule could fail
+		and we would have to roll back all the syncs
+	*/
+
+	/*
+		Only do the command helper release (which takes the server sync
+		operations if the acquire succeeded
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		ui32CmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext));
+	}
+#endif
+	RGXCmdHelperReleaseCmdCCB(1,
+	                          psCmdHelper,
+	                          "TQ_TDM",
+	                          FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext).ui32Addr);
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* The following is used to determine the offset of the command header containing
+		   the workload estimation data so that can be accessed when the KCCB is read */
+		ui32TDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(psCmdHelper);
+
+		ui32TDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32CmdOffset < ui32TDMCmdOffsetWrapCheck)
+		{
+			ui32TDMWorkloadDataRO = ui32CmdOffset;
+		}
+		else
+		{
+			ui32TDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/*
+		Even if we failed to acquire the client CCB space we might still need
+		to kick the HW to process a padding packet to release space for us next
+		time round
+	*/
+	{
+		RGXFWIF_KCCB_CMD sTDMKCCBCmd;
+		IMG_UINT32 ui32FWAddr = FWCommonContextGetFWAddress(
+		        psTransferContext->sTDMData.psServerCommonContext).ui32Addr;
+
+		/* Construct the kernel 3D CCB command. */
+		sTDMKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		sTDMKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced
+			 * when the KCCB command reaches the FW */
+			sTDMKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32TDMWorkloadDataRO + ui32TDMCmdHeaderOffset;
+		}
+#endif
+
+		/* HTBLOGK(HTB_SF_MAIN_KICK_TDM, */
+		/*		s3DKCCBCmd.uCmdData.sCmdKickData.psContext, */
+		/*		ui323DCmdOffset); */
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext).ui32Addr,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQTDM,
+		                  iCheckFence,
+		                  iUpdateFence,
+		                  iUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  uiUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = RGXScheduleCommandWithoutPowerLock(psDeviceNode->pvDevice,
+			                             RGXFWIF_DM_TDM,
+			                             & sTDMKCCBCmd,
+			                             ui32PDumpFlags);
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXTDMSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError));
+			goto fail_2dcmdacquire;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWAddr, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQTDM);
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointSignalNoHW(psExportFenceSyncCheckpoint);
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	* piUpdateFence = iUpdateFence;
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, iUpdateFence, pvUpdateFenceFinaliseData,
+		                            psUpdateSyncCheckpoint, szUpdateFenceName);
+	}
+
+	OSFreeMem(psCmdHelper);
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+/*
+	No resources are created in this function so there is nothing to free
+	unless we had to merge syncs.
+	If we fail after the client CCB acquire there is still nothing to do
+	as only the client CCB release will modify the client CCB
+*/
+fail_2dcmdacquire:
+fail_acquirepowerlock:
+fail_3dcmdacquire:
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+fail_invalfbsc:
+#endif
+	SyncAddrListRollbackCheckpoints(psTransferContext->psDeviceNode, &psTransferContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psTransferContext->psDeviceNode, &psTransferContext->sSyncAddrListUpdate);
+
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+fail_alloc_update_values_mem:
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+fail_check_fence_includes_export_fence:
+fail_resolve_export_fence:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+
+fail_resolve_input_fence:
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+fail_populate_sync_addr_list:
+	OSFreeMem(psCmdHelper);
+fail_allochelper:
+
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	OSLockRelease(psTransferContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+	IMG_UINT32                 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD  sKCCBCmd;
+	PVRSRV_ERROR      eError;
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* Schedule the firmware command */
+	sKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE;
+	sKCCBCmd.uCmdData.sWriteOffsetUpdateData.psContext = FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psTransferContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_TDM,
+		                            &sKCCBCmd,
+		                            ui32PDumpFlags);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to schedule the FW command %d (%s)",
+				__func__, eError, PVRSRVGETERRORSTRING(eError)));
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                      RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+                                                      IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	if (psTransferContext->sTDMData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psTransferContext->sTDMData.psServerCommonContext,
+		                            psConnection,
+		                            psTransferContext->psDeviceNode->pvDevice,
+		                            i32Priority,
+		                            RGXFWIF_DM_TDM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority (%s)", __func__, PVRSRVGetErrorString(eError)));
+
+			OSLockRelease(psTransferContext->hLock);
+			return eError;
+		}
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPropertyKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+													  RGX_CONTEXT_PROPERTY eContextProperty,
+													  IMG_UINT64 ui64Input,
+													  IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psTransferContext->hLock);
+			eError = FWCommonContextSetFlags(psTransferContext->sTDMData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			OSLockRelease(psTransferContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpTDMTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                              void *pvDumpDebugFile,
+                              IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hTDMCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTDMCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_TDM_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_TDM_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerTransferCtx->sTDMData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTDMCtxListLock);
+}
+
+
+IMG_UINT32 CheckForStalledClientTDMTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hTDMCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTDMCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_TDM_CONTEXT *psCurrentServerTransferCtx =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_TDM_CONTEXT, sListNode);
+
+		if (CheckStalledClientCommonContext(
+				psCurrentServerTransferCtx->sTDMData.psServerCommonContext, RGX_KICK_TYPE_DM_TDM_2D)
+				== PVRSRV_ERROR_CCCB_STALLED) {
+			ui32ContextBitMask = RGX_KICK_TYPE_DM_TDM_2D;
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTDMCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/**************************************************************************//**
+ End of file (rgxtdmtransfer.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.h
new file mode 100644
index 000000000000..b0f003cf3ad3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtdmtransfer.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@File           rgxtdmtransfer.h
+@Title          RGX Transfer queue 2 Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Transfer queue Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTDMTRANSFER_H)
+#define RGXTDMTRANSFER_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "connection_server.h"
+
+typedef struct _RGX_SERVER_TQ_TDM_CONTEXT_ RGX_SERVER_TQ_TDM_CONTEXT;
+
+
+PVRSRV_ERROR PVRSRVRGXTDMCreateTransferContextKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	IMG_INT32                   i32Priority,
+	IMG_UINT32                  ui32FrameworkCommandSize,
+	IMG_PBYTE                   pabyFrameworkCommand,
+	IMG_HANDLE                  hMemCtxPrivData,
+	IMG_UINT32					ui32PackedCCBSizeU88,
+	IMG_UINT32                  ui32ContextFlags,
+	IMG_UINT64					ui64RobustnessAddress,
+	IMG_UINT32                  ui32MaxDeadlineMS,
+	RGX_SERVER_TQ_TDM_CONTEXT **ppsTransferContext);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMDestroyTransferContextKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransferKM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * pui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32					ui32TDMCharacteristic1,
+	IMG_UINT32					ui32TDMCharacteristic2,
+	IMG_UINT64					ui64DeadlineInus);
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransfer3KM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	PVRSRV_FENCE                iExportFenceToSignal,
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * pui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32					ui32TDMCharacteristic1,
+	IMG_UINT32					ui32TDMCharacteristic2,
+	IMG_UINT64					ui64DeadlineInus);
+
+PVRSRV_ERROR PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+	IMG_UINT32                 ui32PDumpFlags);
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDeviceNode,
+												   RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+												   IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPropertyKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+													 RGX_CONTEXT_PROPERTY eContextProperty,
+													 IMG_UINT64 ui64Input,
+													 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of TDM transfer contexts on this device */
+void DumpTDMTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                              void *pvDumpDebugFile,
+                              IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client transfer contexts are stalled */
+IMG_UINT32 CheckForStalledClientTDMTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+
+#endif /* RGXTDMTRANSFER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.c
new file mode 100644
index 000000000000..bea5cd1c98ef
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.c
@@ -0,0 +1,791 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific time correlation and calibration routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific time correlation and calibration routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxtimecorr.h"
+#include "rgxfwutils.h"
+#include "htbserver.h"
+#include "pvrsrv_apphint.h"
+#include "rgxpower.h"
+
+/******************************************************************************
+ *
+ * - A calibration period is started on power-on and after a DVFS transition,
+ *   and it's closed before a power-off and before a DVFS transition
+ *   (so power-on -> dfvs -> dvfs -> power-off , power on -> dvfs -> dvfs...,
+ *   where each arrow is a calibration period).
+ *
+ * - The timers on the Host and on the FW are correlated at the beginning of
+ *   each period together with the current GPU frequency.
+ *
+ * - Correlation and calibration are also done at regular intervals using
+ *   a best effort approach.
+ *
+ *****************************************************************************/
+
+/*
+	AppHint interfaces
+*/
+
+static PVRSRV_ERROR _SetClock(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 ui32Value)
+{
+	static __maybe_unused const char* const apszClocks[] = {
+		"mono", "mono_raw", "sched"
+	};
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	if (ui32Value >= RGXTIMECORR_CLOCK_LAST)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid clock source type (%u)", ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING,
+	        "Setting time correlation clock from \"%s\" to \"%s\"",
+	        apszClocks[psDevInfo->ui32ClockSource],
+	        apszClocks[ui32Value]));
+
+	/* PVRSRVPowerLock() fails only when power is off. */
+	if (PVRSRVPowerLock((PVRSRV_DEVICE_NODE *) psDeviceNode) == PVRSRV_OK)
+	{
+		RGXTimeCorrEnd((PVRSRV_DEVICE_NODE *) psDeviceNode,
+		               RGXTIMECORR_EVENT_CLOCK_CHANGE);
+
+		psDevInfo->ui32ClockSource = ui32Value;
+
+		RGXTimeCorrBegin((PVRSRV_DEVICE_NODE *) psDeviceNode,
+		                 RGXTIMECORR_EVENT_CLOCK_CHANGE);
+
+		PVRSRVPowerUnlock((PVRSRV_DEVICE_NODE *)psDeviceNode);
+	}
+	else
+	{
+		/* Set the new clock source without updating the time correlation
+		 * data. This is going to be accounted for during the next power up. */
+		psDevInfo->ui32ClockSource = ui32Value;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _GetClock(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 *pui32Value)
+{
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	*pui32Value =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource;
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return PVRSRV_OK;
+}
+
+void RGXTimeCorrInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_SecondaryOSClockSource,
+	                                    _GetClock, _SetClock, psDeviceNode,
+	                                   NULL);
+}
+
+/*
+	End of AppHint interface
+*/
+
+IMG_UINT64 RGXTimeCorrGetClockns64(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT64 ui64Clock;
+
+	switch (((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource) {
+		case RGXTIMECORR_CLOCK_MONO:
+			return ((void) OSClockMonotonicns64(&ui64Clock), ui64Clock);
+		case RGXTIMECORR_CLOCK_MONO_RAW:
+			return OSClockMonotonicRawns64();
+		case RGXTIMECORR_CLOCK_SCHED:
+			return OSClockns64();
+		default:
+			PVR_ASSERT(IMG_FALSE);
+			return 0;
+	}
+}
+
+IMG_UINT64 RGXTimeCorrGetClockus64(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 rem;
+	return OSDivide64r64(RGXTimeCorrGetClockns64(psDeviceNode), 1000, &rem);
+}
+
+static IMG_UINT64 RGXTimeGetDeviceTimestampInTicks(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT64 ui64DeviceTimestamp;
+
+	/* In case of powerlock acquire fails or device is powered off, return
+	 * last device power off timestamp plus ticks corresponding to device off time. */
+	ui64DeviceTimestamp = psDeviceNode->ui64LastDeviceOffTimestamp +
+			       RGXTimeCorrDeltaOSNsToDeltaCR(psDeviceNode,
+						OSClockns64() - psDeviceNode->ui64LastDeviceOffHostTimestampNs);
+
+	if (PVRSRV_OK == PVRSRVPowerLock(psDeviceNode))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+
+		if (PVRSRVIsDevicePowered(psDeviceNode))
+		{
+			ui64DeviceTimestamp = RGXReadHWTimerReg(psDevInfo) + psGpuUtilFW->i64DeviceTimestampOffset;
+		}
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return ui64DeviceTimestamp;
+}
+
+void RGXGetTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+							RGXFWIF_TIME_CORR *psTimeCorrs,
+							IMG_UINT32 ui32NumOut)
+{
+	PVRSRV_RGXDEV_INFO    *psDevInfo     = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW   *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32CurrentIndex;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32CurrentIndex = psGpuUtilFW->ui32TimeCorrSeqCount;
+
+	RGXFwSharedMemCacheOpExec(&psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32CurrentIndex - ui32NumOut)],
+	                          sizeof(psGpuUtilFW->sTimeCorr[0]) * ui32NumOut,
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+	while (ui32NumOut--)
+	{
+		*(psTimeCorrs++) = psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32CurrentIndex)];
+		ui32CurrentIndex--;
+	}
+}
+
+static __maybe_unused const IMG_CHAR* _EventToString(RGXTIMECORR_EVENT eEvent)
+{
+	switch (eEvent)
+	{
+		case RGXTIMECORR_EVENT_POWER:
+			return "power";
+		case RGXTIMECORR_EVENT_DVFS:
+			return "dvfs";
+		case RGXTIMECORR_EVENT_PERIODIC:
+			return "periodic";
+		case RGXTIMECORR_EVENT_CLOCK_CHANGE:
+			return "clock source";
+		default:
+			return "n/a";
+	}
+}
+
+static inline IMG_UINT32 _RGXGetSystemLayerGPUClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+
+	return psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+}
+
+static inline IMG_UINT32 _RGXGetEstimatedGPUClockSpeed(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_GPU_DVFS_TABLE *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+
+	psTrackingData = &psGpuDVFSTable->asTrackingData[psGpuDVFSTable->ui32FreqIndex];
+
+	return psTrackingData->ui32EstCoreClockSpeed;
+}
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+static inline void _DumpTimerCorrelationHistory(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_GPU_DVFS_TABLE *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	IMG_UINT32 i = psGpuDVFSTable->ui32HistoryIndex;
+
+	PVR_DPF((PVR_DBG_ERROR, "Dumping history of timer correlation data (latest first):"));
+
+	do
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "  Begin times: OS %" IMG_UINT64_FMTSPEC ", CR %" IMG_UINT64_FMTSPEC ", "
+				 "End times: OS %" IMG_UINT64_FMTSPEC ", CR %" IMG_UINT64_FMTSPEC ", "
+				 "Core clk %u, Estimated clk %u",
+				 psGpuDVFSTable->asTrackingHistory[i].ui64BeginOSTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64BeginCRTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64EndOSTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64EndCRTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui32CoreClockSpeed,
+				 psGpuDVFSTable->asTrackingHistory[i].ui32EstCoreClockSpeed));
+
+		i = (i - 1) % RGX_GPU_FREQ_TRACKING_SIZE;
+
+	} while (i != psGpuDVFSTable->ui32HistoryIndex);
+}
+#endif
+
+static void _RGXMakeTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32NewSeqCount;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	RGXFWIF_TIME_CORR sTimeCorr = {0};
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32NewSeqCount = psGpuUtilFW->ui32TimeCorrSeqCount + 1;
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32NewSeqCount)], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32NewSeqCount)];
+
+	/*
+	 * The following reads must be done as close together as possible, because
+	 * they represent the same current time sampled from different clock sources.
+	 */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		if (OSClockMonotonicns64(&sTimeCorr.ui64OSMonoTimeStamp) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "_RGXMakeTimeCorrData: System Monotonic Clock not available."));
+			PVR_ASSERT(0);
+		}
+	}
+
+	sTimeCorr.ui64CRTimeStamp = RGXReadHWTimerReg(psDevInfo);
+	sTimeCorr.ui64OSTimeStamp =
+		((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource == RGXTIMECORR_CLOCK_MONO
+		? sTimeCorr.ui64OSMonoTimeStamp
+		: RGXTimeCorrGetClockns64(psDeviceNode);
+	sTimeCorr.ui32CoreClockSpeed = _RGXGetEstimatedGPUClockSpeed(psDevInfo);
+	sTimeCorr.ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(sTimeCorr.ui32CoreClockSpeed);
+
+	if (sTimeCorr.ui64CRDeltaToOSDeltaKNs == 0)
+	{
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+		_DumpTimerCorrelationHistory(psDevInfo);
+#endif
+
+		/* Revert to original clock speed (error already printed) */
+		sTimeCorr.ui32CoreClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+		sTimeCorr.ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(sTimeCorr.ui32CoreClockSpeed);
+	}
+
+	OSCachedMemCopy(psTimeCorr, &sTimeCorr, sizeof(sTimeCorr));
+	/* Make sure the values are written to memory before updating the index of the current entry */
+	OSWriteMemoryBarrier(psTimeCorr);
+	RGXFwSharedMemCacheOpPtr(psTimeCorr, FLUSH);
+
+
+	/* Update the index of the current entry in the timer correlation array */
+	psGpuUtilFW->ui32TimeCorrSeqCount = ui32NewSeqCount;
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, FLUSH);
+
+
+	if (RGXTIMECORR_EVENT_POWER == eEvent && psDeviceNode->ui64LastDeviceOffTimestamp)
+	{
+		/* Capture effective timestamp offset on device power on */
+		psGpuUtilFW->i64DeviceTimestampOffset = psDeviceNode->ui64LastDeviceOffTimestamp - RGXReadHWTimerReg(psDevInfo) +
+								RGXTimeCorrDeltaOSNsToDeltaCR(psDeviceNode,
+									OSClockns64() - psDeviceNode->ui64LastDeviceOffHostTimestampNs);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "Timer correlation data (post %s event): OS %" IMG_UINT64_FMTSPEC " ns, "
+	         "CR %" IMG_UINT64_FMTSPEC ", GPU freq. %u Hz (given as %u Hz)",
+	         _EventToString(eEvent),
+	         sTimeCorr.ui64OSTimeStamp,
+	         sTimeCorr.ui64CRTimeStamp,
+	         RGXFWIF_ROUND_TO_KHZ(sTimeCorr.ui32CoreClockSpeed),
+	         _RGXGetSystemLayerGPUClockSpeed(psDeviceNode)));
+
+	/*
+	 * Don't log timing data to the HTB log after a power(-on) event.
+	 * Otherwise this will be logged before the HTB partition marker, breaking
+	 * the log sync grammar. This data will be automatically repeated when the
+	 * partition marker is written.
+	 */
+	HTBSyncScale(eEvent != RGXTIMECORR_EVENT_POWER,
+	             sTimeCorr.ui64OSTimeStamp,
+	             sTimeCorr.ui64CRTimeStamp,
+	             sTimeCorr.ui32CoreClockSpeed);
+}
+
+static void _RGXCheckTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+#if !defined(NO_HARDWARE) && !defined(VIRTUAL_PLATFORM) && defined(DEBUG)
+#define SCALING_FACTOR (10)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32Index;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	IMG_UINT64 ui64EstimatedTime, ui64CRTimeStamp, ui64OSTimeStamp;
+	IMG_UINT64 ui64CRTimeDiff, ui64OSTimeDiff;
+	IMG_INT64 i64Diff;
+	IMG_UINT32 ui32Ratio, ui32Remainder;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32Index = RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount);
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32Index], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32Index];
+
+	/*
+	 * The following reads must be done as close together as possible, because
+	 * they represent the same current time sampled from different clock sources.
+	 */
+	ui64CRTimeStamp = RGXReadHWTimerReg(psDevInfo);
+	ui64OSTimeStamp = RGXTimeCorrGetClockns64(psDeviceNode);
+
+	if ((ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp) < (1 << SCALING_FACTOR))
+	{
+		/*
+		 * Less than ~1us has passed since the timer correlation data was generated.
+		 * A time frame this short is probably not enough to get an estimate
+		 * of how good the timer correlation data was.
+		 * Skip calculations for the above reason and to avoid a division by 0 below.
+		 */
+		return;
+	}
+
+
+	/* Calculate an estimated timestamp based on the latest timer correlation data */
+	ui64CRTimeDiff = ui64CRTimeStamp - psTimeCorr->ui64CRTimeStamp;
+	ui64OSTimeDiff = RGXFWIF_GET_DELTA_OSTIME_NS(ui64CRTimeDiff,
+	                                             psTimeCorr->ui64CRDeltaToOSDeltaKNs);
+	ui64EstimatedTime = psTimeCorr->ui64OSTimeStamp + ui64OSTimeDiff;
+
+	/* Get difference between estimated timestamp and current timestamp, in ns */
+	i64Diff = ui64EstimatedTime - ui64OSTimeStamp;
+
+	/*
+	 * Calculate ratio between estimated time diff and real time diff:
+	 * ratio% : 100% = (OSestimate - OStimecorr) : (OSreal - OStimecorr)
+	 *
+	 * The operands are scaled down (approximately from ns to us) so at least
+	 * the divisor fits on 32 bit.
+	 */
+	ui32Ratio = OSDivide64(((ui64EstimatedTime - psTimeCorr->ui64OSTimeStamp) * 100ULL) >> SCALING_FACTOR,
+	                       (ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp) >> SCALING_FACTOR,
+	                       &ui32Remainder);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "Estimated timestamp check: diff %" IMG_INT64_FMTSPECd " ns over "
+	         "period %" IMG_UINT64_FMTSPEC " ns, estimated timer speed %u%%",
+	         i64Diff,
+	         ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp,
+	         ui32Ratio));
+
+	/* Warn if the estimated timestamp is not within +/- 1% of the current time */
+	if (ui32Ratio < 99 || ui32Ratio > 101)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Estimated timestamps generated in the last %" IMG_UINT64_FMTSPEC " ns "
+		         "were %s the real time (increasing at %u%% speed)",
+		         ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp,
+		         i64Diff > 0 ? "ahead of" : "behind",
+		         ui32Ratio));
+
+		/* Higher ratio == higher delta OS == higher delta CR == frequency higher than expected (and viceversa) */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Current GPU frequency %u Hz (given as %u Hz) is probably %s than expected",
+		         RGXFWIF_ROUND_TO_KHZ(psTimeCorr->ui32CoreClockSpeed),
+		         _RGXGetSystemLayerGPUClockSpeed(psDeviceNode),
+		         i64Diff > 0 ? "lower" : "higher"));
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psGpuDVFSTable);
+#endif
+}
+
+static inline IMG_UINT32 _RGXGPUFreqGetIndex(RGX_GPU_DVFS_TABLE *psGpuDVFSTable, IMG_UINT32 ui32CoreClockSpeed)
+{
+	IMG_UINT32 *paui32GPUFrequencies = psGpuDVFSTable->aui32GPUFrequency;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGX_GPU_DVFS_TABLE_SIZE; i++)
+	{
+		if (paui32GPUFrequencies[i] == ui32CoreClockSpeed)
+		{
+			return i;
+		}
+
+		if (paui32GPUFrequencies[i] == 0)
+		{
+			paui32GPUFrequencies[i] = ui32CoreClockSpeed;
+			return i;
+		}
+	}
+
+	i--;
+
+	PVR_DPF((PVR_DBG_ERROR, "GPU frequency table in the driver is full! "
+	         "Table size should be increased! Overriding last entry (%u) with %u",
+	         paui32GPUFrequencies[i], ui32CoreClockSpeed));
+
+	paui32GPUFrequencies[i] = ui32CoreClockSpeed;
+
+	return i;
+}
+
+static void _RGXGPUFreqCalibrationPeriodStart(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+	IMG_UINT32 ui32CoreClockSpeed, ui32Index;
+
+	IMG_UINT64 ui64CRTimestamp = RGXReadHWTimerReg(psDevInfo);
+	IMG_UINT64 ui64OSTimestamp = RGXTimeCorrGetClockus64(psDeviceNode);
+
+	psGpuDVFSTable->ui64CalibrationCRTimestamp = ui64CRTimestamp;
+	psGpuDVFSTable->ui64CalibrationOSTimestamp = ui64OSTimestamp;
+
+	ui32CoreClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+	ui32Index          = _RGXGPUFreqGetIndex(psGpuDVFSTable, ui32CoreClockSpeed);
+	psTrackingData     = &psGpuDVFSTable->asTrackingData[ui32Index];
+
+	/* Set the time needed to (re)calibrate the GPU frequency */
+	if (psTrackingData->ui32CalibrationCount == 0) /* We never met this frequency */
+	{
+		psTrackingData->ui32EstCoreClockSpeed = ui32CoreClockSpeed;
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US;
+	}
+	else if (psTrackingData->ui32CalibrationCount == 1) /* We calibrated this frequency only once */
+	{
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US;
+	}
+	else
+	{
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US;
+	}
+
+	/* Update the index to the DVFS table */
+	psGpuDVFSTable->ui32FreqIndex = ui32Index;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui32CoreClockSpeed    = ui32CoreClockSpeed;
+		psTrackingHistory->ui32EstCoreClockSpeed = psTrackingData->ui32EstCoreClockSpeed;
+		psTrackingHistory->ui64BeginCRTimestamp  = ui64CRTimestamp;
+		psTrackingHistory->ui64BeginOSTimestamp  = ui64OSTimestamp;
+		psTrackingHistory->ui64EndCRTimestamp    = 0ULL;
+		psTrackingHistory->ui64EndOSTimestamp    = 0ULL;
+	}
+#endif
+}
+
+static void _RGXGPUFreqCalibrationPeriodStop(PVRSRV_DEVICE_NODE *psDeviceNode,
+											 RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	IMG_UINT64 ui64CRTimestamp = RGXReadHWTimerReg(psDevInfo);
+	IMG_UINT64 ui64OSTimestamp = RGXTimeCorrGetClockus64(psDeviceNode);
+
+	psGpuDVFSTable->ui64CalibrationCRTimediff =
+	    ui64CRTimestamp - psGpuDVFSTable->ui64CalibrationCRTimestamp;
+	psGpuDVFSTable->ui64CalibrationOSTimediff =
+	    ui64OSTimestamp - psGpuDVFSTable->ui64CalibrationOSTimestamp;
+
+	/* Check if the current timer correlation data is good enough */
+	_RGXCheckTimeCorrData(psDeviceNode, psGpuDVFSTable);
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui64EndCRTimestamp = ui64CRTimestamp;
+		psTrackingHistory->ui64EndOSTimestamp = ui64OSTimestamp;
+	}
+#endif
+}
+
+static void _RGXGPUFreqCalibrationCalculate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            RGX_GPU_DVFS_TABLE *psGpuDVFSTable,
+                                            RGXTIMECORR_EVENT   eEvent)
+{
+#if !defined(DISABLE_GPU_FREQUENCY_CALIBRATION)
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+	IMG_UINT32 ui32EstCoreClockSpeed, ui32PrevCoreClockSpeed, ui32SysGPUClockSpeed;
+	IMG_INT32  i32Diff;
+	IMG_UINT32 ui32Remainder;
+
+	ui32SysGPUClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+	/*
+	 * Find out what the GPU frequency was in the last period.
+	 * This should return a value very close to the frequency passed by the system layer.
+	 */
+	ui32EstCoreClockSpeed =
+	    RGXFWIF_GET_GPU_CLOCK_FREQUENCY_HZ(psGpuDVFSTable->ui64CalibrationCRTimediff,
+	                                       psGpuDVFSTable->ui64CalibrationOSTimediff,
+	                                       ui32Remainder);
+
+	/* Update GPU frequency used by the driver for a given system layer frequency */
+	psTrackingData = &psGpuDVFSTable->asTrackingData[psGpuDVFSTable->ui32FreqIndex];
+
+	ui32PrevCoreClockSpeed = psTrackingData->ui32EstCoreClockSpeed;
+	psTrackingData->ui32EstCoreClockSpeed = ui32EstCoreClockSpeed;
+	psTrackingData->ui32CalibrationCount++;
+
+	i32Diff = (IMG_INT32) (ui32EstCoreClockSpeed - ui32PrevCoreClockSpeed);
+
+	if ((i32Diff < -1000000) || (i32Diff > 1000000))
+	{
+		/* Warn if the frequency changed by more than 1 MHz between recalculations */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "GPU frequency calibration of system layer frequency %u Hz (pre %s event): "
+		         "more than 1 MHz difference between old and new value "
+		         "(%u Hz -> %u Hz over %"  IMG_UINT64_FMTSPEC " us)",
+		         ui32SysGPUClockSpeed,
+		         _EventToString(eEvent),
+		         RGXFWIF_ROUND_TO_KHZ(ui32PrevCoreClockSpeed),
+		         RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed),
+		         psGpuDVFSTable->ui64CalibrationOSTimediff));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "GPU frequency calibration of system layer frequency %u Hz (pre %s event): "
+		         "%u Hz -> %u Hz done over %" IMG_UINT64_FMTSPEC " us",
+		         ui32SysGPUClockSpeed,
+		         _EventToString(eEvent),
+		         RGXFWIF_ROUND_TO_KHZ(ui32PrevCoreClockSpeed),
+		         RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed),
+		         psGpuDVFSTable->ui64CalibrationOSTimediff));
+	}
+
+	if (eEvent == RGXTIMECORR_EVENT_PERIODIC)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+
+		i32Diff = (IMG_INT32) (ui32EstCoreClockSpeed - ui32SysGPUClockSpeed);
+
+		/*
+		 * Notify the Firmware about unexpected frequency differences observed during
+		 * periodic frequency calibration events only.
+		 * Other events like PDVFS and power transitions are already likely to call
+		 * the pre/post clock callback directly to set frequencies as needed.
+		 * Platforms without PDVFS or APM need a method to correct the Firmware's
+		 * internal timing measurements.
+		 */
+		if (((i32Diff < -1000000) || (i32Diff > 1000000)) &&
+			(PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState) == PVRSRV_OK))
+		{
+			PVRSRV_ERROR eError = RGXPreClockSpeedChange((IMG_HANDLE)psDeviceNode, ePowerState);
+
+			PVR_LOG_IF_ERROR(eError, "RGXPreClockSpeedChange");
+			if (eError == PVRSRV_OK)
+			{
+				RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+
+				/* Update the internal core frequency variable and notify the Firmware of the change */
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed = RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed);
+
+				eError = RGXPostClockSpeedChange((IMG_HANDLE)psDeviceNode, ePowerState);
+				PVR_LOG_IF_ERROR(eError, "RGXPostClockSpeedChange");
+			}
+		}
+	}
+
+	/* Reset time deltas to avoid recalibrating the same frequency over and over again */
+	psGpuDVFSTable->ui64CalibrationCRTimediff = 0;
+	psGpuDVFSTable->ui64CalibrationOSTimediff = 0;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui32EstCoreClockSpeed = ui32EstCoreClockSpeed;
+		psGpuDVFSTable->ui32HistoryIndex =
+			(psGpuDVFSTable->ui32HistoryIndex + 1) % RGX_GPU_FREQ_TRACKING_SIZE;
+	}
+#endif
+
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psGpuDVFSTable);
+	PVR_UNREFERENCED_PARAMETER(eEvent);
+#endif
+}
+
+void RGXTimeCorrBegin(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_DEVICE_NODE  *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO  *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE  *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	_RGXGPUFreqCalibrationPeriodStart(psDeviceNode, psGpuDVFSTable);
+	_RGXMakeTimeCorrData(psDeviceNode, eEvent);
+}
+
+void RGXTimeCorrEnd(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_DEVICE_NODE  *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO  *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE  *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	_RGXGPUFreqCalibrationPeriodStop(psDeviceNode, psGpuDVFSTable);
+
+	if (psGpuDVFSTable->ui64CalibrationOSTimediff >= psGpuDVFSTable->ui32CalibrationPeriod)
+	{
+		_RGXGPUFreqCalibrationCalculate(psDeviceNode, psGpuDVFSTable, eEvent);
+	}
+
+	if (RGXTIMECORR_EVENT_POWER == eEvent)
+	{
+		RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+
+		/* Capture effective device power off timestamp */
+		psDeviceNode->ui64LastDeviceOffTimestamp = RGXReadHWTimerReg(psDevInfo) + psGpuUtilFW->i64DeviceTimestampOffset;
+		psDeviceNode->ui64LastDeviceOffHostTimestampNs = OSClockns64();
+	}
+}
+
+void RGXTimeCorrRestartPeriodic(IMG_HANDLE hDevHandle)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO     *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE     *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	IMG_UINT64             ui64TimeNow     = RGXTimeCorrGetClockus64(psDeviceNode);
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_DEFAULT;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	if (psGpuDVFSTable == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Required data not initialised yet", __func__));
+		return;
+	}
+
+	/* Check if it's the right time to recalibrate the GPU clock frequency */
+	if ((ui64TimeNow - psGpuDVFSTable->ui64CalibrationOSTimestamp) < psGpuDVFSTable->ui32CalibrationPeriod) return;
+
+	/* Try to acquire the powerlock, if not possible then don't wait */
+	if (PVRSRVPowerTryLock(psDeviceNode) != PVRSRV_OK) return;
+
+	/* If the GPU is off then we can't do anything */
+	PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+	if (ePowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return;
+	}
+
+	/* All checks passed, we can calibrate and correlate */
+	RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_PERIODIC);
+	RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_PERIODIC);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+/*
+	RGXTimeCorrGetClockSource
+*/
+RGXTIMECORR_CLOCK_TYPE RGXTimeCorrGetClockSource(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource;
+}
+
+/*
+	RGXTimeCorrSetClockSource
+*/
+PVRSRV_ERROR RGXTimeCorrSetClockSource(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       RGXTIMECORR_CLOCK_TYPE eClockType)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psRGXDevNode = psDeviceNode->pvDevice;
+	RGXTIMECORR_CLOCK_TYPE eLastClock = psRGXDevNode->ui32ClockSource;
+
+	eError = _SetClock(psDeviceNode, NULL, eClockType);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_SetClock");
+
+	psRGXDevNode->ui32LastClockSource = eLastClock;
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXCurrentTime(CONNECTION_DATA    *psConnection,
+                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT8           ui8TimestampType,
+                     IMG_UINT64         *pui64Time)
+{
+	RGX_QUERY_TIMESTAMP_TYPE eTimestampType = (RGX_QUERY_TIMESTAMP_TYPE)ui8TimestampType;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	switch (eTimestampType)
+	{
+		case RGX_QUERY_HOST_TIMESTAMP:
+		*pui64Time = RGXTimeCorrGetClockns64(psDeviceNode);
+		break;
+
+		case RGX_QUERY_DEVICE_TIMESTAMP:
+		*pui64Time = RGXTimeGetDeviceTimestampInTicks(psDeviceNode);
+		break;
+
+		default:
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		break;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxtimecorr.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.h
new file mode 100644
index 000000000000..9758e61909c8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimecorr.h
@@ -0,0 +1,316 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX time correlation and calibration header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX time correlation and calibration routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTIMECORR_H)
+#define RGXTIMECORR_H
+
+#include "img_types.h"
+#include "device.h"
+#include "osfunc.h"
+#include "connection_server.h"
+#include "rgxdevice.h"
+
+typedef enum
+{
+	RGXTIMECORR_CLOCK_MONO,
+	RGXTIMECORR_CLOCK_MONO_RAW,
+	RGXTIMECORR_CLOCK_SCHED,
+
+	RGXTIMECORR_CLOCK_LAST
+} RGXTIMECORR_CLOCK_TYPE;
+
+typedef enum
+{
+	RGXTIMECORR_EVENT_POWER,
+	RGXTIMECORR_EVENT_DVFS,
+	RGXTIMECORR_EVENT_PERIODIC,
+	RGXTIMECORR_EVENT_CLOCK_CHANGE
+} RGXTIMECORR_EVENT;
+
+/*
+ * Calibrated GPU frequencies are rounded to the nearest multiple of 1 KHz
+ * before use, to reduce the noise introduced by calculations done with
+ * imperfect operands (correlated timers not sampled at exactly the same
+ * time, GPU CR timer incrementing only once every 256 GPU cycles).
+ * This also helps reducing the variation between consecutive calculations.
+ */
+#define RGXFWIF_CONVERT_TO_KHZ(freq)   (((freq) + 500) / 1000)
+#define RGXFWIF_ROUND_TO_KHZ(freq)    ((((freq) + 500) / 1000) * 1000)
+
+/* Constants used in different calculations */
+#define SECONDS_TO_MICROSECONDS          (1000000ULL)
+#define CRTIME_TO_CYCLES_WITH_US_SCALE   (RGX_CRTIME_TICK_IN_CYCLES * SECONDS_TO_MICROSECONDS)
+
+/*
+ * Use this macro to get a more realistic GPU core clock speed than the one
+ * given by the upper layers (used when doing GPU frequency calibration)
+ */
+#define RGXFWIF_GET_GPU_CLOCK_FREQUENCY_HZ(deltacr_us, deltaos_us, remainder) \
+    OSDivide64((deltacr_us) * CRTIME_TO_CYCLES_WITH_US_SCALE, (deltaos_us), &(remainder))
+
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetConversionFactor
+
+ @Description Generate constant used to convert a GPU time difference into
+              an OS time difference (for more info see rgx_fwif_km.h).
+
+ @Input       ui32ClockSpeed : GPU clock speed
+
+ @Return      0 on failure, conversion factor otherwise
+
+******************************************************************************/
+static inline IMG_UINT64 RGXTimeCorrGetConversionFactor(IMG_UINT32 ui32ClockSpeed)
+{
+	IMG_UINT32 ui32Remainder;
+
+	if (RGXFWIF_CONVERT_TO_KHZ(ui32ClockSpeed) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: GPU clock frequency %u is too low",
+				 __func__, ui32ClockSpeed));
+
+		return 0;
+	}
+
+	return OSDivide64r64(CRTIME_TO_CYCLES_WITH_US_SCALE << RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT,
+	                     RGXFWIF_CONVERT_TO_KHZ(ui32ClockSpeed), &ui32Remainder);
+}
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrDeltaOSNsToDeltaCR
+
+ @Description Convert OS timestamp difference in ns to device tick difference
+
+ @Input       psDeviceNode, ui64DeltaOSNs
+
+ @Return      0 on failure, deltaCR otherwise
+
+******************************************************************************/
+static inline IMG_UINT64 RGXTimeCorrDeltaOSNsToDeltaCR(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64DeltaOSNs)
+{
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	IMG_UINT32 ui32Remainder;
+
+	PVR_ASSERT(psDeviceNode && psDeviceNode->pvDevice);
+	psDevInfo = psDeviceNode->pvDevice;
+	psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	PVR_ASSERT(psGpuUtilFW);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount)];
+
+	if (psTimeCorr->ui64CRDeltaToOSDeltaKNs == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui64CRDeltaToOSDeltaKNs is 0", __func__));
+		return 0;
+	}
+
+	return OSDivide64r64(ui64DeltaOSNs << RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT,
+	                     psTimeCorr->ui64CRDeltaToOSDeltaKNs, &ui32Remainder);
+}
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrBegin
+
+ @Description Generate new timer correlation data, and start tracking
+              the current GPU frequency.
+
+ @Input       hDevHandle : RGX Device Node
+ @Input       eEvent     : Event associated with the beginning of a timer
+                           correlation period
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrBegin(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrEnd
+
+ @Description Stop tracking the CPU and GPU timers, and if possible
+              recalculate the GPU frequency to a value which makes the timer
+              correlation data more accurate.
+
+ @Input       hDevHandle : RGX Device Node
+ @Input       eEvent     : Event associated with the end of a timer
+                           correlation period
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrEnd(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrRestartPeriodic
+
+ @Description Perform actions from RGXTimeCorrEnd and RGXTimeCorrBegin,
+              but only if enough time has passed since the last timer
+              correlation data was generated.
+
+ @Input       hDevHandle : RGX Device Node
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrRestartPeriodic(IMG_HANDLE hDevHandle);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockns64
+
+ @Description Returns value of currently selected clock (in ns).
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock value from currently selected clock source
+
+******************************************************************************/
+IMG_UINT64 RGXTimeCorrGetClockns64(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockus64
+
+ @Description Returns value of currently selected clock (in us).
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock value from currently selected clock source
+
+******************************************************************************/
+IMG_UINT64 RGXTimeCorrGetClockus64(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockSource
+
+ @Description Returns currently selected clock source
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock source type
+
+******************************************************************************/
+RGXTIMECORR_CLOCK_TYPE RGXTimeCorrGetClockSource(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrSetClockSource
+
+ @Description Sets clock source for correlation data.
+
+ @Input       psDeviceNode : RGX Device Node
+ @Input       eClockType : clock source type
+
+ @Return      error code
+
+******************************************************************************/
+PVRSRV_ERROR RGXTimeCorrSetClockSource(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       RGXTIMECORR_CLOCK_TYPE eClockType);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrInitAppHintCallbacks
+
+ @Description Initialise apphint callbacks for timer correlation
+              related apphints.
+
+ @Input       psDeviceNode : RGX Device Node
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXGetTimeCorrData
+
+ @Description Get a number of the most recent time correlation data points
+
+ @Input       psDeviceNode : RGX Device Node
+ @Output      psTimeCorrs  : Output array of RGXFWIF_TIME_CORR elements
+                             for data to be written to
+ @Input       ui32NumOut   : Number of elements to be written out
+
+ @Return      void
+
+******************************************************************************/
+void RGXGetTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+							RGXFWIF_TIME_CORR *psTimeCorrs,
+							IMG_UINT32 ui32NumOut);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVRGXCurrentTime
+
+ @Description Server-side implementation of RGXCurrentTime
+
+ @Input       psConnection     : Connection handle
+ @Input       psDeviceNode     : RGX Device Node
+ @Input       ui8TimestampType : Timestamp type
+ @Output      pui64Time        : Timestamp
+
+ @Return      PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXCurrentTime(CONNECTION_DATA    *psConnection,
+                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT8           ui8TimestampType,
+                     IMG_UINT64         *pui64Time);
+#endif /* RGXTIMECORR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.c
new file mode 100644
index 000000000000..82900863ed26
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.c
@@ -0,0 +1,225 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Timer queries
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Timer queries
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxtimerquery.h"
+#include "rgxdevice.h"
+#include "rgxtimecorr.h"
+
+#include "rgxfwutils.h"
+#include "pdump_km.h"
+
+PVRSRV_ERROR
+PVRSRVRGXBeginTimerQueryKM(CONNECTION_DATA    * psConnection,
+                           PVRSRV_DEVICE_NODE * psDeviceNode,
+                           IMG_UINT32         ui32QueryId)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32QueryId >= RGX_MAX_TIMER_QUERIES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	psDevInfo->bSaveStart = IMG_TRUE;
+	psDevInfo->bSaveEnd   = IMG_TRUE;
+
+	/* clear the stamps, in case there is no Kick */
+	psDevInfo->pui64StartTimeById[ui32QueryId] = 0UL;
+	psDevInfo->pui64EndTimeById[ui32QueryId]   = 0UL;
+	OSWriteMemoryBarrier(&psDevInfo->pui64EndTimeById[ui32QueryId]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui64StartTimeById[ui32QueryId], FLUSH);
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui64EndTimeById[ui32QueryId], FLUSH);
+
+	/* save of the active query index */
+	psDevInfo->ui32ActiveQueryId = ui32QueryId;
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXEndTimerQueryKM(CONNECTION_DATA    * psConnection,
+                         PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	/* clear off the flags set by Begin(). Note that _START_TIME is
+	 * probably already cleared by Kick()
+	 */
+	psDevInfo->bSaveStart = IMG_FALSE;
+	psDevInfo->bSaveEnd   = IMG_FALSE;
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXQueryTimerKM(CONNECTION_DATA    * psConnection,
+                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                      IMG_UINT32         ui32QueryId,
+                      IMG_UINT64         * pui64StartTime,
+                      IMG_UINT64         * pui64EndTime)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT32         ui32Scheduled;
+	IMG_UINT32         ui32Completed;
+	PVRSRV_ERROR       eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32QueryId >= RGX_MAX_TIMER_QUERIES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	ui32Scheduled = psDevInfo->aui32ScheduledOnId[ui32QueryId];
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32CompletedById[ui32QueryId], INVALIDATE);
+	ui32Completed = psDevInfo->pui32CompletedById[ui32QueryId];
+
+	/* if there was no kick since the Begin() on this id we return 0-s as Begin cleared
+	 * the stamps. If there was no begin the returned data is undefined - but still
+	 * safe from services pov
+	 */
+	if (ui32Completed >= ui32Scheduled)
+	{
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui64StartTimeById[ui32QueryId], INVALIDATE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui64EndTimeById[ui32QueryId], INVALIDATE);
+		* pui64StartTime = psDevInfo->pui64StartTimeById[ui32QueryId];
+		* pui64EndTime   = psDevInfo->pui64EndTimeById[ui32QueryId];
+
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+	return eError;
+}
+
+
+
+/******************************************************************************
+ NOT BRIDGED/EXPORTED FUNCS
+******************************************************************************/
+
+void
+RGX_GetTimestampCmdHelper(PVRSRV_RGXDEV_INFO      * psDevInfo,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPreAddr,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPostAddr,
+                          PRGXFWIF_UFO_ADDR       * ppUpdate)
+{
+	if (ppPreAddr != NULL)
+	{
+		if (psDevInfo->bSaveStart)
+		{
+			/* drop the SaveStart on the first Kick */
+			psDevInfo->bSaveStart = IMG_FALSE;
+
+			RGXSetFirmwareAddress(ppPreAddr,
+			                      psDevInfo->psStartTimeMemDesc,
+			                      sizeof(IMG_UINT64) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+		}
+		else
+		{
+			ppPreAddr->ui32Addr = 0;
+		}
+	}
+
+	if (ppPostAddr != NULL && ppUpdate != NULL)
+	{
+		if (psDevInfo->bSaveEnd)
+		{
+			RGXSetFirmwareAddress(ppPostAddr,
+			                      psDevInfo->psEndTimeMemDesc,
+			                      sizeof(IMG_UINT64) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+
+			psDevInfo->aui32ScheduledOnId[psDevInfo->ui32ActiveQueryId]++;
+
+			RGXSetFirmwareAddress(ppUpdate,
+			                      psDevInfo->psCompletedMemDesc,
+			                      sizeof(IMG_UINT32) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+		}
+		else
+		{
+			ppUpdate->ui32Addr   = 0;
+			ppPostAddr->ui32Addr = 0;
+		}
+	}
+}
+
+
+/******************************************************************************
+ End of file (rgxtimerquery.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.h
new file mode 100644
index 000000000000..7a43e7966f88
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxtimerquery.h
@@ -0,0 +1,117 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Timer queries
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Timer queries functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_TIMERQUERIES_H)
+#define RGX_TIMERQUERIES_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "rgxdevice.h"
+
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXBeginTimerQueryKM
+@Description    Opens a new timer query.
+
+@Input          ui32QueryId an identifier between [ 0 and RGX_MAX_TIMER_QUERIES - 1 ]
+@Return         PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXBeginTimerQueryKM(CONNECTION_DATA    * psConnection,
+                           PVRSRV_DEVICE_NODE * psDeviceNode,
+                           IMG_UINT32         ui32QueryId);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXEndTimerQueryKM
+@Description    Closes a timer query
+
+                The lack of ui32QueryId argument expresses the fact that there
+                can't be overlapping queries open.
+@Return         PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXEndTimerQueryKM(CONNECTION_DATA    * psConnection,
+                         PVRSRV_DEVICE_NODE * psDeviceNode);
+
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXQueryTimerKM
+@Description    Queries the state of the specified timer
+
+@Input          ui32QueryId an identifier between [ 0 and RGX_MAX_TIMER_QUERIES - 1 ]
+@Out            pui64StartTime
+@Out            pui64EndTime
+@Return         PVRSRV_OK                         on success.
+                PVRSRV_ERROR_RESOURCE_UNAVAILABLE if the device is still busy with
+                                                  operations from the queried period
+                other error code                  otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXQueryTimerKM(CONNECTION_DATA    * psConnection,
+                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                      IMG_UINT32         ui32QueryId,
+                      IMG_UINT64         * pui64StartTime,
+                      IMG_UINT64         * pui64EndTime);
+
+
+
+/******************************************************************************
+ NON BRIDGED/EXPORTED interface
+******************************************************************************/
+
+/* get the relevant data from the Kick to the helper*/
+void
+RGX_GetTimestampCmdHelper(PVRSRV_RGXDEV_INFO      * psDevInfo,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPreAddr,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPostAddr,
+                          PRGXFWIF_UFO_ADDR       * ppUpdate);
+
+#endif /* RGX_TIMERQUERIES_H */
+
+/******************************************************************************
+ End of file (rgxtimerquery.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.c
new file mode 100644
index 000000000000..09fb841e0e60
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.c
@@ -0,0 +1,550 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "power.h"
+#include "pvrsrv.h"
+#include "sync_internal.h"
+#include "rgxfwutils.h"
+#include "rgxlayer.h"
+#include "rgxmmudefs_km.h"
+#include "rgxta3d.h"
+
+PVRSRV_ERROR RGXQueryAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 *pui32State)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psDevInfo = psDeviceNode->pvDevice;
+	*pui32State = psDevInfo->eActivePMConf;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 ui32State)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if !defined(NO_HARDWARE)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (RGX_ACTIVEPM_FORCE_OFF != ui32State)
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+#if !defined(NO_HARDWARE)
+	psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pvAPMISRData)
+	{
+		psDevInfo->eActivePMConf = RGX_ACTIVEPM_FORCE_OFF;
+		psDevInfo->pvAPMISRData = NULL;
+		eError = PVRSRVSetDeviceDefaultPowerState((PPVRSRV_DEVICE_NODE)psDeviceNode,
+		                                          PVRSRV_DEV_POWER_STATE_ON);
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXQueryPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL *pbDisabled)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	*pbDisabled = !psDevInfo->bPDPEnabled;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL bDisable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	psDevInfo->bPDPEnabled = !bDisable;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 *pui32DeviceFlags)
+{
+	if (!pui32DeviceFlags || !psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui32DeviceFlags = psDevInfo->ui32DeviceFlags;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_BOOL bSetNotClear)
+{
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((ui32Config & ~RGXKM_DEVICE_STATE_MASK) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Bits outside of device state mask set (input: 0x%x, mask: 0x%x)",
+				 __func__, ui32Config, RGXKM_DEVICE_STATE_MASK));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (bSetNotClear)
+	{
+		psDevInfo->ui32DeviceFlags |= ui32Config;
+	}
+	else
+	{
+		psDevInfo->ui32DeviceFlags &= ~ui32Config;
+	}
+
+	return PVRSRV_OK;
+}
+
+inline const char * RGXStringifyKickTypeDM(RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	PVR_ASSERT(eKickTypeDM < RGX_KICK_TYPE_DM_LAST);
+
+	switch (eKickTypeDM) {
+		case RGX_KICK_TYPE_DM_GP:
+			return "GP ";
+		case RGX_KICK_TYPE_DM_TDM_2D:
+			return "TDM/2D ";
+		case RGX_KICK_TYPE_DM_TA:
+			return "TA ";
+		case RGX_KICK_TYPE_DM_3D:
+			return "3D ";
+		case RGX_KICK_TYPE_DM_CDM:
+			return "CDM ";
+		case RGX_KICK_TYPE_DM_RTU:
+			return "RTU ";
+		case RGX_KICK_TYPE_DM_SHG:
+			return "SHG ";
+		case RGX_KICK_TYPE_DM_TQ2D:
+			return "TQ2D ";
+		case RGX_KICK_TYPE_DM_TQ3D:
+			return "TQ3D ";
+		default:
+			return "Invalid DM ";
+	}
+}
+
+PHYS_HEAP_POLICY RGXPhysHeapGetLMAPolicy(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP_POLICY ui32Policy;
+
+	if (OSIsMapPhysNonContigSupported())
+	{
+		ui32Policy = PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG;
+
+		if (BITMASK_ANY(ui32UsageFlags,
+			(PHYS_HEAP_USAGE_FW_SHARED    |
+			 PHYS_HEAP_USAGE_FW_PRIVATE   |
+			 PHYS_HEAP_USAGE_FW_PREMAP_PT |
+			 PHYS_HEAP_USAGE_FW_CODE      |
+			 PHYS_HEAP_USAGE_FW_PRIV_DATA)))
+		{
+			if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				/* Guest Firmware heaps are always premapped */
+				ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+			}
+#if defined(RGX_PREMAP_FW_HEAPS)
+			else if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+			{
+				/* All Firmware heaps are premapped under AutoVz*/
+				ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+			}
+#endif
+		}
+
+		if (BITMASK_ANY(ui32UsageFlags, PHYS_HEAP_USAGE_FW_PREMAP))
+		{
+			ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+		}
+	}
+	else
+	{
+		ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+	}
+
+	return ui32Policy;
+}
+
+IMG_BOOL RGXIsErrorAndDeviceRecoverable(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_ERROR *peError)
+{
+	IMG_BOOL bRecoverable = IMG_TRUE;
+
+	if (*peError == PVRSRV_OK)
+	{
+		/* No recovery required */
+		return IMG_FALSE;
+	}
+
+	if (!PVRSRVIsStatusRecoverable(OSAtomicRead(&psDeviceNode->eHealthStatus)))
+	{
+		bRecoverable = IMG_FALSE;
+	}
+	else
+	{
+		RGXUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+
+		if (!PVRSRVIsStatusRecoverable(OSAtomicRead(&psDeviceNode->eHealthStatus)))
+		{
+			bRecoverable = IMG_FALSE;
+		}
+	}
+
+	if (bRecoverable && !PVRSRVIsRetryError(*peError))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device is recoverable. Changing error type (%s) to retry.",
+				 __func__, PVRSRVGetErrorString(*peError)));
+		*peError = PVRSRV_ERROR_RETRY;
+	}
+
+	if (!bRecoverable && PVRSRVIsRetryError(*peError))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device is not recoverable. Error type should not be retry (%s).",
+				 __func__, PVRSRVGetErrorString(*peError)));
+		*peError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return bRecoverable;
+}
+
+/*
+ *  Function that returns the MList Size required for a given max PB size.
+ *
+ *  The maximum MList size required always depends on the maximum PB Size
+ *  chosen and must also take into account the additional pages that will
+ *  be provided by a local PB.
+ */
+IMG_UINT32 RGXCalcMListSize(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT64 ui64MaxLocalPBSize,
+                            IMG_UINT64 ui64MaxGlobalPBSize)
+{
+	IMG_UINT32  ui32PTEPages = 0, ui32PDEPages = 0, ui32PCEPages = 0, ui32MListSize = 0;
+	IMG_UINT32  ui32NumOfPipes = 1;
+	IMG_UINT64  ui64TotalPages = 0;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	/*
+	 *  Assert if Size of PB exceeds maximum theoretical limit
+	 *  RGX_PM_MAX_PB_VIRT_ADDR_SPACE represents the 16G address space#
+	 */
+	PVR_ASSERT(ui64MaxLocalPBSize+ui64MaxGlobalPBSize <= RGX_PM_MAX_PB_VIRT_ADDR_SPACE);
+
+	/* Calculate the total number of pages which is the number of Page table entries */
+	ui64TotalPages = ((ui64MaxLocalPBSize+ui64MaxGlobalPBSize)/RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	/* Calculate the total number of pages required for the PTE's (minimum of 1) */
+	ui32PTEPages = (IMG_UINT32)(ui64TotalPages/RGX_MMUCTRL_ENTRIES_PT_VALUE);
+	if (ui32PTEPages == 0U)
+	{
+		ui32PTEPages = 1;
+	}
+
+	/* Calculate the total number of pages required to hold the PDE's (minimum of 1) */
+	ui32PDEPages = ui32PTEPages/RGX_MMUCTRL_ENTRIES_PD_VALUE;
+	if (ui32PDEPages == 0U)
+	{
+		ui32PDEPages = 1;
+	}
+
+	/* Calculate the total number of pages required to hold the PCE's (minimum of 1) */
+	ui32PCEPages = ui32PDEPages/RGX_MMUCTRL_ENTRIES_PC_VALUE;
+	if (ui32PCEPages == 0U)
+	{
+		ui32PCEPages = 1;
+	}
+
+	/* Calculate the maximum number of TA/VCE pipes */
+#if defined(RGX_FEATURE_SCALABLE_TE_ARCH_IDX)
+	{
+		IMG_UINT32 ui32Val = RGX_GET_FEATURE_VALUE(psDevInfo, RGX_FEATURE_SCALABLE_TE_ARCH);
+		if (ui32Val > ui32NumOfPipes)
+		{
+			ui32NumOfPipes = ui32Val;
+		}
+	}
+#endif
+
+#if defined(RGX_FEATURE_SCALABLE_TE_ARCH_IDX)
+	{
+		IMG_UINT32 ui32Val = RGX_GET_FEATURE_VALUE(psDevInfo, RGX_FEATURE_SCALABLE_VCE);
+		if (ui32Val > ui32NumOfPipes)
+		{
+			ui32NumOfPipes = ui32Val;
+		}
+	}
+#endif
+
+	/*
+	 *  Calculate the MList size considering the total number of pages in PB are shared
+	 *  among all the PM address spaces...
+	 */
+	ui32MListSize = (ui32PCEPages + ui32PDEPages + ui32PTEPages) *
+					RGX_NUM_PM_ADDR_SPACES * ui32NumOfPipes * RGX_MLIST_ENTRY_STRIDE;
+
+	/* Round it off to the nearest page granularity */
+	ui32MListSize = PVR_ALIGN(ui32MListSize, RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	return ui32MListSize;
+}
+
+/*
+ * Critical PMRs are PMRs that are created by client that might contain physical page addresses.
+ * We need to validate if they were allocated with proper flags.
+ */
+PVRSRV_ERROR ValidateCriticalPMR(PMR* psPMR, IMG_DEVMEM_SIZE_T ui64MinSize)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	IMG_BOOL bCPUCacheSnoop =
+		(PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		 psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY);
+
+	PMR_FLAGS_T uiFlags = PMR_Flags(psPMR);
+
+	/* Critical PMR cannot be user CPU mappable */
+	if (PVRSRV_CHECK_CPU_READABLE(uiFlags) ||
+	    PVRSRV_CHECK_CPU_WRITEABLE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Critical PMR allows CPU mapping (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	/* Critical PMR must not be user CPU cacheable (unless snooping is on) */
+	if (!bCPUCacheSnoop &&
+	    (PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags) ||
+	     PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+	     PVRSRV_CHECK_CPU_CACHED(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Critical PMR allows CPU caching (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	/* Critical PMRs must be allocated with PMMETA_PROTECT */
+	if ((uiFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Critical PMR must have PMMETA_PROTECT set",
+		        __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Critical PMR must not have OS_LINUX_MOVABLE set",
+		        __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+#endif
+
+	if (PMR_LogicalSize(psPMR) < ui64MinSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Critical PMR doesn't have sufficient size",
+		         __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	return PVRSRV_OK;
+return_error:
+	return eError;
+}
+
+/* Check if all global freelists have the same size and if all local freelists have the same size.*/
+PVRSRV_ERROR ValidateFreeListSizes(RGX_FREELIST* apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                   IMG_UINT32*   pui32LocalFLMaxPages,
+                                   IMG_UINT32*   pui32GlobalFLMaxPages)
+{
+	IMG_UINT32 i,j;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32GlobalFLMaxPages = apsFreeLists[RGXFW_GLOBAL_FREELIST]->ui32MaxFLPages;
+	IMG_UINT32 ui32LocalFLMaxPages = apsFreeLists[RGXFW_LOCAL_FREELIST]->ui32MaxFLPages;
+	IMG_UINT32 ui32NumFLPerGD = RGXMKIF_NUM_RTDATA_FREELISTS/RGXMKIF_NUM_GEOMDATAS;
+
+	for (i=0; i<RGXMKIF_NUM_GEOMDATAS; i++)
+	{
+		/* Check if all local freelists have the same size */
+		if (apsFreeLists[ui32NumFLPerGD * i + RGXFW_LOCAL_FREELIST]->ui32MaxFLPages != ui32LocalFLMaxPages)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Check if all global freelists have the same size */
+		for (j=RGXFW_GLOBAL_FREELIST; j<ui32NumFLPerGD; j++)
+		{
+			if (apsFreeLists[ui32NumFLPerGD * i + j]->ui32MaxFLPages != ui32GlobalFLMaxPages)
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+		}
+	}
+
+	*pui32LocalFLMaxPages = ui32LocalFLMaxPages;
+	*pui32GlobalFLMaxPages = ui32GlobalFLMaxPages;
+
+	return eError;
+}
+
+PVRSRV_ERROR
+AcquireValidateRefCriticalBuffer(PVRSRV_DEVICE_NODE*     psDevNode,
+                                 DEVMEMINT_RESERVATION*  psReservation,
+                                 IMG_DEVMEM_SIZE_T       ui64MinSize,
+                                 PMR**                   ppsPMR,
+                                 IMG_DEV_VIRTADDR*       psDevVAddr)
+{
+	PVRSRV_ERROR eError;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psReservation))
+	{
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		PVR_LOG_GOTO_IF_ERROR_VA(eError, ReturnError,
+		    "%s: Failed to acquire reservation for critical buffer", __func__);
+	}
+
+	eError = DevmemIntGetReservationData(psReservation, ppsPMR, psDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR_VA(eError, RollbackReservation,
+	    "%s: Error from DevmemIntGetReservationData for critical buffer: %s",
+	    __func__, PVRSRVGetErrorString(eError));
+
+
+	/* Check buffer sizes and flags are as required */
+	eError = ValidateCriticalPMR(*ppsPMR, ui64MinSize);
+	PVR_LOG_GOTO_IF_ERROR_VA(eError, RollbackReservation,
+	    "%s: Validation of critical PMR failed: %s",
+	    __func__, PVRSRVGetErrorString(eError));
+
+	/* If no error on validation ref the PMR */
+	(void) PMRRefPMR(*ppsPMR);
+
+	return PVRSRV_OK;
+
+RollbackReservation:
+	DevmemIntReservationRelease(psReservation);
+ReturnError:
+	return eError;
+}
+
+void UnrefAndReleaseCriticalBuffer(DEVMEMINT_RESERVATION* psReservation)
+{
+	PVRSRV_ERROR eError;
+	PMR* psPMR;
+	IMG_DEV_VIRTADDR sDummy;
+	/* Skip error check. If this function is called it means we already
+	   Acquired a reservation and confirmed that mapping exists. */
+	eError = DevmemIntGetReservationData(psReservation, &psPMR, &sDummy);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError,
+	    "Error when trying to obtain reservation data in %s", __func__);
+
+	eError = PMRUnrefPMR(psPMR);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError,
+	    "Error on PMR unref in %s", __func__);
+
+	DevmemIntReservationRelease(psReservation);
+}
+
+/******************************************************************************
+ End of file (rgxutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.h
new file mode 100644
index 000000000000..cf5046433826
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxutils.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific utility routines declarations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Inline functions/structures specific to RGX
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_error.h"
+#include "rgxta3d.h"
+
+/*!
+******************************************************************************
+
+ @Function      RGXQueryAPMState
+
+ @Description   Query the state of the APM configuration
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Output        pui32State   : The APM configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXQueryAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 *pui32State);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetAPMState
+
+ @Description   Set the APM configuration state. Currently only 'OFF' is
+                supported
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         ui32State    : The requested APM configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 ui32State);
+
+/*!
+******************************************************************************
+
+ @Function      RGXQueryPdumpPanicDisable
+
+ @Description   Get the PDump Panic Enable configuration state.
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         pbDisabled    : IMG_TRUE if PDump Panic is disabled
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXQueryPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL *pbDisabled);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetPdumpPanicDisable
+
+ @Description   Set the PDump Panic Enable flag
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         bDisable      : The requested configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL bDisable);
+
+/*!
+******************************************************************************
+
+ @Function      RGXGetDeviceFlags
+
+ @Description   Get the device flags for a given device
+
+ @Input         psDevInfo        : The device descriptor query
+
+ @Output        pui32DeviceFlags : The current state of the device flags
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXGetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 *pui32DeviceFlags);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetDeviceFlags
+
+ @Description   Set the device flags for a given device
+
+ @Input         psDevInfo : The device descriptor to modify
+
+ @Input         ui32Config : The device flags to modify
+
+ @Input         bSetNotClear : Set or clear the specified flags
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_BOOL bSetNotClear);
+
+/*!
+******************************************************************************
+
+ @Function    RGXStringifyKickTypeDM
+
+ @Description Gives the kick type DM name stringified
+
+ @Input       Kick type DM
+
+ @Return      Array containing the kick type DM name
+
+******************************************************************************/
+const char* RGXStringifyKickTypeDM(RGX_KICK_TYPE_DM eKickTypeDM);
+
+/*************************************************************************/ /*!
+
+@Function       RGXPhysHeapGetLMAPolicy
+
+@Description    Returns the optimal LMA allocation policy based on a heap's
+                usage flags
+
+@Input          ui32UsageFlags Flags specifying a heap's intended use
+@Input          psDeviceNode The device node.
+
+@Return         PHYS_HEAP_POLICY The recommended LMA policy
+
+*/ /**************************************************************************/
+PHYS_HEAP_POLICY RGXPhysHeapGetLMAPolicy(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#define RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(bitmask, eKickTypeDM) bitmask & eKickTypeDM ? RGXStringifyKickTypeDM(eKickTypeDM) : ""
+
+/*************************************************************************/ /*!
+@Function       RGXIsErrorAndDeviceRecoverable
+@Description    This function is used to check if device (and firmware) is in
+                a state that can be recovered from without a full reset of the
+                device.
+@Input          psDeviceNode The device node.
+@Input          peError      Pointer to error. Can be changed to retry type.
+@Return         IMG_BOOL   Return true if device is recoverable.
+*/ /**************************************************************************/
+IMG_BOOL RGXIsErrorAndDeviceRecoverable(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_ERROR *peError);
+
+/*
+ * To avoid repeated calls and avoid double frees, the error value is set to PVRSRV_OK
+ * if RGXIsErrorAndDeviceRecoverable is false.
+ */
+#define RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode, eError, cleanupFunc) \
+	do \
+	{ \
+		if (RGXIsErrorAndDeviceRecoverable(psDeviceNode, &eError)) \
+		{ \
+			return eError; \
+		} \
+		else if (eError != PVRSRV_OK) \
+		{ \
+			PVR_LOG(("%s: Unexpected error from " #cleanupFunc "(%s)", \
+					__func__, \
+					PVRSRVGetErrorString(eError))); \
+			/* Device is dead. \
+			 * Change error type to make callers destroy the resource handle. \
+			 * This is to prevent repeated calls to this function. \
+			 */ \
+			eError = PVRSRV_OK; \
+		} \
+	} while (false)
+
+/*************************************************************************/ /*!
+@Function       RGXCalcMListSize
+@Description    Function that calculates the MList Size required for
+                given local and global PB sizes.
+@Input          psDeviceNode The device node.
+@Input          ui64MaxLocalPBSize Maximum local PB size in bytes
+@Input          ui64MaxGlobalPBSize Maximum global PB size in bytes
+
+@Return         IMG_UINT32 Returns size of the mlist in bytes aligned to
+                RGX_BIF_PM_PHYSICAL_PAGE_SIZE.
+*/ /**************************************************************************/
+IMG_UINT32 RGXCalcMListSize(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT64 ui64MaxLocalPBSize,
+                            IMG_UINT64 ui64MaxGlobalPBSize);
+
+/*************************************************************************/ /*!
+@Function       ValidateCriticalPMR
+@Description    Validate if critical PMR has proper flags and size.
+@Input          psPMR Pointer to PMR to validate.
+@Input          ui64MinSize Minimum size that PMR needs to have
+
+@Return         PVRSRV_ERROR PVRSRV_OK if validation successful.
+                Appropriate error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR ValidateCriticalPMR(PMR* psPMR, IMG_DEVMEM_SIZE_T ui64MinSize);
+
+/*************************************************************************/ /*!
+@Function       ValidateFreeListSizes
+@Description    Helper function for RGXCreateHWRTDataSet
+                For the freelist array passed to RGXCreateHWRTDataSet, validate
+                if all global freelists have the same size and if all local
+                freelists have the same size. Return the sizes in output params.
+
+@Output         pui32LocalFLMaxPages Max number of pages for local freelist
+@Output         pui32GlobalFLMaxPages Max number of pages for global freelist
+
+@Return         PVRSRV_ERROR PVRSRV_OK if validation successful.
+                Appropriate error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR ValidateFreeListSizes(RGX_FREELIST* apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                   IMG_UINT32*   pui32LocalFLMaxPages,
+                                   IMG_UINT32*   pui32GlobalFLMaxPages);
+
+/*************************************************************************/ /*!
+@Function       AcquireValidateRefCriticalBuffer
+@Description    Helper function for RGXCreateHWRTDataSet
+                Acquire the reservation validate if the underlying PMR is
+                appropriate for use as critical buffer and ref it.
+@Input          psDevNode The device node.
+@Input          psReservation The reservation describing the critical buffer
+@Input          ui64MinSize Minimum size that buffer needs to have
+@Output         ppsPMR Pointer to be written with the PMR on success.
+@Output         psDevVAddr The device vaddress to be written on success.
+
+@Return         PVRSRV_ERROR PVRSRV_OK if validation successful.
+                Appropriate error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR AcquireValidateRefCriticalBuffer(PVRSRV_DEVICE_NODE*     psDevNode,
+                                              DEVMEMINT_RESERVATION*  psReservation,
+                                              IMG_DEVMEM_SIZE_T       ui64MinSize,
+                                              PMR**                   ppsPMR,
+                                              IMG_DEV_VIRTADDR*       psDevVAddr);
+
+
+/*************************************************************************/ /*!
+@Function       UnrefAndReleaseCriticalBuffer
+@Description    Helper function for RGXCreateHWRTDataSet
+                Unref the critical buffer and release the reservation object.
+@Input          psReservation The reservation describing the critical buffer
+
+*/ /**************************************************************************/
+void UnrefAndReleaseCriticalBuffer(DEVMEMINT_RESERVATION* psReservation);
+
+/******************************************************************************
+ End of file (rgxutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.c
new file mode 100644
index 000000000000..1143910295c5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.c
@@ -0,0 +1,673 @@
+/*************************************************************************/ /*!
+@File           rgxworkest.c
+@Title          RGX Workload Estimation Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxworkest.h"
+#include "rgxfwutils.h"
+#include "rgxpdvfs.h"
+#include "rgx_options.h"
+#include "device.h"
+#include "hash.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_SOC_TIMER)
+#include "rgxtimecorr.h"
+#endif
+
+#define ROUND_DOWN_TO_NEAREST_1024(number) (((number) >> 10) << 10)
+
+static inline IMG_BOOL _WorkEstEnabled(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData->sDriverInfo.sKMBuildInfo.ui32BuildOptions &
+	    psPVRSRVData->sDriverInfo.sUMBuildInfo.ui32BuildOptions &
+	    OPTIONS_WORKLOAD_ESTIMATION_EN)
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+inline IMG_UINT32 _WorkEstDoHash(IMG_UINT32 ui32Input)
+{
+	IMG_UINT32 ui32HashPart;
+
+	/* Hash function borrowed from hash.c */
+	ui32HashPart = ui32Input;
+	ui32HashPart += (ui32HashPart << 12);
+	ui32HashPart ^= (ui32HashPart >> 22);
+	ui32HashPart += (ui32HashPart << 4);
+	ui32HashPart ^= (ui32HashPart >> 9);
+	ui32HashPart += (ui32HashPart << 10);
+	ui32HashPart ^= (ui32HashPart >> 2);
+	ui32HashPart += (ui32HashPart << 7);
+	ui32HashPart ^= (ui32HashPart >> 12);
+
+	return ui32HashPart;
+}
+
+/*! Hash functions for TA/3D workload estimation */
+IMG_BOOL WorkEstHashCompareTA3D(size_t uKeySize, void *pKey1, void *pKey2);
+IMG_UINT32 WorkEstHashFuncTA3D(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*! Hash functions for compute workload estimation */
+IMG_BOOL WorkEstHashCompareCompute(size_t uKeySize, void *pKey1, void *pKey2);
+IMG_UINT32 WorkEstHashFuncCompute(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*! Hash functions for TDM/transfer workload estimation */
+IMG_BOOL WorkEstHashCompareTDM(size_t uKeySize, void *pKey1, void *pKey2);
+IMG_UINT32 WorkEstHashFuncTDM(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+
+IMG_BOOL WorkEstHashCompareTA3D(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sTA3D.ui32RenderTargetSize  == psWorkload2->sTA3D.ui32RenderTargetSize &&
+		    psWorkload1->sTA3D.ui32NumberOfDrawCalls == psWorkload2->sTA3D.ui32NumberOfDrawCalls &&
+		    psWorkload1->sTA3D.ui32NumberOfIndices   == psWorkload2->sTA3D.ui32NumberOfIndices &&
+		    psWorkload1->sTA3D.ui32NumberOfMRTs      == psWorkload2->sTA3D.ui32NumberOfMRTs)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_UINT32 WorkEstHashFuncTA3D(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on multiple render target attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32RenderTargetSize);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32NumberOfDrawCalls);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32NumberOfIndices);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTA3D.ui32NumberOfMRTs);
+
+	return ui32HashKey;
+}
+
+IMG_BOOL WorkEstHashCompareCompute(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sCompute.ui32NumberOfWorkgroups == psWorkload2->sCompute.ui32NumberOfWorkgroups &&
+		    psWorkload1->sCompute.ui32NumberOfWorkitems  == psWorkload2->sCompute.ui32NumberOfWorkitems)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_UINT32 WorkEstHashFuncCompute(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on multiple render target attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sCompute.ui32NumberOfWorkgroups);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sCompute.ui32NumberOfWorkitems);
+	return ui32HashKey;
+}
+
+IMG_BOOL WorkEstHashCompareTDM(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sTransfer.ui32Characteristic1 == psWorkload2->sTransfer.ui32Characteristic1 &&
+		    psWorkload1->sTransfer.ui32Characteristic2 == psWorkload2->sTransfer.ui32Characteristic2)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_UINT32 WorkEstHashFuncTDM(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on transfer src/dest attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTransfer.ui32Characteristic1);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sTransfer.ui32Characteristic2);
+
+	return ui32HashKey;
+}
+
+void WorkEstHashLockCreate(POS_LOCK *ppsHashLock)
+{
+	if (*ppsHashLock == NULL)
+	{
+		OSLockCreate(ppsHashLock);
+	}
+}
+
+void WorkEstHashLockDestroy(POS_LOCK psWorkEstHashLock)
+{
+	if (psWorkEstHashLock != NULL)
+	{
+		OSLockDestroy(psWorkEstHashLock);
+		psWorkEstHashLock = NULL;
+	}
+}
+
+void WorkEstCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_WORKEST_FWCCB_CMD *psFwCCBCmd;
+	IMG_UINT8 *psFWCCB = psDevInfo->psWorkEstFirmwareCCB;
+	RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psWorkEstFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psWorkEstFirmwareCCBCtlLocal;
+
+	RGXFwSharedMemCacheOpPtr(psFWCCBCtl, INVALIDATE);
+	while (psFWCCBCtlLocal->ui32ReadOffset != psFWCCBCtl->ui32WriteOffset)
+	{
+		PVRSRV_ERROR eError;
+
+		/* Point to the next command */
+		psFwCCBCmd = (RGXFWIF_WORKEST_FWCCB_CMD *)((uintptr_t)psFWCCB + psFWCCBCtlLocal->ui32ReadOffset * sizeof(RGXFWIF_WORKEST_FWCCB_CMD));
+
+		eError = WorkEstRetire(psDevInfo, psFwCCBCmd);
+		PVR_LOG_IF_ERROR(eError, "WorkEstCheckFirmwareCCB: WorkEstRetire failed");
+
+		/* Update read offset */
+		psFWCCBCtlLocal->ui32ReadOffset = (psFWCCBCtlLocal->ui32ReadOffset + 1) & psFWCCBCtlLocal->ui32WrapMask;
+		psFWCCBCtl->ui32ReadOffset = psFWCCBCtlLocal->ui32ReadOffset;
+	}
+	RGXFwSharedMemCacheOpValue(psFWCCBCtl->ui32ReadOffset, FLUSH);
+}
+
+PVRSRV_ERROR WorkEstPrepare(PVRSRV_RGXDEV_INFO        *psDevInfo,
+                            WORKEST_HOST_DATA         *psWorkEstHostData,
+                            WORKLOAD_MATCHING_DATA    *psWorkloadMatchingData,
+                            const RGXFWIF_CCB_CMD_TYPE eDMCmdType,
+                            const RGX_WORKLOAD        *psWorkloadCharsIn,
+                            IMG_UINT64                ui64DeadlineInus,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData)
+{
+	RGX_WORKLOAD          *psWorkloadCharacteristics;
+	IMG_UINT64            *pui64CyclePrediction;
+	IMG_UINT64            ui64CurrentTime;
+	WORKEST_RETURN_DATA   *psReturnData;
+	IMG_UINT32            ui32ReturnDataWO;
+#if defined(SUPPORT_SOC_TIMER)
+	PVRSRV_DEVICE_CONFIG  *psDevConfig;
+	IMG_UINT64            ui64CurrentSoCTime;
+#endif
+	PVRSRV_ERROR          eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (!_WorkEstEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	if (eDMCmdType == RGXFWIF_CCB_CMD_TYPE_NULL)
+	{
+		/* No workload, only fence updates */
+		return PVRSRV_OK;
+	}
+
+#if !defined(PVRSRV_NEED_PVR_DPF)
+	PVR_UNREFERENCED_PARAMETER(eDMCmdType);
+#endif
+
+	/* Validate all required objects required for preparing work estimation */
+	PVR_LOG_RETURN_IF_FALSE(psDevInfo, "device info not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkEstHostData, "host data not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkloadMatchingData, "Workload Matching Data not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkloadMatchingData->psHashLock, "hash lock not available", eError);
+	PVR_LOG_RETURN_IF_FALSE(psWorkloadMatchingData->psHashTable, "hash table not available", eError);
+
+#if defined(SUPPORT_SOC_TIMER)
+	psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	if (psDevConfig->pfnSoCTimerRead)
+	{
+		ui64CurrentSoCTime = psDevConfig->pfnSoCTimerRead(psDevConfig->hSysData);
+	}
+	else
+	{
+		/* Fallback to OS clock */
+		ui64CurrentSoCTime = 0;
+	}
+#endif
+
+	eError = OSClockMonotonicus64(&ui64CurrentTime);
+	PVR_LOG_RETURN_IF_ERROR(eError, "unable to access System Monotonic clock");
+
+	OSLockAcquire(psDevInfo->hWorkEstLock);
+
+	/* Select the next index for the return data and update it (is this thread safe?) */
+	ui32ReturnDataWO = psDevInfo->ui32ReturnDataWO;
+	psDevInfo->ui32ReturnDataWO = (ui32ReturnDataWO + 1) & RETURN_DATA_ARRAY_WRAP_MASK;
+
+	/* Index for the return data passed to/from the firmware. */
+	psWorkEstKickData->ui16ReturnDataIndex = ui32ReturnDataWO;
+	if (ui64DeadlineInus > ui64CurrentTime)
+	{
+		/* Rounding is done to reduce multiple deadlines with minor spread flooding the fw workload array. */
+#if defined(SUPPORT_SOC_TIMER)
+		if (psDevConfig->pfnSoCTimerRead)
+		{
+			RGX_DATA *psRGXData = (RGX_DATA*)psDevConfig->hDevData;
+			RGX_TIMING_INFORMATION *psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+			IMG_UINT32 ui32Remainder;
+			IMG_UINT64 ui64TimeDelta =
+				OSDivide64r64((ui64DeadlineInus - ui64CurrentTime) * psRGXTimingInfo->ui32SOCClockSpeed, SECONDS_TO_MICROSECONDS, &ui32Remainder);
+
+			psWorkEstKickData->ui64Deadline = ROUND_DOWN_TO_NEAREST_1024(ui64CurrentSoCTime + ui64TimeDelta);
+			PVR_DPF((PVR_DBG_MESSAGE, "Current SOC time: %llu. New deadline: %llu SOC ticks",
+					 ui64CurrentSoCTime, psWorkEstKickData->ui64Deadline));
+		}
+		else
+		{
+			psWorkEstKickData->ui64Deadline = ROUND_DOWN_TO_NEAREST_1024(ui64DeadlineInus);
+		}
+#else
+		psWorkEstKickData->ui64Deadline = ROUND_DOWN_TO_NEAREST_1024(ui64DeadlineInus);
+#endif
+	}
+	else
+	{
+		/* If deadline has already passed, assign zero to suggest full frequency */
+		psWorkEstKickData->ui64Deadline = 0;
+	}
+
+	/* Set up data for the return path to process the workload; the matching data is needed
+	   as it holds the hash data, the host data is needed for completion updates */
+	psReturnData = &psDevInfo->asReturnData[ui32ReturnDataWO];
+	psReturnData->psWorkloadMatchingData = psWorkloadMatchingData;
+	psReturnData->psWorkEstHostData = psWorkEstHostData;
+
+	/* The workload characteristic is needed in the return data for the matching
+	   of future workloads via the hash. */
+	psWorkloadCharacteristics = &psReturnData->sWorkloadCharacteristics;
+	memcpy(psWorkloadCharacteristics, psWorkloadCharsIn, sizeof(RGX_WORKLOAD));
+
+	OSLockRelease(psDevInfo->hWorkEstLock);
+
+	/* Acquire the lock to access hash */
+	OSLockAcquire(psWorkloadMatchingData->psHashLock);
+
+	/* Check if there is a prediction for this workload */
+	pui64CyclePrediction = (IMG_UINT64*) HASH_Retrieve(psWorkloadMatchingData->psHashTable,
+													   (uintptr_t)psWorkloadCharacteristics);
+
+	/* Release lock */
+	OSLockRelease(psWorkloadMatchingData->psHashLock);
+
+	if (pui64CyclePrediction != NULL)
+	{
+		/* Cycle prediction is available, store this prediction */
+		psWorkEstKickData->ui32CyclesPrediction = *pui64CyclePrediction;
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+		if (*pui64CyclePrediction >= IMG_UINT32_MAX)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Workload estimate overflow:"
+					" %" IMG_UINT64_FMTSPEC, *pui64CyclePrediction));
+		}
+
+		switch (eDMCmdType)
+		{
+			case RGXFWIF_CCB_CMD_TYPE_GEOM:
+			case RGXFWIF_CCB_CMD_TYPE_3D:
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: RT size = %u, draw count = %u, indices = %u, prediction = " IMG_DEVMEM_SIZE_FMTSPEC,
+					 __func__,
+					 psWorkloadCharacteristics->sTA3D.ui32RenderTargetSize,
+					 psWorkloadCharacteristics->sTA3D.ui32NumberOfDrawCalls,
+					 psWorkloadCharacteristics->sTA3D.ui32NumberOfIndices,
+					 *pui64CyclePrediction));
+				break;
+			case RGXFWIF_CCB_CMD_TYPE_CDM:
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Number of workgroups = %u, max workgroup size = %u, prediction = " IMG_DEVMEM_SIZE_FMTSPEC,
+					 __func__,
+					 psWorkloadCharacteristics->sCompute.ui32NumberOfWorkgroups,
+					 psWorkloadCharacteristics->sCompute.ui32NumberOfWorkitems,
+					 *pui64CyclePrediction));
+				break;
+			case RGXFWIF_CCB_CMD_TYPE_TQ_TDM:
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Dest size = %u, Pixel format ID = %u, prediction = " IMG_DEVMEM_SIZE_FMTSPEC,
+					 __func__,
+					 psWorkloadCharacteristics->sTransfer.ui32Characteristic1,
+					 psWorkloadCharacteristics->sTransfer.ui32Characteristic2,
+					 *pui64CyclePrediction));
+				break;
+
+			default:
+				break;
+		}
+#endif
+	}
+	else
+	{
+		/* There is no prediction */
+		psWorkEstKickData->ui32CyclesPrediction = 0;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR WorkEstRetire(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   RGXFWIF_WORKEST_FWCCB_CMD *psReturnCmd)
+{
+	RGX_WORKLOAD           *psWorkloadCharacteristics;
+	WORKLOAD_MATCHING_DATA *psWorkloadMatchingData;
+	IMG_UINT64             *paui64WorkloadHashData;
+	RGX_WORKLOAD           *pasWorkloadHashKeys;
+	IMG_UINT32             ui32HashArrayWO;
+	IMG_UINT64             *pui64CyclesTaken;
+	IMG_UINT64              ui64ActualCyclesTaken;
+	WORKEST_RETURN_DATA    *psReturnData;
+	WORKEST_HOST_DATA      *psWorkEstHostData;
+
+	if (!_WorkEstEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(psReturnCmd,
+	                        "WorkEstRetire: Missing return command",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	RGXFwSharedMemCacheOpPtr(psReturnCmd, INVALIDATE);
+
+	if (psReturnCmd->ui16ReturnDataIndex >= RETURN_DATA_ARRAY_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "WorkEstRetire: Handle reference out-of-bounds:"
+		        " %u >= %u",
+		        psReturnCmd->ui16ReturnDataIndex,
+		        RETURN_DATA_ARRAY_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquire(psDevInfo->hWorkEstLock);
+
+	/* Retrieve/validate the return data from this completed workload */
+	psReturnData = &psDevInfo->asReturnData[psReturnCmd->ui16ReturnDataIndex];
+	psWorkloadCharacteristics = &psReturnData->sWorkloadCharacteristics;
+	psWorkEstHostData = psReturnData->psWorkEstHostData;
+	PVR_LOG_GOTO_IF_FALSE(psWorkEstHostData,
+	                      "WorkEstRetire: Missing host data",
+	                      unlock_workest);
+
+	/* Skip if cycle data unavailable */
+	PVR_LOG_GOTO_IF_FALSE(psReturnCmd->ui32CyclesTaken,
+	                      "WorkEstRetire: Cycle data not available",
+	                      unlock_workest);
+
+	/* Retrieve/validate completed workload matching data */
+	psWorkloadMatchingData = psReturnData->psWorkloadMatchingData;
+	PVR_LOG_GOTO_IF_FALSE(psWorkloadMatchingData,
+	                      "WorkEstRetire: Missing matching data",
+	                      unlock_workest);
+	PVR_LOG_GOTO_IF_FALSE(psWorkloadMatchingData->psHashTable,
+	                      "WorkEstRetire: Missing hash",
+	                      unlock_workest);
+	PVR_LOG_GOTO_IF_FALSE(psWorkloadMatchingData->psHashLock,
+	                      "WorkEstRetire: Missing hash/lock",
+	                      unlock_workest);
+	paui64WorkloadHashData = psWorkloadMatchingData->aui64HashData;
+	pasWorkloadHashKeys = psWorkloadMatchingData->asHashKeys;
+	ui32HashArrayWO = psWorkloadMatchingData->ui32HashArrayWO;
+
+	OSLockRelease(psDevInfo->hWorkEstLock);
+
+	OSLockAcquire(psWorkloadMatchingData->psHashLock);
+
+	/* Update workload prediction by removing old hash entry (if any)
+	 * & inserting new hash entry */
+	pui64CyclesTaken = (IMG_UINT64*) HASH_Remove(psWorkloadMatchingData->psHashTable,
+												 (uintptr_t)psWorkloadCharacteristics);
+
+	if (paui64WorkloadHashData[ui32HashArrayWO] > 0)
+	{
+		/* Out-of-space so remove the oldest hash data before it becomes
+		 * overwritten */
+		RGX_WORKLOAD *psWorkloadHashKey = &pasWorkloadHashKeys[ui32HashArrayWO];
+		(void) HASH_Remove(psWorkloadMatchingData->psHashTable, (uintptr_t)psWorkloadHashKey);
+	}
+
+	ui64ActualCyclesTaken = (IMG_UINT64)psReturnCmd->ui16CyclesTakenHigh << 32U;
+	ui64ActualCyclesTaken += psReturnCmd->ui32CyclesTaken;
+
+	if (pui64CyclesTaken == NULL)
+	{
+		/* There is no existing entry for this workload characteristics,
+		 * store it */
+		paui64WorkloadHashData[ui32HashArrayWO] = ui64ActualCyclesTaken;
+		pasWorkloadHashKeys[ui32HashArrayWO] = *psWorkloadCharacteristics;
+	}
+	else
+	{
+		/* Found prior entry for workload characteristics, average with
+		 * completed; also reset the old value to 0 so it is known to be
+		 * invalid */
+		paui64WorkloadHashData[ui32HashArrayWO] = (*pui64CyclesTaken + ui64ActualCyclesTaken)/2;
+		pasWorkloadHashKeys[ui32HashArrayWO] = *psWorkloadCharacteristics;
+		*pui64CyclesTaken = 0;
+	}
+
+	/* Hash insertion should not fail but if it does best we can do is to exit
+	 * gracefully and not update the FW received counter */
+	if (IMG_TRUE != HASH_Insert((HASH_TABLE*)psWorkloadMatchingData->psHashTable,
+								(uintptr_t)&pasWorkloadHashKeys[ui32HashArrayWO],
+								(uintptr_t)&paui64WorkloadHashData[ui32HashArrayWO]))
+	{
+		PVR_ASSERT(0);
+		PVR_LOG(("WorkEstRetire: HASH_Insert failed"));
+	}
+
+#if defined(DEBUG)
+	/* Zero the current entry in the return data table.
+	 * Helps detect invalid ReturnDataIndex values from the
+	 * firmware before the hash table is corrupted. */
+	memset(psReturnData, 0, sizeof(WORKEST_RETURN_DATA));
+#endif
+
+	psWorkloadMatchingData->ui32HashArrayWO = (ui32HashArrayWO + 1) & WORKLOAD_HASH_WRAP_MASK;
+
+	OSLockRelease(psWorkloadMatchingData->psHashLock);
+
+	/* Update the received counter so that the FW is able to check as to whether
+	 * all the workloads connected to a render context are finished.
+	 * Note: needs to be done also for *unlock_workest* label below. */
+	psWorkEstHostData->ui32WorkEstCCBReceived++;
+
+	return PVRSRV_OK;
+
+unlock_workest:
+	OSLockRelease(psDevInfo->hWorkEstLock);
+
+	PVR_ASSERT(psWorkEstHostData);
+	if (psWorkEstHostData)
+	{
+		psWorkEstHostData->ui32WorkEstCCBReceived++;
+	}
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+void _WorkEstInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						 WORKLOAD_MATCHING_DATA *psWorkloadMatchingData,
+						 HASH_FUNC *pfnWorkEstHashFunc,
+						 HASH_KEY_COMP *pfnWorkEstHashCompare)
+{
+	HASH_TABLE *psWorkloadHashTable;
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+
+	/* Create a lock to protect the per-DM hash table */
+	WorkEstHashLockCreate(&psWorkloadMatchingData->psHashLock);
+
+	/* Create hash table for the per-DM workload matching */
+	psWorkloadHashTable = HASH_Create_Extended(WORKLOAD_HASH_SIZE,
+											   sizeof(RGX_WORKLOAD *),
+											   pfnWorkEstHashFunc,
+											   pfnWorkEstHashCompare);
+	psWorkloadMatchingData->psHashTable = psWorkloadHashTable;
+}
+
+void _WorkEstDeInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   WORKLOAD_MATCHING_DATA *psWorkloadMatchingData)
+{
+	HASH_TABLE        *psWorkloadHashTable;
+	RGX_WORKLOAD      *pasWorkloadHashKeys;
+	RGX_WORKLOAD      *psWorkloadHashKey;
+	IMG_UINT64        *paui64WorkloadCycleData;
+	IMG_UINT32        ui32Itr;
+
+	/* Tear down per-DM hash */
+	pasWorkloadHashKeys = psWorkloadMatchingData->asHashKeys;
+	paui64WorkloadCycleData = psWorkloadMatchingData->aui64HashData;
+	psWorkloadHashTable = psWorkloadMatchingData->psHashTable;
+
+	if (psWorkloadHashTable)
+	{
+		for (ui32Itr = 0; ui32Itr < WORKLOAD_HASH_SIZE; ui32Itr++)
+		{
+			if (paui64WorkloadCycleData[ui32Itr] > 0)
+			{
+				psWorkloadHashKey = &pasWorkloadHashKeys[ui32Itr];
+				HASH_Remove(psWorkloadHashTable, (uintptr_t)psWorkloadHashKey);
+			}
+		}
+
+		HASH_Delete(psWorkloadHashTable);
+	}
+
+	/* Remove the hash lock */
+	WorkEstHashLockDestroy(psWorkloadMatchingData->psHashLock);
+}
+
+void WorkEstInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sTA3D.sDataTA,
+		(HASH_FUNC *)WorkEstHashFuncTA3D,
+		(HASH_KEY_COMP *)WorkEstHashCompareTA3D);
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sTA3D.sData3D,
+		(HASH_FUNC *)WorkEstHashFuncTA3D,
+		(HASH_KEY_COMP *)WorkEstHashCompareTA3D);
+}
+
+void WorkEstDeInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sTA3D.sDataTA);
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sTA3D.sData3D);
+}
+
+void WorkEstInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sCompute.sDataCDM,
+		(HASH_FUNC *)WorkEstHashFuncCompute,
+		(HASH_KEY_COMP *)WorkEstHashCompareCompute);
+}
+
+void WorkEstDeInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sCompute.sDataCDM);
+}
+
+void WorkEstInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sTransfer.sDataTDM,
+		(HASH_FUNC *)WorkEstHashFuncTDM,
+		(HASH_KEY_COMP *)WorkEstHashCompareTDM);
+}
+
+void WorkEstDeInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sTransfer.sDataTDM);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.h
new file mode 100644
index 000000000000..972c5199a3d8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rgxworkest.h
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File           rgxworkest.h
+@Title          RGX Workload Estimation Functionality
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXWORKEST_H
+#define RGXWORKEST_H
+
+#include "img_types.h"
+#include "rgxta3d.h"
+
+
+void WorkEstInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstDeInitTA3D(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstDeInitCompute(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+void WorkEstDeInitTDM(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+PVRSRV_ERROR WorkEstPrepare(PVRSRV_RGXDEV_INFO        *psDevInfo,
+                            WORKEST_HOST_DATA         *psWorkEstHostData,
+                            WORKLOAD_MATCHING_DATA    *psWorkloadMatchingData,
+                            const RGXFWIF_CCB_CMD_TYPE eDMCmdType,
+                            const RGX_WORKLOAD        *psWorkloadCharsIn,
+                            IMG_UINT64                ui64DeadlineInus,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData);
+
+PVRSRV_ERROR WorkEstRetire(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   RGXFWIF_WORKEST_FWCCB_CMD *psReturnCmd);
+
+void WorkEstHashLockCreate(POS_LOCK *ppsHashLock);
+
+void WorkEstHashLockDestroy(POS_LOCK psHashLock);
+
+void WorkEstCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void _WorkEstInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						 WORKLOAD_MATCHING_DATA *psWorkloadMatchingData,
+						 HASH_FUNC *pfnWorkEstHashFunc,
+						 HASH_KEY_COMP *pfnWorkEstHashCompare);
+
+void _WorkEstDeInit(PVRSRV_RGXDEV_INFO *psDevInfo,
+						 WORKLOAD_MATCHING_DATA *psWorkloadMatchingData);
+
+inline IMG_UINT32 _WorkEstDoHash(IMG_UINT32 ui32Input);
+
+#endif /* RGXWORKEST_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdebug.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdebug.c
new file mode 100644
index 000000000000..93738fdf6f9a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdebug.c
@@ -0,0 +1,2580 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "cache_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+#include "rgxutils.h"
+#include "tlstream.h"
+#include "rgxfwriscv.h"
+#include "pvrsrv.h"
+#include "services_km.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_utils.h"
+#include "rgx_fwif_km.h"
+#include "rgx_fwif_sf.h"
+#include "debug_common.h"
+
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxcompute.h"
+#include "rgxtransfer.h"
+#include "rgxtdmtransfer.h"
+#include "rgxtimecorr.h"
+#include "rgx_options.h"
+#include "rgxinit.h"
+#include "rgxlayer_impl.h"
+#include "devicemem_history_server.h"
+
+#define DD_SUMMARY_INDENT  ""
+
+#define RGX_DEBUG_STR_SIZE			(150U)
+
+#define RGX_CR_BIF_CAT_BASE0                              (0x1200U)
+#define RGX_CR_BIF_CAT_BASE1                              (0x1208U)
+
+#define RGX_CR_BIF_CAT_BASEN(n) \
+	RGX_CR_BIF_CAT_BASE0 + \
+	((RGX_CR_BIF_CAT_BASE1 - RGX_CR_BIF_CAT_BASE0) * n)
+
+
+#define RGXDBG_BIF_IDS \
+	X(BIF0)\
+	X(BIF1)\
+	X(TEXAS_BIF)\
+	X(DPX_BIF) \
+	X(FWCORE)
+
+#define RGXDBG_SIDEBAND_TYPES \
+	X(META)\
+	X(TLA)\
+	X(DMA)\
+	X(VDMM)\
+	X(CDM)\
+	X(IPP)\
+	X(PM)\
+	X(TILING)\
+	X(MCU)\
+	X(PDS)\
+	X(PBE)\
+	X(VDMS)\
+	X(IPF)\
+	X(ISP)\
+	X(TPF)\
+	X(USCS)\
+	X(PPP)\
+	X(VCE)\
+	X(TPF_CPF)\
+	X(IPF_CPF)\
+	X(FBCDC)
+
+typedef enum
+{
+#define X(NAME) RGXDBG_##NAME,
+	RGXDBG_BIF_IDS
+#undef X
+} RGXDBG_BIF_ID;
+
+typedef enum
+{
+#define X(NAME) RGXDBG_##NAME,
+	RGXDBG_SIDEBAND_TYPES
+#undef X
+} RGXDBG_SIDEBAND_TYPE;
+
+static const IMG_CHAR *const pszPowStateName[] =
+{
+#define X(NAME)	#NAME,
+	RGXFWIF_POW_STATES
+#undef X
+};
+
+static const IMG_CHAR *const pszBIFNames[] =
+{
+#define X(NAME)	#NAME,
+	RGXDBG_BIF_IDS
+#undef X
+};
+
+static const IMG_FLAGS2DESC asHwrState2Description[] =
+{
+	{RGXFWIF_HWR_HARDWARE_OK, " HWR OK;"},
+	{RGXFWIF_HWR_GENERAL_LOCKUP, " General lockup;"},
+	{RGXFWIF_HWR_DM_RUNNING_OK, " DM running ok;"},
+	{RGXFWIF_HWR_DM_STALLING, " DM stalling;"},
+	{RGXFWIF_HWR_FW_FAULT, " FW fault;"},
+	{RGXFWIF_HWR_RESTART_REQUESTED, " Restart requested;"},
+};
+
+static const IMG_FLAGS2DESC asDmState2Description[] =
+{
+	{RGXFWIF_DM_STATE_READY_FOR_HWR, " ready for hwr;"},
+	{RGXFWIF_DM_STATE_NEEDS_SKIP, " needs skip;"},
+	{RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP, " needs PR cleanup;"},
+	{RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR, " needs trace clear;"},
+	{RGXFWIF_DM_STATE_GUILTY_LOCKUP, " guilty lockup;"},
+	{RGXFWIF_DM_STATE_INNOCENT_LOCKUP, " innocent lockup;"},
+	{RGXFWIF_DM_STATE_GUILTY_OVERRUNING, " guilty overrunning;"},
+	{RGXFWIF_DM_STATE_INNOCENT_OVERRUNING, " innocent overrunning;"},
+	{RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH, " hard context switching;"},
+	{RGXFWIF_DM_STATE_GPU_ECC_HWR, " GPU ECC hwr;"},
+};
+
+static const IMG_CHAR * const apszFwOsStateName[RGXFW_CONNECTION_FW_STATE_COUNT] =
+{
+	"offline",
+	"ready",
+	"active",
+	"offloading",
+	"cooldown"
+};
+
+#if defined(PVR_ENABLE_PHR)
+static const IMG_FLAGS2DESC asPHRConfig2Description[] =
+{
+	{BIT_ULL(RGXFWIF_PHR_MODE_OFF), "off"},
+	{BIT_ULL(RGXFWIF_PHR_MODE_RD_RESET), "reset RD hardware"},
+};
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodePMPC
+
+ @Description
+
+ Return the name for the PM managed Page Catalogues
+
+ @Input ui32PC	 - Page Catalogue number
+
+ @Return   void
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodePMPC(IMG_UINT32 ui32PC)
+{
+	const IMG_CHAR* pszPMPC = " (-)";
+
+	switch (ui32PC)
+	{
+		case 0x8: pszPMPC = " (PM-VCE0)"; break;
+		case 0x9: pszPMPC = " (PM-TE0)"; break;
+		case 0xA: pszPMPC = " (PM-ZLS0)"; break;
+		case 0xB: pszPMPC = " (PM-ALIST0)"; break;
+		case 0xC: pszPMPC = " (PM-VCE1)"; break;
+		case 0xD: pszPMPC = " (PM-TE1)"; break;
+		case 0xE: pszPMPC = " (PM-ZLS1)"; break;
+		case 0xF: pszPMPC = " (PM-ALIST1)"; break;
+	}
+
+	return pszPMPC;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeBIFReqTags
+
+ @Description
+
+ Decode the BIF Tag ID and sideband data fields from BIF_FAULT_BANK_REQ_STATUS regs
+
+ @Input eBankID             - BIF identifier
+ @Input ui32TagID           - Tag ID value
+ @Input ui32TagSB           - Tag Sideband data
+ @Output ppszTagID          - Decoded string from the Tag ID
+ @Output ppszTagSB          - Decoded string from the Tag SB
+ @Output pszScratchBuf      - Buffer provided to the function to generate the debug strings
+ @Input ui32ScratchBufSize  - Size of the provided buffer
+
+ @Return   void
+
+******************************************************************************/
+#include "rgxmhdefs_km.h"
+
+static void _RGXDecodeBIFReqTagsXE(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								   IMG_UINT32	ui32TagID,
+								   IMG_UINT32	ui32TagSB,
+								   IMG_CHAR		**ppszTagID,
+								   IMG_CHAR		**ppszTagSB,
+								   IMG_CHAR		*pszScratchBuf,
+								   IMG_UINT32	ui32ScratchBufSize)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+	IMG_BOOL bNewTagEncoding = IMG_FALSE;
+
+	PVR_ASSERT(ppszTagID != NULL);
+	PVR_ASSERT(ppszTagSB != NULL);
+
+	/* tags updated for all cores (auto & consumer) with branch > 36 or only auto cores with branch = 36 */
+	if ((psDevInfo->sDevFeatureCfg.ui32B > 36) ||
+	    (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TILE_REGION_PROTECTION) && (psDevInfo->sDevFeatureCfg.ui32B == 36)))
+	{
+		bNewTagEncoding = IMG_TRUE;
+	}
+
+	switch (ui32TagID)
+	{
+		/* MMU tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MMU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_MMU:	    pszTagID = "MMU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:	pszTagID = "CPU MMU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:	pszTagID = "CPU IFU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:	pszTagID = "CPU LSU"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PT_REQUEST:		pszTagSB = "PT"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PD_REQUEST:		pszTagSB = "PD"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PC_REQUEST:		pszTagSB = "PC"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PT_REQUEST:	pszTagSB = "PM PT"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_REQUEST:	pszTagSB = "PM PD"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_REQUEST:	pszTagSB = "PM PC"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_WREQUEST:	pszTagSB = "PM PD W"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_WREQUEST:	pszTagSB = "PM PC W"; break;
+			}
+			break;
+		}
+
+		/* MIPS */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MIPS:
+		{
+			pszTagID = "MIPS";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_OPCODE_FETCH:	pszTagSB = "Opcode"; break;
+				case RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_DATA_ACCESS:	pszTagSB = "Data"; break;
+			}
+			break;
+		}
+
+		/* CDM tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0:	pszTagID = "CDM Stage 0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1:	pszTagID = "CDM Stage 1"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2:	pszTagID = "CDM Stage 2"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3:	pszTagID = "CDM Stage 3"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTROL_STREAM:	pszTagSB = "Control"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_INDIRECT_DATA:	pszTagSB = "Indirect"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_EVENT_DATA:		pszTagSB = "Event"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTEXT_STATE:	pszTagSB = "Context"; break;
+			}
+			break;
+		}
+
+		/* VDM tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0:	pszTagID = "VDM Stage 0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1:	pszTagID = "VDM Stage 1"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2:	pszTagID = "VDM Stage 2"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3:	pszTagID = "VDM Stage 3"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4:	pszTagID = "VDM Stage 4"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTROL:	pszTagSB = "Control"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STATE:		pszTagSB = "State"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_INDEX:		pszTagSB = "Index"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STACK:		pszTagSB = "Stack"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTEXT:	pszTagSB = "Context"; break;
+			}
+			break;
+		}
+
+		/* PDS */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PDS_0:
+			pszTagID = "PDS req 0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PDS_1:
+			pszTagID = "PDS req 1"; break;
+
+		/* MCU */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCA:
+			pszTagID = "MCU USCA"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCB:
+			pszTagID = "MCU USCB"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCC:
+			pszTagID = "MCU USCC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCD:
+			pszTagID = "MCU USCD"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCA:
+			pszTagID = "MCU PDS USCA"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCB:
+			pszTagID = "MCU PDS USCB"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCC:
+			pszTagID = "MCU PDS USCC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCD:
+			pszTagID = "MCU PDSUSCD"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDSRW:
+			pszTagID = "MCU PDS PDSRW"; break;
+
+		/* TCU */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TCU_0:
+			pszTagID = "TCU req 0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TCU_1:
+			pszTagID = "TCU req 1"; break;
+
+		/* FBCDC */
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_0:
+			pszTagID = bNewTagEncoding ? "TFBDC_TCU0" : "FBCDC0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_1:
+			pszTagID = bNewTagEncoding ? "TFBDC_ZLS0" : "FBCDC1"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_2:
+			pszTagID = bNewTagEncoding ? "TFBDC_TCU1" : "FBCDC2"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_3:
+			pszTagID = bNewTagEncoding ? "TFBDC_ZLS1" : "FBCDC3"; break;
+
+		/* USC Shared */
+		case RGX_MH_TAG_ENCODING_MH_TAG_USC:
+			pszTagID = "USCS"; break;
+
+		/* ISP */
+		case RGX_MH_TAG_ENCODING_MH_TAG_ISP_ZLS:
+			pszTagID = "ISP0 ZLS"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_ISP_DS:
+			pszTagID = "ISP0 DS"; break;
+
+		/* TPF */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF:
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS:
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF:           pszTagID = "TPF0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS:  pszTagID = "TPF0 DBIAS"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF:       pszTagID = "TPF0 SPF"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_PDS_STATE:	pszTagSB = "PDS state"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DEPTH_BIAS:	pszTagSB = "Depth bias"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_FLOOR_OFFSET_DATA:	pszTagSB = "Floor offset"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DELTA_DATA:	pszTagSB = "Delta"; break;
+			}
+			break;
+		}
+
+		/* IPF */
+		case RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ:
+		case RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ:      pszTagID = "IPF0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS:    pszTagID = "IPF0"; break;
+			}
+
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_ISP_IPP_PIPES))
+			{
+				if (ui32TagID < RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+				{
+					OSSNPrintf(pszScratchBuf, ui32ScratchBufSize, "CReq%d", ui32TagID);
+					pszTagSB = pszScratchBuf;
+				}
+				else if (ui32TagID < 2 * RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+				{
+					ui32TagID -= RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES);
+					OSSNPrintf(pszScratchBuf, ui32ScratchBufSize, "PReq%d", ui32TagID);
+					pszTagSB = pszScratchBuf;
+				}
+				else
+				{
+					switch (ui32TagSB - 2 * RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+					{
+						case 0:	pszTagSB = "RReq"; break;
+						case 1:	pszTagSB = "DBSC"; break;
+						case 2:	pszTagSB = "CPF"; break;
+						case 3:	pszTagSB = "Delta"; break;
+					}
+				}
+			}
+			break;
+		}
+
+		/* VDM Stage 5 (temporary) */
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG5:
+			pszTagID = "VDM Stage 5"; break;
+
+		/* TA */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PPP:
+			pszTagID = "PPP"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPWRTC:
+			pszTagID = "TPW RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TEACRTC:
+			pszTagID = "TEAC RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGRTC:
+			pszTagID = "PSG RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGREGION:
+			pszTagID = "PSG Region"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGSTREAM:
+			pszTagID = "PSG Stream"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPW:
+			pszTagID = "TPW"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPC:
+			pszTagID = "TPC"; break;
+
+		/* PM */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PM_ALLOC:
+		{
+			pszTagID = "PMA";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAFSTACK:	pszTagSB = "TA Fstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMLIST:		pszTagSB = "TA MList"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DFSTACK:	pszTagSB = "3D Fstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMLIST:		pszTagSB = "3D MList"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX0:		pszTagSB = "Context0"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX1:		pszTagSB = "Context1"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_MAVP:		pszTagSB = "MAVP"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_UFSTACK:		pszTagSB = "UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMMUSTACK:	pszTagSB = "TA MMUstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMMUSTACK:	pszTagSB = "3D MMUstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAUFSTACK:	pszTagSB = "TA UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DUFSTACK:	pszTagSB = "3D UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAVFP:		pszTagSB = "TA VFP"; break;
+			}
+			break;
+		}
+		case RGX_MH_TAG_ENCODING_MH_TAG_PM_DEALLOC:
+		{
+			pszTagID = "PMD";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAFSTACK:	pszTagSB = "TA Fstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMLIST:		pszTagSB = "TA MList"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DFSTACK:	pszTagSB = "3D Fstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMLIST:		pszTagSB = "3D MList"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX0:		pszTagSB = "Context0"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX1:		pszTagSB = "Context1"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_UFSTACK:		pszTagSB = "UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMMUSTACK:	pszTagSB = "TA MMUstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMMUSTACK:	pszTagSB = "3D MMUstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAUFSTACK:	pszTagSB = "TA UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DUFSTACK:	pszTagSB = "3D UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAVFP:		pszTagSB = "TA VFP"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DVFP:		pszTagSB = "3D VFP"; break;
+			}
+			break;
+		}
+
+		/* TDM */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TDM_DMA:
+		{
+			pszTagID = "TDM DMA";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTL_STREAM: pszTagSB = "Ctl stream"; break;
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTX_BUFFER: pszTagSB = "Ctx buffer"; break;
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_QUEUE_CTL:  pszTagSB = "Queue ctl"; break;
+			}
+			break;
+		}
+		case RGX_MH_TAG_ENCODING_MH_TAG_TDM_CTL:
+		{
+			pszTagID = "TDM CTL";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_FENCE:   pszTagSB = "Fence"; break;
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_CONTEXT: pszTagSB = "Context"; break;
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_QUEUE:   pszTagSB = "Queue"; break;
+			}
+			break;
+		}
+
+		/* PBE */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE0:
+			pszTagID = "PBE0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE1:
+			pszTagID = "PBE1"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE2:
+			pszTagID = "PBE2"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE3:
+			pszTagID = "PBE3"; break;
+	}
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+/* RISC-V pf tags */
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU  (0x00000001U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU  (0x00000002U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU  (0x00000003U)
+
+static void _RGXDecodeBIFReqTagsFwcore(PVRSRV_RGXDEV_INFO *psDevInfo,
+									   IMG_UINT32 ui32TagID,
+									   IMG_UINT32 ui32TagSB,
+									   IMG_CHAR **ppszTagID,
+									   IMG_CHAR **ppszTagSB)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		pszTagSB = "RISC-V";
+
+		switch (ui32TagID)
+		{
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:	pszTagID = "RISC-V MMU"; break;
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:	pszTagID = "RISC-V Instruction Fetch Unit"; break;
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:	pszTagID = "RISC-V Load/Store Unit"; break; /* Or Debug Module System Bus */
+		}
+	}
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+static void _RGXDecodeBIFReqTags(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								 RGXDBG_BIF_ID	eBankID,
+								 IMG_UINT32		ui32TagID,
+								 IMG_UINT32		ui32TagSB,
+								 IMG_CHAR		**ppszTagID,
+								 IMG_CHAR		**ppszTagSB,
+								 IMG_CHAR		*pszScratchBuf,
+								 IMG_UINT32		ui32ScratchBufSize)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+
+	PVR_ASSERT(ppszTagID != NULL);
+	PVR_ASSERT(ppszTagSB != NULL);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		if (eBankID == RGXDBG_FWCORE)
+		{
+			_RGXDecodeBIFReqTagsFwcore(psDevInfo, ui32TagID, ui32TagSB, ppszTagID, ppszTagSB);
+		}
+		else
+		{
+			_RGXDecodeBIFReqTagsXE(psDevInfo, ui32TagID, ui32TagSB, ppszTagID, ppszTagSB, pszScratchBuf, ui32ScratchBufSize);
+		}
+		return;
+	}
+
+	switch (ui32TagID)
+	{
+		case 0x0:
+		{
+			pszTagID = "MMU";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Table"; break;
+				case 0x1: pszTagSB = "Directory"; break;
+				case 0x2: pszTagSB = "Catalogue"; break;
+			}
+			break;
+		}
+		case 0x1:
+		{
+			pszTagID = "TLA";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Pixel data"; break;
+				case 0x1: pszTagSB = "Command stream data"; break;
+				case 0x2: pszTagSB = "Fence or flush"; break;
+			}
+			break;
+		}
+		case 0x2:
+		{
+			pszTagID = "HOST";
+			break;
+		}
+		case 0x3:
+		{
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+			{
+					pszTagID = "META";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "DCache - Thread 0"; break;
+						case 0x1: pszTagSB = "ICache - Thread 0"; break;
+						case 0x2: pszTagSB = "JTag - Thread 0"; break;
+						case 0x3: pszTagSB = "Slave bus - Thread 0"; break;
+						case 0x4: pszTagSB = "DCache - Thread "; break;
+						case 0x5: pszTagSB = "ICache - Thread 1"; break;
+						case 0x6: pszTagSB = "JTag - Thread 1"; break;
+						case 0x7: pszTagSB = "Slave bus - Thread 1"; break;
+					}
+			}
+			else if (RGX_IS_ERN_SUPPORTED(psDevInfo, 57596))
+			{
+				pszTagID="TCU";
+			}
+			else
+			{
+				/* Unreachable code */
+				PVR_ASSERT(IMG_FALSE);
+			}
+			break;
+		}
+		case 0x4:
+		{
+			pszTagID = "USC";
+			OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+			           "Cache line %d", (ui32TagSB & 0x3f));
+			pszTagSB = pszScratchBuf;
+			break;
+		}
+		case 0x5:
+		{
+			pszTagID = "PBE";
+			break;
+		}
+		case 0x6:
+		{
+			pszTagID = "ISP";
+			switch (ui32TagSB)
+			{
+				case 0x00: pszTagSB = "ZLS"; break;
+				case 0x20: pszTagSB = "Occlusion Query"; break;
+			}
+			break;
+		}
+		case 0x7:
+		{
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+			{
+				if (eBankID == RGXDBG_TEXAS_BIF)
+				{
+					pszTagID = "IPF";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "CPF"; break;
+						case 0x1: pszTagSB = "DBSC"; break;
+						case 0x2:
+						case 0x4:
+						case 0x6:
+						case 0x8: pszTagSB = "Control Stream"; break;
+						case 0x3:
+						case 0x5:
+						case 0x7:
+						case 0x9: pszTagSB = "Primitive Block"; break;
+					}
+				}
+				else
+				{
+					pszTagID = "IPP";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "Macrotile Header"; break;
+						case 0x1: pszTagSB = "Region Header"; break;
+					}
+				}
+			}
+			else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SIMPLE_INTERNAL_PARAMETER_FORMAT))
+			{
+				pszTagID = "IPF";
+				switch (ui32TagSB)
+				{
+					case 0x0: pszTagSB = "Region Header"; break;
+					case 0x1: pszTagSB = "DBSC"; break;
+					case 0x2: pszTagSB = "CPF"; break;
+					case 0x3: pszTagSB = "Control Stream"; break;
+					case 0x4: pszTagSB = "Primitive Block"; break;
+				}
+			}
+			else
+			{
+				pszTagID = "IPF";
+				switch (ui32TagSB)
+				{
+					case 0x0: pszTagSB = "Macrotile Header"; break;
+					case 0x1: pszTagSB = "Region Header"; break;
+					case 0x2: pszTagSB = "DBSC"; break;
+					case 0x3: pszTagSB = "CPF"; break;
+					case 0x4:
+					case 0x6:
+					case 0x8: pszTagSB = "Control Stream"; break;
+					case 0x5:
+					case 0x7:
+					case 0x9: pszTagSB = "Primitive Block"; break;
+				}
+			}
+			break;
+		}
+		case 0x8:
+		{
+			pszTagID = "CDM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Control Stream"; break;
+				case 0x1: pszTagSB = "Indirect Data"; break;
+				case 0x2: pszTagSB = "Event Write"; break;
+				case 0x3: pszTagSB = "Context State"; break;
+			}
+			break;
+		}
+		case 0x9:
+		{
+			pszTagID = "VDM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Control Stream"; break;
+				case 0x1: pszTagSB = "PPP State"; break;
+				case 0x2: pszTagSB = "Index Data"; break;
+				case 0x4: pszTagSB = "Call Stack"; break;
+				case 0x8: pszTagSB = "Context State"; break;
+			}
+			break;
+		}
+		case 0xA:
+		{
+			pszTagID = "PM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "PMA_TAFSTACK"; break;
+				case 0x1: pszTagSB = "PMA_TAMLIST"; break;
+				case 0x2: pszTagSB = "PMA_3DFSTACK"; break;
+				case 0x3: pszTagSB = "PMA_3DMLIST"; break;
+				case 0x4: pszTagSB = "PMA_PMCTX0"; break;
+				case 0x5: pszTagSB = "PMA_PMCTX1"; break;
+				case 0x6: pszTagSB = "PMA_MAVP"; break;
+				case 0x7: pszTagSB = "PMA_UFSTACK"; break;
+				case 0x8: pszTagSB = "PMD_TAFSTACK"; break;
+				case 0x9: pszTagSB = "PMD_TAMLIST"; break;
+				case 0xA: pszTagSB = "PMD_3DFSTACK"; break;
+				case 0xB: pszTagSB = "PMD_3DMLIST"; break;
+				case 0xC: pszTagSB = "PMD_PMCTX0"; break;
+				case 0xD: pszTagSB = "PMD_PMCTX1"; break;
+				case 0xF: pszTagSB = "PMD_UFSTACK"; break;
+				case 0x10: pszTagSB = "PMA_TAMMUSTACK"; break;
+				case 0x11: pszTagSB = "PMA_3DMMUSTACK"; break;
+				case 0x12: pszTagSB = "PMD_TAMMUSTACK"; break;
+				case 0x13: pszTagSB = "PMD_3DMMUSTACK"; break;
+				case 0x14: pszTagSB = "PMA_TAUFSTACK"; break;
+				case 0x15: pszTagSB = "PMA_3DUFSTACK"; break;
+				case 0x16: pszTagSB = "PMD_TAUFSTACK"; break;
+				case 0x17: pszTagSB = "PMD_3DUFSTACK"; break;
+				case 0x18: pszTagSB = "PMA_TAVFP"; break;
+				case 0x19: pszTagSB = "PMD_3DVFP"; break;
+				case 0x1A: pszTagSB = "PMD_TAVFP"; break;
+			}
+			break;
+		}
+		case 0xB:
+		{
+			pszTagID = "TA";
+			switch (ui32TagSB)
+			{
+				case 0x1: pszTagSB = "VCE"; break;
+				case 0x2: pszTagSB = "TPC"; break;
+				case 0x3: pszTagSB = "TE Control Stream"; break;
+				case 0x4: pszTagSB = "TE Region Header"; break;
+				case 0x5: pszTagSB = "TE Render Target Cache"; break;
+				case 0x6: pszTagSB = "TEAC Render Target Cache"; break;
+				case 0x7: pszTagSB = "VCE Render Target Cache"; break;
+				case 0x8: pszTagSB = "PPP Context State"; break;
+			}
+			break;
+		}
+		case 0xC:
+		{
+			pszTagID = "TPF";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "TPF0: Primitive Block"; break;
+				case 0x1: pszTagSB = "TPF0: Depth Bias"; break;
+				case 0x2: pszTagSB = "TPF0: Per Primitive IDs"; break;
+				case 0x3: pszTagSB = "CPF - Tables"; break;
+				case 0x4: pszTagSB = "TPF1: Primitive Block"; break;
+				case 0x5: pszTagSB = "TPF1: Depth Bias"; break;
+				case 0x6: pszTagSB = "TPF1: Per Primitive IDs"; break;
+				case 0x7: pszTagSB = "CPF - Data: Pipe 0"; break;
+				case 0x8: pszTagSB = "TPF2: Primitive Block"; break;
+				case 0x9: pszTagSB = "TPF2: Depth Bias"; break;
+				case 0xA: pszTagSB = "TPF2: Per Primitive IDs"; break;
+				case 0xB: pszTagSB = "CPF - Data: Pipe 1"; break;
+				case 0xC: pszTagSB = "TPF3: Primitive Block"; break;
+				case 0xD: pszTagSB = "TPF3: Depth Bias"; break;
+				case 0xE: pszTagSB = "TPF3: Per Primitive IDs"; break;
+				case 0xF: pszTagSB = "CPF - Data: Pipe 2"; break;
+			}
+			break;
+		}
+		case 0xD:
+		{
+			pszTagID = "PDS";
+			break;
+		}
+		case 0xE:
+		{
+			pszTagID = "MCU";
+			{
+				IMG_UINT32 ui32Burst = (ui32TagSB >> 5) & 0x7;
+				IMG_UINT32 ui32GroupEnc = (ui32TagSB >> 2) & 0x7;
+				IMG_UINT32 ui32Group = ui32TagSB & 0x3;
+
+				IMG_CHAR* pszBurst = "";
+				IMG_CHAR* pszGroupEnc = "";
+				IMG_CHAR* pszGroup = "";
+
+				switch (ui32Burst)
+				{
+					case 0x0:
+					case 0x1: pszBurst = "128bit word within the Lower 256bits"; break;
+					case 0x2:
+					case 0x3: pszBurst = "128bit word within the Upper 256bits"; break;
+					case 0x4: pszBurst = "Lower 256bits"; break;
+					case 0x5: pszBurst = "Upper 256bits"; break;
+					case 0x6: pszBurst = "512 bits"; break;
+				}
+				switch (ui32GroupEnc)
+				{
+					case 0x0: pszGroupEnc = "TPUA_USC"; break;
+					case 0x1: pszGroupEnc = "TPUB_USC"; break;
+					case 0x2: pszGroupEnc = "USCA_USC"; break;
+					case 0x3: pszGroupEnc = "USCB_USC"; break;
+					case 0x4: pszGroupEnc = "PDS_USC"; break;
+					case 0x5:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 > RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "PDSRW";
+						} else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "UPUC_USC";
+						}
+						break;
+					case 0x6:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "TPUC_USC";
+						}
+						break;
+					case 0x7:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "PDSRW";
+						}
+						break;
+				}
+				switch (ui32Group)
+				{
+					case 0x0: pszGroup = "Banks 0-3"; break;
+					case 0x1: pszGroup = "Banks 4-7"; break;
+					case 0x2: pszGroup = "Banks 8-11"; break;
+					case 0x3: pszGroup = "Banks 12-15"; break;
+				}
+
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+								"%s, %s, %s", pszBurst, pszGroupEnc, pszGroup);
+				pszTagSB = pszScratchBuf;
+			}
+			break;
+		}
+		case 0xF:
+		{
+			pszTagID = "FB_CDC";
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+			{
+				IMG_UINT32 ui32Req   = (ui32TagSB >> 0) & 0xf;
+				IMG_UINT32 ui32MCUSB = (ui32TagSB >> 4) & 0x3;
+				IMG_CHAR* pszReqOrig = "";
+
+				switch (ui32Req)
+				{
+					case 0x0: pszReqOrig = "FBC Request, originator ZLS"; break;
+					case 0x1: pszReqOrig = "FBC Request, originator PBE"; break;
+					case 0x2: pszReqOrig = "FBC Request, originator Host"; break;
+					case 0x3: pszReqOrig = "FBC Request, originator TLA"; break;
+					case 0x4: pszReqOrig = "FBDC Request, originator ZLS"; break;
+					case 0x5: pszReqOrig = "FBDC Request, originator MCU"; break;
+					case 0x6: pszReqOrig = "FBDC Request, originator Host"; break;
+					case 0x7: pszReqOrig = "FBDC Request, originator TLA"; break;
+					case 0x8: pszReqOrig = "FBC Request, originator ZLS Requester Fence"; break;
+					case 0x9: pszReqOrig = "FBC Request, originator PBE Requester Fence"; break;
+					case 0xa: pszReqOrig = "FBC Request, originator Host Requester Fence"; break;
+					case 0xb: pszReqOrig = "FBC Request, originator TLA Requester Fence"; break;
+					case 0xc: pszReqOrig = "Reserved"; break;
+					case 0xd: pszReqOrig = "Reserved"; break;
+					case 0xe: pszReqOrig = "FBDC Request, originator FBCDC(Host) Memory Fence"; break;
+					case 0xf: pszReqOrig = "FBDC Request, originator FBCDC(TLA) Memory Fence"; break;
+				}
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+				           "%s, MCU sideband 0x%X", pszReqOrig, ui32MCUSB);
+				pszTagSB = pszScratchBuf;
+			}
+			else
+			{
+				IMG_UINT32 ui32Req   = (ui32TagSB >> 2) & 0x7;
+				IMG_UINT32 ui32MCUSB = (ui32TagSB >> 0) & 0x3;
+				IMG_CHAR* pszReqOrig = "";
+
+				switch (ui32Req)
+				{
+					case 0x0: pszReqOrig = "FBC Request, originator ZLS";   break;
+					case 0x1: pszReqOrig = "FBC Request, originator PBE";   break;
+					case 0x2: pszReqOrig = "FBC Request, originator Host";  break;
+					case 0x3: pszReqOrig = "FBC Request, originator TLA";   break;
+					case 0x4: pszReqOrig = "FBDC Request, originator ZLS";  break;
+					case 0x5: pszReqOrig = "FBDC Request, originator MCU";  break;
+					case 0x6: pszReqOrig = "FBDC Request, originator Host"; break;
+					case 0x7: pszReqOrig = "FBDC Request, originator TLA";  break;
+				}
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+				           "%s, MCU sideband 0x%X", pszReqOrig, ui32MCUSB);
+				pszTagSB = pszScratchBuf;
+			}
+			break;
+		}
+	} /* switch (TagID) */
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeMMULevel
+
+ @Description
+
+ Return the name for the MMU level that faulted.
+
+ @Input ui32MMULevel	 - MMU level
+
+ @Return   IMG_CHAR* to the string describing the MMU level that faulted.
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodeMMULevel(IMG_UINT32 ui32MMULevel)
+{
+	const IMG_CHAR* pszMMULevel = "";
+
+	switch (ui32MMULevel)
+	{
+		case 0x0: pszMMULevel = " (Page Table)"; break;
+		case 0x1: pszMMULevel = " (Page Directory)"; break;
+		case 0x2: pszMMULevel = " (Page Catalog)"; break;
+		case 0x3: pszMMULevel = " (Cat Base Reg)"; break;
+	}
+
+	return pszMMULevel;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpRGXBIFBank
+
+ @Description
+
+ Dump BIF Bank state in human readable form.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo            - RGX device info
+ @Input eBankID              - BIF identifier
+ @Input ui64MMUStatus        - MMU Status register value
+ @Input ui64ReqStatus        - BIF request Status register value
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpRGXBIFBank(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					RGXDBG_BIF_ID eBankID,
+					IMG_UINT64 ui64MMUStatus,
+					IMG_UINT64 ui64ReqStatus,
+					const IMG_CHAR *pszIndent)
+{
+	if (ui64MMUStatus == 0x0)
+	{
+		PVR_DUMPDEBUG_LOG("%s - OK", pszBIFNames[eBankID]);
+	}
+	else
+	{
+		IMG_UINT32 ui32PageSize;
+		IMG_UINT32 ui32PC =
+			(ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+				RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+
+		/* Bank 0 & 1 share the same fields */
+		PVR_DUMPDEBUG_LOG("%s%s - FAULT:",
+						  pszIndent,
+						  pszBIFNames[eBankID]);
+
+		/* MMU Status */
+		{
+			IMG_UINT32 ui32MMUDataType =
+				(ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT;
+
+			IMG_BOOL bROFault = (ui64MMUStatus & RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN) != 0;
+			IMG_BOOL bProtFault = (ui64MMUStatus & RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN) != 0;
+
+			ui32PageSize = (ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+						RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+
+			PVR_DUMPDEBUG_LOG("%s  * MMU status (0x%016" IMG_UINT64_FMTSPECx "): PC = %d%s, Page Size = %d%s%s%s.",
+						pszIndent,
+						ui64MMUStatus,
+						ui32PC,
+						(ui32PC < 0x8)?"":_RGXDecodePMPC(ui32PC),
+						ui32PageSize,
+						(bROFault)?", Read Only fault":"",
+						(bProtFault)?", PM/META protection fault":"",
+						_RGXDecodeMMULevel(ui32MMUDataType));
+		}
+
+		/* Req Status */
+		{
+			IMG_CHAR *pszTagID;
+			IMG_CHAR *pszTagSB;
+			IMG_CHAR aszScratch[RGX_DEBUG_STR_SIZE];
+			IMG_BOOL bRead;
+			IMG_UINT32 ui32TagSB, ui32TagID;
+			IMG_UINT64 ui64Addr;
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+			{
+				bRead = (ui64ReqStatus & RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN) != 0;
+				ui32TagSB = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT;
+				ui32TagID = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT;
+			}
+			else
+			{
+				bRead = (ui64ReqStatus & RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN) != 0;
+				ui32TagSB = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT;
+				ui32TagID = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT;
+			}
+			ui64Addr = ((ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK) >>
+				RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT) <<
+				RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT;
+
+			_RGXDecodeBIFReqTags(psDevInfo, eBankID, ui32TagID, ui32TagSB, &pszTagID, &pszTagSB, &aszScratch[0], RGX_DEBUG_STR_SIZE);
+
+			PVR_DUMPDEBUG_LOG("%s  * Request (0x%016" IMG_UINT64_FMTSPECx
+						"): %s (%s), %s " IMG_DEV_VIRTADDR_FMTSPEC ".",
+							  pszIndent,
+							  ui64ReqStatus,
+			                  pszTagID,
+			                  pszTagSB,
+			                  (bRead)?"Reading from":"Writing to",
+			                  ui64Addr);
+		}
+	}
+}
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT mismatch!");
+
+static_assert((RGX_CR_MMU_FAULT_STATUS_CONTEXT_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_CONTEXT_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_ADDRESS_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_ADDRESS_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TAG_SB_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TAG_SB_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_REQ_ID_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_REQ_ID_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_LEVEL_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_LEVEL_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_RNW_EN == RGX_CR_MMU_FAULT_STATUS_META_RNW_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_FAULT_EN == RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+
+
+#if !defined(NO_HARDWARE)
+static inline IMG_CHAR const *_GetRISCVException(IMG_UINT32 ui32Mcause)
+{
+	switch (ui32Mcause)
+	{
+#define X(value, fatal, description) \
+		case value: \
+			if (fatal) \
+				return description; \
+			return NULL;
+
+		RGXRISCVFW_MCAUSE_TABLE
+#undef X
+
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "Invalid RISC-V FW mcause value 0x%08x", ui32Mcause));
+			return NULL;
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+static const IMG_FLAGS2DESC asHWErrorState[] =
+{
+	{RGX_HW_ERR_NA, "N/A"},
+	{RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL, "Primitive ID failure during DM kill."},
+};
+
+/*
+ *  Translate ID code to descriptive string.
+ *  Returns on the first match.
+ */
+static void _ID2Description(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, const IMG_FLAGS2DESC *psConvTable, IMG_UINT32 ui32TableSize, IMG_UINT32 ui32ID)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (ui32ID == psConvTable[ui32Idx].uiFlag)
+		{
+			OSStringSafeCopy(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+			return;
+		}
+	}
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWAssert
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psRGXFWIfTraceBufCtl - RGX FW trace buffer
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWAssert(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl)
+{
+	const IMG_CHAR *pszTraceAssertPath;
+	const IMG_CHAR *pszTraceAssertInfo;
+	IMG_INT32 ui32TraceAssertLine;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf, INVALIDATE);
+		pszTraceAssertPath = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szPath;
+		pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szInfo;
+		ui32TraceAssertLine = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.ui32LineNum;
+
+		/* print non-null assert strings */
+		if (*pszTraceAssertInfo)
+		{
+			PVR_DUMPDEBUG_LOG("FW-T%d Assert: %.*s (%.*s:%d)",
+			                  i, RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertPath, ui32TraceAssertLine);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWFaults
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psFwSysData       - RGX FW shared system data
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWFaults(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             const RGXFWIF_SYSDATA *psFwSysData)
+{
+	if (psFwSysData->ui32FWFaults > 0)
+	{
+		IMG_UINT32	ui32StartFault = psFwSysData->ui32FWFaults - RGXFWIF_FWFAULTINFO_MAX;
+		IMG_UINT32	ui32EndFault   = psFwSysData->ui32FWFaults - 1;
+		IMG_UINT32  ui32Index;
+
+		if (psFwSysData->ui32FWFaults < RGXFWIF_FWFAULTINFO_MAX)
+		{
+			ui32StartFault = 0;
+		}
+
+		for (ui32Index = ui32StartFault; ui32Index <= ui32EndFault; ui32Index++)
+		{
+			const RGX_FWFAULTINFO *psFaultInfo = &psFwSysData->sFaultInfo[ui32Index % RGXFWIF_FWFAULTINFO_MAX];
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			/* Split OS timestamp in seconds and nanoseconds */
+			RGXConvertOSTimestampToSAndNS(psFaultInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+			PVR_DUMPDEBUG_LOG("FW Fault %d: %.*s (%.*s:%d)",
+			                  ui32Index+1, RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szPath,
+			                  psFaultInfo->sFaultBuf.ui32LineNum);
+			PVR_DUMPDEBUG_LOG("            Data = 0x%016"IMG_UINT64_FMTSPECx", CRTimer = 0x%012"IMG_UINT64_FMTSPECx", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+			                  psFaultInfo->ui64Data,
+			                  psFaultInfo->ui64CRTimer,
+			                  ui64Seconds, ui64Nanoseconds);
+		}
+	}
+}
+
+static void _RGXDumpFWPoll(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_SYSDATA *psFwSysData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psFwSysData->aui32CrPollAddr[i])
+		{
+			PVR_DUMPDEBUG_LOG("T%u polling %s (reg:0x%08X mask:0x%08X)",
+			                  i,
+			                  ((psFwSysData->aui32CrPollAddr[i] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+			                  psFwSysData->aui32CrPollAddr[i] & ~RGXFW_POLL_TYPE_SET,
+			                  psFwSysData->aui32CrPollMask[i]);
+		}
+	}
+
+}
+
+static void _RGXDumpFWHWRInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							  void *pvDumpDebugFile,
+							  const RGXFWIF_SYSDATA *psFwSysData,
+							  const RGXFWIF_HWRINFOBUF *psHWRInfoBuf,
+							  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL          bAnyLocked = IMG_FALSE;
+	IMG_UINT32        dm, i;
+	IMG_UINT32        ui32LineSize;
+	IMG_CHAR          *pszLine, *pszTemp;
+	const IMG_CHAR    *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+	const IMG_CHAR    szMsgHeader[] = "Number of HWR: ";
+	const IMG_CHAR    szMsgFalse[] = "FALSE(";
+	IMG_CHAR          *pszLockupType = "";
+	const IMG_UINT32  ui32MsgHeaderCharCount = ARRAY_SIZE(szMsgHeader) - 1; /* size includes the null */
+	const IMG_UINT32  ui32MsgFalseCharCount = ARRAY_SIZE(szMsgFalse) - 1;
+	IMG_UINT32        ui32HWRRecoveryFlags;
+	IMG_UINT32        ui32ReadIndex;
+
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+	{
+		apszDmNames[RGXFWIF_DM_TDM] = "2D";
+	}
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] ||
+		    psHWRInfoBuf->aui32HwrDmOverranCount[dm])
+		{
+			bAnyLocked = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) && !bAnyLocked && (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_HARDWARE_OK))
+	{
+		/* No HWR situation, print nothing */
+		return;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_BOOL bAnyHWROccurred = IMG_FALSE;
+
+		for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+		{
+			if (psHWRInfoBuf->aui32HwrDmRecoveredCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm] !=0)
+				{
+					bAnyHWROccurred = IMG_TRUE;
+					break;
+				}
+		}
+
+		if (!bAnyHWROccurred)
+		{
+			return;
+		}
+	}
+
+/* <DM name + left parenthesis> + <UINT32 max num of digits> + <slash> + <UINT32 max num of digits> +
+   <plus> + <UINT32 max num of digits> + <right parenthesis + comma + space> */
+#define FWHWRINFO_DM_STR_SIZE (5U + 10U + 1U + 10U + 1U + 10U + 3U)
+
+	ui32LineSize = sizeof(IMG_CHAR) * (
+			ui32MsgHeaderCharCount +
+			(psDevInfo->sDevFeatureCfg.ui32MAXDMCount * FWHWRINFO_DM_STR_SIZE) +
+			ui32MsgFalseCharCount + 1 + (psDevInfo->sDevFeatureCfg.ui32MAXDMCount*6) + 1
+				/* 'FALSE(' + ')' + (UINT16 max num + comma) per DM + \0 */
+			);
+
+	pszLine = OSAllocMem(ui32LineSize);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Out of mem allocating line string (size: %d)",
+			__func__,
+			ui32LineSize));
+		return;
+	}
+
+	OSStringSafeCopy(pszLine, szMsgHeader, ui32LineSize);
+	pszTemp = pszLine + ui32MsgHeaderCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				FWHWRINFO_DM_STR_SIZE,
+				"%s(%u/%u+%u), ",
+				apszDmNames[dm],
+				psHWRInfoBuf->aui32HwrDmRecoveredCount[dm],
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm],
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm]);
+	}
+
+	OSStringLCat(pszLine, szMsgFalse, ui32LineSize);
+	pszTemp += ui32MsgFalseCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				10 + 1 + 1 /* UINT32 max num + comma + \0 */,
+				(dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1 ? "%u," : "%u)"),
+				psHWRInfoBuf->aui32HwrDmFalseDetectCount[dm]);
+	}
+
+	PVR_DUMPDEBUG_LOG("%s", pszLine);
+
+	OSFreeMem(pszLine);
+
+	/* Print out per HWR info */
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (dm == RGXFWIF_DM_GP)
+		{
+			PVR_DUMPDEBUG_LOG("DM %d (GP)", dm);
+		}
+		else
+		{
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+			{
+				IMG_UINT32 ui32HWRRecoveryFlags = psFwSysData->aui32HWRRecoveryFlags[dm];
+				IMG_CHAR sPerDmHwrDescription[RGX_DEBUG_STR_SIZE];
+				sPerDmHwrDescription[0] = '\0';
+
+				if (ui32HWRRecoveryFlags == RGXFWIF_DM_STATE_WORKING)
+				{
+					OSStringSafeCopy(sPerDmHwrDescription, " working;", RGX_DEBUG_STR_SIZE);
+				}
+				else
+				{
+					DebugCommonFlagStrings(sPerDmHwrDescription, RGX_DEBUG_STR_SIZE,
+						asDmState2Description, ARRAY_SIZE(asDmState2Description),
+						ui32HWRRecoveryFlags);
+				}
+				PVR_DUMPDEBUG_LOG("DM %d (HWRflags 0x%08x:%.*s)", dm, ui32HWRRecoveryFlags,
+								  RGX_DEBUG_STR_SIZE, sPerDmHwrDescription);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("DM %d", dm);
+			}
+		}
+
+		ui32ReadIndex = 0;
+		for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+		{
+			IMG_BOOL bPMFault = IMG_FALSE;
+			IMG_UINT32 ui32PC;
+			IMG_UINT32 ui32PageSize = 0;
+			IMG_DEV_PHYADDR sPCDevPAddr = { 0 };
+			const RGX_HWRINFO *psHWRInfo = &psHWRInfoBuf->sHWRInfo[ui32ReadIndex];
+
+			if (ui32ReadIndex >= RGXFWIF_HWINFO_MAX)
+			{
+				PVR_DUMPDEBUG_LOG("HWINFO index error: %u", ui32ReadIndex);
+				break;
+			}
+
+			if ((psHWRInfo->eDM == dm) && (psHWRInfo->ui32HWRNumber != 0))
+			{
+				IMG_CHAR aui8RecoveryNum[10+10+1];
+				IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+				IMG_BOOL bPageFault = IMG_FALSE;
+				IMG_DEV_VIRTADDR sFaultDevVAddr;
+
+				/* Split OS timestamp in seconds and nanoseconds */
+				RGXConvertOSTimestampToSAndNS(psHWRInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+				ui32HWRRecoveryFlags = psHWRInfo->ui32HWRRecoveryFlags;
+				if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_LOCKUP) { pszLockupType = ", Guilty Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_LOCKUP) { pszLockupType = ", Innocent Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_OVERRUNING) { pszLockupType = ", Guilty Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_OVERRUNING) { pszLockupType = ", Innocent Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH) { pszLockupType = ", Hard Context Switch"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_ECC_HWR) { pszLockupType = ", GPU ECC HWR"; }
+
+				OSSNPrintf(aui8RecoveryNum, sizeof(aui8RecoveryNum), "Recovery %d:", psHWRInfo->ui32HWRNumber);
+				if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+				{
+					PVR_DUMPDEBUG_LOG("  %s Core = %u, PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32CoreID,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+
+				if (psHWRInfo->eHWErrorCode != RGX_HW_ERR_NA)
+				{
+					IMG_CHAR sHWDebugInfo[RGX_DEBUG_STR_SIZE] = "";
+
+					_ID2Description(sHWDebugInfo, RGX_DEBUG_STR_SIZE, asHWErrorState, ARRAY_SIZE(asHWErrorState),
+						psHWRInfo->eHWErrorCode);
+					PVR_DUMPDEBUG_LOG("  HW error code = 0x%X: %s",
+									  psHWRInfo->eHWErrorCode, sHWDebugInfo);
+				}
+
+				pszTemp = &aui8RecoveryNum[0];
+				while (*pszTemp != '\0')
+				{
+					*pszTemp++ = ' ';
+				}
+
+				/* There's currently no time correlation for the Guest OSes on the Firmware so there's no point printing OS Timestamps on Guests */
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   ui64Seconds,
+									   ui64Nanoseconds,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+
+				if (psHWRInfo->ui64CRTimeHWResetFinish != 0)
+				{
+					if (psHWRInfo->ui64CRTimeFreelistReady != 0)
+					{
+						/* If ui64CRTimeFreelistReady is less than ui64CRTimeHWResetFinish it means APM kicked in and the time is not valid. */
+						if (psHWRInfo->ui64CRTimeHWResetFinish < psHWRInfo->ui64CRTimeFreelistReady)
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalRecoveryTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimeHWResetFinish)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimer)*256);
+						}
+						else
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = <not_timed>, TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+						}
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+										   aui8RecoveryNum,
+										   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+					}
+				}
+
+#if defined(HW_ERN_65104_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 65104))
+				{
+					PVR_DUMPDEBUG_LOG("    Active PDS DM USCs = 0x%08x", psHWRInfo->ui32PDSActiveDMUSCs);
+				}
+#endif
+
+#if defined(HW_ERN_69700_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 69700))
+				{
+					PVR_DUMPDEBUG_LOG("    DMs stalled waiting on PDS Store space = 0x%08x", psHWRInfo->ui32PDSStalledDMs);
+				}
+#endif
+
+				switch (psHWRInfo->eHWRType)
+				{
+					case RGX_HWRTYPE_BIF0FAULT:
+					case RGX_HWRTYPE_BIF1FAULT:
+					{
+						_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXFWIF_HWRTYPE_BIF_BANK_GET(psHWRInfo->eHWRType),
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+										DD_NORMAL_INDENT);
+
+						bPageFault = IMG_TRUE;
+						sFaultDevVAddr.uiAddr = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK);
+						ui32PC = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+								RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+						bPMFault = (ui32PC >= 8);
+						ui32PageSize = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+									RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+						sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+					}
+					break;
+					case RGX_HWRTYPE_TEXASBIF0FAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+						{
+							_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+										DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							sFaultDevVAddr.uiAddr = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK);
+							ui32PC = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+									RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+							bPMFault = (ui32PC >= 8);
+							ui32PageSize = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+										RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_ECCFAULT:
+					{
+						PVR_DUMPDEBUG_LOG("    ECC fault GPU=0x%08x", psHWRInfo->uHWRData.sECCInfo.ui32FaultGPU);
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUFAULT:
+					{
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUMETAFAULT:
+					{
+					}
+					break;
+
+					case RGX_HWRTYPE_POLLFAILURE:
+					{
+						PVR_DUMPDEBUG_LOG("    T%u polling %s (reg:0x%08X mask:0x%08X last:0x%08X)",
+										  psHWRInfo->uHWRData.sPollInfo.ui32ThreadNum,
+										  ((psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & ~RGXFW_POLL_TYPE_SET,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollMask,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollLastValue);
+					}
+					break;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+					case RGX_HWRTYPE_MIPSTLBFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+						{
+							IMG_UINT32 ui32EntryLo = psHWRInfo->uHWRData.sTLBInfo.ui32EntryLo;
+
+							/* This is not exactly what the MMU code does, but the result should be the same */
+							const IMG_UINT32 ui32UnmappedEntry =
+								((IMG_UINT32)(MMU_BAD_PHYS_ADDR & 0xffffffff) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) | RGXMIPSFW_ENTRYLO_UNCACHED;
+
+							PVR_DUMPDEBUG_LOG("    MIPS TLB fault: BadVA = 0x%08X, EntryLo = 0x%08X"
+											  " (page PA 0x%" IMG_UINT64_FMTSPECx", V %u)",
+											  psHWRInfo->uHWRData.sTLBInfo.ui32BadVAddr,
+											  ui32EntryLo,
+											  RGXMIPSFW_TLB_GET_PA(ui32EntryLo),
+											  ui32EntryLo & RGXMIPSFW_TLB_VALID ? 1 : 0);
+
+							if (ui32EntryLo == ui32UnmappedEntry)
+							{
+								PVR_DUMPDEBUG_LOG("    Potential use-after-free detected");
+							}
+						}
+					}
+					break;
+#endif
+
+					case RGX_HWRTYPE_MMURISCVFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+						{
+							_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_FWCORE,
+											psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+											psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+											DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							bPMFault = IMG_FALSE;
+							sFaultDevVAddr.uiAddr =
+								(psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus &
+								 ~RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK);
+							ui32PageSize =
+								(psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus &
+								 ~RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+								RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_SHIFT;
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_OVERRUN:
+					case RGX_HWRTYPE_UNKNOWNFAILURE:
+					{
+						/* Nothing to dump */
+					}
+					break;
+
+					default:
+					{
+						PVR_DUMPDEBUG_LOG("    Unknown HWR Info type: 0x%x", psHWRInfo->eHWRType);
+					}
+					break;
+				}
+
+				if (bPageFault)
+				{
+					RGXDumpFaultInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psHWRInfo,
+					                 ui32ReadIndex, &sFaultDevVAddr, &sPCDevPAddr, bPMFault, ui32PageSize);
+				}
+
+			}
+
+			if (ui32ReadIndex == RGXFWIF_HWINFO_MAX_FIRST - 1)
+				ui32ReadIndex = psHWRInfoBuf->ui32WriteIndex;
+			else
+				ui32ReadIndex = (ui32ReadIndex + 1) - (ui32ReadIndex / RGXFWIF_HWINFO_LAST_INDEX) * RGXFWIF_HWINFO_MAX_LAST;
+		}
+	}
+}
+
+
+
+
+#if !defined(NO_HARDWARE)
+
+/*!
+*******************************************************************************
+
+ @Function	_CheckForPendingPage
+
+ @Description
+
+ Check if the MMU indicates it is blocked on a pending page
+ MMU4 does not support pending pages, so return false.
+
+ @Input psDevInfo	 - RGX device info
+
+ @Return   IMG_BOOL      - IMG_TRUE if there is a pending page
+
+******************************************************************************/
+static INLINE IMG_BOOL _CheckForPendingPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* MMU4 doesn't support pending pages */
+	return (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) < 4) &&
+		   (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_ENTRY) & RGX_CR_MMU_ENTRY_PENDING_EN);
+#else
+	IMG_UINT32 ui32BIFMMUEntry;
+
+	ui32BIFMMUEntry = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY);
+
+	if (ui32BIFMMUEntry & RGX_CR_BIF_MMU_ENTRY_PENDING_EN)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetPendingPageInfo
+
+ @Description
+
+ Get information about the pending page from the MMU status registers
+
+ @Input psDevInfo	 - RGX device info
+ @Output psDevVAddr      - The device virtual address of the pending MMU address translation
+ @Output pui32CatBase    - The page catalog base
+
+ @Return   void
+
+******************************************************************************/
+static void _GetPendingPageInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_VIRTADDR *psDevVAddr,
+								IMG_UINT32 *pui32CatBase)
+{
+	IMG_UINT64 ui64BIFMMUEntryStatus;
+
+	ui64BIFMMUEntryStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY_STATUS);
+
+	psDevVAddr->uiAddr = (ui64BIFMMUEntryStatus & ~RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_CLRMSK);
+
+	*pui32CatBase = (ui64BIFMMUEntryStatus & ~RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_CLRMSK) >>
+								RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_SHIFT;
+}
+
+#endif
+
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON)
+{
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	IMG_UINT32 ui32DriverID;
+	const RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	/* space for the current clock speed and 3 previous */
+	RGXFWIF_TIME_CORR asTimeCorrs[4];
+	IMG_UINT32 ui32NumClockSpeedChanges;
+
+	/* Should invalidate all reads below including when passed to functions. */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, INVALIDATE);
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(bRGXPoweredON);
+#else
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		{
+			IMG_UINT64	ui64RegValMMUStatus, ui64RegValREQStatus;
+
+			ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK0_MMU_STATUS);
+			ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK0_REQ_STATUS);
+
+			_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_BIF0, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+
+			if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, SINGLE_BIF)))
+			{
+				ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK1_MMU_STATUS);
+				ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK1_REQ_STATUS);
+				_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_BIF1, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+			}
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+			{
+				ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS);
+				ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS);
+				_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_FWCORE, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+			}
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+			{
+				IMG_UINT32  ui32PhantomCnt = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) ?  RGX_REQ_NUM_PHANTOMS(RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS)) : 0;
+
+				if (ui32PhantomCnt > 1)
+				{
+					IMG_UINT32  ui32Phantom;
+					for (ui32Phantom = 0;  ui32Phantom < ui32PhantomCnt;  ui32Phantom++)
+					{
+						/* This can't be done as it may interfere with the FW... */
+						/*OSWriteHWReg64(RGX_CR_TEXAS_INDIRECT, ui32Phantom);*/
+
+						ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS);
+						ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS);
+
+						_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+					}
+				}else
+				{
+					ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS);
+					ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS);
+
+					_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+				}
+			}
+		}
+
+		if (_CheckForPendingPage(psDevInfo))
+		{
+			IMG_UINT32 ui32CatBase;
+			IMG_DEV_VIRTADDR sDevVAddr;
+
+			PVR_DUMPDEBUG_LOG("MMU Pending page: Yes");
+
+			_GetPendingPageInfo(psDevInfo, &sDevVAddr, &ui32CatBase);
+
+			if (ui32CatBase >= 8)
+			{
+				PVR_DUMPDEBUG_LOG("Cannot check address on PM cat base %u", ui32CatBase);
+			}
+			else
+			{
+				IMG_DEV_PHYADDR sPCDevPAddr;
+				MMU_FAULT_DATA sFaultData;
+				IMG_BOOL bIsValid = IMG_TRUE;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+				IMG_UINT64 ui64CBaseMapping;
+				IMG_UINT32 ui32CBaseMapCtxReg;
+
+				if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_EN);
+				}
+				else
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING_INVALID_EN);
+				}
+#else
+				sPCDevPAddr.uiAddr = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_CAT_BASEN(ui32CatBase));
+#endif
+
+				PVR_DUMPDEBUG_LOG("Checking device virtual address " IMG_DEV_VIRTADDR_FMTSPEC
+							" on cat base %u. PC Addr = 0x%" IMG_UINT64_FMTSPECx " is %s",
+								sDevVAddr.uiAddr,
+								ui32CatBase,
+								sPCDevPAddr.uiAddr,
+								bIsValid ? "valid":"invalid");
+				RGXCheckFaultAddress(psDevInfo, &sDevVAddr, &sPCDevPAddr, &sFaultData);
+				RGXDumpFaultAddressHostView(&sFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_SUMMARY_INDENT);
+			}
+		}
+	}
+#endif /* NO_HARDWARE */
+
+#if !defined(NO_HARDWARE)
+	/* Determine the type virtualisation support used */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+#if defined(SUPPORT_AUTOVZ)
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with HW register support");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with shared memory");
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with static Fw heap allocation");
+#endif /* defined(SUPPORT_AUTOVZ) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with dynamic Fw heap allocation");
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+	}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_CONNECTION_FW_STATE eFwState;
+		RGXFWIF_CONNECTION_OS_STATE eOsState;
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+		eFwState = KM_GET_FW_CONNECTION(psDevInfo);
+		eOsState = KM_GET_OS_CONNECTION(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation firmware connection state: %s (Fw=%s; OS=%s)",
+						  ((eFwState == RGXFW_CONNECTION_FW_ACTIVE) && (eOsState == RGXFW_CONNECTION_OS_ACTIVE)) ? ("UP") : ("DOWN"),
+						  (eFwState < RGXFW_CONNECTION_FW_STATE_COUNT) ? (apszFwOsStateName[eFwState]) : ("invalid"),
+						  (eOsState < RGXFW_CONNECTION_OS_STATE_COUNT) ? (apszFwOsStateName[eOsState]) : ("invalid"));
+
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32FwAliveTS;
+		IMG_UINT32 ui32OsAliveTS;
+
+		KM_ALIVE_TOKEN_CACHEOP(Fw, INVALIDATE);
+		KM_ALIVE_TOKEN_CACHEOP(Os, INVALIDATE);
+
+		ui32FwAliveTS = KM_GET_FW_ALIVE_TOKEN(psDevInfo);
+		ui32OsAliveTS = KM_GET_OS_ALIVE_TOKEN(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation watchdog timestamps (in GPU timer ticks): Fw=%u; OS=%u; diff(FW, OS) = %u",
+						  ui32FwAliveTS, ui32OsAliveTS, ui32FwAliveTS - ui32OsAliveTS);
+	}
+#endif
+#endif /* !defined(NO_HARDWARE) */
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_CHAR sHwrStateDescription[RGX_DEBUG_STR_SIZE];
+		IMG_BOOL bDriverIsolationEnabled = IMG_FALSE;
+		IMG_UINT32 ui32HostIsolationGroup;
+
+		sHwrStateDescription[0] = '\0';
+
+		DebugCommonFlagStrings(sHwrStateDescription, RGX_DEBUG_STR_SIZE,
+			asHwrState2Description, ARRAY_SIZE(asHwrState2Description),
+			psFwSysData->ui32HWRStateFlags);
+		PVR_DUMPDEBUG_LOG("RGX HWR State 0x%08x:%s", psFwSysData->ui32HWRStateFlags, sHwrStateDescription);
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: %s (APM %s: %d ok, %d denied, %d non-idle, %d retry, %d other, %d total. Latency: %u ms)",
+		                  (psFwSysData->ePowState < ARRAY_SIZE(pszPowStateName) ? pszPowStateName[psFwSysData->ePowState] : "???"),
+		                  (psDevInfo->pvAPMISRData)?"enabled":"disabled",
+		                  psDevInfo->ui32ActivePMReqOk - psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqDenied,
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqRetry,
+		                  psDevInfo->ui32ActivePMReqTotal -
+		                  psDevInfo->ui32ActivePMReqOk -
+		                  psDevInfo->ui32ActivePMReqDenied -
+		                  psDevInfo->ui32ActivePMReqRetry -
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqTotal,
+		                  psRuntimeCfg->ui32ActivePMLatencyms);
+
+		ui32NumClockSpeedChanges = (IMG_UINT32) OSAtomicRead(&psDevInfo->psDeviceNode->iNumClockSpeedChanges);
+		RGXGetTimeCorrData(psDevInfo->psDeviceNode, asTimeCorrs, ARRAY_SIZE(asTimeCorrs));
+
+		PVR_DUMPDEBUG_LOG("RGX DVFS: %u frequency changes. "
+		                  "Current frequency: %u.%03u MHz (sampled at %" IMG_UINT64_FMTSPEC " ns). "
+		                  "FW frequency: %u.%03u MHz.",
+		                  ui32NumClockSpeedChanges,
+		                  asTimeCorrs[0].ui32CoreClockSpeed / 1000000,
+		                  (asTimeCorrs[0].ui32CoreClockSpeed / 1000) % 1000,
+		                  asTimeCorrs[0].ui64OSTimeStamp,
+		                  psRuntimeCfg->ui32CoreClockSpeed / 1000000,
+		                  (psRuntimeCfg->ui32CoreClockSpeed / 1000) % 1000);
+		if (ui32NumClockSpeedChanges > 0)
+		{
+			PVR_DUMPDEBUG_LOG("          Previous frequencies: %u.%03u, %u.%03u, %u.%03u MHz (Sampled at "
+							"%" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ")",
+												asTimeCorrs[1].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[1].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[2].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[2].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[3].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[3].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[1].ui64OSTimeStamp,
+												asTimeCorrs[2].ui64OSTimeStamp,
+												asTimeCorrs[3].ui64OSTimeStamp);
+		}
+
+		ui32HostIsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID];
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			RGXFWIF_OS_RUNTIME_FLAGS sFwRunFlags = psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+			IMG_UINT32 ui32IsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID];
+			IMG_BOOL bMTSEnabled = IMG_FALSE;
+
+#if !defined(NO_HARDWARE)
+			if (bRGXPoweredON)
+			{
+				bMTSEnabled = (
+#if defined(FIX_HW_BRN_64502_BIT_MASK)
+								RGX_IS_BRN_SUPPORTED(psDevInfo, 64502) ||
+#endif
+								!RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_VIRTUALISATION)) ?
+								IMG_TRUE : ((OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE_ENABLE) & BIT(ui32DriverID)) != 0);
+			}
+#endif
+
+			PVR_DUMPDEBUG_LOG("RGX FW OS %u - State: %s; Freelists: %s%s; Priority: %u; Isolation group: %u;  Time Slice%s: %u%% (%ums); %s", ui32DriverID,
+							  apszFwOsStateName[sFwRunFlags.bfOsState],
+							  (sFwRunFlags.bfFLOk) ? "Ok" : "Not Ok",
+							  (sFwRunFlags.bfFLGrowPending) ? "; Grow Request Pending" : "",
+							  psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID],
+							  ui32IsolationGroup,
+							  (psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] != 0) ? "" : "*",
+							  psFwSysData->aui32TSMirror[ui32DriverID],
+							  (psFwSysData->aui32TSMirror[ui32DriverID] *
+							   psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs / 100),
+							  (bMTSEnabled) ? "MTS on;" : "MTS off;"
+							 );
+
+			if (ui32IsolationGroup != ui32HostIsolationGroup)
+			{
+				bDriverIsolationEnabled = IMG_TRUE;
+			}
+
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+			{
+				/* don't print guest information on native mode drivers */
+				break;
+			}
+		}
+
+#if defined(PVR_ENABLE_PHR)
+		{
+			IMG_CHAR sPHRConfigDescription[RGX_DEBUG_STR_SIZE];
+
+			sPHRConfigDescription[0] = '\0';
+			DebugCommonFlagStrings(sPHRConfigDescription, RGX_DEBUG_STR_SIZE,
+			                   asPHRConfig2Description, ARRAY_SIZE(asPHRConfig2Description),
+			                   BIT_ULL(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode));
+
+			PVR_DUMPDEBUG_LOG("RGX PHR configuration: (%d) %.*s", psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, RGX_DEBUG_STR_SIZE, sPHRConfigDescription);
+		}
+#endif
+
+		if (bRGXPoweredON && RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			if (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM) > 1U)
+			{
+				PVR_DUMPDEBUG_LOG("RGX MC Configuration: 0x%X (1:primary, 0:secondary)", psFwSysData->ui32McConfig);
+			}
+		}
+
+		if (bDriverIsolationEnabled)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Hard Context Switch deadline: %u ms", psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+		}
+
+		_RGXDumpFWAssert(pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl);
+		_RGXDumpFWFaults(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+		_RGXDumpFWPoll(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("RGX FW State: Unavailable under Guest Mode of operation");
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: Unavailable under Guest Mode of operation");
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+	_RGXDumpFWHWRInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData, psDevInfo->psRGXFWIfHWRInfoBufCtl, psDevInfo);
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK)
+	/* Dump all non-zero values in lines of 8... */
+	{
+		IMG_CHAR    pszLine[(9*RGXFWIF_STATS_FRAMEWORK_LINESIZE)+1];
+		const IMG_UINT32 *pui32FWStatsBuf = psFwSysData->aui32FWStatsBuf;
+		IMG_UINT32  ui32Index1, ui32Index2;
+
+		PVR_DUMPDEBUG_LOG("STATS[START]: RGXFWIF_STATS_FRAMEWORK_MAX=%d", RGXFWIF_STATS_FRAMEWORK_MAX);
+		for (ui32Index1 = 0;  ui32Index1 < RGXFWIF_STATS_FRAMEWORK_MAX;  ui32Index1 += RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+		{
+			IMG_UINT32  ui32OrOfValues = 0;
+			IMG_CHAR    *pszBuf = pszLine;
+
+			/* Print all values in this line and skip if all zero... */
+			for (ui32Index2 = 0;  ui32Index2 < RGXFWIF_STATS_FRAMEWORK_LINESIZE;  ui32Index2++)
+			{
+				ui32OrOfValues |= pui32FWStatsBuf[ui32Index1+ui32Index2];
+				OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32FWStatsBuf[ui32Index1+ui32Index2]);
+				pszBuf += 9; /* write over the '\0' */
+			}
+
+			if (ui32OrOfValues != 0)
+			{
+				PVR_DUMPDEBUG_LOG("STATS[%08x]:%s", ui32Index1, pszLine);
+			}
+		}
+		PVR_DUMPDEBUG_LOG("STATS[END]");
+	}
+#endif
+}
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	RGXRISCVFW_STATE sRiscvState;
+	const IMG_CHAR *pszException;
+	PVRSRV_ERROR eError;
+
+	DDLOG64(FWCORE_MEM_CAT_BASE0);
+	DDLOG64(FWCORE_MEM_CAT_BASE1);
+	DDLOG64(FWCORE_MEM_CAT_BASE2);
+	DDLOG64(FWCORE_MEM_CAT_BASE3);
+	DDLOG64(FWCORE_MEM_CAT_BASE4);
+	DDLOG64(FWCORE_MEM_CAT_BASE5);
+	DDLOG64(FWCORE_MEM_CAT_BASE6);
+	DDLOG64(FWCORE_MEM_CAT_BASE7);
+
+	/* Limit dump to what is currently being used */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4);
+	}
+	else
+#endif
+	{
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG4);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG5);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG6);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG12);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG13);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG14);
+	}
+
+	DDLOG32(FWCORE_MEM_FAULT_MMU_STATUS);
+	DDLOG64(FWCORE_MEM_FAULT_REQ_STATUS);
+	DDLOG32(FWCORE_MEM_MMU_STATUS);
+	DDLOG32(FWCORE_MEM_READS_EXT_STATUS);
+	DDLOG32(FWCORE_MEM_READS_INT_STATUS);
+
+	PVR_DUMPDEBUG_LOG("---- [ RISC-V internal state ] ----");
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGXRiscvIsHalted(psDevInfo))
+	{
+		/* Avoid resuming the RISC-V FW as most operations
+		 * on the debug module require a halted core */
+		PVR_DUMPDEBUG_LOG("(skipping as RISC-V found halted)");
+		return PVRSRV_OK;
+	}
+#endif
+
+	eError = RGXRiscvHalt(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+#define X(name, address)												\
+	eError = RGXRiscvReadReg(psDevInfo, address, &sRiscvState.name);	\
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXRiscvReadReg", _RISCVDMError);	\
+	DDLOGVAL32(#name, sRiscvState.name);
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+
+	eError = RGXRiscvResume(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+	pszException = _GetRISCVException(sRiscvState.mcause);
+	if (pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG("RISC-V FW hit an exception: %s", pszException);
+
+		eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+		}
+	}
+
+	return PVRSRV_OK;
+
+_RISCVDMError:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to communicate with the Debug Module"));
+
+	return eError;
+}
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBVNC_COREID_REG))
+	{
+		DDLOG64(CORE_ID__PBVNC);
+	}
+	else
+	{
+		DDLOG32(CORE_ID);
+		DDLOG32(CORE_REVISION);
+	}
+	DDLOG32(DESIGNER_REV_FIELD1);
+	DDLOG32(DESIGNER_REV_FIELD2);
+	DDLOG64(CHANGESET_NUMBER);
+}
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(MULTICORE_SYSTEM);
+	DDLOG32(MULTICORE_GPU);
+}
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG64(CLK_CTRL);
+	DDLOG64(CLK_STATUS);
+	DDLOG64(CLK_CTRL2);
+	DDLOG64(CLK_STATUS2);
+}
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	{
+		DDLOG32(BIF_FAULT_BANK0_MMU_STATUS);
+		DDLOG64(BIF_FAULT_BANK0_REQ_STATUS);
+		DDLOG32(BIF_FAULT_BANK1_MMU_STATUS);
+		DDLOG64(BIF_FAULT_BANK1_REQ_STATUS);
+	}
+	DDLOG32(BIF_MMU_STATUS);
+	DDLOG32(BIF_MMU_ENTRY);
+	DDLOG64(BIF_MMU_ENTRY_STATUS);
+
+	{
+		if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE)))
+		{
+			DDLOG32(BIF_STATUS_MMU);
+			DDLOG32(BIF_READS_EXT_STATUS);
+			DDLOG32(BIF_READS_INT_STATUS);
+		}
+		DDLOG32(BIFPM_STATUS_MMU);
+		DDLOG32(BIFPM_READS_EXT_STATUS);
+		DDLOG32(BIFPM_READS_INT_STATUS);
+	}
+
+	{
+		DDLOG64(BIF_CAT_BASE_INDEX);
+		DDLOG64(BIF_CAT_BASE0);
+		DDLOG64(BIF_CAT_BASE1);
+		DDLOG64(BIF_CAT_BASE2);
+		DDLOG64(BIF_CAT_BASE3);
+		DDLOG64(BIF_CAT_BASE4);
+		DDLOG64(BIF_CAT_BASE5);
+		DDLOG64(BIF_CAT_BASE6);
+		DDLOG64(BIF_CAT_BASE7);
+	}
+
+	DDLOG32(BIF_CTRL_INVAL);
+	DDLOG32(BIF_CTRL);
+}
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	IMG_UINT32   ui32TACycles, ui323DCycles, ui32TAOr3DCycles, ui32TAAnd3DCycles;
+
+	DDLOG64(BIF_PM_CAT_BASE_VCE0);
+	DDLOG64(BIF_PM_CAT_BASE_TE0);
+	DDLOG64(BIF_PM_CAT_BASE_ALIST0);
+	DDLOG64(BIF_PM_CAT_BASE_VCE1);
+	DDLOG64(BIF_PM_CAT_BASE_TE1);
+	DDLOG64(BIF_PM_CAT_BASE_ALIST1);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		DDLOG32(MULTICORE_GEOMETRY_CTRL_COMMON);
+		DDLOG32(MULTICORE_FRAGMENT_CTRL_COMMON);
+		DDLOG32(MULTICORE_COMPUTE_CTRL_COMMON);
+	}
+
+	DDLOG32(PERF_TA_PHASE);
+	DDLOG32(PERF_TA_CYCLE);
+	DDLOG32(PERF_3D_PHASE);
+	DDLOG32(PERF_3D_CYCLE);
+
+	ui32TACycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_TA_CYCLE);
+	ui323DCycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_3D_CYCLE);
+	ui32TAOr3DCycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_TA_OR_3D_CYCLE);
+	ui32TAAnd3DCycles = ((ui32TACycles + ui323DCycles) > ui32TAOr3DCycles) ? (ui32TACycles + ui323DCycles - ui32TAOr3DCycles) : 0;
+	DDLOGVAL32("PERF_TA_OR_3D_CYCLE", ui32TAOr3DCycles);
+	DDLOGVAL32("PERF_TA_AND_3D_CYCLE", ui32TAAnd3DCycles);
+
+	DDLOG32(PERF_COMPUTE_PHASE);
+	DDLOG32(PERF_COMPUTE_CYCLE);
+
+	DDLOG32(PM_PARTIAL_RENDER_ENABLE);
+
+	DDLOG32(ISP_RENDER);
+	DDLOG64(TLA_STATUS);
+	DDLOG64(MCU_FENCE);
+
+	DDLOG32(VDM_CONTEXT_STORE_STATUS);
+	DDLOG64(VDM_CONTEXT_STORE_TASK0);
+	DDLOG64(VDM_CONTEXT_STORE_TASK1);
+	DDLOG64(VDM_CONTEXT_STORE_TASK2);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK0);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK1);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK2);
+
+	DDLOG32(ISP_CTL);
+	DDLOG32(ISP_STATUS);
+
+	DDLOG32(CDM_CONTEXT_STORE_STATUS);
+	DDLOG64(CDM_CONTEXT_PDS0);
+	DDLOG64(CDM_CONTEXT_PDS1);
+	DDLOG64(CDM_TERMINATE_PDS);
+	DDLOG64(CDM_TERMINATE_PDS1);
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 47025))
+	{
+		DDLOG64(CDM_CONTEXT_LOAD_PDS0);
+		DDLOG64(CDM_CONTEXT_LOAD_PDS1);
+	}
+}
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	{
+		DDLOG32(SLC_IDLE);
+		DDLOG32(SLC_STATUS0);
+		DDLOG64(SLC_STATUS1);
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_BANKS) && RGX_GET_FEATURE_VALUE(psDevInfo, SLC_BANKS))
+		{
+			DDLOG64(SLC_STATUS2);
+		}
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+		{
+			DDLOG64(SLC_CTRL_BYPASS);
+		}
+		else
+		{
+			DDLOG32(SLC_CTRL_BYPASS);
+		}
+		DDLOG64(SLC_CTRL_MISC);
+	}
+}
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(SIDEKICK_IDLE);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ROGUEXE) &&
+		RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER))
+	{
+		DDLOG32(SAFETY_EVENT_STATUS__ROGUEXE);
+		DDLOG32(MTS_SAFETY_EVENT_ENABLE__ROGUEXE);
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER))
+	{
+		DDLOG32(FWCORE_WDT_CTRL);
+	}
+
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevInfo->psDeviceNode, LAYOUT_MARS) > 0)
+	{
+		DDLOG32(SCRATCH0);
+		DDLOG32(SCRATCH1);
+		DDLOG32(SCRATCH2);
+		DDLOG32(SCRATCH3);
+		DDLOG32(SCRATCH4);
+		DDLOG32(SCRATCH5);
+		DDLOG32(SCRATCH6);
+		DDLOG32(SCRATCH7);
+		DDLOG32(SCRATCH8);
+		DDLOG32(SCRATCH9);
+		DDLOG32(SCRATCH10);
+		DDLOG32(SCRATCH11);
+		DDLOG32(SCRATCH12);
+		DDLOG32(SCRATCH13);
+		DDLOG32(SCRATCH14);
+		DDLOG32(SCRATCH15);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#undef REG32_FMTSPEC
+#undef REG64_FMTSPEC
+#undef DDLOG32
+#undef DDLOG64
+#undef DDLOG32_DPX
+#undef DDLOG64_DPX
+#undef DDLOGVAL32
+
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32VerbLevel)
+{
+	DumpTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	DumpRenderCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DumpKickSyncCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		DumpComputeCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		DumpTDMTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+}
+
+/******************************************************************************
+ End of file (rgxdebug.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdevice.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdevice.h
new file mode 100644
index 000000000000..95e0a0b53cca
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxdevice.h
@@ -0,0 +1,938 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX device node header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX device node
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEVICE_H)
+#define RGXDEVICE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_device_types.h"
+#include "mmu_common.h"
+#include "rgx_fwif_km.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "osfunc.h"
+#include "rgxlayer_impl.h"
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "hash.h"
+#endif
+typedef struct _RGX_SERVER_COMMON_CONTEXT_ RGX_SERVER_COMMON_CONTEXT;
+
+typedef struct {
+	DEVMEM_MEMDESC		*psFWFrameworkMemDesc;
+} RGX_COMMON_CONTEXT_INFO;
+
+
+/*!
+ ******************************************************************************
+ * Device state flags
+ *****************************************************************************/
+#define RGXKM_DEVICE_STATE_ZERO_FREELIST                          (0x1)  /*!< Zeroing the physical pages of reconstructed free lists */
+#define RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN                  (0x2)  /*!< Used to disable the Devices Watchdog logging */
+#define RGXKM_DEVICE_STATE_GPU_UNITS_POWER_CHANGE_EN              (0x4)  /*!< Used for validation to inject power units state change every DM kick */
+#define RGXKM_DEVICE_STATE_CCB_GROW_EN                            (0x8)  /*!< Used to indicate CCB grow is permitted */
+#define RGXKM_DEVICE_STATE_ENABLE_SPU_UNITS_POWER_MASK_CHANGE_EN  (0x10) /*!< Used for validation to enable SPU power state mask change */
+#define RGXKM_DEVICE_STATE_MASK                                   (0x1F)
+
+/*!
+ ******************************************************************************
+ * ECC RAM Fault Validation
+ *****************************************************************************/
+#define RGXKM_ECC_ERR_INJ_DISABLE 0
+#define RGXKM_ECC_ERR_INJ_SLC     1
+#define RGXKM_ECC_ERR_INJ_USC     2
+#define RGXKM_ECC_ERR_INJ_TPU     3
+#define RGXKM_ECC_ERR_INJ_RASCAL  4
+#define RGXKM_ECC_ERR_INJ_MARS    5
+
+#define RGXKM_ECC_ERR_INJ_INTERVAL 10U
+
+/*!
+ ******************************************************************************
+ * GPU DVFS Table
+ *****************************************************************************/
+
+#define RGX_GPU_DVFS_TABLE_SIZE                      32
+#define RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US       25000     /* Time required to calibrate a clock frequency the first time */
+#define RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US  150000    /* Time required for a recalibration after a DVFS transition */
+#define RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US    10000000  /* Time before the next periodic calibration and correlation */
+
+
+/*!
+ ******************************************************************************
+ * Global flags for driver validation
+ *****************************************************************************/
+#define RGX_VAL_FBDC_SIG_CHECK_NOERR_EN           (0x2U)  /*!< Enable FBDC signature check. Signatures must match */
+#define RGX_VAL_FBDC_SIG_CHECK_ERR_EN             (0x4U)  /*!< Enable FBDC signature check. Signatures must not match */
+#define RGX_VAL_GPUSTATEPIN_EN                    (0x8U)  /*!< Enable GPU state pin check */
+#define RGX_VAL_WGP_SIG_CHECK_NOERR_EN           (0x10U)  /*!< Enable WGP signature check. Signatures must match */
+#define RGX_VAL_WGP_SIG_CHECK_ERR_EN             (0x20U)  /*!< Enable WGP signature check. Signatures must not match */
+#define RGX_VAL_TRP_SIG_CHECK_NOERR_EN           (0x40U)  /*!< Enable TRP signature check. Signatures must match */
+#define RGX_VAL_TRP_SIG_CHECK_ERR_EN             (0x80U)  /*!< Enable TRP signature check. Signatures must not match */
+
+/*!
+ ******************************************************************************
+ * HWPerf L2 Stream ID type definition.
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_HWPERF_L2_STREAM_ID;
+/* HWPerf stream for Client HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_HWPERF 0U
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+/* HWPerf stream for FTrace HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_FTRACE 1U
+#define RGX_HWPERF_L2_STREAM_LAST   2U
+#else
+#define RGX_HWPERF_L2_STREAM_LAST   1U
+#endif
+
+typedef struct _GPU_FREQ_TRACKING_DATA_
+{
+	/* Core clock speed estimated by the driver */
+	IMG_UINT32 ui32EstCoreClockSpeed;
+
+	/* Amount of successful calculations of the estimated core clock speed */
+	IMG_UINT32 ui32CalibrationCount;
+} GPU_FREQ_TRACKING_DATA;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+#define RGX_GPU_FREQ_TRACKING_SIZE 16
+
+typedef struct
+{
+	IMG_UINT64 ui64BeginCRTimestamp;
+	IMG_UINT64 ui64BeginOSTimestamp;
+
+	IMG_UINT64 ui64EndCRTimestamp;
+	IMG_UINT64 ui64EndOSTimestamp;
+
+	IMG_UINT32 ui32EstCoreClockSpeed;
+	IMG_UINT32 ui32CoreClockSpeed;
+} GPU_FREQ_TRACKING_HISTORY;
+#endif
+
+typedef struct _RGX_GPU_DVFS_TABLE_
+{
+	/* Beginning of current calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimestamp;
+	IMG_UINT64 ui64CalibrationOSTimestamp;
+
+	/* Calculated calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimediff;
+	IMG_UINT64 ui64CalibrationOSTimediff;
+
+	/* Current calibration period (in us) */
+	IMG_UINT32 ui32CalibrationPeriod;
+
+	/* System layer frequency table and frequency tracking data */
+	IMG_UINT32 ui32FreqIndex;
+	IMG_UINT32 aui32GPUFrequency[RGX_GPU_DVFS_TABLE_SIZE];
+	GPU_FREQ_TRACKING_DATA asTrackingData[RGX_GPU_DVFS_TABLE_SIZE];
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	IMG_UINT32 ui32HistoryIndex;
+	GPU_FREQ_TRACKING_HISTORY asTrackingHistory[RGX_GPU_FREQ_TRACKING_SIZE];
+#endif
+} RGX_GPU_DVFS_TABLE;
+
+
+/*!
+ ******************************************************************************
+ * GPU utilisation statistics
+ *****************************************************************************/
+
+typedef struct _RGXFWIF_TEMP_GPU_UTIL_STATS_
+{
+	IMG_UINT64 aaaui64DMOSTmpCounters[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 aaui64DMOSTmpLastWord[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastState[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastPeriod[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastTime[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+} RGXFWIF_TEMP_GPU_UTIL_STATS;
+
+typedef struct _RGXFWIF_GPU_UTIL_STATS_
+{
+	IMG_BOOL   bValid;                /* If TRUE, statistics are valid.
+	                                     FALSE if the driver couldn't get reliable stats. */
+	IMG_UINT64 ui64GpuStatActive;     /* GPU active statistic */
+	IMG_UINT64 ui64GpuStatBlocked;    /* GPU blocked statistic */
+	IMG_UINT64 ui64GpuStatIdle;       /* GPU idle statistic */
+	IMG_UINT64 ui64GpuStatCumulative; /* Sum of active/blocked/idle stats */
+
+	IMG_UINT64 aaui64DMOSStatInactive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];   /* Per-DM per-OS sum of idle and blocked stats */
+	IMG_UINT64 aaui64DMOSStatActive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];     /* Per-DM per-OS active statistic */
+	IMG_UINT64 aaui64DMOSStatCumulative[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED]; /* Per-DM per-OS sum of active/blocked/idle stats */
+
+	IMG_UINT64 ui64TimeStamp;         /* Timestamp of the most recent sample of the GPU stats */
+
+	RGXFWIF_TEMP_GPU_UTIL_STATS sTempGpuStats; /* Temporary data used to calculate the per-DM per-OS statistics */
+} RGXFWIF_GPU_UTIL_STATS;
+
+
+typedef struct _RGX_REG_CONFIG_
+{
+	IMG_BOOL               bEnabled;
+	RGXFWIF_REG_CFG_TYPE   eRegCfgTypeToPush;
+	IMG_UINT32             ui32NumRegRecords;
+	POS_LOCK               hLock;
+} RGX_REG_CONFIG;
+
+typedef struct _PVRSRV_STUB_PBDESC_ PVRSRV_STUB_PBDESC;
+
+
+typedef struct _PVRSRV_DEVICE_FEATURE_CONFIG_
+{
+	IMG_UINT64 ui64ErnsBrns;
+	IMG_UINT64 ui64Features;
+	IMG_UINT32 ui32B;
+	IMG_UINT32 ui32V;
+	IMG_UINT32 ui32N;
+	IMG_UINT32 ui32C;
+	IMG_UINT32 ui32FeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX];
+	IMG_UINT32 ui32MAXDMCount;
+	IMG_UINT32 ui32MAXPowUnitCount;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	IMG_UINT32 ui32MAXRACCount;
+#endif
+	IMG_UINT32 ui32SLCSizeInBytes;
+	IMG_PCHAR  pszBVNCString;
+}PVRSRV_DEVICE_FEATURE_CONFIG;
+
+/* This is used to get the value of a specific feature.
+ * Note that it will assert if the feature is disabled or value is invalid. */
+#define RGX_GET_FEATURE_VALUE(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] )
+
+/* This is used to check if the feature value (e.g. with an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] < RGX_FEATURE_VALUE_DISABLED )
+
+/* This is used to check if the Boolean feature (e.g. WITHOUT an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_SUPPORTED(psDevInfo, Feature) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64Features, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if the ERN is available for the currently running BVNC or not */
+#define RGX_IS_ERN_SUPPORTED(psDevInfo, ERN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if the BRN is available for the currently running BVNC or not */
+#define RGX_IS_BRN_SUPPORTED(psDevInfo, BRN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+/* there is a corresponding define in rgxapi.h */
+#define RGX_MAX_TIMER_QUERIES 16U
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*!
+ * The host maintains a 512-deep cache of submitted workloads per device,
+ * i.e. a global look-up table for TA, 3D and compute (depending on the RGX
+ * hardware support present)
+ */
+
+/*
+ * For the workload estimation return data array, the max amount of commands the
+ * MTS can have is 255, therefore 512 (LOG2 = 9) is large enough to account for
+ * all corner cases
+ */
+#define RETURN_DATA_ARRAY_SIZE_LOG2 (9)
+#define RETURN_DATA_ARRAY_SIZE      ((1U) << RETURN_DATA_ARRAY_SIZE_LOG2)
+#define RETURN_DATA_ARRAY_WRAP_MASK (RETURN_DATA_ARRAY_SIZE - 1)
+
+#define WORKLOAD_HASH_SIZE_LOG2		6
+#define WORKLOAD_HASH_SIZE			((1U) << WORKLOAD_HASH_SIZE_LOG2)
+#define WORKLOAD_HASH_WRAP_MASK		(WORKLOAD_HASH_SIZE - 1)
+
+/*!
+ * Workload characteristics for supported data masters.
+ * All characteristics must match for the workload estimate to be used/updated.
+ */
+typedef union _RGX_WORKLOAD_
+{
+	struct
+	{
+		IMG_UINT32				ui32RenderTargetSize;
+		IMG_UINT32				ui32NumberOfDrawCalls;
+		IMG_UINT32				ui32NumberOfIndices;
+		IMG_UINT32				ui32NumberOfMRTs;
+	} sTA3D;
+
+	struct
+	{
+		IMG_UINT32				ui32NumberOfWorkgroups;
+		IMG_UINT32				ui32NumberOfWorkitems;
+	} sCompute;
+
+	struct
+	{
+		IMG_UINT32				ui32Characteristic1;
+		IMG_UINT32				ui32Characteristic2;
+	} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	struct
+	{
+		IMG_UINT32				ui32DispatchSize;
+		IMG_UINT32				ui32AccStructSize;
+	} sRay;
+#endif
+
+} RGX_WORKLOAD;
+
+/*!
+ * Host data used to match the return data (actual cycles count) to the
+ * submitted command packet.
+ * The hash table is a per-DM circular buffer containing a key based on the
+ * workload characteristics. On job completion, the oldest workload data
+ * is evicted if the CB is full and the driver matches the characteristics
+ * to the matching data.
+ *
+ * o If the driver finds a match the existing cycle estimate is averaged with
+ *   the actual cycles used.
+ * o Otherwise a new hash entry is created with the actual cycles for this
+ *   workload.
+ *
+ * Subsequently if a match is found during command submission, the estimate
+ * is passed to the scheduler, e.g. adjust the GPU frequency if PDVFS is enabled.
+ */
+typedef struct _WORKLOAD_MATCHING_DATA_
+{
+	POS_LOCK				psHashLock;
+	HASH_TABLE				*psHashTable;		/*! existing workload cycle estimates for this DM */
+	RGX_WORKLOAD			asHashKeys[WORKLOAD_HASH_SIZE];
+	IMG_UINT64				aui64HashData[WORKLOAD_HASH_SIZE];
+	IMG_UINT32				ui32HashArrayWO;	/*! track the most recent workload estimates */
+} WORKLOAD_MATCHING_DATA;
+
+/*!
+ * A generic container for the workload matching data for GPU contexts:
+ * rendering (TA, 3D), compute, etc.
+ */
+typedef struct _WORKEST_HOST_DATA_
+{
+	union
+	{
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTA;	/*!< matching data for TA commands */
+			WORKLOAD_MATCHING_DATA	sData3D;	/*!< matching data for 3D commands */
+		} sTA3D;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataCDM;	/*!< matching data for CDM commands */
+		} sCompute;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTDM;	/*!< matching data for TDM-TQ commands */
+		} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataRDM;	/*!< matching data for RDM commands */
+		} sRay;
+#endif
+	} uWorkloadMatchingData;
+
+	/*
+	 * This is a per-context property, hence the TA and 3D share the same
+	 * per render context counter.
+	 */
+	IMG_UINT32				ui32WorkEstCCBReceived;	/*!< Used to ensure all submitted work
+														 estimation commands are received
+														 by the host before clean up. */
+} WORKEST_HOST_DATA;
+
+/*!
+ * Entries in the list of submitted workloads, used when the completed command
+ * returns data to the host.
+ *
+ * - the matching data is needed as it holds the hash data
+ * - the host data is needed for completion updates, ensuring memory is not
+ *   freed while workload estimates are in-flight.
+ * - the workload characteristic is used in the hash table look-up.
+ */
+typedef struct _WORKEST_RETURN_DATA_
+{
+	WORKEST_HOST_DATA		*psWorkEstHostData;
+	WORKLOAD_MATCHING_DATA	*psWorkloadMatchingData;
+	RGX_WORKLOAD			sWorkloadCharacteristics;
+} WORKEST_RETURN_DATA;
+#endif
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+#define RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES    4
+#endif
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+typedef struct
+{
+#if defined(PDUMP)
+	IMG_HANDLE      hPdumpPages;
+#endif
+	PG_HANDLE       sPages;
+	IMG_DEV_PHYADDR sPhysAddr;
+} RGX_MIPS_ADDRESS_TRAMPOLINE;
+#endif
+
+
+/*!
+ ******************************************************************************
+ * RGX Device error counts
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_ERROR_COUNTS_
+{
+	IMG_UINT32 ui32WGPErrorCount;		/*!< count of the number of WGP checksum errors */
+	IMG_UINT32 ui32TRPErrorCount;		/*!< count of the number of TRP checksum errors */
+} PVRSRV_RGXDEV_ERROR_COUNTS;
+
+/*!
+ ******************************************************************************
+ * RGX Debug dump firmware trace log type
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_FWT_LOGTYPE;
+#define RGX_FWT_LOGTYPE_NONE    0U
+#define RGX_FWT_LOGTYPE_BINARY  1U
+#define RGX_FWT_LOGTYPE_DECODED 2U
+#define RGX_FWT_LOGTYPE_PARTIAL 3U
+
+/*!
+ ******************************************************************************
+ * RGX Device info
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_INFO_
+{
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+
+	PVRSRV_DEVICE_FEATURE_CONFIG	sDevFeatureCfg;
+
+	IMG_BOOL				bDevInit2Done;
+
+	IMG_BOOL				bFirmwareInitialised;
+	IMG_BOOL				bPDPEnabled;
+
+	IMG_HANDLE				hDbgReqNotify;
+
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvRegsBaseKM;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvSecureRegsBaseKM;
+#endif
+
+	IMG_HANDLE				hRegMapping;
+
+	/* System physical address of device registers */
+	IMG_CPU_PHYADDR			sRegsPhysBase;
+	/* Register region size in bytes */
+	IMG_UINT32				ui32RegSize;
+
+	PVRSRV_STUB_PBDESC		*psStubPBDescListKM;
+
+	/* Firmware memory context info */
+	DEVMEM_CONTEXT			*psKernelDevmemCtx;
+	DEVMEM_HEAP				*psFirmwareMainHeap;
+	DEVMEM_HEAP				*psFirmwareConfigHeap;
+	MMU_CONTEXT				*psKernelMMUCtx;
+
+	void					*pvDeviceMemoryHeap;
+
+	/* Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBCtlMemDesc;      /*!< memdesc for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtl;             /*!< kernel mapping for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtlLocal;        /*!< cpu local copy of Kernel CCB control */
+	DEVMEM_MEMDESC			*psKernelCCBMemDesc;         /*!< memdesc for Kernel CCB */
+	IMG_UINT8				*psKernelCCB;                /*!< kernel mapping for Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBRtnSlotsMemDesc; /*!< Return slot array for Kernel CCB commands */
+	IMG_UINT32				*pui32KernelCCBRtnSlots;     /*!< kernel mapping for return slot array */
+
+	/* Firmware CCB */
+	DEVMEM_MEMDESC			*psFirmwareCCBCtlMemDesc;   /*!< memdesc for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtl;          /*!< kernel mapping for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtlLocal;     /*!< cpu local copy of Firmware CCB control */
+	DEVMEM_MEMDESC			*psFirmwareCCBMemDesc;      /*!< memdesc for Firmware CCB */
+	IMG_UINT8				*psFirmwareCCB;             /*!< kernel mapping for Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBCtlMemDesc;   /*!< memdesc for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtl;          /*!< kernel mapping for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtlLocal;     /*!< cpu local copy of Workload Estimation Firmware CCB control */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBMemDesc;      /*!< memdesc for Workload Estimation Firmware CCB */
+	IMG_UINT8				*psWorkEstFirmwareCCB;             /*!< kernel mapping for Workload Estimation Firmware CCB */
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	/* Counter dumping */
+	DEVMEM_MEMDESC			*psCounterBufferMemDesc;      /*!< mem desc for counter dumping buffer */
+	POS_LOCK				hCounterDumpingLock;          /*!< Lock for guarding access to counter dumping buffer */
+#endif
+
+	PVRSRV_MEMALLOCFLAGS_T  uiFWPoisonOnFreeFlag;           /*!< Flag for poisoning FW allocations when freed */
+
+	IMG_BOOL				bIgnoreHWReportedBVNC;			/*!< Ignore BVNC reported by HW */
+
+    /* multicore configuration information */
+    IMG_UINT32              ui32MultiCoreNumCores;      /* total cores primary + secondaries. 0 for non-multi core */
+    IMG_UINT32              ui32MultiCorePrimaryId;     /* primary core id for this device */
+    IMG_UINT64             *pui64MultiCoreCapabilities; /* capabilities for each core */
+
+	/*
+		if we don't preallocate the pagetables we must
+		insert newly allocated page tables dynamically
+	*/
+	void					*pvMMUContextList;
+
+	IMG_UINT32				ui32ClkGateStatusReg;
+	IMG_UINT32				ui32ClkGateStatusMask;
+
+	DEVMEM_MEMDESC			*psRGXFWCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCodeDevVAddrBase;
+	IMG_UINT32			ui32FWCodeSizeInBytes;
+	DEVMEM_MEMDESC			*psRGXFWDataMemDesc;
+	IMG_DEV_VIRTADDR		sFWDataDevVAddrBase;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	RGX_MIPS_ADDRESS_TRAMPOLINE	*psTrampoline;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWCorememCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememCodeDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememCodeFWAddr;
+	IMG_UINT32			ui32FWCorememCodeSizeInBytes;
+
+	DEVMEM_MEMDESC			*psRGXFWIfCorememDataStoreMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememDataStoreDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememDataStoreFWAddr;
+
+	DEVMEM_MEMDESC			*psRGXFWAlignChecksMemDesc;
+
+#if defined(PDUMP)
+	DEVMEM_MEMDESC			*psRGXFWSigTAChecksMemDesc;
+	IMG_UINT32				ui32SigTAChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSig3DChecksMemDesc;
+	IMG_UINT32				ui32Sig3DChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSigTDMChecksMemDesc;
+	IMG_UINT32				ui32SigTDMChecksSize;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DEVMEM_MEMDESC			*psRGXFWSigRDMChecksMemDesc;
+	IMG_UINT32				ui32SigRDMChecksSize;
+#endif
+
+	IMG_BOOL				bDumpedKCCBCtlAlready;
+
+	POS_SPINLOCK			hSyncCheckpointSignalSpinLock;						/*!< Guards data shared between an atomic & sleepable-context */
+#endif
+
+	POS_LOCK				hRGXFWIfBufInitLock;								/*!< trace buffer lock for initialisation phase */
+
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufCtlMemDesc;						/*!< memdesc of trace buffer control structure */
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufferMemDesc[RGXFW_THREAD_NUM];		/*!< memdesc of actual FW trace (log) buffer(s) */
+	IMG_PUINT32				apui32TraceBuffer[RGXFW_THREAD_NUM];				/*!< Trace buffer address (Host address), to be used by host when reading from trace buffer */
+	IMG_UINT32				ui32TraceBufSizeInDWords;							/*!< CPU local copy of FW Trace buffer size in dwords */
+	RGXFWIF_TRACEBUF		*psRGXFWIfTraceBufCtl;								/*!< structure containing trace control data and actual trace buffer */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwSysDataMemDesc;							/*!< memdesc of the firmware-shared system data structure */
+	RGXFWIF_SYSDATA			*psRGXFWIfFwSysData;								/*!< structure containing km-firmware shared system data */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwOsDataMemDesc;							/*!< memdesc of the firmware-shared os structure */
+	RGXFWIF_OSDATA			*psRGXFWIfFwOsData;									/*!< structure containing km-firmware shared os data */
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	DEVMEM_MEMDESC			*psRGXFWIfTBIBufferMemDesc;							/*!< memdesc of actual FW TBI buffer */
+	RGXFWIF_DEV_VIRTADDR	sRGXFWIfTBIBuffer;									/*!< TBI buffer data */
+	IMG_UINT32				ui32FWIfTBIBufferSize;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWRInfoBufCtlMemDesc;
+	RGXFWIF_HWRINFOBUF		*psRGXFWIfHWRInfoBufCtl;
+	IMG_UINT32				ui32ClockSource;
+	IMG_UINT32				ui32LastClockSource;
+
+	DEVMEM_MEMDESC			*psRGXFWIfGpuUtilFWCtlMemDesc;
+	RGXFWIF_GPU_UTIL_FW     *psRGXFWIfGpuUtilFW;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfBufMemDesc;
+	IMG_BYTE				*psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32RGXFWIfHWPerfBufSize; /* in bytes */
+	IMG_UINT32				ui32RGXL2HWPerfBufSize;   /* in bytes */
+
+	DEVMEM_MEMDESC			*psRGXFWIfRegCfgMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfCountersMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfConnectionCtlMemDesc;
+	RGXFWIF_CONNECTION_CTL	*psRGXFWIfConnectionCtl;
+
+	DEVMEM_MEMDESC			*psRGXFWHeapGuardPageReserveMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfSysInitMemDesc;
+	RGXFWIF_SYSINIT			*psRGXFWIfSysInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfOsInitMemDesc;
+	RGXFWIF_OSINIT			*psRGXFWIfOsInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfRuntimeCfgMemDesc;
+	RGXFWIF_RUNTIME_CFG		*psRGXFWIfRuntimeCfg;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	DEVMEM_MEMDESC			*psRGXFWIfActiveContextBufDesc;
+	RGXFWIF_ACTIVE_CONTEXT_BUF_DATA		*psRGXFWIfActiveContextBuf;
+#endif
+
+	/* Premapped firmware memory context info */
+	DEVMEM_HEAP				*psPremappedFwRawHeap[RGX_NUM_DRIVERS_SUPPORTED];
+	DEVMEM_MEMDESC			*psPremappedFwRawMemDesc[RGX_NUM_DRIVERS_SUPPORTED];
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Array to store data needed for workload estimation when a workload
+	   has finished and its cycle time is returned to the host.	 */
+	WORKEST_RETURN_DATA     asReturnData[RETURN_DATA_ARRAY_SIZE];
+	IMG_UINT32              ui32ReturnDataWO;
+	POS_LOCK                hWorkEstLock;
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/**
+	 * Host memdesc and pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory on changing the core clock rate over
+	 * GPIO.
+	 * Note: Shared memory needs atomic access from Host driver and firmware,
+	 * hence size should not be greater than memory transaction granularity.
+	 * Currently it is chosen to be 32 bits.
+	 */
+	DEVMEM_MEMDESC			*psRGXFWIFCoreClkRateMemDesc;
+	volatile IMG_UINT32		*pui32RGXFWIFCoreClkRate;
+	/**
+	 * Last sampled core clk rate.
+	 */
+	volatile IMG_UINT32		ui32CoreClkRateSnapshot;
+#endif
+
+	/*
+	   HWPerf data for the RGX device
+	 */
+
+	POS_LOCK    hHWPerfLock;  /*! Critical section lock that protects HWPerf code
+	                           *  from multiple thread duplicate init/deinit
+	                           *  and loss/freeing of FW & Host resources while in
+	                           *  use in another thread e.g. MSIR. */
+
+	IMG_UINT64  ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_LAST];         /*! Event filter for FW events (settable by AppHint) */
+	IMG_HANDLE  hHWPerfStream[RGX_HWPERF_L2_STREAM_LAST];            /*! TL Stream buffer (L2) for firmware event stream */
+	IMG_UINT32  ui32L2BufMaxPacketSize[RGX_HWPERF_L2_STREAM_LAST];   /*! Max allowed packet size in FW HWPerf TL (L2) buffer */
+	IMG_BOOL    bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_LAST]; /*! Flag to indicate if copying HWPerf data is suspended */
+	IMG_UINT64  ui64HWPerfFwFilter;                                  /*! Event filter for FW events created from OR-ing ui64HWPerfFilter values. */
+	IMG_UINT32  uiHWPerfStreamCount;                                 /*! Value indicating if any of the HWPerf streams has been created */
+
+	IMG_UINT32  ui32HWPerfHostFilter;      /*! Event filter for HWPerfHost stream (settable by AppHint) */
+	POS_LOCK    hLockHWPerfHostStream;     /*! Lock guarding access to HWPerfHost stream from multiple threads */
+	IMG_HANDLE  hHWPerfHostStream;         /*! TL Stream buffer for host only event stream */
+	IMG_UINT32  ui32HWPerfHostBufSize;     /*! Host side buffer size in bytes */
+	IMG_UINT32  ui32HWPerfHostLastOrdinal; /*! Ordinal of the last packet emitted in HWPerfHost TL stream.
+	                                        *  Guarded by hLockHWPerfHostStream */
+	IMG_UINT32  ui32HWPerfHostNextOrdinal; /*! Ordinal number for HWPerfHost events. Guarded by hHWPerfHostSpinLock */
+	IMG_UINT8   *pui8DeferredEvents;       /*! List of HWPerfHost events yet to be emitted in the TL stream.
+	                                        *  Events generated from atomic context are deferred "emitted"
+											*  as the "emission" code can sleep */
+	IMG_UINT16  ui16DEReadIdx;             /*! Read index in the above deferred events buffer */
+	IMG_UINT16  ui16DEWriteIdx;            /*! Write index in the above deferred events buffer */
+	void        *pvHostHWPerfMISR;         /*! MISR to emit pending/deferred events in HWPerfHost TL stream */
+	POS_SPINLOCK hHWPerfHostSpinLock;      /*! Guards data shared between an atomic & sleepable-context */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32  ui32DEHighWatermark;       /*! High watermark of deferred events buffer usage. Protected by
+	                                        *! hHWPerfHostSpinLock */
+	/* Max number of times DeferredEmission waited for an atomic-context to "finish" packet write */
+	IMG_UINT32  ui32WaitForAtomicCtxPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an atomic-context packet loss (due to too long wait for "write" finish) */
+	IMG_BOOL    bWarnedAtomicCtxPktLost;
+	/* Max number of times DeferredEmission scheduled-out to give a chance to the right-ordinal packet to be emitted */
+	IMG_UINT32  ui32WaitForRightOrdPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an packet loss (due to too long wait for right ordinal to emit) */
+	IMG_BOOL    bWarnedPktOrdinalBroke;
+#endif
+
+	void        *pvGpuFtraceData;
+
+	/* Poll data for detecting firmware fatal errors */
+	IMG_UINT32				aui32CrLastPollCount[RGXFW_THREAD_NUM];
+	IMG_UINT32				ui32KCCBCmdsExecutedLastTime;
+	IMG_BOOL				bKCCBCmdsWaitingLastTime;
+	IMG_UINT32				ui32GEOTimeoutsLastTime;
+	IMG_UINT32				ui32InterruptCountLastTime;
+	IMG_UINT32				ui32MissingInterruptsLastTime;
+
+	/* Client stall detection */
+	IMG_UINT32				ui32StalledClientMask;
+
+	IMG_BOOL				bWorkEstEnabled;
+	IMG_BOOL				bPDVFSEnabled;
+
+	void					*pvLISRData;
+	void					*pvMISRData;
+	void					*pvAPMISRData;
+	RGX_ACTIVEPM_CONF		eActivePMConf;
+
+	volatile IMG_UINT32		aui32SampleIRQCount[RGXFW_THREAD_NUM];
+
+	DEVMEM_MEMDESC			*psRGXFaultAddressMemDesc;
+
+	DEVMEM_MEMDESC			*psSLC3FenceMemDesc;
+
+	/* If we do 10 deferred memory allocations per second, then the ID would wrap around after 13 years */
+	IMG_UINT32				ui32ZSBufferCurrID;	/*!< ID assigned to the next deferred devmem allocation */
+	IMG_UINT32				ui32FreelistCurrID;	/*!< ID assigned to the next freelist */
+
+	POS_LOCK				hLockZSBuffer;		/*!< Lock to protect simultaneous access to ZSBuffers */
+	DLLIST_NODE				sZSBufferHead;		/*!< List of on-demand ZSBuffers */
+	POS_LOCK				hLockFreeList;		/*!< Lock to protect simultaneous access to Freelists */
+	DLLIST_NODE				sFreeListHead;		/*!< List of growable Freelists */
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+	PVRSRV_CLIENT_SYNC_PRIM	*psPowSyncPrim;
+
+	IMG_UINT32				ui32ActivePMReqOk;
+	IMG_UINT32				ui32ActivePMReqDenied;
+	IMG_UINT32				ui32ActivePMReqNonIdle;
+	IMG_UINT32				ui32ActivePMReqRetry;
+	IMG_UINT32				ui32ActivePMReqTotal;
+
+	IMG_HANDLE				hProcessQueuesMISR;
+
+	IMG_UINT32				ui32DeviceFlags;		/*!< Flags to track general device state */
+
+	/* GPU DVFS Table */
+	RGX_GPU_DVFS_TABLE		*psGpuDVFSTable;
+
+	/* Pointer to function returning the GPU utilisation statistics since the last
+	 * time the function was called. Supports different users at the same time.
+	 *
+	 * psReturnStats [out]: GPU utilisation statistics (active high/active low/idle/blocked)
+	 *                      in microseconds since the last time the function was called
+	 *                      by a specific user (identified by hGpuUtilUser)
+	 *
+	 * Returns PVRSRV_OK in case the call completed without errors,
+	 * some other value otherwise.
+	 */
+	PVRSRV_ERROR (*pfnGetGpuUtilStats) (PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                    IMG_HANDLE hGpuUtilUser,
+	                                    RGXFWIF_GPU_UTIL_STATS *psReturnStats);
+
+	/* Pointer to function that checks if the physical GPU IRQ
+	 * line has been asserted and clears it if so */
+	IMG_BOOL (*pfnRGXAckIrq) (struct _PVRSRV_RGXDEV_INFO_ *psDevInfo);
+
+	POS_LOCK				hGPUUtilLock;
+
+	/* Register configuration */
+	RGX_REG_CONFIG			sRegConfig;
+
+	IMG_BOOL				bRGXPowered;
+	DLLIST_NODE				sMemoryContextList;
+
+	POSWR_LOCK				hRenderCtxListLock;
+	POSWR_LOCK				hComputeCtxListLock;
+	POSWR_LOCK				hTransferCtxListLock;
+	POSWR_LOCK				hTDMCtxListLock;
+	POSWR_LOCK				hMemoryCtxListLock;
+	POSWR_LOCK				hKickSyncCtxListLock;
+
+	/* Linked list of deferred KCCB commands due to a full KCCB.
+	 * Access to members sKCCBDeferredCommandsListHead and ui32KCCBDeferredCommandsCount
+	 * are protected by the hLockKCCBDeferredCommandsList spin lock. */
+	POS_SPINLOCK			hLockKCCBDeferredCommandsList; /*!< Protects deferred KCCB commands list */
+	DLLIST_NODE				sKCCBDeferredCommandsListHead;
+	IMG_UINT32				ui32KCCBDeferredCommandsCount; /*!< No of commands in the deferred list */
+
+	/* Linked lists of contexts on this device */
+	DLLIST_NODE				sRenderCtxtListHead;
+	DLLIST_NODE				sComputeCtxtListHead;
+	DLLIST_NODE				sTransferCtxtListHead;
+	DLLIST_NODE				sTDMCtxtListHead;
+	DLLIST_NODE				sKickSyncCtxtListHead;
+
+	DLLIST_NODE				sCommonCtxtListHead;
+	POSWR_LOCK				hCommonCtxtListLock;
+	IMG_UINT32				ui32CommonCtxtCurrentID;	/*!< ID assigned to the next common context */
+
+	POS_LOCK				hDebugFaultInfoLock;	/*!< Lock to protect the debug fault info list */
+	POS_LOCK				hMMUCtxUnregLock;		/*!< Lock to protect list of unregistered MMU contexts */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	POS_LOCK				hNMILock; /*!< Lock to protect NMI operations */
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32				ui32AvailablePowUnitsMask;
+	IMG_UINT32				ui32AvailableRACMask;
+#endif
+
+	RGX_LAYER_PARAMS		sLayerParams;
+
+	RGXFWIF_DM				eBPDM;					/*!< Current breakpoint data master */
+	IMG_BOOL				bBPSet;					/*!< A Breakpoint has been set */
+	POS_LOCK				hBPLock;				/*!< Lock for break point operations */
+
+	IMG_UINT32				ui32CoherencyTestsDone;
+
+	ATOMIC_T				iCCBSubmissionOrdinal; /* Rolling count used to indicate CCB submission order (all CCBs) */
+	POS_LOCK				hCCBRecoveryLock;      /* Lock to protect pvEarliestStalledClientCCB and ui32OldestSubmissionOrdinal variables */
+	void					*pvEarliestStalledClientCCB; /* Will point to cCCB command to unblock in the event of a stall */
+	IMG_UINT32				ui32OldestSubmissionOrdinal; /* Earliest submission ordinal of CCB entry found so far */
+	IMG_UINT32				ui32SLRHoldoffCounter;   /* Decremented each time health check is called until zero. SLR only happen when zero. */
+
+	POS_LOCK				hCCBStallCheckLock; /* Lock used to guard against multiple threads simultaneously checking for stalled CCBs */
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Firmware gcov buffer */
+	DEVMEM_MEMDESC			*psFirmwareGcovBufferMemDesc;      /*!< mem desc for Firmware gcov dumping buffer */
+	IMG_UINT32				ui32FirmwareGcovSize;
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* Value to store for each page size range config register in MMU4 */
+	IMG_UINT64				aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES];
+#endif
+
+
+
+	IMG_HANDLE				hTQCLISharedMem;		/*!< TQ Client Shared Mem PMR */
+	IMG_HANDLE				hTQUSCSharedMem;		/*!< TQ USC Shared Mem PMR */
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	DEVMEM_MEMDESC			*psRGXFWIfSecureBufMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfNonSecureBufMemDesc;
+#endif
+
+	/* Timer Queries */
+	IMG_UINT32				ui32ActiveQueryId;		/*!< id of the active line */
+	IMG_BOOL				bSaveStart;				/*!< save the start time of the next kick on the device*/
+	IMG_BOOL				bSaveEnd;				/*!< save the end time of the next kick on the device*/
+
+	DEVMEM_MEMDESC			*psStartTimeMemDesc;    /*!< memdesc for Start Times */
+	IMG_UINT64				*pui64StartTimeById;    /*!< CPU mapping of the above */
+
+	DEVMEM_MEMDESC			*psEndTimeMemDesc;      /*!< memdesc for End Timer */
+	IMG_UINT64				*pui64EndTimeById;      /*!< CPU mapping of the above */
+
+	IMG_UINT32				aui32ScheduledOnId[RGX_MAX_TIMER_QUERIES];	/*!< kicks Scheduled on QueryId */
+	DEVMEM_MEMDESC			*psCompletedMemDesc;	/*!< kicks Completed on QueryId */
+	IMG_UINT32				*pui32CompletedById;	/*!< CPU mapping of the above */
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	POS_LOCK				hTimerQueryLock;		/*!< lock to protect simultaneous access to timer query members */
+#endif
+
+	PVRSRV_RGXDEV_ERROR_COUNTS sErrorCounts;		/*!< struct containing device error counts */
+
+	IMG_UINT32				ui32HostSafetyEventMask;/*!< mask of the safety events handled by the driver */
+
+	RGX_CONTEXT_RESET_REASON	eLastDeviceError;	/*!< device error reported to client */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	PMR						*psGenHeapSecMem;		/*!< An allocation of secure memory mapped to
+													  the general devmem heap. The allocation is
+													  created and mapped at driver init. It's used for
+													  various purposes. See rgx_fwif_km.h for all use cases. */
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	DEVMEM_MEMDESC			*psRGXFWScratchBufMemDesc;
+#endif
+
+	RGX_FWT_LOGTYPE			eDebugDumpFWTLogType;
+
+	RGX_FW_INFO_HEADER      sFWInfoHeader;
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	IMG_UINT32              ui32TFBCLossyGroup;     /*!< TFBCCompressionControlGroup
+													  setting for those cores which support
+													  this feature. */
+#endif
+	RGXFWIF_GPU_UTIL_STATS	sGpuUtilStats;          /*!< GPU usage statistics */
+	POS_LOCK				hGpuUtilStatsLock;
+
+} PVRSRV_RGXDEV_INFO;
+
+
+
+typedef struct _RGX_TIMING_INFORMATION_
+{
+	/*! GPU default core clock speed in Hz */
+	IMG_UINT32			ui32CoreClockSpeed;
+
+	/*! Default SOC clock speed in Hz */
+	IMG_UINT32			ui32SOCClockSpeed;
+
+	/*! Active Power Management: GPU actively requests the host driver to be powered off */
+	IMG_BOOL			bEnableActivePM;
+
+	/*! Enable the GPU to power off internal Power Islands independently from the host driver */
+	IMG_BOOL			bEnableRDPowIsland;
+
+	/*! Active Power Management: Delay between the GPU idle and the request to the host */
+	IMG_UINT32			ui32ActivePMLatencyms;
+
+} RGX_TIMING_INFORMATION;
+
+typedef struct _RGX_DATA_
+{
+	/*! Timing information */
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo;
+} RGX_DATA;
+
+typedef enum _RGX_QUERY_TIMESTAMP_TYPE_
+{
+	RGX_QUERY_HOST_TIMESTAMP,
+	RGX_QUERY_DEVICE_TIMESTAMP,
+} RGX_QUERY_TIMESTAMP_TYPE;
+
+
+/*
+	RGX PDUMP register bank name (prefix)
+*/
+#define RGX_PDUMPREG_NAME		"RGXREG"
+
+#endif /* RGXDEVICE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxfwutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxfwutils.c
new file mode 100644
index 000000000000..2f65106d359e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxfwutils.c
@@ -0,0 +1,6563 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rogue firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Rogue firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+#include "cache_km.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxfwriscv.h"
+#include "rgx_options.h"
+#include "rgx_fwif_alignchecks.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_pdump_panics.h"
+#include "fwtrace_string.h"
+#include "rgxheapconfig.h"
+#include "pvrsrv.h"
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "rgxccb.h"
+#include "rgxcompute.h"
+#include "rgxtransfer.h"
+#include "rgxtdmtransfer.h"
+#include "rgxpower.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+#include "rgxmem.h"
+#include "rgxmmudefs_km.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#endif
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxutils.h"
+#include "rgxtimecorr.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_external.h"
+#include "tlstream.h"
+#include "devicemem_server_utils.h"
+#include "htbserver.h"
+#include "rgx_bvnc_defs_km.h"
+#include "info_page.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#ifdef __linux__
+#include <linux/kernel.h>	/* sprintf */
+#else
+#include <stdio.h>
+#include <string.h>
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF)
+#include "pvr_dvfs_common.h"
+#endif
+
+
+#include "vz_vmm_pvz.h"
+#include "rgx_heaps.h"
+
+/*!
+ ******************************************************************************
+ * HWPERF
+ *****************************************************************************/
+/* Size of the Firmware L1 HWPERF buffer in bytes (2MB). Accessed by the
+ * Firmware and host driver. */
+#define RGXFW_HWPERF_L1_SIZE_MIN        (16U)
+#define RGXFW_HWPERF_L1_SIZE_DEFAULT    PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB
+#define RGXFW_HWPERF_L1_SIZE_MAX        (12288U)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#if defined(DEBUG)
+/* Catch the use of auto-increment when meta_registers_unpacked_accesses feature is
+ * present in case we ever use it. No WA exists so it must not be used */
+#define CHECK_HWBRN_68777(v) \
+	do { \
+		PVR_ASSERT(((v) & RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN) == 0); \
+	} while (0)
+#else
+#define CHECK_HWBRN_68777(v)
+#endif
+#endif
+
+/* Firmware CCB length */
+#if defined(NO_HARDWARE) && defined(PDUMP)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (10)
+#elif defined(SUPPORT_PDVFS) || defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (8)
+#else
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (5)
+#endif
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS] = {IRQ_COUNTER_STORAGE_REGS};
+#endif
+
+/* Workload Estimation Firmware CCB length */
+#define RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2   (7)
+
+/* Size of memory buffer for firmware gcov data
+ * The actual data size is several hundred kilobytes. The buffer is an order of magnitude larger. */
+#define RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE (4*1024*1024)
+
+typedef struct
+{
+	RGXFWIF_KCCB_CMD        sKCCBcmd;
+	DLLIST_NODE             sListNode;
+	PDUMP_FLAGS_T           uiPDumpFlags;
+	PVRSRV_RGXDEV_INFO      *psDevInfo;
+} RGX_DEFERRED_KCCB_CMD;
+
+typedef struct
+{
+	IMG_INT32  i32Priority;
+	IMG_UINT32 ui32IsolationGroups;
+	IMG_UINT32 ui32TSPercentage;
+}RGX_QOS_DEFAULTS;
+
+#define RGX_QOS_DEFAULTS_INIT(osid) \
+	{RGX_DRIVERID_##osid##_DEFAULT_PRIORITY,\
+	RGX_DRIVERID_##osid##_DEFAULT_ISOLATION_GROUP,\
+	RGX_DRIVERID_##osid##_DEFAULT_TIME_SLICE}
+
+#if defined(PDUMP)
+/* ensure PIDs are 32-bit because a 32-bit PDump load is generated for the
+ * PID filter example entries
+ */
+static_assert(sizeof(IMG_PID) == sizeof(IMG_UINT32),
+		"FW PID filtering assumes the IMG_PID type is 32-bits wide as it "
+		"generates WRW commands for loading the PID values");
+#endif
+
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+static_assert(RGX_DRIVER_DEFAULT_TIME_SLICES_SUM <= PVRSRV_VZ_TIME_SLICE_MAX, "Invalid driverid time slice aggregate");
+#endif
+
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo);
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+
+static void __MTSScheduleWrite(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Value)
+{
+	/* This should *NOT* happen. Try to trace what caused this and avoid a NPE
+	 * with the Write/Read at the foot of the function.
+	 */
+	PVR_ASSERT((psDevInfo != NULL));
+	if (psDevInfo == NULL)
+	{
+		return;
+	}
+
+	/* Kick MTS to wake firmware. */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE, ui32Value);
+
+	/* Uncached device/IO mapping will ensure MTS kick leaves CPU, read back
+	 * will ensure it reaches the regbank via inter-connects (AXI, PCIe etc)
+	 * before continuing.
+	 */
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          ppsMemDesc      pointer to the allocation's memory descriptor
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO*  psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL bClearByMemset;
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiAllocFlags))
+	{
+		/* Under AutoVz the ZERO_ON_ALLOC flag is avoided as it causes the memory to
+		 * be allocated from a different PMR than an allocation without the flag.
+		 * When the content of an allocation needs to be recovered from physical memory
+		 * on a later driver reboot, the memory then cannot be zeroed but the allocation
+		 * addresses must still match.
+		 * If the memory requires clearing, perform a memset after the allocation. */
+		uiAllocFlags &= ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+		bClearByMemset = IMG_TRUE;
+	}
+	else
+	{
+		bClearByMemset = IMG_FALSE;
+	}
+#endif
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate %s", pszName);
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui32Size,
+							  uiAllocFlags,
+							  pszName,
+							  ppsMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate %u bytes for %s (%u)",
+				 __func__,
+				 ui32Size,
+				 pszName,
+				 eError));
+		goto fail_alloc;
+	}
+
+	if (psFwPtr)
+	{
+		eError = RGXSetFirmwareAddress(psFwPtr, *ppsMemDesc, 0, ui32DevVAFlags);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire firmware virtual address for %s (%u)",
+					 __func__,
+					 pszName,
+					 eError));
+			goto fail_fwaddr;
+		}
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	if ((bClearByMemset) || (ppvCpuPtr))
+#else
+	if (ppvCpuPtr)
+#endif
+	{
+		void *pvTempCpuPtr;
+
+		eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, &pvTempCpuPtr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to acquire CPU virtual address for %s (%u)",
+					__func__,
+					 pszName,
+					eError));
+			goto fail_cpuva;
+		}
+
+#if defined(SUPPORT_AUTOVZ)
+		if (bClearByMemset)
+		{
+			if (PVRSRV_CHECK_CPU_WRITE_COMBINE(uiAllocFlags))
+			{
+				OSCachedMemSetWMB(pvTempCpuPtr, 0, ui32Size);
+			}
+			else
+			{
+				OSDeviceMemSet(pvTempCpuPtr, 0, ui32Size);
+			}
+		}
+		if (ppvCpuPtr)
+#endif
+		{
+			*ppvCpuPtr = pvTempCpuPtr;
+		}
+#if defined(SUPPORT_AUTOVZ)
+		else
+		{
+			DevmemReleaseCpuVirtAddr(*ppsMemDesc);
+			pvTempCpuPtr = NULL;
+		}
+#endif
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s set up at Fw VA 0x%x and CPU VA 0x%p with alloc flags 0x%" IMG_UINT64_FMTSPECX,
+			 __func__, pszName,
+			 (psFwPtr)   ? (psFwPtr->ui32Addr) : (0),
+			 (ppvCpuPtr) ? (*ppvCpuPtr)        : (NULL),
+			 uiAllocFlags));
+
+	return eError;
+
+fail_cpuva:
+	if (psFwPtr)
+	{
+		RGXUnsetFirmwareAddress(*ppsMemDesc);
+	}
+fail_fwaddr:
+	DevmemFree(*ppsMemDesc);
+fail_alloc:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHwPerfBufferSize
+
+@Description    Computes the effective size of the HW Perf Buffer
+@Input          ui32HWPerfFWBufSizeKB       Device Info struct
+@Return         HwPerfBufferSize
+*/ /**************************************************************************/
+static IMG_UINT32 GetHwPerfBufferSize(IMG_UINT32 ui32HWPerfFWBufSizeKB)
+{
+	IMG_UINT32 HwPerfBufferSize;
+
+	/* HWPerf: Determine the size of the FW buffer */
+	if (ui32HWPerfFWBufSizeKB == 0 ||
+			ui32HWPerfFWBufSizeKB == RGXFW_HWPERF_L1_SIZE_DEFAULT)
+	{
+		/* Under pvrsrvctl 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. Set it to the default
+		 * size, no logging.
+		 */
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_DEFAULT<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MAX))
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too big, using maximum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MAX));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MAX<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MIN))
+	{
+		/* Size specified as in AppHint HWPerfFWBufSizeInKB */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Using HWPerf FW buffer size of %u KB",
+				__func__,
+				ui32HWPerfFWBufSizeKB));
+		HwPerfBufferSize = ui32HWPerfFWBufSizeKB<<10;
+	}
+	else
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too small, using minimum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MIN));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MIN<<10;
+	}
+
+	return HwPerfBufferSize;
+}
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupSignatureChecks
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupSignatureChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
+                                              DEVMEM_MEMDESC**    ppsSigChecksMemDesc,
+                                              IMG_UINT32          ui32SigChecksBufSize,
+                                              RGXFWIF_SIGBUF_CTL* psSigBufCtl)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+								  ui32SigChecksBufSize,
+								  "FwSignatureChecks",
+								  ppsSigChecksMemDesc,
+								  &psSigBufCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	DevmemPDumpLoadMem(	*ppsSigChecksMemDesc,
+			0,
+			ui32SigChecksBufSize,
+			PDUMP_FLAGS_CONTINUOUS);
+
+	psSigBufCtl->ui32LeftSizeInRegs = ui32SigChecksBufSize / sizeof(IMG_UINT32);
+fail:
+	return eError;
+}
+#endif
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupFirmwareGcovBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupFirmwareGcovBuffer(PVRSRV_RGXDEV_INFO*			psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32FirmwareGcovBufferSize,
+		RGXFWIF_FIRMWARE_GCOV_CTL*	psFirmwareGcovCtl,
+		const IMG_CHAR*				pszBufferName)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for gcov */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32FirmwareGcovBufferSize,
+								  pszBufferName,
+								  ppsBufferMemDesc,
+								  &psFirmwareGcovCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psFirmwareGcovCtl->ui32Size = ui32FirmwareGcovBufferSize;
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+/*!
+ ******************************************************************************
+ @Function		RGXFWSetupCounterBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+ *****************************************************************************/
+static PVRSRV_ERROR RGXFWSetupCounterBuffer(PVRSRV_RGXDEV_INFO* psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32CounterDataBufferSize,
+		RGXFWIF_COUNTER_DUMP_CTL*	psCounterDumpCtl)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32CounterDataBufferSize,
+								  "FwCounterBuffer",
+								  ppsBufferMemDesc,
+								  &psCounterDumpCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psCounterDumpCtl->ui32SizeInDwords = ui32CounterDataBufferSize >> 2;
+	RGXFwSharedMemCacheOpValue(psCounterDumpCtl->ui32SizeInDwords, FLUSH);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+ @Function      RGXFWSetupAlignChecks
+ @Description   This functions allocates and fills memory needed for the
+                aligns checks of the UM and KM structures shared with the
+                firmware. The format of the data in the memory is as follows:
+                    <number of elements in the KM array>
+                    <array of KM structures' sizes and members' offsets>
+                    <number of elements in the UM array>
+                    <array of UM structures' sizes and members' offsets>
+                The UM array is passed from the user side. Now the firmware is
+                responsible for filling this part of the memory. If that
+                happens the check of the UM structures will be performed
+                by the host driver on client's connect.
+                If the macro is not defined the client driver fills the memory
+                and the firmware checks for the alignment of all structures.
+ @Input			psDeviceNode
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupAlignChecks(PVRSRV_DEVICE_NODE *psDeviceNode,
+								RGXFWIF_DEV_VIRTADDR	*psAlignChecksDevFW)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32			aui32RGXFWAlignChecksKM[] = { RGXFW_ALIGN_CHECKS_INIT_KM };
+	IMG_UINT32			ui32RGXFWAlignChecksTotal;
+	IMG_UINT32*			paui32AlignChecks;
+	PVRSRV_ERROR		eError;
+
+	/* In this case we don't know the number of elements in UM array.
+	 * We have to assume something so we assume RGXFW_ALIGN_CHECKS_UM_MAX.
+	 */
+	ui32RGXFWAlignChecksTotal = sizeof(aui32RGXFWAlignChecksKM)
+	                            + RGXFW_ALIGN_CHECKS_UM_MAX * sizeof(IMG_UINT32)
+	                            + 2 * sizeof(IMG_UINT32);
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32RGXFWAlignChecksTotal,
+								  "FwAlignmentChecks",
+								  &psDevInfo->psRGXFWAlignChecksMemDesc,
+								  psAlignChecksDevFW,
+								  (void**) &paui32AlignChecks,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Copy the values */
+		*paui32AlignChecks++ = ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+		OSCachedMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0],
+		                sizeof(aui32RGXFWAlignChecksKM));
+		paui32AlignChecks += ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+
+		*paui32AlignChecks = 0;
+
+		OSWriteMemoryBarrier(paui32AlignChecks);
+		RGXFwSharedMemCacheOpExec(paui32AlignChecks - (ARRAY_SIZE(aui32RGXFWAlignChecksKM) + 1),
+		                          ui32RGXFWAlignChecksTotal,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+
+
+	DevmemPDumpLoadMem(	psDevInfo->psRGXFWAlignChecksMemDesc,
+						0,
+						ui32RGXFWAlignChecksTotal,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXFWFreeAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo)
+{
+	if (psDevInfo->psRGXFWAlignChecksMemDesc != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWAlignChecksMemDesc);
+		psDevInfo->psRGXFWAlignChecksMemDesc = NULL;
+	}
+}
+
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+						   DEVMEM_MEMDESC		*psSrc,
+						   IMG_UINT32			uiExtraOffset,
+						   IMG_UINT32			ui32Flags)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	psDevVirtAddr;
+	PVRSRV_DEVICE_NODE	*psDeviceNode;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+
+	psDeviceNode = (PVRSRV_DEVICE_NODE *) DevmemGetConnection(psSrc);
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		IMG_UINT32          ui32Offset;
+		IMG_BOOL            bCachedInMETA;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+		IMG_UINT32          uiGPUCacheMode;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in META memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the META cache flags */
+		bCachedInMETA = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		/* Honour the SLC cache flags */
+		eError = DevmemDeviceCacheMode(uiDevFlags, &uiGPUCacheMode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemDeviceCacheMode", failDevCacheMode);
+
+		/*
+		 * Choose Meta virtual address based on Meta and SLC cacheability.
+		 */
+		ui32Offset += RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+		if (bCachedInMETA)
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+		}
+
+		if (PVRSRV_CHECK_GPU_CACHED(uiGPUCacheMode))
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_GOTO_IF_ERROR(eError, failDevVAAcquire);
+
+		ppDest->ui32Addr = (IMG_UINT32)((psDevVirtAddr.uiAddr + uiExtraOffset) & 0xFFFFFFFF);
+	}
+#endif
+	else
+	{
+		IMG_UINT32      ui32Offset;
+		IMG_BOOL        bCachedInRISCV;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in RISCV memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the RISCV cache flags */
+		bCachedInRISCV = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		if (bCachedInRISCV)
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_CACHED_DATA_BASE;
+		}
+		else
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_UNCACHED_DATA_BASE;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+
+	if ((ppDest->ui32Addr & 0x3U) != 0)
+	{
+		IMG_CHAR *pszAnnotation;
+		/* It is expected that the annotation returned by DevmemGetAnnotation() is always valid */
+		DevmemGetAnnotation(psSrc, &pszAnnotation);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: %s @ 0x%x is not aligned to 32 bit",
+				 __func__, pszAnnotation, ppDest->ui32Addr));
+
+		return PVRSRV_ERROR_INVALID_ALIGNMENT;
+	}
+
+	if (ui32Flags & RFW_FWADDR_NOREF_FLAG)
+	{
+		DevmemReleaseDevVirtAddr(psSrc);
+	}
+
+	return PVRSRV_OK;
+
+failDevCacheMode:
+	DevmemReleaseDevVirtAddr(psSrc);
+failDevVAAcquire:
+	return eError;
+}
+
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	sDevVirtAddr;
+
+	eError = DevmemAcquireDevVirtAddr(psSrcMemDesc, &sDevVirtAddr);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psDest->psDevVirtAddr.uiAddr = sDevVirtAddr.uiAddr;
+	psDest->psDevVirtAddr.uiAddr += uiOffset;
+	psDest->pbyFWAddr.ui32Addr = psSrcFWDevVAddr->ui32Addr;
+
+	DevmemReleaseDevVirtAddr(psSrcMemDesc);
+}
+
+
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc)
+{
+	DevmemReleaseDevVirtAddr(psSrc);
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCB
+ @Description	Free the kernel or firmware CCB
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppvCCBCtlLocal
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+******************************************************************************/
+static void RGXFreeCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtl,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+					   DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+					   IMG_UINT8			**ppui8CCB,
+					   DEVMEM_MEMDESC		**ppsCCBMemDesc)
+{
+	if (*ppsCCBMemDesc != NULL)
+	{
+		if (*ppui8CCB != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBMemDesc);
+			*ppui8CCB = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBMemDesc);
+		*ppsCCBMemDesc = NULL;
+	}
+	if (*ppsCCBCtlMemDesc != NULL)
+	{
+		if (*ppsCCBCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBCtlMemDesc);
+			*ppsCCBCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBCtlMemDesc);
+		*ppsCCBCtlMemDesc = NULL;
+	}
+	if (*ppsCCBCtlLocal != NULL)
+	{
+		OSFreeMem(*ppsCCBCtlLocal);
+		*ppsCCBCtlLocal = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCBReturnSlots
+ @Description	Free the kernel CCB's return slot array and associated mappings
+ @Input			psDevInfo              Device Info struct
+ @Input			ppui32CCBRtnSlots      CPU mapping of slot array
+ @Input			ppsCCBRtnSlotsMemDesc  Slot array's device memdesc
+******************************************************************************/
+static void RGXFreeCCBReturnSlots(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32         **ppui32CCBRtnSlots,
+								  DEVMEM_MEMDESC     **ppsCCBRtnSlotsMemDesc)
+{
+	/* Free the return slot array if allocated */
+	if (*ppsCCBRtnSlotsMemDesc != NULL)
+	{
+		/* Before freeing, ensure the CPU mapping as well is released */
+		if (*ppui32CCBRtnSlots != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBRtnSlotsMemDesc);
+			*ppui32CCBRtnSlots = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBRtnSlotsMemDesc);
+		*ppsCCBRtnSlotsMemDesc = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXSetupCCB
+ @Description	Allocate and initialise a circular command buffer
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppui8CCB
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+ @Input			ui32NumCmdsLog2
+ @Input			ui32CmdSize
+ @Input			uiCCBMemAllocFlags
+ @Input			pszName
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								RGXFWIF_CCB_CTL		**ppsCCBCtl,
+								RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+								DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+								IMG_UINT8			**ppui8CCB,
+								DEVMEM_MEMDESC		**ppsCCBMemDesc,
+								PRGXFWIF_CCB_CTL	*psCCBCtlFWAddr,
+								PRGXFWIF_CCB		*psCCBFWAddr,
+								IMG_UINT32			ui32NumCmdsLog2,
+								IMG_UINT32			ui32CmdSize,
+								PVRSRV_MEMALLOCFLAGS_T uiCCBMemAllocFlags,
+								const IMG_CHAR		*pszName)
+{
+	PVRSRV_ERROR		eError;
+	RGXFWIF_CCB_CTL		*psCCBCtl;
+	IMG_UINT32		ui32CCBSize = (1U << ui32NumCmdsLog2);
+	IMG_CHAR		szCCBCtlName[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_INT32		iStrLen;
+
+	/* Append "Control" to the name for the control struct. */
+	iStrLen = OSSNPrintf(szCCBCtlName, sizeof(szCCBCtlName), "%sControl", pszName);
+	PVR_ASSERT(iStrLen < sizeof(szCCBCtlName));
+
+	if (unlikely(iStrLen < 0))
+	{
+		OSStringSafeCopy(szCCBCtlName, "FwCCBControl", DEVMEM_ANNOTATION_MAX_LEN);
+	}
+
+	/* Allocate memory for the CCB control.*/
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_CCB_CTL),
+								  szCCBCtlName,
+								  ppsCCBCtlMemDesc,
+								  psCCBCtlFWAddr,
+								  (void**) ppsCCBCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Allocate memory for the CCB.
+	 * (this will reference further command data in non-shared CCBs)
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  uiCCBMemAllocFlags,
+								  ui32CCBSize * ui32CmdSize,
+								  pszName,
+								  ppsCCBMemDesc,
+								  psCCBFWAddr,
+								  (void**) ppui8CCB,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Initialise the CCB control.
+	 */
+	psCCBCtl = OSAllocZMem(sizeof(*psCCBCtl));
+	PVR_LOG_GOTO_IF_NOMEM(psCCBCtl, eError, fail);
+
+	psCCBCtl->ui32WrapMask = ui32CCBSize - 1;
+
+	OSDeviceMemCopy(*ppsCCBCtl, psCCBCtl, sizeof(*psCCBCtl));
+	RGXFwSharedMemCacheOpPtr(psCCBCtl, FLUSH);
+
+	*ppsCCBCtlLocal = psCCBCtl;
+
+	/* Pdump the CCB control */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Initialise %s", szCCBCtlName);
+	DevmemPDumpLoadMem(*ppsCCBCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCB_CTL),
+					   0);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeCCB(psDevInfo,
+			   ppsCCBCtl,
+			   ppsCCBCtlLocal,
+			   ppsCCBCtlMemDesc,
+			   ppui8CCB,
+			   ppsCCBMemDesc);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXSetupFaultReadRegisterRollback(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+#endif
+	{
+		if (psDevInfo->psRGXFaultAddressMemDesc)
+		{
+			if (DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR) == PVRSRV_OK)
+			{
+				PMRUnlockSysPhysAddresses(psPMR);
+			}
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+			psDevInfo->psRGXFaultAddressMemDesc = NULL;
+		}
+	}
+}
+
+static PVRSRV_ERROR RGXSetupFaultReadRegister(PVRSRV_DEVICE_NODE *psDeviceNode, RGXFWIF_SYSINIT *psFwSysInit)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			*pui32MemoryVirtAddr;
+	IMG_UINT32			i;
+	size_t				ui32PageSize = OSGetPageSize();
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PMR					*psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	/* Allocate page of memory to use for page faults on non-blocking memory transactions.
+	 * Doesn't need to be cleared as it is initialised with the 0xDEADBEEF pattern below. */
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+			ui32PageSize,
+			ui32PageSize,
+			RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS & ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+			"FwExFaultAddress",
+			&psDevInfo->psRGXFaultAddressMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate mem for fault address (%u)",
+		         __func__, eError));
+		goto failFaultAddressDescAlloc;
+	}
+
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc,
+										  (void **)&pui32MemoryVirtAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire mem for fault address (%u)",
+					 __func__, eError));
+			goto failFaultAddressDescAqCpuVirt;
+		}
+
+		/* fill the page with a known pattern when booting the firmware */
+		for (i = 0; i < ui32PageSize/sizeof(IMG_UINT32); i++)
+		{
+			*(pui32MemoryVirtAddr + i) = 0xDEADBEEF;
+		}
+
+		OSWriteMemoryBarrier(pui32MemoryVirtAddr);
+		RGXFwSharedMemCacheOpExec(pui32MemoryVirtAddr, ui32PageSize, PVRSRV_CACHE_OP_FLUSH);
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
+	}
+
+	eError = DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Error getting PMR for fault address (%u)",
+		         __func__, eError));
+
+		goto failFaultAddressDescGetPMR;
+	}
+	else
+	{
+		IMG_BOOL bValid;
+		IMG_UINT32 ui32Log2PageSize = OSGetPageShift();
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error locking physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescLockPhys;
+		}
+
+		eError = PMR_DevPhysAddr(psPMR,ui32Log2PageSize, 1, 0, &(psFwSysInit->sFaultPhysAddr), &bValid, DEVICE_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error getting physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescGetPhys;
+		}
+
+		if (!bValid)
+		{
+			psFwSysInit->sFaultPhysAddr.uiAddr = 0;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed getting physical address for fault address MemDesc - invalid page (0x%" IMG_UINT64_FMTSPECX ")",
+			         __func__, psFwSysInit->sFaultPhysAddr.uiAddr));
+
+			goto failFaultAddressDescGetPhys;
+		}
+	}
+
+	return PVRSRV_OK;
+
+failFaultAddressDescGetPhys:
+	PMRUnlockSysPhysAddresses(psPMR);
+
+failFaultAddressDescLockPhys:
+failFaultAddressDescGetPMR:
+failFaultAddressDescAqCpuVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+
+failFaultAddressDescAlloc:
+
+	return eError;
+}
+
+#if defined(PDUMP)
+/* Replace the DevPhy address with the one Pdump allocates at pdump_player run time */
+static PVRSRV_ERROR RGXPDumpFaultReadRegister(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	PMR *psFWInitPMR, *psFaultAddrPMR;
+	IMG_UINT32 ui32Dstoffset;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	psFWInitPMR = (PMR *)(psDevInfo->psRGXFWIfSysInitMemDesc->psImport->hPMR);
+	ui32Dstoffset = psDevInfo->psRGXFWIfSysInitMemDesc->uiOffset + offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr.uiAddr);
+
+	psFaultAddrPMR = (PMR *)(psDevInfo->psRGXFaultAddressMemDesc->psImport->hPMR);
+
+	eError = PDumpMemLabelToMem64(psFaultAddrPMR,
+				psFWInitPMR,
+				0,
+				ui32Dstoffset,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dump of Fault Page Phys address failed(%u)", __func__, eError));
+	}
+	return eError;
+}
+#endif
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW tbi buffer
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTBIBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTBIBufferMemDesc);
+	psDevInfo->psRGXFWIfTBIBufferMemDesc = NULL;
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = 0;
+	psDevInfo->ui32RGXL2HWPerfBufSize = 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferInitOnDemandResources
+
+@Description    Allocates the firmware TBI buffer required for reading SFs
+		strings and initialize it with SFs.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         i, ui32Len;
+	const IMG_UINT32   ui32FWTBIBufsize = g_ui32SFsCount * sizeof(RGXFW_STID_FMT);
+	RGXFW_STID_FMT     *psFW_SFs = NULL;
+
+	/* Firmware address should not be already set */
+	if (psDevInfo->sRGXFWIfTBIBuffer.ui32Addr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW address for FWTBI is already set. Resetting it with newly allocated one",
+		         __func__));
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS,
+								  ui32FWTBIBufsize,
+								  "FwTBIBuffer",
+								  &psDevInfo->psRGXFWIfTBIBufferMemDesc,
+								  &psDevInfo->sRGXFWIfTBIBuffer,
+								  (void**)&psFW_SFs,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* Copy SFs entries to FW buffer */
+	for (i = 0; i < g_ui32SFsCount; i++)
+	{
+		OSCachedMemCopy(&psFW_SFs[i].ui32Id, &SFs[i].ui32Id, sizeof(SFs[i].ui32Id));
+		ui32Len = OSStringLength(SFs[i].psName);
+		OSCachedMemCopy(psFW_SFs[i].sName, SFs[i].psName, MIN(ui32Len, IMG_SF_STRING_MAX_SIZE - 1));
+	}
+
+	/* flush write buffers for psFW_SFs */
+	OSWriteMemoryBarrier(psFW_SFs);
+
+	/* Set size of TBI buffer */
+	psDevInfo->ui32FWIfTBIBufferSize = ui32FWTBIBufsize;
+
+	/* release CPU mapping */
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTBIBufferMemDesc);
+
+	return PVRSRV_OK;
+fail:
+	RGXTBIBufferDeinit(psDevInfo);
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW trace buffer(s)
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTraceBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufferMemDesc[i])
+		{
+			if (psDevInfo->apui32TraceBuffer[i] != NULL)
+			{
+				DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+				psDevInfo->apui32TraceBuffer[i] = NULL;
+			}
+
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+			psDevInfo->psRGXFWIfTraceBufferMemDesc[i] = NULL;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferInitOnDemandResources
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+		info from the firmware.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo,
+												 PVRSRV_MEMALLOCFLAGS_T uiAllocFlags)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         ui32FwThreadNum;
+	IMG_UINT32         ui32DefaultTraceBufSize;
+	IMG_DEVMEM_SIZE_T  uiTraceBufSizeInBytes;
+	void               *pvAppHintState = NULL;
+	IMG_CHAR           pszBufferName[] = "FwTraceBuffer_Thread0";
+
+	/* Check AppHint value for module-param FWTraceBufSizeInDWords */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32DefaultTraceBufSize = RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,
+						 pvAppHintState,
+						 FWTraceBufSizeInDWords,
+						 &ui32DefaultTraceBufSize,
+						 &psDevInfo->ui32TraceBufSizeInDWords);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/* Write tracebuf size once to devmem */
+	psTraceBufCtl->ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (psDevInfo->ui32TraceBufSizeInDWords < RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS ||
+		psDevInfo->ui32TraceBufSizeInDWords > RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Requested trace buffer size (%u) out of its minimum (%u) & maximum (%u) range. Exiting error.",
+				  __func__,
+				  psDevInfo->ui32TraceBufSizeInDWords,
+				  RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS,
+				  RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS));
+		eError = PVRSRV_ERROR_OUT_OF_RANGE;
+		goto exit_error;
+	}
+
+	uiTraceBufSizeInBytes = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+
+	for (ui32FwThreadNum = 0; ui32FwThreadNum < RGXFW_THREAD_NUM; ui32FwThreadNum++)
+	{
+#if !defined(SUPPORT_AUTOVZ)
+		/* Ensure allocation API is only called when not already allocated */
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum] == NULL);
+		/* Firmware address should not be already set */
+		PVR_ASSERT(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer.ui32Addr == 0x0);
+#endif
+
+		/* update the firmware thread number in the Trace Buffer's name */
+		pszBufferName[sizeof(pszBufferName) - 2] += ui32FwThreadNum;
+
+		eError = RGXSetupFwAllocation(psDevInfo,
+									  uiAllocFlags,
+									  uiTraceBufSizeInBytes,
+									  pszBufferName,
+									  &psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum],
+									  &psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer,
+									  (void**)&psDevInfo->apui32TraceBuffer[ui32FwThreadNum],
+									  RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+	}
+
+	return PVRSRV_OK;
+
+fail:
+	RGXTraceBufferDeinit(psDevInfo);
+exit_error:
+	return eError;
+}
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       RGXPDumpLoadFWInitData
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+                info from the firmware.
+
+@Input          psDevInfo RGX device info
+ */ /*************************************************************************/
+static void RGXPDumpLoadFWInitData(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   RGX_INIT_APPHINTS  *psApphints,
+								   IMG_UINT32         ui32HWPerfCountersDataSize)
+{
+	IMG_UINT32 ui32ConfigFlags  = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+	IMG_UINT32 ui32FwOsCfgFlags = psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Dump RGXFW Init data");
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "(to enable rgxfw signatures place the following line after the RTCONF line)");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, asSigBufCtl),
+						   sizeof(RGXFWIF_SIGBUF_CTL)*(RGXFWIF_DM_MAX),
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump initial state of FW runtime configuration");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_RUNTIME_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw hwperfctl structure");
+	DevmemPDumpLoadZeroMem (psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+							0,
+							ui32HWPerfCountersDataSize,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw trace control structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TRACEBUF),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware system data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware OS data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgx TBI buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTBIBufferMemDesc,
+					   0,
+					   psDevInfo->ui32FWIfTBIBufferSize,
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw register configuration buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRegCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_REG_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_USER_REGISTER_CONFIGURATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw system init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw os init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfOsInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* RGXFW Init structure needs to be loaded before we overwrite FaultPhysAddr, else this address patching won't have any effect */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Overwrite FaultPhysAddr of FwSysInit in pdump with actual physical address");
+	RGXPDumpFaultReadRegister(psDevInfo);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "RTCONF: run-time configuration");
+
+	/* Dump the config options so they can be edited. */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW system config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Rand mode:                      0x%08x)", RGXFWIF_INICFG_CTXSWITCH_MODE_RAND);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Soft Reset Enable:              0x%08x)", RGXFWIF_INICFG_CTXSWITCH_SRESET_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable HWPerf:                             0x%08x)", RGXFWIF_INICFG_HWPERF_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Rascal+Dust Power Island:                  0x%08x)", RGXFWIF_INICFG_POW_RASCALDUST);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( FBCDC Version 3.1 Enable:                  0x%08x)", RGXFWIF_INICFG_FBCDC_V3_1_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Check MList:                               0x%08x)", RGXFWIF_INICFG_CHECK_MLIST_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable Auto Clock Gating:                 0x%08x)", RGXFWIF_INICFG_DISABLE_CLKGATING_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+				 "( Try overlapping DM pipelines:              0x%08x)", RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable DM pipeline roadblocks:             0x%08x)", RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#endif
+				 "( Enable register configuration:             0x%08x)", RGXFWIF_INICFG_REGCONFIG_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on TA Out-of-Memory:                0x%08x)", RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable HWPerf custom counter filter:      0x%08x)", RGXFWIF_INICFG_HWP_DISABLE_FILTER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable Ctx Switch profile mode: 0x%08x (none=b'000, fast=b'001, medium=b'010, slow=b'011, nodelay=b'100))", RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable DM overlap (except TA during SPM): 0x%08x)", RGXFWIF_INICFG_DISABLE_DM_OVERLAP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on HWR trigger (page fault, lockup, overrun or poll failure): 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable IRQ validation:                     0x%08x)", RGXFWIF_INICFG_VALIDATE_IRQ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPU power state mask change Enable:        0x%08x)", RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN);
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Workload Estimation:                0x%08x)", RGXFWIF_INICFG_WORKEST);
+#if defined(SUPPORT_PDVFS)
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Proactive DVFS:                     0x%08x)", RGXFWIF_INICFG_PDVFS);
+#endif /* defined(SUPPORT_PDVFS) */
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( ISP Scheduling Mode (v1=b'01, v2=b'10):    0x%08x)", RGXFWIF_INICFG_ISPSCHEDMODE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Validate SOC & USC timers:                 0x%08x)", RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+							offsetof(RGXFWIF_SYSDATA, ui32ConfigFlags),
+							ui32ConfigFlags,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Extended FW system config options not used.)");
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW OS config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch TDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch GEOM Enable:                    0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch 3D Enable:                      0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch CDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch RDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN);
+#endif
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  2D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  TA Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  3D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch CDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch RDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM);
+#endif
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+							  offsetof(RGXFWIF_OSDATA, ui32FwOsConfigFlags),
+							  ui32FwOsCfgFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	{
+		PDUMP_FLAGS_T      ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+		IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+		IMG_BOOL           bRunTimeUpdate = IMG_FALSE;
+		IMG_UINT32         ui32DstOffset = psDevInfo->psRGXFWIfRuntimeCfgMemDesc->uiOffset + offsetof(RGXFWIF_RUNTIME_CFG, ui32PowUnitsState);
+		IMG_CHAR           aszPowUnitsMaskRegVar[] = ":SYSMEM:$1";
+		IMG_CHAR           aszPowUnitsEnable[] = "RUNTIME_POW_UNITS_MASK";
+		PMR                *psPMR = (PMR *)(psDevInfo->psRGXFWIfRuntimeCfgMemDesc->psImport->hPMR);
+
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPIF(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+		                      "Load initial value power units mask in FW runtime configuration");
+		DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  ui32DstOffset,
+								  psDevInfo->psRGXFWIfRuntimeCfg->ui32PowUnitsState,
+								  ui32PDumpFlags);
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPELSE(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+			PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags, "Read initial SPU mask value from HW registers");
+			PDumpRegRead32ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_SPU_ENABLE, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, aszPowUnitsMaskRegVar, ui32AllPowUnitsMask, ui32PDumpFlags);
+			PDumpInternalVarToMemLabel(psPMR, ui32DstOffset, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDUMPFI(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+	}
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Select one or more security tests here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW private data from non-FW contexts: 0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW code from non-FW contexts:         0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from non-secure memory:          0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from secure (non-FW) memory:     0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui32SecurityTestFlags),
+							  psDevInfo->psRGXFWIfSysInit->ui32SecurityTestFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_SECURITY_VALIDATION) */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter type: %X=INCLUDE_ALL_EXCEPT, %X=EXCLUDE_ALL_EXCEPT)",
+				 RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+				 RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, sPIDFilter.eMode),
+							  psDevInfo->psRGXFWIfSysInit->sPIDFilter.eMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter PID/DriverID list (Up to %u entries. Terminate with a zero PID))",
+				 RGXFWIF_PID_FILTER_MAX_NUM_PIDS);
+	{
+		IMG_UINT32 i;
+
+		/* generate a few WRWs in the pdump stream as an example */
+		for (i = 0; i < MIN(RGXFWIF_PID_FILTER_MAX_NUM_PIDS, 8); i++)
+		{
+			/*
+			 * Some compilers cannot cope with the uses of offsetof() below - the specific problem being the use of
+			 * a non-const variable in the expression, which it needs to be const. Typical compiler output is
+			 * "expression must have a constant value".
+			 */
+			const IMG_DEVMEM_OFFSET_T uiPIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].uiPID);
+
+			const IMG_DEVMEM_OFFSET_T uiDriverIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].ui32DriverID);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID and DriverID pair %u)", i);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiPIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(DriverID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiDriverIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	/*
+	 * Dump the log config so it can be edited.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the log config here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Log Type: TRACE mode using shared memory buffer: 0x00000001)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           TBI mode via external interface or sim support: 0x00000000)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           Note: TBI mode will hang on most hardware devices!)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MAIN Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MAIN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MTS Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MTS);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CLEANUP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CLEANUP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CSW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CSW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( BIF Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_BIF);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_PM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( RTD Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RTD);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SPM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( POW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_POW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWR Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWR);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MISC Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MISC);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VZ Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VZ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SAFETY Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SAFETY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VERBOSE Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VERBOSE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CUSTOMER Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CUSTOMER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( DEBUG Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DEBUG);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+							  offsetof(RGXFWIF_TRACEBUF, ui32LogType),
+							  psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Set the HWPerf Filter config here, see \"hwperfbin2jsont -h\"");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui64HWPerfFilter),
+							  psDevInfo->psRGXFWIfSysInit->ui64HWPerfFilter,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Number of registers configurations for types(byte index): pow on(%d), dust change(%d), ta(%d), 3d(%d), cdm(%d), "
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 "tla(%d), "
+#endif
+				 "tdm(%d))",
+				 RGXFWIF_REG_CFG_TYPE_PWR_ON,
+				 RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,
+				 RGXFWIF_REG_CFG_TYPE_TA,
+				 RGXFWIF_REG_CFG_TYPE_3D,
+				 RGXFWIF_REG_CFG_TYPE_CDM,
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 RGXFWIF_REG_CFG_TYPE_TLA,
+#endif
+				 RGXFWIF_REG_CFG_TYPE_TDM);
+
+	{
+		IMG_UINT32 i;
+
+		/* Write 32 bits in each iteration as required by PDUMP WRW command */
+		for (i = 0; i < RGXFWIF_REG_CFG_TYPE_ALL; i += sizeof(IMG_UINT32))
+		{
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
+									offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[i]),
+									0,
+									PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "(Set registers here: address, mask, value)");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Addr),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Mask),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Value),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* SUPPORT_USER_REGISTER_CONFIGURATION */
+}
+#endif /* defined(PDUMP) */
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwGuardPage
+
+ @Description Allocate a Guard Page at the start of a Guest's Main Heap
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwGuardPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)),
+								  OSGetPageSize(),
+								  "FwGuardPage",
+								  &psDevInfo->psRGXFWHeapGuardPageReserveMemDesc,
+								  NULL,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwSysData
+
+ @Description Sets up all system-wide firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwSysData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									  RGX_INIT_APPHINTS        *psApphints,
+									  IMG_UINT32               ui32ConfigFlags,
+									  IMG_UINT32               ui32ConfigFlagsExt,
+									  IMG_UINT32               ui32HWPerfCountersDataSize)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32         ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+	RGXFWIF_SYSINIT    *psFwSysInitScratch = NULL;
+#if defined(PDUMP)
+	IMG_UINT32         ui32SignatureChecksBufSize = psApphints->ui32SignatureChecksBufSize;
+#endif
+
+	psFwSysInitScratch = OSAllocZMem(sizeof(*psFwSysInitScratch));
+	PVR_LOG_GOTO_IF_NOMEM(psFwSysInitScratch, eError, fail);
+
+	/* Sys Fw init data */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              (RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+	                              PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSINIT),
+	                              "FwSysInitStructure",
+	                              &psDevInfo->psRGXFWIfSysInitMemDesc,
+	                              NULL,
+	                              (void**) &psDevInfo->psRGXFWIfSysInit,
+	                              RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Sys Init structure allocation", fail);
+
+	/* Setup Fault read register */
+	eError = RGXSetupFaultReadRegister(psDeviceNode, psFwSysInitScratch);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Fault read register setup", fail);
+
+#if defined(SUPPORT_AUTOVZ)
+	psFwSysInitScratch->ui32VzWdgPeriod = PVR_AUTOVZ_WDG_PERIOD_MS;
+#endif
+
+	/* RD Power Island */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableRDPowIsland = psRGXData->psRGXTimingInfo->bEnableRDPowIsland;
+		IMG_BOOL bEnableRDPowIsland = ((psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_DEFAULT) && bSysEnableRDPowIsland) ||
+		                               (psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_FORCE_ON);
+
+		ui32ConfigFlags |= bEnableRDPowIsland? RGXFWIF_INICFG_POW_RASCALDUST : 0;
+	}
+
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		ui32ConfigFlags |= RGXFWIF_INICFG_WORKEST;
+#endif
+#if defined(SUPPORT_FW_OPP_TABLE)
+		{
+			RGXFWIF_OPP_INFO    *psOPPInfo;
+			IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg;
+
+			/* Pro-active DVFS depends on Workload Estimation */
+			psOPPInfo = &psFwSysInitScratch->sOPPInfo;
+			psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+#if defined(CONFIG_OF) && defined(CONFIG_PM_OPP) && !defined(NO_HARDWARE)
+			if (psDVFSDeviceCfg->bDTConfig)
+			{
+				/* OPP table configured from Device tree */
+				eError = DVFSCopyOPPTable(psDeviceNode,
+										  psOPPInfo,
+										  ARRAY_SIZE(psOPPInfo->asOPPValues));
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Unable to copy OPP table to FW init buffer (%u)", eError));
+					goto fail;
+				}
+			}
+#endif
+			if (!psDVFSDeviceCfg->bDTConfig)
+			{
+				PVR_LOG_IF_FALSE(psDVFSDeviceCfg->pasOPPTable, "RGXSetupFwSysData: Missing OPP Table");
+
+				if (psDVFSDeviceCfg->pasOPPTable != NULL)
+				{
+					if (psDVFSDeviceCfg->ui32OPPTableSize > ARRAY_SIZE(psOPPInfo->asOPPValues))
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								 "%s: OPP Table too large: Size = %u, Maximum size = %lu",
+								 __func__,
+								 psDVFSDeviceCfg->ui32OPPTableSize,
+								 (unsigned long)(ARRAY_SIZE(psOPPInfo->asOPPValues))));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto fail;
+					}
+
+					OSDeviceMemCopy(psOPPInfo->asOPPValues,
+									psDVFSDeviceCfg->pasOPPTable,
+									sizeof(psOPPInfo->asOPPValues));
+
+					psOPPInfo->ui32MaxOPPPoint = psDVFSDeviceCfg->ui32OPPTableSize - 1;
+
+				}
+			}
+
+#if defined(SUPPORT_PDVFS)
+			ui32ConfigFlags |= RGXFWIF_INICFG_PDVFS;
+#endif
+		}
+#endif /* defined(SUPPORT_FW_OPP_TABLE) */
+	}
+
+	/* FW trace control structure */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_TRACEBUF),
+								  "FwTraceCtlStruct",
+								  &psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								  &psFwSysInitScratch->sTraceBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfTraceBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Set initial firmware log type/group(s) */
+		if (psApphints->ui32LogType & ~RGXFWIF_LOG_TYPE_MASK)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid initial log type (0x%X)",
+			         __func__, psApphints->ui32LogType));
+			goto fail;
+		}
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = psApphints->ui32LogType;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+	}
+
+	/* When PDUMP is enabled, ALWAYS allocate on-demand trace buffer resource
+	 * (irrespective of loggroup(s) enabled), given that logtype/loggroups can
+	 * be set during PDump playback in logconfig, at any point of time,
+	 * Otherwise, allocate only if required. */
+#if !defined(PDUMP)
+#if defined(SUPPORT_AUTOVZ)
+	/* always allocate trace buffer for AutoVz Host drivers to allow
+	 * deterministic addresses of all SysData structures */
+	if ((PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo)) || (RGXTraceBufferIsInitRequired(psDevInfo)))
+#else
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+#endif
+#endif
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo,
+													 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+													 RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp));
+	}
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXTraceBufferInitOnDemandResources", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSDATA),
+	                              "FwSysData",
+	                              &psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                              &psFwSysInitScratch->sFwSysData,
+	                              (void**) &psDevInfo->psRGXFWIfFwSysData,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* GPIO validation setup */
+	psFwSysInitScratch->eGPIOValidationMode = RGXFWIF_GPIO_VAL_OFF;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = RGXFWSetupCounterBuffer(psDevInfo,
+									 &psDevInfo->psCounterBufferMemDesc,
+									 OSGetPageSize(),
+									 &psFwSysInitScratch->sCounterDumpCtl);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Counter Buffer allocation", fail);
+
+	PVR_DPF((PVR_DBG_WARNING, "Counter buffer allocated at %p, size %zu Bytes.", psDevInfo->psCounterBufferMemDesc, OSGetPageSize()));
+#endif /* defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = RGXFWSetupFirmwareGcovBuffer(psDevInfo,
+	                                      &psDevInfo->psFirmwareGcovBufferMemDesc,
+	                                      RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE,
+	                                      &psFwSysInitScratch->sFirmwareGcovCtl,
+	                                      "FirmwareGcovBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware GCOV buffer allocation", fail);
+	psDevInfo->ui32FirmwareGcovSize = RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE;
+#endif /* defined(SUPPORT_FIRMWARE_GCOV) */
+
+#if defined(PDUMP)
+	/* Require a minimum amount of memory for the signature buffers */
+	if (ui32SignatureChecksBufSize < RGXFW_SIG_BUFFER_SIZE_MIN)
+	{
+		ui32SignatureChecksBufSize = RGXFW_SIG_BUFFER_SIZE_MIN;
+	}
+
+	/* Setup Signature and Checksum Buffers */
+	psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	psDevInfo->ui32SigTDMChecksSize = 0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* Buffer allocated only when feature present because all known TDM
+		 * signature registers are dependent on this feature being present */
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TDM Signature check setup", fail);
+		psDevInfo->ui32SigTDMChecksSize = ui32SignatureChecksBufSize;
+	}
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GEOM Signature check setup", fail);
+	psDevInfo->ui32SigTAChecksSize = ui32SignatureChecksBufSize;
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "3D Signature check setup", fail);
+	psDevInfo->ui32Sig3DChecksSize = ui32SignatureChecksBufSize;
+
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RDM Signature check setup", fail);
+		psDevInfo->ui32SigRDMChecksSize = ui32SignatureChecksBufSize;
+	}
+#endif
+
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_CDM].sBuffer.ui32Addr = 0x0;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY].sBuffer.ui32Addr = 0x0;
+#endif
+	}
+#endif /* defined(PDUMP) */
+
+	eError = RGXFWSetupAlignChecks(psDeviceNode,
+	                               &psFwSysInitScratch->sAlignChecks);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Alignment checks setup", fail);
+
+	psFwSysInitScratch->ui32FilterFlags = psApphints->ui32FilterFlags;
+
+	/* Fill the remaining bits of fw the init data */
+	psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_HEAP_BASE;
+	psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_HEAP_BASE;
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/* Fill the remaining bits of fw the init data */
+		psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_BRN_65273_HEAP_BASE;
+		psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_BRN_65273_HEAP_BASE;
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/* Core clock rate */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(IMG_UINT32),
+	                              "FwPDVFSCoreClkRate",
+	                              &psDevInfo->psRGXFWIFCoreClkRateMemDesc,
+	                              &psFwSysInitScratch->sCoreClockRate,
+	                              (void**) &psDevInfo->pui32RGXFWIFCoreClkRate,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDVFS core clock rate memory setup", fail);
+#endif
+	{
+	/* Timestamps */
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags =
+		PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+		PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		PVRSRV_MEMALLOCFLAG_GPU_READABLE | /* XXX ?? */
+		PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+	/*
+	  the timer query arrays
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate timer query arrays (FW)");
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwStartTimesArray",
+	                          & psDevInfo->psStartTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psStartTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64StartTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwEndTimesArray",
+	                          & psDevInfo->psEndTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psEndTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64EndTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT32) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags,
+	                          "FwCompletedOpsArray",
+	                          & psDevInfo->psCompletedMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to completed ops array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCompletedMemDesc,
+	                                  (void **)& psDevInfo->pui32CompletedById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map completed ops array",
+				__func__));
+		goto fail;
+	}
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	eError = OSLockCreate(&psDevInfo->hTimerQueryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate log for timer query",
+				__func__));
+		goto fail;
+	}
+#endif
+#if defined(SUPPORT_TBI_INTERFACE)
+#if !defined(PDUMP)
+	/* allocate only if required */
+	if (RGXTBIBufferIsInitRequired(psDevInfo))
+#endif /* !defined(PDUMP) */
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand TBI buffer resource
+		 * (irrespective of loggroup(s) enabled), given that logtype/loggroups
+		 * can be set during PDump playback in logconfig, at any point of time
+		 */
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXTBIBufferInitOnDemandResources", fail);
+	}
+
+	psFwSysInitScratch->sTBIBuf = psDevInfo->sRGXFWIfTBIBuffer;
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+	/* Allocate shared buffer for GPU utilisation.
+	 * Enable FIRMWARE_CACHED to reduce read latency in the FW.
+	 * The FW flushes the cache after any writes.
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_GPU_UTIL_FW),
+								  "FwGPUUtilisationBuffer",
+								  &psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc,
+								  &psFwSysInitScratch->sGpuUtilFWCtl,
+								  (void**) &psDevInfo->psRGXFWIfGpuUtilFW,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GPU Utilisation Buffer ctl allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_RUNTIME_CFG),
+								  "FwRuntimeCfg",
+								  &psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  &psFwSysInitScratch->sRuntimeCfg,
+								  (void**) &psDevInfo->psRGXFWIfRuntimeCfg,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware runtime configuration memory allocation", fail);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_REG_CFG),
+								  "FwRegisterConfigStructure",
+								  &psDevInfo->psRGXFWIfRegCfgMemDesc,
+								  &psFwSysInitScratch->sRegCfg,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware register user configuration structure allocation", fail);
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS | PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  RGXFW_SCRATCH_BUF_SIZE,
+								  "FwScratchBuf",
+								  &psDevInfo->psRGXFWScratchBufMemDesc,
+								  &psFwSysInitScratch->pbFwScratchBuf,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware scratch buffer allocation", fail);
+#endif
+
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = GetHwPerfBufferSize(psApphints->ui32HWPerfFWBufSize);
+
+	/* Host L2 HWPERF buffer size in bytes must be bigger than the L1 buffer
+	 * accessed by the FW. The MISR may try to write one packet the size of the L1
+	 * buffer in some scenarios. When logging is enabled in the MISR, it can be seen
+	 * if the L2 buffer hits a full condition. The closer in size the L2 and L1 buffers
+	 * are the more chance of this happening.
+	 * Size chosen to allow MISR to write an L1 sized packet and for the client
+	 * application/daemon to drain a L1 sized packet e.g. ~ 1.5*L1.
+	 */
+	psDevInfo->ui32RGXL2HWPerfBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize +
+	        (psDevInfo->ui32RGXFWIfHWPerfBufSize>>1);
+
+	/* Second stage initialisation or HWPerf, hHWPerfLock created in first
+	 * stage. See RGXRegisterDevice() call to RGXHWPerfInit(). */
+	if (psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF] == 0)
+	{
+		psFwSysInitScratch->ui64HWPerfFilter =
+		    RGXHWPerfFwSetEventFilter(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF,
+		                              (IMG_UINT64) psApphints->ui32HWPerfFilter0 |
+		                              ((IMG_UINT64) psApphints->ui32HWPerfFilter1 << 32));
+	}
+	else
+	{
+		/* The filter has already been modified. This can happen if
+		 * pvr/apphint/EnableFTraceGPU was enabled. */
+		psFwSysInitScratch->ui64HWPerfFilter = psDevInfo->ui64HWPerfFwFilter;
+	}
+
+#if !defined(PDUMP)
+	/* Allocate if HWPerf filter has already been set. This is possible either
+	 * by setting a proper AppHint or enabling GPU ftrace events. */
+	if (psFwSysInitScratch->ui64HWPerfFilter != 0)
+#endif
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand HWPerf resources
+		 * (irrespective of HWPerf enabled or not), given that HWPerf can be
+		 * enabled during PDump playback via RTCONF at any point of time. */
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL1Buffer", fail);
+
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL2Stream", fail);
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWCOMCTX_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32HWPerfCountersDataSize,
+								  "FwHWPerfControlStructure",
+								  &psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+								  &psFwSysInitScratch->sHWPerfCtl,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware HW Perf control struct allocation", fail);
+
+	psDevInfo->bPDPEnabled = (ui32ConfigFlags & RGXFWIF_INICFG_DISABLE_PDP_EN)
+							  ? IMG_FALSE : IMG_TRUE;
+
+	psFwSysInitScratch->eFirmwarePerf = psApphints->eFirmwarePerf;
+
+#if defined(PDUMP)
+	/* default: no filter */
+	psFwSysInitScratch->sPIDFilter.eMode = RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT;
+	psFwSysInitScratch->sPIDFilter.asItems[0].uiPID = 0;
+#endif
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	{
+		PVRSRV_MEMALLOCFLAGS_T uiFlags = RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS;
+		PVRSRV_SET_PHYS_HEAP_HINT(FW_PRIV_DATA, uiFlags);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate non-secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+											"FwExNonSecureBuffer",
+											&psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Non-secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbNonSecureBuffer,
+									   psDevInfo->psRGXFWIfNonSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											uiFlags,
+											"FwExSecureBuffer",
+											&psDevInfo->psRGXFWIfSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbSecureBuffer,
+									   psDevInfo->psRGXFWIfSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail);
+	}
+#endif /* SUPPORT_SECURITY_VALIDATION */
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) || RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		psFwSysInitScratch->ui32TFBCCompressionControl =
+			(ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >> RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+	}
+#endif /* RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK */
+
+	/* Initialize FW started flag */
+	psFwSysInitScratch->bFirmwareStarted = IMG_FALSE;
+	psFwSysInitScratch->ui32MarkerVal = 1;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = RGX_VZ_CONNECTION_COOLDOWN_PERIOD;
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		IMG_UINT32 ui32DriverID;
+
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+
+		/* Required info by FW to calculate the ActivePM idle timer latency */
+		psFwSysInitScratch->ui32InitialCoreClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psFwSysInitScratch->ui32InitialSOCClockSpeed = psRGXData->psRGXTimingInfo->ui32SOCClockSpeed;
+#endif
+		psFwSysInitScratch->ui32InitialActivePMLatencyms = psRGXData->psRGXTimingInfo->ui32ActivePMLatencyms;
+
+		/* Initialise variable runtime configuration to the system defaults */
+		psRuntimeCfg->ui32CoreClockSpeed = psFwSysInitScratch->ui32InitialCoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psRuntimeCfg->ui32SOCClockSpeed = psFwSysInitScratch->ui32InitialSOCClockSpeed;
+#endif
+		psRuntimeCfg->ui32ActivePMLatencyms = psFwSysInitScratch->ui32InitialActivePMLatencyms;
+		psRuntimeCfg->bActivePMLatencyPersistant = IMG_TRUE;
+		psRuntimeCfg->ui32HCSDeadlineMS = RGX_HCS_DEFAULT_DEADLINE_MS;
+
+		if ((RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US > 0U) && (RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US < 1000U))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_LOG_GOTO_IF_ERROR(eError,
+								  "RGXSetupFwSysData: RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US must be either 0 (disabled) or greater than 1000",
+								  fail);
+		}
+		else
+		{
+			psRuntimeCfg->ui32WdgPeriodUs = RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+		{
+			psRuntimeCfg->ai32DriverPriority[RGXFW_HOST_DRIVER_ID] = 0;
+			psRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID] = RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP;
+			psRuntimeCfg->aui32TSPercentage[RGXFW_HOST_DRIVER_ID] = (IMG_UINT8)RGX_DRIVERID_0_DEFAULT_TIME_SLICE;
+		}
+		else
+		{
+			const RGX_QOS_DEFAULTS asQosDefaults[RGXFW_MAX_NUM_OSIDS] = {
+					RGX_QOS_DEFAULTS_INIT(0),
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+					RGX_QOS_DEFAULTS_INIT(1),
+#if (RGXFW_MAX_NUM_OSIDS > 2)
+					RGX_QOS_DEFAULTS_INIT(2),
+					RGX_QOS_DEFAULTS_INIT(3),
+					RGX_QOS_DEFAULTS_INIT(4),
+					RGX_QOS_DEFAULTS_INIT(5),
+					RGX_QOS_DEFAULTS_INIT(6),
+					RGX_QOS_DEFAULTS_INIT(7),
+#if (RGXFW_MAX_NUM_OSIDS > 8)
+					RGX_QOS_DEFAULTS_INIT(8),
+					RGX_QOS_DEFAULTS_INIT(9),
+					RGX_QOS_DEFAULTS_INIT(10),
+					RGX_QOS_DEFAULTS_INIT(11),
+					RGX_QOS_DEFAULTS_INIT(12),
+					RGX_QOS_DEFAULTS_INIT(13),
+					RGX_QOS_DEFAULTS_INIT(14),
+					RGX_QOS_DEFAULTS_INIT(15),
+					RGX_QOS_DEFAULTS_INIT(16),
+					RGX_QOS_DEFAULTS_INIT(17),
+					RGX_QOS_DEFAULTS_INIT(18),
+					RGX_QOS_DEFAULTS_INIT(19),
+					RGX_QOS_DEFAULTS_INIT(20),
+					RGX_QOS_DEFAULTS_INIT(21),
+					RGX_QOS_DEFAULTS_INIT(22),
+					RGX_QOS_DEFAULTS_INIT(23),
+					RGX_QOS_DEFAULTS_INIT(24),
+					RGX_QOS_DEFAULTS_INIT(25),
+					RGX_QOS_DEFAULTS_INIT(26),
+					RGX_QOS_DEFAULTS_INIT(27),
+					RGX_QOS_DEFAULTS_INIT(28),
+					RGX_QOS_DEFAULTS_INIT(29),
+					RGX_QOS_DEFAULTS_INIT(30),
+					RGX_QOS_DEFAULTS_INIT(31),
+#if (RGXFW_MAX_NUM_OSIDS > 32)
+#error "Support for more than 32 OSIDs not implemented."
+#endif
+#endif /* RGXFW_MAX_NUM_OSIDS > 8 */
+#endif /* RGXFW_MAX_NUM_OSIDS > 2 */
+
+#endif /* RGXFW_MAX_NUM_OSIDS > 1 */
+			};
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				/* Set up initial priorities between different OSes */
+				psRuntimeCfg->ai32DriverPriority[ui32DriverID] = asQosDefaults[ui32DriverID].i32Priority;
+				psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = asQosDefaults[ui32DriverID].ui32IsolationGroups;
+				psRuntimeCfg->aui32TSPercentage[ui32DriverID] = (asQosDefaults[ui32DriverID].ui32TSPercentage <=
+																				PVRSRV_VZ_TIME_SLICE_MAX) ?
+																				asQosDefaults[ui32DriverID].ui32TSPercentage:(0);
+			}
+		}
+		psRuntimeCfg->ui32TSIntervalMs = RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL;
+
+#if defined(PVR_ENABLE_PHR) && defined(PDUMP)
+		psRuntimeCfg->ui32PHRMode = RGXFWIF_PHR_MODE_RD_RESET;
+#else
+		psRuntimeCfg->ui32PHRMode = 0;
+#endif
+
+		/* Initialize the PowUnitsState Field to Max Dusts */
+		psRuntimeCfg->ui32PowUnitsState = psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount;
+
+		/* flush write buffers for psDevInfo->psRGXFWIfRuntimeCfg */
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfRuntimeCfg);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, FLUSH);
+
+		/* Setup FW coremem data */
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psFwSysInitScratch->sCorememDataStore.pbyFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+		}
+
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags    = ui32ConfigFlags    & RGXFWIF_INICFG_ALL;
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt = ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_ALL;
+
+		/* Initialise GPU utilisation buffer */
+		{
+			IMG_UINT64 ui64LastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64(), RGXFWIF_GPU_UTIL_STATE_IDLE);
+			RGXFWIF_DM eDM;
+			IMG_UINT64 ui64LastWordTimeShifted =
+			        RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64() >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT, RGXFWIF_GPU_UTIL_STATE_IDLE);
+			IMG_UINT32 ui32DriverID;
+
+			psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord = ui64LastWord;
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				RGXFWIF_GPU_STATS *psStats = &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID];
+
+				for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+				{
+					psStats->aui32DMOSLastWord[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted & IMG_UINT32_MAX);
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted >> 32);
+				}
+			}
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+		}
+
+		/* init HWPERF data */
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfUt = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfDropCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32FirstDropOrdinal = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32LastDropOrdinal = 0;
+
+		psDevInfo->psRGXFWIfFwSysData->ui32MemFaultCheck = 0;
+
+		// flush write buffers for psRGXFWIfFwSysData
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwSysData);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, FLUSH);
+
+		/*Send through the BVNC Feature Flags*/
+		eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, &psFwSysInitScratch->sBvncKmFeatureFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags", fail);
+
+		/* populate the real FwOsInit structure with the values stored in the scratch copy */
+		OSCachedMemCopyWMB(psDevInfo->psRGXFWIfSysInit, psFwSysInitScratch, sizeof(RGXFWIF_SYSINIT));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit,
+		                         FLUSH);
+	}
+
+	OSFreeMem(psFwSysInitScratch);
+
+
+	return PVRSRV_OK;
+
+fail:
+	if (psFwSysInitScratch)
+	{
+		OSFreeMem(psFwSysInitScratch);
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwOsData
+
+ @Description Sets up all os-specific firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwOsData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									 IMG_UINT32               ui32KCCBSizeLog2,
+									 IMG_UINT32               ui32HWRDebugDumpLimit,
+									 IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_OSINIT     sFwOsInitScratch;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSCachedMemSet(&sFwOsInitScratch, 0, sizeof(RGXFWIF_OSINIT));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXSetupFwGuardPage(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware heap guard pages", fail);
+	}
+
+	/* Memory tracking the connection state should be non-volatile and
+	 * is not cleared on allocation to prevent loss of pre-reset information */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS &
+								  ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+								  sizeof(RGXFWIF_CONNECTION_CTL),
+								  "FwConnectionCtl",
+								  &psDevInfo->psRGXFWIfConnectionCtlMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfConnectionCtl,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Connection Control structure allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED),
+								  sizeof(RGXFWIF_OSINIT),
+								  "FwOsInitStructure",
+								  &psDevInfo->psRGXFWIfOsInitMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfOsInit,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Os Init structure allocation", fail);
+
+	/* init HWR frame info */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_HWRINFOBUF),
+								  "FwHWRInfoBuffer",
+								  &psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
+								  &sFwOsInitScratch.sRGXFWIfHWRInfoBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfHWRInfoBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HWR Info Buffer allocation", fail);
+
+	/* Might be uncached. Be conservative and use a DeviceMemSet */
+	OSDeviceMemSet(psDevInfo->psRGXFWIfHWRInfoBufCtl, 0, sizeof(RGXFWIF_HWRINFOBUF));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, FLUSH);
+
+	/* Allocate a sync for power management */
+	eError = SyncPrimContextCreate(psDevInfo->psDeviceNode,
+	                               &psDevInfo->hSyncPrimContext);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive context allocation", fail);
+
+	eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psDevInfo->psPowSyncPrim, "fw power ack");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive allocation", fail);
+
+	/* Set up kernel CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psKernelCCBCtl,
+						 &psDevInfo->psKernelCCBCtlLocal,
+						 &psDevInfo->psKernelCCBCtlMemDesc,
+						 &psDevInfo->psKernelCCB,
+						 &psDevInfo->psKernelCCBMemDesc,
+						 &sFwOsInitScratch.psKernelCCBCtl,
+						 &sFwOsInitScratch.psKernelCCB,
+						 ui32KCCBSizeLog2,
+						 sizeof(RGXFWIF_KCCB_CMD),
+						 (RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS |
+						 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+						 "FwKernelCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB allocation", fail);
+
+	/* KCCB additionally uses a return slot array for FW to be able to send back
+	 * return codes for each required command
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  (1U << ui32KCCBSizeLog2) * sizeof(IMG_UINT32),
+								  "FwKernelCCBRtnSlots",
+								  &psDevInfo->psKernelCCBRtnSlotsMemDesc,
+								  &sFwOsInitScratch.psKernelCCBRtnSlots,
+								  (void**) &psDevInfo->pui32KernelCCBRtnSlots,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB return slot array allocation", fail);
+
+	/* Set up firmware CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psFirmwareCCBCtl,
+						 &psDevInfo->psFirmwareCCBCtlLocal,
+						 &psDevInfo->psFirmwareCCBCtlMemDesc,
+						 &psDevInfo->psFirmwareCCB,
+						 &psDevInfo->psFirmwareCCBMemDesc,
+						 &sFwOsInitScratch.psFirmwareCCBCtl,
+						 &sFwOsInitScratch.psFirmwareCCB,
+						 RGXFWIF_FWCCB_NUMCMDS_LOG2,
+						 sizeof(RGXFWIF_FWCCB_CMD),
+						 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+						 "FwCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware CCB allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_OSDATA),
+								  "FwOsData",
+								  &psDevInfo->psRGXFWIfFwOsDataMemDesc,
+								  &sFwOsInitScratch.sFwOsData,
+								  (void**) &psDevInfo->psRGXFWIfFwOsData,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags = ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_ALL;
+
+	eError = SyncPrimGetFirmwareAddr(psDevInfo->psPowSyncPrim, &psDevInfo->psRGXFWIfFwOsData->sPowerSync.ui32Addr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Get Sync Prim FW address", fail);
+
+	/* flush write buffers for psRGXFWIfFwOsData */
+	OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwOsData);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         FLUSH);
+
+	sFwOsInitScratch.ui32HWRDebugDumpLimit = ui32HWRDebugDumpLimit;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Set up Workload Estimation firmware CCB */
+		eError = RGXSetupCCB(psDevInfo,
+							 &psDevInfo->psWorkEstFirmwareCCBCtl,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+							 &psDevInfo->psWorkEstFirmwareCCB,
+							 &psDevInfo->psWorkEstFirmwareCCBMemDesc,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCBCtl,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCB,
+							 RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2,
+							 sizeof(RGXFWIF_WORKEST_FWCCB_CMD),
+							 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+							 "FwWEstCCB");
+		PVR_LOG_GOTO_IF_ERROR(eError, "Workload Estimation Firmware CCB allocation", fail);
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise the compatibility check data */
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sFWBVNC);
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sHWBVNC);
+	}
+
+	/* populate the real FwOsInit structure with the values stored in the scratch copy */
+	OSCachedMemCopyWMB(psDevInfo->psRGXFWIfOsInit, &sFwOsInitScratch, sizeof(RGXFWIF_OSINIT));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfOsInit,
+	                         FLUSH);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeFwOsData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFirmware
+
+ @Description Sets up all firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+							  RGX_INIT_APPHINTS        *psApphints,
+							  IMG_UINT32               ui32ConfigFlags,
+							  IMG_UINT32               ui32ConfigFlagsExt,
+							  IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32HWPerfCountersDataSize;
+
+	eError = RGXSetupFwOsData(psDeviceNode,
+							  psApphints->ui32KCCBSizeLog2,
+							  psApphints->ui32HWRDebugDumpLimit,
+							  ui32FwOsCfgFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware os data", fail);
+
+	ui32HWPerfCountersDataSize = sizeof(RGXFWIF_HWPERF_CTL);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest drivers do not configure system-wide firmware data */
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+	else
+	{
+		/* Native and Host drivers must initialise the firmware's system data */
+		eError = RGXSetupFwSysData(psDeviceNode,
+								   psApphints,
+								   ui32ConfigFlags,
+								   ui32ConfigFlagsExt,
+								   ui32HWPerfCountersDataSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware system data", fail);
+	}
+
+	psDevInfo->bFirmwareInitialised = IMG_TRUE;
+
+#if defined(PDUMP)
+	RGXPDumpLoadFWInitData(psDevInfo,
+	                       psApphints,
+	                       ui32HWPerfCountersDataSize);
+#endif /* PDUMP */
+
+fail:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwSysData
+
+ @Description Frees all system-wide firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	psDevInfo->bFirmwareInitialised = IMG_FALSE;
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc)
+	{
+		RGXFWFreeAlignChecks(psDevInfo);
+	}
+
+#if defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM) &&
+	    psDevInfo->psRGXFWSigTDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTDMChecksMemDesc);
+		psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSigTAChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTAChecksMemDesc);
+		psDevInfo->psRGXFWSigTAChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSig3DChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSig3DChecksMemDesc);
+		psDevInfo->psRGXFWSig3DChecksMemDesc = NULL;
+	}
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1 &&
+	   psDevInfo->psRGXFWSigRDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigRDMChecksMemDesc);
+		psDevInfo->psRGXFWSigRDMChecksMemDesc = NULL;
+	}
+#endif
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	if (psDevInfo->psCounterBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCounterBufferMemDesc);
+		psDevInfo->psCounterBufferMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	if (psDevInfo->psFirmwareGcovBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psFirmwareGcovBufferMemDesc);
+		psDevInfo->psFirmwareGcovBufferMemDesc = NULL;
+	}
+#endif
+
+	RGXSetupFaultReadRegisterRollback(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfGpuUtilFW != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+			psDevInfo->psRGXFWIfGpuUtilFW = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+		psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfRuntimeCfgMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfRuntimeCfg != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+			psDevInfo->psRGXFWIfRuntimeCfg = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+		psDevInfo->psRGXFWIfRuntimeCfgMemDesc = NULL;
+	}
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+		}
+	}
+
+	if (psDevInfo->psRGXFWIfTraceBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufCtl != NULL)
+		{
+			/* deinit/free the tracebuffer allocation */
+			RGXTraceBufferDeinit(psDevInfo);
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+			psDevInfo->psRGXFWIfTraceBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+		psDevInfo->psRGXFWIfTraceBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwSysDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwSysData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwSysDataMemDesc);
+			psDevInfo->psRGXFWIfFwSysData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwSysDataMemDesc);
+		psDevInfo->psRGXFWIfFwSysDataMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	if (psDevInfo->psRGXFWIfTBIBufferMemDesc)
+	{
+		RGXTBIBufferDeinit(psDevInfo);
+	}
+#endif
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	if (psDevInfo->psRGXFWIfRegCfgMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRegCfgMemDesc);
+		psDevInfo->psRGXFWIfRegCfgMemDesc = NULL;
+	}
+#endif
+	if (psDevInfo->psRGXFWIfHWPerfCountersMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		psDevInfo->psRGXFWIfHWPerfCountersMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDevInfo->psRGXFWIfNonSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		psDevInfo->psRGXFWIfNonSecureBufMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSecureBufMemDesc);
+		psDevInfo->psRGXFWIfSecureBufMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	if (psDevInfo->psRGXFWIFCoreClkRateMemDesc)
+	{
+		if (psDevInfo->pui32RGXFWIFCoreClkRate != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+			psDevInfo->pui32RGXFWIFCoreClkRate = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+		psDevInfo->psRGXFWIFCoreClkRateMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+		psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwOsData
+
+ @Description Frees all os-specific firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFreeCCBReturnSlots(psDevInfo,
+	                      &psDevInfo->pui32KernelCCBRtnSlots,
+	                      &psDevInfo->psKernelCCBRtnSlotsMemDesc);
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psKernelCCBCtl,
+	           &psDevInfo->psKernelCCBCtlLocal,
+	           &psDevInfo->psKernelCCBCtlMemDesc,
+	           &psDevInfo->psKernelCCB,
+	           &psDevInfo->psKernelCCBMemDesc);
+
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psFirmwareCCBCtl,
+	           &psDevInfo->psFirmwareCCBCtlLocal,
+	           &psDevInfo->psFirmwareCCBCtlMemDesc,
+	           &psDevInfo->psFirmwareCCB,
+	           &psDevInfo->psFirmwareCCBMemDesc);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFreeCCB(psDevInfo,
+		           &psDevInfo->psWorkEstFirmwareCCBCtl,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+		           &psDevInfo->psWorkEstFirmwareCCB,
+		           &psDevInfo->psWorkEstFirmwareCCBMemDesc);
+	}
+#endif
+
+	if (psDevInfo->psPowSyncPrim != NULL)
+	{
+		SyncPrimFree(psDevInfo->psPowSyncPrim);
+		psDevInfo->psPowSyncPrim = NULL;
+	}
+
+	if (psDevInfo->hSyncPrimContext != (IMG_HANDLE) NULL)
+	{
+		SyncPrimContextDestroy(psDevInfo->hSyncPrimContext);
+		psDevInfo->hSyncPrimContext = (IMG_HANDLE) NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfHWRInfoBufCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+			psDevInfo->psRGXFWIfHWRInfoBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+		psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwOsDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwOsData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwOsDataMemDesc);
+			psDevInfo->psRGXFWIfFwOsData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwOsDataMemDesc);
+		psDevInfo->psRGXFWIfFwOsDataMemDesc = NULL;
+	}
+
+	if (psDevInfo->psCompletedMemDesc)
+	{
+		if (psDevInfo->pui32CompletedById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psCompletedMemDesc);
+			psDevInfo->pui32CompletedById = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCompletedMemDesc);
+		psDevInfo->psCompletedMemDesc = NULL;
+	}
+	if (psDevInfo->psEndTimeMemDesc)
+	{
+		if (psDevInfo->pui64EndTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psEndTimeMemDesc);
+			psDevInfo->pui64EndTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psEndTimeMemDesc);
+		psDevInfo->psEndTimeMemDesc = NULL;
+	}
+	if (psDevInfo->psStartTimeMemDesc)
+	{
+		if (psDevInfo->pui64StartTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psStartTimeMemDesc);
+			psDevInfo->pui64StartTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psStartTimeMemDesc);
+		psDevInfo->psStartTimeMemDesc = NULL;
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	if (psDevInfo->hTimerQueryLock)
+	{
+		OSLockDestroy(psDevInfo->hTimerQueryLock);
+		psDevInfo->hTimerQueryLock = NULL;
+	}
+#endif
+
+	if (psDevInfo->psRGXFWHeapGuardPageReserveMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWHeapGuardPageReserveMemDesc);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFirmware
+
+ @Description Frees all the firmware-related allocations
+
+ @Input       psDevInfo
+******************************************************************************/
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFreeFwOsData(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfConnectionCtl)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtl = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfConnectionCtlMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInitMemDesc = NULL;
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+	if (psDevInfo->psRGXFWIfSysInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSysInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInitMemDesc = NULL;
+	}
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalKCCBRoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT32 ui32ReadOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update KCCB read offset */
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+	ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+	if (ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psKCCBCtlLocal->ui32ReadOffset = ui32ReadOffset;
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalFWCCBWoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT32 ui32WriteOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update FWCCB write offset */
+	RGXFwSharedMemCacheOpValue(psFWCCBCtl->ui32WriteOffset, INVALIDATE);
+	ui32WriteOffset = psFWCCBCtl->ui32WriteOffset;
+
+	if (ui32WriteOffset > psFWCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psFWCCBCtlLocal->ui32WriteOffset = ui32WriteOffset;
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireKernelCCBSlot
+
+ PURPOSE	: Attempts to obtain a slot in the Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32		*pui32Offset)
+{
+	IMG_UINT32 ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+#if defined(PDUMP)
+	/* Wait for sufficient CCB space to become available */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, 0,
+	                      "Wait for space to write kCCB woff=%u", psKCCBCtlLocal->ui32WriteOffset);
+	DevmemPDumpCBP(psDevInfo->psKernelCCBCtlMemDesc,
+	               offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+	               psKCCBCtlLocal->ui32WriteOffset,
+	               1,
+	               (psKCCBCtlLocal->ui32WrapMask + 1));
+#endif
+
+	if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+	}
+	*pui32Offset = ui32NextWriteOffset;
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXPollKernelCCBSlot
+
+ PURPOSE	: Poll for space in Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXPollKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32	ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+	if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+	{
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_OK;
+		}
+
+		/*
+		 * The following check doesn't impact performance, since the
+		 * CPU has to wait for the GPU anyway (full kernel CCB).
+		 */
+		if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return PVRSRV_ERROR_KERNEL_CCB_FULL;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXGetCmdMemCopySize
+
+ PURPOSE	: Calculates actual size of KCCB command getting used
+
+ PARAMETERS	: eCmdType     Type of KCCB command
+
+ RETURNS	: Returns actual size of KCCB command on success else zero
+******************************************************************************/
+static IMG_UINT32 RGXGetCmdMemCopySize(RGXFWIF_KCCB_CMD_TYPE eCmdType)
+{
+	/* First get offset of uCmdData inside the struct RGXFWIF_KCCB_CMD
+	 * This will account alignment requirement of uCmdData union
+	 *
+	 * Then add command-data size depending on command type to calculate actual
+	 * command size required to do mem copy
+	 *
+	 * NOTE: Make sure that uCmdData is the last member of RGXFWIF_KCCB_CMD struct.
+	 */
+	switch (eCmdType)
+	{
+		case RGXFWIF_KCCB_CMD_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_MMUCACHE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_MMUCACHEDATA);
+		}
+#if defined(SUPPORT_USC_BREAKPOINT)
+		case RGXFWIF_KCCB_CMD_BP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_BPDATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_SLCFLUSHINVAL:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_SLCFLUSHINVALDATA);
+		}
+		case RGXFWIF_KCCB_CMD_CLEANUP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CLEANUP_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_POW:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_POWER_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE:
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_ZSBUFFER_BACKING_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELIST_GS_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELISTS_RECONSTRUCTION_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_WRITE_OFFSET_UPDATE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FORCE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA);
+		}
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+		case RGXFWIF_KCCB_CMD_REGCONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_REGCONFIG_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS);
+		}
+#if defined(SUPPORT_PDVFS)
+		case RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_PDVFS_MAX_FREQ_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_OS_STATE_CHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COUNTER_DUMP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_COUNTER_DUMP_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_DA_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CORECLKSPEEDCHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL:
+		case RGXFWIF_KCCB_CMD_WDG_CFG:
+		case RGXFWIF_KCCB_CMD_PHR_CFG:
+		case RGXFWIF_KCCB_CMD_HEALTH_CHECK:
+		case RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL:
+		{
+			/* No command specific data */
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData);
+		}
+		case RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_DEV_VIRTADDR);
+		}
+		case RGXFWIF_KCCB_CMD_CANCEL_WORK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CANCEL_WORK_DATA);
+		}
+		default:
+		{
+			/* Invalid (OR) Unused (OR) Newly added command type */
+			return 0; /* Error */
+		}
+	}
+}
+
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+	                           INVALIDATE);
+	eError = PVRSRVWaitForValueKM(
+	              (IMG_UINT32 __iomem *)&psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFwSharedMemCacheOpExecPfn);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVWaitForValueKM");
+
+#if defined(PDUMP)
+	/* PDumping conditions same as RGXSendCommandRaw for the actual command and poll command to go in harmony */
+	if (PDumpCheckFlagsWrite(psDevInfo->psDeviceNode, ui32PDumpFlags))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Poll on KCCB slot %u for value %u (mask: 0x%x)", ui32SlotNum,
+					 RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED, RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED);
+
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+										ui32SlotNum * sizeof(IMG_UINT32),
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										PDUMP_POLL_OPERATOR_EQUAL,
+										ui32PDumpFlags);
+		PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXSendCommandRaw(PVRSRV_RGXDEV_INFO  *psDevInfo,
+									  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+									  IMG_UINT32          uiPDumpFlags,
+									  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE	*psDeviceNode = psDevInfo->psDeviceNode;
+	RGXFWIF_CCB_CTL		*psKCCBCtl;
+	RGXFWIF_CCB_CTL		*psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT8			*pui8KCCB = psDevInfo->psKernelCCB;
+	IMG_UINT32			ui32NewWriteOffset;
+	IMG_UINT32			ui32OldWriteOffset;
+	IMG_UINT32			ui32CmdMemCopySize;
+
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+#else
+	IMG_BOOL bContCaptureOn = PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER); /* client connected or in pdump init phase */
+	IMG_BOOL bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, uiPDumpFlags); /* Are we in capture range or continuous and not in a power transition */
+
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			if (!psDevInfo->bDumpedKCCBCtlAlready)
+			{
+				/* entering capture range */
+				psDevInfo->bDumpedKCCBCtlAlready = IMG_TRUE;
+
+				/* Wait for the live FW to catch up */
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: waiting on fw to catch-up, roff: %d, woff: %d",
+						__func__,
+						psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset));
+				PVRSRVPollForValueKM(psDeviceNode,
+				                     (IMG_UINT32 __iomem *)&psKCCBCtl->ui32ReadOffset,
+				                     ui32OldWriteOffset, 0xFFFFFFFF,
+				                     POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+				                     NULL);
+
+				/* Dump Init state of Kernel CCB control (read and write offset) */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+						"Initial state of kernel CCB Control, roff: %d, woff: %d",
+						psKCCBCtl->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset);
+
+				DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+						0,
+						sizeof(RGXFWIF_CCB_CTL),
+						uiPDumpFlags);
+			}
+		}
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		if ((likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+			(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo)))) ||
+			(KM_FW_CONNECTION_IS(READY, psDevInfo) && KM_OS_CONNECTION_IS(READY, psDevInfo)))
+		{
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: The firmware-driver connection is invalid:"
+									"driver state = %u / firmware state = %u;"
+									"expected READY (%u/%u) or ACTIVE (%u/%u) or in transition (%u/%u);",
+									__func__, KM_GET_OS_CONNECTION(psDevInfo), KM_GET_FW_CONNECTION(psDevInfo),
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_READY,
+									RGXFW_CONNECTION_OS_ACTIVE, RGXFW_CONNECTION_FW_ACTIVE,
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_ACTIVE));
+			eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+			goto _RGXSendCommandRaw_Exit;
+		}
+	}
+#endif
+
+	if (!OSLockIsLocked(psDeviceNode->hPowerLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called without power lock held!",
+				__func__));
+		PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+	}
+
+	/* Acquire a slot in the CCB */
+	eError = RGXAcquireKernelCCBSlot(psDevInfo, &ui32NewWriteOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto _RGXSendCommandRaw_Exit;
+	}
+
+	/* Calculate actual size of command to optimize device mem copy */
+	ui32CmdMemCopySize = RGXGetCmdMemCopySize(psKCCBCmd->eCmdType);
+	PVR_LOG_RETURN_IF_FALSE(ui32CmdMemCopySize !=0, "RGXGetCmdMemCopySize failed", PVRSRV_ERROR_INVALID_CCB_COMMAND);
+
+	/* Copy the command into the CCB */
+	OSCachedMemCopy(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)],
+	                psKCCBCmd, ui32CmdMemCopySize);
+	RGXFwSharedMemCacheOpExec(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)], ui32CmdMemCopySize, PVRSRV_CACHE_OP_FLUSH);
+
+	/* If non-NULL pui32CmdKCCBSlot passed-in, return the kCCB slot in which the command was enqueued */
+	if (pui32CmdKCCBSlot)
+	{
+		*pui32CmdKCCBSlot = ui32OldWriteOffset;
+
+		/* Each such command enqueue needs to reset the slot value first. This is so that a caller
+		 * doesn't get to see stale/false value in allotted slot */
+		OSWriteDeviceMem32WithWMB(&psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                          RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                           FLUSH);
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+							  "Reset kCCB slot number %u", ui32OldWriteOffset);
+		DevmemPDumpLoadMem(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+		                   ui32OldWriteOffset * sizeof(IMG_UINT32),
+						   sizeof(IMG_UINT32),
+						   uiPDumpFlags);
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Device (%p) KCCB slot %u reset with value %u for command type %x",
+		         __func__, psDevInfo, ui32OldWriteOffset, RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE, psKCCBCmd->eCmdType));
+	}
+
+	/* Memory barrier before KCCB write offset update. */
+	OSWriteMemoryBarrier(NULL);
+
+
+	/* Move past the current command */
+	psKCCBCtlLocal->ui32WriteOffset = ui32NewWriteOffset;
+	psKCCBCtl->ui32WriteOffset = ui32NewWriteOffset;
+	/* Read-back of memory before Kick MTS */
+	OSWriteMemoryBarrier(&psKCCBCtl->ui32WriteOffset);
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32WriteOffset, FLUSH);
+
+#if defined(PDUMP)
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			/* Dump new Kernel CCB content */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCB cmd woff = %d",
+					ui32OldWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBMemDesc,
+					ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD),
+					ui32CmdMemCopySize,
+					uiPDumpFlags);
+
+			/* Dump new kernel CCB write offset */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCBCtl woff: %d",
+					ui32NewWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					sizeof(IMG_UINT32),
+					uiPDumpFlags);
+
+			/* mimic the read-back of the write from above */
+			DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					ui32NewWriteOffset,
+					0xFFFFFFFF,
+					PDUMP_POLL_OPERATOR_EQUAL,
+					uiPDumpFlags);
+		}
+		/* out of capture range */
+		else
+		{
+			eError = RGXPdumpDrainKCCB(psDevInfo, ui32OldWriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", _RGXSendCommandRaw_Exit);
+		}
+	}
+#endif
+
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags, "MTS kick for kernel CCB");
+	/*
+	 * Kick the MTS to schedule the firmware.
+	 */
+	__MTSScheduleWrite(psDevInfo, RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK);
+
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_MTS_SCHEDULE,
+	           RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK, uiPDumpFlags);
+
+#if defined(SUPPORT_AUTOVZ)
+	RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+
+#if defined(NO_HARDWARE)
+	/* keep the roff updated because fw isn't there to update it */
+	psKCCBCtl->ui32ReadOffset = psKCCBCtlLocal->ui32WriteOffset;
+#endif
+
+_RGXSendCommandRaw_Exit:
+	return eError;
+}
+
+/******************************************************************************
+ FUNCTION	: _AllocDeferredCommand
+
+ PURPOSE	: Allocate a KCCB command and add it to KCCB deferred list
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: eKCCBType		Firmware Command type
+			: psKCCBCmd		Firmware Command
+			: uiPDumpFlags	Pdump flags
+
+ RETURNS	: PVRSRV_OK	If all went good, PVRSRV_ERROR_RETRY otherwise.
+******************************************************************************/
+static PVRSRV_ERROR _AllocDeferredCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                          RGXFWIF_KCCB_CMD   *psKCCBCmd,
+                                          IMG_UINT32         uiPDumpFlags)
+{
+	RGX_DEFERRED_KCCB_CMD *psDeferredCommand;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psDeferredCommand = OSAllocMem(sizeof(*psDeferredCommand));
+
+	if (!psDeferredCommand)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Deferring a KCCB command failed: allocation failure: requesting retry"));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeferredCommand->sKCCBcmd = *psKCCBCmd;
+	psDeferredCommand->uiPDumpFlags = uiPDumpFlags;
+	psDeferredCommand->psDevInfo = psDevInfo;
+
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_add_to_tail(&(psDevInfo->sKCCBDeferredCommandsListHead), &(psDeferredCommand->sListNode));
+	psDevInfo->ui32KCCBDeferredCommandsCount++;
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: _FreeDeferredCommand
+
+ PURPOSE	: Remove from the deferred list the sent deferred KCCB command
+
+ PARAMETERS	: psNode			Node in deferred list
+			: psDeferredKCCBCmd	KCCB Command to free
+
+ RETURNS	: None
+******************************************************************************/
+static void _FreeDeferredCommand(DLLIST_NODE *psNode, RGX_DEFERRED_KCCB_CMD *psDeferredKCCBCmd)
+{
+	dllist_remove_node(psNode);
+	psDeferredKCCBCmd->psDevInfo->ui32KCCBDeferredCommandsCount--;
+	OSFreeMem(psDeferredKCCBCmd);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXSendCommandsFromDeferredList
+
+ PURPOSE	: Try send KCCB commands in deferred list to KCCB
+		  Should be called by holding PowerLock
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: bPoll		Poll for space in KCCB
+
+ RETURNS	: PVRSRV_OK	If all commands in deferred list are sent to KCCB,
+			  PVRSRV_ERROR_KERNEL_CCB_FULL otherwise.
+******************************************************************************/
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psNode, *psNext;
+	RGX_DEFERRED_KCCB_CMD *psTempDeferredKCCBCmd;
+	DLLIST_NODE sCommandList;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDevInfo->psDeviceNode));
+
+	/* !!! Important !!!
+	 *
+	 * The idea of moving the whole list hLockKCCBDeferredCommandsList below
+	 * to the temporary list is only valid under the principle that all of the
+	 * operations are also protected by the power lock. It must be held
+	 * so that the order of the commands doesn't get messed up while we're
+	 * performing the operations on the local list.
+	 *
+	 * The necessity of releasing the hLockKCCBDeferredCommandsList comes from
+	 * the fact that _FreeDeferredCommand() is allocating memory and it can't
+	 * be done in atomic context (inside section protected by a spin lock).
+	 *
+	 * We're using spin lock here instead of mutex to quickly perform a check
+	 * if the list is empty in MISR without a risk that the MISR is going
+	 * to sleep due to a lock.
+	 */
+
+	/* move the whole list to a local list so it can be processed without lock */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_replace_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (dllist_is_empty(&sCommandList))
+		{
+			return PVRSRV_OK;
+		}
+
+		/* For every deferred KCCB command, try to send it*/
+		dllist_foreach_node(&sCommandList, psNode, psNext)
+		{
+			psTempDeferredKCCBCmd = IMG_CONTAINER_OF(psNode, RGX_DEFERRED_KCCB_CMD, sListNode);
+			eError = RGXSendCommandRaw(psTempDeferredKCCBCmd->psDevInfo,
+			                           &psTempDeferredKCCBCmd->sKCCBcmd,
+			                           psTempDeferredKCCBCmd->uiPDumpFlags,
+			                           NULL /* We surely aren't interested in kCCB slot number of deferred command */);
+			if (eError != PVRSRV_OK)
+			{
+				if (!bPoll)
+				{
+					eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+					goto cleanup_;
+				}
+				break;
+			}
+
+			_FreeDeferredCommand(psNode, psTempDeferredKCCBCmd);
+		}
+
+		if (bPoll)
+		{
+			PVRSRV_ERROR eErrPollForKCCBSlot;
+
+			/* Don't overwrite eError because if RGXPollKernelCCBSlot returns OK and the
+			 * outer loop times-out, we'll still want to return KCCB_FULL to caller
+			 */
+			eErrPollForKCCBSlot = RGXPollKernelCCBSlot(psDevInfo);
+			if (eErrPollForKCCBSlot == PVRSRV_ERROR_KERNEL_CCB_FULL)
+			{
+				eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+				goto cleanup_;
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+cleanup_:
+	/* if the local list is not empty put it back to the deferred list head
+	 * so that the old order of commands is retained */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_insert_list_at_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+										  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+										  IMG_UINT32          uiPDumpFlags,
+										  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	IMG_BOOL     bPoll = (pui32CmdKCCBSlot != NULL);
+	PVRSRV_ERROR eError;
+
+	/*
+	 * First try to Flush all the cmds in deferred list.
+	 *
+	 * We cannot defer an incoming command if the caller is interested in
+	 * knowing the command's kCCB slot: it plans to poll/wait for a
+	 * response from the FW just after the command is enqueued, so we must
+	 * poll for space to be available.
+	 */
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, bPoll);
+	if (eError == PVRSRV_OK)
+	{
+		eError = RGXSendCommandRaw(psDevInfo,
+								   psKCCBCmd,
+								   uiPDumpFlags,
+								   pui32CmdKCCBSlot);
+	}
+
+	/*
+	 * If we don't manage to enqueue one of the deferred commands or the command
+	 * passed as argument because the KCCB is full, insert the latter into the deferred commands list.
+	 * The deferred commands will also be flushed eventually by:
+	 *  - one more KCCB command sent for any DM
+	 *  - RGX_MISRHandler_CheckFWActivePowerState
+	 */
+	if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+	{
+		if (pui32CmdKCCBSlot == NULL)
+		{
+			eError = _AllocDeferredCommand(psDevInfo, psKCCBCmd, uiPDumpFlags);
+		}
+		else
+		{
+			/* Let the caller retry. Otherwise if we deferred the command and returned OK,
+			 * the caller can end up looking in a stale CCB slot.
+			 */
+			PVR_DPF((PVR_DBG_WARNING, "%s: Couldn't flush the deferred queue for a command (Type:%d) "
+			                        "- will be retried", __func__, psKCCBCmd->eCmdType));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32         *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+
+	/* Ensure Rogue is powered up before kicking MTS */
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: failed to acquire powerlock (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVPowerLock_Exit;
+	}
+
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition Rogue to ON (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+										  psKCCBCmd,
+										  ui32PDumpFlags,
+	                                      pui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to schedule command (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+#if defined(DEBUG)
+		/* PVRSRVDebugRequest must be called without powerlock */
+		PVRSRVPowerUnlock(psDeviceNode);
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+		goto _PVRSRVPowerLock_Exit;
+#endif
+	}
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_PVRSRVPowerLock_Exit:
+	return eError;
+}
+
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hCmdCompHandle;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSScheduleMISR(psDevInfo->hProcessQueuesMISR);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGX_MISRHandler_ScheduleProcessQueues
+
+ @Description - Sends uncounted kick to all the DMs (the FW will process all
+				the queue for all the DMs)
+******************************************************************************/
+static void RGX_MISRHandler_ScheduleProcessQueues(void *pvData)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO     *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR           eError;
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_OFF;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Check whether it's worth waking up the GPU */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guests are not permitted to change the device power state */
+		if ((eError != PVRSRV_OK) || (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+	else
+	{
+		if ((eError == PVRSRV_OK) && (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			RGXFWIF_GPU_UTIL_FW    *psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+			IMG_BOOL               bGPUHasWorkWaiting;
+
+			/* Check whether it's worth waking up the GPU */
+			RGXFwSharedMemCacheOpValue(psUtilFW->ui64GpuLastWord, INVALIDATE);
+			bGPUHasWorkWaiting =
+			    (RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord) == RGXFWIF_GPU_UTIL_STATE_BLOCKED);
+
+			if (!bGPUHasWorkWaiting)
+			{
+				/* all queues are empty, don't wake up the GPU */
+				PVRSRVPowerUnlock(psDeviceNode);
+				return;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		/* wake up the GPU */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition device to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+
+	/* Memory barrier before Kick MTS */
+	OSWriteMemoryBarrier(NULL);
+
+	/* uncounted kick to the FW */
+	HTBLOGK(HTB_SF_MAIN_KICK_UNCOUNTED);
+	__MTSScheduleWrite(psDevInfo, (RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return OSInstallMISR(phMISR,
+			RGX_MISRHandler_ScheduleProcessQueues,
+			psDeviceNode,
+			"RGX_ScheduleProcessQueues");
+}
+
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+                                              RGXFWIF_DM          eKCCBType,
+                                              RGXFWIF_KCCB_CMD    *psKCCBCmd,
+                                              IMG_UINT32          ui32PDumpFlags,
+                                              IMG_UINT32          *pui32CmdKCCBSlot,
+                                              IMG_BOOL            bCallerHasPwrLock)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiMMUSyncUpdate;
+
+	/* Don't send the command/power up request if device not available. */
+	if (unlikely((psDevInfo == NULL) ||
+	             (psDevInfo->psDeviceNode == NULL) ||
+				 (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)))
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	/* Don't send the command/power up request if device in deinit phase.
+	 * The de-init thread could destroy the device whilst the power up
+	 * sequence below is accessing the HW registers.
+	 * Not yet safe to free resources. Caller should retry later.
+	 */
+	if (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	    psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+
+
+	if (!bCallerHasPwrLock)
+	{
+		/* PVRSRVPowerLock guarantees atomicity between commands. This is helpful
+		   in a scenario with several applications allocating resources. */
+		eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+
+			/* If system is found powered OFF, Retry scheduling the command */
+			if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+			{
+				eError = PVRSRV_ERROR_RETRY;
+			}
+
+			goto RGXScheduleCommand_exit;
+		}
+	}
+
+	if (unlikely(psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	             psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING))
+	{
+		/* If we have the power lock the device is valid but the deinit
+		 * thread could be waiting for the lock. */
+		eError = PVRSRV_ERROR_RETRY;
+		goto _PVRSRVInvalidDeviceError_Exit;
+	}
+
+	/* Ensure device is powered up before sending any commands */
+	PDUMPPOWCMDSTART(psDevInfo->psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDevInfo->psDeviceNode,
+	                                     PVRSRV_DEV_POWER_STATE_ON,
+	                                     PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXPreKickCacheCommand(psDevInfo, eKCCBType, &uiMMUSyncUpdate);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+_PVRSRVInvalidDeviceError_Exit:
+	if (!bCallerHasPwrLock)
+	{
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+	}
+RGXScheduleCommand_exit:
+	return eError;
+}
+
+/*
+ * RGXCheckFirmwareCCB
+ */
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT8 *psFWCCB = psDevInfo->psFirmwareCCB;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo) ||
+								 (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+								  (KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))),
+								 "FW-KM connection is down");
+#endif
+
+	eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+		return;
+	}
+
+	while (psFWCCBCtlLocal->ui32ReadOffset != psFWCCBCtlLocal->ui32WriteOffset)
+	{
+		/* Point to the next command */
+		const RGXFWIF_FWCCB_CMD *psFwCCBCmd = ((RGXFWIF_FWCCB_CMD *)psFWCCB) + psFWCCBCtlLocal->ui32ReadOffset;
+		RGXFwSharedMemCacheOpPtr(psFwCCBCmd, INVALIDATE);
+
+
+		HTBLOGK(HTB_SF_MAIN_FWCCB_CMD, psFwCCBCmd->eCmdType);
+		switch (psFwCCBCmd->eCmdType)
+		{
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_BACKING, "Request to add backing to ZSBuffer");
+				}
+				RGXProcessRequestZSBufferBacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_UNBACKING, "Request to remove backing from ZSBuffer");
+				}
+				RGXProcessRequestZSBufferUnbacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELIST_GROW:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELIST_GROW, "Request to grow the free list");
+				}
+				RGXProcessRequestGrow(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELISTS_RECONSTRUCTION, "Request to reconstruct free lists");
+				}
+
+				if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d/%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psDevInfo->psRGXFWIfHWRInfoBufCtl->ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+
+				RGXProcessRequestFreelistsReconstruction(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.aui32FreelistIDs);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION:
+			{
+				/* Notify client drivers */
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+					const RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA *psCmdFwPagefault =
+							&psFwCCBCmd->uCmdData.sCmdFWPagefault;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+						sErrorData.uErrData.sFwPFErrData.sFWFaultAddr.uiAddr = psCmdFwPagefault->sFWFaultAddr.uiAddr;
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+														  &sErrorData);
+					}
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
+			{
+				const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification =
+						&psFwCCBCmd->uCmdData.sCmdContextResetNotification;
+				IMG_UINT32 ui32ErrorPid = 0;
+
+				FWCommonContextListSetLastResetReason(psDevInfo,
+				                                      &ui32ErrorPid,
+				                                      psCmdContextResetNotification);
+
+				/* Increment error counter (if appropriate) */
+				if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32WGPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32WGPErrorCount++;
+					}
+				}
+				else if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32TRPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32TRPErrorCount++;
+					}
+				}
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = psCmdContextResetNotification->eResetReason;
+						sErrorData.pid = ui32ErrorPid;
+
+						/* Populate error data according to reset reason */
+						switch (psCmdContextResetNotification->eResetReason)
+						{
+							case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+							case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+							{
+								sErrorData.uErrData.sChecksumErrData.ui32ExtJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+								sErrorData.uErrData.sChecksumErrData.eDM = psCmdContextResetNotification->eDM;
+								break;
+							}
+							default:
+							{
+								break;
+							}
+						}
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+						                                  &sErrorData);
+					}
+				}
+
+				/* Notify if a page fault */
+				if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF)
+				{
+					DevmemIntPFNotify(psDevInfo->psDeviceNode,
+							psCmdContextResetNotification->ui64PCAddress,
+							psCmdContextResetNotification->sFaultAddress);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_DEBUG_DUMP:
+			{
+				PVRSRV_ERROR eError;
+				PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+				OSAtomicWrite(&psDevInfo->psDeviceNode->eDebugDumpRequested, PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE);
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal FW Cmd debug dump event, dumping now instead", __func__));
+					PVRSRVDebugRequest(psDevInfo->psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_UPDATE_STATS:
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				IMG_PID pidTmp = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.pidOwner;
+				IMG_INT32 i32AdjustmentValue = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.i32AdjustmentValue;
+
+				switch (psFwCCBCmd->uCmdData.sCmdUpdateStatsData.eElementToUpdate)
+				{
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,i32AdjustmentValue,0,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,i32AdjustmentValue,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,i32AdjustmentValue,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,i32AdjustmentValue,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,i32AdjustmentValue,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,i32AdjustmentValue,0,pidTmp);
+						break;
+					}
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_RAY_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,0,i32AdjustmentValue,pidTmp);
+						break;
+					}
+#endif
+				}
+#endif
+				break;
+			}
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+			case RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE:
+			{
+				RGX_PROCESS_CORE_CLK_RATE_CHANGE(psDevInfo,
+											  psFwCCBCmd->uCmdData.sCmdCoreClkRateChange.ui32CoreClkRate);
+				break;
+			}
+#endif
+			case RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART:
+			{
+				RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+				if (psDevInfo->psRGXFWIfFwSysData != NULL  &&
+				    psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)
+				{
+					PVRSRV_ERROR eError;
+
+					/* Power down... */
+					eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+															 PVRSRV_SYS_POWER_STATE_OFF,
+															 PVRSRV_POWER_FLAGS_NONE);
+					if (eError == PVRSRV_OK)
+					{
+						/* Clear the FW faulted flags... */
+						psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags &= ~(RGXFWIF_HWR_FW_FAULT|RGXFWIF_HWR_RESTART_REQUESTED);
+						OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags);
+						RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags,
+						                           FLUSH);
+
+						/* Power back up again... */
+						eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+																 PVRSRV_SYS_POWER_STATE_ON,
+																 PVRSRV_POWER_FLAGS_NONE);
+
+						/* Send a dummy KCCB command to ensure the FW wakes up and checks the queues... */
+						if (eError == PVRSRV_OK)
+						{
+							LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+							{
+								eError = RGXFWHealthCheckCmd(psDevInfo);
+								if (eError != PVRSRV_ERROR_RETRY)
+								{
+									break;
+								}
+								OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+							} END_LOOP_UNTIL_TIMEOUT_US();
+						}
+					}
+
+					/* Notify client drivers and system layer of FW fault */
+					{
+						PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+						PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+						/* Client notification of device error will be achieved by
+						 * clients calling UM function RGXGetLastDeviceError() */
+						psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+
+						/* Notify system layer */
+						if (psDevConfig->pfnSysDevErrorNotify)
+						{
+							PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+							sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+							psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+							                                  &sErrorData);
+						}
+					}
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Failed firmware restart (%s)",
+								 __func__, PVRSRVGetErrorString(eError)));
+					}
+				}
+				break;
+			}
+			default:
+			{
+				/* unknown command */
+				PVR_DPF((PVR_DBG_WARNING, "%s: Unknown Command (eCmdType=0x%08x)",
+				         __func__, psFwCCBCmd->eCmdType));
+				/* Assert on magic value corruption */
+				PVR_ASSERT((((IMG_UINT32)psFwCCBCmd->eCmdType & RGX_CMD_MAGIC_DWORD_MASK) >> RGX_CMD_MAGIC_DWORD_SHIFT) == RGX_CMD_MAGIC_DWORD);
+			}
+		}
+
+		/* Update read offset */
+		psFWCCBCtlLocal->ui32ReadOffset = (psFWCCBCtlLocal->ui32ReadOffset + 1) & psFWCCBCtlLocal->ui32WrapMask;
+		OSMemoryBarrier(NULL);
+		psFWCCBCtl->ui32ReadOffset = psFWCCBCtlLocal->ui32ReadOffset;
+		OSWriteMemoryBarrier(NULL);
+
+		if (psFWCCBCtlLocal->ui32ReadOffset == psFWCCBCtlLocal->ui32WriteOffset)
+		{
+			eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+				return;
+			}
+		}
+	}
+}
+
+/*
+ * PVRSRVRGXFrameworkCopyCommand
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+		DEVMEM_MEMDESC	*psFWFrameworkMemDesc,
+		IMG_PBYTE		pbyGPUFRegisterList,
+		IMG_UINT32		ui32FrameworkRegisterSize)
+{
+	PVRSRV_ERROR	eError;
+	RGXFWIF_RF_REGISTERS	*psRFReg;
+
+	eError = DevmemAcquireCpuVirtAddr(psFWFrameworkMemDesc,
+			(void **)&psRFReg);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	OSDeviceMemCopy(psRFReg, pbyGPUFRegisterList, ui32FrameworkRegisterSize);
+	RGXFwSharedMemCacheOpPtr(psRFReg, FLUSH);
+
+	/* Release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psFWFrameworkMemDesc);
+
+	/*
+	 * Dump the FW framework buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FWFramework buffer");
+	DevmemPDumpLoadMem(psFWFrameworkMemDesc, 0, ui32FrameworkRegisterSize, PDUMP_FLAGS_CONTINUOUS);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXFrameworkCreateKM
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE	*psDeviceNode,
+		DEVMEM_MEMDESC		**ppsFWFrameworkMemDesc,
+		IMG_UINT32			ui32FrameworkCommandSize)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+		Allocate device memory for the firmware GPU framework state.
+		Sufficient info to kick one or more DMs should be contained in this buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware framework state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+			ui32FrameworkCommandSize,
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwGPUFrameworkState",
+			ppsFWFrameworkMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware framework state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem *pui32LinMemAddr,
+												IMG_UINT32			ui32Value,
+												IMG_UINT32			ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32CurrentQueueLength, ui32MaxRetries;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDevNode->pvDevice;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+	ui32CurrentQueueLength = (psKCCBCtlLocal->ui32WrapMask+1 +
+					psKCCBCtlLocal->ui32WriteOffset -
+					psKCCBCtlLocal->ui32ReadOffset) & psKCCBCtlLocal->ui32WrapMask;
+	ui32CurrentQueueLength += psDevInfo->ui32KCCBDeferredCommandsCount;
+
+	for (ui32MaxRetries = ui32CurrentQueueLength + 1;
+				ui32MaxRetries > 0;
+				ui32MaxRetries--)
+	{
+
+		/*
+		 * PVRSRVPollForValueKM flags are set to POLL_FLAG_NONE in this case so that the function
+		 * does not generate an error message. In this case, the PollForValueKM is expected to
+		 * timeout as there is work ongoing on the GPU which may take longer than the timeout period.
+		 */
+		eError = PVRSRVPollForValueKM(psDevNode, pui32LinMemAddr, ui32Value, ui32Mask, POLL_FLAG_NONE, NULL);
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			break;
+		}
+
+		RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		         __func__, PVRSRVGetErrorString(eError),
+		         pui32LinMemAddr, ui32Value));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32ConfigState,
+				IMG_BOOL bSetNotClear)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	RGXFWIF_KCCB_CMD sStateFlagCmd = { 0 };
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_SYSDATA *psFwSysData;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (NULL == psFwSysData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Fw Sys Config is not mapped into CPU space", __func__));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* apply change and ensure the new data is written to memory
+	 * before requesting the FW to read it
+	 */
+	ui32Config = ui32Config & RGXFWIF_INICFG_ALL;
+	if (bSetNotClear)
+	{
+		psFwSysData->ui32ConfigFlags |= ui32Config;
+	}
+	else
+	{
+		psFwSysData->ui32ConfigFlags &= ~ui32Config;
+	}
+	OSWriteMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+	RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags, FLUSH);
+
+	/* return current/new value to caller */
+	if (pui32ConfigState)
+	{
+		*pui32ConfigState = psFwSysData->ui32ConfigFlags;
+	}
+
+	OSMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	/* notify FW to update setting */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of the value */
+		sStateFlagCmd.eCmdType = RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sStateFlagCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock(psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the value to be updated as the FW validates
+		 * the parameters and modifies the ui32ConfigFlags
+		 * accordingly
+		 * (for completeness as registered callbacks should also
+		 *  not permit invalid transitions)
+		 */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXScheduleCleanupCommand(PVRSRV_RGXDEV_INFO	*psDevInfo,
+									   RGXFWIF_DM			eDM,
+									   RGXFWIF_KCCB_CMD		*psKCCBCmd,
+									   RGXFWIF_CLEANUP_TYPE	eCleanupType,
+									   IMG_UINT32			ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32kCCBCommandSlot;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus);
+
+	PVR_LOG_RETURN_IF_FALSE((eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+							(eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT),
+							"Cleanup aborted: Device in bad state", PVRSRV_OK);
+#endif
+
+
+	/* Clean-up commands sent during frame capture intervals must be dumped even when not in capture range... */
+	ui32PDumpFlags |= PDUMP_FLAGS_INTERVAL;
+
+	psKCCBCmd->eCmdType = RGXFWIF_KCCB_CMD_CLEANUP;
+	psKCCBCmd->uCmdData.sCleanupData.eCleanupType = eCleanupType;
+
+	/*
+		Send the cleanup request to the firmware. If the resource is still busy
+		the firmware will tell us and we'll drop out with a retry.
+	*/
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+											  eDM,
+											  psKCCBCmd,
+											  ui32PDumpFlags,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR ,"RGXScheduleCommandAndGetKCCBSlot() failed (%s) in %s()",
+			         PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		goto fail_command;
+	}
+
+	/* Wait for command kCCB slot to be updated by FW */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+						  "Wait for the firmware to reply to the cleanup command");
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot,
+									  ui32PDumpFlags);
+	/*
+		If the firmware hasn't got back to us in a timely manner
+		then bail and let the caller retry the command.
+	 */
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: RGXWaitForKCCBSlotUpdate timed out. Dump debug information.",
+		         __func__));
+
+		eError = PVRSRV_ERROR_RETRY;
+#if defined(DEBUG)
+		PVRSRVDebugRequest(psDevInfo->psDeviceNode,
+				DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+#endif
+		goto fail_poll;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		goto fail_poll;
+	}
+
+#if defined(PDUMP)
+	/*
+	 * The cleanup request to the firmware will tell us if a given resource is busy or not.
+	 * If the RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY flag is set, this means that the resource is
+	 * still in use. In this case we return a PVRSRV_ERROR_RETRY error to the client drivers
+	 * and they will re-issue the cleanup request until it succeed.
+	 *
+	 * Since this retry mechanism doesn't work for pdumps, client drivers should ensure
+	 * that cleanup requests are only submitted if the resource is unused.
+	 * If this is not the case, the following poll will block infinitely, making sure
+	 * the issue doesn't go unnoticed.
+	 */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+					"Cleanup: If this poll fails, the following resource is still in use (DM=%u, type=%u, address=0x%08x), which is incorrect in pdumps",
+					eDM,
+					psKCCBCmd->uCmdData.sCleanupData.eCleanupType,
+					psKCCBCmd->uCmdData.sCleanupData.uCleanupData.psContext.ui32Addr);
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+									ui32kCCBCommandSlot * sizeof(IMG_UINT32),
+									0,
+									RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+#endif
+
+	/*
+		If the command has was run but a resource was busy, then the request
+		will need to be retried.
+	*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot],
+	                           INVALIDATE);
+
+	if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY))
+	{
+		if (psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+		eError = PVRSRV_ERROR_RETRY;
+		goto fail_requestbusy;
+	}
+
+	return PVRSRV_OK;
+
+fail_requestbusy:
+fail_poll:
+fail_command:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	RGXRequestCommonContextCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD			sRCCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+	PRGXFWIF_FWCOMMONCONTEXT	psFWCommonContextFWAddr;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	/* Force retry if this context's CCB is currently being dumped
+	 * as part of the stalled CCB debug */
+	if (psDevInfo->pvEarliestStalledClientCCB == (void*)FWCommonContextGetClientCCB(psServerCommonContext))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: [%u/%d]: Forcing retry as psDevInfo->pvEarliestStalledClientCCB = psCtxClientCCB <%p>",
+		         __func__,
+		         psDeviceNode->sDevId.ui32InternalID,
+		         psDeviceNode->sDevId.i32KernelDeviceID,
+		         psDevInfo->pvEarliestStalledClientCCB));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psFWCommonContextFWAddr = FWCommonContextGetFWAddress(psServerCommonContext);
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Common ctx cleanup Request DM%d [context = 0x%08x]",
+			eDM, psFWCommonContextFWAddr.ui32Addr);
+	PDUMPCOMMENT(psDeviceNode, "Wait for CCB to be empty before common ctx cleanup");
+
+	RGXCCBPDumpDrainCCB(FWCommonContextGetClientCCB(psServerCommonContext), ui32PDumpFlags);
+#endif
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sRCCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psContext = psFWCommonContextFWAddr;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+									   eDM,
+									   &sRCCleanUpCmd,
+									   RGXFWIF_CLEANUP_FWCOMMONCONTEXT,
+									   ui32PDumpFlags);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+/*
+ * RGXFWRequestHWRTDataCleanUp
+ */
+
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         PRGXFWIF_HWRTDATA psHWRTData)
+{
+	RGXFWIF_KCCB_CMD			sHWRTDataCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDeviceNode, "HW RTData cleanup Request [HWRTData = 0x%08x]", psHWRTData.ui32Addr);
+
+	sHWRTDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWRTData = psHWRTData;
+
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+	                                   RGXFWIF_DM_GP,
+	                                   &sHWRTDataCleanUpCmd,
+	                                   RGXFWIF_CLEANUP_HWRTDATA,
+	                                   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a HWRTData cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestFreeListCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_FREELIST psFWFreeList)
+{
+	RGXFWIF_KCCB_CMD			sFLCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free list cleanup Request [FreeList = 0x%08x]", psFWFreeList.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psFreelist = psFWFreeList;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_GP,
+									   &sFLCleanUpCmd,
+									   RGXFWIF_CLEANUP_FREELIST,
+									   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a memory context cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestZSBufferCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer)
+{
+	RGXFWIF_KCCB_CMD			sZSBufferCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "ZS Buffer cleanup Request [ZS Buffer = 0x%08x]", psFWZSBuffer.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sZSBufferCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psZSBuffer = psFWZSBuffer;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_3D,
+									   &sZSBufferCleanUpCmd,
+									   RGXFWIF_CLEANUP_ZSBUFFER,
+									   PDUMP_FLAGS_NONE);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS = ui32HCSDeadlineMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Hard Context Switching deadline inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32HCSDeadlineMS),
+							  ui32HCSDeadlineMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock)
+{
+	RGXFWIF_KCCB_CMD	sCmpKCCBCmd = { 0 };
+
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+	if (bCallerHasPwrLock)
+	{
+		return	RGXScheduleCommandWithoutPowerLock(psDevInfo,
+												   RGXFWIF_DM_GP,
+												   &sCmpKCCBCmd,
+												   PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		return	RGXScheduleCommand(psDevInfo,
+								   RGXFWIF_DM_GP,
+								   &sCmpKCCBCmd,
+								   PDUMP_FLAGS_CONTINUOUS);
+	}
+}
+
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32CBaseMapCtxReg;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+	else
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+#else
+	/*
+	 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+	 * FW crash from its memory context
+	 */
+	RGXWriteKernelMMUPC64(&psDevInfo->sLayerParams,
+				                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+				                      ((0xDEADBEEF
+				                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+				                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+				                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               IMG_UINT32 ui32DriverID,
+                               RGXFWIF_OS_STATE_CHANGE eOSOnlineState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sOSOnlineStateCmd = { 0 };
+	const RGXFWIF_SYSDATA *psFwSysData;
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	sOSOnlineStateCmd.eCmdType = RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.ui32DriverID = ui32DriverID;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = eOSOnlineState;
+
+#if defined(SUPPORT_AUTOVZ)
+	{
+		IMG_BOOL bConnectionDown = IMG_FALSE;
+
+		PVR_UNREFERENCED_PARAMETER(psFwSysData);
+		sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = RGXFWIF_OS_OFFLINE;
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			/* Send the offline command regardless if power lock is held or not.
+			 * Under AutoVz this is done during regular driver deinit, store-to-ram suspend
+			 * or (optionally) from a kernel panic callback. Deinit and suspend operations
+			 * take the lock in the rgx pre/post power functions as expected.
+			 * The kernel panic callback is a last resort way of letting the firmware know that
+			 * the VM is unrecoverable and the vz connection must be disabled. It cannot wait
+			 * on other kernel threads to finish and release the lock. */
+			eError = RGXSendCommand(psDevInfo,
+									&sOSOnlineStateCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Guests and Host going offline should wait for confirmation
+		 * from the Firmware of the state change. If this fails, break
+		 * the connection on the OS Driver's end as backup. */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (ui32DriverID == RGXFW_HOST_DRIVER_ID))
+		{
+			LOOP_UNTIL_TIMEOUT_US(SECONDS_TO_MICROSECONDS/2)
+			{
+				KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+				if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+				{
+					bConnectionDown = IMG_TRUE;
+					break;
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (!bConnectionDown)
+			{
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+		}
+	}
+#else
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* no reason for Guests to update their state or any other VM's.
+		 * This is the Hypervisor and Host driver's responsibility. */
+		return PVRSRV_OK;
+	}
+	else
+	{
+		const volatile RGXFWIF_OS_RUNTIME_FLAGS *psFwRunFlags;
+
+		PVR_ASSERT(psFwSysData != NULL);
+
+		psFwRunFlags = (const volatile RGXFWIF_OS_RUNTIME_FLAGS*) &psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+		/* Attempt several times until the FW manages to offload the OS */
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			IMG_UINT32 ui32kCCBCommandSlot;
+
+			/* Send request */
+			eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+													  RGXFWIF_DM_GP,
+													  &sOSOnlineStateCmd,
+													  PDUMP_FLAGS_CONTINUOUS,
+													  &ui32kCCBCommandSlot);
+			if (unlikely(eError == PVRSRV_ERROR_RETRY)) continue;
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommand", return_);
+
+			/* Wait for FW to process the cmd */
+			eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", return_);
+
+			/* read the OS state */
+			OSMemoryBarrier(NULL);
+			/* check if FW finished offloading the driver and is stopped */
+			if ((eOSOnlineState == RGXFWIF_OS_ONLINE && (psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_READY || psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_ACTIVE)) ||
+			    (eOSOnlineState == RGXFWIF_OS_OFFLINE && psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_OFFLINE))
+			{
+				eError = PVRSRV_OK;
+				break;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_TIMEOUT;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+
+return_ :
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgPHRCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE((ui32PHRMode == RGXFWIF_PHR_MODE_OFF) ||
+							(ui32PHRMode == RGXFWIF_PHR_MODE_RD_RESET),
+							"Invalid PHR Mode.", PVRSRV_ERROR_INVALID_PARAMS);
+
+	sCfgPHRCmd.eCmdType = RGXFWIF_KCCB_CMD_PHR_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode = ui32PHRMode;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Periodic Hardware Reset Mode inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32PHRMode),
+							  ui32PHRMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GP,
+		                            &sCfgPHRCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+							IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgWdgCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCfgWdgCmd.eCmdType = RGXFWIF_KCCB_CMD_WDG_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs = ui32WdgPeriodUs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the firmware watchdog period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32WdgPeriodUs),
+							  ui32WdgPeriodUs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sCfgWdgCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious)
+{
+	/* Attempt to detect and deal with any stalled client contexts.
+	 * bIgnorePrevious may be set by the caller if they know a context to be
+	 * stalled, as otherwise this function will only identify stalled
+	 * contexts which have not been previously reported.
+	 */
+
+	IMG_UINT32 ui32StalledClientMask = 0;
+
+	if (!(OSTryLockAcquire(psDevInfo->hCCBStallCheckLock)))
+	{
+		PVR_LOG(("RGXCheckForStalledClientContexts: Failed to acquire hCCBStallCheckLock, returning..."));
+		return;
+	}
+
+	ui32StalledClientMask |= CheckForStalledClientTransferCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientRenderCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientComputeCtxt(psDevInfo);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	ui32StalledClientMask |= CheckForStalledClientKickSyncCtxt(psDevInfo);
+#endif
+	/* If at least one DM stalled bit is different than before */
+	if (bIgnorePrevious || (psDevInfo->ui32StalledClientMask != ui32StalledClientMask))
+	{
+		if (ui32StalledClientMask > 0)
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Print all the stalled DMs */
+			PVR_LOG(("Possible stalled client RGX contexts detected: %s%s%s%s%s%s%s",
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_GP),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TDM_2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TA),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_3D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_CDM),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ3D)));
+
+			PVR_LOG(("Trying to identify stalled context...(%s) [%d]",
+			         pszStalledAction, bIgnorePrevious));
+
+			DumpStalledContextInfo(psDevInfo);
+		}
+		else
+		{
+			if (psDevInfo->ui32StalledClientMask> 0)
+			{
+				/* Indicate there are no stalled DMs */
+				PVR_LOG(("No further stalled client contexts exist"));
+			}
+		}
+		psDevInfo->ui32StalledClientMask = ui32StalledClientMask;
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+	OSLockRelease(psDevInfo->hCCBStallCheckLock);
+}
+
+/*
+	RGXUpdateHealthStatus
+*/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed)
+{
+	const PVRSRV_DATA*           psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_HEALTH_STATUS  eNewStatus   = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	PVRSRV_DEVICE_HEALTH_REASON  eNewReason   = PVRSRV_DEVICE_HEALTH_REASON_NONE;
+	PVRSRV_RGXDEV_INFO*          psDevInfo;
+	const RGXFWIF_TRACEBUF*      psRGXFWIfTraceBufCtl;
+	const RGXFWIF_SYSDATA*       psFwSysData;
+	const RGXFWIF_OSDATA*        psFwOsData;
+	const RGXFWIF_CCB_CTL*       psKCCBCtl;
+	RGXFWIF_CCB_CTL*             psKCCBCtlLocal;
+	IMG_UINT32                   ui32ThreadCount;
+	IMG_BOOL                     bKCCBCmdsWaiting;
+	PVRSRV_ERROR                 eError;
+
+	PVR_ASSERT(psDevNode != NULL);
+	psDevInfo = psDevNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, PVRSRV_OK);
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_OK;
+	}
+
+	psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData,
+								 INVALIDATE);
+		psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         INVALIDATE);
+	psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+
+	/* If this is a quick update, then include the last current value... */
+	if (!bCheckAfterTimePassed)
+	{
+		eNewStatus = OSAtomicRead(&psDevNode->eHealthStatus);
+		eNewReason = OSAtomicRead(&psDevNode->eHealthReason);
+	}
+
+	/* Decrement the SLR holdoff counter (if non-zero) */
+	if (psDevInfo->ui32SLRHoldoffCounter > 0)
+	{
+		psDevInfo->ui32SLRHoldoffCounter--;
+	}
+
+	/* Take power lock, retry if it's in use in another task. */
+	eError = PVRSRVPowerTryLockWaitForTimeout(psDevNode);
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		/* Skip health status update if timeout */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Power lock timeout, increase OS_POWERLOCK_TIMEOUT_US.", __func__));
+		goto _RGXUpdateHealthStatus_Exit;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerTryLockWaitForTimeout");
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+		return PVRSRV_OK;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		/* On a PCI error all reads from the PCI bar may return 0xFFFFFFFF.
+		   This value is not valid for a core ID. */
+		if (psFwSysData->ui32MemFaultCheck == RGX_PCI_ERROR_VALUE_DWORD)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: PCI error", __func__));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR;
+			PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_PCI_ERROR);
+			PVRSRVPowerUnlock(psDevNode);
+			goto _RGXUpdateHealthStatus_Exit;
+		}
+	}
+
+	/* If Rogue is not powered on, just skip ahead and check for stalled client CCBs */
+	if (PVRSRVIsDevicePowered(psDevNode))
+	{
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			/*
+			   Firmware thread checks...
+			 */
+			for (ui32ThreadCount = 0; ui32ThreadCount < RGXFW_THREAD_NUM; ui32ThreadCount++)
+			{
+				const IMG_CHAR* pszTraceAssertInfo;
+
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf, INVALIDATE);
+				pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szInfo;
+
+				/*
+				Check if the FW has hit an assert...
+				*/
+				if (*pszTraceAssertInfo != '\0')
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: Firmware thread %d has asserted: %.*s (%.*s:%d)",
+							__func__, ui32ThreadCount, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							pszTraceAssertInfo, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szPath,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.ui32LineNum));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_ASSERTED;
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+				{
+					/*
+					   Check the threads to see if they are in the same poll locations as last time...
+					*/
+					if (bCheckAfterTimePassed)
+					{
+						if (psFwSysData->aui32CrPollAddr[ui32ThreadCount] != 0  &&
+							psFwSysData->aui32CrPollCount[ui32ThreadCount] == psDevInfo->aui32CrLastPollCount[ui32ThreadCount])
+						{
+							PVR_DPF((PVR_DBG_WARNING, "%s: Firmware stuck on CR poll: T%u polling %s (reg:0x%08X mask:0x%08X)",
+									__func__, ui32ThreadCount,
+									((psFwSysData->aui32CrPollAddr[ui32ThreadCount] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+									psFwSysData->aui32CrPollAddr[ui32ThreadCount] & ~RGXFW_POLL_TYPE_SET,
+									psFwSysData->aui32CrPollMask[ui32ThreadCount]));
+							eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+							eNewReason = PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING;
+							PVRSRVPowerUnlock(psDevNode);
+							goto _RGXUpdateHealthStatus_Exit;
+						}
+						psDevInfo->aui32CrLastPollCount[ui32ThreadCount] = psFwSysData->aui32CrPollCount[ui32ThreadCount];
+					}
+				}
+			}
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+			{
+				/*
+				Check if the FW has faulted...
+				*/
+				if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_FW_FAULT)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+							"%s: Firmware has faulted and needs to restart",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_FAULT;
+					if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_RESTART_REQUESTED)
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_RESTARTING;
+					}
+					else
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_IDLING;
+					}
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+			}
+		}
+
+		/*
+		   Event Object Timeouts check...
+		*/
+		if (!bCheckAfterTimePassed)
+		{
+			if (psDevInfo->ui32GEOTimeoutsLastTime > 1 && psPVRSRVData->ui32GEOConsecutiveTimeouts > psDevInfo->ui32GEOTimeoutsLastTime)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Global Event Object Timeouts have risen (from %d to %d)",
+				         __func__,
+				         psDevInfo->ui32GEOTimeoutsLastTime, psPVRSRVData->ui32GEOConsecutiveTimeouts));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS;
+			}
+			psDevInfo->ui32GEOTimeoutsLastTime = psPVRSRVData->ui32GEOConsecutiveTimeouts;
+		}
+
+		/*
+		   Check the Kernel CCB pointer is valid. If any commands were waiting last time, then check
+		   that some have executed since then.
+		*/
+		bKCCBCmdsWaiting = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+		psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+		if (psKCCBCtl != NULL && psKCCBCtlLocal != NULL)
+		{
+			/* update KCCB read offset */
+			RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+			psKCCBCtlLocal->ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+			if (psKCCBCtlLocal->ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask  ||
+				psKCCBCtlLocal->ui32WriteOffset > psKCCBCtlLocal->ui32WrapMask)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: KCCB has invalid offset (ROFF=%d WOFF=%d)",
+						__func__, psKCCBCtlLocal->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+			}
+
+			if (psKCCBCtlLocal->ui32ReadOffset != psKCCBCtlLocal->ui32WriteOffset)
+			{
+				bKCCBCmdsWaiting = IMG_TRUE;
+			}
+		}
+
+		if (bCheckAfterTimePassed && psFwOsData != NULL)
+		{
+			IMG_UINT32 ui32KCCBCmdsExecuted = psFwOsData->ui32KCCBCmdsExecuted;
+
+			if (psDevInfo->ui32KCCBCmdsExecutedLastTime == ui32KCCBCmdsExecuted)
+			{
+				/*
+				   If something was waiting last time then the Firmware has stopped processing commands.
+				*/
+				if (psDevInfo->bKCCBCmdsWaitingLastTime)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: No KCCB commands executed since check!",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+				}
+
+				/*
+				   If no commands are currently pending and nothing happened since the last poll, then
+				   schedule a dummy command to ping the firmware so we know it is alive and processing.
+				*/
+				if ((!bKCCBCmdsWaiting) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+				{
+					/* Protect the PDumpLoadMem. RGXScheduleCommand() cannot take the
+					 * PMR lock itself, because some bridge functions will take the PMR lock
+					 * before calling RGXScheduleCommand
+					 */
+					PVRSRV_ERROR eError = RGXFWHealthCheckCmdWithoutPowerLock(psDevNode->pvDevice);
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Cannot schedule Health Check command! (0x%x)",
+								__func__, eError));
+					}
+					else
+					{
+						bKCCBCmdsWaiting = IMG_TRUE;
+					}
+				}
+			}
+
+			psDevInfo->bKCCBCmdsWaitingLastTime     = bKCCBCmdsWaiting;
+			psDevInfo->ui32KCCBCmdsExecutedLastTime = ui32KCCBCmdsExecuted;
+		}
+	}
+
+	/*
+	   Interrupt counts check...
+	*/
+	if (bCheckAfterTimePassed  &&  psFwOsData != NULL)
+	{
+		IMG_UINT32  ui32LISRCount   = 0;
+		IMG_UINT32  ui32FWCount     = 0;
+		IMG_UINT32  ui32MissingInts = 0;
+
+		/* Add up the total number of interrupts issued, sampled/received and missed... */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* Only the Host OS has a sample count, so only one counter to check. */
+		ui32LISRCount += psDevInfo->aui32SampleIRQCount[RGXFW_HOST_DRIVER_ID];
+		ui32FWCount   += OSReadHWReg32(psDevInfo->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[RGXFW_HOST_DRIVER_ID]);
+#else
+		IMG_UINT32  ui32Index;
+
+		for (ui32Index = 0;  ui32Index < RGXFW_THREAD_NUM;  ui32Index++)
+		{
+			ui32LISRCount += psDevInfo->aui32SampleIRQCount[ui32Index];
+			ui32FWCount   += psFwOsData->aui32InterruptCount[ui32Index];
+		}
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+		if (ui32LISRCount < ui32FWCount)
+		{
+			ui32MissingInts = (ui32FWCount-ui32LISRCount);
+		}
+
+		if (ui32LISRCount == psDevInfo->ui32InterruptCountLastTime  &&
+		    ui32MissingInts >= psDevInfo->ui32MissingInterruptsLastTime  &&
+		    psDevInfo->ui32MissingInterruptsLastTime > 1)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: LISR has not received the last %d interrupts",
+					__func__, ui32MissingInts));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+
+			/* Schedule the MISRs to help mitigate the problems of missing interrupts. */
+			OSScheduleMISR(psDevInfo->pvMISRData);
+			if (psDevInfo->pvAPMISRData != NULL)
+			{
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+		psDevInfo->ui32InterruptCountLastTime    = ui32LISRCount;
+		psDevInfo->ui32MissingInterruptsLastTime = ui32MissingInts;
+	}
+
+	/* Release power lock before RGXCheckForStalledClientContexts */
+	PVRSRVPowerUnlock(psDevNode);
+
+	/*
+	   Stalled CCB check...
+	*/
+	if (bCheckAfterTimePassed && (PVRSRV_DEVICE_HEALTH_STATUS_OK==eNewStatus))
+	{
+		RGXCheckForStalledClientContexts(psDevInfo, IMG_FALSE);
+	}
+
+	/* Notify client driver and system layer of any eNewStatus errors */
+	if (eNewStatus > PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		/* Client notification of device error will be achieved by
+		 * clients calling UM function RGXGetLastDeviceError() */
+		psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+
+		/* Notify system layer */
+		{
+			PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+			PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+			if (psDevConfig->pfnSysDevErrorNotify)
+			{
+				PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+				sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+				sErrorData.uErrData.sHostWdgData.ui32Status = (IMG_UINT32)eNewStatus;
+				sErrorData.uErrData.sHostWdgData.ui32Reason = (IMG_UINT32)eNewReason;
+
+				psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+				                                  &sErrorData);
+			}
+		}
+	}
+
+	/*
+	   Finished, save the new status...
+	*/
+_RGXUpdateHealthStatus_Exit:
+	OSAtomicWrite(&psDevNode->eHealthStatus, eNewStatus);
+	OSAtomicWrite(&psDevNode->eHealthReason, eNewReason);
+	RGXSRV_HWPERF_DEVICE_INFO_HEALTH(psDevInfo, eNewStatus, eNewReason);
+
+	/*
+	 * Attempt to service the HWPerf buffer to regularly transport idle/periodic
+	 * packets to host buffer.
+	 */
+	if (psDevNode->pfnServiceHWPerf != NULL)
+	{
+		PVRSRV_ERROR eError = psDevNode->pfnServiceHWPerf(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: "
+			         "Error occurred when servicing HWPerf buffer (%d)",
+			         __func__, eError));
+		}
+	}
+
+	/* Attempt to refresh timer correlation data */
+	RGXTimeCorrRestartPeriodic(psDevNode);
+
+	return PVRSRV_OK;
+} /* RGXUpdateHealthStatus */
+
+#if defined(SUPPORT_AUTOVZ)
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+		(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))))
+	{
+		/* read and write back the alive token value to confirm to the
+		 * virtualisation watchdog that this connection is healthy */
+		KM_SET_OS_ALIVE_TOKEN(KM_GET_FW_ALIVE_TOKEN(psDevInfo), psDevInfo);
+		KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH);
+	}
+}
+
+/*
+	RGXUpdateAutoVzWatchdog
+*/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode)
+{
+	if (likely(psDevNode != NULL))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+
+		if (unlikely((psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || !psDevInfo->bRGXPowered ||
+					  psDevInfo->pvRegsBaseKM == NULL || psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+					  psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)))
+		{
+			/* If the firmware is not initialised, stop here */
+			return;
+		}
+		else
+		{
+			PVRSRV_ERROR eError = PVRSRVPowerLock(psDevNode);
+			PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PVRSRVPowerLock");
+
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+	}
+}
+
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DriverID;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+	                           INVALIDATE);
+
+	for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+	     ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED;
+	     ui32DriverID++)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+			psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+		if (eGuestState == RGXFW_CONNECTION_FW_ACTIVE ||
+		    eGuestState == RGXFW_CONNECTION_FW_READY)
+		{
+			PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+		}
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer)
+{
+	PVRSRV_ERROR eError;
+	PRGXFWIF_CLEANUP_CTL *psCleanupCtlWrite = apsCleanupCtl;
+
+	PVR_ASSERT((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+	PVR_RETURN_IF_INVALID_PARAM((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+
+	if (bKick)
+	{
+		if (psKMHWRTDataSet)
+		{
+			PRGXFWIF_CLEANUP_CTL psCleanupCtl;
+
+			eError = RGXSetFirmwareAddress(&psCleanupCtl, psKMHWRTDataSet->psHWRTDataFwMemDesc,
+					offsetof(RGXFWIF_HWRTDATA, sCleanupState),
+					RFW_FWADDR_NOREF_FLAG);
+			PVR_RETURN_IF_ERROR(eError);
+
+			*(psCleanupCtlWrite++) = psCleanupCtl;
+		}
+
+		if (eDM == RGXFWIF_DM_3D)
+		{
+			RGXFWIF_PRBUFFER_TYPE eBufferType;
+			RGX_ZSBUFFER_DATA *psBuffer = NULL;
+
+			for (eBufferType = RGXFWIF_PRBUFFER_START; eBufferType < RGXFWIF_PRBUFFER_MAXSUPPORTED; eBufferType++)
+			{
+				switch (eBufferType)
+				{
+				case RGXFWIF_PRBUFFER_ZSBUFFER:
+					psBuffer = psZSBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MSAABUFFER:
+					psBuffer = psMSAAScratchBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MAXSUPPORTED:
+					psBuffer = NULL;
+					break;
+				}
+				if (psBuffer)
+				{
+					(psCleanupCtlWrite++)->ui32Addr = psBuffer->sZSBufferFWDevVAddr.ui32Addr +
+									offsetof(RGXFWIF_PRBUFFER, sCleanupState);
+					psBuffer = NULL;
+				}
+			}
+		}
+	}
+
+	*pui32NumCleanupCtl = psCleanupCtlWrite - apsCleanupCtl;
+	PVR_ASSERT(*pui32NumCleanupCtl <= RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO       *psDevInfo;
+	RGXFWIF_HWRINFOBUF       *psHWRInfoBuf;
+	IMG_UINT32               i;
+
+	if (psDevNode->pvDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVINFO;
+	}
+	psDevInfo = psDevNode->pvDevice;
+
+	psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+	for (i = 0 ; i < RGXFWIF_DM_MAX ; i++)
+	{
+		/* Reset the HWR numbers */
+		psHWRInfoBuf->aui32HwrDmLockedUpCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmFalseDetectCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmRecoveredCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmOverranCount[i] = 0;
+	}
+
+	for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+	{
+		psHWRInfoBuf->sHWRInfo[i].ui32HWRNumber = 0;
+	}
+
+	psHWRInfoBuf->ui32WriteIndex = 0;
+	psHWRInfoBuf->ui32DDReqCount = 0;
+
+	OSWriteMemoryBarrier(&psHWRInfoBuf->ui32DDReqCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid)
+{
+
+	PVRSRV_ERROR eError;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+								 ui32Log2PageSize,
+								 ui32NumOfPages,
+								 ui32LogicalOffset,
+								 psPhyAddr,
+								 bValid,
+								 DEVICE_USE);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR_DevPhysAddr failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRUnLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	return eError;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32WriteOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_OK);
+
+	if (psDevInfo->bDumpedKCCBCtlAlready)
+	{
+		/* exiting capture range or pdump block */
+		psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+		/* make sure previous cmd is drained in pdump in case we will 'jump' over some future cmds */
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+                              PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER,
+                              "kCCB(%p): Draining rgxfw_roff (0x%x) == woff (0x%x)",
+                              psDevInfo->psKernelCCBCtl,
+                              ui32WriteOffset,
+                              ui32WriteOffset);
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+				offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+				ui32WriteOffset,
+				0xffffffff,
+				PDUMP_POLL_OPERATOR_EQUAL,
+				PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: problem pdumping POL for kCCBCtl (%d)", __func__, eError));
+		}
+	}
+
+	return eError;
+
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXClientConnectCompatCheck_ClientAgainstFW
+
+ @Description
+
+ Check compatibility of client and firmware (build options)
+ at the connection time.
+
+ @Input psDeviceNode - device node
+ @Input ui32ClientBuildOptions - build options for the client
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+******************************************************************************/
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32ClientBuildOptions)
+{
+	IMG_UINT32		ui32BuildOptionsMismatch;
+	IMG_UINT32		ui32BuildOptionsFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_FW_INFO_HEADER *psFWInfoHeader;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psDevInfo == NULL || psDevInfo->psRGXFWIfOsInitMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Cannot acquire kernel fw compatibility check info, RGXFWIF_OSINIT structure not allocated.",
+		         __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *) &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+#endif
+
+	ui32BuildOptionsFW = psFWInfoHeader->ui32Flags;
+	ui32BuildOptionsMismatch = ui32ClientBuildOptions ^ ui32BuildOptionsFW;
+
+	if (ui32BuildOptionsMismatch != 0)
+	{
+		if ((ui32ClientBuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in client: (0x%x). Please check rgx_options.h",
+					 ui32ClientBuildOptions & ui32BuildOptionsMismatch ));
+		}
+
+		if ((ui32BuildOptionsFW & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFW & ui32BuildOptionsMismatch ));
+		}
+
+		return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware and client build options match. [ OK ]", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapAllocMap
+
+ @Description Register firmware heap for the specified driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID - Guest driver
+ @Input sDevPAddr    - Heap address
+ @Input ui64DevPSize - Heap size
+
+ @Return   PVRSRV_ERROR - PVRSRV_OK if heap setup was successful.
+
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR szRegionRAName[RA_MAX_NAME_LENGTH];
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_MEMALLOCFLAGS_T uiRawFwHeapAllocFlags = (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+													PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PREMAP0 + ui32DriverID));
+	PHYS_HEAP_CONFIG *psFwHeapConfig = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+	PHYS_HEAP_CONFIG sFwHeapConfig;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psFwHeapConfig == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW_MAIN heap config not found."));
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	OSSNPrintf(szRegionRAName, sizeof(szRegionRAName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	if (!ui64DevPSize ||
+		!sDevPAddr.uiAddr ||
+		ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED ||
+		ui64DevPSize != RGX_FIRMWARE_RAW_HEAP_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid parameters for %s", szRegionRAName));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sFwHeapConfig = *psFwHeapConfig;
+	sFwHeapConfig.eType = PHYS_HEAP_TYPE_LMA;
+	sFwHeapConfig.ui32UsageFlags = PHYS_HEAP_USAGE_FW_PREMAP;
+	sFwHeapConfig.uConfig.sLMA.sStartAddr.uiAddr = 0;
+	sFwHeapConfig.uConfig.sLMA.sCardBase.uiAddr = sDevPAddr.uiAddr;
+	sFwHeapConfig.uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+	eError = PhysmemCreateHeapLMA(psDeviceNode,
+	                              RGXPhysHeapGetLMAPolicy(sFwHeapConfig.ui32UsageFlags, psDeviceNode),
+	                              &sFwHeapConfig,
+	                              szRegionRAName,
+	                              &psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysmemCreateHeapLMA:PREMAP [%d]", ui32DriverID);
+
+	eError = PhysHeapAcquire(psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysHeapAcquire:PREMAP [%d]", ui32DriverID);
+
+	psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID] = psDeviceNode->apsFWPremapPhysHeap[ui32DriverID];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate and map raw firmware heap for DriverID: [%d]", ui32DriverID);
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* don't clear the heap of other guests on allocation */
+	uiRawFwHeapAllocFlags &= (ui32DriverID > RGXFW_HOST_DRIVER_ID) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL);
+#endif
+
+	/* if the firmware is already powered up, consider the firmware heaps are pre-mapped. */
+	if (psDeviceNode->bAutoVzFwIsUp)
+	{
+		uiRawFwHeapAllocFlags &= RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Allocation and mapping for Firmware heaps done by TEE.", __func__));
+#else
+	eError = DevmemFwAllocate(psDevInfo,
+							  RGX_FIRMWARE_RAW_HEAP_SIZE,
+							  uiRawFwHeapAllocFlags,
+							  psDevInfo->psPremappedFwRawHeap[ui32DriverID]->pszName,
+							  &psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+#endif
+
+	/* Mark this devmem heap as premapped so allocations will not require device mapping. */
+	DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+
+	if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+	{
+		/* if the Host's raw fw heap is premapped, mark its main & config sub-heaps accordingly
+		 * No memory allocated from these sub-heaps will be individually mapped into the device's
+		 * address space so they can remain marked permanently as premapped. */
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapUnmapFree
+
+ @Description Unregister firmware heap for the specified guest driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID
+
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* remove the premap status, so the heap can be unmapped and freed */
+	if (psDevInfo->psPremappedFwRawHeap[ui32DriverID])
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_FALSE);
+	}
+
+	if (psDevInfo->psPremappedFwRawMemDesc[ui32DriverID])
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+		psDevInfo->psPremappedFwRawMemDesc[ui32DriverID] = NULL;
+	}
+}
+
+/*
+	RGXReadMETAAddr
+*/
+static PVRSRV_ERROR RGXReadMETAAddr(PVRSRV_RGXDEV_INFO	*psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 *pui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem  *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem  *pui8RegBase = pvRegBase;
+	IMG_UINT32 ui32PollValue;
+	IMG_UINT32 ui32PollMask;
+	IMG_UINT32 ui32PollRegOffset;
+	IMG_UINT32 ui32ReadOffset;
+	IMG_UINT32 ui32WriteOffset;
+	IMG_UINT32 ui32WriteValue;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+	{
+		ui32PollValue = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollMask = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1;
+		ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0;
+		ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN;
+		ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX;
+	}
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteUncheckedHWReg32(pvRegBase, ui32WriteOffset, ui32WriteValue);
+	(void)OSReadUncheckedHWReg32(pvRegBase, ui32WriteOffset);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadUncheckedHWReg32(pvRegBase, ui32ReadOffset);
+#else
+	IMG_UINT8 __iomem  *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteHWReg32(
+			psDevInfo->pvRegsBaseKM,
+			RGX_CR_META_SP_MSLVCTRL0,
+			ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAX);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGXWriteMETAAddr
+*/
+static PVRSRV_ERROR RGXWriteMETAAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 ui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem *pui8RegBase = pvRegBase;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA, ui32Value);
+		}
+		else
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA, ui32Value);
+		}
+	}
+	else
+	{
+		/* Wait for Slave Port to be Ready */
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+				(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Issue the Write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0); /* Fence write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT); /* Fence write */
+	}
+#else
+	IMG_UINT8 __iomem *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Write */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 *pui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXReadMETAAddr(psDevInfo, ui32FWAddr, pui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvReadMem(psDevInfo, ui32FWAddr, pui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 ui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXWriteMETAAddr(psDevInfo, ui32FWAddr, ui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvWriteMem(psDevInfo, ui32FWAddr, ui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32FwVA,
+                             IMG_CPU_PHYADDR *psCpuPA,
+                             IMG_DEV_PHYADDR *psDevPA,
+                             IMG_UINT64 *pui64RawPTE)
+{
+	PVRSRV_ERROR eError       = PVRSRV_OK;
+	IMG_CPU_PHYADDR sCpuPA    = {0U};
+	IMG_DEV_PHYADDR sDevPA    = {0U};
+	IMG_UINT64 ui64RawPTE     = 0U;
+	MMU_FAULT_DATA sFaultData = {0U};
+	MMU_CONTEXT *psFwMMUCtx   = psDevInfo->psKernelMMUCtx;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (RGX_NUM_DRIVERS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+	IMG_UINT32 ui32DriverID   = (ui32FwVA - ui32FwHeapBase) / RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT32 ui32HeapId;
+	PHYS_HEAP *psPhysHeap;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* MIPS uses the same page size as the OS, while others default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) ?
+	                             OSGetPageSize() : BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#else
+	/* default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#endif
+
+	IMG_UINT32 ui32PageOffset = (ui32FwVA & (ui32FwPageSize - 1));
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM((ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((psCpuPA != NULL) ||
+	                               (psDevPA != NULL) ||
+	                               (pui64RawPTE != NULL)),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((ui32FwVA >= ui32FwHeapBase) &&
+	                              (ui32FwVA < ui32FwHeapEnd)),
+	                              eError, ErrorExit);
+
+	ui32HeapId = (ui32DriverID == RGXFW_HOST_DRIVER_ID) ?
+	              PVRSRV_PHYS_HEAP_FW_MAIN : (PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID);
+	psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[ui32HeapId];
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS is equipped with a dedicated MMU */
+		RGXMipsCheckFaultAddress(psFwMMUCtx, ui32FwVA, &sFaultData);
+	}
+	else
+#endif
+	{
+		IMG_UINT64 ui64FwDataBaseMask;
+		IMG_DEV_VIRTADDR sDevVAddr;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			ui64FwDataBaseMask = ~(RGXFW_SEGMMU_DATA_META_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_BASE_ADDRESS);
+		}
+#if !defined(EMULATOR)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			ui64FwDataBaseMask = ~RGXRISCVFW_REGION_MASK;
+		}
+#endif
+		else
+		{
+			PVR_LOG_GOTO_WITH_ERROR("RGXGetFwMapping", eError, PVRSRV_ERROR_NOT_IMPLEMENTED, ErrorExit);
+		}
+
+		sDevVAddr.uiAddr = (ui32FwVA & ui64FwDataBaseMask) | RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Fw CPU shares a subset of the GPU's VA space */
+		MMU_CheckFaultAddress(psFwMMUCtx, &sDevVAddr, &sFaultData);
+	}
+
+	ui64RawPTE = sFaultData.sLevelData[MMU_LEVEL_1].ui64Address;
+
+	if (eError == PVRSRV_OK)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		IMG_BOOL bValidPage = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+		                       BITMASK_HAS(ui64RawPTE, RGXMIPSFW_TLB_VALID) :
+		                       BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#else
+		IMG_BOOL bValidPage = BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#endif
+
+		if (!bValidPage)
+		{
+			/* don't report invalid pages */
+			eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+			sDevPA.uiAddr = ui32PageOffset + ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			                RGXMIPSFW_TLB_GET_PA(ui64RawPTE) :
+			                (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK));
+#else
+			sDevPA.uiAddr = ui32PageOffset + (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK);
+#endif
+
+			/* Only the Host's Firmware heap is present in the Host's CPU IPA space */
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPA, &sDevPA);
+			}
+			else
+			{
+				sCpuPA.uiAddr = 0U;
+			}
+		}
+	}
+
+	if (psCpuPA != NULL)
+	{
+		*psCpuPA = sCpuPA;
+	}
+
+	if (psDevPA != NULL)
+	{
+		*psDevPA = sDevPA;
+	}
+
+	if (pui64RawPTE != NULL)
+	{
+		*pui64RawPTE = ui64RawPTE;
+	}
+
+ErrorExit:
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = ui32VzConnectionCooldownPeriodInSec;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Vz reconnect request cooldown period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32VzConnectionCooldownPeriodInSec),
+							  ui32VzConnectionCooldownPeriodInSec,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxfwutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxhwperf.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxhwperf.c
new file mode 100644
index 000000000000..e43fc499d9db
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxhwperf.c
@@ -0,0 +1,1030 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+
+#include "info_page_defs.h"
+
+/* This is defined by default to enable producer callbacks.
+ * Clients of the TL interface can disable the use of the callback
+ * with PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK. */
+#define SUPPORT_TL_PRODUCER_CALLBACK 1
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+/* Defines size of buffers returned from acquire/release calls */
+#define FW_STREAM_BUFFER_SIZE (0x80000)
+#define HOST_STREAM_BUFFER_SIZE (0x20000)
+
+/* Must be at least as large as two tl packets of maximum size */
+static_assert(HOST_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "HOST_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+static_assert(FW_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "FW_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+
+/******************************************************************************
+ * RGX HW Performance Profiling Server API(s)
+ *****************************************************************************/
+
+static IMG_BOOL RGXServerFeatureFlagsToHWPerfFlagsAddBlock(
+	RGX_HWPERF_BVNC_BLOCK	* const psBlocks,
+	IMG_UINT16				* const pui16Count,
+	const IMG_UINT16		ui16BlockID, /* see RGX_HWPERF_CNTBLK_ID */
+	const IMG_UINT16		ui16NumCounters,
+	const IMG_UINT16		ui16NumBlocks)
+{
+	const IMG_UINT16 ui16Count = *pui16Count;
+
+	if (ui16Count < RGX_HWPERF_MAX_BVNC_BLOCK_LEN)
+	{
+		RGX_HWPERF_BVNC_BLOCK * const psBlock = &psBlocks[ui16Count];
+
+		/* If the GROUP is non-zero, convert from e.g. RGX_CNTBLK_ID_USC0 to RGX_CNTBLK_ID_USC_ALL. The table stores the former (plus the
+		number of blocks and counters) but PVRScopeServices expects the latter (plus the number of blocks and counters). The conversion
+		could always be moved to PVRScopeServices, but it's less code this way. */
+		psBlock->ui16BlockID		= (ui16BlockID & RGX_CNTBLK_ID_GROUP_MASK) ? (ui16BlockID | RGX_CNTBLK_ID_UNIT_ALL_MASK) : ui16BlockID;
+		if ((ui16BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+		{
+			psBlock->ui16NumCounters	= RGX_CNTBLK_COUNTERS_MAX;
+		}
+		else
+		{
+			psBlock->ui16NumCounters	= ui16NumCounters;
+		}
+		psBlock->ui16NumBlocks		= ui16NumBlocks;
+
+		*pui16Count = ui16Count + 1;
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_HWPERF_BVNC *psBVNC)
+{
+	IMG_PCHAR pszBVNC;
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDevInfo), "psDevInfo invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((pszBVNC = RGXDevBVNCString(psDevInfo)))
+	{
+		size_t uiStringLength = OSStringNLength(pszBVNC, RGX_HWPERF_MAX_BVNC_LEN - 1);
+		OSStringSafeCopy(psBVNC->aszBvncString, pszBVNC, uiStringLength + 1);
+		memset(&psBVNC->aszBvncString[uiStringLength], 0, RGX_HWPERF_MAX_BVNC_LEN - uiStringLength);
+	}
+	else
+	{
+		*psBVNC->aszBvncString = 0;
+	}
+
+	psBVNC->ui32BvncKmFeatureFlags = 0x0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERFBUS_FLAG;
+	}
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_XT_TOP_INFRASTRUCTURE_FLAG;
+	}
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERF_COUNTER_BATCH))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERF_COUNTER_BATCH_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ROGUEXE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_ROGUEXE_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PBE2_IN_XE_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_MULTICORE_FLAG;
+	}
+
+#ifdef SUPPORT_WORKLOAD_ESTIMATION
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Not a part of BVNC feature line and so doesn't need the feature supported check */
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION;
+	}
+#endif
+
+	/* Define the HW counter block counts. */
+	{
+		RGX_HWPERF_BVNC_BLOCK					* const psBlocks	= psBVNC->aBvncBlocks;
+		IMG_UINT16								* const pui16Count	= &psBVNC->ui16BvncBlocks;
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	*asCntBlkTypeModel;
+		const IMG_UINT32						ui32CntBlkModelLen	= RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+		IMG_UINT32								ui32BlkCfgIdx;
+		size_t									uiCount;
+		IMG_BOOL								bOk					= IMG_TRUE;
+
+		// Initialise to zero blocks
+		*pui16Count = 0;
+
+		// Add all the blocks
+		for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+		{
+			const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	* const psCntBlkInfo = &asCntBlkTypeModel[ui32BlkCfgIdx];
+			RGX_HWPERF_CNTBLK_RT_INFO				sCntBlkRtInfo;
+			/* psCntBlkInfo->ui8NumUnits gives compile-time info. For BVNC agnosticism, we use this: */
+			if (psCntBlkInfo->pfnIsBlkPresent(psCntBlkInfo, psDevInfo, &sCntBlkRtInfo))
+			{
+				bOk &= RGXServerFeatureFlagsToHWPerfFlagsAddBlock(psBlocks, pui16Count, psCntBlkInfo->ui32CntBlkIdBase, psCntBlkInfo->ui8NumCounters, sCntBlkRtInfo.ui32NumUnits);
+			}
+		}
+
+		/* If this fails, consider why the static_assert didn't fail, and consider increasing RGX_HWPERF_MAX_BVNC_BLOCK_LEN */
+		PVR_ASSERT(bOk);
+
+		// Zero the remaining entries
+		uiCount = *pui16Count;
+		OSDeviceMemSet(&psBlocks[uiCount], 0, (RGX_HWPERF_MAX_BVNC_BLOCK_LEN - uiCount) * sizeof(*psBlocks));
+	}
+
+	/* The GPU core count is overwritten by the FW */
+	psBVNC->ui16BvncGPUCores = 0;
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVRGXConfigMuxHWPerfCountersKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigMuxHWPerfCountersKM(
+		CONNECTION_DATA               *psConnection,
+		PVRSRV_DEVICE_NODE            *psDeviceNode,
+		IMG_UINT32                     ui32ArrayLen,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK  *psBlockConfigs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	DEVMEM_MEMDESC*		psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_MUX_CNTBLK* psFwArray;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed
+	 */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.sBlockConfigs,
+	                      psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM parameters set, calling FW"));*/
+
+	/* Ask the FW to carry out the HWPerf configuration command
+	 */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM command scheduled for FW"));*/
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration
+	 */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM firmware completed"));*/
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED", ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+/*
+	PVRSRVRGXConfigCustomCountersReadingHWPerfKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigCustomCountersKM(
+		CONNECTION_DATA             * psConnection,
+		PVRSRV_DEVICE_NODE          * psDeviceNode,
+		IMG_UINT16                    ui16CustomBlockID,
+		IMG_UINT16                    ui16NumCustomCounters,
+		IMG_UINT32                  * pui32CustomCounterIDs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	DEVMEM_MEMDESC*		psFwSelectCntrsMemDesc = NULL;
+	IMG_UINT32*			psFwArray;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVRGXSelectCustomCountersKM: configure block %u to read %u counters", ui16CustomBlockID, ui16NumCustomCounters));
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS;
+	sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.ui16NumCounters = ui16NumCustomCounters;
+	sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.ui16CustomBlock = ui16CustomBlockID;
+
+	if (ui16NumCustomCounters > 0)
+	{
+		PVR_ASSERT(pui32CustomCounterIDs);
+
+		/* used for passing counters config to the Firmware, write-only for the CPU */
+		eError = DevmemFwAllocate(psDevice,
+		                          sizeof(IMG_UINT32) * ui16NumCustomCounters,
+		                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+		                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+		                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+		                          "FwHWPerfConfigCustomCounters",
+		                          &psFwSelectCntrsMemDesc);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+		eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.sCustomCounterIDs,
+		                      psFwSelectCntrsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+		eError = DevmemAcquireCpuVirtAddr(psFwSelectCntrsMemDesc, (void **)&psFwArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+		OSCachedMemCopyWMB(psFwArray, pui32CustomCounterIDs, sizeof(IMG_UINT32) * ui16NumCustomCounters);
+		DevmemPDumpLoadMem(psFwSelectCntrsMemDesc,
+		                   0,
+		                   sizeof(IMG_UINT32) * ui16NumCustomCounters,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	/* Push in the KCCB the command to configure the custom counters block */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail3);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXSelectCustomCountersKM: Command scheduled"));
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXSelectCustomCountersKM: FW operation completed"));
+
+	if (ui16NumCustomCounters > 0)
+	{
+		/* Release temporary memory used for block configuration */
+		RGXUnsetFirmwareAddress(psFwSelectCntrsMemDesc);
+		DevmemReleaseCpuVirtAddr(psFwSelectCntrsMemDesc);
+		DevmemFwUnmapAndFree(psDevice, psFwSelectCntrsMemDesc);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf custom counters %u reading will be sent with the next HW events", ui16NumCustomCounters));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	if (psFwSelectCntrsMemDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psFwSelectCntrsMemDesc);
+	}
+fail2:
+	if (psFwSelectCntrsMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psFwSelectCntrsMemDesc);
+	}
+fail1:
+	if (psFwSelectCntrsMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevice, psFwSelectCntrsMemDesc);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*
+	PVRSRVRGXConfigureHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+		CONNECTION_DATA          * psConnection,
+		PVRSRV_DEVICE_NODE       * psDeviceNode,
+		IMG_UINT32                 ui32CtrlWord,
+		IMG_UINT32                 ui32ArrayLen,
+		RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs)
+{
+	PVRSRV_ERROR             eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD         sKccbCmd;
+	DEVMEM_MEMDESC           *psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_CNTBLK *psFwArray;
+	IMG_UINT32               ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO       *psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(ui32CtrlWord);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgDABlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_CNTBLK) * ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersDAConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgDABlks.sBlockConfigs,
+	                               psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/* Ask the FW to carry out the HWPerf configuration command. */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration. */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED",
+	         ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfConfigMuxCounters(
+		RGX_HWPERF_CONNECTION           *psHWPerfConnection,
+		IMG_UINT32					     ui32NumBlocks,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK	*asBlockConfigs)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !asBlockConfigs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXConfigMuxHWPerfCountersKM(NULL,
+		                                            psDevData->psRgxDevNode,
+		                                            ui32NumBlocks,
+		                                            asBlockConfigs);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXConfigMuxHWPerfCountersKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfConfigureAndEnableCustomCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT16              ui16CustomBlockID,
+		IMG_UINT16          ui16NumCustomCounters,
+		IMG_UINT32         *pui32CustomCounterIDs)
+{
+	PVRSRV_ERROR            eError;
+	RGX_HWPERF_DEVICE       *psHWPerfDev;
+
+	/* Validate input arguments supplied by the caller */
+	PVR_LOG_RETURN_IF_FALSE((NULL != psHWPerfConnection), "psHWPerfConnection invalid",
+	                   PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((0 != ui16NumCustomCounters), "uiNumBlocks invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((NULL != pui32CustomCounterIDs),"asBlockConfigs invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Check # of blocks */
+	PVR_LOG_RETURN_IF_FALSE((!(ui16CustomBlockID > RGX_HWPERF_MAX_CUSTOM_BLKS)),"ui16CustomBlockID invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Check # of counters */
+	PVR_LOG_RETURN_IF_FALSE((!(ui16NumCustomCounters > RGX_HWPERF_MAX_CUSTOM_CNTRS)),"ui16NumCustomCounters invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		RGX_KM_HWPERF_DEVDATA *psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		eError = PVRSRVRGXConfigCustomCountersKM(NULL,
+				                                 psDevData->psRgxDevNode,
+												 ui16CustomBlockID, ui16NumCustomCounters, pui32CustomCounterIDs);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXCtrlCustHWPerfKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHWPerfBlockTypeByID
+@Description    Lookup function to obtain a block type descriptor for a given
+                counter block identifier.
+@Input          psDevInfo                       A pointer to current device info.
+@Input          ui32BlockID                     The block ID for which a type
+                                                descriptor should be retrieved.
+@Return         RGXFW_HWPERF_CNTBLK_TYPE_MODEL  Block type descriptor.
+*/ /**************************************************************************/
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *
+GetHWPerfBlockTypeByID(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32BlockID)
+{
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	IMG_UINT32 ui32TableIdx = 0xFFFF;
+	RGX_HWPERF_CNTBLK_RT_INFO sRtInfo; /* Only used to satisfy pfnIsBlkPresent requirements. */
+
+#if defined(HWPERF_UNIFIED)
+	IMG_UINT32 uiBlockID = (IMG_UINT32)(ui32BlockID & ~(RGX_CNTBLK_ID_UNIT_ALL_MASK|RGX_CNTBLK_ID_DA_MASK));
+#else
+	IMG_UINT32 uiBlockID = (IMG_UINT32)(ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK);
+#endif
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	/* Is it a direct block? */
+	if (uiBlockID < RGX_CNTBLK_ID_DIRECT_LAST)
+	{
+		ui32TableIdx = uiBlockID;
+	}
+	/* Is it an indirect block */
+	else if ((uiBlockID > RGX_CNTBLK_ID_DIRECT_LAST) && (uiBlockID < RGX_CNTBLK_ID_LAST))
+	{
+		ui32TableIdx = RGX_CNTBLK_ID_DIRECT_LAST + (((uiBlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT) - 1U);
+	}
+	/* Unknown mapping from CNTBLK_ID to Table index */
+	else
+	{
+		return NULL;
+	}
+
+	PVR_ASSERT(ui32TableIdx < ui32CntBlkModelLen);
+
+	if (psDevInfo == NULL)
+	{
+		PVR_LOG(("psDevInfo invalid"));
+		return NULL;
+	}
+
+	if ((ui32TableIdx < ui32CntBlkModelLen) &&
+		(asCntBlkTypeModel[ui32TableIdx].pfnIsBlkPresent(&asCntBlkTypeModel[ui32TableIdx], psDevInfo, &sRtInfo) != IMG_FALSE))
+	{
+		return &asCntBlkTypeModel[ui32TableIdx];
+	}
+
+	/* Fall through, block not valid from run-time validation */
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(CONNECTION_DATA *psConnection,
+                                                       PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                       const IMG_UINT32 ui32BlockID,
+                                                       RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXAcquireHWPerfCtlCPUAddr");
+
+	eError = PVRSRVRGXGetConfiguredHWPerfMuxCounters(psDeviceNode,
+	                                                 psHWPerfCtl,
+	                                                 ui32BlockID,
+	                                                 psConfiguredMuxCounters);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRGXGetConfiguredHWPerfMuxCounters");
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                     RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                     IMG_UINT32 ui32BlockID,
+                                                     RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredMuxCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) < RGX_CNTBLK_ID_LAST)
+	{
+		RGXFWIF_HWPERF_CTL_BLK *psBlock = rgxfw_hwperf_get_block_ctl(ui32BlockID, psHWPerfCtl);
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *psBlkTypeDesc;
+		IMG_UINT32 i, ui32LastCountIdx = 0, ui8CurCountIdx = 0;
+		RGX_HWPERF_CONFIG_MUX_CNTBLK sBlockConfig;
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		if (!psBlock->ui32Enabled || !psBlock->ui32Valid)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not %s",
+			         ui32BlockID,
+			         !psBlock->ui32Enabled ? "enabled." : "configured."));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		psBlkTypeDesc = GetHWPerfBlockTypeByID(psDevInfo, psBlock->eBlockID);
+		if (psBlkTypeDesc == NULL)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->eBlockID;
+		sBlockConfig.ui8Mode = 0;
+
+		for (i = 0; ((psBlock->uiCounterMask >> i) != 0) &&
+		     (ui8CurCountIdx < psBlkTypeDesc->ui8NumCounters); i++)
+		{
+			if (psBlock->uiCounterMask & (1 << i))
+			{
+				IMG_UINT8 ui8Mode = 0;
+
+				ui8Mode = (psBlock->aui64CounterCfg[i] >> psBlkTypeDesc->ui8SelectRegModeShift) & 1U;
+				sBlockConfig.ui8Mode |= ui8Mode << ui32LastCountIdx;
+
+				sBlockConfig.aui8GroupSelect[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_SHIFT) & 0x1F;
+
+				sBlockConfig.aui16BitSelect[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BIT_SELECT_SHIFT) & 0x7FFF;
+
+#if defined(RGX_FEATURE_PERF_COUNTER_BATCH)
+				sBlockConfig.aui32BatchMax[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BATCH_MAX_SHIFT) & 0x1FFF;
+
+				sBlockConfig.aui32BatchMin[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BATCH_MIN_SHIFT) & 0x1FFF;
+#endif
+				ui32LastCountIdx++;
+				ui8CurCountIdx++;
+			}
+		}
+
+		sBlockConfig.ui8CounterSelect = (1 << ui32LastCountIdx) - 1;
+		*psConfiguredMuxCounters = sBlockConfig;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	RGX_HWPERF_CONFIG_CNTBLK sBlockConfig;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & RGX_CNTBLK_ID_CUSTOM_MASK) >= RGX_CNTBLK_ID_LAST)
+	{
+		/* Validate block ID */
+		switch (ui32BlockID)
+		{
+			case RGX_CNTBLK_ID_CUSTOM0:
+			case RGX_CNTBLK_ID_CUSTOM1:
+			case RGX_CNTBLK_ID_CUSTOM2:
+			case RGX_CNTBLK_ID_CUSTOM3:
+			case RGX_CNTBLK_ID_CUSTOM4_FW:
+			{
+				PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+				/* Check to see if this block is enabled */
+				if (psHWPerfCtl->ui32SelectedCountersBlockMask & (1 << (ui32BlockID & 0x0F)))
+				{
+					RGXFW_HWPERF_SELECT *psBlock = &psHWPerfCtl->SelCntr[ui32BlockID & 0x0F];
+
+					sBlockConfig.ui16BlockID = ui32BlockID;
+					sBlockConfig.ui16NumCounters = psBlock->ui32NumSelectedCounters;
+
+					for (i = 0; i < psBlock->ui32NumSelectedCounters; i++)
+					{
+						sBlockConfig.ui16Counters[i] = psBlock->aui32SelectedCountersIDs[i];
+					}
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+					PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+				}
+				break;
+			}
+			default:
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+			}
+		}
+	}
+#if defined(HWPERF_UNIFIED)
+	else if ((ui32BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+	{
+		RGXFWIF_HWPERF_DA_BLK *psBlock = rgxfw_hwperf_get_da_block_ctl(ui32BlockID, psHWPerfCtl);
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		if (!psBlock->uiEnabled)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->eBlockID;
+		sBlockConfig.ui16NumCounters = psBlock->uiNumCounters;
+
+		for (i = 0; i < psBlock->uiNumCounters; i++)
+		{
+			sBlockConfig.ui16Counters[i] = psBlock->aui32Counters[i];
+		}
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	if (eError == PVRSRV_OK)
+	{
+		*psConfiguredCounters = sBlockConfig;
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLen,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	IMG_UINT32 ui32LastIdx = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(pui32BlockCount != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	*pui32BlockCount = 0;
+
+	if (ui32ArrayLen > 0 && pui32EnabledBlockIDs == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen is greater than 0 but pui32EnabledBlockIDs is NULL"));
+	}
+
+	PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+	for (i = 0; i < RGX_HWPERF_MAX_MUX_BLKS; i++)
+	{
+		if (psHWPerfCtl->sBlkCfg[i].ui32Enabled && psHWPerfCtl->sBlkCfg[i].ui32Valid)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx + 1 > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sBlkCfg[i].eBlockID;
+			ui32LastIdx++;
+		}
+	}
+
+	for (i = 0; i < RGX_HWPERF_MAX_CUSTOM_BLKS; i++)
+	{
+		if (psHWPerfCtl->ui32SelectedCountersBlockMask == 0)
+		{
+			break;
+		}
+
+		if (psHWPerfCtl->ui32SelectedCountersBlockMask & (1 << i))
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx + 1 > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = RGX_CNTBLK_ID_CUSTOM0 + i;
+			ui32LastIdx++;
+		}
+	}
+
+#if defined(HWPERF_UNIFIED)
+	for (i = 0; i < RGX_HWPERF_MAX_DA_BLKS; i++)
+	{
+		if (psHWPerfCtl->sDABlkCfg[i].uiEnabled)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sDABlkCfg[i].eBlockID;
+			ui32LastIdx++;
+		}
+	}
+#endif
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxinit.c
new file mode 100644
index 000000000000..d6096e6a9681
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxinit.c
@@ -0,0 +1,5608 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_bridge_init.h"
+#include "rgx_bridge_init.h"
+#include "syscommon.h"
+#include "rgx_heaps_server.h"
+#include "rgxheapconfig.h"
+#include "rgxpower.h"
+#include "tlstream.h"
+#include "pvrsrv_tlstreams.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "pvr_ricommon.h"
+#endif
+
+#include "rgxinit.h"
+#include "rgxbvnc.h"
+#include "rgxmulticore.h"
+
+#include "pdump_km.h"
+#include "handle.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxmem.h"
+#include "sync_internal.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "rgxfwdbg.h"
+#include "info_page.h"
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxfwimageutils.h"
+#endif
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+#include "rgxmmuinit.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#include "physmem.h"
+#endif
+#include "devicemem_utils.h"
+#include "devicemem_server.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "rgxta3d.h"
+#include "rgxtimecorr.h"
+#include "rgxshader.h"
+
+#if defined(PDUMP)
+#include "rgxstartstop.h"
+#endif
+
+#include "rgx_fwif_alignchecks.h"
+#include "vmm_pvz_client.h"
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+
+#if defined(PDUMP) && defined(SUPPORT_SECURITY_VALIDATION)
+#include "pdump_physmem.h"
+#endif
+
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_CHAR **ppszVersionString);
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PUINT32 pui32RGXClockSpeed);
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID);
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/* Services internal heap identification used in this file only */
+#define RGX_FIRMWARE_MAIN_HEAP_IDENT   "FwMain"   /*!< RGX Main Firmware Heap identifier */
+#define RGX_FIRMWARE_CONFIG_HEAP_IDENT "FwConfig" /*!< RGX Config firmware Heap identifier */
+
+#define RGX_MMU_PAGE_SIZE_4KB   (   4 * 1024)
+#define RGX_MMU_PAGE_SIZE_16KB  (  16 * 1024)
+#define RGX_MMU_PAGE_SIZE_64KB  (  64 * 1024)
+#define RGX_MMU_PAGE_SIZE_256KB ( 256 * 1024)
+#define RGX_MMU_PAGE_SIZE_1MB   (1024 * 1024)
+#define RGX_MMU_PAGE_SIZE_2MB   (2048 * 1024)
+#define RGX_MMU_PAGE_SIZE_MIN RGX_MMU_PAGE_SIZE_4KB
+#define RGX_MMU_PAGE_SIZE_MAX RGX_MMU_PAGE_SIZE_2MB
+
+#define VAR(x) #x
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@Function       SampleIRQCount
+@Description    Utility function taking snapshots of RGX FW interrupt count.
+@Input          psDevInfo    Device Info structure
+
+@Return         IMG_BOOL     Returns IMG_TRUE if RGX FW IRQ is not equal to
+                             sampled RGX FW IRQ count for any RGX FW thread.
+ */ /**************************************************************************/
+static INLINE IMG_BOOL SampleIRQCount(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bReturnVal = IMG_FALSE;
+	volatile IMG_UINT32 *pui32SampleIrqCount = psDevInfo->aui32SampleIRQCount;
+	IMG_UINT32 ui32IrqCnt;
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bReturnVal = IMG_TRUE;
+	}
+	else
+	{
+		get_irq_cnt_val(ui32IrqCnt, RGXFW_HOST_DRIVER_ID, psDevInfo);
+
+		if (ui32IrqCnt != pui32SampleIrqCount[RGXFW_THREAD_0])
+		{
+			pui32SampleIrqCount[RGXFW_THREAD_0] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#else
+	IMG_UINT32 ui32TID;
+
+	for_each_irq_cnt(ui32TID)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32TID, psDevInfo);
+
+		/* treat unhandled interrupts here to align host count with fw count */
+		if (pui32SampleIrqCount[ui32TID] != ui32IrqCnt)
+		{
+			pui32SampleIrqCount[ui32TID] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#endif
+
+	return bReturnVal;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHostSafetyEvents
+@Description    Returns the event status masked to keep only the safety
+                events handled by the Host
+@Input          psDevInfo    Device Info structure
+@Return         IMG_UINT32   Status of Host-handled safety events
+ */ /**************************************************************************/
+static INLINE IMG_UINT32 RGXHostSafetyEvents(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (psDevInfo->ui32HostSafetyEventMask == 0))
+	{
+		return 0;
+	}
+	else
+	{
+		IMG_UINT32 ui32SafetyEventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE);
+		return (ui32SafetyEventStatus & psDevInfo->ui32HostSafetyEventMask);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventCheck
+@Description    Clears the Event Status register and checks if any of the
+                safety events need Host handling
+@Input          psDevInfo    Device Info structure
+@Return         IMG_BOOL     Are there any safety events for Host to handle ?
+ */ /**************************************************************************/
+static INLINE IMG_BOOL RGXSafetyEventCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bSafetyEvent = IMG_FALSE;
+
+	if (psDevInfo->ui32HostSafetyEventMask != 0)
+	{
+		IMG_UINT32 ui32EventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_STATUS);
+
+		if (BIT_ISSET(ui32EventStatus, RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_SHIFT))
+		{
+			/* clear the safety event */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_CLEAR, RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_EN);
+
+			/* report if there is anything for the Host to handle */
+			bSafetyEvent = (RGXHostSafetyEvents(psDevInfo) != 0);
+		}
+	}
+
+	return bSafetyEvent;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventHandler
+@Description    Handles the Safety Events that the Host is responsible for
+@Input          psDevInfo    Device Info structure
+ */ /**************************************************************************/
+static void RGXSafetyEventHandler(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	if (psDevInfo->bRGXPowered)
+	{
+		RGX_CONTEXT_RESET_REASON eResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+		IMG_UINT32 ui32HostSafetyStatus = RGXHostSafetyEvents(psDevInfo);
+
+		if (ui32HostSafetyStatus != 0)
+		{
+			/* clear the safety bus events handled by the Host */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE, ui32HostSafetyStatus);
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_SHIFT))
+			{
+				IMG_UINT32 ui32FaultFlag;
+				IMG_UINT32 ui32FaultFW = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_STATUS);
+				IMG_UINT32 ui32CorrectedBitOffset = RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_SHIFT -
+													RGX_CR_FAULT_FW_STATUS_CPU_DETECT_SHIFT;
+
+				PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety fault status: 0x%X", __func__, ui32FaultFW));
+
+				for (ui32FaultFlag = 0; ui32FaultFlag < ui32CorrectedBitOffset; ui32FaultFlag++)
+				{
+					if (BIT_ISSET(ui32FaultFW, ui32FaultFlag))
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety hardware fault detected (0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_ERR;
+					}
+					else if BIT_ISSET(ui32FaultFW, ui32FaultFlag + ui32CorrectedBitOffset)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Firmware safety hardware fault corrected.(0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+
+						/* Only report this if we haven't detected a more serious error */
+						if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR)
+						{
+							eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_OK;
+						}
+					}
+				}
+
+				OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_CLEAR, ui32FaultFW);
+			}
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT))
+			{
+				volatile RGXFWIF_POW_STATE ePowState;
+
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+										   INVALIDATE);
+				ePowState = psDevInfo->psRGXFWIfFwSysData->ePowState;
+
+				if (ePowState != RGXFWIF_POW_OFF)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Safety Watchdog Trigger !", __func__));
+
+					/* Only report this if we haven't detected a more serious error */
+					if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR)
+					{
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_WATCHDOG;
+					}
+				}
+			}
+
+			/* Notify client and system layer of any error */
+			if (eResetReason != RGX_CONTEXT_RESET_REASON_NONE)
+			{
+				PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+				PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = eResetReason;
+
+				/* Notify system layer of any error */
+				if (psDevConfig->pfnSysDevErrorNotify)
+				{
+					PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+					sErrorData.eResetReason = eResetReason;
+
+					psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+													  &sErrorData);
+				}
+			}
+		}
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+static IMG_BOOL _WaitForInterruptsTimeoutCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32idx;
+#endif
+
+	RGXDEBUG_PRINT_IRQ_COUNT(psDevInfo);
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVR_DPF((PVR_DBG_ERROR,
+	        "Last RGX_LISRHandler State (DevID %u): 0x%08X Clock: %" IMG_UINT64_FMTSPEC,
+			 psDeviceNode->sDevId.ui32InternalID,
+			 psDeviceNode->sLISRExecutionInfo.ui32Status,
+			 psDeviceNode->sLISRExecutionInfo.ui64Clockns));
+
+	for_each_irq_cnt(ui32idx)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				MSG_IRQ_CNT_TYPE " %u: InterruptCountSnapshot: 0x%X",
+				ui32idx, psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[ui32idx]));
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR, "No further information available. Please enable PVRSRV_DEBUG_LISR_EXECUTION"));
+#endif
+
+	return SampleIRQCount(psDevInfo);
+}
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bScheduleMISR;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bScheduleMISR = IMG_TRUE;
+	}
+	else
+	{
+		bScheduleMISR = _WaitForInterruptsTimeoutCheck(psDevInfo);
+	}
+
+	if (bScheduleMISR)
+	{
+		OSScheduleMISR(psDevInfo->pvMISRData);
+
+		if (psDevInfo->pvAPMISRData != NULL)
+		{
+			OSScheduleMISR(psDevInfo->pvAPMISRData);
+		}
+	}
+}
+
+static inline IMG_BOOL RGXAckHwIrq(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_UINT32 ui32IRQStatusReg,
+								   IMG_UINT32 ui32IRQStatusEventMsk,
+								   IMG_UINT32 ui32IRQClearReg,
+								   IMG_UINT32 ui32IRQClearMask)
+{
+	IMG_UINT32 ui32IRQStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQStatusReg);
+
+	/* clear only the pending bit of the thread that triggered this interrupt */
+	ui32IRQClearMask &= ui32IRQStatus;
+
+	if (ui32IRQStatus & ui32IRQStatusEventMsk)
+	{
+		/* acknowledge and clear the interrupt */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQClearReg, ui32IRQClearMask);
+		return IMG_TRUE;
+	}
+	else
+	{
+		/* spurious interrupt */
+		return IMG_FALSE;
+	}
+}
+
+static __maybe_unused IMG_BOOL RGXAckIrqMETA(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static IMG_BOOL RGXAckIrqMIPS(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN);
+}
+#endif
+
+static IMG_BOOL RGXAckIrqDedicated(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+		/* status & clearing registers are available on both Host and Guests
+		 * and are agnostic of the Fw CPU type. Due to the remappings done by
+		 * the 2nd stage device MMU, all drivers assume they are accessing
+		 * register bank 0 */
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_IRQ_OS0_EVENT_STATUS,
+					   ~RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK,
+					   RGX_CR_IRQ_OS0_EVENT_CLEAR,
+					   ~RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK);
+}
+
+static PVRSRV_ERROR RGXSetAckIrq(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, IRQ_PER_OS)) && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo)))
+	{
+		psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psDevInfo->pfnRGXAckIrq = NULL;
+	}
+	else
+	{
+		/* native and host drivers must clear the unique GPU physical interrupt */
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqMIPS;
+		}
+		else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqMETA;
+		}
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU IRQ clearing mechanism not implemented "
+									"for this architecture.", __func__));
+			return PVRSRV_ERROR_NOT_IMPLEMENTED;
+		}
+	}
+
+#if defined(RGX_IRQ_HYPERV_HANDLER)
+	/* The hypervisor receives and acknowledges the GPU irq, then it injects an
+	 * irq only in the recipient OS. The KM driver doesn't handle the GPU irq line */
+	psDevInfo->pfnRGXAckIrq = NULL;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static IMG_BOOL RGX_LISRHandler(void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bIrqAcknowledged = IMG_FALSE;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	IMG_UINT32 ui32idx, ui32IrqCnt;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+		UPDATE_LISR_DBG_SNAPSHOT(ui32idx, ui32IrqCnt);
+	}
+
+	UPDATE_LISR_DBG_STATUS(RGX_LISR_INIT);
+	UPDATE_LISR_DBG_TIMESTAMP();
+#endif
+
+	UPDATE_LISR_DBG_COUNTER();
+
+	if (psDevInfo->bRGXPowered)
+	{
+		IMG_BOOL bSafetyEvent = RGXSafetyEventCheck(psDevInfo);
+
+		if ((psDevInfo->pfnRGXAckIrq == NULL) || psDevInfo->pfnRGXAckIrq(psDevInfo) || bSafetyEvent)
+		{
+			bIrqAcknowledged = IMG_TRUE;
+
+			if (bSafetyEvent || SampleIRQCount(psDevInfo))
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_PROCESSED);
+				UPDATE_MISR_DBG_COUNTER();
+
+				OSScheduleMISR(psDevInfo->pvMISRData);
+
+#if defined(SUPPORT_AUTOVZ)
+				RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+				if (psDevInfo->pvAPMISRData != NULL)
+				{
+					OSScheduleMISR(psDevInfo->pvAPMISRData);
+				}
+			}
+			else
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED);
+			}
+		}
+		else
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_NOT_TRIGGERED_BY_HW);
+		}
+	}
+	else
+	{
+#if defined(SUPPORT_AUTOVZ)
+		/* AutoVz drivers rebooting while the firmware is active must acknowledge
+		 * and clear the hw IRQ line before the RGXInit() has finished. */
+		if ((psDevInfo->pfnRGXAckIrq != NULL) &&
+			 psDevInfo->pfnRGXAckIrq(psDevInfo))
+		{
+			bIrqAcknowledged = IMG_TRUE;
+		}
+		else
+#endif
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_DEVICE_NOT_POWERED);
+		}
+	}
+
+	return bIrqAcknowledged;
+}
+
+static void RGX_MISR_ProcessKCCBDeferredList(PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* First check whether there are pending commands in Deferred KCCB List */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	if (dllist_is_empty(&psDevInfo->sKCCBDeferredCommandsListHead))
+	{
+		OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+		return;
+	}
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	/* Powerlock to avoid further Power transition requests
+	   while KCCB deferred list is being processed */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Try to send deferred KCCB commands Do not Poll from here*/
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s could not flush Deferred KCCB list, KCCB is full.",
+				 __func__));
+	}
+}
+
+static void RGX_MISRHandler_CheckFWActivePowerState(void *psDevice)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevice;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_ON || psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		/* The FW is IDLE and therefore could be shut down */
+		eError = RGXActivePowerRequest(psDeviceNode);
+
+		if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED))
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+					"%s: Failed RGXActivePowerRequest call (device: %p) with %s",
+					__func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+			else
+			{
+				/* Re-schedule the power down request as it was deferred. */
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+	}
+
+}
+
+/* Shorter defines to keep the code a bit shorter */
+#define GPU_IDLE       RGXFWIF_GPU_UTIL_STATE_IDLE
+#define GPU_ACTIVE     RGXFWIF_GPU_UTIL_STATE_ACTIVE
+#define GPU_BLOCKED    RGXFWIF_GPU_UTIL_STATE_BLOCKED
+#define GPU_INACTIVE   RGXFWIF_GPU_UTIL_STATE_INACTIVE
+#define MAX_ITERATIONS 64
+#define MAX_DIFF_TIME_NS (300000ULL)
+#define MAX_DIFF_DM_TIME_NS (MAX_DIFF_TIME_NS >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT)
+
+static PVRSRV_ERROR RGXGetGpuUtilStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_HANDLE hGpuUtilUser,
+                                       RGXFWIF_GPU_UTIL_STATS *psReturnStats)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_STATS sStats;
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+	IMG_UINT64 (*paaui64DMOSTmpCounters)[RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 (*paui64DMOSTmpLastWord)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastState)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastPeriod)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastTime)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT64 ui64TimeNowShifted;
+	IMG_UINT32 ui32Attempts;
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32MaxDMCount;
+	RGXFWIF_DM eDM;
+
+	/***** (1) Initialise return stats *****/
+
+	psReturnStats->bValid = IMG_FALSE;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psAggregateStats = hGpuUtilUser;
+
+	/* decrease by 1 to account for excluding GP DM from the statics */;
+	ui32MaxDMCount =  psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1;
+
+	/* Reset temporary counters used in the attempts loop */
+	paaui64DMOSTmpCounters  = &psAggregateStats->sTempGpuStats.aaaui64DMOSTmpCounters[0];
+	paui64DMOSTmpLastWord   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastWord[0];
+	paui64DMOSTmpLastState  = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastState[0];
+	paui64DMOSTmpLastPeriod = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastPeriod[0];
+	paui64DMOSTmpLastTime   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastTime[0];
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	/* Try to acquire GPU utilisation counters and repeat if the FW is in the middle of an update */
+	for (ui32Attempts = 0; ui32Attempts < 4; ui32Attempts++)
+	{
+		IMG_UINT64 aui64GpuTmpCounters[RGXFWIF_GPU_UTIL_STATE_NUM] = {0};
+		IMG_UINT64 ui64GpuLastPeriod = 0, ui64GpuLastWord = 0, ui64GpuLastState = 0, ui64GpuLastTime = 0;
+
+		/***** (2) Get latest data from shared area *****/
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			IMG_UINT64 aui64StatsCountersNew[RGXFWIF_GPU_UTIL_STATE_NUM];
+			IMG_UINT64 ui64GpuLastWordNew;
+			RGXFWIF_GPU_STATS sStatsNew;
+			IMG_UINT32 i = 0;
+
+			ui64GpuLastWord = 0;
+			ui64GpuLastState = 0;
+
+			OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+			/* Copy data from device memory */
+			memcpy(&sStatsNew, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+			memcpy(&ui64GpuLastWordNew, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+			memcpy(aui64StatsCountersNew, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64StatsCountersNew));
+
+			/*
+			 * First attempt at detecting if the FW is in the middle of an update.
+			 * This should also help if the FW is in the middle of a 64 bit variable update.
+			 * This loop must be fast. Faster than FW updates the stats.
+			 */
+			for (i = 0; i < MAX_ITERATIONS; i++)
+			{
+				IMG_UINT32 j,k;
+				IMG_BOOL bRetry = IMG_FALSE;
+
+				if (i > 0)
+				{
+					/* On retry keep previous data */
+					ui64GpuLastWordNew = ui64GpuLastWord;
+					memcpy(aui64StatsCountersNew, aui64GpuTmpCounters, sizeof(aui64StatsCountersNew));
+					memcpy(&sStatsNew, &sStats, sizeof(sStatsNew));
+				}
+
+				/* Copy data from device memory */
+				memcpy(&sStats, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+				memcpy(&ui64GpuLastWord, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+				memcpy(aui64GpuTmpCounters, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64GpuTmpCounters));
+
+				/* Check for abnormal time difference between reads */
+				if (RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord) - RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWordNew) > MAX_DIFF_TIME_NS)
+				{
+					bRetry = IMG_TRUE;
+					continue;
+				}
+
+				for (j = 0; j < RGXFWIF_GPU_UTIL_STATE_NUM; j++)
+				{
+					/* Check for abnormal time difference between reads */
+					if (aui64GpuTmpCounters[j] - aui64StatsCountersNew[j] > MAX_DIFF_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+				}
+
+				if (bRetry)
+				{
+					continue;
+				}
+
+				/* Check for DM counters wrapped or
+				   abnormal time difference between reads.
+				   The DM time is shifted by RGXFWIF_DM_OS_TIMESTAMP_SHIFT */
+				for (j = 0; j < RGXFWIF_GPU_UTIL_DM_MAX; j++)
+				{
+					if (sStats.aui32DMOSLastWordWrap[j] != sStatsNew.aui32DMOSLastWordWrap[j] ||
+						RGXFWIF_GPU_UTIL_GET_TIME32(sStats.aui32DMOSLastWord[j]) - RGXFWIF_GPU_UTIL_GET_TIME32(sStatsNew.aui32DMOSLastWord[j]) > MAX_DIFF_DM_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+
+					for (k = 0; k < RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM; k++)
+					{
+						if (sStats.aaui32DMOSCountersWrap[j][k] != sStatsNew.aaui32DMOSCountersWrap[j][k] ||
+							sStats.aaui32DMOSStatsCounters[j][k] - sStatsNew.aaui32DMOSStatsCounters[j][k] > MAX_DIFF_DM_TIME_NS)
+						{
+							bRetry = IMG_TRUE;
+							break;
+						}
+
+					}
+
+					if (bRetry)
+					{
+						break;
+					}
+				}
+
+				if (!bRetry)
+				{
+					/* Stats are good*/
+					break;
+				}
+			}
+
+			OSLockRelease(psDevInfo->hGPUUtilLock);
+
+			ui64GpuLastState = RGXFWIF_GPU_UTIL_GET_STATE(ui64GpuLastWord);
+
+			if (i == MAX_ITERATIONS)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						 "RGXGetGpuUtilStats could not get reliable data after trying %u times", i));
+
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+			{
+				paui64DMOSTmpLastWord[eDM][ui32DriverID]  =
+					((IMG_UINT64)sStats.aui32DMOSLastWordWrap[eDM] << 32) + sStats.aui32DMOSLastWord[eDM];
+				paui64DMOSTmpLastState[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_STATE(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				if (paui64DMOSTmpLastState[eDM][ui32DriverID] != GPU_ACTIVE)
+				{
+					paui64DMOSTmpLastState[eDM][ui32DriverID] = GPU_INACTIVE;
+				}
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE] = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_INACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_INACTIVE] << 32);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE]  = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_ACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_ACTIVE] << 32);
+			}
+
+		} /* FOREACH_SUPPORTED_DRIVER(ui32DriverID) */
+
+
+		/***** (3) Compute return stats *****/
+
+		/* Update temp counters to account for the time since the last update to the shared ones */
+		OSMemoryBarrier(NULL); /* Ensure the current time is read after the loop above */
+		ui64TimeNow    = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDeviceNode));
+
+		ui64GpuLastTime   = RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord);
+		ui64GpuLastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64GpuLastTime);
+		aui64GpuTmpCounters[ui64GpuLastState] += ui64GpuLastPeriod;
+
+		/* Get statistics for a user since its last request */
+		psReturnStats->ui64GpuStatIdle = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_IDLE],
+		                                                             psAggregateStats->ui64GpuStatIdle);
+		psReturnStats->ui64GpuStatActive = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_ACTIVE],
+		                                                               psAggregateStats->ui64GpuStatActive);
+		psReturnStats->ui64GpuStatBlocked = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_BLOCKED],
+		                                                                psAggregateStats->ui64GpuStatBlocked);
+		psReturnStats->ui64GpuStatCumulative = psReturnStats->ui64GpuStatIdle +
+		                                       psReturnStats->ui64GpuStatActive +
+		                                       psReturnStats->ui64GpuStatBlocked;
+
+		/* convert time into the same units as used by fw */
+		ui64TimeNowShifted  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+		for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+		{
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				paui64DMOSTmpLastTime[eDM][ui32DriverID]   = RGXFWIF_GPU_UTIL_GET_TIME(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				paui64DMOSTmpLastPeriod[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNowShifted , paui64DMOSTmpLastTime[eDM][ui32DriverID]);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][paui64DMOSTmpLastState[eDM][ui32DriverID]] += paui64DMOSTmpLastPeriod[eDM][ui32DriverID];
+				/* Get statistics for a user since its last request */
+				psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE],
+				                                                             psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE],
+				                                                               psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatCumulative[eDM][ui32DriverID] = psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] +
+				                                       psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+			}
+		}
+
+		if (psAggregateStats->ui64TimeStamp != 0)
+		{
+			IMG_UINT64 ui64TimeSinceLastCall = ui64TimeNow - psAggregateStats->ui64TimeStamp;
+			/* We expect to return at least 75% of the time since the last call in GPU stats */
+			IMG_UINT64 ui64MinReturnedStats = ui64TimeSinceLastCall - (ui64TimeSinceLastCall / 4);
+
+			/*
+			 * If the returned stats are substantially lower than the time since
+			 * the last call, then the Host might have read a partial update from the FW.
+			 * If this happens, try sampling the shared counters again.
+			 */
+			if (psReturnStats->ui64GpuStatCumulative < ui64MinReturnedStats)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s: Return stats (%" IMG_UINT64_FMTSPEC ") too low "
+				         "(call period %" IMG_UINT64_FMTSPEC ")",
+				         __func__, psReturnStats->ui64GpuStatCumulative, ui64TimeSinceLastCall));
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Attempt #%u has failed, trying again",
+				         __func__, ui32Attempts));
+				continue;
+			}
+		}
+
+		break;
+	}
+
+	/***** (4) Update aggregate stats for the current user *****/
+
+	psAggregateStats->ui64GpuStatIdle    += psReturnStats->ui64GpuStatIdle;
+	psAggregateStats->ui64GpuStatActive  += psReturnStats->ui64GpuStatActive;
+	psAggregateStats->ui64GpuStatBlocked += psReturnStats->ui64GpuStatBlocked;
+	psAggregateStats->ui64TimeStamp       = ui64TimeNow;
+
+	for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+	{
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]    += psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID];
+			psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]  += psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+		}
+	}
+
+	/***** (5) Convert return stats to microseconds *****/
+
+	psReturnStats->ui64GpuStatIdle       = OSDivide64(psReturnStats->ui64GpuStatIdle, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatActive     = OSDivide64(psReturnStats->ui64GpuStatActive, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatBlocked    = OSDivide64(psReturnStats->ui64GpuStatBlocked, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatCumulative = OSDivide64(psReturnStats->ui64GpuStatCumulative, 1000, &ui32Remainder);
+
+	/* Check that the return stats make sense */
+	if (psReturnStats->ui64GpuStatCumulative == 0)
+	{
+		/* We can enter here only if allocating the temporary stats
+		 * buffers failed, or all the RGXFWIF_GPU_UTIL_GET_PERIOD
+		 * returned 0. The latter could happen if the GPU frequency value
+		 * is not well calibrated and the FW is updating the GPU state
+		 * while the Host is reading it.
+		 * When such an event happens frequently, timers or the aggregate
+		 * stats might not be accurate...
+		 */
+#if defined(VIRTUAL_PLATFORM)
+		/* To avoid spamming the console logging system on emulated devices,
+		 * we special-case so that we will only produce a single message per
+		 * driver invocation. This should reduce the time spent logging
+		 * information which is not relevant for very slow timers found in
+		 * VP device configurations
+		 */
+		static IMG_BOOL bFirstTime = IMG_TRUE;
+
+		if (bFirstTime)
+		{
+			bFirstTime = IMG_FALSE;
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "RGXGetGpuUtilStats could not get reliable data."));
+#if defined(VIRTUAL_PLATFORM)
+		}
+#endif	/* defined(VIRTUAL_PLATFORM) */
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psReturnStats->bValid = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psAggregateStats = OSAllocZMemNoStats(sizeof(RGXFWIF_GPU_UTIL_STATS));
+	if (psAggregateStats == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	*phGpuUtilUser = psAggregateStats;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAggregateStats = hGpuUtilUser;
+	OSFreeMemNoStats(psAggregateStats);
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGX MISR Handler
+*/
+static void RGX_MISRHandler_Main (void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* Give the HWPerf service a chance to transfer some data from the FW
+	 * buffer to the host driver transport layer buffer.
+	 */
+	RGXHWPerfDataStoreCB(psDeviceNode);
+
+	/* Inform other services devices that we have finished an operation */
+	PVRSRVNotifyCommandCompletion(psDeviceNode);
+
+#if defined(SUPPORT_PDVFS) && defined(RGXFW_META_SUPPORT_2ND_THREAD)
+	/* Normally, firmware CCB only exists for the primary FW thread unless PDVFS
+	   is running on the second[ary] FW thread, here we process said CCB */
+	RGXPDVFSCheckCoreClkRateChange(psDeviceNode->pvDevice);
+#endif
+
+	/* Only execute SafetyEventHandler if RGX_FEATURE_SAFETY_EVENT is on */
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, ECC_RAMS) > 0 ||
+		PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, WATCHDOG_TIMER))
+	{
+		/* Handle Safety events if necessary */
+		RGXSafetyEventHandler(psDeviceNode->pvDevice);
+	}
+
+	/* Signal the global event object */
+	PVRSRVSignalDriverWideEO();
+
+	/* Process the Firmware CCB for pending commands */
+	RGXCheckFirmwareCCB(psDeviceNode->pvDevice);
+
+	/* Calibrate the GPU frequency and recorrelate Host and GPU timers (done every few seconds) */
+	RGXTimeCorrRestartPeriodic(psDeviceNode);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Process Workload Estimation Specific commands from the FW */
+		WorkEstCheckFirmwareCCB(psDeviceNode->pvDevice);
+	}
+#endif
+
+	if (psDevInfo->pvAPMISRData == NULL)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+static PVRSRV_ERROR RGXPDumpBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+		PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psFWDataPMR;
+	RGXMIPSFW_BOOT_DATA *psBootData;
+	IMG_DEV_PHYADDR sTmpAddr;
+	IMG_UINT32 ui32BootConfOffset, ui32ParamOffset, i;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+	ui32BootConfOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+	ui32BootConfOffset += RGXMIPSFW_BOOTLDR_CONF_OFFSET;
+
+	/* The physical addresses used by a pdump player will be different
+	 * than the ones we have put in the MIPS bootloader configuration data.
+	 * We have to tell the pdump player to replace the original values with the real ones.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Pass new boot parameters to the FW");
+
+	/* Rogue Registers physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64RegBase);
+
+	eError = PDumpRegLabelToMem64(RGX_PDUMPREG_NAME,
+			0x0,
+			psFWDataPMR,
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of Rogue registers phy address failed (%u)", eError));
+		return eError;
+	}
+
+	/* Page Table physical Address */
+	eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sTmpAddr);
+	if (eError !=  PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXBootldrDataInit: MMU_AcquireBaseAddr failed (%u)",
+		         eError));
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+									 (void **)&psBootData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire pointer to FW data (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	psBootData = IMG_OFFSET_ADDR(psBootData, ui32BootConfOffset);
+
+	for (i = 0; i < psBootData->ui32PTNumPages; i++)
+	{
+		ui32ParamOffset = ui32BootConfOffset +
+			offsetof(RGXMIPSFW_BOOT_DATA, aui64PTPhyAddr[0])
+			+ i * sizeof(psBootData->aui64PTPhyAddr[0]);
+
+		eError = PDumpPTBaseObjectToMem64(psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				psFWDataPMR,
+				0,
+				ui32ParamOffset,
+				PDUMP_FLAGS_CONTINUOUS,
+				MMU_LEVEL_1,
+				sTmpAddr.uiAddr,
+				i << psBootData->ui32PTLog2PageSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of page tables phy address failed (%u)", eError));
+			return eError;
+		}
+	}
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+
+	/* Stack physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64StackPhyAddr);
+
+	eError = PDumpMemLabelToMem64(psFWDataPMR,
+			psFWDataPMR,
+			RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of stack phy address failed (%u)", eError));
+		return eError;
+	}
+
+	return eError;
+}
+#endif /* PDUMP */
+
+static PVRSRV_ERROR RGXSetPowerParams(PVRSRV_RGXDEV_INFO   *psDevInfo,
+                                      PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR sKernelMMUCtxPCAddr;
+	IMG_BOOL bPremappedFw;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVCFG, psDevConfig, PVRSRV_OK);
+
+	/* Save information used on power transitions for later
+	 * (when RGXStart and RGXStop are executed)
+	 */
+#if defined(PDUMP)
+	psDevInfo->sLayerParams.ui32PdumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	/* Rogue drivers with security support and premapped fw heaps
+	 * always have their fw heap premapped by the TEE */
+	bPremappedFw = IMG_TRUE;
+#else
+	/* If AutoVz firmware is up at this stage, the driver initialised it
+	 * during a previous life-cycle. The firmware's memory is already pre-mapped
+	 * and the MMU page tables reside in the predetermined memory carveout.
+	 * The Kernel MMU Context created in this life-cycle is a dummy structure
+	 * that is not used for mapping.
+	 * To program the Device's BIF with the correct PC address, use the base
+	 * address of the carveout reserved for MMU mappings as Kernel MMU PC Address */
+	bPremappedFw = psDevInfo->psDeviceNode->bAutoVzFwIsUp;
+#endif
+
+	if (bPremappedFw)
+	{
+		IMG_DEV_PHYADDR sDevPAddr;
+		PHYS_HEAP *psFwPageTableHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_RETURN_IF_FALSE((NULL != psFwPageTableHeap),
+								"Firmware Page Table heap not defined.",
+								PVRSRV_ERROR_INVALID_HEAP);
+
+		PhysHeapGetDevPAddr(psFwPageTableHeap, &sDevPAddr);
+		sKernelMMUCtxPCAddr.uiAddr = sDevPAddr.uiAddr;
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+	else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx,
+		                             &sKernelMMUCtxPCAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire Kernel MMU Ctx page catalog"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		PMR *psFWCodePMR = (PMR *)(psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR);
+		PMR *psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+		IMG_DEV_PHYADDR sPhyAddr;
+		IMG_BOOL bValid;
+
+#if defined(SUPPORT_ALT_REGBASE)
+		psDevInfo->sLayerParams.sGPURegAddr = psDevConfig->sAltRegsGpuPBase;
+#else
+		/* The physical address of the GPU registers needs to be translated
+		 * in case we are in a LMA scenario
+		 */
+		PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+				1,
+				&sPhyAddr,
+				&(psDevConfig->sRegsCpuPBase));
+
+		psDevInfo->sLayerParams.sGPURegAddr = sPhyAddr;
+#endif
+
+		/* Register bank must be aligned to 512KB (as per the core integration) to
+		 * prevent the FW accessing incorrect registers */
+		if ((psDevInfo->sLayerParams.sGPURegAddr.uiAddr & 0x7FFFFU) != 0U)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Register bank must be aligned to 512KB, but current address (0x%016"IMG_UINT64_FMTSPECX") is not",
+						psDevInfo->sLayerParams.sGPURegAddr.uiAddr));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE),
+				OSGetPageShift(), /* FW will be using the same page size as the OS */
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI code address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sBootRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWDataPMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI data address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sDataRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW exceptions address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sCodeRemapAddr = sPhyAddr;
+
+		psDevInfo->sLayerParams.sTrampolineRemapAddr.uiAddr = psDevInfo->psTrampoline->sPhysAddr.uiAddr;
+
+		psDevInfo->sLayerParams.bDevicePA0IsValid = psDevConfig->bDevicePA0IsValid;
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	/* Send information used on power transitions to the trusted device as
+	 * in this setup the driver cannot start/stop the GPU and perform resets
+	 */
+	if (psDevConfig->pfnTDSetPowerParams)
+	{
+		PVRSRV_TD_POWER_PARAMS sTDPowerParams;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+		    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			sTDPowerParams.sPCAddr = psDevInfo->sLayerParams.sPCAddr;
+		}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			sTDPowerParams.sGPURegAddr    = psDevInfo->sLayerParams.sGPURegAddr;
+			sTDPowerParams.sBootRemapAddr = psDevInfo->sLayerParams.sBootRemapAddr;
+			sTDPowerParams.sCodeRemapAddr = psDevInfo->sLayerParams.sCodeRemapAddr;
+			sTDPowerParams.sDataRemapAddr = psDevInfo->sLayerParams.sDataRemapAddr;
+		}
+#endif
+
+		eError = psDevConfig->pfnTDSetPowerParams(psDevConfig->hSysData,
+												  &sTDPowerParams);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: TDSetPowerParams not implemented!"));
+		eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	return eError;
+}
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*
+	RGXSystemGetFabricCoherency
+*/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode)
+{
+	IMG_CHAR *aszLabels[] = {"none", "acelite", "fullace", "unknown"};
+	PVRSRV_DEVICE_SNOOP_MODE eAppHintCacheSnoopingMode;
+	PVRSRV_DEVICE_SNOOP_MODE eDeviceCacheSnoopingMode;
+	IMG_UINT32 ui32AppHintFabricCoherency;
+	IMG_UINT32 ui32DeviceFabricCoherency;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+#if !defined(NO_HARDWARE)
+	void *pvRegsBaseKM;
+	IMG_BOOL bPowerDown = IMG_TRUE;
+	PVRSRV_ERROR eError;
+#endif
+
+	if (!sRegsCpuPBase.uiAddr || !ui32RegsSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Invalid RGX register base/size parameters"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(NO_HARDWARE)
+	pvRegsBaseKM = OSMapPhysToLin(sRegsCpuPBase, ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvRegsBaseKM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Failed to create RGX register mapping"));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	bPowerDown = ! PVRSRVIsSystemPowered(psDevConfig->psDevNode);
+
+	/* Power-up the device as required to read the registers */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+	}
+
+	/* AXI support within the SoC, bitfield COHERENCY_SUPPORT [1 .. 0]
+		value NO_COHERENCY        0x0 {SoC does not support any form of Coherency}
+		value ACE_LITE_COHERENCY  0x1 {SoC supports ACE-Lite or I/O Coherency}
+		value FULL_ACE_COHERENCY  0x2 {SoC supports full ACE or 2-Way Coherency} */
+	ui32DeviceFabricCoherency = OSReadHWReg32((void __iomem *)pvRegsBaseKM, RGX_CR_SOC_AXI);
+	PVR_LOG(("AXI fabric coherency (RGX_CR_SOC_AXI): 0x%x", ui32DeviceFabricCoherency));
+#if defined(DEBUG)
+	if (ui32DeviceFabricCoherency & ~((IMG_UINT32)RGX_CR_SOC_AXI_MASKFULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid RGX_CR_SOC_AXI value.", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+	ui32DeviceFabricCoherency &= ~((IMG_UINT32)RGX_CR_SOC_AXI_COHERENCY_SUPPORT_CLRMSK);
+	ui32DeviceFabricCoherency >>= RGX_CR_SOC_AXI_COHERENCY_SUPPORT_SHIFT;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+	}
+
+	/* UnMap Regs */
+	OSUnMapPhysToLin(pvRegsBaseKM, ui32RegsSize);
+
+	switch (ui32DeviceFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_FULLACE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+	default:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_NONE;
+		break;
+	}
+#else /* !defined(NO_HARDWARE) */
+    *peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+    eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+    ui32DeviceFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY;
+#endif /* !defined(NO_HARDWARE) */
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, FabricCoherencyOverride,
+						 &ui32AppHintDefault, &ui32AppHintFabricCoherency);
+	OSFreeAppHintState(pvAppHintState);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	/* Temporarily disable coherency */
+	ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY;
+#endif
+
+	/* Suppress invalid AppHint value */
+	switch (ui32AppHintFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"Invalid FabricCoherencyOverride AppHint %d, ignoring",
+				ui32AppHintFabricCoherency));
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+		break;
+	}
+
+	if (ui32AppHintFabricCoherency < ui32DeviceFabricCoherency)
+	{
+		PVR_LOG(("Downgrading device fabric coherency from %s to %s",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+		eDeviceCacheSnoopingMode = eAppHintCacheSnoopingMode;
+	}
+	else if (ui32AppHintFabricCoherency > ui32DeviceFabricCoherency)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Cannot upgrade device fabric coherency from %s to %s, not supported by device!",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+
+		/* Override requested-for app-hint with actual app-hint value being used */
+		ui32AppHintFabricCoherency = ui32DeviceFabricCoherency;
+	}
+
+	*peCacheSnoopingMode = eDeviceCacheSnoopingMode;
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXSystemHasFBCDCVersion31
+*/
+static IMG_BOOL RGXSystemHasFBCDCVersion31(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(HW_ERN_66622_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 66622))
+#endif
+	{
+		{
+			if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+			{
+				return IMG_TRUE;
+			}
+		}
+	}
+#if defined(HW_ERN_66622_BIT_MASK)
+	else
+	{
+
+#if !defined(NO_HARDWARE)
+		if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: System uses FBCDC3.1 but GPU doesn't support it!",
+			         __func__));
+		}
+#endif
+	}
+#endif /* defined(HW_ERN_66622_BIT_MASK) */
+
+	return IMG_FALSE;
+}
+
+/*
+	RGXGetTFBCLossyGroup
+*/
+static IMG_UINT32 RGXGetTFBCLossyGroup(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	return psDevInfo->ui32TFBCLossyGroup;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return 0;
+#endif
+}
+
+/*
+	RGXDevMMUAttributes
+*/
+static MMU_DEVICEATTRIBS *RGXDevMMUAttributes(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              IMG_BOOL bKernelFWMemoryCtx)
+{
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = NULL;
+
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			psMMUDevAttrs = bKernelFWMemoryCtx ?
+			                psDeviceNode->psFirmwareMMUDevAttrs :
+			                psDeviceNode->psMMUDevAttrs;
+		}
+		else
+#endif
+		{
+			PVR_UNREFERENCED_PARAMETER(bKernelFWMemoryCtx);
+			psMMUDevAttrs = psDeviceNode->psMMUDevAttrs;
+		}
+	}
+
+	return psMMUDevAttrs;
+}
+
+/*
+	RGXDevSnoopMode
+*/
+static PVRSRV_DEVICE_SNOOP_MODE RGXDevSnoopMode(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+	{
+		return PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+	}
+
+	return PVRSRV_DEVICE_SNOOP_NONE;
+}
+
+/*
+ * RGXInitDevPart2
+ */
+PVRSRV_ERROR RGXInitDevPart2(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                             RGX_INIT_APPHINTS	*psApphints)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEV_POWER_STATE	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32			ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32			ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 2");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		RGXPDumpBootldrData(psDeviceNode, psDevInfo);
+	}
+#endif
+#if defined(TIMING) || defined(DEBUG)
+	OSUserModeAccessToPerfCountersEn();
+#endif
+
+	/* Initialise Device Flags */
+	psDevInfo->ui32DeviceFlags = 0;
+	RGXSetDeviceFlags(psDevInfo, psApphints->ui32DeviceFlags, IMG_TRUE);
+
+	/* Allocate DVFS Table (needs to be allocated before GPU trace events
+	 *  component is initialised because there is a dependency between them) */
+	psDevInfo->psGpuDVFSTable = OSAllocZMem(sizeof(*(psDevInfo->psGpuDVFSTable)));
+	PVR_LOG_GOTO_IF_NOMEM(psDevInfo->psGpuDVFSTable, eError, ErrorExit);
+
+	if (psDevInfo->ui32HWPerfHostFilter == 0)
+	{
+		RGXHWPerfHostSetEventFilter(psDevInfo, psApphints->ui32HWPerfHostFilter);
+	}
+
+	/* If HWPerf enabled allocate all resources for the host side buffer. */
+	if (psDevInfo->ui32HWPerfHostFilter != 0)
+	{
+		if (RGXHWPerfHostInitOnDemandResources(psDevInfo) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "HWPerfHost buffer on demand"
+			        " initialisation failed."));
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise work estimation lock */
+		eError = OSLockCreate(&psDevInfo->hWorkEstLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(WorkEstLock)", ErrorExit);
+	}
+#endif
+
+	/* Initialise lists of ZSBuffers */
+	eError = OSLockCreate(&psDevInfo->hLockZSBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockZSBuffer)", ErrorExit);
+	dllist_init(&psDevInfo->sZSBufferHead);
+	psDevInfo->ui32ZSBufferCurrID = 1;
+
+	/* Initialise lists of growable Freelists */
+	eError = OSLockCreate(&psDevInfo->hLockFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockFreeList)", ErrorExit);
+	dllist_init(&psDevInfo->sFreeListHead);
+	psDevInfo->ui32FreelistCurrID = 1;
+
+	eError = OSLockCreate(&psDevInfo->hDebugFaultInfoLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(DebugFaultInfoLock)", ErrorExit);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		eError = OSLockCreate(&psDevInfo->hMMUCtxUnregLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(MMUCtxUnregLock)", ErrorExit);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = OSLockCreate(&psDevInfo->hNMILock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(NMILock)", ErrorExit);
+	}
+#endif
+
+	/* Setup GPU utilisation stats update callback */
+	eError = OSLockCreate(&psDevInfo->hGPUUtilLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(GPUUtilLock)", ErrorExit);
+#if !defined(NO_HARDWARE)
+	psDevInfo->pfnGetGpuUtilStats = RGXGetGpuUtilStats;
+#endif
+
+	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	psDevInfo->eActivePMConf = psApphints->eRGXActivePMConf;
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	/* Validate the SPU mask and initialize to number of SPUs to power up */
+	if ((psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s:Invalid SPU mask (All=0x%X, Non Fused=0x%X). At-least one SPU must to be powered up.",
+		         __func__,
+		         ui32AllPowUnitsMask,
+		         psApphints->ui32AvailablePowUnitsMask));
+		PVR_LOG_GOTO_WITH_ERROR("ui32AvailablePowUnitsMask", eError, PVRSRV_ERROR_INVALID_SPU_MASK, ErrorExit);
+	}
+
+	psDevInfo->ui32AvailablePowUnitsMask = psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask;
+
+	psDevInfo->ui32AvailableRACMask = psApphints->ui32AvailableRACMask & ui32AllRACMask;
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* set-up the Active Power Mgmt callback */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableAPM = psRGXData->psRGXTimingInfo->bEnableActivePM;
+		IMG_BOOL bEnableAPM = ((psApphints->eRGXActivePMConf == RGX_ACTIVEPM_DEFAULT) && bSysEnableAPM) ||
+							   (psApphints->eRGXActivePMConf == RGX_ACTIVEPM_FORCE_ON);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1) && defined(SUPPORT_AUTOVZ)
+		/* The AutoVz driver enables a virtualisation watchdog not compatible with APM */
+		if (bEnableAPM && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Active Power Management disabled in AutoVz mode", __func__));
+			bEnableAPM = IMG_FALSE;
+		}
+
+		PVR_ASSERT(bEnableAPM == IMG_FALSE);
+#endif
+
+		if (bEnableAPM)
+		{
+			eError = OSInstallMISR(&psDevInfo->pvAPMISRData,
+					RGX_MISRHandler_CheckFWActivePowerState,
+					psDeviceNode,
+					"RGX_CheckFWActivePower");
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(APMISR)", ErrorExit);
+
+			/* Prevent the device being woken up before there is something to do. */
+			eDefaultPowerState = PVRSRV_DEV_POWER_STATE_OFF;
+		}
+	}
+#endif
+
+	psDevInfo->eDebugDumpFWTLogType = psApphints->eDebugDumpFWTLogType;
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableAPM,
+	                                    RGXQueryAPMState,
+	                                    RGXSetAPMState,
+	                                    psDeviceNode,
+	                                    NULL);
+
+	RGXTimeCorrInitAppHintCallbacks(psDeviceNode);
+
+	/* Register the device with the power manager */
+	eError = PVRSRVRegisterPowerDevice(psDeviceNode,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPrePowerState : &RGXVzPrePowerState,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPostPowerState : &RGXVzPostPowerState,
+	                                   psDevConfig->pfnPrePowerState, psDevConfig->pfnPostPowerState,
+	                                   &RGXPreClockSpeedChange, &RGXPostClockSpeedChange,
+	                                   &RGXForcedIdleRequest, &RGXCancelForcedIdleRequest, &RGXCancelForcedIdleRequestAsync,
+	                                   &RGXPowUnitsChange,
+	                                   (IMG_HANDLE)psDeviceNode,
+	                                   PVRSRV_DEV_POWER_STATE_OFF,
+	                                   eDefaultPowerState);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterPowerDevice", ErrorExit);
+
+	eError = RGXSetPowerParams(psDevInfo, psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetPowerParams", ErrorExit);
+
+
+#if defined(PDUMP)
+#if defined(NO_HARDWARE)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT, "Wait for the FW to signal idle");
+
+	/* Kick the FW once, in case it still needs to detect and set the idle state */
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+			   RGX_CR_MTS_SCHEDULE,
+			   RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK,
+			   PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                                offsetof(RGXFWIF_SYSDATA, ePowState),
+	                                RGXFWIF_POW_IDLE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", ErrorExit);
+#endif
+
+	/* Run RGXStop with the correct PDump flags to feed the last-frame deinit buffer */
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT,
+	                      "RGX deinitialisation commands");
+
+	psDevInfo->sLayerParams.ui32PdumpFlags |= PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW;
+
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXStop(&psDevInfo->sLayerParams);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXStop", ErrorExit);
+	}
+
+	psDevInfo->sLayerParams.ui32PdumpFlags &= ~(PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW);
+#endif
+
+#if !defined(NO_HARDWARE)
+	eError = RGXInstallProcessQueuesMISR(&psDevInfo->hProcessQueuesMISR, psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInstallProcessQueuesMISR", ErrorExit);
+
+	/* Register RGX to receive notifies when other devices complete some work */
+	PVRSRVRegisterCmdCompleteNotify(&psDeviceNode->hCmdCompNotify, &RGXScheduleProcessQueuesKM, psDeviceNode);
+
+	/* Register the interrupt handlers */
+	eError = OSInstallMISR(&psDevInfo->pvMISRData,
+			RGX_MISRHandler_Main,
+			psDeviceNode,
+			"RGX_Main");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(MISR)", ErrorExit);
+
+	/* Register appropriate mechanism for clearing hw interrupts */
+	eError = RGXSetAckIrq(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetAckIrq", ErrorExit);
+
+	eError = SysInstallDeviceLISR(psDevConfig->hSysData,
+								  psDevConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  RGX_LISRHandler,
+								  psDeviceNode,
+								  &psDevInfo->pvLISRData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysInstallDeviceLISR", ErrorExit);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(PDUMP)
+    if (PVRSRVSystemSnoopingOfCPUCache(psDevConfig))
+    {
+        PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+                              "System has CPU cache snooping");
+    }
+    else
+    {
+        PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+                              "System has NO cache snooping");
+    }
+#endif
+
+#if defined(RGX_FEATURE_COMPUTE_ONLY_BIT_MASK)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE_ONLY))
+#endif
+	{
+		eError = PVRSRVTQLoadShaders(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVTQLoadShaders", ErrorExit);
+	}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	eError = OSAllocateSecBuf(psDeviceNode, RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE, "SharedSecMem", &psDevInfo->psGenHeapSecMem);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSAllocateSecBuf", ErrorExit);
+#endif
+
+	psDevInfo->bDevInit2Done = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	DevPart2DeInitRGX(psDeviceNode);
+
+	return eError;
+}
+
+#define VZ_RGX_FW_FILENAME_SUFFIX ".vz"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#define RGX_64K_FW_FILENAME_SUFFIX ".64k"
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX) + sizeof(RGX_64K_FW_FILENAME_SUFFIX)))
+#else
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX)))
+#endif
+
+static void _GetFWFileName(PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_CHAR *pszFWFilenameStr,
+		IMG_CHAR *pszFWpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const IMG_CHAR * const pszFWFilenameSuffix =
+			PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) ? "" : VZ_RGX_FW_FILENAME_SUFFIX;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	const IMG_CHAR * const pszFWFilenameSuffix2 =
+			((OSGetPageSize() == RGX_MMU_PAGE_SIZE_64KB) &&
+			 RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+			? RGX_64K_FW_FILENAME_SUFFIX : "";
+#else
+	const IMG_CHAR * const pszFWFilenameSuffix2 = "";
+#endif
+
+	OSSNPrintf(pszFWFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STR_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+
+	OSSNPrintf(pszFWpFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STRP_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+}
+
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW)
+{
+	IMG_CHAR aszFWFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR aszFWpFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR *pszLoadedFwStr;
+	PVRSRV_ERROR eErr;
+
+	/* Prepare the image filenames to use in the following code */
+	_GetFWFileName(psDeviceNode, aszFWFilenameStr, aszFWpFilenameStr);
+
+	/* Get pointer to Firmware image */
+	pszLoadedFwStr = aszFWFilenameStr;
+	eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+	if (eErr == PVRSRV_ERROR_NOT_FOUND)
+	{
+		pszLoadedFwStr = aszFWpFilenameStr;
+		eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+		if (eErr == PVRSRV_ERROR_NOT_FOUND)
+		{
+			pszLoadedFwStr = RGX_FW_FILENAME;
+			eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+			if (eErr == PVRSRV_ERROR_NOT_FOUND)
+			{
+				PVR_DPF((PVR_DBG_FATAL, "All RGX Firmware image loads failed for '%s' (%s)",
+						aszFWFilenameStr, PVRSRVGetErrorString(eErr)));
+			}
+		}
+	}
+
+	if (eErr == PVRSRV_OK)
+	{
+		PVR_LOG(("RGX Firmware image '%s' loaded", pszLoadedFwStr));
+	}
+
+	return eErr;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return PVRSRV_OK;
+}
+#endif
+
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* set up fw memory contexts */
+	PVRSRV_RGXDEV_INFO   *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	IMG_BOOL  bNativeFwUMAHeap = PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) &&
+	                             (PhysHeapGetType(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM]) == PHYS_HEAP_TYPE_UMA);
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)) && (!psDeviceNode->bAutoVzFwIsUp) && (!bNativeFwUMAHeap))
+	{
+		PHYS_HEAP *psFwPageTableHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_GOTO_IF_INVALID_PARAM((psFwPageTableHeap != NULL),
+		                              eError, failed_to_create_ctx);
+
+		/* Temporarily swap the MMU and default GPU physheap to allow the page
+		 * tables of all memory mapped by the FwKernel context to be placed
+		 * in a dedicated memory carveout. This should allow the firmware mappings to
+		 * persist after a Host kernel crash or driver reset. */
+		psDeviceNode->psMMUPhysHeap = psFwPageTableHeap;
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+	/* Set the device fabric coherency before FW context creation */
+	eError = RGXSystemGetFabricCoherency(psDevConfig,
+										 psDevConfig->sRegsCpuPBase,
+										 psDevConfig->ui32RegsSize,
+										 &psDeviceNode->eDevFabricType,
+										 &psDevConfig->eCacheSnoopingMode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed RGXSystemGetFabricCoherency (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+#endif
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	RGXFwSharedMemCheckSnoopMode(psDevConfig);
+
+	/* Create the memory context for the firmware. */
+	eError = DevmemCreateContext(psDeviceNode, DEVMEM_HEAPCFG_FORFW,
+	                             &psDevInfo->psKernelDevmemCtx);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemCreateContext (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_MAIN_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareMainHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_CONFIG_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareConfigHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+#if (defined(RGX_PREMAP_FW_HEAPS)) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_CHAR szHeapName[RA_MAX_NAME_LENGTH];
+
+			OSSNPrintf(szHeapName, sizeof(szHeapName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+			eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, szHeapName,
+										  &psDevInfo->psPremappedFwRawHeap[ui32DriverID]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "DevmemFindHeapByName", failed_to_find_heap);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && !bNativeFwUMAHeap)
+	{
+		IMG_DEV_PHYADDR sPhysHeapBase;
+		IMG_UINT32 ui32DriverID;
+		void *pvAppHintState = NULL;
+		IMG_UINT64 ui64DefaultHeapStride;
+		IMG_UINT64 ui64GuestHeapDevBaseStride;
+
+		OSCreateAppHintState(&pvAppHintState);
+		ui64DefaultHeapStride = PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+		OSGetAppHintUINT64(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							GuestFWHeapStride,
+							&ui64DefaultHeapStride,
+							&ui64GuestHeapDevBaseStride);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		eError = PhysHeapGetDevPAddr(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM], &sPhysHeapBase);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", failed_to_find_heap);
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_DEV_PHYADDR sRawFwHeapBase = {sPhysHeapBase.uiAddr + (ui32DriverID * ui64GuestHeapDevBaseStride)};
+
+			eError = RGXFwRawHeapAllocMap(psDeviceNode,
+										  ui32DriverID,
+										  sRawFwHeapBase,
+										  RGX_FIRMWARE_RAW_HEAP_SIZE);
+			if (eError != PVRSRV_OK)
+			{
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+				}
+				PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", failed_to_find_heap);
+			}
+		}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+		/* restore default Px setup */
+		psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* On setups with dynamically mapped Guest heaps, the Guest makes
+	 * a PVZ call to the Host to request the mapping during init. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PvzClientMapDevPhysHeap(psDevConfig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PvzClientMapDevPhysHeap", failed_to_find_heap);
+	}
+#endif /* !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+
+failed_to_find_heap:
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+	DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+	psDevInfo->psKernelDevmemCtx = NULL;
+failed_to_create_ctx:
+	return eError;
+}
+
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR       eError;
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(RGX_PREMAP_FW_HEAPS)
+		psDeviceNode->psMMUPhysHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		if (!psDeviceNode->bAutoVzFwIsUp)
+#endif
+		{
+			IMG_UINT32 ui32DriverID;
+
+			FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+			{
+				RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+			}
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		(void) PvzClientUnmapDevPhysHeap(psDeviceNode->psDevConfig);
+
+		if (psDevInfo->psFirmwareMainHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_FALSE);
+		}
+		if (psDevInfo->psFirmwareConfigHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_FALSE);
+		}
+	}
+#endif
+
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+
+	if (psDevInfo->psKernelDevmemCtx)
+	{
+		eError = DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+}
+
+static PVRSRV_ERROR RGXAlignmentCheck(PVRSRV_DEVICE_NODE *psDevNode,
+                                      IMG_UINT32 ui32AlignChecksSizeUM,
+                                      IMG_UINT32 aui32AlignChecksUM[])
+{
+	static const IMG_UINT32 aui32AlignChecksKM[] = {RGXFW_ALIGN_CHECKS_INIT_KM};
+	IMG_UINT32 ui32UMChecksOffset = ARRAY_SIZE(aui32AlignChecksKM) + 1;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32 i, *paui32FWAlignChecks;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Skip the alignment check if the driver is guest
+	   since there is no firmware to check against */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, eError);
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW Alignment Check Mem Descriptor is NULL",
+		         __func__));
+		return PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc,
+	                                  (void **) &paui32FWAlignChecks);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire kernel address for alignment checks (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	paui32FWAlignChecks += ui32UMChecksOffset;
+	/* Invalidate the size value, check the next region size (UM) and invalidate */
+	RGXFwSharedMemCacheOpPtr(paui32FWAlignChecks, INVALIDATE);
+	if (*paui32FWAlignChecks++ != ui32AlignChecksSizeUM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Mismatching sizes of RGXFW_ALIGN_CHECKS_INIT"
+		         " array between UM(%d) and FW(%d)",
+		         __func__,
+		         ui32AlignChecksSizeUM,
+		         *paui32FWAlignChecks));
+		eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		goto return_;
+	}
+
+	RGXFwSharedMemCacheOpExec(paui32FWAlignChecks,
+	                          ui32AlignChecksSizeUM * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	for (i = 0; i < ui32AlignChecksSizeUM; i++)
+	{
+		if (aui32AlignChecksUM[i] != paui32FWAlignChecks[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: size/offset mismatch in RGXFW_ALIGN_CHECKS_INIT[%d]"
+					" between UM(%d) and FW(%d)",
+					__func__, i, aui32AlignChecksUM[i], paui32FWAlignChecks[i]));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		}
+	}
+
+	if (eError == PVRSRV_ERROR_INVALID_ALIGNMENT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Check for FW/KM structure"
+				" alignment failed.", __func__));
+	}
+
+return_:
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXAllocateFWMemoryRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_DEVMEM_SIZE_T ui32Size,
+                                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags,
+                                       const IMG_PCHAR pszText,
+                                       DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Align = OSGetPageShift();
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(SUPPORT_MIPS_CONTIGUOUS_FW_MEMORY)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		uiLog2Align = RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiLog2Align);
+#endif
+
+	uiMemAllocFlags = (uiMemAllocFlags |
+					   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+					   PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &
+	                   RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	uiMemAllocFlags &= PVRSRV_MEMALLOCFLAGS_TDFWMASK;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate FW %s memory", pszText);
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										ui32Size,
+										1ULL << uiLog2Align,
+										uiMemAllocFlags,
+										pszText,
+										ppsMemDescPtr);
+
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver
+
+ @Description
+
+ Validate the FW build options against KM driver build options (KM build options only)
+
+ Following check is redundant, because next check checks the same bits.
+ Redundancy occurs because if client-server are build-compatible and client-firmware are
+ build-compatible then server-firmware are build-compatible as well.
+
+ This check is left for clarity in error messages if any incompatibility occurs.
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+
+	IMG_UINT32			ui32BuildOptions, ui32BuildOptionsFWKMPart, ui32BuildOptionsMismatch;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+	IMG_UINT8               ui8FwOsCount;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32BuildOptions = (RGX_BUILD_OPTIONS_KM & RGX_BUILD_OPTIONS_MASK_FW);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32BuildOptionsFWKMPart = psFwOsInit->sRGXCompChecks.ui32BuildOptions & RGX_BUILD_OPTIONS_MASK_FW;
+
+		ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+		if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32BuildOptionsFWKMPart = psFWInfoHeader->ui32Flags & RGX_BUILD_OPTIONS_MASK_FW;
+
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo) && BITMASK_HAS(psFWInfoHeader->ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psFWInfoHeader->ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+				return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+			}
+		}
+	}
+
+	/* Check if the FW is missing support for any features required by the driver */
+	if (~ui32BuildOptionsFWKMPart & ui32BuildOptions)
+	{
+		ui32BuildOptionsMismatch = ui32BuildOptions ^ ui32BuildOptionsFWKMPart;
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask non-critical options out as we do support combining them in UM & KM */
+		ui32BuildOptionsMismatch &= FW_OPTIONS_STRICT;
+#endif
+		if ((ui32BuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and KM driver build options; "
+					 "extra options present in the KM driver: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptions & ui32BuildOptionsMismatch));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+
+		if ((ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware-side and KM driver build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch ));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "RGXDevInitCompatCheck: Firmware and KM driver build options differ."));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: Firmware and KM driver build options match. [ OK ]"));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK version against driver DDK version
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32			ui32KMDDKVersion;
+	IMG_UINT32			ui32FWDDKVersion;
+	PVRSRV_ERROR		eError;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32KMDDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKVersion = psFwOsInit->sRGXCompChecks.ui32DDKVersion;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+		ui32FWDDKVersion = PVRVERSION_PACK(psFWInfoHeader->ui16PVRVersionMajor, psFWInfoHeader->ui16PVRVersionMinor);
+	}
+
+	if (ui32FWDDKVersion != ui32KMDDKVersion)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Incompatible driver DDK version (%u.%u) / Firmware DDK version (%u.%u).",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_UNPACK_MAJ(ui32FWDDKVersion),
+				 PVRVERSION_UNPACK_MIN(ui32FWDDKVersion)));
+		eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK version (%u.%u) and Firmware DDK version (%u.%u) match. [ OK ]",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK build against driver DDK build
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32KMDDKBuild;
+	IMG_UINT32			ui32FWDDKBuild;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	ui32KMDDKBuild = PVRVERSION_BUILD;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKBuild = psFwOsInit->sRGXCompChecks.ui32DDKBuild;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32FWDDKBuild = psFWInfoHeader->ui32PVRVersionBuild;
+	}
+
+	if (ui32FWDDKBuild != ui32KMDDKBuild)
+	{
+		PVR_LOG(("(WARN) RGXDevInitCompatCheck: Different driver DDK build version (%d) / Firmware DDK build version (%d).",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		eError = PVRSRV_ERROR_DDK_BUILD_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK build version (%d) and Firmware DDK build version (%d) match. [ OK ]",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+	}
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_BVNC_FWAgainstDriver
+
+ @Description
+
+ Validate FW BVNC against driver BVNC
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_BVNC_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+
+	RGX_FW_INFO_HEADER	*psFWInfoHeader;
+	IMG_UINT64			ui64KMBVNC;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+	ui64KMBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+							   psDevInfo->sDevFeatureCfg.ui32V,
+							   psDevInfo->sDevFeatureCfg.ui32N,
+							   psDevInfo->sDevFeatureCfg.ui32C);
+
+	if (ui64KMBVNC != psFWInfoHeader->ui64BVNC)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in KM driver BVNC (%u.%u.%u.%u) and Firmware BVNC (%u.%u.%u.%u)",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_ERROR_BVNC_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: KM driver BVNC (%d.%d.%d.%d) and FW BVNC (%d.%d.%d.%d) match. [ OK ]",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevInitCompatCheck
+
+ @Description
+
+ Check compatibility of host driver and firmware (DDK and build options)
+ for RGX devices at services/device initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32			ui32FwTimeout = MAX_HW_TIME_US;
+
+		LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Flush covers this instance and the reads in the functions below */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                               INVALIDATE);
+		if (!*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+		{
+			eError = PVRSRV_ERROR_TIMEOUT;
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU Firmware not responding: failed to supply compatibility info (%u)",
+					__func__, eError));
+
+			PVR_DPF((PVR_DBG_ERROR, "%s: Potential causes: firmware not initialised or the current Guest driver's "
+									"OsConfig initialisation data was not accepted by the firmware", __func__));
+			goto chk_exit;
+		}
+	}
+#endif /* defined(NO_HARDWARE) */
+
+	eError = RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDevInitCompatCheck_BVNC_FWAgainstDriver(psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			goto chk_exit;
+		}
+	}
+
+	eError = PVRSRV_OK;
+chk_exit:
+
+	return eError;
+}
+
+static void _RGXSoftResetToggle(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT64  ui64ResetValue1,
+                                IMG_UINT64  ui64ResetValue2)
+{
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, ui64ResetValue1);
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
+}
+
+/**************************************************************************/ /*!
+@Function       RGXSoftReset
+@Description    Resets some modules of the RGX device
+@Input          psDeviceNode		Device node
+@Input          ui64ResetValue1 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET
+                                register).
+@Input          ui64ResetValue2 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET2
+                                register).
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT64  ui64ResetValue1,
+                                 IMG_UINT64  ui64ResetValue2)
+{
+	PVRSRV_RGXDEV_INFO        *psDevInfo;
+	IMG_BOOL	bSoftReset = IMG_FALSE;
+	IMG_UINT64	ui64SoftResetMask = 0;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+	if (((ui64ResetValue1 & ui64SoftResetMask) != ui64ResetValue1) || bSoftReset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Set in soft-reset */
+	_RGXSoftResetToggle(psDevInfo, ui64ResetValue1, ui64ResetValue2);
+
+	/* Take the modules out of reset... */
+	_RGXSoftResetToggle(psDevInfo, 0, 0);
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static const RGX_MIPS_ADDRESS_TRAMPOLINE sNullTrampoline;
+
+static void RGXFreeTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+			psDevInfo->psTrampoline->hPdumpPages,
+#endif
+			&psDevInfo->psTrampoline->sPages);
+
+	if (psDevInfo->psTrampoline != &sNullTrampoline)
+	{
+		OSFreeMem(psDevInfo->psTrampoline);
+	}
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+}
+
+#define RANGES_OVERLAP(x,y,size) (x < (y+size) && y < (x+size))
+#define TRAMPOLINE_ALLOC_MAX_RETRIES (3)
+
+static PVRSRV_ERROR RGXAllocTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT32 i, j;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_MIPS_ADDRESS_TRAMPOLINE *pasTrampoline[TRAMPOLINE_ALLOC_MAX_RETRIES];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate pages for trampoline");
+
+	/* Retry the allocation of the trampoline block (16KB), retaining any
+	 * previous allocations overlapping  with the target range until we get an
+	 * allocation that doesn't overlap with the target range.
+	 * Any allocation like this will require a maximum of 3 tries as we are
+	 * allocating a physical contiguous block of memory, not individual pages.
+	 * Free the unused allocations at the end only after the desired range
+	 * is obtained to prevent the alloc function from returning the same bad
+	 * range repeatedly.
+	 */
+	for (i = 0; i < TRAMPOLINE_ALLOC_MAX_RETRIES; i++)
+	{
+		pasTrampoline[i] = OSAllocMem(sizeof(RGX_MIPS_ADDRESS_TRAMPOLINE));
+		eError = DevPhysMemAlloc(psDeviceNode,
+				RGXMIPSFW_TRAMPOLINE_SIZE,
+				RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE,
+				0,         // (init) u8Value
+				IMG_FALSE, // bInitPage,
+#if defined(PDUMP)
+				psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				"TrampolineRegion",
+				&pasTrampoline[i]->hPdumpPages,
+#endif
+				PVR_SYS_ALLOC_PID,
+				&pasTrampoline[i]->sPages,
+				&pasTrampoline[i]->sPhysAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s failed (%u)",
+			         __func__,
+			         eError));
+			goto fail;
+		}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		/* Set the persistent uiOSid value so that we free from the correct
+		 * base arena when unloading the driver and freeing the trampoline.
+		 */
+		pasTrampoline[i]->sPages.uiOSid = 0;	/* Firmware global arena */
+#endif
+
+		if (!RANGES_OVERLAP(pasTrampoline[i]->sPhysAddr.uiAddr,
+				RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+				RGXMIPSFW_TRAMPOLINE_SIZE))
+		{
+			break;
+		}
+	}
+	if (TRAMPOLINE_ALLOC_MAX_RETRIES == i)
+	{
+		/* Failed to find a physical allocation after 3 attempts */
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s failed to allocate non-overlapping pages (%u)",
+				__func__, eError));
+		/* Fall through, clean up and return error. */
+	}
+	else
+	{
+		/* Remember the last physical block allocated, it will not be freed */
+		psDevInfo->psTrampoline = pasTrampoline[i];
+	}
+
+fail:
+	/* free all unused allocations */
+	for (j = 0; j < i; j++)
+	{
+		DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+				pasTrampoline[j]->hPdumpPages,
+#endif
+				&pasTrampoline[j]->sPages);
+		OSFreeMem(pasTrampoline[j]);
+	}
+
+	return eError;
+}
+
+#undef RANGES_OVERLAP
+#endif
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    uiFWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR        eError;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEVMEM_SIZE_T	uiDummyLen;
+	DEVMEM_MEMDESC		*psDummyMemDesc = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		eError = RGXAllocTrampoline(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Failed to allocate trampoline region (%u)",
+					eError));
+			goto failTrampolineMemDescAlloc;
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW code section
+	 */
+	uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+	                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE);
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWCodeLen,
+	                                   uiMemAllocFlags,
+	                                   "FwExCodeRegion",
+	                                   &psDevInfo->psRGXFWCodeMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc,
+	                                  &psDevInfo->sFWCodeDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAqDevVirt;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) || (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))))
+#endif
+	{
+		/*
+		 * The FW code must be the first allocation in the firmware heap, otherwise
+		 * the bootloader will not work (the FW will not be able to find the bootloader).
+		 */
+		PVR_ASSERT(psDevInfo->sFWCodeDevVAddrBase.uiAddr == RGX_FIRMWARE_RAW_HEAP_BASE);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/*
+		 * Allocate Dummy Pages so that Data segment allocation gets the same
+		 * device virtual address as specified in MIPS firmware linker script
+		 */
+		uiDummyLen = RGXGetFWImageSectionMaxSize(NULL, MIPS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_EXCEPTIONS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_BOOT_CODE) -
+				uiFWCodeLen; /* code actual size */
+
+		if (uiDummyLen > 0)
+		{
+			eError = DevmemFwAllocateExportable(psDeviceNode,
+					uiDummyLen,
+					OSGetPageSize(),
+					uiMemAllocFlags,
+					"FwExDummyPages",
+					&psDummyMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "Failed to allocate fw dummy mem (%u)",
+				         eError));
+				goto failDummyMemDescAlloc;
+			}
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW data section
+	 */
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWDataLen,
+	                                   RGX_FWCODEDATA_ALLOCFLAGS |
+	                                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA),
+	                                   "FwExDataRegion",
+	                                   &psDevInfo->psRGXFWDataMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+	                                  &psDevInfo->sFWDataDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAqDevVirt;
+	}
+
+	if (uiFWCorememCodeLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem code section
+		 */
+		uiMemAllocFlags = (RGX_FWCODEDATA_ALLOCFLAGS |
+		                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE)) &
+		                   ~PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE;
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+		                                   uiFWCorememCodeLen,
+		                                   uiMemAllocFlags,
+		                                   "FwExCorememCodeRegion",
+		                                   &psDevInfo->psRGXFWCorememCodeMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem code mem, size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememCodeLen, uiMemAllocFlags, eError));
+			goto failFWCorememCodeMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc,
+		                                  &psDevInfo->sFWCorememCodeDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem code (%u)",
+			         eError));
+			goto failFWCorememCodeMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememCodeFWAddr,
+		                      psDevInfo->psRGXFWCorememCodeMemDesc,
+		                      0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", failFWCorememCodeMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememCodeDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememCodeFWAddr.ui32Addr = 0;
+	}
+
+	if (uiFWCorememDataLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem data section
+		 */
+		uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+		                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA);
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+				uiFWCorememDataLen,
+				uiMemAllocFlags,
+				"FwExCorememDataRegion",
+				&psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem data mem, "
+			         "size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememDataLen,
+			         uiMemAllocFlags,
+			         eError));
+			goto failFWCorememDataMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				&psDevInfo->sFWCorememDataStoreDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem data (%u)",
+			         eError));
+			goto failFWCorememDataMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememDataStoreFWAddr,
+				psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", failFWCorememDataMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememDataStoreDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememDataStoreFWAddr.ui32Addr = 0;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Free Dummy Pages */
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failFWCorememDataMemDescFwAddr:
+failFWCorememDataMemDescAqDevVirt:
+	if (uiFWCorememDataLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+failFWCorememDataMemDescAlloc:
+failFWCorememCodeMemDescFwAddr:
+failFWCorememCodeMemDescAqDevVirt:
+	if (uiFWCorememCodeLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+failFWCorememCodeMemDescAlloc:
+failFWDataMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+	psDevInfo->psRGXFWDataMemDesc = NULL;
+failFWDataMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+failDummyMemDescAlloc:
+#endif
+failFWCodeMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+	psDevInfo->psRGXFWCodeMemDesc = NULL;
+failFWCodeMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		RGXFreeTrampoline(psDeviceNode);
+	}
+failTrampolineMemDescAlloc:
+#endif
+	return eError;
+}
+
+/*
+	AppHint parameter interface
+ */
+static
+PVRSRV_ERROR RGXFWTraceQueryFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	*pui32Value &= RGXFWIF_LOG_TYPE_GROUP_MASK;
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceQueryLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	if (PVRSRV_OK == eResult)
+	{
+		if (*pui32Value & RGXFWIF_LOG_TYPE_TRACE)
+		{
+			*pui32Value = 0; /* Trace */
+		}
+		else
+		{
+			*pui32Value = 1; /* TBI */
+		}
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 const void *psPrivate,
+                                 IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType;
+
+	eResult = RGXFWTraceQueryLogType(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK == eResult)
+	{
+		if (0 == ui32RGXFWLogType)
+		{
+			BITMASK_SET(ui32Value, RGXFWIF_LOG_TYPE_TRACE);
+		}
+		eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32Value);
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType = ui32Value;
+
+	eResult = RGXFWTraceQueryFilter(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK != eResult)
+	{
+		return eResult;
+	}
+
+	/* 0 - trace, 1 - tbi */
+	if (0 == ui32Value)
+	{
+		BITMASK_SET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	else if (1 == ui32Value)
+	{
+		BITMASK_UNSET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid parameter %u specified to set FW log type AppHint.",
+		         __func__, ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32RGXFWLogType);
+	return eResult;
+}
+
+#if defined(DEBUG)
+static
+PVRSRV_ERROR RGXQueryFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL *pbValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pbValue = (PVRSRV_MEMALLOCFLAG_POISON_ON_FREE == psDevInfo->uiFWPoisonOnFreeFlag)
+		? IMG_TRUE
+		: IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXSetFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL bValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	psDevInfo->uiFWPoisonOnFreeFlag = bValue
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+ * RGXInitFirmware
+ */
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+#if defined(DEBUG)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bEnableFWPoisonOnFree = IMG_FALSE;
+#endif
+
+	eError = RGXSetupFirmware(psDeviceNode,
+	                          psApphints,
+	                          ui32ConfigFlags,
+	                          ui32ConfigFlagsExt,
+	                          ui32FwOsCfgFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXInitFirmwareKM: RGXSetupFirmware failed (%u)",
+		         eError));
+		goto failed_init_firmware;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableLogGroup,
+		                                    RGXFWTraceQueryFilter,
+		                                    RGXFWTraceSetFilter,
+		                                    psDeviceNode,
+		                                    NULL);
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_FirmwareLogType,
+		                                    RGXFWTraceQueryLogType,
+		                                    RGXFWTraceSetLogType,
+		                                    psDeviceNode,
+		                                    NULL);
+	}
+
+#if defined(DEBUG)
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFWPOISONONFREE;
+	OSGetAppHintBOOL(psDeviceNode,
+			pvAppHintState,
+			EnableFWPoisonOnFree,
+			&bAppHintDefault,
+			&bEnableFWPoisonOnFree);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFWPoisonOnFree,
+	                                   RGXQueryFWPoisonOnFree,
+	                                   RGXSetFWPoisonOnFree,
+	                                   psDeviceNode,
+	                                   NULL);
+
+	psDevInfo->uiFWPoisonOnFreeFlag = bEnableFWPoisonOnFree
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+#else
+	psDevInfo->uiFWPoisonOnFreeFlag = 0ULL;
+#endif
+
+	psDevInfo->ui32ClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	psDevInfo->ui32LastClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+
+	return PVRSRV_OK;
+
+failed_init_firmware:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* See device.h for function declaration */
+static PVRSRV_ERROR RGXAllocUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+									 IMG_UINT32 ui32RequestedSize,
+									 DEVMEM_MEMDESC **psMemDesc,
+									 IMG_UINT32 *puiSyncPrimVAddr,
+									 IMG_UINT32 *puiSyncPrimBlockSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR pFirmwareAddr;
+	IMG_DEVMEM_ALIGN_T uiUFOBlockAlign = MAX(sizeof(IMG_UINT32), sizeof(SYNC_CHECKPOINT_FW_OBJ));
+	IMG_DEVMEM_SIZE_T uiUFOBlockSize = PVR_ALIGN(ui32RequestedSize, uiUFOBlockAlign);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Size and align are 'expanded' because we request an Exportalign allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiUFOBlockSize,
+	                                             &uiUFOBlockAlign);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										uiUFOBlockSize,
+										uiUFOBlockAlign,
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_UNCACHED,
+										"FwExUFOBlock",
+										psMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = RGXSetFirmwareAddress(&pFirmwareAddr, *psMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = pFirmwareAddr.ui32Addr;
+	*puiSyncPrimBlockSize = TRUNCATE_64BITS_TO_32BITS(uiUFOBlockSize);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, *psMemDesc);
+e0:
+	return eError;
+}
+
+/* See device.h for function declaration */
+static void RGXFreeUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+							DEVMEM_MEMDESC *psMemDesc)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	RGXUnsetFirmwareAddress(psMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psMemDesc);
+}
+
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	psDevInfo->bDevInit2Done = IMG_FALSE;
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	if (psDevInfo->psGenHeapSecMem != NULL)
+	{
+		OSFreeSecBuf(psDevInfo->psGenHeapSecMem);
+	}
+#endif
+
+#if defined(RGX_FEATURE_COMPUTE_ONLY_BIT_MASK)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE_ONLY))
+#endif
+	{
+		if ((psDevInfo->hTQUSCSharedMem != NULL) &&
+		    (psDevInfo->hTQCLISharedMem != NULL))
+		{
+			PVRSRVTQUnloadShaders(psDeviceNode);
+		}
+	}
+
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvLISRData != NULL)
+	{
+		(void) SysUninstallDeviceLISR(psDevInfo->pvLISRData);
+	}
+	if (psDevInfo->pvMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvMISRData);
+	}
+	if (psDevInfo->hProcessQueuesMISR != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->hProcessQueuesMISR);
+	}
+	if (psDevInfo->pvAPMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvAPMISRData);
+	}
+	if (psDeviceNode->hCmdCompNotify != NULL)
+	{
+		/* Cancel notifications to this device */
+		PVRSRVUnregisterCmdCompleteNotify(psDeviceNode->hCmdCompNotify);
+		psDeviceNode->hCmdCompNotify = NULL;
+	}
+#endif /* !NO_HARDWARE */
+
+	/* Remove the device from the power manager */
+	PVRSRVRemovePowerDevice(psDeviceNode);
+
+	psDevInfo->pfnGetGpuUtilStats = NULL;
+	if (psDevInfo->hGPUUtilLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hGPUUtilLock);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(psDevInfo->hNMILock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hNMILock);
+	}
+#endif
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(psDevInfo->hMMUCtxUnregLock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	if (psDevInfo->hDebugFaultInfoLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hDebugFaultInfoLock);
+	}
+
+	/* De-init Freelists/ZBuffers... */
+	if (psDevInfo->hLockFreeList != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockFreeList);
+	}
+
+	if (psDevInfo->hLockZSBuffer != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockZSBuffer);
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* De-init work estimation lock */
+		if (psDevInfo->hWorkEstLock != NULL)
+		{
+			OSLockDestroy(psDevInfo->hWorkEstLock);
+		}
+	}
+#endif
+
+	/* Free DVFS Table */
+	if (psDevInfo->psGpuDVFSTable != NULL)
+	{
+		OSFreeMem(psDevInfo->psGpuDVFSTable);
+		psDevInfo->psGpuDVFSTable = NULL;
+	}
+}
+
+/*
+	DevDeInitRGX
+ */
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO		*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	PVRSRV_ERROR			eError;
+	DEVICE_MEMORY_INFO		*psDevMemoryInfo;
+
+	if (!psDevInfo)
+	{
+		/* Can happen if DevInitRGX failed */
+		PVR_DPF((PVR_DBG_ERROR, "DevDeInitRGX: Null DevInfo"));
+		return PVRSRV_OK;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	RGXUnregisterBridges(psDevInfo);
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	OSLockDestroy(psDevInfo->hCounterDumpingLock);
+#endif
+
+	/* Unregister debug request notifiers first as they could depend on anything. */
+
+	RGXDebugDeinit(psDevInfo);
+
+	/* De-initialise in reverse order, so stage 2 init is undone first. */
+	if (psDevInfo->bDevInit2Done)
+	{
+		DevPart2DeInitRGX(psDeviceNode);
+	}
+
+	/* Unregister MMU related stuff */
+	eError = RGXMMUInit_Unregister(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevDeInitRGX: Failed RGXMMUInit_Unregister (0x%x)",
+		         eError));
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* Unregister MMU related stuff */
+		eError = RGXMipsMMUInit_Unregister(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed RGXMipsMMUInit_Unregister (0x%x)",
+			         eError));
+		}
+	}
+#endif
+
+	/* UnMap Regs */
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+						 psDevInfo->ui32RegSize);
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvRegsBaseKM = NULL;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+			(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* undo the VA offset performed in RGXRegisterDevice() to allow the allocation to be unmapped */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM, RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvSecureRegsBaseKM = NULL;
+	}
+#endif
+
+#if 0 /* not required at this time */
+	if (psDevInfo->hTimer)
+	{
+		eError = OSRemoveTimer(psDevInfo->hTimer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed to remove timer"));
+			return eError;
+		}
+		psDevInfo->hTimer = NULL;
+	}
+#endif
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+
+	RGXDeInitHeaps(psDevMemoryInfo, psDeviceNode);
+
+	if (psDevInfo->psRGXFWCodeMemDesc)
+	{
+		/* Free fw code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+		psDevInfo->psRGXFWCodeMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware code memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		if (psDevInfo->psTrampoline->sPages.u.pvHandle)
+		{
+			/* Free trampoline region */
+			PDUMPCOMMENT(psDeviceNode, "Freeing trampoline memory");
+			RGXFreeTrampoline(psDeviceNode);
+		}
+	}
+#endif
+
+	if (psDevInfo->psRGXFWDataMemDesc)
+	{
+		/* Free fw data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW data memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+		psDevInfo->psRGXFWDataMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware data memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+	if (psDevInfo->psRGXFWCorememCodeMemDesc)
+	{
+		/* Free fw core mem code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+	{
+		/* Free fw core mem data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem data store memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+
+	/*
+	   Free the firmware allocations.
+	 */
+	RGXFreeFirmware(psDevInfo);
+
+
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+
+	/* De-initialise non-device specific (TL) users of RGX device memory */
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+		{
+			RGXHWPerfDeinitL2Stream(psDevInfo, i);
+		}
+
+		RGXHWPerfDeinit(psDevInfo);
+	}
+
+	RGXDeInitDestroyFWKernelMemoryContext(psDeviceNode);
+
+	RGXHWPerfHostDeInit(psDevInfo);
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+
+	/* destroy the stalled CCB locks */
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+
+	/* destroy the context list locks */
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+	OSLockDestroy(psDevInfo->hBPLock);
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+
+	/* Free device BVNC string */
+	if (NULL != psDevInfo->sDevFeatureCfg.pszBVNCString)
+	{
+		OSFreeMem(psDevInfo->sDevFeatureCfg.pszBVNCString);
+	}
+
+
+	/* DeAllocate devinfo */
+	OSFreeMem(psDevInfo);
+
+	psDeviceNode->pvDevice = NULL;
+
+	return PVRSRV_OK;
+}
+
+#if defined(PDUMP)
+static
+PVRSRV_ERROR RGXResetPDump(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/* Takes a log2 page size parameter and calculates a suitable page size
+ * for the RGX heaps. Returns 0 if parameter is wrong.*/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize)
+{
+	IMG_BOOL bFound = IMG_FALSE;
+	IMG_UINT32 ui32PageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	/* OS page shift must be at least RGX_HEAP_4KB_PAGE_SHIFT,
+	 * max RGX_HEAP_2MB_PAGE_SHIFT, non-zero and a power of two*/
+	if (uiLog2PageSize == 0U ||
+	    (uiLog2PageSize < RGX_HEAP_4KB_PAGE_SHIFT) ||
+	    (uiLog2PageSize > RGX_HEAP_2MB_PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Provided incompatible log2 page size %u",
+				__func__,
+				uiLog2PageSize));
+		PVR_ASSERT(0);
+		return 0;
+	}
+
+	do
+	{
+		if ((IMG_PAGE2BYTES32(uiLog2PageSize) & ui32PageSizeMask) == 0)
+		{
+			/* We have to fall back to a smaller device
+			 * page size than given page size because there
+			 * is no exact match for any supported size. */
+			uiLog2PageSize -= 1U;
+		}
+		else
+		{
+			/* All good, RGX page size equals given page size
+			 * => use it as default for heaps */
+			bFound = IMG_TRUE;
+		}
+	} while (!bFound);
+
+	return uiLog2PageSize;
+}
+
+/* First 16-bits define possible types */
+#define HEAP_INST_VALUE_MASK     (0xFFFF)
+#define HEAP_INST_DEFAULT_VALUE  (1U)  /* Used to show either the heap is always instantiated by default (pfn = NULL)
+	                                      OR
+	                                      that this is the default configuration of the heap with an Alternative BRN */
+#define HEAP_INST_BRN_DEP_VALUE  (2U)  /* The inclusion of this heap is dependent on the brn being present */
+#define HEAP_INST_FEAT_DEP_VALUE (3U)  /* The inclusion of this heap is dependent on the feature being present */
+#define HEAP_INST_BRN_ALT_VALUE  (4U)  /* This entry is a possible alternative to the default determined by a BRN */
+#define HEAP_INST_FEAT_ALT_VALUE (5U)  /* The entry is a possible alternative to the default determined by a Feature define */
+
+/* Latter 16-bits define other flags we may need */
+#define HEAP_INST_NON4K_FLAG     (1 << 16U) /* This is a possible NON4K Entry and we should use the device
+                                               NON4K size when instantiating */
+
+typedef struct RGX_HEAP_INFO_TAG RGX_HEAP_INFO; // Forward declaration
+typedef IMG_BOOL (*PFN_IS_PRESENT)(PVRSRV_RGXDEV_INFO*, const RGX_HEAP_INFO*);
+typedef void (*PFN_HEAP_DYNAMIC)(PVRSRV_DEVICE_NODE*, RGX_HEAP_INFO*);
+
+struct RGX_HEAP_INFO_TAG
+{
+	IMG_CHAR           *pszName;
+	IMG_UINT64         ui64HeapBase;
+	IMG_DEVMEM_SIZE_T  uiHeapLength;
+	IMG_DEVMEM_SIZE_T  uiHeapReservedRegionLength;
+	IMG_UINT32         ui32Log2ImportAlignment;
+	PFN_IS_PRESENT     pfnIsHeapPresent;
+	PFN_HEAP_DYNAMIC   pfnDynamicBaseSize; /* May modify the psHeapInfo's base & length. May be NULL.
+	                                          Only called once if the heap is present otherwise never. */
+	PFN_HEAP_INIT      pfnInit;
+	PFN_HEAP_DEINIT    pfnDeInit;
+	IMG_UINT32         ui32HeapInstanceFlags;
+};
+/* RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES is the total amount of reserved space, to be specified in gasRGXHeapLayoutApp[] */
+#define RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES	(RGX_HEAP_UM_GENERAL_RESERVED_SIZE + RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE)
+
+/* Private heap data struct. */
+typedef struct RGX_HEAP_DATA_TAG
+{
+	DEVMEMINT_RESERVATION  *psMemReservation;
+} RGX_HEAP_DATA;
+
+static PVRSRV_ERROR HeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_HEAP *psDevmemHeap,
+							 PMR *psPMR,
+							 IMG_DEVMEM_SIZE_T uiSize,
+							 IMG_UINT64 ui64Offset,
+							 IMG_BOOL bWriteAble,
+							 IMG_HANDLE *phPrivData)
+{
+	RGX_HEAP_DATA *psHeapData;
+	IMG_DEV_VIRTADDR sCarveOutAddr;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap, "psDevmemHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phPrivData, "phPrivData");
+
+	psHeapData = OSAllocMem(sizeof(*psHeapData));
+	PVR_LOG_RETURN_IF_NOMEM(psHeapData, "psHeapData");
+
+	/* Map the per device secure mem PMR allocation to the general devmem heap carveout. */
+	sCarveOutAddr = DevmemIntHeapGetBaseAddr(psDevmemHeap);
+	sCarveOutAddr.uiAddr += ui64Offset;
+
+	eError = DevmemIntReserveRange(NULL,
+	                               NULL,
+	                               psDevmemHeap,
+	                               sCarveOutAddr,
+	                               uiSize,
+	                               PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                               (bWriteAble ? PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE : 0),
+	                               &psHeapData->psMemReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeHeapData);
+
+	eError = DevmemIntMapPMR(psHeapData->psMemReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	*phPrivData = (IMG_HANDLE)psHeapData;
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+ErrorFreeHeapData:
+	OSFreeMem(psHeapData);
+
+	return eError;
+}
+
+/* Deinit callback function for general heap. */
+static void HeapDeInit(IMG_HANDLE hPrivData)
+{
+	RGX_HEAP_DATA *psHeapData = (RGX_HEAP_DATA*)hPrivData;
+
+	PVR_ASSERT(hPrivData);
+
+	DevmemIntUnmapPMR(psHeapData->psMemReservation);
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+
+	OSFreeMem(psHeapData);
+}
+
+/* Init callback function for general heap. */
+static PVRSRV_ERROR USCHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_FALSE;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	uiSize = PMR_PhysicalSize(psDevInfo->hTQUSCSharedMem);
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->hTQUSCSharedMem,
+					  uiSize,
+					  RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+/* Init callback function for general heap. */
+static PVRSRV_ERROR GeneralHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_TRUE;
+	PVRSRV_ERROR eError;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->psGenHeapSecMem,
+					  RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE,
+					  RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#define GeneralHeapDeInit HeapDeInit
+#else
+/* Callbacks not used */
+#define GeneralHeapInit NULL
+#define GeneralHeapDeInit NULL
+#endif
+
+static void SVMHeapDynamic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           RGX_HEAP_INFO *psHeapInfo)
+{
+	IMG_UINT64 ui64OSPageSize = OSGetPageSize();
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+
+	/* Ensures the SVM heap has the correct alignment & size for any OS page size.
+	 *
+	 * The SVM heap's base must be the smallest possible address mappable by UM.
+	 * This is 32KB unless the page size is larger than 32KB. [1]
+	 * If the page size > 32KB, raise the SVM heap base to the next page boundary.
+	 * Also reduce the length to ensure it's still page aligned and doesn't go
+	 * into another heap.
+	 *
+	 * [1]: https://chromium.googlesource.com/chromium/src/+/fe24932ee14aa93e1fe4d3e7003b9362591a54d4/docs/security/faq.md#why-aren_t-null-pointer-dereferences-considered-security-bugs
+	 */
+	IMG_UINT64 ui64Base = PVR_ALIGN(psHeapInfo->ui64HeapBase, ui64OSPageSize);
+	IMG_UINT64 ui64BaseDiff = ui64Base - psHeapInfo->ui64HeapBase;
+	psHeapInfo->ui64HeapBase = ui64Base;
+	if (psHeapInfo->uiHeapLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapLength -= ui64BaseDiff;
+	if (psHeapInfo->uiHeapReservedRegionLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapReservedRegionLength -= ui64BaseDiff;
+
+	/* The device shared-virtual-memory heap address-space size is stored on the device for
+	   faster look-up without having to walk the device heap configuration structures during
+	   client device connection  (i.e. this size is relative to a zero-based offset) */
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = 0;
+	}
+	else
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = psHeapInfo->ui64HeapBase + psHeapInfo->uiHeapLength;
+	}
+}
+
+/* Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+static IMG_BOOL BRN65273IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		return (((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_ALT_VALUE) ||
+		        ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_DEP_VALUE)) ?
+		        IMG_TRUE : IMG_FALSE;
+	}
+	else
+	{
+		return ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_DEFAULT_VALUE) ? IMG_TRUE : IMG_FALSE;
+	}
+}
+#endif
+
+#if defined(FIX_HW_BRN_63142_BIT_MASK)
+static IMG_BOOL BRN63142IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 63142))
+	{
+		PVR_ASSERT((pksHeapInfo->ui64HeapBase & IMG_UINT64_C(0x3FFFFFFFF)) +
+		            pksHeapInfo->uiHeapLength == IMG_UINT64_C(0x400000000));
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FBCDescriptorIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, FBC_MAX_DEFAULT_DESCRIPTORS))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_BOOL FBCLargeDescriptorIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, FBC_MAX_LARGE_DESCRIPTORS))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_BOOL TextureStateIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+#if defined(RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, BINDLESS_IMAGE_AND_TEXTURE_STATE))
+	{
+		return IMG_TRUE;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+#endif
+	return IMG_FALSE;
+}
+
+/* FW Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+static IMG_BOOL FWBRN65101IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap size and base at run time VS the current table instance
+	 */
+	IMG_UINT64 ui64MainSubHeapSize;
+
+	/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)))
+	{
+		if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101;
+		}
+		else
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL;
+		}
+	}
+	else
+	{
+		ui64MainSubHeapSize = RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE;
+	}
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->uiHeapLength == ui64MainSubHeapSize &&
+	        pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_MAIN_HEAP_BASE) ?
+	        IMG_TRUE : IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FWVZConfigPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap base at run time VS the current table instance
+	 */
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_CONFIG_HEAP_BASE) ? IMG_TRUE : IMG_FALSE;
+}
+
+/* Blueprint array. note: not all heaps are available to clients*/
+
+static RGX_HEAP_INFO gasRGXHeapLayoutApp[] =
+{
+	/* Name                             HeapBase                                 HeapLength                               HeapReservedRegionLength                     Log2ImportAlignment pfnIsHeapPresent             pfnDynamicBaseSize pfnInit           pfnDeInit          HeapInstanceFlags   */
+	{RGX_GENERAL_SVM_HEAP_IDENT,        RGX_GENERAL_SVM_HEAP_BASE,               RGX_GENERAL_SVM_HEAP_SIZE,               0,                                           0,                  NULL,                        SVMHeapDynamic,    NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_HEAP_BASE,                   RGX_GENERAL_HEAP_SIZE,                   RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  BRN65273IsPresent,           NULL,              GeneralHeapInit,  GeneralHeapDeInit, HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_BRN_65273_HEAP_BASE,         RGX_GENERAL_BRN_65273_HEAP_SIZE,         RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_HEAP_BASE,             RGX_GENERAL_NON4K_HEAP_SIZE,             0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_BRN_65273_HEAP_BASE,   RGX_GENERAL_NON4K_BRN_65273_HEAP_SIZE,   0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_HEAP_BASE,               RGX_PDSCODEDATA_HEAP_SIZE,               RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_BRN_65273_HEAP_BASE,     RGX_PDSCODEDATA_BRN_65273_HEAP_SIZE,     RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_RGNHDR_BRN_63142_HEAP_IDENT,   RGX_RGNHDR_BRN_63142_HEAP_BASE,          RGX_RGNHDR_BRN_63142_HEAP_SIZE,          0,                                           0,                  BRN63142IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_HEAP_BASE,                   RGX_USCCODE_HEAP_SIZE,                   RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_DEFAULT_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_BRN_65273_HEAP_BASE,         RGX_USCCODE_BRN_65273_HEAP_SIZE,         RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_BRN_ALT_VALUE },
+	{RGX_TQ3DPARAMETERS_HEAP_IDENT,     RGX_TQ3DPARAMETERS_HEAP_BASE,            RGX_TQ3DPARAMETERS_HEAP_SIZE,            0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TQ3DPARAMETERS_HEAP_IDENT,     RGX_TQ3DPARAMETERS_BRN_65273_HEAP_BASE,  RGX_TQ3DPARAMETERS_BRN_65273_HEAP_SIZE,  0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_VK_CAPT_REPLAY_HEAP_IDENT,     RGX_VK_CAPT_REPLAY_HEAP_BASE,            RGX_VK_CAPT_REPLAY_HEAP_SIZE,            0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_FBCDC_HEAP_IDENT,              RGX_FBCDC_HEAP_BASE,                     RGX_FBCDC_HEAP_SIZE,                     0,                                           0,                  FBCDescriptorIsPresent,      NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_FBCDC_LARGE_HEAP_IDENT,        RGX_FBCDC_LARGE_HEAP_BASE,               RGX_FBCDC_LARGE_HEAP_SIZE,               0,                                           0,                  FBCLargeDescriptorIsPresent, NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_CMP_MISSION_RMW_HEAP_IDENT,    RGX_CMP_MISSION_RMW_HEAP_BASE,           RGX_CMP_MISSION_RMW_HEAP_SIZE,           0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_CMP_SAFETY_RMW_HEAP_IDENT,     RGX_CMP_SAFETY_RMW_HEAP_BASE,            RGX_CMP_SAFETY_RMW_HEAP_SIZE,            0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TEXTURE_STATE_HEAP_IDENT,      RGX_TEXTURE_STATE_HEAP_BASE,             RGX_TEXTURE_STATE_HEAP_SIZE,             0,                                           0,                  TextureStateIsPresent,       NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_HEAP_BASE,           RGX_VISIBILITY_TEST_HEAP_SIZE,           0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_BRN_65273_HEAP_BASE, RGX_VISIBILITY_TEST_BRN_65273_HEAP_SIZE, 0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_MMU_INIA_BRN_65273_HEAP_IDENT, RGX_MMU_INIA_BRN_65273_HEAP_BASE,        RGX_MMU_INIA_BRN_65273_HEAP_SIZE,        0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+	{RGX_MMU_INIB_BRN_65273_HEAP_IDENT, RGX_MMU_INIB_BRN_65273_HEAP_BASE,        RGX_MMU_INIB_BRN_65273_HEAP_SIZE,        0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+};
+
+static RGX_HEAP_INFO gasRGXHeapLayoutFW[] =
+{
+	/* Name                          HeapBase                        HeapLength                                      HeapReservedRegionLength Log2ImportAlignment pfnIsHeapPresent     pfnDynamicBaseSize pfnInit pfnDeInit HeapInstanceFlags*/
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE,            0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL,   0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101, 0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_BRN_ALT_VALUE},
+	{RGX_FIRMWARE_CONFIG_HEAP_IDENT, RGX_FIRMWARE_CONFIG_HEAP_BASE,  RGX_FIRMWARE_CONFIG_HEAP_SIZE,                  0,                       0,                  FWVZConfigPresent,   NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+};
+
+static INLINE IMG_BOOL IsFwHeapLayout(const RGX_HEAP_INFO *psHeapInfo)
+{
+	return psHeapInfo->pszName[0] == 'F' &&
+	       psHeapInfo->pszName[1] == 'w' ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE void CheckHeapAlignment(const RGX_HEAP_INFO *psHeapInfo,
+                                      const PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 uiAlignment = RGX_HEAP_BASE_SIZE_ALIGN - 1;
+
+	if (IsFwHeapLayout(psHeapInfo) ||
+	    RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/*
+		 * 1) Main heap starts at 2MB offset 0xEC10000000UL
+		 * 2) Config Sub heap is created at the end of the main heap making the entire unit start and end at 2MB offset aligned,
+		 *    these 2 heaps will always have the same page size
+		 *    There are no other heaps in between these two heaps, there is no risk for another devmem heap be created between them.
+		 */
+		return;
+	}
+
+	/* General SVM Heap must be placed below 2MiB boundary so we need to adjust
+	 * the validity condition. This is because an OS might return virtual
+	 * addresses below 2MiB threshold. By default (based on testing on Linux)
+	 * this is around 32KiB. */
+	if (OSStringNCompare(psHeapInfo->pszName, RGX_GENERAL_SVM_HEAP_IDENT,
+	                     sizeof(RGX_GENERAL_SVM_HEAP_IDENT)) == 0)
+	{
+		uiAlignment = RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT - 1;
+	}
+
+	/* All UM accessible heap bases should be aligned to 2MB */
+	if (psHeapInfo->ui64HeapBase & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Base not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Base: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Base (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->ui64HeapBase,
+		         psHeapInfo->ui64HeapBase,
+		         uiAlignment + 1));
+	}
+
+	/* All UM accessible heaps should also be size aligned to 2MB */
+	if (psHeapInfo->uiHeapLength & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Size not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Size (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_SIZE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->uiHeapLength,
+		         psHeapInfo->uiHeapLength,
+		         uiAlignment + 1));
+	}
+}
+
+/* Generic counting method. */
+static void _CountRequiredHeaps(PVRSRV_RGXDEV_INFO  *psDevInfo,
+	                            const RGX_HEAP_INFO  pksHeapInfo[],
+	                            IMG_UINT32           ui32HeapListSize,
+	                            IMG_UINT32*          ui32HeapCount)
+{
+	IMG_UINT32 i;
+
+	/* Loop over rows in the heap data array using callback to decide if we
+	 * should include the heap
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo = &pksHeapInfo[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		(*ui32HeapCount)++;
+	}
+}
+/* Generic heap instantiator */
+static void _InstantiateRequiredHeaps(PVRSRV_RGXDEV_INFO     *psDevInfo,
+                                      RGX_HEAP_INFO           psHeapInfos[],
+                                      IMG_UINT32              ui32HeapListSize,
+                                      const IMG_UINT32        ui32Log2RgxDefaultPageShift,
+                                      DEVMEM_HEAP_BLUEPRINT **psDeviceMemoryHeapCursor)
+{
+	IMG_UINT32 i;
+
+#if defined DEBUG
+	IMG_UINT32 ui32heapListCnt;
+	bool bHeapPageSizeMisMatch = false;
+
+	/*
+	 * To ensure all heaps within a 2MB region have the same page sizes
+	 */
+	for (ui32heapListCnt = 0; ui32heapListCnt < (ui32HeapListSize-1); ui32heapListCnt++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo1 = &psHeapInfos[ui32heapListCnt];
+		const RGX_HEAP_INFO *psHeapInfo2 = &psHeapInfos[ui32heapListCnt+1];
+
+		if (((psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) &&
+			((psHeapInfo1->ui64HeapBase + psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) ==
+			((psHeapInfo2->ui64HeapBase) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)))
+		{
+			if (psHeapInfo1->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				if (!(psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG))
+				{
+					bHeapPageSizeMisMatch = true;
+				}
+			}
+			else if (psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				bHeapPageSizeMisMatch = true;
+			}
+		}
+		if (bHeapPageSizeMisMatch)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Two Heap with Different Page Size allocated in the same PD space(2MB)\n"
+			         "Invalid Heaps 1) \"%s\" and 2) \"%s\"",
+			         __func__,
+			         psHeapInfo1->pszName,
+			         psHeapInfo2->pszName));
+		}
+	}
+#endif
+
+	/* We now have a list of the heaps to include and so we should loop over this
+	 * list and instantiate.
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		IMG_UINT32 ui32Log2DataPageSize = 0;
+		RGX_HEAP_INFO *psHeapInfo = &psHeapInfos[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		if (psHeapInfo->pfnDynamicBaseSize != NULL)
+		{
+			psHeapInfo->pfnDynamicBaseSize(psDevInfo->psDeviceNode, psHeapInfo);
+		}
+
+		if (psHeapInfo->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+		{
+			ui32Log2DataPageSize = psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2;
+		}
+		else
+		{
+			ui32Log2DataPageSize = ui32Log2RgxDefaultPageShift;
+		}
+
+		CheckHeapAlignment(psHeapInfo, psDevInfo);
+
+		HeapCfgBlueprintInit(psHeapInfo->pszName,
+		                     psHeapInfo->ui64HeapBase,
+		                     psHeapInfo->uiHeapLength,
+		                     psHeapInfo->uiHeapReservedRegionLength,
+		                     ui32Log2DataPageSize,
+		                     psHeapInfo->ui32Log2ImportAlignment,
+		                     psHeapInfo->pfnInit,
+		                     psHeapInfo->pfnDeInit,
+		                     *psDeviceMemoryHeapCursor);
+
+		(*psDeviceMemoryHeapCursor)++;
+	}
+}
+
+static PVRSRV_ERROR RGXInitHeaps(PVRSRV_RGXDEV_INFO *psDevInfo,
+	                             DEVICE_MEMORY_INFO *psNewMemoryInfo)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor;
+
+	IMG_UINT32 ui32AppHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutApp);
+	IMG_UINT32 ui32FWHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutFW);
+	IMG_UINT32 ui32CountedHeapSize;
+
+	IMG_UINT32 ui32AppHeapCount = 0;
+	IMG_UINT32 ui32FWHeapCount = 0;
+
+	IMG_UINT32 ui32Log2DefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	if (ui32Log2DefaultPageShift == 0)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2DefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2DefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e0;
+		}
+	}
+#endif
+
+	/* Count heaps required for the app heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutApp,
+		                ui32AppHeapListSize,
+		                &ui32AppHeapCount);
+
+	/* Count heaps required for the FW heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutFW,
+		                ui32FWHeapListSize,
+		                &ui32FWHeapCount);
+
+	ui32CountedHeapSize = (ui32AppHeapCount + ui32FWHeapCount + RGX_NUM_DRIVERS_SUPPORTED);
+
+	psNewMemoryInfo->psDeviceMemoryHeap = OSAllocMem(sizeof(DEVMEM_HEAP_BLUEPRINT) * ui32CountedHeapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeap, eError, e0);
+
+	/* Initialise the heaps */
+	psDeviceMemoryHeapCursor = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	/* Instantiate App Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutApp,
+	                          ui32AppHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* Instantiate FW Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutFW,
+	                          ui32FWHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* set the heap count */
+	psNewMemoryInfo->ui32HeapCount = (IMG_UINT32)(psDeviceMemoryHeapCursor - psNewMemoryInfo->psDeviceMemoryHeap);
+
+	/* Check we have allocated the correct # of heaps, minus any VZ heaps as these
+	 * have not been created at this point
+	 */
+	PVR_ASSERT(psNewMemoryInfo->ui32HeapCount == (ui32CountedHeapSize - RGX_NUM_DRIVERS_SUPPORTED));
+
+	/*
+	   In the new heap setup, we initialise 2 configurations:
+		1 - One will be for the firmware only (index 1 in array)
+			a. This primarily has the firmware heap in it.
+			b. It also has additional guest OSID firmware heap(s)
+				- Only if the number of support firmware OSID > 1
+		2 - Others shall be for clients only (index 0 in array)
+			a. This has all the other client heaps in it.
+	 */
+	psNewMemoryInfo->uiNumHeapConfigs = 2;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray = OSAllocMem(sizeof(DEVMEM_HEAP_CONFIG) * psNewMemoryInfo->uiNumHeapConfigs);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeapConfigArray, eError, e1);
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].pszName = "Default Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].uiNumHeaps = psNewMemoryInfo->ui32HeapCount - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].psHeapBlueprintArray = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].pszName = "Firmware Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps = RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].psHeapBlueprintArray = psDeviceMemoryHeapCursor - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4)
+	{
+		IMG_UINT32 i;
+		const IMG_UINT32 ui32GeneralNon4KHeapPageSize = (1 << psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2);
+		const IMG_UINT32 ui32RgxDefaultPageSize = (1 << RGXHeapDerivePageSize(OSGetPageShift()));
+
+		/*
+		 * Initialise all MMU Page Size Range Config register to the default page size
+		 * used by the OS, leaving the address range 0;
+		 */
+		for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[i] =
+					RGXMMUInit_GetConfigRangeValue(ui32RgxDefaultPageSize,
+												   0,
+												   (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+		}
+
+
+		/* set the last MMU config range covering the entire virtual memory to the OS's page size */
+		psDevInfo->aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES - 1] =
+				RGXMMUInit_GetConfigRangeValue(ui32RgxDefaultPageSize, 0, (1ULL << 40));
+
+		/*
+		 * If the Non4K heap has a different page size than the OS's page size
+		 * (used as default for all other heaps), configure one MMU config range
+		 * for the Non4K heap
+		 */
+		if (ui32GeneralNon4KHeapPageSize != ui32RgxDefaultPageSize)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[0] =
+					RGXMMUInit_GetConfigRangeValue(ui32GeneralNon4KHeapPageSize,
+												   RGX_GENERAL_NON4K_HEAP_BASE,
+												   RGX_GENERAL_NON4K_HEAP_SIZE);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		/* Create additional raw firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVINFO, psDevInfo)
+		{
+			eError = RGXInitFwRawHeap(psDevInfo, psDeviceMemoryHeapCursor, ui32DriverID);
+			if (eError != PVRSRV_OK)
+			{
+				/* if any allocation fails, free previously allocated heaps and abandon initialisation */
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+					psDeviceMemoryHeapCursor--;
+				}
+				goto e1;
+			}
+
+			/* Append additional firmware heaps to host driver firmware context heap configuration */
+			psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps += 1;
+
+			/* advance to the next heap */
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	return PVRSRV_OK;
+e1:
+	OSFreeMem(psNewMemoryInfo->psDeviceMemoryHeap);
+e0:
+	return eError;
+}
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+		DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor = psDevMemoryInfo->psDeviceMemoryHeap;
+
+		/* Delete all guest firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeapConfigArray);
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeap);
+}
+
+static PVRSRV_ERROR RGXInitSharedFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psSysHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* VZ heap validation */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg != NULL,
+								"FW Main heap is required for VZ Guest.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+#endif
+
+	if (psSysHeapCfg != NULL)
+	{
+		/* Check FW_SHARED for multiple usage flags. Because FW_SHARED is divided
+		   into subheaps, shared usage with other heaps is not allowed.  */
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_SHARED,
+								"FW_SHARED phys heap config not specified with more than one usage."
+								"FW_SHARED heap must be exclusively used as FW_SHARED.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	if (psSysHeapCfg == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap not set", __func__));
+		/* Nothing to do. Default to the physheap fallback option */
+	}
+	else if (psSysHeapCfg->eType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses OS System memory (UMA)", __func__));
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psSysHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG] = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	}
+	else /* PHYS_HEAP_TYPE_LMA or PHYS_HEAP_TYPE_DMA */
+	{
+		PHYS_HEAP_CONFIG sFwMainHeapCfg, sFwCfgHeapCfg;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses local memory managed by the driver (LMA)", __func__));
+
+
+		/* Subheap layout: Main + (optional MIPS reserved range) + Config */
+		sFwMainHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		/* Reserve space for the Config heap */
+		sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+			/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)))
+			{
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+				if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101;
+				}
+				else
+#endif
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL;
+				}
+			}
+		}
+#endif
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwMainHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwMainHeapCfg,
+		                              "Fw Main subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:MAIN", ErrorDeinit);
+
+		sFwCfgHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		sFwCfgHeapCfg.uConfig.sLMA.sStartAddr.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+		sFwCfgHeapCfg.uConfig.sLMA.sCardBase.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		sFwCfgHeapCfg.uConfig.sLMA.uiSize = RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwCfgHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwCfgHeapCfg,
+		                              "Fw Cfg subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:CFG", ErrorDeinit);
+	}
+
+	/* Acquire FW heaps */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_MAIN, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_MAIN", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CONFIG, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CONFIG", ErrorDeinit);
+
+	return eError;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitPrivateFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psFwCodeHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_CODE);
+	PHYS_HEAP_CONFIG *psFwDataHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIV_DATA);
+	PHYS_HEAP_CONFIG *psFwPrivateHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIVATE);
+	PHYS_HEAP_CONFIG sFwPrivateTempCfg;
+
+	if (psFwPrivateHeapCfg != NULL)
+	{
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg == NULL) && (psFwDataHeapCfg == NULL),
+								"FW_PRIVATE and the FW_CODE & FW_PRIV_DATA usage flags "
+								"achieve the same goal and are mutually exclusive.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Fw code and data are both allocated from this unified heap */
+		sFwPrivateTempCfg = *psFwPrivateHeapCfg;
+		sFwPrivateTempCfg.ui32UsageFlags = PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+
+		psFwCodeHeapCfg = &sFwPrivateTempCfg;
+		psFwDataHeapCfg = &sFwPrivateTempCfg;
+	}
+
+	if ((psFwCodeHeapCfg == NULL) || (psFwDataHeapCfg == NULL))
+	{
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware code and data heaps must be either both defined
+			 * or both undefined. There is no point in isolating one but not
+			 * the other.*/
+			eError = PVRSRV_ERROR_PHYSHEAP_CONFIG;
+			PVR_LOG_GOTO_IF_ERROR(eError, "PrivateFwPhysHeap check", ErrorDeinit);
+		}
+		else
+		{
+			/* No dedicated heaps, default to the physheap fallback option */
+		}
+	}
+	else if (psFwCodeHeapCfg == psFwDataHeapCfg)
+	{
+		if (psFwCodeHeapCfg->ui32UsageFlags ==
+			(PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA))
+		{
+			/* Fw code and private data allocations come from the same system heap
+			 * Instantiate one physheap and share it between them. */
+
+			eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+												  psFwCodeHeapCfg,
+												  NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig");
+		}
+		else
+		{
+			/* Not an exclusive heap, can be used for other purposes (e.g. secure buffers).
+			 * Expect the PVR layer to have already created a heap for the other uses. */
+		}
+	}
+	else
+	{
+		/*
+		 * Separating private Firmware code and data is allowed for backwards compatibility
+		 * purposes. New platforms should use the unified FW_PRIVATE heap instead.
+		 *
+		 * Early security implementations on Rogue cores required separate FW_PRIV_DATA
+		 * and FW_CODE heaps, as access permissions to Firmware were granted differently
+		 * based on the transaction types (code or data).
+		 */
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_CODE) &&
+								(psFwDataHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PRIV_DATA),
+								"Dedicated private heaps for Fw code and "
+								"data must have one usage flag exclusively.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Dedicated Fw code heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwCodeHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		/* Dedicated Fw private data heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwDataHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+	/* When premapping distinct private and shared Firmware phys heaps
+	 * inside the same virtual devmem heap, their sizes must add up to
+	 * the fixed RGX_FIRMWARE_RAW_HEAP_SIZE for the premapping to work */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwSharedHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		PHYS_HEAP_USAGE_FW_SHARED);
+		IMG_UINT64 ui64FwCodeHeapSize = PhysHeapConfigGetSize(psFwCodeHeapCfg);
+		IMG_UINT64 ui64FwDataHeapSize = PhysHeapConfigGetSize(psFwDataHeapCfg);
+		IMG_UINT64 ui64FwSharedHeapSize = PhysHeapConfigGetSize(psFwSharedHeapCfg);
+		IMG_UINT64 ui64FwPrivateHeapSize;
+
+		PVR_LOG_GOTO_IF_FALSE((psFwCodeHeapCfg != NULL) && (psFwDataHeapCfg != NULL),
+							  "Security support requires Fw code and data memory be"
+							  " separate from the heap shared with the kernel driver.", FailDeinit);
+
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware allocations come from 2 different heaps */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize + ui64FwDataHeapSize;
+		}
+		else
+		{
+			/* Private Firmware allocations come from a single heap */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize;
+		}
+
+		PVR_LOG_GOTO_IF_FALSE((ui64FwSharedHeapSize + ui64FwPrivateHeapSize) == RGX_FIRMWARE_RAW_HEAP_SIZE,
+		                      "Invalid firmware physical heap size.", FailDeinit);
+	}
+#endif
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CODE, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CODE]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CODE", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_PRIV_DATA, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PRIV_DATA]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_DATA", ErrorDeinit);
+
+	return eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+FailDeinit:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitFwPageTableHeap(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwPageTableHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		  PHYS_HEAP_USAGE_FW_PREMAP_PT);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg != NULL),
+								"The Firmware Page Table phys heap config not found.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PREMAP_PT),
+								"The Firmware Page Table heap must be used exclusively for this purpose",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_LMA) ||
+								(psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_DMA),
+								"The Firmware Page Table heap must be LMA or DMA memory.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->uConfig.sLMA.uiSize >= RGX_FIRMWARE_MAX_PAGETABLE_SIZE),
+								"The Firmware Page Table heap must be able to hold the maximum "
+								"number of pagetables needed to cover the Firmware's VA space.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwPageTableHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig:FwPageTableHeap");
+
+		eError = PhysHeapAcquire(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquire:FwPageTableHeap");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) */
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXInitFwPageTableHeap(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitFwPageTableHeap", ErrorDeinit);
+	eError = RGXInitSharedFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitSharedFwPhysHeaps", ErrorDeinit);
+	eError = RGXInitPrivateFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitPrivateFwPhysHeaps", ErrorDeinit);
+
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDeviceFWMainHeapMemCheck
+@Description    Checks the free memory in FW Main PhysHeap of a device to ensure
+                there is enough for a connection to be made.
+
+@Input          psDeviceNode    The device of the FW Main PhysHeap to be checked.
+
+@Return         On success PVRSRV_OK, else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDeviceFWMainHeapMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP *psFWMainPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psFWMainPhysHeap = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	if (psFWMainPhysHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's FW Main PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psFWMainPhysHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		const IMG_UINT32 ui32MinMemInKBs = RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION;
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psFWMainPhysHeap,
+		                              KB2B(ui32MinMemInKBs),
+		                              &ui64FreePhysHeapMem);
+
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FW_MAIN PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"KB "
+				"Minimum required: %uKB",
+				B2KB(ui64FreePhysHeapMem),
+				ui32MinMemInKBs));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate, IMG_UINT32 *pui32Log2Non4KPgShift)
+{
+	IMG_UINT32 ui32GeneralNon4KHeapPageSize;
+	IMG_UINT32 uiLog2OSPageShift = OSGetPageShift();
+
+	/* We support Non4K pages only on platforms with 4KB pages. On all platforms
+	 * where OS pages are larger than 4KB we must ensure the non4K device memory
+	 * heap matches the page size used in all other device memory heaps, which
+	 * is the OS page size, see RGXHeapDerivePageSize. */
+	if (uiLog2OSPageShift > RGX_HEAP_4KB_PAGE_SHIFT)
+	{
+		*pui32Log2Non4KPgShift = RGXHeapDerivePageSize(uiLog2OSPageShift);
+	}
+	else
+	{
+		void *pvAppHintState = NULL;
+		IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE;
+		IMG_UINT32 ui32Log2Non4KPgShift;
+
+		/* Get the page size for the dummy page from the NON4K heap apphint */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState,
+		                   GeneralNon4KHeapPageSize, &ui32AppHintDefault,
+		                   &ui32GeneralNon4KHeapPageSize);
+		OSFreeAppHintState(pvAppHintState);
+
+		/* Validate the specified parameter to be one of the supported values */
+		ui32Log2Non4KPgShift = RGXHeapDerivePageSize(ExactLog2(ui32GeneralNon4KHeapPageSize));
+		if (ui32AppHintDefault != ui32GeneralNon4KHeapPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid Non4K Page-size, default=%u, requested=%u,"
+			         " actual=%u, pageshift 0x%x",
+			         __func__, ui32AppHintDefault, ui32GeneralNon4KHeapPageSize,
+			         (1U << ui32Log2Non4KPgShift), ui32Log2Non4KPgShift));
+		}
+
+		if (ui32Log2Non4KPgShift == 0U)
+		{
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+
+		*pui32Log2Non4KPgShift = ui32Log2Non4KPgShift;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 71317))
+	{
+		if (*pui32Log2Non4KPgShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || *pui32Log2Non4KPgShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Page sizes of 2MB or 1MB cause page faults."));
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+#endif
+
+	/* Check the Non4k page size is at least the size of the OS page size
+	 * or larger. The Non4k page size also has to be a multiple of the OS page
+	 * size but since we have the log2 value from the apphint we know powers of 2
+	 * will always be multiples. If the Non4k page size is less than OS page size
+	 * we notify and upgrade the size.
+	 */
+	if (*pui32Log2Non4KPgShift < uiLog2OSPageShift)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "Non4K page size smaller than OS page size, upgrading to "
+		                          "match OS page size."));
+		*pui32Log2Non4KPgShift = uiLog2OSPageShift;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* RGXRegisterDevice
+ *
+ * WARNING!
+ *
+ * No PDUMP statements are allowed until device initialisation starts.
+ */
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	DEVICE_MEMORY_INFO *psDevMemoryInfo;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault = HWPERF_HOST_TL_STREAM_SIZE_DEFAULT, ui32HWPerfHostBufSizeKB;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault, &ui32HWPerfHostBufSizeKB);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/*********************
+	 * Device node setup *
+	 *********************/
+	/* Setup static data and callbacks on the device agnostic device node */
+#if defined(PDUMP)
+	psDeviceNode->sDevId.pszPDumpRegName	= RGX_PDUMPREG_NAME;
+	psDeviceNode->sDevId.pszPDumpDevName	= PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+	psDeviceNode->pfnPDumpInitDevice = &RGXResetPDump;
+#endif /* PDUMP */
+
+	OSAtomicWrite(&psDeviceNode->eHealthStatus, PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	OSAtomicWrite(&psDeviceNode->eHealthReason, PVRSRV_DEVICE_HEALTH_REASON_NONE);
+
+	/* Configure MMU specific stuff */
+	RGXMMUInit_Register(psDeviceNode);
+
+	psDeviceNode->pfnInvalFBSCTable = NULL;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = NULL;
+
+	/* Callback for getting the MMU device attributes */
+	psDeviceNode->pfnGetMMUDeviceAttributes = RGXDevMMUAttributes;
+	psDeviceNode->pfnGetDeviceSnoopMode = RGXDevSnoopMode;
+	psDeviceNode->pfnMMUCacheInvalidate = RGXMMUCacheInvalidate;
+	psDeviceNode->pfnMMUCacheInvalidateKick = RGXMMUCacheInvalidateKick;
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = RGXMapBRN71422TargetPhysicalAddress;
+#else
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = NULL;
+#endif
+	/* pfnMMUTweakProtFlags is set later on once BVNC features are setup */
+	psDeviceNode->pfnMMUTweakProtFlags = NULL;
+
+	psDeviceNode->pfnInitDeviceCompatCheck	= &RGXDevInitCompatCheck;
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	/* Register callbacks for Unified Fence Objects */
+	psDeviceNode->pfnAllocUFOBlock = RGXAllocUFOBlock;
+	psDeviceNode->pfnFreeUFOBlock = RGXFreeUFOBlock;
+
+	/* Register callback for checking the device's health */
+	psDeviceNode->pfnUpdateHealthStatus = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? NULL : RGXUpdateHealthStatus;
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Register callback for updating the virtualization watchdog */
+	psDeviceNode->pfnUpdateAutoVzWatchdog = RGXUpdateAutoVzWatchdog;
+#endif
+
+	/* Register method to service the FW HWPerf buffer */
+	psDeviceNode->pfnServiceHWPerf = RGXHWPerfDataStoreCB;
+
+	/* Register callback for getting the device version information string */
+	psDeviceNode->pfnDeviceVersionString = RGXDevVersionString;
+
+	/* Register callback for getting the device clock speed */
+	psDeviceNode->pfnDeviceClockSpeed = RGXDevClockSpeed;
+
+	/* Register callback for soft resetting some device modules */
+	psDeviceNode->pfnSoftReset = RGXSoftReset;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnResetHWRLogs = RGXResetHWRLogs;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnVerifyBVNC = RGXVerifyBVNC;
+
+	/* Register callback for checking alignment of UM structures */
+	psDeviceNode->pfnAlignmentCheck = RGXAlignmentCheck;
+
+	/*Register callback for checking the supported features and getting the
+	 * corresponding values */
+	psDeviceNode->pfnCheckDeviceFeature = RGXBvncCheckFeatureSupported;
+	psDeviceNode->pfnGetDeviceFeatureValue = RGXBvncGetSupportedFeatureValue;
+
+	/* Callback for checking if system layer supports FBC 3.1 */
+	psDeviceNode->pfnHasFBCDCVersion31 = RGXSystemHasFBCDCVersion31;
+
+	/* Callback for getting TFBC configuration */
+	psDeviceNode->pfnGetTFBCLossyGroup = RGXGetTFBCLossyGroup;
+
+	/* Register callback for initialising device-specific physical memory heaps */
+	psDeviceNode->pfnPhysMemDeviceHeapsInit = RGXPhysMemDeviceHeapsInit;
+
+	/* Register callback for checking a device's FW Main physical heap for sufficient free memory */
+	psDeviceNode->pfnCheckForSufficientFWPhysMem = RGXDeviceFWMainHeapMemCheck;
+
+	/* Register callback for determining the appropriate LMA allocation policy for a phys heap */
+	psDeviceNode->pfnPhysHeapGetLMAPolicy = RGXPhysHeapGetLMAPolicy;
+
+	/*********************
+	 * Device info setup *
+	 *********************/
+	/* Allocate device control block */
+	psDevInfo = OSAllocZMem(sizeof(*psDevInfo));
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevInitRGXPart1 : Failed to alloc memory for DevInfo"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* initialise the layer parameters needed for early hw feature checks */
+	psDevInfo->sLayerParams.psDevInfo = psDevInfo;
+	psDevInfo->sLayerParams.psDevConfig = psDeviceNode->psDevConfig;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Default psTrampoline to point to null struct */
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+#endif
+
+	/* create locks for the context lists stored in the DevInfo structure.
+	 * these lists are modified on context create/destroy and read by the
+	 * watchdog thread
+	 */
+
+	eError = OSWRLockCreate(&(psDevInfo->hRenderCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create render context list lock", __func__));
+		goto e0;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hComputeCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create compute context list lock", __func__));
+		goto e1;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTransferCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create transfer context list lock", __func__));
+		goto e2;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTDMCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create TDM context list lock", __func__));
+		goto e3;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hKickSyncCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create kick sync context list lock", __func__));
+		goto e4;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hMemoryCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create memory context list lock", __func__));
+		goto e5;
+	}
+
+	eError = OSSpinLockCreate(&psDevInfo->hLockKCCBDeferredCommandsList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to KCCB deferred commands list lock", __func__));
+		goto e6;
+	}
+	dllist_init(&(psDevInfo->sKCCBDeferredCommandsListHead));
+
+	dllist_init(&(psDevInfo->sRenderCtxtListHead));
+	dllist_init(&(psDevInfo->sComputeCtxtListHead));
+	dllist_init(&(psDevInfo->sTransferCtxtListHead));
+	dllist_init(&(psDevInfo->sTDMCtxtListHead));
+	dllist_init(&(psDevInfo->sKickSyncCtxtListHead));
+
+	dllist_init(&(psDevInfo->sCommonCtxtListHead));
+	psDevInfo->ui32CommonCtxtCurrentID = 1;
+
+
+	eError = OSWRLockCreate(&psDevInfo->hCommonCtxtListLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create common context list lock", __func__));
+		goto e7;
+	}
+
+	eError = OSLockCreate(&psDevInfo->sRegConfig.hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create register configuration lock", __func__));
+		goto e8;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hBPLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for break points", __func__));
+		goto e9;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hRGXFWIfBufInitLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for trace buffers", __func__));
+		goto e10;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hCCBStallCheckLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB checking lock", __func__));
+		goto e11;
+	}
+	eError = OSLockCreate(&psDevInfo->hCCBRecoveryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB recovery lock", __func__));
+		goto e12;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hGpuUtilStatsLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create GPU stats lock", __func__));
+		goto e13;
+	}
+
+	dllist_init(&psDevInfo->sMemoryContextList);
+
+	/* initialise ui32SLRHoldoffCounter */
+	if (RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS > DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT)
+	{
+		psDevInfo->ui32SLRHoldoffCounter = RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS / DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	}
+	else
+	{
+		psDevInfo->ui32SLRHoldoffCounter = 0;
+	}
+
+	/* Setup static data and callbacks on the device specific device info */
+	psDevInfo->psDeviceNode		= psDeviceNode;
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+	psDevInfo->pvDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
+
+	/*
+	 * Map RGX Registers
+	 */
+	psDevInfo->ui32RegSize = psDeviceNode->psDevConfig->ui32RegsSize;
+	psDevInfo->sRegsPhysBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+#if !defined(NO_HARDWARE)
+	psDevInfo->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create RGX register mapping",
+		         __func__));
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto e14;
+	}
+#endif /* !NO_HARDWARE */
+
+	psDeviceNode->pvDevice = psDevInfo;
+
+	eError = RGXBvncInitialiseConfiguration(psDeviceNode);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Unsupported HW device detected by driver",
+		         __func__));
+		goto e15;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/*
+	 * We must now setup the SECURITY mappings if supported. We cannot
+	 * check on the features until we have reached here as the BVNC is
+	 * not setup before now.
+	 */
+#if !defined(NO_HARDWARE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+	{
+		IMG_CPU_PHYADDR sHostSecureRegBankBase = {psDeviceNode->psDevConfig->sRegsCpuPBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET};
+
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *) OSMapPhysToLin(sHostSecureRegBankBase,
+																		RGX_HOST_SECURE_REGBANK_SIZE,
+																		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (psDevInfo->pvSecureRegsBaseKM == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "PVRSRVRGXInitDevPart2KM: Failed to create RGX secure register mapping"));
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			goto e15;
+		}
+
+		/*
+		 * The secure register bank is mapped into the CPU VA space starting from
+		 * the base of the normal register bank + an offset of RGX_HOST_SECURE_REGBAK_OFFSET.
+		 * The hardware register addresses are all indexed from the base of the regular register bank.
+		 * For the RegBankBase+RegOffset computation to still be accurate for host-secure registers,
+		 * we need to compensate for offsets of registers in the secure bank
+		 */
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM - RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+	else
+	{
+		psDevInfo->pvSecureRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	}
+#endif /* !NO_HARDWARE */
+#endif /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	psDeviceNode->pfnMMUTweakProtFlags = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			RGXMMUTweakProtFlags : NULL;
+#endif
+
+	eError = RGXGetNon4KHeapPageShift(&psDevInfo->sLayerParams,
+	                                &psDeviceNode->ui32Non4KPageSizeLog2);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXGetNon4KHeapPageSize", e16);
+
+	eError = RGXInitHeaps(psDevInfo, psDevMemoryInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto e16;
+	}
+
+	eError = RGXHWPerfInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInit", e16);
+
+	eError = RGXHWPerfHostInit(psDeviceNode->pvDevice, ui32HWPerfHostBufSizeKB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfHostInit", ErrorDeInitHWPerfFw);
+
+
+	/* Register callback for dumping debug info */
+	eError = RGXDebugInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDebugInit", e17);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Register callback for fw mmu init */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDeviceNode->pfnFwMMUInit = RGXMipsMMUInit_Register;
+	}
+#endif
+
+	if (NULL != psDeviceNode->psDevConfig->pfnSysDevFeatureDepInit)
+	{
+		psDeviceNode->psDevConfig->pfnSysDevFeatureDepInit(psDeviceNode->psDevConfig,
+				psDevInfo->sDevFeatureCfg.ui64Features);
+	}
+
+	/* Initialise the device dependent bridges */
+	eError = RGXRegisterBridges(psDevInfo);
+	PVR_LOG_IF_ERROR(eError, "RGXRegisterBridges");
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = OSLockCreate(&psDevInfo->hCounterDumpingLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for counter sampling.", __func__));
+		goto ErrorDeInitDeviceDepBridge;
+	}
+#endif
+
+	/* Initialise error counters */
+	memset(&psDevInfo->sErrorCounts, 0, sizeof(PVRSRV_RGXDEV_ERROR_COUNTS));
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+ErrorDeInitDeviceDepBridge:
+	RGXUnregisterBridges(psDevInfo);
+#endif
+
+e17:
+	RGXHWPerfHostDeInit(psDevInfo);
+ErrorDeInitHWPerfFw:
+	RGXHWPerfDeinit(psDevInfo);
+e16:
+#if !defined(NO_HARDWARE)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+		/* Adjust pvSecureRegsBaseKM if device has SECURITY_VERSION > 1 */
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		    (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* Undo the VA offset adjustment to unmap correct VAddr */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM,
+			                 RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+	}
+#endif
+#endif /* !NO_HARDWARE */
+e15:
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+		                 psDevInfo->ui32RegSize);
+	}
+e14:
+#endif /* !NO_HARDWARE */
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+e13:
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+e12:
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+e11:
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+e10:
+	OSLockDestroy(psDevInfo->hBPLock);
+e9:
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+e8:
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+e7:
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+e6:
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+e5:
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+e4:
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+e3:
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+e2:
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+e1:
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+e0:
+	OSFreeMem(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_PCHAR psz = psDevInfo->sDevFeatureCfg.pszBVNCString;
+	if (NULL == psz)
+	{
+		IMG_CHAR pszBVNCInfo[RGX_HWPERF_MAX_BVNC_LEN];
+		size_t uiBVNCStringSize;
+		size_t uiStringLength;
+
+		uiStringLength = OSSNPrintf(pszBVNCInfo, RGX_HWPERF_MAX_BVNC_LEN, "%d.%d.%d.%d",
+				psDevInfo->sDevFeatureCfg.ui32B,
+				psDevInfo->sDevFeatureCfg.ui32V,
+				psDevInfo->sDevFeatureCfg.ui32N,
+				psDevInfo->sDevFeatureCfg.ui32C);
+		PVR_ASSERT(uiStringLength < RGX_HWPERF_MAX_BVNC_LEN);
+
+		uiBVNCStringSize = (uiStringLength + 1) * sizeof(IMG_CHAR);
+		psz = OSAllocMem(uiBVNCStringSize);
+		if (NULL != psz)
+		{
+			OSCachedMemCopy(psz, pszBVNCInfo, uiBVNCStringSize);
+			psDevInfo->sDevFeatureCfg.pszBVNCString = psz;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Allocating memory for BVNC Info string failed",
+					__func__));
+		}
+	}
+
+	return psz;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDevVersionString
+@Description    Gets the version string for the given device node and returns
+                a pointer to it in ppszVersionString. It is then the
+                responsibility of the caller to free this memory.
+@Input          psDeviceNode            Device node from which to obtain the
+                                        version string
+@Output	        ppszVersionString	Contains the version string upon return
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR **ppszVersionString)
+{
+#if defined(NO_HARDWARE) || defined(EMULATOR)
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (SW)";
+#else
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (HW)";
+#endif
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_PCHAR pszBVNC;
+	size_t uiStringLength;
+
+	if (psDeviceNode == NULL || ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	pszBVNC = RGXDevBVNCString(psDevInfo);
+
+	if (NULL == pszBVNC)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	uiStringLength = OSStringLength(pszBVNC);
+	uiStringLength += (sizeof(szFormatString) - 2); /* sizeof includes the null, -2 for "%s" */
+	*ppszVersionString = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (*ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSSNPrintf(*ppszVersionString, uiStringLength, szFormatString,
+		pszBVNC);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       RGXDevClockSpeed
+@Description    Gets the clock speed for the given device node and returns
+                it in pui32RGXClockSpeed.
+@Input          psDeviceNode		Device node
+@Output         pui32RGXClockSpeed  Variable for storing the clock speed
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_PUINT32  pui32RGXClockSpeed)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* get clock speed */
+	*pui32RGXClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/*!
+ *******************************************************************************
+
+ @Function		RGXInitFwRawHeap
+
+ @Description	Called to perform additional initialisation
+ ******************************************************************************/
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID)
+{
+	IMG_UINT32 uiStringLength;
+	IMG_UINT32 uiStringLengthMax = 32;
+
+	IMG_UINT32 ui32Log2RgxDefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	PVR_RETURN_IF_FALSE(ui32Log2RgxDefaultPageShift != 0, PVRSRV_ERROR_INVALID_PARAMS);
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2RgxDefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2RgxDefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	uiStringLength = MIN(sizeof(RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT), uiStringLengthMax + 1);
+
+	/* Start by allocating memory for this DriverID heap identification string */
+	psDevMemHeap->pszName = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (psDevMemHeap->pszName == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Append the DriverID number to the RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT string */
+	OSSNPrintf((IMG_CHAR *)psDevMemHeap->pszName, uiStringLength, RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	/* Use the common blueprint template support function to initialise the heap */
+	HeapCfgBlueprintInit(psDevMemHeap->pszName,
+		                 RGX_FIRMWARE_RAW_HEAP_BASE + (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE),
+		                 RGX_FIRMWARE_RAW_HEAP_SIZE,
+		                 0,
+		                 ui32Log2RgxDefaultPageShift,
+		                 0,
+		                 NULL,
+		                 NULL,
+		                 psDevMemHeap);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function		RGXDeInitFwRawHeap
+
+ @Description	Called to perform additional deinitialisation
+ ******************************************************************************/
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap)
+{
+	IMG_UINT64 uiBase = RGX_FIRMWARE_RAW_HEAP_BASE + RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT64 uiSpan = uiBase + ((RGX_NUM_DRIVERS_SUPPORTED - 1) * RGX_FIRMWARE_RAW_HEAP_SIZE);
+
+	/* Safe to do as the guest firmware heaps are last in the list */
+	if (psDevMemHeap->sHeapBaseAddr.uiAddr >= uiBase &&
+	    psDevMemHeap->sHeapBaseAddr.uiAddr < uiSpan)
+	{
+		void *pszName = (void*)psDevMemHeap->pszName;
+		OSFreeMem(pszName);
+	}
+}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/******************************************************************************
+ End of file (rgxinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxlayer_impl.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxlayer_impl.c
new file mode 100644
index 000000000000..535326d7e786
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxlayer_impl.c
@@ -0,0 +1,478 @@
+/*************************************************************************/ /*!
+@File
+@Title          DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwimageutils.h"
+#include "cache_km.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+
+#define MAX_NUM_COHERENCY_TESTS  (10)
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	if (psDevInfo->ui32CoherencyTestsDone >= MAX_NUM_COHERENCY_TESTS)
+	{
+		return IMG_FALSE;
+	}
+
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	return PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig);
+}
+
+/*
+ * The fabric coherency test is performed when platform supports fabric coherency
+ * either in the form of ACE-lite or Full-ACE. This test is done quite early
+ * with the firmware processor quiescent and makes exclusive use of the slave
+ * port interface for reading/writing through the device memory hierarchy. The
+ * rationale for the test is to ensure that what the CPU writes to its dcache
+ * is visible to the GPU via coherency snoop miss/hit and vice-versa without
+ * any intervening cache maintenance by the writing agent.
+ */
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 *pui32FabricCohTestBufferCpuVA;
+	DEVMEM_MEMDESC *psFabricCohTestBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR sFabricCohTestBufferDevVA;
+	IMG_DEVMEM_SIZE_T uiFabricCohTestBlockSize = sizeof(IMG_UINT64);
+	IMG_DEVMEM_ALIGN_T uiFabricCohTestBlockAlign = sizeof(IMG_UINT64);
+	IMG_UINT32 ui32SLCCTRL = 0;
+	IMG_UINT32 ui32OddEven;
+	IMG_UINT32 ui32OddEvenSeed = 1;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bFullTestPassed = IMG_TRUE;
+	IMG_BOOL bExit = IMG_FALSE;
+#if defined(DEBUG)
+	IMG_BOOL bSubTestPassed = IMG_FALSE;
+#endif
+	enum TEST_TYPE {
+		CPU_WRITE_GPU_READ_SM=0, GPU_WRITE_CPU_READ_SM,
+		CPU_WRITE_GPU_READ_SH,   GPU_WRITE_CPU_READ_SH
+	} eTestType;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	PVR_LOG(("Starting fabric coherency test ....."));
+
+	{
+		/* Bypass the SLC when IO coherency is enabled */
+		ui32SLCCTRL = RGXReadReg32(hPrivate, RGX_CR_SLC_CTRL_BYPASS);
+		RGXWriteReg32(hPrivate,
+				RGX_CR_SLC_CTRL_BYPASS,
+				ui32SLCCTRL | RGX_CR_SLC_CTRL_BYPASS_BYP_CC_EN);
+	}
+
+	/* Size and align are 'expanded' because we request an export align allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiFabricCohTestBlockSize,
+	                                             &uiFabricCohTestBlockAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemExportalignAdjustSizeAndAlign() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	/* Allocate, acquire cpu address and set firmware address */
+	eError = DevmemFwAllocateExportable(psDevInfo->psDeviceNode,
+										uiFabricCohTestBlockSize,
+										uiFabricCohTestBlockAlign,
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT |
+										PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwExFabricCoherencyTestBuffer",
+										&psFabricCohTestBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemFwAllocateExportable() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psFabricCohTestBufferMemDesc, (void **) &pui32FabricCohTestBufferCpuVA);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemAcquireCpuVirtAddr() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e1;
+	}
+
+	/* Create a FW address which is uncached in the Meta DCache and in the SLC using the Meta bootloader segment.
+	   This segment is the only one configured correctly out of reset (when this test is meant to be executed) */
+
+	eError = RGXSetFirmwareAddress(&sFabricCohTestBufferDevVA,
+						  psFabricCohTestBufferMemDesc,
+						  0,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", e2);
+
+	/* Undo most of the FW mappings done by RGXSetFirmwareAddress */
+	sFabricCohTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_META_CACHE_MASK;
+	sFabricCohTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK;
+	sFabricCohTestBufferDevVA.ui32Addr -= RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+	/* Map the buffer in the bootloader segment as uncached */
+	sFabricCohTestBufferDevVA.ui32Addr |= RGXFW_BOOTLDR_META_ADDR;
+	sFabricCohTestBufferDevVA.ui32Addr |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+
+	for (eTestType = CPU_WRITE_GPU_READ_SH; eTestType <= GPU_WRITE_CPU_READ_SH && bExit == IMG_FALSE; eTestType++)
+	{
+		IMG_CPU_PHYADDR sCpuPhyAddr;
+		IMG_BOOL bValid;
+		PMR *psPMR;
+
+		/* Acquire underlying PMR CpuPA in preparation for cache maintenance */
+		(void) DevmemLocalGetImportHandle(psFabricCohTestBufferMemDesc, (void**)&psPMR);
+		eError = PMR_CpuPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sCpuPhyAddr, &bValid, CPU_USE);
+		if (eError != PVRSRV_OK || bValid == IMG_FALSE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"PMR_CpuPhysAddr error: %s, exiting",
+					PVRSRVGetErrorString(eError)));
+			bExit = IMG_TRUE;
+			continue;
+		}
+
+		/* Here we do two passes mostly to account for the effects of using a different
+		   seed (i.e. ui32OddEvenSeed) value to read and write */
+		for (ui32OddEven = 1; ui32OddEven < 3 && bExit == IMG_FALSE; ui32OddEven++)
+		{
+			IMG_UINT32 i;
+
+#if defined(DEBUG)
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+
+			/* Do multiple sub-dword cache line tests */
+			for (i = 0; i < 2 && bExit == IMG_FALSE; i++)
+			{
+				IMG_UINT32 ui32FWAddr;
+				IMG_UINT32 ui32FWValue;
+				IMG_UINT32 ui32FWValue2;
+				IMG_CPU_PHYADDR sCpuPhyAddrStart;
+				IMG_CPU_PHYADDR sCpuPhyAddrEnd;
+				IMG_UINT32 ui32LastFWValue = ~0;
+				IMG_UINT32 ui32Offset = i * sizeof(IMG_UINT32);
+
+				/* Calculate next address and seed value to write/read from slave-port */
+				ui32FWAddr = sFabricCohTestBufferDevVA.ui32Addr + ui32Offset;
+				sCpuPhyAddrStart.uiAddr = sCpuPhyAddr.uiAddr + ui32Offset;
+				sCpuPhyAddrEnd.uiAddr = sCpuPhyAddrStart.uiAddr;
+				ui32OddEvenSeed += 1;
+
+				if (eTestType == GPU_WRITE_CPU_READ_SM || eTestType == GPU_WRITE_CPU_READ_SH)
+				{
+					/* Clean dcache to ensure there is no stale data in dcache that might over-write
+					   what we are about to write via slave-port here because if it drains from the CPU
+					   dcache before we read it, it would corrupt what we are going to read back via
+					   the CPU */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_FLUSH);
+
+					/* Calculate a new value to write */
+					ui32FWValue = i + ui32OddEvenSeed;
+
+					/* Write the value using the RGX slave-port interface */
+					eError = RGXWriteFWModuleAddr(psDevInfo, ui32FWAddr, ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXWriteFWModuleAddr error: %s, exiting",
+						          PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Read back value using RGX slave-port interface, this is used
+					   as a sort of memory barrier for the above write */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue2);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXReadFWModuleAddr error: %s, exiting",
+						         PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+					else if (ui32FWValue != ui32FWValue2)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW via SlavePort failed: expected: %x, got: %x",
+								i,
+								ui32FWValue,
+								ui32FWValue2));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Invalidate dcache to ensure that any prefetched data by the CPU from this memory
+					   region is discarded before we read (i.e. next read must trigger a cache miss).
+					   Previously there was snooping of device cache, where prefetching done by the CPU
+					   would reflect the most up to date datum writing by GPU into said location,
+					   that is to say prefetching was coherent so CPU d-flush was not needed */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_INVALIDATE);
+				}
+				else
+				{
+					IMG_UINT32 ui32RAWCpuValue;
+
+					/* Ensures line is in dcache */
+					ui32FWValue = IMG_UINT32_MAX;
+
+					/* Dirty allocation in dcache */
+					ui32RAWCpuValue = i + ui32OddEvenSeed;
+					pui32FabricCohTestBufferCpuVA[i] = i + ui32OddEvenSeed;
+
+					/* Flush possible cpu store-buffer(ing) on LMA */
+					OSWriteMemoryBarrier(&pui32FabricCohTestBufferCpuVA[i]);
+
+					if (eTestType == CPU_WRITE_GPU_READ_SM)
+					{
+						/* Flush dcache to force subsequent incoming CPU-bound snoop to miss so
+						   memory is coherent before the SlavePort reads */
+						sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+						CacheOpExec(psDevInfo->psDeviceNode,
+								(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+								(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+								sCpuPhyAddrStart,
+								sCpuPhyAddrEnd,
+								PVRSRV_CACHE_OP_FLUSH);
+					}
+
+					/* Read back value using RGX slave-port interface */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"RGXReadFWModuleAddr error: %s, exiting",
+								PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Being mostly paranoid here, verify that CPU RAW operation is valid
+					   after the above slave port read */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_FLUSH);
+					if (pui32FabricCohTestBufferCpuVA[i] != ui32RAWCpuValue)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW by CPU failed: expected: %x, got: %x",
+								i,
+								ui32RAWCpuValue,
+								pui32FabricCohTestBufferCpuVA[i]));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				/* Compare to see if sub-test passed */
+				if (pui32FabricCohTestBufferCpuVA[i] == ui32FWValue)
+				{
+#if defined(DEBUG)
+					bSubTestPassed = IMG_TRUE;
+#endif
+				}
+				else
+				{
+					bFullTestPassed = IMG_FALSE;
+					eError = PVRSRV_ERROR_INIT_FAILURE;
+#if defined(DEBUG)
+					bSubTestPassed = IMG_FALSE;
+#endif
+					if (ui32LastFWValue != ui32FWValue)
+					{
+#if defined(DEBUG)
+						PVR_LOG(("At Offset: %d, Expected: %x, Got: %x",
+								 i,
+								 (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i],
+								 (eTestType & 0x1) ? pui32FabricCohTestBufferCpuVA[i] : ui32FWValue));
+#endif
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"test encountered unexpected error, exiting"));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				ui32LastFWValue = (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i];
+			}
+
+#if defined(DEBUG)
+			if (bExit)
+			{
+				continue;
+			}
+
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+		}
+	}
+
+	RGXUnsetFirmwareAddress(psFabricCohTestBufferMemDesc);
+e2:
+	DevmemReleaseCpuVirtAddr(psFabricCohTestBufferMemDesc);
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, psFabricCohTestBufferMemDesc);
+
+e0:
+	{
+		/* Restore SLC bypass settings */
+		RGXWriteReg32(hPrivate, RGX_CR_SLC_CTRL_BYPASS, ui32SLCCTRL);
+	}
+
+	bFullTestPassed = bExit ? IMG_FALSE: bFullTestPassed;
+	if (bFullTestPassed)
+	{
+		PVR_LOG(("fabric coherency test: PASSED"));
+		psDevInfo->ui32CoherencyTestsDone = MAX_NUM_COHERENCY_TESTS + 1;
+	}
+	else
+	{
+		PVR_LOG(("fabric coherency test: FAILED"));
+		psDevInfo->ui32CoherencyTestsDone++;
+	}
+
+	return eError;
+}
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.c
new file mode 100644
index 000000000000..e0b17bf3dedd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.c
@@ -0,0 +1,1068 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxmipsmmuinit.h"
+
+#include "device.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "mmu_common.h"
+#include "pdump_mmu.h"
+#include "rgxheapconfig.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "rgx_memallocflags.h"
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "log2.h"
+
+/*
+ * Bits of PT, PD and PC not involving addresses
+ */
+
+/* Currently there is no page directory for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PDE_PROTMASK        0
+/* Currently there is no page catalog for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PCE_PROTMASK	     0
+
+
+static MMU_PxE_CONFIG sRGXMMUPCEConfig;
+static MMU_DEVVADDR_CONFIG sRGXMMUTopLevelDevVAddrConfig;
+
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_4KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_4KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_4KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig4KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_16KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_16KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_16KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig16KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_64KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_64KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_64KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig64KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_256KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_256KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig256KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 1MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_1MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_1MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_1MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig1MB;
+
+
+/*
+ *
+ *  Configuration for heaps with 2MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_2MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_2MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_2MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig2MB;
+
+
+/* Forward declaration of protection bits derivation functions, for
+   the following structure */
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags);
+
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+										   const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+										   const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+										   const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+										   IMG_HANDLE *phPriv);
+
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv);
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+
+static MMU_DEVICEATTRIBS sRGXMMUDeviceAttributes;
+
+/* Cached policy */
+static IMG_UINT32 gui32CachedPolicy;
+
+static PVRSRV_ERROR RGXCheckTrampolineAddrs(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                            MMU_DEVICEATTRIBS *psDevAttrs,
+                                            IMG_UINT64 *pui64Addr);
+
+PVRSRV_ERROR RGXMipsMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bPhysBusAbove32Bit = 0;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PHYS_BUS_WIDTH))
+	{
+		bPhysBusAbove32Bit = RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32;
+	}
+
+	sRGXMMUDeviceAttributes.pszMMUPxPDumpMemSpaceName =
+		PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+
+	/*
+	 * Setup sRGXMMUPCEConfig, no PC in MIPS MMU currently
+	 */
+	sRGXMMUPCEConfig.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPCEConfig.pszPxLevelStr = "UnD";
+	sRGXMMUPCEConfig.uiBytesPerEntry = 0; /* 32 bit entries */
+
+	sRGXMMUPCEConfig.uiAddrMask = 0; /* Mask to get significant address bits of PC entry */
+	sRGXMMUPCEConfig.uiAddrShift = 0; /* Shift this many bits to get PD address in PC entry */
+	sRGXMMUPCEConfig.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K; /* Alignment of PD AND PC */
+
+	sRGXMMUPCEConfig.uiProtMask = RGX_MIPS_MMUCTRL_PCE_PROTMASK; /* Mask to get the status bits of the PC */
+	sRGXMMUPCEConfig.uiProtShift = 0; /* Shift this many bits to have status bits starting with bit 0 */
+
+	sRGXMMUPCEConfig.uiValidEnMask = RGX_MIPS_MMUCTRL_PC_DATA_VALID_EN; /* Mask to get entry valid bit of the PC */
+	sRGXMMUPCEConfig.uiValidEnShift = RGX_MIPS_MMUCTRL_PC_DATA_VALID_SHIFT; /* Shift this many bits to have entry valid bit starting with bit 0 */
+
+	/*
+	 *  Setup sRGXMMUTopLevelDevVAddrConfig
+	 */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask = 0; /* Get the PC address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift = 0;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPC = 0;
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask = 0; /* Get the PD address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift = 0;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPD = 0;
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPTIndexMask = IMG_UINT64_C(0xfffffff000); /* Get the PT address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPTIndexShift = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_4KBDP. No PD in MIPS MMU currently
+	 */
+	sRGXMMUPDEConfig_4KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_4KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_4KBDP.uiBytesPerEntry = 0;
+
+	/* No PD used for MIPS */
+	sRGXMMUPDEConfig_4KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_4KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_4KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlMask = IMG_UINT64_C(0x0);
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiProtMask = RGX_MIPS_MMUCTRL_PDE_PROTMASK;
+	sRGXMMUPDEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiValidEnMask = RGX_MIPS_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_4KBDP.uiValidEnShift = RGX_MIPS_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_4KBDP.
+	 */
+	sRGXMMUPTEConfig_4KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_4KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+
+	if (bPhysBusAbove32Bit)
+	{
+		sRGXMMUPTEConfig_4KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT;
+	}
+	else
+	{
+		sRGXMMUPTEConfig_4KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY;
+	}
+
+	sRGXMMUPTEConfig_4KBDP.uiAddrShift = RGXMIPSFW_ENTRYLO_PFN_SHIFT;
+	sRGXMMUPTEConfig_4KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPTEConfig_4KBDP.uiProtMask = RGXMIPSFW_ENTRYLO_DVG | ~RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK |
+	                                    RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN | RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN;
+	sRGXMMUPTEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_4KBDP.uiValidEnMask = RGXMIPSFW_ENTRYLO_VALID_EN;
+	sRGXMMUPTEConfig_4KBDP.uiValidEnShift = RGXMIPSFW_ENTRYLO_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_4KBDP
+	 */
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPC = 0;
+
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask = ~RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift = RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000000fff);
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiOffsetInBytes = RGX_FIRMWARE_RAW_HEAP_BASE & IMG_UINT64_C(0x00ffffffff);
+
+	/*
+	 * Setup gsPageSizeConfig4KB
+	 */
+	gsPageSizeConfig4KB.psPDEConfig = &sRGXMMUPDEConfig_4KBDP;
+	gsPageSizeConfig4KB.psPTEConfig = &sRGXMMUPTEConfig_4KBDP;
+	gsPageSizeConfig4KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_4KBDP;
+	gsPageSizeConfig4KB.uiRefCount = 0;
+	gsPageSizeConfig4KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_16KBDP
+	 */
+	sRGXMMUPDEConfig_16KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_16KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_16KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiAddrShift = 0; /* These are for a page directory ENTRY, meaning the address of a PT cropped to suit the PD */
+	sRGXMMUPDEConfig_16KBDP.uiAddrLog2Align = 0; /* Alignment of the page tables NOT directories */
+
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_16KBDP. Not supported yet
+	 */
+	sRGXMMUPTEConfig_16KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_16KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiAddrShift = 0; /* These are for a page table ENTRY, meaning the address of a PAGE cropped to suit the PD */
+	sRGXMMUPTEConfig_16KBDP.uiAddrLog2Align = 0; /* Alignment of the pages NOT tables */
+
+	sRGXMMUPTEConfig_16KBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_16KBDP
+	 */
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig16KB
+	 */
+	gsPageSizeConfig16KB.psPDEConfig = &sRGXMMUPDEConfig_16KBDP;
+	gsPageSizeConfig16KB.psPTEConfig = &sRGXMMUPTEConfig_16KBDP;
+	gsPageSizeConfig16KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_16KBDP;
+	gsPageSizeConfig16KB.uiRefCount = 0;
+	gsPageSizeConfig16KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size. Not supported yet
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_64KBDP
+	 */
+	sRGXMMUPDEConfig_64KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_64KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_64KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_64KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_64KBDP.
+	 *
+	 */
+	sRGXMMUPTEConfig_64KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_64KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+	if (bPhysBusAbove32Bit)
+	{
+		sRGXMMUPTEConfig_64KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT;
+	}
+	else
+	{
+		sRGXMMUPTEConfig_64KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY;
+	}
+
+	/* Even while using 64K pages, MIPS still aligns addresses to 4K */
+	sRGXMMUPTEConfig_64KBDP.uiAddrShift = RGXMIPSFW_ENTRYLO_PFN_SHIFT;
+	sRGXMMUPTEConfig_64KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPTEConfig_64KBDP.uiProtMask = RGXMIPSFW_ENTRYLO_DVG | ~RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK |
+	                                     RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN | RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN;
+	sRGXMMUPTEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_64KBDP.uiValidEnMask = RGXMIPSFW_ENTRYLO_VALID_EN;
+	sRGXMMUPTEConfig_64KBDP.uiValidEnShift = RGXMIPSFW_ENTRYLO_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_64KBDP.
+	 */
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask = IMG_UINT64_C(0x00ffff0000);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_64K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000000ffff);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiOffsetInBytes = RGX_FIRMWARE_RAW_HEAP_BASE & IMG_UINT64_C(0x00ffffffff);
+
+	/*
+	 * Setup gsPageSizeConfig64KB.
+	 */
+	gsPageSizeConfig64KB.psPDEConfig = &sRGXMMUPDEConfig_64KBDP;
+	gsPageSizeConfig64KB.psPTEConfig = &sRGXMMUPTEConfig_64KBDP;
+	gsPageSizeConfig64KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_64KBDP;
+	gsPageSizeConfig64KB.uiRefCount = 0;
+	gsPageSizeConfig64KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size. Not supported yet
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_256KBDP
+	 */
+	sRGXMMUPDEConfig_256KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_256KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_256KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_256KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP
+	 */
+	sRGXMMUPTEConfig_256KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_256KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_256KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_256KBDP
+	 */
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig256KB
+	 */
+	gsPageSizeConfig256KB.psPDEConfig = &sRGXMMUPDEConfig_256KBDP;
+	gsPageSizeConfig256KB.psPTEConfig = &sRGXMMUPTEConfig_256KBDP;
+	gsPageSizeConfig256KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_256KBDP;
+	gsPageSizeConfig256KB.uiRefCount = 0;
+	gsPageSizeConfig256KB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUPDEConfig_1MBDP.  Not supported yet
+	 */
+	sRGXMMUPDEConfig_1MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_1MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_1MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_1MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_1MBDP
+	 */
+	sRGXMMUPTEConfig_1MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_1MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_1MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_1MBDP
+	 */
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig1MB
+	 */
+	gsPageSizeConfig1MB.psPDEConfig = &sRGXMMUPDEConfig_1MBDP;
+	gsPageSizeConfig1MB.psPTEConfig = &sRGXMMUPTEConfig_1MBDP;
+	gsPageSizeConfig1MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_1MBDP;
+	gsPageSizeConfig1MB.uiRefCount = 0;
+	gsPageSizeConfig1MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUPDEConfig_2MBDP. Not supported yet
+	 */
+	sRGXMMUPDEConfig_2MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_2MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_2MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_2MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_2MBDP
+	 */
+	sRGXMMUPTEConfig_2MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_2MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_2MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_2MBDP
+	 */
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig2MB
+	 */
+	gsPageSizeConfig2MB.psPDEConfig = &sRGXMMUPDEConfig_2MBDP;
+	gsPageSizeConfig2MB.psPTEConfig = &sRGXMMUPTEConfig_2MBDP;
+	gsPageSizeConfig2MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_2MBDP;
+	gsPageSizeConfig2MB.uiRefCount = 0;
+	gsPageSizeConfig2MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUDeviceAttributes
+	 */
+	sRGXMMUDeviceAttributes.eMMUType = PDUMP_MMU_TYPE_MIPS_MICROAPTIV;
+
+	/*
+	 * The page table fits in one or more big physically adjacent pages,
+	 * at most as big as the page table itself.
+	 * To calculate its alignment/page size, calculate the log2 size of the page
+	 * table taking into account all OSes, then round that down to a valid MIPS
+	 * log2 page size (12, 14, 16 for a 4K, 16K, 64K page size).
+	 */
+	sRGXMMUDeviceAttributes.ui32BaseAlign =
+		(CeilLog2(RGX_NUM_DRIVERS_SUPPORTED) + RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) & ~1U;
+
+	/* 256K alignment might be too hard to achieve, fall back to 64K */
+	sRGXMMUDeviceAttributes.ui32BaseAlign =
+		MIN(sRGXMMUDeviceAttributes.ui32BaseAlign, RGXMIPSFW_LOG2_PAGE_SIZE_64K);
+
+
+
+	/* The base configuration is set to 4kB pages*/
+	sRGXMMUDeviceAttributes.psBaseConfig = &sRGXMMUPTEConfig_4KBDP;
+	sRGXMMUDeviceAttributes.psTopLevelDevVAddrConfig = &sRGXMMUTopLevelDevVAddrConfig;
+
+	/* Functions for deriving page table/dir/cat protection bits */
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt8 = RGXDerivePCEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt4 = RGXDerivePCEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXDerivePDEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt4 = RGXDerivePDEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt8 = RGXDerivePTEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt4 = RGXDerivePTEProt4;
+
+	/* Functions for establishing configurations for PDE/PTE/DEVVADDR
+	   on per-heap basis */
+	sRGXMMUDeviceAttributes.pfnGetPageSizeConfiguration = RGXGetPageSizeConfigCB;
+	sRGXMMUDeviceAttributes.pfnPutPageSizeConfiguration = RGXPutPageSizeConfigCB;
+
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE4 = RGXGetPageSizeFromPDE4;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXGetPageSizeFromPDE8;
+
+	psDeviceNode->psFirmwareMMUDevAttrs = &sRGXMMUDeviceAttributes;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = RGXCheckTrampolineAddrs;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXCheckTrampolineAddrs(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                            MMU_DEVICEATTRIBS *psDevAttrs,
+                                            IMG_UINT64 *pui64Addr)
+{
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, MIPS))
+	{
+		/*
+		 * If mapping for the MIPS FW context, check for sensitive PAs
+		 */
+		if (psDevAttrs == psDevNode->psFirmwareMMUDevAttrs)
+		{
+			PVRSRV_RGXDEV_INFO *psDevice = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice;
+
+			if ((RGX_GET_FEATURE_VALUE(psDevice, PHYS_BUS_WIDTH) == 32) &&
+				 RGXMIPSFW_SENSITIVE_ADDR(*pui64Addr))
+			{
+				*pui64Addr = psDevice->psTrampoline->sPhysAddr.uiAddr + RGXMIPSFW_TRAMPOLINE_OFFSET(*pui64Addr);
+			}
+			/* FIX_HW_BRN_63553 is mainlined for all MIPS cores */
+			else if (*pui64Addr == 0x0 && !psDevice->sLayerParams.bDevicePA0IsValid)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s attempt to map addr 0x0 in the FW but 0x0 is not considered valid.", __func__));
+				return PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE;
+			}
+		}
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXMipsMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	psDeviceNode->pfnMMUGetContextID = NULL;
+#endif
+
+	psDeviceNode->psFirmwareMMUDevAttrs = NULL;
+
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_MESSAGE, "Variable Page Size Heap Stats:"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 4K page heaps: %d",
+			 gsPageSizeConfig4KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 4K page heaps (should be 0): %d",
+			 gsPageSizeConfig4KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 16K page heaps: %d",
+			 gsPageSizeConfig16KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 16K page heaps (should be 0): %d",
+			 gsPageSizeConfig16KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 64K page heaps: %d",
+			 gsPageSizeConfig64KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 64K page heaps (should be 0): %d",
+			 gsPageSizeConfig64KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 256K page heaps: %d",
+			 gsPageSizeConfig256KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 256K page heaps (should be 0): %d",
+			 gsPageSizeConfig256KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 1M page heaps: %d",
+			 gsPageSizeConfig1MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 1M page heaps (should be 0): %d",
+			 gsPageSizeConfig1MB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 2M page heaps: %d",
+			 gsPageSizeConfig2MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 2M page heaps (should be 0): %d",
+			 gsPageSizeConfig2MB.uiRefCount));
+#endif
+	if (gsPageSizeConfig4KB.uiRefCount > 0 ||
+		gsPageSizeConfig16KB.uiRefCount > 0 ||
+		gsPageSizeConfig64KB.uiRefCount > 0 ||
+		gsPageSizeConfig256KB.uiRefCount > 0 ||
+		gsPageSizeConfig1MB.uiRefCount > 0 ||
+		gsPageSizeConfig2MB.uiRefCount > 0
+		)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXMMUInit_Unregister: Unbalanced MMU API Usage (Internal error)"));
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt4
+@Description    calculate the PCE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_DPF((PVR_DBG_ERROR, "Page Catalog not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt8
+@Description    calculate the PCE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "Page Catalog not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt4
+@Description    derive the PDE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "Page Directory not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize The log2 of the required page size.
+				E.g, for 4KiB pages, this parameter must be 12.
+				For 2MiB pages, it must be set to 21.
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "Page Directory not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt4
+@Description    calculate the PTE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags)
+{
+	IMG_UINT32 ui32MMUFlags = 0;
+
+	if (((MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE) & uiProtFlags) == (MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE))
+	{
+		/* read/write */
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_DIRTY_EN;
+	}
+	else if (MMU_PROTFLAGS_READABLE & uiProtFlags)
+	{
+		/* read only */
+	}
+	else if (MMU_PROTFLAGS_WRITEABLE & uiProtFlags)
+	{
+		/* write only */
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN;
+	}
+	else if ((MMU_PROTFLAGS_INVALID & uiProtFlags) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt4: neither read nor write specified..."));
+	}
+
+	/* cache coherency */
+	if (MMU_PROTFLAGS_CACHE_COHERENT & uiProtFlags)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt4: cache coherency not supported for MIPS caches"));
+	}
+
+	/* cache setup */
+	if ((MMU_PROTFLAGS_CACHED & uiProtFlags) == 0)
+	{
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_UNCACHED;
+	}
+	else
+	{
+		ui32MMUFlags |= gui32CachedPolicy <<
+		                RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT;
+	}
+
+	if ((uiProtFlags & MMU_PROTFLAGS_INVALID) == 0)
+	{
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_VALID_EN;
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_GLOBAL_EN;
+	}
+
+	if (MMU_PROTFLAGS_DEVICE(PMMETA_PROTECT) & uiProtFlags)
+	{
+		/* PVR_DPF((PVR_DBG_WARNING, "RGXDerivePTEProt4: PMMETA Protect not existent for MIPS, option discarded")); */
+	}
+
+	return ui32MMUFlags;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt8
+@Description    calculate the PTE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "8-byte PTE not supported on this device"));
+
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXGetPageSizeConfig
+@Description    Set up configuration for variable sized data pages.
+				RGXPutPageSizeConfigCB has to be called to ensure correct
+				refcounting.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+										   const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+										   const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+										   const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+										   IMG_HANDLE *phPriv)
+{
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGXMIPSFW_LOG2_PAGE_SIZE_64K:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGXMIPSFW_LOG2_PAGE_SIZE_4K:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				 "RGXGetPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				 uiLog2DataPageSize));
+		*phPriv = NULL;
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Refer caller's pointers to the data */
+	*ppsMMUPDEConfig = psPageSizeConfig->psPDEConfig;
+	*ppsMMUPTEConfig = psPageSizeConfig->psPTEConfig;
+	*ppsMMUDevVAddrConfig = psPageSizeConfig->psDevVAddrConfig;
+
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	/* Increment ref-count - not that we're allocating anything here
+	   (I'm using static structs), but one day we might, so we want
+	   the Get/Put code to be balanced properly */
+	psPageSizeConfig->uiRefCount++;
+
+	/* This is purely for debug statistics */
+	psPageSizeConfig->uiMaxRefCount = MAX(psPageSizeConfig->uiMaxRefCount,
+										  psPageSizeConfig->uiRefCount);
+#endif
+
+	*phPriv = (IMG_HANDLE)(uintptr_t)uiLog2DataPageSize;
+	PVR_ASSERT (uiLog2DataPageSize == (IMG_UINT32)(uintptr_t)*phPriv);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPutPageSizeConfig
+@Description    Tells this code that the mmu module is done with the
+				configurations set in RGXGetPageSizeConfig.  This can
+				be a no-op.
+				Called after RGXGetPageSizeConfigCB.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv)
+{
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+	IMG_UINT32 uiLog2DataPageSize;
+
+	uiLog2DataPageSize = (IMG_UINT32)(uintptr_t) hPriv;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGXMIPSFW_LOG2_PAGE_SIZE_64K:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGXMIPSFW_LOG2_PAGE_SIZE_4K:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				 "RGXPutPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				 uiLog2DataPageSize));
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Ref-count here is not especially useful, but it's an extra
+	   check that the API is being used correctly */
+	psPageSizeConfig->uiRefCount--;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "PDE not supported on MIPS"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "PDE not supported on MIPS"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+void RGXMipsCheckFaultAddress(MMU_CONTEXT *psFwMMUCtx,
+                              IMG_UINT32 ui32FwVA,
+                              MMU_FAULT_DATA *psOutFaultData)
+{
+	IMG_UINT32 *pui32PageTable = NULL;
+	PVRSRV_ERROR eError = MMU_AcquireCPUBaseAddr(psFwMMUCtx, (void**) &pui32PageTable);
+	MMU_LEVEL_DATA *psMMULevelData;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32PageSize = OSGetPageSize();
+
+	/* MIPS Firmware CPU must use the same page size as the Host */
+	IMG_UINT32 ui32PTEIndex = ((ui32FwVA & ~(ui32PageSize - 1)) - ui32FwHeapBase) / ui32PageSize;
+
+	psOutFaultData->eTopLevel = MMU_LEVEL_1;
+	psOutFaultData->eType = MMU_FAULT_TYPE_NON_PM;
+
+	psMMULevelData = &psOutFaultData->sLevelData[MMU_LEVEL_1];
+	psMMULevelData->uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	psMMULevelData->ui32Index = ui32PTEIndex;
+	psMMULevelData->ui32NumOfEntries = RGX_FIRMWARE_RAW_HEAP_SIZE / ui32PageSize;
+
+	if ((eError == PVRSRV_OK) && (pui32PageTable != NULL))
+	{
+		psMMULevelData->ui64Address = pui32PageTable[ui32PTEIndex];
+	}
+	else
+	{
+		psMMULevelData->ui64Address = 0U;
+	}
+
+	psMMULevelData->psDebugStr = BITMASK_HAS(psMMULevelData->ui64Address,
+	                                         RGXMIPSFW_TLB_VALID) ?
+	                             ("valid") : ("not valid");
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.h
new file mode 100644
index 000000000000..b2b39402cafe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmipsmmuinit.h
@@ -0,0 +1,97 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation for the MIPS firmware
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* NB: this file is not to be included arbitrarily.  It exists solely
+   for the linkage between rgxinit.c and rgxmmuinit.c, the former
+   being otherwise cluttered by the contents of the latter */
+
+#ifndef SRVKM_RGXMIPSMMUINIT_H
+#define SRVKM_RGXMIPSMMUINIT_H
+
+#include "device.h"
+#include "img_types.h"
+#include "mmu_common.h"
+#include "img_defs.h"
+#include "rgx_mips.h"
+
+/*
+
+		Labelling of fields within virtual address. No PD and PC are used currently for
+		the MIPS MMU
+*/
+/*
+Page Table entry #
+*/
+#define RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_SHIFT        (12U)
+#define RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_CLRMSK       (IMG_UINT64_C(0XFFFFFFFF00000FFF))
+
+
+/* PC entries related definitions */
+/* No PC is currently used for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_EN            (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_SHIFT         (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_CLRMSK        (0U)
+
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_SHIFT     (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_CLRMSK    (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_EN        (0U)
+
+/* PD entries related definitions */
+/* No PD is currently used for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_EN            (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_SHIFT         (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_CLRMSK        (0U)
+
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_SHIFT     (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_CLRMSK    (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_EN        (0U)
+
+
+PVRSRV_ERROR RGXMipsMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR RGXMipsMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void RGXMipsCheckFaultAddress(MMU_CONTEXT *psFwMMUCtx,
+                              IMG_UINT32 ui32FwVA,
+                              MMU_FAULT_DATA *psOutFaultData);
+
+#endif /* #ifndef SRVKM_RGXMIPSMMUINIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmulticore.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmulticore.c
new file mode 100644
index 000000000000..447c50a78e00
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxmulticore.c
@@ -0,0 +1,328 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.c
+@Title          Functions related to multicore devices
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxdefs_km.h"
+#include "pdump_km.h"
+#include "rgxmulticore.h"
+#include "multicore_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwmemctx.h"
+
+/*
+ * check that register defines match our hardcoded definitions.
+ * Rogue has these, volcanic does not.
+ */
+#if ((RGX_MULTICORE_CAPABILITY_FRAGMENT_EN != RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_EN) || \
+     (RGX_MULTICORE_CAPABILITY_GEOMETRY_EN != RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_EN) || \
+     (RGX_MULTICORE_CAPABILITY_COMPUTE_EN  != RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_EN) || \
+     (RGX_MULTICORE_CAPABILITY_PRIMARY_EN  != RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN) || \
+     (RGX_MULTICORE_ID_CLRMSK              != RGX_CR_MULTICORE_GPU_ID_CLRMSK))
+#error "Rogue definitions for RGX_CR_MULTICORE_GPU register have changed"
+#endif
+
+
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores,
+                                        IMG_UINT64 *pui64Caps);
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Return multicore information to clients.
+ * Return not supported on cores without multicore.
+ */
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32CapsSize,
+                                 IMG_UINT32 *pui32NumCores,
+                                 IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDevInfo->ui32MultiCoreNumCores == 0)
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+	else
+	{
+		*pui32NumCores = psDevInfo->ui32MultiCoreNumCores;
+		if (ui32CapsSize > 0)
+		{
+			if (ui32CapsSize < psDevInfo->ui32MultiCoreNumCores)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore caps buffer too small"));
+				eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			}
+			else
+			{
+				IMG_UINT32 i;
+
+				for (i = 0; i < psDevInfo->ui32MultiCoreNumCores; ++i)
+				{
+					pui64Caps[i] = psDevInfo->pui64MultiCoreCapabilities[i];
+				}
+			}
+		}
+	}
+
+	return eError;
+}
+
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Read multicore HW registers and fill in data structure for clients.
+ * Return not supported on cores without multicore.
+ */
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		/* we only set this up once */
+		return PVRSRV_OK;
+	}
+
+	/* defaults for non-multicore devices */
+	psDevInfo->ui32MultiCoreNumCores = 0;
+	psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	psDevInfo->pui64MultiCoreCapabilities = NULL;
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		IMG_BOOL bPowerWasDown;
+		IMG_UINT32 ui32MulticoreRegBankOffset = (1 << RGX_GET_FEATURE_VALUE(psDevInfo, XPU_MAX_REGBANKS_ADDR_WIDTH));
+		IMG_UINT32 ui32MulticoreGPUReg = RGX_CR_MULTICORE_GPU;
+		IMG_UINT32 ui32NumCores;
+		IMG_UINT32 i;
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(XPU_MAX_REGBANKS_ADDR_WIDTH)
+		/* Ensure the HOST_SECURITY reg bank definitions are correct */
+		if ((RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE) != ui32MulticoreRegBankOffset)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Register bank definitions for HOST_SECURITY don't match core's configuration.", __func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+#endif
+		bPowerWasDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			PVRSRVPowerLock(psDeviceNode);
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVSetSystemPowerState ON failed (%u)", __func__, eError));
+				PVRSRVPowerUnlock(psDeviceNode);
+				return eError;
+			}
+		}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			IMG_UINT32 ui32FwTimeout = MAX_HW_TIME_US;
+
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores,
+		                               INVALIDATE);
+				if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores))
+				{
+					/* No need to wait if the FW has already updated the values */
+					break;
+				}
+				OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores) == 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore info not available for guest"));
+				return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+			}
+
+			ui32NumCores = psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores;
+
+			PVR_LOG(("RGX Guest Device initialised with %u %s",
+					 ui32NumCores, (ui32NumCores == 1U) ? "core" : "cores"));
+		}
+		else
+#endif
+		{
+			ui32NumCores = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM);
+		}
+#if !defined(NO_HARDWARE)
+		/* check that the number of cores reported is in-bounds */
+		if (ui32NumCores > (RGX_CR_MULTICORE_SYSTEM_MASKFULL >> RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "invalid return (%u) read from MULTICORE_SYSTEM", ui32NumCores));
+			return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+		}
+#else
+		/* for nohw set to max so clients can allocate enough memory for all pdump runs on any config */
+		ui32NumCores = RGX_MULTICORE_MAX_NOHW_CORES;
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "Multicore system has %u cores", ui32NumCores));
+		PDUMPCOMMENT(psDeviceNode, "RGX Multicore has %d cores\n", ui32NumCores);
+
+		/* allocate storage for capabilities */
+		psDevInfo->pui64MultiCoreCapabilities = OSAllocMem(ui32NumCores * sizeof(psDevInfo->pui64MultiCoreCapabilities[0]));
+		PVR_LOG_GOTO_IF_NOMEM(psDevInfo->pui64MultiCoreCapabilities, eError, err);
+
+		psDevInfo->ui32MultiCoreNumCores = ui32NumCores;
+
+		for (i = 0; i < ui32NumCores; ++i)
+		{
+	#if !defined(NO_HARDWARE)
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+			if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				psDevInfo->pui64MultiCoreCapabilities[i] = psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.aui64MultiCoreCapabilities[i];
+			}
+			else
+#endif
+			{
+				IMG_BOOL bMulticoreRegsMapped = (psDeviceNode->psDevConfig->ui32RegsSize > ui32MulticoreGPUReg);
+				void __iomem *pvCoreRegBase;
+				IMG_INT32 ui32MulticoreRegOffset;
+
+				if (bMulticoreRegsMapped)
+				{
+					pvCoreRegBase = psDevInfo->pvRegsBaseKM;
+					ui32MulticoreRegOffset = ui32MulticoreGPUReg;
+				}
+				else
+				{
+					/* the register bank of this core is not mapped */
+					IMG_CPU_PHYADDR sMultiCoreRegsBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+					sMultiCoreRegsBase.uiAddr += i*ui32MulticoreRegBankOffset;
+					pvCoreRegBase = (void __iomem *) OSMapPhysToLin(sMultiCoreRegsBase, psDeviceNode->psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+					PVR_LOG_GOTO_IF_NOMEM(pvCoreRegBase, eError, err);
+
+					/* adjust the register offset to point inside the newly mapped range */
+					ui32MulticoreRegOffset = RGX_CR_MULTICORE_GPU;
+				}
+
+				psDevInfo->pui64MultiCoreCapabilities[i] =
+							OSReadHWReg64(pvCoreRegBase, ui32MulticoreRegOffset) & RGX_CR_MULTICORE_GPU_MASKFULL;
+
+				if (!bMulticoreRegsMapped)
+				{
+					OSUnMapPhysToLin((void __force *) pvCoreRegBase, psDeviceNode->psDevConfig->ui32RegsSize);
+				}
+			}
+	#else
+			/* emulation for what we think caps are */
+			psDevInfo->pui64MultiCoreCapabilities[i] =
+							   i | ((i == 0) ? (RGX_MULTICORE_CAPABILITY_PRIMARY_EN
+											  | RGX_MULTICORE_CAPABILITY_GEOMETRY_EN) : 0)
+							   | RGX_MULTICORE_CAPABILITY_COMPUTE_EN
+							   | RGX_MULTICORE_CAPABILITY_FRAGMENT_EN;
+	#endif
+			PVR_DPF((PVR_DBG_MESSAGE, "Core %d has capabilities value 0x%x", i, (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[i] ));
+			PDUMPCOMMENT(psDeviceNode, "\tCore %d has caps 0x%08x\n", i,
+			             (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[i]);
+
+			if (psDevInfo->pui64MultiCoreCapabilities[i] & RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN)
+			{
+				psDevInfo->ui32MultiCorePrimaryId = (psDevInfo->pui64MultiCoreCapabilities[i]
+														& ~RGX_CR_MULTICORE_GPU_ID_CLRMSK)
+														>> RGX_CR_MULTICORE_GPU_ID_SHIFT;
+			}
+
+			ui32MulticoreGPUReg += ui32MulticoreRegBankOffset;
+		}
+
+        /* revert power state to what it was on entry to this function */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Register callback to return info about multicore setup to client bridge */
+		psDeviceNode->pfnGetMultiCoreInfo = RGXGetMultiCoreInfo;
+	}
+	else
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	return eError;
+
+err:
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+	return eError;
+}
+
+
+/*
+ * RGXDeinitMultiCoreInfo:
+ * Release resources and clear the MultiCore values in the DeviceNode.
+ */
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pui64MultiCoreCapabilities != NULL)
+	{
+		OSFreeMem(psDevInfo->pui64MultiCoreCapabilities);
+		psDevInfo->pui64MultiCoreCapabilities = NULL;
+		psDevInfo->ui32MultiCoreNumCores = 0;
+		psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	}
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxpdump.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxpdump.c
new file mode 100644
index 000000000000..409d901ff3a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxpdump.c
@@ -0,0 +1,776 @@
+/*************************************************************************/ /*!
+@File           rgxpdump.c
+@Title          Device specific pdump routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific pdump functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(PDUMP)
+#include "pvrsrv.h"
+#include "devicemem_utils.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+#include "rgxpdump.h"
+#include "rgxpdump_common.h"
+#include "rgx_bvnc_defs_km.h"
+#include "pdumpdesc.h"
+
+/*
+ * There are two different set of functions one for META/RISCV and one for MIPS
+ * because the Pdump player does not implement the support for
+ * the MIPS MMU yet. So for MIPS builds we cannot use DevmemPDumpSaveToFileVirtual,
+ * we have to use DevmemPDumpSaveToFile instead.
+ */
+static PVRSRV_ERROR _FWDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DECLARE_DLLIST(sFirmwareGcovBufferValidRegions);
+#endif
+	DECLARE_DLLIST(sSigTDMChecksValidRegions);
+	DECLARE_DLLIST(sSigTAChecksValidRegions);
+	DECLARE_DLLIST(sSig3DChecksValidRegions);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DECLARE_DLLIST(sSigRDMChecksValidRegions);
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psFirmwareGcovBufferMemDesc,
+	                             psDevInfo->ui32FirmwareGcovSize,
+	                             &sFirmwareGcovBufferValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrReturnError);
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                             psDevInfo->ui32SigTDMChecksSize,
+		                             &sSigTDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeGcovBufferRegions);
+	}
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                             psDevInfo->ui32SigTAChecksSize,
+	                             &sSigTAChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTDMChecksRegions);
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                             psDevInfo->ui32Sig3DChecksSize,
+	                             &sSig3DChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTAChecksRegions);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+	    RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                             psDevInfo->ui32SigRDMChecksSize,
+		                             &sSigRDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSig3DChecksRegions);
+	}
+#endif
+
+	PDUMPIF(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+	PDUMPELSE(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Gcov */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Gcov Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psFirmwareGcovBufferMemDesc,
+	                       &sFirmwareGcovBufferValidRegions,
+	                       "firmware_gcov.img",
+	                       ui32PDumpFlags);
+#endif
+
+	/* TDM signatures */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                       &sSigTDMChecksValidRegions,
+		                       "out.tdmsig",
+		                       ui32PDumpFlags);
+	}
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                       &sSigTAChecksValidRegions,
+	                       "out.tasig",
+	                       ui32PDumpFlags);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                       &sSig3DChecksValidRegions,
+	                       "out.3dsig",
+	                       ui32PDumpFlags);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		/* RDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump RDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                       &sSigRDMChecksValidRegions,
+		                       "out.rdmsig",
+		                       ui32PDumpFlags);
+	}
+#endif
+
+	PDUMPFI(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+	return PVRSRV_OK;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+ErrFreeSig3DChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSig3DChecksValidRegions);
+#endif
+ErrFreeSigTAChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTAChecksValidRegions);
+ErrFreeSigTDMChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTDMChecksValidRegions);
+ErrFreeGcovBufferRegions:
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DevmemIntPDumpFreeValidRegions(&sFirmwareGcovBufferValidRegions);
+ErrReturnError:
+#endif
+	return eError;
+}
+
+static PVRSRV_ERROR _FWDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32	ui32ThreadNum, ui32Size, ui32OutFileOffset;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufThreadNumOff =
+				(IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufThreadNumOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+								0, /* 0 offset in the trace buffer mem desc */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, sTraceBuf) /* move to first element of sTraceBuf */
+									+ ui32ThreadNum * sizeof(RGXFWIF_TRACEBUF_SPACE) /* skip required number of sTraceBuf elements */
+									+ offsetof(RGXFWIF_TRACEBUF_SPACE, sAssertBuf), /* offset into its sAssertBuf, to be pdumped */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset,
+								ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+	}
+
+	/* FW HWPerf buffer is always allocated when PDUMP is defined, irrespective of HWPerf events being enabled/disabled */
+	PVR_ASSERT(psDevInfo->psRGXFWIfHWPerfBufMemDesc);
+
+	/* Dump hwperf buffer */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	DevmemPDumpSaveToFileVirtual(psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+								 0,
+								 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+								 "out.hwperf",
+								 0,
+								 ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static PVRSRV_ERROR _MipsDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTAChecksMemDesc,
+								 0,
+								 psDevInfo->ui32SigTAChecksSize,
+								 "out.tasig",
+								 0);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSig3DChecksMemDesc,
+								 0,
+								 psDevInfo->ui32Sig3DChecksSize,
+								 "out.3dsig",
+								 0);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* TDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTDMChecksMemDesc,
+									 0,
+									 psDevInfo->ui32SigTDMChecksSize,
+									 "out.tdmsig",
+									 0);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MipsDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32		ui32ThreadNum, ui32Size, ui32OutFileOffset;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* Same again... */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufSpaceAssertBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF_SPACE *)0)->sAssertBuf);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+								0, /* 0 offset in the trace buffer mem desc */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff + uiTraceBufSpaceAssertBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+	}
+
+	/* Dump hwperf buffer */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+								 0,
+								 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+								 "out.hwperf",
+								 0);
+
+	return PVRSRV_OK;
+}
+#endif
+
+
+/*
+ * PVRSRVPDumpSignatureBufferKM
+ */
+PVRSRV_ERROR PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_UINT32			ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpSignatureBufferKM(psConnection,
+											  psDeviceNode,
+											  ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpSignatureBufferKM(psConnection,
+											psDeviceNode,
+											ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+
+
+PVRSRV_ERROR PVRSRVPDumpCRCSignatureCheckKM(CONNECTION_DATA * psConnection,
+                                            PVRSRV_DEVICE_NODE * psDeviceNode,
+                                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+/*
+ * PVRSRVPDumpValCheckPreCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPreCommandKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVPDumpValCheckPostCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPostCommandKM(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE * psDeviceNode,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR PVRSRVPDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputImageDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_PBYTE pbyPDumpImageHdr)
+{
+	IMG_PUINT32 pui32Word;
+	IMG_UINT32 ui32HeaderDataSize;
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK) || defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK) || defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32TFBCControl = (psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >>
+																					  RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+#endif
+
+	/* Validate parameters */
+	if (((IMAGE_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((IMAGE_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	memset(pbyPDumpImageHdr, 0, IMAGE_HEADER_SIZE);
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpImageHdr, 0);
+	pui32Word[0] = (IMAGE_HEADER_TYPE << HEADER_WORD0_TYPE_SHIFT);
+	pui32Word[1] = (IMAGE_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+				   (IMAGE_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+
+	ui32HeaderDataSize = ui32DataSize;
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		ui32HeaderDataSize += ui32HeaderSize;
+	}
+	pui32Word[2] = ui32HeaderDataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+	pui32Word[3] = ui32LogicalWidth << IMAGE_HEADER_WORD3_LOGICAL_WIDTH_SHIFT;
+	pui32Word[4] = ui32LogicalHeight << IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_SHIFT;
+
+	pui32Word[5] = ePixFmt << IMAGE_HEADER_WORD5_FORMAT_SHIFT;
+
+	pui32Word[6] = ui32PhysicalWidth << IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_SHIFT;
+	pui32Word[7] = ui32PhysicalHeight << IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_SHIFT;
+
+	pui32Word[8] = IMAGE_HEADER_WORD8_STRIDE_POSITIVE | IMAGE_HEADER_WORD8_BIFTYPE_NONE;
+
+	switch (eMemLayout)
+	{
+	case IMG_MEMLAYOUT_STRIDED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_STRIDED;
+		break;
+	case IMG_MEMLAYOUT_TWIDDLED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_NTWIDDLE;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported memory layout - %d", eMemLayout));
+		return PVRSRV_ERROR_UNSUPPORTED_MEMORY_LAYOUT;
+	}
+
+	pui32Word[9] = 0;
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		switch (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, FBCDC_ALGORITHM))
+		{
+		case 1:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_BASE;
+			break;
+		case 2:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V2;
+			break;
+		case 3:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT2;
+			break;
+		case 4:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V4;
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_ON;
+			}
+
+			pui32Word[9] |= (eFBCSwizzle << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT) & IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK;
+
+			break;
+		case 50:
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_TFBC;
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_25;
+			}
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_37;
+			}
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_50;
+			}
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_75;
+			}
+
+			pui32Word[9] |= (eFBCSwizzle << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT) & IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK;
+
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Unsupported algorithm - %d",
+					PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, FBCDC_ALGORITHM)));
+			return PVRSRV_ERROR_NOT_ENABLED;
+		}
+	}
+
+	switch (GET_FBCDC_BLOCK_TYPE(eFBCompression))
+	{
+	case IMG_FB_COMPRESSION_NONE:
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_8x8:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_8X8;
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_16x4:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_16x4;
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_32x2:
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported compression mode - %d", eFBCompression));
+		return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+	}
+
+	pui32Word[10] = paui32FBCClearColour[0];
+	pui32Word[11] = paui32FBCClearColour[1];
+	pui32Word[12] = paui32FBCClearColour[2];
+	pui32Word[13] = paui32FBCClearColour[3];
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP */
+		IMG_UINT32 ui32TFBCGroup  = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK;
+		switch (ui32TFBCGroup)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_0:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_1:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_37_50;
+				break;
+		}
+	}
+	else
+#endif
+	{
+		pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+	}
+
+#if defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME */
+		IMG_UINT32 ui32TFBCScheme = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK;
+		switch (ui32TFBCScheme)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_DEFAULT:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_ALL;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD_AND_CORRELATION:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_CORR;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR, "Unsupported TFBC compression control scheme - %d", ui32TFBCScheme));
+				return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+		}
+	}
+	else
+#endif
+	{
+		/* Should always be set to 2 ("TFBC delta standard only") on cores without this feature */
+		pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+	}
+
+#if defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_NATIVE_YUV10))
+	{
+		IMG_UINT32 ui32TFBCOverrideYUV10 = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN);
+
+		if (ui32TFBCOverrideYUV10)
+		{
+			pui32Word[14] |= IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_EN;
+		}
+	}
+#endif
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, TFBC_VERSION) >= 20U)
+		{
+			IMG_UINT32 ui32TFBCOverrideLossyMinChannel = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN);
+
+			if (ui32TFBCOverrideLossyMinChannel)
+			{
+				pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+			}
+		}
+		else
+		{
+			/* Should be set on TFBC version < 2.0 cores */
+			pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputDataDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_PBYTE pbyPDumpDataHdr)
+{
+	IMG_PUINT32 pui32Word;
+
+	/* Validate parameters */
+	if (((DATA_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((DATA_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpDataHdr, 0);
+
+	if (ui32HeaderType == DATA_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (DATA_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(DATA_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+		pui32Word[3] = ui32ElementType << DATA_HEADER_WORD3_ELEMENT_TYPE_SHIFT;
+		pui32Word[4] = ui32ElementCount << DATA_HEADER_WORD4_ELEMENT_COUNT_SHIFT;
+	}
+
+	if (ui32HeaderType == IBIN_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (IBIN_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(IBIN_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/******************************************************************************
+ End of file (rgxpdump.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxsrvinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxsrvinit.c
new file mode 100644
index 000000000000..77d63732e4e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxsrvinit.c
@@ -0,0 +1,1711 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvinit.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "km_apphint_defs.h"
+#include "htbuffer_types.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+
+#include "rgxinit_apphints.h"
+#include "rgxinit.h"
+#include "rgxmulticore.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "osfunc.h"
+
+#include "rgxdefs_km.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "rgx_fwif_hwperf.h"
+#include "rgx_hwperf_table.h"
+
+#include "fwload.h"
+#include "rgxlayer_impl.h"
+#include "rgxfwimageutils.h"
+#include "rgxfwutils.h"
+
+#include "rgx_hwperf.h"
+#include "rgx_bvnc_defs_km.h"
+
+#include "rgxdevice.h"
+
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#include "rgxdevice.h"
+#include "pvrsrv_device.h"
+#endif
+
+#define	HW_PERF_FILTER_DEFAULT         0x00000000 /* Default to no HWPerf */
+#define HW_PERF_FILTER_DEFAULT_ALL_ON  0xFFFFFFFF /* All events */
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+#define AVAIL_POW_UNITS_MASK_DEFAULT   (PVRSRV_APPHINT_HWVALAVAILABLESPUMASK)
+#define AVAIL_RAC_MASK_DEFAULT         (PVRSRV_APPHINT_HWVALAVAILABLERACMASK)
+#endif
+
+/* Kernel CCB size */
+
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE 4
+#endif
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE 16
+#endif
+
+#if PVRSRV_APPHINT_KCCB_SIZE_LOG2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too low.
+#elif PVRSRV_APPHINT_KCCB_SIZE_LOG2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too high.
+#endif
+
+
+#include "os_apphint.h"
+
+/*
+ * _ParseHTBAppHints:
+ *
+ * Generate necessary references to the globally visible AppHints which are
+ * declared in the above #include "km_apphint_defs.h"
+ * Without these local references some compiler tool-chains will treat
+ * unreferenced declarations as fatal errors. This function duplicates the
+ * HTB_specific apphint references which are made in htbserver.c:HTBInit()
+ * However, it makes absolutely *NO* use of these hints.
+ */
+static void
+_ParseHTBAppHints(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvAppHintState)
+{
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32OpMode;
+	IMG_UINT32 ui32BufferSize;
+
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEHTBLOGGROUP;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, EnableHTBLogGroup,
+	                     &ui32AppHintDefault, &ui32LogType);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBOPERATIONMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBOperationMode,
+	                     &ui32AppHintDefault, &ui32OpMode);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBUFFERSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+	                     &ui32AppHintDefault, &ui32BufferSize);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetFilterFlags
+
+ @Description   Initialise and return filter flags
+
+ @Input         bFilteringMode : Enable new TPU filtering mode
+ @Input         ui32TruncateMode : TPU Truncate mode
+
+ @Return        IMG_UINT32 : Filter flags
+
+******************************************************************************/
+static INLINE IMG_UINT32 GetFilterFlags(IMG_BOOL bFilteringMode, IMG_UINT32 ui32TruncateMode)
+{
+	IMG_UINT32 ui32FilterFlags = 0;
+
+	ui32FilterFlags |= bFilteringMode ? RGXFWIF_FILTCFG_NEW_FILTER_MODE : 0;
+	if (ui32TruncateMode == 2)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_INT;
+	}
+	else if (ui32TruncateMode == 3)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_HALF;
+	}
+
+	return ui32FilterFlags;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      InitDeviceFlags
+
+ @Description   Initialise and return device flags
+
+ @Input         psDeviceNode     : Pointer to device node
+ @Input         pvAppHintState   : Pointer to apphint state
+ @Input         psHints          : Apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void InitDeviceFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   void *pvAppHintState,
+                                   RGX_INIT_APPHINTS *psHints)
+{
+	IMG_UINT32 ui32DeviceFlags = 0;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bZeroFreelist;
+	IMG_BOOL bDisableFEDLogging;
+
+
+	bAppHintDefault = PVRSRV_APPHINT_ZEROFREELIST;
+	OSGetAppHintBOOL(psDeviceNode,  pvAppHintState,  ZeroFreelist,
+	                   &bAppHintDefault,         &bZeroFreelist);
+	ui32DeviceFlags |= bZeroFreelist ? RGXKM_DEVICE_STATE_ZERO_FREELIST : 0;
+
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEFEDLOGGING;
+	OSGetAppHintBOOL(psDeviceNode,    pvAppHintState,    DisableFEDLogging,
+	                   &bAppHintDefault,        &bDisableFEDLogging);
+	ui32DeviceFlags |= bDisableFEDLogging ? RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN : 0;
+
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	BITMASK_SET(ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+#endif
+
+	psHints->ui32DeviceFlags = ui32DeviceFlags;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetApphints
+
+ @Description   Read init time apphints and initialise apphints structure
+
+ @Input         psHints : Pointer to apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetApphints(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_INIT_APPHINTS *psHints)
+{
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+	IMG_UINT32 ui32ParamTemp;
+#if defined(__linux__)
+	IMG_UINT64 ui64AppHintDefault;
+#endif
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESIGNATURECHECKS;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSignatureChecks,
+	                     &bAppHintDefault,    &psHints->bEnableSignatureChecks);
+	ui32AppHintDefault = PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    SignatureChecksBufSize,
+	                     &ui32AppHintDefault,    &psHints->ui32SignatureChecksBufSize);
+
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTOUTOFMEMORY;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOutOfMemory,
+	                   &bAppHintDefault,        &psHints->bAssertOnOutOfMem);
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTONHWRTRIGGER;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOnHWRTrigger,
+	                   &bAppHintDefault,        &psHints->bAssertOnHWRTrigger);
+	bAppHintDefault = PVRSRV_APPHINT_CHECKMLIST;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    CheckMList,
+	                   &bAppHintDefault,        &psHints->bCheckMlist);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLECLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableClockGating,
+	                   &bAppHintDefault,        &psHints->bDisableClockGating);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEDMOVERLAP;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableDMOverlap,
+	                   &bAppHintDefault,        &psHints->bDisableDMOverlap);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEAPM;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    EnableAPM,
+	                     &ui32AppHintDefault,      &ui32ParamTemp);
+	psHints->eRGXActivePMConf = ui32ParamTemp;
+	bAppHintDefault = PVRSRV_APPHINT_ENABLECDMKILLINGRANDMODE;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableCDMKillingRandMode,
+	                   &bAppHintDefault,      &psHints->bEnableDMKillRand);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableRandomContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableRandomCsw);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSoftResetContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableSoftResetCsw);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableFWContextSwitch,
+	                     &ui32AppHintDefault,    &psHints->ui32EnableFWContextSwitch);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLERDPOWERISLAND;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableRDPowerIsland,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eRGXRDPowerIslandConf = ui32ParamTemp;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESPUCLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    EnableSPUClockGating,
+	                   &bAppHintDefault,           &psHints->bSPUClockGating);
+#endif
+	ui32AppHintDefault = PVRSRV_APPHINT_FIRMWAREPERF;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FirmwarePerf,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eFirmwarePerf = ui32ParamTemp;
+	ui32AppHintDefault = PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FWContextSwitchProfile,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchProfile);
+
+	/*
+	 * HWPerf apphints *
+	 */
+	bAppHintDefault = PVRSRV_APPHINT_HWPERFDISABLECUSTOMCOUNTERFILTER;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    HWPerfDisableCustomCounterFilter,
+	                     &bAppHintDefault,    &psHints->bHWPerfDisableCounterFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfHostBufSize);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfFWBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfFWBufSize);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_KCCB_SIZE_LOG2;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,  KernelCCBSizeLog2,
+	                     &ui32AppHintDefault,    &psHints->ui32KCCBSizeLog2);
+
+	if (psHints->ui32KCCBSizeLog2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too low, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE;
+	}
+	else if (psHints->ui32KCCBSizeLog2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too high, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE;
+	}
+
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AppHintDefault = PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  ISPSchedulingLatencyMode,
+	                     &ui32AppHintDefault,  &psHints->ui32ISPSchedulingLatencyMode);
+#endif
+
+#if defined(__linux__)
+	/* name changes */
+	{
+		IMG_UINT64 ui64Tmp;
+		bAppHintDefault = PVRSRV_APPHINT_DISABLEPDUMPPANIC;
+		OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,  DisablePDumpPanic,
+		                   &bAppHintDefault,           &psHints->bDisablePDP);
+		ui64AppHintDefault = PVRSRV_APPHINT_HWPERFFWFILTER;
+		OSGetAppHintUINT64(psDevInfo->psDeviceNode,  pvAppHintState,  HWPerfFWFilter,
+		                   &ui64AppHintDefault,        &ui64Tmp);
+		psHints->ui32HWPerfFilter0 = (IMG_UINT32)(ui64Tmp & 0xffffffffllu);
+		psHints->ui32HWPerfFilter1 = (IMG_UINT32)((ui64Tmp >> 32) & 0xffffffffllu);
+	}
+#endif
+
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTFILTER;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWPerfHostFilter,
+	                     &ui32AppHintDefault,          &psHints->ui32HWPerfHostFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    SecondaryOSClockSource,
+	                         &ui32AppHintDefault,      &psHints->ui32SecondaryOSClockSource);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWRDebugDumpLimit,
+	                     &ui32AppHintDefault,          &ui32ParamTemp);
+	psHints->ui32HWRDebugDumpLimit = MIN(ui32ParamTemp, RGXFWIF_HWR_DEBUG_DUMP_ALL);
+
+	{
+		IMG_BOOL bFilteringMode = IMG_FALSE;
+		IMG_UINT32 ui32TruncateMode = 0U;
+
+#if defined(HW_ERN_42290_BIT_MASK) && defined(RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42290) && RGX_IS_FEATURE_SUPPORTED(psDevInfo, TPU_FILTERING_MODE_CONTROL))
+#endif
+		{
+			bAppHintDefault = PVRSRV_APPHINT_NEWFILTERINGMODE;
+			OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  NewFilteringMode,
+			                   &bAppHintDefault,  &bFilteringMode);
+		}
+
+#if defined(HW_ERN_42606_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42606))
+#endif
+		{
+			ui32AppHintDefault = PVRSRV_APPHINT_TRUNCATEMODE;
+			OSGetAppHintUINT32(APPHINT_NO_DEVICE,  pvAppHintState,  TruncateMode,
+			                     &ui32AppHintDefault,  &ui32TruncateMode);
+		}
+
+		psHints->ui32FilterFlags = GetFilterFlags(bFilteringMode, ui32TruncateMode);
+	}
+
+#if defined(EMULATOR)
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#endif
+	{
+		bAppHintDefault = PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG;
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  EnableTrustedDeviceAceConfig,
+		                   &bAppHintDefault,     &psHints->bEnableTrustedDeviceAceConfig);
+	}
+#endif
+
+#if defined(__linux__)
+	ui32AppHintDefault = 0;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  FWContextSwitchCrossDM,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchCrossDM);
+#endif
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	ui32AppHintDefault = PVRSRV_APPHINT_PHYSMEMTESTPASSES;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  PhysMemTestPasses,
+	                     &ui32AppHintDefault,    &psHints->ui32PhysMemTestPasses);
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	psHints->ui32AvailablePowUnitsMask = AVAIL_POW_UNITS_MASK_DEFAULT;
+	psHints->ui32AvailableRACMask = AVAIL_RAC_MASK_DEFAULT;
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	ui32AppHintDefault = PVRSRV_APPHINT_TFBCVERSION;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, TFBCVersionDowngrade,
+	 &ui32AppHintDefault, &psHints->ui32TFBCVersion);
+
+	if (ui32AppHintDefault != psHints->ui32TFBCVersion)
+	{
+		PVR_LOG(("TFBCVersionDowngrade set to %u", psHints->ui32TFBCVersion));
+	}
+
+	psHints->bTFBCCompressionControlLossyMinChannel = false;
+	psHints->bTFBCCompressionControlYUVFormat = false;
+	psHints->ui32TFBCCompressionControlScheme =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME;
+	psHints->ui32TFBCCompressionControlGroup =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	ui32AppHintDefault = PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  DebugDumpFWTLogType,
+	                     &ui32AppHintDefault,  &psHints->eDebugDumpFWTLogType);
+	if ((IMG_UINT32)psHints->eDebugDumpFWTLogType > RGX_FWT_LOGTYPE_PARTIAL)
+	{
+		psHints->eDebugDumpFWTLogType = RGX_FWT_LOGTYPE_NONE;
+		PVR_DPF((PVR_DBG_WARNING, "Invalid value for DebugDumpFWTLogType. Setting to 0 (disabled)."));
+	}
+
+	/*
+	 * FW logs apphints
+	 */
+	{
+		IMG_UINT32 ui32LogGroup, ui32TraceOrTBI;
+
+		ui32AppHintDefault = PVRSRV_APPHINT_ENABLELOGGROUP;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, EnableLogGroup, &ui32AppHintDefault, &ui32LogGroup);
+		ui32AppHintDefault = PVRSRV_APPHINT_FIRMWARELOGTYPE;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, FirmwareLogType, &ui32AppHintDefault, &ui32TraceOrTBI);
+
+		/* Defaulting to TRACE */
+		BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+		if (ui32TraceOrTBI == 1 /* TBI */)
+		{
+			if ((ui32LogGroup & RGXFWIF_LOG_TYPE_GROUP_MASK) == 0)
+			{
+				/* No groups configured - defaulting to MAIN group */
+				BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_GROUP_MAIN);
+			}
+			BITMASK_UNSET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+		}
+#endif
+		psHints->ui32LogType = ui32LogGroup;
+	}
+
+	_ParseHTBAppHints(psDevInfo->psDeviceNode, pvAppHintState);
+
+	InitDeviceFlags(psDevInfo->psDeviceNode, pvAppHintState, psHints);
+
+	OSFreeAppHintState(pvAppHintState);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      GetFWConfigFlags
+
+ @Description   Initialise and return FW config flags
+
+ @Input         psHints            : Apphints container
+ @Input         pui32FWConfigFlags : Pointer to config flags
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetFWConfigFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    RGX_INIT_APPHINTS *psHints,
+                                    IMG_UINT32 *pui32FWConfigFlags,
+                                    IMG_UINT32 *pui32FWConfigFlagsExt,
+                                    IMG_UINT32 *pui32FwOsCfgFlags)
+{
+	IMG_UINT32 ui32FWConfigFlags = 0;
+	IMG_UINT32 ui32FWConfigFlagsExt = 0;
+	IMG_UINT32 ui32FwOsCfgFlags = psHints->ui32FWContextSwitchCrossDM |
+	                              (psHints->ui32EnableFWContextSwitch & ~RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK);
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32FWConfigFlagsSupValExt = 0;
+	IMG_UINT32 ui32TFBCVersion = 0U;
+#endif
+
+	ui32FWConfigFlags |= psHints->bAssertOnOutOfMem ? RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY : 0;
+	ui32FWConfigFlags |= psHints->bAssertOnHWRTrigger ? RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER : 0;
+	ui32FWConfigFlags |= psHints->bCheckMlist ? RGXFWIF_INICFG_CHECK_MLIST_EN : 0;
+	ui32FWConfigFlags |= psHints->bDisableClockGating ? RGXFWIF_INICFG_DISABLE_CLKGATING_EN : 0;
+
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PIPELINED_DATAMASTERS_VERSION)  &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PIPELINED_DATAMASTERS_VERSION) > 0) &&
+		!RGX_IS_FEATURE_SUPPORTED(psDevInfo, ERYX_TOP_INFRASTRUCTURE))
+	{
+		/* Pipeline DM roadblocks are currently enabled pre-Eryx. */
+		ui32FWConfigFlags |= RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN;
+	}
+#endif
+
+	ui32FWConfigFlags |= psHints->bDisableDMOverlap ? RGXFWIF_INICFG_DISABLE_DM_OVERLAP : 0;
+
+	if ((RGX_GET_FEATURE_VALUE(psDevInfo, SLC_SIZE_IN_KILOBYTES) <= 2))
+	{
+		ui32FWConfigFlags |= RGXFWIF_INICFG_DISABLE_DM_OVERLAP;
+	}
+
+	ui32FWConfigFlags |= psHints->bDisablePDP ? RGXFWIF_INICFG_DISABLE_PDP_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableDMKillRand ? RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableRandomCsw ? RGXFWIF_INICFG_CTXSWITCH_MODE_RAND : 0;
+	ui32FWConfigFlags |= psHints->bEnableSoftResetCsw ? RGXFWIF_INICFG_CTXSWITCH_SRESET_EN : 0;
+	ui32FWConfigFlags |= (psHints->ui32HWPerfFilter0 != 0 || psHints->ui32HWPerfFilter1 != 0) ? RGXFWIF_INICFG_HWPERF_EN : 0;
+	ui32FWConfigFlags |= psHints->bHWPerfDisableCounterFilter ? RGXFWIF_INICFG_HWP_DISABLE_FILTER : 0;
+	ui32FWConfigFlags |= (psHints->ui32FWContextSwitchProfile << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT) & RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK;
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32FWConfigFlags |= (psHints->ui32ISPSchedulingLatencyMode << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT) & RGXFWIF_INICFG_ISPSCHEDMODE_MASK;
+#endif
+
+	{
+		ui32FWConfigFlags |= psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode) ? RGXFWIF_INICFG_FBCDC_V3_1_EN : 0;
+	}
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+
+	/* Determine if we need to present a TFBC v1.0, v1.1 or native
+	 * behaviour. For V1.0 we need to set the following features:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x1
+	 *  TFBCCompressionControlScheme = 0x2
+	 *  TFBCCompressionControlGroup = 0x0
+	 * For V1.1 we need to set the following:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x0
+	 *  TFBCCompressionControlScheme = 0x1
+	 *  TFBCCompressionControlGroup = 0 / 1 (depends on LOSSY_37_PERCENT)
+	 * The gating for these values depends on whether the GPU supports
+	 * RGX_FEATURE_TFBC_VERSION = 20U
+	 */
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		ui32TFBCVersion = RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION);
+
+		if (ui32TFBCVersion >= 20U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 11:	/* TFBC Version 1.1 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = false;
+					psHints->ui32TFBCCompressionControlScheme = 1U;
+					psHints->ui32TFBCCompressionControlGroup =
+					    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.1, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+					         " Ignoring. Using value %u instead",
+					         __func__, psHints->ui32TFBCVersion,
+					         ui32TFBCVersion));
+					break;
+			}
+
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else if (ui32TFBCVersion == 11U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+				             " Ignoring. Using value %u instead",
+				             __func__, psHints->ui32TFBCVersion,
+				             ui32TFBCVersion));
+					break;
+			}
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else	/* TFBC v1.0 */
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32FWConfigFlagsSupValExt);
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = 0;
+#endif
+			if ((psHints->ui32TFBCVersion != 0U) &&
+			    (psHints->ui32TFBCVersion != ui32TFBCVersion))
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s: Cannot specify TFBC version %u"
+				         " on a version %u GPU core", __func__,
+				         psHints->ui32TFBCVersion, ui32TFBCVersion));
+			}
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	*pui32FWConfigFlags    = ui32FWConfigFlags;
+	*pui32FWConfigFlagsExt = ui32FWConfigFlagsExt;
+	*pui32FwOsCfgFlags     = ui32FwOsCfgFlags;
+}
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+/*!
+*******************************************************************************
+
+ @Function      RGXTDProcessFWImage
+
+ @Description   Fetch and send data used by the trusted device to complete
+                the FW image setup
+
+ @Input         psDeviceNode : Device node
+ @Input         psFWParams : Firmware and parameters used by the FW
+
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXTDProcessFWImage(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        PVRSRV_FW_PARAMS *psFWParams)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+	PVRSRV_ERROR eError;
+
+	if (psDevConfig->pfnTDSendFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TDSendFWImage not implemented!", __func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		if (psFWParams->uFWP.sMips.ui32FWPageTableNumPages > TD_MAX_NUM_MIPS_PAGETABLE_PAGES)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Number of page table pages %u greater "
+					 "than what is allowed by the TD interface (%u), FW might "
+					 "not work properly!", __func__,
+					 psFWParams->uFWP.sMips.ui32FWPageTableNumPages,
+					 TD_MAX_NUM_MIPS_PAGETABLE_PAGES));
+		}
+	}
+#endif
+
+	eError = psDevConfig->pfnTDSendFWImage(psDevConfig->hSysData, psFWParams);
+
+	return eError;
+}
+#endif
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXAcquireMipsBootldrData
+
+ @Description   Acquire MIPS bootloader data parameters
+
+ @Input         psDeviceNode : Device node
+ @Input         puFWParams   : FW boot parameters
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireMipsBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+	MMU_DEVICEATTRIBS *psFWMMUDevAttrs = psDevInfo->psDeviceNode->psFirmwareMMUDevAttrs;
+	IMG_DEV_PHYADDR sAddr;
+	IMG_UINT32 ui32PTSize, i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bValid;
+
+	/* Rogue Registers physical address */
+#if defined(SUPPORT_ALT_REGBASE)
+	puFWParams->sMips.sGPURegAddr = psDeviceNode->psDevConfig->sAltRegsGpuPBase;
+#else
+	PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+	                           1,
+	                           &puFWParams->sMips.sGPURegAddr,
+	                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+#endif
+
+	/* MIPS Page Table physical address */
+	MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sAddr);
+
+	/* MIPS Page Table allocation is contiguous. Pass one or more addresses
+	 * to the FW depending on the Page Table size and alignment. */
+
+	ui32PTSize = (psFWMMUDevAttrs->psTopLevelDevVAddrConfig->uiNumEntriesPT)
+	             << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	ui32PTSize = PVR_ALIGN(ui32PTSize, 1U << psFWMMUDevAttrs->ui32BaseAlign);
+
+	puFWParams->sMips.ui32FWPageTableLog2PageSize = psFWMMUDevAttrs->ui32BaseAlign;
+	puFWParams->sMips.ui32FWPageTableNumPages = ui32PTSize >> psFWMMUDevAttrs->ui32BaseAlign;
+
+	if (puFWParams->sMips.ui32FWPageTableNumPages > 4U)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Page table cannot be mapped by the FW "
+		         "(size 0x%x, log2 page size %u, %u pages)",
+		         __func__, ui32PTSize, puFWParams->sMips.ui32FWPageTableLog2PageSize,
+		         puFWParams->sMips.ui32FWPageTableNumPages));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Confirm page alignment fits in 64-bits */
+	if (psFWMMUDevAttrs->ui32BaseAlign > 63)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid page alignment "
+		         "(psFWMMUDevAttrs->ui32BaseAlign = %u)",
+		         __func__, psFWMMUDevAttrs->ui32BaseAlign));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	for (i = 0; i < puFWParams->sMips.ui32FWPageTableNumPages; i++)
+	{
+		puFWParams->sMips.asFWPageTableAddr[i].uiAddr =
+		    sAddr.uiAddr + i * (1ULL << psFWMMUDevAttrs->ui32BaseAlign);
+	}
+
+	/* MIPS Stack Pointer Physical Address */
+	eError = RGXGetPhyAddr(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+	                       &puFWParams->sMips.sFWStackAddr,
+	                       RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+	                       OSGetPageShift(),
+	                       1,
+	                       &bValid);
+
+	return eError;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      InitFirmware
+
+ @Description   Allocate, initialise and pdump Firmware code and data memory
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitFirmware(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	OS_FW_IMAGE       *psRGXFW = NULL;
+	PVRSRV_FW_PARAMS  sFWParams = {0};
+
+	/* FW code memory */
+	IMG_DEVMEM_SIZE_T uiFWCodeAllocSize;
+	void              *pvFWCodeHostAddr;
+
+	/* FW data memory */
+	IMG_DEVMEM_SIZE_T uiFWDataAllocSize;
+	void              *pvFWDataHostAddr;
+
+	/* FW coremem code memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememCodeAllocSize;
+	void              *pvFWCorememCodeHostAddr = NULL;
+
+	/* FW coremem data memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememDataAllocSize;
+	void              *pvFWCorememDataHostAddr = NULL;
+
+	PVRSRV_FW_BOOT_PARAMS uFWParams;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_BOOL bUseSecureFWData =
+								RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	                            (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+	                             RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32) ||
+#endif
+	                            RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR);
+#endif
+
+	/*
+	 * Get pointer to Firmware image
+	 */
+	eError = RGXLoadAndGetFWData(psDeviceNode, &psRGXFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Error or confirmation message generated in RGXLoadAndGetFWData */
+		goto fw_load_fail;
+	}
+
+	/*
+	 * Get pointer and size
+	 */
+	sFWParams.pvFirmware       = OSFirmwareData(psRGXFW);
+	sFWParams.ui32FirmwareSize = OSFirmwareSize(psRGXFW);
+
+	/*
+	 * Allow it to be pre-processed by the platform hook
+	 */
+	if (psDeviceNode->psDevConfig->pfnPrepareFWImage != NULL)
+	{
+		eError = psDeviceNode->psDevConfig->pfnPrepareFWImage(
+				psDeviceNode->psDevConfig->hSysData, &sFWParams);
+		if (eError != PVRSRV_OK)
+			goto cleanup_initfw;
+	}
+
+	/*
+	 * Allocate Firmware memory
+	 */
+
+	eError = RGXGetFWImageAllocSize(&psDevInfo->sLayerParams,
+	                                sFWParams.pvFirmware,
+	                                sFWParams.ui32FirmwareSize,
+	                                &uiFWCodeAllocSize,
+	                                &uiFWDataAllocSize,
+	                                &uiFWCorememCodeAllocSize,
+	                                &uiFWCorememDataAllocSize,
+	                                &psDevInfo->sFWInfoHeader);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXGetFWImageAllocSize failed",
+		        __func__));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Initiate FW compatibility check for Native and Host.
+	 * Guest compatibility check must be done after FW boot.
+	 */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed compatibility check for device %p (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+	}
+
+	psDevInfo->ui32FWCodeSizeInBytes = uiFWCodeAllocSize;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+    PVR_DPF((PVR_DBG_WARNING,
+            "%s: META DMA not available, disabling core memory code/data",
+            __func__));
+    uiFWCorememCodeAllocSize = 0;
+    uiFWCorememDataAllocSize = 0;
+#endif
+
+	psDevInfo->ui32FWCorememCodeSizeInBytes = uiFWCorememCodeAllocSize;
+
+	eError = RGXInitAllocFWImgMem(psDeviceNode,
+	                              uiFWCodeAllocSize,
+	                              uiFWDataAllocSize,
+	                              uiFWCorememCodeAllocSize,
+	                              uiFWCorememDataAllocSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXInitAllocFWImgMem failed (%d)",
+		        __func__,
+		        eError));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Acquire pointers to Firmware allocations
+	 */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCodeHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, &pvFWCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", cleanup_initfw);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWDataHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc, &pvFWDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_code);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCorememCodeHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, &pvFWCorememCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_data);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		pvFWCorememDataHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememDataAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc, &pvFWCorememDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_corememcode);
+	}
+
+	/*
+	 * Prepare FW boot parameters
+	 */
+	OSCachedMemSet(&uFWParams, 0, sizeof(PVRSRV_FW_BOOT_PARAMS));
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		uFWParams.sMeta.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sMeta.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeFWAddr = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sMeta.uiFWCorememCodeSize = uiFWCorememCodeAllocSize;
+		uFWParams.sMeta.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sMeta.sFWCorememDataFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+		uFWParams.sMeta.ui32NumThreads = 2;
+#else
+		uFWParams.sMeta.ui32NumThreads = 1;
+#endif
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = RGXAcquireMipsBootldrData(psDeviceNode, &uFWParams);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXAcquireMipsBootldrData failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+	}
+#endif
+	else
+	{
+		uFWParams.sRISCV.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+
+		uFWParams.sRISCV.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememCodeFWAddr   = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sRISCV.uiFWCorememCodeSize    = uiFWCorememCodeAllocSize;
+
+		uFWParams.sRISCV.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememDataFWAddr   = psDevInfo->sFWCorememDataStoreFWAddr;
+		uFWParams.sRISCV.uiFWCorememDataSize    = uiFWCorememDataAllocSize;
+	}
+
+
+	/*
+	 * Process the Firmware image and setup code and data segments.
+	 *
+	 * When the trusted device is enabled and the FW code lives
+	 * in secure memory we will only setup the data segments here,
+	 * while the code segments will be loaded to secure memory
+	 * by the trusted device.
+	 */
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = RGXProcessFWImage(&psDevInfo->sLayerParams,
+								   sFWParams.pvFirmware,
+								   pvFWCodeHostAddr,
+								   pvFWDataHostAddr,
+								   pvFWCorememCodeHostAddr,
+								   pvFWCorememDataHostAddr,
+								   &uFWParams);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXProcessFWImage failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+		RGXFwSharedMemCacheOpExec(pvFWCodeHostAddr,
+		                          sizeof(psDevInfo->psRGXFWCodeMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememCodeAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememCodeHostAddr,
+			                          sizeof(psDevInfo->psRGXFWCorememCodeMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+
+		RGXFwSharedMemCacheOpExec(pvFWDataHostAddr,
+		                          sizeof(psDevInfo->psRGXFWDataMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememDataAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememDataHostAddr,
+			                          sizeof(psDevInfo->psRGXFWIfCorememDataStoreMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+#endif
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	sFWParams.uFWP = uFWParams;
+	RGXTDProcessFWImage(psDeviceNode, &sFWParams);
+#endif
+
+
+	/*
+	 * PDump Firmware allocations
+	 */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Dump firmware code image");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWCodeMemDesc,
+	                   0,
+	                   uiFWCodeAllocSize,
+	                   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware data image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWDataMemDesc,
+		                   0,
+		                   uiFWDataAllocSize,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem code image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWCorememCodeMemDesc,
+						   0,
+						   uiFWCorememCodeAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem data store image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+						   0,
+						   uiFWCorememDataAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	/*
+	 * Release Firmware allocations and clean up
+	 */
+release_fw_allocations:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+	}
+
+release_corememcode:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && (uiFWCorememCodeAllocSize != 0))
+#else
+	if (uiFWCorememCodeAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+
+release_data:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+	}
+
+release_code:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+cleanup_initfw:
+	OSUnloadFirmware(psRGXFW);
+fw_load_fail:
+
+	return eError;
+}
+
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function      InitialiseHWPerfCounters
+
+ @Description   Initialisation of hardware performance counters and dumping
+                them out to pdump, so that they can be modified at a later
+                point.
+
+ @Input         pvDevice
+ @Input         psHWPerfDataMemDesc
+ @Input         psHWPerfInitDataInt
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     void *pvDevice,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc,
+                                     RGXFWIF_HWPERF_CTL *psHWPerfInitDataInt)
+{
+	RGXFWIF_HWPERF_CTL_BLK *psHWPerfInitBlkData;
+#if defined(HWPERF_UNIFIED)
+	RGXFWIF_HWPERF_DA_BLK *psHWPerfInitDABlkData;
+#endif
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc;
+	IMG_UINT32 ui32BlockID, ui32BlkCfgIdx, ui32CounterIdx;
+	RGX_HWPERF_CNTBLK_RT_INFO sCntBlkRtInfo;
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "HWPerf Counter Config starts here.");
+
+	for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+	{
+		__maybe_unused IMG_UINT32 uiUnit;
+		__maybe_unused IMG_BOOL bDirect;
+
+		/* Exit early if this core does not have any of these counter blocks
+		 * due to core type/BVNC features.... */
+		psBlkTypeDesc = &asCntBlkTypeModel[ui32BlkCfgIdx];
+		if (psBlkTypeDesc->pfnIsBlkPresent(psBlkTypeDesc, pvDevice, &sCntBlkRtInfo) == IMG_FALSE)
+		{
+			continue;
+		}
+
+		/* Program all counters in one block so those already on may
+		 * be configured off and vice-versa. */
+		for (ui32BlockID = psBlkTypeDesc->ui32CntBlkIdBase;
+			 ui32BlockID < psBlkTypeDesc->ui32CntBlkIdBase+sCntBlkRtInfo.ui32NumUnits;
+			 ui32BlockID++)
+		{
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "Unit %d Block : %s",
+			                      ui32BlockID-psBlkTypeDesc->ui32CntBlkIdBase,
+			                      psBlkTypeDesc->pszBlockNameComment);
+
+			/* Get the block configure store to update from the global store of
+			 * block configuration. This is used to remember the configuration
+			 * between configurations and core power on in APM.
+			 * For HWPERF_UNIFIED layout we will have a different
+			 * structure type to decode the HWPerf block. This is indicated by
+			 * the RGX_CNTBLK_ID_DA_MASK bit being set in the block-ID value. */
+
+			bDirect = (psBlkTypeDesc->ui32IndirectReg == 0U);
+			uiUnit = ui32BlockID - psBlkTypeDesc->ui32CntBlkIdBase;
+
+#if defined(HWPERF_UNIFIED)
+			if ((ui32BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+			{
+				psHWPerfInitDABlkData = rgxfw_hwperf_get_da_block_ctl(ui32BlockID, psHWPerfInitDataInt);
+
+				PVR_ASSERT(psHWPerfInitDABlkData);
+
+				psHWPerfInitDABlkData->eBlockID = ui32BlockID;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "eBlockID: The Block ID for the layout block. See RGX_HWPERF_CNTBLK_ID for further information.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->eBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->eBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitDABlkData->uiEnabled = 0U;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->uiEnabled) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->uiEnabled,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitDABlkData->uiNumCounters = 0U;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiNumCounters (X): Specifies the number of valid counters"
+			                      " [0..%d] which follow.", RGX_CNTBLK_COUNTERS_MAX);
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->uiNumCounters) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->uiNumCounters,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				for (ui32CounterIdx = 0; ui32CounterIdx < RGX_CNTBLK_COUNTERS_MAX; ui32CounterIdx++)
+				{
+					psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx] = IMG_UINT32_C(0x00000000);
+
+					if (bDirect)
+					{
+						PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+					                      "%s_COUNTER_%d",
+					                      psBlkTypeDesc->pszBlockNameComment,
+					                      ui32CounterIdx);
+					}
+					else
+					{
+						PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+					                      "%s%d_COUNTER_%d",
+					                      psBlkTypeDesc->pszBlockNameComment,
+					                      uiUnit, ui32CounterIdx);
+					}
+
+					DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+					        psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx],
+							PDUMP_FLAGS_CONTINUOUS);
+				}
+			}
+			else
+#endif	/* defined(HWPERF_UNIFIED) */
+			{
+				psHWPerfInitBlkData = rgxfw_hwperf_get_block_ctl(ui32BlockID, psHWPerfInitDataInt);
+				/* Assert to check for HWPerf block mis-configuration */
+				PVR_ASSERT(psHWPerfInitBlkData);
+
+				psHWPerfInitBlkData->ui32Valid = IMG_TRUE;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "bValid: This specifies if the layout block is valid for the given BVNC.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->ui32Valid) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->ui32Valid,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->ui32Enabled = IMG_FALSE;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "bEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->ui32Enabled) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->ui32Enabled,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->eBlockID = ui32BlockID;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "eBlockID: The Block ID for the layout block. See RGX_HWPERF_CNTBLK_ID for further information.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->eBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->eBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->uiCounterMask = 0x00;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiCounterMask: Bitmask for selecting the counters that need to be configured. (Bit 0 - counter0, bit 1 - counter1 and so on.)");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->uiCounterMask) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->uiCounterMask,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				for (ui32CounterIdx = RGX_CNTBLK_COUNTER0_ID; ui32CounterIdx < psBlkTypeDesc->ui8NumCounters; ui32CounterIdx++)
+				{
+					psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx] = IMG_UINT64_C(0x0000000000000000);
+
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+				                      "%s_COUNTER_%d", psBlkTypeDesc->pszBlockNameComment, ui32CounterIdx);
+					DevmemPDumpLoadMemValue64(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx],
+							PDUMP_FLAGS_CONTINUOUS);
+
+				}
+			}
+		}
+	}
+}
+/*!
+*******************************************************************************
+
+ @Function      InitialiseCustomCounters
+
+ @Description   Initialisation of custom counters and dumping them out to
+                pdump, so that they can be modified at a later point.
+
+ @Input         psHWPerfDataMemDesc
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseCustomCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc)
+{
+	IMG_UINT32 ui32CustomBlock, ui32CounterID;
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "ui32SelectedCountersBlockMask - The Bitmask of the custom counters that are to be selected");
+	DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+						offsetof(RGXFWIF_HWPERF_CTL, ui32SelectedCountersBlockMask),
+						0,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	for (ui32CustomBlock = 0; ui32CustomBlock < RGX_HWPERF_MAX_CUSTOM_BLKS; ui32CustomBlock++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiOffsetOfCustomBlockSelectedCounters
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_HWPERF_CTL *)0)->SelCntr[ui32CustomBlock].ui32NumSelectedCounters);
+
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "ui32NumSelectedCounters - The Number of counters selected for this Custom Block: %d",ui32CustomBlock );
+		DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+					uiOffsetOfCustomBlockSelectedCounters,
+					0,
+					PDUMP_FLAGS_CONTINUOUS);
+
+		for (ui32CounterID = 0; ui32CounterID < RGX_HWPERF_MAX_CUSTOM_CNTRS; ui32CounterID++ )
+		{
+			const IMG_DEVMEM_OFFSET_T uiOffsetOfCustomBlockSelectedCounterIDs
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_HWPERF_CTL *)0)->SelCntr[ui32CustomBlock].aui32SelectedCountersIDs[ui32CounterID]);
+
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "CUSTOMBLK_%d_COUNTERID_%d",ui32CustomBlock, ui32CounterID);
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+					uiOffsetOfCustomBlockSelectedCounterIDs,
+					0,
+					PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitialiseAllCounters
+
+ @Description   Initialise HWPerf and custom counters
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitialiseAllCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	RGXFWIF_HWPERF_CTL *psHWPerfInitData;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc, (void **)&psHWPerfInitData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", failHWPerfCountersMemDescAqCpuVirt);
+
+	InitialiseHWPerfCounters(psDeviceNode, psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc, psHWPerfInitData);
+	InitialiseCustomCounters(psDeviceNode, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+	RGXFwSharedMemCacheOpPtr(psHWPerfInitData, FLUSH);
+
+failHWPerfCountersMemDescAqCpuVirt:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+
+	return eError;
+}
+#endif /* PDUMP */
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR RGXValidateTDHeap(PVRSRV_DEVICE_NODE *psDeviceNode,
+									  PVRSRV_PHYS_HEAP ePhysHeap,
+									  PHYS_HEAP_USAGE_FLAGS ui32RequiredFlags)
+{
+	PHYS_HEAP *psHeap = psDeviceNode->apsPhysHeap[ePhysHeap];
+	PHYS_HEAP_USAGE_FLAGS ui32HeapFlags = PhysHeapGetFlags(psHeap);
+	PHYS_HEAP_USAGE_FLAGS ui32InvalidFlags = ~(PHYS_HEAP_USAGE_FW_PRIV_DATA | PHYS_HEAP_USAGE_FW_CODE
+											   | PHYS_HEAP_USAGE_GPU_SECURE | PHYS_HEAP_USAGE_FW_PRIVATE);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32RequiredFlags) != 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap is missing required flags. flags: 0x%x / required:0x%x",
+							   ui32HeapFlags,
+							   ui32RequiredFlags);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32InvalidFlags) == 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap uses invalid flags. flags: 0x%x / invalid:0x%x",
+							   ui32HeapFlags,
+							   ui32InvalidFlags);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXValidateTDHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_PRIV_DATA, PHYS_HEAP_USAGE_FW_PRIV_DATA);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_PRIV_DATA");
+
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_CODE, PHYS_HEAP_USAGE_FW_CODE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_CODE");
+	}
+
+	eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_GPU_SECURE, PHYS_HEAP_USAGE_GPU_SECURE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:GPU_SECURE");
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInit
+
+ @Description   RGX Initialisation
+
+ @Input         psDeviceNode
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Services initialisation parameters */
+	RGX_INIT_APPHINTS sApphints = {0};
+	IMG_UINT32 ui32FWConfigFlags, ui32FWConfigFlagsExt, ui32FwOsCfgFlags;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 1");
+
+	PDUMPCOMMENT(psDeviceNode, "Device Name: %s",
+	             psDeviceNode->psDevConfig->pszName);
+	PDUMPCOMMENT(psDeviceNode, "Device ID: %u (%d)",
+	             psDeviceNode->sDevId.ui32InternalID,
+	             psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->psDevConfig->pszVersion)
+	{
+		PDUMPCOMMENT(psDeviceNode, "Device Version: %s",
+		             psDeviceNode->psDevConfig->pszVersion);
+	}
+
+	/* pdump info about the core */
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGX Version Information (KM): %d.%d.%d.%d",
+	             psDevInfo->sDevFeatureCfg.ui32B,
+	             psDevInfo->sDevFeatureCfg.ui32V,
+	             psDevInfo->sDevFeatureCfg.ui32N,
+	             psDevInfo->sDevFeatureCfg.ui32C);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	eError = RGXValidateTDHeaps(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeaps");
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		/* The RGX_CR_MTS_DM4_INTERRUPT_ENABLE register is always set by the firmware during initialisation
+		 * and it provides a good method of determining if the firmware has been booted previously */
+		psDeviceNode->bAutoVzFwIsUp = (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_DM4_INTERRUPT_ENABLE) != 0);
+
+		PVR_LOG(("AutoVz startup check: firmware is %s;",
+				(psDeviceNode->bAutoVzFwIsUp) ? "already running" : "powered down"));
+		PVR_LOG(("AutoVz allow GPU powerdown is %s:",
+				(psDeviceNode->bAutoVzAllowGPUPowerdown) ? "enabled" : "disabled"));
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest assumes the firmware is always available */
+		psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+		psDeviceNode->bAutoVzAllowGPUPowerdown = IMG_FALSE;
+	}
+	else
+#endif
+	{
+		/* Firmware does not follow the AutoVz life-cycle */
+		psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+	{
+		/* set the device power state here as the regular power
+		 * callbacks will not be executed on this driver */
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+
+	/* Set which HW Safety Events will be handled by the driver */
+	psDevInfo->ui32HostSafetyEventMask |= RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER) ?
+										  RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_EN : 0;
+	psDevInfo->ui32HostSafetyEventMask |= (RGX_DEVICE_HAS_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS)
+										   && (RGX_DEVICE_GET_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS) > 0)) ?
+										  RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_EN : 0;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Register defs revision: %d", RGX_CR_DEFS_KM_REVISION);
+#endif
+
+	/* Services initialisation parameters */
+	GetApphints(psDevInfo, &sApphints);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	if ((sApphints.bEnableTrustedDeviceAceConfig) &&
+		(RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE)))
+	{
+		SetTrustedDeviceAceEnabled(psDeviceNode->psDevConfig->hSysData);
+	}
+#endif
+#endif
+
+	eError = RGXInitCreateFWKernelMemoryContext(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create FW kernel memory context (%u)",
+		         __func__, eError));
+		goto cleanup;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitFirmware(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitFirmware failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Setup Firmware initialisation data
+	 */
+
+	GetFWConfigFlags(psDeviceNode, &sApphints, &ui32FWConfigFlags, &ui32FWConfigFlagsExt, &ui32FwOsCfgFlags);
+
+	eError = RGXInitFirmware(psDeviceNode,
+	                         &sApphints,
+	                         ui32FWConfigFlags,
+	                         ui32FWConfigFlagsExt,
+	                         ui32FwOsCfgFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitFirmware failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+#if defined(PDUMP)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitialiseAllCounters(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitialiseAllCounters failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+#endif
+
+	/*
+	 * Perform second stage of RGX initialisation
+	 */
+	eError = RGXInitDevPart2(psDeviceNode,
+	                         &sApphints);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitDevPart2 failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+
+	eError = PVRSRV_OK;
+
+cleanup:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxsrvinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxstartstop.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxstartstop.c
new file mode 100644
index 000000000000..2b93b46ff3ef
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxstartstop.c
@@ -0,0 +1,1245 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific start/stop routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific start/stop routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxstartstop.h"
+
+#define SOC_FEATURE_STRICT_SAME_ADDRESS_WRITE_ORDERING
+
+/*
+	RGXWriteMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAT;
+	}
+
+	eError = RGXPollReg32(hPrivate, ui32StateReg, ui32StateReadyFlag, ui32StateReadyFlag);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Write */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+		RGXWriteReg32(hPrivate, ui32DataReg, ui32RegValue);
+		(void) RGXReadReg32(hPrivate, ui32DataReg); /* Fence write */
+	}
+
+	return eError;
+}
+
+/*
+	RGXReadMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN);
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN);
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAX;
+		BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	}
+
+	/* Wait for Slave Port to be Ready */
+	eError = RGXPollReg32(hPrivate,
+						  ui32StateReg,
+						  ui32StateReadyFlag,
+						  ui32StateReadyFlag);
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Read */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+
+		/* Wait for Slave Port to be Ready */
+		eError = RGXPollReg32(hPrivate,
+				  ui32StateReg,
+				  ui32StateReadyFlag,
+				  ui32StateReadyFlag);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+#if !defined(NO_HARDWARE)
+	*ui32RegValue = RGXReadReg32(hPrivate, ui32DataReg);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32DataReg);
+	*ui32RegValue = 0xFFFFFFFF;
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXWriteMetaCoreRegThoughSP(const void *hPrivate,
+                                                IMG_UINT32 ui32CoreReg,
+                                                IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 i = 0;
+
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXDT_OFFSET, ui32Value);
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, ui32CoreReg & ~META_CR_TXUXXRXRQ_RDnWR_BIT);
+
+	do
+	{
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, &ui32Value);
+	} while (((ui32Value & META_CR_TXUXXRXRQ_DREADY_BIT) != META_CR_TXUXXRXRQ_DREADY_BIT) && (i++ < 1000));
+
+	if (i == 1000)
+	{
+		RGXCommentLog(hPrivate, "RGXWriteMetaCoreRegThoughSP: Timeout");
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXStartFirmware(const void *hPrivate)
+{
+	PVRSRV_ERROR eError;
+
+	/* Give privilege to debug and slave port */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_SYSC_JTAG_THREAD, META_CR_SYSC_JTAG_THREAD_PRIV_EN);
+
+	/* Point Meta to the bootloader address, global (uncached) range */
+	eError = RGXWriteMetaCoreRegThoughSP(hPrivate,
+	                                     PC_ACCESS(0),
+	                                     RGXFW_BOOTLDR_META_ADDR | META_MEM_GLOBAL_RANGE_BIT);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start failed!");
+		return eError;
+	}
+
+	/* Enable minim encoding */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXPRIVEXT, META_CR_TXPRIVEXT_MINIM_EN);
+
+	/* Enable Meta thread */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_T0ENABLE_OFFSET, META_CR_TXENABLE_ENABLE_BIT);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMetaProcWrapper
+
+ @Description   Configures the hardware wrapper of the META processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMetaProcWrapper(const void *hPrivate)
+{
+	IMG_UINT64 ui64GartenConfig;
+
+	/* Garten IDLE bit controlled by META */
+	ui64GartenConfig = RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META;
+
+#if defined(RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT)
+	/* Set the Garten Wrapper BIF Fence address */
+	{
+		/* Set PC = 0 for fences */
+		ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK;
+		ui64GartenConfig |= (IMG_UINT64)MMU_CONTEXT_MAPPING_FWPRIV
+		                    << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT;
+
+		/* Set SLC DM=META */
+		ui64GartenConfig |= ((IMG_UINT64) RGXFW_SEGMMU_META_BIFDM_ID) << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT;
+	}
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure META wrapper");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, ui64GartenConfig);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMipsProcWrapper
+
+ @Description   Configures the hardware wrapper of the MIPS processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMipsProcWrapper(const void *hPrivate)
+{
+	IMG_DEV_PHYADDR sPhyAddr;
+	IMG_UINT64 ui64RemapSettings = RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE; /* Same for all remap registers */
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure MIPS wrapper");
+
+	/*
+	 * MIPS wrapper (registers transaction ID and ISA mode) setup
+	 */
+
+	RGXCommentLog(hPrivate, "RGXStart: Write wrapper config register");
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		RGXWriteReg32(hPrivate,
+		              RGX_CR_MIPS_WRAPPER_CONFIG,
+		              (RGXMIPSFW_REGISTERS_VIRTUAL_BASE >>
+		              RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN) |
+		              RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+	else
+	{
+		RGXAcquireGPURegsAddr(hPrivate, &sPhyAddr);
+
+		RGXMIPSWrapperConfig(hPrivate,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG,
+		                     sPhyAddr.uiAddr,
+		                     RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+
+	/*
+	 * Boot remap setup
+	 */
+
+	RGXAcquireBootRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write boot remap registers");
+	RGXBootRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG1,
+	                   RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+#if defined(FIX_HW_BRN_63553_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 63553))
+	{
+		IMG_BOOL bPhysBusAbove32Bit = RGXGetDevicePhysBusWidth(hPrivate) > 32;
+		IMG_BOOL bDevicePA0IsValid  = RGXDevicePA0IsValid(hPrivate);
+
+		/* WA always required on 36 bit cores, to avoid continuous unmapped memory accesses to address 0x0 */
+		if (bPhysBusAbove32Bit || !bDevicePA0IsValid)
+		{
+			RGXCodeRemapConfig(hPrivate,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG1,
+					0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG2,
+					sPhyAddr.uiAddr,
+					~RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK,
+					ui64RemapSettings);
+		}
+	}
+#endif
+
+	/*
+	 * Data remap setup
+	 */
+
+	RGXAcquireDataRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		/* Remapped private data in secure memory */
+		ui64RemapSettings |= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN;
+	}
+	else
+	{
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+	}
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+	RGXCommentLog(hPrivate, "RGXStart: Write data remap registers");
+	RGXDataRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG1,
+	                   RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	/*
+	 * Code remap setup
+	 */
+
+	RGXAcquireCodeRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write exceptions remap registers");
+	RGXCodeRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG1,
+	                   RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) == 32)
+	{
+		/*
+		 * Trampoline remap setup
+		 */
+
+		RGXAcquireTrampolineRemapAddr(hPrivate, &sPhyAddr);
+		ui64RemapSettings = RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+		ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+		RGXCommentLog(hPrivate, "RGXStart: Write trampoline remap registers");
+		RGXTrampolineRemapConfig(hPrivate,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG1,
+		                         sPhyAddr.uiAddr | RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG2,
+		                         RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+		                         ~RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK,
+		                         ui64RemapSettings);
+	}
+
+	/* Garten IDLE bit controlled by MIPS */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to MIPS");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+
+	/* Turn on the EJTAG probe (only useful driver live) */
+	RGXWriteReg32(hPrivate, RGX_CR_MIPS_DEBUG_CONFIG, 0);
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitRiscvProcWrapper
+
+ @Description   Configures the hardware wrapper of the RISCV processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitRiscvProcWrapper(const void *hPrivate)
+{
+	IMG_UINT32 ui32BootCodeRemap = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+	IMG_UINT32 ui32BootDataRemap = RGXRISCVFW_BOOTLDR_DATA_REMAP;
+	IMG_DEV_VIRTADDR sTmp;
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure RISCV wrapper");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) < 4)
+#endif
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Write boot code remap");
+		RGXAcquireBootCodeAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootCodeRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN);
+
+		RGXCommentLog(hPrivate, "RGXStart: Write boot data remap");
+		RGXAcquireBootDataAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootDataRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN)
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN |
+#endif
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN);
+	}
+
+	/* Garten IDLE bit controlled by RISCV */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to RISCV");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      __RGXInitSLC
+
+ @Description   Initialise RGX SLC
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void __RGXInitSLC(const void *hPrivate)
+{
+	{
+		IMG_UINT32 ui32Reg;
+		IMG_UINT32 ui32RegVal;
+		IMG_UINT64 ui64RegVal;
+
+		/*
+		 * SLC Bypass control
+		 */
+		ui32Reg = RGX_CR_SLC_CTRL_BYPASS;
+		ui64RegVal = 0;
+
+#if defined(RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN)
+		if ((RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, SLC_SIZE_IN_KILOBYTES) == 8)  ||
+		    RGX_DEVICE_HAS_BRN(hPrivate, 61450))
+		{
+			RGXCommentLog(hPrivate, "Bypass SLC for IPF_OBJ and IPF_CPF");
+			ui64RegVal |= (IMG_UINT64) RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN |
+						(IMG_UINT64) RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_EN;
+		}
+#endif
+
+		if (ui64RegVal != 0)
+		{
+			RGXReadModifyWriteReg64(hPrivate, ui32Reg, ui64RegVal, ~ui64RegVal);
+		}
+
+		/*
+		 * SLC Misc control.
+		 *
+		 * Note: This is a 64bit register and we set only the lower 32bits leaving the top
+		 *       32bits (RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS) unchanged from the HW default.
+		 */
+		ui32Reg = RGX_CR_SLC_CTRL_MISC;
+		ui32RegVal = RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH1;
+		ui32RegVal |= RGXReadReg32(hPrivate, ui32Reg) & RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+
+#if defined(FIX_HW_BRN_60084_BIT_MASK)
+		if (RGX_DEVICE_HAS_BRN(hPrivate, 60084))
+		{
+#if !defined(SOC_FEATURE_STRICT_SAME_ADDRESS_WRITE_ORDERING)
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+#else
+			if (RGX_DEVICE_HAS_ERN(hPrivate, 61389))
+			{
+				ui32RegVal |= RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+			}
+#endif
+		}
+#endif
+
+		/* Bypass burst combiner if SLC line size is smaller than 1024 bits */
+		if (RGXGetDeviceCacheLineSize(hPrivate) < 1024)
+		{
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_EN;
+		}
+
+		if (RGX_DEVICE_HAS_BRN(hPrivate, 71242) && !RGX_DEVICE_HAS_FEATURE(hPrivate, GPU_MULTICORE_SUPPORT))
+		{
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_EN;
+		}
+
+		RGXWriteReg32(hPrivate, ui32Reg, ui32RegVal);
+	}
+}
+
+
+static void RGXWriteKernelCatBase(const void *hPrivate, IMG_DEV_PHYADDR sPCAddr)
+{
+	{
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC64(hPrivate,
+		                      BIF_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT,
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT,
+		                      ((sPCAddr.uiAddr
+		                      >> RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT)
+		                      << RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT)
+		                      & ~RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK);
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Keep catbase registers in sync */
+			RGXWriteKernelMMUPC64(hPrivate,
+			                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+			                      ((sPCAddr.uiAddr
+			                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+			                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+			                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+		}
+
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXWriteKernelCatBase: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitBIF
+
+ @Description   Initialise RGX BIF
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitBIF(const void *hPrivate)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXInitBIF: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+	else
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+	{
+		IMG_DEV_PHYADDR sPCAddr;
+
+		/*
+		 * Acquire the address of the Kernel Page Catalogue.
+		 */
+		RGXAcquireKernelMMUPC(hPrivate, &sPCAddr);
+
+		/*
+		 * Write the kernel catalogue base.
+		 */
+		RGXCommentLog(hPrivate, "RGX firmware MMU Page Catalogue");
+
+		RGXWriteKernelCatBase(hPrivate, sPCAddr);
+	}
+}
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/**************************************************************************/ /*!
+@Function       RGXInitMMURangeRegisters
+@Description    Initialises MMU range registers for Non4K pages.
+@Input          hPrivate           Implementation specific data
+@Return         void
+ */ /**************************************************************************/
+static void RGXInitMMURangeRegisters(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psParams->psDevInfo;
+	IMG_UINT32 ui32RegAddr = RGX_CR_MMU_PAGE_SIZE_RANGE_ONE;
+	IMG_UINT32 i;
+
+	for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i, ui32RegAddr += sizeof(IMG_UINT64))
+	{
+		RGXWriteReg64(hPrivate, ui32RegAddr, psDevInfo->aui64MMUPageSizeRangeValue[i]);
+	}
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitAXIACE
+
+ @Description    Initialise AXI-ACE interface
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitAXIACE(const void *hPrivate)
+{
+	IMG_UINT64 ui64RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	ui32RegAddr = RGX_CR_AXI_ACE_LITE_CONFIGURATION;
+
+	/* Setup AXI-ACE config. Set everything to outer cache */
+	ui64RegVal = (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT) |
+	             (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT)  |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT);
+
+#if defined(FIX_HW_BRN_42321_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 42321))
+	{
+		ui64RegVal |= (((IMG_UINT64) 1) << RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT);
+	}
+#endif
+
+#if defined(FIX_HW_BRN_68186_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 68186))
+	{
+		/* default value for reg_enable_fence_out is zero. Force to 1 to allow core_clk < mem_clk */
+		ui64RegVal |= (IMG_UINT64)1 << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT;
+	}
+#endif
+#else /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+	ui32RegAddr = RGX_CR_ACE_CTRL;
+
+	/**
+	 * The below configuration is only applicable for RGX cores supporting
+	 * ACE/ACE-lite protocol and connected to ACE coherent interconnect.
+	 */
+
+	/**
+	 * Configure AxDomain and AxCache for MMU transactions.
+	 * AxDomain set to non sharable (0x0).
+	 */
+	ui64RegVal = RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE |
+				 RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE;
+
+	/**
+	 * Configure AxCache for PM/MMU transactions.
+	 * Set to same value (i.e WBRWALLOC caching, rgxmmunit.c:RGXDerivePTEProt8)
+	 * as non-coherent PTEs
+	 */
+	ui64RegVal |= (IMG_UINT64_C(0xF)) << RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_SHIFT;
+
+	/**
+	 * Configure AxDomain for non MMU transactions.
+	 */
+	ui64RegVal |= (IMG_UINT64)(RGX_CR_ACE_CTRL_COH_DOMAIN_OUTER_SHAREABLE |
+							   RGX_CR_ACE_CTRL_NON_COH_DOMAIN_NON_SHAREABLE);
+#endif /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+
+	RGXCommentLog(hPrivate, "Init AXI-ACE interface");
+	RGXWriteReg64(hPrivate, ui32RegAddr, ui64RegVal);
+}
+
+static void RGXResetSequence(const void *hPrivate, const IMG_CHAR *pcRGXFW_PROCESSOR)
+{
+	IMG_UINT64 ui64SoftResetMask;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXCommentLog(hPrivate, "RGXStart: soft reset cpu core");
+		RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 0);
+	}
+
+	{
+		/* Set RGX in soft-reset */
+		RGXCommentLog(hPrivate, "RGXStart: soft reset everything");
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, ui64SoftResetMask);
+
+		/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+		/* Take Rascal and Dust out of reset */
+		RGXCommentLog(hPrivate, "RGXStart: Rascal and Dust out of reset");
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, ui64SoftResetMask ^ RGX_CR_SOFT_RESET_RASCALDUSTS_EN);
+
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+		/* Take everything out of reset but the FW processor */
+		RGXCommentLog(hPrivate, "RGXStart: Take everything out of reset but %s", pcRGXFW_PROCESSOR);
+
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
+
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+	}
+}
+
+static void DeassertMetaReset(const void *hPrivate)
+{
+	/* Need to wait for at least 32 cycles before taking the FW processor out of reset ... */
+	RGXWaitCycles(hPrivate, 32, 3);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, 0x0);
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	/* ... and afterwards */
+	RGXWaitCycles(hPrivate, 32, 3);
+}
+
+PVRSRV_ERROR RGXStart(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR *pcRGXFW_PROCESSOR;
+	IMG_BOOL bDoFWSlaveBoot = IMG_FALSE;
+	IMG_BOOL bMetaFW = IMG_FALSE;
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_RISCV;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_MIPS;
+	}
+#endif
+	else
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_META;
+		bMetaFW = IMG_TRUE;
+		bDoFWSlaveBoot = RGXDoFWSlaveBoot(hPrivate);
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SYS_BUS_SECURE_RESET))
+	{
+		/* Disable the default sys_bus_secure protection to perform minimal setup */
+		RGXCommentLog(hPrivate, "RGXStart: Disable sys_bus_secure");
+		RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, 0);
+		(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+	}
+
+#if defined(RGX_FEATURE_HYPERVISOR_MMU_BIT_MASK)
+	/* Only bypass HMMU if the module is present */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, HYPERVISOR_MMU))
+	{
+		/* Always set HMMU in bypass mode */
+		RGXWriteReg32(hPrivate, RGX_CR_HMMU_BYPASS, RGX_CR_HMMU_BYPASS_MASKFULL);
+		(void) RGXReadReg32(hPrivate, RGX_CR_HMMU_BYPASS);
+	}
+#endif
+
+	/*!
+	 * Start FW init sequence
+	 */
+	RGXResetSequence(hPrivate, pcRGXFW_PROCESSOR);
+
+	/*
+	 * Initialise SLC.
+	 */
+	__RGXInitSLC(hPrivate);
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, ECC_RAMS) > 0)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Enable safety events");
+		RGXWriteReg32(hPrivate, RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE,
+		                        RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL);
+	}
+
+	if (bMetaFW)
+	{
+		if (bDoFWSlaveBoot)
+		{
+			/* Configure META to Slave boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Slave boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, 0);
+		}
+		else
+		{
+			/* Configure META to Master boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Master boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, RGX_CR_META_BOOT_MODE_EN);
+		}
+	}
+
+	/*
+	 * Initialise Firmware wrapper
+	 */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXInitRiscvProcWrapper(hPrivate);
+	}
+	else if (bMetaFW)
+	{
+		RGXInitMetaProcWrapper(hPrivate);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		RGXInitMipsProcWrapper(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, MMU_VERSION) >= 4)
+	{
+		/* initialise the MMU range based config registers for Non4K pages */
+		RGXInitMMURangeRegisters(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, AXI_ACELITE))
+#endif
+	{
+		/* We must init the AXI-ACE interface before 1st BIF transaction */
+		RGXInitAXIACE(hPrivate);
+	}
+
+	/*
+	 * Initialise BIF.
+	 */
+	RGXInitBIF(hPrivate);
+
+	RGXSetPoweredState(hPrivate, IMG_TRUE);
+
+	RGXCommentLog(hPrivate, "RGXStart: Take %s out of reset", pcRGXFW_PROCESSOR);
+	DeassertMetaReset(hPrivate);
+
+	if (bMetaFW && bDoFWSlaveBoot)
+	{
+		eError = RGXFabricCoherencyTest(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start");
+		eError = RGXStartFirmware(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+	}
+	else
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Master boot Start");
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Bring Debug Module out of reset */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) >= 4)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_EN);
+			}
+			else
+#endif
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL, RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+			}
+
+			/* Boot the FW */
+			RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 1);
+			RGXWaitCycles(hPrivate, 32, 3);
+		}
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	RGXCommentLog(hPrivate, "RGXStart: Enable sys_bus_secure");
+	RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, RGX_CR_SYS_BUS_SECURE_ENABLE_EN);
+	(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStop(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMetaFW = RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, META);
+
+	RGXDeviceAckIrq(hPrivate);
+
+	/* Set FW power state OFF to disable LISR handler */
+	RGXSetPoweredState(hPrivate, IMG_FALSE);
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper
+	 * For LAYOUT_MARS = 1, SIDEKICK would have been powered down by FW
+	 */
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN),
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN));
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		/*
+		 * Wait for SLC to signal IDLE
+		 * For LAYOUT_MARS = 1, SLC would have been powered down by FW
+		 */
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SLC_IDLE,
+					RGX_CR_SLC_IDLE_MASKFULL,
+					RGX_CR_SLC_IDLE_MASKFULL);
+		}
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	/* Unset MTS DM association with threads */
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL);
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL);
+
+	if (bMetaFW)
+	{
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL);
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL);
+	}
+
+#if defined(PDUMP)
+	if (bMetaFW)
+	{
+		/* Disabling threads is only required for pdumps to stop the fw gracefully */
+
+		/* Disable thread 0 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T0ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Disable thread 1 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T1ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Clear down any irq raised by META (done after disabling the FW
+		 * threads to avoid a race condition).
+		 * This is only really needed for PDumps but we do it anyway driver-live.
+		 */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		/* Wait for the Slave Port to finish all the transactions */
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+		{
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+			else
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+		}
+		else
+#endif
+		{
+			RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS, 0x0);
+			(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS); /* Fence write */
+
+			eError = RGXPollReg32(hPrivate,
+								  RGX_CR_META_SP_MSLVCTRL1,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN);
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+#endif
+
+	/* Extra Idle checks */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIF_STATUS_MMU,
+	                      0,
+	                      RGX_CR_BIF_STATUS_MMU_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIFPM_STATUS_MMU,
+	                      0,
+	                      RGX_CR_BIFPM_STATUS_MMU_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	if (!RGX_DEVICE_HAS_FEATURE(hPrivate, XT_TOP_INFRASTRUCTURE))
+	{
+		eError = RGXPollReg32(hPrivate,
+		                      RGX_CR_BIF_READS_EXT_STATUS,
+		                      0,
+		                      RGX_CR_BIF_READS_EXT_STATUS_MASKFULL);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIFPM_READS_EXT_STATUS,
+	                      0,
+	                      RGX_CR_BIFPM_READS_EXT_STATUS_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg64(hPrivate,
+	                      RGX_CR_SLC_STATUS1,
+	                      0,
+		                  RGX_CR_SLC_STATUS1_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	if (4 == RGXGetDeviceSLCBanks(hPrivate))
+	{
+		eError = RGXPollReg64(hPrivate,
+		                      RGX_CR_SLC_STATUS2,
+		                      0,
+		                      RGX_CR_SLC_STATUS2_MASKFULL);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		/*
+		 * Wait for SLC to signal IDLE
+		 * For LAYOUT_MARS = 1, SLC would have been powered down by FW
+		 */
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SLC_IDLE,
+					RGX_CR_SLC_IDLE_MASKFULL,
+					RGX_CR_SLC_IDLE_MASKFULL);
+		}
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper
+	 * For LAYOUT_MARS = 1, SIDEKICK would have been powered down by FW
+	 */
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN),
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN));
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (bMetaFW)
+	{
+		IMG_UINT32 ui32RegValue;
+
+		eError = RGXReadMetaRegThroughSP(hPrivate,
+		                                 META_CR_TxVECINT_BHALT,
+		                                 &ui32RegValue);
+		if (eError != PVRSRV_OK) return eError;
+
+		if ((ui32RegValue & 0xFFFFFFFFU) == 0x0)
+		{
+			/* Wait for Sidekick/Jones to signal IDLE including
+			 * the Garten Wrapper if there is no debugger attached
+			 * (TxVECINT_BHALT = 0x0) */
+			{
+				eError = RGXPollReg32(hPrivate,
+				                      RGX_CR_SIDEKICK_IDLE,
+				                      RGX_CR_SIDEKICK_IDLE_GARTEN_EN,
+				                      RGX_CR_SIDEKICK_IDLE_GARTEN_EN);
+				if (eError != PVRSRV_OK) return eError;
+			}
+		}
+	}
+	else
+	{
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0)
+		{
+			/* As FW core has been moved from SIDEKICK to the new MARS domain, checking
+			 * idle bits for CPU & System Arbiter excluding SOCIF which will never be Idle
+			 * if Host polling on this register
+			 */
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_MARS_IDLE,
+					RGX_CR_MARS_IDLE_CPU_EN | RGX_CR_MARS_IDLE_MH_SYSARB0_EN,
+					RGX_CR_MARS_IDLE_CPU_EN | RGX_CR_MARS_IDLE_MH_SYSARB0_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+		else
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_GARTEN_EN,
+					RGX_CR_SIDEKICK_IDLE_GARTEN_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxstartstop.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.c
new file mode 100644
index 000000000000..515b988e938f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.c
@@ -0,0 +1,5785 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA/3D routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX TA/3D routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* for the offsetof macro */
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxta3d.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "ri_server.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "rgx_memallocflags.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxsyncutils.h"
+#include "htbserver.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "physmem.h"
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "process_stats.h"
+
+#if !defined(PM_INTERACTIVE_MODE)
+#include "rgxpmdefs.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+#include "rgxmmudefs_km.h"
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TA3D_UFO_DUMP	0
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static INLINE
+void _DebugSyncValues(const IMG_CHAR *pszFunction,
+					  const IMG_UINT32 *pui32UpdateValues,
+					  const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", pszFunction, i, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+
+static INLINE
+void _DebugSyncCheckpoints(const IMG_CHAR *pszFunction,
+						   const IMG_CHAR *pszDMName,
+						   const PSYNC_CHECKPOINT *apsSyncCheckpoints,
+						   const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFence%sSyncCheckpoints[%d]=<%p>", pszFunction, pszDMName, i, *(apsSyncCheckpoints + i)));
+	}
+}
+
+#else
+#define CHKPT_DBG(X)
+#endif
+
+/* define the number of commands required to be set up by the CCB helper */
+/* 1 command for the TA */
+#define CCB_CMD_HELPER_NUM_TA_COMMANDS 1
+/* Up to 3 commands for the 3D (partial render fence, partial render, and render) */
+#define CCB_CMD_HELPER_NUM_3D_COMMANDS 3
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define WORKEST_CYCLES_PREDICTION_GET(x) ((x).ui32CyclesPrediction)
+#else
+#define WORKEST_CYCLES_PREDICTION_GET(x) (NO_CYCEST)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_TA_DATA;
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_3D_DATA;
+
+struct _RGX_SERVER_RENDER_CONTEXT_ {
+	/* this lock protects usage of the render context.
+	 * it ensures only one kick is being prepared and/or submitted on
+	 * this render context at any time
+	 */
+	POS_LOCK				hLock;
+	RGX_CCB_CMD_HELPER_DATA asTACmdHelperData[CCB_CMD_HELPER_NUM_TA_COMMANDS];
+	RGX_CCB_CMD_HELPER_DATA as3DCmdHelperData[CCB_CMD_HELPER_NUM_3D_COMMANDS];
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWRenderContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	RGX_SERVER_RC_TA_DATA		sTAData;
+	RGX_SERVER_RC_3D_DATA		s3DData;
+	IMG_UINT32					ui32CleanupStatus;
+#define RC_CLEANUP_TA_COMPLETE		(1 << 0)
+#define RC_CLEANUP_3D_COMPLETE		(1 << 1)
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListTAFence;
+	SYNC_ADDR_LIST				sSyncAddrListTAUpdate;
+	SYNC_ADDR_LIST				sSyncAddrList3DFence;
+	SYNC_ADDR_LIST				sSyncAddrList3DUpdate;
+	ATOMIC_T					hIntJobRef;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+
+/*
+	Static functions used by render context code
+*/
+
+static
+PVRSRV_ERROR _DestroyTAContext(RGX_SERVER_RC_TA_DATA *psTAData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  psTAData->psServerCommonContext,
+											  RGXFWIF_DM_GEOM,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(psTAData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, psTAData->psContextStateMemDesc);
+	psTAData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Destroy3DContext(RGX_SERVER_RC_3D_DATA *ps3DData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psContextStateMemDesc);
+	ps3DData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static void _RGXDumpPMRPageList(DLLIST_NODE *psNode)
+{
+	RGX_PMR_NODE *psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	PVRSRV_ERROR			eError;
+
+	eError = PMRDumpPageList(psPMRNode->psPMR,
+							RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Error (%s) printing pmr %p",
+				PVRSRVGetErrorString(eError),
+				psPMRNode->psPMR));
+	}
+}
+
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_LOG(("Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016" IMG_UINT64_FMTSPECx,
+				psFreeList->sFreeListFWDevVAddr.ui32Addr,
+				psFreeList->ui32FreelistID,
+				psFreeList->ui64FreelistChecksum));
+
+	/* Dump Init FreeList page list */
+	PVR_LOG(("  Initial Memory block"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	/* Dump Grow FreeList page list */
+	PVR_LOG(("  Grow Memory blocks"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	return IMG_TRUE;
+}
+
+static void _CheckFreelist(RGX_FREELIST *psFreeList,
+						   IMG_UINT32 ui32NumOfPagesToCheck,
+						   IMG_UINT64 ui64ExpectedCheckSum,
+						   IMG_UINT64 *pui64CalculatedCheckSum)
+{
+#if defined(NO_HARDWARE)
+	/* No checksum needed as we have all information in the pdumps */
+	PVR_UNREFERENCED_PARAMETER(psFreeList);
+	PVR_UNREFERENCED_PARAMETER(ui32NumOfPagesToCheck);
+	PVR_UNREFERENCED_PARAMETER(ui64ExpectedCheckSum);
+	*pui64CalculatedCheckSum = 0;
+#else
+	PVRSRV_ERROR eError;
+	size_t uiNumBytes;
+	IMG_UINT8* pui8Buffer;
+	IMG_UINT32* pui32Buffer;
+	IMG_UINT32 ui32CheckSumAdd = 0;
+	IMG_UINT32 ui32CheckSumXor = 0;
+	IMG_UINT32 ui32Entry;
+	IMG_UINT32 ui32Entry2;
+	IMG_BOOL bFreelistBad = IMG_FALSE;
+
+	*pui64CalculatedCheckSum = 0;
+
+	PVR_ASSERT(ui32NumOfPagesToCheck <= (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+	/* Allocate Buffer of the size of the freelist */
+	pui8Buffer = OSAllocMem(ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+	if (pui8Buffer == NULL)
+	{
+		PVR_LOG(("%s: Failed to allocate buffer to check freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	/* Copy freelist content into Buffer */
+	eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
+					psFreeList->uiFreeListPMROffset +
+					(((psFreeList->ui32MaxFLPages -
+					   psFreeList->ui32CurrentFLPages - psFreeList->ui32ReadyFLPages) * sizeof(IMG_UINT32)) &
+					 ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1)),
+					 pui8Buffer,
+					 ui32NumOfPagesToCheck * sizeof(IMG_UINT32),
+					&uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pui8Buffer);
+		PVR_LOG(("%s: Failed to get freelist data for freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	PVR_ASSERT(uiNumBytes == ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+
+	/* Generate checksum (skipping the first page if not allocated) */
+	pui32Buffer = (IMG_UINT32 *)pui8Buffer;
+	ui32Entry = ((psFreeList->ui32GrowFLPages == 0  &&  psFreeList->ui32CurrentFLPages > 1) ? 1 : 0);
+	for (/*ui32Entry*/ ; ui32Entry < ui32NumOfPagesToCheck; ui32Entry++)
+	{
+		ui32CheckSumAdd += pui32Buffer[ui32Entry];
+		ui32CheckSumXor ^= pui32Buffer[ui32Entry];
+
+		/* Check for double entries */
+		for (ui32Entry2 = ui32Entry+1; ui32Entry2 < ui32NumOfPagesToCheck; ui32Entry2++)
+		{
+			if (pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2])
+			{
+				PVR_LOG(("%s: Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d of %d",
+						__func__,
+						psFreeList->sFreeListFWDevVAddr.ui32Addr,
+						pui32Buffer[ui32Entry2],
+						ui32Entry,
+						ui32Entry2,
+						psFreeList->ui32CurrentFLPages));
+				bFreelistBad = IMG_TRUE;
+				break;
+			}
+		}
+	}
+
+	OSFreeMem(pui8Buffer);
+
+	/* Check the calculated checksum against the expected checksum... */
+	*pui64CalculatedCheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
+
+	if (ui64ExpectedCheckSum != 0  &&  ui64ExpectedCheckSum != *pui64CalculatedCheckSum)
+	{
+		PVR_LOG(("%s: Checksum mismatch for freelist %p! Expected 0x%016" IMG_UINT64_FMTSPECx " calculated 0x%016" IMG_UINT64_FMTSPECx,
+				__func__, psFreeList,
+				ui64ExpectedCheckSum, *pui64CalculatedCheckSum));
+		bFreelistBad = IMG_TRUE;
+	}
+
+	if (bFreelistBad)
+	{
+		PVR_LOG(("%s: Sleeping for ever!", __func__));
+		PVR_ASSERT(!bFreelistBad);
+	}
+#endif
+}
+
+
+/*
+ *  Function to work out the number of freelist pages to reserve for growing
+ *  within the FW without having to wait for the host to progress a grow
+ *  request.
+ *
+ *  The number of pages must be a multiple of 4 to align the PM addresses
+ *  for the initial freelist allocation and also be less than the grow size.
+ *
+ *  If the threshold or grow size means less than 4 pages, then the feature
+ *  is not used.
+ */
+static IMG_UINT32 _CalculateFreelistReadyPages(RGX_FREELIST *psFreeList,
+                                               IMG_UINT32  ui32FLPages)
+{
+	IMG_UINT32  ui32ReadyFLPages = ((ui32FLPages * psFreeList->ui32GrowThreshold) / 100) &
+	                               ~((RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE/sizeof(IMG_UINT32))-1);
+
+	if (ui32ReadyFLPages > psFreeList->ui32GrowFLPages)
+	{
+		ui32ReadyFLPages = psFreeList->ui32GrowFLPages;
+	}
+
+	return ui32ReadyFLPages;
+}
+
+
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader)
+{
+	RGX_PMR_NODE	*psPMRNode;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32  ui32MappingTable = 0;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_DEVMEM_SIZE_T uistartPage;
+	PVRSRV_ERROR eError;
+	static const IMG_CHAR szAllocName[] = "Free List";
+
+	/* Are we allowed to grow ? */
+	if (psFreeList->ui32MaxFLPages - (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) < ui32NumPages)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: grow by %u pages denied. "
+				"Max PB size reached (current pages %u+%u/%u)",
+				psFreeList,
+				ui32NumPages,
+				psFreeList->ui32CurrentFLPages,
+				psFreeList->ui32ReadyFLPages,
+				psFreeList->ui32MaxFLPages));
+		return PVRSRV_ERROR_PBSIZE_ALREADY_MAX;
+	}
+
+	/* Allocate kernel memory block structure */
+	psPMRNode = OSAllocMem(sizeof(*psPMRNode));
+	if (psPMRNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to allocate host data structure",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocHost;
+	}
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages
+	 */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+
+	/*
+	 *  The PM never takes the last page in a freelist, so if this block
+	 *  of pages is the first one and there is no ability to grow, then
+	 *  we can skip allocating one 4K page for the lowest entry.
+	 */
+	if (OSGetPageSize() > RGX_BIF_PM_PHYSICAL_PAGE_SIZE)
+	{
+		/*
+		 * Allocation size will be rounded up to the OS page size,
+		 * any attempt to change it a bit now will be invalidated later.
+		 */
+		psPMRNode->bFirstPageMissing = IMG_FALSE;
+	}
+	else
+	{
+		psPMRNode->bFirstPageMissing = (psFreeList->ui32GrowFLPages == 0  &&  ui32NumPages > 1);
+	}
+
+	psPMRNode->ui32NumPages = ui32NumPages;
+	psPMRNode->psFreeList = psFreeList;
+
+	/* Allocate Memory Block */
+	PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Allocate PB Block (Pages %08X)", ui32NumPages);
+	uiSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	if (psPMRNode->bFirstPageMissing)
+	{
+		uiSize -= RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	}
+
+	eError = PhysmemNewRamBackedPMR(psFreeList->psConnection,
+			psFreeList->psDevInfo->psDeviceNode,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+			PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+			PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_PRIVATE) |
+			PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP,
+			sizeof(szAllocName),
+			szAllocName,
+			psFreeList->ownerPid,
+			&psPMRNode->psPMR,
+			PDUMP_NONE,
+			NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate PB block of size: 0x%016" IMG_UINT64_FMTSPECX,
+				 __func__,
+				 (IMG_UINT64)uiSize));
+		goto ErrorBlockAlloc;
+	}
+
+	/* Zeroing physical pages pointed by the PMR */
+	if (psFreeList->psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	uistartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((uistartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRNode->psPMR,
+	                                    psFreeList->ownerPid);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: call to RIWritePMREntryWithOwnerKM failed (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+	}
+
+	 /* Attach RI information */
+	eError = RIWriteMEMDESCEntryKM(psFreeList->psConnection,
+	                               psFreeList->psDevInfo->psDeviceNode,
+	                               psPMRNode->psPMR,
+	                               OSStringNLength(szAllocName, DEVMEM_ANNOTATION_MAX_LEN),
+	                               szAllocName,
+	                               0,
+	                               uiSize,
+	                               0,
+	                               &psPMRNode->hRIHandle);
+	PVR_LOG_IF_ERROR(eError, "RIWriteMEMDESCEntryKM");
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	/* write Freelist with Memory Block physical addresses */
+	eError = PMRWritePMPageList(
+						/* Target PMR, offset, and length */
+						psFreeList->psFreeListPMR,
+						(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+						(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+						/* Referenced PMR, and "page" granularity */
+						psPMRNode->psPMR,
+						RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+						&psPMRNode->psPageList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to write pages of Node %p",
+				 __func__,
+				 psPMRNode));
+		goto ErrorPopulateFreelist;
+	}
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+	/* Copy freelist memory to shadow freelist */
+	{
+		const IMG_UINT32 ui32FLMaxSize = psFreeList->ui32MaxFLPages * sizeof(IMG_UINT32);
+		const IMG_UINT32 ui32MapSize = ui32FLMaxSize * 2;
+		const IMG_UINT32 ui32CopyOffset = uiOffset - psFreeList->uiFreeListPMROffset;
+		IMG_BYTE *pFLMapAddr;
+		size_t uiNumBytes;
+		PVRSRV_ERROR res;
+		IMG_HANDLE hMapHandle;
+
+		/* Check for overflow. Validate size and offset. */
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize > psFreeList->uiFreeListPMROffset,
+		                          eError, ErrorPopulateFreelist);
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize <= PMR_LogicalSize(psFreeList->psFreeListPMR),
+		                          eError, ErrorPopulateFreelist);
+
+		/* Map both the FL and the shadow FL */
+		res = PMRAcquireKernelMappingData(psFreeList->psFreeListPMR, psFreeList->uiFreeListPMROffset, ui32MapSize,
+		                                  (void**) &pFLMapAddr, &uiNumBytes, &hMapHandle);
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map freelist (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+
+		/* Copy only the newly added memory */
+		OSCachedMemCopy(pFLMapAddr + ui32FLMaxSize + ui32CopyOffset, pFLMapAddr + ui32CopyOffset , uiLength);
+		OSWriteMemoryBarrier(pFLMapAddr);
+
+#if defined(PDUMP)
+		PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Initialize shadow freelist");
+
+		/* Translate memcpy to pdump */
+		{
+			IMG_DEVMEM_OFFSET_T uiCurrOffset;
+
+			for (uiCurrOffset = uiOffset; (uiCurrOffset - uiOffset) < uiLength; uiCurrOffset += sizeof(IMG_UINT32))
+			{
+				PMRPDumpCopyMem32(psFreeList->psFreeListPMR,
+				                  uiCurrOffset + ui32FLMaxSize,
+				                  psFreeList->psFreeListPMR,
+				                  uiCurrOffset,
+				                  ":SYSMEM:$1",
+				                  PDUMP_FLAGS_CONTINUOUS);
+			}
+		}
+#endif
+
+
+		res = PMRReleaseKernelMappingData(psFreeList->psFreeListPMR, hMapHandle);
+
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to release freelist mapping (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+	}
+#endif
+
+	/* We add It must be added to the tail, otherwise the freelist population won't work */
+	dllist_add_to_head(pListHeader, &psPMRNode->sMemoryBlock);
+
+	/* Update number of available pages */
+	psFreeList->ui32CurrentFLPages += ui32NumPages;
+
+	/* Update statistics (needs to happen before the ReadyFL calculation to also count those pages) */
+	if (psFreeList->ui32NumHighPages < psFreeList->ui32CurrentFLPages)
+	{
+		psFreeList->ui32NumHighPages = psFreeList->ui32CurrentFLPages;
+	}
+
+	/* Reserve a number ready pages to allow the FW to process OOM quickly and asynchronously request a grow. */
+	psFreeList->ui32ReadyFLPages    = _CalculateFreelistReadyPages(psFreeList, psFreeList->ui32CurrentFLPages);
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+
+	if (psFreeList->bCheckFreelist)
+	{
+		/*
+		 *  We can only calculate the freelist checksum when the list is full
+		 *  (e.g. at initial creation time). At other times the checksum cannot
+		 *  be calculated and has to be disabled for this freelist.
+		 */
+		if ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages)
+		{
+			_CheckFreelist(psFreeList, ui32NumPages, 0, &psFreeList->ui64FreelistChecksum);
+		}
+		else
+		{
+			psFreeList->ui64FreelistChecksum = 0;
+		}
+	}
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: %s %u pages (pages=%u+%u/%u checksum=0x%016" IMG_UINT64_FMTSPECx "%s)",
+			psFreeList,
+	        ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages ? "Create initial" : "Grow by"),
+			ui32NumPages,
+			psFreeList->ui32CurrentFLPages,
+			psFreeList->ui32ReadyFLPages,
+			psFreeList->ui32MaxFLPages,
+			psFreeList->ui64FreelistChecksum,
+			(psPMRNode->bFirstPageMissing ? " - lowest page not allocated" : "")));
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+ErrorPopulateFreelist:
+	(void) PMRUnrefPMR(psPMRNode->psPMR);
+
+ErrorBlockAlloc:
+	OSFreeMem(psPMRNode);
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+ErrorAllocHost:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR RGXShrinkFreeList(PDLLIST_NODE pListHeader,
+										RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode;
+	RGX_PMR_NODE *psPMRNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32OldValue;
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages value
+	 */
+	PVR_ASSERT(pListHeader);
+	PVR_ASSERT(psFreeList);
+	PVR_ASSERT(psFreeList->psDevInfo);
+	PVR_ASSERT(psFreeList->psDevInfo->hLockFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+	/* Get node from head of list and remove it */
+	psNode = dllist_get_next_node(pListHeader);
+	if (psNode)
+	{
+		dllist_remove_node(psNode);
+
+		psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+		PVR_ASSERT(psPMRNode);
+		PVR_ASSERT(psPMRNode->psPMR);
+		PVR_ASSERT(psPMRNode->psFreeList);
+
+		/* remove block from freelist list */
+
+		/* Unwrite Freelist with Memory Block physical addresses */
+		eError = PMRUnwritePMPageList(psPMRNode->psPageList);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to unwrite pages of Node %p",
+					 __func__,
+					 psPMRNode));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+		if (psPMRNode->hRIHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			eError = RIDeleteMEMDESCEntryKM(psPMRNode->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "RIDeleteMEMDESCEntryKM");
+		}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+		/* Free PMR (We should be the only one that holds a ref on the PMR) */
+		eError = PMRUnrefPMR(psPMRNode->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to free PB block %p (%s)",
+					 __func__,
+					 psPMRNode->psPMR,
+					 PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+		/* update available pages in freelist */
+		ui32OldValue = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+
+		/*
+		 * Deallocated pages should first be deducted from ReadyPages bank, once
+		 * there are no more left, start deducting them from CurrentPage bank.
+		 */
+		if (psPMRNode->ui32NumPages > psFreeList->ui32ReadyFLPages)
+		{
+			psFreeList->ui32CurrentFLPages -= psPMRNode->ui32NumPages - psFreeList->ui32ReadyFLPages;
+			psFreeList->ui32ReadyFLPages = 0;
+		}
+		else
+		{
+			psFreeList->ui32ReadyFLPages -= psPMRNode->ui32NumPages;
+		}
+
+		/* check underflow */
+		PVR_ASSERT(ui32OldValue > (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: shrink by %u pages (current pages %u/%u)",
+								psFreeList,
+								psPMRNode->ui32NumPages,
+								psFreeList->ui32CurrentFLPages,
+								psFreeList->ui32MaxFLPages));
+
+		OSFreeMem(psPMRNode);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: shrink denied. PB already at initial PB size (%u pages)",
+								psFreeList,
+								psFreeList->ui32InitFLPages));
+		eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
+	}
+
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	return eError;
+}
+
+static RGX_FREELIST *FindFreeList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FreelistID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_FREELIST *psFreeList = NULL;
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psThisFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		if (psThisFreeList->ui32FreelistID == ui32FreelistID)
+		{
+			psFreeList = psThisFreeList;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevInfo->hLockFreeList);
+	return psFreeList;
+}
+
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT32 ui32FreelistID)
+{
+	RGX_FREELIST *psFreeList = NULL;
+	RGXFWIF_KCCB_CMD s3DCCBCmd;
+	IMG_UINT32 ui32GrowValue;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	psFreeList = FindFreeList(psDevInfo, ui32FreelistID);
+	if (psFreeList == NULL)
+	{
+		/* Should never happen */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "FreeList Lookup for FreeList ID 0x%08x failed (Populate)",
+		         ui32FreelistID));
+		PVR_ASSERT(IMG_FALSE);
+		return;
+	}
+
+	/* Since the FW made the request, it has already consumed the ready pages, update the host struct */
+	psFreeList->ui32CurrentFLPages += psFreeList->ui32ReadyFLPages;
+	psFreeList->ui32ReadyFLPages = 0;
+
+	/* Try to grow the freelist */
+	eError = RGXGrowFreeList(psFreeList,
+	                         psFreeList->ui32GrowFLPages,
+	                         &psFreeList->sMemoryBlockHead);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Grow successful, return size of grow size */
+		ui32GrowValue = psFreeList->ui32GrowFLPages;
+
+		psFreeList->ui32NumGrowReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		/* Update Stats */
+		PVRSRVStatsUpdateFreelistStats(psDevInfo->psDeviceNode,
+		                               0,
+		                               1, /* Add 1 to the appropriate counter (Requests by FW) */
+		                               psFreeList->ui32InitFLPages,
+		                               psFreeList->ui32NumHighPages,
+		                               psFreeList->ownerPid);
+
+#endif
+
+	}
+	else
+	{
+		/* Grow failed */
+		ui32GrowValue = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Grow for FreeList %p failed (%s)",
+		         psFreeList,
+		         PVRSRVGetErrorString(eError)));
+	}
+
+	/* send feedback */
+	s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE;
+	s3DCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32DeltaPages = ui32GrowValue;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages = psFreeList->ui32ReadyFLPages;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: Grow pages=%u, new pages=%u, ready pages=%u, counter=%d",
+	         psFreeList,
+	         ui32GrowValue,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages,
+	         psFreeList->ui32NumGrowReqByFW));
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_3D,
+		                            &s3DCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+static void _RGXFreeListReconstruction(PDLLIST_NODE psNode)
+{
+
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	RGX_FREELIST			*psFreeList;
+	RGX_PMR_NODE			*psPMRNode;
+	PVRSRV_ERROR			eError;
+	IMG_DEVMEM_OFFSET_T		uiOffset;
+	IMG_DEVMEM_SIZE_T		uiLength;
+	IMG_UINT32				ui32StartPage;
+
+	psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	psFreeList = psPMRNode->psFreeList;
+	PVR_ASSERT(psFreeList);
+	psDevInfo = psFreeList->psDevInfo;
+	PVR_ASSERT(psDevInfo);
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	ui32StartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((ui32StartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+	PMRUnwritePMPageList(psPMRNode->psPageList);
+	psPMRNode->psPageList = NULL;
+	eError = PMRWritePMPageList(
+			/* Target PMR, offset, and length */
+			psFreeList->psFreeListPMR,
+			(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+			(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+			/* Referenced PMR, and "page" granularity */
+			psPMRNode->psPMR,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			&psPMRNode->psPageList);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error (%s) writing FL 0x%08x",
+				__func__,
+				PVRSRVGetErrorString(eError),
+				(IMG_UINT32)psFreeList->ui32FreelistID));
+	}
+
+	/* Zeroing physical pages pointed by the reconstructed freelist */
+	if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	psFreeList->ui32CurrentFLPages += psPMRNode->ui32NumPages;
+}
+
+
+static PVRSRV_ERROR RGXReconstructFreeList(RGX_FREELIST *psFreeList)
+{
+	IMG_UINT32        ui32OriginalFLPages;
+	DLLIST_NODE       *psNode, *psNext;
+	RGXFWIF_FREELIST  *psFWFreeList;
+	PVRSRV_ERROR      eError;
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: Reconstructing freelist %p (ID=%u)", psFreeList, psFreeList->ui32FreelistID));
+
+	/* Do the FreeList Reconstruction */
+	ui32OriginalFLPages            = psFreeList->ui32CurrentFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+
+	/* Reconstructing Init FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Reconstructing Grow FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Ready pages are allocated but kept hidden until OOM occurs. */
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+	if (psFreeList->ui32CurrentFLPages != ui32OriginalFLPages)
+	{
+		PVR_ASSERT(psFreeList->ui32CurrentFLPages == ui32OriginalFLPages);
+		return PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED;
+	}
+
+	/* Update firmware freelist structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psFWFreeList->ui32MaxPages     = psFreeList->ui32MaxFLPages;
+	psFWFreeList->ui32GrowPages    = psFreeList->ui32GrowFLPages;
+	psFWFreeList->ui32CurrentPages = psFreeList->ui32CurrentFLPages;
+	psFWFreeList->ui32FreeListID   = psFreeList->ui32FreelistID;
+	psFWFreeList->bGrowPending     = IMG_FALSE;
+	psFWFreeList->ui32ReadyPages   = psFreeList->ui32ReadyFLPages;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psFWFreeList->ui32CurrentStackTop       = psFWFreeList->ui32CurrentPages - 1;
+	psFWFreeList->ui64CurrentDevVAddr       = (psFWFreeList->psFreeListDevVAddr.uiAddr +
+				((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+				~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+	psFWFreeList->ui32AllocatedPageCount    = 0;
+	psFWFreeList->ui32AllocatedMMUPageCount = 0;
+#else
+	psFWFreeList->bUpdatePending                   = IMG_FALSE;
+	psFWFreeList->ui32UpdateNewPages               = 0;
+	psFWFreeList->ui32UpdateNewReadyPages          = 0;
+	psFWFreeList->sFreeListLastGrowDevVAddr.uiAddr = 0;
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+#if !defined(PM_INTERACTIVE_MODE)
+	/* Reset freelist state buffer */
+	{
+		RGX_PM_FREELISTSTATE_BUFFER		sFLState;
+		size_t uiNbBytes;
+		IMG_DEV_VIRTADDR sFLBaseAddr;
+
+		eError = PMR_ReadBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+
+		sFLBaseAddr.uiAddr = (psFreeList->sFreeListBaseDevVAddr.uiAddr +
+		                      ((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+		                      ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+		/* Note: Freelist base address is packed shifted down. */
+		RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(sFLState, sFLBaseAddr.uiAddr >> RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(sFLState, psFreeList->ui32CurrentFLPages - 1);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(sFLState, 0);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(sFLState, 0);
+
+		eError = PMR_WriteBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+	}
+#endif
+
+	/* Check the Freelist checksum if required (as the list is fully populated) */
+	if (psFreeList->bCheckFreelist)
+	{
+		IMG_UINT64  ui64CheckSum;
+
+		_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+	}
+
+	return eError;
+}
+
+
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                              IMG_UINT32 ui32FreelistsCount,
+                                              const IMG_UINT32 *paui32Freelists)
+{
+	PVRSRV_ERROR      eError = PVRSRV_OK;
+	DLLIST_NODE       *psNode, *psNext;
+	IMG_UINT32        ui32Loop;
+	RGXFWIF_KCCB_CMD  sTACCBCmd;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE       *psNodeHWRTData, *psNextHWRTData;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+	RGXFWIF_HWRTDATA     *psHWRTData;
+#endif
+	IMG_UINT32        ui32PIDCount = 0;
+	IMG_UINT32        aui32PIDList[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+
+	PVR_ASSERT(psDevInfo != NULL);
+	PVR_ASSERT(ui32FreelistsCount <= RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT);
+	if (ui32FreelistsCount > RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT)
+	{
+		ui32FreelistsCount = RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT;
+	}
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: %u freelist(s) requested for reconstruction", ui32FreelistsCount));
+
+	/*
+	 *  Initialise the response command (in case we don't find a freelist ID).
+	 *  Also copy the list to the 'final' freelist array.
+	 */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE;
+	sTACCBCmd.uCmdData.sFreeListsReconstructionData.ui32FreelistsCount = ui32FreelistsCount;
+
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] = paui32Freelists[ui32Loop] |
+				RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+	}
+
+	/*
+	 *  All freelists belonging to the same PID will be reconstructed.
+	 *  This simplifies tracking for AGP, since there is no longer a
+	 *  single global freelist per local freelist. The list of unique
+	 *  PIDs is calculated from the list of freelists in this first pass.
+	 */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList   = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+		{
+			IMG_UINT32 ui32PIDLoop;
+
+			if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+			{
+				for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+				{
+					if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+					{
+						break;
+					}
+				}
+
+				if (ui32PIDLoop == ui32PIDCount)
+				{
+					aui32PIDList[ui32PIDCount++] = psFreeList->ownerPid;
+				}
+			}
+		}
+	}
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList  = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		IMG_BOOL      bReconstruct = IMG_FALSE;
+		IMG_UINT32    ui32PIDLoop;
+
+		/*
+		 *  Check if this freelist needs to be reconstructed (is it in the list
+		 *  of PIDs which will have every single one of their freelists reconstructed)
+		 */
+		for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+		{
+			if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+			{
+				bReconstruct = IMG_TRUE;
+				break;
+			}
+		}
+
+		if (bReconstruct)
+		{
+			eError = RGXReconstructFreeList(psFreeList);
+			if (eError == PVRSRV_OK)
+			{
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+				/* Mark all HWRTData's of reconstructing local freelists as HWR (applies to TA/3D's not finished yet) */
+				dllist_foreach_node(&psFreeList->sNodeHWRTDataHead, psNodeHWRTData, psNextHWRTData)
+				{
+					psKMHWRTDataSet = IMG_CONTAINER_OF(psNodeHWRTData, RGX_KM_HW_RT_DATASET, sNodeHWRTData);
+					eError = DevmemAcquireCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc, (void **)&psHWRTData);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+									"Devmem AcquireCpuVirtAddr Failed during Reconstructing of FreeList, FwMemDesc(%p),psHWRTData(%p)",
+									psKMHWRTDataSet->psHWRTDataFwMemDesc,
+									psHWRTData));
+						continue;
+					}
+
+					psHWRTData->eState = RGXFWIF_RTDATA_STATE_HWR;
+					psHWRTData->ui32HWRTDataFlags &= ~HWRTDATA_HAS_LAST_TA;
+					RGXFwSharedMemCacheOpValue(psHWRTData->eState, FLUSH);
+					RGXFwSharedMemCacheOpValue(psHWRTData->ui32HWRTDataFlags, FLUSH);
+
+					DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+				}
+#endif
+
+				/* Update the response for this freelist if it was specifically requested for reconstruction. */
+				for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+				{
+					if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+					{
+						/* Reconstruction of this requested freelist was successful... */
+						sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &= ~RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+						break;
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Reconstructing of FreeList %p failed (%s)",
+						psFreeList,
+						PVRSRVGetErrorString(eError)));
+			}
+		}
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Check that all freelists were found and reconstructed... */
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		PVR_ASSERT((sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &
+		            RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG) == 0);
+	}
+
+	/* send feedback */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+/* Create a single HWRTData instance */
+static PVRSRV_ERROR RGXCreateHWRTData_aux(CONNECTION_DATA		*psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_DEV_VIRTADDR		psVHeapTableDevVAddr,
+                                          IMG_DEV_VIRTADDR		sPMDataDevVAddr, /* per-HWRTData */
+                                          RGX_FREELIST			*apsFreeLists[RGXFW_MAX_FREELISTS],
+                                          IMG_DEV_VIRTADDR		sTailPtrsDevVAddr,
+                                          IMG_DEV_VIRTADDR		sMacrotileArrayDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR		sRgnHeaderDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR		sRTCDevVAddr,
+                                          IMG_UINT16			ui16MaxRTs,
+                                          RGX_HWRTDATA_COMMON_COOKIE	*psHWRTDataCommonCookie,
+                                          RGX_KM_HW_RT_DATASET	**ppsKMHWRTDataSet) /* per-HWRTData */
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	/* KM cookie storing all the FW/HW data */
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+
+	/* local pointers for memory descriptors of FW allocations */
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc = NULL;
+
+	/* local pointer for CPU-mapped [FW]HWRTData */
+	RGXFWIF_HWRTDATA *psHWRTData = NULL;
+
+	/*
+	 * Fill in a local copy of RGXFWIF_HWRTDATA first to reduces reads and writes to
+	 * device memory. Using a byte array buffer avoids the limitation that some OSs
+	 * cannot align local variables to 64 bytes, usually needed by RGXFWIF_HWRTDATA
+	 */
+	IMG_BYTE aui8HWRTDataLocal[sizeof(RGXFWIF_HWRTDATA)] = {0};
+	RGXFWIF_HWRTDATA *psHWRTDataLocal = (RGXFWIF_HWRTDATA *)aui8HWRTDataLocal;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Prepare the HW RT DataSet struct */
+	psKMHWRTDataSet = OSAllocZMem(sizeof(*psKMHWRTDataSet));
+	if (psKMHWRTDataSet == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto AllocError;
+	}
+
+	*ppsKMHWRTDataSet = psKMHWRTDataSet;
+	psKMHWRTDataSet->psDeviceNode = psDeviceNode;
+
+	psKMHWRTDataSet->psHWRTDataCommonCookie = psHWRTDataCommonCookie;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+	 * This FW RT-Data is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency and write-combine will
+	 * suffice on the CPU side. (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(RGXFWIF_HWRTDATA),
+	                          RGX_FWCOMCTX_ALLOCFLAGS,
+	                          "FwHwRTData",
+	                          &psHWRTDataFwMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: DevmemAllocate for RGX_FWIF_HWRTDATA failed",
+				__func__));
+		goto FWRTDataAllocateError;
+	}
+
+	psKMHWRTDataSet->psHWRTDataFwMemDesc = psHWRTDataFwMemDesc;
+	eError = RGXSetFirmwareAddress( &psKMHWRTDataSet->sHWRTDataFwAddr,
+							psHWRTDataFwMemDesc,
+							0,
+							RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", FWRTDataFwAddrError);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataFwMemDesc,
+									  (void **)&psHWRTData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTDataCpuMapError);
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psVHeapTableDevVAddr = psVHeapTableDevVAddr;
+#endif
+
+	psHWRTDataLocal->sHWRTDataCommonFwAddr = psHWRTDataCommonCookie->sHWRTDataCommonFwAddr;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psPMMListDevVAddr = sPMDataDevVAddr;
+#else
+	psHWRTDataLocal->sPMRenderStateDevVAddr = sPMDataDevVAddr;
+#endif
+
+	psHWRTDataLocal->sTailPtrsDevVAddr     = sTailPtrsDevVAddr;
+	psHWRTDataLocal->sMacrotileArrayDevVAddr = sMacrotileArrayDevVAddr;
+	psHWRTDataLocal->sRgnHeaderDevVAddr		= sRgnHeaderDevVAddr;
+	psHWRTDataLocal->sRTCDevVAddr			= sRTCDevVAddr;
+
+	{
+		RGXFWIF_RTA_CTL *psRTACtl = &psHWRTDataLocal->sRTACtl;
+
+		psRTACtl->ui32RenderTargetIndex = 0;
+		psRTACtl->ui32ActiveRenderTargets = 0;
+		psRTACtl->sValidRenderTargets.ui32Addr = 0;
+		psRTACtl->sRTANumPartialRenders.ui32Addr = 0;
+		psRTACtl->ui32MaxRTs = (IMG_UINT32) ui16MaxRTs;
+
+		if (ui16MaxRTs > 1)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for shadow render target cache");
+			eError = DevmemFwAllocate(	psDevInfo,
+										ui16MaxRTs * sizeof(IMG_UINT32),
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwShadowRTCache",
+										&psRTArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTArryError;
+			}
+
+			psKMHWRTDataSet->psRTArrayFwMemDesc = psRTArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sValidRenderTargets,
+									psRTArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", FWAllocateRTArryFwAddrError);
+
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for tracking renders accumulation");
+			eError = DevmemFwAllocate(psDevInfo,
+			                          ui16MaxRTs * sizeof(IMG_UINT32),
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+					                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+					                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+					                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+					                  "FwRendersAccumulation",
+					                  &psRendersAccArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s) (renders accumulation)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTAccArryError;
+			}
+
+			psKMHWRTDataSet->psRendersAccArrayFwMemDesc = psRendersAccArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sRTANumPartialRenders,
+									psRendersAccArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", FWAllocRTAccArryFwAddrError);
+		}
+	}
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		psKMHWRTDataSet->apsFreeLists[ui32Loop] = apsFreeLists[ui32Loop];
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount++;
+		psHWRTDataLocal->apsFreeLists[ui32Loop].ui32Addr = psKMHWRTDataSet->apsFreeLists[ui32Loop]->sFreeListFWDevVAddr.ui32Addr;
+		/* Invalid initial snapshot value. The snapshot is always taken during the first
+		 * kick and hence this value gets replaced, so it's safe to set it to zero.
+		 */
+		psHWRTDataLocal->aui32FreeListHWRSnapshot[ui32Loop] = 0;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_add_to_tail(&apsFreeLists[RGXFW_LOCAL_FREELIST]->sNodeHWRTDataHead, &(psKMHWRTDataSet->sNodeHWRTData));
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	OSCachedMemCopy(psHWRTData, psHWRTDataLocal, sizeof(*psHWRTDataLocal));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTData 0x%08X", psKMHWRTDataSet->sHWRTDataFwAddr.ui32Addr);
+	DevmemPDumpLoadMem(psKMHWRTDataSet->psHWRTDataFwMemDesc, 0, sizeof(*psHWRTData), PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psHWRTData, FLUSH);
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	return PVRSRV_OK;
+
+FWAllocRTAccArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRendersAccArrayFwMemDesc);
+FWAllocateRTAccArryError:
+	RGXUnsetFirmwareAddress(psRTArrayFwMemDesc);
+FWAllocateRTArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRTArrayFwMemDesc);
+FWAllocateRTArryError:
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataCpuMapError:
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataAllocateError:
+	*ppsKMHWRTDataSet = NULL;
+	OSFreeMem(psKMHWRTDataSet);
+
+AllocError:
+	return eError;
+}
+
+static void RGXDestroyHWRTData_aux(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	if (psKMHWRTDataSet == NULL)
+	{
+		return;
+	}
+
+	psDevInfo = psKMHWRTDataSet->psDeviceNode->pvDevice;
+
+	if (psKMHWRTDataSet->psRTArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRTArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRTArrayFwMemDesc);
+	}
+
+	if (psKMHWRTDataSet->psRendersAccArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+	}
+
+	/* Decrease freelist refcount */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		PVR_ASSERT(psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount > 0);
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount--;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_remove_node(&psKMHWRTDataSet->sNodeHWRTData);
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Freeing the memory has to happen _after_ removing the HWRTData from the freelist
+	 * otherwise we risk traversing the freelist to find a pointer from a freed data structure */
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+
+	OSFreeMem(psKMHWRTDataSet);
+}
+
+/* Create set of HWRTData(s) and bind it with a shared FW HWRTDataCommon */
+PVRSRV_ERROR RGXCreateHWRTDataSet(
+		CONNECTION_DATA			*psConnection,
+		PVRSRV_DEVICE_NODE		*psDeviceNode,
+		IMG_DEV_VIRTADDR	asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_DEV_VIRTADDR		asPMMListDevVAddr[RGXMKIF_NUM_RTDATAS],
+		RGX_FREELIST			*apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+		IMG_UINT32           ui32ScreenPixelMax,
+		IMG_UINT64           ui64MultiSampleCtl,
+		IMG_UINT64           ui64FlippedMultiSampleCtl,
+		IMG_UINT32           ui32TPCStride,
+		IMG_DEV_VIRTADDR		asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT32           ui32TPCSize,
+		IMG_UINT32           ui32TEScreen,
+		IMG_UINT32           ui32TEAA,
+		IMG_UINT32           ui32TEMTILE1,
+		IMG_UINT32           ui32TEMTILE2,
+		IMG_UINT32           ui32MTileStride,
+		IMG_UINT32                 ui32ISPMergeLowerX,
+		IMG_UINT32                 ui32ISPMergeLowerY,
+		IMG_UINT32                 ui32ISPMergeUpperX,
+		IMG_UINT32                 ui32ISPMergeUpperY,
+		IMG_UINT32                 ui32ISPMergeScaleX,
+		IMG_UINT32                 ui32ISPMergeScaleY,
+		IMG_DEV_VIRTADDR	asMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR	asRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR	asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT32			uiRgnHeaderSize,
+		IMG_UINT32			ui32ISPMtileSize,
+		IMG_UINT16			ui16MaxRTs,
+		RGX_KM_HW_RT_DATASET	*pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(asVHeapTableDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asPMMListDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(apsFreeLists);
+	PVR_UNREFERENCED_PARAMETER(ui32ScreenPixelMax);
+	PVR_UNREFERENCED_PARAMETER(ui64MultiSampleCtl);
+	PVR_UNREFERENCED_PARAMETER(ui64FlippedMultiSampleCtl);
+	PVR_UNREFERENCED_PARAMETER(ui32TPCStride);
+	PVR_UNREFERENCED_PARAMETER(asTailPtrsDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32TPCSize);
+	PVR_UNREFERENCED_PARAMETER(ui32TEScreen);
+	PVR_UNREFERENCED_PARAMETER(ui32TEAA);
+	PVR_UNREFERENCED_PARAMETER(ui32TEMTILE1);
+	PVR_UNREFERENCED_PARAMETER(ui32TEMTILE2);
+	PVR_UNREFERENCED_PARAMETER(ui32MTileStride);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeLowerX);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeLowerY);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeUpperX);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeUpperY);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeScaleX);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMergeScaleY);
+	PVR_UNREFERENCED_PARAMETER(asMacrotileArrayDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asRgnHeaderDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asRTCDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(uiRgnHeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ISPMtileSize);
+	PVR_UNREFERENCED_PARAMETER(ui16MaxRTs);
+	PVR_UNREFERENCED_PARAMETER(pasKMHWRTDataSet);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+/* Create set of HWRTData(s) and bind it with a shared FW HWRTDataCommon */
+PVRSRV_ERROR RGXCreateHWRTDataSet2(
+        CONNECTION_DATA         *psConnection,
+        PVRSRV_DEVICE_NODE      *psDeviceNode,
+        IMG_DEV_VIRTADDR        asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        DEVMEMINT_RESERVATION   *psPMMListsReservation,
+        RGX_FREELIST            *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+        IMG_UINT32              ui32ScreenPixelMax,
+        IMG_UINT64              ui64MultiSampleCtl,
+        IMG_UINT64              ui64FlippedMultiSampleCtl,
+        IMG_UINT32              ui32TPCStride,
+        IMG_DEV_VIRTADDR        asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        IMG_UINT32              ui32TPCSize,
+        IMG_UINT32              ui32TEScreen,
+        IMG_UINT32              ui32TEAA,
+        IMG_UINT32              ui32TEMTILE1,
+        IMG_UINT32              ui32TEMTILE2,
+        IMG_UINT32              ui32MTileStride,
+        IMG_UINT32              ui32ISPMergeLowerX,
+        IMG_UINT32              ui32ISPMergeLowerY,
+        IMG_UINT32              ui32ISPMergeUpperX,
+        IMG_UINT32              ui32ISPMergeUpperY,
+        IMG_UINT32              ui32ISPMergeScaleX,
+        IMG_UINT32              ui32ISPMergeScaleY,
+        IMG_DEV_VIRTADDR        asMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+        IMG_DEV_VIRTADDR        asRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+        IMG_DEV_VIRTADDR        asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        IMG_UINT32              uiRgnHeaderSize,
+        IMG_UINT32              ui32ISPMtileSize,
+        IMG_UINT16              ui16MaxRTs,
+        RGX_KM_HW_RT_DATASET    *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RTDataID;
+	IMG_UINT32 ui32GlobalFLMaxPages, ui32LocalFLMaxPages;
+	IMG_DEVMEM_SIZE_T ui64MListSize;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	RGX_HWRTDATA_COMMON_COOKIE *psHWRTDataCommonCookie;
+	RGXFWIF_HWRTDATA_COMMON *psHWRTDataCommon;
+	DEVMEM_MEMDESC *psHWRTDataCommonFwMemDesc;
+	RGXFWIF_DEV_VIRTADDR sHWRTDataCommonFwAddr;
+
+	PMR* psMListsPMR = NULL;
+	IMG_DEV_VIRTADDR sMListsDevVAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(0 < ui16MaxRTs && ui16MaxRTs <= RGX_MAX_TA_RENDER_TARGETS, "Number of TA Render targets outside the range (0, RGX_MAX_TA_RENDER_TARGETS) is unsupported");
+
+	/* Check if freelists have correct sizes */
+	eError = ValidateFreeListSizes(apsFreeLists,
+	                               &ui32LocalFLMaxPages,
+	                               &ui32GlobalFLMaxPages);
+	PVR_LOG_RETURN_IF_ERROR(eError, "Invalid freelist sizes");
+
+	ui64MListSize = RGXCalcMListSize(psDeviceNode,
+	    ui32LocalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE,
+	    ui32GlobalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMMListsReservation,
+	                                          ui64MListSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psMListsPMR,
+	                                          &sMListsDevVAddr);
+	PVR_LOG_RETURN_IF_ERROR(eError, "Failed to obtain or validate MLIST buffer");
+
+	/* Prepare KM cleanup object for HWRTDataCommon FW object */
+	psHWRTDataCommonCookie = OSAllocZMem(sizeof(*psHWRTDataCommonCookie));
+	if (psHWRTDataCommonCookie == NULL)
+	{
+	    eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	    goto err_HWRTDataCommonCookieAlloc;
+	}
+
+	/*
+	 * This FW common context is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(RGXFWIF_HWRTDATA_COMMON),
+	                          RGX_FWCOMCTX_ALLOCFLAGS,
+	                          "FwHWRTDataCommon",
+	                          &psHWRTDataCommonFwMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+	    PVR_DPF((PVR_DBG_ERROR, "%s: DevmemAllocate for FwHWRTDataCommon failed", __func__));
+	    goto err_HWRTDataCommonAlloc;
+	}
+	eError = RGXSetFirmwareAddress(&sHWRTDataCommonFwAddr, psHWRTDataCommonFwMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", err_HWRTDataCommonFwAddr);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataCommonFwMemDesc, (void **)&psHWRTDataCommon);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", err_HWRTDataCommonVA);
+
+	psHWRTDataCommon->bTACachesNeedZeroing = IMG_FALSE;
+	psHWRTDataCommon->ui32ScreenPixelMax    = ui32ScreenPixelMax;
+	psHWRTDataCommon->ui64MultiSampleCtl = ui64MultiSampleCtl;
+	psHWRTDataCommon->ui64FlippedMultiSampleCtl = ui64FlippedMultiSampleCtl;
+	psHWRTDataCommon->ui32TPCStride         = ui32TPCStride;
+	psHWRTDataCommon->ui32TPCSize           = ui32TPCSize;
+	psHWRTDataCommon->ui32TEScreen          = ui32TEScreen;
+	psHWRTDataCommon->ui32TEAA              = ui32TEAA;
+	psHWRTDataCommon->ui32TEMTILE1          = ui32TEMTILE1;
+	psHWRTDataCommon->ui32TEMTILE2          = ui32TEMTILE2;
+	psHWRTDataCommon->ui32MTileStride       = ui32MTileStride;
+	psHWRTDataCommon->ui32ISPMergeLowerX    = ui32ISPMergeLowerX;
+	psHWRTDataCommon->ui32ISPMergeLowerY    = ui32ISPMergeLowerY;
+	psHWRTDataCommon->ui32ISPMergeUpperX    = ui32ISPMergeUpperX;
+	psHWRTDataCommon->ui32ISPMergeUpperY    = ui32ISPMergeUpperY;
+	psHWRTDataCommon->ui32ISPMergeScaleX    = ui32ISPMergeScaleX;
+	psHWRTDataCommon->ui32ISPMergeScaleY    = ui32ISPMergeScaleY;
+	psHWRTDataCommon->uiRgnHeaderSize       = uiRgnHeaderSize;
+	psHWRTDataCommon->ui32ISPMtileSize      = ui32ISPMtileSize;
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTDataCommon");
+	DevmemPDumpLoadMem(psHWRTDataCommonFwMemDesc, 0, sizeof(*psHWRTDataCommon), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psHWRTDataCommon, FLUSH);
+	DevmemReleaseCpuVirtAddr(psHWRTDataCommonFwMemDesc);
+
+	psHWRTDataCommonCookie->ui32RefCount = 0;
+	psHWRTDataCommonCookie->psHWRTDataCommonFwMemDesc = psHWRTDataCommonFwMemDesc;
+	psHWRTDataCommonCookie->sHWRTDataCommonFwAddr = sHWRTDataCommonFwAddr;
+
+	psHWRTDataCommonCookie->psPMMListsReservation = psPMMListsReservation;
+
+	/* Here we are creating a set of HWRTData(s)
+	   the number of elements in the set equals RGXMKIF_NUM_RTDATAS.
+	*/
+
+	for (ui32RTDataID = 0; ui32RTDataID < RGXMKIF_NUM_RTDATAS; ui32RTDataID++)
+	{
+		IMG_DEV_VIRTADDR sMListDevVAddr;
+		sMListDevVAddr.uiAddr = sMListsDevVAddr.uiAddr + ui32RTDataID * ui64MListSize;
+
+		eError = RGXCreateHWRTData_aux(
+		    psConnection,
+		    psDeviceNode,
+		    asVHeapTableDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+		    sMListDevVAddr,
+		    &apsFreeLists[(ui32RTDataID % RGXMKIF_NUM_GEOMDATAS) * RGXFW_MAX_FREELISTS],
+		    asTailPtrsDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+		    asMacrotileArrayDevVAddr[ui32RTDataID],
+		    asRgnHeaderDevVAddr[ui32RTDataID],
+		    asRTCDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+		    ui16MaxRTs,
+		    psHWRTDataCommonCookie,
+		    &pasKMHWRTDataSet[ui32RTDataID]);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Failed to create HWRTData [slot %u] (%s)",
+			        __func__,
+			        ui32RTDataID,
+			        PVRSRVGetErrorString(eError)));
+			goto err_HWRTDataAlloc;
+		}
+		psHWRTDataCommonCookie->ui32RefCount += 1;
+	}
+
+	return PVRSRV_OK;
+
+err_HWRTDataAlloc:
+	PVR_DPF((PVR_DBG_WARNING, "%s: err_HWRTDataAlloc %u",
+			 __func__, psHWRTDataCommonCookie->ui32RefCount));
+	if (pasKMHWRTDataSet)
+	{
+		for (ui32RTDataID = psHWRTDataCommonCookie->ui32RefCount; ui32RTDataID > 0; ui32RTDataID--)
+		{
+			if (pasKMHWRTDataSet[ui32RTDataID-1] != NULL)
+			{
+				RGXDestroyHWRTData_aux(pasKMHWRTDataSet[ui32RTDataID-1]);
+				pasKMHWRTDataSet[ui32RTDataID-1] = NULL;
+			}
+		}
+	}
+err_HWRTDataCommonVA:
+	RGXUnsetFirmwareAddress(psHWRTDataCommonFwMemDesc);
+err_HWRTDataCommonFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psHWRTDataCommonFwMemDesc);
+err_HWRTDataCommonAlloc:
+	OSFreeMem(psHWRTDataCommonCookie);
+err_HWRTDataCommonCookieAlloc:
+	UnrefAndReleaseCriticalBuffer(psPMMListsReservation);
+	return eError;
+}
+
+/* Destroy a single instance of HWRTData.
+   Additionally, destroy the HWRTDataCommon{Cookie} objects
+   when it is the last HWRTData within a corresponding set of HWRTDatas.
+*/
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_ERROR eError;
+	PRGXFWIF_HWRTDATA psHWRTData;
+	RGX_HWRTDATA_COMMON_COOKIE *psCommonCookie;
+
+	PVR_ASSERT(psKMHWRTDataSet);
+
+	psDevNode = psKMHWRTDataSet->psDeviceNode;
+
+	eError = RGXSetFirmwareAddress(&psHWRTData,
+	                               psKMHWRTDataSet->psHWRTDataFwMemDesc, 0,
+	                               RFW_FWADDR_NOREF_FLAG);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Cleanup HWRTData */
+	eError = RGXFWRequestHWRTDataCleanUp(psDevNode, psHWRTData);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDevNode,
+						   eError,
+						   RGXFWRequestHWRTDataCleanUp);
+
+	psCommonCookie = psKMHWRTDataSet->psHWRTDataCommonCookie;
+
+	RGXDestroyHWRTData_aux(psKMHWRTDataSet);
+
+	/* We've got past potential PVRSRV_ERROR_RETRY events, so we are sure
+	   that the HWRTDATA instance will be destroyed during this call.
+	   Consequently, we decrease the ref count for HWRTDataCommonCookie.
+
+	   NOTE: This ref count does not require locks or atomics.
+	   -------------------------------------------------------
+	     HWRTDatas bound into one pair are always destroyed sequentially,
+	     within a single loop on the Client side.
+	     The Common/Cookie objects always belong to only one pair of
+	     HWRTDatas, and ref count is used to ensure that the Common/Cookie
+	     objects will be destroyed after destruction of all HWRTDatas
+	     within a single pair.
+	*/
+	psCommonCookie->ui32RefCount--;
+
+	/* When ref count for HWRTDataCommonCookie hits ZERO
+	 * we have to destroy the HWRTDataCommon [FW object] and the cookie
+	 * [KM object] afterwards. */
+	if (psCommonCookie->ui32RefCount == 0)
+	{
+		RGXUnsetFirmwareAddress(psCommonCookie->psHWRTDataCommonFwMemDesc);
+
+		/* We don't need to flush the SLC before freeing.
+		 * FW RequestCleanUp has already done that for HWRTData, so we're fine
+		 * now. */
+
+		DevmemFwUnmapAndFree(psDevNode->pvDevice,
+		                     psCommonCookie->psHWRTDataCommonFwMemDesc);
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMMListsReservation);
+		OSFreeMem(psCommonCookie);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA			*psConnection,
+                               PVRSRV_DEVICE_NODE			*psDeviceNode,
+                               IMG_HANDLE				hMemCtxPrivData,
+                               IMG_UINT32				ui32MaxFLPages,
+                               IMG_UINT32				ui32InitFLPages,
+                               IMG_UINT32				ui32GrowFLPages,
+                               IMG_UINT32				ui32GrowParamThreshold,
+                               RGX_FREELIST				*psGlobalFreeList,
+                               IMG_BOOL					bCheckFreelist,
+                               DEVMEMINT_RESERVATION		*psFreeListReservation,
+                               RGX_FREELIST				**ppsFreeList)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_FREELIST   *psFWFreeList;
+	RGXFWIF_FREELIST   sFWFreeList = {{0}};
+	DEVMEM_MEMDESC     *psFWFreelistMemDesc;
+	RGX_FREELIST       *psFreeList;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR   sFreeListDevVAddr;
+	PMR*               psFreeListPMR = NULL;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psFreeListReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Failed to acquire reservation for freelist buffer",
+		        __func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	eError = DevmemIntGetReservationData(psFreeListReservation, &psFreeListPMR, &sFreeListDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Error from DevmemIntGetReservationData: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+
+		goto ErrorAllocHost;
+	}
+
+	/* Check if client properly allocated PMMETA_PROTECT */
+	if ((PMR_Flags(psFreeListPMR) & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must have PMMETA_PROTECT set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psFreeListPMR)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must not have OS_LINUX_MOVABLE set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+#endif
+
+	if (PMR_IsSparse(psFreeListPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Free list PMR cannot be sparse!",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto ErrorAllocHost;
+	}
+
+	/* Ref the PMR to prevent resource being destroyed before use */
+	eError = PMRRefPMR(psFreeListPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorAllocHost);
+
+	if (OSGetPageShift() > RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+	{
+		IMG_UINT32 ui32Size, ui32NewInitFLPages, ui32NewMaxFLPages, ui32NewGrowFLPages;
+
+		/* Round up number of FL pages to the next multiple of the OS page size */
+
+		ui32Size = ui32InitFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewInitFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32GrowFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewGrowFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32MaxFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewMaxFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Increased number of PB pages: Init %u -> %u, Grow %u -> %u, Max %u -> %u",
+		        __func__, ui32InitFLPages, ui32NewInitFLPages, ui32GrowFLPages, ui32NewGrowFLPages, ui32MaxFLPages, ui32NewMaxFLPages));
+
+		ui32InitFLPages = ui32NewInitFLPages;
+		ui32GrowFLPages = ui32NewGrowFLPages;
+		ui32MaxFLPages = ui32NewMaxFLPages;
+	}
+
+	/* Allocate kernel freelist struct */
+	psFreeList = OSAllocZMem(sizeof(*psFreeList));
+	if (psFreeList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: failed to allocate host data structure",
+		        __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRefPMR;
+	}
+
+	/*
+	 * This FW FreeList context is only mapped into kernel for initialisation
+	 * and reconstruction (at other times it is not mapped and only used by the
+	 * FW).
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(*psFWFreeList),
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+	                          PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwFreeList",
+	                          &psFWFreelistMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: DevmemAllocate for RGXFWIF_FREELIST failed",
+		        __func__));
+		goto FWFreeListAlloc;
+	}
+
+	/* Initialise host data structures */
+	psFreeList->psDevInfo = psDevInfo;
+	psFreeList->psConnection = psConnection;
+
+	psFreeList->psFreeListPMR = psFreeListPMR;
+	psFreeList->psFreeListReservation = psFreeListReservation;
+
+	psFreeList->uiFreeListPMROffset = 0U;
+
+	psFreeList->psFWFreelistMemDesc = psFWFreelistMemDesc;
+	eError = RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	/* psFreeList->ui32FreelistID set below with lock... */
+	psFreeList->ui32FreelistGlobalID = (psGlobalFreeList ? psGlobalFreeList->ui32FreelistID : 0);
+	psFreeList->ui32MaxFLPages = ui32MaxFLPages;
+	psFreeList->ui32InitFLPages = ui32InitFLPages;
+	psFreeList->ui32GrowFLPages = ui32GrowFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+	psFreeList->ui32ReadyFLPages = 0;
+	psFreeList->ui32GrowThreshold = ui32GrowParamThreshold;
+	psFreeList->ui64FreelistChecksum = 0;
+	psFreeList->ui32RefCount = 0;
+	psFreeList->bCheckFreelist = bCheckFreelist;
+	dllist_init(&psFreeList->sMemoryBlockHead);
+	dllist_init(&psFreeList->sMemoryBlockInitHead);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_init(&psFreeList->sNodeHWRTDataHead);
+#endif
+	psFreeList->ownerPid = OSGetCurrentClientProcessIDKM();
+
+
+	/* Add to list of freelists */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	psFreeList->ui32FreelistID = psDevInfo->ui32FreelistCurrID++;
+	dllist_add_to_tail(&psDevInfo->sFreeListHead, &psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+
+	/* Initialise FW data structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWFreeListCpuMap);
+
+	{
+		const IMG_UINT32 ui32ReadyPages = _CalculateFreelistReadyPages(psFreeList, ui32InitFLPages);
+
+		sFWFreeList.ui32MaxPages = ui32MaxFLPages;
+		sFWFreeList.ui32CurrentPages = ui32InitFLPages - ui32ReadyPages;
+		sFWFreeList.ui32GrowPages = ui32GrowFLPages;
+		sFWFreeList.psFreeListDevVAddr = sFreeListDevVAddr;
+#if defined(PM_INTERACTIVE_MODE)
+		sFWFreeList.ui32CurrentStackTop = sFWFreeList.ui32CurrentPages - 1;
+		sFWFreeList.ui64CurrentDevVAddr = (sFreeListDevVAddr.uiAddr +
+		        ((ui32MaxFLPages - sFWFreeList.ui32CurrentPages) * sizeof(IMG_UINT32))) &
+		                ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+#endif
+		sFWFreeList.ui32FreeListID = psFreeList->ui32FreelistID;
+		sFWFreeList.bGrowPending = IMG_FALSE;
+		sFWFreeList.ui32ReadyPages = ui32ReadyPages;
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+		/* Get the FW Memory Context address... */
+		eError = RGXSetFirmwareAddress(&sFWFreeList.psFWMemContext,
+		                               RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData),
+		                               0, RFW_FWADDR_NOREF_FLAG);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: RGXSetFirmwareAddress for RGXFWIF_FWMEMCONTEXT failed",
+			        __func__));
+			DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+			goto FWFreeListCpuMap;
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(hMemCtxPrivData);
+#endif
+	}
+
+	OSCachedMemCopy(psFWFreeList, &sFWFreeList, sizeof(sFWFreeList));
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "Freelist %p created: Max pages 0x%08x, Init pages 0x%08x, "
+	        "Max FL base address 0x%016" IMG_UINT64_FMTSPECx ", "
+	        "Init FL base address 0x%016" IMG_UINT64_FMTSPECx,
+	        psFreeList,
+	        ui32MaxFLPages,
+	        ui32InitFLPages,
+	        sFreeListDevVAddr.uiAddr,
+	        sFWFreeList.ui64CurrentDevVAddr));
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FW FreeList");
+	DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWFreeList), PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(PM_INTERACTIVE_MODE)
+	/*
+	 * Separate dump of the Freelist's number of Pages and stack pointer.
+	 * This allows to easily modify the PB size in the out2.txt files.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "FreeList TotalPages");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentPages),
+	                          sFWFreeList.ui32CurrentPages,
+	                          PDUMP_FLAGS_CONTINUOUS);
+	PDUMPCOMMENT(psDeviceNode, "FreeList StackPointer");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentStackTop),
+	                          sFWFreeList.ui32CurrentStackTop,
+	                          PDUMP_FLAGS_CONTINUOUS);
+#endif
+#endif
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+	/* Add initial PB block */
+	eError = RGXGrowFreeList(psFreeList,
+	                         ui32InitFLPages,
+	                         &psFreeList->sMemoryBlockInitHead);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: failed to allocate initial memory block for free list 0x%016" IMG_UINT64_FMTSPECx " (%s)",
+		         __func__,
+		         sFreeListDevVAddr.uiAddr,
+		         PVRSRVGetErrorString(eError)));
+		goto FWFreeListCpuMap;
+	}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Update Stats */
+	PVRSRVStatsUpdateFreelistStats(psDeviceNode,
+	                               1, /* Add 1 to the appropriate counter (Requests by App)*/
+	                               0,
+	                               psFreeList->ui32InitFLPages,
+	                               psFreeList->ui32NumHighPages,
+	                               psFreeList->ownerPid);
+
+#endif
+
+	/* return values */
+	*ppsFreeList = psFreeList;
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+
+FWFreeListCpuMap:
+	/* Remove freelists from list */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+	RGXUnsetFirmwareAddress(psFWFreelistMemDesc);
+
+ErrorSetFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psFWFreelistMemDesc);
+
+FWFreeListAlloc:
+	OSFreeMem(psFreeList);
+
+ErrorRefPMR:
+	(void) PMRUnrefPMR(psFreeListPMR);
+
+ErrorAllocHost:
+	DevmemIntReservationRelease(psFreeListReservation);
+
+ErrorReservationAcquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyFreeList
+*/
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RefCount;
+
+	PVR_ASSERT(psFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	ui32RefCount = psFreeList->ui32RefCount;
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	if (ui32RefCount != 0)
+	{
+		/* Freelist still busy */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Freelist is not in use => start firmware cleanup */
+	eError = RGXFWRequestFreeListCleanUp(psFreeList->psDevInfo,
+										 psFreeList->sFreeListFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psFreeList->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestFreeListCleanUp);
+
+	/* Device becomes unrecoverable if the firmware took too long to
+	 * handle the cleanup request, or if SLC-flushes didn't go through
+	 * (due to some GPU lockup) */
+
+	/* Remove FreeList from linked list before we destroy it... */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* Confirm all HWRTData nodes are freed before releasing freelist */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sNodeHWRTDataHead));
+#endif
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+#if defined(PM_INTERACTIVE_MODE)
+	if (psFreeList->bCheckFreelist)
+	{
+		RGXFWIF_FREELIST  *psFWFreeList;
+		IMG_UINT32        ui32CurrentStackTop;
+		IMG_UINT64        ui64CheckSum;
+
+		/* Get the current stack pointer for this free list */
+		DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+		RGXFwSharedMemCacheOpValue(psFWFreeList->ui32CurrentStackTop, INVALIDATE);
+		ui32CurrentStackTop = psFWFreeList->ui32CurrentStackTop;
+		DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+		if (ui32CurrentStackTop > psFreeList->ui32MaxFLPages)
+		{
+			PVR_LOG(("%s: FW freelist corrupted (%d)",
+					 __func__,
+					 ui32CurrentStackTop));
+		}
+		else if (ui32CurrentStackTop == psFreeList->ui32CurrentFLPages-1)
+		{
+			/* Do consistency tests (as the list is fully populated) */
+			_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+		}
+		else
+		{
+			/* Check for duplicate pages, but don't check the checksum as the list is not fully populated */
+			_CheckFreelist(psFreeList, ui32CurrentStackTop+1, 0, &ui64CheckSum);
+		}
+	}
+#endif
+
+	/* Destroy FW structures */
+	RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
+	DevmemFwUnmapAndFree(psFreeList->psDevInfo, psFreeList->psFWFreelistMemDesc);
+
+	/* Remove grow shrink blocks */
+	while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
+	{
+		eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* Remove initial PB block */
+	eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockInitHead, psFreeList);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* consistency checks */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockInitHead));
+	PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);
+
+	/* Remove reference from the PMR and reservation resources */
+	eError = PMRUnrefPMR(psFreeList->psFreeListPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	DevmemIntReservationRelease(psFreeList->psFreeListReservation);
+
+	/* free Freelist */
+	OSFreeMem(psFreeList);
+
+	return eError;
+}
+
+
+/*
+	RGXCreateZSBuffer
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_PRBUFFER			*psFWZSBuffer;
+	RGXFWIF_PRBUFFER			sFWZSBuffer = {0};
+	RGX_ZSBUFFER_DATA			*psZSBuffer;
+	DEVMEM_MEMDESC				*psFWZSBufferMemDesc;
+	IMG_BOOL					bOnDemand = PVRSRV_CHECK_ON_DEMAND(uiMapFlags) ? IMG_TRUE : IMG_FALSE;
+
+	/* Allocate host data structure */
+	psZSBuffer = OSAllocZMem(sizeof(*psZSBuffer));
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate cleanup data structure for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocCleanup;
+	}
+
+	/* Populate Host data */
+	psZSBuffer->psDevInfo = psDevInfo;
+	psZSBuffer->psReservation = psReservation;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psZSBuffer->psReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to acquire reservation for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	psZSBuffer->psPMR = psPMR;
+	/* Obtain reference to PMR */
+	eError = PMRRefPMR(psZSBuffer->psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorRefPMR);
+
+	psZSBuffer->ui32RefCount = 0;
+	psZSBuffer->bIsBacked = IMG_FALSE;
+	psZSBuffer->bOnDemand = bOnDemand;
+
+	/* Allocate firmware memory for ZS-Buffer. */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware ZS-Buffer data structure");
+	eError = DevmemFwAllocate(psDevInfo,
+							sizeof(*psFWZSBuffer),
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+							PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+							PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+							PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+							PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+							"FwZSBuffer",
+							&psFWZSBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAllocFWZSBuffer;
+	}
+	psZSBuffer->psFWZSBufferMemDesc = psFWZSBufferMemDesc;
+
+	/* Temporarily map the firmware render context to the kernel. */
+	eError = DevmemAcquireCpuVirtAddr(psFWZSBufferMemDesc,
+	                                  (void **)&psFWZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAcquireFWZSBuffer;
+	}
+
+	/* Get firmware address of ZS-Buffer. */
+	eError = RGXSetFirmwareAddress(&psZSBuffer->sZSBufferFWDevVAddr, psFWZSBufferMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	if (bOnDemand)
+	{
+		/* psZSBuffer->ui32ZSBufferID set below with lock... */
+		OSLockAcquire(psDevInfo->hLockZSBuffer);
+		psZSBuffer->ui32ZSBufferID = psDevInfo->ui32ZSBufferCurrID++;
+		dllist_add_to_tail(&psDevInfo->sZSBufferHead, &psZSBuffer->sNode);
+		OSLockRelease(psDevInfo->hLockZSBuffer);
+	}
+
+	/* Populate FW ZS-Buffer data structure */
+	sFWZSBuffer.bOnDemand = bOnDemand;
+	sFWZSBuffer.eState = (bOnDemand) ? RGXFWIF_PRBUFFER_UNBACKED : RGXFWIF_PRBUFFER_BACKED;
+	sFWZSBuffer.ui32BufferID = psZSBuffer->ui32ZSBufferID;
+
+	OSCachedMemCopy(psFWZSBuffer, &sFWZSBuffer, sizeof(sFWZSBuffer));
+	/* Dump the ZS-Buffer and the memory content */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump firmware ZS-Buffer");
+	DevmemPDumpLoadMem(psFWZSBufferMemDesc, 0, sizeof(*psFWZSBuffer), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWZSBuffer, FLUSH);
+
+	/* Release address acquired above. */
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+
+
+	/* define return value */
+	*ppsZSBuffer = psZSBuffer;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] created (%s)",
+							psZSBuffer,
+							(bOnDemand) ? "On-Demand": "Up-front"));
+
+	psZSBuffer->owner=OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+
+	/* error handling */
+
+ErrorSetFwAddr:
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+ErrorAcquireFWZSBuffer:
+	DevmemFwUnmapAndFree(psDevInfo, psFWZSBufferMemDesc);
+
+ErrorAllocFWZSBuffer:
+	(void) PMRUnrefPMR(psZSBuffer->psPMR);
+ErrorRefPMR:
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+ErrorReservationAcquire:
+	OSFreeMem(psZSBuffer);
+
+ErrorAllocCleanup:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyZSBuffer
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError, eError2;
+
+	PVR_ASSERT(psZSBuffer);
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	/* Request ZS Buffer cleanup */
+	eError = RGXFWRequestZSBufferCleanUp(psZSBuffer->psDevInfo,
+										psZSBuffer->sZSBufferFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psZSBuffer->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestZSBufferCleanUp);
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->ui32RefCount != 0)
+	{
+		/* ZS-Buffer is still referenced */
+		OSLockRelease(hLockZSBuffer);
+		PVR_DPF((PVR_DBG_WARNING, "Trying to destroy a ZS-Buffer [%p] that's still in use.", psZSBuffer));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Free the firmware render context. */
+	RGXUnsetFirmwareAddress(psZSBuffer->psFWZSBufferMemDesc);
+	DevmemFwUnmapAndFree(psZSBuffer->psDevInfo, psZSBuffer->psFWZSBufferMemDesc);
+
+	/* Remove Deferred Allocation from list */
+	if (psZSBuffer->bOnDemand)
+	{
+		PVR_ASSERT(dllist_node_is_in_list(&psZSBuffer->sNode));
+		dllist_remove_node(&psZSBuffer->sNode);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] destroyed", psZSBuffer));
+
+	/* Release reference to reservation object and the PMR */
+	eError2 = PMRUnrefPMR(psZSBuffer->psPMR);
+	PVR_LOG_IF_ERROR(eError2, "PMRUnrefPMR");
+
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+
+	/* Free ZS-Buffer host data structure */
+	OSFreeMem(psZSBuffer);
+
+	OSLockRelease(hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (!psZSBuffer->bOnDemand)
+	{
+		/* Only deferred allocations can be populated */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bIsBacked == IMG_FALSE)
+	{
+		IMG_HANDLE hDevmemHeap;
+
+		/* Get Heap */
+		eError = DevmemServerGetHeapHandle(psZSBuffer->psReservation, &hDevmemHeap);
+		if (unlikely(hDevmemHeap == (IMG_HANDLE)NULL))
+		{
+			OSLockRelease(hLockZSBuffer);
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+
+		eError = DevmemIntMapPMR(psZSBuffer->psReservation, psZSBuffer->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable populate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+
+		}
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing acquired",
+		                          psZSBuffer,
+		                          psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_TRUE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+					RGX_POPULATION **ppsPopulation)
+{
+	RGX_POPULATION *psPopulation;
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount++;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	psZSBuffer->ui32NumReqByApp++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psZSBuffer->psDevInfo->psDeviceNode,
+	                               1, 0, psZSBuffer->owner);
+#endif
+
+	/* Do the backing */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		goto OnErrorBacking;
+	}
+
+	/* Create the handle to the backing */
+	psPopulation = OSAllocMem(sizeof(*psPopulation));
+	if (psPopulation == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto OnErrorAlloc;
+	}
+
+	psPopulation->psZSBuffer = psZSBuffer;
+
+	/* return value */
+	*ppsPopulation = psPopulation;
+
+	return PVRSRV_OK;
+
+OnErrorAlloc:
+	RGXUnbackingZSBuffer(psZSBuffer);
+
+OnErrorBacking:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_ASSERT(psZSBuffer->ui32RefCount);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing removal requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bOnDemand && psZSBuffer->bIsBacked == IMG_TRUE)
+	{
+		eError = DevmemIntUnmapPMR(psZSBuffer->psReservation);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable to unpopulate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing removed",
+									psZSBuffer,
+									psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_FALSE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation)
+{
+	PVRSRV_ERROR eError;
+	POS_LOCK hLockZSBuffer;
+	if (!psPopulation)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	hLockZSBuffer = psPopulation->psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	eError = RGXUnbackingZSBuffer(psPopulation->psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	OSLockAcquire(hLockZSBuffer);
+	psPopulation->psZSBuffer->ui32RefCount--;
+	OSLockRelease(hLockZSBuffer);
+
+	OSFreeMem(psPopulation);
+
+	return PVRSRV_OK;
+}
+
+static RGX_ZSBUFFER_DATA *FindAndRefZSBuffer(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32ZSBufferID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_ZSBUFFER_DATA *psZSBuffer = NULL;
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+
+	dllist_foreach_node(&psDevInfo->sZSBufferHead, psNode, psNext)
+	{
+		RGX_ZSBUFFER_DATA *psThisZSBuffer = IMG_CONTAINER_OF(psNode, RGX_ZSBUFFER_DATA, sNode);
+
+		if (psThisZSBuffer->ui32ZSBufferID == ui32ZSBufferID)
+		{
+			psZSBuffer = psThisZSBuffer;
+
+			psZSBuffer->ui32RefCount++;
+			break;
+		}
+	}
+
+
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+	return psZSBuffer;
+}
+
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID)
+{
+	IMG_BOOL bBackingDone = IMG_TRUE;
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (Populate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Populate ZLS */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Populating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		bBackingDone = IMG_FALSE;
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = bBackingDone;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psZSBuffer->ui32NumReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psDevInfo->psDeviceNode,
+	                               0, 1, psZSBuffer->owner);
+#endif
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+
+}
+
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32 ui32ZSBufferID)
+{
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (UnPopulate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Unpopulate ZLS */
+	eError = RGXUnbackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"UnPopulating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		PVR_ASSERT(IMG_FALSE);
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = IMG_TRUE;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+}
+
+static
+PVRSRV_ERROR _CreateTAContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_DEV_VIRTADDR sVDMCallStackAddr,
+                              IMG_UINT32 ui32CallStackDepth,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_TA_DATA *psTAData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_TACTX_STATE *psContextState;
+	IMG_UINT32 uiCoreIdx;
+	PVRSRV_ERROR eError;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TA context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_TACTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwTAContextState",
+							  &psTAData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacontextsuspendalloc;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psTAData->psContextStateMemDesc,
+			(void **)&psContextState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_suspendcpuvirtacquire;
+	}
+
+	for (uiCoreIdx = 0; uiCoreIdx < RGX_NUM_GEOM_CORES; uiCoreIdx++)
+	{
+		psContextState->asGeomCore[uiCoreIdx].uTAReg_VDM_CALL_STACK_POINTER_Init =
+			sVDMCallStackAddr.uiAddr + (uiCoreIdx * ui32CallStackDepth * sizeof(IMG_UINT64));
+	}
+	RGXFwSharedMemCacheOpPtr(psContextState->asGeomCore, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psTAData->psContextStateMemDesc);
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TA,
+	                                 RGXFWIF_DM_GEOM,
+	                                 NULL,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 psTAData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TA_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TA_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &psTAData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init TA fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump the TA context suspend state buffer");
+	DevmemPDumpLoadMem(psTAData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TACTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	psTAData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_tacommoncontext:
+fail_suspendcpuvirtacquire:
+	DevmemFwUnmapAndFree(psDevInfo, psTAData->psContextStateMemDesc);
+fail_tacontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Create3DContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_3D_DATA *ps3DData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	uiNumISPStoreRegs;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware 3D context suspend state");
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	uiNumISPStoreRegs = RGXFWIF_IPP_RESUME_REG_COUNT; /* default 1 register for IPP_resume */
+
+	uiNumISPStoreRegs += (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU) *
+								RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_PER_SPU) *
+								RGXFWIF_PIPE_COUNT_PER_ISP);
+
+
+	if (uiNumISPStoreRegs > (RGXFWIF_ISP_PIPE_COUNT_MAX + RGXFWIF_IPP_RESUME_REG_COUNT))
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+#else /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+	/* Size of the CS buffer */
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			(uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]));
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui3DRegISPStateStoreSize,
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "Fw3DContextState",
+							  &ps3DData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcontextsuspendalloc;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_3D,
+	                                 RGXFWIF_DM_3D,
+	                                 NULL,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init 3D fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Dump the 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_3DCTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_3dcommoncontext:
+	DevmemFwUnmapAndFree(psDevInfo, ps3DData->psContextStateMemDesc);
+fail_3dcontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVRGXCreateRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+                                            PVRSRV_DEVICE_NODE			*psDeviceNode,
+                                            IMG_INT32					i32Priority,
+                                            IMG_DEV_VIRTADDR			sVDMCallStackAddr,
+                                            IMG_UINT32					ui32CallStackDepth,
+                                            IMG_UINT32					ui32FrameworkRegisterSize,
+                                            IMG_PBYTE					pabyFrameworkRegisters,
+                                            IMG_HANDLE					hMemCtxPrivData,
+                                            IMG_UINT32					ui32StaticRenderContextStateSize,
+                                            IMG_PBYTE					pStaticRenderContextState,
+                                            IMG_UINT32					ui32PackedCCBSizeU8888,
+                                            IMG_UINT32					ui32ContextFlags,
+                                            IMG_UINT64					ui64RobustnessAddress,
+                                            IMG_UINT32					ui32MaxTADeadlineMS,
+                                            IMG_UINT32					ui32Max3DDeadlineMS,
+                                            RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_RENDER_CONTEXT	*psRenderContext;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	RGXFWIF_FWRENDERCONTEXT		*psFWRenderContext;
+
+	*ppsRenderContext = NULL;
+
+	if (ui32StaticRenderContextStateSize > RGXFWIF_STATIC_RENDERCONTEXT_SIZE)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRenderContext = OSAllocZMem(sizeof(*psRenderContext));
+	if (psRenderContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psRenderContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_lock;
+	}
+
+	psRenderContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Create the FW render context, this has the TA and 3D FW common
+		contexts embedded within it
+	*/
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_FWRENDERCONTEXT),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwRenderContext",
+							  &psRenderContext->psFWRenderContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwrendercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkRegisterSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psRenderContext->psFWFrameworkMemDesc,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psRenderContext->psFWFrameworkMemDesc,
+				pabyFrameworkRegisters,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+		sInfo.psFWFrameworkMemDesc = psRenderContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DContext(psConnection,
+	                          psDeviceNode,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),
+	                          psFWMemContextMemDesc,
+	                          i32Priority,
+	                          ui32Max3DDeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->s3DData,
+	                          U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcontext;
+	}
+
+	eError = _CreateTAContext(psConnection,
+	                          psDeviceNode,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),
+	                          psFWMemContextMemDesc,
+	                          sVDMCallStackAddr,
+	                          ui32CallStackDepth,
+	                          i32Priority,
+	                          ui32MaxTADeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->sTAData,
+	                          U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tacontext;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+			(void **)&psFWRenderContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	/* Copy the static render context data */
+	OSDeviceMemCopy(&psFWRenderContext->sStaticRenderContextState, pStaticRenderContextState, ui32StaticRenderContextStateSize);
+#if defined(SUPPORT_TRP)
+	psFWRenderContext->eTRPGeomCoreAffinity = RGXFWIF_DM_MAX;
+#endif
+	DevmemPDumpLoadMem(psRenderContext->psFWRenderContextMemDesc, 0, sizeof(RGXFWIF_FWRENDERCONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpPtr(psFWRenderContext, FLUSH);
+	DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psRenderContext->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-ta3d");
+	if (IS_ERR(psRenderContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(psRenderContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	}
+
+	*ppsRenderContext = psRenderContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _DestroyTAContext(&psRenderContext->sTAData,
+		                                         psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_tacontext:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _Destroy3DContext(&psRenderContext->s3DData,
+		                                         psRenderContext->psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_3dcontext:
+fail_frameworkcopy:
+	if (psRenderContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+fail_fwrendercontext:
+	OSLockDestroy(psRenderContext->hLock);
+fail_lock:
+	OSFreeMem(psRenderContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+ * PVRSRVRGXDestroyRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psRenderContext->psDeviceNode->pvDevice;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_remove_node(&(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* Check psBufferSyncContext has not been destroyed already (by a previous
+	 * call to this function which then later returned PVRSRV_ERROR_RETRY)
+	 */
+	if (psRenderContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psRenderContext->psBufferSyncContext);
+		psRenderContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+	/* Cleanup the TA if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_TA_COMPLETE) == 0)
+	{
+		eError = _DestroyTAContext(&psRenderContext->sTAData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_TA_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+	/* Cleanup the 3D if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_3D_COMPLETE) == 0)
+	{
+		eError = _Destroy3DContext(&psRenderContext->s3DData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_3D_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+	{
+		RGXFWIF_FWRENDERCONTEXT	*psFWRenderContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+				(void **)&psFWRenderContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware render context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+		RGXFwSharedMemCacheOpValue(psFWRenderContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+
+		ui32WorkEstCCBSubmitted = psFWRenderContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psRenderContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psRenderContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			eError = PVRSRV_ERROR_RETRY;
+			goto e0;
+		}
+	}
+#endif
+
+	/*
+		Only if both TA and 3D contexts have been cleaned up can we
+		free the shared resources
+	*/
+	if (psRenderContext->ui32CleanupStatus == (RC_CLEANUP_3D_COMPLETE | RC_CLEANUP_TA_COMPLETE))
+	{
+		if (psRenderContext->psFWFrameworkMemDesc)
+		{
+			/* Free the framework buffer */
+			DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+		}
+
+		/* Free the firmware render context */
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAUpdate);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DUpdate);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			WorkEstDeInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+		}
+#endif
+
+		OSLockDestroy(psRenderContext->hLock);
+
+		OSFreeMem(psRenderContext);
+	}
+
+	return PVRSRV_OK;
+
+e0:
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	return eError;
+}
+
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+static void DumpUfoList(IMG_UINT32 ui32ClientTAFenceCount,
+                        IMG_UINT32 ui32ClientTAUpdateCount,
+                        IMG_UINT32 ui32Client3DFenceCount,
+                        IMG_UINT32 ui32Client3DUpdateCount,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAFenceUFOAddress,
+                        IMG_UINT32 *paui32ClientTAFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAUpdateUFOAddress,
+                        IMG_UINT32 *paui32ClientTAUpdateValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DFenceUFOAddress,
+                        IMG_UINT32 *paui32Client3DFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DUpdateUFOAddress,
+                        IMG_UINT32 *paui32Client3DUpdateValue)
+{
+	IMG_UINT32 i;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: ~~~ After populating sync prims ~~~",
+			 __func__));
+
+	/* Dump Fence syncs, Update syncs and PR Update syncs */
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA fence syncs:",
+	        __func__, ui32ClientTAFenceCount));
+	for (i = 0; i < ui32ClientTAFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr,
+			        *paui32ClientTAFenceValue,
+			        *paui32ClientTAFenceValue));
+			paui32ClientTAFenceValue++;
+		}
+		pauiClientTAFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA update syncs:",
+			 __func__, ui32ClientTAUpdateCount));
+	for (i = 0; i < ui32ClientTAUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr,
+			        *paui32ClientTAUpdateValue,
+			        *paui32ClientTAUpdateValue));
+			paui32ClientTAUpdateValue++;
+		}
+		pauiClientTAUpdateUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D fence syncs:",
+			 __func__, ui32Client3DFenceCount));
+	for (i = 0; i < ui32Client3DFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr,
+			        *paui32Client3DFenceValue,
+			        *paui32Client3DFenceValue));
+			paui32Client3DFenceValue++;
+		}
+		pauiClient3DFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D update syncs:",
+			 __func__, ui32Client3DUpdateCount));
+	for (i = 0; i < ui32Client3DUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr,
+			        *paui32Client3DUpdateValue,
+			        *paui32Client3DUpdateValue));
+			paui32Client3DUpdateValue++;
+		}
+		pauiClient3DUpdateUFOAddress++;
+	}
+}
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+/*
+ * PVRSRVRGXKickTA3DKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRFenceSyncPrimBlock,
+								 IMG_UINT32					ui32PRFenceSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckTAFence,
+								 PVRSRV_TIMELINE			iUpdateTATimeline,
+								 PVRSRV_FENCE				*piUpdateTAFence,
+								 IMG_CHAR					szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheck3DFence,
+								 PVRSRV_TIMELINE			iUpdate3DTimeline,
+								 PVRSRV_FENCE				*piUpdate3DFence,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus)
+{
+	/* per-context helper structures */
+	RGX_CCB_CMD_HELPER_DATA *pasTACmdHelperData = psRenderContext->asTACmdHelperData;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelperData = psRenderContext->as3DCmdHelperData;
+
+	IMG_UINT32                ui32TACmdCount=0;
+	IMG_UINT32                ui323DCmdCount=0;
+	__maybe_unused IMG_UINT32 ui32TACmdOffset=0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset=0;
+
+	RGXFWIF_UFO        sPRUFO;
+	IMG_UINT32         i;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	PVRSRV_ERROR       eError2 = PVRSRV_OK;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psRenderContext->s3DData.psServerCommonContext);
+	IMG_UINT32         ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_BOOL           bCCBStateOpen = IMG_FALSE;
+
+	IMG_UINT32         ui32ClientPRUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR  *pauiClientPRUpdateUFOAddress = NULL;
+	IMG_UINT32         *paui32ClientPRUpdateValue = NULL;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	PRGXFWIF_UFO_ADDR  *pauiClientTAFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClientTAUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  uiPRFenceUFOAddress;
+
+	IMG_UINT64         uiCheckTAFenceUID = 0;
+	IMG_UINT64         uiCheck3DFenceUID = 0;
+	IMG_UINT64         uiUpdateTAFenceUID = 0;
+	IMG_UINT64         uiUpdate3DFenceUID = 0;
+
+	IMG_BOOL bUseCombined3DAnd3DPR = bKickPR && bKick3D && !pui83DPRDMCmd;
+
+	RGXFWIF_KCCB_CMD_KICK_DATA sTACmdKickData;
+	RGXFWIF_KCCB_CMD_KICK_DATA s3DCmdKickData;
+	IMG_BOOL bUseSingleFWCommand = bKickTA && (bKickPR || bKick3D);
+
+	IMG_UINT32 ui32TACmdSizeTmp = 0, ui323DCmdSizeTmp = 0;
+
+	IMG_BOOL bTAFenceOnSyncCheckpointsOnly = IMG_FALSE;
+
+	PVRSRV_FENCE iUpdateTAFence = PVRSRV_NO_FENCE;
+	PVRSRV_FENCE iUpdate3DFence = PVRSRV_NO_FENCE;
+
+	IMG_BOOL b3DFenceOnSyncCheckpointsOnly = IMG_FALSE;
+	IMG_UINT32 ui32TAFenceTimelineUpdateValue = 0;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+
+	/*
+	 * Count of the number of TA and 3D update values (may differ from number of
+	 * TA and 3D updates later, as sync checkpoints do not need to specify a value)
+	 */
+	IMG_UINT32 ui32ClientPRUpdateValueCount = 0;
+	IMG_UINT32 ui32ClientTAUpdateValueCount = ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32Client3DUpdateValueCount = ui32Client3DUpdateCount;
+	PSYNC_CHECKPOINT *apsFenceTASyncCheckpoints = NULL;				/*!< TA fence checkpoints */
+	PSYNC_CHECKPOINT *apsFence3DSyncCheckpoints = NULL;				/*!< 3D fence checkpoints */
+	IMG_UINT32 ui32FenceTASyncCheckpointCount = 0;
+	IMG_UINT32 ui32Fence3DSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psUpdateTASyncCheckpoint = NULL;				/*!< TA update checkpoint (output) */
+	PSYNC_CHECKPOINT psUpdate3DSyncCheckpoint = NULL;				/*!< 3D update checkpoint (output) */
+	PVRSRV_CLIENT_SYNC_PRIM *psTAFenceTimelineUpdateSync = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	void *pvTAUpdateFenceFinaliseData = NULL;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+
+	RGX_SYNC_DATA sTASyncData = {NULL};		/*!< Contains internal update syncs for TA */
+	RGX_SYNC_DATA s3DSyncData = {NULL};		/*!< Contains internal update syncs for 3D */
+
+	IMG_BOOL bTestSLRAdd3DCheck = IMG_FALSE;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTA = {0};
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickData3D = {0};
+	IMG_UINT32 ui32TACommandOffset = 0;
+	IMG_UINT32 ui323DCommandOffset = 0;
+	IMG_UINT32 ui32TACmdHeaderOffset = 0;
+	IMG_UINT32 ui323DCmdHeaderOffset = 0;
+	IMG_UINT32 ui323DFullRenderCommandOffset = 0;
+	IMG_UINT32 ui32TACmdOffsetWrapCheck = 0;
+	IMG_UINT32 ui323DCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT32 ui32TAFenceCount, ui323DFenceCount;
+	IMG_UINT32 ui32TAUpdateCount, ui323DUpdateCount;
+	IMG_UINT32 ui32PRUpdateCount;
+
+	IMG_PID uiCurrentProcess = OSGetCurrentClientProcessIDKM();
+
+	IMG_UINT32 ui32Client3DFenceCount = 0;
+
+	/* Ensure we haven't been given a null ptr to
+	 * TA fence values if we have been told we
+	 * have TA sync prim fences
+	 */
+	if (ui32ClientTAFenceCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAFenceValue != NULL,
+		                        "paui32ClientTAFenceValue NULL but "
+		                        "ui32ClientTAFenceCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * TA update values if we have been told we
+	 * have TA updates
+	 */
+	if (ui32ClientTAUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAUpdateValue != NULL,
+		                        "paui32ClientTAUpdateValue NULL but "
+		                        "ui32ClientTAUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * 3D update values if we have been told we
+	 * have 3D updates
+	 */
+	if (ui32Client3DUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32Client3DUpdateValue != NULL,
+		                        "paui32Client3DUpdateValue NULL but "
+		                        "ui32Client3DUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Write FW addresses into CMD SHARED BLOCKs */
+	{
+		CMDTA3D_SHARED *psGeomCmdShared = (CMDTA3D_SHARED *)pui8TADMCmd;
+		CMDTA3D_SHARED *ps3DCmdShared = (CMDTA3D_SHARED *)pui83DDMCmd;
+		CMDTA3D_SHARED *psPR3DCmdShared = (CMDTA3D_SHARED *)pui83DPRDMCmd;
+		RGXFWIF_DEV_VIRTADDR sNullFWAddr = {0};
+
+		if (psKMHWRTDataSet == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "KMHWRTDataSet is a null-pointer"));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Write FW address for TA CMD
+		*/
+		if (psGeomCmdShared != NULL)
+		{
+			if (ui32TACmdSize < sizeof(*psGeomCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid TACmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psGeomCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for 3D CMD
+		*/
+		if (ps3DCmdShared != NULL)
+		{
+			if (ui323DCmdSize < sizeof(*ps3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			ps3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for PR3D CMD
+		*/
+		if (psPR3DCmdShared != NULL)
+		{
+			if (ui323DPRCmdSize < sizeof(*psPR3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DPRCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psPR3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+	}
+
+	if (unlikely(iUpdateTATimeline >= 0 && !piUpdateTAFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	if (unlikely(iUpdate3DTimeline >= 0 && !piUpdate3DFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d, "
+			   "ui32Client3DFenceCount=%d, ui32Client3DUpdateCount=%d",
+			   __func__,
+			   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+			   ui32Client3DFenceCount, ui32Client3DUpdateCount));
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRenderContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	/* Double-check we have a PR kick if there are client fences */
+	if (unlikely(!bKickPR && ui32Client3DFenceCount != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: 3D fence passed without a PR kick",
+		        __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+	szFenceName3D[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	ui32TAFenceCount = ui32ClientTAFenceCount;
+	ui323DFenceCount = ui32Client3DFenceCount;
+	ui32TAUpdateCount = ui32ClientTAUpdateCount;
+	ui323DUpdateCount = ui32Client3DUpdateCount;
+	ui32PRUpdateCount = ui32ClientPRUpdateCount;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32SyncPMRCount)
+	{
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling"
+		          " pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psRenderContext->psBufferSyncContext,
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%d)",
+				        __func__, eError));
+			}
+			OSLockRelease(psRenderContext->hLock);
+
+			return eError;
+		}
+
+#if !defined(SUPPORT_STRIP_RENDERING)
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+		{
+			ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+#else /* !defined(SUPPORT_STRIP_RENDERING) */
+		ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+
+		PVR_UNREFERENCED_PARAMETER(bTAFenceOnSyncCheckpointsOnly);
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+
+		if (psBufferUpdateSyncCheckpoint != NULL)
+		{
+			if (bKick3D)
+			{
+				ui323DUpdateCount++;
+			}
+			else
+			{
+				ui32PRUpdateCount++;
+			}
+		}
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2
+#error "Invalid value for UPDATE_FENCE_CHECKPOINT_COUNT. Must be either 1 or 2."
+#endif /* !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2 */
+
+	if (iCheckTAFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[TA]"
+		          " (iCheckFence=%d),"
+		          " psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheckTAFence,
+		          (void *) psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheckTAFence,
+		    &ui32FenceTASyncCheckpointCount,
+		    &apsFenceTASyncCheckpoints,
+		    &uiCheckTAFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_ta_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheckTAFence, ui32FenceTASyncCheckpointCount,
+		          (void *) apsFenceTASyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFenceTASyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "TA", apsFenceTASyncCheckpoints,
+			                      ui32FenceTASyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheck3DFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[3D]"
+		          " (iCheckFence=%d), "
+		          "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheck3DFence,
+		          (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheck3DFence,
+		    &ui32Fence3DSyncCheckpointCount,
+		    &apsFence3DSyncCheckpoints,
+		    &uiCheck3DFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_3d_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheck3DFence, ui32Fence3DSyncCheckpointCount,
+		          (void*)apsFence3DSyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFence3DSyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "3D", apsFence3DSyncCheckpoints,
+			                      ui32Fence3DSyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		IMG_UINT32 i;
+
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32FenceTASyncCheckpointCount;
+
+			for (i = 0; i < ui32Fence3DSyncCheckpointCount; i++)
+			{
+				if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) !=
+				    uiCurrentProcess)
+				{
+					ui32TAFenceCount++;
+				}
+			}
+		}
+
+		if (bKick3D)
+		{
+			ui323DFenceCount += ui32Fence3DSyncCheckpointCount;
+		}
+
+		ui32TAUpdateCount += iUpdateTATimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui323DUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui32PRUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE && !bKick3D ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+	}
+
+
+	if (bKickTA)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+		          __func__, ui32TAFenceCount, ui32TAUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+		    ui32TAFenceCount,
+		    ui32TAUpdateCount,
+		    ui32TACmdSize,
+		    &pPreAddr,
+		    (bKick3D ? NULL : &pPostAddr),
+		    (bKick3D ? NULL : &pRMWUFOAddr),
+		    pasTACmdHelperData
+		);
+	}
+
+	if (bKickPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+		    ui323DFenceCount,
+		    0,
+		    sizeof(sPRUFO),
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickPR && !bUseCombined3DAnd3DPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32PRUpdateCount=%d", __func__,
+		          ui32PRUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+		    0,
+		    0,
+		    ui32PRUpdateCount,
+		    /* if the client has not provided a 3DPR command, the regular 3D
+		     * command should be used instead */
+		    pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize,
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKick3D || bAbort)
+	{
+		if (!bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+		}
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+			bKickTA ? 0 : ui323DFenceCount,
+		    ui323DUpdateCount,
+		    ui323DCmdSize,
+			(bKickTA ? NULL : & pPreAddr),
+			&pPostAddr,
+			&pRMWUFOAddr,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickTA)
+	{
+		ui32TACmdSizeTmp = RGXCmdHelperGetCommandSize(1, pasTACmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext),
+		    ui32TACmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	if (ui323DCmdCount > 0)
+	{
+		ui323DCmdSizeTmp = RGXCmdHelperGetCommandSize(ui323DCmdCount, pas3DCmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext),
+		    ui323DCmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	/* need to reset the counter here */
+
+	ui323DCmdCount = 0;
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAFence, %d fences)...",
+			   __func__, ui32ClientTAFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
+										ui32ClientTAFenceCount,
+										apsClientTAFenceSyncPrimBlock,
+										paui32ClientTAFenceSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_fence;
+	}
+
+	if (ui32ClientTAFenceCount)
+	{
+		pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAUpdate, %d updates)...",
+			   __func__, ui32ClientTAUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
+										ui32ClientTAUpdateCount,
+										apsClientTAUpdateSyncPrimBlock,
+										paui32ClientTAUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_update;
+	}
+
+	if (ui32ClientTAUpdateCount)
+	{
+		pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAUpdateUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DFence, %d fences)...",
+			   __func__, ui32Client3DFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
+										ui32Client3DFenceCount,
+										NULL,
+										NULL);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_fence;
+	}
+
+	if (ui32Client3DFenceCount)
+	{
+		pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DUpdate, %d updates)...",
+			   __func__, ui32Client3DUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
+										ui32Client3DUpdateCount,
+										apsClient3DUpdateSyncPrimBlock,
+										paui32Client3DUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_update;
+	}
+
+	if (ui32Client3DUpdateCount || (iUpdate3DTimeline != PVRSRV_NO_TIMELINE && piUpdate3DFence && bKick3D))
+	{
+		pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DUpdateUFOAddress));
+
+	eError = SyncPrimitiveBlockToFWAddr(psPRFenceSyncPrimBlock, ui32PRFenceSyncOffset, &uiPRFenceUFOAddress);
+
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_pr_fence_address;
+	}
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	if (ui32SyncPMRCount)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+#if !defined(SUPPORT_STRIP_RENDERING)
+		/* Append buffer sync fences to TA fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0 && bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to TA Fence "
+					   "(&psRenderContext->sSyncAddrListTAFence=<%p>, "
+					   "pauiClientTAFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrListTAFence ,
+					   (void*)pauiClientTAFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClientTAFenceUFOAddress)
+			{
+				pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+			}
+			if (ui32ClientTAFenceCount == 0)
+			{
+				bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32ClientTAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+		/* Append buffer sync fences to 3D fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to 3D Fence "
+					   "(&psRenderContext->sSyncAddrList3DFence=<%p>, "
+					   "pauiClient3DFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrList3DFence,
+					   (void*)pauiClient3DFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+					ui32BufferFenceSyncCheckpointCount,
+					apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClient3DFenceUFOAddress)
+			{
+				pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+			}
+			if (ui32Client3DFenceCount == 0)
+			{
+				b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32Client3DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			/* If we have a 3D kick append update to the 3D updates else append to the PR update */
+			if (bKick3D)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   Append 1 buffer sync checkpoint<%p> to 3D Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClient3DUpdateUFOAddress=<%p>)...",
+						   __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClient3DUpdateUFOAddress));
+				/* Append buffer sync update to 3D updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClient3DUpdateUFOAddress)
+				{
+					pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32Client3DUpdateCount++;
+			}
+			else
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+				           "%s:   Append 1 buffer sync checkpoint<%p> to PR Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClientPRUpdateUFOAddress=<%p>)...",
+				           __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClientPRUpdateUFOAddress));
+				/* Attach update to the 3D (used for PR) Updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClientPRUpdateUFOAddress)
+				{
+					pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32ClientPRUpdateCount++;
+			}
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   (after buffer_sync) ui32ClientTAFenceCount=%d, "
+				   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+				   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+				   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+				   ui32ClientPRUpdateCount));
+
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Buffer sync not supported but got %u buffers",
+				 __func__, ui32SyncPMRCount));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_no_buffer_sync_invalid_params;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	/*
+	 * The hardware requires a PR to be submitted if there is a TA (otherwise
+	 * it can wedge if we run out of PB space with no PR to run)
+	 *
+	 * If we only have a TA, attach native checks to the TA and updates to the PR
+	 * If we have a TA and 3D, attach checks to TA, updates to 3D
+	 * If we only have a 3D, attach checks and updates to the 3D
+	 *
+	 * Note that 'updates' includes the cleanup syncs for 'check' fence FDs, in
+	 * addition to the update fence FD (if supplied)
+	 *
+	 * Currently, the client driver never kicks only the 3D, so we only support
+	 * that for the time being.
+	 */
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [TA] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheckTAFence, iUpdateTATimeline));
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [3D] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheck3DFence, iUpdate3DTimeline));
+
+		{
+			/* Create the output fence for TA (if required) */
+			if (iUpdateTATimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdateTAFence, iUpdateTATimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceNameTA,
+						iUpdateTATimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdateTAFence,
+						&uiUpdateTAFenceUID,
+						&pvTAUpdateFenceFinaliseData,
+						&psUpdateTASyncCheckpoint,
+						(void*)&psTAFenceTimelineUpdateSync,
+						&ui32TAFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[TA] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_ta_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdateTAFence,
+						   (void*)psTAFenceTimelineUpdateSync,
+						   ui32TAFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClientTAIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClientTAIntUpdateUFOAddress */
+					pauiClientTAIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdateTASyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[TA]->ui32Addr=0x%x",
+							   __func__, pauiClientTAIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the TA timeline (if required) */
+			if (psTAFenceTimelineUpdateSync)
+			{
+				sTASyncData.ui32ClientUpdateCount		 = ui32ClientTAUpdateCount;
+				sTASyncData.ui32ClientUpdateValueCount	 = ui32ClientTAUpdateValueCount;
+				sTASyncData.ui32ClientPRUpdateValueCount = (bKick3D) ? 0 : ui32ClientPRUpdateValueCount;
+				sTASyncData.paui32ClientUpdateValue		 = paui32ClientTAUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui32TAFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrListTAUpdate,
+											(bKick3D) ? NULL : &psRenderContext->sSyncAddrList3DUpdate,
+											psTAFenceTimelineUpdateSync,
+											&sTASyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_TA;
+				}
+
+				paui32ClientTAUpdateValue = sTASyncData.paui32ClientUpdateValue;
+				ui32ClientTAUpdateValueCount = sTASyncData.ui32ClientUpdateValueCount;
+				pauiClientTAUpdateUFOAddress = sTASyncData.pauiClientUpdateUFOAddress;
+				ui32ClientTAUpdateCount = sTASyncData.ui32ClientUpdateCount;
+			}
+
+			/* Create the output fence for 3D (if required) */
+			if (iUpdate3DTimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdate3DFence, iUpdate3DTimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceName3D,
+						iUpdate3DTimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdate3DFence,
+						&uiUpdate3DFenceUID,
+						&pv3DUpdateFenceFinaliseData,
+						&psUpdate3DSyncCheckpoint,
+						(void*)&ps3DFenceTimelineUpdateSync,
+						&ui323DFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[3D] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_3d_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdate3DFence,
+						   (void*)ps3DFenceTimelineUpdateSync,
+						   ui323DFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClient3DIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClient3DIntUpdateUFOAddress */
+					pauiClient3DIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdate3DSyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[3D]->ui32Addr=0x%x",
+							   __func__, pauiClient3DIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the 3D timeline (if required) */
+			if (ps3DFenceTimelineUpdateSync)
+			{
+				s3DSyncData.ui32ClientUpdateCount = ui32Client3DUpdateCount;
+				s3DSyncData.ui32ClientUpdateValueCount = ui32Client3DUpdateValueCount;
+				s3DSyncData.ui32ClientPRUpdateValueCount = ui32ClientPRUpdateValueCount;
+				s3DSyncData.paui32ClientUpdateValue = paui32Client3DUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui323DFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrList3DUpdate,
+											&psRenderContext->sSyncAddrList3DUpdate,	/*!< PR update: is this required? */
+											ps3DFenceTimelineUpdateSync,
+											&s3DSyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_3D;
+				}
+
+				paui32Client3DUpdateValue = s3DSyncData.paui32ClientUpdateValue;
+				ui32Client3DUpdateValueCount = s3DSyncData.ui32ClientUpdateValueCount;
+				pauiClient3DUpdateUFOAddress = s3DSyncData.pauiClientUpdateUFOAddress;
+				ui32Client3DUpdateCount = s3DSyncData.ui32ClientUpdateCount;
+
+				if (!bKick3D)
+				{
+					paui32ClientPRUpdateValue = s3DSyncData.paui32ClientPRUpdateValue;
+					ui32ClientPRUpdateValueCount = s3DSyncData.ui32ClientPRUpdateValueCount;
+					pauiClientPRUpdateUFOAddress = s3DSyncData.pauiClientPRUpdateUFOAddress;
+					ui32ClientPRUpdateCount = s3DSyncData.ui32ClientPRUpdateCount;
+				}
+			}
+
+			/*
+			 * The hardware requires a PR to be submitted if there is a TA OOM.
+			 * If we only have a TA, attach native checks and updates to the TA
+			 * and 3D updates to the PR.
+			 * If we have a TA and 3D, attach the native TA checks and updates
+			 * to the TA and similarly for the 3D.
+			 * Note that 'updates' includes the cleanup syncs for 'check' fence
+			 * FDs, in addition to the update fence FD (if supplied).
+			 * Currently, the client driver never kicks only the 3D, so we don't
+			 * support that for the time being.
+			 */
+
+			{
+				if (bKickTA)
+				{
+					/* Attach checks and updates to TA */
+
+					/* Checks (from input fence) */
+					if (ui32FenceTASyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to TA Fence (apsFenceSyncCheckpoints=<%p>)...",
+								   __func__,
+								   ui32FenceTASyncCheckpointCount,
+								   (void*)apsFenceTASyncCheckpoints));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+													  ui32FenceTASyncCheckpointCount,
+													  apsFenceTASyncCheckpoints);
+						if (!pauiClientTAFenceUFOAddress)
+						{
+							pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+								   __func__, ui32ClientTAFenceCount,
+								   ui32ClientTAFenceCount + ui32FenceTASyncCheckpointCount));
+						if (ui32ClientTAFenceCount == 0)
+						{
+							bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32ClientTAFenceCount += ui32FenceTASyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32ClientTAFenceCount now %d}",
+							   __func__, ui32ClientTAFenceCount));
+
+					if (psUpdateTASyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Update...",
+								   __func__, (void*)psUpdateTASyncCheckpoint,
+								   SyncCheckpointGetId(psUpdateTASyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAUpdate,
+													  1,
+													  &psUpdateTASyncCheckpoint);
+						if (!pauiClientTAUpdateUFOAddress)
+						{
+							pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+						}
+						ui32ClientTAUpdateCount++;
+					}
+
+					if (!bKick3D && psUpdate3DSyncCheckpoint)
+					{
+						/* Attach update to the 3D (used for PR) Updates */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D(PR) Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClientPRUpdateUFOAddress)
+						{
+							pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32ClientPRUpdateCount++;
+					}
+				}
+
+				if (bKick3D)
+				{
+					/* Attach checks and updates to the 3D */
+
+					/* Checks (from input fence) */
+					if (ui32Fence3DSyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to 3D Fence...",
+								   __func__, ui32Fence3DSyncCheckpointCount));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+													  ui32Fence3DSyncCheckpointCount,
+													  apsFence3DSyncCheckpoints);
+						if (!pauiClient3DFenceUFOAddress)
+						{
+							pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32Client3DFenceCount was %d, now %d}",
+								   __func__, ui32Client3DFenceCount,
+								   ui32Client3DFenceCount + ui32Fence3DSyncCheckpointCount));
+						if (ui32Client3DFenceCount == 0)
+						{
+							b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32Client3DFenceCount += ui32Fence3DSyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32Client3DFenceCount was %d}",
+							   __func__, ui32Client3DFenceCount));
+
+					if (psUpdate3DSyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClient3DUpdateUFOAddress)
+						{
+							pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32Client3DUpdateCount++;
+					}
+				}
+
+				/*
+				 * Relocate sync check points from the 3D fence that are
+				 * external to the current process, to the TA fence.
+				 * This avoids a sync lockup when dependent renders are
+				 * submitted out-of-order and a PR must be scheduled.
+				 */
+				if (bKickTA)
+				{
+					/* Search for external timeline dependencies */
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: Checking 3D fence for external sync points (%d)...",
+							   __func__, ui32Fence3DSyncCheckpointCount));
+
+					for (i=0; i<ui32Fence3DSyncCheckpointCount; i++)
+					{
+						/* Check to see if the checkpoint is on a TL owned by
+						 * another process.
+						 */
+						if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) != uiCurrentProcess)
+						{
+							/* 3D Sync point represents cross process
+							 * dependency, copy sync point to TA command fence. */
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Fence...",
+									   __func__, (void*)apsFence3DSyncCheckpoints[i],
+									   SyncCheckpointGetId(apsFence3DSyncCheckpoints[i])));
+
+							SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+														  1,
+														  &apsFence3DSyncCheckpoints[i]);
+
+							if (!pauiClientTAFenceUFOAddress)
+							{
+								pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+							}
+
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+									   __func__,
+									   ui32ClientTAFenceCount,
+									   ui32ClientTAFenceCount + 1));
+
+							if (ui32ClientTAFenceCount == 0)
+							{
+								bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+							}
+
+							ui32ClientTAFenceCount++;
+						}
+					}
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   (after pvr_sync) ui32ClientTAFenceCount=%d, "
+						   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+						   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+						   __func__,
+						   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+						   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+						   ui32ClientPRUpdateCount));
+			}
+		}
+
+		if (ui32ClientTAFenceCount)
+		{
+			PVR_ASSERT(pauiClientTAFenceUFOAddress);
+			if (!bTAFenceOnSyncCheckpointsOnly)
+			{
+				PVR_ASSERT(paui32ClientTAFenceValue);
+			}
+		}
+		if (ui32ClientTAUpdateCount)
+		{
+			PVR_ASSERT(pauiClientTAUpdateUFOAddress);
+			if (ui32ClientTAUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientTAUpdateValue);
+			}
+		}
+		if (ui32Client3DFenceCount)
+		{
+			PVR_ASSERT(pauiClient3DFenceUFOAddress);
+			PVR_ASSERT(b3DFenceOnSyncCheckpointsOnly);
+		}
+		if (ui32Client3DUpdateCount)
+		{
+			PVR_ASSERT(pauiClient3DUpdateUFOAddress);
+			if (ui32Client3DUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32Client3DUpdateValue);
+			}
+		}
+		if (ui32ClientPRUpdateCount)
+		{
+			PVR_ASSERT(pauiClientPRUpdateUFOAddress);
+			if (ui32ClientPRUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientPRUpdateValue);
+			}
+		}
+
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, pauiClientTAFenceUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAFenceCount,
+			   (void*)paui32ClientTAFenceValue));
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAUpdateCount=%d, pauiClientTAUpdateUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAUpdateCount,
+			   (void*)pauiClientTAUpdateUFOAddress));
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	/* Command size check */
+	if (ui32TAFenceCount != ui32ClientTAFenceCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAFenceCount, ui32ClientTAFenceCount));
+	}
+	if (ui32TAUpdateCount != ui32ClientTAUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAUpdateCount, ui32ClientTAUpdateCount));
+	}
+	if (!bTestSLRAdd3DCheck && (ui323DFenceCount != ui32Client3DFenceCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui323DFenceCount, ui32Client3DFenceCount));
+	}
+	if (ui323DUpdateCount != ui32Client3DUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui323DUpdateCount, ui32Client3DUpdateCount));
+	}
+	if (ui32PRUpdateCount != ui32ClientPRUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PR pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32PRUpdateCount, ui32ClientPRUpdateCount));
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode)) && (bKickTA || bKick3D || bAbort))
+	{
+		sWorkloadCharacteristics.sTA3D.ui32RenderTargetSize  = ui32RenderTargetSize;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfDrawCalls = ui32NumberOfDrawCalls;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfIndices   = ui32NumberOfIndices;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfMRTs      = ui32NumberOfMRTs;
+	}
+#endif
+
+	/* Init and acquire to TA command if required */
+	if (bKickTA)
+	{
+		RGX_SERVER_RC_TA_DATA *psTAData = &psRenderContext->sTAData;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sDataTA,
+					RGXFWIF_CCB_CMD_TYPE_GEOM,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTA);
+		}
+#endif
+
+		/* Init the TA command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(psTAData->psServerCommonContext),
+		                                 ui32ClientTAFenceCount,
+		                                 pauiClientTAFenceUFOAddress,
+		                                 paui32ClientTAFenceValue,
+		                                 ui32ClientTAUpdateCount,
+		                                 pauiClientTAUpdateUFOAddress,
+		                                 paui32ClientTAUpdateValue,
+		                                 ui32TACmdSize,
+		                                 pui8TADMCmd,
+								         &pPreAddr,
+								         (bKick3D ? NULL : & pPostAddr),
+								         (bKick3D ? NULL : & pRMWUFOAddr),
+		                                 RGXFWIF_CCB_CMD_TYPE_GEOM,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickDataTA,
+#else
+		                                 NULL,
+#endif
+		                                 "TA",
+		                                 bCCBStateOpen,
+		                                 pasTACmdHelperData);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following is used to determine the offset of the command header containing
+			   the workload estimation data so that can be accessed when the KCCB is read */
+			ui32TACmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(pasTACmdHelperData);
+		}
+#endif
+
+		eError = RGXCmdHelperAcquireCmdCCB(CCB_CMD_HELPER_NUM_TA_COMMANDS, pasTACmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taacquirecmd;
+		}
+		else
+		{
+			ui32TACmdCount++;
+		}
+	}
+
+	/* Only kick the 3D if required */
+	if (bKickPR)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+
+		/*
+			The command helper doesn't know about the PR fence so create
+			the command with all the fences against it and later create
+			the PR command itself which _must_ come after the PR fence.
+		*/
+		sPRUFO.puiAddrUFO = uiPRFenceUFOAddress;
+		sPRUFO.ui32Value = ui32PRFenceValue;
+
+		/* Init the PR fence command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32Client3DFenceCount=%d",
+				   __func__, ui32Client3DFenceCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+		                                 pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 0,
+		                                 NULL,
+		                                 NULL,
+		                                 sizeof(sPRUFO),
+		                                 (IMG_UINT8*) &sPRUFO,
+		                                 NULL,
+		                                 NULL,
+		                                 NULL,
+		                                 RGXFWIF_CCB_CMD_TYPE_FENCE_PR,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+		                                 NULL,
+		                                 "3D-PR-Fence",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+		/* Init the 3D PR command helper */
+		/*
+			Updates for Android (fence sync and Timeline sync prim) are provided in the PR-update
+			if no 3D is present. This is so the timeline update cannot happen out of order with any
+			other 3D already in flight for the same timeline (PR-updates are done in the 3D cCCB).
+			This out of order timeline sync prim update could happen if we attach it to the TA update.
+		*/
+		if (ui32ClientPRUpdateCount)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s: Line %d, ui32ClientPRUpdateCount=%d, "
+					   "pauiClientPRUpdateUFOAddress=0x%x, "
+					   "ui32ClientPRUpdateValueCount=%d, "
+					   "paui32ClientPRUpdateValue=0x%x",
+					   __func__, __LINE__, ui32ClientPRUpdateCount,
+					   pauiClientPRUpdateUFOAddress->ui32Addr,
+					   ui32ClientPRUpdateValueCount,
+					   (ui32ClientPRUpdateValueCount == 0) ? PVRSRV_SYNC_CHECKPOINT_SIGNALLED : *paui32ClientPRUpdateValue));
+		}
+
+		if (!bUseCombined3DAnd3DPR)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientPRUpdateCount=%d",
+					   __func__, ui32ClientPRUpdateCount));
+			RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+			                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+			                                 0,
+			                                 NULL,
+			                                 NULL,
+			                                 ui32ClientPRUpdateCount,
+			                                 pauiClientPRUpdateUFOAddress,
+			                                 paui32ClientPRUpdateValue,
+			                                 pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize, // If the client has not provided a 3DPR command, the regular 3D command should be used instead
+			                                 pui83DPRDMCmd ? pui83DPRDMCmd : pui83DDMCmd,
+			                                 NULL,
+			                                 NULL,
+			                                 NULL,
+			                                 RGXFWIF_CCB_CMD_TYPE_3D_PR,
+			                                 ui32ExtJobRef,
+			                                 ui32IntJobRef,
+			                                 ui32PDumpFlags,
+			                                 NULL,
+			                                 "3D-PR",
+			                                 bCCBStateOpen,
+			                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+		}
+	}
+
+	if (bKick3D || bAbort)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+		const RGXFWIF_CCB_CMD_TYPE e3DCmdType = bAbort ? RGXFWIF_CCB_CMD_TYPE_ABORT : RGXFWIF_CCB_CMD_TYPE_3D;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sData3D,
+					e3DCmdType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickData3D);
+		}
+#endif
+
+		/* Init the 3D command helper */
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 bKickTA ? 0 : ui32Client3DFenceCount,  /* For a kick with a TA, the 3D fences are added before the PR command instead */
+		                                 bKickTA ? NULL : pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 ui32Client3DUpdateCount,
+		                                 pauiClient3DUpdateUFOAddress,
+		                                 paui32Client3DUpdateValue,
+		                                 ui323DCmdSize,
+		                                 pui83DDMCmd,
+		                                 (bKickTA ? NULL : & pPreAddr),
+		                                 &pPostAddr,
+		                                 &pRMWUFOAddr,
+		                                 e3DCmdType,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickData3D,
+#else
+		                                 NULL,
+#endif
+		                                 "3D",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following are used to determine the offset of the command header containing the workload estimation
+			   data so that can be accessed when the KCCB is read */
+			ui323DCmdHeaderOffset =	RGXCmdHelperGetDMCommandHeaderOffset(&pas3DCmdHelperData[ui323DCmdCount - 1]);
+			ui323DFullRenderCommandOffset =	RGXCmdHelperGetCommandOffset(pas3DCmdHelperData, ui323DCmdCount - 1);
+		}
+#endif
+	}
+
+	/* Protect against array overflow in RGXCmdHelperAcquireCmdCCB() */
+	if (unlikely(ui323DCmdCount > CCB_CMD_HELPER_NUM_3D_COMMANDS))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+		goto fail_3dcmdinit;
+	}
+
+	if (ui323DCmdCount)
+	{
+		PVR_ASSERT(bKickPR || bKick3D);
+
+		/* Acquire space for all the 3D command(s) */
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount, pas3DCmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			/* If RGXCmdHelperAcquireCmdCCB fails we skip the scheduling
+			 * of a new TA command with the same Write offset in Kernel CCB.
+			 */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+			goto fail_3dacquirecmd;
+		}
+	}
+
+	if (ui32TACmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &sTACmdKickData.apsCleanupCtl,
+				&sTACmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_GEOM,
+				bKickTA,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taattachcleanupctls;
+		}
+	}
+
+	if (ui323DCmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &s3DCmdKickData.apsCleanupCtl,
+				&s3DCmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_3D,
+				bKick3D,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_3dattachcleanupctls;
+		}
+	}
+
+	/*
+		We should acquire the space in the kernel CCB here as after this point
+		we release the commands which will take operations on server syncs
+		which can't be undone
+	*/
+
+	/*
+		Everything is ready to go now, release the commands
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui32TACmdCount)
+	{
+		ui32TACmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui32TACmdCount,
+								  pasTACmdHelperData,
+								  "TA",
+								  FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui32TACmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+
+			/* This checks if the command would wrap around at the end of the CCB and therefore  would start at an
+			   offset of 0 rather than the current command offset */
+			if (ui32TACmdOffset < ui32TACmdOffsetWrapCheck)
+			{
+				ui32TACommandOffset = ui32TACmdOffset;
+			}
+			else
+			{
+				ui32TACommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  pas3DCmdHelperData,
+								  "3D",
+								  FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui323DCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+
+			if (ui323DCmdOffset < ui323DCmdOffsetWrapCheck)
+			{
+				ui323DCommandOffset = ui323DCmdOffset;
+			}
+			else
+			{
+				ui323DCommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui32TACmdCount)
+	{
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext);
+		CMDTA3D_SHARED *psGeomCmdShared = IMG_OFFSET_ADDR(pui8TADMCmd, 0);
+
+		sTACmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext);
+		sTACmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTACmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Add the Workload data into the KCCB kick */
+			sTACmdKickData.ui32WorkEstCmdHeaderOffset = ui32TACommandOffset + ui32TACmdHeaderOffset;
+		}
+#endif
+
+		if (psGeomCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_TA,
+					sTACmdKickData.psContext,
+					ui32TACmdOffset,
+					psGeomCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_GEOM,
+		                  iCheckTAFence,
+		                  iUpdateTAFence,
+		                  iUpdateTATimeline,
+		                  uiCheckTAFenceUID,
+		                  uiUpdateTAFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickDataTA));
+
+		if (!bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA CCB command. */
+			RGXFWIF_KCCB_CMD sTAKCCBCmd;
+			sTAKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			sTAKCCBCmd.uCmdData.sCmdKickData = sTACmdKickData;
+
+			LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+			{
+				eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+						RGXFWIF_DM_GEOM,
+						&sTAKCCBCmd,
+						ui32PDumpFlags);
+				if (eError2 != PVRSRV_ERROR_RETRY)
+				{
+					break;
+				}
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+		}
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			/* Mark the error and bail out */
+			eError = eError2;
+			goto fail_tasubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_GEOM);
+	}
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd = { 0 };
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext);
+		CMDTA3D_SHARED *ps3DCmdShared = IMG_OFFSET_ADDR(pui83DDMCmd, 0);
+
+		s3DCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+		s3DCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		s3DCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced when the KCCB command reaches the FW */
+			s3DCmdKickData.ui32WorkEstCmdHeaderOffset = ui323DCommandOffset + ui323DCmdHeaderOffset + ui323DFullRenderCommandOffset;
+		}
+#endif
+
+		if (ps3DCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_3D,
+					s3DCmdKickData.psContext,
+					ui323DCmdOffset,
+					ps3DCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_3D,
+		                  iCheck3DFence,
+		                  iUpdate3DFence,
+		                  iUpdate3DTimeline,
+		                  uiCheck3DFenceUID,
+		                  uiUpdate3DFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickData3D));
+
+		if (bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA/3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.sTACmdKickData = sTACmdKickData;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.s3DCmdKickData = s3DCmdKickData;
+		}
+		else
+		{
+			/* Construct the kernel 3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			s3DKCCBCmd.uCmdData.sCmdKickData = s3DCmdKickData;
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+										 RGXFWIF_DM_3D,
+										 &s3DKCCBCmd,
+										 ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			if (eError == PVRSRV_OK)
+			{
+				eError = eError2;
+			}
+			goto fail_3dsubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_3D);
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (unlikely(eError != PVRSRV_OK ))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+				   __func__, eError));
+		goto fail_3dsubmitcmd;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateTASyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [TA] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateTASyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateTASyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateTASyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateTASyncCheckpoint);
+	}
+	if (psTAFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [TA] <%p> to %d",
+				   __func__, (void*)psTAFenceTimelineUpdateSync,
+				   ui32TAFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psTAFenceTimelineUpdateSync, ui32TAFenceTimelineUpdateValue);
+	}
+
+	if (psUpdate3DSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [3D] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdate3DSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdate3DSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdate3DSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdate3DSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [3D] <%p> to %d",
+				   __func__, (void*)ps3DFenceTimelineUpdateSync,
+				   ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling pvr_buffer_sync_kick_succeeded(psBufferSyncData=<%p>)...",
+				   __func__, (void*)psBufferSyncData));
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	if (piUpdateTAFence)
+	{
+		*piUpdateTAFence = iUpdateTAFence;
+	}
+	if (piUpdate3DFence)
+	{
+		*piUpdate3DFence = iUpdate3DFence;
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+
+	if (pvTAUpdateFenceFinaliseData && (iUpdateTAFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdateTAFence,
+									pvTAUpdateFenceFinaliseData,
+									psUpdateTASyncCheckpoint, szFenceNameTA);
+	}
+	if (pv3DUpdateFenceFinaliseData && (iUpdate3DFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdate3DFence,
+									pv3DUpdateFenceFinaliseData,
+									psUpdate3DSyncCheckpoint, szFenceName3D);
+	}
+
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_3dsubmitcmd:
+fail_tasubmitcmd:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_3dattachcleanupctls:
+fail_taattachcleanupctls:
+fail_3dacquirecmd:
+fail_3dcmdinit:
+fail_taacquirecmd:
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DUpdate);
+	/* Where a TA-only kick (ie no 3D) is submitted, the PR update will make use of the unused 3DUpdate list.
+	 * If this has happened, performing a rollback on pauiClientPRUpdateUFOAddress will simply repeat what
+	 * has already been done for the sSyncAddrList3DUpdate above and result in a double decrement of the
+	 * sync checkpoint's hEnqueuedCCBCount, so we need to check before rolling back the PRUpdate.
+	 */
+	if (pauiClientPRUpdateUFOAddress && (pauiClientPRUpdateUFOAddress != psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs))
+	{
+		SyncCheckpointRollbackFromUFO(psRenderContext->psDeviceNode, pauiClientPRUpdateUFOAddress->ui32Addr);
+	}
+
+fail_alloc_update_values_mem_3D:
+	if (iUpdate3DFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdate3DFence, pv3DUpdateFenceFinaliseData);
+	}
+fail_create_3d_fence:
+fail_alloc_update_values_mem_TA:
+	if (iUpdateTAFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateTAFence, pvTAUpdateFenceFinaliseData);
+	}
+fail_create_ta_fence:
+#if !defined(SUPPORT_BUFFER_SYNC)
+err_no_buffer_sync_invalid_params:
+#endif /* !defined(SUPPORT_BUFFER_SYNC) */
+err_pr_fence_address:
+err_populate_sync_addr_list_3d_update:
+err_populate_sync_addr_list_3d_fence:
+err_populate_sync_addr_list_ta_update:
+err_populate_sync_addr_list_ta_fence:
+err_not_enough_space:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		SyncAddrListDeRefCheckpoints(ui32BufferFenceSyncCheckpointCount,
+					     apsBufferFenceSyncCheckpoints);
+	}
+#endif
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+fail_resolve_input_3d_fence:
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+fail_resolve_input_ta_fence:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	PVR_ASSERT(eError != PVRSRV_OK);
+	OSLockRelease(psRenderContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD		sDisableZSStoreCmd = { 0 };
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	sDisableZSStoreCmd.eCmdType = RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableZStore = bDisableDepthStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableSStore = bDisableStencilStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.ui32ExtJobRefToDisableZSStore = ui32ExtJobRefToDisableZSStore;
+
+	if (psRenderContext->ui32CleanupStatus & (RC_CLEANUP_TA_COMPLETE | RC_CLEANUP_3D_COMPLETE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: submit disable depth and stencil store command to render context that has been cleaned up (%u)",
+				__func__, PVRSRV_ERROR_INVALID_CONTEXT));
+		return PVRSRV_ERROR_INVALID_CONTEXT;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_3D,
+									&sDisableZSStoreCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit disable depth and stencil store command (%u)",
+				__func__,
+				eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	if (psRenderContext->sTAData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->sTAData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_GEOM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the TA part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_tacontext;
+		}
+		psRenderContext->sTAData.i32Priority = i32Priority;
+	}
+
+	if (psRenderContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->s3DData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the 3D part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_3dcontext;
+		}
+		psRenderContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+fail_tacontext:
+	OSLockRelease(psRenderContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 RGX_CONTEXT_PROPERTY eContextProperty,
+                                                 IMG_UINT64 ui64Input,
+                                                 IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psRenderContext->hLock);
+			eError = FWCommonContextSetFlags(psRenderContext->sTAData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psRenderContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psRenderContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->sTAData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->s3DData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+		if (NULL != psCurrentServerRenderCtx->sTAData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->sTAData.psServerCommonContext, RGX_KICK_TYPE_DM_TA) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TA;
+			}
+		}
+
+		if (NULL != psCurrentServerRenderCtx->s3DData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_3D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * RGXRenderContextStalledKM
+ */
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	RGXCheckForStalledClientContexts((PVRSRV_RGXDEV_INFO *) psRenderContext->psDeviceNode->pvDevice, IMG_TRUE);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxta3d.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.h
new file mode 100644
index 000000000000..ff8c33d90893
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxta3d.h
@@ -0,0 +1,540 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA and 3D Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX TA and 3D Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXTA3D_H
+#define RGXTA3D_H
+
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_shared.h"
+#include "rgx_fwif_resetframework.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+typedef struct _RGX_SERVER_RENDER_CONTEXT_ RGX_SERVER_RENDER_CONTEXT;
+typedef struct _RGX_FREELIST_ RGX_FREELIST;
+typedef struct _RGX_PMR_NODE_ RGX_PMR_NODE;
+
+/*****************************************************************************
+ * The Design of Data Storage System for Render Targets                      *
+ * ====================================================                      *
+ *   Relevant for                                                            *
+ *     understanding RGXCreateHWRTDataSet & RGXDestroyHWRTDataSet            *
+ *                                                                           *
+ *                                                                           *
+ *        +=========================================+                        *
+ *        |           RenderTargetDataSet           |                        *
+ *        +---------------|---------|---------------+                        *
+ *                        |         |                                        *
+ *                        V         V                                        *
+ *  +- - - - - - - - - - - - +   +- - - - - - - - - - - - +                  *
+ *  | KM_HW_RT_DATA_HANDLE_0 |   | KM_HW_RT_DATA_HANDLE_1 |                  *
+ *  +- - -|- - - - - - - - - +   +- - - - - - - - - | - - +                  *
+ *        |                                         |                        *
+ *        |                                         |           [UM]Client   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |               Bridge   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |           [KM]Server   *
+ *        |                                         |                        *
+ *        | KM-ptr                                  | KM-ptr                 *
+ *        V                                         V                        *
+ *  +====================+           +====================+                  *
+ *  |  KM_HW_RT_DATA_0   |           |   KM_HW_RT_DATA_1  |                  *
+ *  +-----|------------|-+           +-|------------|-----+                  *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            | KM-ptr        | KM-ptr     |                        *
+ *        |            V               V            |                        *
+ *        |      +==========================+       |                        *
+ *        |      | HW_RT_DATA_COMMON_COOKIE |       |                        *
+ *        |      +--------------------------+       |                        *
+ *        |                   |                     |                        *
+ *        |                   |                     |                        *
+ *  ------|-------------------|---------------------|----------------------- *
+ *        |                   |                     |         [FW]Firmware   *
+ *        |                   |                     |                        *
+ *        | FW-addr           |                     | FW-addr                *
+ *        V                   |                     V                        *
+ *  +===============+         |           +===============+                  *
+ *  | HW_RT_DATA_0  |         |           | HW_RT_DATA_1  |                  *
+ *  +------------|--+         |           +--|------------+                  *
+ *               |            |              |                               *
+ *               | FW-addr    | FW-addr      | FW-addr                       *
+ *               V            V              V                               *
+ *        +=========================================+                        *
+ *        |           HW_RT_DATA_COMMON             |                        *
+ *        +-----------------------------------------+                        *
+ *                                                                           *
+ *****************************************************************************/
+
+typedef struct _RGX_HWRTDATA_COMMON_COOKIE_
+{
+	DEVMEM_MEMDESC			*psHWRTDataCommonFwMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sHWRTDataCommonFwAddr;
+	DEVMEMINT_RESERVATION	*psPMMListsReservation;
+	IMG_UINT32				ui32RefCount;
+
+} RGX_HWRTDATA_COMMON_COOKIE;
+
+typedef struct _RGX_KM_HW_RT_DATASET_
+{
+	RGX_HWRTDATA_COMMON_COOKIE *psHWRTDataCommonCookie;
+
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_DEV_VIRTADDR sHWRTDataFwAddr;
+
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc;
+
+	RGX_FREELIST *apsFreeLists[RGXFW_MAX_FREELISTS];
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE			sNodeHWRTData;
+#endif
+
+} RGX_KM_HW_RT_DATASET;
+
+struct _RGX_FREELIST_ {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	CONNECTION_DATA			*psConnection;
+
+	/* Free list PMR */
+	PMR						*psFreeListPMR;
+	IMG_DEVMEM_OFFSET_T		uiFreeListPMROffset;
+
+	DEVMEMINT_RESERVATION* psFreeListReservation;
+
+	/* Freelist config */
+	IMG_UINT32				ui32MaxFLPages;
+	IMG_UINT32				ui32InitFLPages;
+	IMG_UINT32				ui32CurrentFLPages;
+	IMG_UINT32				ui32GrowFLPages;
+	IMG_UINT32				ui32ReadyFLPages;
+	IMG_UINT32				ui32GrowThreshold;		/* Percentage of FL memory used that should trigger a new grow request */
+	IMG_UINT32				ui32FreelistID;
+	IMG_UINT32				ui32FreelistGlobalID;	/* related global freelist for this freelist */
+	IMG_UINT64				ui64FreelistChecksum;	/* checksum over freelist content */
+	IMG_BOOL				bCheckFreelist;			/* freelist check enabled */
+	IMG_UINT32				ui32RefCount;			/* freelist reference counting */
+
+	IMG_UINT32				ui32NumGrowReqByApp;	/* Total number of grow requests by Application */
+	IMG_UINT32				ui32NumGrowReqByFW;		/* Total Number of grow requests by Firmware */
+	IMG_UINT32				ui32NumHighPages;		/* High Mark of pages in the freelist */
+
+	IMG_PID					ownerPid;				/* Pid of the owner of the list */
+
+	/* Memory Blocks */
+	DLLIST_NODE				sMemoryBlockHead;
+	DLLIST_NODE				sMemoryBlockInitHead;
+	DLLIST_NODE				sNode;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* HWRTData nodes linked to local freelist */
+	DLLIST_NODE				sNodeHWRTDataHead;
+#endif
+
+	/* FW data structures */
+	DEVMEM_MEMDESC			*psFWFreelistMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;
+};
+
+struct _RGX_PMR_NODE_ {
+	RGX_FREELIST			*psFreeList;
+	PMR						*psPMR;
+	PMR_PAGELIST			*psPageList;
+	DLLIST_NODE				sMemoryBlock;
+	IMG_UINT32				ui32NumPages;
+	IMG_BOOL				bFirstPageMissing;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RI_HANDLE				hRIHandle;
+#endif
+};
+
+typedef struct {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	DEVMEM_MEMDESC			*psFWZSBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;
+
+	DEVMEMINT_RESERVATION	*psReservation;
+	PMR						*psPMR;
+	IMG_UINT32				ui32ZSBufferID;
+	IMG_UINT32				ui32RefCount;
+	IMG_BOOL				bOnDemand;
+	IMG_BOOL				bIsBacked;
+
+	IMG_UINT32				ui32NumReqByApp;		/* Number of Backing Requests from Application */
+	IMG_UINT32				ui32NumReqByFW;			/* Number of Backing Requests from Firmware */
+
+	IMG_PID					owner;
+
+	DLLIST_NODE	sNode;
+}RGX_ZSBUFFER_DATA;
+
+typedef struct {
+	RGX_ZSBUFFER_DATA		*psZSBuffer;
+} RGX_POPULATION;
+
+/* Dump the physical pages of a freelist */
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList);
+
+
+/* Create set of HWRTData(s) */
+PVRSRV_ERROR RGXCreateHWRTDataSet(CONNECTION_DATA          *psConnection,
+                                  PVRSRV_DEVICE_NODE       *psDeviceNode,
+                                  IMG_DEV_VIRTADDR          asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_DEV_VIRTADDR          psPMMListDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  RGX_FREELIST	           *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                  IMG_UINT32                ui32ScreenPixelMax,
+                                  IMG_UINT64                ui64MultiSampleCtl,
+                                  IMG_UINT64                ui64FlippedMultiSampleCtl,
+                                  IMG_UINT32                ui32TPCStride,
+                                  IMG_DEV_VIRTADDR          asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                ui32TPCSize,
+                                  IMG_UINT32                ui32TEScreen,
+                                  IMG_UINT32                ui32TEAA,
+                                  IMG_UINT32                ui32TEMTILE1,
+                                  IMG_UINT32                ui32TEMTILE2,
+                                  IMG_UINT32                ui32MTileStride,
+                                  IMG_UINT32                ui32ISPMergeLowerX,
+                                  IMG_UINT32                ui32ISPMergeLowerY,
+                                  IMG_UINT32                ui32ISPMergeUpperX,
+                                  IMG_UINT32                ui32ISPMergeUpperY,
+                                  IMG_UINT32                ui32ISPMergeScaleX,
+                                  IMG_UINT32                ui32ISPMergeScaleY,
+                                  IMG_DEV_VIRTADDR          sMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          sRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                uiRgnHeaderSize,
+                                  IMG_UINT32                ui32ISPMtileSize,
+                                  IMG_UINT16                ui16MaxRTs,
+                                  RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+PVRSRV_ERROR RGXCreateHWRTDataSet2(CONNECTION_DATA          *psConnection,
+                                  PVRSRV_DEVICE_NODE        *psDeviceNode,
+                                  IMG_DEV_VIRTADDR          asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  DEVMEMINT_RESERVATION     *psPMMListsReservation,
+                                  RGX_FREELIST	             *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                  IMG_UINT32                ui32ScreenPixelMax,
+                                  IMG_UINT64                ui64MultiSampleCtl,
+                                  IMG_UINT64                ui64FlippedMultiSampleCtl,
+                                  IMG_UINT32                ui32TPCStride,
+                                  IMG_DEV_VIRTADDR          asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                ui32TPCSize,
+                                  IMG_UINT32                ui32TEScreen,
+                                  IMG_UINT32                ui32TEAA,
+                                  IMG_UINT32                ui32TEMTILE1,
+                                  IMG_UINT32                ui32TEMTILE2,
+                                  IMG_UINT32                ui32MTileStride,
+                                  IMG_UINT32                ui32ISPMergeLowerX,
+                                  IMG_UINT32                ui32ISPMergeLowerY,
+                                  IMG_UINT32                ui32ISPMergeUpperX,
+                                  IMG_UINT32                ui32ISPMergeUpperY,
+                                  IMG_UINT32                ui32ISPMergeScaleX,
+                                  IMG_UINT32                ui32ISPMergeScaleY,
+                                  IMG_DEV_VIRTADDR          sMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          sRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                uiRgnHeaderSize,
+                                  IMG_UINT32                ui32ISPMtileSize,
+                                  IMG_UINT16                ui16MaxRTs,
+                                  RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+/* Destroy HWRTDataSet */
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet);
+
+/*
+	RGXCreateZSBufferKM
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer);
+
+/*
+	RGXDestroyZSBufferKM
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+
+/*
+ * RGXBackingZSBuffer()
+ *
+ * Backs ZS-Buffer with physical pages
+ */
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXPopulateZSBufferKM()
+ *
+ * Backs ZS-Buffer with physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+								   RGX_POPULATION **ppsPopulation);
+
+/*
+ * RGXUnbackingZSBuffer()
+ *
+ * Frees ZS-Buffer's physical pages
+ */
+PVRSRV_ERROR RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXUnpopulateZSBufferKM()
+ *
+ * Frees ZS-Buffer's physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation);
+
+/*
+	RGXProcessRequestZSBufferBacking
+*/
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXProcessRequestZSBufferUnbacking
+*/
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXGrowFreeList
+*/
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader);
+
+/* Create free list */
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA      *psConnection,
+                               PVRSRV_DEVICE_NODE	*psDeviceNode,
+                               IMG_HANDLE			hMemCtxPrivData,
+                               IMG_UINT32			ui32MaxFLPages,
+                               IMG_UINT32			ui32InitFLPages,
+                               IMG_UINT32			ui32GrowFLPages,
+                               IMG_UINT32           ui32GrowParamThreshold,
+                               RGX_FREELIST			*psGlobalFreeList,
+                               IMG_BOOL				bCheckFreelist,
+                               DEVMEMINT_RESERVATION* psFreeListReservation,
+                               RGX_FREELIST			**ppsFreeList);
+
+/* Destroy free list */
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList);
+
+/*
+	RGXProcessRequestGrow
+*/
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   IMG_UINT32 ui32FreelistID);
+
+
+/* Reconstruct free list after Hardware Recovery */
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+											  IMG_UINT32 ui32FreelistsCount,
+											  const IMG_UINT32 *paui32Freelists);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateRenderContextKM
+
+ @Description
+	Server-side implementation of RGXCreateRenderContext
+
+ @Input psConnection -
+ @Input psDeviceNode - device node
+ @Input i32Priority - context priority
+ @Input sVDMCallStackAddr - VDM call stack device virtual address
+ @Input ui32CallStackDepth - VDM call stack depth
+ @Input ui32FrameworkCommandSize - framework command size
+ @Input pabyFrameworkCommand - ptr to framework command
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32StaticRenderContextStateSize - size of fixed render state
+ @Input pStaticRenderContextState - ptr to fixed render state buffer
+ @Input ui32PackedCCBSizeU8888 :
+		ui8TACCBAllocSizeLog2 - TA CCB size
+		ui8TACCBMaxAllocSizeLog2 - maximum size to which TA CCB can grow
+		ui83DCCBAllocSizeLog2 - 3D CCB size
+		ui83DCCBMaxAllocSizeLog2 - maximum size to which 3D CCB can grow
+ @Input ui32ContextFlags - flags which specify properties of the context
+ @Output ppsRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+											PVRSRV_DEVICE_NODE			*psDeviceNode,
+											IMG_INT32					i32Priority,
+											IMG_DEV_VIRTADDR			sVDMCallStackAddr,
+											IMG_UINT32					ui32CallStackDepth,
+											IMG_UINT32					ui32FrameworkCommandSize,
+											IMG_PBYTE					pabyFrameworkCommand,
+											IMG_HANDLE					hMemCtxPrivData,
+											IMG_UINT32					ui32StaticRenderContextStateSize,
+											IMG_PBYTE					pStaticRenderContextState,
+											IMG_UINT32					ui32PackedCCBSizeU8888,
+											IMG_UINT32					ui32ContextFlags,
+											IMG_UINT64					ui64RobustnessAddress,
+											IMG_UINT32					ui32MaxTADeadlineMS,
+											IMG_UINT32					ui32Max3DDeadlineMS,
+											RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyRenderContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyRenderContext
+
+ @Input psRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXKickTA3DKM
+
+ @Description
+	Server-side implementation of RGXKickTA3D
+
+ @Input psRTDataCleanup - RT data associated with the kick (or NULL)
+ @Input psZBuffer - Z-buffer associated with the kick (or NULL)
+ @Input psSBuffer - S-buffer associated with the kick (or NULL)
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRSyncPrimBlock,
+								 IMG_UINT32					ui32PRSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckFence,
+								 PVRSRV_TIMELINE			iUpdateTimeline,
+								 PVRSRV_FENCE				*piUpdateFence,
+								 IMG_CHAR					szFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheckFence3D,
+								 PVRSRV_TIMELINE			iUpdateTimeline3D,
+								 PVRSRV_FENCE				*piUpdateFence3D,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus);
+
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore);
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDevNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+												 RGX_CONTEXT_PROPERTY eContextProperty,
+												 IMG_UINT64 ui64Input,
+												 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of render contexts on this device */
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client contexts are stalled */
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+#endif /* RGXTA3D_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.c
new file mode 100644
index 000000000000..63379f57a032
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.c
@@ -0,0 +1,1990 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific transfer queue routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxtransfer.h"
+#include "rgx_tq_shared.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_memallocflags.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "htbserver.h"
+#include "rgxshader.h"
+
+#include "pdump_km.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_bvnc_defs_km.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TQ_UFO_DUMP	0
+
+//#define TRANSFER_CHECKPOINT_DEBUG 1
+
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psFWContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_3D_DATA;
+
+typedef struct {
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_2D_DATA;
+
+struct _RGX_SERVER_TQ_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC              *psFWTransferContextMemDesc;
+	IMG_UINT32					ui32Flags;
+#define RGX_SERVER_TQ_CONTEXT_FLAGS_2D		(1<<0)
+#define RGX_SERVER_TQ_CONTEXT_FLAGS_3D		(1<<1)
+	RGX_SERVER_TQ_3D_DATA		s3DData;
+	RGX_SERVER_TQ_2D_DATA		s2DData;
+	DLLIST_NODE					sListNode;
+	ATOMIC_T			hIntJobRef;
+	IMG_UINT32			ui32PDumpFlags;
+	/* per-prepare sync address lists */
+	SYNC_ADDR_LIST			asSyncAddrListFence[TQ_MAX_PREPARES_PER_SUBMIT];
+	SYNC_ADDR_LIST			asSyncAddrListUpdate[TQ_MAX_PREPARES_PER_SUBMIT];
+	POS_LOCK				hLock;
+};
+
+/*
+	Static functions used by transfer context code
+*/
+static PVRSRV_ERROR _Create3DTransferContext(CONNECTION_DATA *psConnection,
+											 PVRSRV_DEVICE_NODE *psDeviceNode,
+											 DEVMEM_MEMDESC *psAllocatedMemDesc,
+											 IMG_UINT32 ui32AllocatedOffset,
+											 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+											 IMG_INT32 i32Priority,
+											 RGX_COMMON_CONTEXT_INFO *psInfo,
+											 RGX_SERVER_TQ_3D_DATA *ps3DData,
+											 IMG_UINT32 ui32CCBAllocSizeLog2,
+											 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+											 IMG_UINT32 ui32ContextFlags,
+											 IMG_UINT64 ui64RobustnessAddress)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+	IMG_UINT	uiNumISPStoreRegs;
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TQ/3D context suspend state");
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+													RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	ps3DData->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-tq3d");
+	if (IS_ERR(ps3DData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(ps3DData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = DevmemFwAllocate(psDevInfo,
+							ui3DRegISPStateStoreSize,
+							RGX_FWCOMCTX_ALLOCFLAGS,
+							"FwTQ3DContext",
+							&ps3DData->psFWContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextswitchstate;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TQ_3D,
+	                                 RGXFWIF_DM_3D,
+									 NULL,
+									 psAllocatedMemDesc,
+									 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psFWContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TQ3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TQ3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 UINT_MAX, /* max deadline MS */
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+
+	PDUMPCOMMENT(psDeviceNode, "Dump 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psFWContextStateMemDesc, 0, sizeof(RGXFWIF_3DCTX_STATE), PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+	DevmemFwUnmapAndFree(psDevInfo, ps3DData->psFWContextStateMemDesc);
+fail_contextswitchstate:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps3DData->psBufferSyncContext);
+	ps3DData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+static PVRSRV_ERROR _Create2DTransferContext(CONNECTION_DATA *psConnection,
+											 PVRSRV_DEVICE_NODE *psDeviceNode,
+											 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+											 IMG_INT32 i32Priority,
+											 RGX_COMMON_CONTEXT_INFO *psInfo,
+											 RGX_SERVER_TQ_2D_DATA *ps2DData,
+											 IMG_UINT32 ui32CCBAllocSizeLog2,
+											 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+											 IMG_UINT32 ui32ContextFlags,
+											 IMG_UINT64 ui64RobustnessAddress)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	ps2DData->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-tqtla");
+	if (IS_ERR(ps2DData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(ps2DData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TQ_2D,
+	                                 RGXFWIF_DM_2D,
+									 NULL,
+	                                 NULL,
+	                                 0,
+	                                 psFWMemContextMemDesc,
+	                                 NULL,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TQ2D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TQ2D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 UINT_MAX, /* max deadline MS */
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps2DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	ps2DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps2DData->psBufferSyncContext);
+	ps2DData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+static PVRSRV_ERROR _Destroy2DTransferContext(RGX_SERVER_TQ_2D_DATA *ps2DData,
+											  PVRSRV_DEVICE_NODE *psDeviceNode,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps2DData->psServerCommonContext,
+											  RGXFWIF_DM_2D,
+											  ui32PDumpFlags);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps2DData->psServerCommonContext);
+	ps2DData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps2DData->psBufferSyncContext);
+	ps2DData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+#endif /* #if defined(RGX_FEATURE_TLA_BIT_MASK) */
+
+static PVRSRV_ERROR _Destroy3DTransferContext(RGX_SERVER_TQ_3D_DATA *ps3DData,
+											  PVRSRV_DEVICE_NODE *psDeviceNode,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  ui32PDumpFlags);
+	if (PVRSRVIsRetryError(eError))
+	{
+		return eError;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+	}
+
+	/* ... it has so we can free its resources */
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psFWContextStateMemDesc);
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	ps3DData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps3DData->psBufferSyncContext);
+	ps3DData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVCreateTransferContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateTransferContextKM(CONNECTION_DATA		*psConnection,
+										   PVRSRV_DEVICE_NODE		*psDeviceNode,
+										   IMG_INT32				i32Priority,
+										   IMG_UINT32				ui32FrameworkCommandSize,
+										   IMG_PBYTE				pabyFrameworkCommand,
+										   IMG_HANDLE				hMemCtxPrivData,
+										   IMG_UINT32				ui32PackedCCBSizeU8888,
+										   IMG_UINT32				ui32ContextFlags,
+										   IMG_UINT64				ui64RobustnessAddress,
+										   RGX_SERVER_TQ_CONTEXT	**ppsTransferContext)
+{
+	RGX_SERVER_TQ_CONTEXT	*psTransferContext;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC			*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO	sInfo = {NULL};
+	PVRSRV_ERROR			eError = PVRSRV_OK;
+
+	/* Allocate the server side structure */
+	*ppsTransferContext = NULL;
+	psTransferContext = OSAllocZMem(sizeof(*psTransferContext));
+	if (psTransferContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW transfer context, this has the TQ common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWTRANSFERCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwTransferContext",
+			&psTransferContext->psFWTransferContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwtransfercontext;
+	}
+
+	eError = OSLockCreate(&psTransferContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+									__func__,
+									PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	psTransferContext->psDeviceNode = psDeviceNode;
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psTransferContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate firmware GPU framework state (%s)",
+						__func__,
+						PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psTransferContext->psFWFrameworkMemDesc,
+				pabyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to populate the framework buffer (%s)",
+						__func__,
+				PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psTransferContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DTransferContext(psConnection,
+									  psDeviceNode,
+									  psTransferContext->psFWTransferContextMemDesc,
+									  offsetof(RGXFWIF_FWTRANSFERCONTEXT, sTQContext),
+									  psFWMemContextMemDesc,
+									  i32Priority,
+									  &sInfo,
+									  &psTransferContext->s3DData,
+									  U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+									  U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+									  ui32ContextFlags,
+									  ui64RobustnessAddress);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dtransfercontext;
+	}
+	psTransferContext->ui32Flags |= RGX_SERVER_TQ_CONTEXT_FLAGS_3D;
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA))
+	{
+		eError = _Create2DTransferContext(psConnection,
+										  psDeviceNode,
+										  psFWMemContextMemDesc,
+										  i32Priority,
+										  &sInfo,
+										  &psTransferContext->s2DData,
+										  U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+										  U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+										  ui32ContextFlags,
+										  ui64RobustnessAddress);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_2dtransfercontext;
+		}
+		psTransferContext->ui32Flags |= RGX_SERVER_TQ_CONTEXT_FLAGS_2D;
+	}
+#endif
+
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sTransferCtxtListHead), &(psTransferContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+		*ppsTransferContext = psTransferContext;
+	}
+
+	return PVRSRV_OK;
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+fail_2dtransfercontext:
+	_Destroy3DTransferContext(&psTransferContext->s3DData,
+							  psTransferContext->psDeviceNode,
+							  psTransferContext->ui32PDumpFlags);
+#endif
+fail_3dtransfercontext:
+fail_frameworkcopy:
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	OSLockDestroy(psTransferContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+fail_fwtransfercontext:
+	OSFreeMem(psTransferContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*ppsTransferContext = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyTransferContextKM(RGX_SERVER_TQ_CONTEXT *psTransferContext)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psTransferContext->psDeviceNode->pvDevice;
+	IMG_UINT32 i;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+	dllist_remove_node(&(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((psTransferContext->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+			(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = _Destroy2DTransferContext(&psTransferContext->s2DData,
+										   psTransferContext->psDeviceNode,
+										   PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_destroy2d;
+		}
+		/* We've freed the 2D context, don't try to free it again */
+		psTransferContext->ui32Flags &= ~RGX_SERVER_TQ_CONTEXT_FLAGS_2D;
+	}
+#endif
+
+	if (psTransferContext->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D)
+	{
+		eError = _Destroy3DTransferContext(&psTransferContext->s3DData,
+										   psTransferContext->psDeviceNode,
+										   PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_destroy3d;
+		}
+		/* We've freed the 3D context, don't try to free it again */
+		psTransferContext->ui32Flags &= ~RGX_SERVER_TQ_CONTEXT_FLAGS_3D;
+	}
+
+	/* free any resources within the per-prepare UFO address stores */
+	for (i = 0; i < TQ_MAX_PREPARES_PER_SUBMIT; i++)
+	{
+		SyncAddrListDeinit(&psTransferContext->asSyncAddrListFence[i]);
+		SyncAddrListDeinit(&psTransferContext->asSyncAddrListUpdate[i]);
+	}
+
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+
+	OSLockDestroy(psTransferContext->hLock);
+
+	OSFreeMem(psTransferContext);
+
+	return PVRSRV_OK;
+
+fail_destroy3d:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+
+fail_destroy2d:
+#endif
+	OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTransferCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+ * PVRSRVSubmitTQ3DKickKM
+ */
+/* Old bridge call for backwards compatibility. */
+PVRSRV_ERROR PVRSRVRGXSubmitTransferKM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									   IMG_UINT32				ui32PrepareCount,
+									   IMG_UINT32				*paui32ClientUpdateCount,
+									   SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									   IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									   IMG_UINT32				**papaui32ClientUpdateValue,
+									   PVRSRV_FENCE				iCheckFence,
+									   PVRSRV_TIMELINE			i2DUpdateTimeline,
+									   PVRSRV_FENCE				*pi2DUpdateFence,
+									   PVRSRV_TIMELINE			i3DUpdateTimeline,
+									   PVRSRV_FENCE				*pi3DUpdateFence,
+									   IMG_CHAR					szFenceName[32],
+									   IMG_UINT32				*paui32FWCommandSize,
+									   IMG_UINT8				**papaui8FWCommand,
+									   IMG_UINT32				*pui32TQPrepareFlags,
+									   IMG_UINT32				ui32ExtJobRef,
+									   IMG_UINT32				ui32SyncPMRCount,
+									   IMG_UINT32				*paui32SyncPMRFlags,
+									   PMR						**ppsSyncPMRs)
+{
+	return PVRSRVRGXSubmitTransfer3KM(psTransferContext,
+									  ui32PrepareCount,
+									  paui32ClientUpdateCount,
+									  papauiClientUpdateUFODevVarBlock,
+									  papaui32ClientUpdateSyncOffset,
+									  papaui32ClientUpdateValue,
+									  iCheckFence,
+									  i2DUpdateTimeline,
+									  pi2DUpdateFence,
+									  i3DUpdateTimeline,
+									  pi3DUpdateFence,
+									  szFenceName,
+									  PVRSRV_NO_FENCE,
+									  paui32FWCommandSize,
+									  papaui8FWCommand,
+									  pui32TQPrepareFlags,
+									  ui32ExtJobRef,
+									  ui32SyncPMRCount,
+									  paui32SyncPMRFlags,
+									  ppsSyncPMRs);
+}
+
+PVRSRV_ERROR PVRSRVRGXSubmitTransfer3KM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									   IMG_UINT32				ui32PrepareCount,
+									   IMG_UINT32				*paui32ClientUpdateCount,
+									   SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									   IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									   IMG_UINT32				**papaui32ClientUpdateValue,
+									   PVRSRV_FENCE				iCheckFence,
+									   PVRSRV_TIMELINE			i2DUpdateTimeline,
+									   PVRSRV_FENCE				*pi2DUpdateFence,
+									   PVRSRV_TIMELINE			i3DUpdateTimeline,
+									   PVRSRV_FENCE				*pi3DUpdateFence,
+									   IMG_CHAR					szFenceName[32],
+									   PVRSRV_FENCE				iExportFenceToSignal,
+									   IMG_UINT32				*paui32FWCommandSize,
+									   IMG_UINT8				**papaui8FWCommand,
+									   IMG_UINT32				*pui32TQPrepareFlags,
+									   IMG_UINT32				ui32ExtJobRef,
+									   IMG_UINT32				ui32SyncPMRCount,
+									   IMG_UINT32				*paui32SyncPMRFlags,
+									   PMR						**ppsSyncPMRs)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelper;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	RGX_CCB_CMD_HELPER_DATA *pas2DCmdHelper;
+#endif
+	IMG_UINT32                ui323DCmdCount = 0;
+	IMG_UINT32                ui323DCmdLast = 0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset = 0;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	IMG_UINT32                ui322DCmdCount = 0;
+	IMG_UINT32                ui322DCmdLast = 0;
+	__maybe_unused IMG_UINT32 ui322DCmdOffset = 0;
+#endif
+	IMG_UINT32 ui32PDumpFlags = PDUMP_FLAGS_NONE;
+	IMG_UINT32 i;
+	IMG_UINT64 uiCheckFenceUID = 0;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	IMG_UINT64 ui2DUpdateFenceUID = 0;
+#endif
+	IMG_UINT64 ui3DUpdateFenceUID = 0;
+
+	PSYNC_CHECKPOINT ps3DUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT ps3DExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *ppsExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui323DIntAllocatedUpdateValues = NULL;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PSYNC_CHECKPOINT ps2DUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT ps2DExportFenceSyncCheckpoint = NULL;
+	IMG_UINT32 *pui322DIntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps2DFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui322DFenceTimelineUpdateValue = 0;
+	void *pv2DUpdateFenceFinaliseData = NULL;
+#endif
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVRSRV_FENCE i2DUpdateFence = PVRSRV_NO_FENCE;
+#endif
+	PVRSRV_FENCE i3DUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT32   ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32   ui32PreparesDone = 0;
+
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+#if !defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVR_UNREFERENCED_PARAMETER(i2DUpdateTimeline);
+	PVR_UNREFERENCED_PARAMETER(pi2DUpdateFence);
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (i2DUpdateTimeline != PVRSRV_NO_TIMELINE && !pi2DUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+	if (i3DUpdateTimeline != PVRSRV_NO_TIMELINE && !pi3DUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Validate sync prim fence/update value ptrs
+	 * for each prepare.
+	 */
+	{
+		IMG_UINT32 ui32Prepare;
+		IMG_UINT32 *pui32UpdateCount = paui32ClientUpdateCount;
+		IMG_UINT32 **papui32UpdateValue = papaui32ClientUpdateValue;
+
+		/* Check that we have not been given a null ptr for
+		 * update count parameters.
+		 */
+		PVR_LOG_RETURN_IF_FALSE((paui32ClientUpdateCount != NULL),
+		                        "paui32ClientUpdateCount NULL",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+		for (ui32Prepare=0; ui32Prepare<ui32PrepareCount; ui32Prepare++)
+		{
+			/* Ensure we haven't been given a null ptr to
+			 * update values if we have been told we
+			 * have updates for this prepare
+			 */
+			if (*pui32UpdateCount > 0)
+			{
+				PVR_LOG_RETURN_IF_FALSE(*papui32UpdateValue != NULL,
+				                        "paui32ClientUpdateValue NULL but "
+				                        "ui32ClientUpdateCount > 0",
+				                        PVRSRV_ERROR_INVALID_PARAMS);
+			}
+			/* Advance local ptr to update values ptr for next prepare. */
+			papui32UpdateValue++;
+			/* Advance local ptr to update count for next prepare. */
+			pui32UpdateCount++;
+		}
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	if ((ui32PrepareCount == 0) || (ui32PrepareCount > TQ_MAX_PREPARES_PER_SUBMIT))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32SyncPMRCount != 0)
+	{
+		if (!ppsSyncPMRs)
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+#if defined(SUPPORT_BUFFER_SYNC)
+		/* PMR sync is valid only when there is no batching */
+		if ((ui32PrepareCount != 1))
+#endif
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* We can't allocate the required amount of stack space on all consumer architectures */
+	pas3DCmdHelper = OSAllocMem(sizeof(*pas3DCmdHelper) * ui32PrepareCount);
+	if (pas3DCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc3dhelper;
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	pas2DCmdHelper = OSAllocMem(sizeof(*pas2DCmdHelper) * ui32PrepareCount);
+	if (pas2DCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc2dhelper;
+	}
+#endif
+
+	if (iCheckFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psDeviceNode->hSyncCheckpointContext));
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(psDeviceNode->hSyncCheckpointContext,
+											iCheckFence,
+											&ui32FenceSyncCheckpointCount,
+											&apsFenceSyncCheckpoints,
+											&uiCheckFenceUID,
+											ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)", __func__, eError));
+			goto fail_resolve_fencesync_input_fence;
+		}
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			IMG_UINT32 ii;
+			for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+			{
+				PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint));
+			}
+		}
+#endif
+	}
+	/*
+		Ensure we do the right thing for server syncs which cross call boundaries
+	*/
+	for (i=0;i<ui32PrepareCount;i++)
+	{
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 3D))
+		{
+			ui323DCmdLast++;
+		}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		else if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			ui322DCmdLast++;
+		}
+#endif
+	}
+
+	/*
+		Init the command helper commands for all the prepares
+	*/
+	for (i=0;i<ui32PrepareCount;i++)
+	{
+		RGX_CLIENT_CCB *psClientCCB;
+		RGX_SERVER_COMMON_CONTEXT *psServerCommonCtx;
+		IMG_CHAR *pszCommandName;
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper;
+		RGXFWIF_CCB_CMD_TYPE eType;
+		PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+		PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+		SYNC_ADDR_LIST *psSyncAddrListFence = &psTransferContext->asSyncAddrListFence[i];
+		SYNC_ADDR_LIST *psSyncAddrListUpdate = &psTransferContext->asSyncAddrListUpdate[i];
+		IMG_UINT32 ui32IntClientFenceCount = 0U;
+		IMG_UINT32 ui32IntClientUpdateCount = paui32ClientUpdateCount[i];
+		IMG_UINT32 *paui32IntUpdateValue = papaui32ClientUpdateValue[i];
+#if defined(SUPPORT_BUFFER_SYNC)
+		struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+
+		PVRSRV_FENCE *piUpdateFence = NULL;
+		PVRSRV_TIMELINE	iUpdateTimeline = PVRSRV_NO_TIMELINE;
+		void **ppvUpdateFenceFinaliseData = NULL;
+		PSYNC_CHECKPOINT * ppsUpdateSyncCheckpoint = NULL;
+		PVRSRV_CLIENT_SYNC_PRIM **ppsFenceTimelineUpdateSync = NULL;
+		IMG_UINT32 *pui32FenceTimelineUpdateValue = NULL;
+		IMG_UINT32 **ppui32IntAllocatedUpdateValues = NULL;
+		IMG_BOOL bCheckFence = IMG_FALSE;
+		IMG_BOOL bUpdateFence = IMG_FALSE;
+		IMG_BOOL bExportFence = IMG_FALSE;
+		IMG_UINT64 *puiUpdateFenceUID = NULL;
+
+		IMG_BOOL bCCBStateOpen = IMG_FALSE;
+
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 3D))
+		{
+			psServerCommonCtx = psTransferContext->s3DData.psServerCommonContext;
+			psClientCCB = FWCommonContextGetClientCCB(psServerCommonCtx);
+			pszCommandName = "TQ-3D";
+			psCmdHelper = &pas3DCmdHelper[ui323DCmdCount++];
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_3D;
+#if defined(SUPPORT_BUFFER_SYNC)
+			psBufferSyncContext = psTransferContext->s3DData.psBufferSyncContext;
+#endif
+			bCheckFence = ui323DCmdCount == 1;
+			bUpdateFence = ui323DCmdCount == ui323DCmdLast
+				&& i3DUpdateTimeline != PVRSRV_NO_TIMELINE;
+			bExportFence = ui323DCmdCount == ui323DCmdLast
+				&& iExportFenceToSignal != PVRSRV_NO_FENCE;
+
+			if (bUpdateFence)
+			{
+				piUpdateFence = &i3DUpdateFence;
+				iUpdateTimeline = i3DUpdateTimeline;
+				ppvUpdateFenceFinaliseData = &pv3DUpdateFenceFinaliseData;
+				ppsUpdateSyncCheckpoint = &ps3DUpdateSyncCheckpoint;
+				ppsFenceTimelineUpdateSync = &ps3DFenceTimelineUpdateSync;
+				pui32FenceTimelineUpdateValue = &ui323DFenceTimelineUpdateValue;
+				ppui32IntAllocatedUpdateValues = &pui323DIntAllocatedUpdateValues;
+				puiUpdateFenceUID = &ui3DUpdateFenceUID;
+			}
+
+			if (bExportFence)
+			{
+				ppsExportFenceSyncCheckpoint = &ps3DExportFenceSyncCheckpoint;
+			}
+		}
+		else
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			psServerCommonCtx = psTransferContext->s2DData.psServerCommonContext;
+			psClientCCB = FWCommonContextGetClientCCB(psServerCommonCtx);
+			pszCommandName = "TQ-2D";
+			psCmdHelper = &pas2DCmdHelper[ui322DCmdCount++];
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_2D;
+#if defined(SUPPORT_BUFFER_SYNC)
+			psBufferSyncContext = psTransferContext->s2DData.psBufferSyncContext;
+#endif
+			bCheckFence = ui322DCmdCount == 1;
+			bUpdateFence = ui322DCmdCount == ui322DCmdLast
+				&& i2DUpdateTimeline != PVRSRV_NO_TIMELINE;
+			bExportFence = ui322DCmdCount == ui322DCmdLast
+				&& iExportFenceToSignal != PVRSRV_NO_FENCE;
+
+			if (bUpdateFence)
+			{
+				piUpdateFence = &i2DUpdateFence;
+				iUpdateTimeline = i2DUpdateTimeline;
+				ppvUpdateFenceFinaliseData = &pv2DUpdateFenceFinaliseData;
+				ppsUpdateSyncCheckpoint = &ps2DUpdateSyncCheckpoint;
+				ppsFenceTimelineUpdateSync = &ps2DFenceTimelineUpdateSync;
+				pui32FenceTimelineUpdateValue = &ui322DFenceTimelineUpdateValue;
+				ppui32IntAllocatedUpdateValues = &pui322DIntAllocatedUpdateValues;
+				puiUpdateFenceUID = &ui2DUpdateFenceUID;
+			}
+
+			if (bExportFence)
+			{
+				ppsExportFenceSyncCheckpoint = &ps2DExportFenceSyncCheckpoint;
+			}
+		}
+		else
+#endif
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto fail_prepare_loop;
+		}
+
+		if (i == 0)
+		{
+			ui32PDumpFlags = ((pui32TQPrepareFlags[i] & TQ_PREP_FLAGS_PDUMPCONTINUOUS) != 0) ? PDUMP_FLAGS_CONTINUOUS : PDUMP_FLAGS_NONE;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags,
+					"%s Command Server Submit on FWCtx %08x", pszCommandName, FWCommonContextGetFWAddress(psServerCommonCtx).ui32Addr);
+			psTransferContext->ui32PDumpFlags |= ui32PDumpFlags;
+		}
+		else
+		{
+			IMG_UINT32 ui32NewPDumpFlags = ((pui32TQPrepareFlags[i] & TQ_PREP_FLAGS_PDUMPCONTINUOUS) != 0) ? PDUMP_FLAGS_CONTINUOUS : PDUMP_FLAGS_NONE;
+			if (ui32NewPDumpFlags != ui32PDumpFlags)
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				PVR_DPF((PVR_DBG_ERROR, "%s: Mixing of continuous and non-continuous command in a batch is not permitted", __func__));
+				goto fail_prepare_loop;
+			}
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncAddrListPopulate(psTransferContext->sSyncAddrListFence, %d fences)", __func__, ui32IntClientFenceCount));
+		eError = SyncAddrListPopulate(psSyncAddrListFence,
+										0,
+										NULL,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_prepare_loop;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncAddrListPopulate(psTransferContext->asSyncAddrListUpdate[], %d updates)", __func__, ui32IntClientUpdateCount));
+		eError = SyncAddrListPopulate(psSyncAddrListUpdate,
+										ui32IntClientUpdateCount,
+										papauiClientUpdateUFODevVarBlock[i],
+										papaui32ClientUpdateSyncOffset[i]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_prepare_loop;
+		}
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after sync prims) ui32IntClientUpdateCount=%d", __func__, ui32IntClientUpdateCount));
+		if (ui32SyncPMRCount)
+		{
+#if defined(SUPPORT_BUFFER_SYNC)
+			int err;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling pvr_buffer_sync_resolve_and_create_fences", __func__));
+			err = pvr_buffer_sync_resolve_and_create_fences(psBufferSyncContext,
+			                                                psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                                ui32SyncPMRCount,
+			                                                ppsSyncPMRs,
+			                                                paui32SyncPMRFlags,
+			                                                &ui32BufferFenceSyncCheckpointCount,
+			                                                &apsBufferFenceSyncCheckpoints,
+			                                                &psBufferUpdateSyncCheckpoint,
+			                                                &psBufferSyncData);
+			if (err)
+			{
+				switch (err)
+				{
+					case -EINTR:
+						eError = PVRSRV_ERROR_RETRY;
+						break;
+					case -ENOMEM:
+						eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+						break;
+					default:
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						break;
+				}
+
+				if (eError != PVRSRV_ERROR_RETRY)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   pvr_buffer_sync_resolve_and_create_fences failed (%s)", __func__, PVRSRVGetErrorString(eError)));
+				}
+				goto fail_resolve_buffersync_input_fence;
+			}
+
+			/* Append buffer sync fences */
+			if (ui32BufferFenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints to TQ Fence (psSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>)...", __func__, ui32BufferFenceSyncCheckpointCount, (void*)psSyncAddrListFence , (void*)pauiIntFenceUFOAddress));
+				SyncAddrListAppendAndDeRefCheckpoints(psSyncAddrListFence,
+													  ui32BufferFenceSyncCheckpointCount,
+													  apsBufferFenceSyncCheckpoints);
+				if (!pauiIntFenceUFOAddress)
+				{
+					pauiIntFenceUFOAddress = psSyncAddrListFence->pasFWAddrs;
+				}
+				ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+			}
+
+			if (psBufferUpdateSyncCheckpoint)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 buffer sync checkpoint<%p> to TQ Update (&psTransferContext->asSyncAddrListUpdate[i]=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)psBufferUpdateSyncCheckpoint, (void*)psSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+				/* Append the update (from output fence) */
+				SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiIntUpdateUFOAddress)
+				{
+					pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+				}
+				ui32IntClientUpdateCount++;
+			}
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after buffer_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers", __func__, ui32SyncPMRCount));
+			PVR_DPF((PVR_DBG_ERROR, "%s:   <--EXIT(%d)", __func__, PVRSRV_ERROR_INVALID_PARAMS));
+			OSLockRelease(psTransferContext->hLock);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+		}
+
+		/* Create the output fence (if required) */
+		if (bUpdateFence)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (piUpdateFence=%p, iUpdateTimeline=%d,  psTranserContext->psDeviceNode->hSyncCheckpointContext=<%p>)", __func__, piUpdateFence, iUpdateTimeline, (void*)psDeviceNode->hSyncCheckpointContext));
+			eError = SyncCheckpointCreateFence(psDeviceNode,
+			                                   szFenceName,
+			                                   iUpdateTimeline,
+			                                   psDeviceNode->hSyncCheckpointContext,
+			                                   piUpdateFence,
+			                                   puiUpdateFenceUID,
+			                                   ppvUpdateFenceFinaliseData,
+			                                   ppsUpdateSyncCheckpoint,
+			                                   (void*)ppsFenceTimelineUpdateSync,
+			                                   pui32FenceTimelineUpdateValue,
+			                                   ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s:   SyncCheckpointCreateFence failed (%s)",
+						__func__,
+						PVRSRVGetErrorString(eError)));
+				goto fail_prepare_loop;
+			}
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: returned from SyncCheckpointCreateFence (piUpdateFence=%p)", __func__, piUpdateFence));
+
+			/* Resolve the iExportFenceToSignal (if required) */
+			if (bExportFence)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+				eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+														  psDeviceNode->hSyncCheckpointContext,
+														  ppsExportFenceSyncCheckpoint,
+														  ui32PDumpFlags);
+				if (eError != PVRSRV_OK)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), *ppsExportFenceSyncCheckpoint));
+					goto fail_prepare_loop;
+				}
+
+				/* Check that the export fence was not also included as part of the
+				 * check fence (which is an error and would lead to a stalled kick).
+				 */
+				if (ui32FenceSyncCheckpointCount > 0)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s:   Checking export fence is not part of check fence...", __func__));
+					CHKPT_DBG((PVR_DBG_ERROR, "%s:   ui32FenceSyncCheckpointCount=%d",
+							   __func__, ui32FenceSyncCheckpointCount));
+					if (ui32FenceSyncCheckpointCount > 0)
+					{
+						IMG_UINT32 iii;
+
+						for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+						{
+							CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+							if (apsFenceSyncCheckpoints[iii] == *ppsExportFenceSyncCheckpoint)
+							{
+								CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, *ppsExportFenceSyncCheckpoint));
+								eError = PVRSRV_ERROR_INVALID_PARAMS;
+								PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+								goto fail_prepare_loop;
+							}
+						}
+					}
+				}
+			}
+
+			/* Append the sync prim update for the timeline (if required) */
+			if (*ppsFenceTimelineUpdateSync)
+			{
+				IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+				/* Allocate memory to hold the list of update values (including our timeline update) */
+				*ppui32IntAllocatedUpdateValues = OSAllocMem(sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				if (!*ppui32IntAllocatedUpdateValues)
+				{
+					/* Failed to allocate memory */
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto fail_prepare_loop;
+				}
+				OSCachedMemSet(*ppui32IntAllocatedUpdateValues, 0xbb, sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+#if defined(SUPPORT_BUFFER_SYNC)
+				if (psBufferUpdateSyncCheckpoint)
+				{
+					/* Copy the update values into the new memory, then append our timeline update value */
+					OSCachedMemCopy(*ppui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount-1));
+					pui32TimelineUpdateWp = *ppui32IntAllocatedUpdateValues + (ui32IntClientUpdateCount-1);
+				}
+				else
+#endif
+				{
+					/* Copy the update values into the new memory, then append our timeline update value */
+					OSCachedMemCopy(*ppui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(**ppui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+					pui32TimelineUpdateWp = *ppui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+				}
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: Appending the additional update value 0x%x)", __func__, *pui32FenceTimelineUpdateValue));
+				/* Now set the additional update value */
+				*pui32TimelineUpdateWp = *pui32FenceTimelineUpdateValue;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+				if (ui32IntClientUpdateCount > 0)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)*ppui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: *ppui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Now append the timeline sync prim addr to the transfer context update list */
+				SyncAddrListAppendSyncPrim(psSyncAddrListUpdate,
+				                           *ppsFenceTimelineUpdateSync);
+				ui32IntClientUpdateCount++;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+				if (ui32IntClientUpdateCount > 0)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)*ppui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: *ppui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: set paui32IntUpdateValue<%p> to point to *ppui32IntAllocatedUpdateValues<%p>", __func__, (void*)paui32IntUpdateValue, (void*)*ppui32IntAllocatedUpdateValues));
+				paui32IntUpdateValue = *ppui32IntAllocatedUpdateValues;
+			}
+		}
+
+		if (bCheckFence && ui32FenceSyncCheckpointCount)
+		{
+			/* Append the checks (from input fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to TQ Fence (psSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)psSyncAddrListFence));
+			SyncAddrListAppendCheckpoints(psSyncAddrListFence,
+										  ui32FenceSyncCheckpointCount,
+										  apsFenceSyncCheckpoints);
+			if (!pauiIntFenceUFOAddress)
+			{
+				pauiIntFenceUFOAddress = psSyncAddrListFence->pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+			if (ui32IntClientFenceCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+				for (iii=0; iii<ui32IntClientFenceCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: psSyncAddrListFence->pasFWAddrs[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+		if (bUpdateFence && *ppsUpdateSyncCheckpoint)
+		{
+			/* Append the update (from output fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to TQ Update (psSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)&psTransferContext->asSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+			SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+										  1,
+										  ppsUpdateSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		if (bExportFence && *ppsExportFenceSyncCheckpoint)
+		{
+			/* Append the update (from export fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Transfer Update (psSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)psSyncAddrListUpdate , (void*)*ppsExportFenceSyncCheckpoint));
+			SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+										  1,
+										  ppsExportFenceSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_TQ_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping TQ fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TQ fence syncs (&psTransferContext->asSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psTransferContext->asSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_ASSERT(psTmpIntFenceUFOAddress->ui32Addr & 0x1);
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TQ update syncs (&psTransferContext->asSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psTransferContext->asSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+		ui32PreparesDone++;
+
+		/*
+			Create the command helper data for this command
+		*/
+		RGXCmdHelperInitCmdCCB(psDevInfo,
+		                       psClientCCB,
+		                       0,
+		                       ui32IntClientFenceCount,
+		                       pauiIntFenceUFOAddress,
+		                       NULL, /* fence value */
+		                       ui32IntClientUpdateCount,
+		                       pauiIntUpdateUFOAddress,
+		                       paui32IntUpdateValue,
+		                       paui32FWCommandSize[i],
+		                       papaui8FWCommand[i],
+							   &pPreAddr,
+							   &pPostAddr,
+							   &pRMWUFOAddr,
+		                       eType,
+		                       ui32ExtJobRef,
+		                       ui32IntJobRef,
+		                       ui32PDumpFlags,
+		                       NULL,
+		                       pszCommandName,
+		                       bCCBStateOpen,
+		                       psCmdHelper);
+
+	}
+
+	/*
+		Acquire space for all the commands in one go
+	*/
+	if (ui323DCmdCount)
+	{
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount,
+										   &pas3DCmdHelper[0]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_cmdacquire;
+		}
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = RGXCmdHelperAcquireCmdCCB(ui322DCmdCount,
+										   &pas2DCmdHelper[0]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_cmdacquire;
+		}
+	}
+#endif
+
+	/*
+		We should acquire the kernel CCB(s) space here as the schedule could fail
+		and we would have to roll back all the syncs
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  &pas3DCmdHelper[0],
+								  "TQ_3D",
+								  FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext).ui32Addr);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		ui322DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->s2DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui322DCmdCount,
+								  &pas2DCmdHelper[0],
+								  "TQ_2D",
+								  FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext).ui32Addr);
+	}
+#endif
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd;
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *ps3DTQCCB = FWCommonContextGetClientCCB(psTransferContext->s3DData.psServerCommonContext);
+
+		/* Take one of the helper data structs and extract the common cmd struct,
+		 * this is used to obtain the frame num. Each command should share the same
+		 * frame number so we can just get the first.
+		 */
+#if defined(PVRSRV_ENABLE_HTB)
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper = &pas3DCmdHelper[0];
+		CMD_COMMON *psTransferCmdCmn = IMG_OFFSET_ADDR(psCmdHelper->pui8DMCmd, 0);
+#endif
+
+		/* Construct the kernel 3D CCB command. */
+		s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		s3DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(ps3DTQCCB);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(ps3DTQCCB);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+		HTBLOGK(HTB_SF_MAIN_KICK_3D,
+				s3DKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui323DCmdOffset,
+				psTransferCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef
+				);
+#endif
+
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQ3D,
+		                  iCheckFence,
+		                  i3DUpdateFence,
+		                  i3DUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  ui3DUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psDevInfo,
+										RGXFWIF_DM_3D,
+										&s3DKCCBCmd,
+										ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			eError = eError2;
+			goto fail_cmdsubmit;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWCtx, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQ3D);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		RGXFWIF_KCCB_CMD s2DKCCBCmd;
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *ps2DTQCCB = FWCommonContextGetClientCCB(psTransferContext->s2DData.psServerCommonContext);
+
+		/* Take one of the helper data structs and extract the common cmd struct,
+		 * this is used to obtain the frame num. Each command should share the same
+		 * frame number so we can just get the first.
+		 */
+#if defined(PVRSRV_ENABLE_HTB)
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper = &pas2DCmdHelper[0];
+		CMD_COMMON *psTransferCmdCmn = IMG_OFFSET_ADDR(psCmdHelper->pui8DMCmd, 0);
+#endif
+
+		/* Construct the kernel 2D CCB command. */
+		s2DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		s2DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(ps2DTQCCB);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(ps2DTQCCB);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(PVRSRV_ENABLE_HTB)
+		HTBLOGK(HTB_SF_MAIN_KICK_2D,
+				s2DKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui322DCmdOffset,
+				psTransferCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef);
+#endif
+
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQ2D,
+		                  iCheckFence,
+		                  i2DUpdateFence,
+		                  i2DUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  ui2DUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psDevInfo,
+										RGXFWIF_DM_2D,
+										&s2DKCCBCmd,
+										ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			eError = eError2;
+			goto fail_cmdsubmit;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWCtx, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQ2D);
+	}
+#endif
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdsubmit;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (ps2DUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling TLA NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)ps2DUpdateSyncCheckpoint, SyncCheckpointGetId(ps2DUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(ps2DUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(ps2DUpdateSyncCheckpoint);
+	}
+	if (ps2DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating TLA NOHW sync prim<%p> to %d", __func__, (void*)ps2DFenceTimelineUpdateSync, ui322DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps2DFenceTimelineUpdateSync, ui322DFenceTimelineUpdateValue);
+	}
+#endif
+	if (ps3DUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling TQ3D NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)ps3DUpdateSyncCheckpoint, SyncCheckpointGetId(ps3DUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(ps3DUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(ps3DUpdateSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating TQ3D NOHW sync prim<%p> to %d", __func__, (void*)ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+	if (ppsExportFenceSyncCheckpoint && *ppsExportFenceSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)*ppsExportFenceSyncCheckpoint, SyncCheckpointGetId(*ppsExportFenceSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(*ppsExportFenceSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(*ppsExportFenceSyncCheckpoint);
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal=%d)", __func__, iExportFenceToSignal));
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pi2DUpdateFence)
+	{
+		*pi2DUpdateFence = i2DUpdateFence;
+	}
+#endif
+	if (pi3DUpdateFence)
+	{
+		*pi3DUpdateFence = i3DUpdateFence;
+	}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pv2DUpdateFenceFinaliseData && (i2DUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, i2DUpdateFence, pv2DUpdateFenceFinaliseData,
+		                            ps2DUpdateSyncCheckpoint, szFenceName);
+	}
+#endif
+	if (pv3DUpdateFenceFinaliseData && (i3DUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, i3DUpdateFence, pv3DUpdateFenceFinaliseData,
+		                            ps3DUpdateSyncCheckpoint, szFenceName);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	OSFreeMem(pas2DCmdHelper);
+#endif
+	OSFreeMem(pas3DCmdHelper);
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pui322DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui322DIntAllocatedUpdateValues);
+		pui322DIntAllocatedUpdateValues = NULL;
+	}
+#endif
+	if (pui323DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui323DIntAllocatedUpdateValues);
+		pui323DIntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+/*
+	No resources are created in this function so there is nothing to free
+	unless we had to merge syncs.
+	If we fail after the client CCB acquire there is still nothing to do
+	as only the client CCB release will modify the client CCB
+*/
+fail_cmdsubmit:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_cmdacquire:
+fail_prepare_loop:
+
+	for (i=0;i<ui32PreparesDone;i++)
+	{
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListFence[i]);
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListUpdate[i]);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32PreparesDone > 0)
+	{
+		/* Prevent duplicate rollback in case of buffer sync. */
+		psBufferUpdateSyncCheckpoint = NULL;
+	}
+#endif
+
+	/* Free memory allocated to hold the internal list of update values */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pui322DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui322DIntAllocatedUpdateValues);
+		pui322DIntAllocatedUpdateValues = NULL;
+	}
+#endif
+	if (pui323DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui323DIntAllocatedUpdateValues);
+		pui323DIntAllocatedUpdateValues = NULL;
+	}
+	if (ppsExportFenceSyncCheckpoint && *ppsExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (i2DUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(i2DUpdateFence, pv2DUpdateFenceFinaliseData);
+	}
+#endif
+	if (i3DUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(i3DUpdateFence, pv3DUpdateFenceFinaliseData);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferUpdateSyncCheckpoint)
+	{
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListUpdate[0]);
+	}
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+fail_resolve_buffersync_input_fence:
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+fail_resolve_fencesync_input_fence:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	OSFreeMem(pas2DCmdHelper);
+fail_alloc2dhelper:
+#endif
+	OSFreeMem(pas3DCmdHelper);
+fail_alloc3dhelper:
+
+	OSLockRelease(psTransferContext->hLock);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDevNode,
+                                                   RGX_SERVER_TQ_CONTEXT *psTransferContext,
+                                                   IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+#endif
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	OSLockAcquire(psTransferContext->hLock);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((psTransferContext->s2DData.i32Priority != i32Priority) &&
+			(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = ContextSetPriority(psTransferContext->s2DData.psServerCommonContext,
+									psConnection,
+									psTransferContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_2D);
+		if (eError != PVRSRV_OK)
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 2D part of the transfercontext (%s)", __func__, PVRSRVGetErrorString(eError)));
+			}
+			goto fail_2dcontext;
+		}
+		psTransferContext->s2DData.i32Priority = i32Priority;
+	}
+#endif
+
+	if (psTransferContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psTransferContext->s3DData.psServerCommonContext,
+									psConnection,
+									psTransferContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 3D part of the transfercontext (%s)", __func__, PVRSRVGetErrorString(eError)));
+			}
+			goto fail_3dcontext;
+		}
+		psTransferContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+
+fail_2dcontext:
+#endif
+	OSLockRelease(psTransferContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPropertyKM(RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   RGX_CONTEXT_PROPERTY eContextProperty,
+												   IMG_UINT64 ui64Input,
+												   IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psTransferContext->hLock);
+			eError = FWCommonContextSetFlags(psTransferContext->s2DData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psTransferContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psTransferContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hTransferCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTransferCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_CONTEXT, sListNode);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			DumpFWCommonContextInfo(psCurrentServerTransferCtx->s2DData.psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+#endif
+
+		if (psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D)
+		{
+			DumpFWCommonContextInfo(psCurrentServerTransferCtx->s3DData.psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTransferCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hTransferCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTransferCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_CONTEXT, sListNode);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+				(NULL != psCurrentServerTransferCtx->s2DData.psServerCommonContext) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerTransferCtx->s2DData.psServerCommonContext, RGX_KICK_TYPE_DM_TQ2D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TQ2D;
+			}
+		}
+#endif
+
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D) && (NULL != psCurrentServerTransferCtx->s3DData.psServerCommonContext))
+		{
+			if ((CheckStalledClientCommonContext(psCurrentServerTransferCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_TQ3D) == PVRSRV_ERROR_CCCB_STALLED))
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TQ3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTransferCtxListLock);
+	return ui32ContextBitMask;
+}
+
+PVRSRV_ERROR PVRSRVRGXTQGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTQReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************//**
+ End of file (rgxtransfer.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.h
new file mode 100644
index 000000000000..3cdc454d3de0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/rogue/rgxtransfer.h
@@ -0,0 +1,179 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Transfer queue Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Transfer queue Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTRANSFER_H)
+#define RGXTRANSFER_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "connection_server.h"
+
+typedef struct _RGX_SERVER_TQ_CONTEXT_ RGX_SERVER_TQ_CONTEXT;
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateTransferContextKM
+
+ @Description
+	Server-side implementation of RGXCreateTransferContext
+
+ @Input pvDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateTransferContextKM(CONNECTION_DATA		*psConnection,
+										   PVRSRV_DEVICE_NODE		*psDeviceNode,
+										   IMG_INT32				i32Priority,
+										   IMG_UINT32				ui32FrameworkCommandSize,
+										   IMG_PBYTE				pabyFrameworkCommand,
+										   IMG_HANDLE				hMemCtxPrivData,
+										   IMG_UINT32				ui32PackedCCBSizeU8888,
+										   IMG_UINT32				ui32ContextFlags,
+										   IMG_UINT64				ui64RobustnessAddress,
+										   RGX_SERVER_TQ_CONTEXT	**ppsTransferContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyTransferContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyTransferContext
+
+ @Input psTransferContext - Transfer context
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyTransferContextKM(RGX_SERVER_TQ_CONTEXT *psTransferContext);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVSubmitTransferKM
+
+ @Description
+	Schedules one or more 2D or 3D HW commands on the firmware
+
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSubmitTransferKM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									IMG_UINT32				ui32PrepareCount,
+									IMG_UINT32				*paui32ClientUpdateCount,
+									SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									IMG_UINT32				**papaui32ClientUpdateValue,
+									PVRSRV_FENCE			iCheckFence,
+									PVRSRV_TIMELINE			i2DUpdateTimeline,
+									PVRSRV_FENCE			*pi2DUpdateFence,
+									PVRSRV_TIMELINE			i3DUpdateTimeline,
+									PVRSRV_FENCE			*pi3DUpdateFence,
+									IMG_CHAR				szFenceName[32],
+									IMG_UINT32				*paui32FWCommandSize,
+									IMG_UINT8				**papaui8FWCommand,
+									IMG_UINT32				*pui32TQPrepareFlags,
+									IMG_UINT32				ui32ExtJobRef,
+									IMG_UINT32				ui32SyncPMRCount,
+									IMG_UINT32				*paui32SyncPMRFlags,
+									PMR						**ppsSyncPMRs);
+
+PVRSRV_ERROR PVRSRVRGXSubmitTransfer3KM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									IMG_UINT32				ui32PrepareCount,
+									IMG_UINT32				*paui32ClientUpdateCount,
+									SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									IMG_UINT32				**papaui32ClientUpdateValue,
+									PVRSRV_FENCE			iCheckFence,
+									PVRSRV_TIMELINE			i2DUpdateTimeline,
+									PVRSRV_FENCE			*pi2DUpdateFence,
+									PVRSRV_TIMELINE			i3DUpdateTimeline,
+									PVRSRV_FENCE			*pi3DUpdateFence,
+									IMG_CHAR				szFenceName[32],
+									PVRSRV_FENCE			iExportFenceToSignal,
+									IMG_UINT32				*paui32FWCommandSize,
+									IMG_UINT8				**papaui8FWCommand,
+									IMG_UINT32				*pui32TQPrepareFlags,
+									IMG_UINT32				ui32ExtJobRef,
+									IMG_UINT32				ui32SyncPMRCount,
+									IMG_UINT32				*paui32SyncPMRFlags,
+									PMR						**ppsSyncPMRs);
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDevNode,
+												   RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPropertyKM(RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   RGX_CONTEXT_PROPERTY eContextProperty,
+												   IMG_UINT64 ui64Input,
+												   IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of transfer contexts on this device */
+void DumpTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client transfer contexts are stalled */
+IMG_UINT32 CheckForStalledClientTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR PVRSRVRGXTQGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+PVRSRV_ERROR PVRSRVRGXTQReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+#endif /* RGXTRANSFER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdebug.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdebug.c
new file mode 100644
index 000000000000..61e808bb976b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdebug.c
@@ -0,0 +1,1798 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "cache_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+#include "rgxutils.h"
+#include "tlstream.h"
+#include "rgxfwriscv.h"
+#include "pvrsrv.h"
+#include "services_km.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_utils.h"
+#include "rgx_fwif_km.h"
+#include "rgx_fwif_sf.h"
+#include "debug_common.h"
+
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxcompute.h"
+#include "rgxtdmtransfer.h"
+#include "rgxtimecorr.h"
+#include "rgx_options.h"
+#include "rgxinit.h"
+#include "rgxlayer_impl.h"
+#include "devicemem_history_server.h"
+
+#define DD_SUMMARY_INDENT  ""
+
+#define RGX_DEBUG_STR_SIZE			(150U)
+
+
+#define RGX_TEXAS_BIF0_ID				(0)
+#define RGX_TEXAS_BIF1_ID				(1)
+
+/*
+ *  The first 7 or 8 cat bases are memory contexts used for PM
+ *  or firmware. The rest are application contexts. The numbering
+ *  is zero-based.
+ */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define MAX_RESERVED_FW_MMU_CONTEXT		(7)
+#else
+#define MAX_RESERVED_FW_MMU_CONTEXT		(6)
+#endif
+
+static const IMG_CHAR *const pszPowStateName[] =
+{
+#define X(NAME)	#NAME,
+	RGXFWIF_POW_STATES
+#undef X
+};
+
+static const IMG_FLAGS2DESC asHwrState2Description[] =
+{
+	{RGXFWIF_HWR_HARDWARE_OK, " HWR OK;"},
+	{RGXFWIF_HWR_RESET_IN_PROGRESS, " Reset ongoing;"},
+	{RGXFWIF_HWR_GENERAL_LOCKUP, " General lockup;"},
+	{RGXFWIF_HWR_DM_RUNNING_OK, " DM running ok;"},
+	{RGXFWIF_HWR_DM_STALLING, " DM stalling;"},
+	{RGXFWIF_HWR_FW_FAULT, " FW fault;"},
+	{RGXFWIF_HWR_RESTART_REQUESTED, " Restart requested;"},
+};
+
+static const IMG_FLAGS2DESC asDmState2Description[] =
+{
+	{RGXFWIF_DM_STATE_READY_FOR_HWR, " ready for hwr;"},
+	{RGXFWIF_DM_STATE_NEEDS_SKIP, " needs skip;"},
+	{RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP, " needs PR cleanup;"},
+	{RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR, " needs trace clear;"},
+	{RGXFWIF_DM_STATE_GUILTY_LOCKUP, " guilty lockup;"},
+	{RGXFWIF_DM_STATE_INNOCENT_LOCKUP, " innocent lockup;"},
+	{RGXFWIF_DM_STATE_GUILTY_OVERRUNING, " guilty overrunning;"},
+	{RGXFWIF_DM_STATE_INNOCENT_OVERRUNING, " innocent overrunning;"},
+	{RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH, " hard context switching;"},
+	{RGXFWIF_DM_STATE_GPU_ECC_HWR, " GPU ECC hwr;"},
+	{RGXFWIF_DM_STATE_GPU_PARITY_HWR, " GPU PARITY hwr;"},
+	{RGXFWIF_DM_STATE_GPU_LATENT_HWR, " GPU LATENT hwr;"},
+	{RGXFWIF_DM_STATE_ICS_HWR, " ICS hwr;"},
+};
+
+static const IMG_CHAR * const apszFwOsStateName[RGXFW_CONNECTION_FW_STATE_COUNT] =
+{
+	"offline",
+	"ready",
+	"active",
+	"offloading",
+	"cooldown"
+};
+
+#if defined(PVR_ENABLE_PHR)
+static const IMG_FLAGS2DESC asPHRConfig2Description[] =
+{
+	{BIT_ULL(RGXFWIF_PHR_MODE_OFF), "off"},
+	{BIT_ULL(RGXFWIF_PHR_MODE_RD_RESET), "reset RD hardware"},
+};
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeMMULevel
+
+ @Description
+
+ Return the name for the MMU level that faulted.
+
+ @Input ui32MMULevel	 - MMU level
+
+ @Return   IMG_CHAR* to the string describing the MMU level that faulted.
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodeMMULevel(IMG_UINT32 ui32MMULevel)
+{
+	const IMG_CHAR* pszMMULevel = "";
+
+	switch (ui32MMULevel)
+	{
+		case 0x0: pszMMULevel = " (Page Table)"; break;
+		case 0x1: pszMMULevel = " (Page Directory)"; break;
+		case 0x2: pszMMULevel = " (Page Catalog)"; break;
+		case 0x3: pszMMULevel = " (Cat Base Reg)"; break;
+	}
+
+	return pszMMULevel;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeMMUReqTags
+
+ @Description
+
+ Decodes the MMU Tag ID and Sideband data fields from RGX_CR_MMU_FAULT_META_STATUS and
+ RGX_CR_MMU_FAULT_STATUS regs.
+
+ @Input psDevInfo           - RGX device info
+ @Input ui32Requester       - Requester ID value
+ @Input ui32BIFModule       - BIF module
+ @Input bWriteBack          - Write Back flag
+ @Input bFBMFault           - FBM Fault flag
+ @Output ppszRequester      - Decoded string from the Requester ID
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDecodeMMUReqTags(PVRSRV_RGXDEV_INFO  *psDevInfo,
+								 IMG_UINT32          ui32Requester,
+								 IMG_UINT32          ui32BIFModule,
+								 IMG_BOOL            bWriteBack,
+								 IMG_BOOL            bFBMFault,
+								 IMG_CHAR            **ppszRequester)
+{
+	IMG_UINT32 ui32BIFsPerSPU = 2;
+	IMG_CHAR   *pszRequester  = "-";
+
+	PVR_ASSERT(ppszRequester != NULL);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ALBIORIX_TOP_INFRASTRUCTURE))
+	{
+		ui32BIFsPerSPU = 4;
+	}
+
+	if (bFBMFault)
+	{
+		if (bWriteBack)
+		{
+			pszRequester = "FBM (Header/state cache request)";
+		}
+		else
+		{
+			pszRequester = "FBM";
+		}
+	}
+	else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_SPU) &&
+	         ui32BIFModule <  RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU)*ui32BIFsPerSPU)
+	{
+		IMG_BOOL  bTexasBIFA = ((ui32BIFModule % 2) == 0);
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ALBIORIX_TOP_INFRASTRUCTURE))
+		{
+			switch (ui32Requester)
+			{
+				case  0: pszRequester = "MCU L1 (IP0 PDS)"; break;
+				case  1: pszRequester = "MCU L1 (IP0 Global)"; break;
+				case  2: pszRequester = "MCU L1 (IP1 PDS)"; break;
+				case  3: pszRequester = "MCU L1 (IP1 Global)"; break;
+				case  4: pszRequester = "MCU L1 (IP2 PDS)"; break;
+				case  5: pszRequester = "MCU L1 (IP2 Global)"; break;
+				case  6: pszRequester = (bTexasBIFA ? "TCU L1" : "BSC"); break;
+				case  7: pszRequester = (bTexasBIFA ? "PBE0" : "BSC"); break;
+				case  8: pszRequester = "PBE0"; break;
+				case  9: pszRequester = "IPF"; break;
+				case 10: pszRequester = "IPF"; break;
+				case 11: pszRequester = "IPF"; break;
+				case 12: pszRequester = (bTexasBIFA ? "IPF" : "TCU L1"); break;
+				case 13: pszRequester = (bTexasBIFA ? "IPF" : "TPF"); break;
+				case 14: pszRequester = (bTexasBIFA ? "IPF" : "TPF CPF"); break;
+				case 15: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 16: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 17: pszRequester = (bTexasBIFA ? "IPF" : "PDSRW cache"); break;
+				case 18: pszRequester = (bTexasBIFA ? "IPF" : "PDS"); break;
+				case 19: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 20: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 21: pszRequester = (bTexasBIFA ? "IPF" : "USC L2"); break;
+				case 22: pszRequester = (bTexasBIFA ? "IPF" : "URI"); break;
+				case 23: pszRequester = "IPF"; break;
+				case 24: pszRequester = "IPF"; break;
+				case 25: pszRequester = "IPF"; break;
+				case 26: pszRequester = "IPF"; break;
+				case 27: pszRequester = "IPF"; break;
+				case 28: pszRequester = "IPF"; break;
+				case 29: pszRequester = "IPF"; break;
+				case 30: pszRequester = "IPF"; break;
+				case 31: pszRequester = "IPF"; break;
+				case 32: pszRequester = "IPF"; break;
+				case 33: pszRequester = "IPF_CPF"; break;
+				case 34: pszRequester = "ISP0"; break;
+				case 35: pszRequester = "ISP0"; break;
+				case 36: pszRequester = "ISP2"; break;
+				case 37: pszRequester = "ISP2"; break;
+				case 38: pszRequester = "GEOM"; break;
+				case 39: pszRequester = "GEOM"; break;
+				case 40: pszRequester = "GEOM"; break;
+				case 41: pszRequester = "GEOM"; break;
+				case 42: pszRequester = "BSC"; break;
+				case 43: pszRequester = "ASC"; break;
+
+				default:
+					PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified Texas BIF Requester ID: %d", __func__, ui32Requester));
+				break;
+			}
+		}
+		else
+		{
+			switch (ui32Requester)
+			{
+				case  0: pszRequester = "MCU L1 (IP0 PDS)"; break;
+				case  1: pszRequester = "MCU L1 (IP0 Global)"; break;
+				case  2: pszRequester = "MCU L1 (IP0 BSC)"; break;
+				case  3: pszRequester = "MCU L1 (IP0 Constants)"; break;
+				case  4: pszRequester = "MCU L1 (IP1 PDS)"; break;
+				case  5: pszRequester = "MCU L1 (IP1 Global)"; break;
+				case  6: pszRequester = "MCU L1 (IP1 BSC)"; break;
+				case  7: pszRequester = "MCU L1 (IP1 Constants)"; break;
+				case  8: pszRequester = "MCU L1 (IP2 PDS)"; break;
+				case  9: pszRequester = "MCU L1 (IP2 Global)"; break;
+				case 10: pszRequester = "MCU L1 (IP2 BSC)"; break;
+				case 11: pszRequester = "MCU L1 (IP2 Constants)"; break;
+				case 12: pszRequester = "TCU L1"; break;
+				case 13: pszRequester = (bTexasBIFA ? "PBE0" : "TPF"); break;
+				case 14: pszRequester = (bTexasBIFA ? "PBE0" : "TPF CPF"); break;
+				case 15: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 16: pszRequester = (bTexasBIFA ? "IPF" : "PBE1"); break;
+				case 17: pszRequester = (bTexasBIFA ? "IPF" : "PDSRW cache"); break;
+				case 18: pszRequester = (bTexasBIFA ? "IPF" : "PDS"); break;
+				case 19: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 20: pszRequester = (bTexasBIFA ? "IPF" : "ISP1"); break;
+				case 21: pszRequester = (bTexasBIFA ? "IPF" : "USC L2"); break;
+				case 22: pszRequester = (bTexasBIFA ? "IPF" : "VDM L2"); break;
+				case 23: pszRequester = (bTexasBIFA ? "IPF" : "RTU FBA L2"); break;
+				case 24: pszRequester = (bTexasBIFA ? "IPF" : "RTU SHR L2"); break;
+				case 25: pszRequester = (bTexasBIFA ? "IPF" : "RTU SHG L2"); break;
+				case 26: pszRequester = (bTexasBIFA ? "IPF" : "RTU TUL L2"); break;
+				case 27: pszRequester = "IPF"; break;
+				case 28: pszRequester = "IPF"; break;
+				case 29: pszRequester = "IPF"; break;
+				case 30: pszRequester = "IPF"; break;
+				case 31: pszRequester = "IPF"; break;
+				case 32: pszRequester = "IPF"; break;
+				case 33: pszRequester = "IPF"; break;
+				case 34: pszRequester = "IPF_CPF"; break;
+				case 35: pszRequester = "PPP"; break;
+				case 36: pszRequester = "ISP0"; break;
+				case 37: pszRequester = "ISP0"; break;
+				case 38: pszRequester = "ISP2"; break;
+				case 39: pszRequester = "ISP2"; break;
+				case 40: pszRequester = "VCE RTC"; break;
+				case 41: pszRequester = "RTU RAC"; break;
+				case 42: pszRequester = "RTU RAC"; break;
+				case 43: pszRequester = "RTU RAC"; break;
+				case 44: pszRequester = "RTU RAC"; break;
+				case 45: pszRequester = "RTU RAC"; break;
+				case 46: pszRequester = "RTU RAC"; break;
+				case 47: pszRequester = "RTU RAC"; break;
+				case 48: pszRequester = "RTU RAC"; break;
+				case 49: pszRequester = "VCE AMC"; break;
+				case 50: pszRequester = "SHF"; break;
+				case 51: pszRequester = "SHF"; break;
+
+				default:
+					PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified Texas BIF Requester ID: %d", __func__, ui32Requester));
+				break;
+			}
+		}
+	}
+	else if (ui32BIFModule == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU)*ui32BIFsPerSPU)
+	{
+		/* Jones BIF */
+		switch (ui32Requester)
+		{
+			case  0: pszRequester = "IPP"; break;
+			case  1: pszRequester = "DCE"; break;
+			case  2: pszRequester = "DCE"; break;
+			case  3: pszRequester = "DCE"; break;
+			case  4: pszRequester = "DCE"; break;
+			case  5: pszRequester = "DCE"; break;
+			case  6: pszRequester = "DCE"; break;
+			case  7: pszRequester = "DCE"; break;
+			case  8: pszRequester = "DCE"; break;
+			case  9: pszRequester = "DCE"; break;
+			case 10: pszRequester = "DCE"; break;
+			case 11: pszRequester = "DCE"; break;
+			case 12: pszRequester = "DCE"; break;
+			case 13: pszRequester = "DCE"; break;
+			case 14: pszRequester = "DCE"; break;
+			case 15: pszRequester = "TDM"; break;
+			case 16: pszRequester = "TDM"; break;
+			case 17: pszRequester = "TDM"; break;
+			case 18: pszRequester = "TDM"; break;
+			case 19: pszRequester = "TDM"; break;
+			case 20: pszRequester = "PM"; break;
+			case 21: pszRequester = "CDM"; break;
+			case 22: pszRequester = "CDM"; break;
+			case 23: pszRequester = "CDM"; break;
+			case 24: pszRequester = "CDM"; break;
+			case 25: pszRequester = "CDM"; break;
+			case 26: pszRequester = "CDM"; break;
+			case 27: pszRequester = "CDM"; break;
+			case 28: pszRequester = "CDM"; break;
+			case 29: pszRequester = "CDM"; break;
+			case 30: pszRequester = "CDM"; break;
+			case 31: pszRequester = "CDM"; break;
+			case 32: pszRequester = "META"; break;
+			case 33: pszRequester = "META DMA"; break;
+			case 34: pszRequester = "TE"; break;
+			case 35: pszRequester = "TE"; break;
+			case 36: pszRequester = "TE"; break;
+			case 37: pszRequester = "TE"; break;
+			case 38: pszRequester = "TE"; break;
+			case 39: pszRequester = "TE"; break;
+			case 40: pszRequester = "TE"; break;
+			case 41: pszRequester = "TE"; break;
+			case 42: pszRequester = "TE"; break;
+			case 43: pszRequester = "TE"; break;
+			case 44: pszRequester = "TE"; break;
+			case 45: pszRequester = "TE"; break;
+			case 46: pszRequester = "TE"; break;
+			case 47: pszRequester = "TE"; break;
+			case 48: pszRequester = "TE"; break;
+			case 49: pszRequester = "TE"; break;
+			case 50: pszRequester = "RCE"; break;
+			case 51: pszRequester = "RCE"; break;
+			case 52: pszRequester = "RCE"; break;
+
+			default:
+				PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified Jones BIF Tag ID: %d", __func__, ui32Requester));
+			break;
+		}
+	}
+	else if (bWriteBack)
+	{
+		pszRequester = "Unknown (Writeback of dirty cacheline)";
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unidentified BIF Module: %d", __func__, ui32BIFModule));
+	}
+
+	*ppszRequester = pszRequester;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpRGXMMUFaultStatus
+
+ @Description
+
+ Dump MMU Fault status in human readable form.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo            - RGX device info
+ @Input ui64MMUStatus        - MMU Status register value
+ @Input pszMetaOrCore        - string representing call is for META or MMU core
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpRGXMMUFaultStatus(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					const IMG_UINT64 aui64MMUStatus[],
+					const IMG_PCHAR pszMetaOrCore,
+					const IMG_CHAR *pszIndent)
+{
+	if (aui64MMUStatus[0] == 0x0)
+	{
+		PVR_DUMPDEBUG_LOG("%sMMU (%s) - OK", pszIndent, pszMetaOrCore);
+	}
+	else
+	{
+		IMG_UINT32 ui32PC        = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT;
+		IMG_UINT64 ui64Addr      = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT) <<  4; /* align shift */
+		IMG_UINT32 ui32Requester = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_REQ_ID_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_REQ_ID_SHIFT;
+		IMG_UINT32 ui32MMULevel  = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_LEVEL_CLRMSK) >>
+		                           RGX_CR_MMU_FAULT_STATUS1_LEVEL_SHIFT;
+		IMG_BOOL bRead           = (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS1_RNW_EN) != 0;
+		IMG_BOOL bFault          = (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS1_FAULT_EN) != 0;
+		IMG_BOOL bROFault        = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK) >>
+		                            RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT) == 0x2;
+		IMG_BOOL bProtFault      = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK) >>
+		                            RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT) == 0x3;
+		IMG_UINT32 ui32BIFModule;
+		IMG_BOOL bWriteBack, bFBMFault;
+		IMG_CHAR *pszRequester = NULL;
+		const IMG_PCHAR pszMetaOrRiscv = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? "META" : "RISCV";
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ALBIORIX_TOP_INFRASTRUCTURE))
+		{
+			ui32BIFModule = (aui64MMUStatus[1] & ~RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_CLRMSK) >>
+										RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__BIF_ID_SHIFT;
+			bWriteBack    = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__WRITEBACK_EN) != 0;
+			bFBMFault     = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2__AXT_INFRA__FBM_FAULT_EN) != 0;
+		}
+		else
+		{
+			ui32BIFModule = (aui64MMUStatus[1] & ~RGX_CR_MMU_FAULT_STATUS2_BIF_ID_CLRMSK) >>
+										RGX_CR_MMU_FAULT_STATUS2_BIF_ID_SHIFT;
+			bWriteBack    = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2_WRITEBACK_EN) != 0;
+			bFBMFault     = (aui64MMUStatus[1] & RGX_CR_MMU_FAULT_STATUS2_FBM_FAULT_EN) != 0;
+		}
+
+		if (strcmp(pszMetaOrCore, "Core") != 0)
+		{
+			ui32PC		= (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT;
+			ui64Addr	= ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT) <<  4; /* align shift */
+			ui32Requester = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT;
+			ui32MMULevel  = (aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT;
+			bRead		= (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS_META_RNW_EN) != 0;
+			bFault      = (aui64MMUStatus[0] & RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN) != 0;
+			bROFault    = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT) == 0x2;
+			bProtFault  = ((aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK) >>
+								RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT) == 0x3;
+		}
+		else
+		{
+			_RGXDecodeMMUReqTags(psDevInfo, ui32Requester, ui32BIFModule, bWriteBack, bFBMFault, &pszRequester);
+		}
+
+		PVR_DUMPDEBUG_LOG("%sMMU (%s) - FAULT:", pszIndent, pszMetaOrCore);
+		PVR_DUMPDEBUG_LOG("%s  * MMU status (0x%016" IMG_UINT64_FMTSPECX " | 0x%08" IMG_UINT64_FMTSPECX "): PC = %d, %s 0x%010" IMG_UINT64_FMTSPECX ", %s%s%s%s%s.",
+						  pszIndent,
+						  aui64MMUStatus[0],
+						  aui64MMUStatus[1],
+						  ui32PC,
+						  (bRead)?"Reading from":"Writing to",
+						  ui64Addr,
+						  (pszRequester) ? pszRequester : pszMetaOrRiscv,
+						  (bFault)?", Fault":"",
+						  (bROFault)?", Read Only fault":"",
+						  (bProtFault)?", PM/FW core protection fault":"",
+						  _RGXDecodeMMULevel(ui32MMULevel));
+
+	}
+}
+
+static_assert((RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_REQ_ID_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_REQ_ID_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_LEVEL_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_LEVEL_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_RNW_EN == RGX_CR_MMU_FAULT_STATUS_META_RNW_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_FAULT_EN == RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS1_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+
+
+#if !defined(NO_HARDWARE)
+static inline IMG_CHAR const *_GetRISCVException(IMG_UINT32 ui32Mcause)
+{
+	switch (ui32Mcause)
+	{
+#define X(value, fatal, description) \
+		case value: \
+			if (fatal) \
+				return description; \
+			return NULL;
+
+		RGXRISCVFW_MCAUSE_TABLE
+#undef X
+
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "Invalid RISC-V FW mcause value 0x%08x", ui32Mcause));
+			return NULL;
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+static const IMG_FLAGS2DESC asHWErrorState[] =
+{
+	{RGX_HW_ERR_NA, "N/A"},
+	{RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL, "Primitive ID failure during DM kill."},
+};
+
+/*
+ *  Translate ID code to descriptive string.
+ *  Returns on the first match.
+ */
+static void _ID2Description(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, const IMG_FLAGS2DESC *psConvTable, IMG_UINT32 ui32TableSize, IMG_UINT32 ui32ID)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (ui32ID == psConvTable[ui32Idx].uiFlag)
+		{
+			OSStringSafeCopy(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+			return;
+		}
+	}
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWAssert
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psRGXFWIfTraceBufCtl - RGX FW trace buffer
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWAssert(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl)
+{
+	const IMG_CHAR *pszTraceAssertPath;
+	const IMG_CHAR *pszTraceAssertInfo;
+	IMG_INT32 ui32TraceAssertLine;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf, INVALIDATE);
+		pszTraceAssertPath = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szPath;
+		pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szInfo;
+		ui32TraceAssertLine = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.ui32LineNum;
+
+		/* print non-null assert strings */
+		if (*pszTraceAssertInfo)
+		{
+			PVR_DUMPDEBUG_LOG("FW-T%d Assert: %.*s (%.*s:%d)",
+			                  i, RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertPath, ui32TraceAssertLine);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWFaults
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psFwSysData       - RGX FW shared system data
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWFaults(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             const RGXFWIF_SYSDATA *psFwSysData)
+{
+	if (psFwSysData->ui32FWFaults > 0)
+	{
+		IMG_UINT32	ui32StartFault = psFwSysData->ui32FWFaults - RGXFWIF_FWFAULTINFO_MAX;
+		IMG_UINT32	ui32EndFault   = psFwSysData->ui32FWFaults - 1;
+		IMG_UINT32  ui32Index;
+
+		if (psFwSysData->ui32FWFaults < RGXFWIF_FWFAULTINFO_MAX)
+		{
+			ui32StartFault = 0;
+		}
+
+		for (ui32Index = ui32StartFault; ui32Index <= ui32EndFault; ui32Index++)
+		{
+			const RGX_FWFAULTINFO *psFaultInfo = &psFwSysData->sFaultInfo[ui32Index % RGXFWIF_FWFAULTINFO_MAX];
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			/* Split OS timestamp in seconds and nanoseconds */
+			RGXConvertOSTimestampToSAndNS(psFaultInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+			PVR_DUMPDEBUG_LOG("FW Fault %d: %.*s (%.*s:%d)",
+			                  ui32Index+1, RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szPath,
+			                  psFaultInfo->sFaultBuf.ui32LineNum);
+			PVR_DUMPDEBUG_LOG("            Data = 0x%016"IMG_UINT64_FMTSPECx", CRTimer = 0x%012"IMG_UINT64_FMTSPECx", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+			                  psFaultInfo->ui64Data,
+			                  psFaultInfo->ui64CRTimer,
+			                  ui64Seconds, ui64Nanoseconds);
+		}
+	}
+}
+
+static void _RGXDumpFWPoll(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_SYSDATA *psFwSysData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psFwSysData->aui32CrPollAddr[i])
+		{
+			PVR_DUMPDEBUG_LOG("T%u polling %s (reg:0x%08X mask:0x%08X)",
+			                  i,
+			                  ((psFwSysData->aui32CrPollAddr[i] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+			                  psFwSysData->aui32CrPollAddr[i] & ~RGXFW_POLL_TYPE_SET,
+			                  psFwSysData->aui32CrPollMask[i]);
+		}
+	}
+
+}
+
+static void _RGXDumpFWHWRInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							  void *pvDumpDebugFile,
+							  const RGXFWIF_SYSDATA *psFwSysData,
+							  const RGXFWIF_HWRINFOBUF *psHWRInfoBuf,
+							  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL          bAnyLocked = IMG_FALSE;
+	IMG_UINT32        dm, i;
+	IMG_UINT32        ui32LineSize;
+	IMG_CHAR          *pszLine, *pszTemp;
+	const IMG_CHAR    *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+	const IMG_CHAR    szMsgHeader[] = "Number of HWR: ";
+	const IMG_CHAR    szMsgFalse[] = "FALSE(";
+	IMG_CHAR          *pszLockupType = "";
+	const IMG_UINT32  ui32MsgHeaderCharCount = ARRAY_SIZE(szMsgHeader) - 1; /* size includes the null */
+	const IMG_UINT32  ui32MsgFalseCharCount = ARRAY_SIZE(szMsgFalse) - 1;
+	IMG_UINT32        ui32HWRRecoveryFlags;
+	IMG_UINT32        ui32ReadIndex;
+
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+	{
+		apszDmNames[RGXFWIF_DM_TDM] = "2D";
+	}
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] ||
+		    psHWRInfoBuf->aui32HwrDmOverranCount[dm])
+		{
+			bAnyLocked = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) && !bAnyLocked && (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_HARDWARE_OK))
+	{
+		/* No HWR situation, print nothing */
+		return;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_BOOL bAnyHWROccurred = IMG_FALSE;
+
+		for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+		{
+			if (psHWRInfoBuf->aui32HwrDmRecoveredCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm] !=0)
+				{
+					bAnyHWROccurred = IMG_TRUE;
+					break;
+				}
+		}
+
+		if (!bAnyHWROccurred)
+		{
+			return;
+		}
+	}
+
+/* <DM name + left parenthesis> + <UINT32 max num of digits> + <slash> + <UINT32 max num of digits> +
+   <plus> + <UINT32 max num of digits> + <right parenthesis + comma + space> */
+#define FWHWRINFO_DM_STR_SIZE (5U + 10U + 1U + 10U + 1U + 10U + 3U)
+
+	ui32LineSize = sizeof(IMG_CHAR) * (
+			ui32MsgHeaderCharCount +
+			(psDevInfo->sDevFeatureCfg.ui32MAXDMCount * FWHWRINFO_DM_STR_SIZE) +
+			ui32MsgFalseCharCount + 1 + (psDevInfo->sDevFeatureCfg.ui32MAXDMCount*6) + 1
+				/* 'FALSE(' + ')' + (UINT16 max num + comma) per DM + \0 */
+			);
+
+	pszLine = OSAllocMem(ui32LineSize);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Out of mem allocating line string (size: %d)",
+			__func__,
+			ui32LineSize));
+		return;
+	}
+
+	OSStringSafeCopy(pszLine, szMsgHeader, ui32LineSize);
+	pszTemp = pszLine + ui32MsgHeaderCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				FWHWRINFO_DM_STR_SIZE,
+				"%s(%u/%u+%u), ",
+				apszDmNames[dm],
+				psHWRInfoBuf->aui32HwrDmRecoveredCount[dm],
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm],
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm]);
+	}
+
+	OSStringLCat(pszLine, szMsgFalse, ui32LineSize);
+	pszTemp += ui32MsgFalseCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				10 + 1 + 1 /* UINT32 max num + comma + \0 */,
+				(dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1 ? "%u," : "%u)"),
+				psHWRInfoBuf->aui32HwrDmFalseDetectCount[dm]);
+	}
+
+	PVR_DUMPDEBUG_LOG("%s", pszLine);
+
+	OSFreeMem(pszLine);
+
+	/* Print out per HWR info */
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (dm == RGXFWIF_DM_GP)
+		{
+			PVR_DUMPDEBUG_LOG("DM %d (GP)", dm);
+		}
+		else
+		{
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+			{
+				IMG_UINT32 ui32HWRRecoveryFlags = psFwSysData->aui32HWRRecoveryFlags[dm];
+				IMG_CHAR sPerDmHwrDescription[RGX_DEBUG_STR_SIZE];
+				sPerDmHwrDescription[0] = '\0';
+
+				if (ui32HWRRecoveryFlags == RGXFWIF_DM_STATE_WORKING)
+				{
+					OSStringSafeCopy(sPerDmHwrDescription, " working;", RGX_DEBUG_STR_SIZE);
+				}
+				else
+				{
+					DebugCommonFlagStrings(sPerDmHwrDescription, RGX_DEBUG_STR_SIZE,
+						asDmState2Description, ARRAY_SIZE(asDmState2Description),
+						ui32HWRRecoveryFlags);
+				}
+				PVR_DUMPDEBUG_LOG("DM %d (HWRflags 0x%08x:%.*s)", dm, ui32HWRRecoveryFlags,
+								  RGX_DEBUG_STR_SIZE, sPerDmHwrDescription);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("DM %d", dm);
+			}
+		}
+
+		ui32ReadIndex = 0;
+		for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+		{
+			IMG_BOOL bPMFault = IMG_FALSE;
+			IMG_UINT32 ui32PC;
+			IMG_UINT32 ui32PageSize = 0;
+			IMG_DEV_PHYADDR sPCDevPAddr = { 0 };
+			const RGX_HWRINFO *psHWRInfo = &psHWRInfoBuf->sHWRInfo[ui32ReadIndex];
+
+			if (ui32ReadIndex >= RGXFWIF_HWINFO_MAX)
+			{
+				PVR_DUMPDEBUG_LOG("HWINFO index error: %u", ui32ReadIndex);
+				break;
+			}
+
+			if ((psHWRInfo->eDM == dm) && (psHWRInfo->ui32HWRNumber != 0))
+			{
+				IMG_CHAR aui8RecoveryNum[10+10+1];
+				IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+				IMG_BOOL bPageFault = IMG_FALSE;
+				IMG_DEV_VIRTADDR sFaultDevVAddr;
+
+				/* Split OS timestamp in seconds and nanoseconds */
+				RGXConvertOSTimestampToSAndNS(psHWRInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+				ui32HWRRecoveryFlags = psHWRInfo->ui32HWRRecoveryFlags;
+				if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_LOCKUP) { pszLockupType = ", Guilty Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_LOCKUP) { pszLockupType = ", Innocent Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_OVERRUNING) { pszLockupType = ", Guilty Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_OVERRUNING) { pszLockupType = ", Innocent Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH) { pszLockupType = ", Hard Context Switch"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_ECC_HWR) { pszLockupType = ", GPU ECC HWR"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_PARITY_HWR) { pszLockupType = ", GPU PARITY HWR"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_LATENT_HWR) { pszLockupType = ", GPU LATENT HWR"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_ICS_HWR) { pszLockupType = ", ICS HWR"; }
+
+				OSSNPrintf(aui8RecoveryNum, sizeof(aui8RecoveryNum), "Recovery %d:", psHWRInfo->ui32HWRNumber);
+				if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+				{
+					PVR_DUMPDEBUG_LOG("  %s Core = %u, PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32CoreID,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+
+				if (psHWRInfo->eHWErrorCode != RGX_HW_ERR_NA)
+				{
+					IMG_CHAR sHWDebugInfo[RGX_DEBUG_STR_SIZE] = "";
+
+					_ID2Description(sHWDebugInfo, RGX_DEBUG_STR_SIZE, asHWErrorState, ARRAY_SIZE(asHWErrorState),
+						psHWRInfo->eHWErrorCode);
+					PVR_DUMPDEBUG_LOG("  HW error code = 0x%X: %s",
+									  psHWRInfo->eHWErrorCode, sHWDebugInfo);
+				}
+
+				pszTemp = &aui8RecoveryNum[0];
+				while (*pszTemp != '\0')
+				{
+					*pszTemp++ = ' ';
+				}
+
+				/* There's currently no time correlation for the Guest OSes on the Firmware so there's no point printing OS Timestamps on Guests */
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   ui64Seconds,
+									   ui64Nanoseconds,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+
+				if (psHWRInfo->ui64CRTimeHWResetFinish != 0)
+				{
+					if (psHWRInfo->ui64CRTimeFreelistReady != 0)
+					{
+						/* If ui64CRTimeFreelistReady is less than ui64CRTimeHWResetFinish it means APM kicked in and the time is not valid. */
+						if (psHWRInfo->ui64CRTimeHWResetFinish < psHWRInfo->ui64CRTimeFreelistReady)
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalRecoveryTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimeHWResetFinish)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimer)*256);
+						}
+						else
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = <not_timed>, TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+						}
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+										   aui8RecoveryNum,
+										   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+					}
+				}
+
+#if defined(HW_ERN_65104_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 65104))
+				{
+					PVR_DUMPDEBUG_LOG("    Active PDS DM USCs = 0x%08x", psHWRInfo->ui32PDSActiveDMUSCs);
+				}
+#endif
+
+#if defined(HW_ERN_69700_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 69700))
+				{
+					PVR_DUMPDEBUG_LOG("    DMs stalled waiting on PDS Store space = 0x%08x", psHWRInfo->ui32PDSStalledDMs);
+				}
+#endif
+
+				switch (psHWRInfo->eHWRType)
+				{
+					case RGX_HWRTYPE_ECCFAULT:
+					{
+						PVR_DUMPDEBUG_LOG("    ECC fault GPU=0x%08x", psHWRInfo->uHWRData.sECCInfo.ui32FaultGPU);
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+						{
+							_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo,
+											&psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0],
+											"Core",
+											DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							sFaultDevVAddr.uiAddr =   psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0];
+							sFaultDevVAddr.uiAddr &=  ~RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK;
+							sFaultDevVAddr.uiAddr >>= RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT;
+							sFaultDevVAddr.uiAddr <<= 4; /* align shift */
+							ui32PC  = (psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0] & ~RGX_CR_MMU_FAULT_STATUS1_CONTEXT_CLRMSK) >>
+													   RGX_CR_MMU_FAULT_STATUS1_CONTEXT_SHIFT;
+							bPMFault = (ui32PC <= 8);
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sMMUInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUMETAFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+						{
+							const IMG_PCHAR pszMetaOrRiscv = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? "Meta" : "RiscV";
+
+							_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo,
+											&psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0],
+											pszMetaOrRiscv,
+											DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							sFaultDevVAddr.uiAddr =   psHWRInfo->uHWRData.sMMUInfo.aui64MMUStatus[0];
+							sFaultDevVAddr.uiAddr &=  ~RGX_CR_MMU_FAULT_STATUS1_ADDRESS_CLRMSK;
+							sFaultDevVAddr.uiAddr >>= RGX_CR_MMU_FAULT_STATUS1_ADDRESS_SHIFT;
+							sFaultDevVAddr.uiAddr <<= 4; /* align shift */
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sMMUInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_POLLFAILURE:
+					{
+						PVR_DUMPDEBUG_LOG("    T%u polling %s (reg:0x%08X mask:0x%08X last:0x%08X)",
+										  psHWRInfo->uHWRData.sPollInfo.ui32ThreadNum,
+										  ((psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & ~RGXFW_POLL_TYPE_SET,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollMask,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollLastValue);
+					}
+					break;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+					case RGX_HWRTYPE_MIPSTLBFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+						{
+							IMG_UINT32 ui32EntryLo = psHWRInfo->uHWRData.sTLBInfo.ui32EntryLo;
+
+							/* This is not exactly what the MMU code does, but the result should be the same */
+							const IMG_UINT32 ui32UnmappedEntry =
+								((IMG_UINT32)(MMU_BAD_PHYS_ADDR & 0xffffffff) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) | RGXMIPSFW_ENTRYLO_UNCACHED;
+
+							PVR_DUMPDEBUG_LOG("    MIPS TLB fault: BadVA = 0x%08X, EntryLo = 0x%08X"
+											  " (page PA 0x%" IMG_UINT64_FMTSPECx", V %u)",
+											  psHWRInfo->uHWRData.sTLBInfo.ui32BadVAddr,
+											  ui32EntryLo,
+											  RGXMIPSFW_TLB_GET_PA(ui32EntryLo),
+											  ui32EntryLo & RGXMIPSFW_TLB_VALID ? 1 : 0);
+
+							if (ui32EntryLo == ui32UnmappedEntry)
+							{
+								PVR_DUMPDEBUG_LOG("    Potential use-after-free detected");
+							}
+						}
+					}
+					break;
+#endif
+
+					case RGX_HWRTYPE_OVERRUN:
+					case RGX_HWRTYPE_UNKNOWNFAILURE:
+					{
+						/* Nothing to dump */
+					}
+					break;
+
+					default:
+					{
+						PVR_DUMPDEBUG_LOG("    Unknown HWR Info type: 0x%x", psHWRInfo->eHWRType);
+					}
+					break;
+				}
+
+				if (bPageFault)
+				{
+					RGXDumpFaultInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psHWRInfo,
+					                 ui32ReadIndex, &sFaultDevVAddr, &sPCDevPAddr, bPMFault, ui32PageSize);
+				}
+
+			}
+
+			if (ui32ReadIndex == RGXFWIF_HWINFO_MAX_FIRST - 1)
+				ui32ReadIndex = psHWRInfoBuf->ui32WriteIndex;
+			else
+				ui32ReadIndex = (ui32ReadIndex + 1) - (ui32ReadIndex / RGXFWIF_HWINFO_LAST_INDEX) * RGXFWIF_HWINFO_MAX_LAST;
+		}
+	}
+}
+
+
+
+
+#if !defined(NO_HARDWARE)
+
+/*!
+*******************************************************************************
+
+ @Function	_CheckForPendingPage
+
+ @Description
+
+ Check if the MMU indicates it is blocked on a pending page
+ MMU4 does not support pending pages, so return false.
+
+ @Input psDevInfo	 - RGX device info
+
+ @Return   IMG_BOOL      - IMG_TRUE if there is a pending page
+
+******************************************************************************/
+static INLINE IMG_BOOL _CheckForPendingPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* MMU4 doesn't support pending pages */
+	return (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) < 4) &&
+		   (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_ENTRY) & RGX_CR_MMU_ENTRY_PENDING_EN);
+#else
+	IMG_UINT32 ui32BIFMMUEntry;
+
+	ui32BIFMMUEntry = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY);
+
+	if (ui32BIFMMUEntry & RGX_CR_BIF_MMU_ENTRY_PENDING_EN)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetPendingPageInfo
+
+ @Description
+
+ Get information about the pending page from the MMU status registers
+
+ @Input psDevInfo	 - RGX device info
+ @Output psDevVAddr      - The device virtual address of the pending MMU address translation
+ @Output pui32CatBase    - The page catalog base
+
+ @Return   void
+
+******************************************************************************/
+static void _GetPendingPageInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_VIRTADDR *psDevVAddr,
+								IMG_UINT32 *pui32CatBase)
+{
+	IMG_UINT64 ui64BIFMMUEntryStatus;
+
+	ui64BIFMMUEntryStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_ENTRY_STATUS);
+
+	psDevVAddr->uiAddr = (ui64BIFMMUEntryStatus & ~RGX_CR_MMU_ENTRY_STATUS_ADDRESS_CLRMSK);
+
+	*pui32CatBase = (ui64BIFMMUEntryStatus & ~RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_CLRMSK) >>
+								RGX_CR_MMU_ENTRY_STATUS_CONTEXT_ID_SHIFT;
+}
+
+#endif
+
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON)
+{
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	IMG_UINT32 ui32DriverID;
+	const RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	/* space for the current clock speed and 3 previous */
+	RGXFWIF_TIME_CORR asTimeCorrs[4];
+	IMG_UINT32 ui32NumClockSpeedChanges;
+
+	/* Should invalidate all reads below including when passed to functions. */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, INVALIDATE);
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(bRGXPoweredON);
+#else
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+		{
+			IMG_UINT64	aui64RegValMMUStatus[2];
+			const IMG_PCHAR pszMetaOrRiscv = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? "Meta" : "RiscV";
+
+			/*
+			 *  BRN72144 prevents reading RGX_CR_MMU_FAULT_STATUS1/2
+			 *  registers while the FW is running...
+			 */
+			if (!RGX_IS_BRN_SUPPORTED(psDevInfo, 72144) ||
+			    (psRGXFWIfTraceBufCtl->sTraceBuf[0].sAssertBuf.szInfo[0] != 0))
+			{
+				aui64RegValMMUStatus[0] = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_FAULT_STATUS1);
+				aui64RegValMMUStatus[1] = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_FAULT_STATUS2);
+				_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, &aui64RegValMMUStatus[0], "Core", DD_SUMMARY_INDENT);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("%sMMU (Core) - Unknown", DD_SUMMARY_INDENT);
+			}
+
+			aui64RegValMMUStatus[0] = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_FAULT_STATUS_META);
+			_RGXDumpRGXMMUFaultStatus(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, &aui64RegValMMUStatus[0], pszMetaOrRiscv, DD_SUMMARY_INDENT);
+		}
+
+		if (_CheckForPendingPage(psDevInfo))
+		{
+			IMG_UINT32 ui32CatBase;
+			IMG_DEV_VIRTADDR sDevVAddr;
+
+			PVR_DUMPDEBUG_LOG("MMU Pending page: Yes");
+
+			_GetPendingPageInfo(psDevInfo, &sDevVAddr, &ui32CatBase);
+
+			if (ui32CatBase <= MAX_RESERVED_FW_MMU_CONTEXT)
+			{
+				PVR_DUMPDEBUG_LOG("Cannot check address on PM cat base %u", ui32CatBase);
+			}
+			else
+			{
+				IMG_DEV_PHYADDR sPCDevPAddr;
+				MMU_FAULT_DATA sFaultData;
+				IMG_BOOL bIsValid = IMG_TRUE;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+				IMG_UINT64 ui64CBaseMapping;
+				IMG_UINT32 ui32CBaseMapCtxReg;
+
+				if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_EN);
+				}
+				else
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING_INVALID_EN);
+				}
+#else
+				sPCDevPAddr.uiAddr = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_CAT_BASEN(ui32CatBase));
+#endif
+
+				PVR_DUMPDEBUG_LOG("Checking device virtual address " IMG_DEV_VIRTADDR_FMTSPEC
+							" on cat base %u. PC Addr = 0x%" IMG_UINT64_FMTSPECx " is %s",
+								sDevVAddr.uiAddr,
+								ui32CatBase,
+								sPCDevPAddr.uiAddr,
+								bIsValid ? "valid":"invalid");
+				RGXCheckFaultAddress(psDevInfo, &sDevVAddr, &sPCDevPAddr, &sFaultData);
+				RGXDumpFaultAddressHostView(&sFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_SUMMARY_INDENT);
+			}
+		}
+	}
+#endif /* NO_HARDWARE */
+
+#if !defined(NO_HARDWARE)
+	/* Determine the type virtualisation support used */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+#if defined(SUPPORT_AUTOVZ)
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with HW register support");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with shared memory");
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with static Fw heap allocation");
+#endif /* defined(SUPPORT_AUTOVZ) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with dynamic Fw heap allocation");
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+	}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_CONNECTION_FW_STATE eFwState;
+		RGXFWIF_CONNECTION_OS_STATE eOsState;
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+		eFwState = KM_GET_FW_CONNECTION(psDevInfo);
+		eOsState = KM_GET_OS_CONNECTION(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation firmware connection state: %s (Fw=%s; OS=%s)",
+						  ((eFwState == RGXFW_CONNECTION_FW_ACTIVE) && (eOsState == RGXFW_CONNECTION_OS_ACTIVE)) ? ("UP") : ("DOWN"),
+						  (eFwState < RGXFW_CONNECTION_FW_STATE_COUNT) ? (apszFwOsStateName[eFwState]) : ("invalid"),
+						  (eOsState < RGXFW_CONNECTION_OS_STATE_COUNT) ? (apszFwOsStateName[eOsState]) : ("invalid"));
+
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32FwAliveTS;
+		IMG_UINT32 ui32OsAliveTS;
+
+		KM_ALIVE_TOKEN_CACHEOP(Fw, INVALIDATE);
+		KM_ALIVE_TOKEN_CACHEOP(Os, INVALIDATE);
+
+		ui32FwAliveTS = KM_GET_FW_ALIVE_TOKEN(psDevInfo);
+		ui32OsAliveTS = KM_GET_OS_ALIVE_TOKEN(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation watchdog timestamps (in GPU timer ticks): Fw=%u; OS=%u; diff(FW, OS) = %u",
+						  ui32FwAliveTS, ui32OsAliveTS, ui32FwAliveTS - ui32OsAliveTS);
+	}
+#endif
+#endif /* !defined(NO_HARDWARE) */
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_CHAR sHwrStateDescription[RGX_DEBUG_STR_SIZE];
+		IMG_BOOL bDriverIsolationEnabled = IMG_FALSE;
+		IMG_UINT32 ui32HostIsolationGroup;
+
+		sHwrStateDescription[0] = '\0';
+
+		DebugCommonFlagStrings(sHwrStateDescription, RGX_DEBUG_STR_SIZE,
+			asHwrState2Description, ARRAY_SIZE(asHwrState2Description),
+			psFwSysData->ui32HWRStateFlags);
+		PVR_DUMPDEBUG_LOG("RGX HWR State 0x%08x:%s", psFwSysData->ui32HWRStateFlags, sHwrStateDescription);
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: %s (APM %s: %d ok, %d denied, %d non-idle, %d retry, %d other, %d total. Latency: %u ms)",
+		                  (psFwSysData->ePowState < ARRAY_SIZE(pszPowStateName) ? pszPowStateName[psFwSysData->ePowState] : "???"),
+		                  (psDevInfo->pvAPMISRData)?"enabled":"disabled",
+		                  psDevInfo->ui32ActivePMReqOk - psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqDenied,
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqRetry,
+		                  psDevInfo->ui32ActivePMReqTotal -
+		                  psDevInfo->ui32ActivePMReqOk -
+		                  psDevInfo->ui32ActivePMReqDenied -
+		                  psDevInfo->ui32ActivePMReqRetry -
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqTotal,
+		                  psRuntimeCfg->ui32ActivePMLatencyms);
+
+		ui32NumClockSpeedChanges = (IMG_UINT32) OSAtomicRead(&psDevInfo->psDeviceNode->iNumClockSpeedChanges);
+		RGXGetTimeCorrData(psDevInfo->psDeviceNode, asTimeCorrs, ARRAY_SIZE(asTimeCorrs));
+
+		PVR_DUMPDEBUG_LOG("RGX DVFS: %u frequency changes. "
+		                  "Current frequency: %u.%03u MHz (sampled at %" IMG_UINT64_FMTSPEC " ns). "
+		                  "FW frequency: %u.%03u MHz.",
+		                  ui32NumClockSpeedChanges,
+		                  asTimeCorrs[0].ui32CoreClockSpeed / 1000000,
+		                  (asTimeCorrs[0].ui32CoreClockSpeed / 1000) % 1000,
+		                  asTimeCorrs[0].ui64OSTimeStamp,
+		                  psRuntimeCfg->ui32CoreClockSpeed / 1000000,
+		                  (psRuntimeCfg->ui32CoreClockSpeed / 1000) % 1000);
+		if (ui32NumClockSpeedChanges > 0)
+		{
+			PVR_DUMPDEBUG_LOG("          Previous frequencies: %u.%03u, %u.%03u, %u.%03u MHz (Sampled at "
+							"%" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ")",
+												asTimeCorrs[1].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[1].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[2].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[2].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[3].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[3].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[1].ui64OSTimeStamp,
+												asTimeCorrs[2].ui64OSTimeStamp,
+												asTimeCorrs[3].ui64OSTimeStamp);
+		}
+
+		ui32HostIsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID];
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			RGXFWIF_OS_RUNTIME_FLAGS sFwRunFlags = psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+			IMG_UINT32 ui32IsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID];
+			IMG_BOOL bMTSEnabled = IMG_FALSE;
+
+#if !defined(NO_HARDWARE)
+			if (bRGXPoweredON)
+			{
+				bMTSEnabled = (
+#if defined(FIX_HW_BRN_64502_BIT_MASK)
+								RGX_IS_BRN_SUPPORTED(psDevInfo, 64502) ||
+#endif
+								!RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_VIRTUALISATION)) ?
+								IMG_TRUE : ((OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE_ENABLE) & BIT(ui32DriverID)) != 0);
+			}
+#endif
+			PVR_DUMPDEBUG_LOG("RGX FW OS %u - State: %s; Freelists: %s%s; Priority: %u; Isolation group: %u;  Time Slice%s: %u%% (%ums); %s", ui32DriverID,
+							  apszFwOsStateName[sFwRunFlags.bfOsState],
+							  (sFwRunFlags.bfFLOk) ? "Ok" : "Not Ok",
+							  (sFwRunFlags.bfFLGrowPending) ? "; Grow Request Pending" : "",
+							  psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID],
+							  ui32IsolationGroup,
+							  (psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] != 0) ? "" : "*",
+							  psFwSysData->aui32TSMirror[ui32DriverID],
+							  (psFwSysData->aui32TSMirror[ui32DriverID] *
+							   psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs / 100),
+							  (bMTSEnabled) ? "MTS on;" : "MTS off;"
+							 );
+
+			if (ui32IsolationGroup != ui32HostIsolationGroup)
+			{
+				bDriverIsolationEnabled = IMG_TRUE;
+			}
+
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+			{
+				/* don't print guest information on native mode drivers */
+				break;
+			}
+		}
+
+#if defined(PVR_ENABLE_PHR)
+		{
+			IMG_CHAR sPHRConfigDescription[RGX_DEBUG_STR_SIZE];
+
+			sPHRConfigDescription[0] = '\0';
+			DebugCommonFlagStrings(sPHRConfigDescription, RGX_DEBUG_STR_SIZE,
+			                   asPHRConfig2Description, ARRAY_SIZE(asPHRConfig2Description),
+			                   BIT_ULL(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode));
+
+			PVR_DUMPDEBUG_LOG("RGX PHR configuration: (%d) %.*s", psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, RGX_DEBUG_STR_SIZE, sPHRConfigDescription);
+		}
+#endif
+
+		if (bDriverIsolationEnabled)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Hard Context Switch deadline: %u ms", psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+		}
+
+		_RGXDumpFWAssert(pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl);
+		_RGXDumpFWFaults(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+		_RGXDumpFWPoll(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("RGX FW State: Unavailable under Guest Mode of operation");
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: Unavailable under Guest Mode of operation");
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+	_RGXDumpFWHWRInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData, psDevInfo->psRGXFWIfHWRInfoBufCtl, psDevInfo);
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK)
+	/* Dump all non-zero values in lines of 8... */
+	{
+		IMG_CHAR    pszLine[(9*RGXFWIF_STATS_FRAMEWORK_LINESIZE)+1];
+		const IMG_UINT32 *pui32FWStatsBuf = psFwSysData->aui32FWStatsBuf;
+		IMG_UINT32  ui32Index1, ui32Index2;
+
+		PVR_DUMPDEBUG_LOG("STATS[START]: RGXFWIF_STATS_FRAMEWORK_MAX=%d", RGXFWIF_STATS_FRAMEWORK_MAX);
+		for (ui32Index1 = 0;  ui32Index1 < RGXFWIF_STATS_FRAMEWORK_MAX;  ui32Index1 += RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+		{
+			IMG_UINT32  ui32OrOfValues = 0;
+			IMG_CHAR    *pszBuf = pszLine;
+
+			/* Print all values in this line and skip if all zero... */
+			for (ui32Index2 = 0;  ui32Index2 < RGXFWIF_STATS_FRAMEWORK_LINESIZE;  ui32Index2++)
+			{
+				ui32OrOfValues |= pui32FWStatsBuf[ui32Index1+ui32Index2];
+				OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32FWStatsBuf[ui32Index1+ui32Index2]);
+				pszBuf += 9; /* write over the '\0' */
+			}
+
+			if (ui32OrOfValues != 0)
+			{
+				PVR_DUMPDEBUG_LOG("STATS[%08x]:%s", ui32Index1, pszLine);
+			}
+		}
+		PVR_DUMPDEBUG_LOG("STATS[END]");
+	}
+#endif
+}
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvSecureRegsBaseKM;
+	RGXRISCVFW_STATE sRiscvState;
+	const IMG_CHAR *pszException;
+	PVRSRV_ERROR eError;
+
+	/* Limit dump to what is currently being used */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4);
+	}
+	else
+#endif
+	{
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG4);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG5);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG6);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG12);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG13);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG14);
+	}
+
+	PVR_DUMPDEBUG_LOG("---- [ RISC-V internal state ] ----");
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGXRiscvIsHalted(psDevInfo))
+	{
+		/* Avoid resuming the RISC-V FW as most operations
+		 * on the debug module require a halted core */
+		PVR_DUMPDEBUG_LOG("(skipping as RISC-V found halted)");
+		return PVRSRV_OK;
+	}
+#endif
+
+	eError = RGXRiscvHalt(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+#define X(name, address)												\
+	eError = RGXRiscvReadReg(psDevInfo, address, &sRiscvState.name);	\
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXRiscvReadReg", _RISCVDMError);	\
+	DDLOGVAL32(#name, sRiscvState.name);
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+
+	eError = RGXRiscvResume(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+	pszException = _GetRISCVException(sRiscvState.mcause);
+	if (pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG("RISC-V FW hit an exception: %s", pszException);
+
+		eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+		}
+	}
+
+	return PVRSRV_OK;
+
+_RISCVDMError:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to communicate with the Debug Module"));
+
+	return eError;
+}
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(CORE_ID);
+}
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG64(MULTICORE);
+	DDLOG32(MULTICORE_SYSTEM);
+	DDLOG32(MULTICORE_DOMAIN);
+
+#if !defined(RGX_CR_MULTICORE_AXI)
+#define RGX_CR_MULTICORE_AXI                              (0x2508U)
+#define RGX_CR_MULTICORE_AXI_ERROR                        (0x2510U)
+#endif
+	DDLOG32(MULTICORE_AXI);
+	DDLOG32(MULTICORE_AXI_ERROR);
+	DDLOG32(MULTICORE_TDM_CTRL_COMMON);
+	DDLOG32(MULTICORE_FRAGMENT_CTRL_COMMON);
+	DDLOG32(MULTICORE_COMPUTE_CTRL_COMMON);
+}
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG64(CLK_CTRL0);
+	DDLOG64(CLK_STATUS0);
+	DDLOG64(CLK_CTRL1);
+	DDLOG64(CLK_STATUS1);
+	DDLOG32(CLK_CTRL2);
+	DDLOG32(CLK_STATUS2);
+}
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	/* BRN72144 prevents reading RGX_CR_MMU_FAULT_STATUS1/2 while the FW is running... */
+	if (!RGX_IS_BRN_SUPPORTED(psDevInfo, 72144)  ||
+		(psDevInfo->psRGXFWIfTraceBufCtl->sTraceBuf[0].sAssertBuf.szInfo[0] != 0))
+	{
+		DDLOG64(MMU_FAULT_STATUS1);
+		DDLOG64(MMU_FAULT_STATUS2);
+	}
+	DDLOG64(MMU_FAULT_STATUS_PM);
+	DDLOG64(MMU_FAULT_STATUS_META);
+	DDLOG64(SLC_STATUS1);
+	DDLOG64(SLC_STATUS2);
+	DDLOG64(SLC_STATUS3);
+	DDLOG64(SLC_STATUS_DEBUG);
+	DDLOG64(MMU_STATUS);
+	DDLOG32(BIF_PFS);
+	DDLOG32(BIF_TEXAS0_PFS);
+	DDLOG32(BIF_TEXAS1_PFS);
+	DDLOG32(BIF_OUTSTANDING_READ);
+	DDLOG32(BIF_TEXAS0_OUTSTANDING_READ);
+	DDLOG32(BIF_TEXAS1_OUTSTANDING_READ);
+	DDLOG32(FBCDC_IDLE);
+	DDLOG32(FBCDC_STATUS);
+	DDLOG32(FBCDC_SIGNATURE_STATUS);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, POWER_ISLAND_VERSION) &&
+	    RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) < 2)
+	{
+		DDLOG32(SPU_ENABLE);
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT))
+	{
+		DDLOG64(CONTEXT_MAPPING0);
+		DDLOG64(CONTEXT_MAPPING2);
+		DDLOG64(CONTEXT_MAPPING3);
+		DDLOG64(CONTEXT_MAPPING4);
+	}
+}
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(PERF_PHASE_2D);
+	DDLOG32(PERF_CYCLE_2D_TOTAL);
+	DDLOG32(PERF_PHASE_GEOM);
+	DDLOG32(PERF_CYCLE_GEOM_TOTAL);
+	DDLOG32(PERF_PHASE_FRAG);
+	DDLOG32(PERF_CYCLE_FRAG_TOTAL);
+	DDLOG32(PERF_CYCLE_GEOM_OR_FRAG_TOTAL);
+	DDLOG32(PERF_CYCLE_GEOM_AND_FRAG_TOTAL);
+	DDLOG32(PERF_PHASE_COMP);
+	DDLOG32(PERF_CYCLE_COMP_TOTAL);
+	DDLOG32(PM_PARTIAL_RENDER_ENABLE);
+
+	DDLOG32(ISP_RENDER);
+	DDLOG32(ISP_CTL);
+
+	DDLOG32(CDM_CONTEXT_STORE_STATUS__CDM_CSF_LT5);
+	DDLOG64(CDM_CONTEXT_PDS0);
+	DDLOG64(CDM_CONTEXT_PDS1);
+	DDLOG64(CDM_TERMINATE_PDS);
+	DDLOG64(CDM_TERMINATE_PDS1);
+	DDLOG64(CDM_CONTEXT_LOAD_PDS0);
+	DDLOG64(CDM_CONTEXT_LOAD_PDS1);
+}
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(SLC_IDLE);
+	DDLOG32(SLC_FAULT_STOP_STATUS);
+}
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(JONES_IDLE);
+
+	DDLOG64(SCRATCH0);
+	DDLOG64(SCRATCH1);
+	DDLOG64(SCRATCH2);
+	DDLOG64(SCRATCH3);
+	DDLOG64(SCRATCH4);
+	DDLOG64(SCRATCH5);
+	DDLOG64(SCRATCH6);
+	DDLOG64(SCRATCH7);
+	DDLOG64(SCRATCH8);
+	DDLOG64(SCRATCH9);
+	DDLOG64(SCRATCH10);
+	DDLOG64(SCRATCH11);
+	DDLOG64(SCRATCH12);
+	DDLOG64(SCRATCH13);
+	DDLOG64(SCRATCH14);
+	DDLOG64(SCRATCH15);
+	DDLOG32(IRQ_OS0_EVENT_STATUS);
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#undef REG32_FMTSPEC
+#undef REG64_FMTSPEC
+#undef DDLOG32
+#undef DDLOG64
+#undef DDLOG32_DPX
+#undef DDLOG64_DPX
+#undef DDLOGVAL32
+
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32VerbLevel)
+{
+	DumpRenderCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DumpKickSyncCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		DumpComputeCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		DumpTDMTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+}
+
+/******************************************************************************
+ End of file (rgxdebug.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdevice.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdevice.h
new file mode 100644
index 000000000000..5e197ec8a0f9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxdevice.h
@@ -0,0 +1,953 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX device node header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX device node
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEVICE_H)
+#define RGXDEVICE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_device_types.h"
+#include "mmu_common.h"
+#include "rgx_fwif_km.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "osfunc.h"
+#include "rgxlayer_impl.h"
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "hash.h"
+#endif
+typedef struct _RGX_SERVER_COMMON_CONTEXT_ RGX_SERVER_COMMON_CONTEXT;
+
+typedef struct {
+	DEVMEM_MEMDESC		*psFWFrameworkMemDesc;
+} RGX_COMMON_CONTEXT_INFO;
+
+
+/*!
+ ******************************************************************************
+ * Device state flags
+ *****************************************************************************/
+#define RGXKM_DEVICE_STATE_ZERO_FREELIST                          (0x1)  /*!< Zeroing the physical pages of reconstructed free lists */
+#define RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN                  (0x2)  /*!< Used to disable the Devices Watchdog logging */
+#define RGXKM_DEVICE_STATE_GPU_UNITS_POWER_CHANGE_EN              (0x4)  /*!< Used for validation to inject power units state change every DM kick */
+#define RGXKM_DEVICE_STATE_CCB_GROW_EN                            (0x8)  /*!< Used to indicate CCB grow is permitted */
+#define RGXKM_DEVICE_STATE_ENABLE_SPU_UNITS_POWER_MASK_CHANGE_EN  (0x10) /*!< Used for validation to enable SPU power state mask change */
+#define RGXKM_DEVICE_STATE_MASK                                   (0x1F)
+
+/*!
+ ******************************************************************************
+ * ECC RAM Fault Validation
+ *****************************************************************************/
+#define RGXKM_ECC_ERR_INJ_DISABLE 0
+#define RGXKM_ECC_ERR_INJ_SLC     1
+#define RGXKM_ECC_ERR_INJ_USC     2
+#define RGXKM_ECC_ERR_INJ_TPU     3
+#define RGXKM_ECC_ERR_INJ_RASCAL  4
+#define RGXKM_ECC_ERR_INJ_MARS    5
+
+#define RGXKM_ECC_ERR_INJ_INTERVAL 10U
+
+/*!
+ ******************************************************************************
+ * GPU DVFS Table
+ *****************************************************************************/
+
+#define RGX_GPU_DVFS_TABLE_SIZE                      32
+#define RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US       25000     /* Time required to calibrate a clock frequency the first time */
+#define RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US  150000    /* Time required for a recalibration after a DVFS transition */
+#define RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US    10000000  /* Time before the next periodic calibration and correlation */
+
+
+/*!
+ ******************************************************************************
+ * Global flags for driver validation
+ *****************************************************************************/
+#define RGX_VAL_FBDC_SIG_CHECK_NOERR_EN           (0x2U)  /*!< Enable FBDC signature check. Signatures must match */
+#define RGX_VAL_FBDC_SIG_CHECK_ERR_EN             (0x4U)  /*!< Enable FBDC signature check. Signatures must not match */
+#define RGX_VAL_GPUSTATEPIN_EN                    (0x8U)  /*!< Enable GPU state pin check */
+#define RGX_VAL_WGP_SIG_CHECK_NOERR_EN           (0x10U)  /*!< Enable WGP signature check. Signatures must match */
+#define RGX_VAL_WGP_SIG_CHECK_ERR_EN             (0x20U)  /*!< Enable WGP signature check. Signatures must not match */
+#define RGX_VAL_TRP_SIG_CHECK_NOERR_EN           (0x40U)  /*!< Enable TRP signature check. Signatures must match */
+#define RGX_VAL_TRP_SIG_CHECK_ERR_EN             (0x80U)  /*!< Enable TRP signature check. Signatures must not match */
+
+/*!
+ ******************************************************************************
+ * HWPerf L2 Stream ID type definition.
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_HWPERF_L2_STREAM_ID;
+/* HWPerf stream for Client HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_HWPERF 0U
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+/* HWPerf stream for FTrace HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_FTRACE 1U
+#define RGX_HWPERF_L2_STREAM_LAST   2U
+#else
+#define RGX_HWPERF_L2_STREAM_LAST   1U
+#endif
+
+/*!
+ ******************************************************************************
+ * Boot-time Faults Injection
+ *****************************************************************************/
+#define RGX_NO_FAULTS_INJ                  0
+#define RGX_AXI_BUS_PARITY_INJ             1
+#define RGX_SAFETY_IRQ_AXI_BUS_PARITY_INJ  2
+#define RGX_XPU_BUS_PARITY_INJ             3
+#define RGX_AXI_MEM_BUS_PARITY_INJ         4
+#define RGX_DCLS_INJ                       5
+#define RGX_MMU_PTE_PARITY_INJ             6
+#define RGX_MMU_PTE_PARITY_INJ_KICK_NAME   "kick-tq"
+#define RGX_MMU_PTE_PARITY_INJ_PMR          "tquscpmr"
+#define RGX_REST_LATENT_ICS_INJ            7
+#define RGX_REST_LATENT_ICS_KICK_NAME      "DoKickTA"
+
+typedef struct _GPU_FREQ_TRACKING_DATA_
+{
+	/* Core clock speed estimated by the driver */
+	IMG_UINT32 ui32EstCoreClockSpeed;
+
+	/* Amount of successful calculations of the estimated core clock speed */
+	IMG_UINT32 ui32CalibrationCount;
+} GPU_FREQ_TRACKING_DATA;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+#define RGX_GPU_FREQ_TRACKING_SIZE 16
+
+typedef struct
+{
+	IMG_UINT64 ui64BeginCRTimestamp;
+	IMG_UINT64 ui64BeginOSTimestamp;
+
+	IMG_UINT64 ui64EndCRTimestamp;
+	IMG_UINT64 ui64EndOSTimestamp;
+
+	IMG_UINT32 ui32EstCoreClockSpeed;
+	IMG_UINT32 ui32CoreClockSpeed;
+} GPU_FREQ_TRACKING_HISTORY;
+#endif
+
+typedef struct _RGX_GPU_DVFS_TABLE_
+{
+	/* Beginning of current calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimestamp;
+	IMG_UINT64 ui64CalibrationOSTimestamp;
+
+	/* Calculated calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimediff;
+	IMG_UINT64 ui64CalibrationOSTimediff;
+
+	/* Current calibration period (in us) */
+	IMG_UINT32 ui32CalibrationPeriod;
+
+	/* System layer frequency table and frequency tracking data */
+	IMG_UINT32 ui32FreqIndex;
+	IMG_UINT32 aui32GPUFrequency[RGX_GPU_DVFS_TABLE_SIZE];
+	GPU_FREQ_TRACKING_DATA asTrackingData[RGX_GPU_DVFS_TABLE_SIZE];
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	IMG_UINT32 ui32HistoryIndex;
+	GPU_FREQ_TRACKING_HISTORY asTrackingHistory[RGX_GPU_FREQ_TRACKING_SIZE];
+#endif
+} RGX_GPU_DVFS_TABLE;
+
+
+/*!
+ ******************************************************************************
+ * GPU utilisation statistics
+ *****************************************************************************/
+
+typedef struct _RGXFWIF_TEMP_GPU_UTIL_STATS_
+{
+	IMG_UINT64 aaaui64DMOSTmpCounters[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 aaui64DMOSTmpLastWord[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastState[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastPeriod[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastTime[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+} RGXFWIF_TEMP_GPU_UTIL_STATS;
+
+typedef struct _RGXFWIF_GPU_UTIL_STATS_
+{
+	IMG_BOOL   bValid;                /* If TRUE, statistics are valid.
+	                                     FALSE if the driver couldn't get reliable stats. */
+	IMG_UINT64 ui64GpuStatActive;     /* GPU active statistic */
+	IMG_UINT64 ui64GpuStatBlocked;    /* GPU blocked statistic */
+	IMG_UINT64 ui64GpuStatIdle;       /* GPU idle statistic */
+	IMG_UINT64 ui64GpuStatCumulative; /* Sum of active/blocked/idle stats */
+
+	IMG_UINT64 aaui64DMOSStatInactive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];   /* Per-DM per-OS sum of idle and blocked stats */
+	IMG_UINT64 aaui64DMOSStatActive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];     /* Per-DM per-OS active statistic */
+	IMG_UINT64 aaui64DMOSStatCumulative[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED]; /* Per-DM per-OS sum of active/blocked/idle stats */
+
+	IMG_UINT64 ui64TimeStamp;         /* Timestamp of the most recent sample of the GPU stats */
+
+	RGXFWIF_TEMP_GPU_UTIL_STATS sTempGpuStats; /* Temporary data used to calculate the per-DM per-OS statistics */
+} RGXFWIF_GPU_UTIL_STATS;
+
+
+typedef struct _RGX_REG_CONFIG_
+{
+	IMG_BOOL               bEnabled;
+	RGXFWIF_REG_CFG_TYPE   eRegCfgTypeToPush;
+	IMG_UINT32             ui32NumRegRecords;
+	POS_LOCK               hLock;
+} RGX_REG_CONFIG;
+
+typedef struct _PVRSRV_STUB_PBDESC_ PVRSRV_STUB_PBDESC;
+
+
+typedef struct _PVRSRV_DEVICE_FEATURE_CONFIG_
+{
+	IMG_UINT64 ui64ErnsBrns;
+	IMG_UINT64 ui64Features;
+	IMG_UINT32 ui32B;
+	IMG_UINT32 ui32V;
+	IMG_UINT32 ui32N;
+	IMG_UINT32 ui32C;
+	IMG_UINT32 ui32FeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX];
+	IMG_UINT32 ui32MAXDMCount;
+	IMG_UINT32 ui32MAXPowUnitCount;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	IMG_UINT32 ui32MAXRACCount;
+#endif
+	IMG_UINT32 ui32SLCSizeInBytes;
+	IMG_PCHAR  pszBVNCString;
+}PVRSRV_DEVICE_FEATURE_CONFIG;
+
+/* This is used to get the value of a specific feature.
+ * Note that it will assert if the feature is disabled or value is invalid. */
+#define RGX_GET_FEATURE_VALUE(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] )
+
+/* This is used to check if the feature value (e.g. with an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] < RGX_FEATURE_VALUE_DISABLED )
+
+/* This is used to check if the Boolean feature (e.g. WITHOUT an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_SUPPORTED(psDevInfo, Feature) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64Features, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if the ERN is available for the currently running BVNC or not */
+#define RGX_IS_ERN_SUPPORTED(psDevInfo, ERN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if the BRN is available for the currently running BVNC or not */
+#define RGX_IS_BRN_SUPPORTED(psDevInfo, BRN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+/* there is a corresponding define in rgxapi.h */
+#define RGX_MAX_TIMER_QUERIES 16U
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*!
+ * The host maintains a 512-deep cache of submitted workloads per device,
+ * i.e. a global look-up table for TA, 3D and compute (depending on the RGX
+ * hardware support present)
+ */
+
+/*
+ * For the workload estimation return data array, the max amount of commands the
+ * MTS can have is 255, therefore 512 (LOG2 = 9) is large enough to account for
+ * all corner cases
+ */
+#define RETURN_DATA_ARRAY_SIZE_LOG2 (9)
+#define RETURN_DATA_ARRAY_SIZE      ((1U) << RETURN_DATA_ARRAY_SIZE_LOG2)
+#define RETURN_DATA_ARRAY_WRAP_MASK (RETURN_DATA_ARRAY_SIZE - 1)
+
+#define WORKLOAD_HASH_SIZE_LOG2		6
+#define WORKLOAD_HASH_SIZE			((1U) << WORKLOAD_HASH_SIZE_LOG2)
+#define WORKLOAD_HASH_WRAP_MASK		(WORKLOAD_HASH_SIZE - 1)
+
+/*!
+ * Workload characteristics for supported data masters.
+ * All characteristics must match for the workload estimate to be used/updated.
+ */
+typedef union _RGX_WORKLOAD_
+{
+	struct
+	{
+		IMG_UINT32				ui32RenderTargetSize;
+		IMG_UINT32				ui32NumberOfDrawCalls;
+		IMG_UINT32				ui32NumberOfIndices;
+		IMG_UINT32				ui32NumberOfMRTs;
+	} sTA3D;
+
+	struct
+	{
+		IMG_UINT32				ui32NumberOfWorkgroups;
+		IMG_UINT32				ui32NumberOfWorkitems;
+	} sCompute;
+
+	struct
+	{
+		IMG_UINT32				ui32Characteristic1;
+		IMG_UINT32				ui32Characteristic2;
+	} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	struct
+	{
+		IMG_UINT32				ui32DispatchSize;
+		IMG_UINT32				ui32AccStructSize;
+	} sRay;
+#endif
+
+} RGX_WORKLOAD;
+
+/*!
+ * Host data used to match the return data (actual cycles count) to the
+ * submitted command packet.
+ * The hash table is a per-DM circular buffer containing a key based on the
+ * workload characteristics. On job completion, the oldest workload data
+ * is evicted if the CB is full and the driver matches the characteristics
+ * to the matching data.
+ *
+ * o If the driver finds a match the existing cycle estimate is averaged with
+ *   the actual cycles used.
+ * o Otherwise a new hash entry is created with the actual cycles for this
+ *   workload.
+ *
+ * Subsequently if a match is found during command submission, the estimate
+ * is passed to the scheduler, e.g. adjust the GPU frequency if PDVFS is enabled.
+ */
+typedef struct _WORKLOAD_MATCHING_DATA_
+{
+	POS_LOCK				psHashLock;
+	HASH_TABLE				*psHashTable;		/*! existing workload cycle estimates for this DM */
+	RGX_WORKLOAD			asHashKeys[WORKLOAD_HASH_SIZE];
+	IMG_UINT64				aui64HashData[WORKLOAD_HASH_SIZE];
+	IMG_UINT32				ui32HashArrayWO;	/*! track the most recent workload estimates */
+} WORKLOAD_MATCHING_DATA;
+
+/*!
+ * A generic container for the workload matching data for GPU contexts:
+ * rendering (TA, 3D), compute, etc.
+ */
+typedef struct _WORKEST_HOST_DATA_
+{
+	union
+	{
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTA;	/*!< matching data for TA commands */
+			WORKLOAD_MATCHING_DATA	sData3D;	/*!< matching data for 3D commands */
+		} sTA3D;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataCDM;	/*!< matching data for CDM commands */
+		} sCompute;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTDM;	/*!< matching data for TDM-TQ commands */
+		} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataRDM;	/*!< matching data for RDM commands */
+		} sRay;
+#endif
+	} uWorkloadMatchingData;
+
+	/*
+	 * This is a per-context property, hence the TA and 3D share the same
+	 * per render context counter.
+	 */
+	IMG_UINT32				ui32WorkEstCCBReceived;	/*!< Used to ensure all submitted work
+														 estimation commands are received
+														 by the host before clean up. */
+} WORKEST_HOST_DATA;
+
+/*!
+ * Entries in the list of submitted workloads, used when the completed command
+ * returns data to the host.
+ *
+ * - the matching data is needed as it holds the hash data
+ * - the host data is needed for completion updates, ensuring memory is not
+ *   freed while workload estimates are in-flight.
+ * - the workload characteristic is used in the hash table look-up.
+ */
+typedef struct _WORKEST_RETURN_DATA_
+{
+	WORKEST_HOST_DATA		*psWorkEstHostData;
+	WORKLOAD_MATCHING_DATA	*psWorkloadMatchingData;
+	RGX_WORKLOAD			sWorkloadCharacteristics;
+} WORKEST_RETURN_DATA;
+#endif
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+typedef struct
+{
+#if defined(PDUMP)
+	IMG_HANDLE      hPdumpPages;
+#endif
+	PG_HANDLE       sPages;
+	IMG_DEV_PHYADDR sPhysAddr;
+} RGX_MIPS_ADDRESS_TRAMPOLINE;
+#endif
+
+
+/*!
+ ******************************************************************************
+ * RGX Device error counts
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_ERROR_COUNTS_
+{
+	IMG_UINT32 ui32WGPErrorCount;		/*!< count of the number of WGP checksum errors */
+	IMG_UINT32 ui32TRPErrorCount;		/*!< count of the number of TRP checksum errors */
+} PVRSRV_RGXDEV_ERROR_COUNTS;
+
+/*!
+ ******************************************************************************
+ * RGX Debug dump firmware trace log type
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_FWT_LOGTYPE;
+#define RGX_FWT_LOGTYPE_NONE    0U
+#define RGX_FWT_LOGTYPE_BINARY  1U
+#define RGX_FWT_LOGTYPE_DECODED 2U
+#define RGX_FWT_LOGTYPE_PARTIAL 3U
+
+/*!
+ ******************************************************************************
+ * RGX Device info
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_INFO_
+{
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+
+	PVRSRV_DEVICE_FEATURE_CONFIG	sDevFeatureCfg;
+
+	IMG_BOOL				bDevInit2Done;
+
+	IMG_BOOL				bFirmwareInitialised;
+	IMG_BOOL				bPDPEnabled;
+
+	IMG_HANDLE				hDbgReqNotify;
+
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvRegsBaseKM;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvSecureRegsBaseKM;
+#endif
+
+	IMG_HANDLE				hRegMapping;
+
+	/* System physical address of device registers */
+	IMG_CPU_PHYADDR			sRegsPhysBase;
+	/* Register region size in bytes */
+	IMG_UINT32				ui32RegSize;
+
+	PVRSRV_STUB_PBDESC		*psStubPBDescListKM;
+
+	/* Firmware memory context info */
+	DEVMEM_CONTEXT			*psKernelDevmemCtx;
+	DEVMEM_HEAP				*psFirmwareMainHeap;
+	DEVMEM_HEAP				*psFirmwareConfigHeap;
+	MMU_CONTEXT				*psKernelMMUCtx;
+
+	void					*pvDeviceMemoryHeap;
+
+	/* Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBCtlMemDesc;      /*!< memdesc for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtl;             /*!< kernel mapping for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtlLocal;        /*!< cpu local copy of Kernel CCB control */
+	DEVMEM_MEMDESC			*psKernelCCBMemDesc;         /*!< memdesc for Kernel CCB */
+	IMG_UINT8				*psKernelCCB;                /*!< kernel mapping for Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBRtnSlotsMemDesc; /*!< Return slot array for Kernel CCB commands */
+	IMG_UINT32				*pui32KernelCCBRtnSlots;     /*!< kernel mapping for return slot array */
+
+	/* Firmware CCB */
+	DEVMEM_MEMDESC			*psFirmwareCCBCtlMemDesc;   /*!< memdesc for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtl;          /*!< kernel mapping for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtlLocal;     /*!< cpu local copy of Firmware CCB control */
+	DEVMEM_MEMDESC			*psFirmwareCCBMemDesc;      /*!< memdesc for Firmware CCB */
+	IMG_UINT8				*psFirmwareCCB;             /*!< kernel mapping for Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBCtlMemDesc;   /*!< memdesc for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtl;          /*!< kernel mapping for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtlLocal;     /*!< cpu local copy of Workload Estimation Firmware CCB control */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBMemDesc;      /*!< memdesc for Workload Estimation Firmware CCB */
+	IMG_UINT8				*psWorkEstFirmwareCCB;             /*!< kernel mapping for Workload Estimation Firmware CCB */
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	/* Counter dumping */
+	DEVMEM_MEMDESC			*psCounterBufferMemDesc;      /*!< mem desc for counter dumping buffer */
+	POS_LOCK				hCounterDumpingLock;          /*!< Lock for guarding access to counter dumping buffer */
+#endif
+
+	PVRSRV_MEMALLOCFLAGS_T  uiFWPoisonOnFreeFlag;           /*!< Flag for poisoning FW allocations when freed */
+
+	IMG_BOOL				bIgnoreHWReportedBVNC;			/*!< Ignore BVNC reported by HW */
+
+    /* multicore configuration information */
+    IMG_UINT32              ui32MultiCoreNumCores;      /* total cores primary + secondaries. 0 for non-multi core */
+    IMG_UINT32              ui32MultiCorePrimaryId;     /* primary core id for this device */
+    IMG_UINT64             *pui64MultiCoreCapabilities; /* capabilities for each core */
+
+	/*
+		if we don't preallocate the pagetables we must
+		insert newly allocated page tables dynamically
+	*/
+	void					*pvMMUContextList;
+
+	IMG_UINT32				ui32ClkGateStatusReg;
+	IMG_UINT32				ui32ClkGateStatusMask;
+
+	DEVMEM_MEMDESC			*psRGXFWCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCodeDevVAddrBase;
+	IMG_UINT32			ui32FWCodeSizeInBytes;
+	DEVMEM_MEMDESC			*psRGXFWDataMemDesc;
+	IMG_DEV_VIRTADDR		sFWDataDevVAddrBase;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	RGX_MIPS_ADDRESS_TRAMPOLINE	*psTrampoline;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWCorememCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememCodeDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememCodeFWAddr;
+	IMG_UINT32			ui32FWCorememCodeSizeInBytes;
+
+	DEVMEM_MEMDESC			*psRGXFWIfCorememDataStoreMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememDataStoreDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememDataStoreFWAddr;
+
+	DEVMEM_MEMDESC			*psRGXFWAlignChecksMemDesc;
+
+#if defined(PDUMP)
+	DEVMEM_MEMDESC			*psRGXFWSigTAChecksMemDesc;
+	IMG_UINT32				ui32SigTAChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSig3DChecksMemDesc;
+	IMG_UINT32				ui32Sig3DChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSigCDMChecksMemDesc;
+	IMG_UINT32				ui32SigCDMChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSigTDMChecksMemDesc;
+	IMG_UINT32				ui32SigTDMChecksSize;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DEVMEM_MEMDESC			*psRGXFWSigRDMChecksMemDesc;
+	IMG_UINT32				ui32SigRDMChecksSize;
+#endif
+
+
+	IMG_BOOL				bDumpedKCCBCtlAlready;
+
+	POS_SPINLOCK			hSyncCheckpointSignalSpinLock;						/*!< Guards data shared between an atomic & sleepable-context */
+#endif
+
+	POS_LOCK				hRGXFWIfBufInitLock;								/*!< trace buffer lock for initialisation phase */
+
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufCtlMemDesc;						/*!< memdesc of trace buffer control structure */
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufferMemDesc[RGXFW_THREAD_NUM];		/*!< memdesc of actual FW trace (log) buffer(s) */
+	IMG_PUINT32				apui32TraceBuffer[RGXFW_THREAD_NUM];				/*!< Trace buffer address (Host address), to be used by host when reading from trace buffer */
+	IMG_UINT32				ui32TraceBufSizeInDWords;							/*!< CPU local copy of FW Trace buffer size in dwords */
+	DEVMEM_MEMDESC			*psRGXFWIfPowMonBufferMemDesc;						/*!< memdesc of FW power monitoring data */
+	RGXFWIF_TRACEBUF		*psRGXFWIfTraceBufCtl;								/*!< structure containing trace control data and actual trace buffer */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwSysDataMemDesc;							/*!< memdesc of the firmware-shared system data structure */
+	RGXFWIF_SYSDATA			*psRGXFWIfFwSysData;								/*!< structure containing km-firmware shared system data */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwOsDataMemDesc;							/*!< memdesc of the firmware-shared os structure */
+	RGXFWIF_OSDATA			*psRGXFWIfFwOsData;									/*!< structure containing km-firmware shared os data */
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	DEVMEM_MEMDESC			*psRGXFWIfTBIBufferMemDesc;							/*!< memdesc of actual FW TBI buffer */
+	RGXFWIF_DEV_VIRTADDR	sRGXFWIfTBIBuffer;									/*!< TBI buffer data */
+	IMG_UINT32				ui32FWIfTBIBufferSize;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWRInfoBufCtlMemDesc;
+	RGXFWIF_HWRINFOBUF		*psRGXFWIfHWRInfoBufCtl;
+	IMG_UINT32				ui32ClockSource;
+	IMG_UINT32				ui32LastClockSource;
+
+	DEVMEM_MEMDESC			*psRGXFWIfGpuUtilFWCtlMemDesc;
+	RGXFWIF_GPU_UTIL_FW     *psRGXFWIfGpuUtilFW;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfBufMemDesc;
+	IMG_BYTE				*psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32RGXFWIfHWPerfBufSize; /* in bytes */
+	IMG_UINT32				ui32RGXL2HWPerfBufSize;   /* in bytes */
+
+	DEVMEM_MEMDESC			*psRGXFWIfRegCfgMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfCountersMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfConnectionCtlMemDesc;
+	RGXFWIF_CONNECTION_CTL	*psRGXFWIfConnectionCtl;
+
+	DEVMEM_MEMDESC			*psRGXFWHeapGuardPageReserveMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfSysInitMemDesc;
+	RGXFWIF_SYSINIT			*psRGXFWIfSysInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfOsInitMemDesc;
+	RGXFWIF_OSINIT			*psRGXFWIfOsInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfRuntimeCfgMemDesc;
+	RGXFWIF_RUNTIME_CFG		*psRGXFWIfRuntimeCfg;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	DEVMEM_MEMDESC			*psRGXFWIfActiveContextBufDesc;
+	RGXFWIF_ACTIVE_CONTEXT_BUF_DATA		*psRGXFWIfActiveContextBuf;
+#endif
+
+	/* Premapped firmware memory context info */
+	DEVMEM_HEAP				*psPremappedFwRawHeap[RGX_NUM_DRIVERS_SUPPORTED];
+	DEVMEM_MEMDESC			*psPremappedFwRawMemDesc[RGX_NUM_DRIVERS_SUPPORTED];
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Array to store data needed for workload estimation when a workload
+	   has finished and its cycle time is returned to the host.	 */
+	WORKEST_RETURN_DATA     asReturnData[RETURN_DATA_ARRAY_SIZE];
+	IMG_UINT32              ui32ReturnDataWO;
+	POS_LOCK                hWorkEstLock;
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/**
+	 * Host memdesc and pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory on changing the core clock rate over
+	 * GPIO.
+	 * Note: Shared memory needs atomic access from Host driver and firmware,
+	 * hence size should not be greater than memory transaction granularity.
+	 * Currently it is chosen to be 32 bits.
+	 */
+	DEVMEM_MEMDESC			*psRGXFWIFCoreClkRateMemDesc;
+	volatile IMG_UINT32		*pui32RGXFWIFCoreClkRate;
+	/**
+	 * Last sampled core clk rate.
+	 */
+	volatile IMG_UINT32		ui32CoreClkRateSnapshot;
+#endif
+
+	/*
+	   HWPerf data for the RGX device
+	 */
+
+	POS_LOCK    hHWPerfLock;  /*! Critical section lock that protects HWPerf code
+	                           *  from multiple thread duplicate init/deinit
+	                           *  and loss/freeing of FW & Host resources while in
+	                           *  use in another thread e.g. MSIR. */
+
+	IMG_UINT64  ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_LAST];         /*! Event filter for FW events (settable by AppHint) */
+	IMG_HANDLE  hHWPerfStream[RGX_HWPERF_L2_STREAM_LAST];            /*! TL Stream buffer (L2) for firmware event stream */
+	IMG_UINT32  ui32L2BufMaxPacketSize[RGX_HWPERF_L2_STREAM_LAST];   /*! Max allowed packet size in FW HWPerf TL (L2) buffer */
+	IMG_BOOL    bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_LAST]; /*! Flag to indicate if copying HWPerf data is suspended */
+	IMG_UINT64  ui64HWPerfFwFilter;                                  /*! Event filter for FW events created from OR-ing ui64HWPerfFilter values. */
+	IMG_UINT32  uiHWPerfStreamCount;                                 /*! Value indicating if any of the HWPerf streams has been created */
+
+	IMG_UINT32  ui32HWPerfHostFilter;      /*! Event filter for HWPerfHost stream (settable by AppHint) */
+	POS_LOCK    hLockHWPerfHostStream;     /*! Lock guarding access to HWPerfHost stream from multiple threads */
+	IMG_HANDLE  hHWPerfHostStream;         /*! TL Stream buffer for host only event stream */
+	IMG_UINT32  ui32HWPerfHostBufSize;     /*! Host side buffer size in bytes */
+	IMG_UINT32  ui32HWPerfHostLastOrdinal; /*! Ordinal of the last packet emitted in HWPerfHost TL stream.
+	                                        *  Guarded by hLockHWPerfHostStream */
+	IMG_UINT32  ui32HWPerfHostNextOrdinal; /*! Ordinal number for HWPerfHost events. Guarded by hHWPerfHostSpinLock */
+	IMG_UINT8   *pui8DeferredEvents;       /*! List of HWPerfHost events yet to be emitted in the TL stream.
+	                                        *  Events generated from atomic context are deferred "emitted"
+											*  as the "emission" code can sleep */
+	IMG_UINT16  ui16DEReadIdx;             /*! Read index in the above deferred events buffer */
+	IMG_UINT16  ui16DEWriteIdx;            /*! Write index in the above deferred events buffer */
+	void        *pvHostHWPerfMISR;         /*! MISR to emit pending/deferred events in HWPerfHost TL stream */
+	POS_SPINLOCK hHWPerfHostSpinLock;      /*! Guards data shared between an atomic & sleepable-context */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32  ui32DEHighWatermark;       /*! High watermark of deferred events buffer usage. Protected by
+	                                        *! hHWPerfHostSpinLock */
+	/* Max number of times DeferredEmission waited for an atomic-context to "finish" packet write */
+	IMG_UINT32  ui32WaitForAtomicCtxPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an atomic-context packet loss (due to too long wait for "write" finish) */
+	IMG_BOOL    bWarnedAtomicCtxPktLost;
+	/* Max number of times DeferredEmission scheduled-out to give a chance to the right-ordinal packet to be emitted */
+	IMG_UINT32  ui32WaitForRightOrdPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an packet loss (due to too long wait for right ordinal to emit) */
+	IMG_BOOL    bWarnedPktOrdinalBroke;
+#endif
+
+	void        *pvGpuFtraceData;
+
+	/* Poll data for detecting firmware fatal errors */
+	IMG_UINT32				aui32CrLastPollCount[RGXFW_THREAD_NUM];
+	IMG_UINT32				ui32KCCBCmdsExecutedLastTime;
+	IMG_BOOL				bKCCBCmdsWaitingLastTime;
+	IMG_UINT32				ui32GEOTimeoutsLastTime;
+	IMG_UINT32				ui32InterruptCountLastTime;
+	IMG_UINT32				ui32MissingInterruptsLastTime;
+
+	/* Client stall detection */
+	IMG_UINT32				ui32StalledClientMask;
+
+	IMG_BOOL				bWorkEstEnabled;
+	IMG_BOOL				bPDVFSEnabled;
+
+	void					*pvLISRData;
+	void					*pvMISRData;
+	void					*pvAPMISRData;
+	RGX_ACTIVEPM_CONF		eActivePMConf;
+
+	volatile IMG_UINT32		aui32SampleIRQCount[RGXFW_THREAD_NUM];
+
+	DEVMEM_MEMDESC			*psRGXFaultAddressMemDesc;
+
+	DEVMEM_MEMDESC			*psSLC3FenceMemDesc;
+
+	/* If we do 10 deferred memory allocations per second, then the ID would wrap around after 13 years */
+	IMG_UINT32				ui32ZSBufferCurrID;	/*!< ID assigned to the next deferred devmem allocation */
+	IMG_UINT32				ui32FreelistCurrID;	/*!< ID assigned to the next freelist */
+
+	POS_LOCK				hLockZSBuffer;		/*!< Lock to protect simultaneous access to ZSBuffers */
+	DLLIST_NODE				sZSBufferHead;		/*!< List of on-demand ZSBuffers */
+	POS_LOCK				hLockFreeList;		/*!< Lock to protect simultaneous access to Freelists */
+	DLLIST_NODE				sFreeListHead;		/*!< List of growable Freelists */
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+	PVRSRV_CLIENT_SYNC_PRIM	*psPowSyncPrim;
+
+	IMG_UINT32				ui32ActivePMReqOk;
+	IMG_UINT32				ui32ActivePMReqDenied;
+	IMG_UINT32				ui32ActivePMReqNonIdle;
+	IMG_UINT32				ui32ActivePMReqRetry;
+	IMG_UINT32				ui32ActivePMReqTotal;
+
+	IMG_HANDLE				hProcessQueuesMISR;
+
+	IMG_UINT32				ui32DeviceFlags;		/*!< Flags to track general device state */
+
+	/* GPU DVFS Table */
+	RGX_GPU_DVFS_TABLE		*psGpuDVFSTable;
+
+	/* Pointer to function returning the GPU utilisation statistics since the last
+	 * time the function was called. Supports different users at the same time.
+	 *
+	 * psReturnStats [out]: GPU utilisation statistics (active high/active low/idle/blocked)
+	 *                      in microseconds since the last time the function was called
+	 *                      by a specific user (identified by hGpuUtilUser)
+	 *
+	 * Returns PVRSRV_OK in case the call completed without errors,
+	 * some other value otherwise.
+	 */
+	PVRSRV_ERROR (*pfnGetGpuUtilStats) (PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                    IMG_HANDLE hGpuUtilUser,
+	                                    RGXFWIF_GPU_UTIL_STATS *psReturnStats);
+
+	/* Pointer to function that checks if the physical GPU IRQ
+	 * line has been asserted and clears it if so */
+	IMG_BOOL (*pfnRGXAckIrq) (struct _PVRSRV_RGXDEV_INFO_ *psDevInfo);
+
+	POS_LOCK				hGPUUtilLock;
+
+	/* Register configuration */
+	RGX_REG_CONFIG			sRegConfig;
+
+	IMG_BOOL				bRGXPowered;
+	DLLIST_NODE				sMemoryContextList;
+
+	POSWR_LOCK				hRenderCtxListLock;
+	POSWR_LOCK				hComputeCtxListLock;
+	POSWR_LOCK				hTransferCtxListLock;
+	POSWR_LOCK				hTDMCtxListLock;
+	POSWR_LOCK				hMemoryCtxListLock;
+	POSWR_LOCK				hKickSyncCtxListLock;
+
+	/* Linked list of deferred KCCB commands due to a full KCCB.
+	 * Access to members sKCCBDeferredCommandsListHead and ui32KCCBDeferredCommandsCount
+	 * are protected by the hLockKCCBDeferredCommandsList spin lock. */
+	POS_SPINLOCK			hLockKCCBDeferredCommandsList; /*!< Protects deferred KCCB commands list */
+	DLLIST_NODE				sKCCBDeferredCommandsListHead;
+	IMG_UINT32				ui32KCCBDeferredCommandsCount; /*!< No of commands in the deferred list */
+
+	/* Linked lists of contexts on this device */
+	DLLIST_NODE				sRenderCtxtListHead;
+	DLLIST_NODE				sComputeCtxtListHead;
+	DLLIST_NODE				sTDMCtxtListHead;
+	DLLIST_NODE				sKickSyncCtxtListHead;
+
+	DLLIST_NODE				sCommonCtxtListHead;
+	POSWR_LOCK				hCommonCtxtListLock;
+	IMG_UINT32				ui32CommonCtxtCurrentID;	/*!< ID assigned to the next common context */
+
+	POS_LOCK				hDebugFaultInfoLock;	/*!< Lock to protect the debug fault info list */
+	POS_LOCK				hMMUCtxUnregLock;		/*!< Lock to protect list of unregistered MMU contexts */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	POS_LOCK				hNMILock; /*!< Lock to protect NMI operations */
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32				ui32AvailablePowUnitsMask;
+	IMG_UINT32				ui32AvailableRACMask;
+#endif
+
+	RGX_LAYER_PARAMS		sLayerParams;
+
+	RGXFWIF_DM				eBPDM;					/*!< Current breakpoint data master */
+	IMG_BOOL				bBPSet;					/*!< A Breakpoint has been set */
+	POS_LOCK				hBPLock;				/*!< Lock for break point operations */
+
+	IMG_UINT32				ui32CoherencyTestsDone;
+
+	ATOMIC_T				iCCBSubmissionOrdinal; /* Rolling count used to indicate CCB submission order (all CCBs) */
+	POS_LOCK				hCCBRecoveryLock;      /* Lock to protect pvEarliestStalledClientCCB and ui32OldestSubmissionOrdinal variables */
+	void					*pvEarliestStalledClientCCB; /* Will point to cCCB command to unblock in the event of a stall */
+	IMG_UINT32				ui32OldestSubmissionOrdinal; /* Earliest submission ordinal of CCB entry found so far */
+	IMG_UINT32				ui32SLRHoldoffCounter;   /* Decremented each time health check is called until zero. SLR only happen when zero. */
+
+	POS_LOCK				hCCBStallCheckLock; /* Lock used to guard against multiple threads simultaneously checking for stalled CCBs */
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Firmware gcov buffer */
+	DEVMEM_MEMDESC			*psFirmwareGcovBufferMemDesc;      /*!< mem desc for Firmware gcov dumping buffer */
+	IMG_UINT32				ui32FirmwareGcovSize;
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* Value to store for each page size range config register in MMU4 */
+	IMG_UINT64				aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES];
+#endif
+
+
+
+	IMG_HANDLE				hTQCLISharedMem;		/*!< TQ Client Shared Mem PMR */
+	IMG_HANDLE				hTQUSCSharedMem;		/*!< TQ USC Shared Mem PMR */
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	DEVMEM_MEMDESC			*psRGXFWIfSecureBufMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfNonSecureBufMemDesc;
+#endif
+
+	/* Timer Queries */
+	IMG_UINT32				ui32ActiveQueryId;		/*!< id of the active line */
+	IMG_BOOL				bSaveStart;				/*!< save the start time of the next kick on the device*/
+	IMG_BOOL				bSaveEnd;				/*!< save the end time of the next kick on the device*/
+
+	DEVMEM_MEMDESC			*psStartTimeMemDesc;    /*!< memdesc for Start Times */
+	IMG_UINT64				*pui64StartTimeById;    /*!< CPU mapping of the above */
+
+	DEVMEM_MEMDESC			*psEndTimeMemDesc;      /*!< memdesc for End Timer */
+	IMG_UINT64				*pui64EndTimeById;      /*!< CPU mapping of the above */
+
+	IMG_UINT32				aui32ScheduledOnId[RGX_MAX_TIMER_QUERIES];	/*!< kicks Scheduled on QueryId */
+	DEVMEM_MEMDESC			*psCompletedMemDesc;	/*!< kicks Completed on QueryId */
+	IMG_UINT32				*pui32CompletedById;	/*!< CPU mapping of the above */
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	POS_LOCK				hTimerQueryLock;		/*!< lock to protect simultaneous access to timer query members */
+#endif
+
+	PVRSRV_RGXDEV_ERROR_COUNTS sErrorCounts;		/*!< struct containing device error counts */
+
+	IMG_UINT32				ui32HostSafetyEventMask;/*!< mask of the safety events handled by the driver */
+
+	RGX_CONTEXT_RESET_REASON	eLastDeviceError;	/*!< device error reported to client */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	PMR						*psGenHeapSecMem;		/*!< An allocation of secure memory mapped to
+													  the general devmem heap. The allocation is
+													  created and mapped at driver init. It's used for
+													  various purposes. See rgx_fwif_km.h for all use cases. */
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	DEVMEM_MEMDESC			*psRGXFWScratchBufMemDesc;
+#endif
+
+	RGX_FWT_LOGTYPE			eDebugDumpFWTLogType;
+
+	RGX_FW_INFO_HEADER      sFWInfoHeader;
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	IMG_UINT32              ui32TFBCLossyGroup;     /*!< TFBCCompressionControlGroup
+													  setting for those cores which support
+													  this feature. */
+#endif
+	RGXFWIF_GPU_UTIL_STATS	sGpuUtilStats;          /*!< GPU usage statistics */
+	POS_LOCK				hGpuUtilStatsLock;
+
+
+} PVRSRV_RGXDEV_INFO;
+
+
+
+typedef struct _RGX_TIMING_INFORMATION_
+{
+	/*! GPU default core clock speed in Hz */
+	IMG_UINT32			ui32CoreClockSpeed;
+
+	/*! Default SOC clock speed in Hz */
+	IMG_UINT32			ui32SOCClockSpeed;
+
+	/*! Active Power Management: GPU actively requests the host driver to be powered off */
+	IMG_BOOL			bEnableActivePM;
+
+	/*! Enable the GPU to power off internal Power Islands independently from the host driver */
+	IMG_BOOL			bEnableRDPowIsland;
+
+	/*! Active Power Management: Delay between the GPU idle and the request to the host */
+	IMG_UINT32			ui32ActivePMLatencyms;
+
+} RGX_TIMING_INFORMATION;
+
+typedef struct _RGX_DATA_
+{
+	/*! Timing information */
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo;
+} RGX_DATA;
+
+typedef enum _RGX_QUERY_TIMESTAMP_TYPE_
+{
+	RGX_QUERY_HOST_TIMESTAMP,
+	RGX_QUERY_DEVICE_TIMESTAMP,
+} RGX_QUERY_TIMESTAMP_TYPE;
+
+
+/*
+	RGX PDUMP register bank name (prefix)
+*/
+#define RGX_PDUMPREG_NAME		"RGXREG"
+
+#endif /* RGXDEVICE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxfwutils.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxfwutils.c
new file mode 100644
index 000000000000..25ead9588808
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxfwutils.c
@@ -0,0 +1,6703 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rogue firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Rogue firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+#include "cache_km.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxfwriscv.h"
+#include "rgx_options.h"
+#include "rgx_fwif_alignchecks.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_pdump_panics.h"
+#include "fwtrace_string.h"
+#include "rgxheapconfig.h"
+#include "pvrsrv.h"
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "rgxccb.h"
+#include "rgxcompute.h"
+#include "rgxtdmtransfer.h"
+#include "rgxpower.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+#include "rgxmem.h"
+#include "rgxmmudefs_km.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#endif
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxutils.h"
+#include "rgxtimecorr.h"
+#include "rgxfwimageutils.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_external.h"
+#include "tlstream.h"
+#include "devicemem_server_utils.h"
+#include "htbserver.h"
+#include "info_page.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#ifdef __linux__
+#include <linux/kernel.h>	/* sprintf */
+#else
+#include <stdio.h>
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF)
+#include "pvr_dvfs_common.h"
+#endif
+
+
+#include "vz_vmm_pvz.h"
+#include "rgx_heaps.h"
+
+/*!
+ ******************************************************************************
+ * HWPERF
+ *****************************************************************************/
+/* Size of the Firmware L1 HWPERF buffer in bytes (2MB). Accessed by the
+ * Firmware and host driver. */
+#define RGXFW_HWPERF_L1_SIZE_MIN        (16U)
+#define RGXFW_HWPERF_L1_SIZE_DEFAULT    PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB
+#define RGXFW_HWPERF_L1_SIZE_MAX        (12288U)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#if defined(DEBUG)
+/* Catch the use of auto-increment when meta_registers_unpacked_accesses feature is
+ * present in case we ever use it. No WA exists so it must not be used */
+#define CHECK_HWBRN_68777(v) \
+	do { \
+		PVR_ASSERT(((v) & RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN) == 0); \
+	} while (0)
+#else
+#define CHECK_HWBRN_68777(v)
+#endif
+#endif
+
+/* Firmware CCB length */
+#if defined(NO_HARDWARE) && defined(PDUMP)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (10)
+#elif defined(SUPPORT_PDVFS) || defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (8)
+#else
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (5)
+#endif
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS] = {IRQ_COUNTER_STORAGE_REGS};
+#endif
+
+/* Workload Estimation Firmware CCB length */
+#define RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2   (7)
+
+/* Size of memory buffer for firmware gcov data
+ * The actual data size is several hundred kilobytes. The buffer is an order of magnitude larger. */
+#define RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE (4*1024*1024)
+
+typedef struct
+{
+	RGXFWIF_KCCB_CMD        sKCCBcmd;
+	DLLIST_NODE             sListNode;
+	PDUMP_FLAGS_T           uiPDumpFlags;
+	PVRSRV_RGXDEV_INFO      *psDevInfo;
+} RGX_DEFERRED_KCCB_CMD;
+
+typedef struct
+{
+	IMG_INT32  i32Priority;
+	IMG_UINT32 ui32IsolationGroups;
+	IMG_UINT32 ui32TSPercentage;
+}RGX_QOS_DEFAULTS;
+
+#define RGX_QOS_DEFAULTS_INIT(osid) \
+	{RGX_DRIVERID_##osid##_DEFAULT_PRIORITY,\
+	RGX_DRIVERID_##osid##_DEFAULT_ISOLATION_GROUP,\
+	RGX_DRIVERID_##osid##_DEFAULT_TIME_SLICE}
+
+#if defined(PDUMP)
+/* ensure PIDs are 32-bit because a 32-bit PDump load is generated for the
+ * PID filter example entries
+ */
+static_assert(sizeof(IMG_PID) == sizeof(IMG_UINT32),
+		"FW PID filtering assumes the IMG_PID type is 32-bits wide as it "
+		"generates WRW commands for loading the PID values");
+#endif
+
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+static_assert(RGX_DRIVER_DEFAULT_TIME_SLICES_SUM <= PVRSRV_VZ_TIME_SLICE_MAX, "Invalid driverid time slice aggregate");
+#endif
+
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo);
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+static PVRSRV_ERROR _AllocateSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo, RGXFWIF_SYSINIT* psFwSysInit)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC** ppsSLC3FenceMemDesc = &psDevInfo->psSLC3FenceMemDesc;
+	IMG_UINT32 ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(
+			RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS));
+
+	PVR_DPF_ENTERED;
+
+	eError = DevmemAllocate(psDevInfo->psFirmwareMainHeap,
+			1,
+			ui32CacheLineSize,
+			PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+			PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+			PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+			"FwSLC3FenceWA",
+			ppsSLC3FenceMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	/* We need to map it so the heap for this allocation is set */
+	eError = DevmemMapToDevice(*ppsSLC3FenceMemDesc,
+							   psDevInfo->psFirmwareMainHeap,
+							   &psFwSysInit->sSLC3FenceDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsSLC3FenceMemDesc);
+		*ppsSLC3FenceMemDesc = NULL;
+	}
+
+	PVR_DPF_RETURN_RC1(eError, *ppsSLC3FenceMemDesc);
+}
+
+static void _FreeSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo)
+{
+	DEVMEM_MEMDESC* psSLC3FenceMemDesc = psDevInfo->psSLC3FenceMemDesc;
+
+	if (psSLC3FenceMemDesc)
+	{
+		DevmemReleaseDevVirtAddr(psSLC3FenceMemDesc);
+		DevmemFree(psSLC3FenceMemDesc);
+	}
+}
+#endif
+
+static void __MTSScheduleWrite(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Value)
+{
+	/* This should *NOT* happen. Try to trace what caused this and avoid a NPE
+	 * with the Write/Read at the foot of the function.
+	 */
+	PVR_ASSERT((psDevInfo != NULL));
+	if (psDevInfo == NULL)
+	{
+		return;
+	}
+
+	/* Kick MTS to wake firmware. */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE, ui32Value);
+
+	/* Uncached device/IO mapping will ensure MTS kick leaves CPU, read back
+	 * will ensure it reaches the regbank via inter-connects (AXI, PCIe etc)
+	 * before continuing.
+	 */
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          ppsMemDesc      pointer to the allocation's memory descriptor
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO*  psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL bClearByMemset;
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiAllocFlags))
+	{
+		/* Under AutoVz the ZERO_ON_ALLOC flag is avoided as it causes the memory to
+		 * be allocated from a different PMR than an allocation without the flag.
+		 * When the content of an allocation needs to be recovered from physical memory
+		 * on a later driver reboot, the memory then cannot be zeroed but the allocation
+		 * addresses must still match.
+		 * If the memory requires clearing, perform a memset after the allocation. */
+		uiAllocFlags &= ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+		bClearByMemset = IMG_TRUE;
+	}
+	else
+	{
+		bClearByMemset = IMG_FALSE;
+	}
+#endif
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate %s", pszName);
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui32Size,
+							  uiAllocFlags,
+							  pszName,
+							  ppsMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate %u bytes for %s (%u)",
+				 __func__,
+				 ui32Size,
+				 pszName,
+				 eError));
+		goto fail_alloc;
+	}
+
+	if (psFwPtr)
+	{
+		eError = RGXSetFirmwareAddress(psFwPtr, *ppsMemDesc, 0, ui32DevVAFlags);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire firmware virtual address for %s (%u)",
+					 __func__,
+					 pszName,
+					 eError));
+			goto fail_fwaddr;
+		}
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	if ((bClearByMemset) || (ppvCpuPtr))
+#else
+	if (ppvCpuPtr)
+#endif
+	{
+		void *pvTempCpuPtr;
+
+		eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, &pvTempCpuPtr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to acquire CPU virtual address for %s (%u)",
+					__func__,
+					 pszName,
+					eError));
+			goto fail_cpuva;
+		}
+
+#if defined(SUPPORT_AUTOVZ)
+		if (bClearByMemset)
+		{
+			if (PVRSRV_CHECK_CPU_WRITE_COMBINE(uiAllocFlags))
+			{
+				OSCachedMemSetWMB(pvTempCpuPtr, 0, ui32Size);
+			}
+			else
+			{
+				OSDeviceMemSet(pvTempCpuPtr, 0, ui32Size);
+			}
+		}
+		if (ppvCpuPtr)
+#endif
+		{
+			*ppvCpuPtr = pvTempCpuPtr;
+		}
+#if defined(SUPPORT_AUTOVZ)
+		else
+		{
+			DevmemReleaseCpuVirtAddr(*ppsMemDesc);
+			pvTempCpuPtr = NULL;
+		}
+#endif
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s set up at Fw VA 0x%x and CPU VA 0x%p with alloc flags 0x%" IMG_UINT64_FMTSPECX,
+			 __func__, pszName,
+			 (psFwPtr)   ? (psFwPtr->ui32Addr) : (0),
+			 (ppvCpuPtr) ? (*ppvCpuPtr)        : (NULL),
+			 uiAllocFlags));
+
+	return eError;
+
+fail_cpuva:
+	if (psFwPtr)
+	{
+		RGXUnsetFirmwareAddress(*ppsMemDesc);
+	}
+fail_fwaddr:
+	DevmemFree(*ppsMemDesc);
+fail_alloc:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHwPerfBufferSize
+
+@Description    Computes the effective size of the HW Perf Buffer
+@Input          ui32HWPerfFWBufSizeKB       Device Info struct
+@Return         HwPerfBufferSize
+*/ /**************************************************************************/
+static IMG_UINT32 GetHwPerfBufferSize(IMG_UINT32 ui32HWPerfFWBufSizeKB)
+{
+	IMG_UINT32 HwPerfBufferSize;
+
+	/* HWPerf: Determine the size of the FW buffer */
+	if (ui32HWPerfFWBufSizeKB == 0 ||
+			ui32HWPerfFWBufSizeKB == RGXFW_HWPERF_L1_SIZE_DEFAULT)
+	{
+		/* Under pvrsrvctl 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. Set it to the default
+		 * size, no logging.
+		 */
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_DEFAULT<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MAX))
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too big, using maximum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MAX));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MAX<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MIN))
+	{
+		/* Size specified as in AppHint HWPerfFWBufSizeInKB */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Using HWPerf FW buffer size of %u KB",
+				__func__,
+				ui32HWPerfFWBufSizeKB));
+		HwPerfBufferSize = ui32HWPerfFWBufSizeKB<<10;
+	}
+	else
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too small, using minimum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MIN));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MIN<<10;
+	}
+
+	return HwPerfBufferSize;
+}
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupSignatureChecks
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupSignatureChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
+                                              DEVMEM_MEMDESC**    ppsSigChecksMemDesc,
+                                              IMG_UINT32          ui32SigChecksBufSize,
+                                              RGXFWIF_SIGBUF_CTL* psSigBufCtl)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+								  ui32SigChecksBufSize,
+								  "FwSignatureChecks",
+								  ppsSigChecksMemDesc,
+								  &psSigBufCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	DevmemPDumpLoadMem(	*ppsSigChecksMemDesc,
+			0,
+			ui32SigChecksBufSize,
+			PDUMP_FLAGS_CONTINUOUS);
+
+	psSigBufCtl->ui32LeftSizeInRegs = ui32SigChecksBufSize / sizeof(IMG_UINT32);
+fail:
+	return eError;
+}
+#endif
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupFirmwareGcovBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupFirmwareGcovBuffer(PVRSRV_RGXDEV_INFO*			psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32FirmwareGcovBufferSize,
+		RGXFWIF_FIRMWARE_GCOV_CTL*	psFirmwareGcovCtl,
+		const IMG_CHAR*				pszBufferName)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for gcov */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32FirmwareGcovBufferSize,
+								  pszBufferName,
+								  ppsBufferMemDesc,
+								  &psFirmwareGcovCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psFirmwareGcovCtl->ui32Size = ui32FirmwareGcovBufferSize;
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+/*!
+ ******************************************************************************
+ @Function		RGXFWSetupCounterBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+ *****************************************************************************/
+static PVRSRV_ERROR RGXFWSetupCounterBuffer(PVRSRV_RGXDEV_INFO* psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32CounterDataBufferSize,
+		RGXFWIF_COUNTER_DUMP_CTL*	psCounterDumpCtl)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32CounterDataBufferSize,
+								  "FwCounterBuffer",
+								  ppsBufferMemDesc,
+								  &psCounterDumpCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psCounterDumpCtl->ui32SizeInDwords = ui32CounterDataBufferSize >> 2;
+	RGXFwSharedMemCacheOpValue(psCounterDumpCtl->ui32SizeInDwords, FLUSH);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+ @Function      RGXFWSetupAlignChecks
+ @Description   This functions allocates and fills memory needed for the
+                aligns checks of the UM and KM structures shared with the
+                firmware. The format of the data in the memory is as follows:
+                    <number of elements in the KM array>
+                    <array of KM structures' sizes and members' offsets>
+                    <number of elements in the UM array>
+                    <array of UM structures' sizes and members' offsets>
+                The UM array is passed from the user side. Now the firmware is
+                responsible for filling this part of the memory. If that
+                happens the check of the UM structures will be performed
+                by the host driver on client's connect.
+                If the macro is not defined the client driver fills the memory
+                and the firmware checks for the alignment of all structures.
+ @Input			psDeviceNode
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupAlignChecks(PVRSRV_DEVICE_NODE *psDeviceNode,
+								RGXFWIF_DEV_VIRTADDR	*psAlignChecksDevFW)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32			aui32RGXFWAlignChecksKM[] = { RGXFW_ALIGN_CHECKS_INIT_KM };
+	IMG_UINT32			ui32RGXFWAlignChecksTotal;
+	IMG_UINT32*			paui32AlignChecks;
+	PVRSRV_ERROR		eError;
+
+	/* In this case we don't know the number of elements in UM array.
+	 * We have to assume something so we assume RGXFW_ALIGN_CHECKS_UM_MAX.
+	 */
+	ui32RGXFWAlignChecksTotal = sizeof(aui32RGXFWAlignChecksKM)
+	                            + RGXFW_ALIGN_CHECKS_UM_MAX * sizeof(IMG_UINT32)
+	                            + 2 * sizeof(IMG_UINT32);
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32RGXFWAlignChecksTotal,
+								  "FwAlignmentChecks",
+								  &psDevInfo->psRGXFWAlignChecksMemDesc,
+								  psAlignChecksDevFW,
+								  (void**) &paui32AlignChecks,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Copy the values */
+		*paui32AlignChecks++ = ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+		OSCachedMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0],
+		                sizeof(aui32RGXFWAlignChecksKM));
+		paui32AlignChecks += ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+
+		*paui32AlignChecks = 0;
+
+		OSWriteMemoryBarrier(paui32AlignChecks);
+		RGXFwSharedMemCacheOpExec(paui32AlignChecks - (ARRAY_SIZE(aui32RGXFWAlignChecksKM) + 1),
+		                          ui32RGXFWAlignChecksTotal,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+
+
+	DevmemPDumpLoadMem(	psDevInfo->psRGXFWAlignChecksMemDesc,
+						0,
+						ui32RGXFWAlignChecksTotal,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXFWFreeAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo)
+{
+	if (psDevInfo->psRGXFWAlignChecksMemDesc != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWAlignChecksMemDesc);
+		psDevInfo->psRGXFWAlignChecksMemDesc = NULL;
+	}
+}
+
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+						   DEVMEM_MEMDESC		*psSrc,
+						   IMG_UINT32			uiExtraOffset,
+						   IMG_UINT32			ui32Flags)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	psDevVirtAddr;
+	PVRSRV_DEVICE_NODE	*psDeviceNode;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+
+	psDeviceNode = (PVRSRV_DEVICE_NODE *) DevmemGetConnection(psSrc);
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		IMG_UINT32          ui32Offset;
+		IMG_BOOL            bCachedInMETA;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+		IMG_UINT32          uiGPUCacheMode;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in META memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the META cache flags */
+		bCachedInMETA = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		/* Honour the SLC cache flags */
+		eError = DevmemDeviceCacheMode(uiDevFlags, &uiGPUCacheMode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemDeviceCacheMode", failDevCacheMode);
+
+		/*
+		 * Choose Meta virtual address based on Meta and SLC cacheability.
+		 */
+		ui32Offset += RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+		if (bCachedInMETA)
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+		}
+
+		if (PVRSRV_CHECK_GPU_CACHED(uiGPUCacheMode))
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_GOTO_IF_ERROR(eError, failDevVAAcquire);
+
+		ppDest->ui32Addr = (IMG_UINT32)((psDevVirtAddr.uiAddr + uiExtraOffset) & 0xFFFFFFFF);
+	}
+#endif
+	else
+	{
+		IMG_UINT32      ui32Offset;
+		IMG_BOOL        bCachedInRISCV;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in RISCV memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the RISCV cache flags */
+		bCachedInRISCV = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		if (bCachedInRISCV)
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_CACHED_DATA_BASE;
+		}
+		else
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_UNCACHED_DATA_BASE;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+
+	if ((ppDest->ui32Addr & 0x3U) != 0)
+	{
+		IMG_CHAR *pszAnnotation;
+		/* It is expected that the annotation returned by DevmemGetAnnotation() is always valid */
+		DevmemGetAnnotation(psSrc, &pszAnnotation);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: %s @ 0x%x is not aligned to 32 bit",
+				 __func__, pszAnnotation, ppDest->ui32Addr));
+
+		return PVRSRV_ERROR_INVALID_ALIGNMENT;
+	}
+
+	if (ui32Flags & RFW_FWADDR_NOREF_FLAG)
+	{
+		DevmemReleaseDevVirtAddr(psSrc);
+	}
+
+	return PVRSRV_OK;
+
+failDevCacheMode:
+	DevmemReleaseDevVirtAddr(psSrc);
+failDevVAAcquire:
+	return eError;
+}
+
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	sDevVirtAddr;
+
+	eError = DevmemAcquireDevVirtAddr(psSrcMemDesc, &sDevVirtAddr);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psDest->psDevVirtAddr.uiAddr = sDevVirtAddr.uiAddr;
+	psDest->psDevVirtAddr.uiAddr += uiOffset;
+	psDest->pbyFWAddr.ui32Addr = psSrcFWDevVAddr->ui32Addr;
+
+	DevmemReleaseDevVirtAddr(psSrcMemDesc);
+}
+
+
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc)
+{
+	DevmemReleaseDevVirtAddr(psSrc);
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCB
+ @Description	Free the kernel or firmware CCB
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppvCCBCtlLocal
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+******************************************************************************/
+static void RGXFreeCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtl,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+					   DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+					   IMG_UINT8			**ppui8CCB,
+					   DEVMEM_MEMDESC		**ppsCCBMemDesc)
+{
+	if (*ppsCCBMemDesc != NULL)
+	{
+		if (*ppui8CCB != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBMemDesc);
+			*ppui8CCB = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBMemDesc);
+		*ppsCCBMemDesc = NULL;
+	}
+	if (*ppsCCBCtlMemDesc != NULL)
+	{
+		if (*ppsCCBCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBCtlMemDesc);
+			*ppsCCBCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBCtlMemDesc);
+		*ppsCCBCtlMemDesc = NULL;
+	}
+	if (*ppsCCBCtlLocal != NULL)
+	{
+		OSFreeMem(*ppsCCBCtlLocal);
+		*ppsCCBCtlLocal = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCBReturnSlots
+ @Description	Free the kernel CCB's return slot array and associated mappings
+ @Input			psDevInfo              Device Info struct
+ @Input			ppui32CCBRtnSlots      CPU mapping of slot array
+ @Input			ppsCCBRtnSlotsMemDesc  Slot array's device memdesc
+******************************************************************************/
+static void RGXFreeCCBReturnSlots(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32         **ppui32CCBRtnSlots,
+								  DEVMEM_MEMDESC     **ppsCCBRtnSlotsMemDesc)
+{
+	/* Free the return slot array if allocated */
+	if (*ppsCCBRtnSlotsMemDesc != NULL)
+	{
+		/* Before freeing, ensure the CPU mapping as well is released */
+		if (*ppui32CCBRtnSlots != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBRtnSlotsMemDesc);
+			*ppui32CCBRtnSlots = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBRtnSlotsMemDesc);
+		*ppsCCBRtnSlotsMemDesc = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXSetupCCB
+ @Description	Allocate and initialise a circular command buffer
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppui8CCB
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+ @Input			ui32NumCmdsLog2
+ @Input			ui32CmdSize
+ @Input			uiCCBMemAllocFlags
+ @Input			pszName
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								RGXFWIF_CCB_CTL		**ppsCCBCtl,
+								RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+								DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+								IMG_UINT8			**ppui8CCB,
+								DEVMEM_MEMDESC		**ppsCCBMemDesc,
+								PRGXFWIF_CCB_CTL	*psCCBCtlFWAddr,
+								PRGXFWIF_CCB		*psCCBFWAddr,
+								IMG_UINT32			ui32NumCmdsLog2,
+								IMG_UINT32			ui32CmdSize,
+								PVRSRV_MEMALLOCFLAGS_T uiCCBMemAllocFlags,
+								const IMG_CHAR		*pszName)
+{
+	PVRSRV_ERROR		eError;
+	RGXFWIF_CCB_CTL		*psCCBCtl;
+	IMG_UINT32		ui32CCBSize = (1U << ui32NumCmdsLog2);
+	IMG_CHAR		szCCBCtlName[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_INT32		iStrLen;
+
+	/* Append "Control" to the name for the control struct. */
+	iStrLen = OSSNPrintf(szCCBCtlName, sizeof(szCCBCtlName), "%sControl", pszName);
+	PVR_ASSERT(iStrLen < sizeof(szCCBCtlName));
+
+	if (unlikely(iStrLen < 0))
+	{
+		OSStringSafeCopy(szCCBCtlName, "FwCCBControl", DEVMEM_ANNOTATION_MAX_LEN);
+	}
+
+	/* Allocate memory for the CCB control.*/
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_CCB_CTL),
+								  szCCBCtlName,
+								  ppsCCBCtlMemDesc,
+								  psCCBCtlFWAddr,
+								  (void**) ppsCCBCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Allocate memory for the CCB.
+	 * (this will reference further command data in non-shared CCBs)
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  uiCCBMemAllocFlags,
+								  ui32CCBSize * ui32CmdSize,
+								  pszName,
+								  ppsCCBMemDesc,
+								  psCCBFWAddr,
+								  (void**) ppui8CCB,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Initialise the CCB control.
+	 */
+	psCCBCtl = OSAllocZMem(sizeof(*psCCBCtl));
+	PVR_LOG_GOTO_IF_NOMEM(psCCBCtl, eError, fail);
+
+	psCCBCtl->ui32WrapMask = ui32CCBSize - 1;
+
+	OSDeviceMemCopy(*ppsCCBCtl, psCCBCtl, sizeof(*psCCBCtl));
+	RGXFwSharedMemCacheOpPtr(psCCBCtl, FLUSH);
+
+	*ppsCCBCtlLocal = psCCBCtl;
+
+	/* Pdump the CCB control */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Initialise %s", szCCBCtlName);
+	DevmemPDumpLoadMem(*ppsCCBCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCB_CTL),
+					   0);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeCCB(psDevInfo,
+			   ppsCCBCtl,
+			   ppsCCBCtlLocal,
+			   ppsCCBCtlMemDesc,
+			   ppui8CCB,
+			   ppsCCBMemDesc);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXSetupFaultReadRegisterRollback(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+#endif
+	{
+		if (psDevInfo->psRGXFaultAddressMemDesc)
+		{
+			if (DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR) == PVRSRV_OK)
+			{
+				PMRUnlockSysPhysAddresses(psPMR);
+			}
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+			psDevInfo->psRGXFaultAddressMemDesc = NULL;
+		}
+	}
+}
+
+static PVRSRV_ERROR RGXSetupFaultReadRegister(PVRSRV_DEVICE_NODE *psDeviceNode, RGXFWIF_SYSINIT *psFwSysInit)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			*pui32MemoryVirtAddr;
+	IMG_UINT32			i;
+	size_t				ui32PageSize = OSGetPageSize();
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PMR					*psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	/* Allocate page of memory to use for page faults on non-blocking memory transactions.
+	 * Doesn't need to be cleared as it is initialised with the 0xDEADBEEF pattern below. */
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+			ui32PageSize,
+			ui32PageSize,
+			RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS & ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+			"FwExFaultAddress",
+			&psDevInfo->psRGXFaultAddressMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate mem for fault address (%u)",
+		         __func__, eError));
+		goto failFaultAddressDescAlloc;
+	}
+
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc,
+										  (void **)&pui32MemoryVirtAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire mem for fault address (%u)",
+					 __func__, eError));
+			goto failFaultAddressDescAqCpuVirt;
+		}
+
+		/* fill the page with a known pattern when booting the firmware */
+		for (i = 0; i < ui32PageSize/sizeof(IMG_UINT32); i++)
+		{
+			*(pui32MemoryVirtAddr + i) = 0xDEADBEEF;
+		}
+
+		OSWriteMemoryBarrier(pui32MemoryVirtAddr);
+		RGXFwSharedMemCacheOpExec(pui32MemoryVirtAddr, ui32PageSize, PVRSRV_CACHE_OP_FLUSH);
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
+	}
+
+	eError = DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Error getting PMR for fault address (%u)",
+		         __func__, eError));
+
+		goto failFaultAddressDescGetPMR;
+	}
+	else
+	{
+		IMG_BOOL bValid;
+		IMG_UINT32 ui32Log2PageSize = OSGetPageShift();
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error locking physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescLockPhys;
+		}
+
+		eError = PMR_DevPhysAddr(psPMR,ui32Log2PageSize, 1, 0, &(psFwSysInit->sFaultPhysAddr), &bValid, DEVICE_USE);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error getting physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescGetPhys;
+		}
+
+		if (!bValid)
+		{
+			psFwSysInit->sFaultPhysAddr.uiAddr = 0;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed getting physical address for fault address MemDesc - invalid page (0x%" IMG_UINT64_FMTSPECX ")",
+			         __func__, psFwSysInit->sFaultPhysAddr.uiAddr));
+
+			goto failFaultAddressDescGetPhys;
+		}
+	}
+
+	return PVRSRV_OK;
+
+failFaultAddressDescGetPhys:
+	PMRUnlockSysPhysAddresses(psPMR);
+
+failFaultAddressDescLockPhys:
+failFaultAddressDescGetPMR:
+failFaultAddressDescAqCpuVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+
+failFaultAddressDescAlloc:
+
+	return eError;
+}
+
+#if defined(PDUMP)
+/* Replace the DevPhy address with the one Pdump allocates at pdump_player run time */
+static PVRSRV_ERROR RGXPDumpFaultReadRegister(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	PMR *psFWInitPMR, *psFaultAddrPMR;
+	IMG_UINT32 ui32Dstoffset;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	psFWInitPMR = (PMR *)(psDevInfo->psRGXFWIfSysInitMemDesc->psImport->hPMR);
+	ui32Dstoffset = psDevInfo->psRGXFWIfSysInitMemDesc->uiOffset + offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr.uiAddr);
+
+	psFaultAddrPMR = (PMR *)(psDevInfo->psRGXFaultAddressMemDesc->psImport->hPMR);
+
+	eError = PDumpMemLabelToMem64(psFaultAddrPMR,
+				psFWInitPMR,
+				0,
+				ui32Dstoffset,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dump of Fault Page Phys address failed(%u)", __func__, eError));
+	}
+	return eError;
+}
+#endif
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW tbi buffer
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTBIBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTBIBufferMemDesc);
+	psDevInfo->psRGXFWIfTBIBufferMemDesc = NULL;
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = 0;
+	psDevInfo->ui32RGXL2HWPerfBufSize = 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferInitOnDemandResources
+
+@Description    Allocates the firmware TBI buffer required for reading SFs
+		strings and initialize it with SFs.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         i, ui32Len;
+	const IMG_UINT32   ui32FWTBIBufsize = g_ui32SFsCount * sizeof(RGXFW_STID_FMT);
+	RGXFW_STID_FMT     *psFW_SFs = NULL;
+
+	/* Firmware address should not be already set */
+	if (psDevInfo->sRGXFWIfTBIBuffer.ui32Addr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW address for FWTBI is already set. Resetting it with newly allocated one",
+		         __func__));
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS,
+								  ui32FWTBIBufsize,
+								  "FwTBIBuffer",
+								  &psDevInfo->psRGXFWIfTBIBufferMemDesc,
+								  &psDevInfo->sRGXFWIfTBIBuffer,
+								  (void**)&psFW_SFs,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* Copy SFs entries to FW buffer */
+	for (i = 0; i < g_ui32SFsCount; i++)
+	{
+		OSDeviceMemCopy(&psFW_SFs[i].ui32Id, &SFs[i].ui32Id, sizeof(SFs[i].ui32Id));
+		ui32Len = OSStringLength(SFs[i].psName);
+		OSDeviceMemCopy(psFW_SFs[i].sName, SFs[i].psName, MIN(ui32Len, IMG_SF_STRING_MAX_SIZE - 1));
+	}
+
+	/* flush write buffers for psFW_SFs */
+	OSWriteMemoryBarrier(psFW_SFs);
+
+	/* Set size of TBI buffer */
+	psDevInfo->ui32FWIfTBIBufferSize = ui32FWTBIBufsize;
+
+	/* release CPU mapping */
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTBIBufferMemDesc);
+
+	return PVRSRV_OK;
+fail:
+	RGXTBIBufferDeinit(psDevInfo);
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW trace buffer(s)
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTraceBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufferMemDesc[i])
+		{
+			if (psDevInfo->apui32TraceBuffer[i] != NULL)
+			{
+				DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+				psDevInfo->apui32TraceBuffer[i] = NULL;
+			}
+
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+			psDevInfo->psRGXFWIfTraceBufferMemDesc[i] = NULL;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferInitOnDemandResources
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+		info from the firmware.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo,
+												 PVRSRV_MEMALLOCFLAGS_T uiAllocFlags)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         ui32FwThreadNum;
+	IMG_UINT32         ui32DefaultTraceBufSize;
+	IMG_DEVMEM_SIZE_T  uiTraceBufSizeInBytes;
+	void               *pvAppHintState = NULL;
+	IMG_CHAR           pszBufferName[] = "FwTraceBuffer_Thread0";
+
+	/* Check AppHint value for module-param FWTraceBufSizeInDWords */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32DefaultTraceBufSize = RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,
+						 pvAppHintState,
+						 FWTraceBufSizeInDWords,
+						 &ui32DefaultTraceBufSize,
+						 &psDevInfo->ui32TraceBufSizeInDWords);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/* Write tracebuf size once to devmem */
+	psTraceBufCtl->ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (psDevInfo->ui32TraceBufSizeInDWords < RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS ||
+		psDevInfo->ui32TraceBufSizeInDWords > RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Requested trace buffer size (%u) out of its minimum (%u) & maximum (%u) range. Exiting error.",
+				  __func__,
+				  psDevInfo->ui32TraceBufSizeInDWords,
+				  RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS,
+				  RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS));
+		eError = PVRSRV_ERROR_OUT_OF_RANGE;
+		goto exit_error;
+	}
+
+	uiTraceBufSizeInBytes = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+
+	for (ui32FwThreadNum = 0; ui32FwThreadNum < RGXFW_THREAD_NUM; ui32FwThreadNum++)
+	{
+#if !defined(SUPPORT_AUTOVZ)
+		/* Ensure allocation API is only called when not already allocated */
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum] == NULL);
+		/* Firmware address should not be already set */
+		PVR_ASSERT(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer.ui32Addr == 0x0);
+#endif
+
+		/* update the firmware thread number in the Trace Buffer's name */
+		pszBufferName[sizeof(pszBufferName) - 2] += ui32FwThreadNum;
+
+		eError = RGXSetupFwAllocation(psDevInfo,
+									  uiAllocFlags,
+									  uiTraceBufSizeInBytes,
+									  pszBufferName,
+									  &psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum],
+									  &psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer,
+									  (void**)&psDevInfo->apui32TraceBuffer[ui32FwThreadNum],
+									  RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+	}
+
+	return PVRSRV_OK;
+
+fail:
+	RGXTraceBufferDeinit(psDevInfo);
+exit_error:
+	return eError;
+}
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       RGXPDumpLoadFWInitData
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+                info from the firmware.
+
+@Input          psDevInfo RGX device info
+ */ /*************************************************************************/
+static void RGXPDumpLoadFWInitData(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   RGX_INIT_APPHINTS  *psApphints,
+								   IMG_UINT32         ui32HWPerfCountersDataSize)
+{
+	IMG_UINT32 ui32ConfigFlags  = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+	IMG_UINT32 ui32FwOsCfgFlags = psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Dump RGXFW Init data");
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "(to enable rgxfw signatures place the following line after the RTCONF line)");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, asSigBufCtl),
+						   sizeof(RGXFWIF_SIGBUF_CTL)*(RGXFWIF_DM_MAX),
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump initial state of FW runtime configuration");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_RUNTIME_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw hwperfctl structure");
+	DevmemPDumpLoadZeroMem (psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+							0,
+							ui32HWPerfCountersDataSize,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw trace control structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TRACEBUF),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware system data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware OS data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgx TBI buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTBIBufferMemDesc,
+					   0,
+					   psDevInfo->ui32FWIfTBIBufferSize,
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw register configuration buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRegCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_REG_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_USER_REGISTER_CONFIGURATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw system init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw os init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfOsInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* RGXFW Init structure needs to be loaded before we overwrite FaultPhysAddr, else this address patching won't have any effect */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Overwrite FaultPhysAddr of FwSysInit in pdump with actual physical address");
+	RGXPDumpFaultReadRegister(psDevInfo);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "RTCONF: run-time configuration");
+
+	/* Dump the config options so they can be edited. */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW system config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Rand mode:                      0x%08x)", RGXFWIF_INICFG_CTXSWITCH_MODE_RAND);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Soft Reset Enable:              0x%08x)", RGXFWIF_INICFG_CTXSWITCH_SRESET_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable HWPerf:                             0x%08x)", RGXFWIF_INICFG_HWPERF_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Rascal+Dust Power Island:                  0x%08x)", RGXFWIF_INICFG_POW_RASCALDUST);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( FBCDC Version 3.1 Enable:                  0x%08x)", RGXFWIF_INICFG_FBCDC_V3_1_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Check MList:                               0x%08x)", RGXFWIF_INICFG_CHECK_MLIST_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable Auto Clock Gating:                 0x%08x)", RGXFWIF_INICFG_DISABLE_CLKGATING_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+				 "( Try overlapping DM pipelines:              0x%08x)", RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable DM pipeline roadblocks:             0x%08x)", RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#endif
+				 "( Enable register configuration:             0x%08x)", RGXFWIF_INICFG_REGCONFIG_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on TA Out-of-Memory:                0x%08x)", RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable HWPerf counter filter:             0x%08x)", RGXFWIF_INICFG_HWP_DISABLE_FILTER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable Ctx Switch profile mode: 0x%08x (none=b'000, fast=b'001, medium=b'010, slow=b'011, nodelay=b'100))", RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable DM overlap (except TA during SPM): 0x%08x)", RGXFWIF_INICFG_DISABLE_DM_OVERLAP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on HWR trigger (page fault, lockup, overrun or poll failure): 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( ICS Fault Injection:                       0x%08x)", RGXFWIF_INICFG_INJECT_ICS_FAULT);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable IRQ validation:                     0x%08x)", RGXFWIF_INICFG_VALIDATE_IRQ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPU power state mask change Enable:        0x%08x)", RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN);
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Workload Estimation:                0x%08x)", RGXFWIF_INICFG_WORKEST);
+#if defined(SUPPORT_PDVFS)
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Proactive DVFS:                     0x%08x)", RGXFWIF_INICFG_PDVFS);
+#endif /* defined(SUPPORT_PDVFS) */
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( ISP Scheduling Mode (v1=b'01, v2=b'10):    0x%08x)", RGXFWIF_INICFG_ISPSCHEDMODE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Validate SOC & USC timers:                 0x%08x)", RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+							offsetof(RGXFWIF_SYSDATA, ui32ConfigFlags),
+							ui32ConfigFlags,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Extended FW system config options not used.)");
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW OS config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch TDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch GEOM Enable:                    0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch 3D Enable:                      0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch CDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch RDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN);
+#endif
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  2D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  TA Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  3D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch CDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch RDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM);
+#endif
+#if defined(SUPPORT_ICS)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing TDM Enable:            0x%08x)", RGXFWIF_INICFG_OS_ICS_TDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing GEOM Enable:           0x%08x)", RGXFWIF_INICFG_OS_ICS_GEOM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing 3D Enable:             0x%08x)", RGXFWIF_INICFG_OS_ICS_3D_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing CDM Enable:            0x%08x)", RGXFWIF_INICFG_OS_ICS_CDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Idle Cycle Stealing RDM Enable:            0x%08x)", RGXFWIF_INICFG_OS_ICS_RDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable FDTI profile mode:                  0x%08x (none=b'00, long=b'01, medium=b'10, short=b'11))", RGXFWIF_INICFG_FDTI_PROFILE_MASK);
+#endif
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+							  offsetof(RGXFWIF_OSDATA, ui32FwOsConfigFlags),
+							  ui32FwOsCfgFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+#if defined(SUPPORT_ICS)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,"Set the FDTI value in usec here");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+								  offsetof(RGXFWIF_SYSINIT, ui32FDTI),
+								  psDevInfo->psRGXFWIfSysInit->ui32FDTI,
+								  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	{
+		PDUMP_FLAGS_T      ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+		IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+		IMG_BOOL           bRunTimeUpdate = IMG_FALSE;
+		IMG_UINT32         ui32DstOffset = psDevInfo->psRGXFWIfRuntimeCfgMemDesc->uiOffset + offsetof(RGXFWIF_RUNTIME_CFG, ui32PowUnitsState);
+		IMG_CHAR           aszPowUnitsMaskRegVar[] = ":SYSMEM:$1";
+		IMG_CHAR           aszPowUnitsEnable[] = "RUNTIME_POW_UNITS_MASK";
+		PMR                *psPMR = (PMR *)(psDevInfo->psRGXFWIfRuntimeCfgMemDesc->psImport->hPMR);
+
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPIF(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+		                      "Load initial value power units mask in FW runtime configuration");
+		DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  ui32DstOffset,
+								  psDevInfo->psRGXFWIfRuntimeCfg->ui32PowUnitsState,
+								  ui32PDumpFlags);
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPELSE(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+			PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags, "Read initial SPU mask value from HW registers");
+			PDumpRegRead32ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_SPU_ENABLE, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, aszPowUnitsMaskRegVar, ui32AllPowUnitsMask, ui32PDumpFlags);
+			PDumpInternalVarToMemLabel(psPMR, ui32DstOffset, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDUMPFI(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+	}
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Select one or more security tests here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW private data from non-FW contexts: 0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW code from non-FW contexts:         0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from non-secure memory:          0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from secure (non-FW) memory:     0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui32SecurityTestFlags),
+							  psDevInfo->psRGXFWIfSysInit->ui32SecurityTestFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_SECURITY_VALIDATION) */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter type: %X=INCLUDE_ALL_EXCEPT, %X=EXCLUDE_ALL_EXCEPT)",
+				 RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+				 RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, sPIDFilter.eMode),
+							  psDevInfo->psRGXFWIfSysInit->sPIDFilter.eMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter PID/DriverID list (Up to %u entries. Terminate with a zero PID))",
+				 RGXFWIF_PID_FILTER_MAX_NUM_PIDS);
+	{
+		IMG_UINT32 i;
+
+		/* generate a few WRWs in the pdump stream as an example */
+		for (i = 0; i < MIN(RGXFWIF_PID_FILTER_MAX_NUM_PIDS, 8); i++)
+		{
+			/*
+			 * Some compilers cannot cope with the uses of offsetof() below - the specific problem being the use of
+			 * a non-const variable in the expression, which it needs to be const. Typical compiler output is
+			 * "expression must have a constant value".
+			 */
+			const IMG_DEVMEM_OFFSET_T uiPIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].uiPID);
+
+			const IMG_DEVMEM_OFFSET_T uiDriverIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].ui32DriverID);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID and DriverID pair %u)", i);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiPIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(DriverID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiDriverIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+	/*
+	 * Dump the log config so it can be edited.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the log config here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Log Type: TRACE mode using shared memory buffer: 0x00000001)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           TBI mode via external interface or sim support: 0x00000000)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           Note: TBI mode will hang on most hardware devices!)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MAIN Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MAIN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MTS Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MTS);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CLEANUP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CLEANUP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CSW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CSW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( BIF Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_BIF);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_PM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( RTD Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RTD);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SPM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( POW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_POW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWR Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWR);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( DMA Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DMA);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MISC Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MISC);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VZ Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VZ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SAFETY Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SAFETY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VERBOSE Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VERBOSE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CUSTOMER Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CUSTOMER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( DEBUG Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DEBUG);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+							  offsetof(RGXFWIF_TRACEBUF, ui32LogType),
+							  psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Set the HWPerf Filter config here, see \"hwperfbin2jsont -h\"");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui64HWPerfFilter),
+							  psDevInfo->psRGXFWIfSysInit->ui64HWPerfFilter,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Number of registers configurations for types(byte index): pow on(%d), dust change(%d), ta(%d), 3d(%d), cdm(%d), "
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 "tla(%d), "
+#endif
+				 "tdm(%d))",
+				 RGXFWIF_REG_CFG_TYPE_PWR_ON,
+				 RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,
+				 RGXFWIF_REG_CFG_TYPE_TA,
+				 RGXFWIF_REG_CFG_TYPE_3D,
+				 RGXFWIF_REG_CFG_TYPE_CDM,
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 RGXFWIF_REG_CFG_TYPE_TLA,
+#endif
+				 RGXFWIF_REG_CFG_TYPE_TDM);
+
+	{
+		IMG_UINT32 i;
+
+		/* Write 32 bits in each iteration as required by PDUMP WRW command */
+		for (i = 0; i < RGXFWIF_REG_CFG_TYPE_ALL; i += sizeof(IMG_UINT32))
+		{
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
+									offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[i]),
+									0,
+									PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "(Set registers here: address, mask, value)");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Addr),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Mask),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Value),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* SUPPORT_USER_REGISTER_CONFIGURATION */
+}
+#endif /* defined(PDUMP) */
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwGuardPage
+
+ @Description Allocate a Guard Page at the start of a Guest's Main Heap
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwGuardPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)),
+								  OSGetPageSize(),
+								  "FwGuardPage",
+								  &psDevInfo->psRGXFWHeapGuardPageReserveMemDesc,
+								  NULL,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwSysData
+
+ @Description Sets up all system-wide firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwSysData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									  RGX_INIT_APPHINTS        *psApphints,
+									  IMG_UINT32               ui32ConfigFlags,
+									  IMG_UINT32               ui32ConfigFlagsExt,
+									  IMG_UINT32               ui32HWPerfCountersDataSize)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32         ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+	RGXFWIF_SYSINIT    *psFwSysInitScratch = NULL;
+#if defined(PDUMP)
+	IMG_UINT32         ui32SignatureChecksBufSize = psApphints->ui32SignatureChecksBufSize;
+#endif
+
+	psFwSysInitScratch = OSAllocZMem(sizeof(*psFwSysInitScratch));
+	PVR_LOG_GOTO_IF_NOMEM(psFwSysInitScratch, eError, fail);
+
+	/* Sys Fw init data */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              (RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+	                              PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSINIT),
+	                              "FwSysInitStructure",
+	                              &psDevInfo->psRGXFWIfSysInitMemDesc,
+	                              NULL,
+	                              (void**) &psDevInfo->psRGXFWIfSysInit,
+	                              RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Sys Init structure allocation", fail);
+
+	/* Setup Fault read register */
+	eError = RGXSetupFaultReadRegister(psDeviceNode, psFwSysInitScratch);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Fault read register setup", fail);
+
+#if defined(SUPPORT_AUTOVZ)
+	psFwSysInitScratch->ui32VzWdgPeriod = PVR_AUTOVZ_WDG_PERIOD_MS;
+#endif
+
+#if defined(SUPPORT_ICS)
+	psFwSysInitScratch->ui32FDTI = psApphints->ui32FDTI;
+	psFwSysInitScratch->ui32ICSThreshold = psApphints->ui32ICSThreshold;
+	psFwSysInitScratch->bTestModeOn = psApphints->bTestModeOn;
+#endif
+
+	/* RD Power Island */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableRDPowIsland = psRGXData->psRGXTimingInfo->bEnableRDPowIsland;
+		IMG_BOOL bEnableRDPowIsland = ((psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_DEFAULT) && bSysEnableRDPowIsland) ||
+		                               (psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_FORCE_ON);
+
+		ui32ConfigFlags |= bEnableRDPowIsland? RGXFWIF_INICFG_POW_RASCALDUST : 0;
+	}
+
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		ui32ConfigFlags |= RGXFWIF_INICFG_WORKEST;
+#endif
+#if defined(SUPPORT_FW_OPP_TABLE)
+		{
+			RGXFWIF_OPP_INFO    *psOPPInfo;
+			IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg;
+
+			/* Pro-active DVFS depends on Workload Estimation */
+			psOPPInfo = &psFwSysInitScratch->sOPPInfo;
+			psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+#if defined(CONFIG_OF) && defined(CONFIG_PM_OPP) && !defined(NO_HARDWARE)
+			if (psDVFSDeviceCfg->bDTConfig)
+			{
+				/* OPP table configured from Device tree */
+				eError = DVFSCopyOPPTable(psDeviceNode,
+										  psOPPInfo,
+										  ARRAY_SIZE(psOPPInfo->asOPPValues));
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Unable to copy OPP table to FW init buffer (%u)", eError));
+					goto fail;
+				}
+			}
+#endif
+			if (!psDVFSDeviceCfg->bDTConfig)
+			{
+				PVR_LOG_IF_FALSE(psDVFSDeviceCfg->pasOPPTable, "RGXSetupFwSysData: Missing OPP Table");
+
+				if (psDVFSDeviceCfg->pasOPPTable != NULL)
+				{
+					if (psDVFSDeviceCfg->ui32OPPTableSize > ARRAY_SIZE(psOPPInfo->asOPPValues))
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								 "%s: OPP Table too large: Size = %u, Maximum size = %lu",
+								 __func__,
+								 psDVFSDeviceCfg->ui32OPPTableSize,
+								 (unsigned long)(ARRAY_SIZE(psOPPInfo->asOPPValues))));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto fail;
+					}
+
+					OSDeviceMemCopy(psOPPInfo->asOPPValues,
+									psDVFSDeviceCfg->pasOPPTable,
+									sizeof(psOPPInfo->asOPPValues));
+
+					psOPPInfo->ui32MaxOPPPoint = psDVFSDeviceCfg->ui32OPPTableSize - 1;
+				}
+			}
+
+#if defined(SUPPORT_PDVFS)
+			ui32ConfigFlags |= RGXFWIF_INICFG_PDVFS;
+#endif
+		}
+#endif /* defined(SUPPORT_FW_OPP_TABLE) */
+	}
+
+	/* FW trace control structure */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_TRACEBUF),
+								  "FwTraceCtlStruct",
+								  &psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								  &psFwSysInitScratch->sTraceBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfTraceBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Set initial firmware log type/group(s) */
+		if (psApphints->ui32LogType & ~RGXFWIF_LOG_TYPE_MASK)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid initial log type (0x%X)",
+			         __func__, psApphints->ui32LogType));
+			goto fail;
+		}
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = psApphints->ui32LogType;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+	}
+
+	/* When PDUMP is enabled, ALWAYS allocate on-demand trace buffer resource
+	 * (irrespective of loggroup(s) enabled), given that logtype/loggroups can
+	 * be set during PDump playback in logconfig, at any point of time,
+	 * Otherwise, allocate only if required. */
+#if !defined(PDUMP)
+#if defined(SUPPORT_AUTOVZ)
+	/* always allocate trace buffer for AutoVz Host drivers to allow
+	 * deterministic addresses of all SysData structures */
+	if ((PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo)) || (RGXTraceBufferIsInitRequired(psDevInfo)))
+#else
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+#endif
+#endif
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo,
+													 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+													 RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp));
+	}
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXTraceBufferInitOnDemandResources", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSDATA),
+	                              "FwSysData",
+	                              &psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                              &psFwSysInitScratch->sFwSysData,
+	                              (void**) &psDevInfo->psRGXFWIfFwSysData,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* GPIO validation setup */
+	psFwSysInitScratch->eGPIOValidationMode = RGXFWIF_GPIO_VAL_OFF;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = RGXFWSetupCounterBuffer(psDevInfo,
+									 &psDevInfo->psCounterBufferMemDesc,
+									 OSGetPageSize(),
+									 &psFwSysInitScratch->sCounterDumpCtl);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Counter Buffer allocation", fail);
+
+	PVR_DPF((PVR_DBG_WARNING, "Counter buffer allocated at %p, size %zu Bytes.", psDevInfo->psCounterBufferMemDesc, OSGetPageSize()));
+#endif /* defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = RGXFWSetupFirmwareGcovBuffer(psDevInfo,
+	                                      &psDevInfo->psFirmwareGcovBufferMemDesc,
+	                                      RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE,
+	                                      &psFwSysInitScratch->sFirmwareGcovCtl,
+	                                      "FirmwareGcovBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware GCOV buffer allocation", fail);
+	psDevInfo->ui32FirmwareGcovSize = RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE;
+#endif /* defined(SUPPORT_FIRMWARE_GCOV) */
+
+#if defined(PDUMP)
+	/* Require a minimum amount of memory for the signature buffers */
+	if (ui32SignatureChecksBufSize < RGXFW_SIG_BUFFER_SIZE_MIN)
+	{
+		ui32SignatureChecksBufSize = RGXFW_SIG_BUFFER_SIZE_MIN;
+	}
+
+	/* Setup Signature and Checksum Buffers */
+	psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	psDevInfo->ui32SigTDMChecksSize = 0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* Buffer allocated only when feature present because all known TDM
+		 * signature registers are dependent on this feature being present */
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TDM Signature check setup", fail);
+		psDevInfo->ui32SigTDMChecksSize = ui32SignatureChecksBufSize;
+	}
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GEOM Signature check setup", fail);
+	psDevInfo->ui32SigTAChecksSize = ui32SignatureChecksBufSize;
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "3D Signature check setup", fail);
+	psDevInfo->ui32Sig3DChecksSize = ui32SignatureChecksBufSize;
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSigCDMChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_CDM]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CDM Signature check setup", fail);
+	psDevInfo->ui32SigCDMChecksSize = ui32SignatureChecksBufSize;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RDM Signature check setup", fail);
+		psDevInfo->ui32SigRDMChecksSize = ui32SignatureChecksBufSize;
+	}
+#endif
+
+
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_CDM].sBuffer.ui32Addr = 0x0;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY].sBuffer.ui32Addr = 0x0;
+#endif
+	}
+#endif /* defined(PDUMP) */
+
+	eError = RGXFWSetupAlignChecks(psDeviceNode,
+	                               &psFwSysInitScratch->sAlignChecks);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Alignment checks setup", fail);
+
+	psFwSysInitScratch->ui32FilterFlags = psApphints->ui32FilterFlags;
+
+	/* Fill the remaining bits of fw the init data */
+	psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_HEAP_BASE;
+	psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_HEAP_BASE;
+	psFwSysInitScratch->sFBCDCStateTableBase.uiAddr = RGX_FBCDC_HEAP_BASE;
+	psFwSysInitScratch->sFBCDCLargeStateTableBase.uiAddr = RGX_FBCDC_LARGE_HEAP_BASE;
+	psFwSysInitScratch->sTextureHeapBase.uiAddr = RGX_TEXTURE_STATE_HEAP_BASE;
+
+	psFwSysInitScratch->sPDSIndirectHeapBase.uiAddr = RGX_PDS_INDIRECT_STATE_HEAP_BASE;
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/* Fill the remaining bits of fw the init data */
+		psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_BRN_65273_HEAP_BASE;
+		psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_BRN_65273_HEAP_BASE;
+	}
+#endif
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT))
+	{
+		eError = _AllocateSLC3Fence(psDevInfo, psFwSysInitScratch);
+		PVR_LOG_GOTO_IF_ERROR(eError, "SLC3Fence memory allocation", fail);
+	}
+#endif
+#if defined(SUPPORT_PDVFS)
+	/* Core clock rate */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(IMG_UINT32),
+	                              "FwPDVFSCoreClkRate",
+	                              &psDevInfo->psRGXFWIFCoreClkRateMemDesc,
+	                              &psFwSysInitScratch->sCoreClockRate,
+	                              (void**) &psDevInfo->pui32RGXFWIFCoreClkRate,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDVFS core clock rate memory setup", fail);
+#endif
+	{
+	/* Timestamps */
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags =
+		PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+		PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		PVRSRV_MEMALLOCFLAG_GPU_READABLE | /* XXX ?? */
+		PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+	/*
+	  the timer query arrays
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate timer query arrays (FW)");
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwStartTimesArray",
+	                          & psDevInfo->psStartTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psStartTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64StartTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwEndTimesArray",
+	                          & psDevInfo->psEndTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psEndTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64EndTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT32) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags,
+	                          "FwCompletedOpsArray",
+	                          & psDevInfo->psCompletedMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to completed ops array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCompletedMemDesc,
+	                                  (void **)& psDevInfo->pui32CompletedById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map completed ops array",
+				__func__));
+		goto fail;
+	}
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	eError = OSLockCreate(&psDevInfo->hTimerQueryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate log for timer query",
+				__func__));
+		goto fail;
+	}
+#endif
+#if defined(SUPPORT_TBI_INTERFACE)
+#if !defined(PDUMP)
+	/* allocate only if required */
+	if (RGXTBIBufferIsInitRequired(psDevInfo))
+#endif /* !defined(PDUMP) */
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand TBI buffer resource
+		 * (irrespective of loggroup(s) enabled), given that logtype/loggroups
+		 * can be set during PDump playback in logconfig, at any point of time
+		 */
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXTBIBufferInitOnDemandResources", fail);
+	}
+
+	psFwSysInitScratch->sTBIBuf = psDevInfo->sRGXFWIfTBIBuffer;
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+	/* Allocate shared buffer for GPU utilisation.
+	 * Enable FIRMWARE_CACHED to reduce read latency in the FW.
+	 * The FW flushes the cache after any writes.
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_GPU_UTIL_FW),
+								  "FwGPUUtilisationBuffer",
+								  &psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc,
+								  &psFwSysInitScratch->sGpuUtilFWCtl,
+								  (void**) &psDevInfo->psRGXFWIfGpuUtilFW,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GPU Utilisation Buffer ctl allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_RUNTIME_CFG),
+								  "FwRuntimeCfg",
+								  &psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  &psFwSysInitScratch->sRuntimeCfg,
+								  (void**) &psDevInfo->psRGXFWIfRuntimeCfg,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware runtime configuration memory allocation", fail);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_REG_CFG),
+								  "FwRegisterConfigStructure",
+								  &psDevInfo->psRGXFWIfRegCfgMemDesc,
+								  &psFwSysInitScratch->sRegCfg,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware register user configuration structure allocation", fail);
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS | PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  RGXFW_SCRATCH_BUF_SIZE,
+								  "FwScratchBuf",
+								  &psDevInfo->psRGXFWScratchBufMemDesc,
+								  &psFwSysInitScratch->pbFwScratchBuf,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware scratch buffer allocation", fail);
+#endif
+
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = GetHwPerfBufferSize(psApphints->ui32HWPerfFWBufSize);
+
+	/* Host L2 HWPERF buffer size in bytes must be bigger than the L1 buffer
+	 * accessed by the FW. The MISR may try to write one packet the size of the L1
+	 * buffer in some scenarios. When logging is enabled in the MISR, it can be seen
+	 * if the L2 buffer hits a full condition. The closer in size the L2 and L1 buffers
+	 * are the more chance of this happening.
+	 * Size chosen to allow MISR to write an L1 sized packet and for the client
+	 * application/daemon to drain a L1 sized packet e.g. ~ 1.5*L1.
+	 */
+	psDevInfo->ui32RGXL2HWPerfBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize +
+	        (psDevInfo->ui32RGXFWIfHWPerfBufSize>>1);
+
+	/* Second stage initialisation or HWPerf, hHWPerfLock created in first
+	 * stage. See RGXRegisterDevice() call to RGXHWPerfInit(). */
+	if (psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF] == 0)
+	{
+		psFwSysInitScratch->ui64HWPerfFilter =
+		    RGXHWPerfFwSetEventFilter(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF,
+		                              (IMG_UINT64) psApphints->ui32HWPerfFilter0 |
+		                              ((IMG_UINT64) psApphints->ui32HWPerfFilter1 << 32));
+	}
+	else
+	{
+		/* The filter has already been modified. This can happen if
+		 * pvr/apphint/EnableFTraceGPU was enabled. */
+		psFwSysInitScratch->ui64HWPerfFilter = psDevInfo->ui64HWPerfFwFilter;
+	}
+
+#if !defined(PDUMP)
+	/* Allocate if HWPerf filter has already been set. This is possible either
+	 * by setting a proper AppHint or enabling GPU ftrace events. */
+	if (psFwSysInitScratch->ui64HWPerfFilter != 0)
+#endif
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand HWPerf resources
+		 * (irrespective of HWPerf enabled or not), given that HWPerf can be
+		 * enabled during PDump playback via RTCONF at any point of time. */
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL1Buffer", fail);
+
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL2Stream", fail);
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		/* Align the hwperf allocation to the DMA block size to ensure
+		 * the shared control structure buffer isn't overflowed when
+		 * Meta DMA writes to it. */
+		ui32HWPerfCountersDataSize = PVR_ALIGN(ui32HWPerfCountersDataSize, RGX_META_DMA_BLOCK_SIZE);
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWCOMCTX_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32HWPerfCountersDataSize,
+								  "FwHWPerfControlStructure",
+								  &psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+								  &psFwSysInitScratch->sHWPerfCtl,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware HW Perf control struct allocation", fail);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		RGXSetMetaDMAAddress(&psDevInfo->psRGXFWIfRuntimeCfg->sHWPerfCtlDMABuf,
+		                     psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+		                     &psFwSysInitScratch->sHWPerfCtl, 0);
+	}
+
+	psDevInfo->bPDPEnabled = (ui32ConfigFlags & RGXFWIF_INICFG_DISABLE_PDP_EN)
+							  ? IMG_FALSE : IMG_TRUE;
+
+	psFwSysInitScratch->eFirmwarePerf = psApphints->eFirmwarePerf;
+
+#if defined(PDUMP)
+	/* default: no filter */
+	psFwSysInitScratch->sPIDFilter.eMode = RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT;
+	psFwSysInitScratch->sPIDFilter.asItems[0].uiPID = 0;
+#endif
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	{
+		PVRSRV_MEMALLOCFLAGS_T uiFlags = RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS;
+		PVRSRV_SET_PHYS_HEAP_HINT(FW_PRIV_DATA, uiFlags);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate non-secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+											"FwExNonSecureBuffer",
+											&psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Non-secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbNonSecureBuffer,
+									   psDevInfo->psRGXFWIfNonSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											uiFlags,
+											"FwExSecureBuffer",
+											&psDevInfo->psRGXFWIfSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbSecureBuffer,
+									   psDevInfo->psRGXFWIfSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail);
+	}
+#endif /* SUPPORT_SECURITY_VALIDATION */
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) || RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		psFwSysInitScratch->ui32TFBCCompressionControl =
+			(ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >> RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+	}
+#endif /* RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK */
+
+	/* Initialize FW started flag */
+	psFwSysInitScratch->bFirmwareStarted = IMG_FALSE;
+	psFwSysInitScratch->ui32MarkerVal = 1;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = RGX_VZ_CONNECTION_COOLDOWN_PERIOD;
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		IMG_UINT32 ui32DriverID;
+
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+
+		/* Required info by FW to calculate the ActivePM idle timer latency */
+		psFwSysInitScratch->ui32InitialCoreClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psFwSysInitScratch->ui32InitialSOCClockSpeed = psRGXData->psRGXTimingInfo->ui32SOCClockSpeed;
+#endif
+		psFwSysInitScratch->ui32InitialActivePMLatencyms = psRGXData->psRGXTimingInfo->ui32ActivePMLatencyms;
+
+		/* Initialise variable runtime configuration to the system defaults */
+		psRuntimeCfg->ui32CoreClockSpeed = psFwSysInitScratch->ui32InitialCoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psRuntimeCfg->ui32SOCClockSpeed = psFwSysInitScratch->ui32InitialSOCClockSpeed;
+#endif
+		psRuntimeCfg->ui32ActivePMLatencyms = psFwSysInitScratch->ui32InitialActivePMLatencyms;
+		psRuntimeCfg->bActivePMLatencyPersistant = IMG_TRUE;
+		psRuntimeCfg->ui32HCSDeadlineMS = RGX_HCS_DEFAULT_DEADLINE_MS;
+
+		if ((RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US > 0U) && (RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US < 1000U))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_LOG_GOTO_IF_ERROR(eError,
+								  "RGXSetupFwSysData: RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US must be either 0 (disabled) or greater than 1000",
+								  fail);
+		}
+		else
+		{
+			psRuntimeCfg->ui32WdgPeriodUs = RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+		{
+			psRuntimeCfg->ai32DriverPriority[RGXFW_HOST_DRIVER_ID] = 0;
+			psRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID] = RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP;
+			psRuntimeCfg->aui32TSPercentage[RGXFW_HOST_DRIVER_ID] = (IMG_UINT8)RGX_DRIVERID_0_DEFAULT_TIME_SLICE;
+		}
+		else
+		{
+			const RGX_QOS_DEFAULTS asQosDefaults[RGXFW_MAX_NUM_OSIDS] = {
+					RGX_QOS_DEFAULTS_INIT(0),
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+					RGX_QOS_DEFAULTS_INIT(1),
+#if (RGXFW_MAX_NUM_OSIDS > 2)
+					RGX_QOS_DEFAULTS_INIT(2),
+					RGX_QOS_DEFAULTS_INIT(3),
+					RGX_QOS_DEFAULTS_INIT(4),
+					RGX_QOS_DEFAULTS_INIT(5),
+					RGX_QOS_DEFAULTS_INIT(6),
+					RGX_QOS_DEFAULTS_INIT(7),
+#if (RGXFW_MAX_NUM_OSIDS > 8)
+					RGX_QOS_DEFAULTS_INIT(8),
+					RGX_QOS_DEFAULTS_INIT(9),
+					RGX_QOS_DEFAULTS_INIT(10),
+					RGX_QOS_DEFAULTS_INIT(11),
+					RGX_QOS_DEFAULTS_INIT(12),
+					RGX_QOS_DEFAULTS_INIT(13),
+					RGX_QOS_DEFAULTS_INIT(14),
+					RGX_QOS_DEFAULTS_INIT(15),
+					RGX_QOS_DEFAULTS_INIT(16),
+					RGX_QOS_DEFAULTS_INIT(17),
+					RGX_QOS_DEFAULTS_INIT(18),
+					RGX_QOS_DEFAULTS_INIT(19),
+					RGX_QOS_DEFAULTS_INIT(20),
+					RGX_QOS_DEFAULTS_INIT(21),
+					RGX_QOS_DEFAULTS_INIT(22),
+					RGX_QOS_DEFAULTS_INIT(23),
+					RGX_QOS_DEFAULTS_INIT(24),
+					RGX_QOS_DEFAULTS_INIT(25),
+					RGX_QOS_DEFAULTS_INIT(26),
+					RGX_QOS_DEFAULTS_INIT(27),
+					RGX_QOS_DEFAULTS_INIT(28),
+					RGX_QOS_DEFAULTS_INIT(29),
+					RGX_QOS_DEFAULTS_INIT(30),
+					RGX_QOS_DEFAULTS_INIT(31),
+#if (RGXFW_MAX_NUM_OSIDS > 32)
+#error "Support for more than 32 OSIDs not implemented."
+#endif
+#endif /* RGXFW_MAX_NUM_OSIDS > 8 */
+#endif /* RGXFW_MAX_NUM_OSIDS > 2 */
+
+#endif /* RGXFW_MAX_NUM_OSIDS > 1 */
+			};
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				/* Set up initial priorities between different OSes */
+				psRuntimeCfg->ai32DriverPriority[ui32DriverID] = asQosDefaults[ui32DriverID].i32Priority;
+				psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = asQosDefaults[ui32DriverID].ui32IsolationGroups;
+				psRuntimeCfg->aui32TSPercentage[ui32DriverID] = (asQosDefaults[ui32DriverID].ui32TSPercentage <=
+																				PVRSRV_VZ_TIME_SLICE_MAX) ?
+																				asQosDefaults[ui32DriverID].ui32TSPercentage:(0);
+			}
+		}
+		psRuntimeCfg->ui32TSIntervalMs = RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL;
+
+#if defined(PVR_ENABLE_PHR) && defined(PDUMP)
+		psRuntimeCfg->ui32PHRMode = RGXFWIF_PHR_MODE_RD_RESET;
+#else
+		psRuntimeCfg->ui32PHRMode = 0;
+#endif
+
+		/* Validate the power units mask and initialize to number of units to power up */
+		if ((psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask) == 0)
+		{
+			eError = PVRSRV_ERROR_INVALID_SPU_MASK;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s:Invalid power units mask (All=0x%X, Non Fused=0x%X). At-least one power unit must to be powered up.",
+			         __func__,
+			         ui32AllPowUnitsMask,
+			         psApphints->ui32AvailablePowUnitsMask));
+			goto fail;
+		}
+		psRuntimeCfg->ui32PowUnitsState = psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask;
+
+		psRuntimeCfg->ui32RACUnitsState = psApphints->ui32AvailableRACMask & ui32AllRACMask;
+
+		/* flush write buffers for psDevInfo->psRGXFWIfRuntimeCfg */
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfRuntimeCfg);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, FLUSH);
+
+		/* Setup FW coremem data */
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psFwSysInitScratch->sCorememDataStore.pbyFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+			{
+				RGXSetMetaDMAAddress(&psFwSysInitScratch->sCorememDataStore,
+									 psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+									 &psFwSysInitScratch->sCorememDataStore.pbyFWAddr,
+									 0);
+			}
+		}
+
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags    = ui32ConfigFlags    & RGXFWIF_INICFG_ALL;
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt = ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_ALL;
+
+		/* Initialise GPU utilisation buffer */
+		{
+			IMG_UINT64 ui64LastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64(), RGXFWIF_GPU_UTIL_STATE_IDLE);
+			RGXFWIF_DM eDM;
+			IMG_UINT64 ui64LastWordTimeShifted =
+			        RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64() >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT, RGXFWIF_GPU_UTIL_STATE_IDLE);
+			IMG_UINT32 ui32DriverID;
+
+			psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord = ui64LastWord;
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				RGXFWIF_GPU_STATS *psStats = &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID];
+
+				for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+				{
+					psStats->aui32DMOSLastWord[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted & IMG_UINT32_MAX);
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted >> 32);
+				}
+			}
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+		}
+
+		/* init HWPERF data */
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfUt = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfDropCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32FirstDropOrdinal = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32LastDropOrdinal = 0;
+
+		psDevInfo->psRGXFWIfFwSysData->ui32MemFaultCheck = 0;
+
+		// flush write buffers for psRGXFWIfFwSysData
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwSysData);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, FLUSH);
+
+		/*Send through the BVNC Feature Flags*/
+		eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, &psFwSysInitScratch->sBvncKmFeatureFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags", fail);
+
+		/* populate the real FwOsInit structure with the values stored in the scratch copy */
+		OSCachedMemCopyWMB(psDevInfo->psRGXFWIfSysInit, psFwSysInitScratch, sizeof(RGXFWIF_SYSINIT));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit,
+		                         FLUSH);
+	}
+
+	OSFreeMem(psFwSysInitScratch);
+
+
+	return PVRSRV_OK;
+
+fail:
+	if (psFwSysInitScratch)
+	{
+		OSFreeMem(psFwSysInitScratch);
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwOsData
+
+ @Description Sets up all os-specific firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwOsData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									 IMG_UINT32               ui32KCCBSizeLog2,
+									 IMG_UINT32               ui32HWRDebugDumpLimit,
+									 IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_OSINIT     sFwOsInitScratch;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSCachedMemSet(&sFwOsInitScratch, 0, sizeof(RGXFWIF_OSINIT));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXSetupFwGuardPage(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware heap guard pages", fail);
+	}
+
+	/* Memory tracking the connection state should be non-volatile and
+	 * is not cleared on allocation to prevent loss of pre-reset information */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS &
+								  ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+								  sizeof(RGXFWIF_CONNECTION_CTL),
+								  "FwConnectionCtl",
+								  &psDevInfo->psRGXFWIfConnectionCtlMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfConnectionCtl,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Connection Control structure allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED),
+								  sizeof(RGXFWIF_OSINIT),
+								  "FwOsInitStructure",
+								  &psDevInfo->psRGXFWIfOsInitMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfOsInit,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Os Init structure allocation", fail);
+
+	/* init HWR frame info */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_HWRINFOBUF),
+								  "FwHWRInfoBuffer",
+								  &psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
+								  &sFwOsInitScratch.sRGXFWIfHWRInfoBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfHWRInfoBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HWR Info Buffer allocation", fail);
+
+	OSCachedMemSetWMB(psDevInfo->psRGXFWIfHWRInfoBufCtl, 0, sizeof(RGXFWIF_HWRINFOBUF));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, FLUSH);
+
+	/* Allocate a sync for power management */
+	eError = SyncPrimContextCreate(psDevInfo->psDeviceNode,
+	                               &psDevInfo->hSyncPrimContext);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive context allocation", fail);
+
+	eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psDevInfo->psPowSyncPrim, "fw power ack");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive allocation", fail);
+
+	/* Set up kernel CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psKernelCCBCtl,
+						 &psDevInfo->psKernelCCBCtlLocal,
+						 &psDevInfo->psKernelCCBCtlMemDesc,
+						 &psDevInfo->psKernelCCB,
+						 &psDevInfo->psKernelCCBMemDesc,
+						 &sFwOsInitScratch.psKernelCCBCtl,
+						 &sFwOsInitScratch.psKernelCCB,
+						 ui32KCCBSizeLog2,
+						 sizeof(RGXFWIF_KCCB_CMD),
+						 (RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS |
+						 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+						 "FwKernelCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB allocation", fail);
+
+	/* KCCB additionally uses a return slot array for FW to be able to send back
+	 * return codes for each required command
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  (1U << ui32KCCBSizeLog2) * sizeof(IMG_UINT32),
+								  "FwKernelCCBRtnSlots",
+								  &psDevInfo->psKernelCCBRtnSlotsMemDesc,
+								  &sFwOsInitScratch.psKernelCCBRtnSlots,
+								  (void**) &psDevInfo->pui32KernelCCBRtnSlots,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB return slot array allocation", fail);
+
+	/* Set up firmware CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psFirmwareCCBCtl,
+						 &psDevInfo->psFirmwareCCBCtlLocal,
+						 &psDevInfo->psFirmwareCCBCtlMemDesc,
+						 &psDevInfo->psFirmwareCCB,
+						 &psDevInfo->psFirmwareCCBMemDesc,
+						 &sFwOsInitScratch.psFirmwareCCBCtl,
+						 &sFwOsInitScratch.psFirmwareCCB,
+						 RGXFWIF_FWCCB_NUMCMDS_LOG2,
+						 sizeof(RGXFWIF_FWCCB_CMD),
+						 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+						 "FwCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware CCB allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_OSDATA),
+								  "FwOsData",
+								  &psDevInfo->psRGXFWIfFwOsDataMemDesc,
+								  &sFwOsInitScratch.sFwOsData,
+								  (void**) &psDevInfo->psRGXFWIfFwOsData,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags = ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_ALL;
+
+	eError = SyncPrimGetFirmwareAddr(psDevInfo->psPowSyncPrim, &psDevInfo->psRGXFWIfFwOsData->sPowerSync.ui32Addr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Get Sync Prim FW address", fail);
+
+	/* flush write buffers for psRGXFWIfFwOsData */
+	OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwOsData);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         FLUSH);
+
+	sFwOsInitScratch.ui32HWRDebugDumpLimit = ui32HWRDebugDumpLimit;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Set up Workload Estimation firmware CCB */
+		eError = RGXSetupCCB(psDevInfo,
+							 &psDevInfo->psWorkEstFirmwareCCBCtl,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+							 &psDevInfo->psWorkEstFirmwareCCB,
+							 &psDevInfo->psWorkEstFirmwareCCBMemDesc,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCBCtl,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCB,
+							 RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2,
+							 sizeof(RGXFWIF_WORKEST_FWCCB_CMD),
+							 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+							 "FwWEstCCB");
+		PVR_LOG_GOTO_IF_ERROR(eError, "Workload Estimation Firmware CCB allocation", fail);
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise the compatibility check data */
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sFWBVNC);
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sHWBVNC);
+	}
+
+	/* populate the real FwOsInit structure with the values stored in the scratch copy */
+	OSCachedMemCopyWMB(psDevInfo->psRGXFWIfOsInit, &sFwOsInitScratch, sizeof(RGXFWIF_OSINIT));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfOsInit,
+	                         FLUSH);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeFwOsData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFirmware
+
+ @Description Sets up all firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+							  RGX_INIT_APPHINTS        *psApphints,
+							  IMG_UINT32               ui32ConfigFlags,
+							  IMG_UINT32               ui32ConfigFlagsExt,
+							  IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32NumHWPerfBlocks;
+	IMG_UINT32 ui32HWPerfCountersDataSize;
+
+	eError = RGXSetupFwOsData(psDeviceNode,
+							  psApphints->ui32KCCBSizeLog2,
+							  psApphints->ui32HWRDebugDumpLimit,
+							  ui32FwOsCfgFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware os data", fail);
+
+	ui32NumHWPerfBlocks = RGXHWPerfMaxDefinedBlks((void *)psDevInfo);
+	ui32HWPerfCountersDataSize = sizeof(RGXFWIF_HWPERF_CTL) +
+		(ui32NumHWPerfBlocks - 1) * sizeof(RGXFWIF_HWPERF_CTL_BLK);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest drivers do not configure system-wide firmware data */
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+	else
+	{
+		/* Native and Host drivers must initialise the firmware's system data */
+		eError = RGXSetupFwSysData(psDeviceNode,
+								   psApphints,
+								   ui32ConfigFlags,
+								   ui32ConfigFlagsExt,
+								   ui32HWPerfCountersDataSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware system data", fail);
+	}
+
+	psDevInfo->bFirmwareInitialised = IMG_TRUE;
+
+#if defined(PDUMP)
+	RGXPDumpLoadFWInitData(psDevInfo,
+	                       psApphints,
+	                       ui32HWPerfCountersDataSize);
+#endif /* PDUMP */
+
+fail:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwSysData
+
+ @Description Frees all system-wide firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	psDevInfo->bFirmwareInitialised = IMG_FALSE;
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc)
+	{
+		RGXFWFreeAlignChecks(psDevInfo);
+	}
+
+#if defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM) &&
+	    psDevInfo->psRGXFWSigTDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTDMChecksMemDesc);
+		psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSigTAChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTAChecksMemDesc);
+		psDevInfo->psRGXFWSigTAChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSig3DChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSig3DChecksMemDesc);
+		psDevInfo->psRGXFWSig3DChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSigCDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigCDMChecksMemDesc);
+		psDevInfo->psRGXFWSigCDMChecksMemDesc = NULL;
+	}
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1 &&
+	   psDevInfo->psRGXFWSigRDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigRDMChecksMemDesc);
+		psDevInfo->psRGXFWSigRDMChecksMemDesc = NULL;
+	}
+#endif
+
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	if (psDevInfo->psCounterBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCounterBufferMemDesc);
+		psDevInfo->psCounterBufferMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	if (psDevInfo->psFirmwareGcovBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psFirmwareGcovBufferMemDesc);
+		psDevInfo->psFirmwareGcovBufferMemDesc = NULL;
+	}
+#endif
+
+	RGXSetupFaultReadRegisterRollback(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfGpuUtilFW != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+			psDevInfo->psRGXFWIfGpuUtilFW = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+		psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfRuntimeCfgMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfRuntimeCfg != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+			psDevInfo->psRGXFWIfRuntimeCfg = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+		psDevInfo->psRGXFWIfRuntimeCfgMemDesc = NULL;
+	}
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+			psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+		}
+	}
+
+	if (psDevInfo->psRGXFWIfTraceBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufCtl != NULL)
+		{
+			/* deinit/free the tracebuffer allocation */
+			RGXTraceBufferDeinit(psDevInfo);
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+			psDevInfo->psRGXFWIfTraceBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+		psDevInfo->psRGXFWIfTraceBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwSysDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwSysData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwSysDataMemDesc);
+			psDevInfo->psRGXFWIfFwSysData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwSysDataMemDesc);
+		psDevInfo->psRGXFWIfFwSysDataMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	if (psDevInfo->psRGXFWIfTBIBufferMemDesc)
+	{
+		RGXTBIBufferDeinit(psDevInfo);
+	}
+#endif
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	if (psDevInfo->psRGXFWIfRegCfgMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRegCfgMemDesc);
+		psDevInfo->psRGXFWIfRegCfgMemDesc = NULL;
+	}
+#endif
+	if (psDevInfo->psRGXFWIfHWPerfCountersMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		psDevInfo->psRGXFWIfHWPerfCountersMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDevInfo->psRGXFWIfNonSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		psDevInfo->psRGXFWIfNonSecureBufMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSecureBufMemDesc);
+		psDevInfo->psRGXFWIfSecureBufMemDesc = NULL;
+	}
+#endif
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT))
+	{
+		_FreeSLC3Fence(psDevInfo);
+	}
+#endif
+#if defined(SUPPORT_PDVFS)
+	if (psDevInfo->psRGXFWIFCoreClkRateMemDesc)
+	{
+		if (psDevInfo->pui32RGXFWIFCoreClkRate != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+			psDevInfo->pui32RGXFWIFCoreClkRate = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+		psDevInfo->psRGXFWIFCoreClkRateMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+		psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwOsData
+
+ @Description Frees all os-specific firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFreeCCBReturnSlots(psDevInfo,
+	                      &psDevInfo->pui32KernelCCBRtnSlots,
+	                      &psDevInfo->psKernelCCBRtnSlotsMemDesc);
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psKernelCCBCtl,
+	           &psDevInfo->psKernelCCBCtlLocal,
+	           &psDevInfo->psKernelCCBCtlMemDesc,
+	           &psDevInfo->psKernelCCB,
+	           &psDevInfo->psKernelCCBMemDesc);
+
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psFirmwareCCBCtl,
+	           &psDevInfo->psFirmwareCCBCtlLocal,
+	           &psDevInfo->psFirmwareCCBCtlMemDesc,
+	           &psDevInfo->psFirmwareCCB,
+	           &psDevInfo->psFirmwareCCBMemDesc);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFreeCCB(psDevInfo,
+		           &psDevInfo->psWorkEstFirmwareCCBCtl,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+		           &psDevInfo->psWorkEstFirmwareCCB,
+		           &psDevInfo->psWorkEstFirmwareCCBMemDesc);
+	}
+#endif
+
+	if (psDevInfo->psPowSyncPrim != NULL)
+	{
+		SyncPrimFree(psDevInfo->psPowSyncPrim);
+		psDevInfo->psPowSyncPrim = NULL;
+	}
+
+	if (psDevInfo->hSyncPrimContext != (IMG_HANDLE) NULL)
+	{
+		SyncPrimContextDestroy(psDevInfo->hSyncPrimContext);
+		psDevInfo->hSyncPrimContext = (IMG_HANDLE) NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfHWRInfoBufCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+			psDevInfo->psRGXFWIfHWRInfoBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+		psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwOsDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwOsData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwOsDataMemDesc);
+			psDevInfo->psRGXFWIfFwOsData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwOsDataMemDesc);
+		psDevInfo->psRGXFWIfFwOsDataMemDesc = NULL;
+	}
+
+	if (psDevInfo->psCompletedMemDesc)
+	{
+		if (psDevInfo->pui32CompletedById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psCompletedMemDesc);
+			psDevInfo->pui32CompletedById = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCompletedMemDesc);
+		psDevInfo->psCompletedMemDesc = NULL;
+	}
+	if (psDevInfo->psEndTimeMemDesc)
+	{
+		if (psDevInfo->pui64EndTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psEndTimeMemDesc);
+			psDevInfo->pui64EndTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psEndTimeMemDesc);
+		psDevInfo->psEndTimeMemDesc = NULL;
+	}
+	if (psDevInfo->psStartTimeMemDesc)
+	{
+		if (psDevInfo->pui64StartTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psStartTimeMemDesc);
+			psDevInfo->pui64StartTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psStartTimeMemDesc);
+		psDevInfo->psStartTimeMemDesc = NULL;
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	if (psDevInfo->hTimerQueryLock)
+	{
+		OSLockDestroy(psDevInfo->hTimerQueryLock);
+		psDevInfo->hTimerQueryLock = NULL;
+	}
+#endif
+
+	if (psDevInfo->psRGXFWHeapGuardPageReserveMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWHeapGuardPageReserveMemDesc);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFirmware
+
+ @Description Frees all the firmware-related allocations
+
+ @Input       psDevInfo
+******************************************************************************/
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFreeFwOsData(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfConnectionCtl)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtl = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfConnectionCtlMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInitMemDesc = NULL;
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+	if (psDevInfo->psRGXFWIfSysInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSysInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInitMemDesc = NULL;
+	}
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalKCCBRoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT32 ui32ReadOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update KCCB read offset */
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+	ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+	if (ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psKCCBCtlLocal->ui32ReadOffset = ui32ReadOffset;
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalFWCCBWoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT32 ui32WriteOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update FWCCB write offset */
+	RGXFwSharedMemCacheOpValue(psFWCCBCtl->ui32WriteOffset, INVALIDATE);
+	ui32WriteOffset = psFWCCBCtl->ui32WriteOffset;
+
+	if (ui32WriteOffset > psFWCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psFWCCBCtlLocal->ui32WriteOffset = ui32WriteOffset;
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireKernelCCBSlot
+
+ PURPOSE	: Attempts to obtain a slot in the Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32		*pui32Offset)
+{
+	IMG_UINT32 ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+#if defined(PDUMP)
+	/* Wait for sufficient CCB space to become available */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, 0,
+	                      "Wait for space to write kCCB woff=%u", psKCCBCtlLocal->ui32WriteOffset);
+	DevmemPDumpCBP(psDevInfo->psKernelCCBCtlMemDesc,
+	               offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+	               psKCCBCtlLocal->ui32WriteOffset,
+	               1,
+	               (psKCCBCtlLocal->ui32WrapMask + 1));
+#endif
+
+	if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+	}
+	*pui32Offset = ui32NextWriteOffset;
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXPollKernelCCBSlot
+
+ PURPOSE	: Poll for space in Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXPollKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32	ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+	if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+	{
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_OK;
+		}
+
+		/*
+		 * The following check doesn't impact performance, since the
+		 * CPU has to wait for the GPU anyway (full kernel CCB).
+		 */
+		if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return PVRSRV_ERROR_KERNEL_CCB_FULL;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXGetCmdMemCopySize
+
+ PURPOSE	: Calculates actual size of KCCB command getting used
+
+ PARAMETERS	: eCmdType     Type of KCCB command
+
+ RETURNS	: Returns actual size of KCCB command on success else zero
+******************************************************************************/
+static IMG_UINT32 RGXGetCmdMemCopySize(RGXFWIF_KCCB_CMD_TYPE eCmdType)
+{
+	/* First get offset of uCmdData inside the struct RGXFWIF_KCCB_CMD
+	 * This will account alignment requirement of uCmdData union
+	 *
+	 * Then add command-data size depending on command type to calculate actual
+	 * command size required to do mem copy
+	 *
+	 * NOTE: Make sure that uCmdData is the last member of RGXFWIF_KCCB_CMD struct.
+	 */
+	switch (eCmdType)
+	{
+		case RGXFWIF_KCCB_CMD_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_MMUCACHE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_MMUCACHEDATA);
+		}
+#if defined(SUPPORT_USC_BREAKPOINT)
+		case RGXFWIF_KCCB_CMD_BP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_BPDATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_SLCFLUSHINVAL:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_SLCFLUSHINVALDATA);
+		}
+		case RGXFWIF_KCCB_CMD_CLEANUP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CLEANUP_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_POW:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_POWER_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE:
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_ZSBUFFER_BACKING_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELIST_GS_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELISTS_RECONSTRUCTION_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_WRITE_OFFSET_UPDATE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FORCE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA);
+		}
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+		case RGXFWIF_KCCB_CMD_REGCONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_REGCONFIG_DATA);
+		}
+#endif
+#if defined(SUPPORT_PDVFS)
+		case RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_PDVFS_MAX_FREQ_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_OS_STATE_CHANGE_DATA);
+		}
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+		case RGXFWIF_KCCB_CMD_COUNTER_DUMP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_COUNTER_DUMP_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CORECLKSPEEDCHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL:
+		case RGXFWIF_KCCB_CMD_WDG_CFG:
+		case RGXFWIF_KCCB_CMD_HEALTH_CHECK:
+		case RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL:
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS_DIRECT:
+		{
+			/* No command specific data */
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData);
+		}
+		case RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_DEV_VIRTADDR);
+		}
+		case RGXFWIF_KCCB_CMD_CANCEL_WORK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CANCEL_WORK_DATA);
+		}
+		default:
+		{
+			/* Invalid (OR) Unused (OR) Newly added command type */
+			return 0; /* Error */
+		}
+	}
+}
+
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+	                           INVALIDATE);
+	eError = PVRSRVWaitForValueKM(
+	              (IMG_UINT32 __iomem *)&psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFwSharedMemCacheOpExecPfn);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVWaitForValueKM");
+
+#if defined(PDUMP)
+	/* PDumping conditions same as RGXSendCommandRaw for the actual command and poll command to go in harmony */
+	if (PDumpCheckFlagsWrite(psDevInfo->psDeviceNode, ui32PDumpFlags))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Poll on KCCB slot %u for value %u (mask: 0x%x)", ui32SlotNum,
+					 RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED, RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED);
+
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+										ui32SlotNum * sizeof(IMG_UINT32),
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										PDUMP_POLL_OPERATOR_EQUAL,
+										ui32PDumpFlags);
+		PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXSendCommandRaw(PVRSRV_RGXDEV_INFO  *psDevInfo,
+									  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+									  IMG_UINT32          uiPDumpFlags,
+									  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE	*psDeviceNode = psDevInfo->psDeviceNode;
+	RGXFWIF_CCB_CTL		*psKCCBCtl;
+	RGXFWIF_CCB_CTL		*psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT8			*pui8KCCB = psDevInfo->psKernelCCB;
+	IMG_UINT32			ui32NewWriteOffset;
+	IMG_UINT32			ui32OldWriteOffset;
+	IMG_UINT32			ui32CmdMemCopySize;
+
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+#else
+	IMG_BOOL bContCaptureOn = PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER); /* client connected or in pdump init phase */
+	IMG_BOOL bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, uiPDumpFlags); /* Are we in capture range or continuous and not in a power transition */
+
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			if (!psDevInfo->bDumpedKCCBCtlAlready)
+			{
+				/* entering capture range */
+				psDevInfo->bDumpedKCCBCtlAlready = IMG_TRUE;
+
+				/* Wait for the live FW to catch up */
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: waiting on fw to catch-up, roff: %d, woff: %d",
+						__func__,
+						psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset));
+				PVRSRVPollForValueKM(psDeviceNode,
+				                     (IMG_UINT32 __iomem *)&psKCCBCtl->ui32ReadOffset,
+				                     ui32OldWriteOffset, 0xFFFFFFFF,
+				                     POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+				                     NULL);
+
+				/* Dump Init state of Kernel CCB control (read and write offset) */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+						"Initial state of kernel CCB Control, roff: %d, woff: %d",
+						psKCCBCtl->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset);
+
+				DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+						0,
+						sizeof(RGXFWIF_CCB_CTL),
+						uiPDumpFlags);
+			}
+		}
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		if ((likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+			(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo)))) ||
+			(KM_FW_CONNECTION_IS(READY, psDevInfo) && KM_OS_CONNECTION_IS(READY, psDevInfo)))
+		{
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: The firmware-driver connection is invalid:"
+									"driver state = %u / firmware state = %u;"
+									"expected READY (%u/%u) or ACTIVE (%u/%u) or in transition (%u/%u);",
+									__func__, KM_GET_OS_CONNECTION(psDevInfo), KM_GET_FW_CONNECTION(psDevInfo),
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_READY,
+									RGXFW_CONNECTION_OS_ACTIVE, RGXFW_CONNECTION_FW_ACTIVE,
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_ACTIVE));
+			eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+			goto _RGXSendCommandRaw_Exit;
+		}
+	}
+#endif
+
+	if (!OSLockIsLocked(psDeviceNode->hPowerLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called without power lock held!",
+				__func__));
+		PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+	}
+
+	/* Acquire a slot in the CCB */
+	eError = RGXAcquireKernelCCBSlot(psDevInfo, &ui32NewWriteOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto _RGXSendCommandRaw_Exit;
+	}
+
+	/* Calculate actual size of command to optimize device mem copy */
+	ui32CmdMemCopySize = RGXGetCmdMemCopySize(psKCCBCmd->eCmdType);
+	PVR_LOG_RETURN_IF_FALSE(ui32CmdMemCopySize !=0, "RGXGetCmdMemCopySize failed", PVRSRV_ERROR_INVALID_CCB_COMMAND);
+
+	/* Copy the command into the CCB */
+	OSCachedMemCopy(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)],
+	                psKCCBCmd, ui32CmdMemCopySize);
+	RGXFwSharedMemCacheOpExec(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)], ui32CmdMemCopySize, PVRSRV_CACHE_OP_FLUSH);
+
+	/* If non-NULL pui32CmdKCCBSlot passed-in, return the kCCB slot in which the command was enqueued */
+	if (pui32CmdKCCBSlot)
+	{
+		*pui32CmdKCCBSlot = ui32OldWriteOffset;
+
+		/* Each such command enqueue needs to reset the slot value first. This is so that a caller
+		 * doesn't get to see stale/false value in allotted slot */
+		OSWriteDeviceMem32WithWMB(&psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                          RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                           FLUSH);
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+							  "Reset kCCB slot number %u", ui32OldWriteOffset);
+		DevmemPDumpLoadMem(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+		                   ui32OldWriteOffset * sizeof(IMG_UINT32),
+						   sizeof(IMG_UINT32),
+						   uiPDumpFlags);
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Device (%p) KCCB slot %u reset with value %u for command type %x",
+		         __func__, psDevInfo, ui32OldWriteOffset, RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE, psKCCBCmd->eCmdType));
+	}
+
+	/* Memory barrier before KCCB write offset update. */
+	OSWriteMemoryBarrier(NULL);
+
+
+	/* Move past the current command */
+	psKCCBCtlLocal->ui32WriteOffset = ui32NewWriteOffset;
+	psKCCBCtl->ui32WriteOffset = ui32NewWriteOffset;
+	/* Read-back of memory before Kick MTS */
+	OSWriteMemoryBarrier(&psKCCBCtl->ui32WriteOffset);
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32WriteOffset, FLUSH);
+
+#if defined(PDUMP)
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			/* Dump new Kernel CCB content */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCB cmd woff = %d",
+					ui32OldWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBMemDesc,
+					ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD),
+					ui32CmdMemCopySize,
+					uiPDumpFlags);
+
+			/* Dump new kernel CCB write offset */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCBCtl woff: %d",
+					ui32NewWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					sizeof(IMG_UINT32),
+					uiPDumpFlags);
+
+			/* mimic the read-back of the write from above */
+			DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					ui32NewWriteOffset,
+					0xFFFFFFFF,
+					PDUMP_POLL_OPERATOR_EQUAL,
+					uiPDumpFlags);
+		}
+		/* out of capture range */
+		else
+		{
+			eError = RGXPdumpDrainKCCB(psDevInfo, ui32OldWriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", _RGXSendCommandRaw_Exit);
+		}
+	}
+#endif
+
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags, "MTS kick for kernel CCB");
+	/*
+	 * Kick the MTS to schedule the firmware.
+	 */
+	__MTSScheduleWrite(psDevInfo, RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK);
+
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_MTS_SCHEDULE,
+	           RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK, uiPDumpFlags);
+
+#if defined(SUPPORT_AUTOVZ)
+	RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+
+#if defined(NO_HARDWARE)
+	/* keep the roff updated because fw isn't there to update it */
+	psKCCBCtl->ui32ReadOffset = psKCCBCtlLocal->ui32WriteOffset;
+#endif
+
+_RGXSendCommandRaw_Exit:
+	return eError;
+}
+
+/******************************************************************************
+ FUNCTION	: _AllocDeferredCommand
+
+ PURPOSE	: Allocate a KCCB command and add it to KCCB deferred list
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: eKCCBType		Firmware Command type
+			: psKCCBCmd		Firmware Command
+			: uiPDumpFlags	Pdump flags
+
+ RETURNS	: PVRSRV_OK	If all went good, PVRSRV_ERROR_RETRY otherwise.
+******************************************************************************/
+static PVRSRV_ERROR _AllocDeferredCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                          RGXFWIF_KCCB_CMD   *psKCCBCmd,
+                                          IMG_UINT32         uiPDumpFlags)
+{
+	RGX_DEFERRED_KCCB_CMD *psDeferredCommand;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psDeferredCommand = OSAllocMem(sizeof(*psDeferredCommand));
+
+	if (!psDeferredCommand)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Deferring a KCCB command failed: allocation failure: requesting retry"));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeferredCommand->sKCCBcmd = *psKCCBCmd;
+	psDeferredCommand->uiPDumpFlags = uiPDumpFlags;
+	psDeferredCommand->psDevInfo = psDevInfo;
+
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_add_to_tail(&(psDevInfo->sKCCBDeferredCommandsListHead), &(psDeferredCommand->sListNode));
+	psDevInfo->ui32KCCBDeferredCommandsCount++;
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: _FreeDeferredCommand
+
+ PURPOSE	: Remove from the deferred list the sent deferred KCCB command
+
+ PARAMETERS	: psNode			Node in deferred list
+			: psDeferredKCCBCmd	KCCB Command to free
+
+ RETURNS	: None
+******************************************************************************/
+static void _FreeDeferredCommand(DLLIST_NODE *psNode, RGX_DEFERRED_KCCB_CMD *psDeferredKCCBCmd)
+{
+	dllist_remove_node(psNode);
+	psDeferredKCCBCmd->psDevInfo->ui32KCCBDeferredCommandsCount--;
+	OSFreeMem(psDeferredKCCBCmd);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXSendCommandsFromDeferredList
+
+ PURPOSE	: Try send KCCB commands in deferred list to KCCB
+		  Should be called by holding PowerLock
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: bPoll		Poll for space in KCCB
+
+ RETURNS	: PVRSRV_OK	If all commands in deferred list are sent to KCCB,
+			  PVRSRV_ERROR_KERNEL_CCB_FULL otherwise.
+******************************************************************************/
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psNode, *psNext;
+	RGX_DEFERRED_KCCB_CMD *psTempDeferredKCCBCmd;
+	DLLIST_NODE sCommandList;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDevInfo->psDeviceNode));
+
+	/* !!! Important !!!
+	 *
+	 * The idea of moving the whole list hLockKCCBDeferredCommandsList below
+	 * to the temporary list is only valid under the principle that all of the
+	 * operations are also protected by the power lock. It must be held
+	 * so that the order of the commands doesn't get messed up while we're
+	 * performing the operations on the local list.
+	 *
+	 * The necessity of releasing the hLockKCCBDeferredCommandsList comes from
+	 * the fact that _FreeDeferredCommand() is allocating memory and it can't
+	 * be done in atomic context (inside section protected by a spin lock).
+	 *
+	 * We're using spin lock here instead of mutex to quickly perform a check
+	 * if the list is empty in MISR without a risk that the MISR is going
+	 * to sleep due to a lock.
+	 */
+
+	/* move the whole list to a local list so it can be processed without lock */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_replace_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (dllist_is_empty(&sCommandList))
+		{
+			return PVRSRV_OK;
+		}
+
+		/* For every deferred KCCB command, try to send it*/
+		dllist_foreach_node(&sCommandList, psNode, psNext)
+		{
+			psTempDeferredKCCBCmd = IMG_CONTAINER_OF(psNode, RGX_DEFERRED_KCCB_CMD, sListNode);
+			eError = RGXSendCommandRaw(psTempDeferredKCCBCmd->psDevInfo,
+			                           &psTempDeferredKCCBCmd->sKCCBcmd,
+			                           psTempDeferredKCCBCmd->uiPDumpFlags,
+			                           NULL /* We surely aren't interested in kCCB slot number of deferred command */);
+			if (eError != PVRSRV_OK)
+			{
+				if (!bPoll)
+				{
+					eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+					goto cleanup_;
+				}
+				break;
+			}
+
+			_FreeDeferredCommand(psNode, psTempDeferredKCCBCmd);
+		}
+
+		if (bPoll)
+		{
+			PVRSRV_ERROR eErrPollForKCCBSlot;
+
+			/* Don't overwrite eError because if RGXPollKernelCCBSlot returns OK and the
+			 * outer loop times-out, we'll still want to return KCCB_FULL to caller
+			 */
+			eErrPollForKCCBSlot = RGXPollKernelCCBSlot(psDevInfo);
+			if (eErrPollForKCCBSlot == PVRSRV_ERROR_KERNEL_CCB_FULL)
+			{
+				eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+				goto cleanup_;
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+cleanup_:
+	/* if the local list is not empty put it back to the deferred list head
+	 * so that the old order of commands is retained */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_insert_list_at_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+										  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+										  IMG_UINT32          uiPDumpFlags,
+										  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	IMG_BOOL     bPoll = (pui32CmdKCCBSlot != NULL);
+	PVRSRV_ERROR eError;
+
+	/*
+	 * First try to Flush all the cmds in deferred list.
+	 *
+	 * We cannot defer an incoming command if the caller is interested in
+	 * knowing the command's kCCB slot: it plans to poll/wait for a
+	 * response from the FW just after the command is enqueued, so we must
+	 * poll for space to be available.
+	 */
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, bPoll);
+	if (eError == PVRSRV_OK)
+	{
+		eError = RGXSendCommandRaw(psDevInfo,
+								   psKCCBCmd,
+								   uiPDumpFlags,
+								   pui32CmdKCCBSlot);
+	}
+
+	/*
+	 * If we don't manage to enqueue one of the deferred commands or the command
+	 * passed as argument because the KCCB is full, insert the latter into the deferred commands list.
+	 * The deferred commands will also be flushed eventually by:
+	 *  - one more KCCB command sent for any DM
+	 *  - RGX_MISRHandler_CheckFWActivePowerState
+	 */
+	if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+	{
+		if (pui32CmdKCCBSlot == NULL)
+		{
+			eError = _AllocDeferredCommand(psDevInfo, psKCCBCmd, uiPDumpFlags);
+		}
+		else
+		{
+			/* Let the caller retry. Otherwise if we deferred the command and returned OK,
+			 * the caller can end up looking in a stale CCB slot.
+			 */
+			PVR_DPF((PVR_DBG_WARNING, "%s: Couldn't flush the deferred queue for a command (Type:%d) "
+			                        "- will be retried", __func__, psKCCBCmd->eCmdType));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32         *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+
+	/* Ensure Rogue is powered up before kicking MTS */
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: failed to acquire powerlock (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVPowerLock_Exit;
+	}
+
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition Rogue to ON (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+										  psKCCBCmd,
+										  ui32PDumpFlags,
+	                                      pui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to schedule command (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+#if defined(DEBUG)
+		/* PVRSRVDebugRequest must be called without powerlock */
+		PVRSRVPowerUnlock(psDeviceNode);
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+		goto _PVRSRVPowerLock_Exit;
+#endif
+	}
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_PVRSRVPowerLock_Exit:
+	return eError;
+}
+
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hCmdCompHandle;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSScheduleMISR(psDevInfo->hProcessQueuesMISR);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGX_MISRHandler_ScheduleProcessQueues
+
+ @Description - Sends uncounted kick to all the DMs (the FW will process all
+				the queue for all the DMs)
+******************************************************************************/
+static void RGX_MISRHandler_ScheduleProcessQueues(void *pvData)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO     *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR           eError;
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_OFF;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guests are not permitted to change the device power state */
+		if ((eError != PVRSRV_OK) || (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+	else
+	{
+		if ((eError == PVRSRV_OK) && (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			RGXFWIF_GPU_UTIL_FW    *psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+			IMG_BOOL               bGPUHasWorkWaiting;
+
+			/* Check whether it's worth waking up the GPU */
+			RGXFwSharedMemCacheOpValue(psUtilFW->ui64GpuLastWord, INVALIDATE);
+			bGPUHasWorkWaiting =
+			    (RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord) == RGXFWIF_GPU_UTIL_STATE_BLOCKED);
+
+			if (!bGPUHasWorkWaiting)
+			{
+				/* all queues are empty, don't wake up the GPU */
+				PVRSRVPowerUnlock(psDeviceNode);
+				return;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		/* wake up the GPU */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition device to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+
+	/* Memory barrier before Kick MTS */
+	OSWriteMemoryBarrier(NULL);
+
+	/* uncounted kick to the FW */
+	HTBLOGK(HTB_SF_MAIN_KICK_UNCOUNTED);
+	__MTSScheduleWrite(psDevInfo, (RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return OSInstallMISR(phMISR,
+			RGX_MISRHandler_ScheduleProcessQueues,
+			psDeviceNode,
+			"RGX_ScheduleProcessQueues");
+}
+
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+                                              RGXFWIF_DM          eKCCBType,
+                                              RGXFWIF_KCCB_CMD    *psKCCBCmd,
+                                              IMG_UINT32          ui32PDumpFlags,
+                                              IMG_UINT32          *pui32CmdKCCBSlot,
+                                              IMG_BOOL            bCallerHasPwrLock)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiMMUSyncUpdate;
+
+	/* Don't send the command/power up request if device not available. */
+	if (unlikely((psDevInfo == NULL) ||
+	             (psDevInfo->psDeviceNode == NULL) ||
+				 (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)))
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	/* Don't send the command/power up request if device in deinit phase.
+	 * The de-init thread could destroy the device whilst the power up
+	 * sequence below is accessing the HW registers.
+	 * Not yet safe to free resources. Caller should retry later.
+	 */
+	if (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	    psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+
+	if (!bCallerHasPwrLock)
+	{
+		/* PVRSRVPowerLock guarantees atomicity between commands. This is helpful
+		   in a scenario with several applications allocating resources. */
+		eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+
+			/* If system is found powered OFF, Retry scheduling the command */
+			if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+			{
+				eError = PVRSRV_ERROR_RETRY;
+			}
+
+			goto RGXScheduleCommand_exit;
+		}
+	}
+
+	if (unlikely(psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	             psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING))
+	{
+		/* If we have the power lock the device is valid but the deinit
+		 * thread could be waiting for the lock. */
+		eError = PVRSRV_ERROR_RETRY;
+		goto _PVRSRVInvalidDeviceError_Exit;
+	}
+
+	/* Ensure device is powered up before sending any commands */
+	PDUMPPOWCMDSTART(psDevInfo->psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDevInfo->psDeviceNode,
+	                                     PVRSRV_DEV_POWER_STATE_ON,
+	                                     PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+
+	eError = RGXPreKickCacheCommand(psDevInfo, eKCCBType, &uiMMUSyncUpdate);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+_PVRSRVInvalidDeviceError_Exit:
+	if (!bCallerHasPwrLock)
+	{
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+	}
+RGXScheduleCommand_exit:
+	return eError;
+}
+
+/*
+ * RGXCheckFirmwareCCB
+ */
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT8 *psFWCCB = psDevInfo->psFirmwareCCB;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo) ||
+								 (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+								  (KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))),
+								 "FW-KM connection is down");
+#endif
+
+	eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+		return;
+	}
+
+	while (psFWCCBCtlLocal->ui32ReadOffset != psFWCCBCtlLocal->ui32WriteOffset)
+	{
+		/* Point to the next command */
+		const RGXFWIF_FWCCB_CMD *psFwCCBCmd = ((RGXFWIF_FWCCB_CMD *)psFWCCB) + psFWCCBCtlLocal->ui32ReadOffset;
+		RGXFwSharedMemCacheOpPtr(psFwCCBCmd, INVALIDATE);
+
+
+		HTBLOGK(HTB_SF_MAIN_FWCCB_CMD, psFwCCBCmd->eCmdType);
+		switch (psFwCCBCmd->eCmdType)
+		{
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_BACKING, "Request to add backing to ZSBuffer");
+				}
+				RGXProcessRequestZSBufferBacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_UNBACKING, "Request to remove backing from ZSBuffer");
+				}
+				RGXProcessRequestZSBufferUnbacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELIST_GROW:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELIST_GROW, "Request to grow the free list");
+				}
+				RGXProcessRequestGrow(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELISTS_RECONSTRUCTION, "Request to reconstruct free lists");
+				}
+
+				if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d/%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psDevInfo->psRGXFWIfHWRInfoBufCtl->ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+
+				RGXProcessRequestFreelistsReconstruction(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.aui32FreelistIDs);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION:
+			{
+				/* Notify client drivers */
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+					const RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA *psCmdFwPagefault =
+							&psFwCCBCmd->uCmdData.sCmdFWPagefault;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+						sErrorData.uErrData.sFwPFErrData.sFWFaultAddr.uiAddr = psCmdFwPagefault->sFWFaultAddr.uiAddr;
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+														  &sErrorData);
+					}
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
+			{
+				const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification =
+						&psFwCCBCmd->uCmdData.sCmdContextResetNotification;
+				IMG_UINT32 ui32ErrorPid = 0;
+
+				FWCommonContextListSetLastResetReason(psDevInfo,
+				                                      &ui32ErrorPid,
+				                                      psCmdContextResetNotification);
+
+				/* Increment error counter (if appropriate) */
+				if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32WGPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32WGPErrorCount++;
+					}
+				}
+				else if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32TRPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32TRPErrorCount++;
+					}
+				}
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = psCmdContextResetNotification->eResetReason;
+						sErrorData.pid = ui32ErrorPid;
+
+						/* Populate error data according to reset reason */
+						switch (psCmdContextResetNotification->eResetReason)
+						{
+							case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+							case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+							{
+								sErrorData.uErrData.sChecksumErrData.ui32ExtJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+								sErrorData.uErrData.sChecksumErrData.eDM = psCmdContextResetNotification->eDM;
+								break;
+							}
+							default:
+							{
+								break;
+							}
+						}
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+						                                  &sErrorData);
+					}
+				}
+
+				/* Notify if a page fault */
+				if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF)
+				{
+					DevmemIntPFNotify(psDevInfo->psDeviceNode,
+							psCmdContextResetNotification->ui64PCAddress,
+							psCmdContextResetNotification->sFaultAddress);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_DEBUG_DUMP:
+			{
+				PVRSRV_ERROR eError;
+				PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+				OSAtomicWrite(&psDevInfo->psDeviceNode->eDebugDumpRequested, PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE);
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal FW Cmd debug dump event, dumping now instead", __func__));
+					PVRSRVDebugRequest(psDevInfo->psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_UPDATE_STATS:
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				IMG_PID pidTmp = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.pidOwner;
+				IMG_INT32 i32AdjustmentValue = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.i32AdjustmentValue;
+
+				switch (psFwCCBCmd->uCmdData.sCmdUpdateStatsData.eElementToUpdate)
+				{
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,i32AdjustmentValue,0,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,i32AdjustmentValue,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,i32AdjustmentValue,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,i32AdjustmentValue,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,i32AdjustmentValue,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,i32AdjustmentValue,0,pidTmp);
+						break;
+					}
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_RAY_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,0,i32AdjustmentValue,pidTmp);
+						break;
+					}
+#endif
+				}
+#endif
+				break;
+			}
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+			case RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE:
+			{
+				RGX_PROCESS_CORE_CLK_RATE_CHANGE(psDevInfo,
+											  psFwCCBCmd->uCmdData.sCmdCoreClkRateChange.ui32CoreClkRate);
+				break;
+			}
+#endif
+			case RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART:
+			{
+				RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+				if (psDevInfo->psRGXFWIfFwSysData != NULL  &&
+				    psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)
+				{
+					PVRSRV_ERROR eError;
+
+					/* Power down... */
+					eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+															 PVRSRV_SYS_POWER_STATE_OFF,
+															 PVRSRV_POWER_FLAGS_NONE);
+					if (eError == PVRSRV_OK)
+					{
+						/* Clear the FW faulted flags... */
+						psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags &= ~(RGXFWIF_HWR_FW_FAULT|RGXFWIF_HWR_RESTART_REQUESTED);
+						OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags);
+						RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags,
+						                           FLUSH);
+
+						/* Power back up again... */
+						eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+																 PVRSRV_SYS_POWER_STATE_ON,
+																 PVRSRV_POWER_FLAGS_NONE);
+
+						/* Send a dummy KCCB command to ensure the FW wakes up and checks the queues... */
+						if (eError == PVRSRV_OK)
+						{
+							LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+							{
+								eError = RGXFWHealthCheckCmd(psDevInfo);
+								if (eError != PVRSRV_ERROR_RETRY)
+								{
+									break;
+								}
+								OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+							} END_LOOP_UNTIL_TIMEOUT_US();
+						}
+					}
+
+					/* Notify client drivers and system layer of FW fault */
+					{
+						PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+						PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+						/* Client notification of device error will be achieved by
+						 * clients calling UM function RGXGetLastDeviceError() */
+						psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+
+						/* Notify system layer */
+						if (psDevConfig->pfnSysDevErrorNotify)
+						{
+							PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+							sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+							psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+							                                  &sErrorData);
+						}
+					}
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Failed firmware restart (%s)",
+								 __func__, PVRSRVGetErrorString(eError)));
+					}
+				}
+				break;
+			}
+			default:
+			{
+				/* unknown command */
+				PVR_DPF((PVR_DBG_WARNING, "%s: Unknown Command (eCmdType=0x%08x)",
+				         __func__, psFwCCBCmd->eCmdType));
+				/* Assert on magic value corruption */
+				PVR_ASSERT((((IMG_UINT32)psFwCCBCmd->eCmdType & RGX_CMD_MAGIC_DWORD_MASK) >> RGX_CMD_MAGIC_DWORD_SHIFT) == RGX_CMD_MAGIC_DWORD);
+			}
+		}
+
+		/* Update read offset */
+		psFWCCBCtlLocal->ui32ReadOffset = (psFWCCBCtlLocal->ui32ReadOffset + 1) & psFWCCBCtlLocal->ui32WrapMask;
+		OSMemoryBarrier(NULL);
+		psFWCCBCtl->ui32ReadOffset = psFWCCBCtlLocal->ui32ReadOffset;
+		OSWriteMemoryBarrier(NULL);
+
+		if (psFWCCBCtlLocal->ui32ReadOffset == psFWCCBCtlLocal->ui32WriteOffset)
+		{
+			eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+				return;
+			}
+		}
+	}
+}
+
+/*
+ * PVRSRVRGXFrameworkCopyCommand
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+		DEVMEM_MEMDESC	*psFWFrameworkMemDesc,
+		IMG_PBYTE		pbyGPUFRegisterList,
+		IMG_UINT32		ui32FrameworkRegisterSize)
+{
+	PVRSRV_ERROR	eError;
+	RGXFWIF_RF_REGISTERS	*psRFReg;
+
+	eError = DevmemAcquireCpuVirtAddr(psFWFrameworkMemDesc,
+			(void **)&psRFReg);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	OSDeviceMemCopy(psRFReg, pbyGPUFRegisterList, ui32FrameworkRegisterSize);
+	RGXFwSharedMemCacheOpPtr(psRFReg, FLUSH);
+
+	/* Release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psFWFrameworkMemDesc);
+
+	/*
+	 * Dump the FW framework buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FWFramework buffer");
+	DevmemPDumpLoadMem(psFWFrameworkMemDesc, 0, ui32FrameworkRegisterSize, PDUMP_FLAGS_CONTINUOUS);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXFrameworkCreateKM
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE	*psDeviceNode,
+		DEVMEM_MEMDESC		**ppsFWFrameworkMemDesc,
+		IMG_UINT32			ui32FrameworkCommandSize)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+		Allocate device memory for the firmware GPU framework state.
+		Sufficient info to kick one or more DMs should be contained in this buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware framework state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+			ui32FrameworkCommandSize,
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwGPUFrameworkState",
+			ppsFWFrameworkMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware framework state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem *pui32LinMemAddr,
+												IMG_UINT32			ui32Value,
+												IMG_UINT32			ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32CurrentQueueLength, ui32MaxRetries;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDevNode->pvDevice;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+	ui32CurrentQueueLength = (psKCCBCtlLocal->ui32WrapMask+1 +
+					psKCCBCtlLocal->ui32WriteOffset -
+					psKCCBCtlLocal->ui32ReadOffset) & psKCCBCtlLocal->ui32WrapMask;
+	ui32CurrentQueueLength += psDevInfo->ui32KCCBDeferredCommandsCount;
+
+	for (ui32MaxRetries = ui32CurrentQueueLength + 1;
+				ui32MaxRetries > 0;
+				ui32MaxRetries--)
+	{
+
+		/*
+		 * PVRSRVPollForValueKM flags are set to POLL_FLAG_NONE in this case so that the function
+		 * does not generate an error message. In this case, the PollForValueKM is expected to
+		 * timeout as there is work ongoing on the GPU which may take longer than the timeout period.
+		 */
+		eError = PVRSRVPollForValueKM(psDevNode, pui32LinMemAddr, ui32Value, ui32Mask, POLL_FLAG_NONE, NULL);
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			break;
+		}
+
+		RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		         __func__, PVRSRVGetErrorString(eError),
+		         pui32LinMemAddr, ui32Value));
+		OSDumpStack();
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32ConfigState,
+				IMG_BOOL bSetNotClear)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	RGXFWIF_KCCB_CMD sStateFlagCmd = { 0 };
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_SYSDATA *psFwSysData;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (NULL == psFwSysData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Fw Sys Config is not mapped into CPU space", __func__));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* apply change and ensure the new data is written to memory
+	 * before requesting the FW to read it
+	 */
+	ui32Config = ui32Config & RGXFWIF_INICFG_ALL;
+	if (bSetNotClear)
+	{
+		psFwSysData->ui32ConfigFlags |= ui32Config;
+	}
+	else
+	{
+		psFwSysData->ui32ConfigFlags &= ~ui32Config;
+	}
+	OSWriteMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+	RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags, FLUSH);
+
+	/* return current/new value to caller */
+	if (pui32ConfigState)
+	{
+		*pui32ConfigState = psFwSysData->ui32ConfigFlags;
+	}
+
+	OSMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	/* notify FW to update setting */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of the value */
+		sStateFlagCmd.eCmdType = RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sStateFlagCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock(psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the value to be updated as the FW validates
+		 * the parameters and modifies the ui32ConfigFlags
+		 * accordingly
+		 * (for completeness as registered callbacks should also
+		 *  not permit invalid transitions)
+		 */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXScheduleCleanupCommand(PVRSRV_RGXDEV_INFO	*psDevInfo,
+									   RGXFWIF_DM			eDM,
+									   RGXFWIF_KCCB_CMD		*psKCCBCmd,
+									   RGXFWIF_CLEANUP_TYPE	eCleanupType,
+									   IMG_UINT32			ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32kCCBCommandSlot;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus);
+
+	PVR_LOG_RETURN_IF_FALSE((eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+							(eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT),
+							"Cleanup aborted: Device in bad state", PVRSRV_OK);
+#endif
+
+
+	/* Clean-up commands sent during frame capture intervals must be dumped even when not in capture range... */
+	ui32PDumpFlags |= PDUMP_FLAGS_INTERVAL;
+
+	psKCCBCmd->eCmdType = RGXFWIF_KCCB_CMD_CLEANUP;
+	psKCCBCmd->uCmdData.sCleanupData.eCleanupType = eCleanupType;
+
+	/*
+		Send the cleanup request to the firmware. If the resource is still busy
+		the firmware will tell us and we'll drop out with a retry.
+	*/
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+											  eDM,
+											  psKCCBCmd,
+											  ui32PDumpFlags,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR ,"RGXScheduleCommandAndGetKCCBSlot() failed (%s) in %s()",
+			         PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		goto fail_command;
+	}
+
+	/* Wait for command kCCB slot to be updated by FW */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+						  "Wait for the firmware to reply to the cleanup command");
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot,
+									  ui32PDumpFlags);
+	/*
+		If the firmware hasn't got back to us in a timely manner
+		then bail and let the caller retry the command.
+	 */
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: RGXWaitForKCCBSlotUpdate timed out. Dump debug information.",
+		         __func__));
+
+		eError = PVRSRV_ERROR_RETRY;
+#if defined(DEBUG)
+		PVRSRVDebugRequest(psDevInfo->psDeviceNode,
+				DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+#endif
+		goto fail_poll;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		goto fail_poll;
+	}
+
+#if defined(PDUMP)
+	/*
+	 * The cleanup request to the firmware will tell us if a given resource is busy or not.
+	 * If the RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY flag is set, this means that the resource is
+	 * still in use. In this case we return a PVRSRV_ERROR_RETRY error to the client drivers
+	 * and they will re-issue the cleanup request until it succeed.
+	 *
+	 * Since this retry mechanism doesn't work for pdumps, client drivers should ensure
+	 * that cleanup requests are only submitted if the resource is unused.
+	 * If this is not the case, the following poll will block infinitely, making sure
+	 * the issue doesn't go unnoticed.
+	 */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+					"Cleanup: If this poll fails, the following resource is still in use (DM=%u, type=%u, address=0x%08x), which is incorrect in pdumps",
+					eDM,
+					psKCCBCmd->uCmdData.sCleanupData.eCleanupType,
+					psKCCBCmd->uCmdData.sCleanupData.uCleanupData.psContext.ui32Addr);
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+									ui32kCCBCommandSlot * sizeof(IMG_UINT32),
+									0,
+									RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+#endif
+
+	/*
+		If the command has was run but a resource was busy, then the request
+		will need to be retried.
+	*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot],
+	                           INVALIDATE);
+
+	if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY))
+	{
+		if (psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+		eError = PVRSRV_ERROR_RETRY;
+		goto fail_requestbusy;
+	}
+
+	return PVRSRV_OK;
+
+fail_requestbusy:
+fail_poll:
+fail_command:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	RGXRequestCommonContextCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD			sRCCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+	PRGXFWIF_FWCOMMONCONTEXT	psFWCommonContextFWAddr;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	/* Force retry if this context's CCB is currently being dumped
+	 * as part of the stalled CCB debug */
+	if (psDevInfo->pvEarliestStalledClientCCB == (void*)FWCommonContextGetClientCCB(psServerCommonContext))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: [%u/%d] Forcing retry as psDevInfo->pvEarliestStalledClientCCB = psCtxClientCCB <%p>",
+		         __func__,
+		         psDeviceNode->sDevId.ui32InternalID,
+		         psDeviceNode->sDevId.i32KernelDeviceID,
+		         psDevInfo->pvEarliestStalledClientCCB));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psFWCommonContextFWAddr = FWCommonContextGetFWAddress(psServerCommonContext);
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Common ctx cleanup Request DM%d [context = 0x%08x]",
+			eDM, psFWCommonContextFWAddr.ui32Addr);
+	PDUMPCOMMENT(psDeviceNode, "Wait for CCB to be empty before common ctx cleanup");
+
+	RGXCCBPDumpDrainCCB(FWCommonContextGetClientCCB(psServerCommonContext), ui32PDumpFlags);
+#endif
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sRCCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psContext = psFWCommonContextFWAddr;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+									   eDM,
+									   &sRCCleanUpCmd,
+									   RGXFWIF_CLEANUP_FWCOMMONCONTEXT,
+									   ui32PDumpFlags);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+/*
+ * RGXFWRequestHWRTDataCleanUp
+ */
+
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         PRGXFWIF_HWRTDATA psHWRTData)
+{
+	RGXFWIF_KCCB_CMD			sHWRTDataCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDeviceNode, "HW RTData cleanup Request [HWRTData = 0x%08x]", psHWRTData.ui32Addr);
+
+	sHWRTDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWRTData = psHWRTData;
+
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+	                                   RGXFWIF_DM_GP,
+	                                   &sHWRTDataCleanUpCmd,
+	                                   RGXFWIF_CLEANUP_HWRTDATA,
+	                                   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a HWRTData cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestFreeListCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_FREELIST psFWFreeList)
+{
+	RGXFWIF_KCCB_CMD			sFLCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free list cleanup Request [FreeList = 0x%08x]", psFWFreeList.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psFreelist = psFWFreeList;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_GP,
+									   &sFLCleanUpCmd,
+									   RGXFWIF_CLEANUP_FREELIST,
+									   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a memory context cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestZSBufferCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer)
+{
+	RGXFWIF_KCCB_CMD			sZSBufferCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "ZS Buffer cleanup Request [ZS Buffer = 0x%08x]", psFWZSBuffer.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sZSBufferCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psZSBuffer = psFWZSBuffer;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_3D,
+									   &sZSBufferCleanUpCmd,
+									   RGXFWIF_CLEANUP_ZSBUFFER,
+									   PDUMP_FLAGS_NONE);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS = ui32HCSDeadlineMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Hard Context Switching deadline inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32HCSDeadlineMS),
+							  ui32HCSDeadlineMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock)
+{
+	RGXFWIF_KCCB_CMD	sCmpKCCBCmd = { 0 };
+
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+	if (bCallerHasPwrLock)
+	{
+		return	RGXScheduleCommandWithoutPowerLock(psDevInfo,
+												   RGXFWIF_DM_GP,
+												   &sCmpKCCBCmd,
+												   PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		return	RGXScheduleCommand(psDevInfo,
+								   RGXFWIF_DM_GP,
+								   &sCmpKCCBCmd,
+								   PDUMP_FLAGS_CONTINUOUS);
+	}
+}
+
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32CBaseMapCtxReg;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+	else
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+#else
+	/*
+	 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+	 * FW crash from its memory context
+	 */
+	RGXWriteKernelMMUPC64(&psDevInfo->sLayerParams,
+				                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+				                      ((0xDEADBEEF
+				                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+				                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+				                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               IMG_UINT32 ui32DriverID,
+                               RGXFWIF_OS_STATE_CHANGE eOSOnlineState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sOSOnlineStateCmd = { 0 };
+	const RGXFWIF_SYSDATA *psFwSysData;
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	sOSOnlineStateCmd.eCmdType = RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.ui32DriverID = ui32DriverID;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = eOSOnlineState;
+
+#if defined(SUPPORT_AUTOVZ)
+	{
+		IMG_BOOL bConnectionDown = IMG_FALSE;
+
+		PVR_UNREFERENCED_PARAMETER(psFwSysData);
+		sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = RGXFWIF_OS_OFFLINE;
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			/* Send the offline command regardless if power lock is held or not.
+			 * Under AutoVz this is done during regular driver deinit, store-to-ram suspend
+			 * or (optionally) from a kernel panic callback. Deinit and suspend operations
+			 * take the lock in the rgx pre/post power functions as expected.
+			 * The kernel panic callback is a last resort way of letting the firmware know that
+			 * the VM is unrecoverable and the vz connection must be disabled. It cannot wait
+			 * on other kernel threads to finish and release the lock. */
+			eError = RGXSendCommand(psDevInfo,
+									&sOSOnlineStateCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Guests and Host going offline should wait for confirmation
+		 * from the Firmware of the state change. If this fails, break
+		 * the connection on the OS Driver's end as backup. */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (ui32DriverID == RGXFW_HOST_DRIVER_ID))
+		{
+			LOOP_UNTIL_TIMEOUT_US(SECONDS_TO_MICROSECONDS/2)
+			{
+				KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+				if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+				{
+					bConnectionDown = IMG_TRUE;
+					break;
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (!bConnectionDown)
+			{
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+		}
+	}
+#else
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* no reason for Guests to update their state or any other VM's.
+		 * This is the Hypervisor and Host driver's responsibility. */
+		return PVRSRV_OK;
+	}
+	else
+	{
+		const volatile RGXFWIF_OS_RUNTIME_FLAGS *psFwRunFlags;
+
+		PVR_ASSERT(psFwSysData != NULL);
+
+		psFwRunFlags = (const volatile RGXFWIF_OS_RUNTIME_FLAGS*) &psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+		/* Attempt several times until the FW manages to offload the OS */
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			IMG_UINT32 ui32kCCBCommandSlot;
+
+			/* Send request */
+			eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+													  RGXFWIF_DM_GP,
+													  &sOSOnlineStateCmd,
+													  PDUMP_FLAGS_CONTINUOUS,
+													  &ui32kCCBCommandSlot);
+			if (unlikely(eError == PVRSRV_ERROR_RETRY)) continue;
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommand", return_);
+
+			/* Wait for FW to process the cmd */
+			eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", return_);
+
+			/* read the OS state */
+			OSMemoryBarrier(NULL);
+			/* check if FW finished offloading the driver and is stopped */
+			if ((eOSOnlineState == RGXFWIF_OS_ONLINE && (psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_READY || psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_ACTIVE)) ||
+			    (eOSOnlineState == RGXFWIF_OS_OFFLINE && psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_OFFLINE))
+			{
+				eError = PVRSRV_OK;
+				break;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_TIMEOUT;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+
+return_ :
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgPHRCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE((ui32PHRMode == RGXFWIF_PHR_MODE_OFF) ||
+							(ui32PHRMode == RGXFWIF_PHR_MODE_RD_RESET),
+							"Invalid PHR Mode.", PVRSRV_ERROR_INVALID_PARAMS);
+
+	sCfgPHRCmd.eCmdType = RGXFWIF_KCCB_CMD_PHR_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode = ui32PHRMode;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Periodic Hardware Reset Mode inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32PHRMode),
+							  ui32PHRMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GP,
+		                            &sCfgPHRCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+							IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgWdgCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCfgWdgCmd.eCmdType = RGXFWIF_KCCB_CMD_WDG_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs = ui32WdgPeriodUs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the firmware watchdog period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32WdgPeriodUs),
+							  ui32WdgPeriodUs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sCfgWdgCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious)
+{
+	/* Attempt to detect and deal with any stalled client contexts.
+	 * bIgnorePrevious may be set by the caller if they know a context to be
+	 * stalled, as otherwise this function will only identify stalled
+	 * contexts which have not been previously reported.
+	 */
+
+	IMG_UINT32 ui32StalledClientMask = 0;
+
+	if (!(OSTryLockAcquire(psDevInfo->hCCBStallCheckLock)))
+	{
+		PVR_LOG(("RGXCheckForStalledClientContexts: Failed to acquire hCCBStallCheckLock, returning..."));
+		return;
+	}
+
+	ui32StalledClientMask |= CheckForStalledClientTDMTransferCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientRenderCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientComputeCtxt(psDevInfo);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	ui32StalledClientMask |= CheckForStalledClientKickSyncCtxt(psDevInfo);
+#endif
+	/* If at least one DM stalled bit is different than before */
+	if (bIgnorePrevious || (psDevInfo->ui32StalledClientMask != ui32StalledClientMask))
+	{
+		if (ui32StalledClientMask > 0)
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Print all the stalled DMs */
+			PVR_LOG(("Possible stalled client RGX contexts detected: %s%s%s%s%s%s%s",
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_GP),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TDM_2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TA),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_3D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_CDM),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ3D)));
+
+			PVR_LOG(("Trying to identify stalled context...(%s) [%d]",
+			         pszStalledAction, bIgnorePrevious));
+
+			DumpStalledContextInfo(psDevInfo);
+		}
+		else
+		{
+			if (psDevInfo->ui32StalledClientMask> 0)
+			{
+				/* Indicate there are no stalled DMs */
+				PVR_LOG(("No further stalled client contexts exist"));
+			}
+		}
+		psDevInfo->ui32StalledClientMask = ui32StalledClientMask;
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+	OSLockRelease(psDevInfo->hCCBStallCheckLock);
+}
+
+/*
+	RGXUpdateHealthStatus
+*/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed)
+{
+	const PVRSRV_DATA*           psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_HEALTH_STATUS  eNewStatus   = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	PVRSRV_DEVICE_HEALTH_REASON  eNewReason   = PVRSRV_DEVICE_HEALTH_REASON_NONE;
+	PVRSRV_RGXDEV_INFO*          psDevInfo;
+	const RGXFWIF_TRACEBUF*      psRGXFWIfTraceBufCtl;
+	const RGXFWIF_SYSDATA*       psFwSysData;
+	const RGXFWIF_OSDATA*        psFwOsData;
+	const RGXFWIF_CCB_CTL*       psKCCBCtl;
+	RGXFWIF_CCB_CTL*             psKCCBCtlLocal;
+	IMG_UINT32                   ui32ThreadCount;
+	IMG_BOOL                     bKCCBCmdsWaiting;
+	PVRSRV_ERROR                 eError;
+
+	PVR_ASSERT(psDevNode != NULL);
+	psDevInfo = psDevNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, PVRSRV_OK);
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_OK;
+	}
+
+	psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData,
+								 INVALIDATE);
+		psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         INVALIDATE);
+	psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+
+	/* If this is a quick update, then include the last current value... */
+	if (!bCheckAfterTimePassed)
+	{
+		eNewStatus = OSAtomicRead(&psDevNode->eHealthStatus);
+		eNewReason = OSAtomicRead(&psDevNode->eHealthReason);
+	}
+
+	/* Decrement the SLR holdoff counter (if non-zero) */
+	if (psDevInfo->ui32SLRHoldoffCounter > 0)
+	{
+		psDevInfo->ui32SLRHoldoffCounter--;
+	}
+
+	/* Take power lock, retry if it's in use in another task. */
+	eError = PVRSRVPowerTryLockWaitForTimeout(psDevNode);
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		/* Skip health status update if timeout */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Power lock timeout, increase OS_POWERLOCK_TIMEOUT_US.", __func__));
+		goto _RGXUpdateHealthStatus_Exit;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerTryLockWaitForTimeout");
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+		return PVRSRV_OK;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		/* On a PCI error all reads from the PCI bar may return 0xFFFFFFFF.
+		   This value is not valid for a core ID. */
+		if (psFwSysData->ui32MemFaultCheck == RGX_PCI_ERROR_VALUE_DWORD)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: PCI error", __func__));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR;
+			PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_PCI_ERROR);
+			PVRSRVPowerUnlock(psDevNode);
+			goto _RGXUpdateHealthStatus_Exit;
+		}
+	}
+
+	/* If Rogue is not powered on, just skip ahead and check for stalled client CCBs */
+	if (PVRSRVIsDevicePowered(psDevNode))
+	{
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			/*
+			   Firmware thread checks...
+			 */
+			for (ui32ThreadCount = 0; ui32ThreadCount < RGXFW_THREAD_NUM; ui32ThreadCount++)
+			{
+				const IMG_CHAR* pszTraceAssertInfo;
+
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf, INVALIDATE);
+				pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szInfo;
+
+				/*
+				Check if the FW has hit an assert...
+				*/
+				if (*pszTraceAssertInfo != '\0')
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: Firmware thread %d has asserted: %.*s (%.*s:%d)",
+							__func__, ui32ThreadCount, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							pszTraceAssertInfo, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szPath,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.ui32LineNum));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_ASSERTED;
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+				{
+					/*
+					   Check the threads to see if they are in the same poll locations as last time...
+					*/
+					if (bCheckAfterTimePassed)
+					{
+						if (psFwSysData->aui32CrPollAddr[ui32ThreadCount] != 0  &&
+							psFwSysData->aui32CrPollCount[ui32ThreadCount] == psDevInfo->aui32CrLastPollCount[ui32ThreadCount])
+						{
+							PVR_DPF((PVR_DBG_WARNING, "%s: Firmware stuck on CR poll: T%u polling %s (reg:0x%08X mask:0x%08X)",
+									__func__, ui32ThreadCount,
+									((psFwSysData->aui32CrPollAddr[ui32ThreadCount] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+									psFwSysData->aui32CrPollAddr[ui32ThreadCount] & ~RGXFW_POLL_TYPE_SET,
+									psFwSysData->aui32CrPollMask[ui32ThreadCount]));
+							eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+							eNewReason = PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING;
+							PVRSRVPowerUnlock(psDevNode);
+							goto _RGXUpdateHealthStatus_Exit;
+						}
+						psDevInfo->aui32CrLastPollCount[ui32ThreadCount] = psFwSysData->aui32CrPollCount[ui32ThreadCount];
+					}
+				}
+			}
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+			{
+				/*
+				Check if the FW has faulted...
+				*/
+				if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_FW_FAULT)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+							"%s: Firmware has faulted and needs to restart",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_FAULT;
+					if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_RESTART_REQUESTED)
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_RESTARTING;
+					}
+					else
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_IDLING;
+					}
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+			}
+		}
+
+		/*
+		   Event Object Timeouts check...
+		*/
+		if (!bCheckAfterTimePassed)
+		{
+			if (psDevInfo->ui32GEOTimeoutsLastTime > 1 && psPVRSRVData->ui32GEOConsecutiveTimeouts > psDevInfo->ui32GEOTimeoutsLastTime)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Global Event Object Timeouts have risen (from %d to %d)",
+				         __func__,
+				         psDevInfo->ui32GEOTimeoutsLastTime, psPVRSRVData->ui32GEOConsecutiveTimeouts));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS;
+			}
+			psDevInfo->ui32GEOTimeoutsLastTime = psPVRSRVData->ui32GEOConsecutiveTimeouts;
+		}
+
+		/*
+		   Check the Kernel CCB pointer is valid. If any commands were waiting last time, then check
+		   that some have executed since then.
+		*/
+		bKCCBCmdsWaiting = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+		psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+		if (psKCCBCtl != NULL && psKCCBCtlLocal != NULL)
+		{
+			/* update KCCB read offset */
+			RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+			psKCCBCtlLocal->ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+			if (psKCCBCtlLocal->ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask  ||
+				psKCCBCtlLocal->ui32WriteOffset > psKCCBCtlLocal->ui32WrapMask)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: KCCB has invalid offset (ROFF=%d WOFF=%d)",
+						__func__, psKCCBCtlLocal->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+			}
+
+			if (psKCCBCtlLocal->ui32ReadOffset != psKCCBCtlLocal->ui32WriteOffset)
+			{
+				bKCCBCmdsWaiting = IMG_TRUE;
+			}
+		}
+
+		if (bCheckAfterTimePassed && psFwOsData != NULL)
+		{
+			IMG_UINT32 ui32KCCBCmdsExecuted = psFwOsData->ui32KCCBCmdsExecuted;
+
+			if (psDevInfo->ui32KCCBCmdsExecutedLastTime == ui32KCCBCmdsExecuted)
+			{
+				/*
+				   If something was waiting last time then the Firmware has stopped processing commands.
+				*/
+				if (psDevInfo->bKCCBCmdsWaitingLastTime)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: No KCCB commands executed since check!",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+				}
+
+				/*
+				   If no commands are currently pending and nothing happened since the last poll, then
+				   schedule a dummy command to ping the firmware so we know it is alive and processing.
+				*/
+				if ((!bKCCBCmdsWaiting) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+				{
+					/* Protect the PDumpLoadMem. RGXScheduleCommand() cannot take the
+					 * PMR lock itself, because some bridge functions will take the PMR lock
+					 * before calling RGXScheduleCommand
+					 */
+					PVRSRV_ERROR eError = RGXFWHealthCheckCmdWithoutPowerLock(psDevNode->pvDevice);
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Cannot schedule Health Check command! (0x%x)",
+								__func__, eError));
+					}
+					else
+					{
+						bKCCBCmdsWaiting = IMG_TRUE;
+					}
+				}
+			}
+
+			psDevInfo->bKCCBCmdsWaitingLastTime     = bKCCBCmdsWaiting;
+			psDevInfo->ui32KCCBCmdsExecutedLastTime = ui32KCCBCmdsExecuted;
+		}
+	}
+
+	/*
+	   Interrupt counts check...
+	*/
+	if (bCheckAfterTimePassed  &&  psFwOsData != NULL)
+	{
+		IMG_UINT32  ui32LISRCount   = 0;
+		IMG_UINT32  ui32FWCount     = 0;
+		IMG_UINT32  ui32MissingInts = 0;
+
+		/* Add up the total number of interrupts issued, sampled/received and missed... */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* Only the Host OS has a sample count, so only one counter to check. */
+		ui32LISRCount += psDevInfo->aui32SampleIRQCount[RGXFW_HOST_DRIVER_ID];
+		ui32FWCount   += OSReadHWReg32(psDevInfo->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[RGXFW_HOST_DRIVER_ID]);
+#else
+		IMG_UINT32  ui32Index;
+
+		for (ui32Index = 0;  ui32Index < RGXFW_THREAD_NUM;  ui32Index++)
+		{
+			ui32LISRCount += psDevInfo->aui32SampleIRQCount[ui32Index];
+			ui32FWCount   += psFwOsData->aui32InterruptCount[ui32Index];
+		}
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+		if (ui32LISRCount < ui32FWCount)
+		{
+			ui32MissingInts = (ui32FWCount-ui32LISRCount);
+		}
+
+		if (ui32LISRCount == psDevInfo->ui32InterruptCountLastTime  &&
+		    ui32MissingInts >= psDevInfo->ui32MissingInterruptsLastTime  &&
+		    psDevInfo->ui32MissingInterruptsLastTime > 1)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: LISR has not received the last %d interrupts",
+					__func__, ui32MissingInts));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+
+			/* Schedule the MISRs to help mitigate the problems of missing interrupts. */
+			OSScheduleMISR(psDevInfo->pvMISRData);
+			if (psDevInfo->pvAPMISRData != NULL)
+			{
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+		psDevInfo->ui32InterruptCountLastTime    = ui32LISRCount;
+		psDevInfo->ui32MissingInterruptsLastTime = ui32MissingInts;
+	}
+
+	/* Release power lock before RGXCheckForStalledClientContexts */
+	PVRSRVPowerUnlock(psDevNode);
+
+	/*
+	   Stalled CCB check...
+	*/
+	if (bCheckAfterTimePassed && (PVRSRV_DEVICE_HEALTH_STATUS_OK==eNewStatus))
+	{
+		RGXCheckForStalledClientContexts(psDevInfo, IMG_FALSE);
+	}
+
+	/* Notify client driver and system layer of any eNewStatus errors */
+	if (eNewStatus > PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		/* Client notification of device error will be achieved by
+		 * clients calling UM function RGXGetLastDeviceError() */
+		psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+
+		/* Notify system layer */
+		{
+			PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+			PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+			if (psDevConfig->pfnSysDevErrorNotify)
+			{
+				PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+				sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+				sErrorData.uErrData.sHostWdgData.ui32Status = (IMG_UINT32)eNewStatus;
+				sErrorData.uErrData.sHostWdgData.ui32Reason = (IMG_UINT32)eNewReason;
+
+				psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+				                                  &sErrorData);
+			}
+		}
+	}
+
+	/*
+	   Finished, save the new status...
+	*/
+_RGXUpdateHealthStatus_Exit:
+	OSAtomicWrite(&psDevNode->eHealthStatus, eNewStatus);
+	OSAtomicWrite(&psDevNode->eHealthReason, eNewReason);
+	RGXSRV_HWPERF_DEVICE_INFO_HEALTH(psDevInfo, eNewStatus, eNewReason);
+
+	/*
+	 * Attempt to service the HWPerf buffer to regularly transport idle/periodic
+	 * packets to host buffer.
+	 */
+	if (psDevNode->pfnServiceHWPerf != NULL)
+	{
+		PVRSRV_ERROR eError = psDevNode->pfnServiceHWPerf(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: "
+			         "Error occurred when servicing HWPerf buffer (%d)",
+			         __func__, eError));
+		}
+	}
+
+	/* Attempt to refresh timer correlation data */
+	RGXTimeCorrRestartPeriodic(psDevNode);
+
+	return PVRSRV_OK;
+} /* RGXUpdateHealthStatus */
+
+#if defined(SUPPORT_AUTOVZ)
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+		(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))))
+	{
+		/* read and write back the alive token value to confirm to the
+		 * virtualisation watchdog that this connection is healthy */
+		KM_SET_OS_ALIVE_TOKEN(KM_GET_FW_ALIVE_TOKEN(psDevInfo), psDevInfo);
+		KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH);
+	}
+}
+
+/*
+	RGXUpdateAutoVzWatchdog
+*/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode)
+{
+	if (likely(psDevNode != NULL))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+
+		if (unlikely((psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || !psDevInfo->bRGXPowered ||
+					  psDevInfo->pvRegsBaseKM == NULL || psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+					  psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)))
+		{
+			/* If the firmware is not initialised, stop here */
+			return;
+		}
+		else
+		{
+			PVRSRV_ERROR eError = PVRSRVPowerLock(psDevNode);
+			PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PVRSRVPowerLock");
+
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+	}
+}
+
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DriverID;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+	                           INVALIDATE);
+
+	for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+	     ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED;
+	     ui32DriverID++)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+			psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+		if (eGuestState == RGXFW_CONNECTION_FW_ACTIVE ||
+		    eGuestState == RGXFW_CONNECTION_FW_READY)
+		{
+			PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+		}
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer)
+{
+	PVRSRV_ERROR eError;
+	PRGXFWIF_CLEANUP_CTL *psCleanupCtlWrite = apsCleanupCtl;
+
+	PVR_ASSERT((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+	PVR_RETURN_IF_INVALID_PARAM((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+
+	if (bKick)
+	{
+		if (psKMHWRTDataSet)
+		{
+			PRGXFWIF_CLEANUP_CTL psCleanupCtl;
+
+			eError = RGXSetFirmwareAddress(&psCleanupCtl, psKMHWRTDataSet->psHWRTDataFwMemDesc,
+					offsetof(RGXFWIF_HWRTDATA, sCleanupState),
+					RFW_FWADDR_NOREF_FLAG);
+			PVR_RETURN_IF_ERROR(eError);
+
+			*(psCleanupCtlWrite++) = psCleanupCtl;
+		}
+
+		if (eDM == RGXFWIF_DM_3D)
+		{
+			RGXFWIF_PRBUFFER_TYPE eBufferType;
+			RGX_ZSBUFFER_DATA *psBuffer = NULL;
+
+			for (eBufferType = RGXFWIF_PRBUFFER_START; eBufferType < RGXFWIF_PRBUFFER_MAXSUPPORTED; eBufferType++)
+			{
+				switch (eBufferType)
+				{
+				case RGXFWIF_PRBUFFER_ZSBUFFER:
+					psBuffer = psZSBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MSAABUFFER:
+					psBuffer = psMSAAScratchBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MAXSUPPORTED:
+					psBuffer = NULL;
+					break;
+				}
+				if (psBuffer)
+				{
+					(psCleanupCtlWrite++)->ui32Addr = psBuffer->sZSBufferFWDevVAddr.ui32Addr +
+									offsetof(RGXFWIF_PRBUFFER, sCleanupState);
+					psBuffer = NULL;
+				}
+			}
+		}
+	}
+
+	*pui32NumCleanupCtl = psCleanupCtlWrite - apsCleanupCtl;
+	PVR_ASSERT(*pui32NumCleanupCtl <= RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO       *psDevInfo;
+	RGXFWIF_HWRINFOBUF       *psHWRInfoBuf;
+	IMG_UINT32               i;
+
+	if (psDevNode->pvDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVINFO;
+	}
+	psDevInfo = psDevNode->pvDevice;
+
+	psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+	for (i = 0 ; i < psDevInfo->sDevFeatureCfg.ui32MAXDMCount ; i++)
+	{
+		/* Reset the HWR numbers */
+		psHWRInfoBuf->aui32HwrDmLockedUpCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmFalseDetectCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmRecoveredCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmOverranCount[i] = 0;
+	}
+
+	for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+	{
+		psHWRInfoBuf->sHWRInfo[i].ui32HWRNumber = 0;
+	}
+
+	psHWRInfoBuf->ui32WriteIndex = 0;
+	psHWRInfoBuf->ui32DDReqCount = 0;
+
+	OSWriteMemoryBarrier(&psHWRInfoBuf->ui32DDReqCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid)
+{
+
+	PVRSRV_ERROR eError;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+								 ui32Log2PageSize,
+								 ui32NumOfPages,
+								 ui32LogicalOffset,
+								 psPhyAddr,
+								 bValid,
+								 DEVICE_USE);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR_DevPhysAddr failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRUnLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	return eError;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32WriteOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_OK);
+
+	if (psDevInfo->bDumpedKCCBCtlAlready)
+	{
+		/* exiting capture range or pdump block */
+		psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+		/* make sure previous cmd is drained in pdump in case we will 'jump' over some future cmds */
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+                              PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER,
+                              "kCCB(%p): Draining rgxfw_roff (0x%x) == woff (0x%x)",
+                              psDevInfo->psKernelCCBCtl,
+                              ui32WriteOffset,
+                              ui32WriteOffset);
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+				offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+				ui32WriteOffset,
+				0xffffffff,
+				PDUMP_POLL_OPERATOR_EQUAL,
+				PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: problem pdumping POL for kCCBCtl (%d)", __func__, eError));
+		}
+	}
+
+	return eError;
+
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXClientConnectCompatCheck_ClientAgainstFW
+
+ @Description
+
+ Check compatibility of client and firmware (build options)
+ at the connection time.
+
+ @Input psDeviceNode - device node
+ @Input ui32ClientBuildOptions - build options for the client
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+******************************************************************************/
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32ClientBuildOptions)
+{
+	IMG_UINT32		ui32BuildOptionsMismatch;
+	IMG_UINT32		ui32BuildOptionsFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_FW_INFO_HEADER *psFWInfoHeader;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psDevInfo == NULL || psDevInfo->psRGXFWIfOsInitMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Cannot acquire kernel fw compatibility check info, RGXFWIF_OSINIT structure not allocated.",
+		         __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *) &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+#endif
+
+	ui32BuildOptionsFW = psFWInfoHeader->ui32Flags;
+	ui32BuildOptionsMismatch = ui32ClientBuildOptions ^ ui32BuildOptionsFW;
+
+	if (ui32BuildOptionsMismatch != 0)
+	{
+		if ((ui32ClientBuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in client: (0x%x). Please check rgx_options.h",
+					 ui32ClientBuildOptions & ui32BuildOptionsMismatch ));
+		}
+
+		if ((ui32BuildOptionsFW & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFW & ui32BuildOptionsMismatch ));
+		}
+
+		return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware and client build options match. [ OK ]", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapAllocMap
+
+ @Description Register firmware heap for the specified driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID - Guest driver
+ @Input sDevPAddr    - Heap address
+ @Input ui64DevPSize - Heap size
+
+ @Return   PVRSRV_ERROR - PVRSRV_OK if heap setup was successful.
+
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR szRegionRAName[RA_MAX_NAME_LENGTH];
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_MEMALLOCFLAGS_T uiRawFwHeapAllocFlags = (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+													PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PREMAP0 + ui32DriverID));
+	PHYS_HEAP_CONFIG *psFwHeapConfig = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+	PHYS_HEAP_CONFIG sFwHeapConfig;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psFwHeapConfig == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW_MAIN heap config not found."));
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	OSSNPrintf(szRegionRAName, sizeof(szRegionRAName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	if (!ui64DevPSize ||
+		!sDevPAddr.uiAddr ||
+		ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED ||
+		ui64DevPSize != RGX_FIRMWARE_RAW_HEAP_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid parameters for %s", szRegionRAName));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sFwHeapConfig = *psFwHeapConfig;
+	sFwHeapConfig.eType = PHYS_HEAP_TYPE_LMA;
+	sFwHeapConfig.ui32UsageFlags = PHYS_HEAP_USAGE_FW_PREMAP;
+	sFwHeapConfig.uConfig.sLMA.sStartAddr.uiAddr = 0;
+	sFwHeapConfig.uConfig.sLMA.sCardBase.uiAddr = sDevPAddr.uiAddr;
+	sFwHeapConfig.uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+	eError = PhysmemCreateHeapLMA(psDeviceNode,
+	                              RGXPhysHeapGetLMAPolicy(sFwHeapConfig.ui32UsageFlags, psDeviceNode),
+	                              &sFwHeapConfig,
+	                              szRegionRAName,
+	                              &psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysmemCreateHeapLMA:PREMAP [%d]", ui32DriverID);
+
+	eError = PhysHeapAcquire(psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysHeapAcquire:PREMAP [%d]", ui32DriverID);
+
+	psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID] = psDeviceNode->apsFWPremapPhysHeap[ui32DriverID];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate and map raw firmware heap for DriverID: [%d]", ui32DriverID);
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* don't clear the heap of other guests on allocation */
+	uiRawFwHeapAllocFlags &= (ui32DriverID > RGXFW_HOST_DRIVER_ID) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL);
+#endif
+
+	/* if the firmware is already powered up, consider the firmware heaps are pre-mapped. */
+	if (psDeviceNode->bAutoVzFwIsUp)
+	{
+		uiRawFwHeapAllocFlags &= RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Allocation and mapping for Firmware heaps done by TEE.", __func__));
+#else
+	eError = DevmemFwAllocate(psDevInfo,
+							  RGX_FIRMWARE_RAW_HEAP_SIZE,
+							  uiRawFwHeapAllocFlags,
+							  psDevInfo->psPremappedFwRawHeap[ui32DriverID]->pszName,
+							  &psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+#endif
+
+	/* Mark this devmem heap as premapped so allocations will not require device mapping. */
+	DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+
+	if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+	{
+		/* if the Host's raw fw heap is premapped, mark its main & config sub-heaps accordingly
+		 * No memory allocated from these sub-heaps will be individually mapped into the device's
+		 * address space so they can remain marked permanently as premapped. */
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapUnmapFree
+
+ @Description Unregister firmware heap for the specified guest driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID
+
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* remove the premap status, so the heap can be unmapped and freed */
+	if (psDevInfo->psPremappedFwRawHeap[ui32DriverID])
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_FALSE);
+	}
+
+	if (psDevInfo->psPremappedFwRawMemDesc[ui32DriverID])
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+		psDevInfo->psPremappedFwRawMemDesc[ui32DriverID] = NULL;
+	}
+}
+
+/*
+	RGXReadMETAAddr
+*/
+static PVRSRV_ERROR RGXReadMETAAddr(PVRSRV_RGXDEV_INFO	*psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 *pui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem  *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem  *pui8RegBase = pvRegBase;
+	IMG_UINT32 ui32PollValue;
+	IMG_UINT32 ui32PollMask;
+	IMG_UINT32 ui32PollRegOffset;
+	IMG_UINT32 ui32ReadOffset;
+	IMG_UINT32 ui32WriteOffset;
+	IMG_UINT32 ui32WriteValue;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+	{
+		ui32PollValue = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollMask = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1;
+		ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0;
+		ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN;
+		ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX;
+	}
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteUncheckedHWReg32(pvRegBase, ui32WriteOffset, ui32WriteValue);
+	(void)OSReadUncheckedHWReg32(pvRegBase, ui32WriteOffset);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadUncheckedHWReg32(pvRegBase, ui32ReadOffset);
+#else
+	IMG_UINT8 __iomem  *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteHWReg32(
+			psDevInfo->pvRegsBaseKM,
+			RGX_CR_META_SP_MSLVCTRL0,
+			ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAX);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGXWriteMETAAddr
+*/
+static PVRSRV_ERROR RGXWriteMETAAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 ui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem *pui8RegBase = pvRegBase;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA, ui32Value);
+		}
+		else
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA, ui32Value);
+		}
+	}
+	else
+	{
+		/* Wait for Slave Port to be Ready */
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+				(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Issue the Write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0); /* Fence write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT); /* Fence write */
+	}
+#else
+	IMG_UINT8 __iomem *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Write */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 *pui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXReadMETAAddr(psDevInfo, ui32FWAddr, pui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvReadMem(psDevInfo, ui32FWAddr, pui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 ui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXWriteMETAAddr(psDevInfo, ui32FWAddr, ui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvWriteMem(psDevInfo, ui32FWAddr, ui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32FwVA,
+                             IMG_CPU_PHYADDR *psCpuPA,
+                             IMG_DEV_PHYADDR *psDevPA,
+                             IMG_UINT64 *pui64RawPTE)
+{
+	PVRSRV_ERROR eError       = PVRSRV_OK;
+	IMG_CPU_PHYADDR sCpuPA    = {0U};
+	IMG_DEV_PHYADDR sDevPA    = {0U};
+	IMG_UINT64 ui64RawPTE     = 0U;
+	MMU_FAULT_DATA sFaultData = {0U};
+	MMU_CONTEXT *psFwMMUCtx   = psDevInfo->psKernelMMUCtx;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (RGX_NUM_DRIVERS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+	IMG_UINT32 ui32DriverID   = (ui32FwVA - ui32FwHeapBase) / RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT32 ui32HeapId;
+	PHYS_HEAP *psPhysHeap;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* MIPS uses the same page size as the OS, while others default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) ?
+	                             OSGetPageSize() : BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#else
+	/* default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#endif
+
+	IMG_UINT32 ui32PageOffset = (ui32FwVA & (ui32FwPageSize - 1));
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM((ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((psCpuPA != NULL) ||
+	                               (psDevPA != NULL) ||
+	                               (pui64RawPTE != NULL)),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((ui32FwVA >= ui32FwHeapBase) &&
+	                              (ui32FwVA < ui32FwHeapEnd)),
+	                              eError, ErrorExit);
+
+	ui32HeapId = (ui32DriverID == RGXFW_HOST_DRIVER_ID) ?
+	              PVRSRV_PHYS_HEAP_FW_MAIN : (PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID);
+	psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[ui32HeapId];
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS is equipped with a dedicated MMU */
+		RGXMipsCheckFaultAddress(psFwMMUCtx, ui32FwVA, &sFaultData);
+	}
+	else
+#endif
+	{
+		IMG_UINT64 ui64FwDataBaseMask;
+		IMG_DEV_VIRTADDR sDevVAddr;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			ui64FwDataBaseMask = ~(RGXFW_SEGMMU_DATA_META_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_BASE_ADDRESS);
+		}
+#if !defined(EMULATOR)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			ui64FwDataBaseMask = ~RGXRISCVFW_REGION_MASK;
+		}
+#endif
+		else
+		{
+			PVR_LOG_GOTO_WITH_ERROR("RGXGetFwMapping", eError, PVRSRV_ERROR_NOT_IMPLEMENTED, ErrorExit);
+		}
+
+		sDevVAddr.uiAddr = (ui32FwVA & ui64FwDataBaseMask) | RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Fw CPU shares a subset of the GPU's VA space */
+		MMU_CheckFaultAddress(psFwMMUCtx, &sDevVAddr, &sFaultData);
+	}
+
+	ui64RawPTE = sFaultData.sLevelData[MMU_LEVEL_1].ui64Address;
+
+	if (eError == PVRSRV_OK)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		IMG_BOOL bValidPage = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+		                       BITMASK_HAS(ui64RawPTE, RGXMIPSFW_TLB_VALID) :
+		                       BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#else
+		IMG_BOOL bValidPage = BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#endif
+
+		if (!bValidPage)
+		{
+			/* don't report invalid pages */
+			eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+			sDevPA.uiAddr = ui32PageOffset + ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			                RGXMIPSFW_TLB_GET_PA(ui64RawPTE) :
+			                (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK));
+#else
+			sDevPA.uiAddr = ui32PageOffset + (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK);
+#endif
+
+			/* Only the Host's Firmware heap is present in the Host's CPU IPA space */
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPA, &sDevPA);
+			}
+			else
+			{
+				sCpuPA.uiAddr = 0U;
+			}
+		}
+	}
+
+	if (psCpuPA != NULL)
+	{
+		*psCpuPA = sCpuPA;
+	}
+
+	if (psDevPA != NULL)
+	{
+		*psDevPA = sDevPA;
+	}
+
+	if (pui64RawPTE != NULL)
+	{
+		*pui64RawPTE = ui64RawPTE;
+	}
+
+ErrorExit:
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = ui32VzConnectionCooldownPeriodInSec;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Vz reconnect request cooldown period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32VzConnectionCooldownPeriodInSec),
+							  ui32VzConnectionCooldownPeriodInSec,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxfwutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxhwperf.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxhwperf.c
new file mode 100644
index 000000000000..67812e3c5d0a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxhwperf.c
@@ -0,0 +1,616 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+
+#include "info_page_defs.h"
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] =
+{
+#define X(a, b, c, d, e, f, g)  {a, b, 0xFF, d, e, f, NULL}
+RGX_CNT_BLK_TYPE_MODEL_DIRECT_LIST,
+RGX_CNT_BLK_TYPE_MODEL_INDIRECT_LIST
+#undef X
+};
+
+IMG_INTERNAL IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel)
+{
+    *ppsModel = gasCntBlkTypeModel;
+    return ARRAY_SIZE(gasCntBlkTypeModel);
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXHWPerfMaxDefinedBlks
+
+ @Description Return the number of valid block-IDs for the given device node
+
+ @Input       (PVRSRV_RGXDEV_INFO *)   pvDevice    device-node to query
+
+ @Returns     (IMG_UINT32)             Number of block-IDs (RGX_CNTBLK_ID)
+                                       valid for this device.
+******************************************************************************/
+IMG_INTERNAL IMG_UINT32
+RGXHWPerfMaxDefinedBlks(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_HWPERF_CNTBLK_RT_INFO sRtInfo;
+	IMG_UINT32  uiRetVal;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *psHWPBlkConfig;
+	IMG_UINT32  uiNumArrayEls, ui;
+
+	uiRetVal = RGX_CNTBLK_ID_DIRECT_LAST;
+
+	uiNumArrayEls = RGXGetHWPerfBlockConfig(&psHWPBlkConfig);
+
+	if (psHWPBlkConfig == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected NULL Config Block", __func__));
+		return 0;
+	}
+	PVR_ASSERT(uiNumArrayEls > 0);
+
+	/* Iterate over each block-ID and find the number of instances of each
+	 * block which are present for this device type. We only query the
+	 * Indirect blocks as their presence varies according to GPU. All direct
+	 * blocks have an entry - but they may not be physically present.
+	 */
+	for (ui = RGX_CNTBLK_ID_DIRECT_LAST; ui < uiNumArrayEls; ui++)
+	{
+		if (rgx_hwperf_blk_present(&psHWPBlkConfig[ui], (void *)psDevInfo, &sRtInfo))
+		{
+			uiRetVal += sRtInfo.uiNumUnits;
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Block %u, NumUnits %u, Total %u",
+			        __func__, ui, sRtInfo.uiNumUnits, uiRetVal));
+		}
+#ifdef DEBUG
+		else
+		{
+			if (psHWPBlkConfig[ui].uiCntBlkIdBase == RGX_CNTBLK_ID_RAC0)
+			{
+				if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevInfo->psDeviceNode,
+				    RAY_TRACING_ARCH) > 2U)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: Block %u *NOT* present",
+					        __func__, ui));
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Block %u *NOT* present",
+				        __func__, ui));
+			}
+		}
+#endif
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Num Units = %u", __func__, uiRetVal));
+
+	return uiRetVal;
+}
+
+static IMG_BOOL RGXServerFeatureFlagsToHWPerfFlagsAddBlock(
+	RGX_HWPERF_BVNC_BLOCK	* const psBlocks,
+	IMG_UINT16				* const pui16Count,
+	const IMG_UINT16		ui16BlockID, /* see RGX_HWPERF_CNTBLK_ID */
+	const IMG_UINT16		ui16NumCounters,
+	const IMG_UINT16		ui16NumBlocks)
+{
+	const IMG_UINT16 ui16Count = *pui16Count;
+
+	if (ui16Count < RGX_HWPERF_MAX_BVNC_BLOCK_LEN)
+	{
+		RGX_HWPERF_BVNC_BLOCK * const psBlock = &psBlocks[ui16Count];
+
+		/* If the GROUP is non-zero, convert from e.g. RGX_CNTBLK_ID_USC0 to
+		 * RGX_CNTBLK_ID_USC_ALL. The table stores the former (plus the
+		 * number of blocks and counters) but PVRScopeServices expects the
+		 * latter (plus the number of blocks and counters). The conversion
+		 * could always be moved to PVRScopeServices, but it's less code this
+		 * way.
+		 * For SLC0 we generate a single SLCBANK_ALL which has NUM_MEMBUS
+		 * instances.
+		 * This replaces the SLC0 .. SLC3 entries.
+		 */
+		if ((ui16BlockID == RGX_CNTBLK_ID_SLCBANK0) || (ui16BlockID & RGX_CNTBLK_ID_GROUP_MASK))
+		{
+			psBlock->ui16BlockID	= ui16BlockID | RGX_CNTBLK_ID_UNIT_ALL_MASK;
+		}
+		else
+		{
+			psBlock->ui16BlockID	= ui16BlockID;
+		}
+		psBlock->ui16NumCounters	= ui16NumCounters;
+		psBlock->ui16NumBlocks		= ui16NumBlocks;
+
+		*pui16Count = ui16Count + 1;
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_HWPERF_BVNC *psBVNC)
+{
+	IMG_PCHAR pszBVNC;
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDevInfo), "psDevInfo invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((pszBVNC = RGXDevBVNCString(psDevInfo)))
+	{
+		size_t uiStringLength = OSStringNLength(pszBVNC, RGX_HWPERF_MAX_BVNC_LEN - 1);
+		OSStringSafeCopy(psBVNC->aszBvncString, pszBVNC, uiStringLength + 1);
+		memset(&psBVNC->aszBvncString[uiStringLength], 0, RGX_HWPERF_MAX_BVNC_LEN - uiStringLength);
+	}
+	else
+	{
+		*psBVNC->aszBvncString = 0;
+	}
+
+	psBVNC->ui32BvncKmFeatureFlags = 0x0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERFBUS_FLAG;
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_MULTICORE_FLAG;
+	}
+
+	/* Determine if we've got the new RAY_TRACING feature supported. This is
+	 * only determined by comparing the RAY_TRACING_ARCHITECTURE value */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_RAYTRACING_FLAG;
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CATURIX_TOP_INFRASTRUCTURE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_CXT_TOP_INFRASTRUCTURE_FLAG;
+	}
+
+#ifdef SUPPORT_WORKLOAD_ESTIMATION
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Not a part of BVNC feature line and so doesn't need the feature supported check */
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION;
+	}
+#endif
+
+	/* Define the HW counter block counts. */
+	{
+		RGX_HWPERF_BVNC_BLOCK					* const psBlocks	= psBVNC->aBvncBlocks;
+		IMG_UINT16								* const pui16Count	= &psBVNC->ui16BvncBlocks;
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	*asCntBlkTypeModel;
+		const IMG_UINT32						ui32CntBlkModelLen	= RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+		IMG_UINT32								ui32BlkCfgIdx;
+		size_t									uiCount;
+		IMG_BOOL								bOk					= IMG_TRUE;
+
+		// Initialise to zero blocks
+		*pui16Count = 0;
+
+		// Add all the blocks
+		for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+		{
+			const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	* const psCntBlkInfo = &asCntBlkTypeModel[ui32BlkCfgIdx];
+			RGX_HWPERF_CNTBLK_RT_INFO				sCntBlkRtInfo;
+			/* psCntBlkInfo->uiNumUnits gives compile-time info. For BVNC agnosticism, we use this: */
+			if (psCntBlkInfo->pfnIsBlkPresent(psCntBlkInfo, psDevInfo, &sCntBlkRtInfo))
+			{
+				IMG_UINT32	uiNumUnits;
+
+				switch (psCntBlkInfo->uiCntBlkIdBase)
+				{
+					case RGX_CNTBLK_ID_SLCBANK0:
+						/* Generate the SLCBANK_ALL block for SLC0..SLC3
+						 * we have to special-case this as the iteration will
+						 * generate entries starting at SLC0 and we need to
+						 * defer until we are processing the last 'present'
+						 * entry.
+						 * The SLC_BLKID_ALL is keyed from SLC0. Need to access
+						 * the NUM_MEMBUS feature to see how many are physically
+						 * present.
+						 * For CXT_TOP_INFRASTRUCTURE systems we present a
+						 * singleton SLCBANK - this provides accumulated counts
+						 * for all registers within the physical SLCBANK instances.
+						 */
+						if (psBVNC->ui32BvncKmFeatureFlags &
+						    RGX_HWPERF_FEATURE_CXT_TOP_INFRASTRUCTURE_FLAG)
+						{
+							uiNumUnits = 1U;
+						}
+						else
+						{
+							uiNumUnits = RGX_GET_FEATURE_VALUE(psDevInfo,
+							                                   NUM_MEMBUS);
+						}
+						break;
+					case RGX_CNTBLK_ID_SLCBANK1:
+					case RGX_CNTBLK_ID_SLCBANK2:
+					case RGX_CNTBLK_ID_SLCBANK3:
+						/* These are contained within SLCBANK_ALL block */
+						continue;
+					default:
+						uiNumUnits = sCntBlkRtInfo.uiNumUnits;
+						break;
+				}
+				bOk &= RGXServerFeatureFlagsToHWPerfFlagsAddBlock(psBlocks, pui16Count, psCntBlkInfo->uiCntBlkIdBase, RGX_CNTBLK_COUNTERS_MAX, uiNumUnits);
+			}
+		}
+
+		/* If this fails, consider why the static_assert didn't fail, and consider increasing RGX_HWPERF_MAX_BVNC_BLOCK_LEN */
+		PVR_ASSERT(bOk);
+
+		// Zero the remaining entries
+		uiCount = *pui16Count;
+		OSDeviceMemSet(&psBlocks[uiCount], 0, (RGX_HWPERF_MAX_BVNC_BLOCK_LEN - uiCount) * sizeof(*psBlocks));
+	}
+
+	/* The GPU core count is overwritten by the FW */
+	psBVNC->ui16BvncGPUCores = 0;
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVRGXConfigureHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+		CONNECTION_DATA          * psConnection,
+		PVRSRV_DEVICE_NODE       * psDeviceNode,
+		IMG_UINT32                 ui32CtrlWord,
+		IMG_UINT32                 ui32ArrayLen,
+		RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs)
+{
+	PVRSRV_ERROR             eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD         sKccbCmd;
+	DEVMEM_MEMDESC           *psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_CNTBLK *psFwArray;
+	IMG_UINT32               ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO       *psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.ui32CtrlWord = ui32CtrlWord;
+	sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_CNTBLK) * ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.sBlockConfigs,
+	                               psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/* Ask the FW to carry out the HWPerf configuration command. */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration. */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED",
+	         ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfConfigureCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32					ui32CtrlWord,
+		IMG_UINT32					ui32NumBlocks,
+		RGX_HWPERF_CONFIG_CNTBLK*	asBlockConfigs)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !asBlockConfigs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXConfigureHWPerfBlocksKM(NULL,
+		                                          psDevData->psRgxDevNode,
+		                                          ui32CtrlWord,
+		                                          ui32NumBlocks,
+		                                          asBlockConfigs);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXConfigureHWPerfBlocksKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) < RGX_CNTBLK_ID_LAST)
+	{
+		RGXFWIF_HWPERF_CTL_BLK *psBlock = NULL;
+		RGX_HWPERF_CONFIG_CNTBLK sBlockConfig;
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		for (i = 0; i < psHWPerfCtl->ui32NumBlocks; i++)
+		{
+			if (psHWPerfCtl->sBlkCfg[i].uiBlockID != ui32BlockID)
+			{
+				continue;
+			}
+			else if (psHWPerfCtl->sBlkCfg[i].uiEnabled == 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+			}
+
+			psBlock = &psHWPerfCtl->sBlkCfg[i];
+			break;
+		}
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->uiBlockID;
+		sBlockConfig.ui16NumCounters = psBlock->uiNumCounters;
+
+		for (i = 0; i < psBlock->uiNumCounters; i++)
+		{
+			sBlockConfig.ui16Counters[i] = psBlock->aui32CounterCfg[i];
+		}
+
+		*psConfiguredCounters = sBlockConfig;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLen,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	IMG_UINT32 ui32LastIdx = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	*pui32BlockCount = 0;
+
+	if (ui32ArrayLen > 0 && pui32EnabledBlockIDs == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen is greater than 0 but pui32EnabledBlockIDs is NULL"));
+	}
+
+	PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+	for (i = 0; i < psHWPerfCtl->ui32NumBlocks; i++)
+	{
+		if (psHWPerfCtl->sBlkCfg[i].uiEnabled)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sBlkCfg[i].uiBlockID;
+			ui32LastIdx++;
+		}
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxinit.c
new file mode 100644
index 000000000000..eca671110a4f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxinit.c
@@ -0,0 +1,5603 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_bridge_init.h"
+#include "rgx_bridge_init.h"
+#include "syscommon.h"
+#include "rgx_heaps_server.h"
+#include "rgxheapconfig.h"
+#include "rgxpower.h"
+#include "tlstream.h"
+#include "pvrsrv_tlstreams.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "pvr_ricommon.h"
+#endif
+
+#include "rgxinit.h"
+#include "rgxbvnc.h"
+#include "rgxmulticore.h"
+
+#include "pdump_km.h"
+#include "handle.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxmem.h"
+#include "sync_internal.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "rgxfwdbg.h"
+#include "info_page.h"
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxfwimageutils.h"
+#endif
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+#include "rgxmmuinit.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#include "physmem.h"
+#endif
+#include "devicemem_utils.h"
+#include "devicemem_server.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "rgxta3d.h"
+#include "rgxtimecorr.h"
+#include "rgxshader.h"
+
+#if defined(PDUMP)
+#include "rgxstartstop.h"
+#endif
+
+#include "rgx_fwif_alignchecks.h"
+#include "vmm_pvz_client.h"
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+
+#if defined(PDUMP) && defined(SUPPORT_SECURITY_VALIDATION)
+#include "pdump_physmem.h"
+#endif
+
+#if defined(PDUMP)
+#include "rgxpdump_common.h"
+#endif
+
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_CHAR **ppszVersionString);
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PUINT32 pui32RGXClockSpeed);
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID);
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/* Services internal heap identification used in this file only */
+#define RGX_FIRMWARE_MAIN_HEAP_IDENT   "FwMain"   /*!< RGX Main Firmware Heap identifier */
+#define RGX_FIRMWARE_CONFIG_HEAP_IDENT "FwConfig" /*!< RGX Config firmware Heap identifier */
+
+#define RGX_MMU_PAGE_SIZE_4KB   (   4 * 1024)
+#define RGX_MMU_PAGE_SIZE_16KB  (  16 * 1024)
+#define RGX_MMU_PAGE_SIZE_64KB  (  64 * 1024)
+#define RGX_MMU_PAGE_SIZE_256KB ( 256 * 1024)
+#define RGX_MMU_PAGE_SIZE_1MB   (1024 * 1024)
+#define RGX_MMU_PAGE_SIZE_2MB   (2048 * 1024)
+#define RGX_MMU_PAGE_SIZE_MIN RGX_MMU_PAGE_SIZE_4KB
+#define RGX_MMU_PAGE_SIZE_MAX RGX_MMU_PAGE_SIZE_2MB
+
+#define VAR(x) #x
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@Function       SampleIRQCount
+@Description    Utility function taking snapshots of RGX FW interrupt count.
+@Input          psDevInfo    Device Info structure
+
+@Return         IMG_BOOL     Returns IMG_TRUE if RGX FW IRQ is not equal to
+                             sampled RGX FW IRQ count for any RGX FW thread.
+ */ /**************************************************************************/
+static INLINE IMG_BOOL SampleIRQCount(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bReturnVal = IMG_FALSE;
+	volatile IMG_UINT32 *pui32SampleIrqCount = psDevInfo->aui32SampleIRQCount;
+	IMG_UINT32 ui32IrqCnt;
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bReturnVal = IMG_TRUE;
+	}
+	else
+	{
+		get_irq_cnt_val(ui32IrqCnt, RGXFW_HOST_DRIVER_ID, psDevInfo);
+
+		if (ui32IrqCnt != pui32SampleIrqCount[RGXFW_THREAD_0])
+		{
+			pui32SampleIrqCount[RGXFW_THREAD_0] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#else
+	IMG_UINT32 ui32TID;
+
+	for_each_irq_cnt(ui32TID)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32TID, psDevInfo);
+
+		/* treat unhandled interrupts here to align host count with fw count */
+		if (pui32SampleIrqCount[ui32TID] != ui32IrqCnt)
+		{
+			pui32SampleIrqCount[ui32TID] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#endif
+
+	return bReturnVal;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHostSafetyEvents
+@Description    Returns the event status masked to keep only the safety
+                events handled by the Host
+@Input          psDevInfo    Device Info structure
+@Return         IMG_UINT32   Status of Host-handled safety events
+ */ /**************************************************************************/
+static INLINE IMG_UINT32 RGXHostSafetyEvents(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (psDevInfo->ui32HostSafetyEventMask == 0))
+	{
+		return 0;
+	}
+	else
+	{
+		IMG_UINT32 ui32EventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_STATUS);
+
+		return (ui32EventStatus & psDevInfo->ui32HostSafetyEventMask);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       GetContextResetReason
+@Description    Maps a bit corresponding to an uncorrected FW fault reported in
+                RGX_CR_FAULT_FW_STATUS register into RGX_CONTEXT_RESET_REASON value
+@Input          ui32FaultFlag Offset into the bitmask of uncorrected FW faults
+                psDevInfo Device Info structure
+@Return         RGX_CONTEXT_RESET_REASON Reset Reason
+ */ /**************************************************************************/
+static inline RGX_CONTEXT_RESET_REASON GetContextResetReason(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FaultFlag)
+{
+	RGX_CONTEXT_RESET_REASON eResetReason;
+
+	if (ui32FaultFlag == RGX_CR_FAULT_FW_STATUS_MMU_PTE_PARITY_DETECT_SHIFT)
+	{
+		IMG_UINT32 ui32ParityStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_PT_PARITY_STATUS);
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_PT_PARITY_CLEAR, ui32ParityStatus);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: MMU PTE parity errors: 0x%x", __func__, ui32ParityStatus));
+
+		eResetReason = RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR;
+	}
+	else if (ui32FaultFlag == RGX_CR_FAULT_FW_STATUS_PARITY_DETECT_SHIFT)
+	{
+		eResetReason = RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR;
+	}
+	else if (ui32FaultFlag == RGX_CR_FAULT_FW_STATUS_LOCKSTEP_DETECT_SHIFT)
+	{
+		eResetReason = RGX_CONTEXT_RESET_REASON_DCLS_ERR;
+	}
+	else
+	{
+		eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_ERR;
+	}
+
+	return eResetReason;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventHandler
+@Description    Handles the Safety Events that the Host is responsible for
+@Input          psDevInfo    Device Info structure
+ */ /**************************************************************************/
+static void RGXSafetyEventHandler(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	if (psDevInfo->bRGXPowered)
+	{
+		RGX_CONTEXT_RESET_REASON eResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+		IMG_UINT32 ui32HostSafetyStatus = RGXHostSafetyEvents(psDevInfo);
+		IMG_UINT32 ui32HostSafetyStatusBitPos = 0;
+
+		if (ui32HostSafetyStatus != 0)
+		{
+			/* clear the safety bus events handled by the Host */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_CLEAR, ui32HostSafetyStatus);
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_EVENT_STATUS_FAULT_FW_SHIFT))
+			{
+				IMG_UINT32 ui32FaultFlag;
+				IMG_UINT32 ui32FaultFW = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_STATUS);
+				IMG_UINT32 ui32CorrectedBitOffset = RGX_CR_FAULT_FW_STATUS_MMU_CORRECT_SHIFT -
+													RGX_CR_FAULT_FW_STATUS_MMU_DETECT_SHIFT;
+
+				PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety fault status: 0x%X", __func__, ui32FaultFW));
+
+				for (ui32FaultFlag = 0; ui32FaultFlag < ui32CorrectedBitOffset; ui32FaultFlag++)
+				{
+					if (BIT_ISSET(ui32FaultFW, ui32FaultFlag))
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety hardware fault detected (0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+						eResetReason = GetContextResetReason(psDevInfo, ui32FaultFlag);
+					}
+					else if BIT_ISSET(ui32FaultFW, ui32FaultFlag + ui32CorrectedBitOffset)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Firmware safety hardware fault corrected.(0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+
+						/* Only report this if we haven't detected a more serious error */
+						if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR &&
+						    eResetReason != RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR &&
+							eResetReason != RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR &&
+							eResetReason != RGX_CONTEXT_RESET_REASON_DCLS_ERR)
+
+						{
+							eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_OK;
+						}
+					}
+				}
+
+				OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_CLEAR, ui32FaultFW);
+			}
+
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PIPELINED_DATAMASTERS_VERSION)  &&
+				RGX_GET_FEATURE_VALUE(psDevInfo, PIPELINED_DATAMASTERS_VERSION) > 0)
+			{
+				ui32HostSafetyStatusBitPos = RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_GT0__WDT_TIMEOUT_SHIFT;
+			}
+			else
+#endif
+			{
+				ui32HostSafetyStatusBitPos = RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_SHIFT;
+			}
+
+			if (BIT_ISSET(ui32HostSafetyStatus, ui32HostSafetyStatusBitPos))
+			{
+				volatile RGXFWIF_POW_STATE ePowState;
+
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+				                           INVALIDATE);
+				ePowState = psDevInfo->psRGXFWIfFwSysData->ePowState;
+
+				if (ePowState != RGXFWIF_POW_OFF)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Safety Watchdog Trigger !", __func__));
+
+					/* Only report this if we haven't detected a more serious error */
+					if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR &&
+					    eResetReason != RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR)
+					{
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_WATCHDOG;
+					}
+				}
+			}
+
+			/* Notify client and system layer of any error */
+			if (eResetReason != RGX_CONTEXT_RESET_REASON_NONE)
+			{
+				PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+				PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = eResetReason;
+
+				/* Notify system layer of any error */
+				if (psDevConfig->pfnSysDevErrorNotify)
+				{
+					PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+					sErrorData.eResetReason = eResetReason;
+
+					psDevConfig->pfnSysDevErrorNotify(psDevConfig->hSysData,
+					                                  &sErrorData);
+				}
+			}
+		}
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+static IMG_BOOL _WaitForInterruptsTimeoutCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32idx;
+#endif
+
+	RGXDEBUG_PRINT_IRQ_COUNT(psDevInfo);
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVR_DPF((PVR_DBG_ERROR,
+	        "Last RGX_LISRHandler State (DevID %u): 0x%08X Clock: %" IMG_UINT64_FMTSPEC,
+			 psDeviceNode->sDevId.ui32InternalID,
+			 psDeviceNode->sLISRExecutionInfo.ui32Status,
+			 psDeviceNode->sLISRExecutionInfo.ui64Clockns));
+
+	for_each_irq_cnt(ui32idx)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				MSG_IRQ_CNT_TYPE " %u: InterruptCountSnapshot: 0x%X",
+				ui32idx, psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[ui32idx]));
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR, "No further information available. Please enable PVRSRV_DEBUG_LISR_EXECUTION"));
+#endif
+
+	return SampleIRQCount(psDevInfo);
+}
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bScheduleMISR;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bScheduleMISR = IMG_TRUE;
+	}
+	else
+	{
+		bScheduleMISR = _WaitForInterruptsTimeoutCheck(psDevInfo);
+	}
+
+	if (bScheduleMISR)
+	{
+		OSScheduleMISR(psDevInfo->pvMISRData);
+
+		if (psDevInfo->pvAPMISRData != NULL)
+		{
+			OSScheduleMISR(psDevInfo->pvAPMISRData);
+		}
+	}
+}
+
+static inline IMG_BOOL RGXAckHwIrq(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_UINT32 ui32IRQStatusReg,
+								   IMG_UINT32 ui32IRQStatusEventMsk,
+								   IMG_UINT32 ui32IRQClearReg,
+								   IMG_UINT32 ui32IRQClearMask)
+{
+	IMG_UINT32 ui32IRQStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQStatusReg);
+
+	/* clear only the pending bit of the thread that triggered this interrupt */
+	ui32IRQClearMask &= ui32IRQStatus;
+
+	if (ui32IRQStatus & ui32IRQStatusEventMsk)
+	{
+		/* acknowledge and clear the interrupt */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQClearReg, ui32IRQClearMask);
+		return IMG_TRUE;
+	}
+	else
+	{
+		/* spurious interrupt */
+		return IMG_FALSE;
+	}
+}
+
+static __maybe_unused IMG_BOOL RGXAckIrqMETA(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static IMG_BOOL RGXAckIrqMIPS(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN);
+}
+#endif
+
+static IMG_BOOL RGXAckIrqDedicated(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+		/* status & clearing registers are available on both Host and Guests
+		 * and are agnostic of the Fw CPU type. Due to the remappings done by
+		 * the 2nd stage device MMU, all drivers assume they are accessing
+		 * register bank 0 */
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_IRQ_OS0_EVENT_STATUS,
+					   ~RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK,
+					   RGX_CR_IRQ_OS0_EVENT_CLEAR,
+					   ~RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK);
+}
+
+static PVRSRV_ERROR RGXSetAckIrq(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_SINGLE_IRQ_WORKAROUND)
+	/* Register appropriate mechanism for clearing hw interrupts */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psDevInfo->pfnRGXAckIrq = NULL;
+	}
+	else
+#endif
+	{
+		psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+	}
+
+	return PVRSRV_OK;
+}
+
+static IMG_BOOL RGX_LISRHandler(void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bIrqAcknowledged = IMG_FALSE;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	IMG_UINT32 ui32idx, ui32IrqCnt;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+		UPDATE_LISR_DBG_SNAPSHOT(ui32idx, ui32IrqCnt);
+	}
+
+	UPDATE_LISR_DBG_STATUS(RGX_LISR_INIT);
+	UPDATE_LISR_DBG_TIMESTAMP();
+#endif
+
+	UPDATE_LISR_DBG_COUNTER();
+
+	if (psDevInfo->bRGXPowered)
+	{
+		IMG_BOOL bSafetyEvent = (RGXHostSafetyEvents(psDevInfo) != 0);
+
+		if ((psDevInfo->pfnRGXAckIrq == NULL) || psDevInfo->pfnRGXAckIrq(psDevInfo) || bSafetyEvent)
+		{
+			bIrqAcknowledged = IMG_TRUE;
+
+			if (bSafetyEvent || SampleIRQCount(psDevInfo))
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_PROCESSED);
+				UPDATE_MISR_DBG_COUNTER();
+
+				OSScheduleMISR(psDevInfo->pvMISRData);
+
+#if defined(SUPPORT_AUTOVZ)
+				RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+				if (psDevInfo->pvAPMISRData != NULL)
+				{
+					OSScheduleMISR(psDevInfo->pvAPMISRData);
+				}
+			}
+			else
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED);
+			}
+		}
+		else
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_NOT_TRIGGERED_BY_HW);
+		}
+	}
+	else
+	{
+#if defined(SUPPORT_AUTOVZ)
+		/* AutoVz drivers rebooting while the firmware is active must acknowledge
+		 * and clear the hw IRQ line before the RGXInit() has finished. */
+		if ((psDevInfo->pfnRGXAckIrq != NULL) &&
+			 psDevInfo->pfnRGXAckIrq(psDevInfo))
+		{
+			bIrqAcknowledged = IMG_TRUE;
+		}
+		else
+#endif
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_DEVICE_NOT_POWERED);
+		}
+	}
+
+	return bIrqAcknowledged;
+}
+
+static void RGX_MISR_ProcessKCCBDeferredList(PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* First check whether there are pending commands in Deferred KCCB List */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	if (dllist_is_empty(&psDevInfo->sKCCBDeferredCommandsListHead))
+	{
+		OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+		return;
+	}
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	/* Powerlock to avoid further Power transition requests
+	   while KCCB deferred list is being processed */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Try to send deferred KCCB commands Do not Poll from here*/
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s could not flush Deferred KCCB list, KCCB is full.",
+				 __func__));
+	}
+}
+
+static void RGX_MISRHandler_CheckFWActivePowerState(void *psDevice)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevice;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_ON || psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		/* The FW is IDLE and therefore could be shut down */
+		eError = RGXActivePowerRequest(psDeviceNode);
+
+		if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED))
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+					"%s: Failed RGXActivePowerRequest call (device: %p) with %s",
+					__func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+			else
+			{
+				/* Re-schedule the power down request as it was deferred. */
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+	}
+
+}
+
+/* Shorter defines to keep the code a bit shorter */
+#define GPU_IDLE       RGXFWIF_GPU_UTIL_STATE_IDLE
+#define GPU_ACTIVE     RGXFWIF_GPU_UTIL_STATE_ACTIVE
+#define GPU_BLOCKED    RGXFWIF_GPU_UTIL_STATE_BLOCKED
+#define GPU_INACTIVE   RGXFWIF_GPU_UTIL_STATE_INACTIVE
+#define MAX_ITERATIONS 64
+#define MAX_DIFF_TIME_NS (300000ULL)
+#define MAX_DIFF_DM_TIME_NS (MAX_DIFF_TIME_NS >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT)
+
+static PVRSRV_ERROR RGXGetGpuUtilStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_HANDLE hGpuUtilUser,
+                                       RGXFWIF_GPU_UTIL_STATS *psReturnStats)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_STATS sStats;
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+	IMG_UINT64 (*paaui64DMOSTmpCounters)[RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 (*paui64DMOSTmpLastWord)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastState)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastPeriod)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastTime)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT64 ui64TimeNowShifted;
+	IMG_UINT32 ui32Attempts;
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32MaxDMCount;
+	RGXFWIF_DM eDM;
+
+	/***** (1) Initialise return stats *****/
+
+	psReturnStats->bValid = IMG_FALSE;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psAggregateStats = hGpuUtilUser;
+
+	/* decrease by 1 to account for excluding GP DM from the statics */;
+	ui32MaxDMCount =  psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1;
+
+	/* Reset temporary counters used in the attempts loop */
+	paaui64DMOSTmpCounters  = &psAggregateStats->sTempGpuStats.aaaui64DMOSTmpCounters[0];
+	paui64DMOSTmpLastWord   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastWord[0];
+	paui64DMOSTmpLastState  = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastState[0];
+	paui64DMOSTmpLastPeriod = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastPeriod[0];
+	paui64DMOSTmpLastTime   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastTime[0];
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	/* Try to acquire GPU utilisation counters and repeat if the FW is in the middle of an update */
+	for (ui32Attempts = 0; ui32Attempts < 4; ui32Attempts++)
+	{
+		IMG_UINT64 aui64GpuTmpCounters[RGXFWIF_GPU_UTIL_STATE_NUM] = {0};
+		IMG_UINT64 ui64GpuLastPeriod = 0, ui64GpuLastWord = 0, ui64GpuLastState = 0, ui64GpuLastTime = 0;
+
+		/***** (2) Get latest data from shared area *****/
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			IMG_UINT64 aui64StatsCountersNew[RGXFWIF_GPU_UTIL_STATE_NUM];
+			IMG_UINT64 ui64GpuLastWordNew;
+			RGXFWIF_GPU_STATS sStatsNew;
+			IMG_UINT32 i = 0;
+
+			ui64GpuLastWord = 0;
+			ui64GpuLastState = 0;
+
+			OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+			/* Copy data from device memory */
+			memcpy(&sStatsNew, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+			memcpy(&ui64GpuLastWordNew, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+			memcpy(aui64StatsCountersNew, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64StatsCountersNew));
+
+			/*
+			 * First attempt at detecting if the FW is in the middle of an update.
+			 * This should also help if the FW is in the middle of a 64 bit variable update.
+			 * This loop must be fast. Faster than FW updates the stats.
+			 */
+			for (i = 0; i < MAX_ITERATIONS; i++)
+			{
+				IMG_UINT32 j,k;
+				IMG_BOOL bRetry = IMG_FALSE;
+
+				if (i > 0)
+				{
+					/* On retry keep previous data */
+					ui64GpuLastWordNew = ui64GpuLastWord;
+					memcpy(aui64StatsCountersNew, aui64GpuTmpCounters, sizeof(aui64StatsCountersNew));
+					memcpy(&sStatsNew, &sStats, sizeof(sStatsNew));
+				}
+
+				/* Copy data from device memory */
+				memcpy(&sStats, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+				memcpy(&ui64GpuLastWord, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+				memcpy(aui64GpuTmpCounters, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64GpuTmpCounters));
+
+				/* Check for abnormal time difference between reads */
+				if (RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord) - RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWordNew) > MAX_DIFF_TIME_NS)
+				{
+					bRetry = IMG_TRUE;
+					continue;
+				}
+
+				for (j = 0; j < RGXFWIF_GPU_UTIL_STATE_NUM; j++)
+				{
+					/* Check for abnormal time difference between reads */
+					if (aui64GpuTmpCounters[j] - aui64StatsCountersNew[j] > MAX_DIFF_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+				}
+
+				if (bRetry)
+				{
+					continue;
+				}
+
+				/* Check for DM counters wrapped or
+				   abnormal time difference between reads.
+				   The DM time is shifted by RGXFWIF_DM_OS_TIMESTAMP_SHIFT */
+				for (j = 0; j < RGXFWIF_GPU_UTIL_DM_MAX; j++)
+				{
+					if (sStats.aui32DMOSLastWordWrap[j] != sStatsNew.aui32DMOSLastWordWrap[j] ||
+						RGXFWIF_GPU_UTIL_GET_TIME32(sStats.aui32DMOSLastWord[j]) - RGXFWIF_GPU_UTIL_GET_TIME32(sStatsNew.aui32DMOSLastWord[j]) > MAX_DIFF_DM_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+
+					for (k = 0; k < RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM; k++)
+					{
+						if (sStats.aaui32DMOSCountersWrap[j][k] != sStatsNew.aaui32DMOSCountersWrap[j][k] ||
+							sStats.aaui32DMOSStatsCounters[j][k] - sStatsNew.aaui32DMOSStatsCounters[j][k] > MAX_DIFF_DM_TIME_NS)
+						{
+							bRetry = IMG_TRUE;
+							break;
+						}
+
+					}
+
+					if (bRetry)
+					{
+						break;
+					}
+				}
+
+				if (!bRetry)
+				{
+					/* Stats are good*/
+					break;
+				}
+			}
+
+			OSLockRelease(psDevInfo->hGPUUtilLock);
+
+			ui64GpuLastState = RGXFWIF_GPU_UTIL_GET_STATE(ui64GpuLastWord);
+
+			if (i == MAX_ITERATIONS)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						 "RGXGetGpuUtilStats could not get reliable data after trying %u times", i));
+
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+			{
+				paui64DMOSTmpLastWord[eDM][ui32DriverID]  =
+					((IMG_UINT64)sStats.aui32DMOSLastWordWrap[eDM] << 32) + sStats.aui32DMOSLastWord[eDM];
+				paui64DMOSTmpLastState[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_STATE(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				if (paui64DMOSTmpLastState[eDM][ui32DriverID] != GPU_ACTIVE)
+				{
+					paui64DMOSTmpLastState[eDM][ui32DriverID] = GPU_INACTIVE;
+				}
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE] = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_INACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_INACTIVE] << 32);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE]  = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_ACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_ACTIVE] << 32);
+			}
+
+		} /* FOREACH_SUPPORTED_DRIVER(ui32DriverID) */
+
+
+		/***** (3) Compute return stats *****/
+
+		/* Update temp counters to account for the time since the last update to the shared ones */
+		OSMemoryBarrier(NULL); /* Ensure the current time is read after the loop above */
+		ui64TimeNow    = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDeviceNode));
+
+		ui64GpuLastTime   = RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord);
+		ui64GpuLastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64GpuLastTime);
+		aui64GpuTmpCounters[ui64GpuLastState] += ui64GpuLastPeriod;
+
+		/* Get statistics for a user since its last request */
+		psReturnStats->ui64GpuStatIdle = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_IDLE],
+		                                                             psAggregateStats->ui64GpuStatIdle);
+		psReturnStats->ui64GpuStatActive = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_ACTIVE],
+		                                                               psAggregateStats->ui64GpuStatActive);
+		psReturnStats->ui64GpuStatBlocked = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_BLOCKED],
+		                                                                psAggregateStats->ui64GpuStatBlocked);
+		psReturnStats->ui64GpuStatCumulative = psReturnStats->ui64GpuStatIdle +
+		                                       psReturnStats->ui64GpuStatActive +
+		                                       psReturnStats->ui64GpuStatBlocked;
+
+		/* convert time into the same units as used by fw */
+		ui64TimeNowShifted  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+		for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+		{
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				paui64DMOSTmpLastTime[eDM][ui32DriverID]   = RGXFWIF_GPU_UTIL_GET_TIME(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				paui64DMOSTmpLastPeriod[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNowShifted , paui64DMOSTmpLastTime[eDM][ui32DriverID]);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][paui64DMOSTmpLastState[eDM][ui32DriverID]] += paui64DMOSTmpLastPeriod[eDM][ui32DriverID];
+				/* Get statistics for a user since its last request */
+				psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE],
+				                                                             psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE],
+				                                                               psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatCumulative[eDM][ui32DriverID] = psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] +
+				                                       psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+			}
+		}
+
+		if (psAggregateStats->ui64TimeStamp != 0)
+		{
+			IMG_UINT64 ui64TimeSinceLastCall = ui64TimeNow - psAggregateStats->ui64TimeStamp;
+			/* We expect to return at least 75% of the time since the last call in GPU stats */
+			IMG_UINT64 ui64MinReturnedStats = ui64TimeSinceLastCall - (ui64TimeSinceLastCall / 4);
+
+			/*
+			 * If the returned stats are substantially lower than the time since
+			 * the last call, then the Host might have read a partial update from the FW.
+			 * If this happens, try sampling the shared counters again.
+			 */
+			if (psReturnStats->ui64GpuStatCumulative < ui64MinReturnedStats)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s: Return stats (%" IMG_UINT64_FMTSPEC ") too low "
+				         "(call period %" IMG_UINT64_FMTSPEC ")",
+				         __func__, psReturnStats->ui64GpuStatCumulative, ui64TimeSinceLastCall));
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Attempt #%u has failed, trying again",
+				         __func__, ui32Attempts));
+				continue;
+			}
+		}
+
+		break;
+	}
+
+	/***** (4) Update aggregate stats for the current user *****/
+
+	psAggregateStats->ui64GpuStatIdle    += psReturnStats->ui64GpuStatIdle;
+	psAggregateStats->ui64GpuStatActive  += psReturnStats->ui64GpuStatActive;
+	psAggregateStats->ui64GpuStatBlocked += psReturnStats->ui64GpuStatBlocked;
+	psAggregateStats->ui64TimeStamp       = ui64TimeNow;
+
+	for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+	{
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]    += psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID];
+			psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]  += psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+		}
+	}
+
+	/***** (5) Convert return stats to microseconds *****/
+
+	psReturnStats->ui64GpuStatIdle       = OSDivide64(psReturnStats->ui64GpuStatIdle, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatActive     = OSDivide64(psReturnStats->ui64GpuStatActive, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatBlocked    = OSDivide64(psReturnStats->ui64GpuStatBlocked, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatCumulative = OSDivide64(psReturnStats->ui64GpuStatCumulative, 1000, &ui32Remainder);
+
+	/* Check that the return stats make sense */
+	if (psReturnStats->ui64GpuStatCumulative == 0)
+	{
+		/* We can enter here only if allocating the temporary stats
+		 * buffers failed, or all the RGXFWIF_GPU_UTIL_GET_PERIOD
+		 * returned 0. The latter could happen if the GPU frequency value
+		 * is not well calibrated and the FW is updating the GPU state
+		 * while the Host is reading it.
+		 * When such an event happens frequently, timers or the aggregate
+		 * stats might not be accurate...
+		 */
+#if defined(VIRTUAL_PLATFORM)
+		/* To avoid spamming the console logging system on emulated devices,
+		 * we special-case so that we will only produce a single message per
+		 * driver invocation. This should reduce the time spent logging
+		 * information which is not relevant for very slow timers found in
+		 * VP device configurations
+		 */
+		static IMG_BOOL bFirstTime = IMG_TRUE;
+
+		if (bFirstTime)
+		{
+			bFirstTime = IMG_FALSE;
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "RGXGetGpuUtilStats could not get reliable data."));
+#if defined(VIRTUAL_PLATFORM)
+		}
+#endif	/* defined(VIRTUAL_PLATFORM) */
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psReturnStats->bValid = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psAggregateStats = OSAllocZMemNoStats(sizeof(RGXFWIF_GPU_UTIL_STATS));
+	if (psAggregateStats == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	*phGpuUtilUser = psAggregateStats;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAggregateStats = hGpuUtilUser;
+	OSFreeMemNoStats(psAggregateStats);
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGX MISR Handler
+*/
+static void RGX_MISRHandler_Main (void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* Give the HWPerf service a chance to transfer some data from the FW
+	 * buffer to the host driver transport layer buffer.
+	 */
+	RGXHWPerfDataStoreCB(psDeviceNode);
+
+	/* Inform other services devices that we have finished an operation */
+	PVRSRVNotifyCommandCompletion(psDeviceNode);
+
+#if defined(SUPPORT_PDVFS) && defined(RGXFW_META_SUPPORT_2ND_THREAD)
+	/* Normally, firmware CCB only exists for the primary FW thread unless PDVFS
+	   is running on the second[ary] FW thread, here we process said CCB */
+	RGXPDVFSCheckCoreClkRateChange(psDeviceNode->pvDevice);
+#endif
+
+	/* Only execute SafetyEventHandler if RGX_FEATURE_SAFETY_EVENT is on */
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, ECC_RAMS) > 0 ||
+		PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, WATCHDOG_TIMER) ||
+		PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, RISCV_DUAL_LOCKSTEP) ||
+		PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, FAULT_DECODE_VERSION) >= 2)
+	{
+		/* Handle Safety events if necessary */
+		RGXSafetyEventHandler(psDeviceNode->pvDevice);
+	}
+
+	/* Signal the global event object */
+	PVRSRVSignalDriverWideEO();
+
+	/* Process the Firmware CCB for pending commands */
+	RGXCheckFirmwareCCB(psDeviceNode->pvDevice);
+
+	/* Calibrate the GPU frequency and recorrelate Host and GPU timers (done every few seconds) */
+	RGXTimeCorrRestartPeriodic(psDeviceNode);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Process Workload Estimation Specific commands from the FW */
+		WorkEstCheckFirmwareCCB(psDeviceNode->pvDevice);
+	}
+#endif
+
+	if (psDevInfo->pvAPMISRData == NULL)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+static PVRSRV_ERROR RGXPDumpBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+		PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psFWDataPMR;
+	RGXMIPSFW_BOOT_DATA *psBootData;
+	IMG_DEV_PHYADDR sTmpAddr;
+	IMG_UINT32 ui32BootConfOffset, ui32ParamOffset, i;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+	ui32BootConfOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+	ui32BootConfOffset += RGXMIPSFW_BOOTLDR_CONF_OFFSET;
+
+	/* The physical addresses used by a pdump player will be different
+	 * than the ones we have put in the MIPS bootloader configuration data.
+	 * We have to tell the pdump player to replace the original values with the real ones.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Pass new boot parameters to the FW");
+
+	/* Rogue Registers physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64RegBase);
+
+	eError = PDumpRegLabelToMem64(RGX_PDUMPREG_NAME,
+			0x0,
+			psFWDataPMR,
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of Rogue registers phy address failed (%u)", eError));
+		return eError;
+	}
+
+	/* Page Table physical Address */
+	eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sTmpAddr);
+	if (eError !=  PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXBootldrDataInit: MMU_AcquireBaseAddr failed (%u)",
+		         eError));
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+									 (void **)&psBootData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire pointer to FW data (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	psBootData = IMG_OFFSET_ADDR(psBootData, ui32BootConfOffset);
+
+	for (i = 0; i < psBootData->ui32PTNumPages; i++)
+	{
+		ui32ParamOffset = ui32BootConfOffset +
+			offsetof(RGXMIPSFW_BOOT_DATA, aui64PTPhyAddr[0])
+			+ i * sizeof(psBootData->aui64PTPhyAddr[0]);
+
+		eError = PDumpPTBaseObjectToMem64(psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				psFWDataPMR,
+				0,
+				ui32ParamOffset,
+				PDUMP_FLAGS_CONTINUOUS,
+				MMU_LEVEL_1,
+				sTmpAddr.uiAddr,
+				i << psBootData->ui32PTLog2PageSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of page tables phy address failed (%u)", eError));
+			return eError;
+		}
+	}
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+
+	/* Stack physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64StackPhyAddr);
+
+	eError = PDumpMemLabelToMem64(psFWDataPMR,
+			psFWDataPMR,
+			RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of stack phy address failed (%u)", eError));
+		return eError;
+	}
+
+	return eError;
+}
+#endif /* PDUMP */
+
+static PVRSRV_ERROR RGXSetPowerParams(PVRSRV_RGXDEV_INFO   *psDevInfo,
+                                      PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR sKernelMMUCtxPCAddr;
+	IMG_BOOL bPremappedFw;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVCFG, psDevConfig, PVRSRV_OK);
+
+	/* Save information used on power transitions for later
+	 * (when RGXStart and RGXStop are executed)
+	 */
+#if defined(PDUMP)
+	psDevInfo->sLayerParams.ui32PdumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	/* Volcanic drivers with security support and premapped fw heaps
+	 * always have their fw heap premapped by the TEE */
+	bPremappedFw = IMG_TRUE;
+#else
+	/* If AutoVz firmware is up at this stage, the driver initialised it
+	 * during a previous life-cycle. The firmware's memory is already pre-mapped
+	 * and the MMU page tables reside in the predetermined memory carveout.
+	 * The Kernel MMU Context created in this life-cycle is a dummy structure
+	 * that is not used for mapping.
+	 * To program the Device's BIF with the correct PC address, use the base
+	 * address of the carveout reserved for MMU mappings as Kernel MMU PC Address */
+	bPremappedFw = psDevInfo->psDeviceNode->bAutoVzFwIsUp;
+#endif
+
+	if (bPremappedFw)
+	{
+		IMG_DEV_PHYADDR sDevPAddr;
+		PHYS_HEAP *psFwPageTableHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_RETURN_IF_FALSE((NULL != psFwPageTableHeap),
+								"Firmware Page Table heap not defined.",
+								PVRSRV_ERROR_INVALID_HEAP);
+
+		PhysHeapGetDevPAddr(psFwPageTableHeap, &sDevPAddr);
+		sKernelMMUCtxPCAddr.uiAddr = sDevPAddr.uiAddr;
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+	else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx,
+		                             &sKernelMMUCtxPCAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire Kernel MMU Ctx page catalog"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		PMR *psFWCodePMR = (PMR *)(psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR);
+		PMR *psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+		IMG_DEV_PHYADDR sPhyAddr;
+		IMG_BOOL bValid;
+
+#if defined(SUPPORT_ALT_REGBASE)
+		psDevInfo->sLayerParams.sGPURegAddr = psDevConfig->sAltRegsGpuPBase;
+#else
+		/* The physical address of the GPU registers needs to be translated
+		 * in case we are in a LMA scenario
+		 */
+		PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+				1,
+				&sPhyAddr,
+				&(psDevConfig->sRegsCpuPBase));
+
+		psDevInfo->sLayerParams.sGPURegAddr = sPhyAddr;
+#endif
+
+		/* Register bank must be aligned to 512KB (as per the core integration) to
+		 * prevent the FW accessing incorrect registers */
+		if ((psDevInfo->sLayerParams.sGPURegAddr.uiAddr & 0x7FFFFU) != 0U)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Register bank must be aligned to 512KB, but current address (0x%016"IMG_UINT64_FMTSPECX") is not",
+						psDevInfo->sLayerParams.sGPURegAddr.uiAddr));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE),
+				OSGetPageShift(), /* FW will be using the same page size as the OS */
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI code address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sBootRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWDataPMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI data address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sDataRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW exceptions address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sCodeRemapAddr = sPhyAddr;
+
+		psDevInfo->sLayerParams.sTrampolineRemapAddr.uiAddr = psDevInfo->psTrampoline->sPhysAddr.uiAddr;
+
+		psDevInfo->sLayerParams.bDevicePA0IsValid = psDevConfig->bDevicePA0IsValid;
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	/* Send information used on power transitions to the trusted device as
+	 * in this setup the driver cannot start/stop the GPU and perform resets
+	 */
+	if (psDevConfig->pfnTDSetPowerParams)
+	{
+		PVRSRV_TD_POWER_PARAMS sTDPowerParams;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+		    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			sTDPowerParams.sPCAddr = psDevInfo->sLayerParams.sPCAddr;
+		}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			sTDPowerParams.sGPURegAddr    = psDevInfo->sLayerParams.sGPURegAddr;
+			sTDPowerParams.sBootRemapAddr = psDevInfo->sLayerParams.sBootRemapAddr;
+			sTDPowerParams.sCodeRemapAddr = psDevInfo->sLayerParams.sCodeRemapAddr;
+			sTDPowerParams.sDataRemapAddr = psDevInfo->sLayerParams.sDataRemapAddr;
+		}
+#endif
+
+		eError = psDevConfig->pfnTDSetPowerParams(psDevConfig->hSysData,
+												  &sTDPowerParams);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: TDSetPowerParams not implemented!"));
+		eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	return eError;
+}
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*
+	RGXSystemGetFabricCoherency
+*/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode)
+{
+	IMG_CHAR *aszLabels[] = {"none", "acelite", "fullace", "unknown"};
+	PVRSRV_DEVICE_SNOOP_MODE eAppHintCacheSnoopingMode;
+	PVRSRV_DEVICE_SNOOP_MODE eDeviceCacheSnoopingMode;
+	IMG_UINT32 ui32AppHintFabricCoherency;
+	IMG_UINT32 ui32DeviceFabricCoherency;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+#if !defined(NO_HARDWARE)
+	void *pvRegsBaseKM;
+	IMG_BOOL bPowerDown = IMG_TRUE;
+	PVRSRV_ERROR eError;
+#endif
+
+	if (!sRegsCpuPBase.uiAddr || !ui32RegsSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Invalid RGX register base/size parameters"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(NO_HARDWARE)
+	pvRegsBaseKM = OSMapPhysToLin(sRegsCpuPBase, ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvRegsBaseKM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Failed to create RGX register mapping"));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	bPowerDown = ! PVRSRVIsSystemPowered(psDevConfig->psDevNode);
+
+	/* Power-up the device as required to read the registers */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+	}
+
+	/* AXI support within the SoC, bitfield COHERENCY_SUPPORT [1 .. 0]
+		value NO_COHERENCY        0x0 {SoC does not support any form of Coherency}
+		value ACE_LITE_COHERENCY  0x1 {SoC supports ACE-Lite or I/O Coherency}
+		value FULL_ACE_COHERENCY  0x2 {SoC supports full ACE or 2-Way Coherency} */
+	ui32DeviceFabricCoherency = OSReadHWReg32((void __iomem *)pvRegsBaseKM, RGX_CR_SOC_AXI);
+	PVR_LOG(("AXI fabric coherency (RGX_CR_SOC_AXI): 0x%x", ui32DeviceFabricCoherency));
+#if defined(DEBUG)
+	if (ui32DeviceFabricCoherency & ~((IMG_UINT32)RGX_CR_SOC_AXI_MASKFULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid RGX_CR_SOC_AXI value.", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+	ui32DeviceFabricCoherency &= ~((IMG_UINT32)RGX_CR_SOC_AXI_COHERENCY_SUPPORT_CLRMSK);
+	ui32DeviceFabricCoherency >>= RGX_CR_SOC_AXI_COHERENCY_SUPPORT_SHIFT;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+	}
+
+	/* UnMap Regs */
+	OSUnMapPhysToLin(pvRegsBaseKM, ui32RegsSize);
+
+	switch (ui32DeviceFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_FULLACE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+	default:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_NONE;
+		break;
+	}
+#else /* !defined(NO_HARDWARE) */
+    *peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+    eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+    ui32DeviceFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY;
+#endif /* !defined(NO_HARDWARE) */
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, FabricCoherencyOverride,
+						 &ui32AppHintDefault, &ui32AppHintFabricCoherency);
+	OSFreeAppHintState(pvAppHintState);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	/* Temporarily disable coherency */
+	ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY;
+#endif
+
+	/* Suppress invalid AppHint value */
+	switch (ui32AppHintFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"Invalid FabricCoherencyOverride AppHint %d, ignoring",
+				ui32AppHintFabricCoherency));
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+		break;
+	}
+
+	if (ui32AppHintFabricCoherency < ui32DeviceFabricCoherency)
+	{
+		PVR_LOG(("Downgrading device fabric coherency from %s to %s",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+		eDeviceCacheSnoopingMode = eAppHintCacheSnoopingMode;
+	}
+	else if (ui32AppHintFabricCoherency > ui32DeviceFabricCoherency)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Cannot upgrade device fabric coherency from %s to %s, not supported by device!",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+
+		/* Override requested-for app-hint with actual app-hint value being used */
+		ui32AppHintFabricCoherency = ui32DeviceFabricCoherency;
+	}
+
+	*peCacheSnoopingMode = eDeviceCacheSnoopingMode;
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXSystemHasFBCDCVersion31
+*/
+static IMG_BOOL RGXSystemHasFBCDCVersion31(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(HW_ERN_66622_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 66622))
+#endif
+	{
+		{
+			if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+			{
+				return IMG_TRUE;
+			}
+		}
+	}
+#if defined(HW_ERN_66622_BIT_MASK)
+	else
+	{
+
+#if !defined(NO_HARDWARE)
+		if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: System uses FBCDC3.1 but GPU doesn't support it!",
+			         __func__));
+		}
+#endif
+	}
+#endif /* defined(HW_ERN_66622_BIT_MASK) */
+
+	return IMG_FALSE;
+}
+
+/*
+	RGXGetTFBCLossyGroup
+*/
+static IMG_UINT32 RGXGetTFBCLossyGroup(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	return psDevInfo->ui32TFBCLossyGroup;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return 0;
+#endif
+}
+
+/*
+	RGXDevMMUAttributes
+*/
+static MMU_DEVICEATTRIBS *RGXDevMMUAttributes(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              IMG_BOOL bKernelFWMemoryCtx)
+{
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = NULL;
+
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			psMMUDevAttrs = bKernelFWMemoryCtx ?
+			                psDeviceNode->psFirmwareMMUDevAttrs :
+			                psDeviceNode->psMMUDevAttrs;
+		}
+		else
+#endif
+		{
+			PVR_UNREFERENCED_PARAMETER(bKernelFWMemoryCtx);
+			psMMUDevAttrs = psDeviceNode->psMMUDevAttrs;
+		}
+	}
+
+	return psMMUDevAttrs;
+}
+
+
+/*
+	RGXDevSnoopMode
+*/
+static PVRSRV_DEVICE_SNOOP_MODE RGXDevSnoopMode(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_DEVICE_SNOOP_NONE;
+}
+
+/*
+ * RGXInitDevPart2
+ */
+PVRSRV_ERROR RGXInitDevPart2(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                             RGX_INIT_APPHINTS	*psApphints)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEV_POWER_STATE	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32			ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32			ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	IMG_UINT32 ui32PDumpFlags = PDUMP_FLAGS_DEINIT;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 2");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		RGXPDumpBootldrData(psDeviceNode, psDevInfo);
+	}
+#endif
+#if defined(TIMING) || defined(DEBUG)
+	OSUserModeAccessToPerfCountersEn();
+#endif
+
+	/* Initialise Device Flags */
+	psDevInfo->ui32DeviceFlags = 0;
+	RGXSetDeviceFlags(psDevInfo, psApphints->ui32DeviceFlags, IMG_TRUE);
+
+	/* Allocate DVFS Table (needs to be allocated before GPU trace events
+	 *  component is initialised because there is a dependency between them) */
+	psDevInfo->psGpuDVFSTable = OSAllocZMem(sizeof(*(psDevInfo->psGpuDVFSTable)));
+	PVR_LOG_GOTO_IF_NOMEM(psDevInfo->psGpuDVFSTable, eError, ErrorExit);
+
+	if (psDevInfo->ui32HWPerfHostFilter == 0)
+	{
+		RGXHWPerfHostSetEventFilter(psDevInfo, psApphints->ui32HWPerfHostFilter);
+	}
+
+	/* If HWPerf enabled allocate all resources for the host side buffer. */
+	if (psDevInfo->ui32HWPerfHostFilter != 0)
+	{
+		if (RGXHWPerfHostInitOnDemandResources(psDevInfo) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "HWPerfHost buffer on demand"
+			        " initialisation failed."));
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise work estimation lock */
+		eError = OSLockCreate(&psDevInfo->hWorkEstLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(WorkEstLock)", ErrorExit);
+	}
+#endif
+
+	/* Initialise lists of ZSBuffers */
+	eError = OSLockCreate(&psDevInfo->hLockZSBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockZSBuffer)", ErrorExit);
+	dllist_init(&psDevInfo->sZSBufferHead);
+	psDevInfo->ui32ZSBufferCurrID = 1;
+
+	/* Initialise lists of growable Freelists */
+	eError = OSLockCreate(&psDevInfo->hLockFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockFreeList)", ErrorExit);
+	dllist_init(&psDevInfo->sFreeListHead);
+	psDevInfo->ui32FreelistCurrID = 1;
+
+	eError = OSLockCreate(&psDevInfo->hDebugFaultInfoLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(DebugFaultInfoLock)", ErrorExit);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		eError = OSLockCreate(&psDevInfo->hMMUCtxUnregLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(MMUCtxUnregLock)", ErrorExit);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = OSLockCreate(&psDevInfo->hNMILock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(NMILock)", ErrorExit);
+	}
+#endif
+
+	/* Setup GPU utilisation stats update callback */
+	eError = OSLockCreate(&psDevInfo->hGPUUtilLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(GPUUtilLock)", ErrorExit);
+#if !defined(NO_HARDWARE)
+	psDevInfo->pfnGetGpuUtilStats = RGXGetGpuUtilStats;
+#endif
+
+	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	psDevInfo->eActivePMConf = psApphints->eRGXActivePMConf;
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	/* Validate the SPU mask and initialize to number of SPUs to power up */
+	if ((psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s:Invalid SPU mask (All=0x%X, Non Fused=0x%X). At-least one SPU must to be powered up.",
+		         __func__,
+		         ui32AllPowUnitsMask,
+		         psApphints->ui32AvailablePowUnitsMask));
+		PVR_LOG_GOTO_WITH_ERROR("ui32AvailablePowUnitsMask", eError, PVRSRV_ERROR_INVALID_SPU_MASK, ErrorExit);
+	}
+
+	psDevInfo->ui32AvailablePowUnitsMask = psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask;
+
+	psDevInfo->ui32AvailableRACMask = psApphints->ui32AvailableRACMask & ui32AllRACMask;
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* set-up the Active Power Mgmt callback */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableAPM = psRGXData->psRGXTimingInfo->bEnableActivePM;
+		IMG_BOOL bEnableAPM = ((psApphints->eRGXActivePMConf == RGX_ACTIVEPM_DEFAULT) && bSysEnableAPM) ||
+							   (psApphints->eRGXActivePMConf == RGX_ACTIVEPM_FORCE_ON);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1) && defined(SUPPORT_AUTOVZ)
+		/* The AutoVz driver enables a virtualisation watchdog not compatible with APM */
+		if (bEnableAPM && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Active Power Management disabled in AutoVz mode", __func__));
+			bEnableAPM = IMG_FALSE;
+		}
+
+		PVR_ASSERT(bEnableAPM == IMG_FALSE);
+#endif
+
+		if (bEnableAPM)
+		{
+			eError = OSInstallMISR(&psDevInfo->pvAPMISRData,
+					RGX_MISRHandler_CheckFWActivePowerState,
+					psDeviceNode,
+					"RGX_CheckFWActivePower");
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(APMISR)", ErrorExit);
+
+			/* Prevent the device being woken up before there is something to do. */
+			eDefaultPowerState = PVRSRV_DEV_POWER_STATE_OFF;
+		}
+	}
+#endif
+
+	psDevInfo->eDebugDumpFWTLogType = psApphints->eDebugDumpFWTLogType;
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableAPM,
+	                                    RGXQueryAPMState,
+	                                    RGXSetAPMState,
+	                                    psDeviceNode,
+	                                    NULL);
+
+	RGXTimeCorrInitAppHintCallbacks(psDeviceNode);
+
+	/* Register the device with the power manager */
+	eError = PVRSRVRegisterPowerDevice(psDeviceNode,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPrePowerState : &RGXVzPrePowerState,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPostPowerState : &RGXVzPostPowerState,
+	                                   psDevConfig->pfnPrePowerState, psDevConfig->pfnPostPowerState,
+	                                   &RGXPreClockSpeedChange, &RGXPostClockSpeedChange,
+	                                   &RGXForcedIdleRequest, &RGXCancelForcedIdleRequest, &RGXCancelForcedIdleRequestAsync,
+	                                   &RGXPowUnitsChange,
+	                                   (IMG_HANDLE)psDeviceNode,
+	                                   PVRSRV_DEV_POWER_STATE_OFF,
+	                                   eDefaultPowerState);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterPowerDevice", ErrorExit);
+
+	eError = RGXSetPowerParams(psDevInfo, psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetPowerParams", ErrorExit);
+
+
+#if defined(PDUMP)
+#if defined(NO_HARDWARE)
+	/* Capture the HWPERF buffer as the last thing we do */
+	PDUMPIF(psDeviceNode, "ENABLE_HWPERF", ui32PDumpFlags);
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+	                                 psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                                 0,
+	                                 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+	                                 "out.hwperf",
+	                                 0,
+	                                 ui32PDumpFlags);
+	PDUMPFI(psDeviceNode, "ENABLE_HWPERF", ui32PDumpFlags);
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT, "Wait for the FW to signal idle");
+
+	/* Kick the FW once, in case it still needs to detect and set the idle state */
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+			   RGX_CR_MTS_SCHEDULE,
+			   RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK,
+			   PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                                offsetof(RGXFWIF_SYSDATA, ePowState),
+	                                RGXFWIF_POW_IDLE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", ErrorExit);
+#endif
+
+	/* Run RGXStop with the correct PDump flags to feed the last-frame deinit buffer */
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT,
+	                      "RGX deinitialisation commands");
+
+	psDevInfo->sLayerParams.ui32PdumpFlags |= PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW;
+
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXStop(&psDevInfo->sLayerParams);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXStop", ErrorExit);
+	}
+
+	psDevInfo->sLayerParams.ui32PdumpFlags &= ~(PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW);
+#endif
+
+#if !defined(NO_HARDWARE)
+	eError = RGXInstallProcessQueuesMISR(&psDevInfo->hProcessQueuesMISR, psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInstallProcessQueuesMISR", ErrorExit);
+
+	/* Register RGX to receive notifies when other devices complete some work */
+	PVRSRVRegisterCmdCompleteNotify(&psDeviceNode->hCmdCompNotify, &RGXScheduleProcessQueuesKM, psDeviceNode);
+
+	/* Register the interrupt handlers */
+	eError = OSInstallMISR(&psDevInfo->pvMISRData,
+			RGX_MISRHandler_Main,
+			psDeviceNode,
+			"RGX_Main");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(MISR)", ErrorExit);
+
+	/* Register appropriate mechanism for clearing hw interrupts */
+	eError = RGXSetAckIrq(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetAckIrq", ErrorExit);
+
+	eError = SysInstallDeviceLISR(psDevConfig->hSysData,
+								  psDevConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  RGX_LISRHandler,
+								  psDeviceNode,
+								  &psDevInfo->pvLISRData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysInstallDeviceLISR", ErrorExit);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(PDUMP)
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_CACHE_HIERARCHY)))
+	{
+		if (PVRSRVSystemSnoopingOfCPUCache(psDevConfig))
+		{
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+				              "System has CPU cache snooping");
+		}
+		else
+		{
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "System has NO cache snooping");
+		}
+	}
+#endif
+
+	eError = PVRSRVTQLoadShaders(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVTQLoadShaders", ErrorExit);
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	eError = OSAllocateSecBuf(psDeviceNode, RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE, "SharedSecMem", &psDevInfo->psGenHeapSecMem);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSAllocateSecBuf", ErrorExit);
+#endif
+
+	psDevInfo->bDevInit2Done = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	DevPart2DeInitRGX(psDeviceNode);
+
+	return eError;
+}
+
+#define VZ_RGX_FW_FILENAME_SUFFIX ".vz"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#define RGX_64K_FW_FILENAME_SUFFIX ".64k"
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX) + sizeof(RGX_64K_FW_FILENAME_SUFFIX)))
+#else
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX)))
+#endif
+
+static void _GetFWFileName(PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_CHAR *pszFWFilenameStr,
+		IMG_CHAR *pszFWpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const IMG_CHAR * const pszFWFilenameSuffix =
+			PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) ? "" : VZ_RGX_FW_FILENAME_SUFFIX;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	const IMG_CHAR * const pszFWFilenameSuffix2 =
+			((OSGetPageSize() == RGX_MMU_PAGE_SIZE_64KB) &&
+			 RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+			? RGX_64K_FW_FILENAME_SUFFIX : "";
+#else
+	const IMG_CHAR * const pszFWFilenameSuffix2 = "";
+#endif
+
+	OSSNPrintf(pszFWFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STR_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+
+	OSSNPrintf(pszFWpFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STRP_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+}
+
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW)
+{
+	IMG_CHAR aszFWFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR aszFWpFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR *pszLoadedFwStr;
+	PVRSRV_ERROR eErr;
+
+	/* Prepare the image filenames to use in the following code */
+	_GetFWFileName(psDeviceNode, aszFWFilenameStr, aszFWpFilenameStr);
+
+	/* Get pointer to Firmware image */
+	pszLoadedFwStr = aszFWFilenameStr;
+	eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+	if (eErr == PVRSRV_ERROR_NOT_FOUND)
+	{
+		pszLoadedFwStr = aszFWpFilenameStr;
+		eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+		if (eErr == PVRSRV_ERROR_NOT_FOUND)
+		{
+			pszLoadedFwStr = RGX_FW_FILENAME;
+			eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+			if (eErr == PVRSRV_ERROR_NOT_FOUND)
+			{
+				PVR_DPF((PVR_DBG_FATAL, "All RGX Firmware image loads failed for '%s' (%s)",
+						aszFWFilenameStr, PVRSRVGetErrorString(eErr)));
+			}
+		}
+	}
+
+	if (eErr == PVRSRV_OK)
+	{
+		PVR_LOG(("RGX Firmware image '%s' loaded", pszLoadedFwStr));
+	}
+
+	return eErr;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	PVRSRV_ERROR eError;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS_DIRECT;
+
+	eError = RGXSendCommandWithPowLock(psDeviceNode->pvDevice,
+	                                   &sKccbCmd,
+	                                   PDUMP_FLAGS_CONTINUOUS);
+
+	return eError;
+}
+#endif
+
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* set up fw memory contexts */
+	PVRSRV_RGXDEV_INFO   *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	IMG_BOOL  bNativeFwUMAHeap = PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) &&
+	                             (PhysHeapGetType(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM]) == PHYS_HEAP_TYPE_UMA);
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)) && (!psDeviceNode->bAutoVzFwIsUp) && (!bNativeFwUMAHeap))
+	{
+		PHYS_HEAP *psFwPageTableHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_GOTO_IF_INVALID_PARAM((psFwPageTableHeap != NULL),
+		                              eError, failed_to_create_ctx);
+
+		/* Temporarily swap the MMU and default GPU physheap to allow the page
+		 * tables of all memory mapped by the FwKernel context to be placed
+		 * in a dedicated memory carveout. This should allow the firmware mappings to
+		 * persist after a Host kernel crash or driver reset. */
+		psDeviceNode->psMMUPhysHeap = psFwPageTableHeap;
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+	/* Set the device fabric coherency before FW context creation */
+	eError = RGXSystemGetFabricCoherency(psDevConfig,
+										 psDevConfig->sRegsCpuPBase,
+										 psDevConfig->ui32RegsSize,
+										 &psDeviceNode->eDevFabricType,
+										 &psDevConfig->eCacheSnoopingMode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed RGXSystemGetFabricCoherency (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+#endif
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	RGXFwSharedMemCheckSnoopMode(psDevConfig);
+
+	/* Create the memory context for the firmware. */
+	eError = DevmemCreateContext(psDeviceNode, DEVMEM_HEAPCFG_FORFW,
+	                             &psDevInfo->psKernelDevmemCtx);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemCreateContext (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_MAIN_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareMainHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_CONFIG_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareConfigHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+#if (defined(RGX_PREMAP_FW_HEAPS)) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_CHAR szHeapName[RA_MAX_NAME_LENGTH];
+
+			OSSNPrintf(szHeapName, sizeof(szHeapName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+			eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, szHeapName,
+										  &psDevInfo->psPremappedFwRawHeap[ui32DriverID]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "DevmemFindHeapByName", failed_to_find_heap);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && !bNativeFwUMAHeap)
+	{
+		IMG_DEV_PHYADDR sPhysHeapBase;
+		IMG_UINT32 ui32DriverID;
+		void *pvAppHintState = NULL;
+		IMG_UINT64 ui64DefaultHeapStride;
+		IMG_UINT64 ui64GuestHeapDevBaseStride;
+
+		OSCreateAppHintState(&pvAppHintState);
+		ui64DefaultHeapStride = PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+		OSGetAppHintUINT64(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							GuestFWHeapStride,
+							&ui64DefaultHeapStride,
+							&ui64GuestHeapDevBaseStride);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		eError = PhysHeapGetDevPAddr(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM], &sPhysHeapBase);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", failed_to_find_heap);
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_DEV_PHYADDR sRawFwHeapBase = {sPhysHeapBase.uiAddr + (ui32DriverID * ui64GuestHeapDevBaseStride)};
+
+			eError = RGXFwRawHeapAllocMap(psDeviceNode,
+										  ui32DriverID,
+										  sRawFwHeapBase,
+										  RGX_FIRMWARE_RAW_HEAP_SIZE);
+			if (eError != PVRSRV_OK)
+			{
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+				}
+				PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", failed_to_find_heap);
+			}
+		}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+		/* restore default Px setup */
+		psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* On setups with dynamically mapped Guest heaps, the Guest makes
+	 * a PVZ call to the Host to request the mapping during init. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PvzClientMapDevPhysHeap(psDevConfig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PvzClientMapDevPhysHeap", failed_to_find_heap);
+	}
+#endif /* !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+
+failed_to_find_heap:
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+	DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+	psDevInfo->psKernelDevmemCtx = NULL;
+failed_to_create_ctx:
+	return eError;
+}
+
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR       eError;
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(RGX_PREMAP_FW_HEAPS)
+		psDeviceNode->psMMUPhysHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		if (!psDeviceNode->bAutoVzFwIsUp)
+#endif
+		{
+			IMG_UINT32 ui32DriverID;
+
+			FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+			{
+				RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+			}
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		(void) PvzClientUnmapDevPhysHeap(psDeviceNode->psDevConfig);
+
+		if (psDevInfo->psFirmwareMainHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_FALSE);
+		}
+		if (psDevInfo->psFirmwareConfigHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_FALSE);
+		}
+	}
+#endif
+
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+
+	if (psDevInfo->psKernelDevmemCtx)
+	{
+		eError = DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+		PVR_LOG_IF_ERROR(eError, "DevmemDestroyContext");
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+}
+
+static PVRSRV_ERROR RGXAlignmentCheck(PVRSRV_DEVICE_NODE *psDevNode,
+                                      IMG_UINT32 ui32AlignChecksSizeUM,
+                                      IMG_UINT32 aui32AlignChecksUM[])
+{
+	static const IMG_UINT32 aui32AlignChecksKM[] = {RGXFW_ALIGN_CHECKS_INIT_KM};
+	IMG_UINT32 ui32UMChecksOffset = ARRAY_SIZE(aui32AlignChecksKM) + 1;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32 i, *paui32FWAlignChecks;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Skip the alignment check if the driver is guest
+	   since there is no firmware to check against */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, eError);
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW Alignment Check Mem Descriptor is NULL",
+		         __func__));
+		return PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc,
+	                                  (void **) &paui32FWAlignChecks);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire kernel address for alignment checks (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	paui32FWAlignChecks += ui32UMChecksOffset;
+	/* Invalidate the size value, check the next region size (UM) and invalidate */
+	RGXFwSharedMemCacheOpPtr(paui32FWAlignChecks, INVALIDATE);
+	if (*paui32FWAlignChecks++ != ui32AlignChecksSizeUM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Mismatching sizes of RGXFW_ALIGN_CHECKS_INIT"
+		         " array between UM(%d) and FW(%d)",
+		         __func__,
+		         ui32AlignChecksSizeUM,
+		         *paui32FWAlignChecks));
+		eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		goto return_;
+	}
+
+	RGXFwSharedMemCacheOpExec(paui32FWAlignChecks,
+	                          ui32AlignChecksSizeUM * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	for (i = 0; i < ui32AlignChecksSizeUM; i++)
+	{
+		if (aui32AlignChecksUM[i] != paui32FWAlignChecks[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: size/offset mismatch in RGXFW_ALIGN_CHECKS_INIT[%d]"
+					" between UM(%d) and FW(%d)",
+					__func__, i, aui32AlignChecksUM[i], paui32FWAlignChecks[i]));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		}
+	}
+
+	if (eError == PVRSRV_ERROR_INVALID_ALIGNMENT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Check for FW/KM structure"
+				" alignment failed.", __func__));
+	}
+
+return_:
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXAllocateFWMemoryRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_DEVMEM_SIZE_T ui32Size,
+                                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags,
+                                       const IMG_PCHAR pszText,
+                                       DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Align = OSGetPageShift();
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(SUPPORT_MIPS_CONTIGUOUS_FW_MEMORY)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		uiLog2Align = RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiLog2Align);
+#endif
+
+	uiMemAllocFlags = (uiMemAllocFlags |
+					   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+					   PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &
+	                   RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	uiMemAllocFlags &= PVRSRV_MEMALLOCFLAGS_TDFWMASK;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate FW %s memory", pszText);
+
+	eError = DevmemFwAllocate(psDeviceNode->pvDevice,
+	                          ui32Size,
+	                          uiMemAllocFlags,
+	                          pszText,
+	                          ppsMemDescPtr);
+
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver
+
+ @Description
+
+ Validate the FW build options against KM driver build options (KM build options only)
+
+ Following check is redundant, because next check checks the same bits.
+ Redundancy occurs because if client-server are build-compatible and client-firmware are
+ build-compatible then server-firmware are build-compatible as well.
+
+ This check is left for clarity in error messages if any incompatibility occurs.
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+
+	IMG_UINT32			ui32BuildOptions, ui32BuildOptionsFWKMPart, ui32BuildOptionsMismatch;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+	IMG_UINT8               ui8FwOsCount;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32BuildOptions = (RGX_BUILD_OPTIONS_KM & RGX_BUILD_OPTIONS_MASK_FW);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32BuildOptionsFWKMPart = psFwOsInit->sRGXCompChecks.ui32BuildOptions & RGX_BUILD_OPTIONS_MASK_FW;
+
+		ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+		if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32BuildOptionsFWKMPart = psFWInfoHeader->ui32Flags & RGX_BUILD_OPTIONS_MASK_FW;
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo) && BITMASK_HAS(psFWInfoHeader->ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psFWInfoHeader->ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+				return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+			}
+		}
+	}
+
+	/* Check if the FW is missing support for any features required by the driver */
+	if (~ui32BuildOptionsFWKMPart & ui32BuildOptions)
+	{
+		ui32BuildOptionsMismatch = ui32BuildOptions ^ ui32BuildOptionsFWKMPart;
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask non-critical options out as we do support combining them in UM & KM */
+		ui32BuildOptionsMismatch &= FW_OPTIONS_STRICT;
+#endif
+		if ((ui32BuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and KM driver build options; "
+					 "extra options present in the KM driver: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptions & ui32BuildOptionsMismatch));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+
+		if ((ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware-side and KM driver build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch ));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "RGXDevInitCompatCheck: Firmware and KM driver build options differ."));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: Firmware and KM driver build options match. [ OK ]"));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK version against driver DDK version
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32			ui32KMDDKVersion;
+	IMG_UINT32			ui32FWDDKVersion;
+	PVRSRV_ERROR		eError;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32KMDDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKVersion = psFwOsInit->sRGXCompChecks.ui32DDKVersion;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+		ui32FWDDKVersion = PVRVERSION_PACK(psFWInfoHeader->ui16PVRVersionMajor, psFWInfoHeader->ui16PVRVersionMinor);
+	}
+
+	if (ui32FWDDKVersion != ui32KMDDKVersion)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Incompatible driver DDK version (%u.%u) / Firmware DDK version (%u.%u).",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_UNPACK_MAJ(ui32FWDDKVersion),
+				 PVRVERSION_UNPACK_MIN(ui32FWDDKVersion)));
+		eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK version (%u.%u) and Firmware DDK version (%u.%u) match. [ OK ]",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK build against driver DDK build
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32KMDDKBuild;
+	IMG_UINT32			ui32FWDDKBuild;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	ui32KMDDKBuild = PVRVERSION_BUILD;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKBuild = psFwOsInit->sRGXCompChecks.ui32DDKBuild;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32FWDDKBuild = psFWInfoHeader->ui32PVRVersionBuild;
+	}
+
+	if (ui32FWDDKBuild != ui32KMDDKBuild)
+	{
+		PVR_LOG(("(WARN) RGXDevInitCompatCheck: Different driver DDK build version (%d) / Firmware DDK build version (%d).",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		eError = PVRSRV_ERROR_DDK_BUILD_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK build version (%d) and Firmware DDK build version (%d) match. [ OK ]",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+	}
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_BVNC_FWAgainstDriver
+
+ @Description
+
+ Validate FW BVNC against driver BVNC
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_BVNC_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+
+	RGX_FW_INFO_HEADER	*psFWInfoHeader;
+	IMG_UINT64			ui64KMBVNC;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+	ui64KMBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+							   psDevInfo->sDevFeatureCfg.ui32V,
+							   psDevInfo->sDevFeatureCfg.ui32N,
+							   psDevInfo->sDevFeatureCfg.ui32C);
+
+	if (ui64KMBVNC != psFWInfoHeader->ui64BVNC)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in KM driver BVNC (%u.%u.%u.%u) and Firmware BVNC (%u.%u.%u.%u)",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_ERROR_BVNC_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: KM driver BVNC (%d.%d.%d.%d) and FW BVNC (%d.%d.%d.%d) match. [ OK ]",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevInitCompatCheck
+
+ @Description
+
+ Check compatibility of host driver and firmware (DDK and build options)
+ for RGX devices at services/device initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32			ui32FwTimeout = MAX_HW_TIME_US;
+
+		LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Flush covers this instance and the reads in the functions below */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                               INVALIDATE);
+		if (!*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+		{
+			eError = PVRSRV_ERROR_TIMEOUT;
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU Firmware not responding: failed to supply compatibility info (%u)",
+					__func__, eError));
+
+			PVR_DPF((PVR_DBG_ERROR, "%s: Potential causes: firmware not initialised or the current Guest driver's "
+									"OsConfig initialisation data was not accepted by the firmware", __func__));
+			goto chk_exit;
+		}
+	}
+#endif /* defined(NO_HARDWARE) */
+
+	eError = RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDevInitCompatCheck_BVNC_FWAgainstDriver(psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			goto chk_exit;
+		}
+	}
+
+	eError = PVRSRV_OK;
+chk_exit:
+
+	return eError;
+}
+
+static void _RGXSoftResetToggle(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT64  ui64ResetValue1,
+                                IMG_UINT64  ui64ResetValue2)
+{
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, ui64ResetValue1);
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, POWER_ISLAND_VERSION) == 1)
+	{
+		OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET_SPU, ui64ResetValue2);
+	}
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, POWER_ISLAND_VERSION) == 1)
+	{
+		(void) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET_SPU);
+	}
+}
+
+/**************************************************************************/ /*!
+@Function       RGXSoftReset
+@Description    Resets some modules of the RGX device
+@Input          psDeviceNode		Device node
+@Input          ui64ResetValue1 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET
+                                register).
+@Input          ui64ResetValue2 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET_SPU
+                                register).
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT64  ui64ResetValue1,
+                                 IMG_UINT64  ui64ResetValue2)
+{
+	PVRSRV_RGXDEV_INFO        *psDevInfo;
+	IMG_BOOL	bSoftReset = IMG_FALSE;
+	IMG_UINT64	ui64SoftResetMask = 0;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+#if defined(RGX_CR_SOFT_RESET2_MASKFULL)
+	if ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE)) &&
+			((ui64ResetValue2 & RGX_CR_SOFT_RESET2_MASKFULL) != ui64ResetValue2))
+	{
+		bSoftReset = IMG_TRUE;
+	}
+#elif defined(RGX_CR_SOFT_RESET_SPU_MASKFULL)
+	if ((ui64ResetValue2 & RGX_CR_SOFT_RESET_SPU_MASKFULL) != ui64ResetValue2)
+	{
+		bSoftReset = IMG_TRUE;
+	}
+#endif
+
+	if (((ui64ResetValue1 & ui64SoftResetMask) != ui64ResetValue1) || bSoftReset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Set in soft-reset */
+	_RGXSoftResetToggle(psDevInfo, ui64ResetValue1, ui64ResetValue2);
+
+	/* Take the modules out of reset... */
+	_RGXSoftResetToggle(psDevInfo, 0, 0);
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static const RGX_MIPS_ADDRESS_TRAMPOLINE sNullTrampoline;
+
+static void RGXFreeTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+			psDevInfo->psTrampoline->hPdumpPages,
+#endif
+			&psDevInfo->psTrampoline->sPages);
+
+	if (psDevInfo->psTrampoline != &sNullTrampoline)
+	{
+		OSFreeMem(psDevInfo->psTrampoline);
+	}
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+}
+
+#define RANGES_OVERLAP(x,y,size) (x < (y+size) && y < (x+size))
+#define TRAMPOLINE_ALLOC_MAX_RETRIES (3)
+
+static PVRSRV_ERROR RGXAllocTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT32 i, j;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_MIPS_ADDRESS_TRAMPOLINE *pasTrampoline[TRAMPOLINE_ALLOC_MAX_RETRIES];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate pages for trampoline");
+
+	/* Retry the allocation of the trampoline block (16KB), retaining any
+	 * previous allocations overlapping  with the target range until we get an
+	 * allocation that doesn't overlap with the target range.
+	 * Any allocation like this will require a maximum of 3 tries as we are
+	 * allocating a physical contiguous block of memory, not individual pages.
+	 * Free the unused allocations at the end only after the desired range
+	 * is obtained to prevent the alloc function from returning the same bad
+	 * range repeatedly.
+	 */
+	for (i = 0; i < TRAMPOLINE_ALLOC_MAX_RETRIES; i++)
+	{
+		pasTrampoline[i] = OSAllocMem(sizeof(RGX_MIPS_ADDRESS_TRAMPOLINE));
+		eError = DevPhysMemAlloc(psDeviceNode,
+				RGXMIPSFW_TRAMPOLINE_SIZE,
+				RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE,
+				0,         // (init) u8Value
+				IMG_FALSE, // bInitPage,
+#if defined(PDUMP)
+				psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				"TrampolineRegion",
+				&pasTrampoline[i]->hPdumpPages,
+#endif
+				PVR_SYS_ALLOC_PID,
+				&pasTrampoline[i]->sPages,
+				&pasTrampoline[i]->sPhysAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s failed (%u)",
+			         __func__,
+			         eError));
+			goto fail;
+		}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		/* Set the persistent uiOSid value so that we free from the correct
+		 * base arena when unloading the driver and freeing the trampoline.
+		 */
+		pasTrampoline[i]->sPages.uiOSid = 0;	/* Firmware global arena */
+#endif
+
+		if (!RANGES_OVERLAP(pasTrampoline[i]->sPhysAddr.uiAddr,
+				RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+				RGXMIPSFW_TRAMPOLINE_SIZE))
+		{
+			break;
+		}
+	}
+	if (TRAMPOLINE_ALLOC_MAX_RETRIES == i)
+	{
+		/* Failed to find a physical allocation after 3 attempts */
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s failed to allocate non-overlapping pages (%u)",
+				__func__, eError));
+		/* Fall through, clean up and return error. */
+	}
+	else
+	{
+		/* Remember the last physical block allocated, it will not be freed */
+		psDevInfo->psTrampoline = pasTrampoline[i];
+	}
+
+fail:
+	/* free all unused allocations */
+	for (j = 0; j < i; j++)
+	{
+		DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+				pasTrampoline[j]->hPdumpPages,
+#endif
+				&pasTrampoline[j]->sPages);
+		OSFreeMem(pasTrampoline[j]);
+	}
+
+	return eError;
+}
+
+#undef RANGES_OVERLAP
+#endif
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    uiFWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR        eError;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEVMEM_SIZE_T	uiDummyLen;
+	DEVMEM_MEMDESC		*psDummyMemDesc = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		eError = RGXAllocTrampoline(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Failed to allocate trampoline region (%u)",
+					eError));
+			goto failTrampolineMemDescAlloc;
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW code section
+	 */
+	uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+	                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE);
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWCodeLen,
+	                                   uiMemAllocFlags,
+	                                   "FwCodeRegion",
+	                                   &psDevInfo->psRGXFWCodeMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc,
+	                                  &psDevInfo->sFWCodeDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAqDevVirt;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) || (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))))
+#endif
+	{
+		/*
+		 * The FW code must be the first allocation in the firmware heap, otherwise
+		 * the bootloader will not work (the FW will not be able to find the bootloader).
+		 */
+		PVR_ASSERT(psDevInfo->sFWCodeDevVAddrBase.uiAddr == RGX_FIRMWARE_RAW_HEAP_BASE);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/*
+		 * Allocate Dummy Pages so that Data segment allocation gets the same
+		 * device virtual address as specified in MIPS firmware linker script
+		 */
+		uiDummyLen = RGXGetFWImageSectionMaxSize(NULL, MIPS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_EXCEPTIONS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_BOOT_CODE) -
+				uiFWCodeLen; /* code actual size */
+
+		if (uiDummyLen > 0)
+		{
+			eError = DevmemFwAllocateExportable(psDeviceNode,
+					uiDummyLen,
+					OSGetPageSize(),
+					uiMemAllocFlags,
+					"FwExDummyPages",
+					&psDummyMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "Failed to allocate fw dummy mem (%u)",
+				         eError));
+				goto failDummyMemDescAlloc;
+			}
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW data section
+	 */
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWDataLen,
+	                                   RGX_FWCODEDATA_ALLOCFLAGS |
+	                                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA),
+	                                   "FwDataRegion",
+	                                   &psDevInfo->psRGXFWDataMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+	                                  &psDevInfo->sFWDataDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAqDevVirt;
+	}
+
+	if (uiFWCorememCodeLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem code section
+		 */
+		uiMemAllocFlags = (RGX_FWCODEDATA_ALLOCFLAGS |
+		                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE)) &
+		                   ~PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE;
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+		                                   uiFWCorememCodeLen,
+		                                   uiMemAllocFlags,
+		                                   "FwCorememCodeRegion",
+		                                   &psDevInfo->psRGXFWCorememCodeMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem code mem, size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememCodeLen, uiMemAllocFlags, eError));
+			goto failFWCorememCodeMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc,
+		                                  &psDevInfo->sFWCorememCodeDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem code (%u)",
+			         eError));
+			goto failFWCorememCodeMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememCodeFWAddr,
+		                      psDevInfo->psRGXFWCorememCodeMemDesc,
+		                      0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", failFWCorememCodeMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememCodeDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememCodeFWAddr.ui32Addr = 0;
+	}
+
+	if (uiFWCorememDataLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem data section
+		 */
+		uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+		                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA);
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+				uiFWCorememDataLen,
+				uiMemAllocFlags,
+				"FwCorememDataRegion",
+				&psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem data mem, "
+			         "size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememDataLen,
+			         uiMemAllocFlags,
+			         eError));
+			goto failFWCorememDataMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				&psDevInfo->sFWCorememDataStoreDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem data (%u)",
+			         eError));
+			goto failFWCorememDataMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememDataStoreFWAddr,
+				psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", failFWCorememDataMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememDataStoreDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememDataStoreFWAddr.ui32Addr = 0;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Free Dummy Pages */
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failFWCorememDataMemDescFwAddr:
+failFWCorememDataMemDescAqDevVirt:
+	if (uiFWCorememDataLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+failFWCorememDataMemDescAlloc:
+failFWCorememCodeMemDescFwAddr:
+failFWCorememCodeMemDescAqDevVirt:
+	if (uiFWCorememCodeLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+failFWCorememCodeMemDescAlloc:
+failFWDataMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+	psDevInfo->psRGXFWDataMemDesc = NULL;
+failFWDataMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+failDummyMemDescAlloc:
+#endif
+failFWCodeMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+	psDevInfo->psRGXFWCodeMemDesc = NULL;
+failFWCodeMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		RGXFreeTrampoline(psDeviceNode);
+	}
+failTrampolineMemDescAlloc:
+#endif
+	return eError;
+}
+
+/*
+	AppHint parameter interface
+ */
+static
+PVRSRV_ERROR RGXFWTraceQueryFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	*pui32Value &= RGXFWIF_LOG_TYPE_GROUP_MASK;
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceQueryLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	if (PVRSRV_OK == eResult)
+	{
+		if (*pui32Value & RGXFWIF_LOG_TYPE_TRACE)
+		{
+			*pui32Value = 0; /* Trace */
+		}
+		else
+		{
+			*pui32Value = 1; /* TBI */
+		}
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 const void *psPrivate,
+                                 IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType;
+
+	eResult = RGXFWTraceQueryLogType(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK == eResult)
+	{
+		if (0 == ui32RGXFWLogType)
+		{
+			BITMASK_SET(ui32Value, RGXFWIF_LOG_TYPE_TRACE);
+		}
+		eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32Value);
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType = ui32Value;
+
+	eResult = RGXFWTraceQueryFilter(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK != eResult)
+	{
+		return eResult;
+	}
+
+	/* 0 - trace, 1 - tbi */
+	if (0 == ui32Value)
+	{
+		BITMASK_SET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	else if (1 == ui32Value)
+	{
+		BITMASK_UNSET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid parameter %u specified to set FW log type AppHint.",
+		         __func__, ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32RGXFWLogType);
+	return eResult;
+}
+
+#if defined(DEBUG)
+static
+PVRSRV_ERROR RGXQueryFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL *pbValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pbValue = (PVRSRV_MEMALLOCFLAG_POISON_ON_FREE == psDevInfo->uiFWPoisonOnFreeFlag)
+		? IMG_TRUE
+		: IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXSetFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL bValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	psDevInfo->uiFWPoisonOnFreeFlag = bValue
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+ * RGXInitFirmware
+ */
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+#if defined(DEBUG)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bEnableFWPoisonOnFree = IMG_FALSE;
+#endif
+
+	eError = RGXSetupFirmware(psDeviceNode,
+	                          psApphints,
+	                          ui32ConfigFlags,
+	                          ui32ConfigFlagsExt,
+	                          ui32FwOsCfgFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXInitFirmwareKM: RGXSetupFirmware failed (%u)",
+		         eError));
+		goto failed_init_firmware;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableLogGroup,
+		                                    RGXFWTraceQueryFilter,
+		                                    RGXFWTraceSetFilter,
+		                                    psDeviceNode,
+		                                    NULL);
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_FirmwareLogType,
+		                                    RGXFWTraceQueryLogType,
+		                                    RGXFWTraceSetLogType,
+		                                    psDeviceNode,
+		                                    NULL);
+	}
+
+#if defined(DEBUG)
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFWPOISONONFREE;
+	OSGetAppHintBOOL(psDeviceNode,
+			pvAppHintState,
+			EnableFWPoisonOnFree,
+			&bAppHintDefault,
+			&bEnableFWPoisonOnFree);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFWPoisonOnFree,
+	                                   RGXQueryFWPoisonOnFree,
+	                                   RGXSetFWPoisonOnFree,
+	                                   psDeviceNode,
+	                                   NULL);
+
+	psDevInfo->uiFWPoisonOnFreeFlag = bEnableFWPoisonOnFree
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+#else
+	psDevInfo->uiFWPoisonOnFreeFlag = 0ULL;
+#endif
+
+	psDevInfo->ui32ClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	psDevInfo->ui32LastClockSource = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+
+	return PVRSRV_OK;
+
+failed_init_firmware:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* See device.h for function declaration */
+static PVRSRV_ERROR RGXAllocUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+									 IMG_UINT32 ui32RequestedSize,
+									 DEVMEM_MEMDESC **psMemDesc,
+									 IMG_UINT32 *puiSyncPrimVAddr,
+									 IMG_UINT32 *puiSyncPrimBlockSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR pFirmwareAddr;
+	IMG_DEVMEM_ALIGN_T uiUFOBlockAlign = MAX(sizeof(IMG_UINT32), sizeof(SYNC_CHECKPOINT_FW_OBJ));
+	IMG_DEVMEM_SIZE_T uiUFOBlockSize = PVR_ALIGN(ui32RequestedSize, uiUFOBlockAlign);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Size and align are 'expanded' because we request an Exportalign allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiUFOBlockSize,
+	                                             &uiUFOBlockAlign);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										uiUFOBlockSize,
+										uiUFOBlockAlign,
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_UNCACHED,
+										"FwExUFOBlock",
+										psMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = RGXSetFirmwareAddress(&pFirmwareAddr, *psMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = pFirmwareAddr.ui32Addr;
+	*puiSyncPrimBlockSize = TRUNCATE_64BITS_TO_32BITS(uiUFOBlockSize);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, *psMemDesc);
+e0:
+	return eError;
+}
+
+/* See device.h for function declaration */
+static void RGXFreeUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+							DEVMEM_MEMDESC *psMemDesc)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	RGXUnsetFirmwareAddress(psMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psMemDesc);
+}
+
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	psDevInfo->bDevInit2Done = IMG_FALSE;
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	if (psDevInfo->psGenHeapSecMem != NULL)
+	{
+		OSFreeSecBuf(psDevInfo->psGenHeapSecMem);
+	}
+#endif
+
+
+	if ((psDevInfo->hTQUSCSharedMem != NULL) &&
+		(psDevInfo->hTQCLISharedMem != NULL))
+	{
+		PVRSRVTQUnloadShaders(psDeviceNode);
+	}
+
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvLISRData != NULL)
+	{
+		(void) SysUninstallDeviceLISR(psDevInfo->pvLISRData);
+	}
+	if (psDevInfo->pvMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvMISRData);
+	}
+	if (psDevInfo->hProcessQueuesMISR != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->hProcessQueuesMISR);
+	}
+	if (psDevInfo->pvAPMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvAPMISRData);
+	}
+	if (psDeviceNode->hCmdCompNotify != NULL)
+	{
+		/* Cancel notifications to this device */
+		PVRSRVUnregisterCmdCompleteNotify(psDeviceNode->hCmdCompNotify);
+		psDeviceNode->hCmdCompNotify = NULL;
+	}
+#endif /* !NO_HARDWARE */
+
+	/* Remove the device from the power manager */
+	PVRSRVRemovePowerDevice(psDeviceNode);
+
+	psDevInfo->pfnGetGpuUtilStats = NULL;
+	if (psDevInfo->hGPUUtilLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hGPUUtilLock);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(psDevInfo->hNMILock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hNMILock);
+	}
+#endif
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(psDevInfo->hMMUCtxUnregLock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	if (psDevInfo->hDebugFaultInfoLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hDebugFaultInfoLock);
+	}
+
+	/* De-init Freelists/ZBuffers... */
+	if (psDevInfo->hLockFreeList != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockFreeList);
+	}
+
+	if (psDevInfo->hLockZSBuffer != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockZSBuffer);
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* De-init work estimation lock */
+		if (psDevInfo->hWorkEstLock != NULL)
+		{
+			OSLockDestroy(psDevInfo->hWorkEstLock);
+		}
+	}
+#endif
+
+	/* Free DVFS Table */
+	if (psDevInfo->psGpuDVFSTable != NULL)
+	{
+		OSFreeMem(psDevInfo->psGpuDVFSTable);
+		psDevInfo->psGpuDVFSTable = NULL;
+	}
+}
+
+/*
+	DevDeInitRGX
+ */
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO		*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	PVRSRV_ERROR			eError;
+	DEVICE_MEMORY_INFO		*psDevMemoryInfo;
+
+	if (!psDevInfo)
+	{
+		/* Can happen if DevInitRGX failed */
+		PVR_DPF((PVR_DBG_ERROR, "DevDeInitRGX: Null DevInfo"));
+		return PVRSRV_OK;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	RGXUnregisterBridges(psDevInfo);
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	OSLockDestroy(psDevInfo->hCounterDumpingLock);
+#endif
+
+	/* Unregister debug request notifiers first as they could depend on anything. */
+
+	RGXDebugDeinit(psDevInfo);
+
+	/* De-initialise in reverse order, so stage 2 init is undone first. */
+	if (psDevInfo->bDevInit2Done)
+	{
+		DevPart2DeInitRGX(psDeviceNode);
+	}
+
+	/* Unregister MMU related stuff */
+	eError = RGXMMUInit_Unregister(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevDeInitRGX: Failed RGXMMUInit_Unregister (0x%x)",
+		         eError));
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* Unregister MMU related stuff */
+		eError = RGXMipsMMUInit_Unregister(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed RGXMipsMMUInit_Unregister (0x%x)",
+			         eError));
+		}
+	}
+#endif
+
+	/* UnMap Regs */
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+						 psDevInfo->ui32RegSize);
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvRegsBaseKM = NULL;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+			(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* undo the VA offset performed in RGXRegisterDevice() to allow the allocation to be unmapped */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM, RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvSecureRegsBaseKM = NULL;
+	}
+#endif
+
+#if 0 /* not required at this time */
+	if (psDevInfo->hTimer)
+	{
+		eError = OSRemoveTimer(psDevInfo->hTimer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed to remove timer"));
+			return eError;
+		}
+		psDevInfo->hTimer = NULL;
+	}
+#endif
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+
+	RGXDeInitHeaps(psDevMemoryInfo, psDeviceNode);
+
+	if (psDevInfo->psRGXFWCodeMemDesc)
+	{
+		/* Free fw code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+		psDevInfo->psRGXFWCodeMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware code memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		if (psDevInfo->psTrampoline->sPages.u.pvHandle)
+		{
+			/* Free trampoline region */
+			PDUMPCOMMENT(psDeviceNode, "Freeing trampoline memory");
+			RGXFreeTrampoline(psDeviceNode);
+		}
+	}
+#endif
+
+	if (psDevInfo->psRGXFWDataMemDesc)
+	{
+		/* Free fw data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW data memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+		psDevInfo->psRGXFWDataMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware data memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+	if (psDevInfo->psRGXFWCorememCodeMemDesc)
+	{
+		/* Free fw core mem code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+	{
+		/* Free fw core mem data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem data store memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+
+	/*
+	   Free the firmware allocations.
+	 */
+	RGXFreeFirmware(psDevInfo);
+
+
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+
+	/* De-initialise non-device specific (TL) users of RGX device memory */
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+		{
+			RGXHWPerfDeinitL2Stream(psDevInfo, i);
+		}
+
+		RGXHWPerfDeinit(psDevInfo);
+	}
+
+	RGXDeInitDestroyFWKernelMemoryContext(psDeviceNode);
+
+	RGXHWPerfHostDeInit(psDevInfo);
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+
+	/* destroy the stalled CCB locks */
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+
+	/* destroy the context list locks */
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+	OSLockDestroy(psDevInfo->hBPLock);
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+
+	/* Free device BVNC string */
+	if (NULL != psDevInfo->sDevFeatureCfg.pszBVNCString)
+	{
+		OSFreeMem(psDevInfo->sDevFeatureCfg.pszBVNCString);
+	}
+
+
+	/* DeAllocate devinfo */
+	OSFreeMem(psDevInfo);
+
+	psDeviceNode->pvDevice = NULL;
+
+	return PVRSRV_OK;
+}
+
+#if defined(PDUMP)
+static
+PVRSRV_ERROR RGXResetPDump(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/* Takes a log2 page size parameter and calculates a suitable page size
+ * for the RGX heaps. Returns 0 if parameter is wrong.*/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize)
+{
+	IMG_BOOL bFound = IMG_FALSE;
+	IMG_UINT32 ui32PageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	/* OS page shift must be at least RGX_HEAP_4KB_PAGE_SHIFT,
+	 * max RGX_HEAP_2MB_PAGE_SHIFT, non-zero and a power of two*/
+	if (uiLog2PageSize == 0U ||
+	    (uiLog2PageSize < RGX_HEAP_4KB_PAGE_SHIFT) ||
+	    (uiLog2PageSize > RGX_HEAP_2MB_PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Provided incompatible log2 page size %u",
+				__func__,
+				uiLog2PageSize));
+		PVR_ASSERT(0);
+		return 0;
+	}
+
+	do
+	{
+		if ((IMG_PAGE2BYTES32(uiLog2PageSize) & ui32PageSizeMask) == 0)
+		{
+			/* We have to fall back to a smaller device
+			 * page size than given page size because there
+			 * is no exact match for any supported size. */
+			uiLog2PageSize -= 1U;
+		}
+		else
+		{
+			/* All good, RGX page size equals given page size
+			 * => use it as default for heaps */
+			bFound = IMG_TRUE;
+		}
+	} while (!bFound);
+
+	return uiLog2PageSize;
+}
+
+/* First 16-bits define possible types */
+#define HEAP_INST_VALUE_MASK     (0xFFFF)
+#define HEAP_INST_DEFAULT_VALUE  (1U)  /* Used to show either the heap is always instantiated by default (pfn = NULL)
+	                                      OR
+	                                      that this is the default configuration of the heap with an Alternative BRN */
+#define HEAP_INST_BRN_DEP_VALUE  (2U)  /* The inclusion of this heap is dependent on the brn being present */
+#define HEAP_INST_FEAT_DEP_VALUE (3U)  /* The inclusion of this heap is dependent on the feature being present */
+#define HEAP_INST_BRN_ALT_VALUE  (4U)  /* This entry is a possible alternative to the default determined by a BRN */
+#define HEAP_INST_FEAT_ALT_VALUE (5U)  /* The entry is a possible alternative to the default determined by a Feature define */
+
+/* Latter 16-bits define other flags we may need */
+#define HEAP_INST_NON4K_FLAG     (1 << 16U) /* This is a possible NON4K Entry and we should use the device
+                                               NON4K size when instantiating */
+
+typedef struct RGX_HEAP_INFO_TAG RGX_HEAP_INFO; // Forward declaration
+typedef IMG_BOOL (*PFN_IS_PRESENT)(PVRSRV_RGXDEV_INFO*, const RGX_HEAP_INFO*);
+typedef void (*PFN_HEAP_DYNAMIC)(PVRSRV_DEVICE_NODE*, RGX_HEAP_INFO*);
+
+struct RGX_HEAP_INFO_TAG
+{
+	IMG_CHAR           *pszName;
+	IMG_UINT64         ui64HeapBase;
+	IMG_DEVMEM_SIZE_T  uiHeapLength;
+	IMG_DEVMEM_SIZE_T  uiHeapReservedRegionLength;
+	IMG_UINT32         ui32Log2ImportAlignment;
+	PFN_IS_PRESENT     pfnIsHeapPresent;
+	PFN_HEAP_DYNAMIC   pfnDynamicBaseSize; /* May modify the psHeapInfo's base & length. May be NULL.
+	                                          Only called once if the heap is present otherwise never. */
+	PFN_HEAP_INIT      pfnInit;
+	PFN_HEAP_DEINIT    pfnDeInit;
+	IMG_UINT32         ui32HeapInstanceFlags;
+};
+
+/* RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES is the total amount of reserved space, to be specified in gasRGXHeapLayoutApp[] */
+#define RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES	(RGX_HEAP_UM_GENERAL_RESERVED_SIZE + RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE)
+
+/* Private heap data struct. */
+typedef struct RGX_HEAP_DATA_TAG
+{
+	DEVMEMINT_RESERVATION  *psMemReservation;
+} RGX_HEAP_DATA;
+
+static PVRSRV_ERROR HeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_HEAP *psDevmemHeap,
+							 PMR *psPMR,
+							 IMG_DEVMEM_SIZE_T uiSize,
+							 IMG_UINT64 ui64Offset,
+							 IMG_BOOL bWriteAble,
+							 IMG_HANDLE *phPrivData)
+{
+	RGX_HEAP_DATA *psHeapData;
+	IMG_DEV_VIRTADDR sCarveOutAddr;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap, "psDevmemHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phPrivData, "phPrivData");
+
+	psHeapData = OSAllocMem(sizeof(*psHeapData));
+	PVR_LOG_RETURN_IF_NOMEM(psHeapData, "psHeapData");
+
+	/* Map the per device secure mem PMR allocation to the general devmem heap carveout. */
+	sCarveOutAddr = DevmemIntHeapGetBaseAddr(psDevmemHeap);
+	sCarveOutAddr.uiAddr += ui64Offset;
+
+	eError = DevmemIntReserveRange(NULL,
+	                               NULL,
+	                               psDevmemHeap,
+	                               sCarveOutAddr,
+	                               uiSize,
+	                               PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                               (bWriteAble ? PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE : 0),
+	                               &psHeapData->psMemReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeHeapData);
+
+	eError = DevmemIntMapPMR(psHeapData->psMemReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	*phPrivData = (IMG_HANDLE)psHeapData;
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+ErrorFreeHeapData:
+	OSFreeMem(psHeapData);
+
+	return eError;
+}
+
+/* Deinit callback function for general heap. */
+static void HeapDeInit(IMG_HANDLE hPrivData)
+{
+	RGX_HEAP_DATA *psHeapData = (RGX_HEAP_DATA*)hPrivData;
+
+	PVR_ASSERT(hPrivData);
+
+	DevmemIntUnmapPMR(psHeapData->psMemReservation);
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+
+	OSFreeMem(psHeapData);
+}
+
+/* Init callback function for general heap. */
+static PVRSRV_ERROR USCHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_FALSE;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	uiSize = PMR_PhysicalSize(psDevInfo->hTQUSCSharedMem);
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->hTQUSCSharedMem,
+					  uiSize,
+					  RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+/* Init callback function for general heap. */
+static PVRSRV_ERROR GeneralHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_TRUE;
+	PVRSRV_ERROR eError;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->psGenHeapSecMem,
+					  RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE,
+					  RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#define GeneralHeapDeInit HeapDeInit
+#else
+/* Callbacks not used */
+#define GeneralHeapInit NULL
+#define GeneralHeapDeInit NULL
+#endif
+
+static void SVMHeapDynamic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           RGX_HEAP_INFO *psHeapInfo)
+{
+	IMG_UINT64 ui64OSPageSize = OSGetPageSize();
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+
+	/* Ensures the SVM heap has the correct alignment & size for any OS page size.
+	 *
+	 * The SVM heap's base must be the smallest possible address mappable by UM.
+	 * This is 32KB unless the page size is larger than 32KB. [1]
+	 * If the page size > 32KB, raise the SVM heap base to the next page boundary.
+	 * Also reduce the length to ensure it's still page aligned and doesn't go
+	 * into another heap.
+	 *
+	 * [1]: https://chromium.googlesource.com/chromium/src/+/fe24932ee14aa93e1fe4d3e7003b9362591a54d4/docs/security/faq.md#why-aren_t-null-pointer-dereferences-considered-security-bugs
+	 */
+	IMG_UINT64 ui64Base = PVR_ALIGN(psHeapInfo->ui64HeapBase, ui64OSPageSize);
+	IMG_UINT64 ui64BaseDiff = ui64Base - psHeapInfo->ui64HeapBase;
+	psHeapInfo->ui64HeapBase = ui64Base;
+	if (psHeapInfo->uiHeapLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapLength -= ui64BaseDiff;
+	if (psHeapInfo->uiHeapReservedRegionLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapReservedRegionLength -= ui64BaseDiff;
+
+	/* The device shared-virtual-memory heap address-space size is stored on the device for
+	   faster look-up without having to walk the device heap configuration structures during
+	   client device connection  (i.e. this size is relative to a zero-based offset) */
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = 0;
+	}
+	else
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = psHeapInfo->ui64HeapBase + psHeapInfo->uiHeapLength;
+	}
+}
+
+/* Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+static IMG_BOOL BRN65273IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		return (((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_ALT_VALUE) ||
+		        ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_DEP_VALUE)) ?
+		        IMG_TRUE : IMG_FALSE;
+	}
+	else
+	{
+		return ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_DEFAULT_VALUE) ? IMG_TRUE : IMG_FALSE;
+	}
+}
+#endif
+
+#if defined(FIX_HW_BRN_63142_BIT_MASK)
+static IMG_BOOL BRN63142IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 63142))
+	{
+		PVR_ASSERT((pksHeapInfo->ui64HeapBase & IMG_UINT64_C(0x3FFFFFFFF)) +
+		            pksHeapInfo->uiHeapLength == IMG_UINT64_C(0x400000000));
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+#endif
+
+/* FW Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+static IMG_BOOL FWBRN65101IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap size and base at run time VS the current table instance
+	 */
+	IMG_UINT64 ui64MainSubHeapSize;
+
+	/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)))
+	{
+		if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101;
+		}
+		else
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL;
+		}
+	}
+	else
+	{
+		ui64MainSubHeapSize = RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE;
+	}
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->uiHeapLength == ui64MainSubHeapSize &&
+	        pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_MAIN_HEAP_BASE) ?
+	        IMG_TRUE : IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FWVZConfigPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap base at run time VS the current table instance
+	 */
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_CONFIG_HEAP_BASE) ? IMG_TRUE : IMG_FALSE;
+}
+
+static IMG_BOOL FWVZMainHeapPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap base at run time VS the current table instance
+	 */
+	return (pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_MAIN_HEAP_BASE) ? IMG_TRUE : IMG_FALSE;
+}
+
+static IMG_BOOL BPHandlerHeapPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+	return IMG_TRUE;
+#else
+	return IMG_FALSE;
+#endif
+}
+
+static RGX_HEAP_INFO gasRGXHeapLayoutApp[] =
+{
+	/* Name                             HeapBase                                 HeapLength                               HeapReservedRegionLength                     Log2ImportAlignment pfnIsHeapPresent      pfnDynamicBaseSize pfnInit           pfnDeInit          HeapInstanceFlags   */
+	{RGX_GENERAL_SVM_HEAP_IDENT,        RGX_GENERAL_SVM_HEAP_BASE,               RGX_GENERAL_SVM_HEAP_SIZE,               0,                                           0,                  NULL,                 SVMHeapDynamic,    NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_HEAP_BASE,                   RGX_GENERAL_HEAP_SIZE,                   RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  NULL,                 NULL,              GeneralHeapInit,  GeneralHeapDeInit, HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_HEAP_BASE,             RGX_GENERAL_NON4K_HEAP_SIZE,             0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_HEAP_BASE,               RGX_PDSCODEDATA_HEAP_SIZE,               RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_HEAP_BASE,                   RGX_USCCODE_HEAP_SIZE,                   RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  NULL,                 NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_DEFAULT_VALUE },
+	{RGX_USCCODE_BPH_HEAP_IDENT,        RGX_USCCODE_BPH_HEAP_BASE,               RGX_USCCODE_BPH_HEAP_SIZE,               0,                                           0,                  BPHandlerHeapPresent, NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_VK_CAPT_REPLAY_HEAP_IDENT,     RGX_VK_CAPT_REPLAY_HEAP_BASE,            RGX_VK_CAPT_REPLAY_HEAP_SIZE,            0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_COMPONENT_CTRL_HEAP_IDENT,     RGX_COMPONENT_CTRL_HEAP_BASE,            RGX_COMPONENT_CTRL_HEAP_SIZE,            0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_FBCDC_HEAP_IDENT,              RGX_FBCDC_HEAP_BASE,                     RGX_FBCDC_HEAP_SIZE,                     0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_FBCDC_LARGE_HEAP_IDENT,        RGX_FBCDC_LARGE_HEAP_BASE,               RGX_FBCDC_LARGE_HEAP_SIZE,               0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_PDS_INDIRECT_STATE_HEAP_IDENT, RGX_PDS_INDIRECT_STATE_HEAP_BASE,        RGX_PDS_INDIRECT_STATE_HEAP_SIZE,        0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_CMP_MISSION_RMW_HEAP_IDENT,    RGX_CMP_MISSION_RMW_HEAP_BASE,           RGX_CMP_MISSION_RMW_HEAP_SIZE,           0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_CMP_SAFETY_RMW_HEAP_IDENT,     RGX_CMP_SAFETY_RMW_HEAP_BASE,            RGX_CMP_SAFETY_RMW_HEAP_SIZE,            0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TEXTURE_STATE_HEAP_IDENT,      RGX_TEXTURE_STATE_HEAP_BASE,             RGX_TEXTURE_STATE_HEAP_SIZE,             0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_HEAP_BASE,           RGX_VISIBILITY_TEST_HEAP_SIZE,           0,                                           0,                  NULL,                 NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+};
+
+static RGX_HEAP_INFO gasRGXHeapLayoutFW[] =
+{
+	/* Name                          HeapBase                        HeapLength                                      HeapReservedRegionLength Log2ImportAlignment pfnIsHeapPresent     pfnDynamicBaseSize pfnInit pfnDeInit HeapInstanceFlags*/
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE,            0,                       0,                  FWVZMainHeapPresent, NULL,              NULL,   NULL,     HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_CONFIG_HEAP_IDENT, RGX_FIRMWARE_CONFIG_HEAP_BASE,  RGX_FIRMWARE_CONFIG_HEAP_SIZE,                  0,                       0,                  FWVZConfigPresent,   NULL,              NULL,   NULL,     HEAP_INST_DEFAULT_VALUE},
+};
+
+static INLINE IMG_BOOL IsFwHeapLayout(const RGX_HEAP_INFO *psHeapInfo)
+{
+	return psHeapInfo->pszName[0] == 'F' &&
+	       psHeapInfo->pszName[1] == 'w' ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE void CheckHeapAlignment(const RGX_HEAP_INFO *psHeapInfo,
+                                      const PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 uiAlignment = RGX_HEAP_BASE_SIZE_ALIGN - 1;
+
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+
+	if (IsFwHeapLayout(psHeapInfo))
+	{
+		/*
+		 * 1) Main heap starts at 2MB offset 0xEC10000000UL
+		 * 2) Config Sub heap is created at the end of the main heap making the entire unit start and end at 2MB offset aligned,
+		 *    these 2 heaps will always have the same page size
+		 *    There are no other heaps in between these two heaps, there is no risk for another devmem heap be created between them.
+		 */
+		return;
+	}
+
+	/* General SVM Heap must be placed below 2MiB boundary so we need to adjust
+	 * the validity condition. This is because an OS might return virtual
+	 * addresses below 2MiB threshold. By default (based on testing on Linux)
+	 * this is around 32KiB. */
+	if (OSStringNCompare(psHeapInfo->pszName, RGX_GENERAL_SVM_HEAP_IDENT,
+	                     sizeof(RGX_GENERAL_SVM_HEAP_IDENT)) == 0)
+	{
+		uiAlignment = RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT - 1;
+	}
+
+	/* All UM accessible heap bases should be aligned to 2MB */
+	if (psHeapInfo->ui64HeapBase & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Base not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Base: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Base (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->ui64HeapBase,
+		         psHeapInfo->ui64HeapBase,
+		         uiAlignment + 1));
+	}
+
+	/* All UM accessible heaps should also be size aligned to 2MB */
+	if (psHeapInfo->uiHeapLength & uiAlignment)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Heap Size not aligned to RGX_HEAP_BASE_SIZE_ALIGN. "
+		         "Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC")\n"
+		         "Heap Size (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_SIZE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->uiHeapLength,
+		         psHeapInfo->uiHeapLength,
+		         uiAlignment + 1));
+	}
+}
+
+/* Generic counting method. */
+static void _CountRequiredHeaps(PVRSRV_RGXDEV_INFO  *psDevInfo,
+	                            const RGX_HEAP_INFO  pksHeapInfo[],
+	                            IMG_UINT32           ui32HeapListSize,
+	                            IMG_UINT32*          ui32HeapCount)
+{
+	IMG_UINT32 i;
+
+	/* Loop over rows in the heap data array using callback to decide if we
+	 * should include the heap
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo = &pksHeapInfo[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		(*ui32HeapCount)++;
+	}
+}
+/* Generic heap instantiator */
+static void _InstantiateRequiredHeaps(PVRSRV_RGXDEV_INFO     *psDevInfo,
+                                      RGX_HEAP_INFO           psHeapInfos[],
+                                      IMG_UINT32              ui32HeapListSize,
+                                      const IMG_UINT32        ui32Log2RgxDefaultPageShift,
+                                      DEVMEM_HEAP_BLUEPRINT **psDeviceMemoryHeapCursor)
+{
+	IMG_UINT32 i;
+
+#if defined DEBUG
+	IMG_UINT32 ui32heapListCnt;
+	bool bHeapPageSizeMisMatch = false;
+
+	/*
+	 * To ensure all heaps within a 2MB region have the same page sizes
+	 */
+	for (ui32heapListCnt = 0; ui32heapListCnt < (ui32HeapListSize-1); ui32heapListCnt++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo1 = &psHeapInfos[ui32heapListCnt];
+		const RGX_HEAP_INFO *psHeapInfo2 = &psHeapInfos[ui32heapListCnt+1];
+
+		if (((psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) &&
+			((psHeapInfo1->ui64HeapBase + psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) ==
+			((psHeapInfo2->ui64HeapBase) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)))
+		{
+			if (psHeapInfo1->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				if (!(psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG))
+				{
+					bHeapPageSizeMisMatch = true;
+				}
+			}
+			else if (psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				bHeapPageSizeMisMatch = true;
+			}
+		}
+		if (bHeapPageSizeMisMatch)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Two Heap with Different Page Size allocated in the same PD space(2MB)\n"
+			         "Invalid Heaps 1) \"%s\" and 2) \"%s\"",
+			         __func__,
+			         psHeapInfo1->pszName,
+			         psHeapInfo2->pszName));
+		}
+	}
+#endif
+
+	/* We now have a list of the heaps to include and so we should loop over this
+	 * list and instantiate.
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		IMG_UINT32 ui32Log2DataPageSize = 0;
+		RGX_HEAP_INFO *psHeapInfo = &psHeapInfos[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		if (psHeapInfo->pfnDynamicBaseSize != NULL)
+		{
+			psHeapInfo->pfnDynamicBaseSize(psDevInfo->psDeviceNode, psHeapInfo);
+		}
+
+		if (psHeapInfo->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+		{
+			ui32Log2DataPageSize = psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2;
+		}
+		else
+		{
+			ui32Log2DataPageSize = ui32Log2RgxDefaultPageShift;
+		}
+
+		CheckHeapAlignment(psHeapInfo, psDevInfo);
+
+		HeapCfgBlueprintInit(psHeapInfo->pszName,
+		                     psHeapInfo->ui64HeapBase,
+		                     psHeapInfo->uiHeapLength,
+		                     psHeapInfo->uiHeapReservedRegionLength,
+		                     ui32Log2DataPageSize,
+		                     psHeapInfo->ui32Log2ImportAlignment,
+		                     psHeapInfo->pfnInit,
+		                     psHeapInfo->pfnDeInit,
+		                     *psDeviceMemoryHeapCursor);
+
+		(*psDeviceMemoryHeapCursor)++;
+	}
+}
+
+static PVRSRV_ERROR RGXInitHeaps(PVRSRV_RGXDEV_INFO *psDevInfo,
+	                             DEVICE_MEMORY_INFO *psNewMemoryInfo)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor;
+
+	IMG_UINT32 ui32AppHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutApp);
+	IMG_UINT32 ui32FWHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutFW);
+	IMG_UINT32 ui32CountedHeapSize;
+
+	IMG_UINT32 ui32AppHeapCount = 0;
+	IMG_UINT32 ui32FWHeapCount = 0;
+
+	IMG_UINT32 ui32Log2DefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	if (ui32Log2DefaultPageShift == 0)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2DefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2DefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e0;
+		}
+	}
+#endif
+
+	/* Count heaps required for the app heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutApp,
+		                ui32AppHeapListSize,
+		                &ui32AppHeapCount);
+
+	/* Count heaps required for the FW heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutFW,
+		                ui32FWHeapListSize,
+		                &ui32FWHeapCount);
+
+	ui32CountedHeapSize = (ui32AppHeapCount + ui32FWHeapCount + RGX_NUM_DRIVERS_SUPPORTED);
+
+	psNewMemoryInfo->psDeviceMemoryHeap = OSAllocMem(sizeof(DEVMEM_HEAP_BLUEPRINT) * ui32CountedHeapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeap, eError, e0);
+
+	/* Initialise the heaps */
+	psDeviceMemoryHeapCursor = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	/* Instantiate App Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutApp,
+	                          ui32AppHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* Instantiate FW Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutFW,
+	                          ui32FWHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* set the heap count */
+	psNewMemoryInfo->ui32HeapCount = (IMG_UINT32)(psDeviceMemoryHeapCursor - psNewMemoryInfo->psDeviceMemoryHeap);
+
+	/* Check we have allocated the correct # of heaps, minus any VZ heaps as these
+	 * have not been created at this point
+	 */
+	PVR_ASSERT(psNewMemoryInfo->ui32HeapCount == (ui32CountedHeapSize - RGX_NUM_DRIVERS_SUPPORTED));
+
+	/*
+	   In the new heap setup, we initialise 2 configurations:
+		1 - One will be for the firmware only (index 1 in array)
+			a. This primarily has the firmware heap in it.
+			b. It also has additional guest OSID firmware heap(s)
+				- Only if the number of support firmware OSID > 1
+		2 - Others shall be for clients only (index 0 in array)
+			a. This has all the other client heaps in it.
+	 */
+	psNewMemoryInfo->uiNumHeapConfigs = 2;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray = OSAllocMem(sizeof(DEVMEM_HEAP_CONFIG) * psNewMemoryInfo->uiNumHeapConfigs);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeapConfigArray, eError, e1);
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].pszName = "Default Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].uiNumHeaps = psNewMemoryInfo->ui32HeapCount - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].psHeapBlueprintArray = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].pszName = "Firmware Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps = RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].psHeapBlueprintArray = psDeviceMemoryHeapCursor - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4)
+	{
+		IMG_UINT32 i;
+
+		/*
+		 * Initialise all MMU Page Size Range Config register to the default page size
+		 * used by the OS, leaving the address range 0;
+		 */
+		for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[i] = RGXMMUInitRangeValue(i);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		/* Create additional raw firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVINFO, psDevInfo)
+		{
+			eError = RGXInitFwRawHeap(psDevInfo, psDeviceMemoryHeapCursor, ui32DriverID);
+			if (eError != PVRSRV_OK)
+			{
+				/* if any allocation fails, free previously allocated heaps and abandon initialisation */
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+					psDeviceMemoryHeapCursor--;
+				}
+				goto e1;
+			}
+
+			/* Append additional firmware heaps to host driver firmware context heap configuration */
+			psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps += 1;
+
+			/* advance to the next heap */
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	return PVRSRV_OK;
+e1:
+	OSFreeMem(psNewMemoryInfo->psDeviceMemoryHeap);
+e0:
+	return eError;
+}
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+		DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor = psDevMemoryInfo->psDeviceMemoryHeap;
+
+		/* Delete all guest firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeapConfigArray);
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeap);
+}
+
+static PVRSRV_ERROR RGXInitSharedFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psSysHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* VZ heap validation */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg != NULL,
+								"FW Main heap is required for VZ Guest.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+#endif
+
+	if (psSysHeapCfg != NULL)
+	{
+		/* Check FW_SHARED for multiple usage flags. Because FW_SHARED is divided
+		   into subheaps, shared usage with other heaps is not allowed.  */
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_SHARED,
+								"FW_SHARED phys heap config not specified with more than one usage."
+								"FW_SHARED heap must be exclusively used as FW_SHARED.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	if (psSysHeapCfg == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap not set", __func__));
+		/* Nothing to do. Default to the physheap fallback option */
+	}
+	else if (psSysHeapCfg->eType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses OS System memory (UMA)", __func__));
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psSysHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG] = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	}
+	else /* PHYS_HEAP_TYPE_LMA or PHYS_HEAP_TYPE_DMA */
+	{
+		PHYS_HEAP_CONFIG sFwMainHeapCfg, sFwCfgHeapCfg;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses local memory managed by the driver (LMA)", __func__));
+
+
+		/* Subheap layout: Main + (optional MIPS reserved range) + Config */
+		sFwMainHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		/* Reserve space for the Config heap */
+		sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+			/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)))
+			{
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+				if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101;
+				}
+				else
+#endif
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL;
+				}
+			}
+		}
+#endif
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwMainHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwMainHeapCfg,
+		                              "Fw Main subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:MAIN", ErrorDeinit);
+
+		sFwCfgHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		sFwCfgHeapCfg.uConfig.sLMA.sStartAddr.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+		sFwCfgHeapCfg.uConfig.sLMA.sCardBase.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		sFwCfgHeapCfg.uConfig.sLMA.uiSize = RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwCfgHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwCfgHeapCfg,
+		                              "Fw Cfg subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:CFG", ErrorDeinit);
+	}
+
+	/* Acquire FW heaps */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_MAIN, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_MAIN", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CONFIG, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CONFIG", ErrorDeinit);
+
+	return eError;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitPrivateFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psFwCodeHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_CODE);
+	PHYS_HEAP_CONFIG *psFwDataHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIV_DATA);
+	PHYS_HEAP_CONFIG *psFwPrivateHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIVATE);
+	PHYS_HEAP_CONFIG sFwPrivateTempCfg;
+
+	if (psFwPrivateHeapCfg != NULL)
+	{
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg == NULL) && (psFwDataHeapCfg == NULL),
+								"FW_PRIVATE and the FW_CODE & FW_PRIV_DATA usage flags "
+								"achieve the same goal and are mutually exclusive.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Fw code and data are both allocated from this unified heap */
+		sFwPrivateTempCfg = *psFwPrivateHeapCfg;
+		sFwPrivateTempCfg.ui32UsageFlags = PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+
+		psFwCodeHeapCfg = &sFwPrivateTempCfg;
+		psFwDataHeapCfg = &sFwPrivateTempCfg;
+	}
+
+	if ((psFwCodeHeapCfg == NULL) || (psFwDataHeapCfg == NULL))
+	{
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware code and data heaps must be either both defined
+			 * or both undefined. There is no point in isolating one but not
+			 * the other.*/
+			eError = PVRSRV_ERROR_PHYSHEAP_CONFIG;
+			PVR_LOG_GOTO_IF_ERROR(eError, "PrivateFwPhysHeap check", ErrorDeinit);
+		}
+		else
+		{
+			/* No dedicated heaps, default to the physheap fallback option */
+		}
+	}
+	else if (psFwCodeHeapCfg == psFwDataHeapCfg)
+	{
+		if (psFwCodeHeapCfg->ui32UsageFlags ==
+			(PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA))
+		{
+			/* Fw code and private data allocations come from the same system heap
+			 * Instantiate one physheap and share it between them. */
+
+			eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+												  psFwCodeHeapCfg,
+												  NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig");
+		}
+		else
+		{
+			/* Not an exclusive heap, can be used for other purposes (e.g. secure buffers).
+			 * Expect the PVR layer to have already created a heap for the other uses. */
+		}
+	}
+	else
+	{
+		/*
+		 * Separating private Firmware code and data is allowed for backwards compatibility
+		 * purposes. New platforms should use the unified FW_PRIVATE heap instead.
+		 *
+		 * Early security implementations on Rogue cores required separate FW_PRIV_DATA
+		 * and FW_CODE heaps, as access permissions to Firmware were granted differently
+		 * based on the transaction types (code or data).
+		 */
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_CODE) &&
+								(psFwDataHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PRIV_DATA),
+								"Dedicated private heaps for Fw code and "
+								"data must have one usage flag exclusively.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Dedicated Fw code heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwCodeHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		/* Dedicated Fw private data heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwDataHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+	/* When premapping distinct private and shared Firmware phys heaps
+	 * inside the same virtual devmem heap, their sizes must add up to
+	 * the fixed RGX_FIRMWARE_RAW_HEAP_SIZE for the premapping to work */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwSharedHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		PHYS_HEAP_USAGE_FW_SHARED);
+		IMG_UINT64 ui64FwCodeHeapSize = PhysHeapConfigGetSize(psFwCodeHeapCfg);
+		IMG_UINT64 ui64FwDataHeapSize = PhysHeapConfigGetSize(psFwDataHeapCfg);
+		IMG_UINT64 ui64FwSharedHeapSize = PhysHeapConfigGetSize(psFwSharedHeapCfg);
+		IMG_UINT64 ui64FwPrivateHeapSize;
+
+		PVR_LOG_GOTO_IF_FALSE((psFwCodeHeapCfg != NULL) && (psFwDataHeapCfg != NULL),
+							  "Security support requires Fw code and data memory be"
+							  " separate from the heap shared with the kernel driver.", FailDeinit);
+
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware allocations come from 2 different heaps */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize + ui64FwDataHeapSize;
+		}
+		else
+		{
+			/* Private Firmware allocations come from a single heap */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize;
+		}
+
+				PVR_LOG_GOTO_IF_FALSE((ui64FwSharedHeapSize +
+									   ui64FwPrivateHeapSize) ==
+									   RGX_FIRMWARE_RAW_HEAP_SIZE,
+									   "Invalid firmware physical heap size.", FailDeinit);
+	}
+#endif
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CODE, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CODE]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CODE", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_PRIV_DATA, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PRIV_DATA]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_DATA", ErrorDeinit);
+
+	return eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+FailDeinit:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitFwPageTableHeap(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwPageTableHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		  PHYS_HEAP_USAGE_FW_PREMAP_PT);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg != NULL),
+								"The Firmware Page Table phys heap config not found.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PREMAP_PT),
+								"The Firmware Page Table heap must be used exclusively for this purpose",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_LMA) ||
+								(psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_DMA),
+								"The Firmware Page Table heap must be LMA or DMA memory.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->uConfig.sLMA.uiSize >= RGX_FIRMWARE_MAX_PAGETABLE_SIZE),
+								"The Firmware Page Table heap must be able to hold the maximum "
+								"number of pagetables needed to cover the Firmware's VA space.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwPageTableHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig:FwPageTableHeap");
+
+		eError = PhysHeapAcquire(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquire:FwPageTableHeap");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) */
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXInitFwPageTableHeap(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitFwPageTableHeap", ErrorDeinit);
+	eError = RGXInitSharedFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitSharedFwPhysHeaps", ErrorDeinit);
+	eError = RGXInitPrivateFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitPrivateFwPhysHeaps", ErrorDeinit);
+
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDeviceFWMainHeapMemCheck
+@Description    Checks the free memory in FW Main PhysHeap of a device to ensure
+                there is enough for a connection to be made.
+
+@Input          psDeviceNode    The device of the FW Main PhysHeap to be checked.
+
+@Return         On success PVRSRV_OK, else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDeviceFWMainHeapMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP *psFWMainPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psFWMainPhysHeap = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	if (psFWMainPhysHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's FW Main PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psFWMainPhysHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		const IMG_UINT32 ui32MinMemInKBs = RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION;
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psFWMainPhysHeap,
+		                              KB2B(ui32MinMemInKBs),
+		                              &ui64FreePhysHeapMem);
+
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FW_MAIN PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"KB "
+				"Minimum required: %uKB",
+				B2KB(ui64FreePhysHeapMem),
+				ui32MinMemInKBs));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate, IMG_UINT32 *pui32Log2Non4KPgShift)
+{
+	IMG_UINT32 uiLog2OSPageShift = OSGetPageShift();
+
+	/* We support Non4K pages only on platforms with 4KB pages. On all platforms
+	 * where OS pages are larger than 4KB we must ensure the non4K device memory
+	 * heap matches the page size used in all other device memory heaps, which
+	 * is the OS page size, see RGXHeapDerivePageSize. */
+	if (uiLog2OSPageShift > RGX_HEAP_4KB_PAGE_SHIFT)
+	{
+		*pui32Log2Non4KPgShift = RGXHeapDerivePageSize(uiLog2OSPageShift);
+	}
+	else
+	{
+		IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE;
+		IMG_UINT32 ui32GeneralNon4KHeapPageSize;
+		IMG_UINT32 ui32GeneralNon4KValidHeapPageSize;
+		IMG_UINT32 ui32Log2Non4KPgShift;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		/* The TEE-DDK is not aware of REE-DDK apphints. Stick to default */
+		ui32GeneralNon4KHeapPageSize = ui32AppHintDefault;
+#else
+		void *pvAppHintState = NULL;
+		/* Get the page size for the dummy page from the NON4K heap apphint */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState,
+		                     GeneralNon4KHeapPageSize, &ui32AppHintDefault,
+		                     &ui32GeneralNon4KHeapPageSize);
+		OSFreeAppHintState(pvAppHintState);
+#endif
+		/* Validate that the given ui32GeneralNon4KHeapPageSize is one of the
+		 * supported range of values.
+		 */
+		ui32Log2Non4KPgShift = RGXHeapDerivePageSize(ExactLog2(ui32GeneralNon4KHeapPageSize));
+		ui32GeneralNon4KValidHeapPageSize = (1U << ui32Log2Non4KPgShift);
+		if (ui32GeneralNon4KValidHeapPageSize != ui32GeneralNon4KHeapPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Non4K Heap Size: Requested"
+			         " Heap size=%u, actual=%u, pageshift 0x%x",
+			         __func__, ui32GeneralNon4KHeapPageSize,
+			         ui32GeneralNon4KValidHeapPageSize, ui32Log2Non4KPgShift));
+		}
+
+		if (ui32Log2Non4KPgShift == 0U)
+		{
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+
+		*pui32Log2Non4KPgShift = ui32Log2Non4KPgShift;
+	}
+
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 71317))
+	{
+		if (*pui32Log2Non4KPgShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || *pui32Log2Non4KPgShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Page sizes of 2MB or 1MB cause page faults."));
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+#endif
+
+	/* Check the Non4k page size is at least the size of the OS page size
+	 * or larger. The Non4k page size also has to be a multiple of the OS page
+	 * size but since we have the log2 value from the apphint we know powers of 2
+	 * will always be multiples. If the Non4k page size is less than OS page size
+	 * we notify and upgrade the size.
+	 */
+	if (*pui32Log2Non4KPgShift < uiLog2OSPageShift)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "Non4K page size smaller than OS page size, upgrading to "
+		                          "match OS page size."));
+		*pui32Log2Non4KPgShift = uiLog2OSPageShift;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* RGXRegisterDevice
+ *
+ * WARNING!
+ *
+ * No PDUMP statements are allowed until device initialisation starts.
+ */
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	DEVICE_MEMORY_INFO *psDevMemoryInfo;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault = HWPERF_HOST_TL_STREAM_SIZE_DEFAULT, ui32HWPerfHostBufSizeKB;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault, &ui32HWPerfHostBufSizeKB);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/*********************
+	 * Device node setup *
+	 *********************/
+	/* Setup static data and callbacks on the device agnostic device node */
+#if defined(PDUMP)
+	psDeviceNode->sDevId.pszPDumpRegName	= RGX_PDUMPREG_NAME;
+	psDeviceNode->sDevId.pszPDumpDevName	= PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+	psDeviceNode->pfnPDumpInitDevice = &RGXResetPDump;
+	psDeviceNode->ui64FBCClearColour = RGX_FBC_CC_DEFAULT;
+
+#endif /* PDUMP */
+
+	OSAtomicWrite(&psDeviceNode->eHealthStatus, PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	OSAtomicWrite(&psDeviceNode->eHealthReason, PVRSRV_DEVICE_HEALTH_REASON_NONE);
+
+	psDeviceNode->pfnInvalFBSCTable = RGXInvalidateFBSCTable;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = NULL;
+
+	/* Callback for getting the MMU device attributes */
+	psDeviceNode->pfnGetMMUDeviceAttributes = RGXDevMMUAttributes;
+	psDeviceNode->pfnGetDeviceSnoopMode = RGXDevSnoopMode;
+	psDeviceNode->pfnMMUCacheInvalidate = RGXMMUCacheInvalidate;
+	psDeviceNode->pfnMMUCacheInvalidateKick = RGXMMUCacheInvalidateKick;
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = RGXMapBRN71422TargetPhysicalAddress;
+#else
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = NULL;
+#endif
+	/* pfnMMUTweakProtFlags is set later on once BVNC features are setup */
+	psDeviceNode->pfnMMUTweakProtFlags = NULL;
+
+	psDeviceNode->pfnInitDeviceCompatCheck	= &RGXDevInitCompatCheck;
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	/* Register callbacks for Unified Fence Objects */
+	psDeviceNode->pfnAllocUFOBlock = RGXAllocUFOBlock;
+	psDeviceNode->pfnFreeUFOBlock = RGXFreeUFOBlock;
+
+	/* Register callback for checking the device's health */
+	psDeviceNode->pfnUpdateHealthStatus = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? NULL : RGXUpdateHealthStatus;
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Register callback for updating the virtualization watchdog */
+	psDeviceNode->pfnUpdateAutoVzWatchdog = RGXUpdateAutoVzWatchdog;
+#endif
+
+	/* Register method to service the FW HWPerf buffer */
+	psDeviceNode->pfnServiceHWPerf = RGXHWPerfDataStoreCB;
+
+	/* Register callback for getting the device version information string */
+	psDeviceNode->pfnDeviceVersionString = RGXDevVersionString;
+
+	/* Register callback for getting the device clock speed */
+	psDeviceNode->pfnDeviceClockSpeed = RGXDevClockSpeed;
+
+	/* Register callback for soft resetting some device modules */
+	psDeviceNode->pfnSoftReset = RGXSoftReset;
+
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnResetHWRLogs = RGXResetHWRLogs;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnVerifyBVNC = RGXVerifyBVNC;
+
+	/* Register callback for checking alignment of UM structures */
+	psDeviceNode->pfnAlignmentCheck = RGXAlignmentCheck;
+
+	/*Register callback for checking the supported features and getting the
+	 * corresponding values */
+	psDeviceNode->pfnCheckDeviceFeature = RGXBvncCheckFeatureSupported;
+	psDeviceNode->pfnGetDeviceFeatureValue = RGXBvncGetSupportedFeatureValue;
+
+	/* Callback for checking if system layer supports FBC 3.1 */
+	psDeviceNode->pfnHasFBCDCVersion31 = RGXSystemHasFBCDCVersion31;
+
+	/* Callback for getting TFBC configuration */
+	psDeviceNode->pfnGetTFBCLossyGroup = RGXGetTFBCLossyGroup;
+
+	/* Register callback for initialising device-specific physical memory heaps */
+	psDeviceNode->pfnPhysMemDeviceHeapsInit = RGXPhysMemDeviceHeapsInit;
+
+	/* Register callback for checking a device's FW Main physical heap for sufficient free memory */
+	psDeviceNode->pfnCheckForSufficientFWPhysMem = RGXDeviceFWMainHeapMemCheck;
+
+	/*********************
+	 * Device info setup *
+	 *********************/
+	/* Allocate device control block */
+	psDevInfo = OSAllocZMem(sizeof(*psDevInfo));
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevInitRGXPart1 : Failed to alloc memory for DevInfo"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* initialise the layer parameters needed for early hw feature checks */
+	psDevInfo->sLayerParams.psDevInfo = psDevInfo;
+	psDevInfo->sLayerParams.psDevConfig = psDeviceNode->psDevConfig;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Default psTrampoline to point to null struct */
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+#endif
+
+	/* create locks for the context lists stored in the DevInfo structure.
+	 * these lists are modified on context create/destroy and read by the
+	 * watchdog thread
+	 */
+
+	eError = OSWRLockCreate(&(psDevInfo->hRenderCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create render context list lock", __func__));
+		goto e0;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hComputeCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create compute context list lock", __func__));
+		goto e1;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTransferCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create transfer context list lock", __func__));
+		goto e2;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTDMCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create TDM context list lock", __func__));
+		goto e3;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hKickSyncCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create kick sync context list lock", __func__));
+		goto e4;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hMemoryCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create memory context list lock", __func__));
+		goto e5;
+	}
+
+	eError = OSSpinLockCreate(&psDevInfo->hLockKCCBDeferredCommandsList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to KCCB deferred commands list lock", __func__));
+		goto e6;
+	}
+	dllist_init(&(psDevInfo->sKCCBDeferredCommandsListHead));
+
+	dllist_init(&(psDevInfo->sRenderCtxtListHead));
+	dllist_init(&(psDevInfo->sComputeCtxtListHead));
+	dllist_init(&(psDevInfo->sTDMCtxtListHead));
+	dllist_init(&(psDevInfo->sKickSyncCtxtListHead));
+
+	dllist_init(&(psDevInfo->sCommonCtxtListHead));
+	psDevInfo->ui32CommonCtxtCurrentID = 1;
+
+
+	eError = OSWRLockCreate(&psDevInfo->hCommonCtxtListLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create common context list lock", __func__));
+		goto e7;
+	}
+
+	eError = OSLockCreate(&psDevInfo->sRegConfig.hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create register configuration lock", __func__));
+		goto e8;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hBPLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for break points", __func__));
+		goto e9;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hRGXFWIfBufInitLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for trace buffers", __func__));
+		goto e10;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hCCBStallCheckLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB checking lock", __func__));
+		goto e11;
+	}
+	eError = OSLockCreate(&psDevInfo->hCCBRecoveryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB recovery lock", __func__));
+		goto e12;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hGpuUtilStatsLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create GPU stats lock", __func__));
+		goto e13;
+	}
+
+	dllist_init(&psDevInfo->sMemoryContextList);
+
+	/* initialise ui32SLRHoldoffCounter */
+	if (RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS > DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT)
+	{
+		psDevInfo->ui32SLRHoldoffCounter = RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS / DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	}
+	else
+	{
+		psDevInfo->ui32SLRHoldoffCounter = 0;
+	}
+
+	/* Setup static data and callbacks on the device specific device info */
+	psDevInfo->psDeviceNode		= psDeviceNode;
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+	psDevInfo->pvDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
+
+	/*
+	 * Map RGX Registers
+	 */
+	psDevInfo->ui32RegSize = psDeviceNode->psDevConfig->ui32RegsSize;
+	psDevInfo->sRegsPhysBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+#if !defined(NO_HARDWARE)
+	psDevInfo->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create RGX register mapping",
+		         __func__));
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto e14;
+	}
+#endif /* !NO_HARDWARE */
+
+	psDeviceNode->pvDevice = psDevInfo;
+
+	eError = RGXBvncInitialiseConfiguration(psDeviceNode);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Unsupported HW device detected by driver",
+		         __func__));
+		goto e15;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/*
+	 * We must now setup the SECURITY mappings if supported. We cannot
+	 * check on the features until we have reached here as the BVNC is
+	 * not setup before now.
+	 */
+#if defined(NO_HARDWARE) || defined(SUPPORT_TRUSTED_DEVICE)
+	/* secure register bank not accessible or unused */
+	psDevInfo->pvSecureRegsBaseKM = 0;
+#else
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+	{
+		IMG_CPU_PHYADDR sHostSecureRegBankBase = {psDeviceNode->psDevConfig->sRegsCpuPBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET};
+
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *) OSMapPhysToLin(sHostSecureRegBankBase,
+																		RGX_HOST_SECURE_REGBANK_SIZE,
+																		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (psDevInfo->pvSecureRegsBaseKM == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "PVRSRVRGXInitDevPart2KM: Failed to create RGX secure register mapping"));
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			goto e15;
+		}
+
+		/*
+		 * The secure register bank is mapped into the CPU VA space starting from
+		 * the base of the normal register bank + an offset of RGX_HOST_SECURE_REGBAK_OFFSET.
+		 * The hardware register addresses are all indexed from the base of the regular register bank.
+		 * For the RegBankBase+RegOffset computation to still be accurate for host-secure registers,
+		 * we need to compensate for offsets of registers in the secure bank
+		 */
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM - RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+	else
+	{
+		psDevInfo->pvSecureRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	}
+#endif /* defined(NO_HARDWARE) || defined(SUPPORT_TRUSTED_DEVICE) */
+#endif /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	psDeviceNode->pfnMMUTweakProtFlags = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			RGXMMUTweakProtFlags : NULL;
+#endif
+
+	eError = RGXGetNon4KHeapPageShift(&psDevInfo->sLayerParams,
+									 &psDeviceNode->ui32Non4KPageSizeLog2);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXGetNon4KHeapPageSize", e16);
+
+	/* Configure MMU specific stuff */
+	RGXMMUInit_Register(psDeviceNode);
+
+	eError = RGXInitHeaps(psDevInfo, psDevMemoryInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto e16;
+	}
+
+	eError = RGXHWPerfInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInit", e16);
+
+	eError = RGXHWPerfHostInit(psDeviceNode->pvDevice, ui32HWPerfHostBufSizeKB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfHostInit", ErrorDeInitHWPerfFw);
+
+
+	/* Register callback for dumping debug info */
+	eError = RGXDebugInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDebugInit", e17);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Register callback for fw mmu init */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDeviceNode->pfnFwMMUInit = RGXMipsMMUInit_Register;
+	}
+#endif
+
+	/* Initialise the device dependent bridges */
+	eError = RGXRegisterBridges(psDevInfo);
+	PVR_LOG_IF_ERROR(eError, "RGXRegisterBridges");
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = OSLockCreate(&psDevInfo->hCounterDumpingLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for counter sampling.", __func__));
+		goto ErrorDeInitDeviceDepBridge;
+	}
+#endif
+
+	/* Initialise error counters */
+	memset(&psDevInfo->sErrorCounts, 0, sizeof(PVRSRV_RGXDEV_ERROR_COUNTS));
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+ErrorDeInitDeviceDepBridge:
+	RGXUnregisterBridges(psDevInfo);
+#endif
+
+e17:
+	RGXHWPerfHostDeInit(psDevInfo);
+ErrorDeInitHWPerfFw:
+	RGXHWPerfDeinit(psDevInfo);
+e16:
+#if !defined(NO_HARDWARE)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+		/* Adjust pvSecureRegsBaseKM if device has SECURITY_VERSION > 1 */
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		    (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* Undo the VA offset adjustment to unmap correct VAddr */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM,
+			                 RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+	}
+#endif
+#endif /* !NO_HARDWARE */
+e15:
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+		                 psDevInfo->ui32RegSize);
+	}
+e14:
+#endif /* !NO_HARDWARE */
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+e13:
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+e12:
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+e11:
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+e10:
+	OSLockDestroy(psDevInfo->hBPLock);
+e9:
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+e8:
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+e7:
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+e6:
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+e5:
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+e4:
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+e3:
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+e2:
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+e1:
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+e0:
+	OSFreeMem(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_PCHAR psz = psDevInfo->sDevFeatureCfg.pszBVNCString;
+	if (NULL == psz)
+	{
+		IMG_CHAR pszBVNCInfo[RGX_HWPERF_MAX_BVNC_LEN];
+		size_t uiBVNCStringSize;
+		size_t uiStringLength;
+
+		uiStringLength = OSSNPrintf(pszBVNCInfo, RGX_HWPERF_MAX_BVNC_LEN, "%d.%d.%d.%d",
+				psDevInfo->sDevFeatureCfg.ui32B,
+				psDevInfo->sDevFeatureCfg.ui32V,
+				psDevInfo->sDevFeatureCfg.ui32N,
+				psDevInfo->sDevFeatureCfg.ui32C);
+		PVR_ASSERT(uiStringLength < RGX_HWPERF_MAX_BVNC_LEN);
+
+		uiBVNCStringSize = (uiStringLength + 1) * sizeof(IMG_CHAR);
+		psz = OSAllocMem(uiBVNCStringSize);
+		if (NULL != psz)
+		{
+			OSCachedMemCopy(psz, pszBVNCInfo, uiBVNCStringSize);
+			psDevInfo->sDevFeatureCfg.pszBVNCString = psz;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Allocating memory for BVNC Info string failed",
+					__func__));
+		}
+	}
+
+	return psz;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDevVersionString
+@Description    Gets the version string for the given device node and returns
+                a pointer to it in ppszVersionString. It is then the
+                responsibility of the caller to free this memory.
+@Input          psDeviceNode            Device node from which to obtain the
+                                        version string
+@Output	        ppszVersionString	Contains the version string upon return
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR **ppszVersionString)
+{
+#if defined(NO_HARDWARE) || defined(EMULATOR)
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (SW)";
+#else
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (HW)";
+#endif
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_PCHAR pszBVNC;
+	size_t uiStringLength;
+
+	if (psDeviceNode == NULL || ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	pszBVNC = RGXDevBVNCString(psDevInfo);
+
+	if (NULL == pszBVNC)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	uiStringLength = OSStringLength(pszBVNC);
+	uiStringLength += (sizeof(szFormatString) - 2); /* sizeof includes the null, -2 for "%s" */
+	*ppszVersionString = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (*ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSSNPrintf(*ppszVersionString, uiStringLength, szFormatString,
+		pszBVNC);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       RGXDevClockSpeed
+@Description    Gets the clock speed for the given device node and returns
+                it in pui32RGXClockSpeed.
+@Input          psDeviceNode		Device node
+@Output         pui32RGXClockSpeed  Variable for storing the clock speed
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_PUINT32  pui32RGXClockSpeed)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* get clock speed */
+	*pui32RGXClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/*!
+ *******************************************************************************
+
+ @Function		RGXInitFwRawHeap
+
+ @Description	Called to perform additional initialisation
+ ******************************************************************************/
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID)
+{
+	IMG_UINT32 uiStringLength;
+	IMG_UINT32 uiStringLengthMax = 32;
+
+	IMG_UINT32 ui32Log2RgxDefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	PVR_RETURN_IF_FALSE(ui32Log2RgxDefaultPageShift != 0, PVRSRV_ERROR_INVALID_PARAMS);
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2RgxDefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2RgxDefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	uiStringLength = MIN(sizeof(RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT), uiStringLengthMax + 1);
+
+	/* Start by allocating memory for this DriverID heap identification string */
+	psDevMemHeap->pszName = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (psDevMemHeap->pszName == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Append the DriverID number to the RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT string */
+	OSSNPrintf((IMG_CHAR *)psDevMemHeap->pszName, uiStringLength, RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	/* Use the common blueprint template support function to initialise the heap */
+	HeapCfgBlueprintInit(psDevMemHeap->pszName,
+		                 RGX_FIRMWARE_RAW_HEAP_BASE + (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE),
+		                 RGX_FIRMWARE_RAW_HEAP_SIZE,
+		                 0,
+		                 ui32Log2RgxDefaultPageShift,
+		                 0,
+		                 NULL,
+		                 NULL,
+		                 psDevMemHeap);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function		RGXDeInitFwRawHeap
+
+ @Description	Called to perform additional deinitialisation
+ ******************************************************************************/
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap)
+{
+	IMG_UINT64 uiBase = RGX_FIRMWARE_RAW_HEAP_BASE + RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT64 uiSpan = uiBase + ((RGX_NUM_DRIVERS_SUPPORTED - 1) * RGX_FIRMWARE_RAW_HEAP_SIZE);
+
+	/* Safe to do as the guest firmware heaps are last in the list */
+	if (psDevMemHeap->sHeapBaseAddr.uiAddr >= uiBase &&
+	    psDevMemHeap->sHeapBaseAddr.uiAddr < uiSpan)
+	{
+		void *pszName = (void*)psDevMemHeap->pszName;
+		OSFreeMem(pszName);
+	}
+}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/******************************************************************************
+ End of file (rgxinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxlayer_impl.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxlayer_impl.c
new file mode 100644
index 000000000000..f95f500dd5b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxlayer_impl.c
@@ -0,0 +1,639 @@
+/*************************************************************************/ /*!
+@File
+@Title          DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxinit.h"
+#include "rgxfwimageutils.h"
+#include "cache_km.h"
+#include "km/rgxdefs_km.h"
+#include "rgx_heaps_server.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+
+#define MAX_NUM_COHERENCY_TESTS  (10)
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	if (psDevInfo->ui32CoherencyTestsDone >= MAX_NUM_COHERENCY_TESTS)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_FALSE;
+}
+
+/*
+ * The fabric coherency test is performed when platform supports fabric coherency
+ * either in the form of ACE-lite or Full-ACE. This test is done quite early
+ * with the firmware processor quiescent and makes exclusive use of the slave
+ * port interface for reading/writing through the device memory hierarchy. The
+ * rationale for the test is to ensure that what the CPU writes to its dcache
+ * is visible to the GPU via coherency snoop miss/hit and vice-versa without
+ * any intervening cache maintenance by the writing agent.
+ */
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 *pui32FabricCohTestBufferCpuVA = NULL;
+	IMG_UINT32 *pui32FabricCohCcTestBufferCpuVA = NULL;
+	IMG_UINT32 *pui32FabricCohNcTestBufferCpuVA = NULL;
+	DEVMEM_MEMDESC *psFabricCohTestBufferMemDesc = NULL;
+	DEVMEM_MEMDESC *psFabricCohCcTestBufferMemDesc = NULL;
+	DEVMEM_MEMDESC *psFabricCohNcTestBufferMemDesc = NULL;
+	RGXFWIF_DEV_VIRTADDR sFabricCohCcTestBufferDevVA;
+	RGXFWIF_DEV_VIRTADDR sFabricCohNcTestBufferDevVA;
+	RGXFWIF_DEV_VIRTADDR *psFabricCohTestBufferDevVA = NULL;
+	IMG_DEVMEM_SIZE_T uiFabricCohTestBlockSize = sizeof(IMG_UINT64);
+	IMG_DEVMEM_ALIGN_T uiFabricCohTestBlockAlign = sizeof(IMG_UINT64);
+	IMG_UINT64 ui64SegOutAddrTopCached = 0;
+	IMG_UINT64 ui64SegOutAddrTopUncached = 0;
+	IMG_UINT32 ui32OddEven;
+	IMG_UINT32 ui32OddEvenSeed = 1;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bFullTestPassed = IMG_TRUE;
+	IMG_BOOL bExit = IMG_FALSE;
+#if defined(DEBUG)
+	IMG_BOOL bSubTestPassed = IMG_FALSE;
+#endif
+	enum TEST_TYPE {
+		CPU_WRITE_GPU_READ_SM=0, GPU_WRITE_CPU_READ_SM,
+		CPU_WRITE_GPU_READ_SH,   GPU_WRITE_CPU_READ_SH
+	} eTestType;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	PVR_LOG(("Starting fabric coherency test ....."));
+
+	/* Size and align are 'expanded' because we request an export align allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiFabricCohTestBlockSize,
+	                                             &uiFabricCohTestBlockAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemExportalignAdjustSizeAndAlign() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	/* Allocate, acquire cpu address and set firmware address for cc=1 buffer */
+	eError = DevmemFwAllocateExportable(psDevInfo->psDeviceNode,
+										uiFabricCohTestBlockSize,
+										uiFabricCohTestBlockAlign,
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT |
+										PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwExFabricCoherencyCcTestBuffer",
+										&psFabricCohCcTestBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemFwAllocateExportable() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psFabricCohCcTestBufferMemDesc, (void **) &pui32FabricCohCcTestBufferCpuVA);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemAcquireCpuVirtAddr() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e1;
+	}
+
+	/* Create a FW address which is uncached in the Meta DCache and in the SLC using the Meta bootloader segment.
+	   This segment is the only one configured correctly out of reset (when this test is meant to be executed) */
+	eError = RGXSetFirmwareAddress(&sFabricCohCcTestBufferDevVA,
+						  psFabricCohCcTestBufferMemDesc,
+						  0,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", e2);
+
+	/* Undo most of the FW mappings done by RGXSetFirmwareAddress */
+	sFabricCohCcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_META_CACHE_MASK;
+	sFabricCohCcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK;
+	sFabricCohCcTestBufferDevVA.ui32Addr -= RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+	/* Map the buffer in the bootloader segment as uncached */
+	sFabricCohCcTestBufferDevVA.ui32Addr |= RGXFW_BOOTLDR_META_ADDR;
+	sFabricCohCcTestBufferDevVA.ui32Addr |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+
+	/* Allocate, acquire cpu address and set firmware address for cc=0 buffer */
+	eError = DevmemFwAllocateExportable(psDevInfo->psDeviceNode,
+										uiFabricCohTestBlockSize,
+										uiFabricCohTestBlockAlign,
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwExFabricCoherencyNcTestBuffer",
+										&psFabricCohNcTestBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemFwAllocateExportable() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e3;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psFabricCohNcTestBufferMemDesc, (void **) &pui32FabricCohNcTestBufferCpuVA);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemAcquireCpuVirtAddr() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e4;
+	}
+
+	eError = RGXSetFirmwareAddress(&sFabricCohNcTestBufferDevVA,
+						  psFabricCohNcTestBufferMemDesc,
+						  0,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", e5);
+
+	/* Undo most of the FW mappings done by RGXSetFirmwareAddress */
+	sFabricCohNcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_META_CACHE_MASK;
+	sFabricCohNcTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK;
+	sFabricCohNcTestBufferDevVA.ui32Addr -= RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+	/* Map the buffer in the bootloader segment as uncached */
+	sFabricCohNcTestBufferDevVA.ui32Addr |= RGXFW_BOOTLDR_META_ADDR;
+	sFabricCohNcTestBufferDevVA.ui32Addr |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+
+	/* Obtain the META segment addresses corresponding to cached and uncached windows into SLC */
+	ui64SegOutAddrTopCached   = RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(MMU_CONTEXT_MAPPING_FWIF);
+	ui64SegOutAddrTopUncached = RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED(MMU_CONTEXT_MAPPING_FWIF);
+
+	/* At the top level, we perform snoop-miss (i.e. to verify slave port) & snoop-hit (i.e. to verify ACE) test.
+	   NOTE: For now, skip snoop-miss test as Services currently forces all firmware allocations to be coherent */
+	for (eTestType = CPU_WRITE_GPU_READ_SH; eTestType <= GPU_WRITE_CPU_READ_SH && bExit == IMG_FALSE; eTestType++)
+	{
+		IMG_CPU_PHYADDR sCpuPhyAddr;
+		IMG_BOOL bValid;
+		PMR *psPMR;
+
+		if (eTestType == CPU_WRITE_GPU_READ_SM)
+		{
+			/* All snoop miss test must bypass the SLC, here memory is region of coherence so
+			   configure META to use SLC bypass cache policy for the bootloader segment. Note
+			   this cannot be done on a cache-coherent (i.e. CC=1) VA, as this violates ACE
+			   standard as one cannot issue a non-coherent request into the bus fabric for
+			   an allocation's VA that is cache-coherent in SLC, so use non-coherent buffer */
+			RGXWriteMetaRegThroughSP(hPrivate, META_CR_MMCU_SEGMENTn_OUTA1(6),
+									(ui64SegOutAddrTopUncached |  RGXFW_BOOTLDR_DEVV_ADDR) >> 32);
+			pui32FabricCohTestBufferCpuVA = pui32FabricCohNcTestBufferCpuVA;
+			psFabricCohTestBufferMemDesc = psFabricCohNcTestBufferMemDesc;
+			psFabricCohTestBufferDevVA = &sFabricCohNcTestBufferDevVA;
+		}
+		else if (eTestType == CPU_WRITE_GPU_READ_SH)
+		{
+			/* All snoop hit test must obviously use SLC, here SLC is region of coherence so
+			   configure META not to bypass the SLC for the bootloader segment */
+			RGXWriteMetaRegThroughSP(hPrivate, META_CR_MMCU_SEGMENTn_OUTA1(6),
+									(ui64SegOutAddrTopCached |  RGXFW_BOOTLDR_DEVV_ADDR) >> 32);
+			pui32FabricCohTestBufferCpuVA = pui32FabricCohCcTestBufferCpuVA;
+			psFabricCohTestBufferMemDesc = psFabricCohCcTestBufferMemDesc;
+			psFabricCohTestBufferDevVA = &sFabricCohCcTestBufferDevVA;
+		}
+
+		if (eTestType == GPU_WRITE_CPU_READ_SH)
+		{
+			/* Cannot perform this test if there is no snooping of device cache */
+			continue;
+		}
+
+		/* Acquire underlying PMR CpuPA in preparation for cache maintenance */
+		(void) DevmemLocalGetImportHandle(psFabricCohTestBufferMemDesc, (void**)&psPMR);
+		eError = PMR_CpuPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sCpuPhyAddr, &bValid, CPU_USE);
+		if (eError != PVRSRV_OK || bValid == IMG_FALSE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"PMR_CpuPhysAddr error: %s, exiting",
+					PVRSRVGetErrorString(eError)));
+			bExit = IMG_TRUE;
+			continue;
+		}
+
+		/* Here we do two passes mostly to account for the effects of using a different
+		   seed (i.e. ui32OddEvenSeed) value to read and write */
+		for (ui32OddEven = 1; ui32OddEven < 3 && bExit == IMG_FALSE; ui32OddEven++)
+		{
+			IMG_UINT32 i;
+
+#if defined(DEBUG)
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+
+			/* Do multiple sub-dword cache line tests */
+			for (i = 0; i < 2 && bExit == IMG_FALSE; i++)
+			{
+				IMG_UINT32 ui32FWAddr;
+				IMG_UINT32 ui32FWValue;
+				IMG_UINT32 ui32FWValue2;
+				IMG_UINT32 ui32LastFWValue = ~0;
+				IMG_UINT32 ui32Offset = i * sizeof(IMG_UINT32);
+
+				/* Calculate next address and seed value to write/read from slave-port */
+				ui32FWAddr = psFabricCohTestBufferDevVA->ui32Addr + ui32Offset;
+				ui32OddEvenSeed += 1;
+
+				if (eTestType == GPU_WRITE_CPU_READ_SM || eTestType == GPU_WRITE_CPU_READ_SH)
+				{
+					/* Clean dcache to ensure there is no stale data in dcache that might over-write
+					   what we are about to write via slave-port here because if it drains from the CPU
+					   dcache before we read it, it would corrupt what we are going to read back via
+					   the CPU */
+					CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_FLUSH);
+
+					/* Calculate a new value to write */
+					ui32FWValue = i + ui32OddEvenSeed;
+
+					/* Write the value using the RGX slave-port interface */
+					eError = RGXWriteFWModuleAddr(psDevInfo, ui32FWAddr, ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXWriteFWModuleAddr error: %s, exiting",
+						          PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Read back value using RGX slave-port interface, this is used
+					   as a sort of memory barrier for the above write */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue2);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXReadFWModuleAddr error: %s, exiting",
+						         PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+					else if (ui32FWValue != ui32FWValue2)
+					{
+						//IMG_UINT32 ui32FWValue3;
+						//RGXReadFWModuleAddr(psDevInfo, 0xC1F00000, &ui32FWValue3);
+
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW via SlavePort failed: expected: %x, got: %x",
+								i,
+								ui32FWValue,
+								ui32FWValue2));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Invalidate dcache to ensure that any prefetched data by the CPU from this memory
+					   region is discarded before we read (i.e. next read must trigger a cache miss).
+					   Previously there was snooping of device cache, where prefetching done by the CPU
+					   would reflect the most up to date datum writing by GPU into said location,
+					   that is to say prefetching was coherent so CPU d-flush was not needed */
+					CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_INVALIDATE);
+				}
+				else
+				{
+					IMG_UINT32 ui32RAWCpuValue;
+
+					/* Ensures line is in dcache */
+					ui32FWValue = IMG_UINT32_MAX;
+
+					/* Dirty allocation in dcache */
+					ui32RAWCpuValue = i + ui32OddEvenSeed;
+					pui32FabricCohTestBufferCpuVA[i] = i + ui32OddEvenSeed;
+
+					/* Flush possible cpu store-buffer(ing) on LMA */
+					OSWriteMemoryBarrier(&pui32FabricCohTestBufferCpuVA[i]);
+
+					if (eTestType == CPU_WRITE_GPU_READ_SM)
+					{
+						/* Flush dcache to force subsequent incoming CPU-bound snoop to miss so
+						   memory is coherent before the SlavePort reads */
+						CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_FLUSH);
+					}
+
+					/* Read back value using RGX slave-port interface */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"RGXReadFWModuleAddr error: %s, exiting",
+								PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Being mostly paranoid here, verify that CPU RAW operation is valid
+					   after the above slave port read */
+					CacheOpValExec(psPMR, 0, ui32Offset, sizeof(IMG_UINT32), PVRSRV_CACHE_OP_INVALIDATE);
+					if (pui32FabricCohTestBufferCpuVA[i] != ui32RAWCpuValue)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW by CPU failed: expected: %x, got: %x",
+								i,
+								ui32RAWCpuValue,
+								pui32FabricCohTestBufferCpuVA[i]));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				/* Compare to see if sub-test passed */
+				if (pui32FabricCohTestBufferCpuVA[i] == ui32FWValue)
+				{
+#if defined(DEBUG)
+					bSubTestPassed = IMG_TRUE;
+#endif
+				}
+				else
+				{
+					bFullTestPassed = IMG_FALSE;
+					eError = PVRSRV_ERROR_INIT_FAILURE;
+#if defined(DEBUG)
+					bSubTestPassed = IMG_FALSE;
+#endif
+					if (ui32LastFWValue != ui32FWValue)
+					{
+#if defined(DEBUG)
+						PVR_LOG(("At Offset: %d, Expected: %x, Got: %x",
+								 i,
+								 (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i],
+								 (eTestType & 0x1) ? pui32FabricCohTestBufferCpuVA[i] : ui32FWValue));
+#endif
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"test encountered unexpected error, exiting"));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				ui32LastFWValue = (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i];
+			}
+
+#if defined(DEBUG)
+			if (bExit)
+			{
+				continue;
+			}
+
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+		}
+	}
+
+	/* Release and free NC/CC test buffers */
+	RGXUnsetFirmwareAddress(psFabricCohCcTestBufferMemDesc);
+e5:
+	DevmemReleaseCpuVirtAddr(psFabricCohCcTestBufferMemDesc);
+e4:
+	DevmemFwUnmapAndFree(psDevInfo, psFabricCohCcTestBufferMemDesc);
+
+e3:
+	RGXUnsetFirmwareAddress(psFabricCohNcTestBufferMemDesc);
+e2:
+	DevmemReleaseCpuVirtAddr(psFabricCohNcTestBufferMemDesc);
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, psFabricCohNcTestBufferMemDesc);
+
+e0:
+	/* Restore bootloader segment settings */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_MMCU_SEGMENTn_OUTA1(6),
+	                         (ui64SegOutAddrTopCached | RGXFW_BOOTLDR_DEVV_ADDR) >> 32);
+
+	bFullTestPassed = bExit ? IMG_FALSE: bFullTestPassed;
+	if (bFullTestPassed)
+	{
+		PVR_LOG(("fabric coherency test: PASSED"));
+		psDevInfo->ui32CoherencyTestsDone = MAX_NUM_COHERENCY_TESTS + 1;
+	}
+	else
+	{
+		PVR_LOG(("fabric coherency test: FAILED"));
+		psDevInfo->ui32CoherencyTestsDone++;
+	}
+
+	return eError;
+}
+
+static IMG_UINT64 RGXMMUComputeRangeValue(IMG_UINT32 ui32DataPageShift, IMG_UINT64 ui64BaseAddress, IMG_UINT64 ui64RangeSize)
+{
+	/* end address of range is inclusive */
+	IMG_UINT64 ui64EndAddress = ui64BaseAddress + ui64RangeSize - (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT);
+	IMG_UINT64 ui64RegValue = 0;
+
+	switch (ui32DataPageShift)
+	{
+		case RGX_HEAP_16KB_PAGE_SHIFT:
+			ui64RegValue = 1;
+			break;
+		case RGX_HEAP_64KB_PAGE_SHIFT:
+			ui64RegValue = 2;
+			break;
+		case RGX_HEAP_256KB_PAGE_SHIFT:
+			ui64RegValue = 3;
+			break;
+		case RGX_HEAP_1MB_PAGE_SHIFT:
+			ui64RegValue = 4;
+			break;
+		case RGX_HEAP_2MB_PAGE_SHIFT:
+			ui64RegValue = 5;
+			break;
+		case RGX_HEAP_4KB_PAGE_SHIFT:
+			/* fall through */
+		default:
+			/* anything we don't support, use 4K */
+			break;
+	}
+
+	/* check that the range is defined by valid 40 bit virtual addresses */
+	PVR_ASSERT((ui64BaseAddress & ~((1ULL << 40) - 1)) == 0);
+	PVR_ASSERT((ui64EndAddress  & ~((1ULL << 40) - 1)) == 0);
+
+	/* the range config register addresses are in 2MB chunks so check 21 lsb are zero */
+	PVR_ASSERT((ui64BaseAddress & ((1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT) - 1)) == 0);
+	PVR_ASSERT((ui64EndAddress  & ((1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT)  - 1)) == 0);
+
+	ui64BaseAddress >>= RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT;
+	ui64EndAddress  >>= RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT;
+
+	ui64RegValue = (ui64RegValue << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT) |
+				   (ui64EndAddress  << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_SHIFT) |
+				   (ui64BaseAddress << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_SHIFT);
+	return ui64RegValue;
+}
+
+IMG_UINT64 RGXMMUInitRangeValue(IMG_UINT32 ui32MMURange)
+{
+	IMG_UINT64 ui64RegVal;
+	IMG_UINT32 ui32Non4KHeapPageShift;
+	IMG_UINT32 ui32RgxDefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+	PVRSRV_ERROR eError;
+
+	switch (ui32MMURange)
+	{
+		case RGX_MMU_RANGE_GLOBAL:
+			/* set the last MMU config range covering the entire virtual memory to the OS's page size */
+			ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1ULL << 40));
+			break;
+		case RGX_MMU_RANGE_NON4KHEAP:
+			/*
+			 * If the Non4K heap has a different page size than the OS's page size
+			 * (used as default for all other heaps), configure one MMU config range
+			 * for the Non4K heap
+			 */
+			eError = RGXGetNon4KHeapPageShift(NULL, &ui32Non4KHeapPageShift);
+			PVR_LOG_IF_ERROR(eError, "RGXGetNon4KHeapPageShift");
+
+			if (eError == PVRSRV_OK)
+			{
+				if (ui32Non4KHeapPageShift != ui32RgxDefaultPageShift)
+				{
+					ui64RegVal = RGXMMUComputeRangeValue(ui32Non4KHeapPageShift, RGX_GENERAL_NON4K_HEAP_BASE, RGX_GENERAL_NON4K_HEAP_SIZE);
+				}
+				else
+				{
+					ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+				}
+			}
+			else
+			{
+				/*
+				 * Error from Non4K Heap shift. Default to the normal PageShift
+				 * to attempt to continue.
+				 */
+				ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+			}
+			break;
+		default:
+			ui64RegVal = RGXMMUComputeRangeValue(ui32RgxDefaultPageShift, 0, (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+			break;
+	}
+
+	return ui64RegVal;
+}
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxmulticore.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxmulticore.c
new file mode 100644
index 000000000000..0c8bc6747805
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxmulticore.c
@@ -0,0 +1,294 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.c
+@Title          Functions related to multicore devices
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxdefs_km.h"
+#include "pdump_km.h"
+#include "rgxmulticore.h"
+#include "multicore_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwmemctx.h"
+
+
+
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores,
+                                        IMG_UINT64 *pui64Caps);
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Return multicore information to clients.
+ * Return not supported on cores without multicore.
+ */
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32CapsSize,
+                                 IMG_UINT32 *pui32NumCores,
+                                 IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDevInfo->ui32MultiCoreNumCores == 0)
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+	else
+	{
+		*pui32NumCores = psDevInfo->ui32MultiCoreNumCores;
+		if (ui32CapsSize > 0)
+		{
+			if (ui32CapsSize < psDevInfo->ui32MultiCoreNumCores)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore caps buffer too small"));
+				eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			}
+			else
+			{
+				IMG_UINT32 i;
+
+				for (i = 0; i < psDevInfo->ui32MultiCoreNumCores; ++i)
+				{
+					pui64Caps[i] = psDevInfo->pui64MultiCoreCapabilities[i];
+				}
+			}
+		}
+	}
+
+	return eError;
+}
+
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Read multicore HW registers and fill in data structure for clients.
+ * Return not supported on cores without multicore.
+ */
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		/* we only set this up once */
+		return PVRSRV_OK;
+	}
+
+	/* defaults for non-multicore devices */
+	psDevInfo->ui32MultiCoreNumCores = 0;
+	psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	psDevInfo->pui64MultiCoreCapabilities = NULL;
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		IMG_BOOL bPowerWasDown;
+		IMG_UINT32 ui32MulticoreInfo;
+		IMG_UINT32 ui32PrimaryCoreIds;
+		IMG_UINT32 ui32PrimaryId;
+		IMG_UINT32 ui32NumCores;
+		IMG_UINT32 i;
+		IMG_UINT32 ui32CoresFoundInDomain = 0;
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(XPU_MAX_REGBANKS_ADDR_WIDTH)
+		IMG_UINT32 ui32MulticoreRegBankOffset = (1 << RGX_GET_FEATURE_VALUE(psDevInfo, XPU_MAX_REGBANKS_ADDR_WIDTH));
+
+		/* Ensure the HOST_SECURITY reg bank definitions are correct */
+		if ((RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE) != ui32MulticoreRegBankOffset)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Register bank definitions for HOST_SECURITY don't match core's configuration.", __func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+#endif
+		bPowerWasDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			PVRSRVPowerLock(psDeviceNode);
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVSetSystemPowerState ON failed (%u)", __func__, eError));
+				PVRSRVPowerUnlock(psDeviceNode);
+				return eError;
+			}
+		}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			RGXFWIF_MULTICORE_INFO *psRGXMulticoreInfo;
+			IMG_UINT32 ui32FwTimeout = MAX_HW_TIME_US;
+
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores,
+		                               INVALIDATE);
+				if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores))
+				{
+					/* No need to wait if the FW has already updated the values */
+					break;
+				}
+				OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores) == 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore info not available for guest"));
+				return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+			}
+
+			psRGXMulticoreInfo = &psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo;
+			ui32NumCores = psRGXMulticoreInfo->ui32NumCores;
+			ui32MulticoreInfo = psRGXMulticoreInfo->ui32MulticoreInfo;
+
+			PVR_LOG(("RGX Guest Device initialised with %u %s",
+					 ui32NumCores, (ui32NumCores == 1U) ? "core" : "cores"));
+		}
+		else
+#endif
+		{
+			ui32NumCores = (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_DOMAIN)
+		                                            & ~RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_CLRMSK)
+		                                            >> RGX_CR_MULTICORE_DOMAIN_GPU_COUNT_SHIFT;
+			ui32MulticoreInfo = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE);
+		}
+#if defined(NO_HARDWARE)
+		/* override to defaults if no hardware */
+		ui32NumCores = RGX_MULTICORE_MAX_NOHW_CORES;
+		ui32MulticoreInfo = 0;  /* primary id 0 with 7 secondaries */
+#endif
+		/* ID for this primary is in this register */
+		ui32PrimaryId = (ui32MulticoreInfo & ~RGX_CR_MULTICORE_ID_CLRMSK) >> RGX_CR_MULTICORE_ID_SHIFT;
+
+		/* allocate storage for capabilities */
+		psDevInfo->pui64MultiCoreCapabilities = OSAllocMem(ui32NumCores * sizeof(psDevInfo->pui64MultiCoreCapabilities[0]));
+		if (psDevInfo->pui64MultiCoreCapabilities == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to alloc memory for Multicore info", __func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+
+		ui32PrimaryCoreIds = (ui32MulticoreInfo & ~RGX_CR_MULTICORE_PRIMARY_CORE_ID_CLRMSK)
+		                                        >> RGX_CR_MULTICORE_PRIMARY_CORE_ID_SHIFT;
+
+		psDevInfo->ui32MultiCorePrimaryId = ui32PrimaryId;
+		psDevInfo->ui32MultiCoreNumCores = ui32NumCores;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Multicore domain has %d cores with primary id %u\n", ui32NumCores, ui32PrimaryId));
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGX Multicore domain has %d cores with primary id %u\n",
+		             ui32NumCores, ui32PrimaryId);
+		for (i = 0; i < RGX_MULTICORE_MAX_NOHW_CORES; i++)
+		{
+			if ((ui32PrimaryCoreIds & 0x7) == ui32PrimaryId)
+			{
+				if (ui32CoresFoundInDomain >= ui32NumCores)
+				{
+					/* Enough cores have already been found in the domain, but there is an additional match.
+					   This is an illegal combination. */
+					PVR_ASSERT(ui32CoresFoundInDomain < ui32NumCores);
+					break;
+				}
+
+				/* currently all cores are identical so have the same capabilities */
+				psDevInfo->pui64MultiCoreCapabilities[ui32CoresFoundInDomain] = i
+				                    | ((i == ui32PrimaryId) ? RGX_MULTICORE_CAPABILITY_PRIMARY_EN : 0)
+				                    | RGX_MULTICORE_CAPABILITY_GEOMETRY_EN
+				                    | RGX_MULTICORE_CAPABILITY_COMPUTE_EN
+				                    | RGX_MULTICORE_CAPABILITY_FRAGMENT_EN;
+				PDUMPCOMMENT(psDeviceNode, "\tCore %u has caps 0x%08x", i,
+				             (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[ui32CoresFoundInDomain]);
+				PVR_DPF((PVR_DBG_MESSAGE, "Core %u has caps 0x%08x", i, (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[ui32CoresFoundInDomain]));
+				ui32CoresFoundInDomain++;
+			}
+			ui32PrimaryCoreIds >>= 3;
+		}
+
+		PVR_ASSERT(ui32CoresFoundInDomain == ui32NumCores);
+
+        /* revert power state to what it was on entry to this function */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Register callback to return info about multicore setup to client bridge */
+		psDeviceNode->pfnGetMultiCoreInfo = RGXGetMultiCoreInfo;
+	}
+	else
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	return eError;
+}
+
+
+/*
+ * RGXDeinitMultiCoreInfo:
+ * Release resources and clear the MultiCore values in the DeviceNode.
+ */
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pui64MultiCoreCapabilities != NULL)
+	{
+		OSFreeMem(psDevInfo->pui64MultiCoreCapabilities);
+		psDevInfo->pui64MultiCoreCapabilities = NULL;
+		psDevInfo->ui32MultiCoreNumCores = 0;
+		psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	}
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxpdump.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxpdump.c
new file mode 100644
index 000000000000..a844e17e9b97
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxpdump.c
@@ -0,0 +1,768 @@
+/*************************************************************************/ /*!
+@File           rgxpdump.c
+@Title          Device specific pdump routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific pdump functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(PDUMP)
+#include "pvrsrv.h"
+#include "devicemem_utils.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+#include "rgxpdump.h"
+#include "rgxpdump_common.h"
+#include "pdumpdesc.h"
+
+/*
+ * There are two different set of functions one for META/RISCV and one for MIPS
+ * because the Pdump player does not implement the support for
+ * the MIPS MMU yet. So for MIPS builds we cannot use DevmemPDumpSaveToFileVirtual,
+ * we have to use DevmemPDumpSaveToFile instead.
+ */
+static PVRSRV_ERROR _FWDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DECLARE_DLLIST(sFirmwareGcovBufferValidRegions);
+#endif
+	DECLARE_DLLIST(sSigTDMChecksValidRegions);
+	DECLARE_DLLIST(sSigTAChecksValidRegions);
+	DECLARE_DLLIST(sSig3DChecksValidRegions);
+	DECLARE_DLLIST(sSigCDMChecksValidRegions);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DECLARE_DLLIST(sSigRDMChecksValidRegions);
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psFirmwareGcovBufferMemDesc,
+	                             psDevInfo->ui32FirmwareGcovSize,
+	                             &sFirmwareGcovBufferValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrReturnError);
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                             psDevInfo->ui32SigTDMChecksSize,
+		                             &sSigTDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeGcovBufferRegions);
+	}
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                             psDevInfo->ui32SigTAChecksSize,
+	                             &sSigTAChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTDMChecksRegions);
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                             psDevInfo->ui32Sig3DChecksSize,
+	                             &sSig3DChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigTAChecksRegions);
+
+	eError = PDumpGetValidRegion(psDeviceNode,
+	                             psDevInfo->psRGXFWSigCDMChecksMemDesc,
+	                             psDevInfo->ui32SigCDMChecksSize,
+	                             &sSigCDMChecksValidRegions);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSig3DChecksRegions);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+	    RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = PDumpGetValidRegion(psDeviceNode,
+		                             psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                             psDevInfo->ui32SigRDMChecksSize,
+		                             &sSigRDMChecksValidRegions);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PDumpGetValidRegion", ErrFreeSigCDMChecksRegions);
+	}
+#endif
+
+
+	PDUMPIF(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+	PDUMPELSE(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Gcov */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Gcov Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psFirmwareGcovBufferMemDesc,
+	                       &sFirmwareGcovBufferValidRegions,
+	                       "firmware_gcov.img",
+	                       ui32PDumpFlags);
+#endif
+
+	/* TDM signatures */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                       &sSigTDMChecksValidRegions,
+		                       "out.tdmsig",
+		                       ui32PDumpFlags);
+	}
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                       &sSigTAChecksValidRegions,
+	                       "out.tasig",
+	                       ui32PDumpFlags);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                       &sSig3DChecksValidRegions,
+	                       "out.3dsig",
+	                       ui32PDumpFlags);
+
+	/* CDM signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump CDM signatures and checksums Buffer");
+	PDumpSaveToFileVirtual(psDeviceNode,
+	                       psDevInfo->psRGXFWSigCDMChecksMemDesc,
+	                       &sSigCDMChecksValidRegions,
+	                       "out.cdmsig",
+	                       ui32PDumpFlags);
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		/* RDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump RDM signatures and checksums Buffer");
+		PDumpSaveToFileVirtual(psDeviceNode,
+		                       psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                       &sSigRDMChecksValidRegions,
+		                       "out.rdmsig",
+		                       ui32PDumpFlags);
+	}
+#endif
+
+	PDUMPFI(psDeviceNode, "DISABLE_SIGNATURE_BUFFER_DUMP", ui32PDumpFlags);
+
+
+	return PVRSRV_OK;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+ErrFreeSigCDMChecksRegions:
+#endif
+	DevmemIntPDumpFreeValidRegions(&sSigCDMChecksValidRegions);
+ErrFreeSig3DChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSig3DChecksValidRegions);
+ErrFreeSigTAChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTAChecksValidRegions);
+ErrFreeSigTDMChecksRegions:
+	DevmemIntPDumpFreeValidRegions(&sSigTDMChecksValidRegions);
+ErrFreeGcovBufferRegions:
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	DevmemIntPDumpFreeValidRegions(&sFirmwareGcovBufferValidRegions);
+ErrReturnError:
+#endif
+	return eError;
+}
+
+static PVRSRV_ERROR _FWDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32	ui32ThreadNum, ui32Size, ui32OutFileOffset;
+
+	const IMG_CHAR acOutTraceFileName[] = "out.trace";
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDUMPIF(psDeviceNode, "ENABLE_TRACEBUF", ui32PDumpFlags);
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufThreadNumOff =
+				(IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+		                                 uiTraceBufThreadNumOff,
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+		                                 offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+		                                 0, /* 0 offset in the trace buffer mem desc */
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		PDumpSaveToFileVirtualNoValidate(psDeviceNode,
+		                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+		                                 /* move to first element of sTraceBuf */
+		                                 offsetof(RGXFWIF_TRACEBUF, sTraceBuf)
+		                                     /* skip required number of sTraceBuf elements */
+		                                     + ui32ThreadNum * sizeof(RGXFWIF_TRACEBUF_SPACE)
+		                                     /* offset into its sAssertBuf, to be pdumped */
+		                                     + offsetof(RGXFWIF_TRACEBUF_SPACE, sAssertBuf),
+		                                 ui32Size,
+		                                 acOutTraceFileName,
+		                                 ui32OutFileOffset,
+		                                 ui32PDumpFlags);
+		ui32OutFileOffset += ui32Size;
+	}
+	PDUMPFI(psDeviceNode, "ENABLE_TRACEBUF", ui32PDumpFlags);
+
+	/* FW HWPerf buffer is always allocated when PDUMP is defined, irrespective of HWPerf events being enabled/disabled */
+	PVR_ASSERT(psDevInfo->psRGXFWIfHWPerfBufMemDesc);
+
+	return PVRSRV_OK;
+}
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static PVRSRV_ERROR _MipsDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* TA signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TA signatures and checksums Buffer");
+
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTAChecksMemDesc,
+								 0,
+								 psDevInfo->ui32SigTAChecksSize,
+								 "out.tasig",
+								 0);
+
+	/* 3D signatures */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump 3D signatures and checksums Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWSig3DChecksMemDesc,
+								 0,
+								 psDevInfo->ui32Sig3DChecksSize,
+								 "out.3dsig",
+								 0);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* TDM signatures */
+		PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump TDM signatures and checksums Buffer");
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWSigTDMChecksMemDesc,
+									 0,
+									 psDevInfo->ui32SigTDMChecksSize,
+									 "out.tdmsig",
+									 0);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MipsDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32		ui32ThreadNum, ui32Size, ui32OutFileOffset;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	/* Dump trace buffers */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump trace buffers");
+	for (ui32ThreadNum = 0, ui32OutFileOffset = 0; ui32ThreadNum < RGXFW_THREAD_NUM; ui32ThreadNum++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF *)0)->sTraceBuf[ui32ThreadNum]);
+
+		/* Same again... */
+		const IMG_DEVMEM_OFFSET_T uiTraceBufSpaceAssertBufOff
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_TRACEBUF_SPACE *)0)->sAssertBuf);
+
+		/* ui32TracePointer tracepointer */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* next, dump size of trace buffer in DWords */
+		ui32Size = sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* trace buffer */
+		ui32Size = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum]);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32ThreadNum],
+								0, /* 0 offset in the trace buffer mem desc */
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+
+		/* assert info buffer */
+		ui32Size = RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ RGXFW_TRACE_BUFFER_ASSERT_SIZE * sizeof(IMG_CHAR)
+				+ sizeof(IMG_UINT32);
+		DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								uiTraceBufOff + uiTraceBufSpaceAssertBufOff,
+								ui32Size,
+								"out.trace",
+								ui32OutFileOffset);
+		ui32OutFileOffset += ui32Size;
+	}
+
+	/* Dump hwperf buffer */
+	PDumpCommentWithFlags(psDeviceNode, ui32PDumpFlags, "** Dump HWPerf Buffer");
+	DevmemPDumpSaveToFile(psDevInfo->psRGXFWIfHWPerfBufMemDesc,
+								 0,
+								 psDevInfo->ui32RGXFWIfHWPerfBufSize,
+								 "out.hwperf",
+								 0);
+
+	return PVRSRV_OK;
+}
+#endif
+
+
+/*
+ * PVRSRVPDumpSignatureBufferKM
+ */
+PVRSRV_ERROR PVRSRVPDumpSignatureBufferKM(CONNECTION_DATA * psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_UINT32			ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpSignatureBufferKM(psConnection,
+											  psDeviceNode,
+											  ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpSignatureBufferKM(psConnection,
+											psDeviceNode,
+											ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+
+
+PVRSRV_ERROR PVRSRVPDumpCRCSignatureCheckKM(CONNECTION_DATA * psConnection,
+                                            PVRSRV_DEVICE_NODE * psDeviceNode,
+                                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+/*
+ * PVRSRVPDumpValCheckPreCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPreCommandKM(CONNECTION_DATA * psConnection,
+                                             PVRSRV_DEVICE_NODE * psDeviceNode,
+                                             IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVPDumpValCheckPostCommand
+ */
+PVRSRV_ERROR PVRSRVPDumpValCheckPostCommandKM(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE * psDeviceNode,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR PVRSRVPDumpTraceBufferKM(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32PDumpFlags)
+{
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			return _MipsDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+		else
+#endif
+		{
+			return _FWDumpTraceBufferKM(psConnection, psDeviceNode, ui32PDumpFlags);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputImageDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderSize,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32LogicalWidth,
+									IMG_UINT32 ui32LogicalHeight,
+									IMG_UINT32 ui32PhysicalWidth,
+									IMG_UINT32 ui32PhysicalHeight,
+									PDUMP_PIXEL_FORMAT ePixFmt,
+									IMG_MEMLAYOUT eMemLayout,
+									IMG_FB_COMPRESSION eFBCompression,
+									const IMG_UINT32 *paui32FBCClearColour,
+									PDUMP_FBC_SWIZZLE eFBCSwizzle,
+									IMG_PBYTE pbyPDumpImageHdr)
+{
+	IMG_PUINT32 pui32Word;
+	IMG_UINT32 ui32HeaderDataSize;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK) || defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK) || defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	IMG_UINT32 ui32TFBCControl = (psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >>
+																					  RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+#endif
+
+	/* Validate parameters */
+	if (((IMAGE_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((IMAGE_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	memset(pbyPDumpImageHdr, 0, IMAGE_HEADER_SIZE);
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpImageHdr, 0);
+	pui32Word[0] = (IMAGE_HEADER_TYPE << HEADER_WORD0_TYPE_SHIFT);
+	pui32Word[1] = (IMAGE_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+				   (IMAGE_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+
+	ui32HeaderDataSize = ui32DataSize;
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		ui32HeaderDataSize += ui32HeaderSize;
+	}
+	pui32Word[2] = ui32HeaderDataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+	pui32Word[3] = ui32LogicalWidth << IMAGE_HEADER_WORD3_LOGICAL_WIDTH_SHIFT;
+	pui32Word[4] = ui32LogicalHeight << IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_SHIFT;
+
+	pui32Word[5] = ePixFmt << IMAGE_HEADER_WORD5_FORMAT_SHIFT;
+
+	pui32Word[6] = ui32PhysicalWidth << IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_SHIFT;
+	pui32Word[7] = ui32PhysicalHeight << IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_SHIFT;
+
+	pui32Word[8] = IMAGE_HEADER_WORD8_STRIDE_POSITIVE | IMAGE_HEADER_WORD8_BIFTYPE_NONE;
+
+	switch (eMemLayout)
+	{
+	case IMG_MEMLAYOUT_STRIDED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_STRIDED;
+		break;
+	case IMG_MEMLAYOUT_TWIDDLED:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_TWIDDLING_ZTWIDDLE;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported memory layout - %d", eMemLayout));
+		return PVRSRV_ERROR_UNSUPPORTED_MEMORY_LAYOUT;
+	}
+
+	switch (eFBCompression)
+	{
+	case IMG_FB_COMPRESSION_NONE:
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_PACKED_8x8:
+	case IMG_FB_COMPRESSION_DIRECT_8x8:
+	case IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_8X8;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_16x4:
+	case IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4:
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_16x4;
+		break;
+	case IMG_FB_COMPRESSION_DIRECT_32x2:
+	case IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2:
+		/* Services Client guards against unsupported FEATURE_FB_CDC_32x2.
+		   We should never pass through the UM|KM bridge on cores lacking the feature.
+		*/
+		pui32Word[8] |= IMAGE_HEADER_WORD8_FBCTYPE_32x2;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "Unsupported compression mode - %d", eFBCompression));
+		return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+	}
+
+	pui32Word[9] = 0;
+
+	if (eFBCompression != IMG_FB_COMPRESSION_NONE)
+	{
+		if ((RGX_GET_FEATURE_VALUE(psDevInfo, FBCDC) == 4) || (RGX_GET_FEATURE_VALUE(psDevInfo, FBCDC) == 5))
+		{
+			pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V4;
+
+			if (eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8  ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 ||
+				eFBCompression == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_LOSSY_ON;
+			}
+
+			pui32Word[9] |= (eFBCSwizzle << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT) & IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK;
+		}
+		else /* 3 or 3.1 */
+		{
+			IMG_BOOL bIsFBC31 = psDevInfo->psRGXFWIfFwSysData->
+					ui32ConfigFlags & RGXFWIF_INICFG_FBCDC_V3_1_EN;
+
+			if (bIsFBC31)
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT2;
+			}
+			else
+			{
+				pui32Word[9] |= IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT2;
+			}
+		}
+
+		pui32Word[9] |= IMAGE_HEADER_WORD9_FBCDECOR_ENABLE;
+	}
+
+	pui32Word[10] = paui32FBCClearColour[0];
+	pui32Word[11] = paui32FBCClearColour[1];
+	pui32Word[12] = (IMG_UINT32) (psDeviceNode->ui64FBCClearColour & 0xFFFFFFFF);
+	pui32Word[13] = (IMG_UINT32) (psDeviceNode->ui64FBCClearColour >> 32);
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP */
+		IMG_UINT32 ui32TFBCGroup  = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK;
+		switch (ui32TFBCGroup)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_0:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_1:
+				pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_37_50;
+				break;
+		}
+	}
+	else
+#endif
+	{
+		pui32Word[14] = IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75;
+	}
+
+#if defined(RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		/* Should match current value of RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME */
+		IMG_UINT32 ui32TFBCScheme = ui32TFBCControl & ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK;
+		switch (ui32TFBCScheme)
+		{
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_DEFAULT:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_ALL;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD_AND_CORRELATION:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_CORR;
+				break;
+			case RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD:
+				pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR, "Unsupported TFBC compression control scheme - %d", ui32TFBCScheme));
+				return PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE;
+		}
+	}
+	else
+#endif
+	{
+		/* Should always be set to 2 ("TFBC delta standard only") on cores without this feature */
+		pui32Word[14] |= IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY;
+	}
+
+#if defined(RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_NATIVE_YUV10))
+	{
+		IMG_UINT32 ui32TFBCOverrideYUV10 = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN);
+
+		if (ui32TFBCOverrideYUV10)
+		{
+			pui32Word[14] |= IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_EN;
+		}
+	}
+#endif
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, TFBC_VERSION) >= 20U)
+		{
+			IMG_UINT32 ui32TFBCOverrideLossyMinChannel = (ui32TFBCControl & RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN);
+
+			if (ui32TFBCOverrideLossyMinChannel)
+			{
+				pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+			}
+		}
+		else
+		{
+			/* Should be set on TFBC version < 2.0 cores */
+			pui32Word[14] |= IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN;
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPDumpPrepareOutputDataDescriptorHdr(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32HeaderType,
+									IMG_UINT32 ui32DataSize,
+									IMG_UINT32 ui32ElementType,
+									IMG_UINT32 ui32ElementCount,
+									IMG_PBYTE pbyPDumpDataHdr)
+{
+	IMG_PUINT32 pui32Word;
+
+	/* Validate parameters */
+	if (((DATA_HEADER_SIZE & ~(HEADER_WORD1_SIZE_CLRMSK >> HEADER_WORD1_SIZE_SHIFT)) != 0) ||
+		((DATA_HEADER_VERSION & ~(HEADER_WORD1_VERSION_CLRMSK >> HEADER_WORD1_VERSION_SHIFT)) != 0))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pui32Word = IMG_OFFSET_ADDR(pbyPDumpDataHdr, 0);
+
+	if (ui32HeaderType == DATA_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (DATA_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(DATA_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+
+		pui32Word[3] = ui32ElementType << DATA_HEADER_WORD3_ELEMENT_TYPE_SHIFT;
+		pui32Word[4] = ui32ElementCount << DATA_HEADER_WORD4_ELEMENT_COUNT_SHIFT;
+	}
+
+	if (ui32HeaderType == IBIN_HEADER_TYPE)
+	{
+		pui32Word[0] = (ui32HeaderType << HEADER_WORD0_TYPE_SHIFT);
+		pui32Word[1] = (IBIN_HEADER_SIZE << HEADER_WORD1_SIZE_SHIFT) |
+			(IBIN_HEADER_VERSION << HEADER_WORD1_VERSION_SHIFT);
+		pui32Word[2] = ui32DataSize << HEADER_WORD2_DATA_SIZE_SHIFT;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/******************************************************************************
+ End of file (rgxpdump.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.c
new file mode 100644
index 000000000000..fb5c757d5d80
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.c
@@ -0,0 +1,894 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Ray routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Ray routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvkm.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxray.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_memallocflags.h"
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#include "rgxworkest_ray.h"
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_RAY_UFO_DUMP	0
+
+//#define RAY_CHECKPOINT_DEBUG 1
+
+#if defined(RAY_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+struct _RGX_SERVER_RAY_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWRayContextMemDesc;
+	POS_LOCK					hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	SYNC_ADDR_LIST				sSyncAddrListFence;
+	SYNC_ADDR_LIST				sSyncAddrListUpdate;
+};
+
+PVRSRV_ERROR PVRSRVRGXCreateRayContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE		*psDeviceNode,
+											 IMG_INT32				i32Priority,
+											 IMG_HANDLE				hMemCtxPrivData,
+											 IMG_UINT32				ui32ContextFlags,
+											 IMG_UINT32				ui32StaticRayContextStateSize,
+											 IMG_PBYTE				pStaticRayContextState,
+											 IMG_UINT64				ui64RobustnessAddress,
+											 IMG_UINT32				ui32MaxDeadlineMS,
+											 RGX_SERVER_RAY_CONTEXT	**ppsRayContext)
+{
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_RAY_CONTEXT		*psRayContext;
+	RGXFWIF_FWRAYCONTEXT		*psFWRayContext;
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	PVRSRV_ERROR				eError;
+
+	*ppsRayContext = NULL;
+
+	psRayContext = OSAllocZMem(sizeof(*psRayContext));
+	if (psRayContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRayContext->psDeviceNode = psDeviceNode;
+	/*
+		Create the FW ray context, this has the RDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWRAYCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwRayContext",
+			&psRayContext->psFWRayContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwraycontext;
+	}
+
+	eError = OSLockCreate(&psRayContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_RAY,
+									 RGXFWIF_DM_RAY,
+									 hMemCtxPrivData,
+									 psRayContext->psFWRayContextMemDesc,
+									 offsetof(RGXFWIF_FWRAYCONTEXT, sRDMContext),
+									 psFWMemContextMemDesc,
+									 NULL,
+									 RGX_RDM_CCB_SIZE_LOG2,
+									 RGX_RDM_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 i32Priority,
+									 ui32MaxDeadlineMS,
+									 ui64RobustnessAddress,
+									 &sInfo,
+									 &psRayContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init Ray fw common context (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_raycommoncontext;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psRayContext->psFWRayContextMemDesc,
+			(void **)&psFWRayContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitRay(psDevInfo, &psRayContext->sWorkEstData);
+	}
+#endif
+
+	OSDeviceMemCopy(&psFWRayContext->sStaticRayContextState, pStaticRayContextState, ui32StaticRayContextStateSize);
+	DevmemPDumpLoadMem(psRayContext->psFWRayContextMemDesc, 0, sizeof(RGXFWIF_FWRAYCONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	DevmemReleaseCpuVirtAddr(psRayContext->psFWRayContextMemDesc);
+
+	SyncAddrListInit(&psRayContext->sSyncAddrListFence);
+	SyncAddrListInit(&psRayContext->sSyncAddrListUpdate);
+
+	*ppsRayContext = psRayContext;
+
+	return PVRSRV_OK;
+fail_acquire_cpu_mapping:
+fail_raycommoncontext:
+	OSLockDestroy(psRayContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psRayContext->psFWRayContextMemDesc);
+fail_fwraycontext:
+	OSFreeMem(psRayContext);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyRayContextKM(RGX_SERVER_RAY_CONTEXT *psRayContext)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psRayContext->psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_FWRAYCONTEXT	*psFWRayContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psRayContext->psFWRayContextMemDesc,
+				(void **)&psFWRayContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware ray context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+		ui32WorkEstCCBSubmitted = psFWRayContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psRayContext->psFWRayContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psRayContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psRayContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psRayContext->psDeviceNode,
+											  psRayContext->psServerCommonContext,
+											  RGXFWIF_DM_RAY,
+											  PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psRayContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(psRayContext->psServerCommonContext);
+
+	psRayContext->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		WorkEstDeInitRay(psDevInfo, &psRayContext->sWorkEstData);
+	}
+#endif
+
+	DevmemFwUnmapAndFree(psDevInfo, psRayContext->psFWRayContextMemDesc);
+
+	OSLockDestroy(psRayContext->hLock);
+	OSFreeMem(psRayContext);
+
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXKickRDMKM(RGX_SERVER_RAY_CONTEXT	*psRayContext,
+								IMG_UINT32				ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK	**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32				*paui32ClientUpdateSyncOffset,
+								IMG_UINT32				*paui32ClientUpdateValue,
+								PVRSRV_FENCE			iCheckFence,
+								PVRSRV_TIMELINE			iUpdateTimeline,
+								PVRSRV_FENCE			*piUpdateFence,
+								IMG_CHAR				pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								IMG_UINT32				ui32CmdSize,
+								IMG_PBYTE				pui8DMCmd,
+								IMG_UINT32				ui32PDumpFlags,
+								IMG_UINT32				ui32ExtJobRef,
+								IMG_UINT32				ui32AccStructSizeInBytes,
+								IMG_UINT32				ui32DispatchSize,
+								IMG_UINT64				ui64DeadlineInus)
+{
+
+	RGXFWIF_KCCB_CMD		sRayKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	PVRSRV_ERROR			eError, eError2;
+	IMG_UINT32				ui32FWCtx;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+	PVRSRV_RGXDEV_INFO      *psDevInfo;
+	RGX_CLIENT_CCB          *psClientCCB;
+	IMG_UINT32              ui32IntJobRef;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataRay = {0};
+	IMG_UINT32 ui32RDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32RDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32RDMCmdOffsetWrapCheck = 0;
+	IMG_UINT32 ui32RDMCmdOffset = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_BOOL				bCCBStateOpen = IMG_FALSE;
+	IMG_UINT64 ui64FBSCEntryMask;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	psDevInfo = FWCommonContextGetRGXDevInfo(psRayContext->psServerCommonContext);
+	psClientCCB = FWCommonContextGetClientCCB(psRayContext->psServerCommonContext);
+	ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psRayContext->hLock);
+
+	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	ui32IntClientUpdateCount = ui32ClientUpdateCount;
+
+	eError = SyncAddrListPopulate(&psRayContext->sSyncAddrListUpdate,
+									ui32ClientUpdateCount,
+									pauiClientUpdateUFODevVarBlock,
+									paui32ClientUpdateSyncOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount && !pauiIntUpdateUFOAddress)
+	{
+		pauiIntUpdateUFOAddress = psRayContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psRayContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psRayContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psRayContext->psDeviceNode->hSyncCheckpointContext,
+										iCheckFence,
+										&ui32FenceSyncCheckpointCount,
+										&apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)", __func__, eError));
+		goto fail_resolve_input_fence;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(RAY_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psRayContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psRayContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psRayContext->psDeviceNode,
+		                                   pcszUpdateFenceName,
+										   iUpdateTimeline,
+										   psRayContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%d)", __func__, eError));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+#if defined(RAY_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the ray context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the ray context update list */
+			SyncAddrListAppendSyncPrim(&psRayContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+#if defined(RAY_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Ray RDM Fence (&psRayContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psRayContext->sSyncAddrListFence));
+#if defined(RAY_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+		SyncAddrListAppendCheckpoints(&psRayContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psRayContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+#if defined(RAY_CHECKPOINT_DEBUG)
+	if (ui32IntClientUpdateCount > 0)
+	{
+		IMG_UINT32 iii;
+		IMG_UINT32 *pui32Tmp = (IMG_UINT32*)paui32IntUpdateValue;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Dumping %d update values (paui32IntUpdateValue=<%p>)...", __func__, ui32IntClientUpdateCount, (void*)paui32IntUpdateValue));
+		for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: paui32IntUpdateValue[%d] = <%p>", __func__, iii, (void*)pui32Tmp));
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: *paui32IntUpdateValue[%d] = 0x%x", __func__, iii, *pui32Tmp));
+			pui32Tmp++;
+		}
+	}
+#endif
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Ray RDM Update (&psRayContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psRayContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psRayContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psRayContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(RAY_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_RAY_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping Ray (RDM) fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Ray (RDM) fence syncs (&psRayContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psRayContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Ray (RDM) update syncs (&psRayContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psRayContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRayContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	/*
+	* Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+	* in other words, take the value and set it to zero afterwards.
+	* FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+	* as it must be ready at the time of context activation.
+	*/
+	{
+		eError = RGXExtractFBSCEntryMaskFromMMUContext(psRayContext->psDeviceNode,
+														FWCommonContextGetServerMMUCtx(psRayContext->psServerCommonContext),
+														&ui64FBSCEntryMask);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%d)", eError));
+			goto fail_invalfbsc;
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		sWorkloadCharacteristics.sRay.ui32AccStructSize = ui32AccStructSizeInBytes;
+		sWorkloadCharacteristics.sRay.ui32DispatchSize = ui32DispatchSize;
+
+		/* Prepare workload estimation */
+		WorkEstPrepare(psRayContext->psDeviceNode->pvDevice,
+				&psRayContext->sWorkEstData,
+				&psRayContext->sWorkEstData.uWorkloadMatchingData.sRay.sDataRDM,
+				RGXFWIF_CCB_CMD_TYPE_RAY,
+				&sWorkloadCharacteristics,
+				ui64DeadlineInus,
+				&sWorkloadKickDataRay);
+
+		if (sWorkloadKickDataRay.ui32CyclesPrediction != 0)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Dispatch size = %u, Acc struct size = %u, prediction = %u",
+					__func__,
+					sWorkloadCharacteristics.sRay.ui32DispatchSize,
+					sWorkloadCharacteristics.sRay.ui32AccStructSize,
+					sWorkloadKickDataRay.ui32CyclesPrediction));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32AccStructSizeInBytes);
+	PVR_UNREFERENCED_PARAMETER(ui32DispatchSize);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       ui64FBSCEntryMask,
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+                           &pPreAddr,
+                           &pPostAddr,
+                           &pRMWUFOAddr,
+	                       RGXFWIF_CCB_CMD_TYPE_RAY,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	                       &sWorkloadKickDataRay,
+#else
+	                       NULL,
+#endif
+	                       "Ray",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (eError == PVRSRV_OK)
+	{
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+		{
+			ui32RDMCmdOffset = RGXGetHostWriteOffsetCCB(psClientCCB);
+		}
+#endif
+		/*
+			All the required resources are ready at this point, we can't fail so
+			take the required server sync operations and commit all the resources
+		*/
+		RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "RDM", FWCommonContextGetFWAddress(psRayContext->psServerCommonContext).ui32Addr);
+	}
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* The following is used to determine the offset of the command header containing
+		 * the workload estimation data so that can be accessed when the KCCB is read */
+		ui32RDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(asCmdHelperData);
+
+		ui32RDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRayContext->psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32RDMCmdOffset < ui32RDMCmdOffsetWrapCheck)
+		{
+			ui32RDMWorkloadDataRO = ui32RDMCmdOffset;
+		}
+		else
+		{
+			ui32RDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/* Construct the kernel ray CCB command. */
+	sRayKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sRayKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psRayContext->psServerCommonContext);
+	sRayKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sRayKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sRayKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Store the offset to the CCCB command header so that it can be referenced
+		 * when the KCCB command reaches the FW */
+		sRayKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32RDMWorkloadDataRO + ui32RDMCmdHeaderOffset;
+	}
+#endif
+	ui32FWCtx = FWCommonContextGetFWAddress(psRayContext->psServerCommonContext).ui32Addr;
+
+	RGXSRV_HWPERF_ENQ(psRayContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_RS,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	/*
+	 * Submit the ray command to the firmware.
+	 */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError2 = RGXScheduleCommandWithoutPowerLock(psRayContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_RAY,
+									&sRayKCCBCmd,
+									ui32PDumpFlags);
+		if (eError2 != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError2 != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError2)));
+		if (eError == PVRSRV_OK)
+		{
+			eError = eError2;
+		}
+	}
+	else
+	{
+		PVRGpuTraceEnqueueEvent(psRayContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_RS);
+	}
+	/*
+	 * Now check eError (which may have returned an error from our earlier call
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdsubmit;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__, (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif /* defined(NO_HARDWARE) */
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRayContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pcszUpdateFenceName);
+	}
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psRayContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_cmdsubmit:
+fail_acquirepowerlock:
+fail_cmdacquire:
+fail_invalfbsc:
+	SyncAddrListRollbackCheckpoints(psRayContext->psDeviceNode, &psRayContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psRayContext->psDeviceNode, &psRayContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+fail_resolve_input_fence:
+
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psRayContext->hLock);
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxray.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.h
new file mode 100644
index 000000000000..ecf23db03a46
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxray.h
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX ray functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX compute functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXRAY_H_)
+#define RGXRAY_H_
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxfwutils.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+
+
+typedef struct _RGX_SERVER_RAY_CONTEXT_ RGX_SERVER_RAY_CONTEXT;
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXCreateRayContextKM
+
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateRayContextKM(CONNECTION_DATA		*psConnection,
+										 PVRSRV_DEVICE_NODE		*psDeviceNode,
+										 IMG_INT32				i32Priority,
+										 IMG_HANDLE				hMemCtxPrivData,
+										 IMG_UINT32				ui32ContextFlags,
+										 IMG_UINT32				ui32StaticRayContextStateSize,
+										 IMG_PBYTE				pStaticRayContextState,
+										 IMG_UINT64				ui64RobustnessAddress,
+										 IMG_UINT32				ui32MaxDeadlineMS,
+										 RGX_SERVER_RAY_CONTEXT	**ppsRayContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDestroyRayContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyRayContext
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyRayContextKM(RGX_SERVER_RAY_CONTEXT *psRayContext);
+
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXKickRDMKM
+
+ @Description
+	Server-side implementation of RGXKickRDM
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickRDMKM(RGX_SERVER_RAY_CONTEXT	*psRayContext,
+								IMG_UINT32				ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK	**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32				*paui32ClientUpdateSyncOffset,
+								IMG_UINT32				*paui32ClientUpdateValue,
+								PVRSRV_FENCE			iCheckFence,
+								PVRSRV_TIMELINE			iUpdateTimeline,
+								PVRSRV_FENCE			*piUpdateFence,
+								IMG_CHAR				pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								IMG_UINT32				ui32CmdSize,
+								IMG_PBYTE				pui8DMCmd,
+								IMG_UINT32				ui32PDumpFlags,
+								IMG_UINT32				ui32ExtJobRef,
+								IMG_UINT32				ui32AccStructSizeInBytes,
+								IMG_UINT32				ui32DispatchSize,
+								IMG_UINT64				ui64DeadlineInus);
+
+
+#endif /* RGXRAY_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxsrvinit.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxsrvinit.c
new file mode 100644
index 000000000000..709cbfd1002e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxsrvinit.c
@@ -0,0 +1,1690 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvinit.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "km_apphint_defs.h"
+#include "htbuffer_types.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "rgx_compat_bvnc.h"
+
+#include "rgxdefs_km.h"
+#include "pvrsrv.h"
+
+#include "rgxinit_apphints.h"
+#include "rgxinit.h"
+#include "rgxmulticore.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "osfunc.h"
+
+#include "rgxdefs_km.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "rgx_fwif_hwperf.h"
+#include "rgxhwperf_common.h"
+
+#include "fwload.h"
+#include "rgxlayer_impl.h"
+#include "rgxfwimageutils.h"
+#include "rgxfwutils.h"
+
+#include "rgx_bvnc_defs_km.h"
+
+#include "rgxdevice.h"
+
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#include "rgxdevice.h"
+#include "pvrsrv_device.h"
+#endif
+
+#define	HW_PERF_FILTER_DEFAULT         0x00000000 /* Default to no HWPerf */
+#define HW_PERF_FILTER_DEFAULT_ALL_ON  0xFFFFFFFF /* All events */
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+#define AVAIL_POW_UNITS_MASK_DEFAULT   (PVRSRV_APPHINT_HWVALAVAILABLESPUMASK)
+#define AVAIL_RAC_MASK_DEFAULT         (PVRSRV_APPHINT_HWVALAVAILABLERACMASK)
+#endif
+
+/* Kernel CCB size */
+
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE 4
+#endif
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE 16
+#endif
+
+#if PVRSRV_APPHINT_KCCB_SIZE_LOG2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too low.
+#elif PVRSRV_APPHINT_KCCB_SIZE_LOG2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too high.
+#endif
+
+
+#include "os_apphint.h"
+
+/*
+ * _ParseHTBAppHints:
+ *
+ * Generate necessary references to the globally visible AppHints which are
+ * declared in the above #include "km_apphint_defs.h"
+ * Without these local references some compiler tool-chains will treat
+ * unreferenced declarations as fatal errors. This function duplicates the
+ * HTB_specific apphint references which are made in htbserver.c:HTBInit()
+ * However, it makes absolutely *NO* use of these hints.
+ */
+static void
+_ParseHTBAppHints(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvAppHintState)
+{
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32OpMode;
+	IMG_UINT32 ui32BufferSize;
+
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEHTBLOGGROUP;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, EnableHTBLogGroup,
+	                     &ui32AppHintDefault, &ui32LogType);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBOPERATIONMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBOperationMode,
+	                     &ui32AppHintDefault, &ui32OpMode);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBUFFERSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+	                     &ui32AppHintDefault, &ui32BufferSize);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetFilterFlags
+
+ @Description   Initialise and return filter flags
+
+ @Input         bFilteringMode : Enable new TPU filtering mode
+ @Input         ui32TruncateMode : TPU Truncate mode
+
+ @Return        IMG_UINT32 : Filter flags
+
+******************************************************************************/
+static INLINE IMG_UINT32 GetFilterFlags(IMG_BOOL bFilteringMode, IMG_UINT32 ui32TruncateMode)
+{
+	IMG_UINT32 ui32FilterFlags = 0;
+
+	ui32FilterFlags |= bFilteringMode ? RGXFWIF_FILTCFG_NEW_FILTER_MODE : 0;
+	if (ui32TruncateMode == 2)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_INT;
+	}
+	else if (ui32TruncateMode == 3)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_HALF;
+	}
+
+	return ui32FilterFlags;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      InitDeviceFlags
+
+ @Description   Initialise and return device flags
+
+ @Input         psDeviceNode     : Pointer to device node
+ @Input         pvAppHintState   : Pointer to apphint state
+ @Input         psHints          : Apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void InitDeviceFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   void *pvAppHintState,
+                                   RGX_INIT_APPHINTS *psHints)
+{
+	IMG_UINT32 ui32DeviceFlags = 0;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bZeroFreelist;
+	IMG_BOOL bDisableFEDLogging;
+
+
+	bAppHintDefault = PVRSRV_APPHINT_ZEROFREELIST;
+	OSGetAppHintBOOL(psDeviceNode,  pvAppHintState,  ZeroFreelist,
+	                   &bAppHintDefault,         &bZeroFreelist);
+	ui32DeviceFlags |= bZeroFreelist ? RGXKM_DEVICE_STATE_ZERO_FREELIST : 0;
+
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEFEDLOGGING;
+	OSGetAppHintBOOL(psDeviceNode,    pvAppHintState,    DisableFEDLogging,
+	                   &bAppHintDefault,        &bDisableFEDLogging);
+	ui32DeviceFlags |= bDisableFEDLogging ? RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN : 0;
+
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	BITMASK_SET(ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+#endif
+
+	psHints->ui32DeviceFlags = ui32DeviceFlags;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetApphints
+
+ @Description   Read init time apphints and initialise apphints structure
+
+ @Input         psHints : Pointer to apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetApphints(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_INIT_APPHINTS *psHints)
+{
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+	IMG_UINT32 ui32ParamTemp;
+#if defined(__linux__)
+	IMG_UINT64 ui64AppHintDefault;
+#endif
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESIGNATURECHECKS;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSignatureChecks,
+	                     &bAppHintDefault,    &psHints->bEnableSignatureChecks);
+	ui32AppHintDefault = PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    SignatureChecksBufSize,
+	                     &ui32AppHintDefault,    &psHints->ui32SignatureChecksBufSize);
+
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTOUTOFMEMORY;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOutOfMemory,
+	                   &bAppHintDefault,        &psHints->bAssertOnOutOfMem);
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTONHWRTRIGGER;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOnHWRTrigger,
+	                   &bAppHintDefault,        &psHints->bAssertOnHWRTrigger);
+	bAppHintDefault = PVRSRV_APPHINT_CHECKMLIST;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    CheckMList,
+	                   &bAppHintDefault,        &psHints->bCheckMlist);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLECLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableClockGating,
+	                   &bAppHintDefault,        &psHints->bDisableClockGating);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEDMOVERLAP;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableDMOverlap,
+	                   &bAppHintDefault,        &psHints->bDisableDMOverlap);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEAPM;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    EnableAPM,
+	                     &ui32AppHintDefault,      &ui32ParamTemp);
+	psHints->eRGXActivePMConf = ui32ParamTemp;
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEDMKILLINGRANDMODE;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    EnableGenericDMKillingRandMode,
+	                   &bAppHintDefault,           &psHints->bEnableDMKillRand);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableRandomContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableRandomCsw);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSoftResetContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableSoftResetCsw);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableFWContextSwitch,
+	                     &ui32AppHintDefault,    &psHints->ui32EnableFWContextSwitch);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLERDPOWERISLAND;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableRDPowerIsland,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eRGXRDPowerIslandConf = ui32ParamTemp;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESPUCLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    EnableSPUClockGating,
+	                   &bAppHintDefault,           &psHints->bSPUClockGating);
+#endif
+	ui32AppHintDefault = PVRSRV_APPHINT_FIRMWAREPERF;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FirmwarePerf,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eFirmwarePerf = ui32ParamTemp;
+	ui32AppHintDefault = PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FWContextSwitchProfile,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchProfile);
+
+	/*
+	 * HWPerf apphints *
+	 */
+	bAppHintDefault = PVRSRV_APPHINT_HWPERFDISABLECOUNTERFILTER;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    HWPerfDisableCounterFilter,
+	                     &bAppHintDefault,    &psHints->bHWPerfDisableCounterFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfHostBufSize);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfFWBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfFWBufSize);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_KCCB_SIZE_LOG2;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,  KernelCCBSizeLog2,
+	                     &ui32AppHintDefault,    &psHints->ui32KCCBSizeLog2);
+
+	if (psHints->ui32KCCBSizeLog2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too low, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE;
+	}
+	else if (psHints->ui32KCCBSizeLog2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too high, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE;
+	}
+
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AppHintDefault = PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  ISPSchedulingLatencyMode,
+	                     &ui32AppHintDefault,  &psHints->ui32ISPSchedulingLatencyMode);
+#endif
+
+#if defined(__linux__)
+	/* name changes */
+	{
+		IMG_UINT64 ui64Tmp;
+		bAppHintDefault = PVRSRV_APPHINT_DISABLEPDUMPPANIC;
+		OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,  DisablePDumpPanic,
+		                   &bAppHintDefault,           &psHints->bDisablePDP);
+		ui64AppHintDefault = PVRSRV_APPHINT_HWPERFFWFILTER;
+		OSGetAppHintUINT64(psDevInfo->psDeviceNode,  pvAppHintState,  HWPerfFWFilter,
+		                   &ui64AppHintDefault,        &ui64Tmp);
+		psHints->ui32HWPerfFilter0 = (IMG_UINT32)(ui64Tmp & 0xffffffffllu);
+		psHints->ui32HWPerfFilter1 = (IMG_UINT32)((ui64Tmp >> 32) & 0xffffffffllu);
+	}
+#endif
+
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTFILTER;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWPerfHostFilter,
+	                     &ui32AppHintDefault,          &psHints->ui32HWPerfHostFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_SECONDARYOSCLOCKSOURCE;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    SecondaryOSClockSource,
+	                         &ui32AppHintDefault,      &psHints->ui32SecondaryOSClockSource);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWRDebugDumpLimit,
+	                     &ui32AppHintDefault,          &ui32ParamTemp);
+	psHints->ui32HWRDebugDumpLimit = MIN(ui32ParamTemp, RGXFWIF_HWR_DEBUG_DUMP_ALL);
+
+	{
+		IMG_BOOL bFilteringMode = IMG_FALSE;
+		IMG_UINT32 ui32TruncateMode = 0U;
+
+#if defined(HW_ERN_42290_BIT_MASK) && defined(RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42290) && RGX_IS_FEATURE_SUPPORTED(psDevInfo, TPU_FILTERING_MODE_CONTROL))
+#endif
+		{
+			bAppHintDefault = PVRSRV_APPHINT_NEWFILTERINGMODE;
+			OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  NewFilteringMode,
+			                   &bAppHintDefault,  &bFilteringMode);
+		}
+
+#if defined(HW_ERN_42606_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42606))
+#endif
+		{
+			ui32AppHintDefault = PVRSRV_APPHINT_TRUNCATEMODE;
+			OSGetAppHintUINT32(APPHINT_NO_DEVICE,  pvAppHintState,  TruncateMode,
+			                     &ui32AppHintDefault,  &ui32TruncateMode);
+		}
+
+		psHints->ui32FilterFlags = GetFilterFlags(bFilteringMode, ui32TruncateMode);
+	}
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#endif
+	{
+		bAppHintDefault = PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG;
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  EnableTrustedDeviceAceConfig,
+		                   &bAppHintDefault,     &psHints->bEnableTrustedDeviceAceConfig);
+	}
+
+#if defined(__linux__)
+	ui32AppHintDefault = 0;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  FWContextSwitchCrossDM,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchCrossDM);
+#endif
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	ui32AppHintDefault = PVRSRV_APPHINT_PHYSMEMTESTPASSES;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  PhysMemTestPasses,
+	                     &ui32AppHintDefault,    &psHints->ui32PhysMemTestPasses);
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	psHints->ui32AvailablePowUnitsMask = AVAIL_POW_UNITS_MASK_DEFAULT;
+	psHints->ui32AvailableRACMask = AVAIL_RAC_MASK_DEFAULT;
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	ui32AppHintDefault = PVRSRV_APPHINT_TFBCVERSION;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, TFBCVersionDowngrade,
+	 &ui32AppHintDefault, &psHints->ui32TFBCVersion);
+
+	if (ui32AppHintDefault != psHints->ui32TFBCVersion)
+	{
+		PVR_LOG(("TFBCVersionDowngrade set to %u", psHints->ui32TFBCVersion));
+	}
+
+	psHints->bTFBCCompressionControlLossyMinChannel = false;
+	psHints->bTFBCCompressionControlYUVFormat = false;
+	psHints->ui32TFBCCompressionControlScheme =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME;
+	psHints->ui32TFBCCompressionControlGroup =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	ui32AppHintDefault = PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  DebugDumpFWTLogType,
+	                     &ui32AppHintDefault,  &psHints->eDebugDumpFWTLogType);
+	if ((IMG_UINT32)psHints->eDebugDumpFWTLogType > RGX_FWT_LOGTYPE_PARTIAL)
+	{
+		psHints->eDebugDumpFWTLogType = RGX_FWT_LOGTYPE_NONE;
+		PVR_DPF((PVR_DBG_WARNING, "Invalid value for DebugDumpFWTLogType. Setting to 0 (disabled)."));
+	}
+#if defined(SUPPORT_ICS)
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableIdleCycleStealing,
+	                     &ui32AppHintDefault,    &psHints->ui32EnableIdleCycleStealing);
+	ui32AppHintDefault = PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  FaultDetectionTimeInterval, &ui32AppHintDefault, &psHints->ui32FDTI);
+	ui32AppHintDefault = PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  ICSTimeIntervalThreshold, &ui32AppHintDefault, &psHints->ui32ICSThreshold);
+	bAppHintDefault = IMG_FALSE;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  ICSTestModeOn, &bAppHintDefault, &psHints->bTestModeOn);
+#endif
+
+	/*
+	 * FW logs apphints
+	 */
+	{
+		IMG_UINT32 ui32LogGroup, ui32TraceOrTBI;
+
+		ui32AppHintDefault = PVRSRV_APPHINT_ENABLELOGGROUP;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, EnableLogGroup, &ui32AppHintDefault, &ui32LogGroup);
+		ui32AppHintDefault = PVRSRV_APPHINT_FIRMWARELOGTYPE;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, FirmwareLogType, &ui32AppHintDefault, &ui32TraceOrTBI);
+
+		/* Defaulting to TRACE */
+		BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+		if (ui32TraceOrTBI == 1 /* TBI */)
+		{
+			if ((ui32LogGroup & RGXFWIF_LOG_TYPE_GROUP_MASK) == 0)
+			{
+				/* No groups configured - defaulting to MAIN group */
+				BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_GROUP_MAIN);
+			}
+			BITMASK_UNSET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+		}
+#endif
+		psHints->ui32LogType = ui32LogGroup;
+	}
+
+	_ParseHTBAppHints(psDevInfo->psDeviceNode, pvAppHintState);
+
+	InitDeviceFlags(psDevInfo->psDeviceNode, pvAppHintState, psHints);
+
+	OSFreeAppHintState(pvAppHintState);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      GetFWConfigFlags
+
+ @Description   Initialise and return FW config flags
+
+ @Input         psHints            : Apphints container
+ @Input         pui32FWConfigFlags : Pointer to config flags
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetFWConfigFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    RGX_INIT_APPHINTS *psHints,
+                                    IMG_UINT32 *pui32FWConfigFlags,
+                                    IMG_UINT32 *pui32FWConfigFlagsExt,
+                                    IMG_UINT32 *pui32FwOsCfgFlags)
+{
+	IMG_UINT32 ui32FWConfigFlags = 0;
+	IMG_UINT32 ui32FWConfigFlagsExt = 0;
+	IMG_UINT32 ui32FwOsCfgFlags = psHints->ui32FWContextSwitchCrossDM |
+	                              (psHints->ui32EnableFWContextSwitch & ~RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK)
+#if defined(SUPPORT_ICS)
+	                              | ((psHints->ui32EnableIdleCycleStealing << RGXFWIF_INICFG_OS_ICS_DM_ALL_SHIFT) &
+	                              ~RGXFWIF_INICFG_OS_ICS_CLRMSK)
+#endif
+;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32FWConfigFlagsSupValExt = 0;
+	IMG_UINT32 ui32TFBCVersion = 0U;
+#endif
+
+	ui32FWConfigFlags |= psHints->bAssertOnOutOfMem ? RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY : 0;
+	ui32FWConfigFlags |= psHints->bAssertOnHWRTrigger ? RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER : 0;
+	ui32FWConfigFlags |= psHints->bCheckMlist ? RGXFWIF_INICFG_CHECK_MLIST_EN : 0;
+	ui32FWConfigFlags |= psHints->bDisableClockGating ? RGXFWIF_INICFG_DISABLE_CLKGATING_EN : 0;
+
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PIPELINED_DATAMASTERS_VERSION)  &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PIPELINED_DATAMASTERS_VERSION) > 0))
+	{
+		if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, ERYX_TOP_INFRASTRUCTURE))
+		{
+			if ((!RGX_IS_FEATURE_SUPPORTED(psDevInfo, DIVANO_TOP_INFRASTRUCTURE)) ||
+				(RGX_GET_FEATURE_VALUE(psDevInfo, ECC_RAMS) == 0))
+			{
+				/* Pipeline DM roadblocks are currently enabled pre-DXS. */
+				ui32FWConfigFlags |= RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN;
+			}
+		}
+	}
+#endif
+
+	ui32FWConfigFlags |= psHints->bDisableDMOverlap ? RGXFWIF_INICFG_DISABLE_DM_OVERLAP : 0;
+	ui32FWConfigFlags |= psHints->bDisablePDP ? RGXFWIF_INICFG_DISABLE_PDP_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableDMKillRand ? RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableRandomCsw ? RGXFWIF_INICFG_CTXSWITCH_MODE_RAND : 0;
+	ui32FWConfigFlags |= psHints->bEnableSoftResetCsw ? RGXFWIF_INICFG_CTXSWITCH_SRESET_EN : 0;
+	ui32FWConfigFlags |= (psHints->ui32HWPerfFilter0 != 0 || psHints->ui32HWPerfFilter1 != 0) ? RGXFWIF_INICFG_HWPERF_EN : 0;
+	ui32FWConfigFlags |= psHints->bHWPerfDisableCounterFilter ? RGXFWIF_INICFG_HWP_DISABLE_FILTER : 0;
+	ui32FWConfigFlags |= (psHints->ui32FWContextSwitchProfile << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT) & RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK;
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32FWConfigFlags |= (psHints->ui32ISPSchedulingLatencyMode << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT) & RGXFWIF_INICFG_ISPSCHEDMODE_MASK;
+#endif
+
+	{
+		ui32FWConfigFlags |= psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode) ? RGXFWIF_INICFG_FBCDC_V3_1_EN : 0;
+	}
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+
+	/* Determine if we need to present a TFBC v1.0, v1.1 or native
+	 * behaviour. For V1.0 we need to set the following features:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x1
+	 *  TFBCCompressionControlScheme = 0x2
+	 *  TFBCCompressionControlGroup = 0x0
+	 * For V1.1 we need to set the following:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x0
+	 *  TFBCCompressionControlScheme = 0x1
+	 *  TFBCCompressionControlGroup = 0 / 1 (depends on LOSSY_37_PERCENT)
+	 * The gating for these values depends on whether the GPU supports
+	 * RGX_FEATURE_TFBC_VERSION = 20U
+	 */
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		ui32TFBCVersion = RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION);
+
+		if (ui32TFBCVersion >= 20U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 11:	/* TFBC Version 1.1 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = false;
+					psHints->ui32TFBCCompressionControlScheme = 1U;
+					psHints->ui32TFBCCompressionControlGroup =
+					    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.1, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+					         " Ignoring. Using value %u instead",
+					         __func__, psHints->ui32TFBCVersion,
+					         ui32TFBCVersion));
+					break;
+			}
+
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else if (ui32TFBCVersion == 11U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+				             " Ignoring. Using value %u instead",
+				             __func__, psHints->ui32TFBCVersion,
+				             ui32TFBCVersion));
+					break;
+			}
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else	/* TFBC v1.0 */
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32FWConfigFlagsSupValExt);
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = 0;
+#endif
+			if ((psHints->ui32TFBCVersion != 0U) &&
+			    (psHints->ui32TFBCVersion != ui32TFBCVersion))
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s: Cannot specify TFBC version %u"
+				         " on a version %u GPU core", __func__,
+				         psHints->ui32TFBCVersion, ui32TFBCVersion));
+			}
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	if ((ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN) &&
+		((ui32FWConfigFlags & RGXFWIF_INICFG_ISPSCHEDMODE_MASK) == RGXFWIF_INICFG_ISPSCHEDMODE_NONE))
+	{
+		ui32FwOsCfgFlags &= ~RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN;
+		PVR_DPF((PVR_DBG_WARNING, "ISPSchedulingLatencyMode=0 implies context switching is inoperable on DM_3D.\n"
+				 "Overriding current value EnableFWContextSwitch=0x%x with new value 0x%x",
+				 psHints->ui32EnableFWContextSwitch,
+				 ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL));
+	}
+
+	*pui32FWConfigFlags    = ui32FWConfigFlags;
+	*pui32FWConfigFlagsExt = ui32FWConfigFlagsExt;
+	*pui32FwOsCfgFlags     = ui32FwOsCfgFlags;
+}
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+/*!
+*******************************************************************************
+
+ @Function      RGXTDProcessFWImage
+
+ @Description   Fetch and send data used by the trusted device to complete
+                the FW image setup
+
+ @Input         psDeviceNode : Device node
+ @Input         psFWParams : Firmware and parameters used by the FW
+
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXTDProcessFWImage(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        PVRSRV_FW_PARAMS *psFWParams)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+	PVRSRV_ERROR eError;
+
+	if (psDevConfig->pfnTDSendFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TDSendFWImage not implemented!", __func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		if (psFWParams->uFWP.sMips.ui32FWPageTableNumPages > TD_MAX_NUM_MIPS_PAGETABLE_PAGES)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Number of page table pages %u greater "
+					 "than what is allowed by the TD interface (%u), FW might "
+					 "not work properly!", __func__,
+					 psFWParams->uFWP.sMips.ui32FWPageTableNumPages,
+					 TD_MAX_NUM_MIPS_PAGETABLE_PAGES));
+		}
+	}
+#endif
+
+	eError = psDevConfig->pfnTDSendFWImage(psDevConfig->hSysData, psFWParams);
+
+	return eError;
+}
+#endif
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXAcquireMipsBootldrData
+
+ @Description   Acquire MIPS bootloader data parameters
+
+ @Input         psDeviceNode : Device node
+ @Input         puFWParams   : FW boot parameters
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireMipsBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+	MMU_DEVICEATTRIBS *psFWMMUDevAttrs = psDevInfo->psDeviceNode->psFirmwareMMUDevAttrs;
+	IMG_DEV_PHYADDR sAddr;
+	IMG_UINT32 ui32PTSize, i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bValid;
+
+	/* Rogue Registers physical address */
+#if defined(SUPPORT_ALT_REGBASE)
+	puFWParams->sMips.sGPURegAddr = psDeviceNode->psDevConfig->sAltRegsGpuPBase;
+#else
+	PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+	                           1,
+	                           &puFWParams->sMips.sGPURegAddr,
+	                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+#endif
+
+	/* MIPS Page Table physical address */
+	MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sAddr);
+
+	/* MIPS Page Table allocation is contiguous. Pass one or more addresses
+	 * to the FW depending on the Page Table size and alignment. */
+
+	ui32PTSize = (psFWMMUDevAttrs->psTopLevelDevVAddrConfig->uiNumEntriesPT)
+	             << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	ui32PTSize = PVR_ALIGN(ui32PTSize, 1U << psFWMMUDevAttrs->ui32BaseAlign);
+
+	puFWParams->sMips.ui32FWPageTableLog2PageSize = psFWMMUDevAttrs->ui32BaseAlign;
+	puFWParams->sMips.ui32FWPageTableNumPages = ui32PTSize >> psFWMMUDevAttrs->ui32BaseAlign;
+
+	if (puFWParams->sMips.ui32FWPageTableNumPages > 4U)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Page table cannot be mapped by the FW "
+		         "(size 0x%x, log2 page size %u, %u pages)",
+		         __func__, ui32PTSize, puFWParams->sMips.ui32FWPageTableLog2PageSize,
+		         puFWParams->sMips.ui32FWPageTableNumPages));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Confirm page alignment fits in 64-bits */
+	if (psFWMMUDevAttrs->ui32BaseAlign > 63)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid page alignment "
+		         "(psFWMMUDevAttrs->ui32BaseAlign = %u)",
+		         __func__, psFWMMUDevAttrs->ui32BaseAlign));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	for (i = 0; i < puFWParams->sMips.ui32FWPageTableNumPages; i++)
+	{
+		puFWParams->sMips.asFWPageTableAddr[i].uiAddr =
+		    sAddr.uiAddr + i * (1ULL << psFWMMUDevAttrs->ui32BaseAlign);
+	}
+
+	/* MIPS Stack Pointer Physical Address */
+	eError = RGXGetPhyAddr(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+	                       &puFWParams->sMips.sFWStackAddr,
+	                       RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+	                       OSGetPageShift(),
+	                       1,
+	                       &bValid);
+
+	return eError;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      InitFirmware
+
+ @Description   Allocate, initialise and pdump Firmware code and data memory
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitFirmware(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	OS_FW_IMAGE       *psRGXFW = NULL;
+	PVRSRV_FW_PARAMS  sFWParams = {0};
+
+	/* FW code memory */
+	IMG_DEVMEM_SIZE_T uiFWCodeAllocSize;
+	void              *pvFWCodeHostAddr;
+
+	/* FW data memory */
+	IMG_DEVMEM_SIZE_T uiFWDataAllocSize;
+	void              *pvFWDataHostAddr;
+
+	/* FW coremem code memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememCodeAllocSize;
+	void              *pvFWCorememCodeHostAddr = NULL;
+
+	/* FW coremem data memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememDataAllocSize;
+	void              *pvFWCorememDataHostAddr = NULL;
+
+	PVRSRV_FW_BOOT_PARAMS uFWParams;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_BOOL bUseSecureFWData =
+								RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	                            (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+	                             RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32) ||
+#endif
+	                            RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR);
+#endif
+
+	/*
+	 * Get pointer to Firmware image
+	 */
+	eError = RGXLoadAndGetFWData(psDeviceNode, &psRGXFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Error or confirmation message generated in RGXLoadAndGetFWData */
+		goto fw_load_fail;
+	}
+
+	/*
+	 * Get pointer and size
+	 */
+	sFWParams.pvFirmware       = OSFirmwareData(psRGXFW);
+	sFWParams.ui32FirmwareSize = OSFirmwareSize(psRGXFW);
+
+	/*
+	 * Allow it to be pre-processed by the platform hook
+	 */
+	if (psDeviceNode->psDevConfig->pfnPrepareFWImage != NULL)
+	{
+		eError = psDeviceNode->psDevConfig->pfnPrepareFWImage(
+				psDeviceNode->psDevConfig->hSysData, &sFWParams);
+		if (eError != PVRSRV_OK)
+			goto cleanup_initfw;
+	}
+
+	/*
+	 * Allocate Firmware memory
+	 */
+
+	eError = RGXGetFWImageAllocSize(&psDevInfo->sLayerParams,
+	                                sFWParams.pvFirmware,
+	                                sFWParams.ui32FirmwareSize,
+	                                &uiFWCodeAllocSize,
+	                                &uiFWDataAllocSize,
+	                                &uiFWCorememCodeAllocSize,
+	                                &uiFWCorememDataAllocSize,
+	                                &psDevInfo->sFWInfoHeader);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXGetFWImageAllocSize failed",
+		        __func__));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Initiate FW compatibility check for Native and Host.
+	 * Guest compatibility check must be done after FW boot.
+	 */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed compatibility check for device %p (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+	}
+
+	psDevInfo->ui32FWCodeSizeInBytes = uiFWCodeAllocSize;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s: META DMA not available, disabling core memory code/data",
+		        __func__));
+		uiFWCorememCodeAllocSize = 0;
+		uiFWCorememDataAllocSize = 0;
+	}
+#endif
+
+	psDevInfo->ui32FWCorememCodeSizeInBytes = uiFWCorememCodeAllocSize;
+
+	eError = RGXInitAllocFWImgMem(psDeviceNode,
+	                              uiFWCodeAllocSize,
+	                              uiFWDataAllocSize,
+	                              uiFWCorememCodeAllocSize,
+	                              uiFWCorememDataAllocSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXInitAllocFWImgMem failed (%d)",
+		        __func__,
+		        eError));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Acquire pointers to Firmware allocations
+	 */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCodeHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, &pvFWCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", cleanup_initfw);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWDataHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc, &pvFWDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_code);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCorememCodeHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, &pvFWCorememCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_data);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		pvFWCorememDataHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememDataAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc, &pvFWCorememDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_corememcode);
+	}
+
+	/*
+	 * Prepare FW boot parameters
+	 */
+	OSCachedMemSet(&uFWParams, 0, sizeof(PVRSRV_FW_BOOT_PARAMS));
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		uFWParams.sMeta.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sMeta.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeFWAddr = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sMeta.uiFWCorememCodeSize = uiFWCorememCodeAllocSize;
+		uFWParams.sMeta.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sMeta.sFWCorememDataFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+		uFWParams.sMeta.ui32NumThreads = 2;
+#else
+		uFWParams.sMeta.ui32NumThreads = 1;
+#endif
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = RGXAcquireMipsBootldrData(psDeviceNode, &uFWParams);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXAcquireMipsBootldrData failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+	}
+#endif
+	else
+	{
+		uFWParams.sRISCV.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+
+		uFWParams.sRISCV.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememCodeFWAddr   = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sRISCV.uiFWCorememCodeSize    = uiFWCorememCodeAllocSize;
+
+		uFWParams.sRISCV.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememDataFWAddr   = psDevInfo->sFWCorememDataStoreFWAddr;
+		uFWParams.sRISCV.uiFWCorememDataSize    = uiFWCorememDataAllocSize;
+	}
+
+
+	/*
+	 * On Volcanic the TEE handles the loading of all Firmware sections.
+	 */
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = RGXProcessFWImage(&psDevInfo->sLayerParams,
+								   sFWParams.pvFirmware,
+								   pvFWCodeHostAddr,
+								   pvFWDataHostAddr,
+								   pvFWCorememCodeHostAddr,
+								   pvFWCorememDataHostAddr,
+								   &uFWParams);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXProcessFWImage failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+		RGXFwSharedMemCacheOpExec(pvFWCodeHostAddr,
+		                          sizeof(psDevInfo->psRGXFWCodeMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememCodeAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememCodeHostAddr,
+			                          sizeof(psDevInfo->psRGXFWCorememCodeMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+
+		RGXFwSharedMemCacheOpExec(pvFWDataHostAddr,
+		                          sizeof(psDevInfo->psRGXFWDataMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememDataAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememDataHostAddr,
+			                          sizeof(psDevInfo->psRGXFWIfCorememDataStoreMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+	}
+#else
+	sFWParams.uFWP = uFWParams;
+	RGXTDProcessFWImage(psDeviceNode, &sFWParams);
+#endif
+
+
+	/*
+	 * PDump Firmware allocations
+	 */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Dump firmware code image");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWCodeMemDesc,
+	                   0,
+	                   uiFWCodeAllocSize,
+	                   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware data image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWDataMemDesc,
+		                   0,
+		                   uiFWDataAllocSize,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem code image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWCorememCodeMemDesc,
+						   0,
+						   uiFWCorememCodeAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem data store image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+						   0,
+						   uiFWCorememDataAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION) || defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/*
+	 * Release Firmware allocations and clean up
+	 */
+release_fw_allocations:
+#endif
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && (uiFWCorememDataAllocSize != 0))
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+	}
+
+release_corememcode:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && (uiFWCorememCodeAllocSize != 0))
+#else
+	if (uiFWCorememCodeAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+
+release_data:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+	}
+
+release_code:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+cleanup_initfw:
+	OSUnloadFirmware(psRGXFW);
+fw_load_fail:
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitialiseHWPerfCounters
+
+ @Description   Initialisation of hardware performance counters and dumping
+                them out to pdump, so that they can be modified at a later
+                point.
+
+ @Input         pvDevice
+ @Input         psHWPerfDataMemDesc
+ @Input         psHWPerfInitDataInt
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     void *pvDevice,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc,
+                                     RGXFWIF_HWPERF_CTL *psHWPerfInitDataInt)
+{
+	RGXFWIF_HWPERF_CTL_BLK *psHWPerfInitBlkData;
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc;
+	IMG_UINT32 ui32BlockID, ui32BlkCfgIdx, ui32CounterIdx;
+	RGX_HWPERF_CNTBLK_RT_INFO sCntBlkRtInfo;
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: #BlockConfig entries = %d", __func__, ui32CntBlkModelLen));
+
+	/* Initialise the number of blocks in the RGXFWIF_HWPERF_CTL structure.
+	 * This allows Firmware to validate that it has been correctly configured.
+	 */
+	psHWPerfInitDataInt->ui32NumBlocks = RGXHWPerfMaxDefinedBlks(pvDevice);
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	    "HWPerf Block count = %u.",
+	    psHWPerfInitDataInt->ui32NumBlocks);
+#if defined(PDUMP)
+	/* Ensure that we record the BVNC specific ui32NumBlocks in the PDUMP data
+	 * so that when we playback we have the correct value present.
+	 */
+	DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+	    (size_t)&(psHWPerfInitDataInt->ui32NumBlocks) - (size_t)(psHWPerfInitDataInt),
+	    psHWPerfInitDataInt->ui32NumBlocks, PDUMP_FLAGS_CONTINUOUS);
+#endif	/* defined(PDUMP) */
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "HWPerf Counter Config starts here.");
+
+	/* Simply iterate over all the RGXFWIW_HWPERF_CTL blocks in order */
+	psHWPerfInitBlkData = &psHWPerfInitDataInt->sBlkCfg[0];
+
+	for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen;
+	     ui32BlkCfgIdx++, psHWPerfInitBlkData++)
+	{
+		IMG_UINT32 uiUnit;
+		IMG_BOOL bDirect;
+		IMG_BOOL bSingleton;
+
+		/* Exit early if this core does not have any of these counter blocks
+		 * due to core type/BVNC features.... */
+		psBlkTypeDesc = &asCntBlkTypeModel[ui32BlkCfgIdx];
+
+		if (psBlkTypeDesc == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected NULL - Index %d / %d",
+			    __func__, ui32BlkCfgIdx, ui32CntBlkModelLen));
+			continue;
+		}
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+		        "%s: CfgIdx = %u, InitBlkData @ 0x%p, BlkTypeDesc @ 0x%p",
+		        __func__, ui32BlkCfgIdx, psHWPerfInitBlkData, psBlkTypeDesc));
+
+		if (psBlkTypeDesc->pfnIsBlkPresent(psBlkTypeDesc, pvDevice, &sCntBlkRtInfo) == IMG_FALSE)
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: %s [ID 0x%x] NOT present", __func__,
+			    psBlkTypeDesc->pszBlockNameComment,
+			    psBlkTypeDesc->uiCntBlkIdBase ));
+			/* Block isn't present, but has an entry in the table. Populate
+			 * the Init data so that we can track the block later.
+			 */
+			psHWPerfInitBlkData->uiBlockID = psBlkTypeDesc->uiCntBlkIdBase;
+			continue;
+		}
+#ifdef DEBUG
+		else
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: %s has %d %s", __func__,
+				psBlkTypeDesc->pszBlockNameComment, sCntBlkRtInfo.uiNumUnits,
+			    (sCntBlkRtInfo.uiNumUnits > 1) ? "units" : "unit"));
+		}
+#endif	/* DEBUG */
+
+		/* Program all counters in one block so those already on may
+		 * be configured off and vice-versa. */
+		bDirect = psBlkTypeDesc->uiIndirectReg == 0;
+
+		/* Set if there is only one instance of this block-ID present */
+		bSingleton = sCntBlkRtInfo.uiNumUnits == 1;
+
+		for (ui32BlockID = psBlkTypeDesc->uiCntBlkIdBase, uiUnit = 0;
+		     ui32BlockID < psBlkTypeDesc->uiCntBlkIdBase+sCntBlkRtInfo.uiNumUnits;
+		     ui32BlockID++, uiUnit++)
+		{
+
+			if (bDirect)
+			{
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+				    "Block : %s", psBlkTypeDesc->pszBlockNameComment);
+			}
+			else
+			{
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+				    PDUMP_FLAGS_CONTINUOUS,
+				    "Unit %d Block : %s%d",
+				    ui32BlockID-psBlkTypeDesc->uiCntBlkIdBase,
+				    psBlkTypeDesc->pszBlockNameComment, uiUnit);
+			}
+
+			psHWPerfInitBlkData->uiBlockID = ui32BlockID;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			    "uiBlockID: The Block ID for the layout block. See RGX_CNTBLK_ID for further information.");
+#if defined(PDUMP)
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->uiBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->uiBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			psHWPerfInitBlkData->uiNumCounters = 0;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			    "uiNumCounters (X): Specifies the number of valid counters"
+			    " [0..%d] which follow.", RGX_CNTBLK_COUNTERS_MAX);
+#if defined(PDUMP)
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+			    (size_t)&(psHWPerfInitBlkData->uiNumCounters) - (size_t)(psHWPerfInitDataInt),
+			    psHWPerfInitBlkData->uiNumCounters,
+			    PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			psHWPerfInitBlkData->uiEnabled = 0;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			    "uiEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+#if defined(PDUMP)
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+			    (size_t)&(psHWPerfInitBlkData->uiEnabled) - (size_t)(psHWPerfInitDataInt),
+			    psHWPerfInitBlkData->uiEnabled,
+			    PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			for (ui32CounterIdx = 0; ui32CounterIdx < RGX_CNTBLK_COUNTERS_MAX; ui32CounterIdx++)
+			{
+				psHWPerfInitBlkData->aui32CounterCfg[ui32CounterIdx] = IMG_UINT32_C(0x00000000);
+
+				if (bDirect)
+				{
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					    PDUMP_FLAGS_CONTINUOUS,
+					    "%s_COUNTER_%d",
+					    psBlkTypeDesc->pszBlockNameComment, ui32CounterIdx);
+				}
+				else
+				{
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					    PDUMP_FLAGS_CONTINUOUS,
+					    "%s%d_COUNTER_%d",
+					    psBlkTypeDesc->pszBlockNameComment,
+					    uiUnit, ui32CounterIdx);
+				}
+#if defined(PDUMP)
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+				    (size_t)&(psHWPerfInitBlkData->aui32CounterCfg[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+				    psHWPerfInitBlkData->aui32CounterCfg[ui32CounterIdx],
+				    PDUMP_FLAGS_CONTINUOUS);
+#endif	/* PDUMP */
+
+			}
+
+			/* Update our block reference for indirect units which have more
+			 * than a single unit present. Only increment if we have more than
+			 * one unit left to process as the external loop counter will be
+			 * incremented after final unit is processed.
+			 */
+			if (!bSingleton && (uiUnit < (sCntBlkRtInfo.uiNumUnits - 1)))
+			{
+				psHWPerfInitBlkData++;
+			}
+		}
+	}
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "HWPerf Counter config finishes here.");
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitialiseAllCounters
+
+ @Description   Initialise HWPerf and custom counters
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitialiseAllCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	RGXFWIF_HWPERF_CTL *psHWPerfInitData;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc, (void **)&psHWPerfInitData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", failHWPerfCountersMemDescAqCpuVirt);
+
+	InitialiseHWPerfCounters(psDeviceNode, psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc, psHWPerfInitData);
+	RGXFwSharedMemCacheOpPtr(psHWPerfInitData, FLUSH);
+
+failHWPerfCountersMemDescAqCpuVirt:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+
+	return eError;
+}
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR RGXValidateTDHeap(PVRSRV_DEVICE_NODE *psDeviceNode,
+									  PVRSRV_PHYS_HEAP ePhysHeap,
+									  PHYS_HEAP_USAGE_FLAGS ui32RequiredFlags)
+{
+	PHYS_HEAP *psHeap = psDeviceNode->apsPhysHeap[ePhysHeap];
+	PHYS_HEAP_USAGE_FLAGS ui32HeapFlags = PhysHeapGetFlags(psHeap);
+	PHYS_HEAP_USAGE_FLAGS ui32InvalidFlags = ~(PHYS_HEAP_USAGE_FW_PRIV_DATA | PHYS_HEAP_USAGE_FW_CODE
+											   | PHYS_HEAP_USAGE_GPU_SECURE | PHYS_HEAP_USAGE_FW_PRIVATE);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32RequiredFlags) != 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap is missing required flags. flags: 0x%x / required:0x%x",
+							   ui32HeapFlags,
+							   ui32RequiredFlags);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32InvalidFlags) == 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap uses invalid flags. flags: 0x%x / invalid:0x%x",
+							   ui32HeapFlags,
+							   ui32InvalidFlags);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXValidateTDHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_PRIV_DATA, PHYS_HEAP_USAGE_FW_PRIV_DATA);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_PRIV_DATA");
+
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_CODE, PHYS_HEAP_USAGE_FW_CODE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_CODE");
+	}
+
+	eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_GPU_SECURE, PHYS_HEAP_USAGE_GPU_SECURE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:GPU_SECURE");
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInit
+
+ @Description   RGX Initialisation
+
+ @Input         psDeviceNode
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Services initialisation parameters */
+	RGX_INIT_APPHINTS sApphints = {0};
+	IMG_UINT32 ui32FWConfigFlags, ui32FWConfigFlagsExt, ui32FwOsCfgFlags;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 1");
+
+	PDUMPCOMMENT(psDeviceNode, "Device Name: %s",
+	             psDeviceNode->psDevConfig->pszName);
+	PDUMPCOMMENT(psDeviceNode, "Device ID: %u (%d)",
+	             psDeviceNode->sDevId.ui32InternalID,
+	             psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->psDevConfig->pszVersion)
+	{
+		PDUMPCOMMENT(psDeviceNode, "Device Version: %s",
+		             psDeviceNode->psDevConfig->pszVersion);
+	}
+
+	/* pdump info about the core */
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGX Version Information (KM): %d.%d.%d.%d",
+	             psDevInfo->sDevFeatureCfg.ui32B,
+	             psDevInfo->sDevFeatureCfg.ui32V,
+	             psDevInfo->sDevFeatureCfg.ui32N,
+	             psDevInfo->sDevFeatureCfg.ui32C);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	eError = RGXValidateTDHeaps(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeaps");
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		/* The RGX_CR_MTS_DM4_INTERRUPT_ENABLE register is always set by the firmware during initialisation
+		 * and it provides a good method of determining if the firmware has been booted previously */
+		psDeviceNode->bAutoVzFwIsUp = (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_DM4_INTERRUPT_ENABLE) != 0);
+
+		PVR_LOG(("AutoVz startup check: firmware is %s;",
+				(psDeviceNode->bAutoVzFwIsUp) ? "already running" : "powered down"));
+		PVR_LOG(("AutoVz allow GPU powerdown is %s:",
+				(psDeviceNode->bAutoVzAllowGPUPowerdown) ? "enabled" : "disabled"));
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest assumes the firmware is always available */
+		psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+		psDeviceNode->bAutoVzAllowGPUPowerdown = IMG_FALSE;
+	}
+	else
+#endif
+	{
+		/* Firmware does not follow the AutoVz life-cycle */
+		psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+	{
+		/* set the device power state here as the regular power
+		 * callbacks will not be executed on this driver */
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+
+	/* Set which HW Safety Events will be handled by the driver */
+	psDevInfo->ui32HostSafetyEventMask |= RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER) ?
+										  RGX_CR_EVENT_STATUS__AXT_IF_EQ2_AND_PIPEDM_EQ0__WDT_TIMEOUT_EN : 0;
+	psDevInfo->ui32HostSafetyEventMask |= (RGX_DEVICE_HAS_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS)
+										   && (RGX_DEVICE_GET_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS) > 0)) ?
+										  RGX_CR_EVENT_STATUS_FAULT_FW_EN : 0;
+	psDevInfo->ui32HostSafetyEventMask |= RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_DUAL_LOCKSTEP) ?
+										  RGX_CR_EVENT_STATUS_FAULT_FW_EN : 0;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Register defs revision: %d", RGX_CR_DEFS_KM_REVISION);
+#endif
+
+	/* Services initialisation parameters */
+	GetApphints(psDevInfo, &sApphints);
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	if ((sApphints.bEnableTrustedDeviceAceConfig) &&
+		(RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACE)))
+	{
+		SetTrustedDeviceAceEnabled(psDeviceNode->psDevConfig->hSysData);
+	}
+#endif
+#endif
+
+	eError = RGXInitCreateFWKernelMemoryContext(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create FW kernel memory context (%u)",
+		         __func__, eError));
+		goto cleanup;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitFirmware(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitFirmware failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Setup Firmware initialisation data
+	 */
+
+	GetFWConfigFlags(psDeviceNode, &sApphints, &ui32FWConfigFlags, &ui32FWConfigFlagsExt, &ui32FwOsCfgFlags);
+
+	eError = RGXInitFirmware(psDeviceNode,
+	                         &sApphints,
+	                         ui32FWConfigFlags,
+	                         ui32FWConfigFlagsExt,
+	                         ui32FwOsCfgFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitFirmware failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitialiseAllCounters(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitialiseAllCounters failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Perform second stage of RGX initialisation
+	 */
+	eError = RGXInitDevPart2(psDeviceNode,
+	                         &sApphints);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitDevPart2 failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+
+	eError = PVRSRV_OK;
+
+cleanup:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxsrvinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxstartstop.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxstartstop.c
new file mode 100644
index 000000000000..042138268df0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxstartstop.c
@@ -0,0 +1,1331 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific start/stop routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific start/stop routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxstartstop.h"
+
+
+
+/*
+	RGXWriteMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAT;
+	}
+
+	eError = RGXPollReg32(hPrivate, ui32StateReg, ui32StateReadyFlag, ui32StateReadyFlag);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Write */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+		RGXWriteReg32(hPrivate, ui32DataReg, ui32RegValue);
+		(void) RGXReadReg32(hPrivate, ui32DataReg); /* Fence write */
+	}
+
+	return eError;
+}
+
+/*
+	RGXReadMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN);
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN);
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAX;
+		BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	}
+
+	/* Wait for Slave Port to be Ready */
+	eError = RGXPollReg32(hPrivate,
+						  ui32StateReg,
+						  ui32StateReadyFlag,
+						  ui32StateReadyFlag);
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Read */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+
+		/* Wait for Slave Port to be Ready */
+		eError = RGXPollReg32(hPrivate,
+				  ui32StateReg,
+				  ui32StateReadyFlag,
+				  ui32StateReadyFlag);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+#if !defined(NO_HARDWARE)
+	*ui32RegValue = RGXReadReg32(hPrivate, ui32DataReg);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32DataReg);
+	*ui32RegValue = 0xFFFFFFFF;
+#endif
+
+	return eError;
+}
+
+/*
+ * Specific fields for RGX_CR_IDLE must not be polled in pdumps
+ * (technical reasons)
+ */
+#define CR_IDLE_UNSELECTED_MASK ((~RGX_CR_SLC_IDLE_ACE_CONVERTERS_CLRMSK) | \
+								 (~RGX_CR_SLC_IDLE_OWDB_CLRMSK) |		\
+								 (RGX_CR_SLC_IDLE_FBCDC_ARB_EN))
+
+static PVRSRV_ERROR RGXWriteMetaCoreRegThoughSP(const void *hPrivate,
+                                                IMG_UINT32 ui32CoreReg,
+                                                IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 i = 0;
+
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXDT_OFFSET, ui32Value);
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, ui32CoreReg & ~META_CR_TXUXXRXRQ_RDnWR_BIT);
+
+	do
+	{
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, &ui32Value);
+	} while (((ui32Value & META_CR_TXUXXRXRQ_DREADY_BIT) != META_CR_TXUXXRXRQ_DREADY_BIT) && (i++ < 1000));
+
+	if (i == 1000)
+	{
+		RGXCommentLog(hPrivate, "RGXWriteMetaCoreRegThoughSP: Timeout");
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXStartFirmware(const void *hPrivate)
+{
+	PVRSRV_ERROR eError;
+
+	/* Give privilege to debug and slave port */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_SYSC_JTAG_THREAD, META_CR_SYSC_JTAG_THREAD_PRIV_EN);
+
+	/* Point Meta to the bootloader address, global (uncached) range */
+	eError = RGXWriteMetaCoreRegThoughSP(hPrivate,
+	                                     PC_ACCESS(0),
+	                                     RGXFW_BOOTLDR_META_ADDR | META_MEM_GLOBAL_RANGE_BIT);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start failed!");
+		return eError;
+	}
+
+	/* Enable minim encoding */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXPRIVEXT, META_CR_TXPRIVEXT_MINIM_EN);
+
+	/* Enable Meta thread */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_T0ENABLE_OFFSET, META_CR_TXENABLE_ENABLE_BIT);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMetaProcWrapper
+
+ @Description   Configures the hardware wrapper of the META processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMetaProcWrapper(const void *hPrivate)
+{
+	IMG_UINT64 ui64GartenConfig;
+
+	/* Garten IDLE bit controlled by META */
+	ui64GartenConfig = RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META;
+
+#if defined(RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT)
+	/* Set the Garten Wrapper BIF Fence address */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, S7_TOP_INFRASTRUCTURE))
+	{
+		/* Set PC = 0 for fences */
+		ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_CLRMSK;
+		ui64GartenConfig |= (IMG_UINT64)MMU_CONTEXT_MAPPING_FWPRIV
+		                    << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT;
+
+	}
+	else
+	{
+		/* Set PC = 0 for fences */
+		ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK;
+		ui64GartenConfig |= (IMG_UINT64)MMU_CONTEXT_MAPPING_FWPRIV
+		                    << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT;
+
+		/* Set SLC DM=META */
+		ui64GartenConfig |= ((IMG_UINT64) RGXFW_SEGMMU_META_BIFDM_ID) << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT;
+	}
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure META wrapper");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, ui64GartenConfig);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMipsProcWrapper
+
+ @Description   Configures the hardware wrapper of the MIPS processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMipsProcWrapper(const void *hPrivate)
+{
+	IMG_DEV_PHYADDR sPhyAddr;
+	IMG_UINT64 ui64RemapSettings = RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE; /* Same for all remap registers */
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure MIPS wrapper");
+
+	/*
+	 * MIPS wrapper (registers transaction ID and ISA mode) setup
+	 */
+
+	RGXCommentLog(hPrivate, "RGXStart: Write wrapper config register");
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		RGXWriteReg32(hPrivate,
+		              RGX_CR_MIPS_WRAPPER_CONFIG,
+		              (RGXMIPSFW_REGISTERS_VIRTUAL_BASE >>
+		              RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN) |
+		              RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+	else
+	{
+		RGXAcquireGPURegsAddr(hPrivate, &sPhyAddr);
+
+		RGXMIPSWrapperConfig(hPrivate,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG,
+		                     sPhyAddr.uiAddr,
+		                     RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+
+	/*
+	 * Boot remap setup
+	 */
+
+	RGXAcquireBootRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write boot remap registers");
+	RGXBootRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG1,
+	                   RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+#if defined(FIX_HW_BRN_63553_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 63553))
+	{
+		IMG_BOOL bPhysBusAbove32Bit = RGXGetDevicePhysBusWidth(hPrivate) > 32;
+		IMG_BOOL bDevicePA0IsValid  = RGXDevicePA0IsValid(hPrivate);
+
+		/* WA always required on 36 bit cores, to avoid continuous unmapped memory accesses to address 0x0 */
+		if (bPhysBusAbove32Bit || !bDevicePA0IsValid)
+		{
+			RGXCodeRemapConfig(hPrivate,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG1,
+					0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG2,
+					sPhyAddr.uiAddr,
+					~RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK,
+					ui64RemapSettings);
+		}
+	}
+#endif
+
+	/*
+	 * Data remap setup
+	 */
+
+	RGXAcquireDataRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		/* Remapped private data in secure memory */
+		ui64RemapSettings |= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN;
+	}
+	else
+	{
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+	}
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+	RGXCommentLog(hPrivate, "RGXStart: Write data remap registers");
+	RGXDataRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG1,
+	                   RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	/*
+	 * Code remap setup
+	 */
+
+	RGXAcquireCodeRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write exceptions remap registers");
+	RGXCodeRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG1,
+	                   RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) == 32)
+	{
+		/*
+		 * Trampoline remap setup
+		 */
+
+		RGXAcquireTrampolineRemapAddr(hPrivate, &sPhyAddr);
+		ui64RemapSettings = RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+		ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+		RGXCommentLog(hPrivate, "RGXStart: Write trampoline remap registers");
+		RGXTrampolineRemapConfig(hPrivate,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG1,
+		                         sPhyAddr.uiAddr | RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG2,
+		                         RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+		                         ~RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK,
+		                         ui64RemapSettings);
+	}
+
+	/* Garten IDLE bit controlled by MIPS */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to MIPS");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+
+	/* Turn on the EJTAG probe (only useful driver live) */
+	RGXWriteReg32(hPrivate, RGX_CR_MIPS_DEBUG_CONFIG, 0);
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitRiscvProcWrapper
+
+ @Description   Configures the hardware wrapper of the RISCV processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitRiscvProcWrapper(const void *hPrivate)
+{
+	IMG_UINT32 ui32BootCodeRemap = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+	IMG_UINT32 ui32BootDataRemap = RGXRISCVFW_BOOTLDR_DATA_REMAP;
+	IMG_DEV_VIRTADDR sTmp;
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure RISCV wrapper");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) < 4)
+#endif
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Write boot code remap");
+		RGXAcquireBootCodeAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootCodeRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN);
+
+		RGXCommentLog(hPrivate, "RGXStart: Write boot data remap");
+		RGXAcquireBootDataAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootDataRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN)
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN |
+#endif
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN);
+	}
+
+	/* Garten IDLE bit controlled by RISCV */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to RISCV");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+}
+
+
+static void RGXWriteKernelCatBase(const void *hPrivate, IMG_DEV_PHYADDR sPCAddr)
+{
+	IMG_UINT32 uiPCAddr;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+	{
+		IMG_UINT32 ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+
+		uiPCAddr = (((sPCAddr.uiAddr >> RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT)
+		             << RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT)
+		            & ~RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK);
+
+		/* Set the mapping context */
+		RGXWriteReg32(hPrivate, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(hPrivate, ui32CBaseMapCtxReg); /* Fence write */
+
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+
+#if (MMU_CONTEXT_MAPPING_FWIF != MMU_CONTEXT_MAPPING_FWPRIV)
+		/* Set-up different MMU ID mapping to the same PC used above */
+		RGXWriteReg32(hPrivate, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWIF);
+		(void)RGXReadReg32(hPrivate, ui32CBaseMapCtxReg); /* Fence write */
+
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+#endif
+	}
+#else /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+	if (!RGX_DEVICE_HAS_FEATURE(hPrivate, SLC_VIVT))
+	{
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC64(hPrivate,
+		                      BIF_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT,
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT,
+		                      ((sPCAddr.uiAddr
+		                      >> RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT)
+		                      << RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT)
+		                      & ~RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK);
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Keep catbase registers in sync */
+			RGXWriteKernelMMUPC64(hPrivate,
+			                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+			                      ((sPCAddr.uiAddr
+			                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+			                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+			                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+		}
+
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXWriteKernelCatBase: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+#endif /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+	else
+	{
+		uiPCAddr = (((sPCAddr.uiAddr >> RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT)
+		             << RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT)
+		            & ~RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK);
+
+		/* Set the mapping context */
+		RGXWriteReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT); /* Fence write */
+
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+
+#if (MMU_CONTEXT_MAPPING_FWIF != MMU_CONTEXT_MAPPING_FWPRIV)
+		/* Set-up different MMU ID mapping to the same PC used above */
+		RGXWriteReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT, MMU_CONTEXT_MAPPING_FWIF);
+		(void)RGXReadReg32(hPrivate, RGX_CR_MMU_CBASE_MAPPING_CONTEXT); /* Fence write */
+
+		RGXWriteKernelMMUPC32(hPrivate,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      uiPCAddr);
+#endif
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitBIF
+
+ @Description   Initialise RGX BIF
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitBIF(const void *hPrivate)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXInitBIF: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+	else
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+	{
+		IMG_DEV_PHYADDR sPCAddr;
+
+		/*
+		 * Acquire the address of the Kernel Page Catalogue.
+		 */
+		RGXAcquireKernelMMUPC(hPrivate, &sPCAddr);
+
+		/*
+		 * Write the kernel catalogue base.
+		 */
+		RGXCommentLog(hPrivate, "RGX firmware MMU Page Catalogue");
+
+		RGXWriteKernelCatBase(hPrivate, sPCAddr);
+	}
+}
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/**************************************************************************/ /*!
+@Function       RGXInitMMURangeRegisters
+@Description    Initialises MMU range registers for Non4K pages.
+@Input          hPrivate           Implementation specific data
+@Return         void
+ */ /**************************************************************************/
+static void RGXInitMMURangeRegisters(const void *hPrivate)
+{
+	IMG_UINT32 ui32RegAddr = RGX_CR_MMU_PAGE_SIZE_RANGE_ONE;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES; ++i, ui32RegAddr += sizeof(IMG_UINT64))
+	{
+		RGXWriteReg64(hPrivate, ui32RegAddr, RGXMMUInitRangeValue(i));
+	}
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitAXIACE
+
+ @Description    Initialise AXI-ACE interface
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitAXIACE(const void *hPrivate)
+{
+	IMG_UINT64 ui64RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	ui32RegAddr = RGX_CR_AXI_ACE_LITE_CONFIGURATION;
+
+	/* Setup AXI-ACE config. Set everything to outer cache */
+	ui64RegVal = (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT) |
+	             (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT)  |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT);
+
+#if defined(FIX_HW_BRN_42321_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 42321))
+	{
+		ui64RegVal |= (((IMG_UINT64) 1) << RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT);
+	}
+#endif
+
+#if defined(FIX_HW_BRN_68186_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 68186))
+	{
+		/* default value for reg_enable_fence_out is zero. Force to 1 to allow core_clk < mem_clk */
+		ui64RegVal |= (IMG_UINT64)1 << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT;
+	}
+#endif
+#else /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+	ui32RegAddr = RGX_CR_ACE_CTRL;
+
+	/**
+	 * The below configuration is only applicable for RGX cores supporting
+	 * ACE/ACE-lite protocol and connected to ACE coherent interconnect.
+	 */
+
+	/**
+	 * Configure AxDomain and AxCache for MMU transactions.
+	 * AxDomain set to non sharable (0x0).
+	 */
+	ui64RegVal = RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE |
+				 RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE;
+
+	/**
+	 * Configure AxCache for PM/MMU transactions.
+	 * Set to same value (i.e WBRWALLOC caching, rgxmmunit.c:RGXDerivePTEProt8)
+	 * as non-coherent PTEs
+	 */
+	ui64RegVal |= (IMG_UINT64_C(0xF)) << RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_SHIFT;
+
+	/**
+	 * Configure AxDomain for non MMU transactions.
+	 */
+	ui64RegVal |= (IMG_UINT64)(RGX_CR_ACE_CTRL_COH_DOMAIN_OUTER_SHAREABLE |
+							   RGX_CR_ACE_CTRL_NON_COH_DOMAIN_NON_SHAREABLE);
+#endif /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+
+	RGXCommentLog(hPrivate, "Init AXI-ACE interface");
+	RGXWriteReg64(hPrivate, ui32RegAddr, ui64RegVal);
+}
+
+static void RGXMercerSoftResetSet(const void *hPrivate, IMG_UINT64 ui32MercerFlags)
+{
+	RGXWriteReg64(hPrivate, RGX_CR_MERCER_SOFT_RESET, ui32MercerFlags & RGX_CR_MERCER_SOFT_RESET_MASKFULL);
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) RGXReadReg64(hPrivate, RGX_CR_MERCER_SOFT_RESET);
+}
+
+static void RGXSPUSoftResetAssert(const void *hPrivate)
+{
+	/* Assert Mercer0 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN);
+	/* Assert Mercer1 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN | RGX_CR_MERCER1_SOFT_RESET_SPU_EN);
+	/* Assert Mercer2 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN | RGX_CR_MERCER1_SOFT_RESET_SPU_EN | RGX_CR_MERCER2_SOFT_RESET_SPU_EN);
+
+	RGXWriteReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET, RGX_CR_SWIFT_SOFT_RESET_MASKFULL);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET);
+
+	RGXWriteReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET, RGX_CR_TEXAS_SOFT_RESET_MASKFULL);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET);
+}
+
+static void RGXSPUSoftResetDeAssert(const void *hPrivate)
+{
+	RGXWriteReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET, 0);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_TEXAS_SOFT_RESET);
+
+
+	RGXWriteReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET, 0);
+	/* Fence the previous write */
+	(void) RGXReadReg32(hPrivate, RGX_CR_SWIFT_SOFT_RESET);
+
+	/* Deassert Mercer2 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN | RGX_CR_MERCER1_SOFT_RESET_SPU_EN);
+	/* Deassert Mercer1 */
+	RGXMercerSoftResetSet(hPrivate, RGX_CR_MERCER0_SOFT_RESET_SPU_EN);
+	/* Deassert Mercer0 */
+	RGXMercerSoftResetSet(hPrivate, 0);
+}
+
+static void RGXResetSequence(const void *hPrivate, const IMG_CHAR *pcRGXFW_PROCESSOR)
+{
+	/* Set RGX in soft-reset */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXCommentLog(hPrivate, "RGXStart: soft reset cpu core");
+		RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 0);
+	}
+
+	RGXCommentLog(hPrivate, "RGXStart: soft reset assert step 1");
+	RGXSPUSoftResetAssert(hPrivate);
+
+	RGXCommentLog(hPrivate, "RGXStart: soft reset assert step 2");
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_SOFT_RESET_JONES_ALL);
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_SOFT_RESET_JONES_ALL | RGX_SOFT_RESET_EXTRA);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	/* Take everything out of reset but the FW processor */
+	RGXCommentLog(hPrivate, "RGXStart: soft reset de-assert step 1 excluding %s", pcRGXFW_PROCESSOR);
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_SOFT_RESET_EXTRA | RGX_CR_SOFT_RESET_GARTEN_EN);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	RGXCommentLog(hPrivate, "RGXStart: soft reset de-assert step 2 excluding %s", pcRGXFW_PROCESSOR);
+	RGXSPUSoftResetDeAssert(hPrivate);
+
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+}
+
+static void DeassertMetaReset(const void *hPrivate)
+{
+	/* Need to wait for at least 32 cycles before taking the FW processor out of reset ... */
+	RGXWaitCycles(hPrivate, 32, 3);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, 0x0);
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	/* ... and afterwards */
+	RGXWaitCycles(hPrivate, 32, 3);
+}
+
+static PVRSRV_ERROR InitJonesECCRAM(const void *hPrivate)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Value;
+	IMG_BOOL bMetaFW = RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, META);
+	IMG_UINT32 ui32Mask;
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, ECC_RAMS) == 0)
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	if (bMetaFW)
+	{
+		/* META must be taken out of reset (without booting) during Coremem initialization. */
+		RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, 0);
+		DeassertMetaReset(hPrivate);
+	}
+
+	/* Clocks must be set to "on" during RAMs initialization. */
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, PIPELINED_DATAMASTERS_VERSION) > 0)
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_ON);
+	}
+	else
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0_ALL_ON);
+	}
+	RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL1, RGX_CR_CLK_CTRL1_ALL_ON);
+	RGXWriteReg32(hPrivate, RGX_CR_CLK_CTRL2, RGX_CR_CLK_CTRL2_ALL_ON);
+
+	if (bMetaFW)
+	{
+		RGXWriteMetaRegThroughSP(hPrivate, META_CR_SYSC_JTAG_THREAD, META_CR_SYSC_JTAG_THREAD_PRIV_EN);
+		RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, META_CR_TXCLKCTRL_ALL_ON);
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, &ui32Value);
+	}
+
+	ui32Mask = bMetaFW ?
+		RGX_CR_JONES_RAM_INIT_KICK_MASKFULL
+		: RGX_CR_JONES_RAM_INIT_KICK_MASKFULL & ~RGX_CR_JONES_RAM_INIT_KICK_GARTEN_EN;
+
+	/* SLC component in CHEST either does not exist or not powered at this point */
+	ui32Mask &= RGX_CR_JONES_RAM_INIT_KICK_SLC_CHEST_CLRMSK;
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, POWER_ISLAND_VERSION) >= 4)
+	{
+		/* FBCDC has components in CHEST which may not be powered at this point */
+		ui32Mask &= RGX_CR_JONES_RAM_INIT_KICK_FBCDC_CLRMSK;
+	}
+
+	RGXWriteReg64(hPrivate, RGX_CR_JONES_RAM_INIT_KICK, ui32Mask);
+	eError = RGXPollReg64(hPrivate, RGX_CR_JONES_RAM_STATUS, ui32Mask, ui32Mask);
+
+	if (bMetaFW)
+	{
+		RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, META_CR_TXCLKCTRL_ALL_AUTO);
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXCLKCTRL, &ui32Value);
+	}
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, PIPELINED_DATAMASTERS_VERSION) > 0)
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0__PIPEDM_GT0__ALL_AUTO);
+	}
+	else
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL0, RGX_CR_CLK_CTRL0_ALL_AUTO);
+	}
+	RGXWriteReg64(hPrivate, RGX_CR_CLK_CTRL1, RGX_CR_CLK_CTRL1_ALL_AUTO);
+	RGXWriteReg32(hPrivate, RGX_CR_CLK_CTRL2, RGX_CR_CLK_CTRL2_ALL_AUTO);
+
+	if (bMetaFW)
+	{
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
+		RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+	}
+
+	return eError;
+}
+
+
+PVRSRV_ERROR RGXStart(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR *pcRGXFW_PROCESSOR;
+	IMG_BOOL bDoFWSlaveBoot = IMG_FALSE;
+	IMG_BOOL bMetaFW = IMG_FALSE;
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_RISCV;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_MIPS;
+	}
+#endif
+	else
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_META;
+		bMetaFW = IMG_TRUE;
+		bDoFWSlaveBoot = RGXDoFWSlaveBoot(hPrivate);
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SYS_BUS_SECURE_RESET))
+	{
+		/* Disable the default sys_bus_secure protection to perform minimal setup */
+		RGXCommentLog(hPrivate, "RGXStart: Disable sys_bus_secure");
+		RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, 0);
+		(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+	}
+
+#if defined(RGX_FEATURE_HYPERVISOR_MMU_BIT_MASK)
+	/* Only bypass HMMU if the module is present */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, HYPERVISOR_MMU))
+	{
+		/* Always set HMMU in bypass mode */
+		RGXWriteReg32(hPrivate, RGX_CR_HMMU_BYPASS, RGX_CR_HMMU_BYPASS_MASKFULL);
+		(void) RGXReadReg32(hPrivate, RGX_CR_HMMU_BYPASS);
+	}
+#endif
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_DUAL_LOCKSTEP))
+	{
+		/* Enable the dual LS mode */
+		RGXCommentLog(hPrivate, "RGXStart: Enable Dual Core Lock Step");
+		RGXWriteReg32(hPrivate, RGX_CR_FIRMWARE_PROCESSOR_LS, (0x3 << RGX_CR_FIRMWARE_PROCESSOR_LS__RDL__ENABLE_SHIFT));
+		(void) RGXReadReg32(hPrivate, RGX_CR_FIRMWARE_PROCESSOR_LS);
+	}
+
+
+	/*!
+	 * Start FW init sequence
+	 */
+	RGXResetSequence(hPrivate, pcRGXFW_PROCESSOR);
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, ECC_RAMS) > 0)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Init Jones ECC RAM");
+		eError = InitJonesECCRAM(hPrivate);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, GPU_MULTICORE_SUPPORT))
+		{
+			/* Set OR reduce for ECC faults to ensure faults are not missed during early boot stages */
+			RGXWriteReg32(hPrivate, RGX_CR_MULTICORE_EVENT_REDUCE, RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_FW_EN | RGX_CR_MULTICORE_EVENT_REDUCE_FAULT_GPU_EN);
+		}
+
+		/* Route fault events to the host */
+		RGXWriteReg32(hPrivate, RGX_CR_EVENT_ENABLE, RGX_CR_EVENT_ENABLE_FAULT_FW_EN);
+	}
+
+	if (bMetaFW)
+	{
+		if (bDoFWSlaveBoot)
+		{
+			/* Configure META to Slave boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Slave boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, 0);
+		}
+		else
+		{
+			/* Configure META to Master boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Master boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, RGX_CR_META_BOOT_MODE_EN);
+		}
+	}
+
+	/*
+	 * Initialise Firmware wrapper
+	 */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXInitRiscvProcWrapper(hPrivate);
+	}
+	else if (bMetaFW)
+	{
+		RGXInitMetaProcWrapper(hPrivate);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		RGXInitMipsProcWrapper(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, MMU_VERSION) >= 4)
+	{
+		/* initialise the MMU range based config registers for Non4K pages */
+		RGXInitMMURangeRegisters(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, AXI_ACELITE))
+#endif
+	{
+		/* We must init the AXI-ACE interface before 1st BIF transaction */
+		RGXInitAXIACE(hPrivate);
+	}
+
+	/*
+	 * Initialise BIF.
+	 */
+	RGXInitBIF(hPrivate);
+
+	RGXSetPoweredState(hPrivate, IMG_TRUE);
+
+	RGXCommentLog(hPrivate, "RGXStart: Take %s out of reset", pcRGXFW_PROCESSOR);
+	DeassertMetaReset(hPrivate);
+
+	if (bMetaFW && bDoFWSlaveBoot)
+	{
+		eError = RGXFabricCoherencyTest(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start");
+		eError = RGXStartFirmware(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+	}
+	else
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Master boot Start");
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Bring Debug Module out of reset */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) >= 4)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_EN);
+			}
+			else
+#endif
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL, RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+			}
+
+			/* Boot the FW */
+			RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 1);
+			RGXWaitCycles(hPrivate, 32, 3);
+		}
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) == 1)
+#endif
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Enable sys_bus_secure");
+		RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, RGX_CR_SYS_BUS_SECURE_ENABLE_EN);
+		(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStop(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMetaFW = RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, META);
+	IMG_UINT32 ui32JonesIdleMask = RGX_CR_JONES_IDLE_MASKFULL^(RGX_CR_JONES_IDLE_AXI2IMG_EN|RGX_CR_JONES_IDLE_SI_AXI2IMG_EN);
+
+	RGXDeviceAckIrq(hPrivate);
+
+	/* Set FW power state OFF to disable LISR handler */
+	RGXSetPoweredState(hPrivate, IMG_FALSE);
+
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper */
+	if (!RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, RAY_TRACING_ARCH) ||
+	    RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, RAY_TRACING_ARCH) < 2)
+	{
+		ui32JonesIdleMask ^= (RGX_CR_JONES_IDLE_ASC_EN|RGX_CR_JONES_IDLE_RCE_EN);
+	}
+
+#if defined(FIX_HW_BRN_74812_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 74812))
+	{
+		/* BRN74812 means the IPP idle signal may not be valid and is therefore ignored... */
+		ui32JonesIdleMask &= RGX_CR_JONES_IDLE_IPP_CLRMSK;
+	}
+#endif
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_JONES_IDLE,
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN),
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN));
+
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Wait for SLC to signal IDLE */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_SLC_IDLE,
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK),
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK));
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Unset MTS DM association with threads */
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL);
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL);
+
+	if (bMetaFW)
+	{
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL);
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL);
+	}
+
+#if defined(PDUMP)
+	if (bMetaFW)
+	{
+		/* Disabling threads is only required for pdumps to stop the fw gracefully */
+
+		/* Disable thread 0 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T0ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Disable thread 1 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T1ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Clear down any irq raised by META (done after disabling the FW
+		 * threads to avoid a race condition).
+		 * This is only really needed for PDumps but we do it anyway driver-live.
+		 */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		/* Wait for the Slave Port to finish all the transactions */
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+		{
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+			else
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+		}
+		else
+#endif
+		{
+			RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS, 0x0);
+			(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS); /* Fence write */
+
+			eError = RGXPollReg32(hPrivate,
+								  RGX_CR_META_SP_MSLVCTRL1,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN);
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+#endif
+
+
+	eError = RGXPollReg64(hPrivate,
+	                      RGX_CR_SLC_STATUS1,
+	                      0,
+	                      (~RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS1_BUS0_OUTSTANDING_WRITES_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS1_BUS1_OUTSTANDING_WRITES_CLRMSK));
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg64(hPrivate,
+	                      RGX_CR_SLC_STATUS2,
+	                      0,
+	                      (~RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_READS_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS2_BUS2_OUTSTANDING_WRITES_CLRMSK |
+	                       ~RGX_CR_SLC_STATUS2_BUS3_OUTSTANDING_WRITES_CLRMSK));
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Wait for SLC to signal IDLE */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_SLC_IDLE,
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK),
+	                      RGX_CR_SLC_IDLE_MASKFULL^(CR_IDLE_UNSELECTED_MASK));
+	if (eError != PVRSRV_OK) return eError;
+
+
+	/* Wait for Jones to signal IDLE except for the Garten Wrapper */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_JONES_IDLE,
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN),
+	                      ui32JonesIdleMask^(RGX_CR_JONES_IDLE_GARTEN_EN|RGX_CR_JONES_IDLE_SOCIF_EN));
+	if (eError != PVRSRV_OK) return eError;
+
+	if (bMetaFW)
+	{
+		IMG_UINT32 ui32RegValue;
+
+		eError = RGXReadMetaRegThroughSP(hPrivate,
+		                                 META_CR_TxVECINT_BHALT,
+		                                 &ui32RegValue);
+		if (eError != PVRSRV_OK) return eError;
+
+		if ((ui32RegValue & 0xFFFFFFFFU) == 0x0)
+		{
+			/* Wait for Sidekick/Jones to signal IDLE including
+			 * the Garten Wrapper if there is no debugger attached
+			 * (TxVECINT_BHALT = 0x0) */
+			eError = RGXPollReg32(hPrivate,
+			                      RGX_CR_JONES_IDLE,
+			                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN,
+			                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+	}
+	else
+	{
+		eError = RGXPollReg32(hPrivate,
+		                      RGX_CR_JONES_IDLE,
+		                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN,
+		                      ui32JonesIdleMask^RGX_CR_JONES_IDLE_SOCIF_EN);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.c
new file mode 100644
index 000000000000..9ebf43429e19
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.c
@@ -0,0 +1,5798 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA/3D routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX TA/3D routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* for the offsetof macro */
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxta3d.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "ri_server.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "rgx_memallocflags.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxsyncutils.h"
+#include "htbserver.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "physmem.h"
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "process_stats.h"
+
+#if !defined(PM_INTERACTIVE_MODE)
+#include "rgxpmdefs.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TA3D_UFO_DUMP	0
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static INLINE
+void _DebugSyncValues(const IMG_CHAR *pszFunction,
+					  const IMG_UINT32 *pui32UpdateValues,
+					  const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", pszFunction, i, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+
+static INLINE
+void _DebugSyncCheckpoints(const IMG_CHAR *pszFunction,
+						   const IMG_CHAR *pszDMName,
+						   const PSYNC_CHECKPOINT *apsSyncCheckpoints,
+						   const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFence%sSyncCheckpoints[%d]=<%p>", pszFunction, pszDMName, i, *(apsSyncCheckpoints + i)));
+	}
+}
+
+#else
+#define CHKPT_DBG(X)
+#endif
+
+/* define the number of commands required to be set up by the CCB helper */
+/* 1 command for the TA */
+#define CCB_CMD_HELPER_NUM_TA_COMMANDS 1
+/* Up to 3 commands for the 3D (partial render fence, partial render, and render) */
+#define CCB_CMD_HELPER_NUM_3D_COMMANDS 3
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define WORKEST_CYCLES_PREDICTION_GET(x) ((x).ui32CyclesPrediction)
+#else
+#define WORKEST_CYCLES_PREDICTION_GET(x) (NO_CYCEST)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_TA_DATA;
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_3D_DATA;
+
+struct _RGX_SERVER_RENDER_CONTEXT_ {
+	/* this lock protects usage of the render context.
+	 * it ensures only one kick is being prepared and/or submitted on
+	 * this render context at any time
+	 */
+	POS_LOCK				hLock;
+	RGX_CCB_CMD_HELPER_DATA asTACmdHelperData[CCB_CMD_HELPER_NUM_TA_COMMANDS];
+	RGX_CCB_CMD_HELPER_DATA as3DCmdHelperData[CCB_CMD_HELPER_NUM_3D_COMMANDS];
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWRenderContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	RGX_SERVER_RC_TA_DATA		sTAData;
+	RGX_SERVER_RC_3D_DATA		s3DData;
+	IMG_UINT32					ui32CleanupStatus;
+#define RC_CLEANUP_TA_COMPLETE		(1 << 0)
+#define RC_CLEANUP_3D_COMPLETE		(1 << 1)
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListTAFence;
+	SYNC_ADDR_LIST				sSyncAddrListTAUpdate;
+	SYNC_ADDR_LIST				sSyncAddrList3DFence;
+	SYNC_ADDR_LIST				sSyncAddrList3DUpdate;
+	ATOMIC_T					hIntJobRef;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+
+/*
+	Static functions used by render context code
+*/
+
+static
+PVRSRV_ERROR _DestroyTAContext(RGX_SERVER_RC_TA_DATA *psTAData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  psTAData->psServerCommonContext,
+											  RGXFWIF_DM_GEOM,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(psTAData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, psTAData->psContextStateMemDesc);
+	psTAData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Destroy3DContext(RGX_SERVER_RC_3D_DATA *ps3DData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psContextStateMemDesc);
+	ps3DData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static void _RGXDumpPMRPageList(DLLIST_NODE *psNode)
+{
+	RGX_PMR_NODE *psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	PVRSRV_ERROR			eError;
+
+	eError = PMRDumpPageList(psPMRNode->psPMR,
+							RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Error (%s) printing pmr %p",
+				PVRSRVGetErrorString(eError),
+				psPMRNode->psPMR));
+	}
+}
+
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_LOG(("Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016" IMG_UINT64_FMTSPECx,
+				psFreeList->sFreeListFWDevVAddr.ui32Addr,
+				psFreeList->ui32FreelistID,
+				psFreeList->ui64FreelistChecksum));
+
+	/* Dump Init FreeList page list */
+	PVR_LOG(("  Initial Memory block"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	/* Dump Grow FreeList page list */
+	PVR_LOG(("  Grow Memory blocks"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	return IMG_TRUE;
+}
+
+static void _CheckFreelist(RGX_FREELIST *psFreeList,
+						   IMG_UINT32 ui32NumOfPagesToCheck,
+						   IMG_UINT64 ui64ExpectedCheckSum,
+						   IMG_UINT64 *pui64CalculatedCheckSum)
+{
+#if defined(NO_HARDWARE)
+	/* No checksum needed as we have all information in the pdumps */
+	PVR_UNREFERENCED_PARAMETER(psFreeList);
+	PVR_UNREFERENCED_PARAMETER(ui32NumOfPagesToCheck);
+	PVR_UNREFERENCED_PARAMETER(ui64ExpectedCheckSum);
+	*pui64CalculatedCheckSum = 0;
+#else
+	PVRSRV_ERROR eError;
+	size_t uiNumBytes;
+	IMG_UINT8* pui8Buffer;
+	IMG_UINT32* pui32Buffer;
+	IMG_UINT32 ui32CheckSumAdd = 0;
+	IMG_UINT32 ui32CheckSumXor = 0;
+	IMG_UINT32 ui32Entry;
+	IMG_UINT32 ui32Entry2;
+	IMG_BOOL bFreelistBad = IMG_FALSE;
+
+	*pui64CalculatedCheckSum = 0;
+
+	PVR_ASSERT(ui32NumOfPagesToCheck <= (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+	/* Allocate Buffer of the size of the freelist */
+	pui8Buffer = OSAllocMem(ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+	if (pui8Buffer == NULL)
+	{
+		PVR_LOG(("%s: Failed to allocate buffer to check freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	/* Copy freelist content into Buffer */
+	eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
+					psFreeList->uiFreeListPMROffset +
+					(((psFreeList->ui32MaxFLPages -
+					   psFreeList->ui32CurrentFLPages - psFreeList->ui32ReadyFLPages) * sizeof(IMG_UINT32)) &
+					 ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1)),
+					 pui8Buffer,
+					 ui32NumOfPagesToCheck * sizeof(IMG_UINT32),
+					&uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pui8Buffer);
+		PVR_LOG(("%s: Failed to get freelist data for freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	PVR_ASSERT(uiNumBytes == ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+
+	/* Generate checksum (skipping the first page if not allocated) */
+	pui32Buffer = (IMG_UINT32 *)pui8Buffer;
+	ui32Entry = ((psFreeList->ui32GrowFLPages == 0  &&  psFreeList->ui32CurrentFLPages > 1) ? 1 : 0);
+	for (/*ui32Entry*/ ; ui32Entry < ui32NumOfPagesToCheck; ui32Entry++)
+	{
+		ui32CheckSumAdd += pui32Buffer[ui32Entry];
+		ui32CheckSumXor ^= pui32Buffer[ui32Entry];
+
+		/* Check for double entries */
+		for (ui32Entry2 = ui32Entry+1; ui32Entry2 < ui32NumOfPagesToCheck; ui32Entry2++)
+		{
+			if (pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2])
+			{
+				PVR_LOG(("%s: Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d of %d",
+						__func__,
+						psFreeList->sFreeListFWDevVAddr.ui32Addr,
+						pui32Buffer[ui32Entry2],
+						ui32Entry,
+						ui32Entry2,
+						psFreeList->ui32CurrentFLPages));
+				bFreelistBad = IMG_TRUE;
+				break;
+			}
+		}
+	}
+
+	OSFreeMem(pui8Buffer);
+
+	/* Check the calculated checksum against the expected checksum... */
+	*pui64CalculatedCheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
+
+	if (ui64ExpectedCheckSum != 0  &&  ui64ExpectedCheckSum != *pui64CalculatedCheckSum)
+	{
+		PVR_LOG(("%s: Checksum mismatch for freelist %p! Expected 0x%016" IMG_UINT64_FMTSPECx " calculated 0x%016" IMG_UINT64_FMTSPECx,
+				__func__, psFreeList,
+				ui64ExpectedCheckSum, *pui64CalculatedCheckSum));
+		bFreelistBad = IMG_TRUE;
+	}
+
+	if (bFreelistBad)
+	{
+		PVR_LOG(("%s: Sleeping for ever!", __func__));
+		PVR_ASSERT(!bFreelistBad);
+	}
+#endif
+}
+
+
+/*
+ *  Function to work out the number of freelist pages to reserve for growing
+ *  within the FW without having to wait for the host to progress a grow
+ *  request.
+ *
+ *  The number of pages must be a multiple of 4 to align the PM addresses
+ *  for the initial freelist allocation and also be less than the grow size.
+ *
+ *  If the threshold or grow size means less than 4 pages, then the feature
+ *  is not used.
+ */
+static IMG_UINT32 _CalculateFreelistReadyPages(RGX_FREELIST *psFreeList,
+                                               IMG_UINT32  ui32FLPages)
+{
+	IMG_UINT32  ui32ReadyFLPages = ((ui32FLPages * psFreeList->ui32GrowThreshold) / 100) &
+	                               ~((RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE/sizeof(IMG_UINT32))-1);
+
+	if (ui32ReadyFLPages > psFreeList->ui32GrowFLPages)
+	{
+		ui32ReadyFLPages = psFreeList->ui32GrowFLPages;
+	}
+
+	return ui32ReadyFLPages;
+}
+
+
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader)
+{
+	RGX_PMR_NODE	*psPMRNode;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32  ui32MappingTable = 0;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_DEVMEM_SIZE_T uistartPage;
+	PVRSRV_ERROR eError;
+	static const IMG_CHAR szAllocName[] = "Free List";
+
+	/* Are we allowed to grow ? */
+	if (psFreeList->ui32MaxFLPages - (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) < ui32NumPages)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: grow by %u pages denied. "
+				"Max PB size reached (current pages %u+%u/%u)",
+				psFreeList,
+				ui32NumPages,
+				psFreeList->ui32CurrentFLPages,
+				psFreeList->ui32ReadyFLPages,
+				psFreeList->ui32MaxFLPages));
+		return PVRSRV_ERROR_PBSIZE_ALREADY_MAX;
+	}
+
+	/* Allocate kernel memory block structure */
+	psPMRNode = OSAllocMem(sizeof(*psPMRNode));
+	if (psPMRNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to allocate host data structure",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocHost;
+	}
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages
+	 */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+
+	/*
+	 *  The PM never takes the last page in a freelist, so if this block
+	 *  of pages is the first one and there is no ability to grow, then
+	 *  we can skip allocating one 4K page for the lowest entry.
+	 */
+	if (OSGetPageSize() > RGX_BIF_PM_PHYSICAL_PAGE_SIZE)
+	{
+		/*
+		 * Allocation size will be rounded up to the OS page size,
+		 * any attempt to change it a bit now will be invalidated later.
+		 */
+		psPMRNode->bFirstPageMissing = IMG_FALSE;
+	}
+	else
+	{
+		psPMRNode->bFirstPageMissing = (psFreeList->ui32GrowFLPages == 0  &&  ui32NumPages > 1);
+	}
+
+	psPMRNode->ui32NumPages = ui32NumPages;
+	psPMRNode->psFreeList = psFreeList;
+
+	/* Allocate Memory Block */
+	PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Allocate PB Block (Pages %08X)", ui32NumPages);
+	uiSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	if (psPMRNode->bFirstPageMissing)
+	{
+		uiSize -= RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	}
+
+	eError = PhysmemNewRamBackedPMR(psFreeList->psConnection,
+			psFreeList->psDevInfo->psDeviceNode,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+			PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+			PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_PRIVATE) |
+			PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP,
+			sizeof(szAllocName),
+			szAllocName,
+			psFreeList->ownerPid,
+			&psPMRNode->psPMR,
+			PDUMP_NONE,
+			NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate PB block of size: 0x%016" IMG_UINT64_FMTSPECX,
+				 __func__,
+				 (IMG_UINT64)uiSize));
+		goto ErrorBlockAlloc;
+	}
+
+	/* Zeroing physical pages pointed by the PMR */
+	if (psFreeList->psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	uistartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((uistartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRNode->psPMR,
+	                                    psFreeList->ownerPid);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: call to RIWritePMREntryWithOwnerKM failed (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+	}
+
+	 /* Attach RI information */
+	eError = RIWriteMEMDESCEntryKM(psFreeList->psConnection,
+	                               psFreeList->psDevInfo->psDeviceNode,
+	                               psPMRNode->psPMR,
+	                               OSStringNLength(szAllocName, DEVMEM_ANNOTATION_MAX_LEN),
+	                               szAllocName,
+	                               0,
+	                               uiSize,
+	                               0,
+	                               &psPMRNode->hRIHandle);
+	PVR_LOG_IF_ERROR(eError, "RIWriteMEMDESCEntryKM");
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	/* write Freelist with Memory Block physical addresses */
+	eError = PMRWritePMPageList(
+						/* Target PMR, offset, and length */
+						psFreeList->psFreeListPMR,
+						(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+						(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+						/* Referenced PMR, and "page" granularity */
+						psPMRNode->psPMR,
+						RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+						&psPMRNode->psPageList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to write pages of Node %p",
+				 __func__,
+				 psPMRNode));
+		goto ErrorPopulateFreelist;
+	}
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+	/* Copy freelist memory to shadow freelist */
+	{
+		const IMG_UINT32 ui32FLMaxSize = psFreeList->ui32MaxFLPages * sizeof(IMG_UINT32);
+		const IMG_UINT32 ui32MapSize = ui32FLMaxSize * 2;
+		const IMG_UINT32 ui32CopyOffset = uiOffset - psFreeList->uiFreeListPMROffset;
+		IMG_BYTE *pFLMapAddr;
+		size_t uiNumBytes;
+		PVRSRV_ERROR res;
+		IMG_HANDLE hMapHandle;
+
+		/* Check for overflow. Validate size and offset. */
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize > psFreeList->uiFreeListPMROffset,
+		                          eError, ErrorPopulateFreelist);
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize <= PMR_LogicalSize(psFreeList->psFreeListPMR),
+		                          eError, ErrorPopulateFreelist);
+
+		/* Map both the FL and the shadow FL */
+		res = PMRAcquireKernelMappingData(psFreeList->psFreeListPMR, psFreeList->uiFreeListPMROffset, ui32MapSize,
+		                                  (void**) &pFLMapAddr, &uiNumBytes, &hMapHandle);
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map freelist (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+
+		/* Copy only the newly added memory */
+		OSCachedMemCopy(pFLMapAddr + ui32FLMaxSize + ui32CopyOffset, pFLMapAddr + ui32CopyOffset , uiLength);
+		OSWriteMemoryBarrier(pFLMapAddr);
+
+#if defined(PDUMP)
+		PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Initialize shadow freelist");
+
+		/* Translate memcpy to pdump */
+		{
+			IMG_DEVMEM_OFFSET_T uiCurrOffset;
+
+			for (uiCurrOffset = uiOffset; (uiCurrOffset - uiOffset) < uiLength; uiCurrOffset += sizeof(IMG_UINT32))
+			{
+				PMRPDumpCopyMem32(psFreeList->psFreeListPMR,
+				                  uiCurrOffset + ui32FLMaxSize,
+				                  psFreeList->psFreeListPMR,
+				                  uiCurrOffset,
+				                  ":SYSMEM:$1",
+				                  PDUMP_FLAGS_CONTINUOUS);
+			}
+		}
+#endif
+
+
+		res = PMRReleaseKernelMappingData(psFreeList->psFreeListPMR, hMapHandle);
+
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to release freelist mapping (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto ErrorPopulateFreelist;
+		}
+	}
+#endif
+
+	/* We add It must be added to the tail, otherwise the freelist population won't work */
+	dllist_add_to_head(pListHeader, &psPMRNode->sMemoryBlock);
+
+	/* Update number of available pages */
+	psFreeList->ui32CurrentFLPages += ui32NumPages;
+
+	/* Update statistics (needs to happen before the ReadyFL calculation to also count those pages) */
+	if (psFreeList->ui32NumHighPages < psFreeList->ui32CurrentFLPages)
+	{
+		psFreeList->ui32NumHighPages = psFreeList->ui32CurrentFLPages;
+	}
+
+	/* Reserve a number ready pages to allow the FW to process OOM quickly and asynchronously request a grow. */
+	psFreeList->ui32ReadyFLPages    = _CalculateFreelistReadyPages(psFreeList, psFreeList->ui32CurrentFLPages);
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+
+	if (psFreeList->bCheckFreelist)
+	{
+		/*
+		 *  We can only calculate the freelist checksum when the list is full
+		 *  (e.g. at initial creation time). At other times the checksum cannot
+		 *  be calculated and has to be disabled for this freelist.
+		 */
+		if ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages)
+		{
+			_CheckFreelist(psFreeList, ui32NumPages, 0, &psFreeList->ui64FreelistChecksum);
+		}
+		else
+		{
+			psFreeList->ui64FreelistChecksum = 0;
+		}
+	}
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: %s %u pages (pages=%u+%u/%u checksum=0x%016" IMG_UINT64_FMTSPECx "%s)",
+			psFreeList,
+	        ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages ? "Create initial" : "Grow by"),
+			ui32NumPages,
+			psFreeList->ui32CurrentFLPages,
+			psFreeList->ui32ReadyFLPages,
+			psFreeList->ui32MaxFLPages,
+			psFreeList->ui64FreelistChecksum,
+			(psPMRNode->bFirstPageMissing ? " - lowest page not allocated" : "")));
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+ErrorPopulateFreelist:
+	(void) PMRUnrefPMR(psPMRNode->psPMR);
+
+ErrorBlockAlloc:
+	OSFreeMem(psPMRNode);
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+ErrorAllocHost:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR RGXShrinkFreeList(PDLLIST_NODE pListHeader,
+										RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode;
+	RGX_PMR_NODE *psPMRNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32OldValue;
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages value
+	 */
+	PVR_ASSERT(pListHeader);
+	PVR_ASSERT(psFreeList);
+	PVR_ASSERT(psFreeList->psDevInfo);
+	PVR_ASSERT(psFreeList->psDevInfo->hLockFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+	/* Get node from head of list and remove it */
+	psNode = dllist_get_next_node(pListHeader);
+	if (psNode)
+	{
+		dllist_remove_node(psNode);
+
+		psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+		PVR_ASSERT(psPMRNode);
+		PVR_ASSERT(psPMRNode->psPMR);
+		PVR_ASSERT(psPMRNode->psFreeList);
+
+		/* remove block from freelist list */
+
+		/* Unwrite Freelist with Memory Block physical addresses */
+		eError = PMRUnwritePMPageList(psPMRNode->psPageList);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to unwrite pages of Node %p",
+					 __func__,
+					 psPMRNode));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+		if (psPMRNode->hRIHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			eError = RIDeleteMEMDESCEntryKM(psPMRNode->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "RIDeleteMEMDESCEntryKM");
+		}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+		/* Free PMR (We should be the only one that holds a ref on the PMR) */
+		eError = PMRUnrefPMR(psPMRNode->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to free PB block %p (%s)",
+					 __func__,
+					 psPMRNode->psPMR,
+					 PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+		/* update available pages in freelist */
+		ui32OldValue = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+
+		/*
+		 * Deallocated pages should first be deducted from ReadyPages bank, once
+		 * there are no more left, start deducting them from CurrentPage bank.
+		 */
+		if (psPMRNode->ui32NumPages > psFreeList->ui32ReadyFLPages)
+		{
+			psFreeList->ui32CurrentFLPages -= psPMRNode->ui32NumPages - psFreeList->ui32ReadyFLPages;
+			psFreeList->ui32ReadyFLPages = 0;
+		}
+		else
+		{
+			psFreeList->ui32ReadyFLPages -= psPMRNode->ui32NumPages;
+		}
+
+		/* check underflow */
+		PVR_ASSERT(ui32OldValue > (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: shrink by %u pages (current pages %u/%u)",
+								psFreeList,
+								psPMRNode->ui32NumPages,
+								psFreeList->ui32CurrentFLPages,
+								psFreeList->ui32MaxFLPages));
+
+		OSFreeMem(psPMRNode);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: shrink denied. PB already at initial PB size (%u pages)",
+								psFreeList,
+								psFreeList->ui32InitFLPages));
+		eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
+	}
+
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	return eError;
+}
+
+static RGX_FREELIST *FindFreeList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FreelistID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_FREELIST *psFreeList = NULL;
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psThisFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		if (psThisFreeList->ui32FreelistID == ui32FreelistID)
+		{
+			psFreeList = psThisFreeList;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevInfo->hLockFreeList);
+	return psFreeList;
+}
+
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT32 ui32FreelistID)
+{
+	RGX_FREELIST *psFreeList = NULL;
+	RGXFWIF_KCCB_CMD s3DCCBCmd;
+	IMG_UINT32 ui32GrowValue;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	psFreeList = FindFreeList(psDevInfo, ui32FreelistID);
+	if (psFreeList == NULL)
+	{
+		/* Should never happen */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "FreeList Lookup for FreeList ID 0x%08x failed (Populate)",
+		         ui32FreelistID));
+		PVR_ASSERT(IMG_FALSE);
+		return;
+	}
+
+	/* Since the FW made the request, it has already consumed the ready pages, update the host struct */
+	psFreeList->ui32CurrentFLPages += psFreeList->ui32ReadyFLPages;
+	psFreeList->ui32ReadyFLPages = 0;
+
+	/* Try to grow the freelist */
+	eError = RGXGrowFreeList(psFreeList,
+	                         psFreeList->ui32GrowFLPages,
+	                         &psFreeList->sMemoryBlockHead);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Grow successful, return size of grow size */
+		ui32GrowValue = psFreeList->ui32GrowFLPages;
+
+		psFreeList->ui32NumGrowReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		/* Update Stats */
+		PVRSRVStatsUpdateFreelistStats(psDevInfo->psDeviceNode,
+		                               0,
+		                               1, /* Add 1 to the appropriate counter (Requests by FW) */
+		                               psFreeList->ui32InitFLPages,
+		                               psFreeList->ui32NumHighPages,
+		                               psFreeList->ownerPid);
+
+#endif
+
+	}
+	else
+	{
+		/* Grow failed */
+		ui32GrowValue = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Grow for FreeList %p failed (%s)",
+		         psFreeList,
+		         PVRSRVGetErrorString(eError)));
+	}
+
+	/* send feedback */
+	s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE;
+	s3DCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32DeltaPages = ui32GrowValue;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages = psFreeList->ui32ReadyFLPages;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: Grow pages=%u, new pages=%u, ready pages=%u, counter=%d",
+	         psFreeList,
+	         ui32GrowValue,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages,
+	         psFreeList->ui32NumGrowReqByFW));
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_3D,
+		                            &s3DCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+static void _RGXFreeListReconstruction(PDLLIST_NODE psNode)
+{
+
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	RGX_FREELIST			*psFreeList;
+	RGX_PMR_NODE			*psPMRNode;
+	PVRSRV_ERROR			eError;
+	IMG_DEVMEM_OFFSET_T		uiOffset;
+	IMG_DEVMEM_SIZE_T		uiLength;
+	IMG_UINT32				ui32StartPage;
+
+	psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	psFreeList = psPMRNode->psFreeList;
+	PVR_ASSERT(psFreeList);
+	psDevInfo = psFreeList->psDevInfo;
+	PVR_ASSERT(psDevInfo);
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	ui32StartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((ui32StartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+	PMRUnwritePMPageList(psPMRNode->psPageList);
+	psPMRNode->psPageList = NULL;
+	eError = PMRWritePMPageList(
+			/* Target PMR, offset, and length */
+			psFreeList->psFreeListPMR,
+			(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+			(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+			/* Referenced PMR, and "page" granularity */
+			psPMRNode->psPMR,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			&psPMRNode->psPageList);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error (%s) writing FL 0x%08x",
+				__func__,
+				PVRSRVGetErrorString(eError),
+				(IMG_UINT32)psFreeList->ui32FreelistID));
+	}
+
+	/* Zeroing physical pages pointed by the reconstructed freelist */
+	if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	psFreeList->ui32CurrentFLPages += psPMRNode->ui32NumPages;
+}
+
+
+static PVRSRV_ERROR RGXReconstructFreeList(RGX_FREELIST *psFreeList)
+{
+	IMG_UINT32        ui32OriginalFLPages;
+	DLLIST_NODE       *psNode, *psNext;
+	RGXFWIF_FREELIST  *psFWFreeList;
+	PVRSRV_ERROR      eError;
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: Reconstructing freelist %p (ID=%u)", psFreeList, psFreeList->ui32FreelistID));
+
+	/* Do the FreeList Reconstruction */
+	ui32OriginalFLPages            = psFreeList->ui32CurrentFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+
+	/* Reconstructing Init FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Reconstructing Grow FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Ready pages are allocated but kept hidden until OOM occurs. */
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+	if (psFreeList->ui32CurrentFLPages != ui32OriginalFLPages)
+	{
+		PVR_ASSERT(psFreeList->ui32CurrentFLPages == ui32OriginalFLPages);
+		return PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED;
+	}
+
+	/* Update firmware freelist structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psFWFreeList->ui32MaxPages     = psFreeList->ui32MaxFLPages;
+	psFWFreeList->ui32GrowPages    = psFreeList->ui32GrowFLPages;
+	psFWFreeList->ui32CurrentPages = psFreeList->ui32CurrentFLPages;
+	psFWFreeList->ui32FreeListID   = psFreeList->ui32FreelistID;
+	psFWFreeList->bGrowPending     = IMG_FALSE;
+	psFWFreeList->ui32ReadyPages   = psFreeList->ui32ReadyFLPages;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psFWFreeList->ui32CurrentStackTop       = psFWFreeList->ui32CurrentPages - 1;
+	psFWFreeList->ui64CurrentDevVAddr       = (psFWFreeList->psFreeListDevVAddr.uiAddr +
+				((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+				~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+	psFWFreeList->ui32AllocatedPageCount    = 0;
+	psFWFreeList->ui32AllocatedMMUPageCount = 0;
+#else
+	psFWFreeList->bUpdatePending                   = IMG_FALSE;
+	psFWFreeList->ui32UpdateNewPages               = 0;
+	psFWFreeList->ui32UpdateNewReadyPages          = 0;
+	psFWFreeList->sFreeListLastGrowDevVAddr.uiAddr = 0;
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+#if !defined(PM_INTERACTIVE_MODE)
+	/* Reset freelist state buffer */
+	{
+		RGX_PM_FREELISTSTATE_BUFFER		sFLState;
+		size_t uiNbBytes;
+		IMG_DEV_VIRTADDR sFLBaseAddr;
+
+		eError = PMR_ReadBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+
+		sFLBaseAddr.uiAddr = (psFreeList->sFreeListBaseDevVAddr.uiAddr +
+		                      ((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+		                      ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+		/* Note: Freelist base address is packed shifted down. */
+		RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(sFLState, sFLBaseAddr.uiAddr >> RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(sFLState, psFreeList->ui32CurrentFLPages - 1);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(sFLState, 0);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(sFLState, 0);
+
+		eError = PMR_WriteBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+	}
+#endif
+
+	/* Check the Freelist checksum if required (as the list is fully populated) */
+	if (psFreeList->bCheckFreelist)
+	{
+		IMG_UINT64  ui64CheckSum;
+
+		_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+	}
+
+	return eError;
+}
+
+
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                              IMG_UINT32 ui32FreelistsCount,
+                                              const IMG_UINT32 *paui32Freelists)
+{
+	PVRSRV_ERROR      eError = PVRSRV_OK;
+	DLLIST_NODE       *psNode, *psNext;
+	IMG_UINT32        ui32Loop;
+	RGXFWIF_KCCB_CMD  sTACCBCmd;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE       *psNodeHWRTData, *psNextHWRTData;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+	RGXFWIF_HWRTDATA     *psHWRTData;
+#endif
+	IMG_UINT32        ui32PIDCount = 0;
+	IMG_UINT32        aui32PIDList[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+
+	PVR_ASSERT(psDevInfo != NULL);
+	PVR_ASSERT(ui32FreelistsCount <= RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT);
+	if (ui32FreelistsCount > RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT)
+	{
+		ui32FreelistsCount = RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT;
+	}
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: %u freelist(s) requested for reconstruction", ui32FreelistsCount));
+
+	/*
+	 *  Initialise the response command (in case we don't find a freelist ID).
+	 *  Also copy the list to the 'final' freelist array.
+	 */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE;
+	sTACCBCmd.uCmdData.sFreeListsReconstructionData.ui32FreelistsCount = ui32FreelistsCount;
+
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] = paui32Freelists[ui32Loop] |
+				RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+	}
+
+	/*
+	 *  All freelists belonging to the same PID will be reconstructed.
+	 *  This simplifies tracking for AGP, since there is no longer a
+	 *  single global freelist per local freelist. The list of unique
+	 *  PIDs is calculated from the list of freelists in this first pass.
+	 */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList   = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+		{
+			IMG_UINT32 ui32PIDLoop;
+
+			if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+			{
+				for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+				{
+					if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+					{
+						break;
+					}
+				}
+
+				if (ui32PIDLoop == ui32PIDCount)
+				{
+					aui32PIDList[ui32PIDCount++] = psFreeList->ownerPid;
+				}
+			}
+		}
+	}
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList  = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		IMG_BOOL      bReconstruct = IMG_FALSE;
+		IMG_UINT32    ui32PIDLoop;
+
+		/*
+		 *  Check if this freelist needs to be reconstructed (is it in the list
+		 *  of PIDs which will have every single one of their freelists reconstructed)
+		 */
+		for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+		{
+			if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+			{
+				bReconstruct = IMG_TRUE;
+				break;
+			}
+		}
+
+		if (bReconstruct)
+		{
+			eError = RGXReconstructFreeList(psFreeList);
+			if (eError == PVRSRV_OK)
+			{
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+				/* Mark all HWRTData's of reconstructing local freelists as HWR (applies to TA/3D's not finished yet) */
+				dllist_foreach_node(&psFreeList->sNodeHWRTDataHead, psNodeHWRTData, psNextHWRTData)
+				{
+					psKMHWRTDataSet = IMG_CONTAINER_OF(psNodeHWRTData, RGX_KM_HW_RT_DATASET, sNodeHWRTData);
+					eError = DevmemAcquireCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc, (void **)&psHWRTData);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+									"Devmem AcquireCpuVirtAddr Failed during Reconstructing of FreeList, FwMemDesc(%p),psHWRTData(%p)",
+									psKMHWRTDataSet->psHWRTDataFwMemDesc,
+									psHWRTData));
+						continue;
+					}
+
+					psHWRTData->eState = RGXFWIF_RTDATA_STATE_HWR;
+					psHWRTData->ui32HWRTDataFlags &= ~HWRTDATA_HAS_LAST_TA;
+					RGXFwSharedMemCacheOpValue(psHWRTData->eState, FLUSH);
+					RGXFwSharedMemCacheOpValue(psHWRTData->ui32HWRTDataFlags, FLUSH);
+
+					DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+				}
+#endif
+
+				/* Update the response for this freelist if it was specifically requested for reconstruction. */
+				for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+				{
+					if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+					{
+						/* Reconstruction of this requested freelist was successful... */
+						sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &= ~RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+						break;
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Reconstructing of FreeList %p failed (%s)",
+						psFreeList,
+						PVRSRVGetErrorString(eError)));
+			}
+		}
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Check that all freelists were found and reconstructed... */
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		PVR_ASSERT((sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &
+		            RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG) == 0);
+	}
+
+	/* send feedback */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+/* Create a single HWRTData instance */
+static PVRSRV_ERROR RGXCreateHWRTData_aux(CONNECTION_DATA		*psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_DEV_VIRTADDR		psVHeapTableDevVAddr,
+                                          IMG_DEV_VIRTADDR		sPMDataDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR sPMSecureDataDevVAddr, /* per-HWRTData */
+                                          RGX_FREELIST			*apsFreeLists[RGXFW_MAX_FREELISTS],
+                                          IMG_DEV_VIRTADDR		sTailPtrsDevVAddr,
+                                          IMG_UINT16			ui16MaxRTs,
+                                          RGX_HWRTDATA_COMMON_COOKIE	*psHWRTDataCommonCookie,
+                                          RGX_KM_HW_RT_DATASET	**ppsKMHWRTDataSet) /* per-HWRTData */
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	/* KM cookie storing all the FW/HW data */
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+
+	/* local pointers for memory descriptors of FW allocations */
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc = NULL;
+
+	/* local pointer for CPU-mapped [FW]HWRTData */
+	RGXFWIF_HWRTDATA *psHWRTData = NULL;
+
+	/*
+	 * Fill in a local copy of RGXFWIF_HWRTDATA first to reduces reads and writes to
+	 * device memory. Using a byte array buffer avoids the limitation that some OSs
+	 * cannot align local variables to 64 bytes, usually needed by RGXFWIF_HWRTDATA
+	 */
+	IMG_BYTE aui8HWRTDataLocal[sizeof(RGXFWIF_HWRTDATA)] = {0};
+	RGXFWIF_HWRTDATA *psHWRTDataLocal = (RGXFWIF_HWRTDATA *)aui8HWRTDataLocal;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Prepare the HW RT DataSet struct */
+	psKMHWRTDataSet = OSAllocZMem(sizeof(*psKMHWRTDataSet));
+	if (psKMHWRTDataSet == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto AllocError;
+	}
+
+	*ppsKMHWRTDataSet = psKMHWRTDataSet;
+	psKMHWRTDataSet->psDeviceNode = psDeviceNode;
+	psKMHWRTDataSet->psHWRTDataCommonCookie = psHWRTDataCommonCookie;
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+	 * This FW RT-Data is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency and write-combine will
+	 * suffice on the CPU side. (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(RGXFWIF_HWRTDATA),
+	                          RGX_FWCOMCTX_ALLOCFLAGS,
+	                          "FwHwRTData",
+	                          &psHWRTDataFwMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: DevmemAllocate for RGX_FWIF_HWRTDATA failed",
+				__func__));
+		goto FWRTDataAllocateError;
+	}
+
+	psKMHWRTDataSet->psHWRTDataFwMemDesc = psHWRTDataFwMemDesc;
+	eError = RGXSetFirmwareAddress( &psKMHWRTDataSet->sHWRTDataFwAddr,
+							psHWRTDataFwMemDesc,
+							0,
+							RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", FWRTDataFwAddrError);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataFwMemDesc,
+									  (void **)&psHWRTData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTDataCpuMapError);
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psVHeapTableDevVAddr = psVHeapTableDevVAddr;
+#endif
+
+	psHWRTDataLocal->sPMSecureRenderStateDevVAddr = sPMSecureDataDevVAddr;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psPMMListDevVAddr = sPMDataDevVAddr;
+#else
+	psHWRTDataLocal->sPMRenderStateDevVAddr = sPMDataDevVAddr;
+#endif
+
+	psHWRTDataLocal->sTailPtrsDevVAddr     = sTailPtrsDevVAddr;
+
+	{
+		RGXFWIF_RTA_CTL *psRTACtl = &psHWRTDataLocal->sRTACtl;
+
+		psRTACtl->ui32RenderTargetIndex = 0;
+		psRTACtl->ui32ActiveRenderTargets = 0;
+		psRTACtl->sValidRenderTargets.ui32Addr = 0;
+		psRTACtl->sRTANumPartialRenders.ui32Addr = 0;
+		psRTACtl->ui32MaxRTs = (IMG_UINT32) ui16MaxRTs;
+
+		if (ui16MaxRTs > 1)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for shadow render target cache");
+			eError = DevmemFwAllocate(	psDevInfo,
+										ui16MaxRTs * sizeof(IMG_UINT32),
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwShadowRTCache",
+										&psRTArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTArryError;
+			}
+
+			psKMHWRTDataSet->psRTArrayFwMemDesc = psRTArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sValidRenderTargets,
+									psRTArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", FWAllocateRTArryFwAddrError);
+
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for tracking renders accumulation");
+			eError = DevmemFwAllocate(psDevInfo,
+			                          ui16MaxRTs * sizeof(IMG_UINT32),
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+					                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+					                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+					                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+					                  "FwRendersAccumulation",
+					                  &psRendersAccArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s) (renders accumulation)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTAccArryError;
+			}
+
+			psKMHWRTDataSet->psRendersAccArrayFwMemDesc = psRendersAccArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sRTANumPartialRenders,
+									psRendersAccArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", FWAllocRTAccArryFwAddrError);
+		}
+	}
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		psKMHWRTDataSet->apsFreeLists[ui32Loop] = apsFreeLists[ui32Loop];
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount++;
+		psHWRTDataLocal->apsFreeLists[ui32Loop].ui32Addr = psKMHWRTDataSet->apsFreeLists[ui32Loop]->sFreeListFWDevVAddr.ui32Addr;
+		/* Invalid initial snapshot value. The snapshot is always taken during the first
+		 * kick and hence this value gets replaced, so it's safe to set it to zero.
+		 */
+		psHWRTDataLocal->aui32FreeListHWRSnapshot[ui32Loop] = 0;
+		psHWRTDataLocal->bRenderStateNeedsReset = IMG_FALSE;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_add_to_tail(&apsFreeLists[RGXFW_LOCAL_FREELIST]->sNodeHWRTDataHead, &(psKMHWRTDataSet->sNodeHWRTData));
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	OSCachedMemCopy(psHWRTData, psHWRTDataLocal, sizeof(*psHWRTDataLocal));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTData 0x%08X", psKMHWRTDataSet->sHWRTDataFwAddr.ui32Addr);
+	DevmemPDumpLoadMem(psKMHWRTDataSet->psHWRTDataFwMemDesc, 0, sizeof(*psHWRTData), PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psHWRTData, FLUSH);
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	return PVRSRV_OK;
+
+FWAllocRTAccArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRendersAccArrayFwMemDesc);
+FWAllocateRTAccArryError:
+	RGXUnsetFirmwareAddress(psRTArrayFwMemDesc);
+FWAllocateRTArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRTArrayFwMemDesc);
+FWAllocateRTArryError:
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataCpuMapError:
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataAllocateError:
+	*ppsKMHWRTDataSet = NULL;
+	OSFreeMem(psKMHWRTDataSet);
+
+AllocError:
+	return eError;
+}
+
+static void RGXDestroyHWRTData_aux(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	if (psKMHWRTDataSet == NULL)
+	{
+		return;
+	}
+
+	psDevInfo = psKMHWRTDataSet->psDeviceNode->pvDevice;
+
+	if (psKMHWRTDataSet->psRTArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRTArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRTArrayFwMemDesc);
+	}
+
+	if (psKMHWRTDataSet->psRendersAccArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+	}
+
+	/* Decrease freelist refcount */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		PVR_ASSERT(psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount > 0);
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount--;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_remove_node(&psKMHWRTDataSet->sNodeHWRTData);
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Freeing the memory has to happen _after_ removing the HWRTData from the freelist
+	 * otherwise we risk traversing the freelist to find a pointer from a freed data structure */
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+
+	OSFreeMem(psKMHWRTDataSet);
+}
+
+/* Create set of HWRTData(s) and bind it with a shared FW HWRTDataCommon */
+PVRSRV_ERROR RGXCreateHWRTDataSet(
+		CONNECTION_DATA			*psConnection,
+		PVRSRV_DEVICE_NODE		*psDeviceNode,
+		IMG_DEV_VIRTADDR		psVHeapTableDevVAddr,
+		IMG_DEV_VIRTADDR		asPMDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR		asPMSecureDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+		RGX_FREELIST			*apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+		IMG_DEV_VIRTADDR		asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT16			ui16MaxRTs,
+		RGX_KM_HW_RT_DATASET	*pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psVHeapTableDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asPMDataDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(asPMSecureDataDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(apsFreeLists);
+	PVR_UNREFERENCED_PARAMETER(asTailPtrsDevVAddr);
+	PVR_UNREFERENCED_PARAMETER(ui16MaxRTs);
+	PVR_UNREFERENCED_PARAMETER(pasKMHWRTDataSet);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+_WritePMStateBuffer(PMR* psRenderStatesPMR,
+                    IMG_DEVMEM_OFFSET_T ui64Offset,
+                    IMG_DEV_VIRTADDR sMListDevVAddr,
+                    IMG_DEV_VIRTADDR sVHeapTableDevVAddr)
+{
+	PVRSRV_ERROR eError;
+
+	size_t uiNbBytes;
+	IMG_UINT32 asRenderStateBuffer[RGX_PM_MAX_RSTATE_SIZE_DWORDS] = {0};
+
+	PVR_ASSERT(PVRSRV_IS_FEATURE_SUPPORTED(PMR_DeviceNode(psRenderStatesPMR), PM_BYTE_ALIGNED_BASE_ADDRESSES));
+	PVR_ASSERT((sMListDevVAddr.uiAddr & (RGX_PM_MLIST_BASE_ADDR_MAX_ALIGNSIZE - 1)) == 0);
+	PVR_ASSERT((sVHeapTableDevVAddr.uiAddr & (RGX_PM_VHEAP_BASE_ADDR_MAX_ALIGNSIZE - 1)) == 0);
+
+	/* Initialise the Mlist base and Vheap base */
+	_RGX_PM_RENDERSTATE_BUFFER_SET_MLIST_BASE_ADDR(asRenderStateBuffer, sMListDevVAddr.uiAddr);
+	_RGX_PM_RENDERSTATE_BUFFER_SET_VHEAP_BASE_ADDR(asRenderStateBuffer, sVHeapTableDevVAddr.uiAddr);
+
+	eError = PMR_WriteBytes(psRenderStatesPMR,
+	                        ui64Offset,
+	                        (IMG_UINT8*)&asRenderStateBuffer,
+	                        sizeof(asRenderStateBuffer),
+	                        &uiNbBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Error from PMR_WriteBytes: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+		goto ErrorExit;
+	}
+
+	PVR_ASSERT(uiNbBytes == sizeof(asRenderStateBuffer));
+
+	PDUMPCOMMENT(PMR_DeviceNode(psRenderStatesPMR), "PMState buffer");
+	PMRPDumpLoadMem(psRenderStatesPMR, ui64Offset, sizeof(asRenderStateBuffer), PDUMP_FLAGS_CONTINUOUS, false);
+
+	return PVRSRV_OK;
+ErrorExit:
+	return eError;
+}
+
+PVRSRV_ERROR RGXCreateHWRTDataSet2(
+        CONNECTION_DATA         *psConnection,
+        PVRSRV_DEVICE_NODE      *psDeviceNode,
+        IMG_DEV_VIRTADDR        sVHeapTableDevVAddr,
+        DEVMEMINT_RESERVATION   *psPMMListsReservation,
+        DEVMEMINT_RESERVATION   *psPMStateReservation,
+        DEVMEMINT_RESERVATION   *psPMSecureStateReservation,
+        RGX_FREELIST            *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+        IMG_DEV_VIRTADDR        asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+        IMG_UINT16              ui16MaxRTs,
+        RGX_KM_HW_RT_DATASET    *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RTDataID;
+	IMG_UINT32 ui32GlobalFLMaxPages, ui32LocalFLMaxPages;
+	IMG_DEVMEM_SIZE_T ui64MListSize, ui64AlignedRenderStateSize;
+	RGX_HWRTDATA_COMMON_COOKIE *psHWRTDataCommonCookie;
+
+	PMR *psPMStatePMR=NULL, *psPMSecureStatePMR=NULL, *psMListsPMR=NULL;
+	IMG_DEV_VIRTADDR sPMStateBaseDevVAddr, sPMSecureStateDevVAddr, sMListsDevVAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(0 < ui16MaxRTs && ui16MaxRTs <= RGX_MAX_RTA_INDICES, "Number of TA Render targets outside the range (0, RGX_MAX_RTA_INDICES) is unsupported");
+
+	ui64AlignedRenderStateSize = PVR_ALIGN(sizeof(RGX_PM_RENDERSTATE_BUFFER),
+	   MAX(RGX_PM_RENDERSTATE_BASE_ADDR_ALIGNSIZE,
+	       GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, SLC_CACHE_LINE_SIZE_BITS))));
+
+	/* Check if freelists have uniform sizes */
+	eError = ValidateFreeListSizes(apsFreeLists,
+	                               &ui32LocalFLMaxPages,
+	                               &ui32GlobalFLMaxPages);
+	PVR_LOG_RETURN_IF_ERROR(eError, "Invalid freelist sizes");
+
+	ui64MListSize = RGXCalcMListSize(psDeviceNode,
+	    ui32LocalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE,
+	    ui32GlobalFLMaxPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMStateReservation,
+	                                          ui64AlignedRenderStateSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psPMStatePMR,
+	                                          &sPMStateBaseDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	    "Validation failed for MListAndState reservations", err_validation_rs);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMSecureStateReservation,
+	                                          ui64AlignedRenderStateSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psPMSecureStatePMR,
+	                                          &sPMSecureStateDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	    "Validation failed for secure data reservations", err_validation_srs);
+
+	eError = AcquireValidateRefCriticalBuffer(psDeviceNode,
+	                                          psPMMListsReservation,
+	                                          ui64MListSize * RGXMKIF_NUM_RTDATAS,
+	                                          &psMListsPMR,
+	                                          &sMListsDevVAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError,
+	    "Validation failed for secure data reservations", err_validation_mlist);
+
+	/* Prepare KM cleanup object for common data */
+	psHWRTDataCommonCookie = OSAllocZMem(sizeof(*psHWRTDataCommonCookie));
+	if (psHWRTDataCommonCookie == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_common_cookie_alloc;
+	}
+
+	psHWRTDataCommonCookie->psPMStateReservation = psPMStateReservation;
+	psHWRTDataCommonCookie->psPMSecureStateReservation = psPMSecureStateReservation;
+	psHWRTDataCommonCookie->psPMMListsReservation = psPMMListsReservation;
+
+	/* Here we are creating a set of HWRTData(s)
+	   the number of elements in the set equals RGXMKIF_NUM_RTDATAS.
+	*/
+	for (ui32RTDataID = 0; ui32RTDataID < RGXMKIF_NUM_RTDATAS; ui32RTDataID++)
+	{
+		IMG_DEV_VIRTADDR sRenderStateDevVirtAddr;
+		IMG_DEV_VIRTADDR sSecureRenderStateDevVirtAddr;
+		IMG_DEV_VIRTADDR sMListDevVirtAddr;
+
+		sRenderStateDevVirtAddr.uiAddr =
+		    sPMStateBaseDevVAddr.uiAddr + ui64AlignedRenderStateSize * ui32RTDataID;
+		sSecureRenderStateDevVirtAddr.uiAddr =
+		    sPMSecureStateDevVAddr.uiAddr + ui64AlignedRenderStateSize * ui32RTDataID;
+		sMListDevVirtAddr.uiAddr =
+		    sMListsDevVAddr.uiAddr + ui64MListSize * ui32RTDataID;
+
+		eError = _WritePMStateBuffer(psPMStatePMR,
+		                             ui64AlignedRenderStateSize * ui32RTDataID,
+		                             sMListDevVirtAddr,
+		                             sVHeapTableDevVAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError,
+		    "Failed to write PM state buffer", err_HWRTDataAlloc);
+
+		eError = RGXCreateHWRTData_aux(
+			psConnection,
+			psDeviceNode,
+			sVHeapTableDevVAddr,
+			sRenderStateDevVirtAddr,
+			sSecureRenderStateDevVirtAddr,
+			&apsFreeLists[(ui32RTDataID % RGXMKIF_NUM_GEOMDATAS) * RGXFW_MAX_FREELISTS],
+			asTailPtrsDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+			ui16MaxRTs,
+			psHWRTDataCommonCookie,
+			&pasKMHWRTDataSet[ui32RTDataID]);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to create HWRTData [slot %u] (%s)",
+					__func__,
+					ui32RTDataID,
+					PVRSRVGetErrorString(eError)));
+			goto err_HWRTDataAlloc;
+		}
+		psHWRTDataCommonCookie->ui32RefCount += 1;
+	}
+
+	return PVRSRV_OK;
+
+err_HWRTDataAlloc:
+	PVR_DPF((PVR_DBG_WARNING, "%s: err_HWRTDataAlloc", __func__));
+	if (pasKMHWRTDataSet)
+	{
+		for (; ui32RTDataID > 0; ui32RTDataID--)
+		{
+			if (pasKMHWRTDataSet[ui32RTDataID-1] != NULL)
+			{
+				RGXDestroyHWRTData_aux(pasKMHWRTDataSet[ui32RTDataID-1]);
+				pasKMHWRTDataSet[ui32RTDataID-1] = NULL;
+			}
+		}
+	}
+	OSFreeMem(psHWRTDataCommonCookie);
+err_common_cookie_alloc:
+	UnrefAndReleaseCriticalBuffer(psPMMListsReservation);
+err_validation_mlist:
+	UnrefAndReleaseCriticalBuffer(psPMSecureStateReservation);
+err_validation_srs:
+	UnrefAndReleaseCriticalBuffer(psPMStateReservation);
+err_validation_rs:
+	return eError;
+}
+
+/* Destroy a single instance of HWRTData.
+   Additionally, destroy the HWRTDataCommon{Cookie} objects
+   when it is the last HWRTData within a corresponding set of HWRTDatas.
+*/
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_ERROR eError;
+	PRGXFWIF_HWRTDATA psHWRTData;
+	RGX_HWRTDATA_COMMON_COOKIE *psCommonCookie;
+	PVR_ASSERT(psKMHWRTDataSet);
+
+	psDevNode = psKMHWRTDataSet->psDeviceNode;
+
+	eError = RGXSetFirmwareAddress(&psHWRTData,
+	                               psKMHWRTDataSet->psHWRTDataFwMemDesc, 0,
+	                               RFW_FWADDR_NOREF_FLAG);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Cleanup HWRTData */
+	eError = RGXFWRequestHWRTDataCleanUp(psDevNode, psHWRTData);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDevNode,
+						   eError,
+						   RGXFWRequestHWRTDataCleanUp);
+
+	psCommonCookie = psKMHWRTDataSet->psHWRTDataCommonCookie;
+
+	RGXDestroyHWRTData_aux(psKMHWRTDataSet);
+
+	/* We've got past potential PVRSRV_ERROR_RETRY events, so we are sure
+	   that the HWRTDATA instance will be destroyed during this call.
+	   Consequently, we decrease the ref count for HWRTDataCommonCookie.
+
+	   NOTE: This ref count does not require locks or atomics.
+	   -------------------------------------------------------
+	     HWRTDatas bound into one pair are always destroyed sequentially,
+	     within a single loop on the Client side.
+	     The Common/Cookie objects always belong to only one pair of
+	     HWRTDatas, and ref count is used to ensure that the Common/Cookie
+	     objects will be destroyed after destruction of all HWRTDatas
+	     within a single pair.
+	*/
+	psCommonCookie->ui32RefCount--;
+
+	/* When ref count for HWRTDataCommonCookie hits ZERO
+	 * we have to destroy the cookie
+	 * [KM object] afterwards. */
+	if (psCommonCookie->ui32RefCount == 0)
+	{
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMStateReservation);
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMSecureStateReservation);
+		UnrefAndReleaseCriticalBuffer(psCommonCookie->psPMMListsReservation);
+
+		OSFreeMem(psCommonCookie);
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA       *psConnection,
+                               PVRSRV_DEVICE_NODE     *psDeviceNode,
+                               IMG_HANDLE             hMemCtxPrivData,
+                               IMG_UINT32             ui32MaxFLPages,
+                               IMG_UINT32             ui32InitFLPages,
+                               IMG_UINT32             ui32GrowFLPages,
+                               IMG_UINT32             ui32GrowParamThreshold,
+                               RGX_FREELIST           *psGlobalFreeList,
+                               IMG_BOOL               bCheckFreelist,
+                               DEVMEMINT_RESERVATION *psFreeListAndStateReservation,
+                               RGX_FREELIST           **ppsFreeList)
+{
+	PVRSRV_ERROR        eError;
+	RGXFWIF_FREELIST    *psFWFreeList;
+	RGXFWIF_FREELIST    sFWFreeList = {{0}};
+	DEVMEM_MEMDESC      *psFWFreelistMemDesc;
+	RGX_FREELIST        *psFreeList;
+	PVRSRV_RGXDEV_INFO  *psDevInfo = psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR    sFreeListBaseDevVAddr;
+	IMG_DEV_VIRTADDR    sFreeListStateDevVAddr;
+	PMR*                psFreeListAndStatePMR = NULL;
+	IMG_UINT32          ui32ReadyPages;
+	IMG_UINT32          uiFreeListOffset = PVR_ALIGN(
+	                        sizeof(RGX_PM_FREELISTSTATE_BUFFER),
+	                        GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, SLC_CACHE_LINE_SIZE_BITS)));
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psFreeListAndStateReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Failed to acquire reservation for freelist buffer",
+		        __func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	eError = DevmemIntGetReservationData(psFreeListAndStateReservation, &psFreeListAndStatePMR, &sFreeListStateDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Error from DevmemIntGetReservationData: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+
+		goto ErrorAllocHost;
+	}
+
+	/* Check if client properly allocated PMMETA_PROTECT */
+	if ((PMR_Flags(psFreeListAndStatePMR) & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must have PMMETA_PROTECT set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psFreeListAndStatePMR)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must not have OS_LINUX_MOVABLE set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+#endif
+
+	/* Ref the PMR to prevent resource being destroyed before use */
+	eError = PMRRefPMR(psFreeListAndStatePMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorAllocHost);
+
+	if (PMR_IsSparse(psFreeListAndStatePMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Free list PMR cannot be sparse!",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto ErrorRefPMR;
+	}
+
+	sFreeListBaseDevVAddr.uiAddr = sFreeListStateDevVAddr.uiAddr + uiFreeListOffset;
+
+	if (OSGetPageShift() > RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+	{
+		IMG_UINT32 ui32Size, ui32NewInitFLPages, ui32NewMaxFLPages, ui32NewGrowFLPages;
+
+		/* Round up number of FL pages to the next multiple of the OS page size */
+
+		ui32Size = ui32InitFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewInitFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32GrowFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewGrowFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32MaxFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewMaxFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Increased number of PB pages: Init %u -> %u, Grow %u -> %u, Max %u -> %u",
+		         __func__, ui32InitFLPages, ui32NewInitFLPages, ui32GrowFLPages, ui32NewGrowFLPages, ui32MaxFLPages, ui32NewMaxFLPages));
+
+		ui32InitFLPages = ui32NewInitFLPages;
+		ui32GrowFLPages = ui32NewGrowFLPages;
+		ui32MaxFLPages = ui32NewMaxFLPages;
+	}
+
+	/* Allocate kernel freelist struct */
+	psFreeList = OSAllocZMem(sizeof(*psFreeList));
+	if (psFreeList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: failed to allocate host data structure",
+		        __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRefPMR;
+	}
+
+	/*
+	 * This FW FreeList context is only mapped into kernel for initialisation
+	 * and reconstruction (at other times it is not mapped and only used by the
+	 * FW).
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(*psFWFreeList),
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+	                          PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_OS_LINUX_DENY_MOVE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwFreeList",
+	                          &psFWFreelistMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: DevmemAllocate for RGXFWIF_FREELIST failed",
+		        __func__));
+		goto FWFreeListAlloc;
+	}
+
+	/* Initialise host data structures */
+	psFreeList->psDevInfo = psDevInfo;
+	psFreeList->psConnection = psConnection;
+
+	psFreeList->psFreeListStatePMR = psFreeListAndStatePMR;
+	psFreeList->uiFreeListStatePMROffset = 0;
+
+	psFreeList->psFreeListPMR = psFreeListAndStatePMR;
+	psFreeList->uiFreeListPMROffset = uiFreeListOffset;
+
+	psFreeList->psFreeListAndStateReservation = psFreeListAndStateReservation;
+
+	psFreeList->psFWFreelistMemDesc = psFWFreelistMemDesc;
+	psFreeList->sFreeListBaseDevVAddr = sFreeListBaseDevVAddr;
+	eError = RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	/* psFreeList->ui32FreelistID set below with lock... */
+	psFreeList->ui32FreelistGlobalID = (psGlobalFreeList ? psGlobalFreeList->ui32FreelistID : 0);
+	psFreeList->ui32MaxFLPages = ui32MaxFLPages;
+	psFreeList->ui32InitFLPages = ui32InitFLPages;
+	psFreeList->ui32GrowFLPages = ui32GrowFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+	psFreeList->ui32ReadyFLPages = 0;
+	psFreeList->ui32GrowThreshold = ui32GrowParamThreshold;
+	psFreeList->ui64FreelistChecksum = 0;
+	psFreeList->ui32RefCount = 0;
+	psFreeList->bCheckFreelist = bCheckFreelist;
+	dllist_init(&psFreeList->sMemoryBlockHead);
+	dllist_init(&psFreeList->sMemoryBlockInitHead);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_init(&psFreeList->sNodeHWRTDataHead);
+#endif
+	psFreeList->ownerPid = OSGetCurrentClientProcessIDKM();
+
+	/* Add to list of freelists */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	psFreeList->ui32FreelistID = psDevInfo->ui32FreelistCurrID++;
+	dllist_add_to_tail(&psDevInfo->sFreeListHead, &psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	ui32ReadyPages = _CalculateFreelistReadyPages(psFreeList, ui32InitFLPages);
+
+	/* Write freelist state buffer */
+	{
+		RGX_PM_FREELISTSTATE_BUFFER sFLState = {0};
+		size_t uiNbBytes;
+		IMG_DEV_VIRTADDR sFLBaseAddr;
+
+		sFLBaseAddr.uiAddr = (psFreeList->sFreeListBaseDevVAddr.uiAddr +
+		                      ((psFreeList->ui32MaxFLPages - ui32InitFLPages + ui32ReadyPages) * sizeof(IMG_UINT32))) &
+		                      ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+
+
+		RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(sFLState, (sFLBaseAddr.uiAddr >> RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT));
+
+		RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(sFLState, ui32InitFLPages - ui32ReadyPages - 1);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(sFLState, 0);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(sFLState, 0);
+
+		eError = PMR_WriteBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Error from PMR_WriteBytes: %s",
+			        __func__, PVRSRVGetErrorString(eError)));
+			goto FWFreeListAlloc;
+		}
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+
+		PDUMPCOMMENT(psDeviceNode, "FreeListState buffer");
+		PMRPDumpLoadMem(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, sizeof(sFLState), PDUMP_FLAGS_CONTINUOUS, false);
+	}
+
+	/* Initialise FW data structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWFreeListCpuMap);
+
+	{
+		sFWFreeList.ui32MaxPages = ui32MaxFLPages;
+		sFWFreeList.ui32CurrentPages = ui32InitFLPages - ui32ReadyPages;
+		sFWFreeList.ui32GrowPages = ui32GrowFLPages;
+		sFWFreeList.bUpdatePending = IMG_FALSE;
+		sFWFreeList.ui32UpdateNewPages = 0;
+		sFWFreeList.ui32UpdateNewReadyPages = 0;
+		sFWFreeList.sFreeListBaseDevVAddr = sFreeListBaseDevVAddr;
+#if defined(PM_INTERACTIVE_MODE)
+		sFWFreeList.ui32CurrentStackTop = sFWFreeList.ui32CurrentPages - 1;
+		sFWFreeList.ui64CurrentDevVAddr = (sFreeListBaseDevVAddr.uiAddr +
+		                                     ((ui32MaxFLPages - sFWFreeList.ui32CurrentPages) * sizeof(IMG_UINT32))) &
+		                                    ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+#endif
+		sFWFreeList.ui32FreeListID = psFreeList->ui32FreelistID;
+		sFWFreeList.bGrowPending = IMG_FALSE;
+		sFWFreeList.ui32ReadyPages = ui32ReadyPages;
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+		/* Get the FW Memory Context address... */
+		eError = RGXSetFirmwareAddress(&sFWFreeList.psFWMemContext,
+		                               RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData),
+		                               0, RFW_FWADDR_NOREF_FLAG);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: RGXSetFirmwareAddress for RGXFWIF_FWMEMCONTEXT failed",
+			        __func__));
+			DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+			goto FWFreeListCpuMap;
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(hMemCtxPrivData);
+#endif
+
+		/*
+		 * Only the PM state buffer address is needed which contains the PM
+		 * state including the freelist base address.
+		 *
+		 * Access to the physical PMR will be used to update the contents of the
+		 * PM state buffer when PB grow occurs following OOM.
+		 */
+		sFWFreeList.sFreeListLastGrowDevVAddr.uiAddr = 0;
+		sFWFreeList.sFreeListStateDevVAddr = sFreeListStateDevVAddr;
+	}
+
+	OSCachedMemCopy(psFWFreeList, &sFWFreeList, sizeof(sFWFreeList));
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "Freelist %p created: Max pages 0x%08x, Init pages 0x%08x, "
+	        "FL base address 0x%016" IMG_UINT64_FMTSPECx ", "
+	        "Current FL base address 0x%016" IMG_UINT64_FMTSPECx ", "
+	        "Current pages %u",
+	        psFreeList,
+	        ui32MaxFLPages,
+	        ui32InitFLPages,
+	        sFreeListBaseDevVAddr.uiAddr,
+	        (sFreeListBaseDevVAddr.uiAddr +
+	            ((ui32MaxFLPages - sFWFreeList.ui32CurrentPages) * sizeof(IMG_UINT32))) &
+	        ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1),
+	        sFWFreeList.ui32CurrentPages - 1));
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FW FreeList");
+	DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWFreeList), PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(PM_INTERACTIVE_MODE)
+	/*
+	 * Separate dump of the Freelist's number of Pages and stack pointer.
+	 * This allows to easily modify the PB size in the out2.txt files.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "FreeList TotalPages");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentPages),
+	                          sFWFreeList.ui32CurrentPages,
+	                          PDUMP_FLAGS_CONTINUOUS);
+	PDUMPCOMMENT(psDeviceNode, "FreeList StackPointer");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentStackTop),
+	                          sFWFreeList.ui32CurrentStackTop,
+	                          PDUMP_FLAGS_CONTINUOUS);
+#endif
+#endif
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+	/* Add initial PB block */
+	eError = RGXGrowFreeList(psFreeList,
+	                         ui32InitFLPages,
+	                         &psFreeList->sMemoryBlockInitHead);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: failed to allocate initial memory block for free list 0x%016" IMG_UINT64_FMTSPECx " (%s)",
+		        __func__,
+		        sFreeListBaseDevVAddr.uiAddr,
+		        PVRSRVGetErrorString(eError)));
+		goto FWFreeListCpuMap;
+	}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Update Stats */
+	PVRSRVStatsUpdateFreelistStats(psDeviceNode,
+	                               1, /* Add 1 to the appropriate counter (Requests by App)*/
+	                               0,
+	                               psFreeList->ui32InitFLPages,
+	                               psFreeList->ui32NumHighPages,
+	                               psFreeList->ownerPid);
+
+#endif
+
+	/* return values */
+	*ppsFreeList = psFreeList;
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+FWFreeListCpuMap:
+	/* Remove freelists from list */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+	RGXUnsetFirmwareAddress(psFWFreelistMemDesc);
+
+ErrorSetFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psFWFreelistMemDesc);
+
+FWFreeListAlloc:
+	OSFreeMem(psFreeList);
+
+ErrorRefPMR:
+	(void) PMRUnrefPMR(psFreeListAndStatePMR);
+
+ErrorAllocHost:
+	DevmemIntReservationRelease(psFreeListAndStateReservation);
+
+ErrorReservationAcquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyFreeList
+*/
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RefCount;
+
+	PVR_ASSERT(psFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	ui32RefCount = psFreeList->ui32RefCount;
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	if (ui32RefCount != 0)
+	{
+		/* Freelist still busy */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Freelist is not in use => start firmware cleanup */
+	eError = RGXFWRequestFreeListCleanUp(psFreeList->psDevInfo,
+										 psFreeList->sFreeListFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psFreeList->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestFreeListCleanUp);
+
+	/* Device becomes unrecoverable if the firmware took too long to
+	 * handle the cleanup request, or if SLC-flushes didn't go through
+	 * (due to some GPU lockup) */
+
+	/* Remove FreeList from linked list before we destroy it... */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* Confirm all HWRTData nodes are freed before releasing freelist */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sNodeHWRTDataHead));
+#endif
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+#if defined(PM_INTERACTIVE_MODE)
+	if (psFreeList->bCheckFreelist)
+	{
+		RGXFWIF_FREELIST  *psFWFreeList;
+		IMG_UINT32        ui32CurrentStackTop;
+		IMG_UINT64        ui64CheckSum;
+
+		/* Get the current stack pointer for this free list */
+		DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+		RGXFwSharedMemCacheOpValue(psFWFreeList->ui32CurrentStackTop, INVALIDATE);
+		ui32CurrentStackTop = psFWFreeList->ui32CurrentStackTop;
+		DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+		if (ui32CurrentStackTop > psFreeList->ui32MaxFLPages)
+		{
+			PVR_LOG(("%s: FW freelist corrupted (%d)",
+					 __func__,
+					 ui32CurrentStackTop));
+		}
+		else if (ui32CurrentStackTop == psFreeList->ui32CurrentFLPages-1)
+		{
+			/* Do consistency tests (as the list is fully populated) */
+			_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+		}
+		else
+		{
+			/* Check for duplicate pages, but don't check the checksum as the list is not fully populated */
+			_CheckFreelist(psFreeList, ui32CurrentStackTop+1, 0, &ui64CheckSum);
+		}
+	}
+#endif
+
+	/* Destroy FW structures */
+	RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
+	DevmemFwUnmapAndFree(psFreeList->psDevInfo, psFreeList->psFWFreelistMemDesc);
+
+	/* Remove grow shrink blocks */
+	while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
+	{
+		eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* Remove initial PB block */
+	eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockInitHead, psFreeList);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* consistency checks */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockInitHead));
+	PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);
+
+	/* Remove references from the PMR resources */
+	eError = PMRUnrefPMR(psFreeList->psFreeListStatePMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	DevmemIntReservationRelease(psFreeList->psFreeListAndStateReservation);
+
+	/* free Freelist */
+	OSFreeMem(psFreeList);
+
+	return eError;
+}
+
+
+/*
+	RGXCreateZSBuffer
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_PRBUFFER			*psFWZSBuffer;
+	RGXFWIF_PRBUFFER			sFWZSBuffer = {0};
+	RGX_ZSBUFFER_DATA			*psZSBuffer;
+	DEVMEM_MEMDESC				*psFWZSBufferMemDesc;
+	IMG_BOOL					bOnDemand = PVRSRV_CHECK_ON_DEMAND(uiMapFlags) ? IMG_TRUE : IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Allocate host data structure */
+	psZSBuffer = OSAllocZMem(sizeof(*psZSBuffer));
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate cleanup data structure for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocCleanup;
+	}
+
+	/* Populate Host data */
+	psZSBuffer->psDevInfo = psDevInfo;
+	psZSBuffer->psReservation = psReservation;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psZSBuffer->psReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to acquire reservation for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	psZSBuffer->psPMR = psPMR;
+	/* Obtain reference to PMR */
+	eError = PMRRefPMR(psZSBuffer->psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", ErrorRefPMR);
+
+	psZSBuffer->ui32RefCount = 0;
+	psZSBuffer->bIsBacked = IMG_FALSE;
+	psZSBuffer->bOnDemand = bOnDemand;
+
+	/* Allocate firmware memory for ZS-Buffer. */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware ZS-Buffer data structure");
+	eError = DevmemFwAllocate(psDevInfo,
+							sizeof(*psFWZSBuffer),
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+							PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+							PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+							PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+							PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+							"FwZSBuffer",
+							&psFWZSBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAllocFWZSBuffer;
+	}
+	psZSBuffer->psFWZSBufferMemDesc = psFWZSBufferMemDesc;
+
+	/* Temporarily map the firmware render context to the kernel. */
+	eError = DevmemAcquireCpuVirtAddr(psFWZSBufferMemDesc,
+	                                  (void **)&psFWZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAcquireFWZSBuffer;
+	}
+
+	/* Get firmware address of ZS-Buffer. */
+	eError = RGXSetFirmwareAddress(&psZSBuffer->sZSBufferFWDevVAddr, psFWZSBufferMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	if (bOnDemand)
+	{
+		/* psZSBuffer->ui32ZSBufferID set below with lock... */
+		OSLockAcquire(psDevInfo->hLockZSBuffer);
+		psZSBuffer->ui32ZSBufferID = psDevInfo->ui32ZSBufferCurrID++;
+		dllist_add_to_tail(&psDevInfo->sZSBufferHead, &psZSBuffer->sNode);
+		OSLockRelease(psDevInfo->hLockZSBuffer);
+	}
+
+	/* Populate FW ZS-Buffer data structure */
+	sFWZSBuffer.bOnDemand = bOnDemand;
+	sFWZSBuffer.eState = (bOnDemand) ? RGXFWIF_PRBUFFER_UNBACKED : RGXFWIF_PRBUFFER_BACKED;
+	sFWZSBuffer.ui32BufferID = psZSBuffer->ui32ZSBufferID;
+
+	OSCachedMemCopy(psFWZSBuffer, &sFWZSBuffer, sizeof(sFWZSBuffer));
+	/* Dump the ZS-Buffer and the memory content */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump firmware ZS-Buffer");
+	DevmemPDumpLoadMem(psFWZSBufferMemDesc, 0, sizeof(*psFWZSBuffer), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWZSBuffer, FLUSH);
+
+	/* Release address acquired above. */
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+
+
+	/* define return value */
+	*ppsZSBuffer = psZSBuffer;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] created (%s)",
+							psZSBuffer,
+							(bOnDemand) ? "On-Demand": "Up-front"));
+
+	psZSBuffer->owner=OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+
+	/* error handling */
+
+ErrorSetFwAddr:
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+ErrorAcquireFWZSBuffer:
+	DevmemFwUnmapAndFree(psDevInfo, psFWZSBufferMemDesc);
+
+ErrorAllocFWZSBuffer:
+	(void) PMRUnrefPMR(psZSBuffer->psPMR);
+ErrorRefPMR:
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+ErrorReservationAcquire:
+	OSFreeMem(psZSBuffer);
+
+ErrorAllocCleanup:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyZSBuffer
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError, eError2;
+
+	PVR_ASSERT(psZSBuffer);
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	/* Request ZS Buffer cleanup */
+	eError = RGXFWRequestZSBufferCleanUp(psZSBuffer->psDevInfo,
+										psZSBuffer->sZSBufferFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psZSBuffer->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestZSBufferCleanUp);
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->ui32RefCount != 0)
+	{
+		/* ZS-Buffer is still referenced */
+		OSLockRelease(hLockZSBuffer);
+		PVR_DPF((PVR_DBG_WARNING, "Trying to destroy a ZS-Buffer [%p] that's still in use.", psZSBuffer));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Free the firmware render context. */
+	RGXUnsetFirmwareAddress(psZSBuffer->psFWZSBufferMemDesc);
+	DevmemFwUnmapAndFree(psZSBuffer->psDevInfo, psZSBuffer->psFWZSBufferMemDesc);
+
+	/* Remove Deferred Allocation from list */
+	if (psZSBuffer->bOnDemand)
+	{
+		PVR_ASSERT(dllist_node_is_in_list(&psZSBuffer->sNode));
+		dllist_remove_node(&psZSBuffer->sNode);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] destroyed", psZSBuffer));
+
+	/* Release reference to reservation object and the PMR */
+	eError2 = PMRUnrefPMR(psZSBuffer->psPMR);
+	PVR_LOG_IF_ERROR(eError2, "PMRUnrefPMR");
+
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+
+	/* Free ZS-Buffer host data structure */
+	OSFreeMem(psZSBuffer);
+
+	OSLockRelease(hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (!psZSBuffer->bOnDemand)
+	{
+		/* Only deferred allocations can be populated */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bIsBacked == IMG_FALSE)
+	{
+		IMG_HANDLE hDevmemHeap;
+
+		/* Get Heap */
+		eError = DevmemServerGetHeapHandle(psZSBuffer->psReservation, &hDevmemHeap);
+		if (unlikely(hDevmemHeap == (IMG_HANDLE)NULL))
+		{
+			OSLockRelease(hLockZSBuffer);
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+
+		eError = DevmemIntMapPMR(psZSBuffer->psReservation, psZSBuffer->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable populate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+
+		}
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing acquired",
+		                          psZSBuffer,
+		                          psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_TRUE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+					RGX_POPULATION **ppsPopulation)
+{
+	RGX_POPULATION *psPopulation;
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount++;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	psZSBuffer->ui32NumReqByApp++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psZSBuffer->psDevInfo->psDeviceNode,
+	                               1, 0, psZSBuffer->owner);
+#endif
+
+	/* Do the backing */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		goto OnErrorBacking;
+	}
+
+	/* Create the handle to the backing */
+	psPopulation = OSAllocMem(sizeof(*psPopulation));
+	if (psPopulation == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto OnErrorAlloc;
+	}
+
+	psPopulation->psZSBuffer = psZSBuffer;
+
+	/* return value */
+	*ppsPopulation = psPopulation;
+
+	return PVRSRV_OK;
+
+OnErrorAlloc:
+	RGXUnbackingZSBuffer(psZSBuffer);
+
+OnErrorBacking:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing removal requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bOnDemand && psZSBuffer->bIsBacked == IMG_TRUE)
+	{
+		eError = DevmemIntUnmapPMR(psZSBuffer->psReservation);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable to unpopulate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing removed",
+									psZSBuffer,
+									psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_FALSE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation)
+{
+	PVRSRV_ERROR eError;
+	POS_LOCK hLockZSBuffer;
+
+	if (!psPopulation)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	hLockZSBuffer = psPopulation->psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	eError = RGXUnbackingZSBuffer(psPopulation->psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	OSLockAcquire(hLockZSBuffer);
+	psPopulation->psZSBuffer->ui32RefCount--;
+	OSLockRelease(hLockZSBuffer);
+
+	OSFreeMem(psPopulation);
+
+	return PVRSRV_OK;
+}
+
+static RGX_ZSBUFFER_DATA *FindAndRefZSBuffer(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32ZSBufferID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_ZSBUFFER_DATA *psZSBuffer = NULL;
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+
+	dllist_foreach_node(&psDevInfo->sZSBufferHead, psNode, psNext)
+	{
+		RGX_ZSBUFFER_DATA *psThisZSBuffer = IMG_CONTAINER_OF(psNode, RGX_ZSBUFFER_DATA, sNode);
+
+		if (psThisZSBuffer->ui32ZSBufferID == ui32ZSBufferID)
+		{
+			psZSBuffer = psThisZSBuffer;
+
+			psZSBuffer->ui32RefCount++;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+	return psZSBuffer;
+}
+
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID)
+{
+	IMG_BOOL bBackingDone = IMG_TRUE;
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (Populate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Populate ZLS */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Populating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		bBackingDone = IMG_FALSE;
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = bBackingDone;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psZSBuffer->ui32NumReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psDevInfo->psDeviceNode,
+	                               0, 1, psZSBuffer->owner);
+#endif
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+
+}
+
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32 ui32ZSBufferID)
+{
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (UnPopulate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Unpopulate ZLS */
+	eError = RGXUnbackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"UnPopulating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		PVR_ASSERT(IMG_FALSE);
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = IMG_TRUE;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+}
+
+static
+PVRSRV_ERROR _CreateTAContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              SERVER_MMU_CONTEXT *psServerMMUContext,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_TA_DATA *psTAData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TA context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_TACTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwTAContextState",
+							  &psTAData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacontextsuspendalloc;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TA,
+	                                 RGXFWIF_DM_GEOM,
+	                                 psServerMMUContext,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 psTAData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TA_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TA_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &psTAData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init TA fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump the TA context suspend state buffer");
+	DevmemPDumpLoadMem(psTAData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TACTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	psTAData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_tacommoncontext:
+	DevmemFree(psTAData->psContextStateMemDesc);
+fail_tacontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Create3DContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              SERVER_MMU_CONTEXT *psServerMMUContext,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_3D_DATA *ps3DData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	uiNumISPStoreRegs;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware 3D context suspend state");
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	uiNumISPStoreRegs = RGXFWIF_IPP_RESUME_REG_COUNT; /* default 1 register for IPP_resume */
+
+	uiNumISPStoreRegs += (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU) *
+								RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_PER_SPU) *
+								RGXFWIF_PIPE_COUNT_PER_ISP);
+
+
+	if (uiNumISPStoreRegs > (RGXFWIF_ISP_PIPE_COUNT_MAX + RGXFWIF_IPP_RESUME_REG_COUNT))
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+#else /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+	/* Size of the CS buffer */
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			(uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]));
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui3DRegISPStateStoreSize,
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "Fw3DContextState",
+							  &ps3DData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcontextsuspendalloc;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_3D,
+	                                 RGXFWIF_DM_3D,
+	                                 psServerMMUContext,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init 3D fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Dump the 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_3DCTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_3dcommoncontext:
+	DevmemFree(ps3DData->psContextStateMemDesc);
+fail_3dcontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVRGXCreateRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+                                            PVRSRV_DEVICE_NODE			*psDeviceNode,
+                                            IMG_INT32					i32Priority,
+                                            IMG_UINT32					ui32FrameworkRegisterSize,
+                                            IMG_PBYTE					pabyFrameworkRegisters,
+                                            IMG_HANDLE					hMemCtxPrivData,
+                                            IMG_UINT32					ui32StaticRenderContextStateSize,
+                                            IMG_PBYTE					pStaticRenderContextState,
+                                            IMG_UINT32					ui32PackedCCBSizeU8888,
+                                            IMG_UINT32					ui32ContextFlags,
+                                            IMG_UINT64					ui64RobustnessAddress,
+                                            IMG_UINT32					ui32MaxTADeadlineMS,
+                                            IMG_UINT32					ui32Max3DDeadlineMS,
+                                            RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_RENDER_CONTEXT	*psRenderContext;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	RGXFWIF_FWRENDERCONTEXT		*psFWRenderContext;
+
+	*ppsRenderContext = NULL;
+
+	if (ui32StaticRenderContextStateSize > RGXFWIF_STATIC_RENDERCONTEXT_SIZE)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRenderContext = OSAllocZMem(sizeof(*psRenderContext));
+	if (psRenderContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psRenderContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_lock;
+	}
+
+	psRenderContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Create the FW render context, this has the TA and 3D FW common
+		contexts embedded within it
+	*/
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_FWRENDERCONTEXT),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwRenderContext",
+							  &psRenderContext->psFWRenderContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwrendercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkRegisterSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psRenderContext->psFWFrameworkMemDesc,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psRenderContext->psFWFrameworkMemDesc,
+				pabyFrameworkRegisters,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+		sInfo.psFWFrameworkMemDesc = psRenderContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DContext(psConnection,
+	                          psDeviceNode,
+	                          hMemCtxPrivData,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),
+	                          psFWMemContextMemDesc,
+	                          i32Priority,
+	                          ui32Max3DDeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->s3DData,
+	                          U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcontext;
+	}
+
+	eError = _CreateTAContext(psConnection,
+	                          psDeviceNode,
+	                          hMemCtxPrivData,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),
+	                          psFWMemContextMemDesc,
+	                          i32Priority,
+	                          ui32MaxTADeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->sTAData,
+	                          U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tacontext;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+			(void **)&psFWRenderContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	/* Copy the static render context data */
+	OSDeviceMemCopy(&psFWRenderContext->sStaticRenderContextState, pStaticRenderContextState, ui32StaticRenderContextStateSize);
+#if defined(SUPPORT_TRP)
+	psFWRenderContext->eTRPGeomCoreAffinity = RGXFWIF_DM_MAX;
+#endif
+	DevmemPDumpLoadMem(psRenderContext->psFWRenderContextMemDesc, 0, sizeof(RGXFWIF_FWRENDERCONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpPtr(psFWRenderContext, FLUSH);
+	DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psRenderContext->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-ta3d");
+	if (IS_ERR(psRenderContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(psRenderContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	}
+
+	*ppsRenderContext = psRenderContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _DestroyTAContext(&psRenderContext->sTAData,
+		                                         psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_tacontext:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _Destroy3DContext(&psRenderContext->s3DData,
+		                                         psRenderContext->psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_3dcontext:
+fail_frameworkcopy:
+	if (psRenderContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+fail_fwrendercontext:
+	OSLockDestroy(psRenderContext->hLock);
+fail_lock:
+	OSFreeMem(psRenderContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+ * PVRSRVRGXDestroyRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psRenderContext->psDeviceNode->pvDevice;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_remove_node(&(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* Check psBufferSyncContext has not been destroyed already (by a previous
+	 * call to this function which then later returned PVRSRV_ERROR_RETRY)
+	 */
+	if (psRenderContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psRenderContext->psBufferSyncContext);
+		psRenderContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+	/* Cleanup the TA if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_TA_COMPLETE) == 0)
+	{
+		eError = _DestroyTAContext(&psRenderContext->sTAData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_TA_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+	/* Cleanup the 3D if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_3D_COMPLETE) == 0)
+	{
+		eError = _Destroy3DContext(&psRenderContext->s3DData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_3D_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+	{
+		RGXFWIF_FWRENDERCONTEXT	*psFWRenderContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+				(void **)&psFWRenderContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware render context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+		RGXFwSharedMemCacheOpValue(psFWRenderContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+
+		ui32WorkEstCCBSubmitted = psFWRenderContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psRenderContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psRenderContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			eError = PVRSRV_ERROR_RETRY;
+			goto e0;
+		}
+	}
+#endif
+
+	/*
+		Only if both TA and 3D contexts have been cleaned up can we
+		free the shared resources
+	*/
+	if (psRenderContext->ui32CleanupStatus == (RC_CLEANUP_3D_COMPLETE | RC_CLEANUP_TA_COMPLETE))
+	{
+		if (psRenderContext->psFWFrameworkMemDesc)
+		{
+			/* Free the framework buffer */
+			DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+		}
+
+		/* Free the firmware render context */
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAUpdate);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DUpdate);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			WorkEstDeInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+		}
+#endif
+
+		OSLockDestroy(psRenderContext->hLock);
+
+		OSFreeMem(psRenderContext);
+	}
+
+	return PVRSRV_OK;
+
+e0:
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	return eError;
+}
+
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+static void DumpUfoList(IMG_UINT32 ui32ClientTAFenceCount,
+                        IMG_UINT32 ui32ClientTAUpdateCount,
+                        IMG_UINT32 ui32Client3DFenceCount,
+                        IMG_UINT32 ui32Client3DUpdateCount,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAFenceUFOAddress,
+                        IMG_UINT32 *paui32ClientTAFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAUpdateUFOAddress,
+                        IMG_UINT32 *paui32ClientTAUpdateValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DFenceUFOAddress,
+                        IMG_UINT32 *paui32Client3DFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DUpdateUFOAddress,
+                        IMG_UINT32 *paui32Client3DUpdateValue)
+{
+	IMG_UINT32 i;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: ~~~ After populating sync prims ~~~",
+			 __func__));
+
+	/* Dump Fence syncs, Update syncs and PR Update syncs */
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA fence syncs:",
+	        __func__, ui32ClientTAFenceCount));
+	for (i = 0; i < ui32ClientTAFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr,
+			        *paui32ClientTAFenceValue,
+			        *paui32ClientTAFenceValue));
+			paui32ClientTAFenceValue++;
+		}
+		pauiClientTAFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA update syncs:",
+			 __func__, ui32ClientTAUpdateCount));
+	for (i = 0; i < ui32ClientTAUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr,
+			        *paui32ClientTAUpdateValue,
+			        *paui32ClientTAUpdateValue));
+			paui32ClientTAUpdateValue++;
+		}
+		pauiClientTAUpdateUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D fence syncs:",
+			 __func__, ui32Client3DFenceCount));
+	for (i = 0; i < ui32Client3DFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr,
+			        *paui32Client3DFenceValue,
+			        *paui32Client3DFenceValue));
+			paui32Client3DFenceValue++;
+		}
+		pauiClient3DFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D update syncs:",
+			 __func__, ui32Client3DUpdateCount));
+	for (i = 0; i < ui32Client3DUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr,
+			        *paui32Client3DUpdateValue,
+			        *paui32Client3DUpdateValue));
+			paui32Client3DUpdateValue++;
+		}
+		pauiClient3DUpdateUFOAddress++;
+	}
+}
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+/*
+ * PVRSRVRGXKickTA3DKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRFenceSyncPrimBlock,
+								 IMG_UINT32					ui32PRFenceSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckTAFence,
+								 PVRSRV_TIMELINE			iUpdateTATimeline,
+								 PVRSRV_FENCE				*piUpdateTAFence,
+								 IMG_CHAR					szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheck3DFence,
+								 PVRSRV_TIMELINE			iUpdate3DTimeline,
+								 PVRSRV_FENCE				*piUpdate3DFence,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus)
+{
+	/* per-context helper structures */
+	RGX_CCB_CMD_HELPER_DATA *pasTACmdHelperData = psRenderContext->asTACmdHelperData;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelperData = psRenderContext->as3DCmdHelperData;
+
+	IMG_UINT64                ui64FBSCEntryMask;
+
+	IMG_UINT32                ui32TACmdCount=0;
+	IMG_UINT32                ui323DCmdCount=0;
+	__maybe_unused IMG_UINT32 ui32TACmdOffset=0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset=0;
+
+	RGXFWIF_UFO        sPRUFO;
+	IMG_UINT32         i;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	PVRSRV_ERROR       eError2 = PVRSRV_OK;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psRenderContext->s3DData.psServerCommonContext);
+	IMG_UINT32         ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_BOOL           bCCBStateOpen = IMG_FALSE;
+
+	IMG_UINT32         ui32ClientPRUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR  *pauiClientPRUpdateUFOAddress = NULL;
+	IMG_UINT32         *paui32ClientPRUpdateValue = NULL;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	PRGXFWIF_UFO_ADDR  *pauiClientTAFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClientTAUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  uiPRFenceUFOAddress;
+
+	IMG_UINT64         uiCheckTAFenceUID = 0;
+	IMG_UINT64         uiCheck3DFenceUID = 0;
+	IMG_UINT64         uiUpdateTAFenceUID = 0;
+	IMG_UINT64         uiUpdate3DFenceUID = 0;
+
+	IMG_BOOL bUseCombined3DAnd3DPR = bKickPR && bKick3D && !pui83DPRDMCmd;
+
+	RGXFWIF_KCCB_CMD_KICK_DATA sTACmdKickData;
+	RGXFWIF_KCCB_CMD_KICK_DATA s3DCmdKickData;
+	IMG_BOOL bUseSingleFWCommand = bKickTA && (bKickPR || bKick3D);
+
+	IMG_UINT32 ui32TACmdSizeTmp = 0, ui323DCmdSizeTmp = 0;
+
+	IMG_BOOL bTAFenceOnSyncCheckpointsOnly = IMG_FALSE;
+
+	PVRSRV_FENCE iUpdateTAFence = PVRSRV_NO_FENCE;
+	PVRSRV_FENCE iUpdate3DFence = PVRSRV_NO_FENCE;
+
+	IMG_BOOL   b3DFenceOnSyncCheckpointsOnly = IMG_FALSE;
+	IMG_UINT32 ui32TAFenceTimelineUpdateValue = 0;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+
+	/*
+	 * Count of the number of TA and 3D update values (may differ from number of
+	 * TA and 3D updates later, as sync checkpoints do not need to specify a value)
+	 */
+	IMG_UINT32 ui32ClientPRUpdateValueCount = 0;
+	IMG_UINT32 ui32ClientTAUpdateValueCount = ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32Client3DUpdateValueCount = ui32Client3DUpdateCount;
+	PSYNC_CHECKPOINT *apsFenceTASyncCheckpoints = NULL;				/*!< TA fence checkpoints */
+	PSYNC_CHECKPOINT *apsFence3DSyncCheckpoints = NULL;				/*!< 3D fence checkpoints */
+	IMG_UINT32 ui32FenceTASyncCheckpointCount = 0;
+	IMG_UINT32 ui32Fence3DSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psUpdateTASyncCheckpoint = NULL;				/*!< TA update checkpoint (output) */
+	PSYNC_CHECKPOINT psUpdate3DSyncCheckpoint = NULL;				/*!< 3D update checkpoint (output) */
+	PVRSRV_CLIENT_SYNC_PRIM *psTAFenceTimelineUpdateSync = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	void *pvTAUpdateFenceFinaliseData = NULL;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+
+	RGX_SYNC_DATA sTASyncData = {NULL};		/*!< Contains internal update syncs for TA */
+	RGX_SYNC_DATA s3DSyncData = {NULL};		/*!< Contains internal update syncs for 3D */
+
+	IMG_BOOL bTestSLRAdd3DCheck = IMG_FALSE;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTA = {0};
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickData3D = {0};
+	IMG_UINT32 ui32TACommandOffset = 0;
+	IMG_UINT32 ui323DCommandOffset = 0;
+	IMG_UINT32 ui32TACmdHeaderOffset = 0;
+	IMG_UINT32 ui323DCmdHeaderOffset = 0;
+	IMG_UINT32 ui323DFullRenderCommandOffset = 0;
+	IMG_UINT32 ui32TACmdOffsetWrapCheck = 0;
+	IMG_UINT32 ui323DCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT32 ui32TAFenceCount, ui323DFenceCount;
+	IMG_UINT32 ui32TAUpdateCount, ui323DUpdateCount;
+	IMG_UINT32 ui32PRUpdateCount;
+
+	IMG_PID uiCurrentProcess = OSGetCurrentClientProcessIDKM();
+
+	IMG_UINT32 ui32Client3DFenceCount = 0;
+
+	/* Ensure we haven't been given a null ptr to
+	 * TA fence values if we have been told we
+	 * have TA sync prim fences
+	 */
+	if (ui32ClientTAFenceCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAFenceValue != NULL,
+		                        "paui32ClientTAFenceValue NULL but "
+		                        "ui32ClientTAFenceCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * TA update values if we have been told we
+	 * have TA updates
+	 */
+	if (ui32ClientTAUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAUpdateValue != NULL,
+		                        "paui32ClientTAUpdateValue NULL but "
+		                        "ui32ClientTAUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * 3D update values if we have been told we
+	 * have 3D updates
+	 */
+	if (ui32Client3DUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32Client3DUpdateValue != NULL,
+		                        "paui32Client3DUpdateValue NULL but "
+		                        "ui32Client3DUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Write FW addresses into CMD SHARED BLOCKs */
+	{
+		CMDTA3D_SHARED *psGeomCmdShared = (CMDTA3D_SHARED *)pui8TADMCmd;
+		CMDTA3D_SHARED *ps3DCmdShared = (CMDTA3D_SHARED *)pui83DDMCmd;
+		CMDTA3D_SHARED *psPR3DCmdShared = (CMDTA3D_SHARED *)pui83DPRDMCmd;
+		RGXFWIF_DEV_VIRTADDR sNullFWAddr = {0};
+
+		if (psKMHWRTDataSet == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "KMHWRTDataSet is a null-pointer"));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Write FW address for TA CMD
+		*/
+		if (psGeomCmdShared != NULL)
+		{
+			if (ui32TACmdSize < sizeof(*psGeomCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid TACmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psGeomCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for 3D CMD
+		*/
+		if (ps3DCmdShared != NULL)
+		{
+			if (ui323DCmdSize < sizeof(*ps3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			ps3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for PR3D CMD
+		*/
+		if (psPR3DCmdShared != NULL)
+		{
+			if (ui323DPRCmdSize < sizeof(*psPR3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DPRCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psPR3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+	}
+
+	if (unlikely(iUpdateTATimeline >= 0 && !piUpdateTAFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	if (unlikely(iUpdate3DTimeline >= 0 && !piUpdate3DFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d, "
+			   "ui32Client3DFenceCount=%d, ui32Client3DUpdateCount=%d",
+			   __func__,
+			   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+			   ui32Client3DFenceCount, ui32Client3DUpdateCount));
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRenderContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	/* Double-check we have a PR kick if there are client fences */
+	if (unlikely(!bKickPR && ui32Client3DFenceCount != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: 3D fence passed without a PR kick",
+		        __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+	szFenceName3D[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	ui32TAFenceCount = ui32ClientTAFenceCount;
+	ui323DFenceCount = ui32Client3DFenceCount;
+	ui32TAUpdateCount = ui32ClientTAUpdateCount;
+	ui323DUpdateCount = ui32Client3DUpdateCount;
+	ui32PRUpdateCount = ui32ClientPRUpdateCount;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32SyncPMRCount)
+	{
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling"
+		          " pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psRenderContext->psBufferSyncContext,
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%d)",
+				        __func__, eError));
+			}
+			OSLockRelease(psRenderContext->hLock);
+
+			return eError;
+		}
+
+#if !defined(SUPPORT_STRIP_RENDERING)
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+		{
+			ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+#else /* !defined(SUPPORT_STRIP_RENDERING) */
+		ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+
+		PVR_UNREFERENCED_PARAMETER(bTAFenceOnSyncCheckpointsOnly);
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+
+		if (psBufferUpdateSyncCheckpoint != NULL)
+		{
+			if (bKick3D)
+			{
+				ui323DUpdateCount++;
+			}
+			else
+			{
+				ui32PRUpdateCount++;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+	PVR_UNREFERENCED_PARAMETER(ppsSyncPMRs);
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2
+#error "Invalid value for UPDATE_FENCE_CHECKPOINT_COUNT. Must be either 1 or 2."
+#endif /* !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2 */
+
+	if (iCheckTAFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[TA]"
+		          " (iCheckFence=%d),"
+		          " psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheckTAFence,
+		          (void *) psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheckTAFence,
+		    &ui32FenceTASyncCheckpointCount,
+		    &apsFenceTASyncCheckpoints,
+		    &uiCheckTAFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_ta_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheckTAFence, ui32FenceTASyncCheckpointCount,
+		          (void *) apsFenceTASyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFenceTASyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "TA", apsFenceTASyncCheckpoints,
+			                      ui32FenceTASyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheck3DFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[3D]"
+		          " (iCheckFence=%d), "
+		          "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheck3DFence,
+		          (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheck3DFence,
+		    &ui32Fence3DSyncCheckpointCount,
+		    &apsFence3DSyncCheckpoints,
+		    &uiCheck3DFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_3d_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheck3DFence, ui32Fence3DSyncCheckpointCount,
+		          (void*)apsFence3DSyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFence3DSyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "3D", apsFence3DSyncCheckpoints,
+			                      ui32Fence3DSyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		IMG_UINT32 i;
+
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32FenceTASyncCheckpointCount;
+
+			for (i = 0; i < ui32Fence3DSyncCheckpointCount; i++)
+			{
+				if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) !=
+				    uiCurrentProcess)
+				{
+					ui32TAFenceCount++;
+				}
+			}
+		}
+
+		if (bKick3D)
+		{
+			ui323DFenceCount += ui32Fence3DSyncCheckpointCount;
+		}
+
+		ui32TAUpdateCount += iUpdateTATimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui323DUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui32PRUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE && !bKick3D ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+	}
+
+
+	/*
+	 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+	 * in other words, take the value and set it to zero afterwards.
+	 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+	 * as it must be ready at the time of context activation.
+	 *
+	 * NOTE: We use sTAData to get the ServerCommonContext giving us the ServerMMUCtx,
+	 *       should we use s3DData in some cases?
+	 *       Under assumption that sTAData and s3DData share the same psServerCommonContext,
+	 *       the answer is NO.
+	 *
+	 *       The ui64FBSCEntryMask filled by the following call gets cleared
+	 *       after the first KICK command and is ignored in the context of partial renders.
+	 */
+	eError = RGXExtractFBSCEntryMaskFromMMUContext(
+	    psRenderContext->psDeviceNode,
+	    FWCommonContextGetServerMMUCtx(psRenderContext->sTAData.psServerCommonContext),
+	    &ui64FBSCEntryMask
+	);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%d)", eError));
+		goto fail_tacmdinvalfbsc;
+	}
+
+	if (bKickTA)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+		          __func__, ui32TAFenceCount, ui32TAUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			ui64FBSCEntryMask,
+		    ui32TAFenceCount,
+		    ui32TAUpdateCount,
+		    ui32TACmdSize,
+		    &pPreAddr,
+		    (bKick3D ? NULL : &pPostAddr),
+		    (bKick3D ? NULL : &pRMWUFOAddr),
+		    pasTACmdHelperData
+		);
+
+		/* Clear the mask as we don't want to invalidate the FBSC multiple times
+		 * with the same value of ui64FBSCEntryMask.
+		 */
+		ui64FBSCEntryMask = 0;
+	}
+
+	if (bKickPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0, /* empty ui64FBSCEntryMask it is assumed that PRs should
+		        * not invalidate FBSC */
+		    ui323DFenceCount,
+		    0,
+		    sizeof(sPRUFO),
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickPR && !bUseCombined3DAnd3DPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32PRUpdateCount=%d", __func__,
+		          ui32PRUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0, /* empty ui64FBSCEntryMask it is assumed that PRs should
+		        * not invalidate FBSC */
+		    0,
+		    ui32PRUpdateCount,
+		    /* if the client has not provided a 3DPR command, the regular 3D
+		     * command should be used instead */
+		    pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize,
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKick3D || bAbort)
+	{
+		if (!bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+		}
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			ui64FBSCEntryMask, /* equals: [a] 0 if 3D is preceded by TA
+		                        *         [b] value from the MMU ctx otherwise */
+			bKickTA ? 0 : ui323DFenceCount,
+		    ui323DUpdateCount,
+		    ui323DCmdSize,
+			(bKickTA ? NULL : & pPreAddr),
+			&pPostAddr,
+			&pRMWUFOAddr,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickTA)
+	{
+		ui32TACmdSizeTmp = RGXCmdHelperGetCommandSize(1, pasTACmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext),
+		    ui32TACmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	if (ui323DCmdCount > 0)
+	{
+		ui323DCmdSizeTmp = RGXCmdHelperGetCommandSize(ui323DCmdCount, pas3DCmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext),
+		    ui323DCmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	/* need to reset the counter here */
+
+	ui323DCmdCount = 0;
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAFence, %d fences)...",
+			   __func__, ui32ClientTAFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
+										ui32ClientTAFenceCount,
+										apsClientTAFenceSyncPrimBlock,
+										paui32ClientTAFenceSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_fence;
+	}
+
+	if (ui32ClientTAFenceCount)
+	{
+		pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAUpdate, %d updates)...",
+			   __func__, ui32ClientTAUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
+										ui32ClientTAUpdateCount,
+										apsClientTAUpdateSyncPrimBlock,
+										paui32ClientTAUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_update;
+	}
+
+	if (ui32ClientTAUpdateCount)
+	{
+		pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAUpdateUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DFence, %d fences)...",
+			   __func__, ui32Client3DFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
+										ui32Client3DFenceCount,
+										NULL,
+										NULL);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_fence;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DUpdate, %d updates)...",
+			   __func__, ui32Client3DUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
+										ui32Client3DUpdateCount,
+										apsClient3DUpdateSyncPrimBlock,
+										paui32Client3DUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_update;
+	}
+
+	if (ui32Client3DUpdateCount || (iUpdate3DTimeline != PVRSRV_NO_TIMELINE && piUpdate3DFence && bKick3D))
+	{
+		pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DUpdateUFOAddress));
+
+	eError = SyncPrimitiveBlockToFWAddr(psPRFenceSyncPrimBlock, ui32PRFenceSyncOffset, &uiPRFenceUFOAddress);
+
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_pr_fence_address;
+	}
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	if (ui32SyncPMRCount)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+#if !defined(SUPPORT_STRIP_RENDERING)
+		/* Append buffer sync fences to TA fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0 && bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to TA Fence "
+					   "(&psRenderContext->sSyncAddrListTAFence=<%p>, "
+					   "pauiClientTAFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrListTAFence ,
+					   (void*)pauiClientTAFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClientTAFenceUFOAddress)
+			{
+				pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+			}
+			if (ui32ClientTAFenceCount == 0)
+			{
+				bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32ClientTAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+		/* Append buffer sync fences to 3D fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to 3D Fence "
+					   "(&psRenderContext->sSyncAddrList3DFence=<%p>, "
+					   "pauiClient3DFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrList3DFence,
+					   (void*)pauiClient3DFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+					ui32BufferFenceSyncCheckpointCount,
+					apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClient3DFenceUFOAddress)
+			{
+				pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+			}
+			if (ui32Client3DFenceCount == 0)
+			{
+				b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32Client3DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			/* If we have a 3D kick append update to the 3D updates else append to the PR update */
+			if (bKick3D)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   Append 1 buffer sync checkpoint<%p> to 3D Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClient3DUpdateUFOAddress=<%p>)...",
+						   __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClient3DUpdateUFOAddress));
+				/* Append buffer sync update to 3D updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClient3DUpdateUFOAddress)
+				{
+					pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32Client3DUpdateCount++;
+			}
+			else
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+				           "%s:   Append 1 buffer sync checkpoint<%p> to PR Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClientPRUpdateUFOAddress=<%p>)...",
+				           __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClientPRUpdateUFOAddress));
+				/* Attach update to the 3D (used for PR) Updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClientPRUpdateUFOAddress)
+				{
+					pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32ClientPRUpdateCount++;
+			}
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   (after buffer_sync) ui32ClientTAFenceCount=%d, "
+				   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+				   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+				   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+				   ui32ClientPRUpdateCount));
+
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Buffer sync not supported but got %u buffers",
+				 __func__, ui32SyncPMRCount));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_no_buffer_sync_invalid_params;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	/*
+	 * The hardware requires a PR to be submitted if there is a TA (otherwise
+	 * it can wedge if we run out of PB space with no PR to run)
+	 *
+	 * If we only have a TA, attach native checks to the TA and updates to the PR
+	 * If we have a TA and 3D, attach checks to TA, updates to 3D
+	 * If we only have a 3D, attach checks and updates to the 3D
+	 *
+	 * Note that 'updates' includes the cleanup syncs for 'check' fence FDs, in
+	 * addition to the update fence FD (if supplied)
+	 *
+	 * Currently, the client driver never kicks only the 3D, so we only support
+	 * that for the time being.
+	 */
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [TA] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheckTAFence, iUpdateTATimeline));
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [3D] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheck3DFence, iUpdate3DTimeline));
+
+		{
+			/* Create the output fence for TA (if required) */
+			if (iUpdateTATimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdateTAFence, iUpdateTATimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceNameTA,
+						iUpdateTATimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdateTAFence,
+						&uiUpdateTAFenceUID,
+						&pvTAUpdateFenceFinaliseData,
+						&psUpdateTASyncCheckpoint,
+						(void*)&psTAFenceTimelineUpdateSync,
+						&ui32TAFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[TA] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_ta_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdateTAFence,
+						   (void*)psTAFenceTimelineUpdateSync,
+						   ui32TAFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClientTAIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClientTAIntUpdateUFOAddress */
+					pauiClientTAIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdateTASyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[TA]->ui32Addr=0x%x",
+							   __func__, pauiClientTAIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the TA timeline (if required) */
+			if (psTAFenceTimelineUpdateSync)
+			{
+				sTASyncData.ui32ClientUpdateCount		 = ui32ClientTAUpdateCount;
+				sTASyncData.ui32ClientUpdateValueCount	 = ui32ClientTAUpdateValueCount;
+				sTASyncData.ui32ClientPRUpdateValueCount = (bKick3D) ? 0 : ui32ClientPRUpdateValueCount;
+				sTASyncData.paui32ClientUpdateValue		 = paui32ClientTAUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui32TAFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrListTAUpdate,
+											(bKick3D) ? NULL : &psRenderContext->sSyncAddrList3DUpdate,
+											psTAFenceTimelineUpdateSync,
+											&sTASyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_TA;
+				}
+
+				paui32ClientTAUpdateValue = sTASyncData.paui32ClientUpdateValue;
+				ui32ClientTAUpdateValueCount = sTASyncData.ui32ClientUpdateValueCount;
+				pauiClientTAUpdateUFOAddress = sTASyncData.pauiClientUpdateUFOAddress;
+				ui32ClientTAUpdateCount = sTASyncData.ui32ClientUpdateCount;
+			}
+
+			/* Create the output fence for 3D (if required) */
+			if (iUpdate3DTimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdate3DFence, iUpdate3DTimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceName3D,
+						iUpdate3DTimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdate3DFence,
+						&uiUpdate3DFenceUID,
+						&pv3DUpdateFenceFinaliseData,
+						&psUpdate3DSyncCheckpoint,
+						(void*)&ps3DFenceTimelineUpdateSync,
+						&ui323DFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[3D] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_3d_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdate3DFence,
+						   (void*)ps3DFenceTimelineUpdateSync,
+						   ui323DFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClient3DIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClient3DIntUpdateUFOAddress */
+					pauiClient3DIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdate3DSyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[3D]->ui32Addr=0x%x",
+							   __func__, pauiClient3DIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the 3D timeline (if required) */
+			if (ps3DFenceTimelineUpdateSync)
+			{
+				s3DSyncData.ui32ClientUpdateCount = ui32Client3DUpdateCount;
+				s3DSyncData.ui32ClientUpdateValueCount = ui32Client3DUpdateValueCount;
+				s3DSyncData.ui32ClientPRUpdateValueCount = ui32ClientPRUpdateValueCount;
+				s3DSyncData.paui32ClientUpdateValue = paui32Client3DUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui323DFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrList3DUpdate,
+											&psRenderContext->sSyncAddrList3DUpdate,	/*!< PR update: is this required? */
+											ps3DFenceTimelineUpdateSync,
+											&s3DSyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_3D;
+				}
+
+				paui32Client3DUpdateValue = s3DSyncData.paui32ClientUpdateValue;
+				ui32Client3DUpdateValueCount = s3DSyncData.ui32ClientUpdateValueCount;
+				pauiClient3DUpdateUFOAddress = s3DSyncData.pauiClientUpdateUFOAddress;
+				ui32Client3DUpdateCount = s3DSyncData.ui32ClientUpdateCount;
+
+				if (!bKick3D)
+				{
+					paui32ClientPRUpdateValue = s3DSyncData.paui32ClientPRUpdateValue;
+					ui32ClientPRUpdateValueCount = s3DSyncData.ui32ClientPRUpdateValueCount;
+					pauiClientPRUpdateUFOAddress = s3DSyncData.pauiClientPRUpdateUFOAddress;
+					ui32ClientPRUpdateCount = s3DSyncData.ui32ClientPRUpdateCount;
+				}
+			}
+
+			/*
+			 * The hardware requires a PR to be submitted if there is a TA OOM.
+			 * If we only have a TA, attach native checks and updates to the TA
+			 * and 3D updates to the PR.
+			 * If we have a TA and 3D, attach the native TA checks and updates
+			 * to the TA and similarly for the 3D.
+			 * Note that 'updates' includes the cleanup syncs for 'check' fence
+			 * FDs, in addition to the update fence FD (if supplied).
+			 * Currently, the client driver never kicks only the 3D, so we don't
+			 * support that for the time being.
+			 */
+
+			{
+				if (bKickTA)
+				{
+					/* Attach checks and updates to TA */
+
+					/* Checks (from input fence) */
+					if (ui32FenceTASyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to TA Fence (apsFenceSyncCheckpoints=<%p>)...",
+								   __func__,
+								   ui32FenceTASyncCheckpointCount,
+								   (void*)apsFenceTASyncCheckpoints));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+													  ui32FenceTASyncCheckpointCount,
+													  apsFenceTASyncCheckpoints);
+						if (!pauiClientTAFenceUFOAddress)
+						{
+							pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+								   __func__, ui32ClientTAFenceCount,
+								   ui32ClientTAFenceCount + ui32FenceTASyncCheckpointCount));
+						if (ui32ClientTAFenceCount == 0)
+						{
+							bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32ClientTAFenceCount += ui32FenceTASyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32ClientTAFenceCount now %d}",
+							   __func__, ui32ClientTAFenceCount));
+
+					if (psUpdateTASyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Update...",
+								   __func__, (void*)psUpdateTASyncCheckpoint,
+								   SyncCheckpointGetId(psUpdateTASyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAUpdate,
+													  1,
+													  &psUpdateTASyncCheckpoint);
+						if (!pauiClientTAUpdateUFOAddress)
+						{
+							pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+						}
+						ui32ClientTAUpdateCount++;
+					}
+
+					if (!bKick3D && psUpdate3DSyncCheckpoint)
+					{
+						/* Attach update to the 3D (used for PR) Updates */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D(PR) Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClientPRUpdateUFOAddress)
+						{
+							pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32ClientPRUpdateCount++;
+					}
+				}
+
+				if (bKick3D)
+				{
+					/* Attach checks and updates to the 3D */
+
+					/* Checks (from input fence) */
+					if (ui32Fence3DSyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to 3D Fence...",
+								   __func__, ui32Fence3DSyncCheckpointCount));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+													  ui32Fence3DSyncCheckpointCount,
+													  apsFence3DSyncCheckpoints);
+						if (!pauiClient3DFenceUFOAddress)
+						{
+							pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32Client3DFenceCount was %d, now %d}",
+								   __func__, ui32Client3DFenceCount,
+								   ui32Client3DFenceCount + ui32Fence3DSyncCheckpointCount));
+						if (ui32Client3DFenceCount == 0)
+						{
+							b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32Client3DFenceCount += ui32Fence3DSyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32Client3DFenceCount was %d}",
+							   __func__, ui32Client3DFenceCount));
+
+					if (psUpdate3DSyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClient3DUpdateUFOAddress)
+						{
+							pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32Client3DUpdateCount++;
+					}
+				}
+
+				/*
+				 * Relocate sync check points from the 3D fence that are
+				 * external to the current process, to the TA fence.
+				 * This avoids a sync lockup when dependent renders are
+				 * submitted out-of-order and a PR must be scheduled.
+				 */
+				if (bKickTA)
+				{
+					/* Search for external timeline dependencies */
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: Checking 3D fence for external sync points (%d)...",
+							   __func__, ui32Fence3DSyncCheckpointCount));
+
+					for (i=0; i<ui32Fence3DSyncCheckpointCount; i++)
+					{
+						/* Check to see if the checkpoint is on a TL owned by
+						 * another process.
+						 */
+						if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) != uiCurrentProcess)
+						{
+							/* 3D Sync point represents cross process
+							 * dependency, copy sync point to TA command fence. */
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Fence...",
+									   __func__, (void*)apsFence3DSyncCheckpoints[i],
+									   SyncCheckpointGetId(apsFence3DSyncCheckpoints[i])));
+
+							SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+														  1,
+														  &apsFence3DSyncCheckpoints[i]);
+
+							if (!pauiClientTAFenceUFOAddress)
+							{
+								pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+							}
+
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+									   __func__,
+									   ui32ClientTAFenceCount,
+									   ui32ClientTAFenceCount + 1));
+
+							if (ui32ClientTAFenceCount == 0)
+							{
+								bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+							}
+
+							ui32ClientTAFenceCount++;
+						}
+					}
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   (after pvr_sync) ui32ClientTAFenceCount=%d, "
+						   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+						   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+						   __func__,
+						   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+						   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+						   ui32ClientPRUpdateCount));
+			}
+		}
+
+		if (ui32ClientTAFenceCount)
+		{
+			PVR_ASSERT(pauiClientTAFenceUFOAddress);
+			if (!bTAFenceOnSyncCheckpointsOnly)
+			{
+				PVR_ASSERT(paui32ClientTAFenceValue);
+			}
+		}
+		if (ui32ClientTAUpdateCount)
+		{
+			PVR_ASSERT(pauiClientTAUpdateUFOAddress);
+			if (ui32ClientTAUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientTAUpdateValue);
+			}
+		}
+		if (ui32Client3DFenceCount)
+		{
+			PVR_ASSERT(pauiClient3DFenceUFOAddress);
+			PVR_ASSERT(b3DFenceOnSyncCheckpointsOnly);
+		}
+		if (ui32Client3DUpdateCount)
+		{
+			PVR_ASSERT(pauiClient3DUpdateUFOAddress);
+			if (ui32Client3DUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32Client3DUpdateValue);
+			}
+		}
+		if (ui32ClientPRUpdateCount)
+		{
+			PVR_ASSERT(pauiClientPRUpdateUFOAddress);
+			if (ui32ClientPRUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientPRUpdateValue);
+			}
+		}
+
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, pauiClientTAFenceUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAFenceCount,
+			   (void*)paui32ClientTAFenceValue));
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAUpdateCount=%d, pauiClientTAUpdateUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAUpdateCount,
+			   (void*)pauiClientTAUpdateUFOAddress));
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	/* Command size check */
+	if (ui32TAFenceCount != ui32ClientTAFenceCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAFenceCount, ui32ClientTAFenceCount));
+	}
+	if (ui32TAUpdateCount != ui32ClientTAUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAUpdateCount, ui32ClientTAUpdateCount));
+	}
+	if (!bTestSLRAdd3DCheck && (ui323DFenceCount != ui32Client3DFenceCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui323DFenceCount, ui32Client3DFenceCount));
+	}
+	if (ui323DUpdateCount != ui32Client3DUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui323DUpdateCount, ui32Client3DUpdateCount));
+	}
+	if (ui32PRUpdateCount != ui32ClientPRUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PR pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32PRUpdateCount, ui32ClientPRUpdateCount));
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)) && (bKickTA || bKick3D || bAbort))
+	{
+		sWorkloadCharacteristics.sTA3D.ui32RenderTargetSize  = ui32RenderTargetSize;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfDrawCalls = ui32NumberOfDrawCalls;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfIndices   = ui32NumberOfIndices;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfMRTs      = ui32NumberOfMRTs;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32RenderTargetSize);
+	PVR_UNREFERENCED_PARAMETER(ui32NumberOfDrawCalls);
+	PVR_UNREFERENCED_PARAMETER(ui32NumberOfIndices);
+	PVR_UNREFERENCED_PARAMETER(ui32NumberOfMRTs);
+#endif
+
+	/* Init and acquire to TA command if required */
+	if (bKickTA)
+	{
+		RGX_SERVER_RC_TA_DATA *psTAData = &psRenderContext->sTAData;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sDataTA,
+					RGXFWIF_CCB_CMD_TYPE_GEOM,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTA);
+		}
+#endif
+
+		/* Init the TA command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(psTAData->psServerCommonContext),
+		                                 ui32ClientTAFenceCount,
+		                                 pauiClientTAFenceUFOAddress,
+		                                 paui32ClientTAFenceValue,
+		                                 ui32ClientTAUpdateCount,
+		                                 pauiClientTAUpdateUFOAddress,
+		                                 paui32ClientTAUpdateValue,
+		                                 ui32TACmdSize,
+		                                 pui8TADMCmd,
+								         &pPreAddr,
+								         (bKick3D ? NULL : & pPostAddr),
+								         (bKick3D ? NULL : & pRMWUFOAddr),
+		                                 RGXFWIF_CCB_CMD_TYPE_GEOM,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickDataTA,
+#else
+		                                 NULL,
+#endif
+		                                 "TA",
+		                                 bCCBStateOpen,
+		                                 pasTACmdHelperData);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following is used to determine the offset of the command header containing
+			   the workload estimation data so that can be accessed when the KCCB is read */
+			ui32TACmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(pasTACmdHelperData);
+		}
+#endif
+
+		eError = RGXCmdHelperAcquireCmdCCB(CCB_CMD_HELPER_NUM_TA_COMMANDS, pasTACmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taacquirecmd;
+		}
+		else
+		{
+			ui32TACmdCount++;
+		}
+	}
+
+	/* Only kick the 3D if required */
+	if (bKickPR)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+
+		/*
+			The command helper doesn't know about the PR fence so create
+			the command with all the fences against it and later create
+			the PR command itself which _must_ come after the PR fence.
+		*/
+		sPRUFO.puiAddrUFO = uiPRFenceUFOAddress;
+		sPRUFO.ui32Value = ui32PRFenceValue;
+
+		/* Init the PR fence command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32Client3DFenceCount=%d",
+				   __func__, ui32Client3DFenceCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+		                                 pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 0,
+		                                 NULL,
+		                                 NULL,
+		                                 sizeof(sPRUFO),
+		                                 (IMG_UINT8*) &sPRUFO,
+		                                 NULL,
+		                                 NULL,
+		                                 NULL,
+		                                 RGXFWIF_CCB_CMD_TYPE_FENCE_PR,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+		                                 NULL,
+		                                 "3D-PR-Fence",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+		/* Init the 3D PR command helper */
+		/*
+			Updates for Android (fence sync and Timeline sync prim) are provided in the PR-update
+			if no 3D is present. This is so the timeline update cannot happen out of order with any
+			other 3D already in flight for the same timeline (PR-updates are done in the 3D cCCB).
+			This out of order timeline sync prim update could happen if we attach it to the TA update.
+		*/
+		if (ui32ClientPRUpdateCount)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s: Line %d, ui32ClientPRUpdateCount=%d, "
+					   "pauiClientPRUpdateUFOAddress=0x%x, "
+					   "ui32ClientPRUpdateValueCount=%d, "
+					   "paui32ClientPRUpdateValue=0x%x",
+					   __func__, __LINE__, ui32ClientPRUpdateCount,
+					   pauiClientPRUpdateUFOAddress->ui32Addr,
+					   ui32ClientPRUpdateValueCount,
+					   (ui32ClientPRUpdateValueCount == 0) ? PVRSRV_SYNC_CHECKPOINT_SIGNALLED : *paui32ClientPRUpdateValue));
+		}
+
+		if (!bUseCombined3DAnd3DPR)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientPRUpdateCount=%d",
+					   __func__, ui32ClientPRUpdateCount));
+			RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+			                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+			                                 0,
+			                                 NULL,
+			                                 NULL,
+			                                 ui32ClientPRUpdateCount,
+			                                 pauiClientPRUpdateUFOAddress,
+			                                 paui32ClientPRUpdateValue,
+			                                 pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize, // If the client has not provided a 3DPR command, the regular 3D command should be used instead
+			                                 pui83DPRDMCmd ? pui83DPRDMCmd : pui83DDMCmd,
+			                                 NULL,
+			                                 NULL,
+			                                 NULL,
+			                                 RGXFWIF_CCB_CMD_TYPE_3D_PR,
+			                                 ui32ExtJobRef,
+			                                 ui32IntJobRef,
+			                                 ui32PDumpFlags,
+			                                 NULL,
+			                                 "3D-PR",
+			                                 bCCBStateOpen,
+			                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+		}
+	}
+
+	if (bKick3D || bAbort)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+		const RGXFWIF_CCB_CMD_TYPE e3DCmdType = bAbort ? RGXFWIF_CCB_CMD_TYPE_ABORT : RGXFWIF_CCB_CMD_TYPE_3D;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sData3D,
+					e3DCmdType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickData3D);
+		}
+#endif
+
+		/* Init the 3D command helper */
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 bKickTA ? 0 : ui32Client3DFenceCount,  /* For a kick with a TA, the 3D fences are added before the PR command instead */
+		                                 bKickTA ? NULL : pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 ui32Client3DUpdateCount,
+		                                 pauiClient3DUpdateUFOAddress,
+		                                 paui32Client3DUpdateValue,
+		                                 ui323DCmdSize,
+		                                 pui83DDMCmd,
+		                                 (bKickTA ? NULL : & pPreAddr),
+		                                 &pPostAddr,
+		                                 &pRMWUFOAddr,
+		                                 e3DCmdType,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickData3D,
+#else
+		                                 NULL,
+#endif
+		                                 "3D",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following are used to determine the offset of the command header containing the workload estimation
+			   data so that can be accessed when the KCCB is read */
+			ui323DCmdHeaderOffset =	RGXCmdHelperGetDMCommandHeaderOffset(&pas3DCmdHelperData[ui323DCmdCount - 1]);
+			ui323DFullRenderCommandOffset =	RGXCmdHelperGetCommandOffset(pas3DCmdHelperData, ui323DCmdCount - 1);
+		}
+#endif
+	}
+
+	/* Protect against array overflow in RGXCmdHelperAcquireCmdCCB() */
+	if (unlikely(ui323DCmdCount > CCB_CMD_HELPER_NUM_3D_COMMANDS))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+		goto fail_3dcmdinit;
+	}
+
+	if (ui323DCmdCount)
+	{
+		PVR_ASSERT(bKickPR || bKick3D);
+
+		/* Acquire space for all the 3D command(s) */
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount, pas3DCmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			/* If RGXCmdHelperAcquireCmdCCB fails we skip the scheduling
+			 * of a new TA command with the same Write offset in Kernel CCB.
+			 */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+			goto fail_3dacquirecmd;
+		}
+	}
+
+	if (ui32TACmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &sTACmdKickData.apsCleanupCtl,
+				&sTACmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_GEOM,
+				bKickTA,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taattachcleanupctls;
+		}
+	}
+
+	if (ui323DCmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &s3DCmdKickData.apsCleanupCtl,
+				&s3DCmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_3D,
+				bKick3D,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_3dattachcleanupctls;
+		}
+	}
+
+	/*
+		We should acquire the space in the kernel CCB here as after this point
+		we release the commands which will take operations on server syncs
+		which can't be undone
+	*/
+
+	/*
+		Everything is ready to go now, release the commands
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui32TACmdCount)
+	{
+		ui32TACmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui32TACmdCount,
+								  pasTACmdHelperData,
+								  "TA",
+								  FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui32TACmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+
+			/* This checks if the command would wrap around at the end of the CCB and therefore  would start at an
+			   offset of 0 rather than the current command offset */
+			if (ui32TACmdOffset < ui32TACmdOffsetWrapCheck)
+			{
+				ui32TACommandOffset = ui32TACmdOffset;
+			}
+			else
+			{
+				ui32TACommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  pas3DCmdHelperData,
+								  "3D",
+								  FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui323DCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+
+			if (ui323DCmdOffset < ui323DCmdOffsetWrapCheck)
+			{
+				ui323DCommandOffset = ui323DCmdOffset;
+			}
+			else
+			{
+				ui323DCommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui32TACmdCount)
+	{
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext);
+		CMDTA3D_SHARED *psGeomCmdShared = IMG_OFFSET_ADDR(pui8TADMCmd, 0);
+
+		sTACmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext);
+		sTACmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTACmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Add the Workload data into the KCCB kick */
+			sTACmdKickData.ui32WorkEstCmdHeaderOffset = ui32TACommandOffset + ui32TACmdHeaderOffset;
+		}
+#endif
+
+		if (psGeomCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_TA,
+					sTACmdKickData.psContext,
+					ui32TACmdOffset,
+					psGeomCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_GEOM,
+		                  iCheckTAFence,
+		                  iUpdateTAFence,
+		                  iUpdateTATimeline,
+		                  uiCheckTAFenceUID,
+		                  uiUpdateTAFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickDataTA));
+
+		if (!bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA CCB command. */
+			RGXFWIF_KCCB_CMD sTAKCCBCmd;
+			sTAKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			sTAKCCBCmd.uCmdData.sCmdKickData = sTACmdKickData;
+
+			LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+			{
+				eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+						RGXFWIF_DM_GEOM,
+						&sTAKCCBCmd,
+						ui32PDumpFlags);
+				if (eError2 != PVRSRV_ERROR_RETRY)
+				{
+					break;
+				}
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+		}
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			/* Mark the error and bail out */
+			eError = eError2;
+			goto fail_tasubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_GEOM);
+	}
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd = { 0 };
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext);
+		CMDTA3D_SHARED *ps3DCmdShared = IMG_OFFSET_ADDR(pui83DDMCmd, 0);
+
+		s3DCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+		s3DCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		s3DCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced when the KCCB command reaches the FW */
+			s3DCmdKickData.ui32WorkEstCmdHeaderOffset = ui323DCommandOffset + ui323DCmdHeaderOffset + ui323DFullRenderCommandOffset;
+		}
+#endif
+
+		if (ps3DCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_3D,
+					s3DCmdKickData.psContext,
+					ui323DCmdOffset,
+					ps3DCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_3D,
+		                  iCheck3DFence,
+		                  iUpdate3DFence,
+		                  iUpdate3DTimeline,
+		                  uiCheck3DFenceUID,
+		                  uiUpdate3DFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickData3D));
+
+		if (bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA/3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.sTACmdKickData = sTACmdKickData;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.s3DCmdKickData = s3DCmdKickData;
+		}
+		else
+		{
+			/* Construct the kernel 3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			s3DKCCBCmd.uCmdData.sCmdKickData = s3DCmdKickData;
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+										 RGXFWIF_DM_3D,
+										 &s3DKCCBCmd,
+										 ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			if (eError == PVRSRV_OK)
+			{
+				eError = eError2;
+			}
+			goto fail_3dsubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_3D);
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (unlikely(eError != PVRSRV_OK ))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+				   __func__, eError));
+		goto fail_3dsubmitcmd;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateTASyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [TA] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateTASyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateTASyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateTASyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateTASyncCheckpoint);
+	}
+	if (psTAFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [TA] <%p> to %d",
+				   __func__, (void*)psTAFenceTimelineUpdateSync,
+				   ui32TAFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psTAFenceTimelineUpdateSync, ui32TAFenceTimelineUpdateValue);
+	}
+
+	if (psUpdate3DSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [3D] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdate3DSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdate3DSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdate3DSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdate3DSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [3D] <%p> to %d",
+				   __func__, (void*)ps3DFenceTimelineUpdateSync,
+				   ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling pvr_buffer_sync_kick_succeeded(psBufferSyncData=<%p>)...",
+				   __func__, (void*)psBufferSyncData));
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	if (piUpdateTAFence)
+	{
+		*piUpdateTAFence = iUpdateTAFence;
+	}
+	if (piUpdate3DFence)
+	{
+		*piUpdate3DFence = iUpdate3DFence;
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+
+	if (pvTAUpdateFenceFinaliseData && (iUpdateTAFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdateTAFence,
+									pvTAUpdateFenceFinaliseData,
+									psUpdateTASyncCheckpoint, szFenceNameTA);
+	}
+	if (pv3DUpdateFenceFinaliseData && (iUpdate3DFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdate3DFence,
+									pv3DUpdateFenceFinaliseData,
+									psUpdate3DSyncCheckpoint, szFenceName3D);
+	}
+
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_3dsubmitcmd:
+fail_tasubmitcmd:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_3dattachcleanupctls:
+fail_taattachcleanupctls:
+fail_3dacquirecmd:
+fail_3dcmdinit:
+fail_taacquirecmd:
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DUpdate);
+	/* Where a TA-only kick (ie no 3D) is submitted, the PR update will make use of the unused 3DUpdate list.
+	 * If this has happened, performing a rollback on pauiClientPRUpdateUFOAddress will simply repeat what
+	 * has already been done for the sSyncAddrList3DUpdate above and result in a double decrement of the
+	 * sync checkpoint's hEnqueuedCCBCount, so we need to check before rolling back the PRUpdate.
+	 */
+	if (pauiClientPRUpdateUFOAddress && (pauiClientPRUpdateUFOAddress != psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs))
+	{
+		SyncCheckpointRollbackFromUFO(psRenderContext->psDeviceNode, pauiClientPRUpdateUFOAddress->ui32Addr);
+	}
+
+fail_alloc_update_values_mem_3D:
+	if (iUpdate3DFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdate3DFence, pv3DUpdateFenceFinaliseData);
+	}
+fail_create_3d_fence:
+fail_alloc_update_values_mem_TA:
+	if (iUpdateTAFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateTAFence, pvTAUpdateFenceFinaliseData);
+	}
+fail_create_ta_fence:
+#if !defined(SUPPORT_BUFFER_SYNC)
+err_no_buffer_sync_invalid_params:
+#endif /* !defined(SUPPORT_BUFFER_SYNC) */
+err_pr_fence_address:
+err_populate_sync_addr_list_3d_update:
+err_populate_sync_addr_list_3d_fence:
+err_populate_sync_addr_list_ta_update:
+err_populate_sync_addr_list_ta_fence:
+err_not_enough_space:
+fail_tacmdinvalfbsc:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		SyncAddrListDeRefCheckpoints(ui32BufferFenceSyncCheckpointCount,
+					     apsBufferFenceSyncCheckpoints);
+	}
+#endif
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+fail_resolve_input_3d_fence:
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+fail_resolve_input_ta_fence:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	PVR_ASSERT(eError != PVRSRV_OK);
+	OSLockRelease(psRenderContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD		sDisableZSStoreCmd = { 0 };
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	sDisableZSStoreCmd.eCmdType = RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableZStore = bDisableDepthStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableSStore = bDisableStencilStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.ui32ExtJobRefToDisableZSStore = ui32ExtJobRefToDisableZSStore;
+
+	if (psRenderContext->ui32CleanupStatus & (RC_CLEANUP_TA_COMPLETE | RC_CLEANUP_3D_COMPLETE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: submit disable depth and stencil store command to render context that has been cleaned up (%u)",
+				__func__, PVRSRV_ERROR_INVALID_CONTEXT));
+		return PVRSRV_ERROR_INVALID_CONTEXT;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_3D,
+									&sDisableZSStoreCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit disable depth and stencil store command (%u)",
+				__func__,
+				eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	if (psRenderContext->sTAData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->sTAData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_GEOM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the TA part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_tacontext;
+		}
+		psRenderContext->sTAData.i32Priority = i32Priority;
+	}
+
+	if (psRenderContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->s3DData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the 3D part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_3dcontext;
+		}
+		psRenderContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+fail_tacontext:
+	OSLockRelease(psRenderContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 RGX_CONTEXT_PROPERTY eContextProperty,
+                                                 IMG_UINT64 ui64Input,
+                                                 IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psRenderContext->hLock);
+			eError = FWCommonContextSetFlags(psRenderContext->sTAData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psRenderContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psRenderContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->sTAData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->s3DData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+		if (NULL != psCurrentServerRenderCtx->sTAData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->sTAData.psServerCommonContext, RGX_KICK_TYPE_DM_TA) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TA;
+			}
+		}
+
+		if (NULL != psCurrentServerRenderCtx->s3DData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_3D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * RGXRenderContextStalledKM
+ */
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	RGXCheckForStalledClientContexts((PVRSRV_RGXDEV_INFO *) psRenderContext->psDeviceNode->pvDevice, IMG_TRUE);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxta3d.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.h
new file mode 100644
index 000000000000..c7cf0edfc108
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxta3d.h
@@ -0,0 +1,506 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA and 3D Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX TA and 3D Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXTA3D_H
+#define RGXTA3D_H
+
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_shared.h"
+#include "rgx_fwif_resetframework.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+typedef struct _RGX_SERVER_RENDER_CONTEXT_ RGX_SERVER_RENDER_CONTEXT;
+typedef struct _RGX_FREELIST_ RGX_FREELIST;
+typedef struct _RGX_PMR_NODE_ RGX_PMR_NODE;
+
+/*****************************************************************************
+ * The Design of Data Storage System for Render Targets                      *
+ * ====================================================                      *
+ *   Relevant for                                                            *
+ *     understanding RGXCreateHWRTDataSet & RGXDestroyHWRTDataSet            *
+ *                                                                           *
+ *                                                                           *
+ *        +=========================================+                        *
+ *        |           RenderTargetDataSet           |                        *
+ *        +---------------|---------|---------------+                        *
+ *                        |         |                                        *
+ *                        V         V                                        *
+ *  +- - - - - - - - - - - - +   +- - - - - - - - - - - - +                  *
+ *  | KM_HW_RT_DATA_HANDLE_0 |   | KM_HW_RT_DATA_HANDLE_1 |                  *
+ *  +- - -|- - - - - - - - - +   +- - - - - - - - - | - - +                  *
+ *        |                                         |                        *
+ *        |                                         |           [UM]Client   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |               Bridge   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |           [KM]Server   *
+ *        |                                         |                        *
+ *        | KM-ptr                                  | KM-ptr                 *
+ *        V                                         V                        *
+ *  +====================+           +====================+                  *
+ *  |  KM_HW_RT_DATA_0   |           |   KM_HW_RT_DATA_1  |                  *
+ *  +-----|------------|-+           +-|------------|-----+                  *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            | KM-ptr        | KM-ptr     |                        *
+ *        |            V               V            |                        *
+ *        |      +==========================+       |                        *
+ *        |      | HW_RT_DATA_COMMON_COOKIE |       |                        *
+ *        |      +--------------------------+       |                        *
+ *        |                   |                     |                        *
+ *        |                   |                     |                        *
+ *  ------|-------------------|---------------------|----------------------- *
+ *        |                   |                     |         [FW]Firmware   *
+ *        |                   |                     |                        *
+ *        | FW-addr           |                     | FW-addr                *
+ *        V                   |                     V                        *
+ *  +===============+         |           +===============+                  *
+ *  | HW_RT_DATA_0  |         |           | HW_RT_DATA_1  |                  *
+ *  +------------|--+         |           +--|------------+                  *
+ *               |            |              |                               *
+ *               | FW-addr    | FW-addr      | FW-addr                       *
+ *               V            V              V                               *
+ *        +=========================================+                        *
+ *        |           HW_RT_DATA_COMMON             |                        *
+ *        +-----------------------------------------+                        *
+ *                                                                           *
+ *****************************************************************************/
+
+typedef struct _RGX_HWRTDATA_COMMON_COOKIE_
+{
+	DEVMEMINT_RESERVATION* psPMStateReservation;
+	DEVMEMINT_RESERVATION* psPMSecureStateReservation;
+	DEVMEMINT_RESERVATION* psPMMListsReservation;
+	IMG_UINT32             ui32RefCount;
+} RGX_HWRTDATA_COMMON_COOKIE;
+
+typedef struct _RGX_KM_HW_RT_DATASET_
+{
+	/* RGX_RTDATA_CLEANUP_DATA */
+	/* RGXMKIF_NUM_RTDATAS */
+	RGX_HWRTDATA_COMMON_COOKIE* psHWRTDataCommonCookie;
+
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_DEV_VIRTADDR sHWRTDataFwAddr;
+
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc;
+
+	RGX_FREELIST *apsFreeLists[RGXFW_MAX_FREELISTS];
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE			sNodeHWRTData;
+#endif
+
+} RGX_KM_HW_RT_DATASET;
+
+struct _RGX_FREELIST_ {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	CONNECTION_DATA			*psConnection;
+
+	/* Free list PMR */
+	PMR						*psFreeListPMR;
+	IMG_DEVMEM_OFFSET_T		uiFreeListPMROffset;
+
+	/* Free list PM state PMR */
+	PMR						*psFreeListStatePMR;
+	IMG_DEVMEM_OFFSET_T		uiFreeListStatePMROffset;
+
+	DEVMEMINT_RESERVATION	*psFreeListAndStateReservation;
+
+	/* Freelist config */
+	IMG_UINT32				ui32MaxFLPages;
+	IMG_UINT32				ui32InitFLPages;
+	IMG_UINT32				ui32CurrentFLPages;
+	IMG_UINT32				ui32GrowFLPages;
+	IMG_UINT32				ui32ReadyFLPages;
+	IMG_UINT32				ui32GrowThreshold;		/* Percentage of FL memory used that should trigger a new grow request */
+	IMG_UINT32				ui32FreelistID;
+	IMG_UINT32				ui32FreelistGlobalID;	/* related global freelist for this freelist */
+	IMG_UINT64				ui64FreelistChecksum;	/* checksum over freelist content */
+	IMG_BOOL				bCheckFreelist;			/* freelist check enabled */
+	IMG_UINT32				ui32RefCount;			/* freelist reference counting */
+
+	IMG_UINT32				ui32NumGrowReqByApp;	/* Total number of grow requests by Application */
+	IMG_UINT32				ui32NumGrowReqByFW;		/* Total Number of grow requests by Firmware */
+	IMG_UINT32				ui32NumHighPages;		/* High Mark of pages in the freelist */
+
+	IMG_PID					ownerPid;				/* Pid of the owner of the list */
+
+	/* Memory Blocks */
+	DLLIST_NODE				sMemoryBlockHead;
+	DLLIST_NODE				sMemoryBlockInitHead;
+	DLLIST_NODE				sNode;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* HWRTData nodes linked to local freelist */
+	DLLIST_NODE				sNodeHWRTDataHead;
+#endif
+
+	/* FW data structures */
+	DEVMEM_MEMDESC			*psFWFreelistMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;
+	IMG_DEV_VIRTADDR		sFreeListBaseDevVAddr;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	HASH_TABLE*				psWorkloadHashTable;
+#endif
+};
+
+struct _RGX_PMR_NODE_ {
+	RGX_FREELIST			*psFreeList;
+	PMR						*psPMR;
+	PMR_PAGELIST			*psPageList;
+	DLLIST_NODE				sMemoryBlock;
+	IMG_UINT32				ui32NumPages;
+	IMG_BOOL				bFirstPageMissing;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RI_HANDLE				hRIHandle;
+#endif
+};
+
+typedef struct {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	DEVMEM_MEMDESC			*psFWZSBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;
+
+	DEVMEMINT_RESERVATION	*psReservation;
+	PMR						*psPMR;
+	IMG_UINT32				ui32ZSBufferID;
+	IMG_UINT32				ui32RefCount;
+	IMG_BOOL				bOnDemand;
+	IMG_BOOL				bIsBacked;
+
+	IMG_UINT32				ui32NumReqByApp;		/* Number of Backing Requests from Application */
+	IMG_UINT32				ui32NumReqByFW;			/* Number of Backing Requests from Firmware */
+
+	IMG_PID					owner;
+
+	DLLIST_NODE	sNode;
+}RGX_ZSBUFFER_DATA;
+
+typedef struct {
+	RGX_ZSBUFFER_DATA		*psZSBuffer;
+} RGX_POPULATION;
+
+/* Dump the physical pages of a freelist */
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList);
+
+PVRSRV_ERROR RGXCreateHWRTDataSet2(CONNECTION_DATA          *psConnection,
+                                   PVRSRV_DEVICE_NODE       *psDeviceNode,
+                                   IMG_DEV_VIRTADDR         psVHeapTableDevVAddr,
+                                   DEVMEMINT_RESERVATION    *psPMMListsReservation,
+                                   DEVMEMINT_RESERVATION    *psPMStateReservation,
+                                   DEVMEMINT_RESERVATION    *psPMSecureStateReservation,
+                                   RGX_FREELIST             *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                   IMG_DEV_VIRTADDR         asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                   IMG_UINT16               ui16MaxRTs,
+                                   RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+/* Create HWRTDataSet */
+PVRSRV_ERROR RGXCreateHWRTDataSet(CONNECTION_DATA          *psConnection,
+                                  PVRSRV_DEVICE_NODE       *psDeviceNode,
+                                  IMG_DEV_VIRTADDR          psVHeapTableDevVAddr,
+                                  IMG_DEV_VIRTADDR          sPMDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          sPMSecureDataDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  RGX_FREELIST	           *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                  IMG_DEV_VIRTADDR          asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT16                ui16MaxRTs,
+                                  RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+/* Destroy HWRTDataSet */
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet);
+
+/*
+	RGXCreateZSBufferKM
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer);
+
+/*
+	RGXDestroyZSBufferKM
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+
+/*
+ * RGXBackingZSBuffer()
+ *
+ * Backs ZS-Buffer with physical pages
+ */
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXPopulateZSBufferKM()
+ *
+ * Backs ZS-Buffer with physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+								   RGX_POPULATION **ppsPopulation);
+
+/*
+ * RGXUnbackingZSBuffer()
+ *
+ * Frees ZS-Buffer's physical pages
+ */
+PVRSRV_ERROR RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXUnpopulateZSBufferKM()
+ *
+ * Frees ZS-Buffer's physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation);
+
+/*
+	RGXProcessRequestZSBufferBacking
+*/
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXProcessRequestZSBufferUnbacking
+*/
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXGrowFreeList
+*/
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader);
+
+/* Create free list */
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA       *psConnection,
+                               PVRSRV_DEVICE_NODE	     *psDeviceNode,
+                               IMG_HANDLE             hMemCtxPrivData,
+                               IMG_UINT32             ui32MaxFLPages,
+                               IMG_UINT32             ui32InitFLPages,
+                               IMG_UINT32             ui32GrowFLPages,
+                               IMG_UINT32             ui32GrowParamThreshold,
+                               RGX_FREELIST           *psGlobalFreeList,
+                               IMG_BOOL               bCheckFreelist,
+                               DEVMEMINT_RESERVATION	 *psFreeListAndStateReservation,
+                               RGX_FREELIST           **ppsFreeList);
+
+/* Destroy free list */
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList);
+
+/*
+	RGXProcessRequestGrow
+*/
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   IMG_UINT32 ui32FreelistID);
+
+
+/* Reconstruct free list after Hardware Recovery */
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+											  IMG_UINT32 ui32FreelistsCount,
+											  const IMG_UINT32 *paui32Freelists);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateRenderContextKM
+
+ @Description
+	Server-side implementation of RGXCreateRenderContext
+
+ @Input psConnection -
+ @Input psDeviceNode - device node
+ @Input i32Priority - context priority
+ @Input ui32FrameworkCommandSize - framework command size
+ @Input pabyFrameworkCommand - ptr to framework command
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32StaticRenderContextStateSize - size of fixed render state
+ @Input pStaticRenderContextState - ptr to fixed render state buffer
+ @Input ui32PackedCCBSizeU8888 :
+		ui8TACCBAllocSizeLog2 - TA CCB size
+		ui8TACCBMaxAllocSizeLog2 - maximum size to which TA CCB can grow
+		ui83DCCBAllocSizeLog2 - 3D CCB size
+		ui83DCCBMaxAllocSizeLog2 - maximum size to which 3D CCB can grow
+ @Input ui32ContextFlags - flags which specify properties of the context
+ @Output ppsRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+											PVRSRV_DEVICE_NODE			*psDeviceNode,
+											IMG_INT32					i32Priority,
+											IMG_UINT32					ui32FrameworkCommandSize,
+											IMG_PBYTE					pabyFrameworkCommand,
+											IMG_HANDLE					hMemCtxPrivData,
+											IMG_UINT32					ui32StaticRenderContextStateSize,
+											IMG_PBYTE					pStaticRenderContextState,
+											IMG_UINT32					ui32PackedCCBSizeU8888,
+											IMG_UINT32					ui32ContextFlags,
+											IMG_UINT64					ui64RobustnessAddress,
+											IMG_UINT32					ui32MaxTADeadlineMS,
+											IMG_UINT32					ui32Max3DDeadlineMS,
+											RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyRenderContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyRenderContext
+
+ @Input psRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXKickTA3DKM
+
+ @Description
+	Server-side implementation of RGXKickTA3D
+
+ @Input psRTDataCleanup - RT data associated with the kick (or NULL)
+ @Input psZBuffer - Z-buffer associated with the kick (or NULL)
+ @Input psSBuffer - S-buffer associated with the kick (or NULL)
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRSyncPrimBlock,
+								 IMG_UINT32					ui32PRSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckFence,
+								 PVRSRV_TIMELINE			iUpdateTimeline,
+								 PVRSRV_FENCE				*piUpdateFence,
+								 IMG_CHAR					szFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheckFence3D,
+								 PVRSRV_TIMELINE			iUpdateTimeline3D,
+								 PVRSRV_FENCE				*piUpdateFence3D,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus);
+
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore);
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDevNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+												 RGX_CONTEXT_PROPERTY eContextProperty,
+												 IMG_UINT64 ui64Input,
+												 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of render contexts on this device */
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client contexts are stalled */
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+#endif /* RGXTA3D_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.c b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.c
new file mode 100644
index 000000000000..9397bf9d0338
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.c
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@File           rgxworkest_ray.c
+@Title          RGX Workload Estimation Functionality for ray datamaster
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality for ray datamaster.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxworkest.h"
+#include "rgxworkest_ray.h"
+#include "rgxfwutils.h"
+#include "rgxpdvfs.h"
+#include "rgx_options.h"
+#include "device.h"
+#include "hash.h"
+#include "pvr_debug.h"
+
+
+static IMG_BOOL WorkEstHashCompareRay(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	RGX_WORKLOAD *psWorkload1;
+	RGX_WORKLOAD *psWorkload2;
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	if (pKey1 && pKey2)
+	{
+		psWorkload1 = *((RGX_WORKLOAD **)pKey1);
+		psWorkload2 = *((RGX_WORKLOAD **)pKey2);
+
+		PVR_ASSERT(psWorkload1);
+		PVR_ASSERT(psWorkload2);
+
+		if (psWorkload1->sRay.ui32DispatchSize == psWorkload2->sRay.ui32DispatchSize &&
+		    psWorkload1->sRay.ui32AccStructSize == psWorkload2->sRay.ui32AccStructSize)
+		{
+			/* This is added to allow this memory to be freed */
+			*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
+			return IMG_TRUE;
+		}
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_UINT32 WorkEstHashFuncRay(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	RGX_WORKLOAD *psWorkload = *((RGX_WORKLOAD**)pKey);
+	IMG_UINT32 ui32HashKey = 0;
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+
+	/* Hash key predicated on transfer src/dest attributes */
+	ui32HashKey += _WorkEstDoHash(psWorkload->sRay.ui32DispatchSize);
+	ui32HashKey += _WorkEstDoHash(psWorkload->sRay.ui32AccStructSize);
+
+	return ui32HashKey;
+}
+
+void WorkEstInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstInit(psDevInfo,
+		&psWorkEstData->uWorkloadMatchingData.sRay.sDataRDM,
+		(HASH_FUNC *)WorkEstHashFuncRay,
+		(HASH_KEY_COMP *)WorkEstHashCompareRay);
+}
+
+void WorkEstDeInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData)
+{
+	_WorkEstDeInit(psDevInfo, &psWorkEstData->uWorkloadMatchingData.sRay.sDataRDM);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.h b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.h
new file mode 100644
index 000000000000..21f3d3d8271a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/devices/volcanic/rgxworkest_ray.h
@@ -0,0 +1,53 @@
+/*************************************************************************/ /*!
+@File           rgxworkest_ray.h
+@Title          RGX Workload Estimation Functionality for ray datamaster
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXWORKEST_RAY_H
+#define RGXWORKEST_RAY_H
+
+#include "img_types.h"
+
+void WorkEstInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+void WorkEstDeInitRay(PVRSRV_RGXDEV_INFO *psDevInfo, WORKEST_HOST_DATA *psWorkEstData);
+
+#endif /* RGXWORKEST_RAY_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Kbuild.mk
new file mode 100644
index 000000000000..a53db2eac5bb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Kbuild.mk
@@ -0,0 +1,695 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+# Define used in code when no feature MASK or IDX flag can determine the
+# PVR_ARCH build config of the KMD server. KMD Server either supports all
+# Rogue cores, or all Volcanic cores, but not both.
+ifeq ($(PVR_ARCH),rogue)
+ ccflags-y += -DPVR_ARCH_ROGUE
+else ifeq ($(PVR_ARCH),volcanic)
+ ccflags-y += -DPVR_ARCH_VOLCANIC
+endif
+
+# Window system
+ccflags-y += -DWINDOW_SYSTEM=\"$(WINDOW_SYSTEM)\"
+
+# Linux kernel headers
+ccflags-y += \
+ -Iinclude \
+ -Iinclude/drm
+
+# Compatibility BVNC
+ccflags-y += -I$(TOP)/services/shared/devices/$(PVR_ARCH_DEFS)
+
+# Errata files
+ccflags-y += -I$(HWDEFS_DIR) -I$(HWDEFS_DIR)/$(RGX_BNC)
+
+# Linux-specific headers
+ccflags-y += \
+ -I$(TOP)/include/drm \
+ -I$(TOP)/services/include/env/linux \
+ -I$(TOP)/services/server/env/linux/$(PVR_ARCH) -I$(TOP)/services/server/env/linux \
+ -I$(TOP)/kernel/drivers/staging/imgtec
+
+# System dir
+ifneq ($(wildcard $(TOP)/services/system/$(PVR_ARCH)/$(PVR_SYSTEM)/Kbuild.mk),)
+SYSTEM_DIR := $(TOP)/services/system/$(PVR_ARCH)/$(PVR_SYSTEM)
+else
+SYSTEM_DIR := $(TOP)/services/system/$(PVR_SYSTEM)
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/dkf_server.o \
+ services/server/env/linux/event.o \
+ services/server/env/linux/fwload.o \
+ services/server/env/linux/km_apphint.o \
+ services/server/env/linux/module_common.o \
+ services/server/env/linux/osmmap_stub.o \
+ services/server/env/linux/osfunc.o \
+ services/server/env/linux/allocmem.o \
+ services/server/env/linux/osconnection_server.o \
+ services/server/env/linux/physmem_osmem_linux.o \
+ services/server/env/linux/pmr_os.o \
+ services/server/env/linux/pmr_env.o \
+ services/server/env/linux/pvr_bridge_k.o \
+ services/server/env/linux/pvr_debug.o \
+ services/server/env/linux/pvr_drm.o \
+ services/server/env/linux/physmem_dmabuf.o \
+ services/server/common/devicemem_heapcfg.o \
+ services/shared/common/devicemem.o \
+ services/shared/common/devicemem_utils.o \
+ services/shared/common/hash.o \
+ services/shared/common/hash_functions.o \
+ services/shared/common/ra.o \
+ services/shared/common/sync.o \
+ services/shared/common/mem_utils.o \
+ services/server/common/devicemem_server.o \
+ services/server/common/handle.o \
+ services/server/common/lists.o \
+ services/server/common/mmu_common.o \
+ services/server/common/connection_server.o \
+ services/server/common/physheap.o \
+ services/server/common/physmem.o \
+ services/server/common/physmem_lma.o \
+ services/server/common/physmem_ramem.o \
+ services/server/common/physmem_osmem.o \
+ services/server/common/physmem_hostmem.o \
+ services/server/common/pmr.o \
+ services/server/common/power.o \
+ services/server/common/process_stats.o \
+ services/server/common/pvr_notifier.o \
+ services/server/common/pvrsrv.o \
+ services/server/common/srvcore.o \
+ services/server/common/sync_checkpoint.o \
+ services/server/common/sync_server.o \
+ services/server/common/tlintern.o \
+ services/shared/common/tlclient.o \
+ services/server/common/tlserver.o \
+ services/server/common/tlstream.o \
+ services/server/common/cache_km.o \
+ services/shared/common/uniq_key_splay_tree.o \
+ services/server/common/pvrsrv_pool.o \
+ services/server/common/pvrsrv_bridge_init.o \
+ services/server/common/info_page_km.o \
+ services/shared/common/pvrsrv_error.o \
+ services/server/common/debug_common.o \
+ services/server/common/di_server.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/$(PVR_ARCH)/common/vmm_type_$(VMM_TYPE).o
+
+ifeq ($(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/common/physmem_dlm.o \
+ services/server/common/physmem_ima.o
+endif
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+$(PVRSRV_MODNAME)-y += \
+ services/shared/common/htbuffer.o \
+ services/server/common/htbserver.o \
+ services/server/common/htb_debug.o
+endif
+
+ifeq ($(SUPPORT_DMA_TRANSFER),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/common/dma_km.o
+endif
+
+# Wrap ExtMem support
+ifeq ($(SUPPORT_WRAP_EXTMEM),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/physmem_extmem_linux.o \
+ services/server/common/physmem_extmem.o
+endif
+
+ifeq ($(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pg_walk_through.o
+endif
+
+ifeq ($(SUPPORT_PHYSMEM_TEST),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/physmem_test.o
+endif
+
+ifneq ($(PVR_LOADER),)
+ ifeq ($(KERNEL_DRIVER_DIR),)
+  $(PVRSRV_MODNAME)-y += services/server/env/linux/$(PVR_LOADER).o
+ else
+  ifneq ($(wildcard $(KERNELDIR)/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM)/$(PVR_LOADER).c),)
+    $(PVRSRV_MODNAME)-y += external/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM)/$(PVR_LOADER).o
+  else
+   ifneq ($(wildcard $(KERNELDIR)/$(KERNEL_DRIVER_DIR)/$(PVR_LOADER).c),)
+     $(PVRSRV_MODNAME)-y += external/$(KERNEL_DRIVER_DIR)/$(PVR_LOADER).o
+   else
+     $(PVRSRV_MODNAME)-y += services/server/env/linux/$(PVR_LOADER).o
+   endif
+  endif
+ endif
+else
+ $(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_platform_drv.o
+endif
+
+ifeq ($(SUPPORT_RGX),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgx_bridge_init.o \
+ services/server/devices/rgxfwdbg.o \
+ services/server/devices/rgxtimerquery.o \
+ services/server/devices/rgxccb.o \
+ services/server/devices/rgxdebug_common.o \
+ services/server/devices/$(PVR_ARCH)/rgxdebug.o \
+ services/server/devices/rgxfwtrace_strings.o \
+ services/server/devices/$(PVR_ARCH)/rgxfwutils.o \
+ services/server/devices/rgxfwcmnctx.o \
+ services/server/devices/rgxfwriscv.o \
+ services/server/devices/$(PVR_ARCH)/rgxinit.o \
+ services/server/devices/rgxbvnc.o \
+ services/server/devices/rgxlayer_impl_common.o \
+ services/server/devices/$(PVR_ARCH)/rgxlayer_impl.o \
+ services/server/devices/rgxmem.o \
+ services/server/devices/rgxmmuinit.o \
+ services/server/devices/rgxregconfig.o \
+ services/server/devices/$(PVR_ARCH)/rgxta3d.o \
+ services/server/devices/rgxsyncutils.o \
+ services/server/devices/rgxtdmtransfer.o \
+ services/server/devices/rgxutils.o \
+ services/server/devices/rgxhwperf_common.o \
+ services/server/devices/$(PVR_ARCH)/rgxhwperf.o \
+ services/server/devices/rgxpower.o \
+ services/server/devices/$(PVR_ARCH)/rgxstartstop.o \
+ services/server/devices/rgxtimecorr.o \
+ services/server/devices/rgxcompute.o \
+ services/server/devices/$(PVR_ARCH)/rgxmulticore.o \
+ services/server/devices/rgxshader.o
+
+# add pvr_gputrace.o if any of these features are enabled
+ifneq (, $(filter 1,$(PVRSRV_TRACE_ROGUE_EVENTS) $(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)))
+$(PVRSRV_MODNAME)-$(CONFIG_EVENT_TRACING) += services/server/env/linux/pvr_gputrace.o
+endif
+
+ifeq ($(PVRSRV_ANDROID_TRACE_GPU_FREQ),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_gpufreq.o
+endif
+
+ifeq ($(SUPPORT_RGXKICKSYNC_BRIDGE),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxkicksync.o
+endif
+
+ifeq ($(SUPPORT_USC_BREAKPOINT),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxbreakpoint.o
+endif
+
+ifeq ($(PVR_ARCH),volcanic)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/$(PVR_ARCH)/rgxray.o
+endif
+
+
+ifeq ($(PVR_ARCH),rogue)
+ $(PVRSRV_MODNAME)-y += \
+  services/server/devices/$(PVR_ARCH)/rgxtransfer.o \
+  services/server/devices/$(PVR_ARCH)/rgxmipsmmuinit.o
+endif
+
+ifeq ($(SUPPORT_PDVFS),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxpdvfs.o
+endif
+
+ifeq ($(SUPPORT_WORKLOAD_ESTIMATION),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxworkest.o
+ ifeq ($(PVR_ARCH),volcanic)
+  $(PVRSRV_MODNAME)-y += \
+  services/server/devices/$(PVR_ARCH)/rgxworkest_ray.o
+ endif
+endif
+
+ifeq ($(SUPPORT_VALIDATION),1)
+ifeq ($(PVR_TESTING_UTILS),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxgpumap.o
+endif
+endif
+
+ifeq ($(SUPPORT_VALIDATION),1)
+ $(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxsoctimer.o
+endif
+endif # SUPPORT_RGX
+
+ifeq ($(SUPPORT_DISPLAY_CLASS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/common/dc_server.o \
+ services/server/common/scp.o
+endif
+
+ifeq ($(SUPPORT_SECURE_EXPORT),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/ossecure_export.o
+endif
+
+ifeq ($(PDUMP),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/common/pdump_server.o \
+ services/server/common/pdump_mmu.o \
+ services/server/common/pdump_physmem.o \
+ services/shared/common/devicemem_pdump.o \
+ services/shared/common/devicememx_pdump.o
+
+ifeq ($(SUPPORT_RGX),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/$(PVR_ARCH)/rgxpdump.o
+ifeq ($(PDUMP),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/rgxpdump_common.o
+endif
+endif
+
+endif
+
+
+
+ifeq ($(PVRSRV_ENABLE_GPU_MEMORY_INFO),1)
+$(PVRSRV_MODNAME)-y += services/server/common/ri_server.o
+endif
+
+ifeq ($(PVR_TESTING_UTILS),1)
+$(PVRSRV_MODNAME)-y += services/server/common/tutils.o
+endif
+
+$(PVRSRV_MODNAME)-y += services/server/common/devicemem_history_server.o
+
+ifeq ($(PVRSRV_PHYSMEM_CPUMAP_HISTORY),1)
+$(PVRSRV_MODNAME)-y += services/server/common/physmem_cpumap_history.o
+endif
+
+ifeq ($(PVR_HANDLE_BACKEND),generic)
+$(PVRSRV_MODNAME)-y += services/server/common/handle_generic.o
+else
+ifeq ($(PVR_HANDLE_BACKEND),idr)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/handle_idr.o
+endif
+endif
+
+ifeq ($(PVRSRV_ENABLE_LINUX_MMAP_STATS),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/mmap_stats.o
+endif
+
+ifeq ($(SUPPORT_BUFFER_SYNC),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_buffer_sync.o \
+ services/server/env/linux/pvr_fence.o
+endif
+
+ifeq ($(SUPPORT_NATIVE_FENCE_SYNC),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_sync_ioctl_common.o
+ifeq ($(USE_PVRSYNC_DEVNODE),1)
+$(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_sync_ioctl_dev.o
+else
+$(PVRSRV_MODNAME)-y += services/server/env/linux/pvr_sync_ioctl_drm.o
+endif
+ifeq ($(SUPPORT_DMA_FENCE),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_sync_file.o \
+ services/server/env/linux/pvr_counting_timeline.o \
+ services/server/env/linux/pvr_sw_fence.o \
+ services/server/env/linux/pvr_export_fence.o \
+ services/server/env/linux/pvr_fence.o
+endif
+else
+ifeq ($(SUPPORT_FALLBACK_FENCE_SYNC),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/common/sync_fallback_server.o \
+ services/server/env/linux/ossecure_export.o
+endif
+endif
+
+ifeq ($(SUPPORT_LINUX_DVFS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_dvfs_device.o \
+ services/server/env/linux/pvr_dvfs_common.o
+else
+ ifeq ($(SUPPORT_PDVFS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_dvfs_common.o
+ endif
+endif
+
+ifeq ($(PVRSRV_ENABLE_PVR_ION_STATS),1)
+$(PVRSRV_MODNAME)-y += \
+ services/server/env/linux/pvr_ion_stats.o
+endif
+
+$(PVRSRV_MODNAME)-$(CONFIG_X86) += services/server/env/linux/osfunc_x86.o
+$(PVRSRV_MODNAME)-$(CONFIG_ARM) += services/server/env/linux/osfunc_arm.o
+$(PVRSRV_MODNAME)-$(CONFIG_ARM64) += services/server/env/linux/osfunc_arm64.o
+$(PVRSRV_MODNAME)-$(CONFIG_METAG) += services/server/env/linux/osfunc_metag.o
+$(PVRSRV_MODNAME)-$(CONFIG_MIPS) += services/server/env/linux/osfunc_mips.o
+$(PVRSRV_MODNAME)-$(CONFIG_RISCV) += services/server/env/linux/osfunc_riscv.o
+
+ifeq ($(SUPPORT_ANDROID_PLATFORM),1)
+ ifeq ($(CONFIG_PROC_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_PROC_FS) += services/server/env/linux/pvr_procfs.o
+ else ifeq ($(CONFIG_DEBUG_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_DEBUG_FS) += services/server/env/linux/pvr_debugfs.o
+ endif
+else
+ ifeq ($(CONFIG_DEBUG_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_DEBUG_FS) += services/server/env/linux/pvr_debugfs.o
+ else ifeq ($(CONFIG_PROC_FS),y)
+ $(PVRSRV_MODNAME)-$(CONFIG_PROC_FS) += services/server/env/linux/pvr_procfs.o
+ endif
+endif
+
+ifeq ($(SUPPORT_DI_BRG_IMPL),1)
+$(PVRSRV_MODNAME)-y += services/server/common/di_impl_brg.o
+endif
+$(PVRSRV_MODNAME)-$(CONFIG_EVENT_TRACING) += services/server/env/linux/trace_events.o
+
+ccflags-y += -I$(OUT)/target_neutral/intermediates/firmware
+
+ifeq ($(SUPPORT_RGX),1)
+# Srvinit headers and source files
+
+$(PVRSRV_MODNAME)-y += \
+ services/server/devices/$(PVR_ARCH)/rgxsrvinit.o \
+ services/server/devices/rgxfwimageutils.o
+ifeq ($(PVR_ARCH),rogue)
+$(PVRSRV_MODNAME)-y += \
+ services/shared/devices/$(PVR_ARCH)/rgx_hwperf_table.o
+endif
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ services/system/$(PVR_ARCH)/common/env/linux/dma_support.o \
+ services/system/common/env/linux/interrupt_support.o
+
+$(PVRSRV_MODNAME)-$(CONFIG_PCI) += \
+ services/system/common/env/linux/pci_support.o
+
+ccflags-y += \
+ -I$(HWDEFS_DIR)/km
+ifeq ($(PVR_ARCH),rogue)
+ccflags-y += \
+ -I$(TOP)/include/$(PVR_ARCH_DEFS)
+endif
+ccflags-y += \
+ -I$(TOP)/include/$(PVR_ARCH) -I$(TOP)/include \
+ -I$(TOP)/include/$(PVR_ARCH)/public -I$(TOP)/include/public \
+ -I$(TOP)/services/include/$(PVR_ARCH) -I$(TOP)/services/include \
+ -I$(TOP)/services/shared/include \
+ -I$(TOP)/services/server/devices/$(PVR_ARCH) -I$(TOP)/services/server/devices \
+ -I$(TOP)/services/server/include/$(PVR_ARCH) -I$(TOP)/services/server/include \
+ -I$(TOP)/services/shared/common \
+ -I$(TOP)/services/shared/devices \
+ -I$(TOP)/services/system/include \
+ -I$(TOP)/services/system/$(PVR_ARCH)/include \
+ -I$(TOP)/services/server/common/$(PVR_ARCH) -I$(TOP)/services/server/common
+
+ifeq ($(KERNEL_DRIVER_DIR),)
+ ccflags-y += -I$(SYSTEM_DIR)
+endif
+
+# Bridge headers and source files
+
+# Keep in sync with:
+# build/linux/common/bridges.mk AND
+# services/bridge/Linux.mk
+
+ccflags-y += \
+ -I$(bridge_base)/mm_bridge \
+ -I$(bridge_base)/cmm_bridge \
+ -I$(bridge_base)/srvcore_bridge \
+ -I$(bridge_base)/sync_bridge \
+ -I$(bridge_base)/synctracking_bridge \
+ -I$(bridge_base)/pvrtl_bridge \
+ -I$(bridge_base)/cache_bridge \
+ -I$(bridge_base)/dmabuf_bridge
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+ccflags-y += \
+ -I$(bridge_base)/htbuffer_bridge
+endif
+
+ifeq ($(SUPPORT_DMA_TRANSFER),1)
+ccflags-y += \
+ -I$(bridge_base)/dma_bridge
+endif
+
+ifeq ($(SUPPORT_RGX),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxtq2_bridge \
+ -I$(bridge_base)/rgxta3d_bridge \
+ -I$(bridge_base)/rgxhwperf_bridge \
+ -I$(bridge_base)/rgxcmp_bridge \
+ -I$(bridge_base)/rgxregconfig_bridge \
+ -I$(bridge_base)/rgxtimerquery_bridge \
+ -I$(bridge_base)/rgxfwdbg_bridge
+ifeq ($(PVR_ARCH),volcanic)
+ccflags-y += \
+ -I$(bridge_base)/rgxray_bridge
+endif
+ifeq ($(PVR_ARCH),rogue)
+ccflags-y += \
+ -I$(bridge_base)/rgxtq_bridge
+endif
+ifeq ($(SUPPORT_USC_BREAKPOINT),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxbreakpoint_bridge
+endif
+ifeq ($(SUPPORT_RGXKICKSYNC_BRIDGE),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxkicksync_bridge
+endif
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/mm_bridge/server_mm_bridge.o \
+ generated/$(PVR_ARCH)/cmm_bridge/server_cmm_bridge.o \
+ generated/$(PVR_ARCH)/srvcore_bridge/server_srvcore_bridge.o \
+ generated/$(PVR_ARCH)/sync_bridge/server_sync_bridge.o \
+ generated/$(PVR_ARCH)/pvrtl_bridge/server_pvrtl_bridge.o \
+ generated/$(PVR_ARCH)/cache_bridge/server_cache_bridge.o \
+ generated/$(PVR_ARCH)/dmabuf_bridge/server_dmabuf_bridge.o
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/htbuffer_bridge/server_htbuffer_bridge.o
+endif
+
+ifeq ($(SUPPORT_DMA_TRANSFER),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/dma_bridge/server_dma_bridge.o
+endif
+
+ifeq ($(SUPPORT_RGX),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxtq2_bridge/server_rgxtq2_bridge.o \
+ generated/$(PVR_ARCH)/rgxta3d_bridge/server_rgxta3d_bridge.o \
+ generated/$(PVR_ARCH)/rgxhwperf_bridge/server_rgxhwperf_bridge.o \
+ generated/$(PVR_ARCH)/rgxcmp_bridge/server_rgxcmp_bridge.o \
+ generated/$(PVR_ARCH)/rgxregconfig_bridge/server_rgxregconfig_bridge.o \
+ generated/$(PVR_ARCH)/rgxtimerquery_bridge/server_rgxtimerquery_bridge.o \
+ generated/$(PVR_ARCH)/rgxfwdbg_bridge/server_rgxfwdbg_bridge.o
+ifeq ($(PVR_ARCH),volcanic)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxray_bridge/server_rgxray_bridge.o
+endif
+ifeq ($(PVR_ARCH),rogue)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxtq_bridge/server_rgxtq_bridge.o
+endif
+ifeq ($(SUPPORT_USC_BREAKPOINT),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxbreakpoint_bridge/server_rgxbreakpoint_bridge.o
+endif
+ifeq ($(SUPPORT_RGXKICKSYNC_BRIDGE),1)
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxkicksync_bridge/server_rgxkicksync_bridge.o
+endif
+endif
+
+ifeq ($(SUPPORT_WRAP_EXTMEM),1)
+ccflags-y += -I$(bridge_base)/mmextmem_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/mmextmem_bridge/server_mmextmem_bridge.o
+endif
+
+ifeq ($(SUPPORT_DISPLAY_CLASS),1)
+ccflags-y += -I$(bridge_base)/dc_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/dc_bridge/server_dc_bridge.o
+endif
+
+ifeq ($(SUPPORT_SECURE_EXPORT),1)
+ccflags-y += -I$(bridge_base)/smm_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/smm_bridge/server_smm_bridge.o
+endif
+
+ifeq ($(PDUMP),1)
+ccflags-y += \
+ -I$(bridge_base)/pdump_bridge \
+ -I$(bridge_base)/pdumpctrl_bridge \
+ -I$(bridge_base)/pdumpmm_bridge
+
+ifeq ($(SUPPORT_RGX),1)
+ccflags-y += \
+ -I$(bridge_base)/rgxpdump_bridge
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/rgxpdump_bridge/server_rgxpdump_bridge.o
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/pdump_bridge/server_pdump_bridge.o \
+ generated/$(PVR_ARCH)/pdumpctrl_bridge/server_pdumpctrl_bridge.o \
+ generated/$(PVR_ARCH)/pdumpmm_bridge/server_pdumpmm_bridge.o
+endif
+
+ifeq ($(PVRSRV_ENABLE_GPU_MEMORY_INFO),1)
+ccflags-y += -I$(bridge_base)/ri_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/ri_bridge/server_ri_bridge.o
+endif
+
+ifeq ($(SUPPORT_VALIDATION),1)
+ccflags-y += -I$(bridge_base)/validation_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/validation_bridge/server_validation_bridge.o
+$(PVRSRV_MODNAME)-y += services/server/common/validation.o
+$(PVRSRV_MODNAME)-y += services/server/common/validation_soc.o
+endif
+
+ifeq ($(PVR_TESTING_UTILS),1)
+ccflags-y += -I$(bridge_base)/tutils_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/tutils_bridge/server_tutils_bridge.o
+endif
+
+ccflags-y += -I$(bridge_base)/devicememhistory_bridge
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/devicememhistory_bridge/server_devicememhistory_bridge.o
+
+ccflags-y += -I$(bridge_base)/synctracking_bridge
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/synctracking_bridge/server_synctracking_bridge.o
+
+ifeq ($(SUPPORT_FALLBACK_FENCE_SYNC),1)
+ccflags-y += \
+ -I$(bridge_base)/syncfallback_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/syncfallback_bridge/server_syncfallback_bridge.o
+endif
+
+ifeq ($(SUPPORT_DI_BRG_IMPL),1)
+ccflags-y += -I$(bridge_base)/di_bridge
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/di_bridge/server_di_bridge.o
+endif
+
+
+# Direct bridges
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/mm_bridge/client_mm_direct_bridge.o \
+ generated/$(PVR_ARCH)/sync_bridge/client_sync_direct_bridge.o \
+ generated/$(PVR_ARCH)/cache_bridge/client_cache_direct_bridge.o \
+ generated/$(PVR_ARCH)/pvrtl_bridge/client_pvrtl_direct_bridge.o
+
+ifeq ($(PVRSRV_ENABLE_HTB),1)
+$(PVRSRV_MODNAME)-y +=\
+ generated/$(PVR_ARCH)/htbuffer_bridge/client_htbuffer_direct_bridge.o
+endif
+
+ifeq ($(PDUMP),1)
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/pdumpmm_bridge/client_pdumpmm_direct_bridge.o
+endif
+
+ifeq ($(PVRSRV_ENABLE_GPU_MEMORY_INFO),1)
+$(PVRSRV_MODNAME)-y += generated/$(PVR_ARCH)/ri_bridge/client_ri_direct_bridge.o
+endif
+
+ifeq ($(PDUMP),1)
+ $(PVRSRV_MODNAME)-y += \
+  generated/$(PVR_ARCH)/pdump_bridge/client_pdump_direct_bridge.o \
+  generated/$(PVR_ARCH)/pdumpctrl_bridge/client_pdumpctrl_direct_bridge.o
+
+ifeq ($(SUPPORT_RGX),1)
+ $(PVRSRV_MODNAME)-y += \
+  generated/$(PVR_ARCH)/rgxpdump_bridge/client_rgxpdump_direct_bridge.o
+endif
+
+endif
+
+# Enable -Werror for all built object files
+ifneq ($(W),1)
+$(foreach _o,$(addprefix CFLAGS_,$($(PVRSRV_MODNAME)-y)),$(eval $(_o) += -Werror))
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/devicememhistory_bridge/client_devicememhistory_direct_bridge.o
+
+$(PVRSRV_MODNAME)-y += \
+ generated/$(PVR_ARCH)/synctracking_bridge/client_synctracking_direct_bridge.o
+
+# Ignore address-of-packed-member warning for all bridge files
+$(foreach _o,$(addprefix CFLAGS_,$(filter generated/%.o,$($(PVRSRV_MODNAME)-y))),$(eval $(_o) += -Wno-address-of-packed-member))
+
+# With certain build configurations, e.g., ARM, Werror, we get a build
+# failure in the ftrace Linux kernel header.  So disable the relevant check.
+CFLAGS_services/server/env/linux/trace_events.o := -Wno-missing-prototypes
+
+# Make sure the mem_utils are built in 'free standing' mode, so the compiler
+# is not encouraged to call out to C library functions
+ifeq ($(CC),clang)
+ ifneq ($(SUPPORT_ANDROID_PLATFORM),1)
+  CFLAGS_services/shared/common/mem_utils.o := -ffreestanding -fforce-enable-int128
+ else
+  CFLAGS_services/shared/common/mem_utils.o := -ffreestanding
+ endif
+endif
+
+# Chrome OS kernel adds some issues
+ccflags-y += -Wno-ignored-qualifiers
+
+# Treat #warning as a warning
+ccflags-y += -Wno-error=cpp
+
+include $(SYSTEM_DIR)/Kbuild.mk
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Linux.mk b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Linux.mk
new file mode 100644
index 000000000000..caf3a077a3a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/Linux.mk
@@ -0,0 +1,46 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+modules := srvkm
+
+srvkm_type := kernel_module
+srvkm_target := $(PVRSRV_MODNAME).ko
+srvkm_makefile := $(THIS_DIR)/Kbuild.mk
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/allocmem.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/allocmem.c
new file mode 100644
index 000000000000..817680376d79
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/allocmem.c
@@ -0,0 +1,377 @@
+/*************************************************************************/ /*!
+@File
+@Title          Host memory management implementation for Linux
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/string.h>
+
+#include "img_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "process_stats.h"
+#include "osfunc.h"
+
+
+/*
+ * When memory statistics are disabled, memory records are used instead.
+ * In order for these to work, the PID of the process that requested the
+ * allocation needs to be stored at the end of the kmalloc'd memory, making
+ * sure 4 extra bytes are allocated to fit the PID.
+ *
+ * There is no need for this extra allocation when memory statistics are
+ * enabled, since all allocations are tracked in DebugFS mem_area files.
+ */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+/* kmalloc guarantees a minimal alignment which is ARCH_KMALLOC_MINALIGN. This
+ * alignment is architecture specific and can be quite big, e.g. on Aarch64
+ * it can be 64 bytes. This is too much for keeping a single PID field and could
+ * lead to a lot of wasted memory. This is a reason why we're defaulting to 8
+ * bytes alignment which should be enough for any architecture.
+ */
+#define ALLOCMEM_PID_SIZE_PADDING PVR_ALIGN(sizeof(IMG_UINT32), 8)
+#else
+#define ALLOCMEM_PID_SIZE_PADDING 0UL
+#endif
+
+/* How many times kmalloc can fail before the allocation threshold is reduced */
+static const IMG_UINT32 g_ui32kmallocFailLimit = 10;
+/* How many kmalloc failures happened since the last allocation threshold change */
+static IMG_UINT32 g_ui32kmallocFailCount = 0;
+/* Current kmalloc threshold value in bytes */
+static IMG_UINT32 g_ui32kmallocThreshold = PVR_LINUX_KMALLOC_ALLOCATION_THRESHOLD;
+/* Spinlock used so that the global variables above may not be modified by more than 1 thread at a time */
+static DEFINE_SPINLOCK(kmalloc_lock);
+
+
+static inline void OSTryDecreaseKmallocThreshold(void)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&kmalloc_lock, flags);
+
+	g_ui32kmallocFailCount++;
+
+	if (g_ui32kmallocFailCount >= g_ui32kmallocFailLimit)
+	{
+		g_ui32kmallocFailCount = 0;
+		if (g_ui32kmallocThreshold > PAGE_SIZE)
+		{
+			g_ui32kmallocThreshold >>= 1;
+			printk(KERN_INFO "Threshold is now set to %d\n", g_ui32kmallocThreshold);
+		}
+	}
+
+	spin_unlock_irqrestore(&kmalloc_lock, flags);
+}
+
+static inline void OSResetKmallocFailCount(void)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&kmalloc_lock, flags);
+
+	g_ui32kmallocFailCount = 0;
+
+	spin_unlock_irqrestore(&kmalloc_lock, flags);
+}
+
+static inline void _pvr_vfree(const void* pvAddr)
+{
+#if defined(DEBUG)
+	/* Size harder to come by for vmalloc and since vmalloc allocates
+	 * a whole number of pages, poison the minimum size known to have
+	 * been allocated.
+	 */
+	OSCachedMemSet((void*)pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE,
+	               PAGE_SIZE);
+#endif
+	vfree(pvAddr);
+}
+
+static inline void _pvr_kfree(const void* pvAddr)
+{
+#if defined(DEBUG)
+	/* Poison whole memory block */
+	OSCachedMemSet((void*)pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE,
+	               ksize(pvAddr));
+#endif
+	kfree(pvAddr);
+}
+
+static inline void *_pvr_alloc_stats_add(void *pvAddr, IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+#if !defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVR_UNREFERENCED_PARAMETER(pvAddr);
+#else
+	if (!is_vmalloc_addr(pvAddr))
+	{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_KMALLOC,
+									  pvAddr,
+									  sCpuPAddr,
+									  ksize(pvAddr),
+									  OSGetCurrentClientProcessIDKM()
+									  DEBUG_MEMSTATS_ARGS);
+#else
+		/* because clang has some features that allow detection out-of-bounds
+		 * access we need to put the metadata in the beginning of the allocation */
+		*(IMG_UINT32 *) pvAddr = OSGetCurrentClientProcessIDKM();
+		PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_KMALLOC, ksize(pvAddr),
+		                            *(IMG_UINT32 *) pvAddr);
+
+		/* because metadata is kept in the beginning of the allocation we need
+		 * to return address offset by the ALLOCMEM_PID_SIZE_PADDING */
+		pvAddr = (IMG_UINT8 *) pvAddr + ALLOCMEM_PID_SIZE_PADDING;
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+	else
+	{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+									  pvAddr,
+									  sCpuPAddr,
+									  PVR_ALIGN(ui32Size, PAGE_SIZE),
+									  OSGetCurrentClientProcessIDKM()
+									  DEBUG_MEMSTATS_ARGS);
+#else
+		PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                    PVR_ALIGN(ui32Size, PAGE_SIZE),
+		                                    (IMG_UINT64)(uintptr_t) pvAddr,
+		                                    OSGetCurrentClientProcessIDKM());
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#endif /* !defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return pvAddr;
+}
+
+static inline void *_pvr_alloc_stats_remove(void *pvAddr)
+{
+#if !defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVR_UNREFERENCED_PARAMETER(pvAddr);
+#else
+	if (!is_vmalloc_addr(pvAddr))
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		/* because metadata is kept in the beginning of the allocation we need
+		 * shift address offset by the ALLOCMEM_PID_SIZE_PADDING to the original
+		 * value */
+		pvAddr = (IMG_UINT8 *) pvAddr - ALLOCMEM_PID_SIZE_PADDING;
+
+		/* first 4 bytes of the allocation are the process' PID */
+		PVRSRVStatsDecrMemKAllocStat(ksize(pvAddr), *(IMG_UINT32 *) pvAddr);
+#else
+		PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_KMALLOC,
+		                                (IMG_UINT64)(uintptr_t) pvAddr,
+		                                OSGetCurrentClientProcessIDKM());
+#endif
+	}
+	else
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                      (IMG_UINT64)(uintptr_t) pvAddr);
+#else
+		PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                (IMG_UINT64)(uintptr_t) pvAddr,
+		                                OSGetCurrentClientProcessIDKM());
+#endif
+	}
+#endif /* !defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return pvAddr;
+}
+
+void *(OSAllocMem)(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+	void *pvRet = NULL;
+
+	if ((ui32Size + ALLOCMEM_PID_SIZE_PADDING) <= g_ui32kmallocThreshold)
+	{
+		pvRet = kmalloc(ui32Size + ALLOCMEM_PID_SIZE_PADDING, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vmalloc(ui32Size);
+	}
+
+	if (pvRet != NULL)
+	{
+		pvRet = _pvr_alloc_stats_add(pvRet, ui32Size DEBUG_MEMSTATS_ARGS);
+	}
+
+	return pvRet;
+}
+
+void *(OSAllocZMem)(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+	void *pvRet = NULL;
+
+	if ((ui32Size + ALLOCMEM_PID_SIZE_PADDING) <= g_ui32kmallocThreshold)
+	{
+		pvRet = kzalloc(ui32Size + ALLOCMEM_PID_SIZE_PADDING, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vzalloc(ui32Size);
+	}
+
+	if (pvRet != NULL)
+	{
+		pvRet = _pvr_alloc_stats_add(pvRet, ui32Size DEBUG_MEMSTATS_ARGS);
+	}
+
+	return pvRet;
+}
+
+/*
+ * The parentheses around OSFreeMem prevent the macro in allocmem.h from
+ * applying, as it would break the function's definition.
+ */
+void (OSFreeMem)(void *pvMem)
+{
+	if (pvMem != NULL)
+	{
+		pvMem = _pvr_alloc_stats_remove(pvMem);
+
+		if (!is_vmalloc_addr(pvMem))
+		{
+			_pvr_kfree(pvMem);
+		}
+		else
+		{
+			_pvr_vfree(pvMem);
+		}
+	}
+}
+
+void *OSAllocMemNoStats(IMG_UINT32 ui32Size)
+{
+	void *pvRet = NULL;
+
+	if (ui32Size <= g_ui32kmallocThreshold)
+	{
+		pvRet = kmalloc(ui32Size, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vmalloc(ui32Size);
+	}
+
+	return pvRet;
+}
+
+void *OSAllocZMemNoStats(IMG_UINT32 ui32Size)
+{
+	void *pvRet = NULL;
+
+	if (ui32Size <= g_ui32kmallocThreshold)
+	{
+		pvRet = kzalloc(ui32Size, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vzalloc(ui32Size);
+	}
+
+	return pvRet;
+}
+
+/*
+ * The parentheses around OSFreeMemNoStats prevent the macro in allocmem.h from
+ * applying, as it would break the function's definition.
+ */
+void (OSFreeMemNoStats)(void *pvMem)
+{
+	if (pvMem != NULL)
+	{
+		if (!is_vmalloc_addr(pvMem))
+		{
+			_pvr_kfree(pvMem);
+		}
+		else
+		{
+			_pvr_vfree(pvMem);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.c
new file mode 100644
index 000000000000..6136b627d9ae
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.c
@@ -0,0 +1,409 @@
+/*************************************************************************/ /*!
+@File         dkf_server.c
+@Title        DRM Key Framework support routines.
+@Copyright    Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "dkf_server.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvrsrv_error.h"
+#include "dkp_impl.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#if defined(SUPPORT_LINUX_FDINFO)
+typedef struct DKF_REGISTERED_DKP_TAG
+{
+	IMG_HANDLE              hPrivHandle;    /*!< Private data - can be NULL */
+	const IMG_CHAR          *pszDKPName;    /*!< DKP provider name */
+	IMG_PID                 pid;            /*!< Process-ID - can be 0 */
+	IMG_UINT32              uiReserved1;    /*!< Reserved field - padding */
+	DKP_PFN_SHOW            *psDKPShowPfn;  /*!< DKP Callback function */
+	DLLIST_NODE             sDKFEntryNode;  /*!< List of DKP entries */
+
+	DKP_CONNECTION_FLAGS    ui32Filter;     /*!< The types of connection to output to */
+} DKF_REGISTERED_DKP;
+
+/* Global sentinel to track all allocated DKP callback key/value pairs */
+typedef struct DKF_TAG
+{
+	POS_LOCK            hDKFListLock;   /*!< Lock for accessing sDKFListNode */
+	IMG_UINT32          ui32NumEntries; /*!< Number of registered DKP entries */
+	IMG_UINT32          uiReserved1;    /*!< Reserved field - padding */
+	DLLIST_NODE         sDKFListNode;   /*!< Head of the DKF_REGISTERED_DKP linked list */
+	DKF_VPRINTF_FUNC    *pfnPrint;  /*!< Default printf-style output fn */
+	void                *pvPrintArg1; /*!< First arg for pfnPrint */
+} DKF;
+
+static DKF *gpsDKF;
+
+static_assert(DKF_CONNECTION_FLAG_INVALID == DKP_CONNECTION_FLAG_INVALID, "DKF and DKP INVALID connection flags do not match");
+static_assert(DKF_CONNECTION_FLAG_SYNC == DKP_CONNECTION_FLAG_SYNC, "DKF and DKP SYNC connection flags do not match");
+static_assert(DKF_CONNECTION_FLAG_SERVICES == DKP_CONNECTION_FLAG_SERVICES, "DKF and DKP SERVICES connection flags do not match");
+
+PVRSRV_ERROR PVRDKFInit(void)
+{
+	DKF *psDKF;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	if (gpsDKF != NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: gpsDKF = %p, NULL expected",
+		        __func__, gpsDKF));
+
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s called", __func__));
+
+	psDKF = OSAllocZMemNoStats(sizeof(*psDKF));
+	PVR_GOTO_IF_NOMEM(psDKF, eError, Error);
+
+	dllist_init(&psDKF->sDKFListNode);
+	eError = OSLockCreate(&psDKF->hDKFListLock);
+	PVR_GOTO_IF_ERROR(eError, ErrorFree);
+
+	gpsDKF = psDKF;
+
+	return PVRSRV_OK;
+
+ErrorFree:
+	OSFreeMemNoStats(psDKF);
+	/* fallthrough */
+
+Error:
+	PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+	return eError;
+}
+
+
+void PVRDKFDeInit(void)
+{
+	IMG_UINT32 uiNumFreed = 0U;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s called", __func__));
+
+	if (gpsDKF == NULL)
+	{
+		return;
+	}
+
+	/* Ensure we leave no data allocated. The DKP instances should have
+	 * been cleaned up by their module deInit processing. Handle badly
+	 * behaved clients here.
+	 */
+	if (gpsDKF->ui32NumEntries > 0)
+	{
+		DLLIST_NODE *psThis, *psNext;
+
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Have %u un-freed allocations remaining", __func__,
+		         gpsDKF->ui32NumEntries));
+
+		OSLockAcquire(gpsDKF->hDKFListLock);
+
+		dllist_foreach_node(&gpsDKF->sDKFListNode, psThis, psNext)
+		{
+			DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(psThis,
+			                                             DKF_REGISTERED_DKP,
+			                                             sDKFEntryNode);
+
+			dllist_remove_node(&psEntry->sDKFEntryNode);
+
+			uiNumFreed++;
+
+			OSFreeMemNoStats(psEntry);
+		}
+
+		if (uiNumFreed != gpsDKF->ui32NumEntries)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could only free %u out of %u",
+			         uiNumFreed, gpsDKF->ui32NumEntries));
+		}
+
+		dllist_remove_node(&gpsDKF->sDKFListNode);
+
+		OSLockRelease(gpsDKF->hDKFListLock);
+	}
+
+	OSLockDestroy(gpsDKF->hDKFListLock);
+
+	OSFreeMemNoStats(gpsDKF);
+
+	gpsDKF = NULL;
+}
+
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    DKF_CONNECTION_FLAGS ui32ConnectionType)
+{
+	PDLLIST_NODE pNext, pNode;
+
+	PVR_ASSERT(gpsDKF != NULL);
+	PVR_ASSERT(pfnPrint != NULL);
+	PVR_ASSERT(ui32ConnectionType != DKF_CONNECTION_FLAG_INVALID);
+
+	OSLockAcquire(gpsDKF->hDKFListLock);
+
+	gpsDKF->pfnPrint = pfnPrint;
+	gpsDKF->pvPrintArg1 = pvArg;
+
+	if (dllist_is_empty(&gpsDKF->sDKFListNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: No DKPs registered", __func__));
+	}
+	else
+	{
+		dllist_foreach_node(&gpsDKF->sDKFListNode, pNode, pNext)
+		{
+			DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(pNode,
+			                                               DKF_REGISTERED_DKP,
+			                                               sDKFEntryNode);
+
+			if (psEntry->psDKPShowPfn != NULL &&
+			    BITMASK_ANY(psEntry->ui32Filter, ui32ConnectionType))
+			{
+				psEntry->psDKPShowPfn(psDevNode, pid,
+				                      psEntry->hPrivHandle);
+			}
+		}
+	}
+
+	OSLockRelease(gpsDKF->hDKFListLock);
+}
+
+/*
+ * Wrapper function to display data using preconfigured DKF-specific output
+ * function.
+ */
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...)
+{
+	DKF_REGISTERED_DKP *psDKFEntry = (DKF_REGISTERED_DKP *)hPrivData;
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list Arglist;
+	DLLIST_NODE *pNode, *pNext;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (psDKFEntry == NULL || gpsDKF == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: NULL DKF entry found (%p, %p)",
+		        __func__, psDKFEntry, gpsDKF));
+		return;
+	}
+
+	PVR_ASSERT(gpsDKF->pfnPrint != NULL);
+
+	/* validate that this is a legitimate output function reference */
+
+	dllist_foreach_node(&gpsDKF->sDKFListNode, pNode, pNext)
+	{
+		DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(pNode,
+		                                               DKF_REGISTERED_DKP,
+		                                               sDKFEntryNode);
+
+		if (psEntry == psDKFEntry)
+		{
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!bFound)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Handle %p not found.", __func__,
+		         hPrivData));
+		return;
+	}
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN, "%s", fmt);
+
+	va_start(Arglist, fmt);
+	gpsDKF->pfnPrint(gpsDKF->pvPrintArg1, szBuffer, &Arglist);
+	va_end(Arglist);
+}
+
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                               const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	DKF_REGISTERED_DKP *psDKFEntry; /* New entry for this DKP */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	/* Check for a NULL argument. Nothing to allocate if we are not provided
+	 * a location to store the DkpHandle reference.
+	 */
+	if (phDkpHandle == NULL || ui32Filter == DKF_CONNECTION_FLAG_INVALID)
+	{
+
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_WARNING, "%s(%p, %s, %p, %p) Called", __func__,
+		         hPrivData, pszDKPName, psShowPfn,
+		         phDkpHandle));
+#endif
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDKFEntry = OSAllocZMemNoStats(sizeof(*psDKFEntry));
+	PVR_GOTO_IF_NOMEM(psDKFEntry, eError, Error);
+
+	OSLockAcquire(gpsDKF->hDKFListLock);
+
+	psDKFEntry->hPrivHandle = hPrivData;
+	psDKFEntry->pszDKPName = pszDKPName;
+	psDKFEntry->psDKPShowPfn = psShowPfn;
+	psDKFEntry->ui32Filter = ui32Filter;
+	dllist_init(&psDKFEntry->sDKFEntryNode);
+
+	/*
+	 * Append the new entry to the end of the gpsDKF list-head. Return a
+	 * reference to the new entry to the caller.
+	 */
+	dllist_add_to_tail(&gpsDKF->sDKFListNode, &psDKFEntry->sDKFEntryNode);
+
+	gpsDKF->ui32NumEntries++;
+
+	OSLockRelease(gpsDKF->hDKFListLock);
+
+	*phDkpHandle = psDKFEntry;
+
+	return PVRSRV_OK;
+
+
+Error:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Error: '%s'", __func__,
+	         PVRSRVGetErrorString(eError)));
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData, PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	DKF_REGISTERED_DKP *psDKFEntry = (DKF_REGISTERED_DKP *)hDkpHandle;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDKFEntry)
+	{
+#if defined(DEBUG)
+		if (psDKFEntry->hPrivHandle == hPrivData)
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Matched %p private handle",
+			         __func__, hDkpHandle));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_VERBOSE,
+			         "%s: Did not find match (%p. vs %p), freeing anyway",
+			         __func__, hPrivData, psDKFEntry->hPrivHandle));
+		}
+#endif	/* DEBUG */
+
+		OSLockAcquire(gpsDKF->hDKFListLock);
+
+		dllist_remove_node(&psDKFEntry->sDKFEntryNode);
+
+		gpsDKF->ui32NumEntries--;
+
+		OSLockRelease(gpsDKF->hDKFListLock);
+
+		OSFreeMemNoStats(psDKFEntry);
+
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+#else
+
+/* Stub routines for earlier kernel versions */
+
+PVRSRV_ERROR PVRDKFInit(void)
+{
+	return PVRSRV_OK;
+}
+
+void PVRDKFDeInit(void)
+{
+}
+
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    IMG_UINT32 ui32ConnectionType)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(pid);
+}
+
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(fmt);
+}
+
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData, const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pszDKPName);
+	PVR_UNREFERENCED_PARAMETER(psShowPfn);
+	PVR_UNREFERENCED_PARAMETER(phDkpHandle);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData, PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(hDkpHandle);
+
+	return PVRSRV_OK;
+}
+#endif	/* SUPPORT_LINUX_FDINFO */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.h
new file mode 100644
index 000000000000..304d9bc71b60
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/dkf_server.h
@@ -0,0 +1,103 @@
+/**************************************************************************/ /*!
+@File           dkf_server.h
+@Title          Functions for supporting the DRM Key Framework
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(DKF_SERVER_H)
+#define DKF_SERVER_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(SUPPORT_LINUX_FDINFO)
+
+#include <drm/drm_print.h>
+typedef void (DKF_VPRINTF_FUNC)(struct drm_printer *p, const char *fmt, va_list *va) __printf(2, 0);
+
+#else /* !defined(SUPPORT_LINUX_FDINFO) */
+typedef void (DKF_VPRINTF_FUNC)(void *p, const char *fmt, ...) __printf(2, 3);
+
+#endif	/* defined(SUPPORT_LINUX_FDINFO) */
+struct _PVRSRV_DEVICE_NODE_;
+
+typedef IMG_UINT32 DKF_CONNECTION_FLAGS;
+
+#define DKF_CONNECTION_FLAG_SYNC        BIT(0)
+#define DKF_CONNECTION_FLAG_SERVICES    BIT(1)
+
+#define DKF_CONNECTION_FLAG_INVALID     IMG_UINT32_C(0)
+
+/*! @Function PVRDKFTraverse
+ *
+ * @Description
+ * Outputs the DKP data associated with the given device node's
+ * framework entries.
+ *
+ * @Input   pfnPrint            The print function callback to be used to output.
+ * @Input   pvArg               Print function first argument.
+ * @Input   psDevNode           Device node associated with fdinfo owner.
+ * @Input   pid                 Process ID of the process owning the FD the fdinfo
+ *                              file relates to.
+ * @Input   ui32ConnectionType  A value indicating the PVR connection type
+ *                              (sync or services).
+ */
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    DKF_CONNECTION_FLAGS ui32ConnectionType);
+
+/* @Function PVRDKFInit
+ *
+ * @Description
+ * Initialises the DKF infrastructure for subsequent usage by the PVR system.
+ *
+ * @Returns  PVRSRV_ERROR.
+ */
+PVRSRV_ERROR PVRDKFInit(void);
+
+/* @Function PVRDKFDeInit
+ *
+ * @Description
+ * Removes and frees all associated system-specific DKF meta-data.
+ */
+void PVRDKFDeInit(void);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/env_connection.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/env_connection.h
new file mode 100644
index 000000000000..d2e4f9743370
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/env_connection.h
@@ -0,0 +1,96 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux specific server side connection management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(ENV_CONNECTION_H)
+#define ENV_CONNECTION_H
+
+#include <linux/version.h>
+#include <linux/list.h>
+#include <linux/types.h>
+
+#include "handle.h"
+#include "pvr_debug.h"
+#include "device.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_sys.h"
+#include "allocmem.h"
+#endif
+
+struct drm_file;
+
+typedef struct _ENV_CONNECTION_PRIVATE_DATA_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	struct drm_file *psDRMFile;
+} ENV_CONNECTION_PRIVATE_DATA;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#define ION_CLIENT_NAME_SIZE	50
+
+typedef struct _ENV_ION_CONNECTION_DATA_
+{
+	IMG_CHAR azIonClientName[ION_CLIENT_NAME_SIZE];
+	struct ion_device *psIonDev;
+	struct ion_client *psIonClient;
+} ENV_ION_CONNECTION_DATA;
+#endif
+
+typedef struct _ENV_CONNECTION_DATA_
+{
+	pid_t owner;
+
+	PVRSRV_DEVICE_NODE *psDevNode;
+	struct drm_file *psDRMFile;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	void *pvPvrSyncPrivateData;
+#endif
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	ENV_ION_CONNECTION_DATA *psIonData;
+#endif
+} ENV_CONNECTION_DATA;
+
+#endif /* !defined(ENV_CONNECTION_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.c
new file mode 100644
index 000000000000..98b029c926df
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.c
@@ -0,0 +1,516 @@
+/*************************************************************************/ /*!
+@File
+@Title          Event Object
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+
+#include <linux/version.h>
+#include <linux/string.h>
+#include <linux/sched.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,11,0)
+#include <linux/sched/signal.h>
+#endif
+#include <linux/interrupt.h>
+#include <asm/hardirq.h>
+#include <linux/timer.h>
+#include <linux/capability.h>
+#include <linux/freezer.h>
+#include <linux/uaccess.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "event.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "pvr_bridge_k.h"
+
+#include "osfunc.h"
+
+/* Uncomment to enable event object stats that are useful for debugging.
+ * The stats can be gotten at any time (during lifetime of event object)
+ * using OSEventObjectDumpdebugInfo API */
+// #define LINUX_EVENT_OBJECT_STATS
+
+
+typedef struct PVRSRV_LINUX_EVENT_OBJECT_LIST_TAG
+{
+	rwlock_t sLock;
+	/* Counts how many times event object was signalled i.e. how many times
+	 * LinuxEventObjectSignal() was called on a given event object.
+	 * Used for detecting pending signals.
+	 * Note that this is in no way related to OS signals. */
+	atomic_t sEventSignalCount;
+	struct list_head sList;
+} PVRSRV_LINUX_EVENT_OBJECT_LIST;
+
+
+typedef struct PVRSRV_LINUX_EVENT_OBJECT_TAG
+{
+	IMG_UINT32 ui32EventSignalCountPrevious;
+#if defined(DEBUG)
+	IMG_UINT ui32Stats;
+#endif
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	POS_LOCK hLock;
+	IMG_UINT32 ui32ScheduleAvoided;
+	IMG_UINT32 ui32ScheduleCalled;
+	IMG_UINT32 ui32ScheduleSleptFully;
+	IMG_UINT32 ui32ScheduleSleptPartially;
+	IMG_UINT32 ui32ScheduleReturnedImmediately;
+#endif
+	wait_queue_head_t sWait;
+	struct list_head sList;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList;
+} PVRSRV_LINUX_EVENT_OBJECT;
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectListCreate
+
+ @Description
+
+ Linux wait object list creation
+
+ @Output    hOSEventKM : Pointer to the event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectListCreate(IMG_HANDLE *phEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psEvenObjectList;
+
+	psEvenObjectList = OSAllocMem(sizeof(*psEvenObjectList));
+	if (psEvenObjectList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectCreate: failed to allocate memory for event list"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_LIST_HEAD(&psEvenObjectList->sList);
+
+	rwlock_init(&psEvenObjectList->sLock);
+	atomic_set(&psEvenObjectList->sEventSignalCount, 0);
+
+	*phEventObjectList = (IMG_HANDLE *) psEvenObjectList;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectListDestroy
+
+ @Description
+
+ Linux wait object list destruction
+
+ @Input    hOSEventKM : Event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectListDestroy(IMG_HANDLE hEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psEvenObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST *) hEventObjectList;
+
+	if (psEvenObjectList)
+	{
+		if (!list_empty(&psEvenObjectList->sList))
+		{
+			 PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectListDestroy: Event List is not empty"));
+			 return PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+		}
+		OSFreeMem(psEvenObjectList);
+		/*not nulling pointer, copy on stack*/
+	}
+	return PVRSRV_OK;
+}
+
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectDelete
+
+ @Description
+
+ Linux wait object removal
+
+ @Input    hOSEventObject : Event object handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectDelete(IMG_HANDLE hOSEventObject)
+{
+	if (hOSEventObject)
+	{
+		PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)hOSEventObject;
+		PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = psLinuxEventObject->psLinuxEventObjectList;
+
+		write_lock_bh(&psLinuxEventObjectList->sLock);
+		list_del(&psLinuxEventObject->sList);
+		write_unlock_bh(&psLinuxEventObjectList->sLock);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+		OSLockDestroy(psLinuxEventObject->hLock);
+#endif
+
+#if defined(DEBUG)
+//		PVR_DPF((PVR_DBG_MESSAGE, "LinuxEventObjectDelete: Event object waits: %u", psLinuxEventObject->ui32Stats));
+#endif
+
+		OSFreeMem(psLinuxEventObject);
+		/*not nulling pointer, copy on stack*/
+
+		return PVRSRV_OK;
+	}
+	return PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectAdd
+
+ @Description
+
+ Linux wait object addition
+
+ @Input    hOSEventObjectList : Event object list handle
+ @Output   phOSEventObject : Pointer to the event object handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectAdd(IMG_HANDLE hOSEventObjectList, IMG_HANDLE *phOSEventObject)
+ {
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST*)hOSEventObjectList;
+
+	/* allocate completion variable */
+	psLinuxEventObject = OSAllocMem(sizeof(*psLinuxEventObject));
+	if (psLinuxEventObject == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectAdd: failed to allocate memory"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_LIST_HEAD(&psLinuxEventObject->sList);
+
+	/* Start with the timestamp at which event object was added to the list */
+	psLinuxEventObject->ui32EventSignalCountPrevious = atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&psLinuxEventObject->hLock), "OSLockCreate");
+	psLinuxEventObject->ui32ScheduleAvoided = 0;
+	psLinuxEventObject->ui32ScheduleCalled = 0;
+	psLinuxEventObject->ui32ScheduleSleptFully = 0;
+	psLinuxEventObject->ui32ScheduleSleptPartially = 0;
+	psLinuxEventObject->ui32ScheduleReturnedImmediately = 0;
+#endif
+
+#if defined(DEBUG)
+	psLinuxEventObject->ui32Stats = 0;
+#endif
+	init_waitqueue_head(&psLinuxEventObject->sWait);
+
+	psLinuxEventObject->psLinuxEventObjectList = psLinuxEventObjectList;
+
+	write_lock_bh(&psLinuxEventObjectList->sLock);
+	list_add(&psLinuxEventObject->sList, &psLinuxEventObjectList->sList);
+	write_unlock_bh(&psLinuxEventObjectList->sLock);
+
+	*phOSEventObject = psLinuxEventObject;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectSignal
+
+ @Description
+
+ Linux wait object signaling function
+
+ @Input    hOSEventObjectList : Event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectSignal(IMG_HANDLE hOSEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST*)hOSEventObjectList;
+	struct list_head *psListEntry, *psListEntryTemp, *psList;
+	psList = &psLinuxEventObjectList->sList;
+
+	/* Move the timestamp ahead for this call, so a potential "Wait" from any
+	 * EventObject/s doesn't wait for the signal to occur before returning. Early
+	 * setting/incrementing of timestamp reduces the window where a concurrent
+	 * "Wait" call might block while "this" Signal call is being processed */
+	atomic_inc(&psLinuxEventObjectList->sEventSignalCount);
+
+	read_lock_bh(&psLinuxEventObjectList->sLock);
+	list_for_each_safe(psListEntry, psListEntryTemp, psList)
+	{
+		psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)list_entry(psListEntry, PVRSRV_LINUX_EVENT_OBJECT, sList);
+		wake_up_interruptible(&psLinuxEventObject->sWait);
+	}
+	read_unlock_bh(&psLinuxEventObjectList->sLock);
+
+	return PVRSRV_OK;
+}
+
+static void _TryToFreeze(void)
+{
+	/* if we reach zero it means that all of the threads called try_to_freeze */
+	LinuxBridgeNumActiveKernelThreadsDecrement();
+
+	/* Returns true if the thread was frozen, should we do anything with this
+	* information? What do we return? Which one is the error case? */
+	try_to_freeze();
+
+	LinuxBridgeNumActiveKernelThreadsIncrement();
+}
+
+void LinuxEventObjectDumpDebugInfo(IMG_HANDLE hOSEventObject)
+{
+#ifdef LINUX_EVENT_OBJECT_STATS
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)hOSEventObject;
+
+	OSLockAcquire(psLinuxEventObject->hLock);
+	PVR_LOG(("%s: EvObj(%p) schedule: Avoided(%u) Called(%u) ReturnedImmediately(%u) SleptFully(%u) SleptPartially(%u)",
+	         __func__, psLinuxEventObject, psLinuxEventObject->ui32ScheduleAvoided,
+			 psLinuxEventObject->ui32ScheduleCalled, psLinuxEventObject->ui32ScheduleReturnedImmediately,
+			 psLinuxEventObject->ui32ScheduleSleptFully, psLinuxEventObject->ui32ScheduleSleptPartially));
+	OSLockRelease(psLinuxEventObject->hLock);
+#else
+	PVR_LOG(("%s: LINUX_EVENT_OBJECT_STATS disabled!", __func__));
+#endif
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectWait
+
+ @Description
+
+ Linux wait object routine
+
+ @Input    hOSEventObject : Event object handle
+
+ @Input   ui64Timeoutus : Time out value in usec
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectWait(IMG_HANDLE hOSEventObject,
+                                  IMG_UINT64 ui64Timeoutus,
+                                  IMG_BOOL bFreezable)
+{
+	IMG_UINT32 ui32EventSignalCount;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32Remainder;
+	long timeOutJiffies;
+#ifdef LINUX_EVENT_OBJECT_STATS
+	long totalTimeoutJiffies;
+	IMG_BOOL bScheduleCalled = IMG_FALSE;
+#endif
+
+	DEFINE_WAIT(sWait);
+
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT*)hOSEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = psLinuxEventObject->psLinuxEventObjectList;
+
+	PVR_ASSERT(psLinuxEventObjectList != NULL);
+
+	/* Check if the driver is good shape */
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* usecs_to_jiffies only takes an uint. So if our timeout is bigger than an
+	 * uint use the msec version. With such a long timeout we really don't need
+	 * the high resolution of usecs. */
+	if (ui64Timeoutus > 0xffffffffULL)
+		timeOutJiffies = msecs_to_jiffies(OSDivide64(ui64Timeoutus, 1000, &ui32Remainder));
+	else
+		timeOutJiffies = usecs_to_jiffies(ui64Timeoutus);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	totalTimeoutJiffies = timeOutJiffies;
+#endif
+
+	do
+	{
+		prepare_to_wait(&psLinuxEventObject->sWait, &sWait, TASK_INTERRUPTIBLE);
+		ui32EventSignalCount = (IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+		if (psLinuxEventObject->ui32EventSignalCountPrevious != ui32EventSignalCount)
+		{
+			/* There is a pending event signal i.e. LinuxEventObjectSignal()
+			 * was called on the event object since the last time we checked.
+			 * Return without waiting. */
+			break;
+		}
+
+		if (signal_pending(current))
+		{
+			/* There is an OS signal pending so return.
+			 * This allows to kill/interrupt user space processes which
+			 * are waiting on this event object. */
+			break;
+		}
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+		bScheduleCalled = IMG_TRUE;
+#endif
+		timeOutJiffies = schedule_timeout(timeOutJiffies);
+
+		if (bFreezable)
+		{
+			_TryToFreeze();
+		}
+
+#if defined(DEBUG)
+		psLinuxEventObject->ui32Stats++;
+#endif
+
+
+	} while (timeOutJiffies);
+
+	finish_wait(&psLinuxEventObject->sWait, &sWait);
+
+	psLinuxEventObject->ui32EventSignalCountPrevious = ui32EventSignalCount;
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	OSLockAcquire(psLinuxEventObject->hLock);
+	if (bScheduleCalled)
+	{
+		psLinuxEventObject->ui32ScheduleCalled++;
+		if (totalTimeoutJiffies == timeOutJiffies)
+		{
+			psLinuxEventObject->ui32ScheduleReturnedImmediately++;
+		}
+		else if (timeOutJiffies == 0)
+		{
+			psLinuxEventObject->ui32ScheduleSleptFully++;
+		}
+		else
+		{
+			psLinuxEventObject->ui32ScheduleSleptPartially++;
+		}
+	}
+	else
+	{
+		psLinuxEventObject->ui32ScheduleAvoided++;
+	}
+	OSLockRelease(psLinuxEventObject->hLock);
+#endif
+
+	if (signal_pending(current) && test_tsk_thread_flag(current, TIF_SIGPENDING))
+	{
+		return PVRSRV_ERROR_INTERRUPTED;
+	}
+	else
+	{
+		return timeOutJiffies ? PVRSRV_OK : PVRSRV_ERROR_TIMEOUT;
+	}
+}
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+PVRSRV_ERROR LinuxEventObjectWaitUntilSignalled(IMG_HANDLE hOSEventObject)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	DEFINE_WAIT(sWait);
+
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject =
+			(PVRSRV_LINUX_EVENT_OBJECT *) hOSEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList =
+			psLinuxEventObject->psLinuxEventObjectList;
+
+	/* Check if the driver is in good shape */
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	prepare_to_wait(&psLinuxEventObject->sWait, &sWait, TASK_INTERRUPTIBLE);
+
+	if (psLinuxEventObject->ui32EventSignalCountPrevious !=
+	    (IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount))
+	{
+		/* There is a pending signal, so return without waiting */
+		goto finish;
+	}
+
+	schedule();
+
+	_TryToFreeze();
+
+finish:
+	finish_wait(&psLinuxEventObject->sWait, &sWait);
+
+	psLinuxEventObject->ui32EventSignalCountPrevious =
+			(IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.h
new file mode 100644
index 000000000000..bb378cb9220f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/event.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@File
+@Title          Event Object
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR LinuxEventObjectListCreate(IMG_HANDLE *phEventObjectList);
+PVRSRV_ERROR LinuxEventObjectListDestroy(IMG_HANDLE hEventObjectList);
+PVRSRV_ERROR LinuxEventObjectAdd(IMG_HANDLE hOSEventObjectList, IMG_HANDLE *phOSEventObject);
+PVRSRV_ERROR LinuxEventObjectDelete(IMG_HANDLE hOSEventObject);
+PVRSRV_ERROR LinuxEventObjectSignal(IMG_HANDLE hOSEventObjectList);
+PVRSRV_ERROR LinuxEventObjectWait(IMG_HANDLE hOSEventObject,
+                                  IMG_UINT64 ui64Timeoutus,
+                                  IMG_BOOL bFreezable);
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+PVRSRV_ERROR LinuxEventObjectWaitUntilSignalled(IMG_HANDLE hOSEventObject);
+#endif
+void LinuxEventObjectDumpDebugInfo(IMG_HANDLE hOSEventObject);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/fwload.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/fwload.c
new file mode 100644
index 000000000000..8c597b3c2e28
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/fwload.c
@@ -0,0 +1,259 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services firmware load and access routines for Linux
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/firmware.h>
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/err.h>
+
+#include "device.h"
+#include "module_common.h"
+#include "fwload.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+
+#if defined(RGX_FW_SIGNED)
+
+#include <linux/verification.h>
+#include <linux/module.h>
+#include <crypto/public_key.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <linux/module_signature.h>
+#else
+#define PKEY_ID_PKCS7 2
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) */
+
+#include "signfw.h"
+#endif /* RGX_FW_SIGNED */
+
+struct OS_FW_IMAGE_t
+{
+	const struct firmware *psFW;
+	size_t                 uSignatureSize;
+};
+
+#if defined(RGX_FW_SIGNED)
+
+static int OSCheckSignature(const struct FirmwareSignatureHeader *psHeader, size_t uSize)
+{
+	if (be32_to_cpu(psHeader->ui32SignatureLen) >= uSize - sizeof(*psHeader))
+	{
+		return -EBADMSG;
+	}
+
+	if (psHeader->ui8IDType != PKEY_ID_PKCS7)
+	{
+		return -ENOPKG;
+	}
+
+	if (psHeader->ui8Algo != 0 || psHeader->ui8HashAlgo != 0 ||
+	    psHeader->ui8SignerLen != 0 || psHeader->ui8KeyIDLen != 0 ||
+	    psHeader->__ui8Padding[0] != 0 || psHeader->__ui8Padding[1] != 0 ||
+	    psHeader->__ui8Padding[2] != 0)
+	{
+		return -EBADMSG;
+	}
+
+	return 0;
+}
+
+bool OSVerifyFirmware(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW        = psFWImage->psFW;
+	const u8              *pui8FWData  = psFW->data;
+	size_t                uFWSize      = psFW->size;
+	uint32_t              ui32MagicLen = sizeof(MODULE_SIG_STRING) - 1;
+	struct FirmwareSignatureHeader sHeader;
+	int                            err;
+
+	if (uFWSize <= ui32MagicLen)
+	{
+		return false;
+	}
+
+	/*
+	 * Linux Kernel's sign-file utility is primarily intended for signing
+	 * modules, and so appends the MODULE_SIG_STRING magic at the end of
+	 * the signature. Only proceed with verification if this magic is found.
+	 */
+	if (memcmp(pui8FWData + uFWSize - ui32MagicLen, MODULE_SIG_STRING, ui32MagicLen) != 0)
+	{
+		return false;
+	}
+
+	uFWSize -= ui32MagicLen;
+	if (uFWSize <= sizeof(sHeader))
+	{
+		return false;
+	}
+
+	/*
+	 * After the magic, a header is placed which informs about the digest /
+	 * crypto algorithm etc. Copy that header and ensure that it has valid
+	 * contents (We only support RSA Crypto, SHA Hash, X509 certificate and
+	 * PKCS#7 signature).
+	 */
+	memcpy(&sHeader, pui8FWData + (uFWSize - sizeof(sHeader)), sizeof(sHeader));
+	if (OSCheckSignature(&sHeader, uFWSize) != 0)
+	{
+		return false;
+	}
+
+	/*
+	 * As all information is now extracted, we can go ahead and ask PKCS
+	 * module to verify the sign.
+	 */
+	uFWSize -= be32_to_cpu(sHeader.ui32SignatureLen) + sizeof(sHeader);
+	err = verify_pkcs7_signature(pui8FWData, uFWSize, pui8FWData + uFWSize,
+				     be32_to_cpu(sHeader.ui32SignatureLen), NULL,
+				     VERIFYING_UNSPECIFIED_SIGNATURE, NULL, NULL);
+	if (err == 0)
+	{
+		psFWImage->uSignatureSize = psFW->size - uFWSize;
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware Successfully Verified",
+						__func__));
+		return true;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Verification Failed (%d)",
+					__func__, err));
+	return false;
+}
+
+#else /* defined(RGX_FW_SIGNED) */
+
+inline bool OSVerifyFirmware(OS_FW_IMAGE *psFWImage)
+{
+	return true;
+}
+
+#endif /* defined(RGX_FW_SIGNED) */
+
+PVRSRV_ERROR
+OSLoadFirmware(PVRSRV_DEVICE_NODE *psDeviceNode, const IMG_CHAR *pszBVNCString,
+               bool (*pfnVerifyFirmware)(OS_FW_IMAGE*), OS_FW_IMAGE **ppsFWImage)
+{
+	const struct firmware *psFW = NULL;
+	OS_FW_IMAGE *psFWImage;
+	IMG_INT32    res;
+	PVRSRV_ERROR eError;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0))
+	res = firmware_request_nowarn(&psFW, pszBVNCString, psDeviceNode->psDevConfig->pvOSDevice);
+#else
+	res = request_firmware(&psFW, pszBVNCString, psDeviceNode->psDevConfig->pvOSDevice);
+#endif
+	if (res != 0)
+	{
+		release_firmware(psFW);
+		if (res == -ENOENT)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: requested firmware('%s') not found (%d)",
+							__func__, pszBVNCString, res));
+			eError = PVRSRV_ERROR_NOT_FOUND;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: requested firmware('%s') not ready (%d)",
+							__func__, pszBVNCString, res));
+			eError = PVRSRV_ERROR_NOT_READY;
+		}
+		goto err_exit;
+	}
+
+	psFWImage = OSAllocZMem(sizeof(*psFWImage));
+	if (psFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: OSAllocZMem('%s') failed.",
+						__func__, pszBVNCString));
+
+		release_firmware(psFW);
+		eError =  PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_exit;
+	}
+
+	psFWImage->psFW = psFW;
+	if (pfnVerifyFirmware != NULL && !pfnVerifyFirmware(psFWImage))
+	{
+		release_firmware(psFW);
+		OSFreeMem(psFWImage);
+		eError = PVRSRV_ERROR_NOT_AUTHENTICATED;
+		goto err_exit;
+	}
+
+	*ppsFWImage = psFWImage;
+	return PVRSRV_OK;
+
+err_exit:
+	*ppsFWImage = NULL;
+	return eError;
+}
+
+void
+OSUnloadFirmware(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+
+	release_firmware(psFW);
+	OSFreeMem(psFWImage);
+}
+
+size_t
+OSFirmwareSize(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+	return psFW->size - psFWImage->uSignatureSize;
+}
+
+const void *
+OSFirmwareData(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+
+	return psFW->data;
+}
+
+/******************************************************************************
+ End of file (fwload.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/handle_idr.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/handle_idr.c
new file mode 100644
index 000000000000..48bb8f4410e3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/handle_idr.c
@@ -0,0 +1,417 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Handle Manager - IDR Back-end
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide IDR based resource handle management back-end
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/gfp.h>
+#include <linux/idr.h>
+#include <linux/pid.h>
+
+#include "handle_impl.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#define ID_VALUE_MIN	1
+#define ID_VALUE_MAX	INT_MAX
+
+#define	ID_TO_HANDLE(i) ((IMG_HANDLE)(uintptr_t)(i))
+#define	HANDLE_TO_ID(h) ((IMG_INT)(uintptr_t)(h))
+
+struct _HANDLE_IMPL_BASE_
+{
+	struct idr sIdr;
+
+	IMG_UINT32 ui32MaxHandleValue;
+
+	IMG_UINT32 ui32TotalHandCount;
+};
+
+typedef struct _HANDLE_ITER_DATA_WRAPPER_
+{
+	PFN_HANDLE_ITER pfnHandleIter;
+	void *pvHandleIterData;
+} HANDLE_ITER_DATA_WRAPPER;
+
+
+static int HandleIterFuncWrapper(int id, void *data, void *iter_data)
+{
+	HANDLE_ITER_DATA_WRAPPER *psIterData = (HANDLE_ITER_DATA_WRAPPER *)iter_data;
+
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	return (int)psIterData->pfnHandleIter(ID_TO_HANDLE(id), psIterData->pvHandleIterData);
+}
+
+/*!
+******************************************************************************
+
+ @Function	AcquireHandle
+
+ @Description	Acquire a new handle
+
+ @Input		psBase - Pointer to handle base structure
+		phHandle - Points to a handle pointer
+		pvData - Pointer to resource to be associated with the handle
+
+ @Output	phHandle - Points to a handle pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR AcquireHandle(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData)
+{
+	int id;
+	int result;
+
+	PVR_ASSERT(psBase != NULL);
+	PVR_ASSERT(phHandle != NULL);
+	PVR_ASSERT(pvData != NULL);
+
+	idr_preload(GFP_KERNEL);
+	id = idr_alloc(&psBase->sIdr, pvData, ID_VALUE_MIN, psBase->ui32MaxHandleValue + 1, 0);
+	idr_preload_end();
+
+	result = id;
+
+	if (result < 0)
+	{
+		if (result == -ENOSPC)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Limit of %u handles reached",
+				 __func__, psBase->ui32MaxHandleValue));
+
+			return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		}
+
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psBase->ui32TotalHandCount++;
+
+	*phHandle = ID_TO_HANDLE(id);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	ReleaseHandle
+
+ @Description	Release a handle that is no longer needed.
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle to release
+		ppvData - Points to a void data pointer
+
+ @Output	ppvData - Points to a void data pointer
+
+ @Return	PVRSRV_OK or PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR ReleaseHandle(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void **ppvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvData;
+
+	PVR_ASSERT(psBase);
+
+	/* Get the data associated with the handle. If we get back NULL then
+	   it's an invalid handle */
+
+	pvData = idr_find(&psBase->sIdr, id);
+	if (likely(pvData))
+	{
+		idr_remove(&psBase->sIdr, id);
+		psBase->ui32TotalHandCount--;
+	}
+
+	if (unlikely(pvData == NULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle out of range (%u > %u)",
+			 __func__, id, psBase->ui32TotalHandCount));
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+
+	if (ppvData)
+	{
+		*ppvData = pvData;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	GetHandleData
+
+ @Description	Get the data associated with the given handle
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle from which data should be retrieved
+		ppvData - Points to a void data pointer
+
+ @Output	ppvData - Points to a void data pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR GetHandleData(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void **ppvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvData;
+
+	PVR_ASSERT(psBase);
+	PVR_ASSERT(ppvData);
+
+	pvData = idr_find(&psBase->sIdr, id);
+	if (likely(pvData))
+	{
+		*ppvData = pvData;
+
+		return PVRSRV_OK;
+	}
+	else
+	{
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+}
+
+/*!
+******************************************************************************
+
+ @Function	SetHandleData
+
+ @Description	Set the data associated with the given handle
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle for which data should be changed
+		pvData - Pointer to new data to be associated with the handle
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR SetHandleData(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void *pvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvOldData;
+
+	PVR_ASSERT(psBase);
+
+	pvOldData = idr_replace(&psBase->sIdr, pvData, id);
+	if (IS_ERR(pvOldData))
+	{
+		if (PTR_ERR(pvOldData) == -ENOENT)
+		{
+			return PVRSRV_ERROR_HANDLE_NOT_ALLOCATED;
+		}
+		else
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR IterateOverHandles(HANDLE_IMPL_BASE *psBase, PFN_HANDLE_ITER pfnHandleIter, void *pvHandleIterData)
+{
+	HANDLE_ITER_DATA_WRAPPER sIterData;
+
+	PVR_ASSERT(psBase);
+	PVR_ASSERT(pfnHandleIter);
+
+	sIterData.pfnHandleIter = pfnHandleIter;
+	sIterData.pvHandleIterData = pvHandleIterData;
+
+	return (PVRSRV_ERROR)idr_for_each(&psBase->sIdr, HandleIterFuncWrapper, &sIterData);
+}
+
+/*!
+******************************************************************************
+
+ @Function	EnableHandlePurging
+
+ @Description	Enable purging for a given handle base
+
+ @Input		psBase - pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR EnableHandlePurging(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+	PVR_ASSERT(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PurgeHandles
+
+ @Description	Purge handles for a given handle base
+
+ @Input		psBase - Pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR PurgeHandles(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+	PVR_ASSERT(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	CreateHandleBase
+
+ @Description	Create a handle base structure
+
+ @Input		ppsBase - pointer to handle base structure pointer
+
+ @Output	ppsBase - points to handle base structure pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR CreateHandleBase(HANDLE_IMPL_BASE **ppsBase)
+{
+	HANDLE_IMPL_BASE *psBase;
+
+	PVR_ASSERT(ppsBase);
+
+	psBase = OSAllocZMem(sizeof(*psBase));
+	if (psBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't allocate generic handle base",
+				 __func__));
+
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	idr_init(&psBase->sIdr);
+
+	psBase->ui32MaxHandleValue = ID_VALUE_MAX;
+	psBase->ui32TotalHandCount = 0;
+
+	*ppsBase = psBase;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	DestroyHandleBase
+
+ @Description	Destroy a handle base structure
+
+ @Input		psBase - pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR DestroyHandleBase(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_ASSERT(psBase);
+
+	/* Finally destroy the idr */
+	idr_destroy(&psBase->sIdr);
+
+	OSFreeMem(psBase);
+
+	return PVRSRV_OK;
+}
+
+static const HANDLE_IMPL_FUNCTAB g_sHandleFuncTab =
+{
+	.pfnAcquireHandle = AcquireHandle,
+	.pfnReleaseHandle = ReleaseHandle,
+	.pfnGetHandleData = GetHandleData,
+	.pfnSetHandleData = SetHandleData,
+	.pfnIterateOverHandles = IterateOverHandles,
+	.pfnEnableHandlePurging = EnableHandlePurging,
+	.pfnPurgeHandles = PurgeHandles,
+	.pfnCreateHandleBase = CreateHandleBase,
+	.pfnDestroyHandleBase = DestroyHandleBase,
+};
+
+PVRSRV_ERROR PVRSRVHandleGetFuncTable(HANDLE_IMPL_FUNCTAB const **ppsFuncs)
+{
+	static IMG_BOOL bAcquired = IMG_FALSE;
+
+	if (bAcquired)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Function table already acquired",
+			 __func__));
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	if (ppsFuncs == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*ppsFuncs = &g_sHandleFuncTab;
+
+	bAcquired = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.c
new file mode 100644
index 000000000000..6e9bfd6f07d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.c
@@ -0,0 +1,1767 @@
+/*************************************************************************/ /*!
+@File           km_apphint.c
+@Title          Apphint routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "di_server.h"
+#include "pvr_uaccess.h"
+#include <linux/moduleparam.h>
+#include <linux/workqueue.h>
+#include <linux/string.h>
+
+/* Common and SO layer */
+#include "img_defs.h"
+#include "sofunc_pvr.h"
+
+/* for action device access */
+#include "pvrsrv.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+#include "rgxutils.h"
+#include "rgxapi_km.h"
+
+
+/* defines for default values */
+#include "rgx_fwif_km.h"
+#include "htbuffer_types.h"
+
+#include "pvr_notifier.h"
+
+#include "km_apphint_defs.h"
+#include "km_apphint.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#include "pdump_km.h"
+#endif
+
+/* Size of temporary buffers used to read and write AppHint data.
+ * Must be large enough to contain any strings read or written but no larger
+ * than 4096: which is the buffer size for the kernel_param_ops .get
+ * function. And less than 1024 to keep the stack frame size within bounds.
+ */
+#define APPHINT_BUFFER_SIZE 512
+
+/* Apphint Debug output level */
+#define APPHINT_DPF_LEVEL PVR_DBG_VERBOSE
+
+/*
+*******************************************************************************
+ * AppHint mnemonic data type helper tables
+******************************************************************************/
+struct apphint_lookup {
+	const char *name;
+	int value;
+};
+
+static const struct apphint_lookup fwt_logtype_tbl[] = {
+	{ "trace", 0},
+	{ "none", 0}
+#if defined(SUPPORT_TBI_INTERFACE)
+	, { "tbi", 1}
+#endif
+};
+
+static const struct apphint_lookup fwt_loggroup_tbl[] = {
+	RGXFWIF_LOG_GROUP_NAME_VALUE_MAP
+};
+
+static const struct apphint_lookup htb_loggroup_tbl[] = {
+#define X(a, b) { #b, HTB_LOG_GROUP_FLAG(a) },
+	HTB_LOG_SFGROUPLIST
+#undef X
+};
+
+static const struct apphint_lookup htb_opmode_tbl[] = {
+	{ "droplatest", HTB_OPMODE_DROPLATEST},
+	{ "dropoldest", HTB_OPMODE_DROPOLDEST},
+	{ "block", HTB_OPMODE_BLOCK}
+};
+
+__maybe_unused
+static const struct apphint_lookup htb_logmode_tbl[] = {
+	{ "all", HTB_LOGMODE_ALLPID},
+	{ "restricted", HTB_LOGMODE_RESTRICTEDPID}
+};
+
+__maybe_unused
+static const struct apphint_lookup timecorr_clk_tbl[] = {
+	{ "mono", 0 },
+	{ "mono_raw", 1 },
+	{ "sched", 2 }
+};
+
+/*
+*******************************************************************************
+ Data types
+******************************************************************************/
+union apphint_value {
+	IMG_UINT64 UINT64;
+	IMG_UINT32 UINT32;
+	IMG_BOOL BOOL;
+	IMG_CHAR *STRING;
+};
+
+union apphint_query_action {
+	PVRSRV_ERROR (*UINT64)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT64 *value);
+	PVRSRV_ERROR (*UINT32)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT32 *value);
+	PVRSRV_ERROR (*BOOL)(const PVRSRV_DEVICE_NODE *device,
+	                     const void *private_data, IMG_BOOL *value);
+	PVRSRV_ERROR (*STRING)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_CHAR **value);
+};
+
+union apphint_set_action {
+	PVRSRV_ERROR (*UINT64)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT64 value);
+	PVRSRV_ERROR (*UINT32)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT32 value);
+	PVRSRV_ERROR (*BOOL)(const PVRSRV_DEVICE_NODE *device,
+	                     const void *private_data, IMG_BOOL value);
+	PVRSRV_ERROR (*STRING)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_CHAR *value);
+};
+
+struct apphint_action {
+	union apphint_query_action query; /*!< Query callbacks. */
+	union apphint_set_action set;     /*!< Set callbacks. */
+	const PVRSRV_DEVICE_NODE *device; /*!< Pointer to the device node.*/
+	const void *private_data;         /*!< Opaque data passed to `query` and
+	                                       `set` callbacks. */
+	union apphint_value stored;       /*!< Value of the AppHint. */
+	bool free;                        /*!< Flag indicating that memory has been
+	                                       allocated for this AppHint and it
+	                                       needs to be freed on deinit. */
+	bool initialised;                 /*!< Flag indicating if the AppHint has
+	                                       been already initialised. */
+};
+
+struct apphint_param {
+	IMG_UINT32 id;
+	APPHINT_DATA_TYPE data_type;
+	const void *data_type_helper;
+	IMG_UINT32 helper_size;
+};
+
+struct apphint_init_data {
+	IMG_UINT32 id;          /* index into AppHint Table */
+	APPHINT_CLASS class;
+	const IMG_CHAR *name;
+	union apphint_value default_value;
+	APPHINT_RT_CLASS guest; /* ALWAYS => present on GUEST,
+	                           NEVER => not present on GUEST */
+};
+
+struct apphint_init_data_mapping {
+	IMG_UINT32 device_apphint_id;
+	IMG_UINT32 modparam_apphint_id;
+};
+
+struct apphint_class_state {
+	APPHINT_CLASS class;
+	IMG_BOOL enabled;
+};
+
+struct apphint_work {
+	struct work_struct work;
+	union apphint_value new_value;
+	struct apphint_action *action;
+};
+
+/*
+*******************************************************************************
+ Initialization / configuration table data
+******************************************************************************/
+#define UINT32Bitfield UINT32
+#define UINT32List UINT32
+
+static const struct apphint_init_data init_data_buildvar[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+};
+
+static const struct apphint_init_data init_data_modparam[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+};
+
+static const struct apphint_init_data init_data_debuginfo[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+};
+
+static const struct apphint_init_data init_data_debuginfo_device[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+};
+
+static const struct apphint_init_data_mapping init_data_debuginfo_device_to_modparams[] = {
+#define X(a, b) \
+	{APPHINT_ID_ ## a, APPHINT_ID_ ## b},
+	APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT_COMMON
+	APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT
+#undef X
+};
+
+#undef UINT32Bitfield
+#undef UINT32List
+
+__maybe_unused static const char NO_PARAM_TABLE[] = {};
+
+static const struct apphint_param param_lookup[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_DATA_TYPE_ ## b, e, ARRAY_SIZE(e) },
+	APPHINT_LIST_ALL
+#undef X
+};
+
+static const struct apphint_class_state class_state[] = {
+#define X(a) {APPHINT_CLASS_ ## a, APPHINT_ENABLED_CLASS_ ## a},
+	APPHINT_CLASS_LIST
+#undef X
+};
+
+/*
+*******************************************************************************
+ Global state
+******************************************************************************/
+/* If the union apphint_value becomes such that it is not possible to read
+ * and write atomically, a mutex may be desirable to prevent a read returning
+ * a partially written state.
+ * This would require a statically initialized mutex outside of the
+ * struct apphint_state to prevent use of an uninitialized mutex when
+ * module_params are provided on the command line.
+ *     static DEFINE_MUTEX(apphint_mutex);
+ */
+static struct apphint_state
+{
+	struct workqueue_struct *workqueue;
+	DI_GROUP *debuginfo_device_rootdir[PVRSRV_MAX_DEVICES];
+	DI_ENTRY *debuginfo_device_entry[PVRSRV_MAX_DEVICES][APPHINT_DEBUGINFO_DEVICE_ID_MAX];
+	DI_GROUP *debuginfo_rootdir;
+	DI_ENTRY *debuginfo_entry[APPHINT_DEBUGINFO_ID_MAX];
+	DI_GROUP *buildvar_rootdir;
+	DI_ENTRY *buildvar_entry[APPHINT_BUILDVAR_ID_MAX];
+
+	unsigned int num_devices;
+	PVRSRV_DEVICE_NODE *devices[PVRSRV_MAX_DEVICES];
+	unsigned int initialized;
+
+	/* Array contains value space for 1 copy of all apphint values defined
+	 * (for device 1) and N copies of device specific apphint values for
+	 * multi-device platforms.
+	 */
+	struct apphint_action val[APPHINT_ID_MAX + ((PVRSRV_MAX_DEVICES-1)*APPHINT_DEBUGINFO_DEVICE_ID_MAX)];
+
+} apphint = {
+/* statically initialise default values to ensure that any module_params
+ * provided on the command line are not overwritten by defaults.
+ */
+	.val = {
+#define UINT32Bitfield UINT32
+#define UINT32List UINT32
+#define X(a, b, c, d, e, f) \
+	{ {NULL}, {NULL}, NULL, NULL, {.b=d}, NULL },
+	APPHINT_LIST_ALL
+#undef X
+#undef UINT32Bitfield
+#undef UINT32List
+	},
+	.initialized = 0,
+	.num_devices = 0
+};
+
+#define APPHINT_DEBUGINFO_DEVICE_ID_OFFSET (APPHINT_ID_MAX-APPHINT_DEBUGINFO_DEVICE_ID_MAX)
+
+static inline void
+get_apphint_id_from_action_addr(const struct apphint_action * const addr,
+                                APPHINT_ID * const id)
+{
+	*id = (APPHINT_ID)(addr - apphint.val);
+	if (*id >= APPHINT_ID_MAX) {
+		*id -= APPHINT_DEBUGINFO_DEVICE_ID_OFFSET;
+		*id %= APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+		*id += APPHINT_DEBUGINFO_DEVICE_ID_OFFSET;
+	}
+}
+
+static inline void
+get_value_offset_from_device(const PVRSRV_DEVICE_NODE * const device,
+                             int * const offset,
+                             APPHINT_ID id)
+{
+	int i;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	/* No device offset if not a device specific apphint */
+	if (APPHINT_OF_DRIVER_NO_DEVICE == device) {
+		*offset = 0;
+		return;
+	}
+
+	/* Check that the specified ID is a device-specific one. If not we
+	 * set the offset to 0 for the global MODPARAM / BUILDVAR etc. AppHint
+	 */
+	for (i = 0; i < ARRAY_SIZE(init_data_debuginfo_device); i++)
+	{
+		const struct apphint_init_data *device_init = &init_data_debuginfo_device[i];
+
+		if ((IMG_UINT32)id == device_init->id) {
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!bFound) {
+		*offset = 0;
+		return;
+	}
+
+	for (i = 0; device && i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i] == device)
+			break;
+	}
+	if (PVRSRV_MAX_DEVICES == i) {
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unregistered device", __func__));
+		i = 0;
+	}
+	*offset = i * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+}
+
+/**
+ * apphint_action_worker - perform an action after an AppHint update has been
+ *                    requested by a UM process
+ *                    And update the record of the current active value
+ */
+static void apphint_action_worker(struct work_struct *work)
+{
+	struct apphint_work *work_pkt = container_of(work,
+	                                             struct apphint_work,
+	                                             work);
+	struct apphint_action *a = work_pkt->action;
+	union apphint_value value = work_pkt->new_value;
+	APPHINT_ID id;
+	PVRSRV_ERROR result = PVRSRV_OK;
+
+	get_apphint_id_from_action_addr(a, &id);
+
+	if (a->set.UINT64) {
+		switch (param_lookup[id].data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			result = a->set.UINT64(a->device,
+			                       a->private_data,
+			                       value.UINT64);
+			break;
+
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+		case APPHINT_DATA_TYPE_UINT32List:
+			result = a->set.UINT32(a->device,
+			                       a->private_data,
+			                       value.UINT32);
+			break;
+
+		case APPHINT_DATA_TYPE_BOOL:
+			result = a->set.BOOL(a->device,
+			                     a->private_data,
+			                     value.BOOL);
+			break;
+
+		case APPHINT_DATA_TYPE_STRING:
+			result = a->set.STRING(a->device,
+								   a->private_data,
+								   value.STRING);
+			kfree(value.STRING);
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: unrecognised data type (%d), index (%d)",
+			         __func__, param_lookup[id].data_type, id));
+		}
+
+		if (PVRSRV_OK != result) {
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: failed (%s)",
+			         __func__, PVRSRVGetErrorString(result)));
+		}
+	} else {
+		if (a->free) {
+			kfree(a->stored.STRING);
+		}
+		a->stored = value;
+		if (param_lookup[id].data_type == APPHINT_DATA_TYPE_STRING) {
+			a->free = true;
+		}
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: AppHint value updated before handler is registered, ID(%d)",
+		         __func__, id));
+	}
+	kfree((void *)work_pkt);
+}
+
+static void apphint_action(union apphint_value new_value,
+                           struct apphint_action *action)
+{
+	struct apphint_work *work_pkt = kmalloc(sizeof(*work_pkt), GFP_KERNEL);
+
+	/* queue apphint update on a serialized workqueue to avoid races */
+	if (work_pkt) {
+		work_pkt->new_value = new_value;
+		work_pkt->action = action;
+		INIT_WORK(&work_pkt->work, apphint_action_worker);
+		if (0 == queue_work(apphint.workqueue, &work_pkt->work)) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to queue apphint change request",
+				__func__));
+			goto err_exit;
+		}
+	} else {
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: failed to alloc memory for apphint change request",
+			__func__));
+		goto err_exit;
+	}
+	return;
+err_exit:
+	kfree(new_value.STRING);
+}
+
+/**
+ * apphint_read - read the different AppHint data types
+ * return -errno or the buffer size
+ */
+static int apphint_read(char *buffer, size_t count, APPHINT_ID ue,
+			 union apphint_value *value)
+{
+	APPHINT_DATA_TYPE data_type = param_lookup[ue].data_type;
+	int result = 0;
+
+	switch (data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		if (kstrtou64(buffer, 0, &value->UINT64) < 0) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid UINT64 input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32:
+		if (kstrtou32(buffer, 0, &value->UINT32) < 0) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid UINT32 input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_BOOL:
+		switch (buffer[0]) {
+		case '0':
+		case 'n':
+		case 'N':
+		case 'f':
+		case 'F':
+			value->BOOL = IMG_FALSE;
+			break;
+		case '1':
+		case 'y':
+		case 'Y':
+		case 't':
+		case 'T':
+			value->BOOL = IMG_TRUE;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid BOOL input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32List:
+	{
+		int i;
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *)
+			param_lookup[ue].data_type_helper;
+		int size = param_lookup[ue].helper_size;
+		/* buffer may include '\n', remove it */
+		char *arg = strsep(&buffer, "\n");
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < size; i++) {
+			if (strcasecmp(lookup[i].name, arg) == 0) {
+				value->UINT32 = lookup[i].value;
+				break;
+			}
+		}
+		if (i == size) {
+			if (OSStringLength(arg) == 0) {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: No value set for AppHint",
+					__func__));
+			} else {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unrecognised AppHint value (%s)",
+					__func__, arg));
+			}
+			result = -EINVAL;
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	{
+		int i;
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *)
+			param_lookup[ue].data_type_helper;
+		int size = param_lookup[ue].helper_size;
+		/* buffer may include '\n', remove it */
+		char *string = strsep(&buffer, "\n");
+		char *token = strsep(&string, ",");
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		value->UINT32 = 0;
+		/* empty string is valid to clear the bitfield */
+		while (token && *token) {
+			for (i = 0; i < size; i++) {
+				if (strcasecmp(lookup[i].name, token) == 0) {
+					value->UINT32 |= lookup[i].value;
+					break;
+				}
+			}
+			if (i == size) {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unrecognised AppHint value (%s)",
+					__func__, token));
+				result = -EINVAL;
+				goto err_exit;
+			}
+			token = strsep(&string, ",");
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_STRING:
+	{
+		/* buffer may include '\n', remove it */
+		char *string = strsep(&buffer, "\n");
+		size_t len = OSStringLength(string);
+
+		if (!len) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		++len;
+
+		value->STRING = kmalloc(len , GFP_KERNEL);
+		if (!value->STRING) {
+			result = -ENOMEM;
+			goto err_exit;
+		}
+
+		OSStringSafeCopy(value->STRING, string, len);
+		break;
+	}
+	default:
+		result = -EINVAL;
+		goto err_exit;
+	}
+
+err_exit:
+	return (result < 0) ? result : count;
+}
+
+static PVRSRV_ERROR get_apphint_value_from_action(const struct apphint_action * const action,
+                                                  union apphint_value * const value,
+                                                  const PVRSRV_DEVICE_NODE * const psDevNode)
+{
+	APPHINT_ID id;
+	APPHINT_DATA_TYPE data_type;
+	PVRSRV_ERROR result = PVRSRV_OK;
+	const PVRSRV_DEVICE_NODE *psDevice;
+
+	get_apphint_id_from_action_addr(action, &id);
+	data_type = param_lookup[id].data_type;
+
+	/* If we've got an entry that is APPHINT_OF_DRIVER_NO_DEVICE we should use
+	 * the higher-level psDevNode value instead. This is the device-node that is
+	 * associated with the original debug_dump request.
+	 * Note: if we're called with psDevNode == APPHINT_OF_DRIVER_NO_DEVICE
+	 * we attempt to use the first registered apphint.devices[0] (if any
+	 * devices have been presented). If we have no devices hooked into the
+	 * apphint mechanism we just return the default value for the AppHint.
+	 */
+	if (psDevNode == APPHINT_OF_DRIVER_NO_DEVICE) {
+		if (action->device == APPHINT_OF_DRIVER_NO_DEVICE) {
+			if (apphint.num_devices > 0) {
+				psDevice = apphint.devices[0];
+			} else {
+				PVR_DPF((PVR_DBG_ERROR,
+				        "Uninitialised AppHint device for AppHint index (%d)",
+				        id));
+				return PVRSRV_ERROR_RETRY;
+			}
+		} else {
+			psDevice = action->device;
+		}
+	} else {
+		if (action->device == APPHINT_OF_DRIVER_NO_DEVICE) {
+			if (psDevNode != action->device) {
+				return PVRSRV_ERROR_INVALID_DEVICE;
+			}
+			psDevice = psDevNode;
+		} else {
+			psDevice = action->device;
+		}
+	}
+
+	if (action->query.UINT64) {
+		switch (data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			result = action->query.UINT64(psDevice,
+			                              action->private_data,
+			                              &value->UINT64);
+			break;
+
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+		case APPHINT_DATA_TYPE_UINT32List:
+			result = action->query.UINT32(psDevice,
+			                              action->private_data,
+			                              &value->UINT32);
+			break;
+
+		case APPHINT_DATA_TYPE_BOOL:
+			result = action->query.BOOL(psDevice,
+			                            action->private_data,
+			                            &value->BOOL);
+			break;
+
+		case APPHINT_DATA_TYPE_STRING:
+			result = action->query.STRING(psDevice,
+			                              action->private_data,
+			                              &value->STRING);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: unrecognised data type (%d), index (%d)",
+			         __func__, data_type, id));
+		}
+	} else {
+		*value = action->stored;
+	}
+
+	if (PVRSRV_OK != result) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed (%d), index (%d)", __func__, result, id));
+	}
+
+	return result;
+}
+
+/*
+ * apphint_write - write the current AppHint data to a buffer
+ *
+ * Returns length written or -errno
+ */
+static int apphint_write(char *buffer, const size_t size,
+                         const struct apphint_action *a)
+{
+	const struct apphint_param *hint;
+	int result = 0;
+	APPHINT_ID id;
+	union apphint_value value;
+
+	get_apphint_id_from_action_addr(a, &id);
+	hint = &param_lookup[id];
+
+	result = get_apphint_value_from_action(a, &value, a->device);
+
+	switch (hint->data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		result += snprintf(buffer + result, size - result,
+				"0x%016llx",
+				value.UINT64);
+		break;
+	case APPHINT_DATA_TYPE_UINT32:
+		result += snprintf(buffer + result, size - result,
+				"0x%08x",
+				value.UINT32);
+		break;
+	case APPHINT_DATA_TYPE_BOOL:
+		result += snprintf(buffer + result, size - result,
+			"%s",
+			value.BOOL ? "Y" : "N");
+		break;
+	case APPHINT_DATA_TYPE_STRING:
+		if (value.STRING) {
+			result += snprintf(buffer + result, size - result,
+				"%s",
+				*value.STRING ? value.STRING : "(none)");
+		} else {
+			result += snprintf(buffer + result, size - result,
+			"(none)");
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32List:
+	{
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *) hint->data_type_helper;
+		IMG_UINT32 i;
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < hint->helper_size; i++) {
+			if (lookup[i].value == value.UINT32) {
+				result += snprintf(buffer + result,
+						size - result,
+						"%s",
+						lookup[i].name);
+				break;
+			}
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	{
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *) hint->data_type_helper;
+		IMG_UINT32 i;
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < hint->helper_size; i++) {
+			if (lookup[i].value & value.UINT32) {
+				result += snprintf(buffer + result,
+						size - result,
+						"%s,",
+						lookup[i].name);
+			}
+		}
+		if (result) {
+			/* remove any trailing ',' */
+			--result;
+			*(buffer + result) = '\0';
+		} else {
+			result += snprintf(buffer + result,
+					size - result, "none");
+		}
+		break;
+	}
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: unrecognised data type (%d), index (%d)",
+			 __func__, hint->data_type, id));
+		result = -EINVAL;
+	}
+
+err_exit:
+	return result;
+}
+
+/*
+*******************************************************************************
+ Module parameters initialization - different from debuginfo
+******************************************************************************/
+/*
+ * apphint_kparam_set - Handle an update of a module parameter
+ *
+ * Returns 0, or -errno.  arg is in kp->arg.
+ */
+static int apphint_kparam_set(const char *val, const struct kernel_param *kp)
+{
+	char val_copy[APPHINT_BUFFER_SIZE];
+	APPHINT_ID id;
+	union apphint_value value;
+	ssize_t result = OSStringSafeCopy(val_copy, val, APPHINT_BUFFER_SIZE);
+
+	/* Document the assumption - we can safely store the result of
+	 * apphint_read() in a ssize_t variable, no need for a separate one */
+#ifndef SSIZE_MAX
+#define SSIZE_MAX ((~(size_t)0) >> 1)
+#endif
+	BUILD_BUG_ON(INT_MAX > SSIZE_MAX);
+
+	if (result < 0) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: String too long", __func__));
+		return (int)result;
+	}
+
+	get_apphint_id_from_action_addr(kp->arg, &id);
+	result = apphint_read(val_copy, result, id, &value);
+	if (result < 0) {
+		return (int)result;
+	}
+
+	((struct apphint_action *)kp->arg)->stored = value;
+	((struct apphint_action *)kp->arg)->initialised = true;
+	if (param_lookup[id].data_type == APPHINT_DATA_TYPE_STRING) {
+		((struct apphint_action *)kp->arg)->free = true;
+	}
+
+	return 0;
+}
+
+/*
+ * apphint_kparam_get - handle a read of a module parameter
+ *
+ * Returns length written or -errno.  Buffer is 4k (ie. be short!)
+ */
+static int apphint_kparam_get(char *buffer, const struct kernel_param *kp)
+{
+	return apphint_write(buffer, PAGE_SIZE, kp->arg);
+}
+
+__maybe_unused
+static const struct kernel_param_ops apphint_kparam_fops = {
+	.set = apphint_kparam_set,
+	.get = apphint_kparam_get,
+};
+
+/*
+ * call module_param_cb() for all AppHints listed in APPHINT_LIST_MODPARAM_COMMON + APPHINT_LIST_MODPARAM
+ * apphint_modparam_class_ ## resolves to apphint_modparam_enable() except for
+ * AppHint classes that have been disabled.
+ */
+
+#define apphint_modparam_enable(name, number, perm) \
+	module_param_cb(name, &apphint_kparam_fops, &apphint.val[number], perm);
+
+#define X(a, b, c, d, e, f) \
+	apphint_modparam_class_ ##c(a, APPHINT_ID_ ## a, 0444)
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+
+/*
+*******************************************************************************
+ Debug Info get (seq file) operations - supporting functions
+******************************************************************************/
+static void *apphint_di_start(OSDI_IMPL_ENTRY *s, IMG_UINT64 *pos)
+{
+	if (*pos == 0) {
+		/* We want only one entry in the sequence, one call to show() */
+		return (void *) 1;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(s);
+
+	return NULL;
+}
+
+static void apphint_di_stop(OSDI_IMPL_ENTRY *s, void *v)
+{
+	PVR_UNREFERENCED_PARAMETER(s);
+	PVR_UNREFERENCED_PARAMETER(v);
+}
+
+static void *apphint_di_next(OSDI_IMPL_ENTRY *s, void *v, IMG_UINT64 *pos)
+{
+	PVR_UNREFERENCED_PARAMETER(s);
+	PVR_UNREFERENCED_PARAMETER(v);
+	(*pos)++;
+	return NULL;
+}
+
+static int apphint_di_show(OSDI_IMPL_ENTRY *s, void *v)
+{
+	IMG_CHAR km_buffer[APPHINT_BUFFER_SIZE];
+	int result;
+	void *private = DIGetPrivData(s);
+
+	PVR_UNREFERENCED_PARAMETER(v);
+
+	result = apphint_write(km_buffer, APPHINT_BUFFER_SIZE, private);
+	if (result < 0) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: failure", __func__));
+	} else {
+		/* debuginfo requires a trailing \n, module_params don't */
+		result += snprintf(km_buffer + result,
+				APPHINT_BUFFER_SIZE - result,
+				"\n");
+		DIPuts(s, km_buffer);
+	}
+
+	/* have to return 0 to see output */
+	return (result < 0) ? result : 0;
+}
+
+/*
+*******************************************************************************
+ Debug Info supporting functions
+******************************************************************************/
+
+/*
+ * apphint_set - Handle a DI value update
+ */
+static IMG_INT64 apphint_set(const IMG_CHAR *buffer, IMG_UINT64 count,
+                             IMG_UINT64 *ppos, void *data)
+{
+	APPHINT_ID id;
+	union apphint_value value;
+	struct apphint_action *action = data;
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	int result = 0;
+
+	if (ppos == NULL)
+		return -EIO;
+
+	if (count >= APPHINT_BUFFER_SIZE) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: String too long (%" IMG_INT64_FMTSPECd ")",
+			__func__, count));
+		result = -EINVAL;
+		goto err_exit;
+	}
+
+	/* apphint_read() modifies the buffer so we need to copy it */
+	memcpy(km_buffer, buffer, count);
+	/* count is larger than real buffer by 1 because DI framework appends
+	 * a '\0' character at the end, but here we're ignoring this */
+	count -= 1;
+	km_buffer[count] = '\0';
+
+	get_apphint_id_from_action_addr(action, &id);
+	result = apphint_read(km_buffer, count, id, &value);
+	if (result >= 0)
+		apphint_action(value, action);
+
+	*ppos += count;
+err_exit:
+	return result;
+}
+
+/*
+ * apphint_debuginfo_init - Create the specified debuginfo entries
+ */
+static int apphint_debuginfo_init(const char *sub_dir,
+		unsigned int device_num,
+		unsigned int init_data_size,
+		const struct apphint_init_data *init_data,
+		DI_GROUP *parentdir,
+		DI_GROUP **rootdir,
+		DI_ENTRY *entry[])
+{
+	PVRSRV_ERROR result;
+	unsigned int i;
+	unsigned int device_value_offset = device_num * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	const DI_ITERATOR_CB iterator = {
+		.pfnStart = apphint_di_start, .pfnStop = apphint_di_stop,
+		.pfnNext  = apphint_di_next,  .pfnShow = apphint_di_show,
+		.pfnWrite = apphint_set,      .ui32WriteLenMax = APPHINT_BUFFER_SIZE
+	};
+
+	/* Determine if we're booted as a GUEST VZ OS */
+	IMG_BOOL bIsGUEST = PVRSRV_VZ_MODE_IS(GUEST, DEVID, device_num);
+
+	if (*rootdir) {
+		PVR_DPF((PVR_DBG_WARNING,
+			"AppHint DebugFS already created, skipping"));
+		result = -EEXIST;
+		goto err_exit;
+	}
+
+	result = DICreateGroup(sub_dir, parentdir, rootdir);
+	if (result != PVRSRV_OK) {
+		PVR_DPF((PVR_DBG_WARNING,
+			"Failed to create \"%s\" DebugFS directory.", sub_dir));
+		goto err_exit;
+	}
+
+	for (i = 0; i < init_data_size; i++) {
+		if (!class_state[init_data[i].class].enabled)
+			continue;
+
+		/* Check to see if this AppHint should appear in a GUEST OS.
+		 * This will have a value in the init_data[i].guest field of ALWAYS
+		 * and if we don't have this set (and we're in GUEST mode) we must
+		 * not present this AppHint to the OS.
+		 */
+		if (bIsGUEST && (init_data[i].guest != APPHINT_RT_CLASS_ALWAYS))
+			continue;
+
+		result = DICreateEntry(init_data[i].name,
+				*rootdir,
+				&iterator,
+				(void *) &apphint.val[init_data[i].id + device_value_offset],
+				DI_ENTRY_TYPE_GENERIC,
+				&entry[i]);
+		if (result != PVRSRV_OK) {
+			PVR_DPF((PVR_DBG_WARNING,
+				"Failed to create \"%s/%s\" DebugFS entry.",
+				sub_dir, init_data[i].name));
+		}
+	}
+
+	return 0;
+
+err_exit:
+	return result;
+}
+
+/*
+ * apphint_debuginfo_deinit- destroy the debuginfo entries
+ */
+static void apphint_debuginfo_deinit(unsigned int num_entries,
+		DI_GROUP **rootdir,
+		DI_ENTRY *entry[])
+{
+	unsigned int i;
+
+	for (i = 0; i < num_entries; i++) {
+		if (entry[i]) {
+			DIDestroyEntry(entry[i]);
+		}
+	}
+
+	if (*rootdir) {
+		DIDestroyGroup(*rootdir);
+		*rootdir = NULL;
+	}
+}
+
+/*
+*******************************************************************************
+ AppHint status dump implementation
+******************************************************************************/
+#if defined(PDUMP)
+static void apphint_pdump_values(void *pvDeviceNode,
+                                 const IMG_CHAR *format, ...)
+{
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS;
+	va_list ap;
+
+	va_start(ap, format);
+	(void)vsnprintf(km_buffer, APPHINT_BUFFER_SIZE, format, ap);
+	va_end(ap);
+
+	/* ui32CommentSize set to 0 here as function does not make use of the value. */
+	PDumpCommentKM(NULL, (PVRSRV_DEVICE_NODE*)pvDeviceNode, 0, km_buffer, ui32Flags);
+}
+#endif
+
+static IMG_BOOL is_apphint_value_equal(const APPHINT_DATA_TYPE data_type,
+									const union apphint_value * const left,
+									const union apphint_value * const right)
+{
+		switch (data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			return left->UINT64 == right->UINT64;
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32List:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+			return left->UINT32 == right->UINT32;
+		case APPHINT_DATA_TYPE_BOOL:
+			return left->BOOL == right->BOOL;
+		case APPHINT_DATA_TYPE_STRING:
+			return (OSStringNCompare(left->STRING, right->STRING, OSStringLength(right->STRING) + 1) == 0 ? IMG_TRUE : IMG_FALSE);
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "%s: unhandled data type (%d)", __func__, data_type));
+			return IMG_FALSE;
+		}
+}
+
+static void apphint_dump_values(const char *group_name,
+			int device_num,
+			const struct apphint_init_data *group_data,
+			int group_size,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile,
+			bool list_all,
+			PVRSRV_DEVICE_NODE *psDevNode)
+{
+	int i, result;
+	int device_value_offset = device_num * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	char count = 0;
+
+	PVR_DUMPDEBUG_LOG("  %s", group_name);
+	for (i = 0; i < group_size; i++)
+	{
+		IMG_UINT32 id = group_data[i].id;
+		APPHINT_DATA_TYPE data_type = param_lookup[id].data_type;
+		const struct apphint_action *action = &apphint.val[id + device_value_offset];
+		union apphint_value value;
+
+		result = get_apphint_value_from_action(action, &value, psDevNode);
+
+		if (PVRSRV_OK != result) {
+			continue;
+		}
+
+		/* List only apphints with non-default values */
+		if (!list_all &&
+			is_apphint_value_equal(data_type, &value, &group_data[i].default_value)) {
+			continue;
+		}
+
+		result = apphint_write(km_buffer, APPHINT_BUFFER_SIZE, action);
+		count++;
+
+		if (result <= 0) {
+			PVR_DUMPDEBUG_LOG("    %s: <Error>",
+				group_data[i].name);
+		} else {
+			PVR_DUMPDEBUG_LOG("    %s: %s",
+				group_data[i].name, km_buffer);
+		}
+	}
+
+	if (count == 0) {
+		PVR_DUMPDEBUG_LOG("    none");
+	}
+}
+
+/*
+ * Callback for debug dump
+ */
+static void apphint_dump_state(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+			IMG_UINT32 ui32VerbLevel,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	int i, result;
+	char description_buffer[50];
+	PVRSRV_DEVICE_NODE *device = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH)) {
+		PVR_DUMPDEBUG_LOG("------[ AppHint Settings ]------");
+
+		apphint_dump_values("Build Vars", 0,
+			init_data_buildvar, ARRAY_SIZE(init_data_buildvar),
+			pfnDumpDebugPrintf, pvDumpDebugFile, true, device);
+
+		apphint_dump_values("Module Params", 0,
+			init_data_modparam, ARRAY_SIZE(init_data_modparam),
+			pfnDumpDebugPrintf, pvDumpDebugFile, false, device);
+
+		apphint_dump_values("Debug Info Params", 0,
+			init_data_debuginfo, ARRAY_SIZE(init_data_debuginfo),
+			pfnDumpDebugPrintf, pvDumpDebugFile, false, device);
+
+		for (i = 0; i < PVRSRV_MAX_DEVICES; i++) {
+			if (!apphint.devices[i]
+			    || (device && device != apphint.devices[i]))
+				continue;
+
+			result = snprintf(description_buffer,
+					  sizeof(description_buffer),
+					  "Debug Info Params Device ID: %d",
+					  i);
+			if (0 > result)
+				continue;
+
+			apphint_dump_values(description_buffer, i,
+					    init_data_debuginfo_device,
+					    ARRAY_SIZE(init_data_debuginfo_device),
+					    pfnDumpDebugPrintf,
+					    pvDumpDebugFile,
+						false, device);
+		}
+	}
+}
+
+/*
+*******************************************************************************
+ Public interface
+******************************************************************************/
+int pvr_apphint_init(void)
+{
+	int result, i;
+
+	if (apphint.initialized) {
+		result = -EEXIST;
+		goto err_out;
+	}
+
+	for (i = 0; i < PVRSRV_MAX_DEVICES; i++)
+		apphint.devices[i] = NULL;
+
+	/* create workqueue with strict execution ordering to ensure no
+	 * race conditions when setting/updating apphints from different
+	 * contexts
+	 */
+	apphint.workqueue = alloc_workqueue("apphint_workqueue",
+	                                    WQ_UNBOUND | WQ_FREEZABLE, 1);
+	if (!apphint.workqueue) {
+		result = -ENOMEM;
+		goto err_out;
+	}
+
+	result = apphint_debuginfo_init("apphint", 0,
+		ARRAY_SIZE(init_data_debuginfo), init_data_debuginfo,
+		NULL,
+		&apphint.debuginfo_rootdir, apphint.debuginfo_entry);
+	if (0 != result)
+		goto err_out;
+
+	result = apphint_debuginfo_init("buildvar", 0,
+		ARRAY_SIZE(init_data_buildvar), init_data_buildvar,
+		NULL,
+		&apphint.buildvar_rootdir, apphint.buildvar_entry);
+
+	apphint.initialized = 1;
+
+err_out:
+	return result;
+}
+
+int pvr_apphint_device_register(PVRSRV_DEVICE_NODE *device)
+{
+	int result, i;
+	unsigned int device_value_offset;
+
+	if (!apphint.initialized) {
+		result = -EAGAIN;
+		goto err_out;
+	}
+
+	if (apphint.num_devices+1 > PVRSRV_MAX_DEVICES) {
+		result = -EMFILE;
+		goto err_out;
+	}
+
+	/* Set the default values for the new device */
+	device_value_offset = apphint.num_devices * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	for (i = 0; i < APPHINT_DEBUGINFO_DEVICE_ID_MAX; i++) {
+		apphint.val[init_data_debuginfo_device[i].id + device_value_offset].stored
+			= init_data_debuginfo_device[i].default_value;
+	}
+
+	/* Set value of an apphint if mapping to module param exists for it
+	 * and this module parameter has been initialised */
+	for (i = 0; i < ARRAY_SIZE(init_data_debuginfo_device_to_modparams); i++) {
+		const struct apphint_init_data_mapping *mapping =
+			&init_data_debuginfo_device_to_modparams[i];
+		const struct apphint_action *modparam_action =
+			&apphint.val[mapping->modparam_apphint_id];
+		struct apphint_action *device_action =
+			&apphint.val[mapping->device_apphint_id + device_value_offset];
+
+		/* Set only if the module parameter was explicitly set during the module
+		 * load. */
+		if (modparam_action->initialised) {
+			device_action->stored = modparam_action->stored;
+		}
+	}
+
+	result = apphint_debuginfo_init("apphint", device->sDevId.ui32InternalID,
+	                              ARRAY_SIZE(init_data_debuginfo_device),
+	                              init_data_debuginfo_device,
+	                              device->sDebugInfo.psGroup,
+	                              &apphint.debuginfo_device_rootdir[device->sDevId.ui32InternalID],
+	                              apphint.debuginfo_device_entry[device->sDevId.ui32InternalID]);
+	if (0 != result)
+		goto err_out;
+
+	apphint.devices[device->sDevId.ui32InternalID] = device;
+	apphint.num_devices++;
+
+	(void)SOPvrDbgRequestNotifyRegister(
+			&device->hAppHintDbgReqNotify,
+			device,
+			apphint_dump_state,
+			DEBUG_REQUEST_APPHINT,
+			device);
+
+err_out:
+	return result;
+}
+
+void pvr_apphint_device_unregister(PVRSRV_DEVICE_NODE *device)
+{
+	int i;
+
+	if (!apphint.initialized)
+		return;
+
+	/* find the device */
+	for (i = 0; i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i] == device)
+			break;
+	}
+
+	if (PVRSRV_MAX_DEVICES == i)
+		return;
+
+	if (device->hAppHintDbgReqNotify) {
+		(void)SOPvrDbgRequestNotifyUnregister(
+			device->hAppHintDbgReqNotify);
+		device->hAppHintDbgReqNotify = NULL;
+	}
+
+	apphint_debuginfo_deinit(APPHINT_DEBUGINFO_DEVICE_ID_MAX,
+	                       &apphint.debuginfo_device_rootdir[i],
+	                       apphint.debuginfo_device_entry[i]);
+
+	apphint.devices[i] = NULL;
+
+	WARN_ON(apphint.num_devices==0);
+	apphint.num_devices--;
+}
+
+void pvr_apphint_deinit(void)
+{
+	int i;
+
+	if (!apphint.initialized)
+		return;
+
+	/* remove any remaining device data */
+	for (i = 0; apphint.num_devices && i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i])
+			pvr_apphint_device_unregister(apphint.devices[i]);
+	}
+
+	/* free all alloc'd string apphints and set to NULL */
+	for (i = 0; i < ARRAY_SIZE(apphint.val); i++) {
+		if (apphint.val[i].free && apphint.val[i].stored.STRING) {
+			kfree(apphint.val[i].stored.STRING);
+			apphint.val[i].stored.STRING = NULL;
+			apphint.val[i].free = false;
+		}
+	}
+
+	apphint_debuginfo_deinit(APPHINT_DEBUGINFO_ID_MAX,
+			&apphint.debuginfo_rootdir, apphint.debuginfo_entry);
+	apphint_debuginfo_deinit(APPHINT_BUILDVAR_ID_MAX,
+			&apphint.buildvar_rootdir, apphint.buildvar_entry);
+
+	destroy_workqueue(apphint.workqueue);
+
+	apphint.initialized = 0;
+}
+
+void pvr_apphint_dump_state(PVRSRV_DEVICE_NODE *device)
+{
+#if defined(PDUMP)
+	/* NB. apphint_pdump_values() is the pfnDumpDebugPrintf
+	 * function used when PDUMP is defined.
+	 * apphintpdump_values() calls PDumpCommentKM(), which
+	 * requires the device but as it is only called as a
+	 * DUMPDEBUG_PRINTF_FUNC it is only passed pvDumpDebugFile
+	 * (which happens to be the 4th parameter in the call to
+	 * apphint_dump_state() below).
+	 * Hence, we also need to pass device in the 4th parameter.
+	 */
+	apphint_dump_state(device, DEBUG_REQUEST_VERBOSITY_HIGH,
+	                   apphint_pdump_values, device);
+#endif
+	apphint_dump_state(device, DEBUG_REQUEST_VERBOSITY_HIGH,
+	                   NULL, NULL);
+}
+
+
+int pvr_apphint_get_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.UINT64;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.UINT64;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.UINT32;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.UINT32;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.BOOL;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.BOOL;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX && apphint.val[ue].stored.STRING) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			if (OSStringSafeCopy(pBuffer, apphint.val[ue + device_offset].stored.STRING, size) >= 0) {
+				error = 0;
+			}
+		}
+		else
+		{
+			if (OSStringSafeCopy(pBuffer, apphint.val[ue].stored.STRING, size) >= 0) {
+				error = 0;
+			}
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_set_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_UINT64)) {
+
+		if (apphint.val[ue + device_offset].set.UINT64) {
+			apphint.val[ue + device_offset].set.UINT64(apphint.val[ue + device_offset].device,
+													 apphint.val[ue + device_offset].private_data,
+													 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.UINT64 = Val;
+		}
+		apphint.val[ue].device = device;
+		error = 0;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_UINT32)) {
+
+		if (apphint.val[ue + device_offset].set.UINT32) {
+			apphint.val[ue + device_offset].set.UINT32(apphint.val[ue + device_offset].device,
+													 apphint.val[ue + device_offset].private_data,
+													 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.UINT32 = Val;
+		}
+		apphint.val[ue].device = device;
+		error = 0;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_BOOL)) {
+
+		error = 0;
+		if (apphint.val[ue + device_offset].set.BOOL) {
+			apphint.val[ue + device_offset].set.BOOL(apphint.val[ue + device_offset].device,
+												 apphint.val[ue + device_offset].private_data,
+												 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.BOOL = Val;
+		}
+		apphint.val[ue].device = device;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		((param_lookup[ue].data_type == APPHINT_DATA_TYPE_STRING) &&
+		apphint.val[ue + device_offset].stored.STRING)) {
+
+		if (apphint.val[ue + device_offset].set.STRING) {
+			error = apphint.val[ue + device_offset].set.STRING(apphint.val[ue + device_offset].device,
+															 apphint.val[ue + device_offset].private_data,
+															 pBuffer);
+		} else if (OSStringSafeCopy(apphint.val[ue + device_offset].stored.STRING, pBuffer, size) >= 0) {
+			error = 0;
+		}
+		apphint.val[ue].device = device;
+	}
+
+	return error;
+}
+
+void pvr_apphint_register_handlers_uint64(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.UINT64 = query,
+		.set.UINT64 = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_uint32(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_UINT32:
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	case APPHINT_DATA_TYPE_UINT32List:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.UINT32 = query,
+		.set.UINT32 = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_bool(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_BOOL:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.BOOL = query,
+		.set.BOOL = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_string(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR **value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR *value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_STRING:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.STRING = query,
+		.set.STRING = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.h
new file mode 100644
index 000000000000..71e2ce94000a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/km_apphint.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File           km_apphint.h
+@Title          Apphint internal header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux kernel AppHint control
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef KM_APPHINT_H
+#define KM_APPHINT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "pvrsrv_apphint.h"
+#include "km_apphint_defs.h"
+#include "device.h"
+
+int pvr_apphint_init(void);
+void pvr_apphint_deinit(void);
+int pvr_apphint_device_register(PVRSRV_DEVICE_NODE *device);
+void pvr_apphint_device_unregister(PVRSRV_DEVICE_NODE *device);
+void pvr_apphint_dump_state(PVRSRV_DEVICE_NODE *device);
+
+int pvr_apphint_get_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 *pVal);
+int pvr_apphint_get_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 *pVal);
+int pvr_apphint_get_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL *pVal);
+int pvr_apphint_get_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size);
+
+int pvr_apphint_set_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 Val);
+int pvr_apphint_set_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 Val);
+int pvr_apphint_set_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL Val);
+int pvr_apphint_set_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size);
+
+void pvr_apphint_register_handlers_uint64(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void * private_data);
+void pvr_apphint_register_handlers_uint32(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+void pvr_apphint_register_handlers_bool(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+void pvr_apphint_register_handlers_string(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR **value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR *value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* KM_APPHINT_H */
+
+/******************************************************************************
+ End of file (km_apphint.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/linkage.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/linkage.h
new file mode 100644
index 000000000000..3f24dc68eff6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/linkage.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux specific Services code internal interfaces
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Interfaces between various parts of the Linux specific
+                Services code, that don't have any other obvious
+                header file to go into.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(LINKAGE_H)
+#define LINKAGE_H
+
+PVRSRV_ERROR PVROSFuncInit(void);
+void PVROSFuncDeInit(void);
+
+#endif /* !defined(LINKAGE_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.c
new file mode 100644
index 000000000000..8df0593db286
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.c
@@ -0,0 +1,787 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common Linux module setup
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+
+#if defined(CONFIG_DEBUG_FS)
+#include "pvr_debugfs.h"
+#endif /* defined(CONFIG_DEBUG_FS) */
+#if defined(CONFIG_PROC_FS)
+#include "pvr_procfs.h"
+#endif /* defined(CONFIG_PROC_FS) */
+#include "di_server.h"
+#include "private_data.h"
+#include "linkage.h"
+#include "power.h"
+#include "env_connection.h"
+#include "process_stats.h"
+#include "module_common.h"
+#include "pvrsrv.h"
+#include "srvcore.h"
+#if defined(SUPPORT_RGX)
+#include "rgxdevice.h"
+#endif
+#include "pvrsrv_error.h"
+#include "pvr_drv.h"
+#include "pvr_bridge_k.h"
+
+#include "pvr_fence.h"
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+#include "pvr_sync.h"
+#if !defined(USE_PVRSYNC_DEVNODE)
+#include "pvr_sync_ioctl_drm.h"
+#endif
+#endif
+
+#include "ospvr_gputrace.h"
+
+#include "km_apphint.h"
+#include "srvinit.h"
+
+#include "pvr_ion_stats.h"
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+#include "physmem_osmem_linux.h"
+#endif
+#endif
+
+#include "dkf_server.h"
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+/* Display class interface */
+#include "kerneldisplay.h"
+EXPORT_SYMBOL(DCRegisterDevice);
+EXPORT_SYMBOL(DCUnregisterDevice);
+EXPORT_SYMBOL(DCDisplayConfigurationRetired);
+EXPORT_SYMBOL(DCDisplayHasPendingCommand);
+EXPORT_SYMBOL(DCImportBufferAcquire);
+EXPORT_SYMBOL(DCImportBufferRelease);
+
+EXPORT_SYMBOL(PVRSRVGetDriverStatus);
+EXPORT_SYMBOL(PVRSRVSystemInstallDeviceLISR);
+EXPORT_SYMBOL(PVRSRVSystemUninstallDeviceLISR);
+
+#include "pvr_notifier.h"
+EXPORT_SYMBOL(PVRSRVCheckStatus);
+#endif /* defined(SUPPORT_DISPLAY_CLASS) */
+
+#if defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+/*
+ * Physmem interface.
+ * Required by LMA DC drivers, and some non-DC LMA display drivers.
+ */
+#include "physheap.h"
+EXPORT_SYMBOL(PhysHeapAcquireByID);
+EXPORT_SYMBOL(PhysHeapRelease);
+EXPORT_SYMBOL(PhysHeapGetType);
+EXPORT_SYMBOL(PhysHeapGetCpuPAddr);
+EXPORT_SYMBOL(PhysHeapGetSize);
+EXPORT_SYMBOL(PhysHeapCpuPAddrToDevPAddr);
+
+#include "pvr_debug.h"
+EXPORT_SYMBOL(PVRSRVGetErrorString);
+EXPORT_SYMBOL(PVRSRVGetDeviceInstance);
+#endif
+
+#if defined(SUPPORT_RGX)
+#include "rgxapi_km.h"
+EXPORT_SYMBOL(RGXHWPerfConnect);
+EXPORT_SYMBOL(RGXHWPerfDisconnect);
+EXPORT_SYMBOL(RGXHWPerfControl);
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+EXPORT_SYMBOL(RGXHWPerfConfigureCounters);
+#else
+EXPORT_SYMBOL(RGXHWPerfConfigMuxCounters);
+EXPORT_SYMBOL(RGXHWPerfConfigureAndEnableCustomCounters);
+#endif
+EXPORT_SYMBOL(RGXHWPerfDisableCounters);
+EXPORT_SYMBOL(RGXHWPerfAcquireEvents);
+EXPORT_SYMBOL(RGXHWPerfReleaseEvents);
+EXPORT_SYMBOL(RGXHWPerfConvertCRTimeStamp);
+#if defined(SUPPORT_KERNEL_HWPERF_TEST)
+EXPORT_SYMBOL(OSAddTimer);
+EXPORT_SYMBOL(OSEnableTimer);
+EXPORT_SYMBOL(OSDisableTimer);
+EXPORT_SYMBOL(OSRemoveTimer);
+#endif
+#endif
+
+static int PVRSRVDeviceSyncOpen(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                                struct drm_file *psDRMFile);
+
+CONNECTION_DATA *LinuxServicesConnectionFromFile(struct file *pFile)
+{
+	if (pFile)
+	{
+		struct drm_file *psDRMFile;
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+
+		psDRMFile = pFile->private_data;
+		PVR_LOG_RETURN_IF_FALSE(psDRMFile != NULL, "psDRMFile is NULL", NULL);
+
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+		PVR_LOG_RETURN_IF_FALSE(psConnectionPriv != NULL, "psConnectionPriv is NULL", NULL);
+
+		return (CONNECTION_DATA*)psConnectionPriv->pvConnectionData;
+	}
+
+	return NULL;
+}
+
+CONNECTION_DATA *LinuxSyncConnectionFromFile(struct file *pFile)
+{
+	if (pFile)
+	{
+		struct drm_file *psDRMFile = pFile->private_data;
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+		return (CONNECTION_DATA*)psConnectionPriv->pvConnectionData;
+#else
+		return (CONNECTION_DATA*)psConnectionPriv->pvSyncConnectionData;
+#endif
+	}
+
+	return NULL;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDriverInit
+@Description  Common one time driver initialisation
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDriverInit(void)
+{
+	PVRSRV_ERROR error;
+	int os_err;
+
+	error = PVROSFuncInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	error = PVRDKFInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+	error = PVRSRVCommonDriverInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	error = pvr_sync_register_functions();
+	if (error != PVRSRV_OK)
+	{
+		return -EPERM;
+	}
+
+	os_err = pvr_sync_init();
+	if (os_err != 0)
+	{
+		return os_err;
+	}
+#endif
+
+	os_err = pvr_apphint_init();
+	if (os_err != 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed AppHint setup(%d)", __func__,
+			     os_err));
+	}
+
+#if defined(SUPPORT_RGX)
+	error = PVRGpuTraceSupportInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif
+
+#if defined(ANDROID)
+#if defined(CONFIG_PROC_FS)
+	error = PVRProcFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#elif defined(CONFIG_DEBUG_FS)
+	error = PVRDebugFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif /* defined(CONFIG_PROC_FS) || defined(CONFIG_DEBUG_FS) */
+#else
+#if defined(CONFIG_DEBUG_FS)
+	error = PVRDebugFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#elif defined(CONFIG_PROC_FS)
+	error = PVRProcFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif /* defined(CONFIG_DEBUG_FS) || defined(CONFIG_PROC_FS) */
+#endif /* defined(ANDROID) */
+
+	error = PVRSRVIonStatsInitialise();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+#if defined(SUPPORT_RGX)
+	/* calling here because we need to handle input from the file even
+	 * before the devices are initialised
+	 * note: we're not passing a device node because apphint callbacks don't
+	 * need it */
+	PVRGpuTraceInitAppHintCallbacks(NULL);
+#endif
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDriverDeinit
+@Description  Common one time driver de-initialisation
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDriverDeinit(void)
+{
+	pvr_apphint_deinit();
+
+	PVRSRVIonStatsDestroy();
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	pvr_sync_deinit();
+#endif
+
+#if defined(SUPPORT_RGX)
+	PVRGpuTraceSupportDeInit();
+#endif
+
+	PVRSRVCommonDriverDeInit();
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	PVRDKFDeInit();
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+	PVROSFuncDeInit();
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceInit
+@Description  Common device related initialisation.
+@Input        psDeviceNode  The device node for which initialisation should be
+                            performed
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	{
+		PVRSRV_ERROR eError = pvr_sync_device_init(psDeviceNode->psDevConfig->pvOSDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: unable to create sync (%d)",
+					 __func__, eError));
+			return -EBUSY;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_RGX)
+	{
+		int error = PVRGpuTraceInitDevice(psDeviceNode);
+		if (error != 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+				 "%s: failed to initialise PVR GPU Tracing on device%d (%d)",
+				 __func__, psDeviceNode->sDevId.i32KernelDeviceID, error));
+		}
+	}
+#endif
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceDeinit
+@Description  Common device related de-initialisation.
+@Input        psDeviceNode  The device node for which de-initialisation should
+                            be performed
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDeviceDeinit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_RGX)
+	PVRGpuTraceDeInitDevice(psDeviceNode);
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	pvr_sync_device_deinit(psDeviceNode->psDevConfig->pvOSDevice);
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	pvr_fence_cleanup();
+#endif
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceShutdown
+@Description  Common device shutdown.
+@Input        psDev  The device node representing the device that should
+                     be shutdown
+@Return       void
+*/ /***************************************************************************/
+
+void PVRSRVDeviceShutdown(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	/* Since this is a shutdown request ignore the returned error and try to
+	 * to power off the device. This is done because there is no way of
+	 * signalling the OS that this call failed. */
+	(void) LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+
+	/* Passing PVRSRV_POWER_FLAGS_NONE as there are no special actions required
+	 * from the shutdown call beside the regular device power off. */
+	(void) PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+	                                       PVRSRV_SYS_POWER_STATE_OFF,
+	                                       PVRSRV_POWER_FLAGS_NONE);
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceSuspend
+@Description  Common device suspend.
+@Input        psDev  The device node representing the device that should
+                     be suspended
+@Return       int    0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_FALSE);
+	PVR_LOG_RETURN_IF_FALSE_VA(eError == PVRSRV_OK, -EFAULT,
+	                           "LinuxBridgeBlockClientsAccess() failed with error %u",
+	                           eError);
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceResume
+@Description  Common device resume.
+@Input        psDev  The device node representing the device that should
+                     be resumed
+@Return       int    0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_ON,
+										PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceServicesOpen
+@Description  Services device open.
+@Input        psDeviceNode  The device node representing the device being
+                            opened by a user mode process
+@Input        psDRMFile     The DRM file data that backs the file handle
+                            returned to the user mode process
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceServicesOpen(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             struct drm_file *psDRMFile)
+{
+	static DEFINE_MUTEX(sDeviceInitMutex);
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	ENV_CONNECTION_PRIVATE_DATA sPrivData;
+	PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+	PVRSRV_ERROR eError;
+	int iErr = 0;
+
+	if (!psPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device data", __func__));
+		iErr = -ENODEV;
+		goto out;
+	}
+
+	mutex_lock(&sDeviceInitMutex);
+	/*
+	 * If the first attempt already set the state to bad,
+	 * there is no point in going the second time, so get out
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_BAD)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Driver already in bad state. Device open failed.",
+				 __func__));
+		iErr = -ENODEV;
+		mutex_unlock(&sDeviceInitMutex);
+		goto out;
+	}
+
+	if (psDRMFile->driver_priv == NULL)
+	{
+		/* Allocate psConnectionPriv (stores private data and release pfn under driver_priv) */
+		psConnectionPriv = kzalloc(sizeof(*psConnectionPriv), GFP_KERNEL);
+		if (!psConnectionPriv)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate driver_priv data", __func__));
+			iErr = -ENOMEM;
+			mutex_unlock(&sDeviceInitMutex);
+			goto fail_alloc_connection_priv;
+		}
+		psConnectionPriv->ui32Type = DKF_CONNECTION_FLAG_SERVICES;
+	}
+	else
+	{
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+	}
+
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_CREATED)
+	{
+		eError = PVRSRVCommonDeviceInitialise(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise device (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			iErr = -ENODEV;
+			mutex_unlock(&sDeviceInitMutex);
+			goto fail_device_init;
+		}
+
+#if defined(SUPPORT_RGX)
+		PVRGpuTraceInitIfEnabled(psDeviceNode);
+#endif
+	}
+	mutex_unlock(&sDeviceInitMutex);
+
+	sPrivData.psDevNode = psDeviceNode;
+	sPrivData.psDRMFile = psDRMFile;
+
+	/*
+	 * Here we pass the file pointer which will passed through to our
+	 * OSConnectionPrivateDataInit function where we can save it so
+	 * we can back reference the file structure from its connection
+	 */
+	eError = PVRSRVCommonConnectionConnect(&psConnectionPriv->pvConnectionData,
+	                                       (void *)&sPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		iErr = -ENOMEM;
+		goto fail_connect;
+	}
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pfDeviceRelease = PVRSRVCommonConnectionDisconnect;
+#endif
+	psDRMFile->driver_priv = (void*)psConnectionPriv;
+
+	goto out;
+
+fail_connect:
+fail_device_init:
+	kfree(psConnectionPriv);
+fail_alloc_connection_priv:
+out:
+	return iErr;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceSyncOpen
+@Description  Sync device open.
+@Input        psDeviceNode  The device node representing the device being
+                            opened by a user mode process
+@Input        psDRMFile     The DRM file data that backs the file handle
+                            returned to the user mode process
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+static int PVRSRVDeviceSyncOpen(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                struct drm_file *psDRMFile)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	CONNECTION_DATA *psConnection = NULL;
+	ENV_CONNECTION_PRIVATE_DATA sPrivData;
+	PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+	PVRSRV_ERROR eError;
+	int iErr = 0;
+
+	if (!psPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device data", __func__));
+		iErr = -ENODEV;
+		goto out;
+	}
+
+	if (psDRMFile->driver_priv == NULL)
+	{
+		/* Allocate psConnectionPriv (stores private data and release pfn under driver_priv) */
+		psConnectionPriv = kzalloc(sizeof(*psConnectionPriv), GFP_KERNEL);
+		if (!psConnectionPriv)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate driver_priv data", __func__));
+			iErr = -ENOMEM;
+			goto out;
+		}
+	}
+	else
+	{
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+	}
+
+	/* Allocate connection data area, no stats since process not registered yet */
+	psConnection = kzalloc(sizeof(*psConnection), GFP_KERNEL);
+	if (!psConnection)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate connection data", __func__));
+		iErr = -ENOMEM;
+		goto fail_alloc_connection;
+	}
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pvConnectionData = (void*)psConnection;
+#else
+	psConnectionPriv->pvSyncConnectionData = (void*)psConnection;
+#endif
+
+	psConnectionPriv->ui32Type = DKF_CONNECTION_FLAG_SYNC;
+
+	sPrivData.psDevNode = psDeviceNode;
+	sPrivData.psDRMFile = psDRMFile;
+
+	/* Call environment specific connection data init function */
+	eError = OSConnectionPrivateDataInit(&psConnection->hOsPrivateData, &sPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSConnectionPrivateDataInit() failed (%s)",
+		        __func__, PVRSRVGetErrorString(eError)));
+		goto fail_private_data_init;
+	}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	iErr = pvr_sync_open(psConnectionPriv->pvConnectionData, psDRMFile);
+#else
+	iErr = pvr_sync_open(psConnectionPriv->pvSyncConnectionData, psDRMFile);
+#endif
+	if (iErr)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: pvr_sync_open() failed(%d)",
+				__func__, iErr));
+		goto fail_pvr_sync_open;
+	}
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pfDeviceRelease = pvr_sync_close;
+#endif
+#endif
+	psDRMFile->driver_priv = psConnectionPriv;
+	goto out;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+fail_pvr_sync_open:
+	OSConnectionPrivateDataDeInit(psConnection->hOsPrivateData);
+#endif
+fail_private_data_init:
+	kfree(psConnection);
+fail_alloc_connection:
+	kfree(psConnectionPriv);
+out:
+	return iErr;
+#else /* if defined(SUPPORT_NATIVE_FENCE_SYNC) */
+	/* Ioctl should not be being called (no DDK support for native sync) */
+	PVR_DPF((PVR_DBG_ERROR, "%s: only supported when SUPPORT_NATIVE_FENCE_SYNC=1", __func__));
+	return -ENOTTY;
+#endif /* if defined(SUPPORT_NATIVE_FENCE_SYNC) */
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceRelease
+@Description  Common device release.
+@Input        psDeviceNode  The device node for the device that the given file
+                            represents
+@Input        psDRMFile     The DRM file data that's being released
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDeviceRelease(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         struct drm_file *psDRMFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	if (psDRMFile->driver_priv)
+	{
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+
+		if (psConnectionPriv->pvConnectionData)
+		{
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+			if (psConnectionPriv->pfDeviceRelease)
+			{
+				psConnectionPriv->pfDeviceRelease(psConnectionPriv->pvConnectionData);
+			}
+#else
+			if (psConnectionPriv->pvConnectionData)
+				PVRSRVCommonConnectionDisconnect(psConnectionPriv->pvConnectionData);
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+			if (psConnectionPriv->pvSyncConnectionData)
+				pvr_sync_close(psConnectionPriv->pvSyncConnectionData);
+#endif
+#endif
+		}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+		LinuxDeregisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+
+		kfree(psDRMFile->driver_priv);
+		psDRMFile->driver_priv = NULL;
+	}
+}
+
+int
+drm_pvr_srvkm_init(struct drm_device *dev, void *arg, struct drm_file *psDRMFile)
+{
+	struct drm_pvr_srvkm_init_data *data = arg;
+	struct pvr_drm_private *priv = dev->dev_private;
+	int iErr = 0;
+
+	switch (data->init_module)
+	{
+		case PVR_SRVKM_SYNC_INIT:
+		{
+			iErr = PVRSRVDeviceSyncOpen(priv->dev_node, psDRMFile);
+			break;
+		}
+		case PVR_SRVKM_SERVICES_INIT:
+		{
+			iErr = PVRSRVDeviceServicesOpen(priv->dev_node, psDRMFile);
+			if (iErr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDeviceServicesOpen() failed(%d)",
+						__func__, iErr));
+				break;
+			}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+			iErr = LinuxRegisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+			break;
+		}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		case PVR_SRVKM_SERVICES_PAGE_MIGRATE_INIT:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+			iErr = LinuxRegisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+			break;
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid init_module (%d)",
+			        __func__, data->init_module));
+			iErr = -EINVAL;
+		}
+	}
+
+	return iErr;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.h
new file mode 100644
index 000000000000..35f33770e2e5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/module_common.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@File           module_common.h
+@Title          Common linux module setup header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef MODULE_COMMON_H
+#define MODULE_COMMON_H
+
+#include "pvr_drm.h"
+
+/* DRVNAME is the name we use to register our driver. */
+#define DRVNAME PVR_LDM_DRIVER_REGISTRATION_NAME
+
+struct _PVRSRV_DEVICE_NODE_;
+struct drm_file;
+struct drm_device;
+
+/* psDRMFile->driver_priv will point to a PVRSV_CONNECTION_PRIV
+ * struct, which will contain a ptr to the CONNECTION_DATA and
+ * a pfn to the release function (which will differ depending
+ * on whether the connection is to Sync or Services).
+ */
+typedef void (*PFN_PVRSRV_DEV_RELEASE)(void *pvData);
+typedef struct
+{
+	/* pvConnectionData is used to hold Services connection data
+	 * for all PVRSRV_DEVICE_INIT_MODE options.
+	 */
+	void *pvConnectionData;
+
+	/* pfDeviceRelease is used to indicate the release function
+	 * to be called when PVRSRV_DEVICE_INIT_MODE is PVRSRV_LINUX_DEV_INIT_ON_CONNECT,
+	 * as we can then have one connections made (either for Services or Sync) per
+	 * psDRMFile, and need to know which type of connection is being released
+	 * (as the ioctl release call is common for both).
+	 */
+	PFN_PVRSRV_DEV_RELEASE pfDeviceRelease;
+
+	/* pvSyncConnectionData is used to hold Sync connection data
+	 * when PVRSRV_DEVICE_INIT_MODE is not PVRSRV_LINUX_DEV_INIT_ON_CONNECT,
+	 * as we can then have two connections made (for Services and Sync) to
+	 * the same psDRMFile.
+	 */
+	void *pvSyncConnectionData;
+
+	/* An integer representing the type of connection (indicated by the
+	 * DKF_CONNECTION_FLAGS).
+	 */
+	uint32_t ui32Type;
+
+} PVRSRV_CONNECTION_PRIV;
+
+int PVRSRVDriverInit(void);
+void PVRSRVDriverDeinit(void);
+
+int PVRSRVDeviceInit(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+void PVRSRVDeviceDeinit(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+void PVRSRVDeviceShutdown(struct drm_device *psDev);
+int PVRSRVDeviceSuspend(struct drm_device *psDev);
+int PVRSRVDeviceResume(struct drm_device *psDev);
+
+int PVRSRVDeviceServicesOpen(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                             struct drm_file *psDRMFile);
+void PVRSRVDeviceRelease(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                         struct drm_file *psDRMFile);
+int drm_pvr_srvkm_init(struct drm_device *dev,
+                       void *arg, struct drm_file *psDRMFile);
+
+#endif /* MODULE_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osconnection_server.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osconnection_server.c
new file mode 100644
index 000000000000..543b4d8ddabf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osconnection_server.c
@@ -0,0 +1,168 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux specific per process data functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "connection_server.h"
+#include "osconnection_server.h"
+
+#include "env_connection.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+
+#include <linux/sched.h>
+
+#if defined(SUPPORT_ION)
+#include <linux/err.h>
+#include PVR_ANDROID_ION_HEADER
+
+/*
+	The ion device (the base object for all requests)
+	gets created by the system and we acquire it via
+	Linux specific functions provided by the system layer
+*/
+#include "ion_sys.h"
+#endif
+
+PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData)
+{
+	ENV_CONNECTION_PRIVATE_DATA *psPrivData = pvOSData;
+	ENV_CONNECTION_DATA *psEnvConnection;
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	ENV_ION_CONNECTION_DATA *psIonConnection;
+#endif
+
+	*phOsPrivateData = OSAllocZMem(sizeof(ENV_CONNECTION_DATA));
+
+	if (*phOsPrivateData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem failed", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psEnvConnection = (ENV_CONNECTION_DATA *)*phOsPrivateData;
+
+	psEnvConnection->owner = current->tgid;
+
+	psEnvConnection->psDevNode = psPrivData->psDevNode;
+	psEnvConnection->psDRMFile = psPrivData->psDRMFile;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	psEnvConnection->pvPvrSyncPrivateData = NULL;
+#endif
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	psIonConnection = (ENV_ION_CONNECTION_DATA *)OSAllocZMem(sizeof(ENV_ION_CONNECTION_DATA));
+	if (psIonConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem failed", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psEnvConnection->psIonData = psIonConnection;
+	/*
+		We can have more than one connection per process, so we need
+		more than the PID to have a unique name.
+	*/
+	psEnvConnection->psIonData->psIonDev = IonDevAcquire();
+	OSSNPrintf(psEnvConnection->psIonData->azIonClientName, ION_CLIENT_NAME_SIZE, "pvr_ion_client-%p-%d", *phOsPrivateData, OSGetCurrentClientProcessIDKM());
+	psEnvConnection->psIonData->psIonClient =
+		ion_client_create(psEnvConnection->psIonData->psIonDev,
+						  psEnvConnection->psIonData->azIonClientName);
+
+	if (IS_ERR_OR_NULL(psEnvConnection->psIonData->psIonClient))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSConnectionPrivateDataInit: Couldn't create "
+								"ion client for per connection data"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+#endif /* SUPPORT_ION && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData)
+{
+	if (hOsPrivateData == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	{
+		ENV_CONNECTION_DATA *psEnvConnection = hOsPrivateData;
+
+		PVR_ASSERT(psEnvConnection->psIonData != NULL);
+
+		PVR_ASSERT(psEnvConnection->psIonData->psIonClient != NULL);
+		ion_client_destroy(psEnvConnection->psIonData->psIonClient);
+
+		IonDevRelease(psEnvConnection->psIonData->psIonDev);
+		OSFreeMem(psEnvConnection->psIonData);
+	}
+#endif
+
+	OSFreeMem(hOsPrivateData);
+	/*not nulling pointer, copy on stack*/
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_DEVICE_NODE *OSGetDevNode(CONNECTION_DATA *psConnection)
+{
+	ENV_CONNECTION_DATA *psEnvConnection;
+
+	psEnvConnection = PVRSRVConnectionPrivateData(psConnection);
+	PVR_ASSERT(psEnvConnection);
+
+	return psEnvConnection->psDevNode;
+}
+
+struct drm_file *OSGetDRMFile(CONNECTION_DATA *psConnection)
+{
+	ENV_CONNECTION_DATA *psEnvConnection;
+
+	psEnvConnection = PVRSRVConnectionPrivateData(psConnection);
+	PVR_ASSERT(psEnvConnection);
+
+	return psEnvConnection->psDRMFile;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc.c
new file mode 100644
index 000000000000..62390830ccd5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc.c
@@ -0,0 +1,2763 @@
+/*************************************************************************/ /*!
+@File
+@Title          Environment related functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <asm/page.h>
+#include <asm/div64.h>
+#include <linux/atomic.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/hugetlb.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/delay.h>
+#include <linux/genalloc.h>
+#include <linux/string.h>
+#include <linux/freezer.h>
+#include <asm/hardirq.h>
+#include <asm/tlbflush.h>
+#include <linux/timer.h>
+#include <linux/capability.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-direction.h>
+#include <linux/kthread.h>
+#include <linux/utsname.h>
+#include <linux/scatterlist.h>
+#include <linux/interrupt.h>
+#include <linux/pfn_t.h>
+#include <linux/pfn.h>
+#include <linux/pid.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+#include <linux/sched/clock.h>
+#include <linux/sched/signal.h>
+#else
+#include <linux/sched.h>
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) */
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+#include <linux/dma-heap.h>
+#include "physmem_dmabuf.h"
+#else
+#include "physmem.h"
+#endif
+#endif
+
+#include "log2.h"
+#include "sysinfo.h"
+#include "osfunc.h"
+#include "cache_km.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "devicemem_server_utils.h"
+#include "event.h"
+#include "linkage.h"
+#include "pvr_uaccess.h"
+#include "pvr_debug.h"
+#include "pvr_bridge_k.h"
+#include "pvrsrv_memallocflags.h"
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+#include "physmem_osmem_linux.h"
+#include "dma_support.h"
+
+#include "pvrsrv_sync_server.h"
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#include "pvr_ricommon.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+#if !defined(EVENT_OBJECT_TIMEOUT_US)
+#error EVENT_OBJECT_TIMEOUT_US should be defined sysinfo.h
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+#define PVR_FOLL_LONGTERM (0x0U)
+#else
+#define PVR_FOLL_LONGTERM FOLL_LONGTERM
+#endif
+
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(5, 6, 0))
+
+#define pvr_pin_user_pages_for_dma(puiAddress, num_pages, bWrite, pages)  get_user_pages_fast( \
+    (unsigned long)puiAddress, \
+    (int)num_pages, \
+    (int) (bWrite ? FOLL_WRITE : 0) | PVR_FOLL_LONGTERM, \
+    pages)
+#define pvr_unpin_user_page_for_dma(p) put_page(p)
+
+#else
+
+#define pvr_pin_user_pages_for_dma(puiAddress, num_pages, bWrite, pages) pin_user_pages_fast( \
+    (unsigned long)puiAddress, \
+    (int) num_pages, \
+    (int) (bWrite ? FOLL_WRITE : 0) | PVR_FOLL_LONGTERM, \
+    pages)
+#define pvr_unpin_user_page_for_dma(p) unpin_user_page(p)
+
+#endif
+
+#define _FREEZABLE IMG_TRUE
+#define _NON_FREEZABLE IMG_FALSE
+
+typedef struct {
+	struct task_struct *kthread;
+	PFN_THREAD pfnThread;
+	void *hData;
+	IMG_CHAR *pszThreadName;
+	IMG_BOOL   bIsThreadRunning;
+	IMG_BOOL   bIsSupportingThread;
+	PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB;
+	DLLIST_NODE sNode;
+} OSThreadData;
+
+void OSSuspendTaskInterruptible(void)
+{
+	set_current_state(TASK_INTERRUPTIBLE);
+	schedule();
+}
+
+static DLLIST_NODE gsThreadListHead;
+
+static void _ThreadListAddEntry(OSThreadData *psThreadListNode)
+{
+	dllist_add_to_tail(&gsThreadListHead, &(psThreadListNode->sNode));
+}
+
+static void _ThreadListRemoveEntry(OSThreadData *psThreadListNode)
+{
+	dllist_remove_node(&(psThreadListNode->sNode));
+}
+
+static void _ThreadSetStopped(OSThreadData *psOSThreadData)
+{
+	psOSThreadData->bIsThreadRunning = IMG_FALSE;
+}
+
+static void _OSInitThreadList(void)
+{
+	dllist_init(&gsThreadListHead);
+}
+
+void OSThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile)
+{
+	PDLLIST_NODE psNodeCurr, psNodeNext;
+	LINUX_THREAD_ACTIVITY_STATS sThreadStats;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	dllist_foreach_node(&gsThreadListHead, psNodeCurr, psNodeNext)
+	{
+		OSThreadData *psThreadListNode;
+		psThreadListNode = IMG_CONTAINER_OF(psNodeCurr, OSThreadData, sNode);
+
+		PVR_DUMPDEBUG_LOG("  %s : %s",
+				  psThreadListNode->pszThreadName,
+				  (psThreadListNode->bIsThreadRunning) ? "Running" : "Stopped");
+
+		if (psThreadListNode->pfnDebugDumpCB)
+		{
+			psThreadListNode->pfnDebugDumpCB(pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+	}
+
+	eError = LinuxGetThreadActivityStats(&sThreadStats);
+	if (eError == PVRSRV_OK)
+	{
+		PVR_DUMPDEBUG_LOG("Active Threads (UM/KM): %d / %d",
+		                  sThreadStats.i32DriverThreadCount,
+		                  sThreadStats.i32KernelThreadCount);
+
+		PVR_DUMPDEBUG_LOG("Suspended Threads: %d",
+		                  sThreadStats.i32SuspendedThreadCount);
+	}
+	else
+	{
+		PVR_LOG_ERROR(eError, "LinuxGetThreadActivityStats");
+	}
+}
+
+PVRSRV_ERROR OSPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+							PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+							IMG_PID uiPid)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+	IMG_CPU_PHYADDR sCpuPAddr;
+	struct page *psPage;
+	IMG_UINT32	ui32Order=0;
+	gfp_t gfp_flags;
+
+	PVR_ASSERT(uiSize != 0);
+	/*Align the size to the page granularity */
+	uiSize = PAGE_ALIGN(uiSize);
+
+	/*Get the order to be used with the allocation */
+	ui32Order = get_order(uiSize);
+
+	gfp_flags = GFP_KERNEL;
+
+#if !defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+	if (psDev)
+	{
+		if (*psDev->dma_mask == DMA_BIT_MASK(32))
+		{
+			/* Limit to 32 bit.
+			 * Achieved by setting __GFP_DMA32 for 64 bit systems */
+			gfp_flags |= __GFP_DMA32;
+		}
+		else if (*psDev->dma_mask < DMA_BIT_MASK(32))
+		{
+			/* Limit to whatever the size of DMA zone is. */
+			gfp_flags |= __GFP_DMA;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDev);
+#endif
+
+	/*allocate the pages */
+	psPage = alloc_pages(gfp_flags, ui32Order);
+	if (psPage == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	uiSize = (1 << ui32Order) * PAGE_SIZE;
+
+	psMemHandle->u.pvHandle = psPage;
+	psMemHandle->uiOrder = ui32Order;
+	sCpuPAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(psPage));
+
+	/*
+	 * Even when more pages are allocated as base MMU object we still need one single physical address because
+	 * they are physically contiguous.
+	 */
+	PhysHeapCpuPAddrToDevPAddr(psPhysHeap, 1, psDevPAddr, &sCpuPAddr);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                    uiSize,
+	                                    (IMG_UINT64)(uintptr_t) psPage,
+	                                    uiPid);
+#else
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                             psPage,
+	                             sCpuPAddr,
+	                             uiSize,
+	                             uiPid
+	                             DEBUG_MEMSTATS_VALUES);
+#endif
+#else
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif
+
+	return PVRSRV_OK;
+}
+
+void OSPhyContigPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle)
+{
+	struct page *psPage = (struct page*) psMemHandle->u.pvHandle;
+	IMG_UINT32 ui32Order;
+
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+	ui32Order = psMemHandle->uiOrder;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                      (IMG_UINT64)(uintptr_t) psPage);
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                (IMG_UINT64)(uintptr_t) psPage,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	__free_pages(psPage, ui32Order);
+	psMemHandle->uiOrder = 0;
+}
+
+PVRSRV_ERROR OSPhyContigPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle,
+						size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+						void **pvPtr)
+{
+	size_t actualSize = 1 << (PAGE_SHIFT + psMemHandle->uiOrder);
+	*pvPtr = kmap((struct page*)psMemHandle->u.pvHandle);
+
+	PVR_UNREFERENCED_PARAMETER(psDevPAddr);
+
+	PVR_UNREFERENCED_PARAMETER(actualSize); /* If we don't take an #ifdef path */
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA, actualSize, OSGetCurrentClientProcessIDKM());
+#else
+	{
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+									 *pvPtr,
+									 sCpuPAddr,
+									 actualSize,
+									 OSGetCurrentClientProcessIDKM()
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+}
+
+void OSPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	/* Mapping is done a page at a time */
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+	                            (1 << (PAGE_SHIFT + psMemHandle->uiOrder)),
+	                            OSGetCurrentClientProcessIDKM());
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+	                                (IMG_UINT64)(uintptr_t)pvPtr,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+	PVR_UNREFERENCED_PARAMETER(pvPtr);
+
+	kunmap((struct page*) psMemHandle->u.pvHandle);
+}
+
+PVRSRV_ERROR OSPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_UINT32 uiOffset,
+                                   IMG_UINT32 uiLength)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct page* psPage = (struct page*) psMemHandle->u.pvHandle;
+
+	void* pvVirtAddrStart = kmap(psPage) + uiOffset;
+	IMG_CPU_PHYADDR sPhysStart, sPhysEnd;
+
+	IMG_UINT32 ui32Order;
+
+	if (uiLength == 0)
+	{
+		goto e0;
+	}
+
+	ui32Order = psMemHandle->uiOrder;
+	if ((uiOffset + uiLength) > ((1 << ui32Order) * PAGE_SIZE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid size params, uiOffset %u, uiLength %u",
+				__func__,
+				uiOffset,
+				uiLength));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	sPhysStart.uiAddr = page_to_phys(psPage) + uiOffset;
+	sPhysEnd.uiAddr = sPhysStart.uiAddr + uiLength;
+
+	CacheOpExec(psDevNode,
+				pvVirtAddrStart,
+				pvVirtAddrStart + uiLength,
+				sPhysStart,
+				sPhysEnd,
+				PVRSRV_CACHE_OP_CLEAN);
+
+e0:
+	kunmap(psPage);
+
+	return eError;
+}
+
+#if defined(__GNUC__)
+#define PVRSRV_MEM_ALIGN __attribute__ ((aligned (0x8)))
+#define PVRSRV_MEM_ALIGN_MASK (0x7)
+#else
+#error "PVRSRV Alignment macros need to be defined for this compiler"
+#endif
+
+IMG_UINT32 OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE eCacheAttribute)
+{
+	IMG_UINT32 uiSize = 0;
+
+	switch (eCacheAttribute)
+	{
+		case OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE:
+			uiSize = cache_line_size();
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid cache attribute type %d",
+					__func__, (IMG_UINT32)eCacheAttribute));
+			PVR_ASSERT(0);
+			break;
+	}
+
+	return uiSize;
+}
+
+IMG_INT OSMemCmp(void *pvBufA, void *pvBufB, size_t uiLen)
+{
+	return (IMG_INT)memcmp(pvBufA, pvBufB, uiLen);
+}
+
+size_t OSStringLCat(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDstSize)
+{
+	/*
+	 * Let strlcat handle any truncation cases correctly.
+	 * We will definitely get a NUL-terminated string set in pszDest
+	 */
+	size_t uSrcSize = strlcat(pszDest, pszSrc, uDstSize);
+
+#if defined(PVR_DEBUG_STRLCPY) && defined(DEBUG)
+	/* Handle truncation by dumping calling stack if debug allows */
+	if (uSrcSize >= uDstSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+			"%s: String truncated Src = '<%s>' %ld bytes, Dest = '%s'",
+			__func__, pszSrc, (long)uDstSize, pszDest));
+		OSDumpStack();
+	}
+#endif /* defined(PVR_DEBUG_STRLCPY) && defined(DEBUG) */
+
+	return uSrcSize;
+}
+
+IMG_INT32 OSSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR *pszFormat, ...)
+{
+	va_list argList;
+	IMG_INT32 iCount;
+
+	va_start(argList, pszFormat);
+	iCount = vsnprintf(pStr, (size_t)ui32Size, pszFormat, argList);
+	va_end(argList);
+
+	return iCount;
+}
+
+IMG_INT32 OSVSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR* pszFormat, va_list vaArgs)
+{
+	return vsnprintf(pStr, ui32Size, pszFormat, vaArgs);
+}
+
+size_t OSStringLength(const IMG_CHAR *pStr)
+{
+	return strlen(pStr);
+}
+
+size_t OSStringNLength(const IMG_CHAR *pStr, size_t uiCount)
+{
+	return strnlen(pStr, uiCount);
+}
+
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                          size_t uiSize)
+{
+#if defined(DEBUG)
+	/* Double-check that we are not passing NULL parameters in. If we are we
+	 * return -1 (for arg1 == NULL, arg2 != NULL)
+	 * 0 (for arg1 == NULL, arg2 == NULL
+	 * +1 (for arg1 != NULL, arg2 == NULL)
+	 * strncmp(arg1, arg2, size) otherwise
+	 */
+	if (pStr1 == NULL)
+	{
+		if (pStr2 == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): Both args NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return 0;	/* Both NULL */
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): arg1 NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return -1;	/* NULL < non-NULL */
+		}
+	}
+	else
+	{
+		if (pStr2 == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): arg2 NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return +1;	/* non-NULL > NULL */
+		}
+		else
+		{
+			return strncmp(pStr1, pStr2, uiSize);
+		}
+	}
+#else
+	return strncmp(pStr1, pStr2, uiSize);
+#endif
+}
+
+PVRSRV_ERROR OSStringToUINT32(const IMG_CHAR *pStr, IMG_UINT32 ui32Base,
+                              IMG_UINT32 *ui32Result)
+{
+	if (kstrtou32(pStr, ui32Base, ui32Result) != 0)
+		return PVRSRV_ERROR_CONVERSION_FAILED;
+
+	return PVRSRV_OK;
+}
+
+IMG_UINT32 OSStringUINT32ToStr(IMG_CHAR *pszBuf, size_t uSize,
+							IMG_UINT32 ui32Num)
+{
+	IMG_UINT32 ui32i, ui32Len = 0, ui32NumCopy = ui32Num;
+
+	/* calculate string length required to hold the number string */
+	do
+	{
+		ui32Len++;
+		ui32NumCopy /= 10;
+	} while (ui32NumCopy != 0);
+
+	if (unlikely(ui32Len >= uSize))
+	{
+		/* insufficient buffer */
+		return 0;
+	}
+
+	for (ui32i = 0; ui32i < ui32Len; ui32i++)
+	{
+		pszBuf[ui32Len - (ui32i + 1)] = '0' + ui32Num % 10;
+		ui32Num = ui32Num / 10;
+	}
+
+	pszBuf[ui32Len] = '\0';
+	return ui32Len;
+}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+static struct workqueue_struct *gpFenceStatusWq;
+static struct workqueue_struct *gpFenceCtxDestroyWq;
+
+static PVRSRV_ERROR _NativeSyncInit(void)
+{
+	gpFenceStatusWq = create_freezable_workqueue("pvr_fence_status");
+	if (!gpFenceStatusWq)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create foreign fence status workqueue",
+				 __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	gpFenceCtxDestroyWq = create_freezable_workqueue("pvr_fence_context_destroy");
+	if (!gpFenceCtxDestroyWq)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create foreign fence context destruction workqueue",
+				 __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void _NativeSyncDeinit(void)
+{
+	destroy_workqueue(gpFenceStatusWq);
+	destroy_workqueue(gpFenceCtxDestroyWq);
+}
+
+struct workqueue_struct *NativeSyncGetFenceStatusWq(void)
+{
+	if (!gpFenceStatusWq)
+	{
+#if defined(DEBUG)
+		PVR_ASSERT(gpFenceStatusWq);
+#endif
+		return NULL;
+	}
+
+	return gpFenceStatusWq;
+}
+
+struct workqueue_struct *NativeSyncGetFenceCtxDestroyWq(void)
+{
+	if (!gpFenceCtxDestroyWq)
+	{
+#if defined(DEBUG)
+		PVR_ASSERT(gpFenceCtxDestroyWq);
+#endif
+		return NULL;
+	}
+
+	return gpFenceCtxDestroyWq;
+}
+#endif
+
+PVRSRV_ERROR OSInitEnvData(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = LinuxInitPhysmem();
+	PVR_GOTO_IF_ERROR(eError, error_out);
+
+	_OSInitThreadList();
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	eError = _NativeSyncInit();
+#endif
+
+error_out:
+	return eError;
+}
+
+void OSDeInitEnvData(void)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	_NativeSyncDeinit();
+#endif
+
+	LinuxDeinitPhysmem();
+}
+
+void OSReleaseThreadQuanta(void)
+{
+	schedule();
+}
+
+void OSMemoryBarrier(volatile void *hReadback)
+{
+	mb();
+
+	if (hReadback)
+	{
+		/* Force a read-back to memory to avoid posted writes on certain buses
+		 * e.g. PCI(E)
+		 */
+		(void) OSReadDeviceMem32(hReadback);
+	}
+}
+
+void OSWriteMemoryBarrier(volatile void *hReadback)
+{
+	wmb();
+
+	if (hReadback)
+	{
+		/* Force a read-back to memory to avoid posted writes on certain buses
+		 * e.g. PCI(E)
+		 */
+		(void) OSReadDeviceMem32(hReadback);
+	}
+}
+
+/* Not matching/aligning this API to the Clockus() API above to avoid necessary
+ * multiplication/division operations in calling code.
+ */
+static inline IMG_UINT64 Clockns64(void)
+{
+	IMG_UINT64 timenow;
+
+	/* Kernel thread preempt protection. Some architecture implementations
+	 * (ARM) of sched_clock are not preempt safe when the kernel is configured
+	 * as such e.g. CONFIG_PREEMPT and others.
+	 */
+	preempt_disable();
+
+	/* Using sched_clock instead of ktime_get since we need a time stamp that
+	 * correlates with that shown in kernel logs and trace data not one that
+	 * is a bit behind. */
+	timenow = sched_clock();
+
+	preempt_enable();
+
+	return timenow;
+}
+
+IMG_UINT64 OSClockns64(void)
+{
+	return Clockns64();
+}
+
+IMG_UINT64 OSClockus64(void)
+{
+	IMG_UINT64 timenow = Clockns64();
+	IMG_UINT32 remainder;
+
+	return OSDivide64r64(timenow, 1000, &remainder);
+}
+
+IMG_UINT32 OSClockus(void)
+{
+	return (IMG_UINT32) OSClockus64();
+}
+
+IMG_UINT32 OSClockms(void)
+{
+	IMG_UINT64 timenow = Clockns64();
+	IMG_UINT32 remainder;
+
+	return OSDivide64(timenow, 1000000, &remainder);
+}
+
+static inline IMG_UINT64 KClockns64(void)
+{
+	ktime_t sTime = ktime_get();
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return sTime;
+#else
+	return sTime.tv64;
+#endif
+}
+
+PVRSRV_ERROR OSClockMonotonicns64(IMG_UINT64 *pui64Time)
+{
+	*pui64Time = KClockns64();
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSClockMonotonicus64(IMG_UINT64 *pui64Time)
+{
+	IMG_UINT64 timenow = KClockns64();
+	IMG_UINT32 remainder;
+
+	*pui64Time = OSDivide64r64(timenow, 1000, &remainder);
+	return PVRSRV_OK;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+IMG_UINT64 OSClockMonotonicRawns64(void)
+{
+	struct timespec64 ts;
+
+	ktime_get_raw_ts64(&ts);
+	return ts.tv_sec * 1000000000 + ts.tv_nsec;
+}
+#else
+IMG_UINT64 OSClockMonotonicRawns64(void)
+{
+	struct timespec ts;
+
+	getrawmonotonic(&ts);
+	return (IMG_UINT64) ts.tv_sec * 1000000000 + ts.tv_nsec;
+}
+#endif
+
+IMG_UINT64 OSClockMonotonicRawus64(void)
+{
+	IMG_UINT32 rem;
+	return OSDivide64r64(OSClockMonotonicRawns64(), 1000, &rem);
+}
+
+/*
+	OSWaitus
+*/
+void OSWaitus(IMG_UINT32 ui32Timeus)
+{
+	udelay(ui32Timeus);
+}
+
+
+/*
+	OSSleepms
+*/
+void OSSleepms(IMG_UINT32 ui32Timems)
+{
+	msleep(ui32Timems);
+}
+
+
+INLINE IMG_UINT64 OSGetCurrentProcessVASpaceSize(void)
+{
+	return (IMG_UINT64)TASK_SIZE;
+}
+
+INLINE IMG_PID OSGetCurrentProcessID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return (IMG_PID)task_tgid_nr(current);
+}
+
+INLINE IMG_PID OSGetCurrentVirtualProcessID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return (IMG_PID)task_tgid_vnr(current);
+}
+
+INLINE IMG_CHAR *OSGetCurrentProcessName(void)
+{
+	return current->comm;
+}
+
+INLINE uintptr_t OSGetCurrentThreadID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return current->pid;
+}
+
+IMG_PID OSGetCurrentClientProcessIDKM(void)
+{
+	return OSGetCurrentProcessID();
+}
+
+IMG_CHAR *OSGetCurrentClientProcessNameKM(void)
+{
+	return OSGetCurrentProcessName();
+}
+
+uintptr_t OSAcquireCurrentPPIDResourceRefKM(void)
+{
+	struct pid *psPPIDResource = find_pid_ns(OSGetCurrentClientProcessIDKM(), &init_pid_ns);
+
+	PVR_ASSERT(psPPIDResource != NULL);
+	/* Take ref on pPid */
+	get_pid(psPPIDResource);
+	return (uintptr_t)psPPIDResource;
+}
+
+void OSReleasePPIDResourceRefKM(uintptr_t psPPIDResource)
+{
+	/* Drop ref on uiProc */
+	put_pid((struct pid*)psPPIDResource);
+}
+
+uintptr_t OSGetCurrentClientThreadIDKM(void)
+{
+	return OSGetCurrentThreadID();
+}
+
+size_t OSGetPageSize(void)
+{
+	return PAGE_SIZE;
+}
+
+size_t OSGetPageShift(void)
+{
+	return PAGE_SHIFT;
+}
+
+size_t OSGetPageMask(void)
+{
+	return (OSGetPageSize()-1);
+}
+
+size_t OSGetOrder(size_t uSize)
+{
+	return get_order(PAGE_ALIGN(uSize));
+}
+
+IMG_UINT64 OSGetRAMSize(void)
+{
+	struct sysinfo SI;
+	si_meminfo(&SI);
+
+	return (PAGE_SIZE * SI.totalram);
+}
+
+typedef struct
+{
+	int os_error;
+	PVRSRV_ERROR pvr_error;
+} error_map_t;
+
+#define PVRSRV_ERROR_TO_OS_ERROR \
+	X(PVRSRV_OK, 0) \
+	X(PVRSRV_ERROR_BRIDGE_EFAULT, EFAULT) \
+	X(PVRSRV_ERROR_BRIDGE_EINVAL, EINVAL) \
+	X(PVRSRV_ERROR_BRIDGE_ENOMEM, ENOMEM) \
+	X(PVRSRV_ERROR_BRIDGE_ERANGE, ERANGE) \
+	X(PVRSRV_ERROR_BRIDGE_EPERM, EPERM) \
+	X(PVRSRV_ERROR_BRIDGE_ENOTTY, ENOTTY) \
+	X(PVRSRV_ERROR_BRIDGE_CALL_FAILED, ENOTTY) \
+	X(PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL, ERANGE) \
+	X(PVRSRV_ERROR_OUT_OF_MEMORY, ENOMEM) \
+	X(PVRSRV_ERROR_PMR_NOT_PERMITTED, EACCES) \
+	X(PVRSRV_ERROR_INVALID_PARAMS, EINVAL) \
+	X(PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING, EPERM) \
+	X(PVRSRV_ERROR_NOT_IMPLEMENTED, ENOSYS) \
+	X(PVRSRV_ERROR_BAD_MAPPING, EINVAL)
+
+/* return -ve versions of POSIX errors as they are used in this form */
+int PVRSRVToNativeError(PVRSRV_ERROR eError)
+{
+	switch (eError)
+	{
+#define X(_PVRSRV_ERROR, _OS_ERROR) \
+	case (_PVRSRV_ERROR): return -(_OS_ERROR);
+
+		PVRSRV_ERROR_TO_OS_ERROR
+
+#undef X
+		default:
+			return -EFAULT;
+	}
+}
+
+typedef struct  _MISR_DATA_ {
+	struct workqueue_struct *psWorkQueue;
+	struct work_struct sMISRWork;
+	const IMG_CHAR* pszName;
+	PFN_MISR pfnMISR;
+	void *hData;
+} MISR_DATA;
+
+/*
+	MISRWrapper
+*/
+static void MISRWrapper(struct work_struct *data)
+{
+	MISR_DATA *psMISRData = container_of(data, MISR_DATA, sMISRWork);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Waking up '%s' MISR %p", psMISRData->pszName, psMISRData));
+
+	psMISRData->pfnMISR(psMISRData->hData);
+}
+
+/*
+	OSInstallMISR
+*/
+PVRSRV_ERROR OSInstallMISR(IMG_HANDLE *hMISRData, PFN_MISR pfnMISR,
+							void *hData, const IMG_CHAR *pszMisrName)
+{
+	MISR_DATA *psMISRData;
+
+	psMISRData = OSAllocMem(sizeof(*psMISRData));
+	PVR_LOG_RETURN_IF_NOMEM(psMISRData, "psMISRData");
+
+	psMISRData->hData = hData;
+	psMISRData->pfnMISR = pfnMISR;
+	psMISRData->pszName = pszMisrName;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Installing MISR with cookie %p", psMISRData));
+
+	psMISRData->psWorkQueue = create_singlethread_workqueue("pvr_misr");
+
+	if (psMISRData->psWorkQueue == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSInstallMISR: create_singlethreaded_workqueue failed"));
+		OSFreeMem(psMISRData);
+		return PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD;
+	}
+
+	INIT_WORK(&psMISRData->sMISRWork, MISRWrapper);
+
+	*hMISRData = (IMG_HANDLE) psMISRData;
+
+	return PVRSRV_OK;
+}
+
+/*
+	OSUninstallMISR
+*/
+PVRSRV_ERROR OSUninstallMISR(IMG_HANDLE hMISRData)
+{
+	MISR_DATA *psMISRData = (MISR_DATA *) hMISRData;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Uninstalling MISR with cookie %p", psMISRData));
+
+	destroy_workqueue(psMISRData->psWorkQueue);
+	OSFreeMem(psMISRData);
+
+	return PVRSRV_OK;
+}
+
+/*
+	OSScheduleMISR
+*/
+PVRSRV_ERROR OSScheduleMISR(IMG_HANDLE hMISRData)
+{
+	MISR_DATA *psMISRData = (MISR_DATA *) hMISRData;
+
+	/*
+		Note:
+
+		In the case of NO_HARDWARE we want the driver to be synchronous so
+		that we don't have to worry about waiting for previous operations
+		to complete
+	*/
+#if defined(NO_HARDWARE)
+	psMISRData->pfnMISR(psMISRData->hData);
+	return PVRSRV_OK;
+#else
+	{
+		bool rc = queue_work(psMISRData->psWorkQueue, &psMISRData->sMISRWork);
+		return rc ? PVRSRV_OK : PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+#endif
+}
+
+void OSSyncIRQ(IMG_UINT32 ui32IRQ)
+{
+	synchronize_irq(ui32IRQ);
+}
+
+/* OS specific values for thread priority */
+static const IMG_INT32 ai32OSPriorityValues[OS_THREAD_LAST_PRIORITY] =
+{
+	  0, /* OS_THREAD_NOSET_PRIORITY */
+	-20, /* OS_THREAD_HIGHEST_PRIORITY */
+	-10, /* OS_THREAD_HIGH_PRIORITY */
+	  0, /* OS_THREAD_NORMAL_PRIORITY */
+	  9, /* OS_THREAD_LOW_PRIORITY */
+	 19, /* OS_THREAD_LOWEST_PRIORITY */
+};
+
+static int OSThreadRun(void *data)
+{
+	OSThreadData *psOSThreadData = data;
+
+	/* count freezable threads */
+	LinuxBridgeNumActiveKernelThreadsIncrement();
+
+	/* Returns true if the thread was frozen, should we do anything with this
+	 * information? What do we return? Which one is the error case? */
+	set_freezable();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Starting Thread '%s'...", psOSThreadData->pszThreadName));
+
+	/* Call the client's kernel thread with the client's data pointer */
+	psOSThreadData->pfnThread(psOSThreadData->hData);
+
+	if (psOSThreadData->bIsSupportingThread)
+	{
+		_ThreadSetStopped(psOSThreadData);
+	}
+
+	/* Wait for OSThreadDestroy() to call kthread_stop() */
+	while (!kthread_freezable_should_stop(NULL))
+	{
+		schedule();
+	}
+
+	LinuxBridgeNumActiveKernelThreadsDecrement();
+
+	return 0;
+}
+
+PVRSRV_ERROR OSThreadCreate(IMG_HANDLE *phThread,
+                            IMG_CHAR *pszThreadName,
+                            PFN_THREAD pfnThread,
+                            PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                            IMG_BOOL bIsSupportingThread,
+                            void *hData)
+{
+	return OSThreadCreatePriority(phThread, pszThreadName, pfnThread,
+	                              pfnDebugDumpCB, bIsSupportingThread, hData,
+	                              OS_THREAD_NOSET_PRIORITY);
+}
+
+PVRSRV_ERROR OSThreadCreatePriority(IMG_HANDLE *phThread,
+                                    IMG_CHAR *pszThreadName,
+                                    PFN_THREAD pfnThread,
+                                    PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                                    IMG_BOOL bIsSupportingThread,
+                                    void *hData,
+                                    OS_THREAD_LEVEL eThreadPriority)
+{
+	OSThreadData *psOSThreadData;
+	PVRSRV_ERROR eError;
+
+	psOSThreadData = OSAllocZMem(sizeof(*psOSThreadData));
+	PVR_LOG_GOTO_IF_NOMEM(psOSThreadData, eError, fail_alloc);
+
+	psOSThreadData->pfnThread = pfnThread;
+	psOSThreadData->hData = hData;
+	psOSThreadData->pszThreadName = pszThreadName;
+	psOSThreadData->kthread = kthread_run(OSThreadRun, psOSThreadData, "%s", pszThreadName);
+
+	if (IS_ERR(psOSThreadData->kthread))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_kthread;
+	}
+
+	if (bIsSupportingThread)
+	{
+		psOSThreadData->pfnDebugDumpCB = pfnDebugDumpCB;
+		psOSThreadData->bIsThreadRunning = IMG_TRUE;
+		psOSThreadData->bIsSupportingThread = IMG_TRUE;
+
+		_ThreadListAddEntry(psOSThreadData);
+	}
+
+	if (eThreadPriority != OS_THREAD_NOSET_PRIORITY &&
+	    eThreadPriority < OS_THREAD_LAST_PRIORITY)
+	{
+		set_user_nice(psOSThreadData->kthread,
+		              ai32OSPriorityValues[eThreadPriority]);
+	}
+
+	*phThread = psOSThreadData;
+
+	return PVRSRV_OK;
+
+fail_kthread:
+	OSFreeMem(psOSThreadData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR OSThreadDestroy(IMG_HANDLE hThread)
+{
+	OSThreadData *psOSThreadData = hThread;
+	int ret;
+
+	/* Let the thread know we are ready for it to end and wait for it. */
+	ret = kthread_stop(psOSThreadData->kthread);
+	if (0 != ret)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "kthread_stop failed(%d)", ret));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	if (psOSThreadData->bIsSupportingThread)
+	{
+		_ThreadListRemoveEntry(psOSThreadData);
+	}
+
+	OSFreeMem(psOSThreadData);
+
+	return PVRSRV_OK;
+}
+
+void OSPanic(void)
+{
+	BUG();
+
+#if defined(__KLOCWORK__)
+	/* Klocwork does not understand that BUG is terminal... */
+	abort();
+#endif
+}
+
+IMG_BOOL OSIsMapPhysNonContigSupported(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) && !defined(CONFIG_VMAP_PFN)
+	return IMG_FALSE;
+#else
+	return IMG_TRUE;
+#endif
+}
+
+void OSUnMapPhysArrayToLin(void *pvLinAddr, void *pvPrivData)
+{
+	if (is_vmalloc_addr(pvLinAddr))
+	{
+#if defined(CONFIG_VMAP_PFN)
+		PVR_UNREFERENCED_PARAMETER(pvPrivData);
+		vunmap(pvLinAddr);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+		unmap_kernel_range((unsigned long) (uintptr_t) pvLinAddr,
+                           get_vm_area_size(pvPrivData));
+		free_vm_area(pvPrivData);
+#else
+		PVR_DPF((PVR_DBG_ERROR,"%s: Cannot map into kernel, no method supported.", __func__));
+		PVR_ASSERT(0);
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Given kernel address is not a vmalloc addr", __func__));
+	}
+}
+
+#define PagePAToPFN(PageAddr) (PageAddr >> PAGE_SHIFT)
+
+PVRSRV_ERROR OSMapPhysArrayToLin(IMG_CPU_PHYADDR pPagePA[],
+                                 IMG_UINT32 uiPagesCount,
+                                 void **ppvLinAddr,
+                                 void **ppvPrivData)
+{
+	if (ppvLinAddr == NULL || ppvPrivData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if defined(CONFIG_VMAP_PFN)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < uiPagesCount; i++)
+		{
+			pPagePA[i].uiAddr = PagePAToPFN(pPagePA[i].uiAddr);
+		}
+
+		*ppvLinAddr = vmap_pfn((unsigned long *)pPagePA,
+							   (unsigned int)uiPagesCount,
+							   pgprot_device(PAGE_KERNEL));
+		if (NULL == *ppvLinAddr)
+		{
+			return PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+		}
+		*ppvPrivData = NULL;
+		return PVRSRV_OK;
+	}
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	{
+		pte_t *pte[32], **pte_array;
+		struct vm_struct *psVMA;
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_UINT32 i = 0;
+
+		pte_array = &pte[0];
+		if (sizeof(pte) < (sizeof(pte[0]) * uiPagesCount))
+		{
+			pte_array = kzalloc(uiPagesCount * sizeof(*pte_array), GFP_KERNEL);
+			if (NULL == pte_array)
+			{
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+		}
+
+		psVMA = alloc_vm_area((size_t)(uiPagesCount << PAGE_SHIFT), pte_array);
+		if (NULL == psVMA)
+		{
+			eError = PVRSRV_ERROR_FAILED_TO_ALLOC_VIRT_MEMORY;
+			goto FreePTEArray;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Allocated VM: %s VMA: %p Addr: %p Size: 0x%lx count: %d", __func__,
+				psVMA, psVMA->addr, psVMA->size, psVMA->nr_pages));
+
+		for (i = 0; i < uiPagesCount; i++)
+		{
+			*(pte_array[i]) = pte_mkspecial(pfn_pte((unsigned long) PagePAToPFN(pPagePA[i].uiAddr),
+													pgprot_device(PAGE_KERNEL)));
+		}
+
+		OSWriteMemoryBarrier(psVMA->addr);
+
+		*ppvLinAddr = psVMA->addr;
+		*ppvPrivData = psVMA;
+
+FreePTEArray:
+		if (pte_array != pte)
+		{
+			kfree(pte_array);
+		}
+
+		return eError;
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR,"%s: Cannot map into kernel, no method supported.", __func__));
+	PVR_ASSERT(0);
+	*ppvLinAddr = NULL;
+	return PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+#endif
+}
+
+void *
+OSMapPhysToLin(IMG_CPU_PHYADDR BasePAddr,
+			   size_t ui32Bytes,
+			   PVRSRV_MEMALLOCFLAGS_T uiMappingFlags)
+{
+	void __iomem *pvLinAddr;
+
+	if (uiMappingFlags & ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Found non-cpu cache mode flag when mapping to the cpu"));
+		return NULL;
+	}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/*
+	  This is required to support DMA physheaps for GPU virtualization.
+	  Unfortunately, if a region of kernel managed memory is turned into
+	  a DMA buffer, conflicting mappings can come about easily on Linux
+	  as the original memory is mapped by the kernel as normal cached
+	  memory whilst DMA buffers are mapped mostly as uncached device or
+	  cache-coherent device memory. In both cases the system will have
+	  two conflicting mappings for the same memory region and will have
+	  "undefined behaviour" for most processors notably ARMv6 onwards
+	  and some x86 micro-architectures. As a result, perform ioremapping
+	  manually for DMA physheap allocations by translating from CPU/VA
+	  to BUS/PA thereby preventing the creation of conflicting mappings.
+	*/
+	pvLinAddr = (void __iomem *) SysDmaDevPAddrToCpuVAddr(BasePAddr.uiAddr, ui32Bytes);
+	if (pvLinAddr != NULL)
+	{
+		return (void __force *) pvLinAddr;
+	}
+#endif
+
+	switch (uiMappingFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+#if defined(CONFIG_X86) || defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+			pvLinAddr = (void __iomem *)ioremap_wc(BasePAddr.uiAddr, ui32Bytes);
+#else
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+#endif
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+#if defined(CONFIG_X86) || defined(CONFIG_ARM)
+			pvLinAddr = (void __iomem *)ioremap_cache(BasePAddr.uiAddr, ui32Bytes);
+#else
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+#endif
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT:
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT:
+			PVR_DPF((PVR_DBG_ERROR, "Unexpected cpu cache mode"));
+			pvLinAddr = NULL;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Unsupported cpu cache mode"));
+			pvLinAddr = NULL;
+			break;
+	}
+
+	return (void __force *) pvLinAddr;
+}
+
+
+IMG_BOOL
+OSUnMapPhysToLin(void *pvLinAddr, size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32Bytes);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (SysDmaCpuVAddrToDevPAddr(pvLinAddr))
+	{
+		return IMG_TRUE;
+	}
+#endif
+
+	iounmap((void __iomem *) pvLinAddr);
+
+	return IMG_TRUE;
+}
+
+#define OS_MAX_TIMERS	8
+
+/* Timer callback structure used by OSAddTimer */
+typedef struct TIMER_CALLBACK_DATA_TAG
+{
+	IMG_BOOL			bInUse;
+	PFN_TIMER_FUNC		pfnTimerFunc;
+	void				*pvData;
+	struct timer_list	sTimer;
+	IMG_UINT32			ui32Delay;
+	IMG_BOOL			bActive;
+	struct work_struct	sWork;
+}TIMER_CALLBACK_DATA;
+
+static struct workqueue_struct *psTimerWorkQueue = NULL;
+
+static TIMER_CALLBACK_DATA sTimers[OS_MAX_TIMERS];
+
+static DEFINE_MUTEX(sTimerStructLock);
+
+static void OSTimerCallbackBody(TIMER_CALLBACK_DATA *psTimerCBData)
+{
+	if (!psTimerCBData->bActive)
+		return;
+
+	/* call timer callback */
+	psTimerCBData->pfnTimerFunc(psTimerCBData->pvData);
+
+	/* reset timer */
+	mod_timer(&psTimerCBData->sTimer, psTimerCBData->sTimer.expires + psTimerCBData->ui32Delay);
+}
+
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+/*************************************************************************/ /*!
+@Function       OSTimerCallbackWrapper
+@Description    OS specific timer callback wrapper function
+@Input          psTimer    Timer list structure
+*/ /**************************************************************************/
+static void OSTimerCallbackWrapper(struct timer_list *psTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = from_timer(psTimerCBData, psTimer, sTimer);
+#else
+/*************************************************************************/ /*!
+@Function       OSTimerCallbackWrapper
+@Description    OS specific timer callback wrapper function
+@Input          uData    Timer callback data
+*/ /**************************************************************************/
+static void OSTimerCallbackWrapper(uintptr_t uData)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = (TIMER_CALLBACK_DATA*)uData;
+#endif
+	int res;
+
+	res = queue_work(psTimerWorkQueue, &psTimerCBData->sWork);
+	if (res == 0)
+	{
+		PVR_LOG(("OSTimerCallbackWrapper: work already queued"));
+	}
+}
+
+
+static void OSTimerWorkQueueCallBack(struct work_struct *psWork)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = container_of(psWork, TIMER_CALLBACK_DATA, sWork);
+
+	OSTimerCallbackBody(psTimerCBData);
+}
+
+IMG_HANDLE OSAddTimer(PFN_TIMER_FUNC pfnTimerFunc, void *pvData, IMG_UINT32 ui32MsTimeout)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData;
+	IMG_UINT32		ui32i;
+
+	/* check callback */
+	if (!pfnTimerFunc)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAddTimer: passed invalid callback"));
+		return NULL;
+	}
+
+	/* Allocate timer callback data structure */
+	mutex_lock(&sTimerStructLock);
+	for (ui32i = 0; ui32i < OS_MAX_TIMERS; ui32i++)
+	{
+		psTimerCBData = &sTimers[ui32i];
+		if (!psTimerCBData->bInUse)
+		{
+			psTimerCBData->bInUse = IMG_TRUE;
+			break;
+		}
+	}
+	mutex_unlock(&sTimerStructLock);
+	if (ui32i >= OS_MAX_TIMERS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAddTimer: all timers are in use"));
+		return NULL;
+	}
+
+	psTimerCBData->pfnTimerFunc = pfnTimerFunc;
+	psTimerCBData->pvData = pvData;
+	psTimerCBData->bActive = IMG_FALSE;
+
+	/*
+		HZ = ticks per second
+		ui32MsTimeout = required ms delay
+		ticks = (Hz * ui32MsTimeout) / 1000
+	*/
+	psTimerCBData->ui32Delay = ((HZ * ui32MsTimeout) < 1000)
+								?	1
+								:	((HZ * ui32MsTimeout) / 1000);
+
+	/* initialise object */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+	timer_setup(&psTimerCBData->sTimer, OSTimerCallbackWrapper, 0);
+#else
+	init_timer(&psTimerCBData->sTimer);
+
+	/* setup timer object */
+	psTimerCBData->sTimer.function = (void *)OSTimerCallbackWrapper;
+	psTimerCBData->sTimer.data = (uintptr_t)psTimerCBData;
+#endif
+
+	return (IMG_HANDLE)(uintptr_t)(ui32i + 1);
+}
+
+
+static inline TIMER_CALLBACK_DATA *GetTimerStructure(IMG_HANDLE hTimer)
+{
+	IMG_UINT32 ui32i = (IMG_UINT32)((uintptr_t)hTimer) - 1;
+
+	PVR_ASSERT(ui32i < OS_MAX_TIMERS);
+
+	return &sTimers[ui32i];
+}
+
+PVRSRV_ERROR OSRemoveTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(!psTimerCBData->bActive);
+
+	/* free timer callback data struct */
+	psTimerCBData->bInUse = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSEnableTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(!psTimerCBData->bActive);
+
+	/* Start timer arming */
+	psTimerCBData->bActive = IMG_TRUE;
+
+	/* set the expire time */
+	psTimerCBData->sTimer.expires = psTimerCBData->ui32Delay + jiffies;
+
+	/* Add the timer to the list */
+	add_timer(&psTimerCBData->sTimer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR OSDisableTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(psTimerCBData->bActive);
+
+	/* Stop timer from arming */
+	psTimerCBData->bActive = IMG_FALSE;
+	smp_mb();
+
+	flush_workqueue(psTimerWorkQueue);
+
+	/* remove timer */
+	del_timer_sync(&psTimerCBData->sTimer);
+
+	/*
+	 * This second flush is to catch the case where the timer ran
+	 * before we managed to delete it, in which case, it will have
+	 * queued more work for the workqueue. Since the bActive flag
+	 * has been cleared, this second flush won't result in the
+	 * timer being rearmed.
+	 */
+	flush_workqueue(psTimerWorkQueue);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR OSEventObjectCreate(const IMG_CHAR *pszName, IMG_HANDLE *hEventObject)
+{
+	PVR_UNREFERENCED_PARAMETER(pszName);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectListCreate(hEventObject);
+}
+
+
+PVRSRV_ERROR OSEventObjectDestroy(IMG_HANDLE hEventObject)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectListDestroy(hEventObject);
+}
+
+PVRSRV_ERROR OSEventObjectWaitTimeout(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus)
+{
+	PVRSRV_ERROR eError;
+
+	if (hOSEventKM && uiTimeoutus > 0)
+	{
+		eError = LinuxEventObjectWait(hOSEventKM, uiTimeoutus, _NON_FREEZABLE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSEventObjectWait: invalid arguments %p, %lld", hOSEventKM, uiTimeoutus));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR OSEventObjectWait(IMG_HANDLE hOSEventKM)
+{
+	return OSEventObjectWaitTimeout(hOSEventKM, EVENT_OBJECT_TIMEOUT_US);
+}
+
+PVRSRV_ERROR OSEventObjectWaitKernel(IMG_HANDLE hOSEventKM,
+                                     IMG_UINT64 uiTimeoutus)
+{
+	PVRSRV_ERROR eError;
+
+	if (hOSEventKM != NULL && uiTimeoutus > 0)
+	{
+		eError = LinuxEventObjectWait(hOSEventKM, uiTimeoutus, _FREEZABLE);
+	}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	else if (hOSEventKM != NULL)
+	{
+		eError = LinuxEventObjectWaitUntilSignalled(hOSEventKM);
+	}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSEventObjectWaitKernel: invalid arguments %p",
+		        hOSEventKM));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+
+void OSEventObjectDumpDebugInfo(IMG_HANDLE hOSEventKM)
+{
+	LinuxEventObjectDumpDebugInfo(hOSEventKM);
+}
+
+PVRSRV_ERROR OSEventObjectOpen(IMG_HANDLE hEventObject, IMG_HANDLE *phOSEvent)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phOSEvent, "phOSEvent");
+	PVR_LOG_GOTO_IF_INVALID_PARAM(hEventObject, eError, error);
+
+	eError = LinuxEventObjectAdd(hEventObject, phOSEvent);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LinuxEventObjectAdd", error);
+
+	return PVRSRV_OK;
+
+error:
+	*phOSEvent = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR OSEventObjectClose(IMG_HANDLE hOSEventKM)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hOSEventKM, "hOSEventKM");
+
+	return LinuxEventObjectDelete(hOSEventKM);
+}
+
+PVRSRV_ERROR OSEventObjectSignal(IMG_HANDLE hEventObject)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectSignal(hEventObject);
+}
+
+PVRSRV_ERROR OSCopyToUser(void *pvProcess,
+						  void __user *pvDest,
+						  const void *pvSrc,
+						  size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(pvProcess);
+
+	if (pvr_copy_to_user(pvDest, pvSrc, ui32Bytes)==0)
+		return PVRSRV_OK;
+	else
+		return PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY;
+}
+
+PVRSRV_ERROR OSCopyFromUser(void *pvProcess,
+							void *pvDest,
+							const void __user *pvSrc,
+							size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(pvProcess);
+
+	if (likely(pvr_copy_from_user(pvDest, pvSrc, ui32Bytes)==0))
+		return PVRSRV_OK;
+	else
+		return PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY;
+}
+
+IMG_UINT64 OSDivide64r64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder)
+{
+	*pui32Remainder = do_div(ui64Divident, ui32Divisor);
+
+	return ui64Divident;
+}
+
+IMG_UINT32 OSDivide64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder)
+{
+	*pui32Remainder = do_div(ui64Divident, ui32Divisor);
+
+	return (IMG_UINT32) ui64Divident;
+}
+
+/* One time osfunc initialisation */
+PVRSRV_ERROR PVROSFuncInit(void)
+{
+	{
+		PVR_ASSERT(!psTimerWorkQueue);
+
+		psTimerWorkQueue = create_freezable_workqueue("pvr_timer");
+		if (psTimerWorkQueue == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: couldn't create timer workqueue",
+					 __func__));
+			return PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD;
+		}
+	}
+
+	{
+		IMG_UINT32 ui32i;
+
+		for (ui32i = 0; ui32i < OS_MAX_TIMERS; ui32i++)
+		{
+			TIMER_CALLBACK_DATA *psTimerCBData = &sTimers[ui32i];
+
+			INIT_WORK(&psTimerCBData->sWork, OSTimerWorkQueueCallBack);
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*
+ * Osfunc deinitialisation.
+ * Note that PVROSFuncInit may not have been called
+ */
+void PVROSFuncDeInit(void)
+{
+	if (psTimerWorkQueue != NULL)
+	{
+		destroy_workqueue(psTimerWorkQueue);
+		psTimerWorkQueue = NULL;
+	}
+}
+
+void OSDumpStack(void)
+{
+	dump_stack();
+}
+
+/*************************************************************************/ /*!
+@Function       OSDebugSignalPID
+@Description    Sends a SIGTRAP signal to a specific PID in user mode for
+                debugging purposes. The user mode process can register a handler
+                against this signal.
+                This is necessary to support the Rogue debugger. If the Rogue
+                debugger is not used then this function may be implemented as
+                a stub.
+@Input          ui32PID    The PID for the signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDebugSignalPID(IMG_UINT32 ui32PID)
+{
+	int err;
+	struct pid *psPID;
+
+	psPID = find_vpid(ui32PID);
+	if (psPID == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get PID struct.", __func__));
+		return PVRSRV_ERROR_NOT_FOUND;
+	}
+
+	err = kill_pid(psPID, SIGTRAP, 0);
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Signal Failure %d", __func__, err));
+		return PVRSRV_ERROR_SIGNAL_FAILED;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSIsKernelThread
+@Description    This API determines if the current running thread is a kernel
+                thread (i.e. one not associated with any userland process,
+                typically an MISR handler.)
+@Return         IMG_TRUE if it is a kernel thread, otherwise IMG_FALSE.
+*/ /**************************************************************************/
+IMG_BOOL OSIsKernelThread(void)
+{
+	/*
+	 * Kernel threads have a NULL memory descriptor.
+	 *
+	 * See https://www.kernel.org/doc/Documentation/vm/active_mm.txt
+	 */
+	return current->mm == NULL;
+}
+
+void OSDumpVersionInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					   void *pvDumpDebugFile)
+{
+	PVR_DUMPDEBUG_LOG("OS kernel info: %s %s %s %s",
+					utsname()->sysname,
+					utsname()->release,
+					utsname()->version,
+					utsname()->machine);
+}
+#if defined(SUPPORT_DMA_TRANSFER)
+
+typedef struct _OS_CLEANUP_DATA_
+{
+	IMG_BOOL bSucceed;
+	IMG_BOOL bAdvanceTimeline;
+	IMG_UINT uiRefCount;
+	IMG_UINT uiNumDMA;
+	IMG_UINT uiCount;
+
+	struct dma_async_tx_descriptor** ppsDescriptors;
+
+
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PFN_SERVER_CLEANUP pfnServerCleanup;
+	void* pvServerCleanupData;
+
+	enum dma_data_direction eDirection;
+	struct sg_table **ppsSg;
+	struct page ***pages;
+	IMG_UINT32* puiNumPages;
+	spinlock_t spinlock;
+
+	struct completion start_cleanup;
+	struct completion *sync_completion;
+
+	/* Sparse PMR transfer information */
+	IMG_BOOL *pbIsSparse;
+	IMG_UINT *uiNumValidPages;
+	struct sg_table ***ppsSgSparse;
+	struct dma_async_tx_descriptor*** ppsDescriptorsSparse;
+
+} OS_CLEANUP_DATA;
+
+static int cleanup_thread(void *pvData)
+{
+	IMG_UINT32 i, j;
+	struct completion *sync_completion = NULL;
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvData;
+	IMG_BOOL bSucceed = psOSCleanup->bSucceed;
+
+	sync_completion = psOSCleanup->sync_completion;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Cleanup thread waiting (%p) on completion", pvData));
+#endif
+
+	wait_for_completion(&psOSCleanup->start_cleanup);
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Cleanup thread notified (%p)", pvData));
+#endif
+	/* Free resources */
+	for (i=0; i<psOSCleanup->uiCount; i++)
+	{
+		if (!psOSCleanup->pbIsSparse[i])
+		{
+			dma_sync_sg_for_cpu(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+								psOSCleanup->ppsSg[i]->sgl,
+								psOSCleanup->ppsSg[i]->nents,
+								psOSCleanup->eDirection);
+
+			dma_unmap_sg(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+						 psOSCleanup->ppsSg[i]->sgl,
+						 psOSCleanup->ppsSg[i]->nents,
+						 psOSCleanup->eDirection);
+
+			sg_free_table(psOSCleanup->ppsSg[i]);
+
+			OSFreeMem(psOSCleanup->ppsSg[i]);
+
+			/* Unpin pages */
+			for (j=0; j<psOSCleanup->puiNumPages[i]; j++)
+			{
+				/*
+				 * using DMA_FROM_DEVICE from enum dma_data_direction instead of DMA_DEV_TO_MEM
+				 * from enum dma_transfer_direction to avoid casting explicitly
+				 */
+				if (psOSCleanup->eDirection == DMA_FROM_DEVICE)
+				{
+					set_page_dirty_lock(psOSCleanup->pages[i][j]);
+				}
+
+				pvr_unpin_user_page_for_dma(psOSCleanup->pages[i][j]);
+			}
+		}
+		else
+		{
+			for (j = 0; j < psOSCleanup->puiNumPages[i]; j++)
+			{
+				if (psOSCleanup->ppsSgSparse[i][j]) {
+					dma_sync_sg_for_cpu(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+										psOSCleanup->ppsSgSparse[i][j]->sgl,
+										psOSCleanup->ppsSgSparse[i][j]->nents,
+										psOSCleanup->eDirection);
+
+
+					dma_unmap_sg(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+								psOSCleanup->ppsSgSparse[i][j]->sgl,
+								psOSCleanup->ppsSgSparse[i][j]->nents,
+								psOSCleanup->eDirection);
+
+					sg_free_table(psOSCleanup->ppsSgSparse[i][j]);
+
+					OSFreeMem(psOSCleanup->ppsSgSparse[i][j]);
+
+				}
+			}
+
+			OSFreeMem(psOSCleanup->ppsSgSparse[i]);
+			OSFreeMem(psOSCleanup->ppsDescriptorsSparse[i]);
+
+			/* Unpin pages */
+			for (j=0; j<psOSCleanup->puiNumPages[i]*2; j++)
+			{
+				/*
+				 * Some pages might've been pinned twice
+				 * Others may have not been pinned at all
+				 */
+				if (psOSCleanup->pages[i][j])
+				{
+					/*
+					 * using DMA_FROM_DEVICE from enum dma_data_direction instead of DMA_DEV_TO_MEM
+					 * from enum dma_transfer_direction to avoid casting explicitly
+					 */
+					if (psOSCleanup->eDirection == DMA_FROM_DEVICE)
+					{
+						set_page_dirty_lock(psOSCleanup->pages[i][j]);
+					}
+
+					pvr_unpin_user_page_for_dma(psOSCleanup->pages[i][j]);
+				}
+			}
+		}
+
+		OSFreeMem(psOSCleanup->pages[i]);
+	}
+
+	psOSCleanup->pfnServerCleanup(psOSCleanup->pvServerCleanupData,
+								  psOSCleanup->bAdvanceTimeline);
+
+	OSFreeMem(psOSCleanup->ppsSg);
+	OSFreeMem(psOSCleanup->pages);
+	OSFreeMem(psOSCleanup->puiNumPages);
+	OSFreeMem(psOSCleanup->ppsSgSparse);
+	OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+	OSFreeMem(psOSCleanup->ppsDescriptors);
+	OSFreeMem(psOSCleanup->pbIsSparse);
+	OSFreeMem(psOSCleanup->uiNumValidPages);
+	OSFreeMem(psOSCleanup);
+
+	if (bSucceed)
+	{
+		kthread_complete_and_exit(sync_completion, 0);
+	}
+	else
+	{
+		kthread_complete_and_exit(NULL, 0);
+	}
+	return 0;
+}
+
+static void dma_callback(void *pvOSCleanup)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvOSCleanup;
+	unsigned long flags;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "dma_callback (%p) refcount decreased to %d", psOSCleanup, psOSCleanup->uiRefCount - 1));
+#endif
+	spin_lock_irqsave(&psOSCleanup->spinlock, flags);
+
+	psOSCleanup->uiRefCount--;
+
+	if (psOSCleanup->uiRefCount==0)
+	{
+		/* Notify the cleanup thread */
+		spin_unlock_irqrestore(&psOSCleanup->spinlock, flags);
+		complete(&psOSCleanup->start_cleanup);
+		return;
+	}
+
+	spin_unlock_irqrestore(&psOSCleanup->spinlock, flags);
+}
+
+#if defined(PVRSRV_DEBUG_DMA)
+static void
+DMADumpPhysicalAddresses(struct page **ppsHostMemPages,
+						 IMG_UINT32 uiNumPages,
+						 IMG_DMA_ADDR *sDmaAddr,
+						 IMG_UINT64 ui64Offset)
+{
+	IMG_CPU_PHYADDR sPagePhysAddr;
+	IMG_UINT32 uiIdx;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "DMA Transfer Address Dump"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Hostmem phys addresses:"));
+
+	for (uiIdx = 0; uiIdx < uiNumPages; uiIdx++)
+	{
+		sPagePhysAddr.uiAddr = page_to_phys(ppsHostMemPages[uiIdx]);
+		if (uiIdx == 0)
+		{
+			sPagePhysAddr.uiAddr += ui64Offset;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "\tHost mem start at 0x%" IMG_UINT64_FMTSPECx,
+					 (IMG_UINT64)sPagePhysAddr.uiAddr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "\tHost Mem Page %d at 0x%" IMG_UINT64_FMTSPECx,
+					 uiIdx, (IMG_UINT64)sPagePhysAddr.uiAddr));
+		}
+	}
+	PVR_DPF((PVR_DBG_MESSAGE,
+			 "Devmem CPU phys address: 0x%" IMG_UINT64_FMTSPECx,
+			 (IMG_UINT64)sDmaAddr->uiAddr));
+}
+#endif
+
+PVRSRV_ERROR OSDmaSubmitTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *pvOSData,
+				    void *pvChan, IMG_BOOL bSynchronous)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvOSData;
+	struct completion* sync_completion = NULL;
+
+	psOSCleanup->bSucceed = IMG_TRUE;
+	psOSCleanup->bAdvanceTimeline = IMG_TRUE;
+
+	if (bSynchronous)
+	{
+		sync_completion = OSAllocZMem(sizeof(struct completion));
+		init_completion(sync_completion);
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	/* Wait only on number of ops scheduled. This might be different to NumDMAs
+	in certain error conditions */
+	psOSCleanup->uiRefCount = psOSCleanup->uiCount;
+	psOSCleanup->sync_completion = sync_completion;
+
+	{
+		IMG_UINT32 i,j;
+		for (i=0; i<psOSCleanup->uiCount; i++)
+		{
+			if (psOSCleanup->pbIsSparse[i])
+			{
+				for (j=0; j<psOSCleanup->puiNumPages[i]; j++)
+				{
+					if (psOSCleanup->ppsDescriptorsSparse[i][j])
+						dmaengine_submit(psOSCleanup->ppsDescriptorsSparse[i][j]);
+				}
+			}
+			else
+			{
+				dmaengine_submit(psOSCleanup->ppsDescriptors[i]);
+			}
+		}
+	}
+
+	dma_async_issue_pending(pvChan);
+
+	if (bSynchronous)
+	{
+		wait_for_completion(sync_completion);
+		OSFreeMem(sync_completion);
+	}
+
+	return PVRSRV_OK;
+}
+
+void OSDmaForceCleanup(PVRSRV_DEVICE_NODE *psDevNode, void *pvChan,
+					   void *pvOSData, void *pvServerCleanupParam,
+					   PFN_SERVER_CLEANUP pfnServerCleanup)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA *)pvOSData;
+	IMG_UINT ui32Retries;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	psOSCleanup->bSucceed = IMG_FALSE;
+	psOSCleanup->bAdvanceTimeline = IMG_TRUE;
+
+	/* Need to wait for outstanding DMA Engine ops before advancing the
+	   user-supplied timeline in case of error. dmaengine_terminate_sync
+	   cannot be called from within atomic context, so cannot invoke it
+	   from inside the cleanup kernel thread. */
+	for (ui32Retries = 0; ui32Retries < DMA_ERROR_SYNC_RETRIES; ui32Retries++)
+	{
+		if (dmaengine_terminate_sync(pvChan) == 0)
+		{
+			break;
+		}
+	}
+	if (ui32Retries == DMA_ERROR_SYNC_RETRIES)
+	{
+		/* We cannot guarantee all outstanding DMAs were terminated
+		 * so we let the UM fence time out as a fallback mechanism */
+		psOSCleanup->bAdvanceTimeline = IMG_FALSE;
+	}
+
+	if (psOSCleanup->uiCount > 0)
+	{
+		complete(&psOSCleanup->start_cleanup);
+	}
+	else
+	{
+		/* Cleanup kthread never run, need to manually wind down */
+		pfnServerCleanup(pvServerCleanupParam, psOSCleanup->bAdvanceTimeline);
+
+		OSFreeMem(psOSCleanup->ppsSg);
+		OSFreeMem(psOSCleanup->pages);
+		OSFreeMem(psOSCleanup->puiNumPages);
+		OSFreeMem(psOSCleanup->ppsSgSparse);
+		OSFreeMem(psOSCleanup->pbIsSparse);
+		OSFreeMem(psOSCleanup->uiNumValidPages);
+		OSFreeMem(psOSCleanup->ppsDescriptors);
+		OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+
+		OSFreeMem(psOSCleanup);
+	}
+}
+
+PVRSRV_ERROR OSDmaAllocData(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 uiNumDMA, void **pvOutData)
+{
+	PVRSRV_ERROR eError;
+	OS_CLEANUP_DATA *psOSCleanup = OSAllocZMem(sizeof(OS_CLEANUP_DATA));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup, eError, e0);
+
+	psOSCleanup->uiNumDMA = uiNumDMA;
+	psOSCleanup->psDevNode = psDevNode;
+
+	spin_lock_init(&psOSCleanup->spinlock);
+
+	init_completion(&psOSCleanup->start_cleanup);
+
+	psOSCleanup->ppsDescriptors = OSAllocZMem(uiNumDMA * sizeof(struct dma_async_tx_descriptor*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsDescriptors, eError, e0);
+
+	psOSCleanup->ppsDescriptorsSparse = OSAllocZMem(uiNumDMA * sizeof(struct dma_async_tx_descriptor*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsDescriptorsSparse, eError, e11);
+
+	psOSCleanup->ppsSg = OSAllocZMem(uiNumDMA * sizeof(struct sg_table*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsSg, eError, e1);
+
+	psOSCleanup->ppsSgSparse = OSAllocZMem(uiNumDMA * sizeof(struct sg_table*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsSgSparse, eError, e12);
+
+	psOSCleanup->pbIsSparse = OSAllocZMem(uiNumDMA * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->pbIsSparse, eError, e13);
+
+	psOSCleanup->uiNumValidPages = OSAllocZMem(uiNumDMA * sizeof(IMG_UINT));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->uiNumValidPages, eError, e14);
+
+	psOSCleanup->pages = OSAllocZMem(uiNumDMA * sizeof(struct page **));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->pages, eError, e2);
+
+	psOSCleanup->puiNumPages = OSAllocZMem(uiNumDMA * sizeof(IMG_UINT32));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->puiNumPages, eError, e3);
+
+	*pvOutData = psOSCleanup;
+
+	return PVRSRV_OK;
+
+e3:
+	OSFreeMem(psOSCleanup->pages);
+e2:
+	OSFreeMem(psOSCleanup->uiNumValidPages);
+e14:
+	OSFreeMem(psOSCleanup->pbIsSparse);
+e13:
+	OSFreeMem(psOSCleanup->ppsSgSparse);
+e12:
+	OSFreeMem(psOSCleanup->ppsSg);
+e1:
+	OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+e11:
+	OSFreeMem(psOSCleanup->ppsDescriptors);
+e0:
+	OSFreeMem(psOSCleanup);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       OSDmaTransfer
+@Description    This API is used to ask OS to perform a DMA transfer operation
+@Return
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDmaPrepareTransfer(PVRSRV_DEVICE_NODE *psDevNode,
+							   void* pvChan,
+							   IMG_DMA_ADDR* psDmaAddr, IMG_UINT64* puiAddress,
+							   IMG_UINT64 uiSize, IMG_BOOL bMemToDev,
+							   void* pvOSData,
+							   void* pvServerCleanupParam, PFN_SERVER_CLEANUP pfnServerCleanup, IMG_BOOL bFirst)
+{
+
+	IMG_INT iRet;
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+	OS_CLEANUP_DATA* psOSCleanupData = pvOSData;
+
+	enum dma_data_direction eDataDirection = bMemToDev ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct dma_slave_config sConfig = {0};
+	struct dma_async_tx_descriptor *psDesc;
+
+	unsigned long offset = (unsigned long)puiAddress & ((1 << PAGE_SHIFT) - 1);
+	unsigned int num_pages = (uiSize + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	int num_pinned_pages = 0;
+
+	struct sg_table *psSg = OSAllocZMem(sizeof(struct sg_table));
+	PVR_LOG_GOTO_IF_NOMEM(psSg, eError, e0);
+
+	psOSCleanupData->pages[psOSCleanupData->uiCount] = OSAllocZMem(num_pages * sizeof(struct page *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->pages[psOSCleanupData->uiCount], eError, e1);
+
+	num_pinned_pages = pvr_pin_user_pages_for_dma(
+		puiAddress,
+		num_pages,
+		!bMemToDev,
+		psOSCleanupData->pages[psOSCleanupData->uiCount]);
+
+	psOSCleanupData->puiNumPages[psOSCleanupData->uiCount] = num_pinned_pages;
+	if (num_pinned_pages != num_pages)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "get_user_pages_fast failed: (%d - %u)", num_pinned_pages, num_pages));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e2; /* Unpin what was pinned and return error */
+	}
+
+#if defined(PVRSRV_DEBUG_DMA)
+	DMADumpPhysicalAddresses(psOSCleanupData->pages[psOSCleanupData->uiCount],
+							 num_pages, psDmaAddr, offset);
+#endif
+
+	if (sg_alloc_table_from_pages(psSg, psOSCleanupData->pages[psOSCleanupData->uiCount], num_pages, offset, uiSize, GFP_KERNEL) != 0)
+	{
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table_from_pages failed"));
+		goto e2;
+	}
+
+	if (bMemToDev)
+	{
+		sConfig.direction = DMA_MEM_TO_DEV;
+		sConfig.src_addr = 0;
+		sConfig.dst_addr = psDmaAddr->uiAddr;
+	}
+	else
+	{
+		sConfig.direction = DMA_DEV_TO_MEM;
+		sConfig.src_addr = psDmaAddr->uiAddr;
+		sConfig.dst_addr = 0;
+	}
+	dmaengine_slave_config(pvChan, &sConfig);
+
+	iRet = dma_map_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+	if (!iRet)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error mapping SG list", __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e3;
+	}
+
+	dma_sync_sg_for_device(psDevConfig->pvOSDevice, psSg->sgl,(unsigned int)iRet, eDataDirection);
+
+	psDesc = dmaengine_prep_slave_sg(pvChan, psSg->sgl, (unsigned int)iRet, sConfig.direction, 0);
+	if (!psDesc)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: dmaengine_prep_slave_sg failed", __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e4;
+	}
+
+	psOSCleanupData->eDirection = eDataDirection;
+	psOSCleanupData->ppsSg[psOSCleanupData->uiCount] = psSg;
+	psOSCleanupData->pfnServerCleanup = pfnServerCleanup;
+	psOSCleanupData->pvServerCleanupData = pvServerCleanupParam;
+
+	psDesc->callback_param = psOSCleanupData;
+	psDesc->callback = dma_callback;
+
+	if (bFirst)
+		kthread_run(cleanup_thread, psOSCleanupData, "dma-cleanup-thread");
+	psOSCleanupData->ppsDescriptors[psOSCleanupData->uiCount] = psDesc;
+
+	psOSCleanupData->uiCount++;
+
+	return PVRSRV_OK;
+
+e4:
+	dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+e3:
+	sg_free_table(psSg);
+e2:
+	{
+		IMG_UINT32 i;
+		/* Unpin pages */
+		for (i=0; i<psOSCleanupData->puiNumPages[psOSCleanupData->uiCount]; i++)
+		{
+			pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][i]);
+		}
+	}
+	OSFreeMem(psOSCleanupData->pages[psOSCleanupData->uiCount]);
+e1:
+	OSFreeMem(psSg);
+e0:
+	return eError;
+}
+
+static IMG_UINT32
+CalculateValidPages(IMG_BOOL *pbValid,
+					IMG_UINT32 ui32SizeInPages)
+{
+	IMG_UINT32 ui32nValid;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0, ui32nValid = 0; ui32Idx < ui32SizeInPages; ui32Idx++)
+	{
+		ui32nValid += pbValid[ui32Idx] ? 1 : 0;
+	}
+
+	return ui32nValid;
+}
+
+PVRSRV_ERROR OSDmaPrepareTransferSparse(PVRSRV_DEVICE_NODE *psDevNode,
+										void* pvChan,
+										IMG_DMA_ADDR* psDmaAddr,
+										IMG_BOOL *pbValid,
+										IMG_UINT64* puiAddress,
+										IMG_UINT64 uiSize,
+										IMG_UINT32 uiOffsetInFirstPMRPage,
+										IMG_UINT32 ui32SizeInPages,
+										IMG_BOOL bMemToDev,
+										void* pvOSData,
+										void* pvServerCleanupParam,
+										PFN_SERVER_CLEANUP pfnServerCleanup,
+										IMG_BOOL bFirst)
+{
+
+	IMG_INT iRet;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+	OS_CLEANUP_DATA* psOSCleanupData = pvOSData;
+	IMG_UINT32 ui32PageSize = OSGetPageSize();
+	void *pvNextAddress = puiAddress;
+	IMG_UINT32 ui32Idx;
+	IMG_INT32 i32Rwd;
+
+	enum dma_data_direction eDataDirection = bMemToDev ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct dma_slave_config sConfig = {0};
+	struct dma_async_tx_descriptor *psDesc;
+
+	unsigned long offset = (unsigned long)puiAddress & ((1 << PAGE_SHIFT) - 1);
+	unsigned int num_pages = (uiSize + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	unsigned int num_valid_pages = CalculateValidPages(pbValid, ui32SizeInPages);
+	unsigned int num_pinned_pages = 0;
+	unsigned int valid_idx;
+	size_t transfer_size;
+	struct page ** next_pages;
+	struct sg_table *psSg;
+
+	psOSCleanupData->uiNumValidPages[psOSCleanupData->uiCount] = num_valid_pages;
+	psOSCleanupData->pbIsSparse[psOSCleanupData->uiCount] = IMG_TRUE;
+
+	/*
+	 * If an SG transfer from virtual memory to card memory goes over a page boundary in
+	 * main memory, it'll span two different pages - therefore, total number of pages to
+	 * keep track of should be twice as many as for a simple transfer. This twice-as-big
+	 * allocation is also necessary because the same virtual memory page might be present
+	 * in more than one SG DMA transfer, because of differences in first-page offset between
+	 * the sparse device PMR and the virtual memory buffer.
+	 */
+	psOSCleanupData->pages[psOSCleanupData->uiCount] = OSAllocZMem(2*num_valid_pages * sizeof(struct page *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->pages[psOSCleanupData->uiCount], eError, e0);
+
+	psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount] = OSAllocZMem(num_valid_pages * sizeof(struct sg_table *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount], eError, e1);
+
+	psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount] = OSAllocZMem(num_valid_pages * sizeof(struct dma_async_tx_descriptor *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount], eError, e11);
+
+	for (ui32Idx = 0, valid_idx = 0; ui32Idx < ui32SizeInPages; ui32Idx++)
+	{
+		if (valid_idx == num_valid_pages)
+		{
+			break;
+		}
+		if (!pbValid[ui32Idx])
+		{
+			pvNextAddress += (ui32Idx == 0) ? ui32PageSize - uiOffsetInFirstPMRPage : ui32PageSize;
+			continue;
+		}
+
+		/* Pick transfer size */
+		if (ui32Idx == 0)
+		{
+			if (uiOffsetInFirstPMRPage + uiSize <= ui32PageSize)
+			{
+				PVR_ASSERT(num_valid_pages == 1);
+				transfer_size = uiSize;
+			}
+			else
+			{
+				transfer_size = ui32PageSize - uiOffsetInFirstPMRPage;
+			}
+		}
+		else
+		{
+			/* Last valid LMA page */
+			if (valid_idx == num_valid_pages - 1)
+			{
+				transfer_size = ((uiOffsetInFirstPMRPage + uiSize - 1) % ui32PageSize) + 1;
+			}
+			else
+			{
+				transfer_size = ui32PageSize;
+			}
+		}
+
+		if (((unsigned long long)pvNextAddress & (ui32PageSize - 1)) + transfer_size > ui32PageSize)
+		{
+			num_pages = 2;
+		}
+		else
+		{
+			num_pages = 1;
+		}
+
+		next_pages = psOSCleanupData->pages[psOSCleanupData->uiCount] + (valid_idx * 2);
+
+		num_pinned_pages = pvr_pin_user_pages_for_dma(
+			pvNextAddress,
+			num_pages,
+			!bMemToDev,
+			next_pages);
+
+		if (num_pinned_pages != num_pages)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "get_user_pages_fast for sparse failed: (%d - %u)", num_pinned_pages, num_pages));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e2;
+		}
+
+#if defined(PVRSRV_DEBUG_DMA)
+		DMADumpPhysicalAddresses(next_pages, num_pages,
+								 &psDmaAddr[ui32Idx],
+								 (unsigned long)pvNextAddress & (ui32PageSize - 1));
+#endif
+
+		psSg = OSAllocZMem(sizeof(struct sg_table));
+		PVR_LOG_GOTO_IF_NOMEM(psSg, eError, e3);
+
+		if (sg_alloc_table_from_pages(psSg, next_pages, num_pages,
+									  (unsigned long)pvNextAddress & (ui32PageSize - 1),
+									  transfer_size,
+									  GFP_KERNEL) != 0)
+		{
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table_from_pages failed"));
+			goto e4;
+		}
+
+		pvNextAddress += transfer_size;
+
+		if (bMemToDev)
+		{
+			sConfig.direction = DMA_MEM_TO_DEV;
+			sConfig.src_addr = 0;
+			sConfig.dst_addr = psDmaAddr[ui32Idx].uiAddr;
+		}
+		else
+		{
+			sConfig.direction = DMA_DEV_TO_MEM;
+			sConfig.src_addr = psDmaAddr[ui32Idx].uiAddr;
+			sConfig.dst_addr = 0;
+		}
+		dmaengine_slave_config(pvChan, &sConfig);
+
+		iRet = dma_map_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+		if (!iRet)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Error mapping SG list", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e5;
+		}
+		dma_sync_sg_for_device(psDevConfig->pvOSDevice, psSg->sgl,(unsigned int)iRet, eDataDirection);
+
+		psDesc = dmaengine_prep_slave_sg(pvChan, psSg->sgl, (unsigned int)iRet, sConfig.direction, 0);
+		if (!psDesc)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: dmaengine_prep_slave_sg failed", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto e6;
+		}
+
+		psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount][valid_idx] = psSg;
+		psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount][valid_idx] = psDesc;
+		psOSCleanupData->puiNumPages[psOSCleanupData->uiCount] = ++valid_idx;
+
+		if (valid_idx == num_valid_pages)
+		{
+			psDesc->callback_param = psOSCleanupData;
+			psDesc->callback = dma_callback;
+
+			if (bFirst)
+			{
+				psOSCleanupData->eDirection = eDataDirection;
+				psOSCleanupData->pfnServerCleanup = pfnServerCleanup;
+				psOSCleanupData->pvServerCleanupData = pvServerCleanupParam;
+
+				kthread_run(cleanup_thread, psOSCleanupData, "dma-cleanup-thread");
+			}
+
+			psOSCleanupData->uiCount++;
+		}
+
+	}
+
+	return PVRSRV_OK;
+
+e6:
+	dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+e5:
+	sg_free_table(psSg);
+e4:
+	OSFreeMem(psSg);
+e3:
+	/* Unpin last */
+	pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx]);
+
+	if (psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx+1])
+	{
+		pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx+1]);
+	}
+e2:
+	/* rewind */
+	for (i32Rwd=valid_idx-1; i32Rwd >= 0; i32Rwd--)
+	{
+		IMG_UINT32 i;
+
+		psSg = psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount][i32Rwd];
+		dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+		sg_free_table(psSg);
+
+		/* Unpin pages */
+		for (i=0; i < psOSCleanupData->puiNumPages[psOSCleanupData->uiCount]*2; i++)
+		{
+			if (psOSCleanupData->pages[psOSCleanupData->uiCount][i])
+			{
+				pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][i]);
+			}
+		}
+	}
+	OSFreeMem(psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount]);
+e11:
+	OSFreeMem(psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount]);
+e1:
+	OSFreeMem(psOSCleanupData->pages[psOSCleanupData->uiCount]);
+e0:
+	return eError;
+}
+
+#endif /* SUPPORT_DMA_TRANSFER */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+IMG_INTERNAL PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR)
+{
+	struct dma_heap *heap;
+	struct dma_buf *buf;
+	struct device *dev;
+	struct dma_buf_attachment *buf_attachment;
+
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_ERROR eError;
+	IMG_CHAR *pszHeapName;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode->psDevConfig->pszSecureDMAHeapName, "pszSecureDMAHeapName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM((OSStringLength(psDeviceNode->psDevConfig->pszSecureDMAHeapName) > 0), "pszSecureDMAHeapName length");
+
+	pszHeapName = psDeviceNode->psDevConfig->pszSecureDMAHeapName;
+	dev = (struct device*)psDeviceNode->psDevConfig->pvOSDevice;
+
+	heap = dma_heap_find(pszHeapName);
+	PVR_LOG_GOTO_IF_NOMEM(heap, eError, ErrorExit);
+
+	buf = dma_heap_buffer_alloc(heap, uiSize, 0, 0);
+	if (IS_ERR_OR_NULL(buf))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("dma_heap_buffer_alloc", eError,
+		                        PVRSRV_ERROR_OUT_OF_MEMORY, ErrorHeapPut);
+	}
+
+	if (buf->size < uiSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: buffer size (%ld) is less than requested (%lld).",
+				 __func__, buf->size, uiSize));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorBufFree;
+	}
+
+	buf_attachment = dma_buf_attach(buf, dev);
+	if (IS_ERR_OR_NULL(buf_attachment))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("dma_buf_attach", eError,
+		                        PVRSRV_ERROR_OUT_OF_MEMORY, ErrorBufFree);
+	}
+
+	eError = PhysmemCreateNewDmaBufBackedPMR(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_EXTERNAL],
+											 buf_attachment,
+											 NULL,
+											 PVRSRV_MEMALLOCFLAG_GPU_READABLE
+											 | PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE,
+											 OSGetCurrentClientProcessIDKM(),
+											 buf->size,
+											 1,
+											 1,
+											 &ui32MappingTable,
+											 OSStringLength(pszName),
+											 pszName,
+											 ppsPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateNewDmaBufBackedPMR", ErrorBufDetach);
+
+	PhysmemSetDmaHeap(psPMR, heap);
+
+	return PVRSRV_OK;
+
+ErrorBufDetach:
+	dma_buf_detach(buf, buf_attachment);
+ErrorBufFree:
+	dma_heap_buffer_free(buf);
+ErrorHeapPut:
+	dma_heap_put(heap);
+ErrorExit:
+	return eError;
+}
+
+IMG_INTERNAL void
+OSFreeSecBuf(PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+
+	struct dma_buf *buf = PhysmemGetDmaBuf(psPMR);
+	struct dma_heap *heap = PhysmemGetDmaHeap(psPMR);
+
+	/* calls dma_buf_put() */
+	dma_heap_buffer_free(buf);
+
+	if (heap != NULL)
+	{
+		PhysmemSetDmaHeap(psPMR, NULL);
+		dma_heap_put(heap);
+	}
+
+	eError = PMRUnrefPMR(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+#else /* PVR_ANDROID_HAS_DMA_HEAP_FIND */
+IMG_INTERNAL PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR)
+{
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_ERROR eError;
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiSize,
+									1,
+									1,
+									&ui32MappingTable,
+									ExactLog2(OSGetPageSize()),
+									PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_SECURE)
+									| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE,
+									OSStringLength(pszName),
+									pszName,
+									OSGetCurrentClientProcessIDKM(),
+									ppsPMR,
+									PDUMP_NONE,
+									NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemNewRamBackedPMR", ErrorExit);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(*ppsPMR, PVR_SYS_ALLOC_PID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RIWritePMREntryWithOwnerKM", ErrorUnrefPMR);
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+ErrorUnrefPMR:
+	(void) PMRUnrefPMR(*ppsPMR);
+#endif
+ErrorExit:
+	return eError;
+}
+
+IMG_INTERNAL void
+OSFreeSecBuf(PMR *psPMR)
+{
+	PVRSRV_ERROR eError = PMRUnrefPMR(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+#endif
+#endif /* SUPPORT_SECURE_ALLOC_KM */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm.c
new file mode 100644
index 000000000000..d3d3ab9f83fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm.c
@@ -0,0 +1,155 @@
+/*************************************************************************/ /*!
+@File
+@Title          arm specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+ #include <linux/dma-map-ops.h>
+#else
+ #include <linux/dma-mapping.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) */
+#include <asm/cacheflush.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+
+static inline size_t pvr_dmac_range_len(const void *pvStart, const void *pvEnd)
+{
+	return (size_t)((char *)pvEnd - (char *)pvStart);
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+		                           sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                           DMA_TO_DEVICE);
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+		                        sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                        DMA_FROM_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_device(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_TO_DEVICE);
+	arm_dma_ops.sync_single_for_cpu(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_FROM_DEVICE);
+#endif
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+		                           sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                           DMA_TO_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_device(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_TO_DEVICE);
+#endif
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+		                        sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                        DMA_FROM_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_cpu(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_FROM_DEVICE);
+#endif
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+}
+
+/* User Enable Register */
+#define PMUSERENR_EN      0x00000001 /* enable user access to the counters */
+
+static void per_cpu_perf_counter_user_access_en(void *data)
+{
+	PVR_UNREFERENCED_PARAMETER(data);
+	/* Enable user-mode access to counters. */
+	asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(PMUSERENR_EN));
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+	on_each_cpu(per_cpu_perf_counter_user_access_en, NULL, 1);
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	/*
+	 * The kernel looks to have always used normal memory under ARM32.
+	 * See osfunc_arm64.c implementation for more details.
+	 */
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm64.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm64.c
new file mode 100644
index 000000000000..5d0419a00d48
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_arm64.c
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File
+@Title          arm64 specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#include <linux/cpumask.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+ #include <linux/dma-map-ops.h>
+#else
+ #include <linux/dma-mapping.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) */
+#include <asm/cacheflush.h>
+#include <linux/uaccess.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#include "kernel_compatibility.h"
+
+#if defined(CONFIG_OUTER_CACHE)
+  /* If you encounter a 64-bit ARM system with an outer cache, you'll need
+   * to add the necessary code to manage that cache. See osfunc_arm.c
+   * for an example of how to do so.
+   */
+	#error "CONFIG_OUTER_CACHE not supported on arm64."
+#endif
+
+static inline void begin_user_mode_access(void)
+{
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	uaccess_enable_privileged();
+#endif
+}
+
+static inline void end_user_mode_access(void)
+{
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	uaccess_disable_privileged();
+#endif
+}
+
+static inline void FlushRange(void *pvRangeAddrStart,
+							  void *pvRangeAddrEnd,
+							  PVRSRV_CACHE_OP eCacheOp)
+{
+	IMG_UINT32 ui32CacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+	IMG_BYTE *pbStart = pvRangeAddrStart;
+	IMG_BYTE *pbEnd = pvRangeAddrEnd;
+	IMG_BYTE *pbBase;
+
+	/*
+	  On arm64, the TRM states in D5.8.1 (data and unified caches) that if cache
+	  maintenance is performed on a memory location using a VA, the effect of
+	  that cache maintenance is visible to all VA aliases of the physical memory
+	  location. So here it's quicker to issue the machine cache maintenance
+	  instruction directly without going via the Linux kernel DMA framework as
+	  this is sufficient to maintain the CPU d-caches on arm64.
+	 */
+
+	begin_user_mode_access();
+
+	pbEnd = (IMG_BYTE *) PVR_ALIGN((uintptr_t)pbEnd, (uintptr_t)ui32CacheLineSize);
+
+	/* Memory-barrier */
+	asm volatile("dsb sy" : : : "memory");
+
+	switch (eCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			for (pbBase = pbStart; pbBase < pbEnd; pbBase += ui32CacheLineSize)
+			{
+				asm volatile ("dc cvac, %0" :: "r" (pbBase));
+			}
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+		case PVRSRV_CACHE_OP_FLUSH:
+			for (pbBase = pbStart; pbBase < pbEnd; pbBase += ui32CacheLineSize)
+			{
+				asm volatile ("dc civac, %0" :: "r" (pbBase));
+			}
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Cache maintenance operation type %d is invalid",
+					__func__, eCacheOp));
+			break;
+	}
+
+	/* Memory-barrier */
+	asm volatile("dsb sy" : : : "memory");
+
+	end_user_mode_access();
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	if (pvVirtStart == NULL)
+	{
+		/*
+		 * Converting the physical addresses to virtual addresses allows us to
+		 * utilize the assembly instruction that makes the Flush + Invalidate
+		 * cache operation atomic.
+		 * There is no state in-between the flush and the invalidate operation.
+		 */
+		pvVirtStart = phys_to_virt(sCPUPhysStart.uiAddr);
+		pvVirtEnd = phys_to_virt(sCPUPhysEnd.uiAddr);
+	}
+
+	FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_FLUSH);
+	return;
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+
+	if (pvVirtStart)
+	{
+		FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_CLEAN);
+		return;
+	}
+
+	dev = psDevNode->psDevConfig->pvOSDevice;
+
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+								   sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+								   DMA_TO_DEVICE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cache operation cannot be completed!"));
+	}
+
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+								 void *pvVirtStart,
+								 void *pvVirtEnd,
+								 IMG_CPU_PHYADDR sCPUPhysStart,
+								 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+
+	if (pvVirtStart)
+	{
+		FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_INVALIDATE);
+		return;
+	}
+
+	dev = psDevNode->psDevConfig->pvOSDevice;
+
+	if (dev)
+	{
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+								sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+								DMA_FROM_DEVICE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cache operation cannot be completed!"));
+	}
+}
+
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	if (!psDevNode->psDevConfig->pvOSDevice)
+	{
+		/* Host Mem device node doesn't have an associated Linux dev ptr.
+		   Use virtual addr ops instead of asking kernel to do physical
+		   maintenance */
+		return OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+	}
+
+	return OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	/*
+	 * Under ARM64 there is the concept of 'device' [0] and 'normal' [1] memory.
+	 * Unaligned access on device memory is explicitly disallowed [2]:
+	 *
+	 * 'Further, unaligned accesses are only allowed to regions marked as Normal
+	 *  memory type.
+	 *  ...
+	 *  Attempts to perform unaligned accesses when not allowed will cause an
+	 *  alignment fault (data abort).'
+	 *
+	 * Write-combine on ARM64 can be implemented as either normal non-cached
+	 * memory (NORMAL_NC) or as device memory with gathering enabled
+	 * (DEVICE_GRE.) Kernel 3.13 changed this from the latter to the former.
+	 *
+	 * [0]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/CHDBDIDF.html
+	 * [1]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch13s01s01.html
+	 * [2]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html
+	 */
+
+	pgprot_t pgprot = pgprot_writecombine(PAGE_KERNEL);
+
+	return (pgprot_val(pgprot) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_NC);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_riscv.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_riscv.c
new file mode 100644
index 000000000000..2d2ad2360433
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_riscv.c
@@ -0,0 +1,173 @@
+/*************************************************************************/ /*!
+@File
+@Title          RISC-V specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "cache_ops.h"
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_FLUSH,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: System doesn't implement cache maintenance. Skipping!",
+		         __func__));
+	}
+#endif
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_CLEAN,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: System doesn't implement cache maintenance. Skipping!",
+		         __func__));
+	}
+#endif
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+								 void *pvVirtStart,
+								 void *pvVirtEnd,
+								 IMG_CPU_PHYADDR sCPUPhysStart,
+								 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_INVALIDATE,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: System doesn't implement cache maintenance. Skipping!",
+		         __func__));
+	}
+#endif
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	/*
+	 * Need to obtain psDevNode here and do the following:
+	 *
+	 * OS_CACHE_OP_ADDR_TYPE eOpAddrType =
+	 *	psDevNode->psDevConfig->bHasPhysicalCacheMaintenance ?
+	 *		OS_CACHE_OP_ADDR_TYPE_PHYSICAL : OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+	 *
+	 * Return BOTH for now on.
+	 *
+	 */
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_BOTH;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+#if !defined(NO_HARDWARE)
+	PVR_DPF((PVR_DBG_WARNING, "%s: Not implemented!", __func__));
+	PVR_ASSERT(0);
+#endif
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+    // EIC770X support unaligned access.
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_x86.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_x86.c
new file mode 100644
index 000000000000..2c5c145e268f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osfunc_x86.c
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File
+@Title          x86 specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/smp.h>
+#include <linux/uaccess.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+static void x86_flush_cache_range(const void *pvStart, const void *pvEnd)
+{
+	IMG_BYTE *pbStart = (IMG_BYTE *)pvStart;
+	IMG_BYTE *pbEnd = (IMG_BYTE *)pvEnd;
+	IMG_BYTE *pbBase;
+
+	pbEnd = (IMG_BYTE *)PVR_ALIGN((uintptr_t)pbEnd,
+	                              (uintptr_t)boot_cpu_data.x86_clflush_size);
+
+	mb();
+
+	__uaccess_begin();
+
+	for (pbBase = pbStart; pbBase < pbEnd; pbBase += boot_cpu_data.x86_clflush_size)
+	{
+		clflush(pbBase);
+	}
+
+	__uaccess_end();
+
+	mb();
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	/* No clean feature on x86 */
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	/* No invalidate-only support */
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+	/* Not applicable to x86 architecture. */
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osmmap_stub.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osmmap_stub.c
new file mode 100644
index 000000000000..74bad7073c58
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/osmmap_stub.c
@@ -0,0 +1,146 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS abstraction for the mmap2 interface for mapping PMRs into
+                User Mode memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* our exported API */
+#include "osmmap.h"
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+/* services/include/ */
+
+/* services/include/srvhelper/ */
+#include "ra.h"
+
+#include "pmr.h"
+
+PVRSRV_ERROR
+OSMMapPMR(IMG_HANDLE hBridge,
+          IMG_HANDLE hPMR,
+          IMG_DEVMEM_SIZE_T uiPMRSize,
+          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+          IMG_HANDLE *phOSMMapPrivDataOut,
+          void **ppvMappingAddressOut,
+          size_t *puiMappingLengthOut)
+{
+    PVRSRV_ERROR eError;
+    PMR *psPMR;
+    void *pvKernelAddress;
+    size_t uiLength;
+    IMG_HANDLE hPriv;
+
+    PVR_UNREFERENCED_PARAMETER(hBridge);
+    PVR_UNREFERENCED_PARAMETER(uiFlags);
+
+    /*
+      Normally this function would mmap a PMR into the memory space of
+      user process, but in this case we're taking a PMR and mapping it
+      into kernel virtual space.  We keep the same function name for
+      symmetry as this allows the higher layers of the software stack
+      to not care whether they are user mode or kernel
+    */
+
+    psPMR = hPMR;
+
+    if (PMR_IsSparse(psPMR))
+    {
+        eError = PMRAcquireSparseKernelMappingData(psPMR,
+                                            0,
+                                            0,
+                                            &pvKernelAddress,
+                                            &uiLength,
+                                            &hPriv);
+    }
+    else
+    {
+        eError = PMRAcquireKernelMappingData(psPMR,
+                                            0,
+                                            0,
+                                            &pvKernelAddress,
+                                            &uiLength,
+                                            &hPriv);
+    }
+    if (eError != PVRSRV_OK)
+    {
+        goto e0;
+    }
+
+    *phOSMMapPrivDataOut = hPriv;
+    *ppvMappingAddressOut = pvKernelAddress;
+    *puiMappingLengthOut = uiLength;
+
+    /* MappingLength might be rounded up to page size */
+    PVR_ASSERT(*puiMappingLengthOut >= uiPMRSize);
+
+    return PVRSRV_OK;
+
+    /*
+      error exit paths follow
+    */
+
+e0:
+    PVR_ASSERT(eError != PVRSRV_OK);
+    return eError;
+}
+
+void
+OSMUnmapPMR(IMG_HANDLE hBridge,
+            IMG_HANDLE hPMR,
+            IMG_HANDLE hOSMMapPrivData,
+            void *pvMappingAddress,
+            size_t uiMappingLength)
+{
+    PMR *psPMR;
+
+    PVR_UNREFERENCED_PARAMETER(hBridge);
+    PVR_UNREFERENCED_PARAMETER(pvMappingAddress);
+    PVR_UNREFERENCED_PARAMETER(uiMappingLength);
+
+    psPMR = hPMR;
+    PMRReleaseKernelMappingData(psPMR,
+                                hOSMMapPrivData);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/ossecure_export.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/ossecure_export.c
new file mode 100644
index 000000000000..c1b4a7083bf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/ossecure_export.c
@@ -0,0 +1,155 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/anon_inodes.h>
+#include <linux/dcache.h>
+#include <linux/mount.h>
+#include <linux/sched.h>
+#include <linux/cred.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "ossecure_export.h"
+#include "private_data.h"
+#include "pvr_debug.h"
+
+#include "kernel_compatibility.h"
+
+typedef struct
+{
+	PVRSRV_ERROR (*pfnReleaseFunc)(void *);
+	void *pvData;
+} OSSecureFileData;
+
+static IMG_INT _OSSecureFileReleaseFunc(struct inode *psInode,
+                                        struct file *psFile)
+{
+	OSSecureFileData *psSecureFileData = (OSSecureFileData *)psFile->private_data;
+	psSecureFileData->pfnReleaseFunc(psSecureFileData->pvData);
+
+	OSFreeMem(psSecureFileData);
+	PVR_UNREFERENCED_PARAMETER(psInode);
+
+	return 0;
+}
+
+static struct file_operations secure_file_fops = {
+	.release	= _OSSecureFileReleaseFunc,
+};
+
+PVRSRV_ERROR OSSecureExport(const IMG_CHAR *pszName,
+                            PVRSRV_ERROR (*pfnReleaseFunc)(void *),
+                            void *pvData,
+                            IMG_SECURE_TYPE *phSecure)
+{
+	struct file *secure_file;
+	int secure_fd;
+	PVRSRV_ERROR eError;
+	OSSecureFileData *psSecureFileData;
+
+	PVR_ASSERT(pfnReleaseFunc != NULL || pvData != NULL);
+
+	psSecureFileData = OSAllocMem(sizeof(*psSecureFileData));
+	if (psSecureFileData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	psSecureFileData->pvData = pvData;
+	psSecureFileData->pfnReleaseFunc = pfnReleaseFunc;
+
+	/* Allocate a fd number */
+	secure_fd = get_unused_fd_flags(0);
+	if (secure_fd < 0)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	/* Create a file with provided name, fops and flags,
+	 * also store the private data in the file */
+	secure_file = anon_inode_getfile(pszName, &secure_file_fops, psSecureFileData, 0);
+	if (IS_ERR(secure_file))
+	{
+		put_unused_fd(secure_fd);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	/* Bind our struct file with it's fd number */
+	fd_install(secure_fd, secure_file);
+
+	*phSecure = secure_fd;
+	return PVRSRV_OK;
+
+e0:
+	OSFreeMem(psSecureFileData);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR OSSecureImport(IMG_SECURE_TYPE hSecure, void **ppvData)
+{
+	struct file *secure_file;
+	PVRSRV_ERROR eError;
+	OSSecureFileData *psSecureFileData;
+
+	secure_file = fget(hSecure);
+	if (!secure_file)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	psSecureFileData = (OSSecureFileData *)secure_file->private_data;
+	*ppvData = psSecureFileData->pvData;
+
+	fput(secure_file);
+	return PVRSRV_OK;
+
+err_out:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf.c
new file mode 100644
index 000000000000..97f84d6095bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf.c
@@ -0,0 +1,2391 @@
+/*************************************************************************/ /*!
+@File           physmem_dmabuf.c
+@Title          dmabuf memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for dmabuf memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "physmem_dmabuf.h"
+#include "physmem_dmabuf_internal.h"
+#include "physmem.h"
+#include "pvrsrv.h"
+#include "pmr.h"
+
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_prime.h>
+#else
+#include <drm/drmP.h>
+#endif
+#include <drm/drm_gem.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pmr_impl.h"
+#include "pmr_env.h"
+#include "hash.h"
+#include "private_data.h"
+#include "module_common.h"
+#include "pvr_ion_stats.h"
+#include "cache_km.h"
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+typedef struct _PMR_DMA_BUF_WRAPPER_
+{
+	PMR *psPMR;
+
+	/*
+	 * Set if the PMR has been exported via GEM. This field should only
+	 * be accessed using smp_store_release and smp_load_acquire.
+	 * In general, the field is not safe to access, as the wrapper doesn't
+	 * hold a reference on the GEM object.
+	 */
+	struct drm_gem_object *psObj;
+
+	struct dma_resv sDmaResv;
+
+	/* kernel mapping */
+	IMG_UINT32 uiKernelMappingRefCnt;
+	IMG_HANDLE hKernelMappingHandle;
+	void *pvKernelMappingAddr;
+	PMR_SIZE_T uiKernelMappingLen;
+	POS_LOCK hKernelMappingLock;
+
+	/* device mapping */
+	IMG_UINT32 uiDeviceMappingRefCnt;
+	POS_LOCK hDeviceMappingLock;
+	struct sg_table *psTable;
+} PMR_DMA_BUF_WRAPPER;
+
+typedef struct _PMR_DMA_BUF_GEM_OBJ {
+	struct drm_gem_object sBase;
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+} PMR_DMA_BUF_GEM_OBJ;
+
+#define TO_PMR_DMA_BUF_GEM_OBJ(psObj) IMG_CONTAINER_OF((psObj), PMR_DMA_BUF_GEM_OBJ, sBase)
+
+static PVRSRV_ERROR _PMRKMap(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                             size_t uiSize,
+                             void **pvAddr)
+{
+	IMG_HANDLE hKernelMapping;
+	void *pvAddrOut = NULL;
+	size_t uiLengthOut = 0;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt++ == 0)
+	{
+		PVR_ASSERT(psPMRWrapper->hKernelMappingHandle == NULL);
+		PVR_ASSERT(psPMRWrapper->pvKernelMappingAddr == NULL);
+
+		if (PMR_IsSparse(psPMRWrapper->psPMR))
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMRWrapper->psPMR,
+			                                           0,
+			                                           uiSize,
+			                                           &pvAddrOut,
+			                                           &uiLengthOut,
+			                                           &hKernelMapping);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData",
+			                      ErrReleaseLock);
+		}
+		else
+		{
+			eError = PMRAcquireKernelMappingData(psPMRWrapper->psPMR,
+			                                     0,
+			                                     uiSize,
+			                                     &pvAddrOut,
+			                                     &uiLengthOut,
+			                                     &hKernelMapping);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData",
+			                      ErrReleaseLock);
+		}
+
+		psPMRWrapper->hKernelMappingHandle = hKernelMapping;
+		psPMRWrapper->pvKernelMappingAddr = pvAddrOut;
+		psPMRWrapper->uiKernelMappingLen = uiLengthOut;
+	}
+
+	*pvAddr = psPMRWrapper->pvKernelMappingAddr;
+	goto ExitUnlock;
+
+ErrReleaseLock:
+	psPMRWrapper->uiKernelMappingRefCnt--;
+
+ExitUnlock:
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+	return eError;
+}
+
+static void _PMRKUnmap(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (--psPMRWrapper->uiKernelMappingRefCnt == 0)
+	{
+		PVRSRV_ERROR eError;
+
+		PVR_ASSERT(psPMRWrapper->hKernelMappingHandle != NULL);
+		PVR_ASSERT(psPMRWrapper->pvKernelMappingAddr != NULL);
+
+		eError = PMRReleaseKernelMappingData(psPMRWrapper->psPMR,
+		                                     psPMRWrapper->hKernelMappingHandle);
+		PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+		psPMRWrapper->hKernelMappingHandle = NULL;
+		psPMRWrapper->pvKernelMappingAddr = NULL;
+		psPMRWrapper->uiKernelMappingLen = 0;
+	}
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+}
+
+static PVRSRV_ERROR _PMRInvalidateCache(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt == 0) {
+		OSLockRelease(psPMRWrapper->hKernelMappingLock);
+		return PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+	}
+
+	eError = CacheOpValExec(psPMRWrapper->psPMR,
+	                        (IMG_UINT64) (uintptr_t) psPMRWrapper->pvKernelMappingAddr,
+	                        0,
+	                        psPMRWrapper->uiKernelMappingLen,
+	                        PVRSRV_CACHE_OP_INVALIDATE);
+	PVR_LOG_IF_ERROR(eError, "CacheOpValExec");
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _PMRCleanCache(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt == 0) {
+		OSLockRelease(psPMRWrapper->hKernelMappingLock);
+		return PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+	}
+
+	eError = CacheOpValExec(psPMRWrapper->psPMR,
+	                        (IMG_UINT64) (uintptr_t) psPMRWrapper->pvKernelMappingAddr,
+	                        0,
+	                        psPMRWrapper->uiKernelMappingLen,
+	                        PVRSRV_CACHE_OP_FLUSH);
+	PVR_LOG_IF_ERROR(eError, "CacheOpValExec");
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+
+	return eError;
+}
+
+/*
+ * dma_buf_ops common code
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ * The following common functions are used by both the dma_buf and GEM
+ * callbacks.
+ */
+
+static int PVRDmaBufOpsAttachCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+							  struct dma_buf_attachment *psAttachment)
+{
+	PMR *psPMR = psPMRWrapper->psPMR;
+
+	if (PMR_GetType(psPMR) == PMR_TYPE_DMABUF)
+	{
+		// don't support exporting PMRs that are itself created from imported
+		// DmaBufs
+		PVR_DPF((PVR_DBG_ERROR, "exporting PMRs of type DMABUF not supported"));
+
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static struct sg_table *PVRDmaBufOpsMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                              struct dma_buf_attachment *psAttachment,
+                                              enum dma_data_direction eDirection)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMR = psPMRWrapper->psPMR;
+	IMG_UINT uiNents;
+	IMG_DEVMEM_SIZE_T uiPhysSize, uiVirtSize;
+	IMG_UINT32 uiNumVirtPages;
+	void *pvPAddrData = NULL;
+	IMG_DEV_PHYADDR asPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC], *psPAddr = asPAddr;
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC], *pbValid = abValid;
+	IMG_UINT32 i;
+	IMG_DEV_PHYADDR sPAddrPrev, sPAddrCurr;
+	struct sg_table *psTable;
+	struct scatterlist *psSg;
+	int iRet = 0;
+	IMG_UINT32 uiDevPageShift, uiDevPageSize;
+
+	OSLockAcquire(psPMRWrapper->hDeviceMappingLock);
+
+	psPMRWrapper->uiDeviceMappingRefCnt++;
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt > 1)
+	{
+		goto OkUnlock;
+	}
+
+	PVR_ASSERT(psPMRWrapper->psTable == NULL);
+
+	uiDevPageShift = PMR_GetLog2Contiguity(psPMR);
+	uiDevPageSize = 1u << uiDevPageShift;
+	uiVirtSize = PMR_LogicalSize(psPMR);
+	uiPhysSize = PMR_PhysicalSize(psPMR);
+	if (uiPhysSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "invalid PMR size"));
+		iRet = PVRSRVToNativeError(PVRSRV_ERROR_BAD_MAPPING);
+		goto ErrUnlockMapping;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): mapping pmr: 0x%p@0x%" IMG_UINT64_FMTSPECx
+	         " from heap: \"%s\" with page size: 0x%x", __func__, psPMR,
+	         uiPhysSize, PhysHeapName(PMR_PhysHeap(psPMR)), uiDevPageSize));
+
+	uiNumVirtPages = uiVirtSize >> uiDevPageShift;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMRLockSysPhysAddresses");
+		iRet = PVRSRVToNativeError(eError);
+		goto ErrUnlockMapping;
+	}
+
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	psTable = OSAllocZMem(sizeof(*psTable));
+	if (psTable == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAllocMem.1() failed"));
+		iRet = -ENOMEM;
+		goto ErrUnlockPhysAddresses;
+	}
+
+	if (uiNumVirtPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		pvPAddrData = OSAllocMem(uiNumVirtPages * (sizeof(*psPAddr) + sizeof(*pbValid)));
+		if (pvPAddrData == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "OSAllocMem.2() failed"));
+			iRet = -ENOMEM;
+			goto ErrFreeTable;
+		}
+
+		psPAddr = IMG_OFFSET_ADDR(pvPAddrData, 0);
+		pbValid = IMG_OFFSET_ADDR(pvPAddrData, uiNumVirtPages * sizeof(*psPAddr));
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+	                         uiDevPageShift,
+	                         uiNumVirtPages,
+	                         0,
+	                         psPAddr,
+	                         pbValid,
+	                         DEVICE_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "PMR_DevPhysAddr");
+		iRet = PVRSRVToNativeError(eError);
+		goto ErrFreePAddrData;
+	}
+
+	/* Calculate how many contiguous regions there are in the PMR. This
+	 * value will be used to allocate one scatter list for every region. */
+
+	/* Find first valid physical address. */
+	for (i = 0; i < uiNumVirtPages && !pbValid[i]; i++);
+
+	sPAddrPrev = psPAddr[i];
+	uiNents = 1;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): %03u paddr: 0x%" IMG_UINT64_FMTSPECx,
+	         __func__, i, sPAddrPrev.uiAddr));
+
+	/* Find the rest of the addresses. */
+	for (i = i + 1; i < uiNumVirtPages; i++)
+	{
+		if (!pbValid[i])
+		{
+			continue;
+		}
+
+		sPAddrCurr = psPAddr[i];
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s(): %03u paddr: 0x%" IMG_UINT64_FMTSPECx
+		         ", pprev: 0x%" IMG_UINT64_FMTSPECx ", valid: %u", __func__,
+		         i, sPAddrCurr.uiAddr, sPAddrPrev.uiAddr, pbValid[i]));
+
+		if (sPAddrCurr.uiAddr != (sPAddrPrev.uiAddr + uiDevPageSize))
+		{
+			uiNents++;
+		}
+
+		sPAddrPrev = sPAddrCurr;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): found %u contiguous regions", __func__,
+	         uiNents));
+
+	/* Allocate scatter lists from all of the contiguous regions. */
+
+	iRet = sg_alloc_table(psTable, uiNents, GFP_KERNEL);
+	if (iRet != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table() failed with error %d", iRet));
+		goto ErrFreePAddrData;
+	}
+
+	/* Fill in all of the physical addresses and sizes for all of the
+	 * contiguous regions that were calculated. */
+
+	for (i = 0; i < uiNumVirtPages && !pbValid[i]; i++);
+
+	psSg = psTable->sgl;
+	sPAddrPrev = psPAddr[i];
+
+	sg_dma_address(psSg) = sPAddrPrev.uiAddr;
+	sg_dma_len(psSg) = uiDevPageSize;
+
+	for (i = i + 1; i < uiNumVirtPages; i++)
+	{
+		if (!pbValid[i])
+		{
+			continue;
+		}
+
+		sPAddrCurr = psPAddr[i];
+
+		if (sPAddrCurr.uiAddr != (sPAddrPrev.uiAddr + uiDevPageSize))
+		{
+			psSg = sg_next(psSg);
+			PVR_ASSERT(psSg != NULL);
+
+			sg_dma_address(psSg) = sPAddrCurr.uiAddr;
+			sg_dma_len(psSg) = uiDevPageSize;
+		}
+		else
+		{
+			sg_dma_len(psSg) += uiDevPageSize;
+		}
+
+		sPAddrPrev = sPAddrCurr;
+	}
+
+	if (pvPAddrData != NULL)
+	{
+		OSFreeMem(pvPAddrData);
+	}
+
+	psPMRWrapper->psTable = psTable;
+
+OkUnlock:
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+
+	return psPMRWrapper->psTable;
+
+ErrFreePAddrData:
+	if (pvPAddrData != NULL)
+	{
+		OSFreeMem(pvPAddrData);
+	}
+ErrFreeTable:
+	OSFreeMem(psTable);
+ErrUnlockPhysAddresses:
+	{
+		PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+	}
+ErrUnlockMapping:
+	psPMRWrapper->uiDeviceMappingRefCnt--;
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+
+	return ERR_PTR(iRet);
+}
+
+static void PVRDmaBufOpsUnmapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                              struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hDeviceMappingLock);
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "reference count on mapping already 0"));
+		goto ErrUnlock;
+	}
+
+	psPMRWrapper->uiDeviceMappingRefCnt--;
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt > 0)
+	{
+		goto ErrUnlock;
+	}
+
+	dma_unmap_sg(psAttachment->dev, psTable->sgl, psTable->nents, eDirection);
+	sg_free_table(psTable);
+
+	eError = PMRUnlockSysPhysAddresses(psPMRWrapper->psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+
+	OSFreeMem(psPMRWrapper->psTable);
+	psPMRWrapper->psTable = NULL;
+
+ErrUnlock:
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+}
+
+static int PVRDmaBufOpsBeginCpuAccessCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                      enum dma_data_direction eDirection)
+{
+	if (PVRSRV_CHECK_CPU_CACHED(PMR_Flags(psPMRWrapper->psPMR)))
+	{
+		PVRSRV_ERROR eError = _PMRInvalidateCache(psPMRWrapper);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PMRInvalidateCache");
+			return OSPVRSRVToNativeError(eError);
+		}
+	}
+
+	return 0;
+}
+
+static int PVRDmaBufOpsEndCpuAccessCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                    enum dma_data_direction eDirection)
+{
+	if (PVRSRV_CHECK_CPU_CACHED(PMR_Flags(psPMRWrapper->psPMR)))
+	{
+		PVRSRV_ERROR eError = _PMRCleanCache(psPMRWrapper);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PMRCleanCache");
+			return OSPVRSRVToNativeError(eError);
+		}
+	}
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, unsigned long uiPageNum)
+{
+	return NULL;
+}
+
+static void PVRDmaBufOpsKUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, unsigned long uiPageNum, void *pvMem)
+{
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	void *pvAddrOut = NULL;
+	PVRSRV_ERROR eError;
+
+	eError = _PMRKMap(psPMRWrapper, 0, &pvAddrOut);
+
+	return eError == PVRSRV_OK ? pvAddrOut : NULL;
+}
+#else
+static int PVRDmaBufOpsVMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct iosys_map *psMap)
+{
+	void *pvAddrOut = NULL;
+	PVRSRV_ERROR eError;
+
+	eError = _PMRKMap(psPMRWrapper, 0, &pvAddrOut);
+
+	if (eError != PVRSRV_OK)
+	{
+		return OSPVRSRVToNativeError(eError);
+	}
+
+	if (PhysHeapGetType(PMR_PhysHeap(psPMRWrapper->psPMR)) == PHYS_HEAP_TYPE_UMA)
+	{
+		iosys_map_set_vaddr(psMap, pvAddrOut);
+	}
+	else
+	{
+		iosys_map_set_vaddr_iomem(psMap, pvAddrOut);
+	}
+
+	return 0;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, void *pvAddr)
+{
+	_PMRKUnmap(psPMRWrapper);
+}
+#else
+static void PVRDmaBufOpsVUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct iosys_map *psMap)
+{
+	_PMRKUnmap(psPMRWrapper);
+
+	iosys_map_clear(psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct vm_area_struct *psVMA)
+{
+	PMR *psPMR = psPMRWrapper->psPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiProtFlags =
+	    (BITMASK_HAS(psVMA->vm_flags, VM_READ) ? PVRSRV_MEMALLOCFLAG_CPU_READABLE : 0) |
+	    (BITMASK_HAS(psVMA->vm_flags, VM_WRITE) ? PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0);
+	PVRSRV_ERROR eError;
+
+	/* Forcibly clear the VM_MAYWRITE flag as this is inherited from the
+	 * kernel mmap code and we do not want to produce a potentially writable
+	 * mapping from a read-only mapping.
+	 */
+	if (!BITMASK_HAS(psVMA->vm_flags, VM_WRITE))
+	{
+		pvr_vm_flags_clear(psVMA, VM_MAYWRITE);
+	}
+
+	eError = PMRMMapPMR(psPMR, psVMA, uiProtFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMRMMapPMR");
+		return OSPVRSRVToNativeError(eError);
+	}
+
+	return 0;
+}
+
+/* end of dma_buf_ops common code*/
+
+/*
+ * dma_buf_ops (non-GEM)
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ */
+
+static int PVRDmaBufOpsAttach(struct dma_buf *psDmaBuf,
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+							  struct device *psDev,
+#endif
+							  struct dma_buf_attachment *psAttachment)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsAttachCommon(psPMRWrapper, psAttachment);
+}
+
+static struct sg_table *PVRDmaBufOpsMap(struct dma_buf_attachment *psAttachment,
+                                        enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psAttachment->dmabuf->priv;
+
+	return PVRDmaBufOpsMapCommon(psPMRWrapper, psAttachment, eDirection);
+}
+
+static void PVRDmaBufOpsUnmap(struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psAttachment->dmabuf->priv;
+
+	PVRDmaBufOpsUnmapCommon(psPMRWrapper, psAttachment, psTable, eDirection);
+}
+
+static void PVRDmaBufOpsRelease(struct dma_buf *psDmaBuf)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+	PMR *psPMR = psPMRWrapper->psPMR;
+
+	PVRSRV_ERROR eError = PMRUnrefPMR(psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+}
+
+static int PVRDmaBufOpsBeginCpuAccess(struct dma_buf *psDmaBuf,
+                                      enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsBeginCpuAccessCommon(psPMRWrapper, eDirection);
+}
+
+static int PVRDmaBufOpsEndCpuAccess(struct dma_buf *psDmaBuf,
+                                    enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+	int iErr;
+
+	iErr = PVRDmaBufOpsEndCpuAccessCommon(psPMRWrapper, eDirection);
+
+	return iErr;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMap(struct dma_buf *psDmaBuf, unsigned long uiPageNum)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsKMapCommon(psPMRWrapper, uiPageNum);
+}
+
+static void PVRDmaBufOpsKUnMap(struct dma_buf *psDmaBuf, unsigned long uiPageNum, void *pvMem)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsKUnMapCommon(psPMRWrapper, uiPageNum, pvMem);
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMap(struct dma_buf *psDmaBuf)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper);
+}
+#else
+static int PVRDmaBufOpsVMap(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMap(struct dma_buf *psDmaBuf, void *pvAddr)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, pvAddr);
+}
+#else
+static void PVRDmaBufOpsVUnMap(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMap(struct dma_buf *psDmaBuf, struct vm_area_struct *psVMA)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): psDmaBuf = %px, psPMR = %px", __func__,
+	         psDmaBuf, psPMRWrapper->psPMR));
+
+	return PVRDmaBufOpsMMapCommon(psPMRWrapper, psVMA);
+}
+
+static const struct dma_buf_ops sPVRDmaBufOps =
+{
+	.attach        = PVRDmaBufOpsAttach,
+	.map_dma_buf   = PVRDmaBufOpsMap,
+	.unmap_dma_buf = PVRDmaBufOpsUnmap,
+	.release       = PVRDmaBufOpsRelease,
+	.begin_cpu_access = PVRDmaBufOpsBeginCpuAccess,
+	.end_cpu_access   = PVRDmaBufOpsEndCpuAccess,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+	.map_atomic    = PVRDmaBufOpsKMap,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	.map           = PVRDmaBufOpsKMap,
+	.unmap         = PVRDmaBufOpsKUnMap,
+#endif
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.kmap_atomic   = PVRDmaBufOpsKMap,
+	.kmap          = PVRDmaBufOpsKMap,
+	.kunmap        = PVRDmaBufOpsKUnMap,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.mmap          = PVRDmaBufOpsMMap,
+
+	.vmap          = PVRDmaBufOpsVMap,
+	.vunmap        = PVRDmaBufOpsVUnMap,
+};
+
+/* end of dma_buf_ops (non-GEM) */
+
+/*
+ * dma_buf_ops (GEM)
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ */
+
+static int PVRDmaBufOpsAttachGEM(struct dma_buf *psDmaBuf,
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+							  struct device *psDev,
+#endif
+							  struct dma_buf_attachment *psAttachment)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsAttachCommon(psPMRWrapper, psAttachment);
+}
+
+static struct sg_table *PVRDmaBufOpsMapGEM(struct dma_buf_attachment *psAttachment,
+                                        enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psAttachment->dmabuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsMapCommon(psPMRWrapper, psAttachment, eDirection);
+}
+
+static void PVRDmaBufOpsUnmapGEM(struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psAttachment->dmabuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsUnmapCommon(psPMRWrapper, psAttachment, psTable, eDirection);
+}
+
+static int PVRDmaBufOpsBeginCpuAccessGEM(struct dma_buf *psDmaBuf,
+                                      enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsBeginCpuAccessCommon(psPMRWrapper, eDirection);
+}
+
+static int PVRDmaBufOpsEndCpuAccessGEM(struct dma_buf *psDmaBuf,
+                                    enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	int iErr;
+
+	iErr = PVRDmaBufOpsEndCpuAccessCommon(psPMRWrapper, eDirection);
+
+	return iErr;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMapGEM(struct dma_buf *psDmaBuf, unsigned long uiPageNum)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsKMapCommon(psPMRWrapper, uiPageNum);
+}
+
+static void PVRDmaBufOpsKUnMapGEM(struct dma_buf *psDmaBuf, unsigned long uiPageNum, void *pvMem)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsKUnMapCommon(psPMRWrapper, uiPageNum, pvMem);
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMapGEM(struct dma_buf *psDmaBuf)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper);
+}
+#else
+static int PVRDmaBufOpsVMapGEM(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMapGEM(struct dma_buf *psDmaBuf, void *pvAddr)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, pvAddr);
+}
+#else
+static void PVRDmaBufOpsVUnMapGEM(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMapGEM(struct dma_buf *psDmaBuf, struct vm_area_struct *psVMA)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): psDmaBuf = %px, psPMR = %px", __func__,
+	         psDmaBuf, psPMRWrapper->psPMR));
+
+	return PVRDmaBufOpsMMapCommon(psPMRWrapper, psVMA);
+}
+
+static const struct dma_buf_ops sPVRDmaBufOpsGEM =
+{
+	.attach        = PVRDmaBufOpsAttachGEM,
+	.map_dma_buf   = PVRDmaBufOpsMapGEM,
+	.unmap_dma_buf = PVRDmaBufOpsUnmapGEM,
+	.release       = drm_gem_dmabuf_release,
+	.begin_cpu_access = PVRDmaBufOpsBeginCpuAccessGEM,
+	.end_cpu_access   = PVRDmaBufOpsEndCpuAccessGEM,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+	.map_atomic    = PVRDmaBufOpsKMapGEM,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	.map           = PVRDmaBufOpsKMapGEM,
+	.unmap         = PVRDmaBufOpsKUnMapGEM,
+#endif
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.kmap_atomic   = PVRDmaBufOpsKMapGEM,
+	.kmap          = PVRDmaBufOpsKMapGEM,
+	.kunmap        = PVRDmaBufOpsKUnMapGEM,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.mmap          = PVRDmaBufOpsMMapGEM,
+
+	.vmap          = PVRDmaBufOpsVMapGEM,
+	.vunmap        = PVRDmaBufOpsVUnMapGEM,
+};
+
+/* end of dma_buf_ops (GEM) */
+
+
+typedef struct _PMR_DMA_BUF_DATA_
+{
+	/* Filled in at PMR create time */
+	PHYS_HEAP *psPhysHeap;
+	struct dma_buf_attachment *psAttachment;
+#if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+	struct dma_heap *psDmaHeap;
+#endif
+	PFN_DESTROY_DMABUF_PMR pfnDestroy;
+	IMG_BOOL bPoisonOnFree;
+	IMG_PID uiOriginPID;
+
+	/* Mapping information. */
+	struct iosys_map sMap;
+
+	/* Modified by PMR lock/unlock */
+	struct sg_table *psSgTable;
+	IMG_DEV_PHYADDR *pasDevPhysAddr;
+	IMG_UINT32 ui32PhysPageCount;
+	IMG_UINT32 ui32VirtPageCount;
+
+	IMG_BOOL bZombie;
+} PMR_DMA_BUF_DATA;
+
+/* Start size of the g_psDmaBufHash hash table */
+#define DMA_BUF_HASH_SIZE 20
+
+static DEFINE_MUTEX(g_FactoryLock);
+
+static HASH_TABLE *g_psDmaBufHash;
+static IMG_UINT32 g_ui32HashRefCount;
+
+#if defined(PVR_ANDROID_ION_USE_SG_LENGTH)
+#define pvr_sg_length(sg) ((sg)->length)
+#else
+#define pvr_sg_length(sg) sg_dma_len(sg)
+#endif
+
+/*
+ * _pvr_dma_resv_lock
+ *
+ * Locks a dma_buf's reservation object (dma_resv). This is required when calling
+ * certain functions from the dma_buf interface (for more info see:
+ * https://docs.kernel.org/driver-api/dma-buf.html#dma-buf-locking-convention).
+ *
+ * This is a wrapper for the kernel's own locking functions which have changed
+ * over a number of versions.
+ *
+ * The dma_resv uses a ww_mutex which provides additional behaviours to prevent
+ * deadlocks. However, this function expects that only a single lock will be
+ * locked in a function at a time making the error code returned is redundant.
+ *
+ * If this changes a ww_acquire_ctx should be used.
+ * For more information about ww_mutex and ww_acquire_ctx see:
+ * https://www.kernel.org/doc/Documentation/locking/ww-mutex-design.txt
+ *
+ * Unlock the dma_resv lock with _pvr_dma_resv_unlock
+ */
+static inline void _pvr_dma_resv_lock(const struct dma_buf *psDMABuf)
+{
+	int iErr = 0;
+
+	PVR_ASSERT(psDMABuf != NULL);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	iErr = dma_resv_lock(psDMABuf->resv, NULL);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	iErr = reservation_object_lock(psDMABuf->resv, NULL);
+#else
+	iErr = ww_mutex_lock(&psDMABuf->resv->lock, NULL);
+#endif
+
+	/* This is not expected to occur, but if it does we are notified. */
+	PVR_LOG_IF_FALSE_VA(PVR_DBG_ERROR,
+	                    iErr == 0,
+	                    "Attempt to lock dma_resv resulted in %d",
+	                    iErr);
+}
+
+/*
+ * _pvr_dma_resv_unlock
+ *
+ * Unlocks a dma_buf's dma_resv struct, previously locked with _pvr_dma_resv_lock.
+ */
+static inline void _pvr_dma_resv_unlock(const struct dma_buf *psDMABuf)
+{
+	PVR_ASSERT(psDMABuf != NULL);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	dma_resv_unlock(psDMABuf->resv);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	reservation_object_unlock(psDMABuf->resv);
+#else
+	ww_mutex_unlock(&psDMABuf->resv->lock);
+#endif
+}
+
+static inline PVRSRV_ERROR _pvr_dma_buf_map_attachment(struct dma_buf_attachment *psAttachment,
+                                                       enum dma_data_direction eDirection,
+                                                       struct sg_table **ppsTable)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0))
+	/* DMA buf reservation locks will be taken internally */
+	*ppsTable = dma_buf_map_attachment_unlocked(psAttachment, eDirection);
+#else
+	_pvr_dma_resv_lock(psAttachment->dmabuf);
+	*ppsTable = dma_buf_map_attachment(psAttachment, eDirection);
+	_pvr_dma_resv_unlock(psAttachment->dmabuf);
+#endif
+
+	if (IS_ERR_OR_NULL(*ppsTable))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map attachment. ppsTable = %p",
+		         __func__,
+		         *ppsTable));
+		eError = PVRSRV_ERROR_DMABUF_ATTACHMENT_MAPPING;
+	}
+
+	return eError;
+}
+
+static inline void _pvr_dma_buf_unmap_attachment(struct dma_buf_attachment *psAttachment,
+                                                 struct sg_table *psTable,
+                                                 enum dma_data_direction eDirection)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0))
+	/* DMA buf reservation locks will be taken internally */
+	dma_buf_unmap_attachment_unlocked(psAttachment, psTable, eDirection);
+#else
+	_pvr_dma_resv_lock(psAttachment->dmabuf);
+	dma_buf_unmap_attachment(psAttachment, psTable, eDirection);
+	_pvr_dma_resv_unlock(psAttachment->dmabuf);
+#endif
+}
+
+static int
+DmaBufSetValue(struct dma_buf *psDmaBuf, int iValue, const char *szFunc)
+{
+	struct iosys_map sMap;
+	int err, err_end_access;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	int i;
+#endif
+
+	err = dma_buf_begin_cpu_access(psDmaBuf, DMA_FROM_DEVICE);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to begin cpu access (err=%d)",
+		                        szFunc, err));
+		goto err_out;
+	}
+
+	_pvr_dma_resv_lock(psDmaBuf);
+	err = dma_buf_vmap(psDmaBuf, &sMap);
+	if (err)
+	{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map page (err=%d)",
+		                        szFunc, err));
+		goto exit_end_access;
+#else
+		for (i = 0; i < psDmaBuf->size / PAGE_SIZE; i++)
+		{
+			void *pvKernAddr;
+
+			pvKernAddr = dma_buf_kmap(psDmaBuf, i);
+			if (IS_ERR_OR_NULL(pvKernAddr))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map page (err=%ld)",
+							szFunc,
+							pvKernAddr ? PTR_ERR(pvKernAddr) : -ENOMEM));
+				err = !pvKernAddr ? -ENOMEM : -EINVAL;
+
+				goto exit_end_access;
+			}
+
+			memset(pvKernAddr, iValue, PAGE_SIZE);
+
+			dma_buf_kunmap(psDmaBuf, i, pvKernAddr);
+		}
+#endif
+	}
+	else
+	{
+		memset(sMap.vaddr, iValue, psDmaBuf->size);
+
+		dma_buf_vunmap(psDmaBuf, &sMap);
+	}
+
+	err = 0;
+
+exit_end_access:
+	_pvr_dma_resv_unlock(psDmaBuf);
+
+	do {
+		err_end_access = dma_buf_end_cpu_access(psDmaBuf, DMA_TO_DEVICE);
+	} while (err_end_access == -EAGAIN || err_end_access == -EINTR);
+
+	if (err_end_access)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to end cpu access (err=%d)",
+		                        szFunc, err_end_access));
+		if (!err)
+		{
+			err = err_end_access;
+		}
+	}
+
+err_out:
+	return err;
+}
+
+/*****************************************************************************
+ *                          PMR callback functions                           *
+ *****************************************************************************/
+
+/* This function is protected by the pfn(Get/Release)PMRFactoryLock() lock
+ * acquired/released in _UnrefAndMaybeDestroy() in pmr.c. */
+static void PMRFinalizeDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+	struct sg_table *psSgTable = psPrivData->psSgTable;
+
+	if (psDmaBuf->ops != &sPVRDmaBufOps && psDmaBuf->ops != &sPVRDmaBufOpsGEM)
+	{
+		if (g_psDmaBufHash)
+		{
+			/* We have a hash table so check if we've seen this dmabuf before */
+			if (HASH_Remove(g_psDmaBufHash, (uintptr_t) psDmaBuf) != 0U)
+			{
+				g_ui32HashRefCount--;
+
+				if (g_ui32HashRefCount == 0)
+				{
+					HASH_Delete(g_psDmaBufHash);
+					g_psDmaBufHash = NULL;
+				}
+			}
+
+			PVRSRVIonRemoveMemAllocRecord(psDmaBuf);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (psPrivData->bZombie)
+	{
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+		                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+		                            psPrivData->uiOriginPID);
+	}
+	else
+#endif
+	{
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+		                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+		                            psPrivData->uiOriginPID);
+	}
+#endif
+
+	psPrivData->ui32PhysPageCount = 0;
+
+	_pvr_dma_buf_unmap_attachment(psAttachment, psSgTable, DMA_BIDIRECTIONAL);
+
+	if (psPrivData->bPoisonOnFree)
+	{
+		int err = DmaBufSetValue(psDmaBuf, PVRSRV_POISON_ON_FREE_VALUE,
+		                         __func__);
+		PVR_LOG_IF_FALSE(err != 0, "Failed to poison allocation before free");
+
+		PVR_ASSERT(err != 0);
+	}
+
+	if (psPrivData->pfnDestroy)
+	{
+		psPrivData->pfnDestroy(psPrivData->psPhysHeap, psPrivData->psAttachment);
+	}
+
+	OSFreeMem(psPrivData->pasDevPhysAddr);
+	OSFreeMem(psPrivData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static PVRSRV_ERROR PMRZombifyDmaBufMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	psPrivData->bZombie = IMG_TRUE;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+#endif
+
+	PVRSRVIonZombifyMemAllocRecord(psDmaBuf);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static PVRSRV_ERROR PMRLockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+{
+	/* The imported memory is assumed to be backed by a permanent allocation.
+	 * The PMR does not need to be (un)locked */
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRUnlockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+                                                 PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR PMRUnlockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	/* The imported memory is assumed to be backed by a permanent allocation.
+	 * The PMR does not need to be (un)locked */
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+	return PVRSRV_OK;
+}
+
+static void PMRFactoryLock(void)
+{
+	mutex_lock(&g_FactoryLock);
+}
+
+static void PMRFactoryUnlock(void)
+{
+	mutex_unlock(&g_FactoryLock);
+}
+
+static PVRSRV_ERROR PMRDevPhysAddrDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+					 IMG_UINT32 ui32Log2PageSize,
+					 IMG_UINT32 ui32NumOfPages,
+					 IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					 IMG_UINT64 ui64IPAPolicyValue,
+					 IMG_UINT64 ui64IPAClearMask,
+#endif
+					 IMG_BOOL *pbValid,
+					 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	IMG_UINT32 ui32PageIndex;
+	IMG_UINT32 idx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (ui32Log2PageSize != PAGE_SHIFT)
+	{
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (idx=0; idx < ui32NumOfPages; idx++)
+	{
+		if (pbValid[idx])
+		{
+			IMG_UINT32 ui32InPageOffset;
+
+			ui32PageIndex = puiOffset[idx] >> PAGE_SHIFT;
+			ui32InPageOffset = puiOffset[idx] - ((IMG_DEVMEM_OFFSET_T)ui32PageIndex << PAGE_SHIFT);
+
+			PVR_LOG_RETURN_IF_FALSE(ui32PageIndex < psPrivData->ui32VirtPageCount,
+			                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+			PVR_ASSERT(ui32InPageOffset < PAGE_SIZE);
+			psDevPAddr[idx].uiAddr = psPrivData->pasDevPhysAddr[ui32PageIndex].uiAddr + ui32InPageOffset;
+#if defined(SUPPORT_STATIC_IPA)
+			/* Modify the physical address with the associated IPA values */
+			psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+			psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+		}
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+				  size_t uiOffset,
+				  size_t uiSize,
+				  void **ppvKernelAddressOut,
+				  IMG_HANDLE *phHandleOut,
+				  PMR_FLAGS_T ulFlags)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if (psPrivData->ui32PhysPageCount != psPrivData->ui32VirtPageCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Kernel mappings for sparse DMABufs "
+				"are not allowed!", __func__));
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail;
+	}
+
+	err = dma_buf_begin_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	if (err)
+	{
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail;
+	}
+
+	err = dma_buf_vmap(psDmaBuf, &psPrivData->sMap);
+	if (err != 0 || psPrivData->sMap.vaddr == NULL)
+	{
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail_kmap;
+	}
+
+	*ppvKernelAddressOut = psPrivData->sMap.vaddr + uiOffset;
+	*phHandleOut = psPrivData->sMap.vaddr;
+
+	return PVRSRV_OK;
+
+fail_kmap:
+	do {
+		err = dma_buf_end_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	} while (err == -EAGAIN || err == -EINTR);
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+					      IMG_HANDLE hHandle)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	int err;
+
+	dma_buf_vunmap(psDmaBuf, &psPrivData->sMap);
+
+	do {
+		err = dma_buf_end_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	} while (err == -EAGAIN || err == -EINTR);
+}
+
+static PVRSRV_ERROR PMRMMapDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+                                  PMR *psPMR,
+                                  PMR_MMAP_DATA pOSMMapData)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	struct vm_area_struct *psVma = pOSMMapData;
+	int err;
+
+	if (psPrivData->ui32PhysPageCount != psPrivData->ui32VirtPageCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Not possible to MMAP sparse DMABufs",
+				__func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	err = dma_buf_mmap(psDmaBuf, psVma, 0);
+	if (err)
+	{
+		return (err == -EINVAL) ? PVRSRV_ERROR_NOT_SUPPORTED : PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	MMapStatsAddOrUpdatePMR(psPMR, psVma->vm_end - psVma->vm_start);
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRDmaBufFuncTab =
+{
+	.pfnLockPhysAddresses = PMRLockPhysAddressesDmaBuf,
+	.pfnUnlockPhysAddresses = PMRUnlockPhysAddressesDmaBuf,
+	.pfnDevPhysAddr = PMRDevPhysAddrDmaBuf,
+	.pfnAcquireKernelMappingData = PMRAcquireKernelMappingDataDmaBuf,
+	.pfnReleaseKernelMappingData = PMRReleaseKernelMappingDataDmaBuf,
+	.pfnMMap = PMRMMapDmaBuf,
+	.pfnFinalize = PMRFinalizeDmaBuf,
+	.pfnGetPMRFactoryLock = PMRFactoryLock,
+	.pfnReleasePMRFactoryLock = PMRFactoryUnlock,
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = PMRZombifyDmaBufMem,
+#endif
+};
+
+/*****************************************************************************
+ *                          Public facing interface                          *
+ *****************************************************************************/
+
+PVRSRV_ERROR
+PhysmemCreateNewDmaBufBackedPMR(PHYS_HEAP *psHeap,
+                                struct dma_buf_attachment *psAttachment,
+                                PFN_DESTROY_DMABUF_PMR pfnDestroy,
+                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                IMG_PID uiPid,
+                                IMG_DEVMEM_SIZE_T uiChunkSize,
+                                IMG_UINT32 ui32NumPhysChunks,
+                                IMG_UINT32 ui32NumVirtChunks,
+                                IMG_UINT32 *pui32MappingTable,
+                                IMG_UINT32 ui32NameSize,
+                                const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                                PMR **ppsPMRPtr)
+{
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+	PMR_DMA_BUF_DATA *psPrivData;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_BOOL bZeroOnAlloc;
+	IMG_BOOL bPoisonOnAlloc;
+	IMG_BOOL bPoisonOnFree;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i, j;
+	IMG_UINT32 uiPagesPerChunk = uiChunkSize >> PAGE_SHIFT;
+	IMG_UINT32 ui32PageCount = 0;
+	struct scatterlist *sg;
+	struct sg_table *table;
+	IMG_UINT32 uiSglOffset;
+	IMG_CHAR pszAnnotation[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_UINT32 ui32ActualDmaBufPageCount;
+
+	bZeroOnAlloc = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags);
+	bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags);
+#if defined(DEBUG)
+	bPoisonOnFree = PVRSRV_CHECK_POISON_ON_FREE(uiFlags);
+#else
+	bPoisonOnFree = IMG_FALSE;
+#endif
+	if (bZeroOnAlloc && bPoisonOnFree)
+	{
+		/* Zero on Alloc and Poison on Alloc are mutually exclusive */
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errReturn;
+	}
+
+	if (!PMRValidateSize((IMG_UINT64) ui32NumVirtChunks * uiChunkSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				 "PMR size exceeds limit #Chunks: %u ChunkSz %"IMG_UINT64_FMTSPECX"",
+				 ui32NumVirtChunks,
+				 uiChunkSize);
+		eError = PVRSRV_ERROR_PMR_TOO_LARGE;
+		goto errReturn;
+	}
+
+	psPrivData = OSAllocZMem(sizeof(*psPrivData));
+	if (psPrivData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto errReturn;
+	}
+
+	psPrivData->psPhysHeap = psHeap;
+	psPrivData->psAttachment = psAttachment;
+	psPrivData->pfnDestroy = pfnDestroy;
+	psPrivData->bPoisonOnFree = bPoisonOnFree;
+	psPrivData->uiOriginPID = uiPid;
+	psPrivData->ui32VirtPageCount =
+			(ui32NumVirtChunks * uiChunkSize) >> PAGE_SHIFT;
+
+	psPrivData->pasDevPhysAddr =
+			OSAllocZMem(sizeof(*(psPrivData->pasDevPhysAddr)) *
+			            psPrivData->ui32VirtPageCount);
+	if (!psPrivData->pasDevPhysAddr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate buffer for physical addresses (oom)",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto errFreePrivData;
+	}
+
+	if (bZeroOnAlloc || bPoisonOnAlloc)
+	{
+		int iValue = bZeroOnAlloc ? 0 : PVRSRV_POISON_ON_ALLOC_VALUE;
+		int err;
+
+		err = DmaBufSetValue(psDmaBuf, iValue, __func__);
+		if (err)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map buffer for %s",
+			                        __func__,
+			                        bZeroOnAlloc ? "zeroing" : "poisoning"));
+
+			eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+			goto errFreePhysAddr;
+		}
+	}
+
+	eError = _pvr_dma_buf_map_attachment(psAttachment, DMA_BIDIRECTIONAL, &table);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_pvr_dma_buf_map_attachment", errFreePhysAddr);
+
+	/*
+	 * We do a two pass process: first work out how many pages there
+	 * are and second, fill in the data.
+	 */
+	for_each_sg(table->sgl, sg, table->nents, i)
+	{
+		ui32PageCount += PAGE_ALIGN(pvr_sg_length(sg)) / PAGE_SIZE;
+	}
+
+	if (WARN_ON(!ui32PageCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of phys. pages must not be zero",
+				 __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errUnmap;
+	}
+
+	/* Obtain actual page count of dma buf */
+	ui32ActualDmaBufPageCount = psAttachment->dmabuf->size / PAGE_SIZE;
+
+	if (WARN_ON(ui32ActualDmaBufPageCount < ui32NumPhysChunks * uiPagesPerChunk))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Requested physical chunks greater than "
+				"number of physical dma buf pages",
+				 __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errUnmap;
+	}
+
+	psPrivData->ui32PhysPageCount = ui32ActualDmaBufPageCount;
+	psPrivData->psSgTable = table;
+	ui32PageCount = 0;
+	sg = table->sgl;
+	uiSglOffset = 0;
+
+
+	/* Fill physical address array */
+	for (i = 0; i < ui32NumPhysChunks; i++)
+	{
+		for (j = 0; j < uiPagesPerChunk; j++)
+		{
+			IMG_UINT32 uiIdx = pui32MappingTable[i] * uiPagesPerChunk + j;
+
+			psPrivData->pasDevPhysAddr[uiIdx].uiAddr =
+					sg_dma_address(sg) + uiSglOffset;
+
+			/* Get the next offset for the current sgl or the next sgl */
+			uiSglOffset += PAGE_SIZE;
+			if (uiSglOffset >= pvr_sg_length(sg))
+			{
+				sg = sg_next(sg);
+				uiSglOffset = 0;
+
+				/* Check that we haven't looped */
+				if (WARN_ON(sg == table->sgl))
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to fill phys. address "
+							"array",
+							 __func__));
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					goto errUnmap;
+				}
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+#endif
+
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+
+	/*
+	 * Check no significant bits were lost in cast due to different
+	 * bit widths for flags
+	 */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (OSSNPrintf((IMG_CHAR *)pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN, "ImpDmaBuf:%s", (IMG_CHAR *)pszName) < 0)
+	{
+		pszAnnotation[0] = '\0';
+	}
+	else
+	{
+		pszAnnotation[DEVMEM_ANNOTATION_MAX_LEN-1] = '\0';
+	}
+
+	eError = PMRCreatePMR(psHeap,
+			      ui32NumVirtChunks * uiChunkSize,
+			      ui32NumPhysChunks,
+			      ui32NumVirtChunks,
+			      pui32MappingTable,
+			      PAGE_SHIFT,
+			      uiPMRFlags,
+			      pszAnnotation,
+			      &_sPMRDmaBufFuncTab,
+			      psPrivData,
+			      PMR_TYPE_DMABUF,
+			      ppsPMRPtr,
+			      PDUMP_NONE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create PMR (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		goto errFreePhysAddr;
+	}
+
+	return PVRSRV_OK;
+
+errUnmap:
+	_pvr_dma_buf_unmap_attachment(psAttachment, table, DMA_BIDIRECTIONAL);
+errFreePhysAddr:
+	OSFreeMem(psPrivData->pasDevPhysAddr);
+errFreePrivData:
+	OSFreeMem(psPrivData);
+errReturn:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PhysmemDestroyDmaBuf(PHYS_HEAP *psHeap,
+                                 struct dma_buf_attachment *psAttachment)
+{
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+
+	PVR_UNREFERENCED_PARAMETER(psHeap);
+
+	/* PMRUnlockSysPhysAddresses(psPMR) is redundant.
+	 * See PMRUnlockPhysAddressesDmaBuf */
+
+	dma_buf_detach(psDmaBuf, psAttachment);
+	dma_buf_put(psDmaBuf);
+}
+
+struct dma_buf *
+PhysmemGetDmaBuf(PMR *psPMR)
+{
+	PMR_DMA_BUF_DATA *psPrivData;
+
+	psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+	if (psPrivData)
+	{
+		return psPrivData->psAttachment->dmabuf;
+	}
+
+	return NULL;
+}
+
+struct dma_resv *
+PhysmemGetDmaResv(PMR *psPMR)
+{
+	struct dma_buf *psDmaBuf = PhysmemGetDmaBuf(psPMR);
+
+	if (psDmaBuf)
+	{
+		return psDmaBuf->resv;
+	}
+	else
+	{
+		PMR_DMA_BUF_WRAPPER *psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+
+		if (psPMRWrapper)
+		{
+			return &psPMRWrapper->sDmaResv;
+
+		}
+
+		return NULL;
+	}
+}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+struct dma_heap *
+PhysmemGetDmaHeap(PMR *psPMR)
+{
+	struct dma_buf *psDmaBuf = PhysmemGetDmaBuf(psPMR);
+
+	psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+	if (psPrivData != NULL)
+	{
+		return psPrivData->psDmaHeap;
+	}
+
+	return NULL;
+}
+
+void
+PhysmemSetDmaHeap(PMR *psPMR, struct dma_heap *psDmaHeap)
+{
+	struct dma_buf *psDmaBuf = PhysmemGetDmaBuf(psPMR);
+
+	psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+	if (psPrivData != NULL)
+	{
+		psPrivData->psDmaHeap = psDmaHeap;
+	}
+}
+#endif /* #if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND) */
+
+static void
+PhysmemDestroyPMRWrapper(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	dma_resv_fini(&psPMRWrapper->sDmaResv);
+
+	OSLockDestroy(psPMRWrapper->hKernelMappingLock);
+	OSLockDestroy(psPMRWrapper->hDeviceMappingLock);
+	OSFreeMem(psPMRWrapper);
+}
+
+void
+PhysmemDmaBufExportFinalize(void *pvDmaBufExportData)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = pvDmaBufExportData;
+
+	PhysmemDestroyPMRWrapper(psPMRWrapper);
+}
+
+static PVRSRV_ERROR
+PhysmemCreatePMRWrapper(PMR *psPMR, PMR_DMA_BUF_WRAPPER **ppsPMRWrapper)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	psPMRWrapper = OSAllocZMem(sizeof(*psPMRWrapper));
+	PVR_LOG_GOTO_IF_NOMEM(psPMRWrapper, eError, fail_alloc_mem);
+
+	psPMRWrapper->psPMR = psPMR;
+
+	eError = OSLockCreate(&psPMRWrapper->hKernelMappingLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.1",
+	                      fail_kernel_mapping_lock_create);
+
+	eError = OSLockCreate(&psPMRWrapper->hDeviceMappingLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.2",
+	                      fail_device_mapping_lock_create);
+
+	dma_resv_init(&psPMRWrapper->sDmaResv);
+
+	*ppsPMRWrapper = psPMRWrapper;
+
+	return PVRSRV_OK;
+fail_device_mapping_lock_create:
+	OSLockDestroy(psPMRWrapper->hKernelMappingLock);
+fail_kernel_mapping_lock_create:
+	OSFreeMem(psPMRWrapper);
+fail_alloc_mem:
+	return eError;
+}
+
+static PVRSRV_ERROR
+PhysmemGetOrCreatePMRWrapper(PMR *psPMR, PMR_DMA_BUF_WRAPPER **ppsPMRWrapper)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+	if (!psPMRWrapper)
+	{
+		PMRFactoryLock();
+
+		/* Check again with the factory lock held */
+		psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+		if (!psPMRWrapper)
+		{
+			eError = PhysmemCreatePMRWrapper(psPMR, &psPMRWrapper);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreatePMRWrapper", fail_create_pmr_wrapper);
+
+			/*
+			 * A PMR memory layout can't change once exported.
+			 * This makes sure the exported and imported parties
+			 * see the same layout of the memory.
+			 */
+			PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+			PMREnvDmaBufSetExportData(psPMR, psPMRWrapper);
+		}
+
+		PMRFactoryUnlock();
+	}
+
+	*ppsPMRWrapper = psPMRWrapper;
+
+	return PVRSRV_OK;
+
+fail_create_pmr_wrapper:
+	PMRFactoryUnlock();
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemExportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    PMR *psPMR,
+                    IMG_INT *piFd)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+
+	eError = PhysmemGetOrCreatePMRWrapper(psPMR, &psPMRWrapper);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemExportDmaBuf", fail_get_pmr_wrapper);
+
+	eError = PMRRefPMR(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", fail_ref_pmr);
+
+	{
+		DEFINE_DMA_BUF_EXPORT_INFO(sDmaBufExportInfo);
+
+		sDmaBufExportInfo.priv  = psPMRWrapper;
+		sDmaBufExportInfo.ops   = &sPVRDmaBufOps;
+		sDmaBufExportInfo.size  = PMR_LogicalSize(psPMR);
+		sDmaBufExportInfo.flags = O_RDWR;
+		sDmaBufExportInfo.resv  = &psPMRWrapper->sDmaResv;
+
+		psDmaBuf = dma_buf_export(&sDmaBufExportInfo);
+	}
+
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to export buffer (err=%ld)",
+		         __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_export;
+	}
+
+	iFd = dma_buf_fd(psDmaBuf, O_RDWR);
+	if (iFd < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf fd (err=%d)",
+		         __func__, iFd));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_dma_buf;
+	}
+
+	*piFd = iFd;
+
+	return PVRSRV_OK;
+
+fail_dma_buf:
+	dma_buf_put(psDmaBuf);
+	return eError;
+
+fail_export:
+	(void) PMRUnrefPMR(psPMR);
+
+fail_ref_pmr:
+	/* No need to destroy psPMRWrapper here. It will be (or already was)
+	 * destroyed alongside the PMR. */
+fail_get_pmr_wrapper:
+	return eError;
+}
+
+struct dma_buf *
+PhysmemGEMPrimeExport(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+			struct drm_device *psDev,
+#endif
+
+			struct drm_gem_object *psObj,
+			int iFlags)
+{
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	struct dma_buf *psDmaBuf = NULL;
+	DEFINE_DMA_BUF_EXPORT_INFO(sDmaBufExportInfo);
+
+
+	sDmaBufExportInfo.priv  = psObj;
+	sDmaBufExportInfo.ops   = &sPVRDmaBufOpsGEM;
+	sDmaBufExportInfo.size  = PMR_LogicalSize(psPMRWrapper->psPMR);
+	sDmaBufExportInfo.flags = iFlags;
+	sDmaBufExportInfo.resv  = &psPMRWrapper->sDmaResv;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	psDmaBuf = drm_gem_dmabuf_export(psObj->dev, &sDmaBufExportInfo);
+#else
+	psDmaBuf = drm_gem_dmabuf_export(psDev, &sDmaBufExportInfo);
+#endif
+	if (IS_ERR(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to export buffer (err=%ld)",
+		         __func__, PTR_ERR(psDmaBuf)));
+	}
+
+	return psDmaBuf;
+}
+
+void
+PhysmemGEMObjectFree(struct drm_gem_object *psObj)
+{
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	drm_gem_object_release(psObj);
+
+	if (smp_load_acquire(&psPMRWrapper->psObj) == psObj)
+	{
+		smp_store_release(&psPMRWrapper->psObj, NULL);
+	}
+
+	eError = PMRUnrefPMR(psPMRWrapper->psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+
+	OSFreeMem(psGEMObj);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static const struct drm_gem_object_funcs sPhysmemGEMObjFuncs = {
+	.export = PhysmemGEMPrimeExport,
+	.free = PhysmemGEMObjectFree,
+};
+#endif
+
+PVRSRV_ERROR
+PhysmemExportGemHandle(CONNECTION_DATA *psConnection,
+		       PVRSRV_DEVICE_NODE *psDevNode,
+		       PMR *psPMR,
+		       IMG_UINT32 *puHandle)
+{
+	struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+	struct drm_device *psDRMDev = dev_get_drvdata(psDev);
+	struct drm_file *psDRMFile = OSGetDRMFile(psConnection);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj;
+	bool bAlreadyExported;
+	PVRSRV_ERROR eError;
+	int iErr;
+
+	eError = PhysmemGetOrCreatePMRWrapper(psPMR, &psPMRWrapper);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemExportGemHandle", fail_get_pmr_wrapper);
+
+	psGEMObj = OSAllocZMem(sizeof(*psGEMObj));
+	PVR_LOG_GOTO_IF_NOMEM(psGEMObj, eError, fail_alloc_mem);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	psGEMObj->sBase.funcs = &sPhysmemGEMObjFuncs;
+#endif
+	psGEMObj->psPMRWrapper = psPMRWrapper;
+
+	eError = PMRRefPMR(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", fail_ref_pmr);
+
+	drm_gem_private_object_init(psDRMDev, &psGEMObj->sBase,
+	                            PMR_LogicalSize(psPMR));
+
+	PMRFactoryLock();
+	bAlreadyExported = smp_load_acquire(&psPMRWrapper->psObj) != NULL;
+	if (!bAlreadyExported)
+	{
+		smp_store_release(&psPMRWrapper->psObj, &psGEMObj->sBase);
+	}
+	PMRFactoryUnlock();
+
+	if (bAlreadyExported)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_STILL_REFERENCED,
+		                    fail_export_check);
+	}
+
+	iErr = drm_gem_handle_create(psDRMFile, &psGEMObj->sBase, puHandle);
+	if (iErr)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY,
+		                    fail_handle_create);
+	}
+
+	/* The handle holds a reference on the object, so drop ours */
+	drm_gem_object_put(&psGEMObj->sBase);
+
+	return PVRSRV_OK;
+
+fail_export_check:
+fail_handle_create:
+	/* Decrement reference on the GEM object and call .free()
+	 * (PhysmemGEMObjectFree()). This will unreference the PMR and call
+	 * OSMemFree() on psGEMObj. */
+	drm_gem_object_put(&psGEMObj->sBase);
+	return eError;
+
+fail_ref_pmr:
+	OSFreeMem(psGEMObj);
+fail_alloc_mem:
+fail_get_pmr_wrapper:
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemImportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_INT fd,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    IMG_UINT32 ui32NameSize,
+                    const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                    PMR **ppsPMRPtr,
+                    IMG_DEVMEM_SIZE_T *puiSize,
+                    IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 ui32MappingTable = 0;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+
+	/* Get the buffer handle */
+	psDmaBuf = dma_buf_get(fd);
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf from fd (err=%ld)",
+				 __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		return PVRSRV_ERROR_BAD_MAPPING;
+
+	}
+
+	uiSize = psDmaBuf->size;
+
+	eError = PhysmemImportSparseDmaBuf(psConnection,
+	                                 psDevNode,
+	                                 fd,
+	                                 uiFlags,
+	                                 uiSize,
+	                                 1,
+	                                 1,
+	                                 &ui32MappingTable,
+	                                 ui32NameSize,
+	                                 pszName,
+	                                 ppsPMRPtr,
+	                                 puiSize,
+	                                 puiAlign);
+
+	dma_buf_put(psDmaBuf);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemImportSparseDmaBuf(CONNECTION_DATA *psConnection,
+                          PVRSRV_DEVICE_NODE *psDevNode,
+                          IMG_INT fd,
+                          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                          IMG_DEVMEM_SIZE_T uiChunkSize,
+                          IMG_UINT32 ui32NumPhysChunks,
+                          IMG_UINT32 ui32NumVirtChunks,
+                          IMG_UINT32 *pui32MappingTable,
+                          IMG_UINT32 ui32NameSize,
+                          const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                          PMR **ppsPMRPtr,
+                          IMG_DEVMEM_SIZE_T *puiSize,
+                          IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PMR *psPMR = NULL;
+	struct dma_buf_attachment *psAttachment;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bHashTableCreated = IMG_FALSE;
+	IMG_PID uiPid = OSGetCurrentClientProcessIDKM();
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode != NULL, eError, errReturn);
+
+	/* Terminate string from bridge to prevent corrupt annotations in RI */
+	if (pszName != NULL)
+	{
+		IMG_CHAR* pszName0 = (IMG_CHAR*) pszName;
+		pszName0[ui32NameSize-1] = '\0';
+	}
+
+	PMRFactoryLock();
+
+	/* Get the buffer handle */
+	psDmaBuf = dma_buf_get(fd);
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf from fd (err=%ld)",
+		        __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, errUnlockReturn);
+	}
+
+	if (psDmaBuf->ops == &sPVRDmaBufOps || psDmaBuf->ops == &sPVRDmaBufOpsGEM)
+	{
+		/* We exported this dma_buf, so we can just get its PMR. */
+		if (psDmaBuf->ops == &sPVRDmaBufOps)
+		{
+			PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+			psPMR = psPMRWrapper->psPMR;
+		}
+		else
+		{
+			struct drm_gem_object *psObj = psDmaBuf->priv;
+			PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+			PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+			psPMR = psPMRWrapper->psPMR;
+		}
+
+		/* However, we can't import it if it belongs to a different device. */
+		if (PMR_DeviceNode(psPMR) != psDevNode)
+		{
+#if defined(PVRSRV_ENABLE_XD_MEM)
+			/* and if it's not in a shareable physical address space. */
+			if (PhysHeapSpasWithDevice(PMR_PhysHeap(psPMR), psDevNode) != PVRSRV_OK)
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PMR invalid for this device",
+				        __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PERMITTED,
+				                    errUnlockAndDMAPut);
+			}
+		}
+	}
+	else if (g_psDmaBufHash != NULL)
+	{
+		/* We have a hash table so check if we've seen this dmabuf
+		 * before. */
+		psPMR = (PMR *) HASH_Retrieve(g_psDmaBufHash, (uintptr_t) psDmaBuf);
+	}
+	else
+	{
+		/* As different processes may import the same dmabuf we need to
+		 * create a hash table so we don't generate a duplicate PMR but
+		 * rather just take a reference on an existing one. */
+		g_psDmaBufHash = HASH_Create(DMA_BUF_HASH_SIZE);
+		PVR_GOTO_IF_NOMEM(g_psDmaBufHash, eError, errUnlockAndDMAPut);
+
+		bHashTableCreated = IMG_TRUE;
+	}
+
+	if (psPMR != NULL)
+	{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		if (PMR_IsZombie(psPMR))
+		{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			PMR_DMA_BUF_DATA *psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+#endif
+
+			PMRReviveZombieAndRef(psPMR);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			if (psPrivData != NULL)
+			{
+				PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+				                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+				                            psPrivData->uiOriginPID);
+				PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+				                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+				                            psPrivData->uiOriginPID);
+			}
+#endif
+
+			PVRSRVIonReviveMemAllocRecord(psDmaBuf);
+		}
+		else
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+		{
+			/* Reuse the PMR we already created */
+			eError = PMRRefPMR(psPMR);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRRefPMR", errUnlockAndDMAPut);
+		}
+
+		/* If an existing PMR is found, the table wasn't created by this func
+		 * call, so the error path can be safely ignored allowing for
+		 * the factory lock can be dropped. */
+		PVR_ASSERT(bHashTableCreated == IMG_FALSE);
+		dma_buf_put(psDmaBuf);
+		PMRFactoryUnlock();
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+		/* The device import can only be registered on an alive & healthy PMR
+		 * therefore we wait for the potential zombie to be dequeued first. */
+		eError = PMR_RegisterDeviceImport(psPMR, psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			/* The factory lock might be taken in PMRUnrefPMR. */
+			(void) PMRUnrefPMR(psPMR);
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to register PMR with device: %u",
+			        __func__, psDevNode->sDevId.ui32InternalID));
+			goto errReturn;
+		}
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+		*ppsPMRPtr = psPMR;
+		*puiSize = PMR_LogicalSize(psPMR);
+		*puiAlign = PAGE_SIZE;
+
+		/* We expect a PMR to be immutable at this point.
+		 * But its explicitly set here to cover a corner case
+		 * where a PMR created through non-DMA interface could be
+		 * imported back again through DMA interface. */
+		PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+		return PVRSRV_OK;
+	}
+
+	/* Do we want this to be a sparse PMR? */
+	if (ui32NumVirtChunks > 1)
+	{
+		/* Parameter validation */
+		if (psDmaBuf->size < (uiChunkSize * ui32NumPhysChunks) ||
+		    uiChunkSize != PAGE_SIZE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Requesting sparse buffer: "
+					"uiChunkSize ("IMG_DEVMEM_SIZE_FMTSPEC") must be equal to "
+					"OS page size (%lu). uiChunkSize * ui32NumPhysChunks "
+					"("IMG_DEVMEM_SIZE_FMTSPEC") must"
+					" not be greater than the buffer size ("IMG_SIZE_FMTSPEC").",
+					 __func__,
+					uiChunkSize,
+					PAGE_SIZE,
+					uiChunkSize * ui32NumPhysChunks,
+					psDmaBuf->size));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS,
+			                    errUnlockAndDMAPut);
+		}
+	}
+	else
+	{
+		/* if ui32NumPhysChunks == 0 then pui32MappingTable == NULL
+		 * this is handled by the generated bridge code.
+		 * Because ui32NumPhysChunks is set to 1 below, we don't allow NULL array */
+		if (pui32MappingTable == NULL)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS,
+			                    errUnlockAndDMAPut);
+		}
+
+		/* Make sure parameters are valid for non-sparse allocations as well */
+		uiChunkSize = psDmaBuf->size;
+		ui32NumPhysChunks = 1;
+		ui32NumVirtChunks = 1;
+	}
+
+	{
+		IMG_DEVMEM_SIZE_T uiSize = ui32NumVirtChunks * uiChunkSize;
+		IMG_UINT32 uiLog2PageSize = PAGE_SHIFT; /* log2(uiChunkSize) */
+		eError = PhysMemValidateParams(psDevNode,
+		                               ui32NumPhysChunks,
+		                               ui32NumVirtChunks,
+		                               pui32MappingTable,
+		                               uiFlags,
+		                               uiPid,
+		                               &uiLog2PageSize,
+		                               &uiSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysMemValidateParams", errUnlockAndDMAPut);
+	}
+
+	psAttachment = dma_buf_attach(psDmaBuf, psDevNode->psDevConfig->pvOSDevice);
+	if (IS_ERR_OR_NULL(psAttachment))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to attach to dma-buf (err=%ld)",
+				 __func__, psAttachment? PTR_ERR(psAttachment) : -ENOMEM));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DMABUF_ATTACH,
+		                    errUnlockAndDMAPut);
+	}
+
+	/*
+	 * Note:
+	 * While we have no way to determine the type of the buffer we just
+	 * assume that all dmabufs are from the same physical heap.
+	 */
+	eError = PhysmemCreateNewDmaBufBackedPMR(psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_EXTERNAL],
+	                                         psAttachment,
+	                                         PhysmemDestroyDmaBuf,
+	                                         uiFlags,
+	                                         uiPid,
+	                                         uiChunkSize,
+	                                         ui32NumPhysChunks,
+	                                         ui32NumVirtChunks,
+	                                         pui32MappingTable,
+	                                         ui32NameSize,
+	                                         pszName,
+	                                         &psPMR);
+	PVR_GOTO_IF_ERROR(eError, errDMADetach);
+
+	/* The imported memory is assumed to be backed by a permanent allocation. */
+	/* PMRLockSysPhysAddresses(psPMR) is redundant.
+	 * See PMRLockPhysAddressesDmaBuf */
+
+	/* First time we've seen this dmabuf so store it in the hash table */
+	HASH_Insert(g_psDmaBufHash, (uintptr_t) psDmaBuf, (uintptr_t) psPMR);
+	g_ui32HashRefCount++;
+
+	PMRFactoryUnlock();
+
+	PVRSRVIonAddMemAllocRecord(psDmaBuf);
+
+	*ppsPMRPtr = psPMR;
+	*puiSize = ui32NumVirtChunks * uiChunkSize;
+	*puiAlign = PAGE_SIZE;
+
+	/* The memory that's just imported is owned by some other entity.
+	 * Hence the memory layout cannot be changed through our API */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	return PVRSRV_OK;
+
+errDMADetach:
+	dma_buf_detach(psDmaBuf, psAttachment);
+
+errUnlockAndDMAPut:
+	if (bHashTableCreated)
+	{
+		HASH_Delete(g_psDmaBufHash);
+		g_psDmaBufHash = NULL;
+	}
+	dma_buf_put(psDmaBuf);
+
+errUnlockReturn:
+	PMRFactoryUnlock();
+
+errReturn:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf_internal.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf_internal.h
new file mode 100644
index 000000000000..50a8ab37f442
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_dmabuf_internal.h
@@ -0,0 +1,69 @@
+/**************************************************************************/ /*!
+@File           physmem_dmabuf_internal.h
+@Title          Internal header for dmabuf PMR factory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks importing Ion allocations
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PHYSMEM_DMABUF_INTERNAL_H)
+#define PHYSMEM_DMABUF_INTERNAL_H
+
+#include <linux/version.h>
+
+struct dma_buf;
+struct drm_device;
+struct drm_gem_object;
+
+struct dma_buf *
+PhysmemGEMPrimeExport(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+			struct drm_device *psDev,
+#endif
+
+			struct drm_gem_object *psObj,
+			int iFlags);
+
+void
+PhysmemGEMObjectFree(struct drm_gem_object *psObj);
+
+void
+PhysmemDmaBufExportFinalize(void *pvDmaBufExportData);
+
+#endif /* !defined(PHYSMEM_DMABUF_INTERNAL_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_linux.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_linux.c
new file mode 100644
index 000000000000..994f675329b8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_linux.c
@@ -0,0 +1,1077 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR functions to wrap non-services allocated
+                memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                implementing the function callbacks for physical memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/uaccess.h>
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "devicemem_server_utils.h"
+
+#include "physmem_extmem.h"
+#include "physmem_extmem_wrap.h"
+#include "pdump_physmem.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "physmem.h"
+#include "cache_km.h"
+
+#include "kernel_compatibility.h"
+
+typedef struct _WRAP_KERNEL_MAP_DATA_
+{
+	void *pvKernLinAddr;
+	IMG_BOOL bVMAP;
+} WRAP_KERNEL_MAP_DATA;
+
+/* Free the PMR private data */
+static void _FreeWrapData(PMR_WRAP_DATA *psPrivData)
+{
+	OSFreeMem(psPrivData->ppsPageArray);
+	OSFreeMem(psPrivData->ppvPhysAddr);
+	psPrivData->psVMArea = NULL;
+	OSFreeMem(psPrivData);
+}
+
+
+/* Allocate the PMR private data */
+static PVRSRV_ERROR _AllocWrapData(PMR_WRAP_DATA **ppsPrivData,
+                            PVRSRV_DEVICE_NODE *psDevNode,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_CPU_VIRTADDR pvCpuVAddr,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_WRAP_DATA *psPrivData;
+	struct vm_area_struct *psVMArea;
+	IMG_UINT32 ui32CPUCacheMode;
+
+	/* Obtain a reader lock on the process mmap lock while we are using
+	 * the psVMArea.
+	 */
+
+	mmap_read_lock(current->mm);
+
+	/* Find the VMA */
+	psVMArea = find_vma(current->mm, (uintptr_t)pvCpuVAddr);
+	if (psVMArea == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Couldn't find memory region containing start address %p",
+				__func__,
+				(void*) pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto eUnlockReturn;
+	}
+
+	/* If requested size is larger than actual allocation
+	 * return error. Can never request more memory to be imported
+	 * than its original allocation */
+	/* Now check the end address is in range */
+	if (((uintptr_t)pvCpuVAddr + uiSize) > psVMArea->vm_end)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: End address %p is outside of the region returned by find_vma",
+				__func__,
+				(void*) (uintptr_t)((uintptr_t)pvCpuVAddr + uiSize)));
+		eError = PVRSRV_ERROR_BAD_PARAM_SIZE;
+		goto eUnlockReturn;
+	}
+
+	/* Find_vma locates a region with an end point past a given
+	 * virtual address. So check the address is actually in the region. */
+	if ((uintptr_t)pvCpuVAddr < psVMArea->vm_start)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Start address %p is outside of the region returned by find_vma",
+				__func__,
+				(void*) pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto eUnlockReturn;
+	}
+
+	eError = DevmemCPUCacheMode(uiFlags, &ui32CPUCacheMode);
+	if (eError != PVRSRV_OK)
+	{
+		goto eUnlockReturn;
+	}
+
+	/* Allocate and initialise private factory data */
+	psPrivData = OSAllocZMem(sizeof(*psPrivData));
+	if (psPrivData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	psPrivData->ui32CPUCacheFlags = ui32CPUCacheMode;
+
+	/* Track the VMA area structure so that it can be checked later */
+	psPrivData->psVMArea = psVMArea;
+
+	psPrivData->psDevNode = psDevNode;
+	psPrivData->uiTotalNumPages = uiSize >> PAGE_SHIFT;
+
+	/* Allocate page and phys address arrays */
+	psPrivData->ppsPageArray = OSAllocZMem(sizeof(*(psPrivData->ppsPageArray)) * psPrivData->uiTotalNumPages);
+	if (psPrivData == NULL)
+	{
+		OSFreeMem(psPrivData);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	psPrivData->ppvPhysAddr = OSAllocZMem(sizeof(*(psPrivData->ppvPhysAddr)) * psPrivData->uiTotalNumPages);
+	if (psPrivData->ppvPhysAddr == NULL)
+	{
+		OSFreeMem(psPrivData->ppsPageArray);
+		OSFreeMem(psPrivData);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	if (uiFlags & (PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+					PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED))
+	{
+		psPrivData->bWrite = IMG_TRUE;
+	}
+
+	*ppsPrivData = psPrivData;
+
+	eError = PVRSRV_OK;
+eUnlockReturn:
+	mmap_read_unlock(current->mm);
+
+	return eError;
+}
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+/* Free the pages we got via _TryFindVMA() */
+static void _FreeFindVMAPages(PMR_WRAP_DATA *psPrivData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i] != NULL)
+		{
+			/* Release the page */
+			put_page(psPrivData->ppsPageArray[i]);
+		}
+	}
+}
+#endif
+
+/* Release pages got via get_user_pages().
+ * As the function comment for GUP states we have
+ * to set the page dirty if it has been written to and
+ * we have to put the page to decrease its refcount. */
+static void _FreeGetUserPages(PMR_WRAP_DATA *psPrivData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i])
+		{
+			if (IMG_TRUE == psPrivData->bWrite)
+			{
+				/* Write data back to fs if necessary */
+				set_page_dirty_lock(psPrivData->ppsPageArray[i]);
+			}
+
+			/* Release the page */
+			put_page(psPrivData->ppsPageArray[i]);
+		}
+	}
+}
+
+/* Get the page structures and physical addresses mapped to
+ * a CPU virtual range via get_user_pages() */
+static PVRSRV_ERROR _TryGetUserPages(PVRSRV_DEVICE_NODE *psDevNode,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_CPU_VIRTADDR pvCpuVAddr,
+                                PMR_WRAP_DATA *psPrivData)
+{
+	IMG_INT32 iMappedPages, i;
+
+	IMG_UINT64 ui64DmaMask = dma_get_mask(psDevNode->psDevConfig->pvOSDevice);
+
+	/* Do the actual call */
+	iMappedPages = get_user_pages_fast((uintptr_t) pvCpuVAddr, uiSize >> PAGE_SHIFT,
+	                                    psPrivData->bWrite, psPrivData->ppsPageArray);
+	if (iMappedPages < 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "get_user_pages_fast() failed, got back %d, expected num pages %d",
+		         iMappedPages,
+		         psPrivData->uiTotalNumPages));
+
+		return PVRSRV_ERROR_FAILED_TO_ACQUIRE_PAGES;
+	}
+
+	/* Fill the physical address array */
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i])
+		{
+			psPrivData->ppvPhysAddr[i].uiAddr = page_to_phys(psPrivData->ppsPageArray[i]);
+			psPrivData->uiNumBackedPages += 1;
+		}
+		else
+		{
+			psPrivData->ppvPhysAddr[i].uiAddr = 0;
+		}
+
+
+		/* Check the data transfer capability of the DMA.
+		 *
+		 * For instance:
+		 * APOLLO test chips TCF5 or ES2 can only access 4G maximum memory from the card.
+		 * Hence pages with a physical address beyond 4G range cannot be accessed by the
+		 * device. An error is reported in such a case. */
+		if (psPrivData->ppvPhysAddr[i].uiAddr & ~ui64DmaMask)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Backed page @pos:%d Physical Address: %pa exceeds GPU "
+					"accessible range(mask): 0x%0llx",
+					i, &psPrivData->ppvPhysAddr[i].uiAddr, ui64DmaMask));
+
+			psPrivData->bWrite = IMG_FALSE;
+
+			/* Free the acquired pages */
+			_FreeGetUserPages(psPrivData);
+
+			return PVRSRV_ERROR_INVALID_PHYS_ADDR;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Release all the pages we got via _TryGetUserPages or _TryFindVMA */
+static PVRSRV_ERROR _WrapExtMemReleasePages(PMR_WRAP_DATA *psPrivData)
+{
+	switch (psPrivData->eWrapType)
+	{
+		case WRAP_TYPE_GET_USER_PAGES:
+			_FreeGetUserPages(psPrivData);
+			break;
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+		case WRAP_TYPE_FIND_VMA:
+			_FreeFindVMAPages(psPrivData);
+			break;
+#endif
+		case WRAP_TYPE_NULL:
+			/* fall through */
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Wrong wrap type, cannot release pages",
+					__func__));
+			return PVRSRV_ERROR_INVALID_WRAP_TYPE;
+	}
+
+	_FreeWrapData(psPrivData);
+
+	return PVRSRV_OK;
+}
+
+/* Try to get pages and physical addresses for a CPU virtual address.
+ * Will try both methods:
+ * get_user_pages or find_vma + page table walk if the first one fails */
+static PVRSRV_ERROR _WrapExtMemAcquirePages(PVRSRV_DEVICE_NODE *psDevNode,
+                                            IMG_DEVMEM_SIZE_T uiSize,
+                                            IMG_CPU_VIRTADDR pvCpuVAddr,
+                                            PMR_WRAP_DATA *psPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	eError = _TryGetUserPages(psDevNode,
+	                            uiSize,
+	                            pvCpuVAddr,
+	                            psPrivData);
+	if (eError == PVRSRV_OK)
+	{
+		psPrivData->eWrapType = WRAP_TYPE_GET_USER_PAGES;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"%s: Used GetUserPages",
+				__func__));
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+	if (PVRSRV_ERROR_INVALID_PHYS_ADDR != eError)
+	{
+		PVRSRV_ERROR eError2;
+		eError2 = _TryFindVMA(uiSize,
+		                      (uintptr_t) pvCpuVAddr,
+		                      psPrivData);
+		if (eError2 == PVRSRV_OK)
+		{
+			psPrivData->eWrapType = WRAP_TYPE_FIND_VMA;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Used FindVMA",
+					__func__));
+			return PVRSRV_OK;
+		}
+
+		PVR_WARN_IF_ERROR(eError, "_TryGetUserPages");
+		PVR_LOG_ERROR(eError2, "_TryFindVMA");
+	}
+	else
+	{
+		PVR_LOG_ERROR(eError, "_TryGetUserPages");
+	}
+#else
+	PVR_LOG_ERROR(eError, "_TryGetUserPages");
+#endif
+
+	psPrivData->eWrapType = WRAP_TYPE_NULL;
+	return eError;
+}
+
+static PVRSRV_ERROR
+PMRSysPhysAddrExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                     IMG_UINT32 ui32Log2PageSize,
+                     IMG_UINT32 ui32NumOfPages,
+                     IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+                     IMG_UINT64 ui64IPAPolicyValue,
+                     IMG_UINT64 ui64IPAClearMask,
+#endif
+                     IMG_BOOL *pbValid,
+                     IMG_DEV_PHYADDR *psDevPAddr)
+{
+	const PMR_WRAP_DATA *psWrapData = pvPriv;
+	IMG_UINT32 uiPageSize = 1U << PAGE_SHIFT;
+	IMG_UINT32 uiInPageOffset;
+	IMG_UINT32 uiPageIndex;
+	IMG_UINT32 uiIdx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+
+	if (PAGE_SHIFT != ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Requested ui32Log2PageSize %u is different from "
+				"OS page shift %u. Not supported.",
+				__func__,
+				ui32Log2PageSize,
+				PAGE_SHIFT));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++)
+	{
+		uiPageIndex = puiOffset[uiIdx] >> PAGE_SHIFT;
+		uiInPageOffset = puiOffset[uiIdx] - ((IMG_DEVMEM_OFFSET_T)uiPageIndex << PAGE_SHIFT);
+
+		PVR_LOG_RETURN_IF_FALSE(uiPageIndex < psWrapData->uiTotalNumPages,
+		                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+		PVR_ASSERT(uiInPageOffset < uiPageSize);
+
+		/* The ExtMem is only enabled in UMA mode, in that mode, the device physical
+		 * address translation will be handled by the PRM factory after this call.
+		 * Here we just copy the physical address like other callback implementations. */
+		psDevPAddr[uiIdx].uiAddr = psWrapData->ppvPhysAddr[uiPageIndex].uiAddr;
+
+		pbValid[uiIdx] = (psDevPAddr[uiIdx].uiAddr)? IMG_TRUE:IMG_FALSE;
+
+		psDevPAddr[uiIdx].uiAddr += uiInPageOffset;
+#if defined(SUPPORT_STATIC_IPA)
+		psDevPAddr[uiIdx].uiAddr &= ~ui64IPAClearMask;
+		psDevPAddr[uiIdx].uiAddr |= ui64IPAPolicyValue;
+#endif	/* SUPPORT_STATIC_IPA */
+	}
+
+	return PVRSRV_OK;
+}
+
+
+static void
+PMRFinalizeExtMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PMR_WRAP_DATA *psWrapData = pvPriv;
+
+	PVRSRV_ERROR eError = _WrapExtMemReleasePages(psWrapData);
+	PVR_LOG_IF_ERROR(eError, "_WrapExtMemReleasePages");
+}
+
+
+static void _UnmapPage(PMR_WRAP_DATA *psWrapData,
+                       WRAP_KERNEL_MAP_DATA *psMapData)
+{
+	IMG_BOOL bSuccess;
+
+	if (psMapData->bVMAP)
+	{
+		vunmap(psMapData->pvKernLinAddr);
+	}
+	else
+	{
+		if (psWrapData->ui32CPUCacheFlags & ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unable to unmap wrapped extmem "
+			        "page - wrong cached mode flags passed. This may leak "
+			        "memory.", __func__));
+			PVR_DPF((PVR_DBG_ERROR, "Found non-cpu cache mode flag when unmapping from "
+					 "the cpu"));
+		}
+		else
+		{
+			bSuccess = OSUnMapPhysToLin(psMapData->pvKernLinAddr, PAGE_SIZE);
+			if (!bSuccess)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Unable to unmap wrapped extmem "
+				        "page. This may leak memory.", __func__));
+			}
+		}
+	}
+}
+
+static void _MapPage(PMR_WRAP_DATA *psWrapData,
+                     IMG_UINT32 uiPageIdx,
+                     WRAP_KERNEL_MAP_DATA *psMapData)
+{
+	IMG_UINT8 *puiLinCPUAddr;
+
+	if (psWrapData->ppsPageArray[uiPageIdx])
+	{
+		pgprot_t prot = PAGE_KERNEL;
+
+		switch (PVRSRV_CPU_CACHE_MODE(psWrapData->ui32CPUCacheFlags))
+		{
+			case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				prot = pgprot_noncached(prot);
+				break;
+
+			case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				prot = pgprot_writecombine(prot);
+				break;
+
+			case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+				break;
+
+			default:
+				break;
+		}
+
+		puiLinCPUAddr = vmap(&psWrapData->ppsPageArray[uiPageIdx], 1, VM_MAP, prot);
+
+		psMapData->bVMAP = IMG_TRUE;
+	}
+	else
+	{
+		puiLinCPUAddr = (IMG_UINT8*) OSMapPhysToLin(psWrapData->ppvPhysAddr[uiPageIdx],
+		                                              PAGE_SIZE,
+		                                              psWrapData->ui32CPUCacheFlags);
+
+		psMapData->bVMAP = IMG_FALSE;
+	}
+
+	psMapData->pvKernLinAddr = puiLinCPUAddr;
+
+}
+
+static PVRSRV_ERROR _CopyBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                     IMG_DEVMEM_OFFSET_T uiOffset,
+                                     IMG_UINT8 *pcBuffer,
+                                     size_t uiBufSz,
+                                     size_t *puiNumBytes,
+                                     IMG_BOOL bWrite)
+{
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	size_t uiBytesToCopy = uiBufSz;
+	IMG_UINT32 uiPageIdx = uiOffset >> PAGE_SHIFT;
+	IMG_BOOL bFirst = IMG_TRUE;
+	WRAP_KERNEL_MAP_DATA sKernMapData;
+
+
+	if ((uiBufSz + uiOffset) > (psWrapData->uiTotalNumPages << PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Trying to read out of bounds of PMR.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Copy the pages */
+	while (uiBytesToCopy != 0)
+	{
+		size_t uiBytesPerLoop;
+		IMG_DEVMEM_OFFSET_T uiCopyOffset;
+
+		if (bFirst)
+		{
+			bFirst = IMG_FALSE;
+			uiCopyOffset = (uiOffset & ~PAGE_MASK);
+			uiBytesPerLoop = (uiBufSz >= (PAGE_SIZE - uiCopyOffset)) ?
+					PAGE_SIZE - uiCopyOffset : uiBufSz;
+		}
+		else
+		{
+			uiCopyOffset = 0;
+			uiBytesPerLoop = (uiBytesToCopy > PAGE_SIZE) ? PAGE_SIZE : uiBytesToCopy;
+		}
+
+		_MapPage(psWrapData,
+		         uiPageIdx,
+		         &sKernMapData);
+
+		if (sKernMapData.pvKernLinAddr == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unable to map wrapped extmem page.",
+					__func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+
+		/* Use 'DeviceMemCopy' because we need to be conservative. We can't
+		 * know whether the wrapped memory was originally imported as cached.
+		 */
+
+		if (bWrite)
+		{
+			OSDeviceMemCopy(sKernMapData.pvKernLinAddr + uiCopyOffset,
+			          pcBuffer,
+			          uiBytesPerLoop);
+		}
+		else
+		{
+			OSDeviceMemCopy(pcBuffer,
+			          sKernMapData.pvKernLinAddr + uiCopyOffset,
+			          uiBytesPerLoop);
+		}
+
+		_UnmapPage(psWrapData,
+		           &sKernMapData);
+
+		uiBytesToCopy -= uiBytesPerLoop;
+		uiPageIdx++;
+	}
+
+	*puiNumBytes = uiBufSz;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRReadBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_UINT8 *pcBuffer,
+                   size_t uiBufSz,
+                   size_t *puiNumBytes)
+{
+
+	return _CopyBytesExtMem(pvPriv,
+	                        uiOffset,
+	                        pcBuffer,
+	                        uiBufSz,
+	                        puiNumBytes,
+	                        IMG_FALSE);
+}
+
+static PVRSRV_ERROR
+PMRWriteBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_UINT8 *pcBuffer,
+                    size_t uiBufSz,
+                    size_t *puiNumBytes)
+{
+	return _CopyBytesExtMem(pvPriv,
+	                        uiOffset,
+	                        pcBuffer,
+	                        uiBufSz,
+	                        puiNumBytes,
+	                        IMG_TRUE);
+}
+
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                  size_t uiOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  IMG_HANDLE *phHandleOut,
+                                  PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	WRAP_KERNEL_MAP_DATA *psKernMapData;
+	IMG_UINT32 ui32PageIndex = uiOffset >> PAGE_SHIFT;
+	IMG_UINT32 ui32PageOffset = (IMG_UINT32)(uiOffset & ~PAGE_MASK);
+
+	/* Offset was out of bounds */
+	if (ui32PageIndex > psWrapData->uiTotalNumPages)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, offset out of PMR bounds.",
+				__func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	/* We can not map in more than one page with ioremap.
+	 * Only possible with physically contiguous pages */
+	if ((ui32PageOffset + uiSize) > PAGE_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, cannot map more than one page for wrapped extmem.",
+				__func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	psKernMapData = OSAllocMem(sizeof(*psKernMapData));
+	if (psKernMapData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, unable to allocate memory for kernel mapping data.",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	_MapPage(psWrapData,
+	         ui32PageIndex,
+	         psKernMapData);
+
+	if (psKernMapData->pvKernLinAddr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Unable to map wrapped extmem page.",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	*ppvKernelAddressOut = ((IMG_CHAR *) psKernMapData->pvKernLinAddr) + ui32PageOffset;
+	*phHandleOut = psKernMapData;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+e1:
+	OSFreeMem(psKernMapData);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                              IMG_HANDLE hHandle)
+{
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	WRAP_KERNEL_MAP_DATA *psKernMapData = (void*) hHandle;
+
+	_UnmapPage(psWrapData,
+	           psKernMapData);
+
+	OSFreeMem(psKernMapData);
+}
+
+static inline void begin_user_mode_access(IMG_UINT *uiState)
+{
+#if defined(CONFIG_ARM) && defined(CONFIG_CPU_SW_DOMAIN_PAN)
+	*uiState = uaccess_save_and_enable();
+#elif defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	uaccess_enable_privileged();
+#elif defined(CONFIG_X86)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	__uaccess_begin();
+#else
+	PVR_UNREFERENCED_PARAMETER(uiState);
+#endif
+
+}
+
+static inline void end_user_mode_access(IMG_UINT uiState)
+{
+#if defined(CONFIG_ARM) && defined(CONFIG_CPU_SW_DOMAIN_PAN)
+	uaccess_restore(uiState);
+#elif defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	uaccess_disable_privileged();
+#elif defined(CONFIG_X86)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	__uaccess_end();
+#else
+	PVR_UNREFERENCED_PARAMETER(uiState);
+#endif
+}
+
+static PVRSRV_ERROR _FlushUMVirtualRange(PVRSRV_DEVICE_NODE *psDevNode,
+							PMR_WRAP_DATA *psPrivData,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_CPU_VIRTADDR pvCpuVAddr)
+{
+	struct vm_area_struct *psVMArea;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT	uiUserAccessState=0;
+
+	mmap_read_lock(current->mm);
+
+	/* Check that the recorded psVMArea matches the one associated with
+	 * this request. If not, fail the request.
+	 */
+	psVMArea = find_vma(current->mm, (uintptr_t)pvCpuVAddr);
+	if ((psVMArea != psPrivData->psVMArea) || (psVMArea == NULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Couldn't find memory region containing start address %p",
+		         __func__, (void *)pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto UMFlushUnlockReturn;
+	}
+
+
+	/*
+	 * Latest kernels enable "Privileged access never" feature in the kernel
+	 * This features leads to a privilege access failure fault (oops) whenever
+	 * the kernel tries to access any user mode address.
+	 * .
+	 * This is an additional security feature that allows kernel
+	 * to be protected against possible software attacks.
+	 * In this case, as we need to flush the cache lines associated
+	 * with the user mode virtual address, we need to explicitly disable
+	 * the feature for the duration of access and re-enable it once done.
+	 * */
+	begin_user_mode_access(&uiUserAccessState);
+	{
+		if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart = {0};
+
+			eError = CacheOpExec(psDevNode,
+								pvCpuVAddr,
+								((IMG_UINT8 *)pvCpuVAddr + uiSize),
+								sCPUPhysStart,
+								sCPUPhysStart,
+								PVRSRV_CACHE_OP_FLUSH);
+		}
+		else if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+			IMG_UINT i = 0;
+
+			for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+			{
+				if (NULL != psPrivData->ppsPageArray[i])
+				{
+					sCPUPhysStart.uiAddr = psPrivData->ppvPhysAddr[i].uiAddr;
+					sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+
+					eError = CacheOpExec(psDevNode,
+										NULL,
+										NULL,
+										sCPUPhysStart,
+										sCPUPhysEnd,
+										PVRSRV_CACHE_OP_FLUSH);
+					if (eError != PVRSRV_OK)
+					{
+						break;
+					}
+				}
+			}
+		}
+		else if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_BOTH)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+			void *pvVirtStart, *pvVirtEnd;
+			IMG_UINT i = 0;
+
+			for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+			{
+				if (NULL != psPrivData->ppsPageArray[i])
+				{
+					pvVirtStart = pvCpuVAddr + (i * PAGE_SIZE);
+					pvVirtEnd = pvVirtStart + PAGE_SIZE;
+					sCPUPhysStart.uiAddr = psPrivData->ppvPhysAddr[i].uiAddr;
+					sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+
+					eError = CacheOpExec(psDevNode,
+										pvVirtStart,
+										pvVirtEnd,
+										sCPUPhysStart,
+										sCPUPhysEnd,
+										PVRSRV_CACHE_OP_FLUSH);
+					if (eError != PVRSRV_OK)
+					{
+						break;
+					}
+				}
+			}
+		}
+
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to clean the virtual region cache %p",
+					__func__,
+					pvCpuVAddr));
+			goto UMFlushFailed;
+		}
+	}
+
+UMFlushFailed:
+	end_user_mode_access(uiUserAccessState);
+
+UMFlushUnlockReturn:
+	mmap_read_unlock(current->mm);
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRWrapPFuncTab = {
+    .pfnLockPhysAddresses = NULL,
+    .pfnUnlockPhysAddresses = NULL,
+    .pfnDevPhysAddr = &PMRSysPhysAddrExtMem,
+    .pfnAcquireKernelMappingData = PMRAcquireKernelMappingDataExtMem,
+    .pfnReleaseKernelMappingData = PMRReleaseKernelMappingDataExtMem,
+    .pfnReadBytes = PMRReadBytesExtMem,
+    .pfnWriteBytes = PMRWriteBytesExtMem,
+    .pfnChangeSparseMem = NULL,
+    .pfnFinalize = &PMRFinalizeExtMem,
+};
+
+static inline PVRSRV_ERROR PhysmemValidateParam( IMG_DEVMEM_SIZE_T uiSize,
+                                                 IMG_CPU_VIRTADDR pvCpuVAddr,
+                                                 PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (!access_ok(pvCpuVAddr, uiSize))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid User mode CPU virtual address"));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* Fail if requesting coherency on one side but uncached on the other */
+	if ((PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) &&
+			(PVRSRV_CHECK_GPU_UNCACHED(uiFlags) || PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for CPU coherency but specifying GPU uncached "
+				"Please use GPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if ((PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+			(PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for GPU coherency but specifying CPU uncached "
+				"Please use CPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+#if !defined(PVRSRV_WRAP_EXTMEM_WRITE_ATTRIB_ENABLE)
+	if (uiFlags & (PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED |
+			PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED |
+			PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+			PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC
+#if defined(DEBUG)
+			| PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+#endif
+		))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Write Attribute not supported. Passed Flags: 0x%"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+				__func__, uiFlags));
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+#endif
+
+	/* Check address and size alignment */
+	if (uiSize & (PAGE_SIZE - 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Given size %llu is not multiple of OS page size (%lu)",
+				__func__,
+				uiSize,
+				PAGE_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (((uintptr_t) pvCpuVAddr) & (PAGE_SIZE - 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Given address %p is not aligned to OS page size (%lu)",
+				__func__,
+				pvCpuVAddr,
+				PAGE_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PhysmemWrapExtMemOS(CONNECTION_DATA * psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    IMG_CPU_VIRTADDR pvCpuVAddr,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    PMR **ppsPMRPtr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32	*pui32MappingTable = NULL;
+	PMR_WRAP_DATA *psPrivData;
+	PMR *psPMR;
+	IMG_UINT uiTotalNumPages = (uiSize >> PAGE_SHIFT);
+	IMG_BOOL bIsPMRDestroyed = IMG_FALSE;
+	IMG_UINT i = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0))
+	/* Ignore the most significant byte. */
+	pvCpuVAddr = (IMG_CPU_VIRTADDR)untagged_addr((uintptr_t)pvCpuVAddr);
+#endif
+
+	eError = PhysmemValidateParam(uiSize,
+	                              pvCpuVAddr,
+	                              uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Allocate private factory data */
+	eError = _AllocWrapData(&psPrivData,
+	                        psDevNode,
+	                        uiSize,
+	                        pvCpuVAddr,
+	                        uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Actually find and acquire the pages and physical addresses */
+	eError = _WrapExtMemAcquirePages(psDevNode,
+	                                 uiSize,
+	                                 pvCpuVAddr,
+	                                 psPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		_FreeWrapData(psPrivData);
+		goto e0;
+	}
+
+
+	pui32MappingTable = OSAllocMem(sizeof(*pui32MappingTable) * uiTotalNumPages);
+	if (pui32MappingTable == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	for (i = 0; i < uiTotalNumPages; i++)
+	{
+		pui32MappingTable[i] = i;
+	}
+
+	/* Avoid creating scratch page or zero page when the entire
+	 * allocation is backed and pinned */
+	if (psPrivData->uiNumBackedPages == psPrivData->uiTotalNumPages)
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING;
+	}
+
+	/* Create a suitable PMR */
+	eError = PMRCreatePMR(psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL],
+	                      uiSize,    /* PMR_SIZE_T uiLogicalSize                  */
+	                      uiTotalNumPages,    /* IMG_UINT32 ui32NumPhysChunks              */
+	                      uiTotalNumPages,    /* IMG_UINT32 ui32NumVirtChunks              */
+	                      pui32MappingTable,
+	                      PAGE_SHIFT,           /* PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee */
+	                      (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK), /* PMR_FLAGS_T uiFlags */
+	                      "WrappedExtMem",      /* const IMG_CHAR *pszAnnotation             */
+	                      &_sPMRWrapPFuncTab,   /* const PMR_IMPL_FUNCTAB *psFuncTab         */
+	                      psPrivData,           /* PMR_IMPL_PRIVDATA pvPrivData              */
+	                      PMR_TYPE_EXTMEM,
+	                      &psPMR,               /* PMR **ppsPMRPtr                           */
+	                      PDUMP_NONE);          /* IMG_UINT32 ui32PDumpFlags                 */
+	if (eError != PVRSRV_OK)
+	{
+		goto e2;
+	}
+
+	if (PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags))
+	{
+		eError = _FlushUMVirtualRange(psDevNode,
+									psPrivData,
+									uiSize,
+									pvCpuVAddr);
+		if (eError != PVRSRV_OK)
+		{
+			goto e3;
+		}
+	}
+
+	/* Mark the PMR such that no layout changes can happen.
+	 * The memory is allocated in the CPU domain and hence
+	 * no changes can be made through any of our API */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	*ppsPMRPtr = psPMR;
+
+	OSFreeMem(pui32MappingTable);
+
+	return PVRSRV_OK;
+e3:
+	(void) PMRUnrefPMR(psPMR);
+	bIsPMRDestroyed = IMG_TRUE;
+e2:
+	OSFreeMem(pui32MappingTable);
+e1:
+	if (!bIsPMRDestroyed)
+	{
+		(void)_WrapExtMemReleasePages(psPrivData);
+	}
+e0:
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_wrap.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_wrap.h
new file mode 100644
index 000000000000..015360736d03
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_extmem_wrap.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File           physmem_extmem_wrap.h
+@Title          Header for wrapping non-services allocated memory PMR factory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the support for wrapping non services allocated
+                memory into GPU space.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_EXTEMEM_WRAP_H
+#define SRVSRV_PHYSMEM_EXTEMEM_WRAP_H
+
+#include <linux/version.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+#include <linux/mm_types.h>
+#include <linux/vmalloc.h>
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <linux/atomic.h>
+
+#include "img_types.h"
+#include "device.h"
+
+typedef enum _WRAP_EXT_MEM_TYPE_
+{
+	WRAP_TYPE_NULL = 0,
+	WRAP_TYPE_GET_USER_PAGES,
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+	WRAP_TYPE_FIND_VMA
+#endif
+} WRAP_EXT_MEM_TYPE;
+
+typedef struct _PMR_WRAP_DATA_
+{
+	/* Device for which this allocation has been made */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* Total Number of pages in the allocation */
+	IMG_UINT32 uiTotalNumPages;
+
+	/* Total number of actual physically backed pages */
+	IMG_UINT32 uiNumBackedPages;
+
+	/* This is only filled if we have page mappings,
+	 * for pfn mappings this stays empty */
+	struct page **ppsPageArray;
+
+	/* VM Area structure reference */
+	struct vm_area_struct *psVMArea;
+
+	/* This should always be filled and hold the physical addresses */
+	IMG_CPU_PHYADDR *ppvPhysAddr;
+
+	/* Did we find the pages via get_user_pages()
+	 * or via FindVMA and a page table walk? */
+	WRAP_EXT_MEM_TYPE eWrapType;
+
+	/* CPU caching modes for the PMR mapping */
+	IMG_UINT32 ui32CPUCacheFlags;
+
+	/* Write permitted ?*/
+	IMG_BOOL bWrite;
+
+} PMR_WRAP_DATA;
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+/* Find the VMA to a given CPU virtual address and do a
+ * page table walk to find the corresponding pfns */
+PVRSRV_ERROR _TryFindVMA(IMG_DEVMEM_SIZE_T uiSize,
+                            uintptr_t pvCpuVAddr,
+                            PMR_WRAP_DATA *psPrivData);
+#endif
+
+#endif /* SRVSRV_PHYSMEM_EXTEMEM_WRAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.c
new file mode 100644
index 000000000000..e624878d6862
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.c
@@ -0,0 +1,4786 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR functions for OS managed memory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                implementing the function callbacks for physical memory borrowed
+                from that normally managed by the operating system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+#include <linux/mm_types.h>
+#include <linux/vmalloc.h>
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <linux/atomic.h>
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#include <linux/migrate.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+#include <drm/drm_file.h>
+#endif
+#endif
+
+#if defined(CONFIG_X86)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0))
+#include <asm/set_memory.h>
+#else
+#include <asm/cacheflush.h>
+#endif
+#endif
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "rgx_pdump_panics.h"
+/* services/server/include/ */
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "pmr_os.h"
+#include "cache_km.h"
+#include "devicemem_server_utils.h"
+#include "pvr_vmap.h"
+#include "physheap.h"
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+#include "physmem_cpumap_history.h"
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+/* services/server/env/linux */
+#include "env_connection.h"
+#endif
+#endif
+
+/* ourselves */
+#include "physmem_osmem.h"
+#include "physmem_osmem_linux.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+#include "hash.h"
+#endif
+#endif
+
+#include "kernel_compatibility.h"
+
+
+static IMG_UINT32 g_uiMaxOrder = PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM;
+
+
+/* Defines how many pages should be mapped at once to the kernel */
+#define PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES 1024 /* 4 MB */
+
+/*
+	These are used to get/set/mask lower-order bits in a dma_addr_t
+	to provide side-band information associated with that address.
+	These includes whether the address was obtained via alloc_page
+	or dma_alloc and if address came allocated pre-aligned or an
+	adjustment was made manually to aligned it. They can also indicate
+	a page entry in the array has been generated by us for ease of mapping
+	and is part of a higher order page, this is known as a CMA Ghost.
+*/
+#define DMA_SET_CMA_GHOST(x)			((x) | ((dma_addr_t)0x03))
+#define DMA_IS_CMA_GHOST(x)				(((x) & ((dma_addr_t)0x03)) == 0x03)
+#define DMA_SET_ADJUSTED_ADDR(x)		((x) | ((dma_addr_t)0x02))
+#define DMA_IS_ADDR_ADJUSTED(x)			(((x) & ((dma_addr_t)0x02)) == 0x02)
+#define DMA_SET_ALLOCPG_ADDR(x)			((x) | ((dma_addr_t)0x01))
+#define DMA_IS_ALLOCPG_ADDR(x)			(((x) & ((dma_addr_t)0x01)) == 0x01)
+#define DMA_GET_ALIGN_ADJUSTMENT(x)		((x>>2) & ((dma_addr_t)0x3ff))
+#define DMA_SET_ALIGN_ADJUSTMENT(x,y)	((x) | (((dma_addr_t)y)<<0x02))
+#define DMA_GET_ADDR(x)					(((dma_addr_t)x) & ((dma_addr_t)~0xfff))
+#define DMA_VADDR_NOT_IN_USE			0xCAFEF00DDEADBEEFULL
+
+#define PVRSRV_ZERO_VALUE 0
+
+typedef struct _PMR_OSPAGEARRAY_DATA_ {
+	/* Device for which this allocation has been made */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+
+	/* DRM Node associated with connection that made the allocation */
+	struct drm_file *psDRMFile;
+#endif
+	/* Protects migration in progress count and action of idling the device */
+	POSWR_LOCK hPagesUnderMigrationRWLock;
+	/* How many pages in the PMR are currently in process of migration,
+	 * Must be protected by above RW lock.
+	 */
+	IMG_INT32 iNumOSPagesUnderMigration;
+
+	/*
+	 * CPU mappings associated with PMR.
+	 * Must be protected with PMR lock.
+	 */
+	DLLIST_NODE sCpuMappingListHead;
+#endif
+
+	/* The pid that made this allocation */
+	IMG_PID uiPid;
+
+	/*
+	 * iNumOSPagesAllocated:
+	 * Number of pages allocated in this PMR so far.
+	 * This allows for up to (2^31 - 1) pages. With 4KB pages, that's 8TB of memory for each PMR.
+	 */
+	IMG_INT32 iNumOSPagesAllocated;
+
+	/*
+	 * uiTotalNumOSPages:
+	 * Total number of pages supported by this PMR. (Fixed as of now due the fixed Page table array size)
+	 *  number of "pages" (a.k.a. macro pages, compound pages, higher order pages, etc...)
+	 */
+	IMG_UINT32 uiTotalNumOSPages;
+
+	/*
+	  uiLog2DevPageSize;
+
+	  size of each requested device "page" -- this would normally be the same as
+	  PAGE_SHIFT, but we support the idea that we may allocate pages
+	  in larger chunks for better contiguity, using order>0 in the
+	  call to alloc_pages()
+	*/
+	IMG_UINT32 uiLog2DevPageSize;
+
+	/*
+	  ui64DmaMask;
+	*/
+	IMG_UINT64 ui64DmaMask;
+
+	/*
+	  For non DMA/CMA allocation, pagearray references the pages
+	  thus allocated; one entry per compound page when compound
+	  pages are used. In addition, for DMA/CMA allocations, we
+	  track the returned cpu virtual and device bus address.
+	*/
+	struct page **pagearray;
+	dma_addr_t *dmaphysarray;
+	void **dmavirtarray;
+
+
+#define FLAG_ZERO              (0U)
+#define FLAG_POISON_ON_FREE    (1U)
+#define FLAG_POISON_ON_ALLOC   (2U)
+#define FLAG_ONDEMAND          (3U)
+
+/* IS CMA ensures the allocation will go through the dma_alloc_coherent path */
+#define FLAG_DMA_CMA           (5U)
+#define FLAG_UNSET_MEMORY_TYPE (6U)
+
+#define FLAG_IS_ZOMBIE         (7U)
+/* Pref CMA appends a flag (if exists) to hint alloc_pages should source from CMA region */
+#define FLAG_PREF_CMA          (8U)
+#define FLAG_IS_MOVABLE        (9U)
+
+	/*
+	 * Allocation flags related to the pages:
+	 * Zero              - Should we Zero memory on alloc
+	 * Poison on free    - Should we Poison the memory on free.
+	 * Poison on alloc   - Should we Poison the memory on alloc.
+	 * On demand         - Is the allocation on Demand i.e Do we defer allocation to time of use.
+	 * CMA               - Is CMA memory allocated via DMA framework
+	 * Unset Memory Type - Upon free do we need to revert the cache type before return to OS
+	 * Zombie            - Pages are part of a zombie PMR
+	 * */
+	IMG_UINT32 ui32AllocFlags;
+
+	/*
+	  The cache mode of the PMR. Additionally carrying the CPU-Cache-Clean
+	  flag, advising us to do cache maintenance on behalf of the caller.
+	  Boolean used to track if we need to revert the cache attributes
+	  of the pages used in this allocation. Depends on OS/architecture.
+	*/
+	IMG_UINT32 ui32CPUCacheFlags;
+	/*
+	 * In CMA allocation path, algorithm can allocate double the size of
+	 * requested allocation size to satisfy the alignment. In this case
+	 * the additional pages allocated are tracked through this additional
+	 * variable and are accounted for in the memory statistics */
+	IMG_UINT32 ui32CMAAdjustedPageCount;
+
+	/*
+	  Handle on the parent PMR
+	 */
+	PMR *hPMR;
+
+} PMR_OSPAGEARRAY_DATA;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+typedef struct _OSMEM_PAGE_PRIVDATA_
+{
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	/* Store index of entry to speed up migration */
+	IMG_UINT32 uiPMRArrIdx;
+} OSMEM_PAGE_PRIVDATA;
+#endif
+
+/***********************************
+ * Page pooling for uncached pages *
+ ***********************************/
+
+static INLINE void
+_FreeOSPage_CMA(struct device *dev,
+				size_t alloc_size,
+				IMG_UINT32 uiOrder,
+				void *virt_addr,
+				dma_addr_t dev_addr,
+				struct page *psPage);
+
+static void
+_FreeOSPage(IMG_UINT32 uiOrder,
+			IMG_BOOL bUnsetMemoryType,
+			PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			struct page *psPage);
+
+static PVRSRV_ERROR
+_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			IMG_UINT32 *pai32FreeIndices,
+			IMG_UINT32 ui32FreePageCount);
+
+static PVRSRV_ERROR
+_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree,
+						   IMG_UINT32 *puiPagesFreed);
+
+static inline PVRSRV_ERROR
+_ApplyOSPagesAttribute(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages,
+					   IMG_BOOL bFlush,
+					   IMG_UINT32 ui32CPUCacheFlags);
+
+/* A struct for our page pool holding an array of zeroed (!) pages.
+ * We always put units of page arrays to the pool but are
+ * able to take individual pages */
+typedef	struct
+{
+	/* Linkage for page pool LRU list */
+	struct list_head sPagePoolItem;
+
+	/* How many items are still in the page array */
+	IMG_UINT32 uiItemsRemaining;
+	/* Array of the actual pages */
+	struct page **ppsPageArray;
+
+} LinuxPagePoolEntry;
+
+/* CleanupThread structure to put allocation in page pool */
+typedef struct
+{
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupWork;
+	IMG_UINT32 ui32CPUCacheMode;
+	LinuxPagePoolEntry *psPoolEntry;
+} LinuxCleanupData;
+
+
+/* Caches to hold page pool and page array structures */
+static struct kmem_cache *g_psLinuxPagePoolCache;
+static struct kmem_cache *g_psLinuxPageArray;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+static struct kmem_cache *g_psLinuxPagePrivateData;
+#endif
+
+/* Track what is live, all protected by pool lock.
+ * x86 needs two page pools because we have to change the memory attributes
+ * of the pages which is expensive due to an implicit flush.
+ * See set_pages_array_uc/wc/wb. */
+static IMG_UINT32 g_ui32PagePoolUCCount;
+#if defined(CONFIG_X86)
+static IMG_UINT32 g_ui32PagePoolWCCount;
+#endif
+/* Tracks asynchronous tasks currently accessing the page pool.
+ * It is incremented if a defer free task
+ * is created. Both will decrement the value when they finished the work.
+ * The atomic prevents piling up of deferred work in case the deferred thread
+ * cannot keep up with the application.*/
+static ATOMIC_T g_iPoolCleanTasks;
+/* We don't want too many asynchronous threads trying to access the page pool
+ * at the same time */
+#define PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS 128
+
+/* Defines how many pages the page cache should hold. */
+#if defined(PVR_LINUX_PHYSMEM_MAX_POOL_PAGES)
+static const IMG_UINT32 g_ui32PagePoolMaxEntries = PVR_LINUX_PHYSMEM_MAX_POOL_PAGES;
+#else
+static const IMG_UINT32 g_ui32PagePoolMaxEntries;
+#endif
+
+/*	We double check if we would exceed this limit if we are below MAX_POOL_PAGES
+	and want to add an allocation to the pool.
+	This prevents big allocations being given back to the OS just because they
+	exceed the MAX_POOL_PAGES limit even though the pool is currently empty. */
+#if defined(PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES)
+static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries = PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES;
+#else
+static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries;
+#endif
+
+#if defined(CONFIG_X86)
+#define PHYSMEM_OSMEM_NUM_OF_POOLS 2
+static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = {
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED,
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC
+};
+#else
+#define PHYSMEM_OSMEM_NUM_OF_POOLS 1
+static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = {
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED
+};
+#endif
+
+/* Global structures we use to manage the page pool */
+static DEFINE_MUTEX(g_sPagePoolMutex);
+
+/* List holding the page array pointers: */
+static LIST_HEAD(g_sPagePoolList_WC);
+static LIST_HEAD(g_sPagePoolList_UC);
+
+
+static IMG_BOOL g_bInitialisedOnAlloc = IMG_FALSE;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+static bool OSMemPageIsolate(struct page *psPage, isolate_mode_t uIsolateMode);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static int OSMemPageMigrate(struct address_space *psAddrSpace,
+		struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode);
+#else
+static int OSMemPageMigrate(struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode);
+#endif
+static void OSMemPagePutback(struct page *psPage);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION_DEBUG)
+#define MIGRATE_DBG_LOG(x) PVR_DPF(x);
+#else
+#define MIGRATE_DBG_LOG(...)
+#endif
+
+static INLINE OSMEM_PAGE_PRIVDATA* GetPrivateDataFromPage(struct page* psPage)
+{
+	return (OSMEM_PAGE_PRIVDATA*) page_private(psPage);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static const struct address_space_operations movable_callbacks =
+{
+	.isolate_page = OSMemPageIsolate,
+	.migratepage = OSMemPageMigrate,
+	.putback_page = OSMemPagePutback
+
+};
+#else
+static const struct movable_operations movable_callbacks =
+{
+	.isolate_page = OSMemPageIsolate,
+	.migrate_page = OSMemPageMigrate,
+	.putback_page = OSMemPagePutback
+};
+#endif
+
+static INLINE void OSMemSetMovablePageAttr(struct page* psPage,
+                                           PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                           IMG_UINT32 uiPageIndex)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	struct drm_file *psDRMFile = psPageArrayData->psDRMFile;
+#endif
+	OSMEM_PAGE_PRIVDATA *psPrivData;
+
+	if (!BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		return;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	/* Allocations marked with movable may occur before we have a valid connection
+	 * and thus DRM file. In this case ignore the request. An example of this would be
+	 * a kernel originated request which wouldn't have a connection associated.
+	 */
+	if (!psDRMFile)
+	{
+		PVR_ASSERT(!"Attempt to make allocation movable without connection associated, "
+		            "possible server origin");
+		return;
+	}
+#endif
+
+	psPrivData = kmem_cache_alloc(g_psLinuxPagePrivateData, GFP_KERNEL);
+	if (psPrivData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: OS refused the memory allocation for the private page data. "
+		         "Page will not be marked movable. PMR UID:%llu",
+				 __func__,
+				 PMRInternalGetUID(psPrivData->psPMRData->hPMR)));
+		return;
+	}
+
+	psPrivData->psPMRData = psPageArrayData;
+	psPrivData->uiPMRArrIdx = uiPageIndex;
+
+	SetPagePrivate(psPage);
+	set_page_private(psPage, (unsigned long)psPrivData);
+
+	/* Take page lock here while we set movable as __SetPageMovable asserts it */
+	lock_page(psPage);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	__SetPageMovable(psPage, psDRMFile->filp->f_inode->i_mapping);
+#else
+	__SetPageMovable(psPage, &movable_callbacks);
+#endif
+	unlock_page(psPage);
+}
+
+/* Page should be locked before calling this function */
+static INLINE void OSMemSetPagePrivate(struct page* psPage)
+{
+	__ClearPageMovable(psPage);
+	set_page_private(psPage, 0);
+	ClearPagePrivate(psPage);
+}
+
+static INLINE void OSMemUnsetMovablePageAttr(struct page* psPage, PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData;
+
+	if (!BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		return;
+	}
+
+	/* Private data should only be set if page is movable */
+	psPrivData = GetPrivateDataFromPage(psPage);
+	if (!psPrivData)
+	{
+		return;
+	}
+
+	lock_page(psPage);
+	OSMemSetPagePrivate(psPage);
+	unlock_page(psPage);
+
+	kmem_cache_free(g_psLinuxPagePrivateData, psPrivData);
+}
+static IMG_BOOL _MigrateGPUIdleGet(PMR_OSPAGEARRAY_DATA *psPMRData)
+{
+	PVRSRV_ERROR eError;
+
+	/* Try take a ref on PMR, this ensures destruction will not occur while migrating */
+	eError = PMRTryRefPMR(psPMRData->hPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorOut);
+
+	OSWRLockAcquireWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+	/* If we just started migrating pages */
+	if ((++psPMRData->iNumOSPagesUnderMigration) == 1)
+	{
+		eError = PVRSRVDeviceIdleLatchedGetKM(psPMRData->psDevNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDeviceIdleLatchedGetKM", ErrorOutDecr);
+
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+		PMRLockPMR(psPMRData->hPMR);
+		PMRNotifyMigrateInProgress(psPMRData->hPMR);
+		PMRUnlockPMR(psPMRData->hPMR);
+	}
+	else
+	{
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	}
+
+	return IMG_TRUE;
+
+ErrorOutDecr:
+	psPMRData->iNumOSPagesUnderMigration--;
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING, "%s: Failed to Idle GPU", __func__));
+#endif
+	OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	(void) PMRUnrefPMR(psPMRData->hPMR);
+ErrorOut:
+	return IMG_FALSE;
+}
+
+static void _MigrateGPUIdlePut(PMR_OSPAGEARRAY_DATA *psPMRData)
+{
+	PVRSRV_ERROR eError;
+
+	OSWRLockAcquireWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+	/* If we just finished migrating pages */
+	if ((--psPMRData->iNumOSPagesUnderMigration) == 0)
+	{
+		eError = PVRSRVDeviceIdleLatchedPutAsyncKM(psPMRData->psDevNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDeviceIdleLatchedPutAsyncKM", ErrorOut);
+
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+
+		PMRLockPMR(psPMRData->hPMR);
+		PMRNotifyMigrateComplete(psPMRData->hPMR);
+		PMRUnlockPMR(psPMRData->hPMR);
+	}
+	else
+	{
+		OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	}
+
+
+	/* Unref parent PMR to allow for subsequent destruction,
+	 * if PMR resource no longer required and the bridge has
+	 * dropped its handles to it, destruction will follow.
+	 */
+	(void) PMRUnrefPMR(psPMRData->hPMR);
+
+	return;
+
+ErrorOut:
+	psPMRData->iNumOSPagesUnderMigration++;
+	OSWRLockReleaseWrite(psPMRData->hPagesUnderMigrationRWLock);
+	PVR_DPF((PVR_DBG_FATAL,"Request to Unidle GPU failed, unrecoverable state!"));
+}
+
+static bool OSMemPageIsolate(struct page *psPage, isolate_mode_t uIsolateMode)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData = GetPrivateDataFromPage(psPage);
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	IMG_BOOL bSuccess;
+
+
+	PVR_GOTO_IF_FALSE(psPrivData != NULL, condition_failed);
+	psPMRData = psPrivData->psPMRData;
+
+	/* Attempt to migrate pages before PMR wrapper has been setup,
+	 * avoid migration at this time.
+	 */
+	PVR_GOTO_IF_FALSE(psPMRData->hPMR, condition_failed);
+
+	bSuccess = _MigrateGPUIdleGet(psPMRData);
+	PVR_GOTO_IF_FALSE(bSuccess, condition_failed);
+
+	PVR_GOTO_IF_FALSE(!PMR_IsMemLayoutFixed(psPMRData->hPMR), clear_movable);
+	PVR_GOTO_IF_FALSE(!PMR_IsGpuMultiMapped(psPMRData->hPMR), clear_movable);
+	PVR_GOTO_IF_FALSE(!PMR_IsKernelCpuMapped(psPMRData->hPMR), condition_failed_decr);
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVR_GOTO_IF_FALSE(!PMR_IsZombie(psPMRData->hPMR), condition_failed_decr);
+#endif
+
+
+	psPrivData->psPMRData->pagearray[psPrivData->uiPMRArrIdx] = NULL;
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Isolated: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, PageAddr 0x%llx",
+	                                PMRInternalGetUID(psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psPage));
+
+	/* Movable */
+	return true;
+
+clear_movable:
+	OSMemSetPagePrivate(psPage);
+	kmem_cache_free(g_psLinuxPagePrivateData, psPrivData);
+
+condition_failed_decr:
+	_MigrateGPUIdlePut(psPMRData);
+
+condition_failed:
+	/* Not movable */
+	return false;
+}
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static int OSMemPageMigrate(struct address_space *psAddrSpace,
+		struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode)
+#else
+static int OSMemPageMigrate(struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode)
+#endif
+{
+	PVRSRV_ERROR eError;
+	int iLinuxError;
+	void* vpDst;
+	void* vpSrc;
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	OSMEM_PAGE_PRIVDATA *psSrcPagePrivData = GetPrivateDataFromPage(psSrcPage);
+	/* Align phys to logical */
+	IMG_UINT32 uiLogicalAlignedOffset;
+
+	PVR_UNREFERENCED_PARAMETER(eMigrateMode);
+
+	if (!psSrcPagePrivData)
+	{
+		/* Shouldn't hit this case */
+		PVR_DPF((PVR_DBG_ERROR, "Migrate invalid, potentially leaked PMR"));
+		return -EINVAL;
+	}
+	psPMRData = psSrcPagePrivData->psPMRData;
+
+	/* Calc logical aligned index offset, dev page aligned down. */
+	uiLogicalAlignedOffset = (
+	                          IMG_PAGES2BYTES64(psSrcPagePrivData->uiPMRArrIdx, PAGE_SHIFT) &
+	                          ~(psPMRData->uiLog2DevPageSize - 1)
+	                         ) >> PAGE_SHIFT;
+
+	/* If layout has been fixed after isolation, cancel the migration */
+	PVR_RETURN_IF_FALSE(!PMR_IsMemLayoutFixed(psPMRData->hPMR), -EPERM);
+
+	/* Unmap the page from any CPU mappings on this PMR. The mapping will
+	 * be repaired by the fault handler installed on the vma that
+	 * represents the CPU mapping.
+	 */
+	OSLinuxPMRUnmapPageInPMR(psPMRData->hPMR,
+	                         &psPMRData->sCpuMappingListHead,
+	                         uiLogicalAlignedOffset);
+
+	/* Set the new page in place, if any errors occur with migration the
+	 * putback callback will reset the original page.
+	 */
+	psPMRData->pagearray[psSrcPagePrivData->uiPMRArrIdx] = psDstPage;
+
+	/* Apply caching reqs to page we receive from OS */
+	eError = _ApplyOSPagesAttribute(psPMRData->psDevNode,
+	                                &psPMRData->pagearray[psSrcPagePrivData->uiPMRArrIdx],
+	                                1,
+	                                IMG_FALSE,
+	                                psPMRData->ui32CPUCacheFlags);
+	PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "_ApplyOSPagesAttribute", -EPERM);
+
+	/* Trigger remapping */
+	eError = PMRRemapGPUPMR(psPMRData->hPMR, uiLogicalAlignedOffset);
+	if (eError != PVRSRV_OK)
+	{
+		iLinuxError = -EPERM;
+		if (eError != PVRSRV_ERROR_DEVICEMEM_REMAP_REJECTED)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PMRRemapGPUPMR() failed to remap page.", __func__));
+		}
+		goto error_reset_page_attrs;
+	}
+
+	vpDst = kmap(psDstPage);
+	vpSrc = kmap(psSrcPage);
+
+
+	OSDeviceMemCopy(vpDst, vpSrc, PAGE_SIZE);
+
+	kunmap(vpDst);
+	kunmap(vpSrc);
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Migrated: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, SrcPageAddr 0x%llx, DstPageAddr 0x%llx",
+	                                PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psSrcPagePrivData->psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psSrcPage,
+	                                (unsigned long long) psDstPage));
+
+	get_page(psDstPage);
+	SetPagePrivate(psDstPage);
+	set_page_private(psDstPage, (unsigned long)psSrcPagePrivData);
+
+	/* DstPage already locked prior to callback */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	__SetPageMovable(psDstPage, psPMRData->psDRMFile->filp->f_inode->i_mapping);
+#else
+	__SetPageMovable(psDstPage, &movable_callbacks);
+#endif
+
+	/* Migration for this page no longer in progress */
+	_MigrateGPUIdlePut(psSrcPagePrivData->psPMRData);
+
+#if defined(CONFIG_X86)
+	/* Reset caching attrs of page we give back to OS */
+	if (!set_pages_array_wb(&psSrcPage, 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute for "
+		                        " page given back to kernel. PMR UID:%llu",
+		         __func__,
+		         (unsigned long long)PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR)));
+	}
+#endif
+
+	set_page_private(psSrcPage, 0);
+	ClearPagePrivate(psSrcPage);
+	put_page(psSrcPage);
+	__ClearPageMovable(psSrcPage);
+
+	return MIGRATEPAGE_SUCCESS;
+
+error_reset_page_attrs:
+#if defined(CONFIG_X86)
+	/* Reset caching attrs of page we give back to OS */
+	if (!set_pages_array_wb(&psDstPage, 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute for "
+		                        " page given from kernel. PMR UID:%llu",
+		         __func__,
+		         (unsigned long long)PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR)));
+	}
+#endif
+	return iLinuxError;
+}
+
+static void OSMemPagePutback(struct page *psPage)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData = GetPrivateDataFromPage(psPage);
+	if (!psPrivData)
+	{
+		/* We shouldn't hit this case */
+		PVR_DPF((PVR_DBG_ERROR, "Putback invalid, potentially leaked PMR"));
+		return;
+	}
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Putback: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, PageAddr 0x%llx",
+	                                PMRInternalGetUID(psPrivData->psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psPrivData->psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psPage));
+
+
+	/* Restore the page to our structures */
+	psPrivData->psPMRData->pagearray[psPrivData->uiPMRArrIdx] = psPage;
+
+	/* Migration for this page no longer in progress */
+	_MigrateGPUIdlePut(psPrivData->psPMRData);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+int LinuxRegisterMigrateCallbacks(struct file* filp)
+{
+	if (filp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "filp NULL, cannot set movable callbacks, PID %u:%s",
+		                        OSGetCurrentClientProcessIDKM(),
+		                        OSGetCurrentClientProcessNameKM()));
+		return -EINVAL;
+	}
+
+	filp->f_inode->i_mapping->a_ops = &movable_callbacks;
+
+	return 0;
+}
+
+void LinuxDeregisterMigrateCallbacks(struct file* filp)
+{
+	if (filp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "filp NULL, cannot unset movable callbacks"));
+		return;
+	}
+
+	filp->f_inode->i_mapping->a_ops = NULL;
+}
+#endif
+#endif /* defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+static inline IMG_BOOL
+_ShouldInitMem(IMG_UINT32 ui32AllocFlags)
+{
+	return BIT_ISSET(ui32AllocFlags, FLAG_ZERO) && !g_bInitialisedOnAlloc;
+}
+
+static inline IMG_UINT32
+_PagesInPoolUnlocked(void)
+{
+	IMG_UINT32 uiCnt = g_ui32PagePoolUCCount;
+#if defined(CONFIG_X86)
+	uiCnt += g_ui32PagePoolWCCount;
+#endif
+	return uiCnt;
+}
+
+static inline void
+_PagePoolLock(void)
+{
+	mutex_lock(&g_sPagePoolMutex);
+}
+
+static inline int
+_PagePoolTrylock(void)
+{
+	return mutex_trylock(&g_sPagePoolMutex);
+}
+
+static inline void
+_PagePoolUnlock(void)
+{
+	mutex_unlock(&g_sPagePoolMutex);
+}
+
+static inline IMG_BOOL
+_GetPoolListHead(IMG_UINT32 ui32CPUCacheFlags,
+				 struct list_head **ppsPoolHead,
+				 IMG_UINT32 **ppuiCounter)
+{
+	switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+#if defined(CONFIG_X86)
+		/*
+			For x86 we need to keep different lists for uncached
+			and write-combined as we must always honour the PAT
+			setting which cares about this difference.
+		*/
+
+			*ppsPoolHead = &g_sPagePoolList_WC;
+			*ppuiCounter = &g_ui32PagePoolWCCount;
+			break;
+#endif
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			*ppsPoolHead = &g_sPagePoolList_UC;
+			*ppuiCounter = &g_ui32PagePoolUCCount;
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unknown CPU caching mode. "
+					 "Using default UC pool.",
+					 __func__));
+			*ppsPoolHead = &g_sPagePoolList_UC;
+			*ppuiCounter = &g_ui32PagePoolUCCount;
+			PVR_ASSERT(0);
+			return IMG_FALSE;
+	}
+	return IMG_TRUE;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+static struct shrinker g_sShrinker;
+static struct shrinker *g_psShrinker = &g_sShrinker;
+#else
+static struct shrinker *g_psShrinker;
+#endif
+
+/* Returning the number of pages that still reside in the page pool. */
+static unsigned long
+_GetNumberOfPagesInPoolUnlocked(void)
+{
+	return _PagesInPoolUnlocked();
+}
+
+/* Linux shrinker function that informs the OS about how many pages we are caching and
+ * it is able to reclaim. */
+static unsigned long
+_CountObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl)
+{
+	int remain;
+
+	PVR_ASSERT(psShrinker == g_psShrinker);
+	(void)psShrinker;
+	(void)psShrinkControl;
+
+	/* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */
+	if (_PagePoolTrylock() == 0)
+		return 0;
+	remain = _GetNumberOfPagesInPoolUnlocked();
+	_PagePoolUnlock();
+
+	return remain;
+}
+
+/* Linux shrinker function to reclaim the pages from our page pool */
+static unsigned long
+_ScanObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl)
+{
+	unsigned long uNumToScan = psShrinkControl->nr_to_scan;
+	IMG_UINT32 uiPagesFreed;
+
+	PVR_ASSERT(psShrinker == g_psShrinker);
+	(void)psShrinker;
+
+	/* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */
+	if (_PagePoolTrylock() == 0)
+		return SHRINK_STOP;
+
+	_FreePagesFromPoolUnlocked(uNumToScan,
+							   &uiPagesFreed);
+	uNumToScan -= uiPagesFreed;
+
+	/* Returning the number of pages freed during the scan */
+	_PagePoolUnlock();
+	return psShrinkControl->nr_to_scan - uNumToScan;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+static struct shrinker g_sShrinker =
+{
+	.count_objects = _CountObjectsInPagePool,
+	.scan_objects = _ScanObjectsInPagePool,
+	.seeks = DEFAULT_SEEKS
+};
+#endif
+
+/* Register the shrinker so Linux can reclaim cached pages */
+PVRSRV_ERROR LinuxInitPhysmem(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	g_psLinuxPagePrivateData = kmem_cache_create("pvr-ppd", sizeof(OSMEM_PAGE_PRIVDATA), 0, 0, NULL);
+	if (!g_psLinuxPagePrivateData)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPagePrivateData",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e1);
+	}
+#endif
+
+	g_psLinuxPageArray = kmem_cache_create("pvr-pa", sizeof(PMR_OSPAGEARRAY_DATA), 0, 0, NULL);
+	if (!g_psLinuxPageArray)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPageArray",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e2);
+	}
+
+	g_psLinuxPagePoolCache = kmem_cache_create("pvr-pp", sizeof(LinuxPagePoolEntry), 0, 0, NULL);
+	if (g_psLinuxPagePoolCache)
+	{
+		/* Only create the shrinker if we created the cache OK */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+		register_shrinker(&g_sShrinker, "pvr-pp");
+#else
+		g_psShrinker = shrinker_alloc(0, "pvr-pp");
+		if (g_psShrinker)
+		{
+			g_psShrinker->count_objects = _CountObjectsInPagePool;
+			g_psShrinker->scan_objects = _ScanObjectsInPagePool;
+			g_psShrinker->seeks = DEFAULT_SEEKS;
+			shrinker_register(g_psShrinker);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Unable to allocate a shrinker for page pool cache"));
+		}
+#endif
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPagePoolCache",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e3);
+	}
+
+	OSAtomicWrite(&g_iPoolCleanTasks, 0);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,3,0))
+/* Check both config and modparam setting */
+#if PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC == 1
+	g_bInitialisedOnAlloc = want_init_on_alloc(0x0);
+
+/* Assume modparam setting not in use on system */
+#elif PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC == 2
+#   if defined(CONFIG_INIT_ON_ALLOC_DEFAULT_ON)
+	g_bInitialisedOnAlloc = IMG_TRUE;
+#   else
+	g_bInitialisedOnAlloc = IMG_FALSE;
+#   endif
+
+/* Ignore both config and modparam settings */
+#else
+	g_bInitialisedOnAlloc = IMG_FALSE;
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+e3:
+	/* Free the page array */
+	kmem_cache_destroy(g_psLinuxPageArray);
+e2:
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Free the page private data */
+	kmem_cache_destroy(g_psLinuxPagePrivateData);
+e1:
+#endif
+
+	return eError;
+}
+
+/* Unregister the shrinker and remove all pages from the pool that are still left */
+void LinuxDeinitPhysmem(void)
+{
+	IMG_UINT32 uiPagesFreed;
+
+	if (OSAtomicRead(&g_iPoolCleanTasks) > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Still deferred cleanup tasks running "
+				"while deinitialising memory subsystem."));
+	}
+
+	_PagePoolLock();
+	if (_FreePagesFromPoolUnlocked(IMG_UINT32_MAX, &uiPagesFreed) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Unable to free all pages from page pool when "
+				"deinitialising memory subsystem."));
+		PVR_ASSERT(0);
+	}
+
+	PVR_ASSERT(_PagesInPoolUnlocked() == 0);
+
+	/* Free the page cache */
+	if (g_psLinuxPagePoolCache)
+		kmem_cache_destroy(g_psLinuxPagePoolCache);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+	unregister_shrinker(&g_sShrinker);
+#else
+	shrinker_free(g_psShrinker);
+#endif
+	_PagePoolUnlock();
+
+	if (g_psLinuxPageArray)
+		kmem_cache_destroy(g_psLinuxPageArray);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (g_psLinuxPagePrivateData)
+		kmem_cache_destroy(g_psLinuxPagePrivateData);
+#endif
+}
+
+static void EnableOOMKiller(void)
+{
+	current->flags &= ~PF_DUMPCORE;
+}
+
+static void DisableOOMKiller(void)
+{
+	/* PF_DUMPCORE is treated by the VM as if the OOM killer was disabled.
+	 *
+	 * As oom_killer_disable() is an inline, non-exported function, we
+	 * can't use it from a modular driver. Furthermore, the OOM killer
+	 * API doesn't look thread safe, which 'current' is.
+	 */
+	WARN_ON(current->flags & PF_DUMPCORE);
+	current->flags |= PF_DUMPCORE;
+}
+
+/* Prints out the addresses in a page array for debugging purposes
+ * Define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY locally to activate: */
+/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY 1 */
+static inline void
+_DumpPageArray(struct page **pagearray, IMG_UINT32 uiPagesToPrint)
+{
+#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY)
+	IMG_UINT32 i;
+	if (pagearray)
+	{
+		printk("Array %p:\n", pagearray);
+		for (i = 0; i < uiPagesToPrint; i++)
+		{
+			printk("%p | ", (pagearray)[i]);
+		}
+		printk("\n");
+	}
+	else
+	{
+		printk("Array is NULL:\n");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(pagearray);
+	PVR_UNREFERENCED_PARAMETER(uiPagesToPrint);
+#endif
+}
+
+/* Debugging function that dumps out the number of pages for every
+ * page array that is currently in the page pool.
+ * Not defined by default. Define locally to activate feature: */
+/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL 1 */
+static void
+_DumpPoolStructure(void)
+{
+#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL)
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 j;
+	IMG_UINT32 *puiCounter;
+
+	printk("\n");
+	/* Empty all pools */
+	for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++)
+	{
+
+		printk("pool = %u\n", j);
+
+		/* Get the correct list for this caching mode */
+		if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter))
+		{
+			break;
+		}
+
+		list_for_each_entry_safe(psPagePoolEntry,
+								 psTempPoolEntry,
+								 psPoolHead,
+								 sPagePoolItem)
+		{
+			printk("%u | ", psPagePoolEntry->uiItemsRemaining);
+		}
+		printk("\n");
+	}
+#endif
+}
+
+/* Free a certain number of pages from the page pool.
+ * Mainly used in error paths or at deinitialisation to
+ * empty the whole pool. */
+static PVRSRV_ERROR
+_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree,
+						   IMG_UINT32 *puiPagesFreed)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 i, j;
+	IMG_UINT32 *puiCounter;
+
+	*puiPagesFreed = uiMaxPagesToFree;
+
+	/* Empty all pools */
+	for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++)
+	{
+
+		/* Get the correct list for this caching mode */
+		if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter))
+		{
+			break;
+		}
+
+		/* Free the pages and remove page arrays from the pool if they are exhausted */
+		list_for_each_entry_safe(psPagePoolEntry,
+								 psTempPoolEntry,
+								 psPoolHead,
+								 sPagePoolItem)
+		{
+			IMG_UINT32 uiItemsToFree;
+			struct page **ppsPageArray;
+
+			/* Check if we are going to free the whole page array or just parts */
+			if (psPagePoolEntry->uiItemsRemaining <= uiMaxPagesToFree)
+			{
+				uiItemsToFree = psPagePoolEntry->uiItemsRemaining;
+				ppsPageArray = psPagePoolEntry->ppsPageArray;
+			}
+			else
+			{
+				uiItemsToFree = uiMaxPagesToFree;
+				ppsPageArray = &(psPagePoolEntry->ppsPageArray[psPagePoolEntry->uiItemsRemaining - uiItemsToFree]);
+			}
+
+#if defined(CONFIG_X86)
+			/* Set the correct page caching attributes on x86 */
+			if (!PVRSRV_CHECK_CPU_CACHED(g_aui32CPUCacheFlags[j]))
+			{
+				int ret;
+				ret = set_pages_array_wb(ppsPageArray, uiItemsToFree);
+				if (ret)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s: Failed to reset page attributes",
+							 __func__));
+					eError = PVRSRV_ERROR_FAILED_TO_FREE_PAGES;
+					goto e_exit;
+				}
+			}
+#endif
+
+			/* Free the actual pages */
+			for (i = 0; i < uiItemsToFree; i++)
+			{
+				__free_pages(ppsPageArray[i], 0);
+				ppsPageArray[i] = NULL;
+			}
+
+			/* Reduce counters */
+			uiMaxPagesToFree -= uiItemsToFree;
+			*puiCounter -= uiItemsToFree;
+			psPagePoolEntry->uiItemsRemaining -= uiItemsToFree;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			/*
+			 * MemStats usually relies on having the bridge lock held, however
+			 * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and
+			 * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so
+			 * the page pool lock is used to ensure these calls are mutually
+			 * exclusive
+			 */
+			PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * uiItemsToFree);
+#endif
+
+			/* Is this pool entry exhausted, delete it */
+			if (psPagePoolEntry->uiItemsRemaining == 0)
+			{
+				OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+				list_del(&psPagePoolEntry->sPagePoolItem);
+				kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+			}
+
+			/* Return if we have all our pages */
+			if (uiMaxPagesToFree == 0)
+			{
+				goto e_exit;
+			}
+		}
+	}
+
+e_exit:
+	*puiPagesFreed -= uiMaxPagesToFree;
+	_DumpPoolStructure();
+	return eError;
+}
+
+/* Get a certain number of pages from the page pool and
+ * copy them directly into a given page array. */
+static void
+_GetPagesFromPoolUnlocked(IMG_UINT32 ui32CPUCacheFlags,
+						  IMG_UINT32 uiMaxNumPages,
+						  struct page **ppsPageArray,
+						  IMG_UINT32 *puiNumReceivedPages)
+{
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 i;
+	IMG_UINT32 *puiCounter;
+
+	*puiNumReceivedPages = 0;
+
+	/* Get the correct list for this caching mode */
+	if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter))
+	{
+		return;
+	}
+
+	/* Check if there are actually items in the list */
+	if (list_empty(psPoolHead))
+	{
+		return;
+	}
+
+	PVR_ASSERT(*puiCounter > 0);
+
+	/* Receive pages from the pool */
+	list_for_each_entry_safe(psPagePoolEntry,
+							 psTempPoolEntry,
+							 psPoolHead,
+							 sPagePoolItem)
+	{
+		/* Get the pages from this pool entry */
+		for (i = psPagePoolEntry->uiItemsRemaining; i != 0 && *puiNumReceivedPages < uiMaxNumPages; i--)
+		{
+			ppsPageArray[*puiNumReceivedPages] = psPagePoolEntry->ppsPageArray[i-1];
+			(*puiNumReceivedPages)++;
+			psPagePoolEntry->uiItemsRemaining--;
+		}
+
+		/* Is this pool entry exhausted, delete it */
+		if (psPagePoolEntry->uiItemsRemaining == 0)
+		{
+			OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+			list_del(&psPagePoolEntry->sPagePoolItem);
+			kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+		}
+
+		/* Return if we have all our pages */
+		if (*puiNumReceivedPages == uiMaxNumPages)
+		{
+			goto exit_ok;
+		}
+	}
+
+exit_ok:
+
+	/* Update counters */
+	*puiCounter -= *puiNumReceivedPages;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* MemStats usually relies on having the bridge lock held, however
+	 * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and
+	 * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so
+	 * the page pool lock is used to ensure these calls are mutually
+	 * exclusive
+	 */
+	PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * (*puiNumReceivedPages));
+#endif
+
+	_DumpPoolStructure();
+}
+
+/* Same as _GetPagesFromPoolUnlocked but handles locking and
+ * checks first whether pages from the pool are a valid option. */
+static inline void
+_GetPagesFromPoolLocked(PVRSRV_DEVICE_NODE *psDevNode,
+						IMG_UINT32 ui32CPUCacheFlags,
+						IMG_UINT32 ui32AllocFlags,
+						IMG_UINT32 uiPagesToAlloc,
+						IMG_UINT32 uiOrder,
+						struct page **ppsPageArray,
+						IMG_UINT32 *puiPagesFromPool)
+{
+#if !defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	/* Don't get pages from pool as it doesn't provide zeroed pages */
+	if (BIT_ISSET(ui32AllocFlags, FLAG_ZERO))
+	{
+		return;
+	}
+#endif
+
+	/* The page pool stores only order 0 pages. If we need zeroed memory we
+	 * directly allocate from the OS because it is faster than
+	 * doing it within the driver. CMA is naturally more commonly used for higher
+	 * order pages and so we reject that also.
+	 */
+	if (uiOrder == 0 &&
+	    !PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags) &&
+	    !BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA) &&
+	    !BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+
+		_PagePoolLock();
+		_GetPagesFromPoolUnlocked(ui32CPUCacheFlags,
+								  uiPagesToAlloc,
+								  ppsPageArray,
+								  puiPagesFromPool);
+		_PagePoolUnlock();
+	}
+}
+
+/* Takes a page array and maps it into the kernel to write zeros */
+static PVRSRV_ERROR
+_MemsetPageArray(IMG_UINT32 uiNumToClean,
+                 struct page **ppsCleanArray,
+                 pgprot_t pgprot,
+                 IMG_UINT8 ui8Pattern)
+{
+	IMG_CPU_VIRTADDR pvAddr;
+	IMG_UINT32 uiMaxPagesToMap = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES,
+	                                 uiNumToClean);
+
+	/* Map and fill the pages with zeros.
+	 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+	 * at a time. */
+	while (uiNumToClean != 0)
+	{
+		IMG_UINT32 uiToClean = MIN(uiNumToClean, uiMaxPagesToMap);
+
+		pvAddr = pvr_vmap(ppsCleanArray, uiToClean, VM_MAP, pgprot);
+		if (!pvAddr)
+		{
+			if (uiMaxPagesToMap <= 1)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				        "%s: Out of vmalloc memory, unable to map pages for %s.",
+				        __func__,
+				        ui8Pattern == PVRSRV_ZERO_VALUE ? "zeroing" : "poisoning"));
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+			else
+			{
+				/* Halve the pages to map at once and try again. */
+				uiMaxPagesToMap = uiMaxPagesToMap >> 1;
+				continue;
+			}
+		}
+
+		if (pgprot_val(pgprot) == pgprot_val(pgprot_noncached(PAGE_KERNEL)))
+		{
+			/* this is most likely unnecessary as all pages must be 8-bytes
+			 * aligned so there unaligned access is impossible */
+			OSDeviceMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		else if (pgprot_val(pgprot) == pgprot_val(pgprot_writecombine(PAGE_KERNEL)))
+		{
+			OSCachedMemSetWMB(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		else
+		{
+			OSCachedMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		pvr_vunmap(pvAddr, uiToClean, pgprot);
+		ppsCleanArray = &(ppsCleanArray[uiToClean]);
+		uiNumToClean -= uiToClean;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_CleanupThread_CleanPages(void *pvData)
+{
+	LinuxCleanupData *psCleanupData = (LinuxCleanupData*) pvData;
+	LinuxPagePoolEntry *psPagePoolEntry = psCleanupData->psPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 *puiCounter = NULL;
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	PVRSRV_ERROR eError;
+	pgprot_t pgprot;
+	IMG_UINT32 i;
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+
+	/* Get the correct pool for this caching mode. */
+	_GetPoolListHead(psCleanupData->ui32CPUCacheMode , &psPoolHead, &puiCounter);
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	switch (PVRSRV_CPU_CACHE_MODE(psCleanupData->ui32CPUCacheMode))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+#if defined(CONFIG_X86)
+			/* For x86 we can only map with the same attributes
+			 * as in the PAT settings*/
+			pgprot = pgprot_noncached(PAGE_KERNEL);
+			break;
+#endif
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			pgprot = pgprot_writecombine(PAGE_KERNEL);
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unknown caching mode to set page protection flags.",
+					__func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto eExit;
+	}
+
+	/* Map and fill the pages with zeros.
+	 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+	 * at a time. */
+	eError = _MemsetPageArray(psPagePoolEntry->uiItemsRemaining,
+	                          psPagePoolEntry->ppsPageArray,
+	                          pgprot, PVRSRV_ZERO_VALUE);
+	if (eError != PVRSRV_OK)
+	{
+		goto eExit;
+	}
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+
+	/* Lock down pool and add item */
+	_PagePoolLock();
+
+	/* Pool counters were already updated so don't do it here again*/
+
+	/* The pages are all zeroed so return them to the pool. */
+	list_add_tail(&psPagePoolEntry->sPagePoolItem, psPoolHead);
+
+	_DumpPoolStructure();
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Calling PVRSRVStatsIncrMemAllocPoolStat and PVRSRVStatsDecrMemAllocPoolStat
+	 * inside page pool lock ensures that the stat reflects the state of the pool. */
+	PVRSRVStatsIncrMemAllocPoolStat(PAGE_SIZE * psPagePoolEntry->uiItemsRemaining);
+#endif
+
+	_PagePoolUnlock();
+
+	OSFreeMem(pvData);
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+
+	return PVRSRV_OK;
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+eExit:
+	/* we failed to zero the pages so return the error so we can
+	 * retry during the next spin */
+	if ((psCleanupData->sCleanupWork.ui32RetryCount - 1) > 0)
+	{
+		return eError;
+	}
+
+	/* this was the last retry, give up and free pages to OS */
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Deferred task error, freeing pages to OS.",
+			__func__));
+	_PagePoolLock();
+
+	*puiCounter -= psPagePoolEntry->uiItemsRemaining;
+
+	_PagePoolUnlock();
+
+	for (i = 0; i < psCleanupData->psPoolEntry->uiItemsRemaining; i++)
+	{
+		_FreeOSPage(0, IMG_TRUE, NULL, psPagePoolEntry->ppsPageArray[i]);
+	}
+	OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+	kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+	OSFreeMem(psCleanupData);
+
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+
+	return PVRSRV_OK;
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+}
+
+static bool _PagesHaveOtherRefs(struct page **ppsPageArray, IMG_UINT32 uiNumPages)
+{
+	IMG_UINT32 ui32pageIndex;
+
+	PVR_DPF_ENTERED;
+
+	if (!ppsPageArray)
+	{
+		PVR_DPF_RETURN_RC(false);
+	}
+
+	/* Iterate pages in ppsPageArray and return
+	   false if any are found to have page_count() > 1
+	*/
+	for (ui32pageIndex = 0; ui32pageIndex < uiNumPages; ui32pageIndex++)
+	{
+		struct page *psPage = ppsPageArray[ui32pageIndex];
+
+		if (page_count(psPage) > 1)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: page %d in page array found with page_count() of %d",
+			         __func__, ui32pageIndex, page_count(psPage)));
+			PVR_DPF_RETURN_RC(true);
+		}
+	}
+
+	PVR_DPF_RETURN_RC(false);
+}
+
+/* Put page array to the page pool.
+ * Handles locking and checks whether the pages are
+ * suitable to be stored in the pool. */
+static inline IMG_BOOL
+_PutPagesToPoolLocked(PVRSRV_DEVICE_NODE *psDevNode,
+                      IMG_UINT32 ui32CPUCacheFlags,
+                      IMG_UINT32 ui32AllocFlags,
+                      struct page **ppsPageArray,
+                      IMG_UINT32 uiOrder,
+                      IMG_UINT32 uiNumPages)
+{
+	LinuxCleanupData *psCleanupData;
+	PVRSRV_CLEANUP_THREAD_WORK *psCleanupThreadFn;
+	IMG_UINT32 uiEntries;
+	IMG_UINT32 *puiCounter;
+	struct list_head *psPoolHead;
+#if defined(SUPPORT_PHYSMEM_TEST)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+#endif
+
+	PVR_DPF_ENTERED;
+
+	/* The page pool stores only order 0 pages. CMA is naturally more
+	 * commonly used for higher order pages and so we reject that also.
+	 */
+	if (uiOrder != 0 ||
+		PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags) ||
+		BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA) ||
+		BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		goto eExitFalse;
+	}
+
+	if (_PagesHaveOtherRefs(ppsPageArray, uiNumPages))
+	{
+		/* Pages still have other references, so
+		   must not be put into our pool */
+		goto eExitFalse;
+	}
+
+	_PagePoolLock();
+
+	uiEntries = _PagesInPoolUnlocked();
+
+	/* Check for number of current page pool entries */
+	if ( (uiEntries >= g_ui32PagePoolMaxEntries) ||
+		 ((uiEntries + uiNumPages) >=
+		  (g_ui32PagePoolMaxEntries + g_ui32PagePoolMaxExcessEntries) ))
+	{
+		goto eUnlock;
+	}
+
+	/* Check for whether we have other asynchronous tasks in-flight */
+	if (OSAtomicIncrement(&g_iPoolCleanTasks) >
+			PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS)
+	{
+		goto eDecrement;
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST)
+	if (!psPVRSRVData->hCleanupThread)
+	{
+		goto eDecrement;
+	}
+#endif
+
+	psCleanupData = OSAllocMem(sizeof(*psCleanupData));
+
+	if (!psCleanupData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get memory for deferred page pool cleanup. "
+				 "Trying to free pages immediately",
+				 __func__));
+		goto eDecrement;
+	}
+
+	psCleanupThreadFn = &psCleanupData->sCleanupWork;
+	psCleanupData->ui32CPUCacheMode = ui32CPUCacheFlags;
+	psCleanupData->psPoolEntry = kmem_cache_alloc(g_psLinuxPagePoolCache, GFP_KERNEL);
+
+	if (!psCleanupData->psPoolEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get memory for deferred page pool cleanup. "
+				 "Trying to free pages immediately",
+				 __func__));
+		goto eFreeCleanupData;
+	}
+
+	if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get correct page pool",
+				 __func__));
+		goto eFreePoolEntry;
+	}
+
+	/* Increase counter here to avoid deferred cleanup tasks piling up */
+	*puiCounter = *puiCounter + uiNumPages;
+
+	psCleanupData->psPoolEntry->ppsPageArray = ppsPageArray;
+	psCleanupData->psPoolEntry->uiItemsRemaining = uiNumPages;
+
+	psCleanupThreadFn->pfnFree = _CleanupThread_CleanPages;
+	psCleanupThreadFn->pvData = psCleanupData;
+	psCleanupThreadFn->bDependsOnHW = IMG_FALSE;
+	psCleanupThreadFn->eCleanupType = PVRSRV_CLEANUP_TYPE_OSMEM;
+	CLEANUP_THREAD_SET_RETRY_COUNT(psCleanupThreadFn,
+								   CLEANUP_THREAD_RETRY_COUNT_DEFAULT);
+
+	/* We must not hold the pool lock when calling AddWork because it might call us back to
+	 * free pooled pages directly when unloading the driver	 */
+	_PagePoolUnlock();
+
+	PVRSRVCleanupThreadAddWork(psDevNode, psCleanupThreadFn);
+
+	PVR_DPF_RETURN_RC(IMG_TRUE);
+
+eFreePoolEntry:
+	OSFreeMem(psCleanupData->psPoolEntry);
+eFreeCleanupData:
+	OSFreeMem(psCleanupData);
+eDecrement:
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+eUnlock:
+	_PagePoolUnlock();
+eExitFalse:
+	PVR_DPF_RETURN_RC(IMG_FALSE);
+}
+
+/* Get the GFP flags that we pass to the page allocator */
+static inline gfp_t
+_GetGFPFlags(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	/* If higher order pages then potentially ask the kernel to zero the allocation
+	 * for us. If zero order we can do it ourselves later in a batch operation
+	 */
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bZero = uiOrder ? _ShouldInitMem(psPageArrayData->ui32AllocFlags) : IMG_FALSE;
+	struct device *psDev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice;
+
+	gfp_t gfp_flags = __GFP_NOWARN | __GFP_NOMEMALLOC;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		gfp_flags |= GFP_HIGHUSER_MOVABLE;
+	}
+	else
+#else
+	{
+		gfp_flags |= GFP_USER;
+	}
+#endif
+
+#if defined(ANDROID)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)) && \
+	(LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	/* Only Android has CMA suggestion flag, base Linux kernel uses heuristics based on memory
+	 * pressure / availability to achieve similar goal.
+	 */
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_PREF_CMA))
+	{
+		gfp_flags |= __GFP_CMA;
+	}
+#endif
+#endif
+
+#if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+	/* Force use of HIGHMEM */
+	gfp_flags |= __GFP_HIGHMEM;
+
+	PVR_UNREFERENCED_PARAMETER(psDev);
+#else
+	if (psDev)
+	{
+#if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE)
+		if (*psDev->dma_mask > DMA_BIT_MASK(32))
+		{
+			/* If our system is able to handle large addresses use highmem */
+			gfp_flags |= __GFP_HIGHMEM;
+		}
+		else if (*psDev->dma_mask == DMA_BIT_MASK(32))
+		{
+			/* Limit to 32 bit.
+			 * Achieved by setting __GFP_DMA32 for 64 bit systems */
+			gfp_flags |= __GFP_DMA32;
+		}
+		else
+		{
+			/* Limit to size of DMA zone. */
+			gfp_flags |= __GFP_DMA;
+		}
+#else
+		if (*psDev->dma_mask < DMA_BIT_MASK(32))
+		{
+			gfp_flags |= __GFP_DMA;
+		}
+		else
+		{
+			gfp_flags |= __GFP_HIGHMEM;
+		}
+#endif /* if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE) */
+	}
+
+#endif /* if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY) */
+
+	if (bZero)
+	{
+		gfp_flags |= __GFP_ZERO;
+	}
+
+	return gfp_flags;
+}
+
+/*
+ * @Function _PoisonDevicePage
+ *
+ * @Description  Poisons a device page. In normal case the device page has the
+ *               same size as the OS page and so the ui32DevPageOrder will be
+ *               equal to 0 and page argument will point to one OS page
+ *               structure. In case of Non4K pages the order will be greater
+ *               than 0 and page argument will point to an array of OS
+ *               allocated pages.
+ *
+ * @Input psDevNode          pointer to the device object
+ * @Input page               array of the pages allocated by from the OS
+ * @Input ui32DevPageOrder   order of the page (same as the one used to allocate
+ *                           the page array by alloc_pages())
+ * @Input ui32CPUCacheFlags  CPU cache flags applied to the page
+ * @Input ui8PoisonValue     value used to poison the page
+ */
+static void
+_PoisonDevicePage(PVRSRV_DEVICE_NODE *psDevNode,
+                  struct page *page,
+                  IMG_UINT32 ui32DevPageOrder,
+                  IMG_UINT32 ui32CPUCacheFlags,
+                  IMG_BYTE ui8PoisonValue)
+{
+	IMG_UINT32 ui32OsPageIdx;
+
+	for (ui32OsPageIdx = 0;
+	     ui32OsPageIdx < (1U << ui32DevPageOrder);
+	     ui32OsPageIdx++)
+	{
+		struct page *current_page = page + ui32OsPageIdx;
+		IMG_CPU_PHYADDR sCPUPhysAddrStart = {page_to_phys(current_page)};
+		IMG_CPU_PHYADDR sCPUPhysAddrEnd = {sCPUPhysAddrStart.uiAddr + PAGE_SIZE};
+
+		void *kvaddr = kmap_atomic(current_page);
+
+		/* kmap_atomic maps pages as cached so it's safe to use OSCachedMemSet
+		 * here (also pages are always 8 bytes aligned anyway) */
+		OSCachedMemSet(kvaddr, ui8PoisonValue, PAGE_SIZE);
+
+		OSCPUCacheFlushRangeKM(psDevNode, kvaddr, kvaddr + PAGE_SIZE,
+		                       sCPUPhysAddrStart, sCPUPhysAddrEnd);
+
+		kunmap_atomic(kvaddr);
+	}
+}
+
+/* Allocate and initialise the structure to hold the metadata of the allocation */
+static PVRSRV_ERROR
+_AllocOSPageArray(PVRSRV_DEVICE_NODE *psDevNode,
+				  PMR_SIZE_T uiSize,
+				  IMG_UINT32 uiLog2DevPageSize,
+				  IMG_UINT32 ui32AllocFlags,
+				  IMG_UINT32 ui32CPUCacheFlags,
+				  IMG_PID uiPid,
+				  PMR_OSPAGEARRAY_DATA **ppsPageArrayDataPtr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiNumOSPageSizeVirtPages;
+	PMR_OSPAGEARRAY_DATA *psPageArrayData;
+	IMG_UINT64 ui64DmaMask = 0;
+
+	/* Use of cast below is justified by the assertion that follows to
+	 * prove that no significant bits have been truncated */
+	uiNumOSPageSizeVirtPages = (IMG_UINT32) (((uiSize - 1) >> PAGE_SHIFT) + 1);
+	PVR_ASSERT(((PMR_SIZE_T) uiNumOSPageSizeVirtPages << PAGE_SHIFT) == uiSize);
+
+	/* Allocate the struct to hold the metadata */
+	psPageArrayData = kmem_cache_alloc(g_psLinuxPageArray, GFP_KERNEL);
+	if (psPageArrayData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: OS refused the memory allocation for the private data.",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e_freed_none;
+	}
+
+	/*
+	 * Allocate the page array
+	 *
+	 * We avoid tracking this memory because this structure might go into the page pool.
+	 * The OS can drain the pool asynchronously and when doing that we have to avoid
+	 * any potential deadlocks.
+	 *
+	 * In one scenario the process stats vmalloc hash table lock is held and then
+	 * the oom-killer softirq is trying to call _ScanObjectsInPagePool(), it must not
+	 * try to acquire the vmalloc hash table lock again.
+	 */
+	psPageArrayData->pagearray = OSAllocZMemNoStats(sizeof(struct page *) * uiNumOSPageSizeVirtPages);
+	if (psPageArrayData->pagearray == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e_free_kmem_cache;
+	}
+
+	if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		/* Allocate additional DMA/CMA cpu kernel virtual address & device bus address array state */
+		psPageArrayData->dmavirtarray = OSAllocZMemNoStats(sizeof(void*) * uiNumOSPageSizeVirtPages);
+		if (psPageArrayData->dmavirtarray == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e_free_pagearray;
+		}
+
+		psPageArrayData->dmaphysarray = OSAllocZMemNoStats(sizeof(dma_addr_t) * uiNumOSPageSizeVirtPages);
+		if (psPageArrayData->dmaphysarray == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e_free_cpuvirtaddrarray;
+		}
+	}
+	else
+	{
+		psPageArrayData->dmavirtarray = NULL;
+		psPageArrayData->dmaphysarray = NULL;
+	}
+
+
+	if (psDevNode->psDevConfig && psDevNode->psDevConfig->pvOSDevice)
+	{
+		struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+		ui64DmaMask = *psDev->dma_mask;
+	}
+
+	/* Init metadata */
+	psPageArrayData->hPMR = NULL;
+	psPageArrayData->psDevNode = psDevNode;
+	psPageArrayData->uiPid = uiPid;
+	psPageArrayData->iNumOSPagesAllocated = 0;
+	psPageArrayData->uiTotalNumOSPages = uiNumOSPageSizeVirtPages;
+	psPageArrayData->uiLog2DevPageSize = uiLog2DevPageSize;
+	psPageArrayData->ui64DmaMask = ui64DmaMask;
+	psPageArrayData->ui32AllocFlags = ui32AllocFlags;
+	psPageArrayData->ui32CPUCacheFlags = ui32CPUCacheFlags;
+	psPageArrayData->ui32CMAAdjustedPageCount = 0;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	eError = OSWRLockCreate(&psPageArrayData->hPagesUnderMigrationRWLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSWRLockCreate", e_free_cpuvirtaddrarray);
+	psPageArrayData->iNumOSPagesUnderMigration = 0;
+	dllist_init(&psPageArrayData->sCpuMappingListHead);
+#endif
+
+	*ppsPageArrayDataPtr = psPageArrayData;
+	return PVRSRV_OK;
+
+/* Error path */
+e_free_cpuvirtaddrarray:
+	OSFreeMemNoStats(psPageArrayData->dmavirtarray);
+
+e_free_pagearray:
+	OSFreeMemNoStats(psPageArrayData->pagearray);
+
+e_free_kmem_cache:
+	kmem_cache_free(g_psLinuxPageArray, psPageArrayData);
+	PVR_DPF((PVR_DBG_ERROR,
+			 "%s: OS refused the memory allocation for the page pointer table. "
+			 "Did you ask for too much?",
+			 __func__));
+
+e_freed_none:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static inline void
+_ApplyCacheMaintenance(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages)
+{
+	void * pvAddr;
+
+	if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+	{
+		pgprot_t pgprot = PAGE_KERNEL;
+
+		IMG_UINT32 uiNumToClean = uiNumPages;
+		struct page **ppsCleanArray = ppsPage;
+
+		/* Map and flush page.
+		 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+		 * at a time. */
+		while (uiNumToClean != 0)
+		{
+			IMG_UINT32 uiToClean = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES,
+			                           uiNumToClean);
+			IMG_CPU_PHYADDR sUnused =
+				{ IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) };
+
+			pvAddr = pvr_vmap(ppsCleanArray, uiToClean, VM_MAP, pgprot);
+			if (!pvAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Unable to flush page cache for new allocation, skipping flush."));
+				return;
+			}
+
+			CacheOpExec(psDevNode,
+						pvAddr,
+						pvAddr + PAGE_SIZE,
+						sUnused,
+						sUnused,
+						PVRSRV_CACHE_OP_FLUSH);
+
+			pvr_vunmap(pvAddr, uiToClean, pgprot);
+			ppsCleanArray = &(ppsCleanArray[uiToClean]);
+			uiNumToClean -= uiToClean;
+		}
+	}
+	else
+	{
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 0; ui32Idx < uiNumPages; ++ui32Idx)
+		{
+			IMG_CPU_PHYADDR sCPUPhysAddrStart, sCPUPhysAddrEnd;
+
+			pvAddr = kmap(ppsPage[ui32Idx]);
+			sCPUPhysAddrStart.uiAddr = page_to_phys(ppsPage[ui32Idx]);
+			sCPUPhysAddrEnd.uiAddr = sCPUPhysAddrStart.uiAddr + PAGE_SIZE;
+
+			/* If we're zeroing, we need to make sure the cleared memory is pushed out
+			 * of the cache before the cache lines are invalidated */
+			CacheOpExec(psDevNode,
+						pvAddr,
+						pvAddr + PAGE_SIZE,
+						sCPUPhysAddrStart,
+						sCPUPhysAddrEnd,
+						PVRSRV_CACHE_OP_FLUSH);
+
+			kunmap(ppsPage[ui32Idx]);
+		}
+	}
+}
+
+/* Change the caching attribute of pages on x86 systems and takes care of
+ * cache maintenance. This function is supposed to be called once for pages that
+ * came from alloc_pages(). It expects an array of OS page sized pages!
+ *
+ * Flush/Invalidate pages in case the allocation is not cached. Necessary to
+ * remove pages from the cache that might be flushed later and corrupt memory. */
+static inline PVRSRV_ERROR
+_ApplyOSPagesAttribute(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages,
+					   IMG_BOOL bFlush,
+					   IMG_UINT32 ui32CPUCacheFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bCPUCached = PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags);
+	IMG_BOOL bCPUUncached = PVRSRV_CHECK_CPU_UNCACHED(ui32CPUCacheFlags);
+	IMG_BOOL bCPUWriteCombine = PVRSRV_CHECK_CPU_WRITE_COMBINE(ui32CPUCacheFlags);
+
+	if (ppsPage != NULL && uiNumPages != 0)
+	{
+#if defined(CONFIG_X86)
+		/* On x86 we have to set page cache attributes for non-cached pages.
+		 * The call is implicitly taking care of all flushing/invalidating
+		 * and therefore we can skip the usual cache maintenance after this. */
+		if (bCPUUncached || bCPUWriteCombine)
+		{
+			/* On x86 if we already have a mapping (e.g. low memory) we need to change the mode of
+				current mapping before we map it ourselves */
+			int ret = IMG_FALSE;
+
+			switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags))
+			{
+				case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+					ret = set_pages_array_uc(ppsPage, uiNumPages);
+					if (ret)
+					{
+						eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE;
+						PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to UC failed, returned %d", ret));
+					}
+					break;
+
+				case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+					ret = set_pages_array_wc(ppsPage, uiNumPages);
+					if (ret)
+					{
+						eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE;
+						PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to WC failed, returned %d", ret));
+					}
+					break;
+
+				case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+					break;
+
+				default:
+					break;
+			}
+		}
+		else
+#endif
+		{
+			if ( bFlush ||
+				 bCPUUncached || bCPUWriteCombine ||
+				 (bCPUCached && PVRSRV_CHECK_CPU_CACHE_CLEAN(ui32CPUCacheFlags)) )
+			{
+				/*  We can be given pages which still remain in the cache.
+					In order to make sure that the data we write through our mappings
+					doesn't get overwritten by later cache evictions we invalidate the
+					pages that are given to us.
+
+					Note:
+					This still seems to be true if we request cold pages, it's just less
+					likely to be in the cache. */
+				_ApplyCacheMaintenance(psDevNode,
+									   ppsPage,
+									   uiNumPages);
+			}
+		}
+	}
+
+	return eError;
+}
+
+/* Same as _AllocOSPage except it uses DMA framework to perform allocation.
+ * uiPageIndex is expected to be the pagearray index where to store the higher order page. */
+static PVRSRV_ERROR
+_AllocOSPage_CMA(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+				gfp_t gfp_flags,
+				IMG_UINT32 ui32AllocOrder,
+				IMG_UINT32 ui32MinOrder,
+				IMG_UINT32 uiPageIndex)
+{
+	void *virt_addr;
+	struct page *page;
+	dma_addr_t bus_addr;
+	IMG_UINT32 uiAllocIsMisaligned;
+	size_t alloc_size = PAGE_SIZE << ui32AllocOrder;
+	struct device *dev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice;
+
+	do
+	{
+		DisableOOMKiller();
+#if defined(PVR_LINUX_PHYSMEM_SUPPRESS_DMA_AC)
+		virt_addr = NULL;
+#else
+		virt_addr = dma_alloc_coherent(dev, alloc_size, &bus_addr, gfp_flags);
+#endif
+		if (virt_addr == NULL)
+		{
+			/* The idea here is primarily to support some older kernels with
+			   broken or non-functioning DMA/CMA implementations (< Linux-3.4)
+			   and to also handle DMA/CMA allocation failures by attempting a
+			   normal page allocation though we expect dma_alloc_coherent()
+			   already attempts this internally also before failing but
+			   nonetheless it does no harm to retry the allocation ourselves */
+			page = alloc_pages(gfp_flags, ui32AllocOrder);
+			if (page)
+			{
+				/* Taint bus_addr as alloc_page, needed when freeing;
+				   also acquire the low memory page address only, this
+				   prevents mapping possible high memory pages into
+				   kernel virtual address space which might exhaust
+				   the VMALLOC address space */
+				bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page));
+				virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE;
+			}
+			else
+			{
+				EnableOOMKiller();
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+		}
+		else
+		{
+#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
+			page = pfn_to_page(bus_addr >> PAGE_SHIFT);
+#else
+			/* Assumes bus address space is identical to physical address space */
+			page = phys_to_page(bus_addr);
+#endif
+		}
+		EnableOOMKiller();
+
+		/* Physical allocation alignment works/hidden behind the scene transparently,
+		   we do this here if the allocated buffer address does not meet its alignment
+		   requirement by over-allocating using the next power-2 order and reporting
+		   aligned-adjusted values back to meet the requested alignment constraint.
+		   Evidently we waste memory by doing this so should only do so if we do not
+		   initially meet the alignment constraint. */
+		uiAllocIsMisaligned = DMA_GET_ADDR(bus_addr) & ((PAGE_SIZE<<ui32MinOrder)-1);
+		if (uiAllocIsMisaligned || ui32AllocOrder > ui32MinOrder)
+		{
+			IMG_BOOL bUsedAllocPages = DMA_IS_ALLOCPG_ADDR(bus_addr);
+			if (ui32AllocOrder == ui32MinOrder)
+			{
+				if (bUsedAllocPages)
+				{
+					__free_pages(page, ui32AllocOrder);
+				}
+				else
+				{
+					dma_free_coherent(dev, alloc_size, virt_addr, bus_addr);
+				}
+
+				ui32AllocOrder = ui32AllocOrder + 1;
+				alloc_size = PAGE_SIZE << ui32AllocOrder;
+
+				PVR_ASSERT(uiAllocIsMisaligned != 0);
+			}
+			else
+			{
+				size_t align_adjust = PAGE_SIZE << ui32MinOrder;
+
+				/* Adjust virtual/bus addresses to meet alignment */
+				bus_addr = bUsedAllocPages ? page_to_phys(page) : bus_addr;
+				align_adjust = PVR_ALIGN((size_t)bus_addr, align_adjust);
+				align_adjust -= (size_t)bus_addr;
+
+				if (align_adjust)
+				{
+					if (bUsedAllocPages)
+					{
+						page += align_adjust >> PAGE_SHIFT;
+						bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page));
+						virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE;
+					}
+					else
+					{
+						bus_addr += align_adjust;
+						virt_addr += align_adjust;
+#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
+						page = pfn_to_page(bus_addr >> PAGE_SHIFT);
+#else
+						/* Assumes bus address space is identical to physical address space */
+						page = phys_to_page(bus_addr);
+#endif
+					}
+
+					/* Store adjustments in PAGE_SIZE counts */
+					align_adjust = align_adjust >> PAGE_SHIFT;
+					bus_addr = DMA_SET_ALIGN_ADJUSTMENT(bus_addr, align_adjust);
+
+					/* Taint bus_addr due to over-allocation, allows us to free
+					 * memory correctly */
+					bus_addr = DMA_SET_ADJUSTED_ADDR(bus_addr);
+				}
+				uiAllocIsMisaligned = 0;
+			}
+		}
+	} while (uiAllocIsMisaligned);
+
+	/* Convert OSPageSize-based index into DevicePageSize-based index */
+	psPageArrayData->ui32CMAAdjustedPageCount += (alloc_size - (PAGE_SIZE << ui32AllocOrder));
+	psPageArrayData->dmavirtarray[uiPageIndex] = virt_addr;
+	psPageArrayData->dmaphysarray[uiPageIndex] = bus_addr;
+	psPageArrayData->pagearray[uiPageIndex] = page;
+
+	return PVRSRV_OK;
+}
+
+/* Allocate a page of order uiAllocOrder and stores it in the page array ppsPage at
+ * position uiPageIndex.
+ *
+ * If the order is higher than 0, it splits the page into multiples and
+ * stores them at position uiPageIndex to uiPageIndex+(1<<uiAllocOrder).
+ *
+ * This function is supposed to be used for uiMinOrder == 0 only! */
+static PVRSRV_ERROR
+_AllocOSPage(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			gfp_t gfp_flags,
+			IMG_UINT32 uiAllocOrder,
+			IMG_UINT32 uiMinOrder,
+			IMG_UINT32 uiPageIndex)
+{
+	struct page *psPage;
+	IMG_UINT32 ui32Count;
+
+	/* Parameter check. If it fails we write into the wrong places in the array. */
+	PVR_ASSERT(uiMinOrder == 0);
+
+	/* Allocate the page */
+	DisableOOMKiller();
+	psPage = alloc_pages(gfp_flags, uiAllocOrder);
+	EnableOOMKiller();
+
+	if (psPage == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* In case we need to, split the higher order page;
+	   this should only be used for order-0 allocations
+	   as higher order allocations should use DMA/CMA */
+	if (uiAllocOrder != 0)
+	{
+		split_page(psPage, uiAllocOrder);
+	}
+
+	/* Store the page (or multiple split pages) in the page array */
+	for (ui32Count = 0; ui32Count < (1 << uiAllocOrder); ui32Count++)
+	{
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		OSMemSetMovablePageAttr(&psPage[ui32Count], psPageArrayData, uiPageIndex + ui32Count);
+#endif
+		psPageArrayData->pagearray[uiPageIndex + ui32Count] = &(psPage[ui32Count]);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+
+static inline void _AddMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                               const struct page *psPage,
+                                               IMG_UINT32 uiOrder)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = { page_to_phys(psPage) };
+	IMG_UINT32 uiPageSize = 1 << psPageArrayData->uiLog2DevPageSize;
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                             NULL, sCPUPhysAddr,
+	                             uiPageSize << uiOrder,
+	                             psPageArrayData->uiPid
+	                             DEBUG_MEMSTATS_VALUES);
+}
+
+static inline void _RemoveMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                                  const struct page *psPage)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	IMG_UINT32 uiStat = BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE)
+		? PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES
+		: PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#else
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#endif
+
+	PVRSRVStatsRemoveMemAllocRecord(uiStat, (IMG_UINT64) page_to_phys(psPage),
+	                                psPageArrayData->uiPid);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static inline void _TransferToMemZombieRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                                       const struct page *psPage)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = { page_to_phys(psPage) };
+
+	PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                                  PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,
+	                                  (IMG_UINT64) sCPUPhysAddr.uiAddr,
+	                                  psPageArrayData->uiPid
+	                                  DEBUG_MEMSTATS_VALUES);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+
+static inline void _IncrMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid)
+{
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                            uiSize, uiPid);
+}
+
+static inline void _DecrMemAllocStat_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                              size_t uiSize, IMG_PID uiPid)
+{
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES;
+	}
+#endif
+
+	PVRSRVStatsDecrMemAllocStat(uiStat, uiSize, uiPid);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static inline void _ZombifyMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid)
+{
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                            uiSize, uiPid);
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,
+	                            uiSize, uiPid);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+/* Allocation of OS pages: We may allocate 2^N order pages at a time for two reasons.
+ *
+ * Firstly to support device pages which are larger than OS. By asking the OS for 2^N
+ * order OS pages at a time we guarantee the device page is contiguous.
+ *
+ * Secondly for performance where we may ask for 2^N order pages to reduce the number
+ * of calls to alloc_pages, and thus reduce time for huge allocations.
+ *
+ * Regardless of page order requested, we need to break them down to track _OS pages.
+ * The maximum order requested is increased if all max order allocations were successful.
+ * If any request fails we reduce the max order.
+ */
+static PVRSRV_ERROR
+_AllocOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiArrayIndex = 0;
+	/* Order of the allocation to request, can be upgraded for large allocations
+	 * to aid efficient allocation. */
+	IMG_UINT32 ui32Order;
+	/* Min order the requested allocation can be. We express this in terms of the device
+	 * page size to the OS page size. The OS page size is the minimum page size we can request
+	 * else we cannot map it. Requesters may request further size / contiguity constraints in form
+	 * of the log2AllocPageSize */
+	IMG_UINT32 ui32MinOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bIncreaseMaxOrder = IMG_TRUE;
+
+	IMG_UINT32 ui32NumPageReq;
+	IMG_UINT32 uiOSPagesToAlloc;
+	IMG_UINT32 uiDevPagesFromPool = 0;
+
+	gfp_t gfp_flags = _GetGFPFlags(psPageArrayData);
+	gfp_t ui32GfpFlags;
+	gfp_t ui32HighOrderGfpFlags = ((gfp_flags & ~__GFP_RECLAIM) | __GFP_NORETRY);
+
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	struct page **ppsPageAttributeArray = NULL;
+
+	uiOSPagesToAlloc = psPageArrayData->uiTotalNumOSPages;
+
+	/* Try to get pages from the pool since it is faster;
+	   the page pool currently only supports zero-order pages
+	   thus currently excludes all DMA/CMA allocated memory.
+	   _ShouldInitMem() must not be used for bZero argument since it only
+	   applies to new pages allocated from the kernel.  */
+	_GetPagesFromPoolLocked(psPageArrayData->psDevNode,
+							psPageArrayData->ui32CPUCacheFlags,
+							psPageArrayData->ui32AllocFlags,
+							uiOSPagesToAlloc,
+							ui32MinOrder,
+							ppsPageArray,
+							&uiDevPagesFromPool);
+
+	uiArrayIndex = uiDevPagesFromPool;
+
+	if ((uiOSPagesToAlloc - uiDevPagesFromPool) < PVR_LINUX_HIGHORDER_ALLOCATION_THRESHOLD &&
+	    !BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{	/* Small allocations: ask for one device page at a time */
+		ui32Order = ui32MinOrder;
+		bIncreaseMaxOrder = IMG_FALSE;
+	}
+	else
+	{
+		/* Large zero-order or none zero-order allocations, ask for
+		   MAX(max-order, min-order) order pages at a time; alloc
+		   failures throttles this down to ZeroOrder allocations */
+		ui32Order = MAX(g_uiMaxOrder, ui32MinOrder);
+	}
+
+	/* Only if asking for more contiguity than we actually need, let it fail */
+	ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+	ui32NumPageReq = (1 << ui32Order);
+
+	while (uiArrayIndex < uiOSPagesToAlloc)
+	{
+		IMG_UINT32 ui32PageRemain = uiOSPagesToAlloc - uiArrayIndex;
+
+		while (ui32NumPageReq > ui32PageRemain)
+		{
+			/* Pages to request is larger than that remaining
+			   so ask for less so never over allocate */
+			ui32Order = MAX(ui32Order >> 1, ui32MinOrder);
+			ui32NumPageReq = (1 << ui32Order);
+			ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+		}
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			/* As the DMA/CMA framework rounds-up request to the
+			   next power-of-two, we request multiple uiMinOrder
+			   pages to satisfy allocation request in order to
+			   minimise wasting memory */
+			eError = _AllocOSPage_CMA(psPageArrayData,
+									  gfp_flags,
+									  ui32Order,
+									  ui32MinOrder,
+									  uiArrayIndex);
+		}
+		else
+		{
+			/* Allocate uiOrder pages at uiArrayIndex */
+			eError = _AllocOSPage(psPageArrayData,
+								  ui32GfpFlags,
+								  ui32Order,
+								  ui32MinOrder,
+								  uiArrayIndex);
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+			if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+			{
+				/* We encode non Ghost entries for CMA so we can record in order chunks */
+				_AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[uiArrayIndex], ui32Order);
+			}
+			else
+			{
+				/* Higher order pages get split using split_page() meaning we can't create a higher
+				 * order record in case the logic below reduces the order request part way through
+				 */
+				IMG_UINT32 i;
+				for (i = 0; i < (1 << ui32Order); i++)
+				{
+					_AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[uiArrayIndex + i], 0);
+				}
+			}
+#endif
+#endif
+			/* Successful request. Move onto next. */
+			uiArrayIndex += ui32NumPageReq;
+		}
+		else
+		{
+			if (ui32Order > ui32MinOrder)
+			{
+				/* Last request failed. Let's ask for less next time */
+				ui32Order = MAX(ui32Order >> 1, ui32MinOrder);
+				bIncreaseMaxOrder = IMG_FALSE;
+				ui32NumPageReq = (1 << ui32Order);
+				ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+				g_uiMaxOrder = ui32Order;
+			}
+			else
+			{
+				/* Failed to alloc pages at required contiguity. Failed allocation */
+				PVR_DPF((PVR_DBG_ERROR, "%s: %s failed to honour request at %u of %u, flags = %x, order = %u (%s)",
+								__func__,
+								BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA) ? "dma_alloc_coherent" : "alloc_pages",
+								uiArrayIndex,
+								uiOSPagesToAlloc,
+								ui32GfpFlags,
+								ui32Order,
+								PVRSRVGetErrorString(eError)));
+				eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+				goto e_free_pages;
+			}
+		}
+	}
+
+	if (bIncreaseMaxOrder && (g_uiMaxOrder < PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM))
+	{	/* All successful allocations on max order. Let's ask for more next time */
+		g_uiMaxOrder++;
+	}
+
+	/* Construct table of page pointers to apply attributes */
+	ppsPageAttributeArray = &ppsPageArray[uiDevPagesFromPool];
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT32 i;
+		dma_addr_t *psBusAddrArray = &psPageArrayData->dmaphysarray[uiDevPagesFromPool];
+
+		/* Iterate over page array generating ghost CMA pages */
+		for (i = 0; i < uiOSPagesToAlloc;)
+		{
+			IMG_UINT32 j;
+			IMG_UINT32 ui32NumPagesPerOrder = 1;
+
+			/* Iterate and populate until we find the next real page */
+			for (j = i + 1;
+			     j < uiOSPagesToAlloc &&
+			     ppsPageAttributeArray[j] == NULL;
+			     j++, ui32NumPagesPerOrder++)
+			{
+				/* Generate ghost page */
+				ppsPageAttributeArray[j] = ppsPageAttributeArray[i];
+				ppsPageAttributeArray[j] += ui32NumPagesPerOrder;
+				/* Mark DMA addr as ghost */
+				psBusAddrArray[j] = DMA_SET_CMA_GHOST(psBusAddrArray[i]);
+			}
+
+			i += ui32NumPagesPerOrder;
+		}
+	}
+
+	if (_ShouldInitMem(psPageArrayData->ui32AllocFlags) && ui32MinOrder == 0)
+	{
+		eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool,
+		                          ppsPageAttributeArray, PAGE_KERNEL,
+		                          PVRSRV_ZERO_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to zero pages (fast)"));
+			goto e_free_pages;
+		}
+	}
+	else if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC))
+	{
+		/* need to call twice because ppsPageArray and ppsPageAttributeArray
+		 * can point to different allocations: first for pages obtained from
+		 * the pool and then the remaining pages */
+		eError = _MemsetPageArray(uiDevPagesFromPool, ppsPageArray, PAGE_KERNEL,
+		                          PVRSRV_POISON_ON_ALLOC_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)"));
+		}
+		eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool,
+		                          ppsPageAttributeArray, PAGE_KERNEL,
+		                          PVRSRV_POISON_ON_ALLOC_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)"));
+		}
+
+		/* for poisoning need to also flush the pool pages as the 0s have
+		 * been overwritten */
+		_ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsPageArray,
+		                       uiDevPagesFromPool);
+	}
+
+	/* Do the cache management as required */
+	eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode,
+									ppsPageAttributeArray,
+									uiOSPagesToAlloc - uiDevPagesFromPool,
+									BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_ZERO) ||
+									BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC),
+									psPageArrayData->ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes"));
+		goto e_free_pages;
+	}
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated = psPageArrayData->uiTotalNumOSPages;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		_IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)),
+		                           psPageArrayData->uiPid);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return PVRSRV_OK;
+
+/* Error path */
+e_free_pages:
+	{
+		IMG_UINT32 ui32PageToFree;
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			IMG_UINT32 uiDevArrayIndex = uiArrayIndex >> ui32Order;
+			IMG_UINT32 uiDevPageSize = PAGE_SIZE << ui32Order;
+			PVR_ASSERT(ui32Order == ui32MinOrder);
+
+			for (ui32PageToFree = 0; ui32PageToFree < uiDevArrayIndex; ui32PageToFree++)
+			{
+				if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[ui32PageToFree]))
+				{
+					_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+									uiDevPageSize,
+									ui32MinOrder,
+									psPageArrayData->dmavirtarray[ui32PageToFree],
+									psPageArrayData->dmaphysarray[ui32PageToFree],
+									ppsPageArray[ui32PageToFree]);
+				}
+				psPageArrayData->dmaphysarray[ui32PageToFree]= (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[ui32PageToFree] = NULL;
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+		}
+		else
+		{
+			/* Free the pages we got from the pool */
+			for (ui32PageToFree = 0; ui32PageToFree < uiDevPagesFromPool; ui32PageToFree++)
+			{
+				/* Pages added to pool are always order 0 */
+				_FreeOSPage(0,
+							BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE),
+							psPageArrayData,
+							ppsPageArray[ui32PageToFree]);
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+
+			for (ui32PageToFree = uiDevPagesFromPool; ui32PageToFree < uiArrayIndex; ui32PageToFree++)
+			{
+				/* Higher order pages from _AllocOSPage are always split into order 0 using split_page */
+				_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsPageArray[ui32PageToFree]);
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+		}
+
+		return eError;
+	}
+}
+
+static INLINE PVRSRV_ERROR
+_CheckIfIndexInRange(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, IMG_UINT32 ui32Limit)
+{
+	if (pui32Indices[ui32Index] >= ui32Limit)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Given alloc index %u at %u is larger than page array %u.",
+		        __func__, pui32Indices[ui32Index], ui32Index, ui32Limit));
+		return PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+_CheckIfPageNotAllocated(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, IMG_UINT32 uiOrder, struct page **ppsPageArray)
+{
+	if (ppsPageArray[pui32Indices[ui32Index] << uiOrder] != NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Mapping Idx Dev:%u|OS:%u at page array index %u already exists. "
+		        "Page struct %p", __func__,
+		        pui32Indices[ui32Index],
+		        pui32Indices[ui32Index] << uiOrder,
+		        ui32Index,
+		        ppsPageArray[pui32Indices[ui32Index] << uiOrder]));
+		return PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Allocation of OS pages: This function is used for sparse allocations.
+ *
+ * Sparse allocations provide only a proportion of sparse physical backing within the total
+ * virtual range. */
+static PVRSRV_ERROR
+_AllocOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+					 IMG_UINT32 *pauiDevPageAllocIndices,
+					 IMG_UINT32 uiDevPagesToAlloc)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_UINT32 uiDevPagesFromPool = 0;
+	IMG_UINT32 uiOSPagesToAlloc = uiDevPagesToAlloc * (1 << uiOrder);
+	IMG_UINT32 uiDevPagesAllocated = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+	const IMG_UINT32 ui32AllocFlags = psPageArrayData->ui32AllocFlags;
+	gfp_t ui32GfpFlags = _GetGFPFlags(psPageArrayData);
+
+	/* We use this page array to receive pages from the pool and then reuse it afterwards to
+	 * store pages that need their cache attribute changed on x86 */
+	struct page **ppsTempPageArray;
+	IMG_UINT32 uiTempPageArrayIndex = 0;
+
+	/* Allocate the temporary page array that we need here to receive pages
+	 * from the pool and to store pages that need their caching attributes changed.
+	 * Allocate number of OS pages to be able to use the attribute function later. */
+	ppsTempPageArray = OSAllocMem(sizeof(struct page*) * uiOSPagesToAlloc);
+	PVR_LOG_GOTO_IF_NOMEM(ppsTempPageArray, eError, e_exit);
+
+	/* Check the requested number of pages if they fit in the page array */
+	if (uiDevPagesAllocated <
+	        ((psPageArrayData->iNumOSPagesAllocated >> uiOrder) + uiDevPagesToAlloc))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Trying to allocate more pages (Order %u) than this buffer can handle, "
+				 "Request + Allocated < Max! Request %u, Allocated %u, Max %u.",
+				 __func__,
+				 uiOrder,
+				 uiDevPagesToAlloc,
+				 psPageArrayData->iNumOSPagesAllocated >> uiOrder,
+				 uiDevPagesAllocated));
+		eError = PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+		goto e_free_temp_array;
+	}
+
+	/* Try to get pages from the pool since it is faster. The pages from pool are going to be
+	 * allocated only if:
+	 * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 1 && uiOrder == 0
+	 * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 0 && uiOrder == 0 &&
+	 *   !(BIT_ISSET(ui32AllocFlags, FLAG_ZERO))
+	 * - !BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA)
+	 * _ShouldInitMem() must not be used for bZero argument since it only
+	 * applies to new pages allocated from the kernel.  */
+	_GetPagesFromPoolLocked(psPageArrayData->psDevNode,
+							psPageArrayData->ui32CPUCacheFlags,
+							psPageArrayData->ui32AllocFlags,
+							uiDevPagesToAlloc,
+							uiOrder,
+							ppsTempPageArray,
+							&uiDevPagesFromPool);
+
+	/* In general device pages can have higher order than 0 but page pool always provides only 0
+	 * order pages so they can be assigned to the OS pages values (in other words if we're
+	 * allocating non-4k pages uiDevPagesFromPool will always be 0) */
+	uiTempPageArrayIndex = uiDevPagesFromPool;
+
+	/* Move pages we got from the pool to the array. */
+	for (i = 0; i < uiDevPagesFromPool; i++)
+	{
+		eError = _CheckIfIndexInRange(i, pauiDevPageAllocIndices, uiDevPagesAllocated);
+		PVR_GOTO_IF_ERROR(eError, e_free_pool_pages);
+		eError = _CheckIfPageNotAllocated(i, pauiDevPageAllocIndices, uiOrder, ppsPageArray);
+		PVR_GOTO_IF_ERROR(eError, e_free_pool_pages);
+
+		/* In this case the allocation order is the same for device and OS page size, we can use
+		 * DevPageAllocIndices here without having to scale up from device to OS page indices.
+		 */
+		PVR_ASSERT(uiOrder == 0);
+		ppsPageArray[pauiDevPageAllocIndices[i]] = ppsTempPageArray[i];
+	}
+
+	/* Allocate pages from the OS */
+	for (i = uiDevPagesFromPool; i < uiDevPagesToAlloc; i++)
+	{
+		IMG_UINT32 uiOSPageAllocIndex = pauiDevPageAllocIndices[i] << uiOrder;
+		/* We can perform range checks in terms of device pages, then we scale the indices
+		 * to OS page size for the storage array.
+		 */
+		eError = _CheckIfIndexInRange(i, pauiDevPageAllocIndices, uiDevPagesAllocated);
+		PVR_GOTO_IF_ERROR(eError, e_free_pages);
+		eError = _CheckIfPageNotAllocated(i, pauiDevPageAllocIndices, uiOrder, ppsPageArray);
+		PVR_GOTO_IF_ERROR(eError, e_free_pages);
+
+		/* Allocated pages and assign them the array. */
+		if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			IMG_UINT32 uiSubPageInOrder;
+			dma_addr_t *psBusAddrArray = &psPageArrayData->dmaphysarray[uiDevPagesFromPool];
+
+			/* As the DMA/CMA framework rounds-up request to the
+			   next power-of-two, we request multiple uiMinOrder
+			   pages to satisfy allocation request in order to
+			   minimise wasting memory */
+			eError = _AllocOSPage_CMA(psPageArrayData,
+									  ui32GfpFlags,
+									  uiOrder,
+									  uiOrder,
+									  uiOSPageAllocIndex);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to alloc CMA pages"));
+				goto e_free_pages;
+			}
+
+			for (uiSubPageInOrder = 1;
+			     uiSubPageInOrder < (1 << uiOrder);
+			     uiSubPageInOrder++)
+			{
+				/* Generate ghost page */
+				ppsPageArray[uiOSPageAllocIndex + uiSubPageInOrder] = ppsPageArray[uiOSPageAllocIndex];
+				ppsPageArray[uiOSPageAllocIndex + uiSubPageInOrder] += uiSubPageInOrder;
+				/* Mark DMA addr as ghost */
+				psBusAddrArray[uiOSPageAllocIndex + uiSubPageInOrder] = DMA_SET_CMA_GHOST(psBusAddrArray[uiOSPageAllocIndex]);
+			}
+		}
+		else
+		{
+			eError = _AllocOSPage(psPageArrayData,
+								  ui32GfpFlags,
+								  uiOrder,
+								  uiOrder,
+								  uiOSPageAllocIndex);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to alloc CMA pages"));
+				goto e_free_pages;
+			}
+		}
+
+		if (ppsPageArray[uiOSPageAllocIndex] != NULL)
+		{
+			/* Append pages to the temporary array so it's easier to process
+			 * them later on. */
+
+			if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+			{
+				IMG_UINT32 idx;
+				struct page *psPageAddr;
+
+				psPageAddr = ppsPageArray[uiOSPageAllocIndex];
+
+				/* "divide" CMA pages into OS pages if they have higher order */
+				for (idx = 0; idx < (1 << uiOrder); idx++)
+				{
+					ppsTempPageArray[uiTempPageArrayIndex + idx] = psPageAddr;
+					psPageAddr++;
+				}
+				uiTempPageArrayIndex += (1 << uiOrder);
+			}
+			else
+			{
+				ppsTempPageArray[uiTempPageArrayIndex] = ppsPageArray[uiOSPageAllocIndex];
+				uiTempPageArrayIndex++;
+			}
+		}
+		else
+		{
+			/* Failed to alloc pages at required contiguity. Failed allocation */
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: alloc_pages failed to honour request at %u of %u, flags = %x, order = %u",
+			        __func__, i, uiDevPagesToAlloc, ui32GfpFlags, uiOrder));
+			eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+			goto e_free_pages;
+		}
+	}
+
+	if (_ShouldInitMem(ui32AllocFlags) && uiOrder == 0)
+	{
+		/* At this point this array contains pages allocated from the page pool at its start
+		 * and pages allocated from the OS after that.
+		 * If there are pages from the pool here they must be zeroed already hence we don't have
+		 * to do it again. This is because if PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES is enabled pool pages
+		 * are zeroed in the cleanup thread. If it's disabled they aren't, and in that case we never
+		 * allocate pages with FLAG_ZERO from the pool. This is why those pages need to be zeroed
+		 * here.
+		 * All of the above is true for the 0 order pages. For higher order we never allocated from
+		 * the pool and those pages are allocated already zeroed from the OS.
+		 * Long story short we can always skip pages allocated from the pool because they are either
+		 * zeroed or we didn't allocate any of them. */
+		eError = _MemsetPageArray(uiTempPageArrayIndex - uiDevPagesFromPool,
+		                          &ppsTempPageArray[uiDevPagesFromPool],
+		                          PAGE_KERNEL, PVRSRV_ZERO_VALUE);
+		PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to zero pages (sparse)", e_free_pages);
+	}
+	else if (BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC))
+	{
+		/* Here we need to poison all of the pages regardless if they were
+		 * allocated from the pool or from the system. */
+		eError = _MemsetPageArray(uiTempPageArrayIndex, ppsTempPageArray,
+		                          PAGE_KERNEL, PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_IF_FALSE(eError == PVRSRV_OK, "failed to poison pages (sparse)");
+
+		/* We need to flush the cache for the poisoned pool pages here. The flush for the pages
+		 * allocated from the system is done below because we also need to add appropriate cache
+		 * attributes to them. Pages allocated from the pool already come with correct caching
+		 * mode. */
+		_ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsTempPageArray, uiDevPagesFromPool);
+	}
+
+	/* Do the cache management as required */
+	eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode,
+	                                &ppsTempPageArray[uiDevPagesFromPool],
+	                                uiTempPageArrayIndex - uiDevPagesFromPool,
+	                                BIT_ISSET(ui32AllocFlags, FLAG_ZERO) ||
+	                                BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC),
+	                                psPageArrayData->ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes"));
+		goto e_free_pages;
+	}
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated += uiOSPagesToAlloc;
+
+	/* Free temporary page array */
+	OSFreeMem(ppsTempPageArray);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiDevPagesToAlloc; i++)
+	{
+		_AddMemAllocRecord_UmaPages(psPageArrayData,
+		                            ppsPageArray[pauiDevPageAllocIndices[i] << uiOrder],
+		                            uiOrder);
+	}
+#else
+	_IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)),
+	                           psPageArrayData->uiPid);
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+e_free_pages:
+	if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder;
+
+		/* Free the pages we just allocated from the CMA */
+		for (; i > uiDevPagesFromPool; i--)
+		{
+			IMG_UINT32 idx = (i - 1) << uiOrder;
+			IMG_UINT32 uiSubPageInOrder;
+			_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+			                uiDevPageSize,
+			                uiOrder,
+			                psPageArrayData->dmavirtarray[pauiDevPageAllocIndices[idx]],
+			                psPageArrayData->dmaphysarray[pauiDevPageAllocIndices[idx]],
+			                ppsPageArray[pauiDevPageAllocIndices[idx]]);
+			psPageArrayData->dmaphysarray[pauiDevPageAllocIndices[idx]]= (dma_addr_t) 0;
+			psPageArrayData->dmavirtarray[pauiDevPageAllocIndices[idx]] = NULL;
+			ppsPageArray[pauiDevPageAllocIndices[idx]] = NULL;
+
+			for (uiSubPageInOrder = 1;
+				 uiSubPageInOrder < (1 << uiOrder);
+				 uiSubPageInOrder++)
+			{
+				ppsPageArray[idx + uiSubPageInOrder] = NULL;
+				psPageArrayData->dmaphysarray[idx + uiSubPageInOrder] = (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[idx + uiSubPageInOrder] = NULL;
+			}
+		}
+
+	}
+	else
+	{
+		/* Free the pages we just allocated from the OS */
+		for (; i > uiDevPagesFromPool; i--)
+		{
+			_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsPageArray[pauiDevPageAllocIndices[i-1]]);
+			ppsPageArray[pauiDevPageAllocIndices[i-1]] = NULL;
+		}
+	}
+
+e_free_pool_pages:
+	/* And now free all of the pages we allocated from the pool. */
+	for (i = 0; i < uiDevPagesFromPool; i++)
+	{
+		_FreeOSPage(0, BIT_ISSET(ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE),
+		            psPageArrayData,
+		            ppsTempPageArray[i]);
+
+		/* not using _CheckIfIndexInRange() to not print error message */
+		if (pauiDevPageAllocIndices[i] < uiDevPagesAllocated)
+		{
+			ppsPageArray[pauiDevPageAllocIndices[i]] = NULL;
+		}
+	}
+
+e_free_temp_array:
+	OSFreeMem(ppsTempPageArray);
+
+e_exit:
+	return eError;
+}
+
+/* Allocate pages for a given page array.
+ *
+ * The executed allocation path depends whether an array with allocation
+ * indices has been passed or not */
+static PVRSRV_ERROR
+_AllocOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			  IMG_UINT32 *puiAllocIndices,
+			  IMG_UINT32 uiPagesToAlloc)
+{
+	PVRSRV_ERROR eError;
+	struct page **ppsPageArray;
+
+	/* Parameter checks */
+	PVR_ASSERT(NULL != psPageArrayData);
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		PVR_ASSERT(psPageArrayData->dmaphysarray != NULL);
+		PVR_ASSERT(psPageArrayData->dmavirtarray != NULL);
+	}
+	PVR_ASSERT(psPageArrayData->pagearray != NULL);
+	PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated);
+
+	ppsPageArray = psPageArrayData->pagearray;
+
+	/* Go the sparse alloc path if we have an array with alloc indices.*/
+	if (puiAllocIndices != NULL)
+	{
+		eError = _AllocOSPages_Sparse(psPageArrayData,
+									  puiAllocIndices,
+									  uiPagesToAlloc);
+	}
+	else
+	{
+		eError = _AllocOSPages_Fast(psPageArrayData);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e_exit;
+	}
+
+	_DumpPageArray(ppsPageArray,
+	               psPageArrayData->uiTotalNumOSPages >>
+	               (psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT) );
+
+	PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: allocated OS memory for PMR @0x%p", psPageArrayData));
+	return PVRSRV_OK;
+
+e_exit:
+	return eError;
+}
+
+/* Same as _FreeOSPage except free memory using DMA framework */
+static INLINE void
+_FreeOSPage_CMA(struct device *dev,
+				size_t alloc_size,
+				IMG_UINT32 uiOrder,
+				void *virt_addr,
+				dma_addr_t dev_addr,
+				struct page *psPage)
+{
+	if (DMA_IS_ALLOCPG_ADDR(dev_addr))
+	{
+#if defined(CONFIG_X86)
+		void *pvPageVAddr = page_address(psPage);
+		if (pvPageVAddr)
+		{
+			int ret = set_memory_wb((unsigned long)pvPageVAddr, 1);
+			if (ret)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to reset page attribute",
+						__func__));
+			}
+		}
+#endif
+
+		if (DMA_IS_ADDR_ADJUSTED(dev_addr))
+		{
+			psPage -= DMA_GET_ALIGN_ADJUSTMENT(dev_addr);
+			uiOrder += 1;
+		}
+
+		__free_pages(psPage, uiOrder);
+	}
+	else
+	{
+		if (DMA_IS_ADDR_ADJUSTED(dev_addr))
+		{
+			size_t align_adjust;
+
+			align_adjust = DMA_GET_ALIGN_ADJUSTMENT(dev_addr);
+			alloc_size = alloc_size << 1;
+
+			dev_addr = DMA_GET_ADDR(dev_addr);
+			dev_addr -= align_adjust << PAGE_SHIFT;
+			virt_addr -= align_adjust << PAGE_SHIFT;
+		}
+
+#if defined(DEBUG)
+		/* We should never attempt to free a CMA Ghost as it is handled in layer above */
+		if (DMA_IS_CMA_GHOST(dev_addr))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Attempt to free CMA Ghost entry!"));
+			return;
+		}
+#endif
+
+		dma_free_coherent(dev, alloc_size, virt_addr, DMA_GET_ADDR(dev_addr));
+	}
+}
+
+/* Free a single page back to the OS.
+ * Make sure the cache type is set back to the default value.
+ *
+ * Note:
+ * We must _only_ check bUnsetMemoryType in the case where we need to free
+ * the page back to the OS since we may have to revert the cache properties
+ * of the page to the default as given by the OS when it was allocated. */
+static void
+_FreeOSPage(IMG_UINT32 uiOrder,
+			IMG_BOOL bUnsetMemoryType,
+			PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			struct page *psPage)
+{
+
+#if defined(CONFIG_X86)
+	void *pvPageVAddr;
+	pvPageVAddr = page_address(psPage);
+
+	if (pvPageVAddr && bUnsetMemoryType)
+	{
+		int ret;
+
+		ret = set_memory_wb((unsigned long)pvPageVAddr, 1);
+		if (ret)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute",
+					 __func__));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(bUnsetMemoryType);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* May call _FreeOSPage from clean pages for pool path, it has no
+	 * association with PMR_OSPAGEARRAY_DATA at this point
+	 */
+	if (psPageArrayData)
+	{
+		OSMemUnsetMovablePageAttr(psPage, psPageArrayData);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psPageArrayData);
+#endif
+
+	__free_pages(psPage, uiOrder);
+}
+
+/* Free the struct holding the metadata */
+static void
+_FreeOSPagesArray(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: freed OS memory for PMR @0x%p", psPageArrayData));
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (!dllist_is_empty(&psPageArrayData->sCpuMappingListHead))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Attempt to free linux cpu mapping data while still in use."));
+		OSWarnOn(1);
+	}
+#endif
+
+	psPageArrayData->hPMR = NULL;
+
+	/* Check if the page array actually still exists.
+	 * It might be the case that has been moved to the page pool */
+	if (psPageArrayData->pagearray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->pagearray);
+	}
+
+	/* Check if we need to free additional DMA/CMA cpu kernel virtual address & device bus address arrays */
+	if (psPageArrayData->dmaphysarray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->dmaphysarray);
+	}
+	if (psPageArrayData->dmavirtarray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->dmavirtarray);
+	}
+
+	kmem_cache_free(g_psLinuxPageArray, psPageArrayData);
+}
+
+/* Free all or some pages from a sparse page array */
+static PVRSRV_ERROR
+_FreeOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+					IMG_UINT32 *paui32DevPageFreeIndices,
+					IMG_UINT32 uiDevPagesToFree)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	IMG_UINT32 uiNumDevPages;
+	struct page **ppsOSPagesToFree;
+	IMG_UINT32 *puiDevIndicesToFree;
+	IMG_UINT32 uiOSPageCount;
+	IMG_UINT32 uiTransferedDevPageCount;
+	IMG_UINT32 i;
+
+	/* We really should have something to free before we call this */
+	PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0);
+
+	if (paui32DevPageFreeIndices == NULL)
+	{
+		uiNumDevPages = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+		uiOSPageCount = psPageArrayData->iNumOSPagesAllocated;
+	}
+	else
+	{
+		uiNumDevPages = uiDevPagesToFree;
+		uiOSPageCount = uiNumDevPages << uiOrder;
+	}
+
+	/* Construct the array of pages before any free operation so error recovery is possible.
+	 * OSAllocMemNoStats required because this code may be run without the bridge lock held */
+	ppsOSPagesToFree = OSAllocMemNoStats(uiOSPageCount * sizeof(*ppsOSPagesToFree));
+	PVR_LOG_GOTO_IF_NOMEM(ppsOSPagesToFree, eError, err_array);
+	puiDevIndicesToFree = OSAllocMemNoStats(uiNumDevPages * sizeof(*puiDevIndicesToFree));
+	PVR_LOG_GOTO_IF_NOMEM(puiDevIndicesToFree, eError, err_indices);
+
+	/* Count how many allocated dev pages have been found. In case we need to unwind */
+	uiTransferedDevPageCount = 0;
+	for (i = 0; i < uiNumDevPages; i++)
+	{
+		IMG_UINT32 uiSubPageInOrder;
+		IMG_UINT32 idx = paui32DevPageFreeIndices ? paui32DevPageFreeIndices[i] : i;
+		/* Scale Device page index to OS page Idx */
+		idx <<= uiOrder;
+
+		if (ppsPageArray[idx] != NULL)
+		{
+			IMG_UINT32 uiOSFreeArrayIdx = uiTransferedDevPageCount << uiOrder;
+			puiDevIndicesToFree[uiTransferedDevPageCount] = idx;
+
+			for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+			{
+				ppsOSPagesToFree[uiOSFreeArrayIdx + uiSubPageInOrder] = ppsPageArray[idx + uiSubPageInOrder];
+				ppsPageArray[idx + uiSubPageInOrder] = NULL;
+			}
+			uiTransferedDevPageCount++;
+		}
+		else if (paui32DevPageFreeIndices != NULL)
+		{
+			IMG_UINT32 j;
+			PVR_DPF((PVR_DBG_ERROR, "Invalid free index (%u), possible duplicate", idx));
+			/* Restore the ppsPageArray to its initial state.
+			 * Reverse the loop and swap the ppsTempPages back into ppsPageArray. */
+			for (j = 0; j < uiTransferedDevPageCount; j++)
+			{
+				IMG_UINT32 uiSubPageInOrder;
+
+				idx = puiDevIndicesToFree[j];
+				ppsPageArray[idx] = ppsOSPagesToFree[j];
+
+				for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+				{
+					ppsPageArray[idx + uiSubPageInOrder] = ppsOSPagesToFree[j + uiSubPageInOrder];
+				}
+			}
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK, err_indices);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiTransferedDevPageCount; i++)
+	{
+		/* Scale Device page index to OS page Idx */
+		IMG_UINT32 idx = i << uiOrder;
+
+		_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsOSPagesToFree[idx]);
+	}
+#endif /* PVRSRV_ENABLE_PROCESS_STATS && PVRSRV_ENABLE_MEMORY_STATS */
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE))
+	{
+		for (i = 0; i < uiTransferedDevPageCount; i++)
+		{
+			/* Scale Device page index to OS page Idx */
+			IMG_UINT32 idx = i << uiOrder;
+
+			_PoisonDevicePage(psPageArrayData->psDevNode,
+			                  ppsOSPagesToFree[idx],
+			                  uiOrder,
+			                  psPageArrayData->ui32CPUCacheFlags,
+			                  PVRSRV_POISON_ON_FREE_VALUE);
+		}
+	}
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT64 ui64DevPageSize = IMG_PAGE2BYTES64(psPageArrayData->uiLog2DevPageSize);
+		for (i = 0; i < uiTransferedDevPageCount; i++)
+		{
+			IMG_UINT32 uiSubPageInOrder;
+			/* Scale Device page index to OS page Idx */
+			IMG_UINT32 uiFreeIdx = i << uiOrder;
+			/* Get the page's original index in ppsPageArray, already scaled to dev pages */
+			IMG_UINT32 idx = puiDevIndicesToFree[i];
+			_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+			                ui64DevPageSize,
+			                uiOrder,
+			                psPageArrayData->dmavirtarray[idx],
+			                psPageArrayData->dmaphysarray[idx],
+			                ppsOSPagesToFree[uiFreeIdx]);
+			for (uiSubPageInOrder = 0;
+			     uiSubPageInOrder < (1 << uiOrder);
+			     uiSubPageInOrder++)
+			{
+				psPageArrayData->dmaphysarray[idx + uiSubPageInOrder] = (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[idx + uiSubPageInOrder] = NULL;
+			}
+		}
+	}
+	else
+	{
+		IMG_BOOL bPagesInPool;
+
+		/* Try to move the temp page array to the pool */
+		bPagesInPool = _PutPagesToPoolLocked(psPageArrayData->psDevNode,
+		                                     psPageArrayData->ui32CPUCacheFlags,
+		                                     psPageArrayData->ui32AllocFlags,
+		                                     ppsOSPagesToFree,
+		                                     0,
+		                                     uiOSPageCount);
+		if (bPagesInPool)
+		{
+			/* Do NOT free ppsPagesToFree if it moved to the pool */
+			ppsOSPagesToFree = NULL;
+		}
+		else
+		{
+			/* Free pages and reset page caching attributes on x86 */
+#if defined(CONFIG_X86)
+			if (uiOSPageCount != 0 && BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE))
+			{
+				int iError;
+				iError = set_pages_array_wb(ppsOSPagesToFree, uiOSPageCount);
+
+				if (iError)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes", __func__));
+				}
+			}
+#endif
+
+			/* Free the pages */
+			for (i = 0; i < uiOSPageCount; i++)
+			{
+				_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsOSPagesToFree[i]);
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	_DecrMemAllocStat_UmaPages(psPageArrayData,
+	                           (uiOSPageCount * PAGE_SIZE) - psPageArrayData->ui32CMAAdjustedPageCount,
+	                           psPageArrayData->uiPid);
+#endif
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated -= uiOSPageCount;
+	PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated);
+
+err_indices:
+	OSFreeMemNoStats(puiDevIndicesToFree);
+err_array:
+	if (ppsOSPagesToFree)
+	{
+		OSFreeMemNoStats(ppsOSPagesToFree);
+	}
+	return eError;
+}
+
+/* Free all the pages in a page array */
+static PVRSRV_ERROR
+_FreeOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	IMG_BOOL bSuccess;
+	IMG_UINT32 i;
+	IMG_UINT32 uiOSNumPages = psPageArrayData->uiTotalNumOSPages;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_UINT32 uiDevNumPages = uiOSNumPages >> uiOrder;
+	IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+
+	/* We really should have something to free before we call this */
+	PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiOSNumPages; i++)
+	{
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+			{
+				_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+			}
+		}
+		else
+		{
+			_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+		}
+	}
+#else
+	_DecrMemAllocStat_UmaPages(psPageArrayData, uiOSNumPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+	                           psPageArrayData->uiPid);
+#endif
+#endif
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE))
+	{
+		for (i = 0; i < uiDevNumPages; i++)
+		{
+			/* Scale Device page index to OS page idx */
+			IMG_UINT32 idx = i << uiOrder;
+
+			_PoisonDevicePage(psPageArrayData->psDevNode,
+			                  ppsPageArray[idx],
+			                  uiOrder,
+			                  psPageArrayData->ui32CPUCacheFlags,
+			                  PVRSRV_POISON_ON_FREE_VALUE);
+		}
+	}
+
+	/* Try to move the page array to the pool */
+	bSuccess = _PutPagesToPoolLocked(psPageArrayData->psDevNode,
+	                                 psPageArrayData->ui32CPUCacheFlags,
+	                                 psPageArrayData->ui32AllocFlags,
+	                                 ppsPageArray,
+	                                 uiOrder,
+	                                 uiOSNumPages);
+	if (bSuccess)
+	{
+		psPageArrayData->pagearray = NULL;
+		goto exit_ok;
+	}
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		for (i = 0; i < uiOSNumPages; i++)
+		{
+			if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+			{
+				_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+								uiDevPageSize,
+								uiOrder,
+								psPageArrayData->dmavirtarray[i],
+								psPageArrayData->dmaphysarray[i],
+								ppsPageArray[i]);
+
+			}
+			psPageArrayData->dmaphysarray[i] = (dma_addr_t)0;
+			psPageArrayData->dmavirtarray[i] = NULL;
+			ppsPageArray[i] = NULL;
+		}
+	}
+	else
+	{
+#if defined(CONFIG_X86)
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE))
+		{
+			int ret;
+
+			ret = set_pages_array_wb(ppsPageArray, uiOSNumPages);
+			if (ret)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes",
+						 __func__));
+			}
+		}
+#endif
+
+		for (i = 0; i < uiOSNumPages; i++)
+		{
+			_FreeOSPage(uiOrder, IMG_FALSE, psPageArrayData, ppsPageArray[i]);
+			ppsPageArray[i] = NULL;
+		}
+	}
+
+exit_ok:
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated = 0;
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRFreeZombiePagesOSMem(PMR_IMPL_ZOMBIEPAGES pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psZombiePageArray = pvPriv;
+
+	eError = _FreeOSPages(psZombiePageArray,
+	                      NULL,
+	                      0 /* Unused */);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	_FreeOSPagesArray(psZombiePageArray);
+
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+
+/* Extracts ui32ExtractPageCount of pages referenced in pai32ExtractIndices from the psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted pages information.
+ */
+static PVRSRV_ERROR
+_ExtractPages(PMR_OSPAGEARRAY_DATA *psSrcPageArrayData,
+			  IMG_UINT32 *pai32ExtractIndices,
+			  IMG_UINT32 ui32ExtractPageCount,
+			  PMR_OSPAGEARRAY_DATA** psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i, uiSubPageInOrder, uiOrder;
+	PMR_OSPAGEARRAY_DATA* psDstPageArrayData;
+
+	/* Alloc PMR_OSPAGEARRAY_DATA for the extracted pages */
+	eError = _AllocOSPageArray(psSrcPageArrayData->psDevNode,
+	                           (IMG_UINT64)ui32ExtractPageCount << psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->ui32AllocFlags,
+	                           psSrcPageArrayData->ui32CPUCacheFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_AllocOSPageArray failed in _ExtractPages"));
+		return eError;
+	}
+
+	uiOrder = psSrcPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+
+	/* Transfer from src pagearray to dst pagearray */
+	for (i = 0; i < ui32ExtractPageCount; i++)
+	{
+		IMG_UINT32 idxSrc = pai32ExtractIndices[i] << uiOrder;
+		IMG_UINT32 idxExtracted = i << uiOrder;
+
+		if (psSrcPageArrayData->pagearray[idxSrc] != NULL)
+		{
+			for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+			{
+				psDstPageArrayData->pagearray[idxExtracted + uiSubPageInOrder] =
+				    psSrcPageArrayData->pagearray[idxSrc + uiSubPageInOrder];
+
+				psSrcPageArrayData->pagearray[idxSrc + uiSubPageInOrder] = NULL;
+			}
+		}
+	}
+
+	/* Do the same for dmaphysarray and dmavirtarray if allocated with CMA */
+	if (BIT_ISSET(psSrcPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		for (i = 0; i < ui32ExtractPageCount; i++)
+		{
+			IMG_UINT32 idxSrc = pai32ExtractIndices[i] << uiOrder;
+			IMG_UINT32 idxDst = i << uiOrder;
+
+			if (psSrcPageArrayData->dmaphysarray[idxSrc] != (dma_addr_t)0 ||
+			    psSrcPageArrayData->dmavirtarray[idxSrc] != NULL)
+			{
+				for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+				{
+					psDstPageArrayData->dmaphysarray[idxDst + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmaphysarray[idxSrc + uiSubPageInOrder];
+					psDstPageArrayData->dmavirtarray[idxDst + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmavirtarray[idxSrc + uiSubPageInOrder];
+
+					psSrcPageArrayData->dmaphysarray[idxSrc + uiSubPageInOrder] = (dma_addr_t)0;
+					psSrcPageArrayData->dmavirtarray[idxSrc + uiSubPageInOrder] = NULL;
+				}
+			}
+		}
+	}
+
+	/* Update page counts */
+	psSrcPageArrayData->iNumOSPagesAllocated -= ui32ExtractPageCount << uiOrder;
+	psDstPageArrayData->iNumOSPagesAllocated += ui32ExtractPageCount << uiOrder;
+
+	*psOutPageArrayData = psDstPageArrayData;
+	return PVRSRV_OK;
+}
+
+/* Extracts all allocated pages referenced psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted
+ * pages information.
+ */
+static PVRSRV_ERROR
+_ExtractAllPages(PMR_OSPAGEARRAY_DATA *psSrcPageArrayData,
+				 PMR_OSPAGEARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	PMR_OSPAGEARRAY_DATA* psDstPageArrayData;
+	IMG_UINT32 uiPagesCopied = 0;
+
+	/* Alloc PMR_OSPAGEARRAY_DATA for the extracted pages */
+	eError = _AllocOSPageArray(psSrcPageArrayData->psDevNode,
+	                           (IMG_UINT64)psSrcPageArrayData->iNumOSPagesAllocated << psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->ui32AllocFlags,
+	                           psSrcPageArrayData->ui32CPUCacheFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_AllocOSPageArray failed in _ExtractPages"));
+		return eError;
+	}
+
+	/* Transfer from src pagearray to dst pagearray */
+	/* Iterate through all pages in psSrcPageArrayData but stop once
+	 * we have copied psSrcPageArrayData->iNumOSPagesAllocated pages to
+	 * psDstPageArrayData.
+	 */
+
+	for (i = 0; ((i < psSrcPageArrayData->uiTotalNumOSPages) &&
+	             (uiPagesCopied < psSrcPageArrayData->iNumOSPagesAllocated)); i++)
+	{
+		if (psSrcPageArrayData->pagearray[i] != NULL)
+		{
+			psDstPageArrayData->pagearray[uiPagesCopied] =
+			        psSrcPageArrayData->pagearray[i];
+
+			psSrcPageArrayData->pagearray[i] = NULL;
+
+			if (BIT_ISSET(psSrcPageArrayData->ui32AllocFlags, FLAG_DMA_CMA) &&
+			    (psSrcPageArrayData->dmaphysarray[i] != (dma_addr_t)0 ||
+			     psSrcPageArrayData->dmavirtarray[i] != NULL))
+			{
+				psDstPageArrayData->dmaphysarray[uiPagesCopied] =
+				    psSrcPageArrayData->dmaphysarray[i];
+				psDstPageArrayData->dmavirtarray[uiPagesCopied] =
+				    psSrcPageArrayData->dmavirtarray[i];
+
+				psSrcPageArrayData->dmaphysarray[i] = (dma_addr_t)0;
+				psSrcPageArrayData->dmavirtarray[i] = NULL;
+			}
+			uiPagesCopied++;
+		}
+	}
+	/* Update page counts */
+	psDstPageArrayData->iNumOSPagesAllocated = psSrcPageArrayData->iNumOSPagesAllocated;
+	psSrcPageArrayData->iNumOSPagesAllocated = 0;
+
+	*psOutPageArrayData = psDstPageArrayData;
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+/* Free pages from a page array.
+ * Takes care of mem stats and chooses correct free path depending on parameters. */
+static PVRSRV_ERROR
+_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			 IMG_UINT32 *pai32FreeIndices,
+			 IMG_UINT32 ui32FreePageCount)
+{
+	PVRSRV_ERROR eError;
+
+	/* Go the sparse or non-sparse path */
+	if (psPageArrayData->iNumOSPagesAllocated != psPageArrayData->uiTotalNumOSPages
+		|| pai32FreeIndices != NULL)
+	{
+		eError = _FreeOSPages_Sparse(psPageArrayData,
+									 pai32FreeIndices,
+									 ui32FreePageCount);
+	}
+	else
+	{
+		eError = _FreeOSPages_Fast(psPageArrayData);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_FreeOSPages_FreePages failed"));
+	}
+
+	_DumpPageArray(psPageArrayData->pagearray,
+	               psPageArrayData->uiTotalNumOSPages >>
+	              (psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT) );
+
+	return eError;
+}
+
+/*
+ *
+ * Implementation of callback functions
+ *
+ */
+
+/* Destruction function is called after last reference disappears,
+ * but before PMR itself is freed.
+ */
+static void
+PMRFinalizeOSMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+
+	/* We can't free pages until now. */
+	if (psOSPageArrayData->iNumOSPagesAllocated != 0)
+	{
+
+		eError = _FreeOSPages(psOSPageArrayData, NULL, 0);
+		PVR_LOG_IF_ERROR(eError, "_FreeOSPages");
+	}
+
+	_FreeOSPagesArray(psOSPageArrayData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static PVRSRV_ERROR PMRZombifyOSMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PMR_OSPAGEARRAY_DATA *psPageArrayData = pvPriv;
+	IMG_UINT32 uiNumOSPages = psPageArrayData->uiTotalNumOSPages;
+
+	BIT_SET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE);
+
+	/* no need to check for free indices as it's always all the memory we're
+	 * freeing */
+	if (psPageArrayData->iNumOSPagesAllocated != psPageArrayData->uiTotalNumOSPages)
+	{
+		/* _FreeOSPages_Sparse() path */
+
+		struct page *const *const ppsPageArray = psPageArrayData->pagearray;
+		const IMG_UINT32 uiFlags = psPageArrayData->ui32AllocFlags;
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_UINT32 i;
+
+		if (BIT_ISSET(uiFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (ppsPageArray[i] != NULL && !DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+		else
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (ppsPageArray[i] != NULL)
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+		const IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+		IMG_UINT32 uiNumDevPages = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+		IMG_UINT32 i, j, uiAllocatedNumPages = 0;
+
+		if (BIT_ISSET(uiFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumDevPages; i++)
+			{
+				IMG_UINT32 idx = i << uiOrder;
+				if (ppsPageArray[idx] != NULL)
+				{
+					uiAllocatedNumPages++;
+				}
+			}
+			uiAllocatedNumPages <<= uiOrder;
+		}
+		else
+		{
+			for (i = 0; i < uiNumDevPages; i++)
+			{
+				IMG_UINT32 idx = i << uiOrder;
+				if (ppsPageArray[idx] != NULL)
+				{
+					for (j = 0; j < (1<<uiOrder); j++)
+					{
+						uiAllocatedNumPages++;
+					}
+				}
+			}
+		}
+
+		_ZombifyMemAllocStat_UmaPages(
+		    uiAllocatedNumPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+		    psPageArrayData->uiPid
+		);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+	else
+	{
+		/* _FreeOSPages_Fast() path */
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		struct page *const *const ppsPageArray = psPageArrayData->pagearray;
+		IMG_UINT32 i;
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+		else
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+			}
+		}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+		_ZombifyMemAllocStat_UmaPages(
+		    uiNumOSPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+		    psPageArrayData->uiPid
+		);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#else /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Callback function for locking the system physical page addresses.
+ * This function must be called before the lookup address func. */
+static PVRSRV_ERROR
+PMRLockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+
+	if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		/* Allocate Memory for deferred allocation */
+		eError = _AllocOSPages(psOSPageArrayData, NULL, psOSPageArrayData->uiTotalNumOSPages);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	}
+
+	eError = PVRSRV_OK;
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv,
+                               PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	/* Just drops the refcount. */
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_OSPAGEARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+	*ppvZombiePages = NULL;
+#endif
+
+	if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND))
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (psOSPageArrayData->iNumOSPagesAllocated == 0)
+		{
+			*ppvZombiePages = NULL;
+			return PVRSRV_OK;
+		}
+
+		eError = _ExtractAllPages(psOSPageArrayData,
+		                          &psExtractedPagesPageArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractAllPages", e0);
+
+		if (psExtractedPagesPageArray)
+		{
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyOSMem(psExtractedPagesPageArray, NULL);
+			PVR_WARN_IF_ERROR(eError, "PMRZombifyOSMem");
+		}
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		/* Free Memory for deferred allocation */
+		eError = _FreeOSPages(psOSPageArrayData,
+							  NULL,
+							  0);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+#endif
+	}
+
+	return eError;
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+#endif
+}
+
+static INLINE IMG_BOOL IsOffsetValid(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData,
+                                     IMG_UINT32 ui32Offset)
+{
+	return (ui32Offset >> psOSPageArrayData->uiLog2DevPageSize) <
+	    psOSPageArrayData->uiTotalNumOSPages;
+}
+
+/* Determine PA for specified offset into page array. */
+static IMG_DEV_PHYADDR GetOffsetPA(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData,
+                                   IMG_UINT32 ui32Offset)
+{
+	IMG_UINT32 ui32PageIndex = ui32Offset >> PAGE_SHIFT;
+	IMG_UINT32 ui32InPageOffset = ui32Offset - (ui32PageIndex << PAGE_SHIFT);
+	IMG_DEV_PHYADDR sPA;
+
+	PVR_ASSERT(ui32InPageOffset < (1U << PAGE_SHIFT));
+
+	sPA.uiAddr = page_to_phys(psOSPageArrayData->pagearray[ui32PageIndex]);
+	sPA.uiAddr += ui32InPageOffset;
+
+	return sPA;
+}
+
+/* N.B. It is assumed that PMRLockSysPhysAddressesOSMem() is called _before_ this function! */
+static PVRSRV_ERROR
+PMRSysPhysAddrOSMem(PMR_IMPL_PRIVDATA pvPriv,
+					IMG_UINT32 ui32Log2PageSize,
+					IMG_UINT32 ui32NumOfPages,
+					IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					IMG_UINT64 ui64IPAPolicyValue,
+					IMG_UINT64 ui64IPAClearMask,
+#endif
+					IMG_BOOL *pbValid,
+					IMG_DEV_PHYADDR *psDevPAddr)
+{
+	const PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+	IMG_UINT32 uiIdx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (psOSPageArrayData->uiLog2DevPageSize < ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "for incompatible contiguity %u!",
+		         __func__,
+		         ui32Log2PageSize));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++)
+	{
+		if (pbValid[uiIdx])
+		{
+			PVR_LOG_RETURN_IF_FALSE(IsOffsetValid(psOSPageArrayData, puiOffset[uiIdx]),
+			                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+			psDevPAddr[uiIdx] = GetOffsetPA(psOSPageArrayData, puiOffset[uiIdx]);
+
+#if !defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+			/* this is just a precaution, normally this should be always
+			 * available */
+			if (psOSPageArrayData->ui64DmaMask)
+			{
+				if (psDevPAddr[uiIdx].uiAddr > psOSPageArrayData->ui64DmaMask)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: physical address"
+							" (%" IMG_UINT64_FMTSPECX ") out of allowable range"
+							" [0; %" IMG_UINT64_FMTSPECX "]", __func__,
+							psDevPAddr[uiIdx].uiAddr,
+							psOSPageArrayData->ui64DmaMask));
+					BUG();
+				}
+			}
+#endif
+#if defined(SUPPORT_STATIC_IPA)
+			/* Modify the physical address with the associated IPA values */
+			psDevPAddr[uiIdx].uiAddr &= ~ui64IPAClearMask;
+			psDevPAddr[uiIdx].uiAddr |= ui64IPAPolicyValue;
+#endif
+
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+typedef struct _PMR_OSPAGEARRAY_KERNMAP_DATA_ {
+	void *pvBase;
+	IMG_UINT32 ui32PageCount;
+	pgprot_t PageProps;
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	IMG_UINT32 ui32CpuCacheFlags;
+#endif
+} PMR_OSPAGEARRAY_KERNMAP_DATA;
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv,
+								 size_t uiOffset,
+								 size_t uiSize,
+								 void **ppvKernelAddressOut,
+								 IMG_HANDLE *phHandleOut,
+								 PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+	void *pvAddress;
+	pgprot_t prot = PAGE_KERNEL;
+	IMG_UINT32 ui32PageOffset=0;
+	size_t uiMapOffset=0;
+	IMG_UINT32 ui32PageCount = 0;
+	IMG_UINT32 uiLog2DevPageSize = psOSPageArrayData->uiLog2DevPageSize;
+	struct page **pagearray;
+	PMR_OSPAGEARRAY_KERNMAP_DATA *psData;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	OSWRLockAcquireRead(psOSPageArrayData->hPagesUnderMigrationRWLock);
+	if (psOSPageArrayData->iNumOSPagesUnderMigration != 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: Requested kernel mapping from PMR "
+		         "in process of page migration, retry.",
+		         __func__));
+		OSWRLockReleaseRead(psOSPageArrayData->hPagesUnderMigrationRWLock);
+		return PVRSRV_ERROR_RETRY;
+	}
+	OSWRLockReleaseRead(psOSPageArrayData->hPagesUnderMigrationRWLock);
+#endif
+
+	/*
+		Zero offset and size as a special meaning which means map in the
+		whole of the PMR, this is due to fact that the places that call
+		this callback might not have access to be able to determine the
+		physical size
+	*/
+	if ((uiOffset == 0) && (uiSize == 0))
+	{
+		ui32PageOffset = 0;
+		uiMapOffset = 0;
+		/* Page count = amount of OS pages */
+		ui32PageCount = psOSPageArrayData->iNumOSPagesAllocated;
+	}
+	else
+	{
+		size_t uiEndoffset;
+
+		ui32PageOffset = uiOffset >> uiLog2DevPageSize;
+		uiMapOffset = uiOffset - (ui32PageOffset << uiLog2DevPageSize);
+		uiEndoffset = uiOffset + uiSize - 1;
+		/* Add one as we want the count, not the offset */
+		/* Page count = amount of device pages (note uiLog2DevPageSize being used) */
+		ui32PageCount = (uiEndoffset >> uiLog2DevPageSize) + 1;
+		ui32PageCount -= ui32PageOffset;
+	}
+
+	switch (PVRSRV_CPU_CACHE_MODE(psOSPageArrayData->ui32CPUCacheFlags))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				prot = pgprot_noncached(prot);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				prot = pgprot_writecombine(prot);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+				break;
+
+		default:
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto e0;
+	}
+
+	pagearray = &psOSPageArrayData->pagearray[ui32PageOffset];
+
+	psData = OSAllocMem(sizeof(*psData));
+	if (psData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	pvAddress = pvr_vmap(pagearray, ui32PageCount, VM_MAP, prot);
+	if (pvAddress == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	*ppvKernelAddressOut = pvAddress + uiMapOffset;
+	psData->pvBase = pvAddress;
+	psData->ui32PageCount = ui32PageCount;
+	psData->PageProps = prot;
+	*phHandleOut = psData;
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	{
+		IMG_CPU_PHYADDR pvAddrPhy;
+		pvAddrPhy.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(*pagearray));
+		InsertMappingRecord(PMR_GetAnnotation(psOSPageArrayData->hPMR),
+							psOSPageArrayData->uiPid,
+							pvAddress,
+							pvAddrPhy,
+							psOSPageArrayData->ui32CPUCacheFlags,
+							uiMapOffset,
+							ui32PageCount);
+
+		psData->ui32CpuCacheFlags = psOSPageArrayData->ui32CPUCacheFlags;
+	}
+#endif
+
+	return PVRSRV_OK;
+	/*
+	  error exit paths follow
+	*/
+e1:
+	OSFreeMem(psData);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv,
+											 IMG_HANDLE hHandle)
+{
+	PMR_OSPAGEARRAY_KERNMAP_DATA *psData = hHandle;
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	{
+		IMG_CPU_PHYADDR pvAddrPhy;
+		pvAddrPhy.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(vmalloc_to_page(psData->pvBase)));
+		InsertUnMappingRecord(psData->pvBase,
+				pvAddrPhy,
+				psData->ui32CpuCacheFlags,
+				psData->ui32PageCount);
+	}
+#endif
+
+	pvr_vunmap(psData->pvBase, psData->ui32PageCount, psData->PageProps);
+	OSFreeMem(psData);
+}
+
+/*************************************************************************/ /*!
+@Function       PMRChangeSparseMemOSMem
+@Description    This function Changes the sparse mapping by allocating and
+                freeing of pages. It changes the GPU and CPU maps accordingly.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PMRChangeSparseMemOSMem(PMR_IMPL_PRIVDATA pPriv,
+						const PMR *psPMR,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 *pai32AllocIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+						PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+						IMG_UINT32 uiFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappingTable(psPMR);
+	PMR_OSPAGEARRAY_DATA *psPMRPageArrayData = (PMR_OSPAGEARRAY_DATA *)pPriv;
+	struct page **psPageArray = psPMRPageArrayData->pagearray;
+	void **psDMAVirtArray = psPMRPageArrayData->dmavirtarray;
+	dma_addr_t *psDMAPhysArray = psPMRPageArrayData->dmaphysarray;
+	IMG_UINT32 ui32AdtnlAllocPages = 0; /*<! Number of pages to alloc from the OS */
+	IMG_UINT32 ui32AdtnlFreePages = 0; /*<! Number of pages to free back to the OS */
+	IMG_UINT32 ui32CommonRequestCount = 0; /*<! Number of pages to move position in the page array */
+	IMG_UINT32 ui32Loop = 0;
+	IMG_UINT32 ui32Index = 0;
+	IMG_UINT32 uiAllocpgidx;
+	IMG_UINT32 uiFreepgidx;
+	IMG_UINT32 uiOrder = psPMRPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bCMA = BIT_ISSET(psPMRPageArrayData->ui32AllocFlags, FLAG_DMA_CMA);
+
+
+	/* Check SPARSE flags and calculate pages to allocate and free */
+	if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH))
+	{
+		ui32CommonRequestCount = (ui32AllocPageCount > ui32FreePageCount) ?
+				ui32FreePageCount : ui32AllocPageCount;
+
+		PDUMP_PANIC(PMR_DeviceNode(psPMR), SPARSEMEM_SWAP, "Request to swap alloc & free pages not supported");
+	}
+
+	if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC))
+	{
+		ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequestCount;
+	}
+	else
+	{
+		ui32AllocPageCount = 0;
+	}
+
+	if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE))
+	{
+		ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequestCount;
+	}
+	else
+	{
+		ui32FreePageCount = 0;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+
+	if (0 == (ui32CommonRequestCount || ui32AdtnlAllocPages || ui32AdtnlFreePages))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Missing parameters for number of pages to alloc/free",
+		         __func__));
+		return eError;
+	}
+
+	/* The incoming request is classified into two operations independent of
+	 * each other: alloc & free pages.
+	 * These operations can be combined with two mapping operations as well
+	 * which are GPU & CPU space mappings.
+	 *
+	 * From the alloc and free page requests, the net amount of pages to be
+	 * allocated or freed is computed. Pages that were requested to be freed
+	 * will be reused to fulfil alloc requests.
+	 *
+	 * The order of operations is:
+	 * 1. Allocate new pages from the OS
+	 * 2. Move the free pages from free request to alloc positions.
+	 * 3. Free the rest of the pages not used for alloc
+	 *
+	 * Alloc parameters are validated at the time of allocation
+	 * and any error will be handled then. */
+
+	/* Validate the free indices */
+	if (ui32FreePageCount)
+	{
+		if (NULL != pai32FreeIndices){
+
+			for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++)
+			{
+				uiFreepgidx = pai32FreeIndices[ui32Loop];
+
+				if (uiFreepgidx >= (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder))
+				{
+					eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+					goto e0;
+				}
+
+				if (NULL == (psPageArray[uiFreepgidx << uiOrder]))
+				{
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					PVR_DPF((PVR_DBG_ERROR,
+					         "%s: Trying to free non-allocated page",
+					         __func__));
+					goto e0;
+				}
+			}
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Given non-zero free count but missing indices array",
+			         __func__));
+			return eError;
+		}
+	}
+
+	/* Validate the alloc indices */
+	for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++)
+	{
+		uiAllocpgidx = pai32AllocIndices[ui32Loop];
+
+		if (uiAllocpgidx >= (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder))
+		{
+			eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+			goto e0;
+		}
+
+		if ((NULL != psPageArray[uiAllocpgidx << uiOrder]) ||
+		    (TRANSLATION_INVALID != psPMRMapTable->aui32Translation[uiAllocpgidx]))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to allocate already allocated page again",
+			         __func__));
+			goto e0;
+		}
+	}
+
+	ui32Loop = 0;
+
+	/* Allocate new pages from the OS */
+	if (0 != ui32AdtnlAllocPages)
+	{
+		eError = _AllocOSPages(psPMRPageArrayData, pai32AllocIndices, ui32AdtnlAllocPages);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "%s: New Addtl Allocation of pages failed",
+					 __func__));
+			goto e0;
+		}
+
+		psPMRMapTable->ui32NumPhysChunks += ui32AdtnlAllocPages;
+		/*Mark the corresponding pages of translation table as valid */
+		for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++)
+		{
+			psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop];
+		}
+	}
+
+
+	ui32Index = ui32Loop;
+
+	/* Move the corresponding free pages to alloc request */
+	for (ui32Loop = 0; ui32Loop < ui32CommonRequestCount; ui32Loop++, ui32Index++)
+	{
+		IMG_UINT32 uiOrderIdx;
+		uiAllocpgidx = pai32AllocIndices[ui32Index] << uiOrder;
+		uiFreepgidx  = pai32FreeIndices[ui32Loop] << uiOrder;
+
+		for (uiOrderIdx = 0; uiOrderIdx < (1 << uiOrder); uiOrderIdx++)
+		{
+			psPageArray[uiAllocpgidx + uiOrderIdx] = psPageArray[uiFreepgidx + uiOrderIdx];
+			if (bCMA)
+			{
+				psDMAVirtArray[uiAllocpgidx + uiOrderIdx] = psDMAVirtArray[uiFreepgidx + uiOrderIdx];
+				psDMAPhysArray[uiAllocpgidx + uiOrderIdx] = psDMAPhysArray[uiFreepgidx + uiOrderIdx];
+				psDMAVirtArray[uiFreepgidx + uiOrderIdx] = NULL;
+				psDMAPhysArray[uiFreepgidx + uiOrderIdx] = (dma_addr_t)0;
+			}
+			psPageArray[uiFreepgidx + uiOrderIdx] = NULL;
+		}
+
+		psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID;
+		psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
+	}
+
+	/* Free or zombie the additional free pages */
+	if (0 != ui32AdtnlFreePages)
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		PMR_OSPAGEARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+		eError = _ExtractPages(psPMRPageArrayData,
+		                       &pai32FreeIndices[ui32Loop],
+		                       ui32AdtnlFreePages,
+		                       &psExtractedPagesPageArray);
+		if (eError != PVRSRV_OK)
+		{
+			goto e0;
+		}
+
+		/* Zombify pages to get proper stats */
+		eError = PMRZombifyOSMem(psExtractedPagesPageArray, NULL);
+		PVR_LOG_IF_ERROR(eError, "psExtractedPagesPageArray");
+
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		eError = _FreeOSPages(psPMRPageArrayData,
+		                      &pai32FreeIndices[ui32Loop],
+		                      ui32AdtnlFreePages);
+		if (eError != PVRSRV_OK)
+		{
+			goto e0;
+		}
+#endif /* SUPPORT_PMR_PAGES_DEFERRED_FREE */
+		psPMRMapTable->ui32NumPhysChunks -= ui32AdtnlFreePages;
+		while (ui32Loop < ui32FreePageCount)
+		{
+			psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Loop]] = TRANSLATION_INVALID;
+			ui32Loop++;
+		}
+	}
+
+	eError = PVRSRV_OK;
+
+e0:
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMROSPFuncTab = {
+	.pfnLockPhysAddresses = &PMRLockSysPhysAddressesOSMem,
+	.pfnUnlockPhysAddresses = &PMRUnlockSysPhysAddressesOSMem,
+	.pfnDevPhysAddr = &PMRSysPhysAddrOSMem,
+	.pfnAcquireKernelMappingData = &PMRAcquireKernelMappingDataOSMem,
+	.pfnReleaseKernelMappingData = &PMRReleaseKernelMappingDataOSMem,
+	.pfnReadBytes = NULL,
+	.pfnWriteBytes = NULL,
+	.pfnChangeSparseMem = &PMRChangeSparseMemOSMem,
+	.pfnFinalize = &PMRFinalizeOSMem,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	.pfnFreeZombiePages = &PMRFreeZombiePagesOSMem,
+#endif
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = &PMRZombifyOSMem,
+#endif
+};
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+DLLIST_NODE *
+LinuxOSGetCPUMappingPrivateDataList(PMR *psPMR)
+{
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = PMRGetPrivateData(psPMR, &_sPMROSPFuncTab);
+	PVR_ASSERT(psOSPageArrayData);
+
+	return &psOSPageArrayData->sCpuMappingListHead;
+}
+#endif
+
+/* Wrapper around OS page allocation. */
+static PVRSRV_ERROR
+DoPageAlloc(PMR_OSPAGEARRAY_DATA *psPrivData,
+            IMG_UINT32 *puiAllocIndices,
+            IMG_UINT32 ui32NumPhysChunks,
+            IMG_UINT32 ui32NumLogicalChunks,
+            IMG_UINT32 ui32Log2AllocPageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Do we fill the whole page array or just parts (sparse)? */
+	if (ui32NumPhysChunks == ui32NumLogicalChunks)
+	{
+		/* Allocate the physical pages */
+		eError = _AllocOSPages(psPrivData,
+		                       NULL,
+		                       psPrivData->uiTotalNumOSPages >>
+		                       (ui32Log2AllocPageSize - PAGE_SHIFT));
+	}
+	else if (ui32NumPhysChunks != 0)
+	{
+		/* Allocate the physical pages */
+		eError = _AllocOSPages(psPrivData, puiAllocIndices,
+		                       ui32NumPhysChunks);
+	}
+
+	return eError;
+}
+
+static void _EncodeAllocationFlags(IMG_UINT32 uiLog2DevPageSize,
+	                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+	                               IMG_UINT32* ui32AllocFlags)
+{
+
+	if (PVRSRV_CHECK_OS_LINUX_PREFER_CMA(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_PREF_CMA);
+	}
+	/*
+	 * Use CMA framework if order is greater than OS page size; please note
+	 * that OSMMapPMRGeneric() has the same expectation as well.
+	 */
+	else if (uiLog2DevPageSize > PAGE_SHIFT)
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_DMA_CMA);
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_IS_MOVABLE);
+	}
+#endif
+
+	/* OnDemand? */
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_ONDEMAND);
+	}
+
+	/* Zero? */
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_ZERO);
+	}
+
+	/* Poison on alloc? */
+	if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_ALLOC);
+	}
+
+#if defined(DEBUG)
+	/* Poison on free? */
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_FREE);
+	}
+#endif
+
+	/* Indicate whether this is an allocation with default caching attribute (i.e cached) or not */
+	if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) ||
+		PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE);
+	}
+
+}
+
+void PhysmemGetOSRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                             IMG_UINT64 *pui64TotalSize,
+                             IMG_UINT64 *pui64FreeSize)
+{
+	struct sysinfo sMeminfo;
+	si_meminfo(&sMeminfo);
+
+	PVR_UNREFERENCED_PARAMETER(pvImplData);
+
+	*pui64TotalSize = sMeminfo.totalram * sMeminfo.mem_unit;
+	*pui64FreeSize = sMeminfo.freeram * sMeminfo.mem_unit;
+
+}
+
+PVRSRV_ERROR
+PhysmemNewOSRamBackedPMR(PHYS_HEAP *psPhysHeap,
+						 CONNECTION_DATA *psConnection,
+						 IMG_DEVMEM_SIZE_T uiSize,
+						 IMG_UINT32 ui32NumPhysChunks,
+						 IMG_UINT32 ui32NumLogicalChunks,
+						 IMG_UINT32 *puiAllocIndices,
+						 IMG_UINT32 uiLog2DevPageSize,
+						 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						 const IMG_CHAR *pszAnnotation,
+						 IMG_PID uiPid,
+						 PMR **ppsPMRPtr,
+						 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	PMR *psPMR;
+	struct _PMR_OSPAGEARRAY_DATA_ *psPrivData;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_UINT32 ui32CPUCacheFlags;
+	IMG_UINT32 ui32AllocFlags = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	IMG_UINT32 ui32ValidPageSizeMask = psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask;
+	IMG_UINT32 ui32PageSize = IMG_PAGE2BYTES32(uiLog2DevPageSize);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32PageSize & ui32ValidPageSizeMask, eError, errorOnParam);
+
+	/*
+	 * The host driver (but not guest) can still use this factory for firmware
+	 * allocations
+	 */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode) && PVRSRV_CHECK_FW_MAIN(uiFlags))
+	{
+		PVR_ASSERT(0);
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errorOnParam;
+	}
+
+	/* Select correct caching mode */
+	eError = DevmemCPUCacheMode(uiFlags, &ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnParam;
+	}
+
+	if (PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags))
+	{
+		ui32CPUCacheFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN;
+	}
+
+	_EncodeAllocationFlags(uiLog2DevPageSize, uiFlags, &ui32AllocFlags);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Reject sparse and non4k PMR marked for migrate */
+	if (BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE) &&
+	   ((ui32NumPhysChunks != ui32NumLogicalChunks|| ui32NumLogicalChunks > 1) ||
+	    uiLog2DevPageSize != PAGE_SHIFT)
+	   )
+	{
+		PVR_LOG_GOTO_WITH_ERROR("PhysmemNewOSRamBackedPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, errorOnParam);
+	}
+#endif
+
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	/* Overwrite flags and always zero pages that could go back to UM */
+	BIT_SET(ui32AllocFlags, FLAG_ZERO);
+	BIT_UNSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC);
+#endif
+
+	/* Physical allocation alignment is generally not supported except under
+	   very restrictive conditions, also there is a maximum alignment value
+	   which must not exceed the largest device page-size. If these are not
+	   met then fail the aligned-requested allocation */
+	if (uiLog2DevPageSize > PAGE_SHIFT)
+	{
+		IMG_UINT64 ui64InvalidSizeMask = ui32PageSize - 1;
+		IMG_UINT32 ui32PageSizeMask = psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask;
+		IMG_UINT32 ui32MaxPageSize = 1;
+
+		while (ui32PageSizeMask > 1)
+		{
+			ui32PageSizeMask >>= 1;
+			ui32MaxPageSize <<= 1;
+		}
+
+		if ((uiSize & ui64InvalidSizeMask) || (ui32PageSize > ui32MaxPageSize))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid PA alignment: size 0x%llx, align 0x%x",
+					__func__, uiSize, ui32PageSize));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+			goto errorOnParam;
+		}
+	}
+
+	/* Create Array structure that hold the physical pages */
+	eError = _AllocOSPageArray(psDevNode,
+							   uiSize,
+							   uiLog2DevPageSize,
+							   ui32AllocFlags,
+							   ui32CPUCacheFlags,
+							   uiPid,
+							   &psPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnAllocPageArray;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	if (psConnection)
+	{
+		psPrivData->psDRMFile = ((ENV_CONNECTION_DATA*)psConnection->hOsPrivateData)->psDRMFile;
+	}
+	else
+	{
+		psPrivData->psDRMFile = NULL;
+	}
+#endif
+#endif
+
+	if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		eError = DoPageAlloc(psPrivData, puiAllocIndices, ui32NumPhysChunks,
+		                     ui32NumLogicalChunks, uiLog2DevPageSize);
+		if (eError != PVRSRV_OK)
+		{
+			goto errorOnAllocPages;
+		}
+	}
+
+	/*
+	 * In this instance, we simply pass flags straight through.
+	 *
+	 * Generically, uiFlags can include things that control the PMR factory, but
+	 * we don't need any such thing (at the time of writing!), and our caller
+	 * specifies all PMR flags so we don't need to meddle with what was given to
+	 * us.
+	 */
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+
+	/*
+	 * Check no significant bits were lost in cast due to different bit widths
+	 * for flags
+	 */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		PDUMPCOMMENT(PhysHeapDeviceNode(psPhysHeap), "Deferred Allocation PMR (UMA)");
+	}
+
+	eError = PMRCreatePMR(psPhysHeap,
+						  uiSize,
+						  ui32NumPhysChunks,
+						  ui32NumLogicalChunks,
+						  puiAllocIndices,
+						  uiLog2DevPageSize,
+						  uiPMRFlags,
+						  pszAnnotation,
+						  &_sPMROSPFuncTab,
+						  psPrivData,
+						  PMR_TYPE_OSMEM,
+						  &psPMR,
+						  ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnCreate;
+	}
+
+	psPrivData->hPMR = psPMR;
+	*ppsPMRPtr = psPMR;
+
+	return PVRSRV_OK;
+
+errorOnCreate:
+	if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		eError2 = _FreeOSPages(psPrivData, NULL, 0);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+errorOnAllocPages:
+	_FreeOSPagesArray(psPrivData);
+
+errorOnAllocPageArray:
+errorOnParam:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.h
new file mode 100644
index 000000000000..2581cceeb219
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_osmem_linux.h
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux OS physmem implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_OSMEM_LINUX_H
+#define PHYSMEM_OSMEM_LINUX_H
+
+PVRSRV_ERROR LinuxInitPhysmem(void);
+void LinuxDeinitPhysmem(void);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+DLLIST_NODE *
+LinuxOSGetCPUMappingPrivateDataList(PMR *psPMR);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+int LinuxRegisterMigrateCallbacks(struct file* filp);
+void LinuxDeregisterMigrateCallbacks(struct file* filp);
+#endif
+#endif
+
+#endif /* PHYSMEM_OSMEM_LINUX_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.c
new file mode 100644
index 000000000000..00c02f16d3e2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.c
@@ -0,0 +1,992 @@
+/*************************************************************************/ /*!
+@Title          Physmem_test
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Single entry point for testing of page factories
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "physmem_test.h"
+#include "device.h"
+#include "syscommon.h"
+#include "pmr.h"
+#include "osfunc.h"
+#include "physmem.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+#include "pvrsrv.h"
+
+#define PHYSMEM_TEST_PAGES        2     /* Mem test pages */
+#define PHYSMEM_TEST_PASSES_MAX   1000  /* Limit number of passes to some reasonable value */
+
+
+/* Test patterns for mem test */
+
+static const IMG_UINT64 gui64Patterns[] = {
+	0,
+	0xffffffffffffffffULL,
+	0x5555555555555555ULL,
+	0xaaaaaaaaaaaaaaaaULL,
+	0x1111111111111111ULL,
+	0x2222222222222222ULL,
+	0x4444444444444444ULL,
+	0x8888888888888888ULL,
+	0x3333333333333333ULL,
+	0x6666666666666666ULL,
+	0x9999999999999999ULL,
+	0xccccccccccccccccULL,
+	0x7777777777777777ULL,
+	0xbbbbbbbbbbbbbbbbULL,
+	0xddddddddddddddddULL,
+	0xeeeeeeeeeeeeeeeeULL,
+	0x7a6c7258554e494cULL,
+};
+
+static const IMG_UINT32 gui32Patterns[] = {
+	0,
+	0xffffffffU,
+	0x55555555U,
+	0xaaaaaaaaU,
+	0x11111111U,
+	0x22222222U,
+	0x44444444U,
+	0x88888888U,
+	0x33333333U,
+	0x66666666U,
+	0x99999999U,
+	0xccccccccU,
+	0x77777777U,
+	0xbbbbbbbbU,
+	0xddddddddU,
+	0xeeeeeeeeU,
+	0x7a6c725cU,
+};
+
+static const IMG_UINT16 gui16Patterns[] = {
+	0,
+	0xffffU,
+	0x5555U,
+	0xaaaaU,
+	0x1111U,
+	0x2222U,
+	0x4444U,
+	0x8888U,
+	0x3333U,
+	0x6666U,
+	0x9999U,
+	0xccccU,
+	0x7777U,
+	0xbbbbU,
+	0xddddU,
+	0xeeeeU,
+	0x7a6cU,
+};
+
+static const IMG_UINT8 gui8Patterns[] = {
+	0,
+	0xffU,
+	0x55U,
+	0xaaU,
+	0x11U,
+	0x22U,
+	0x44U,
+	0x88U,
+	0x33U,
+	0x66U,
+	0x99U,
+	0xccU,
+	0x77U,
+	0xbbU,
+	0xddU,
+	0xeeU,
+	0x6cU,
+};
+
+static PVRSRV_ERROR
+PMRContiguousSparseMappingTest(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError, eError1;
+	PHYS_HEAP *psHeap;
+	PHYS_HEAP_POLICY psHeapPolicy;
+
+	PMR *psPMR = NULL;
+	PMR *psSpacingPMR = NULL, *psSecondSpacingPMR = NULL;
+	IMG_UINT32 aui32MappingTableFirstAlloc[4] = {0,1,2,3};
+	IMG_UINT32 aui32MappingTableSecondAlloc[8] = {4,5,6,7,8,9,10,11};
+	IMG_UINT32 aui32MappingTableThirdAlloc[4] = {12,13,14,15};
+	IMG_UINT32 ui32NoMappingTable = 0;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_BOOL *pbValid;
+	IMG_DEV_PHYADDR *apsDevPAddr;
+	IMG_UINT32 ui32NumOfPages = 16;
+	size_t uiMappedSize, uiPageSize;
+	IMG_UINT32 i, uiAttempts;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	eError = PhysHeapAcquireByID(PVRSRV_GET_PHYS_HEAP_HINT(uiFlags),
+	                             psDeviceNode,
+	                             &psHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorReturn);
+
+	psHeapPolicy = PhysHeapGetPolicy(psHeap);
+
+	PhysHeapRelease(psHeap);
+
+	/* If this is the case then it's not supported and so don't attempt the test */
+	if (psHeapPolicy != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	{
+		return PVRSRV_OK;
+	}
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate OS memory for PMR page list */
+	apsDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+	PVR_LOG_RETURN_IF_NOMEM(apsDevPAddr, "OSAllocMem");
+
+	/* Allocate OS memory for PMR page state */
+	pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrorFreePMRPageListMem);
+
+	/* Allocate OS memory for write buffer */
+	pcWriteBuffer = OSAllocMem(uiPageSize * ui32NumOfPages);
+	PVR_LOG_GOTO_IF_NOMEM(pcWriteBuffer, eError, ErrorFreePMRPageStateMem);
+	OSCachedMemSet(pcWriteBuffer, 0xF, uiPageSize);
+
+	/* Allocate OS memory for read buffer */
+	pcReadBuffer = OSAllocMem(uiPageSize * ui32NumOfPages);
+	PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+	/* Allocate Sparse PMR with SPARSE | READ | WRITE | UNCACHED_WC attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/*
+	 * Construct a sparse PMR attempting to ensure the allocations
+	 * are physically non contiguous but sequentially placed in the mapping
+	 * table.
+	 */
+	for (uiAttempts = 3; uiAttempts > 0; uiAttempts--)
+	{
+		/* Allocate a sparse PMR from given physical heap - CPU/GPU/FW */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										4,
+										ui32NumOfPages,
+										aui32MappingTableFirstAlloc,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorFreeReadBuffer;
+		}
+
+		/* Allocate some memory from the same physheap so that we can ensure
+		 * the allocations aren't linear
+		 */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										1,
+										1,
+										&ui32NoMappingTable,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psSpacingPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorUnrefPMR;
+		}
+
+		/* Allocate 8 more physical pages on the Sparse PMR */
+		eError = PMR_ChangeSparseMem(psPMR,
+									 8,
+									 aui32MappingTableSecondAlloc,
+									 0,
+									 NULL,
+									 uiFlags | SPARSE_RESIZE_ALLOC);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_ChangeSparseMem", ErrorUnrefSpacingPMR);
+
+		/* Allocate some more memory from the same physheap so that we can ensure
+		 * the allocations aren't linear
+		 */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										1,
+										1,
+										&ui32NoMappingTable,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psSecondSpacingPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorUnrefSpacingPMR;
+		}
+
+		/* Allocate final 4 physical pages on the Sparse PMR */
+		eError = PMR_ChangeSparseMem(psPMR,
+									 4,
+									 aui32MappingTableThirdAlloc,
+									 0,
+									 NULL,
+									 uiFlags | SPARSE_RESIZE_ALLOC);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_ChangeSparseMem", ErrorUnrefSecondSpacingPMR);
+
+		/*
+		 * Check we have in fact managed to obtain a PMR with non contiguous
+		 * physical pages.
+		 */
+		eError = PMRLockSysPhysAddresses(psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to lock PMR"));
+			goto ErrorUnrefSecondSpacingPMR;
+		}
+
+		/* Get the Device physical addresses of the pages */
+		eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), ui32NumOfPages, 0, apsDevPAddr, pbValid, CPU_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to map PMR pages into device physical addresses"));
+			goto ErrorUnlockPhysAddresses;
+		}
+
+		{
+			IMG_BOOL bPhysicallyContiguous = IMG_TRUE;
+			IMG_DEV_PHYADDR sPrevDevPAddr = apsDevPAddr[0];
+			for (i = 1; i < ui32NumOfPages && bPhysicallyContiguous; i++)
+			{
+				if (apsDevPAddr[i].uiAddr != sPrevDevPAddr.uiAddr + uiPageSize)
+				{
+					bPhysicallyContiguous = IMG_FALSE;
+				}
+				sPrevDevPAddr = apsDevPAddr[i];
+			}
+
+			if (bPhysicallyContiguous)
+			{
+				/* We haven't yet managed to create the mapping scenario we
+				 * require: unwind and attempt again.
+				 */
+				eError1 = PMRUnlockSysPhysAddresses(psPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+				}
+				eError1 = PMRUnrefPMR(psPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+				}
+				eError1 = PMRUnrefPMR(psSpacingPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free Spacing PMR"));
+				}
+				eError1 = PMRUnrefPMR(psSecondSpacingPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free Second Spacing PMR"));
+				}
+			} else {
+				/* We have the scenario, break out of the attempt loop */
+				break;
+			}
+		}
+	}
+
+	if (uiAttempts == 0)
+	{
+		/* We can't create the scenario, very unlikely this would happen */
+		PVR_LOG_GOTO_IF_ERROR(PVRSRV_ERROR_MEMORY_TEST_FAILED,
+		                      "Unable to create Non Contiguous PMR scenario",
+		                      ErrorFreeReadBuffer);
+	}
+
+	/* We have the PMR scenario to test, now attempt to map the whole PMR,
+	 * write and then read from it
+	 */
+	eError = PMRAcquireSparseKernelMappingData(psPMR, 0, ui32NumOfPages * uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	OSCachedMemCopyWMB(pvKernAddr, pcWriteBuffer, ui32NumOfPages * uiPageSize);
+
+	eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+	PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+	/*
+	 * Release and reacquire the mapping to exercise the mapping paths
+	 */
+	eError = PMRAcquireSparseKernelMappingData(psPMR, 0, ui32NumOfPages * uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	OSCachedMemSetWMB(pcReadBuffer, 0x0, ui32NumOfPages * uiPageSize);
+	OSCachedMemCopyWMB(pcReadBuffer, pvKernAddr, ui32NumOfPages * uiPageSize);
+
+	eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+	PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+	for (i = 0; i < ui32NumOfPages * uiPageSize; i++)
+	{
+		if (pcReadBuffer[i] != pcWriteBuffer[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)! @ %u",
+			         __func__, pcReadBuffer[i], pcWriteBuffer[i], i));
+			eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+			goto ErrorUnlockPhysAddresses;
+		}
+	}
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it */
+	eError1 = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+	}
+
+ErrorUnrefSecondSpacingPMR:
+	eError1 = PMRUnrefPMR(psSecondSpacingPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free Second Spacing PMR"));
+	}
+ErrorUnrefSpacingPMR:
+	eError1 = PMRUnrefPMR(psSpacingPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free Spacing PMR"));
+	}
+ErrorUnrefPMR:
+	eError1 = PMRUnrefPMR(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+	}
+
+ErrorFreeReadBuffer:
+	OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+	OSFreeMem(pcWriteBuffer);
+ErrorFreePMRPageStateMem:
+	OSFreeMem(pbValid);
+ErrorFreePMRPageListMem:
+	OSFreeMem(apsDevPAddr);
+ErrorReturn:
+	return eError;
+}
+
+/* Test for PMR factory validation */
+static PVRSRV_ERROR
+PMRValidationTest(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError, eError1;
+	IMG_UINT32 i = 0, j = 0, ui32Index = 0;
+	IMG_UINT32 *pui32MappingTable = NULL;
+	PMR *psPMR = NULL;
+	IMG_BOOL *pbValid;
+	IMG_DEV_PHYADDR *apsDevPAddr;
+	IMG_UINT32 ui32NumOfPages = 10, ui32NumOfPhysPages = 5;
+	size_t uiMappedSize, uiPageSize;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate OS memory for PMR page list */
+	apsDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+	PVR_LOG_RETURN_IF_NOMEM(apsDevPAddr, "OSAllocMem");
+
+	/* Allocate OS memory for PMR page state */
+	pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrorFreePMRPageListMem);
+
+	/* Allocate OS memory for write buffer */
+	pcWriteBuffer = OSAllocMem(uiPageSize);
+	PVR_LOG_GOTO_IF_NOMEM(pcWriteBuffer, eError, ErrorFreePMRPageStateMem);
+	OSCachedMemSet(pcWriteBuffer, 0xF, uiPageSize);
+
+	/* Allocate OS memory for read buffer */
+	pcReadBuffer = OSAllocMem(uiPageSize);
+	PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+	/* Allocate OS memory for mapping table */
+	pui32MappingTable = (IMG_UINT32 *)OSAllocMem(ui32NumOfPhysPages * sizeof(*pui32MappingTable));
+	PVR_LOG_GOTO_IF_NOMEM(pui32MappingTable, eError, ErrorFreeReadBuffer);
+
+	/* Pages having even index will have physical backing in PMR */
+	for (ui32Index=0; ui32Index < ui32NumOfPages; ui32Index+=2)
+	{
+		pui32MappingTable[i++] = ui32Index;
+	}
+
+	/* Allocate Sparse PMR with SPARSE | READ | WRITE | UNCACHED_WC attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/* Allocate a sparse PMR from given physical heap - CPU/GPU/FW */
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									ui32NumOfPages * uiPageSize,
+									ui32NumOfPhysPages,
+									ui32NumOfPages,
+									pui32MappingTable,
+									OSGetPageShift(),
+									uiFlags,
+									sizeof("PMR ValidationTest"),
+									"PMR ValidationTest",
+									OSGetCurrentClientProcessIDKM(),
+									&psPMR,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+		goto ErrorFreeMappingTable;
+	}
+
+	/* Check whether allocated PMR can be locked and obtain physical addresses
+	 * of underlying memory pages.
+	 */
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to lock PMR"));
+		goto ErrorUnrefPMR;
+	}
+
+	/* Get the Device physical addresses of the pages */
+	eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), ui32NumOfPages, 0, apsDevPAddr, pbValid, CPU_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to map PMR pages into device physical addresses"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	/* Check whether device address of each physical page is OS PAGE_SIZE aligned */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			if ((apsDevPAddr[i].uiAddr & OSGetPageMask()) != 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Physical memory of PMR is not page aligned"));
+				eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+				goto ErrorUnlockPhysAddresses;
+			}
+		}
+	}
+
+	/* Acquire kernel virtual address of each physical page and write to it
+	 * and then release it.
+	 */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMR, (i * uiPageSize), uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+				goto ErrorUnlockPhysAddresses;
+			}
+			OSCachedMemCopyWMB(pvKernAddr, pcWriteBuffer, OSGetPageSize());
+
+			eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+			PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+		}
+	}
+
+	/* Acquire kernel virtual address of each physical page and read
+	 * from it and check where contents are intact.
+	 */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMR, (i * uiPageSize), uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+				goto ErrorUnlockPhysAddresses;
+			}
+			OSCachedMemSetWMB(pcReadBuffer, 0x0, uiPageSize);
+			OSCachedMemCopyWMB(pcReadBuffer, pvKernAddr, uiMappedSize);
+
+			eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+			PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+			for (j = 0; j < uiPageSize; j++)
+			{
+				if (pcReadBuffer[j] != pcWriteBuffer[j])
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+					         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!",
+					         __func__, pcReadBuffer[j], pcWriteBuffer[j]));
+					eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+					goto ErrorUnlockPhysAddresses;
+				}
+			}
+		}
+	}
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it */
+	eError1 = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+	}
+
+ErrorUnrefPMR:
+	eError1 = PMRUnrefPMR(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+	}
+ErrorFreeMappingTable:
+	OSFreeMem(pui32MappingTable);
+ErrorFreeReadBuffer:
+	OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+	OSFreeMem(pcWriteBuffer);
+ErrorFreePMRPageStateMem:
+	OSFreeMem(pbValid);
+ErrorFreePMRPageListMem:
+	OSFreeMem(apsDevPAddr);
+
+	return eError;
+}
+
+#define DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, Patterns, NumOfPatterns, Error, ptr, i) \
+	for (i = 0; i < NumOfPatterns; i++) \
+	{ \
+		/* Write pattern */ \
+		for (ptr = StartAddr; ptr < EndAddr; ptr++) \
+		{ \
+			*ptr = Patterns[i]; \
+		} \
+		\
+		/* Read back and validate pattern */ \
+		for (ptr = StartAddr; ptr < EndAddr ; ptr++) \
+		{ \
+			if (*ptr != Patterns[i]) \
+			{ \
+				Error = PVRSRV_ERROR_MEMORY_TEST_FAILED; \
+				break; \
+			} \
+		} \
+		\
+		if (Error != PVRSRV_OK) \
+		{ \
+			break; \
+		} \
+	}
+
+static PVRSRV_ERROR
+TestPatternU8(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT8 *StartAddr = (IMG_UINT8 *) pvKernAddr;
+	IMG_UINT8 *EndAddr = ((IMG_UINT8 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT8));
+	IMG_UINT8 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT8)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui8Patterns, sizeof(gui8Patterns)/sizeof(IMG_UINT8), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!",
+		         __func__, *p, gui8Patterns[i]));
+	}
+
+	return eError;
+}
+
+
+static PVRSRV_ERROR
+TestPatternU16(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT16 *StartAddr = (IMG_UINT16 *) pvKernAddr;
+	IMG_UINT16 *EndAddr = ((IMG_UINT16 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT16));
+	IMG_UINT16 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT16)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui16Patterns, sizeof(gui16Patterns)/sizeof(IMG_UINT16), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%hx), expected (0x%hx)!",
+		         __func__, *p, gui16Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestPatternU32(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT32 *StartAddr = (IMG_UINT32 *) pvKernAddr;
+	IMG_UINT32 *EndAddr = ((IMG_UINT32 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT32));
+	IMG_UINT32 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT32)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui32Patterns, sizeof(gui32Patterns)/sizeof(IMG_UINT32), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%x), expected (0x%x)!",
+		         __func__, *p, gui32Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestPatternU64(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT64 *StartAddr = (IMG_UINT64 *) pvKernAddr;
+	IMG_UINT64 *EndAddr = ((IMG_UINT64 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT64));
+	IMG_UINT64 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT64)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui64Patterns, sizeof(gui64Patterns)/sizeof(IMG_UINT64), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%llx), expected (0x%llx)!",
+		         __func__, *p, gui64Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestSplitCacheline(void *pvKernAddr, size_t uiMappedSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	size_t uiCacheLineSize;
+	size_t uiBlockSize;
+	size_t j;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_UINT8 *StartAddr = (IMG_UINT8 *) pvKernAddr;
+	IMG_UINT8 *EndAddr, *p;
+
+	uiCacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+
+	if (uiCacheLineSize > 0)
+	{
+		uiBlockSize = (uiCacheLineSize * 2)/3; /* split cacheline */
+
+		pcWriteBuffer = OSAllocMem(uiBlockSize);
+		PVR_LOG_RETURN_IF_NOMEM(pcWriteBuffer, "OSAllocMem");
+
+		/* Fill the write buffer with test data, 0xAB*/
+		OSCachedMemSet(pcWriteBuffer, 0xAB, uiBlockSize);
+
+		pcReadBuffer = OSAllocMem(uiBlockSize);
+		PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+		/* Fit only complete blocks in uiMappedSize, ignore leftover bytes */
+		EndAddr = StartAddr + (uiBlockSize * (uiMappedSize / uiBlockSize));
+
+		/* Write blocks into the memory */
+		for (p = StartAddr; p < EndAddr; p += uiBlockSize)
+		{
+			OSCachedMemCopy(p, pcWriteBuffer, uiBlockSize);
+		}
+
+		/* Read back blocks and check */
+		for (p = StartAddr; p < EndAddr; p += uiBlockSize)
+		{
+			OSCachedMemCopy(pcReadBuffer, p, uiBlockSize);
+
+			for (j = 0; j < uiBlockSize; j++)
+			{
+				if (pcReadBuffer[j] != pcWriteBuffer[j])
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!", __func__, pcReadBuffer[j], pcWriteBuffer[j]));
+					eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+					goto ErrorMemTestFailed;
+				}
+			}
+		}
+
+ErrorMemTestFailed:
+		OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+		OSFreeMem(pcWriteBuffer);
+	}
+
+	return eError;
+}
+
+/* Memory test - writes and reads back different patterns to memory and validate the same */
+static PVRSRV_ERROR
+MemTestPatterns(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32MappingTable = 0;
+	PMR *psPMR = NULL;
+	size_t uiMappedSize, uiPageSize;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate PMR with READ | WRITE | WRITE_COMBINE attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+			   PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+			   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/*Allocate a PMR from given physical heap */
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiPageSize * PHYSMEM_TEST_PAGES,
+									1,
+									1,
+									&ui32MappingTable,
+									OSGetPageShift(),
+									uiFlags,
+									sizeof("PMR PhysMemTest"),
+									"PMR PhysMemTest",
+									OSGetCurrentClientProcessIDKM(),
+									&psPMR,
+									PDUMP_NONE,
+									NULL);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewRamBackedPMR");
+
+	/* Check whether allocated PMR can be locked and obtain physical
+	 * addresses of underlying memory pages.
+	 */
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", ErrorUnrefPMR);
+
+	/* Map the physical page(s) into kernel space, acquire kernel mapping
+	 * for PMR.
+	 */
+	eError = PMRAcquireKernelMappingData(psPMR, 0, uiPageSize * PHYSMEM_TEST_PAGES, &pvKernAddr, &uiMappedSize, &hPrivData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", ErrorUnlockPhysAddresses);
+
+	PVR_ASSERT((uiPageSize * PHYSMEM_TEST_PAGES) == uiMappedSize);
+
+	/* Test various patterns */
+	eError = TestPatternU64(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU32(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU16(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU8(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	/* Test split cachelines */
+	eError = TestSplitCacheline(pvKernAddr, uiMappedSize);
+
+ErrorReleaseKernelMappingData:
+	(void) PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it, ignore returned value */
+	(void) PMRUnlockSysPhysAddresses(psPMR);
+ErrorUnrefPMR:
+	(void) PMRUnrefPMR(psPMR);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+PhysMemTestRun(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags, IMG_UINT32 ui32Passes)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	/* PMR validation test */
+	eError = PMRValidationTest(psDeviceNode, uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR Contiguous PhysHeap self test failed! %"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         __func__,
+		         uiFlags));
+		return eError;
+	}
+
+	eError = PMRContiguousSparseMappingTest(psDeviceNode, uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR Non-contiguous PhysHeap self test failed! %"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         __func__,
+		         uiFlags));
+		return eError;
+	}
+
+
+	for (i = 0; i < ui32Passes; i++)
+	{
+		/* Mem test */
+		eError = MemTestPatterns(psDeviceNode, uiFlags);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: [Pass#%u] MemTestPatterns failed!",
+			         __func__, i));
+			break;
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysMemTest(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvDevConfig, IMG_UINT32 ui32MemTestPasses)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = pvDevConfig;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* validate memtest passes requested */
+	ui32MemTestPasses = (ui32MemTestPasses > PHYSMEM_TEST_PASSES_MAX)? PHYSMEM_TEST_PASSES_MAX : ui32MemTestPasses;
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		PHYS_HEAP_CONFIG *psHeapConfig = &psDevConfig->pasPhysHeaps[i];
+
+		if (psHeapConfig->ui32UsageFlags & PHYS_HEAP_USAGE_GPU_LOCAL)
+		{
+			/* GPU local mem (should be only up to 1 heap) */
+			eError = PhysMemTestRun(psDeviceNode, PHYS_HEAP_USAGE_GPU_LOCAL, ui32MemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "GPU local memory test failed!", ErrorPhysMemTestEnd);
+		}
+
+		if (psHeapConfig->ui32UsageFlags & PHYS_HEAP_USAGE_CPU_LOCAL)
+		{
+			/* CPU local mem (should be only up to 1 heap) */
+			eError = PhysMemTestRun(psDeviceNode, PHYS_HEAP_USAGE_CPU_LOCAL, ui32MemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "CPU local memory test failed!", ErrorPhysMemTestEnd);
+		}
+	}
+
+
+ErrorPhysMemTestEnd:
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PhysMemTest: Failed."));
+	}
+	else
+	{
+		PVR_LOG(("PhysMemTest: Passed."));
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.h
new file mode 100644
index 000000000000..75795afe9cab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/physmem_test.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@Title          Physmem test header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for single entry point for testing of page factories
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_TEST_H
+#define SRVSRV_PHYSMEM_TEST_H
+
+#include "device.h"
+
+/*
+ * PhysMemTest
+ */
+PVRSRV_ERROR
+PhysMemTest(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvDevConfig, IMG_UINT32 ui32MemTestPasses);
+
+#endif /* SRVSRV_PHYSMEM_TEST_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.c
new file mode 100644
index 000000000000..e0c74b4b508d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.c
@@ -0,0 +1,83 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux environment PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/string.h>
+
+#include "img_defs.h"
+
+#include "pmr_env.h"
+#include "physmem_dmabuf_internal.h"
+
+void
+PMREnvInitialize(PMR_ENV *psPMREnv)
+{
+	memset(psPMREnv, 0, sizeof(*psPMREnv));
+}
+
+void
+PMREnvFinalize(PMR_ENV *psPMREnv)
+{
+	void *pvDmaBufExportData;
+
+	pvDmaBufExportData = smp_load_acquire(&psPMREnv->pvDmaBufExportData);
+	if (pvDmaBufExportData)
+		PhysmemDmaBufExportFinalize(pvDmaBufExportData);
+}
+
+void
+PMREnvDmaBufSetExportData(PMR *psPMR, void *pvData)
+{
+	PMR_ENV *psPMREnv = PMREnvGetData(psPMR);
+
+	smp_store_release(&psPMREnv->pvDmaBufExportData, pvData);
+}
+
+void *
+PMREnvDmaBufGetExportData(PMR *psPMR)
+{
+	PMR_ENV *psPMREnv = PMREnvGetData(psPMR);
+	void *pvDmaBufExportData;
+
+	pvDmaBufExportData = smp_load_acquire(&psPMREnv->pvDmaBufExportData);
+
+	return pvDmaBufExportData;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.h
new file mode 100644
index 000000000000..52a172e80b82
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_env.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          Environment PMR functions and data
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Environment specific PMR functions and data
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PMR_ENV_H)
+#define PMR_ENV_H
+
+#include "pvrsrv_memallocflags.h"
+#include "powervr/mem_types.h"
+#include "pmr_impl.h"
+
+/* Structures, functions and definitions shared with the generic PMR code */
+struct _PMR_ENV_
+{
+	/*
+	 * The smp_store_release and smp_load_acquire functions are used
+	 * to access this field, in order to avoid taking locks, where
+	 * possible.
+	 */
+	void *pvDmaBufExportData;
+};
+
+typedef struct _PMR_ENV_ PMR_ENV;
+
+#define	DECLARE_PMR_ENV_DATA(name) PMR_ENV name;
+
+/* The generic PMR code calls these functions, via the macros that follow */
+void PMREnvInitialize(PMR_ENV *psPMREnv);
+void PMREnvFinalize(PMR_ENV *psPMREnv);
+
+#define	PMR_ENV_INITIALIZE(psPMR, sEnvData) PMREnvInitialize(&psPMR->sEnvData)
+#define	PMR_ENV_FINALIZE(psPMR, sEnvData) PMREnvFinalize(&psPMR->sEnvData)
+
+/* The generic PMR code provides this function, using the macro that follows */
+PMR_ENV *PMREnvGetData(PMR *psPMR);
+
+#define	DEFINE_PMR_ENV_GET_DATA(psPMR, sEnvData) \
+	PMR_ENV *PMREnvGetData(PMR *psPMR) { return &psPMR->sEnvData; }
+
+/*
+ * Structures, functions and definitions used by the environment specific
+ * PMR code.
+ */
+
+void
+PMREnvDmaBufSetExportData(PMR *psPMR, void *pvData);
+
+void *
+PMREnvDmaBufGetExportData(PMR *psPMR);
+
+#endif /* !defined(PMR_ENV_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_os.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_os.c
new file mode 100644
index 000000000000..63645c930dba
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pmr_os.c
@@ -0,0 +1,815 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux OS PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/version.h>
+#include <linux/pfn_t.h>
+#include <linux/pfn.h>
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "devicemem_server_utils.h"
+#include "pmr.h"
+#include "pmr_os.h"
+#include "physmem_osmem_linux.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION_DEBUG)
+#define MIGRATE_DBG_LOG(x) PVR_DPF(x);
+#else
+#define MIGRATE_DBG_LOG(...)
+#endif
+
+typedef struct _PMR_OS_CPU_MAPPING_
+{
+	DLLIST_NODE sListNode;
+	struct vm_area_struct *ps_vma;
+} PMR_OS_CPU_MAPPING;
+
+
+static PVRSRV_ERROR
+PMROSLinkCPUMapping(PMR *psPMR, struct vm_area_struct *psVMArea)
+{
+	PMR_OS_CPU_MAPPING *psCPUMapping;
+	DLLIST_NODE *psListHead;
+
+	PMRLockHeldAssert(psPMR);
+
+	psCPUMapping = OSAllocMem(sizeof(*psCPUMapping));
+	PVR_RETURN_IF_NOMEM(psCPUMapping);
+
+	psCPUMapping->ps_vma = psVMArea;
+
+	psListHead = LinuxOSGetCPUMappingPrivateDataList(psPMR);
+	PVR_ASSERT(psListHead);
+
+	dllist_add_to_head(psListHead, &psCPUMapping->sListNode);
+
+	return PVRSRV_OK;
+}
+
+static void
+PMROSUnlinkCPUMapping(PMR *psPMR, struct vm_area_struct *psTargetVMArea)
+{
+	PDLLIST_NODE pNext, pNode;
+	DLLIST_NODE *psListHead;
+	PMR_OS_CPU_MAPPING *psExtractedMapping = NULL;
+
+	PMRLockHeldAssert(psPMR);
+
+	psListHead = LinuxOSGetCPUMappingPrivateDataList(psPMR);
+	PVR_ASSERT(psListHead);
+
+	dllist_foreach_node(psListHead, pNode, pNext)
+	{
+		PMR_OS_CPU_MAPPING *psCheckMapping = IMG_CONTAINER_OF(pNode,
+		                                                      PMR_OS_CPU_MAPPING,
+		                                                      sListNode);
+		if (psTargetVMArea == psCheckMapping->ps_vma)
+		{
+			psExtractedMapping = psCheckMapping;
+			break;
+		}
+	}
+
+	PVR_ASSERT(psExtractedMapping != NULL);
+
+	if (psExtractedMapping)
+	{
+		dllist_remove_node(&psExtractedMapping->sListNode);
+		OSFreeMem(psExtractedMapping);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unable to find vm_area_struct", __func__));
+	}
+}
+#endif
+
+/*
+ * x86_32:
+ * Use vm_insert_page because remap_pfn_range has issues when mapping HIGHMEM
+ * pages with default memory attributes; these HIGHMEM pages are skipped in
+ * set_pages_array_[uc,wc] during allocation; see reserve_pfn_range().
+ * Also vm_insert_page is faster.
+ *
+ * x86_64:
+ * Use vm_insert_page because it is faster.
+ *
+ * Other platforms:
+ * Use remap_pfn_range by default because it does not issue a cache flush.
+ * It is known that ARM32 benefits from this. When other platforms become
+ * available it has to be investigated if this assumption holds for them as well.
+ *
+ * Since vm_insert_page does more precise memory accounting we have the build
+ * flag PVR_MMAP_USE_VM_INSERT that forces its use. This is useful as a debug
+ * feature.
+ *
+ */
+#if defined(CONFIG_X86) || defined(PVR_MMAP_USE_VM_INSERT)
+#define PMR_OS_USE_VM_INSERT_PAGE 1
+#endif
+
+static void MMapPMROpen(struct vm_area_struct *ps_vma)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	PVRSRV_ERROR eError;
+
+	/* VM_DONTCOPY should ensure this function never gets called */
+	PVR_DPF((PVR_DBG_WARNING,
+			 "%s: Unexpected mmap open call, this is probably an application bug.",
+			 __func__));
+	PVR_DPF((PVR_DBG_WARNING,
+			 "%s: vma struct: 0x%p, vAddr: %#lX, length: %#lX, PMR pointer: 0x%p",
+			 __func__,
+			 ps_vma,
+			 ps_vma->vm_start,
+			 ps_vma->vm_end - ps_vma->vm_start,
+			 psPMR));
+
+	/* Should this entry-point be called for one of our PMRs we must increase
+	 * the refcount and lock down the physical addresses.
+	 */
+	eError = PMRRefPMR(psPMR);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PMRRefPMR");
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", ErrUnref);
+
+	/* MMapPMROpen() is called when a process is forked, but only if
+	 * mappings are to be inherited - so increment mapping count of the
+	 * PMR to prevent its layout being changed (if sparse).
+	 */
+	PMRClientCpuMapCountIncr(psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRLockPMR(psPMR);
+	eError = PMROSLinkCPUMapping(psPMR, ps_vma);
+	PMRUnlockPMR(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMROSLinkCPUMapping", ErrUnlock);
+#endif
+
+	return;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+ErrUnlock:
+	PMRUnlockSysPhysAddresses(psPMR);
+#endif
+ErrUnref:
+	(void) PMRUnrefPMR(psPMR);
+}
+
+static void MMapPMRClose(struct vm_area_struct *ps_vma)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	PVRSRV_ERROR eError;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		uintptr_t vAddr = ps_vma->vm_start;
+
+		while (vAddr < ps_vma->vm_end)
+		{
+			/* USER MAPPING */
+			PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+			                                (IMG_UINT64)vAddr,
+			                                OSGetCurrentClientProcessIDKM());
+			vAddr += PAGE_SIZE;
+		}
+	}
+#else
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+	                            ps_vma->vm_end - ps_vma->vm_start,
+	                            OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRLockPMR(psPMR);
+	PMROSUnlinkCPUMapping(psPMR, ps_vma);
+	PMRUnlockPMR(psPMR);
+#endif
+
+	PMRUnlockSysPhysAddresses(psPMR);
+	/* Decrement the mapping count before Unref of PMR (as Unref could destroy the PMR) */
+	PMRClientCpuMapCountDecr(psPMR);
+	eError = PMRUnrefPMR(psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnrefPMR");
+}
+
+/*
+ * This vma operation is used to read data from mmap regions. It is called
+ * by access_process_vm, which is called to handle PTRACE_PEEKDATA ptrace
+ * requests and reads from /proc/<pid>/mem.
+ */
+static int MMapVAccess(struct vm_area_struct *ps_vma, unsigned long addr,
+		       void *buf, int len, int write)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	unsigned long ulOffset = addr - ps_vma->vm_start;
+	size_t uiBytesCopied;
+	PVRSRV_ERROR eError;
+	int iRetVal = -EINVAL;
+
+	if (write)
+	{
+		eError = PMR_WriteBytes(psPMR,
+					(IMG_DEVMEM_OFFSET_T) ulOffset,
+					buf,
+					len,
+					&uiBytesCopied);
+	}
+	else
+	{
+		eError = PMR_ReadBytes(psPMR,
+				       (IMG_DEVMEM_OFFSET_T) ulOffset,
+				       buf,
+				       len,
+				       &uiBytesCopied);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error from %s (%d)",
+			 __func__,
+			 write ? "PMR_WriteBytes" : "PMR_ReadBytes",
+			 eError));
+	}
+	else
+	{
+		iRetVal = uiBytesCopied;
+	}
+
+	return iRetVal;
+}
+
+static INLINE int _OSMMapPMR(PVRSRV_DEVICE_NODE *psDevNode,
+							struct vm_area_struct *ps_vma,
+							IMG_DEVMEM_OFFSET_T uiOffset,
+							IMG_CPU_PHYADDR *psCpuPAddr,
+							IMG_UINT32 uiLog2PageSize,
+							IMG_BOOL bUseVMInsertPage,
+							IMG_BOOL bUseMixedMap)
+{
+	IMG_INT32 iStatus;
+	pfn_t sPFN;
+
+	sPFN = phys_to_pfn_t(psCpuPAddr->uiAddr, 0);
+
+	/*
+	 * vm_insert_page() allows insertion of individual pages into user
+	 * VMA space _only_ if page is a order-zero allocated page
+	 */
+	if (bUseVMInsertPage)
+	{
+		if (bUseMixedMap)
+		{
+			/*
+			 * This path is just for debugging. It should be
+			 * equivalent to the remap_pfn_range() path.
+			 */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0))
+			vm_fault_t vmf;
+
+			vmf = vmf_insert_mixed(ps_vma,
+									ps_vma->vm_start + uiOffset,
+									sPFN);
+			if (vmf & VM_FAULT_ERROR)
+			{
+				iStatus = vm_fault_to_errno(vmf, 0);
+			}
+			else
+			{
+				iStatus = 0;
+			}
+#else
+			iStatus = vm_insert_mixed(ps_vma,
+									  ps_vma->vm_start + uiOffset,
+									  sPFN);
+#endif
+		}
+		else
+		{
+			/* Since kernel 3.7 this sets VM_MIXEDMAP internally */
+			iStatus = vm_insert_page(ps_vma,
+									 ps_vma->vm_start + uiOffset,
+									 pfn_t_to_page(sPFN));
+		}
+	}
+	else
+	{
+		/*
+		   NOTE: Regarding absence of dma_mmap_coherent() in _OSMMapPMR()
+
+		   The current services mmap model maps in a PMR's full-length size
+		   into the user VMA & applies any user specified offset to the kernel
+		   returned zero-offset based VA in services client; this essentially
+		   means services server ignores ps_vma->vm_pgoff (this houses hPMR)
+		   during a mmap call.
+
+		   Furthermore, during a DMA/CMA memory allocation, multiple order-n
+		   pages are used to satisfy an allocation request due to DMA/CMA
+		   framework rounding-up allocation size to next power-of-two which
+		   can lead to wasted memory (so we don't allocate using single call).
+
+		   The combination of the above two issues mean that we cannot use the
+		   dma_mmap_coherent() for a number of reasons outlined below:
+
+		     - Services mmap semantics does not fit with dma_mmap_coherent()
+		       which requires proper ps_vma->vm_pgoff; seeing this houses a
+		       hPMR handle value, calls into dma_mmap_coherent() fails. This
+		       could be avoided by forcing ps_vma->vm_pgoff to zero but the
+		       ps_vma->vm_pgoff is applied to DMA bus address PFN and not
+		       user VMA which is always mapped at ps_vma->vm_start.
+
+		     - As multiple order-n pages are used for DMA/CMA allocations, a
+		       single dma_mmap_coherent() call with a vma->vm_pgoff set to
+		       zero cannot (maybe) be used because there is no guarantee that
+		       all of the multiple order-n pages in the PMR are physically
+		       contiguous from the first entry to the last. Whilst this is
+		       highly likely to be the case, there is no guarantee that it
+		       will be so we cannot depend on this being the case.
+
+		   The solution is to manually mmap DMA/CMA pages into user VMA
+		   using remap_pfn_range() directly. Furthermore, accounting is
+		   always compromised for DMA/CMA allocations.
+		*/
+		size_t uiNumContiguousBytes = 1ULL << uiLog2PageSize;
+
+		iStatus = remap_pfn_range(ps_vma,
+								  ps_vma->vm_start + uiOffset,
+								  pfn_t_to_pfn(sPFN),
+								  uiNumContiguousBytes,
+								  ps_vma->vm_page_prot);
+	}
+
+	return iStatus;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+static vm_fault_t MMapPMRFault(struct vm_fault *ps_vmf)
+{
+	PVRSRV_ERROR eError;
+	vm_fault_t iFault;
+	PMR *psPMR = ps_vmf->vma->vm_private_data;
+
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_BOOL bValid;
+	IMG_UINT64 uiAddrOffsetBytes = ps_vmf->address - ps_vmf->vma->vm_start;
+	unsigned long ulPFN;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	if (!PVRSRV_CHECK_OS_LINUX_MOVABLE(PMR_Flags(psPMR)))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Attempt to fault non movable PMR."));
+		return VM_FAULT_SIGBUS;
+	}
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Fault offset: %llu, addr 0x%llx, vm_start 0x%llx, addr offset b 0x%llx, PMR 0x%llx",
+	                (unsigned long long) ps_vmf->pgoff,
+	                (unsigned long long) ps_vmf->address,
+	                (unsigned long long) ps_vmf->address - ps_vmf->vma->vm_start,
+	                (unsigned long long) uiAddrOffsetBytes,
+	                (unsigned long long) psPMR
+	                ));
+
+	PMRLockPMR(psPMR);
+
+	/* Obtain map range pfns */
+	eError = PMR_CpuPhysAddr(psPMR,
+	                         PAGE_SHIFT,
+	                         1,
+	                         uiAddrOffsetBytes,
+	                         &sCpuPAddr,
+	                         &bValid,
+	                         MAPPING_USE | CPU_USE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMR_CpuPhysAddr", ErrSigFaultReturn);
+	PVR_LOG_GOTO_IF_FALSE(bValid, "Attempted remap with invalid PMR offset", ErrSigFaultReturn);
+
+	ulPFN = pfn_t_to_pfn(phys_to_pfn_t(sCpuPAddr.uiAddr, 0));
+
+	iFault = vmf_insert_pfn_prot(ps_vmf->vma,
+	                             ps_vmf->address,
+	                             ulPFN,
+	                             ps_vmf->vma->vm_page_prot);
+
+	if (iFault != VM_FAULT_NOPAGE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "vmf_insert_pfn_prot failed in MMapPMRFault"));
+		goto ErrReturn;
+	}
+
+
+	PMRUnlockPMR(psPMR);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+								(void*)(uintptr_t)(ps_vmf->vma->vm_start + uiAddrOffsetBytes),
+								sCpuPAddr,
+								PAGE_SIZE,
+								OSGetCurrentClientProcessIDKM()
+								DEBUG_MEMSTATS_VALUES);
+#endif
+
+	return iFault;
+
+ErrSigFaultReturn:
+	iFault = VM_FAULT_SIGBUS;
+ErrReturn:
+	PMRUnlockPMR(psPMR);
+	return iFault;
+}
+#endif
+
+static const struct vm_operations_struct gsMMapOps =
+{
+	.open = &MMapPMROpen,
+	.close = &MMapPMRClose,
+	.access = &MMapVAccess,
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	.fault = &MMapPMRFault
+#endif
+};
+
+PVRSRV_ERROR
+OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData)
+{
+	struct vm_area_struct *ps_vma = pOSMMapData;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+	PVRSRV_ERROR eError;
+	size_t uiLength;
+	IMG_INT32 iStatus;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_UINT32 ui32CPUCacheFlags;
+	pgprot_t sPageProt;
+	IMG_CPU_PHYADDR asCpuPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 uiOffsetIdx;
+	IMG_UINT32 uiNumOfPFNs;
+	IMG_UINT32 uiLog2PageSize;
+	IMG_CPU_PHYADDR *psCpuPAddr;
+	IMG_BOOL *pbValid;
+	IMG_BOOL bUseMixedMap = IMG_FALSE;
+	IMG_BOOL bUseVMInsertPage = IMG_FALSE;
+	vm_flags_t uVMFlags = ps_vma->vm_flags;
+
+	/* if writeable but not shared mapping is requested then fail */
+	PVR_RETURN_IF_INVALID_PARAM(((uVMFlags & VM_WRITE) == 0) ||
+	                            ((uVMFlags & VM_SHARED) != 0));
+
+	uiLength = ps_vma->vm_end - ps_vma->vm_start;
+
+	/* Check early if the requested mapping size doesn't exceed the virtual
+	 * PMR size. */
+	if (PMR_LogicalSize(psPMR) < uiLength)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, ErrReturn);
+	}
+
+	uiLog2PageSize = PMR_GetLog2Contiguity(psPMR);
+
+	/* Check the number of PFNs to be mapped is valid. */
+	uiNumOfPFNs = uiLength >> uiLog2PageSize;
+	if (uiNumOfPFNs == 0)
+	{
+		/* print as 64-bit value to avoid Smatch warning */
+		PVR_LOG_VA(PVR_DBG_ERROR,
+		           "uiLength is invalid. Must be >= %" IMG_UINT64_FMTSPEC ".",
+		           IMG_UINT64_C(1) << uiLog2PageSize);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, ErrReturn);
+	}
+
+	/*
+	 * Take a reference on the PMR so that it can't be freed while mapped
+	 * into the user process.
+	 */
+	eError = PMRRefPMR(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrReturn);
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrUnrefPMR);
+
+	/* Increment mapping count of the PMR so that its layout cannot be
+	 * changed (if sparse).
+	 */
+	PMRLockPMR(psPMR);
+
+	PMRClientCpuMapCountIncr(psPMR);
+	sPageProt = vm_get_page_prot(uVMFlags);
+
+	eError = DevmemCPUCacheMode(PMR_Flags(psPMR), &ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrUnlockPhysAddr;
+	}
+
+	switch (ui32CPUCacheFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				sPageProt = pgprot_noncached(sPageProt);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				sPageProt = pgprot_writecombine(sPageProt);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+		{
+/* Do not set to write-combine for plato */
+#if !defined(PLATO_MEMORY_CONFIG)
+				PHYS_HEAP *psPhysHeap = PMR_PhysHeap(psPMR);
+
+				if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA)
+					sPageProt = pgprot_writecombine(sPageProt);
+#endif
+				break;
+		}
+
+		default:
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto ErrUnlockPhysAddr;
+	}
+	ps_vma->vm_page_prot = sPageProt;
+
+	uVMFlags |= VM_IO;
+
+	/* Don't include the mapping in core dumps */
+	uVMFlags |= VM_DONTDUMP;
+
+	/*
+	 * Disable mremap because our nopage handler assumes all
+	 * page requests have already been validated.
+	 */
+	uVMFlags |= VM_DONTEXPAND;
+
+	/* Don't allow mapping to be inherited across a process fork */
+	uVMFlags |= VM_DONTCOPY;
+
+#if defined(PMR_OS_USE_VM_INSERT_PAGE)
+	/* Is this mmap targeting non order-zero pages or does it use pfn mappings?
+	 * If yes, don't use vm_insert_page */
+	bUseVMInsertPage = (uiLog2PageSize == PAGE_SHIFT) && (PMR_GetType(psPMR) != PMR_TYPE_EXTMEM);
+#endif
+
+	/* Can we use stack allocations */
+	if (uiNumOfPFNs > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psCpuPAddr = OSAllocMem(uiNumOfPFNs * sizeof(*psCpuPAddr));
+		PVR_LOG_GOTO_IF_NOMEM(psCpuPAddr, eError, ErrUnlockPhysAddr);
+
+		/* Should allocation fail, clean-up here before exiting */
+		pbValid = OSAllocMem(uiNumOfPFNs * sizeof(*pbValid));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrFreeCpuPAddr);
+	}
+	else
+	{
+		psCpuPAddr = asCpuPAddr;
+		pbValid = abValid;
+	}
+
+	/* Obtain map range pfns */
+	eError = PMR_CpuPhysAddr(psPMR,
+				 uiLog2PageSize,
+				 uiNumOfPFNs,
+				 0,
+				 psCpuPAddr,
+				 pbValid,
+				 CPU_USE | MAPPING_USE);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrFreeValid;
+	}
+
+	/*
+	 * Scan the map range for pfns without struct page* handling. If
+	 * we find one, this is a mixed map, and we can't use vm_insert_page()
+	 * NOTE: vm_insert_page() allows insertion of individual pages into user
+	 * VMA space _only_ if said page is an order-zero allocated page.
+	 */
+	if (bUseVMInsertPage)
+	{
+		pfn_t sPFN;
+
+		for (uiOffsetIdx = 0; uiOffsetIdx < uiNumOfPFNs; ++uiOffsetIdx)
+		{
+			if (pbValid[uiOffsetIdx])
+			{
+				sPFN = phys_to_pfn_t(psCpuPAddr[uiOffsetIdx].uiAddr, 0);
+
+				if (!pfn_t_valid(sPFN) || page_count(pfn_t_to_page(sPFN)) == 0)
+				{
+					bUseMixedMap = IMG_TRUE;
+					break;
+				}
+			}
+		}
+
+		if (bUseMixedMap)
+		{
+			uVMFlags |= VM_MIXEDMAP;
+		}
+	}
+	else
+	{
+		uVMFlags |= VM_PFNMAP;
+	}
+
+	/* Actually initialise the flags */
+	pvr_vm_flags_init(ps_vma, uVMFlags);
+
+	/* For each PMR page-size contiguous bytes, map page(s) into user VMA */
+	for (uiOffset = 0; uiOffset < uiLength; uiOffset += 1ULL<<uiLog2PageSize)
+	{
+		uiOffsetIdx = uiOffset >> uiLog2PageSize;
+		/*
+		 * Only map in pages that are valid, any that aren't will be
+		 * picked up by the nopage handler which will return a zeroed
+		 * page for us.
+		 */
+		if (pbValid[uiOffsetIdx])
+		{
+			iStatus = _OSMMapPMR(psDevNode,
+								 ps_vma,
+								 uiOffset,
+								 &psCpuPAddr[uiOffsetIdx],
+								 uiLog2PageSize,
+								 bUseVMInsertPage,
+								 bUseMixedMap);
+			if (iStatus)
+			{
+				/* Failure error code doesn't get propagated */
+				eError = PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+				PVR_ASSERT(0);
+				goto ErrFreeValid;
+			}
+		}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+#define PMR_OS_BAD_CPUADDR 0x0BAD0BAD
+		{
+			IMG_CPU_PHYADDR sPAddr;
+			sPAddr.uiAddr = pbValid[uiOffsetIdx] ?
+					psCpuPAddr[uiOffsetIdx].uiAddr :
+					IMG_CAST_TO_CPUPHYADDR_UINT(PMR_OS_BAD_CPUADDR);
+
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+										(void*)(uintptr_t)(ps_vma->vm_start + uiOffset),
+										sPAddr,
+										1<<uiLog2PageSize,
+										OSGetCurrentClientProcessIDKM()
+										DEBUG_MEMSTATS_VALUES);
+		}
+#undef PMR_OS_BAD_CPUADDR
+#endif
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES, uiNumOfPFNs * PAGE_SIZE, OSGetCurrentClientProcessIDKM());
+#endif
+
+	if (psCpuPAddr != asCpuPAddr)
+	{
+		OSFreeMem(psCpuPAddr);
+		OSFreeMem(pbValid);
+	}
+
+	/* let us see the PMR so we can unlock it later */
+	ps_vma->vm_private_data = psPMR;
+
+	/* Install open and close handlers for ref-counting */
+	ps_vma->vm_ops = &gsMMapOps;
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	/* record the stats */
+	MMapStatsAddOrUpdatePMR(psPMR, uiLength);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	eError = PMROSLinkCPUMapping(psPMR, ps_vma);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLinkCpuMapping", ErrUnlockPhysAddr);
+#endif
+
+	PMRUnlockPMR(psPMR);
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+ErrFreeValid:
+	if (pbValid != abValid)
+	{
+		OSFreeMem(pbValid);
+	}
+ErrFreeCpuPAddr:
+	if (psCpuPAddr != asCpuPAddr)
+	{
+		OSFreeMem(psCpuPAddr);
+	}
+ErrUnlockPhysAddr:
+	PMRClientCpuMapCountDecr(psPMR);
+	PMRUnlockPMR(psPMR);
+	PMRUnlockSysPhysAddresses(psPMR);
+ErrUnrefPMR:
+	(void) PMRUnrefPMR(psPMR);
+ErrReturn:
+	return eError;
+}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+static void
+OSLinuxUnmapPageInCPUMapping(PMR *psPMR, struct vm_area_struct *ps_vma, IMG_UINT32 ui32LogicalPgOffset)
+{
+	IMG_UINT64 ui64OffsetBytes = IMG_PAGES2BYTES64(ui32LogicalPgOffset, PAGE_SHIFT);
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "CPU PTE Zap, PMR 0x%llx, Offset %u, OffsetB 0x%llx, vm_start 0x%llx, vm_addr 0x%llx",
+	                (unsigned long long) psPMR,
+	                ui32LogicalPgOffset,
+	                (unsigned long long) ui64OffsetBytes,
+	                (unsigned long long) ps_vma->vm_start,
+	                (unsigned long long) ps_vma->vm_start + ui64OffsetBytes));
+
+	zap_vma_ptes(ps_vma, ps_vma->vm_start + ui64OffsetBytes, PAGE_SIZE);
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+	                                (IMG_UINT64)ps_vma->vm_start + ui64OffsetBytes,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+}
+
+void
+OSLinuxPMRUnmapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset)
+{
+	DLLIST_NODE *psNext, *psNode;
+
+	PMRLockPMR(psPMR);
+
+	dllist_foreach_node(psMappingListHead, psNode, psNext)
+	{
+		PMR_OS_CPU_MAPPING *psCPUMapping = IMG_CONTAINER_OF(psNode,
+		                                                 PMR_OS_CPU_MAPPING,
+		                                                 sListNode);
+
+		OSLinuxUnmapPageInCPUMapping(psPMR, psCPUMapping->ps_vma, ui32LogicalPgOffset);
+	}
+
+	PMRUnlockPMR(psPMR);
+
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/private_data.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/private_data.h
new file mode 100644
index 000000000000..60a1fac0b970
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/private_data.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux private data structure
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(INCLUDED_PRIVATE_DATA_H)
+#define INCLUDED_PRIVATE_DATA_H
+
+#include <linux/fs.h>
+
+#include "connection_server.h"
+#include "pvr_drm.h"
+
+#define PVR_SRVKM_PRIV_DATA_IDX 0
+#define PVR_SYNC_PRIV_DATA_IDX  1
+
+#define PVR_NUM_PRIV_DATA_IDXS  2
+
+CONNECTION_DATA *LinuxServicesConnectionFromFile(struct file *pFile);
+CONNECTION_DATA *LinuxSyncConnectionFromFile(struct file *pFile);
+
+#endif /* !defined(INCLUDED_PRIVATE_DATA_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.c
new file mode 100644
index 000000000000..77739189b0aa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.c
@@ -0,0 +1,643 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Receives calls from the user portion of services and
+                despatches them to functions in the kernel portion.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include <linux/mm_types.h>
+
+#include "img_defs.h"
+#include "pvr_bridge.h"
+#include "pvr_bridge_k.h"
+#include "connection_server.h"
+#include "syscommon.h"
+#include "pvr_debug.h"
+#include "di_server.h"
+#include "private_data.h"
+#include "linkage.h"
+#include "pmr.h"
+#include "rgx_bvnc_defs_km.h"
+#include "pvrsrv_bridge_init.h"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_print.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include "pvr_drm.h"
+#include "pvr_drv.h"
+
+#include "env_connection.h"
+#include <linux/sched.h>
+#include <linux/freezer.h>
+
+/* RGX: */
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+
+#include "srvcore.h"
+#include "common_srvcore_bridge.h"
+#include "kernel_compatibility.h"
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+#if defined(MODULE_TEST)
+/************************************************************************/
+// additional includes for services testing
+/************************************************************************/
+#include "pvr_test_bridge.h"
+#include "kern_test.h"
+/************************************************************************/
+// end of additional includes
+/************************************************************************/
+#endif
+
+/* The mmap code has its own mutex, to prevent possible re-entrant issues
+ * when the same PMR is mapped from two different connections/processes.
+ */
+static DEFINE_MUTEX(g_sMMapMutex);
+
+#define _SUSPENDED 1
+#define _NOT_SUSPENDED 0
+static ATOMIC_T g_iDriverSuspendCount;
+static ATOMIC_T g_iNumActiveDriverThreads;
+static ATOMIC_T g_iNumActiveKernelThreads;
+static IMG_HANDLE g_hDriverThreadEventObject;
+
+
+#if defined(DEBUG_BRIDGE_KM)
+static DI_ENTRY *gpsDIBridgeStatsEntry;
+
+static void *BridgeStatsDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psDispatchTable = DIGetPrivData(psEntry);
+
+	if (psDispatchTable == NULL || *pui64Pos > BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		return NULL;
+	}
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	return &(psDispatchTable[*pui64Pos - 1]);
+}
+
+static void BridgeStatsDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+}
+
+static void *BridgeStatsDINext(OSDI_IMPL_ENTRY *psEntry, void *pvData,
+                               IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psDispatchTable = DIGetPrivData(psEntry);
+	IMG_UINT64 uiItemAskedFor = *pui64Pos; /* pui64Pos on entry is the index to return */
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	/* Is the item asked for (starts at 0) a valid table index? */
+	if (uiItemAskedFor < BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		(*pui64Pos)++; /* on exit it is the next DI index to ask for */
+		return &(psDispatchTable[uiItemAskedFor]);
+	}
+
+	/* Now passed the end of the table to indicate stop */
+	return NULL;
+}
+
+static int BridgeStatsDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	if (pvData == DI_START_TOKEN)
+	{
+		BridgeGlobalStatsLock();
+		DIPrintf(psEntry,
+		         "Total ioctl call count = %u\n"
+		         "Total number of bytes copied via copy_from_user = %u\n"
+		         "Total number of bytes copied via copy_to_user = %u\n"
+		         "Total number of bytes copied via copy_*_user = %u\n\n"
+		         "%3s: %-60s | %-48s | %10s | %20s | %20s | %20s | %20s\n",
+		         g_BridgeGlobalStats.ui32IOCTLCount,
+		         g_BridgeGlobalStats.ui32TotalCopyFromUserBytes,
+		         g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		         g_BridgeGlobalStats.ui32TotalCopyFromUserBytes +
+		             g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		         "#",
+		         "Bridge Name",
+		         "Wrapper Function",
+		         "Call Count",
+		         "copy_from_user (B)",
+		         "copy_to_user (B)",
+		         "Total Time (us)",
+		         "Max Time (us)");
+		BridgeGlobalStatsUnlock();
+	}
+	else if (pvData != NULL)
+	{
+		PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psTableEntry = pvData;
+		IMG_UINT32 ui32Remainder;
+
+		BridgeGlobalStatsLock();
+		DIPrintf(psEntry,
+		         "%3d: %-60s   %-48s   %-10u   %-20u   %-20u   %-20" IMG_UINT64_FMTSPEC "   %-20" IMG_UINT64_FMTSPEC "\n",
+		         (IMG_UINT32)(((size_t)psTableEntry-(size_t)g_BridgeDispatchTable)/sizeof(*g_BridgeDispatchTable)),
+		         psTableEntry->pszIOCName,
+		         (psTableEntry->pfFunction != NULL) ? psTableEntry->pszFunctionName : "(null)",
+		         psTableEntry->ui32CallCount,
+		         psTableEntry->ui32CopyFromUserTotalBytes,
+		         psTableEntry->ui32CopyToUserTotalBytes,
+		         OSDivide64r64(psTableEntry->ui64TotalTimeNS, 1000, &ui32Remainder),
+		         OSDivide64r64(psTableEntry->ui64MaxTimeNS, 1000, &ui32Remainder));
+		BridgeGlobalStatsUnlock();
+	}
+
+	return 0;
+}
+
+static IMG_INT64 BridgeStatsWrite(const IMG_CHAR *pcBuffer,
+                                  IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                                  void *pvData)
+{
+	IMG_UINT32 i;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[0] == '0', -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	/* Reset stats. */
+
+	BridgeGlobalStatsLock();
+
+	g_BridgeGlobalStats.ui32IOCTLCount = 0;
+	g_BridgeGlobalStats.ui32TotalCopyFromUserBytes = 0;
+	g_BridgeGlobalStats.ui32TotalCopyToUserBytes = 0;
+
+	for (i = 0; i < ARRAY_SIZE(g_BridgeDispatchTable); i++)
+	{
+		g_BridgeDispatchTable[i].ui32CallCount = 0;
+		g_BridgeDispatchTable[i].ui32CopyFromUserTotalBytes = 0;
+		g_BridgeDispatchTable[i].ui32CopyToUserTotalBytes = 0;
+		g_BridgeDispatchTable[i].ui64TotalTimeNS = 0;
+		g_BridgeDispatchTable[i].ui64MaxTimeNS = 0;
+	}
+
+	BridgeGlobalStatsUnlock();
+
+	return ui64Count;
+}
+
+#endif /* defined(DEBUG_BRIDGE_KM) */
+
+PVRSRV_ERROR LinuxGetThreadActivityStats(LINUX_THREAD_ACTIVITY_STATS *psThreadStats)
+{
+	PVR_RETURN_IF_FALSE(psThreadStats != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psThreadStats->i32KernelThreadCount = OSAtomicRead(&g_iNumActiveKernelThreads);
+	psThreadStats->i32DriverThreadCount = OSAtomicRead(&g_iNumActiveDriverThreads);
+	psThreadStats->i32SuspendedThreadCount = OSAtomicRead(&g_iDriverSuspendCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSPlatformBridgeInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = InitDMABUFBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDMABUFBridge");
+
+	OSAtomicWrite(&g_iDriverSuspendCount, 0);
+	OSAtomicWrite(&g_iNumActiveDriverThreads, 0);
+	OSAtomicWrite(&g_iNumActiveKernelThreads, 0);
+
+	eError = OSEventObjectCreate("Global driver thread event object",
+	                             &g_hDriverThreadEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", EventCreateError);
+
+#if defined(DEBUG_BRIDGE_KM)
+	{
+		DI_ITERATOR_CB sIter = {
+			.pfnStart = BridgeStatsDIStart,
+			.pfnStop = BridgeStatsDIStop,
+			.pfnNext = BridgeStatsDINext,
+			.pfnShow = BridgeStatsDIShow,
+			.pfnWrite = BridgeStatsWrite,
+
+			//Expects '0' + Null terminator
+			.ui32WriteLenMax = ((1U)+1U)
+		};
+
+		eError = DICreateEntry("bridge_stats", NULL, &sIter,
+		                       &g_BridgeDispatchTable[0],
+		                       DI_ENTRY_TYPE_GENERIC,
+		                       &gpsDIBridgeStatsEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", DIEntryCreateError);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(DEBUG_BRIDGE_KM)
+DIEntryCreateError:
+#endif
+EventCreateError:
+	OSEventObjectDestroy(g_hDriverThreadEventObject);
+	g_hDriverThreadEventObject = NULL;
+
+	return eError;
+}
+
+void OSPlatformBridgeDeInit(void)
+{
+#if defined(DEBUG_BRIDGE_KM)
+	DIDestroyEntry(gpsDIBridgeStatsEntry);
+#endif
+
+	OSEventObjectDestroy(g_hDriverThreadEventObject);
+	g_hDriverThreadEventObject = NULL;
+
+	DeinitDMABUFBridge();
+}
+
+PVRSRV_ERROR LinuxBridgeBlockClientsAccess(struct pvr_drm_private *psDevPriv,
+                                           IMG_BOOL bShutdown)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	__maybe_unused IMG_INT iSuspendCount;
+
+	eError = OSEventObjectOpen(g_hDriverThreadEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return eError;
+	}
+
+	iSuspendCount = OSAtomicIncrement(&g_iDriverSuspendCount);
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver suspended %d times.", __func__,
+	         iSuspendCount));
+
+	if (OSAtomicCompareExchange(&psDevPriv->suspended, _NOT_SUSPENDED,
+	                            _SUSPENDED) == _SUSPENDED)
+	{
+		OSAtomicDecrement(&g_iDriverSuspendCount);
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p already suspended", __func__,
+		        psDevPriv->dev_node));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto CloseEventObject;
+	}
+
+	/* now wait for any threads currently in the server to exit */
+	while (OSAtomicRead(&g_iNumActiveDriverThreads) != 0 ||
+		   (OSAtomicRead(&g_iNumActiveKernelThreads) != 0 && !bShutdown))
+	{
+		if (OSAtomicRead(&g_iNumActiveDriverThreads) != 0)
+		{
+			PVR_LOG(("%s: waiting for user threads (%d)", __func__,
+			        OSAtomicRead(&g_iNumActiveDriverThreads)));
+		}
+		if (OSAtomicRead(&g_iNumActiveKernelThreads) != 0)
+		{
+			PVR_LOG(("%s: waiting for kernel threads (%d)", __func__,
+			        OSAtomicRead(&g_iNumActiveKernelThreads)));
+		}
+		/* Regular wait is called here (and not OSEventObjectWaitKernel) because
+		 * this code is executed by the caller of .suspend/.shutdown callbacks
+		 * which is most likely PM (or other actor responsible for suspend
+		 * process). Because of that this thread shouldn't and most likely
+		 * event cannot be frozen. */
+		OSEventObjectWait(hEvent);
+	}
+
+CloseEventObject:
+	OSEventObjectClose(hEvent);
+
+	return eError;
+}
+
+PVRSRV_ERROR LinuxBridgeUnblockClientsAccess(struct pvr_drm_private *psDevPriv)
+{
+	PVRSRV_ERROR eError;
+	__maybe_unused IMG_INT iSuspendCount;
+
+	/* resume the driver and then signal so any waiting threads wake up */
+	if (OSAtomicCompareExchange(&psDevPriv->suspended, _SUSPENDED,
+	                            _NOT_SUSPENDED) == _NOT_SUSPENDED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device is not suspended", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	iSuspendCount = OSAtomicDecrement(&g_iDriverSuspendCount);
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver suspended %d times.", __func__,
+	         iSuspendCount));
+
+	eError = OSEventObjectSignal(g_hDriverThreadEventObject);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSEventObjectSignal failed: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR LinuxBridgeSignalIfSuspended(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (OSAtomicRead(&g_iDriverSuspendCount) > 0)
+	{
+		PVRSRV_ERROR eError = OSEventObjectSignal(g_hDriverThreadEventObject);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal driver thread event"
+			        " object: %s", __func__, PVRSRVGetErrorString(eError)));
+		}
+	}
+
+	return eError;
+}
+
+void LinuxBridgeNumActiveKernelThreadsIncrement(void)
+{
+	OSAtomicIncrement(&g_iNumActiveKernelThreads);
+}
+
+void LinuxBridgeNumActiveKernelThreadsDecrement(void)
+{
+	OSAtomicDecrement(&g_iNumActiveKernelThreads);
+	PVR_ASSERT(OSAtomicRead(&g_iNumActiveKernelThreads) >= 0);
+
+	/* Signal on every decrement in case LinuxBridgeBlockClientsAccess() is
+	 * waiting for the threads to freeze.
+	 * (error is logged in called function so ignore, we can't do much with
+	 * it anyway) */
+	(void) LinuxBridgeSignalIfSuspended();
+}
+
+static PVRSRV_ERROR _WaitForDriverUnsuspend(void)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+
+	eError = OSEventObjectOpen(g_hDriverThreadEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return eError;
+	}
+
+	while (OSAtomicRead(&g_iDriverSuspendCount) == 0)
+	{
+		/* we should be able to use normal (not kernel) wait here since
+		 * we were just unfrozen and most likely we're not going to
+		 * be frozen again (?) */
+		OSEventObjectWait(hEvent);
+	}
+
+	OSEventObjectClose(hEvent);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVDriverThreadEnter(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	CONNECTION_DATA *psConnection = (CONNECTION_DATA *)pvData;
+
+	/* Block if the associated device has been placed into a FROZEN state.
+	 * In this case we must await a PVRSRVDeviceThaw() completion request.
+	 * Device is obtained from the incoming psConnection if pvData is non-NULL.
+	 */
+	if (likely(pvData != NULL))
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+		PVRSRVBlockIfFrozen(psDevNode);
+		OSAtomicIncrement(&psDevNode->iThreadsActive);
+	}
+
+	/* increment first so there is no race between this value and
+	 * g_iDriverSuspendCount in LinuxBridgeBlockClientsAccess() */
+	OSAtomicIncrement(&g_iNumActiveDriverThreads);
+
+	if (OSAtomicRead(&g_iDriverSuspendCount) > 0)
+	{
+		/* decrement here because the driver is going to be suspended and
+		 * this thread is going to be frozen so we don't want to wait for
+		 * it in LinuxBridgeBlockClientsAccess() */
+		OSAtomicDecrement(&g_iNumActiveDriverThreads);
+
+		/* during suspend procedure this will put the current thread to
+		 * the freezer but during shutdown this will just return */
+		try_to_freeze();
+
+		/* if the thread was unfrozen but the number of suspends is non-0 wait
+		 * for it
+		 * in case this is a shutdown the thread was not frozen so we'll
+		 * wait here indefinitely but this is ok (and this is in fact what
+		 * we want) because no thread should be entering the driver in such
+		 * case */
+		eError = _WaitForDriverUnsuspend();
+
+		/* increment here because that means that the thread entered the
+		 * driver */
+		OSAtomicIncrement(&g_iNumActiveDriverThreads);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to wait for driver"
+			        " unsuspend: %s", __func__,
+			        PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVDriverThreadExit(void *pvData)
+{
+	CONNECTION_DATA *psConnection = (CONNECTION_DATA *)pvData;
+	OSAtomicDecrement(&g_iNumActiveDriverThreads);
+
+	/* Decrement the number of threads active on this device if the
+	 * connection is known.
+	 */
+	if (psConnection != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+		OSAtomicDecrement(&psDevNode->iThreadsActive);
+	}
+	/* if the driver is being suspended then we need to signal the
+	 * event object as the thread suspending the driver is waiting
+	 * for active threads to exit
+	 * error is logged in called function so ignore returned error
+	 */
+	(void) LinuxBridgeSignalIfSuspended();
+}
+
+int
+PVRSRV_BridgeDispatchKM(struct drm_device __maybe_unused *dev, void *arg, struct drm_file *pDRMFile)
+{
+	struct drm_pvr_srvkm_cmd *psSrvkmCmd = (struct drm_pvr_srvkm_cmd *) arg;
+	PVRSRV_BRIDGE_PACKAGE sBridgePackageKM = { 0 };
+	CONNECTION_DATA *psConnection = LinuxServicesConnectionFromFile(pDRMFile->filp);
+	PVRSRV_ERROR error;
+
+	if (psConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid connection data"));
+		return -EFAULT;
+	}
+
+	PVR_ASSERT(psSrvkmCmd != NULL);
+
+	DRM_DEBUG("tgid=%d, tgid_connection=%d, bridge_id=%d, func_id=%d",
+			  task_tgid_nr(current),
+			  ((ENV_CONNECTION_DATA *)PVRSRVConnectionPrivateData(psConnection))->owner,
+			  psSrvkmCmd->bridge_id,
+			  psSrvkmCmd->bridge_func_id);
+
+	error = PVRSRVDriverThreadEnter(psConnection);
+	PVR_LOG_GOTO_IF_ERROR(error, "PVRSRVDriverThreadEnter", e0);
+
+	sBridgePackageKM.ui32BridgeID = psSrvkmCmd->bridge_id;
+	sBridgePackageKM.ui32FunctionID = psSrvkmCmd->bridge_func_id;
+	sBridgePackageKM.ui32Size = sizeof(sBridgePackageKM);
+	sBridgePackageKM.pvParamIn = (void __user *)(uintptr_t)psSrvkmCmd->in_data_ptr;
+	sBridgePackageKM.ui32InBufferSize = psSrvkmCmd->in_data_size;
+	sBridgePackageKM.pvParamOut = (void __user *)(uintptr_t)psSrvkmCmd->out_data_ptr;
+	sBridgePackageKM.ui32OutBufferSize = psSrvkmCmd->out_data_size;
+
+	error = BridgedDispatchKM(psConnection, &sBridgePackageKM);
+
+e0:
+	PVRSRVDriverThreadExit(psConnection);
+
+	return OSPVRSRVToNativeError(error);
+}
+
+int
+PVRSRV_MMap(struct file *pFile, struct vm_area_struct *ps_vma)
+{
+	CONNECTION_DATA *psConnection = LinuxServicesConnectionFromFile(pFile);
+	IMG_HANDLE hSecurePMRHandle = (IMG_HANDLE)((uintptr_t)ps_vma->vm_pgoff);
+	PMR *psPMR;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiProtFlags =
+	    (BITMASK_HAS(ps_vma->vm_flags, VM_READ) ? PVRSRV_MEMALLOCFLAG_CPU_READABLE : 0) |
+	    (BITMASK_HAS(ps_vma->vm_flags, VM_WRITE) ? PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0);
+
+	if (psConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid connection data"));
+		return -ENOENT;
+	}
+
+	eError = PVRSRVDriverThreadEnter(psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDriverThreadEnter", e0);
+
+	/*
+	 * The bridge lock used here to protect PVRSRVLookupHandle is replaced
+	 * by a specific lock considering that the handle functions have now
+	 * their own lock. This change was necessary to solve the lockdep issues
+	 * related with the PVRSRV_MMap.
+	 */
+
+	eError = PVRSRVLookupHandle(psConnection->psHandleBase,
+								(void **)&psPMR,
+								hSecurePMRHandle,
+								PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								IMG_TRUE);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	mutex_lock(&g_sMMapMutex);
+
+	/* Forcibly clear the VM_MAYWRITE flag as this is inherited from the
+	 * kernel mmap code and we do not want to produce a potentially writable
+	 * mapping from a read-only mapping.
+	 */
+	if (!BITMASK_HAS(ps_vma->vm_flags, VM_WRITE))
+	{
+		pvr_vm_flags_clear(ps_vma, VM_MAYWRITE);
+	}
+
+	/* Note: PMRMMapPMR will take a reference on the PMR.
+	 * Unref the handle immediately, because we have now done
+	 * the required operation on the PMR (whether it succeeded or not)
+	 */
+	eError = PMRMMapPMR(psPMR, ps_vma, uiProtFlags);
+	mutex_unlock(&g_sMMapMutex);
+	PVRSRVReleaseHandle(psConnection->psHandleBase, hSecurePMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMRMMapPMR failed (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	PVRSRVDriverThreadExit(psConnection);
+
+	return 0;
+
+e0:
+	PVRSRVDriverThreadExit(psConnection);
+
+	PVR_DPF((PVR_DBG_ERROR, "Failed with error: %s", PVRSRVGetErrorString(eError)));
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return OSPVRSRVToNativeError(eError);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.h
new file mode 100644
index 000000000000..2885dd465eaf
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_bridge_k.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Receives calls from the user portion of services and
+                despatches them to functions in the kernel portion.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BRIDGE_K_H
+#define PVR_BRIDGE_K_H
+
+#include "pvrsrv_error.h"
+#include "pvr_drv.h"
+
+typedef struct LINUX_THREAD_ACTIVITY_STATS
+{
+	IMG_INT32 i32KernelThreadCount;
+	IMG_INT32 i32DriverThreadCount;
+	IMG_INT32 i32SuspendedThreadCount;
+} LINUX_THREAD_ACTIVITY_STATS;
+
+/*!
+******************************************************************************
+ @Function      LinuxBridgeBlockClientsAccess
+ @Description   This function will wait for any existing threads in the Server
+                to exit and then disable access to the driver. New threads will
+                not be allowed to enter the Server until the driver is
+                unsuspended (see LinuxBridgeUnblockClientsAccess).
+ @Input         psDevPriv pointer to devices OS specific data
+ @Input         bShutdown this flag indicates that the function was called
+                          from a shutdown callback and therefore it will
+                          not wait for the kernel threads to get frozen
+                          (because this doesn't happen during shutdown
+                          procedure)
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR LinuxBridgeBlockClientsAccess(struct pvr_drm_private *psDevPriv,
+                                           IMG_BOOL bShutdown);
+
+/*!
+******************************************************************************
+ @Function      LinuxBridgeUnblockClientsAccess
+ @Description   This function will re-enable the bridge and allow any threads
+                waiting to enter the Server to continue.
+ @Input         psDevPriv pointer to devices OS specific data
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR LinuxBridgeUnblockClientsAccess(struct pvr_drm_private *psDevPriv);
+
+void LinuxBridgeNumActiveKernelThreadsIncrement(void);
+void LinuxBridgeNumActiveKernelThreadsDecrement(void);
+
+/*************************************************************************/ /*!
+ @Function      LinuxGetThreadActivityStats
+ @Description   Getter for active and suspended thread stats.
+
+ @Output        psThreadStats   Struct to be populated with thread activity
+                                stats.
+
+ @Return        PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR LinuxGetThreadActivityStats(LINUX_THREAD_ACTIVITY_STATS *psThreadStats);
+
+/*!
+******************************************************************************
+ @Function      PVRSRVDriverThreadEnter
+ @Description   Increments number of client threads currently operating
+                in the driver's context.
+                If the driver is currently being suspended this function
+                will call try_to_freeze() on behalf of the client thread.
+                When the driver is resumed the function will exit and allow
+                the thread into the driver.
+ @Input         Reference to Connection data. NULL if no associated
+                 connection / device.
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDriverThreadEnter(void *pvData);
+
+/*!
+******************************************************************************
+ @Function      PVRSRVDriverThreadExit
+ @Description   Decrements the number of client threads currently operating
+                in the driver's context to match the call to
+                PVRSRVDriverThreadEnter().
+                The function also signals the driver that a thread left the
+                driver context so if it's waiting to suspend it knows that
+                the number of threads decreased.
+ @Input         Reference to Connection data. NULL if no associated
+                 connection / device.
+******************************************************************************/
+void PVRSRVDriverThreadExit(void *pvData);
+
+#endif /* PVR_BRIDGE_K_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.c
new file mode 100644
index 000000000000..529cafd2bcf3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.c
@@ -0,0 +1,733 @@
+/*
+ * @File
+ * @Title       Linux buffer sync interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/dma-buf.h>
+
+#include "services_kernel_client.h"
+#include "pvr_dma_resv.h"
+#include "pvr_buffer_sync.h"
+#include "pvr_buffer_sync_shared.h"
+#include "pvr_drv.h"
+#include "pvr_fence.h"
+
+struct pvr_buffer_sync_context {
+	struct mutex ctx_lock;
+	struct pvr_fence_context *fence_ctx;
+	struct ww_acquire_ctx acquire_ctx;
+};
+
+struct pvr_buffer_sync_check_data {
+	struct dma_fence_cb base;
+
+	u32 nr_fences;
+	struct pvr_fence **fences;
+};
+
+struct pvr_buffer_sync_append_data {
+	struct pvr_buffer_sync_context *ctx;
+
+	u32 nr_pmrs;
+	struct _PMR_ **pmrs;
+	u32 *pmr_flags;
+
+	struct pvr_fence *update_fence;
+	struct pvr_buffer_sync_check_data *check_data;
+};
+
+static struct dma_resv *
+pmr_reservation_object_get(struct _PMR_ *pmr)
+{
+	return PhysmemGetDmaResv(pmr);
+}
+
+static int
+pvr_buffer_sync_pmrs_lock(struct pvr_buffer_sync_context *ctx,
+			  u32 nr_pmrs,
+			  struct _PMR_ **pmrs)
+{
+	struct dma_resv *resv, *cresv = NULL, *lresv = NULL;
+	int i, err;
+	struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx;
+
+	mutex_lock(&ctx->ctx_lock);
+
+	ww_acquire_init(acquire_ctx, &reservation_ww_class);
+retry:
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (!resv) {
+			pr_err("%s: Failed to get reservation object from pmr %p\n",
+			       __func__, pmrs[i]);
+			err = -EINVAL;
+			goto fail;
+		}
+
+		if (resv != lresv) {
+			err = ww_mutex_lock_interruptible(&resv->lock,
+							  acquire_ctx);
+			if (err) {
+				cresv = (err == -EDEADLK) ? resv : NULL;
+				goto fail;
+			}
+		} else {
+			lresv = NULL;
+		}
+	}
+
+	ww_acquire_done(acquire_ctx);
+
+	return 0;
+
+fail:
+	while (i--) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+		ww_mutex_unlock(&resv->lock);
+	}
+
+	if (lresv)
+		ww_mutex_unlock(&lresv->lock);
+
+	if (cresv) {
+		err = ww_mutex_lock_slow_interruptible(&cresv->lock,
+						       acquire_ctx);
+		if (!err) {
+			lresv = cresv;
+			cresv = NULL;
+			goto retry;
+		}
+	}
+
+	ww_acquire_fini(acquire_ctx);
+
+	mutex_unlock(&ctx->ctx_lock);
+	return err;
+}
+
+static void
+pvr_buffer_sync_pmrs_unlock(struct pvr_buffer_sync_context *ctx,
+			    u32 nr_pmrs,
+			    struct _PMR_ **pmrs)
+{
+	struct dma_resv *resv;
+	int i;
+	struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+		ww_mutex_unlock(&resv->lock);
+	}
+
+	ww_acquire_fini(acquire_ctx);
+
+	mutex_unlock(&ctx->ctx_lock);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+
+static void
+dma_resv_count_fences(struct dma_resv *resv, u32 *read_fence_count_out, u32 *write_fence_count_out)
+{
+	struct dma_resv_iter cursor;
+	u32 write_fence_count = 0;
+	u32 read_fence_count = 0;
+	struct dma_fence *fence;
+
+	dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_READ);
+	dma_resv_for_each_fence_unlocked(&cursor, fence) {
+		if (dma_resv_iter_is_restarted(&cursor)) {
+			read_fence_count = 0;
+			write_fence_count = 0;
+		}
+		if (dma_resv_iter_usage(&cursor) == DMA_RESV_USAGE_READ)
+			read_fence_count++;
+		else if (dma_resv_iter_usage(&cursor) == DMA_RESV_USAGE_WRITE)
+			write_fence_count++;
+	}
+
+	*read_fence_count_out = read_fence_count;
+	*write_fence_count_out = write_fence_count;
+}
+
+static u32
+pvr_buffer_sync_pmrs_fence_count(u32 nr_pmrs, struct _PMR_ **pmrs,
+				 u32 *pmr_flags)
+{
+	struct dma_resv *resv;
+	u32 fence_count = 0;
+	bool exclusive;
+	int i;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		u32 write_fence_count = 0;
+		u32 read_fence_count = 0;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		dma_resv_count_fences(resv, &read_fence_count, &write_fence_count);
+
+		if (!exclusive || !read_fence_count)
+			fence_count += write_fence_count;
+		if (exclusive)
+			fence_count += read_fence_count;
+	}
+
+	return fence_count;
+}
+
+static struct pvr_buffer_sync_check_data *
+pvr_buffer_sync_check_fences_create(struct pvr_fence_context *fence_ctx,
+				    PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+				    u32 nr_pmrs,
+				    struct _PMR_ **pmrs,
+				    u32 *pmr_flags)
+{
+	struct pvr_buffer_sync_check_data *data;
+	struct dma_resv *resv;
+	struct dma_fence *fence;
+	u32 fence_count;
+	bool exclusive;
+	int i;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return NULL;
+
+	fence_count = pvr_buffer_sync_pmrs_fence_count(nr_pmrs, pmrs,
+						       pmr_flags);
+	if (fence_count) {
+		data->fences = kcalloc(fence_count, sizeof(*data->fences),
+				       GFP_KERNEL);
+		if (!data->fences)
+			goto err_check_data_free;
+	}
+
+	for (i = 0; i < nr_pmrs; i++) {
+		struct dma_resv_iter cursor;
+		bool include_write_fences;
+		bool include_read_fences;
+		u32 write_fence_count = 0;
+		u32 read_fence_count = 0;
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		dma_resv_count_fences(resv, &read_fence_count, &write_fence_count);
+
+		include_write_fences = (!exclusive || !read_fence_count);
+		include_read_fences = exclusive;
+
+		dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_READ);
+		dma_resv_for_each_fence_unlocked(&cursor, fence) {
+			enum dma_resv_usage usage = dma_resv_iter_usage(&cursor);
+
+			if ((!include_write_fences && usage == DMA_RESV_USAGE_WRITE) ||
+			    (!include_read_fences && usage == DMA_RESV_USAGE_READ))
+				continue;
+
+			data->fences[data->nr_fences++] =
+				pvr_fence_create_from_fence(fence_ctx,
+							    sync_checkpoint_ctx,
+							    fence,
+							    PVRSRV_NO_FENCE,
+							    (usage == DMA_RESV_USAGE_WRITE) ?
+							    "write check fence" :
+							    "read check fence");
+			if (!data->fences[data->nr_fences - 1]) {
+				data->nr_fences--;
+				PVR_FENCE_TRACE(fence,
+						(usage == DMA_RESV_USAGE_WRITE) ?
+						"waiting on write fence" :
+						"waiting on read fence\n");
+				WARN_ON(dma_fence_wait(fence, true) <= 0);
+			}
+		}
+	}
+
+	WARN_ON(i != nr_pmrs);
+
+	return data;
+
+err_check_data_free:
+	kfree(data);
+	return NULL;
+}
+
+#else
+
+static u32
+pvr_buffer_sync_pmrs_fence_count(u32 nr_pmrs, struct _PMR_ **pmrs,
+				 u32 *pmr_flags)
+{
+	struct dma_resv *resv;
+	struct dma_resv_list *resv_list;
+	struct dma_fence *fence;
+	u32 fence_count = 0;
+	bool exclusive;
+	int i;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		resv_list = dma_resv_shared_list(resv);
+		fence = dma_resv_excl_fence(resv);
+
+		if (fence &&
+		    (!exclusive || !resv_list || !resv_list->shared_count))
+			fence_count++;
+
+		if (exclusive && resv_list)
+			fence_count += resv_list->shared_count;
+	}
+
+	return fence_count;
+}
+
+static struct pvr_buffer_sync_check_data *
+pvr_buffer_sync_check_fences_create(struct pvr_fence_context *fence_ctx,
+				    PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+				    u32 nr_pmrs,
+				    struct _PMR_ **pmrs,
+				    u32 *pmr_flags)
+{
+	struct pvr_buffer_sync_check_data *data;
+	struct dma_resv *resv;
+	struct dma_resv_list *resv_list;
+	struct dma_fence *fence;
+	u32 fence_count;
+	bool exclusive;
+	int i, j;
+	int err;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return NULL;
+
+	fence_count = pvr_buffer_sync_pmrs_fence_count(nr_pmrs, pmrs,
+						       pmr_flags);
+	if (fence_count) {
+		data->fences = kcalloc(fence_count, sizeof(*data->fences),
+				       GFP_KERNEL);
+		if (!data->fences)
+			goto err_check_data_free;
+	}
+
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+		if (!exclusive) {
+			err = dma_resv_reserve_shared(resv
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+						      , 1
+#endif
+				);
+			if (err)
+				goto err_destroy_fences;
+		}
+
+		resv_list = dma_resv_shared_list(resv);
+		fence = dma_resv_excl_fence(resv);
+
+		if (fence &&
+		    (!exclusive || !resv_list || !resv_list->shared_count)) {
+			data->fences[data->nr_fences++] =
+				pvr_fence_create_from_fence(fence_ctx,
+							    sync_checkpoint_ctx,
+							    fence,
+							    PVRSRV_NO_FENCE,
+							    "exclusive check fence");
+			if (!data->fences[data->nr_fences - 1]) {
+				data->nr_fences--;
+				PVR_FENCE_TRACE(fence,
+						"waiting on exclusive fence\n");
+				WARN_ON(dma_fence_wait(fence, true) <= 0);
+			}
+		}
+
+		if (exclusive && resv_list) {
+			for (j = 0; j < resv_list->shared_count; j++) {
+				fence = rcu_dereference_protected(resv_list->shared[j],
+								  dma_resv_held(resv));
+				data->fences[data->nr_fences++] =
+					pvr_fence_create_from_fence(fence_ctx,
+								    sync_checkpoint_ctx,
+								    fence,
+								    PVRSRV_NO_FENCE,
+								    "check fence");
+				if (!data->fences[data->nr_fences - 1]) {
+					data->nr_fences--;
+					PVR_FENCE_TRACE(fence,
+							"waiting on non-exclusive fence\n");
+					WARN_ON(dma_fence_wait(fence, true) <= 0);
+				}
+			}
+		}
+	}
+
+	WARN_ON((i != nr_pmrs) || (data->nr_fences != fence_count));
+
+	return data;
+
+err_destroy_fences:
+	for (i = 0; i < data->nr_fences; i++)
+		pvr_fence_destroy(data->fences[i]);
+	kfree(data->fences);
+err_check_data_free:
+	kfree(data);
+	return NULL;
+}
+
+#endif
+
+static void
+pvr_buffer_sync_check_fences_destroy(struct pvr_buffer_sync_check_data *data)
+{
+	int i;
+
+	for (i = 0; i < data->nr_fences; i++)
+		pvr_fence_destroy(data->fences[i]);
+
+	kfree(data->fences);
+	kfree(data);
+}
+
+struct pvr_buffer_sync_context *
+pvr_buffer_sync_context_create(struct device *dev, const char *name)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+	struct pvr_buffer_sync_context *ctx;
+	int err;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+
+	ctx->fence_ctx = pvr_fence_context_create(priv->dev_node,
+						  NativeSyncGetFenceStatusWq(),
+						  name);
+	if (!ctx->fence_ctx) {
+		err = -ENOMEM;
+		goto err_free_ctx;
+	}
+
+	mutex_init(&ctx->ctx_lock);
+
+	return ctx;
+
+err_free_ctx:
+	kfree(ctx);
+err_exit:
+	return ERR_PTR(err);
+}
+
+void
+pvr_buffer_sync_context_destroy(struct pvr_buffer_sync_context *ctx)
+{
+	pvr_fence_context_destroy(ctx->fence_ctx);
+	kfree(ctx);
+}
+
+int
+pvr_buffer_sync_resolve_and_create_fences(struct pvr_buffer_sync_context *ctx,
+					  PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+					  u32 nr_pmrs,
+					  struct _PMR_ **pmrs,
+					  u32 *pmr_flags,
+					  u32 *nr_fence_checkpoints_out,
+					  PSYNC_CHECKPOINT **fence_checkpoints_out,
+					  PSYNC_CHECKPOINT *update_checkpoints_out,
+					  struct pvr_buffer_sync_append_data **data_out)
+{
+	struct pvr_buffer_sync_append_data *data;
+	PSYNC_CHECKPOINT *fence_checkpoints;
+	const size_t data_size = sizeof(*data);
+	const size_t pmrs_size = sizeof(*pmrs) * nr_pmrs;
+	const size_t pmr_flags_size = sizeof(*pmr_flags) * nr_pmrs;
+	int i;
+	int j;
+	int err;
+
+	if (unlikely((nr_pmrs && !(pmrs && pmr_flags)) ||
+	    !nr_fence_checkpoints_out || !fence_checkpoints_out ||
+	    !update_checkpoints_out))
+		return -EINVAL;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		if (unlikely(!(pmr_flags[i] & PVR_BUFFER_FLAG_MASK))) {
+			pr_err("%s: Invalid flags %#08x for pmr %p\n",
+			       __func__, pmr_flags[i], pmrs[i]);
+			return -EINVAL;
+		}
+	}
+
+#if defined(NO_HARDWARE)
+	/*
+	 * For NO_HARDWARE there's no checking or updating of sync checkpoints
+	 * which means SW waits on our fences will cause a deadlock (since they
+	 * will never be signalled). Avoid this by not creating any fences.
+	 */
+	nr_pmrs = 0;
+#endif
+
+	if (!nr_pmrs) {
+		*nr_fence_checkpoints_out = 0;
+		*fence_checkpoints_out = NULL;
+		*update_checkpoints_out = NULL;
+		*data_out = NULL;
+
+		return 0;
+	}
+
+	data = kzalloc(data_size + pmrs_size + pmr_flags_size, GFP_KERNEL);
+	if (unlikely(!data))
+		return -ENOMEM;
+
+	data->ctx = ctx;
+	data->pmrs = (struct _PMR_ **)(void *)(data + 1);
+	data->pmr_flags = (u32 *)(void *)(data->pmrs + nr_pmrs);
+
+	/*
+	 * It's expected that user space will provide a set of unique PMRs
+	 * but, as a PMR can have multiple handles, it's still possible to
+	 * end up here with duplicates. Take this opportunity to filter out
+	 * any remaining duplicates (updating flags when necessary) before
+	 * trying to process them further.
+	 */
+	for (i = 0; i < nr_pmrs; i++) {
+		for (j = 0; j < data->nr_pmrs; j++) {
+			if (data->pmrs[j] == pmrs[i]) {
+				data->pmr_flags[j] |= pmr_flags[i];
+				break;
+			}
+		}
+
+		if (j == data->nr_pmrs) {
+			data->pmrs[j] = pmrs[i];
+			data->pmr_flags[j] = pmr_flags[i];
+			data->nr_pmrs++;
+		}
+	}
+
+	err = pvr_buffer_sync_pmrs_lock(ctx, data->nr_pmrs, data->pmrs);
+	if (unlikely(err)) {
+		/*
+		 * -EINTR is returned if a signal arrives while trying to acquire a PMR
+		 * lock. In this case the operation should be retried after the signal
+		 * has been serviced. As this is expected behaviour, don't print an
+		 * error in this case.
+		 */
+		if (err != -EINTR) {
+			pr_err("%s: failed to lock pmrs (errno=%d)\n",
+			       __func__, err);
+		}
+		goto err_free_data;
+	}
+
+	/* create the check data */
+	data->check_data = pvr_buffer_sync_check_fences_create(ctx->fence_ctx,
+							 sync_checkpoint_ctx,
+							 data->nr_pmrs,
+							 data->pmrs,
+							 data->pmr_flags);
+	if (unlikely(!data->check_data)) {
+		err = -ENOMEM;
+		goto err_pmrs_unlock;
+	}
+
+	fence_checkpoints = kcalloc(data->check_data->nr_fences,
+				    sizeof(*fence_checkpoints),
+				    GFP_KERNEL);
+	if (fence_checkpoints) {
+		pvr_fence_get_checkpoints(data->check_data->fences,
+					  data->check_data->nr_fences,
+					  fence_checkpoints);
+	} else {
+		if (unlikely(data->check_data->nr_fences)) {
+			err = -ENOMEM;
+			goto err_free_check_data;
+		}
+	}
+
+	/* create the update fence */
+	data->update_fence = pvr_fence_create(ctx->fence_ctx,
+			sync_checkpoint_ctx,
+			SYNC_CHECKPOINT_FOREIGN_CHECKPOINT, "update fence");
+	if (unlikely(!data->update_fence)) {
+		err = -ENOMEM;
+		goto err_free_fence_checkpoints;
+	}
+
+	/*
+	 * We need to clean up the fences once the HW has finished with them.
+	 * We can do this using fence callbacks. However, instead of adding a
+	 * callback to every fence, which would result in more work, we can
+	 * simply add one to the update fence since this will be the last fence
+	 * to be signalled. This callback can do all the necessary clean up.
+	 *
+	 * Note: we take an additional reference on the update fence in case
+	 * it signals before we can add it to a reservation object.
+	 */
+	PVR_FENCE_TRACE(&data->update_fence->base,
+			"create fence calling dma_fence_get\n");
+	dma_fence_get(&data->update_fence->base);
+
+	*nr_fence_checkpoints_out = data->check_data->nr_fences;
+	*fence_checkpoints_out = fence_checkpoints;
+	*update_checkpoints_out = pvr_fence_get_checkpoint(data->update_fence);
+	*data_out = data;
+
+	return 0;
+
+err_free_fence_checkpoints:
+	kfree(fence_checkpoints);
+err_free_check_data:
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+err_pmrs_unlock:
+	pvr_buffer_sync_pmrs_unlock(ctx, data->nr_pmrs, data->pmrs);
+err_free_data:
+	kfree(data);
+	return err;
+}
+
+void
+pvr_buffer_sync_kick_succeeded(struct pvr_buffer_sync_append_data *data)
+{
+	struct dma_resv *resv;
+	int i;
+
+	dma_fence_enable_sw_signaling(&data->update_fence->base);
+
+	for (i = 0; i < data->nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(data->pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+		dma_resv_reserve_fences(resv, 1);
+#endif
+		if (data->pmr_flags[i] & PVR_BUFFER_FLAG_WRITE) {
+			PVR_FENCE_TRACE(&data->update_fence->base,
+					"added exclusive fence (%s) to resv %p\n",
+					data->update_fence->name, resv);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+			dma_resv_add_fence(resv,
+					   &data->update_fence->base,
+					   DMA_RESV_USAGE_WRITE);
+#else
+			dma_resv_add_excl_fence(resv,
+						&data->update_fence->base);
+#endif
+		} else if (data->pmr_flags[i] & PVR_BUFFER_FLAG_READ) {
+			PVR_FENCE_TRACE(&data->update_fence->base,
+					"added non-exclusive fence (%s) to resv %p\n",
+					data->update_fence->name, resv);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+			dma_resv_add_fence(resv,
+					   &data->update_fence->base,
+					   DMA_RESV_USAGE_READ);
+#else
+			dma_resv_add_shared_fence(resv,
+						  &data->update_fence->base);
+#endif
+		}
+	}
+
+	/*
+	 * Now that the fence has been added to the necessary
+	 * reservation objects we can safely drop the extra reference
+	 * we took in pvr_buffer_sync_resolve_and_create_fences().
+	 */
+	dma_fence_put(&data->update_fence->base);
+	pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs,
+					data->pmrs);
+
+	/* destroy the check fences */
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+	/* destroy the update fence */
+	pvr_fence_destroy(data->update_fence);
+
+	/* free the append data */
+	kfree(data);
+}
+
+void
+pvr_buffer_sync_kick_failed(struct pvr_buffer_sync_append_data *data)
+{
+
+	/* drop the extra reference we took on the update fence in
+	 * pvr_buffer_sync_resolve_and_create_fences().
+	 */
+	dma_fence_put(&data->update_fence->base);
+
+	if (data->nr_pmrs > 0)
+		pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs,
+					    data->pmrs);
+
+	/* destroy the check fences */
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+	/* destroy the update fence */
+	pvr_fence_destroy(data->update_fence);
+
+	/* free the append data */
+	kfree(data);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.h
new file mode 100644
index 000000000000..b6aadf940479
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_buffer_sync.h
@@ -0,0 +1,125 @@
+/*
+ * @File        pvr_buffer_sync.h
+ * @Title       PowerVR Linux buffer sync interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PVR_BUFFER_SYNC_H
+#define PVR_BUFFER_SYNC_H
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/types.h>
+
+struct _PMR_;
+struct pvr_buffer_sync_context;
+struct pvr_buffer_sync_append_data;
+
+/**
+ * pvr_buffer_sync_context_create - creates a buffer sync context
+ * @dev: Linux device
+ * @name: context name (used for debugging)
+ *
+ * pvr_buffer_sync_context_destroy() should be used to clean up the buffer
+ * sync context.
+ *
+ * Return: A buffer sync context or NULL if it fails for any reason.
+ */
+struct pvr_buffer_sync_context *
+pvr_buffer_sync_context_create(struct device *dev, const char *name);
+
+/**
+ * pvr_buffer_sync_context_destroy() - frees a buffer sync context
+ * @ctx: buffer sync context
+ */
+void
+pvr_buffer_sync_context_destroy(struct pvr_buffer_sync_context *ctx);
+
+/**
+ * pvr_buffer_sync_resolve_and_create_fences() - create checkpoints from
+ *                                               buffers
+ * @ctx: buffer sync context
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @nr_pmrs: number of buffer objects (PMRs)
+ * @pmrs: buffer array
+ * @pmr_flags: internal flags
+ * @nr_fence_checkpoints_out: returned number of fence sync checkpoints
+ * @fence_checkpoints_out: returned array of fence sync checkpoints
+ * @update_checkpoint_out: returned update sync checkpoint
+ * @data_out: returned buffer sync data
+ *
+ * After this call, either pvr_buffer_sync_kick_succeeded() or
+ * pvr_buffer_sync_kick_failed() must be called.
+ *
+ * Return: 0 on success or an error code otherwise.
+ */
+int
+pvr_buffer_sync_resolve_and_create_fences(struct pvr_buffer_sync_context *ctx,
+					  PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+					  u32 nr_pmrs,
+					  struct _PMR_ **pmrs,
+					  u32 *pmr_flags,
+					  u32 *nr_fence_checkpoints_out,
+					  PSYNC_CHECKPOINT **fence_checkpoints_out,
+					  PSYNC_CHECKPOINT *update_checkpoint_out,
+					  struct pvr_buffer_sync_append_data **data_out);
+
+/**
+ * pvr_buffer_sync_kick_succeeded() - cleans up after a successful kick
+ *                                    operation
+ * @data: buffer sync data returned by
+ *        pvr_buffer_sync_resolve_and_create_fences()
+ *
+ * Should only be called following pvr_buffer_sync_resolve_and_create_fences().
+ */
+void
+pvr_buffer_sync_kick_succeeded(struct pvr_buffer_sync_append_data *data);
+
+/**
+ * pvr_buffer_sync_kick_failed() - cleans up after a failed kick operation
+ * @data: buffer sync data returned by
+ *        pvr_buffer_sync_resolve_and_create_fences()
+ *
+ * Should only be called following pvr_buffer_sync_resolve_and_create_fences().
+ */
+void
+pvr_buffer_sync_kick_failed(struct pvr_buffer_sync_append_data *data);
+
+#endif /* PVR_BUFFER_SYNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.c
new file mode 100644
index 000000000000..240872e0b0fd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.c
@@ -0,0 +1,308 @@
+/*
+ * @File
+ * @Title       PowerVR Linux software "counting" timeline fence implementation
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/kref.h>
+
+#include "services_kernel_client.h"
+#include "pvr_counting_timeline.h"
+#include "pvr_sw_fence.h"
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+struct pvr_counting_fence_timeline {
+	struct pvr_sw_fence_context *context;
+
+	void *dbg_request_handle;
+
+	spinlock_t active_fences_lock;
+	u64 current_value; /* guarded by active_fences_lock */
+	u64 next_value; /* guarded by active_fences_lock */
+	struct list_head active_fences;
+
+	struct kref kref;
+};
+
+struct pvr_counting_fence {
+	u64 value;
+	struct dma_fence *fence;
+	struct list_head active_list_entry;
+};
+
+void pvr_counting_fence_timeline_dump_timeline(
+	void *data,
+	DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+	void *dump_debug_file)
+{
+
+	struct pvr_counting_fence_timeline *timeline =
+		(struct pvr_counting_fence_timeline *) data;
+	unsigned long flags;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "TL:%s SeqNum: %llu/%llu",
+					  pvr_sw_fence_context_name(
+							  timeline->context),
+					  timeline->current_value,
+					  timeline->next_value);
+
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+}
+
+static void
+pvr_counting_fence_timeline_debug_request(void *data, u32 verbosity,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	struct pvr_counting_fence_timeline *timeline =
+		(struct pvr_counting_fence_timeline *)data;
+	struct pvr_counting_fence *obj;
+	unsigned long flags;
+	char value[128];
+
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM)) {
+		spin_lock_irqsave(&timeline->active_fences_lock, flags);
+		pvr_sw_fence_context_value_str(timeline->context, value,
+					       sizeof(value));
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  "sw: %s @%s cur=%llu",
+				  pvr_sw_fence_context_name(timeline->context),
+				  value, timeline->current_value);
+		list_for_each_entry(obj, &timeline->active_fences,
+				    active_list_entry) {
+			obj->fence->ops->fence_value_str(obj->fence,
+							 value, sizeof(value));
+			PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+					  " @%s: val=%llu", value, obj->value);
+		}
+		spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+	}
+}
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_create(
+	const char *name)
+{
+	PVRSRV_ERROR srv_err;
+	struct pvr_counting_fence_timeline *timeline =
+		kzalloc(sizeof(*timeline), GFP_KERNEL);
+
+	if (!timeline)
+		goto err_out;
+
+	timeline->context = pvr_sw_fence_context_create(name,
+							"pvr_sw_sync");
+	if (!timeline->context)
+		goto err_free_timeline;
+
+	timeline->current_value = 0;
+	timeline->next_value = 1;
+	kref_init(&timeline->kref);
+	spin_lock_init(&timeline->active_fences_lock);
+	INIT_LIST_HEAD(&timeline->active_fences);
+
+	srv_err = PVRSRVRegisterDriverDbgRequestNotify(
+				&timeline->dbg_request_handle,
+				pvr_counting_fence_timeline_debug_request,
+				DEBUG_REQUEST_LINUXFENCE,
+				timeline);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+			   __func__, PVRSRVGetErrorString(srv_err));
+		goto err_free_timeline_ctx;
+	}
+
+err_out:
+	return timeline;
+
+err_free_timeline_ctx:
+	pvr_sw_fence_context_destroy(timeline->context);
+
+err_free_timeline:
+	kfree(timeline);
+	timeline = NULL;
+	goto err_out;
+}
+
+void pvr_counting_fence_timeline_force_complete(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	struct list_head *entry, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+#if defined(DEBUG) && !defined(SUPPORT_AUTOVZ)
+	/* This is just a safety measure. Normally we should never see any
+	 * unsignaled sw fences when we come here. Warn if we still do!
+	 */
+	WARN_ON(!list_empty(&timeline->active_fences));
+#endif
+
+	list_for_each_safe(entry, tmp, &timeline->active_fences) {
+		struct pvr_counting_fence *fence =
+			list_entry(entry, struct pvr_counting_fence,
+			active_list_entry);
+		dma_fence_signal(fence->fence);
+		dma_fence_put(fence->fence);
+		fence->fence = NULL;
+		list_del(&fence->active_list_entry);
+		kfree(fence);
+	}
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+}
+
+static void pvr_counting_fence_timeline_destroy(
+	struct kref *kref)
+{
+	struct pvr_counting_fence_timeline *timeline =
+		container_of(kref, struct pvr_counting_fence_timeline, kref);
+
+	WARN_ON(!list_empty(&timeline->active_fences));
+
+	PVRSRVUnregisterDriverDbgRequestNotify(timeline->dbg_request_handle);
+
+	pvr_sw_fence_context_destroy(timeline->context);
+	kfree(timeline);
+}
+
+void pvr_counting_fence_timeline_put(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	kref_put(&timeline->kref, pvr_counting_fence_timeline_destroy);
+}
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_get(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	if (!timeline)
+		return NULL;
+	kref_get(&timeline->kref);
+	return timeline;
+}
+
+struct dma_fence *pvr_counting_fence_create(
+	struct pvr_counting_fence_timeline *timeline, u64 *sync_pt_idx)
+{
+	unsigned long flags;
+	struct dma_fence *sw_fence;
+	struct pvr_counting_fence *fence = kmalloc(sizeof(*fence), GFP_KERNEL);
+
+	if (!fence)
+		return NULL;
+
+	sw_fence = pvr_sw_fence_create(timeline->context);
+	if (!sw_fence)
+		goto err_free_fence;
+
+	fence->fence = dma_fence_get(sw_fence);
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	fence->value = timeline->next_value++;
+	if (sync_pt_idx)
+		*sync_pt_idx = fence->value;
+
+	list_add_tail(&fence->active_list_entry, &timeline->active_fences);
+
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+
+	/* Counting fences can be signalled any time after creation */
+	dma_fence_enable_sw_signaling(sw_fence);
+
+	return sw_fence;
+
+err_free_fence:
+	kfree(fence);
+	return NULL;
+}
+
+bool pvr_counting_fence_timeline_inc(
+	struct pvr_counting_fence_timeline *timeline, u64 *sync_pt_idx)
+{
+	struct list_head *entry, *tmp;
+	unsigned long flags;
+	bool res;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	if (timeline->current_value == timeline->next_value-1) {
+		res = false;
+		goto exit_unlock;
+	}
+
+	timeline->current_value++;
+
+	if (sync_pt_idx)
+		*sync_pt_idx = timeline->current_value;
+
+	list_for_each_safe(entry, tmp, &timeline->active_fences) {
+		struct pvr_counting_fence *fence =
+			list_entry(entry, struct pvr_counting_fence,
+			active_list_entry);
+		if (fence->value <= timeline->current_value) {
+			dma_fence_signal(fence->fence);
+			dma_fence_put(fence->fence);
+			fence->fence = NULL;
+			list_del(&fence->active_list_entry);
+			kfree(fence);
+		}
+	}
+
+	res = true;
+
+exit_unlock:
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+
+	return res;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.h
new file mode 100644
index 000000000000..2cb8db1e5c50
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_counting_timeline.h
@@ -0,0 +1,68 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_COUNTING_TIMELINE_H__)
+#define __PVR_COUNTING_TIMELINE_H__
+
+#include "pvr_linux_fence.h"
+
+struct pvr_counting_fence_timeline;
+
+void pvr_counting_fence_timeline_dump_timeline(
+	void *data,
+	DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+	void *dump_debug_file);
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_create(
+	const char *name);
+void pvr_counting_fence_timeline_put(
+	struct pvr_counting_fence_timeline *fence_timeline);
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_get(
+	struct pvr_counting_fence_timeline *fence_timeline);
+struct dma_fence *pvr_counting_fence_create(
+	struct pvr_counting_fence_timeline *fence_timeline, u64 *sync_pt_idx);
+bool pvr_counting_fence_timeline_inc(
+	struct pvr_counting_fence_timeline *fence_timeline, u64 *sync_pt_idx);
+void pvr_counting_fence_timeline_force_complete(
+	struct pvr_counting_fence_timeline *fence_timeline);
+
+#endif /* !defined(__PVR_COUNTING_TIMELINE_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debug.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debug.c
new file mode 100644
index 000000000000..cf59a2d5aa3e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debug.c
@@ -0,0 +1,487 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides kernel side Debug Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/sched.h>
+#include <linux/pid.h>
+#include <linux/moduleparam.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "linkage.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "di_server.h"
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/******** BUFFERED LOG MESSAGES ********/
+
+/* Because we don't want to have to handle CCB wrapping, each buffered
+ * message is rounded up to PVRSRV_DEBUG_CCB_MESG_MAX bytes. This means
+ * there is the same fixed number of messages that can be stored,
+ * regardless of message length.
+ */
+
+#if defined(PVRSRV_DEBUG_CCB_MAX)
+
+#define PVRSRV_DEBUG_CCB_MESG_MAX	PVR_MAX_DEBUG_MESSAGE_LEN
+
+typedef struct
+{
+	const IMG_CHAR *pszFile;
+	IMG_INT iLine;
+	IMG_UINT32 ui32TID;
+	IMG_UINT32 ui32PID;
+	IMG_CHAR pcMesg[PVRSRV_DEBUG_CCB_MESG_MAX];
+	struct timeval sTimeVal;
+}
+PVRSRV_DEBUG_CCB;
+
+static PVRSRV_DEBUG_CCB gsDebugCCB[PVRSRV_DEBUG_CCB_MAX];
+
+static IMG_UINT giOffset;
+
+/* protects access to gsDebugCCB */
+static DEFINE_SPINLOCK(gsDebugCCBLock);
+
+static void
+AddToBufferCCB(const IMG_CHAR *pszFileName, IMG_UINT32 ui32Line,
+			   const IMG_CHAR *szBuffer)
+{
+	unsigned long uiFlags;
+
+	spin_lock_irqsave(&gsDebugCCBLock, uiFlags);
+
+	gsDebugCCB[giOffset].pszFile = pszFileName;
+	gsDebugCCB[giOffset].iLine   = ui32Line;
+	gsDebugCCB[giOffset].ui32TID = current->pid;
+	gsDebugCCB[giOffset].ui32PID = current->tgid;
+
+	do_gettimeofday(&gsDebugCCB[giOffset].sTimeVal);
+
+	OSStringSafeCopy(gsDebugCCB[giOffset].pcMesg, szBuffer,
+	              PVRSRV_DEBUG_CCB_MESG_MAX);
+
+	giOffset = (giOffset + 1) % PVRSRV_DEBUG_CCB_MAX;
+
+	spin_unlock_irqrestore(&gsDebugCCBLock, uiFlags);
+}
+
+void PVRSRVDebugPrintfDumpCCB(void)
+{
+	int i;
+	unsigned long uiFlags;
+
+	spin_lock_irqsave(&gsDebugCCBLock, uiFlags);
+
+	for (i = 0; i < PVRSRV_DEBUG_CCB_MAX; i++)
+	{
+		PVRSRV_DEBUG_CCB *psDebugCCBEntry =
+			&gsDebugCCB[(giOffset + i) % PVRSRV_DEBUG_CCB_MAX];
+
+		/* Early on, we won't have PVRSRV_DEBUG_CCB_MAX messages */
+		if (!psDebugCCBEntry->pszFile)
+		{
+			continue;
+		}
+
+		printk(KERN_ERR "%s:%d: (%ld.%ld, tid=%u, pid=%u) %s\n",
+			   psDebugCCBEntry->pszFile,
+			   psDebugCCBEntry->iLine,
+			   (long)psDebugCCBEntry->sTimeVal.tv_sec,
+			   (long)psDebugCCBEntry->sTimeVal.tv_usec,
+			   psDebugCCBEntry->ui32TID,
+			   psDebugCCBEntry->ui32PID,
+			   psDebugCCBEntry->pcMesg);
+
+		/* Clear this entry so it doesn't get printed the next time again. */
+		psDebugCCBEntry->pszFile = NULL;
+	}
+
+	spin_unlock_irqrestore(&gsDebugCCBLock, uiFlags);
+}
+
+#else /* defined(PVRSRV_DEBUG_CCB_MAX) */
+
+static INLINE void
+AddToBufferCCB(const IMG_CHAR *pszFileName, IMG_UINT32 ui32Line,
+			   const IMG_CHAR *szBuffer)
+{
+	(void)pszFileName;
+	(void)szBuffer;
+	(void)ui32Line;
+}
+
+void PVRSRVDebugPrintfDumpCCB(void)
+{
+	/* Not available */
+}
+
+#endif /* defined(PVRSRV_DEBUG_CCB_MAX) */
+
+static IMG_UINT32 PVRDebugLevel =
+	(
+	 DBGPRIV_FATAL | DBGPRIV_ERROR | DBGPRIV_WARNING
+#if defined(PVRSRV_DEBUG_CCB_MAX)
+	 | DBGPRIV_BUFFERED
+#endif /* defined(PVRSRV_DEBUG_CCB_MAX) */
+#if defined(PVR_DPF_ADHOC_DEBUG_ON)
+	 | DBGPRIV_DEBUG
+#endif /* defined(PVR_DPF_ADHOC_DEBUG_ON) */
+	);
+
+module_param(PVRDebugLevel, uint, 0644);
+MODULE_PARM_DESC(PVRDebugLevel,
+                 "Sets the level of debug output (default 0x7)");
+
+IMG_UINT32 OSDebugLevel(void)
+{
+	return PVRDebugLevel;
+}
+
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVRDebugLevel = ui32DebugLevel;
+}
+
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	return (PVRDebugLevel & ui32DebugLevel) != 0;
+}
+
+#else /* defined(PVRSRV_NEED_PVR_DPF) */
+
+IMG_UINT32 OSDebugLevel(void)
+{
+	return 0;
+}
+
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32DebugLevel);
+}
+
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32DebugLevel);
+	return IMG_FALSE;
+}
+
+#endif /* defined(PVRSRV_NEED_PVR_DPF) */
+
+#define	PVR_MAX_MSG_LEN PVR_MAX_DEBUG_MESSAGE_LEN
+
+/* Message buffer for messages */
+static IMG_CHAR gszBuffer[PVR_MAX_MSG_LEN + 1];
+
+/* The lock is used to control access to gszBuffer */
+static DEFINE_SPINLOCK(gsDebugLock);
+
+/*
+ * Append a string to a buffer using formatted conversion.
+ * The function takes a variable number of arguments, pointed
+ * to by the var args list.
+ */
+__printf(3, 0)
+static IMG_BOOL VBAppend(IMG_CHAR *pszBuf, IMG_UINT32 ui32BufSiz, const IMG_CHAR *pszFormat, va_list VArgs)
+{
+	IMG_UINT32 ui32Used;
+	IMG_UINT32 ui32Space;
+	IMG_INT32 i32Len;
+
+	ui32Used = OSStringLength(pszBuf);
+	BUG_ON(ui32Used >= ui32BufSiz);
+	ui32Space = ui32BufSiz - ui32Used;
+
+	i32Len = vsnprintf(&pszBuf[ui32Used], ui32Space, pszFormat, VArgs);
+	pszBuf[ui32BufSiz - 1] = 0;
+
+	/* Return true if string was truncated */
+	return i32Len < 0 || i32Len >= (IMG_INT32)ui32Space;
+}
+
+void PVRSRVReleasePrintfVArgs(const IMG_CHAR *pszFormat, va_list vaArgs)
+{
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+	IMG_INT32  result;
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	result = snprintf(pszBuf, (ui32BufSiz - 2), "PVR_K:  %u: ", current->pid);
+	PVR_ASSERT(result>0);
+	ui32BufSiz -= result;
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, vaArgs))
+	{
+		printk(KERN_INFO "%s (truncated)\n", pszBuf);
+	}
+	else
+	{
+		printk(KERN_INFO "%s\n", pszBuf);
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVReleasePrintf
+@Description    To output an important message to the user in release builds
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVReleasePrintf(const IMG_CHAR *pszFormat, ...)
+{
+	va_list vaArgs;
+
+	va_start(vaArgs, pszFormat);
+	PVRSRVReleasePrintfVArgs(pszFormat, vaArgs);
+	va_end(vaArgs);
+}
+
+#if defined(PVRSRV_NEED_PVR_TRACE)
+
+/*************************************************************************/ /*!
+@Function       PVRTrace
+@Description    To output a debug message to the user
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVTrace(const IMG_CHAR *pszFormat, ...)
+{
+	va_list VArgs;
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+	IMG_INT32  result;
+
+	va_start(VArgs, pszFormat);
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	result = snprintf(pszBuf, (ui32BufSiz - 2), "PVR: %u: ", current->pid);
+	PVR_ASSERT(result>0);
+	ui32BufSiz -= result;
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, VArgs))
+	{
+		printk(KERN_ERR "PVR_K:(Message Truncated): %s\n", pszBuf);
+	}
+	else
+	{
+		printk(KERN_ERR "%s\n", pszBuf);
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+
+	va_end(VArgs);
+}
+
+#endif /* defined(PVRSRV_NEED_PVR_TRACE) */
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/*
+ * Append a string to a buffer using formatted conversion.
+ * The function takes a variable number of arguments, calling
+ * VBAppend to do the actual work.
+ */
+__printf(3, 4)
+static IMG_BOOL BAppend(IMG_CHAR *pszBuf, IMG_UINT32 ui32BufSiz, const IMG_CHAR *pszFormat, ...)
+{
+	va_list VArgs;
+	IMG_BOOL bTrunc;
+
+	va_start (VArgs, pszFormat);
+
+	bTrunc = VBAppend(pszBuf, ui32BufSiz, pszFormat, VArgs);
+
+	va_end (VArgs);
+
+	return bTrunc;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintf
+@Description    To output a debug message to the user
+@Input          uDebugLevel The current debug level
+@Input          pszFile     The source file generating the message
+@Input          uLine       The line of the source file
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVDebugPrintf(IMG_UINT32 ui32DebugLevel,
+			   const IMG_CHAR *pszFullFileName,
+			   IMG_UINT32 ui32Line,
+			   const IMG_CHAR *pszFormat,
+			   ...)
+{
+	const IMG_CHAR *pszFileName = pszFullFileName;
+	IMG_CHAR *pszLeafName;
+	va_list vaArgs;
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+
+	if (!(PVRDebugLevel & ui32DebugLevel))
+	{
+		return;
+	}
+
+	va_start(vaArgs, pszFormat);
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	switch (ui32DebugLevel)
+	{
+		case DBGPRIV_FATAL:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Fatal): ", ui32BufSiz);
+			PVRSRV_REPORT_ERROR();
+			break;
+		}
+		case DBGPRIV_ERROR:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Error): ", ui32BufSiz);
+			PVRSRV_REPORT_ERROR();
+			break;
+		}
+		case DBGPRIV_WARNING:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Warn):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_MESSAGE:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Mesg):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_VERBOSE:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Verb):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_DEBUG:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Debug): ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_CALLTRACE:
+		case DBGPRIV_ALLOC:
+		case DBGPRIV_BUFFERED:
+		default:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K: ", ui32BufSiz);
+			break;
+		}
+	}
+
+	if (current->pid == task_tgid_nr(current))
+	{
+		(void) BAppend(pszBuf, ui32BufSiz, "%5u: ", current->pid);
+	}
+	else
+	{
+		(void) BAppend(pszBuf, ui32BufSiz, "%5u-%5u: ", task_tgid_nr(current) /* pid id of group*/, current->pid /* task id */);
+	}
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, vaArgs))
+	{
+		printk(KERN_ERR "%s (truncated)\n", pszBuf);
+	}
+	else
+	{
+		IMG_BOOL bTruncated = IMG_FALSE;
+
+#if !defined(__sh__)
+		pszLeafName = (IMG_CHAR *)strrchr (pszFileName, '/');
+
+		if (pszLeafName)
+		{
+			pszFileName = pszLeafName+1;
+		}
+#endif /* __sh__ */
+
+#if defined(DEBUG)
+		{
+			static const IMG_CHAR *lastFile;
+
+			if (lastFile == pszFileName)
+			{
+				bTruncated = BAppend(pszBuf, ui32BufSiz, " [%u]", ui32Line);
+			}
+			else
+			{
+				bTruncated = BAppend(pszBuf, ui32BufSiz, " [%s:%u]", pszFileName, ui32Line);
+				lastFile = pszFileName;
+			}
+		}
+#else
+		bTruncated = BAppend(pszBuf, ui32BufSiz, " [%u]", ui32Line);
+#endif
+
+		if (bTruncated)
+		{
+			printk(KERN_ERR "%s (truncated)\n", pszBuf);
+		}
+		else
+		{
+			if (ui32DebugLevel & DBGPRIV_BUFFERED)
+			{
+				AddToBufferCCB(pszFileName, ui32Line, pszBuf);
+			}
+			else
+			{
+				printk(KERN_ERR "%s\n", pszBuf);
+			}
+		}
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+
+	va_end (vaArgs);
+}
+
+#endif /* PVRSRV_NEED_PVR_DPF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.c
new file mode 100644
index 000000000000..63c557bc8459
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.c
@@ -0,0 +1,611 @@
+/*************************************************************************/ /*!
+@File
+@Title          DebugFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via DebugFS.
+
+                Note about locking in DebugFS module.
+
+                Access to DebugFS is protected against the race where any
+                file could be removed while being accessed or accessed while
+                being removed. Any calls to debugfs_remove() will block
+                until all operations are finished.
+
+                See implementation of proxy file operations (FULL_PROXY_FUNC)
+                and implementation of debugfs_file_[get|put]() in
+                fs/debugfs/file.c in Linux kernel sources for more details.
+
+                Not about locking for sequential files.
+
+                The seq_file objects have a mutex that protects access
+                to all of the file operations hence all of the sequential
+                *read* operations are protected.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvr_debugfs.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_bridge_k.h"
+#include "pvr_uaccess.h"
+#include "osdi_impl.h"
+
+#define _DRIVER_THREAD_ENTER() \
+	do { \
+		PVRSRV_ERROR eLocalError = PVRSRVDriverThreadEnter(NULL); \
+		if (eLocalError != PVRSRV_OK) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDriverThreadEnter failed: %s", \
+				__func__, PVRSRVGetErrorString(eLocalError))); \
+			return OSPVRSRVToNativeError(eLocalError); \
+		} \
+	} while (0)
+
+#define _DRIVER_THREAD_EXIT() \
+	PVRSRVDriverThreadExit(NULL)
+
+#define PVR_DEBUGFS_PVR_DPF_LEVEL PVR_DBG_ERROR
+
+typedef struct DFS_DIR
+{
+	struct dentry *psDirEntry;
+	struct DFS_DIR *psParentDir;
+} DFS_DIR;
+
+typedef struct DFS_ENTRY
+{
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+} DFS_ENTRY;
+
+typedef struct DFS_FILE
+{
+	struct dentry *psFileEntry;
+	struct DFS_DIR *psParentDir;
+	const struct seq_operations *psSeqOps;
+	struct DFS_ENTRY sEntry;
+	DI_ENTRY_TYPE eType;
+} DFS_FILE;
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	seq_write(pvNativeHandle, pvData, uiSize);
+}
+
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	seq_vprintf(pvNativeHandle, pszFmt, pArgs);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	seq_puts(pvNativeHandle, pszStr);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	struct seq_file *psSeqFile = pvNativeHandle;
+	return seq_has_overflowed(psSeqFile);
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- sequential file operations ----------------------------------------- */
+
+static void *_Start(struct seq_file *psSeqFile, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	void *pvRet = psEntry->sIterCb.pfnStart(&psEntry->sImplEntry, puiPos);
+
+	if (pvRet == DI_START_TOKEN)
+	{
+		return SEQ_START_TOKEN;
+	}
+
+	return pvRet;
+}
+
+static void _Stop(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	psEntry->sIterCb.pfnStop(&psEntry->sImplEntry, pvPriv);
+}
+
+static void *_Next(struct seq_file *psSeqFile, void *pvPriv, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	return psEntry->sIterCb.pfnNext(&psEntry->sImplEntry, pvPriv, puiPos);
+}
+
+static int _Show(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	if (pvPriv == SEQ_START_TOKEN)
+	{
+		pvPriv = DI_START_TOKEN;
+	}
+
+	return psEntry->sIterCb.pfnShow(&psEntry->sImplEntry, pvPriv);
+}
+
+static struct seq_operations _g_sSeqOps = {
+	.start = _Start,
+	.stop = _Stop,
+	.next = _Next,
+	.show = _Show
+};
+
+/* ----- file operations ---------------------------------------------------- */
+
+static int _Open(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psINode != NULL && psINode->i_private != NULL,
+	                        "psDFSFile is NULL", -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	psDFSFile = psINode->i_private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_open(psFile, psDFSFile->psSeqOps);
+	}
+	else
+	{
+		/* private data is NULL as it's going to be set below */
+		iRes = single_open(psFile, _Show, NULL);
+	}
+
+	if (iRes == 0)
+	{
+		struct seq_file *psSeqFile = psFile->private_data;
+
+		DFS_ENTRY *psEntry = OSAllocMem(sizeof(*psEntry));
+		if (psEntry == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem() failed", __func__));
+			iRes = -ENOMEM;
+			goto return_;
+		}
+
+		*psEntry = psDFSFile->sEntry;
+		psSeqFile->private = psEntry;
+		psEntry->sImplEntry.pvNative = psSeqFile;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to seq_open psFile, returning %d",
+		        __func__, iRes));
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static int _Close(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile = psINode->i_private;
+	DFS_ENTRY *psEntry;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* save pointer to DFS_ENTRY */
+	psEntry = ((struct seq_file *) psFile->private_data)->private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_release(psINode, psFile);
+	}
+	else
+	{
+		iRes = single_release(psINode, psFile);
+	}
+
+	/* free DFS_ENTRY allocated in _Open */
+	OSFreeMem(psEntry);
+
+	/* Validation check as seq_release (and single_release which calls it)
+	 * never fail */
+	if (iRes != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release psFile, returning %d",
+		        __func__, iRes));
+	}
+
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Read(struct file *psFile, char __user *pcBuffer,
+                     size_t uiCount, loff_t *puiPos)
+{
+	DFS_FILE *psDFSFile = psFile->f_path.dentry->d_inode->i_private;
+	ssize_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_read(psFile, pcBuffer, uiCount, puiPos);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file, pfnRead() "
+			        "returned %zd", __func__, iRes));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Count = uiCount;
+
+		IMG_CHAR *pcLocalBuffer = OSAllocMem(uiCount);
+		PVR_GOTO_IF_FALSE(pcLocalBuffer != NULL, return_);
+
+		iRes = psEntry->sIterCb.pfnRead(pcLocalBuffer, ui64Count, puiPos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file, pfnRead() "
+			        "returned %zd", __func__, iRes));
+			OSFreeMem(pcLocalBuffer);
+			goto return_;
+		}
+
+		if (pvr_copy_to_user(pcBuffer, pcLocalBuffer, iRes) != 0)
+		{
+			iRes = -1;
+		}
+
+		OSFreeMem(pcLocalBuffer);
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static loff_t _LSeek(struct file *psFile, loff_t iOffset, int iOrigin)
+{
+	DFS_FILE *psDFSFile = psFile->f_path.dentry->d_inode->i_private;
+	loff_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_lseek(psFile, iOffset, iOrigin);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position in psFile<%p> to offset "
+			        "%lld, iOrigin %d, seq_lseek() returned %lld (dentry='%s')", __func__,
+			        psFile, iOffset, iOrigin, iRes, psFile->f_path.dentry->d_name.name));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Pos;
+
+		switch (iOrigin)
+		{
+			case SEEK_SET:
+				ui64Pos = psFile->f_pos + iOffset;
+				break;
+			case SEEK_CUR:
+				ui64Pos = iOffset;
+				break;
+			case SEEK_END:
+				/* not supported as we don't know the file size here */
+				/* fall through */
+			default:
+				return -1;
+		}
+
+		/* only pass the absolute position to the callback, it's up to the
+		 * implementer to determine if the position is valid */
+
+		iRes = psEntry->sIterCb.pfnSeek(ui64Pos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position to offset "
+			        "%lld, pfnSeek() returned %lld", __func__,
+			        iOffset, iRes));
+			goto return_;
+		}
+
+		psFile->f_pos = ui64Pos;
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Write(struct file *psFile, const char __user *pszBuffer,
+                      size_t uiCount, loff_t *puiPos)
+{
+	struct inode *psINode = psFile->f_path.dentry->d_inode;
+	DFS_FILE *psDFSFile = psINode->i_private;
+	DI_ITERATOR_CB *psIter = &psDFSFile->sEntry.sIterCb;
+	IMG_CHAR *pcLocalBuffer;
+	IMG_UINT64 ui64Count;
+	IMG_INT64 i64Res = -EIO;
+	IMG_UINT64 ui64Pos = *puiPos;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+	PVR_LOG_RETURN_IF_FALSE(psIter->pfnWrite != NULL, "pfnWrite is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* Make sure we allocate the smallest amount of needed memory*/
+	ui64Count = psIter->ui32WriteLenMax;
+	PVR_LOG_GOTO_IF_FALSE(uiCount <= ui64Count, "uiCount too long", return_);
+	ui64Count = MIN(uiCount + 1, ui64Count);
+
+	/* allocate buffer with one additional byte for NUL character */
+	pcLocalBuffer = OSAllocMem(ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(pcLocalBuffer != NULL, "OSAllocMem() failed",
+	                      return_);
+
+	i64Res = pvr_copy_from_user(pcLocalBuffer, pszBuffer, ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(i64Res == 0, "pvr_copy_from_user() failed",
+	                      free_local_buffer_);
+
+	/* ensure that the framework user gets a NUL terminated buffer */
+	pcLocalBuffer[ui64Count - 1] = '\0';
+
+	i64Res = psIter->pfnWrite(pcLocalBuffer, ui64Count, &ui64Pos,
+	                          psDFSFile->sEntry.sImplEntry.pvPrivData);
+	PVR_LOG_GOTO_IF_FALSE(i64Res >= 0, "pfnWrite failed", free_local_buffer_);
+
+	*puiPos = ui64Pos;
+
+free_local_buffer_:
+	OSFreeMem(pcLocalBuffer);
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return i64Res;
+}
+
+static const struct file_operations _g_psFileOpsGen = {
+	.owner = THIS_MODULE,
+	.open = _Open,
+	.release = _Close,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+static const struct file_operations _g_psFileOpsRndAcc = {
+	.owner = THIS_MODULE,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	return PVRSRV_OK;
+}
+
+static void _DeInit(void)
+{
+}
+
+static PVRSRV_ERROR _CreateFile(const IMG_CHAR *pszName,
+                                DI_ENTRY_TYPE eType,
+                                const DI_ITERATOR_CB *psIterCb,
+                                void *pvPrivData,
+                                void *pvParentDir,
+                                void **pvFile)
+{
+	DFS_DIR *psParentDir = pvParentDir;
+	DFS_FILE *psFile;
+	umode_t uiMode = S_IFREG;
+	struct dentry *psEntry;
+	const struct file_operations *psFileOps = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvFile != NULL, "pvFile");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentDir != NULL, "pvParentDir");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			psFileOps = &_g_psFileOpsGen;
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			psFileOps = &_g_psFileOpsRndAcc;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto return_;
+	}
+
+	psFile = OSAllocMem(sizeof(*psFile));
+	PVR_LOG_GOTO_IF_NOMEM(psFile, eError, return_);
+
+	uiMode |= psIterCb->pfnShow != NULL || psIterCb->pfnRead != NULL ?
+	        S_IRUGO : 0;
+	uiMode |= psIterCb->pfnWrite != NULL ? S_IWUSR : 0;
+
+	psEntry = debugfs_create_file(pszName, uiMode, psParentDir->psDirEntry,
+	                              psFile, psFileOps);
+	if (IS_ERR_OR_NULL(psEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create debugfs '%s' file",
+		        __func__, pszName));
+
+		eError = psEntry == NULL ?
+		        PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_ERROR_INVALID_DEVICE;
+		goto free_file_;
+	}
+
+	psFile->eType = eType;
+	psFile->psSeqOps = &_g_sSeqOps;
+	psFile->sEntry.sIterCb = *psIterCb;
+	psFile->sEntry.sImplEntry.pvPrivData = pvPrivData;
+	psFile->sEntry.sImplEntry.pvNative = NULL;
+	psFile->sEntry.sImplEntry.psCb = &_g_sEntryCallbacks;
+	psFile->psParentDir = psParentDir;
+	psFile->psFileEntry = psEntry;
+
+	*pvFile = psFile;
+
+	return PVRSRV_OK;
+
+free_file_:
+	OSFreeMem(psFile);
+
+return_:
+	return eError;
+}
+
+static void _DestroyFile(void *pvFile)
+{
+	DFS_FILE *psFile = pvFile;
+
+	PVR_ASSERT(psFile != NULL);
+
+	psFile->psFileEntry->d_inode->i_private = NULL;
+
+	debugfs_remove(psFile->psFileEntry);
+	OSFreeMem(psFile);
+}
+
+static PVRSRV_ERROR _CreateDir(const IMG_CHAR *pszName,
+                               void *pvParentDir,
+                               void **ppvDir)
+{
+	DFS_DIR *psNewDir;
+	struct dentry *psDirEntry, *psParentDir = NULL;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvDir != NULL, "ppvDir");
+
+	psNewDir = OSAllocMem(sizeof(*psNewDir));
+	PVR_LOG_RETURN_IF_NOMEM(psNewDir, "OSAllocMem");
+
+	psNewDir->psParentDir = pvParentDir;
+
+	if (pvParentDir != NULL)
+	{
+		psParentDir = psNewDir->psParentDir->psDirEntry;
+	}
+
+	psDirEntry = debugfs_create_dir(pszName, psParentDir);
+	if (IS_ERR_OR_NULL(psDirEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create '%s' debugfs directory",
+		        __func__, pszName));
+		OSFreeMem(psNewDir);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psNewDir->psDirEntry = psDirEntry;
+	*ppvDir = psNewDir;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyDir(void *pvDir)
+{
+	DFS_DIR *psDir = pvDir;
+
+	PVR_ASSERT(psDir != NULL);
+
+	debugfs_remove(psDir->psDirEntry);
+	OSFreeMem(psDir);
+}
+
+PVRSRV_ERROR PVRDebugFsRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateFile,
+		.pfnDestroyEntry = _DestroyFile,
+		.pfnCreateGroup = _CreateDir,
+		.pfnDestroyGroup = _DestroyDir
+	};
+
+	return DIRegisterImplementation("debugfs", &sImplCb);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.h
new file mode 100644
index 000000000000..23ae55b12069
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_debugfs.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Title          DebugFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DEBUGFS_H
+#define PVR_DEBUGFS_H
+
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR PVRDebugFsRegister(void);
+
+#endif /* PVR_DEBUGFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drm.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drm.c
new file mode 100644
index 000000000000..145f40c5b884
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drm.c
@@ -0,0 +1,551 @@
+/*
+ * @File
+ * @Title       PowerVR DRM driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#include <drm/drm.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_print.h>
+#include <linux/dma-mapping.h>
+#else
+#include <drm/drmP.h> /* include before drm_crtc.h for kernels older than 3.9 */
+#endif
+
+#include <drm/drm_crtc.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+
+#include "module_common.h"
+#include "pvr_drm.h"
+#include "pvr_drv.h"
+#include "pvrversion.h"
+#include "services_kernel_client.h"
+#include "pvr_sync_ioctl_drm.h"
+#include "physmem_dmabuf_internal.h"
+
+#include "kernel_compatibility.h"
+
+#include "dkf_server.h"
+#include "dkp_impl.h"
+
+#define PVR_DRM_DRIVER_NAME PVR_DRM_NAME
+#define PVR_DRM_DRIVER_DESC "Imagination Technologies PVR DRM"
+#define	PVR_DRM_DRIVER_DATE "20170530"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#define	PVR_DRM_DRIVER_PRIME 0
+#else
+#define	PVR_DRM_DRIVER_PRIME DRIVER_PRIME
+#endif
+
+/*
+ * Protects global PVRSRV_DATA on a multi device system. i.e. this is used to
+ * protect the PVRSRVCommonDeviceXXXX() APIs in the Server common layer which
+ * are not re-entrant for device creation and initialisation.
+ */
+static DEFINE_MUTEX(g_device_mutex);
+
+/* Executed before sleep/suspend-to-RAM/S3. During this phase the content
+ * of the video memory is preserved (copied to system RAM). This step is
+ * necessary because the device can be powered off and the content of the
+ * video memory lost.
+ */
+static int pvr_pm_suspend(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("device %p\n", dev);
+
+	return PVRSRVDeviceSuspend(ddev);
+}
+
+/* Executed after the system is woken up from sleep/suspend-to-RAM/S3. This
+ * phase restores the content of the video memory from the system RAM.
+ */
+static int pvr_pm_resume(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("device %p\n", dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+/* Executed before the hibernation image is created. This callback allows to
+ * preserve the content of the video RAM into the system RAM which in turn
+ * is then stored into a disk.
+ */
+static int pvr_pm_freeze(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceSuspend(ddev);
+}
+
+/* Executed after the hibernation image is created or if the creation of the
+ * image has failed. This callback should undo whatever was done in
+ * pvr_pm_freeze to allow the device to operate in the same way as before the
+ * call to pvr_pm_freeze.
+ */
+static int pvr_pm_thaw(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+/* Executed after the hibernation image is created. This callback should not
+ * preserve the content of the video memory since this was already done
+ * in pvr_pm_freeze.
+ *
+ * Note: from the tests performed on a TestChip this callback is not executed
+ *       and driver's pvr_shutdown() is executed instead.
+ */
+static int pvr_pm_poweroff(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	PVRSRVDeviceShutdown(ddev);
+
+	return 0;
+}
+
+/* Executed after the content of the system memory is restored from the
+ * hibernation image. This callback restored video RAM from the system RAM
+ * and performs any necessary device setup required for the device to operate
+ * properly.
+ */
+static int pvr_pm_restore(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+const struct dev_pm_ops pvr_pm_ops = {
+	/* Sleep (suspend-to-RAM/S3) callbacks.
+	 * This mode saves the content of the video RAM to the system RAM and
+	 * powers off the device to reduce the power consumption. Because the
+	 * video RAM can be powered off, it needs to be preserved beforehand.
+	 */
+	.suspend = pvr_pm_suspend,
+	.resume = pvr_pm_resume,
+
+	/* Hibernation (suspend-to-disk/S4) callbacks.
+	 * This mode saves the content of the video RAM to the system RAM and then
+	 * dumps the system RAM to disk (swap partition or swap file). The system
+	 * then powers off. After power on the system RAM content is loaded from
+	 * the disk and then video RAM is restored from the system RAM.
+	 *
+	 * The procedure is executed in following order
+	 *
+	 * - Suspend-to-disk is triggered
+	 *   At this point the OS goes through the list of all registered devices and
+	 *   calls provided callbacks.
+	 * -- pvr_pm_freeze() is called
+	 *         The GPU is powered of and submitting new work is blocked.
+	 *         The content of the video RAM is saved to the system RAM, and
+	 *         other actions required to suspend the device are performed.
+	 * -- system RAM image is created and saved on the disk
+	 *         The disk now contains a snapshot for the DDK Driver for the
+	 *         moment when pvr_pm_freeze() was called.
+	 * -- pvr_pm_thaw() is called
+	 *         All actions taken in pvr_pm_freeze() are undone. The memory
+	 *         allocated for the video RAM is freed and all actions necessary
+	 *         to bring the device to operational state are taken.
+	 *         This makes sure that regardless if image was created successfully
+	 *         or not the device remains operational.
+	 *
+	 * - System is powered off
+	 * -- pvr_shutdown() is called
+	 *         No actions are required beside powering off the GPU.
+	 *
+	 * - System is powered up
+	 * -- system RAM image is read from the disk
+	 *         This restores the snapshot of the DDK driver along with the saved
+	 *         video RAM buffer.
+	 * -- pvr_pm_restore() is called
+	 *         Video RAM is restored from the buffer located in the system RAM.
+	 *         Actions to reset the device and bring it back to working state
+	 *         are taken. Video RAM buffer is freed.
+	 *         In summary the same procedure as in the case of pvr_pm_thaw() is
+	 *         performed.
+	 */
+	.freeze = pvr_pm_freeze,
+	.thaw = pvr_pm_thaw,
+	.poweroff = pvr_pm_poweroff,
+	.restore = pvr_pm_restore,
+};
+
+#if defined(SUPPORT_LINUX_FDINFO)
+static void pvr_drm_show_drm_info(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	int pid, void *hPrivHandle)
+{
+	struct pvr_drm_private *priv;
+	struct drm_device *pdev;
+
+	pdev = (struct drm_device *)hPrivHandle;
+
+	/* The only field possibly valid in 'priv' is the dev_node field */
+	priv = (struct pvr_drm_private *)pdev->dev_private;
+
+	if (priv->dev_node == psDevNode) {
+
+		/* For kernels post 6.5.0 the mandatory driver fields are produced
+		 * by the 'drm_show_fdinfo' routine in the kernel.
+		 * Avoid duplicating this information here.
+		 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0))
+		PVRDKPOutput(priv->hDeviceDKPRef,
+		             "drm-driver:\t%s\n",
+		             PVR_DRM_DRIVER_NAME);
+#if defined(CONFIG_PCI)
+		if (dev_is_pci(pdev->dev)) {
+			struct pci_dev *pcidev = to_pci_dev(pdev->dev);
+
+			PVRDKPOutput(priv->hDeviceDKPRef,
+			             "drm-pdev:\t%04x:%02x:%02x.%d\n",
+			             pci_domain_nr(pcidev->bus),
+			             pcidev->bus->number,
+			             PCI_SLOT(pcidev->devfn),
+			             PCI_FUNC(pcidev->devfn));
+		}
+#endif /* defined(CONFIG_PCI) */
+#endif	/* LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0) */
+	}
+}
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+int pvr_drm_load(struct drm_device *ddev, unsigned long flags)
+{
+	struct pvr_drm_private *priv;
+	enum PVRSRV_ERROR_TAG srv_err;
+	int err, deviceId;
+
+	DRM_DEBUG_DRIVER("device %p\n", ddev->dev);
+
+	dev_set_drvdata(ddev->dev, ddev);
+
+	if (ddev->render)
+		deviceId = ddev->render->index;
+	else /* when render node is NULL, fallback to primary node */
+		deviceId = ddev->primary->index;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+	ddev->dev_private = priv;
+
+	if (!ddev->dev->dma_parms)
+		ddev->dev->dma_parms = &priv->dma_parms;
+	dma_set_max_seg_size(ddev->dev, DMA_BIT_MASK(32));
+
+	mutex_lock(&g_device_mutex);
+
+	srv_err = PVRSRVCommonDeviceCreate(ddev->dev, deviceId, &priv->dev_node);
+	if (srv_err != PVRSRV_OK) {
+		DRM_ERROR("failed to create device node for device %p (%s)\n",
+			  ddev->dev, PVRSRVGetErrorString(srv_err));
+		if (srv_err == PVRSRV_ERROR_PROBE_DEFER)
+			err = -EPROBE_DEFER;
+		else
+			err = -ENODEV;
+		goto err_unset_dma_parms;
+	}
+
+	err = PVRSRVDeviceInit(priv->dev_node);
+	if (err) {
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+			  ddev->dev, err);
+		goto err_device_destroy;
+	}
+
+	drm_mode_config_init(ddev);
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+	srv_err = PVRSRVCommonDeviceInitialise(priv->dev_node);
+	if (srv_err != PVRSRV_OK) {
+		err = -ENODEV;
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+			  ddev->dev, err);
+		goto err_device_deinit;
+	}
+#endif
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	srv_err = PVRSRVRegisterDKP(ddev,
+	                            "drm-pvr-drm",
+	                            pvr_drm_show_drm_info,
+	                            DKP_CONNECTION_FLAG_ALL,
+	                            &priv->hDeviceDKPRef);
+
+	if (srv_err != PVRSRV_OK) {
+		err = -ENODEV;
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+				  ddev->dev, err);
+		goto err_device_deinit;
+	}
+#endif
+
+	mutex_unlock(&g_device_mutex);
+
+	return 0;
+
+#if defined(SUPPORT_LINUX_FDINFO) || (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+err_device_deinit:
+#endif
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+	drm_mode_config_cleanup(ddev);
+	PVRSRVDeviceDeinit(priv->dev_node);
+#endif
+err_device_destroy:
+	PVRSRVCommonDeviceDestroy(priv->dev_node);
+err_unset_dma_parms:
+	mutex_unlock(&g_device_mutex);
+	if (ddev->dev->dma_parms == &priv->dma_parms)
+		ddev->dev->dma_parms = NULL;
+	kfree(priv);
+err_exit:
+	return err;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+int pvr_drm_unload(struct drm_device *ddev)
+#else
+void pvr_drm_unload(struct drm_device *ddev)
+#endif
+{
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	DRM_DEBUG_DRIVER("device %p\n", ddev->dev);
+
+	drm_mode_config_cleanup(ddev);
+
+	PVRSRVDeviceDeinit(priv->dev_node);
+
+	mutex_lock(&g_device_mutex);
+	PVRSRVCommonDeviceDestroy(priv->dev_node);
+	mutex_unlock(&g_device_mutex);
+
+	if (ddev->dev->dma_parms == &priv->dma_parms)
+		ddev->dev->dma_parms = NULL;
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	PVRSRVUnRegisterDKP(ddev, priv->hDeviceDKPRef);
+#endif
+
+	kfree(priv);
+	ddev->dev_private = NULL;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	return 0;
+#endif
+}
+
+static int pvr_drm_open(struct drm_device *ddev, struct drm_file *dfile)
+{
+#if (PVRSRV_DEVICE_INIT_MODE != PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	struct pvr_drm_private *priv = ddev->dev_private;
+	int err;
+#endif
+
+	if (!try_module_get(THIS_MODULE)) {
+		DRM_ERROR("failed to get module reference\n");
+		return -ENOENT;
+	}
+
+#if (PVRSRV_DEVICE_INIT_MODE != PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	err = PVRSRVDeviceServicesOpen(priv->dev_node, dfile);
+	if (err)
+		module_put(THIS_MODULE);
+
+	return err;
+#else
+	return 0;
+#endif
+}
+
+static void pvr_drm_release(struct drm_device *ddev, struct drm_file *dfile)
+{
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	PVRSRVDeviceRelease(priv->dev_node, dfile);
+
+	module_put(THIS_MODULE);
+}
+
+static struct drm_ioctl_desc pvr_drm_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(PVR_SRVKM_CMD, PVRSRV_BridgeDispatchKM,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SRVKM_INIT, drm_pvr_srvkm_init,
+			  DRM_RENDER_ALLOW),
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_RENAME_CMD, pvr_sync_rename_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_FORCE_SW_ONLY_CMD, pvr_sync_force_sw_only_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SW_SYNC_CREATE_FENCE_CMD, pvr_sw_sync_create_fence_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SW_SYNC_INC_CMD, pvr_sw_sync_inc_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_EXP_FENCE_SYNC_FORCE_CMD, pvr_sync_ioctl_force_exp_only,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_CREATE_EXPORT_FENCE_CMD,
+			  pvr_export_fence_sync_create_fence_ioctl,
+			  DRM_RENDER_ALLOW),
+#endif
+};
+
+#if defined(CONFIG_COMPAT)
+static long pvr_compat_ioctl(struct file *file, unsigned int cmd,
+			     unsigned long arg)
+{
+	unsigned int nr = DRM_IOCTL_NR(cmd);
+
+	if (nr < DRM_COMMAND_BASE)
+		return drm_compat_ioctl(file, cmd, arg);
+
+	return drm_ioctl(file, cmd, arg);
+}
+#endif /* defined(CONFIG_COMPAT) */
+
+#if defined(SUPPORT_LINUX_FDINFO)
+/*
+ * Produce the PVR specific fdinfo (utilization figures etc.) when queried.
+ *
+ * For kernels post 6.5 there is a helper function called 'drm_show_fdinfo'
+ * which will generate the mandatory keys (drm-driver, drm-pdev, drm-client-id)
+ * so we do not need to generate these if running on a later kernel etc.
+ *
+ */
+static void pvr_show_fdinfo(struct seq_file *seq_file, struct file *file)
+{
+	struct drm_file *dfile = file->private_data;
+	struct drm_device *dev = dfile->minor->dev;
+	struct drm_printer p = drm_seq_file_printer(seq_file);
+	PVRSRV_CONNECTION_PRIV *pvr_connection = dfile->driver_priv;
+	struct pvr_drm_private *priv;
+	int my_pid;
+
+	/* Grab the PID from the associated drm_file->pid->numbers[0].nr */
+	my_pid = dfile->pid->numbers[0].nr;
+
+	priv = (struct pvr_drm_private *)dev->dev_private;
+
+	/* Generate driver-specific keys */
+	PVRDKFTraverse((DKF_VPRINTF_FUNC*)drm_vprintf,
+	               &p,
+	               priv->dev_node,
+	               my_pid,
+	               pvr_connection->ui32Type);
+
+	/* Call into OS-specific drm_show_fdinfo if it is supported */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0))
+	drm_show_fdinfo(seq_file, file);
+#endif	/* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0) */
+}
+#endif /* SUPPORT_LINUX_FDINFO */
+
+const struct file_operations pvr_drm_fops = {
+	.owner			= THIS_MODULE,
+	.open			= drm_open,
+	.release		= drm_release,
+	.unlocked_ioctl		= drm_ioctl,
+#if defined(CONFIG_COMPAT)
+	.compat_ioctl		= pvr_compat_ioctl,
+#endif
+	.mmap			= PVRSRV_MMap,
+	.poll			= drm_poll,
+	.read			= drm_read,
+#if defined(SUPPORT_LINUX_FDINFO)
+	.show_fdinfo	= pvr_show_fdinfo,
+#endif /* SUPPORT_LINUX_FDINFO */
+};
+
+const struct drm_driver pvr_drm_generic_driver = {
+	.driver_features	= DRIVER_MODESET | DRIVER_RENDER |
+				  DRIVER_GEM | PVR_DRM_DRIVER_PRIME,
+
+	.load			= NULL,
+	.unload			= NULL,
+	.open			= pvr_drm_open,
+	.postclose		= pvr_drm_release,
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	/* prime_fd_to_handle is not supported */
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_prime_export	= PhysmemGEMPrimeExport,
+	.gem_free_object	= PhysmemGEMObjectFree,
+#endif
+	.ioctls			= pvr_drm_ioctls,
+	.num_ioctls		= ARRAY_SIZE(pvr_drm_ioctls),
+	.fops			= &pvr_drm_fops,
+
+	.name			= PVR_DRM_DRIVER_NAME,
+	.desc			= PVR_DRM_DRIVER_DESC,
+	.date			= PVR_DRM_DRIVER_DATE,
+	.major			= PVRVERSION_MAJ,
+	.minor			= PVRVERSION_MIN,
+	.patchlevel		= PVRVERSION_BUILD,
+};
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drv.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drv.h
new file mode 100644
index 000000000000..827145d60c50
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_drv.h
@@ -0,0 +1,116 @@
+/*
+ * @File
+ * @Title       PowerVR DRM driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_DRV_H__)
+#define __PVR_DRV_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#include <linux/device.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <linux/pm.h>
+
+struct file;
+struct _PVRSRV_DEVICE_NODE_;
+struct workqueue_struct;
+struct vm_area_struct;
+
+/* This structure is used to store Linux specific per-device information. */
+struct pvr_drm_private {
+	struct _PVRSRV_DEVICE_NODE_ *dev_node;
+
+	/*
+	 * This is needed for devices that don't already have their own dma
+	 * parameters structure, e.g. platform devices, and, if necessary, will
+	 * be assigned to the 'struct device' during device initialisation. It
+	 * should therefore never be accessed directly via this structure as
+	 * this may not be the version of dma parameters in use.
+	 */
+	struct device_dma_parameters dma_parms;
+
+	/* PVR Sync debug notify handle */
+	void *sync_debug_notify_handle;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	/* Only used in fence sync as sync_debug_notify_handle is used
+	 * to print a header only. Content is registered separately.
+	 * Used to print foreign sync debug
+	 */
+	void *sync_foreign_debug_notify_handle;
+#endif
+
+	/* Flag stating if the device was suspended/resumed. If this is 0 then
+	 * the device was either resumed or no suspend was called but if 1 then
+	 * the OS called suspend on this device.
+	 */
+	atomic_t suspended;
+
+	/* Padding for 8-byte alignment */
+	unsigned int uiPad1;
+
+	/* Handle associated with DKP for this DRM device */
+	void *hDeviceDKPRef;
+};
+
+extern const struct dev_pm_ops pvr_pm_ops;
+extern const struct drm_driver pvr_drm_generic_driver;
+extern const struct file_operations pvr_drm_fops;
+
+int pvr_drm_load(struct drm_device *ddev, unsigned long flags);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+int pvr_drm_unload(struct drm_device *ddev);
+#else
+void pvr_drm_unload(struct drm_device *ddev);
+#endif
+
+int PVRSRV_BridgeDispatchKM(struct drm_device *dev, void *arg,
+			    struct drm_file *file);
+int PVRSRV_MMap(struct file *file, struct vm_area_struct *ps_vma);
+
+#endif /* !defined(__PVR_DRV_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.c
new file mode 100644
index 000000000000..30640d542b0e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.c
@@ -0,0 +1,363 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_common.c
+@Title          PowerVR devfreq device common utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux DVFS and PDVFS shared code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(NO_HARDWARE)
+
+#include <linux/version.h>
+#include <linux/device.h>
+#include <drm/drm.h>
+#if defined(CONFIG_PM_OPP)
+#include <linux/pm_opp.h>
+#endif
+
+#include "pvrsrv.h"
+
+/*
+ * Common DVFS support code shared between SUPPORT_LINUX_DVFS and
+ * SUPPORT_PDVFS, primarily for OPP table support.
+ *
+ * Note that PDVFS implements the Linux/OS devfreq module in
+ * the firmware, so no devfreq API calls should be used here.
+ */
+#include "pvr_dvfs.h"
+#include "pvr_dvfs_common.h"
+
+#include "kernel_compatibility.h"
+
+
+/*************************************************************************/ /*!
+@Function       GetOPPValues
+
+@Description    Common code to store the OPP points in the freq_table,
+                for use with the IMG DVFS/devfreq implementation, or with
+                the Proactive DVFS.
+                Requires CONFIG_PM_OPP support in the kernel.
+
+@Input          dev        OS Device node
+@Output         min_freq   Min clock freq (Hz)
+@Output         min_volt   Min voltage (V)
+@Output         max_freq   Max clock freq (Hz)
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(CONFIG_PM_OPP)
+int GetOPPValues(struct device *dev,
+                 unsigned long *min_freq,
+                 unsigned long *min_volt,
+                 unsigned long *max_freq,
+                 struct pvr_opp_freq_table *pvr_freq_table)
+{
+	struct dev_pm_opp *opp;
+	int count, i, err = 0;
+	unsigned long freq;
+
+	unsigned long *freq_table;
+
+	count = dev_pm_opp_get_opp_count(dev);
+	if (count <= 0)
+	{
+		dev_err(dev, "Could not fetch OPP count, %d\n", count);
+		return -EINVAL;
+	}
+
+	dev_info(dev, "Found %d OPP points.\n", count);
+
+	freq_table = devm_kcalloc(dev, count, sizeof(*freq_table), GFP_ATOMIC);
+	if (! freq_table)
+	{
+		return -ENOMEM;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	/* Start RCU read-side critical section to map frequency to OPP */
+	rcu_read_lock();
+#endif
+
+	/* Iterate over OPP table; Iteration 0 finds "opp w/ freq >= 0 Hz". */
+	freq = 0;
+	opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+	if (IS_ERR(opp))
+	{
+		err = PTR_ERR(opp);
+		dev_err(dev, "Couldn't find lowest frequency, %d\n", err);
+		goto exit;
+	}
+
+	*min_volt = dev_pm_opp_get_voltage(opp);
+	*max_freq = *min_freq = freq_table[0] = freq;
+	dev_info(dev, "opp[%d/%d]: (%lu Hz, %lu uV)\n", 1, count, freq, *min_volt);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	dev_pm_opp_put(opp);
+#endif
+
+	/* Iteration i > 0 finds "opp w/ freq >= (opp[i-1].freq + 1)". */
+	for (i = 1; i < count; i++)
+	{
+		freq++;
+		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+		if (IS_ERR(opp))
+		{
+			err = PTR_ERR(opp);
+			dev_err(dev, "Couldn't find %dth frequency, %d\n", i, err);
+			goto exit;
+		}
+
+		freq_table[i] = freq;
+		*max_freq = freq;
+		dev_info(dev,
+				 "opp[%d/%d]: (%lu Hz, %lu uV)\n",
+				  i + 1,
+				  count,
+				  freq,
+				  dev_pm_opp_get_voltage(opp));
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+		dev_pm_opp_put(opp);
+#endif
+	}
+
+exit:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#endif
+
+	if (!err)
+	{
+		pvr_freq_table->freq_table = freq_table;
+		pvr_freq_table->num_levels = count;
+	}
+	else
+	{
+		devm_kfree(dev, freq_table);
+	}
+
+	return err;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       DVFSCopyOPPTable
+
+@Description    Copies the OPP points (voltage/frequency) to the target
+                firmware structure RGXFWIF_OPP_INFO which must be pre-allocated.
+                The source values are expected to be read into the kernel
+                Power Manager from the platform's Device Tree.
+                Requires CONFIG_OF and CONFIG_PM_OPP support in the kernel.
+
+@Input          psDeviceNode       Device node
+@Input          psOPPInfo          Target OPP data buffer
+@Input          ui32MaxOPPLevels   Maximum number of OPP levels allowed in buffer.
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+PVRSRV_ERROR DVFSCopyOPPTable(PPVRSRV_DEVICE_NODE psDeviceNode,
+							  RGXFWIF_OPP_INFO   *psOPPInfo,
+							  IMG_UINT32          ui32MaxOPPLevels)
+{
+	PVRSRV_ERROR            eError = PVRSRV_OK;
+	struct device          *psDev = NULL;
+	OPP_LEVEL              *psOPPValue;
+	struct dev_pm_opp      *opp;
+	struct pvr_opp_freq_table pvr_freq_table = {0};
+	unsigned long           min_freq = 0, max_freq = 0, min_volt = 0;
+	unsigned int            i, err;
+	unsigned long *freq_table;
+
+	if (!psDeviceNode || !psOPPInfo)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid device or argument", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+	err = GetOPPValues(psDev, &min_freq, &min_volt, &max_freq, &pvr_freq_table);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: DVFS OPP table not initialised.", __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	freq_table = pvr_freq_table.freq_table;
+
+	if (pvr_freq_table.num_levels > ui32MaxOPPLevels)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Too many OPP levels (%u), max (%u).", __func__,
+		         pvr_freq_table.num_levels, ui32MaxOPPLevels));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto exit;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	/* Start RCU read-side critical section to map frequency to OPP */
+	rcu_read_lock();
+#endif
+
+	/* Loop over the OPP/frequency levels */
+	psOPPValue = &psOPPInfo->asOPPValues[0];
+	for (i=0; i<pvr_freq_table.num_levels; i++)
+	{
+		psOPPValue->ui32Freq = freq_table[i];
+		opp = dev_pm_opp_find_freq_exact(psDev, freq_table[i], IMG_TRUE);
+		if (IS_ERR(opp))
+		{
+			err = PTR_ERR(opp);
+			dev_err(psDev, "Couldn't find %dth frequency, %d\n", i, err);
+			eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+			goto exit;
+		}
+		psOPPValue->ui32Volt = dev_pm_opp_get_voltage(opp);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+		dev_pm_opp_put(opp);
+#endif
+		psOPPValue++;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Copied %u OPP points to the FW processor table.", __func__,
+	         pvr_freq_table.num_levels));
+	psOPPInfo->ui32MaxOPPPoint = pvr_freq_table.num_levels - 1;
+exit:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#endif
+
+	/* Free memory allocated in GetOPPValues */
+	devm_kfree(psDev, freq_table);
+
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+                Prepares the OPP table from the devicetree, if enabled.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_PDVFS)
+PVRSRV_ERROR InitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if !(defined(CONFIG_PM_OPP) && defined(CONFIG_OF))
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_OK;
+#else
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	struct device          *psDev;
+	int                     err;
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_ASSERT(psDeviceNode->psDevConfig);
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	/* Setup the OPP table from the device tree for Proactive DVFS. */
+	err = dev_pm_opp_of_add_table(psDev);
+	if (err == 0)
+	{
+		psDVFSDeviceCfg->bDTConfig = IMG_TRUE;
+	}
+	else
+	{
+		/*
+		 * If there are no device tree or system layer provided operating points
+		 * then return an error
+		 */
+		if (psDVFSDeviceCfg->pasOPPTable)
+		{
+			psDVFSDeviceCfg->bDTConfig = IMG_FALSE;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "No system or device tree opp points found, %d", err));
+			return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		}
+	}
+	return PVRSRV_OK;
+#endif
+}
+
+/*************************************************************************/ /*!
+@Function       DeinitPDVFS
+
+@Description    De-Initialise the device for Proactive DVFS support.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+void DeinitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if !(defined(CONFIG_PM_OPP) && defined(CONFIG_OF))
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#else
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = NULL;
+	struct device *psDev = NULL;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	if (psDVFSDeviceCfg->bDTConfig)
+	{
+		/*
+		 * Remove OPP entries for this device; only static entries from
+		 * the device tree are present.
+		 */
+		dev_pm_opp_of_remove_table(psDev);
+	}
+#endif
+}
+#endif /* SUPPORT_PDVFS */
+
+#endif /* !NO_HARDWARE */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.h
new file mode 100644
index 000000000000..87eccce5762e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_common.h
@@ -0,0 +1,123 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_common.h
+@Title          System level interface for DVFS and PDVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_COMMON_H
+#define PVR_DVFS_COMMON_H
+
+#include "opaque_types.h"
+#include "pvrsrv_error.h"
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+#include "rgx_fwif_km.h"
+#endif
+
+struct pvr_opp_freq_table
+{
+	unsigned long *freq_table;
+	int num_levels;
+};
+
+/*************************************************************************/ /*!
+@Function       GetOPPValues
+
+@Description    Common code to store the OPP points in the pvr_freq_table,
+                for use with the IMG DVFS/devfreq implementation, or with
+                the Proactive DVFS.
+                Requires CONFIG_PM_OPP support in the kernel.
+
+@Input          dev        OS Device node
+@Output         min_freq   Min clock freq (Hz)
+@Output         min_volt   Min voltage (V)
+@Output         max_freq   Max clock freq (Hz)
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(CONFIG_PM_OPP)
+int GetOPPValues(struct device *dev,
+                 unsigned long *min_freq,
+                 unsigned long *min_volt,
+                 unsigned long *max_freq,
+                 struct pvr_opp_freq_table *pvr_freq_table);
+#endif
+
+/*************************************************************************/ /*!
+@Function       DVFSCopyOPPTable
+
+@Description    Copies the OPP points (voltage/frequency) to the target
+                firmware structure RGXFWIF_OPP_INFO which must be pre-allocated.
+                The source values are expected to be read into the kernel
+                Power Manager from the platform's Device Tree.
+                Requires CONFIG_OF and CONFIG_PM_OPP support in the kernel.
+
+@Input          psDeviceNode       Device node
+@Input          psOPPInfo          Target OPP data buffer
+@Input          ui32MaxOPPLevels   Maximum number of OPP levels allowed in buffer.
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+PVRSRV_ERROR DVFSCopyOPPTable(PPVRSRV_DEVICE_NODE psDeviceNode,
+							  RGXFWIF_OPP_INFO   *psOPPInfo,
+							  IMG_UINT32          ui32MaxOPPLevels);
+#endif
+
+#if defined(SUPPORT_PDVFS)
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+                Prepares the OPP table from the devicetree, if enabled.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR InitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+void DeinitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+#endif /* PVR_DVFS_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.c
new file mode 100644
index 000000000000..f5e1a057a0ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.c
@@ -0,0 +1,1412 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_device.c
+@Title          PowerVR devfreq device implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux DVFS module setup
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(NO_HARDWARE)
+
+#include <linux/devfreq.h>
+#if defined(CONFIG_DEVFREQ_THERMAL)
+#include <linux/devfreq_cooling.h>
+#endif
+#include <linux/version.h>
+#include <linux/device.h>
+#include <drm/drm.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include "power.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+
+#include "rgxdevice.h"
+#include "rgxinit.h"
+#include "sofunc_rgx.h"
+
+#include "syscommon.h"
+
+#include "pvr_dvfs.h"
+#include "pvr_dvfs_device.h"
+#include "pvr_dvfs_common.h"
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+
+#if !defined(CONFIG_PM_DEVFREQ)
+#error "PVR DVFS governor requires kernel support for devfreq (CONFIG_PM_DEVFREQ = 1)"
+#endif
+#include "linux/governor.h"
+#include "pvr_dvfs_governor.h"
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+#include "linux/devfreq-event.h"
+#include "pvr_dvfs_events.h"
+#endif
+#endif /* SUPPORT_PVR_DVFS_GOVERNOR */
+
+#include "kernel_compatibility.h"
+
+
+/* Default constants for PVR-Balanced (PVR) governor */
+#define PVR_UPTHRESHOLD			(90)
+#define PVR_DOWNDIFFERENTIAL	(5)
+
+static int _device_get_devid(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	int deviceId;
+
+	if (ddev->render)
+		deviceId = ddev->render->index;
+	else /* when render node is NULL, fallback to primary node */
+		deviceId = ddev->primary->index;
+
+	return deviceId;
+}
+
+static IMG_INT32 devfreq_target(struct device *dev, unsigned long *requested_freq, IMG_UINT32 flags)
+{
+	int deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	RGX_DATA		*psRGXData = NULL;
+	IMG_DVFS_DEVICE		*psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG	*psDVFSDeviceCfg = NULL;
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo = NULL;
+	IMG_UINT32		ui32Freq, ui32CurFreq, ui32Volt;
+	struct dev_pm_opp *opp;
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	/* Check the RGX device is initialised */
+	if (!psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+	if (psDVFSDevice->eState != PVR_DVFS_STATE_READY)
+	{
+		*requested_freq = psRGXTimingInfo->ui32CoreClockSpeed;
+		return 0;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_lock();
+#endif
+
+	opp = devfreq_recommended_opp(dev, requested_freq, flags);
+	if (IS_ERR(opp)) {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+		rcu_read_unlock();
+#endif
+		PVR_DPF((PVR_DBG_ERROR, "Invalid OPP"));
+		return PTR_ERR(opp);
+	}
+
+	ui32Freq = dev_pm_opp_get_freq(opp);
+	ui32Volt = dev_pm_opp_get_voltage(opp);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#else
+	dev_pm_opp_put(opp);
+#endif
+
+	ui32CurFreq = psRGXTimingInfo->ui32CoreClockSpeed;
+
+	if (ui32CurFreq == ui32Freq)
+	{
+		return 0;
+	}
+
+	if (PVRSRV_OK != PVRSRVDevicePreClockSpeedChange(psDeviceNode,
+													 psDVFSDeviceCfg->bIdleReq,
+													 NULL))
+	{
+		dev_err(dev, "PVRSRVDevicePreClockSpeedChange failed\n");
+		return -EPERM;
+	}
+
+	/* Increasing frequency, change voltage first */
+	if (ui32Freq > ui32CurFreq)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, ui32Volt);
+	}
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDeviceNode->psDevConfig->hSysData, ui32Freq);
+
+	/* Decreasing frequency, change frequency first */
+	if (ui32Freq < ui32CurFreq)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, ui32Volt);
+	}
+
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32Freq;
+
+	PVRSRVDevicePostClockSpeedChange(psDeviceNode, psDVFSDeviceCfg->bIdleReq,
+									 NULL);
+
+	return 0;
+}
+
+static int devfreq_get_dev_status(struct device *dev, struct devfreq_dev_status *stat)
+{
+	int                      deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE      *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	PVRSRV_RGXDEV_INFO      *psDevInfo = NULL;
+	IMG_DVFS_DEVICE         *psDVFSDevice = NULL;
+	RGX_DATA                *psRGXData = NULL;
+	RGX_TIMING_INFORMATION  *psRGXTimingInfo = NULL;
+	RGXFWIF_GPU_UTIL_STATS  *psGpuUtilStats = NULL;
+	PVRSRV_ERROR             eError;
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	struct pvr_profiling_dev_status *pvr_stat = stat->private_data;
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* Check the RGX device is initialised */
+	if (!psDevInfo || !psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+	stat->current_frequency = psRGXTimingInfo->ui32CoreClockSpeed;
+
+	if (psDevInfo->pfnGetGpuUtilStats == NULL)
+	{
+		/* Not yet ready. So set times to something sensible. */
+		stat->busy_time = 0;
+		stat->total_time = 0;
+		return 0;
+	}
+
+	psGpuUtilStats = kzalloc(sizeof(*psGpuUtilStats), GFP_KERNEL);
+
+	if (!psGpuUtilStats)
+	{
+		return -ENOMEM;
+	}
+
+	eError = psDevInfo->pfnGetGpuUtilStats(psDeviceNode,
+						psDVFSDevice->hGpuUtilUserDVFS,
+						psGpuUtilStats);
+
+	if (eError != PVRSRV_OK)
+	{
+		kfree(psGpuUtilStats);
+		return -EAGAIN;
+	}
+
+	stat->busy_time = psGpuUtilStats->ui64GpuStatActive;
+	stat->total_time = psGpuUtilStats->ui64GpuStatCumulative;
+
+	kfree(psGpuUtilStats);
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	err = pvr_get_dev_status_get_events(psDVFSDevice->psProfilingDevice, pvr_stat);
+	if (err)
+		return err;
+#endif
+
+	return 0;
+}
+
+static IMG_INT32 devfreq_cur_freq(struct device *dev, unsigned long *freq)
+{
+	int deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	RGX_DATA *psRGXData = NULL;
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* Check the RGX device is initialised */
+	if (!psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	*freq = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return 0;
+}
+
+static struct devfreq_dev_profile img_devfreq_dev_profile =
+{
+	.polling_ms	        = 10,
+	.target             = devfreq_target,
+	.get_dev_status     = devfreq_get_dev_status,
+	.get_cur_freq       = devfreq_cur_freq,
+};
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+
+/* DEVFREQ governor name */
+#define DEVFREQ_GOV_PVR_BALANCED	"pvr_balanced"
+
+static unsigned long _get_max_freq(struct devfreq *devfreq_dev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+	return devfreq_dev->scaling_max_freq;
+#else
+	return devfreq_dev->max_freq;
+#endif
+}
+
+/*
+ * Calculate the utilisation percentage of X cycles as a proportion
+ * of Y cycles
+ */
+static inline
+unsigned long UTILISATION_PC(unsigned long X, unsigned long Y)
+{
+	PVR_ASSERT(Y > 0);
+
+/*
+	if (X < UINT_MAX / 100UL)
+		return min(100UL, X * 100UL / Y);
+	else
+		return min(100UL, X / (Y / 100UL));
+*/
+	/* Prevent overflow */
+	if (X < UINT32_MAX / 100U)
+		return (X * 100U / Y);
+	else
+		return (X / (Y / 100U));
+}
+
+static int pvr_governor_get_target(struct devfreq *devfreq_dev,
+								   unsigned long *freq)
+{
+	struct devfreq_dev_status *stat;
+	struct pvr_profiling_dev_status *pvr_stat;
+	IMG_DVFS_GOVERNOR_CFG *data = devfreq_dev->data;
+	unsigned long long a, b;
+	unsigned int pvr_upthreshold = PVR_UPTHRESHOLD;
+	unsigned int pvr_downdifferential = PVR_DOWNDIFFERENTIAL;
+	unsigned long ui32Util, ui32UtilDM, ui32UtilBus;
+	int iGeomEventID, iFragEventID, iCompEventID, iSLCReadEventID, iSLCWriteEventID;
+	int iSLCReadEventID2, iSLCWriteEventID2;
+	int err;
+
+	if (!devfreq_dev)
+	{
+		return -ENODEV;
+	}
+
+	/*
+	 *	Governor main function for setting target frequency, based on
+	 *	metrics for
+		- utilisation
+		- memory bus bandwidth
+		- ALU counters etc.
+	 */
+
+	err = devfreq_update_stats(devfreq_dev);
+	if (err)
+	{
+		dev_warn(&devfreq_dev->dev, "get_dev_status: error (%d)", err);
+		return err;
+	}
+
+	stat = &devfreq_dev->last_status;
+	pvr_stat = stat->private_data;
+	if (!pvr_stat)
+	{
+		return -EINVAL;
+	}
+
+	iGeomEventID = pvr_find_event_by_name("geom-cycle");
+	iFragEventID = pvr_find_event_by_name("3d-cycle");
+	iCompEventID = pvr_find_event_by_name("comp-cycle");
+	if (iGeomEventID < 0 || iFragEventID < 0 || iCompEventID < 0)
+	{
+		return -EINVAL;
+	}
+
+	if (pvr_stat->event_data[iGeomEventID].total_count == 0 ||
+	    pvr_stat->event_data[iFragEventID].total_count == 0 ||
+	    pvr_stat->event_data[iCompEventID].total_count == 0)
+	{
+		/* no error to avoid verbose kernel logs before the firmware/GPU
+		 * is available. */
+		return 0;
+	}
+
+	/* GEOM utilisation */
+	if (iGeomEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iGeomEventID].load_count,
+						   pvr_stat->event_data[iGeomEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	/* 3D utilisation */
+	if (iFragEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iFragEventID].load_count,
+						   pvr_stat->event_data[iFragEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	/* Compute utilisation */
+	if (iCompEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iCompEventID].load_count,
+						   pvr_stat->event_data[iCompEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	if (ui32UtilDM > 1000U)
+	{
+		/* Illegal value, keep current frequency */
+		*freq = stat->current_frequency;
+		return 0;
+	}
+
+	iSLCReadEventID =   pvr_find_event_by_name("slc-read");
+	iSLCWriteEventID =  pvr_find_event_by_name("slc-write");
+	iSLCReadEventID2 =  pvr_find_event_by_name("slc-read-2");
+	iSLCWriteEventID2 = pvr_find_event_by_name("slc-write-2");
+	if (iSLCReadEventID2 >= 0 && iSLCWriteEventID2 >= 0)
+	{
+		/* Use XE-series memory bus counters */
+
+		ui32UtilBus = 0;
+
+		while (iSLCWriteEventID < ARRAY_SIZE(g_pvr_governor_events))
+		{
+			ui32Util =
+				UTILISATION_PC(pvr_stat->event_data[iSLCReadEventID].load_count,
+							   pvr_stat->event_data[iSLCReadEventID].total_count);
+			ui32UtilBus = ui32UtilBus + ui32Util;
+
+			ui32Util =
+				UTILISATION_PC(pvr_stat->event_data[iSLCWriteEventID].load_count,
+							   pvr_stat->event_data[iSLCWriteEventID].total_count);
+			ui32UtilBus = ui32UtilBus + ui32Util;
+			iSLCReadEventID += 2;
+			iSLCWriteEventID += 2;
+		}
+	}
+	else if (iSLCReadEventID >= 0 && iSLCWriteEventID >= 0)
+	{
+		/* Use DXT-series memory bus counters */
+
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iSLCReadEventID].load_count / data->uiNumMembus,
+						   pvr_stat->event_data[iSLCReadEventID].total_count);
+		ui32UtilBus = ui32Util;
+
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iSLCWriteEventID].load_count / data->uiNumMembus,
+						   pvr_stat->event_data[iSLCWriteEventID].total_count);
+		ui32UtilBus = ui32UtilBus + ui32Util;
+	}
+
+	if (ui32UtilBus > 1000U)
+	{
+		/* Illegal value, use current frequency */
+		*freq = stat->current_frequency;
+		return 0;
+	}
+
+#if defined(DEBUG)
+	iSLCReadEventID =  pvr_find_event_by_name("slc-read");
+	iSLCWriteEventID = pvr_find_event_by_name("slc-write");
+	dev_info(&devfreq_dev->dev, "Geom: %10lu, 3D: %10lu, Comp: %10lu, SLC: %10lu cycles. DM %lu pc, Bus %lu pc.",
+			pvr_stat->event_data[iGeomEventID].load_count,
+			pvr_stat->event_data[iFragEventID].load_count,
+			pvr_stat->event_data[iCompEventID].load_count,
+			pvr_stat->event_data[iSLCReadEventID].load_count + pvr_stat->event_data[iSLCWriteEventID].load_count,
+			ui32UtilDM,
+			ui32UtilBus);
+#endif
+
+	if (ui32UtilDM > 2 * ui32UtilBus) {
+		// DM pipeline limited
+		ui32Util = ui32UtilDM;
+	}
+	else if (ui32UtilBus > 2 * ui32UtilDM) {
+		// Mem bandwidth limited
+		ui32Util = ui32UtilBus;
+	}
+	else {
+		// Average utilisation to smooth random fluctuations
+		ui32Util = (ui32UtilDM + ui32UtilBus) / 2;
+	}
+
+	/* Set MAX if it's busy enough */
+	if (ui32Util > pvr_upthreshold)
+	{
+		*freq = _get_max_freq(devfreq_dev);
+		return 0;
+	}
+
+	/* Set MAX if we do not know the initial frequency */
+	if (stat->current_frequency == 0) {
+		*freq = _get_max_freq(devfreq_dev);
+		return 0;
+	}
+
+	a = (unsigned long long) ui32Util * stat->current_frequency;
+	b = div_u64(a, (pvr_upthreshold - pvr_downdifferential / 2));
+	*freq = (unsigned long) b;
+	return 0;
+}
+
+static void pvr_governor_suspend(struct devfreq *devfreq_dev)
+{
+	/* Nothing to do */
+}
+
+static void pvr_governor_resume(struct devfreq *devfreq_dev)
+{
+	int deviceId;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_DVFS_DEVICE		*psDVFSDevice = NULL;
+	struct pvr_profiling_device *psProfilingDevice;
+
+	if (!devfreq_dev || !dev_get_drvdata(&devfreq_dev->dev))
+		return;
+
+	deviceId = _device_get_devid(&devfreq_dev->dev);
+	psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psProfilingDevice = psDVFSDevice->psProfilingDevice;
+
+	/* Reset the profiling events */
+	pvr_governor_reset_events(psProfilingDevice);
+}
+
+static int pvr_governor_event_handler(struct devfreq *devfreq_dev,
+									  unsigned int event, void *data)
+{
+	if (!devfreq_dev)
+	{
+		pr_err("%s: devfreq_dev not ready.\n", __func__);
+		return -ENODEV;
+	}
+
+	/*
+	 * We cannot take the deviceId here, as the DRM device
+	 * may not be initialised. Null pointer in
+	 * struct drm_device *ddev = dev_get_drvdata(dev)
+	 */
+
+	switch (event) {
+	case DEVFREQ_GOV_START:
+		dev_info(&devfreq_dev->dev,"GOV_START event.\n");
+		devfreq_monitor_start(devfreq_dev);
+		break;
+
+	case DEVFREQ_GOV_STOP:
+		dev_info(&devfreq_dev->dev,"GOV_STOP event.\n");
+		devfreq_monitor_stop(devfreq_dev);
+		break;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	case DEVFREQ_GOV_UPDATE_INTERVAL:
+		dev_info(&devfreq_dev->dev,"GOV_UPDATE_INTERVAL event.\n");
+		devfreq_update_interval(devfreq_dev, (unsigned int *)data);
+		break;
+
+#else
+	case DEVFREQ_GOV_INTERVAL:
+		dev_info(&devfreq_dev->dev,"GOV_INTERVAL event.\n");
+		devfreq_interval_update(devfreq_dev, (unsigned int *)data);
+		break;
+#endif
+
+	case DEVFREQ_GOV_SUSPEND:
+		dev_info(&devfreq_dev->dev,"GOV_SUSPEND event.\n");
+		pvr_governor_suspend(devfreq_dev);
+		devfreq_monitor_suspend(devfreq_dev);
+		break;
+
+	case DEVFREQ_GOV_RESUME:
+		dev_info(&devfreq_dev->dev,"GOV_RESUME event.\n");
+		devfreq_monitor_resume(devfreq_dev);
+		pvr_governor_resume(devfreq_dev);
+		break;
+
+	default:
+		printk("Unknown event.\n");
+		break;
+	}
+
+	return 0;
+}
+
+static struct devfreq_governor pvr_balanced_governor = {
+	.name = DEVFREQ_GOV_PVR_BALANCED,
+	.get_target_freq = pvr_governor_get_target,
+	.event_handler   = pvr_governor_event_handler,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	.attrs = DEVFREQ_GOV_ATTR_POLLING_INTERVAL
+		| DEVFREQ_GOV_ATTR_TIMER,
+#else
+	.immutable = true,
+#endif
+};
+
+int pvr_governor_init(void)
+{
+	int ret;
+
+	ret = devfreq_add_governor(&pvr_balanced_governor);
+	if (ret)
+	{
+		pr_err("%s: failed to install governor %d\n", __func__, ret);
+	}
+
+	return ret;
+}
+
+void pvr_governor_exit(void)
+{
+	int ret;
+
+	ret = devfreq_remove_governor(&pvr_balanced_governor);
+	if (ret)
+	{
+		pr_err("Failed to remove governor (%u)\n", ret);
+	}
+}
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+
+static int pvr_get_dev_status_get_events(struct pvr_profiling_device *pvr_prof_dev,
+										 struct pvr_profiling_dev_status *pvr_stat)
+{
+	struct devfreq_event_data event_data;
+	int i, ret = 0;
+
+	for (i = 0; i < pvr_prof_dev->num_events; i++)
+	{
+		if (!pvr_prof_dev->edev[i])
+			continue;
+
+		ret = devfreq_event_get_event(pvr_prof_dev->edev[i], &event_data);
+		if (ret < 0)
+			return ret;
+
+		pvr_stat->event_data[i].load_count = event_data.load_count;
+		pvr_stat->event_data[i].total_count = event_data.total_count;
+	}
+
+	return ret;
+}
+
+static int pvr_governor_reset_events(struct pvr_profiling_device *pvr_prof_dev)
+{
+	int i, ret = 0;
+
+	for (i = 0; i < pvr_prof_dev->num_events; i++)
+	{
+		if (!pvr_prof_dev->edev[i])
+			continue;
+
+		ret = devfreq_event_set_event(pvr_prof_dev->edev[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int pvr_events_enable(struct devfreq_event_dev *edev)
+{
+	/* Enable counters */
+
+	dev_dbg(&edev->dev, "%s (Enable)\n", edev->desc->name);
+
+	return 0;
+}
+
+static int pvr_events_disable(struct devfreq_event_dev *edev)
+{
+	/* Disable counters */
+
+	dev_dbg(&edev->dev, "%s (Disable)\n", edev->desc->name);
+
+	return 0;
+}
+
+static int pvr_find_event_by_name(const char *name)
+{
+	int idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(g_pvr_governor_events); idx++)
+		if (strcmp(name, g_pvr_governor_events[idx].name) == 0)
+			return idx;
+
+	return -EINVAL;
+}
+
+static int pvr_governor_set_event(struct devfreq_event_dev *edev)
+{
+	int id = pvr_find_event_by_name(edev->desc->name);
+	if (id < 0)
+		return -EINVAL;
+
+	/* clear the last value */
+	g_pvr_governor_events[id].last_value = 0;
+
+	return 0;
+}
+
+static int pvr_governor_get_event(struct devfreq_event_dev *edev,
+				struct devfreq_event_data *edata)
+{
+	struct pvr_profiling_device *info = devfreq_event_get_drvdata(edev);
+	volatile unsigned long perf_ctr;
+	int id = pvr_find_event_by_name(edev->desc->name);
+	if (id < 0)
+		return -EINVAL;
+
+	/*
+	 * Read performance counter
+	 */
+	perf_ctr = OSReadHWReg32(info->pvRegsBaseKM, g_pvr_governor_events[id].cntr);
+
+	/* Calculate deltas */
+	if (perf_ctr >= g_pvr_governor_events[id].last_value)
+	{
+		edata->load_count = perf_ctr - g_pvr_governor_events[id].last_value;
+	}
+	else
+	{
+		edata->load_count = UINT32_MAX - g_pvr_governor_events[id].last_value + perf_ctr + 1;
+	}
+
+	g_pvr_governor_events[id].last_value = perf_ctr;
+
+	/*
+	 * Calc the utilisation as a fraction of the total cycles
+	 */
+	if (edev->desc->event_type & PVR_EVENT_OP_PERF_CNTR)
+	{
+		edata->total_count = info->stat->event_data[0].load_count;
+
+		/*
+		 * Timer has 256-cycle granularity
+		 * Cycle counters have 1-cycle or 256-cycle granularity depending on GPU variant
+		 * SLC read/write counters have 1 event granularity
+		 */
+		if ((g_pvr_governor_events[0].shift > 0) && (g_pvr_governor_events[id].shift == 0))
+		{
+			int timer_shift = g_pvr_governor_events[0].shift;
+			if (edata->total_count < (UINT32_MAX >> timer_shift))
+			{
+				edata->total_count <<= timer_shift;	/*!< convert total to cycles */
+			}
+			else
+			{
+				/* avoid overflow but some loss of precision will occur */
+				edata->load_count >>= timer_shift;	/*!< convert load to ticks */
+			}
+		}
+	}
+	else
+	{
+		/* timer runs continuously, effective load is 100% */
+		edata->total_count = edata->load_count;
+	}
+
+	return 0;
+}
+
+static const struct devfreq_event_ops pvr_governor_event_ops = {
+	.enable = pvr_events_enable,
+	.disable = pvr_events_disable,
+	.set_event = pvr_governor_set_event,
+	.get_event = pvr_governor_get_event,
+};
+
+int pvr_events_register(struct device *dev, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice;
+	struct pvr_profiling_device *pvr_prof_dev;
+	struct devfreq_event_dev **edev;
+	struct devfreq_event_desc *desc;
+	int size, err;
+	int idx;
+
+	if (!psDeviceNode)
+		return -ENODEV;
+
+	pvr_prof_dev = devm_kzalloc(dev, sizeof(*pvr_prof_dev), GFP_KERNEL);
+	if (!pvr_prof_dev)
+		return -ENOMEM;
+
+	pvr_prof_dev->dev = dev;
+	pvr_prof_dev->num_events = ARRAY_SIZE(g_pvr_governor_events);
+
+	/* Alloc the event descriptor table */
+	desc = devm_kcalloc(dev, pvr_prof_dev->num_events, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	for (idx = 0; idx < pvr_prof_dev->num_events; idx++)
+	{
+		if (!g_pvr_governor_events[idx].name)
+			continue;
+
+		desc[idx].name = g_pvr_governor_events[idx].name;
+		desc[idx].event_type = (idx < PERF_CNTR_OFF) ? PVR_EVENT_OP_TIMER : PVR_EVENT_OP_PERF_CNTR;
+		desc[idx].event_type |= PVR_EVENT_OP_PERF_VER;
+		desc[idx].driver_data = pvr_prof_dev;
+		desc[idx].ops = &pvr_governor_event_ops;
+	}
+	pvr_prof_dev->desc = desc;
+
+	size = sizeof(struct devfreq_event_dev *) * pvr_prof_dev->num_events;
+	pvr_prof_dev->edev = devm_kzalloc(dev, size, GFP_KERNEL);
+	if (!pvr_prof_dev->edev)
+		return -ENOMEM;
+
+	/* Populate the events */
+	for (idx = 0, edev = pvr_prof_dev->edev; idx < pvr_prof_dev->num_events; idx++)
+	{
+		edev[idx] = devm_devfreq_event_add_edev(dev, &desc[idx]);
+		if (IS_ERR(edev[idx]))
+		{
+			dev_err(dev, "failed to add devfreq-event device (idx=%d)\n", idx);
+			return PTR_ERR(edev[idx]);
+		}
+
+		/* enable */
+		err = devfreq_event_enable_edev(edev[idx]);
+		if (err)
+		{
+			dev_err(dev, "failed to enable devfreq-event device (idx=%d)\n", idx);
+			return PTR_ERR(edev[idx]);
+		}
+
+		dev_info(dev, "%s: new PVR devfreq-event device registered %s (%s)\n",
+			__func__, dev_name(dev), desc[idx].name);
+	}
+
+	/* Allocate the profiling stats, passed to governor */
+	pvr_prof_dev->stat = devm_kzalloc(dev, sizeof(struct pvr_profiling_dev_status), GFP_KERNEL);
+	if (!pvr_prof_dev->stat)
+		return -ENOMEM;
+
+	/* Map the reg bank */
+	pvr_prof_dev->reg_size = psDeviceNode->psDevConfig->ui32RegsSize;
+	pvr_prof_dev->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvr_prof_dev->pvRegsBaseKM)
+	{
+		dev_err(dev, "failed to map register bank.");
+		return -ENODEV;
+	}
+
+	dev_info(dev, "Mapped regbank at %p, size 0x%x Bytes.\n", pvr_prof_dev->pvRegsBaseKM,
+			pvr_prof_dev->reg_size);
+
+	/* Update the DVFS device */
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDevice->psProfilingDevice = pvr_prof_dev;
+
+	return 0;
+}
+
+void pvr_events_unregister(struct device *dev, IMG_DVFS_DEVICE *psDVFSDevice)
+{
+	struct pvr_profiling_device *pvr_prof_dev = psDVFSDevice->psProfilingDevice;
+	struct devfreq_event_dev **edev;
+	int idx, err;
+
+	/* Remove the events */
+	for (idx = 0, edev = pvr_prof_dev->edev; idx < pvr_prof_dev->num_events; idx++)
+	{
+		err = devfreq_event_disable_edev(edev[idx]);
+		if (err)
+		{
+			dev_warn(dev, "failed to disable devfreq-event device (idx=%d)\n", idx);
+		}
+		devm_devfreq_event_remove_edev(dev, edev[idx]);
+	}
+
+	/* Unmap the register bank */
+	if (pvr_prof_dev != NULL && pvr_prof_dev->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) pvr_prof_dev->pvRegsBaseKM,
+		                 pvr_prof_dev->reg_size);
+	}
+
+	/* devfreq_event resources are managed by the kernel */
+}
+#endif /* CONFIG_PM_DEVFREQ_EVENT */
+
+#endif /* SUPPORT_PVR_DVFS_GOVERNOR */
+
+static int FillOPPTable(struct device *dev, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	const IMG_OPP *iopp;
+	int i, err = 0;
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = NULL;
+
+	/* Check the device exists */
+	if (!dev || !psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	if (!psDVFSDeviceCfg->pasOPPTable)
+	{
+		dev_err(dev, "No DVFS OPP table provided in system layer and no device tree support.");
+		return -ENODATA;
+	}
+
+	for (i = 0, iopp = psDVFSDeviceCfg->pasOPPTable;
+	     i < psDVFSDeviceCfg->ui32OPPTableSize;
+	     i++, iopp++)
+	{
+		err = dev_pm_opp_add(dev, iopp->ui32Freq, iopp->ui32Volt);
+		if (err) {
+			dev_err(dev, "Could not add OPP entry, %d\n", err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+static int RegisterCoolingDevice(struct device *dev,
+								 IMG_DVFS_DEVICE *psDVFSDevice,
+								 struct devfreq_cooling_power *powerOps)
+{
+	struct device_node *of_node;
+	int err = 0;
+
+	if (!psDVFSDevice)
+	{
+		return -EINVAL;
+	}
+
+	if (!powerOps)
+	{
+		dev_info(dev, "Cooling: power ops not registered, not enabling cooling");
+		return 0;
+	}
+
+	of_node = of_node_get(dev->of_node);
+
+	psDVFSDevice->psDevfreqCoolingDevice = of_devfreq_cooling_register_power(
+		of_node, psDVFSDevice->psDevFreq, powerOps);
+
+	if (IS_ERR(psDVFSDevice->psDevfreqCoolingDevice))
+	{
+		err = PTR_ERR(psDVFSDevice->psDevfreqCoolingDevice);
+		dev_err(dev, "Failed to register as devfreq cooling device %d", err);
+	}
+
+	of_node_put(of_node);
+
+	return err;
+}
+#endif
+
+#define TO_IMG_ERR(err) ((err == -EPROBE_DEFER) ? PVRSRV_ERROR_PROBE_DEFER : PVRSRV_ERROR_INIT_FAILURE)
+
+PVRSRV_ERROR InitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE        *psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	struct device          *psDev;
+	PVRSRV_ERROR            eError;
+	int                     err;
+
+#if !defined(CONFIG_PM_OPP)
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	PVR_ASSERT(psDeviceNode->psDevConfig);
+
+	if (psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState == PVR_DVFS_STATE_INIT_PENDING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "DVFS initialise pending for device node %p",
+				 psDeviceNode));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState = PVR_DVFS_STATE_INIT_PENDING;
+
+#if defined(SUPPORT_SOC_TIMER)
+	if (! psDeviceNode->psDevConfig->pfnSoCTimerRead)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "System layer SoC timer callback not implemented"));
+		//return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	eError = SORgxGpuUtilStatsRegister(&psDVFSDevice->hGpuUtilUserDVFS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to register to the GPU utilisation stats, %d", eError));
+		return eError;
+	}
+
+	err = dev_pm_opp_of_add_table(psDev);
+	if (err == 0)
+	{
+		psDVFSDeviceCfg->bDTConfig = IMG_TRUE;
+	}
+	else
+	{
+		/*
+		 * If there are no device tree or system layer provided operating points
+		 * then return an error
+		 */
+		if ((err == -ENOTSUPP || err == -ENODEV) && psDVFSDeviceCfg->pasOPPTable)
+		{
+			err = FillOPPTable(psDev, psDeviceNode);
+			if (err != 0 && err != -ENODATA)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to fill OPP table with data, %d", err));
+				eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+				goto err_exit;
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to init opp table from devicetree, %d", err));
+			eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+			goto err_exit;
+		}
+	}
+
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+	err = pvr_events_register(psDev, psDeviceNode);
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to init PVR profiling events, %d", err));
+		goto err_exit;
+	}
+#endif
+
+	err = pvr_governor_init();
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to init PVR governor, %d", err));
+		goto err_exit;
+	}
+	psDVFSDevice->bGovernorReady = true;
+#endif
+
+	PVR_ASSERT(psDev);
+	PVR_ASSERT(psDeviceNode);
+
+	PVR_TRACE(("PVR DVFS init pending: dev = %p, PVR device = %p",
+			   psDev, psDeviceNode));
+
+	return PVRSRV_OK;
+
+err_exit:
+	DeinitDVFS(psDeviceNode);
+	return eError;
+}
+
+PVRSRV_ERROR RegisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE        *psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	IMG_DVFS_GOVERNOR_CFG  *psDVFSGovernorCfg = NULL;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = NULL;
+	struct device          *psDev;
+	struct pvr_opp_freq_table	pvr_freq_table = {0};
+	unsigned long           min_freq = 0, max_freq = 0, min_volt = 0;
+	PVRSRV_ERROR            eError;
+	int                     err;
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState != PVR_DVFS_STATE_INIT_PENDING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "DVFS initialise not yet pending for device node %p",
+				 psDeviceNode));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	psDVFSGovernorCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSGovernorCfg;
+	psRGXTimingInfo = ((RGX_DATA *)psDeviceNode->psDevConfig->hDevData)->psRGXTimingInfo;
+	psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState = PVR_DVFS_STATE_READY;
+
+	err = GetOPPValues(psDev, &min_freq, &min_volt, &max_freq, &pvr_freq_table);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to read OPP points, %d", err));
+		eError = TO_IMG_ERR(err);
+		goto err_exit;
+	}
+
+	img_devfreq_dev_profile.freq_table = pvr_freq_table.freq_table;
+	img_devfreq_dev_profile.max_state = pvr_freq_table.num_levels;
+	img_devfreq_dev_profile.initial_freq = min_freq;
+	img_devfreq_dev_profile.polling_ms = psDVFSDeviceCfg->ui32PollMs;
+
+	psRGXTimingInfo->ui32CoreClockSpeed = min_freq;
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDeviceNode->psDevConfig->hSysData, min_freq);
+	psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, min_volt);
+
+#if !defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* Use the Linux 'simple_ondemand' governor */
+	psDVFSDevice->data.upthreshold = psDVFSGovernorCfg->ui32UpThreshold;
+	psDVFSDevice->data.downdifferential = psDVFSGovernorCfg->ui32DownDifferential;
+#endif
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	psDVFSDevice->data.ui32UpThreshold = psDVFSGovernorCfg->ui32UpThreshold;
+	psDVFSDevice->data.ui32DownDifferential = psDVFSGovernorCfg->ui32DownDifferential;
+
+#if defined(SUPPORT_RGX)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_MEMBUS))
+		{
+			psDVFSDevice->data.uiNumMembus = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_MEMBUS);
+		}
+		else
+		{
+			psDVFSDevice->data.uiNumMembus = 1;
+		}
+	}
+#endif
+#endif
+
+	psDVFSDevice->psDevFreq = devm_devfreq_add_device(psDev,
+													  &img_devfreq_dev_profile,
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+													  "pvr_balanced",
+													  &psDVFSDevice->data);
+#else
+													  "simple_ondemand",
+													  &psDVFSDevice->data);
+#endif
+
+	if (IS_ERR(psDVFSDevice->psDevFreq))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "Failed to add as devfreq device %p, %ld",
+				 psDVFSDevice->psDevFreq,
+				 PTR_ERR(psDVFSDevice->psDevFreq)));
+		eError = TO_IMG_ERR(PTR_ERR(psDVFSDevice->psDevFreq));
+		goto err_exit;
+	}
+#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 50)))
+	/* Handle Linux kernel bug where a NULL return can occur. */
+	if (psDVFSDevice->psDevFreq == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "Failed to add as devfreq device %p, NULL return",
+				 psDVFSDevice->psDevFreq));
+		eError = TO_IMG_ERR(-EINVAL);
+		goto err_exit;
+	}
+#endif
+
+	eError = SuspendDVFS(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVInit: Failed to suspend DVFS"));
+		goto err_exit;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+	psDVFSDevice->psDevFreq->scaling_min_freq = min_freq;
+	psDVFSDevice->psDevFreq->scaling_max_freq = max_freq;
+#else
+	psDVFSDevice->psDevFreq->min_freq = min_freq;
+	psDVFSDevice->psDevFreq->max_freq = max_freq;
+#endif
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	psDVFSDevice->psDevFreq->last_status.private_data =
+		(void *)psDVFSDevice->psProfilingDevice->stat;
+#endif
+
+	err = devfreq_register_opp_notifier(psDev, psDVFSDevice->psDevFreq);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to register opp notifier, %d", err));
+		eError = TO_IMG_ERR(err);
+		goto err_exit;
+	}
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		err = RegisterCoolingDevice(psDev, psDVFSDevice, psDVFSDeviceCfg->psPowerOps);
+		if (err)
+		{
+			eError = TO_IMG_ERR(err);
+			goto err_exit;
+		}
+	}
+#endif
+
+	PVR_TRACE(("PVR DVFS activated: %lu-%lu Hz, Period: %ums",
+			   min_freq,
+			   max_freq,
+			   psDVFSDeviceCfg->ui32PollMs));
+
+	return PVRSRV_OK;
+
+err_exit:
+	UnregisterDVFSDevice(psDeviceNode);
+	return eError;
+}
+
+void UnregisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice = NULL;
+	struct device *psDev = NULL;
+	IMG_INT32 i32Error;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+	if (! psDVFSDevice)
+	{
+		return;
+	}
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	if (!IS_ERR_OR_NULL(psDVFSDevice->psDevfreqCoolingDevice))
+	{
+		devfreq_cooling_unregister(psDVFSDevice->psDevfreqCoolingDevice);
+		psDVFSDevice->psDevfreqCoolingDevice = NULL;
+	}
+#endif
+
+	if (psDVFSDevice->psDevFreq)
+	{
+		i32Error = devfreq_unregister_opp_notifier(psDev, psDVFSDevice->psDevFreq);
+		if (i32Error < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to unregister OPP notifier"));
+		}
+
+		devm_devfreq_remove_device(psDev, psDVFSDevice->psDevFreq);
+		psDVFSDevice->psDevFreq = NULL;
+	}
+
+	psDVFSDevice->eState = PVR_DVFS_STATE_DEINIT;
+}
+
+void DeinitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice = NULL;
+	struct device *psDev = NULL;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	if (psDVFSDevice->psProfilingDevice)
+	{
+		pvr_events_unregister(psDev, psDVFSDevice);
+	}
+	if (psDVFSDevice->bGovernorReady)
+	{
+		pvr_governor_exit();
+		psDVFSDevice->bGovernorReady = false;
+	}
+#endif
+
+	/*
+	 * Remove OPP entries for this device; both static entries from
+	 * the device tree and dynamic entries.
+	 */
+	dev_pm_opp_remove_table(psDev);
+
+	SORgxGpuUtilStatsUnregister(psDVFSDevice->hGpuUtilUserDVFS);
+	psDVFSDevice->hGpuUtilUserDVFS = NULL;
+	psDVFSDevice->eState = PVR_DVFS_STATE_NONE;
+}
+
+PVRSRV_ERROR SuspendDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE	*psDVFSDevice = NULL;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	if (psDVFSDevice->eState == PVR_DVFS_STATE_DEINIT)
+	{
+		/* Device is shutting down, nothing to do. */
+		return PVRSRV_OK;
+	}
+	psDVFSDevice->eState = PVR_DVFS_STATE_OFF;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* Communicate power suspend to devfreq framework */
+	err = devfreq_suspend_device(psDVFSDevice->psDevFreq);
+	if (err < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Failed to suspend DVFS (%d)", err));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR ResumeDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE	*psDVFSDevice = NULL;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+
+	/* Not supported in GuestOS drivers */
+	psDVFSDevice->eState = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? PVR_DVFS_STATE_NONE : PVR_DVFS_STATE_READY;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Communicate power resume to devfreq framework */
+		err = devfreq_resume_device(psDVFSDevice->psDevFreq);
+		if (err < 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Failed to resume DVFS (%d)", err));
+			return PVRSRV_ERROR_INVALID_DEVICE;
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+#endif /* !NO_HARDWARE */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.h
new file mode 100644
index 000000000000..a246b240d9ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_dvfs_device.h
@@ -0,0 +1,62 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs.c
+@Title          System level interface for DVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_DEVICE_H
+#define PVR_DVFS_DEVICE_H
+
+#include "opaque_types.h"
+#include "pvrsrv_error.h"
+
+
+PVRSRV_ERROR InitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+void DeinitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR RegisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+void UnregisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR SuspendDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR ResumeDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+#endif /* PVR_DVFS_DEVICE_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.c
new file mode 100644
index 000000000000..9a192b1b6fd1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.c
@@ -0,0 +1,472 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/spinlock_types.h>
+#include <linux/atomic.h>
+#include <linux/slab.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+
+#include "pvr_export_fence.h"
+#include "osfunc_common.h"
+
+struct pvr_exp_fence_context {
+	struct kref kref;
+	unsigned int context;
+	char context_name[32];
+	char driver_name[32];
+	atomic_t seqno;
+	atomic_t fence_count;
+
+	void *cmd_complete_handle;
+
+	/* lock for signal and fence lists */
+	spinlock_t list_lock;
+	struct list_head signal_list;
+	struct list_head fence_list;
+};
+
+struct pvr_exp_fence {
+	struct dma_fence base;
+	struct pvr_exp_fence_context *fence_context;
+	PSYNC_CHECKPOINT checkpoint_handle;
+	/* dma_fence fd (needed for hwperf) */
+	int fd;
+	/* Lock for the dma fence */
+	spinlock_t lock;
+	/* fence will point to the dma_fence in base */
+	struct dma_fence *fence;
+	struct list_head fence_head;
+	struct list_head signal_head;
+};
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+static inline bool
+pvr_exp_fence_sync_is_signaled(struct pvr_exp_fence *exp_fence, u32 fence_sync_flags)
+{
+	if (exp_fence->checkpoint_handle) {
+		return SyncCheckpointIsSignalled(exp_fence->checkpoint_handle,
+						 fence_sync_flags);
+	}
+	return false;
+}
+
+const char *pvr_exp_fence_context_name(struct pvr_exp_fence_context *fctx)
+{
+	return fctx->context_name;
+}
+
+void pvr_exp_fence_context_value_str(struct pvr_exp_fence_context *fctx,
+				    char *str, int size)
+{
+	snprintf(str, size, "%d", atomic_read(&fctx->seqno));
+}
+
+static void
+pvr_exp_fence_context_fences_dump(struct pvr_exp_fence_context *fctx,
+				  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				  void *pvDumpDebugFile)
+{
+	unsigned long flags;
+	char value[128];
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	pvr_exp_fence_context_value_str(fctx, value, sizeof(value));
+	PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+			 "exp_fence_ctx: @%s", value);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+}
+
+static inline unsigned
+pvr_exp_fence_context_seqno_next(struct pvr_exp_fence_context *fence_context)
+{
+	if (fence_context)
+		return atomic_inc_return(&fence_context->seqno) - 1;
+	else
+		return 0xfeedface;
+}
+
+static void
+pvr_exp_fence_context_signal_fences(void *data)
+{
+	struct pvr_exp_fence_context *fctx = (struct pvr_exp_fence_context *)data;
+	struct pvr_exp_fence *pvr_exp_fence, *tmp;
+	unsigned long flags1;
+	int chkpt_ct = 0;
+	int chkpt_sig_ct = 0;
+
+	LIST_HEAD(signal_list);
+
+	/*
+	 * We can't call fence_signal while holding the lock as we can end up
+	 * in a situation whereby pvr_fence_foreign_signal_sync, which also
+	 * takes the list lock, ends up being called as a result of the
+	 * fence_signal below, i.e. fence_signal(fence) -> fence->callback()
+	 *  -> fence_signal(foreign_fence) -> foreign_fence->callback() where
+	 * the foreign_fence callback is pvr_fence_foreign_signal_sync.
+	 *
+	 * So extract the items we intend to signal and add them to their own
+	 * queue.
+	 */
+	spin_lock_irqsave(&fctx->list_lock, flags1);
+	list_for_each_entry_safe(pvr_exp_fence, tmp, &fctx->signal_list, signal_head) {
+		chkpt_ct++;
+		if (pvr_exp_fence_sync_is_signaled(pvr_exp_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT)) {
+			chkpt_sig_ct++;
+			list_move_tail(&pvr_exp_fence->signal_head, &signal_list);
+		}
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags1);
+
+	list_for_each_entry_safe(pvr_exp_fence, tmp, &signal_list, signal_head) {
+		spin_lock_irqsave(&pvr_exp_fence->fence_context->list_lock, flags1);
+		list_del(&pvr_exp_fence->signal_head);
+		spin_unlock_irqrestore(&pvr_exp_fence->fence_context->list_lock, flags1);
+		dma_fence_signal(pvr_exp_fence->fence);
+		dma_fence_put(pvr_exp_fence->fence);
+	}
+}
+
+static const char *pvr_exp_fence_get_driver_name(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		return pvr_exp_fence->fence_context->driver_name;
+	else
+		return "***NO_DRIVER***";
+}
+
+static const char *pvr_exp_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		return pvr_exp_fence_context_name(pvr_exp_fence->fence_context);
+	else
+		return "***NO_TIMELINE***";
+}
+
+static void pvr_exp_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	snprintf(str, size, "%llu", (u64) fence->seqno);
+}
+
+static void pvr_exp_fence_timeline_value_str(struct dma_fence *fence,
+					    char *str, int size)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		pvr_exp_fence_context_value_str(pvr_exp_fence->fence_context, str, size);
+}
+
+static bool pvr_exp_fence_enable_signaling(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *exp_fence = to_pvr_exp_fence(fence);
+	unsigned long flags;
+
+	if (!exp_fence)
+		return false;
+
+	if (pvr_exp_fence_sync_is_signaled(exp_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return false;
+
+	dma_fence_get(&exp_fence->base);
+
+	spin_lock_irqsave(&exp_fence->fence_context->list_lock, flags);
+	list_add_tail(&exp_fence->signal_head, &exp_fence->fence_context->signal_list);
+	spin_unlock_irqrestore(&exp_fence->fence_context->list_lock, flags);
+
+	return true;
+}
+
+static void pvr_exp_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_exp_fence_context *fence_context =
+		container_of(kref, struct pvr_exp_fence_context, kref);
+	unsigned int fence_count;
+
+	if (WARN_ON(!list_empty_careful(&fence_context->fence_list)))
+		pvr_exp_fence_context_fences_dump(fence_context, NULL, NULL);
+
+	PVRSRVUnregisterCmdCompleteNotify(fence_context->cmd_complete_handle);
+
+	fence_count = atomic_read(&fence_context->fence_count);
+	if (WARN_ON(fence_count))
+		pr_debug("%s context has %u fence(s) remaining\n",
+			 fence_context->context_name, fence_count);
+
+	kfree(fence_context);
+}
+
+static void pvr_exp_fence_release(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+	unsigned long flags;
+
+	if (pvr_exp_fence) {
+		if (pvr_exp_fence->fence_context) {
+			spin_lock_irqsave(&pvr_exp_fence->fence_context->list_lock, flags);
+			list_del(&pvr_exp_fence->fence_head);
+			atomic_dec(&pvr_exp_fence->fence_context->fence_count);
+			spin_unlock_irqrestore(&pvr_exp_fence->fence_context->list_lock, flags);
+
+			kref_put(&pvr_exp_fence->fence_context->kref,
+				 pvr_exp_fence_context_destroy_kref);
+		}
+
+		if (pvr_exp_fence->checkpoint_handle) {
+			SyncCheckpointFree(pvr_exp_fence->checkpoint_handle);
+			pvr_exp_fence->checkpoint_handle = NULL;
+		}
+
+		kfree(pvr_exp_fence);
+	}
+}
+
+static const struct dma_fence_ops pvr_exp_fence_ops = {
+	.get_driver_name = pvr_exp_fence_get_driver_name,
+	.get_timeline_name = pvr_exp_fence_get_timeline_name,
+	.fence_value_str = pvr_exp_fence_value_str,
+	.timeline_value_str = pvr_exp_fence_timeline_value_str,
+	.enable_signaling = pvr_exp_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+	.release = pvr_exp_fence_release,
+};
+
+struct pvr_exp_fence_context *
+pvr_exp_fence_context_create(const char *context_name, const char *driver_name)
+{
+	struct pvr_exp_fence_context *fence_context;
+	PVRSRV_ERROR srv_err;
+
+	fence_context = kzalloc(sizeof(*fence_context), GFP_KERNEL);
+	if (!fence_context)
+		return NULL;
+
+	fence_context->context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fence_context->context_name, context_name,
+		sizeof(fence_context->context_name));
+	OSStringSafeCopy(fence_context->driver_name, driver_name,
+		sizeof(fence_context->driver_name));
+	atomic_set(&fence_context->seqno, 0);
+	atomic_set(&fence_context->fence_count, 0);
+	kref_init(&fence_context->kref);
+
+	spin_lock_init(&fence_context->list_lock);
+	INIT_LIST_HEAD(&fence_context->signal_list);
+	INIT_LIST_HEAD(&fence_context->fence_list);
+
+	srv_err = PVRSRVRegisterCmdCompleteNotify(&fence_context->cmd_complete_handle,
+				pvr_exp_fence_context_signal_fences,
+				fence_context);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register command complete callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+		kfree(fence_context);
+		return NULL;
+	}
+
+	return fence_context;
+}
+
+void pvr_exp_fence_context_destroy(struct pvr_exp_fence_context *fence_context)
+{
+	if (fence_context) {
+		kref_put(&fence_context->kref, pvr_exp_fence_context_destroy_kref);
+	}
+}
+
+struct dma_fence *
+pvr_exp_fence_create(struct pvr_exp_fence_context *fence_context, int fd, u64 *sync_pt_idx)
+{
+	struct pvr_exp_fence *pvr_exp_fence;
+	unsigned long flags;
+	unsigned int seqno;
+	struct pvr_exp_fence_context *pfence_context = fence_context;
+
+	if (WARN_ON(!fence_context))
+		return NULL;
+
+	pvr_exp_fence = kzalloc(sizeof(*pvr_exp_fence), GFP_KERNEL);
+	if (WARN_ON(!pvr_exp_fence))
+		return NULL;
+
+	spin_lock_init(&pvr_exp_fence->lock);
+
+	INIT_LIST_HEAD(&pvr_exp_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_exp_fence->signal_head);
+
+	pvr_exp_fence->fence_context = pfence_context;
+
+	/* No sync checkpoint is assigned until attached to a kick */
+	pvr_exp_fence->checkpoint_handle = NULL;
+
+	seqno = pvr_exp_fence_context_seqno_next(pfence_context);
+
+	pvr_exp_fence->fence = &pvr_exp_fence->base;
+
+	dma_fence_init(&pvr_exp_fence->base, &pvr_exp_fence_ops,
+		       &pvr_exp_fence->lock, pfence_context->context, seqno);
+
+	atomic_inc(&pfence_context->fence_count);
+
+	spin_lock_irqsave(&fence_context->list_lock, flags);
+	list_add_tail(&pvr_exp_fence->fence_head, &fence_context->fence_list);
+	spin_unlock_irqrestore(&fence_context->list_lock, flags);
+
+	kref_get(&pfence_context->kref);
+
+	pvr_exp_fence->fd = fd;
+
+	*sync_pt_idx = pvr_exp_fence_context_seqno_next(pfence_context);
+
+	return &pvr_exp_fence->base;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_exp_fence_assign_checkpoint(PVRSRV_FENCE fence_to_resolve,
+				struct dma_fence *fence,
+				PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+				PSYNC_CHECKPOINT *assigned_checkpoint)
+{
+	struct SYNC_CHECKPOINT_TAG *new_sync_checkpoint;
+	struct pvr_exp_fence *pvr_exp_fence;
+	PVRSRV_FENCE export_fence_fd = fence_to_resolve;
+	PVRSRV_ERROR err;
+
+	pvr_exp_fence = to_pvr_exp_fence(fence);
+	if (!pvr_exp_fence)	{
+		pr_err("%s: Invalid fence_to_resolve\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (pvr_exp_fence->checkpoint_handle) {
+		/* export fence already has a sync checkpoint assigned */
+		*assigned_checkpoint = pvr_exp_fence->checkpoint_handle;
+		return PVRSRV_OK;
+	}
+
+	/* Ensure fd assigned to export fence when it was created is passed
+	 * to SyncCheckpointAlloc() (so correct fd is used in HWPerf event)
+	 * as the export fence may only be part of the check fence.
+	 */
+	if (pvr_exp_fence->fd != PVRSRV_NO_FENCE) {
+		export_fence_fd = pvr_exp_fence->fd;
+	}
+
+	err = SyncCheckpointAlloc(checkpoint_context,
+				  PVRSRV_NO_TIMELINE, export_fence_fd,
+				  fence->ops->get_timeline_name(fence), &new_sync_checkpoint);
+	if (unlikely(err != PVRSRV_OK)) {
+		pr_err("%s: SyncCheckpointAlloc() failed (err%d)\n",
+		       __func__, err);
+		*assigned_checkpoint = NULL;
+		goto err_out;
+	}
+
+	pvr_exp_fence->checkpoint_handle = new_sync_checkpoint;
+	*assigned_checkpoint = new_sync_checkpoint;
+	return PVRSRV_OK;
+
+err_out:
+	return err;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_exp_fence_rollback(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence;
+	PVRSRV_ERROR err;
+
+	pvr_exp_fence = to_pvr_exp_fence(fence);
+	if (!pvr_exp_fence) {
+		pr_err("%s: Invalid fence\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (pvr_exp_fence->checkpoint_handle) {
+		/* Free the assigned sync checkpoint */
+		SyncCheckpointFree(pvr_exp_fence->checkpoint_handle);
+		pvr_exp_fence->checkpoint_handle = NULL;
+	}
+
+	return PVRSRV_OK;
+
+err_out:
+	return err;
+}
+
+bool pvr_is_exp_fence(struct dma_fence *fence)
+{
+	return (fence->ops == &pvr_exp_fence_ops);
+}
+
+struct pvr_exp_fence *to_pvr_exp_fence(struct dma_fence *fence)
+{
+	if (pvr_is_exp_fence(fence))
+		return container_of(fence, struct pvr_exp_fence, base);
+
+	return NULL;
+}
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_exp_fence_get_checkpoint(struct pvr_exp_fence *export_fence)
+{
+	if (export_fence) {
+		return export_fence->checkpoint_handle;
+	}
+	else
+		return NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.h
new file mode 100644
index 000000000000..0ffe8a11ce40
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_export_fence.h
@@ -0,0 +1,77 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_EXPORT_FENCES_H__)
+#define __PVR_EXPORT_FENCES_H__
+
+#include "pvr_linux_fence.h"
+#include "services_kernel_client.h"
+
+struct pvr_exp_fence_context;
+struct pvr_exp_fence;
+
+struct pvr_exp_fence_context *pvr_exp_fence_context_create(const char *name,
+				const char *driver_name);
+void pvr_exp_fence_context_destroy(struct pvr_exp_fence_context *fence_context);
+struct dma_fence *pvr_exp_fence_create(struct pvr_exp_fence_context *fence_context,
+				       int fd,
+				       u64 *sync_pt_idx);
+
+const char *pvr_exp_fence_context_name(struct pvr_exp_fence_context *fctx);
+void pvr_exp_fence_context_value_str(struct pvr_exp_fence_context *fctx,
+				    char *str, int size);
+
+enum PVRSRV_ERROR_TAG pvr_exp_fence_assign_checkpoint(PVRSRV_FENCE fence_to_resolve,
+						      struct dma_fence *fence,
+						      PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+						      PSYNC_CHECKPOINT *assigned_checkpoint);
+
+enum PVRSRV_ERROR_TAG pvr_exp_fence_rollback(struct dma_fence *fence);
+
+bool pvr_is_exp_fence(struct dma_fence *fence);
+
+struct pvr_exp_fence *to_pvr_exp_fence(struct dma_fence *fence);
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_exp_fence_get_checkpoint(struct pvr_exp_fence *export_fence);
+
+#endif /* !defined(__PVR_EXPORT_FENCES_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.c
new file mode 100644
index 000000000000..130c68d825fc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.c
@@ -0,0 +1,1166 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include "pvr_fence.h"
+#include "services_kernel_client.h"
+#include "sync_checkpoint_external.h"
+#include "osfunc_common.h"
+
+#define CREATE_TRACE_POINTS
+#include "pvr_fence_trace.h"
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+/* Global kmem_cache for pvr_fence object allocations */
+static struct kmem_cache *pvr_fence_cache;
+static DEFINE_MUTEX(pvr_fence_cache_mutex);
+static u32 pvr_fence_cache_refcount;
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+static inline void
+pvr_fence_sync_signal(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
+{
+	SyncCheckpointSignal(pvr_fence->sync_checkpoint, fence_sync_flags);
+}
+
+static inline bool
+pvr_fence_sync_is_signaled(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
+{
+	return SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
+					 fence_sync_flags);
+}
+
+static inline u32
+pvr_fence_sync_value(struct pvr_fence *pvr_fence)
+{
+	if (SyncCheckpointIsErrored(pvr_fence->sync_checkpoint,
+				    PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return PVRSRV_SYNC_CHECKPOINT_ERRORED;
+	else if (SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
+					   PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+	else
+		return PVRSRV_SYNC_CHECKPOINT_ACTIVE;
+}
+
+static void
+pvr_fence_context_check_status(struct work_struct *data)
+{
+	PVRSRVCheckStatus(NULL);
+}
+
+void
+pvr_context_value_str(struct pvr_fence_context *fctx, char *str, int size)
+{
+	snprintf(str, size,
+		 "%u ctx=%llu refs=%u",
+		 atomic_read(&fctx->fence_seqno),
+		 fctx->fence_context,
+		 refcount_read(&fctx->kref.refcount));
+}
+
+static void
+pvr_fence_context_fences_dump(struct pvr_fence_context *fctx,
+			      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			      void *pvDumpDebugFile)
+{
+	struct pvr_fence *pvr_fence;
+	unsigned long flags;
+	char value[128];
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	pvr_context_value_str(fctx, value, sizeof(value));
+	PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+			 "%s: @%s", fctx->name, value);
+	list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
+		struct dma_fence *fence = pvr_fence->fence;
+		const char *timeline_value_str = "unknown timeline value";
+		const char *fence_value_str = "unknown fence value";
+
+		pvr_fence->base.ops->fence_value_str(&pvr_fence->base, value,
+						     sizeof(value));
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " @%s", value);
+
+		if (is_pvr_fence(fence))
+			continue;
+
+		if (fence->ops->timeline_value_str) {
+			fence->ops->timeline_value_str(fence, value,
+						       sizeof(value));
+			timeline_value_str = value;
+		}
+
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " | %s: %s (driver: %s)",
+				  fence->ops->get_timeline_name(fence),
+				  timeline_value_str,
+				  fence->ops->get_driver_name(fence));
+
+		if (fence->ops->fence_value_str) {
+			fence->ops->fence_value_str(fence, value,
+						    sizeof(value));
+			fence_value_str = value;
+		}
+
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " |  @%s (foreign)", fence_value_str);
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+}
+
+static inline unsigned int
+pvr_fence_context_seqno_next(struct pvr_fence_context *fctx)
+{
+	return atomic_inc_return(&fctx->fence_seqno) - 1;
+}
+
+/* This function prepends seqno to fence name */
+static inline void
+pvr_fence_prepare_name(char *fence_name, size_t fence_name_size,
+		const char *name, unsigned int seqno)
+{
+	unsigned int len;
+
+	len = OSStringUINT32ToStr(fence_name, fence_name_size, seqno);
+	if (likely((len > 0) && (fence_name_size >= (len + 1)))) {
+		fence_name[len] = '-';
+		fence_name[len + 1] = '\0';
+	}
+	strlcat(fence_name, name, fence_name_size);
+}
+
+static void
+pvr_fence_sched_free(struct rcu_head *rcu)
+{
+	struct pvr_fence *pvr_fence = container_of(rcu, struct pvr_fence, rcu);
+
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+}
+
+static inline void
+pvr_fence_context_free_deferred(struct pvr_fence_context *fctx)
+{
+	struct pvr_fence *pvr_fence, *tmp;
+	LIST_HEAD(deferred_free_list);
+	unsigned long flags;
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_for_each_entry_safe(pvr_fence, tmp,
+				 &fctx->deferred_free_list,
+				 fence_head)
+		list_move(&pvr_fence->fence_head, &deferred_free_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+	list_for_each_entry_safe(pvr_fence, tmp,
+				 &deferred_free_list,
+				 fence_head) {
+		list_del(&pvr_fence->fence_head);
+		SyncCheckpointFree(pvr_fence->sync_checkpoint);
+		call_rcu(&pvr_fence->rcu, pvr_fence_sched_free);
+		module_put(THIS_MODULE);
+	}
+}
+
+void
+pvr_fence_context_free_deferred_callback(void *data)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+
+	/*
+	 * Free up any fence objects we have deferred freeing.
+	 */
+	pvr_fence_context_free_deferred(fctx);
+}
+
+static void
+pvr_fence_context_signal_fences(void *data)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+	struct pvr_fence *pvr_fence, *tmp;
+	unsigned long flags1;
+
+	LIST_HEAD(signal_list);
+
+	/*
+	 * We can't call fence_signal while holding the lock as we can end up
+	 * in a situation whereby pvr_fence_foreign_signal_sync, which also
+	 * takes the list lock, ends up being called as a result of the
+	 * fence_signal below, i.e. fence_signal(fence) -> fence->callback()
+	 *  -> fence_signal(foreign_fence) -> foreign_fence->callback() where
+	 * the foreign_fence callback is pvr_fence_foreign_signal_sync.
+	 *
+	 * So extract the items we intend to signal and add them to their own
+	 * queue.
+	 */
+	spin_lock_irqsave(&fctx->list_lock, flags1);
+	list_for_each_entry_safe(pvr_fence, tmp, &fctx->signal_list, signal_head) {
+		if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+			list_move_tail(&pvr_fence->signal_head, &signal_list);
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags1);
+
+	list_for_each_entry_safe(pvr_fence, tmp, &signal_list, signal_head) {
+
+		PVR_FENCE_TRACE(&pvr_fence->base, "signalled fence (%s)\n",
+				pvr_fence->name);
+		trace_pvr_fence_signal_fence(pvr_fence);
+		spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags1);
+		list_del(&pvr_fence->signal_head);
+		spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags1);
+		dma_fence_signal(pvr_fence->fence);
+		dma_fence_put(pvr_fence->fence);
+	}
+
+	/*
+	 * Take this opportunity to free up any fence objects we
+	 * have deferred freeing.
+	 */
+	pvr_fence_context_free_deferred(fctx);
+}
+
+void
+pvr_fence_context_signal_fences_nohw(void *data)
+{
+	pvr_fence_context_signal_fences(data);
+}
+
+static void
+pvr_fence_context_destroy_internal(struct pvr_fence_context *fctx)
+{
+	pvr_fence_context_free_deferred(fctx);
+
+	if (WARN_ON(!list_empty_careful(&fctx->fence_list)))
+		pvr_fence_context_fences_dump(fctx, NULL, NULL);
+
+	PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
+
+	// wait for all fences to be freed before kmem_cache_destroy() is called
+	rcu_barrier();
+
+	/* Destroy pvr_fence object cache, if no one is using it */
+	WARN_ON(pvr_fence_cache == NULL);
+	mutex_lock(&pvr_fence_cache_mutex);
+	if (--pvr_fence_cache_refcount == 0)
+		kmem_cache_destroy(pvr_fence_cache);
+	mutex_unlock(&pvr_fence_cache_mutex);
+
+	kfree(fctx);
+}
+
+static void
+pvr_fence_context_unregister_dbg(void *dbg_request_handle)
+{
+	PVRSRVUnregisterDeviceDbgRequestNotify(dbg_request_handle);
+}
+
+static void
+pvr_fence_foreign_context_destroy_work(struct work_struct *data)
+{
+	struct pvr_fence_context *fctx =
+		container_of(data, struct pvr_fence_context, destroy_work);
+
+	pvr_fence_context_destroy_internal(fctx);
+}
+
+static void
+pvr_fence_context_destroy_work(struct work_struct *data)
+{
+	struct pvr_fence_context *fctx =
+		container_of(data, struct pvr_fence_context, destroy_work);
+
+	pvr_fence_context_unregister_dbg(fctx->dbg_request_handle);
+	pvr_fence_context_destroy_internal(fctx);
+}
+
+static void
+pvr_fence_context_debug_request(void *data, u32 verbosity,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+		pvr_fence_context_fences_dump(fctx, pfnDumpDebugPrintf,
+					      pvDumpDebugFile);
+}
+
+static struct pvr_fence_context *
+pvr_fence_context_create_internal(struct workqueue_struct *fence_status_wq,
+			const char *name,
+			work_func_t destroy_callback)
+{
+	struct pvr_fence_context *fctx;
+	PVRSRV_ERROR srv_err;
+
+	fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
+	if (!fctx)
+		return NULL;
+
+	spin_lock_init(&fctx->lock);
+	atomic_set(&fctx->fence_seqno, 0);
+	INIT_WORK(&fctx->check_status_work, pvr_fence_context_check_status);
+	INIT_WORK(&fctx->destroy_work, destroy_callback);
+	spin_lock_init(&fctx->list_lock);
+	INIT_LIST_HEAD(&fctx->signal_list);
+	INIT_LIST_HEAD(&fctx->fence_list);
+	INIT_LIST_HEAD(&fctx->deferred_free_list);
+
+	fctx->fence_wq = fence_status_wq;
+
+	fctx->fence_context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fctx->name, name, sizeof(fctx->name));
+
+	srv_err = PVRSRVRegisterCmdCompleteNotify(&fctx->cmd_complete_handle,
+				pvr_fence_context_signal_fences,
+				fctx);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register command complete callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+		goto err_free_fctx;
+	}
+
+	/* Create pvr_fence object cache, if not already created */
+	mutex_lock(&pvr_fence_cache_mutex);
+	if (pvr_fence_cache_refcount == 0) {
+		pvr_fence_cache = KMEM_CACHE(pvr_fence, 0);
+		if (!pvr_fence_cache) {
+			pr_err("%s: failed to allocate pvr_fence cache\n",
+					__func__);
+			mutex_unlock(&pvr_fence_cache_mutex);
+			goto err_unregister_cmd_complete_notify;
+		}
+	}
+	pvr_fence_cache_refcount++;
+	mutex_unlock(&pvr_fence_cache_mutex);
+
+	kref_init(&fctx->kref);
+
+	PVR_FENCE_CTX_TRACE(fctx, "created fence context (%s)\n", name);
+	trace_pvr_fence_context_create(fctx);
+
+	return fctx;
+
+err_unregister_cmd_complete_notify:
+	PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
+err_free_fctx:
+	kfree(fctx);
+	return NULL;
+}
+
+/**
+ * pvr_fence_context_register_dbg - registers the debug handler for a
+ * fence context
+ *
+ * @dbg_request_handle: handle used to keep a reference for deregister
+ * @dev: device to attach the debug notifier.
+ * @pvr_fence_context: context used as data to the callback for debug
+ *
+ * Registers a debug notifier for a given context for a given device.
+ *
+ * Returns PVRSRV_OK if successful.
+ */
+PVRSRV_ERROR pvr_fence_context_register_dbg(void *dbg_request_handle,
+				void *dev,
+				struct pvr_fence_context *fctx)
+{
+	PVRSRV_ERROR srv_err;
+
+	srv_err = PVRSRVRegisterDeviceDbgRequestNotify(dbg_request_handle,
+				dev,
+				pvr_fence_context_debug_request,
+				DEBUG_REQUEST_LINUXFENCE,
+				fctx);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+	}
+
+	return srv_err;
+}
+
+/**
+ * pvr_fence_foreign_context_create - creates a PVR fence context
+ * @fence_status_wq: linux workqueue used to signal foreign fences
+ * @name: context name (used for debugging)
+ *
+ * Creates a PVR foreign fence context that can be used to create PVR fences
+ * or to create PVR fences from an existing fence.
+ *
+ * pvr_fence_context_destroy should be called to clean up the fence context.
+ *
+ * Returns NULL if a context cannot be created.
+ */
+struct pvr_fence_context *
+pvr_fence_foreign_context_create(struct workqueue_struct *fence_status_wq,
+		const char *name)
+{
+	return pvr_fence_context_create_internal(fence_status_wq, name,
+							pvr_fence_foreign_context_destroy_work);
+}
+
+/**
+ * pvr_fence_context_create - creates a PVR fence context
+ * @dev_cookie: services device cookie
+ * @fence_status_wq: Status workqueue to queue fence update CBs.
+ * @name: context name (used for debugging)
+ *
+ * Creates a PVR fence context that can be used to create PVR fences or to
+ * create PVR fences from an existing fence.
+ *
+ * pvr_fence_context_destroy should be called to clean up the fence context.
+ *
+ * Returns NULL if a context cannot be created.
+ */
+struct pvr_fence_context *
+pvr_fence_context_create(void *dev_cookie,
+			 struct workqueue_struct *fence_status_wq,
+			 const char *name)
+{
+	struct pvr_fence_context *fctx;
+	PVRSRV_ERROR eError;
+
+	fctx = pvr_fence_context_create_internal(fence_status_wq, name,
+					pvr_fence_context_destroy_work);
+	if (fctx == NULL) {
+		pr_err("%s: failed to create fence context", __func__);
+		goto err_out;
+	}
+
+	fctx->dev_cookie = dev_cookie;
+
+	eError = pvr_fence_context_register_dbg(&fctx->dbg_request_handle,
+					dev_cookie,
+					fctx);
+	if (eError != PVRSRV_OK) {
+		pr_err("%s: failed to register fence context debug (%s)\n",
+		       __func__, PVRSRVGetErrorString(eError));
+		goto err_destroy_ctx;
+	}
+
+	PVR_FENCE_CTX_TRACE(fctx,
+						"%s: Created fence context (%s)[%p], dev_cookie %p\n",
+						__func__, fctx->name, fctx, dev_cookie);
+
+	return fctx;
+
+err_destroy_ctx:
+	pvr_fence_context_destroy(fctx);
+err_out:
+	return NULL;
+}
+
+static void pvr_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_fence_context *fctx =
+		container_of(kref, struct pvr_fence_context, kref);
+
+	PVR_FENCE_CTX_TRACE(fctx, "destroyed fence context (%s)\n", fctx->name);
+
+	trace_pvr_fence_context_destroy_kref(fctx);
+
+	queue_work(NativeSyncGetFenceCtxDestroyWq(), &fctx->destroy_work);
+}
+
+/**
+ * pvr_fence_context_destroy - destroys a context
+ * @fctx: PVR fence context to destroy
+ *
+ * Destroys a PVR fence context with the expectation that all fences have been
+ * destroyed.
+ */
+void
+pvr_fence_context_destroy(struct pvr_fence_context *fctx)
+{
+	trace_pvr_fence_context_destroy(fctx);
+
+	kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+}
+
+static const char *
+pvr_fence_get_driver_name(struct dma_fence *fence)
+{
+	return PVR_LDM_DRIVER_REGISTRATION_NAME;
+}
+
+static const char *
+pvr_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		return pvr_fence->fctx->name;
+	return NULL;
+}
+
+static
+void pvr_fence_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (!pvr_fence)
+		return;
+
+	snprintf(str, size,
+		 "%llu: (%s%s) refs=%u fwaddr=%#08x enqueue=%u status=%-9s %s%s",
+		 (u64) pvr_fence->fence->seqno,
+		 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 refcount_read(&pvr_fence->fence->refcount.refcount),
+		 SyncCheckpointGetFirmwareAddr(
+			 pvr_fence->sync_checkpoint),
+		 SyncCheckpointGetEnqueuedCount(pvr_fence->sync_checkpoint),
+		 SyncCheckpointGetStateString(pvr_fence->sync_checkpoint),
+		 pvr_fence->name,
+		 (&pvr_fence->base != pvr_fence->fence) ?
+		 "(foreign)" : "");
+}
+
+static
+void pvr_fence_timeline_value_str(struct dma_fence *fence, char *str, int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		pvr_context_value_str(pvr_fence->fctx, str, size);
+}
+
+static bool
+pvr_fence_enable_signaling(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (!pvr_fence)
+		return false;
+
+	WARN_ON_SMP(!spin_is_locked(&pvr_fence->fctx->lock));
+
+	if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return false;
+
+	dma_fence_get(&pvr_fence->base);
+
+	spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->signal_head, &pvr_fence->fctx->signal_list);
+	spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "signalling enabled (%s)\n",
+			pvr_fence->name);
+	trace_pvr_fence_enable_signaling(pvr_fence);
+
+	return true;
+}
+
+static bool
+pvr_fence_is_signaled(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		return pvr_fence_sync_is_signaled(pvr_fence,
+						  PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT);
+	return false;
+}
+
+static void
+pvr_fence_release(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (pvr_fence) {
+		struct pvr_fence_context *fctx = pvr_fence->fctx;
+
+		PVR_FENCE_TRACE(&pvr_fence->base, "released fence (%s)\n",
+				pvr_fence->name);
+		trace_pvr_fence_release(pvr_fence);
+
+		spin_lock_irqsave(&fctx->list_lock, flags);
+		list_move(&pvr_fence->fence_head,
+			  &fctx->deferred_free_list);
+		spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+		kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+	}
+}
+
+const struct dma_fence_ops pvr_fence_ops = {
+	.get_driver_name = pvr_fence_get_driver_name,
+	.get_timeline_name = pvr_fence_get_timeline_name,
+	.fence_value_str = pvr_fence_fence_value_str,
+	.timeline_value_str = pvr_fence_timeline_value_str,
+	.enable_signaling = pvr_fence_enable_signaling,
+	.signaled = pvr_fence_is_signaled,
+	.wait = dma_fence_default_wait,
+	.release = pvr_fence_release,
+};
+
+/**
+ * pvr_fence_create - creates a PVR fence
+ * @fctx: PVR fence context on which the PVR fence should be created
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @timeline_fd: timeline on which the PVR fence should be created
+ * @name: PVR fence name (used for debugging)
+ *
+ * Creates a PVR fence.
+ *
+ * Once the fence is finished with, pvr_fence_destroy should be called.
+ *
+ * Returns NULL if a PVR fence cannot be created.
+ */
+struct pvr_fence *
+pvr_fence_create(struct pvr_fence_context *fctx,
+		struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+		int timeline_fd, const char *name)
+{
+	struct pvr_fence *pvr_fence;
+	unsigned int seqno;
+	unsigned long flags;
+	PVRSRV_ERROR srv_err;
+
+	if (!try_module_get(THIS_MODULE))
+		goto err_exit;
+
+	/* Note: As kmem_cache is used to allocate pvr_fence objects,
+	 * make sure that all members of pvr_fence struct are initialized
+	 * here
+	 */
+	pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
+	if (unlikely(!pvr_fence))
+		goto err_module_put;
+
+	srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
+				      (PVRSRV_TIMELINE) timeline_fd, PVRSRV_NO_FENCE,
+				      name, &pvr_fence->sync_checkpoint);
+	if (unlikely(srv_err != PVRSRV_OK))
+		goto err_free_fence;
+
+	INIT_LIST_HEAD(&pvr_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_fence->signal_head);
+	pvr_fence->fctx = fctx;
+	seqno = pvr_fence_context_seqno_next(fctx);
+	/* Add the seqno to the fence name for easier debugging */
+	pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
+			name, seqno);
+
+	/* Reset cb to zero */
+	memset(&pvr_fence->cb, 0, sizeof(pvr_fence->cb));
+	pvr_fence->fence = &pvr_fence->base;
+
+	dma_fence_init(&pvr_fence->base, &pvr_fence_ops, &fctx->lock,
+		       fctx->fence_context, seqno);
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+	kref_get(&fctx->kref);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "created fence (%s)\n", name);
+	trace_pvr_fence_create(pvr_fence);
+
+	return pvr_fence;
+
+err_free_fence:
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+err_module_put:
+	module_put(THIS_MODULE);
+err_exit:
+	return NULL;
+}
+
+static const char *
+pvr_fence_foreign_get_driver_name(struct dma_fence *fence)
+{
+	return PVR_LDM_DRIVER_REGISTRATION_NAME;
+}
+
+static const char *
+pvr_fence_foreign_get_timeline_name(struct dma_fence *fence)
+{
+	return "foreign";
+}
+
+static
+void pvr_fence_foreign_fence_value_str(struct dma_fence *fence, char *str,
+				       int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	u32 sync_addr = 0;
+	u32 sync_value_next;
+
+	if (WARN_ON(!pvr_fence))
+		return;
+
+	sync_addr = SyncCheckpointGetFirmwareAddr(pvr_fence->sync_checkpoint);
+	sync_value_next = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+
+	/*
+	 * Include the fence flag bits from the foreign fence instead of our
+	 * shadow copy. This is done as the shadow fence flag bits aren't used.
+	 */
+	snprintf(str, size,
+		 "%llu: (%s%s) refs=%u fwaddr=%#08x cur=%#08x nxt=%#08x %s",
+		 (u64) fence->seqno,
+		 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 refcount_read(&fence->refcount.refcount),
+		 sync_addr,
+		 pvr_fence_sync_value(pvr_fence),
+		 sync_value_next,
+		 pvr_fence->name);
+}
+
+static
+void pvr_fence_foreign_timeline_value_str(struct dma_fence *fence, char *str,
+					  int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		pvr_context_value_str(pvr_fence->fctx, str, size);
+}
+
+static bool
+pvr_fence_foreign_enable_signaling(struct dma_fence *fence)
+{
+	WARN_ON("cannot enable signalling on foreign fence");
+	return false;
+}
+
+static signed long
+pvr_fence_foreign_wait(struct dma_fence *fence, bool intr, signed long timeout)
+{
+	WARN_ON("cannot wait on foreign fence");
+	return 0;
+}
+
+static void
+pvr_fence_foreign_release(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (pvr_fence) {
+		struct pvr_fence_context *fctx = pvr_fence->fctx;
+		struct dma_fence *foreign_fence = pvr_fence->fence;
+
+		PVR_FENCE_TRACE(&pvr_fence->base,
+				"released fence for foreign fence %llu#%d (%s)\n",
+				(u64) pvr_fence->fence->context,
+				(int)pvr_fence->fence->seqno, pvr_fence->name);
+		trace_pvr_fence_foreign_release(pvr_fence);
+
+		spin_lock_irqsave(&fctx->list_lock, flags);
+		list_move(&pvr_fence->fence_head,
+			  &fctx->deferred_free_list);
+		spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+		dma_fence_put(foreign_fence);
+
+		kref_put(&fctx->kref,
+			 pvr_fence_context_destroy_kref);
+	}
+}
+
+const struct dma_fence_ops pvr_fence_foreign_ops = {
+	.get_driver_name = pvr_fence_foreign_get_driver_name,
+	.get_timeline_name = pvr_fence_foreign_get_timeline_name,
+	.fence_value_str = pvr_fence_foreign_fence_value_str,
+	.timeline_value_str = pvr_fence_foreign_timeline_value_str,
+	.enable_signaling = pvr_fence_foreign_enable_signaling,
+	.wait = pvr_fence_foreign_wait,
+	.release = pvr_fence_foreign_release,
+};
+
+static void
+pvr_fence_foreign_signal_sync(struct dma_fence *fence, struct dma_fence_cb *cb)
+{
+	struct pvr_fence *pvr_fence = container_of(cb, struct pvr_fence, cb);
+	struct pvr_fence_context *fctx = pvr_fence->fctx;
+
+	/* Callback registered by dma_fence_add_callback can be called from an atomic ctx */
+	pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_CTX_ATOMIC);
+
+	trace_pvr_fence_foreign_signal(pvr_fence);
+
+	queue_work(fctx->fence_wq, &fctx->check_status_work);
+
+	PVR_FENCE_TRACE(&pvr_fence->base,
+			"foreign fence %llu#%d signalled (%s)\n",
+			(u64) pvr_fence->fence->context,
+			(int)pvr_fence->fence->seqno, pvr_fence->name);
+
+	/* Drop the reference on the base fence */
+	dma_fence_put(&pvr_fence->base);
+}
+
+/**
+ * pvr_fence_create_from_fence - creates a PVR fence from a fence
+ * @fctx: PVR fence context on which the PVR fence should be created
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @fence: fence from which the PVR fence should be created
+ * @fence_fd: fd for the sync file to which the fence belongs. If it doesn't
+ *            belong to a sync file then PVRSRV_NO_FENCE should be given
+ *            instead.
+ * @name: PVR fence name (used for debugging)
+ *
+ * Creates a PVR fence from an existing fence. If the fence is a foreign fence,
+ * i.e. one that doesn't originate from a PVR fence context, then a new PVR
+ * fence will be created using the specified sync_checkpoint_context.
+ * Otherwise, a reference will be taken on the underlying fence and the PVR
+ * fence will be returned.
+ *
+ * Once the fence is finished with, pvr_fence_destroy should be called.
+ *
+ * Returns NULL if a PVR fence cannot be created.
+ */
+
+struct pvr_fence *
+pvr_fence_create_from_fence(struct pvr_fence_context *fctx,
+			    struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+			    struct dma_fence *fence,
+			    PVRSRV_FENCE fence_fd,
+			    const char *name)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned int seqno;
+	unsigned long flags;
+	PVRSRV_ERROR srv_err;
+	int err;
+	bool mirror_other_dev_fence = false;
+	char tempString[40] = {0};
+
+	PVR_FENCE_TRACE(fence, "%s: fence @%p, pvr_fence = %p fctx = %p, dev %p\n",
+			__func__, fence, pvr_fence, fctx, (fctx ? fctx->dev_cookie : NULL));
+
+	if (pvr_fence) {
+		if ((SyncCheckpointCommonDeviceIDs(sync_checkpoint_ctx,
+						pvr_fence->fctx->dev_cookie)) ||
+		    (fctx->dev_cookie == pvr_fence->fctx->dev_cookie)) {
+			if (WARN_ON(fence->ops == &pvr_fence_foreign_ops))
+				return NULL;
+			dma_fence_get(fence);
+
+			PVR_FENCE_TRACE(fence, "created fence from PVR fence (%s)\n",
+					name);
+			return pvr_fence;
+		} else {
+			snprintf(tempString, sizeof(tempString), "Mirror(FWAddr0x%x)",
+				 SyncCheckpointGetFirmwareAddr(pvr_fence->sync_checkpoint));
+			mirror_other_dev_fence = true;
+
+			PVR_FENCE_TRACE(fence, "MIRRORED FENCE '%s'", tempString);
+		}
+	}
+
+	if (!try_module_get(THIS_MODULE))
+		goto err_exit;
+
+	/* Note: As kmem_cache is used to allocate pvr_fence objects,
+	 * make sure that all members of pvr_fence struct are initialized
+	 * here
+	 */
+	pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
+	if (!pvr_fence) {
+		pr_err("%s: kmem_cache_alloc() failed",
+			   __func__);
+		goto err_module_put;
+	}
+
+	srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
+					  (mirror_other_dev_fence ?
+					  SYNC_CHECKPOINT_MIRRORED_CHECKPOINT :
+					  SYNC_CHECKPOINT_FOREIGN_CHECKPOINT),
+					  fence_fd,
+					  mirror_other_dev_fence ? tempString : name,
+					  &pvr_fence->sync_checkpoint);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: SyncCheckpointAlloc() failed (srv_err=%d)",
+			   __func__, srv_err);
+		goto err_free_pvr_fence;
+	}
+
+	INIT_LIST_HEAD(&pvr_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_fence->signal_head);
+	pvr_fence->fctx = fctx;
+	pvr_fence->fence = dma_fence_get(fence);
+	seqno = pvr_fence_context_seqno_next(fctx);
+	/* Add the seqno to the fence name for easier debugging */
+	pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
+			name, seqno);
+
+	/*
+	 * We use the base fence to refcount the PVR fence and to do the
+	 * necessary clean up once the refcount drops to 0.
+	 */
+	dma_fence_init(&pvr_fence->base, &pvr_fence_foreign_ops, &fctx->lock,
+		       fctx->fence_context, seqno);
+
+	/*
+	 * Take an extra reference on the base fence that gets dropped when the
+	 * foreign fence is signalled.
+	 */
+	dma_fence_get(&pvr_fence->base);
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	kref_get(&fctx->kref);
+
+	PVR_FENCE_TRACE(&pvr_fence->base,
+			"created fence from foreign fence %llu#%d (%s)\n",
+			(u64) pvr_fence->fence->context,
+			(int)pvr_fence->fence->seqno, name);
+
+	err = dma_fence_add_callback(fence, &pvr_fence->cb,
+				     pvr_fence_foreign_signal_sync);
+	if (err) {
+		if (err != -ENOENT) {
+			pr_err("%s: failed to add fence callback (err=%d)",
+			       __func__, err);
+			goto err_put_ref;
+		}
+
+
+		/*
+		 * The fence has already signalled so set the sync as signalled.
+		 * The "signalled" hwperf packet should be emitted because the
+		 * callback won't be called for already signalled fence hence,
+		 * PVRSRV_FENCE_FLAG_NONE flag.
+		 */
+		pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
+		PVR_FENCE_TRACE(&pvr_fence->base,
+				"foreign fence %llu#%d already signaled (%s)\n",
+				(u64) pvr_fence->fence->context,
+				(int)pvr_fence->fence->seqno,
+				name);
+		dma_fence_put(&pvr_fence->base);
+	}
+
+	trace_pvr_fence_foreign_create(pvr_fence);
+
+	return pvr_fence;
+
+err_put_ref:
+	kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_del(&pvr_fence->fence_head);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	SyncCheckpointFree(pvr_fence->sync_checkpoint);
+err_free_pvr_fence:
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+err_module_put:
+	module_put(THIS_MODULE);
+err_exit:
+	return NULL;
+}
+
+/**
+ * pvr_fence_destroy - destroys a PVR fence
+ * @pvr_fence: PVR fence to destroy
+ *
+ * Destroys a PVR fence. Upon return, the PVR fence may still exist if something
+ * else still references the underlying fence, e.g. a reservation object, or if
+ * software signalling has been enabled and the fence hasn't yet been signalled.
+ */
+void
+pvr_fence_destroy(struct pvr_fence *pvr_fence)
+{
+	PVR_FENCE_TRACE(&pvr_fence->base, "destroyed fence (%s)\n",
+			pvr_fence->name);
+
+	dma_fence_put(&pvr_fence->base);
+}
+
+/**
+ * pvr_fence_sw_signal - signals a PVR fence sync
+ * @pvr_fence: PVR fence to signal
+ *
+ * Sets the PVR fence sync value to signalled.
+ *
+ * Returns -EINVAL if the PVR fence represents a foreign fence.
+ */
+int
+pvr_fence_sw_signal(struct pvr_fence *pvr_fence)
+{
+	if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
+		return -EINVAL;
+
+	pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
+
+	queue_work(pvr_fence->fctx->fence_wq,
+		   &pvr_fence->fctx->check_status_work);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync signalled (%s)\n",
+			pvr_fence->name);
+
+	return 0;
+}
+
+/**
+ * pvr_fence_sw_error - errors the sync checkpoint backing a PVR fence
+ * @pvr_fence: PVR fence to error
+ *
+ * Sets the PVR fence sync checkpoint value to errored.
+ *
+ * Returns -EINVAL if the PVR fence represents a foreign fence.
+ */
+int
+pvr_fence_sw_error(struct pvr_fence *pvr_fence)
+{
+	if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
+		return -EINVAL;
+
+	SyncCheckpointError(pvr_fence->sync_checkpoint, PVRSRV_FENCE_FLAG_NONE);
+	PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync errored (%s)\n",
+			pvr_fence->name);
+
+	return 0;
+}
+
+int
+pvr_fence_get_checkpoints(struct pvr_fence **pvr_fences, u32 nr_fences,
+			  struct SYNC_CHECKPOINT_TAG **fence_checkpoints)
+{
+	struct SYNC_CHECKPOINT_TAG **next_fence_checkpoint = fence_checkpoints;
+	struct pvr_fence **next_pvr_fence = pvr_fences;
+	int fence_checkpoint_idx;
+
+	if (nr_fences > 0) {
+
+		for (fence_checkpoint_idx = 0; fence_checkpoint_idx < nr_fences;
+		     fence_checkpoint_idx++) {
+			struct pvr_fence *next_fence = *next_pvr_fence++;
+			*next_fence_checkpoint++ = next_fence->sync_checkpoint;
+			/* Take reference on sync checkpoint (will be dropped
+			 * later by kick code)
+			 */
+			SyncCheckpointTakeRef(next_fence->sync_checkpoint);
+		}
+	}
+
+	return 0;
+}
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_fence_get_checkpoint(struct pvr_fence *update_fence)
+{
+	return update_fence->sync_checkpoint;
+}
+
+/**
+ * pvr_fence_dump_info_on_stalled_ufos - displays debug
+ * information on a native fence associated with any of
+ * the ufos provided. This function will be called from
+ * pvr_sync_file.c if the driver determines any GPU work
+ * is stuck waiting for a sync checkpoint representing a
+ * foreign sync to be signalled.
+ * @fctx:    fence context
+ * @nr_ufos: number of ufos in vaddrs
+ * @vaddrs:  array of FW addresses of UFOs which the
+ *           driver is waiting on.
+ *
+ * Output debug information to kernel log on linux fences
+ * which would be responsible for signalling the sync
+ * checkpoints indicated by the ufo vaddresses.
+ *
+ * Returns the number of ufos in the array which were found
+ * to be associated with foreign syncs.
+ */
+u32 pvr_fence_dump_info_on_stalled_ufos(struct pvr_fence_context *fctx,
+					u32 nr_ufos, u32 *vaddrs)
+{
+	int our_ufo_ct = 0;
+	struct pvr_fence *pvr_fence;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	/* dump info on any ufos in our active list */
+	list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
+		u32 *this_ufo_vaddr = vaddrs;
+		int ufo_num;
+		DUMPDEBUG_PRINTF_FUNC *pfnDummy = NULL;
+
+		for (ufo_num = 0; ufo_num < nr_ufos; ufo_num++) {
+			struct SYNC_CHECKPOINT_TAG *checkpoint =
+				pvr_fence->sync_checkpoint;
+			const u32 fence_ufo_addr =
+				SyncCheckpointGetFirmwareAddr(checkpoint);
+
+			if (fence_ufo_addr != this_ufo_vaddr[ufo_num])
+				continue;
+
+			/* Dump sync info */
+			PVR_DUMPDEBUG_LOG(pfnDummy, NULL,
+					  "\tSyncID = %d, FWAddr = 0x%08x: TLID = %d (Foreign Fence - [%p] %s)",
+					  SyncCheckpointGetId(checkpoint),
+					  fence_ufo_addr,
+					  SyncCheckpointGetTimeline(checkpoint),
+					  pvr_fence->fence,
+					  pvr_fence->name);
+			our_ufo_ct++;
+		}
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	return our_ufo_ct;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.h
new file mode 100644
index 000000000000..4e2b17096669
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence.h
@@ -0,0 +1,234 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_FENCE_H__)
+#define __PVR_FENCE_H__
+
+#include "services_kernel_client.h"
+#include "pvr_linux_fence.h"
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+
+struct SYNC_CHECKPOINT_CONTEXT_TAG;
+struct SYNC_CHECKPOINT_TAG;
+
+/**
+ * pvr_fence_context - PVR fence context used to create and manage PVR fences
+ * @lock: protects the context and fences created on the context
+ * @name: fence context name (used for debugging)
+ * @dbg_request_handle: handle for callback used to dump debug data
+ * @fence_context: fence context with which to associate fences
+ * @fence_seqno: sequence number to use for the next fence
+ * @fence_wq: work queue for signalled fence work
+ * @check_status_work: work item used to inform services when a foreign fence
+ * has signalled
+ * @cmd_complete_handle: handle for callback used to signal fences when fence
+ * syncs are met
+ * @list_lock: protects the active and active foreign lists
+ * @signal_list: list of fences waiting to be signalled
+ * @fence_list: list of fences (used for debugging)
+ * @deferred_free_list: list of fences that we will free when we are no longer
+ * holding spinlocks.  The frees get implemented when an update fence is
+ * signalled or the context is freed.
+ */
+struct pvr_fence_context {
+	spinlock_t lock;
+	char name[32];
+	void *dbg_request_handle;
+	u64 fence_context;
+	atomic_t fence_seqno;
+
+	struct workqueue_struct *fence_wq;
+	struct work_struct check_status_work;
+
+	void *cmd_complete_handle;
+
+	spinlock_t list_lock;
+	struct list_head signal_list;
+	struct list_head fence_list;
+	struct list_head deferred_free_list;
+
+	struct kref kref;
+	struct work_struct destroy_work;
+	void *dev_cookie;
+};
+
+/**
+ * pvr_fence - PVR fence that represents both native and foreign fences
+ * @base: fence structure
+ * @fctx: fence context on which this fence was created
+ * @name: fence name (used for debugging)
+ * @fence: pointer to base fence structure or foreign fence
+ * @sync_checkpoint: services sync checkpoint used by hardware
+ * @fence_head: entry on the context fence and deferred free list
+ * @signal_head: entry on the context signal list
+ * @cb: foreign fence callback to set the sync to signalled
+ */
+struct pvr_fence {
+	struct dma_fence base;
+	struct pvr_fence_context *fctx;
+	char name[32];
+
+	struct dma_fence *fence;
+	struct SYNC_CHECKPOINT_TAG *sync_checkpoint;
+
+	struct list_head fence_head;
+	struct list_head signal_head;
+	struct dma_fence_cb cb;
+	struct rcu_head rcu;
+};
+
+extern const struct dma_fence_ops pvr_fence_ops;
+extern const struct dma_fence_ops pvr_fence_foreign_ops;
+
+static inline bool is_our_fence(struct pvr_fence_context *fctx,
+				struct dma_fence *fence)
+{
+	return (fence->context == fctx->fence_context);
+}
+
+static inline bool is_pvr_fence(struct dma_fence *fence)
+{
+	return ((fence->ops == &pvr_fence_ops) ||
+		(fence->ops == &pvr_fence_foreign_ops));
+}
+
+static inline struct pvr_fence *to_pvr_fence(struct dma_fence *fence)
+{
+	if (is_pvr_fence(fence))
+		return container_of(fence, struct pvr_fence, base);
+
+	return NULL;
+}
+
+PVRSRV_ERROR pvr_fence_context_register_dbg(void *dbg_request_handle,
+				void *dev,
+				struct pvr_fence_context *fctx);
+struct pvr_fence_context *
+pvr_fence_foreign_context_create(struct workqueue_struct *fence_status_wq,
+		const char *name);
+struct pvr_fence_context *
+pvr_fence_context_create(void *dev_cookie,
+			 struct workqueue_struct *fence_status_wq,
+			 const char *name);
+void pvr_fence_context_destroy(struct pvr_fence_context *fctx);
+void pvr_context_value_str(struct pvr_fence_context *fctx, char *str, int size);
+
+struct pvr_fence *
+pvr_fence_create(struct pvr_fence_context *fctx,
+		 struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+		 int timeline_fd, const char *name);
+struct pvr_fence *
+pvr_fence_create_from_fence(struct pvr_fence_context *fctx,
+			    struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+			    struct dma_fence *fence,
+			    PVRSRV_FENCE fence_fd,
+			    const char *name);
+void pvr_fence_destroy(struct pvr_fence *pvr_fence);
+int pvr_fence_sw_signal(struct pvr_fence *pvr_fence);
+int pvr_fence_sw_error(struct pvr_fence *pvr_fence);
+
+int pvr_fence_get_checkpoints(struct pvr_fence **pvr_fences, u32 nr_fences,
+			      struct SYNC_CHECKPOINT_TAG **fence_checkpoints);
+struct SYNC_CHECKPOINT_TAG *
+pvr_fence_get_checkpoint(struct pvr_fence *update_fence);
+
+void pvr_fence_context_signal_fences_nohw(void *data);
+
+void pvr_fence_context_free_deferred_callback(void *data);
+
+u32 pvr_fence_dump_info_on_stalled_ufos(struct pvr_fence_context *fctx,
+					u32 nr_ufos,
+					u32 *vaddrs);
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+static inline void pvr_fence_cleanup(void)
+{
+	/*
+	 * Ensure all PVR fence contexts have been destroyed, by flushing
+	 * the context destruction workqueue.
+	 */
+	flush_workqueue(NativeSyncGetFenceCtxDestroyWq());
+}
+#endif
+
+#if defined(PVR_FENCE_DEBUG)
+#define PVR_FENCE_CTX_TRACE(c, fmt, ...)                                   \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_err("c %llu: (PVR) " fmt, (u64) __fctx->fence_context,  \
+		       ## __VA_ARGS__);                                    \
+	} while (0)
+#else
+#define PVR_FENCE_CTX_TRACE(c, fmt, ...)
+#endif
+
+#define PVR_FENCE_CTX_WARN(c, fmt, ...)                                    \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_warn("c %llu: (PVR) " fmt, (u64) __fctx->fence_context, \
+			## __VA_ARGS__);                                   \
+	} while (0)
+
+#define PVR_FENCE_CTX_ERR(c, fmt, ...)                                     \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_err("c %llu: (PVR) " fmt, (u64) __fctx->fence_context,  \
+		       ## __VA_ARGS__);                                    \
+	} while (0)
+
+#if defined(PVR_FENCE_DEBUG)
+#define PVR_FENCE_TRACE(f, fmt, ...)                                       \
+	DMA_FENCE_ERR(f, "(PVR) " fmt, ## __VA_ARGS__)
+#else
+#define PVR_FENCE_TRACE(f, fmt, ...)
+#endif
+
+#define PVR_FENCE_WARN(f, fmt, ...)                                        \
+	DMA_FENCE_WARN(f, "(PVR) " fmt, ## __VA_ARGS__)
+
+#define PVR_FENCE_ERR(f, fmt, ...)                                         \
+	DMA_FENCE_ERR(f, "(PVR) " fmt, ## __VA_ARGS__)
+
+#endif /* !defined(__PVR_FENCE_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence_trace.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence_trace.h
new file mode 100644
index 000000000000..95b2a0dc6c7c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_fence_trace.h
@@ -0,0 +1,225 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM pvr_fence
+
+#if !defined(_TRACE_PVR_FENCE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_PVR_FENCE_H
+
+#include <linux/tracepoint.h>
+
+struct pvr_fence;
+struct pvr_fence_context;
+
+DECLARE_EVENT_CLASS(pvr_fence_context,
+
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx),
+
+	TP_STRUCT__entry(
+		__string(name, fctx->name)
+		__array(char, val, 128)
+	),
+
+	TP_fast_assign(
+		__assign_str(name, fctx->name)
+		pvr_context_value_str(fctx, __entry->val,
+			sizeof(__entry->val));
+	),
+
+	TP_printk("name=%s val=%s",
+		  __get_str(name),
+		  __entry->val
+	)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_create,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_destroy,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_destroy_kref,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_signal_fences,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DECLARE_EVENT_CLASS(pvr_fence,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence),
+
+	TP_STRUCT__entry(
+		__string(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__string(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		__array(char, val, 128)
+		__field(u64, context)
+	),
+
+	TP_fast_assign(
+		__assign_str(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__assign_str(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		fence->base.ops->fence_value_str(&fence->base,
+			__entry->val, sizeof(__entry->val));
+		__entry->context = fence->base.context;
+	),
+
+	TP_printk("driver=%s timeline=%s ctx=%llu val=%s",
+		  __get_str(driver), __get_str(timeline),
+		  __entry->context, __entry->val
+	)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_create,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_release,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_enable_signaling,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_signal_fence,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DECLARE_EVENT_CLASS(pvr_fence_foreign,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence),
+
+	TP_STRUCT__entry(
+		__string(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__string(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		__array(char, val, 128)
+		__field(u64, context)
+		__string(foreign_driver,
+			fence->fence->ops->get_driver_name ?
+			fence->fence->ops->get_driver_name(fence->fence) :
+			"unknown")
+		__string(foreign_timeline,
+			fence->fence->ops->get_timeline_name ?
+			fence->fence->ops->get_timeline_name(fence->fence) :
+			"unknown")
+		__array(char, foreign_val, 128)
+		__field(u64, foreign_context)
+	),
+
+	TP_fast_assign(
+		__assign_str(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__assign_str(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		fence->base.ops->fence_value_str(&fence->base, __entry->val,
+			sizeof(__entry->val));
+		__entry->context = fence->base.context;
+		__assign_str(foreign_driver,
+			fence->fence->ops->get_driver_name ?
+			fence->fence->ops->get_driver_name(fence->fence) :
+			"unknown")
+		__assign_str(foreign_timeline,
+			fence->fence->ops->get_timeline_name ?
+			fence->fence->ops->get_timeline_name(fence->fence) :
+			"unknown")
+		fence->fence->ops->fence_value_str ?
+			fence->fence->ops->fence_value_str(
+				fence->fence, __entry->foreign_val,
+				sizeof(__entry->foreign_val)) :
+			(void) OSStringSafeCopy(__entry->foreign_val,
+				"unknown", sizeof(__entry->foreign_val));
+		__entry->foreign_context = fence->fence->context;
+	),
+
+	TP_printk("driver=%s timeline=%s ctx=%llu val=%s foreign: driver=%s timeline=%s ctx=%llu val=%s",
+		  __get_str(driver), __get_str(timeline), __entry->context,
+		  __entry->val, __get_str(foreign_driver),
+		  __get_str(foreign_timeline), __entry->foreign_context,
+		  __entry->foreign_val
+	)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_create,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_release,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_signal,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+#endif /* _TRACE_PVR_FENCE_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+
+/* This is needed because the name of this file doesn't match TRACE_SYSTEM. */
+#define TRACE_INCLUDE_FILE pvr_fence_trace
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_gputrace.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_gputrace.c
new file mode 100644
index 000000000000..bbfa3504495a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_gputrace.c
@@ -0,0 +1,1680 @@
+/*************************************************************************/ /*!
+@File           pvr_gputrace.c
+@Title          PVR GPU Trace module Linux implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/fs.h>
+#include <linux/trace_events.h>
+
+#include "pvrsrv_error.h"
+#include "pvrsrv_apphint.h"
+#include "pvr_debug.h"
+#include "ospvr_gputrace.h"
+#include "rgxhwperf.h"
+#include "rgxtimecorr.h"
+#include "device.h"
+#include "trace_events.h"
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "tlclient.h"
+#include "tlstream.h"
+#include "pvr_debug.h"
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#define CREATE_TRACE_POINTS
+#include "rogue_trace_events.h"
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#define CREATE_TRACE_POINTS
+#include "gpu_work.h"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#if defined(SUPPORT_RGX)
+
+#define TRACE_CLK     RGXTIMECORR_CLOCK_SCHED
+#define TRACE_CLK_STR "local\n"
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#define TRACE_FS_CLK "/sys/kernel/tracing/trace_clock"
+#else /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)*/
+#define TRACE_FS_CLK "/sys/kernel/debug/tracing/trace_clock"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+
+#endif /* defined(SUPPORT_RGX) */
+
+/******************************************************************************
+ Module internal implementation
+******************************************************************************/
+
+typedef enum {
+	PVR_GPUTRACE_SWITCH_TYPE_UNDEF = 0,
+
+	PVR_GPUTRACE_SWITCH_TYPE_BEGIN = 1,
+	PVR_GPUTRACE_SWITCH_TYPE_END = 2,
+	PVR_GPUTRACE_SWITCH_TYPE_SINGLE = 3
+} PVR_GPUTRACE_SWITCH_TYPE;
+
+typedef struct RGX_HWPERF_FTRACE_DATA {
+	/* This lock ensures the HWPerf TL stream reading resources are not destroyed
+	 * by one thread disabling it while another is reading from it. Keeps the
+	 * state and resource create/destroy atomic and consistent. */
+	POS_LOCK    hFTraceResourceLock;
+
+	IMG_HANDLE  hGPUTraceCmdCompleteHandle;
+	IMG_HANDLE  hGPUFTraceTLStream;
+	IMG_UINT64  ui64LastSampledTimeCorrOSTimeStamp;
+	IMG_UINT32  ui32FTraceLastOrdinal;
+	IMG_BOOL    bTrackOrdinals;
+} RGX_HWPERF_FTRACE_DATA;
+
+/* This lock ensures state change of GPU_TRACING on/off is done atomically */
+static POS_LOCK ghGPUTraceStateLock;
+static IMG_BOOL gbFTraceGPUEventsEnabled = PVRSRV_APPHINT_ENABLEFTRACEGPU;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+/* This is the FTrace clock source prior to driver initialisation */
+static IMG_CHAR gszLastClockSource[32] = {0};
+#endif
+
+/* This lock ensures that the reference counting operation on the FTrace UFO
+ * events and enable/disable operation on firmware event are performed as
+ * one atomic operation. This should ensure that there are no race conditions
+ * between reference counting and firmware event state change.
+ * See below comment for guiUfoEventRef.
+ */
+static POS_LOCK ghLockFTraceEventLock;
+
+/* Multiple FTrace UFO events are reflected in the firmware as only one event. When
+ * we enable FTrace UFO event we want to also at the same time enable it in
+ * the firmware. Since there is a multiple-to-one relation between those events
+ * we count how many FTrace UFO events is enabled. If at least one event is
+ * enabled we enabled the firmware event. When all FTrace UFO events are disabled
+ * we disable firmware event. */
+static IMG_UINT guiUfoEventRef;
+
+/******************************************************************************
+ Module In-bound API
+******************************************************************************/
+
+static PVRSRV_ERROR _GpuTraceDisable(
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+	IMG_BOOL bDeInit);
+
+static void _GpuTraceCmdCompleteNotify(PVRSRV_CMDCOMP_HANDLE);
+static void _GpuTraceProcessPackets(PVRSRV_RGXDEV_INFO *psDevInfo, void *pBuffer,
+                                    IMG_UINT32 ui32ReadLen);
+
+static void _FTrace_FWOnReaderOpenCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_FTRACE] = IMG_FALSE;
+}
+
+static void _FTrace_FWOnReaderCloseCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_FTRACE] = IMG_TRUE;
+}
+
+/* Currently supported by default */
+#if defined(SUPPORT_TL_PRODUCER_CALLBACK)
+static PVRSRV_ERROR GPUTraceTLCB(IMG_HANDLE hStream,
+                                  IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser)
+{
+	/* IN DEV: Not required as the streams goal is to be a copy of HWPerf */
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(hStream);
+	PVR_UNREFERENCED_PARAMETER(ui32ReqOp);
+	PVR_UNREFERENCED_PARAMETER(ui32Resp);
+	PVR_UNREFERENCED_PARAMETER(pvUser);
+
+	return eError;
+}
+#endif /* defined(SUPPORT_TL_PRODUCER_CALLBACK) */
+
+#if defined(SUPPORT_RGX)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+/* Configure the FTrace clock source to use the DDK apphint clock source */
+static void PVRGpuTraceInitFTraceClockSource(void)
+{
+	int ret, i, j;
+	bool bFound = false;
+	loff_t pos = 0;
+	char str[64];
+
+	/*  Just force the value to be what the DDK says it is
+	    Note for filp_open, the mode is only used for O_CREAT
+		Hence its value doesn't matter in this context
+	 */
+	struct file *filep = filp_open(TRACE_FS_CLK, O_RDWR, 0);
+	PVR_LOG_RETURN_VOID_IF_FALSE(!IS_ERR(filep), "TraceFS not found");
+
+	PVR_LOG_VA(PVR_DBG_MESSAGE,
+	        "Writing %s to %s to enable parallel HWPerf and FTrace support",
+	        TRACE_CLK_STR, TRACE_FS_CLK);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_read(filep, str, sizeof(str)-1, &pos);
+#else
+	ret = kernel_read(filep, pos, str, sizeof(str)-1);
+#endif
+	PVR_LOG_RETURN_VOID_IF_FALSE((ret > 0), "TraceFS Read failed");
+	str[ret] = 0;
+	pos = 0;
+
+	/* Determine clock value. trace_clock value is stored like [<clock_string>]
+	   File example: [local] global counter mono mono_raw x86-tsc
+	*/
+	for (i = 0, j = 0; i < sizeof(str) && j < sizeof(gszLastClockSource); i++)
+	{
+		if (str[i] == ']')
+		{
+			break;
+		}
+		else if (str[i] == '[')
+		{
+			bFound = true;
+		}
+		else if (bFound)
+		{
+			gszLastClockSource[j] = str[i];
+			j++;
+		}
+	}
+	PVR_LOG_VA(PVR_DBG_MESSAGE, "Got %s from FTraceFS", gszLastClockSource);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_write(filep, TRACE_CLK_STR, sizeof(TRACE_CLK_STR), &pos);
+#else
+	ret = kernel_write(filep, TRACE_CLK_STR, sizeof(TRACE_CLK_STR), pos);
+#endif
+	PVR_LOG_IF_FALSE((ret > 0), "Setting FTrace clock source failed");
+
+	filp_close(filep, NULL);
+}
+
+/* Reset the FTrace clock source to the original clock source */
+static void PVRGpuTraceDeinitFTraceClockSource(void)
+{
+	int ret;
+	loff_t pos = 0;
+
+	/* Return the original value of the file */
+	struct file *filep = filp_open(TRACE_FS_CLK, O_WRONLY, 0);
+	PVR_LOG_RETURN_VOID_IF_FALSE(!IS_ERR(filep), "TraceFS not found");
+
+	/* FTraceFS write will ignore any writes to it that don't match a clock source */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_write(filep, gszLastClockSource, sizeof(gszLastClockSource), &pos);
+#else
+	ret = kernel_write(filep, gszLastClockSource, sizeof(gszLastClockSource), pos);
+#endif
+	PVR_LOG_IF_FALSE((ret >= 0), "Setting FTrace clock source failed");
+
+	filp_close(filep, NULL);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+#endif /* defined(SUPPORT_RGX) */
+
+PVRSRV_ERROR PVRGpuTraceSupportInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	if (ghLockFTraceEventLock != NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FTrace Support is already initialized"));
+		return PVRSRV_OK;
+	}
+
+	/* common module params initialization */
+	eError = OSLockCreate(&ghLockFTraceEventLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	eError = OSLockCreate(&ghGPUTraceStateLock);
+	PVR_LOG_RETURN_IF_ERROR (eError, "OSLockCreate");
+
+#if defined(SUPPORT_RGX)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+	PVRGpuTraceInitFTraceClockSource();
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+#endif /* defined(SUPPORT_RGX) */
+
+	return PVRSRV_OK;
+}
+
+void PVRGpuTraceSupportDeInit(void)
+{
+#if defined(SUPPORT_RGX)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+	PVRGpuTraceDeinitFTraceClockSource();
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+#endif /* defined(SUPPORT_RGX) */
+
+	if (ghGPUTraceStateLock)
+	{
+		OSLockDestroy(ghGPUTraceStateLock);
+	}
+
+	if (ghLockFTraceEventLock)
+	{
+		OSLockDestroy(ghLockFTraceEventLock);
+		ghLockFTraceEventLock = NULL;
+	}
+}
+
+/* Called from RGXHWPerfInitOnDemandL2Stream() which is alway called from
+ * a critical section protected by hHWPerfLock. */
+PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszFTraceStreamName[sizeof(PVRSRV_TL_FTRACE_RGX_FW_STREAM) + 4];
+	/* + 4 is used to allow names up to "ftrace_999", which is enough */
+
+	IMG_HANDLE hStream = NULL;
+
+	/* form the FTrace stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszFTraceStreamName, sizeof(pszFTraceStreamName), "%s%d",
+				   PVRSRV_TL_FTRACE_RGX_FW_STREAM,
+				   psDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to form FTrace stream name for device %d",
+				__func__,
+				psDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = TLStreamCreate(&hStream,
+							pszFTraceStreamName,
+							psDevInfo->ui32RGXL2HWPerfBufSize,
+							TL_OPMODE_DROP_NEWER | TL_FLAG_NO_SIGNAL_ON_COMMIT,
+							_FTrace_FWOnReaderOpenCB, psDevInfo,
+							_FTrace_FWOnReaderCloseCB, psDevInfo,
+#if !defined(SUPPORT_TL_PRODUCER_CALLBACK)
+							NULL, NULL
+#else
+							/* Not enabled by default */
+							GPUTraceTLCB, psDevInfo
+#endif
+							);
+	PVR_RETURN_IF_ERROR(eError);
+
+	psDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_FTRACE] = hStream;
+	psDevInfo->uiHWPerfStreamCount++;
+	PVR_ASSERT(psDevInfo->uiHWPerfStreamCount <= RGX_HWPERF_L2_STREAM_LAST);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRGpuTraceInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	RGX_HWPERF_FTRACE_DATA *psData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psData = OSAllocZMem(sizeof(RGX_HWPERF_FTRACE_DATA));
+	psDevInfo->pvGpuFtraceData = psData;
+	PVR_LOG_GOTO_IF_NOMEM(psData, eError, e0);
+
+	/* We initialise it only once because we want to track if any
+	 * packets were dropped. */
+	psData->ui32FTraceLastOrdinal = IMG_UINT32_MAX - 1;
+
+	eError = OSLockCreate(&psData->hFTraceResourceLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	return PVRSRV_OK;
+
+e0:
+	PVRGpuTraceDeInitDevice(psDeviceNode);
+	return eError;
+}
+
+void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_HWPERF_FTRACE_DATA *psData = psDevInfo->pvGpuFtraceData;
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+	if (psData)
+	{
+		/* first disable the tracing, to free up TL resources */
+		if (psData->hFTraceResourceLock)
+		{
+			OSLockAcquire(psData->hFTraceResourceLock);
+			_GpuTraceDisable(psDeviceNode->pvDevice, IMG_TRUE);
+			OSLockRelease(psData->hFTraceResourceLock);
+
+			/* now free all the FTrace resources */
+			OSLockDestroy(psData->hFTraceResourceLock);
+		}
+		OSFreeMem(psData);
+		psDevInfo->pvGpuFtraceData = NULL;
+	}
+}
+
+static INLINE IMG_BOOL PVRGpuTraceIsEnabled(void)
+{
+	return gbFTraceGPUEventsEnabled;
+}
+
+void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (PVRGpuTraceIsEnabled())
+	{
+		IMG_BOOL bEnable = IMG_FALSE;
+
+		PVRSRV_ERROR eError = PVRGpuTraceSetEnabled(psDeviceNode, IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to initialise GPU event tracing"
+					" (%s)", PVRSRVGetErrorString(eError)));
+		}
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		/* below functions will enable FTrace events which in turn will
+		 * execute HWPerf callbacks that set appropriate filter values
+		 * note: unfortunately the functions don't allow to pass private
+		 *       data so they enable events for all of the devices
+		 *       at once, which means that this can happen more than once
+		 *       if there is more than one device */
+
+		/* single events can be enabled by calling trace_set_clr_event()
+		 * with the event name, e.g.:
+		 * trace_set_clr_event("rogue", "rogue_ufo_update", 1) */
+		if (trace_set_clr_event("rogue", NULL, 1) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to enable \"rogue\" event"
+			        " group"));
+		}
+		else
+		{
+			PVR_LOG(("FTrace events from \"rogue\" group enabled"));
+			bEnable = IMG_TRUE;
+		}
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+		if (trace_set_clr_event("power", "gpu_work_period", 1) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to enable \"gpu_work_period\" event"));
+		}
+		else
+		{
+			PVR_LOG(("FTrace event from \"gpu_work_period\" enabled"));
+			bEnable = IMG_TRUE;
+		}
+#endif
+
+		if (bEnable)
+		{
+			/* this enables FTrace globally (if not enabled nothing will appear
+			 * in the FTrace buffer) */
+			tracing_on();
+		}
+	}
+}
+
+/* Caller must now hold hFTraceResourceLock before calling this method.
+ */
+static PVRSRV_ERROR _GpuTraceEnable(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+	PVRSRV_DEVICE_NODE *psRgxDevNode = psRgxDevInfo->psDeviceNode;
+	IMG_CHAR pszFTraceStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 4];
+	IMG_UINT64 uiFilter;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	PVR_ASSERT(OSLockIsLocked(psFtraceData->hFTraceResourceLock));
+
+	/* return if already enabled */
+	if (psFtraceData->hGPUFTraceTLStream != NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	uiFilter =
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	    RGX_HWPERF_EVENT_MASK_HW_KICKFINISH | RGX_HWPERF_EVENT_MASK_FW_SED | RGX_HWPERF_EVENT_MASK_FW_UFO |
+#endif
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD) |
+#endif
+	    0;
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	/* Signal FW to enable event generation */
+	if (psRgxDevInfo->bFirmwareInitialised)
+	{
+		eError = PVRSRVRGXCtrlHWPerfFW(psRgxDevNode,
+		                               RGX_HWPERF_L2_STREAM_FTRACE,
+		                               uiFilter, HWPERF_FILTER_OPERATION_SET);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfFW", err_out);
+	}
+	else
+#endif
+	{
+		/* only set filter and exit */
+		(void) RGXHWPerfFwSetEventFilter(psRgxDevInfo, RGX_HWPERF_L2_STREAM_FTRACE, uiFilter);
+
+		return PVRSRV_OK;
+	}
+
+	/* form the FTrace stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszFTraceStreamName, sizeof(pszFTraceStreamName), "%s%d",
+					PVRSRV_TL_FTRACE_RGX_FW_STREAM, psRgxDevNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to form FTrace stream name for device %d",
+		         __func__,
+		         psRgxDevNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* In case there was any data copied from the L1 buffer to the FTrace L2
+	 * buffer in the narrow window between disabling the FTrace and MISR
+	 * running, drop this data to make the ordinal tracking correct. */
+	TLStreamReset(psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_FTRACE]);
+
+	/* Open the TL Stream for FTrace data consumption */
+	eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+								pszFTraceStreamName,
+								PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING,
+								&psFtraceData->hGPUFTraceTLStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLClientOpenStream", err_out);
+
+#if defined(SUPPORT_RGX)
+	if (RGXTimeCorrGetClockSource(psRgxDevNode) != TRACE_CLK)
+	{
+		/* Set clock source for timer correlation data to hwperf clock */
+		eError = RGXTimeCorrSetClockSource(psRgxDevNode, TRACE_CLK);
+		PVR_LOG_IF_ERROR(eError, "RGXTimeCorrSetClockSource");
+	}
+#endif
+
+	/* Reset the ordinal tracking flag. We should skip checking for gaps in
+	 * ordinal on the first run after FTrace is enabled in case another HWPerf
+	 * consumer was connected while FTrace was disabled. */
+	psFtraceData->bTrackOrdinals = IMG_FALSE;
+
+	/* Reset the OS timestamp coming from the timer correlation data
+	 * associated with the latest HWPerf event we processed.
+	 */
+	psFtraceData->ui64LastSampledTimeCorrOSTimeStamp = 0;
+
+	/* Register a notifier to collect HWPerf data whenever the HW completes
+	 * an operation.
+	 */
+	eError = PVRSRVRegisterCmdCompleteNotify(
+		&psFtraceData->hGPUTraceCmdCompleteHandle,
+		&_GpuTraceCmdCompleteNotify,
+		psRgxDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterCmdCompleteNotify", err_close_stream);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+err_close_stream:
+	TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+						psFtraceData->hGPUFTraceTLStream);
+	psFtraceData->hGPUFTraceTLStream = NULL;
+	goto err_out;
+}
+
+/* Caller must now hold hFTraceResourceLock before calling this method.
+ */
+static PVRSRV_ERROR _GpuTraceDisable(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_BOOL bDeInit)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_NODE *psRgxDevNode = psRgxDevInfo->psDeviceNode;
+#endif
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	PVR_ASSERT(OSLockIsLocked(psFtraceData->hFTraceResourceLock));
+
+	/* if FW is not yet initialised, just set filter and exit */
+	if (!psRgxDevInfo->bFirmwareInitialised)
+	{
+		(void) RGXHWPerfFwSetEventFilter(psRgxDevInfo, RGX_HWPERF_L2_STREAM_FTRACE,
+		                                 RGX_HWPERF_EVENT_MASK_NONE);
+		return PVRSRV_OK;
+	}
+
+	if (psFtraceData->hGPUFTraceTLStream == NULL)
+	{
+		/* Tracing already disabled, just return */
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_RGX)
+	if (!bDeInit)
+	{
+		eError = PVRSRVRGXCtrlHWPerfFW(psRgxDevNode,
+		                               RGX_HWPERF_L2_STREAM_FTRACE,
+		                               RGX_HWPERF_EVENT_MASK_NONE,
+		                               HWPERF_FILTER_OPERATION_SET);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfFW");
+	}
+#endif
+
+	if (psFtraceData->hGPUTraceCmdCompleteHandle)
+	{
+		/* Tracing is being turned off. Unregister the notifier. */
+		eError = PVRSRVUnregisterCmdCompleteNotify(
+				psFtraceData->hGPUTraceCmdCompleteHandle);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVUnregisterCmdCompleteNotify");
+		psFtraceData->hGPUTraceCmdCompleteHandle = NULL;
+	}
+
+	/* Let close stream perform the release data on the outstanding acquired
+	 * data */
+	eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+	                             psFtraceData->hGPUFTraceTLStream);
+	PVR_LOG_IF_ERROR(eError, "TLClientCloseStream");
+
+	psFtraceData->hGPUFTraceTLStream = NULL;
+
+#if defined(SUPPORT_RGX)
+	if (psRgxDevInfo->ui32LastClockSource != TRACE_CLK)
+	{
+		RGXTimeCorrSetClockSource(psRgxDevNode, psRgxDevInfo->ui32LastClockSource);
+	}
+#endif
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                        IMG_BOOL bNewValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	/* About to create/destroy FTrace resources, lock critical section
+	 * to avoid HWPerf MISR thread contention.
+	 */
+	OSLockAcquire(psFtraceData->hFTraceResourceLock);
+
+	eError = (bNewValue ? _GpuTraceEnable(psRgxDevInfo)
+					   : _GpuTraceDisable(psRgxDevInfo, IMG_FALSE));
+
+	OSLockRelease(psFtraceData->hFTraceResourceLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabledForAllDevices(IMG_BOOL bNewValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	/* enable/disable GPU trace on all devices */
+	while (psDeviceNode)
+	{
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			eError = _GpuTraceSetEnabled(psRgxDevInfo, bNewValue);
+			if (eError != PVRSRV_OK)
+				break;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR PVRGpuTraceSetEnabled(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_BOOL bNewValue)
+{
+	return _GpuTraceSetEnabled(psDeviceNode->pvDevice, bNewValue);
+}
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+/* ----- HWPerf to FTrace packet processing and events injection ------------ */
+
+static const IMG_CHAR *_HWPerfKickTypeToStr(RGX_HWPERF_KICK_TYPE eKickType)
+{
+	static const IMG_CHAR *aszKickType[RGX_HWPERF_KICK_TYPE2_LAST+1] = {
+		"TA3D", /* Deprecated */
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+		/* Volcanic deprecated kick types */
+		"CDM", "RS", "SHG", "TQTDM", "SYNC", "TA", "3D", "LAST",
+
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+#else
+		/* Rogue deprecated kick types */
+		"TQ2D", "TQ3D", "CDM", "RS", "VRDM", "TQTDM", "SYNC", "TA", "3D", "LAST",
+
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+		"<UNKNOWN>",
+#endif
+		"TQ2D", "TQ3D", "TQTDM", "CDM", "GEOM", "3D", "SYNC", "RS", "LAST"
+	};
+
+	/* cast in case of negative value */
+	if (((IMG_UINT32) eKickType) >= RGX_HWPERF_KICK_TYPE2_LAST)
+	{
+		return "<UNKNOWN>";
+	}
+
+	return aszKickType[eKickType];
+}
+void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExtJobRef,
+		IMG_UINT32 ui32IntJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType)
+{
+	const IMG_CHAR *pszKickType = _HWPerfKickTypeToStr(eKickType);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "PVRGpuTraceEnqueueEvent(%s): contextId %u, "
+	        "jobId %u", pszKickType, ui32FirmwareCtx, ui32IntJobRef));
+
+	if (PVRGpuTraceIsEnabled())
+	{
+		trace_rogue_job_enqueue(psDevNode->sDevId.ui32InternalID, ui32FirmwareCtx,
+		                        ui32IntJobRef, ui32ExtJobRef, pszKickType);
+	}
+}
+
+static void _GpuTraceWorkSwitch(
+		IMG_UINT64 ui64HWTimestampInOSTime,
+		IMG_UINT32 ui32GpuId,
+		IMG_UINT32 ui32CtxId,
+		IMG_UINT32 ui32CtxPriority,
+		IMG_UINT32 ui32ExtJobRef,
+		IMG_UINT32 ui32IntJobRef,
+		const IMG_CHAR* pszWorkType,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	PVR_ASSERT(pszWorkType);
+	trace_rogue_sched_switch(pszWorkType, eSwType, ui64HWTimestampInOSTime,
+	                         ui32GpuId, ui32CtxId, 2-ui32CtxPriority, ui32IntJobRef,
+	                         ui32ExtJobRef);
+}
+
+static void _GpuTraceUfo(
+		IMG_UINT64 ui64OSTimestamp,
+		const RGX_HWPERF_UFO_EV eEvType,
+		const IMG_UINT32 ui32GpuId,
+		const IMG_UINT32 ui32CtxId,
+		const IMG_UINT32 ui32ExtJobRef,
+		const IMG_UINT32 ui32IntJobRef,
+		const IMG_UINT32 ui32UFOCount,
+		const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	switch (eEvType) {
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			trace_rogue_ufo_updates(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, ui32UFOCount, puData);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+			trace_rogue_ufo_checks_success(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_FALSE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			trace_rogue_ufo_checks_success(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_TRUE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+			trace_rogue_ufo_checks_fail(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_FALSE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			trace_rogue_ufo_checks_fail(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_TRUE, ui32UFOCount,
+					puData);
+			break;
+		default:
+			break;
+	}
+}
+
+static void _GpuTraceFirmware(
+		IMG_UINT64 ui64HWTimestampInOSTime,
+		IMG_UINT32 ui32GpuId,
+		const IMG_CHAR* pszWorkType,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	trace_rogue_firmware_activity(ui64HWTimestampInOSTime, ui32GpuId, pszWorkType, eSwType);
+}
+
+static void _GpuTraceEventsLost(
+		const RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT32 ui32GpuId,
+		const IMG_UINT32 ui32LastOrdinal,
+		const IMG_UINT32 ui32CurrOrdinal)
+{
+	trace_rogue_events_lost(eStreamId, ui32GpuId, ui32LastOrdinal, ui32CurrOrdinal);
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+/* Calculate the OS timestamp given an RGX timestamp in the HWPerf event. */
+static uint64_t CalculateEventTimestamp(
+	PVRSRV_RGXDEV_INFO *psDevInfo,
+	uint32_t ui32TimeCorrIndex,
+	uint64_t ui64EventTimestamp)
+{
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData = psDevInfo->pvGpuFtraceData;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	uint64_t ui64CRTimeStamp;
+	uint64_t ui64OSTimeStamp;
+	uint64_t ui64CRDeltaToOSDeltaKNs;
+	uint64_t ui64EventOSTimestamp, deltaRgxTimer, delta_ns;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32TimeCorrIndex], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32TimeCorrIndex];
+	ui64CRTimeStamp = psTimeCorr->ui64CRTimeStamp;
+	/* This is configurable using the AppHint 'SecondaryTimerCorrOSClockSource'
+	 * and can be: sched, mono, mono_raw. */
+	ui64OSTimeStamp = psTimeCorr->ui64OSTimeStamp;
+	ui64CRDeltaToOSDeltaKNs = psTimeCorr->ui64CRDeltaToOSDeltaKNs;
+
+	if (psFtraceData->ui64LastSampledTimeCorrOSTimeStamp > ui64OSTimeStamp)
+	{
+		/* The previous packet had a time reference (time correlation data) more
+		 * recent than the one in the current packet, it means the timer
+		 * correlation array wrapped too quickly (buffer too small) and in the
+		 * previous call to _GpuTraceUfoEvent we read one of the
+		 * newest timer correlations rather than one of the oldest ones.
+		 */
+		PVR_DPF((PVR_DBG_ERROR, "%s: The timestamps computed so far could be "
+				 "wrong! The time correlation array size should be increased "
+				 "to avoid this.", __func__));
+	}
+
+	psFtraceData->ui64LastSampledTimeCorrOSTimeStamp = ui64OSTimeStamp;
+
+	/* RGX CR timer ticks delta */
+	deltaRgxTimer = ui64EventTimestamp - ui64CRTimeStamp;
+	/* RGX time delta in nanoseconds */
+	delta_ns = RGXFWIF_GET_DELTA_OSTIME_NS(deltaRgxTimer, ui64CRDeltaToOSDeltaKNs);
+	/* Calculate OS time of HWPerf event */
+	ui64EventOSTimestamp = ui64OSTimeStamp + delta_ns;
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: psCurrentDvfs RGX %llu, OS %llu, DVFSCLK %u",
+			 __func__, ui64CRTimeStamp, ui64OSTimeStamp,
+			 psTimeCorr->ui32CoreClockSpeed));
+
+	return ui64EventOSTimestamp;
+}
+
+static void _GpuTraceSwitchEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt, const IMG_CHAR* pszWorkName,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	IMG_UINT64 ui64Timestamp;
+	RGX_HWPERF_HW_DATA* psHWPerfPktData;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psHWPerfPkt);
+	PVR_ASSERT(pszWorkName);
+
+	psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceSwitchEvent: %s ui32ExtJobRef=%d, ui32IntJobRef=%d, eSwType=%d",
+			 pszWorkName != NULL ? pszWorkName : "?", psHWPerfPktData->ui32DMContext,
+			 psHWPerfPktData->ui32IntJobRef, eSwType));
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	_GpuTraceWorkSwitch(ui64Timestamp,
+	                    psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+	                    psHWPerfPktData->ui32DMContext,
+	                    psHWPerfPktData->ui32CtxPriority,
+	                    psHWPerfPktData->ui32ExtJobRef,
+	                    psHWPerfPktData->ui32IntJobRef,
+	                    pszWorkName,
+	                    eSwType);
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+	PVR_DPF_RETURN;
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+static void _GpuTraceUfoEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt)
+{
+	IMG_UINT64 ui64Timestamp;
+	RGX_HWPERF_UFO_DATA *psHWPerfPktData;
+	IMG_UINT32 ui32UFOCount;
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData;
+
+	psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui32UFOCount = RGX_HWPERF_GET_UFO_STREAMSIZE(psHWPerfPktData->ui32StreamInfo);
+	puData = (RGX_HWPERF_UFO_DATA_ELEMENT *) IMG_OFFSET_ADDR(psHWPerfPktData, RGX_HWPERF_GET_UFO_STREAMOFFSET(psHWPerfPktData->ui32StreamInfo));
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceUfoEvent: ui32ExtJobRef=%d, "
+	        "ui32IntJobRef=%d", psHWPerfPktData->ui32ExtJobRef,
+	        psHWPerfPktData->ui32IntJobRef));
+
+	_GpuTraceUfo(ui64Timestamp, psHWPerfPktData->eEvType,
+	             psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+	             psHWPerfPktData->ui32DMContext, psHWPerfPktData->ui32ExtJobRef,
+	             psHWPerfPktData->ui32IntJobRef, ui32UFOCount, puData);
+}
+
+static void _GpuTraceFirmwareEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt, const IMG_CHAR* pszWorkName,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+
+{
+	uint64_t ui64Timestamp;
+	RGX_HWPERF_FW_DATA *psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	_GpuTraceFirmware(ui64Timestamp, psDevInfo->psDeviceNode->sDevId.ui32InternalID, pszWorkName,
+	                  eSwType);
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+static IMG_BOOL ValidAndEmitFTraceEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt)
+{
+	RGX_HWPERF_EVENT_TYPE eType;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData = psDevInfo->pvGpuFtraceData;
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	IMG_UINT32 ui32HwEventTypeIndex;
+	static const struct {
+		IMG_CHAR* pszName;
+		PVR_GPUTRACE_SWITCH_TYPE eSwType;
+	} aszHwEventTypeMap[] = {
+#define _T(T) PVR_GPUTRACE_SWITCH_TYPE_##T
+		{ "BG",             _T(BEGIN)  }, /* RGX_HWPERF_FW_BGSTART */
+		{ "BG",             _T(END)    }, /* RGX_HWPERF_FW_BGEND */
+		{ "IRQ",            _T(BEGIN)  }, /* RGX_HWPERF_FW_IRQSTART */
+		{ "IRQ",            _T(END)    }, /* RGX_HWPERF_FW_IRQEND */
+		{ "DBG",            _T(BEGIN)  }, /* RGX_HWPERF_FW_DBGSTART */
+		{ "DBG",            _T(END)    }, /* RGX_HWPERF_FW_DBGEND */
+		{ "PMOOM_TAPAUSE",  _T(END)    }, /* RGX_HWPERF_HW_PMOOM_TAPAUSE */
+		{ "TA",             _T(BEGIN)  }, /* RGX_HWPERF_HW_TAKICK */
+		{ "TA",             _T(END)    }, /* RGX_HWPERF_HW_TAFINISHED */
+		{ "TQ3D",           _T(BEGIN)  }, /* RGX_HWPERF_HW_3DTQKICK */
+		{ "3D",             _T(BEGIN)  }, /* RGX_HWPERF_HW_3DKICK */
+		{ "3D",             _T(END)    }, /* RGX_HWPERF_HW_3DFINISHED */
+		{ "CDM",            _T(BEGIN)  }, /* RGX_HWPERF_HW_CDMKICK */
+		{ "CDM",            _T(END)    }, /* RGX_HWPERF_HW_CDMFINISHED */
+		{ "TQ2D",           _T(BEGIN)  }, /* RGX_HWPERF_HW_TLAKICK */
+		{ "TQ2D",           _T(END)    }, /* RGX_HWPERF_HW_TLAFINISHED */
+		{ "3DSPM",          _T(BEGIN)  }, /* RGX_HWPERF_HW_3DSPMKICK */
+		{ NULL,             0          }, /* RGX_HWPERF_HW_PERIODIC (unsupported) */
+		{ "RTU",            _T(BEGIN)  }, /* RGX_HWPERF_HW_RTUKICK */
+		{ "RTU",            _T(END)    }, /* RGX_HWPERF_HW_RTUFINISHED */
+		{ "SHG",            _T(BEGIN)  }, /* RGX_HWPERF_HW_SHGKICK */
+		{ "SHG",            _T(END)    }, /* RGX_HWPERF_HW_SHGFINISHED */
+		{ "TQ3D",           _T(END)    }, /* RGX_HWPERF_HW_3DTQFINISHED */
+		{ "3DSPM",          _T(END)    }, /* RGX_HWPERF_HW_3DSPMFINISHED */
+		{ "PMOOM_TARESUME", _T(BEGIN)  }, /* RGX_HWPERF_HW_PMOOM_TARESUME */
+		{ "TDM",            _T(BEGIN)  }, /* RGX_HWPERF_HW_TDMKICK */
+		{ "TDM",            _T(END)    }, /* RGX_HWPERF_HW_TDMFINISHED */
+		{ "NULL",           _T(SINGLE) }, /* RGX_HWPERF_HW_NULLKICK */
+		{ "GPU_WORK_PERIOD",_T(SINGLE) }, /* RGX_HWPERF_HW_GPU_WORK_PERIOD */
+#undef _T
+	};
+	static_assert(RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE == RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE + 1,
+				  "FW and HW events are not contiguous in RGX_HWPERF_EVENT_TYPE");
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+	PVR_ASSERT(psHWPerfPkt);
+	eType = RGX_HWPERF_GET_TYPE(psHWPerfPkt);
+
+	if (psFtraceData->bTrackOrdinals)
+	{
+		if (psFtraceData->ui32FTraceLastOrdinal != psHWPerfPkt->ui32Ordinal - 1)
+		{
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+			_GpuTraceEventsLost(RGX_HWPERF_GET_STREAM_ID(psHWPerfPkt),
+			                    psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+			                    psFtraceData->ui32FTraceLastOrdinal,
+			                    psHWPerfPkt->ui32Ordinal);
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+			PVR_DPF((PVR_DBG_WARNING, "FTrace events lost (stream_id = %u, ordinal: last = %u, current = %u)",
+			         RGX_HWPERF_GET_STREAM_ID(psHWPerfPkt), psFtraceData->ui32FTraceLastOrdinal,
+			         psHWPerfPkt->ui32Ordinal));
+		}
+	}
+	else
+	{
+		psFtraceData->bTrackOrdinals = IMG_TRUE;
+	}
+
+	psFtraceData->ui32FTraceLastOrdinal = psHWPerfPkt->ui32Ordinal;
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	/* Process UFO packets */
+	if (eType == RGX_HWPERF_UFO)
+	{
+		_GpuTraceUfoEvent(psDevInfo, psHWPerfPkt);
+		return IMG_TRUE;
+	}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	if (eType <= RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE)
+	{
+		/* this ID belongs to range 0, so index directly in range 0 */
+		ui32HwEventTypeIndex = eType - RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+	}
+	else
+	{
+		/* this ID belongs to range 1, so first index in range 1 and skip number of slots used up for range 0 */
+		ui32HwEventTypeIndex = (eType - RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE) +
+		                       (RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE - RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE + 1);
+	}
+
+	if (ui32HwEventTypeIndex >= ARRAY_SIZE(aszHwEventTypeMap))
+	{
+		goto err_unsupported;
+	}
+
+	if (aszHwEventTypeMap[ui32HwEventTypeIndex].pszName == NULL)
+	{
+		/* Not supported map entry, ignore event */
+		goto err_unsupported;
+	}
+
+	if (HWPERF_PACKET_IS_HW_TYPE(eType))
+	{
+		if (aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType == PVR_GPUTRACE_SWITCH_TYPE_SINGLE)
+		{
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     PVR_GPUTRACE_SWITCH_TYPE_BEGIN);
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     PVR_GPUTRACE_SWITCH_TYPE_END);
+		}
+		else
+		{
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType);
+		}
+
+		return IMG_TRUE;
+	}
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	else if (psDevInfo->psDeviceNode->bGPUWorkPeriodFTraceEnabled &&
+		 eType == RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD &&
+		 RGX_HWPERF_GET_OSID(psHWPerfPkt) == RGXFW_HOST_DRIVER_ID)
+	{
+		RGX_HWPERF_GPU_WORK_PERIOD_DATA *psHWPerfPktData;
+		IMG_UINT64 ui64StartTimestamp, ui64EndTimestamp;
+
+		psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+		ui64StartTimestamp = CalculateEventTimestamp(psDevInfo,
+					psHWPerfPktData->ui32StartTimeCorrIndex,
+					psHWPerfPktData->ui64GPUWorkPeriodStartTime);
+		ui64EndTimestamp = CalculateEventTimestamp(psDevInfo,
+					psHWPerfPktData->ui32TimeCorrIndex,
+					psHWPerfPkt->ui64Timestamp);
+
+		PVR_ASSERT(ui64EndTimestamp > ui64StartTimestamp);
+
+		trace_gpu_work_period(psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+				psHWPerfPktData->ui32UID,
+				ui64StartTimestamp,
+				ui64EndTimestamp,
+				ui64EndTimestamp - ui64StartTimestamp);
+
+		return IMG_TRUE;
+	}
+#endif
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	if (HWPERF_PACKET_IS_FW_TYPE(eType))
+	{
+		_GpuTraceFirmwareEvent(psDevInfo, psHWPerfPkt,
+										aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+										aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType);
+
+		return IMG_TRUE;
+	}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+err_unsupported:
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Unsupported event type %d", __func__, eType));
+	return IMG_FALSE;
+}
+
+static void _GpuTraceProcessPackets(PVRSRV_RGXDEV_INFO *psDevInfo, void *pBuffer,
+                                    IMG_UINT32 ui32ReadLen)
+{
+	IMG_UINT32			ui32TlPackets = 0;
+	IMG_UINT32			ui32HWPerfPackets = 0;
+	IMG_UINT32			ui32HWPerfPacketsSent = 0;
+	void				*pBufferEnd;
+	PVRSRVTL_PPACKETHDR psHDRptr;
+	PVRSRVTL_PACKETTYPE ui16TlType;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDevInfo);
+	PVR_ASSERT(pBuffer);
+	PVR_ASSERT(ui32ReadLen);
+
+	/* Process the TL Packets
+	 */
+	pBufferEnd = IMG_OFFSET_ADDR(pBuffer, ui32ReadLen);
+	psHDRptr = GET_PACKET_HDR(pBuffer);
+	while ( psHDRptr < (PVRSRVTL_PPACKETHDR)pBufferEnd )
+	{
+		ui16TlType = GET_PACKET_TYPE(psHDRptr);
+		if (ui16TlType == PVRSRVTL_PACKETTYPE_DATA)
+		{
+			IMG_UINT16 ui16DataLen = GET_PACKET_DATA_LEN(psHDRptr);
+			if (0 == ui16DataLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "_GpuTraceProcessPackets: ZERO Data in TL data packet: %p", psHDRptr));
+			}
+			else
+			{
+				RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt;
+				RGX_HWPERF_V2_PACKET_HDR* psHWPerfEnd;
+
+				/* Check for lost hwperf data packets */
+				psHWPerfEnd = RGX_HWPERF_GET_PACKET(GET_PACKET_DATA_PTR(psHDRptr)+ui16DataLen);
+				psHWPerfPkt = RGX_HWPERF_GET_PACKET(GET_PACKET_DATA_PTR(psHDRptr));
+				do
+				{
+					if (ValidAndEmitFTraceEvent(psDevInfo, psHWPerfPkt))
+					{
+						ui32HWPerfPacketsSent++;
+					}
+					ui32HWPerfPackets++;
+					psHWPerfPkt = RGX_HWPERF_GET_NEXT_PACKET(psHWPerfPkt);
+				}
+				while (psHWPerfPkt < psHWPerfEnd);
+			}
+		}
+		else if (ui16TlType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "_GpuTraceProcessPackets: Indication that the transport buffer was full"));
+		}
+		else
+		{
+			/* else Ignore padding packet type and others */
+			PVR_DPF((PVR_DBG_MESSAGE, "_GpuTraceProcessPackets: Ignoring TL packet, type %d", ui16TlType ));
+		}
+
+		psHDRptr = GET_NEXT_PACKET_ADDR(psHDRptr);
+		ui32TlPackets++;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceProcessPackets: TL "
+			"Packets processed %03d, HWPerf packets %03d, sent %03d",
+			ui32TlPackets, ui32HWPerfPackets, ui32HWPerfPacketsSent));
+
+	PVR_DPF_RETURN;
+}
+
+static void _GpuTraceCmdCompleteNotify(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_RGXDEV_INFO *psDeviceInfo = hCmdCompHandle;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+	PVRSRV_ERROR eError;
+	IMG_PBYTE pBuffer;
+	IMG_UINT32 ui32ReadLen;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDeviceInfo != NULL);
+
+	psFtraceData = psDeviceInfo->pvGpuFtraceData;
+
+	/* Command-complete notifiers can run concurrently. If this is happening,
+	 * just bail out and let the previous call finish.
+	 * This is ok because we can process the queued packets on the next call.
+	 */
+	if (!OSTryLockAcquire(psFtraceData->hFTraceResourceLock))
+	{
+		PVR_DPF_RETURN;
+	}
+
+	/* If this notifier is called, it means the TL resources will be valid
+	 * at-least until the end of this call, since the DeInit function will wait
+	 * on the hFTraceResourceLock to clean-up the TL resources and un-register
+	 * the notifier, so just assert here.
+	 */
+	PVR_ASSERT(psFtraceData->hGPUFTraceTLStream != NULL);
+
+	/* If we have a valid stream attempt to acquire some data */
+	eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+	                             psFtraceData->hGPUFTraceTLStream,
+	                             &pBuffer, &ui32ReadLen);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_LOG_ERROR(eError, "TLClientAcquireData");
+		}
+
+		goto unlock;
+	}
+
+	/* Process the HWPerf packets and release the data */
+	if (ui32ReadLen > 0)
+	{
+		PVR_DPF((PVR_DBG_VERBOSE, "%s: DATA AVAILABLE offset=%p, length=%d",
+		        __func__, pBuffer, ui32ReadLen));
+
+		/* Process the transport layer data for HWPerf packets... */
+		_GpuTraceProcessPackets(psDeviceInfo, pBuffer, ui32ReadLen);
+
+		eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE,
+		                             psFtraceData->hGPUFTraceTLStream);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "TLClientReleaseData");
+
+			/* Serious error, disable FTrace GPU events */
+			_GpuTraceDisable(psDeviceInfo, IMG_FALSE);
+		}
+	}
+
+unlock:
+	OSLockRelease(psFtraceData->hFTraceResourceLock);
+
+	PVR_DPF_RETURN;
+}
+
+/* ----- AppHint interface -------------------------------------------------- */
+
+static PVRSRV_ERROR _GpuTraceIsEnabledCallback(
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data,
+	IMG_BOOL *value)
+{
+	PVR_UNREFERENCED_PARAMETER(private_data);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (device && PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, device))
+	{
+		*value = IMG_FALSE;
+
+		return PVRSRV_OK;
+	}
+	else
+#endif
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+		PVRSRV_DEVICE_NODE *psDeviceNode = psPVRSRVData ? psPVRSRVData->psDeviceNodeList : NULL;
+
+		while (psDeviceNode)
+		{
+			PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+			{
+				*value = gbFTraceGPUEventsEnabled;
+				return PVRSRV_OK;
+			}
+
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		*value = IMG_FALSE;
+
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#else
+		*value = gbFTraceGPUEventsEnabled;
+
+		return PVRSRV_OK;
+#endif
+	}
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabledCallback(
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data,
+	IMG_BOOL value)
+{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (device && PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, device))
+	{
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	/* Lock down the state to avoid concurrent writes */
+	OSLockAcquire(ghGPUTraceStateLock);
+
+	if (value != gbFTraceGPUEventsEnabled)
+	{
+#if defined(PVRSRV_NEED_PVR_DPF)
+		const IMG_CHAR *pszOperation = value ? "enable" : "disable";
+		/* in case MESSAGE level is compiled out */
+		PVR_UNREFERENCED_PARAMETER(pszOperation);
+#endif
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		if (trace_set_clr_event("rogue", NULL, (int) value) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s GPU FTrace event group",
+			        pszOperation));
+			goto err_restore_state;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "FTrace events from \"rogue\" group %sd",
+			        pszOperation));
+		}
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+		if (trace_set_clr_event("power", "gpu_work_period", (int) value) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s gpu_work_period GPU FTrace event",
+			        pszOperation));
+			goto err_restore_state;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "FTrace event from \"gpu_work_period\" %sd",
+			        pszOperation));
+		}
+#endif
+
+		if (value)
+		{
+			/* this enables FTrace globally (if not enabled nothing will appear
+			 * in the FTrace buffer) */
+			tracing_on();
+		}
+
+		/*  The HWPerf supplier is activated here,
+		    The FTrace consumer is activated above,
+		    The consumer should be active before the supplier    */
+		if (_GpuTraceSetEnabledForAllDevices(value) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s GPU FTrace for all devices",
+			        pszOperation));
+			goto err_restore_state;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s GPU FTrace", value ? "ENABLED" : "DISABLED"));
+		gbFTraceGPUEventsEnabled = value;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "GPU FTrace already %s!",
+		        value ? "enabled" : "disabled"));
+	}
+
+	OSLockRelease(ghGPUTraceStateLock);
+
+	return PVRSRV_OK;
+
+err_restore_state:
+	/* On failure, partial enable/disable might have resulted. Try best to
+	 * restore to previous state. Ignore error */
+	(void) _GpuTraceSetEnabledForAllDevices(gbFTraceGPUEventsEnabled);
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	(void) trace_set_clr_event("rogue", NULL,
+	                           (int) gbFTraceGPUEventsEnabled);
+#endif
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	(void) trace_set_clr_event("power", "gpu_work_period",
+	                           (int) gbFTraceGPUEventsEnabled);
+#endif
+
+	OSLockRelease(ghGPUTraceStateLock);
+
+	return PVRSRV_ERROR_UNABLE_TO_ENABLE_EVENT;
+}
+
+void PVRGpuTraceInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFTraceGPU,
+	                                  _GpuTraceIsEnabledCallback,
+	                                  _GpuTraceSetEnabledCallback,
+	                                  psDeviceNode, NULL);
+}
+
+/* ----- FTrace event callbacks -------------------------------------------- */
+
+void PVRGpuTraceEnableUfoCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	/* Lock down events state, for consistent value of guiUfoEventRef */
+	OSLockAcquire(ghLockFTraceEventLock);
+	if (guiUfoEventRef++ == 0)
+	{
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+		/* make sure UFO events are enabled on all rogue devices */
+		while (psDeviceNode)
+		{
+#if defined(SUPPORT_RGX)
+			PVRSRV_ERROR eError;
+
+			/* Small chance exists that ui64HWPerfFilter can be changed here and
+			 * the newest filter value will be changed to the old one + UFO event.
+			 * This is not a critical problem. */
+			eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+			                               RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO),
+			                               HWPERF_FILTER_OPERATION_BIT_OR);
+			if (eError == PVRSRV_ERROR_NOT_INITIALISED)
+			{
+				/* If we land here that means that the FW is not initialised yet.
+				 * We stored the filter and it will be passed to the firmware
+				 * during its initialisation phase. So ignore. */
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Could not enable UFO HWPerf events on device %d", psDeviceNode->sDevId.i32KernelDeviceID));
+			}
+#endif
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+	OSLockRelease(ghLockFTraceEventLock);
+}
+
+void PVRGpuTraceDisableUfoCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	/* We have to check if lock is valid because on driver unload
+	 * PVRGpuTraceSupportDeInit is called before kernel disables the ftrace
+	 * events. This means that the lock will be destroyed before this callback
+	 * is called.
+	 * We can safely return if that situation happens because driver will be
+	 * unloaded so we don't care about HWPerf state anymore. */
+	if (ghLockFTraceEventLock == NULL)
+		return;
+
+	/* Lock down events state, for consistent value of guiUfoEventRef */
+	OSLockAcquire(ghLockFTraceEventLock);
+	if (--guiUfoEventRef == 0)
+	{
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+		/* make sure UFO events are disabled on all rogue devices */
+		while (psDeviceNode)
+		{
+#if defined(SUPPORT_RGX)
+			PVRSRV_ERROR eError;
+
+			/* Small chance exists that ui64HWPerfFilter can be changed here and
+			 * the newest filter value will be changed to the old one + UFO event.
+			 * This is not a critical problem. */
+			eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+			                               RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO),
+			                               HWPERF_FILTER_OPERATION_BIT_CLR);
+			if (eError == PVRSRV_ERROR_NOT_INITIALISED)
+			{
+				/* If we land here that means that the FW is not initialised yet.
+				 * We stored the filter and it will be passed to the firmware
+				 * during its initialisation phase. So ignore. */
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Could not disable UFO HWPerf events on device %d",
+				        psDeviceNode->sDevId.i32KernelDeviceID));
+			}
+#endif
+
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+	OSLockRelease(ghLockFTraceEventLock);
+}
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_UINT64 uiFilter = 0;
+	int i;
+
+	for (i = RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+		 i <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE; i++)
+	{
+		uiFilter |= RGX_HWPERF_EVENT_MASK_VALUE(i);
+	}
+
+	OSLockAcquire(ghLockFTraceEventLock);
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	/* Enable all FW events on all the devices */
+	while (psDeviceNode)
+	{
+		PVRSRV_ERROR eError;
+
+		eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+		                               uiFilter, HWPERF_FILTER_OPERATION_BIT_OR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could not enable HWPerf event for firmware"
+			        " task timings (%s).", PVRSRVGetErrorString(eError)));
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	OSLockRelease(ghLockFTraceEventLock);
+#endif /* defined(SUPPORT_RGX) */
+}
+
+void PVRGpuTraceDisableFirmwareActivityCallback(void)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_UINT64 uiFilter = 0;
+	int i;
+
+	/* We have to check if lock is valid because on driver unload
+	 * PVRGpuTraceSupportDeInit is called before kernel disables the ftrace
+	 * events. This means that the lock will be destroyed before this callback
+	 * is called.
+	 * We can safely return if that situation happens because driver will be
+	 * unloaded so we don't care about HWPerf state anymore. */
+	if (ghLockFTraceEventLock == NULL)
+		return;
+
+	OSLockAcquire(ghLockFTraceEventLock);
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	for (i = RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+		 i <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE; i++)
+	{
+		uiFilter |= RGX_HWPERF_EVENT_MASK_VALUE(i);
+	}
+
+	/* Disable all FW events on all the devices */
+	while (psDeviceNode)
+	{
+		if (PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+		                          uiFilter, HWPERF_FILTER_OPERATION_BIT_CLR) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could not disable HWPerf event for firmware task timings."));
+		}
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	OSLockRelease(ghLockFTraceEventLock);
+#endif /* defined(SUPPORT_RGX) */
+}
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableWorkPeriodCallback(void)
+#else
+void PVRGpuTraceEnableWorkPeriodCallback(void)
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	/* enable/disable GPU trace on all devices */
+	while (psDeviceNode)
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			psDeviceNode->bGPUWorkPeriodFTraceEnabled = true;
+			eError = PVRSRV_OK;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	if (eError != PVRSRV_OK)
+		return -ENODEV;
+	return 0;
+#else
+	return;
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+}
+
+void PVRGpuTraceDisableWorkPeriodCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	if (!psPVRSRVData)
+		return;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	while (psDeviceNode)
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			psDeviceNode->bGPUWorkPeriodFTraceEnabled = false;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+}
+#endif
+/******************************************************************************
+ End of file (pvr_gputrace.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_ion_stats.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_ion_stats.h
new file mode 100644
index 000000000000..c4748138ccd8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_ion_stats.h
@@ -0,0 +1,96 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for recording ION memory stats.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_ION_STATS_H
+#define PVR_ION_STATS_H
+
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+struct dma_buf;
+
+#if defined(PVRSRV_ENABLE_PVR_ION_STATS)
+PVRSRV_ERROR PVRSRVIonStatsInitialise(void);
+
+void PVRSRVIonStatsDestroy(void);
+
+void PVRSRVIonAddMemAllocRecord(struct dma_buf *psDmaBuf);
+
+void PVRSRVIonRemoveMemAllocRecord(struct dma_buf *psDmaBuf);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void PVRSRVIonZombifyMemAllocRecord(const struct dma_buf *psDmaBuf);
+void PVRSRVIonReviveMemAllocRecord(const struct dma_buf *psDmaBuf);
+#endif
+#else
+static INLINE PVRSRV_ERROR PVRSRVIonStatsInitialise(void)
+{
+	return PVRSRV_OK;
+}
+
+static INLINE void PVRSRVIonStatsDestroy(void)
+{
+}
+
+static INLINE void PVRSRVIonAddMemAllocRecord(struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+
+static INLINE void PVRSRVIonRemoveMemAllocRecord(struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static INLINE void PVRSRVIonZombifyMemAllocRecord(const struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+static INLINE void PVRSRVIonReviveMemAllocRecord(const struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+#endif
+#endif /* defined(PVRSRV_ENABLE_PVR_ION_STATS) */
+
+#endif /* PVR_ION_STATS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_platform_drv.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_platform_drv.c
new file mode 100644
index 000000000000..57e27ebcc1c8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_platform_drv.c
@@ -0,0 +1,318 @@
+/*
+ * @File
+ * @Title       PowerVR DRM platform driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_print.h>
+#include <linux/mod_devicetable.h>
+#include <linux/dma-mapping.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "module_common.h"
+#include "pvr_drv.h"
+#include "pvrmodule.h"
+#include "sysinfo.h"
+
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+unsigned int _corefreq_div = 800000000;
+module_param_named(corefreq_div, _corefreq_div, uint, 0644);
+MODULE_PARM_DESC(corefreq_div, "core frequency div from 1600M, default 800M");
+MODULE_IMPORT_NS(DMA_BUF);
+
+static struct drm_driver pvr_drm_platform_driver;
+
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+/*
+ * This is an arbitrary value. If it's changed then the 'num_devices' module
+ * parameter description should also be updated to match.
+ */
+#define MAX_DEVICES 16
+static unsigned int pvr_num_devices = 1;
+static struct platform_device **pvr_devices;
+
+#if defined(NO_HARDWARE)
+static int pvr_num_devices_set(const char *val,
+			       const struct kernel_param *param)
+{
+	int err;
+
+	err = param_set_uint(val, param);
+	if (err)
+		return err;
+
+	if (pvr_num_devices == 0 || pvr_num_devices > PVRSRV_MAX_DEVICES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct kernel_param_ops pvr_num_devices_ops = {
+	.set = pvr_num_devices_set,
+	.get = param_get_uint,
+};
+
+#define STR(s) #s
+#define STRINGIFY(s) STR(s)
+
+module_param_cb(num_devices, &pvr_num_devices_ops, &pvr_num_devices, 0444);
+MODULE_PARM_DESC(num_devices,
+		 "Number of platform devices to register (default: 1 - max: "
+		 STRINGIFY(PVRSRV_MAX_DEVICES) ")");
+#endif /* defined(NO_HARDWARE) */
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+
+static int pvr_devices_register(void)
+{
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+	struct platform_device_info pvr_dev_info = {
+		.name = SYS_RGX_DEV_NAME,
+		.id = -2,
+#if defined(NO_HARDWARE)
+		/* Not all cores have 40 bit physical support, but this
+		 * will work unless > 32 bit address is returned on those cores.
+		 * In the future this will be fixed more correctly.
+		 */
+		.dma_mask = DMA_BIT_MASK(40),
+#else
+		.dma_mask = DMA_BIT_MASK(32),
+#endif
+	};
+	unsigned int i;
+
+	BUG_ON(pvr_num_devices == 0 || pvr_num_devices > PVRSRV_MAX_DEVICES);
+
+	pvr_devices = kmalloc_array(pvr_num_devices, sizeof(*pvr_devices),
+				    GFP_KERNEL);
+	if (!pvr_devices)
+		return -ENOMEM;
+
+	for (i = 0; i < pvr_num_devices; i++) {
+		pvr_devices[i] = platform_device_register_full(&pvr_dev_info);
+		if (IS_ERR(pvr_devices[i])) {
+			DRM_ERROR("unable to register device %u (err=%ld)\n",
+				  i, PTR_ERR(pvr_devices[i]));
+			pvr_devices[i] = NULL;
+			return -ENODEV;
+		}
+	}
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+
+	return 0;
+}
+
+static void pvr_devices_unregister(void)
+{
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+	unsigned int i;
+
+	BUG_ON(!pvr_devices);
+
+	for (i = 0; i < pvr_num_devices && pvr_devices[i]; i++)
+		platform_device_unregister(pvr_devices[i]);
+
+	kfree(pvr_devices);
+	pvr_devices = NULL;
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+}
+
+static int pvr_probe(struct platform_device *pdev)
+{
+	struct drm_device *ddev;
+	int ret;
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	ddev = drm_dev_alloc(&pvr_drm_platform_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+
+	/*
+	 * The load callback, called from drm_dev_register, is deprecated,
+	 * because of potential race conditions. Calling the function here,
+	 * before calling drm_dev_register, avoids those potential races.
+	 */
+	BUG_ON(pvr_drm_platform_driver.load != NULL);
+	ret = pvr_drm_load(ddev, 0);
+	if (ret)
+		goto err_drm_dev_put;
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret)
+		goto err_drm_dev_unload;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+		pvr_drm_platform_driver.name,
+		pvr_drm_platform_driver.major,
+		pvr_drm_platform_driver.minor,
+		pvr_drm_platform_driver.patchlevel,
+		pvr_drm_platform_driver.date,
+		ddev->primary->index);
+#endif
+	return 0;
+
+err_drm_dev_unload:
+	pvr_drm_unload(ddev);
+err_drm_dev_put:
+	drm_dev_put(ddev);
+	return	ret;
+}
+
+static int pvr_remove(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	drm_dev_unregister(ddev);
+
+	/* The unload callback, called from drm_dev_unregister, is
+	 * deprecated. Call the unload function directly.
+	 */
+	BUG_ON(pvr_drm_platform_driver.unload != NULL);
+	pvr_drm_unload(ddev);
+
+	drm_dev_put(ddev);
+	return 0;
+}
+
+static void pvr_shutdown(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	PVRSRVDeviceShutdown(ddev);
+}
+
+static const struct of_device_id pvr_of_ids[] = {
+#if defined(SYS_RGX_OF_COMPATIBLE)
+	{ .compatible = SYS_RGX_OF_COMPATIBLE, },
+#endif
+	{},
+};
+
+#if !defined(CHROMIUMOS_KERNEL)
+MODULE_DEVICE_TABLE(of, pvr_of_ids);
+#endif
+
+static struct platform_device_id pvr_platform_ids[] = {
+#if defined(SYS_RGX_DEV_NAME)
+	{ SYS_RGX_DEV_NAME, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_0)
+	{ SYS_RGX_DEV_NAME_0, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_1)
+	{ SYS_RGX_DEV_NAME_1, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_2)
+	{ SYS_RGX_DEV_NAME_2, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_3)
+	{ SYS_RGX_DEV_NAME_3, 0 },
+#endif
+	{ }
+};
+
+#if !defined(CHROMIUMOS_KERNEL)
+MODULE_DEVICE_TABLE(platform, pvr_platform_ids);
+#endif
+
+static struct platform_driver pvr_platform_driver = {
+	.driver = {
+		.name		= DRVNAME,
+		.of_match_table	= of_match_ptr(pvr_of_ids),
+		.pm		= &pvr_pm_ops,
+	},
+	.id_table		= pvr_platform_ids,
+	.probe			= pvr_probe,
+	.remove			= pvr_remove,
+	.shutdown		= pvr_shutdown,
+};
+
+static int __init pvr_init(void)
+{
+	int err;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	pvr_drm_platform_driver = pvr_drm_generic_driver;
+
+	err = PVRSRVDriverInit();
+	if (err)
+		return err;
+
+	err = platform_driver_register(&pvr_platform_driver);
+	if (err)
+		return err;
+
+	return pvr_devices_register();
+}
+
+static void __exit pvr_exit(void)
+{
+	DRM_DEBUG_DRIVER("\n");
+
+	pvr_devices_unregister();
+	platform_driver_unregister(&pvr_platform_driver);
+	PVRSRVDriverDeinit();
+
+	DRM_DEBUG_DRIVER("done\n");
+}
+
+module_init(pvr_init);
+module_exit(pvr_exit);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.c
new file mode 100644
index 000000000000..34408fc64875
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.c
@@ -0,0 +1,633 @@
+/*************************************************************************/ /*!
+@File
+@Title          ProcFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via ProcFS.
+
+                Note about locking in ProcFS module.
+
+                Access to ProcFS is protected against the race where any
+                file could be removed while being accessed or accessed while
+                being removed. Any calls to proc_remove() will block until all
+                operations are finished.
+
+                See implementation of file operations (proc_reg_*()) and
+                implementation of (un)use_pde() and proc_entry_rundown() in
+                source/fs/proc/inode.c in Linux kernel sources for more details.
+
+                Not about locking for sequential files.
+
+                The seq_file objects have a mutex that protects access
+                to all of the file operations hence all of the sequential
+                *read* operations are protected.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvr_procfs.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_bridge_k.h"
+#include "pvr_uaccess.h"
+#include "osdi_impl.h"
+#include "kernel_compatibility.h"
+
+#define _DRIVER_THREAD_ENTER() \
+	do { \
+		PVRSRV_ERROR eLocalError = PVRSRVDriverThreadEnter(NULL); \
+		if (eLocalError != PVRSRV_OK) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDriverThreadEnter failed: %s", \
+				__func__, PVRSRVGetErrorString(eLocalError))); \
+			return OSPVRSRVToNativeError(eLocalError); \
+		} \
+	} while (0)
+
+#define _DRIVER_THREAD_EXIT() \
+	PVRSRVDriverThreadExit(NULL)
+
+#define PVR_DEBUGFS_PVR_DPF_LEVEL PVR_DBG_ERROR
+
+typedef struct DFS_DIR
+{
+	struct proc_dir_entry *psDirEntry;
+	struct DFS_DIR *psParentDir;
+} DFS_DIR;
+
+typedef struct DFS_ENTRY
+{
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+} DFS_ENTRY;
+
+typedef struct DFS_FILE
+{
+	struct proc_dir_entry *psFileEntry;
+	struct DFS_DIR *psParentDir;
+	const struct seq_operations *psSeqOps;
+	struct DFS_ENTRY sEntry;
+	DI_ENTRY_TYPE eType;
+} DFS_FILE;
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	seq_write(pvNativeHandle, pvData, uiSize);
+}
+
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	seq_vprintf(pvNativeHandle, pszFmt, pArgs);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	seq_puts(pvNativeHandle, pszStr);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	struct seq_file *psSeqFile = pvNativeHandle;
+	return seq_has_overflowed(psSeqFile);
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- sequential file operations ----------------------------------------- */
+
+static void *_Start(struct seq_file *psSeqFile, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	void *pvRet = psEntry->sIterCb.pfnStart(&psEntry->sImplEntry, puiPos);
+
+	if (pvRet == DI_START_TOKEN)
+	{
+		return SEQ_START_TOKEN;
+	}
+
+	return pvRet;
+}
+
+static void _Stop(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	psEntry->sIterCb.pfnStop(&psEntry->sImplEntry, pvPriv);
+}
+
+static void *_Next(struct seq_file *psSeqFile, void *pvPriv, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	return psEntry->sIterCb.pfnNext(&psEntry->sImplEntry, pvPriv, puiPos);
+}
+
+static int _Show(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	if (pvPriv == SEQ_START_TOKEN)
+	{
+		pvPriv = DI_START_TOKEN;
+	}
+
+	return psEntry->sIterCb.pfnShow(&psEntry->sImplEntry, pvPriv);
+}
+
+static struct seq_operations _g_sSeqOps = {
+	.start = _Start,
+	.stop = _Stop,
+	.next = _Next,
+	.show = _Show
+};
+
+/* ----- file operations ---------------------------------------------------- */
+
+static int _Open(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile = pde_data(psINode);
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL", -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_open(psFile, psDFSFile->psSeqOps);
+	}
+	else
+	{
+		/* private data is NULL as it's going to be set below */
+		iRes = single_open(psFile, _Show, NULL);
+	}
+
+	if (iRes == 0)
+	{
+		struct seq_file *psSeqFile = psFile->private_data;
+
+		DFS_ENTRY *psEntry = OSAllocMem(sizeof(*psEntry));
+		if (psEntry == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem() failed", __func__));
+			iRes = -ENOMEM;
+			goto return_;
+		}
+
+		*psEntry = psDFSFile->sEntry;
+		psSeqFile->private = psEntry;
+		psEntry->sImplEntry.pvNative = psSeqFile;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to seq_open psFile, returning %d",
+		        __func__, iRes));
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static int _Close(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile = pde_data(psINode);
+	DFS_ENTRY *psEntry;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* save pointer to DFS_ENTRY */
+	psEntry = ((struct seq_file *) psFile->private_data)->private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_release(psINode, psFile);
+	}
+	else
+	{
+		iRes = single_release(psINode, psFile);
+	}
+
+	/* free DFS_ENTRY allocated in _Open */
+	OSFreeMem(psEntry);
+
+	/* Validation check as seq_release (and single_release which calls it)
+	 * never fail */
+	if (iRes != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release psFile, returning %d",
+		        __func__, iRes));
+	}
+
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Read(struct file *psFile, char __user *pcBuffer,
+                     size_t uiCount, loff_t *puiPos)
+{
+	DFS_FILE *psDFSFile = pde_data(psFile->f_path.dentry->d_inode);
+	ssize_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_read(psFile, pcBuffer, uiCount, puiPos);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file pfnRead() "
+			        "returned %zd", __func__, iRes));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Count = uiCount, ui64Pos;
+
+		IMG_CHAR *pcLocalBuffer = OSAllocMem(uiCount);
+		PVR_GOTO_IF_FALSE(pcLocalBuffer != NULL, return_);
+
+		iRes = psEntry->sIterCb.pfnRead(pcLocalBuffer, ui64Count, &ui64Pos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file pfnRead() "
+			        "returned %zd", __func__, iRes));
+			OSFreeMem(pcLocalBuffer);
+			goto return_;
+		}
+
+		if (pvr_copy_to_user(pcBuffer, pcLocalBuffer, iRes) != 0)
+		{
+			iRes = -1;
+		}
+
+		OSFreeMem(pcLocalBuffer);
+
+		*puiPos = ui64Pos;
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static loff_t _LSeek(struct file *psFile, loff_t iOffset, int iOrigin)
+{
+	DFS_FILE *psDFSFile = pde_data(psFile->f_path.dentry->d_inode);
+	loff_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_lseek(psFile, iOffset, iOrigin);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position to "
+			        "offset %lld, pfnSeek() returned %lld", __func__,
+			        iOffset, iRes));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Pos;
+
+		switch (iOrigin)
+		{
+			case SEEK_SET:
+				ui64Pos = psFile->f_pos + iOffset;
+				break;
+			case SEEK_CUR:
+				ui64Pos = iOffset;
+				break;
+			case SEEK_END:
+				/* not supported as we don't know the file size here */
+				/* fall through */
+			default:
+				return -1;
+		}
+
+		/* only pass the absolute position to the callback, it's up to the
+		 * implementer to determine if the position is valid */
+
+		iRes = psEntry->sIterCb.pfnSeek(ui64Pos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position to "
+			        "offset %lld, pfnSeek() returned %lld", __func__,
+			        iOffset, iRes));
+			goto return_;
+		}
+
+		psFile->f_pos = ui64Pos;
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Write(struct file *psFile, const char __user *pszBuffer,
+                      size_t uiCount, loff_t *puiPos)
+{
+	struct inode *psINode = psFile->f_path.dentry->d_inode;
+	DFS_FILE *psDFSFile = pde_data(psINode);
+	DI_ITERATOR_CB *psIter = &psDFSFile->sEntry.sIterCb;
+	IMG_CHAR *pcLocalBuffer;
+	IMG_UINT64 ui64Count;
+	IMG_INT64 i64Res = -EIO;
+	IMG_UINT64 ui64Pos = *puiPos;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+	PVR_LOG_RETURN_IF_FALSE(psIter->pfnWrite != NULL, "pfnWrite is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* Make sure we allocate the smallest amount of needed memory*/
+	ui64Count = psIter->ui32WriteLenMax;
+	PVR_LOG_GOTO_IF_FALSE(uiCount <= ui64Count, "uiCount too long", return_);
+	ui64Count = MIN(uiCount+1, ui64Count);
+
+	/* allocate buffer with one additional byte for NUL character */
+	pcLocalBuffer = OSAllocMem(ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(pcLocalBuffer != NULL, "OSAllocMem() failed",
+	                      return_);
+
+	i64Res = pvr_copy_from_user(pcLocalBuffer, pszBuffer, ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(i64Res == 0, "pvr_copy_from_user() failed",
+	                      free_local_buffer_);
+
+	/* ensure that the framework user gets a NUL terminated buffer */
+	pcLocalBuffer[ui64Count - 1] = '\0';
+
+	i64Res = psIter->pfnWrite(pcLocalBuffer, ui64Count, &ui64Pos,
+	                          psDFSFile->sEntry.sImplEntry.pvPrivData);
+	PVR_LOG_GOTO_IF_FALSE(i64Res >= 0, "pfnWrite failed", free_local_buffer_);
+
+	*puiPos = ui64Pos;
+
+free_local_buffer_:
+	OSFreeMem(pcLocalBuffer);
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return i64Res;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+
+static const struct file_operations _g_psFileOpsGen = {
+	.owner = THIS_MODULE,
+	.open = _Open,
+	.release = _Close,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+static const struct file_operations _g_psFileOpsRndAcc = {
+	.owner = THIS_MODULE,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+#else
+
+static const struct proc_ops _g_psFileOpsGen = {
+	.proc_open    = _Open,
+	.proc_read    = _Read,
+	.proc_write   = _Write,
+	.proc_lseek   = _LSeek,
+	.proc_release = _Close,
+};
+
+static const struct proc_ops _g_psFileOpsRndAcc = {
+	.proc_read  = _Read,
+	.proc_write = _Write,
+	.proc_lseek = _LSeek,
+};
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)) */
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	return PVRSRV_OK;
+}
+
+static void _DeInit(void)
+{
+}
+
+static PVRSRV_ERROR _CreateFile(const IMG_CHAR *pszName,
+                                DI_ENTRY_TYPE eType,
+                                const DI_ITERATOR_CB *psIterCb,
+                                void *pvPrivData,
+                                void *pvParentDir,
+                                void **pvFile)
+{
+	DFS_DIR *psParentDir = pvParentDir;
+	DFS_FILE *psFile;
+	umode_t uiMode = S_IFREG;
+	struct proc_dir_entry *psEntry;
+	PVRSRV_ERROR eError;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	const struct file_operations *psProcOps = NULL;
+#else
+	const struct proc_ops *psProcOps = NULL;
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)) */
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvFile != NULL, "pvFile");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentDir != NULL, "pvParentDir");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			psProcOps = &_g_psFileOpsGen;
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			psProcOps = &_g_psFileOpsRndAcc;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto return_;
+	}
+
+	psFile = OSAllocMem(sizeof(*psFile));
+	PVR_LOG_GOTO_IF_NOMEM(psFile, eError, return_);
+
+	uiMode |= psIterCb->pfnShow != NULL || psIterCb->pfnRead != NULL ?
+	        S_IRUGO : 0;
+	uiMode |= psIterCb->pfnWrite != NULL ? S_IWUSR : 0;
+
+	psEntry = proc_create_data(pszName, uiMode, psParentDir->psDirEntry,
+	                           psProcOps, psFile);
+	if (IS_ERR_OR_NULL(psEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create debugfs '%s' file",
+		        __func__, pszName));
+
+		eError = psEntry == NULL ?
+		        PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_ERROR_INVALID_DEVICE;
+		goto free_file_;
+	}
+
+	psFile->eType = eType;
+	psFile->psSeqOps = &_g_sSeqOps;
+	psFile->sEntry.sIterCb = *psIterCb;
+	psFile->sEntry.sImplEntry.pvPrivData = pvPrivData;
+	psFile->sEntry.sImplEntry.pvNative = NULL;
+	psFile->sEntry.sImplEntry.psCb = &_g_sEntryCallbacks;
+	psFile->psParentDir = psParentDir;
+	psFile->psFileEntry = psEntry;
+
+	*pvFile = psFile;
+
+	return PVRSRV_OK;
+
+free_file_:
+	OSFreeMem(psFile);
+
+return_:
+	return eError;
+}
+
+static void _DestroyFile(void *pvFile)
+{
+	DFS_FILE *psFile = pvFile;
+
+	PVR_ASSERT(psFile != NULL);
+
+	proc_remove(psFile->psFileEntry);
+	OSFreeMem(psFile);
+}
+
+static PVRSRV_ERROR _CreateDir(const IMG_CHAR *pszName,
+                               void *pvParentDir,
+                               void **ppvDir)
+{
+	DFS_DIR *psNewDir;
+	struct proc_dir_entry *psDirEntry, *psParentDir = NULL;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvDir != NULL, "ppvDir");
+
+	psNewDir = OSAllocMem(sizeof(*psNewDir));
+	PVR_LOG_RETURN_IF_NOMEM(psNewDir, "OSAllocMem");
+
+	psNewDir->psParentDir = pvParentDir;
+
+	if (pvParentDir != NULL)
+	{
+		psParentDir = psNewDir->psParentDir->psDirEntry;
+	}
+
+	psDirEntry = proc_mkdir(pszName, psParentDir);
+	if (IS_ERR_OR_NULL(psDirEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create '%s' debugfs directory",
+		        __func__, pszName));
+		OSFreeMem(psNewDir);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psNewDir->psDirEntry = psDirEntry;
+	*ppvDir = psNewDir;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyDir(void *pvDir)
+{
+	DFS_DIR *psDir = pvDir;
+
+	PVR_ASSERT(psDir != NULL);
+
+	proc_remove(psDir->psDirEntry);
+	OSFreeMem(psDir);
+}
+
+PVRSRV_ERROR PVRProcFsRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateFile,
+		.pfnDestroyEntry = _DestroyFile,
+		.pfnCreateGroup = _CreateDir,
+		.pfnDestroyGroup = _DestroyDir
+	};
+
+	return DIRegisterImplementation("procfs", &sImplCb);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.h
new file mode 100644
index 000000000000..61a1f0ee28d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_procfs.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Title          ProcFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_PROCFS_H
+#define PVR_PROCFS_H
+
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR PVRProcFsRegister(void);
+
+#endif /* PVR_PROCFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.c
new file mode 100644
index 000000000000..cccc4a64e321
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.c
@@ -0,0 +1,200 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/spinlock_types.h>
+#include <linux/atomic.h>
+#include <linux/slab.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+
+#include "pvr_sw_fence.h"
+#include "osfunc_common.h"
+
+struct pvr_sw_fence_context {
+	struct kref kref;
+	unsigned int context;
+	char context_name[32];
+	char driver_name[32];
+	atomic_t seqno;
+	atomic_t fence_count;
+};
+
+struct pvr_sw_fence {
+	struct dma_fence base;
+	struct pvr_sw_fence_context *fence_context;
+	spinlock_t lock;
+};
+
+#define to_pvr_sw_fence(fence) container_of(fence, struct pvr_sw_fence, base)
+
+const char *pvr_sw_fence_context_name(struct pvr_sw_fence_context *fctx)
+{
+	return fctx->context_name;
+}
+
+void pvr_sw_fence_context_value_str(struct pvr_sw_fence_context *fctx,
+				    char *str, int size)
+{
+	snprintf(str, size, "%d", atomic_read(&fctx->seqno));
+}
+
+static inline unsigned
+pvr_sw_fence_context_seqno_next(struct pvr_sw_fence_context *fence_context)
+{
+	return atomic_inc_return(&fence_context->seqno) - 1;
+}
+
+static const char *pvr_sw_fence_get_driver_name(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	return pvr_sw_fence->fence_context->driver_name;
+}
+
+static const char *pvr_sw_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	return pvr_sw_fence_context_name(pvr_sw_fence->fence_context);
+}
+
+static void pvr_sw_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	snprintf(str, size, "%llu", (u64) fence->seqno);
+}
+
+static void pvr_sw_fence_timeline_value_str(struct dma_fence *fence,
+					    char *str, int size)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	pvr_sw_fence_context_value_str(pvr_sw_fence->fence_context, str, size);
+}
+
+static bool pvr_sw_fence_enable_signaling(struct dma_fence *fence)
+{
+	return true;
+}
+
+static void pvr_sw_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_sw_fence_context *fence_context =
+		container_of(kref, struct pvr_sw_fence_context, kref);
+	unsigned int fence_count;
+
+	fence_count = atomic_read(&fence_context->fence_count);
+	if (WARN_ON(fence_count))
+		pr_debug("%s context has %u fence(s) remaining\n",
+			 fence_context->context_name, fence_count);
+
+	kfree(fence_context);
+}
+
+static void pvr_sw_fence_release(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	atomic_dec(&pvr_sw_fence->fence_context->fence_count);
+	kref_put(&pvr_sw_fence->fence_context->kref,
+		pvr_sw_fence_context_destroy_kref);
+	kfree(pvr_sw_fence);
+}
+
+static const struct dma_fence_ops pvr_sw_fence_ops = {
+	.get_driver_name = pvr_sw_fence_get_driver_name,
+	.get_timeline_name = pvr_sw_fence_get_timeline_name,
+	.fence_value_str = pvr_sw_fence_value_str,
+	.timeline_value_str = pvr_sw_fence_timeline_value_str,
+	.enable_signaling = pvr_sw_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+	.release = pvr_sw_fence_release,
+};
+
+struct pvr_sw_fence_context *
+pvr_sw_fence_context_create(const char *context_name, const char *driver_name)
+{
+	struct pvr_sw_fence_context *fence_context;
+
+	fence_context = kmalloc(sizeof(*fence_context), GFP_KERNEL);
+	if (!fence_context)
+		return NULL;
+
+	fence_context->context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fence_context->context_name, context_name,
+		sizeof(fence_context->context_name));
+	OSStringSafeCopy(fence_context->driver_name, driver_name,
+		sizeof(fence_context->driver_name));
+	atomic_set(&fence_context->seqno, 0);
+	atomic_set(&fence_context->fence_count, 0);
+	kref_init(&fence_context->kref);
+
+	return fence_context;
+}
+
+void pvr_sw_fence_context_destroy(struct pvr_sw_fence_context *fence_context)
+{
+	kref_put(&fence_context->kref, pvr_sw_fence_context_destroy_kref);
+}
+
+struct dma_fence *
+pvr_sw_fence_create(struct pvr_sw_fence_context *fence_context)
+{
+	struct pvr_sw_fence *pvr_sw_fence;
+	unsigned int seqno;
+
+	pvr_sw_fence = kmalloc(sizeof(*pvr_sw_fence), GFP_KERNEL);
+	if (!pvr_sw_fence)
+		return NULL;
+
+	spin_lock_init(&pvr_sw_fence->lock);
+	pvr_sw_fence->fence_context = fence_context;
+
+	seqno = pvr_sw_fence_context_seqno_next(fence_context);
+	dma_fence_init(&pvr_sw_fence->base, &pvr_sw_fence_ops,
+		       &pvr_sw_fence->lock, fence_context->context, seqno);
+
+	atomic_inc(&fence_context->fence_count);
+	kref_get(&fence_context->kref);
+
+	return &pvr_sw_fence->base;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.h
new file mode 100644
index 000000000000..bebbcb75e23b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sw_fence.h
@@ -0,0 +1,60 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_SW_FENCES_H__)
+#define __PVR_SW_FENCES_H__
+
+#include "pvr_linux_fence.h"
+
+struct pvr_sw_fence_context;
+
+struct pvr_sw_fence_context *pvr_sw_fence_context_create(const char *name,
+				const char *driver_name);
+void pvr_sw_fence_context_destroy(struct pvr_sw_fence_context *fence_context);
+struct dma_fence *pvr_sw_fence_create(struct pvr_sw_fence_context *
+				      fence_context);
+
+const char *pvr_sw_fence_context_name(struct pvr_sw_fence_context *fctx);
+void pvr_sw_fence_context_value_str(struct pvr_sw_fence_context *fctx,
+				    char *str, int size);
+
+#endif /* !defined(__PVR_SW_FENCES_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync.h
new file mode 100644
index 000000000000..a8ecd8b2caa0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync.h
@@ -0,0 +1,120 @@
+/*
+ * @File        pvr_sync.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_H
+#define _PVR_SYNC_H
+
+#include <linux/device.h>
+
+#include "pvr_fd_sync_kernel.h"
+#include "services_kernel_client.h"
+
+
+/* Services internal interface */
+
+/**
+ * pvr_sync_register_functions()
+ *
+ * Return: PVRSRV_OK on success.
+ */
+enum PVRSRV_ERROR_TAG pvr_sync_register_functions(void);
+
+/**
+ *  pvr_sync_init - register the pvr_sync misc device
+ *
+ *  Return: error code, 0 on success.
+ */
+int pvr_sync_init(void);
+
+/**
+ * pvr_sync_deinit - unregister the pvr_sync misc device
+ */
+void pvr_sync_deinit(void);
+
+/**
+ * pvr_sync_device_init() - create an internal sync context
+ * @dev: Linux device
+ *
+ * Return: PVRSRV_OK on success.
+ */
+enum PVRSRV_ERROR_TAG pvr_sync_device_init(struct device *dev);
+
+/**
+ * pvr_sync_device_deinit() - destroy an internal sync context
+ *
+ * Drains any work items with outstanding sync fence updates/dependencies.
+ */
+void pvr_sync_device_deinit(struct device *dev);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_wait(void *fence, u32 timeout_in_ms);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_release(void *fence);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_get(int fence_fd, void **fence_out);
+
+enum PVRSRV_ERROR_TAG
+pvr_sync_sw_timeline_fence_create(struct _PVRSRV_DEVICE_NODE_ *pvrsrv_dev_node,
+				  int timeline_fd,
+				  const char *fence_name,
+				  int *fence_fd_out,
+				  u64 *sync_pt_idx);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_advance(void *timeline,
+					       u64 *sync_pt_idx);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_release(void *timeline);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_get(int timeline_fd,
+					   void **timeline_out);
+
+enum PVRSRV_ERROR_TAG
+sync_dump_fence(void *sw_fence_obj,
+		DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		void *dump_debug_file);
+
+enum PVRSRV_ERROR_TAG
+sync_sw_dump_timeline(void *sw_timeline_obj,
+		      DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		      void *dump_debug_file);
+
+#endif /* _PVR_SYNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_api.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_api.h
new file mode 100644
index 000000000000..97dd746439b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_api.h
@@ -0,0 +1,65 @@
+/*
+ * @File        pvr_sync_api.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_API_H
+#define _PVR_SYNC_API_H
+
+#include "img_types.h"
+
+int pvr_sync_api_init(void *file_handle, void **api_priv);
+int pvr_sync_api_deinit(void *api_priv, bool is_sw);
+int pvr_sync_api_rename(void *api_priv, void *user_data);
+int pvr_sync_api_force_sw_only(void *api_priv, void **api_priv_new);
+int pvr_sync_api_sw_create_fence(void *api_priv, void *user_data);
+int pvr_sync_api_sw_inc(void *api_priv, void *user_data);
+int pvr_sync_api_force_exp_only(void *api_priv, void *api_data);
+int pvr_sync_api_create_export_fence(void *api_priv, void *user_data);
+
+struct file;
+
+int pvr_sync_ioctl_init(void);
+void pvr_sync_ioctl_deinit(void);
+void *pvr_sync_get_api_priv(struct file *file);
+struct file *pvr_sync_get_file_struct(void *file_handle);
+
+#endif /* _PVR_SYNC_API_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_file.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_file.c
new file mode 100644
index 000000000000..c6cec6d9dd0d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_file.c
@@ -0,0 +1,1309 @@
+/*
+ * @File        pvr_sync_file.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "services_kernel_client.h"
+#include "pvr_drv.h"
+#include "pvr_sync.h"
+#include "pvr_fence.h"
+#include "pvr_counting_timeline.h"
+#include "pvr_export_fence.h"
+
+#include "linux_sw_sync.h"
+
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/sync_file.h>
+#include <linux/file.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+
+#include "pvr_sync_api.h"
+#include "osfunc_common.h"
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)) && !defined(CHROMIUMOS_KERNEL)
+#define sync_file_user_name(s)	((s)->name)
+#else
+#define sync_file_user_name(s)	((s)->user_name)
+#endif
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+#define	FILE_NAME "pvr_sync_file"
+
+struct sw_sync_create_fence_data {
+	__u32 value;
+	char name[32];
+	__s32 fence;
+};
+#define SW_SYNC_IOC_MAGIC 'W'
+#define SW_SYNC_IOC_CREATE_FENCE \
+	(_IOWR(SW_SYNC_IOC_MAGIC, 0, struct sw_sync_create_fence_data))
+#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
+
+/* Global data for the sync driver */
+static struct {
+	struct pvr_fence_context *foreign_fence_context;
+	PFN_SYNC_CHECKPOINT_STRUCT sync_checkpoint_ops;
+} pvr_sync_data;
+
+#if defined(NO_HARDWARE)
+static DEFINE_MUTEX(pvr_timeline_active_list_lock);
+static struct list_head pvr_timeline_active_list;
+#endif
+
+/* This is the actual timeline metadata. We might keep this around after the
+ * base sync driver has destroyed the pvr_sync_timeline_wrapper object.
+ */
+struct pvr_sync_timeline {
+	char name[32];
+	void *file_handle;
+	bool is_sw;
+	bool is_export;
+	/* Fence context used for hw fences */
+	struct pvr_fence_context *hw_fence_context;
+	/* Timeline and context for sw fences */
+	union {
+		struct pvr_counting_fence_timeline *sw_fence_timeline;
+		struct pvr_exp_fence_context *exp_fence_context;
+	};
+#if defined(NO_HARDWARE)
+	/* List of all timelines (used to advance all timelines in nohw builds) */
+	struct list_head list;
+#endif
+};
+
+static
+void pvr_sync_free_checkpoint_list_mem(void *mem_ptr)
+{
+	kfree(mem_ptr);
+}
+
+#if defined(NO_HARDWARE)
+/* function used to signal pvr fence in nohw builds */
+static
+void pvr_sync_nohw_signal_fence(void *fence_data_to_signal)
+{
+	struct pvr_sync_timeline *this_timeline;
+
+	mutex_lock(&pvr_timeline_active_list_lock);
+	list_for_each_entry(this_timeline, &pvr_timeline_active_list, list) {
+		pvr_fence_context_signal_fences_nohw(this_timeline->hw_fence_context);
+	}
+	mutex_unlock(&pvr_timeline_active_list_lock);
+}
+static
+void pvr_sync_nohw_signal_exp_fence(PVRSRV_FENCE fence_to_signal)
+{
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_signal);
+	if (fence && pvr_is_exp_fence(fence))
+		dma_fence_signal(fence);
+
+	dma_fence_put(fence);
+}
+#endif
+
+static struct pvr_sync_timeline *pvr_sync_timeline_fget(int fd)
+{
+	struct file *file = fget(fd);
+	struct pvr_sync_timeline *timeline;
+
+	if (!file)
+		return NULL;
+
+	timeline = pvr_sync_get_api_priv(file);
+	if (!timeline)
+		fput(file);
+
+	return timeline;
+}
+
+static void pvr_sync_timeline_fput(struct pvr_sync_timeline *timeline)
+{
+	struct file *file = pvr_sync_get_file_struct(timeline->file_handle);
+
+	if (file)
+		fput(file);
+	else
+		pr_err(FILE_NAME ": %s: Timeline incomplete\n", __func__);
+}
+
+/* ioctl and fops handling */
+
+int pvr_sync_api_init(void *file_handle, void **api_priv)
+{
+	struct pvr_sync_timeline *timeline;
+	char task_comm[TASK_COMM_LEN];
+
+	get_task_comm(task_comm, current);
+
+	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
+	if (!timeline)
+		return -ENOMEM;
+
+	OSStringSafeCopy(timeline->name, task_comm, sizeof(timeline->name));
+	timeline->file_handle = file_handle;
+	timeline->is_sw = false;
+	timeline->is_export = false;
+
+	*api_priv = (void *)timeline;
+
+	return 0;
+}
+
+int pvr_sync_api_deinit(void *api_priv, bool is_sw)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	if (!timeline)
+		return 0;
+
+	if (timeline->is_sw && timeline->sw_fence_timeline) {
+		/* This makes sure any outstanding SW syncs are marked as
+		 * complete at timeline close time. Otherwise it'll leak the
+		 * timeline (as outstanding fences hold a ref) and possibly
+		 * wedge the system if something is waiting on one of those
+		 * fences
+		 */
+		pvr_counting_fence_timeline_force_complete(
+			timeline->sw_fence_timeline);
+		pvr_counting_fence_timeline_put(timeline->sw_fence_timeline);
+	} else if (timeline->is_export && timeline->exp_fence_context) {
+		pvr_exp_fence_context_destroy(timeline->exp_fence_context);
+	}
+
+	if (timeline->hw_fence_context) {
+#if defined(NO_HARDWARE)
+		mutex_lock(&pvr_timeline_active_list_lock);
+		list_del(&timeline->list);
+		mutex_unlock(&pvr_timeline_active_list_lock);
+#endif
+		pvr_fence_context_destroy(timeline->hw_fence_context);
+	}
+
+	kfree(timeline);
+
+	return 0;
+}
+
+/*
+ * This is the function that kick code will call in order to 'finalise' a
+ * created output fence just prior to returning from the kick function.
+ * The OS native sync code needs to implement a function meeting this
+ * specification - the implementation may be a nop if the OS does not need
+ * to perform any actions at this point.
+ *
+ * Input: fence_fd            The PVRSRV_FENCE to be 'finalised'. This value
+ *                            will have been returned by an earlier call to
+ *                            pvr_sync_create_fence().
+ * Input: finalise_data       The finalise data returned by an earlier call
+ *                            to pvr_sync_create_fence().
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_finalise_fence(PVRSRV_FENCE fence_fd, void *finalise_data)
+{
+	struct sync_file *sync_file = finalise_data;
+	struct pvr_fence *pvr_fence;
+
+	if (!sync_file || (fence_fd < 0)) {
+		pr_err(FILE_NAME ": %s: Invalid input fence\n", __func__);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pvr_fence = to_pvr_fence(sync_file->fence);
+
+	if (!pvr_fence) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr fence\n", __func__);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* pvr fences can be signalled any time after creation */
+	dma_fence_enable_sw_signaling(&pvr_fence->base);
+
+	fd_install(fence_fd, sync_file->file);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain a new
+ * PVRSRV_FENCE from the OS native sync code and the PSYNC_CHECKPOINT used
+ * in that fence. The OS native sync code needs to implement a function
+ * meeting this specification.
+ *
+ * Input: device                   Device node to use in creating a hw_fence_ctx
+ * Input: fence_name               A string to annotate the fence with (for
+ *                                 debug).
+ * Input: timeline                 The timeline on which the new fence is to be
+ *                                 created.
+ * Output: new_fence               The new PVRSRV_FENCE to be returned by the
+ *                                 kick call.
+ * Output: fence_uid               Unique ID of the update fence.
+ * Output: fence_finalise_data     Pointer to data needed to finalise the fence.
+ * Output: new_checkpoint_handle   The PSYNC_CHECKPOINT used by the new fence.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_create_fence(
+		      struct _PVRSRV_DEVICE_NODE_ *device,
+		      const char *fence_name,
+		      PVRSRV_TIMELINE new_fence_timeline,
+		      PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		      PVRSRV_FENCE *new_fence, u64 *fence_uid,
+		      void **fence_finalise_data,
+		      PSYNC_CHECKPOINT *new_checkpoint_handle,
+		      void **timeline_update_sync,
+		      __u32 *timeline_update_value)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	PVRSRV_FENCE new_fence_fd = -1;
+	struct pvr_sync_timeline *timeline;
+	struct pvr_fence *pvr_fence;
+	PSYNC_CHECKPOINT checkpoint;
+	struct sync_file *sync_file;
+
+	if (new_fence_timeline < 0 || !new_fence || !new_checkpoint_handle
+		|| !fence_finalise_data) {
+		pr_err(FILE_NAME ": %s: Invalid input params\n", __func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	/* We reserve the new fence FD before taking any operations
+	 * as we do not want to fail (e.g. run out of FDs)
+	 */
+	new_fence_fd = get_unused_fd_flags(O_CLOEXEC);
+	if (new_fence_fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to get fd\n", __func__);
+		err = PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		goto err_out;
+	}
+
+	timeline = pvr_sync_timeline_fget(new_fence_timeline);
+	if (!timeline) {
+		pr_err(FILE_NAME ": %s: Failed to open supplied timeline fd (%d)\n",
+			__func__, new_fence_timeline);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_fd;
+	}
+
+	if (timeline->is_sw) {
+		/* This should never happen! */
+		pr_err(FILE_NAME ": %s: Request to create a pvr fence on sw timeline (%d)\n",
+			__func__, new_fence_timeline);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_timeline;
+	}
+
+	if (!timeline->hw_fence_context) {
+		/* First time we use this timeline, so create a context. */
+		timeline->hw_fence_context =
+			pvr_fence_context_create(
+				device,
+				NativeSyncGetFenceStatusWq(),
+				timeline->name);
+		if (!timeline->hw_fence_context) {
+			pr_err(FILE_NAME ": %s: Failed to create fence context (%d)\n",
+			       __func__, new_fence_timeline);
+			err = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto err_put_timeline;
+		}
+#if defined(NO_HARDWARE)
+		/* Add timeline to active list */
+		INIT_LIST_HEAD(&timeline->list);
+		mutex_lock(&pvr_timeline_active_list_lock);
+		list_add_tail(&timeline->list, &pvr_timeline_active_list);
+		mutex_unlock(&pvr_timeline_active_list_lock);
+#endif
+	}
+
+	pvr_fence = pvr_fence_create(timeline->hw_fence_context,
+								 psSyncCheckpointContext,
+								 new_fence_timeline,
+								 fence_name);
+	if (!pvr_fence) {
+		pr_err(FILE_NAME ": %s: Failed to create new pvr_fence\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_timeline;
+	}
+
+	checkpoint = pvr_fence_get_checkpoint(pvr_fence);
+	if (!checkpoint) {
+		pr_err(FILE_NAME ": %s: Failed to get fence checkpoint\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_destroy_fence;
+	}
+
+	sync_file = sync_file_create(&pvr_fence->base);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create sync_file\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_destroy_fence;
+	}
+	OSStringSafeCopy(sync_file_user_name(sync_file),
+		pvr_fence->name,
+		sizeof(sync_file_user_name(sync_file)));
+	dma_fence_put(&pvr_fence->base);
+
+	*new_fence = new_fence_fd;
+	*fence_finalise_data = sync_file;
+	*new_checkpoint_handle = checkpoint;
+	*fence_uid = OSGetCurrentClientProcessIDKM();
+	*fence_uid = (*fence_uid << 32) | (new_fence_fd & U32_MAX);
+	/* not used but don't want to return dangling pointers */
+	*timeline_update_sync = NULL;
+	*timeline_update_value = 0;
+
+	pvr_sync_timeline_fput(timeline);
+err_out:
+	return err;
+
+err_destroy_fence:
+	pvr_fence_destroy(pvr_fence);
+err_put_timeline:
+	pvr_sync_timeline_fput(timeline);
+err_put_fd:
+	put_unused_fd(new_fence_fd);
+	*fence_uid = PVRSRV_NO_FENCE;
+	goto err_out;
+}
+
+/*
+ * This is the function that kick code will call in order to 'rollback' a
+ * created output fence should an error occur when submitting the kick.
+ * The OS native sync code needs to implement a function meeting this
+ * specification.
+ *
+ * Input: fence_to_rollback The PVRSRV_FENCE to be 'rolled back'. The fence
+ *                          should be destroyed and any actions taken due to
+ *                          its creation that need to be undone should be
+ *                          reverted.
+ * Input: finalise_data     The finalise data for the fence to be 'rolled back'.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_rollback_fence_data(PVRSRV_FENCE fence_to_rollback,
+			     void *fence_data_to_rollback)
+{
+	struct sync_file *sync_file = fence_data_to_rollback;
+	struct pvr_fence *pvr_fence;
+
+	if (!sync_file || fence_to_rollback < 0) {
+		pr_err(FILE_NAME ": %s: Invalid fence (%d)\n", __func__,
+			fence_to_rollback);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pvr_fence = to_pvr_fence(sync_file->fence);
+	if (!pvr_fence) {
+		pr_err(FILE_NAME
+			": %s: Non-PVR fence (%p)\n",
+			__func__, sync_file->fence);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	fput(sync_file->file);
+
+	put_unused_fd(fence_to_rollback);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain a list of
+ * the PSYNC_CHECKPOINTs for a given PVRSRV_FENCE passed to a kick function.
+ * The OS native sync code will allocate the memory to hold the returned list
+ * of PSYNC_CHECKPOINT ptrs. The caller will free this memory once it has
+ * finished referencing it.
+ *
+ * Input: fence                     The input (check) fence
+ * Output: nr_checkpoints           The number of PVRSRV_SYNC_CHECKPOINT ptrs
+ *                                  returned in the checkpoint_handles
+ *                                  parameter.
+ * Output: fence_uid                Unique ID of the check fence
+ * Input/Output: checkpoint_handles The returned list of PVRSRV_SYNC_CHECKPOINTs.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_resolve_fence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		       PVRSRV_FENCE fence_to_resolve, u32 *nr_checkpoints,
+		       PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid)
+{
+	PSYNC_CHECKPOINT *checkpoints = NULL;
+	unsigned int i, num_fences = 0, num_used_fences = 0;
+	struct dma_fence **fences = NULL;
+	struct dma_fence *fence;
+	PVRSRV_ERROR err = PVRSRV_OK;
+
+	if (!nr_checkpoints || !checkpoint_handles || !fence_uid) {
+		pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
+			__func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	*nr_checkpoints = 0;
+	*checkpoint_handles = NULL;
+	*fence_uid = 0;
+
+	if (fence_to_resolve < 0)
+		goto err_out;
+
+	fence = sync_file_get_fence(fence_to_resolve);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_resolve);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
+
+		if (array) {
+			fences = array->fences;
+			num_fences = array->num_fences;
+		}
+	} else {
+		fences = &fence;
+		num_fences = 1;
+	}
+
+	checkpoints = kmalloc_array(num_fences, sizeof(PSYNC_CHECKPOINT),
+			      GFP_KERNEL);
+	if (!checkpoints) {
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fence;
+	}
+	for (i = 0; i < num_fences; i++) {
+		/*
+		 * Only return the checkpoint if the fence is still active.
+		 * Don't checked for signalled on PDUMP drivers as we need
+		 * to make sure that all fences make it to the pdump.
+		 */
+#if !defined(PDUMP)
+		if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			      &fences[i]->flags))
+#endif
+		{
+			struct pvr_fence *pvr_fence;
+
+			/* Check if fences[i] is an export fence */
+			if (pvr_is_exp_fence(fences[i])) {
+				/* Assign export fence a sync checkpoint if it does not already
+				 * have one.
+				 */
+				err = pvr_exp_fence_assign_checkpoint(PVRSRV_NO_FENCE,
+				                                      fences[i],
+				                                      psSyncCheckpointContext,
+				                                      &checkpoints[num_used_fences]);
+				SyncCheckpointTakeRef(checkpoints[num_used_fences]);
+				++num_used_fences;
+			}
+			else {
+				pvr_fence = pvr_fence_create_from_fence(
+				            pvr_sync_data.foreign_fence_context,
+			            psSyncCheckpointContext,
+			            fences[i],
+			            fence_to_resolve,
+			            "foreign");
+				if (!pvr_fence) {
+					pr_err(FILE_NAME ": %s: Failed to create fence\n",
+					       __func__);
+					err = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto err_free_checkpoints;
+				}
+				checkpoints[num_used_fences] =
+				        pvr_fence_get_checkpoint(pvr_fence);
+				SyncCheckpointTakeRef(checkpoints[num_used_fences]);
+				++num_used_fences;
+				dma_fence_put(&pvr_fence->base);
+			}
+		}
+	}
+	/* If we don't return any checkpoints, delete the array because
+	 * the caller will not.
+	 */
+	if (num_used_fences == 0) {
+		kfree(checkpoints);
+		checkpoints = NULL;
+	}
+
+	*checkpoint_handles = checkpoints;
+	*nr_checkpoints = num_used_fences;
+	*fence_uid = OSGetCurrentClientProcessIDKM();
+	*fence_uid = (*fence_uid << 32) | (fence_to_resolve & U32_MAX);
+
+err_put_fence:
+	dma_fence_put(fence);
+err_out:
+	return err;
+
+err_free_checkpoints:
+	for (i = 0; i < num_used_fences; i++) {
+		if (checkpoints[i])
+			SyncCheckpointDropRef(checkpoints[i]);
+	}
+	kfree(checkpoints);
+	goto err_put_fence;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain the
+ * PSYNC_CHECKPOINT for a given export fence (PVRSRV_FENCE) passed to a kick
+ * function.
+ * If export_fence is not a valid export fence, (ie fops != &pvr_exp_fence_ops)
+ * this function will return (error).
+ *
+ * Input:  export_fence              The export fence to resolve
+ * Input:  checkpoint_context        The context in which to create the new
+ *                                   sync checkpoint for the export fence
+ * Output: checkpoint_handle         The returned PVRSRV_SYNC_CHECKPOINT.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_resolve_export_fence(PVRSRV_FENCE fence_to_resolve,
+			      PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+			      PSYNC_CHECKPOINT *checkpoint_handle)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_resolve);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_resolve);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (!pvr_is_exp_fence(fence)) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr export fence\n", __func__);
+		dma_fence_put(fence);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	err = pvr_exp_fence_assign_checkpoint(fence_to_resolve,
+					      fence,
+					      checkpoint_context,
+					      checkpoint_handle);
+	if (err != PVRSRV_OK) {
+		pr_err(FILE_NAME ": %s: Failed to assign export fence a sync checkpoint\n",
+		       __func__);
+	}
+
+	dma_fence_put(fence);
+
+err_out:
+	return err;
+}
+
+/*
+ * This is the function that kick code will call in order to rollback the
+ * PSYNC_CHECKPOINT assigned to an export fence (PVRSRV_FENCE) passed to a kick
+ * function.
+ * If export_fence is not a valid export fence, (ie fops != &pvr_exp_fence_ops)
+ * this function will return (error).
+ *
+ * Input:  fence_to_rollback              The export fence to rollback
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_rollback_export_fence(PVRSRV_FENCE fence_to_rollback)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_rollback);
+	if (!fence) {
+		pr_err("%s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_rollback);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_get_fence;
+	}
+
+	if (!pvr_is_exp_fence(fence)) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr export fence\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_is_exp_fence;
+	}
+
+	err = pvr_exp_fence_rollback(fence);
+	if (err != PVRSRV_OK) {
+		pr_err("%s: Failed to rollback export fence\n",
+		       __func__);
+	}
+
+err_is_exp_fence:
+	dma_fence_put(fence);
+
+err_get_fence:
+	return err;
+}
+
+/*
+ * This is the function that driver code will call in order to request the
+ * sync implementation to output debug information relating to any sync
+ * checkpoints it may have created which appear in the provided array of
+ * FW addresses of Unified Fence Objects (UFOs).
+ *
+ * Input: nr_ufos             The number of FW addresses provided in the
+ *                            vaddrs parameter.
+ * Input: vaddrs              The array of FW addresses of UFOs. The sync
+ *                            implementation should check each of these to
+ *                            see if any relate to sync checkpoints it has
+ *                            created and where they do output debug information
+ *                            pertaining to the native/fallback sync with
+ *                            which it is associated.
+ */
+static u32
+pvr_sync_dump_info_on_stalled_ufos(u32 nr_ufos, u32 *vaddrs)
+{
+	return pvr_fence_dump_info_on_stalled_ufos(pvr_sync_data.foreign_fence_context,
+						   nr_ufos,
+						   vaddrs);
+}
+
+#if defined(PDUMP)
+static enum PVRSRV_ERROR_TAG
+pvr_sync_fence_get_checkpoints(PVRSRV_FENCE fence_to_pdump, u32 *nr_checkpoints,
+				struct SYNC_CHECKPOINT_TAG ***checkpoint_handles)
+{
+	struct dma_fence **fences = NULL;
+	struct dma_fence *fence;
+	struct pvr_fence *pvr_fence;
+	struct SYNC_CHECKPOINT_TAG **checkpoints = NULL;
+	unsigned int i, num_fences, num_used_fences = 0;
+	enum PVRSRV_ERROR_TAG err;
+
+	if (fence_to_pdump < 0) {
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (!nr_checkpoints || !checkpoint_handles) {
+		pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
+			__func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	fence = sync_file_get_fence(fence_to_pdump);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_pdump);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
+
+		fences = array->fences;
+		num_fences = array->num_fences;
+	} else {
+		fences = &fence;
+		num_fences = 1;
+	}
+
+	checkpoints = kmalloc_array(num_fences, sizeof(*checkpoints),
+			      GFP_KERNEL);
+	if (!checkpoints) {
+		pr_err("pvr_sync_file: %s: Failed to alloc memory for returned list of sync checkpoints\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fence;
+	}
+
+	for (i = 0; i < num_fences; i++) {
+		if (is_pvr_fence(fences[i])) {
+			pvr_fence = to_pvr_fence(fences[i]);
+			if (pvr_fence) {
+				checkpoints[num_used_fences] = pvr_fence_get_checkpoint(pvr_fence);
+				++num_used_fences;
+			}
+		} else if (pvr_is_exp_fence(fences[i])) {
+			struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fences[i]);
+
+			checkpoints[num_used_fences] = pvr_exp_fence_get_checkpoint(pvr_exp_fence);
+			if (checkpoints[num_used_fences])
+				++num_used_fences;
+		} else
+			continue;
+	}
+
+	*checkpoint_handles = checkpoints;
+	*nr_checkpoints = num_used_fences;
+	err =  PVRSRV_OK;
+
+err_put_fence:
+	dma_fence_put(fence);
+err_out:
+	return err;
+}
+#endif
+
+int pvr_sync_api_rename(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sync_rename_ioctl_data *data = user_data;
+
+	data->szName[sizeof(data->szName) - 1] = '\0';
+	OSStringSafeCopy(timeline->name, data->szName, sizeof(timeline->name));
+	if (timeline->hw_fence_context)
+		OSStringSafeCopy(timeline->hw_fence_context->name, data->szName,
+			sizeof(timeline->hw_fence_context->name));
+
+	return 0;
+}
+
+int pvr_sync_api_force_sw_only(void *api_priv, void **api_priv_new)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	/* Already in SW mode? */
+	if (timeline->sw_fence_timeline)
+		return 0;
+
+	/* Create a sw_sync timeline with the old GPU timeline's name */
+	timeline->sw_fence_timeline = pvr_counting_fence_timeline_create(
+		timeline->name);
+	if (!timeline->sw_fence_timeline)
+		return -ENOMEM;
+
+	timeline->is_sw = true;
+
+	*api_priv_new = (void *)timeline;
+
+	return 0;
+}
+
+/* We simply treat an export-fence as a SW fence and tweak
+ * the timeline structure to flag it as an 'is_export' type.
+ */
+int pvr_sync_api_force_exp_only(void *api_priv, void *api_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	if (timeline->is_export) {
+		pr_err(FILE_NAME ": %s: Already marked export timeline\n", __func__);
+		return 0;
+	}
+
+	timeline->exp_fence_context = pvr_exp_fence_context_create("pvr_exp_fence_ctx", "pvr_sync");
+	if (!timeline->exp_fence_context)
+		return -ENOMEM;
+
+	timeline->is_export = true;
+
+	return 0;
+}
+
+int pvr_sync_api_create_export_fence(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_exp_fence_context *exp_fence_context = timeline->exp_fence_context;
+	pvr_exp_sync_create_fence_data_t *data = user_data;
+	struct sync_file *sync_file;
+	int fd;
+	struct dma_fence *fence;
+	int err;
+
+	if (data == NULL) {
+		pr_err(FILE_NAME ": %s: Unexpected NULL user_data\n", __func__);
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
+		       __func__, fd);
+		err = -EMFILE;
+		goto err_out;
+	}
+
+	fence = pvr_exp_fence_create(exp_fence_context, fd, &data->sync_pt_idx);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to create a dma fence (%d)\n",
+		       __func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync_file (%d)\n",
+			__func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	data->fence = fd;
+
+	fd_install(fd, sync_file->file);
+
+	return 0;
+
+err_put_fd:
+	put_unused_fd(fd);
+err_out:
+	return err;
+}
+
+int pvr_sync_api_sw_create_fence(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sw_sync_create_fence_data *data = user_data;
+	struct sync_file *sync_file;
+	int fd = get_unused_fd_flags(O_CLOEXEC);
+	struct dma_fence *fence;
+	int err;
+
+	if (fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
+		       __func__, fd);
+		err = -EMFILE;
+		goto err_out;
+	}
+
+	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, &data->sync_pt_idx);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
+		       __func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
+			__func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	data->fence = fd;
+
+	fd_install(fd, sync_file->file);
+
+	return 0;
+
+err_put_fd:
+	put_unused_fd(fd);
+err_out:
+	return err;
+}
+
+int pvr_sync_api_sw_inc(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sw_timeline_advance_data *data = user_data;
+	bool res;
+
+	res = pvr_counting_fence_timeline_inc(timeline->sw_fence_timeline, &data->sync_pt_idx);
+
+	/* pvr_counting_fence_timeline_inc won't allow sw timeline to be
+	 * advanced beyond the last defined point
+	 */
+	if (!res) {
+		pr_err("pvr_sync_file: attempt to advance SW timeline beyond last defined point\n");
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static void
+pvr_sync_debug_request_heading(void *data, u32 verbosity,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  "------[ Native Fence Sync: timelines ]------");
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_register_functions(void)
+{
+	/* Register the resolve fence and create fence functions with
+	 * sync_checkpoint.c
+	 * The pvr_fence context registers its own EventObject callback to
+	 * update sync status
+	 */
+	/* Initialise struct and register with sync_checkpoint.c */
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceResolve = pvr_sync_resolve_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceCreate = pvr_sync_create_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceDataRollback = pvr_sync_rollback_fence_data;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceFinalise = pvr_sync_finalise_fence;
+#if defined(NO_HARDWARE)
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = pvr_sync_nohw_signal_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWSignalExpFence = pvr_sync_nohw_signal_exp_fence;
+#else
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = NULL;
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWSignalExpFence = NULL;
+#endif
+	pvr_sync_data.sync_checkpoint_ops.pfnFreeCheckpointListMem =
+		pvr_sync_free_checkpoint_list_mem;
+	pvr_sync_data.sync_checkpoint_ops.pfnDumpInfoOnStalledUFOs =
+		pvr_sync_dump_info_on_stalled_ufos;
+	OSStringSafeCopy(pvr_sync_data.sync_checkpoint_ops.pszImplName, "pvr_sync_file",
+		SYNC_CHECKPOINT_IMPL_MAX_STRLEN);
+#if defined(PDUMP)
+	pvr_sync_data.sync_checkpoint_ops.pfnSyncFenceGetCheckpoints =
+		pvr_sync_fence_get_checkpoints;
+#endif
+	pvr_sync_data.sync_checkpoint_ops.pfnExportFenceResolve = pvr_sync_resolve_export_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnExportFenceRollback = pvr_sync_rollback_export_fence;
+
+	return SyncCheckpointRegisterFunctions(&pvr_sync_data.sync_checkpoint_ops);
+}
+
+int pvr_sync_init(void)
+{
+	int err;
+
+	pvr_sync_data.foreign_fence_context =
+			pvr_fence_foreign_context_create(
+					NativeSyncGetFenceStatusWq(),
+					"foreign_sync");
+	if (!pvr_sync_data.foreign_fence_context) {
+		pr_err(FILE_NAME ": %s: Failed to create foreign sync context\n",
+			__func__);
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+#if defined(NO_HARDWARE)
+	INIT_LIST_HEAD(&pvr_timeline_active_list);
+#endif
+
+	err = pvr_sync_ioctl_init();
+	if (err) {
+		pr_err(FILE_NAME ": %s: Failed to register pvr_sync device (%d)\n",
+		       __func__, err);
+		goto err_ioctl_init;
+	}
+
+	return 0;
+
+err_ioctl_init:
+	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
+	pvr_fence_cleanup();
+err_out:
+	return err;
+}
+
+void pvr_sync_deinit(void)
+{
+	pvr_sync_ioctl_deinit();
+	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
+	pvr_fence_cleanup();
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_device_init(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+	enum PVRSRV_ERROR_TAG error;
+
+	error = PVRSRVRegisterDeviceDbgRequestNotify(
+				&priv->sync_debug_notify_handle,
+				priv->dev_node,
+				pvr_sync_debug_request_heading,
+				DEBUG_REQUEST_LINUXFENCE,
+				NULL);
+	if (error != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(error));
+		goto err_out;
+	}
+
+	/* Register the foreign sync context debug notifier on each device */
+	error = pvr_fence_context_register_dbg(
+				&priv->sync_foreign_debug_notify_handle,
+				priv->dev_node,
+				pvr_sync_data.foreign_fence_context);
+	if (error != PVRSRV_OK) {
+		pr_err("%s: failed to register fence debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(error));
+		goto err_context_regdbg;
+	}
+
+#if defined(NO_HARDWARE)
+	INIT_LIST_HEAD(&pvr_timeline_active_list);
+#endif
+
+	return PVRSRV_OK;
+
+err_context_regdbg:
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
+err_out:
+	return error;
+}
+
+void pvr_sync_device_deinit(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_foreign_debug_notify_handle);
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_wait(void *fence, u32 timeout_in_ms)
+{
+	long timeout = msecs_to_jiffies(timeout_in_ms);
+	int err;
+
+	err = dma_fence_wait_timeout(fence, true, timeout);
+	/*
+	 * dma_fence_wait_timeout returns:
+	 * - the remaining timeout on success
+	 * - 0 on timeout
+	 * - -ERESTARTSYS if interrupted
+	 */
+	if (err > 0)
+		return PVRSRV_OK;
+	else if (err == 0)
+		return PVRSRV_ERROR_TIMEOUT;
+
+	return PVRSRV_ERROR_FAILED_DEPENDENCIES;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_release(void *fence)
+{
+	dma_fence_put(fence);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_get(int fence_fd, void **fence_out)
+{
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_fd);
+	if (fence == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	*fence_out = fence;
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_sync_sw_timeline_fence_create(struct _PVRSRV_DEVICE_NODE_ *pvrsrv_dev_node,
+				  int timeline_fd,
+				  const char *fence_name,
+				  int *fence_fd_out,
+				  u64 *sync_pt_idx)
+{
+	enum PVRSRV_ERROR_TAG srv_err;
+	struct pvr_sync_timeline *timeline;
+	struct dma_fence *fence = NULL;
+	struct sync_file *sync_file = NULL;
+	int fd;
+
+	(void)(pvrsrv_dev_node);
+
+	fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fd < 0)
+		return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+
+	timeline = pvr_sync_timeline_fget(timeline_fd);
+	if (!timeline) {
+		/* unrecognised timeline */
+		srv_err = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		goto err_put_fd;
+	}
+	if (!timeline->is_sw) {
+		pvr_sync_timeline_fput(timeline);
+		srv_err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_fd;
+	}
+
+	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, sync_pt_idx);
+	pvr_sync_timeline_fput(timeline);
+	if (!fence) {
+		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fd;
+	}
+
+	fd_install(fd, sync_file->file);
+
+	*fence_fd_out = fd;
+
+	return PVRSRV_OK;
+
+err_put_fd:
+	put_unused_fd(fd);
+	return srv_err;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_advance(void *timeline, u64 *sync_pt_idx)
+{
+	if (timeline == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	pvr_counting_fence_timeline_inc(timeline, sync_pt_idx);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_release(void *timeline)
+{
+	if (timeline == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	pvr_counting_fence_timeline_put(timeline);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_get(int timeline_fd,
+					   void **timeline_out)
+{
+	struct pvr_counting_fence_timeline *sw_timeline;
+	struct pvr_sync_timeline *timeline;
+
+	timeline = pvr_sync_timeline_fget(timeline_fd);
+	if (!timeline)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	sw_timeline =
+		pvr_counting_fence_timeline_get(timeline->sw_fence_timeline);
+	pvr_sync_timeline_fput(timeline);
+	if (!sw_timeline)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	*timeline_out = sw_timeline;
+
+	return PVRSRV_OK;
+}
+static void _dump_sync_point(struct dma_fence *fence,
+							  DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+							  void *dump_debug_file)
+{
+	const struct dma_fence_ops *fence_ops = fence->ops;
+	bool signaled = dma_fence_is_signaled(fence);
+	char time[16] = { '\0' };
+
+	fence_ops->timeline_value_str(fence, time, sizeof(time));
+
+	PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "<%p> Seq#=%llu TS=%s State=%s TLN=%s",
+					  fence,
+					  (u64) fence->seqno,
+					  time,
+					  (signaled) ? "Signalled" : "Active",
+					  fence_ops->get_timeline_name(fence));
+}
+
+static void _dump_fence(struct dma_fence *fence,
+			DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+			void *dump_debug_file)
+{
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *fence_array = to_dma_fence_array(fence);
+		int i;
+
+		if (fence_array) {
+			PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "Fence: [%p] Sync Points:\n",
+					  fence_array);
+
+			for (i = 0; i < fence_array->num_fences; i++)
+				_dump_sync_point(fence_array->fences[i],
+						 dump_debug_printf,
+						 dump_debug_file);
+		}
+
+	} else {
+		_dump_sync_point(fence, dump_debug_printf, dump_debug_file);
+	}
+}
+
+enum PVRSRV_ERROR_TAG
+sync_dump_fence(void *sw_fence_obj,
+		DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		void *dump_debug_file)
+{
+	struct dma_fence *fence = (struct dma_fence *) sw_fence_obj;
+
+	_dump_fence(fence, dump_debug_printf, dump_debug_file);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG
+sync_sw_dump_timeline(void *sw_timeline_obj,
+		      DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		      void *dump_debug_file)
+{
+	pvr_counting_fence_timeline_dump_timeline(sw_timeline_obj,
+						  dump_debug_printf,
+						  dump_debug_file);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.c
new file mode 100644
index 000000000000..0e1facde86cb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.c
@@ -0,0 +1,405 @@
+/*
+ * @File        pvr_sync_ioctl_common.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/slab.h>
+
+#include "pvr_drm.h"
+#include "pvr_sync_api.h"
+#include "pvr_sync_ioctl_common.h"
+
+#include "pvr_debug.h"
+#include "pvr_uaccess.h"
+
+/*
+ * The PVR Sync API is unusual in that some operations configure the
+ * timeline for use, and are no longer allowed once the timeline is
+ * in use. A locking mechanism, such as a read/write semaphore, would
+ * be one method of helping to ensure the API rules are followed, but
+ * this would add unnecessary overhead once the timeline has been
+ * configured, as read locks would continue to have to be taken after
+ * the timeline is in use. To avoid locks, two atomic variables are used,
+ * together with memory barriers. The in_setup variable indicates a "rename"
+ * or "force software only" ioctl is in progress. At most one of these two
+ * configuration ioctls can be in progress at any one time, and they can't
+ * overlap with any other Sync ioctl. The in_use variable indicates one
+ * of the other Sync ioctls has started. Once set, in_use stays set, and
+ * prevents any further configuration ioctls. Non-configuration ioctls
+ * are allowed to overlap.
+ * It is possible for a configuration and non-configuration ioctl to race,
+ * but at most one will be allowed to proceed, and perhaps neither.
+ * Given the intended usage of the API in user space, where the timeline
+ * is fully configured before being used, the race behaviour won't be
+ * an issue.
+ */
+
+struct pvr_sync_file_data {
+	atomic_t in_setup;
+	atomic_t in_use;
+	void *api_private;
+	bool is_sw;
+	bool is_export;
+};
+
+static bool pvr_sync_set_in_use(struct pvr_sync_file_data *fdata)
+{
+	if (atomic_read(&fdata->in_use) < 2) {
+		atomic_set(&fdata->in_use, 1);
+		/* Ensure in_use change is visible before in_setup is read */
+		smp_mb();
+		if (atomic_read(&fdata->in_setup) != 0)
+			return false;
+
+		atomic_set(&fdata->in_use, 2);
+	} else {
+		/* Ensure stale private data isn't read */
+		smp_rmb();
+	}
+
+	return true;
+}
+
+static bool pvr_sync_set_in_setup(struct pvr_sync_file_data *fdata)
+{
+	int in_setup;
+
+	in_setup = atomic_inc_return(&fdata->in_setup);
+	if (in_setup > 1 || atomic_read(&fdata->in_use) != 0) {
+		atomic_dec(&fdata->in_setup);
+		return false;
+	}
+
+	return true;
+}
+
+static inline void pvr_sync_reset_in_setup(struct pvr_sync_file_data *fdata)
+{
+	/*
+	 * Ensure setup changes are visible before allowing other
+	 * operations to proceed.
+	 */
+	smp_mb__before_atomic();
+	atomic_dec(&fdata->in_setup);
+}
+
+void *pvr_sync_get_api_priv_common(struct file *file)
+{
+	if (file != NULL && pvr_sync_is_timeline(file)) {
+		struct pvr_sync_file_data *fdata = pvr_sync_get_private_data(file);
+
+		if (fdata != NULL && pvr_sync_set_in_use(fdata))
+			return fdata->api_private;
+	}
+
+	return NULL;
+}
+
+int pvr_sync_open_common(void *connection_data, void *file_handle)
+{
+	void *data = NULL;
+	struct pvr_sync_file_data *fdata;
+	int err;
+
+	fdata = kzalloc(sizeof(*fdata), GFP_KERNEL);
+	if (!fdata)
+		return -ENOMEM;
+
+	atomic_set(&fdata->in_setup, 0);
+	atomic_set(&fdata->in_use, 0);
+
+	if (!pvr_sync_set_private_data(connection_data, fdata)) {
+		kfree(fdata);
+		return -EINVAL;
+	}
+
+	err = pvr_sync_api_init(file_handle, &data);
+	if (err)
+		kfree(fdata);
+	else
+		fdata->api_private = data;
+
+	return err;
+}
+
+int pvr_sync_close_common(void *connection_data)
+{
+	struct pvr_sync_file_data *fdata;
+
+	fdata = pvr_sync_connection_private_data(connection_data);
+	if (fdata) {
+		int err;
+
+		err = pvr_sync_api_deinit(fdata->api_private, fdata->is_sw);
+
+		kfree(fdata);
+
+		return err;
+	}
+
+	return 0;
+}
+
+#define PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(name, type) \
+static inline int pvr_sync_ioctl_common_internal_ ## name(struct pvr_sync_file_data *fdata, type data)
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(rename, struct pvr_sync_rename_ioctl_data *)
+{
+	return pvr_sync_api_rename(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(force_sw_only, void*)
+{
+	void *new_api_private = fdata->api_private;
+	int err;
+
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	err = pvr_sync_api_force_sw_only(fdata->api_private, &new_api_private);
+	if (!err) {
+		if (new_api_private != fdata->api_private)
+			fdata->api_private = new_api_private;
+
+		fdata->is_sw = true;
+	}
+
+	return err;
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(force_exp_only, void*)
+{
+	int err;
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	err = pvr_sync_api_force_exp_only(fdata->api_private, data);
+	if (!err)
+		fdata->is_export = true;
+
+	return err;
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(create_export_fence, void*)
+{
+	return pvr_sync_api_create_export_fence(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(sw_create_fence, struct pvr_sw_sync_create_fence_data *)
+{
+	return pvr_sync_api_sw_create_fence(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(sw_inc, struct pvr_sw_timeline_advance_data *)
+{
+	return pvr_sync_api_sw_inc(fdata->api_private, data);
+}
+
+/*
+ * enum pvr_sync_ioctl_dispatch_type
+ * @pvr_sync_ioctl_dispatch_type_setup: The command may only be used during the setup phase
+ * @pvr_sync_ioctl_dispatch_type_export: The command may only be used with an export only timeline.
+ * @pvr_sync_ioctl_dispatch_type_software: The command may only be used with a software only timeline.
+ */
+enum pvr_sync_ioctl_dispatch_type {
+	pvr_sync_ioctl_dispatch_type_setup,
+	pvr_sync_ioctl_dispatch_type_export,
+	pvr_sync_ioctl_dispatch_type_software,
+};
+
+/* Generates a function `from` which performs validation on the data before passing it to `into` */
+#define PVR_SYNC_IOCTL_DISPATCH_VALIDATE(name, structure, type)                                                                        \
+PVR_SYNC_IOCTL_DISPATCH_DECL(name)                                                                                                     \
+{                                                                                                                                      \
+	int err = -ENOTTY;                                                                                                                 \
+	structure server_data;                                                                                                             \
+                                                                                                                                       \
+	struct pvr_sync_file_data *fdata = pvr_sync_get_private_data(file);                                                                \
+	if (unlikely(!fdata))                                                                                                              \
+		return -EINVAL;                                                                                                                \
+                                                                                                                                       \
+	/* Check if the device is busy and the operation is valid for the timelines current state */                                       \
+	if (type == pvr_sync_ioctl_dispatch_type_setup)                                                                                    \
+	{                                                                                                                                  \
+		if (!pvr_sync_set_in_setup(fdata))                                                                                             \
+			return -EBUSY;                                                                                                             \
+		if (fdata->is_sw || fdata->is_export)                                                                                          \
+			goto return_;                                                                                                              \
+	}                                                                                                                                  \
+	else                                                                                                                               \
+	{                                                                                                                                  \
+		if (!pvr_sync_set_in_use(fdata))                                                                                               \
+			return -EBUSY;                                                                                                             \
+                                                                                                                                       \
+		switch (type)                                                                                                                  \
+		{                                                                                                                              \
+		case pvr_sync_ioctl_dispatch_type_software: {                                                                                  \
+			if (!fdata->is_sw) /* Not a software timeline, software operations cannot be used */                                       \
+				goto return_;                                                                                                          \
+		}                                                                                                                              \
+		break;                                                                                                                         \
+		case pvr_sync_ioctl_dispatch_type_export: {                                                                                    \
+			if (!fdata->is_export) /* Not an export timeline, export operations cannot be used */                                      \
+				goto return_;                                                                                                          \
+		}                                                                                                                              \
+		break;                                                                                                                         \
+		default:                                                                                                                       \
+			goto return_; /* Invalid Type */                                                                                           \
+		}                                                                                                                              \
+	}                                                                                                                                  \
+                                                                                                                                       \
+	/* copy_from_user */                                                                                                               \
+	err = pvr_sync_ioctl_dispatch_copy_in__##name((structure __user *)user_data, &server_data);                                        \
+	if (unlikely(err))                                                                                                                 \
+		goto return_;                                                                                                                  \
+                                                                                                                                       \
+	/* Continue into api */                                                                                                            \
+	err = pvr_sync_ioctl_common_internal_ ## name(fdata, (structure __force *) PVR_SYNC_IOCTL_DISPATCH_DATA(user_data, &server_data)); \
+                                                                                                                                       \
+	if (likely(!err))                                                                                                                  \
+	{                                                                                                                                  \
+		/* copy_to_user */                                                                                                             \
+		err = pvr_sync_ioctl_dispatch_copy_out__##name((structure __user *)user_data, &server_data);                                   \
+	}                                                                                                                                  \
+                                                                                                                                       \
+return_:                                                                                                                               \
+	if (type == pvr_sync_ioctl_dispatch_type_setup)                                                                                    \
+		pvr_sync_reset_in_setup(fdata);                                                                                                \
+	return err;                                                                                                                        \
+}
+
+#if !defined(USE_PVRSYNC_DEVNODE)
+/* drm_ioctl() already copies the data over, see comment on drm_ioctl_t */
+#define PVR_SYNC_IOCTL_DISPATCH_DATA(pUM, pKM) pUM
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(dir, name, structure, copy) \
+INLINE static int pvr_sync_ioctl_dispatch_copy_ ## dir ## __ ## name (structure __user *pUM, structure *pKM) \
+{ return 0; }
+#else /* !defined(USE_PVRSYNC_DEVNODE) */
+/* Generates a function to copy over the arguments to/from user-mode */
+#define PVR_SYNC_IOCTL_DISPATCH_DATA(pUM, pKM) pKM
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(dir, name, structure, copy)                                     \
+INLINE static int pvr_sync_ioctl_dispatch_copy_ ## dir ## __ ## name (structure __user *pUM, structure *pKM) \
+{                                                                                                            \
+	/* May be unused if there are no in/out args */                                                          \
+	PVR_UNREFERENCED_PARAMETER(pUM);                                                                         \
+	PVR_UNREFERENCED_PARAMETER(pKM);                                                                         \
+	/* Copy over the data */                                                                                 \
+	{ copy }                                                                                                 \
+	return 0;                                                                                                \
+}
+#endif /* !defined(USE_PVRSYNC_DEVNODE) */
+
+/*************************************************************************/ /*!
+@Function       PVR_SYNC_IOCTL_DISPATCH_COPY
+@Description    Generates the code to copy from/to user mode depending on @dir.
+@Input          dir  Either `from` or `to` corresponding to `copy_from_user`
+                    and `copy_to_user` respectively.
+@Input          to   Pointer for the dest.
+@Input          from Pointer for the src.
+*/ /**************************************************************************/
+#define PVR_SYNC_IOCTL_DISPATCH_COPY(dir, to, from)        \
+if (pvr_copy_##dir##_user(to, from, sizeof(*pKM)))         \
+{                                                          \
+	PVR_DPF((PVR_DBG_ERROR, "Failed copy " #dir " user")); \
+	return -EFAULT;                                        \
+}
+
+/* Copy data from user */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_IN \
+PVR_SYNC_IOCTL_DISPATCH_COPY(from, pKM, pUM)
+
+/* Copy data to user */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_OUT \
+PVR_SYNC_IOCTL_DISPATCH_COPY(to, pUM, pKM)
+
+/* Copy no data */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_NONE
+
+/*************************************************************************/ /*!
+@Function       PVR_SYNC_IOCTL_DISPATCH_FUNCTION
+@Description    Generates a dispatch function which validates the ioctl command
+                and dispatches it to the internal an function.
+                Generates a dispatch function
+                `pvr_sync_ioctl_common_##name(struct file*, void*)`
+                which validates the ioctl command and dispatches it to
+                an internal function
+                `pvr_sync_ioctl_common_internal_##name(struct pvr_sync_file_data*, structure*)`
+@Input          name      A name for the dispatch functions, must be unique.
+@Input          structure The type of the user_data.
+@Input          type      The type specifies when the ioctls are allowed or
+                          if the ioctl is allowed on the current
+                          timeline configuration.
+@Input          copy_in   Either `COPY_IN` or `COPY_NONE`
+@Input          copy_out  Either `COPY_OUT` or `COPY_NONE`
+*/ /**************************************************************************/
+#define PVR_SYNC_IOCTL_DISPATCH_FUNCTION(name, structure, type, copy_in, copy_out) \
+	PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(in, name, structure, PVR_SYNC_IOCTL_DISPATCH_ ## copy_in) \
+	PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(out, name, structure, PVR_SYNC_IOCTL_DISPATCH_ ## copy_out) \
+	PVR_SYNC_IOCTL_DISPATCH_VALIDATE(name, structure, pvr_sync_ioctl_dispatch_type_ ## type)
+
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(rename, struct pvr_sync_rename_ioctl_data,
+                                 setup,
+                                 COPY_IN,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(force_sw_only, void*,
+                                 setup,
+                                 COPY_NONE,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(force_exp_only, void*,
+                                 setup,
+                                 COPY_NONE,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(sw_create_fence, struct pvr_sw_sync_create_fence_data,
+                                 software,
+                                 COPY_IN,
+                                 COPY_OUT);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(create_export_fence, pvr_exp_sync_create_fence_data_t,
+                                 export,
+                                 COPY_IN,
+                                 COPY_OUT);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(sw_inc, struct pvr_sw_timeline_advance_data,
+                                 software,
+                                 COPY_NONE,
+                                 COPY_OUT);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.h
new file mode 100644
index 000000000000..0959e7673164
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_common.h
@@ -0,0 +1,85 @@
+/*
+ * @File        pvr_sync_ioctl_common.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_IOCTL_COMMON_H
+#define _PVR_SYNC_IOCTL_COMMON_H
+
+struct file;
+
+/* Functions provided by pvr_sync_ioctl_common.c */
+
+int pvr_sync_open_common(void *connection_data, void *file_handle);
+int pvr_sync_close_common(void *connection_data);
+void *pvr_sync_get_api_priv_common(struct file *file);
+
+#if defined(USE_PVRSYNC_DEVNODE)
+#define PVR_SYNC_IOCTL_DISPATCH_DECL(name) \
+	int pvr_sync_ioctl_common_ ## name(struct file *file, void __user *user_data)
+#else /* defined(USE_PVRSYNC_DEVNODE) */
+#define PVR_SYNC_IOCTL_DISPATCH_DECL(name) \
+	int pvr_sync_ioctl_common_ ## name(struct file *file, void *user_data)
+#endif /* defined(USE_PVRSYNC_DEVNODE) */
+
+PVR_SYNC_IOCTL_DISPATCH_DECL(rename);
+PVR_SYNC_IOCTL_DISPATCH_DECL(force_sw_only);
+PVR_SYNC_IOCTL_DISPATCH_DECL(force_exp_only);
+PVR_SYNC_IOCTL_DISPATCH_DECL(sw_create_fence);
+PVR_SYNC_IOCTL_DISPATCH_DECL(create_export_fence);
+PVR_SYNC_IOCTL_DISPATCH_DECL(sw_inc);
+
+struct pvr_sync_file_data;
+
+/* Functions required by pvr_sync_ioctl_common.c */
+
+bool pvr_sync_set_private_data(void *connection_data,
+			       struct pvr_sync_file_data *fdata);
+
+struct pvr_sync_file_data *
+pvr_sync_connection_private_data(void *connection_data);
+
+struct pvr_sync_file_data *
+pvr_sync_get_private_data(struct file *file);
+
+bool pvr_sync_is_timeline(struct file *file);
+
+#endif /* _PVR_SYNC_IOCTL_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.c
new file mode 100644
index 000000000000..313393d09c1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.c
@@ -0,0 +1,184 @@
+/*
+ * @File        pvr_sync_ioctl_drm.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "pvr_drv.h"
+#include "pvr_drm.h"
+#include "private_data.h"
+#include "env_connection.h"
+#include "pvr_sync_api.h"
+#include "pvr_sync_ioctl_common.h"
+#include "pvr_sync_ioctl_drm.h"
+
+bool pvr_sync_set_private_data(void *connection_data,
+			       struct pvr_sync_file_data *fdata)
+{
+	if (connection_data) {
+		ENV_CONNECTION_DATA *env_data;
+
+		env_data = PVRSRVConnectionPrivateData(connection_data);
+		if (env_data) {
+			env_data->pvPvrSyncPrivateData = fdata;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+struct pvr_sync_file_data *
+pvr_sync_connection_private_data(void *connection_data)
+{
+	if (connection_data) {
+		ENV_CONNECTION_DATA *env_data;
+
+		env_data = PVRSRVConnectionPrivateData(connection_data);
+
+		if (env_data)
+			return env_data->pvPvrSyncPrivateData;
+	}
+
+	return NULL;
+}
+
+struct pvr_sync_file_data *
+pvr_sync_get_private_data(struct file *file)
+{
+	CONNECTION_DATA *connection_data = LinuxSyncConnectionFromFile(file);
+
+	return pvr_sync_connection_private_data(connection_data);
+}
+
+bool pvr_sync_is_timeline(struct file *file)
+{
+	return file->f_op == &pvr_drm_fops;
+}
+
+void *pvr_sync_get_api_priv(struct file *file)
+{
+	return pvr_sync_get_api_priv_common(file);
+}
+
+struct file *pvr_sync_get_file_struct(void *file_handle)
+{
+	if (file_handle) {
+		struct drm_file *file = file_handle;
+
+		return file->filp;
+	}
+
+	return NULL;
+}
+
+int pvr_sync_open(void *connection_data, struct drm_file *file)
+{
+	/*
+	 * The file structure pointer (file->filp) may not have been
+	 * initialised at this point, so pass down a pointer to the
+	 * drm_file structure instead.
+	 */
+	return pvr_sync_open_common(connection_data, file);
+}
+
+void pvr_sync_close(void *connection_data)
+{
+	int iErr = pvr_sync_close_common(connection_data);
+
+	if (iErr < 0)
+		pr_err("%s: ERROR (%d) returned by pvr_sync_close_common()\n",
+		       __func__, iErr);
+
+	iErr = OSConnectionPrivateDataDeInit(PVRSRVConnectionPrivateData(connection_data));
+	if (iErr != 0)
+		pr_err("%s: ERROR (%d) returned by OSConnectionPrivateDataDeInit()\n",
+		       __func__, iErr);
+
+	if (connection_data)
+		kfree(connection_data);
+}
+
+
+int pvr_sync_rename_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_rename(file->filp, arg);
+}
+
+int pvr_sync_force_sw_only_ioctl(struct drm_device __maybe_unused *dev,
+				 void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_force_sw_only(file->filp, arg);
+}
+
+int pvr_sw_sync_create_fence_ioctl(struct drm_device __maybe_unused *dev,
+				   void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_sw_create_fence(file->filp, arg);
+}
+
+int pvr_sw_sync_inc_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_sw_inc(file->filp, arg);
+}
+
+int pvr_sync_ioctl_force_exp_only(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_force_exp_only(file->filp, arg);
+}
+
+int pvr_export_fence_sync_create_fence_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_create_export_fence(file->filp, arg);
+}
+
+int pvr_sync_ioctl_init(void)
+{
+	return 0;
+}
+
+void pvr_sync_ioctl_deinit(void)
+{
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.h
new file mode 100644
index 000000000000..93182205ca0b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_sync_ioctl_drm.h
@@ -0,0 +1,66 @@
+/*
+ * @File        pvr_sync_ioctl_drm.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_IOCTL_DRM_H
+#define _PVR_SYNC_IOCTL_DRM_H
+
+struct drm_device;
+struct drm_file;
+
+int pvr_sync_open(void *connection_data, struct drm_file *file);
+void pvr_sync_close(void *connection_data);
+
+int pvr_sync_rename_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_sync_force_sw_only_ioctl(struct drm_device *dev, void *arg,
+				 struct drm_file *file);
+int pvr_sw_sync_create_fence_ioctl(struct drm_device *dev, void *arg,
+				   struct drm_file *file);
+int pvr_sw_sync_inc_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_sync_ioctl_force_exp_only(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_export_fence_sync_create_fence_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+
+#endif /* _PVR_SYNC_IOCTL_DRM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_uaccess.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_uaccess.h
new file mode 100644
index 000000000000..13864eab4c7a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/pvr_uaccess.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Title          Utility functions for user space access
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_UACCESS_H
+#define PVR_UACCESS_H
+
+#include <linux/uaccess.h>
+#include <linux/version.h>
+
+static inline unsigned long pvr_copy_to_user(void __user *pvTo, const void *pvFrom, unsigned long ulBytes)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+	if (access_ok(VERIFY_WRITE, pvTo, ulBytes))
+#else
+	if (access_ok(pvTo, ulBytes))
+#endif
+	{
+		return __copy_to_user(pvTo, pvFrom, ulBytes);
+	}
+
+	return ulBytes;
+}
+
+
+#if defined(__KLOCWORK__)
+	/* this part is only to tell Klocwork not to report false positive because
+	   it doesn't understand that pvr_copy_from_user will initialise the memory
+	   pointed to by pvTo */
+#include <linux/string.h> /* get the memset prototype */
+static inline unsigned long pvr_copy_from_user(void *pvTo, const void __user *pvFrom, unsigned long ulBytes)
+{
+	if (pvTo != NULL)
+	{
+		memset(pvTo, 0xAA, ulBytes);
+		return 0;
+	}
+	return 1;
+}
+
+#else /* real implementation */
+
+static inline unsigned long pvr_copy_from_user(void *pvTo, const void __user *pvFrom, unsigned long ulBytes)
+{
+	/*
+	 * The compile time correctness checking introduced for copy_from_user in
+	 * Linux 2.6.33 isn't fully compatible with our usage of the function.
+	 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+	if (access_ok(VERIFY_READ, pvFrom, ulBytes))
+#else
+	if (access_ok(pvFrom, ulBytes))
+#endif
+	{
+		return __copy_from_user(pvTo, pvFrom, ulBytes);
+	}
+
+	return ulBytes;
+}
+#endif /* klocworks */
+
+#endif /* PVR_UACCESS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/rogue_trace_events.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/rogue_trace_events.h
new file mode 100644
index 000000000000..8952bec0f102
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/rogue_trace_events.h
@@ -0,0 +1,592 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rogue
+
+#if !defined(ROGUE_TRACE_EVENTS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define ROGUE_TRACE_EVENTS_H
+
+#include <linux/version.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+#include <linux/time.h>
+
+#define show_secs_from_ns(ns) \
+	({ \
+		u64 t = ns + (NSEC_PER_USEC / 2); \
+		do_div(t, NSEC_PER_SEC); \
+		t; \
+	})
+
+#define show_usecs_from_ns(ns) \
+	({ \
+		u64 t = ns + (NSEC_PER_USEC / 2); \
+		do_div(t, NSEC_PER_USEC); \
+		do_div(t, USEC_PER_SEC); \
+	})
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_update_enabled_callback(void);
+#else
+void trace_fence_update_enabled_callback(void);
+#endif
+void trace_fence_update_disabled_callback(void);
+
+TRACE_EVENT_FN(rogue_fence_update,
+
+	TP_PROTO(const char *comm, const char *cmd, const char *dm, u32 gpu_id, u32 ctx_id, u32 offset,
+		u32 sync_fwaddr, u32 sync_value),
+
+	TP_ARGS(comm, cmd, dm, gpu_id, ctx_id, offset, sync_fwaddr, sync_value),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       cmd,            cmd             )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+		__field(        u32,            offset          )
+		__field(        u32,            sync_fwaddr     )
+		__field(        u32,            sync_value      )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm, comm);
+		__assign_str(cmd, cmd);
+		__assign_str(dm, dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->offset = offset;
+		__entry->sync_fwaddr = sync_fwaddr;
+		__entry->sync_value = sync_value;
+	),
+
+	TP_printk("comm=%s cmd=%s dm=%s gpu=%lu ctx_id=%lu offset=%lu sync_fwaddr=%#lx sync_value=%#lx",
+		__get_str(comm),
+		__get_str(cmd),
+		__get_str(dm),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->offset,
+		(unsigned long) __entry->sync_fwaddr,
+		(unsigned long) __entry->sync_value
+	),
+
+	trace_fence_update_enabled_callback,
+	trace_fence_update_disabled_callback
+);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_check_enabled_callback(void);
+#else
+void trace_fence_check_enabled_callback(void);
+#endif
+void trace_fence_check_disabled_callback(void);
+
+TRACE_EVENT_FN(rogue_fence_check,
+
+	TP_PROTO(const char *comm, const char *cmd, const char *dm, u32 gpu_id, u32 ctx_id, u32 offset,
+		u32 sync_fwaddr, u32 sync_value),
+
+	TP_ARGS(comm, cmd, dm, gpu_id, ctx_id, offset, sync_fwaddr, sync_value),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       cmd,            cmd             )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+		__field(        u32,            offset          )
+		__field(        u32,            sync_fwaddr     )
+		__field(        u32,            sync_value      )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm, comm);
+		__assign_str(cmd, cmd);
+		__assign_str(dm, dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->offset = offset;
+		__entry->sync_fwaddr = sync_fwaddr;
+		__entry->sync_value = sync_value;
+	),
+
+	TP_printk("comm=%s cmd=%s dm=%s gpu=%lu ctx_id=%lu offset=%lu sync_fwaddr=%#lx sync_value=%#lx",
+		__get_str(comm),
+		__get_str(cmd),
+		__get_str(dm),
+		(unsigned long)__entry->gpu_id,
+		(unsigned long)__entry->ctx_id,
+		(unsigned long)__entry->offset,
+		(unsigned long)__entry->sync_fwaddr,
+		(unsigned long)__entry->sync_value
+	),
+
+	trace_fence_check_enabled_callback,
+	trace_fence_check_disabled_callback
+);
+
+TRACE_EVENT(rogue_job_enqueue,
+
+	TP_PROTO(u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         const char *kick_type),
+
+	TP_ARGS(gpu_id, ctx_id, int_id, ext_id, kick_type),
+
+	TP_STRUCT__entry(
+		__field(u32, gpu_id)
+		__field(u32, ctx_id)
+		__field(u32, int_id)
+		__field(u32, ext_id)
+		__string(kick_type, kick_type)
+	),
+
+	TP_fast_assign(
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__assign_str(kick_type, kick_type);
+	),
+
+	TP_printk("gpu=%lu, ctx_id=%lu int_id=%lu ext_id=%lu kick_type=%s",
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		__get_str(kick_type)
+	)
+);
+
+TRACE_EVENT(rogue_sched_switch,
+
+	TP_PROTO(const char *work_type, u32 switch_type, u64 timestamp, u32 gpu_id, u32 next_ctx_id,
+	         u32 next_prio, u32 next_int_id, u32 next_ext_id),
+
+	TP_ARGS(work_type, switch_type, timestamp, gpu_id, next_ctx_id, next_prio, next_int_id,
+	        next_ext_id),
+
+	TP_STRUCT__entry(
+		__string(work_type, work_type)
+		__field(u32, switch_type)
+		__field(u64, timestamp)
+		__field(u32, gpu_id)
+		__field(u32, next_ctx_id)
+		__field(u32, next_prio)
+		__field(u32, next_int_id)
+		__field(u32, next_ext_id)
+	),
+
+	TP_fast_assign(
+		__assign_str(work_type, work_type);
+		__entry->switch_type = switch_type;
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->next_ctx_id = next_ctx_id;
+		__entry->next_prio = next_prio;
+		__entry->next_int_id = next_int_id;
+		__entry->next_ext_id = next_ext_id;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu next_ctx_id=%lu next_int_id=%lu next_ext_id=%lu"
+		" next_prio=%lu work_type=%s switch_type=%s",
+		(unsigned long long) show_secs_from_ns(__entry->timestamp),
+		(unsigned long) show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->next_ctx_id,
+		(unsigned long) __entry->next_int_id,
+		(unsigned long) __entry->next_ext_id,
+		(unsigned long) __entry->next_prio,
+		__get_str(work_type),
+		__print_symbolic(__entry->switch_type,
+			/* These values are from ospvr_gputrace.h. */
+			{ 1, "begin" },
+			{ 2, "end" })
+	)
+);
+
+TRACE_EVENT(rogue_create_fw_context,
+
+	TP_PROTO(const char *comm, const char *dm, u32 gpu_id, u32 ctx_id),
+
+	TP_ARGS(comm, dm, gpu_id, ctx_id),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm, comm);
+		__assign_str(dm, dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+	),
+
+	TP_printk("comm=%s dm=%s gpu=%lu ctx_id=%lu",
+		__get_str(comm),
+		__get_str(dm),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id
+	)
+);
+
+void PVRGpuTraceEnableUfoCallback(void);
+void PVRGpuTraceDisableUfoCallback(void);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableUfoCallbackWrapper(void);
+#else
+#define PVRGpuTraceEnableUfoCallbackWrapper \
+		PVRGpuTraceEnableUfoCallback
+#endif
+
+TRACE_EVENT_FN(rogue_ufo_update,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 old_value, u32 new_value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, old_value,
+	        new_value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            old_value   )
+		__field(        u32,            new_value   )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->old_value = old_value;
+		__entry->new_value = new_value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx old_value=%#lx new_value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->old_value,
+		(unsigned long) __entry->new_value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_check_fail,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value, u32 required),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value, required),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+		__field(        u32,            required    )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+		__entry->required = required;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx required=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value,
+		(unsigned long) __entry->required
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_pr_check_fail,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value, u32 required),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value, required),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+		__field(        u32,            required    )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+		__entry->required = required;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx required=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value,
+		(unsigned long) __entry->required
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_check_success,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_pr_check_success,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT(rogue_events_lost,
+
+	TP_PROTO(u32 event_source, u32 gpu_id, u32 last_ordinal, u32 curr_ordinal),
+
+	TP_ARGS(event_source, gpu_id, last_ordinal, curr_ordinal),
+
+	TP_STRUCT__entry(
+		__field(        u32,            event_source     )
+		__field(        u32,            gpu_id           )
+		__field(        u32,            last_ordinal     )
+		__field(        u32,            curr_ordinal     )
+	),
+
+	TP_fast_assign(
+		__entry->event_source = event_source;
+		__entry->gpu_id = gpu_id;
+		__entry->last_ordinal = last_ordinal;
+		__entry->curr_ordinal = curr_ordinal;
+	),
+
+	TP_printk("event_source=%s gpu=%u last_ordinal=%u curr_ordinal=%u",
+		__print_symbolic(__entry->event_source, {0, "GPU"}, {1, "Host"}),
+		__entry->gpu_id,
+		__entry->last_ordinal,
+		__entry->curr_ordinal)
+);
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void);
+void PVRGpuTraceDisableFirmwareActivityCallback(void);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableFirmwareActivityCallbackWrapper(void);
+#else
+#define PVRGpuTraceEnableFirmwareActivityCallbackWrapper \
+		PVRGpuTraceEnableFirmwareActivityCallback
+#endif
+
+TRACE_EVENT_FN(rogue_firmware_activity,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, const char *task, u32 fw_event),
+
+	TP_ARGS(timestamp, gpu_id, task, fw_event),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp       )
+		__field(        u32,            gpu_id             )
+		__string(       task,           task            )
+		__field(        u32,            fw_event        )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__assign_str(task, task);
+		__entry->fw_event = fw_event;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu task=%s event=%s",
+		(unsigned long long) show_secs_from_ns(__entry->timestamp),
+		(unsigned long) show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		__get_str(task),
+		__print_symbolic(__entry->fw_event,
+			/* These values are from ospvr_gputrace.h. */
+			{ 1, "begin" },
+			{ 2, "end" })),
+
+	PVRGpuTraceEnableFirmwareActivityCallbackWrapper,
+	PVRGpuTraceDisableFirmwareActivityCallback
+);
+
+#undef show_secs_from_ns
+#undef show_usecs_from_ns
+
+#endif /* ROGUE_TRACE_EVENTS_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+
+/* This is needed because the name of this file doesn't match TRACE_SYSTEM. */
+#define TRACE_INCLUDE_FILE rogue_trace_events
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/services_kernel_client.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/services_kernel_client.h
new file mode 100644
index 000000000000..4ec5356abbf0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/services_kernel_client.h
@@ -0,0 +1,308 @@
+/*************************************************************************/ /*!
+@File           services_kernel_client.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* This file contains a partial redefinition of the PowerVR Services 5
+ * interface for use by components which are checkpatch clean. This
+ * header is included by the unrefined, non-checkpatch clean headers
+ * to ensure that prototype/typedef/macro changes break the build.
+ */
+
+#ifndef __SERVICES_KERNEL_CLIENT__
+#define __SERVICES_KERNEL_CLIENT__
+
+#include "pvrsrv_error.h"
+
+#include <linux/types.h>
+
+#include "pvrsrv_sync_km.h"
+#include "sync_checkpoint_external.h"
+
+/* included for the define PVRSRV_LINUX_DEV_INIT_ON_PROBE */
+#include "pvr_drm.h"
+
+#include "pvr_dma_resv.h"
+
+#ifndef __pvrsrv_defined_struct_enum__
+
+/* sync_external.h */
+
+struct PVRSRV_CLIENT_SYNC_PRIM_TAG {
+	volatile __u32 *pui32LinAddr;
+};
+
+struct PVRSRV_CLIENT_SYNC_PRIM_OP {
+	__u32 ui32Flags;
+	struct pvrsrv_sync_prim *psSync;
+	__u32 ui32FenceValue;
+	__u32 ui32UpdateValue;
+};
+
+#else /* __pvrsrv_defined_struct_enum__ */
+
+struct PVRSRV_CLIENT_SYNC_PRIM_TAG;
+struct PVRSRV_CLIENT_SYNC_PRIM_OP;
+
+#endif /* __pvrsrv_defined_struct_enum__ */
+
+struct _PMR_;
+struct _PVRSRV_DEVICE_NODE_;
+struct dma_buf;
+struct dma_resv;
+struct SYNC_PRIM_CONTEXT_TAG;
+
+/* pvr_notifier.h */
+
+#ifndef CMDCOMPNOTIFY_PFN
+typedef void (*PFN_CMDCOMP_NOTIFY)(void *hCmdCompHandle);
+#define CMDCOMPNOTIFY_PFN
+#endif
+enum PVRSRV_ERROR_TAG PVRSRVRegisterCmdCompleteNotify(void **phNotify,
+	PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify, void *hPrivData);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterCmdCompleteNotify(void *hNotify);
+void PVRSRVCheckStatus(void *hCmdCompCallerHandle);
+
+#define DEBUG_REQUEST_DC               0
+#define DEBUG_REQUEST_SYNCTRACKING     1
+#define DEBUG_REQUEST_SRV              2
+#define DEBUG_REQUEST_SYS              3
+#define DEBUG_REQUEST_RGX              4
+#define DEBUG_REQUEST_ANDROIDSYNC      5
+#define DEBUG_REQUEST_LINUXFENCE       6
+#define DEBUG_REQUEST_SYNCCHECKPOINT   7
+#define DEBUG_REQUEST_HTB              8
+#define DEBUG_REQUEST_APPHINT          9
+#define DEBUG_REQUEST_FALLBACKSYNC     10
+
+#define DEBUG_REQUEST_VERBOSITY_LOW    0
+#define DEBUG_REQUEST_VERBOSITY_MEDIUM 1
+#define DEBUG_REQUEST_VERBOSITY_HIGH   2
+#define DEBUG_REQUEST_VERBOSITY_MAX    DEBUG_REQUEST_VERBOSITY_HIGH
+
+#define DD_VERB_LVL_ENABLED(_verbLvl, _verbLvlChk) ((_verbLvl) >= (_verbLvlChk))
+
+#ifndef DBGNOTIFY_PFNS
+typedef void (DUMPDEBUG_PRINTF_FUNC)(void *pvDumpDebugFile,
+	const char *fmt, ...) __printf(2, 3);
+typedef void (*PFN_DBGREQ_NOTIFY) (void *hDebugRequestHandle,
+	__u32 ui32VerbLevel,
+	DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	void *pvDumpDebugFile);
+#define DBGNOTIFY_PFNS
+#endif
+enum PVRSRV_ERROR_TAG PVRSRVRegisterDeviceDbgRequestNotify(void **phNotify,
+	struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	__u32 ui32RequesterID,
+	void *hDbgRequestHandle);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterDeviceDbgRequestNotify(void *hNotify);
+enum PVRSRV_ERROR_TAG PVRSRVRegisterDriverDbgRequestNotify(void **phNotify,
+	PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	__u32 ui32RequesterID,
+	void *hDbgRequestHandle);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterDriverDbgRequestNotify(void *hNotify);
+
+/* physmem_dmabuf.h */
+
+struct dma_buf *PhysmemGetDmaBuf(struct _PMR_ *psPMR);
+
+struct dma_resv *PhysmemGetDmaResv(struct _PMR_ *psPMR);
+
+/* pvrsrv.h */
+
+enum PVRSRV_ERROR_TAG PVRSRVAcquireGlobalEventObjectKM(void **phGlobalEventObject);
+enum PVRSRV_ERROR_TAG PVRSRVReleaseGlobalEventObjectKM(void *hGlobalEventObject);
+
+/* sync.h */
+
+enum PVRSRV_ERROR_TAG SyncPrimContextCreate(
+	struct _PVRSRV_DEVICE_NODE_ *psDevConnection,
+	struct SYNC_PRIM_CONTEXT_TAG **phSyncPrimContext);
+void SyncPrimContextDestroy(struct SYNC_PRIM_CONTEXT_TAG *hSyncPrimContext);
+
+enum PVRSRV_ERROR_TAG SyncPrimAlloc(struct SYNC_PRIM_CONTEXT_TAG *hSyncPrimContext,
+	struct PVRSRV_CLIENT_SYNC_PRIM_TAG **ppsSync, const char *pszClassName);
+enum PVRSRV_ERROR_TAG SyncPrimFree(struct PVRSRV_CLIENT_SYNC_PRIM_TAG *psSync);
+enum PVRSRV_ERROR_TAG SyncPrimGetFirmwareAddr(
+	struct PVRSRV_CLIENT_SYNC_PRIM_TAG *psSync,
+	__u32 *sync_addr);
+
+/* osfunc.h */
+enum PVRSRV_ERROR_TAG OSEventObjectWait(void *hOSEventKM);
+enum PVRSRV_ERROR_TAG OSEventObjectOpen(void *hEventObject, void **phOSEventKM);
+enum PVRSRV_ERROR_TAG OSEventObjectClose(void *hOSEventKM);
+__u32 OSGetCurrentClientProcessIDKM(void);
+__u32 OSStringUINT32ToStr(char *pszBuf, size_t uSize, __u32 ui32Num);
+
+/* srvkm.h */
+
+enum PVRSRV_ERROR_TAG PVRSRVCommonDeviceCreate(void *pvOSDevice,
+	int i32KernelDeviceID,
+	struct _PVRSRV_DEVICE_NODE_ **ppsDeviceNode);
+void PVRSRVCommonDeviceDestroy(
+	struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+const char *PVRSRVGetErrorString(enum PVRSRV_ERROR_TAG eError);
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+enum PVRSRV_ERROR_TAG PVRSRVCommonDeviceInitialise(
+	struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN)(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext, PVRSRV_FENCE fence, u32 *nr_checkpoints, PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid);
+
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN)(PVRSRV_FENCE export_fence, PSYNC_CHECKPOINT_CONTEXT checkpoint_context, PSYNC_CHECKPOINT *checkpoint_handle);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN)(PVRSRV_FENCE export_fence);
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN)(
+		struct _PVRSRV_DEVICE_NODE_ *device,
+		const char *fence_name,
+		PVRSRV_TIMELINE timeline,
+		PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		PVRSRV_FENCE *new_fence,
+		u64 *fence_uid,
+		void **fence_finalise_data,
+		PSYNC_CHECKPOINT *new_checkpoint_handle,
+		void **timeline_update_sync,
+		__u32 *timeline_update_value);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN)(PVRSRV_FENCE fence_to_rollback, void *finalise_data);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN)(PVRSRV_FENCE fence_to_finalise, void *finalise_data);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef __u32 (*PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN)(__u32 num_ufos, __u32 *vaddrs);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef bool (*PFN_SYNC_CHECKPOINT_UFO_HAS_SIGNALLED_FN)(
+	__u32 ui32FwAddr, __u32 ui32Value);
+typedef enum PVRSRV_ERROR_TAG (*PFN_SYNC_CHECKPOINT_SIGNAL_WAITERS_FN)(void);
+typedef void(*PFN_SYNC_CHECKPOINT_CHECK_STATE_FN)(void);
+#if defined(PDUMP)
+typedef PVRSRV_ERROR(*PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN)(PVRSRV_FENCE iFence,
+						IMG_UINT32 *puiNumCheckpoints,
+		                                PSYNC_CHECKPOINT **papsCheckpoints);
+#endif
+#endif
+
+/* This is the function that kick code will call in a NO_HARDWARE build only after
+ * sync checkpoints have been manually signalled, to allow the OS native sync
+ * implementation to update its timelines (as the usual callback notification
+ * of signalled checkpoints is not supported for NO_HARDWARE).
+ */
+#ifndef CHECKPOINT_PFNS
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN)(void *private_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN)(PVRSRV_FENCE fence_to_signal);
+typedef void (*PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN)(void *mem_ptr);
+
+#define SYNC_CHECKPOINT_IMPL_MAX_STRLEN 20
+
+typedef struct {
+	PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN pfnFenceResolve;
+	PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN pfnFenceCreate;
+	PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN pfnFenceDataRollback;
+	PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN pfnFenceFinalise;
+	PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN pfnNoHWUpdateTimelines;
+	PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN pfnNoHWSignalExpFence;
+	PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN pfnFreeCheckpointListMem;
+	PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN pfnDumpInfoOnStalledUFOs;
+	char pszImplName[SYNC_CHECKPOINT_IMPL_MAX_STRLEN];
+#if defined(PDUMP)
+	PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN pfnSyncFenceGetCheckpoints;
+#endif
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN pfnExportFenceResolve;
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN pfnExportFenceRollback;
+} PFN_SYNC_CHECKPOINT_STRUCT;
+
+enum PVRSRV_ERROR_TAG SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns);
+
+#define CHECKPOINT_PFNS
+#endif
+
+/* sync_checkpoint.h */
+enum PVRSRV_ERROR_TAG SyncCheckpointContextCreate(struct _PVRSRV_DEVICE_NODE_ *psDevConnection, PSYNC_CHECKPOINT_CONTEXT *phSyncCheckpointContext);
+enum PVRSRV_ERROR_TAG SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext);
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext);
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext);
+enum PVRSRV_ERROR_TAG SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext, PVRSRV_TIMELINE timeline, PVRSRV_FENCE fence, const char *pszCheckpointName, PSYNC_CHECKPOINT *ppsSyncCheckpoint);
+void SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+void SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+bool SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+bool SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+enum PVRSRV_ERROR_TAG SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+enum PVRSRV_ERROR_TAG SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+void SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+void SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+PVRSRV_TIMELINE SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint);
+const char *SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint);
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+struct _PVRSRV_DEVICE_NODE_ *SyncCheckpointGetAssociatedDevice(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+#endif
+IMG_BOOL SyncCheckpointCommonDeviceIDs(PSYNC_CHECKPOINT_CONTEXT psSyncContext, IMG_HANDLE hDevRef);
+enum PVRSRV_ERROR_TAG SyncCheckpointGetCounters(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 *puiInUse, IMG_UINT32 *puiMax, IMG_UINT32 *puiXDInUse, IMG_UINT32*puiXDMax);
+
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+/*************************************************************************/ /*!
+@Function       NativeSyncGetFenceStatusWq
+@Description    Called to get the Foreign Fence status workqueue used in
+                Fence sync and Buffer sync.
+@Return         struct workqueue_struct ptr on success, NULL otherwise.
+*/ /**************************************************************************/
+struct workqueue_struct *NativeSyncGetFenceStatusWq(void);
+/*************************************************************************/ /*!
+@Function       NativeSyncGetFenceCtxDestroyWq
+@Return         struct workqueue_struct ptr on success, NULL otherwise.
+*/ /**************************************************************************/
+struct workqueue_struct *NativeSyncGetFenceCtxDestroyWq(void);
+#endif
+
+#endif /* __SERVICES_KERNEL_CLIENT__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.c b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.c
new file mode 100644
index 000000000000..9fdd8598f2ef
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.c
@@ -0,0 +1,269 @@
+/*************************************************************************/ /*!
+@Title          Linux trace event helper functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/sched.h>
+
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+#if !defined(CONFIG_TRACE_GPU_MEM)
+#define CREATE_TRACE_POINTS
+#include <trace/events/gpu_mem.h>
+#undef CREATE_TRACE_POINTS
+#else /* !defined(CONFIG_TRACE_GPU_MEM) */
+#include <trace/events/gpu_mem.h>
+#endif /* !defined(CONFIG_TRACE_GPU_MEM) */
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+
+#include "img_types.h"
+#include "trace_events.h"
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#include "rogue_trace_events.h"
+#endif
+#include "sync_checkpoint_external.h"
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+static bool fence_update_event_enabled, fence_check_event_enabled;
+
+bool trace_rogue_are_fence_updates_traced(void)
+{
+	return fence_update_event_enabled;
+}
+
+bool trace_rogue_are_fence_checks_traced(void)
+{
+	return fence_check_event_enabled;
+}
+
+/*
+ * Call backs referenced from rogue_trace_events.h. Note that these are not
+ * thread-safe, however, since running trace code when tracing is not enabled is
+ * simply a no-op, there is no harm in it.
+ */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_update_enabled_callback(void)
+#else
+void trace_fence_update_enabled_callback(void)
+#endif
+{
+	fence_update_event_enabled = true;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return 0;
+#endif
+}
+
+void trace_fence_update_disabled_callback(void)
+{
+	fence_update_event_enabled = false;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_check_enabled_callback(void)
+#else
+void trace_fence_check_enabled_callback(void)
+#endif
+{
+	fence_check_event_enabled = true;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return 0;
+#endif
+}
+
+void trace_fence_check_disabled_callback(void)
+{
+	fence_check_event_enabled = false;
+}
+
+#if defined(SUPPORT_RGX)
+/* This is a helper that calls trace_rogue_fence_update for each fence in an
+ * array.
+ */
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values)
+{
+	IMG_UINT i;
+	for (i = 0; i < uCount; i++)
+	{
+		trace_rogue_fence_update(current->comm, cmd, dm, ui32GpuId, ui32FWContext, ui32Offset,
+								 pauiAddresses[i].ui32Addr, PVRSRV_SYNC_CHECKPOINT_SIGNALLED);
+	}
+}
+
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values)
+{
+	IMG_UINT i;
+	for (i = 0; i < uCount; i++)
+	{
+		trace_rogue_fence_check(current->comm, cmd, dm, ui32GpuId, ui32FWContext, ui32Offset,
+							  pauiAddresses[i].ui32Addr, PVRSRV_SYNC_CHECKPOINT_SIGNALLED);
+	}
+}
+
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		trace_rogue_ufo_update(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+				ui32IntJobRef,
+				ui32ExtJobRef,
+				puData->sUpdate.ui32FWAddr,
+				puData->sUpdate.ui32OldValue,
+				puData->sUpdate.ui32NewValue);
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sUpdate));
+	}
+}
+
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		if (bPrEvent)
+		{
+			trace_rogue_ufo_pr_check_success(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckSuccess.ui32FWAddr,
+					puData->sCheckSuccess.ui32Value);
+		}
+		else
+		{
+			trace_rogue_ufo_check_success(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckSuccess.ui32FWAddr,
+					puData->sCheckSuccess.ui32Value);
+		}
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sCheckSuccess));
+	}
+}
+
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		if (bPrEvent)
+		{
+			trace_rogue_ufo_pr_check_fail(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckFail.ui32FWAddr,
+					puData->sCheckFail.ui32Value,
+					puData->sCheckFail.ui32Required);
+		}
+		else
+		{
+			trace_rogue_ufo_check_fail(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckFail.ui32FWAddr,
+					puData->sCheckFail.ui32Value,
+					puData->sCheckFail.ui32Required);
+		}
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sCheckFail));
+	}
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+
+int PVRGpuTraceEnableUfoCallbackWrapper(void)
+{
+	PVRGpuTraceEnableUfoCallback();
+	return 0;
+}
+
+int PVRGpuTraceEnableFirmwareActivityCallbackWrapper(void)
+{
+	PVRGpuTraceEnableFirmwareActivityCallback();
+	return 0;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+#endif /* defined(SUPPORT_RGX) */
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size)
+{
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+	trace_gpu_mem_total(ui8GPUId, 0, ui64Size);
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+}
+
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size)
+{
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+	trace_gpu_mem_total(ui8GPUId, ui32Pid, ui64Size);
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.h b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.h
new file mode 100644
index 000000000000..5a4d0567688b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/env/linux/trace_events.h
@@ -0,0 +1,196 @@
+/*************************************************************************/ /*!
+@Title          Linux trace events and event helper functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TRACE_EVENTS_H)
+#define TRACE_EVENTS_H
+
+#include "rgx_fwif_km.h"
+#include "rgx_hwperf.h"
+
+/* We need to make these functions do nothing if CONFIG_EVENT_TRACING isn't
+ * enabled, just like the actual trace event functions that the kernel
+ * defines for us.
+ */
+
+#if defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+bool trace_rogue_are_fence_checks_traced(void);
+
+bool trace_rogue_are_fence_updates_traced(void);
+
+#if defined(SUPPORT_RGX)
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values);
+
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values);
+
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+#endif /* if defined(SUPPORT_RGX) */
+#else /* defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+static inline
+bool trace_rogue_are_fence_checks_traced(void)
+{
+	return false;
+}
+
+static inline
+bool trace_rogue_are_fence_updates_traced(void)
+{
+	return false;
+}
+
+#if defined(SUPPORT_RGX)
+static inline
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values)
+{
+}
+
+static inline
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values)
+{
+}
+
+static inline
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+
+static inline
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+
+static inline
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+#endif /* if defined(SUPPORT_RGX)*/
+#endif /* defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(CONFIG_EVENT_TRACING)
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size);
+
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size);
+#else
+static inline
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size)
+{
+}
+
+static inline
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size)
+{
+}
+#endif /* defined(CONFIG_EVENT_TRACING) */
+
+#endif /* TRACE_EVENTS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/cache_km.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/cache_km.h
new file mode 100644
index 000000000000..282ff5bc5f0f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/cache_km.h
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File           cache_km.h
+@Title          CPU cache management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef CACHE_KM_H
+#define CACHE_KM_H
+
+#if defined(__linux__)
+#include <linux/version.h>
+#else
+#define KERNEL_VERSION
+#endif
+
+#include "pvrsrv_error.h"
+#include "os_cpu_cache.h"
+#include "img_types.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "pmr.h"
+
+typedef IMG_UINT32 PVRSRV_CACHE_OP_ADDR_TYPE;	/*!< Represents CPU address type required for CPU d-cache maintenance */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL	0x1	/*!< Operation requires CPU virtual address only */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_PHYSICAL	0x2	/*!< Operation requires CPU physical address only */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_BOTH		0x3	/*!< Operation requires both CPU virtual & physical addresses */
+
+#include "connection_server.h"
+
+/*
+ * CacheOpInit() & CacheOpDeInit()
+ *
+ * This must be called to initialise the KM cache maintenance framework.
+ * This is called early during the driver/module (un)loading phase.
+ */
+PVRSRV_ERROR CacheOpInit(void);
+void CacheOpDeInit(void);
+
+/*
+ * CacheOpInit2() & CacheOpDeInit2()
+ *
+ * This must be called to initialise the UM cache maintenance framework.
+ * This is called when the driver is loaded/unloaded from the kernel.
+ */
+PVRSRV_ERROR CacheOpInit2(void);
+void CacheOpDeInit2(void);
+
+/*
+ * CacheOpExec()
+ *
+ * This is the primary CPU data-cache maintenance interface and it is
+ * always guaranteed to be synchronous; the arguments supplied must be
+ * pre-validated for performance reasons else the d-cache maintenance
+ * operation might cause the underlying OS kernel to fault.
+ */
+PVRSRV_ERROR CacheOpExec(PPVRSRV_DEVICE_NODE psDevNode,
+						void *pvVirtStart,
+						void *pvVirtEnd,
+						IMG_CPU_PHYADDR sCPUPhysStart,
+						IMG_CPU_PHYADDR sCPUPhysEnd,
+						PVRSRV_CACHE_OP uiCacheOp);
+
+/*
+ * CacheOpValExec()
+ *
+ * Same as CacheOpExec(), except arguments are _Validated_ before being
+ * presented to the underlying OS kernel for CPU data-cache maintenance.
+ * The uiAddress is the start CPU virtual address for the to-be d-cache
+ * maintained PMR, it can be NULL in which case a remap will be performed
+ * internally, if required for cache maintenance. This is primarily used
+ * as the services client bridge call handler for synchronous user-mode
+ * cache maintenance requests.
+ */
+PVRSRV_ERROR CacheOpValExec(PMR *psPMR,
+							IMG_UINT64 uiAddress,
+							IMG_DEVMEM_OFFSET_T uiOffset,
+							IMG_DEVMEM_SIZE_T uiSize,
+							PVRSRV_CACHE_OP uiCacheOp);
+
+/*
+ * CacheOpQueue()
+ *
+ * This is the secondary cache maintenance interface and it is not
+ * guaranteed to be synchronous in that requests could be deferred
+ * and executed asynchronously. This interface is primarily meant
+ * as services client bridge call handler. Both uiInfoPgGFSeqNum
+ * and ui32[Current,Next]FenceSeqNum implements an internal client
+ * server queueing protocol so making use of this interface outside
+ * of services client is not recommended and should not be done.
+ */
+PVRSRV_ERROR CacheOpQueue(CONNECTION_DATA *psConnection,
+						PPVRSRV_DEVICE_NODE psDevNode,
+						IMG_UINT32 ui32OpCount,
+						PMR **ppsPMR,
+						IMG_UINT64 *puiAddress,
+						IMG_DEVMEM_OFFSET_T *puiOffset,
+						IMG_DEVMEM_SIZE_T *puiSize,
+						PVRSRV_CACHE_OP *puiCacheOp,
+						IMG_UINT32 ui32OpTimeline);
+
+/*
+ * CacheOpLog()
+ *
+ * This is used for logging client cache maintenance operations that
+ * was executed in user-space.
+ */
+PVRSRV_ERROR CacheOpLog(PMR *psPMR,
+						IMG_UINT64 uiAddress,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT64 ui64StartTime,
+						IMG_UINT64 ui64EndTime,
+						PVRSRV_CACHE_OP uiCacheOp);
+
+#endif	/* CACHE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/connection_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/connection_server.h
new file mode 100644
index 000000000000..673c37c02a3c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/connection_server.h
@@ -0,0 +1,143 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    API for server side connection management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(CONNECTION_SERVER_H)
+#define CONNECTION_SERVER_H
+
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "handle.h"
+#include "pvrsrv_cleanup.h"
+
+/* Variable used to hold in memory the timeout for the current time slice*/
+extern IMG_UINT64 gui64TimesliceLimit;
+/* Counter number of handle data freed during the current time slice */
+extern IMG_UINT32 gui32HandleDataFreeCounter;
+/* Set the maximum time the freeing of the resources can keep the lock */
+#define CONNECTION_DEFERRED_CLEANUP_TIMESLICE_NS (3000 * 1000) /* 3ms */
+
+typedef struct _CONNECTION_DATA_
+{
+	PVRSRV_HANDLE_BASE		*psHandleBase;
+	PROCESS_HANDLE_BASE		*psProcessHandleBase;
+	struct _SYNC_CONNECTION_DATA_	*psSyncConnectionData;
+	struct _PDUMP_CONNECTION_DATA_	*psPDumpConnectionData;
+
+	/* Holds the client flags supplied at connection time */
+	IMG_UINT32          ui32ClientFlags;
+
+	/*
+	 * OS specific data can be stored via this handle.
+	 * See osconnection_server.h for a generic mechanism
+	 * for initialising this field.
+	 */
+	IMG_HANDLE          hOsPrivateData;
+
+#define PVRSRV_CONNECTION_PROCESS_NAME_LEN (16)
+	IMG_PID             pid;
+	IMG_PID				vpid;
+	IMG_UINT32			tid;
+	IMG_CHAR            pszProcName[PVRSRV_CONNECTION_PROCESS_NAME_LEN];
+
+	IMG_HANDLE          hProcessStats;
+
+	IMG_HANDLE          hClientTLStream;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/*
+	 * Connection-based values per application which can be modified by the
+	 * AppHint settings 'OSid, OSidReg, bOSidAxiProtReg' for each application.
+	 * These control where the connection's memory allocation is sourced from.
+	 * ui32OSid, ui32OSidReg range from 0..(GPUVIRT_VALIDATION_NUM_OS - 1).
+	 */
+	IMG_UINT32          ui32OSid;
+	IMG_UINT32          ui32OSidReg;
+	IMG_BOOL            bOSidAxiProtReg;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	IMG_BOOL            bAcceptDmaRequests;
+	ATOMIC_T            ui32NumDmaTransfersInFlight;
+	IMG_HANDLE          hDmaEventObject;
+#endif
+	/* Structure which is hooked into the cleanup thread work list */
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+
+	DLLIST_NODE         sConnectionListNode;
+
+	/* List navigation for deferred freeing of connection data */
+	struct _CONNECTION_DATA_	**ppsThis;
+	struct _CONNECTION_DATA_	*psNext;
+} CONNECTION_DATA;
+
+#include "osconnection_server.h"
+
+PVRSRV_ERROR PVRSRVCommonConnectionConnect(void **ppvPrivData, void *pvOSData);
+void PVRSRVCommonConnectionDisconnect(void *pvPrivData);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVGetPurgeConnectionPid
+
+@Description    Returns PID associated with Connection currently being purged by
+                Cleanup Thread. If no Connection is purged 0 is returned.
+
+@Return         PID associated with currently purged connection or 0 if no
+                connection is being purged
+*/ /***************************************************************************/
+IMG_PID PVRSRVGetPurgeConnectionPid(void);
+
+void PVRSRVConnectionDebugNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile);
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVConnectionPrivateData)
+#endif
+static INLINE
+IMG_HANDLE PVRSRVConnectionPrivateData(CONNECTION_DATA *psConnection)
+{
+	return (psConnection != NULL) ? psConnection->hOsPrivateData : NULL;
+}
+
+#endif /* !defined(CONNECTION_SERVER_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/debug_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/debug_common.h
new file mode 100644
index 000000000000..aba576d273a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/debug_common.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common debug definitions and functions.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEBUG_COMMON_H
+#define DEBUG_COMMON_H
+
+#include "pvrsrv_error.h"
+#include "device.h"
+
+PVRSRV_ERROR DebugCommonInitDriver(void);
+void DebugCommonDeInitDriver(void);
+
+const IMG_CHAR *PVRSRVGetDebugDevStateString(PVRSRV_DEVICE_STATE eDevState);
+const IMG_CHAR *PVRSRVGetDebugHealthStatusString(PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus);
+const IMG_CHAR *PVRSRVGetDebugHealthReasonString(PVRSRV_DEVICE_HEALTH_REASON eHealthReason);
+
+PVRSRV_ERROR DebugCommonInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+void DebugCommonDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+typedef struct _IMG_FLAGS2DESC_
+{
+	IMG_UINT32		uiFlag;
+	const IMG_CHAR	*pszLabel;
+} IMG_FLAGS2DESC;
+
+void DebugCommonFlagStrings(IMG_CHAR *psDesc,
+							IMG_UINT32 ui32DescSize,
+							const IMG_FLAGS2DESC *psConvTable,
+							IMG_UINT32 ui32TableSize,
+							IMG_UINT32 ui32Flags);
+
+#endif /* DEBUG_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/device.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/device.h
new file mode 100644
index 000000000000..e301ef1738e8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/device.h
@@ -0,0 +1,681 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common Device header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device related function templates and defines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DEVICE_H
+#define DEVICE_H
+
+#include "devicemem_heapcfg.h"
+#include "mmu_common.h"
+#include "ra.h"			/* RA_ARENA */
+#include "pvrsrv_device.h"
+#include "sync_checkpoint.h"
+#include "srvkm.h"
+#include "physheap.h"
+#include "sync_internal.h"
+#include "dllist.h"
+
+#include "rgx_bvnc_defs_km.h"
+
+#include "lock.h"
+
+#include "power.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+typedef struct _PVRSRV_POWER_DEV_TAG_ *PPVRSRV_POWER_DEV;
+
+struct SYNC_RECORD;
+
+struct _CONNECTION_DATA_;
+
+/*************************************************************************/ /*!
+ @Function      AllocUFOBlockCallback
+ @Description   Device specific callback for allocation of a UFO block
+
+ @Input         psDeviceNode          Pointer to device node to allocate
+                                      the UFO for.
+ @Input         ui32RequestedSize     Minimum size of allocation requested
+ @Output        ppsMemDesc            Pointer to pointer for the memdesc of
+                                      the allocation
+ @Output        pui32SyncAddr         FW Base address of the UFO block
+ @Output        puiSyncPrimBlockSize  Size of the UFO block
+
+ @Return        PVRSRV_OK if allocation was successful
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*AllocUFOBlockCallback)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+											  IMG_UINT32 ui32RequestedSize,
+											  DEVMEM_MEMDESC **ppsMemDesc,
+											  IMG_UINT32 *pui32SyncAddr,
+											  IMG_UINT32 *puiSyncPrimBlockSize);
+
+/*************************************************************************/ /*!
+ @Function      FreeUFOBlockCallback
+ @Description   Device specific callback for freeing of a UFO
+
+ @Input         psDeviceNode    Pointer to device node that the UFO block was
+                                allocated from.
+ @Input         psMemDesc       Pointer to pointer for the memdesc of the UFO
+                                block to free.
+*/ /**************************************************************************/
+typedef void (*FreeUFOBlockCallback)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+									 DEVMEM_MEMDESC *psMemDesc);
+
+typedef struct _PVRSRV_DEVICE_IDENTIFIER_
+{
+	/* Pdump memory and register bank names */
+	IMG_CHAR				*pszPDumpDevName;
+	IMG_CHAR				*pszPDumpRegName;
+
+	/* Under Linux, this is the minor number of RenderNode corresponding to this Device */
+	IMG_INT32				i32KernelDeviceID;
+	/* Services layer enumeration of the device used in pvrdebug */
+	IMG_UINT32				ui32InternalID;
+} PVRSRV_DEVICE_IDENTIFIER;
+
+typedef struct _DEVICE_MEMORY_INFO_
+{
+	/* Heap count. Doesn't include additional heaps from PVRSRVCreateDeviceMemHeap */
+	IMG_UINT32				ui32HeapCount;
+
+	/* Blueprints for creating new device memory contexts */
+	IMG_UINT32              uiNumHeapConfigs;
+	DEVMEM_HEAP_CONFIG      *psDeviceMemoryHeapConfigArray;
+	DEVMEM_HEAP_BLUEPRINT   *psDeviceMemoryHeap;
+} DEVICE_MEMORY_INFO;
+
+#define MMU_BAD_PHYS_ADDR (0xbadbad00badULL)
+
+typedef struct __DEFAULT_PAGE__
+{
+	/*Page handle for the page allocated (UMA/LMA)*/
+	PG_HANDLE	sPageHandle;
+	POS_LOCK	psPgLock;
+	/*Default page size in terms of log2 */
+	IMG_UINT32	ui32Log2PgSize;
+	IMG_UINT64	ui64PgPhysAddr;
+#if defined(PDUMP)
+	IMG_HANDLE hPdumpPg;
+#endif
+} PVRSRV_DEF_PAGE;
+
+#define PVRSRV_DEVICE_STATE_LIST \
+	X(UNDEFINED)                 \
+	X(CREATING)                  \
+	X(CREATED)                   \
+	X(ACTIVE)                    \
+	X(FROZEN)                    \
+	X(DEINIT)                    \
+	X(DESTRUCTING)               \
+	X(BAD)                       \
+	X(PCI_ERROR)                 \
+	X(LAST)                      \
+
+typedef enum _PVRSRV_DEVICE_STATE_
+{
+#define X(_name) PVRSRV_DEVICE_STATE_ ## _name,
+	PVRSRV_DEVICE_STATE_LIST
+#undef X
+
+} PVRSRV_DEVICE_STATE;
+
+#define PVRSRV_DEVICE_HEALTH_STATUS_LIST \
+	X(UNDEFINED)                         \
+	X(OK)                                \
+	X(NOT_RESPONDING)                    \
+	X(DEAD)                              \
+	X(FAULT)                             \
+	X(LAST)                              \
+
+typedef enum _PVRSRV_DEVICE_HEALTH_STATUS_
+{
+#define X(_name) PVRSRV_DEVICE_HEALTH_STATUS_ ## _name,
+	PVRSRV_DEVICE_HEALTH_STATUS_LIST
+#undef X
+
+} PVRSRV_DEVICE_HEALTH_STATUS;
+
+#define PVRSRV_DEVICE_HEALTH_REASON_LIST \
+	X(NONE)                              \
+	X(ASSERTED)                          \
+	X(POLL_FAILING)                      \
+	X(TIMEOUTS)                          \
+	X(QUEUE_CORRUPT)                     \
+	X(QUEUE_STALLED)                     \
+	X(IDLING)                            \
+	X(RESTARTING)                        \
+	X(MISSING_INTERRUPTS)                \
+	X(PCI_ERROR)                         \
+	X(LAST)                              \
+
+typedef enum _PVRSRV_DEVICE_HEALTH_REASON_
+{
+#define X(_name) PVRSRV_DEVICE_HEALTH_REASON_ ## _name,
+	PVRSRV_DEVICE_HEALTH_REASON_LIST
+#undef X
+
+} PVRSRV_DEVICE_HEALTH_REASON;
+
+typedef enum _PVRSRV_DEVICE_DEBUG_DUMP_STATUS_
+{
+	PVRSRV_DEVICE_DEBUG_DUMP_NONE = 0,
+	PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE
+} PVRSRV_DEVICE_DEBUG_DUMP_STATUS;
+
+#ifndef DI_GROUP_DEFINED
+#define DI_GROUP_DEFINED
+typedef struct DI_GROUP DI_GROUP;
+#endif
+#ifndef DI_ENTRY_DEFINED
+#define DI_ENTRY_DEFINED
+typedef struct DI_ENTRY DI_ENTRY;
+#endif
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#ifndef DI_VZ_DATA_DEFINED
+#define DI_VZ_DATA_DEFINED
+typedef struct DI_VZ_DATA DI_VZ_DATA;
+#endif
+#endif
+
+typedef struct _PVRSRV_DEVICE_DEBUG_INFO_
+{
+	DI_GROUP *psGroup;
+	DI_ENTRY *psDumpDebugEntry;
+#ifdef SUPPORT_RGX
+	DI_ENTRY *psFWTraceEntry;
+#ifdef SUPPORT_FIRMWARE_GCOV
+	DI_ENTRY *psFWGCOVEntry;
+#endif
+	DI_ENTRY *psFWMappingsEntry;
+#if  defined(SUPPORT_RISCV_GDB)
+	DI_ENTRY *psRiscvDmiDIEntry;
+	IMG_UINT64 ui64RiscvDmi;
+#endif
+	DI_ENTRY *psDevMemEntry;
+	IMG_HANDLE hGpuUtilUserDebugFS;
+#endif /* SUPPORT_RGX */
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	DI_ENTRY *psPowerDataEntry;
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	DI_ENTRY *psPowerTimingStatsEntry;
+#endif
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	DI_GROUP *psVZGroup;
+	DI_GROUP *apsVZDriverGroups[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *apsVZDriverPriorityDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *apsVZDriverTimeSliceDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *psVZDriverTimeSliceIntervalDIEntry;
+	DI_ENTRY *apsVZDriverIsolationGroupDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_VZ_DATA *apsVZDriverData[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *psVZDriverConnectionCooldownPeriodDIEntry;
+#endif
+} PVRSRV_DEVICE_DEBUG_INFO;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+#define RGX_LISR_INIT							(0U)
+#define RGX_LISR_DEVICE_NOT_POWERED				(1U)
+#define RGX_LISR_NOT_TRIGGERED_BY_HW			(2U)
+#define RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED		(3U)
+#define RGX_LISR_PROCESSED						(4U)
+
+typedef IMG_UINT32 LISR_STATUS;
+
+typedef struct _LISR_EXECUTION_INFO_
+{
+	/* status of last LISR invocation */
+	LISR_STATUS ui32Status;
+
+	/* snapshot from the last LISR invocation */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	IMG_UINT32 aui32InterruptCountSnapshot[RGX_NUM_DRIVERS_SUPPORTED];
+#else
+	IMG_UINT32 aui32InterruptCountSnapshot[RGXFW_THREAD_NUM];
+#endif
+
+	/* time of the last LISR invocation */
+	IMG_UINT64 ui64Clockns;
+} LISR_EXECUTION_INFO;
+
+#if !defined(__CHECKER__)
+#define UPDATE_LISR_DBG_STATUS(status)		\
+	do { \
+		psDeviceNode->sLISRExecutionInfo.ui32Status = (status); \
+		if ((status > RGX_LISR_INIT) && (status < RGX_LISR_PROCESSED)) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: IRQ %" IMG_UINT64_FMTSPEC " rejected: %s", __func__, psDeviceNode->ui64nLISR, #status)); \
+		} \
+	} while (0)
+
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)	psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[idx] = (val)
+#define UPDATE_LISR_DBG_TIMESTAMP()			psDeviceNode->sLISRExecutionInfo.ui64Clockns = OSClockns64()
+#define UPDATE_LISR_DBG_COUNTER()			psDeviceNode->ui64nLISR++
+#define UPDATE_MISR_DBG_COUNTER()			psDeviceNode->ui64nMISR++
+#else /*!defined(__CHECKER__) */
+#define UPDATE_LISR_DBG_STATUS(status)
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)
+#define UPDATE_LISR_DBG_TIMESTAMP()
+#define UPDATE_LISR_DBG_COUNTER()
+#define UPDATE_MISR_DBG_COUNTER()
+#endif /* !defined(__CHECKER__) */
+#else /* defined(PVRSRV_DEBUG_LISR_EXECUTION) */
+#define UPDATE_LISR_DBG_STATUS(status)
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)
+#define UPDATE_LISR_DBG_TIMESTAMP()
+#define UPDATE_LISR_DBG_COUNTER()
+#define UPDATE_MISR_DBG_COUNTER()
+#endif /* defined(PVRSRV_DEBUG_LISR_EXECUTION) */
+
+typedef struct _PVRSRV_DEVICE_NODE_
+{
+	PVRSRV_DEVICE_IDENTIFIER	sDevId;
+	PVRSRV_DEVICE_STATE			eDevState; /* Set using PVRSRVDeviceSetState, not directly. */
+	PVRSRV_DEVICE_FABRIC_TYPE	eDevFabricType;
+
+	ATOMIC_T					eHealthStatus; /* Holds values from PVRSRV_DEVICE_HEALTH_STATUS */
+	ATOMIC_T					eHealthReason; /* Holds values from PVRSRV_DEVICE_HEALTH_REASON */
+	ATOMIC_T					eDebugDumpRequested; /* Holds values from PVRSRV_DEVICE_DEBUG_DUMP_STATUS */
+
+	IMG_HANDLE					*hDebugTable;
+
+	/* device specific MMU attributes */
+	MMU_DEVICEATTRIBS      *psMMUDevAttrs;
+	/* Device specific MMU firmware attributes, used only in some devices */
+	MMU_DEVICEATTRIBS      *psFirmwareMMUDevAttrs;
+	/* Physical Heap where MMU PT pages are allocated from, normally the
+	 * system's default physical heap but can be different for AutoVz driver. */
+	PHYS_HEAP              *psMMUPhysHeap;
+
+	/* lock for power state transitions */
+	POS_LOCK				hPowerLock;
+	IMG_PID                 uiPwrLockOwnerPID; /* Only valid between lock and corresponding unlock
+	                                              operations of hPowerLock */
+#if defined(DEBUG)
+	struct
+	{
+		const char        *pszFile;			    /* Power lock acquired location (File) */
+		IMG_UINT32         ui32LineNum;		    /* Power lock acquired location (Line number) */
+		IMG_UINT64         ui64Timestamp;       /* Power lock acquired timestamp */
+	} sPowerLockOwner;
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+	IMG_UINT32              uiPowerOffCounter; /* Counts how many times the device has been powered
+	                                              off. Incremented in PVRSRVSetDeviceCurrentPowerState().*/
+	IMG_UINT32              uiPowerOffCounterNext; /* Value of next update to uiPowerOffCounter. */
+#endif
+
+	/* current system device power state */
+	PVRSRV_SYS_POWER_STATE	eCurrentSysPowerState;
+	PPVRSRV_POWER_DEV	psPowerDev;
+
+	/*
+		callbacks the device must support:
+	*/
+
+	PVRSRV_ERROR (*pfnInvalFBSCTable)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+									  MMU_CONTEXT *psMMUContext,
+									  IMG_UINT64 ui64FBSCEntries);
+
+	PVRSRV_ERROR (*pfnValidateOrTweakPhysAddrs)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												MMU_DEVICEATTRIBS *psDevAttrs,
+												IMG_UINT64 *pui64Addr);
+
+	void (*pfnMMUCacheInvalidate)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+								  MMU_CONTEXT *psMMUContext,
+								  MMU_LEVEL eLevel,
+								  IMG_BOOL bUnmap);
+
+	PVRSRV_ERROR (*pfnMMUCacheInvalidateKick)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	                                          IMG_UINT32 *pui32NextMMUInvalidateUpdate);
+
+	IMG_UINT32 (*pfnMMUCacheGetInvalidateCounter)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	/* Callback pfnMMUTopLevelPxWorkarounds may be NULL if not required */
+	void (*pfnMMUTopLevelPxWorkarounds)(struct _CONNECTION_DATA_ *psConnection,
+			                            struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+			                            IMG_DEV_PHYADDR sPhysAddrL1Px,
+			                            void *pxL1PxCpuVAddr);
+
+	/* Callback pfnMMUTweakProtFlags may be NULL if not required */
+	void (*pfnMMUTweakProtFlags)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+			                     MMU_DEVICEATTRIBS *psDevAttrs,
+			                     PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+			                     MMU_PROTFLAGS_T *uiMMUProtFlags);
+
+
+	void (*pfnDumpDebugInfo)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnUpdateHealthStatus)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	                                      IMG_BOOL bIsTimerPoll);
+
+#if defined(SUPPORT_AUTOVZ)
+	void (*pfnUpdateAutoVzWatchdog)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+#endif
+
+	PVRSRV_ERROR (*pfnValidationGPUUnitsPowerChange)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32NewState);
+
+	PVRSRV_ERROR (*pfnResetHWRLogs)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnVerifyBVNC)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask);
+
+	/* Method to drain device HWPerf packets from firmware buffer to host buffer */
+	PVRSRV_ERROR (*pfnServiceHWPerf)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnDeviceVersionString)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_CHAR **ppszVersionString);
+
+	PVRSRV_ERROR (*pfnDeviceClockSpeed)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_PUINT32 pui32RGXClockSpeed);
+
+	PVRSRV_ERROR (*pfnSoftReset)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+
+	PVRSRV_ERROR (*pfnAlignmentCheck)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32FWAlignChecksSize, IMG_UINT32 aui32FWAlignChecks[]);
+	IMG_BOOL	(*pfnCheckDeviceFeature)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64FeatureMask);
+
+	IMG_INT32	(*pfnGetDeviceFeatureValue)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, enum _RGX_FEATURE_WITH_VALUE_INDEX_ eFeatureIndex);
+
+    PVRSRV_ERROR (*pfnGetMultiCoreInfo)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores, IMG_UINT64 *pui64Caps);
+
+	IMG_BOOL (*pfnHasFBCDCVersion31)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	IMG_UINT32 (*pfnGetTFBCLossyGroup)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	MMU_DEVICEATTRIBS* (*pfnGetMMUDeviceAttributes)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_BOOL bKernelMemoryCtx);
+
+	PVRSRV_DEVICE_SNOOP_MODE (*pfnGetDeviceSnoopMode)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_DEVICE_CONFIG	*psDevConfig;
+
+	/* device post-finalise compatibility check */
+	PVRSRV_ERROR			(*pfnInitDeviceCompatCheck) (struct _PVRSRV_DEVICE_NODE_*);
+
+	/* initialise device-specific physheaps */
+	PVRSRV_ERROR			(*pfnPhysMemDeviceHeapsInit) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* determining the appropriate LMA allocation policy */
+	PHYS_HEAP_POLICY		(*pfnPhysHeapGetLMAPolicy) (PHYS_HEAP_USAGE_FLAGS, struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	/* initialise fw mmu, if FW not using GPU mmu, NULL otherwise. */
+	PVRSRV_ERROR			(*pfnFwMMUInit) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* Check device's FW Main physheap free memory */
+	PVRSRV_ERROR			(*pfnCheckForSufficientFWPhysMem) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* information about the device's address space and heaps */
+	DEVICE_MEMORY_INFO		sDevMemoryInfo;
+
+	/* device's shared-virtual-memory heap max virtual address */
+	IMG_UINT64				ui64GeneralSVMHeapTopVA;
+
+	ATOMIC_T				iNumClockSpeedChanges;
+
+	/* private device information */
+	void					*pvDevice;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	RA_ARENA				*psOSSharedArena;
+	RA_ARENA				*psOSidSubArena[GPUVIRT_VALIDATION_NUM_OS];
+#endif
+
+	/* When virtualisation support is enabled the Firmware heaps of virtualised
+	 * drivers can be entirely premapped into the Fw's VA space, during init
+	 * or during runtime on explicit request from Guest drivers. */
+	PHYS_HEAP               *apsFWPremapPhysHeap[RGX_NUM_DRIVERS_SUPPORTED];
+
+	/* Head of the physical heap list. Tracks PhysHeap objects created from
+	 * the PHYS_HEAP_CONFIG definitions supplied by the system layer at
+	 * device creation time. There could be 1 or more and varies from system
+	 * to system.
+	 */
+	struct _PHYS_HEAP_		*psPhysHeapList;
+	POS_LOCK				hPhysHeapLock;
+
+	/* The apsPhysHeap array is a mapping table to the system's, often fewer,
+	 * physical memory heaps defined for this device. It contains
+	 * PVRSRV_PHYS_HEAP_LAST entries, one for each possible physical
+	 * heaps allowed in the design. Each PhysHeap in the design is acquired
+	 * and stored in the mapping table during device create. Fall-back logic
+	 * is employed to ensure a valid heap is always found from the set defined
+	 * in the system layer for the device. Responsibility for this is shared
+	 * between the common layer (PhysHeapInitDeviceHeaps) and sub-device
+	 * layer (pfnPhysMemDeviceHeapsInit).
+	 * It is used in the PhysMem module to create PMRs from a given PhysHeap
+	 * of memory. See PhysHeapCreatePMR()
+	 */
+	PHYS_HEAP				*apsPhysHeap[PVRSRV_PHYS_HEAP_LAST];
+	IMG_UINT32				ui32UserAllocHeapCount;
+
+	/* Flag indicating if the firmware has been initialised during the
+	 * 1st boot of the Host driver according to the AutoVz life-cycle. */
+	IMG_BOOL				bAutoVzFwIsUp;
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL				bAutoVzAllowGPUPowerdown;
+#endif
+
+	/* Flags indicating VM state and if PVZ calls originating from it are valid */
+	IMG_UINT32				ui32VmState;
+
+	struct _PVRSRV_DEVICE_NODE_	*psNext;
+	struct _PVRSRV_DEVICE_NODE_	**ppsThis;
+
+	/* Functions for notification about memory contexts */
+	PVRSRV_ERROR			(*pfnRegisterMemoryContext)(struct _PVRSRV_DEVICE_NODE_	*psDeviceNode,
+														MMU_CONTEXT					*psMMUContext,
+														IMG_HANDLE					*hPrivData);
+	void					(*pfnUnregisterMemoryContext)(IMG_HANDLE hPrivData);
+
+	/* Functions for allocation/freeing of UFOs */
+	AllocUFOBlockCallback	pfnAllocUFOBlock;	/*!< Callback for allocation of a block of UFO memory */
+	FreeUFOBlockCallback	pfnFreeUFOBlock;	/*!< Callback for freeing of a block of UFO memory */
+
+	IMG_HANDLE				hSyncServerRecordNotify;
+	POS_LOCK				hSyncServerRecordLock;
+	IMG_UINT32				ui32SyncServerRecordCount;
+	IMG_UINT32				ui32SyncServerRecordCountHighWatermark;
+	DLLIST_NODE				sSyncServerRecordList;
+	struct SYNC_RECORD		*apsSyncServerRecordsFreed[PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN];
+	IMG_UINT32				uiSyncServerRecordFreeIdx;
+
+	IMG_HANDLE				hSyncCheckpointRecordNotify;
+	POS_LOCK				hSyncCheckpointRecordLock;
+	IMG_UINT32				ui32SyncCheckpointRecordCount;
+	IMG_UINT32				ui32SyncCheckpointRecordCountHighWatermark;
+	DLLIST_NODE				sSyncCheckpointRecordList;
+	struct SYNC_CHECKPOINT_RECORD	*apsSyncCheckpointRecordsFreed[PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN];
+	IMG_UINT32				uiSyncCheckpointRecordFreeIdx;
+
+	IMG_HANDLE				hSyncCheckpointNotify;
+	POS_SPINLOCK			hSyncCheckpointListLock; /*!< Protects sSyncCheckpointSyncsList */
+	DLLIST_NODE				sSyncCheckpointSyncsList;
+
+	PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext;
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+
+	/* With this sync-prim we make sure the MMU cache is flushed
+	 * before we free the page table memory */
+	PVRSRV_CLIENT_SYNC_PRIM	*psMMUCacheSyncPrim;
+	IMG_UINT32				ui32NextMMUInvalidateUpdate;
+
+	IMG_HANDLE				hCmdCompNotify;
+	IMG_HANDLE				hDbgReqNotify;
+	IMG_HANDLE				hAppHintDbgReqNotify;
+	IMG_HANDLE				hPhysHeapDbgReqNotify;
+
+	/* Device MMU common module can support one other larger page size
+	 * (e.g. 16KB) in addition to the default OS page size (often 4KB).
+	 * If supported, device code will store the size here as a log2 value
+	 * during device creation/registration. A 0 value will imply it is
+	 * not supported.
+	 */
+	IMG_UINT32              ui32Non4KPageSizeLog2;
+
+	PVRSRV_DEF_PAGE			sScratchPage;
+	PVRSRV_DEF_PAGE			sDevZeroPage;
+
+	/* Lock protects access to sMemoryContextPageFaultNotifyListHead and
+	 * per memory context DEVMEMINT_CTX::sProcessNotifyListHead lists. */
+	POSWR_LOCK				hPageFaultNotifyLock;
+	DLLIST_NODE				sMemoryContextPageFaultNotifyListHead;
+
+	/* System DMA channels */
+	IMG_UINT32				ui32RefCountDMA;
+	IMG_HANDLE				hDmaTxChan;
+	IMG_HANDLE				hDmaRxChan;
+	POS_LOCK				hDmaTxLock;
+	POS_LOCK				hDmaRxLock;
+
+#if defined(PDUMP)
+	/*
+	 * FBC clear color register default value to use.
+	 */
+	IMG_UINT64				ui64FBCClearColour;
+
+	/* Device-level callback which is called when pdump.exe starts.
+	 * Should be implemented in device-specific init code, e.g. rgxinit.c
+	 */
+	PVRSRV_ERROR			(*pfnPDumpInitDevice)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+	/* device-level callback to return pdump ID associated to a memory context */
+	IMG_UINT32				(*pfnMMUGetContextID)(IMG_HANDLE hDevMemContext);
+
+	IMG_UINT8			*pui8DeferredSyncCPSignal;	/*! Deferred fence events buffer */
+
+	IMG_UINT16			ui16SyncCPReadIdx;		/*! Read index in the above deferred fence events buffer */
+
+	IMG_UINT16			ui16SyncCPWriteIdx;		/*! Write index in the above deferred fence events buffer */
+
+	POS_LOCK			hSyncCheckpointSignalLock;	/*! Guards data shared between an sleepable-contexts */
+
+	void				*pvSyncCPMISR;			/*! MISR to emit pending/deferred fence signals */
+
+	void				*hTransition;			/*!< SyncCheckpoint PdumpTransition Cookie */
+
+	DLLIST_NODE			sSyncCheckpointContextListHead;	/*!< List head for the sync chkpt contexts */
+
+	POS_LOCK			hSyncCheckpointContextListLock;	/*! lock for accessing sync chkpt contexts list */
+
+#endif
+
+
+	/* Members for linking which connections are open on this device */
+	POS_LOCK                hConnectionsLock;    /*!< Lock protecting sConnections */
+	DLLIST_NODE             sConnections;        /*!< The list of currently active connection objects for this device node */
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	LISR_EXECUTION_INFO     sLISRExecutionInfo;  /*!< Information about the last execution of the LISR */
+	IMG_UINT64              ui64nLISR;           /*!< Number of LISR calls seen */
+	IMG_UINT64              ui64nMISR;           /*!< Number of MISR calls made */
+#endif
+
+	PVRSRV_DEVICE_DEBUG_INFO sDebugInfo;
+	IMG_BOOL                bEnablePFDebug;      /*!< EnablePageFaultDebug AppHint setting for device */
+
+	DLLIST_NODE             sCleanupThreadWorkList; /*!< List of work for the cleanup thread associated with the device */
+	ATOMIC_T                i32NumCleanupItems;   /*!< Number of cleanup thread work items. Includes items being freed. */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Data for the deferred freeing of a PMR physical pages for a given device */
+	DLLIST_NODE             sPMRZombieList;       /*!< List of PMRs to free */
+	POS_LOCK                hPMRZombieListLock;   /*!< List lock */
+	IMG_UINT32              uiPMRZombieCount;     /*!< Number of elements in the list */
+	IMG_UINT32              uiPMRZombieCountInCleanup; /*!< Number of elements in cleanup items */
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+	ATOMIC_T                eFrozen;              /*< Frozen / Unfrozen indicator */
+	IMG_HANDLE              hDeviceThreadEvObj;   /*< Event Object for Freeze indicator */
+	IMG_HANDLE              hDeviceFreezeThaw;    /*< Event handle for Freeze/Thaw */
+	POS_LOCK                hFreezeThawLock;      /*< Freeze/Thaw lock */
+	ATOMIC_T                iFreezeCount;         /*< Number of blocked on frozen tasks */
+	ATOMIC_T                iTotalFreezes;        /*< Total number of times device frozen */
+	ATOMIC_T                iThreadsActive;       /*< Number of threads active on this device */
+	IMG_UINT64              ui64LastDeviceOffTimestamp; /* Last device power off timestamp */
+	IMG_UINT64              ui64LastDeviceOffHostTimestampNs; /* Last device power off host timestamp */
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	IMG_BOOL bGPUWorkPeriodFTraceEnabled;
+#endif
+} PVRSRV_DEVICE_NODE;
+
+/*
+ * Macros to be used instead of calling directly the pfns since these macros
+ * will expand the feature passed as argument into the bitmask/index to work
+ * with the macros defined in rgx_bvnc_defs_km.h
+ */
+#define PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, Feature) \
+		psDevNode->pfnCheckDeviceFeature(psDevNode, RGX_FEATURE_##Feature##_BIT_MASK)
+#define PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, Feature) \
+		psDevNode->pfnGetDeviceFeatureValue(psDevNode, RGX_FEATURE_##Feature##_IDX)
+
+PVRSRV_ERROR PVRSRVDeviceFinalise(PVRSRV_DEVICE_NODE *psDeviceNode,
+											   IMG_BOOL bInitSuccessful);
+
+PVRSRV_ERROR PVRSRVDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE * psDeviceNode, IMG_UINT32 ui32ClientBuildOptions);
+
+void PVRSRVDeviceSetState(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_STATE eNewDevState);
+
+#define PVRSRVIsStatusRecoverable(eStatus) \
+	(((eStatus == PVRSRV_DEVICE_HEALTH_STATUS_DEAD)) ? \
+	 IMG_FALSE : IMG_TRUE)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+/* Determines if a 32-bit `uiCurrent` counter advanced to or beyond
+ * `uiRequired` value. The function takes into consideration that the
+ * counter could have wrapped around. */
+static INLINE IMG_BOOL PVRSRVHasCounter32Advanced(IMG_UINT32 uiCurrent,
+                                                  IMG_UINT32 uiRequired)
+{
+	return uiCurrent >= uiRequired ?
+	    /* ... with the counter wrapped around ...
+	     * There can't be ~4 billion transactions completed, so consider wrapped */
+	    (((uiCurrent - uiRequired) > 0xF0000000UL) ? IMG_FALSE : IMG_TRUE) :
+	    /* There can't be ~4 billion transactions pending, so consider wrapped */
+	    (((uiRequired - uiCurrent) > 0xF0000000UL) ? IMG_TRUE : IMG_FALSE);
+}
+#endif
+
+#endif /* DEVICE_H */
+
+/******************************************************************************
+ End of file (device.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_heapcfg.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_heapcfg.h
new file mode 100644
index 000000000000..3b8c0aa08921
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_heapcfg.h
@@ -0,0 +1,213 @@
+/**************************************************************************/ /*!
+@File
+@Title          Device Heap Configuration Helper Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DEVICEMEMHEAPCFG_H
+#define DEVICEMEMHEAPCFG_H
+
+#include <powervr/mem_types.h>
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+struct _PVRSRV_DEVICE_NODE_;
+struct _CONNECTION_DATA_;
+struct _DEVMEMINT_HEAP_;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_HEAP_INIT
+@Description    Device heap initialisation function. Called in server devmem
+                heap create if the callback pointer in RGX_HEAP_INFO is
+                not NULL.
+@Input          psDeviceNode       The device node.
+@Input          psDevmemHeap       Server internal devmem heap.
+@Output         phPrivData         Private data handle. Allocated resources
+                                   can be freed in PFN_HEAP_DEINIT.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_HEAP_INIT)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                                      struct _DEVMEMINT_HEAP_ *psDevmemHeap,
+                                      IMG_HANDLE *phPrivData);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_HEAP_DEINIT
+@Description    Device heap deinit function. Called in server devmem
+                heap create if the callback pointer in RGX_HEAP_INFO is
+                not NULL.
+@Input          hPrivData          Private data handle. To free any resources.
+*/ /**************************************************************************/
+typedef void (*PFN_HEAP_DEINIT)(IMG_HANDLE hPrivData);
+
+/*
+  A "heap config" is a blueprint to be used for initial setting up of heaps
+  when a device memory context is created.
+
+  We define a data structure to define this, but it's really down to the
+  caller to populate it. This is all expected to be in-kernel. We provide an
+  API that client code can use to enquire about the blueprint, such that it may
+  do the heap set-up during the context creation call on behalf of the user.
+*/
+
+/* Blueprint for a single heap */
+typedef struct _DEVMEM_HEAP_BLUEPRINT_
+{
+	/* Name of this heap - for debug purposes, and perhaps for lookup
+	by name */
+	const IMG_CHAR *pszName;
+
+	/* Virtual address of the beginning of the heap.  This _must_ be a
+	multiple of the data page size for the heap.  It is
+	_recommended_ that it be coarser than that - especially, it
+	should begin on a boundary appropriate to the MMU for the
+	device.  For Rogue, this is a Page Directory boundary, or 1GB
+	(virtual address a multiple of 0x0040000000). */
+	IMG_DEV_VIRTADDR sHeapBaseAddr;
+
+	/* Length of the heap.  Given that the END address of the heap has
+	a similar restriction to that of the _beginning_ of the heap.
+	That is the heap length _must_ be a whole number of data pages.
+	Again, the recommendation is that it ends on a 1GB boundary.
+	Again, this is not essential, but we do know that (at the time
+	of writing) the current implementation of mmu_common.c is such
+	that no two heaps may share a page directory, thus the
+	remaining virtual space would be wasted if the length were not
+	a multiple of 1GB */
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+
+	/* VA space starting sHeapBaseAddr to uiReservedRegionLength-1 are reserved
+	for statically defined addresses (shared/known between clients and FW).
+	Services never maps allocations into this reserved address space _unless_
+	explicitly requested via PVRSRVMapToDeviceAddress by passing sDevVirtAddr
+	which falls within this reserved range. Since this range is completely for
+	clients to manage (where allocations are page granular), it _must_ again be
+	a whole number of data pages. Additionally, another constraint enforces this
+	to be a multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY (which evaluates to
+	max page size supported) to support varied pages sizes */
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+
+	/* Data page size.  This is the page size that is going to get
+	programmed into the MMU, so it needs to be a valid one for the
+	device.  Importantly, the start address and length _must_ be
+	multiples of this page size.  Note that the page size is
+	specified as the log 2 relative to 1 byte (e.g. 12 indicates
+	4kB) */
+	IMG_UINT32 uiLog2DataPageSize;
+
+	/* Import alignment.  Force imports to this heap to be
+	aligned to at least this value */
+	IMG_UINT32 uiLog2ImportAlignment;
+
+	/* Callback function for device specific heap init. */
+	PFN_HEAP_INIT pfnInit;
+
+	/* Callback function for device specific heap deinit. */
+	PFN_HEAP_DEINIT pfnDeInit;
+
+} DEVMEM_HEAP_BLUEPRINT;
+
+void HeapCfgBlueprintInit(const IMG_CHAR        *pszName,
+	                      IMG_UINT64             ui64HeapBaseAddr,
+	                      IMG_DEVMEM_SIZE_T      uiHeapLength,
+	                      IMG_DEVMEM_SIZE_T      uiReservedRegionLength,
+	                      IMG_UINT32             ui32Log2DataPageSize,
+	                      IMG_UINT32             uiLog2ImportAlignment,
+	                      PFN_HEAP_INIT          pfnInit,
+	                      PFN_HEAP_DEINIT        pfnDeInit,
+	                      DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint);
+
+/* Entire named heap config */
+typedef struct _DEVMEM_HEAP_CONFIG_
+{
+    /* Name of this heap config - for debug and maybe lookup */
+    const IMG_CHAR *pszName;
+
+    /* Number of heaps in this config */
+    IMG_UINT32 uiNumHeaps;
+
+    /* Array of individual heap blueprints as defined above */
+    DEVMEM_HEAP_BLUEPRINT *psHeapBlueprintArray;
+} DEVMEM_HEAP_CONFIG;
+
+
+PVRSRV_ERROR
+HeapCfgHeapConfigCount(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 *puiNumHeapConfigsOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapCount(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 *puiNumHeapsOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapConfigName(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 uiHeapConfigNameBufSz,
+    IMG_CHAR *pszHeapConfigNameOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapDetails(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 uiHeapIndex,
+    IMG_UINT32 uiHeapNameBufSz,
+    IMG_CHAR *pszHeapNameOut,
+    IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+    IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+    IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+    IMG_UINT32 *puiLog2DataPageSizeOut,
+    IMG_UINT32 *puiLog2ImportAlignmentOut
+);
+
+PVRSRV_ERROR
+HeapCfgGetCallbacks(const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    PFN_HEAP_INIT *ppfnInit,
+                    PFN_HEAP_DEINIT *ppfnDeinit);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_history_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_history_server.h
new file mode 100644
index 000000000000..81fe702371ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_history_server.h
@@ -0,0 +1,167 @@
+/*************************************************************************/ /*!
+@File           devicemem_history_server.h
+@Title          Resource Information abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Devicemem History functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_HISTORY_SERVER_H
+#define DEVICEMEM_HISTORY_SERVER_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxmem.h"
+#include "devicemem_utils.h"
+#include "connection_server.h"
+
+PVRSRV_ERROR DevicememHistoryInitKM(void);
+
+void DevicememHistoryDeInitKM(void);
+
+PVRSRV_ERROR DevicememHistoryDeviceInit(PVRSRV_DEVICE_NODE *psDevNode);
+PVRSRV_ERROR DevicememHistoryDeviceCreate(PVRSRV_DEVICE_NODE *psDevNode);
+void DevicememHistoryDeviceDestroy(PVRSRV_DEVICE_NODE *psDevNode);
+
+PVRSRV_ERROR DevicememHistoryMapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryUnmapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryMapVRangeKM(CONNECTION_DATA *psConnection,
+							PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							IMG_DEVMEM_SIZE_T uiAllocSize,
+							const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *ui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(CONNECTION_DATA *psConnection,
+							PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							IMG_DEVMEM_SIZE_T uiAllocSize,
+							const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *ui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocPageCount,
+							IMG_UINT32 *paui32AllocPageIndices,
+							IMG_UINT32 ui32FreePageCount,
+							IMG_UINT32 *pauiFreePageIndices,
+							IMG_UINT32 AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+/* used when the PID does not matter */
+#define DEVICEMEM_HISTORY_PID_ANY 0xFFFFFFFE
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_IN_
+{
+	IMG_PID uiPID;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_DEVICE_NODE *psDevNode;
+} DEVICEMEM_HISTORY_QUERY_IN;
+
+/* Store up to 4 results for a lookup. In the case of the faulting page being
+ * re-mapped between the page fault occurring on HW and the page fault analysis
+ * being done, the second result entry will show the allocation being unmapped.
+ * A further 2 entries are added to cater for multiple buffers in the same page.
+ */
+#define DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS 4
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_OUT_RESULT_
+{
+	IMG_CHAR szString[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_BOOL bMap;
+	IMG_BOOL bRange;
+	IMG_BOOL bAll;
+	IMG_UINT64 ui64When;
+	IMG_UINT64 ui64Age;
+	/* info for sparse map/unmap operations (i.e. bRange=IMG_TRUE) */
+	IMG_UINT32 ui32StartPage;
+	IMG_UINT32 ui32PageCount;
+	IMG_DEV_VIRTADDR sMapStartAddr;
+	IMG_DEV_VIRTADDR sMapEndAddr;
+	RGXMEM_PROCESS_INFO sProcessInfo;
+} DEVICEMEM_HISTORY_QUERY_OUT_RESULT;
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_OUT_
+{
+	IMG_UINT32 ui32NumResults;
+	IMG_UINT64 ui64SearchCount;
+	/* result 0 is the newest */
+	DEVICEMEM_HISTORY_QUERY_OUT_RESULT sResults[DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS];
+} DEVICEMEM_HISTORY_QUERY_OUT;
+
+void DevicememHistoryDumpRecordStats(PVRSRV_DEVICE_NODE *psDevNode,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile);
+
+IMG_BOOL
+DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
+                      DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+                      IMG_UINT32 ui32PageSizeBytes,
+                      IMG_BOOL bMatchAnyAllocInPage);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server.h
new file mode 100644
index 000000000000..8e000c663970
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server.h
@@ -0,0 +1,786 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server side component for device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_SERVER_H
+#define DEVICEMEM_SERVER_H
+
+#include "device.h" /* For device node */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "connection_server.h"
+#include "pmr.h"
+
+typedef struct _DEVMEMINT_CTX_ DEVMEMINT_CTX;
+typedef struct _DEVMEMINT_CTX_EXPORT_ DEVMEMINT_CTX_EXPORT;
+typedef struct _DEVMEMINT_HEAP_ DEVMEMINT_HEAP;
+
+typedef struct _DEVMEMINT_RESERVATION_ DEVMEMINT_RESERVATION;
+typedef struct _DEVMEMXINT_RESERVATION_ DEVMEMXINT_RESERVATION;
+typedef struct _DEVMEMINT_PF_NOTIFY_ DEVMEMINT_PF_NOTIFY;
+
+/*
+ * DevmemServerGetImportHandle()
+ *
+ * For given exportable memory descriptor returns PMR handle
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+                            IMG_HANDLE *phImport);
+
+/*
+ * DevmemServerGetHeapHandle()
+ *
+ * For given reservation returns the Heap handle
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetHeapHandle(DEVMEMINT_RESERVATION *psReservation,
+                          IMG_HANDLE *phHeap);
+
+/*
+ * DevmemServerGetContext()
+ *
+ * For given heap returns the context.
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetContext(DEVMEMINT_HEAP *psDevmemHeap,
+                       DEVMEMINT_CTX **ppsDevmemCtxPtr);
+
+/*
+ * DevmemServerGetPrivData()
+ *
+ * For given context returns the private data handle.
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetPrivData(DEVMEMINT_CTX *psDevmemCtx,
+                        IMG_HANDLE *phPrivData);
+
+/*
+ * DevmemIntCtxCreate()
+ *
+ * Create a Server-side Device Memory Context. This is usually the counterpart
+ * of the client side memory context, and indeed is usually created at the
+ * same time.
+ *
+ * You must have one of these before creating any heaps.
+ *
+ * All heaps must have been destroyed before calling
+ * DevmemIntCtxDestroy()
+ *
+ * If you call DevmemIntCtxCreate() (and it succeeds) you are promising to
+ * later call DevmemIntCtxDestroy()
+ *
+ * Note that this call will cause the device MMU code to do some work for
+ * creating the device memory context, but it does not guarantee that a page
+ * catalogue will have been created, as this may be deferred until the first
+ * allocation.
+ *
+ * Caller to provide storage for a pointer to the DEVMEM_CTX object that will
+ * be created by this call.
+ */
+PVRSRV_ERROR
+DevmemIntCtxCreate(CONNECTION_DATA *psConnection,
+                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                   /* devnode / perproc etc */
+                   IMG_BOOL bKernelMemoryCtx,
+                   DEVMEMINT_CTX **ppsDevmemCtxPtr,
+                   IMG_HANDLE *hPrivData,
+                   IMG_UINT32 *pui32CPUCacheLineSize);
+/*
+ * DevmemIntCtxDestroy()
+ *
+ * Undoes a prior DevmemIntCtxCreate or DevmemIntCtxImport.
+ */
+PVRSRV_ERROR
+DevmemIntCtxDestroy(DEVMEMINT_CTX *psDevmemCtx);
+
+/*
+ * DevmemIntHeapCreate()
+ *
+ * Creates a new heap in this device memory context.  This will cause a call
+ * into the MMU code to allocate various data structures for managing this
+ * heap. It will not necessarily cause any page tables to be set up, as this
+ * can be deferred until first allocation. (i.e. we shouldn't care - it's up
+ * to the MMU code)
+ *
+ * Note that the data page size must be specified (as log 2). The data page
+ * size as specified here will be communicated to the mmu module, and thus may
+ * determine the page size configured in page directory entries for subsequent
+ * allocations from this heap. It is essential that the page size here is less
+ * than or equal to the "minimum contiguity guarantee" of any PMR that you
+ * subsequently attempt to map to this heap.
+ *
+ * If you call DevmemIntHeapCreate() (and the call succeeds) you are promising
+ * that you shall subsequently call DevmemIntHeapDestroy()
+ *
+ * Caller to provide storage for a pointer to the DEVMEM_HEAP object that will
+ * be created by this call.
+ */
+PVRSRV_ERROR
+DevmemIntHeapCreate(DEVMEMINT_CTX *psDevmemCtx,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    DEVMEMINT_HEAP **ppsDevmemHeapPtr);
+/*
+ * DevmemIntHeapDestroy()
+ *
+ * Destroys a heap previously created with DevmemIntHeapCreate()
+ *
+ * All allocations from his heap must have been freed before this
+ * call.
+ */
+PVRSRV_ERROR
+DevmemIntHeapDestroy(DEVMEMINT_HEAP *psDevmemHeap);
+
+/* DevmemIntHeapGetBaseAddr()
+ *
+ * Get heap base address pre carveouts.
+ */
+IMG_DEV_VIRTADDR
+DevmemIntHeapGetBaseAddr(DEVMEMINT_HEAP *psDevmemHeap);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntReserveRange()
+ * @Description Reserves a number of virtual addresses starting sReservationVAddr
+ *              and continuing until sReservationVAddr + uiVirtualSize - 1.
+ *
+ *              If you call DevmemIntReserveRange() (and the call succeeds)
+ *              then you are promising that you shall later call DevmemIntUnreserveRange()
+ *
+ * @Input       psConnectionData    The connection data from the bridge. Used
+ *                                  to determine where the call to this function
+ *                                  originated from.
+ * @Input       psDeviceNode        The device node (unused).
+ * @Input       psDevmemHeap        The virtual heap the DevVAddr is within.
+ * @Input       sReservationVAddr   The first virtual address of the range.
+ * @Input       uiVirtualSize       The number of bytes in the virtual range.
+ * @Input       uiFlags             Mem alloc flags
+ * @Output      ppsReservationPtr   A pointer to the created reservation.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntReserveRange(CONNECTION_DATA *psConnectionData,
+                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                      DEVMEMINT_HEAP *psDevmemHeap,
+                      IMG_DEV_VIRTADDR sReservationVAddr,
+                      IMG_DEVMEM_SIZE_T uiVirtualSize,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      DEVMEMINT_RESERVATION **ppsReservationPtr);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntUnreserveRange()
+ * @Description Unreserves the specified virtual range. In the case that the
+ *              virtual range has not been unmapped, it will be unmapped.
+ *              If any references are held on the reservation PVRSRV_ERROR_RETRY
+ *              will be returned.
+ *
+ * @Input       psDevmemReservation The reservation to unreserve
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntUnreserveRange(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntMapPMR
+ *
+ * @Description Maps the given PMR to the virtual range previously reserved with
+ *              DevmemIntReserveRange(). When calling this function, the reservation
+ *              must be valid, and not mapped. Additionally, the PMRs logical
+ *              size and the reservations virtual size must be equal.
+ *
+ *              If appropriate, the PMR must have had its physical backing
+ *              committed, as this call will call into the MMU code to set
+ *              up the page tables for this allocation, which shall in turn
+ *              request the physical addresses from the PMR. Alternatively,
+ *              the PMR implementation can choose to do so off the back of
+ *              the "lock" callback, which it will receive as a result
+ *              (indirectly) of this call.
+ *
+ *              If you call DevmemIntMapPMR() (and the call succeeds) then you
+ *              are promising that you shall later call DevmemIntUnmapPMR()
+ *
+ * @Input       psReservation    The reservation the PMR will be mapped into.
+ * @Input       psPMR            The PMR to be mapped.
+ *
+ * @Return      PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntMapPMR(DEVMEMINT_RESERVATION *psReservation, PMR *psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntRemapPageInPMR
+ *
+ * @Description Distributes calls to the MMU module to remap a given PMR
+ *              page offset into all associated mappings.
+ *
+ * @Input       psPMR                The PMR to be mapped.
+ * @Input       psMappingListHead    The mapping node list head where nodes are
+ *                                   associated with the PMR via calls to
+ *                                   PMRLinkGPUMapping.
+ *                                   Expected type:
+ *                                   DLLIST_NODE list head from the PMR
+ *                                   (sGpuMappingListHead)
+ * @Input       ui32LogicalPgOffset  The logical page offset into the
+ *                                   PMR and reservation.
+ *
+ * @Return      PVRSRV_ERROR failure code.
+ *              PVRSRV_ERROR_DEVICEMEM_REJECT_REMAP_REQUEST can be returned
+ *              if remap is not possible on the given page offset.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntRemapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntUnmapPMR()
+ *
+ * @Description Unmaps a previously mapped virtual range.
+ *
+ * @Input       psReservation   The virtual range to unmap.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntUnmapPMR(DEVMEMINT_RESERVATION *psReservation);
+
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntReserveRangeAndMapPMR()
+ *
+ * @Description Reserve (with DevmemIntReserveRange), and map a virtual range
+ *              to a PMR (with DevmemIntMapPMR).
+ *
+ * @Input       psConnectionData    The connection data from the bridge. Used
+ *                                  to determine where the call to this function
+ *                                  originated from.
+ * @Input       psDeviceNode        The device node.
+ * @Input       psDevmemHeap        The virtual heap DevVAddr is within.
+ * @Input       sReservationVAddr   The first virtual address of the range.
+ * @Input       uiVirtualSize       The number of bytes in the virtual range.
+ * @Input       psPMR               The PMR to be mapped.
+ * @Input       uiFlags             Mem alloc flags
+ * @Output      ppsReservation      A pointer to the created reservation.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntReserveRangeAndMapPMR(CONNECTION_DATA *psConnectionData,
+                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                               DEVMEMINT_HEAP *psDevmemHeap,
+                               IMG_DEV_VIRTADDR sReservationVAddr,
+                               IMG_DEVMEM_SIZE_T uiVirtualSize,
+                               PMR *psPMR,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               DEVMEMINT_RESERVATION **ppsReservation);
+
+/*************************************************************************/ /*!
+ * @Function       DevmemIntChangeSparse
+ * @Description    Changes the sparse allocations of a PMR by allocating and freeing
+ *                 pages and changing their corresponding GPU mapping.
+ *
+ *                 Prior to calling this function DevmemIntMapPMR
+ *                 or DevmemIntReserveRangeAndMapPMR must be used.
+ *
+ * @Input          psReservation         The reservation that the PMR is mapped to.
+ * @Input          ui32AllocPageCount    Number of pages to allocate
+ * @Input          pai32AllocIndices     The logical PMR indices where pages will
+ *                                       be allocated. May be NULL.
+ * @Input          ui32FreePageCount     Number of pages to free
+ * @Input          pai32FreeIndices      The logical PMR indices where pages will
+ *                                       be freed. May be NULL.
+ * @Input          uiSparseFlags         Flags passed in to determine which kind
+ *                                       of sparse change the user wanted.
+ *                                       See devicemem_typedefs.h for details.
+ * @Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntChangeSparse(IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiSparseFlags,
+                      DEVMEMINT_RESERVATION *psReservation);
+
+PVRSRV_ERROR
+DevmemIntGetReservationData(DEVMEMINT_RESERVATION* psReservation, PMR** ppsPMR, IMG_DEV_VIRTADDR* psDevVAddr);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntReserveRange()
+ * @Description Indicates that the specified range should be reserved from the
+ *              given heap.
+ *
+ *              In turn causes the page tables to be allocated to cover the
+ *              specified range.
+ *
+ *              If you call DevmemIntReserveRange() (and the call succeeds)
+ *              then you are promising that you shall later call
+ *              DevmemIntUnreserveRange().
+ *
+ * @Input       psDevmemHeap        Pointer to the heap the reservation is made
+ *                                  on
+ * @Input       sReservationVAddr   Virtual address of the reservation
+ * @Input       uiVirtualSize       Size of the reservation (in bytes)
+ * @Input       ppsRsrv             Return pointer to the reservation object
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                       IMG_DEV_VIRTADDR sReservationVAddr,
+                       IMG_DEVMEM_SIZE_T uiVirtualSize,
+                       DEVMEMXINT_RESERVATION **ppsRsrv);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntUnreserveRange()
+ * @Description Undoes the state change caused by DevmemXIntReserveRage()
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntUnreserveRange(DEVMEMXINT_RESERVATION *psRsrv);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationAcquire
+@Description    Acquire a reference to the provided device memory reservation.
+                Prevents releasing of the reservation if external device
+                resource components still require it.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+IMG_BOOL
+DevmemIntReservationAcquire(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationRelease
+@Description    Release the reference to the provided device memory reservation.
+                Once these references have been released the
+                reservation is allowed to be released from UM.
+@Return         None.
+*/ /**************************************************************************/
+void
+DevmemIntReservationRelease(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntMapPages()
+ * @Description Maps an arbitrary amount of pages from a PMR to a reserved range
+ *              and takes references on the PMR.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       psPMR              PMR that is mapped
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 mapped
+ * @Input       uiPhysPageOffset   Logical offset in the PMR (measured in pages)
+ * @Input       uiFlags            Mapping flags
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntMapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                   PMR *psPMR,
+                   IMG_UINT32 uiPageCount,
+                   IMG_UINT32 uiPhysPageOffset,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   IMG_UINT32 uiVirtPageOffset);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntUnmapPages()
+ * @Description Unmaps an arbitrary amount of pages from a reserved range and
+ *              releases references on associated PMRs.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 unmapped
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntUnmapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                     IMG_UINT32 uiVirtPageOffset,
+                     IMG_UINT32 uiPageCount);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntMapVRangeToBackingPage()
+ * @Description Maps a kernel internal backing page to a reserved range.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 mapped
+ * @Input       uiFlags            Mapping flags
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntMapVRangeToBackingPage(DEVMEMXINT_RESERVATION *psRsrv,
+                                 IMG_UINT32 uiPageCount,
+                                 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                 IMG_UINT32 uiVirtPageOffset);
+
+/*
+ * DevmemIntInvalidateFBSCTable()
+ *
+ * Invalidate selected FBSC table indices.
+ *
+ */
+PVRSRV_ERROR
+DevmemIntInvalidateFBSCTable(DEVMEMINT_CTX *psDevmemCtx,
+                             IMG_UINT64 ui64FBSCEntryMask);
+
+PVRSRV_ERROR
+DevmemIntIsVDevAddrValid(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDevNode,
+                         DEVMEMINT_CTX *psDevMemContext,
+                         IMG_DEV_VIRTADDR sDevAddr);
+
+PVRSRV_ERROR
+DevmemIntGetFaultAddress(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDevNode,
+                         DEVMEMINT_CTX *psDevMemContext,
+                         IMG_DEV_VIRTADDR *psFaultAddress);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntRegisterPFNotifyKM
+@Description    Registers a PID to be notified when a page fault occurs on a
+                specific device memory context.
+@Input          psDevmemCtx    The context to be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntRegisterPFNotifyKM(DEVMEMINT_CTX *psDevmemCtx,
+                            IMG_BOOL      bRegister);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntPFNotify
+@Description    Notifies any processes that have registered themselves to be
+                notified when a page fault happens on a specific device memory
+                context.
+@Input          *psDevNode           The device node.
+@Input          ui64FaultedPCAddress The page catalogue address that faulted.
+@Input          sFaultAddress        The address that triggered the fault.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntPFNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                               IMG_UINT64         ui64FaultedPCAddress,
+                               IMG_DEV_VIRTADDR   sFaultAddress);
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize,
+                              DLLIST_NODE *psValidRegionsList);
+
+void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psValidRegionsList);
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * DevmemIntPDumpSaveToFileVirtual()
+ *
+ * Writes out PDump "SAB" commands with the data found in memory at
+ * the given virtual address.
+ */
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 uiArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * DevmemIntPDumpSaveToFileVirtualNoValidate()
+ *
+ * Writes out PDump "SAB" commands with the data found in memory at
+ * the given virtual address. Doesn't perform address validation.
+ */
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          DEVMEMINT_CTX *psDevmemCtx,
+                                          IMG_DEV_VIRTADDR sDevAddrStart,
+                                          IMG_DEVMEM_SIZE_T uiSize,
+                                          const IMG_CHAR *pszFilename,
+                                          IMG_UINT32 ui32FileOffset,
+                                          IMG_UINT32 ui32PDumpFlags);
+
+IMG_UINT32
+DevmemIntMMUContextID(DEVMEMINT_CTX *psDevMemContext);
+
+PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32LogicalWidth,
+                              IMG_UINT32 ui32LogicalHeight,
+                              IMG_UINT32 ui32PhysicalWidth,
+                              IMG_UINT32 ui32PhysicalHeight,
+                              PDUMP_PIXEL_FORMAT ePixFmt,
+                              IMG_MEMLAYOUT eMemLayout,
+                              IMG_FB_COMPRESSION eFBCompression,
+                              const IMG_UINT32 *paui32FBCClearColour,
+                              PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                              IMG_DEV_VIRTADDR sHeader,
+                              IMG_UINT32 ui32HeaderSize,
+                              IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             DEVMEMINT_CTX *psDevMemContext,
+                             IMG_UINT32 ui32Size,
+                             const IMG_CHAR *pszFileName,
+                             IMG_DEV_VIRTADDR sData,
+                             IMG_UINT32 ui32DataSize,
+                             IMG_UINT32 ui32HeaderType,
+                             IMG_UINT32 ui32ElementType,
+                             IMG_UINT32 ui32ElementCount,
+                             IMG_UINT32 ui32PDumpFlags);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpGetValidRegions)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(sDevAddrStart);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpFreeValidRegions)
+#endif
+static INLINE void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psDevAddrRegions)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevAddrRegions);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpSaveFromRegionListToFileVirtual)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(psDevAddrRegions);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpSaveToFileVirtual)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 uiArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(sDevAddrStart);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpImageDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32LogicalWidth,
+                              IMG_UINT32 ui32LogicalHeight,
+                              IMG_UINT32 ui32PhysicalWidth,
+                              IMG_UINT32 ui32PhysicalHeight,
+                              PDUMP_PIXEL_FORMAT ePixFmt,
+                              IMG_MEMLAYOUT eMemLayout,
+                              IMG_FB_COMPRESSION eFBCompression,
+                              const IMG_UINT32 *paui32FBCClearColour,
+                              PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                              IMG_DEV_VIRTADDR sHeader,
+                              IMG_UINT32 ui32HeaderSize,
+                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevMemContext);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	PVR_UNREFERENCED_PARAMETER(pszFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpDataDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32ElementType,
+                              IMG_UINT32 ui32ElementCount,
+                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevMemContext);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	PVR_UNREFERENCED_PARAMETER(pszFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#endif /* PDUMP */
+
+PVRSRV_ERROR
+DevmemIntInit(void);
+
+PVRSRV_ERROR
+DevmemIntDeInit(void);
+
+PVRSRV_ERROR
+DevmemIntExportCtx(DEVMEMINT_CTX *psContext,
+                   PMR *psPMR,
+                   DEVMEMINT_CTX_EXPORT **ppsContextExport);
+
+PVRSRV_ERROR
+DevmemIntUnexportCtx(DEVMEMINT_CTX_EXPORT *psContextExport);
+
+PVRSRV_ERROR
+DevmemIntAcquireRemoteCtx(PMR *psPMR,
+                          DEVMEMINT_CTX **ppsContext,
+                          IMG_HANDLE *phPrivData);
+
+#endif /* DEVICEMEM_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server_utils.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server_utils.h
new file mode 100644
index 000000000000..923bfc9a8f18
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/devicemem_server_utils.h
@@ -0,0 +1,179 @@
+/**************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header file utilities that are specific to device memory functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv.h"
+
+static INLINE PVRSRV_ERROR DevmemCPUCacheMode(PVRSRV_MEMALLOCFLAGS_T ulFlags,
+											  IMG_UINT32 *pui32Ret)
+{
+	IMG_UINT32 ui32CPUCacheMode = PVRSRV_CPU_CACHE_MODE(ulFlags);
+	IMG_UINT32 ui32Ret;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32CPUCacheMode == PVRSRV_CPU_CACHE_MODE(ulFlags));
+
+	switch (ui32CPUCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_CACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT:
+
+			/*
+			 * If system has no coherency but coherency has been requested for CPU
+			 * and GPU we currently fall back to write-combine.
+			 * This avoids errors on arm64 when uncached is turned into ordered device memory
+			 * and suffers from problems with unaligned access.
+			 */
+			if (PVRSRV_GPU_CACHE_MODE(ulFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+			}
+			else
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_CACHED;
+			}
+			break;
+
+		default:
+			PVR_LOG(("DevmemCPUCacheMode: Unknown CPU cache mode 0x%08x", ui32CPUCacheMode));
+			PVR_ASSERT(0);
+			/*
+				We should never get here, but if we do then setting the mode
+				to uncached is the safest thing to do.
+			*/
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED;
+			eError = PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+			break;
+	}
+
+	*pui32Ret = ui32Ret;
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR DevmemDeviceCacheMode(PVRSRV_MEMALLOCFLAGS_T ulFlags,
+												 IMG_UINT32 *pui32Ret)
+{
+	IMG_UINT32 ui32DeviceCacheMode = PVRSRV_GPU_CACHE_MODE(ulFlags);
+	IMG_UINT32 ui32Ret;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32DeviceCacheMode == PVRSRV_GPU_CACHE_MODE(ulFlags));
+
+	switch (ui32DeviceCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_CACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT:
+
+			/*
+			 * If system has no coherency but coherency has been requested for CPU
+			 * and GPU we currently fall back to write-combine.
+			 * This avoids errors on arm64 when uncached is turned into ordered device memory
+			 * and suffers from problems with unaligned access.
+			 */
+			if (PVRSRV_CPU_CACHE_MODE(ulFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC;
+			}
+			else
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_CACHED;
+			}
+			break;
+
+		default:
+			PVR_LOG(("DevmemDeviceCacheMode: Unknown device cache mode 0x%08x", ui32DeviceCacheMode));
+			PVR_ASSERT(0);
+			/*
+				We should never get here, but if we do then setting the mode
+				to uncached is the safest thing to do.
+			*/
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED;
+			eError = PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+			break;
+	}
+
+	*pui32Ret = ui32Ret;
+
+	return eError;
+}
+
+static INLINE IMG_BOOL DevmemDeviceCacheCoherency(PVRSRV_DEVICE_NODE *psDeviceNode,
+												  PVRSRV_MEMALLOCFLAGS_T ulFlags)
+{
+	IMG_UINT32 ui32DeviceCacheMode = PVRSRV_GPU_CACHE_MODE(ulFlags);
+	IMG_BOOL bRet = IMG_FALSE;
+
+	PVR_ASSERT(ui32DeviceCacheMode == PVRSRV_GPU_CACHE_MODE(ulFlags));
+
+	if (ui32DeviceCacheMode == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+	{
+		bRet = PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig);
+	}
+	return bRet;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/di_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_common.h
new file mode 100644
index 000000000000..a10178708a75
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_common.h
@@ -0,0 +1,236 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common types for Debug Info framework.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DI_COMMON_H
+#define DI_COMMON_H
+
+#include "img_types.h"
+
+/* Token that signals that a header should be printed. */
+#define DI_START_TOKEN ((void *) 1)
+
+/* This is a public handle to an entry. */
+#ifndef DI_GROUP_DEFINED
+#define DI_GROUP_DEFINED
+typedef struct DI_GROUP DI_GROUP;
+#endif
+#ifndef DI_ENTRY_DEFINED
+#define DI_ENTRY_DEFINED
+typedef struct DI_ENTRY DI_ENTRY;
+#endif
+typedef struct OSDI_IMPL_ENTRY OSDI_IMPL_ENTRY;
+
+/*! Debug Info entries types. */
+typedef enum DI_ENTRY_TYPE
+{
+    DI_ENTRY_TYPE_GENERIC,          /*!< generic entry type, implements
+                                         start/stop/next/show iterator
+                                         interface */
+    DI_ENTRY_TYPE_RANDOM_ACCESS,    /*!< random access entry, implements
+                                         seek/read iterator interface */
+} DI_ENTRY_TYPE;
+
+/*! @Function DI_PFN_START
+ *
+ * @Description
+ * Start operation returns first entry and passes it to Show operation.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @InOut pui64Pos current data position in the entry
+ *
+ * @Return pointer to data that will be passed to the other iterator
+ *         functions in pvData argument
+ */
+typedef void *(*DI_PFN_START)(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos);
+
+/*! @Function DI_PFN_STOP
+ *
+ * @Description
+ * Stop operations is called after iterator reaches end of data.
+ *
+ * If pvData was allocated in pfnStart it should be freed here.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ */
+typedef void (*DI_PFN_STOP)(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*! @Function DI_PFN_NEXT
+ *
+ * @Description
+ * Next returns next data entry and passes it to Show operation.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ * @InOut pui64Pos current data position in the entry
+ */
+typedef void *(*DI_PFN_NEXT)(OSDI_IMPL_ENTRY *psEntry, void *pvData,
+                             IMG_UINT64 *pui64Pos);
+
+/*! @Function DI_PFN_SHOW
+ *
+ * @Description
+ * Outputs the data element.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ */
+typedef int (*DI_PFN_SHOW)(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*! @Function DI_PFN_SEEK
+ *
+ * @Description
+ * Changes position of the entry data pointer
+ *
+ * @Input uiOffset new entry offset (absolute)
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_SEEK)(IMG_UINT64 ui64Offset, void *pvData);
+
+/*! @Function DI_PFN_READ
+ *
+ * @Description
+ * Retrieves data from the entry from position previously set by Seek.
+ *
+ * @Input pszBuffer output buffer
+ * @Input ui64Count length of the output buffer
+ * @InOut pui64Pos pointer to the current position in the entry
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_READ)(IMG_CHAR *pszBuffer, IMG_UINT64 ui64Count,
+                                 IMG_UINT64 *pui64Pos, void *pvData);
+
+/*! @Function DI_PFN_WRITE
+ *
+ * @Description
+ * Handle writes operation to the entry.
+ *
+ * @Input pszBuffer NUL-terminated buffer containing written data
+ * @Input ui64Count length of the data in pszBuffer (length of the buffer)
+ * @InOut pui64Pos pointer to the current position in the entry
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_WRITE)(const IMG_CHAR *pszBuffer,
+                                  IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                                  void *pvData);
+
+/*! Debug info entry iterator.
+ *
+ * This covers all entry types: GENERIC and RANDOM_ACCESS.
+ *
+ * The GENERIC entry type
+ *
+ * The GENERIC type should implement either a full set of following callbacks:
+ * pfnStart, pfnStop, pfnNext and pfnShow, or pfnShow only. If only pfnShow
+ * callback is given the framework will use default handlers in place of the
+ * other ones.
+ *
+ * e.g. for generic entry:
+ *
+ *   struct sIter = {
+ *     .pfnStart = StartCb, .pfnStop = StopCb, pfnNext = NextCb,
+ *     .pfnShow = ShowCb
+ *   };
+ *
+ * The use case for implementing pfnShow only is if the data for the given
+ * entry is short and can be printed in one go because the pfnShow callback
+ * will be called only once.
+ *
+ * e.g. for one-shot print generic entry:
+ *
+ *   struct sIter = {
+ *     .pfnShow = SingleShowCb
+ *   };
+ *
+ * The DICreateEntry() function will return error if DI_ENTRY_TYPE_GENERIC
+ * type is used and invalid combination of callbacks is given.
+ *
+ * The RANDOM_ACCESS entry
+ *
+ * The RANDOM_ACCESS type should implement either both pfnSeek and pfnRead
+ * or pfnRead only callbacks.
+ *
+ * e.g. of seekable and readable random access entry:
+ *
+ *   struct sIter = {
+ *     .pfnSeek = SeekCb, .pfnRead = ReadCb
+ *   };
+ *
+ * The DICreateEntry() function will return error if DI_ENTRY_TYPE_RANDOM_ACCESS
+ * type is used and invalid combination of callbacks is given.
+ *
+ * Writing to file (optional)
+ *
+ * The iterator allows also to pass a pfnWrite callback that allows implementing
+ * write operation on the entry. The write operation is entry type agnostic
+ * which means that it can be defined for both GENERIC and RANDOM_ACCESS
+ * entries.
+ *
+ * e.g. for writable one-shot print generic entry
+ *
+ *   struct sIter = {
+ *     .pfnShow = SingleShowCb, .pfnWrite = WriteCb
+ *   };
+ */
+typedef struct DI_ITERATOR_CB
+{
+    /* Generic entry interface. */
+
+    DI_PFN_START pfnStart; /*!< Starts iteration and returns first element
+                                of entry's data. */
+    DI_PFN_STOP pfnStop;   /*!< Stops iteration. */
+    DI_PFN_NEXT pfnNext;   /*!< Returns next element of entry's data. */
+    DI_PFN_SHOW pfnShow;   /*!< Shows current data element of an entry. */
+
+    /* Optional random access entry interface. */
+
+    DI_PFN_SEEK pfnSeek;   /*!< Sets data pointer in an entry. */
+    DI_PFN_READ pfnRead;   /*!< Reads data from an entry. */
+
+    /* Optional writing to entry interface. Null terminated. */
+
+    DI_PFN_WRITE pfnWrite; /*!< Performs write operation on an entry. */
+    IMG_UINT32   ui32WriteLenMax; /*!< Maximum char length of entry
+                                       accepted for write. Includes \0 */
+} DI_ITERATOR_CB;
+
+#endif /* DI_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/di_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_server.h
new file mode 100644
index 000000000000..a797890c069a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/di_server.h
@@ -0,0 +1,220 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for creating Debug Info groups and entries.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DI_SERVER_H
+#define DI_SERVER_H
+
+#if defined(__KERNEL__) && defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __KERNEL__ && __linux__ */
+
+#include "di_common.h"
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+/*! @Function DIInit
+ *
+ * @Description
+ * Initialises Debug Info framework. This function will create common resources
+ * for the framework.
+ *
+ * Note: This function must be called before first call to
+ *       DIRegisterImplementation() all of the implementations.
+ */
+PVRSRV_ERROR DIInit(void);
+
+/*! @Function DIDeInit
+ *
+ * @Description
+ * De-initialises Debug Info framework. This function will call pfnDeInit()
+ * on each implementation and clean up common resources.
+ *
+ * In case some of the entries and groups have not been cleaned up this function
+ * will also perform recursive sweep and remove all entries and group for
+ * all implementations.
+ */
+void DIDeInit(void);
+
+/*! @Function DICreateEntry
+ *
+ * @Description
+ * Creates debug info entry. Depending on different implementations the entry
+ * might be for example a DebugFS file or something totally different.
+ *
+ * The entry will belong to a parent group if provided or to the root group
+ * if not.
+ *
+ * @Input pszName: name of the new entry
+ * @Input psDiGroup: parent group, if NULL entry will belong to the root group
+ * @Input psIterCb: implementation of the iterator for the entry
+ * @Input psPriv: private data that will be passed to the iterator operations
+ * @Input eType: type of the entry
+ *
+ * @Output ppsEntry: handle to the newly created entry
+ *
+ * @Return   PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DICreateEntry(const IMG_CHAR *pszName,
+                           DI_GROUP *psGroup,
+                           const DI_ITERATOR_CB *psIterCb,
+                           void *psPriv,
+                           DI_ENTRY_TYPE eType,
+                           DI_ENTRY **ppsEntry);
+
+/*! @Function DIDestroyEntry
+ *
+ * @Description
+ * Destroys debug info entry.
+ *
+ * @Input psEntry: handle to the entry
+ */
+void DIDestroyEntry(DI_ENTRY *psEntry);
+
+/*! @Function DICreateGroup
+ *
+ * @Description
+ * Creates debug info group. Depending on different implementations the group
+ * might be for example a DebugFS directory or something totally different.
+ *
+ * The group will belong to a parent group if provided or to the root group
+ * if not.
+ *
+ * @Input pszName: name of the new entry
+ * @Input psParent: parent group, if NULL entry will belong to the root group
+ *
+ * @Output ppsGroup: handle to the newly created entry
+ *
+ * @Return   PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DICreateGroup(const IMG_CHAR *pszName,
+                           DI_GROUP *psParent,
+                           DI_GROUP **ppsGroup);
+
+/*! @Function DIDestroyGroup
+ *
+ * @Description
+ * Destroys debug info group.
+ *
+ * @Input psGroup: handle to the group
+ */
+void DIDestroyGroup(DI_GROUP *psGroup);
+
+/*! @Function DIGetPrivData
+ *
+ * @Description
+ * Retrieves private data from psEntry. The data is either passed during
+ * entry creation via psPriv parameter of DICreateEntry() function
+ * or by explicitly setting it with DIGetPrivData() function.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ *
+ * @Returns pointer to the private data (can be NULL if private data
+ *          has not been specified)
+ */
+void *DIGetPrivData(const OSDI_IMPL_ENTRY *psEntry);
+
+/*! @Function DIWrite
+ *
+ * @Description
+ * Writes the binary data of the DI entry to the output sync, whatever that may
+ * be for the DI implementation.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pvData data
+ * @Input uiSize pvData length
+ */
+void DIWrite(const OSDI_IMPL_ENTRY *psEntry, const void *pvData,
+             IMG_UINT32 uiSize);
+
+/*! @Function DIPrintf
+ *
+ * @Description
+ * Prints formatted string to the DI entry.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated format string
+ */
+__printf(2, 3)
+void DIPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt, ...);
+
+/*! @Function DIVPrintf
+ *
+ * @Description
+ * Prints formatted string to the DI entry. Equivalent to DIPrintf but takes
+ * va_list instead of a variable number of arguments.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated format string
+ * @Input pArgs vs_list object
+ */
+__printf(2, 0)
+void DIVPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt,
+               va_list pArgs);
+
+/*! @Function DIPrintf
+ *
+ * @Description
+ * Prints a string to the DI entry.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated string
+ */
+void DIPuts(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszStr);
+
+/*! @Function DIHasOverflowed
+ *
+ * @Description
+ * Checks if the DI buffer has overflowed.
+ *
+ * @Return IMG_TRUE if buffer overflowed
+ */
+IMG_BOOL DIHasOverflowed(const OSDI_IMPL_ENTRY *psEntry);
+
+#endif /* DI_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/dkp_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/dkp_impl.h
new file mode 100644
index 000000000000..b5f47f8910a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/dkp_impl.h
@@ -0,0 +1,163 @@
+/**************************************************************************/ /*!
+@File           dkp_impl.h
+@Title          Functions for supporting the DRM Key Provider
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DKP_IMPL_H
+#define DKP_IMPL_H
+
+typedef IMG_HANDLE PVRDKF_DKP_HANDLE;
+typedef PVRDKF_DKP_HANDLE * PPVRDKF_DKP_HANDLE;
+
+/*! @Function DKP_PFN_SHOW
+ *
+ * @Description
+ *
+ * Describes the function called by the DKF infrastructure to request the DKP
+ * outputs all the mandatory and optional keys it supports. Each
+ * key / value pair should be output one per line using the provided
+ * PVRDPFOutput routine.
+ *
+ * The function should check that the passed 'psDevNode' references the
+ * same device as that associated with the module-specific 'hPrivData' handle.
+ * If it doesn't, no data should be displayed.
+ *
+ * @Input psDevNode   Pointer to device node for which data must be shown.
+ * @Input pid         Process-ID for which data must be shown.
+ * @Input hPrivData   Private data passed to DKF from DKP_Register. The DKP
+ *                     must use this within calls to the DKP_PFN_SHOW to limit
+ *                     the data key/value pairs to only those that are relevant
+ *                     to this instance.
+ */
+typedef void (DKP_PFN_SHOW)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, int pid,
+                            IMG_HANDLE hPrivData);
+
+typedef IMG_UINT32 DKP_CONNECTION_FLAGS;
+
+#define DKP_CONNECTION_FLAG_SYNC        BIT(0)
+#define DKP_CONNECTION_FLAG_SERVICES    BIT(1)
+
+#define DKP_CONNECTION_FLAG_INVALID     IMG_UINT32_C(0)
+#define DKP_CONNECTION_FLAG_ALL (DKP_CONNECTION_FLAG_SYNC | DKP_CONNECTION_FLAG_SERVICES)
+
+/* @Function PVRSRVRegisterDKP
+ *
+ * @Description
+ * Registers a DKP with the specified module's DKF entry.
+ *
+ * @Input   hPrivData               DKP Private data handle that the DKF will pass into
+ *                                  the ShowPFN when called. Can be NULL if there is no
+ *                                  DKP instance private data.
+ * @Input   pszDKPName              Provider name associated with the caller.
+ * @Input   psShowPfn               Function to be used when the fdinfo statistics
+ *                                  are queried.
+ * @Input   ui32Filter              The connection types this DKP should be output
+ *                                  on.
+ * @Output  phDkpHandle             Location to store generated handle created by this
+ *                                  function.
+ *
+ * @Returns  PVRSRV_ERROR
+ */
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                               const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle);
+
+/* @Function PVRSRVUnRegisterDKP
+ *
+ * @Description
+ * Removes a previously registered key from the device's DKF entry.
+ *
+ * @Input  hPrivData      Private data handle.
+ * @Input  hDkpHandle     Handle to the DKP's registered entry.
+ *                         Obtained from earlier PVRSRV_DKP_Register call.
+ *
+ * @Returns  PVRSRV_ERROR
+ */
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData,
+                                 PVRDKF_DKP_HANDLE hDkpHandle);
+
+
+/* @Function PVRDKPOutput
+ * Wrapper function which passes the printf-style arguments to the registered
+ * DKF output function associated with the registered DKP handle.
+ *
+ * @Input   hPrivData   Handle for associated DKP instance producing the output.
+ * @Input   fmt         printf-style format string
+ *
+ */
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...) __printf(2, 3);
+
+
+#if !defined(__linux__) || defined(INTEGRITY_OS) || defined(__QNXNTO__)
+
+/* Stub routines follow */
+static inline PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                                             const char *pszDKPName,
+                                             DKP_PFN_SHOW *psShowPfn,
+                                             DKP_CONNECTION_FLAGS ui32Filter,
+                                             PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pszDKPName);
+	PVR_UNREFERENCED_PARAMETER(psShowPfn);
+	PVR_UNREFERENCED_PARAMETER(phDkpHandle);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+static inline PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData,
+                                               PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(hDkpHandle);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+static inline void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...) __printf(2, 3)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(fmt);
+}
+#endif /* !__linux__ || INTEGRITY_OS || __QNXNTO__ */
+
+#endif /* DKP_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/dma_km.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/dma_km.h
new file mode 100644
index 000000000000..9bf633dfd98f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/dma_km.h
@@ -0,0 +1,85 @@
+/*************************************************************************/ /*!
+@File           dma_km.h
+@Title          DMA transfer module header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_KM_H
+#define DMA_KM_H
+
+#if defined(__linux__)
+#include <linux/version.h>
+#else
+#define KERNEL_VERSION
+#endif
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "pmr.h"
+#include "pvrsrv_sync_km.h"
+#include "connection_server.h"
+
+PVRSRV_ERROR DmaDeviceParams(CONNECTION_DATA *psConnection,
+							 PVRSRV_DEVICE_NODE *psDevNode,
+							 IMG_UINT32 *ui32DmaBuffAlign,
+							 IMG_UINT32 *ui32DmaTransferMult);
+
+PVRSRV_ERROR DmaSparseMappingTable(PMR *psPMR,
+								   IMG_DEVMEM_OFFSET_T uiOffset,
+								   IMG_UINT32 ui32SizeInPages,
+								   IMG_BOOL *pbTable);
+
+PVRSRV_ERROR DmaTransfer(CONNECTION_DATA *psConnection,
+		    PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 uiNumDMAs,
+			PMR** ppsPMR,
+			IMG_UINT64 *puiAddress,
+			IMG_DEVMEM_OFFSET_T *puiOffset,
+			IMG_DEVMEM_SIZE_T *puiSize,
+			IMG_UINT32 uiFlags,
+			PVRSRV_TIMELINE iUpdateTimeline);
+
+PVRSRV_ERROR PVRSRVInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode,
+								 CONNECTION_DATA *psConnectionData);
+void PVRSRVDeInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   CONNECTION_DATA *psConnectionData);
+
+#endif /* DMA_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/fwload.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwload.h
new file mode 100644
index 000000000000..08e7f533b666
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwload.h
@@ -0,0 +1,158 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services RGX OS Interface for loading the firmware
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines the OS interface through which the RGX
+                device initialisation code in the kernel/server will obtain
+                the RGX firmware binary image. The API is used during the
+                initialisation of an RGX device via the
+                PVRSRVCommonDeviceInitialise()
+                call sequence.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef FWLOAD_H
+#define FWLOAD_H
+
+#include "img_defs.h"
+#include "device_connection.h"
+#include "device.h"
+
+/*! Opaque type handle defined and known to the OS layer implementation of this
+ * fwload.h OS API. This private data is allocated in the implementation of
+ * OSLoadFirmware() and contains whatever data and information needed to be
+ * able to acquire and return the firmware binary image to the Services
+ * kernel/server during initialisation.
+ * It is no longer required and may be freed when OSUnloadFirmware() is called.
+ */
+typedef struct OS_FW_IMAGE_t OS_FW_IMAGE;
+
+#if defined(__linux__)
+
+bool OSVerifyFirmware(OS_FW_IMAGE* psFWImage);
+
+#endif
+
+/*************************************************************************/ /*!
+@Function     OSLoadFirmware
+@Description  The OS implementation must load or acquire the firmware (FW)
+              image binary needed by the driver stack.
+              A handle to the common layer device node is given to identify
+              which device instance in the system is being initialised. The
+              BVNC string is also supplied so that the implementation knows
+              which FW image to retrieve since each FW image only supports one
+              GPU type/revision.
+              The calling server code supports multiple GPU types and revisions
+              and will detect the specific GPU type and revision before calling
+              this API. It will also have runtime configuration of the VZ mode,
+              hence this API must be able to retrieve different FW binary
+              images based on the pszBVNCString given. The purpose of the end
+              platform/system is key to understand which FW images must be
+              available to the kernel server.
+              On exit the implementation must return a pointer to some private
+              data it uses to hold the FW image information and data. It will
+              be passed onto later API calls by the kernel server code.
+              NULL should be returned if the FW image could not be retrieved.
+              The format of the BVNC string is as follows ([x] denotes
+              optional field):
+                "rgx.fw[.signed].B.V[p].N.C[.vz]"
+              The implementation must first try to load the FW identified
+              by the pszBVpNCString parameter. If this is not available then it
+              should drop back to retrieving the FW identified by the
+              pszBVNCString parameter. The fields in the string are:
+                B, V, N, C are all unsigned integer identifying type/revision.
+                [.signed] is present when RGX_FW_SIGNED=1 is defined in the
+                  server build.
+                [p] denotes a provisional (pre-silicon) GPU configuration.
+                [.vz] is present when the kernel server is loaded on the HOST
+                  of a virtualised platform. See the DriverMode server
+                  AppHint for details.
+
+@Input        psDeviceNode       Device instance identifier.
+@Input        pszBVNCString      Identifier string of the FW image to
+                                 be loaded/acquired in production driver.
+@Input        pfnVerifyFirmware  Callback which checks validity of FW image.
+@Output       ppsFWImage         Ptr to private data on success,
+                                 NULL otherwise.
+@Return       PVRSRV_ERROR       PVRSRV_OK on success,
+                                 PVRSRV_ERROR_NOT_READY if filesystem is not
+                                                        ready/initialised,
+                                 PVRSRV_ERROR_NOT_FOUND if no suitable FW
+                                                        image could be found
+                                 PVRSRV_ERROR_OUT_OF_MEMORY if unable to alloc
+                                                        memory for FW image
+                                 PVRSRV_ERROR_NOT_AUTHENTICATED if FW image
+                                                        cannot be verified.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSLoadFirmware(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            const IMG_CHAR *pszBVNCString,
+                            bool (*pfnVerifyFirmware)(OS_FW_IMAGE*),
+                            OS_FW_IMAGE **ppsFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSFirmwareData
+@Description  This function returns a pointer to the start of the FW image
+              binary data held in memory. It must remain valid until
+              OSUnloadFirmware() is called.
+@Input        psFWImage  Private data opaque handle
+@Return       void*      Ptr to FW binary image to start on GPU.
+*/ /**************************************************************************/
+const void* OSFirmwareData(OS_FW_IMAGE *psFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSFirmwareSize
+@Description  This function returns the size of the FW image binary data.
+@Input        psFWImage  Private data opaque handle
+@Return       size_t     Size in bytes of the firmware binary image
+*/ /**************************************************************************/
+size_t OSFirmwareSize(OS_FW_IMAGE *psFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSUnloadFirmware
+@Description  This is called when the server has completed firmware
+              initialisation and no longer needs the private data, possibly
+              allocated by OSLoadFirmware().
+@Input        psFWImage  Private data opaque handle
+*/ /**************************************************************************/
+void OSUnloadFirmware(OS_FW_IMAGE *psFWImage);
+
+#endif /* FWLOAD_H */
+
+/******************************************************************************
+ End of file (fwload.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/fwtrace_string.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwtrace_string.h
new file mode 100644
index 000000000000..a2ab95c3a815
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/fwtrace_string.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File           fwtrace_string.h
+@Title          RGX Firmware trace strings for KM
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       Generic
+@Description    This file defines SFs tuple.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef KM_TRACE_STRING_H
+#define KM_TRACE_STRING_H
+
+#include "rgx_fwif_sf.h"
+
+extern const RGXKM_STID_FMT SFs[];
+extern const IMG_UINT32 g_ui32SFsCount;
+
+#endif /* KM_TRACE_STRING_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/handle.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle.h
new file mode 100644
index 000000000000..2c132901be1e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle.h
@@ -0,0 +1,211 @@
+/**************************************************************************/ /*!
+@File
+@Title          Handle Manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(HANDLE_API_H)
+#define HANDLE_API_H
+
+#include "lock_types.h"
+
+/*
+ * Handle API
+ * ----------
+ * The handle API is intended to provide handles for kernel resources, which
+ * can then be passed back to user space processes.
+ *
+ * The following functions comprise the API. Each function takes a pointer to
+ * a PVRSRV_HANDLE_BASE structure, one of which is allocated for each process,
+ * and stored in the per-process data area. Use KERNEL_HANDLE_BASE for handles
+ * not allocated for a particular process, or for handles that need to be
+ * allocated before the PVRSRV_HANDLE_BASE structure for the process is
+ * available.
+ *
+ * PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType,
+ *      PVRSRV_HANDLE_ALLOC_FLAG eFlag);
+ *
+ * Allocate a handle phHandle, for the resource of type eType pointed to by
+ * pvData.
+ *
+ * For handles that have a definite lifetime, where the corresponding resource
+ * is explicitly created and destroyed, eFlag should be zero.
+ *
+ * If a particular resource may be referenced multiple times by a given
+ * process, setting eFlag to PVRSRV_HANDLE_ALLOC_FLAG_MULTI will allow multiple
+ * handles to be allocated for the resource. Such handles cannot be found with
+ * PVRSRVFindHandle.
+ *
+ * PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType,
+ *      PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+ *
+ * This function is similar to PVRSRVAllocHandle, except that the allocated
+ * handles are associated with a parent handle, hParent, that has been
+ * allocated previously. Subhandles are automatically deallocated when their
+ * parent handle is deallocated.
+ * Subhandles can be treated as ordinary handles. For example, they may have
+ * subhandles of their own, and may be explicitly deallocated using
+ * PVRSRVReleaseHandle (see below).
+ *
+ * PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Find the handle previously allocated for the resource pointed to by pvData,
+ * of type eType. Handles allocated with the flag
+ * PVRSRV_HANDLE_ALLOC_FLAG_MULTI cannot be found using this function.
+ *
+ * PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Given a handle for a resource of type eType, return the pointer to the
+ * resource.
+ *
+ * PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType,
+ *      IMH_HANDLE hAncestor);
+ *
+ * Similar to PVRSRVLookupHandle, but checks the handle is a descendant
+ * of hAncestor.
+ *
+ * PVRSRV_ERROR PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Deallocate a handle of given type.
+ *
+ * Return the parent of a handle in *phParent, or NULL if the handle has
+ * no parent.
+ */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "hash.h"
+
+typedef enum
+{
+	#define HANDLETYPE(x) PVRSRV_HANDLE_TYPE_##x,
+	#include "handle_types.h"
+	#undef HANDLETYPE
+} PVRSRV_HANDLE_TYPE;
+
+static_assert(PVRSRV_HANDLE_TYPE_NONE == 0, "PVRSRV_HANDLE_TYPE_NONE must be zero");
+
+typedef enum
+{
+	PVRSRV_HANDLE_BASE_TYPE_CONNECTION,
+	PVRSRV_HANDLE_BASE_TYPE_PROCESS,
+	PVRSRV_HANDLE_BASE_TYPE_GLOBAL
+} PVRSRV_HANDLE_BASE_TYPE;
+
+
+typedef enum
+{
+	/* No flags */
+	PVRSRV_HANDLE_ALLOC_FLAG_NONE =    0,
+	/* Multiple handles can point at the given data pointer */
+	PVRSRV_HANDLE_ALLOC_FLAG_MULTI =   0x01,
+	/* Subhandles are allocated in a private handle space */
+	PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE = 0x02
+} PVRSRV_HANDLE_ALLOC_FLAG;
+
+typedef struct _HANDLE_BASE_ PVRSRV_HANDLE_BASE;
+
+typedef struct _PROCESS_HANDLE_BASE_
+{
+	PVRSRV_HANDLE_BASE *psHandleBase;
+	ATOMIC_T iRefCount;
+	uintptr_t uiHashKey;
+} PROCESS_HANDLE_BASE;
+
+extern PVRSRV_HANDLE_BASE *gpsKernelHandleBase;
+#define	KERNEL_HANDLE_BASE (gpsKernelHandleBase)
+
+#define HANDLE_DEBUG_LISTING_MAX_NUM 20
+
+typedef PVRSRV_ERROR (*PFN_HANDLE_RELEASE)(void *pvData);
+
+PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, PFN_HANDLE_RELEASE pfnReleaseData);
+PVRSRV_ERROR PVRSRVAllocHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, PFN_HANDLE_RELEASE pfnReleaseData);
+
+PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+PVRSRV_ERROR PVRSRVAllocSubHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+
+PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVFindHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_BOOL bRef);
+PVRSRV_ERROR PVRSRVLookupHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_BOOL bRef);
+
+PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_HANDLE hAncestor);
+
+void PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+void PVRSRVReleaseHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVDestroyHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVDestroyHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVDestroyHandleStagedUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVPurgeHandles(PVRSRV_HANDLE_BASE *psBase);
+
+PVRSRV_ERROR PVRSRVAllocHandleBase(PVRSRV_HANDLE_BASE **ppsBase,
+                                   PVRSRV_HANDLE_BASE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVFreeHandleBase(PVRSRV_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime);
+
+PVRSRV_ERROR PVRSRVFreeKernelHandles(PVRSRV_HANDLE_BASE *psBase);
+
+PVRSRV_ERROR PVRSRVHandleInit(void);
+
+PVRSRV_ERROR PVRSRVHandleDeInit(void);
+
+/* Only called from sync_fallback_server.c */
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+PVRSRV_HANDLE_BASE *PVRSRVRetrieveProcessHandleBase(void);
+#endif
+
+PVRSRV_ERROR PVRSRVAcquireProcessHandleBase(PROCESS_HANDLE_BASE **ppsBase);
+PVRSRV_ERROR PVRSRVReleaseProcessHandleBase(PROCESS_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime);
+
+void LockHandle(PVRSRV_HANDLE_BASE *psBase);
+void UnlockHandle(PVRSRV_HANDLE_BASE *psBase);
+
+
+#endif /* !defined(HANDLE_API_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_impl.h
new file mode 100644
index 000000000000..e965e536fb5e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_impl.h
@@ -0,0 +1,90 @@
+/**************************************************************************/ /*!
+@File
+@Title          Implementation Callbacks for Handle Manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the handle manager API. This file is for declarations
+                and definitions that are private/internal to the handle manager
+                API but need to be shared between the generic handle manager
+                code and the various handle manager backends, i.e. the code that
+                implements the various callbacks.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(HANDLE_IMPL_H)
+#define HANDLE_IMPL_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+typedef struct _HANDLE_IMPL_BASE_ HANDLE_IMPL_BASE;
+
+typedef PVRSRV_ERROR (*PFN_HANDLE_ITER)(IMG_HANDLE hHandle, void *pvData);
+
+typedef struct _HANDLE_IMPL_FUNCTAB_
+{
+	/* Acquire a new handle which is associated with the given data */
+	PVRSRV_ERROR (*pfnAcquireHandle)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE *phHandle, void *pvData);
+
+	/* Release the given handle (optionally returning the data associated with it) */
+	PVRSRV_ERROR (*pfnReleaseHandle)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void **ppvData);
+
+	/* Get the data associated with the given handle */
+	PVRSRV_ERROR (*pfnGetHandleData)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void **ppvData);
+
+	/* Set the data associated with the given handle */
+	PVRSRV_ERROR (*pfnSetHandleData)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void *pvData);
+
+	PVRSRV_ERROR (*pfnIterateOverHandles)(HANDLE_IMPL_BASE *psHandleBase, PFN_HANDLE_ITER pfnHandleIter, void *pvHandleIterData);
+
+	/* Enable handle purging on the given handle base */
+	PVRSRV_ERROR (*pfnEnableHandlePurging)(HANDLE_IMPL_BASE *psHandleBase);
+
+	/* Purge handles on the given handle base */
+	PVRSRV_ERROR (*pfnPurgeHandles)(HANDLE_IMPL_BASE *psHandleBase);
+
+	/* Create handle base */
+	PVRSRV_ERROR (*pfnCreateHandleBase)(HANDLE_IMPL_BASE **psHandleBase);
+
+	/* Destroy handle base */
+	PVRSRV_ERROR (*pfnDestroyHandleBase)(HANDLE_IMPL_BASE *psHandleBase);
+
+} HANDLE_IMPL_FUNCTAB;
+
+PVRSRV_ERROR PVRSRVHandleGetFuncTable(HANDLE_IMPL_FUNCTAB const **ppsFuncs);
+
+#endif /* !defined(HANDLE_IMPL_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_types.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_types.h
new file mode 100644
index 000000000000..776ab86cc097
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/handle_types.h
@@ -0,0 +1,87 @@
+/**************************************************************************/ /*!
+@File
+@Title          Handle Manager handle types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+/* NOTE: Do not add include guards to this file */
+
+HANDLETYPE(NONE)
+HANDLETYPE(SHARED_EVENT_OBJECT)
+HANDLETYPE(EVENT_OBJECT_CONNECT)
+HANDLETYPE(PMR_LOCAL_EXPORT_HANDLE)
+HANDLETYPE(PHYSMEM_PMR)
+HANDLETYPE(PHYSMEM_PMR_EXPORT)
+HANDLETYPE(PHYSMEM_PMR_SECURE_EXPORT)
+HANDLETYPE(DEVMEMINT_CTX)
+HANDLETYPE(DEVMEMINT_CTX_EXPORT)
+HANDLETYPE(DEVMEMINT_HEAP)
+HANDLETYPE(DEVMEMINT_RESERVATION)
+HANDLETYPE(DEVMEMXINT_RESERVATION)
+HANDLETYPE(DEVMEMINT_MAPPING)
+HANDLETYPE(RGX_FW_MEMDESC)
+HANDLETYPE(RGX_FREELIST)
+HANDLETYPE(RGX_MEMORY_BLOCK)
+HANDLETYPE(RGX_SERVER_RENDER_CONTEXT)
+HANDLETYPE(RGX_SERVER_TQ_CONTEXT)
+HANDLETYPE(RGX_SERVER_TQ_TDM_CONTEXT)
+HANDLETYPE(RGX_SERVER_COMPUTE_CONTEXT)
+HANDLETYPE(RGX_SERVER_RAY_CONTEXT)
+HANDLETYPE(RGX_SERVER_KICKSYNC_CONTEXT)
+HANDLETYPE(SYNC_PRIMITIVE_BLOCK)
+HANDLETYPE(SYNC_RECORD_HANDLE)
+HANDLETYPE(PVRSRV_TIMELINE_SERVER)
+HANDLETYPE(PVRSRV_FENCE_SERVER)
+HANDLETYPE(PVRSRV_FENCE_EXPORT)
+HANDLETYPE(RGX_KM_HW_RT_DATASET)
+HANDLETYPE(RGX_FWIF_ZSBUFFER)
+HANDLETYPE(RGX_POPULATION)
+HANDLETYPE(DC_DEVICE)
+HANDLETYPE(DC_DISPLAY_CONTEXT)
+HANDLETYPE(DC_BUFFER)
+HANDLETYPE(DC_PIN_HANDLE)
+HANDLETYPE(DEVMEM_MEM_IMPORT)
+HANDLETYPE(PHYSMEM_PMR_PAGELIST)
+HANDLETYPE(PVR_TL_SD)
+HANDLETYPE(RI_HANDLE)
+HANDLETYPE(DEV_PRIV_DATA)
+HANDLETYPE(MM_PLAT_CLEANUP)
+HANDLETYPE(WORKEST_RETURN_DATA)
+HANDLETYPE(DI_CONTEXT)
+HANDLETYPE(PVR_HWPERF_SD)
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/htbserver.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/htbserver.h
new file mode 100644
index 000000000000..4aaa3dd35e08
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/htbserver.h
@@ -0,0 +1,240 @@
+/*************************************************************************/ /*!
+@File           htbserver.h
+@Title          Host Trace Buffer server implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+
+                A Host Trace can be merged with a corresponding Firmware Trace.
+                This is achieved by inserting synchronisation data into both
+                traces and post processing to merge them.
+
+                The FW Trace will contain a "Sync Partition Marker". This is
+                updated every time the RGX is brought out of reset (RGX clock
+                timestamps reset at this point) and is repeated when the FW
+                Trace buffer wraps to ensure there is always at least 1
+                partition marker in the Firmware Trace buffer whenever it is
+                read.
+
+                The Host Trace will contain corresponding "Sync Partition
+                Markers" - #HTBSyncPartitionMarker(). Each partition is then
+                subdivided into "Sync Scale" sections - #HTBSyncScale(). The
+                "Sync Scale" data allows the timestamps from the two traces to
+                be correlated. The "Sync Scale" data is updated as part of the
+                standard RGX time correlation code (rgxtimecorr.c) and is
+                updated periodically including on power and clock changes.
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HTBSERVER_H
+#define HTBSERVER_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "htbuffer_types.h"
+
+#define HTBLOGK(SF, args...) do { if (HTB_GROUP_ENABLED(SF)) { (void)HTBLogSimple(SF, ## args); } } while (0)
+
+/* macros to cast 64 or 32-bit pointers into 32-bit integer components for Host Trace */
+#define HTBLOG_PTR_BITS_HIGH(p) ((IMG_UINT32)((((IMG_UINT64)((uintptr_t)p))>>32)&0xffffffff))
+#define HTBLOG_PTR_BITS_LOW(p)  ((IMG_UINT32)(((IMG_UINT64)((uintptr_t)p))&0xffffffff))
+
+/* macros to cast 64-bit integers into 32-bit integer components for Host Trace */
+#define HTBLOG_U64_BITS_HIGH(u) ((IMG_UINT32)((u>>32)&0xffffffff))
+#define HTBLOG_U64_BITS_LOW(u)  ((IMG_UINT32)(u&0xffffffff))
+
+/* Host Trace Buffer name */
+#define HTB_STREAM_NAME	"PVRHTBuffer"
+
+/************************************************************************/ /*!
+ @Function      HTBInit
+ @Description   Initialise the Host Trace Buffer and allocate all resources
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBInit_Impl(void);
+
+/************************************************************************/ /*!
+ @Function      HTBDeInit
+ @Description   Close the Host Trace Buffer and free all resources
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBDeInit_Impl(void);
+
+/*************************************************************************/ /*!
+ @Function      HTBControlKM
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control the behaviour of the data buffer
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBControlKM_Impl(const IMG_UINT32 ui32NumFlagGroups,
+			 const IMG_UINT32 *aui32GroupEnable,
+			 const IMG_UINT32 ui32LogLevel,
+			 const IMG_UINT32 ui32EnablePID,
+			 const HTB_LOGMODE_CTRL eLogMode,
+			 const HTB_OPMODE_CTRL eOpMode);
+
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarker
+ @Description   Write an HTB sync partition marker to the HTB log
+
+ @Input         ui32Marker      Marker value
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarker_Impl(const IMG_UINT32 ui32Marker);
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarkerRpt
+ @Description   Write a HTB sync partition marker to the HTB log, given
+                the previous values to repeat.
+
+ @Input         ui32Marker      Marker value
+ @Input         ui64SyncOSTS    previous OSTS
+ @Input         ui64SyncCRTS    previous CRTS
+ @Input         ui32ClkSpeed    previous Clockspeed
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarkerRepeat_Impl(const IMG_UINT32 ui32Marker,
+							 const IMG_UINT64 ui64SyncOSTS,
+							 const IMG_UINT64 ui64SyncCRTS,
+							 const IMG_UINT32 ui32ClkSpeed);
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncScale
+ @Description   Write FW-Host synchronisation data to the HTB log when clocks
+                change or are re-calibrated
+
+ @Input         bLogValues      IMG_TRUE if value should be immediately written
+                                out to the log
+
+ @Input         ui64OSTS        OS Timestamp
+
+ @Input         ui64CRTS        Rogue timestamp
+
+ @Input         ui32CalcClkSpd  Calculated clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncScale_Impl(const IMG_BOOL bLogValues, const IMG_UINT64 ui64OSTS,
+			 const IMG_UINT64 ui64CRTS, const IMG_UINT32 ui32CalcClkSpd);
+
+/*************************************************************************/ /*!
+ @Function      HTBLogSimple
+ @Description   Record a Host Trace Buffer log event with implicit PID and Timestamp
+
+ @Input         SF              The log event ID
+
+ @Input         ...             Log parameters
+
+ @Return        PVRSRV_OK       Success.
+
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBLogSimple_Impl(IMG_UINT32 SF, ...);
+
+/*  DEBUG log group enable */
+#if !defined(HTB_DEBUG_LOG_GROUP)
+#undef HTB_LOG_TYPE_DBG    /* No trace statements in this log group should be checked in */
+#define HTB_LOG_TYPE_DBG    __BUILDERROR__
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+/*************************************************************************/ /*!
+ @Function      HTBIsConfigured
+ @Description   Determine if HTB stream has been configured
+
+ @Input         none
+
+ @Return        IMG_FALSE       Stream has not been configured
+                IMG_TRUE        Stream has been configured
+
+*/ /**************************************************************************/
+IMG_BOOL
+HTBIsConfigured_Impl(void);
+
+#define HTBIsConfigured HTBIsConfigured_Impl
+#define HTBLogSimple HTBLogSimple_Impl
+#define HTBSyncScale(bLogValues, ui64OSTS, ui64CRTS, ui32CalcClkSpd) \
+  HTBSyncScale_Impl((bLogValues), (ui64OSTS), (ui64CRTS), (ui32CalcClkSpd))
+#define HTBSyncPartitionMarkerRepeat(ui32Marker, ui64SyncOSTS, ui64SyncCRTS, ui32ClkSpeed) \
+  HTBSyncPartitionMarkerRepeat_Impl((ui32Marker), (ui64SyncOSTS), (ui64SyncCRTS), (ui32ClkSpeed))
+#define HTBSyncPartitionMarker(a) HTBSyncPartitionMarker_Impl((a))
+#define HTBControlKM(ui32NumFlagGroups, aui32GroupEnable, ui32LogLevel, ui32EnablePID, eLogMode, eOpMode) \
+  HTBControlKM_Impl((ui32NumFlagGroups), (aui32GroupEnable), (ui32LogLevel), (ui32EnablePID), (eLogMode), (eOpMode))
+#define HTBInit() HTBInit_Impl()
+#define HTBDeInit() HTBDeInit_Impl()
+#else	/* !PVRSRV_ENABLE_HTB) */
+#define HTBIsConfigured()  IMG_FALSE
+#define HTBLogSimple(SF, args...)  PVRSRV_OK
+#define HTBSyncScale(a, b, c, d)
+#define HTBSyncPartitionMarkerRepeat(a, b, c, d)
+#define HTBSyncPartitionMarker(a)
+#define HTBControlKM(a, b, c, d, e, f) PVRSRV_OK
+#define HTBDeInit() PVRSRV_OK
+#define HTBInit() PVRSRV_OK
+#endif	/* PVRSRV_ENABLE_HTB */
+#endif /* HTBSERVER_H */
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/info_page.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/info_page.h
new file mode 100644
index 000000000000..5816125d742f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/info_page.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose memory shared between kernel driver and user
+                mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_KM_H
+#define INFO_PAGE_KM_H
+
+#include "pvrsrv_error.h"
+
+#include "pmr.h"
+#include "pvrsrv.h"
+#include "info_page_defs.h"
+
+/**
+ * @Function InfoPageCreate
+ * @Description Allocates resources for global information page.
+ * @Input psData pointer to PVRSRV data
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+PVRSRV_ERROR InfoPageCreate(PVRSRV_DATA *psData);
+
+/**
+ * @Function InfoPageDestroy
+ * @Description Frees all of the resource of global information page.
+ * @Input psData pointer to PVRSRV data
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+void InfoPageDestroy(PVRSRV_DATA *psData);
+
+/**
+ * @Function PVRSRVAcquireInfoPageKM()
+ * @Description This interface is used for obtaining the global information page
+ *              which acts as a general purpose shared memory between KM and UM.
+ *              The use of this information page outside of services is _not_
+ *              recommended.
+ * @Output ppsPMR handle to exported PMR
+ * @Return
+ */
+PVRSRV_ERROR PVRSRVAcquireInfoPageKM(PMR **ppsPMR);
+
+/**
+ * @Function PVRSRVReleaseInfoPageKM()
+ * @Description This function matches PVRSRVAcquireInfoPageKM().
+ * @Input psPMR handle to exported PMR
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+PVRSRV_ERROR PVRSRVReleaseInfoPageKM(PMR *psPMR);
+
+/**
+ * @Function GetInfoPageDebugFlagsKM()
+ * @Description Return info page debug flags
+ * @Return info page debug flags
+ */
+static INLINE IMG_UINT32 GetInfoPageDebugFlagsKM(void)
+{
+	return (PVRSRVGetPVRSRVData())->pui32InfoPage[DEBUG_FEATURE_FLAGS];
+}
+
+#endif /* INFO_PAGE_KM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/lists.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/lists.h
new file mode 100644
index 000000000000..2e2c29a0b7b0
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/lists.h
@@ -0,0 +1,367 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linked list shared functions templates.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Definition of the linked list function templates.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LISTS_UTILS_H
+#define LISTS_UTILS_H
+
+/* instruct QAC to ignore warnings about the following custom formatted macros */
+/* PRQA S 0881,3410 ++ */
+
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "img_types.h"
+#include "device.h"
+#include "power.h"
+
+/*
+ - USAGE -
+
+ The list functions work with any structure that provides the fields psNext and
+ ppsThis. In order to make a function available for a given type, it is required
+ to use the function template macro that creates the actual code.
+
+ There are 5 main types of functions:
+ - INSERT      : given a pointer to the head pointer of the list and a pointer
+                 to the node, inserts it as the new head.
+ - INSERT TAIL : given a pointer to the head pointer of the list and a pointer
+                 to the node, inserts the node at the tail of the list.
+ - REMOVE      : given a pointer to a node, removes it from its list.
+ - FOR EACH    : apply a function over all the elements of a list.
+ - ANY         : apply a function over the elements of a list, until one of them
+                 return a non null value, and then returns it.
+
+ The two last functions can have a variable argument form, with allows to pass
+ additional parameters to the callback function. In order to do this, the
+ callback function must take two arguments, the first is the current node and
+ the second is a list of variable arguments (va_list).
+
+ The ANY functions have also another for which specifies the return type of the
+ callback function and the default value returned by the callback function.
+
+*/
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_ForEach
+@Description    Apply a callback function to all the elements of a list.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+*/ /**************************************************************************/
+#define DECLARE_LIST_FOR_EACH(TYPE) \
+void List_##TYPE##_ForEach(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_FOR_EACH(TYPE) \
+void List_##TYPE##_ForEach(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))\
+{\
+	while (psHead)\
+	{\
+		pfnCallBack(psHead);\
+		psHead = psHead->psNext;\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_ForEachSafe
+@Description    Apply a callback function to all the elements of a list. Do it
+                in a safe way that handles the fact that a node might remove
+                itself from the list during the iteration.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+*/ /**************************************************************************/
+#define DECLARE_LIST_FOR_EACH_SAFE(TYPE) \
+void List_##TYPE##_ForEachSafe(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_FOR_EACH_SAFE(TYPE) \
+void List_##TYPE##_ForEachSafe(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))\
+{\
+	TYPE *psNext;\
+\
+	while (psHead)\
+	{\
+		psNext = psHead->psNext; \
+		pfnCallBack(psHead);\
+		psHead = psNext;\
+	}\
+}
+
+
+#define DECLARE_LIST_FOR_EACH_VA(TYPE) \
+void List_##TYPE##_ForEach_va(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_FOR_EACH_VA(TYPE) \
+void List_##TYPE##_ForEach_va(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode, va_list va), ...) \
+{\
+	va_list ap;\
+	while (psHead)\
+	{\
+		va_start(ap, pfnCallBack);\
+		pfnCallBack(psHead, ap);\
+		psHead = psHead->psNext;\
+		va_end(ap);\
+	}\
+}
+
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Any
+@Description    Applies a callback function to the elements of a list until
+                the function returns a non null value, then returns it.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+@Return         The first non null value returned by the callback function.
+*/ /**************************************************************************/
+#define DECLARE_LIST_ANY(TYPE) \
+void* List_##TYPE##_Any(TYPE *psHead, void* (*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_ANY(TYPE) \
+void* List_##TYPE##_Any(TYPE *psHead, void* (*pfnCallBack)(TYPE* psNode))\
+{ \
+	void *pResult;\
+	TYPE *psNextNode;\
+	pResult = NULL;\
+	psNextNode = psHead;\
+	while (psHead && !pResult)\
+	{\
+		psNextNode = psNextNode->psNext;\
+		pResult = pfnCallBack(psHead);\
+		psHead = psNextNode;\
+	}\
+	return pResult;\
+}
+
+
+/*with variable arguments, that will be passed as a va_list to the callback function*/
+
+#define DECLARE_LIST_ANY_VA(TYPE) \
+void* List_##TYPE##_Any_va(TYPE *psHead, void*(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_ANY_VA(TYPE) \
+void* List_##TYPE##_Any_va(TYPE *psHead, void*(*pfnCallBack)(TYPE* psNode, va_list va), ...)\
+{\
+	va_list ap;\
+	TYPE *psNextNode;\
+	void* pResult = NULL;\
+	while (psHead && !pResult)\
+	{\
+		psNextNode = psHead->psNext;\
+		va_start(ap, pfnCallBack);\
+		pResult = pfnCallBack(psHead, ap);\
+		va_end(ap);\
+		psHead = psNextNode;\
+	}\
+	return pResult;\
+}
+
+/*those ones are for extra type safety, so there's no need to use castings for the results*/
+
+#define DECLARE_LIST_ANY_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any(TYPE *psHead, RTYPE (*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_ANY_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any(TYPE *psHead, RTYPE (*pfnCallBack)(TYPE* psNode))\
+{ \
+	RTYPE result;\
+	TYPE *psNextNode;\
+	result = CONTINUE;\
+	psNextNode = psHead;\
+	while (psHead && result == CONTINUE)\
+	{\
+		psNextNode = psNextNode->psNext;\
+		result = pfnCallBack(psHead);\
+		psHead = psNextNode;\
+	}\
+	return result;\
+}
+
+
+#define DECLARE_LIST_ANY_VA_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any_va(TYPE *psHead, RTYPE(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_ANY_VA_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any_va(TYPE *psHead, RTYPE(*pfnCallBack)(TYPE* psNode, va_list va), ...)\
+{\
+	va_list ap;\
+	TYPE *psNextNode;\
+	RTYPE result = CONTINUE;\
+	while (psHead && result == CONTINUE)\
+	{\
+		psNextNode = psHead->psNext;\
+		va_start(ap, pfnCallBack);\
+		result = pfnCallBack(psHead, ap);\
+		va_end(ap);\
+		psHead = psNextNode;\
+	}\
+	return result;\
+}
+
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Remove
+@Description    Removes a given node from the list.
+@Input          psNode      The pointer to the node to be removed.
+*/ /**************************************************************************/
+#define DECLARE_LIST_REMOVE(TYPE) \
+void List_##TYPE##_Remove(TYPE *psNode)
+
+#define IMPLEMENT_LIST_REMOVE(TYPE) \
+void List_##TYPE##_Remove(TYPE *psNode)\
+{\
+	(*psNode->ppsThis)=psNode->psNext;\
+	if (psNode->psNext)\
+	{\
+		psNode->psNext->ppsThis = psNode->ppsThis;\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Insert
+@Description    Inserts a given node at the beginning of the list.
+@Input          psHead   The pointer to the pointer to the head node.
+@Input          psNode   The pointer to the node to be inserted.
+*/ /**************************************************************************/
+#define DECLARE_LIST_INSERT(TYPE) \
+void List_##TYPE##_Insert(TYPE **ppsHead, TYPE *psNewNode)
+
+#define IMPLEMENT_LIST_INSERT(TYPE) \
+void List_##TYPE##_Insert(TYPE **ppsHead, TYPE *psNewNode)\
+{\
+	psNewNode->ppsThis = ppsHead;\
+	psNewNode->psNext = *ppsHead;\
+	*ppsHead = psNewNode;\
+	if (psNewNode->psNext)\
+	{\
+		psNewNode->psNext->ppsThis = &(psNewNode->psNext);\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_InsertTail
+@Description    Inserts a given node at the end of the list.
+@Input          psHead   The pointer to the pointer to the head node.
+@Input          psNode   The pointer to the node to be inserted.
+*/ /**************************************************************************/
+#define DECLARE_LIST_INSERT_TAIL(TYPE) \
+void List_##TYPE##_InsertTail(TYPE **ppsHead, TYPE *psNewNode)
+
+#define IMPLEMENT_LIST_INSERT_TAIL(TYPE) \
+void List_##TYPE##_InsertTail(TYPE **ppsHead, TYPE *psNewNode)\
+{\
+	TYPE *psTempNode = *ppsHead;\
+	if (psTempNode != NULL)\
+	{\
+		while (psTempNode->psNext)\
+			psTempNode = psTempNode->psNext;\
+		ppsHead = &psTempNode->psNext;\
+	}\
+	psNewNode->ppsThis = ppsHead;\
+	psNewNode->psNext = NULL;\
+	*ppsHead = psNewNode;\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Reverse
+@Description    Reverse a list in place
+@Input          ppsHead    The pointer to the pointer to the head node.
+*/ /**************************************************************************/
+#define DECLARE_LIST_REVERSE(TYPE) \
+void List_##TYPE##_Reverse(TYPE **ppsHead)
+
+#define IMPLEMENT_LIST_REVERSE(TYPE) \
+void List_##TYPE##_Reverse(TYPE **ppsHead)\
+{\
+	TYPE *psTmpNode1; \
+	TYPE *psTmpNode2; \
+	TYPE *psCurNode; \
+	psTmpNode1 = NULL; \
+	psCurNode = *ppsHead; \
+	while (psCurNode) { \
+		psTmpNode2 = psCurNode->psNext; \
+		psCurNode->psNext = psTmpNode1; \
+		psTmpNode1 = psCurNode; \
+		psCurNode = psTmpNode2; \
+		if (psCurNode) \
+		{ \
+			psTmpNode1->ppsThis = &(psCurNode->psNext); \
+		} \
+		else \
+		{ \
+			psTmpNode1->ppsThis = ppsHead; \
+		} \
+	} \
+	*ppsHead = psTmpNode1; \
+}
+
+#define IS_LAST_ELEMENT(x) ((x)->psNext == NULL)
+
+
+DECLARE_LIST_ANY(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_ANY_2(PVRSRV_DEVICE_NODE, IMG_BOOL, IMG_FALSE);
+DECLARE_LIST_ANY_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK);
+DECLARE_LIST_ANY_VA(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_ANY_VA_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK);
+DECLARE_LIST_FOR_EACH(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_FOR_EACH_VA(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_INSERT_TAIL(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_REMOVE(PVRSRV_DEVICE_NODE);
+
+#undef DECLARE_LIST_ANY_2
+#undef DECLARE_LIST_ANY_VA
+#undef DECLARE_LIST_ANY_VA_2
+#undef DECLARE_LIST_FOR_EACH
+#undef DECLARE_LIST_FOR_EACH_VA
+#undef DECLARE_LIST_INSERT
+#undef DECLARE_LIST_REMOVE
+
+#endif
+
+/* re-enable warnings */
+/* PRQA S 0881,3410 -- */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/mmu_common.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/mmu_common.h
new file mode 100644
index 000000000000..d0c5e9f03aed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/mmu_common.h
@@ -0,0 +1,875 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef MMU_COMMON_H
+#define MMU_COMMON_H
+
+/*
+	The Memory Management Unit (MMU) performs device virtual to physical
+	translation.
+
+	Terminology:
+	 - page catalogue, PC	(optional, 3 tier MMU)
+	 - page directory, PD
+	 - page table, PT (can be variable sized)
+	 - data page, DP (can be variable sized)
+	Note: PD and PC are fixed size and can't be larger than the native
+	      physical (CPU) page size
+	Shifts and AlignShift variables:
+	 - 'xxxShift' represent the number of bits a bitfield is shifted left from bit0
+	 - 'xxxAlignShift' is used to convert a bitfield (based at bit0) into byte units
+	   by applying a bit shift left by 'xxxAlignShift' bits
+*/
+
+/*
+	Device Virtual Address Config:
+
+	Incoming Device Virtual Address is deconstructed into up to 4
+	fields, where the virtual address is up to 64bits:
+	MSB-----------------------------------------------LSB
+	| PC Index:   | PD Index:  | PT Index: | DP offset: |
+	| d bits      | c bits     | b-v bits  |  a+v bits  |
+	-----------------------------------------------------
+	where v is the variable page table modifier, e.g.
+			v == 0 -> 4KB DP
+			v == 2 -> 16KB DP
+			v == 4 -> 64KB DP
+			v == 6 -> 256KB DP
+			v == 8 -> 1MB DP
+			v == 10 -> 4MB DP
+*/
+
+/* services/server/include/ */
+#include "pmr.h"
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv_error.h"
+#include "servicesext.h"
+#include "sync_prim_internal.h"
+
+/*!
+	The level of the MMU
+*/
+typedef enum
+{
+	MMU_LEVEL_0 = 0,    /* Level 0 = Page */
+
+	MMU_LEVEL_1,        /* Level 1 = PT */
+	MMU_LEVEL_2,        /* Level 2 = PD */
+	MMU_LEVEL_3,        /* Level 3 = PC */
+	MMU_LEVEL_LAST
+} MMU_LEVEL;
+
+/* moved after declaration of MMU_LEVEL, as pdump_mmu.h references it */
+#include "pdump_mmu.h"
+
+#define MMU_MAX_LEVEL 3
+
+
+typedef struct _MMU_CONTEXT_ MMU_CONTEXT;
+
+
+/*
+	P(C/D/T) Entry Config:
+
+	MSB-----------------------------------------------LSB
+	| PT Addr:   | variable PT ctrl | protection flags: |
+	| bits c+v   | b bits           | a bits            |
+	-----------------------------------------------------
+	where v is the variable page table modifier and is optional
+*/
+/*!
+	Generic MMU entry description. This is used to describe PC, PD and PT entries.
+*/
+typedef struct _MMU_PxE_CONFIG_
+{
+	MMU_LEVEL		ePxLevel;        /*! MMU Level this config describes */
+	const IMG_CHAR	*pszPxLevelStr;  /*! Px string for this level */
+	IMG_UINT8		uiBytesPerEntry; /*! Size of an entry in bytes */
+
+	IMG_UINT64	uiAddrMask;      /*! Physical address mask */
+	IMG_UINT8	uiAddrShift;     /*! Physical address shift */
+	IMG_UINT8	uiAddrLog2Align; /*! Physical address Log 2 alignment */
+
+	IMG_UINT64	uiVarCtrlMask;   /*! Variable control mask */
+	IMG_UINT8	uiVarCtrlShift;  /*! Variable control shift */
+
+	IMG_UINT64	uiProtMask;      /*! Protection flags mask */
+	IMG_UINT8	uiProtShift;     /*! Protection flags shift */
+	IMG_UINT64	uiPendingEnMask; /*! Entry pending bit mask */
+	IMG_UINT64	uiValidEnMask;   /*! Entry valid bit mask */
+	IMG_UINT8	uiValidEnShift;  /*! Entry valid bit shift */
+	IMG_UINT64	uiParityBitMask;   /*! Entry parity bit mask */
+	IMG_UINT8	uiParityBitShift;  /*! Entry parity bit shift */
+} MMU_PxE_CONFIG;
+
+/*!
+	MMU virtual address split
+*/
+typedef struct _MMU_DEVVADDR_CONFIG_
+{
+	/*! Page catalogue index mask */
+	IMG_UINT64	uiPCIndexMask;
+	/*! Page catalogue index shift */
+	IMG_UINT8	uiPCIndexShift;
+	/*! Total number of PC entries */
+	IMG_UINT32	uiNumEntriesPC;
+
+	/*! Page directory mask */
+	IMG_UINT64	uiPDIndexMask;
+	/*! Page directory shift */
+	IMG_UINT8	uiPDIndexShift;
+	/*! Total number of PD entries */
+	IMG_UINT32	uiNumEntriesPD;
+
+	/*! Page table mask */
+	IMG_UINT64	uiPTIndexMask;
+	/*! Page index shift */
+	IMG_UINT8	uiPTIndexShift;
+	/*! Total number of PT entries */
+	IMG_UINT32	uiNumEntriesPT;
+
+	/*! Page offset mask */
+	IMG_UINT64	uiPageOffsetMask;
+	/*! Page offset shift */
+	IMG_UINT8	uiPageOffsetShift;
+
+	/*! First virtual address mappable for this config */
+	IMG_UINT64	uiOffsetInBytes;
+
+} MMU_DEVVADDR_CONFIG;
+
+/*!
+	MMU device attributes. This structure is the interface between the generic
+	MMU code and the device specific MMU code.
+*/
+typedef struct _MMU_DEVICEATTRIBS_
+{
+	/*! Page and address type */
+	PDUMP_MMU_TYPE eMMUType;
+
+	/*! Name string of the PDUMP memory space to use */
+	IMG_CHAR *pszMMUPxPDumpMemSpaceName;
+
+	/*! Alignment requirement of the top/base object */
+	IMG_UINT32 ui32BaseAlign;
+
+	/*! HW config of the top/base object */
+	struct _MMU_PxE_CONFIG_ *psBaseConfig;
+
+	/*! Address split for the base object */
+	const struct _MMU_DEVVADDR_CONFIG_ *psTopLevelDevVAddrConfig;
+
+	/*! Supported page sizes validation mask */
+	IMG_UINT32 ui32ValidPageSizeMask;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_UINT64* pui64PrecomputedAllocParity[2];
+#endif
+
+	/*! Callback for creating protection bits for the page catalogue entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePCEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page catalogue entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePCEProt4)(IMG_UINT32 uiProtFlags);
+	/*! Callback for creating protection bits for the page directory entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePDEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page directory entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePDEProt4)(IMG_UINT32 uiProtFlags);
+	/*! Callback for creating protection bits for the page table entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePTEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page table entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePTEProt4)(IMG_UINT32 uiProtFlags);
+
+	/*! Callback for getting the MMU configuration based on the specified page size */
+	PVRSRV_ERROR (*pfnGetPageSizeConfiguration)(IMG_UINT32 ui32DataPageSize,
+												const struct _MMU_PxE_CONFIG_ **ppsMMUPDEConfig,
+												const struct _MMU_PxE_CONFIG_ **ppsMMUPTEConfig,
+												const struct _MMU_DEVVADDR_CONFIG_ **ppsMMUDevVAddrConfig,
+												IMG_HANDLE *phPriv2);
+	/*! Callback for putting the MMU configuration obtained from pfnGetPageSizeConfiguration */
+	PVRSRV_ERROR (*pfnPutPageSizeConfiguration)(IMG_HANDLE hPriv);
+
+	/*! Callback for getting the page size from the PDE for the page table entry with 4 byte entry */
+	PVRSRV_ERROR (*pfnGetPageSizeFromPDE4)(IMG_UINT32, IMG_UINT32 *);
+	/*! Callback for getting the page size from the PDE for the page table entry with 8 byte entry */
+	PVRSRV_ERROR (*pfnGetPageSizeFromPDE8)(IMG_UINT64, IMG_UINT32 *);
+	/*! Callback for getting the page size directly from the address. Supported on MMU4 */
+	PVRSRV_ERROR (*pfnGetPageSizeFromVirtAddr)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_DEV_VIRTADDR, IMG_UINT32 *);
+
+	/*! Private data handle */
+	IMG_HANDLE hGetPageSizeFnPriv;
+} MMU_DEVICEATTRIBS;
+
+
+/* MMU Protection flags */
+
+/* These are specified generically and in a h/w independent way, and
+   are interpreted at each level (PC/PD/PT) separately. */
+
+/* The following flags are for internal use only, and should not
+   traverse the API */
+#define MMU_PROTFLAGS_INVALID 0x80000000U
+
+typedef IMG_UINT32 MMU_PROTFLAGS_T;
+
+/* The following flags should be supplied by the caller: */
+#define MMU_PROTFLAGS_READABLE					(1U<<0)
+#define MMU_PROTFLAGS_WRITEABLE					(1U<<1)
+#define MMU_PROTFLAGS_CACHE_COHERENT			(1U<<2)
+#define MMU_PROTFLAGS_CACHED					(1U<<3)
+
+/* Device specific flags*/
+#define MMU_PROTFLAGS_DEVICE_OFFSET		16
+#define MMU_PROTFLAGS_DEVICE_MASK		0x000f0000UL
+#define MMU_PROTFLAGS_DEVICE(n)	\
+			(((n) << MMU_PROTFLAGS_DEVICE_OFFSET) & \
+			MMU_PROTFLAGS_DEVICE_MASK)
+
+
+struct _PVRSRV_DEVICE_NODE_;
+
+struct _CONNECTION_DATA_;
+
+typedef struct _MMU_PAGESIZECONFIG_
+{
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT32 uiRefCount;
+	IMG_UINT32 uiMaxRefCount;
+} MMU_PAGESIZECONFIG;
+
+/*************************************************************************/ /*!
+@Function       MMU_InitDevice
+
+@Description    Creates MMU device specific resources.
+
+@Input          psDevNode               Device node of the device to create the
+                                        MMU context for
+
+@Return         PVRSRV_OK if the initialisation process was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_InitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       MMU_DeInitDevice
+
+@Description    Clean-up MMU device specific resources.
+
+@Input          psDevNode               Device node of the device
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_DeInitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextCreate
+
+@Description    Create a new MMU context
+
+@Input          psConnection            Connection requesting the MMU context
+                                        creation. Can be NULL for kernel/FW
+                                        memory context.
+@Input          psDevNode               Device node of the device to create the
+                                        MMU context for
+@Output         ppsMMUContext           The created MMU context
+
+@Return         PVRSRV_OK if the MMU context was successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ContextCreate(struct _CONNECTION_DATA_ *psConnection,
+                  struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                  MMU_CONTEXT **ppsMMUContext,
+                  MMU_DEVICEATTRIBS *psDevAttrs);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextDestroy
+
+@Description    Destroy a MMU context
+
+@Input          psMMUContext            MMU context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+MMU_ContextDestroy(MMU_CONTEXT *psMMUContext);
+
+/*************************************************************************/ /*!
+@Function       MMU_Alloc
+
+@Description    Allocate the page tables required for the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uSize                   The size of the allocation
+
+@Input          uiProtFlags             Generic MMU protection flags
+
+@Input          uDevVAddrAlignment      Alignment requirement of the virtual
+                                        allocation
+
+@Input          psDevVAddr              Virtual address to start the allocation
+                                        from
+
+@Return         PVRSRV_OK if the allocation of the page tables was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_Alloc(MMU_CONTEXT *psMMUContext,
+          IMG_DEVMEM_SIZE_T uSize,
+          IMG_UINT32 uiProtFlags,
+          IMG_DEVMEM_SIZE_T uDevVAddrAlignment,
+          IMG_DEV_VIRTADDR *psDevVAddr,
+          IMG_UINT32 uiLog2PageSize);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_Free
+
+@Description    Free the page tables of the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddr               Virtual address to start the free
+                                        from
+
+@Input          uiSize                  The size of the allocation
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+MMU_Free(MMU_CONTEXT *psMMUContext,
+         IMG_DEV_VIRTADDR sDevVAddr,
+         IMG_DEVMEM_SIZE_T uiSize,
+         IMG_UINT32 uiLog2DataPageSize);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_MapPages
+
+@Description    Map pages to the MMU.
+                Two modes of operation: One requires a list of physical page
+                indices that are going to be mapped, the other just takes
+                the PMR and a possible offset to map parts of it.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+
+@Input          psPMR                   PMR to map
+
+@Input          ui32PhysPgOffset        Physical offset into the PMR
+
+@Input          ui32MapPageCount        Number of pages to map
+
+@Input          paui32MapIndices        List of page indices to map,
+                                         can be NULL
+
+@Input          uiLog2PageSize          Log2 page size of the pages to map
+
+@Return         PVRSRV_OK if the mapping was successful
+                PVRSRV_ERROR_RETRY if SUPPORT_LINUX_OSPAGE_MIGRATION is
+                enabled and migrate is in progress. Requests to MMU_MapPages
+                may return retry if the target PMR to map is
+                in migrate state. This is because in order for migrate
+                to complete higher locking primitives are required to give
+                way to the migrate path. Expect PVRSRV_ERROR_RETRY return
+                from this function if this give way is required.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_MapPages(MMU_CONTEXT *psMMUContext,
+             PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+             IMG_DEV_VIRTADDR sDevVAddrBase,
+             PMR *psPMR,
+             IMG_UINT32 ui32PhysPgOffset,
+             IMG_UINT32 ui32MapPageCount,
+             IMG_UINT32 *paui32MapIndices,
+             IMG_UINT32 uiLog2PageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_UnmapPages
+
+@Description    Unmap pages from the MMU.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddr               Device virtual address of the 1st page
+
+@Input          ui32PageCount           Number of pages to unmap
+
+@Input          pai32UnmapIndicies      Array of page indices to be unmapped
+
+@Input          uiLog2PageSize          log2 size of the page
+
+
+@Input          uiMemAllocFlags         Indicates if the unmapped regions need
+                                        to be backed by dummy or zero page
+
+@Return         PVRSRV_OK if the unmap operation was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_UnmapPages(MMU_CONTEXT *psMMUContext,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_DEV_VIRTADDR sDevVAddr,
+               IMG_UINT32 ui32PageCount,
+               IMG_UINT32 *pai32UnmapIndicies,
+               IMG_UINT32 uiLog2PageSize,
+               PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags);
+
+/*************************************************************************/ /*!
+@Function       MMUX_MapVRangeToBackingPage
+
+@Description    Map virtual range to a backing page in the MMU.
+                Used in DevmemX calls which don't tie a PMR to a virtual
+                range implicitly.
+
+@Input          psMMUContext            MMU context to operate on
+@Input          uiMappingFlags          Memalloc flags for the mapping
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+@Input          ui32MapPageCount        Number of pages to map
+@Input          uiLog2PageSize          Log2 page size of the pages to map
+
+@Return         PVRSRV_OK if the mapping was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMUX_MapVRangeToBackingPage(MMU_CONTEXT *psMMUContext,
+                            PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                            IMG_DEV_VIRTADDR sDevVAddrBase,
+                            IMG_UINT32 ui32MapPageCount,
+                            IMG_UINT32 uiLog2HeapPageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_MapPMRFast
+
+@Description    Map a PMR into the MMU. Must be not sparse.
+                This is supposed to cover most mappings and, as the name suggests,
+                should be as fast as possible.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddr               Device virtual address to map the PMR
+                                        into
+
+@Input          psPMR                   PMR to map
+
+@Input          uiSizeBytes             Size in bytes to map
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Return         PVRSRV_OK if the PMR was successfully mapped.
+                PVRSRV_ERROR_RETRY if SUPPORT_LINUX_OSPAGE_MIGRATION is
+                enabled and migrate is in progress. Requests to MMU_MapPages
+                may return retry if the target PMR to map is
+                in migrate state. This is because in order for migrate
+                to complete higher locking primitives are required to give
+                way to the migrate path. Expect PVRSRV_ERROR_RETRY return
+                from this function if this give way is required.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_MapPMRFast(MMU_CONTEXT *psMMUContext,
+               IMG_DEV_VIRTADDR sDevVAddr,
+               PMR *psPMR,
+               IMG_DEVMEM_SIZE_T uiSizeBytes,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_UINT32 uiLog2PageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_UnmapPMRFast
+
+@Description    Unmap pages from the MMU as fast as possible.
+                PMR must be non-sparse!
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+
+@Input          ui32PageCount           Number of pages to unmap
+
+@Input          uiLog2PageSize          log2 size of the page
+
+@Return         PVRSRV_OK if the PMR was successfully unmapped
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_UnmapPMRFast(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrBase,
+                 IMG_UINT32 ui32PageCount,
+                 IMG_UINT32 uiLog2PageSize);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+/*************************************************************************/ /*!
+@Function       MMU_RemapPage
+
+@Description    Remap a single page from a PMR in place.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddr               Device virtual address of the page
+
+@Input          uiLog2HeapPageSize      log2 size of the page
+
+@Input          psOriginPMR             PMR to remap
+
+@Input          ui32LogicalPgOffset     Page offset into the PMR of the page
+                                        to remap.
+
+@Return         PVRSRV_OK if the PMR was successfully re-mapped
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_RemapPage(MMU_CONTEXT *psMMUContext,
+              PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+              IMG_DEV_VIRTADDR sDevVAddr,
+              IMG_UINT32 uiLog2HeapPageSize,
+              PMR *psOriginPMR,
+              IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquireBaseAddr
+
+@Description    Acquire the device physical address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         psPhysAddr              Device physical address of the base level
+                                        MMU object
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquireBaseAddr(MMU_CONTEXT *psMMUContext, IMG_DEV_PHYADDR *psPhysAddr);
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquireCPUBaseAddr
+
+@Description    Acquire the CPU Virtual Address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         ppvCPUVAddr             CPU Virtual Address of the base level
+                                        MMU object
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquireCPUBaseAddr(MMU_CONTEXT *psMMUContext, void **ppvCPUVAddr);
+
+/*************************************************************************/ /*!
+@Function       MMU_ReleaseBaseAddr
+
+@Description    Release the device physical address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+void
+MMU_ReleaseBaseAddr(MMU_CONTEXT *psMMUContext);
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+/***********************************************************************************/ /*!
+@Function       MMU_SetOSid
+
+@Description    Set the OSid associated with the application (and the MMU Context)
+
+@Input          psMMUContext            MMU context to store the OSid on
+
+@Input          ui32OSid                the OSid in question
+
+@Input          ui32OSidReg             The value that the firmware will assign to the
+                                        registers.
+
+@Input          bOSidAxiProt            Toggles whether the AXI prot bit will be set or
+                                        not.
+@Return None
+*/
+/***********************************************************************************/
+
+void MMU_SetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32OSid,
+                  IMG_UINT32 ui32OSidReg, IMG_BOOL bOSidAxiProt);
+
+/***********************************************************************************/ /*!
+@Function       MMU_GetOSid
+
+@Description    Retrieve the OSid associated with the MMU context.
+
+@Input          psMMUContext            MMU context in which the OSid is stored
+
+@Output         pui32OSid               The OSid in question
+
+@Output         pui32OSidReg            The OSid that the firmware will assign to the
+                                        registers.
+
+@Output         pbOSidAxiProt           Toggles whether the AXI prot bit will be set or
+                                        not.
+@Return None
+*/
+/***********************************************************************************/
+
+void MMU_GetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 * pui32OSid,
+                  IMG_UINT32 * pui32OSidReg, IMG_BOOL *pbOSidAxiProt);
+#endif
+
+/*************************************************************************/ /*!
+@Function       MMU_AppendCacheFlags
+
+@Description    Set the cache flags to the bitwise or of themselves and the
+                specified input flags, i.e. ui32CacheFlags |= ui32NewCacheFlags,
+                atomically.
+
+@Input          psMMUContext            MMU context
+
+@Input          ui32NewCacheFlags       Cache flags to append.
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_AppendCacheFlags(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32NewCacheFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_GetAndResetCacheFlags
+
+@Description    Clears MMU context flags, atomically.
+
+@Input          psMMUContext            MMU context
+
+@Return         Previous MMU context cache flags.
+*/
+/*****************************************************************************/
+IMG_UINT32 MMU_GetAndResetCacheFlags(MMU_CONTEXT *psMMUContext);
+
+typedef struct _MMU_LEVEL_DATA_
+{
+	IMG_UINT32	ui32Index;
+	IMG_UINT32	ui32NumOfEntries;
+	const IMG_CHAR *psDebugStr;
+	IMG_UINT8	uiBytesPerEntry;
+	IMG_UINT64	ui64Address;
+} MMU_LEVEL_DATA;
+
+typedef enum _MMU_FAULT_TYPE_
+{
+	MMU_FAULT_TYPE_UNKNOWN = 0, /* If fault is not analysed by Host */
+	MMU_FAULT_TYPE_PM,
+	MMU_FAULT_TYPE_NON_PM,
+} MMU_FAULT_TYPE;
+
+typedef struct _MMU_FAULT_DATA_
+{
+	MMU_LEVEL	eTopLevel;
+	MMU_FAULT_TYPE	eType;
+	MMU_LEVEL_DATA	sLevelData[MMU_LEVEL_LAST];
+} MMU_FAULT_DATA;
+
+/*************************************************************************/ /*!
+@Function       MMU_CheckFaultAddress
+
+@Description    Check the specified MMU context to see if the provided address
+                should be valid
+
+@Input          psMMUContext            MMU context to store the data on
+
+@Input          psDevVAddr              Address to check
+
+@Output          psOutFaultData          To store fault details after checking
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_CheckFaultAddress(MMU_CONTEXT *psMMUContext,
+                           IMG_DEV_VIRTADDR *psDevVAddr,
+                           MMU_FAULT_DATA *psOutFaultData);
+
+/*************************************************************************/ /*!
+@Function       MMU_IsVDevAddrValid
+@Description    Checks if given address is valid.
+@Input          psMMUContext MMU context to store the data on
+@Input          uiLog2PageSize page size
+@Input          sDevVAddr Address to check
+@Return         IMG_TRUE of address is valid
+*/ /**************************************************************************/
+IMG_BOOL MMU_IsVDevAddrValid(MMU_CONTEXT *psMMUContext,
+                             IMG_UINT32 uiLog2PageSize,
+                             IMG_DEV_VIRTADDR sDevVAddr);
+
+#if defined(PDUMP)
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextDerivePCPDumpSymAddr
+
+@Description    Derives a PDump Symbolic address for the top level MMU object
+
+@Input          psMMUContext                    MMU context to operate on
+
+@Input          pszPDumpSymbolicNameBuffer      Buffer to write the PDump symbolic
+                                                address to
+
+@Input          uiPDumpSymbolicNameBufferSize   Size of the buffer
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ContextDerivePCPDumpSymAddr(MMU_CONTEXT *psMMUContext,
+                                IMG_CHAR *pszPDumpSymbolicNameBuffer,
+                                size_t uiPDumpSymbolicNameBufferSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_PDumpWritePageCatBase
+
+@Description    PDump write of the top level MMU object to a device register
+
+@Input          psMMUContext        MMU context to operate on
+
+@Input          pszSpaceName        PDump name of the mem/reg space
+
+@Input          uiOffset            Offset to write the address to
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                                       const IMG_CHAR *pszSpaceName,
+                                       IMG_DEVMEM_OFFSET_T uiOffset,
+                                       IMG_UINT32 ui32WordSize,
+                                       IMG_UINT32 ui32AlignShift,
+                                       IMG_UINT32 ui32Shift,
+                                       PDUMP_FLAGS_T uiPdumpFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquirePDumpMMUContext
+
+@Description    Acquire a reference to the PDump MMU context for this MMU
+                context
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         pui32PDumpMMUContextID  PDump MMU context ID
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquirePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                           IMG_UINT32 *pui32PDumpMMUContextID,
+                           IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_ReleasePDumpMMUContext
+
+@Description    Release a reference to the PDump MMU context for this MMU context
+
+@Input          psMMUContext            MMU context to operate on
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ReleasePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                           IMG_UINT32 ui32PDumpFlags);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(MMU_PDumpWritePageCatBase)
+#endif
+static INLINE void
+MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                          const IMG_CHAR *pszSpaceName,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32WordSize,
+                          IMG_UINT32 ui32AlignShift,
+                          IMG_UINT32 ui32Shift,
+                          PDUMP_FLAGS_T uiPdumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMMUContext);
+	PVR_UNREFERENCED_PARAMETER(pszSpaceName);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32WordSize);
+	PVR_UNREFERENCED_PARAMETER(ui32AlignShift);
+	PVR_UNREFERENCED_PARAMETER(ui32Shift);
+	PVR_UNREFERENCED_PARAMETER(uiPdumpFlags);
+}
+#endif /* PDUMP */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*************************************************************************/ /*!
+@Function       MMU_CacheInvalidateKick
+
+@Description    Kicks the Firmware to invalidate caches
+
+@Input          psDeviceNode            Pointer to the device node
+@Output         puiRequiredSyncValue    Value the associated sync prim will be
+                                        updated to after the kick is finished
+                                        (parameter ignored if NULL)
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_CacheInvalidateKick(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                     IMG_UINT32 *puiRequiredSyncValue);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#endif /* #ifdef MMU_COMMON_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/opaque_types.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/opaque_types.h
new file mode 100644
index 000000000000..766bc22ea418
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/opaque_types.h
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File
+@Title          Opaque Types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines opaque types for various services types
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef SERVICES_OPAQUE_TYPES_H
+#define SERVICES_OPAQUE_TYPES_H
+
+#include "img_defs.h"
+#include "img_types.h"
+
+typedef struct _PVRSRV_DEVICE_NODE_ *PPVRSRV_DEVICE_NODE;
+typedef const struct _PVRSRV_DEVICE_NODE_ *PCPVRSRV_DEVICE_NODE;
+
+#endif /* SERVICES_OPAQUE_TYPES_H */
+
+/******************************************************************************
+ End of file (opaque_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/os_apphint.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/os_apphint.h
new file mode 100644
index 000000000000..93c9272498e5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/os_apphint.h
@@ -0,0 +1,186 @@
+/*************************************************************************/ /*!
+@File           os_apphint.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS-independent interface for retrieving apphints
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_defs.h"
+#if defined(__linux__)
+#include "km_apphint.h"
+#include "device.h"
+#else
+#include "services_client_porting.h"
+#endif
+#if !defined(OS_APPHINT_H)
+#define OS_APPHINT_H
+
+/*! Supplied to os_get_apphint_XXX() functions when the param/AppHint is
+ * applicable to all devices and not a specific device. Typically used
+ * for server-wide build and module AppHints.
+ */
+#define APPHINT_NO_DEVICE (NULL)
+
+#if defined(__linux__) && !defined(DOXYGEN)
+static INLINE IMG_UINT os_get_apphint_UINT32(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT32 *pAppHintDefault, IMG_UINT32 *pVal) {
+	return !pvr_apphint_get_uint32(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_UINT64(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT64 *pAppHintDefault, IMG_UINT64 *pVal) {
+	return !pvr_apphint_get_uint64(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_BOOL(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_BOOL *pAppHintDefault, IMG_BOOL *pVal) {
+	return !pvr_apphint_get_bool(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_STRING(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_CHAR *pAppHintDefault, IMG_CHAR *buffer, size_t size) {
+	return !pvr_apphint_get_string(device, id, buffer, size);
+}
+
+#define OSGetAppHintUINT32(device, state, name, appHintDefault, value) \
+	os_get_apphint_UINT32(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintUINT64(device, state, name, appHintDefault, value) \
+	os_get_apphint_UINT64(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintBOOL(device, state, name, appHintDefault, value) \
+	os_get_apphint_BOOL(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	os_get_apphint_STRING(device, state, APPHINT_ID_ ## name, appHintDefault, buffer, size)
+
+
+#define OSCreateAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#define OSFreeAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#else /* defined(__linux__) && !defined(DOXYGEN) */
+
+/**************************************************************************/ /*!
+@def OSGetAppHintUINT32(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint32 server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintUINT32(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintUINT64(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint64 server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintUINT64(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintBOOL(state, name, appHintDefault, value)
+@Description    Interface for retrieval of IMG_BOOL server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintBOOL(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_BOOL_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintSTRING(state, name, appHintDefault, buffer, size)
+@Description    Interface for retrieval of string server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         buffer            Buffer used to return app hint string.
+@Input			size			  Size of the buffer.
+ */ /**************************************************************************/
+#define OSGetAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	(PVR_UNREFERENCED_PARAMETER(size), PVRSRVGetAppHint(state, # name, IMG_STRING_TYPE, appHintDefault, buffer))
+
+/**************************************************************************/ /*!
+@def OSCreateAppHintState(state)
+@Description    Creates the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSCreateAppHintState(state) \
+	PVRSRVCreateAppHintState(IMG_SRV_UM, 0, state)
+
+/**************************************************************************/ /*!
+@def OSFreeAppHintState
+@Description    Free the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSFreeAppHintState(state) \
+	PVRSRVFreeAppHintState(IMG_SRV_UM, state)
+
+#endif /* defined(__linux__) */
+
+#endif /* OS_APPHINT_H */
+
+/******************************************************************************
+ End of file (os_apphint.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/osconnection_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/osconnection_server.h
new file mode 100644
index 000000000000..88f7aab29dca
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/osconnection_server.h
@@ -0,0 +1,137 @@
+/**************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    API for OS specific callbacks from server side connection
+                management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#ifndef OSCONNECTION_SERVER_H
+#define OSCONNECTION_SERVER_H
+
+#include "handle.h"
+#include "osfunc.h"
+
+/*! Function not implemented definition. */
+#define OSCONNECTION_SERVER_NOT_IMPLEMENTED 0
+/*! Assert used for OSCONNECTION_SERVER_NOT_IMPLEMENTED. */
+#define OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT() PVR_ASSERT(OSCONNECTION_SERVER_NOT_IMPLEMENTED)
+
+#if defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData);
+PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData);
+
+PVRSRV_ERROR OSConnectionSetHandleOptions(PVRSRV_HANDLE_BASE *psHandleBase);
+
+PVRSRV_DEVICE_NODE* OSGetDevNode(CONNECTION_DATA *psConnection);
+
+struct drm_file;
+
+struct drm_file* OSGetDRMFile(CONNECTION_DATA *psConnection);
+
+#else	/* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionPrivateDataInit)
+#endif
+/*************************************************************************/ /*!
+@Function       OSConnectionPrivateDataInit
+@Description    Allocates and initialises any OS-specific private data
+                relating to a connection.
+                Called from PVRSRVCommonConnectionConnect().
+@Input          pvOSData            pointer to any OS private data
+@Output         phOsPrivateData     handle to the created connection
+                                    private data
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData)
+{
+	PVR_UNREFERENCED_PARAMETER(phOsPrivateData);
+	PVR_UNREFERENCED_PARAMETER(pvOSData);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionPrivateDataDeInit)
+#endif
+/*************************************************************************/ /*!
+@Function       OSConnectionPrivateDataDeInit
+@Description    Frees previously allocated OS-specific private data
+                relating to a connection.
+@Input          hOsPrivateData      handle to the connection private data
+                                    to be freed
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData)
+{
+	PVR_UNREFERENCED_PARAMETER(hOsPrivateData);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionSetHandleOptions)
+#endif
+static INLINE PVRSRV_ERROR OSConnectionSetHandleOptions(PVRSRV_HANDLE_BASE *psHandleBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psHandleBase);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSGetDevNode)
+#endif
+static INLINE PVRSRV_DEVICE_NODE* OSGetDevNode(CONNECTION_DATA *psConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return NULL;
+}
+#endif	/* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+
+
+#endif /* OSCONNECTION_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/osdi_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/osdi_impl.h
new file mode 100644
index 000000000000..5be78d6c2b62
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/osdi_impl.h
@@ -0,0 +1,213 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions and types for creating Debug Info implementations.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSDI_IMPL_H
+#define OSDI_IMPL_H
+
+#if defined(__KERNEL__) && defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __KERNEL__ && __linux__ */
+
+#include "di_common.h"
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+/*! Implementation callbacks. Those operations are performed on native
+ * implementation handles. */
+typedef struct OSDI_IMPL_ENTRY_CB
+{
+    /*! @Function pfnWrite
+     *
+     * @Description
+     * Writes the binary data of the DI entry to the output sync, whatever that
+     * may be for the DI implementation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pvData data
+     * @Input uiSize pvData length
+     */
+    void (*pfnWrite)(void  *pvNativeHandle, const void *pvData,
+                     IMG_UINT32 uiSize);
+
+    /*! @Function pfnVPrintf
+     *
+     * @Description
+     * Implementation of the 'vprintf' operation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pszFmt NUL-terminated format string
+     * @Input va_list variable length argument list
+     */
+    __printf(2, 0)
+    void (*pfnVPrintf)(void *pvNativeHandle, const IMG_CHAR *pszFmt, va_list pArgs);
+
+    /*! @Function pfnPuts
+     *
+     * @Description
+     * Implementation of the 'puts' operation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pszStr NUL-terminated string
+     */
+    void (*pfnPuts)(void *pvNativeHandle, const IMG_CHAR *pszStr);
+
+    /*! @Function pfnHasOverflowed
+     *
+     * @Description
+     * Checks if the native implementation's buffer has overflowed.
+     *
+     * @Input pvNativeHandle native implementation handle
+     */
+    IMG_BOOL (*pfnHasOverflowed)(void *pvNativeHandle);
+} OSDI_IMPL_ENTRY_CB;
+
+/*! Debug Info entry specialisation. */
+struct OSDI_IMPL_ENTRY
+{
+    /*! Pointer to the private data. The data originates from DICreateEntry()
+     *  function. */
+    void *pvPrivData;
+    /*! Pointer to the implementation native handle. */
+    void *pvNative;
+    /*! Implementation entry callbacks. */
+    OSDI_IMPL_ENTRY_CB *psCb;
+}; /* OSDI_IMPL_ENTRY is already typedef-ed in di_common.h */
+
+/*! Debug Info implementation callbacks. */
+typedef struct OSDI_IMPL_CB
+{
+    /*! Initialise implementation callback.
+     */
+    PVRSRV_ERROR (*pfnInit)(void);
+
+    /*! De-initialise implementation callback.
+     */
+    void (*pfnDeInit)(void);
+
+    /*! @Function pfnCreateEntry
+     *
+     * @Description
+     * Creates entry of eType type with pszName in the pvNativeGroup parent
+     * group. The entry is an abstract term which depends on the implementation,
+     * e.g.: a file in DebugFS.
+     *
+     * @Input pszName: name of the entry
+     * @Input eType: type of the entry
+     * @Input psIterCb: iterator implementation for the entry
+     * @Input pvPrivData: data that will be passed to the iterator callbacks
+     *                    in OSDI_IMPL_ENTRY - it can be retrieved by calling
+     *                    DIGetPrivData() function
+     * @Input pvNativeGroup: implementation specific handle to the parent group
+     *
+     * @Output pvNativeEntry: implementation specific handle to the entry
+     *
+     * return PVRSRV_ERROR error code
+     */
+    PVRSRV_ERROR (*pfnCreateEntry)(const IMG_CHAR *pszName,
+                                   DI_ENTRY_TYPE eType,
+                                   const DI_ITERATOR_CB *psIterCb,
+                                   void *pvPrivData,
+                                   void *pvNativeGroup,
+                                   void **pvNativeEntry);
+
+    /*! @Function pfnDestroyEntry
+     *
+     * @Description
+     * Destroys native entry.
+     *
+     * @Input psNativeEntry: handle to the entry
+     */
+    void (*pfnDestroyEntry)(void *psNativeEntry);
+
+    /*! @Function pfnCreateGroup
+     *
+     * @Description
+     * Creates group with pszName in the psNativeParentGroup parent group.
+     * The group is an abstract term which depends on the implementation,
+     * e.g.: a directory in DebugFS.
+     *
+     * @Input pszName: name of the entry
+     * @Input psNativeParentGroup: implementation specific handle to the parent
+     *                             group
+     *
+     * @Output psNativeGroup: implementation specific handle to the group
+     *
+     * return PVRSRV_ERROR error code
+     */
+    PVRSRV_ERROR (*pfnCreateGroup)(const IMG_CHAR *pszName,
+                                   void *psNativeParentGroup,
+                                   void **psNativeGroup);
+
+    /*! @Function pfnDestroyGroup
+     *
+     * @Description
+     * Destroys native group.
+     *
+     * @Input psNativeGroup: handle to the group
+     */
+    void (*pfnDestroyGroup)(void *psNativeGroup);
+} OSDI_IMPL_CB;
+
+/*! @Function DIRegisterImplementation
+ *
+ * @Description
+ * Registers Debug Info implementations with the framework. The framework takes
+ * the ownership of the implementation and will clean up the resources when
+ * it's de-initialised.
+ *
+ * @Input pszName: name of the implementation
+ * @Input psImplCb: implementation callbacks
+ *
+ * @Return PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DIRegisterImplementation(const IMG_CHAR *pszName,
+                                      const OSDI_IMPL_CB *psImplCb);
+
+#endif /* OSDI_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/osfunc.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/osfunc.h
new file mode 100644
index 000000000000..67ed3d24895b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/osfunc.h
@@ -0,0 +1,1771 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS functions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifdef DEBUG_RELEASE_BUILD
+#pragma optimize( "", off )
+#define DEBUG		1
+#endif
+
+#ifndef OSFUNC_H
+/*! @cond Doxygen_Suppress */
+#define OSFUNC_H
+/*! @endcond */
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include "kernel_nospec.h"
+#if !defined(NO_HARDWARE)
+#include <linux/io.h>
+
+#endif
+#endif
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#if defined(__QNXNTO__)
+#include <stdio.h>
+#include <string.h>
+#endif
+
+#if defined(INTEGRITY_OS)
+#include <stdio.h>
+#include <string.h>
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "device.h"
+#include "pvrsrv_device.h"
+#include "cache_ops.h"
+#include "osfunc_common.h"
+#if defined(SUPPORT_DMA_TRANSFER)
+#include "dma_km.h"
+#include "pmr.h"
+#endif
+
+/******************************************************************************
+ * Static defines
+ *****************************************************************************/
+/*!
+ * Returned by OSGetCurrentProcessID() and OSGetCurrentThreadID() if the OS
+ * is currently operating in the interrupt context.
+ */
+#define KERNEL_ID			0xffffffffL
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define OSConfineArrayIndexNoSpeculation(index, size) array_index_nospec((index), (size))
+#elif defined(__QNXNTO__)
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#elif defined(INTEGRITY_OS)
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#else
+/*************************************************************************/ /*!
+@Function       OSConfineArrayIndexNoSpeculation
+@Description    This macro aims to avoid code exposure to Cache Timing
+                Side-Channel Mechanisms which rely on speculative code
+                execution (Variant 1). It does so by ensuring a value to be
+                used as an array index will be set to zero if outside of the
+                bounds of the array, meaning any speculative execution of code
+                which uses this suitably adjusted index value will not then
+                attempt to load data from memory outside of the array bounds.
+                Code calling this macro must still first verify that the
+                original unmodified index value is within the bounds of the
+                array, and should then only use the modified value returned
+                by this function when accessing the array itself.
+                NB. If no OS-specific implementation of this macro is
+                defined, the original index is returned unmodified and no
+                protection against the potential exploit is provided.
+@Input          index    The original array index value that would be used to
+                         access the array.
+@Input          size     The number of elements in the array being accessed.
+@Return         The value to use for the array index, modified so that it
+                remains within array bounds.
+*/ /**************************************************************************/
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#if !defined(DOXYGEN)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#endif
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSClockns64
+@Description    This function returns the number of ticks since system boot
+                expressed in nanoseconds. Unlike OSClockns, OSClockns64 has
+                a near 64-bit range.
+@Return         The 64-bit clock value, in nanoseconds.
+*/ /**************************************************************************/
+IMG_UINT64 OSClockns64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockus64
+@Description    This function returns the number of ticks since system boot
+                expressed in microseconds. Unlike OSClockus, OSClockus64 has
+                a near 64-bit range.
+@Return         The 64-bit clock value, in microseconds.
+*/ /**************************************************************************/
+IMG_UINT64 OSClockus64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockus
+@Description    This function returns the number of ticks since system boot
+                in microseconds.
+@Return         The 32-bit clock value, in microseconds.
+*/ /**************************************************************************/
+IMG_UINT32 OSClockus(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockms
+@Description    This function returns the number of ticks since system boot
+                in milliseconds.
+@Return         The 32-bit clock value, in milliseconds.
+*/ /**************************************************************************/
+IMG_UINT32 OSClockms(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicns64
+@Description    This function returns a clock value based on the system
+                monotonic clock.
+@Output         pui64Time     The 64-bit clock value, in nanoseconds.
+@Return         Error Code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSClockMonotonicns64(IMG_UINT64 *pui64Time);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicus64
+@Description    This function returns a clock value based on the system
+                monotonic clock.
+@Output         pui64Time     The 64-bit clock value, in microseconds.
+@Return         Error Code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSClockMonotonicus64(IMG_UINT64 *pui64Time);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicRawns64
+@Description    This function returns a clock value based on the system
+                monotonic raw clock.
+@Return         64bit ns timestamp
+*/ /**************************************************************************/
+IMG_UINT64 OSClockMonotonicRawns64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicRawus64
+@Description    This function returns a clock value based on the system
+                monotonic raw clock.
+@Return         64bit us timestamp
+*/ /**************************************************************************/
+IMG_UINT64 OSClockMonotonicRawus64(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageSize
+@Description    This function returns the page size.
+                If the OS is not using memory mappings it should return a
+                default value of 4096.
+@Return         The size of a page, in bytes.
+*/ /**************************************************************************/
+size_t OSGetPageSize(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageShift
+@Description    This function returns the page size expressed as a power of
+                two. A number of pages, left-shifted by this value, gives the
+                equivalent size in bytes.
+                If the OS is not using memory mappings it should return a
+                default value of 12.
+@Return         The page size expressed as a power of two.
+*/ /**************************************************************************/
+size_t OSGetPageShift(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageMask
+@Description    This function returns a bitmask that may be applied to an
+                address to mask off the least-significant bits so as to
+                leave the start address of the page containing that address.
+@Return         The page mask.
+*/ /**************************************************************************/
+size_t OSGetPageMask(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetOrder
+@Description    This function returns the order of power of two for a given
+                size. Eg. for a uSize of 4096 bytes the function would
+                return 12 (4096 = 2^12).
+@Input          uSize     The size in bytes.
+@Return         The order of power of two.
+*/ /**************************************************************************/
+size_t OSGetOrder(size_t uSize);
+
+/*************************************************************************/ /*!
+@Function       OSGetRAMSize
+@Description    This function returns the total amount of GPU-addressable
+                memory provided by the system. In other words, after loading
+                the driver this would be the largest allocation an
+                application would reasonably expect to be able to make.
+                Note that this is function is not expected to return the
+                current available memory but the amount which would be
+                available on startup.
+@Return         Total GPU-addressable memory size, in bytes.
+*/ /**************************************************************************/
+IMG_UINT64 OSGetRAMSize(void);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a Mid-level Interrupt Service Routine (MISR).
+@Input  pvData  Pointer to MISR specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_MISR)(void *pvData);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a thread entry point function.
+@Input  pvData  Pointer to thread specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_THREAD)(void *pvData);
+
+/*************************************************************************/ /*!
+@Function       OSInstallMISR
+@Description    Installs a Mid-level Interrupt Service Routine (MISR)
+                which handles higher-level processing of interrupts from
+                the device (GPU).
+                An MISR runs outside of interrupt context, and so may be
+                descheduled. This means it can contain code that would
+                not be permitted in the LISR.
+                An MISR is invoked when OSScheduleMISR() is called. This
+                call should be made by installed LISR once it has completed
+                its interrupt processing.
+                Multiple MISRs may be installed by the driver to handle
+                different causes of interrupt.
+@Input          pfnMISR       pointer to the function to be installed
+                              as the MISR
+@Input          hData         private data provided to the MISR
+@Input          pszMisrName   Name describing purpose of MISR worker thread
+                              (Must be a string literal).
+@Output         hMISRData     handle to the installed MISR (to be used
+                              for a subsequent uninstall)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInstallMISR(IMG_HANDLE *hMISRData,
+							PFN_MISR pfnMISR,
+							void *hData,
+							const IMG_CHAR *pszMisrName);
+
+/*************************************************************************/ /*!
+@Function       OSUninstallMISR
+@Description    Uninstalls a Mid-level Interrupt Service Routine (MISR).
+@Input          hMISRData     handle to the installed MISR
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSUninstallMISR(IMG_HANDLE hMISRData);
+
+/*************************************************************************/ /*!
+@Function       OSScheduleMISR
+@Description    Schedules a Mid-level Interrupt Service Routine (MISR) to be
+                executed. An MISR should be executed outside of interrupt
+                context, for example in a work queue.
+@Input          hMISRData     handle to the installed MISR
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSScheduleMISR(IMG_HANDLE hMISRData);
+
+/*************************************************************************/ /*!
+@Function       OSSyncIRQ
+@Description    Wait for LISR to complete. If you use this function while
+                holding a resource which the IRQ handler also requires,
+                you will deadlock.
+@Input          ui32IRQ     IRQ number
+*/ /**************************************************************************/
+void OSSyncIRQ(IMG_UINT32 ui32IRQ);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a function implementing debug dump of thread-specific
+                data.
+@Input          pfnDumpDebugPrintf      Used to specify the print function used
+                                        to dump any debug information. If this
+                                        argument is NULL then a default print
+                                        function will be used.
+@Input          pvDumpDebugFile         File identifier to be passed to the
+                                        print function if specified.
+*/ /**************************************************************************/
+
+typedef void (*PFN_THREAD_DEBUG_DUMP)(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                                      void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSThreadCreate
+@Description    Creates a kernel thread and starts it running. The caller
+                is responsible for informing the thread that it must finish
+                and return from the pfnThread function. It is not possible
+                to kill or terminate it. The new thread runs with the default
+                priority provided by the Operating System.
+                Note: Kernel threads are freezable which means that they
+                can be frozen by the kernel on for example driver suspend.
+                Because of that only OSEventObjectWaitKernel() function should
+                be used to put kernel threads in waiting state.
+@Output         phThread            Returned handle to the thread.
+@Input          pszThreadName       Name to assign to the thread.
+@Input          pfnThread           Thread entry point function.
+@Input          pfnDebugDumpCB      Used to dump info of the created thread
+@Input          bIsSupportingThread Set, if summary of this thread needs to
+                                    be dumped in debug_dump
+@Input          hData               Thread specific data pointer for pfnThread().
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR OSThreadCreate(IMG_HANDLE *phThread,
+                            IMG_CHAR *pszThreadName,
+                            PFN_THREAD pfnThread,
+                            PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                            IMG_BOOL bIsSupportingThread,
+                            void *hData);
+
+/*! Available priority levels for the creation of a new Kernel Thread. */
+typedef enum priority_levels
+{
+	OS_THREAD_NOSET_PRIORITY = 0,   /* With this option the priority level is the default for the given OS */
+	OS_THREAD_HIGHEST_PRIORITY,
+	OS_THREAD_HIGH_PRIORITY,
+	OS_THREAD_NORMAL_PRIORITY,
+	OS_THREAD_LOW_PRIORITY,
+	OS_THREAD_LOWEST_PRIORITY,
+	OS_THREAD_LAST_PRIORITY     /* This must be always the last entry */
+} OS_THREAD_LEVEL;
+
+/*************************************************************************/ /*!
+@Function       OSThreadCreatePriority
+@Description    As OSThreadCreate, this function creates a kernel thread and
+                starts it running. The difference is that with this function
+                is possible to specify the priority used to schedule the new
+                thread.
+
+@Output         phThread            Returned handle to the thread.
+@Input          pszThreadName       Name to assign to the thread.
+@Input          pfnThread           Thread entry point function.
+@Input          pfnDebugDumpCB      Used to dump info of the created thread
+@Input          bIsSupportingThread Set, if summary of this thread needs to
+                                    be dumped in debug_dump
+@Input          hData               Thread specific data pointer for pfnThread().
+@Input          eThreadPriority     Priority level to assign to the new thread.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSThreadCreatePriority(IMG_HANDLE *phThread,
+                                    IMG_CHAR *pszThreadName,
+                                    PFN_THREAD pfnThread,
+                                    PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                                    IMG_BOOL bIsSupportingThread,
+                                    void *hData,
+                                    OS_THREAD_LEVEL eThreadPriority);
+
+/*************************************************************************/ /*!
+@Function       OSThreadDestroy
+@Description    Waits for the thread to end and then destroys the thread
+                handle memory. This function will block and wait for the
+                thread to finish successfully, thereby providing a sync point
+                for the thread completing its work. No attempt is made to kill
+                or otherwise terminate the thread.
+@Input          hThread   The thread handle returned by OSThreadCreate().
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSThreadDestroy(IMG_HANDLE hThread);
+
+/*************************************************************************/ /*!
+@Function       OSIsMapPhysNonContigSupported
+@Description    Determine if the kernel mapping of physically non-contiguous
+                pages is supported in the OS layer
+
+                Note: For this function to return IMG_TRUE a full implementation
+                of the following functions is required:
+                OSMapPhysArrayToLin
+                OSUnMapPhysArrayToLin
+
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+IMG_BOOL OSIsMapPhysNonContigSupported(void);
+
+/*************************************************************************/ /*!
+@Function       OSUnMapPhysArrayToLin
+@Description    UnMap a kernel virtual address that was produced by mapping
+                a number of Pages in OSMapPhysArrayToLin.
+
+                Note: This function is only required if the Non contiguous
+                allocation feature is required, in this case
+                OSIsMapPhysNonContigSupported should return IMG_TRUE.
+                If not required this function should return
+                PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED
+
+@Input          pvLinAddr       The linear mapping to be unmapped
+@Input          pvPrivData      Optional implementation specific data.
+
+@Return         None
+*/ /**************************************************************************/
+void OSUnMapPhysArrayToLin(void *pvLinAddr, void *pvPrivData);
+
+/*************************************************************************/ /*!
+@Function       OSMapPhysArrayToLin
+@Description    Given an array of OS page physical addresses and a count
+                of said Pages, this function will map those pages into a
+                virtually contiguous range, this allows for non physically
+                contiguous allocations to be mapped into the kernel.
+                Page size is assumed to be OS page size.
+
+                Note: This function is only required if the Non contiguous
+                allocation feature is required, in this case
+                OSIsMapPhysNonContigSupported should return IMG_TRUE.
+                If not required this function should return
+                PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED
+
+@Input          pPagePA          Array of Pages
+@Input          uiPageCount      Page count of pulPages
+@Output         ppvLinAddr       Pointer to a virtual kernel address of the
+                                 mapped Pages.
+@Output         ppvPrivData      Optional implementation specific data.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSMapPhysArrayToLin(IMG_CPU_PHYADDR pPagePA[],
+                                 IMG_UINT32 uiPageCount,
+                                 void **ppvLinAddr,
+                                 void **ppvPrivData);
+
+/*************************************************************************/ /*!
+@Function       OSMapPhysToLin
+@Description    Maps physical memory into a linear address range.
+@Input          BasePAddr    physical CPU address
+@Input          ui32Bytes    number of bytes to be mapped
+@Input          uiFlags      flags denoting the caching mode to be employed
+                             for the mapping (uncached/write-combined,
+                             cached coherent or cached incoherent).
+                             See pvrsrv_memallocflags.h for full flag bit
+                             definitions.
+@Return         Pointer to the new mapping if successful, NULL otherwise.
+*/ /**************************************************************************/
+void *OSMapPhysToLin(IMG_CPU_PHYADDR BasePAddr, size_t ui32Bytes, PVRSRV_MEMALLOCFLAGS_T uiFlags);
+
+/*************************************************************************/ /*!
+@Function       OSUnMapPhysToLin
+@Description    Unmaps physical memory previously mapped by OSMapPhysToLin().
+@Input          pvLinAddr    the linear mapping to be unmapped
+@Input          ui32Bytes    number of bytes to be unmapped
+@Return         IMG_TRUE if unmapping was successful, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSUnMapPhysToLin(void *pvLinAddr, size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheFlushRangeKM
+@Description    Clean and invalidate the CPU cache for the specified
+                address range.
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              flushed
+@Input          pvVirtEnd     virtual end address of the range to be
+                              flushed
+@Input          sCPUPhysStart physical start address of the range to be
+                              flushed
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              flushed
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheCleanRangeKM
+@Description    Clean the CPU cache for the specified address range.
+                This writes out the contents of the cache and clears the
+                'dirty' bit (which indicates the physical memory is
+                consistent with the cache contents).
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              cleaned
+@Input          pvVirtEnd     virtual end address of the range to be
+                              cleaned
+@Input          sCPUPhysStart physical start address of the range to be
+                              cleaned
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              cleaned
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheInvalidateRangeKM
+@Description    Invalidate the CPU cache for the specified address range.
+                The cache must reload data from those addresses if they
+                are accessed.
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              invalidated
+@Input          pvVirtEnd     virtual end address of the range to be
+                              invalidated
+@Input          sCPUPhysStart physical start address of the range to be
+                              invalidated
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              invalidated
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*! CPU Cache operations address domain type */
+typedef enum
+{
+	OS_CACHE_OP_ADDR_TYPE_VIRTUAL,    /*!< Operation requires CPU virtual address only */
+	OS_CACHE_OP_ADDR_TYPE_PHYSICAL,   /*!< Operation requires CPU physical address only */
+	OS_CACHE_OP_ADDR_TYPE_BOTH        /*!< Operation requires both CPU virtual & physical addresses */
+} OS_CACHE_OP_ADDR_TYPE;
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheOpAddressType
+@Description    Returns the address type (i.e. virtual/physical/both) the CPU
+                architecture performs cache maintenance operations under.
+                This is used to infer whether the virtual or physical address
+                supplied to the OSCPUCacheXXXRangeKM functions can be omitted
+                when called.
+@Input          psDevNode   device on which the allocation was made
+@Return         OS_CACHE_OP_ADDR_TYPE
+*/ /**************************************************************************/
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*! CPU Cache attributes available for retrieval, DCache unless specified */
+typedef enum _OS_CPU_CACHE_ATTRIBUTE_
+{
+	OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE, /*!< The cache line size */
+	OS_CPU_CACHE_ATTRIBUTE_COUNT      /*!< The number of attributes (must be last) */
+} OS_CPU_CACHE_ATTRIBUTE;
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheAttributeSize
+@Description    Returns the size of a given cache attribute.
+                Typically this function is used to return the cache line
+                size, but may be extended to return the size of other
+                cache attributes.
+@Input          eCacheAttribute   the cache attribute whose size should
+                                  be returned.
+@Return         The size of the specified cache attribute, in bytes.
+*/ /**************************************************************************/
+IMG_UINT32 OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE eCacheAttribute);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessID
+@Description    Returns ID of current process (thread group)
+@Return         ID of current process
+*****************************************************************************/
+IMG_PID OSGetCurrentProcessID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentVirtualProcessID
+@Description    Returns ID of current process (thread group of current
+                PID namespace)
+@Return         ID of current process in PID namespace
+*****************************************************************************/
+IMG_PID OSGetCurrentVirtualProcessID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessName
+@Description    Gets the name of current process
+@Return         Process name
+*****************************************************************************/
+IMG_CHAR *OSGetCurrentProcessName(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessVASpaceSize
+@Description    Returns the CPU virtual address space size of current process
+@Return         Process VA space size
+*/ /**************************************************************************/
+IMG_UINT64 OSGetCurrentProcessVASpaceSize(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentThreadID
+@Description    Returns ID for current thread
+@Return         ID of current thread
+*****************************************************************************/
+uintptr_t OSGetCurrentThreadID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientProcessIDKM
+@Description    Returns ID of current client process (thread group) which
+                has made a bridge call into the server.
+                For some operating systems, this may simply be the current
+                process id. For others, it may be that a dedicated thread
+                is used to handle the processing of bridge calls and that
+                some additional processing is required to obtain the ID of
+                the client process making the bridge call.
+@Return         ID of current client process
+*****************************************************************************/
+IMG_PID OSGetCurrentClientProcessIDKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientProcessNameKM
+@Description    Gets the name of current client process
+@Return         Client process name
+*****************************************************************************/
+IMG_CHAR *OSGetCurrentClientProcessNameKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSAcquireCurrentPPIDResourceRefKM
+@Description    Returns a unique process identifier for the current client
+                parent process (thread group) and takes a reference on it
+                (if required) to prevent it being freed/re-allocated.
+                This value may then be used as a unique reference to the
+                process rather than using the PID value which might be
+                reallocated to represent a further process on process
+                destruction.
+                Note that the value to be returned is an address relating to
+                the parent process (thread group) and not to just one thread.
+                It is the caller's responsibility to ensure the reference is
+                subsequently dropped (by calling OSReleasePPIDResourceRefKM())
+                to allow it to be freed when no longer required.
+@Return         Address of a kernel resource allocated for the current client
+                parent process (thread group)
+*****************************************************************************/
+uintptr_t OSAcquireCurrentPPIDResourceRefKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSReleasePPIDResourceRefKM
+@Description    Drops a reference on the unique process identifier provided.
+@Return         None
+*****************************************************************************/
+void OSReleasePPIDResourceRefKM(uintptr_t psPPIDResource);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientThreadIDKM
+@Description    Returns ID for current client thread
+                For some operating systems, this may simply be the current
+                thread id. For others, it may be that a dedicated thread
+                is used to handle the processing of bridge calls and that
+                some additional processing is require to obtain the ID of
+                the client thread making the bridge call.
+@Return         ID of current client thread
+*****************************************************************************/
+uintptr_t OSGetCurrentClientThreadIDKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSMemCmp
+@Description    Compares two blocks of memory for equality.
+@Input          pvBufA      Pointer to the first block of memory
+@Input          pvBufB      Pointer to the second block of memory
+@Input          uiLen       The number of bytes to be compared
+@Return         Value < 0 if pvBufA is less than pvBufB.
+                Value > 0 if pvBufB is less than pvBufA.
+                Value = 0 if pvBufA is equal to pvBufB.
+*****************************************************************************/
+IMG_INT OSMemCmp(void *pvBufA, void *pvBufB, size_t uiLen);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesAlloc
+@Description    Allocates a number of contiguous physical pages.
+                If allocations made by this function are CPU cached then
+                OSPhyContigPagesClean has to be implemented to write the
+                cached data to memory.
+@Input          psPhysHeap    the heap from which to allocate
+@Input          uiSize        the size of the required allocation (in bytes)
+@Output         psMemHandle   a returned handle to be used to refer to this
+                              allocation
+@Output         psDevPAddr    the physical address of the allocation
+@Input          uiPid         the process ID that this allocation should
+                              be associated with
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+							PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+							IMG_PID uiPid);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesFree
+@Description    Frees a previous allocation of contiguous physical pages
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be freed
+@Return         None.
+*****************************************************************************/
+void OSPhyContigPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesMap
+@Description    Maps the specified allocation of contiguous physical pages
+                to a kernel virtual address
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be mapped
+@Input          uiSize        the size of the allocation (in bytes)
+@Input          psDevPAddr    the physical address of the allocation
+@Output         pvPtr         the virtual kernel address to which the
+                              allocation is now mapped
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle,
+						size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+						void **pvPtr);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesUnmap
+@Description    Unmaps the kernel mapping for the specified allocation of
+                contiguous physical pages
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be unmapped
+@Input          pvPtr         the virtual kernel address to which the
+                              allocation is currently mapped
+@Return         None.
+*****************************************************************************/
+void OSPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesClean
+@Description    Write the content of the specified allocation from CPU cache to
+                memory from (start + uiOffset) to (start + uiOffset + uiLength)
+                It is expected to be implemented as a cache clean operation but
+                it is allowed to fall back to a cache clean + invalidate
+                (i.e. flush).
+                If allocations returned by OSPhyContigPagesAlloc are always
+                uncached this can be implemented as nop.
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be flushed
+@Input          uiOffset      the offset in bytes from the start of the
+                              allocation from where to start flushing
+@Input          uiLength      the amount to flush from the offset in bytes
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_UINT32 uiOffset,
+                                   IMG_UINT32 uiLength);
+
+
+/*************************************************************************/ /*!
+@Function       OSInitEnvData
+@Description    Called to initialise any environment-specific data. This
+                could include initialising the bridge calling infrastructure
+                or device memory management infrastructure.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInitEnvData(void);
+
+/*************************************************************************/ /*!
+@Function       OSDeInitEnvData
+@Description    The counterpart to OSInitEnvData(). Called to free any
+                resources which may have been allocated by OSInitEnvData().
+@Return         None.
+*/ /**************************************************************************/
+void OSDeInitEnvData(void);
+
+/*************************************************************************/ /*!
+@Function       OSStringLCat
+@Description    OS function to support the BSD C strlcat() function.
+*/ /**************************************************************************/
+size_t OSStringLCat(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDstSize);
+
+/*************************************************************************/ /*!
+@Function       OSSNPrintf
+@Description    OS function to support the standard C snprintf() function.
+@Output         pStr        char array to print into
+@Input          ui32Size    maximum size of data to write (chars)
+@Input          pszFormat   format string
+*/ /**************************************************************************/
+IMG_INT32 OSSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR *pszFormat, ...) __printf(3, 4);
+
+/*************************************************************************/ /*!
+@Function       OSVSNPrintf
+@Description    Printf to IMG string using variable args (see stdarg.h).
+                This is necessary because the '...' notation does not
+                support nested function calls.
+@Input          ui32Size           maximum size of data to write (chars)
+@Input          pszFormat          format string
+@Input          vaArgs             variable args structure (from stdarg.h)
+@Output         pStr               char array to print into
+@Return         Number of character written in buffer if successful other wise -1 on error
+*/ /**************************************************************************/
+IMG_INT32 OSVSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR* pszFormat, va_list vaArgs) __printf(3, 0);
+
+/*************************************************************************/ /*!
+@Function       OSStringLength
+@Description    OS function to support the standard C strlen() function.
+*/ /**************************************************************************/
+size_t OSStringLength(const IMG_CHAR *pStr);
+
+/*************************************************************************/ /*!
+@Function       OSStringNLength
+@Description    Return the length of a string, excluding the terminating null
+                byte ('\0'), but return at most 'uiCount' bytes. Only the first
+                'uiCount' bytes of 'pStr' are interrogated.
+@Input          pStr     pointer to the string
+@Input          uiCount  the maximum length to return
+@Return         Length of the string if less than 'uiCount' bytes, otherwise
+                'uiCount'.
+*/ /**************************************************************************/
+size_t OSStringNLength(const IMG_CHAR *pStr, size_t uiCount);
+
+/*************************************************************************/ /*!
+@Function       OSStringNCompare
+@Description    OS function to support the standard C strncmp() function.
+*/ /**************************************************************************/
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                           size_t uiSize);
+
+/*************************************************************************/ /*!
+@Function       OSStringToUINT32
+@Description    Changes string to IMG_UINT32.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSStringToUINT32(const IMG_CHAR *pStr, IMG_UINT32 ui32Base,
+                              IMG_UINT32 *ui32Result);
+
+/*************************************************************************/ /*!
+@Function       OSStringUINT32ToStr
+@Description    Changes IMG_UINT32 to string
+@Input          pszBuf         Buffer to write output number string
+@Input          uSize          Size of buffer provided, i.e. size of pszBuf
+@Input          ui32Num        Number to convert to string
+@Return         Returns 0 if buffer is not sufficient to hold the number string,
+                else returns length of number string
+*/ /**************************************************************************/
+IMG_UINT32 OSStringUINT32ToStr(IMG_CHAR *pszBuf, size_t uSize, IMG_UINT32 ui32Num);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectCreate
+@Description    Create an event object.
+@Input          pszName         name to assign to the new event object.
+@Output         EventObject     the created event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectCreate(const IMG_CHAR *pszName,
+								 IMG_HANDLE *EventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectDestroy
+@Description    Destroy an event object.
+@Input          hEventObject    the event object to destroy.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectDestroy(IMG_HANDLE hEventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectSignal
+@Description    Signal an event object. Any thread waiting on that event
+                object will be woken.
+@Input          hEventObject    the event object to signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectSignal(IMG_HANDLE hEventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWait
+@Description    Wait for an event object to signal. The function is passed
+                an OS event object handle (which allows the OS to have the
+                calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after a default timeout
+                period (defined in EVENT_OBJECT_TIMEOUT_MS), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+
+
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectWait(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWaitKernel
+@Description    Wait for an event object to signal. The function is passed
+                an OS event object handle (which allows the OS to have the
+                calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after a default timeout
+                period (defined in EVENT_OBJECT_TIMEOUT_MS), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+
+                Note: This function should be used only by kernel thread.
+                This is because all kernel threads are freezable and
+                this function allows the kernel to freeze the threads
+                when waiting.
+
+                See OSEventObjectWait() for more details.
+
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__)
+PVRSRV_ERROR OSEventObjectWaitKernel(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus);
+#else
+#define OSEventObjectWaitKernel OSEventObjectWaitTimeout
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSSuspendTaskInterruptible
+@Description    Suspend the current task into interruptible state.
+@Return         none.
+*/ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__)
+void OSSuspendTaskInterruptible(void);
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWaitTimeout
+@Description    Wait for an event object to signal or timeout. The function
+                is passed an OS event object handle (which allows the OS to
+                have the calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after the specified
+                timeout period (passed in 'uiTimeoutus'), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Input          uiTimeoutus   the timeout period (in usecs)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectWaitTimeout(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectDumpDebugInfo
+@Description    Emits debug counters/stats related to the event object passed
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         None.
+*/ /**************************************************************************/
+void OSEventObjectDumpDebugInfo(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectOpen
+@Description    Open an OS handle on the specified event object.
+                This OS handle may then be used to make a thread wait for
+                that event object to signal.
+@Input          hEventObject    Event object handle.
+@Output         phOSEvent       OS handle to the returned event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectOpen(IMG_HANDLE hEventObject,
+											IMG_HANDLE *phOSEvent);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectClose
+@Description    Close an OS handle previously opened for an event object.
+@Input          hOSEventKM      OS event object handle to close.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectClose(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function      OSWaitus
+@Description   Implements a busy wait of the specified number of microseconds.
+               This function does NOT release thread quanta.
+@Input         ui32Timeus     The duration of the wait period (in us)
+@Return        None.
+*/ /**************************************************************************/
+void OSWaitus(IMG_UINT32 ui32Timeus);
+
+/*************************************************************************/ /*!
+@Function       OSSleepms
+@Description    Implements a sleep of the specified number of milliseconds.
+                This function may allow pre-emption, meaning the thread
+                may potentially not be rescheduled for a longer period.
+@Input          ui32Timems    The duration of the sleep (in ms)
+@Return         None.
+*/ /**************************************************************************/
+void OSSleepms(IMG_UINT32 ui32Timems);
+
+/*************************************************************************/ /*!
+@Function       OSReleaseThreadQuanta
+@Description    Relinquishes the current thread's execution time-slice,
+                permitting the OS scheduler to schedule another thread.
+@Return         None.
+*/ /**************************************************************************/
+void OSReleaseThreadQuanta(void);
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define OSReadMemoryBarrier() rmb()
+#else
+/*************************************************************************/ /*!
+@Function       OSReadMemoryBarrier
+@Description    Insert a read memory barrier.
+                The read memory barrier guarantees that all load (read)
+                operations specified before the barrier will appear to happen
+                before all of the load operations specified after the barrier.
+*/ /**************************************************************************/
+void OSReadMemoryBarrier(void);
+#endif
+/*************************************************************************/ /*!
+@Function       OSMemoryBarrier
+@Description    Insert a read/write memory barrier.
+                The read and write memory barrier guarantees that all load
+                (read) and all store (write) operations specified before the
+                barrier will appear to happen before all of the load/store
+                operations specified after the barrier.
+@Input          hReadback     Optional pointer to memory to read back, can be
+                useful for flushing queues in bus interconnects to RAM before
+                device (GPU) access the shared memory.
+@Return         None.
+*/ /**************************************************************************/
+void OSMemoryBarrier(volatile void *hReadback);
+/*************************************************************************/ /*!
+@Function       OSWriteMemoryBarrier
+@Description    Insert a write memory barrier.
+                The write memory barrier guarantees that all store operations
+                (writes) specified before the barrier will appear to happen
+                before all of the store operations specified after the barrier.
+@Input          hReadback     Optional pointer to memory to read back, can be
+                useful for flushing queues in bus interconnects to RAM before
+                device (GPU) access the shared memory.
+@Return         None.
+*/ /**************************************************************************/
+void OSWriteMemoryBarrier(volatile void *hReadback);
+
+/*************************************************************************/ /*!
+*/ /**************************************************************************/
+
+/* The access method is dependent on the location of the physical memory that
+ * makes up the PhyHeaps defined for the system and the CPU architecture. These
+ * macros may change in future to accommodate different access requirements.
+ */
+/*! Performs a 32 bit word read from the device memory. */
+#define OSReadDeviceMem32(addr)        (*((volatile IMG_UINT32 __force *)((void*)addr)))
+/*! Performs a 32 bit word write to the device memory. */
+#define OSWriteDeviceMem32(addr, val)  (*((volatile IMG_UINT32 __force *)((void*)addr)) = (IMG_UINT32)(val))
+/*! Performs a 32 bit word write to the device memory and issues a write memory barrier */
+#define OSWriteDeviceMem32WithWMB(addr, val) \
+	do { \
+		*((volatile IMG_UINT32 __force *)((void*)addr)) = (IMG_UINT32)(val); \
+		OSWriteMemoryBarrier(addr); \
+	} while (0)
+
+#if defined(NO_HARDWARE)
+	/* OSReadHWReg and OSWriteHWReg operations are skipped to no-op in nohw builds */
+	#define OSReadUncheckedHWReg32(addr, off) ((void)(addr), (void)(off), 0x30f73a4eU)
+#if defined(__QNXNTO__) && __SIZEOF_LONG__ == 8
+	/* This is needed for 64-bit QNX builds where the size of a long is 64 bits */
+	#define OSReadUncheckedHWReg64(addr, off) ((void)(addr), (void)(off), 0x5b376c9d30f73a4eUL)
+#else
+	#define OSReadUncheckedHWReg64(addr, off) ((void)(addr), (void)(off), 0x5b376c9d30f73a4eULL)
+#endif
+
+	#define OSWriteUncheckedHWReg32(addr, off, val) ((void)(addr), (void)(off), (void)(val))
+	#define OSWriteUncheckedHWReg64(addr, off, val) ((void)(addr), (void)(off), (void)(val))
+
+	#define OSReadHWReg32(addr, off) OSReadUncheckedHWReg32(addr, off)
+	#define OSReadHWReg64(addr, off) OSReadUncheckedHWReg64(addr, off)
+
+	#define OSWriteHWReg32(addr, off, val) OSWriteUncheckedHWReg32(addr, off, val)
+	#define OSWriteHWReg64(addr, off, val) OSWriteUncheckedHWReg64(addr, off, val)
+
+#else
+
+#if defined(__linux__) && defined(__KERNEL__)
+	#define OSReadUncheckedHWReg32(addr, off) ((IMG_UINT32)readl((IMG_BYTE __iomem *)(addr) + (off)))
+
+	/* Little endian support only */
+	#define OSReadUncheckedHWReg64(addr, off) \
+			({ \
+				__typeof__(addr) _addr = addr; \
+				__typeof__(off) _off = off; \
+				(IMG_UINT64) \
+				( \
+					( (IMG_UINT64)(readl((IMG_BYTE __iomem *)(_addr) + (_off) + 4)) << 32) \
+					| readl((IMG_BYTE __iomem *)(_addr) + (_off)) \
+				); \
+			})
+
+	#define OSWriteUncheckedHWReg32(addr, off, val) writel((IMG_UINT32)(val), (IMG_BYTE __iomem *)(addr) + (off))
+	/* Little endian support only */
+	#define OSWriteUncheckedHWReg64(addr, off, val) do \
+			{ \
+				__typeof__(addr) _addr = addr; \
+				__typeof__(off) _off = off; \
+				__typeof__(val) _val = val; \
+				writel((IMG_UINT32)((_val) & 0xffffffff), (IMG_BYTE __iomem *)(_addr) + (_off)); \
+				writel((IMG_UINT32)(((IMG_UINT64)(_val) >> 32) & 0xffffffff), (IMG_BYTE __iomem *)(_addr) + (_off) + 4); \
+			} while (0)
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+/*************************************************************************/ /*!
+@Function       OSReadUncheckedHWReg32
+@Description    Read from a 32-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to read from a location
+                but instead returns a constant value.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be read.
+@Return         The long word read.
+*/ /**************************************************************************/
+	IMG_UINT32 OSReadUncheckedHWReg32(volatile void *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset);
+
+/*************************************************************************/ /*!
+@Function       OSReadUncheckedHWReg64
+@Description    Read from a 64-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to read from a location
+                but instead returns a constant value.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be read.
+@Return         The long long word read.
+*/ /**************************************************************************/
+	IMG_UINT64 OSReadUncheckedHWReg64(volatile void *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset);
+/*************************************************************************/ /*!
+@Function       OSWriteUncheckedHWReg32
+@Description    Write to a 32-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to write to a location.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be written to.
+@Input          ui32Value          The long word to be written to the register.
+@Return         None.
+*/ /**************************************************************************/
+	void OSWriteUncheckedHWReg32(volatile void *pvLinRegBaseAddr,
+								 IMG_UINT32 ui32Offset,
+								 IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+@Function       OSWriteUncheckedHWReg64
+@Description    Write to a 64-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to write to a location.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be written to.
+@Input          ui64Value          The long long word to be written to the
+                                   register.
+@Return         None.
+*/ /**************************************************************************/
+	void OSWriteUncheckedHWReg64(volatile void *pvLinRegBaseAddr,
+								 IMG_UINT32 ui32Offset,
+								 IMG_UINT64 ui64Value);
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#if !defined(DOXYGEN)
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(DEBUG)
+static INLINE bool _NonSecureRegister(IMG_UINT32 ui32Offset)
+{
+	const IMG_UINT32 ui32PerCoreRegBankSize = RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE;
+	const IMG_UINT32 ui32RegOffsetInCoreBank = ui32Offset % ui32PerCoreRegBankSize;
+
+	if (ui32RegOffsetInCoreBank < RGX_HOST_SECURE_REGBANK_OFFSET)
+	{
+		return true;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Secure register (0x%X) accessed incorrectly. "
+								"Call OS<Read/Write>UncheckedHWReg<Width> instead with "
+								"psDevInfo->pvSecureRegsBaseKM as a register base.",
+								ui32RegOffsetInCoreBank));
+		return false;
+	}
+
+}
+#else
+#define _NonSecureRegister(ui32Offset) (true)
+#endif
+
+	/* systems using real hardware must check that regular register
+	 * operations don't attempt to access secure registers */
+	static INLINE IMG_UINT32 OSReadHWReg32(volatile void __iomem *pvLinRegBaseAddr,
+										   IMG_UINT32 ui32Offset)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		return OSReadUncheckedHWReg32(pvLinRegBaseAddr, ui32Offset);
+	}
+
+	static INLINE IMG_UINT64 OSReadHWReg64(volatile void __iomem *pvLinRegBaseAddr,
+										   IMG_UINT32 ui32Offset)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		return OSReadUncheckedHWReg64(pvLinRegBaseAddr, ui32Offset);
+	}
+
+	static INLINE void OSWriteHWReg32(volatile void __iomem *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset,
+									  IMG_UINT32 ui32Value)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		OSWriteUncheckedHWReg32(pvLinRegBaseAddr, ui32Offset, ui32Value);
+	}
+
+	static INLINE void OSWriteHWReg64(volatile void __iomem *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset,
+									  IMG_UINT64 ui64Value)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		OSWriteUncheckedHWReg64(pvLinRegBaseAddr, ui32Offset, ui64Value);
+	}
+
+#endif /* !defined(DOXYGEN) */
+#endif /* defined(NO_HARDWARE) */
+
+/*************************************************************************/ /*!
+@Description    Pointer to a timer callback function.
+@Input          pvData  Pointer to timer specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_TIMER_FUNC)(void* pvData);
+
+/*************************************************************************/ /*!
+@Function       OSAddTimer
+@Description    OS specific function to install a timer callback. The
+                timer will then need to be enabled, as it is disabled by
+                default.
+                When enabled, the callback will be invoked once the specified
+                timeout has elapsed.
+@Input          pfnTimerFunc    Timer callback
+@Input          *pvData         Callback data
+@Input          ui32MsTimeout   Callback period
+@Return         Valid handle on success, NULL if a failure
+*/ /**************************************************************************/
+IMG_HANDLE OSAddTimer(PFN_TIMER_FUNC pfnTimerFunc, void *pvData, IMG_UINT32 ui32MsTimeout);
+
+/*************************************************************************/ /*!
+@Function       OSRemoveTimer
+@Description    Removes the specified timer. The handle becomes invalid and
+                should no longer be used.
+@Input          hTimer          handle of the timer to be removed
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSRemoveTimer(IMG_HANDLE hTimer);
+
+/*************************************************************************/ /*!
+@Function       OSEnableTimer
+@Description    Enable the specified timer. after enabling, the timer will
+                invoke the associated callback at an interval determined by
+                the configured timeout period until disabled.
+@Input          hTimer          handle of the timer to be enabled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEnableTimer(IMG_HANDLE hTimer);
+
+/*************************************************************************/ /*!
+@Function       OSDisableTimer
+@Description    Disable the specified timer
+@Input          hTimer          handle of the timer to be disabled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDisableTimer(IMG_HANDLE hTimer);
+
+
+/*************************************************************************/ /*!
+ @Function      OSPanic
+ @Description   Take action in response to an unrecoverable driver error
+ @Return        None
+*/ /**************************************************************************/
+void __noreturn OSPanic(void);
+
+/*************************************************************************/ /*!
+@Function       OSCopyToUser
+@Description    Copy data to user-addressable memory from kernel-addressable
+                memory.
+                Note that pvDest may be an invalid address or NULL and the
+                function should return an error in this case.
+                For operating systems that do not have a user/kernel space
+                distinction, this function should be implemented as a stub
+                which simply returns PVRSRV_ERROR_NOT_SUPPORTED.
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination User memory
+@Input          pvSrc            pointer to the source Kernel memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSCopyToUser(void *pvProcess, void __user *pvDest, const void *pvSrc, size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSCopyFromUser
+@Description    Copy data from user-addressable memory to kernel-addressable
+                memory.
+                Note that pvSrc may be an invalid address or NULL and the
+                function should return an error in this case.
+                For operating systems that do not have a user/kernel space
+                distinction, this function should be implemented as a stub
+                which simply returns PVRSRV_ERROR_NOT_SUPPORTED.
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination Kernel memory
+@Input          pvSrc            pointer to the source User memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSCopyFromUser(void *pvProcess, void *pvDest, const void __user *pvSrc, size_t ui32Bytes);
+
+#if defined(__linux__) || defined(INTEGRITY_OS)
+#define OSBridgeCopyFromUser OSCopyFromUser
+#define OSBridgeCopyToUser OSCopyToUser
+#else
+/*************************************************************************/ /*!
+@Function       OSBridgeCopyFromUser
+@Description    Copy data from user-addressable memory into kernel-addressable
+                memory as part of a bridge call operation.
+                For operating systems that do not have a user/kernel space
+                distinction, this function will require whatever implementation
+                is needed to pass data for making the bridge function call.
+                For operating systems which do have a user/kernel space
+                distinction (such as Linux) this function may be defined so
+                as to equate to a call to OSCopyFromUser().
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination Kernel memory
+@Input          pvSrc            pointer to the source User memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSBridgeCopyFromUser (void *pvProcess,
+						void *pvDest,
+						const void *pvSrc,
+						size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSBridgeCopyToUser
+@Description    Copy data to user-addressable memory from kernel-addressable
+                memory as part of a bridge call operation.
+                For operating systems that do not have a user/kernel space
+                distinction, this function will require whatever implementation
+                is needed to pass data for making the bridge function call.
+                For operating systems which do have a user/kernel space
+                distinction (such as Linux) this function may be defined so
+                as to equate to a call to OSCopyToUser().
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination User memory
+@Input          pvSrc            pointer to the source Kernel memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSBridgeCopyToUser (void *pvProcess,
+						void *pvDest,
+						const void *pvSrc,
+						size_t ui32Bytes);
+#endif
+
+/* To be increased if required in future */
+#define PVRSRV_MAX_BRIDGE_IN_SIZE      0x2000    /*!< Size of the memory block used to hold data passed in to a bridge call */
+#define PVRSRV_MAX_BRIDGE_OUT_SIZE     0x1000    /*!< Size of the memory block used to hold data returned from a bridge call */
+
+/*************************************************************************/ /*!
+@Function       OSPlatformBridgeInit
+@Description    Called during device creation to allow the OS port to register
+                other bridge modules and related resources that it requires.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPlatformBridgeInit(void);
+
+/*************************************************************************/ /*!
+@Function       OSPlatformBridgeDeInit
+@Description    Called during device destruction to allow the OS port to
+                deregister its OS specific bridges and clean up other
+                related resources.
+*/ /**************************************************************************/
+void OSPlatformBridgeDeInit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVToNativeError
+@Description    Returns the OS-specific equivalent error number/code for
+                the specified PVRSRV_ERROR value.
+                If there is no equivalent, or the PVRSRV_ERROR value is
+                PVRSRV_OK (no error), 0 is returned.
+@Return         The OS equivalent error code.
+*/ /**************************************************************************/
+int PVRSRVToNativeError(PVRSRV_ERROR e);
+/** See PVRSRVToNativeError(). */
+#define OSPVRSRVToNativeError(e) ( (PVRSRV_OK == e)? 0: PVRSRVToNativeError(e) )
+
+
+#if defined(__linux__) && defined(__KERNEL__)
+
+/* Provide LockDep friendly definitions for Services RW locks */
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include "allocmem.h"
+
+#define OSWRLockCreate(ppsLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(ppsLock) = OSAllocMem(sizeof(struct rw_semaphore)); \
+	if (*(ppsLock)) { init_rwsem(*(ppsLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSWRLockDestroy(psLock) ({OSFreeMem(psLock); PVRSRV_OK;})
+
+#define OSWRLockAcquireRead(psLock) ({down_read(psLock); PVRSRV_OK;})
+#define OSWRLockAcquireReadNested(psLock, subclass) ({down_read_nested((psLock), (subclass)); PVRSRV_OK;})
+#define OSWRLockReleaseRead(psLock) ({up_read(psLock); PVRSRV_OK;})
+#define OSWRLockAcquireWrite(psLock) ({down_write(psLock); PVRSRV_OK;})
+#define OSWRLockReleaseWrite(psLock) ({up_write(psLock); PVRSRV_OK;})
+
+#elif defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+/* User-mode unit tests use these definitions on Linux */
+
+PVRSRV_ERROR OSWRLockCreate(POSWR_LOCK *ppsLock);
+void OSWRLockDestroy(POSWR_LOCK psLock);
+void OSWRLockAcquireRead(POSWR_LOCK psLock);
+#define OSWRLockAcquireReadNested(psLock, subclass) OSWRLockAcquireRead((psLock))
+void OSWRLockReleaseRead(POSWR_LOCK psLock);
+void OSWRLockAcquireWrite(POSWR_LOCK psLock);
+void OSWRLockReleaseWrite(POSWR_LOCK psLock);
+
+#else
+
+#if !defined(DOXYGEN)
+#define OSFUNC_NOT_IMPLEMENTED 0
+#define OSFUNC_NOT_IMPLEMENTED_ASSERT() PVR_ASSERT(OSFUNC_NOT_IMPLEMENTED)
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSWRLockCreate
+@Description    Create a writer/reader lock.
+                This type of lock allows multiple concurrent readers but
+                only a single writer, allowing for optimized performance.
+@Output         ppsLock     A handle to the created WR lock.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSWRLockCreate(POSWR_LOCK *ppsLock)
+{
+	PVR_UNREFERENCED_PARAMETER(ppsLock);
+
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockDestroy
+@Description    Destroys a writer/reader lock.
+@Input          psLock     The handle of the WR lock to be destroyed.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockDestroy(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireRead
+@Description    Acquire a writer/reader read lock.
+                If the write lock is already acquired, the caller will
+                block until it is released.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           reading.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireRead(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireReadNested
+@Description    Acquire a nested writer/reader read lock.
+                If the write lock is already acquired, the caller will
+                block until it is released.
+                For operating systems other than Linux, this equates to an
+                OSWRLockAcquireRead() call. On Linux, this function wraps a call
+                to down_read_nested(). This recognises the scenario where
+                there may be multiple subclasses within a particular class
+                of lock. In such cases, the order in which the locks belonging
+                these various subclasses are acquired is important and must be
+                validated.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           reading.
+@Input          iSubclass  The subclass of the lock.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireReadNested(POSWR_LOCK psLock, IMG_INT iSubclass)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	PVR_UNREFERENCED_PARAMETER(iSubclass);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockReleaseRead
+@Description    Release a writer/reader read lock.
+@Input          psLock     The handle of the WR lock whose read lock is to
+                           be released.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockReleaseRead(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireWrite
+@Description    Acquire a writer/reader write lock.
+                If the write lock or any read lock are already acquired,
+                the caller will block until all are released.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           writing.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireWrite(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockReleaseWrite
+@Description    Release a writer/reader write lock.
+@Input          psLock     The handle of the WR lock whose write lock is to
+                           be released.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void OSWRLockReleaseWrite(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSDivide64r64
+@Description    Divide a 64-bit value by a 32-bit value. Return the 64-bit
+                quotient.
+                The remainder is also returned in 'pui32Remainder'.
+@Input          ui64Divident        The number to be divided.
+@Input          ui32Divisor         The 32-bit value 'ui64Divident' is to
+                                    be divided by.
+@Output         pui32Remainder      The remainder of the division.
+@Return         The 64-bit quotient (result of the division).
+*/ /**************************************************************************/
+IMG_UINT64 OSDivide64r64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder);
+
+/*************************************************************************/ /*!
+@Function       OSDivide64
+@Description    Divide a 64-bit value by a 32-bit value. Return a 32-bit
+                quotient.
+                The remainder is also returned in 'pui32Remainder'.
+                This function allows for a more optimal implementation
+                of a 64-bit division when the result is known to be
+                representable in 32-bits.
+@Input          ui64Divident        The number to be divided.
+@Input          ui32Divisor         The 32-bit value 'ui64Divident' is to
+                                    be divided by.
+@Output         pui32Remainder      The remainder of the division.
+@Return         The 32-bit quotient (result of the division).
+*/ /**************************************************************************/
+IMG_UINT32 OSDivide64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder);
+
+/*************************************************************************/ /*!
+@Function       OSDumpStack
+@Description    Dump the current task information and its stack trace.
+@Return         None
+*/ /**************************************************************************/
+void OSDumpStack(void);
+
+/*************************************************************************/ /*!
+@Function       OSUserModeAccessToPerfCountersEn
+@Description    Permit User-mode access to CPU performance counter
+                registers.
+                This function is called during device initialisation.
+                Certain CPU architectures may need to explicitly permit
+                User mode access to performance counters - if this is
+                required, the necessary code should be implemented inside
+                this function.
+@Return         None.
+*/ /**************************************************************************/
+void OSUserModeAccessToPerfCountersEn(void);
+
+/*************************************************************************/ /*!
+@Function       OSDebugSignalPID
+@Description    Sends a SIGTRAP signal to a specific PID in user mode for
+                debugging purposes. The user mode process can register a handler
+                against this signal.
+                This is necessary to support the Rogue debugger. If the Rogue
+                debugger is not used then this function may be implemented as
+                a stub.
+@Input          ui32PID    The PID for the signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDebugSignalPID(IMG_UINT32 ui32PID);
+
+#if defined(__linux__) && defined(__KERNEL__) && !defined(DOXYGEN)
+#define OSWarnOn(a) WARN_ON(a)
+#else
+/*************************************************************************/ /*!
+@Function       OSWarnOn
+@Description    This API allows the driver to emit a special token and stack
+                dump to the server log when an issue is detected that needs the
+                OS to be notified. The token or call may be used to trigger
+                log collection by the OS environment.
+                PVR_DPF log messages will have been emitted prior to this call.
+@Input          a    Expression to evaluate, if true trigger Warn signal
+@Return         None
+*/ /**************************************************************************/
+#define OSWarnOn(a) do { if ((a)) { OSDumpStack(); } } while (0)
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSIsKernelThread
+@Description    This API determines if the current running thread is a kernel
+                thread (i.e. one not associated with any userland process,
+                typically an MISR handler.)
+@Return         IMG_TRUE if it is a kernel thread, otherwise IMG_FALSE.
+*/ /**************************************************************************/
+IMG_BOOL OSIsKernelThread(void);
+
+/*************************************************************************/ /*!
+@Function       OSThreadDumpInfo
+@Description    Traverse the thread list and call each of the stored
+                callbacks to dump the info in debug_dump.
+@Input          pfnDumpDebugPrintf  The 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+*/ /**************************************************************************/
+void OSThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSDumpVersionInfo
+@Description    Store OS version information in debug dump.
+@Input          pfnDumpDebugPrintf  The 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+*/ /**************************************************************************/
+void OSDumpVersionInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                       void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSIsWriteCombineUnalignedSafe
+@Description    Determine if unaligned accesses to write-combine memory are
+                safe to perform, i.e. whether we are safe from a CPU fault
+                occurring. This test is specifically aimed at ARM64 platforms
+                which cannot provide this guarantee if the memory is 'device'
+                memory rather than 'normal' under the ARM memory architecture.
+@Return         IMG_TRUE if safe, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void);
+
+/*************************************************************************/ /*!
+@Function       OSDebugLevel
+@Description    Returns current value of the debug level.
+@Return         Debug level.
+*/ /**************************************************************************/
+IMG_UINT32 OSDebugLevel(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVSetDebugLevel
+@Description    Sets the current value of the debug level to ui32DebugLevel.
+@Input          ui32DebugLevel New debug level value.
+*/ /**************************************************************************/
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVIsDebugLevel
+@Description    Tests if a given debug level is enabled.
+@Input          ui32DebugLevel IMG_TRUE if debug level is enabled
+                and IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel);
+
+#if defined(SUPPORT_DMA_TRANSFER)
+
+typedef void (*PFN_SERVER_CLEANUP)(void *pvData, IMG_BOOL bAdvanceTimeline);
+
+#define DMA_COMPLETION_TIMEOUT_MS 60000
+#define DMA_ERROR_SYNC_RETRIES 100
+
+PVRSRV_ERROR OSDmaPrepareTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *psChan,
+							   IMG_DMA_ADDR* psDmaAddr, IMG_UINT64* puiAddress,
+							   IMG_UINT64 uiSize, IMG_BOOL bMemToDev,
+							   IMG_HANDLE pvOSData,
+							   IMG_HANDLE pvServerCleanupParam,PFN_SERVER_CLEANUP pfnServerCleanup, IMG_BOOL bFirst);
+
+PVRSRV_ERROR OSDmaPrepareTransferSparse(PVRSRV_DEVICE_NODE *psDevNode, IMG_HANDLE pvChan,
+										IMG_DMA_ADDR* psDmaAddr, IMG_BOOL *pbValid,
+										IMG_UINT64* puiAddress, IMG_UINT64 uiSize,
+										IMG_UINT32 uiOffsetInPage,
+										IMG_UINT32 ui32SizeInPages,
+										IMG_BOOL bMemToDev,
+										IMG_HANDLE pvOSData,
+										IMG_HANDLE pvServerCleanupParam, PFN_SERVER_CLEANUP pfnServerCleanup,
+										IMG_BOOL bFirst);
+
+PVRSRV_ERROR OSDmaAllocData(PVRSRV_DEVICE_NODE *psDevNode,IMG_UINT32 uiNumDMA, void **pvAllocedData);
+PVRSRV_ERROR OSDmaSubmitTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *pvOSData, void *psChan, IMG_BOOL bSynchronous);
+void OSDmaForceCleanup(PVRSRV_DEVICE_NODE *psDevNode, void *pvChan,
+					   void *pvOSData, IMG_HANDLE pvServerCleanupParam,
+					   PFN_SERVER_CLEANUP pfnServerCleanup);
+#endif
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR);
+
+void
+OSFreeSecBuf(PMR *psPMR);
+#endif
+#endif /* OSFUNC_H */
+
+/******************************************************************************
+ End of file (osfunc.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/ospvr_gputrace.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/ospvr_gputrace.h
new file mode 100644
index 000000000000..6bafc4d6042f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/ospvr_gputrace.h
@@ -0,0 +1,185 @@
+/*************************************************************************/ /*!
+@File           ospvr_gputrace.h
+@Title          PVR GPU Trace module common environment interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_GPUTRACE_H_
+#define PVR_GPUTRACE_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_hwperf.h"
+#include "rgxdevice.h"
+#include "device.h"
+
+#if defined(__linux__) && defined(CONFIG_EVENT_TRACING) && \
+	(defined(PVRSRV_TRACE_ROGUE_EVENTS) || \
+	 defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD))
+
+/* Early initialisation of GPU Trace events logic.
+ * This function is called on *driver* initialisation. */
+PVRSRV_ERROR PVRGpuTraceSupportInit(void);
+
+/* GPU Trace resources final cleanup.
+ * This function is called on driver de-initialisation. */
+void PVRGpuTraceSupportDeInit(void);
+
+/* Initialisation for AppHints callbacks.
+ * This function is called during the late stage of driver initialisation but
+ * before the device initialisation but after the debugfs sub-system has been
+ * initialised. */
+void PVRGpuTraceInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* Per-device initialisation of the GPU Trace resources */
+PVRSRV_ERROR PVRGpuTraceInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* TL Initialisation of FTrace */
+PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/* Per-device cleanup for the GPU Trace resources. */
+void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* Enables the gpu trace sub-system for a given device. */
+PVRSRV_ERROR PVRGpuTraceSetEnabled(
+		PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_BOOL bNewValue);
+
+/* Performs some initialisation steps if the feature was enabled
+ * on driver startup. */
+void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* FTrace events callbacks interface */
+
+void PVRGpuTraceEnableUfoCallback(void);
+void PVRGpuTraceDisableUfoCallback(void);
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void);
+void PVRGpuTraceDisableFirmwareActivityCallback(void);
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+PVRSRV_ERROR
+PVRSRVGpuTraceWorkPeriodEventStatsRegister(IMG_HANDLE*
+		phGpuWorkPeriodEventStats);
+void PVRSRVGpuTraceWorkPeriodEventStatsUnregister(
+		IMG_HANDLE hGpuWorkPeriodEventStats);
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#else /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) */
+
+static inline PVRSRV_ERROR PVRGpuTraceSupportInit(void) {
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceSupportDeInit(void) {}
+
+static inline void PVRGpuTraceInitAppHintCallbacks(
+		const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceInitDevice(
+		PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceSetEnabled(
+		PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_BOOL bNewValue)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(bNewValue);
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline void PVRGpuTraceEnableUfoCallback(void) {}
+static inline void PVRGpuTraceDisableUfoCallback(void) {}
+
+static inline void PVRGpuTraceEnableFirmwareActivityCallback(void) {}
+static inline void PVRGpuTraceDisableFirmwareActivityCallback(void) {}
+
+#endif /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) */
+
+#if defined(__linux__) && defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExternalJobRef,
+		IMG_UINT32 ui32InternalJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType);
+
+#else
+
+static inline void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExternalJobRef,
+		IMG_UINT32 ui32InternalJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ui32FirmwareCtx);
+	PVR_UNREFERENCED_PARAMETER(ui32ExternalJobRef);
+	PVR_UNREFERENCED_PARAMETER(ui32InternalJobRef);
+	PVR_UNREFERENCED_PARAMETER(eKickType);
+}
+
+#endif /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#endif /* PVR_GPUTRACE_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/ossecure_export.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/ossecure_export.h
new file mode 100644
index 000000000000..020f5f57f45e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/ossecure_export.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       OSSecureExport
+@Description    Assigns an OS-specific 'token' to allow a resource
+                to be securely referenced by another process.
+                A process wishing to reference the exported resource
+                should call OSSecureImport(), passing its OS-specific
+                reference to the same resource.
+                For the export/import to be deemed 'secure', the
+                implementation should ensure that the OS-specific
+                reference can only be meaningfully used by a process
+                which is permitted to do so.
+@Input          pszName        name of the "class" of new secure file
+@Input          pfnReleaseFunc  pointer to the function to be called
+                               while closing secure file
+@Input          pvData         pointer to the actual resource that
+                               is being exported
+@Output         phSecure       the returned secure token
+@Return         PVRSRV_OK      on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSSecureExport(const IMG_CHAR *pszName,
+                            PVRSRV_ERROR (*pfnReleaseFunc)(void *),
+                            void *pvData,
+                            IMG_SECURE_TYPE *phSecure);
+
+/*************************************************************************/ /*!
+@Function       OSSecureImport
+@Description    Imports an OS-specific 'token' that allows a resource
+                allocated by another process to be securely referenced by
+                the current process.
+@Input          hSecure             the secure token for the resource to
+                                    be imported
+@Output         ppvData             pointer to the actual resource that
+                                    is being referenced
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSSecureImport(IMG_SECURE_TYPE hSecure, void **ppvData);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_km.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_km.h
new file mode 100644
index 000000000000..58713cdfa17d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_km.h
@@ -0,0 +1,1159 @@
+/*************************************************************************/ /*!
+@File
+@Title          pdump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for pdump functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PDUMP_KM_H
+#define PDUMP_KM_H
+
+#if defined(PDUMP)
+ #if defined(__linux__)
+  #include <linux/version.h>
+  #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+   #include <linux/stdarg.h>
+  #else
+   #include <stdarg.h>
+  #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+ #else
+  #include <stdarg.h>
+ #endif /* __linux__ */
+#endif /* PDUMP */
+
+/* services/srvkm/include/ */
+#include "device.h"
+
+/* include/ */
+#include "pvrsrv_error.h"
+
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#include "connection_server.h"
+/* Pull in pdump flags from services include */
+#include "pdump.h"
+#include "pdumpdefs.h"
+
+/* Define this to enable the PDUMP_HERE trace in the server */
+#undef PDUMP_TRACE
+
+#if defined(PDUMP_TRACE)
+#define PDUMP_HERE_VAR  __maybe_unused IMG_UINT32 here = 0;
+#define PDUMP_HERE(a)	{ here = (a); if (ui32Flags & PDUMP_FLAGS_DEBUG) PVR_DPF((PVR_DBG_WARNING, "HERE %d", (a))); }
+#define PDUMP_HEREA(a)	{ here = (a); PVR_DPF((PVR_DBG_WARNING, "HERE ALWAYS %d", (a))); }
+#else
+#define PDUMP_HERE_VAR  __maybe_unused IMG_UINT32 here = 0;
+#define PDUMP_HERE(a)	here = (a);
+#define PDUMP_HEREA(a)	here = (a);
+#endif
+
+#define PDUMP_PD_UNIQUETAG	(IMG_HANDLE)0
+#define PDUMP_PT_UNIQUETAG	(IMG_HANDLE)0
+
+/* Defines for CMD:SetParity's state word */
+#define PDUMP_SET_PARITY_STATE_WORD_VERSION_SHIFT		(0U)
+#define PDUMP_SET_PARITY_STATE_WORD_VERSION_MASK		(0x00000007U)
+
+#define PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_SHIFT	(3U)
+#define PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_MASK	(0x000001F8U)
+
+#define PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_SHIFT		(9U)
+#define PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_MASK		(0x00000200U)
+
+/* Invalid value for PDump block number */
+#define PDUMP_BLOCKNUM_INVALID      IMG_UINT32_MAX
+
+typedef struct _PDUMP_CONNECTION_DATA_ PDUMP_CONNECTION_DATA;
+
+/* PDump transition events */
+typedef enum _PDUMP_TRANSITION_EVENT_
+{
+	PDUMP_TRANSITION_EVENT_NONE,              /* No event */
+	PDUMP_TRANSITION_EVENT_BLOCK_FINISHED,    /* Block mode event, current PDump-block has finished */
+	PDUMP_TRANSITION_EVENT_BLOCK_STARTED,     /* Block mode event, new PDump-block has started */
+	PDUMP_TRANSITION_EVENT_RANGE_ENTERED,     /* Transition into capture range */
+	PDUMP_TRANSITION_EVENT_RANGE_APPEND,      /* Append to an already entered capture range */
+	PDUMP_TRANSITION_EVENT_RANGE_EXITED,      /* Transition out of capture range */
+} PDUMP_TRANSITION_EVENT;
+
+typedef PVRSRV_ERROR (*PFN_PDUMP_TRANSITION)(void *pvData, void *pvDevice, PDUMP_TRANSITION_EVENT eEvent, IMG_UINT32 ui32PDumpFlags);
+typedef void (*PFN_PDUMP_SYNCBLOCKS)(PVRSRV_DEVICE_NODE *psDevNode, void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+
+typedef PVRSRV_ERROR (*PFN_PDUMP_TRANSITION_FENCE_SYNC)(void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+
+#ifdef PDUMP
+
+/*! Macro used to record a panic in the PDump script stream */
+#define PDUMP_PANIC(_dev, _id, _msg) do \
+		{ PVRSRV_ERROR _eE;\
+			_eE = PDumpPanic((_dev), ((RGX_PDUMP_PANIC_ ## _id) & 0xFFFF), _msg, __func__, __LINE__);	\
+			PVR_LOG_IF_ERROR(_eE, "PDumpPanic");\
+		MSC_SUPPRESS_4127\
+		} while (0)
+
+/*! Macro used to record a driver error in the PDump script stream to invalidate the capture */
+#define PDUMP_ERROR(_dev, _err, _msg) \
+	(void)PDumpCaptureError((_dev), _err, _msg, __func__, __LINE__)
+
+#define SZ_MSG_SIZE_MAX			PVRSRV_PDUMP_MAX_COMMENT_SIZE
+#define SZ_SCRIPT_SIZE_MAX		PVRSRV_PDUMP_MAX_COMMENT_SIZE
+#define SZ_FILENAME_SIZE_MAX	(PVRSRV_PDUMP_MAX_FILENAME_SIZE+sizeof(PDUMP_PARAM_N_FILE_NAME))
+
+#define PDUMP_GET_SCRIPT_STRING()																			\
+	IMG_HANDLE hScript;																						\
+	void *pvScriptAlloc;																					\
+	IMG_UINT32 ui32MaxLen = SZ_SCRIPT_SIZE_MAX-1;															\
+	pvScriptAlloc = OSAllocMem( SZ_SCRIPT_SIZE_MAX );														\
+	if (!pvScriptAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_STRING() failed to allocate memory for script buffer"));	\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+																											\
+	hScript = (IMG_HANDLE) pvScriptAlloc;
+
+#define PDUMP_GET_MSG_STRING()																				\
+	IMG_CHAR *pszMsg;																						\
+	void *pvMsgAlloc;																						\
+	IMG_UINT32 ui32MaxLen = SZ_MSG_SIZE_MAX-1;																\
+	pvMsgAlloc = OSAllocMem( SZ_MSG_SIZE_MAX );																\
+	if (!pvMsgAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_MSG_STRING() failed to allocate memory for message buffer"));	\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+	pszMsg = (IMG_CHAR *)pvMsgAlloc;
+
+#define PDUMP_GET_SCRIPT_AND_FILE_STRING()																	\
+	IMG_HANDLE hScript;																						\
+	IMG_CHAR *pszFileName;																					\
+	IMG_UINT32 ui32MaxLenScript = SZ_SCRIPT_SIZE_MAX-1;														\
+	void *pvScriptAlloc;																					\
+	void *pvFileAlloc;																						\
+	pvScriptAlloc = OSAllocMem( SZ_SCRIPT_SIZE_MAX );														\
+	if (!pvScriptAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_AND_FILE_STRING() failed to allocate memory for script buffer"));		\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+																											\
+	hScript = (IMG_HANDLE) pvScriptAlloc;																	\
+	pvFileAlloc = OSAllocMem( SZ_FILENAME_SIZE_MAX );														\
+	if (!pvFileAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_AND_FILE_STRING() failed to allocate memory for filename buffer"));	\
+		OSFreeMem(pvScriptAlloc);																			\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+	pszFileName = (IMG_CHAR *)pvFileAlloc;
+
+#define PDUMP_RELEASE_SCRIPT_STRING()																		\
+	if (pvScriptAlloc)																						\
+	{																										\
+		OSFreeMem(pvScriptAlloc);																			\
+		pvScriptAlloc = NULL;																				\
+	}
+
+#define PDUMP_RELEASE_MSG_STRING()																			\
+	if (pvMsgAlloc)																							\
+	{																										\
+		OSFreeMem(pvMsgAlloc);																				\
+		pvMsgAlloc = NULL;																					\
+	}
+
+#define PDUMP_RELEASE_FILE_STRING()																			\
+	if (pvFileAlloc)																						\
+	{																										\
+		OSFreeMem(pvFileAlloc);																				\
+		pvFileAlloc = NULL;																					\
+	}
+
+#define PDUMP_RELEASE_SCRIPT_AND_FILE_STRING()																\
+	if (pvScriptAlloc)																						\
+	{																										\
+		OSFreeMem(pvScriptAlloc);																			\
+		pvScriptAlloc = NULL;																				\
+	}																										\
+	if (pvFileAlloc)																						\
+	{																										\
+		OSFreeMem(pvFileAlloc);																				\
+		pvFileAlloc = NULL;																					\
+	}
+
+
+/* Shared across pdump_x files */
+PVRSRV_ERROR PDumpInitCommon(void);
+void PDumpDeInitCommon(void);
+PVRSRV_ERROR PDumpValidateUMFlags(PDUMP_FLAGS_T uiFlags);
+PVRSRV_ERROR PDumpReady(void);
+void PDumpGetParameterZeroPageInfo(PDUMP_FILEOFFSET_T *puiZeroPageOffset,
+                                   size_t *puiZeroPageSize,
+                                   const IMG_CHAR **ppszZeroPageFilename);
+
+void PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode);
+void PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32Frame);
+PVRSRV_ERROR PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 *pui32Frame);
+PVRSRV_ERROR PDumpCommentKM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT32 ui32CommentSize,
+                            IMG_CHAR *pszComment,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_UINT32 ui32Mode,
+                                            IMG_UINT32 ui32Start,
+                                            IMG_UINT32 ui32End,
+                                            IMG_UINT32 ui32Interval,
+                                            IMG_UINT32 ui32MaxParamFileSize,
+                                            IMG_UINT32 ui32AutoTermTimeout);
+
+
+PVRSRV_ERROR PDumpReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_CHAR *pszPDumpRegName,
+                        IMG_UINT32 ui32RegAddr,
+                        IMG_UINT32 ui32RegValue,
+                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_CHAR *pszPDumpRegName,
+                        IMG_UINT32 ui32RegAddr,
+                        IMG_UINT64 ui64RegValue,
+                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32RegDst,
+                                  IMG_UINT32 ui32RegSrc,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpPhysHandleToInternalVar64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_CHAR *pszInternalVar,
+                                            IMG_HANDLE hPdumpPages,
+                                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToInternalVar64(IMG_CHAR *pszInternalVar,
+                                          PMR *psPMR,
+                                          IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                          IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpInternalVarToMemLabel(PMR *psPMR,
+                                        IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpWriteVarORValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const IMG_CHAR *pszInternalVariable,
+                                    const IMG_UINT64 ui64Value,
+                                    const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarANDValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarSHRValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarORVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const IMG_CHAR *pszInternalVar,
+                                  const IMG_CHAR *pszInternalVar2,
+                                  const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarANDVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const IMG_CHAR *pszInternalVar,
+                                   const IMG_CHAR *pszInternalVar2,
+                                   const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpInternalVarToReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpInternalVarToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToMem32(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToMem64(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToMem32(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToMem64(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR *pszPDumpRegName,
+                                        IMG_UINT32 ui32Reg,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpSAW(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR      *pszDevSpaceName,
+                      IMG_UINT32    ui32HPOffsetBytes,
+                      IMG_UINT32    ui32NumSaveBytes,
+                      IMG_CHAR      *pszOutfileName,
+                      IMG_UINT32    ui32OutfileOffsetByte,
+                      PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR PDumpRegPolKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           IMG_CHAR            *pszPDumpRegName,
+                           IMG_UINT32          ui32RegAddr,
+                           IMG_UINT32          ui32RegValue,
+                           IMG_UINT32          ui32Mask,
+                           IMG_UINT32          ui32Flags,
+                           PDUMP_POLL_OPERATOR eOperator);
+
+
+/**************************************************************************/ /*!
+@Function       PDumpImageDescriptor
+@Description    PDumps image data out as an IMGBv2 data section
+@Input          psDeviceNode         Pointer to device node.
+@Input          ui32MMUContextID     PDUMP MMU context ID.
+@Input          pszSABFileName       Pointer to string containing file name of
+                                     Image being SABed
+@Input          sData                GPU virtual address of this surface.
+@Input          ui32DataSize         Image data size
+@Input          ui32LogicalWidth     Image logical width
+@Input          ui32LogicalHeight    Image logical height
+@Input          ui32PhysicalWidth    Image physical width
+@Input          ui32PhysicalHeight   Image physical height
+@Input          ePixFmt              Image pixel format
+@Input          eFBCompression       FB compression mode
+@Input          paui32FBCClearColour FB clear colour (Only applicable to FBC surfaces)
+@Input          eFBCSwizzle          FBC channel swizzle (Only applicable to FBC surfaces)
+@Input          sHeader              GPU virtual address of the headers of this
+                                     surface (Only applicable to FBC surfaces)
+@Input          ui32HeaderSize       Header size (Only applicable to FBC surfaces)
+@Input          ui32PDumpFlags       PDUMP flags
+@Return         PVRSRV_ERROR:        PVRSRV_OK on success. Otherwise, a PVRSRV_
+                                     error code
+*/ /***************************************************************************/
+PVRSRV_ERROR PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32MMUContextID,
+                                  IMG_CHAR *pszSABFileName,
+                                  IMG_DEV_VIRTADDR sData,
+                                  IMG_UINT32 ui32DataSize,
+                                  IMG_UINT32 ui32LogicalWidth,
+                                  IMG_UINT32 ui32LogicalHeight,
+                                  IMG_UINT32 ui32PhysicalWidth,
+                                  IMG_UINT32 ui32PhysicalHeight,
+                                  PDUMP_PIXEL_FORMAT ePixFmt,
+                                  IMG_MEMLAYOUT eMemLayout,
+                                  IMG_FB_COMPRESSION eFBCompression,
+                                  const IMG_UINT32 *paui32FBCClearColour,
+                                  PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                                  IMG_DEV_VIRTADDR sHeader,
+                                  IMG_UINT32 ui32HeaderSize,
+                                  IMG_UINT32 ui32PDumpFlags);
+
+/**************************************************************************/ /*!
+@Function       PDumpDataDescriptor
+@Description    PDumps non-image data out as an IMGCv1 data section
+@Input          psDeviceNode         Pointer to device node.
+@Input          ui32MMUContextID     PDUMP MMU context ID.
+@Input          pszSABFileName       Pointer to string containing file name of
+                                     Data being SABed
+@Input          sData                GPU virtual address of this data.
+@Input          ui32DataSize         Data size
+@Input          ui32HeaderType       Header type
+@Input          ui32ElementType      Data element type
+@Input          ui32ElementCount     Number of data elements
+@Input          ui32PDumpFlags       PDUMP flags
+@Return         PVRSRV_ERROR:        PVRSRV_OK on success. Otherwise, a PVRSRV_
+                                     error code
+*/ /***************************************************************************/
+PVRSRV_ERROR PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32MMUContextID,
+                                 IMG_CHAR *pszSABFileName,
+                                 IMG_DEV_VIRTADDR sData,
+                                 IMG_UINT32 ui32DataSize,
+                                 IMG_UINT32 ui32HeaderType,
+                                 IMG_UINT32 ui32ElementType,
+                                 IMG_UINT32 ui32ElementCount,
+                                 IMG_UINT32 ui32PDumpFlags);
+
+
+PVRSRV_ERROR PDumpReadRegKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            IMG_CHAR *pszFileName,
+                            IMG_UINT32 ui32FileOffset,
+                            IMG_UINT32 ui32Address,
+                            IMG_UINT32 ui32Size,
+                            IMG_UINT32 ui32PDumpFlags);
+
+__printf(3, 4)
+PVRSRV_ERROR PDumpCommentWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_UINT32 ui32Flags,
+                                   IMG_CHAR* pszFormat,
+                                   ...);
+
+PVRSRV_ERROR PDumpCommentWithFlagsVA(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32Flags,
+                                     const IMG_CHAR * pszFormat,
+                                     va_list args);
+
+PVRSRV_ERROR PDumpPanic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT32      ui32PanicNo,
+                        IMG_CHAR*       pszPanicMsg,
+                        const IMG_CHAR* pszPPFunc,
+                        IMG_UINT32      ui32PPline);
+
+PVRSRV_ERROR PDumpCaptureError(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               PVRSRV_ERROR    ui32ErrorNo,
+                               IMG_CHAR*       pszErrorMsg,
+                               const IMG_CHAR* pszPPFunc,
+                               IMG_UINT32      ui32PPline);
+
+PVRSRV_ERROR PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame);
+
+PVRSRV_ERROR PDumpGetStateKM(IMG_UINT64 *ui64State);
+
+PVRSRV_ERROR PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PDumpRegRead32ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_CHAR *pszPDumpRegName,
+                                         IMG_UINT32 ui32RegOffset,
+                                         IMG_CHAR *pszInternalVar,
+                                         IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            const IMG_UINT32 dwRegOffset,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            const IMG_UINT32 dwRegOffset,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead64ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_CHAR *pszPDumpRegName,
+                                         IMG_CHAR *pszInternalVar,
+                                         const IMG_UINT32 dwRegOffset,
+                                         IMG_UINT32	ui32Flags);
+
+PVRSRV_ERROR PDumpIDLWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Clocks,
+                               IMG_UINT32 ui32Flags);
+PVRSRV_ERROR PDumpIDL(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32Clocks);
+
+PVRSRV_ERROR PDumpRegBasedCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_CHAR   *pszPDumpRegName,
+                              IMG_UINT32 ui32RegOffset,
+                              IMG_UINT32 ui32WPosVal,
+                              IMG_UINT32 ui32PacketSize,
+                              IMG_UINT32 ui32BufferSize,
+                              IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpTRG(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR *pszMemSpace,
+                      IMG_UINT32 ui32MMUCtxID,
+                      IMG_UINT32 ui32RegionID,
+                      IMG_BOOL bEnable,
+                      IMG_UINT64 ui64VAddr,
+                      IMG_UINT64 ui64LenBytes,
+                      IMG_UINT32 ui32XStride,
+                      IMG_UINT32 ui32Flags);
+
+void PDumpLock(void);
+void PDumpUnlock(void);
+
+PVRSRV_ERROR PDumpRegCondStr(IMG_CHAR            **ppszPDumpCond,
+                             IMG_CHAR            *pszPDumpRegName,
+                             IMG_UINT32          ui32RegAddr,
+                             IMG_UINT32          ui32RegValue,
+                             IMG_UINT32          ui32Mask,
+                             IMG_UINT32          ui32Flags,
+                             PDUMP_POLL_OPERATOR eOperator);
+
+PVRSRV_ERROR PDumpInternalValCondStr(IMG_CHAR            **ppszPDumpCond,
+                                     IMG_CHAR            *pszInternalVar,
+                                     IMG_UINT32          ui32RegValue,
+                                     IMG_UINT32          ui32Mask,
+                                     IMG_UINT32          ui32Flags,
+                                     PDUMP_POLL_OPERATOR eOperator);
+
+PVRSRV_ERROR PDumpIfKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpElseKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpFiKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpStartDoLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpEndDoWhileLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_CHAR *pszPDumpWhileCond,
+                                   IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpCOMCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32PDumpFlags,
+                             const IMG_CHAR *pszPDump);
+
+void PDumpPowerTransitionStart(PVRSRV_DEVICE_NODE *psDeviceNode);
+void PDumpPowerTransitionEnd(PVRSRV_DEVICE_NODE *psDeviceNode);
+IMG_BOOL PDumpCheckFlagsWrite(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_UINT32 ui32Flags);
+
+/*!
+ * @name	PDumpWriteParameter
+ * @brief	General function for writing to PDump stream. Used
+ *          mainly for memory dumps to parameter stream.
+ *          Usually more convenient to use PDumpWriteScript below
+ *          for the script stream.
+ * @param	psDeviceNode - device PDump pertains to
+ * @param	psui8Data - data to write
+ * @param	ui32Size - size of write
+ * @param	ui32Flags - PDump flags
+ * @param   pui32FileOffset - on return contains the file offset to
+ *                            the start of the parameter data
+ * @param   aszFilenameStr - pointer to at least a 20 char buffer to
+ *                           return the parameter filename
+ * @return	error
+ */
+PVRSRV_ERROR PDumpWriteParameter(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT8 *psui8Data, IMG_UINT32 ui32Size,
+                                 IMG_UINT32 ui32Flags, IMG_UINT32* pui32FileOffset,
+                                 IMG_CHAR* aszFilenameStr);
+
+/*!
+ * @name	PDumpWriteScript
+ * @brief	Write an PDumpOS created string to the "script" output stream
+ * @param	psDeviceNode - device PDump pertains to
+ * @param	hString - PDump OS layer handle of string buffer to write
+ * @param	ui32Flags - PDump flags
+ * @return	IMG_TRUE on success.
+ */
+IMG_BOOL PDumpWriteScript(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          IMG_HANDLE hString, IMG_UINT32 ui32Flags);
+
+/**************************************************************************/ /*!
+@Function       PDumpSNPrintf
+@Description    Printf to OS-specific PDump state buffer. This function is
+                only called if PDUMP is defined.
+@Input          hBuf               handle of buffer to write into
+@Input          ui32ScriptSizeMax  maximum size of data to write (chars)
+@Input          pszFormat          format string
+@Return         None
+*/ /**************************************************************************/
+__printf(3, 4)
+PVRSRV_ERROR PDumpSNPrintf(IMG_HANDLE hBuf, IMG_UINT32 ui32ScriptSizeMax, IMG_CHAR* pszFormat, ...);
+
+
+/*
+	PDumpWriteShiftedMaskedValue():
+
+	loads the "reference" address into an internal PDump register,
+	optionally shifts it right,
+	optionally shifts it left,
+	optionally masks it
+	then finally writes the computed value to the given destination address
+
+	i.e. it emits pdump language equivalent to this expression:
+
+	 dest = ((&ref) >> SHRamount << SHLamount) & MASK
+*/
+PVRSRV_ERROR
+PDumpWriteShiftedMaskedValue(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const IMG_CHAR *pszDestRegspaceName,
+                             const IMG_CHAR *pszDestSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiDestOffset,
+                             const IMG_CHAR *pszRefRegspaceName,
+                             const IMG_CHAR *pszRefSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiRefOffset,
+                             IMG_UINT32 uiSHRAmount,
+                             IMG_UINT32 uiSHLAmount,
+                             IMG_UINT32 uiMask,
+                             IMG_DEVMEM_SIZE_T uiWordSize,
+                             IMG_UINT32 uiPDumpFlags);
+
+/*
+	PDumpWriteSymbAddress():
+	writes the address of the "reference" to the offset given
+*/
+PVRSRV_ERROR
+PDumpWriteSymbAddress(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      const IMG_CHAR *pszDestSpaceName,
+                      IMG_DEVMEM_OFFSET_T uiDestOffset,
+                      const IMG_CHAR *pszRefSymbolicName,
+                      IMG_DEVMEM_OFFSET_T uiRefOffset,
+                      const IMG_CHAR *pszPDumpDevName,
+                      IMG_UINT32 ui32WordSize,
+                      IMG_UINT32 ui32AlignShift,
+                      IMG_UINT32 ui32Shift,
+                      IMG_UINT32 uiPDumpFlags);
+
+/* Register the connection with the PDump subsystem */
+PVRSRV_ERROR
+PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        void *hSyncPrivData,
+                        PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                        PDUMP_CONNECTION_DATA **ppsPDumpConnectionData);
+
+/* Unregister the connection with the PDump subsystem */
+void
+PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          PDUMP_CONNECTION_DATA *psPDumpConnectionData);
+
+/* Register for notification of PDump Transition into/out of capture range */
+PVRSRV_ERROR
+PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                PFN_PDUMP_TRANSITION pfnCallback,
+                                void *hPrivData,
+                                void *pvDevice,
+                                void **ppvHandle);
+
+/* Unregister notification of PDump Transition */
+void
+PDumpUnregisterTransitionCallback(void *pvHandle);
+
+PVRSRV_ERROR
+PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+                                         PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback,
+                                         void **ppvHandle);
+
+void
+PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle);
+
+/* Notify PDump of a Transition into/out of capture range */
+PVRSRV_ERROR
+PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                PDUMP_TRANSITION_EVENT eEvent,
+                IMG_UINT32 ui32PDumpFlags);
+
+/* Check if writing to a PDump file is permitted for the given device */
+IMG_BOOL PDumpIsDevicePermitted(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_LOCK(_ui32PDumpFlags) do \
+	{ if ((_ui32PDumpFlags & PDUMP_FLAGS_PDUMP_LOCK_HELD) == 0)\
+		{\
+			PDumpLock();\
+		}\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_UNLOCK(_ui32PDumpFlags) do \
+	{ if ((_ui32PDumpFlags & PDUMP_FLAGS_PDUMP_LOCK_HELD) == 0)\
+		{\
+			PDumpUnlock();\
+		}\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#define PDUMPINIT				PDumpInitCommon
+#define PDUMPDEINIT				PDumpDeInitCommon
+#define PDUMPREG32				PDumpReg32
+#define PDUMPREG64				PDumpReg64
+#define PDUMPREGREAD32			PDumpRegRead32
+#define PDUMPREGREAD64			PDumpRegRead64
+#define PDUMPCOMMENT(d, ...)	PDumpCommentWithFlags(d, PDUMP_FLAGS_CONTINUOUS, __VA_ARGS__)
+#define PDUMPCOMMENTWITHFLAGS	PDumpCommentWithFlags
+#define PDUMPREGPOL				PDumpRegPolKM
+#define PDUMPREGBASEDCBP		PDumpRegBasedCBP
+#define PDUMPENDINITPHASE		PDumpStopInitPhase
+#define PDUMPIDLWITHFLAGS		PDumpIDLWithFlags
+#define PDUMPIDL				PDumpIDL
+#define PDUMPPOWCMDSTART		PDumpPowerTransitionStart
+#define PDUMPPOWCMDEND			PDumpPowerTransitionEnd
+#define PDUMPCOM				PDumpCOMCommand
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_BLKSTART(_ui32PDumpFlags) do \
+	{ PDUMP_LOCK(_ui32PDumpFlags);\
+	_ui32PDumpFlags |= PDUMP_FLAGS_PDUMP_LOCK_HELD;\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_BLKEND(_ui32PDumpFlags) do \
+	{ _ui32PDumpFlags &= ~PDUMP_FLAGS_PDUMP_LOCK_HELD;\
+	PDUMP_UNLOCK(_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMPIF(_dev,_msg,_ui32PDumpFlags) do \
+	{PDUMP_BLKSTART(_ui32PDumpFlags);\
+	PDumpIfKM(_dev,_msg,_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#define PDUMPELSE				PDumpElseKM
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMPFI(_dev,_msg,_ui32PDumpFlags) do \
+	{ PDumpFiKM(_dev,_msg,_ui32PDumpFlags);\
+	PDUMP_BLKEND(_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#else
+/*
+	We should be clearer about which functions can be called
+	across the bridge as this looks rather unbalanced
+*/
+
+/*! Macro used to record a panic in the PDump script stream */
+#define PDUMP_PANIC(_dev, _id, _msg)  ((void)0)
+
+/*! Macro used to record a driver error in the PDump script stream to invalidate the capture */
+#define PDUMP_ERROR(_dev, _err, _msg) ((void)0)
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpInitCommon)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpInitCommon(void)
+{
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpConnectionNotify)
+#endif
+static INLINE void
+PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpDisconnectionNotify)
+#endif
+static INLINE void
+PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpLock)
+#endif
+static INLINE void
+PDumpLock(void)
+{
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnlock)
+#endif
+static INLINE void
+PDumpUnlock(void)
+{
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpStopInitPhase)
+#endif
+static INLINE void
+PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpSetFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE *psDeviceNode,
+                IMG_UINT32 ui32Frame)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32Frame);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE *psDeviceNode,
+                IMG_UINT32 *pui32Frame)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pui32Frame);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpCommentKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpCommentKM(CONNECTION_DATA *psConnection,
+               PVRSRV_DEVICE_NODE *psDeviceNode,
+               IMG_UINT32 ui32CommentSize,
+               IMG_CHAR *pszComment,
+               IMG_UINT32 ui32Flags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+    PVR_UNREFERENCED_PARAMETER(ui32CommentSize);
+	PVR_UNREFERENCED_PARAMETER(pszComment);
+	PVR_UNREFERENCED_PARAMETER(ui32Flags);
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpSetDefaultCaptureParamsKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Mode,
+                               IMG_UINT32 ui32Start,
+                               IMG_UINT32 ui32End,
+                               IMG_UINT32 ui32Interval,
+                               IMG_UINT32 ui32MaxParamFileSize,
+                               IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32Mode);
+	PVR_UNREFERENCED_PARAMETER(ui32Start);
+	PVR_UNREFERENCED_PARAMETER(ui32End);
+	PVR_UNREFERENCED_PARAMETER(ui32Interval);
+	PVR_UNREFERENCED_PARAMETER(ui32MaxParamFileSize);
+	PVR_UNREFERENCED_PARAMETER(ui32AutoTermTimeout);
+
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpPanic)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpPanic(IMG_UINT32      ui32PanicNo,
+           IMG_CHAR*       pszPanicMsg,
+           const IMG_CHAR* pszPPFunc,
+           IMG_UINT32      ui32PPline)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PanicNo);
+	PVR_UNREFERENCED_PARAMETER(pszPanicMsg);
+	PVR_UNREFERENCED_PARAMETER(pszPPFunc);
+	PVR_UNREFERENCED_PARAMETER(ui32PPline);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpCaptureError)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpCaptureError(PVRSRV_ERROR    ui32ErrorNo,
+                  IMG_CHAR*       pszErrorMsg,
+                  const IMG_CHAR* pszPPFunc,
+                  IMG_UINT32      ui32PPline)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32ErrorNo);
+	PVR_UNREFERENCED_PARAMETER(pszErrorMsg);
+	PVR_UNREFERENCED_PARAMETER(pszPPFunc);
+	PVR_UNREFERENCED_PARAMETER(ui32PPline);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpIsLastCaptureFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame)
+{
+	*pbIsLastCaptureFrame = IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetStateKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetStateKM(IMG_UINT64 *ui64State)
+{
+	*ui64State = 0;
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpForceCaptureStopKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                        PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpImageDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT32 ui32MMUContextID,
+                     IMG_CHAR *pszSABFileName,
+                     IMG_DEV_VIRTADDR sData,
+                     IMG_UINT32 ui32DataSize,
+                     IMG_UINT32 ui32LogicalWidth,
+                     IMG_UINT32 ui32LogicalHeight,
+                     IMG_UINT32 ui32PhysicalWidth,
+                     IMG_UINT32 ui32PhysicalHeight,
+                     PDUMP_PIXEL_FORMAT ePixFmt,
+                     IMG_MEMLAYOUT eMemLayout,
+                     IMG_FB_COMPRESSION eFBCompression,
+                     const IMG_UINT32 *paui32FBCClearColour,
+                     PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                     IMG_DEV_VIRTADDR sHeader,
+                     IMG_UINT32 ui32HeaderSize,
+                     IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpDataDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    IMG_UINT32 ui32MMUContextID,
+                    IMG_CHAR *pszSABFileName,
+                    IMG_DEV_VIRTADDR sData,
+                    IMG_UINT32 ui32DataSize,
+                    IMG_UINT32 ui32ElementType,
+                    IMG_UINT32 ui32ElementCount,
+                    IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterConnection)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        void *hSyncPrivData,
+                        PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                        PDUMP_CONNECTION_DATA **ppsPDumpConnectionData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hSyncPrivData);
+	PVR_UNREFERENCED_PARAMETER(pfnPDumpSyncBlocks);
+	PVR_UNREFERENCED_PARAMETER(ppsPDumpConnectionData);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterConnection)
+#endif
+static INLINE void
+PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterTransitionCallback)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                PFN_PDUMP_TRANSITION pfnCallback,
+                                void *hPrivData,
+                                void *pvDevice,
+                                void **ppvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+	PVR_UNREFERENCED_PARAMETER(pfnCallback);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pvDevice);
+	PVR_UNREFERENCED_PARAMETER(ppvHandle);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterTransitionCallback)
+#endif
+static INLINE void
+PDumpUnregisterTransitionCallback(void *pvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvHandle);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterTransitionCallback)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+                                         PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback,
+                                         void **ppvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pfnCallback);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(ppvHandle);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterTransitionCallbackFenceSync)
+#endif
+static INLINE void
+PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvHandle);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpTransition)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                PDUMP_TRANSITION_EVENT eEvent,
+                IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+	PVR_UNREFERENCED_PARAMETER(eEvent);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#if defined(__linux__) || defined(GCC_IA32) || defined(GCC_ARM) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+	#define PDUMPINIT	PDumpInitCommon
+	#define PDUMPDEINIT(args...)
+	#define PDUMPREG32(args...)
+	#define PDUMPREG64(args...)
+	#define PDUMPREGREAD32(args...)
+	#define PDUMPREGREAD64(args...)
+	#define PDUMPCOMMENT(args...)
+	#define PDUMPREGPOL(args...)
+	#define PDUMPSYNC(args...)
+	#define PDUMPCOPYTOMEM(args...)
+	#define PDUMPWRITE(args...)
+	#define PDUMPREGBASEDCBP(args...)
+	#define PDUMPCOMMENTWITHFLAGS(args...)
+	#define PDUMPENDINITPHASE(args...)
+	#define PDUMPIDLWITHFLAGS(args...)
+	#define PDUMPIDL(args...)
+	#define PDUMPPOWCMDSTART(args...)
+	#define PDUMPPOWCMDEND(args...)
+	#define PDUMP_LOCK(args...)
+	#define PDUMP_UNLOCK(args...)
+	#define PDUMPIF(args...)
+	#define PDUMPFI(args...)
+	#define PDUMPCOM(args...)
+#else
+	#error Compiler not specified
+#endif
+
+#endif /* PDUMP */
+
+#endif /* PDUMP_KM_H */
+
+/******************************************************************************
+ End of file (pdump_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_mmu.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_mmu.h
new file mode 100644
index 000000000000..5d03e430febd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_mmu.h
@@ -0,0 +1,150 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_PDUMP_MMU_H
+#define SRVKM_PDUMP_MMU_H
+
+/* services/server/include/ */
+#include "pdump_symbolicaddr.h"
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "mmu_common.h"
+
+#include "opaque_types.h"
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PDumpMMUMalloc(PPVRSRV_DEVICE_NODE psDeviceNode,
+               const IMG_CHAR *pszPDumpDevName,
+               MMU_LEVEL eMMULevel,
+               IMG_DEV_PHYADDR *psDevPAddr,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32Align,
+               PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpMMUFree(PPVRSRV_DEVICE_NODE psDeviceNode,
+             const IMG_CHAR *pszPDumpDevName,
+             MMU_LEVEL eMMULevel,
+             IMG_DEV_PHYADDR *psDevPAddr,
+             PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpPTBaseObjectToMem64(const IMG_CHAR *pszPDumpDevName,
+                         PMR *psPMRDest,
+                         IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                         IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                         IMG_UINT32 ui32Flags,
+                         MMU_LEVEL eMMULevel,
+                         IMG_UINT64 ui64PxSymAddr,
+                         IMG_UINT64 ui64PxOffset);
+
+PVRSRV_ERROR
+PDumpMMUDumpPxEntries(PPVRSRV_DEVICE_NODE psDeviceNode,
+                      MMU_LEVEL eMMULevel,
+                      const IMG_CHAR *pszPDumpDevName,
+                      void *pvPxMem,
+                      IMG_DEV_PHYADDR sPxDevPAddr,
+                      IMG_UINT32 uiFirstEntry,
+                      IMG_UINT32 uiNumEntries,
+                      const IMG_CHAR *pszMemspaceName,
+                      const IMG_CHAR *pszSymbolicAddr,
+                      IMG_UINT64 uiSymbolicAddrOffset,
+                      IMG_UINT32 uiBytesPerEntry,
+                      IMG_UINT32 uiLog2Align,
+                      IMG_UINT32 uiAddrShift,
+                      IMG_UINT64 uiAddrMask,
+                      IMG_UINT64 uiPxEProtMask,
+                      IMG_UINT64 uiDataValidEnable,
+                      IMG_UINT32 ui32Flags,
+                      IMG_UINT32 ui32VAParity,
+                      IMG_UINT32 ui32ParityShift,
+                      IMG_UINT64 ui64ParityMask,
+                      PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpMMUAllocMMUContext(PPVRSRV_DEVICE_NODE psDeviceNode,
+                        const IMG_CHAR *pszPDumpMemSpaceName,
+                        IMG_DEV_PHYADDR sPCDevPAddr,
+                        PDUMP_MMU_TYPE eMMUType,
+                        IMG_UINT32 *pui32MMUContextID,
+                        IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMMUFreeMMUContext(PPVRSRV_DEVICE_NODE psDeviceNode,
+                       const IMG_CHAR *pszPDumpMemSpaceName,
+                       IMG_UINT32 ui32MMUContextID,
+                       IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMMUSAB(PPVRSRV_DEVICE_NODE psDeviceNode,
+            const IMG_CHAR *pszPDumpMemNamespace,
+            IMG_UINT32 uiPDumpMMUCtx,
+            IMG_DEV_VIRTADDR sDevAddrStart,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            IMG_UINT32 ui32PDumpFlags);
+
+#define PDUMP_MMU_ALLOC_MMUCONTEXT(psDevNode, pszPDumpMemDevName, sPCDevPAddr, eMMUType, puiPDumpCtxID, ui32PDumpFlags) \
+        PDumpMMUAllocMMUContext(psDevNode,                              \
+                                pszPDumpMemDevName,                     \
+                                sPCDevPAddr,                            \
+                                eMMUType,                               \
+                                puiPDumpCtxID,                          \
+                                ui32PDumpFlags)
+
+#define PDUMP_MMU_FREE_MMUCONTEXT(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags) \
+        PDumpMMUFreeMMUContext(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags)
+#else /* PDUMP */
+
+#define PDUMP_MMU_ALLOC_MMUCONTEXT(pszPDumpMemDevName, sPCDevPAddr, eMMUType, puiPDumpCtxID, ui32PDumpFlags) \
+        ((void)0)
+#define PDUMP_MMU_FREE_MMUCONTEXT(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags) \
+        ((void)0)
+
+#endif /* PDUMP */
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_physmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_physmem.h
new file mode 100644
index 000000000000..bf38df94f3fa
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_physmem.h
@@ -0,0 +1,300 @@
+/**************************************************************************/ /*!
+@File
+@Title          pdump functions to assist with physmem allocations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PDUMP_PHYSMEM_H
+#define SRVSRV_PDUMP_PHYSMEM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pmr.h"
+#include "device.h" /* For device node */
+
+#define PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH 40
+#define PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH 60
+#define PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH (PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH + PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH)
+
+typedef struct _PDUMP_PHYSMEM_INFO_T_ PDUMP_PHYSMEM_INFO_T;
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                     IMG_CHAR **ppszSymbolicAddress);
+
+PVRSRV_ERROR
+PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicAddress,
+            IMG_UINT64 ui64Size,
+            /* alignment is alignment of start of buffer _and_
+               minimum contiguity - i.e. smallest allowable
+               page-size. */
+            IMG_DEVMEM_ALIGN_T uiAlign,
+            IMG_BOOL bInitialise,
+            IMG_UINT8 ui8InitValue,
+            IMG_HANDLE *phHandlePtr,
+            IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    const IMG_CHAR *pszDevSpace,
+                    const IMG_CHAR *pszSymbolicAddress,
+                    IMG_UINT64 ui64Size,
+                    /* alignment is alignment of start of buffer _and_
+                       minimum contiguity - i.e. smallest allowable
+                       page-size. */
+                    IMG_DEVMEM_ALIGN_T uiAlign,
+                    IMG_BOOL bInitialise,
+                    IMG_UINT8 ui8InitValue,
+                    IMG_HANDLE *phHandlePtr,
+                    IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+          IMG_HANDLE hPDumpAllocationInfoHandle);
+
+PVRSRV_ERROR
+PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                  IMG_HANDLE hPDumpAllocationInfoHandle);
+
+void
+PDumpMakeStringValid(IMG_CHAR *pszString,
+                     IMG_UINT32 ui32StrLen);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetSymbolicAddr)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                     IMG_CHAR **ppszSymbolicAddress)
+{
+	PVR_UNREFERENCED_PARAMETER(hPhysmemPDumpHandle);
+	PVR_UNREFERENCED_PARAMETER(ppszSymbolicAddress);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicAddress,
+            IMG_UINT64 ui64Size,
+            IMG_DEVMEM_ALIGN_T uiAlign,
+            IMG_BOOL bInitialise,
+            IMG_UINT8 ui8InitValue,
+            IMG_HANDLE *phHandlePtr,
+            IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pszDevSpace);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(uiAlign);
+	PVR_UNREFERENCED_PARAMETER(bInitialise);
+	PVR_UNREFERENCED_PARAMETER(ui8InitValue);
+	PVR_UNREFERENCED_PARAMETER(phHandlePtr);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    const IMG_CHAR *pszDevSpace,
+                    const IMG_CHAR *pszSymbolicAddress,
+                    IMG_UINT64 ui64Size,
+                    IMG_DEVMEM_ALIGN_T uiAlign,
+                    IMG_BOOL bInitialise,
+                    IMG_UINT8 ui8InitValue,
+                    IMG_HANDLE *phHandlePtr,
+                    IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pszDevSpace);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(uiAlign);
+	PVR_UNREFERENCED_PARAMETER(bInitialise);
+	PVR_UNREFERENCED_PARAMETER(ui8InitValue);
+	PVR_UNREFERENCED_PARAMETER(phHandlePtr);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+          IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hPDumpAllocationInfoHandle);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                  IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hPDumpAllocationInfoHandle);
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+#define PMR_DEFAULT_PREFIX "PMR"
+#define PMR_SPARSE_PREFIX "SPMR"
+#define PMR_SYMBOLICADDR_FMTSPEC "%s%"IMG_UINT64_FMTSPEC"_%"IMG_UINT64_FMTSPEC"_%s"
+#define PMR_MEMSPACE_FMTSPEC "%s"
+#define PMR_MEMSPACE_CACHE_COHERENT_FMTSPEC "CC_%s"
+
+PVRSRV_ERROR
+PDumpPMRWRW32(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT32 ui32Value,
+              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW32InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRRDW32MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW64(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT64 ui64Value,
+              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW64InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRRDW64MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRLDB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRSAB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFileName,
+            IMG_UINT32 uiFileOffset);
+
+/*
+  PDumpPMRPOL()
+
+  Emits a POL to the PDUMP.
+*/
+PVRSRV_ERROR
+PDumpPMRPOL(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMempaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_UINT32 ui32Value,
+            IMG_UINT32 ui32Mask,
+            PDUMP_POLL_OPERATOR eOperator,
+            IMG_UINT32 uiCount,
+            IMG_UINT32 uiDelay,
+            PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMemspaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize);
+
+/*
+ * PDumpWriteParameterBlob()
+ *
+ * Writes a binary blob to the pdump param stream containing the current
+ * contents of the memory, and returns the filename and offset of where
+ * that blob is located (for use in a subsequent LDB, for example).
+ *
+ * Caller to provide buffer to receive filename, and declare the size of
+ * that buffer.
+ */
+PVRSRV_ERROR
+PDumpWriteParameterBlob(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiNumBytes,
+                        PDUMP_FLAGS_T uiPDumpFlags,
+                        IMG_CHAR *pszFilenameOut,
+                        size_t uiFilenameBufSz,
+                        PDUMP_FILEOFFSET_T *puiOffsetOut);
+
+#endif /* #ifndef SRVSRV_PDUMP_PHYSMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_symbolicaddr.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_symbolicaddr.h
new file mode 100644
index 000000000000..ed912a5096c9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pdump_symbolicaddr.h
@@ -0,0 +1,55 @@
+/**************************************************************************/ /*!
+@File
+@Title          Abstraction of PDUMP symbolic address derivation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Allows pdump functions to derive symbolic addresses on-the-fly
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_PDUMP_SYMBOLICADDR_H
+#define SRVKM_PDUMP_SYMBOLICADDR_H
+
+#include "img_types.h"
+
+#include "pvrsrv_error.h"
+
+/* pdump symbolic addresses are generated on-the-fly with a callback */
+
+typedef PVRSRV_ERROR (*PVRSRV_SYMADDRFUNCPTR)(IMG_HANDLE hPriv, IMG_UINT32 uiOffset, IMG_CHAR *pszSymbolicAddr, IMG_UINT32 ui32SymbolicAddrLen, IMG_UINT32 *pui32NewOffset);
+
+#endif /* #ifndef SRVKM_PDUMP_SYMBOLICADDR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem.h
new file mode 100644
index 000000000000..6e2469f73e65
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem.h
@@ -0,0 +1,303 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for common entry point for creation of RAM backed PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_H
+#define SRVSRV_PHYSMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/* Valid values for TC_MEMORY_CONFIG configuration option */
+#define TC_MEMORY_LOCAL			(1)
+#define TC_MEMORY_HOST			(2)
+#define TC_MEMORY_HYBRID		(3)
+
+/* Valid values for the PLATO_MEMORY_CONFIG configuration option */
+#define PLATO_MEMORY_LOCAL		(1)
+#define PLATO_MEMORY_HOST		(2)
+#define PLATO_MEMORY_HYBRID		(3)
+
+/*************************************************************************/ /*!
+@Function       DevPhysMemAlloc
+@Description    Allocate memory from device specific heaps directly.
+@Input          psDevNode             device node to operate on
+@Input          ui32MemSize           Size of the memory to be allocated
+@Input          u8Value               Value to be initialised to.
+@Input          bInitPage             Flag to control initialisation
+@Input          pszDevSpace           PDUMP memory space in which the
+                                        allocation is to be done
+@Input          pszSymbolicAddress    Symbolic name of the allocation
+@Input          phHandlePtr           PDUMP handle to the allocation
+@Input          uiPid                 PID of the process owning the allocation
+                                        (or PVR_SYS_ALLOC_PID if the allocation
+                                        belongs to the driver)
+@Output         hMemHandle            Handle to the allocated memory
+@Output         psDevPhysAddr         Device Physical address of allocated
+                                        page
+@Return         PVRSRV_OK if the allocation is successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevPhysMemAlloc(PVRSRV_DEVICE_NODE *psDevNode,
+                IMG_UINT32 ui32MemSize,
+                IMG_UINT32 ui32Log2Align,
+                const IMG_UINT8 u8Value,
+                IMG_BOOL bInitPage,
+#if defined(PDUMP)
+                const IMG_CHAR *pszDevSpace,
+                const IMG_CHAR *pszSymbolicAddress,
+                IMG_HANDLE *phHandlePtr,
+#endif
+                IMG_PID uiPid,
+                IMG_HANDLE hMemHandle,
+                IMG_DEV_PHYADDR *psDevPhysAddr);
+
+/*************************************************************************/ /*!
+@Function       DevPhysMemFree
+@Description    Free memory to device specific heaps directly.
+@Input          psDevNode             device node to operate on
+@Input          hPDUMPMemHandle       Pdump handle to allocated memory
+@Input          hMemHandle            Devmem handle to allocated memory
+@Return         None
+*/ /**************************************************************************/
+void
+DevPhysMemFree(PVRSRV_DEVICE_NODE *psDevNode,
+#if defined(PDUMP)
+               IMG_HANDLE hPDUMPMemHandle,
+#endif
+               IMG_HANDLE hMemHandle);
+
+/*
+ * PhysmemNewRamBackedPMR
+ *
+ * This function will create a RAM backed PMR using the device specific
+ * callback, this allows control at a per-devicenode level to select the
+ * memory source thus supporting mixed UMA/LMA systems.
+ *
+ * The size must be a multiple of page size. The page size is specified in
+ * log2. It should be regarded as a minimum contiguity of which the
+ * resulting memory must be a multiple. It may be that this should be a fixed
+ * number. It may be that the allocation size needs to be a multiple of some
+ * coarser "page size" than that specified in the page size argument.
+ * For example, take an OS whose page granularity is a fixed 16kB, but the
+ * caller requests memory in page sizes of 4kB. The request can be satisfied
+ * if and only if the SIZE requested is a multiple of 16kB. If the arguments
+ * supplied are such that this OS cannot grant the request,
+ * PVRSRV_ERROR_INVALID_PARAMS will be returned.
+ *
+ * The caller should supply storage of a pointer. Upon successful return a
+ * PMR object will have been created and a pointer to it returned in the
+ * PMROut argument.
+ *
+ * A PMR successfully created should be destroyed with PhysmemUnrefPMR.
+ *
+ * Note that this function may cause memory allocations and on some operating
+ * systems this may cause scheduling events, so it is important that this
+ * function be called with interrupts enabled and in a context where
+ * scheduling events and memory allocations are permitted.
+ *
+ * The flags may be used by the implementation to change its behaviour if
+ * required. The flags will also be stored in the PMR as immutable metadata
+ * and returned to mmu_common when it asks for it.
+ *
+ * The PID specified is used to tie this allocation to the process context
+ * that the allocation is made on behalf of.
+ */
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR(CONNECTION_DATA * psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMROut,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags);
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR_direct(CONNECTION_DATA * psConnection,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  IMG_DEVMEM_SIZE_T uiSize,
+							  IMG_UINT32 ui32NumPhysChunks,
+							  IMG_UINT32 ui32NumVirtChunks,
+							  IMG_UINT32 *pui32MappingTable,
+							  IMG_UINT32 uiLog2PageSize,
+							  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							  IMG_UINT32 uiAnnotationLength,
+							  const IMG_CHAR *pszAnnotation,
+							  IMG_PID uiPid,
+							  PMR **ppsPMROut,
+							  IMG_UINT32 ui32PDumpFlags,
+							  PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysmemImportPMR
+@Description    Import PMR a previously exported PMR
+@Input          psPMRExport           The exported PMR token
+@Input          uiPassword            Authorisation password
+                                      for the PMR being imported
+@Input          uiSize                Size of the PMR being imported
+                                      (for verification)
+@Input          uiLog2Contig          Log2 continuity of the PMR being
+                                      imported (for verification)
+@Output         ppsPMR                The imported PMR
+@Return         PVRSRV_ERROR_PMR_NOT_PERMITTED if not for the same device
+                PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR if password incorrect
+                PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES if size or contiguity incorrect
+                PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemImportPMR(CONNECTION_DATA *psConnection,
+                 PVRSRV_DEVICE_NODE *psDevNode,
+                 PMR_EXPORT *psPMRExport,
+                 PMR_PASSWORD_T uiPassword,
+                 PMR_SIZE_T uiSize,
+                 PMR_LOG2ALIGN_T uiLog2Contig,
+                 PMR **ppsPMR);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDefaultPhysicalHeapKM
+@Description    For the specified device, get the physical heap used for
+                allocations when the PVRSRV_PHYS_HEAP_DEFAULT
+                physical heap hint is set in memalloc flags.
+@Output         peHeap                 Default Heap return value
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVGetDefaultPhysicalHeapKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDevNode,
+                               PVRSRV_PHYS_HEAP *peHeap);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVPhysHeapGetMemInfoKM
+@Description    Get the memory usage statistics for a given physical heap ID
+@Input          ui32PhysHeapCount      Physical Heap count
+@Input          paePhysHeapID          Array of Physical Heap ID's
+@Output         paPhysHeapMemStats     Buffer to hold the memory statistics
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfoKM(CONNECTION_DATA *psConnection,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           IMG_UINT32 ui32PhysHeapCount,
+                           PVRSRV_PHYS_HEAP *paePhysHeapID,
+                           PHYS_HEAP_MEM_STATS_V1 *paPhysHeapMemStats);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVPhysHeapGetMemInfoKM
+@Description    Get the memory usage statistics for a given physical heap ID
+@Input          ui32PhysHeapCount      Physical Heap count
+@Input          paePhysHeapID          Array of Physical Heap ID's
+@Output         paPhysHeapMemStats     Buffer to hold the memory statistics
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfo2KM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDevNode,
+                            IMG_UINT32 ui32PhysHeapCount,
+                            PVRSRV_PHYS_HEAP *paePhysHeapID,
+                            PHYS_HEAP_MEM_STATS_V2 *paPhysHeapMemStats);
+
+/*************************************************************************/ /*!
+@Function       PhysMemValidateMappingTable
+@Description    Checks the PMR mapping table provided is valid (ie has
+                no entries with an index value outside the valid range
+                of indices for the allocation and has no repeated indices)
+
+@Input          ui32TotalNumVirtChunks Total number of virtual chunks
+                                       the allocation has.
+@Input          ui32IndexCount         Number of entries in the mapping
+                                       table.
+@Input          pui32MappingTable      Mapping Table.
+@Return         PVRSRV_OK              if parameters are valid.
+                PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY if mapping table
+                                       contains an index out of range or a
+                                       repeated index
+                PVRSRV_ERROR_OUT_OF_MEMORY if unable to allocate memory
+                                       for the index tracking table
+                                       (used internally in this function)
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysMemValidateMappingTable(IMG_UINT32 ui32TotalNumVirtChunks,
+                            IMG_UINT32 ui32IndexCount,
+                            const IMG_UINT32 *pui32MappingTable);
+
+/*************************************************************************/ /*!
+@Function       PhysMemValidateParams
+@Description    Checks the PMR creation parameters and adjusts them
+                if possible and necessary
+
+@Input          psDevNode              The associated device node.
+@Input          ui32NumPhysChunks      Number of physical chunks.
+@Input          ui32NumVirtChunks      Number of virtual chunks.
+@Input          pui32MappingTable      Mapping Table.
+@Input          uiFlags                Allocation flags.
+@Input          uiPid                  PID of current process.
+@Inout          puiLog2AllocPageSize   Log2 of allocation page size.
+                                       May be adjusted.
+@Inout          puiSize                Size of the allocation.
+                                       May be adjusted.
+@Return         PVRSRV_OK if parameters are valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysMemValidateParams(PVRSRV_DEVICE_NODE *psDevNode,
+                      IMG_UINT32 ui32NumPhysChunks,
+                      IMG_UINT32 ui32NumVirtChunks,
+                      IMG_UINT32 *pui32MappingTable,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      IMG_PID uiPid,
+                      IMG_UINT32 *puiLog2AllocPageSize,
+                      IMG_DEVMEM_SIZE_T *puiSize);
+
+#endif /* SRVSRV_PHYSMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_cpumap_history.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_cpumap_history.h
new file mode 100644
index 000000000000..0f78191d9746
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_cpumap_history.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File
+@Title          Definitions of PMR Mapping History for OS managed memory
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PHYSMEM_CPUMAP_HISTORY_H_
+#define PHYSMEM_CPUMAP_HISTORY_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include <powervr/mem_types.h>
+
+/*!
+*******************************************************************************
+ @Function	  CPUMappingHistoryInit
+ @Description CPU Mapping history initialisation
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR CPUMappingHistoryInit(void);
+
+/*!
+*******************************************************************************
+ @Function	  CPUMappingHistoryDeInit
+ @Description CPU Mapping history De-initialisation
+ @Return      None
+******************************************************************************/
+void CPUMappingHistoryDeInit(void);
+
+/*!
+*******************************************************************************
+ @Function	  InsertMappingRecord
+ @Description CPU Mapping history Map Record insertion.
+              This function given relevant mapping information, inserts a
+              mapping record to the history buffer.
+ @Input       pszAnnotation - The annotation related to the allocation to be
+                              mapped.
+ @Input       uiPID         - The PID of the process mapping the allocation.
+ @Input       pvAddress     - The CPU virtual address of the newly mapped
+                              allocation.
+ @Input       sCPUPhyAddr       - The CPU Physical address of the newly mapped
+                                  allocation.
+ @Input       ui32CPUCacheFlags - The CPU Caching flags associated with the
+                                  mapping.
+ @Input       uiMapOffset       - The offset into the PMR at which the mapping
+                                  resides.
+ @input       ui32PageCount     - The number of pages mapped.
+ @Return      None
+******************************************************************************/
+void InsertMappingRecord(const IMG_CHAR *pszAnnotation,
+                         IMG_PID uiPID,
+                         IMG_CPU_VIRTADDR pvAddress,
+                         IMG_CPU_PHYADDR sCpuPhyAddr,
+                         IMG_UINT32 ui32CPUCacheFlags,
+                         size_t uiMapOffset,
+                         IMG_UINT32 ui32PageCount);
+
+/*!
+*******************************************************************************
+ @Function	  InsertUnMappingRecord
+ @Description CPU Mapping history UnMap Record insertion.
+              This function given relevant mapping information, inserts an
+              un-mapping record to the history buffer.
+ @Input       pvAddress     - The CPU virtual address of the un-mapped
+                              allocation.
+ @Input       sCPUPhyAddr       - The CPU Physical address of the un-mapped
+                                  allocation.
+ @Input       ui32CPUCacheFlags - The CPU Caching flags associated with the
+                                  mapping.
+ @input       ui32PageCount     - The number of pages un-mapped.
+ @Return      None
+******************************************************************************/
+void InsertUnMappingRecord(IMG_CPU_VIRTADDR pvAddress,
+                           IMG_CPU_PHYADDR sCpuPhyAddr,
+                           IMG_UINT32 ui32CPUCacheFlags,
+                           IMG_UINT32 ui32PageCount);
+
+
+#endif /* PHYSMEM_CPUMAP_HISTORY_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dlm.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dlm.h
new file mode 100644
index 000000000000..7d18fd781f2f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dlm.h
@@ -0,0 +1,88 @@
+/*************************************************************************/ /*!
+@File           physmem_dlm.h
+@Title          Dedicated Local Memory allocator for Physical Memory Blocks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for dedicated local memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_DLM_H
+#define PHYSMEM_DLM_H
+
+#include "img_types.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+
+/* PMBDestroy
+ *
+ * Destroys a given PMB used to represent a block of memory
+ * obtained from a DLM heap.
+ */
+void
+PMBDestroy(PMB *psPMB);
+
+/* PMBGetAnnotation
+ *
+ * Obtain the annotation of the allocation represented
+ * by this PMB
+ */
+const IMG_CHAR *
+PMBGetAnnotation(PMB *psPMB);
+
+/* DLM */
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapDLM
+@Description    Create and register new DLM heap.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Input          pszLabel     Debug identifier label
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapDLM(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* #ifndef PHYSMEM_DLM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dmabuf.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dmabuf.h
new file mode 100644
index 000000000000..1896204d3f6e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_dmabuf.h
@@ -0,0 +1,133 @@
+/**************************************************************************/ /*!
+@File           physmem_dmabuf.h
+@Title          Header for dmabuf PMR factory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks importing Ion allocations
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PHYSMEM_DMABUF_H)
+#define PHYSMEM_DMABUF_H
+
+#include <linux/dma-buf.h>
+
+#include "pvr_dma_resv.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+#include "pmr.h"
+
+typedef void (*PFN_DESTROY_DMABUF_PMR)(PHYS_HEAP *psHeap,
+                                       struct dma_buf_attachment *psAttachment);
+
+PVRSRV_ERROR
+PhysmemCreateNewDmaBufBackedPMR(PHYS_HEAP *psHeap,
+                                struct dma_buf_attachment *psAttachment,
+                                PFN_DESTROY_DMABUF_PMR pfnDestroy,
+                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                IMG_PID uiPid,
+                                IMG_DEVMEM_SIZE_T uiChunkSize,
+                                IMG_UINT32 ui32NumPhysChunks,
+                                IMG_UINT32 ui32NumVirtChunks,
+                                IMG_UINT32 *pui32MappingTable,
+		                        IMG_UINT32 ui32NameSize,
+		                        const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                                PMR **ppsPMRPtr);
+
+struct dma_buf *
+PhysmemGetDmaBuf(PMR *psPMR);
+
+struct dma_resv *
+PhysmemGetDmaResv(PMR *psPMR);
+
+#if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+struct dma_heap *
+PhysmemGetDmaHeap(PMR *psPMR);
+
+void
+PhysmemSetDmaHeap(PMR *psPMR, struct dma_heap *psDmaHeap);
+#endif /* #if defined(SUPPORT_SECURE_ALLOC_KM) && defined(PVR_ANDROID_HAS_DMA_HEAP_FIND) */
+
+PVRSRV_ERROR
+PhysmemExportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    PMR *psPMR,
+                    IMG_INT *piFd);
+
+PVRSRV_ERROR
+PhysmemExportGemHandle(CONNECTION_DATA *psConnection,
+		       PVRSRV_DEVICE_NODE *psDevNode,
+		       PMR *psPMR,
+		       IMG_UINT32 *puHandle);
+
+PVRSRV_ERROR
+PhysmemImportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_INT fd,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    IMG_UINT32 ui32NameSize,
+                    const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                    PMR **ppsPMRPtr,
+                    IMG_DEVMEM_SIZE_T *puiSize,
+                    IMG_DEVMEM_ALIGN_T *puiAlign);
+
+PVRSRV_ERROR
+PhysmemImportSparseDmaBuf(CONNECTION_DATA *psConnection,
+                          PVRSRV_DEVICE_NODE *psDevNode,
+                          IMG_INT fd,
+                          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                          IMG_DEVMEM_SIZE_T uiChunkSize,
+                          IMG_UINT32 ui32NumPhysChunks,
+                          IMG_UINT32 ui32NumVirtChunks,
+                          IMG_UINT32 *pui32MappingTable,
+                          IMG_UINT32 ui32NameSize,
+                          const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                          PMR **ppsPMRPtr,
+                          IMG_DEVMEM_SIZE_T *puiSize,
+                          IMG_DEVMEM_ALIGN_T *puiAlign);
+
+#endif /* !defined(PHYSMEM_DMABUF_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_extmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_extmem.h
new file mode 100644
index 000000000000..cc7d23297d43
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_extmem.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem_extmem header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for common entry point for creation of wrapped memory PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_EXTMEM_H
+#define SRVSRV_PHYSMEM_EXTMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/*
+ * Based on PhysmemNewRamBackedPMR but uses a passed in virtual address.
+ */
+PVRSRV_ERROR
+PhysmemWrapExtMemOS(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    IMG_CPU_VIRTADDR pvCpuVAddr,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    PMR **ppsPMRPtr);
+
+PVRSRV_ERROR
+PhysmemWrapExtMem(CONNECTION_DATA *psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT64 pvCpuVAddr,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  PMR **ppsPMROut);
+
+#endif /* SRVSRV_PHYSMEM_EXTMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_hostmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_hostmem.h
new file mode 100644
index 000000000000..cfa453de343a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_hostmem.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File           physmem_hostmem.h
+@Title          Host memory device node header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PHYSMEM_HOSTMEM_H)
+#define PHYSMEM_HOSTMEM_H
+
+#include "pvrsrv_device.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       HostMemDeviceCreate
+@Description    Allocate memory for and create host memory device node.
+@Output         ppsDeviceNode Pointer to device node pointer.
+@Return         PVRSRV_ERROR  PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR HostMemDeviceCreate(PVRSRV_DEVICE_NODE **ppsDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       HostMemDeviceDestroy
+@Description    Destroy host memory device node.
+@Input          psDeviceNode  Pointer to device node.
+*/ /**************************************************************************/
+void HostMemDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* !defined(PHYSMEM_HOSTMEM_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ima.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ima.h
new file mode 100644
index 000000000000..4744dacacb31
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ima.h
@@ -0,0 +1,77 @@
+/**************************************************************************/ /*!
+@File
+@Title          Header import memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of memory management. This module is responsible for
+                implementing the function callbacks for local card memory when
+                used under a shared heap system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PHYSMEM_IMA_H
+#define PHYSMEM_IMA_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapIMA
+@Description    Create and register new IMA heap with IMA specific details and
+                a DLM heap backing.
+@Input          psDevNode     Pointer to device node struct.
+@Input          uiPolicy      Heap allocation policy flags
+@Input          psConfig      Heap configuration.
+@Input          pszLabel      Debug identifier label
+@Input          psDLMHeap     DLM heap backing this LMA heap.
+@Input          uiLog2PMBSize Log 2 of PMB Size in bytes supported by DLM heap.
+@Output         ppsPhysHeap   Pointer to the created heap.
+@Return         PVRSRV_ERROR  PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapIMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP *psDLMHeap,
+                     IMG_UINT32 uiLog2PMBSize,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* PHYSMEM_IMA_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_lma.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_lma.h
new file mode 100644
index 000000000000..af42d8826dac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_lma.h
@@ -0,0 +1,75 @@
+/**************************************************************************/ /*!
+@File
+@Title          Header for local card memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for local card memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_LMA_H
+#define SRVSRV_PHYSMEM_LMA_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "device.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapLMA
+@Description    Create and register new LMA heap with LMA specific details.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Input          pszLabel     Debug identifier label
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapLMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* #ifndef SRVSRV_PHYSMEM_LMA_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_osmem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_osmem.h
new file mode 100644
index 000000000000..5e76513477b6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_osmem.h
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File           physmem_osmem.h
+@Title          OS memory PMR factory API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                OS memory PMR factory API that must be defined so that the
+                common & device layer code in the Services Server can allocate
+                new PMRs back with pages from the OS page allocator. Applicable
+                for UMA based platforms, such platforms must implement this API
+                in the OS Porting layer, in the "env" directory for that
+                system.
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_OSMEM_H
+#define PHYSMEM_OSMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "connection_server.h"
+#include "physheap.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapOSMEM
+@Description    Create and register new OSMEM heap with OSMEM specific details.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapOSMEM(PVRSRV_DEVICE_NODE *psDevNode,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       PHYS_HEAP_CONFIG *psConfig,
+                       PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysmemNewOSRamBackedPMR
+@Description    Rogue Services will call this function to allocate GPU device
+                memory from the PMR factory supported by the OS DDK port. This
+                factory typically obtains physical memory from the kernel/OS
+                API that allocates memory from the default heap of shared
+                system memory available on the platform. The allocated memory
+                must be page-aligned and be a whole number of pages.
+                After allocating the required memory, the implementation must
+                then call PMRCreatePMR() to obtain the PMR structure that
+                describes this allocation to the upper layers of the Services.
+                memory management sub-system.
+                NB. Implementation of this function is mandatory. If shared
+                system memory is not to be used in the OS port then the
+                implementation must return PVRSRV_ERROR_NOT_SUPPORTED.
+
+@Input          psPhysHeap       The phys heap
+@Input          psConnection     The connection to the originator process
+@Input          uiSize           The size of the allocation
+                                 (must be a multiple of page size)
+@Input          ui32NumPhysChunks     When sparse allocations are requested,
+                                      this is the number of physical chunks
+                                      to be allocated.
+                                      For regular allocations, this will be 1.
+@Input          ui32NumLogicalChunks  When sparse allocations are requested,
+                                      this is the number of logical chunks
+                                      covering the sparse allocation.
+                                      For regular allocations, this will be 1.
+@Input          pui32MappingTable  When sparse allocations are requested,
+                                   this is the list of the indices of
+                                   each physically-backed virtual chunk
+                                   For regular allocations, this will
+                                   be NULL.
+@Input          uiLog2DevPageSize  The physical dev pagesize in log2(bytes).
+@Input          uiFlags            The allocation flags.
+@Input          pszAnnotation      String describing the PMR (for debug).
+                                   This should be passed into the function
+                                   PMRCreatePMR().
+@Input          uiPid              The process ID that this allocation should
+                                   be associated with.
+@Output         ppsPMROut          Pointer to the PMR created for the
+                                   new allocation
+@Input          ui32PDumpFlags     The pdump flags.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemNewOSRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                         CONNECTION_DATA *psConnection,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         IMG_UINT32 ui32NumPhysChunks,
+                         IMG_UINT32 ui32NumLogicalChunks,
+                         IMG_UINT32 *pui32MappingTable,
+                         IMG_UINT32 uiLog2DevPageSize,
+                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                         const IMG_CHAR *pszAnnotation,
+                         IMG_PID uiPid,
+                         PMR **ppsPMROut,
+                         IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysmemGetOSRamMemStats
+@Description    Function that gets the OS memory usage statistics
+@Input          pvImplData     Physical heap private data.
+@Output         pui64TotalSize Buffer that holds the total OS memory size
+@Output         pui64FreeSize  Buffer that holds the free OS memory size
+@Return         None.
+*/ /**************************************************************************/
+void PhysmemGetOSRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                            IMG_UINT64 *pui64TotalSize,
+                            IMG_UINT64 *pui64FreeSize);
+
+#endif /* PHYSMEM_OSMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ramem.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ramem.h
new file mode 100644
index 000000000000..9dbf668e21b9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/physmem_ramem.h
@@ -0,0 +1,109 @@
+/*************************************************************************/ /*!
+@File           physmem_ramem.h
+@Title          Resource allocator managed PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                RA managed memory PMR factory API that is shared between local
+                physheap implementations (LMA & IMA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_RAMEM_H
+#define PHYSMEM_RAMEM_H
+
+#include "img_types.h"
+#include "pvrsrv_memallocflags.h"
+#include "physheap.h"
+#include "ra.h"
+#include "connection_server.h"
+#include "pmr_impl.h"
+
+IMG_UINT32
+RAMemGetPageShift(void);
+
+PVRSRV_ERROR
+RAMemDoPhyContigPagesAlloc(RA_ARENA *pArena,
+                           size_t uiSize,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_PID uiPid);
+
+void
+RAMemDoPhyContigPagesFree(RA_ARENA *pArena,
+                          PG_HANDLE *psMemHandle);
+
+PVRSRV_ERROR
+RAMemPhyContigPagesMap(PHYS_HEAP *psPhysHeap,
+                  PG_HANDLE *psMemHandle,
+                  size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                  void **pvPtr);
+
+void
+RAMemPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap,
+                    PG_HANDLE *psMemHandle,
+                    void *pvPtr);
+
+PVRSRV_ERROR
+RAMemPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                    PG_HANDLE *psMemHandle,
+                    IMG_UINT32 uiOffset,
+                    IMG_UINT32 uiLength);
+
+/*
+ * PhysmemNewRAMemRamBackedPMR
+ *
+ * This function will create a PMR using the memory managed by
+ * the PMR factory and is OS agnostic.
+ */
+PVRSRV_ERROR
+PhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                            RA_ARENA *pArena,
+                            CONNECTION_DATA *psConnection,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_UINT32 ui32NumPhysChunks,
+                            IMG_UINT32 ui32NumVirtChunks,
+                            IMG_UINT32 *pui32MappingTable,
+                            IMG_UINT32 uiLog2PageSize,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                            const IMG_CHAR *pszAnnotation,
+                            IMG_PID uiPid,
+                            PMR **ppsPMRPtr,
+                            IMG_UINT32 ui32PDumpFlags);
+
+#endif /* PHYSMEM_RAMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr.h
new file mode 100644
index 000000000000..c932f983ba5c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr.h
@@ -0,0 +1,1309 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                the "PMR" abstraction.  A PMR (Physical Memory Resource)
+                represents some unit of physical memory which is
+                allocated/freed/mapped/unmapped as an indivisible unit
+                (higher software levels provide an abstraction above that
+                to deal with dividing this down into smaller manageable units).
+                Importantly, this module knows nothing of virtual memory, or
+                of MMUs etc., with one excusable exception.  We have the
+                concept of a "page size", which really means nothing in
+                physical memory, but represents a "contiguity quantum" such
+                that the higher level modules which map this memory are able
+                to verify that it matches the needs of the page size for the
+                virtual realm into which it is being mapped.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PMR_H
+#define SRVSRV_PMR_H
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"			/* Required for export DEVMEM_EXPORTCOOKIE */
+
+/* services/include */
+#include "pdump.h"
+#include "physheap.h"
+
+/* services/server/include/ */
+#include "pmr_impl.h"
+#include "opaque_types.h"
+
+#define PMR_MAX_TRANSLATION_STACK_ALLOC				(32)
+
+/* Maximum size PMR can have is 8G of memory */
+#define PMR_MAX_SUPPORTED_SIZE IMG_UINT64_C(0x200000000)
+/* Max number of pages in a PMR at 4k page size */
+#define PMR_MAX_SUPPORTED_4K_PAGE_COUNT (PMR_MAX_SUPPORTED_SIZE >> 12ULL)
+
+typedef IMG_UINT64 PMR_BASE_T;
+typedef IMG_UINT64 PMR_SIZE_T;
+#define PMR_SIZE_FMTSPEC "0x%010"IMG_UINT64_FMTSPECX
+#define PMR_VALUE32_FMTSPEC "0x%08X"
+#define PMR_VALUE64_FMTSPEC "0x%016"IMG_UINT64_FMTSPECX
+typedef IMG_UINT32 PMR_LOG2ALIGN_T;
+typedef IMG_UINT64 PMR_PASSWORD_T;
+
+struct _PMR_MAPPING_TABLE_
+{
+	PMR_SIZE_T uiChunkSize;            /*!< Size of a "chunk" */
+	IMG_UINT32 ui32NumPhysChunks;      /*!< Number of physical chunks that are valid */
+	IMG_UINT32 ui32NumLogicalChunks;   /*!< Number of logical chunks in the mapping */
+	/* Must be last */
+	IMG_UINT32 aui32Translation[IMG_FLEX_ARRAY_MEMBER];    /*!< Translation mapping for "logical" to physical */
+};
+
+#define TRANSLATION_INVALID 0xFFFFFFFFUL
+
+typedef struct _PMR_EXPORT_ PMR_EXPORT;
+
+typedef struct _PMR_PAGELIST_ PMR_PAGELIST;
+
+IMG_INT32 PMRGetLiveCount(void);
+
+/*
+ * PMRValidateSize
+ *
+ * Given a size value, check the value against the max supported
+ * PMR size of 1GB. Return IMG_FALSE if size exceeds max, IMG_TRUE
+ * otherwise.
+ */
+static inline IMG_BOOL PMRValidateSize(IMG_UINT64 uiSize)
+{
+	return (uiSize > PMR_MAX_SUPPORTED_SIZE) ? IMG_FALSE : IMG_TRUE;
+}
+
+/*
+ * PMRCreatePMR
+ *
+ * Not to be called directly, only via implementations of PMR
+ * factories, e.g. in physmem_osmem.c, deviceclass.c, etc.
+ *
+ * Creates a PMR object, with callbacks and private data as per the
+ * FuncTab/PrivData args.
+ *
+ * Note that at creation time the PMR must set in stone the "logical
+ * size" and the "contiguity guarantee"
+ *
+ * Flags are also set at this time.  (T.B.D.  flags also immutable for
+ * the life of the PMR?)
+ *
+ * Logical size is the amount of Virtual space this allocation would
+ * take up when mapped.  Note that this does not have to be the same
+ * as the actual physical size of the memory.  For example, consider
+ * the sparsely allocated non-power-of-2 texture case.  In this
+ * instance, the "logical size" would be the virtual size of the
+ * rounded-up power-of-2 texture.  That some pages of physical memory
+ * may not exist does not affect the logical size calculation.
+ *
+ * The PMR must also supply the "contiguity guarantee" which is the
+ * finest granularity of alignment and size of physical pages that the
+ * PMR will provide after LockSysPhysAddresses is called.  Note that
+ * the calling code may choose to call PMRSysPhysAddr with a finer
+ * granularity than this, for example if it were to map into a device
+ * MMU with a smaller page size, and it's also OK for the PMR to
+ * supply physical memory in larger chunks than this.  But
+ * importantly, never the other way around.
+ *
+ * More precisely, the following inequality must be maintained
+ * whenever mappings and/or physical addresses exist:
+ *
+ *       (device MMU page size) <= 2**(uiLog2ContiguityGuarantee) <= (actual contiguity of physical memory)
+ *
+ * The function table will contain the following callbacks which may
+ * be overridden by the PMR implementation:
+ *
+ * pfnLockPhysAddresses
+ *
+ *      Called when someone locks requests that Physical pages are to
+ *      be locked down via the PMRLockSysPhysAddresses() API.  Note
+ *      that if physical pages are prefaulted at PMR creation time and
+ *      therefore static, it would not be necessary to override this
+ *      function, in which case NULL may be supplied.
+ *
+ * pfnUnlockPhysAddresses
+ *
+ *      The reverse of pfnLockPhysAddresses.  Note that this should be
+ *      NULL if and only if pfnLockPhysAddresses is NULL
+ *
+ * pfnSysPhysAddr
+ *
+ *      This function is mandatory.  This is the one which returns the
+ *      system physical address for a given offset into this PMR.  The
+ *      "lock" function will have been called, if overridden, before
+ *      this function, thus the implementation should not increase any
+ *      refcount when answering this call.  Refcounting, if necessary,
+ *      should be done in the lock/unlock calls.  Refcounting would
+ *      not be necessary in the prefaulted/static scenario, as the
+ *      pmr.c abstraction will handle the refcounting for the whole
+ *      PMR.
+ *
+ * pfnFinalize
+ *
+ *      Called when the PMR's refcount reaches zero and it gets
+ *      destroyed.  This allows the implementation to free up any
+ *      resource acquired during creation time.
+ *
+ */
+PVRSRV_ERROR
+PMRCreatePMR(PHYS_HEAP *psPhysHeap,
+             PMR_SIZE_T uiLogicalSize,
+             IMG_UINT32 ui32NumPhysChunks,
+             IMG_UINT32 ui32NumLogicalChunks,
+             IMG_UINT32 *pui32MappingTable,
+             PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+             PMR_FLAGS_T uiFlags,
+             const IMG_CHAR *pszAnnotation,
+             const PMR_IMPL_FUNCTAB *psFuncTab,
+             PMR_IMPL_PRIVDATA pvPrivData,
+             PMR_IMPL_TYPE eType,
+             PMR **ppsPMRPtr,
+             IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * PMRLockSysPhysAddresses()
+ *
+ * Calls the relevant callback to lock down the system physical addresses of
+ * the memory that makes up the whole PMR.
+ *
+ * Before this call, it is not valid to use any of the information
+ * getting APIs: PMR_Flags(), PMR_SysPhysAddr(),
+ * [ see note below about lock/unlock semantics ]
+ *
+ * The caller of this function does not have to care about how the PMR
+ * is implemented.  He only has to know that he is allowed access to
+ * the physical addresses _after_ calling this function and _until_
+ * calling PMRUnlockSysPhysAddresses().
+ *
+ *
+ * Notes to callback implementers (authors of PMR Factories):
+ *
+ * Some PMR implementations will be such that the physical memory exists for
+ * the lifetime of the PMR, with a static address, (and normally flags and
+ * symbolic address are static too) and so it is legal for a PMR
+ * implementation to not provide an implementation for the lock callback.
+ *
+ * Some PMR implementation may wish to page memory in from secondary storage
+ * on demand. The lock/unlock callbacks _may_ be the place to do this.
+ * (More likely, there would be a separate API for doing this, but this API
+ * provides a useful place to assert that it has been done)
+ */
+
+PVRSRV_ERROR
+PMRLockSysPhysAddresses(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRLockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount);
+
+PVRSRV_ERROR
+PMRLockSysPhysAddressesNested(PMR *psPMR,
+                           IMG_UINT32 uiLockCount,
+                           IMG_UINT32 ui32NestingLevel);
+
+/*
+ * PMRUnlockSysPhysAddresses()
+ *
+ * the reverse of PMRLockSysPhysAddresses()
+ */
+PVRSRV_ERROR
+PMRUnlockSysPhysAddresses(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesN(PMR *psPMR, IMG_UINT32 uiLockCount);
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesNested(PMR *psPMR,
+                             IMG_UINT32 uiLockCount,
+                             IMG_UINT32 ui32NestingLevel);
+
+/*
+ * PhysmemPMRExport()
+ *
+ * Given a PMR, creates a PMR "Export", which is a handle that
+ * provides sufficient data to be able to "import" this PMR elsewhere.
+ * The PMR Export is an object in its own right, whose existence
+ * implies a reference on the PMR, thus the PMR cannot be destroyed
+ * while the PMR Export exists.  The intention is that the PMR Export
+ * will be wrapped in the devicemem layer by a cross process handle,
+ * and some IPC by which to communicate the handle value and password
+ * to other processes.  The receiving process is able to unwrap this
+ * to gain access to the same PMR Export in this layer, and, via
+ * PhysmemPMRImport(), obtain a reference to the original PMR.
+ *
+ * The caller receives, along with the PMR Export object, information
+ * about the size and contiguity guarantee for the PMR, and also the
+ * PMRs secret password, in order to authenticate the subsequent
+ * import.
+ *
+ * N.B.  If you call PMRExportPMR() (and it succeeds), you are
+ * promising to later call PMRUnexportPMR()
+ */
+PVRSRV_ERROR
+PMRExportPMR(PMR *psPMR,
+             PMR_EXPORT **ppsPMRExport,
+             PMR_SIZE_T *puiSize,
+             PMR_LOG2ALIGN_T *puiLog2Contig,
+             PMR_PASSWORD_T *puiPassword);
+
+/*!
+*******************************************************************************
+
+ @Function      PMRMakeLocalImportHandle
+
+ @Description
+
+ Transform a general handle type into one that we are able to import.
+ Takes a PMR reference.
+
+ @Input   psPMR     The input PMR.
+ @Output  ppsPMR    The output PMR that is going to be transformed to the
+                    correct handle type.
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PMRMakeLocalImportHandle(PMR *psPMR,
+                         PMR **ppsPMR);
+
+/*!
+*******************************************************************************
+
+ @Function      PMRUnmakeLocalImportHandle
+
+ @Description
+
+ Take a PMR, destroy the handle and release a reference.
+ Counterpart to PMRMakeServerExportClientExport().
+
+ @Input   psPMR       PMR to destroy.
+                      Created by PMRMakeLocalImportHandle().
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PMRUnmakeLocalImportHandle(PMR *psPMR);
+
+/*
+ * PMRUnexportPMR()
+ *
+ * The reverse of PMRExportPMR().  This causes the PMR to no longer be
+ * exported.  If the PMR has already been imported, the imported PMR
+ * reference will still be valid, but no further imports will be possible.
+ */
+PVRSRV_ERROR
+PMRUnexportPMR(PMR_EXPORT *psPMRExport);
+
+/*
+ * PMRImportPMR()
+ *
+ * Takes a PMR Export object, as obtained by PMRExportPMR(), and
+ * obtains a reference to the original PMR.
+ *
+ * The password must match, and is assumed to have been (by whatever
+ * means, IPC etc.) preserved intact from the former call to
+ * PMRExportPMR()
+ *
+ * The size and contiguity arguments are entirely irrelevant for the
+ * import, however they are verified in order to trap bugs.
+ *
+ * N.B.  If you call PhysmemPMRImport() (and it succeeds), you are
+ * promising to later call PhysmemPMRUnimport()
+ */
+PVRSRV_ERROR
+PMRImportPMR(PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR);
+
+/* Function that alters the mutability property
+ * of the PMR
+ * Setting it to TRUE makes sure the PMR memory layout
+ * can't be changed through future calls */
+void
+PMR_SetLayoutFixed(PMR *psPMR, IMG_BOOL bFlag);
+
+IMG_BOOL PMR_IsMemLayoutFixed(PMR *psPMR);
+
+/*
+ * PMRUnimportPMR()
+ *
+ * releases the reference on the PMR as obtained by PMRImportPMR()
+ */
+PVRSRV_ERROR
+PMRUnimportPMR(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRLocalImportPMR(PMR *psPMR,
+                  PMR **ppsPMR,
+                  IMG_DEVMEM_SIZE_T *puiSize,
+                  IMG_DEVMEM_ALIGN_T *puiAlign);
+
+/*
+ * Equivalent mapping functions when in kernel mode.
+ */
+PVRSRV_ERROR
+PMRAcquireKernelMappingData(PMR *psPMR,
+                            size_t uiLogicalOffset,
+                            size_t uiSize,
+                            void **ppvKernelAddressOut,
+                            size_t *puiLengthOut,
+                            IMG_HANDLE *phPrivOut);
+
+PVRSRV_ERROR
+PMRAcquireSparseKernelMappingData(PMR *psPMR,
+                                  size_t uiLogicalOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  size_t *puiLengthOut,
+                                  IMG_HANDLE *phPrivOut);
+
+PVRSRV_ERROR
+PMRReleaseKernelMappingData(PMR *psPMR,
+                            IMG_HANDLE hPriv);
+
+/*
+ * PMR_ReadBytes()
+ *
+ * calls into the PMR implementation to read up to uiBufSz bytes,
+ * returning the actual number read in *puiNumBytes
+ *
+ * this will read up to the end of the PMR, or the next symbolic name
+ * boundary, or until the requested number of bytes is read, whichever
+ * comes first
+ *
+ * In the case of sparse PMR's the caller doesn't know what offsets are
+ * valid and which ones aren't so we will just write 0 to invalid offsets
+ */
+PVRSRV_ERROR
+PMR_ReadBytes(PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT8 *pcBuffer,
+              size_t uiBufSz,
+              size_t *puiNumBytes);
+
+/*
+ * PMR_WriteBytes()
+ *
+ * calls into the PMR implementation to write up to uiBufSz bytes,
+ * returning the actual number read in *puiNumBytes
+ *
+ * this will write up to the end of the PMR, or the next symbolic name
+ * boundary, or until the requested number of bytes is written, whichever
+ * comes first
+ *
+ * In the case of sparse PMR's the caller doesn't know what offsets are
+ * valid and which ones aren't so we will just ignore data at invalid offsets
+ */
+PVRSRV_ERROR
+PMR_WriteBytes(PMR *psPMR,
+               IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+               IMG_UINT8 *pcBuffer,
+               size_t uiBufSz,
+               size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Function       PMRMMapPMR
+@Description    Performs the necessary steps to map the PMR into a user process
+                address space. The caller does not need to call
+                PMRLockSysPhysAddresses before calling this function.
+
+@Input          psPMR            PMR to map.
+
+@Input          pOSMMapData      OS specific data needed to create a mapping.
+
+@Input          uiCpuAccessFlags Flags to indicate if the mapping request
+                                 requires read, write or both access.
+
+@Return         PVRSRV_ERROR:   PVRSRV_OK on success or an error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PMRMMapPMR(PMR *psPMR,
+           PMR_MMAP_DATA pOSMMapData,
+           PVRSRV_MEMALLOCFLAGS_T uiCpuAccessFlags);
+
+/*
+ * PMRRefPMR()
+ *
+ * Take a reference on the passed in PMR
+ */
+PVRSRV_ERROR
+PMRRefPMR(PMR *psPMR);
+
+/*
+ * PMRUnrefPMR()
+ *
+ * This undoes a call to any of the PhysmemNew* family of APIs
+ * (i.e. any PMR factory "constructor")
+ *
+ * This relinquishes a reference to the PMR, and, where the refcount
+ * reaches 0, causes the PMR to be destroyed (calling the finalizer
+ * callback on the PMR, if there is one)
+ */
+PVRSRV_ERROR
+PMRUnrefPMR(PMR *psPMR);
+
+/*
+ * PMRRefPMRN()
+ *
+ * Take a reference N-times on the passed in PMR.
+ *
+ * This function does not perform address locking as opposed to PMRRefPMR().
+ */
+PVRSRV_ERROR
+PMRRefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount);
+
+/*
+ * PMRUnrefPMRN()
+ *
+ * This undoes a call to any of the PhysmemNew* family of APIs
+ * (i.e. any PMR factory "constructor").
+ *
+ * This relinquishes N references to the PMR, and, where the refcount
+ * reaches 0, causes the PMR to be destroyed (calling the finalizer
+ * callback on the PMR, if there is one)
+ */
+PVRSRV_ERROR
+PMRUnrefPMRN(PMR *psPMR, IMG_UINT32 uiRefCount);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+/*
+ * PMRTryRefPMR()
+ *
+ * This attempts to take a reference on the PMR but only succeeds if
+ * the PMR is not at refcount 0. Other Ref functions would class this
+ * attempt as a logical error. This function is free to attempt and return
+ * an error if PMR is in a free in progress state.
+ */
+PVRSRV_ERROR
+PMRTryRefPMR(PMR *psPMR);
+
+/*
+ * PMRKernelCpuMapCountIncr()
+ *
+ * Increment count of the number of current kernel CPU mappings of the PMR.
+ */
+void
+PMRKernelCpuMapCountIncr(PMR *psPMR);
+
+/*
+ * PMRKernelCpuMapCountDecr()
+ *
+ * Decrement count of the number of current kernel CPU mappings of the PMR.
+ */
+void
+PMRKernelCpuMapCountDecr(PMR *psPMR);
+
+IMG_BOOL
+PMR_IsKernelCpuMapped(PMR *psPMR);
+#endif /* #if defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+/*
+ * PMRClientCpuMapCountIncr()
+ *
+ * Increment count of the number of current client CPU mappings of the PMR.
+ */
+void
+PMRClientCpuMapCountIncr(PMR *psPMR);
+
+/*
+ * PMRClientCpuMapCountDecr()
+ *
+ * Decrement count of the number of current client CPU mappings of the PMR.
+ */
+void
+PMRClientCpuMapCountDecr(PMR *psPMR);
+
+IMG_BOOL
+PMR_IsClientCpuMapped(PMR *psPMR);
+
+
+/*
+ * PMRLinkGPUMapping()
+ *
+ * Link a GPU mapping with a PMR creating an association.
+ * Must be protected by PMR lock.
+ */
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRLinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode);
+#else
+void
+PMRLinkGPUMapping(PMR *psPMR);
+#endif
+
+/*
+ * PMRUnlinkGPUMapping()
+ *
+ * Unlink a GPU mapping with a PMR destroying an association.
+ * Must be protected by PMR lock.
+ */
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+void
+PMRUnlinkGPUMapping(PMR *psPMR, DLLIST_NODE *psMappingNode);
+#else
+void
+PMRUnlinkGPUMapping(PMR *psPMR);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+/*
+ * PMRNotifyMigrateInProgress()
+ *
+ * Used to notify the PMR that the pages backing the PMR are
+ * in the process of migration.
+ */
+void
+PMRNotifyMigrateInProgress(PMR *psPMR);
+
+/*
+ * PMRNotifyMigrateComplete()
+ *
+ * Used to notify the PMR that migration of backing
+ * pages has completed.
+ */
+void
+PMRNotifyMigrateComplete(PMR *psPMR);
+
+/*
+ * PMRRemapGPUPMR()
+ *
+ * Trigger Remap of a PMR page with each associated mapping.
+ * This is called for UMA page migration and will overwrite
+ * an existing page mapping with the new one at logical
+ * pg offset in the PMR.
+ * PVRSRV_ERROR_DEVICEMEM_REJECT_REMAP_REQUEST can be returned
+ * if remap is not possible for the given page offset.
+ */
+PVRSRV_ERROR
+PMRRemapGPUPMR(PMR *psPMR, IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+/*
+ * PMR_IsGpuMultiMapped()
+ *
+ * Must be protected by PMR lock.
+ *
+ */
+IMG_BOOL
+PMR_IsGpuMultiMapped(PMR *psPMR);
+
+PPVRSRV_DEVICE_NODE
+PMR_DeviceNode(const PMR *psPMR);
+
+/*
+ * PMR_Flags()
+ *
+ * Flags are static and guaranteed for the life of the PMR.  Thus this
+ * function is idempotent and acquire/release semantics is not required.
+ *
+ * Returns the flags as specified on the PMR.  The flags are to be
+ * interpreted as mapping permissions
+ */
+PMR_FLAGS_T
+PMR_Flags(const PMR *psPMR);
+
+IMG_BOOL
+PMR_IsSparse(const PMR *psPMR);
+
+IMG_DEVMEM_SIZE_T
+PMR_PhysicalSize(const PMR *psPMR);
+
+/*
+ * PMR_IsOffsetValid()
+ *
+ * Returns if an address offset inside a PMR has a valid
+ * physical backing.
+ */
+PVRSRV_ERROR
+PMR_IsOffsetValid(const PMR *psPMR,
+				IMG_UINT32 ui32Log2PageSize,
+				IMG_UINT32 ui32NumOfPages,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_BOOL *pbValid);
+
+PHYS_HEAP *
+PMR_PhysHeap(const PMR *psPMR);
+
+PMR_MAPPING_TABLE *
+PMR_GetMappingTable(const PMR *psPMR);
+
+IMG_UINT32
+PMR_GetLog2Contiguity(const PMR *psPMR);
+
+IMG_DEVMEM_SIZE_T
+PMR_LogicalSize(const PMR *psPMR);
+
+/*
+ * PMR_GetLogicalChunkCount
+ *
+ * Retrieve the maximum number of chunks supported by the PMR.
+ * This property is fixed at creation time.
+ */
+IMG_UINT32 PMR_GetLogicalChunkCount(const PMR *psPMR);
+
+const IMG_CHAR *
+PMR_GetAnnotation(const PMR *psPMR);
+
+PMR_IMPL_TYPE
+PMR_GetType(const PMR *psPMR);
+
+IMG_CHAR *
+PMR_GetTypeStr(const PMR *psPMR);
+
+IMG_INT32
+PMR_GetRefCount(const PMR *psPMR);
+
+/* PMR usage type for callers of PMR_DevPhysAddr() */
+typedef IMG_UINT32 PMR_PHYSADDRMODE_TYPE;
+
+#define CPU_USE BIT(0) /* CPU use, disable IPA policy */
+#define DEVICE_USE BIT(1) /* Device use, enable IPA policy */
+#define MAPPING_USE BIT(2) /* Mapping use, dev phys addrs obtained in a mapping path */
+
+/*
+ * PMR_DevPhysAddr()
+ *
+ * A note regarding Lock/Unlock semantics
+ * ======================================
+ *
+ * PMR_DevPhysAddr may only be called after PMRLockSysPhysAddresses()
+ * has been called.  The data returned may be used only until
+ * PMRUnlockSysPhysAddresses() is called after which time the licence
+ * to use the data is revoked and the information may be invalid.
+ *
+ * Given an offset, this function returns the device physical address of the
+ * corresponding page in the PMR.  It may be called multiple times
+ * until the address of all relevant pages has been determined.
+ *
+ * If caller only wants one physical address it is sufficient to pass in:
+ * ui32Log2PageSize==0 and ui32NumOfPages==1
+ *
+ * Returns PVRSRV_OK if successful.
+ * PVRSRV_ERROR_RETRY if SUPPORT_LINUX_OSPAGE_MIGRATION is
+ * enabled and migrate is in progress. If we support page migration this
+ * call may return PVRSRV_ERROR_RETRY back to the requester. This is
+ * because the function is used to create device mappings and these mappings
+ * need to be synchronised across both reservations and PMRs. To do this we
+ * must take both locks in any mapping path. In order for migrate to complete,
+ * a UM requested Device mapping must temporarily back off the locks.
+ * Retry signifies this should happen.
+ */
+PVRSRV_ERROR
+PMR_DevPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEV_PHYADDR *psDevAddr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage);
+
+/*
+ * PMR_CpuPhysAddr()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * Given an offset, this function returns the CPU physical address of the
+ * corresponding page in the PMR.  It may be called multiple times
+ * until the address of all relevant pages has been determined.
+ *
+ */
+PVRSRV_ERROR
+PMR_CpuPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_CPU_PHYADDR *psCpuAddrPtr,
+                IMG_BOOL *pbValid,
+                PMR_PHYSADDRMODE_TYPE uiPMRUsage);
+
+/* PMRGetUID()
+ *
+ * Used for bridge calls that expect a PVRSRV_ERROR returned
+ * */
+PVRSRV_ERROR
+PMRGetUID(PMR *psPMR,
+          IMG_UINT64 *pui64UID);
+
+IMG_UINT64
+PMRInternalGetUID(PMR *psPMR);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*
+ * PMR_IsZombie()
+ *
+ * Indicates if a PMR is a "zombie" PMR. This function **must** be called
+ * inside a PMR factory lock.
+ */
+IMG_BOOL
+PMR_IsZombie(const PMR *psPMR);
+
+/*
+ * PMRMarkForDeferFree
+ *
+ * Sets sync value required for this PMR to be freed.
+ */
+void
+PMRMarkForDeferFree(PMR *psPMR);
+
+/*
+ * PMRQueueZombiesForCleanup
+ *
+ * Defers cleanup of all zombie PMRs to the CleanupThread.
+ *
+ * Returns IMG_TRUE if any PMRs were queued for free and IMG_FALSE if no PMRs
+ * were queued.
+ */
+IMG_BOOL
+PMRQueueZombiesForCleanup(PPVRSRV_DEVICE_NODE psDevNode);
+
+/*
+ * PMRReviveZombieAndRef
+ *
+ * Removed the PMR either form zombie list or cleanup item's list
+ * and references it.
+ */
+void
+PMRReviveZombieAndRef(PMR *psPMR);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/*
+ * PMR_ChangeSparseMemUnlocked()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * This function alters the memory map of the given PMR in device space by
+ * adding/deleting the pages as requested. PMR lock must be taken
+ * before calling this function.
+ *
+ */
+PVRSRV_ERROR PMR_ChangeSparseMemUnlocked(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags);
+/*
+ * PMR_ChangeSparseMem()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * This function alters the memory map of the given PMR in device space by
+ *  adding/deleting the pages as requested.
+ *
+ */
+PVRSRV_ERROR PMR_ChangeSparseMem(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32	uiSparseFlags);
+
+#if defined(PDUMP)
+
+/*
+ * PMR_PDumpSymbolicAddr()
+ *
+ * Given an offset, returns the pdump memspace name and symbolic
+ * address of the corresponding page in the PMR.
+ *
+ * Note that PDump memspace names and symbolic addresses are static
+ * and valid for the lifetime of the PMR, therefore we don't require
+ * acquire/release semantics here.
+ *
+ * Note that it is expected that the pdump "mapping" code will call
+ * this function multiple times as each page is mapped in turn
+ *
+ * Note that NextSymName is the offset from the base of the PMR to the
+ * next pdump symbolic address (or the end of the PMR if the PMR only
+ * had one PDUMPMALLOC
+ */
+PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32NamespaceNameLen,
+                      IMG_CHAR *pszNamespaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+		      IMG_DEVMEM_OFFSET_T *puiNextSymName
+                      );
+
+/*
+ * PMRPDumpLoadMemValue32()
+ *
+ * writes the current contents of a dword in PMR memory to the pdump
+ * script stream. Useful for patching a buffer by simply editing the
+ * script output file in ASCII plain text.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT32 ui32Value,
+                     PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpCopyMem32
+ *
+ * Adds in the pdump script stream a copy of a dword in one PMR memory
+ * location to another PMR memory location.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem32(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpLoadMemValue64()
+ *
+ * writes the current contents of a dword in PMR memory to the pdump
+ * script stream. Useful for patching a buffer by simply editing the
+ * script output file in ASCII plain text.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT64 ui64Value,
+                     PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpCopyMem64
+ *
+ * Adds in the pdump script stream a copy of a quadword in one PMR memory
+ * location to another PMR memory location.
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem64(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpLoadMem()
+ *
+ * Writes the current contents of the PMR memory to the pdump PRM stream,
+ * and emits some PDump code to the script stream to LDB said bytes from
+ * said file. If bZero is IMG_TRUE then the PDump zero page is used as the
+ * source for the LDB.
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero);
+
+/*
+ * PMRPDumpSaveToFile()
+ *
+ * Emits some PDump that does an SAB (save bytes) using the PDump symbolic
+ * address of the PMR. Note that this is generally not the preferred way to
+ * dump the buffer contents. There is an equivalent function in
+ * devicemem_server.h which also emits SAB but using the virtual address,
+ * which is the "right" way to dump the buffer contents to a file.
+ * This function exists just to aid testing by providing a means to dump
+ * the PMR directly by symbolic address also.
+ */
+PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMR_PDumpSymbolicAddr)
+#endif
+static INLINE PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32NamespaceNameLen,
+                      IMG_CHAR *pszNamespaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                      IMG_DEVMEM_OFFSET_T *puiNextSymName)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32NamespaceNameLen);
+	PVR_UNREFERENCED_PARAMETER(pszNamespaceName);
+	PVR_UNREFERENCED_PARAMETER(ui32SymbolicAddrLen);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddr);
+	PVR_UNREFERENCED_PARAMETER(puiNewOffset);
+	PVR_UNREFERENCED_PARAMETER(puiNextSymName);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMemValue32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT32 ui32Value,
+                     PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMemValue64)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT64 ui64Value,
+                     PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui64Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMem)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(bZero);
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpSaveToFile)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(uiFileOffset);
+	return PVRSRV_OK;
+}
+
+#endif	/* PDUMP */
+
+/* This function returns the private data that a pmr subtype embedded in
+ * here. We use the function table pointer as "authorisation" that this
+ * function is being called by the pmr subtype implementation. We can
+ * assume (assert) that. It would be a bug in the implementation of the
+ * pmr subtype if this assertion ever fails.
+ */
+void *
+PMRGetPrivateData(const PMR *psPMR,
+                  const PMR_IMPL_FUNCTAB *psFuncTab);
+
+PVRSRV_ERROR
+PMRZeroingPMR(PMR *psPMR,
+				IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize);
+
+PVRSRV_ERROR
+PMRDumpPageList(PMR *psReferencePMR,
+					IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize);
+
+PVRSRV_ERROR
+PMRWritePMPageList(/* Target PMR, offset, and length */
+                   PMR *psPageListPMR,
+                   IMG_DEVMEM_OFFSET_T uiTableOffset,
+                   IMG_DEVMEM_SIZE_T  uiTableLength,
+                   /* Referenced PMR, and "page" granularity */
+                   PMR *psReferencePMR,
+                   IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+                   PMR_PAGELIST **ppsPageList);
+
+/* Doesn't actually erase the page list - just releases
+ * the appropriate refcounts
+ */
+PVRSRV_ERROR // should be void, surely
+PMRUnwritePMPageList(PMR_PAGELIST *psPageList);
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiFlags);
+
+PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize);
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpPol32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpCheck32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpCBP)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif
+
+PPVRSRV_DEVICE_NODE PMRGetExportDeviceNode(PMR_EXPORT *psExportPMR);
+
+/*
+ * PMRInit()
+ *
+ * To be called once and only once to initialise the internal data in
+ * the PMR module (mutexes and such)
+ *
+ * Not for general use.  Only PVRSRVInit(); should be calling this.
+ */
+PVRSRV_ERROR
+PMRInit(void);
+
+/*
+ * PMRDeInit()
+ *
+ * To be called once and only once to deinitialise the internal data in
+ * the PMR module (mutexes and such) and for debug checks
+ *
+ * Not for general use.  Only PVRSRVDeInit(); should be calling this.
+ */
+PVRSRV_ERROR
+PMRDeInit(void);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*
+ * PMRInitDevice()
+ *
+ * Initialised device specific PMR data.
+ */
+PVRSRV_ERROR
+PMRInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRFreeZombies()
+ *
+ * Free deferred PMRs.
+ */
+void
+PMRFreeZombies(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRFreeZombies()
+ *
+ * Print all zombies to the log.
+ */
+void
+PMRDumpZombies(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRDeInitDevice()
+ *
+ * Cleans up device specific PMR data.
+ */
+void
+PMRDeInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR
+PMRStoreRIHandle(PMR *psPMR, void *hRIHandle);
+#endif
+
+#if defined(DEBUG)
+void PMRLockHeldAssert(const PMR *psPMR);
+#else
+#define PMRLockHeldAssert(x)
+#endif
+
+/*
+ * PMRLockPMR()
+ *
+ * To be called when the PMR must not be modified by any other call-stack.
+ * Acquires the mutex on the passed in PMR.
+ */
+void
+PMRLockPMR(const PMR *psPMR);
+
+/*
+ * PMRUnlockPMR()
+ *
+ * To be called when the PMR is no longer being modified.
+ * Releases the per-PMR mutex.
+ */
+void
+PMRUnlockPMR(const PMR *psPMR);
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRModifyIPAPolicy(PMR *psPMR, IMG_UINT8 ui8NewIPAPolicy);
+
+PVRSRV_ERROR
+PMRGetIPAPolicy(PMR *psPMR, IMG_UINT8 *pui8IPAPolicy);
+
+PVRSRV_ERROR
+PMRGetIPAInfo(PMR *psPMR, IMG_UINT32 *pui32IPAPolicy, IMG_UINT32 *pui32IPAShift,
+              IMG_UINT32 *pui32IPAMask, IMG_UINT32 *pui32IPAFlagsValue);
+#endif
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+/*
+ * PMR_ImportedDevicesMask()
+ *
+ * Returns a bitmask of devices this PMR is imported to.
+ * Explicitly does NOT include the PMR_DeviceNode's device.
+ * For each device registered with the PMR: 1 << psDevNode->sDevId.ui32InternalID
+ */
+IMG_UINT64
+PMR_ImportedDevicesMask(const PMR* psPMR);
+static_assert((sizeof(IMG_UINT64) * 8) >= PVRSRV_MAX_DEVICES, "PMR_ImportedDevicesMask() needs to be updated");
+#endif /* defined(PVRSRV_ENABLE_XD_MEM) */
+
+#if defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM)
+/*
+ *
+ * PMR_RegisterDeviceImport()
+ *
+ * Register the PMR with the device node.
+ * This is required for the PMR to be marked as a XD PMR.
+ * Silently handles if the PMR is already registered with the device.
+ */
+PVRSRV_ERROR
+PMR_RegisterDeviceImport(PMR* psPMR, PPVRSRV_DEVICE_NODE psDevNode);
+#endif /* defined(SUPPORT_PMR_DEVICE_IMPORT_DEFERRED_FREE) || defined(PVRSRV_ENABLE_XD_MEM) */
+
+#endif /* #ifdef SRVSRV_PMR_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_impl.h
new file mode 100644
index 000000000000..3c290ca3a6e7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_impl.h
@@ -0,0 +1,605 @@
+/**************************************************************************/ /*!
+@File
+@Title          Implementation Callbacks for Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This file is for definitions
+                that are private to the world of PMRs, but that need to be
+                shared between pmr.c itself and the modules that implement the
+                callbacks for the PMR.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PMR_IMPL_H
+#define SRVSRV_PMR_IMPL_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+/*! Physical Memory Resource type.
+ */
+typedef struct _PMR_ PMR;
+
+/*! Per-flavour callbacks need to be shared with generic implementation
+ * (pmr.c).
+ */
+typedef void *PMR_IMPL_PRIVDATA;
+
+/*! Type for holding flags passed to the PMR factory.
+ */
+typedef PVRSRV_MEMALLOCFLAGS_T PMR_FLAGS_T;
+
+/*! Mapping table for the allocation.
+ *
+ * PMR's can be sparse in which case not all the logical addresses in it are
+ * valid. The mapping table translates logical offsets into physical offsets.
+ *
+ * This table is always passed to the PMR factory regardless if the memory is
+ * sparse or not. In case of non-sparse memory all virtual offsets are mapped
+ * to physical offsets.
+ */
+typedef struct _PMR_MAPPING_TABLE_ PMR_MAPPING_TABLE;
+
+/*! Private data passed to the ::PFN_MMAP_FN function.
+ */
+typedef void *PMR_MMAP_DATA;
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+typedef void *PMR_IMPL_ZOMBIEPAGES;
+#endif
+
+#define PMR_IMPL_TYPES \
+	X(NONE), \
+	X(OSMEM), \
+	X(LMA), \
+	X(DMABUF), \
+	X(EXTMEM), \
+	X(DC), \
+	X(TDFWMEM), \
+	X(TDSECBUF), \
+	X(LAST)
+
+/*! PMR factory type.
+ */
+typedef enum _PMR_IMPL_TYPE_
+{
+#define X(type) PMR_TYPE_##type
+	PMR_IMPL_TYPES
+#undef X
+} PMR_IMPL_TYPE;
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_LOCK_PHYS_ADDRESSES_FN
+
+@Description    Called to lock down the physical addresses for all pages
+                allocated for a PMR.
+                The default implementation is to simply increment a
+                lock-count for debugging purposes.
+                If overridden, the PFN_LOCK_PHYS_ADDRESSES_FN function will
+                be called when someone first requires a physical address,
+                and the PFN_UNLOCK_PHYS_ADDRESSES_FN counterpart will be
+                called when the last such reference is released.
+                The PMR implementation may assume that physical addresses
+                will have been "locked" in this manner before any call is
+                made to the pfnDevPhysAddr() callback
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_LOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_UNLOCK_PHYS_ADDRESSES_FN
+
+@Description    Called to release the lock taken on the physical addresses
+                for all pages allocated for a PMR.
+                The default implementation is to simply decrement a
+                lock-count for debugging purposes.
+                If overridden, the PFN_UNLOCK_PHYS_ADDRESSES_FN will be
+                called when the last reference taken on the PMR is
+                released.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Output         ppvZombiePages        Zombie pages object. If non-null is returned
+                                      caller is obligated to call pfnFreeZombiePages
+                                      at an appropriate time to prevent memory leaks.
+                                      If support for deferred freeing of pages is not
+                                      provided, the implementation must set
+                                      *ppvZombiePages to NULL.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+typedef PVRSRV_ERROR (*PFN_UNLOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                                                     PMR_IMPL_ZOMBIEPAGES *ppvZombiePages);
+#else
+typedef PVRSRV_ERROR (*PFN_UNLOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv);
+#endif
+
+#if defined(SUPPORT_STATIC_IPA)
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_DEV_PHYS_ADDR_FN
+
+@Description    Called to obtain one or more physical addresses for given
+                offsets within a PMR.
+
+                The PFN_LOCK_PHYS_ADDRESSES_FN callback (if overridden) is
+                guaranteed to have been called prior to calling the
+                PFN_DEV_PHYS_ADDR_FN callback and the caller promises not to
+                rely on the physical address thus obtained after the
+                PFN_UNLOCK_PHYS_ADDRESSES_FN callback is called.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          ui32Log2PageSize      The log2 page size.
+@Input          ui32NumOfAddr         The number of addresses to be returned
+@Input          puiOffset             The offset from the start of the PMR
+                                      (in bytes) for which the physical
+                                      address is required. Where multiple
+                                      addresses are requested, this will
+                                      contain a list of offsets.
+@Input          ui64IPAPolicyValue    The Intermediate Physical Address (IPA)
+                                      Policy value to be applied to
+                                      the physical address
+@Input          ui64IPAClearMask      The IPA Clear mask to be applied to
+                                      the physical address when setting policy.
+@Output         pbValid               List of boolean flags indicating which
+                                      addresses in the returned list
+                                      (psDevAddrPtr) are valid (for sparse
+                                      allocations, not all pages may have a
+                                      physical backing)
+@Output         psDevAddrPtr          Returned list of physical addresses
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#else
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_DEV_PHYS_ADDR_FN
+
+@Description    Called to obtain one or more physical addresses for given
+                offsets within a PMR.
+
+                The PFN_LOCK_PHYS_ADDRESSES_FN callback (if overridden) is
+                guaranteed to have been called prior to calling the
+                PFN_DEV_PHYS_ADDR_FN callback and the caller promises not to
+                rely on the physical address thus obtained after the
+                PFN_UNLOCK_PHYS_ADDRESSES_FN callback is called.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          ui32Log2PageSize      The log2 page size.
+@Input          ui32NumOfAddr         The number of addresses to be returned
+@Input          puiOffset             The offset from the start of the PMR
+                                      (in bytes) for which the physical
+                                      address is required. Where multiple
+                                      addresses are requested, this will
+                                      contain a list of offsets.
+@Output         pbValid               List of boolean flags indicating which
+                                      addresses in the returned list
+                                      (psDevAddrPtr) are valid (for sparse
+                                      allocations, not all pages may have a
+                                      physical backing)
+@Output         psDevAddrPtr          Returned list of physical addresses
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#endif
+typedef PVRSRV_ERROR (*PFN_DEV_PHYS_ADDR_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_UINT32 ui32Log2PageSize,
+                      IMG_UINT32 ui32NumOfAddr,
+                      IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+                      IMG_UINT64 ui64IPAPolicyValue,
+                      IMG_UINT64 ui64IPAClearMask,
+#endif
+                      IMG_BOOL *pbValid,
+                      IMG_DEV_PHYADDR *psDevAddrPtr);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN
+
+@Description    Called to obtain a kernel-accessible address (mapped to a
+                virtual address if required) for the PMR for use internally
+                in Services.
+
+    Implementation of this function for the (default) PMR factory providing
+    OS-allocations is mandatory (the driver will expect to be able to call
+    this function for OS-provided allocations).
+    For other PMR factories, implementation of this function is only necessary
+    where an MMU mapping is required for the Kernel to be able to access the
+    allocated memory.
+    If no mapping is needed, this function can remain unimplemented and the
+    pfn may be set to NULL.
+@Input          pvPriv                Private data (which was generated by
+                                      the PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR
+                                      at which mapping is to start
+@Input          uiSize                Size of mapping (in bytes)
+@Output         ppvKernelAddressOut   Mapped kernel address
+@Output         phHandleOut           Returned handle of the new mapping
+@Input          ulFlags               Mapping flags
+
+@Return         PVRSRV_OK if the mapping was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      size_t uiOffset,
+                      size_t uiSize,
+                      void **ppvKernelAddressOut,
+                      IMG_HANDLE *phHandleOut,
+                      PMR_FLAGS_T ulFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_RELEASE_KERNEL_MAPPING_DATA_FN
+
+@Description    Called to release a mapped kernel virtual address
+
+   Implementation of this callback is mandatory if
+   PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN is provided for the PMR factory,
+   otherwise this function can remain unimplemented and the pfn may be set
+   to NULL.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          hHandle               Handle of the mapping to be released
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_RELEASE_KERNEL_MAPPING_DATA_FN)(PMR_IMPL_PRIVDATA pvPriv,
+              IMG_HANDLE hHandle);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_READ_BYTES_FN
+
+@Description    Called to read bytes from an unmapped allocation
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will use PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN to map the entire
+   PMR (if an MMU mapping is required for the Kernel to be able to access the
+   allocated memory).
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR at
+                                      which to begin reading
+@Output         pcBuffer              Buffer in which to return the read data
+@Input          uiBufSz               Number of bytes to be read
+@Output         puiNumBytes           Number of bytes actually read (may be
+                                      less than uiBufSz)
+
+@Return         PVRSRV_OK if the read was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_READ_BYTES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_UINT8 *pcBuffer,
+                      size_t uiBufSz,
+                      size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_WRITE_BYTES_FN
+
+@Description    Called to write bytes into an unmapped allocation
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will use PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN to map the entire
+   PMR (if an MMU mapping is required for the Kernel to be able to access the
+   allocated memory).
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR at
+                                      which to begin writing
+@Input          pcBuffer              Buffer containing the data to be written
+@Input          uiBufSz               Number of bytes to be written
+@Output         puiNumBytes           Number of bytes actually written (may be
+                                      less than uiBufSz)
+
+@Return         PVRSRV_OK if the write was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_WRITE_BYTES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_UINT8 *pcBuffer,
+                      size_t uiBufSz,
+                      size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_CHANGE_SPARSE_MEM_FN
+
+@Description    Called to modify the physical backing for a given sparse
+                allocation.
+                The caller provides a list of the pages within the sparse
+                allocation which should be backed with a physical allocation
+                and a list of the pages which do not require backing.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the sparse allocation to be
+                                      modified
+@Input          ui32AllocPageCount    The number of pages specified in
+                                      pai32AllocIndices
+@Input          pai32AllocIndices     The list of pages in the sparse
+                                      allocation that should be backed with a
+                                      physical allocation. Pages are
+                                      referenced by their index within the
+                                      sparse allocation (e.g. in a 10 page
+                                      allocation, pages are denoted by
+                                      indices 0 to 9)
+@Input          ui32FreePageCount     The number of pages specified in
+                                      pai32FreeIndices
+@Input          pai32FreeIndices      The list of pages in the sparse
+                                      allocation that do not require
+                                      a physical allocation.
+@Input          ui32Flags             Allocation flags
+@Output         ppvZombiePages        Zombie pages object. If non-null is returned
+                                      caller is obligated to call pfnFreeZombiePages
+                                      at an appropriate time to prevent memory leaks
+
+@Return         PVRSRV_OK if the sparse allocation physical backing was updated
+                successfully, an error code otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CHANGE_SPARSE_MEM_FN)(PMR_IMPL_PRIVDATA pPriv,
+                      const PMR *psPMR,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+                      PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+                      IMG_UINT32 uiFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_CHANGE_SPARSE_MEM_CPU_MAP_FN
+
+@Description    Called to modify which pages are mapped for a given sparse
+                allocation.
+                The caller provides a list of the pages within the sparse
+                allocation which should be given a CPU mapping and a list
+                of the pages which do not require a CPU mapping.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the sparse allocation to be
+                                      modified
+@Input          sCpuVAddrBase         The virtual base address of the sparse
+                                      allocation
+@Input          ui32AllocPageCount    The number of pages specified in
+                                      pai32AllocIndices
+@Input          pai32AllocIndices     The list of pages in the sparse
+                                      allocation that should be given a CPU
+                                      mapping. Pages are referenced by their
+                                      index within the sparse allocation (e.g.
+                                      in a 10 page allocation, pages are
+                                      denoted by indices 0 to 9)
+@Input          ui32FreePageCount     The number of pages specified in
+                                      pai32FreeIndices
+@Input          pai32FreeIndices      The list of pages in the sparse
+                                      allocation that do not require a CPU
+                                      mapping.
+
+@Return         PVRSRV_OK if the page mappings were updated successfully, an
+                error code otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CHANGE_SPARSE_MEM_CPU_MAP_FN)(PMR_IMPL_PRIVDATA pPriv,
+                      const PMR *psPMR,
+                      IMG_UINT64 sCpuVAddrBase,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_MMAP_FN
+
+@Description    Called to map pages in the specified PMR.
+
+   Implementation of this callback is optional.
+   Where it is provided, it will be used in place of OSMMapPMRGeneric().
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the allocation to be mapped
+@Input          pMMapData             OS-specific data to describe how mapping
+                                      should be performed
+
+@Return         PVRSRV_OK if the mapping was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_MMAP_FN)(PMR_IMPL_PRIVDATA pPriv,
+                                    PMR *psPMR,
+                                    PMR_MMAP_DATA pMMapData);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_FINALIZE_FN
+
+@Description    Called to destroy the PMR.
+                This callback will be called only when all references to
+                the PMR have been dropped.
+                The PMR was created via a call to PhysmemNewRamBackedPMR()
+                and is destroyed via this callback.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_FINALIZE_FN)(PMR_IMPL_PRIVDATA pvPriv);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ACQUIRE_PMR_FACTORY_LOCK_FN
+
+@Description    Called to acquire the PMR factory's global lock, if it has one,
+                hence callback optional. Factories which support entry points
+                in addition to the normal bridge calls, for example, from the
+                native OS that manipulate the PMR reference count should
+                create a factory lock and implementations for these call backs.
+
+   Implementation of this callback is optional.
+
+@Return         None
+*/
+/*****************************************************************************/
+typedef void (*PFN_ACQUIRE_PMR_FACTORY_LOCK_FN)(void);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_RELEASE_PMR_FACTORY_LOCK_FN
+
+@Description    Called to release the PMR factory's global lock acquired by calling
+                pfn_acquire_pmr_factory_lock callback.
+
+   Implementation of this callback is optional.
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_RELEASE_PMR_FACTORY_LOCK_FN)(void);
+
+#ifdef SUPPORT_PMR_DEFERRED_FREE
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ZOMBIFY_FN
+
+@Description    Called to perform factory actions necessary when PMR becomes
+                a zombie PMR.
+
+                This function should at least adjust the driver/process memory
+                stats to reflect the amount of memory is occupied by the zombie
+                PMRs and at the same time subtract the memory from the main
+                memory stat the pages are accounted under.
+
+   Implementation of this callback is required when SUPPORT_PMR_DEFERRED_FREE=1.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_ZOMBIFY_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                                       PMR *psPMR);
+#endif
+
+#ifdef SUPPORT_PMR_PAGES_DEFERRED_FREE
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_FREE_ZOMBIE_PAGES_FN
+
+@Description    Called to perform factory actions to free zombie pages object
+                previously returned by PFN_CHANGE_SPARSE_MEM_FN.
+
+                This function should free the pages described in the
+                pvZombiePages parameter and do any associated actions related
+                to freeing such as poisoning or returning to the page pool.
+
+                Implementation of this callback is required when
+                SUPPORT_PMR_PAGES_DEFERRED_FREE=1.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise. If error is returned, the PMR layer might retry.
+                On error, factory implementations should modify the contents
+                of the PMR_IMPL_ZOMBIEPAGES object reflecting any changes in
+                underlying memory as a result of the initial (failed) call.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_FREE_ZOMBIE_PAGES_FN)(PMR_IMPL_ZOMBIEPAGES pvZombiePages);
+#endif
+
+/*! PMR factory callback table.
+ */
+struct _PMR_IMPL_FUNCTAB_ {
+    /*! Callback function pointer, see ::PFN_LOCK_PHYS_ADDRESSES_FN */
+    PFN_LOCK_PHYS_ADDRESSES_FN pfnLockPhysAddresses;
+    /*! Callback function pointer, see ::PFN_UNLOCK_PHYS_ADDRESSES_FN */
+    PFN_UNLOCK_PHYS_ADDRESSES_FN pfnUnlockPhysAddresses;
+
+    /*! Callback function pointer, see ::PFN_DEV_PHYS_ADDR_FN */
+    PFN_DEV_PHYS_ADDR_FN pfnDevPhysAddr;
+
+    /*! Callback function pointer, see ::PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN */
+    PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN pfnAcquireKernelMappingData;
+    /*! Callback function pointer, see ::PFN_RELEASE_KERNEL_MAPPING_DATA_FN */
+    PFN_RELEASE_KERNEL_MAPPING_DATA_FN pfnReleaseKernelMappingData;
+
+    /*! Callback function pointer, see ::PFN_READ_BYTES_FN */
+    PFN_READ_BYTES_FN pfnReadBytes;
+    /*! Callback function pointer, see ::PFN_WRITE_BYTES_FN */
+    PFN_WRITE_BYTES_FN pfnWriteBytes;
+
+    /*! Callback function pointer, see ::PFN_CHANGE_SPARSE_MEM_FN */
+    PFN_CHANGE_SPARSE_MEM_FN pfnChangeSparseMem;
+
+#ifdef SUPPORT_PMR_PAGES_DEFERRED_FREE
+    /*! Callback function pointer, see ::PFN_FREE_ZOMBIE_PAGES_FN */
+    PFN_FREE_ZOMBIE_PAGES_FN pfnFreeZombiePages;
+#endif
+
+    /*! Callback function pointer, see ::PFN_MMAP_FN */
+    PFN_MMAP_FN pfnMMap;
+
+    /*! Callback function pointer, see ::PFN_FINALIZE_FN */
+    PFN_FINALIZE_FN pfnFinalize;
+
+    /*! Callback function pointer, see ::PFN_ACQUIRE_PMR_FACTORY_LOCK_FN */
+    PFN_ACQUIRE_PMR_FACTORY_LOCK_FN pfnGetPMRFactoryLock;
+
+    /*! Callback function pointer, see ::PFN_RELEASE_PMR_FACTORY_LOCK_FN */
+    PFN_RELEASE_PMR_FACTORY_LOCK_FN pfnReleasePMRFactoryLock;
+
+#ifdef SUPPORT_PMR_DEFERRED_FREE
+    /*! Callback function pointer, see ::PFN_ZOMBIFY_FN */
+    PFN_ZOMBIFY_FN pfnZombify;
+#endif
+};
+
+/*! PMR factory callback table.
+ */
+typedef struct _PMR_IMPL_FUNCTAB_ PMR_IMPL_FUNCTAB;
+
+#endif /* SRVSRV_PMR_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_os.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_os.h
new file mode 100644
index 000000000000..84a3a6c400ee
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pmr_os.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific PMR functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PMR_OS_H)
+#define PMR_OS_H
+
+#include "pmr_impl.h"
+
+/*************************************************************************/ /*!
+@Function       OSMMapPMRGeneric
+@Description    Implements a generic PMR mapping function, which is used
+                to CPU map a PMR where the PMR does not have a mapping
+                function defined by the creating PMR factory.
+@Input          psPMR               the PMR to be mapped
+@Output         pOSMMapData         pointer to any private data
+                                    needed by the generic mapping function
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+/*************************************************************************/ /*!
+ * @Function    OSLinuxPMRUnmapPageInPMR
+ *
+ * @Description Distributes calls in the CPU mmap module to unmap a given PMR
+ *              page offset into all associated mappings.
+ *
+ * @Input       psPMR                The PMR to be mapped.
+ * @Input       psMappingListHead    List of Linux mappings to adjust.
+ * @Input       ui32LogicalPgOffset  The logical page offset into the
+ *                                   PMR.
+ *
+ * @Return      None
+*/ /**************************************************************************/
+void
+OSLinuxPMRUnmapPageInPMR(PMR *psPMR, DLLIST_NODE *psMappingListHead, IMG_UINT32 ui32LogicalPgOffset);
+#endif
+
+#endif /* !defined(PMR_OS_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/power.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/power.h
new file mode 100644
index 000000000000..159fa17781a8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/power.h
@@ -0,0 +1,611 @@
+/*************************************************************************/ /*!
+@File
+@Title          Power Management Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for power management functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef POWER_H
+#define POWER_H
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_error.h"
+#include "servicesext.h"
+#include "opaque_types.h"
+#include "di_common.h"
+
+/*!
+ *****************************************************************************
+ *	Power management
+ *****************************************************************************/
+
+typedef struct _PVRSRV_POWER_DEV_TAG_ PVRSRV_POWER_DEV;
+
+typedef IMG_BOOL (*PFN_SYS_DEV_IS_DEFAULT_STATE_OFF)(PVRSRV_POWER_DEV *psPowerDevice);
+
+/* Power transition handler prototypes */
+
+/*!
+  Typedef for a pointer to a Function that will be called before a transition
+  from one power state to another. See also PFN_POST_POWER.
+ */
+typedef PVRSRV_ERROR (*PFN_PRE_POWER)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      PVRSRV_DEV_POWER_STATE eNewPowerState,
+                                      PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                                      PVRSRV_POWER_FLAGS ePwrFlags);
+/*!
+  Typedef for a pointer to a Function that will be called after a transition
+  from one power state to another. See also PFN_PRE_POWER.
+ */
+typedef PVRSRV_ERROR (*PFN_POST_POWER)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                       PVRSRV_DEV_POWER_STATE eNewPowerState,
+                                       PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                                       PVRSRV_POWER_FLAGS ePwrFlags);
+
+/* Clock speed handler prototypes */
+
+/*!
+  Typedef for a pointer to a Function that will be called before a transition
+  from one clock speed to another. See also PFN_POST_CLOCKSPEED_CHANGE.
+ */
+typedef PVRSRV_ERROR (*PFN_PRE_CLOCKSPEED_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                  PVRSRV_DEV_POWER_STATE eCurrentPowerState);
+
+/*!
+  Typedef for a pointer to a Function that will be called after a transition
+  from one clock speed to another. See also PFN_PRE_CLOCKSPEED_CHANGE.
+ */
+typedef PVRSRV_ERROR (*PFN_POST_CLOCKSPEED_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                   PVRSRV_DEV_POWER_STATE eCurrentPowerState);
+
+/*!
+  Typedef for a pointer to a function that will be called to transition the
+  device to a forced idle state. Used in unison with (forced) power requests,
+  DVFS and cluster count changes.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_REQUEST)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                IMG_BOOL bDeviceOffPermitted);
+
+/*!
+  Typedef for a pointer to a function that will be called to cancel a forced
+  idle state and return the firmware back to a state where the hardware can be
+  scheduled.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_CANCEL_REQUEST)(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+  Typedef for a pointer to a function that will be called to cancel a forced
+  idle state and return the firmware back to a state where the hardware can be
+  scheduled. This function does not wait for a response from the FW.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC)(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+
+typedef PVRSRV_ERROR (*PFN_GPU_UNITS_POWER_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                   IMG_UINT32 ui32SESPowerState);
+
+const char *PVRSRVSysPowerStateToString(PVRSRV_SYS_POWER_STATE eState);
+const char *PVRSRVDevPowerStateToString(PVRSRV_DEV_POWER_STATE eState);
+
+PVRSRV_ERROR PVRSRVPowerLockInit(PPVRSRV_DEVICE_NODE psDeviceNode);
+void PVRSRVPowerLockDeInit(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerLock
+
+ @Description	Obtain the mutex for power transitions. Only allowed when
+                system power is on.
+
+ @Return	PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or PVRSRV_OK
+
+******************************************************************************/
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                   const char *pszFile, const unsigned int ui32LineNum);
+#define PVRSRVPowerLock(DEV_NODE)	PVRSRVPowerLock_Debug(DEV_NODE, __FILE__, __LINE__)
+#else
+PVRSRV_ERROR PVRSRVPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerUnlock
+
+ @Description	Release the mutex for power transitions
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+void PVRSRVPowerUnlock(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerTryLock
+
+ @Description	Try to obtain the mutex for power transitions. Only allowed when
+		system power is on.
+
+ @Return	PVRSRV_ERROR_RETRY or PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or
+		PVRSRV_OK
+
+******************************************************************************/
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerTryLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      const char *pszFile, const unsigned int ui32LineNum);
+#define PVRSRVPowerTryLock(DEV_NODE)	PVRSRVPowerTryLock_Debug(DEV_NODE, __FILE__, __LINE__)
+#else
+PVRSRV_ERROR PVRSRVPowerTryLock(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerTryLockWaitForTimeout
+
+ @Description	Try to obtain the mutex for power transitions. Only allowed when
+		system power is on. The call blocks until either the lock is acquired,
+		or the timeout is reached.
+
+		*** Debug only. DO NOT use in core GPU functions which cannot fail. ***
+		If the power lock cannot be taken the device may be powered down at
+		any time in another worker thread.
+
+ @Return	PVRSRV_ERROR_RETRY or PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or
+		PVRSRV_OK
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPowerTryLockWaitForTimeout(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function     PVRSRVPwrLockIsLockedByMe
+
+ @Description  Determine if the calling context is holding the device power-lock
+
+ @Return       IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVPwrLockIsLockedByMe(PCPVRSRV_DEVICE_NODE psDeviceNode);
+IMG_BOOL PVRSRVDeviceIsDefaultStateOFF(PVRSRV_POWER_DEV *psPowerDevice);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVSetDevicePowerStateKM
+
+ @Description	Set the Device into a new state
+
+ @Input		psDeviceNode : Device node
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDevicePowerStateKM(PPVRSRV_DEVICE_NODE	psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+										 PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVSetDeviceSystemPowerState
+@Description  Set the device into a new power state based on the systems power
+              state
+@Input        psDeviceNode       Device node
+@Input        eNewSysPowerState  New system power state
+@Input        ePwrFlags          Power state change flags
+@Return       PVRSRV_ERROR       PVRSRV_OK on success or an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState,
+											 PVRSRV_POWER_FLAGS ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetDeviceDefaultPowerState
+
+ @Description   Set the default device power state to eNewPowerState
+
+ @Input         psDeviceNode : Device node
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceDefaultPowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+					PVRSRV_DEV_POWER_STATE eNewPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetDeviceCurrentPowerState
+
+ @Description   Set the current device power state to eNewPowerState
+
+ @Input         psPowerDevice : Power device
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceCurrentPowerState(PVRSRV_POWER_DEV *psPowerDevice,
+					PVRSRV_DEV_POWER_STATE eNewPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetSystemPowerState
+
+ @Description   Set the system power state to eNewPowerState
+
+ @Input         psDeviceConfig : Device config
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetSystemPowerState(PVRSRV_DEVICE_CONFIG * psDeviceConfig,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetPowerCallbacks
+
+ @Description   Initialise the Power Device's function pointers
+                to the appropriate callbacks depending on driver mode and
+                system setup.
+
+ @Input         psDeviceNode : Device node
+ @Input         psPowerDevice : Power device
+ @Input         pfnDevicePrePower : regular device pre power callback
+ @Input         pfnDevicePostPower : regular device post power callback
+ @Input         pfnSystemPrePower : regular system pre power callback
+ @Input         pfnSystemPostPower : regular device pre power callback
+ @Input         pfnForcedIdleRequest : forced idle request callback
+ @Input         pfnForcedIdleCancelRequest : forced idle request cancel callback
+ @Input         pfnForcedIdleCancelRequestAsync : forced idle request cancel callback,
+                                                  doesn't wait for response.
+
+******************************************************************************/
+void PVRSRVSetPowerCallbacks(PPVRSRV_DEVICE_NODE				psDeviceNode,
+							 PVRSRV_POWER_DEV					*psPowerDevice,
+							 PFN_PRE_POWER						pfnDevicePrePower,
+							 PFN_POST_POWER						pfnDevicePostPower,
+							 PFN_SYS_PRE_POWER					pfnSystemPrePower,
+							 PFN_SYS_POST_POWER					pfnSystemPostPower,
+							 PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST		pfnForcedIdleCancelRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync);
+
+/* Type PFN_DC_REGISTER_POWER */
+PVRSRV_ERROR PVRSRVRegisterPowerDevice(PPVRSRV_DEVICE_NODE				psDeviceNode,
+									   PFN_PRE_POWER					pfnDevicePrePower,
+									   PFN_POST_POWER					pfnDevicePostPower,
+									   PFN_SYS_PRE_POWER			    pfnSystemPrePower,
+									   PFN_SYS_POST_POWER			    pfnSystemPostPower,
+									   PFN_PRE_CLOCKSPEED_CHANGE		pfnPreClockSpeedChange,
+									   PFN_POST_CLOCKSPEED_CHANGE		pfnPostClockSpeedChange,
+									   PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST_ASYNC pfnForcedIdleCancelRequestAsync,
+									   PFN_GPU_UNITS_POWER_CHANGE		pfnGPUUnitsPowerChange,
+									   IMG_HANDLE						hDevCookie,
+									   PVRSRV_DEV_POWER_STATE			eCurrentPowerState,
+									   PVRSRV_DEV_POWER_STATE			eDefaultPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVRemovePowerDevice
+
+ @Description
+
+ Removes device from power management register. Device is located by Device Index
+
+ @Input		psDeviceNode : Device node
+
+******************************************************************************/
+void PVRSRVRemovePowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVGetDevicePowerState
+
+ @Description
+
+	Return the device power state
+
+ @Input		psDeviceNode : Device node
+ @Output	pePowerState : Current power state
+
+ @Return	PVRSRV_ERROR_UNKNOWN_POWER_STATE if device could not be found.
+            PVRSRV_OK otherwise.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVGetDevicePowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+									   PPVRSRV_DEV_POWER_STATE pePowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVIsDevicePowered
+
+ @Description
+
+	Whether the device is powered, for the purposes of lockup detection.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVIsDevicePowered(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVGetSystemPowerState
+
+ @Description
+
+	Return the system power state
+
+ @Input		psDeviceNode : Device node
+ @Output	peCurrentSysPowerState : Current power state
+
+ @Return	PVRSRV_ERROR_UNKNOWN_POWER_STATE if device could not be found.
+            PVRSRV_OK otherwise.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVGetSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+	                                   PPVRSRV_SYS_POWER_STATE peCurrentSysPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVIsSystemPowered
+
+ @Description
+
+	Whether the system layer is powered, for ensuring the RGX regbank is powered
+	during initial GPU driver configuration.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVIsSystemPowered(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVDevicePreClockSpeedChange
+
+@Description    This function is called before a voltage/frequency change is
+                made to the GPU HW. It informs the host driver of the intention
+                to make a DVFS change. If allows the host driver to idle
+                the GPU and begin a hold off period from starting new work
+                on the GPU.
+                When this call succeeds the caller *must* call
+                PVRSRVDevicePostClockSpeedChange() to end the hold off period
+                to allow new work to be submitted to the GPU.
+
+                Called from system layer or OS layer implementation that
+                is responsible for triggering a GPU DVFS transition.
+
+@Input          psDeviceNode pointer to the device affected by DVFS transition.
+@Input          bIdleDevice  when True, the driver will wait for the GPU to
+                             reach an idle state before the call returns.
+@Input          pvInfo       unused
+
+@Return         PVRSRV_OK    on success, power lock acquired and held on exit,
+                             GPU idle.
+                PVRSRV_ERROR on failure, power lock not held on exit, do not
+                             call PVRSRVDevicePostClockSpeedChange().
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDevicePreClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 IMG_BOOL	bIdleDevice,
+											 void	*pvInfo);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVDevicePostClockSpeedChange
+
+@Description    This function is called after a voltage/frequency change has
+                been made to the GPU HW following a call to
+                PVRSRVDevicePreClockSpeedChange().
+                Before calling this function, the caller must ensure the system
+                data RGX_DATA->RGX_TIMING_INFORMATION->ui32CoreClockSpeed has
+                been updated with the new frequency set, measured in Hz.
+                The function informs the host driver that the DVFS change has
+                completed. The driver will end the work hold off period, cancel
+                the device idle period and update its time data records.
+                When this call returns work submissions are unblocked and
+                are submitted to the GPU as normal.
+                This function *must* not be called if the preceding call to
+                PVRSRVDevicePreClockSpeedChange() failed.
+
+                Called from system layer or OS layer implementation that
+                is responsible for triggering a GPU DVFS transition.
+
+@Input          psDeviceNode pointer to the device affected by DVFS transition.
+@Input          bIdleDevice  when True, the driver will cancel the GPU
+                             device idle state before the call returns. Value
+                             given must match that used in the call to
+                             PVRSRVDevicePreClockSpeedChange() otherwise
+                             undefined behaviour will result.
+@Input          pvInfo       unused
+
+@Return         void         power lock released, no longer held on exit.
+*/ /**************************************************************************/
+void PVRSRVDevicePostClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+									  IMG_BOOL		bIdleDevice,
+									  void		*pvInfo);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVDeviceIdleRequestKM
+
+ @Description Perform device-specific processing required to force the device
+              idle. The device power-lock might be temporarily released (and
+              again re-acquired) during the course of this call, hence to
+              maintain lock-ordering power-lock should be the last acquired
+              lock before calling this function
+
+ @Input       psDeviceNode         : Device node
+
+ @Input       pfnIsDefaultStateOff : When specified, the idle request is only
+                                     processed if this function passes.
+
+ @Input       bDeviceOffPermitted  : IMG_TRUE if the transition should not fail
+                                       if device off
+                                     IMG_FALSE if the transition should fail if
+                                       device off
+
+ @Return      PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED
+                                     When re-acquisition of power-lock failed.
+                                     This error NEEDS EXPLICIT HANDLING at call
+                                     site as it signifies the caller needs to
+                                     AVOID calling PVRSRVPowerUnlock, since
+                                     power-lock is no longer "possessed" by
+                                     this context.
+
+              PVRSRV_OK              When idle request succeeded.
+              PVRSRV_ERROR           Other system errors.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF	pfnIsDefaultStateOff,
+					IMG_BOOL				bDeviceOffPermitted);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceIdleCancelRequestKM
+
+ @Description Perform device-specific processing required to cancel the forced idle state
+              on the device, returning to normal operation.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleCancelRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVDeviceIdleLatchedGetKM
+
+ @Description Perform device-specific processing required to force the device
+              idle. The device power-lock might be temporarily released (and
+              again re-acquired) during the course of this call, hence to
+              maintain lock-ordering power-lock should be the last acquired
+              lock before calling this function. Latched based on power
+              device internal refcount. Only actions idle on initial reference
+              increment via calling this function.
+
+ @Input       psDeviceNode         : Device node
+
+ @Return      PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED
+                                     When re-acquisition of power-lock failed.
+                                     Handled internally but still represents error
+                                     occurred.
+
+              PVRSRV_OK              When idle request succeeded.
+              PVRSRV_ERROR           Other system errors.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedGetKM(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceIdleLatchedPutAsyncKM
+
+ @Description Perform device-specific processing required to cancel the forced idle state
+              on the device, returning to normal operation. Does not wait for FW response.
+              Latched based on power device internal refcount. Only actions idle cancel
+              once final reference is dropped via calling this function.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleLatchedPutAsyncKM(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+@Function       PVRSRVDeviceGPUUnitsPowerChange
+@Description    Request from system layer for changing power state of GPU
+                units
+@Input          psDeviceNode            RGX Device Node.
+@Input          ui32NewValue            Value indicating the new power state
+                                        of GPU units. how this is interpreted
+                                        depends upon the device-specific
+                                        function subsequently called by the
+                                        server via a pfn.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceGPUUnitsPowerChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+					IMG_UINT32	ui32NewValue);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+void PVRSRVSetFirmwareStartTime(PVRSRV_POWER_DEV *psPowerDevice, IMG_UINT32 ui32TimeStamp);
+
+void PVRSRVSetFirmwareHandshakeIdleTime(PVRSRV_POWER_DEV *psPowerDevice, IMG_UINT64 ui64Duration);
+
+int PVRSRVPowerStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+#endif
+
+#endif /* POWER_H */
+
+/******************************************************************************
+ End of file (power.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/process_stats.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/process_stats.h
new file mode 100644
index 000000000000..c410736e1749
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/process_stats.h
@@ -0,0 +1,220 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for creating and reading proc filesystem entries.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PROCESS_STATS_H
+#define PROCESS_STATS_H
+
+#include <powervr/mem_types.h>
+
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "connection_server.h"
+
+/*
+ * The publishing of Process Stats is controlled by the
+ * PVRSRV_ENABLE_PROCESS_STATS build option. The recording of all Memory
+ * allocations is controlled by the PVRSRV_ENABLE_MEMORY_STATS build option.
+ *
+ * Note: There will be a performance degradation with memory allocation
+ *       recording enabled!
+ */
+
+
+/*
+ * Memory types which can be tracked...
+ */
+typedef enum {
+	PVRSRV_MEM_ALLOC_TYPE_KMALLOC,				/* memory allocated by kmalloc() */
+	PVRSRV_MEM_ALLOC_TYPE_VMALLOC,				/* memory allocated by vmalloc() */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,	/* pages allocated from UMA to hold page table information */
+	PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,			/* ALLOC_PAGES_PT_UMA mapped to kernel address space */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,	/* pages allocated from LMA to hold page table information */
+	PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,		/* ALLOC_PAGES_PT_LMA mapped to kernel address space */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,		/* pages allocated from LMA */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,		/* pages allocated from UMA */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES,		/* zombie pages allocated from LMA */
+	PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,		/* zombie pages allocated from UMA */
+#endif
+	PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,	/* mapped UMA/LMA pages */
+	PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES,		/* pages in the page pool */
+	PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,       /* dma-buf imports */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,       /* dma-buf zombie */
+#endif
+
+	/* Must be the last enum...*/
+	PVRSRV_MEM_ALLOC_TYPE_COUNT
+} PVRSRV_MEM_ALLOC_TYPE;
+
+/*
+ * Functions for managing the processes recorded...
+ */
+PVRSRV_ERROR PVRSRVStatsInitialise(void);
+void PVRSRVStatsDestroy(void);
+void PVRSRVStatsDestroyDI(void);
+
+PVRSRV_ERROR PVRSRVStatsRegisterProcess(IMG_HANDLE* phProcessStats);
+
+void PVRSRVStatsDeregisterProcess(IMG_HANDLE hProcessStats);
+
+PVRSRV_ERROR PVRSRVStatsDeviceConnect(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void PVRSRVStatsDeviceDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*
+ * Functions for recording the statistics...
+ */
+
+void PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  void *pvCpuVAddr,
+								  IMG_CPU_PHYADDR sCpuPAddr,
+								  size_t uiBytes,
+								  IMG_PID uiPid
+								  DEBUG_MEMSTATS_PARAMS);
+
+void PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									 IMG_UINT64 ui64Key,
+									 IMG_PID uiPid);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eCurrentType,
+                                       PVRSRV_MEM_ALLOC_TYPE eTargetType,
+                                       IMG_UINT64 ui64Key,
+                                       IMG_PID currentPid
+                                       DEBUG_MEMSTATS_PARAMS);
+#endif
+
+void PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								 size_t uiBytes,
+								 IMG_PID uiPid);
+
+/*
+ * Increases the memory stat for eAllocType. Tracks the allocation size value
+ * by inserting a value into a hash table with uiCpuVAddr as key.
+ * Pair with PVRSRVStatsDecrMemAllocStatAndUntrack().
+ */
+void PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+										 size_t uiBytes,
+										 IMG_UINT64 uiCpuVAddr,
+										 IMG_PID uiPid);
+
+void PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								 size_t uiBytes,
+								 IMG_PID uiPid);
+
+void PVRSRVStatsDecrMemKAllocStat(size_t uiBytes,
+								  IMG_PID decrPID);
+
+/*
+ * Decrease the memory stat for eAllocType. Takes the allocation size value
+ * from the hash table with uiCpuVAddr as key.
+ * Pair with PVRSRVStatsIncrMemAllocStatAndTrack().
+ */
+void PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									IMG_UINT64 uiCpuVAddr);
+
+void
+PVRSRVStatsIncrMemAllocPoolStat(size_t uiBytes);
+
+void
+PVRSRVStatsDecrMemAllocPoolStat(size_t uiBytes);
+
+PVRSRV_ERROR
+PVRSRVStatsUpdateOOMStat(CONNECTION_DATA *psConnection,
+						  PVRSRV_DEVICE_NODE *psDeviceNode,
+						  IMG_UINT32 ui32OOMStatType,
+						  IMG_PID pidOwner);
+
+void PVRSRVStatsUpdateRenderContextStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+										 IMG_UINT32 ui32TotalNumPartialRenders,
+										 IMG_UINT32 ui32TotalNumOutOfMemory,
+										 IMG_UINT32 ui32TotalTAStores,
+										 IMG_UINT32 ui32Total3DStores,
+										 IMG_UINT32 ui32TotalCDMStores,
+										 IMG_UINT32 ui32TotalTDMStores,
+										 IMG_UINT32 ui32NumRayStores,
+										 IMG_PID owner);
+
+void PVRSRVStatsUpdateZSBufferStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32NumReqByApp,
+									IMG_UINT32 ui32NumReqByFW,
+									IMG_PID owner);
+
+void PVRSRVStatsUpdateFreelistStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32NumGrowReqByApp,
+									IMG_UINT32 ui32NumGrowReqByFW,
+									IMG_UINT32 ui32InitFLPages,
+									IMG_UINT32 ui32NumHighPages,
+									IMG_PID    ownerPid);
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void PVRSRVStatsUpdateCacheOpStats(PVRSRV_CACHE_OP uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+								   IMG_DEV_VIRTADDR sDevVAddr,
+								   IMG_DEV_PHYADDR sDevPAddr,
+#endif
+								   IMG_DEVMEM_SIZE_T uiOffset,
+								   IMG_DEVMEM_SIZE_T uiSize,
+								   IMG_UINT64 ui64ExecuteTimeMs,
+								   IMG_BOOL bUserModeFlush,
+								   IMG_PID ownerPid);
+#endif
+
+/* Functions used for calculating the memory usage statistics of a process */
+PVRSRV_ERROR PVRSRVFindProcessMemStats(IMG_PID pid,
+                                       IMG_UINT32 ui32ArrSize,
+                                       IMG_BOOL bAllProcessStats,
+                                       IMG_UINT64 *pui64MemoryStats);
+
+typedef struct {
+	IMG_UINT32 ui32Pid;
+	IMG_UINT64 ui64KernelMemUsage;
+	IMG_UINT64 ui64GraphicsMemUsage;
+} PVRSRV_PER_PROCESS_MEM_USAGE;
+
+PVRSRV_ERROR PVRSRVGetProcessMemUsage(IMG_UINT64 *pui64TotalMem,
+                                      IMG_UINT32 *pui32NumberOfLivePids,
+                                      PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsageData);
+
+#endif /* PROCESS_STATS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_dvfs.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_dvfs.h
new file mode 100644
index 000000000000..0529b6d20a10
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_dvfs.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs.h
+@Title          System level interface for DVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_H
+#define PVR_DVFS_H
+
+#if defined(SUPPORT_LINUX_DVFS)
+ #include <linux/devfreq.h>
+ #include <linux/thermal.h>
+
+ #if defined(CONFIG_DEVFREQ_THERMAL)
+  #include <linux/devfreq_cooling.h>
+ #endif
+
+ #include <linux/pm_opp.h>
+#endif
+
+#include "img_types.h"
+
+typedef void (*PFN_SYS_DEV_DVFS_SET_FREQUENCY)(IMG_HANDLE hSysData, IMG_UINT32 ui32Freq);
+typedef void (*PFN_SYS_DEV_DVFS_SET_VOLTAGE)(IMG_HANDLE hSysData, IMG_UINT32 ui32Volt);
+
+typedef struct _IMG_OPP_
+{
+	IMG_UINT32			ui32Volt;
+	/*
+	 * Unit of frequency in Hz.
+	 */
+	IMG_UINT32			ui32Freq;
+} IMG_OPP;
+
+typedef struct _IMG_DVFS_DEVICE_CFG_
+{
+	const IMG_OPP  *pasOPPTable;
+	IMG_UINT32      ui32OPPTableSize;
+#if defined(SUPPORT_LINUX_DVFS)
+	IMG_UINT32      ui32PollMs;
+#endif
+	IMG_BOOL        bIdleReq;
+	IMG_BOOL        bDTConfig;
+	PFN_SYS_DEV_DVFS_SET_FREQUENCY  pfnSetFrequency;
+	PFN_SYS_DEV_DVFS_SET_VOLTAGE    pfnSetVoltage;
+
+#if defined(CONFIG_DEVFREQ_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+	struct devfreq_cooling_power *psPowerOps;
+#endif
+} IMG_DVFS_DEVICE_CFG;
+
+#if defined(SUPPORT_LINUX_DVFS)
+typedef struct _IMG_DVFS_GOVERNOR_
+{
+	IMG_BOOL			bEnabled;
+} IMG_DVFS_GOVERNOR;
+
+typedef struct _IMG_DVFS_GOVERNOR_CFG_
+{
+	IMG_UINT32			ui32UpThreshold;
+	IMG_UINT32			ui32DownDifferential;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* custom thresholds */
+	IMG_UINT32			uiNumMembus;
+#endif
+} IMG_DVFS_GOVERNOR_CFG;
+#endif
+
+#if defined(__linux__)
+#if defined(SUPPORT_LINUX_DVFS)
+typedef enum
+{
+	PVR_DVFS_STATE_NONE	= 0,
+	PVR_DVFS_STATE_INIT_PENDING,
+	PVR_DVFS_STATE_READY,
+	PVR_DVFS_STATE_OFF,
+	PVR_DVFS_STATE_DEINIT
+} PVR_DVFS_STATE;
+
+typedef struct _IMG_DVFS_DEVICE_
+{
+	struct dev_pm_opp		*psOPP;
+	struct devfreq			*psDevFreq;
+	PVR_DVFS_STATE		eState;
+	IMG_HANDLE			hGpuUtilUserDVFS;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	IMG_DVFS_GOVERNOR_CFG data;
+	IMG_BOOL			bGovernorReady;
+#else
+	struct devfreq_simple_ondemand_data data;
+#endif
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	struct thermal_cooling_device	*psDevfreqCoolingDevice;
+#endif
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	struct pvr_profiling_device *psProfilingDevice;
+#endif
+} IMG_DVFS_DEVICE;
+#endif
+
+typedef struct _IMG_DVFS_
+{
+#if defined(SUPPORT_LINUX_DVFS)
+	IMG_DVFS_DEVICE			sDVFSDevice;
+	IMG_DVFS_GOVERNOR		sDVFSGovernor;
+	IMG_DVFS_GOVERNOR_CFG	sDVFSGovernorCfg;
+#endif
+	IMG_DVFS_DEVICE_CFG		sDVFSDeviceCfg;
+} PVRSRV_DVFS;
+#endif/* (__linux__) */
+
+#endif /* PVR_DVFS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_notifier.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_notifier.h
new file mode 100644
index 000000000000..b3757494e4b7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvr_notifier.h
@@ -0,0 +1,326 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR notifier interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVR_NOTIFIER_H)
+#define PVR_NOTIFIER_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+
+
+/*************************************************************************/ /*!
+Command Complete Notifier Interface
+*/ /**************************************************************************/
+
+typedef IMG_HANDLE PVRSRV_CMDCOMP_HANDLE;
+#ifndef CMDCOMPNOTIFY_PFN
+typedef void (*PFN_CMDCOMP_NOTIFY)(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle);
+#define CMDCOMPNOTIFY_PFN
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCmdCompleteInit
+@Description    Performs initialisation of the command complete notifier
+                interface.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVCmdCompleteInit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCmdCompleteDeinit
+@Description    Performs cleanup for the command complete notifier interface.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+void
+PVRSRVCmdCompleteDeinit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterCmdCompleteNotify
+@Description    Register a callback function that is called when some device
+                finishes some work, which is signalled via a call to
+                PVRSRVCheckStatus.
+@Output         phNotify             On success, points to command complete
+                                     notifier handle
+@Input          pfnCmdCompleteNotify Function callback
+@Input          hPrivData            Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterCmdCompleteNotify(IMG_HANDLE *phNotify,
+                                PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify,
+                                PVRSRV_CMDCOMP_HANDLE hPrivData);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterCmdCompleteNotify
+@Description    Unregister a previously registered callback function.
+@Input          hNotify              Command complete notifier handle
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterCmdCompleteNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCheckStatus
+@Description    Calls PVRSRVNotifyCommandCompletion() to notify registered
+                command complete handlers of work completion and then calls
+                PVRSRVSignalGlobalEO() to signal the driver wide event objects.
+@Input          hCmdCompCallerHandle Used to prevent a handler from being
+                                     notified. A NULL value results in all
+                                     handlers being notified.
+*/ /**************************************************************************/
+void
+PVRSRVCheckStatus(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVNotifyCommandCompletion
+@Description    Notify any registered command complete handlers that some work
+                has been finished (unless hCmdCompCallerHandle matches a
+                handler's hPrivData).
+@Input          hCmdCompCallerHandle Used to prevent a handler from being
+                                     notified. A NULL value results in all
+                                     handlers being notified.
+*/ /**************************************************************************/
+void
+PVRSRVNotifyCommandCompletion(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVSignalDriverWideEO
+@Description    Signals the driver wide event objects.
+*/ /**************************************************************************/
+void
+PVRSRVSignalDriverWideEO(void);
+
+
+/*************************************************************************/ /*!
+Debug Notifier Interface
+*/ /**************************************************************************/
+
+#define DEBUG_REQUEST_DC                0
+#define DEBUG_REQUEST_SYNCTRACKING      1
+#define DEBUG_REQUEST_SRV               2
+#define DEBUG_REQUEST_SYS               3
+#define DEBUG_REQUEST_RGX               4
+#define DEBUG_REQUEST_ANDROIDSYNC       5
+#define DEBUG_REQUEST_LINUXFENCE        6
+#define DEBUG_REQUEST_SYNCCHECKPOINT    7
+#define DEBUG_REQUEST_HTB               8
+#define DEBUG_REQUEST_APPHINT           9
+#define DEBUG_REQUEST_FALLBACKSYNC      10
+
+#define DEBUG_REQUEST_VERBOSITY_LOW     0
+#define DEBUG_REQUEST_VERBOSITY_MEDIUM  1
+#define DEBUG_REQUEST_VERBOSITY_HIGH    2
+#define DEBUG_REQUEST_VERBOSITY_MAX     DEBUG_REQUEST_VERBOSITY_HIGH
+
+#define DD_VERB_LVL_ENABLED(_verbLvl, _verbLvlChk) ((_verbLvl) >= (_verbLvlChk))
+
+/*
+ * Macro used within debug dump functions to send output either to PVR_LOG or
+ * a custom function. The custom function should be stored as a function
+ * pointer in a local variable called 'pfnDumpDebugPrintf'. 'pvDumpDebugFile'
+ * is also required as a local variable to serve as a file identifier for the
+ * printf function if required.
+ */
+#define PVR_DUMPDEBUG_LOG(...)                                \
+	do                                                        \
+	{                                                         \
+		if (pfnDumpDebugPrintf)                               \
+			pfnDumpDebugPrintf(pvDumpDebugFile, __VA_ARGS__); \
+		else                                                  \
+			PVR_LOG((__VA_ARGS__));                           \
+	} while (0)
+
+struct _PVRSRV_DEVICE_NODE_;
+
+typedef IMG_HANDLE PVRSRV_DBGREQ_HANDLE;
+#ifndef DBGNOTIFY_PFNS
+typedef void (DUMPDEBUG_PRINTF_FUNC)(void *pvDumpDebugFile,
+					const IMG_CHAR *pszFormat, ...);
+typedef void (*PFN_DBGREQ_NOTIFY)(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile);
+#define DBGNOTIFY_PFNS
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDeviceDbgTable
+@Description    Registers a debug requester table for the given device. The
+                order in which the debug requester IDs appear in the
+                table determine the order in which a set of notifier callbacks
+                will be called. In other words, the requester ID that appears
+                first will have all of its associated debug notifier callbacks
+                called first. This will then be followed by all the callbacks
+                associated with the next requester ID in the table and so on.
+                The order table is handled internally.
+@Input          psDevNode     Device node to register requester table with
+@Return         PVRSRV_ERROR  PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgTable(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDriverDbgTable
+@Description    Registers a debug requester table for the driver. The
+                order in which the debug requester IDs appear in the
+                table determine the order in which a set of notifier callbacks
+                will be called. In other words, the requester ID that appears
+                first will have all of its associated debug notifier callbacks
+                called first. This will then be followed by all the callbacks
+                associated with the next requester ID in the table and so on.
+                The order table is handled internally.
+@Return         PVRSRV_ERROR  PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgTable(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDeviceDbgTable
+@Description    Unregisters a debug requester table.
+@Input          psDevNode     Device node for which the requester table should
+                              be unregistered
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVUnregisterDeviceDbgTable(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDriverDbgTable
+@Description    Unregisters the driver debug requester table.
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVUnregisterDriverDbgTable(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDeviceDbgRequestNotify
+@Description    Register a callback function on a given device that is called
+                when a debug request is made via a call PVRSRVDebugRequest.
+                There are a number of verbosity levels ranging from
+                DEBUG_REQUEST_VERBOSITY_LOW up to
+                DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+                for each level up to the highest level specified to
+                PVRSRVDebugRequest.
+@Output         phNotify             Points to debug notifier handle on success
+@Input          psDevNode            Device node for which the debug callback
+                                     should be registered
+@Input          pfnDbgRequestNotify  Function callback
+@Input          ui32RequesterID      Requester ID. This is used to determine
+                                     the order in which callbacks are called
+@Input          hDbgReqeustHandle    Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgRequestNotify(IMG_HANDLE *phNotify,
+                                     struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                     PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+                                     IMG_UINT32 ui32RequesterID,
+                                     PVRSRV_DBGREQ_HANDLE hDbgReqeustHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDriverDbgRequestNotify
+@Description    Register a callback function that is called when a debug request
+                is made via a call PVRSRVDebugRequest. There are a number of
+                verbosity levels ranging from DEBUG_REQUEST_VERBOSITY_LOW up to
+                DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+                for each level up to the highest level specified to
+                PVRSRVDebugRequest.
+@Output         phNotify             Points to debug notifier handle on success
+@Input          pfnDbgRequestNotify  Function callback
+@Input          ui32RequesterID      Requester ID. This is used to determine
+                                     the order in which callbacks are called
+@Input          hDbgReqeustHandle    Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgRequestNotify(IMG_HANDLE *phNotify,
+									 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+									 IMG_UINT32 ui32RequesterID,
+									 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDeviceDbgRequestNotify
+@Description    Unregister a previously registered (device context) callback
+                function.
+@Input          hNotify              Debug notifier handle.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterDeviceDbgRequestNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDriverDbgRequestNotify
+@Description    Unregister a previously registered (driver context) callback
+                function.
+@Input          hNotify              Debug notifier handle.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterDriverDbgRequestNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugRequest
+@Description    Notify any registered debug request handlers that a debug
+                request has been made and at what level.
+@Input          psDevNode           Device node for which the debug request
+                                    has been made
+@Input          ui32VerbLevel       The maximum verbosity level to dump
+@Input          pfnDumpDebugPrintf  Used to specify the print function that
+                                    should be used to dump any debug
+                                    information. If this argument is NULL then
+                                    PVR_LOG() will be used as the default
+                                    print function.
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVDebugRequest(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                   IMG_UINT32 ui32VerbLevel,
+                   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                   void *pvDumpDebugFile);
+
+#endif /* !defined(PVR_NOTIFIER_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv.h
new file mode 100644
index 000000000000..e9a87ba8ad06
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv.h
@@ -0,0 +1,521 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR services server header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_H
+#define PVRSRV_H
+
+#include "connection_server.h"
+#include "pvrsrv_pool.h"
+#include "device.h"
+#include "power.h"
+#include "syscommon.h"
+#include "sysinfo.h"
+#include "physheap.h"
+#include "cache_ops.h"
+#include "pvr_notifier.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "dma_support.h"
+#include "vz_vmm_pvz.h"
+
+/*!
+ * For OSThreadDestroy(), which may require a retry
+ * Try for 100 ms to destroy an OS thread before failing
+ */
+#define OS_THREAD_DESTROY_TIMEOUT_US 100000ULL
+#define OS_THREAD_DESTROY_RETRY_COUNT 10
+
+typedef enum _POLL_FLAGS_
+{
+	POLL_FLAG_NONE = 0, /* No message or dump is printed on poll timeout */
+	POLL_FLAG_LOG_ERROR = 1, /* Log error on poll timeout */
+	POLL_FLAG_DEBUG_DUMP = 2 /* Print debug dump on poll timeout */
+} POLL_FLAGS;
+
+typedef struct _BUILD_INFO_
+{
+	IMG_UINT32	ui32BuildOptions;
+	IMG_UINT32	ui32BuildVersion;
+	IMG_UINT32	ui32BuildRevision;
+	IMG_UINT32	ui32BuildType;
+#define BUILD_TYPE_DEBUG	0
+#define BUILD_TYPE_RELEASE	1
+	/* The above fields are self explanatory */
+	/* B.V.N.C can be added later if required */
+} BUILD_INFO;
+
+typedef struct _DRIVER_INFO_
+{
+	BUILD_INFO	sUMBuildInfo;
+	BUILD_INFO	sKMBuildInfo;
+	IMG_UINT8	ui8UMSupportedArch;
+	IMG_UINT8	ui8KMBitArch;
+
+#define	BUILD_ARCH_64BIT			(1 << 0)
+#define	BUILD_ARCH_32BIT			(1 << 1)
+#define	BUILD_ARCH_BOTH		(BUILD_ARCH_32BIT | BUILD_ARCH_64BIT)
+	IMG_BOOL	bIsNoMatch;
+}DRIVER_INFO;
+
+
+typedef struct PVRSRV_DATA_TAG
+{
+	DRIVER_INFO           sDriverInfo;
+	IMG_UINT32            ui32DPFErrorCount;                 /*!< Number of Fatal/Error DPFs */
+
+	POSWR_LOCK            hDeviceNodeListLock;            /*!< Read-Write lock to protect the list of devices */
+	PVRSRV_DEVICE_NODE    *psDeviceNodeList;              /*!< List head of device nodes */
+	IMG_UINT32            ui32RegisteredDevices;
+	PVRSRV_DEVICE_NODE    *psHostMemDeviceNode;           /*!< DeviceNode to be used for device independent
+	                                                        host based memory allocations where the DevMem
+	                                                        framework is to be used e.g. TL */
+	PVRSRV_SERVICES_STATE eServicesState;                 /*!< global driver state */
+
+	IMG_HANDLE            hGlobalEventObject;             /*!< OS Global Event Object */
+	IMG_UINT32            ui32GEOConsecutiveTimeouts;     /*!< OS Global Event Object Timeouts */
+
+	IMG_HANDLE            hCleanupThread;                 /*!< Cleanup thread */
+	IMG_HANDLE            hCleanupEventObject;            /*!< Event object to drive cleanup thread */
+	POS_SPINLOCK          hCleanupThreadWorkListLock;     /*!< Lock protecting the cleanup thread work list */
+	IMG_PID               cleanupThreadPid;               /*!< Cleanup thread process id */
+	uintptr_t             cleanupThreadTid;               /*!< Cleanup thread id */
+	ATOMIC_T              i32NumCleanupItemsQueued;       /*!< Number of items in cleanup thread work list */
+	ATOMIC_T              i32NumCleanupItemsNotCompleted; /*!< Number of items dropped from cleanup thread work list
+	                                                           after retry limit reached */
+	ATOMIC_T              i32CleanupItemTypes[PVRSRV_CLEANUP_TYPE_LAST];         /*!< Array containing the counts for different cleanup item types. */
+
+	IMG_HANDLE            hDevicesWatchdogThread;         /*!< Devices watchdog thread */
+	IMG_HANDLE            hDevicesWatchdogEvObj;          /*! Event object to drive devices watchdog thread */
+	volatile IMG_UINT32   ui32DevicesWatchdogPwrTrans;    /*! Number of off -> on power state transitions */
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	volatile IMG_UINT32   ui32DevicesWatchdogTimeout;     /*! Timeout for the Devices watchdog Thread */
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_HANDLE            hAutoVzWatchdogThread;          /*!< Devices watchdog thread */
+	IMG_HANDLE            hAutoVzWatchdogEvObj;           /*! Event object to drive devices watchdog thread */
+#endif
+
+	POS_LOCK              hHWPerfHostPeriodicThread_Lock; /*!< Lock for the HWPerf Host periodic thread */
+	IMG_HANDLE            hHWPerfHostPeriodicThread;      /*!< HWPerf Host periodic thread */
+	IMG_HANDLE            hHWPerfHostPeriodicEvObj;       /*! Event object to drive HWPerf thread */
+	volatile IMG_BOOL     bHWPerfHostThreadStop;
+	IMG_UINT32            ui32HWPerfHostThreadTimeout;
+
+	POS_LOCK              hClientStreamTableLock;         /*!< Lock for the client stream hash */
+	HASH_TABLE            *psClientStreamTable;           /*!< Hash table for client streams */
+
+	IMG_HANDLE            hPvzConnection;                 /*!< PVZ connection used for cross-VM hyper-calls */
+	POS_LOCK              hPvzConnectionLock;             /*!< Lock protecting PVZ connection */
+
+	IMG_BOOL              bUnload;                        /*!< Driver unload is in progress */
+
+	IMG_HANDLE            hTLCtrlStream;                  /*! Control plane for TL streams */
+
+	IMG_HANDLE            hDriverThreadEventObject;       /*! Event object relating to multi-threading in the Server */
+	IMG_BOOL              bDriverSuspended;               /*! if TRUE, the driver is suspended and new threads should not enter */
+	ATOMIC_T              iNumActiveDriverThreads;        /*! Number of threads active in the Server */
+
+	PMR                   *psInfoPagePMR;                 /*! Handle to exportable PMR of the information page. */
+	IMG_UINT32            *pui32InfoPage;                 /*! CPU memory mapping for information page. */
+	DEVMEM_MEMDESC        *psInfoPageMemDesc;             /*! Memory descriptor of the information page. */
+	POS_LOCK              hInfoPageLock;                  /*! Lock guarding access to information page. */
+
+	IMG_HANDLE            hThreadsDbgReqNotify;
+
+
+	IMG_UINT32            ui32PDumpBoundDevice;           /*!< PDump is bound to the device first connected to */
+	ATOMIC_T              iNumDriverTasksActive;          /*!< Number of device-agnostic tasks active in the server */
+	PVRSRV_DRIVER_MODE    aeModuleParamDriverMode[PVRSRV_MAX_DEVICES]; /*!< Driver Mode for each device requested at launch */
+} PVRSRV_DATA;
+
+/* Function pointer used to invalidate cache between loops in wait/poll for value functions */
+typedef PVRSRV_ERROR (*PFN_INVALIDATE_CACHEFUNC)(const volatile void*, IMG_UINT64, PVRSRV_CACHE_OP);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVGetPVRSRVData
+
+ @Description	Get a pointer to the global data
+
+ @Return   PVRSRV_DATA *
+******************************************************************************/
+PVRSRV_DATA *PVRSRVGetPVRSRVData(void);
+PVRSRV_DRIVER_MODE PVRSRVGetVzModeByDevNum(IMG_UINT32 ui32DevNum);
+
+#define PVRSRV_KM_ERRORS                     ( PVRSRVGetPVRSRVData() ? PVRSRVGetPVRSRVData()->ui32DPFErrorCount : IMG_UINT32_MAX)
+#define PVRSRV_ERROR_LIMIT_REACHED                (PVRSRV_KM_ERRORS == IMG_UINT32_MAX)
+#define PVRSRV_REPORT_ERROR()                do { if (!(PVRSRV_ERROR_LIMIT_REACHED)) { PVRSRVGetPVRSRVData()->ui32DPFErrorCount++; } } while (0)
+
+#define PVRSRV_VZ_MODE_FROM_DEVNODE(pnode)     (pnode->psDevConfig->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVINFO(pdevinfo)  (pdevinfo->psDeviceNode->psDevConfig->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVCFG(pdevcfg)    (pdevcfg->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVID(devid)       (PVRSRVGetVzModeByDevNum(devid))
+
+#define PVRSRV_VZ_MODE_IS(_expr, _struct, dev) (DRIVER_MODE_##_expr == PVRSRV_VZ_MODE_FROM_##_struct(dev))
+
+#define PVRSRV_VZ_RETN_IF_MODE(_expr, _struct, dev)         do { if (  PVRSRV_VZ_MODE_IS(_expr, _struct, dev)) { return; } } while (0)
+#define PVRSRV_VZ_RET_IF_MODE(_expr, _struct, dev, _rc)     do { if (  PVRSRV_VZ_MODE_IS(_expr, _struct, dev)) { return (_rc); } } while (0)
+
+#define PVRSRV_VZ_TIME_SLICE_MAX	(100UL)
+
+typedef struct _PHYS_HEAP_ITERATOR_ PHYS_HEAP_ITERATOR;
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorCreate
+
+ @Description
+ Creates iterator for traversing physical heap requested by ePhysHeap. The
+ iterator will go through all of the segments (a segment is physically
+ contiguous) of the physical heap and return their CPU physical address and
+ size.
+
+ @Input psDevNode: Pointer to device node struct.
+ @Input ePhysHeap: Find the matching heap.
+ @Output ppsIter: Pointer to the iterator object.
+
+ @Return PVRSRV_OK upon success and PVRSRV_ERROR otherwise.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorCreate(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_PHYS_HEAP ePhysHeap,
+                                    PHYS_HEAP_ITERATOR **ppsIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorDestroy
+
+ @Description
+ Frees the iterator object created with LMA_HeapIteratorCreate.
+
+ @Input psIter: Pointer to the iterator object.
+******************************************************************************/
+void LMA_HeapIteratorDestroy(PHYS_HEAP_ITERATOR *psIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorReset
+
+ @Description
+ Resets the iterator the first segment of the physical heap.
+
+ @Input psIter: Pointer to the iterator object.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorReset(PHYS_HEAP_ITERATOR *psIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorNext
+
+ @Description
+ Retrieves current segment's physical device address and size and moves the
+ iterator to the next element (if exists). If the iterator reached an end of
+ the heap and no segment was retrieved, this function returns IMG_FALSE.
+
+ @Input psIter: Pointer to the iterator object.
+ @Output psDevPAddr: Device physical address of the current segment.
+ @Output puiSize: Size of the current segment.
+
+ @Return IMG TRUE if a segment was found and retrieved, IMG_FALSE otherwise.
+******************************************************************************/
+IMG_BOOL LMA_HeapIteratorNext(PHYS_HEAP_ITERATOR *psIter,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              IMG_UINT64 *puiSize);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorGetHeapStats
+
+ @Description
+ Retrieves phys heap's usage statistics.
+
+ @Input psPhysHeap: Pointer to the physical heap object.
+ @Output puiTotalSize: Total size of the physical heap.
+ @Output puiInUseSize: Used space in the physical heap.
+
+ @Return PVRSRV_OK upon success and PVRSRV_otherwise.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorGetHeapStats(PHYS_HEAP_ITERATOR *psIter,
+                                          IMG_UINT64 *puiTotalSize,
+                                          IMG_UINT64 *puiInUseSize);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVPollForValueKM
+
+ @Description
+ Polls for a value to match a masked read
+
+ @Input psDevNode : Pointer to device node struct
+ @Input pui32LinMemAddr : CPU linear address to poll
+ @Input ui32Value : required value
+ @Input ui32Mask : Mask
+ @Input bDebugDumpOnFailure : Whether poll failure should result into a debug
+        dump. CAUTION: When calling this function from code paths which are
+        also used by debug-dumping code, this argument MUST be IMG_FALSE
+        otherwise, we might end up requesting debug-dump in recursion and
+        eventually blow-up call stack.
+ @Input pfnFwInvalidate : Function pointer to invalidation function used
+        each loop / poll. This is only used for FWmemctx allocations.
+
+ @Return   PVRSRV_ERROR :
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPollForValueKM(PVRSRV_DEVICE_NODE *psDevNode,
+		volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+		IMG_UINT32                   ui32Value,
+		IMG_UINT32                   ui32Mask,
+		POLL_FLAGS                   ePollFlags,
+		PFN_INVALIDATE_CACHEFUNC     pfnFwInvalidate);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVWaitForValueKM
+
+ @Description
+ Waits (using EventObjects) for a value to match a masked read
+
+ @Input  pui32LinMemAddr       : CPU linear address to poll
+ @Input  ui32Value             : Required value
+ @Input  ui32Mask              : Mask to be applied before checking against
+                                 ui32Value
+ @Input  pfnFwInvalidate       : Function pointer to invalidation function used
+                                 each loop / wait. This is only used for
+                                 FWmemctx allocations.
+ @Return PVRSRV_ERROR          :
+******************************************************************************/
+PVRSRV_ERROR
+PVRSRVWaitForValueKM(volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+                     IMG_UINT32                  ui32Value,
+                     IMG_UINT32                  ui32Mask,
+                     PFN_INVALIDATE_CACHEFUNC    pfnFwInvalidate);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemHasCacheSnooping
+
+ @Description	: Returns whether the system has cache snooping
+
+ @Return : IMG_TRUE if the system has cache snooping
+******************************************************************************/
+IMG_BOOL PVRSRVSystemHasCacheSnooping(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemSnoopingIsEmulated
+
+ @Description : Returns whether system cache snooping support is emulated
+
+ @Return : IMG_TRUE if the system cache snooping is emulated in software
+******************************************************************************/
+IMG_BOOL PVRSRVSystemSnoopingIsEmulated(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemSnoopingOfCPUCache
+
+ @Description	: Returns whether the system supports snooping of the CPU cache
+
+ @Return : IMG_TRUE if the system has CPU cache snooping
+******************************************************************************/
+IMG_BOOL PVRSRVSystemSnoopingOfCPUCache(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemHasNonMappableLocalMemory
+
+ @Description	: Returns whether the device has non-mappable part of local memory
+
+ @Return : IMG_TRUE if the device has non-mappable part of local memory
+******************************************************************************/
+IMG_BOOL PVRSRVSystemHasNonMappableLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemWaitCycles
+
+ @Description	: Waits for at least ui32Cycles of the Device clk.
+******************************************************************************/
+void PVRSRVSystemWaitCycles(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT32 ui32Cycles);
+
+PVRSRV_ERROR PVRSRVSystemInstallDeviceLISR(void *pvOSDevice,
+										   IMG_UINT32 ui32IRQ,
+										   const IMG_CHAR *pszName,
+										   PFN_LISR pfnLISR,
+										   void *pvData,
+										   IMG_HANDLE *phLISRData);
+
+PVRSRV_ERROR PVRSRVSystemUninstallDeviceLISR(IMG_HANDLE hLISRData);
+
+int PVRSRVGetDriverStatus(void);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVIsBridgeEnabled
+
+ @Description	: Returns whether the given bridge group is enabled
+
+ @Return : IMG_TRUE if the given bridge group is enabled
+******************************************************************************/
+static inline IMG_BOOL PVRSRVIsBridgeEnabled(IMG_HANDLE hServices, IMG_UINT32 ui32BridgeGroup)
+{
+	IMG_UINT32 ui32Bridges;
+	IMG_UINT32 ui32Offset;
+
+	PVR_UNREFERENCED_PARAMETER(hServices);
+
+#if defined(SUPPORT_RGX)
+	if (ui32BridgeGroup >= PVRSRV_BRIDGE_RGX_FIRST)
+	{
+		ui32Bridges = gui32RGXBridges;
+		ui32Offset = PVRSRV_BRIDGE_RGX_FIRST;
+	}
+	else
+#endif /* SUPPORT_RGX */
+	{
+		ui32Bridges = gui32PVRBridges;
+		ui32Offset = PVRSRV_BRIDGE_FIRST;
+	}
+
+	return (IMG_BOOL)(((1U << (ui32BridgeGroup - ui32Offset)) & ui32Bridges) != 0);
+}
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	void SetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState);
+	void SetTrustedDeviceAceEnabled(IMG_HANDLE hSysData);
+#endif
+#endif
+
+/*!
+******************************************************************************
+ @Function			: PVRSRVCreateHWPerfHostThread
+
+ @Description		: Creates HWPerf event object and thread unless already created
+
+ @Input ui32Timeout	: Initial timeout (ms) between updates on the HWPerf thread
+
+ @Return			: PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+******************************************************************************/
+PVRSRV_ERROR PVRSRVCreateHWPerfHostThread(IMG_UINT32 ui32Timeout);
+
+/*!
+******************************************************************************
+ @Function			: PVRSRVDestroyHWPerfHostThread
+
+ @Description		: Destroys HWPerf event object and thread if created
+
+ @Return			: PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHWPerfHostThread(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVFindPhysHeapConfig
+@Description    Find Phys Heap Config from Device Config.
+@Input          psDevConfig  Pointer to device config.
+@Input          ui32Flags    Find heap that matches flags.
+@Return         PHYS_HEAP_CONFIG*  Return a config, or NULL if not found.
+*/ /**************************************************************************/
+PHYS_HEAP_CONFIG* PVRSRVFindPhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										   PHYS_HEAP_USAGE_FLAGS ui32Flags);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDeviceInstance
+@Description    Return the specified device instance from Device node list.
+@Input          ui32Instance         Device instance to find
+@Return         PVRSRV_DEVICE_NODE*  Return a device node, or NULL if not found.
+*/ /**************************************************************************/
+PVRSRV_DEVICE_NODE* PVRSRVGetDeviceInstance(IMG_UINT32 ui32Instance);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDeviceInstanceByKernelDevID
+@Description    Return the specified device instance by OS Id.
+@Input          i32OSInstance        OS device Id to find
+@Return         PVRSRV_DEVICE_NODE*  Return a device node, or NULL if not found.
+*/ /**************************************************************************/
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstanceByKernelDevID(IMG_INT32 i32OSInstance);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVAcquireInternalID
+@Description    Returns the lowest free device ID.
+@Output         pui32InternalID  The device ID
+@Return         PVRSRV_ERROR     PVRSRV_OK or an error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVAcquireInternalID(IMG_UINT32 *pui32InternalID);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDeviceFreeze
+@Description    Stops further processing from occurring on the device after the
+                 current work ends and blocks user-mode tasks on the specified
+                 device. Device node is put into POWER-DOWN state.
+@Input          PVRSRV_DEVICE_NODE*     Device node
+@Return         PVRSRV_ERROR            PVRSRV_OK on success otherwise a
+                                        PVRSRV_ERR_ code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceFreeze(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDeviceThaw
+@Description    Unfreezes a previously frozen device. Restarts work on the
+                 device, if work queues are non-empty, and unblocks any
+                 user-mode tasks on the specified device.
+@Input          PVRSRV_DEVICE_NODE*     Device node
+@Return         PVRSRV_ERROR            PVRSRV_OK on success otherwise a
+                                        PVRSRV_ERR_ code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceThaw(PVRSRV_DEVICE_NODE *psDevNode);
+
+#endif /* PVRSRV_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_apphint.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_apphint.h
new file mode 100644
index 000000000000..e35426680795
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_apphint.h
@@ -0,0 +1,71 @@
+/**************************************************************************/ /*!
+@File
+@Title          PowerVR AppHint generic interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PVRSRV_APPHINT_H)
+#define PVRSRV_APPHINT_H
+
+/* Supplied to PVRSRVAppHintRegisterHandlers*() functions when the apphint
+ * is a global driver apphint, i.e. apphints not present in
+ * APPHINT_DEBUGFS_DEVICE_ID, i.e. not per device.
+ */
+#define APPHINT_OF_DRIVER_NO_DEVICE ((void*)-1U)
+
+#if defined(__linux__)
+
+#include "km_apphint.h"
+#define PVRSRVAppHintDumpState(d) pvr_apphint_dump_state(d)
+#define PVRSRVAppHintRegisterHandlersUINT64(i,q,s,d,p) pvr_apphint_register_handlers_uint64(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersUINT32(i,q,s,d,p) pvr_apphint_register_handlers_uint32(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersBOOL(i,q,s,d,p) pvr_apphint_register_handlers_bool(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersSTRING(i,q,s,d,p) pvr_apphint_register_handlers_string(i,q,s,d,p)
+
+#else
+
+#define PVRSRVAppHintDumpState(d)
+#define PVRSRVAppHintRegisterHandlersUINT64(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersUINT32(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersBOOL(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersSTRING(i,q,s,d,p)
+
+#endif
+
+#endif /* PVRSRV_APPHINT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_bridge_init.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_bridge_init.h
new file mode 100644
index 000000000000..b225da6d368e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_bridge_init.h
@@ -0,0 +1,53 @@
+/**************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the common PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_BRIDGE_INIT_H
+#define PVRSRV_BRIDGE_INIT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR ServerBridgeInit(void);
+void ServerBridgeDeInit(void);
+
+#endif /* PVRSRV_BRIDGE_INIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_cleanup.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_cleanup.h
new file mode 100644
index 000000000000..a197ca9ec908
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_cleanup.h
@@ -0,0 +1,256 @@
+/**************************************************************************/ /*!
+@File
+@Title          PowerVR SrvKM cleanup thread deferred work interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_CLEANUP_H
+#define PVRSRV_CLEANUP_H
+
+#include "dllist.h"
+#include "device.h"
+
+/**************************************************************************/ /*!
+@Brief          CLEANUP_THREAD_FN
+
+@Description    This is the function prototype for the pfnFree member found in
+                the structure PVRSRV_CLEANUP_THREAD_WORK. The function is
+                responsible for carrying out the clean up work and if successful
+                freeing the memory originally supplied to the call
+                PVRSRVCleanupThreadAddWork().
+
+@Input          pvParam  This is private data originally supplied by the caller
+                         to PVRSRVCleanupThreadAddWork() when registering the
+                         clean up work item, psDAta->pvData. Itr can be cast
+                         to a relevant type within the using module.
+
+@Return         PVRSRV_OK if the cleanup operation was successful and the
+                callback has freed the PVRSRV_CLEANUP_THREAD_WORK* work item
+                memory original supplied to PVRSRVCleanupThreadAddWork()
+                Any other error code will lead to the work item
+                being re-queued and hence the original
+                PVRSRV_CLEANUP_THREAD_WORK* must not be freed.
+*/ /***************************************************************************/
+
+typedef PVRSRV_ERROR (*CLEANUP_THREAD_FN)(void *pvParam);
+
+
+/* Typical number of times a caller should want the work to be retried in case
+ * of the callback function (pfnFree) returning an error.
+ * Callers to PVRSRVCleanupThreadAddWork should provide this value as the retry
+ * count (ui32RetryCount) unless there are special requirements.
+ * A value of 200 corresponds to around ~20s (200 * 100ms). If it is not
+ * successful by then give up as an unrecoverable problem has occurred.
+ */
+#define CLEANUP_THREAD_RETRY_COUNT_DEFAULT 200u
+/* Like for CLEANUP_THREAD_RETRY_COUNT_DEFAULT but call will wait for
+ * a specified amount of time rather than number of retries.
+ */
+#define CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT 20000u /* 20s */
+
+/* Use to set retry count on a cleanup item.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * _count - retry count
+ */
+#define CLEANUP_THREAD_SET_RETRY_COUNT(_item,_count) \
+	do { \
+		(_item)->ui32RetryCount = (_count); \
+		(_item)->ui32TimeStart = 0; \
+		(_item)->ui32TimeEnd = 0; \
+	} while (0)
+
+/* Use to set timeout deadline on a cleanup item.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * _timeout - timeout in milliseconds, if 0
+ *            CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT is used
+ */
+#define CLEANUP_THREAD_SET_RETRY_TIMEOUT(_item,_timeout) \
+	do { \
+		(_item)->ui32RetryCount = 0; \
+		(_item)->ui32TimeStart = OSClockms(); \
+		(_item)->ui32TimeEnd = (_item)->ui32TimeStart + ((_timeout) > 0 ? \
+				(_timeout) : CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT); \
+	} while (0)
+
+/* Indicates if the timeout on a given item has been reached.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ */
+#define CLEANUP_THREAD_RETRY_TIMEOUT_NOT_REACHED(_item) \
+	((_item)->ui32TimeEnd - (_item)->ui32TimeStart >= \
+			OSClockms() - (_item)->ui32TimeStart)
+
+/* Indicates if the current item is waiting on timeout or retry count.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * */
+#define CLEANUP_THREAD_IS_RETRY_TIMEOUT(_item) \
+	((_item)->ui32TimeStart != (_item->ui32TimeEnd))
+
+#define CLEANUP_TYPE_LIST \
+	X(UNDEF)      /**/ \
+	X(CONNECTION)     /**/ \
+	X(MMU)     /**/ \
+	X(OSMEM)   /**/ \
+	X(PMR)       /**/ \
+	X(LAST)      /**/ \
+
+#define CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE 11     /* CONNECTION\0 */
+#define CLEANUP_TYPE_ITEM_DPF " %1.11s : %1.5d"
+#define CLEANUP_TYPE_ITEM_DPF_MAX_SIZE CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE+sizeof(" : ")+5+1
+
+typedef enum _PVRSRV_CLEANUP_TYPE_
+{
+#define X(_name) PVRSRV_CLEANUP_TYPE_ ## _name,
+	CLEANUP_TYPE_LIST
+#undef X
+
+} PVRSRV_CLEANUP_TYPE;
+
+#if defined(CLEANUP_TYPE_STRINGS)
+
+static const char *const _pszCleanupStrings[] = {
+#define X(_name) #_name,
+	CLEANUP_TYPE_LIST
+#undef X
+};
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetCleanupName
+@Description    Returns the name of a Cleanup Type.
+
+@Input          eCleanupType   The enum value of the cleanup type.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetCleanupName(PVRSRV_CLEANUP_TYPE eCleanupType)
+{
+	if (eCleanupType < 0 || eCleanupType > PVRSRV_CLEANUP_TYPE_LAST)
+	{
+		return "Undefined";
+	}
+
+	PVR_ASSERT(sizeof(_pszCleanupStrings[eCleanupType]) < CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE);
+
+	return _pszCleanupStrings[eCleanupType];
+}
+
+#endif /* CLEANUP_TYPE_STRINGS */
+
+/* Clean up work item specifics so that the task can be managed by the
+* pvr_defer_free cleanup thread in the Server.
+*/
+typedef struct _PVRSRV_CLEANUP_THREAD_WORK_
+{
+	DLLIST_NODE sNode;               /*!< List node used internally by the cleanup
+	                                    thread */
+	CLEANUP_THREAD_FN pfnFree;       /*!< Pointer to the function to be called to
+	                                    carry out the deferred cleanup */
+	void *pvData;                    /*!< private data for pfnFree, usually a way back
+	                                    to the original PVRSRV_CLEANUP_THREAD_WORK*
+	                                    pointer supplied in the call to
+	                                    PVRSRVCleanupThreadAddWork(). */
+	IMG_UINT32 ui32TimeStart;        /*!< Timestamp in ms of the moment when
+	                                    cleanup item has been created. */
+	IMG_UINT32 ui32TimeEnd;          /*!< Time in ms after which no further retry
+	                                    attempts will be made, item discard and
+	                                    error logged when this is reached. */
+	IMG_UINT32 ui32RetryCount;       /*!< Number of times the callback should be
+	                                    re-tried when it returns error. */
+	IMG_BOOL bDependsOnHW;           /*!< Don't drop the cleanup task if retry limit
+	                                      is reached, we could depend on event from
+	                                      device to continue. */
+	PVRSRV_CLEANUP_TYPE eCleanupType;/*!< Type of work item added to queue */
+#if defined(DEBUG)
+	const char *pszFun;
+	unsigned int ui32LineNum;
+#endif
+} PVRSRV_CLEANUP_THREAD_WORK;
+
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadAddWork
+
+@Description    Add a work item to be called from the cleanup thread
+
+@Input          psDevNode : Pointer to the device node
+@Input          psData :    The function pointer and private data for the
+                            callback
+
+@Return         None
+*/ /***************************************************************************/
+#if defined(DEBUG)
+#define PVRSRVCleanupThreadAddWork(DEV, DATA)		PVRSRVCleanupThreadAddWork_Debug(DEV, DATA, __FILE__, __LINE__)
+void PVRSRVCleanupThreadAddWork_Debug(PVRSRV_DEVICE_NODE *psDevNode,
+                                      PVRSRV_CLEANUP_THREAD_WORK *psData,
+                                      const char *pszFun, const unsigned int ui32LineNum);
+#else
+#define PVRSRVCleanupThreadAddWork					PVRSRVCleanupThreadAddWork_Int
+void PVRSRVCleanupThreadAddWork_Int(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_CLEANUP_THREAD_WORK *psData);
+#endif
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadWaitForDevice
+
+@Description    Blocking wait for all of the device's items to be cleaned.
+
+@Input          psDevNode : Pointer to the device node
+
+@Return         None
+*/ /***************************************************************************/
+void PVRSRVCleanupThreadWaitForDevice(PVRSRV_DEVICE_NODE *psDevNode);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadGetPid
+
+@Description    Returns Cleanup Thread's PID.
+
+@Return         PID of the Cleanup Thread
+*/ /***************************************************************************/
+IMG_PID PVRSRVCleanupThreadGetPid(void);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadGetTid
+
+@Description    Returns Cleanup Thread's TID.
+
+@Return         TID of the Cleanup Thread
+*/ /***************************************************************************/
+uintptr_t PVRSRVCleanupThreadGetTid(void);
+
+#endif /* PVRSRV_CLEANUP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_device.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_device.h
new file mode 100644
index 000000000000..8d55e1f2f745
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_device.h
@@ -0,0 +1,434 @@
+/**************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_DEVICE_H
+#define PVRSRV_DEVICE_H
+
+#include "img_types.h"
+#include "physheap_config.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "rgx_firmware_boot.h"
+#include "rgx_fwif_km.h"
+#include "servicesext.h"
+#include "cache_ops.h"
+#include "opaque_types.h"
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+#include "pvr_dvfs.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+typedef struct _PVRSRV_DEVICE_CONFIG_ PVRSRV_DEVICE_CONFIG;
+typedef enum _DRIVER_MODE_
+{
+/* Do not use these enumerations directly, to query the
+   current driver mode, use the PVRSRV_VZ_MODE_IS()
+   macro */
+	DRIVER_MODE_NATIVE	= 0,
+	DRIVER_MODE_HOST	= 1,
+	DRIVER_MODE_GUEST	= 2,
+	DRIVER_MODE_DEFAULT	= 3,
+} PVRSRV_DRIVER_MODE;
+
+typedef enum
+{
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_MAPPABLE = 0,
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_NON_MAPPABLE = 1,
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_LAST
+} PVRSRV_DEVICE_LOCAL_MEMORY_ARENA;
+
+typedef enum _PVRSRV_DEVICE_SNOOP_MODE_
+{
+	PVRSRV_DEVICE_SNOOP_NONE = 0,
+	PVRSRV_DEVICE_SNOOP_CPU_ONLY,
+	PVRSRV_DEVICE_SNOOP_DEVICE_ONLY,
+	PVRSRV_DEVICE_SNOOP_CROSS,
+	PVRSRV_DEVICE_SNOOP_EMULATED,
+} PVRSRV_DEVICE_SNOOP_MODE;
+
+#if defined(SUPPORT_SOC_TIMER)
+typedef IMG_UINT64
+(*PFN_SYS_DEV_SOC_TIMER_READ)(IMG_HANDLE hSysData);
+#endif
+
+typedef enum _PVRSRV_DEVICE_FABRIC_TYPE_
+{
+	PVRSRV_DEVICE_FABRIC_NONE = 0,
+	PVRSRV_DEVICE_FABRIC_ACELITE,
+	PVRSRV_DEVICE_FABRIC_FULLACE,
+} PVRSRV_DEVICE_FABRIC_TYPE;
+
+typedef IMG_UINT32
+(*PFN_SYS_DEV_CLK_FREQ_GET)(IMG_HANDLE hSysData);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_PRE_POWER)(IMG_HANDLE hSysData,
+						 PVRSRV_SYS_POWER_STATE eNewPowerState,
+						 PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+						 PVRSRV_POWER_FLAGS ePwrFlags);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_POST_POWER)(IMG_HANDLE hSysData,
+						  PVRSRV_SYS_POWER_STATE eNewPowerState,
+						  PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+						  PVRSRV_POWER_FLAGS ePwrFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_SYS_GET_POWER
+
+@Description    This function queries the SoC power registers to determine
+                if the power domain on which the GPU resides is powered on.
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will assume the domain power state depending on driver type:
+   regular drivers assume it is unpowered at startup, while drivers with
+   AutoVz support expect the GPU domain to be powered on initially. The power
+   state will be then tracked internally according to the pfnPrePowerState
+   and pfnPostPowerState calls using a fallback function.
+
+@Input          psDevNode                  Pointer to node struct of the
+                                           device being initialised
+
+@Return         PVRSRV_SYS_POWER_STATE_ON  if the respective device's hardware
+                                           domain is powered on
+                PVRSRV_SYS_POWER_STATE_OFF if the domain is powered off
+*/ /**************************************************************************/
+typedef PVRSRV_SYS_POWER_STATE
+(*PFN_SYS_GET_POWER)(PPVRSRV_DEVICE_NODE psDevNode);
+
+typedef void
+(*PFN_SYS_DEV_INTERRUPT_HANDLED)(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_DEV_CHECK_MEM_ALLOC_SIZE)(IMG_HANDLE hSysData,
+									IMG_UINT64 ui64MemSize);
+
+typedef void (*PFN_SYS_DEV_FEAT_DEP_INIT)(PVRSRV_DEVICE_CONFIG *, IMG_UINT64);
+
+typedef void
+(*PFN_SYS_DEV_HOST_CACHE_MAINTENANCE)(IMG_HANDLE hSysData,
+									PVRSRV_CACHE_OP eRequestType,
+									void *pvVirtStart,
+									void *pvVirtEnd,
+									IMG_CPU_PHYADDR sCPUPhysStart,
+									IMG_CPU_PHYADDR sCPUPhysEnd);
+
+typedef void*
+(*PFN_SLAVE_DMA_CHAN)(PVRSRV_DEVICE_CONFIG*, char*);
+
+typedef void
+(*PFN_SLAVE_DMA_FREE)(PVRSRV_DEVICE_CONFIG*,
+					  void*);
+
+typedef void
+(*PFN_DEV_PHY_ADDR_2_DMA_ADDR)(PVRSRV_DEVICE_CONFIG *,
+							   IMG_DMA_ADDR *,
+							   IMG_DEV_PHYADDR *,
+							   IMG_BOOL *,
+							   IMG_UINT32,
+							   IMG_BOOL);
+
+
+typedef struct _PVRSRV_FW_PARAMS_
+{
+	const void *pvFirmware;
+	IMG_UINT32 ui32FirmwareSize;
+	const void *pvSignature;
+	IMG_UINT32 ui32SignatureSize;
+	PVRSRV_FW_BOOT_PARAMS uFWP;
+} PVRSRV_FW_PARAMS;
+
+typedef PVRSRV_ERROR
+(*PFN_PREPARE_FW_IMAGE)(IMG_HANDLE hSysData,
+						PVRSRV_FW_PARAMS *psFWParams);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+
+typedef PVRSRV_ERROR
+(*PFN_TD_SEND_FW_IMAGE)(IMG_HANDLE hSysData,
+						PVRSRV_FW_PARAMS *psTDFWParams);
+
+typedef struct _PVRSRV_TD_POWER_PARAMS_
+{
+	IMG_DEV_PHYADDR sPCAddr;
+
+	/* MIPS-only fields */
+	IMG_DEV_PHYADDR sGPURegAddr;
+	IMG_DEV_PHYADDR sBootRemapAddr;
+	IMG_DEV_PHYADDR sCodeRemapAddr;
+	IMG_DEV_PHYADDR sDataRemapAddr;
+} PVRSRV_TD_POWER_PARAMS;
+
+typedef PVRSRV_ERROR
+(*PFN_TD_SET_POWER_PARAMS)(IMG_HANDLE hSysData,
+						   PVRSRV_TD_POWER_PARAMS *psTDPowerParams);
+
+typedef PVRSRV_ERROR
+(*PFN_TD_RGXSTART)(IMG_HANDLE hSysData);
+
+typedef PVRSRV_ERROR
+(*PFN_TD_RGXSTOP)(IMG_HANDLE hSysData);
+
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef void (*PFN_SYS_INIT_FIREWALL)(IMG_HANDLE hSysData,
+                                      IMG_UINT64[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                                      IMG_UINT64[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG_
+{
+	IMG_UINT32 ui32Status;     /*!< FW status */
+	IMG_UINT32 ui32Reason;     /*!< Reason for FW status */
+} PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG;
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF_
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;     /*!< FW page fault address */
+} PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF;
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM_
+{
+	IMG_UINT32 ui32ExtJobRef;     /*!< External Job Reference of any affected GPU work */
+	RGXFWIF_DM eDM;               /*!< Data Master which was running any affected GPU work */
+} PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM;
+
+typedef struct _PVRSRV_ROBUSTNESS_NOTIFY_DATA_
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reason for error/reset */
+	IMG_PID                  pid;          /*!< Pid of process which created the errored context */
+	union
+	{
+		PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM sChecksumErrData; /*!< Data returned for checksum errors */
+		PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF    sFwPFErrData;     /*!< Data returned for FW page faults */
+		PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG sHostWdgData;     /*!< Data returned for Host Wdg FW faults */
+	} uErrData;
+} PVRSRV_ROBUSTNESS_NOTIFY_DATA;
+
+typedef void
+(*PFN_SYS_DEV_ERROR_NOTIFY)(IMG_HANDLE hSysData,
+						    PVRSRV_ROBUSTNESS_NOTIFY_DATA *psRobustnessErrorData);
+
+struct _PVRSRV_DEVICE_CONFIG_
+{
+	/*! OS device passed to SysDevInit (linux: 'struct device') */
+	void *pvOSDevice;
+
+	/*!
+	 *! Service representation of pvOSDevice. Should be set to NULL when the
+	 *! config is created in SysDevInit. Set by Services once a device node has
+	 *! been created for this config and unset before SysDevDeInit is called.
+	 */
+	struct _PVRSRV_DEVICE_NODE_ *psDevNode;
+
+	/*! Name of the device */
+	const IMG_CHAR *pszName;
+
+	/*! Version of the device (optional) */
+	IMG_CHAR *pszVersion;
+
+	/*! Register bank address */
+	IMG_CPU_PHYADDR sRegsCpuPBase;
+	/*! Register bank size */
+	IMG_UINT32 ui32RegsSize;
+	/*! Device interrupt number */
+	IMG_UINT32 ui32IRQ;
+
+	PVRSRV_DEVICE_SNOOP_MODE eCacheSnoopingMode;
+
+	/*! Device specific data handle */
+	IMG_HANDLE hDevData;
+
+	/*! System specific data that gets passed into system callback functions. */
+	IMG_HANDLE hSysData;
+
+	IMG_BOOL bHasNonMappableLocalMemory;
+
+	/*! Indicates if system supports FBCDC v3.1 */
+	IMG_BOOL bHasFBCDCVersion31;
+
+	/*! Physical Heap definitions for this device.
+	 * eDefaultHeap must be set to GPU_LOCAL or CPU_LOCAL. Specifying any other value
+	 *    (e.g. DEFAULT) will lead to an error at device discovery.
+	 * pasPhysHeap array must contain at least one PhysHeap, the declared default heap.
+	 */
+	PVRSRV_PHYS_HEAP  eDefaultHeap;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 ui32PhysHeapCount;
+
+	/*!
+	 *! Callbacks to change system device power state at the beginning and end
+	 *! of a power state change (optional).
+	 */
+	PFN_SYS_PRE_POWER pfnPrePowerState;
+	PFN_SYS_POST_POWER pfnPostPowerState;
+	PFN_SYS_GET_POWER  pfnGpuDomainPower;
+
+	/*! Callback to obtain the clock frequency from the device (optional). */
+	PFN_SYS_DEV_CLK_FREQ_GET pfnClockFreqGet;
+
+#if defined(SUPPORT_SOC_TIMER)
+	/*! Callback to read SoC timer register value (mandatory). */
+	PFN_SYS_DEV_SOC_TIMER_READ	pfnSoCTimerRead;
+#endif
+
+	/*!
+	 *! Callback to perform host CPU cache maintenance. Might be needed for
+	 *! architectures which allow extensions such as RISC-V (optional).
+	 */
+	PFN_SYS_DEV_HOST_CACHE_MAINTENANCE pfnHostCacheMaintenance;
+	IMG_BOOL bHasPhysicalCacheMaintenance;
+
+	/*!
+	 *! Callback to prepare FW image after it has been loaded.  This may
+	 *! be used to separate a signature/header from the firmware proper,
+	 *! potentially modifying pvFirmware and ui32FirmwareSize to point to the
+	 *! actual firmware to be loaded.
+	 */
+	PFN_PREPARE_FW_IMAGE pfnPrepareFWImage;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/*!
+	 *! Callback to send FW image and FW boot time parameters to the trusted
+	 *! device.
+	 */
+	PFN_TD_SEND_FW_IMAGE pfnTDSendFWImage;
+
+	/*!
+	 *! Callback to send parameters needed in a power transition to the trusted
+	 *! device.
+	 */
+	PFN_TD_SET_POWER_PARAMS pfnTDSetPowerParams;
+
+	/*! Callbacks to ping the trusted device to securely run RGXStart/Stop() */
+	PFN_TD_RGXSTART pfnTDRGXStart;
+	PFN_TD_RGXSTOP pfnTDRGXStop;
+
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+	/*! Name of DMA heap to allocate secure memory from. Used with dma_heap_find. */
+	IMG_CHAR *pszSecureDMAHeapName;
+#endif
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+	/*! Function that does device feature specific system layer initialisation */
+	PFN_SYS_DEV_FEAT_DEP_INIT	pfnSysDevFeatureDepInit;
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	PVRSRV_DVFS sDVFS;
+#endif
+
+#if defined(SUPPORT_ALT_REGBASE)
+	IMG_DEV_PHYADDR sAltRegsGpuPBase;
+#endif
+
+	/*!
+	 *! Indicates if device physical address 0x0 might be used as GPU memory
+	 *! (e.g. LMA system or UMA system with CPU PA 0x0 reserved by the OS,
+	 *!  but CPU PA != device PA and device PA 0x0 available for the GPU)
+	 */
+	IMG_BOOL bDevicePA0IsValid;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/*!
+	 *! System-specific function to initialise firewall mechanism needed to
+	 *! validate the correctness of memory accesses tagged with OSIDs.
+	 *! Used for hardware validation of virtualization features only.
+	 */
+	PFN_SYS_INIT_FIREWALL		pfnSysInitFirewall;
+#endif
+
+	/*!
+	 *! Callback to notify system layer of device errors.
+	 *! NB. implementers should ensure that the minimal amount of work is
+	 *! done in the callback function, as it will be executed in the main
+	 *! RGX MISR. (e.g. any blocking or lengthy work should be performed by
+	 *! a worker queue/thread instead.)
+	 */
+	PFN_SYS_DEV_ERROR_NOTIFY	pfnSysDevErrorNotify;
+
+	/*!
+	 *!  Slave DMA channel request callbacks
+	 */
+	PFN_SLAVE_DMA_CHAN pfnSlaveDMAGetChan;
+	PFN_SLAVE_DMA_FREE pfnSlaveDMAFreeChan;
+	/*!
+	 *!  Conversion of device memory to DMA addresses
+	 */
+	PFN_DEV_PHY_ADDR_2_DMA_ADDR pfnDevPhysAddr2DmaAddr;
+	/*!
+	 *!  DMA channel names
+	 */
+	IMG_CHAR *pszDmaTxChanName;
+	IMG_CHAR *pszDmaRxChanName;
+	/*!
+	 *!  DMA device transfer restrictions
+	 */
+	IMG_UINT32 ui32DmaAlignment;
+	IMG_UINT32 ui32DmaTransferUnit;
+	/*!
+	 *!  System-wide presence of DMA capabilities
+	 */
+	IMG_BOOL bHasDma;
+
+	/*!
+	 *!  ES770x clk&reset required
+	 */
+    struct clk *aclk;
+    struct clk *gray_clk;
+    struct clk *cfg_clk;
+    struct reset_control    *rstc_axi;
+    struct reset_control    *rstc_cfg;
+    struct reset_control    *rstc_gray;
+    struct reset_control    *rstc_jones;
+    struct reset_control    *rstc_spu;
+
+	/*!
+	 *!  DriverMode required
+	 */
+	PVRSRV_DRIVER_MODE eDriverMode;
+};
+
+#endif /* PVRSRV_DEVICE_H*/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_pool.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_pool.h
new file mode 100644
index 000000000000..2272fc50ce6e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_pool.h
@@ -0,0 +1,135 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services pool implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a generic pool implementation.
+                The pool allows to dynamically retrieve and return entries from
+                it using functions pair PVRSRVPoolGet/PVRSRVPoolPut. The entries
+                are created in lazy manner which means not until first usage.
+                The pool API allows to pass and allocation/free functions
+                pair that will allocate entry's private data and return it
+                to the caller on every entry 'Get'.
+                The pool will keep up to ui32MaxEntries entries allocated.
+                Every entry that exceeds this number and is 'Put' back to the
+                pool will be freed on the spot instead being returned to the
+                pool.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PVRSRVPOOL_H)
+#define PVRSRVPOOL_H
+
+/**************************************************************************/ /*!
+ @Description  Callback function called during creation of the new element. This
+               function allocates an object that will be stored in the pool.
+               The object can be retrieved from the pool by calling
+               PVRSRVPoolGet.
+ @Input        pvPrivData      Private data passed to the alloc function.
+ @Output       pvOut           Allocated object.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+typedef PVRSRV_ERROR (PVRSRV_POOL_ALLOC_FUNC)(void *pvPrivData, void **pvOut);
+
+/**************************************************************************/ /*!
+ @Description  Callback function called to free the object allocated by
+               the counterpart alloc function.
+ @Input        pvPrivData      Private data passed to the free function.
+ @Output       pvFreeData      Object allocated by PVRSRV_POOL_ALLOC_FUNC.
+*/ /***************************************************************************/
+typedef void (PVRSRV_POOL_FREE_FUNC)(void *pvPrivData, void *pvFreeData);
+
+typedef IMG_HANDLE PVRSRV_POOL_TOKEN;
+
+typedef struct _PVRSRV_POOL_ PVRSRV_POOL;
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolCreate
+ @Description  Creates new buffer pool.
+ @Input        pfnAlloc        Allocation function pointer. Function is used
+                               to allocate new pool entries' data.
+ @Input        pfnFree         Free function pointer. Function is used to
+                               free memory allocated by pfnAlloc function.
+ @Input        ui32MaxEntries  Total maximum number of entries in the pool.
+ @Input        pszName         Name of the pool. String has to be NULL
+                               terminated.
+ @Input        pvPrivData      Private data that will be passed to pfnAlloc and
+                               pfnFree functions.
+ @Output       ppsPool         New buffer pool object.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolCreate(PVRSRV_POOL_ALLOC_FUNC *pfnAlloc,
+					PVRSRV_POOL_FREE_FUNC *pfnFree,
+					IMG_UINT32 ui32MaxEntries,
+					const IMG_CHAR *pszName,
+					void *pvPrivData,
+					PVRSRV_POOL **ppsPool);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolDestroy
+ @Description  Destroys pool created by PVRSRVPoolCreate.
+ @Input        psPool          Buffer pool object meant to be destroyed.
+*/ /***************************************************************************/
+void PVRSRVPoolDestroy(PVRSRV_POOL *psPool);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolGet
+ @Description  Retrieves an entry from a pool. If no free elements are
+               available new entry will be allocated.
+ @Input        psPool          Pointer to the pool.
+ @Output       hToken          Pointer to the entry handle.
+ @Output       ppvDataOut      Pointer to data stored in the entry (the data
+                               allocated by the pfnAlloc function).
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolGet(PVRSRV_POOL *psPool,
+						PVRSRV_POOL_TOKEN *hToken,
+						void **ppvDataOut);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolPut
+ @Description  Returns entry to the pool. If number of entries is greater
+               than ui32MaxEntries set during pool creation the entry will
+               be freed instead.
+ @Input        psPool          Pointer to the pool.
+ @Input        hToken          Entry handle.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolPut(PVRSRV_POOL *psPool,
+						PVRSRV_POOL_TOKEN hToken);
+
+#endif /* PVRSRVPOOL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_sync_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_sync_server.h
new file mode 100644
index 000000000000..5d1a10cd0a9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/pvrsrv_sync_server.h
@@ -0,0 +1,278 @@
+/**************************************************************************/ /*!
+@File
+@Title          Fence sync server interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_SYNC_SERVER_H
+#define PVRSRV_SYNC_SERVER_H
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#include "sync_fallback_server.h"
+#include "pvr_notifier.h"
+#include "img_types.h"
+#include "pvrsrv_sync_km.h"
+#elif defined(SUPPORT_NATIVE_FENCE_SYNC)
+#include "pvr_sync.h"
+#endif
+
+#include "rgxhwperf.h"
+
+#define SYNC_SW_TIMELINE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+#define SYNC_SW_FENCE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+
+typedef struct _SYNC_TIMELINE_OBJ_
+{
+	void *pvTlObj; /* Implementation specific timeline object */
+
+	PVRSRV_TIMELINE hTimeline; /* Reference to implementation-independent timeline object */
+} SYNC_TIMELINE_OBJ;
+
+typedef struct _SYNC_FENCE_OBJ_
+{
+	void *pvFenceObj; /* Implementation specific fence object */
+
+	PVRSRV_FENCE hFence; /* Reference to implementation-independent fence object */
+} SYNC_FENCE_OBJ;
+
+static inline void SyncClearTimelineObj(SYNC_TIMELINE_OBJ *psSTO)
+{
+	psSTO->pvTlObj = NULL;
+	psSTO->hTimeline = PVRSRV_NO_TIMELINE;
+}
+
+static inline IMG_BOOL SyncIsTimelineObjValid(const SYNC_TIMELINE_OBJ *psSTO)
+{
+	return (IMG_BOOL)(psSTO->pvTlObj != NULL);
+}
+
+static inline void SyncClearFenceObj(SYNC_FENCE_OBJ *psSFO)
+{
+	psSFO->pvFenceObj = NULL;
+	psSFO->hFence = PVRSRV_NO_FENCE;
+}
+
+static inline IMG_BOOL SyncIsFenceObjValid(const SYNC_FENCE_OBJ *psSFO)
+{
+	return (IMG_BOOL)(psSFO->pvFenceObj != NULL);
+}
+
+
+/* Mapping of each required function to its appropriate sync-implementation function */
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	#define SyncFenceWaitKM_                SyncFbFenceWait
+	#define SyncGetFenceObj_                SyncFbGetFenceObj
+	#define SyncFenceReleaseKM_             SyncFbFenceReleaseKM
+	#define SyncSWTimelineFenceCreateKM_    SyncFbSWTimelineFenceCreateKM
+	#define SyncSWTimelineAdvanceKM_        SyncFbSWTimelineAdvanceKM
+	#define SyncSWGetTimelineObj_           SyncFbSWGetTimelineObj
+	#define SyncSWTimelineReleaseKM_        SyncFbTimelineRelease
+	#define SyncDumpFence_                  SyncFbDumpFenceKM
+	#define SyncSWDumpTimeline_             SyncFbSWDumpTimelineKM
+#elif defined(SUPPORT_NATIVE_FENCE_SYNC)
+	#define SyncFenceWaitKM_                pvr_sync_fence_wait
+	#define SyncGetFenceObj_                pvr_sync_fence_get
+	#define SyncFenceReleaseKM_             pvr_sync_fence_release
+	#define SyncSWTimelineFenceCreateKM_    pvr_sync_sw_timeline_fence_create
+	#define SyncSWTimelineAdvanceKM_        pvr_sync_sw_timeline_advance
+	#define SyncSWGetTimelineObj_           pvr_sync_sw_timeline_get
+	#define SyncSWTimelineReleaseKM_        pvr_sync_sw_timeline_release
+	#define SyncDumpFence_                  sync_dump_fence
+	#define SyncSWDumpTimeline_             sync_sw_dump_timeline
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncFenceWaitKM
+
+@Description    Wait for all the sync points in the fence to be signalled.
+
+@Input          psFenceObj          Fence to wait on
+
+@Input          ui32TimeoutInMs     Maximum time to wait (in milliseconds)
+
+@Return         PVRSRV_OK               once the fence has been passed (all
+                                        containing check points have either
+                                        signalled or errored)
+                PVRSRV_ERROR_TIMEOUT    if the poll has exceeded the timeout
+                PVRSRV_ERROR_FAILED_DEPENDENCIES Other sync-impl specific error
+*/ /**************************************************************************/
+static inline PVRSRV_ERROR
+SyncFenceWaitKM(PVRSRV_DEVICE_NODE *psDevNode,
+                const SYNC_FENCE_OBJ *psFenceObj,
+                IMG_UINT32 ui32TimeoutInMs)
+{
+	PVRSRV_ERROR eError;
+
+	RGXSRV_HWPERF_SYNC_FENCE_WAIT(psDevNode->pvDevice,
+								  BEGIN,
+								  OSGetCurrentProcessID(),
+								  psFenceObj->hFence,
+								  ui32TimeoutInMs);
+
+	eError = SyncFenceWaitKM_(psFenceObj->pvFenceObj, ui32TimeoutInMs);
+
+	RGXSRV_HWPERF_SYNC_FENCE_WAIT(psDevNode->pvDevice,
+								  END,
+								  OSGetCurrentProcessID(),
+								  psFenceObj->hFence,
+								  ((eError == PVRSRV_OK) ?
+									  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_PASSED :
+									  ((eError == PVRSRV_ERROR_TIMEOUT) ?
+										  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_TIMEOUT :
+										  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_ERROR)));
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       SyncGetFenceObj
+
+@Description    Get the implementation specific server fence object from
+                opaque implementation independent PVRSRV_FENCE type.
+                When successful, this function gets a reference on the base
+                fence, which needs to be dropped using SyncFenceReleaseKM,
+                when fence object is no longer in use.
+
+@Input          iFence        Input opaque fence object
+
+@Output         psFenceObj    Pointer to implementation specific fence object
+
+@Return         PVRSRV_ERROR  PVRSRV_OK, on success
+*/ /**************************************************************************/
+static inline PVRSRV_ERROR
+SyncGetFenceObj(PVRSRV_FENCE iFence,
+                SYNC_FENCE_OBJ *psFenceObj)
+{
+	psFenceObj->hFence = iFence;
+	return SyncGetFenceObj_(iFence, &psFenceObj->pvFenceObj);
+}
+
+/*************************************************************************/ /*!
+@Function       SyncFenceReleaseKM
+
+@Description    Release reference on this fence.
+
+@Input          psFenceObj     Fence to be released
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static inline
+PVRSRV_ERROR SyncFenceReleaseKM(const SYNC_FENCE_OBJ *psFenceObj)
+{
+	return SyncFenceReleaseKM_(psFenceObj->pvFenceObj);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                      SW TIMELINE SPECIFIC FUNCTIONS                       */
+/*                                                                           */
+/*****************************************************************************/
+
+static inline PVRSRV_ERROR
+SyncSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            PVRSRV_TIMELINE hSWTimeline,
+                            const IMG_CHAR *pszFenceName,
+                            PVRSRV_FENCE *phOutFence)
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+	eError = SyncSWTimelineFenceCreateKM_(psDevNode,
+	                                      hSWTimeline,
+	                                      pszFenceName,
+	                                      phOutFence,
+	                                      &ui64SyncPtIdx);
+	if (eError == PVRSRV_OK)
+	{
+		RGXSRV_HWPERF_ALLOC_SW_FENCE(psDevNode, OSGetCurrentProcessID(),
+		                             *phOutFence, hSWTimeline, ui64SyncPtIdx,
+		                             pszFenceName, OSStringLength(pszFenceName));
+	}
+	return eError;
+}
+
+static inline PVRSRV_ERROR
+SyncSWTimelineAdvanceKM(PVRSRV_DEVICE_NODE *psDevNode,
+                        const SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+	eError = SyncSWTimelineAdvanceKM_(psSWTimelineObj->pvTlObj,
+	                                  &ui64SyncPtIdx);
+
+	if (eError == PVRSRV_OK)
+	{
+		RGXSRV_HWPERF_SYNC_SW_TL_ADV(psDevNode->pvDevice,
+		                             OSGetCurrentProcessID(),
+		                             psSWTimelineObj->hTimeline,
+		                             ui64SyncPtIdx);
+	}
+	return eError;
+}
+
+static inline PVRSRV_ERROR
+SyncSWGetTimelineObj(PVRSRV_TIMELINE hSWTimeline,
+                     SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	psSWTimelineObj->hTimeline = hSWTimeline;
+	return SyncSWGetTimelineObj_(hSWTimeline, &psSWTimelineObj->pvTlObj);
+}
+
+static inline PVRSRV_ERROR
+SyncSWTimelineReleaseKM(const SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	return SyncSWTimelineReleaseKM_(psSWTimelineObj->pvTlObj);
+}
+
+static inline PVRSRV_ERROR
+SyncDumpFence(const SYNC_FENCE_OBJ *psFenceObj,
+              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+              void *pvDumpDebugFile)
+{
+	return SyncDumpFence_(psFenceObj->pvFenceObj, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+static inline PVRSRV_ERROR
+SyncSWDumpTimeline(const SYNC_TIMELINE_OBJ *psSWTimelineObj,
+                   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                   void *pvDumpDebugFile)
+{
+	return SyncSWDumpTimeline_(psSWTimelineObj->pvTlObj, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+
+#endif /* PVRSRV_SYNC_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/rgx_firmware_boot.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/rgx_firmware_boot.h
new file mode 100644
index 000000000000..b0a14683e1a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/rgx_firmware_boot.h
@@ -0,0 +1,91 @@
+/**************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef RGX_FIRMWARE_BOOT_H
+#define RGX_FIRMWARE_BOOT_H
+
+#include "img_types.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+
+#define TD_MAX_NUM_MIPS_PAGETABLE_PAGES (4U)
+
+typedef union _PVRSRV_FW_BOOT_PARAMS_
+{
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCodeDevVAddr;
+		IMG_DEV_VIRTADDR sFWDataDevVAddr;
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_UINT32 ui32NumThreads;
+	} sMeta;
+
+	struct
+	{
+		IMG_DEV_PHYADDR sGPURegAddr;
+		IMG_DEV_PHYADDR asFWPageTableAddr[TD_MAX_NUM_MIPS_PAGETABLE_PAGES];
+		IMG_DEV_PHYADDR sFWStackAddr;
+		IMG_UINT32 ui32FWPageTableLog2PageSize;
+		IMG_UINT32 ui32FWPageTableNumPages;
+	} sMips;
+
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCodeDevVAddr;
+		IMG_DEV_VIRTADDR sFWDataDevVAddr;
+
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememDataSize;
+	} sRISCV;
+
+} PVRSRV_FW_BOOT_PARAMS;
+
+
+#endif /* RGX_FIRMWARE_BOOT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/ri_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/ri_server.h
new file mode 100644
index 000000000000..a8bcd9f32039
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/ri_server.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File           ri_server.h
+@Title          Resource Information abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RI_SERVER_H
+#define RI_SERVER_H
+
+#include "img_defs.h"
+#include "ri_typedefs.h"
+#include "pmr.h"
+#include "pvrsrv_error.h"
+#include "physheap.h"
+#include "connection_server.h"
+#include "device.h"
+
+PVRSRV_ERROR RIInitKM(void);
+void RIDeInitKM(void);
+
+void RILockAcquireKM(void);
+void RILockReleaseKM(void);
+
+PVRSRV_ERROR RIWritePMREntryKM(PMR *psPMR);
+
+PVRSRV_ERROR RIWritePMREntryWithOwnerKM(PMR *psPMR,
+                                        IMG_PID ui32Owner);
+
+PVRSRV_ERROR RIWriteMEMDESCEntryKM(void* psConnection,
+                                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   PMR *psPMR,
+                                   IMG_UINT32 ui32TextBSize,
+                                   const IMG_CHAR *psz8TextB,
+                                   IMG_UINT64 uiOffset,
+                                   IMG_UINT64 uiSize,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   RI_HANDLE *phRIHandle);
+
+PVRSRV_ERROR RIWriteProcListEntryKM(void* psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32TextBSize,
+                                    const IMG_CHAR *psz8TextB,
+                                    IMG_UINT64 ui64Size,
+                                    IMG_UINT64 ui64DevVAddr,
+                                    RI_HANDLE *phRIHandle);
+
+PVRSRV_ERROR RIUpdateMEMDESCAddrKM(RI_HANDLE hRIHandle,
+                                   IMG_DEV_VIRTADDR sVAddr);
+
+PVRSRV_ERROR RIDeletePMREntryKM(RI_HANDLE hRIHandle);
+PVRSRV_ERROR RIDeleteMEMDESCEntryKM(RI_HANDLE hRIHandle);
+
+PVRSRV_ERROR RIDeleteListKM(void);
+
+PVRSRV_ERROR RIDumpListKM(PMR *psPMR);
+
+PVRSRV_ERROR RIDumpAllKM(void);
+
+PVRSRV_ERROR RIDumpProcessKM(IMG_PID pid);
+
+#if defined(DEBUG)
+PVRSRV_ERROR RIDumpProcessListKM(PMR *psPMR,
+                                 IMG_PID pid,
+                                 IMG_UINT64 ui64Offset,
+                                 IMG_DEV_VIRTADDR *psDevVAddr);
+#endif
+
+void RIConnectionClosed(void* psConnection);
+
+PVRSRV_ERROR RIDeleteEntriesForPID(IMG_PID pid);
+
+IMG_BOOL RIGetListEntryKM(IMG_PID pid,
+                          IMG_HANDLE **ppHandle,
+                          IMG_CHAR **ppszEntryString);
+
+IMG_INT32 RITotalAllocProcessUnlocked(IMG_PID pid, PHYS_HEAP_TYPE ePhysHeapType);
+
+#endif /* RI_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_pvr.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_pvr.h
new file mode 100644
index 000000000000..48bc94d8ef0b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_pvr.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          SO Interface header file for common PVR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains SO interface functions. These functions are defined in
+                the common layer and are called from the env layer OS specific
+                implementation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SOFUNC_PVR_H_)
+#define SOFUNC_PVR_H_
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "device.h"
+#include "pvr_notifier.h"
+
+
+/**************************************************************************/ /*!
+ @Function     SOPvrDbgRequestNotifyRegister
+ @Description  SO Interface function called from the OS layer implementation.
+               Register a callback function that is called when a debug request
+               is made via a call PVRSRVDebugRequest. There are a number of
+               verbosity levels ranging from DEBUG_REQUEST_VERBOSITY_LOW up to
+               DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+               for each level up to the highest level specified to
+               PVRSRVDebugRequest.
+@Output        phNotify             On success, points to debug notifier handle
+@Input         psDevNode            Device node for which the debug callback
+                                    should be registered
+@Input         pfnDbgRequestNotify  Function callback
+@Input         ui32RequesterID      Requester ID. This is used to determine
+                                    the order in which callbacks are called,
+                                    see DEBUG_REQUEST_*
+@Input         hDbgReqeustHandle    Data to be passed back to the caller via
+                                    the callback function
+@Return        PVRSRV_ERROR         PVRSRV_OK on success and an error otherwise
+*/ /******************************************************************** ******/
+PVRSRV_ERROR SOPvrDbgRequestNotifyRegister(IMG_HANDLE *phNotify,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+							  IMG_UINT32 ui32RequesterID,
+							  PVRSRV_DBGREQ_HANDLE hDbgRequestHandle);
+
+/**************************************************************************/ /*!
+ @Function     SOPvrDbgRequestNotifyUnregister
+ @Description  SO Interface function called from the OS layer implementation.
+               Remove and clean up the specified notifier registration so that
+               it does not receive any further callbacks.
+ @Input        hNotify     Handle returned to caller from
+                           SOPvrDbgRequestNotifyRegister().
+ @Return       PVRSRV_ERROR
+*/ /***************************************************************************/
+PVRSRV_ERROR SOPvrDbgRequestNotifyUnregister(IMG_HANDLE hNotify);
+
+
+#endif /* SOFUNC_PVR_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_rgx.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_rgx.h
new file mode 100644
index 000000000000..be9594d95242
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sofunc_rgx.h
@@ -0,0 +1,95 @@
+/*************************************************************************/ /*!
+@File
+@Title          SO Interface header file for devices/RGX functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains SO interface functions. These functions are defined in
+                the common devices layer and are called from the env layer OS
+                specific implementation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SOFUNC_RGX_H_)
+#define SOFUNC_RGX_H_
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+
+#if !defined(NO_HARDWARE)
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsRegister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Initialise data used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument). This function must be called only once for each
+               different user/handle.
+
+ @Input        phGpuUtilUser - Pointer to handle used to identify a user of
+                               RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser);
+
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsUnregister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Free data previously used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument).
+
+ @Input        hGpuUtilUser - Handle used to identify a user of
+                              RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser);
+#endif /* !defined(NO_HARDWARE) */
+
+
+
+#endif /* SOFUNC_RGX_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/srvcore.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvcore.h
new file mode 100644
index 000000000000..8d1385f618d4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvcore.h
@@ -0,0 +1,274 @@
+/**************************************************************************/ /*!
+@File
+@Title          PVR Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the PVR Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVCORE_H
+#define SRVCORE_H
+
+#include "lock_types.h"
+#include "connection_server.h"
+#include "pvr_debug.h"
+
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+                    IMG_UINT32 ui32DispatchTableEntry,
+                    void *pvDest,
+                    void __user *pvSrc,
+                    IMG_UINT32 ui32Size);
+PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+                  IMG_UINT32 ui32DispatchTableEntry,
+                  void __user *pvDest,
+                  void *pvSrc,
+                  IMG_UINT32 ui32Size);
+#else
+FORCE_INLINE PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+					IMG_UINT32 ui32DispatchTableEntry,
+					void *pvDest,
+					void __user *pvSrc,
+					IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER (ui32DispatchTableEntry);
+	return OSBridgeCopyFromUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+FORCE_INLINE PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+				  IMG_UINT32 ui32DispatchTableEntry,
+				  void __user *pvDest,
+				  void *pvSrc,
+				  IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER (ui32DispatchTableEntry);
+	return OSBridgeCopyToUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+#endif
+
+IMG_INT
+DummyBW(IMG_UINT32 ui32DispatchTableEntry,
+        IMG_UINT8 *psBridgeIn,
+        IMG_UINT8 *psBridgeOut,
+        CONNECTION_DATA *psConnection);
+
+typedef PVRSRV_ERROR (*ServerResourceDestroyFunction)(IMG_HANDLE, IMG_HANDLE);
+
+typedef IMG_INT (*BridgeWrapperFunction)(IMG_UINT32 ui32DispatchTableEntry,
+									 IMG_UINT8 *psBridgeIn,
+									 IMG_UINT8 *psBridgeOut,
+									 CONNECTION_DATA *psConnection);
+
+typedef struct _PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY
+{
+	BridgeWrapperFunction pfFunction; /*!< The wrapper function that validates the ioctl
+	                                    arguments before calling into srvkm proper */
+	POS_LOCK hBridgeLock; /*!< The bridge lock which needs to be acquired
+                                   before calling the above wrapper */
+
+	IMG_UINT32 ui32InBufferSize; /*!< The expected size of the in buffer given by the client */
+	IMG_UINT32 ui32OutBufferSize; /*!< The expected size of the out buffer given by the client */
+
+#if defined(DEBUG_BRIDGE_KM)
+	const IMG_CHAR *pszIOCName; /*!< Name of the ioctl: e.g. "PVRSRV_BRIDGE_CONNECT_SERVICES" */
+	const IMG_CHAR *pszFunctionName; /*!< Name of the wrapper function: e.g. "PVRSRVConnectBW" */
+	const IMG_CHAR *pszBridgeLockName; /*!< Name of bridge lock which will be acquired */
+	IMG_UINT32 ui32CallCount; /*!< The total number of times the ioctl has been called */
+	IMG_UINT32 ui32CopyFromUserTotalBytes; /*!< The total number of bytes copied from
+	                                         userspace within this ioctl */
+	IMG_UINT32 ui32CopyToUserTotalBytes; /*!< The total number of bytes copied from
+	                                       userspace within this ioctl */
+	IMG_UINT64 ui64TotalTimeNS; /*!< The total amount of time spent in this bridge function */
+	IMG_UINT64 ui64MaxTimeNS; /*!< The maximum amount of time for a single call to this bridge function */
+#endif
+}PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY;
+
+#if defined(SUPPORT_RGX)
+	#define BRIDGE_DISPATCH_TABLE_ENTRY_COUNT  (PVRSRV_BRIDGE_RGX_DISPATCH_LAST+1)
+	#define BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT  (PVRSRV_BRIDGE_RGX_LAST+1)
+#else
+	#define BRIDGE_DISPATCH_TABLE_ENTRY_COUNT  (PVRSRV_BRIDGE_DISPATCH_LAST+1)
+	#define BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT  (PVRSRV_BRIDGE_LAST+1)
+#endif
+
+extern PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY g_BridgeDispatchTable[BRIDGE_DISPATCH_TABLE_ENTRY_COUNT];
+
+void BridgeDispatchTableStartOffsetsInit(void);
+
+void
+_SetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+                       IMG_UINT32 ui32Index,
+                       const IMG_CHAR *pszIOCName,
+                       BridgeWrapperFunction pfFunction,
+                       const IMG_CHAR *pszFunctionName,
+                       POS_LOCK hBridgeLock,
+                       const IMG_CHAR* pszBridgeLockName,
+                       IMG_UINT32 ui32InBufferSize,
+                       IMG_UINT32 ui32OutBufferSize);
+void
+UnsetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+                        IMG_UINT32 ui32Index);
+
+
+/* PRQA S 0884,3410 2*/ /* macro relies on the lack of brackets */
+#define SetDispatchTableEntry(ui32BridgeGroup, ui32Index, pfFunction,\
+                              hBridgeLock, ui32InBufferSize, ui32OutBufferSize) \
+	do \
+	{ \
+		static_assert((ui32InBufferSize) <= PVRSRV_MAX_BRIDGE_IN_SIZE, "Bridge input buffer is too small for bridge function: " #pfFunction); \
+		static_assert((ui32OutBufferSize) <= PVRSRV_MAX_BRIDGE_OUT_SIZE, "Bridge output buffer is too small for bridge function: " #pfFunction); \
+		_SetDispatchTableEntry(ui32BridgeGroup, ui32Index, #ui32Index, (BridgeWrapperFunction)pfFunction, #pfFunction,\
+		                       (POS_LOCK)hBridgeLock, #hBridgeLock, ui32InBufferSize, ui32OutBufferSize); \
+	} while (0)
+
+#define DISPATCH_TABLE_GAP_THRESHOLD 5
+
+
+#if defined(DEBUG_BRIDGE_KM)
+typedef struct _PVRSRV_BRIDGE_GLOBAL_STATS
+{
+	IMG_UINT32 ui32IOCTLCount;
+	IMG_UINT32 ui32TotalCopyFromUserBytes;
+	IMG_UINT32 ui32TotalCopyToUserBytes;
+} PVRSRV_BRIDGE_GLOBAL_STATS;
+
+void BridgeGlobalStatsLock(void);
+void BridgeGlobalStatsUnlock(void);
+
+/* OS specific code may want to report the stats held here and within the
+ * BRIDGE_DISPATCH_TABLE_ENTRYs (E.g. on Linux we report these via a
+ * debugfs entry /(sys/kernel/debug|proc)/pvr/bridge_stats) */
+extern PVRSRV_BRIDGE_GLOBAL_STATS g_BridgeGlobalStats;
+#endif
+
+PVRSRV_ERROR BridgeDispatcherInit(void);
+void BridgeDispatcherDeinit(void);
+
+PVRSRV_ERROR
+BridgedDispatchKM(CONNECTION_DATA * psConnection,
+                  PVRSRV_BRIDGE_PACKAGE   * psBridgePackageKM);
+
+PVRSRV_ERROR
+PVRSRVConnectKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE * psDeviceNode,
+                IMG_UINT32 ui32Flags,
+                IMG_UINT32 ui32ClientBuildOptions,
+                IMG_UINT32 ui32ClientDDKVersion,
+                IMG_UINT32 ui32ClientDDKBuild,
+                IMG_UINT8  *pui8KernelArch,
+                IMG_UINT32 *ui32CapabilityFlags,
+                IMG_UINT64 *ui64PackedBvnc);
+
+PVRSRV_ERROR
+PVRSRVDisconnectKM(void);
+
+PVRSRV_ERROR
+PVRSRVAcquireGlobalEventObjectKM(IMG_HANDLE *phGlobalEventObject);
+
+PVRSRV_ERROR
+PVRSRVReleaseGlobalEventObjectKM(IMG_HANDLE hGlobalEventObject);
+
+PVRSRV_ERROR
+PVRSRVDumpDebugInfoKM(CONNECTION_DATA *psConnection,
+                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32VerbLevel);
+
+PVRSRV_ERROR
+PVRSRVGetDevClockSpeedKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_PUINT32  pui32RGXClockSpeed);
+
+PVRSRV_ERROR
+PVRSRVHWOpTimeoutKM(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PVRSRVAlignmentCheckKM(CONNECTION_DATA *psConnection,
+                                    PVRSRV_DEVICE_NODE * psDeviceNode,
+                                    IMG_UINT32 ui32FWAlignChecksSize,
+                                    IMG_UINT32 aui32FWAlignChecks[]);
+
+PVRSRV_ERROR PVRSRVGetDeviceStatusKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 *pui32DeviceStatus);
+
+PVRSRV_ERROR PVRSRVGetMultiCoreInfoKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32CapsSize,
+                                     IMG_UINT32 *pui32NumCores,
+                                     IMG_UINT64 *pui64Caps);
+
+PVRSRV_ERROR PVRSRVFindProcessMemStatsKM(IMG_PID pid,
+                                         IMG_UINT32 ui32ArrSize,
+                                         IMG_BOOL bAllProcessStats,
+                                         IMG_UINT64 *pui64MemoryStats);
+
+static INLINE
+PVRSRV_ERROR DestroyServerResource(const SHARED_DEV_CONNECTION hConnection,
+                                   IMG_HANDLE hEvent,
+                                   ServerResourceDestroyFunction pfnDestroyCall,
+                                   IMG_HANDLE hResource)
+{
+    PVR_UNREFERENCED_PARAMETER(hEvent);
+
+    return pfnDestroyCall(GetBridgeHandle(hConnection), hResource);
+}
+
+/*************************************************************************/ /*!
+@Function      PVRSRVBlockIfFrozen
+@Description   Puts caller into a blocking wait (using event-objects) if the
+                specified device is in a FROZEN state. Routine completes and
+                returns control to the caller once the underlying device state
+                clears its FROZEN state.
+@Input         psDeviceNode     Device Node reference
+@Return        Nothing.         Execution blocks if device is frozen.
+*/ /**************************************************************************/
+void PVRSRVBlockIfFrozen(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* SRVCORE_H */
+
+/******************************************************************************
+ End of file (srvcore.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/srvinit.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvinit.h
new file mode 100644
index 000000000000..48e6863eae1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvinit.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          Initialisation server internal header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the connections between the various parts of the
+                initialisation server.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVINIT_H
+#define SRVINIT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "device_connection.h"
+#include "device.h"
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* SRVINIT_H */
+
+/******************************************************************************
+ End of file (srvinit.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/srvkm.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvkm.h
new file mode 100644
index 000000000000..29418764daa5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/srvkm.h
@@ -0,0 +1,144 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services kernel module internal header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_H
+#define SRVKM_H
+
+#include "servicesext.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+struct _PVRSRV_DEVICE_NODE_;
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDriverInit
+@Description  Performs one time driver initialisation of Services Common and
+              Device layers.
+@Return       PVRSRV_ERROR   PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVCommonDriverInit(void);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDriverInit
+@Description  Performs one time driver de-initialisation of Services.
+@Return       void
+*/ /**************************************************************************/
+void PVRSRVCommonDriverDeInit(void);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceCreate
+@Description  Creates and initialises a common layer Services device node
+              for an OS native device. First stage device discovery.
+@Input        pvOSDevice      OS native device
+@Input        i32KernelDeviceID A unique identifier which helps recognise this
+                              Device in the UM space provided by the OS.
+@Output       ppsDeviceNode   Points to the new device node on success
+@Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVCommonDeviceCreate(void *pvOSDevice, IMG_INT32 i32KernelDeviceID,
+				   struct _PVRSRV_DEVICE_NODE_ **ppsDeviceNode);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceInitialise
+@Description  Initialises the device layer specifics (e.g. boot FW etc)
+              for the supplied device node, created previously by
+              PVRSRVCommonDeviceCreate. The device is ready for use when this
+              second stage device initialisation returns successfully.
+@Input        psDeviceNode  Device node of the device to be initialised
+@Return       PVRSRV_ERROR  PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVCommonDeviceInitialise(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceDestroy
+@Description  Destroys a PVR Services device node.
+@Input        psDeviceNode  Device node to destroy
+*/ /**************************************************************************/
+void
+PVRSRVCommonDeviceDestroy(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+/******************
+HIGHER LEVEL MACROS
+*******************/
+
+/*----------------------------------------------------------------------------
+Repeats the body of the loop for a certain minimum time, or until the body
+exits by its own means (break, return, goto, etc.)
+
+Example of usage:
+
+LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+{
+	if (psQueueInfo->ui32ReadOffset == psQueueInfo->ui32WriteOffset)
+	{
+		bTimeout = IMG_FALSE;
+		break;
+	}
+
+	OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+} END_LOOP_UNTIL_TIMEOUT_US();
+
+-----------------------------------------------------------------------------*/
+
+/* uiNotLastLoop will remain at 1 until the timeout has expired, at which time
+ * it will be decremented and the loop executed one final time. This is
+ * necessary when preemption is enabled.
+ */
+/* PRQA S 3411,3431 12 */ /* critical format, leave alone */
+#define LOOP_UNTIL_TIMEOUT_US(TIMEOUT_US) \
+{\
+	IMG_UINT32 uiOffset, uiStart, uiCurrent; \
+	IMG_INT32 iNotLastLoop;					 \
+	for (uiOffset = 0, uiStart = OSClockus(), uiCurrent = uiStart + 1, iNotLastLoop = 1;\
+		((uiCurrent - uiStart + uiOffset) < (TIMEOUT_US)) || iNotLastLoop--;				\
+		uiCurrent = OSClockus(),													\
+		uiOffset = uiCurrent < uiStart ? IMG_UINT32_MAX - uiStart : uiOffset,		\
+		uiStart = uiCurrent < uiStart ? 0 : uiStart)
+
+#define END_LOOP_UNTIL_TIMEOUT_US() \
+}
+
+#endif /* SRVKM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint.h
new file mode 100644
index 000000000000..3c3846d7f0cd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint.h
@@ -0,0 +1,783 @@
+/*************************************************************************/ /*!
+@File
+@Title          Synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_H
+#define SYNC_CHECKPOINT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_sync_km.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+#include "dllist.h"
+#include "pvr_debug.h"
+#include "device_connection.h"
+#include "opaque_types.h"
+
+#ifndef CHECKPOINT_TYPES
+#define CHECKPOINT_TYPES
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG *PSYNC_CHECKPOINT_CONTEXT;
+
+typedef struct SYNC_CHECKPOINT_TAG *PSYNC_CHECKPOINT;
+#endif
+
+/* definitions for functions to be implemented by OS-specific sync - the OS-specific sync code
+   will call SyncCheckpointRegisterFunctions() when initialised, in order to register functions
+   we can then call */
+#ifndef CHECKPOINT_PFNS
+#define CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN)(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                                             PVRSRV_FENCE fence,
+                                                             IMG_UINT32 *nr_checkpoints,
+                                                             PSYNC_CHECKPOINT **checkpoint_handles,
+                                                             IMG_UINT64 *pui64FenceUID);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN)(PPVRSRV_DEVICE_NODE device,
+                                                            const IMG_CHAR *fence_name,
+                                                            PVRSRV_TIMELINE timeline,
+                                                            PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                                            PVRSRV_FENCE *new_fence,
+                                                            IMG_UINT64 *pui64FenceUID,
+                                                            void **ppvFenceFinaliseData,
+                                                            PSYNC_CHECKPOINT *new_checkpoint_handle,
+                                                            IMG_HANDLE *timeline_update_sync,
+                                                            IMG_UINT32 *timeline_update_value);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN)(PVRSRV_FENCE fence_to_rollback, void *finalise_data);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN)(PVRSRV_FENCE fence_to_finalise, void *finalise_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN)(void *private_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN)(PVRSRV_FENCE fence_to_signal);
+typedef void (*PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN)(void *mem_ptr);
+typedef IMG_UINT32 (*PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN)(IMG_UINT32 num_ufos, IMG_UINT32 *vaddrs);
+#if defined(PDUMP)
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN)(PVRSRV_FENCE iFence,
+									IMG_UINT32 *puiNumCheckpoints,
+									PSYNC_CHECKPOINT **papsCheckpoints);
+#endif
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN)(PVRSRV_FENCE iExportFence,
+                                                             PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+                                                             PSYNC_CHECKPOINT *checkpoint_handle);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN)(PVRSRV_FENCE iExportFence);
+
+#define SYNC_CHECKPOINT_IMPL_MAX_STRLEN 20
+
+typedef struct
+{
+	PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN pfnFenceResolve;
+	PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN pfnFenceCreate;
+	PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN pfnFenceDataRollback;
+	PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN pfnFenceFinalise;
+	PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN pfnNoHWUpdateTimelines;
+	PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN pfnNoHWSignalExpFence;
+	PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN pfnFreeCheckpointListMem;
+	PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN pfnDumpInfoOnStalledUFOs;
+	IMG_CHAR pszImplName[SYNC_CHECKPOINT_IMPL_MAX_STRLEN];
+#if defined(PDUMP)
+	PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN pfnSyncFenceGetCheckpoints;
+#endif
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN pfnExportFenceResolve;
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN pfnExportFenceRollback;
+} PFN_SYNC_CHECKPOINT_STRUCT;
+
+PVRSRV_ERROR SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns);
+
+#endif /* ifndef CHECKPOINT_PFNS */
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextCreate
+
+@Description    Create a new synchronisation checkpoint context
+
+@Input          psDevNode                 Device node
+
+@Output         ppsSyncCheckpointContext  Handle to the created synchronisation
+                                          checkpoint context
+
+@Return         PVRSRV_OK if the synchronisation checkpoint context was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointContextCreate(PPVRSRV_DEVICE_NODE psDevNode,
+                            PSYNC_CHECKPOINT_CONTEXT *ppsSyncCheckpointContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextDestroy
+
+@Description    Destroy a synchronisation checkpoint context
+
+@Input          psSyncCheckpointContext  Handle to the synchronisation
+                                         checkpoint context to destroy
+
+@Return         PVRSRV_OK if the synchronisation checkpoint context was
+                successfully destroyed.
+                PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT if the context still
+                has sync checkpoints defined
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextRef
+
+@Description    Takes a reference on a synchronisation checkpoint context
+
+@Input          psContext  Handle to the synchronisation checkpoint context
+                           on which a ref is to be taken
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextUnref
+
+@Description    Drops a reference taken on a synchronisation checkpoint
+                context
+
+@Input          psContext  Handle to the synchronisation checkpoint context
+                           on which the ref is to be dropped
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointAlloc
+
+@Description    Allocate a new synchronisation checkpoint on the specified
+                synchronisation checkpoint context
+
+@Input          hSyncCheckpointContext  Handle to the synchronisation
+                                        checkpoint context
+
+@Input          hTimeline               Timeline on which this sync
+                                        checkpoint is being created
+
+@Input          hFence                  Fence as passed into pfnFenceResolve
+                                        API, when the API encounters a non-PVR
+                                        fence as part of its input fence. From
+                                        all other places this argument must be
+                                        PVRSRV_NO_FENCE.
+
+@Input          pszClassName            Sync checkpoint source annotation
+                                        (will be truncated to at most
+                                         PVRSRV_SYNC_NAME_LENGTH chars)
+
+@Output         ppsSyncCheckpoint       Created synchronisation checkpoint
+
+@Return         PVRSRV_OK if the synchronisation checkpoint was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                    PVRSRV_TIMELINE hTimeline,
+                    PVRSRV_FENCE hFence,
+                    const IMG_CHAR *pszCheckpointName,
+                    PSYNC_CHECKPOINT *ppsSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFree
+
+@Description    Free a synchronisation checkpoint
+                The reference count held for the synchronisation checkpoint
+                is decremented - if it has becomes zero, it is also freed.
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to free
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointSignal
+
+@Description    Signal the synchronisation checkpoint
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to signal
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointSignalNoHW
+
+@Description    Signal the synchronisation checkpoint in NO_HARDWARE build
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to signal
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointSignalNoHW(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointError
+
+@Description    Error the synchronisation checkpoint
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to error
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointStateFromUFO
+
+@Description    Returns the current state of the synchronisation checkpoint
+                which has the given UFO firmware address
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint
+
+@Return         The current state (32-bit value) of the sync checkpoint
+*/
+/*****************************************************************************/
+IMG_UINT32 SyncCheckpointStateFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointErrorFromUFO
+
+@Description    Error the synchronisation checkpoint which has the
+                given UFO firmware address
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to be errored
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to be errored
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointErrorFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackFromUFO
+
+@Description    Drop the enqueued count reference taken on the synchronisation
+                checkpoint on behalf of the firmware.
+                Called in the event of a DM Kick failing.
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to be rolled back
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to be rolled back
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointRollbackFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointIsSignalled
+
+@Description    Returns IMG_TRUE if the synchronisation checkpoint is
+                signalled or errored
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to test
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_BOOL
+SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint,
+                          IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointIsErrored
+
+@Description    Returns IMG_TRUE if the synchronisation checkpoint is
+                errored
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to test
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_BOOL
+SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint,
+                        IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointTakeRef
+
+@Description    Take a reference on a synchronisation checkpoint
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to take a
+                                        reference on
+
+@Return         PVRSRV_OK if a reference was taken on the synchronisation
+                primitive
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDropRef
+
+@Description    Drop a reference on a synchronisation checkpoint
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to drop a
+                                        reference on
+
+@Return         PVRSRV_OK if a reference was dropped on the synchronisation
+                primitive
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointResolveFence
+
+@Description    Resolve a fence, returning a list of the sync checkpoints
+                that fence contains.
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          psSyncCheckpointContext The sync checkpoint context
+                                        on which checkpoints should be
+                                        created (in the event of the fence
+                                        having a native sync pt with no
+                                        associated sync checkpoint)
+
+@Input          hFence                  The fence to be resolved
+
+@Output         pui32NumSyncCheckpoints The number of sync checkpoints the
+                                        fence contains. Can return 0 if
+                                        passed a null (-1) fence.
+
+@Output         papsSyncCheckpoints     List of sync checkpoints the fence
+                                        contains
+
+@Output         puiFenceUID             Unique ID of the resolved fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointResolveFence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PVRSRV_FENCE hFence,
+                           IMG_UINT32 *pui32NumSyncCheckpoints,
+                           PSYNC_CHECKPOINT **papsSyncCheckpoints,
+                           IMG_UINT64 *puiFenceUID,
+                           PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointResolveExportFence
+
+@Description    Resolve an export fence, returning the sync checkpoint
+                that fence contains.
+                This function in turn calls a function provided by the
+                sync implementation.
+
+@Input          hExportFence            The export fence to be resolved
+
+@Input          psSyncCheckpointContext The context in which to create the
+                                        new sync checkpoint for the export fence
+
+@Output         ppsSyncCheckpoint       The sync checkpoint the fence
+                                        contains
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointResolveExportFence(PVRSRV_FENCE hExportFence,
+                           PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PSYNC_CHECKPOINT *ppsSyncCheckpoint,
+                           PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackExportFence
+
+@Description    Rollback an export fence, freeing the sync checkpoint
+                that fence was assigned.
+                This function in turn calls a function provided by the
+                sync implementation.
+
+@Input          hExportFence            The export fence to be rolled back
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointRollbackExportFence(PVRSRV_FENCE hExportFence);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCreateFence
+
+@Description    Create a fence containing a single sync checkpoint.
+                Return the fence and a ptr to sync checkpoint it contains.
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          pszFenceName            String to assign to the new fence
+                                        (for debugging purposes)
+
+@Input          hTimeline               Timeline on which the new fence is
+                                        to be created
+
+@Input          psSyncCheckpointContext Sync checkpoint context to be used
+                                        when creating the new fence
+
+@Output         phNewFence              The newly created fence
+
+@Output         pui64FenceUID           Unique ID of the created fence
+
+@Output         ppvFenceFinaliseData    Any data needed to finalise the fence
+                                        in a later call to the function
+                                        SyncCheckpointFinaliseFence()
+
+@Output         psNewSyncCheckpoint     The sync checkpoint contained in
+                                        the new fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointCreateFence(PPVRSRV_DEVICE_NODE psDeviceNode,
+                          const IMG_CHAR *pszFenceName,
+                          PVRSRV_TIMELINE hTimeline,
+                          PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                          PVRSRV_FENCE *phNewFence,
+                          IMG_UINT64 *pui64FenceUID,
+                          void **ppvFenceFinaliseData,
+                          PSYNC_CHECKPOINT *psNewSyncCheckpoint,
+                          void **ppvTimelineUpdateSyncPrim,
+                          IMG_UINT32 *pui32TimelineUpdateValue,
+                          PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackFenceData
+
+@Description    'Rolls back' the fence specified (destroys the fence and
+                takes any other required actions to undo the fence
+                creation (eg if the implementation wishes to revert the
+                incrementing of the fence's timeline, etc).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          hFence                  Fence to be 'rolled back'
+
+@Input          pvFinaliseData          Data needed to finalise the
+                                        fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointRollbackFenceData(PVRSRV_FENCE hFence, void *pvFinaliseData);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFinaliseFence
+
+@Description    'Finalise' the fence specified (performs any actions the
+                underlying implementation may need to perform just prior
+                to the fence being returned to the client.
+                This function in turn calls a function provided by the
+                OS native sync implementation - if the native sync
+                implementation does not need to perform any actions at
+                this time, this function does not need to be registered.
+
+@Input          psDevNode               Device node
+
+@Input          hFence                  Fence to be 'finalised'
+
+@Input          pvFinaliseData          Data needed to finalise the fence
+
+@Input          psSyncCheckpoint        Base sync checkpoint that this fence
+                                        is formed of
+
+@Input          pszName                 Fence annotation
+
+@Return         PVRSRV_OK if a valid fence and finalise data were provided.
+                PVRSRV_ERROR_INVALID_PARAMS if an invalid fence or finalise
+                data were provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function (permitted).
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointFinaliseFence(PPVRSRV_DEVICE_NODE psDevNode,
+                            PVRSRV_FENCE hFence,
+                            void *pvFinaliseData,
+                            PSYNC_CHECKPOINT psSyncCheckpoint,
+                            const IMG_CHAR *pszName);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFreeCheckpointListMem
+
+@Description    Free memory the memory which was allocated by the sync
+                implementation and used to return the list of sync
+                checkpoints when resolving a fence.
+                to the fence being returned to the client.
+                This function in turn calls a free function registered by
+                the sync implementation (if a function has been registered).
+
+@Input          pvCheckpointListMem     Pointer to the memory to be freed
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointFreeCheckpointListMem(void *pvCheckpointListMem);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointNoHWUpdateTimelines
+
+@Description    Called by the DDK in a NO_HARDWARE build only.
+                After syncs have been manually signalled by the DDK, this
+                function is called to allow the OS native sync implementation
+                to update its timelines (as the usual callback notification
+                of signalled checkpoints is not supported for NO_HARDWARE).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          pvPrivateData            Any data the OS native sync
+                                         implementation might require.
+
+@Return         PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function, otherwise
+                PVRSRV_OK.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointNoHWUpdateTimelines(void *pvPrivateData);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointNoHWSignalExportFence
+
+@Description    Called by the DDK in a NO_HARDWARE build only.
+                After syncs have been manually signalled by the DDK, this
+                function is called to allow the OS native sync implementation
+                to signal the export fence (as the usual callback notification
+                of signalled checkpoints is not supported for NO_HARDWARE).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          iExportFenceToSignal     The export fence to signal.
+
+@Return         PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function, otherwise
+                PVRSRV_OK.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointNoHWSignalExportFence(PVRSRV_FENCE iExportFenceToSignal);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDumpInfoOnStalledUFOs
+
+@Description    Called by the DDK in the event of the health check watchdog
+                examining the CCBs and determining that one has failed to
+                progress after 10 second when the GPU is idle due to waiting
+                on one or more UFO fences.
+                The DDK will pass a list of UFOs on which the CCB is waiting
+                and the sync implementation will check them to see if any
+                relate to sync points it has created. If so, the
+                implementation should dump debug information on those sync
+                points to the kernel log or other suitable output (which will
+                allow the unsignalled syncs to be identified).
+                The function shall return the number of syncs in the provided
+                array that were syncs which it had created.
+
+@Input          ui32NumUFOs           The number of UFOs in the array passed
+                                      in the pui32VAddrs parameter.
+                pui32Vaddr            The array of UFOs the CCB is waiting on.
+
+@Output         pui32NumSyncOwnedUFOs The number of UFOs in pui32Vaddr which
+                                      relate to syncs created by the sync
+                                      implementation.
+
+@Return         PVRSRV_OK if a valid pointer is provided in pui32NumSyncOwnedUFOs.
+                PVRSRV_ERROR_INVALID_PARAMS if a NULL value is provided in
+                pui32NumSyncOwnedUFOs.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointDumpInfoOnStalledUFOs(IMG_UINT32 ui32NumUFOs,
+                                    IMG_UINT32 *pui32Vaddrs,
+                                    IMG_UINT32 *pui32NumSyncOwnedUFOs);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetStateString
+
+@Description    Called to get a string representing the current state of a
+                sync checkpoint.
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get the
+                                        state for.
+
+@Return         The string representing the current state of this checkpoint
+*/
+/*****************************************************************************/
+const IMG_CHAR *
+SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRecordLookup
+
+@Description    Returns a debug string with information about the
+                sync checkpoint.
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to lookup
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to lookup
+
+@Input          pszSyncInfo             Character array to write to
+
+@Input          len                     Len of the character array
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointRecordLookup(PPVRSRV_DEVICE_NODE psDevNode,
+                           IMG_UINT32 ui32FwAddr,
+                           IMG_CHAR * pszSyncInfo, size_t len);
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       PVRSRVSyncCheckpointFencePDumpPolKM
+
+@Description    Called to insert a poll into the PDump script on a given
+                Fence being signalled or errored.
+
+@Input          hFence        Fence for PDump to poll on
+
+@Return         PVRSRV_OK if a valid sync checkpoint was provided.
+*/
+/*****************************************************************************/
+
+PVRSRV_ERROR PVRSRVSyncCheckpointSignalledPDumpPolKM(PVRSRV_FENCE hFence);
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCommonDeviceIDs
+
+@Description    Determine if the DeviceIDs referenced by the checkpoint context
+                and device reference are to the same physical device.
+
+@Input          psSyncContext  Handle to the synchronisation checkpoint context
+
+@Input          hDevRef        Handle to the device reference to check against
+
+@Return         IMG_TRUE if devices refer to the same device, or if one of the
+                parameters is NULL.
+                IMG_FALSE if the two references are to different physical
+                devices.
+*/
+/*****************************************************************************/
+IMG_BOOL SyncCheckpointCommonDeviceIDs(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                                       IMG_HANDLE hDevRef);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetCounters
+
+@Description    Return the Current In Use and Max In Use Sync Checkpoint
+                counters for the specified sync context.
+
+@Input          psDevNode      Handle to the device-node
+
+@Output         puiInUse       Number of Sync CPs currently in-use
+
+@Output         puiMax         Maximum number of Sync CPs allocated
+
+@Output         puiXDInUse     Number of X-D Sync CPs currently in-use
+
+@Output         puiXDMax       Maximum number of X-D Sync CPs used
+
+@Return         PVRSRV_OK if a valid sync checkpoint context is passed.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR SyncCheckpointGetCounters(PPVRSRV_DEVICE_NODE psDevNode,
+                                       IMG_UINT32 *puiInUse,
+                                       IMG_UINT32 *puiMax,
+                                       IMG_UINT32 *puiXDInUse,
+                                       IMG_UINT32 *puiXDMax);
+#endif /* SYNC_CHECKPOINT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint_init.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint_init.h
new file mode 100644
index 000000000000..94f2e000ded9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_checkpoint_init.h
@@ -0,0 +1,82 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation checkpoint initialisation interface
+                header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines synchronisation checkpoint structures that are visible
+                internally and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_INIT_H
+#define SYNC_CHECKPOINT_INIT_H
+
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointInit
+
+@Description    Initialise the sync checkpoint driver by giving it the
+                device node (needed to determine the pfnUFOAlloc function
+                to call in order to allocate sync block memory).
+
+@Input          psDevNode               Device for which sync checkpoints
+                                        are being initialised
+
+@Return         PVRSRV_OK               initialised successfully,
+                PVRSRV_ERROR_<error>    otherwise
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointInit(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDeinit
+
+@Description    Deinitialise the sync checkpoint driver.
+                Frees resources allocated during initialisation.
+
+@Input          psDevNode               Device for which sync checkpoints
+                                        are being de-initialised
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+#endif /* SYNC_CHECKPOINT_INIT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_fallback_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_fallback_server.h
new file mode 100644
index 000000000000..ac6bd4755b0e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_fallback_server.h
@@ -0,0 +1,204 @@
+/**************************************************************************/ /*!
+@File
+@Title          Fallback sync interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SYNC_FALLBACK_SERVER_H
+#define SYNC_FALLBACK_SERVER_H
+
+#include "img_types.h"
+#include "sync_checkpoint.h"
+#include "device.h"
+#include "connection_server.h"
+
+
+typedef struct _PVRSRV_TIMELINE_SERVER_ PVRSRV_TIMELINE_SERVER;
+typedef struct _PVRSRV_FENCE_SERVER_ PVRSRV_FENCE_SERVER;
+typedef struct _PVRSRV_FENCE_EXPORT_ PVRSRV_FENCE_EXPORT;
+
+typedef struct _PVRSRV_SYNC_PT_ PVRSRV_SYNC_PT;
+
+#define SYNC_FB_TIMELINE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+#define SYNC_FB_FENCE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         SW SPECIFIC FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbTimelineCreateSW(IMG_UINT32 uiTimelineNameSize,
+                                    const IMG_CHAR *pszTimelineName,
+                                    PVRSRV_TIMELINE_SERVER **ppsTimeline);
+
+PVRSRV_ERROR SyncFbFenceCreateSW(CONNECTION_DATA *psConnection,
+                                 PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 PVRSRV_TIMELINE_SERVER *psTimeline,
+                                 IMG_UINT32 uiFenceNameSize,
+                                 const IMG_CHAR *pszFenceName,
+                                 PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                 IMG_UINT64 *pui64SyncPtIdx);
+PVRSRV_ERROR SyncFbSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           PVRSRV_TIMELINE iSWTimeline,
+                                           const IMG_CHAR *pszFenceName,
+                                           PVRSRV_FENCE *piOutputFence,
+                                           IMG_UINT64* pui64SyncPtIdx);
+
+PVRSRV_ERROR SyncFbTimelineAdvanceSW(PVRSRV_TIMELINE_SERVER *psTimeline,
+                                     IMG_UINT64 *pui64SyncPtIdx);
+PVRSRV_ERROR SyncFbSWTimelineAdvanceKM(void *pvSWTimelineObj,
+                                       IMG_UINT64* pui64SyncPtIdx);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         PVR SPECIFIC FUNCTIONS                            */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbTimelineCreatePVR(IMG_UINT32 uiTimelineNameSize,
+                                     const IMG_CHAR *pszTimelineName,
+                                     PVRSRV_TIMELINE_SERVER **ppsTimeline);
+
+PVRSRV_ERROR SyncFbFenceCreatePVR(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const IMG_CHAR *pszName,
+                                  PVRSRV_TIMELINE iTl,
+                                  PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext,
+                                  PVRSRV_FENCE *piOutFence,
+                                  IMG_UINT64 *puiFenceUID,
+                                  void **ppvFenceFinaliseData,
+                                  PSYNC_CHECKPOINT *ppsOutCheckpoint,
+                                  void **ppvTimelineUpdateSync,
+                                  IMG_UINT32 *puiTimelineUpdateValue);
+
+PVRSRV_ERROR SyncFbFenceResolvePVR(PSYNC_CHECKPOINT_CONTEXT psContext,
+                                   PVRSRV_FENCE iFence,
+                                   IMG_UINT32 *puiNumCheckpoints,
+                                   PSYNC_CHECKPOINT **papsCheckpoints,
+                                   IMG_UINT64 *puiFenceUID);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         GENERIC FUNCTIONS                                 */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbGetFenceObj(PVRSRV_FENCE iFence,
+                               void **ppvFenceObj);
+
+PVRSRV_ERROR SyncFbSWGetTimelineObj(PVRSRV_TIMELINE iSWTimeline,
+                                    void **ppvSWTimelineObj);
+
+PVRSRV_ERROR SyncFbTimelineRelease(PVRSRV_TIMELINE_SERVER *psTl);
+
+PVRSRV_ERROR SyncFbFenceRelease(PVRSRV_FENCE_SERVER *psFence);
+PVRSRV_ERROR SyncFbFenceReleaseKM(void *pvFenceObj);
+
+PVRSRV_ERROR SyncFbFenceDup(PVRSRV_FENCE_SERVER *psInFence,
+                            PVRSRV_FENCE_SERVER **ppsOutFence);
+
+PVRSRV_ERROR SyncFbFenceMerge(PVRSRV_FENCE_SERVER *psInFence1,
+                              PVRSRV_FENCE_SERVER *psInFence2,
+                              IMG_UINT32 uiFenceNameSize,
+                              const IMG_CHAR *pszFenceName,
+                              PVRSRV_FENCE_SERVER **ppsOutFence);
+
+PVRSRV_ERROR SyncFbFenceWait(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiTimeout);
+
+PVRSRV_ERROR SyncFbFenceDump(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiLine,
+                             IMG_UINT32 uiFileNameLength,
+                             const IMG_CHAR *pszFile,
+                             IMG_UINT32 uiModuleLength,
+                             const IMG_CHAR *pszModule,
+                             IMG_UINT32 uiDescLength,
+                             const IMG_CHAR *pszDesc);
+
+PVRSRV_ERROR SyncFbDumpFenceKM(void *pvSWFenceObj,
+	                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	                           void *pvDumpDebugFile);
+
+PVRSRV_ERROR SyncFbSWDumpTimelineKM(void *pvSWTimelineObj,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile);
+
+PVRSRV_ERROR SyncFbRegisterSyncFunctions(void);
+
+PVRSRV_ERROR SyncFbRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR SyncFbDeregisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+IMG_UINT32 SyncFbDumpInfoOnStalledUFOs(IMG_UINT32 nr_ufos, IMG_UINT32 *vaddrs);
+
+IMG_BOOL SyncFbCheckpointHasSignalled(IMG_UINT32 ui32FwAddr, IMG_UINT32 ui32Value);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                       IMPORT/EXPORT FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR SyncFbFenceExportInsecure(PVRSRV_FENCE_SERVER *psFence,
+                                       PVRSRV_FENCE_EXPORT **ppExport);
+
+PVRSRV_ERROR SyncFbFenceExportDestroyInsecure(PVRSRV_FENCE_EXPORT *psExport);
+
+PVRSRV_ERROR SyncFbFenceImportInsecure(CONNECTION_DATA *psConnection,
+                                       PVRSRV_DEVICE_NODE *psDevice,
+                                       PVRSRV_FENCE_EXPORT *psImport,
+                                       PVRSRV_FENCE_SERVER **psFence);
+#endif /* defined(SUPPORT_INSECURE_EXPORT) */
+
+PVRSRV_ERROR SyncFbFenceExportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE * psDevNode,
+                                     PVRSRV_FENCE_SERVER *psFence,
+                                     IMG_SECURE_TYPE *phSecure,
+                                     PVRSRV_FENCE_EXPORT **ppsExport,
+                                     CONNECTION_DATA **ppsSecureConnection);
+
+PVRSRV_ERROR SyncFbFenceExportDestroySecure(PVRSRV_FENCE_EXPORT *psExport);
+
+PVRSRV_ERROR SyncFbFenceImportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDevice,
+                                     IMG_SECURE_TYPE hSecure,
+                                     PVRSRV_FENCE_SERVER **psFence);
+
+#endif /* SYNC_FALLBACK_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_server.h
new file mode 100644
index 000000000000..e35682970af6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/sync_server.h
@@ -0,0 +1,249 @@
+/**************************************************************************/ /*!
+@File
+@Title          Server side synchronisation interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Describes the server side synchronisation functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv.h"
+#include "device.h"
+#include "devicemem.h"
+#include "pdump.h"
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+#include "pdump_km.h"
+
+#ifndef SYNC_SERVER_H
+#define SYNC_SERVER_H
+
+typedef struct _SYNC_PRIMITIVE_BLOCK_ SYNC_PRIMITIVE_BLOCK;
+typedef struct _SYNC_CONNECTION_DATA_ SYNC_CONNECTION_DATA;
+typedef struct SYNC_RECORD* SYNC_RECORD_HANDLE;
+
+typedef struct _SYNC_ADDR_LIST_
+{
+	IMG_UINT32 ui32NumSyncs;
+	PRGXFWIF_UFO_ADDR *pasFWAddrs;
+} SYNC_ADDR_LIST;
+
+PVRSRV_ERROR
+SyncPrimitiveBlockToFWAddr(SYNC_PRIMITIVE_BLOCK *psSyncPrimBlock,
+						IMG_UINT32 ui32Offset,
+						PRGXFWIF_UFO_ADDR *psAddrOut);
+
+void
+SyncAddrListInit(SYNC_ADDR_LIST *psList);
+
+void
+SyncAddrListDeinit(SYNC_ADDR_LIST *psList);
+
+PVRSRV_ERROR
+SyncAddrListPopulate(SYNC_ADDR_LIST *psList,
+						IMG_UINT32 ui32NumSyncs,
+						SYNC_PRIMITIVE_BLOCK **apsSyncPrimBlock,
+						IMG_UINT32 *paui32SyncOffset);
+
+PVRSRV_ERROR
+SyncAddrListAppendSyncPrim(SYNC_ADDR_LIST          *psList,
+						   PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim);
+PVRSRV_ERROR
+SyncAddrListAppendCheckpoints(SYNC_ADDR_LIST *psList,
+								IMG_UINT32 ui32NumCheckpoints,
+								PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+PVRSRV_ERROR
+SyncAddrListAppendAndDeRefCheckpoints(SYNC_ADDR_LIST *psList,
+									  IMG_UINT32 ui32NumCheckpoints,
+									  PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+void
+SyncAddrListDeRefCheckpoints(IMG_UINT32 ui32NumCheckpoints,
+							 PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+PVRSRV_ERROR
+SyncAddrListRollbackCheckpoints(PVRSRV_DEVICE_NODE *psDevNode, SYNC_ADDR_LIST *psList);
+
+PVRSRV_ERROR
+PVRSRVAllocSyncPrimitiveBlockKM(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+								SYNC_PRIMITIVE_BLOCK **ppsSyncBlk,
+								IMG_UINT32 *puiSyncPrimVAddr,
+								IMG_UINT32 *puiSyncPrimBlockSize,
+								PMR        **ppsSyncPMR);
+
+PVRSRV_ERROR
+PVRSRVExportSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+								 DEVMEM_EXPORTCOOKIE **psExportCookie);
+
+PVRSRV_ERROR
+PVRSRVUnexportSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk);
+
+PVRSRV_ERROR
+PVRSRVFreeSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *ppsSyncBlk);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimSetKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Index,
+					IMG_UINT32 ui32Value);
+
+PVRSRV_ERROR
+PVRSRVSyncAllocEventKM(CONNECTION_DATA *psConnection,
+					   PVRSRV_DEVICE_NODE *psDevNode,
+					   IMG_BOOL bServerSync,
+                       IMG_UINT32 ui32FWAddr,
+                       IMG_UINT32 ui32ClassNameSize,
+                       const IMG_CHAR *pszClassName);
+
+PVRSRV_ERROR
+PVRSRVSyncFreeEventKM(CONNECTION_DATA *psConnection,
+					   PVRSRV_DEVICE_NODE *psDevNode,
+					   IMG_UINT32 ui32FWAddr);
+
+PVRSRV_ERROR
+PVRSRVSyncRecordAddKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDevNode,
+					  SYNC_RECORD_HANDLE *phRecord,
+					  SYNC_PRIMITIVE_BLOCK *hServerSyncPrimBlock,
+					  IMG_UINT32 ui32FwBlockAddr,
+					  IMG_UINT32 ui32SyncOffset,
+					  IMG_BOOL bServerSync,
+					  IMG_UINT32 ui32ClassNameSize,
+					  const IMG_CHAR *pszClassName);
+
+PVRSRV_ERROR
+PVRSRVSyncRecordRemoveByHandleKM(
+			SYNC_RECORD_HANDLE hRecord);
+void SyncRecordLookup(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 ui32FwAddr,
+					  IMG_CHAR * pszSyncInfo, size_t len);
+
+PVRSRV_ERROR SyncRegisterConnection(SYNC_CONNECTION_DATA **ppsSyncConnectionData);
+void SyncUnregisterConnection(SYNC_CONNECTION_DATA *ppsSyncConnectionData);
+void SyncConnectionPDumpSyncBlocks(PVRSRV_DEVICE_NODE *psDevNode, void *hSyncPrivData, PDUMP_TRANSITION_EVENT eEvent);
+
+/*!
+******************************************************************************
+@Function      SyncServerInit
+
+@Description   Per-device initialisation for the ServerSync module
+******************************************************************************/
+PVRSRV_ERROR SyncServerInit(PVRSRV_DEVICE_NODE *psDevNode);
+void SyncServerDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+							IMG_UINT32 ui32Value);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiDumpFlags);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize);
+
+#else	/* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpValueKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+							IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpPolKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpCBPKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif	/* PDUMP */
+#endif	/*SYNC_SERVER_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/tlintern.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlintern.h
new file mode 100644
index 000000000000..114f90f98548
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlintern.h
@@ -0,0 +1,347 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer internals
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer header used by TL internally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TLINTERN_H
+#define TLINTERN_H
+
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+#include "lock.h"
+#include "tlstream.h"
+
+/* Forward declarations */
+typedef struct _TL_SNODE_* PTL_SNODE;
+
+/* To debug buffer utilisation enable this macro here and define
+ * PVRSRV_NEED_PVR_TRACE in the server pvr_debug.c and in tutils.c
+ * before the inclusion of pvr_debug.h.
+ * Issue pvrtutils 6 on target to see stream buffer utilisation. */
+//#define TL_BUFFER_STATS 1
+
+/*! TL stream structure container.
+ *    pbyBuffer   holds the circular buffer.
+ *    ui32Read    points to the beginning of the buffer, ie to where data to
+ *                  Read begin.
+ *    ui32Write   points to the end of data that have been committed, ie this is
+ *                  where new data will be written.
+ *    ui32Pending number of bytes reserved in last reserve call which have not
+ *                  yet been submitted. Therefore these data are not ready to
+ *                  be transported.
+ *    hStreamWLock - provides atomic protection for the ui32Pending & ui32Write
+ *                   members of the structure for when they are checked and/or
+ *                   updated in the context of a stream writer (producer)
+ *                   calling DoTLStreamReserve() & TLStreamCommit().
+ *                 - Reader context is not multi-threaded, only one client per
+ *                   stream is allowed. Also note the read context may be in an
+ *                   ISR which prevents a design where locks can be held in the
+ *                   AcquireData/ReleaseData() calls. Thus this lock only
+ *                   protects the stream members from simultaneous writers.
+ *
+ *      ui32Read < ui32Write <= ui32Pending
+ *        where < and <= operators are overloaded to make sense in a circular way.
+ */
+typedef struct _TL_STREAM_
+{
+	IMG_CHAR                szName[PRVSRVTL_MAX_STREAM_NAME_SIZE];  /*!< String name identifier */
+	TL_OPMODE               eOpMode;                                /*!< Mode of Operation of TL Buffer */
+
+	IMG_BOOL                bWaitForEmptyOnDestroy;                 /*!< Flag: On destroying a non-empty stream block until
+                                                                         *         stream is drained. */
+	IMG_BOOL                bNoSignalOnCommit;                      /*!< Flag: Used to avoid the TL signalling waiting consumers
+                                                                         *         that new data is available on every commit. Producers
+                                                                         *         using this flag will need to manually signal when
+                                                                         *         appropriate using the TLStreamSync() API */
+
+	void                    (*pfOnReaderOpenCallback)(void *pvArg); /*!< Optional on reader connect callback */
+	void                    *pvOnReaderOpenUserData;                /*!< On reader connect user data */
+	void                    (*pfOnReaderCloseCallback)(void *pvArg);/*!< Optional on reader disconnect callback */
+	void                    *pvOnReaderCloseUserData;               /*!< On reader disconnect user data */
+	void                    (*pfProducerCallback)(void);            /*!< Optional producer callback of type TL_STREAM_SOURCECB */
+	void                    *pvProducerUserData;                    /*!< Producer callback user data */
+
+	struct _TL_STREAM_      *psNotifStream;                         /*!< Pointer to the stream to which notification will be sent */
+
+	volatile IMG_UINT32     ui32Read;                               /*!< Pointer to the beginning of available data */
+	volatile IMG_UINT32     ui32Write;                              /*!< Pointer to already committed data which are ready to be
+                                                                         *   copied to user space */
+	IMG_UINT32              ui32Pending;                            /*!< Count pending bytes reserved in buffer */
+	IMG_UINT32              ui32Size;                               /*!< Buffer size */
+	IMG_UINT32              ui32ThresholdUsageForSignal;            /*!< Buffer usage threshold at which a TL writer signals a blocked/
+	                                                                     *    waiting reader when transitioning from empty->non-empty */
+	IMG_UINT32              ui32MaxPacketSize;                      /*! Max TL packet size */
+	IMG_BYTE                *pbyBuffer;                             /*!< Actual data buffer */
+
+	PTL_SNODE               psNode;                                 /*!< Ptr to parent stream node */
+	DEVMEM_MEMDESC          *psStreamMemDesc;                       /*!< MemDescriptor used to allocate buffer space through PMR */
+
+	IMG_HANDLE              hProducerEvent;	                        /*!< Handle to wait on if there is not enough space */
+	IMG_HANDLE              hProducerEventObj;                      /*!< Handle to signal blocked reserve calls */
+	IMG_BOOL                bSignalPending;                         /*!< Tracks if a "signal" is pending to be sent to a blocked/
+	                                                                     *    waiting reader */
+
+	POS_LOCK                hStreamWLock;                           /*!< Writers Lock for ui32Pending & ui32Write*/
+	POS_LOCK                hReadLock;                              /*!< Readers Lock for bReadPending & ui32Read*/
+	IMG_BOOL                bReadPending;                           /*!< Tracks if a read operation is pending or not*/
+	IMG_BOOL                bNoWrapPermanent;                       /*!< Flag: Prevents buffer wrap and subsequent data loss
+	                                                                     *    as well as resetting the read position on close. */
+
+#if defined(TL_BUFFER_STATS)
+	IMG_UINT32              ui32CntReadFails;                       /*!< Tracks how many times reader failed to acquire read lock */
+	IMG_UINT32              ui32CntReadSuccesses;                   /*!< Tracks how many times reader acquires read lock successfully */
+	IMG_UINT32              ui32CntWriteSuccesses;                  /*!< Tracks how many times writer acquires read lock successfully */
+	IMG_UINT32              ui32CntWriteWaits;                      /*!< Tracks how many times writer had to wait to acquire read lock */
+	IMG_UINT32              ui32CntNumWriteSuccess;	                /*!< Tracks how many write operations were successful*/
+	IMG_UINT32              ui32BufferUt;                           /*!< Buffer utilisation high watermark, see TL_BUFFER_STATS above */
+	IMG_UINT32              ui32MaxReserveWatermark;                /*!< Max stream reserve size that was ever requested by a writer */
+	IMG_UINT32              ui32SignalsSent;                        /*!< Number of signals that were actually sent by the write API */
+	ATOMIC_T                bNoReaderSinceFirstReserve;             /*!< Tracks if a read has been done since the buffer was last found empty */
+	IMG_UINT32              ui32TimeStart;                          /*!< Time at which a write (Reserve call) was done into an empty buffer.
+	                                                                     *    Guarded by hStreamWLock. */
+	IMG_UINT32              ui32MinTimeToFullInUs;                  /*!< Minimum time taken to (nearly) fully fill an empty buffer. Guarded
+	                                                                     *    by hStreamWLock. */
+	/* Behaviour counters, protected by hStreamLock in case of
+	 * multi-threaded access */
+	IMG_UINT32              ui32NumCommits;     /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32SignalNotSent;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32ManSyncs;       /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32ProducerByteCount; /*!< Counters used to analysing stream performance, see ++ loc */
+
+	/* Not protected by the lock, inc in the reader thread which is currently singular */
+	IMG_UINT32              ui32AcquireRead1;   /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32AcquireRead2;   /*!< Counters used to analysing stream performance, see ++ loc */
+#endif
+
+} TL_STREAM, *PTL_STREAM;
+
+/* there need to be enough space reserved in the buffer for 2 minimal packets
+ * and it needs to be aligned the same way the buffer is or there will be a
+ * compile error.*/
+#define BUFFER_RESERVED_SPACE (2 * PVRSRVTL_PACKET_ALIGNMENT)
+
+/* ensure the space reserved follows the buffer's alignment */
+static_assert(!(BUFFER_RESERVED_SPACE&(PVRSRVTL_PACKET_ALIGNMENT-1)),
+			  "BUFFER_RESERVED_SPACE must be a multiple of PVRSRVTL_PACKET_ALIGNMENT");
+
+/* Define the largest value that a uint that matches the
+ * PVRSRVTL_PACKET_ALIGNMENT size can hold */
+#define MAX_UINT 0xffffFFFF
+
+/*! Defines the value used for TL_STREAM.ui32Pending when no reserve is
+ * outstanding on the stream. */
+#define NOTHING_PENDING IMG_UINT32_MAX
+
+
+/*
+ * Transport Layer Stream Descriptor types/defs
+ */
+typedef struct _TL_STREAM_DESC_
+{
+	PTL_SNODE   psNode;         /*!< Ptr to parent stream node */
+	IMG_UINT32  ui32Flags;      /*!< Flags supplied by client on stream open */
+	IMG_HANDLE  hReadEvent;     /*!< For wait call (only used/set in reader descriptors) */
+	IMG_INT     uiRefCount;     /*!< Reference count to the SD */
+
+#if defined(TL_BUFFER_STATS)
+	/* Behaviour counters, no multi-threading protection need as they are
+	 * incremented in a single thread due to only supporting one reader
+	 * at present */
+	IMG_UINT32  ui32AcquireCount;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32NoData;        /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32NoDataSleep;   /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32Signalled;     /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32TimeoutEmpty;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32TimeoutData;   /*!< Counters used to analysing stream performance, see ++ loc */
+#endif
+	IMG_UINT32  ui32ReadLimit;     /*!< Limit buffer reads to data present in the
+                                        buffer at the time of stream open. */
+	IMG_UINT32  ui32ReadLen;       /*!< Size of data returned by initial Acquire */
+} TL_STREAM_DESC, *PTL_STREAM_DESC;
+
+PTL_STREAM_DESC TLMakeStreamDesc(PTL_SNODE f1, IMG_UINT32 f2, IMG_HANDLE f3);
+
+#define TL_STREAM_KM_FLAG_MASK	0xFFFF0000
+#define TL_STREAM_FLAG_TEST		0x10000000
+#define TL_STREAM_FLAG_WRAPREAD	0x00010000
+
+#define TL_STREAM_UM_FLAG_MASK	0x0000FFFF
+
+#if defined(TL_BUFFER_STATS)
+#	define TL_COUNTER_INC(a)    ((a)++)
+#	define TL_COUNTER_ADD(a,b)  ((a) += (b))
+#else
+#	define TL_COUNTER_INC(a)    (void)(0)
+#	define TL_COUNTER_ADD(a,b)  (void)(0)
+#endif
+/*
+ * Transport Layer stream list node
+ */
+typedef struct _TL_SNODE_
+{
+	struct _TL_SNODE_*	psNext;				/*!< Linked list next element */
+	IMG_HANDLE			hReadEventObj;		/*!< Readers 'wait for data' event */
+	PTL_STREAM			psStream;			/*!< TL Stream object */
+	IMG_INT				uiWRefCount;		/*!< Stream writer reference count */
+	PTL_STREAM_DESC		psRDesc;			/*!< Stream reader 0 or ptr only */
+	PTL_STREAM_DESC		psWDesc;			/*!< Stream writer 0 or ptr only */
+} TL_SNODE;
+
+PTL_SNODE TLMakeSNode(IMG_HANDLE f2, TL_STREAM *f3, TL_STREAM_DESC *f4);
+
+/*
+ * Transport Layer global top types and variables
+ * Use access function to obtain pointer.
+ *
+ * hTLGDLock - provides atomicity over read/check/write operations and
+ *             sequence of operations on uiClientCnt, psHead list of SNODEs and
+ *             the immediate members in a list element SNODE structure.
+ *           - This larger scope of responsibility for this lock helps avoid
+ *             the need for a lock in the SNODE structure.
+ *           - Lock held in the client (reader) context when streams are
+ *             opened/closed and in the server (writer) context when streams
+ *             are created/open/closed.
+ */
+typedef struct _TL_GDATA_
+{
+	IMG_HANDLE hTLEventObj;        /* Global TL signal object, new streams, etc */
+
+	IMG_UINT   uiClientCnt;        /* Counter to track the number of client stream connections. */
+	PTL_SNODE  psHead;             /* List of TL streams and associated client handle */
+
+	POS_LOCK   hTLGDLock;          /* Lock for structure AND psHead SNODE list */
+} TL_GLOBAL_DATA, *PTL_GLOBAL_DATA;
+
+/*
+ * Transport Layer Internal Kernel-Mode Server API
+ */
+TL_GLOBAL_DATA* TLGGD(void);		/* TLGetGlobalData() */
+
+PVRSRV_ERROR TLInit(void);
+void TLDeInit(void);
+
+void TLAddStreamNode(PTL_SNODE psAdd);
+PTL_SNODE TLFindStreamNodeByName(const IMG_CHAR *pszName);
+PTL_SNODE TLFindStreamNodeByDesc(PTL_STREAM_DESC psDesc);
+IMG_UINT32 TLDiscoverStreamNodes(const IMG_CHAR *pszNamePattern,
+                          IMG_CHAR aaszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+                          IMG_UINT32 ui32Max);
+PTL_SNODE TLFindAndGetStreamNodeByDesc(PTL_STREAM_DESC psDesc);
+void TLReturnStreamNode(PTL_SNODE psNode);
+
+/******************************************************************************
+ Function Name	: TLTryRemoveStreamAndFreeStreamNode
+
+ Inputs		: PTL_SNODE	Pointer to the TL_SNODE whose stream is requested
+			to be removed from TL_GLOBAL_DATA's list
+
+ Return Value	: IMG_TRUE	-	If the stream was made NULL and this
+					TL_SNODE was removed from the
+					TL_GLOBAL_DATA's list
+
+		  IMG_FALSE	-	If the stream wasn't made NULL as there
+					is a client connected to this stream
+
+ Description	: If there is no client currently connected to this stream then,
+			This function removes this TL_SNODE from the
+			TL_GLOBAL_DATA's list. The caller is responsible for the
+			cleanup of the TL_STREAM whose TL_SNODE may be removed
+
+		  Otherwise, this function does nothing
+******************************************************************************/
+IMG_BOOL  TLTryRemoveStreamAndFreeStreamNode(PTL_SNODE psRemove);
+
+/******************************************************************************
+ Function Name	: TLUnrefDescAndTryFreeStreamNode
+
+ Inputs		: PTL_SNODE	Pointer to the TL_SNODE whose descriptor is
+			requested to be removed
+			: PTL_STREAM_DESC	Pointer to the STREAM_DESC
+
+ Return Value	: IMG_TRUE	-	If this	TL_SNODE was removed from the
+					TL_GLOBAL_DATA's list
+
+		  IMG_FALSE	-	Otherwise
+
+ Description	: This function removes the stream descriptor from this TL_SNODE
+			and, if there is no writer (producer context) currently bound to this
+			stream, this function removes this TL_SNODE from the TL_GLOBAL_DATA's
+			list. The caller is responsible for the cleanup of the TL_STREAM
+			whose TL_SNODE may be removed
+******************************************************************************/
+IMG_BOOL TLUnrefDescAndTryFreeStreamNode(PTL_SNODE psRemove, PTL_STREAM_DESC psSD);
+
+/*
+ * Transport Layer stream interface to server part declared here to avoid
+ * circular dependency.
+ */
+IMG_UINT32 TLStreamAcquireReadPos(PTL_STREAM psStream,
+                                  IMG_BOOL bDisableCallback,
+                                  IMG_UINT32* puiReadOffset);
+PVRSRV_ERROR TLStreamAdvanceReadPos(PTL_STREAM psStream,
+                                    IMG_UINT32 uiReadLen,
+                                    IMG_UINT32 uiOrigReadLen);
+void TLStreamResetReadPos(PTL_STREAM psStream);
+
+DEVMEM_MEMDESC* TLStreamGetBufferPointer(PTL_STREAM psStream);
+IMG_BOOL TLStreamOutOfData(IMG_HANDLE psStream);
+
+/******************************************************************************
+ Function Name	: TLStreamDestroy
+
+ Inputs		: PTL_STREAM	Pointer to the TL_STREAM to be destroyed
+
+ Description	: This function performs all the clean-up operations required for
+			destruction of this stream
+******************************************************************************/
+void TLStreamDestroy(PTL_STREAM psStream);
+
+/*
+ * Test related functions
+ */
+PVRSRV_ERROR TUtilsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR TUtilsDeinit(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+
+#endif /* TLINTERN_H */
+/******************************************************************************
+ End of file (tlintern.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/tlserver.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlserver.h
new file mode 100644
index 000000000000..7ac2958eac1e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlserver.h
@@ -0,0 +1,97 @@
+/*************************************************************************/ /*!
+@File
+@Title          KM server Transport Layer implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main bridge APIs for Transport Layer client functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef TLSERVER_H
+#define TLSERVER_H
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+
+#include "tlintern.h"
+
+/*
+ * Transport Layer Client API Kernel-Mode bridge implementation
+ */
+
+PVRSRV_ERROR TLServerConnectKM(CONNECTION_DATA *psConnection);
+PVRSRV_ERROR TLServerDisconnectKM(CONNECTION_DATA *psConnection);
+
+PVRSRV_ERROR TLServerOpenStreamKM(const IMG_CHAR* pszName,
+			   IMG_UINT32 ui32Mode,
+			   PTL_STREAM_DESC* ppsSD,
+			   PMR** ppsTLPMR);
+
+PVRSRV_ERROR TLServerCloseStreamKM(PTL_STREAM_DESC psSD);
+
+PVRSRV_ERROR TLServerDiscoverStreamsKM(const IMG_CHAR *pszNamePattern,
+                          IMG_UINT32 ui32Max,
+                          IMG_CHAR *pszStreams,
+                          IMG_UINT32 *pui32NumFound);
+
+PVRSRV_ERROR TLServerReserveStreamKM(PTL_STREAM_DESC psSD,
+                                     IMG_UINT32* ui32BufferOffset,
+                                     IMG_UINT32 ui32Size,
+                                     IMG_UINT32 ui32SizeMin,
+                                     IMG_UINT32* pui32Available);
+
+PVRSRV_ERROR TLServerCommitStreamKM(PTL_STREAM_DESC psSD,
+                                    IMG_UINT32 ui32Size);
+
+PVRSRV_ERROR TLServerAcquireDataKM(PTL_STREAM_DESC psSD,
+			   IMG_UINT32* puiReadOffset,
+			   IMG_UINT32* puiReadLen);
+
+PVRSRV_ERROR TLServerReleaseDataKM(PTL_STREAM_DESC psSD,
+				 IMG_UINT32 uiReadOffset,
+				 IMG_UINT32 uiReadLen);
+
+PVRSRV_ERROR TLServerWriteDataKM(PTL_STREAM_DESC psSD,
+                                 IMG_UINT32 ui32Size,
+                                 IMG_BYTE *pui8Data);
+
+#endif /* TLSERVER_H */
+
+/******************************************************************************
+ End of file (tlserver.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/tlstream.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlstream.h
new file mode 100644
index 000000000000..aa237bba4d79
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/tlstream.h
@@ -0,0 +1,619 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    TL provides driver components with a way to copy data from kernel
+                space to user space (e.g. screen/file).
+
+                Data can be passed to the Transport Layer through the
+                TL Stream (kernel space) API interface.
+
+                The buffer provided to every stream is a modified version of a
+                circular buffer. Which CB version is created is specified by
+                relevant flags when creating a stream. Currently two types
+                of buffer are available:
+                - TL_OPMODE_DROP_NEWER:
+                  When the buffer is full, incoming data are dropped
+                  (instead of overwriting older data) and a marker is set
+                  to let the user know that data have been lost.
+                - TL_OPMODE_BLOCK:
+                  When the circular buffer is full, reserve/write calls block
+                  until enough space is freed.
+                - TL_OPMODE_DROP_OLDEST:
+                  When the circular buffer is full, the oldest packets in the
+                  buffer are dropped and a flag is set in header of next packet
+                  to let the user know that data have been lost.
+
+                All size/space requests are in bytes. However, the actual
+                implementation uses native word sizes (i.e. 4 byte aligned).
+
+                The user does not need to provide space for the stream buffer
+                as the TL handles memory allocations and usage.
+
+                Inserting data to a stream's buffer can be done either:
+                - by using TLReserve/TLCommit: User is provided with a buffer
+                                                 to write data to.
+                - or by using TLWrite:         User provides a buffer with
+                                                 data to be committed. The TL
+                                                 copies the data from the
+                                                 buffer into the stream buffer
+                                                 and returns.
+                Users should be aware that there are implementation overheads
+                associated with every stream buffer. If you find that less
+                data are captured than expected then try increasing the
+                stream buffer size or use TLInfo to obtain buffer parameters
+                and calculate optimum required values at run time.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TLSTREAM_H
+#define TLSTREAM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_tlcommon.h"
+#include "device.h"
+
+/*! Extract TL stream opmode from the given stream create flags.
+ * Last 3 bits of streamFlag is used for storing opmode, hence
+ * opmode mask is set as following. */
+#define TL_OPMODE_MASK 0x7
+
+/*
+ * NOTE: This enum is used to directly access the HTB_OPMODE_xxx values
+ * within htbserver.c.
+ * As such we *MUST* keep the values matching in order of declaration.
+ */
+/*! Opmode specifying circular buffer behaviour */
+typedef enum
+{
+	/*! Undefined operation mode */
+	TL_OPMODE_UNDEF = 0,
+
+	/*! Reject new data if the buffer is full, producer may then decide to
+	 *    drop the data or retry after some time. */
+	TL_OPMODE_DROP_NEWER,
+
+	/*! When buffer is full, advance the tail/read position to accept the new
+	 * reserve call (size permitting), effectively overwriting the oldest
+	 * data in the circular buffer. Not supported yet. */
+	TL_OPMODE_DROP_OLDEST,
+
+	/*! Block Reserve (subsequently Write) calls if there is not enough space
+	 *    until some space is freed via a client read operation. */
+	TL_OPMODE_BLOCK,
+
+	/*!< For error checking */
+	TL_OPMODE_LAST
+
+} TL_OPMODE;
+
+typedef enum {
+	/* Enum to be used in conjunction with new Flags feature */
+
+	/* Flag set when Drop Oldest is set and packets have been dropped */
+	TL_FLAG_OVERWRITE_DETECTED = (1 << 0),
+	/* Prevents DoTLStreamReserve() from adding from injecting
+	 * PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED */
+	TL_FLAG_NO_WRITE_FAILED = (1 << 1),
+} TL_Flags;
+
+static_assert(TL_OPMODE_LAST <= TL_OPMODE_MASK,
+	      "TL_OPMODE_LAST must not exceed TL_OPMODE_MASK");
+
+/*! Flags specifying stream behaviour */
+/*! Do not destroy stream if there still are data that have not been
+ *     copied in user space. Block until the stream is emptied. */
+#define TL_FLAG_FORCE_FLUSH            (1U<<8)
+/*! Do not signal consumers on commit automatically when the stream buffer
+ * transitions from empty to non-empty. Producer responsible for signal when
+ * it chooses. */
+#define TL_FLAG_NO_SIGNAL_ON_COMMIT    (1U<<9)
+
+/*! When a stream has this property it never wraps around and
+ * overwrites existing data, hence it is a fixed size persistent
+ * buffer, data written is permanent. Producers need to ensure
+ * the buffer is big enough for their needs.
+ * When a stream is opened for reading the client will always
+ * find the read position at the start of the buffer/data. */
+#define TL_FLAG_PERMANENT_NO_WRAP      (1U<<10)
+
+/*! Defer allocation of stream's shared memory until first open. */
+#define TL_FLAG_ALLOCATE_ON_FIRST_OPEN (1U<<11)
+
+/*! Structure used to pass internal TL stream sizes information to users.*/
+typedef struct _TL_STREAM_INFO_
+{
+    IMG_UINT32 headerSize;          /*!< Packet header size in bytes */
+    IMG_UINT32 minReservationSize;  /*!< Minimum data size reserved in bytes */
+    IMG_UINT32 pageSize;            /*!< Page size in bytes */
+    IMG_UINT32 pageAlign;           /*!< Page alignment in bytes */
+    IMG_UINT32 maxTLpacketSize;     /*! Max allowed TL packet size*/
+} TL_STREAM_INFO, *PTL_STREAM_INFO;
+
+/*! Callback operations or notifications that a stream producer may handle
+ * when requested by the Transport Layer.
+ */
+#define TL_SOURCECB_OP_CLIENT_EOS 0x01  /*!< Client has reached end of stream,
+                                         * can anymore data be supplied?
+                                         * ui32Resp ignored in this operation */
+
+/*! Function pointer type for the callback handler into the "producer" code
+ * that writes data to the TL stream.  Producer should handle the notification
+ * or operation supplied in ui32ReqOp on stream hStream. The
+ * Operations and notifications are defined above in TL_SOURCECB_OP */
+typedef PVRSRV_ERROR (*TL_STREAM_SOURCECB)(IMG_HANDLE hStream,
+		IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser);
+
+typedef void (*TL_STREAM_ONREADEROPENCB)(void *pvArg);
+typedef void (*TL_STREAM_ONREADERCLOSECB)(void *pvArg);
+
+/*************************************************************************/ /*!
+ @Function      TLAllocSharedMemIfNull
+ @Description   Allocates shared memory for the stream.
+ @Input         hStream     Stream handle.
+ @Return        eError      Internal services call returned eError error
+                            number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLAllocSharedMemIfNull(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLFreeSharedMem
+ @Description   Frees stream's shared memory.
+ @Input         phStream    Stream handle.
+*/ /**************************************************************************/
+void
+TLFreeSharedMem(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamCreate
+ @Description   Request the creation of a new stream and open a handle.
+				If creating a stream which should continue to exist after the
+				current context is finished, then TLStreamCreate must be
+				followed by a TLStreamOpen call. On any case, the number of
+				create/open calls must balance with the number of close calls
+				used. This ensures the resources of a stream are released when
+				it is no longer required.
+ @Output        phStream        Pointer to handle to store the new stream.
+ @Input         szStreamName    Name of stream, maximum length:
+                                PRVSRVTL_MAX_STREAM_NAME_SIZE.
+                                If a longer string is provided,creation fails.
+ @Input         ui32Size        Desired buffer size in bytes.
+ @Input         ui32StreamFlags Used to configure buffer behaviour. See above.
+ @Input         pfOnReaderOpenCB    Optional callback called when a client
+                                    opens this stream, may be null.
+ @Input         pvOnReaderOpenUD    Optional user data for pfOnReaderOpenCB,
+                                    may be null.
+ @Input         pfOnReaderCloseCB   Optional callback called when a client
+                                    closes this stream, may be null.
+ @Input         pvOnReaderCloseUD   Optional user data for pfOnReaderCloseCB,
+                                    may be null.
+ @Input         pfProducerCB    Optional callback, may be null.
+ @Input         pvProducerUD    Optional user data for callback, may be null.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle or string name
+                                             exceeded MAX_STREAM_NAME_SIZE
+ @Return        PVRSRV_ERROR_OUT_OF_MEMORY   Failed to allocate space for
+                                             stream handle.
+ @Return        PVRSRV_ERROR_DUPLICATE_VALUE There already exists a stream with
+                                             the same stream name string.
+ @Return        eError                       Internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamCreate(IMG_HANDLE *phStream,
+               const IMG_CHAR *szStreamName,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32StreamFlags,
+               TL_STREAM_ONREADEROPENCB pfOnReaderOpenCB,
+               void *pvOnReaderOpenUD,
+               TL_STREAM_ONREADERCLOSECB pfOnReaderCloseCB,
+               void *pvOnReaderCloseUD,
+               TL_STREAM_SOURCECB pfProducerCB,
+               void *pvProducerUD);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamOpen
+ @Description   Attach to existing stream that has already been created by a
+                  TLStreamCreate call. A handle is returned to the stream.
+ @Output        phStream        Pointer to handle to store the stream.
+ @Input         szStreamName    Name of stream, should match an already
+                                  existing stream name
+ @Return        PVRSRV_ERROR_NOT_FOUND       None of the streams matched the
+                                             requested stream name.
+				PVRSRV_ERROR_INVALID_PARAMS  Non-NULL pointer to stream
+                                             handler is required.
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamOpen(IMG_HANDLE     *phStream,
+             const IMG_CHAR *szStreamName);
+
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReset
+ @Description   Resets read and write pointers and pending flag.
+ @Output        phStream Pointer to stream's handle
+*/ /**************************************************************************/
+void TLStreamReset(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamSetNotifStream
+ @Description   Registers a "notification stream" which will be used to
+                publish information about state change of the "hStream"
+                stream. Notification can inform about events such as stream
+                open/close, etc.
+ @Input         hStream         Handle to stream to update.
+ @Input         hNotifStream    Handle to the stream which will be used for
+                                publishing notifications.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  If either of the parameters is
+                                             NULL
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamSetNotifStream(IMG_HANDLE hStream, IMG_HANDLE hNotifStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReconfigure
+ @Description   Request the stream flags controlling buffer behaviour to
+                be updated.
+                In the case where TL_OPMODE_BLOCK is to be used,
+                TLStreamCreate should be called without that flag and this
+                function used to change the stream mode once a consumer process
+                has been started. This avoids a deadlock scenario where the
+                TLStreaWrite/TLStreamReserve call will hold the Bridge Lock
+                while blocking if the TL buffer is full.
+                The TL_OPMODE_BLOCK should never drop the Bridge Lock
+                as this leads to another deadlock scenario where the caller to
+                TLStreamWrite/TLStreamReserve has already acquired another lock
+                (e.g. gHandleLock) which is not dropped. This then leads to that
+                thread acquiring locks out of order.
+ @Input         hStream         Handle to stream to update.
+ @Input         ui32StreamFlags Flags that configure buffer behaviour. See above.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle or inconsistent
+                                             stream flags.
+ @Return        PVRSRV_ERROR_NOT_READY       Stream is currently being written to
+                                             try again later.
+ @Return        eError                       Internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReconfigure(IMG_HANDLE hStream,
+                    IMG_UINT32 ui32StreamFlags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamClose
+ @Description   Detach from the stream associated with the given handle. If
+                  the current handle is the last one accessing the stream
+				  (i.e. the number of TLStreamCreate+TLStreamOpen calls matches
+				  the number of TLStreamClose calls) then the stream is also
+				  deleted.
+				On return the handle is no longer valid.
+ @Input         hStream     Handle to stream that will be closed.
+ @Return        None.
+*/ /**************************************************************************/
+void
+TLStreamClose(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserve
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Number of bytes to reserve in buffer.
+ @Return        PVRSRV_INVALID_PARAMS       NULL stream handler.
+ @Return        PVRSRV_ERROR_NOT_READY      There are data previously reserved
+                                              that are pending to be committed.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE  Misusing the stream by trying to
+                                              reserve more space than the
+                                              buffer size.
+ @Return        PVRSRV_ERROR_STREAM_FULL    The reserve size requested
+                                            is larger than the free
+                                            space.
+ @Return         PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED  The reserve size
+                                                            requested is larger
+                                                            than max TL packet size
+ @Return        PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE Permanent stream buffer
+                                                     does not have enough space
+                                                     for the reserve.
+ @Return        PVRSRV_OK                   Success, output arguments valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserve(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserve2
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Ideal number of bytes to reserve in buffer.
+ @Input         ui32SizeMin     Minimum number of bytes to reserve in buffer.
+ @Input         pui32Available  Optional, but when present and the
+                                  RESERVE_TOO_BIG error is returned, a size
+                                  suggestion is returned in this argument which
+                                  the caller can attempt to reserve again for a
+                                  successful allocation.
+ @Output        pbIsReaderConnected Let writing clients know if reader is
+                                    connected or not, in case of error.
+ @Return        PVRSRV_INVALID_PARAMS        NULL stream handler.
+ @Return        PVRSRV_ERROR_NOT_READY       There are data previously reserved
+                                             that are pending to be committed.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE   Misusing the stream by trying to
+                                             reserve more space than the
+                                             buffer size.
+ @Return        PVRSRV_ERROR_STREAM_FULL     The reserve size requested
+                                             is larger than the free
+                                             space.
+                                             Check the pui32Available
+                                             value for the correct
+                                             reserve size to use.
+ @Return         PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED   The reserve size
+                                                             requested is larger
+                                                             than max TL packet size
+ @Return        PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE Permanent stream buffer
+                                                     does not have enough space
+                                                     for the reserve.
+ @Return        PVRSRV_OK                   Success, output arguments valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserve2(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size,
+                IMG_UINT32 ui32SizeMin,
+                IMG_UINT32* pui32Available,
+                IMG_BOOL* pbIsReaderConnected);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserveReturnFlags
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Ideal number of bytes to reserve in buffer.
+ @Output        pui32Flags      Output parameter to return flags generated within
+                                the reserve function.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserveReturnFlags(IMG_HANDLE hStream,
+        IMG_UINT8  **ppui8Data,
+        IMG_UINT32 ui32Size,
+		IMG_UINT32* pui32Flags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamGetUT
+ @Description   Returns the current stream utilisation in bytes
+ @Input         hStream     Stream handle.
+ @Return        IMG_UINT32  Stream utilisation
+*/ /**************************************************************************/
+IMG_UINT32 TLStreamGetUT(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamCommit
+ @Description   Notify TL that data have been written in the stream buffer.
+                  Should always follow and match TLStreamReserve call.
+ @Input         hStream         Stream handle.
+ @Input         ui32Size        Number of bytes that have been added to the
+                                  stream.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE   Commit results in more data
+                                             committed than the buffer size,
+                                             the stream is misused.
+ @Return        eError                       Commit was successful but
+                                             internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamCommit(IMG_HANDLE hStream,
+               IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamWrite
+ @Description   Combined Reserve/Commit call. This function Reserves space in
+                  the specified stream buffer, copies ui32Size bytes of data
+                  from the array pui8Src points to and Commits in an "atomic"
+                  style operation.
+ @Input         hStream         Stream handle.
+ @Input         pui8Src         Source to read data from.
+ @Input         ui32Size        Number of bytes to copy and commit.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                               Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamWrite(IMG_HANDLE hStream,
+              IMG_UINT8  *pui8Src,
+              IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamWriteRetFlags
+ @Description   Combined Reserve/Commit call. This function Reserves space in
+                  the specified stream buffer, copies ui32Size bytes of data
+                  from the array pui8Src points to and Commits in an "atomic"
+                  style operation. Also accepts a pointer to a bit flag value
+                  for returning write status flags.
+ @Input         hStream         Stream handle.
+ @Input         pui8Src         Source to read data from.
+ @Input         ui32Size        Number of bytes to copy and commit.
+ @Output        pui32Flags      Output parameter for write status info
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                               Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamWriteRetFlags(IMG_HANDLE hStream,
+                      IMG_UINT8 *pui8Src,
+					  IMG_UINT32 ui32Size,
+					  IMG_UINT32 *pui32Flags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamSync
+ @Description   Signal the consumer to start acquiring data from the stream
+                buffer. Called by producers that use the flag
+                TL_FLAG_NO_SIGNAL_ON_COMMIT to manually control when
+                consumers starting reading the stream.
+                Used when multiple small writes need to be batched.
+ @Input         hStream         Stream handle.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle.
+ @Return        eError                       Error codes returned by either
+                                             Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamSync(IMG_HANDLE hStream);
+
+
+/*************************************************************************/ /*!
+ @Function      TLStreamMarkEOS
+ @Description   Insert a EOS marker packet in the given stream.
+ @Input         hStream         Stream handle.
+ @Input         bRemoveOld      if TRUE, remove old stream record file before
+                                splitting to new file.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                             Reserve or Commit.
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkEOS(IMG_HANDLE hStream, IMG_BOOL bRemoveOld);
+
+/*************************************************************************/ /*!
+@Function       TLStreamMarkStreamOpen
+@Description    Puts *open* stream packet into hStream's notification stream,
+                if set, error otherwise."
+@Input          hStream Stream handle.
+@Return         PVRSRV_OK on success and error code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkStreamOpen(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+@Function       TLStreamMarkStreamClose
+@Description    Puts *close* stream packet into hStream's notification stream,
+                if set, error otherwise."
+@Input          hStream Stream handle.
+@Return         PVRSRV_OK on success and error code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkStreamClose(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamInfo
+ @Description   Run time information about buffer elemental sizes.
+                It sets psInfo members accordingly. Users can use those values
+                to calculate the parameters they use in TLStreamCreate and
+                TLStreamReserve.
+ @Output        psInfo          pointer to stream info structure.
+ @Return        None.
+*/ /**************************************************************************/
+void
+TLStreamInfo(IMG_HANDLE hStream, PTL_STREAM_INFO psInfo);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamIsOpenForReading
+ @Description   Query if a stream has any readers connected.
+ @Input         hStream         Stream handle.
+ @Return        IMG_BOOL        True if at least one reader is connected,
+                                false otherwise
+*/ /**************************************************************************/
+IMG_BOOL
+TLStreamIsOpenForReading(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamOutOfData
+ @Description   Query if the stream is empty (no data waiting to be read).
+ @Input         hStream         Stream handle.
+ @Return        IMG_BOOL        True if read==write, no data waiting,
+                                false otherwise
+*/ /**************************************************************************/
+IMG_BOOL TLStreamOutOfData(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamResetProducerByteCount
+ @Description   Reset the producer byte counter on the specified stream.
+ @Input         hStream         Stream handle.
+ @Input         ui32Value       Value to reset counter to, often 0.
+ @Return        PVRSRV_OK                   Success.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE  Success but the read and write
+                                            positions did not match,
+                                            stream not empty.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR
+TLStreamResetProducerByteCount(IMG_HANDLE hStream, IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamGetMaxTransfer
+ @Description   Obtain the maximum number of bytes that can be submitted to
+                 a given stream.
+ @Input         uiXferSize         Requested transfer size from producer.
+ @Input         hConsumerStream    Stream handle for consumer.
+ @Return        IMG_UINT32         Amount of data that can be submitted to
+                                    the consumer stream.
+*/ /**************************************************************************/
+IMG_UINT32
+TLStreamGetMaxTransfer(IMG_UINT32 uiXferSize, IMG_HANDLE hConsumerStream);
+
+#endif /* TLSTREAM_H */
+/*****************************************************************************
+ End of file (tlstream.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_impl.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_impl.h
new file mode 100644
index 000000000000..928a3f6bc9ac
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_impl.h
@@ -0,0 +1,206 @@
+/*************************************************************************/ /*!
+@File           vmm_impl.h
+@Title          Common VM manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides common VM manager definitions that need to
+                be shared by system virtualization layer itself and modules that
+                implement the actual VM manager types.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_IMPL_H
+#define VMM_IMPL_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_device.h"
+
+typedef enum _VMM_CONF_PARAM_
+{
+	VMM_CONF_PRIO_DRV0 = 0,
+	VMM_CONF_PRIO_DRV1 = 1,
+	VMM_CONF_PRIO_DRV2 = 2,
+	VMM_CONF_PRIO_DRV3 = 3,
+	VMM_CONF_PRIO_DRV4 = 4,
+	VMM_CONF_PRIO_DRV5 = 5,
+	VMM_CONF_PRIO_DRV6 = 6,
+	VMM_CONF_PRIO_DRV7 = 7,
+	VMM_CONF_HCS_DEADLINE = 8,
+	VMM_CONF_ISOLATION_GROUP_DRV0 = 9,
+	VMM_CONF_ISOLATION_GROUP_DRV1 = 10,
+	VMM_CONF_ISOLATION_GROUP_DRV2 = 11,
+	VMM_CONF_ISOLATION_GROUP_DRV3 = 12,
+	VMM_CONF_ISOLATION_GROUP_DRV4 = 13,
+	VMM_CONF_ISOLATION_GROUP_DRV5 = 14,
+	VMM_CONF_ISOLATION_GROUP_DRV6 = 15,
+	VMM_CONF_ISOLATION_GROUP_DRV7 = 16,
+	VMM_CONF_TIME_SLICE_DRV0 = 17,
+	VMM_CONF_TIME_SLICE_DRV1 = 18,
+	VMM_CONF_TIME_SLICE_DRV2 = 19,
+	VMM_CONF_TIME_SLICE_DRV3 = 20,
+	VMM_CONF_TIME_SLICE_DRV4 = 21,
+	VMM_CONF_TIME_SLICE_DRV5 = 22,
+	VMM_CONF_TIME_SLICE_DRV6 = 23,
+	VMM_CONF_TIME_SLICE_DRV7 = 24,
+	VMM_CONF_TIME_SLICE_INTERVAL = 25,
+	VMM_CONF_VZ_CONNECTION_COOLDOWN_PERIOD = 26,
+} VMM_CONF_PARAM;
+
+/*
+	Virtual machine manager (hypervisor) para-virtualization (PVZ) connection:
+		- Type is implemented by host and guest drivers
+			- Assumes synchronous function call semantics
+			- Unidirectional semantics
+				- For Host  (vmm -> host)
+				- For Guest (guest -> vmm)
+			- Parameters can be IN/OUT/INOUT
+
+		- Host pvz entries are pre-implemented by IMG
+			- For host implementation, see vmm_pvz_server.c
+			- Called by host side hypercall handler or VMM
+
+		- Guest pvz entries are supplied by 3rd-party
+			- These are specific to hypervisor (VMM) type
+			- These implement the actual hypercalls mechanism
+
+	Para-virtualization (PVZ) call runtime sequence:
+		1 - Guest driver in guest VM calls PVZ function
+		1.1 - Guest PVZ connection calls
+		1.2 - Guest VM Manager type which
+		1.2.1 - Performs any pre-processing like parameter packing, etc.
+		1.2.2 - Issues hypercall (blocking synchronous call)
+
+		2 - VM Manager (hypervisor) receives hypercall
+		2.1 - Hypercall handler:
+		2.1.1 - Performs any pre-processing
+		2.1.2 - If call terminates in VM Manager: perform action and return from hypercall
+		2.1.3 - Otherwise forward to host driver (implementation specific call)
+
+		3 - Host driver receives call from VM Manager
+		3.1 - Host VM manager type:
+		3.1.1 - Performs any pre-processing like parameter unpacking, etc.
+		3.1.2 - Acquires host driver PVZ handler and calls the appropriate entry
+		3.2 - Host PVZ connection calls corresponding host system virtualisation layer
+		3.3 - Host driver system virtualisation layer:
+		3.3.1 - Perform action requested by guest driver
+		3.3.2 - Return to host VM Manager type
+		3.4 - Host VM Manager type:
+		3.4.1 - Prepare to return from hypercall
+		3.4.2 - Perform any post-processing like result packing, etc.
+		3.4.3 - Issue return from hypercall
+
+		4 - VM Manager (hypervisor)
+		4.1 - Perform any post-processing
+		4.2 - Return control to guest driver
+
+		5 - Guest driver in guest VM
+		5.1 - Perform any post-processing like parameter unpacking, etc.
+		5.2 - Continue execution in guest VM
+ */
+typedef struct _VMM_PVZ_CONNECTION_
+{
+	struct {
+		/*
+		   This pair must be implemented if the guest is responsible
+		   for allocating the physical heap that backs its firmware
+		   allocations, this is the default configuration. The physical
+		   heap is allocated within the guest VM IPA space and this
+		   IPA Addr/Size must be translated into the host's IPA space
+		   by the VM manager before forwarding request to host.
+		   If not implemented, return PVRSRV_ERROR_NOT_IMPLEMENTED.
+		 */
+		PVRSRV_ERROR (*pfnMapDevPhysHeap)(IMG_UINT64 ui64Size,
+										  IMG_UINT64 ui64PAddr);
+
+		PVRSRV_ERROR (*pfnUnmapDevPhysHeap)(void);
+	} sClientFuncTab;
+
+	struct {
+		/*
+			Corresponding server side entries to handle guest PVZ calls
+			NOTE:
+				 - Additional PVZ function ui32DriverID parameter
+					 - Driver ID determination is responsibility of VM manager
+					 - Actual Driver ID value must be supplied by VM manager
+						- This can be done either in client/VMM/host side
+					 - Must be done before host pvz function(s) are called
+					 - Host pvz function validates incoming Driver ID values
+		 */
+		PVRSRV_ERROR (*pfnMapDevPhysHeap)(IMG_UINT32 ui32DriverID,
+										  IMG_UINT32 ui32DevID,
+										  IMG_UINT64 ui64Size,
+										  IMG_UINT64 ui64PAddr);
+
+		PVRSRV_ERROR (*pfnUnmapDevPhysHeap)(IMG_UINT32 ui32DriverID,
+											IMG_UINT32 ui32DevID);
+	} sServerFuncTab;
+
+	struct {
+		/*
+		   This is used by the VM manager to report pertinent runtime guest VM
+		   information to the host; these events may in turn be forwarded to
+		   the firmware
+		 */
+		PVRSRV_ERROR (*pfnOnVmOnline)(IMG_UINT32 ui32DriverID,
+									  IMG_UINT32 ui32DevID);
+
+		PVRSRV_ERROR (*pfnOnVmOffline)(IMG_UINT32 ui32DriverID,
+									   IMG_UINT32 ui32DevID);
+
+		PVRSRV_ERROR (*pfnVMMConfigure)(VMM_CONF_PARAM eVMMParamType,
+										IMG_UINT32 ui32ParamValue,
+										IMG_UINT32 ui32DevID);
+
+	} sVmmFuncTab;
+} VMM_PVZ_CONNECTION;
+
+/*!
+*******************************************************************************
+ @Function      VMMCreatePvzConnection() and VMMDestroyPvzConnection()
+ @Description   Both the guest and VM manager call this in order to obtain a
+                PVZ connection to the VM and host respectively; that is, guest
+                calls it to obtain connection to VM, VM calls it to obtain a
+                connection to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig);
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+#endif /* VMM_IMPL_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_client.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_client.h
new file mode 100644
index 000000000000..205dc4ae93b5
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_client.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           vmm_pvz_client.h
+@Title          Guest VM manager client para-virtualization routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides guest VMM client para-virtualization APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_PVZ_CLIENT_H
+#define VMM_PVZ_CLIENT_H
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "vmm_impl.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzClientMapDevPhysHeap
+ @Description   The guest front-end to initiate a pfnMapDevPhysHeap PVZ call
+                to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzClientMapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+*******************************************************************************
+ @Function      PvzClientUnmapDevPhysHeap
+ @Description   The guest front-end to initiate a pfnUnmapDevPhysHeap PVZ call
+                to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzClientUnmapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+#endif /* VMM_PVZ_CLIENT_H */
+
+/******************************************************************************
+ End of file (vmm_pvz_client.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_server.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_server.h
new file mode 100644
index 000000000000..02e2683c0c19
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vmm_pvz_server.h
@@ -0,0 +1,121 @@
+/*************************************************************************/ /*!
+@File           vmm_pvz_server.h
+@Title          VM manager para-virtualization interface helper routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides API(s) available to VM manager, this must be
+                called to close the loop during guest para-virtualization calls.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_PVZ_SERVER_H
+#define VMM_PVZ_SERVER_H
+
+#include "vmm_impl.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzServerMapDevPhysHeap
+ @Description   The VM manager calls this in response to guest PVZ interface
+                call pfnMapDevPhysHeap.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerMapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						IMG_UINT32 ui32DevID,
+						IMG_UINT64 ui64Size,
+						IMG_UINT64 ui64PAddr);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerUnmapDevPhysHeap
+ @Description   The VM manager calls this in response to guest PVZ interface
+                call pfnUnmapDevPhysHeap.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerUnmapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						  IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerOnVmOnline
+ @Description   The VM manager calls this when guest VM machine comes online.
+                The host driver will initialize the FW if it has not done so
+                already.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerOnVmOnline(IMG_UINT32 ui32DriverID,
+					IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerOnVmOffline
+ @Description   The VM manager calls this when a guest VM machine is about to
+                go offline. The VM manager might have unmapped the GPU kick
+                register for such VM but not the GPU memory until the call
+                returns. Once the function returns, the FW does not hold any
+                reference for such VM and no workloads from it are running in
+                the GPU and it is safe to remove the memory for such VM.
+ @Return        PVRSRV_OK on success. PVRSRV_ERROR_TIMEOUT if for some reason
+                the FW is taking too long to clean-up the resources of the
+                DriverID. Otherwise, a PVRSRV_ERROR code.
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerOnVmOffline(IMG_UINT32 ui32DriverID,
+					 IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerVMMConfigure
+ @Description   The VM manager calls this to configure several parameters like
+                HCS or isolation.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+					  IMG_UINT32 ui32ParamValue,
+					  IMG_UINT32 ui32DevID);
+
+#endif /* VMM_PVZ_SERVER_H */
+
+/******************************************************************************
+ End of file (vmm_pvz_server.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vm.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vm.h
new file mode 100644
index 000000000000..ee4a7fb5b3a7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vm.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File			vz_vm.h
+@Title          System virtualization VM support APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides VM management support APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VZ_VM_H
+#define VZ_VM_H
+
+#include "vmm_impl.h"
+
+bool IsVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzOnVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzOnVmOffline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+							 IMG_UINT32 ui32ParamValue,
+							 IMG_UINT32 ui32DevID);
+
+#endif /* VZ_VM_H */
+
+/*****************************************************************************
+ End of file (vz_vm.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vmm_pvz.h b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vmm_pvz.h
new file mode 100644
index 000000000000..03d5525936b8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/server/include/vz_vmm_pvz.h
@@ -0,0 +1,79 @@
+/*************************************************************************/ /*!
+@File           vz_vmm_pvz.h
+@Title          System virtualization VM manager management APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides VM manager para-virtz management APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VZ_VMM_PVZ_H
+#define VZ_VMM_PVZ_H
+
+#include "img_types.h"
+#include "vmm_impl.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzConnectionInit() and PvzConnectionDeInit()
+ @Description   PvzConnectionInit initializes the VM manager para-virt
+                which is used subsequently for communication between guest and
+                host; depending on the underlying VM setup, this could either
+                be a hyper-call or cross-VM call
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR PvzConnectionInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+void PvzConnectionDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+*******************************************************************************
+ @Function      PvzConnectionAcquire() and PvzConnectionRelease()
+ @Description   These are to acquire/release a handle to the VM manager
+                para-virtz connection to make a pvz call; on the client, use it
+                to make the actual pvz call and on the server handler /
+                VM manager, use it to complete the processing for the pvz call
+                or make a VM manager to host pvzbridge call
+@Return         VMM_PVZ_CONNECTION* on success. Otherwise NULL
+******************************************************************************/
+VMM_PVZ_CONNECTION* PvzConnectionAcquire(void);
+void PvzConnectionRelease(VMM_PVZ_CONNECTION *psPvzConnection);
+
+#endif /* VZ_VMM_PVZ_H */
+
+/******************************************************************************
+ End of file (vz_vmm_pvz.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem.c
new file mode 100644
index 000000000000..c032094ff0b3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem.c
@@ -0,0 +1,2754 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Front End (nominally Client side part, but now invokable
+                from server too) of device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "devicemem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "osfunc.h"
+#include "osmmap.h"
+#include "devicemem_utils.h"
+#include "client_mm_bridge.h"
+#include "client_cache_bridge.h"
+#include "services_km.h"
+
+#if defined(PDUMP)
+#if defined(__KERNEL__)
+#include "pdump_km.h"
+#else
+#include "pdump_um.h"
+#endif
+#include "devicemem_pdump.h"
+#endif
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "client_ri_bridge.h"
+#endif
+#include "client_devicememhistory_bridge.h"
+#include "info_page_client.h"
+
+#include "rgx_heaps.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "pvr_ricommon.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "srvcore.h"
+#if defined(__linux__)
+#include "linux/kernel.h"
+#endif
+#else
+#include "srvcore_intern.h"
+#include "rgxdefs.h"
+#endif
+
+#if defined(__KERNEL__) && defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+extern PVRSRV_ERROR RIDumpAllKM(void);
+#endif
+
+#if defined(__KERNEL__)
+#define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError)
+#else
+#define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError)
+#endif
+
+#if defined(__KERNEL__)
+/* Derive the virtual from the hPMR */
+static
+IMG_UINT64 _GetPremappedVA(PMR *psPMR, PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+
+	IMG_DEV_PHYADDR sDevAddr;
+	IMG_BOOL bValid;
+	PHYS_HEAP *psPhysHeap = psDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+	IMG_DEV_PHYADDR sHeapAddr;
+
+	eError = PhysHeapGetDevPAddr(psPhysHeap, &sHeapAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", fail);
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+{
+	if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_UMA ||
+			PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_DMA)
+	{
+		IMG_DEV_PHYADDR sDevPAddrCorrected;
+
+		PhysHeapCpuPAddrToDevPAddr(psPhysHeap, 1, &sDevPAddrCorrected, (IMG_CPU_PHYADDR *)&sHeapAddr);
+		sHeapAddr.uiAddr = sDevPAddrCorrected.uiAddr;
+	}
+}
+#endif
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddr", fail);
+
+	eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sDevAddr, &bValid, DEVICE_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMR_DevPhysAddr");
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddr");
+		goto fail;
+	}
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddr");
+
+	ui64OptionalMapAddress = RGX_FIRMWARE_RAW_HEAP_BASE | (sDevAddr.uiAddr - sHeapAddr.uiAddr);
+
+	PVR_DPF((PVR_DBG_ALLOC, "%s: sDevAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" sHeapAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" => ui64OptionalMapAddress = 0x%"IMG_UINT64_FMTSPECx,
+	         __func__, sDevAddr.uiAddr, sHeapAddr.uiAddr, ui64OptionalMapAddress));
+fail:
+	return ui64OptionalMapAddress;
+}
+#endif
+
+/*****************************************************************************
+ *                    Sub allocation internals                               *
+ *****************************************************************************/
+static INLINE PVRSRV_MEMALLOCFLAGS_T
+DevmemOverrideFlagsOrPassThrough(SHARED_DEV_CONNECTION hDevConnection, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+#if defined(__KERNEL__) && defined(RGX_FEATURE_GPU_CPU_COHERENCY_BIT_MASK)
+	/*
+	 *  Override the requested memory flags of FW allocations only,
+	 *  non-FW allocations pass-through unmodified.
+	 *
+	 *  On ace-lite platforms:
+	 *    - If the allocation is not CPU cached, then there is nothing
+	 *      for the GPU to snoop regardless of the GPU cache setting.
+	 *    - If the allocation is not GPU cached, then the SLC will not
+	 *      be used and will not snoop the CPU even if it is CPU cached.
+	 *    - Therefore only the GPU setting can be upgraded to coherent
+	 *      if it is already GPU cached incoherent and the CPU is cached.
+	 *
+	 *  All other platforms:
+	 *    - Do not modify the allocation flags.
+	 */
+
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDevConnection;
+
+	if (psDevNode->pvDevice != NULL && PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, GPU_CPU_COHERENCY))
+	{
+		if (PVRSRV_CHECK_FW_MAIN(uiFlags))
+		{
+			if ((PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) || PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags)) &&
+			         (PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags)) &&
+			         PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+			         psDevNode->eDevFabricType == PVRSRV_DEVICE_FABRIC_ACELITE)
+			{
+				/* Upgrade the allocation from GPU cached incoherent to GPU cached coherent. */
+				uiFlags &= ~PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK;
+				uiFlags |= PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hDevConnection);
+#endif
+
+	return uiFlags;
+}
+
+static INLINE void
+CheckAnnotationLength(const IMG_CHAR *pszAnnotation)
+{
+	IMG_UINT32 length = OSStringLength(pszAnnotation);
+
+	if (length >= DEVMEM_ANNOTATION_MAX_LEN)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Annotation \"%s\" has been truncated to %d characters from %d characters",
+				__func__, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1, length));
+	}
+}
+
+static PVRSRV_ERROR
+AllocateDeviceMemory(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiLog2Quantum,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_UINT32 ui32NumPhysChunks,
+		IMG_UINT32 ui32NumVirtChunks,
+		IMG_UINT32 *pui32MappingTable,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		IMG_BOOL bExportable,
+		const IMG_CHAR *pszAnnotation,
+		DEVMEM_IMPORT **ppsImport)
+{
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failAlloc);
+
+	/* check if shift value is not too big (sizeof(1ULL)) */
+	PVR_ASSERT(uiLog2Quantum < sizeof(unsigned long long) * 8);
+	/* Check the size is a multiple of the quantum */
+	PVR_ASSERT((uiSize & ((1ULL<<uiLog2Quantum)-1)) == 0);
+
+	CheckAnnotationLength(pszAnnotation);
+
+	/* Pass only the PMR flags down */
+	uiOutFlags = uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK;
+	eError = BridgePhysmemNewRamBackedPMR(GetBridgeHandle(hDevConnection),
+			uiSize,
+			ui32NumPhysChunks,
+			ui32NumVirtChunks,
+			pui32MappingTable,
+			uiLog2Quantum,
+			uiOutFlags,
+			OSStringNLength(pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1) + 1,
+			pszAnnotation,
+			OSGetCurrentProcessID(),
+			&hPMR,
+			PDUMP_NONE,
+			&uiOutFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate memory for %s (%s)",
+				__func__,
+				pszAnnotation,
+				PVRSRVGETERRORSTRING(eError)));
+		goto failPMR;
+	}
+
+	uiFlags &= ~PVRSRV_PHYS_HEAP_HINT_MASK;
+	uiFlags |= (uiOutFlags & PVRSRV_PHYS_HEAP_HINT_MASK);
+
+	DevmemImportStructInit(psImport,
+			uiSize,
+			uiAlign,
+			uiFlags,
+			hPMR,
+			bExportable ? DEVMEM_PROPERTIES_EXPORTABLE : 0);
+
+	*ppsImport = psImport;
+	return PVRSRV_OK;
+
+failPMR:
+	DevmemImportDiscard(psImport);
+failAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*****************************************************************************
+ *                    Sub allocation internals                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc,
+		IMG_UINT32 ui32AllocPageCount,
+		IMG_UINT32 *paui32AllocPageIndices,
+		IMG_UINT32 ui32FreePageCount,
+		IMG_UINT32 *pauiFreePageIndices,
+		SPARSE_MEM_RESIZE_FLAGS uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_IMPORT *psImport;
+	SHARED_DEV_CONNECTION hDevConnection;
+	IMG_HANDLE hPMR;
+	DEVMEM_HEAP *psHeap;
+	IMG_HANDLE hReservation;
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	DEVMEM_PROPERTIES_T uiProperties;
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(psMemDesc->psImport != NULL);
+
+	psImport = psMemDesc->psImport;
+	hDevConnection = psImport->hDevConnection;
+	hPMR = psImport->hPMR;
+	psHeap = psImport->sDeviceImport.psHeap;
+	hReservation = psImport->sDeviceImport.hReservation;
+
+	PVR_ASSERT(hDevConnection != NULL);
+	PVR_ASSERT(hPMR != NULL);
+	PVR_ASSERT(psHeap != NULL);
+	PVR_ASSERT(BITMASK_ANY(uiSparseFlags, SPARSE_RESIZE_BOTH));
+	PVR_ASSERT(hReservation != LACK_OF_RESERVATION_POISON);
+
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	PVR_ASSERT(!BITMASK_HAS(uiProperties, DEVMEM_PROPERTIES_SECURE));
+	PVR_ASSERT(!BITMASK_HAS(uiProperties, DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE));
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount == 0);
+
+	OSLockAcquire(psImport->hLock);
+
+	eError = BridgeChangeSparseMem(GetBridgeHandle(hDevConnection),
+	                               ui32AllocPageCount,
+	                               paui32AllocPageIndices,
+	                               ui32FreePageCount,
+	                               pauiFreePageIndices,
+	                               uiSparseFlags,
+	                               hReservation);
+
+	OSLockRelease(psImport->hLock);
+
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (GetInfoPageDebugFlags(hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistorySparseChange(GetBridgeHandle(hDevConnection),
+		                                   hPMR,
+		                                   psMemDesc->uiOffset,
+		                                   psMemDesc->sDeviceMemDesc.sDevVAddr,
+		                                   psMemDesc->uiAllocSize,
+		                                   psMemDesc->szText,
+		                                   DevmemGetHeapLog2PageSize(psHeap),
+		                                   ui32AllocPageCount,
+		                                   paui32AllocPageIndices,
+		                                   ui32FreePageCount,
+		                                   pauiFreePageIndices,
+		                                   psMemDesc->ui32AllocationIndex,
+		                                   &psMemDesc->ui32AllocationIndex);
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+FreeDeviceMemory(DEVMEM_IMPORT *psImport)
+{
+	DevmemImportStructRelease(psImport);
+}
+
+static PVRSRV_ERROR
+SubAllocImportAlloc(RA_PERARENA_HANDLE hArena,
+		RA_LENGTH_T uiSize,
+		RA_FLAGS_T _flags,
+		RA_LENGTH_T uBaseAlignment,
+		const IMG_CHAR *pszAnnotation,
+		/* returned data */
+		RA_IMPORT *psRAImport)
+{
+	/* When suballocations need a new lump of memory, the RA calls
+	   back here.  Later, in the kernel, we must construct a new PMR
+	   and a pairing between the new lump of virtual memory and the
+	   PMR (whether or not such PMR is backed by physical memory) */
+	DEVMEM_HEAP *psHeap;
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags = (PVRSRV_MEMALLOCFLAGS_T) _flags;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+
+	/* Per-arena private handle is, for us, the heap */
+	psHeap = hArena;
+
+	/* The RA should not have invoked us with a size that is not a
+	   multiple of the quantum anyway */
+	PVR_ASSERT((uiSize & ((1ULL<<psHeap->uiLog2Quantum)-1)) == 0);
+
+	eError = AllocateDeviceMemory(psHeap->psCtx->hDevConnection,
+			psHeap->uiLog2Quantum,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			uBaseAlignment,
+			uiFlags,
+			IMG_FALSE,
+			"PMR sub-allocated",
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failAlloc);
+
+#if defined(PDUMP) && defined(DEBUG)
+#if defined(__KERNEL__)
+	PDUMPCOMMENTWITHFLAGS(PMR_DeviceNode((PMR*)psImport->hPMR), PDUMP_CONT,
+			"Created PMR for sub-allocations with handle ID: 0x%p Annotation: \"%s\" (PID %u)",
+			psImport->hPMR, pszAnnotation, OSGetCurrentProcessID());
+#else
+	PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS,
+			"Created PMR for sub-allocations with handle ID: %p Annotation: \"%s\" (PID %u)",
+			psImport->hPMR, pszAnnotation, OSGetCurrentProcessID());
+#endif
+#else
+	PVR_UNREFERENCED_PARAMETER(pszAnnotation);
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+#if defined(__KERNEL__)
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection;
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice;
+
+		PVR_ASSERT(PVRSRV_CHECK_FW_MAIN(uiFlags));
+
+		/* If allocation is made by the Kernel from the firmware heap, account for it
+		 * under the PVR_SYS_ALLOC_PID.
+		 */
+		if ((psHeap == psDevInfo->psFirmwareMainHeap) || (psHeap == psDevInfo->psFirmwareConfigHeap))
+		{
+			eError = BridgeRIWritePMREntryWithOwner (GetBridgeHandle(psImport->hDevConnection),
+					psImport->hPMR,
+					PVR_SYS_ALLOC_PID);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntryWithOwner");
+		}
+		else
+#endif
+		{
+			eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+					psImport->hPMR);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+		}
+	}
+#endif
+
+#if defined(__KERNEL__)
+	if (psHeap->bPremapped)
+	{
+		ui64OptionalMapAddress = _GetPremappedVA(psImport->hPMR, psHeap->psCtx->hDevConnection);
+	}
+#endif
+
+	/*
+		Suballocations always get mapped into the device was we need to
+		key the RA off something and as we can't export suballocations
+		there is no valid reason to request an allocation an not map it
+	 */
+	eError = DevmemImportStructDevMap(psHeap,
+			IMG_TRUE,
+			psImport,
+			ui64OptionalMapAddress);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	OSLockAcquire(psImport->hLock);
+	/* Mark this import struct as zeroed so we can save some PDump LDBs
+	 * and do not have to CPU map + mem set()*/
+	if (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC)
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_ZEROED;
+	}
+	else if (uiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC)
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_POISONED;
+	}
+	psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_CLEAN;
+	OSLockRelease(psImport->hLock);
+
+	psRAImport->base = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	psRAImport->uSize = uiSize;
+	psRAImport->hPriv = psImport;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failMap:
+	FreeDeviceMemory(psImport);
+failAlloc:
+
+	return eError;
+}
+
+static void
+SubAllocImportFree(RA_PERARENA_HANDLE hArena,
+		RA_BASE_T uiBase,
+		RA_PERISPAN_HANDLE hImport)
+{
+	DEVMEM_IMPORT *psImport = hImport;
+#if !defined(PVRSRV_NEED_PVR_ASSERT)
+	PVR_UNREFERENCED_PARAMETER(hArena);
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+#endif
+
+	PVR_ASSERT(psImport != NULL);
+	PVR_ASSERT(hArena == psImport->sDeviceImport.psHeap);
+	PVR_ASSERT(uiBase == psImport->sDeviceImport.sDevVAddr.uiAddr);
+
+	(void) DevmemImportStructDevUnmap(psImport);
+	(void) DevmemImportStructRelease(psImport);
+}
+
+/*****************************************************************************
+ *                    Devmem context internals                               *
+ *****************************************************************************/
+
+static PVRSRV_ERROR
+PopulateContextFromBlueprint(struct DEVMEM_CONTEXT_TAG *psCtx,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	struct DEVMEM_HEAP_TAG **ppsHeapArray;
+	IMG_UINT32 uiNumHeaps;
+	IMG_UINT32 uiHeapsToUnwindOnError;
+	IMG_UINT32 uiHeapIndex;
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_CHAR aszHeapName[DEVMEM_HEAPNAME_MAXLENGTH];
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2DataPageSize;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2ImportAlignment;
+
+	eError = DevmemHeapCount(psCtx->hDevConnection,
+			uiHeapBlueprintID,
+			&uiNumHeaps);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (uiNumHeaps == 0)
+	{
+		ppsHeapArray = NULL;
+	}
+	else
+	{
+		ppsHeapArray = OSAllocMem(sizeof(*ppsHeapArray) * uiNumHeaps);
+		PVR_GOTO_IF_NOMEM(ppsHeapArray, eError, e0);
+	}
+
+	uiHeapsToUnwindOnError = 0;
+
+	for (uiHeapIndex = 0; uiHeapIndex < uiNumHeaps; uiHeapIndex++)
+	{
+		eError = DevmemHeapDetails(psCtx->hDevConnection,
+				uiHeapBlueprintID,
+				uiHeapIndex,
+				&aszHeapName[0],
+				sizeof(aszHeapName),
+				&sDevVAddrBase,
+				&uiHeapLength,
+				&uiReservedRegionLength,
+				&uiLog2DataPageSize,
+				&uiLog2ImportAlignment);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		eError = DevmemCreateHeap(psCtx,
+				sDevVAddrBase,
+				uiHeapLength,
+				uiReservedRegionLength,
+				uiLog2DataPageSize,
+				uiLog2ImportAlignment,
+				aszHeapName,
+				uiHeapBlueprintID,
+				uiHeapIndex,
+				&ppsHeapArray[uiHeapIndex]);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		uiHeapsToUnwindOnError = uiHeapIndex + 1;
+	}
+
+	psCtx->uiAutoHeapCount = uiNumHeaps;
+	psCtx->ppsAutoHeapArray = ppsHeapArray;
+
+	PVR_ASSERT(psCtx->uiNumHeaps >= psCtx->uiAutoHeapCount);
+	PVR_ASSERT(psCtx->uiAutoHeapCount == uiNumHeaps);
+
+	return PVRSRV_OK;
+
+	/* error exit paths */
+e1:
+	for (uiHeapIndex = 0; uiHeapIndex < uiHeapsToUnwindOnError; uiHeapIndex++)
+	{
+		eError2 = DevmemDestroyHeap(ppsHeapArray[uiHeapIndex]);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+	if (uiNumHeaps != 0)
+	{
+		OSFreeMem(ppsHeapArray);
+	}
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+UnpopulateContextFromBlueprint(struct DEVMEM_CONTEXT_TAG *psCtx)
+{
+	PVRSRV_ERROR eReturn = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+	IMG_UINT32 uiHeapIndex;
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+
+	for (uiHeapIndex = 0; uiHeapIndex < psCtx->uiAutoHeapCount; uiHeapIndex++)
+	{
+		if (!psCtx->ppsAutoHeapArray[uiHeapIndex])
+		{
+			continue;
+		}
+
+		eError2 = DevmemDestroyHeap(psCtx->ppsAutoHeapArray[uiHeapIndex]);
+		if (eError2 != PVRSRV_OK)
+		{
+			eReturn = eError2;
+		}
+		else
+		{
+			psCtx->ppsAutoHeapArray[uiHeapIndex] = NULL;
+		}
+	}
+
+	if ((!bDoCheck || (eReturn == PVRSRV_OK)) && psCtx->ppsAutoHeapArray)
+	{
+		OSFreeMem(psCtx->ppsAutoHeapArray);
+		psCtx->ppsAutoHeapArray = NULL;
+		psCtx->uiAutoHeapCount = 0;
+	}
+
+	return eReturn;
+}
+
+/*****************************************************************************
+ *                    Devmem context functions                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID,
+		DEVMEM_CONTEXT **ppsCtxPtr)
+{
+	PVRSRV_ERROR		eError;
+	DEVMEM_CONTEXT		*psCtx;
+	/* handle to the server-side counterpart of the device memory
+	   context (specifically, for handling mapping to device MMU) */
+	IMG_HANDLE			hDevMemServerContext;
+	IMG_HANDLE			hPrivData;
+	IMG_BOOL			bHeapCfgFWId = (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW);
+
+	PVR_GOTO_IF_NOMEM(ppsCtxPtr, eError, e0);
+
+	psCtx = OSAllocMem(sizeof(*psCtx));
+	PVR_GOTO_IF_NOMEM(psCtx, eError, e0);
+
+	psCtx->uiNumHeaps = 0;
+
+	psCtx->hDevConnection = hDevConnection;
+
+	/* Create (server-side) Device Memory context */
+	eError = BridgeDevmemIntCtxCreate(GetBridgeHandle(psCtx->hDevConnection),
+			bHeapCfgFWId,
+			&hDevMemServerContext,
+			&hPrivData,
+			&psCtx->ui32CPUCacheLineSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	psCtx->hDevMemServerContext = hDevMemServerContext;
+	psCtx->hPrivData = hPrivData;
+
+	/* automagic heap creation */
+	psCtx->uiAutoHeapCount = 0;
+
+	eError = PopulateContextFromBlueprint(psCtx, uiHeapBlueprintID);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	*ppsCtxPtr = psCtx;
+
+	PVR_ASSERT(psCtx->uiNumHeaps == psCtx->uiAutoHeapCount);
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+e2:
+	PVR_ASSERT(psCtx->uiAutoHeapCount == 0);
+	PVR_ASSERT(psCtx->uiNumHeaps == 0);
+	BridgeDevmemIntCtxDestroy(GetBridgeHandle(psCtx->hDevConnection), hDevMemServerContext);
+
+e1:
+	OSFreeMem(psCtx);
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
+		IMG_HANDLE *hPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(psCtx, eError, e0);
+	PVR_GOTO_IF_INVALID_PARAM(hPrivData, eError, e0);
+
+	*hPrivData = psCtx->hPrivData;
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(psCtx, eError, e0);
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemFindHeapByName(const struct DEVMEM_CONTEXT_TAG *psCtx,
+		const IMG_CHAR *pszHeapName,
+		struct DEVMEM_HEAP_TAG **ppsHeapRet)
+{
+	IMG_UINT32 uiHeapIndex;
+
+	/* N.B.  This func is only useful for finding "automagic" heaps by name */
+	for (uiHeapIndex = 0;
+			uiHeapIndex < psCtx->uiAutoHeapCount;
+			uiHeapIndex++)
+	{
+		if (!OSStringNCompare(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName, pszHeapName, OSStringLength(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName) + 1))
+		{
+			*ppsHeapRet = psCtx->ppsAutoHeapArray[uiHeapIndex];
+			return PVRSRV_OK;
+		}
+	}
+
+	return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemDestroyContext(DEVMEM_CONTEXT *psCtx)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bDoCheck = IMG_TRUE;
+
+#if defined(__KERNEL__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+
+	PVR_RETURN_IF_INVALID_PARAM(psCtx);
+
+	eError = UnpopulateContextFromBlueprint(psCtx);
+	if (bDoCheck && eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: UnpopulateContextFromBlueprint failed (%d) leaving %d heaps",
+				__func__, eError, psCtx->uiNumHeaps));
+		goto e1;
+	}
+
+	eError = DestroyServerResource(psCtx->hDevConnection,
+	                               NULL,
+	                               BridgeDevmemIntCtxDestroy,
+	                               psCtx->hDevMemServerContext);
+	if (bDoCheck)
+	{
+		PVR_LOG_GOTO_IF_ERROR(eError, "BridgeDevMemIntCtxDestroy", e1);
+
+		/* should be no more heaps left */
+		if (psCtx->uiNumHeaps)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Additional heaps remain in DEVMEM_CONTEXT",
+					__func__));
+			eError = PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT;
+			goto e1;
+		}
+	}
+
+	OSCachedMemSet(psCtx, 0, sizeof(*psCtx));
+	OSFreeMem(psCtx);
+
+e1:
+	return eError;
+}
+
+/*****************************************************************************
+ *                 Devmem heap query functions                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 *puiNumHeapConfigsOut)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapConfigCount(GetBridgeHandle(hDevConnection),
+			puiNumHeapConfigsOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_UINT32 *puiNumHeapsOut)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapCount(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			puiNumHeapsOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapConfigName(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_CHAR *pszConfigNameOut,
+		IMG_UINT32 uiConfigNameBufSz)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapConfigName(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			uiConfigNameBufSz,
+			pszConfigNameOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_UINT32 uiHeapIndex,
+		IMG_CHAR *pszHeapNameOut,
+		IMG_UINT32 uiHeapNameBufSz,
+		IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+		IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+		IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+		IMG_UINT32 *puiLog2DataPageSizeOut,
+		IMG_UINT32 *puiLog2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgeHeapCfgHeapDetails(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			uiHeapIndex,
+			uiHeapNameBufSz,
+			pszHeapNameOut,
+			psDevVAddrBaseOut,
+			puiHeapLengthOut,
+			puiReservedRegionLengthOut,
+			puiLog2DataPageSizeOut,
+			puiLog2ImportAlignmentOut);
+
+	VG_MARK_INITIALIZED(pszHeapNameOut, uiHeapNameBufSz);
+
+	return eError;
+}
+
+/*****************************************************************************
+ *                    Devmem heap functions                                  *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapInt(DEVMEM_HEAP *psHeap,
+		IMG_HANDLE *phDevmemHeap)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+	*phDevmemHeap = psHeap->hDevMemServerHeap;
+	return PVRSRV_OK;
+}
+
+/* See devicemem.h for important notes regarding the arguments
+   to this function */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemCreateHeap(DEVMEM_CONTEXT *psCtx,
+		IMG_DEV_VIRTADDR sBaseAddress,
+		IMG_DEVMEM_SIZE_T uiLength,
+		IMG_DEVMEM_SIZE_T uiReservedRegionLength,
+		IMG_UINT32 ui32Log2Quantum,
+		IMG_UINT32 ui32Log2ImportAlignment,
+		const IMG_CHAR *pszName,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID,
+		IMG_UINT32 uiHeapIndex,
+		DEVMEM_HEAP **ppsHeapPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+	DEVMEM_HEAP *psHeap;
+	/* handle to the server-side counterpart of the device memory heap
+	   (specifically, for handling mapping to device MMU) */
+	IMG_HANDLE hDevMemServerHeap;
+	IMG_UINT32 ui32Policy = RA_POLICY_DEFAULT, ui32PolicyVMRA;
+
+	IMG_CHAR aszBuf[100];
+	IMG_CHAR *pszStr;
+	IMG_UINT32 ui32pszStrSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsHeapPtr, "ppsHeapPtr");
+
+	/* Reserved VA space of a heap must always be multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY.
+	 * Granularity has been chosen to support the max possible practically used OS page size. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uiReservedRegionLength % DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY) == 0, "uiReservedRegionLength");
+
+	ui32PolicyVMRA = RA_POLICY_DEFAULT;
+
+	PVR_ASSERT(uiReservedRegionLength + DEVMEM_HEAP_MINIMUM_SIZE <= uiLength);
+
+	psHeap = OSAllocMem(sizeof(*psHeap));
+	PVR_GOTO_IF_NOMEM(psHeap, eError, HeapAllocError);
+
+	/* Need to keep local copy of heap name, so caller may free theirs */
+	ui32pszStrSize = OSStringLength(pszName) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	PVR_GOTO_IF_NOMEM(pszStr, eError, SubAllocRANameCopyError);
+	OSStringSafeCopy(pszStr, pszName, ui32pszStrSize);
+	psHeap->pszName = pszStr;
+
+	psHeap->uiSize = uiLength;
+	psHeap->uiReservedRegionSize = uiReservedRegionLength;
+	psHeap->sBaseAddress = sBaseAddress;
+	psHeap->bPremapped = IMG_FALSE;
+	OSAtomicWrite(&psHeap->hImportCount, 0);
+
+	OSSNPrintf(aszBuf, sizeof(aszBuf),
+			"NDM heap '%s' (suballocs) ctx:%p",
+			pszName, psCtx);
+	ui32pszStrSize = OSStringLength(aszBuf) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	PVR_GOTO_IF_NOMEM(pszStr, eError, SubAllocRANameError);
+	OSStringSafeCopy(pszStr, aszBuf, ui32pszStrSize);
+	psHeap->pszSubAllocRAName = pszStr;
+
+#if defined(__KERNEL__)
+	if (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW)
+	{
+		void *pvAppHintState = NULL;
+		IMG_UINT32 ui32FirmwarePolicydefault = PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY;
+		IMG_UINT32 ui32FirmwarePolicy = PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY;
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, DevMemFWHeapPolicy,
+						&ui32FirmwarePolicydefault, &ui32FirmwarePolicy);
+		ui32PolicyVMRA = ui32Policy = ui32FirmwarePolicy;
+		OSFreeAppHintState(pvAppHintState);
+
+		/* Flag the change from default setting */
+		if (ui32FirmwarePolicy != ui32FirmwarePolicydefault)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: %s: DevMemFWHeapPolicy set to %u, default %u",
+			         __func__, pszStr,
+			         ui32FirmwarePolicy, ui32FirmwarePolicydefault));
+		}
+	}
+#endif
+
+#if defined(PDUMP)
+	/* The META heap is shared globally so a single physical memory import
+	 * may be used to satisfy allocations of different processes.
+	 * This is problematic when PDumping because the physical memory
+	 * import used to satisfy a new allocation may actually have been
+	 * imported (and thus the PDump MALLOC generated) before the PDump
+	 * client was started, leading to the MALLOC being missing.
+	 *
+	 * This is solved by disabling splitting of imports for the META physmem
+	 * RA, meaning that every firmware allocation gets its own import, thus
+	 * ensuring the MALLOC is present for every allocation made within the
+	 * pdump capture range
+	 */
+	if (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW)
+	{
+		ui32Policy |= RA_POLICY_NO_SPLIT;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiHeapBlueprintID);
+#endif
+
+	psHeap->psSubAllocRA = RA_Create(psHeap->pszSubAllocRAName,
+			/* Subsequent imports: */
+			ui32Log2Quantum,
+			RA_LOCKCLASS_3,
+			SubAllocImportAlloc,
+			SubAllocImportFree,
+			(RA_PERARENA_HANDLE) psHeap,
+			ui32Policy);
+	if (psHeap->psSubAllocRA == NULL)
+	{
+		eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
+		goto SubAllocRACreateError;
+	}
+
+	psHeap->uiLog2ImportAlignment = ui32Log2ImportAlignment;
+	psHeap->uiLog2Quantum = ui32Log2Quantum;
+
+	if (!OSStringNCompare(pszName, RGX_GENERAL_SVM_HEAP_IDENT, sizeof(RGX_GENERAL_SVM_HEAP_IDENT)))
+	{
+		/* The SVM heap normally starts out as this type though
+		   it may transition to DEVMEM_HEAP_MANAGER_USER
+		   on platforms with more processor virtual address
+		   bits than device virtual address bits */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_KERNEL;
+	}
+	else if (uiReservedRegionLength != 0)
+	{
+		/* Heaps which specify reserved VA space range are dual managed:
+		 * - sBaseAddress to (sBaseAddress+uiReservedRegionLength-1): User managed
+		 * - (sBaseAddress+uiReservedRegionLength) to uiLength: RA managed
+		 */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_DUAL_USER_RA;
+	}
+	else
+	{
+		/* Otherwise, heap manager is decided (USER or RA) at first map */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_UNKNOWN;
+	}
+
+	/* Mark the heap to be managed by RA */
+	if (!OSStringNCompare(pszName, RGX_VK_CAPT_REPLAY_HEAP_IDENT,
+	                      sizeof(RGX_VK_CAPT_REPLAY_HEAP_IDENT)))
+	{
+		psHeap->ui32HeapManagerFlags |= DEVMEM_HEAP_MANAGER_RA;
+	}
+
+	OSSNPrintf(aszBuf, sizeof(aszBuf),
+			"NDM heap '%s' (QVM) ctx:%p",
+			pszName, psCtx);
+	ui32pszStrSize = OSStringLength(aszBuf) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	if (pszStr == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto VMRANameError;
+	}
+	OSStringSafeCopy(pszStr, aszBuf, ui32pszStrSize);
+	psHeap->pszQuantizedVMRAName = pszStr;
+
+	psHeap->psQuantizedVMRA = RA_Create(psHeap->pszQuantizedVMRAName,
+			/* Subsequent import: */
+			0, RA_LOCKCLASS_1, NULL, NULL,
+			(RA_PERARENA_HANDLE) psHeap,
+			ui32PolicyVMRA);
+	if (psHeap->psQuantizedVMRA == NULL)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA, VMRACreateError);
+	}
+
+	if (!RA_Add(psHeap->psQuantizedVMRA,
+			/* Make sure the VMRA doesn't allocate from reserved VAs */
+			(RA_BASE_T)sBaseAddress.uiAddr + uiReservedRegionLength,
+			(RA_LENGTH_T)uiLength,
+			(RA_FLAGS_T)0, /* This RA doesn't use or need flags */
+			NULL /* per ispan handle */))
+	{
+		RA_Delete(psHeap->psQuantizedVMRA);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA, VMRACreateError);
+	}
+
+	psHeap->psCtx = psCtx;
+
+
+	/* Create server-side counterpart of Device Memory heap */
+	eError = BridgeDevmemIntHeapCreate(GetBridgeHandle(psCtx->hDevConnection),
+	                                   psCtx->hDevMemServerContext,
+	                                   uiHeapBlueprintID,
+	                                   uiHeapIndex,
+	                                   &hDevMemServerHeap);
+	PVR_GOTO_IF_ERROR(eError, ServerHeapCreateError);
+
+	psHeap->hDevMemServerHeap = hDevMemServerHeap;
+
+	eError = OSLockCreate(&psHeap->hLock);
+	PVR_GOTO_IF_ERROR(eError, LockCreateErr);
+
+	psHeap->psCtx->uiNumHeaps++;
+	*ppsHeapPtr = psHeap;
+
+	return PVRSRV_OK;
+
+	/* error exit paths */
+LockCreateErr:
+	eError2 = BridgeDevmemIntHeapDestroy(GetBridgeHandle(psCtx->hDevConnection),
+			psHeap->hDevMemServerHeap);
+	PVR_ASSERT (eError2 == PVRSRV_OK);
+ServerHeapCreateError:
+	RA_Delete(psHeap->psQuantizedVMRA);
+VMRACreateError:
+	OSFreeMem(psHeap->pszQuantizedVMRAName);
+VMRANameError:
+	RA_Delete(psHeap->psSubAllocRA);
+SubAllocRACreateError:
+	OSFreeMem(psHeap->pszSubAllocRAName);
+SubAllocRANameError:
+	OSFreeMem(psHeap->pszName);
+SubAllocRANameCopyError:
+	OSFreeMem(psHeap);
+HeapAllocError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapBaseDevVAddr(struct DEVMEM_HEAP_TAG *psHeap,
+		IMG_DEV_VIRTADDR *pDevVAddr)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+
+	*pDevVAddr = psHeap->sBaseAddress;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL DEVMEM_SIZE_T
+DevmemGetHeapSize(struct DEVMEM_HEAP_TAG *psHeap)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap != NULL);
+
+	return psHeap->uiSize;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum,
+		IMG_DEVMEM_SIZE_T *puiSize,
+		IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	IMG_DEVMEM_SIZE_T uiSize = *puiSize;
+	IMG_DEVMEM_ALIGN_T uiAlign = *puiAlign;
+
+	/* Just in case someone changes definition of IMG_DEVMEM_ALIGN_T. */
+	static_assert(sizeof(unsigned long long) == sizeof(uiAlign),
+	              "invalid uiAlign size");
+	/* This value is used for shifting so it cannot be greater than number
+	 * of bits in unsigned long long (sizeof(1ULL)). Using greater value is
+	 * undefined behaviour. */
+	if (uiLog2Quantum >= sizeof(unsigned long long) * 8)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((1ULL << uiLog2Quantum) > uiAlign)
+	{
+		uiAlign = 1ULL << uiLog2Quantum;
+	}
+	uiSize = PVR_ALIGN(uiSize, uiAlign);
+
+	*puiSize = uiSize;
+	*puiAlign = uiAlign;
+
+	return PVRSRV_OK;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemDestroyHeap(DEVMEM_HEAP *psHeap)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT uiImportCount;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__)
+	if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+#endif
+
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+
+	uiImportCount = OSAtomicRead(&psHeap->hImportCount);
+	if (uiImportCount > 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%d(%s) leaks remain", uiImportCount, psHeap->pszName));
+#if defined(__KERNEL__)
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		PVR_DPF((PVR_DBG_ERROR, "Details of remaining allocated device memory (for all processes):"));
+		RIDumpAllKM();
+#else
+		PVR_DPF((PVR_DBG_ERROR, "Compile with PVRSRV_ENABLE_GPU_MEMORY_INFO=1 to get a full "
+				"list of all driver allocations."));
+#endif
+#endif
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		if (bDoCheck)
+#endif
+		{
+			return PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP;
+		}
+	}
+
+	eError = DestroyServerResource(psHeap->psCtx->hDevConnection,
+	                               NULL,
+	                               BridgeDevmemIntHeapDestroy,
+	                               psHeap->hDevMemServerHeap);
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (bDoCheck)
+#endif
+	{
+		PVR_LOG_RETURN_IF_ERROR(eError, "BridgeDevmemIntHeapDestroy");
+	}
+
+	PVR_ASSERT(psHeap->psCtx->uiNumHeaps > 0);
+	psHeap->psCtx->uiNumHeaps--;
+
+	OSLockDestroy(psHeap->hLock);
+
+	if (psHeap->psQuantizedVMRA)
+	{
+		RA_Delete(psHeap->psQuantizedVMRA);
+	}
+	if (psHeap->pszQuantizedVMRAName)
+	{
+		OSFreeMem(psHeap->pszQuantizedVMRAName);
+	}
+
+	RA_Delete(psHeap->psSubAllocRA);
+	OSFreeMem(psHeap->pszSubAllocRAName);
+
+	OSFreeMem(psHeap->pszName);
+
+	OSCachedMemSet(psHeap, 0, sizeof(*psHeap));
+	OSFreeMem(psHeap);
+
+	return PVRSRV_OK;
+}
+
+/*****************************************************************************
+ *                Devmem allocation/free functions                           *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemSubAllocateAndMap(IMG_UINT8 uiPreAllocMultiplier,
+			DEVMEM_HEAP *psHeap,
+			IMG_DEVMEM_SIZE_T uiSize,
+			IMG_DEVMEM_ALIGN_T uiAlign,
+			PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			const IMG_CHAR *pszText,
+			DEVMEM_MEMDESC **ppsMemDescPtr,
+			IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemSubAllocate(uiPreAllocMultiplier,
+				psHeap,
+				uiSize,
+				uiAlign,
+				uiFlags,
+				pszText,
+				ppsMemDescPtr);
+	PVR_GOTO_IF_ERROR(eError, fail_alloc);
+
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+				psHeap,
+				psDevVirtAddr);
+	PVR_GOTO_IF_ERROR(eError, fail_map);
+
+	return PVRSRV_OK;
+
+fail_map:
+	DevmemFree(*ppsMemDescPtr);
+fail_alloc:
+	*ppsMemDescPtr = NULL;
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	RA_BASE_T uiAllocatedAddr = 0;
+	RA_LENGTH_T uiAllocatedSize;
+	RA_PERISPAN_HANDLE hImport; /* the "import" from which this sub-allocation came */
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	IMG_DEVMEM_OFFSET_T uiOffset = 0;
+	DEVMEM_IMPORT *psImport;
+	IMG_UINT32 ui32CPUCacheLineSize;
+	void *pvAddr = NULL;
+
+	IMG_BOOL bImportClean;
+	IMG_BOOL bCPUCleanFlag = PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags);
+	IMG_BOOL bZero = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags);
+	IMG_BOOL bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags);
+	IMG_BOOL bCPUCached = (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+			PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags));
+	IMG_BOOL bGPUCached = (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) ||
+			PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags));
+	IMG_BOOL bAlign = ! (PVRSRV_CHECK_NO_CACHE_LINE_ALIGN(uiFlags));
+	PVRSRV_CACHE_OP eOp = PVRSRV_CACHE_OP_INVALIDATE;
+	IMG_UINT32	ui32CacheLineSize = 0;
+	DEVMEM_PROPERTIES_T uiProperties;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+		bPoisonOnAlloc = IMG_TRUE;
+	}
+#endif
+
+	if (uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		/* Deferred Allocation not supported on SubAllocs*/
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failParams);
+	}
+
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap->psCtx, eError, failParams);
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(psHeap->psCtx->hDevConnection, uiFlags);
+
+#if defined(__KERNEL__)
+	{
+		/* The hDevConnection holds two different types of pointers depending on the
+		 * address space in which it is used.
+		 * In this instance the variable points to the device node in server */
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection;
+		ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, SLC_CACHE_LINE_SIZE_BITS));
+	}
+#else
+	ui32CacheLineSize = ROGUE_CACHE_LINE_SIZE;
+#endif
+
+	/* The following logic makes sure that any cached memory is aligned to both the CPU and GPU.
+	 * To be aligned on both you have to take the Lowest Common Multiple (LCM) of the cache line sizes of each.
+	 * As the possibilities are all powers of 2 then simply the largest number can be picked as the LCM.
+	 * Therefore this algorithm just picks the highest from the CPU, GPU and given alignments.
+	 */
+	ui32CPUCacheLineSize = psHeap->psCtx->ui32CPUCacheLineSize;
+	/* If the CPU cache line size is larger than the alignment given then it is the lowest common multiple
+	 * Also checking if the allocation is going to be cached on the CPU
+	 * Currently there is no check for the validity of the cache coherent option.
+	 * In this case, the alignment could be applied but the mode could still fall back to uncached.
+	 */
+	if (bAlign && ui32CPUCacheLineSize > uiAlign && bCPUCached)
+	{
+		uiAlign = ui32CPUCacheLineSize;
+	}
+
+	/* If the GPU cache line size is larger than the alignment given then it is the lowest common multiple
+	 * Also checking if the allocation is going to be cached on the GPU via checking for any of the cached options.
+	 * Currently there is no check for the validity of the cache coherent option.
+	 * In this case, the alignment could be applied but the mode could still fall back to uncached.
+	 */
+	if (bAlign && ui32CacheLineSize > uiAlign && bGPUCached)
+	{
+		uiAlign = ui32CacheLineSize;
+	}
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	/* No request for exportable memory so use the RA */
+	eError = RA_Alloc(psHeap->psSubAllocRA,
+			uiSize,
+			uiPreAllocMultiplier,
+			uiFlags,
+			uiAlign,
+			pszText,
+			&uiAllocatedAddr,
+			&uiAllocatedSize,
+			&hImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	psImport = hImport;
+
+	/* This assignment is assuming the RA returns an hImport where suballocations
+	 * can be made from if uiSize is NOT a page multiple of the passed heap.
+	 *
+	 * So we check if uiSize is a page multiple and mark it as exportable
+	 * if it is not.
+	 * */
+	OSLockAcquire(psImport->hLock);
+	if (!(uiSize & (IMG_PAGE2BYTES64(psHeap->uiLog2Quantum) - 1)) &&
+	    (uiPreAllocMultiplier == RA_NO_IMPORT_MULTIPLIER))
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_EXPORTABLE;
+	}
+	psImport->uiProperties |= DEVMEM_PROPERTIES_SUBALLOCATABLE;
+	uiProperties = psImport->uiProperties;
+	OSLockRelease(psImport->hLock);
+
+	uiOffset = uiAllocatedAddr - psImport->sDeviceImport.sDevVAddr.uiAddr;
+
+#if defined(PDUMP) && defined(DEBUG)
+#if defined(__KERNEL__)
+	PDUMPCOMMENTWITHFLAGS(PMR_DeviceNode((PMR*)psImport->hPMR), PDUMP_CONT,
+			"Suballocated %u Byte for \"%s\" from PMR with handle ID: 0x%p (PID %u)",
+			(IMG_UINT32) uiSize, pszText, psImport->hPMR, OSGetCurrentProcessID());
+#else
+	PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS,
+			"Suballocated %u Byte for \"%s\" from PMR with handle ID: %p (PID %u)",
+			(IMG_UINT32) uiSize,
+			pszText,
+			psImport->hPMR,
+			OSGetCurrentProcessID());
+#endif
+#endif
+
+	DevmemMemDescInit(psMemDesc,
+			uiOffset,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	bImportClean = ((uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_CLEAN) != 0);
+
+	/* Zero the memory */
+	if (bZero)
+	{
+		/* Has the import been zeroed on allocation and were no suballocations returned to it so far? */
+		bImportClean = bImportClean && ((uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_ZEROED) != 0);
+
+		if (!bImportClean)
+		{
+			eOp = PVRSRV_CACHE_OP_FLUSH;
+
+			eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
+			PVR_GOTO_IF_ERROR(eError, failMaintenance);
+
+			/* uiSize is a 64-bit quantity whereas the 3rd argument
+			 * to OSDeviceMemSet is a 32-bit quantity on 32-bit systems
+			 * hence a compiler warning of implicit cast and loss of data.
+			 * Added explicit cast and assert to remove warning.
+			 */
+			PVR_ASSERT(uiSize < IMG_UINT32_MAX);
+
+			DevmemCPUMemSet(pvAddr, 0, uiSize, uiFlags);
+
+#if defined(PDUMP)
+			DevmemPDumpLoadZeroMem(psMemDesc, 0, uiSize, PDUMP_FLAGS_CONTINUOUS);
+#endif
+		}
+	}
+	else if (bPoisonOnAlloc)
+	{
+		/* Has the import been poisoned on allocation and were no suballocations returned to it so far? */
+		bPoisonOnAlloc = (uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_POISONED) != 0;
+
+		if (!bPoisonOnAlloc)
+		{
+			eOp = PVRSRV_CACHE_OP_FLUSH;
+
+			eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
+			PVR_GOTO_IF_ERROR(eError, failMaintenance);
+
+			DevmemCPUMemSet(pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE, uiSize, uiFlags);
+
+			bPoisonOnAlloc = IMG_TRUE;
+		}
+	}
+
+	/* Flush or invalidate */
+	if (bCPUCached && !bImportClean && (bZero || bCPUCleanFlag || bPoisonOnAlloc))
+	{
+		eError = BridgeCacheOpExec (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				(IMG_UINT64)(uintptr_t)
+				pvAddr - psMemDesc->uiOffset,
+				psMemDesc->uiOffset,
+				psMemDesc->uiAllocSize,
+				eOp);
+		PVR_GOTO_IF_ERROR(eError, failMaintenance);
+	}
+
+	if (pvAddr)
+	{
+		DevmemReleaseCpuVirtAddr(psMemDesc);
+		pvAddr = NULL;
+	}
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				OSStringNLength(psMemDesc->szText, DEVMEM_ANNOTATION_MAX_LEN),
+				psMemDesc->szText,
+				psMemDesc->uiOffset,
+				uiAllocatedSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_SUBALLOC,
+				&(psMemDesc->hRIHandle));
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+		}
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	*ppsMemDescPtr = psMemDesc;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failMaintenance:
+	if (pvAddr)
+	{
+		DevmemReleaseCpuVirtAddr(psMemDesc);
+		pvAddr = NULL;
+	}
+	DevmemMemDescRelease(psMemDesc);
+	psMemDesc = NULL;	/* Make sure we don't do a discard after the release */
+failDeviceMemAlloc:
+	if (psMemDesc)
+	{
+		DevmemMemDescDiscard(psMemDesc);
+	}
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		IMG_UINT32 uiLog2HeapPageSize,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_UINT32 ui32MappingTable = 0;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+	}
+#endif
+
+	eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(hDevConnection, uiFlags);
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = AllocateDeviceMemory(hDevConnection,
+			uiLog2HeapPageSize,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			uiAlign,
+			uiFlags,
+			IMG_TRUE,
+			pszText,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	*ppsMemDescPtr = psMemDesc;
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				uiSize,
+				uiFlags,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failDeviceMemAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_UINT32 ui32NumPhysChunks,
+		IMG_UINT32 ui32NumVirtChunks,
+		IMG_UINT32 *pui32MappingTable,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		IMG_UINT32 uiLog2HeapPageSize,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+	}
+#endif
+
+	eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(hDevConnection, uiFlags);
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = AllocateDeviceMemory(hDevConnection,
+			uiLog2HeapPageSize,
+			uiSize,
+			ui32NumPhysChunks,
+			ui32NumVirtChunks,
+			pui32MappingTable,
+			uiAlign,
+			uiFlags,
+			IMG_TRUE,
+			pszText,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				uiSize,
+				uiFlags,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	*ppsMemDescPtr = psMemDesc;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failDeviceMemAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hServerHandle,
+		IMG_HANDLE *hLocalImportHandle)
+{
+	return BridgePMRMakeLocalImportHandle(GetBridgeHandle(hDevConnection),
+			hServerHandle,
+			hLocalImportHandle);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hLocalImportHandle)
+{
+	return DestroyServerResource(hDevConnection,
+	                             NULL,
+	                             BridgePMRUnmakeLocalImportHandle,
+	                             hLocalImportHandle);
+}
+
+/*****************************************************************************
+ *                Devmem unsecure export functions                           *
+ *****************************************************************************/
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static PVRSRV_ERROR
+_Mapping_Export(DEVMEM_IMPORT *psImport,
+		DEVMEM_EXPORTHANDLE *phPMRExportHandlePtr,
+		DEVMEM_EXPORTKEY *puiExportKeyPtr,
+		DEVMEM_SIZE_T *puiSize,
+		DEVMEM_LOG2ALIGN_T *puiLog2Contig)
+{
+	/* Gets an export handle and key for the PMR used for this mapping */
+	/* Can only be done if there are no suballocations for this mapping */
+
+	PVRSRV_ERROR eError;
+	DEVMEM_EXPORTHANDLE hPMRExportHandle;
+	DEVMEM_EXPORTKEY uiExportKey;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig;
+
+	PVR_GOTO_IF_INVALID_PARAM(psImport, eError, failParams);
+
+	if ((GetImportProperties(psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION, failParams);
+	}
+
+	eError = BridgePMRExportPMR(GetBridgeHandle(psImport->hDevConnection),
+			psImport->hPMR,
+			&hPMRExportHandle,
+			&uiSize,
+			&uiLog2Contig,
+			&uiExportKey);
+	PVR_GOTO_IF_ERROR(eError, failExport);
+
+	PVR_ASSERT(uiSize == psImport->uiSize);
+
+	*phPMRExportHandlePtr = hPMRExportHandle;
+	*puiExportKeyPtr = uiExportKey;
+	*puiSize = uiSize;
+	*puiLog2Contig = uiLog2Contig;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failExport:
+failParams:
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static void
+_Mapping_Unexport(DEVMEM_IMPORT *psImport,
+		DEVMEM_EXPORTHANDLE hPMRExportHandle)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT (psImport != NULL);
+
+	eError = DestroyServerResource(psImport->hDevConnection,
+	                               NULL,
+	                               BridgePMRUnexportPMR,
+	                               hPMRExportHandle);
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExport(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_EXPORTCOOKIE *psExportCookie)
+{
+	/* Caller to provide storage for export cookie struct */
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hPMRExportHandle = 0;
+	IMG_UINT64 uiPMRExportPassword = 0;
+	IMG_DEVMEM_SIZE_T uiSize = 0;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig = 0;
+
+	PVR_GOTO_IF_INVALID_PARAM(psMemDesc, eError, e0);
+	PVR_GOTO_IF_INVALID_PARAM(psExportCookie, eError, e0);
+
+	if (DEVMEM_PROPERTIES_EXPORTABLE !=
+			(GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This Memory (0x%p) cannot be exported!...",
+				__func__, psMemDesc));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_REQUEST, e0);
+	}
+
+	eError = _Mapping_Export(psMemDesc->psImport,
+			&hPMRExportHandle,
+			&uiPMRExportPassword,
+			&uiSize,
+			&uiLog2Contig);
+	if (eError != PVRSRV_OK)
+	{
+		psExportCookie->uiSize = 0;
+		goto e0;
+	}
+
+	psExportCookie->hPMRExportHandle = hPMRExportHandle;
+	psExportCookie->uiPMRExportPassword = uiPMRExportPassword;
+	psExportCookie->uiSize = uiSize;
+	psExportCookie->uiLog2ContiguityGuarantee = uiLog2Contig;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_EXPORTCOOKIE *psExportCookie)
+{
+	_Mapping_Unexport(psMemDesc->psImport,
+			psExportCookie->hPMRExportHandle);
+
+	psExportCookie->uiSize = 0;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemImport(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_EXPORTCOOKIE *psCookie,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, failImportAlloc);
+	}
+
+	/* Get a handle to the PMR (inc refcount) */
+	eError = BridgePMRImportPMR(GetBridgeHandle(hDevConnection),
+			psCookie->hPMRExportHandle,
+			psCookie->uiPMRExportPassword,
+			psCookie->uiSize, /* not trusted - just for validation */
+			psCookie->uiLog2ContiguityGuarantee, /* not trusted - just for validation */
+			&hPMR);
+	PVR_GOTO_IF_ERROR(eError, failImport);
+
+	DevmemImportStructInit(psImport,
+			psCookie->uiSize,
+			1ULL << psCookie->uiLog2ContiguityGuarantee,
+			uiFlags,
+			hPMR,
+			DEVMEM_PROPERTIES_IMPORTED |
+			DEVMEM_PROPERTIES_EXPORTABLE);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			psImport->uiSize);
+
+	*ppsMemDescPtr = psMemDesc;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				psMemDesc->psImport->uiSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_IMPORT | PVRSRV_MEMALLOCFLAG_RI_SUBALLOC,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failImport:
+	DevmemImportDiscard(psImport);
+failImportAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+/*****************************************************************************
+ *                   Common MemDesc functions                                *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetSize(DEVMEM_MEMDESC *psMemDesc, IMG_DEVMEM_SIZE_T* puiSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	*puiSize = psMemDesc->uiAllocSize;
+
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemGetAnnotation(DEVMEM_MEMDESC *psMemDesc, IMG_CHAR **pszAnnotation)
+{
+	/*
+	 * It is expected that psMemDesc->szText is a valid NUL-terminated string,
+	 * since DevmemMemDescAlloc uses OSAllocZMem to create the memdesc.
+	 */
+	*pszAnnotation = psMemDesc->szText;
+}
+
+/*
+	This function is called for freeing any class of memory
+ */
+IMG_INTERNAL IMG_BOOL
+DevmemFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_SECURE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Please use methods dedicated to secure buffers.",
+				__func__));
+		return IMG_FALSE;
+	}
+
+	return DevmemMemDescRelease(psMemDesc);
+}
+
+IMG_INTERNAL IMG_BOOL
+DevmemReleaseDevAddrAndFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	DevmemReleaseDevVirtAddr(psMemDesc);
+	return DevmemFree(psMemDesc);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	DEVMEM_IMPORT *psImport;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bMap = IMG_TRUE;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED, failCheck);
+	}
+
+	/* Don't map memory for deferred allocations */
+	if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		PVR_ASSERT(uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
+		bMap = IMG_FALSE;
+	}
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	psImport = psMemDesc->psImport;
+	DevmemMemDescAcquire(psMemDesc);
+
+#if defined(__KERNEL__)
+	if (psHeap->bPremapped)
+	{
+		ui64OptionalMapAddress = _GetPremappedVA(psImport->hPMR, psHeap->psCtx->hDevConnection);
+	}
+#endif
+
+	eError = DevmemImportStructDevMap(psHeap,
+			bMap,
+			psImport,
+			ui64OptionalMapAddress);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	sDevVAddr.uiAddr += psMemDesc->uiOffset;
+	psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
+
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				psMemDesc->uiOffset,
+				psMemDesc->sDeviceMemDesc.sDevVAddr,
+				psMemDesc->uiAllocSize,
+				psMemDesc->szText,
+				DevmemGetHeapLog2PageSize(psHeap),
+				psMemDesc->ui32AllocationIndex,
+				&psMemDesc->ui32AllocationIndex);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		if (psMemDesc->hRIHandle)
+		{
+			eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection),
+					psMemDesc->hRIHandle,
+					psImport->sDeviceImport.sDevVAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIUpdateMEMDESCAddr");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failMap:
+	bDestroyed = DevmemMemDescRelease(psMemDesc);
+failCheck:
+failParams:
+	if (!bDestroyed)
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEV_VIRTADDR sDevVirtAddr)
+{
+	DEVMEM_IMPORT *psImport;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bMap = IMG_TRUE;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED, failCheck);
+	}
+
+	/* Don't map memory for deferred allocations */
+	if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		PVR_ASSERT(uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
+		bMap = IMG_FALSE;
+	}
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	psImport = psMemDesc->psImport;
+	DevmemMemDescAcquire(psMemDesc);
+
+	eError = DevmemImportStructDevMap(psHeap,
+			bMap,
+			psImport,
+			sDevVirtAddr.uiAddr);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	sDevVAddr.uiAddr += psMemDesc->uiOffset;
+	psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				psMemDesc->uiOffset,
+				psMemDesc->sDeviceMemDesc.sDevVAddr,
+				psMemDesc->uiAllocSize,
+				psMemDesc->szText,
+				DevmemGetHeapLog2PageSize(psHeap),
+				psMemDesc->ui32AllocationIndex,
+				&psMemDesc->ui32AllocationIndex);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		if (psMemDesc->hRIHandle)
+		{
+			eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection),
+					psMemDesc->hRIHandle,
+					psImport->sDeviceImport.sDevVAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIUpdateMEMDESCAddr");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failMap:
+	bDestroyed = DevmemMemDescRelease(psMemDesc);
+failCheck:
+failParams:
+	if (!bDestroyed)
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL IMG_DEV_VIRTADDR
+DevmemGetDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_DEVICEMEM_NO_MAPPING, "DevmemGetDevVirtAddr");
+	}
+
+	PVR_ASSERT(psMemDesc->sDeviceMemDesc.sDevVAddr.uiAddr !=0 );
+
+	return psMemDesc->sDeviceMemDesc.sDevVAddr;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_NO_MAPPING, failRelease);
+	}
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	return PVRSRV_OK;
+
+failRelease:
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount-1);
+
+	PVR_ASSERT(psMemDesc->sDeviceMemDesc.ui32RefCount != 0);
+
+	if (--psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+		{
+			BridgeDevicememHistoryUnmap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+					psMemDesc->psImport->hPMR,
+					psMemDesc->uiOffset,
+					psMemDesc->sDeviceMemDesc.sDevVAddr,
+					psMemDesc->uiAllocSize,
+					psMemDesc->szText,
+					DevmemGetHeapLog2PageSize(psMemDesc->psImport->sDeviceImport.psHeap),
+					psMemDesc->ui32AllocationIndex,
+					&psMemDesc->ui32AllocationIndex);
+		}
+
+		/* When device mapping destroyed, zero Dev VA so DevmemGetDevVirtAddr()
+		 * returns 0 */
+		if (DevmemImportStructDevUnmap(psMemDesc->psImport) == IMG_TRUE)
+		{
+			psMemDesc->sDeviceMemDesc.sDevVAddr.uiAddr = 0;
+		}
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+		DevmemMemDescRelease(psMemDesc);
+	}
+	else
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		void **ppvCpuVirtAddr)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(ppvCpuVirtAddr != NULL);
+
+	eError = DevmemCPUMapCheckImportProperties(psMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemCPUMapCheckImportProperties");
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount+1);
+
+	if (psMemDesc->sCPUMemDesc.ui32RefCount++ == 0)
+	{
+		DEVMEM_IMPORT *psImport = psMemDesc->psImport;
+		IMG_UINT8 *pui8CPUVAddr;
+
+		DevmemMemDescAcquire(psMemDesc);
+		eError = DevmemImportStructCPUMap(psImport);
+		PVR_GOTO_IF_ERROR(eError, failMap);
+
+		pui8CPUVAddr = psImport->sCPUImport.pvCPUVAddr;
+		pui8CPUVAddr += psMemDesc->uiOffset;
+		psMemDesc->sCPUMemDesc.pvCPUVAddr = pui8CPUVAddr;
+	}
+	*ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
+
+	VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
+
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+
+	return PVRSRV_OK;
+
+failMap:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	psMemDesc->sCPUMemDesc.ui32RefCount--;
+
+	if (!DevmemMemDescRelease(psMemDesc))
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	}
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemReacquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		void **ppvCpuVirtAddr)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(ppvCpuVirtAddr != NULL);
+
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU UnMapping is not possible on this allocation!",
+				__func__));
+		return;
+	}
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount+1);
+
+	*ppvCpuVirtAddr = NULL;
+	if (psMemDesc->sCPUMemDesc.ui32RefCount)
+	{
+		*ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
+		psMemDesc->sCPUMemDesc.ui32RefCount += 1;
+	}
+
+	VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+}
+
+IMG_INTERNAL void
+DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU UnMapping is not possible on this allocation!",
+				__func__));
+		return;
+	}
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount-1);
+
+	PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount != 0);
+
+	if (--psMemDesc->sCPUMemDesc.ui32RefCount == 0)
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+		DevmemImportStructCPUUnmap(psMemDesc->psImport);
+		DevmemMemDescRelease(psMemDesc);
+	}
+	else
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	}
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *phImport)
+{
+	if ((GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION;
+	}
+
+	*phImport = psMemDesc->psImport->hPMR;
+
+	return PVRSRV_OK;
+}
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
+		IMG_UINT64 *pui64UID)
+{
+	DEVMEM_IMPORT *psImport = psMemDesc->psImport;
+	PVRSRV_ERROR eError;
+
+	if (!(GetImportProperties(psImport) & (DEVMEM_PROPERTIES_IMPORTED |
+		                        DEVMEM_PROPERTIES_EXPORTABLE)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This Memory (0x%p) doesn't support the functionality requested...",
+				__func__, psMemDesc));
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+
+	eError = BridgePMRGetUID(GetBridgeHandle(psImport->hDevConnection),
+			psImport->hPMR,
+			pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *hReservation)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*hReservation = psImport->sDeviceImport.hReservation;
+
+	return PVRSRV_OK;
+}
+
+#endif /* !__KERNEL__ */
+
+/* Kernel usage of this function will only work with
+ * memdescs of buffers allocated in the FW memory context
+ * that is created in the Server
+ */
+void
+DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *phPMR,
+		IMG_DEVMEM_OFFSET_T *puiPMROffset)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	*puiPMROffset = psMemDesc->uiOffset;
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*phPMR = psImport->hPMR;
+}
+
+#if defined(__KERNEL__)
+IMG_INTERNAL void
+DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
+		PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*puiFlags = psImport->uiFlags;
+}
+
+IMG_INTERNAL SHARED_DEV_CONNECTION
+DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc)
+{
+	return psMemDesc->psImport->hDevConnection;
+}
+#endif /* __KERNEL__ */
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemLocalImport(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hExtHandle,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		DEVMEM_MEMDESC **ppsMemDescPtr,
+		IMG_DEVMEM_SIZE_T *puiSizePtr,
+		const IMG_CHAR *pszAnnotation)
+{
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, failImportAlloc);
+	}
+
+	/* Get the PMR handle and its size from the server */
+	eError = BridgePMRLocalImportPMR(GetBridgeHandle(hDevConnection),
+			hExtHandle,
+			&hPMR,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failImport);
+
+	DevmemImportStructInit(psImport,
+			uiSize,
+			uiAlign,
+			uiFlags,
+			hPMR,
+			DEVMEM_PROPERTIES_IMPORTED |
+			DEVMEM_PROPERTIES_EXPORTABLE);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+	*ppsMemDescPtr = psMemDesc;
+	if (puiSizePtr)
+		*puiSizePtr = uiSize;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information.
+		 * Set backed size to 0 since this allocation has been allocated
+		 * by the same process and has been accounted for. */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				psMemDesc->psImport->uiSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_IMPORT,
+				&(psMemDesc->hRIHandle));
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+
+	/* Copy the allocation descriptive name and size so it can be passed
+	 * to DevicememHistory when the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszAnnotation);
+	OSStringSafeCopy(psMemDesc->szText, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN);
+
+	return PVRSRV_OK;
+
+failImport:
+	DevmemImportDiscard(psImport);
+failImportAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
+		IMG_DEV_VIRTADDR sDevVAddr)
+{
+	return BridgeDevmemIsVDevAddrValid(GetBridgeHandle(psContext->hDevConnection),
+	                                   psContext->hDevMemServerContext,
+	                                   sDevVAddr);
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetFaultAddress(DEVMEM_CONTEXT *psContext,
+		IMG_DEV_VIRTADDR *psFaultAddress)
+{
+	return BridgeDevmemGetFaultAddress(GetBridgeHandle(psContext->hDevConnection),
+	                                   psContext->hDevMemServerContext,
+	                                   psFaultAddress);
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemInvalidateFBSCTable(DEVMEM_CONTEXT *psContext,
+                          IMG_UINT64 ui64FBSCEntries)
+{
+	return BridgeDevmemInvalidateFBSCTable(GetBridgeHandle(psContext->hDevConnection),
+	                                       psContext->hDevMemServerContext,
+	                                       ui64FBSCEntries);
+}
+#endif
+
+#endif /* !__KERNEL__ */
+
+IMG_INTERNAL IMG_UINT32
+DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap)
+{
+	return psHeap->uiLog2Quantum;
+}
+
+IMG_INTERNAL PVRSRV_MEMALLOCFLAGS_T
+DevmemGetMemAllocFlags(DEVMEM_MEMDESC *psMemDesc)
+{
+	return psMemDesc->psImport->uiFlags;
+}
+
+IMG_INTERNAL IMG_DEVMEM_SIZE_T
+DevmemGetHeapReservedSize(DEVMEM_HEAP *psHeap)
+{
+	return psHeap->uiReservedRegionSize;
+}
+
+#if !defined(__KERNEL__)
+/**************************************************************************/ /*!
+@Function       RegisterDevMemPFNotify
+@Description    Registers that the application wants to be signaled when a page
+                fault occurs.
+
+@Input          psContext      Memory context the process that would like to
+                               be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR:  PVRSRV_OK on success. Otherwise, a PVRSRV_
+                               error code
+ */ /***************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext,
+		IMG_BOOL       bRegister)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgeDevmemIntRegisterPFNotifyKM(GetBridgeHandle(psContext->hDevConnection),
+			psContext->hDevMemServerContext,
+			bRegister);
+	if (eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
+	{
+		PVR_LOG_ERROR(eError, "BridgeDevmemIntRegisterPFNotifyKM");
+	}
+
+	return eError;
+}
+#endif /* !__KERNEL__ */
+
+IMG_INTERNAL void
+DevmemHeapSetPremapStatus(DEVMEM_HEAP *psHeap, IMG_BOOL IsPremapped)
+{
+	psHeap->bPremapped = IsPremapped;
+}
+
+/******************************************************************************
+ End of file (devicemem.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_pdump.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_pdump.c
new file mode 100644
index 000000000000..c86441217b2a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_pdump.c
@@ -0,0 +1,364 @@
+/*************************************************************************/ /*!
+@File
+@Title          Shared device memory management PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common (client & server) PDump functions for the
+                memory management code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#if defined(PDUMP)
+
+#include "allocmem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pdump.h"
+#include "devicemem.h"
+#include "devicemem_utils.h"
+#include "devicemem_pdump.h"
+#include "client_pdump_bridge.h"
+#include "client_pdumpmm_bridge.h"
+#if defined(__linux__) && !defined(__KERNEL__)
+#include <stdio.h>
+#endif
+
+IMG_INTERNAL void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiOffset + uiSize <= psMemDesc->psImport->uiSize);
+
+	eError = BridgePMRPDumpLoadMem(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                               psMemDesc->psImport->hPMR,
+	                               psMemDesc->uiOffset + uiOffset,
+	                               uiSize,
+	                               uiPDumpFlags,
+	                               IMG_FALSE);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMem");
+}
+
+IMG_INTERNAL void
+DevmemPDumpLoadZeroMem(DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiOffset + uiSize <= psMemDesc->psImport->uiSize);
+
+	eError = BridgePMRPDumpLoadMem(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                               psMemDesc->psImport->hPMR,
+	                               psMemDesc->uiOffset + uiOffset,
+	                               uiSize,
+	                               uiPDumpFlags,
+	                               IMG_TRUE);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMem");
+}
+
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgePMRPDumpLoadMemValue32(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+                                        psMemDesc->psImport->hPMR,
+                                        psMemDesc->uiOffset + uiOffset,
+                                        ui32Value,
+                                        uiPDumpFlags);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMemValue32");
+}
+
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgePMRPDumpLoadMemValue64(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                      psMemDesc->psImport->hPMR,
+	                                      psMemDesc->uiOffset + uiOffset,
+	                                      ui64Value,
+	                                      uiPDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMemValue64");
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC		*psMemDesc,
+                              IMG_DEVMEM_OFFSET_T	*puiMemOffset,
+                              IMG_CHAR				*pszName,
+                              IMG_UINT32			ui32Size)
+{
+	PVRSRV_ERROR		eError;
+	IMG_CHAR			aszMemspaceName[100];
+	IMG_CHAR			aszSymbolicName[100];
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+
+	*puiMemOffset += psMemDesc->uiOffset;
+
+	eError = BridgePMRPDumpSymbolicAddr(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                    psMemDesc->psImport->hPMR,
+	                                    *puiMemOffset,
+	                                    sizeof(aszMemspaceName),
+	                                    &aszMemspaceName[0],
+	                                    sizeof(aszSymbolicName),
+	                                    &aszSymbolicName[0],
+	                                    puiMemOffset,
+	                                    &uiNextSymName);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpSymbolicAddr");
+
+	OSSNPrintf(pszName, ui32Size, "%s:%s", &aszMemspaceName[0], &aszSymbolicName[0]);
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgePMRPDumpSaveToFile(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                  psMemDesc->psImport->hPMR,
+	                                  psMemDesc->uiOffset + uiOffset,
+	                                  uiSize,
+	                                  OSStringLength(pszFilename) + 1,
+	                                  pszFilename,
+	                                  uiFileOffset);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpSaveToFile");
+}
+
+IMG_INTERNAL void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevAddrStart;
+
+	sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset;
+	sDevAddrStart.uiAddr += uiOffset;
+
+	eError = BridgeDevmemIntPDumpSaveToFileVirtual(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                               psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+	                                               sDevAddrStart,
+	                                               uiSize,
+	                                               OSStringLength(pszFilename) + 1,
+	                                               pszFilename,
+	                                               ui32FileOffset,
+	                                               ui32PdumpFlags);
+
+	PVR_LOG_IF_ERROR(eError, "BridgeDevmemIntPDumpSaveToFileVirtual");
+}
+
+IMG_INTERNAL void
+DevmemPDumpDataDescriptor(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_DEVMEM_SIZE_T uiSize,
+                          const IMG_CHAR *pszFilename,
+                          IMG_UINT32 ui32HeaderType,
+                          IMG_UINT32 ui32ElementType,
+                          IMG_UINT32 ui32ElementCount,
+                          IMG_UINT32 ui32PdumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevAddrStart;
+
+	sDevAddrStart = psMemDesc->psImport->sDeviceImport.sDevVAddr;
+	sDevAddrStart.uiAddr += psMemDesc->uiOffset;
+	sDevAddrStart.uiAddr += uiOffset;
+
+	eError = BridgePDumpDataDescriptor(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                                   psMemDesc->psImport->sDeviceImport.psHeap->psCtx->hDevMemServerContext,
+                                       OSStringLength(pszFilename) + 1,
+	                                   pszFilename,
+	                                   sDevAddrStart,
+	                                   uiSize,
+	                                   ui32HeaderType,
+	                                   ui32ElementType,
+	                                   ui32ElementCount,
+	                                   ui32PdumpFlags);
+
+	PVR_LOG_IF_ERROR(eError, "BridgePDumpDataDescriptor");
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEVMEM_SIZE_T uiNumBytes;
+
+	uiNumBytes = 4;
+
+	if (psMemDesc->uiOffset + uiOffset + uiNumBytes > psMemDesc->psImport->uiSize)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+	}
+
+	eError = BridgePMRPDumpPol32(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                             psMemDesc->psImport->hPMR,
+	                             psMemDesc->uiOffset + uiOffset,
+	                             ui32Value,
+	                             ui32Mask,
+	                             eOperator,
+	                             ui32PDumpFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow
+	 */
+
+e0:
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError != PVRSRV_OK);
+	}
+	return eError;
+}
+
+#if defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpDevmemCheck32(const DEVMEM_MEMDESC *psMemDesc,
+                         IMG_DEVMEM_OFFSET_T uiOffset,
+                         IMG_UINT32 ui32Value,
+                         IMG_UINT32 ui32Mask,
+                         PDUMP_POLL_OPERATOR eOperator,
+                         PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEVMEM_SIZE_T uiNumBytes;
+
+	uiNumBytes = 4;
+
+	if (psMemDesc->uiOffset + uiOffset + uiNumBytes >= psMemDesc->psImport->uiSize)
+	{
+		eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+		goto e0;
+	}
+
+	eError = BridgePMRPDumpCheck32(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                             psMemDesc->psImport->hPMR,
+	                             psMemDesc->uiOffset + uiOffset,
+	                             ui32Value,
+	                             ui32Mask,
+	                             eOperator,
+	                             ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow
+	 */
+
+e0:
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError != PVRSRV_OK);
+	}
+	return eError;
+}
+#endif /* defined(__KERNEL__) */
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+
+	if ((psMemDesc->uiOffset + uiReadOffset) > psMemDesc->psImport->uiSize)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+	}
+
+	eError = BridgePMRPDumpCBP(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+	                           psMemDesc->psImport->hPMR,
+	                           psMemDesc->uiOffset + uiReadOffset,
+	                           uiWriteOffset,
+	                           uiPacketSize,
+	                           uiBufferSize);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+e0:
+	/* If PDump was rejected for this device, suppress silently */
+	if (eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+	{
+		PVR_ASSERT(eError != PVRSRV_OK);
+	}
+	return eError;
+}
+
+#endif /* PDUMP */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_utils.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_utils.c
new file mode 100644
index 000000000000..52bb9c838b52
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicemem_utils.c
@@ -0,0 +1,1245 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management internal utility functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally by device memory management
+                code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "allocmem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "ra.h"
+#include "devicemem_utils.h"
+#include "client_mm_bridge.h"
+#include "client_cache_bridge.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "client_ri_bridge.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#else
+#include "pvr_bridge_client.h"
+#endif
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "proc_stats.h"
+#endif
+
+#if defined(__KERNEL__)
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+/*
+	SVM heap management support functions for CPU (un)mapping
+ */
+#define DEVMEM_MAP_SVM_USER_MANAGED_RETRY				2
+
+static inline PVRSRV_ERROR
+DevmemCPUMapSVMKernelManaged(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT64 *ui64MapAddress)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64SvmMapAddr;
+	IMG_UINT64 ui64SvmMapAddrEnd;
+	IMG_UINT64 ui64SvmHeapAddrEnd;
+
+	/* SVM heap management always has XXX_MANAGER_KERNEL unless we
+	   have triggered the fall back code-path in which case we
+	   should not be calling into this code-path */
+	PVR_ASSERT(psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_KERNEL);
+
+	/* By acquiring the CPU virtual address here, it essentially
+	   means we lock-down the virtual address for the duration
+	   of the life-cycle of the allocation until a de-allocation
+	   request comes in. Thus the allocation is guaranteed not to
+	   change its virtual address on the CPU during its life-time.
+	   NOTE: Import might have already been CPU Mapped before now,
+	   normally this is not a problem, see fall back */
+	eError = DevmemImportStructCPUMap(psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "DevmemImportStructCPUMap");
+		eError = PVRSRV_ERROR_DEVICEMEM_MAP_FAILED;
+		goto failSVM;
+	}
+
+	/* Supplied kernel mmap virtual address is also device virtual address;
+	   calculate the heap & kernel supplied mmap virtual address limits */
+	ui64SvmMapAddr = (IMG_UINT64)(uintptr_t)psImport->sCPUImport.pvCPUVAddr;
+	ui64SvmHeapAddrEnd = psHeap->sBaseAddress.uiAddr + psHeap->uiSize;
+	ui64SvmMapAddrEnd = ui64SvmMapAddr + psImport->uiSize;
+	PVR_ASSERT(ui64SvmMapAddr != (IMG_UINT64)0);
+
+	/* SVM limit test may fail if processor has more virtual address bits than device */
+	if ((ui64SvmMapAddr >= ui64SvmHeapAddrEnd || ui64SvmMapAddrEnd > ui64SvmHeapAddrEnd) ||
+		(ui64SvmMapAddr & ~(ui64SvmHeapAddrEnd - 1)))
+	{
+		/* Unmap incompatible SVM virtual address, this
+		   may not release address if it was elsewhere
+		   CPU Mapped before call into this function */
+		DevmemImportStructCPUUnmap(psImport);
+
+		/* Flag incompatible SVM mapping */
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto failSVM;
+	}
+
+	*ui64MapAddress = ui64SvmMapAddr;
+failSVM:
+	/* either OK, MAP_FAILED or BAD_MAPPING */
+	return eError;
+}
+
+static inline void
+DevmemCPUUnmapSVMKernelManaged(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	PVR_UNREFERENCED_PARAMETER(psHeap);
+	DevmemImportStructCPUUnmap(psImport);
+}
+
+static inline PVRSRV_ERROR
+DevmemCPUMapSVMUserManaged(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT uiAlign,
+		IMG_UINT64 *ui64MapAddress)
+{
+	RA_LENGTH_T uiAllocatedSize;
+	RA_BASE_T uiAllocatedAddr;
+	IMG_UINT64 ui64SvmMapAddr;
+	IMG_UINT uiRetry = 0;
+	PVRSRV_ERROR eError;
+
+	/* If SVM heap management has transitioned to XXX_MANAGER_USER,
+	   this is essentially a fall back approach that ensures we
+	   continue to satisfy SVM alloc. This approach is not without
+	   hazards in that we may specify a virtual address that is
+	   already in use by the user process */
+	PVR_ASSERT(psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER);
+
+	/* Normally, for SVM heap allocations, CPUMap _must_ be done
+	   before DevMap; ideally the initial CPUMap should be done by
+	   SVM functions though this is not a hard requirement as long
+	   as the prior elsewhere obtained CPUMap virtual address meets
+	   SVM address requirements. This is a fall-back code-pathway
+	   so we have to test that this assumption holds before we
+	   progress any further */
+	OSLockAcquire(psImport->sCPUImport.hLock);
+
+	if (psImport->sCPUImport.ui32RefCount)
+	{
+		/* Already CPU Mapped SVM heap allocation, this prior elsewhere
+		   obtained virtual address is responsible for the above
+		   XXX_MANAGER_KERNEL failure. As we are not responsible for
+		   this, we cannot progress any further so need to fail */
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Previously obtained CPU map address not SVM compatible"
+				, __func__));
+
+		/* Revert SVM heap to DEVMEM_HEAP_MANAGER_KERNEL */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_KERNEL;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"%s: Reverting SVM heap back to kernel managed",
+				__func__));
+
+		OSLockRelease(psImport->sCPUImport.hLock);
+
+		/* Do we need a more specific error code here */
+		eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED;
+		goto failSVM;
+	}
+
+	OSLockRelease(psImport->sCPUImport.hLock);
+
+	do
+	{
+		/* Next we proceed to instruct the kernel to use the RA_Alloc supplied
+		   virtual address to map-in this SVM import suballocation; there is no
+		   guarantee that this RA_Alloc virtual address may not collide with an
+		   already in-use VMA range in the process */
+		eError = RA_Alloc(psHeap->psQuantizedVMRA,
+				psImport->uiSize,
+				RA_NO_IMPORT_MULTIPLIER,
+				0, /* flags: this RA doesn't use flags*/
+				uiAlign,
+				"SVM_Virtual_Alloc",
+				&uiAllocatedAddr,
+				&uiAllocatedSize,
+				NULL /* don't care about per-import priv data */);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "RA_Alloc");
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			if (eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+			{
+				PVRSRV_ERROR eErr;
+				eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+								  PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT,
+								  OSGetCurrentProcessID());
+				PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+			}
+#endif
+			goto failSVM;
+		}
+
+		/* No reason for allocated virtual size to be different from
+		   the PMR's size */
+		psImport->sCPUImport.pvCPUVAddr = (void*)(uintptr_t)uiAllocatedAddr;
+		PVR_ASSERT(uiAllocatedSize == psImport->uiSize);
+
+		/* Map the import or allocation using the RA_Alloc virtual address;
+		   the kernel may fail the request if the supplied virtual address
+		   is already in-use in which case we re-try using another virtual
+		   address obtained from the RA_Alloc */
+		eError = DevmemImportStructCPUMap(psImport);
+		if (eError != PVRSRV_OK)
+		{
+			/* For now we simply discard failed RA_Alloc() obtained virtual
+			   address (i.e. plenty of virtual space), this prevents us from
+			   re-using these and furthermore essentially blacklists these
+			   addresses from future SVM consideration; We exit fall-back
+			   attempt if retry exceeds the fall-back retry limit */
+			if (uiRetry++ > DEVMEM_MAP_SVM_USER_MANAGED_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Cannot find SVM compatible address, bad mapping",
+						__func__));
+				eError = PVRSRV_ERROR_BAD_MAPPING;
+				goto failSVM;
+			}
+		}
+		else
+		{
+			/* Found compatible SVM virtual address, set as device virtual address */
+			ui64SvmMapAddr = (IMG_UINT64)(uintptr_t)psImport->sCPUImport.pvCPUVAddr;
+		}
+	} while (eError != PVRSRV_OK);
+
+	*ui64MapAddress = ui64SvmMapAddr;
+failSVM:
+	return eError;
+}
+
+static inline void
+DevmemCPUUnmapSVMUserManaged(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	RA_BASE_T uiAllocatedAddr;
+
+	/* We only free SVM compatible addresses, all addresses in
+	   the blacklist are essentially excluded from future RA_Alloc */
+	uiAllocatedAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	RA_Free(psHeap->psQuantizedVMRA, uiAllocatedAddr);
+
+	DevmemImportStructCPUUnmap(psImport);
+}
+
+static inline PVRSRV_ERROR
+DevmemImportStructDevMapSVM(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT uiAlign,
+		IMG_UINT64 *ui64MapAddress)
+{
+	PVRSRV_ERROR eError;
+
+	switch (psHeap->ui32HeapManagerFlags)
+	{
+	case DEVMEM_HEAP_MANAGER_KERNEL:
+		eError = DevmemCPUMapSVMKernelManaged(psHeap,
+				psImport,
+				ui64MapAddress);
+		if (eError == PVRSRV_ERROR_BAD_MAPPING)
+		{
+			/* If the SVM map address is outside of SVM heap limits,
+				   change heap type to DEVMEM_HEAP_MANAGER_USER */
+			psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_USER;
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Kernel managed SVM heap is now user managed",
+					__func__));
+
+			/* Retry using user managed fall-back approach */
+			eError = DevmemCPUMapSVMUserManaged(psHeap,
+					psImport,
+					uiAlign,
+					ui64MapAddress);
+		}
+		break;
+
+	case DEVMEM_HEAP_MANAGER_USER:
+		eError = DevmemCPUMapSVMUserManaged(psHeap,
+				psImport,
+				uiAlign,
+				ui64MapAddress);
+		break;
+
+	default:
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		break;
+	}
+
+	return eError;
+}
+
+static inline void
+DevmemImportStructDevUnmapSVM(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	switch (psHeap->ui32HeapManagerFlags)
+	{
+	case DEVMEM_HEAP_MANAGER_KERNEL:
+		DevmemCPUUnmapSVMKernelManaged(psHeap, psImport);
+		break;
+
+	case DEVMEM_HEAP_MANAGER_USER:
+		DevmemCPUUnmapSVMUserManaged(psHeap, psImport);
+		break;
+
+	default:
+		break;
+	}
+}
+
+/*
+	The Devmem import structure is the structure we use
+	to manage memory that is "imported" (which is page
+	granular) from the server into our process, this
+	includes allocations.
+
+	This allows memory to be imported without requiring
+	any CPU or device mapping. Memory can then be mapped
+	into the device or CPU on demand, but neither is
+	required.
+ */
+
+IMG_INTERNAL
+void DevmemImportStructAcquire(DEVMEM_IMPORT *psImport)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psImport->hRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+	PVR_ASSERT(iRefCount != 1);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			iRefCount-1,
+			iRefCount);
+}
+
+IMG_INTERNAL
+IMG_BOOL DevmemImportStructRelease(DEVMEM_IMPORT *psImport)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psImport->hRefCount);
+	PVR_ASSERT(iRefCount >= 0);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			iRefCount+1,
+			iRefCount);
+
+	if (iRefCount == 0)
+	{
+		PVRSRV_ERROR eError = DestroyServerResource(psImport->hDevConnection,
+		                                            NULL,
+		                                            BridgePMRUnrefPMR,
+		                                            psImport->hPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		OSLockDestroy(psImport->sCPUImport.hLock);
+		OSLockDestroy(psImport->sDeviceImport.hLock);
+		OSLockDestroy(psImport->hLock);
+		OSFreeMem(psImport);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_INTERNAL
+void DevmemImportDiscard(DEVMEM_IMPORT *psImport)
+{
+	PVR_ASSERT(OSAtomicRead(&psImport->hRefCount) == 0);
+	OSLockDestroy(psImport->sCPUImport.hLock);
+	OSLockDestroy(psImport->sDeviceImport.hLock);
+	OSLockDestroy(psImport->hLock);
+	OSFreeMem(psImport);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR DevmemMemDescAlloc(DEVMEM_MEMDESC **ppsMemDesc)
+{
+	DEVMEM_MEMDESC *psMemDesc;
+	PVRSRV_ERROR eError;
+
+	/* Must be zeroed in case it needs to be freed before it is initialised */
+	psMemDesc = OSAllocZMem(sizeof(DEVMEM_MEMDESC));
+	PVR_GOTO_IF_NOMEM(psMemDesc, eError, failAlloc);
+
+	eError = OSLockCreate(&psMemDesc->hLock);
+	PVR_GOTO_IF_ERROR(eError, failMDLock);
+
+	eError = OSLockCreate(&psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_ERROR(eError, failDMDLock);
+
+	eError = OSLockCreate(&psMemDesc->sCPUMemDesc.hLock);
+	PVR_GOTO_IF_ERROR(eError, failCMDLock);
+
+	OSAtomicWrite(&psMemDesc->hRefCount, 0);
+
+	*ppsMemDesc = psMemDesc;
+
+	return PVRSRV_OK;
+
+failCMDLock:
+	OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+failDMDLock:
+	OSLockDestroy(psMemDesc->hLock);
+failMDLock:
+	OSFreeMem(psMemDesc);
+failAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	Init the MemDesc structure
+ */
+IMG_INTERNAL
+void DevmemMemDescInit(DEVMEM_MEMDESC *psMemDesc,
+		IMG_DEVMEM_OFFSET_T uiOffset,
+		DEVMEM_IMPORT *psImport,
+		IMG_DEVMEM_SIZE_T uiSize)
+{
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			0,
+			1);
+
+	psMemDesc->psImport = psImport;
+	psMemDesc->uiOffset = uiOffset;
+
+	psMemDesc->sDeviceMemDesc.ui32RefCount = 0;
+	psMemDesc->sCPUMemDesc.ui32RefCount = 0;
+	psMemDesc->uiAllocSize = uiSize;
+	psMemDesc->hPrivData = NULL;
+	psMemDesc->ui32AllocationIndex = DEVICEMEM_HISTORY_ALLOC_INDEX_NONE;
+
+#if defined(DEBUG)
+	psMemDesc->bPoisonOnFree = IMG_FALSE;
+#endif
+
+	OSAtomicWrite(&psMemDesc->hRefCount, 1);
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+void DevmemMemDescSetPoF(DEVMEM_MEMDESC *psMemDesc, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		psMemDesc->bPoisonOnFree = IMG_TRUE;
+	}
+}
+#endif
+
+IMG_INTERNAL
+void DevmemMemDescAcquire(DEVMEM_MEMDESC *psMemDesc)
+{
+	IMG_INT iRefCount = 0;
+
+	iRefCount = OSAtomicIncrement(&psMemDesc->hRefCount);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			iRefCount-1,
+			iRefCount);
+
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+#if defined(DEBUG)
+static void _DevmemPoisonOnFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	void *pvAddr = NULL;
+	IMG_UINT8 *pui8CPUVAddr;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemCPUMapCheckImportProperties(psMemDesc);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "DevmemCPUMapCheckImportProperties");
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	eError = DevmemImportStructCPUMap(psMemDesc->psImport);
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "DevmemImportStructCPUMap");
+
+	pui8CPUVAddr = psMemDesc->psImport->sCPUImport.pvCPUVAddr;
+	pui8CPUVAddr += psMemDesc->uiOffset;
+	pvAddr = pui8CPUVAddr;
+
+	DevmemCPUMemSet(pvAddr,
+	                PVRSRV_POISON_ON_FREE_VALUE,
+	                psMemDesc->uiAllocSize,
+	                psMemDesc->psImport->uiFlags);
+
+	if (PVRSRV_CHECK_CPU_CACHE_COHERENT(psMemDesc->psImport->uiFlags) ||
+	    PVRSRV_CHECK_CPU_CACHE_INCOHERENT(psMemDesc->psImport->uiFlags))
+	{
+		eError = BridgeCacheOpExec(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+								   psMemDesc->psImport->hPMR,
+								   (IMG_UINT64) (uintptr_t)
+								   pvAddr - psMemDesc->uiOffset,
+								   psMemDesc->uiOffset,
+								   psMemDesc->uiAllocSize,
+								   PVRSRV_CACHE_OP_FLUSH);
+		PVR_LOG_IF_ERROR(eError, "BridgeCacheOpExec");
+	}
+
+	DevmemImportStructCPUUnmap(psMemDesc->psImport);
+	pvAddr = NULL;
+}
+#endif
+
+IMG_INTERNAL
+IMG_BOOL DevmemMemDescRelease(DEVMEM_MEMDESC *psMemDesc)
+{
+	IMG_INT iRefCount;
+	PVR_ASSERT(psMemDesc != NULL);
+
+	iRefCount = OSAtomicDecrement(&psMemDesc->hRefCount);
+	PVR_ASSERT(iRefCount >= 0);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			iRefCount+1,
+			iRefCount);
+
+	if (iRefCount == 0)
+	{
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI) &&
+		    (psMemDesc->hRIHandle))
+		{
+			PVRSRV_ERROR eError;
+
+			eError = DestroyServerResource(psMemDesc->psImport->hDevConnection,
+			                               NULL,
+			                               BridgeRIDeleteMEMDESCEntry,
+			                               psMemDesc->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIDeleteMEMDESCEntry");
+		}
+#endif
+
+		OSLockAcquire(psMemDesc->psImport->hLock);
+		if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_SUBALLOCATABLE)
+		{
+			/* As soon as the first sub-allocation on the psImport is freed
+			 * we might get dirty memory when reusing it.
+			 * We have to delete the ZEROED, CLEAN & POISONED flag */
+			psMemDesc->psImport->uiProperties &=
+					~(DEVMEM_PROPERTIES_IMPORT_IS_ZEROED |
+							DEVMEM_PROPERTIES_IMPORT_IS_CLEAN |
+							DEVMEM_PROPERTIES_IMPORT_IS_POISONED);
+
+			OSLockRelease(psMemDesc->psImport->hLock);
+
+#if defined(DEBUG)
+			if (psMemDesc->bPoisonOnFree)
+			{
+				_DevmemPoisonOnFree(psMemDesc);
+			}
+#endif
+
+			RA_Free(psMemDesc->psImport->sDeviceImport.psHeap->psSubAllocRA,
+					psMemDesc->psImport->sDeviceImport.sDevVAddr.uiAddr +
+					psMemDesc->uiOffset);
+		}
+		else
+		{
+			OSLockRelease(psMemDesc->psImport->hLock);
+			DevmemImportStructRelease(psMemDesc->psImport);
+		}
+
+		OSLockDestroy(psMemDesc->sCPUMemDesc.hLock);
+		OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+		OSLockDestroy(psMemDesc->hLock);
+		OSFreeMem(psMemDesc);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_INTERNAL
+void DevmemMemDescDiscard(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(OSAtomicRead(&psMemDesc->hRefCount) == 0);
+
+	OSLockDestroy(psMemDesc->sCPUMemDesc.hLock);
+	OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+	OSLockDestroy(psMemDesc->hLock);
+	OSFreeMem(psMemDesc);
+}
+
+
+IMG_INTERNAL
+PVRSRV_ERROR DevmemValidateParams(IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &&
+			(*puiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Zero on Alloc and Poison on Alloc are mutually exclusive.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiAlign & (uiAlign-1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: The requested alignment is not a power of two.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Please request a non-zero size value.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* If zero or poison flags are set we have to have write access to the page. */
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(*puiFlags) ||
+	    PVRSRV_CHECK_POISON_ON_ALLOC(*puiFlags) ||
+#if defined(DEBUG)
+	    PVRSRV_CHECK_POISON_ON_FREE(*puiFlags) ||
+#endif
+	    PVRSRV_CHECK_CPU_WRITEABLE(*puiFlags))
+	{
+		(*puiFlags) |= PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	Allocate and init an import structure
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructAlloc(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_IMPORT **ppsImport)
+{
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_ERROR eError;
+
+	psImport = OSAllocMem(sizeof(*psImport));
+	PVR_RETURN_IF_FALSE(psImport != NULL, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	/* Setup some known bad values for things we don't have yet */
+	psImport->sDeviceImport.hReservation = LACK_OF_RESERVATION_POISON;
+	psImport->sDeviceImport.hMapping = LACK_OF_MAPPING_POISON;
+	psImport->sDeviceImport.psHeap = NULL;
+	psImport->sDeviceImport.bMapped = IMG_FALSE;
+
+	eError = OSLockCreate(&psImport->sDeviceImport.hLock);
+	PVR_GOTO_IF_ERROR(eError, failDIOSLockCreate);
+
+	psImport->sCPUImport.hOSMMapData = NULL;
+	psImport->sCPUImport.pvCPUVAddr = NULL;
+
+	eError = OSLockCreate(&psImport->sCPUImport.hLock);
+	PVR_GOTO_IF_ERROR(eError, failCIOSLockCreate);
+
+	/* Set up common elements */
+	psImport->hDevConnection = hDevConnection;
+
+	/* Setup properties */
+	psImport->uiProperties = 0;
+
+	/* Setup refcounts */
+	psImport->sDeviceImport.ui32RefCount = 0;
+	psImport->sCPUImport.ui32RefCount = 0;
+	OSAtomicWrite(&psImport->hRefCount, 0);
+
+	/* Create the lock */
+	eError = OSLockCreate(&psImport->hLock);
+	PVR_GOTO_IF_ERROR(eError, failILockAlloc);
+
+	*ppsImport = psImport;
+
+	return PVRSRV_OK;
+
+failILockAlloc:
+	OSLockDestroy(psImport->sCPUImport.hLock);
+failCIOSLockCreate:
+	OSLockDestroy(psImport->sDeviceImport.hLock);
+failDIOSLockCreate:
+	OSFreeMem(psImport);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	Initialise the import structure
+ */
+IMG_INTERNAL
+void DevmemImportStructInit(DEVMEM_IMPORT *psImport,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		IMG_HANDLE hPMR,
+		DEVMEM_PROPERTIES_T uiProperties)
+{
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			0,
+			1);
+
+	psImport->uiSize = uiSize;
+	psImport->uiAlign = uiAlign;
+	psImport->uiFlags = uiFlags;
+	psImport->hPMR = hPMR;
+	psImport->uiProperties = uiProperties;
+	OSAtomicWrite(&psImport->hRefCount, 1);
+}
+
+/* Allocate the requested device virtual address region
+ * from the heap */
+static PVRSRV_ERROR DevmemReserveVARange(DEVMEM_HEAP *psHeap,
+                                         DEVMEM_SIZE_T uiSize,
+                                         IMG_UINT uiAlign,
+                                         RA_LENGTH_T *puiAllocatedSize,
+                                         IMG_UINT64 ui64OptionalMapAddress)
+{
+	PVRSRV_ERROR eError;
+
+	/* Allocate space in the VM */
+	eError = RA_Alloc_Range(psHeap->psQuantizedVMRA,
+							uiSize,
+							0,
+							uiAlign,
+							ui64OptionalMapAddress,
+							puiAllocatedSize);
+
+	if (PVRSRV_OK != eError)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		if ((eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL) ||
+				(eError == PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL))
+		{
+			PVRSRV_ERROR eErr;
+			eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+											PVRSRV_DEVICE_STAT_TYPE_INVALID_VIRTMEM,
+											OSGetCurrentProcessID());
+			PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+		}
+#endif
+		return eError;
+	}
+
+	/* No reason for the allocated virtual size to be different from
+					   the PMR's size */
+	PVR_ASSERT(*puiAllocatedSize == uiSize);
+
+	return PVRSRV_OK;
+}
+
+/*
+	Map an import to the device
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructDevMap(DEVMEM_HEAP *psHeap,
+		IMG_BOOL bMap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT64 ui64OptionalMapAddress)
+{
+	DEVMEM_DEVICE_IMPORT *psDeviceImport;
+	RA_BASE_T uiAllocatedAddr;
+	RA_LENGTH_T uiAllocatedSize;
+	IMG_DEV_VIRTADDR sBase;
+	PVRSRV_ERROR eError;
+	IMG_UINT uiAlign;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+
+	/* Round the provided import alignment to the configured heap alignment */
+	uiAlign = 1ULL << psHeap->uiLog2ImportAlignment;
+	uiAlign = PVR_ALIGN(psImport->uiAlign, uiAlign);
+
+	psDeviceImport = &psImport->sDeviceImport;
+
+	OSLockAcquire(psDeviceImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psDeviceImport->ui32RefCount,
+			psDeviceImport->ui32RefCount+1);
+
+	if (psDeviceImport->ui32RefCount++ == 0)
+	{
+		DevmemImportStructAcquire(psImport);
+
+		OSAtomicIncrement(&psHeap->hImportCount);
+
+		if (PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+		{
+			/*  SVM (shared virtual memory) imports or allocations always
+				need to acquire CPU virtual address first as address is
+				used to map the allocation into the device virtual address
+				space; i.e. the virtual address of the allocation for both
+				the CPU/GPU must be identical. */
+			eError = DevmemImportStructDevMapSVM(psHeap,
+					psImport,
+					uiAlign,
+					&ui64OptionalMapAddress);
+			PVR_GOTO_IF_ERROR(eError, failVMRAAlloc);
+		}
+
+		if (ui64OptionalMapAddress == 0)
+		{
+			/* If heap is _completely_ managed by USER or KERNEL, we shouldn't
+			 * be here, as this is RA manager code-path */
+			if (psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ||
+				psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_KERNEL)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ?
+						"%s: Heap is user managed, please use PVRSRVMapToDeviceAddress().":
+						"%s: Heap is kernel managed, use right allocation flags (e.g. SVM).",
+						__func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			if (psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_UNKNOWN)
+			{
+				/* Only set the heap manager (to RA) at first map when heap manager
+				 * is unknown. It might be a dual heap (both, user and RA managed),
+				 * in which case heap manager is set at creation time */
+				psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_RA;
+			}
+
+			/* Allocate space in the VM */
+			eError = RA_Alloc(psHeap->psQuantizedVMRA,
+					psImport->uiSize,
+					RA_NO_IMPORT_MULTIPLIER,
+					0, /* flags: this RA doesn't use flags*/
+					uiAlign,
+					"Virtual_Alloc",
+					&uiAllocatedAddr,
+					&uiAllocatedSize,
+					NULL /* don't care about per-import priv data */
+			);
+			if (PVRSRV_OK != eError)
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				if (eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+				{
+					PVRSRV_ERROR eErr;
+					eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+									  PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT,
+									  OSGetCurrentProcessID());
+					PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+				}
+#endif
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_DEVICE_VM, failVMRAAlloc);
+			}
+
+			/* No reason for the allocated virtual size to be different from
+			   the PMR's size */
+			PVR_ASSERT(uiAllocatedSize == psImport->uiSize);
+
+			sBase.uiAddr = uiAllocatedAddr;
+
+		}
+		else
+		{
+			IMG_UINT64 ui64ValidEndAddr;
+
+			/* Ensure supplied ui64OptionalMapAddress is within heap range */
+			ui64ValidEndAddr = psHeap->sBaseAddress.uiAddr + psHeap->uiSize;
+			if ((ui64OptionalMapAddress + psImport->uiSize > ui64ValidEndAddr) ||
+					(ui64OptionalMapAddress < psHeap->sBaseAddress.uiAddr))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: ui64OptionalMapAddress %p is outside of heap limits <%p:%p>."
+						, __func__
+						, (void*)(uintptr_t)ui64OptionalMapAddress
+						, (void*)(uintptr_t)psHeap->sBaseAddress.uiAddr
+						, (void*)(uintptr_t)ui64ValidEndAddr));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			switch (psHeap->ui32HeapManagerFlags)
+			{
+				case DEVMEM_HEAP_MANAGER_UNKNOWN:
+					/* DEVMEM_HEAP_MANAGER_USER can apply to _any_ heap and can only
+					 * be determined here. This heap type transitions from
+					 * DEVMEM_HEAP_MANAGER_UNKNOWN to DEVMEM_HEAP_MANAGER_USER on
+					 * 1st alloc. */
+					psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_USER;
+					break;
+
+				case DEVMEM_HEAP_MANAGER_USER:
+				case DEVMEM_HEAP_MANAGER_KERNEL:
+					if (! psHeap->uiSize)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ?
+										"%s: Heap DEVMEM_HEAP_MANAGER_USER is disabled.":
+										"%s: Heap DEVMEM_HEAP_MANAGER_KERNEL is disabled."
+										, __func__));
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAP, failVMRAAlloc);
+					}
+					break;
+
+				case DEVMEM_HEAP_MANAGER_DUAL_USER_RA:
+					/* When the heap is dual managed, ensure supplied ui64OptionalMapAddress
+					 * and import size are within heap address space range */
+					if (ui64OptionalMapAddress + psImport->uiSize <=
+							psHeap->sBaseAddress.uiAddr + psHeap->uiReservedRegionSize)
+					{
+						break;
+					}
+					else
+					{
+						/* Allocate requested VM range */
+						eError = DevmemReserveVARange(psHeap,
+													psImport->uiSize,
+													uiAlign,
+													&uiAllocatedSize,
+													ui64OptionalMapAddress);
+						if (eError != PVRSRV_OK)
+						{
+							PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED, failVMRAAlloc);
+						}
+
+					}
+					break;
+				case DEVMEM_HEAP_MANAGER_RA:
+					/* Allocate requested VM range */
+					eError = DevmemReserveVARange(psHeap,
+												psImport->uiSize,
+												uiAlign,
+												&uiAllocatedSize,
+												ui64OptionalMapAddress);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED, failVMRAAlloc);
+					}
+					break;
+
+				default:
+					break;
+			}
+
+			if (ui64OptionalMapAddress & ((1 << psHeap->uiLog2Quantum) - 1))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid address to map to. Please provide an "
+						"address aligned to a page multiple of the heap."
+						, __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			if (psImport->uiSize & ((1 << psHeap->uiLog2Quantum) - 1))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid heap to map to. "
+						"Please choose a heap that can handle smaller page sizes."
+						, __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			uiAllocatedAddr = ui64OptionalMapAddress;
+			uiAllocatedSize = psImport->uiSize;
+			sBase.uiAddr = uiAllocatedAddr;
+		}
+
+		if (psHeap->bPremapped)
+		{
+			/* No virtual address reservation and mapping are required for
+			 * memory that is already pre-mapped e.g. FW heaps in VZ configs */
+			psDeviceImport->hReservation = LACK_OF_RESERVATION_POISON;
+			psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+		}
+		else
+		{
+			PVRSRV_MEMALLOCFLAGS_T uiFlags;
+			uiFlags = psImport->uiFlags & PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK;
+			if (bMap)
+			{
+				eError = BridgeDevmemIntReserveRangeAndMapPMR(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+				                                              psHeap->hDevMemServerHeap,
+				                                              sBase,
+				                                              uiAllocatedSize,
+				                                              psImport->hPMR,
+				                                              uiFlags,
+				                                              &psDeviceImport->hReservation);
+
+				psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+				PVR_GOTO_IF_ERROR(eError, failReserve);
+
+				psDeviceImport->bMapped = IMG_TRUE;
+			}
+			else
+			{
+				/* Setup page tables for the allocated VM space */
+				eError = BridgeDevmemIntReserveRange(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+				                                     psHeap->hDevMemServerHeap,
+				                                     sBase,
+				                                     uiAllocatedSize,
+				                                     uiFlags,
+				                                     &psDeviceImport->hReservation);
+				PVR_GOTO_IF_ERROR(eError, failReserve);
+			}
+		}
+
+		/* Setup device mapping specific parts of the mapping info */
+		psDeviceImport->sDevVAddr.uiAddr = uiAllocatedAddr;
+		psDeviceImport->psHeap = psHeap;
+	}
+	else
+	{
+		/*
+			Check that we've been asked to map it into the
+			same heap 2nd time around
+		 */
+		if (psHeap != psDeviceImport->psHeap)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAP, failParams);
+		}
+	}
+	OSLockRelease(psDeviceImport->hLock);
+
+	return PVRSRV_OK;
+
+failReserve:
+	if (ui64OptionalMapAddress == 0)
+	{
+		RA_Free(psHeap->psQuantizedVMRA,
+				uiAllocatedAddr);
+	}
+failVMRAAlloc:
+	if ((ui64OptionalMapAddress) && PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+	{
+		DevmemImportStructDevUnmapSVM(psHeap, psImport);
+	}
+	bDestroyed = DevmemImportStructRelease(psImport);
+	OSAtomicDecrement(&psHeap->hImportCount);
+failParams:
+	if (!bDestroyed)
+	{
+		psDeviceImport->ui32RefCount--;
+		OSLockRelease(psDeviceImport->hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+	Unmap an import from the Device
+ */
+IMG_INTERNAL
+IMG_BOOL DevmemImportStructDevUnmap(DEVMEM_IMPORT *psImport)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_DEVICE_IMPORT *psDeviceImport;
+
+	psDeviceImport = &psImport->sDeviceImport;
+
+	OSLockAcquire(psDeviceImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psDeviceImport->ui32RefCount,
+			psDeviceImport->ui32RefCount-1);
+
+	if (--psDeviceImport->ui32RefCount == 0)
+	{
+		DEVMEM_HEAP *psHeap = psDeviceImport->psHeap;
+
+		if (!psHeap->bPremapped)
+		{
+			eError = DestroyServerResource(psImport->hDevConnection,
+			                               NULL,
+			                               BridgeDevmemIntUnreserveRange,
+			                               psDeviceImport->hReservation);
+			PVR_ASSERT(eError == PVRSRV_OK);
+		}
+
+		psDeviceImport->bMapped = IMG_FALSE;
+		psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+		psDeviceImport->hReservation = LACK_OF_RESERVATION_POISON;
+
+		/* DEVMEM_HEAP_MANAGER_RA can also come from a dual managed heap in which case,
+		   we need to check if the allocated VA falls within RA managed range */
+		if ((psHeap->ui32HeapManagerFlags & DEVMEM_HEAP_MANAGER_RA) &&
+		    psDeviceImport->sDevVAddr.uiAddr >= (psHeap->sBaseAddress.uiAddr + psHeap->uiReservedRegionSize) &&
+		    psDeviceImport->sDevVAddr.uiAddr < (psHeap->sBaseAddress.uiAddr + psHeap->uiSize))
+		{
+			RA_Free(psHeap->psQuantizedVMRA, psDeviceImport->sDevVAddr.uiAddr);
+		}
+
+		if (PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+		{
+			DevmemImportStructDevUnmapSVM(psHeap, psImport);
+		}
+
+		OSLockRelease(psDeviceImport->hLock);
+
+		DevmemImportStructRelease(psImport);
+
+		OSAtomicDecrement(&psHeap->hImportCount);
+
+		return IMG_TRUE;
+	}
+	else
+	{
+		OSLockRelease(psDeviceImport->hLock);
+		return IMG_FALSE;
+	}
+}
+
+/*
+	Map an import into the CPU
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructCPUMap(DEVMEM_IMPORT *psImport)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_CPU_IMPORT *psCPUImport;
+	size_t uiMappingLength;
+
+	psCPUImport = &psImport->sCPUImport;
+
+	OSLockAcquire(psCPUImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psCPUImport->ui32RefCount,
+			psCPUImport->ui32RefCount+1);
+
+	if (psCPUImport->ui32RefCount++ == 0)
+	{
+		DevmemImportStructAcquire(psImport);
+
+		eError = OSMMapPMR(GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				psImport->uiSize,
+				psImport->uiFlags,
+				&psCPUImport->hOSMMapData,
+				&psCPUImport->pvCPUVAddr,
+				&uiMappingLength);
+		PVR_GOTO_IF_ERROR(eError, failMap);
+
+		/* MappingLength might be rounded up to page size */
+		PVR_ASSERT(uiMappingLength >= psImport->uiSize);
+	}
+	OSLockRelease(psCPUImport->hLock);
+
+	return PVRSRV_OK;
+
+failMap:
+	psCPUImport->ui32RefCount--;
+	if (!DevmemImportStructRelease(psImport))
+	{
+		OSLockRelease(psCPUImport->hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+	Unmap an import from the CPU
+ */
+IMG_INTERNAL
+void DevmemImportStructCPUUnmap(DEVMEM_IMPORT *psImport)
+{
+	DEVMEM_CPU_IMPORT *psCPUImport;
+
+	psCPUImport = &psImport->sCPUImport;
+
+	OSLockAcquire(psCPUImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psCPUImport->ui32RefCount,
+			psCPUImport->ui32RefCount-1);
+
+	if (--psCPUImport->ui32RefCount == 0)
+	{
+		/* psImport->uiSize is a 64-bit quantity whereas the 5th
+		 * argument to OSUnmapPMR is a 32-bit quantity on 32-bit systems
+		 * hence a compiler warning of implicit cast and loss of data.
+		 * Added explicit cast and assert to remove warning.
+		 */
+#if defined(__linux__) && defined(__i386__)
+		PVR_ASSERT(psImport->uiSize<IMG_UINT32_MAX);
+#endif
+		OSMUnmapPMR(GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				psCPUImport->hOSMMapData,
+				psCPUImport->pvCPUVAddr,
+				(size_t)psImport->uiSize);
+
+		psCPUImport->hOSMMapData = NULL;
+		psCPUImport->pvCPUVAddr = NULL;
+
+		OSLockRelease(psCPUImport->hLock);
+
+		DevmemImportStructRelease(psImport);
+	}
+	else
+	{
+		OSLockRelease(psCPUImport->hLock);
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicememx_pdump.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicememx_pdump.c
new file mode 100644
index 000000000000..7b74db5f2cca
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/devicememx_pdump.c
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@File
+@Title          Shared X device memory management PDump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common (client & server) PDump functions for the
+                memory management code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#if defined(PDUMP)
+
+#include "devicememx_pdump.h"
+#include "pdump.h"
+#include "client_pdumpmm_bridge.h"
+#include "devicemem_utils.h"
+
+IMG_INTERNAL void
+DevmemXPDumpLoadMem(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiSize != 0);
+	PVR_ASSERT(uiOffset + uiSize <= (psMemDescPhys->uiNumPages <<
+			psMemDescPhys->uiLog2PageSize));
+
+	eError = BridgePMRPDumpLoadMem(GetBridgeHandle(psMemDescPhys->hConnection),
+	                               psMemDescPhys->hPMR,
+	                               uiOffset,
+	                               uiSize,
+	                               uiPDumpFlags,
+	                               IMG_FALSE);
+	PVR_LOG_IF_ERROR(eError, "BridgePMRPDumpLoadMem");
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash.c
new file mode 100644
index 000000000000..434c6e967bc4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash.c
@@ -0,0 +1,747 @@
+/*************************************************************************/ /*!
+@File
+@Title          Self scaling hash tables.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+   Implements simple self scaling hash tables. Hash collisions are handled by
+   chaining entries together. Hash tables are increased in size when they
+   become more than (50%?) full and decreased in size when less than (25%?)
+   full. Hash tables are never decreased below their initial size.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* include/ */
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+/* services/shared/include/ */
+#include "hash.h"
+
+/* services/client/include/ or services/server/include/ */
+#include "osfunc_common.h"
+#include "allocmem.h"
+
+//#define PERF_DBG_RESIZE
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+#include <sys/time.h>
+#endif
+
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#endif
+
+#define	KEY_TO_INDEX(pHash, key, uSize) \
+	((pHash)->pfnHashFunc((pHash)->uKeySize, (key), (uSize)) % (uSize))
+
+#define	KEY_COMPARE(pHash, pKey1, pKey2) \
+	((pHash)->pfnKeyComp((pHash)->uKeySize, (pKey1), (pKey2)))
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define _AllocMem OSAllocMemNoStats
+#define _AllocZMem OSAllocZMemNoStats
+#define _FreeMem OSFreeMemNoStats
+#else
+#define _AllocMem OSAllocMem
+#define _AllocZMem OSAllocZMem
+#define _FreeMem OSFreeMem
+#endif
+
+#define NO_SHRINK 0
+
+/* Each entry in a hash table is placed into a bucket */
+typedef struct _BUCKET_
+{
+	struct _BUCKET_ *pNext; /*!< the next bucket on the same chain */
+	uintptr_t v;            /*!< entry value */
+	uintptr_t k[];          /* PRQA S 0642 */
+	                        /* override dynamic array declaration warning */
+} BUCKET;
+
+struct _HASH_TABLE_
+{
+	IMG_UINT32 uSize;            /*!< current size of the hash table */
+	IMG_UINT32 uCount;           /*!< number of entries currently in the hash table */
+	IMG_UINT32 uMinimumSize;     /*!< the minimum size that the hash table should be re-sized to */
+	IMG_UINT32 uKeySize;         /*!< size of key in bytes */
+	IMG_UINT32 uShrinkThreshold; /*!< The threshold at which to trigger a shrink */
+	IMG_UINT32 uGrowThreshold;   /*!< The threshold at which to trigger a grow */
+	HASH_FUNC*     pfnHashFunc;  /*!< hash function */
+	HASH_KEY_COMP* pfnKeyComp;   /*!< key comparison function */
+	BUCKET**   ppBucketTable;    /*!< the hash table array */
+#if defined(DEBUG)
+	const char*      pszFile;
+	unsigned int     ui32LineNum;
+#endif
+};
+
+/*************************************************************************/ /*!
+@Function       HASH_Func_Default
+@Description    Hash function intended for hashing keys composed of uintptr_t
+                arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey         A pointer to the key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_UINT32
+HASH_Func_Default(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	uintptr_t *p = (uintptr_t *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(uintptr_t);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	PVR_ASSERT((uKeySize % sizeof(uintptr_t)) == 0);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = (IMG_UINT32)*p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares keys composed of uintptr_t arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey1        Pointer to first hash key to compare.
+@Input          pKey2        Pointer to second hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Key_Comp_Default(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	uintptr_t *p1 = (uintptr_t *)pKey1;
+	uintptr_t *p2 = (uintptr_t *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(uintptr_t);
+	IMG_UINT32 ui;
+
+	PVR_ASSERT((uKeySize % sizeof(uintptr_t)) == 0);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _ChainInsert
+@Description    Insert a bucket into the appropriate hash table chain.
+@Input          pBucket       The bucket
+@Input          ppBucketTable The hash table
+@Input          uSize         The size of the hash table
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static void
+_ChainInsert(HASH_TABLE *pHash, BUCKET *pBucket, BUCKET **ppBucketTable, IMG_UINT32 uSize)
+{
+	IMG_UINT32 uIndex;
+
+	/* We assume that all parameters passed by the caller are valid. */
+	PVR_ASSERT(pBucket != NULL);
+	PVR_ASSERT(ppBucketTable != NULL);
+	PVR_ASSERT(uSize != 0);
+
+	uIndex = KEY_TO_INDEX(pHash, pBucket->k, uSize);	/* PRQA S 0432,0541 */ /* ignore dynamic array warning */
+	pBucket->pNext = ppBucketTable[uIndex];
+	ppBucketTable[uIndex] = pBucket;
+}
+
+/*************************************************************************/ /*!
+@Function       _Rehash
+@Description    Iterate over every entry in an old hash table and rehash into
+                the new table.
+@Input          ppOldTable   The old hash table
+@Input          uOldSize     The size of the old hash table
+@Input          ppNewTable   The new hash table
+@Input          uNewSize     The size of the new hash table
+@Return         None
+*/ /**************************************************************************/
+static void
+_Rehash(HASH_TABLE *pHash,
+		 BUCKET **ppOldTable, IMG_UINT32 uOldSize,
+		 BUCKET **ppNewTable, IMG_UINT32 uNewSize)
+{
+	IMG_UINT32 uIndex;
+	for (uIndex=0; uIndex< uOldSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		pBucket = ppOldTable[uIndex];
+		while (pBucket != NULL)
+		{
+			BUCKET *pNextBucket = pBucket->pNext;
+			_ChainInsert(pHash, pBucket, ppNewTable, uNewSize);
+			pBucket = pNextBucket;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       _Resize
+@Description    Attempt to resize a hash table, failure to allocate a new
+                larger hash table is not considered a hard failure. We simply
+                continue and allow the table to fill up, the effect is to
+                allow hash chains to become longer.
+@Input          pHash        Hash table to resize.
+@Input          uNewSize     Required table size.
+@Return         IMG_TRUE Success
+                IMG_FALSE Failed
+*/ /**************************************************************************/
+static IMG_BOOL
+_Resize(HASH_TABLE *pHash, IMG_UINT32 uNewSize)
+{
+	BUCKET **ppNewTable;
+	IMG_UINT32 uiThreshold = uNewSize >> 2;
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	struct timeval start, end;
+#endif
+
+	if (uNewSize == pHash->uSize)
+	{
+		return IMG_TRUE;
+	}
+
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	gettimeofday(&start, NULL);
+#endif
+
+	ppNewTable = _AllocZMem(sizeof(BUCKET *) * uNewSize);
+	if (ppNewTable == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	_Rehash(pHash, pHash->ppBucketTable, pHash->uSize, ppNewTable, uNewSize);
+
+	_FreeMem(pHash->ppBucketTable);
+
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	gettimeofday(&end, NULL);
+	if (start.tv_usec > end.tv_usec)
+	{
+		end.tv_usec = 1000000 - start.tv_usec + end.tv_usec;
+	}
+	else
+	{
+		end.tv_usec -= start.tv_usec;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: H:%p O:%d N:%d C:%d G:%d S:%d T:%06luus", __func__, pHash, pHash->uSize, uNewSize, pHash->uCount, pHash->uGrowThreshold, pHash->uShrinkThreshold, end.tv_usec));
+#endif
+
+	/*not nulling pointer, being reassigned just below*/
+	pHash->ppBucketTable = ppNewTable;
+	pHash->uSize = uNewSize;
+
+	pHash->uGrowThreshold = uiThreshold * 3;
+	pHash->uShrinkThreshold = (uNewSize <= pHash->uMinimumSize) ? NO_SHRINK : uiThreshold;
+
+	return IMG_TRUE;
+}
+
+
+/*************************************************************************/ /*!
+@Function       HASH_Create_Extended
+@Description    Create a self scaling hash table, using the supplied key size,
+                and the supplied hash and key comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Input          uKeySize     The size of the key, in bytes.
+@Input          pfnHashFunc  Pointer to hash function.
+@Input          pfnKeyComp   Pointer to key comparison function.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Extended_Int (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp)
+{
+	HASH_TABLE *pHash;
+
+	if (uInitialLen == 0 || uKeySize == 0 || pfnHashFunc == NULL || pfnKeyComp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid input parameters", __func__));
+		return NULL;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: InitialSize=0x%x", __func__, uInitialLen));
+
+	pHash = _AllocMem(sizeof(HASH_TABLE));
+	if (pHash == NULL)
+	{
+		return NULL;
+	}
+
+	pHash->uCount = 0;
+	pHash->uSize = uInitialLen;
+	pHash->uMinimumSize = uInitialLen;
+	pHash->uKeySize = uKeySize;
+	pHash->uGrowThreshold = (uInitialLen >> 2) * 3;
+	pHash->uShrinkThreshold = NO_SHRINK;
+	pHash->pfnHashFunc = pfnHashFunc;
+	pHash->pfnKeyComp = pfnKeyComp;
+
+	pHash->ppBucketTable = _AllocZMem(sizeof(BUCKET *) * pHash->uSize);
+	if (pHash->ppBucketTable == NULL)
+	{
+		_FreeMem(pHash);
+		/*not nulling pointer, out of scope*/
+		return NULL;
+	}
+
+	return pHash;
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Extended_Debug (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp,
+										 const char *file, const unsigned int line)
+{
+	HASH_TABLE *hash;
+	hash = HASH_Create_Extended_Int(uInitialLen, uKeySize,
+									pfnHashFunc, pfnKeyComp);
+	if (hash)
+	{
+		hash->pszFile = file;
+		hash->ui32LineNum = line;
+	}
+	return hash;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Create
+@Description    Create a self scaling hash table with a key consisting of a
+                single uintptr_t, and using the default hash and key
+                comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Int (IMG_UINT32 uInitialLen)
+{
+	return HASH_Create_Extended_Int(uInitialLen, sizeof(uintptr_t),
+									&HASH_Func_Default, &HASH_Key_Comp_Default);
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Debug(IMG_UINT32 uInitialLen, const char *file, const unsigned int line)
+{
+	HASH_TABLE *hash;
+	hash = HASH_Create_Extended_Int(uInitialLen, sizeof(uintptr_t),
+									&HASH_Func_Default, &HASH_Key_Comp_Default);
+	if (hash)
+	{
+		hash->pszFile = file;
+		hash->ui32LineNum = line;
+	}
+	return hash;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete_Extended
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table should have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Input          bWarn        Set false to suppress warnings in the case of
+                             deletion with active entries.
+*/ /**************************************************************************/
+IMG_INTERNAL void
+HASH_Delete_Extended(HASH_TABLE *pHash, IMG_BOOL bWarn)
+{
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__) && !defined(__QNXNTO__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData != NULL)
+	{
+		if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			bDoCheck = IMG_FALSE;
+		}
+	}
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	else
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+#endif
+	if (pHash != NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HASH_Delete"));
+
+		if (bDoCheck)
+		{
+			PVR_ASSERT(pHash->uCount==0);
+		}
+		if (pHash->uCount != 0)
+		{
+			IMG_UINT32 i;
+			if (bWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Leak detected in hash table!", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Removing remaining %u hash entries.", __func__, pHash->uCount));
+#if defined(DEBUG)
+				PVR_DPF ((PVR_DBG_ERROR, "%s: Hash %p created at %s:%u.", __func__, (uintptr_t*)pHash, pHash->pszFile, pHash->ui32LineNum));
+#endif
+			}
+
+			for (i = 0; i < pHash->uSize; i++)
+			{
+				BUCKET *pBucket = pHash->ppBucketTable[i];
+				while (pBucket != NULL)
+				{
+					BUCKET *pNextBucket = pBucket->pNext;
+					_FreeMem(pBucket);
+					pBucket = pNextBucket;
+				}
+			}
+
+		}
+		_FreeMem(pHash->ppBucketTable);
+		pHash->ppBucketTable = NULL;
+		_FreeMem(pHash);
+		/*not nulling pointer, copy on stack*/
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table must have been removed
+                before calling this function.
+@Input          pHash        Hash table
+*/ /**************************************************************************/
+IMG_INTERNAL void
+HASH_Delete(HASH_TABLE *pHash)
+{
+	HASH_Delete_Extended(pHash, IMG_TRUE);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert_Extended
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to the key.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Insert_Extended(HASH_TABLE *pHash, void *pKey, uintptr_t v)
+{
+	BUCKET *pBucket;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter", __func__));
+		return IMG_FALSE;
+	}
+
+	pBucket = _AllocMem(sizeof(BUCKET) + pHash->uKeySize);
+	if (pBucket == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pBucket->v = v;
+	/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k (linux)*/
+	OSCachedMemCopy(pBucket->k, pKey, pHash->uKeySize);
+
+	_ChainInsert(pHash, pBucket, pHash->ppBucketTable, pHash->uSize);
+
+	pHash->uCount++;
+
+	/* check if we need to think about re-balancing */
+	if (pHash->uCount > pHash->uGrowThreshold)
+	{
+		/* Ignore the return code from _Resize because the hash table is
+		   still in a valid state and although not ideally sized, it is still
+		   functional */
+		_Resize(pHash, pHash->uSize << 1);
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Insert(HASH_TABLE *pHash, uintptr_t k, uintptr_t v)
+{
+	return HASH_Insert_Extended(pHash, &k, v);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove_Extended
+@Description    Remove a key from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Remove_Extended(HASH_TABLE *pHash, void *pKey)
+{
+	BUCKET **ppBucket;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Null hash table", __func__));
+		return 0;
+	}
+
+	uIndex = KEY_TO_INDEX(pHash, pKey, pHash->uSize);
+
+	for (ppBucket = &(pHash->ppBucketTable[uIndex]); *ppBucket != NULL; ppBucket = &((*ppBucket)->pNext))
+	{
+		/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k */
+		if (KEY_COMPARE(pHash, (*ppBucket)->k, pKey))
+		{
+			BUCKET *pBucket = *ppBucket;
+			uintptr_t v = pBucket->v;
+			(*ppBucket) = pBucket->pNext;
+
+			_FreeMem(pBucket);
+			/*not nulling original pointer, already overwritten*/
+
+			pHash->uCount--;
+
+			/* check if we need to think about re-balancing, when the shrink
+			 * threshold is 0 we are at the minimum size, no further shrink */
+			if (pHash->uCount < pHash->uShrinkThreshold)
+			{
+				/* Ignore the return code from _Resize because the
+				   hash table is still in a valid state and although
+				   not ideally sized, it is still functional */
+				_Resize(pHash, MAX(pHash->uSize >> 1, pHash->uMinimumSize));
+			}
+
+			return v;
+		}
+	}
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove
+@Description    Remove a key value pair from a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Remove(HASH_TABLE *pHash, uintptr_t k)
+{
+	return HASH_Remove_Extended(pHash, &k);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve_Extended
+@Description    Retrieve a value from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Retrieve_Extended(HASH_TABLE *pHash, void *pKey)
+{
+	BUCKET **ppBucket;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Null hash table", __func__));
+		return 0;
+	}
+
+	uIndex = KEY_TO_INDEX(pHash, pKey, pHash->uSize);
+
+	for (ppBucket = &(pHash->ppBucketTable[uIndex]); *ppBucket != NULL; ppBucket = &((*ppBucket)->pNext))
+	{
+		/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k */
+		if (KEY_COMPARE(pHash, (*ppBucket)->k, pKey))
+		{
+			BUCKET *pBucket = *ppBucket;
+			uintptr_t v = pBucket->v;
+
+			return v;
+		}
+	}
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve
+@Description    Retrieve a value from a hash table created with HASH_Create.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Retrieve(HASH_TABLE *pHash, uintptr_t k)
+{
+	return HASH_Retrieve_Extended(pHash, &k);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Iterate
+@Description    Iterate over every entry in the hash table.
+@Input          pHash        Hash table to iterate.
+@Input          pfnCallback  Callback to call with the key and data for each
+.                            entry in the hash table
+@Return         Callback error if any, otherwise PVRSRV_OK
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HASH_Iterate(HASH_TABLE *pHash, HASH_pfnCallback pfnCallback, void* args)
+{
+	IMG_UINT32 uIndex;
+	for (uIndex=0; uIndex < pHash->uSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		pBucket = pHash->ppBucketTable[uIndex];
+		while (pBucket != NULL)
+		{
+			PVRSRV_ERROR eError;
+			BUCKET *pNextBucket = pBucket->pNext;
+
+			eError = pfnCallback((uintptr_t) ((void *) *(pBucket->k)), pBucket->v, args);
+
+			/* The callback might want us to break out early */
+			if (eError != PVRSRV_OK)
+				return eError;
+
+			pBucket = pNextBucket;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Count
+@Description    Retrieve the number of entries in the hash table.
+@Input          pHash        The hash table.
+@Return         The number of entries.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_UINT32
+HASH_Count(HASH_TABLE *pHash)
+{
+	return pHash->uCount;
+}
+
+
+#ifdef HASH_TRACE
+/*************************************************************************/ /*!
+@Function       HASH_Dump
+@Description    Dump out some information about a hash table.
+@Input          pHash         The hash table.
+*/ /**************************************************************************/
+void
+HASH_Dump(HASH_TABLE *pHash)
+{
+	IMG_UINT32 uIndex;
+	IMG_UINT32 uMaxLength=0;
+	IMG_UINT32 uEmptyCount=0;
+
+	PVR_ASSERT(pHash != NULL);
+	for (uIndex=0; uIndex<pHash->uSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		IMG_UINT32 uLength = 0;
+		if (pHash->ppBucketTable[uIndex] == NULL)
+		{
+			uEmptyCount++;
+		}
+		for (pBucket=pHash->ppBucketTable[uIndex];
+				pBucket != NULL;
+				pBucket = pBucket->pNext)
+		{
+			uLength++;
+		}
+		uMaxLength = MAX(uMaxLength, uLength);
+	}
+
+	PVR_TRACE(("hash table: uMinimumSize=%d  size=%d  count=%d",
+			   pHash->uMinimumSize, pHash->uSize, pHash->uCount));
+	PVR_TRACE(("  empty=%d  max=%d", uEmptyCount, uMaxLength));
+}
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash_functions.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash_functions.c
new file mode 100644
index 000000000000..517dae297ff8
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/hash_functions.c
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File
+@Title          Reusable hash functions for hash.c.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common hash functions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "hash_functions.h"
+#include "img_defs.h"
+
+/* Declaring function here to avoid dependencies that are introduced by
+ * including osfunc.h. */
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                           size_t uiSize);
+
+IMG_UINT32 HASH_Djb2_Hash(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_CHAR *pszStr = pKey;
+	IMG_UINT32 ui32Hash = 5381, ui32Char;
+
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	while ((ui32Char = *pszStr++) != '\0')
+	{
+		ui32Hash = ((ui32Hash << 5) + ui32Hash) + ui32Char;
+	}
+
+	return ui32Hash;
+}
+
+IMG_BOOL HASH_Djb2_Compare(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_CHAR *pszKey1 = pKey1, *pszKey2 = pKey2;
+
+	return OSStringNCompare(pszKey1, pszKey2, uKeySize) == 0;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/htbuffer.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/htbuffer.c
new file mode 100644
index 000000000000..5f83127a5462
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/htbuffer.c
@@ -0,0 +1,106 @@
+/*************************************************************************/ /*!
+@File           htbuffer.c
+@Title          Host Trace Buffer shared API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "htbuffer.h"
+#include "osfunc.h"
+#include "client_htbuffer_bridge.h"
+
+/* The group flags array of ints large enough to store all the group flags
+ * NB: This will only work while all logging is in the kernel
+ */
+IMG_INTERNAL HTB_FLAG_EL_T g_auiHTBGroupEnable[HTB_FLAG_NUM_EL] = {0};
+
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+ @Input         hSrvHandle      Server Handle
+ @Input         ui32NumFlagGroups Number of group enable flags words
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+ @Input         ui32LogLevel    Log level to record
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+ @Input         eLogPidMode     Enable logging for all or specific processes,
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogPidMode,
+	HTB_OPMODE_CTRL eOpMode
+)
+{
+	return BridgeHTBControl(
+			hSrvHandle,
+			ui32NumFlagGroups,
+			aui32GroupEnable,
+			ui32LogLevel,
+			ui32EnablePID,
+			eLogPidMode,
+			eOpMode
+			);
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/mem_utils.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/mem_utils.c
new file mode 100644
index 000000000000..59f808cfcb20
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/mem_utils.c
@@ -0,0 +1,445 @@
+/*************************************************************************/ /*!
+@File
+@Title          Memory manipulation functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory related functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "osfunc_common.h"
+#include "img_defs.h"
+
+/* This workaround is only *required* on ARM64. Avoid building or including
+ * it by default on other architectures, unless the 'safe memcpy' test flag
+ * is enabled. (The code should work on other architectures.)
+ */
+
+
+
+/* NOTE: This C file is compiled with -ffreestanding to avoid pattern matching
+ *       by the compiler to stdlib functions, and it must only use the below
+ *       headers. Do not include any IMG or services headers in this file.
+ */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/* The attribute "vector_size" will generate floating point instructions
+ * and use FPU registers. In kernel OS, the FPU registers might be corrupted
+ * when CPU is doing context switch because FPU registers are not expected to
+ * be stored.
+ * GCC enables compiler option, -mgeneral-regs-only, by default.
+ * This option restricts the generated code to use general registers only
+ * so that we don't have issues on that.
+ */
+#if defined(__KERNEL__) && defined(__clang__)
+
+#define DEVICE_MEMSETCPY_NON_VECTOR_KM
+/* Loading or storing 16 or 32 bytes is only supported on 64-bit machines. */
+#if DEVICE_MEMSETCPY_ALIGN_IN_BYTES > 8
+typedef __uint128_t uint128_t;
+
+typedef struct
+{
+	uint128_t ui128DataFields[2];
+}
+uint256_t;
+#endif
+
+#endif
+
+/* This file is only intended to be used on platforms which use GCC or Clang,
+ * due to its requirement on __attribute__((vector_size(n))), typeof() and
+ * __SIZEOF__ macros.
+ */
+
+#if defined(__GNUC__)
+
+#ifndef MIN
+#define MIN(a, b) \
+ ({__typeof(a) _a = (a); __typeof(b) _b = (b); _a > _b ? _b : _a;})
+#endif
+
+#if !defined(DEVICE_MEMSETCPY_ALIGN_IN_BYTES)
+#define DEVICE_MEMSETCPY_ALIGN_IN_BYTES __SIZEOF_LONG__
+#endif
+#if (DEVICE_MEMSETCPY_ALIGN_IN_BYTES & (DEVICE_MEMSETCPY_ALIGN_IN_BYTES - 1)) != 0
+#error "DEVICE_MEMSETCPY_ALIGN_IN_BYTES must be a power of 2"
+#endif
+#if DEVICE_MEMSETCPY_ALIGN_IN_BYTES < 4
+#error "DEVICE_MEMSETCPY_ALIGN_IN_BYTES must be equal or greater than 4"
+#endif
+
+#if __SIZEOF_POINTER__ != __SIZEOF_LONG__
+#error No support for architectures where void* and long are sized differently
+#endif
+
+#if __SIZEOF_LONG__ > DEVICE_MEMSETCPY_ALIGN_IN_BYTES
+/* Meaningless, and harder to do correctly */
+# error Cannot handle DEVICE_MEMSETCPY_ALIGN_IN_BYTES < sizeof(long)
+typedef unsigned long block_t;
+#elif __SIZEOF_LONG__ <= DEVICE_MEMSETCPY_ALIGN_IN_BYTES
+# if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+#  if DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+    typedef uint64_t block_t;
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+    typedef uint128_t block_t;
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+    typedef uint256_t block_t;
+#  endif
+# else
+typedef unsigned int block_t
+	__attribute__((vector_size(DEVICE_MEMSETCPY_ALIGN_IN_BYTES)));
+# endif
+# if defined(__arm64__) || defined(__aarch64__)
+#  if   DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+#   define DEVICE_MEMSETCPY_ARM64
+#   define REGSZ "w"
+#   define REGCL "w"
+#   define BVCLB "r"
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+#   define DEVICE_MEMSETCPY_ARM64
+#   define REGSZ "x"
+#   define REGCL "x"
+#   define BVCLB "r"
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+#   if defined(__ARM_NEON_FP)
+#    define DEVICE_MEMSETCPY_ARM64
+#    define REGSZ "q"
+#    define REGCL "v"
+#    define BVCLB "w"
+#   endif
+#  endif
+#  if defined(DEVICE_MEMSETCPY_ARM64)
+#   if defined(DEVICE_MEMSETCPY_ARM64_NON_TEMPORAL)
+#    define NSHLD() __asm__ ("dmb nshld")
+#    define NSHST() __asm__ ("dmb nshst")
+#    define LDP "ldnp"
+#    define STP "stnp"
+#   else
+#    define NSHLD()
+#    define NSHST()
+#    define LDP "ldp"
+#    define STP "stp"
+#   endif
+#   if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+#    if DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+typedef uint32_t block_half_t;
+#    elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+typedef uint64_t block_half_t;
+#    elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+typedef uint128_t block_half_t;
+#    endif
+#   else
+ typedef unsigned int block_half_t
+	__attribute__((vector_size(DEVICE_MEMSETCPY_ALIGN_IN_BYTES / 2)));
+#   endif
+#  endif
+# endif
+#endif
+
+__attribute__((visibility("hidden")))
+void DeviceMemCopy(void *pvDst, const void *pvSrc, size_t uSize)
+{
+	volatile const char *pcSrc = pvSrc;
+	volatile char *pcDst = pvDst;
+	size_t uPreambleBytes;
+	int bBlockCopy = 0;
+
+	size_t uSrcUnaligned = (size_t)pcSrc % sizeof(block_t);
+	size_t uDstUnaligned = (size_t)pcDst % sizeof(block_t);
+
+	if (!uSrcUnaligned && !uDstUnaligned)
+	{
+		/* Neither pointer is unaligned. Optimal case. */
+		bBlockCopy = 1;
+	}
+	else
+	{
+		if (uSrcUnaligned == uDstUnaligned)
+		{
+			/* Neither pointer is usefully aligned, but they are misaligned in
+			 * the same way, so we can copy a preamble in a slow way, then
+			 * optimize the rest.
+			 */
+			uPreambleBytes = MIN(sizeof(block_t) - uDstUnaligned, uSize);
+			uSize -= uPreambleBytes;
+			while (uPreambleBytes)
+			{
+				*pcDst++ = *pcSrc++;
+				uPreambleBytes--;
+			}
+
+			bBlockCopy = 1;
+		}
+		else if ((uSrcUnaligned | uDstUnaligned) % sizeof(int) == 0)
+		{
+			/* Both pointers are at least 32-bit aligned, and we assume that
+			 * the processor must handle all kinds of 32-bit load-stores.
+			 * NOTE: Could we optimize this with a non-temporal version?
+			 */
+			if (uSize >= sizeof(int))
+			{
+				volatile int *piSrc = (int *)((void *)pcSrc);
+				volatile int *piDst = (int *)((void *)pcDst);
+
+				while (uSize >= sizeof(int))
+				{
+					*piDst++ = *piSrc++;
+					uSize -= sizeof(int);
+				}
+
+				pcSrc = (char *)((void *)piSrc);
+				pcDst = (char *)((void *)piDst);
+			}
+		}
+	}
+
+	if (bBlockCopy && uSize >= sizeof(block_t))
+	{
+		volatile block_t *pSrc = (block_t *)((void *)pcSrc);
+		volatile block_t *pDst = (block_t *)((void *)pcDst);
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHLD();
+#endif
+
+		while (uSize >= sizeof(block_t))
+		{
+#if defined(DEVICE_MEMSETCPY_ARM64)
+			__asm__ (LDP " " REGSZ "0, " REGSZ "1, [%[pSrc]]\n\t"
+			         STP " " REGSZ "0, " REGSZ "1, [%[pDst]]"
+						:
+						: [pSrc] "r" (pSrc), [pDst] "r" (pDst)
+						: "memory", REGCL "0", REGCL "1");
+#else
+			*pDst = *pSrc;
+#endif
+			pDst++;
+			pSrc++;
+			uSize -= sizeof(block_t);
+		}
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHST();
+#endif
+
+		pcSrc = (char *)((void *)pSrc);
+		pcDst = (char *)((void *)pDst);
+	}
+
+	while (uSize)
+	{
+		*pcDst++ = *pcSrc++;
+		uSize--;
+	}
+}
+
+__attribute__((visibility("hidden")))
+void DeviceMemSet(void *pvDst, unsigned char ui8Value, size_t uSize)
+{
+	volatile char *pcDst = pvDst;
+	size_t uPreambleBytes;
+
+	size_t uDstUnaligned = (size_t)pcDst % sizeof(block_t);
+
+	if (uDstUnaligned)
+	{
+		uPreambleBytes = MIN(sizeof(block_t) - uDstUnaligned, uSize);
+		uSize -= uPreambleBytes;
+		while (uPreambleBytes)
+		{
+			*pcDst++ = ui8Value;
+			uPreambleBytes--;
+		}
+	}
+
+	if (uSize >= sizeof(block_t))
+	{
+		volatile block_t *pDst = (block_t *)((void *)pcDst);
+		size_t i, uBlockSize;
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		typedef block_half_t BLK_t;
+#else
+		typedef block_t BLK_t;
+#endif /* defined(DEVICE_MEMSETCPY_ARM64) */
+
+#if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+		BLK_t bValue = 0;
+
+		uBlockSize = sizeof(BLK_t) / sizeof(ui8Value);
+
+		for (i = 0; i < uBlockSize; i++)
+		{
+			bValue |= (BLK_t)ui8Value << ((uBlockSize - i - 1) * BITS_PER_BYTE);
+		}
+#else
+		BLK_t bValue = {0};
+
+		uBlockSize = sizeof(bValue) / sizeof(unsigned int);
+		for (i = 0; i < uBlockSize; i++)
+			bValue[i] = ui8Value << 24U |
+			            ui8Value << 16U |
+			            ui8Value <<  8U |
+			            ui8Value;
+#endif /* defined(DEVICE_MEMSETCPY_NON_VECTOR_KM) */
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHLD();
+#endif
+
+		while (uSize >= sizeof(block_t))
+		{
+#if defined(DEVICE_MEMSETCPY_ARM64)
+			__asm__ (STP " %" REGSZ "[bValue], %" REGSZ "[bValue], [%[pDst]]"
+						:
+						: [bValue] BVCLB (bValue), [pDst] "r" (pDst)
+						: "memory");
+#else
+			*pDst = bValue;
+#endif
+			pDst++;
+			uSize -= sizeof(block_t);
+		}
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHST();
+#endif
+
+		pcDst = (char *)((void *)pDst);
+	}
+
+	while (uSize)
+	{
+		*pcDst++ = ui8Value;
+		uSize--;
+	}
+}
+
+#endif /* defined(__GNUC__) */
+
+/* Potentially very slow (but safe) fallbacks for non-GNU C compilers */
+IMG_INTERNAL
+void DeviceMemCopyBytes(void *pvDst, const void *pvSrc, size_t uSize)
+{
+	volatile const char *pcSrc = pvSrc;
+	volatile char *pcDst = pvDst;
+
+	while (uSize)
+	{
+		*pcDst++ = *pcSrc++;
+		uSize--;
+	}
+}
+
+IMG_INTERNAL
+void DeviceMemSetBytes(void *pvDst, unsigned char ui8Value, size_t uSize)
+{
+	volatile char *pcDst = pvDst;
+
+	while (uSize)
+	{
+		*pcDst++ = ui8Value;
+		uSize--;
+	}
+}
+
+#if defined(__linux__) && defined(__KERNEL__)
+#if defined(DEBUG)
+/*
+ * In case of Linux kernel-mode in a debug build, choose the variant of
+ * OSStringSafeCopy that uses strscpy and logs truncation via a stack dump. For
+ * Linux kernel-mode in a release build, strscpy alone is used.
+ */
+IMG_INTERNAL
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	/*
+	 * Let strscpy handle any truncation cases correctly.
+	 * We will definitely get a NUL-terminated string set in pszDest
+	 */
+	ssize_t sCopiedCnt = strscpy(pszDest, pszSrc, uDataSize);
+
+#if defined(PVR_DEBUG_STRLCPY)
+	/* Handle truncation by dumping calling stack if debug allows */
+	if (sCopiedCnt < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+			"%s: String truncated Src = '<%s>' %zu bytes, Dest = '%s'",
+			__func__, pszSrc, uDataSize, pszDest));
+		OSDumpStack();
+	}
+#endif /* defined(PVR_DEBUG_STRLCPY) && defined(DEBUG) */
+
+	return sCopiedCnt;
+}
+#endif /* defined(DEBUG) */
+#else /* defined(__linux__) && defined(__KERNEL__) */
+/*
+ * For every other platform, make use of the strnlen and strncpy implementation
+ * of OSStringSafeCopy.
+ *
+ * NOTE: It is crucial to avoid memcpy as this has a hidden side-effect of
+ * dragging in whatever the build-environment flavour of GLIBC is which can
+ * cause unexpected failures for host-side command execution.
+ */
+IMG_INTERNAL
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	/* Match the specification of Linux strscpy - if destination size is 0,
+	 * return negative value (-E2BIG to be precise, but it's probably best
+	 * not to use Linux-specific return values, so -1 will do). */
+	if (uDataSize && (uDataSize <= SSIZE_MAX))
+	{
+		size_t uSrcSize = strnlen(pszSrc, uDataSize);
+
+		(void)strncpy(pszDest, pszSrc, MIN(uSrcSize + 1, uDataSize));
+		if (uSrcSize == uDataSize)
+		{
+			pszDest[uSrcSize-1] = '\0';
+			return -1;
+		}
+
+		return (ssize_t)uSrcSize;
+	}
+
+	return -1;
+}
+#endif /* defined(__linux__) && defined(__KERNEL__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/pvrsrv_error.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/pvrsrv_error.c
new file mode 100644
index 000000000000..5cd02a28d5a1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/pvrsrv_error.c
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services error support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+IMG_EXPORT
+const IMG_CHAR *PVRSRVGetErrorString(PVRSRV_ERROR eError)
+{
+	switch (eError)
+	{
+		case PVRSRV_OK:
+			return "PVRSRV_OK";
+#define PVRE(x) \
+		case x: \
+			return #x;
+#include "pvrsrv_errors.h"
+#undef PVRE
+		default:
+			return "Unknown PVRSRV error number";
+	}
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/ra.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/ra.c
new file mode 100644
index 000000000000..d1646b309f6d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/ra.c
@@ -0,0 +1,4040 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Description
+ Implements generic resource allocation. The resource allocator was originally
+ intended to manage address spaces. In practice the resource allocator is
+ generic and can manage arbitrary sets of integers.
+
+ Resources are allocated from arenas. Arenas can be created with an initial
+ span of resources. Further resources spans can be added to arenas. A
+ callback mechanism allows an arena to request further resource spans on
+ demand.
+
+ Each arena maintains an ordered list of resource segments each described by a
+ boundary tag. Each boundary tag describes a segment of resources which are
+ either 'free', available for allocation, or 'busy' currently allocated.
+ Adjacent 'free' segments are always coalesced to avoid fragmentation.
+
+ For allocation, all 'free' segments are kept on lists of 'free' segments in
+ a table index by pvr_log2(segment size) i.e., each table index n holds 'free'
+ segments in the size range 2^n -> 2^(n+1) - 1.
+
+ Allocation policy is based on an *almost* good fit strategy.
+
+ Allocated segments are inserted into a self-scaling hash table which maps
+ the base resource of the span to the relevant boundary tag. This allows the
+ code to get back to the boundary tag without exporting explicit boundary tag
+ references through the API.
+
+ Each arena has an associated quantum size, all allocations from the arena are
+ made in multiples of the basic quantum.
+
+ On resource exhaustion in an arena, a callback if provided will be used to
+ request further resources. Resource spans allocated by the callback mechanism
+ will be returned when freed (through one of the two callbacks).
+*/ /**************************************************************************/
+
+/* Issues:
+ * - flags, flags are passed into the resource allocator but are not currently used.
+ * - determination, of import size, is currently braindead.
+ * - debug code should be moved out to own module and #ifdef'd
+ */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "uniq_key_splay_tree.h"
+
+#include "hash.h"
+#include "ra.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "osfunc.h"
+#include "allocmem.h"
+#include "lock.h"
+#include "pvr_intrinsics.h"
+
+/* The initial, and minimum size of the live address -> boundary tag structure
+ * hash table. The value 64 is a fairly arbitrary choice. The hash table
+ * resizes on demand so the value chosen is not critical.
+ */
+#define MINIMUM_HASH_SIZE (64)
+
+/* #define RA_VALIDATE */
+
+#if defined(__KLOCWORK__)
+	/* Make sure Klocwork analyses all the code (including the debug one) */
+	#if !defined(RA_VALIDATE)
+		#define RA_VALIDATE
+	#endif
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_ASSERT) || !defined(RA_VALIDATE)
+/* Disable the asserts unless explicitly told otherwise.
+ * They slow the driver too much for other people
+ */
+
+#undef PVR_ASSERT
+/* Use a macro that really do not do anything when compiling in release
+ * mode!
+ */
+#define PVR_ASSERT(x)
+#endif
+
+/* boundary tags, used to describe a resource segment */
+struct _BT_
+{
+	enum bt_type
+	{
+		btt_free,		/* free resource segment */
+		btt_live		/* allocated resource segment */
+	} type;
+
+	unsigned int is_leftmost;
+	unsigned int is_rightmost;
+	unsigned int free_import;
+
+	/* The base resource and extent of this segment */
+	RA_BASE_T base;
+	RA_LENGTH_T uSize;
+
+	/* doubly linked ordered list of all segments within the arena */
+	struct _BT_ *pNextSegment;
+	struct _BT_ *pPrevSegment;
+
+	/* doubly linked un-ordered list of free segments with the same flags. */
+	struct _BT_ *next_free;
+	struct _BT_ *prev_free;
+
+	/* A user reference associated with this span, user references are
+	 * currently only provided in the callback mechanism
+	 */
+	IMG_HANDLE hPriv;
+
+	/* Flags to match on this span */
+	RA_FLAGS_T uFlags;
+
+};
+typedef struct _BT_ BT;
+
+
+/* resource allocation arena */
+struct _RA_ARENA_
+{
+	/* arena name for diagnostics output */
+	IMG_CHAR name[RA_MAX_NAME_LENGTH];
+
+	/* Spans / Imports within this arena are at least quantum sized
+	 * and are a multiple of the uQuantum. This also has the effect of
+	 * aligning these Spans to the uQuantum.
+	 */
+	RA_LENGTH_T uQuantum;
+
+	/* import interface, if provided */
+	union {
+		void* pImportAlloc;
+		PFN_RA_IMPORT_ALLOC_SINGLE pImportAllocSingle;
+		PFN_RA_IMPORT_ALLOC_MULTI pImportAllocMulti;
+	};
+	enum RA_IMPORT_ALLOC_METHOD {
+		RA_IMPORT_ALLOC_METHOD_SINGLE,
+		RA_IMPORT_ALLOC_METHOD_MULTI,
+	} eImportAllocMethod;
+
+	PFN_RA_IMPORT_FREE pImportFree;
+
+	/* Arbitrary handle provided by arena owner to be passed into the
+	 * import alloc and free hooks
+	 */
+	void *pImportHandle;
+
+	IMG_PSPLAY_TREE per_flags_buckets;
+
+	/* resource segment list */
+	BT *pHeadSegment;
+
+	/* segment address to boundary tag hash table */
+	HASH_TABLE *pSegmentHash;
+
+	/* Lock for this arena */
+	POS_LOCK hLock;
+
+	/* Policies that govern the resource area */
+	RA_POLICY_T ui32PolicyFlags;
+
+	/* LockClass of this arena. This is used within lockdep to decide if a
+	 * recursive call sequence with the same lock class is allowed or not.
+	 */
+	IMG_UINT32 ui32LockClass;
+
+	/* Total Size of the Arena */
+	IMG_UINT64	ui64TotalArenaSize;
+
+	/* Size available for allocation in the arena */
+	IMG_UINT64	ui64FreeArenaSize;
+
+};
+
+static_assert(sizeof(void*) == sizeof(PFN_RA_IMPORT_ALLOC_SINGLE), "pImportAlloc == pImportAllocSingle. Import callbacks must be pointers of the same size.");
+static_assert(sizeof(void*) == sizeof(PFN_RA_IMPORT_ALLOC_MULTI), "pImportAlloc == pImportAllocMulti. Import callbacks must be pointers of the same size.");
+
+struct _RA_ARENA_ITERATOR_
+{
+	RA_ARENA *pArena;
+	BT *pCurrent;
+	IMG_BOOL bIncludeFreeSegments;
+};
+
+/* Enum for selecting behaviour of _ConvertAndFree* functions*/
+typedef enum {
+	CONVERT_AND_FREE = 0,
+	CONVERT_DONT_FREE = 1
+} RA_CONVERT_AND_FREE_BEHAVIOUR;
+
+static PVRSRV_ERROR _RA_FreeMultiUnlocked(RA_ARENA *pArena,
+                                   RA_BASE_ARRAY_T aBaseArray,
+                                   RA_BASE_ARRAY_SIZE_T uiBaseArraySize);
+static PVRSRV_ERROR
+_RA_FreeMultiUnlockedSparse(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 *puiFreeIndices,
+                             IMG_UINT32 *puiFreeCount);
+
+/*************************************************************************/ /*!
+@Function       _RequestAllocFail
+@Description    Default callback allocator used if no callback is specified,
+                always fails to allocate further resources to the arena.
+@Input          _h - callback handle
+@Input          _uSize - requested allocation size
+@Input          _uflags - allocation flags
+@Input          _uBaseAlignment - Alignment for the returned allocated base
+
+
+@Output         _psImport - The base and size of the span to insert.
+@Return         PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL, this function always fails
+                to allocate.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_RequestAllocFail(RA_PERARENA_HANDLE _h,
+                  RA_LENGTH_T _uSize,
+                  RA_FLAGS_T _uFlags,
+                  RA_LENGTH_T _uBaseAlignment,
+                  const IMG_CHAR *_pszAnnotation,
+                  RA_IMPORT *_psImport)
+{
+	PVR_UNREFERENCED_PARAMETER(_h);
+	PVR_UNREFERENCED_PARAMETER(_uSize);
+	PVR_UNREFERENCED_PARAMETER(_uFlags);
+	PVR_UNREFERENCED_PARAMETER(_uBaseAlignment);
+	PVR_UNREFERENCED_PARAMETER(_pszAnnotation);
+	PVR_UNREFERENCED_PARAMETER(_psImport);
+
+	return PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL;
+}
+
+
+#if defined(PVR_CTZLL)
+	/* Make sure to trigger an error if someone change the buckets or the bHasEltsMapping size
+	   the bHasEltsMapping is used to quickly determine the smallest bucket containing elements.
+	   therefore it must have at least as many bits has the buckets array have buckets. The RA
+	   implementation actually uses one more bit. */
+	static_assert(ARRAY_SIZE(((IMG_PSPLAY_TREE)0)->buckets)
+				  < 8 * sizeof(((IMG_PSPLAY_TREE) 0)->bHasEltsMapping),
+				  "Too many buckets for bHasEltsMapping bitmap");
+#endif
+
+
+/*************************************************************************/ /*!
+@Function       pvr_log2
+@Description    Computes the floor of the log base 2 of a unsigned integer
+@Input          n       Unsigned integer
+@Return         Floor(Log2(n))
+*/ /**************************************************************************/
+#if defined(PVR_CLZLL)
+/* make sure to trigger a problem if someone changes the RA_LENGTH_T type
+   indeed the __builtin_clzll is for unsigned long long variables.
+
+   if someone changes RA_LENGTH to unsigned long, then use __builtin_clzl
+   if it changes to unsigned int, use __builtin_clz
+
+   if it changes for something bigger than unsigned long long,
+   then revert the pvr_log2 to the classic implementation */
+static_assert(sizeof(RA_LENGTH_T) == sizeof(unsigned long long),
+			  "RA log routines not tuned for sizeof(RA_LENGTH_T)");
+
+static inline IMG_UINT32 pvr_log2(RA_LENGTH_T n)
+{
+	PVR_ASSERT(n != 0); /* Log2 is not defined on 0 */
+
+	return (8 * sizeof(RA_LENGTH_T)) - 1 - PVR_CLZLL(n);
+}
+#else
+static IMG_UINT32
+pvr_log2(RA_LENGTH_T n)
+{
+	IMG_UINT32 l = 0;
+
+	PVR_ASSERT(n != 0); /* Log2 is not defined on 0 */
+
+	n >>= 1;
+	while (n > 0)
+	{
+		n >>= 1;
+		l++;
+	}
+	return l;
+}
+#endif
+
+static INLINE void _FreeTableLimitBoundsCheck(IMG_UINT32 *uiIndex)
+{
+	if (*uiIndex >= FREE_TABLE_LIMIT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Index exceeds FREE_TABLE_LIMIT (1TB), "
+		                        "Clamping Index to FREE_TABLE_LIMIT"));
+		*uiIndex = FREE_TABLE_LIMIT - 1;
+	}
+}
+
+
+#if defined(RA_VALIDATE)
+/*************************************************************************/ /*!
+@Function       _IsInSegmentList
+@Description    Tests if a BT is in the segment list.
+@Input          pArena           The arena.
+@Input          pBT              The boundary tag to look for.
+@Return         IMG_FALSE  BT was not in the arena's segment list.
+                IMG_TRUE   BT was in the arena's segment list.
+*/ /**************************************************************************/
+static IMG_BOOL
+_IsInSegmentList(RA_ARENA *pArena, BT *pBT)
+{
+	BT* pBTScan;
+
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(pBT != NULL);
+
+	/* Walk the segment list until we see the BT pointer... */
+	pBTScan = pArena->pHeadSegment;
+	while (pBTScan != NULL  &&  pBTScan != pBT)
+	{
+		pBTScan = pBTScan->pNextSegment;
+	}
+
+	/* Test if we found it and then return */
+	return (pBTScan == pBT);
+}
+
+/*************************************************************************/ /*!
+@Function       _IsInFreeList
+@Description    Tests if a BT is in the free list.
+@Input          pArena           The arena.
+@Input          pBT              The boundary tag to look for.
+@Return         IMG_FALSE  BT was not in the arena's free list.
+                IMG_TRUE   BT was in the arena's free list.
+*/ /**************************************************************************/
+static IMG_BOOL
+_IsInFreeList(RA_ARENA *pArena, BT *pBT)
+{
+	BT* pBTScan;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(pBT != NULL);
+
+	/* Look for the free list that holds BTs of this size... */
+	uIndex = pvr_log2(pBT->uSize);
+	PVR_ASSERT(uIndex < FREE_TABLE_LIMIT);
+
+	pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != pBT->uFlags))
+	{
+		return 0;
+	}
+	else
+	{
+		pBTScan = pArena->per_flags_buckets->buckets[uIndex];
+		while (pBTScan != NULL  &&  pBTScan != pBT)
+		{
+			pBTScan = pBTScan->next_free;
+		}
+
+		/* Test if we found it and then return */
+		return (pBTScan == pBT);
+	}
+}
+
+/* is_arena_valid should only be used in debug mode.
+ * It checks that some properties an arena must have are verified
+ */
+static int is_arena_valid(struct _RA_ARENA_ *arena)
+{
+	struct _BT_ *chunk;
+#if defined(PVR_CTZLL)
+	unsigned int i;
+#endif
+
+	for (chunk = arena->pHeadSegment; chunk != NULL; chunk = chunk->pNextSegment)
+	{
+		/* if next segment is NULL, then it must be a rightmost */
+		PVR_ASSERT((chunk->pNextSegment != NULL) || (chunk->is_rightmost));
+		/* if prev segment is NULL, then it must be a leftmost */
+		PVR_ASSERT((chunk->pPrevSegment != NULL) || (chunk->is_leftmost));
+
+		if (chunk->type == btt_free)
+		{
+			/* checks the correctness of the type field */
+			PVR_ASSERT(_IsInFreeList(arena, chunk));
+
+			/* check that there can't be two consecutive free chunks.
+			   Indeed, instead of having two consecutive free chunks,
+			   there should be only one that span the size of the two. */
+			PVR_ASSERT((chunk->is_leftmost) || (chunk->pPrevSegment->type != btt_free));
+			PVR_ASSERT((chunk->is_rightmost) || (chunk->pNextSegment->type != btt_free));
+		}
+		else
+		{
+			/* checks the correctness of the type field */
+			PVR_ASSERT(!_IsInFreeList(arena, chunk));
+		}
+
+		PVR_ASSERT((chunk->is_leftmost) || (chunk->pPrevSegment->base + chunk->pPrevSegment->uSize == chunk->base));
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->base + chunk->uSize == chunk->pNextSegment->base));
+
+		/* all segments of the same imports must have the same flags ... */
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->uFlags == chunk->pNextSegment->uFlags));
+		/* ... and the same import handle */
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->hPriv == chunk->pNextSegment->hPriv));
+
+
+		/* if a free chunk spans a whole import, then it must be an 'not to free import'.
+		   Otherwise it should have been freed. */
+		PVR_ASSERT((!chunk->is_leftmost) || (!chunk->is_rightmost) || (chunk->type == btt_live) || (!chunk->free_import));
+	}
+
+#if defined(PVR_CTZLL)
+	if (arena->per_flags_buckets != NULL)
+	{
+		for (i = 0; i < FREE_TABLE_LIMIT; ++i)
+		{
+			/* verify that the bHasEltsMapping is correct for this flags bucket */
+			PVR_ASSERT(
+				((arena->per_flags_buckets->buckets[i] == NULL) &&
+				 (((arena->per_flags_buckets->bHasEltsMapping & ((IMG_ELTS_MAPPINGS) 1 << i)) == 0)))
+				||
+				((arena->per_flags_buckets->buckets[i] != NULL) &&
+				 (((arena->per_flags_buckets->bHasEltsMapping & ((IMG_ELTS_MAPPINGS) 1 << i)) != 0)))
+				);
+		}
+	}
+#endif
+
+	/* if arena was not valid, an earlier assert should have triggered */
+	return 1;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _SegmentListInsertAfter
+@Description    Insert a boundary tag into an arena segment list after a
+                specified boundary tag.
+@Input          pInsertionPoint  The insertion point.
+@Input          pBT              The boundary tag to insert.
+*/ /**************************************************************************/
+static INLINE void
+_SegmentListInsertAfter(BT *pInsertionPoint,
+                        BT *pBT)
+{
+	PVR_ASSERT(pBT != NULL);
+	PVR_ASSERT(pInsertionPoint != NULL);
+
+	pBT->pNextSegment = pInsertionPoint->pNextSegment;
+	pBT->pPrevSegment = pInsertionPoint;
+	if (pInsertionPoint->pNextSegment != NULL)
+	{
+		pInsertionPoint->pNextSegment->pPrevSegment = pBT;
+	}
+	pInsertionPoint->pNextSegment = pBT;
+}
+
+/*************************************************************************/ /*!
+@Function       _SegmentListInsert
+@Description    Insert a boundary tag into an arena segment list
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag to insert.
+*/ /**************************************************************************/
+static INLINE void
+_SegmentListInsert(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(!_IsInSegmentList(pArena, pBT));
+
+	/* insert into the segment chain */
+	pBT->pNextSegment = pArena->pHeadSegment;
+	pArena->pHeadSegment = pBT;
+	if (pBT->pNextSegment != NULL)
+	{
+		pBT->pNextSegment->pPrevSegment = pBT;
+	}
+
+	pBT->pPrevSegment = NULL;
+}
+
+/*************************************************************************/ /*!
+@Function       _SegmentListRemove
+@Description    Remove a boundary tag from an arena segment list.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag to remove.
+*/ /**************************************************************************/
+static void
+_SegmentListRemove(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(_IsInSegmentList(pArena, pBT));
+
+	if (pBT->pPrevSegment == NULL)
+		pArena->pHeadSegment = pBT->pNextSegment;
+	else
+		pBT->pPrevSegment->pNextSegment = pBT->pNextSegment;
+
+	if (pBT->pNextSegment != NULL)
+		pBT->pNextSegment->pPrevSegment = pBT->pPrevSegment;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _BuildBT
+@Description    Construct a boundary tag for a free segment.
+@Input          base     The base of the resource segment.
+@Input          uSize    The extent of the resource segment.
+@Input          uFlags   The flags to give to the boundary tag
+@Return         Boundary tag or NULL
+*/ /**************************************************************************/
+static BT *
+_BuildBT(RA_BASE_T base, RA_LENGTH_T uSize, RA_FLAGS_T uFlags)
+{
+	BT *pBT;
+
+	pBT = OSAllocZMem(sizeof(BT));
+	if (pBT == NULL)
+	{
+		return NULL;
+	}
+
+	pBT->is_leftmost = 1;
+	pBT->is_rightmost = 1;
+	/* pBT->free_import = 0; */
+	pBT->type = btt_live;
+	pBT->base = base;
+	pBT->uSize = uSize;
+	pBT->uFlags = uFlags;
+
+	return pBT;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _SegmentSplit
+@Description    Split a segment into two, maintain the arena segment list. The
+                boundary tag should not be in the free table. Neither the
+                original or the new neighbour boundary tag will be in the free
+                table.
+@Input          pBT       The boundary tag to split.
+@Input          uSize     The required segment size of boundary tag after
+                          splitting.
+@Return         New neighbour boundary tag or NULL.
+*/ /**************************************************************************/
+static BT *
+_SegmentSplit(BT *pBT, RA_LENGTH_T uSize)
+{
+	BT *pNeighbour;
+
+	pNeighbour = _BuildBT(pBT->base + uSize, pBT->uSize - uSize, pBT->uFlags);
+	if (pNeighbour == NULL)
+	{
+		return NULL;
+	}
+
+	_SegmentListInsertAfter(pBT, pNeighbour);
+
+	pNeighbour->is_leftmost = 0;
+	pNeighbour->is_rightmost = pBT->is_rightmost;
+	pNeighbour->free_import = pBT->free_import;
+	pBT->is_rightmost = 0;
+	pNeighbour->hPriv = pBT->hPriv;
+	pBT->uSize = uSize;
+	pNeighbour->uFlags = pBT->uFlags;
+
+	return pNeighbour;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeListInsert
+@Description    Insert a boundary tag into an arena free table.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag.
+*/ /**************************************************************************/
+static void
+_FreeListInsert(RA_ARENA *pArena, BT *pBT)
+{
+	IMG_UINT32 uIndex;
+	BT *pBTTemp = NULL;
+	uIndex = pvr_log2(pBT->uSize);
+
+	_FreeTableLimitBoundsCheck(&uIndex);
+
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+
+	pBT->type = btt_free;
+
+	pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+	/* the flags item in the splay tree must have been created before-hand by
+	   _InsertResource */
+	PVR_ASSERT(pArena->per_flags_buckets != NULL);
+
+	/* Handle NULL values for RELEASE builds and/or disabled ASSERT DEBUG builds */
+	if (unlikely(pArena->per_flags_buckets == NULL))
+	{
+		return;
+	}
+
+	/* Get the first node in the bucket */
+	pBTTemp = pArena->per_flags_buckets->buckets[uIndex];
+
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_ALLOC_NODE_SELECT_MASK) == RA_POLICY_ALLOC_OPTIMAL))
+	{
+		/* Add the node to the start if the bucket is empty */
+		if (NULL == pBTTemp)
+		{
+			pArena->per_flags_buckets->buckets[uIndex] = pBT;
+			pBT->next_free = NULL;
+			pBT->prev_free = NULL;
+
+		}
+		else
+		{
+			BT *pBTPrev = NULL;
+			/* Traverse the list and identify the appropriate
+			 * place based on the size of the Boundary being inserted */
+			while (pBTTemp && (pBTTemp->uSize < pBT->uSize))
+			{
+				pBTPrev = pBTTemp;
+				pBTTemp = pBTTemp->next_free;
+			}
+			/* point the new node to the first higher size element */
+			pBT->next_free = pBTTemp;
+			pBT->prev_free = pBTPrev;
+
+			if (pBTPrev)
+			{
+				/* Set the lower size element in the
+				 * chain to point new node */
+				pBTPrev->next_free = pBT;
+			}
+			else
+			{
+				/* Assign the new node to the start of the bucket
+				 * if the bucket is empty */
+				pArena->per_flags_buckets->buckets[uIndex] = pBT;
+			}
+			/* Make sure the higher size element in the chain points back
+			 * to the new node to be introduced */
+			if (pBTTemp)
+			{
+				pBTTemp->prev_free = pBT;
+			}
+		}
+	}
+	else
+	{
+		pBT->next_free =  pBTTemp;
+		if (pBT->next_free != NULL)
+		{
+			pBT->next_free->prev_free = pBT;
+		}
+		pBT->prev_free = NULL;
+		pArena->per_flags_buckets->buckets[uIndex] = pBT;
+	}
+
+#if defined(PVR_CTZLL)
+	/* tells that bucket[index] now contains elements */
+	pArena->per_flags_buckets->bHasEltsMapping |= ((IMG_ELTS_MAPPINGS) 1 << uIndex);
+#endif
+
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeListRemove
+@Description    Remove a boundary tag from an arena free table.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag.
+*/ /**************************************************************************/
+static void
+_FreeListRemove(RA_ARENA *pArena, BT *pBT)
+{
+	IMG_UINT32 uIndex;
+	uIndex = pvr_log2(pBT->uSize);
+
+	_FreeTableLimitBoundsCheck(&uIndex);
+
+	PVR_ASSERT(_IsInFreeList(pArena, pBT));
+
+	if (pBT->next_free != NULL)
+	{
+		pBT->next_free->prev_free = pBT->prev_free;
+	}
+
+	if (pBT->prev_free != NULL)
+	{
+		pBT->prev_free->next_free = pBT->next_free;
+	}
+	else
+	{
+		pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+		/* the flags item in the splay tree must have already been created
+		   (otherwise how could there be a segment with these flags */
+		PVR_ASSERT(pArena->per_flags_buckets != NULL);
+
+		/* Handle unlikely NULL values for RELEASE or ASSERT-disabled builds */
+		if (unlikely(pArena->per_flags_buckets == NULL))
+		{
+			pBT->type = btt_live;
+			return;
+		}
+
+		pArena->per_flags_buckets->buckets[uIndex] = pBT->next_free;
+#if defined(PVR_CTZLL)
+		if (pArena->per_flags_buckets->buckets[uIndex] == NULL)
+		{
+			/* there is no more elements in this bucket. Update the mapping. */
+			pArena->per_flags_buckets->bHasEltsMapping &= ~((IMG_ELTS_MAPPINGS) 1 << uIndex);
+		}
+#endif
+	}
+
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+	pBT->type = btt_live;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _InsertResource
+@Description    Add a free resource segment to an arena.
+@Input          pArena    The arena.
+@Input          base      The base of the resource segment.
+@Input          uSize     The extent of the resource segment.
+@Input          uFlags    The flags of the new resources.
+@Return         New bucket pointer
+                NULL on failure
+*/ /**************************************************************************/
+static BT *
+_InsertResource(RA_ARENA *pArena, RA_BASE_T base, RA_LENGTH_T uSize,
+                RA_FLAGS_T uFlags)
+{
+	BT *pBT;
+	PVR_ASSERT(pArena!=NULL);
+
+	pBT = _BuildBT(base, uSize, uFlags);
+
+	if (pBT != NULL)
+	{
+		IMG_PSPLAY_TREE tmp = PVRSRVInsert(pBT->uFlags, pArena->per_flags_buckets);
+		if (tmp == NULL)
+		{
+			OSFreeMem(pBT);
+			return NULL;
+		}
+
+		pArena->per_flags_buckets = tmp;
+		_SegmentListInsert(pArena, pBT);
+		_FreeListInsert(pArena, pBT);
+	}
+	return pBT;
+}
+
+/*************************************************************************/ /*!
+@Function       _InsertResourceSpan
+@Description    Add a free resource span to an arena, marked for free_import.
+@Input          pArena    The arena.
+@Input          base      The base of the resource segment.
+@Input          uSize     The extent of the resource segment.
+@Return         The boundary tag representing the free resource segment,
+                or NULL on failure.
+*/ /**************************************************************************/
+static INLINE BT *
+_InsertResourceSpan(RA_ARENA *pArena,
+                    RA_BASE_T base,
+                    RA_LENGTH_T uSize,
+                    RA_FLAGS_T uFlags)
+{
+	BT *pBT = _InsertResource(pArena, base, uSize, uFlags);
+	if (pBT != NULL)
+	{
+		pBT->free_import = 1;
+	}
+	return pBT;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _RemoveResourceSpan
+@Description    Frees a resource span from an arena, returning the imported
+                span via the callback.
+@Input          pArena     The arena.
+@Input          pBT        The boundary tag to free.
+@Return         IMG_FALSE failure - span was still in use
+                IMG_TRUE  success - span was removed and returned
+*/ /**************************************************************************/
+static INLINE IMG_BOOL
+_RemoveResourceSpan(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(pBT!=NULL);
+
+	if (pBT->free_import &&
+		pBT->is_leftmost &&
+		pBT->is_rightmost)
+	{
+		_SegmentListRemove(pArena, pBT);
+		pArena->ui64TotalArenaSize -= pBT->uSize;
+		pArena->ui64FreeArenaSize -= pBT->uSize;
+		pArena->pImportFree(pArena->pImportHandle, pBT->base, pBT->hPriv);
+		OSFreeMem(pBT);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeBT
+@Description    Free a boundary tag taking care of the segment list and the
+                boundary tag free table.
+@Input          pArena     The arena.
+@Input          pBT        The boundary tag to free.
+*/ /**************************************************************************/
+static void
+_FreeBT(RA_ARENA *pArena, BT *pBT)
+{
+	BT *pNeighbour;
+
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(pBT!=NULL);
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+
+	/* try and coalesce with left neighbour */
+	pNeighbour = pBT->pPrevSegment;
+	if ((!pBT->is_leftmost)	&& (pNeighbour->type == btt_free))
+	{
+		/* Verify list correctness */
+		PVR_ASSERT(pNeighbour->base + pNeighbour->uSize == pBT->base);
+
+		_FreeListRemove(pArena, pNeighbour);
+		_SegmentListRemove(pArena, pNeighbour);
+		pBT->base = pNeighbour->base;
+
+		pBT->uSize += pNeighbour->uSize;
+		pBT->is_leftmost = pNeighbour->is_leftmost;
+		OSFreeMem(pNeighbour);
+	}
+
+	/* try to coalesce with right neighbour */
+	pNeighbour = pBT->pNextSegment;
+	if ((!pBT->is_rightmost) && (pNeighbour->type == btt_free))
+	{
+		/* Verify list correctness */
+		PVR_ASSERT(pBT->base + pBT->uSize == pNeighbour->base);
+
+		_FreeListRemove(pArena, pNeighbour);
+		_SegmentListRemove(pArena, pNeighbour);
+		pBT->uSize += pNeighbour->uSize;
+		pBT->is_rightmost = pNeighbour->is_rightmost;
+		OSFreeMem(pNeighbour);
+	}
+
+	if (_RemoveResourceSpan(pArena, pBT) == IMG_FALSE)
+	{
+		_FreeListInsert(pArena, pBT);
+		PVR_ASSERT((!pBT->is_rightmost) || (!pBT->is_leftmost) || (!pBT->free_import));
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+}
+
+
+/*
+  This function returns the first element in a bucket that can be split
+  in a way that one of the sub-segments can meet the size and alignment
+  criteria.
+
+  The first_elt is the bucket to look into. Remember that a bucket is
+  implemented as a pointer to the first element of the linked list.
+
+  nb_max_try is used to limit the number of elements considered.
+  This is used to only consider the first nb_max_try elements in the
+  free-list. The special value ~0 is used to say unlimited i.e. consider
+  all elements in the free list
+ */
+static INLINE
+struct _BT_ *find_chunk_in_bucket(struct _BT_ * first_elt,
+                                  RA_LENGTH_T uSize,
+                                  RA_LENGTH_T uAlignment,
+                                  unsigned int nb_max_try)
+{
+	struct _BT_ *walker;
+
+	for (walker = first_elt; (walker != NULL) && (nb_max_try != 0); walker = walker->next_free)
+	{
+		const RA_BASE_T aligned_base = (uAlignment > 1) ?
+			PVR_ALIGN(walker->base, uAlignment)
+			: walker->base;
+
+		if (walker->base + walker->uSize >= aligned_base + uSize)
+		{
+			return walker;
+		}
+
+		/* 0xFFFF...FFFF is used has nb_max_try = infinity. */
+		if (nb_max_try != (unsigned int) ~0)
+		{
+			nb_max_try--;
+		}
+	}
+
+	return NULL;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FreeMultiBaseArray
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 free the region given as the array and size. This function
+ *                 should be used only when it is known that multiple Real
+ *                 bases will be freed from the array.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_FreeMultiBaseArray(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	IMG_UINT32 i;
+	for (i = 0; i < uiBaseArraySize; i++)
+	{
+		if (RA_BASE_IS_REAL(aBaseArray[i]))
+		{
+			BT *pBT;
+			pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &aBaseArray[i]);
+
+			if (pBT)
+			{
+				pArena->ui64FreeArenaSize += pBT->uSize;
+
+				PVR_ASSERT(pBT->base == aBaseArray[i]);
+				_FreeBT(pArena, pBT);
+				aBaseArray[i] = INVALID_BASE_ADDR;
+			}
+			else
+			{
+				/* Did we attempt to remove a ghost page?
+				 * Essentially the base was marked real but was actually a ghost.
+				 */
+				PVR_ASSERT(!"Attempt to free non-existing real base!");
+				return PVRSRV_ERROR_INVALID_REQUEST;
+			}
+		}
+#if defined(DEBUG)
+		else
+		{
+			aBaseArray[i] = INVALID_BASE_ADDR;
+		}
+#endif
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FreeSingleBaseArray
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 free the region given as the array and size. This function
+ *                 should be used only when it is known that a single Real
+ *                 base will be freed from the array. All Bases will be
+ *                 sanitised after the real has been freed.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from, entry 0 should be a
+ *                       Real base
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_FreeSingleBaseArray(RA_ARENA *pArena,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	BT *pBT;
+	PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[0]));
+
+	pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &aBaseArray[0]);
+
+	if (pBT)
+	{
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		PVR_ASSERT(pBT->base == aBaseArray[0]);
+		_FreeBT(pArena, pBT);
+	}
+	else
+	{
+		/* Did we attempt to remove a ghost page?
+		 * Essentially the base was marked real but was actually ghost.
+		 */
+		PVR_ASSERT(!"Attempt to free non-existing real base!");
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+
+	/* Set all entries to INVALID_BASE_ADDR */
+	OSCachedMemSet(aBaseArray, 0xFF, uiBaseArraySize * sizeof(RA_BASE_T));
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _GenerateGhostBases
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 generate Ghost bases for the allocation and size.
+ *
+ *  @Input  uiBase - The Real base to generate Ghost Bases from.
+ *  @Input  uiBaseSize - The size of the Real Base
+ *  @Input  uiChunkSize - The Base chunk size used to generate Ghost
+ *                           bases on specific boundaries.
+ *  @Input  aBaseArray - The array to add the Ghost bases to.
+ *
+ *  @Return array index of element past last Ghost base of given array.
+*/ /**************************************************************************/
+static IMG_UINT32
+_GenerateGhostBases(RA_BASE_T uiRealBase,
+                    RA_LENGTH_T uiBaseSize,
+                    RA_LENGTH_T uiChunkSize,
+                    RA_BASE_ARRAY_T aBaseArray)
+{
+	IMG_UINT32 ui32Index = 0;
+	RA_LENGTH_T uiRemaining = uiBaseSize - uiChunkSize;
+	RA_LENGTH_T uiCurrentBase = uiRealBase + uiChunkSize;
+	aBaseArray[ui32Index] = uiRealBase;
+
+	for (ui32Index = 1; uiRemaining != 0; ui32Index++)
+	{
+		aBaseArray[ui32Index] = RA_BASE_SET_GHOST_BIT(uiCurrentBase);
+		uiCurrentBase += uiChunkSize;
+		uiRemaining -= uiChunkSize;
+	}
+
+	return ui32Index;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FindRealBaseFromGhost
+ *
+ *  @Description   Given an array and an index into that array for the Ghost Base
+ *                 find the Real Base hosting the Ghost base in the RA.
+ *  @Input aBaseArray - The array the Ghost and Real base reside on.
+ *  @Input ui32GhostBaseIndex - The index into the given array for the Ghost Base.
+ *  @Output pRealBase - The Real Base hosting the Ghost base.
+ *  @Output pui32RealBaseIndex -  The index of the Real Base found in the array.
+ *
+ *  @Return None.
+*/ /**************************************************************************/
+static void
+_FindRealBaseFromGhost(RA_BASE_ARRAY_T aBaseArray,
+                       IMG_UINT32 ui32GhostBaseIndex,
+                       RA_BASE_T *pRealBase,
+                       IMG_UINT32 *pui32RealBaseIndex)
+{
+	IMG_UINT32 ui32Index = ui32GhostBaseIndex;
+
+	PVR_ASSERT(RA_BASE_IS_GHOST(aBaseArray[ui32GhostBaseIndex]));
+
+	while (ui32Index != 0 &&
+	       RA_BASE_IS_GHOST(aBaseArray[ui32Index]))
+	{
+		ui32Index--;
+	}
+
+	*pRealBase = aBaseArray[ui32Index];
+	*pui32RealBaseIndex = ui32Index;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertGhostBaseToReal
+ *
+ *  @Description   Convert the given Ghost Base to a Real Base in the
+ *                 RA. This is mainly used in free paths so we can be
+ *                 agile with memory regions.
+ *  @Input pArena - The RA Arena to convert the base on.
+ *  @Input aBaseArray - The Base array to convert the base on.
+ *  @Input uiRealBase - The Base hosting the Ghost base to convert.
+ *  @Input ui32RealBaseArrayIndex - The index in the array of the Real Base.
+ *  @Input ui32GhostBaseArrayIndex - The index in the array of the Ghost Base.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost bases on.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertGhostBaseToReal(RA_ARENA *pArena,
+                        RA_BASE_ARRAY_T aBaseArray,
+                        RA_BASE_T uiRealBase,
+                        IMG_UINT32 ui32RealBaseArrayIndex,
+                        IMG_UINT32 ui32GhostBaseArrayIndex,
+                        RA_LENGTH_T uiChunkSize)
+{
+	BT *pOrigRealBT;
+	BT *pNewRealBT;
+
+	pOrigRealBT = (BT *) HASH_Retrieve_Extended(pArena->pSegmentHash, &uiRealBase);
+	pNewRealBT = _SegmentSplit(pOrigRealBT,
+	                           uiChunkSize *
+	                            (ui32GhostBaseArrayIndex - ui32RealBaseArrayIndex));
+	PVR_LOG_RETURN_IF_FALSE(pNewRealBT != NULL,
+	                        "Unable to split BT, no memory available to allocate new BT",
+	                        PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	if (!HASH_Insert_Extended(pArena->pSegmentHash, &pNewRealBT->base, (uintptr_t) pNewRealBT))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_UNABLE_TO_INSERT_HASH_VALUE, "HASH_Insert_Extended");
+	}
+
+	aBaseArray[ui32GhostBaseArrayIndex] = pNewRealBT->base;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeStartFromGhost
+ *
+ *  @Description   Given a ghost base and size, convert and free contiguous
+ *                 ghost bases. This has the effect of shrinking the size of
+ *                 the real base holding the range to free. If ghost pages
+ *                 remain after the free region, a new Real base will be
+ *                 created to host them.
+ *
+ *  @Input pArena - The RA Arena to free the Ghost Bases from.
+ *  @Input aBaseArray - The array to remove bases from
+ *  @Input uiBaseArraySize - The size of the Base array to free from.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost Bases.
+ *  @Input ui32GhostBaseIndex - The index into the array of the initial Ghost base to free
+ *  @Input ui32FreeCount - The number of Ghost bases to free from the Real base.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeStartFromGhost(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                              RA_LENGTH_T uiChunkSize,
+                              IMG_UINT32 ui32GhostBaseIndex,
+                              IMG_UINT32 ui32FreeCount,
+                              RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiRealBase;
+	IMG_UINT32 ui32RealBaseIndex;
+	IMG_UINT32 ui32FreeEndIndex;
+
+	_FindRealBaseFromGhost(aBaseArray,
+	                       ui32GhostBaseIndex,
+	                       &uiRealBase,
+	                       &ui32RealBaseIndex);
+
+	/* Make the first Ghost Base to free, real. */
+	eError = _ConvertGhostBaseToReal(pArena,
+	                                 aBaseArray,
+	                                 uiRealBase,
+	                                 ui32RealBaseIndex,
+	                                 ui32GhostBaseIndex,
+	                                 uiChunkSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+
+	/* Calculate the Base after the last to free. */
+	ui32FreeEndIndex = ui32GhostBaseIndex + ui32FreeCount;
+
+	/*
+	 * If the end of the free region is a Ghost base then we need to
+	 * make it a real base so that we can free the intended middle region.
+	 */
+	if (ui32FreeEndIndex != uiBaseArraySize &&
+	    RA_BASE_IS_GHOST(aBaseArray[ui32FreeEndIndex]))
+	{
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 aBaseArray[ui32GhostBaseIndex],
+		                                 ui32GhostBaseIndex,
+		                                 ui32FreeEndIndex,
+		                                 uiChunkSize);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	if (eBehaviour == CONVERT_AND_FREE)
+	{
+		/* Free the region calculated */
+		eError = _FreeSingleBaseArray(pArena,
+		                              &aBaseArray[ui32GhostBaseIndex],
+		                              ui32FreeCount);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeStartFromReal
+ *
+ *  @Description   Used in the case that we want to keep some indices that are ghost pages
+ *                 but the indices to free start with the real base. In this case we can
+ *                 convert the keep point to a real base, then free the original real base
+ *                 along with all ghost bases prior to the new real.
+ *
+ *  @Input pArena - The RA Arena to free the bases from.
+ *  @Input aBaseArray - The Base array to free from.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost bases.
+ *  @Input uiGhostBaseIndex - The index into the array of the Ghost base to convert.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeStartFromReal(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 uiRealBaseIndex,
+                             IMG_UINT32 uiGhostBaseIndex,
+                             RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiRealBase = aBaseArray[uiRealBaseIndex];
+
+	eError = _ConvertGhostBaseToReal(pArena,
+	                                 aBaseArray,
+	                                 uiRealBase,
+	                                 uiRealBaseIndex,
+	                                 uiGhostBaseIndex,
+	                                 uiChunkSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+
+	if (eBehaviour == CONVERT_AND_FREE)
+	{
+		eError = _FreeSingleBaseArray(pArena,
+		                              &aBaseArray[uiRealBaseIndex],
+		                              uiGhostBaseIndex - uiRealBaseIndex);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_FreeBaseArray");
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeBaseArraySlice
+ *
+ *  @Description   Convert and maybe free Bases in an Array Slice.
+ *                 This function might convert some ghosts into real bases.
+ *                 This function assumes that the slice is within a single Real base alloc.
+ *                 i.e the uiFreeStartIndex and uiFreeCount remain fully within a single real
+ *                 base alloc and do not cross into another Real base region.
+ *
+ *  @Input pArena - The RA Arena to free bases from.
+ *  @Input aBaseArray - The Base array to free from.
+ *  @Input uiBaseArraySize - The size of the Base array to free from.
+ *  @Input uiChunkSize - The base chunk size used to generate the Ghost bases.
+ *  @Input uiFreeStartIndex - The index in the array to start freeing from
+ *  @Input uiFreeCount - The number of bases to free.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeBaseArraySlice(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 uiFreeStartIndex,
+                             IMG_UINT32 uiFreeCount,
+                             RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	/*3 cases:
+	 * Key: () = Region to Free
+	 *      [R] = Newly Real
+	 *      R = Real Base
+	 *      G = Ghost Base
+	 * 1. We free the whole Realbase (inc all Ghost bases)
+	 *    e.g. (RGGGGG)
+	 *    e.g. RGGG(R)RGG
+	 * 2 .We free the Real base but not all the Ghost bases meaning the first
+	 *    ghost base after the last freed will become a real base.
+	 *    e.g. RGGGG(RGGGG)[R]GGG
+	 *    e.g. (RGGGG)[R]GGGG
+	 * 3. We free some ghost bases from the real base
+	 *    e.g. RGGG(GGG)
+	 *    e.g. RGGG(GGG)[R]GGG
+	 *
+	 * Invalid Scenarios:
+	 * 1. RGG(GR)GGGRG
+	 * 2. RGG(GRG)GGRG
+	 * Higher levels should prevent these situations by ensuring that the free
+	 * index and count always focus on a single real base.
+	 * Scenario 1 & 2, correctly handled, would be a case 3. followed by a case 2.
+	 */
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(uiBaseArraySize >= uiFreeStartIndex &&
+	                        uiBaseArraySize >= uiFreeStartIndex + (uiFreeCount - 1),
+	                        "Free Index given out of array bounds",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Find which case we have */
+
+	/* Case 1 or 2 */
+	if (RA_BASE_IS_REAL(aBaseArray[uiFreeStartIndex]))
+	{
+		/* Case 1 */
+		if (uiFreeStartIndex + uiFreeCount == uiBaseArraySize ||
+		    RA_BASE_IS_REAL(aBaseArray[uiFreeStartIndex + uiFreeCount]) ||
+		    RA_BASE_IS_INVALID(aBaseArray[uiFreeStartIndex + uiFreeCount]))
+		{
+			if (eBehaviour == CONVERT_AND_FREE)
+			{
+				eError = _FreeSingleBaseArray(pArena,
+				                              &aBaseArray[uiFreeStartIndex],
+				                              uiFreeCount);
+				PVR_LOG_RETURN_IF_ERROR(eError, "_FreeBaseArray");
+			}
+		}
+		/* Case 2*/
+		else
+		{
+			eError = _ConvertAndFreeStartFromReal(pArena,
+			                                      aBaseArray,
+			                                      uiChunkSize,
+			                                      uiFreeStartIndex,
+			                                      uiFreeStartIndex + uiFreeCount,
+			                                      eBehaviour);
+			PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeStartFromReal");
+		}
+	}
+	/* Case 3 */
+	else if (RA_BASE_IS_GHOST(aBaseArray[uiFreeStartIndex]))
+	{
+		eError = _ConvertAndFreeStartFromGhost(pArena,
+		                                       aBaseArray,
+		                                       uiBaseArraySize,
+		                                       uiChunkSize,
+		                                       uiFreeStartIndex,
+		                                       uiFreeCount,
+		                                       eBehaviour);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeStartFromGhost");
+	}
+	/* Attempt to free an invalid base, this could be a duplicated
+	 * value in the free sparse index array */
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Attempt to free already free base Index %u", uiFreeStartIndex));
+		PVR_ASSERT(!"Attempted double free.");
+		return PVRSRV_ERROR_RA_FREE_INVALID_CHUNK;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _AllocAlignSplit
+@Description    Given a valid BT, trim the start and end of the BT according
+                to alignment and size requirements. Also add the resulting
+                BT to the live hash table.
+@Input          pArena       The arena.
+@Input          pBT          The BT to trim and add to live hash table
+@Input          uSize        The requested allocation size.
+@Input          uAlignment   The alignment requirements of the allocation
+                             Required uAlignment, or 0.
+                             Must be a power of 2 if not 0
+@Output         pBase        Allocated, corrected, resource base
+                             (non-optional, must not be NULL)
+@Output         phPriv       The user references associated with
+                             the imported segment. (optional)
+@Return         IMG_FALSE failure
+                IMG_TRUE success
+*/ /**************************************************************************/
+static IMG_BOOL
+_AllocAlignSplit(RA_ARENA *pArena,
+                 BT *pBT,
+                 RA_LENGTH_T uSize,
+                 RA_LENGTH_T uAlignment,
+                 RA_BASE_T *pBase,
+                 RA_PERISPAN_HANDLE *phPriv)
+{
+	RA_BASE_T aligned_base;
+
+	aligned_base = (uAlignment > 1) ? PVR_ALIGN(pBT->base, uAlignment) : pBT->base;
+
+	_FreeListRemove(pArena, pBT);
+
+	if ((pArena->ui32PolicyFlags & RA_POLICY_NO_SPLIT_MASK) == RA_POLICY_NO_SPLIT)
+	{
+		goto nosplit;
+	}
+
+	/* with uAlignment we might need to discard the front of this segment */
+	if (aligned_base > pBT->base)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, (RA_LENGTH_T)(aligned_base - pBT->base));
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Front split failed", __func__));
+			/* Put pBT back in the list */
+			_FreeListInsert(pArena, pBT);
+			return IMG_FALSE;
+		}
+
+		_FreeListInsert(pArena, pBT);
+		pBT = pNeighbour;
+	}
+
+	/* the segment might be too big, if so, discard the back of the segment */
+	if (pBT->uSize > uSize)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, uSize);
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Back split failed", __func__));
+			/* Put pBT back in the list */
+			_FreeListInsert(pArena, pBT);
+			return IMG_FALSE;
+		}
+
+		_FreeListInsert(pArena, pNeighbour);
+	}
+nosplit:
+	pBT->type = btt_live;
+
+	if (!HASH_Insert_Extended(pArena->pSegmentHash, &aligned_base, (uintptr_t)pBT))
+	{
+		_FreeBT(pArena, pBT);
+		return IMG_FALSE;
+	}
+
+	if (phPriv != NULL)
+		*phPriv = pBT->hPriv;
+
+	*pBase = aligned_base;
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptAllocAligned
+@Description    Attempt an allocation from an arena.
+@Input          pArena       The arena.
+@Input          uSize        The requested allocation size.
+@Input          uFlags       Allocation flags
+@Output         phPriv       The user references associated with
+                             the imported segment. (optional)
+@Input          uAlignment   Required uAlignment, or 0.
+                             Must be a power of 2 if not 0
+@Output         base         Allocated resource base (non-optional, must not
+                             be NULL)
+@Return         IMG_FALSE failure
+                IMG_TRUE success
+*/ /**************************************************************************/
+static IMG_BOOL
+_AttemptAllocAligned(RA_ARENA *pArena,
+                     RA_LENGTH_T uSize,
+                     RA_FLAGS_T uFlags,
+                     RA_LENGTH_T uAlignment,
+                     RA_BASE_T *base,
+                     RA_PERISPAN_HANDLE *phPriv) /* this is the "per-import" private data */
+{
+
+	IMG_UINT32 index_low;
+	IMG_UINT32 index_high;
+	IMG_UINT32 i;
+	struct _BT_ *pBT = NULL;
+
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(base != NULL);
+
+	pArena->per_flags_buckets = PVRSRVSplay(uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != uFlags))
+	{
+		/* no chunks with these flags. */
+		return IMG_FALSE;
+	}
+
+	index_low = pvr_log2(uSize);
+	if (uAlignment)
+	{
+		index_high = pvr_log2(uSize + uAlignment - 1);
+	}
+	else
+	{
+		index_high = index_low;
+	}
+
+	_FreeTableLimitBoundsCheck(&index_high);
+	_FreeTableLimitBoundsCheck(&index_low);
+
+	PVR_ASSERT(index_low <= index_high);
+
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_BUCKET_MASK) == RA_POLICY_BUCKET_BEST_FIT))
+	{
+		/* This policy ensures the selection of the first lowest size bucket that
+		 * satisfies the request size is selected */
+#if defined(PVR_CTZLL)
+		i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_low )) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+		i = index_low;
+#endif
+		for ( ; (i < FREE_TABLE_LIMIT) && (pBT == NULL); ++i)
+		{
+			if (pArena->per_flags_buckets->buckets[i])
+			{
+				pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, (unsigned int) ~0);
+			}
+		}
+	}
+	else
+	{
+#if defined(PVR_CTZLL)
+		i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_high + 1)) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+		for (i = index_high + 1; (i < FREE_TABLE_LIMIT) && (pArena->per_flags_buckets->buckets[i] == NULL); ++i)
+		{
+		}
+#endif
+		PVR_ASSERT(i <= FREE_TABLE_LIMIT);
+
+		if (i != FREE_TABLE_LIMIT)
+		{
+			/* since we start at index_high + 1, we are guaranteed to exit */
+			pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, 1);
+		}
+		else
+		{
+			for (i = index_high; (i != index_low - 1) && (pBT == NULL); --i)
+			{
+				pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, (unsigned int) ~0);
+			}
+		}
+	}
+
+	if (pBT == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	return _AllocAlignSplit(pArena, pBT, uSize, uAlignment, base, phPriv);
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptAllocAlignedAssured
+@Description    Attempt an allocation from an arena. If the arena allows
+                non-contiguous allocations, the allocation is guaranteed
+                given there is enough memory to satisfy the full allocation.
+@Input          pArena              The arena.
+@Input          uSize               The requested allocation size.
+@Input          uLog2MinContigSize  The Log2 minimum contiguity of the bases returned.
+@Input          uFlags              Allocation flags
+@Input          uAlignment          Required uAlignment, or 0.
+                                    Must be a power of 2 if not 0
+@Input          aBaseArray          Array to allocate bases to.
+@Input          bSparseAlloc        Is the allocation we are making sparse.
+@Output         bPhysContig         Is the allocation we made physically contiguous
+                                    or did we use the scoop logic
+@Return         Success: PVRSRV_OK
+                Fail:    PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptAllocAlignedAssured(RA_ARENA *pArena,
+                            RA_LENGTH_T uSize,
+                            IMG_UINT32 uLog2MinContigSize,
+                            RA_FLAGS_T uFlags,
+                            RA_LENGTH_T uAlignment,
+                            RA_BASE_ARRAY_T aBaseArray,
+                            IMG_BOOL bSparseAlloc,
+                            IMG_BOOL *bPhysContig)
+{
+	IMG_UINT32 index_low;  /* log2 Lowest contiguity required */
+	IMG_UINT32 index_high; /* log2 Size of full alloc */
+	IMG_UINT32 i;
+	struct _BT_ *pBT = NULL;
+	RA_PERISPAN_HANDLE phPriv;
+	RA_LENGTH_T uiRemaining = uSize;
+	RA_BASE_T uiBase;
+	IMG_UINT32 uiCurrentArrayIndex = 0;
+
+	PVR_ASSERT(pArena != NULL);
+
+	pArena->per_flags_buckets = PVRSRVSplay(uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != uFlags))
+	{
+		/* no chunks with these flags. */
+		return PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS;
+	}
+
+	if (pArena->ui64FreeArenaSize < uSize)
+	{
+		/* Not enough memory to accommodate kick back for a chance to import more */
+		return PVRSRV_ERROR_RA_OUT_OF_RESOURCE;
+	}
+
+	if (uLog2MinContigSize && uAlignment)
+	{
+		index_low = uLog2MinContigSize;
+		index_high = pvr_log2(uSize);
+	}
+	else if (uLog2MinContigSize)
+	{
+		index_low = uLog2MinContigSize;
+		index_high = pvr_log2(uSize);
+	}
+	else if (uAlignment)
+	{
+		index_low = 0;
+		index_high = pvr_log2(uSize + uAlignment - 1);
+	}
+	else
+	{
+		index_low = 0;
+		index_high = pvr_log2(uSize);
+	}
+
+	PVR_ASSERT(index_low < FREE_TABLE_LIMIT);
+	PVR_ASSERT(index_high < FREE_TABLE_LIMIT);
+	PVR_ASSERT(index_low <= index_high);
+
+	/* Start at index_high + 1 as then we can check all buckets larger than the desired alloc
+	 * If we don't find one larger then we could still find one of requested size in index_high and
+	 * shortcut the non-contiguous allocation path. We check index_high + 1 first as it is
+	 * guaranteed to have a free region of the requested size if the bucket has entries. Whereas
+	 * index_high is not guaranteed to have an allocation that meets the size requested due to it
+	 * representing all free regions of size 2^bucket index to 2^bucket index +1. e.g we could have
+	 * a request for 19*4k Pages which would be represented by bucket 16, bucket 16 represents free
+	 * entries from 16*4k pages to 31*4k Pages in size, if this bucket only had free entries of
+	 * 17*4k pages the search would fail, hence not guaranteed at index_high.
+	 */
+#if defined(PVR_CTZLL)
+	i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_high + 1)) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+	for (i = index_high + 1; (i < FREE_TABLE_LIMIT) && (pArena->per_flags_buckets->buckets[i] == NULL); i++)
+	{
+	}
+#endif
+
+	PVR_ASSERT(i <= FREE_TABLE_LIMIT);
+
+	if (i != FREE_TABLE_LIMIT)
+	{
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, 1);
+	}
+	else
+	{
+		/* In this case we have searched all buckets index_high + 1 to FREE_TABLE_LIMIT and not found an
+		 * available bucket with the required allocation size.
+		 * Because we haven't found an allocation of the requested size in index_high + 1 there is still a chance
+		 * that we can find an allocation of correct size in index_high, when index_high references the bucket
+		 * containing the largest free chunks in the RA Arena. i.e All buckets > index_high == NULL.
+		 * We do a final search in that bucket here before we attempt to scoop memory or return NULL.
+		 */
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[index_high], uSize, uAlignment, 1);
+	}
+
+	/* We managed to find a contiguous allocation block of sufficient size */
+	if (pBT != NULL)
+	{
+		IMG_BOOL bResult;
+		bResult =  _AllocAlignSplit(pArena, pBT, uSize, uAlignment, &uiBase, &phPriv);
+		if (bResult)
+		{
+			if (!bSparseAlloc)
+			{
+				aBaseArray[0] = uiBase;
+			}
+			else
+			{
+				_GenerateGhostBases(uiBase, uSize, 1ULL << uLog2MinContigSize, aBaseArray);
+			}
+		}
+		else
+		{
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+		*bPhysContig = IMG_TRUE;
+
+		return PVRSRV_OK;
+	}
+
+	/*
+	 * If this arena doesn't have the non-contiguous allocation functionality enabled, then
+	 * don't attempt to scoop for non physically contiguous allocations. Sparse allocations
+	 * are still able to use the scoop functionality as they map in a chunk at a time in the
+	 * worst case.
+	 */
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == 0) &&
+	    !bSparseAlloc)
+	{
+		return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+	}
+
+	/* Attempt to Scoop memory from non-contiguous blocks */
+	for (i = index_high; i >= index_low && uiRemaining != 0; i--)
+	{
+		/* While we have chunks of at least our contig size in the bucket to use */
+		for (
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], 1ULL << uLog2MinContigSize, uAlignment,(unsigned int) ~0);
+		pBT != NULL && uiRemaining != 0;
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], 1ULL << uLog2MinContigSize, uAlignment,(unsigned int) ~0))//~0 Try all elements in bucket
+		{
+			/* Grab largest chunk possible that is a multiple of our min contiguity size
+			 * N.B: C always rounds towards 0 so this effectively floors for us */
+			IMG_BOOL bResult;
+			RA_BASE_T uiAlignedBase =
+			(uAlignment > 1) ? PVR_ALIGN(pBT->base, uAlignment) : pBT->base;
+			RA_LENGTH_T uiMaxSizeAvailable = (pBT->uSize - (uiAlignedBase - pBT->base));
+			RA_LENGTH_T uiMaxMultipleOfContig = (uiMaxSizeAvailable >> uLog2MinContigSize) << uLog2MinContigSize;
+
+			/*
+			 * If the size of the BT is larger than the remaining memory to allocate
+			 * then just allocate what we need. The rest will be trimmed and put back
+			 * into the pool in _AllocAlignSplit
+			 */
+			if (uiMaxMultipleOfContig > uiRemaining)
+			{
+				uiMaxMultipleOfContig = uiRemaining;
+			}
+
+			bResult = _AllocAlignSplit(pArena, pBT, uiMaxMultipleOfContig, uAlignment, &uiBase, &phPriv);
+			if (!bResult)
+			{
+				/* Something went wrong with splitting or adding to hash,
+				 * We can try find another chunk, although this should
+				 * never occur.
+				 */
+				PVR_ASSERT(!"_AllocAlignSplit issue.");
+				continue;
+			}
+
+			uiRemaining -= uiMaxMultipleOfContig;
+
+			uiCurrentArrayIndex += _GenerateGhostBases(uiBase,
+			                                           uiMaxMultipleOfContig,
+			                                           1ULL << uLog2MinContigSize,
+			                                           &aBaseArray[uiCurrentArrayIndex]);
+		}
+	}
+
+	/* If we didn't manage to scoop enough memory then we need to unwind the allocations we just made */
+	if (uiRemaining != 0)
+	{
+		goto error_unwind;
+	}
+	*bPhysContig = IMG_FALSE;
+
+	return PVRSRV_OK;
+
+error_unwind:
+	_RA_FreeMultiUnlocked(pArena,
+	                       aBaseArray,
+	                       uiCurrentArrayIndex);
+	return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+}
+
+/* Used as an inout parameter to reduce required memory allocations.
+ * The import callbacks return one or more RA_IMPORT which is then converted into a BT*.
+ * Once a RA_IMPORT is converted to a BT*, it is no longer needed. Therefore,
+ * the space taken up by the RA_IMPORT can be reused. */
+typedef union {
+	RA_IMPORT import;
+	BT *pBT;
+} RA_IMPORT_INTERNAL;
+
+
+static_assert(sizeof(RA_IMPORT_INTERNAL) == sizeof(RA_IMPORT),
+              "An RA_IMPORT[] array must be equivalent to an RA_IMPORT_INTERNAL[] array. "
+              "This allows RA_IMPORT[] returned by callbacks to be reused internally "
+              "as BT*[] without leaking scope to the user API.");
+
+static_assert(offsetof(RA_IMPORT_INTERNAL, pBT) == 0,
+              "The pBT member must be at offset 0 to match a BT*[].");
+
+/* Support importing 16 spans at once before allocating more memory.
+ * Almost all imports will only require a single span.
+ * However, if importing multiple spans, shouldn't always require memory allocations.
+ * Nominal use of multi imports shouldn't require many spans. */
+#define RA_DEFAULT_IMPORTS_BUFFER_SIZE (16 * (sizeof(RA_IMPORT) / sizeof(BT*)))
+
+static_assert((RA_DEFAULT_IMPORTS_BUFFER_SIZE * sizeof(BT*)) >= sizeof(RA_IMPORT),
+              "The default must be able to hold at least a single RA_IMPORT.");
+
+typedef struct {
+	BT **apsBTs;
+	IMG_UINT32 uiBTCount;
+} BT_PTR_ARRAY;
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAllocSingle
+@Description    Attempt to import more memory and create a new span.
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form a new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation.
+@Output         psImportInternal  The pBT member must be set to the new span.
+                                  Unless an error is returned.
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAllocSingle(RA_ARENA *pArena,
+                              RA_LENGTH_T uRequestSize,
+                              RA_FLAGS_T uImportFlags,
+                              RA_LENGTH_T uAlignment,
+                              const IMG_CHAR *pszAnnotation,
+                              RA_IMPORT_INTERNAL *psImportInternal)
+{
+	PVRSRV_ERROR eError;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+	RA_IMPORT *psImport = &psImportInternal->import;
+	BT *pBT;
+
+	eError = pArena->pImportAllocSingle(pArena->pImportHandle,
+	                                    uRequestSize, uImportFlags,
+	                                    uAlignment,
+	                                    pszAnnotation,
+	                                    psImport);
+	if (PVRSRV_OK != eError)
+	{
+		return eError;
+	}
+
+	/* If we successfully import more resource, create a span to
+	 * represent it else free the resource we imported.
+	 */
+	pBT = _InsertResourceSpan(pArena, psImport->base, psImport->uSize, uFlags);
+	if (pBT == NULL)
+	{
+		/* insufficient resources to insert the newly acquired span,
+		   so free it back again */
+		pArena->pImportFree(pArena->pImportHandle, psImport->base, psImport->hPriv);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+		        "size=0x%llx failed!", __func__, pArena->name,
+		        (unsigned long long)uRequestSize));
+		/* RA_Dump (arena); */
+
+		return PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	}
+
+	pBT->hPriv = psImport->hPriv;
+	/* Replace the RA_IMPORT with a BT*.
+	 * To free an import we only need the BT so we can reuse the elements slot in ppsImports. */
+	psImportInternal->pBT = pBT;
+
+	pArena->ui64TotalArenaSize += pBT->uSize;
+	pArena->ui64FreeArenaSize += pBT->uSize;
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAllocMulti
+@Description    Attempt to import more memory and create new span(s).
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form at least one new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation.
+@Inout          puiBTCount        The number of elements in *ppsImports. The
+                                  count will be modified to reflect how many
+                                  BT*s have been created.
+@Inout          ppsImports        Pointer to an array of RA_IMPORT_INTERNAL.
+                                  If the must be array resized, this will point
+                                  to the new array. It will be freed higher up
+                                  the callstack if necessary. If successful,
+                                  the pBT member must be set to a new span for
+                                  every element in the array of length puiBTCount.
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAllocMulti(RA_ARENA *pArena,
+                             RA_LENGTH_T uRequestSize,
+                             RA_FLAGS_T uImportFlags,
+                             RA_LENGTH_T uAlignment,
+                             const IMG_CHAR *pszAnnotation,
+                             IMG_UINT32 *puiImportCount,
+                             RA_IMPORT_INTERNAL **ppsImports)
+{
+	PVRSRV_ERROR eError;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+	IMG_UINT32 i;
+
+	eError = pArena->pImportAllocMulti(pArena->pImportHandle,
+	                                   uRequestSize, uImportFlags,
+	                                   uAlignment,
+	                                   pszAnnotation,
+	                                   puiImportCount,
+	                                   (RA_IMPORT**) ppsImports);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* An error should be returned by the callback if the count == 0. */
+	PVR_ASSERT(*puiImportCount >= 1);
+	/* An error should be returned by the callback if *ppsImports == NULL. */
+	PVR_ASSERT(*ppsImports != NULL);
+
+	/* If we successfully import more resource, create spans to represent it. */
+	for (i = 0; i < *puiImportCount; i++)
+	{
+		BT *pBT;
+		RA_IMPORT *psImport = &(*ppsImports)[i].import;
+
+		pBT = _InsertResourceSpan(pArena, psImport->base, psImport->uSize, uFlags);
+		PVR_GOTO_IF_FALSE(pBT != NULL, release_bts);
+
+		pBT->hPriv = psImport->hPriv;
+		pArena->ui64TotalArenaSize += pBT->uSize;
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		/* Replace the RA_IMPORT with a BT*.
+		 * To free an import we only need the BT so we can reuse the elements slot in ppsImports. */
+		(*ppsImports)[i].pBT = pBT;
+	}
+
+	return PVRSRV_OK;
+
+release_bts:
+	{
+		/* Free the just imported span that failed to allocate a BT. */
+		IMG_UINT32 j;
+		RA_IMPORT *psImport = &(*ppsImports)[i].import;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+		        "request=0x%llx size=0x%llx failed!", __func__, pArena->name,
+		        (unsigned long long)uRequestSize, (unsigned long long)psImport->uSize));
+
+		/* Insufficient resources to insert the newly acquired span so free it back again. */
+		pArena->pImportFree(pArena->pImportHandle, psImport->base, psImport->hPriv);
+
+		/* Remove the previously inserted BTs. */
+		for (j = 0; j < i; j++)
+		{
+			BT *pBT = (*ppsImports)[j].pBT;
+			pArena->ui64TotalArenaSize -= pBT->uSize;
+			pArena->ui64FreeArenaSize -= pBT->uSize;
+			_FreeBT(pArena, pBT);
+		}
+
+		return PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAlloc
+@Description    Attempt to import more memory and create new span(s).
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form a new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportMultiplier Import x-times more for future requests if
+                                  we have to import new memory.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation
+@Inout          psBTArray         A container to a BT*[] and it's length.
+                                  If the array is not large enough, a new array
+                                  will be returned and must be freed with OSFreeMem
+                                  regardless of the return error code.
+                                  On error, the array may be NULL!
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAlloc(RA_ARENA *pArena,
+                        RA_LENGTH_T uRequestSize,
+                        IMG_UINT8 uImportMultiplier,
+                        RA_FLAGS_T uImportFlags,
+                        RA_LENGTH_T uAlignment,
+                        const IMG_CHAR *pszAnnotation,
+                        BT_PTR_ARRAY *psBTArray)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	RA_IMPORT_INTERNAL *psImports;
+
+	/* Interpret the BT*[] as a RA_IMPORT_INTERNAL[].
+	 * Any extra space is ignored.
+	 * Example:
+	 * |---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|
+	 * |----RA_IMPORT_INTERNAL----|----RA_IMPORT_INTERNAL----|........|
+	 *
+	 * It must be able to hold at least 1 RA_IMPORT_INTERNAL.
+	 * This should never fail if RA_DEFAULT_IMPORTS_BUFFER_SIZE is used. */
+	PVR_ASSERT(psBTArray->uiBTCount * sizeof(BT*) >= sizeof(RA_IMPORT_INTERNAL));
+	PVR_ASSERT(psBTArray != NULL && psBTArray->apsBTs != NULL);
+	psImports = (RA_IMPORT_INTERNAL*) psBTArray->apsBTs;
+
+	/* apply over-allocation multiplier after all alignment adjustments */
+	uRequestSize *= uImportMultiplier;
+	/* ensure that we import according to the quanta of this arena */
+	uRequestSize = PVR_ALIGN(uRequestSize, pArena->uQuantum);
+
+	switch (pArena->eImportAllocMethod)
+	{
+	/* Exactly 1 BT will be created. */
+	case RA_IMPORT_ALLOC_METHOD_SINGLE:
+	{
+		eError = _AttemptImportSpanAllocSingle(pArena, uRequestSize, uImportFlags, uAlignment, pszAnnotation,  &psImports[0]);
+		PVR_GOTO_IF_ERROR(eError, err_);
+		psBTArray->uiBTCount = 1;
+
+		/* Reinterpret the RA_IMPORT_INTERNAL[] back into a BT*[].
+		 * We only care about psBTArray->apsBTs[0] being valid as we return a count of 1. */
+		psBTArray->apsBTs = (BT**) psImports;
+		break;
+	}
+	/* 1 or more BTs will be created. */
+	case RA_IMPORT_ALLOC_METHOD_MULTI:
+	{
+		/* The number of RA_IMPORT_INTERNAL that can be stored using the same space. */
+		IMG_UINT32 uiImportCount = (psBTArray->uiBTCount) / (sizeof(RA_IMPORT_INTERNAL) / sizeof(BT*));
+
+		eError = _AttemptImportSpanAllocMulti(pArena, uRequestSize, uImportFlags, uAlignment, pszAnnotation, &uiImportCount, &psImports);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_AttemptImportSpanAllocMulti");
+
+		/* Reinterpret the RA_IMPORT_INTERNAL[] back into a BT*[]. */
+		psBTArray->apsBTs = (BT**) psImports;
+
+		/* If a BT* is smaller than an RA_IMPORT_INTERNAL, the array of BT*s will not be tightly packed.
+		 * Therefore, it is not a valid BT*[] and the elements need to be compressed.
+		 * Example: Given the array below
+		 * |----RA_IMPORT_INTERNAL-0--|----RA_IMPORT_INTERNAL-1--|........|
+		 * |--BT*-0-|.................|--BT*-1-|..........................|
+		 * It must be packed into this:
+		 * |--BT*-0-|--BT*-1-|............................................| */
+		if (sizeof(BT*) < sizeof(RA_IMPORT_INTERNAL))
+		{
+			IMG_UINT32 i;
+			/* The first element will always be in the correct place. Skip redundant move. */
+			for (i = 1; i < uiImportCount; i++)
+			{
+				psBTArray->apsBTs[i] = psImports[i].pBT;
+			}
+		}
+		psBTArray->uiBTCount = uiImportCount;
+		break;
+	}
+	default:
+		PVR_ASSERT(!"unreachable");
+		break;
+	}
+
+	return eError;
+
+err_:
+	if ((void*) psImports != (void*) psBTArray->apsBTs && psImports != NULL)
+	{
+		OSFreeMem(psImports);
+	}
+	return eError;
+}
+
+static RA_ARENA*
+RA_CreateCommon(IMG_CHAR *name,
+                 RA_LOG2QUANTUM_T uLog2Quantum,
+                 IMG_UINT32 ui32LockClass,
+                 enum RA_IMPORT_ALLOC_METHOD eImportAllocMethod,
+                 void* imp_alloc,
+                 PFN_RA_IMPORT_FREE imp_free,
+                 RA_PERARENA_HANDLE arena_handle,
+                 RA_POLICY_T ui32PolicyFlags)
+{
+	RA_ARENA *pArena;
+	PVRSRV_ERROR eError;
+
+	if (name == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter 'name' (NULL not accepted)", __func__));
+		return NULL;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s'", __func__, name));
+
+	pArena = OSAllocMem(sizeof(*pArena));
+	if (pArena == NULL)
+	{
+		goto arena_fail;
+	}
+
+	eError = OSLockCreate(&pArena->hLock);
+	if (eError != PVRSRV_OK)
+	{
+		goto lock_fail;
+	}
+
+	pArena->pSegmentHash = HASH_Create_Extended(MINIMUM_HASH_SIZE, sizeof(RA_BASE_T), HASH_Func_Default, HASH_Key_Comp_Default);
+
+	if (pArena->pSegmentHash==NULL)
+	{
+		goto hash_fail;
+	}
+
+	if (imp_alloc == NULL) {
+		eImportAllocMethod = RA_IMPORT_ALLOC_METHOD_SINGLE;
+		imp_alloc = &_RequestAllocFail;
+	}
+
+	OSStringSafeCopy(pArena->name, name, RA_MAX_NAME_LENGTH);
+	pArena->eImportAllocMethod = eImportAllocMethod;
+	pArena->pImportAlloc = imp_alloc; /* import callback union */
+	pArena->pImportFree = imp_free;
+	pArena->pImportHandle = arena_handle;
+	pArena->pHeadSegment = NULL;
+	pArena->uQuantum = IMG_PAGE2BYTES64(uLog2Quantum);
+	pArena->per_flags_buckets = NULL;
+	pArena->ui32LockClass = ui32LockClass;
+	pArena->ui32PolicyFlags = ui32PolicyFlags;
+	pArena->ui64TotalArenaSize = 0;
+	pArena->ui64FreeArenaSize = 0;
+
+	PVR_ASSERT(is_arena_valid(pArena));
+	return pArena;
+
+hash_fail:
+	OSLockDestroy(pArena->hLock);
+lock_fail:
+	OSFreeMem(pArena);
+	/* not nulling pointer, out of scope */
+arena_fail:
+	return NULL;
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_Create(IMG_CHAR *name,
+          RA_LOG2QUANTUM_T uLog2Quantum,
+          IMG_UINT32 ui32LockClass,
+          PFN_RA_IMPORT_ALLOC_SINGLE imp_alloc,
+          PFN_RA_IMPORT_FREE imp_free,
+          RA_PERARENA_HANDLE arena_handle,
+          RA_POLICY_T ui32PolicyFlags)
+{
+	return RA_CreateCommon(name,
+	                       uLog2Quantum,
+	                       ui32LockClass,
+	                       RA_IMPORT_ALLOC_METHOD_SINGLE,
+	                       imp_alloc,
+	                       imp_free,
+	                       arena_handle,
+	                       ui32PolicyFlags
+	);
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_CreateMulti(IMG_CHAR *name,
+               RA_LOG2QUANTUM_T uLog2Quantum,
+               IMG_UINT32 ui32LockClass,
+               PFN_RA_IMPORT_ALLOC_MULTI imp_alloc,
+               PFN_RA_IMPORT_FREE imp_free,
+               RA_PERARENA_HANDLE arena_handle,
+               RA_POLICY_T ui32PolicyFlags)
+{
+	return RA_CreateCommon(name,
+	                       uLog2Quantum,
+	                       ui32LockClass,
+	                       RA_IMPORT_ALLOC_METHOD_MULTI,
+	                       imp_alloc,
+	                       imp_free,
+	                       arena_handle,
+	                       ui32PolicyFlags
+	);
+}
+
+static void _LogRegionCreation(const char *pszMemType,
+                               IMG_UINT64 ui64CpuPA,
+                               IMG_UINT64 ui64DevPA,
+                               IMG_UINT64 ui64Size)
+{
+#if !defined(DEBUG)
+	PVR_UNREFERENCED_PARAMETER(pszMemType);
+	PVR_UNREFERENCED_PARAMETER(ui64CpuPA);
+	PVR_UNREFERENCED_PARAMETER(ui64DevPA);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+#else
+	if ((ui64CpuPA != 0) && (ui64DevPA != 0) && (ui64CpuPA != ui64DevPA))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		        "Creating RA for \"%s\" memory"
+		        " - Cpu PA 0x%016" IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx
+		        " - Dev PA 0x%016" IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx,
+		        pszMemType,
+		        ui64CpuPA, ui64CpuPA + ui64Size,
+		        ui64DevPA, ui64DevPA + ui64Size));
+	}
+	else
+	{
+		__maybe_unused IMG_UINT64 ui64PA =
+			ui64CpuPA != 0 ? ui64CpuPA : ui64DevPA;
+		__maybe_unused const IMG_CHAR *pszAddrType =
+			ui64CpuPA == ui64DevPA ? "Cpu/Dev" : (ui64CpuPA != 0 ? "Cpu" : "Dev");
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+		        "Creating RA for \"%s\" memory - %s PA 0x%016"
+		        IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx,
+		        pszMemType, pszAddrType,
+		        ui64PA, ui64PA + ui64Size));
+	}
+#endif
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_Create_With_Span(IMG_CHAR *name,
+                    RA_LOG2QUANTUM_T uLog2Quantum,
+                    IMG_UINT64 ui64CpuBase,
+                    IMG_UINT64 ui64SpanDevBase,
+                    IMG_UINT64 ui64SpanSize,
+                    RA_POLICY_T ui32PolicyFlags)
+{
+	RA_ARENA *psRA;
+	IMG_BOOL bSuccess;
+
+	psRA = RA_Create(name,
+	                 uLog2Quantum,       /* Use OS page size, keeps things simple */
+	                 RA_LOCKCLASS_0,     /* This arena doesn't use any other arenas. */
+	                 NULL,               /* No Import */
+	                 NULL,               /* No free import */
+	                 NULL,               /* No import handle */
+	                 ui32PolicyFlags);   /* No restriction on import splitting */
+	PVR_LOG_GOTO_IF_FALSE(psRA != NULL, "RA_Create() failed", return_);
+
+	bSuccess = RA_Add(psRA, (RA_BASE_T) ui64SpanDevBase, (RA_LENGTH_T) ui64SpanSize, 0, NULL);
+	PVR_LOG_GOTO_IF_FALSE(bSuccess, "RA_Add() failed", cleanup_);
+
+	_LogRegionCreation(name, ui64CpuBase, ui64SpanDevBase, ui64SpanSize);
+
+	return psRA;
+
+cleanup_:
+	RA_Delete(psRA);
+return_:
+	return NULL;
+}
+
+IMG_INTERNAL void
+RA_Delete(RA_ARENA *pArena)
+{
+	IMG_UINT32 uIndex;
+	IMG_BOOL bPrintWarn = IMG_TRUE;
+
+	PVR_ASSERT(pArena != NULL);
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return;
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			  "%s: name='%s'", __func__, pArena->name));
+
+	while (pArena->pHeadSegment != NULL)
+	{
+		BT *pBT = pArena->pHeadSegment;
+
+		if (pBT->type != btt_free)
+		{
+#if !defined(DEBUG)
+			if (bPrintWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Allocations still exist in the arena that is being destroyed. Arena Name: %s", __func__, pArena->name));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: base = 0x%llx size=0x%llx", __func__,
+					  (unsigned long long)pBT->base, (unsigned long long)pBT->uSize));
+				PVR_DPF((PVR_DBG_ERROR, "%s: This warning will be issued only once for the first allocation found!", __func__));
+				bPrintWarn = IMG_FALSE;
+			}
+#else
+			if (bPrintWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Allocations still exist in the arena that is being destroyed. Arena Name: %s", __func__, pArena->name));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				bPrintWarn = IMG_FALSE;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: base = 0x%llx size=0x%llx flags=0x%llx", __func__,
+				  (unsigned long long)pBT->base, (unsigned long long)pBT->uSize,
+				  (unsigned long long)pBT->uFlags));
+#endif
+		}
+		else
+		{
+			_FreeListRemove(pArena, pBT);
+		}
+
+		_SegmentListRemove(pArena, pBT);
+		OSFreeMem(pBT);
+		/* not nulling original pointer, it has changed */
+	}
+
+	while (pArena->per_flags_buckets != NULL)
+	{
+		for (uIndex=0; uIndex<FREE_TABLE_LIMIT; uIndex++)
+		{
+			PVR_ASSERT(pArena->per_flags_buckets->buckets[uIndex] == NULL);
+		}
+
+		pArena->per_flags_buckets = PVRSRVDelete(pArena->per_flags_buckets->uiFlags, pArena->per_flags_buckets);
+	}
+
+	HASH_Delete(pArena->pSegmentHash);
+	OSLockDestroy(pArena->hLock);
+	OSFreeMem(pArena);
+	/* not nulling pointer, copy on stack */
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_Add(RA_ARENA *pArena,
+       RA_BASE_T base,
+       RA_LENGTH_T uSize,
+       RA_FLAGS_T uFlags,
+       RA_PERISPAN_HANDLE hPriv)
+{
+	struct _BT_* bt;
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(uSize != 0);
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return IMG_FALSE;
+	}
+
+	if (uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid size 0 added to arena %s", __func__, pArena->name));
+		return IMG_FALSE;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+			 "base=0x%llx, size=0x%llx", __func__, pArena->name,
+			 (unsigned long long)base, (unsigned long long)uSize));
+
+	uSize = PVR_ALIGN(uSize, pArena->uQuantum);
+	bt = _InsertResource(pArena, base, uSize, uFlags);
+	if (bt != NULL)
+	{
+		bt->hPriv = hPriv;
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64TotalArenaSize += uSize;
+	pArena->ui64FreeArenaSize += uSize;
+	OSLockRelease(pArena->hLock);
+
+	return bt != NULL;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_Alloc(RA_ARENA *pArena,
+         RA_LENGTH_T uRequestSize,
+         IMG_UINT8 uImportMultiplier,
+         RA_FLAGS_T uImportFlags,
+         RA_LENGTH_T uAlignment,
+         const IMG_CHAR *pszAnnotation,
+         RA_BASE_T *base,
+         RA_LENGTH_T *pActualSize,
+         RA_PERISPAN_HANDLE *phPriv)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bResult;
+	RA_LENGTH_T uSize = uRequestSize;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+
+	if (pArena == NULL || uImportMultiplier == 0 || uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: One of the necessary parameters is 0", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	if (pActualSize != NULL)
+	{
+		*pActualSize = uSize;
+	}
+
+	/* Must be a power of 2 or 0 */
+	PVR_ASSERT((uAlignment == 0) || (uAlignment & (uAlignment - 1)) == 0);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: arena='%s', size=0x%llx(0x%llx), "
+	        "alignment=0x%llx", __func__, pArena->name,
+	        (unsigned long long)uSize, (unsigned long long)uRequestSize,
+	        (unsigned long long)uAlignment));
+
+	/* if allocation failed then we might have an import source which
+	   can provide more resource, else we will have to fail the
+	   allocation to the caller. */
+	bResult = _AttemptAllocAligned(pArena, uSize, uFlags, uAlignment, base, phPriv);
+	if (!bResult)
+	{
+		BT *apsBTsStatic[RA_DEFAULT_IMPORTS_BUFFER_SIZE];
+		BT_PTR_ARRAY sBTs = {
+			.uiBTCount = ARRAY_SIZE(apsBTsStatic),
+			.apsBTs = apsBTsStatic,
+		};
+
+		eError = _AttemptImportSpanAlloc(pArena,
+		                                 uSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 uAlignment,
+		                                 pszAnnotation,
+		                                 &sBTs);
+		if (eError != PVRSRV_OK)
+		{
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			OSLockRelease(pArena->hLock);
+			return eError;
+		}
+
+		bResult = _AttemptAllocAligned(pArena, uSize, uFlags, uAlignment, base, phPriv);
+		if (!bResult)
+		{
+			IMG_UINT32 i;
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: name='%s' second alloc failed!",
+			        __func__, pArena->name));
+
+			/*
+			  On failure of _AttemptAllocAligned() depending on the exact point
+			  of failure, the imported segment may have been used and freed, or
+			  left untouched. If the later, we need to return it.
+			*/
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				_FreeBT(pArena, sBTs.apsBTs[i]);
+			}
+
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			OSLockRelease(pArena->hLock);
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+		else
+		{
+			IMG_UINT32 i;
+			IMG_BOOL bBaseInNewSpans = IMG_FALSE;
+
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				BT *pBT = sBTs.apsBTs[i];
+				/* Check if the new allocation was in a span we just added... */
+				if (*base >= pBT->base && *base < (pBT->base + pBT->uSize))
+				{
+					bBaseInNewSpans = IMG_TRUE;
+					break;
+				}
+			}
+			if (!bBaseInNewSpans)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: name='%s' alloc did not occur in the imported span!",
+				         __func__, pArena->name));
+				/* Remove the imported spans which should not be in use (if it is then that is okay).
+				 * No spans should exist and not be used unless added through the import multiplier. */
+				for (i = 0; i < sBTs.uiBTCount; i++)
+				{
+					_FreeBT(pArena, sBTs.apsBTs[i]);
+				}
+			}
+		}
+		if (sBTs.apsBTs != apsBTsStatic)
+		{
+			OSFreeMem(sBTs.apsBTs);
+		}
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', size=0x%llx, "
+	        "*base=0x%llx = %d", __func__, pArena->name, (unsigned long long)uSize,
+	        (unsigned long long)*base, bResult));
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+	OSLockRelease(pArena->hLock);
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_RA_AllocMultiUnlocked(RA_ARENA *pArena,
+                       RA_LENGTH_T uRequestSize,
+                       IMG_UINT32 uiLog2ChunkSize,
+                       IMG_UINT8 uImportMultiplier,
+                       RA_FLAGS_T uImportFlags,
+                       const IMG_CHAR *pszAnnotation,
+                       RA_BASE_ARRAY_T aBaseArray,
+                       RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                       IMG_BOOL bSparseAlloc,
+                       IMG_BOOL *bPhysContig)
+{
+	PVRSRV_ERROR eError;
+	RA_LENGTH_T uSize = uRequestSize;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+
+	PVR_LOG_RETURN_IF_FALSE(pArena != NULL && uImportMultiplier != 0 && uSize != 0,
+	                        "One of the necessary parameters is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT((uRequestSize & (IMG_PAGE2BYTES64(uiLog2ChunkSize) - 1)) == 0);
+	PVR_LOG_RETURN_IF_FALSE((uRequestSize & (IMG_PAGE2BYTES64(uiLog2ChunkSize) - 1)) == 0,
+	                        "Require uiLog2ChunkSize pow 2 & multiple of uRequestSize",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Enforce these constraints so we can use those bits to handle Ghost bases. */
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure Base Array is large enough for intended allocation */
+	PVR_LOG_RETURN_IF_FALSE(((RA_LENGTH_T)uiBaseArraySize * (IMG_PAGE2BYTES64(uiLog2ChunkSize)) >= uRequestSize),
+		                    "Not enough array space to store alloc bases",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: arena='%s', size=0x%llx(0x%llx), "
+	        "log2ChunkSize=0x%llx", __func__, pArena->name,
+	        (unsigned long long)uSize, (unsigned long long)uRequestSize,
+	        (unsigned long long)uiLog2ChunkSize));
+
+	/* if allocation failed then we might have an import source which
+	   can provide more resource, else we will have to fail the
+	   allocation to the caller. */
+	eError = _AttemptAllocAlignedAssured(pArena,
+	                                      uSize,
+	                                      uiLog2ChunkSize,
+	                                      uFlags,
+	                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                      aBaseArray,
+	                                      bSparseAlloc,
+	                                      bPhysContig);
+	if (eError)
+	{
+		BT *apsBTsStatic[RA_DEFAULT_IMPORTS_BUFFER_SIZE];
+		BT_PTR_ARRAY sBTs = {
+			.uiBTCount = ARRAY_SIZE(apsBTsStatic),
+			.apsBTs = apsBTsStatic,
+		};
+
+		if (eError == PVRSRV_ERROR_RA_OUT_OF_RESOURCE)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA out of resource, attempt to import more if possible:"
+			                         " uSize:0x%llx"
+			                         " uFlags:0x%llx",
+			                         (unsigned long long) uSize,
+			                         (unsigned long long) uFlags));
+		}
+		else if (eError == PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA no resource for flags, attempt to import some if possible:"
+			                         " uSize:0x%llx"
+			                         " uFlags:0x%llx",
+			                         (unsigned long long) uSize,
+			                         (unsigned long long) uFlags));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA Failed to Allocate, could be fragmented, attempt to import"
+			                         " more resource if possible."));
+		}
+
+		eError = _AttemptImportSpanAlloc(pArena,
+		                                 uSize,
+		                                 uImportMultiplier,
+		                                 uFlags,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                 pszAnnotation,
+		                                 &sBTs);
+		if (eError != PVRSRV_OK)
+		{
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			return eError;
+		}
+
+		eError = _AttemptAllocAlignedAssured(pArena,
+		                                      uSize,
+		                                      uiLog2ChunkSize,
+		                                      uFlags,
+		                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                      aBaseArray,
+		                                      bSparseAlloc,
+		                                      bPhysContig);
+		if (eError)
+		{
+			IMG_UINT32 i;
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: name='%s' second alloc failed!",
+					 __func__, pArena->name));
+			/*
+			  On failure of _AttemptAllocAligned() depending on the exact point
+			  of failure, the imported segment may have been used and freed, or
+			  left untouched. If the later, we need to return it.
+			*/
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				_FreeBT(pArena, sBTs.apsBTs[i]);
+			}
+
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+#if defined(DEBUG)
+		/*
+		 * This block of code checks to see if the extra memory we just imported was
+		 * used for the second allocation. If we imported memory but did not use it,
+		 * it indicates there is a bug in the allocation logic. We can still recover by
+		 * freeing the imported span(s) but we emit an error to signal that there is an
+		 * issue.
+		 * */
+		else
+		{
+			IMG_UINT32 bt_index;
+			IMG_UINT32 base_index;
+			IMG_BOOL bBasesInNewSpan = IMG_FALSE;
+
+			for (bt_index = 0; bt_index < sBTs.uiBTCount && !bBasesInNewSpan; bt_index++)
+			{
+				BT *pBT = sBTs.apsBTs[bt_index];
+				for (base_index = 0; base_index < uiBaseArraySize; base_index++)
+				{
+					RA_BASE_T uiBase = RA_BASE_STRIP_GHOST_BIT(aBaseArray[base_index]);
+
+					/* If the base hasn't been allocated then skip it */
+					if (aBaseArray[base_index] == INVALID_BASE_ADDR)
+					{
+						continue;
+					}
+
+					if (uiBase >= pBT->base &&
+					    uiBase <= pBT->base + pBT->uSize)
+					{
+						bBasesInNewSpan = IMG_TRUE;
+						break;
+					}
+				}
+			}
+			if (!bBasesInNewSpan)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: name='%s' alloc did not occur in the imported span!",
+						__func__, pArena->name));
+				/* Remove the imported spans which should not be in use (if it is then that is okay).
+				 * No spans should exist and not be used unless added through the import multiplier. */
+				for (bt_index = 0; bt_index < sBTs.uiBTCount; bt_index++)
+				{
+					_FreeBT(pArena, sBTs.apsBTs[bt_index]);
+				}
+			}
+		}
+#endif
+		if (sBTs.apsBTs != apsBTsStatic)
+		{
+			OSFreeMem(sBTs.apsBTs);
+		}
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_AllocMulti(RA_ARENA *pArena,
+              RA_LENGTH_T uRequestSize,
+              IMG_UINT32 uiLog2ChunkSize,
+              IMG_UINT8 uImportMultiplier,
+              RA_FLAGS_T uImportFlags,
+              const IMG_CHAR *pszAnnotation,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+              IMG_BOOL *bPhysContig)
+{
+	PVRSRV_ERROR eError;
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_AllocMultiUnlocked(pArena,
+	                                 uRequestSize,
+	                                 uiLog2ChunkSize,
+	                                 uImportMultiplier,
+	                                 uImportFlags,
+	                                 pszAnnotation,
+	                                 aBaseArray,
+	                                 uiBaseArraySize,
+	                                 IMG_FALSE, /* Sparse alloc */
+	                                 bPhysContig);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_AllocMultiSparse(RA_ARENA *pArena,
+                     IMG_UINT32 uiLog2ChunkSize,
+                     IMG_UINT8 uImportMultiplier,
+                     RA_FLAGS_T uImportFlags,
+                     const IMG_CHAR *pszAnnotation,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                     IMG_UINT32 *puiAllocIndices,
+                     IMG_UINT32 uiAllocCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bPhysContig;
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+
+	/*
+	 * In this case the arguments given show the allocation is
+	 * sparse but has no specific indices, this indicates
+	 * we want to populate the full aBaseArray
+	 */
+	if (puiAllocIndices == NULL)
+	{
+		RA_LENGTH_T uRequestSize = (RA_LENGTH_T) uiAllocCount << uiLog2ChunkSize;
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 uRequestSize,
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 aBaseArray,
+		                                 uiBaseArraySize,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_IF_ERROR(eError, "RA_AllocMulti");
+		OSLockRelease(pArena->hLock);
+		return eError;
+	}
+
+	/*
+	 * This case is optimised for single allocations as we can skip
+	 * some of the iteration logic in the full allocation path.
+	 */
+	if (uiAllocCount == 1)
+	{
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 &aBaseArray[puiAllocIndices[0]],
+		                                 1,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_IF_ERROR(eError, "RA_AllocMulti");
+		OSLockRelease(pArena->hLock);
+		return eError;
+	}
+
+	/*
+	 * By consolidating / grouping the indices given we can perform sparse allocations
+	 * in blocks, this has the effect of reducing fragmentation and creating optimal free
+	 * scenarios. Free can be performed in blocks rather than a chunk at a time, this reduces
+	 * the amount of BT merging cycles we perform.
+	 */
+	for (i = 0; i < uiAllocCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		for (j = i;
+		     j + 1 != uiAllocCount &&
+		     puiAllocIndices[j + 1] == puiAllocIndices[j] + 1;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 IMG_PAGES2BYTES64(uiConsolidate, uiLog2ChunkSize),
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 &aBaseArray[puiAllocIndices[i]],
+		                                 uiConsolidate,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RA_AllocMulti", unwind_alloc);
+		i += uiConsolidate;
+	}
+
+	OSLockRelease(pArena->hLock);
+	return PVRSRV_OK;
+
+unwind_alloc:
+	if (i != 0)
+	{
+		PVRSRV_ERROR eFreeError;
+		eFreeError = _RA_FreeMultiUnlockedSparse(pArena,
+		                                          aBaseArray,
+		                                          uiBaseArraySize,
+		                                          IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                          puiAllocIndices,
+		                                          &i);
+		PVR_LOG_IF_ERROR(eFreeError, "_RA_FreeMultiUnlockedSparse");
+	}
+
+	OSLockRelease(pArena->hLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RA_Find_BT_VARange
+@Description    To find the boundary tag associated with the given device
+                                  virtual address.
+@Input          pArena            The arena
+@input          base              Allocated base resource
+@Input          uRequestSize      The size of resource segment requested.
+@Input          uImportFlags            Flags influencing allocation policy.
+@Return         Boundary Tag - success, NULL on failure
+*/ /**************************************************************************/
+static BT *RA_Find_BT_VARange(RA_ARENA *pArena,
+                              RA_BASE_T base,
+                              RA_LENGTH_T uRequestSize,
+                              RA_FLAGS_T uImportFlags)
+{
+	IMG_PSPLAY_TREE psSplaynode;
+	IMG_UINT32 uIndex;
+
+	/* Find the splay node associated with these import flags */
+	psSplaynode = PVRSRVFindNode(uImportFlags, pArena->per_flags_buckets);
+
+	if (psSplaynode == NULL)
+	{
+		return NULL;
+	}
+
+	uIndex = pvr_log2(uRequestSize);
+
+	/* Find the free Boundary Tag from the bucket that holds the requested range */
+	while (uIndex < FREE_TABLE_LIMIT)
+	{
+		BT *pBT = psSplaynode->buckets[uIndex];
+
+		while (pBT)
+		{
+			if ((pBT->base <= base) && ((pBT->base + pBT->uSize) >= (base + uRequestSize)))
+			{
+				if (pBT->type == btt_free)
+				{
+					return pBT;
+				}
+				else
+				{
+					PVR_ASSERT(pBT->type == btt_free);
+				}
+			}
+			else{
+				pBT = pBT->next_free;
+			}
+		}
+
+#if defined(PVR_CTZLL)
+		/* This could further be optimised to get the next valid bucket */
+		while (!(psSplaynode->bHasEltsMapping & (1ULL << ++uIndex)));
+#else
+		uIndex++;
+#endif
+	}
+
+	return NULL;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_Alloc_Range(RA_ARENA *pArena,
+               RA_LENGTH_T uRequestSize,
+               RA_FLAGS_T uImportFlags,
+               RA_LENGTH_T uAlignment,
+               RA_BASE_T base,
+               RA_LENGTH_T *pActualSize)
+{
+	RA_LENGTH_T uSize = uRequestSize;
+	BT *pBT = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (pArena == NULL || uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: One of the necessary parameters is 0", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	/* Align the requested size to the Arena Quantum */
+	uSize = PVR_ALIGN(uSize, pArena->uQuantum);
+
+	/* Must be a power of 2 or 0 */
+	PVR_ASSERT((uAlignment == 0) || (uAlignment & (uAlignment - 1)) == 0);
+
+	if (uAlignment > 1)
+	{
+		if (base != PVR_ALIGN(base, uAlignment))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, unlock_);
+		}
+	}
+
+	/* Find if the segment in the range exists and is free
+	 * Check if the segment can be split
+	 * Find the bucket that points to this segment
+	 * Find the free segment is in the free list
+	 * remove the free segment
+	 * split the segment into three segments one prior free, alloc range,
+	 *     free segment after the range.
+	 * remove the allocated range segment from the free list
+	 * hook up the prior and after segments back to free list
+	 * For each free, find the bucket the segment should go to
+	 */
+
+	pBT = RA_Find_BT_VARange(pArena, base, uSize, uImportFlags);
+
+	if (pBT == NULL)
+	{
+		PVR_GOTO_WITH_ERROR(eError,
+		                    PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL,
+		                    unlock_);
+	}
+
+	/* Remove the boundary tag from the free list */
+	_FreeListRemove (pArena, pBT);
+
+	/* if requested VA start in the middle of the BT, split the BT accordingly */
+	if (base > pBT->base)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit (pBT, (RA_LENGTH_T)(base - pBT->base));
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			/* Put pBT back in the list */
+			_FreeListInsert (pArena, pBT);
+			PVR_LOG_GOTO_WITH_ERROR("_SegmentSplit (1)", eError,
+			                        PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL,
+			                        unlock_);
+		}
+
+		/* Insert back the free BT to the free list */
+		_FreeListInsert(pArena, pBT);
+		pBT = pNeighbour;
+	}
+
+	/* the segment might be too big, if so, discard the back of the segment */
+	if (pBT->uSize > uSize)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, uSize);
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			/* Put pBT back in the list */
+			_FreeListInsert (pArena, pBT);
+			PVR_LOG_GOTO_WITH_ERROR("_SegmentSplit (2)", eError,
+			                        PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL,
+			                        unlock_);
+		}
+
+		/* Insert back the free BT to the free list */
+		_FreeListInsert (pArena, pNeighbour);
+	}
+
+	pBT->type = btt_live;
+
+	if (!HASH_Insert_Extended (pArena->pSegmentHash, &base, (uintptr_t)pBT))
+	{
+		_FreeBT (pArena, pBT);
+		PVR_GOTO_WITH_ERROR(eError,
+		                    PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED,
+		                    unlock_);
+	}
+
+	if (pActualSize != NULL)
+	{
+		*pActualSize = uSize;
+	}
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+unlock_:
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+IMG_INTERNAL void
+RA_Free(RA_ARENA *pArena, RA_BASE_T base)
+{
+	BT *pBT;
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', base=0x%llx", __func__, pArena->name,
+	        (unsigned long long)base));
+
+	pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &base);
+	PVR_ASSERT(pBT != NULL);
+
+	if (pBT)
+	{
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		PVR_ASSERT(pBT->base == base);
+		_FreeBT(pArena, pBT);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: no resource span found for given base (0x%llX) in arena %s",
+		         __func__, (unsigned long long) base, pArena->name));
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+	OSLockRelease(pArena->hLock);
+}
+
+static PVRSRV_ERROR
+_RA_FreeMultiUnlocked(RA_ARENA *pArena,
+                       RA_BASE_ARRAY_T aBaseArray,
+                       RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	PVRSRV_ERROR eError;
+
+	/* Free the whole array */
+	if (uiBaseArraySize == 1)
+	{
+		eError = _FreeSingleBaseArray(pArena, aBaseArray, uiBaseArraySize);
+		PVR_LOG_IF_ERROR(eError, "_FreeSingleBaseArray");
+	}
+	else
+	{
+		eError = _FreeMultiBaseArray(pArena, aBaseArray, uiBaseArraySize);
+		PVR_LOG_IF_ERROR(eError, "_FreeMultiBaseArray");
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_FreeMulti(RA_ARENA *pArena,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	PVRSRV_ERROR eError;
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_FreeMultiUnlocked(pArena,
+	                                aBaseArray,
+	                                uiBaseArraySize);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_RA_FreeMultiUnlockedSparse(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 *puiFreeIndices,
+                             IMG_UINT32 *puiFreeCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiFreeCount = *puiFreeCount;
+	*puiFreeCount = 0;
+
+	/* Handle case where we only have 1 base to free. */
+	if (uiFreeCount == 1)
+	{
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiFreeIndices[0],
+		                                      1,
+		                                      CONVERT_AND_FREE);
+		PVR_LOG_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+		if (eError == PVRSRV_OK)
+		{
+			*puiFreeCount = uiFreeCount;
+		}
+		return eError;
+	}
+
+	for (i = 0; i < uiFreeCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[i]));
+
+		for (j = i;
+		     puiFreeIndices[j + 1] == puiFreeIndices[j] + 1 &&
+		     RA_BASE_IS_GHOST(aBaseArray[puiFreeIndices[j + 1]]) &&
+		     j + 1 < uiFreeCount;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiFreeIndices[i],
+		                                      uiConsolidate,
+		                                      CONVERT_AND_FREE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+
+		i += uiConsolidate;
+		*puiFreeCount += uiConsolidate;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_FreeMultiSparse(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                    IMG_UINT32 uiLog2ChunkSize,
+                    IMG_UINT32 *puiFreeIndices,
+                    IMG_UINT32 *puiFreeCount)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(puiFreeCount != NULL,
+		                        "puiFreeCount Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure Base Array is large enough for intended free */
+	PVR_LOG_RETURN_IF_FALSE(uiBaseArraySize >= *puiFreeCount,
+		                        "Attempt to free more bases than array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(puiFreeIndices != NULL,
+		                        "puiFreeIndices Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_FreeMultiUnlockedSparse(pArena,
+	                                      aBaseArray,
+	                                      uiBaseArraySize,
+	                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                      puiFreeIndices,
+	                                      puiFreeCount);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/*************************************************************************/ /*!
+ *  @Function   _RA_RealiseMultiSparseIndicesUnlocked
+ *
+ *  @Description   Given an array of indices to extract, prepares the base
+ *                 array so that the indices in puiExtractIndices can be
+ *                 moved to another base array.
+ *                 Called when some pages of the base array need to be
+ *                 transferred to another base array. As a result of this call,
+ *                 some ghost addresses in aBaseArray might be converted to
+ *                 real addresses.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *  @Input  uiChunkSize -  The Base chunk size used to generate ghost entries
+ *  @Input  puiExtractIndices - Array of indices to extract
+ *  @Input  puiExtractCount - Number of indices to extract
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_RA_RealiseMultiSparseIndicesUnlocked(RA_ARENA *pArena,
+                                      RA_BASE_ARRAY_T aBaseArray,
+                                      RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                                      RA_LENGTH_T uiChunkSize,
+                                      IMG_UINT32 *puiExtractIndices,
+                                      IMG_UINT32 *puiExtractCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiExtractCount = *puiExtractCount;
+	*puiExtractCount = 0;
+
+	/* Handle case where we only have 1 base to extract. */
+	if (uiExtractCount == 1)
+	{
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiExtractIndices[0],
+		                                      1,
+		                                      CONVERT_DONT_FREE);
+		PVR_LOG_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+		if (eError == PVRSRV_OK)
+		{
+			*puiExtractCount = uiExtractCount;
+		}
+		return eError;
+	}
+
+	for (i = 0; i < uiExtractCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[i]));
+
+		for (j = i;
+		     puiExtractIndices[j + 1] == puiExtractIndices[j] + 1 &&
+		     RA_BASE_IS_GHOST(aBaseArray[puiExtractIndices[j + 1]]) &&
+		     j + 1 < uiExtractCount;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiExtractIndices[i],
+		                                      uiConsolidate,
+		                                      CONVERT_DONT_FREE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+
+		i += uiConsolidate;
+		*puiExtractCount += uiConsolidate;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_TransferMultiSparseIndices(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aSrcBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiSrcBaseArraySize,
+                              RA_BASE_ARRAY_T aDstBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiDstBaseArraySize,
+                              IMG_UINT32 uiLog2ChunkSize,
+                              IMG_UINT32 *puiTransferIndices,
+                              IMG_UINT32 *puiTransferCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_FALSE(puiTransferCount != NULL,
+		                        "puiTransferCount Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure source Base Array is large enough for intended extract */
+	PVR_LOG_RETURN_IF_FALSE(uiSrcBaseArraySize >= *puiTransferCount,
+		                        "Attempt to transfer more bases than src array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure dst Base Array is large enough for intended extract */
+	PVR_LOG_RETURN_IF_FALSE(uiDstBaseArraySize <= *puiTransferCount,
+		                        "Attempt to transfer more bases than dst array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(puiTransferIndices != NULL,
+		                        "puiTransferIndices Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	/* First prepare the base array for subsequent transfer */
+	eError = _RA_RealiseMultiSparseIndicesUnlocked(pArena,
+	                                               aSrcBaseArray,
+	                                               uiSrcBaseArraySize,
+	                                               IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                               puiTransferIndices,
+	                                               puiTransferCount);
+	OSLockRelease(pArena->hLock);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	/* Now do the transfer */
+	for (i=0; i<*puiTransferCount; i++)
+	{
+		IMG_UINT32 idxSrc = puiTransferIndices[i];
+		aDstBaseArray[i] = aSrcBaseArray[idxSrc];
+		aSrcBaseArray[idxSrc] = INVALID_BASE_ADDR;
+	}
+
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_TrimBlockMakeReal(RA_ARENA *pArena,
+                   RA_BASE_ARRAY_T aBaseArray,
+                   RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                   IMG_UINT32 uiLog2ChunkSize,
+                   IMG_UINT32 uiStartIndex,
+                   IMG_UINT32 uiEndIndex)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RA_BASE_T sRealBase;
+	IMG_UINT32 uiRealBaseIndex;
+
+	/* Note: Error return paths in this function do not require unwinding.
+	 * Free logic is performed based upon indices and detection of Real base regions,
+	 * performance wise it would be more costly to unwind the conversion here than to
+	 * just free a smaller Real base region.
+	 */
+
+	/* Check Start index is real, if not make it real */
+	if (RA_BASE_IS_GHOST(aBaseArray[uiStartIndex]))
+	{
+		_FindRealBaseFromGhost(aBaseArray,
+		                       uiStartIndex,
+		                       &sRealBase,
+		                       &uiRealBaseIndex);
+
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 sRealBase,
+		                                 uiRealBaseIndex,
+		                                 uiStartIndex,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize));
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	/* Check end +1 is real or end of array , if ghost make real */
+	if (uiEndIndex + 1 != uiBaseArraySize &&
+	    RA_BASE_IS_GHOST(aBaseArray[uiEndIndex + 1]))
+	{
+		_FindRealBaseFromGhost(aBaseArray,
+		                       uiEndIndex + 1,
+		                       &sRealBase,
+		                       &uiRealBaseIndex);
+
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 sRealBase,
+		                                 uiRealBaseIndex,
+		                                 uiEndIndex + 1,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize));
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_SwapSparseMem(RA_ARENA *pArena,
+                  RA_BASE_ARRAY_T aBaseArray,
+                  RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                  IMG_UINT32 uiLog2ChunkSize,
+                  IMG_UINT32 *puiXIndices,
+                  IMG_UINT32 *puiYIndices,
+                  IMG_UINT32 uiSwapCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiSwapped = 0;
+	IMG_UINT32 uiStartIndex;
+	/* Consolidation values counting the bases after the start index*/
+	IMG_UINT32 uiXConsol;
+	IMG_UINT32 uiYConsol;
+	/* Consolidation limit, the smallest consecutive indices between the
+	 * two inputs
+	 */
+	IMG_UINT32 uiConsolidateLimit;
+	IMG_UINT32 uiTotalSwapCount;
+	IMG_UINT32 i;
+
+	/*
+	 * The algorithm below aims to swap the desired indices whilst also
+	 * maintaining a maximum contiguity of allocation blocks where possible.
+	 * It does this by:
+	 * Consolidating the contiguous indices of X and Y.
+	 * Selecting the smallest of these consolidations as a range to swap in a block.
+	 * Trim both block ranges using the indices range to ensure that Real bases are
+	 * created to represent regions that have been split due to the indices.
+	 * Perform the swap and update the swapped count ready for the next iteration.
+	 * Note: Maintaining contiguity improves performance of free logic for sparse
+	 * allocations because we can free in regions rather than chunks.
+	 */
+	while (uiSwapped != uiSwapCount)
+	{
+		IMG_UINT32 x, y;
+		uiTotalSwapCount = 1;
+		uiStartIndex = uiSwapped;
+		uiXConsol = 0;
+		uiYConsol = 0;
+
+		/* Calculate contiguous indices at X */
+		for (x = uiStartIndex;
+		     x < uiSwapCount &&
+		     puiXIndices[x] + 1 == puiXIndices[x + 1];
+		     x++)
+		{
+			uiXConsol++;
+		}
+
+		/* Calculate contiguous indices at Y */
+		for (y = uiStartIndex;
+		     y < uiSwapCount &&
+		     puiYIndices[y] + 1 == puiYIndices[y + 1];
+		     y++)
+		{
+			uiYConsol++;
+		}
+
+		/* Find lowest consolidation value */
+		uiConsolidateLimit = (uiXConsol < uiYConsol) ? uiXConsol : uiYConsol;
+
+		/* Perform RealBase translation where required */
+		eError = _TrimBlockMakeReal(pArena,
+		                            aBaseArray,
+		                            uiBaseArraySize,
+		                            uiLog2ChunkSize,
+		                            puiXIndices[uiStartIndex],
+		                            puiXIndices[uiStartIndex + uiConsolidateLimit]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_TrimBlockMakeReal", unwind);
+
+		eError = _TrimBlockMakeReal(pArena,
+		                            aBaseArray,
+		                            uiBaseArraySize,
+		                            uiLog2ChunkSize,
+		                            puiYIndices[uiStartIndex],
+		                            puiYIndices[uiStartIndex + uiConsolidateLimit]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_TrimBlockMakeReal", unwind);
+
+		uiTotalSwapCount += uiConsolidateLimit;
+		uiSwapped += uiTotalSwapCount;
+		i = uiStartIndex;
+
+		do
+		{
+			SWAP(aBaseArray[puiXIndices[i]], aBaseArray[puiYIndices[i]]);
+			uiTotalSwapCount--;
+			i++;
+		}
+		while (uiTotalSwapCount != 0);
+	}
+
+	return PVRSRV_OK;
+
+unwind:
+	/* If we hit an error when Trimming we should revert the swapping
+	 * that has already been performed.
+	 */
+	for (i = 0; i < uiSwapped; i++)
+	{
+		SWAP(aBaseArray[puiXIndices[i]], aBaseArray[puiYIndices[i]]);
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL void
+RA_Get_Usage_Stats(RA_ARENA *pArena, PRA_USAGE_STATS psRAStats)
+{
+	psRAStats->ui64TotalArenaSize = pArena->ui64TotalArenaSize;
+	psRAStats->ui64FreeArenaSize = pArena->ui64FreeArenaSize;
+}
+
+IMG_INTERNAL IMG_CHAR *
+RA_GetArenaName(RA_ARENA *pArena)
+{
+	return pArena->name;
+}
+
+/* #define _DBG(...) PVR_LOG((__VA_ARGS__)) */
+#define _DBG(...)
+
+IMG_INTERNAL RA_ARENA_ITERATOR *
+RA_IteratorAcquire(RA_ARENA *pArena, IMG_BOOL bIncludeFreeSegments)
+{
+	RA_ARENA_ITERATOR *pIter = OSAllocMem(sizeof(*pIter));
+	PVR_LOG_RETURN_IF_FALSE(pIter != NULL, "OSAllocMem", NULL);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+
+	pIter->pArena = pArena;
+	pIter->bIncludeFreeSegments = bIncludeFreeSegments;
+
+	RA_IteratorReset(pIter);
+
+	return pIter;
+}
+
+IMG_INTERNAL void
+RA_IteratorRelease(RA_ARENA_ITERATOR *pIter)
+{
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		return;
+	}
+
+	OSLockRelease(pIter->pArena->hLock);
+
+	OSFreeMem(pIter);
+}
+
+IMG_INTERNAL void
+RA_IteratorReset(RA_ARENA_ITERATOR *pIter)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	pNext = pIter->pArena->pHeadSegment;
+
+	/* find next element if we're not including the free ones */
+	if (!pIter->bIncludeFreeSegments)
+	{
+		while (pNext != NULL && pNext->type != btt_live)
+		{
+			_DBG("(%s()) skipping segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+			     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+			     pNext->type);
+			pNext = pNext->pNextSegment;
+		}
+	}
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	/* if bIncludeFreeSegments then pNext here is either a valid pointer to
+	 * "live" segment or NULL and if !bIncludeFreeSegments then it's either
+	 * a valid pointer to any next segment or NULL */
+	pIter->pCurrent = pNext;
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNext(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	if (pIter->pCurrent == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pNext = pIter->pCurrent;
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+	     pNext->type);
+
+	pData->uiAddr = pIter->pCurrent->base;
+	pData->uiSize = pIter->pCurrent->uSize;
+	pData->bFree = pIter->pCurrent->type == btt_free;
+
+	/* combine contiguous segments */
+	while ((pNext = pNext->pNextSegment) != NULL &&
+	        pNext->type == pNext->pPrevSegment->type &&
+	        pNext->type == btt_live &&
+	        pNext->base == pData->uiAddr + pData->uiSize)
+	{
+		_DBG("(%s()) combining segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+		     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+		     pNext->type);
+		pData->uiSize += pNext->uSize;
+	}
+
+	/* advance to next */
+	if (!pIter->bIncludeFreeSegments)
+	{
+		while (pNext != NULL && pNext->type != btt_live)
+		{
+			_DBG("(%s()) skipping segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+			     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+			     pNext->type);
+			pNext = pNext->pNextSegment;
+		}
+	}
+
+	_DBG("(%s()) next segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	/* if bIncludeFreeSegments then pNext here is either a valid pointer to
+	 * "live" segment or NULL and if !bIncludeFreeSegments then it's either
+	 * a valid pointer to any next segment or NULL */
+	pIter->pCurrent = pNext;
+
+	return IMG_TRUE;
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNextSpan(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	/* We have no more iterations to perform */
+	if (pIter->pCurrent == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pNext = pIter->pCurrent;
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+	     pNext->type);
+
+	/* Extract details of current span */
+	pData->uiAddr = pIter->pCurrent->base;
+	pData->uiSize = pIter->pCurrent->uSize;
+	pData->bFree = pIter->pCurrent->free_import;
+
+	/* If this BT is a full span, can occur if we have no allocations
+	 * on a span.
+	 */
+	if (pNext->is_leftmost == 1 &&
+	    pNext->is_rightmost == 1)
+	{
+		pNext = pNext->pNextSegment;
+	}
+	else
+	{
+		/* combine contiguous segments of span, these are allocations
+		 * and free segments in a span.
+		 */
+		while ((pNext = pNext->pNextSegment) != NULL &&
+		        pNext->is_leftmost == 0)
+		{
+			_DBG("(%s()) combining segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+				 "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+				 pNext->type);
+			pData->uiSize += pNext->uSize;
+		}
+	}
+
+	_DBG("(%s()) next segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	pIter->pCurrent = pNext;
+
+	return IMG_TRUE;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_BlockDump(RA_ARENA *pArena, void (*pfnLogDump)(void*, IMG_CHAR*, ...), void *pPrivData)
+{
+	RA_ARENA_ITERATOR *pIter = NULL;
+	RA_ITERATOR_DATA sIterData;
+	const IMG_UINT32 uiLineWidth = 64;
+
+	IMG_UINT32 **papRegionArray = NULL;
+	IMG_UINT32 uiRegionCount = 0;
+
+	const IMG_UINT32 uiChunkSize = 32; /* 32-bit chunks */
+	const IMG_UINT32 uiChunkCount = (uiLineWidth / uiChunkSize) * 2; /* This should equal 2 or a multiple of 2 */
+	const IMG_UINT32 uiRegionSize = uiChunkSize * uiChunkCount;
+
+	IMG_UINT32 uiRecognisedQuantum = 0;
+
+	IMG_UINT64 uiLastBase = 0;
+	IMG_UINT64 uiLastSize = 0;
+
+	IMG_UINT32 i;
+	IMG_UINT32 uiRemainder;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	IMG_UINT64 uiLargestFreeSegmentSize = 0;
+	IMG_UINT32 uiFragPercentage = 0;
+
+	/* -- papRegionArray Structure --
+	 *  papRegionArray Indexes
+	 *  |         Chunk 0      Chunk 1      Chunk 2      Chunk 3
+	 *  v     |------------|------------|------------|------------|
+	 * [0] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 | -- |
+	 * [1] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [2] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [3] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    | Regions
+	 * [4] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [5] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [6] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 | -- |
+	 * ...
+	 */
+
+	if (pArena == NULL || pfnLogDump == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pIter = RA_IteratorAcquire(pArena, IMG_TRUE);
+	PVR_LOG_RETURN_IF_NOMEM(pIter, "RA_IteratorAcquire");
+
+	uiRecognisedQuantum = pArena->uQuantum > 0 ? pArena->uQuantum : 4096;
+
+	while (RA_IteratorNext(pIter, &sIterData))
+	{
+		if (!sIterData.bFree && sIterData.uiAddr >= uiLastBase)
+		{
+			uiLastBase = sIterData.uiAddr;
+			uiLastSize = sIterData.uiSize;
+		}
+	}
+
+	uiRegionCount = OSDivide64(uiLastBase + uiLastSize, uiRecognisedQuantum,
+	                           &uiRemainder);
+	uiRegionCount = OSDivide64(uiRegionCount, uiRegionSize, &uiRemainder);
+	if (uiRemainder != 0 || uiRegionCount == 0)
+	{
+		uiRegionCount += 1;
+	}
+
+	papRegionArray = OSAllocZMem(sizeof(IMG_UINT32*) * uiRegionCount);
+	PVR_LOG_GOTO_IF_NOMEM(papRegionArray, eError, cleanup_array);
+
+	RA_IteratorReset(pIter);
+
+	while (RA_IteratorNext(pIter, &sIterData))
+	{
+		IMG_UINT64 uiDataDivRecQuant;
+
+		IMG_UINT32 uiAddrRegionIdx = 0;
+		IMG_UINT32 uiAddrRegionOffset = 0;
+		IMG_UINT32 uiAddrChunkIdx = 0;
+		IMG_UINT32 uiAddrChunkOffset = 0;
+		IMG_UINT32 uiAddrChunkShift; /* The bit-shift needed to fill the chunk */
+
+		IMG_UINT32 uiQuantisedSize;
+		IMG_UINT32 uiQuantisedSizeMod;
+		IMG_UINT32 uiAllocLastRegionIdx = 0; /* The last region that this alloc appears in */
+		IMG_UINT32 uiAllocChunkSize = 0; /* The number of chunks this alloc spans */
+
+		IMG_INT32 iBitSetCount = 0;
+		IMG_INT32 iOverflowCheck = 0;
+		IMG_INT32 iOverflow = 0;
+		IMG_UINT32 uiRegionIdx = 0;
+		IMG_UINT32 uiChunkIdx = 0;
+
+		/* If the current data is for a free block, use it to track largest
+		 * contiguous free segment size.
+		 */
+		if (sIterData.bFree && sIterData.uiSize > uiLargestFreeSegmentSize)
+		{
+			uiLargestFreeSegmentSize = sIterData.uiSize;
+			continue;
+		}
+
+		uiDataDivRecQuant = OSDivide64(sIterData.uiAddr, uiRecognisedQuantum,
+		                               &uiRemainder);
+		uiAddrRegionIdx = OSDivide64(uiDataDivRecQuant, uiRegionSize,
+		                             &uiAddrRegionOffset);
+		uiQuantisedSize = OSDivide64(sIterData.uiSize, uiRecognisedQuantum,
+		                             &uiQuantisedSizeMod);
+
+		uiAddrChunkIdx = uiAddrRegionOffset / uiChunkSize;
+		uiAddrChunkOffset = uiAddrRegionOffset % uiChunkSize;
+		uiAddrChunkShift = uiChunkSize - uiAddrChunkOffset;
+		uiRegionIdx = uiAddrRegionIdx;
+		uiChunkIdx = uiAddrChunkIdx;
+
+		if ((uiQuantisedSize == 0) || (uiQuantisedSizeMod != 0))
+		{
+			uiQuantisedSize += 1;
+		}
+
+		uiAllocLastRegionIdx = OSDivide64(uiDataDivRecQuant + uiQuantisedSize - 1,
+		                                  uiRegionSize, &uiRemainder);
+		uiAllocChunkSize = (uiAddrChunkOffset + uiQuantisedSize) / uiChunkSize;
+
+		if ((uiAddrChunkOffset + uiQuantisedSize) % uiChunkSize > 0)
+		{
+			uiAllocChunkSize += 1;
+		}
+
+		iBitSetCount = uiQuantisedSize;
+		iOverflowCheck = uiQuantisedSize - uiAddrChunkShift;
+
+		if (iOverflowCheck > 0)
+		{
+			iOverflow = iOverflowCheck;
+			iBitSetCount = uiQuantisedSize - iOverflow;
+		}
+
+		/**
+		 * Allocate memory to represent the chunks for each region the allocation
+		 * spans. If one was already allocated before don't do it again.
+		 */
+		for (i = 0; uiAddrRegionIdx + i <= uiAllocLastRegionIdx; i++)
+		{
+			if (papRegionArray[uiAddrRegionIdx + i] == NULL)
+			{
+				papRegionArray[uiAddrRegionIdx + i] = OSAllocZMem(sizeof(IMG_UINT32) * uiChunkCount);
+				PVR_LOG_GOTO_IF_NOMEM(papRegionArray[uiAddrRegionIdx + i], eError, cleanup_regions);
+			}
+		}
+
+		for (i = 0; i < uiAllocChunkSize; i++)
+		{
+			if (uiChunkIdx >= uiChunkCount)
+			{
+				uiRegionIdx++;
+				uiChunkIdx = 0;
+			}
+
+			if ((IMG_UINT32)iBitSetCount != uiChunkSize)
+			{
+				IMG_UINT32 uiBitMask = 0;
+
+				uiBitMask = (1U << iBitSetCount) - 1;
+				uiBitMask <<= (uiAddrChunkShift - iBitSetCount);
+
+				papRegionArray[uiRegionIdx][uiChunkIdx] |= uiBitMask;
+			}
+			else
+			{
+				papRegionArray[uiRegionIdx][uiChunkIdx] |= 0xFFFFFFFF;
+			}
+
+			uiChunkIdx++;
+			iOverflow -= uiChunkSize;
+			iBitSetCount = iOverflow >= 0 ? uiChunkSize : uiChunkSize + iOverflow;
+			if (iOverflow < 0)
+			{
+				uiAddrChunkShift = 32;
+			}
+		}
+	}
+	if (pArena->ui64FreeArenaSize && uiLargestFreeSegmentSize)
+	{
+		/* N.B This can look strange in a dual RA when comparing to the dump visualisation
+		 * as spans that are freed are not included in the segment list, regardless it is
+		 * an accurate representation for the spans in the Arena.
+		 */
+		uiFragPercentage = OSDivide64(100 * pArena->ui64FreeArenaSize,
+		                              pArena->ui64FreeArenaSize + uiLargestFreeSegmentSize,
+		                              &uiRemainder);
+	}
+
+	pfnLogDump(pPrivData, "~~~ '%s' Resource Arena Block Dump", pArena->name);
+	pfnLogDump(pPrivData, "    Block Size: %uB", uiRecognisedQuantum);
+	pfnLogDump(pPrivData,
+	           "    Span Memory Usage: %"IMG_UINT64_FMTSPEC"B"
+	           "    Free Span Memory: %"IMG_UINT64_FMTSPEC"B"
+	           "    Largest Free Region Size: %"IMG_UINT64_FMTSPEC"B"
+	           "    Percent Fragmented %u%%",
+	           pArena->ui64TotalArenaSize,
+	           pArena->ui64FreeArenaSize,
+	           uiLargestFreeSegmentSize,
+	           uiFragPercentage);
+	pfnLogDump(pPrivData,
+	           "===============================================================================");
+
+	for (i = 0; i < uiRegionCount; i++)
+	{
+		static IMG_BOOL bEmptyRegion = IMG_FALSE;
+		if (papRegionArray[i] != NULL)
+		{
+			IMG_CHAR pszLine[65];
+			IMG_UINT32 j;
+
+			bEmptyRegion = IMG_FALSE;
+			pszLine[64] = '\0';
+
+			for (j = 0; j < uiChunkCount; j+=2)
+			{
+				IMG_UINT8 uiBit = 0;
+				IMG_UINT32 k;
+				IMG_UINT64 uiLineAddress =
+				    (i * uiRegionSize + (j >> 1) * uiLineWidth) * uiRecognisedQuantum;
+
+				/**
+				 * Move through each of the 32 bits in the chunk and check their
+				 * value. If it is 1 we set the corresponding character to '#',
+				 * otherwise it is set to '.' representing empty space
+				 */
+				for (k = 1 << 31; k != 0; k >>= 1)
+				{
+					pszLine[uiBit] = papRegionArray[i][j] & k ? '#' : '.';
+					pszLine[32 + uiBit] = papRegionArray[i][j+1] & k ? '#' : '.';
+					uiBit++;
+				}
+
+				pfnLogDump(pPrivData,
+				           "| 0x%08"IMG_UINT64_FMTSPECx" | %s",
+				           uiLineAddress,
+				           pszLine);
+			}
+			OSFreeMem(papRegionArray[i]);
+		}
+		else
+		{
+			/* We only print this once per gap of n regions */
+			if (!bEmptyRegion)
+			{
+				pfnLogDump(pPrivData, "     ....");
+				bEmptyRegion = IMG_TRUE;
+			}
+		}
+	}
+
+	RA_IteratorRelease(pIter);
+
+	OSFreeMem(papRegionArray);
+	return eError;
+
+cleanup_regions:
+	for (i = 0; i < uiRegionCount; i++)
+	{
+		if (papRegionArray[i] != NULL)
+		{
+			OSFreeMem(papRegionArray[i]);
+		}
+	}
+
+cleanup_array:
+	OSFreeMem(papRegionArray);
+	RA_IteratorRelease(pIter);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/sync.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/sync.c
new file mode 100644
index 000000000000..3b6e8914d446
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/sync.c
@@ -0,0 +1,822 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements client side code for services synchronisation
+                interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "client_sync_bridge.h"
+#include "client_synctracking_bridge.h"
+#include "info_page_client.h"
+#include "pvr_bridge.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pvr_debug.h"
+#include "dllist.h"
+#include "sync.h"
+#include "sync_internal.h"
+#include "lock.h"
+#include "log2.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+
+#define SYNC_BLOCK_LIST_CHUNCK_SIZE	10
+
+/*
+	This defines the maximum amount of synchronisation memory
+	that can be allocated per SyncPrim context.
+	In reality this number is meaningless as we would run out
+	of synchronisation memory before we reach this limit, but
+	we need to provide a size to the span RA.
+ */
+#define MAX_SYNC_MEM				(4 * 1024 * 1024)
+
+/* forward declaration */
+static PVRSRV_ERROR
+_SyncPrimSetValue(SYNC_PRIM *psSyncInt, IMG_UINT32 ui32Value);
+
+/*
+	Internal interfaces for management of SYNC_PRIM_CONTEXT
+ */
+static void
+_SyncPrimContextUnref(SYNC_PRIM_CONTEXT *psContext)
+{
+	if (!OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: context already freed", __func__));
+	}
+	else if (0 == OSAtomicDecrement(&psContext->hRefCount))
+	{
+		/* SyncPrimContextDestroy only when no longer referenced */
+		RA_Delete(psContext->psSpanRA);
+		RA_Delete(psContext->psSubAllocRA);
+		OSFreeMem(psContext);
+	}
+}
+
+static void
+_SyncPrimContextRef(SYNC_PRIM_CONTEXT *psContext)
+{
+	if (!OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: context use after free", __func__));
+	}
+	else
+	{
+		OSAtomicIncrement(&psContext->hRefCount);
+	}
+}
+
+/*
+	Internal interfaces for management of synchronisation block memory
+ */
+static PVRSRV_ERROR
+AllocSyncPrimitiveBlock(SYNC_PRIM_CONTEXT *psContext,
+                        SYNC_PRIM_BLOCK **ppsSyncBlock)
+{
+	SYNC_PRIM_BLOCK *psSyncBlk;
+	IMG_HANDLE hSyncPMR;
+	IMG_HANDLE hSyncImportHandle;
+	IMG_DEVMEM_SIZE_T uiImportSize;
+	PVRSRV_ERROR eError;
+
+	psSyncBlk = OSAllocMem(sizeof(SYNC_PRIM_BLOCK));
+	PVR_GOTO_IF_NOMEM(psSyncBlk, eError, fail_alloc);
+
+	psSyncBlk->psContext = psContext;
+
+	/* Allocate sync prim block */
+	eError = BridgeAllocSyncPrimitiveBlock(GetBridgeHandle(psContext->hDevConnection),
+	                                       &psSyncBlk->hServerSyncPrimBlock,
+	                                       &psSyncBlk->ui32FirmwareAddr,
+	                                       &psSyncBlk->ui32SyncBlockSize,
+	                                       &hSyncPMR);
+	PVR_GOTO_IF_ERROR(eError, fail_blockalloc);
+
+	/* Make it mappable by the client */
+	eError = DevmemMakeLocalImportHandle(psContext->hDevConnection,
+	                                     hSyncPMR,
+	                                     &hSyncImportHandle);
+	PVR_GOTO_IF_ERROR(eError, fail_export);
+
+	/* Get CPU mapping of the memory block */
+	eError = DevmemLocalImport(psContext->hDevConnection,
+	                           hSyncImportHandle,
+	                           PVRSRV_MEMALLOCFLAG_CPU_READABLE,
+	                           &psSyncBlk->hMemDesc,
+	                           &uiImportSize,
+	                           "SyncPrimitiveBlock");
+
+	/*
+		Regardless of success or failure we "undo" the export
+	 */
+	DevmemUnmakeLocalImportHandle(psContext->hDevConnection,
+	                              hSyncImportHandle);
+
+	PVR_GOTO_IF_ERROR(eError, fail_import);
+
+	eError = DevmemAcquireCpuVirtAddr(psSyncBlk->hMemDesc,
+	                                  (void **) &psSyncBlk->pui32LinAddr);
+	PVR_GOTO_IF_ERROR(eError, fail_cpuvaddr);
+
+	*ppsSyncBlock = psSyncBlk;
+	return PVRSRV_OK;
+
+fail_cpuvaddr:
+	DevmemFree(psSyncBlk->hMemDesc);
+fail_import:
+fail_export:
+	BridgeFreeSyncPrimitiveBlock(GetBridgeHandle(psContext->hDevConnection),
+	                             psSyncBlk->hServerSyncPrimBlock);
+fail_blockalloc:
+	OSFreeMem(psSyncBlk);
+fail_alloc:
+	return eError;
+}
+
+static void
+FreeSyncPrimitiveBlock(SYNC_PRIM_BLOCK *psSyncBlk)
+{
+	SYNC_PRIM_CONTEXT *psContext = psSyncBlk->psContext;
+
+	DevmemReleaseCpuVirtAddr(psSyncBlk->hMemDesc);
+	DevmemFree(psSyncBlk->hMemDesc);
+	(void) DestroyServerResource(psContext->hDevConnection,
+	                             NULL,
+	                             BridgeFreeSyncPrimitiveBlock,
+	                             psSyncBlk->hServerSyncPrimBlock);
+	OSFreeMem(psSyncBlk);
+}
+
+static PVRSRV_ERROR
+SyncPrimBlockImport(RA_PERARENA_HANDLE hArena,
+                    RA_LENGTH_T uSize,
+                    RA_FLAGS_T uFlags,
+                    RA_LENGTH_T uBaseAlignment,
+                    const IMG_CHAR *pszAnnotation,
+                    RA_IMPORT *psImport)
+{
+	SYNC_PRIM_CONTEXT *psContext = hArena;
+	SYNC_PRIM_BLOCK *psSyncBlock = NULL;
+	RA_LENGTH_T uiSpanSize;
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(uFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	/* Check we've not been called with an unexpected size */
+	PVR_LOG_GOTO_IF_INVALID_PARAM(hArena, eError, e0);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(uSize == sizeof(IMG_UINT32), eError, e0);
+
+	/*
+		Ensure the synprim context doesn't go away while we have sync blocks
+		attached to it
+	 */
+	_SyncPrimContextRef(psContext);
+
+	/* Allocate the block of memory */
+	eError = AllocSyncPrimitiveBlock(psContext, &psSyncBlock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "AllocSyncPrimitiveBlock", fail_syncblockalloc);
+
+	/* Allocate a span for it */
+	eError = RA_Alloc(psContext->psSpanRA,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  pszAnnotation,
+	                  &psSyncBlock->uiSpanBase,
+	                  &uiSpanSize,
+	                  NULL);
+	PVR_GOTO_IF_ERROR(eError, fail_spanalloc);
+
+	/*
+		There is no reason the span RA should return an allocation larger
+		then we request
+	 */
+	PVR_ASSERT(uiSpanSize == psSyncBlock->ui32SyncBlockSize);
+
+	psImport->base = psSyncBlock->uiSpanBase;
+	psImport->uSize = psSyncBlock->ui32SyncBlockSize;
+	psImport->hPriv = psSyncBlock;
+	return PVRSRV_OK;
+
+fail_spanalloc:
+	FreeSyncPrimitiveBlock(psSyncBlock);
+fail_syncblockalloc:
+	_SyncPrimContextUnref(psContext);
+e0:
+	return eError;
+}
+
+static void
+SyncPrimBlockUnimport(RA_PERARENA_HANDLE hArena,
+                      RA_BASE_T uiBase,
+                      RA_PERISPAN_HANDLE hImport)
+{
+	SYNC_PRIM_CONTEXT *psContext = hArena;
+	SYNC_PRIM_BLOCK *psSyncBlock = hImport;
+
+	if (!psContext || !psSyncBlock || uiBase != psSyncBlock->uiSpanBase)
+	{
+		/* Invalid input params */
+		return;
+	}
+
+	/* Free the span this import is using */
+	RA_Free(psContext->psSpanRA, uiBase);
+
+	/* Free the syncpim block */
+	FreeSyncPrimitiveBlock(psSyncBlock);
+
+	/*	Drop our reference to the syncprim context */
+	_SyncPrimContextUnref(psContext);
+}
+
+static INLINE IMG_UINT32 SyncPrimGetOffset(SYNC_PRIM *psSyncInt)
+{
+	IMG_UINT64 ui64Temp = psSyncInt->uiSpanAddr - psSyncInt->psSyncBlock->uiSpanBase;
+
+	PVR_ASSERT(ui64Temp<IMG_UINT32_MAX);
+
+	return TRUNCATE_64BITS_TO_32BITS(ui64Temp);
+}
+
+static void SyncPrimGetCPULinAddr(SYNC_PRIM *psSyncInt)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock = psSyncInt->psSyncBlock;
+
+	psSyncInt->sCommon.pui32LinAddr = psSyncBlock->pui32LinAddr +
+			(SyncPrimGetOffset(psSyncInt)/sizeof(IMG_UINT32));
+}
+
+static void SyncPrimLocalFree(SYNC_PRIM *psSyncInt, IMG_BOOL bFreeFirstSyncPrim)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+#if !defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	PVR_UNREFERENCED_PARAMETER(bFreeFirstSyncPrim);
+#else
+	/* Defer freeing the first allocated sync prim in the sync context */
+	if (psSyncInt != psContext->hFirstSyncPrim || bFreeFirstSyncPrim)
+#endif
+	{
+		PVRSRV_ERROR eError;
+		SHARED_DEV_CONNECTION hDevConnection =
+			psSyncInt->psSyncBlock->psContext->hDevConnection;
+
+		if (GetInfoPageDebugFlags(hDevConnection) & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+		{
+			if (psSyncInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = DestroyServerResource(hDevConnection,
+				                               NULL,
+				                               BridgeSyncRecordRemoveByHandle,
+				                               psSyncInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "BridgeSyncRecordRemoveByHandle");
+			}
+		}
+		else
+		{
+			IMG_UINT32 ui32FWAddr = psSyncBlock->ui32FirmwareAddr +
+					SyncPrimGetOffset(psSyncInt);
+
+			eError = BridgeSyncFreeEvent(GetBridgeHandle(hDevConnection), ui32FWAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeSyncFreeEvent");
+		}
+#if defined(PVRSRV_ENABLE_SYNC_POISONING)
+		(void) _SyncPrimSetValue(psSyncInt, LOCAL_SYNC_PRIM_POISON_VALUE);
+#else
+		/* reset the sync prim value as it is freed.
+		 * this guarantees the client sync allocated to the client will
+		 * have a value of zero and the client does not need to
+		 * explicitly initialise the sync value to zero.
+		 * the allocation of the backing memory for the sync prim block
+		 * is done with ZERO_ON_ALLOC so the memory is initially all zero.
+		 */
+		(void) _SyncPrimSetValue(psSyncInt, LOCAL_SYNC_PRIM_RESET_VALUE);
+#endif
+
+		RA_Free(psContext->psSubAllocRA, psSyncInt->uiSpanAddr);
+		OSFreeMem(psSyncInt);
+		_SyncPrimContextUnref(psContext);
+	}
+}
+
+static void SyncPrimLocalUnref(SYNC_PRIM *psSyncInt)
+{
+	if (!OSAtomicRead(&psSyncInt->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncPrimLocalUnref sync already freed"));
+	}
+	else if (0 == OSAtomicDecrement(&psSyncInt->hRefCount))
+	{
+		SyncPrimLocalFree(psSyncInt, IMG_FALSE);
+	}
+}
+
+static IMG_UINT32 SyncPrimGetFirmwareAddrLocal(SYNC_PRIM *psSyncInt)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	return psSyncBlock->ui32FirmwareAddr + SyncPrimGetOffset(psSyncInt);
+}
+
+static INLINE IMG_UINT32 _Log2(IMG_UINT32 ui32Align)
+{
+	PVR_ASSERT(IsPower2(ui32Align));
+	return ExactLog2(ui32Align);
+}
+
+/*
+	External interfaces
+ */
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimContextCreate(SHARED_DEV_CONNECTION hDevConnection,
+                      PSYNC_PRIM_CONTEXT *phSyncPrimContext)
+{
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	psContext = OSAllocMem(sizeof(SYNC_PRIM_CONTEXT));
+	PVR_GOTO_IF_NOMEM(psContext, eError, fail_alloc);
+
+	psContext->hDevConnection = hDevConnection;
+
+	OSSNPrintf(psContext->azName, SYNC_PRIM_NAME_SIZE, "Sync Prim RA-%p", psContext);
+	OSSNPrintf(psContext->azSpanName, SYNC_PRIM_NAME_SIZE, "Sync Prim span RA-%p", psContext);
+
+	/*
+		Create the RA for sub-allocations of the SynPrim's
+
+		Note:
+		The import size doesn't matter here as the server will pass
+		back the blocksize when does the import which overrides
+		what we specify here.
+	 */
+
+	psContext->psSubAllocRA = RA_Create(psContext->azName,
+	                                    /* Params for imports */
+	                                    _Log2(sizeof(IMG_UINT32)),
+	                                    RA_LOCKCLASS_3,
+	                                    SyncPrimBlockImport,
+	                                    SyncPrimBlockUnimport,
+	                                    psContext,
+	                                    RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_NOMEM(psContext->psSubAllocRA, eError, fail_suballoc);
+
+	/*
+		Create the span-management RA
+
+		The RA requires that we work with linear spans. For our use
+		here we don't require this behaviour as we're always working
+		within offsets of blocks (imports). However, we need to keep
+		the RA happy so we create the "span" management RA which
+		ensures that all are imports are added to the RA in a linear
+		fashion
+	 */
+	psContext->psSpanRA = RA_Create(psContext->azSpanName,
+	                                /* Params for imports */
+	                                0,
+	                                RA_LOCKCLASS_0,
+	                                NULL,
+	                                NULL,
+	                                NULL,
+	                                RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_NOMEM(psContext->psSpanRA, eError, fail_span);
+
+	if (!RA_Add(psContext->psSpanRA, 0, MAX_SYNC_MEM, 0, NULL))
+	{
+		RA_Delete(psContext->psSpanRA);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, fail_span);
+	}
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	psContext->hFirstSyncPrim = NULL;
+#endif
+
+	OSAtomicWrite(&psContext->hRefCount, 1);
+
+	*phSyncPrimContext = psContext;
+	return PVRSRV_OK;
+fail_span:
+	RA_Delete(psContext->psSubAllocRA);
+fail_suballoc:
+	OSFreeMem(psContext);
+fail_alloc:
+	return eError;
+}
+
+IMG_INTERNAL void SyncPrimContextDestroy(PSYNC_PRIM_CONTEXT hSyncPrimContext)
+{
+	SYNC_PRIM_CONTEXT *psContext = hSyncPrimContext;
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	/* Free the first sync prim that was allocated as part of this context */
+	if (psContext->hFirstSyncPrim)
+	{
+		SyncPrimLocalFree((SYNC_PRIM *)psContext->hFirstSyncPrim, IMG_TRUE);
+		psContext->hFirstSyncPrim = NULL;
+	}
+#endif
+
+	if (1 != OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s attempted with active references, may be the result of a race", __func__));
+	}
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+#if defined(__KERNEL__)
+	if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Forcing context destruction due to bad driver state", __func__));
+		OSAtomicWrite(&psContext->hRefCount, 1);
+	}
+#endif
+#endif
+	_SyncPrimContextUnref(psContext);
+}
+
+static PVRSRV_ERROR _SyncPrimAlloc(PSYNC_PRIM_CONTEXT hSyncPrimContext,
+                                   PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+                                   const IMG_CHAR *pszClassName,
+                                   IMG_BOOL bServerSync)
+{
+	SYNC_PRIM_CONTEXT *psContext = hSyncPrimContext;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM *psNewSync;
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiSpanAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hSyncPrimContext, "hSyncPrimeContext");
+
+	psNewSync = OSAllocMem(sizeof(SYNC_PRIM));
+	PVR_GOTO_IF_NOMEM(psNewSync, eError, fail_alloc);
+
+	eError = RA_Alloc(psContext->psSubAllocRA,
+	                  sizeof(IMG_UINT32),
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  sizeof(IMG_UINT32),
+	                  "Sync_Prim",
+	                  &uiSpanAddr,
+	                  NULL,
+	                  (RA_PERISPAN_HANDLE *) &psSyncBlock);
+	PVR_GOTO_IF_ERROR(eError, fail_raalloc);
+
+	OSAtomicWrite(&psNewSync->hRefCount, 1);
+	psNewSync->uiSpanAddr = uiSpanAddr;
+	psNewSync->psSyncBlock = psSyncBlock;
+	SyncPrimGetCPULinAddr(psNewSync);
+	*ppsSync = &psNewSync->sCommon;
+	_SyncPrimContextRef(psContext);
+#if defined(PVRSRV_ENABLE_SYNC_POISONING)
+	(void) _SyncPrimSetValue(psNewSync, LOCAL_SYNC_PRIM_RESET_VALUE);
+#endif
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	/* If this is the first sync prim allocated in the context, keep a handle to it */
+	if (psSyncBlock->uiSpanBase == 0 && psNewSync->uiSpanAddr == 0)
+	{
+		psContext->hFirstSyncPrim = psNewSync;
+	}
+#endif
+
+	if (GetInfoPageDebugFlags(psSyncBlock->psContext->hDevConnection) & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		IMG_CHAR szClassName[PVRSRV_SYNC_NAME_LENGTH];
+		size_t uiSize;
+
+		if (pszClassName)
+		{
+			uiSize = OSStringNLength(pszClassName, PVRSRV_SYNC_NAME_LENGTH);
+			/* Copy the class name annotation into a fixed-size array */
+			OSCachedMemCopy(szClassName, pszClassName, uiSize);
+			if (uiSize == PVRSRV_SYNC_NAME_LENGTH)
+				szClassName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+			else
+				szClassName[uiSize++] = '\0';
+		}
+		else
+		{
+			/* No class name annotation */
+			uiSize = 0;
+			szClassName[0] = '\0';
+		}
+
+		/* record this sync */
+		eError = BridgeSyncRecordAdd(
+				GetBridgeHandle(psSyncBlock->psContext->hDevConnection),
+				&psNewSync->hRecord,
+				psSyncBlock->hServerSyncPrimBlock,
+				psSyncBlock->ui32FirmwareAddr,
+				SyncPrimGetOffset(psNewSync),
+				bServerSync,
+				uiSize,
+				szClassName);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to add SyncRecord \"%s\" (%s)",
+					__func__,
+					szClassName,
+					PVRSRVGETERRORSTRING(eError)));
+			psNewSync->hRecord = NULL;
+		}
+	}
+	else
+	{
+		size_t	uiSize;
+
+		uiSize = OSStringNLength(pszClassName, PVRSRV_SYNC_NAME_LENGTH);
+
+		if (uiSize < PVRSRV_SYNC_NAME_LENGTH)
+			uiSize++;
+		/* uiSize now reflects size used for pszClassName + NUL byte */
+
+		eError = BridgeSyncAllocEvent(GetBridgeHandle(hSyncPrimContext->hDevConnection),
+		                              bServerSync,
+		                              psSyncBlock->ui32FirmwareAddr + SyncPrimGetOffset(psNewSync),
+		                              uiSize,
+		                              pszClassName);
+		PVR_LOG_IF_ERROR(eError, "BridgeSyncAllocEvent");
+	}
+
+	return PVRSRV_OK;
+
+fail_raalloc:
+	OSFreeMem(psNewSync);
+fail_alloc:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimAlloc(PSYNC_PRIM_CONTEXT hSyncPrimContext,
+                                        PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+                                        const IMG_CHAR *pszClassName)
+{
+	return _SyncPrimAlloc(hSyncPrimContext,
+	                      ppsSync,
+	                      pszClassName,
+	                      IMG_FALSE);
+}
+
+static PVRSRV_ERROR
+_SyncPrimSetValue(SYNC_PRIM *psSyncInt, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimSet(GetBridgeHandle(psContext->hDevConnection),
+								psSyncBlock->hServerSyncPrimBlock,
+								SyncPrimGetOffset(psSyncInt)/sizeof(IMG_UINT32),
+								ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimFree(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	SyncPrimLocalUnref(psSyncInt);
+
+
+err_out:
+	return eError;
+}
+
+#if defined(NO_HARDWARE)
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimNoHwUpdate(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	eError =  _SyncPrimSetValue(psSyncInt, ui32Value);
+
+err_out:
+	return eError;
+}
+#endif
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimSet(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	eError = _SyncPrimSetValue(psSyncInt, ui32Value);
+
+#if defined(PDUMP)
+	SyncPrimPDump(psSync);
+#endif
+err_out:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimLocalGetHandleAndOffset(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                                          IMG_HANDLE *phBlock,
+                                                          IMG_UINT32 *pui32Offset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(phBlock, eError, err_out);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(pui32Offset, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	*phBlock = psSyncInt->psSyncBlock->hServerSyncPrimBlock;
+	*pui32Offset = psSyncInt->uiSpanAddr - psSyncInt->psSyncBlock->uiSpanBase;
+
+err_out:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimGetFirmwareAddr(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 *pui32FwAddr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	*pui32FwAddr = 0;
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+	*pui32FwAddr = SyncPrimGetFirmwareAddrLocal(psSyncInt);
+
+err_out:
+	return eError;
+}
+
+#if defined(PDUMP)
+IMG_INTERNAL void SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDump(GetBridgeHandle(psContext->hDevConnection),
+	                             psSyncBlock->hServerSyncPrimBlock,
+	                             SyncPrimGetOffset(psSyncInt));
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDump");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDumpValue(GetBridgeHandle(psContext->hDevConnection),
+	                                  psSyncBlock->hServerSyncPrimBlock,
+	                                  SyncPrimGetOffset(psSyncInt),
+	                                  ui32Value);
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpValue");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                   IMG_UINT32 ui32Value,
+                                   IMG_UINT32 ui32Mask,
+                                   PDUMP_POLL_OPERATOR eOperator,
+                                   IMG_UINT32 ui32PDumpFlags)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDumpPol(GetBridgeHandle(psContext->hDevConnection),
+	                                psSyncBlock->hServerSyncPrimBlock,
+	                                SyncPrimGetOffset(psSyncInt),
+	                                ui32Value,
+	                                ui32Mask,
+	                                eOperator,
+	                                ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpPol");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                   IMG_UINT64 uiWriteOffset,
+                                   IMG_UINT64 uiPacketSize,
+                                   IMG_UINT64 uiBufferSize)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+#if defined(__linux__) && defined(__i386__)
+	PVR_ASSERT(uiWriteOffset<IMG_UINT32_MAX);
+	PVR_ASSERT(uiPacketSize<IMG_UINT32_MAX);
+	PVR_ASSERT(uiBufferSize<IMG_UINT32_MAX);
+#endif
+	eError = BridgeSyncPrimPDumpCBP(GetBridgeHandle(psContext->hDevConnection),
+	                                psSyncBlock->hServerSyncPrimBlock,
+	                                SyncPrimGetOffset(psSyncInt),
+	                                TRUNCATE_64BITS_TO_32BITS(uiWriteOffset),
+	                                TRUNCATE_64BITS_TO_32BITS(uiPacketSize),
+	                                TRUNCATE_64BITS_TO_32BITS(uiBufferSize));
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpCBP");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/tlclient.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/tlclient.c
new file mode 100644
index 000000000000..a72f8118483b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/tlclient.c
@@ -0,0 +1,499 @@
+/*************************************************************************/ /*!
+@File           tlclient.c
+@Title          Services Transport Layer shared API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common API used in both clients and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* DESIGN NOTE
+ * This transport layer consumer-role API was created as a shared API when a
+ * client wanted to read the data of a TL stream from within the KM server
+ * driver. This was in addition to the existing clients supported externally
+ * by the UM client library component via PVR API layer.
+ * This shared API is thus used by the PVR TL API in the client library and
+ * by clients internal to the server driver module. It depends on
+ * client entry points of the TL and DEVMEM bridge modules. These entry points
+ * encapsulate from the TL shared API whether a direct bridge or an indirect
+ * (ioctl) bridge is used.
+ * One reason for needing this layer centres around the fact that some of the
+ * API functions make multiple bridge calls and the logic that glues these
+ * together is common regardless of client location. Further this layer has
+ * allowed the defensive coding that checks parameters to move into the PVR
+ * API layer where untrusted clients enter giving a more efficient KM code path.
+ */
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+
+#include "allocmem.h"
+#include "devicemem.h"
+
+#include "tlclient.h"
+#include "pvrsrv_tlcommon.h"
+#include "client_pvrtl_bridge.h"
+
+#if defined(__KERNEL__)
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+/* Defines/Constants
+ */
+
+#define NO_ACQUIRE             0xffffffffU
+
+/* User-side stream descriptor structure.
+ */
+typedef struct _TL_STREAM_DESC_
+{
+	/* Handle on kernel-side stream descriptor*/
+	IMG_HANDLE		hServerSD;
+
+	/* Stream data buffer variables */
+	DEVMEM_MEMDESC*			psUMmemDesc;
+	IMG_PBYTE				pBaseAddr;
+
+	/* Offset in bytes into the circular buffer and valid only after
+	 * an Acquire call and undefined after a release. */
+	IMG_UINT32	uiReadOffset;
+
+	/* Always a positive integer when the Acquire call returns and a release
+	 * is outstanding. Undefined at all other times. */
+	IMG_UINT32	uiReadLen;
+
+	/* Counter indicating how many writes to a stream failed.
+	 * It's used to reduce number of errors in output log. */
+	IMG_UINT32 ui32WritesFailed;
+
+	/* Name of the stream. */
+	IMG_CHAR szName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+} TL_STREAM_DESC, *PTL_STREAM_DESC;
+
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientOpenStream(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR* pszName,
+		IMG_UINT32   ui32Mode,
+		IMG_HANDLE*  phSD)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	TL_STREAM_DESC *psSD = NULL;
+	IMG_HANDLE hTLPMR;
+	IMG_HANDLE hTLImportHandle;
+	IMG_DEVMEM_SIZE_T uiImportSize;
+	PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(pszName);
+	PVR_ASSERT(phSD);
+	*phSD = NULL;
+
+	/* Allocate memory for the stream descriptor object, initialise with
+	 * "no data read" yet. */
+	psSD = OSAllocZMem(sizeof(TL_STREAM_DESC));
+	PVR_LOG_GOTO_IF_NOMEM(psSD, eError, e0);
+	psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+
+	/* Send open stream request to kernel server to get stream handle and
+	 * buffer cookie so we can get access to the buffer in this process. */
+	eError = BridgeTLOpenStream(GetBridgeHandle(hDevConnection), pszName,
+			ui32Mode, &psSD->hServerSD, &hTLPMR);
+	if (eError != PVRSRV_OK)
+	{
+		if ((ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WAIT) &&
+			(eError == PVRSRV_ERROR_TIMEOUT))
+		{
+			goto e1;
+		}
+		PVR_LOG_GOTO_IF_ERROR(eError, "BridgeTLOpenStream", e1);
+	}
+
+	/* Convert server export cookie into a cookie for use by this client */
+	eError = DevmemMakeLocalImportHandle(hDevConnection,
+			hTLPMR, &hTLImportHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemMakeLocalImportHandle", e2);
+
+	uiMemFlags |= ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	        PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0ULL;
+	/* Now convert client cookie into a client handle on the buffer's
+	 * physical memory region */
+	eError = DevmemLocalImport(hDevConnection,
+	                           hTLImportHandle,
+	                           uiMemFlags,
+	                           &psSD->psUMmemDesc,
+	                           &uiImportSize,
+	                           "TLBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemImport", e3);
+
+	/* Now map the memory into the virtual address space of this process. */
+	eError = DevmemAcquireCpuVirtAddr(psSD->psUMmemDesc, (void **)
+															&psSD->pBaseAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", e4);
+
+	/* Ignore error, not much that can be done */
+	(void) DevmemUnmakeLocalImportHandle(hDevConnection,
+			hTLImportHandle);
+
+	/* Copy stream name */
+	OSStringSafeCopy(psSD->szName, pszName, PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	/* Return client descriptor handle to caller */
+	*phSD = psSD;
+	return PVRSRV_OK;
+
+/* Clean up post buffer setup */
+e4:
+	DevmemFree(psSD->psUMmemDesc);
+e3:
+	(void) DevmemUnmakeLocalImportHandle(hDevConnection,
+				&hTLImportHandle);
+/* Clean up post stream open */
+e2:
+	BridgeTLCloseStream(GetBridgeHandle(hDevConnection), psSD->hServerSD);
+
+/* Clean up post allocation of the descriptor object */
+e1:
+	OSFreeMem(psSD);
+
+e0:
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCloseStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	/* Check the caller provided connection is valid */
+	if (!psSD->hServerSD)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: descriptor already "
+				"closed/not open", __func__));
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	/* Check if acquire is outstanding, perform release if it is, ignore result
+	 * as there is not much we can do if it is an error other than close */
+	if (psSD->uiReadLen != NO_ACQUIRE)
+	{
+		(void) BridgeTLReleaseData(GetBridgeHandle(hDevConnection),
+				psSD->hServerSD, psSD->uiReadOffset, psSD->uiReadLen);
+		psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+	}
+
+	/* Clean up DevMem resources used for this stream in this client */
+	DevmemReleaseCpuVirtAddr(psSD->psUMmemDesc);
+
+	DevmemFree(psSD->psUMmemDesc);
+
+	/* Send close to server to clean up kernel mode resources for this
+	 * handle and release the memory. */
+	eError = DestroyServerResource(hDevConnection,
+	                               NULL,
+	                               BridgeTLCloseStream,
+	                               psSD->hServerSD);
+	PVR_LOG_IF_ERROR(eError, "BridgeTLCloseStream");
+
+	if (psSD->ui32WritesFailed != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s() %u writes failed to stream %s (%c)",
+		        __func__, psSD->ui32WritesFailed, psSD->szName,
+		        psSD->ui32WritesFailed == IMG_UINT32_MAX ? 'T' : 'F'));
+	}
+
+	OSCachedMemSet(psSD, 0x00, sizeof(TL_STREAM_DESC));
+	OSFreeMem(psSD);
+
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientDiscoverStreams(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR *pszNamePattern,
+		IMG_CHAR aszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+		IMG_UINT32 *pui32NumFound)
+{
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(pszNamePattern);
+	PVR_ASSERT(pui32NumFound);
+
+	return BridgeTLDiscoverStreams(GetBridgeHandle(hDevConnection),
+			pszNamePattern,
+			/* we need to treat this as one dimensional array */
+			*pui32NumFound * PRVSRVTL_MAX_STREAM_NAME_SIZE,
+			(IMG_CHAR *) aszStreams,
+			pui32NumFound);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+	IMG_UINT32 ui32BufferOffset, ui32Unused;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppui8Data);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLReserveStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &ui32BufferOffset, ui32Size, ui32Size, &ui32Unused);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppui8Data = psSD->pBaseAddr + ui32BufferOffset;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream2(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size,
+		IMG_UINT32 ui32SizeMin,
+		IMG_UINT32 *pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+	IMG_UINT32 ui32BufferOffset;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppui8Data);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLReserveStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &ui32BufferOffset, ui32Size, ui32SizeMin,
+			pui32Available);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppui8Data = psSD->pBaseAddr + ui32BufferOffset;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCommitStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLCommitStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, ui32Size);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientAcquireData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE  hSD,
+		IMG_PBYTE*  ppPacketBuf,
+		IMG_UINT32* pui32BufLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppPacketBuf);
+	PVR_ASSERT(pui32BufLen);
+
+	/* In case of non-blocking acquires, which can return no data, and
+	 * error paths ensure we clear the output parameters first. */
+	*ppPacketBuf = NULL;
+	*pui32BufLen = 0;
+
+	/* Check Acquire has not been called twice in a row without a release */
+	if (psSD->uiReadOffset != NO_ACQUIRE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: acquire already "
+				"outstanding, ReadOffset(%d), ReadLength(%d)",
+				__func__, psSD->uiReadOffset, psSD->uiReadLen));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Ask the kernel server for the next chunk of data to read */
+	eError = BridgeTLAcquireData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &psSD->uiReadOffset, &psSD->uiReadLen);
+	if (eError != PVRSRV_OK)
+	{
+		/* Mask reporting of the errors seen under normal operation */
+		if ((eError != PVRSRV_ERROR_TIMEOUT) &&
+			(eError != PVRSRV_ERROR_STREAM_READLIMIT_REACHED))
+		{
+			PVR_LOG_ERROR(eError, "BridgeTLAcquireData");
+		}
+		psSD->uiReadOffset = psSD->uiReadLen = NO_ACQUIRE;
+		return eError;
+	}
+	/* else PVRSRV_OK */
+
+	/* Return the data offset and length to the caller if bytes are available
+	 * to be read. Could be zero for non-blocking mode so pass back cleared
+	 * values above */
+	if (psSD->uiReadLen)
+	{
+		*ppPacketBuf = psSD->pBaseAddr + psSD->uiReadOffset;
+		*pui32BufLen = psSD->uiReadLen;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _TLClientReleaseDataLen(
+		SHARED_DEV_CONNECTION hDevConnection,
+		TL_STREAM_DESC* psSD,
+		IMG_UINT32 uiReadLen)
+{
+	PVRSRV_ERROR eError;
+
+	/* the previous acquire did not return any data, this is a no-operation */
+	if (psSD->uiReadLen == 0)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Check release has not been called twice in a row without an acquire */
+	if (psSD->uiReadOffset == NO_ACQUIRE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: no acquire to release", __func__));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Inform the kernel to release the data from the buffer */
+	eError = BridgeTLReleaseData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD,
+			psSD->uiReadOffset, uiReadLen);
+	PVR_LOG_IF_ERROR(eError, "BridgeTLReleaseData");
+
+	/* Reset state to indicate no outstanding acquire */
+	psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD)
+{
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	return _TLClientReleaseDataLen(hDevConnection, psSD, psSD->uiReadLen);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseDataLess(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD, IMG_UINT32 uiActualReadLen)
+{
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	/* Check the specified size is within the size returned by Acquire */
+	if (uiActualReadLen > psSD->uiReadLen)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: no acquire to release", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return _TLClientReleaseDataLen(hDevConnection, psSD, uiActualReadLen);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientWriteData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size,
+		IMG_BYTE *pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ui32Size);
+	PVR_ASSERT(pui8Data);
+
+	eError = BridgeTLWriteData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, ui32Size, pui8Data);
+
+	if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		if (psSD->ui32WritesFailed == 0)
+		{
+			PVR_LOG_ERROR(eError, "BridgeTLWriteData");
+		}
+		if (psSD->ui32WritesFailed != IMG_UINT32_MAX)
+		{
+			psSD->ui32WritesFailed++;
+		}
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "BridgeTLWriteData");
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (tlclient.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.c b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.c
new file mode 100644
index 000000000000..2d4f2fa8ce66
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.c
@@ -0,0 +1,280 @@
+/*************************************************************************/ /*!
+@File
+@Title          Provides splay-trees.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implementation of splay-trees.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "allocmem.h" /* for OSMemAlloc / OSMemFree */
+#include "osfunc.h" /* for OSMemFree */
+#include "pvr_debug.h"
+#include "uniq_key_splay_tree.h"
+
+/**
+ * PVRSRVSplay - perform a simple top down splay
+ * @ui32Flags: flags that must splayed to the root (if possible)
+ * @psTree:    psTree The tree to splay.
+ *
+ * Return the resulting tree after the splay operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVSplay (IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_SPLAY_TREE sTmp1;
+	IMG_PSPLAY_TREE psLeft;
+	IMG_PSPLAY_TREE psRight;
+	IMG_PSPLAY_TREE psTmp2;
+
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	sTmp1.psLeft = NULL;
+	sTmp1.psRight = NULL;
+
+	psLeft = &sTmp1;
+	psRight = &sTmp1;
+
+	for (;;)
+	{
+		if (uiFlags < psTree->uiFlags)
+		{
+			if (psTree->psLeft == NULL)
+			{
+				break;
+			}
+
+			if (uiFlags < psTree->psLeft->uiFlags)
+			{
+				/* if we get to this point, we need to rotate right the tree */
+				psTmp2 = psTree->psLeft;
+				psTree->psLeft = psTmp2->psRight;
+				psTmp2->psRight = psTree;
+				psTree = psTmp2;
+				if (psTree->psLeft == NULL)
+				{
+					break;
+				}
+			}
+
+			/* if we get to this point, we need to link right */
+			psRight->psLeft = psTree;
+			psRight = psTree;
+			psTree = psTree->psLeft;
+		}
+		else
+		{
+			if (uiFlags > psTree->uiFlags)
+			{
+				if (psTree->psRight == NULL)
+				{
+					break;
+				}
+
+				if (uiFlags > psTree->psRight->uiFlags)
+				{
+					/* if we get to this point, we need to rotate left the tree */
+					psTmp2 = psTree->psRight;
+					psTree->psRight = psTmp2->psLeft;
+					psTmp2->psLeft = psTree;
+					psTree = psTmp2;
+					if (psTree->psRight == NULL)
+					{
+						break;
+					}
+				}
+
+				/* if we get to this point, we need to link left */
+				psLeft->psRight = psTree;
+				psLeft = psTree;
+				psTree = psTree->psRight;
+			}
+			else
+			{
+				break;
+			}
+		}
+	}
+
+	/* at this point re-assemble the tree */
+	psLeft->psRight = psTree->psLeft;
+	psRight->psLeft = psTree->psRight;
+	psTree->psLeft = sTmp1.psRight;
+	psTree->psRight = sTmp1.psLeft;
+	return psTree;
+}
+
+
+/**
+ * PVRSRVInsert - insert a node into the Tree (unless it is already present, in
+ * which case it is equivalent to performing only a splay operation
+ * @ui32Flags: the key of the new node
+ * @psTree:    tree into which one wants to add a new node
+ *
+ * Return the resulting tree after the splay operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVInsert(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_PSPLAY_TREE psNew;
+
+	if (psTree != NULL)
+	{
+		psTree = PVRSRVSplay(uiFlags, psTree);
+		if (psTree->uiFlags == uiFlags)
+		{
+			return psTree;
+		}
+	}
+
+	psNew = (IMG_PSPLAY_TREE) OSAllocMem(sizeof(IMG_SPLAY_TREE));
+	if (psNew == NULL)
+	{
+		PVR_DPF ((PVR_DBG_ERROR, "Error: failed to allocate memory to add a node to the splay tree."));
+		return NULL;
+	}
+
+	psNew->uiFlags = uiFlags;
+	OSCachedMemSet(&(psNew->buckets[0]), 0, sizeof(psNew->buckets));
+
+#if defined(PVR_CTZLL)
+	psNew->bHasEltsMapping = ~(((IMG_ELTS_MAPPINGS) 1 << (sizeof(psNew->buckets) / (sizeof(psNew->buckets[0])))) - 1);
+#endif
+
+	if (psTree == NULL)
+	{
+		psNew->psLeft  = NULL;
+		psNew->psRight = NULL;
+		return psNew;
+	}
+
+	if (uiFlags < psTree->uiFlags)
+	{
+		psNew->psLeft  = psTree->psLeft;
+		psNew->psRight = psTree;
+		psTree->psLeft = NULL;
+	}
+	else
+	{
+		psNew->psRight  = psTree->psRight;
+		psNew->psLeft   = psTree;
+		psTree->psRight = NULL;
+	}
+
+	return psNew;
+}
+
+
+/**
+ * PVRSRVDelete - delete a node from the tree (unless it is not there, in which
+ * case it is equivalent to a splay operation)
+ * @ui32Flags: value of the node to remove
+ * @psTree:    tree into which the node must be removed
+ *
+ * Return the resulting tree.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVDelete(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_PSPLAY_TREE psTmp;
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	psTree = PVRSRVSplay(uiFlags, psTree);
+	if (uiFlags == psTree->uiFlags)
+	{
+		/* The value was present in the tree */
+		if (psTree->psLeft == NULL)
+		{
+			psTmp = psTree->psRight;
+		}
+		else
+		{
+			psTmp = PVRSRVSplay(uiFlags, psTree->psLeft);
+			psTmp->psRight = psTree->psRight;
+		}
+		OSFreeMem(psTree);
+		return psTmp;
+	}
+
+	/* The value was not present in the tree, so just return it as is
+	 * (after the splay) */
+	return psTree;
+}
+
+/**
+ * PVRSRVFindNode - pick up the appropriate node for the given flags
+ * @ui32Flags: flags that must associated with the node
+ * @psTree:    current splay tree node
+ *
+ * Return the resulting tree node after the search operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVFindNode(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	while (psTree)
+	{
+		if (uiFlags == psTree->uiFlags)
+		{
+			return psTree;
+		}
+
+		if (uiFlags < psTree->uiFlags)
+		{
+			psTree = psTree->psLeft;
+			continue;
+		}
+
+		if (uiFlags > psTree->uiFlags)
+		{
+			psTree = psTree->psRight;
+			continue;
+		}
+	}
+
+	return NULL;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.h b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.h
new file mode 100644
index 000000000000..75ec9297e9c2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/common/uniq_key_splay_tree.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          Splay trees interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides debug functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef UNIQ_KEY_SPLAY_TREE_H_
+#define UNIQ_KEY_SPLAY_TREE_H_
+
+#include "img_types.h"
+#include "pvr_intrinsics.h"
+
+#if defined(PVR_CTZLL)
+  /* map the is_bucket_n_free to an int.
+   * This way, the driver can find the first non empty without loop
+   */
+  typedef IMG_UINT64 IMG_ELTS_MAPPINGS;
+#endif
+
+typedef IMG_UINT64 IMG_PSPLAY_FLAGS_T;
+
+/* head of list of free boundary tags for indexed by pvr_log2 of the
+   boundary tag size */
+
+#define FREE_TABLE_LIMIT 40
+
+struct _BT_;
+
+typedef struct img_splay_tree
+{
+	/* left child/subtree */
+    struct img_splay_tree * psLeft;
+
+	/* right child/subtree */
+    struct img_splay_tree * psRight;
+
+    /* Flags to match on this span, used as the key. */
+    IMG_PSPLAY_FLAGS_T uiFlags;
+#if defined(PVR_CTZLL)
+	/* each bit of this int is a boolean telling if the corresponding
+	   bucket is empty or not */
+    IMG_ELTS_MAPPINGS bHasEltsMapping;
+#endif
+	struct _BT_ * buckets[FREE_TABLE_LIMIT];
+} IMG_SPLAY_TREE, *IMG_PSPLAY_TREE;
+
+IMG_PSPLAY_TREE PVRSRVSplay (IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVInsert(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVDelete(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVFindNode(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+
+
+#endif /* !UNIQ_KEY_SPLAY_TREE_H_ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.c b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.c
new file mode 100644
index 000000000000..78d731c98eb6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.c
@@ -0,0 +1,629 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance counter table
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance counters table
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(__KERNEL__)
+#include "rgxdefs_km.h"
+#else
+#include "rgxdefs.h"
+#endif
+#include "rgx_hwperf_table.h"
+
+/* Includes needed for PVRSRVKM (Server) context */
+#	include "rgx_bvnc_defs_km.h"
+#	if defined(__KERNEL__)
+#		include "rgxdevice.h"
+#	endif
+
+/* Shared compile-time context ASSERT macro */
+#if defined(RGX_FIRMWARE)
+#	include "rgxfw_utils.h"
+/* firmware context */
+#	define DBG_ASSERT(_c) RGXFW_ASSERT((_c))
+#else
+#	include "pvr_debug.h"
+/* host client/server context */
+#	define DBG_ASSERT(_c) PVR_ASSERT((_c))
+#endif
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered()
+
+ Referenced in gasCntBlkTypeModel[] table below and only called from
+ RGX_FIRMWARE run-time context. Therefore compile time configuration is used.
+ *****************************************************************************/
+
+#if defined(RGX_FIRMWARE) && defined(RGX_FEATURE_PERFBUS)
+#	include "rgxfw_pow.h"
+#	include "rgxfw_utils.h"
+
+static bool rgxfw_hwperf_pow_st_direct(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	PVR_UNREFERENCED_PARAMETER(eBlkType);
+	PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+	/* S6XT: TA, TORNADO */
+	return true;
+#else
+	/* S6  : TA, HUB, RASTER (RASCAL) */
+	return rgxfw_pow_is_rd_on();
+#endif
+}
+
+/* Only use conditional compilation when counter blocks appear in different
+ * islands for different Rogue families.
+ */
+static bool rgxfw_hwperf_pow_st_indirect(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	IMG_UINT32 ui32NumDustsEnabled = rgxfw_pow_get_enabled_units();
+
+	if (rgxfw_pow_is_rd_on() && (ui32NumDustsEnabled > 0U))
+	{
+#if defined(RGX_FEATURE_DYNAMIC_DUST_POWER)
+		IMG_UINT32 ui32NumUscEnabled = ui32NumDustsEnabled*2U;
+
+		switch (eBlkType)
+		{
+		case RGX_CNTBLK_ID_TPU_MCU0:                   /* S6 and S6XT */
+			if (ui8UnitId >= ui32NumDustsEnabled)
+			{
+				return false;
+			}
+			break;
+		case RGX_CNTBLK_ID_USC0:                       /* S6, S6XT, S7 */
+		case RGX_CNTBLK_ID_PBE0:                       /* S7, PBE2_IN_XE */
+			/* Handle single cluster cores */
+			if (ui8UnitId >= ((ui32NumUscEnabled > RGX_FEATURE_NUM_CLUSTERS) ? RGX_FEATURE_NUM_CLUSTERS : ui32NumUscEnabled))
+			{
+				return false;
+			}
+			break;
+		case RGX_CNTBLK_ID_BLACKPEARL0:                /* S7 */
+		case RGX_CNTBLK_ID_RASTER0:                    /* S6XT */
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+		case RGX_CNTBLK_ID_TEXAS0:                     /* S6XT */
+#endif
+			if (ui8UnitId >= (RGX_REQ_NUM_PHANTOMS(ui32NumUscEnabled)))
+			{
+				return false;
+			}
+			break;
+		default:
+			RGXFW_ASSERT(false); /* should never get here, table error */
+			break;
+		}
+#else
+		/* Always true, no fused DUSTs, all powered so do not check unit */
+		PVR_UNREFERENCED_PARAMETER(eBlkType);
+		PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+#endif
+	}
+	else
+	{
+		return false;
+	}
+	return true;
+}
+
+#else /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+# define rgxfw_hwperf_pow_st_direct   ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+# define rgxfw_hwperf_pow_st_indirect ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+
+#endif /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered() end
+ *****************************************************************************/
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() start
+
+ Referenced in gasCntBlkTypeModel[] table below and called from all build
+ contexts:
+ RGX_FIRMWARE, PVRSRVCTL (UM) and PVRSRVKM (Server).
+
+ Therefore each function has two implementations, one for compile time and one
+ run time configuration depending on the context. The functions will inform the
+ caller whether this block is valid for this particular RGX device. Other
+ run-time dependent data is returned in psRtInfo for the caller to use.
+ *****************************************************************************/
+
+/* Used for block types: USC */
+static IMG_BOOL rgx_hwperf_blk_present_perfbus(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_USC0);
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+		{
+			psRtInfo->ui32NumUnits = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) ? RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) : 0;
+			psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+			return IMG_TRUE;
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+# if defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: Direct RASTERISATION, HUB */
+static IMG_BOOL rgx_hwperf_blk_present_not_clustergrouping(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_HUB));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if ((!RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING)) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS)))
+		{
+			psRtInfo->ui32NumUnits = 1;
+			psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+			return IMG_TRUE;
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+#if defined(__KERNEL__) /* Server context */
+static IMG_UINT32 rgx_units_indirect_by_phantom(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_INDIRECT_BY_PHANTOM */
+	return ((psFeatCfg->ui64Features & RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK) == 0) ? 1
+			: (psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX]+3)/4;
+}
+
+static IMG_UINT32 rgx_units_phantom_indirect_by_dust(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST */
+	return MAX((psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX]>>1),1);
+}
+
+static IMG_UINT32 rgx_units_phantom_indirect_by_cluster(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER */
+	return psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX];
+}
+#endif /* defined(__KERNEL__) */
+
+/* Used for block types: TORNADO, TEXAS, Indirect RASTERISATION */
+static IMG_BOOL rgx_hwperf_blk_present_xttop(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TORNADO) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER0));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TORNADO)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TEXAS_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: JONES, TPU_MCU, TEXAS, BLACKPEARL, PBE */
+static IMG_BOOL rgx_hwperf_blk_present_s7top(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_JONES) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_BLACKPEARL0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0));
+
+#if defined(__KERNEL__) /* Server context */
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE_IDX)
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TPU_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TEXAS3_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_BLACKPEARL0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_cluster(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_JONES)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+		}
+	}
+#endif
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return IMG_TRUE;
+# else
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: TA, TPU_MCU. Also PBE when PBE2_IN_XE is present */
+static IMG_BOOL rgx_hwperf_blk_present_not_s7top(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TA) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TA)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+			{
+				if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+				{
+					/* PBE counters are not present on this config */
+					return IMG_FALSE;
+				}
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_cluster(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TPU_MCU_L0_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS)
+#  if !defined(RGX_FEATURE_PBE2_IN_XE)
+	if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+	{
+		/* No support for PBE counters without PBE2_IN_XE */
+		return IMG_FALSE;
+	}
+#  endif
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+static IMG_BOOL rgx_hwperf_blk_present_check_s7top_or_not(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+#if defined(__KERNEL__)
+	return (rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo)
+	     || rgx_hwperf_blk_present_not_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo));
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+
+#elif defined(RGX_FEATURE_PBE2_IN_XE) || defined(RGX_FEATURE_PERFBUS)
+	return rgx_hwperf_blk_present_not_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	return IMG_FALSE;
+#endif
+}
+
+static IMG_BOOL rgx_hwperf_blk_present_check_s7top_or_xttop(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+#if defined(__KERNEL__)
+	return (rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo)
+	     || rgx_hwperf_blk_present_xttop(psBlkTypeDesc, pvDev_km, pvRtInfo));
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+
+#elif defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_xttop(psBlkTypeDesc, pvDev_km, pvRtInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	return IMG_FALSE;
+#endif
+}
+
+#if !defined(__KERNEL__) /* Firmware or User-mode context */
+static IMG_BOOL rgx_hwperf_blk_present_false(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+
+	/* Some functions not used on some BVNCs, silence compiler warnings */
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_perfbus);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_not_clustergrouping);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_xttop);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_s7top);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_not_s7top);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_check_s7top_or_not);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_check_s7top_or_xttop);
+
+	return IMG_FALSE;
+}
+
+/* Used to instantiate a null row in the block type model table below where the
+ * block is not supported for a given build BVNC in firmware/user mode context.
+ * This is needed as the blockid to block type lookup uses the table as well
+ * and clients may try to access blocks not in the hardware. */
+#define RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(_blkid) {_blkid, 0, 0, 0, 0, 0, 0, 0, 0, #_blkid, NULL, rgx_hwperf_blk_present_false}
+
+#endif
+
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() end
+ *****************************************************************************/
+
+#if defined(__KERNEL__) /* Values will be calculated at run-time */
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+#define RGX_INDIRECT_REG_TEXAS 0xFFFFFFFF
+#define RGX_INDIRECT_REG_TPU 0xFFFFFFFF
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST
+#define RGX_INDIRECT_REG_TEXAS RGX_CR_TEXAS3_PERF_INDIRECT
+#define RGX_INDIRECT_REG_TPU RGX_CR_TPU_PERF_INDIRECT
+
+#else
+
+#if defined(RGX_FEATURE_PERFBUS)
+#define RGX_INDIRECT_REG_TPU RGX_CR_TPU_MCU_L0_PERF_INDIRECT
+#endif
+
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_INDIRECT_BY_PHANTOM
+#define RGX_INDIRECT_REG_TEXAS RGX_CR_TEXAS_PERF_INDIRECT
+#endif
+
+#endif
+
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] table
+
+ This table holds the entries for the performance counter block type model.
+ Where the block is not present on an RGX device in question the
+ pfnIsBlkPresent() returns false, if valid and present it returns true.
+ Columns in the table with a ** indicate the value is a default and the
+ value returned in RGX_HWPERF_CNTBLK_RT_INFO when calling pfnIsBlkPresent()
+ should be used at runtime by the caller. These columns are only valid for
+ compile time BVNC configured contexts.
+
+ Order of table rows must match order of counter block IDs in the enumeration
+ RGX_HWPERF_CNTBLK_ID.
+ *****************************************************************************/
+
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] =
+{
+		/*   ui32CntBlkIdBase,         ui32IndirectReg,                  ui32PerfReg,                  ui32Select0BaseReg,                    ui32Counter0BaseReg                   ui8NumCounters,  ui32NumUnits**,                  ui8SelectRegModeShift, ui8SelectRegOffsetShift,            pfnIsBlkPowered               pfnIsBlkPresent
+		 *                                                                                                                                                                                                                                                   pszBlockNameComment,  */
+		/*RGX_CNTBLK_ID_TA*/
+#if !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_TA,       0, /* direct */                RGX_CR_TA_PERF,             RGX_CR_TA_PERF_SELECT0,              RGX_CR_TA_PERF_COUNTER_0,             4,              1,                              21,                  3,  "RGX_CR_TA_PERF",              rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TA),
+#endif
+
+		/*RGX_CNTBLK_ID_RASTER*/
+#if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_RASTER,   0, /* direct */                RGX_CR_RASTERISATION_PERF,  RGX_CR_RASTERISATION_PERF_SELECT0,   RGX_CR_RASTERISATION_PERF_COUNTER_0,  4,              1,                              21,                  3,  "RGX_CR_RASTERISATION_PERF",   rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_clustergrouping },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_RASTER),
+#endif
+
+		/*RGX_CNTBLK_ID_HUB*/
+#if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_HUB,      0, /* direct */                RGX_CR_HUB_BIFPMCACHE_PERF, RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0,  RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0, 4,              1,                              21,                  3,  "RGX_CR_HUB_BIFPMCACHE_PERF",  rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_clustergrouping },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_HUB),
+#endif
+
+		/*RGX_CNTBLK_ID_TORNADO*/
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_TORNADO,  0, /* direct */                RGX_CR_TORNADO_PERF,        RGX_CR_TORNADO_PERF_SELECT0,         RGX_CR_TORNADO_PERF_COUNTER_0,        4,              1,                              21,                  4,  "RGX_CR_TORNADO_PERF",         rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TORNADO),
+#endif
+
+		/*RGX_CNTBLK_ID_JONES*/
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_JONES,   0, /* direct */                 RGX_CR_JONES_PERF,          RGX_CR_JONES_PERF_SELECT0,           RGX_CR_JONES_PERF_COUNTER_0,          4,              1,                              21,                  3,  "RGX_CR_JONES_PERF",           rgxfw_hwperf_pow_st_direct,    rgx_hwperf_blk_present_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_JONES),
+#endif
+
+		/*RGX_CNTBLK_ID_TPU_MCU0*/
+#if defined(__KERNEL__) || (defined(RGX_FEATURE_PERFBUS) && !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+		{RGX_CNTBLK_ID_TPU_MCU0, RGX_INDIRECT_REG_TPU,      RGX_CR_TPU_MCU_L0_PERF,   RGX_CR_TPU_MCU_L0_PERF_SELECT0,     RGX_CR_TPU_MCU_L0_PERF_COUNTER_0,     4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST,    21,          3,  "RGX_CR_TPU_MCU_L0_PERF",      rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_not },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TPU_MCU0),
+#endif
+
+		/*RGX_CNTBLK_ID_USC0*/
+#if defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_USC0,    RGX_CR_USC_PERF_INDIRECT,       RGX_CR_USC_PERF,            RGX_CR_USC_PERF_SELECT0,            RGX_CR_USC_PERF_COUNTER_0,            4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER, 21,          3,  "RGX_CR_USC_PERF",             rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_perfbus },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_USC0),
+#endif
+
+		/*RGX_CNTBLK_ID_TEXAS0*/
+#if defined(__KERNEL__) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+		{RGX_CNTBLK_ID_TEXAS0,  RGX_INDIRECT_REG_TEXAS,      RGX_CR_TEXAS_PERF,          RGX_CR_TEXAS_PERF_SELECT0,          RGX_CR_TEXAS_PERF_COUNTER_0,          6,              RGX_HWPERF_NUM_BLOCK_UNITS,             31,          3,  "RGX_CR_TEXAS_PERF",           rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TEXAS0),
+#endif
+
+		/*RGX_CNTBLK_ID_RASTER0*/
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_RASTER0, RGX_CR_RASTERISATION_PERF_INDIRECT, RGX_CR_RASTERISATION_PERF, RGX_CR_RASTERISATION_PERF_SELECT0, RGX_CR_RASTERISATION_PERF_COUNTER_0,  4,            RGX_HWPERF_INDIRECT_BY_PHANTOM,         21,          3,  "RGX_CR_RASTERISATION_PERF",   rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_RASTER0),
+#endif
+
+		/*RGX_CNTBLK_ID_BLACKPEARL0*/
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_BLACKPEARL0, RGX_CR_BLACKPEARL_PERF_INDIRECT, RGX_CR_BLACKPEARL_PERF, RGX_CR_BLACKPEARL_PERF_SELECT0,    RGX_CR_BLACKPEARL_PERF_COUNTER_0,     6,              RGX_HWPERF_INDIRECT_BY_PHANTOM,         21,          3,  "RGX_CR_BLACKPEARL_PERF",      rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_BLACKPEARL0),
+#endif
+
+		/*RGX_CNTBLK_ID_PBE0*/
+#if defined(__KERNEL__) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(RGX_FEATURE_PBE2_IN_XE)
+		{RGX_CNTBLK_ID_PBE0,    RGX_CR_PBE_PERF_INDIRECT,        RGX_CR_PBE_PERF,            RGX_CR_PBE_PERF_SELECT0,            RGX_CR_PBE_PERF_COUNTER_0,            4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER, 21,          3,  "RGX_CR_PBE_PERF",             rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_not },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_PBE0),
+#endif
+};
+
+
+IMG_INTERNAL IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel)
+{
+	*ppsModel = gasCntBlkTypeModel;
+	return ARRAY_SIZE(gasCntBlkTypeModel);
+}
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.h b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.h
new file mode 100644
index 000000000000..449885cdcb64
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/devices/rogue/rgx_hwperf_table.h
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File
+@Title          HWPerf counter table header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_HWPERF_TABLE_H
+#define RGX_HWPERF_TABLE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(__KERNEL__)
+#include "rgxdevice.h"
+#endif
+/*****************************************************************************/
+
+/* Forward declaration */
+typedef struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_ RGXFW_HWPERF_CNTBLK_TYPE_MODEL;
+
+/* Function pointer type for functions to check dynamic power state of
+ * counter block instance. Used only in firmware. */
+typedef bool (*PFN_RGXFW_HWPERF_CNTBLK_POWERED)(
+		RGX_HWPERF_CNTBLK_ID eBlkType,
+		IMG_UINT8 ui8UnitId);
+
+#if defined(__KERNEL__)
+/* Counter block run-time info */
+typedef struct
+{
+	IMG_UINT32 ui32IndirectReg;          /* 0 if direct type otherwise the indirect control register to select indirect unit */
+	IMG_UINT32 ui32NumUnits;             /* Number of instances of this block type in the core */
+} RGX_HWPERF_CNTBLK_RT_INFO;
+#endif
+
+/* Function pointer type for functions to check block is valid and present
+ * on that RGX Device at runtime. It may have compile logic or run-time
+ * logic depending on where the code executes: server, srvinit or firmware.
+ * Values in the psRtInfo output parameter are only valid if true returned.
+ */
+typedef IMG_BOOL (*PFN_RGXFW_HWPERF_CNTBLK_PRESENT)(
+		const struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_* psBlkTypeDesc,
+		const void *pvDev_km,
+		void *pvRtInfo);
+
+/* This structure encodes properties of a type of performance counter block.
+ * The structure is sometimes referred to as a block type descriptor. These
+ * properties contained in this structure represent the columns in the block
+ * type model table variable below. These values vary depending on the build
+ * BVNC and core type.
+ * Each direct block has a unique type descriptor and each indirect group has
+ * a type descriptor.
+ */
+struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_
+{
+	/* Could use RGXFW_ALIGN_DCACHEL here but then we would waste 40% of the cache line? */
+	IMG_UINT32 ui32CntBlkIdBase;         /* The starting block id for this block type */
+	IMG_UINT32 ui32IndirectReg;          /* 0 if direct type otherwise the indirect control register to select indirect unit */
+	IMG_UINT32 ui32PerfReg;              /* RGX_CR_*_PERF register for this block type */
+	IMG_UINT32 ui32Select0BaseReg;       /* RGX_CR_*_PERF_SELECT0 register for this block type */
+	IMG_UINT32 ui32Counter0BaseReg;      /* RGX_CR_*_PERF_COUNTER_0 register for this block type */
+	IMG_UINT8  ui8NumCounters;          /* Number of counters in this block type */
+	IMG_UINT8  ui8NumUnits;             /* Number of instances of this block type in the core */
+	IMG_UINT8  ui8SelectRegModeShift;   /* Mode field shift value of select registers */
+	IMG_UINT8  ui8SelectRegOffsetShift; /* Interval between select registers, either 8 bytes or 16, hence << 3 or << 4 */
+	const IMG_CHAR *pszBlockNameComment;             /* Name of the PERF register. Used while dumping the perf counters to pdumps */
+	PFN_RGXFW_HWPERF_CNTBLK_POWERED pfnIsBlkPowered; /* A function to determine dynamic power state for the block type */
+	PFN_RGXFW_HWPERF_CNTBLK_PRESENT pfnIsBlkPresent; /* A function to determine presence on RGX Device at run-time */
+};
+
+/*****************************************************************************/
+
+IMG_INTERNAL IMG_UINT32 RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel);
+
+#endif /* RGX_HWPERF_TABLE_H */
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/allocmem.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/allocmem.h
new file mode 100644
index 000000000000..3de9e6781fbd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/allocmem.h
@@ -0,0 +1,224 @@
+/*************************************************************************/ /*!
+@File           allocmem.h
+@Title          memory allocation header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory-Allocation API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef ALLOCMEM_H
+#define ALLOCMEM_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * PVRSRV_ENABLE_PROCESS_STATS enables process statistics regarding events,
+ *     resources and memory across all processes
+ * PVRSRV_ENABLE_MEMORY_STATS enables recording of Linux kernel memory
+ *     allocations, provided that PVRSRV_ENABLE_PROCESS_STATS is enabled
+ *   - Output can be found in:
+ *     /(sys/kernel/debug|proc)/pvr/proc_stats/[live|retired]_pids_stats/mem_area
+ * PVRSRV_DEBUG_LINUX_MEMORY_STATS provides more details about memory
+ *     statistics in conjunction with PVRSRV_ENABLE_MEMORY_STATS
+ * PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON is defined to encompass both memory
+ *     allocation statistics functionalities described above in a single macro
+ */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS) && defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) && defined(DEBUG)
+#define PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+#endif
+
+/*
+ * When using detailed memory allocation statistics, the line number and
+ * file name where the allocation happened are also provided.
+ * When this feature is not used, these parameters are not needed.
+ */
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+#define DEBUG_MEMSTATS_PARAMS ,void *pvAllocFromFile, IMG_UINT32 ui32AllocFromLine
+#define DEBUG_MEMSTATS_ARGS   ,pvAllocFromFile, ui32AllocFromLine
+#define DEBUG_MEMSTATS_UNREF  (void)pvAllocFromFile; (void)ui32AllocFromLine;
+#define DEBUG_MEMSTATS_VALUES ,__FILE__, __LINE__
+#else
+#define DEBUG_MEMSTATS_PARAMS /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_ARGS   /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_UNREF  /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_VALUES  /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#endif
+
+
+/**************************************************************************/ /*!
+@Function       OSAllocMem
+@Description    Allocates CPU memory. Contents are uninitialized.
+                If passed a size of zero, function should not assert,
+                but just return a NULL pointer.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+#if defined(DOXYGEN)
+void *OSAllocMem(IMG_UINT32 ui32Size);
+#else
+void *OSAllocMem(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS);
+#define OSAllocMem(_size)	(OSAllocMem)((_size) DEBUG_MEMSTATS_VALUES)
+#endif
+
+/**************************************************************************/ /*!
+@Function       OSAllocZMem
+@Description    Allocates CPU memory and initializes the contents to zero.
+                If passed a size of zero, function should not assert,
+                but just return a NULL pointer.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+#if defined(DOXYGEN)
+void *OSAllocZMem(IMG_UINT32 ui32Size);
+#else
+void *OSAllocZMem(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS);
+#define OSAllocZMem(_size)	(OSAllocZMem)((_size) DEBUG_MEMSTATS_VALUES)
+#endif
+
+
+/**************************************************************************/ /*!
+@Function       OSAllocMemNoStats
+@Description    Allocates CPU memory. Contents are uninitialized.
+                 If passed a size of zero, function should not assert,
+                 but just return a NULL pointer.
+                 The allocated memory is not accounted for by process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSAllocMem() and OSAllocMemNoStats() equate to
+                 the same operation.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+void *OSAllocMemNoStats(IMG_UINT32 ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSAllocZMemNoStats
+@Description    Allocates CPU memory and initializes the contents to zero.
+                 If passed a size of zero, function should not assert,
+                 but just return a NULL pointer.
+                 The allocated memory is not accounted for by process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSAllocZMem() and OSAllocZMemNoStats() equate to
+                 the same operation.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+void *OSAllocZMemNoStats(IMG_UINT32 ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSFreeMem
+@Description    Frees previously allocated CPU memory.
+@Input          pvCpuVAddr       Pointer to the memory to be freed.
+@Return         None.
+ */ /**************************************************************************/
+void OSFreeMem(void *pvCpuVAddr);
+
+/**************************************************************************/ /*!
+@Function       OSFreeMemNoStats
+@Description    Frees previously allocated CPU memory.
+                 The freed memory does not update the figures in process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSFreeMem() and OSFreeMemNoStats() equate to the
+                 same operation.
+@Input          pvCpuVAddr       Pointer to the memory to be freed.
+@Return         None.
+ */ /**************************************************************************/
+void OSFreeMemNoStats(void *pvCpuVAddr);
+
+/*
+ * These macros allow us to catch double-free bugs on DEBUG builds and
+ * prevent crashes on RELEASE builds.
+ */
+
+/*! @cond Doxygen_Suppress */
+#if defined(DEBUG)
+#define double_free_sentinel ((void *)&OSFreeMem)
+#define ALLOCMEM_ASSERT(exp) PVR_ASSERT(exp)
+#else
+#define double_free_sentinel NULL
+#define ALLOCMEM_ASSERT(exp) do {} while (0)
+#endif
+/*! @endcond */
+
+/*! Frees memory allocated by OSAllocMem(). */
+#define OSFreeMem(_ptr) do { \
+		ALLOCMEM_ASSERT((_ptr) != double_free_sentinel); \
+		(OSFreeMem)(_ptr); \
+		(_ptr) = double_free_sentinel; \
+		MSC_SUPPRESS_4127 \
+	} while (0)
+
+/*! Frees memory allocated by OSAllocMemNoStats(). */
+#define OSFreeMemNoStats(_ptr) do { \
+		ALLOCMEM_ASSERT((_ptr) != double_free_sentinel); \
+		(OSFreeMemNoStats)(_ptr); \
+		(_ptr) = double_free_sentinel; \
+		MSC_SUPPRESS_4127 \
+	} while (0)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* ALLOCMEM_H */
+
+/******************************************************************************
+ End of file (allocmem.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/device_connection.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/device_connection.h
new file mode 100644
index 000000000000..b80983c2d87d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/device_connection.h
@@ -0,0 +1,130 @@
+/*************************************************************************/ /*!
+@File           device_connection.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DEVICE_CONNECTION_H)
+#define DEVICE_CONNECTION_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(__KERNEL__)
+typedef struct _PVRSRV_DEVICE_NODE_ *SHARED_DEV_CONNECTION;
+#else
+#include "connection.h"
+typedef const struct PVRSRV_DEV_CONNECTION_TAG *SHARED_DEV_CONNECTION;
+#endif
+
+/******************************************************************************
+ * Device capability flags and masks
+ *
+ * Following bitmask shows allocated ranges and values for our device
+ * capability settings:
+ *
+ * 31 27  23  19  15  11   7   3  0
+ * |...|...|...|...|...|...|...|...
+ *                               ** CACHE_COHERENT                   [0x1..0x2]
+ *                                x  PVRSRV_CACHE_COHERENT_DEVICE_FLAG
+ *                               x.  PVRSRV_CACHE_COHERENT_CPU_FLAG
+ *                             *... NONMAPPABLE_MEMORY                    [0x8]
+ *                             x...  PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG
+ *                            *.... PDUMP_IS_RECORDING                   [0x10]
+ *                            x....  PVRSRV_PDUMP_IS_RECORDING
+ *                      ***........ DEVMEM_SVM_ALLOC             [0x100..0x400]
+ *                        x........  PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED
+ *                       x.........  PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED
+ *                      x..........  PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL
+ *                     *........... FBCDC_V3_1             [0x800]
+ *                     x...........  FBCDC_V3_1_USED
+ *                    *............ PVRSRV_SYSTEM_DMA
+ *                    x............  PVRSRV_SYSTEM_DMA_USED
+ *                   *............. TFBC_LOSSY_GROUP
+ *                   x.............  TFBC_LOSSY_GROUP_1
+ * |...|...|...|...|...|...|...|...
+ *****************************************************************************/
+
+/* Flag to be passed over the bridge during connection stating whether CPU cache coherent is available*/
+#define PVRSRV_CACHE_COHERENT_SHIFT (0)
+#define	PVRSRV_CACHE_COHERENT_DEVICE_FLAG (1U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define	PVRSRV_CACHE_COHERENT_CPU_FLAG (2U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define	PVRSRV_CACHE_COHERENT_EMULATE_FLAG (4U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define PVRSRV_CACHE_COHERENT_MASK (7U << PVRSRV_CACHE_COHERENT_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether CPU non-mappable memory is present */
+#define PVRSRV_NONMAPPABLE_MEMORY_PRESENT_SHIFT (7)
+#define PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG (1U << PVRSRV_NONMAPPABLE_MEMORY_PRESENT_SHIFT)
+
+/* Flag to be passed over the bridge to indicate PDump activity */
+#define PVRSRV_PDUMP_IS_RECORDING_SHIFT (4)
+#define PVRSRV_PDUMP_IS_RECORDING (1U << PVRSRV_PDUMP_IS_RECORDING_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating SVM allocation availability */
+#define PVRSRV_DEVMEM_SVM_ALLOC_SHIFT (8)
+#define PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED (1U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+#define PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED (2U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+#define PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL (4U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether GPU uses FBCDC v3.1 */
+#define PVRSRV_FBCDC_V3_1_USED_SHIFT (11)
+#define PVRSRV_FBCDC_V3_1_USED (1U << PVRSRV_FBCDC_V3_1_USED_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether System has
+   DMA transfer capability to and from device memory */
+#define PVRSRV_SYSTEM_DMA_SHIFT (12)
+#define PVRSRV_SYSTEM_DMA_USED (1U << PVRSRV_SYSTEM_DMA_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether GPU supports TFBC and is
+   configured to use lossy compression control group 1 (25% / 37.5% / 50%) */
+#define PVRSRV_TFBC_LOSSY_GROUP_SHIFT (13)
+#define PVRSRV_TFBC_LOSSY_GROUP_1 (1U << PVRSRV_TFBC_LOSSY_GROUP_SHIFT)
+
+static INLINE IMG_HANDLE GetBridgeHandle(SHARED_DEV_CONNECTION hDevConnection)
+{
+#if defined(__KERNEL__)
+    return hDevConnection;
+#else
+    return hDevConnection->hServices;
+#endif
+}
+
+
+#endif /* !defined(DEVICE_CONNECTION_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem.h
new file mode 100644
index 000000000000..0a7d550bf194
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem.h
@@ -0,0 +1,694 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management core internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to core device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVCLIENT_DEVICEMEM_H
+#define SRVCLIENT_DEVICEMEM_H
+
+/******************************************************************************
+ *                                                                            *
+ *  +------------+   +------------+      +--------------+   +--------------+  *
+ *  | a   sub-   |   | a   sub-   |      |  an          |   | allocation   |  *
+ *  | allocation |   | allocation |      |  allocation  |   | also mapped  |  *
+ *  |            |   |            |      |  in proc 1   |   | into proc 2  |  *
+ *  +------------+   +------------+      +--------------+   +--------------+  *
+ *            |         |                       |                  |          *
+ *         +--------------+              +--------------+   +--------------+  *
+ *         | page   gran- |              | page   gran- |   | page   gran- |  *
+ *         | ular mapping |              | ular mapping |   | ular mapping |  *
+ *         +--------------+              +--------------+   +--------------+  *
+ *                |                                   |       |               *
+ *                |                                   |       |               *
+ *                |                                   |       |               *
+ *         +--------------+                       +--------------+            *
+ *         |              |                       |              |            *
+ *         | A  "P.M.R."  |                       | A  "P.M.R."  |            *
+ *         |              |                       |              |            *
+ *         +--------------+                       +--------------+            *
+ *                                                                            *
+ ******************************************************************************/
+
+/*
+    All device memory allocations are ultimately a view upon (not
+    necessarily the whole of) a "PMR".
+
+    A PMR is a "Physical Memory Resource", which may be a
+    "pre-faulted" lump of physical memory, or it may be a
+    representation of some physical memory that will be instantiated
+    at some future time.
+
+    PMRs always represent multiple of some power-of-2 "contiguity"
+    promised by the PMR, which will allow them to be mapped in whole
+    pages into the device MMU.  As memory allocations may be smaller
+    than a page, these mappings may be suballocated and thus shared
+    between multiple allocations in one process.  A PMR may also be
+    mapped simultaneously into multiple device memory contexts
+    (cross-process scenario), however, for security reasons, it is not
+    legal to share a PMR "both ways" at once, that is, mapped into
+    multiple processes and divided up amongst several suballocations.
+
+    This PMR terminology is introduced here for background
+    information, but is generally of little concern to the caller of
+    this API.  This API handles suballocations and mappings, and the
+    caller thus deals primarily with MEMORY DESCRIPTORS representing
+    an allocation or suballocation, HEAPS representing ranges of
+    virtual addresses in a CONTEXT.
+*/
+
+/*
+   |<---------------------------context------------------------------>|
+   |<-------heap------->|   |<-------heap------->|<-------heap------->|
+   |<-alloc->|          |   |<-alloc->|<-alloc->||   |<-alloc->|      |
+*/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "pdump.h"
+
+#include "device_connection.h"
+
+
+typedef IMG_UINT32 DEVMEM_HEAPCFGID;
+#define DEVMEM_HEAPCFG_FORCLIENTS 0
+#define DEVMEM_HEAPCFG_FORFW 1
+
+
+/*
+  In order to call the server side functions, we need a bridge handle.
+  We abstract that here, as we may wish to change its form.
+ */
+
+typedef IMG_HANDLE DEVMEM_BRIDGE_HANDLE;
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapInt(DEVMEM_HEAP *psHeap,
+				 IMG_HANDLE *phDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetSize(DEVMEM_MEMDESC *psMemDesc,
+			  IMG_DEVMEM_SIZE_T* puiSize);
+
+IMG_INTERNAL void
+DevmemGetAnnotation(DEVMEM_MEMDESC *psMemDesc,
+			        IMG_CHAR **pszAnnotation);
+
+/*
+ * DevmemCreateContext()
+ *
+ * Create a device memory context
+ *
+ * This must be called before any heap is created in this context
+ *
+ * Caller to provide bridge handle which will be recorded internally and used
+ * for all future operations on items from this memory context.  Caller also
+ * to provide devicenode handle, as this is used for MMU configuration and
+ * also to determine the heap configuration for the auto-instantiated heaps.
+ *
+ * Note that when compiled in services/server, the hBridge is not used and
+ * is thrown away by the "fake" direct bridge.  (This may change. It is
+ * recommended that NULL be passed for the handle for now.)
+ *
+ * hDeviceNode and uiHeapBlueprintID shall together dictate which heap-config
+ * to use.
+ *
+ * This will cause the server side counterpart to be created also.
+ *
+ * If you call DevmemCreateContext() (and the call succeeds) you are promising
+ * that you will later call Devmem_ContextDestroy(), except for abnormal
+ * process termination in which case it is expected it will be destroyed as
+ * part of handle clean up.
+ *
+ * Caller to provide storage for the pointer to the newly created
+ * NEWDEVMEM_CONTEXT object.
+ */
+PVRSRV_ERROR
+DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection,
+                    DEVMEM_HEAPCFGID uiHeapBlueprintID,
+                    DEVMEM_CONTEXT **ppsCtxPtr);
+
+/*
+ * DevmemAcquireDevPrivData()
+ *
+ * Acquire the device private data for this memory context
+ */
+PVRSRV_ERROR
+DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
+                         IMG_HANDLE *hPrivData);
+
+/*
+ * DevmemReleaseDevPrivData()
+ *
+ * Release the device private data for this memory context
+ */
+PVRSRV_ERROR
+DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx);
+
+/*
+ * DevmemDestroyContext()
+ *
+ * Undoes that done by DevmemCreateContext()
+ */
+PVRSRV_ERROR
+DevmemDestroyContext(DEVMEM_CONTEXT *psCtx);
+
+/*
+ * DevmemCreateHeap()
+ *
+ * Create a heap in the given context.
+ *
+ * N.B.  Not intended to be called directly, though it can be.
+ * Normally, heaps are instantiated at context creation time according
+ * to the specified blueprint.  See DevmemCreateContext() for details.
+ *
+ * This will cause MMU code to set up data structures for the heap,
+ * but may not cause page tables to be modified until allocations are
+ * made from the heap.
+ *
+ * uiReservedRegionLength Reserved address space for static VAs shared
+ * between clients and firmware
+ *
+ * The "Quantum" is both the device MMU page size to be configured for
+ * this heap, and the unit multiples of which "quantized" allocations
+ * are made (allocations smaller than this, known as "suballocations"
+ * will be made from a "sub alloc RA" and will "import" chunks
+ * according to this quantum)
+ *
+ * Where imported PMRs (or, for example, PMRs created by device class
+ * buffers) are mapped into this heap, it is important that the
+ * physical contiguity guarantee offered by the PMR is greater than or
+ * equal to the quantum size specified here, otherwise the attempt to
+ * map it will fail.  "Normal" allocations via Devmem_Allocate
+ * shall automatically meet this requirement, as each "import" will
+ * trigger the creation of a PMR with the desired contiguity.  The
+ * supported quantum sizes in that case shall be dictated by the OS
+ * specific implementation of PhysmemNewOSRamBackedPMR() (see)
+ */
+PVRSRV_ERROR
+DevmemCreateHeap(DEVMEM_CONTEXT *psCtxPtr,
+                 /* base and length of heap */
+                 IMG_DEV_VIRTADDR sBaseAddress,
+                 IMG_DEVMEM_SIZE_T uiLength,
+                 IMG_DEVMEM_SIZE_T uiReservedRegionLength,
+                 /* log2 of allocation quantum, i.e. "page" size.
+                    All allocations (that go to server side) are
+                    multiples of this.  We use a client-side RA to
+                    make sub-allocations from this */
+                 IMG_UINT32 ui32Log2Quantum,
+                 /* The minimum import alignment for this heap */
+                 IMG_UINT32 ui32Log2ImportAlignment,
+                 /* Name of heap for debug */
+                 /* N.B.  Okay to exist on caller's stack - this
+                    func takes a copy if it needs it. */
+                 const IMG_CHAR *pszName,
+                 DEVMEM_HEAPCFGID uiHeapBlueprintID,
+                 IMG_UINT32 uiHeapIndex,
+                 DEVMEM_HEAP **ppsHeapPtr);
+/*
+ * DevmemDestroyHeap()
+ *
+ * Reverses DevmemCreateHeap()
+ *
+ * N.B. All allocations must have been freed and all mappings must
+ * have been unmapped before invoking this call
+ */
+PVRSRV_ERROR
+DevmemDestroyHeap(DEVMEM_HEAP *psHeap);
+
+/*
+ * DevmemExportalignAdjustSizeAndAlign()
+ * Compute the Size and Align passed to avoid suballocations
+ * (used when allocation with PVRSRV_MEMALLOCFLAG_EXPORTALIGN).
+ *
+ * Returns PVRSRV_ERROR_INVALID_PARAMS if uiLog2Quantum has invalid value.
+ */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum,
+                                    IMG_DEVMEM_SIZE_T *puiSize,
+                                    IMG_DEVMEM_ALIGN_T *puiAlign);
+
+/*
+ * DevmemSubAllocate()
+ *
+ * Makes an allocation (possibly a "suballocation", as described
+ * below) of device virtual memory from this heap.
+ *
+ * The size and alignment of the allocation will be honoured by the RA
+ * that allocates the "suballocation".  The resulting allocation will
+ * be mapped into GPU virtual memory and the physical memory to back
+ * it will exist, by the time this call successfully completes.
+ *
+ * The size must be a positive integer multiple of the alignment.
+ * (i.e. the alignment specifies the alignment of both the start and
+ * the end of the resulting allocation.)
+ *
+ * Allocations made via this API are routed through a "suballocation
+ * RA" which is responsible for ensuring that small allocations can be
+ * made without wasting physical memory in the server.  Furthermore,
+ * such suballocations can be made entirely client side without
+ * needing to go to the server unless the allocation spills into a new
+ * page.
+ *
+ * Such suballocations cause many allocations to share the same "PMR".
+ * This happens only when the flags match exactly.
+ *
+ */
+
+PVRSRV_ERROR
+DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier,
+                  DEVMEM_HEAP *psHeap,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_DEVMEM_ALIGN_T uiAlign,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  const IMG_CHAR *pszText,
+                  DEVMEM_MEMDESC **ppsMemDescPtr);
+
+#define DevmemAllocate(...) \
+    DevmemSubAllocate(DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER, __VA_ARGS__)
+
+PVRSRV_ERROR
+DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         IMG_DEVMEM_ALIGN_T uiAlign,
+                         IMG_UINT32 uiLog2HeapPageSize,
+                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                         const IMG_CHAR *pszText,
+                         DEVMEM_MEMDESC **ppsMemDescPtr);
+
+PVRSRV_ERROR
+DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *paui32AllocPageIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pauiFreePageIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiFlags);
+
+PVRSRV_ERROR
+DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection,
+                     IMG_DEVMEM_SIZE_T uiSize,
+                     IMG_UINT32 ui32NumPhysChunks,
+                     IMG_UINT32 ui32NumVirtChunks,
+                     IMG_UINT32 *pui32MappingTable,
+                     IMG_DEVMEM_ALIGN_T uiAlign,
+                     IMG_UINT32 uiLog2HeapPageSize,
+                     PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                     const IMG_CHAR *pszText,
+                     DEVMEM_MEMDESC **ppsMemDescPtr);
+
+PVRSRV_ERROR
+DevmemSubAllocateAndMap(IMG_UINT8 uiPreAllocMultiplier,
+			DEVMEM_HEAP *psHeap,
+			IMG_DEVMEM_SIZE_T uiSize,
+			IMG_DEVMEM_ALIGN_T uiAlign,
+			PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			const IMG_CHAR *pszText,
+			DEVMEM_MEMDESC **ppsMemDescPtr,
+			IMG_DEV_VIRTADDR *psDevVirtAddr);
+
+#define DevmemAllocateAndMap(...) \
+	DevmemSubAllocateAndMap(DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER, __VA_ARGS__)
+
+/*
+ * DevmemFree()
+ *
+ * Reverses that done by DevmemSubAllocate() N.B.  The underlying
+ * mapping and server side allocation _may_ not be torn down, for
+ * example, if the allocation has been exported, or if multiple
+ * allocations were suballocated from the same mapping, but this is
+ * properly refcounted, so the caller does not have to care.
+ */
+
+IMG_BOOL
+DevmemFree(DEVMEM_MEMDESC *psMemDesc);
+
+IMG_BOOL
+DevmemReleaseDevAddrAndFree(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+	DevmemMapToDevice:
+
+	Map an allocation to the device it was allocated from.
+	This function _must_ be called before any call to
+	DevmemAcquireDevVirtAddr is made as it binds the allocation
+	to the heap.
+	DevmemReleaseDevVirtAddr is used to release the reference
+	to the device mapping this function created, but it doesn't
+	mean that the memory will actually be unmapped from the
+	device as other references to the mapping obtained via
+	DevmemAcquireDevVirtAddr could still be active.
+*/
+PVRSRV_ERROR DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
+							   DEVMEM_HEAP *psHeap,
+							   IMG_DEV_VIRTADDR *psDevVirtAddr);
+
+/*
+	DevmemMapToDeviceAddress:
+
+	Same as DevmemMapToDevice but the caller chooses the address
+	to map to.
+*/
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc,
+                         DEVMEM_HEAP *psHeap,
+                         IMG_DEV_VIRTADDR sDevVirtAddr);
+
+/*
+	DevmemGetDevVirtAddr
+
+	Obtain the MemDesc's device virtual address.
+	This function _must_ be called after DevmemMapToDevice(Address)
+	and is expected to be used be functions which didn't allocate
+	the MemDesc but need to know it's address.
+	It will PVR_ASSERT if no device mapping exists and 0 is returned.
+ */
+IMG_DEV_VIRTADDR
+DevmemGetDevVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+	DevmemAcquireDevVirtAddr
+
+	Acquire the MemDesc's device virtual address.
+	This function _must_ be called after DevmemMapToDevice
+	and is expected to be used be functions which didn't allocate
+	the MemDesc but need to know it's address
+ */
+PVRSRV_ERROR DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                      IMG_DEV_VIRTADDR *psDevVirtAddrRet);
+
+/*
+ * DevmemReleaseDevVirtAddr()
+ *
+ * give up the licence to use the device virtual address that was
+ * acquired by "Acquire" or "MapToDevice"
+ */
+void
+DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+ * DevmemAcquireCpuVirtAddr()
+ *
+ * Acquires a license to use the cpu virtual address of this mapping.
+ * Note that the memory may not have been mapped into cpu virtual
+ * memory prior to this call.  On first "acquire" the memory will be
+ * mapped in (if it wasn't statically mapped in) and on last put it
+ * _may_ become unmapped.  Later calling "Acquire" again, _may_ cause
+ * the memory to be mapped at a different address.
+ */
+PVRSRV_ERROR DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                      void **ppvCpuVirtAddr);
+
+/*
+ * DevmemReacquireCpuVirtAddr()
+ *
+ * (Re)acquires license to use the cpu virtual address of this mapping
+ * if (and only if) there is already a pre-existing license to use the
+ * cpu virtual address for the mapping, returns NULL otherwise.
+ */
+void DevmemReacquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                void **ppvCpuVirtAddr);
+
+/*
+ * DevmemReleaseDevVirtAddr()
+ *
+ * give up the licence to use the cpu virtual address that was granted
+ * with the "Get" call.
+ */
+void
+DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+/*
+ * DevmemExport()
+ *
+ * Given a memory allocation allocated with DevmemAllocateExportable()
+ * create a "cookie" that can be passed intact by the caller's own choice
+ * of secure IPC to another process and used as the argument to "map"
+ * to map this memory into a heap in the target processes.  N.B.  This can
+ * also be used to map into multiple heaps in one process, though that's not
+ * the intention.
+ *
+ * Note, the caller must later call Unexport before freeing the
+ * memory.
+ */
+PVRSRV_ERROR DevmemExport(DEVMEM_MEMDESC *psMemDesc,
+                          DEVMEM_EXPORTCOOKIE *psExportCookie);
+
+
+void DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
+					DEVMEM_EXPORTCOOKIE *psExportCookie);
+
+PVRSRV_ERROR
+DevmemImport(SHARED_DEV_CONNECTION hDevConnection,
+			 DEVMEM_EXPORTCOOKIE *psCookie,
+			 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			 DEVMEM_MEMDESC **ppsMemDescPtr);
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+/*
+ * DevmemMakeLocalImportHandle()
+ *
+ * This is a "special case" function for making a server export cookie
+ * which went through the direct bridge into an export cookie that can
+ * be passed through the client bridge.
+ */
+PVRSRV_ERROR
+DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+                            IMG_HANDLE hServerExport,
+                            IMG_HANDLE *hClientExport);
+
+/*
+ * DevmemUnmakeLocalImportHandle()
+ *
+ * Free any resource associated with the Make operation
+ */
+PVRSRV_ERROR
+DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+                              IMG_HANDLE hClientExport);
+
+/*
+ *
+ * The following set of functions is specific to the heap "blueprint"
+ * stuff, for automatic creation of heaps when a context is created
+ *
+ */
+
+
+/* Devmem_HeapConfigCount: returns the number of heap configs that
+   this device has.  Note that there is no acquire/release semantics
+   required, as this data is guaranteed to be constant for the
+   lifetime of the device node */
+PVRSRV_ERROR
+DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection,
+                      IMG_UINT32 *puiNumHeapConfigsOut);
+
+/* Devmem_HeapCount: returns the number of heaps that a given heap
+   config on this device has.  Note that there is no acquire/release
+   semantics required, as this data is guaranteed to be constant for
+   the lifetime of the device node */
+PVRSRV_ERROR
+DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection,
+                IMG_UINT32 uiHeapConfigIndex,
+                IMG_UINT32 *puiNumHeapsOut);
+/* Devmem_HeapConfigName: return the name of the given heap config.
+   The caller is to provide the storage for the returned string and
+   indicate the number of bytes (including null terminator) for such
+   string in the BufSz arg.  Note that there is no acquire/release
+   semantics required, as this data is guaranteed to be constant for
+   the lifetime of the device node.
+ */
+PVRSRV_ERROR
+DevmemHeapConfigName(SHARED_DEV_CONNECTION hsDevConnection,
+                     IMG_UINT32 uiHeapConfigIndex,
+                     IMG_CHAR *pszConfigNameOut,
+                     IMG_UINT32 uiConfigNameBufSz);
+
+/* Devmem_HeapDetails: fetches all the metadata that is recorded in
+   this heap "blueprint".  Namely: heap name (caller to provide
+   storage, and indicate buffer size (including null terminator) in
+   BufSz arg), device virtual address and length, log2 of data page
+   size (will be one of 12, 14, 16, 18, 20, 21, at time of writing).
+   Note that there is no acquire/release semantics required, as this
+   data is guaranteed to be constant for the lifetime of the device
+   node. */
+PVRSRV_ERROR
+DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection,
+                  IMG_UINT32 uiHeapConfigIndex,
+                  IMG_UINT32 uiHeapIndex,
+                  IMG_CHAR *pszHeapNameOut,
+                  IMG_UINT32 uiHeapNameBufSz,
+                  IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+                  IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+                  IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+                  IMG_UINT32 *puiLog2DataPageSize,
+                  IMG_UINT32 *puiLog2ImportAlignmentOut);
+
+/*
+ * Devmem_FindHeapByName()
+ *
+ * returns the heap handle for the named _automagic_ heap in this
+ * context.  "automagic" heaps are those that are born with the
+ * context from a blueprint
+ */
+PVRSRV_ERROR
+DevmemFindHeapByName(const DEVMEM_CONTEXT *psCtx,
+                     const IMG_CHAR *pszHeapName,
+                     DEVMEM_HEAP **ppsHeapRet);
+
+/*
+ * DevmemGetHeapBaseDevVAddr()
+ *
+ * returns the device virtual address of the base of the heap.
+ */
+PVRSRV_ERROR
+DevmemGetHeapBaseDevVAddr(DEVMEM_HEAP *psHeap,
+			  IMG_DEV_VIRTADDR *pDevVAddr);
+
+/*
+ * DevmemGetHeapSize()
+ *
+ * returns the size of the heap.
+ */
+IMG_INTERNAL DEVMEM_SIZE_T
+DevmemGetHeapSize(struct DEVMEM_HEAP_TAG *psHeap);
+
+PVRSRV_ERROR
+DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+			   IMG_HANDLE *phImport);
+
+PVRSRV_ERROR
+DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
+		   IMG_UINT64 *pui64UID);
+
+PVRSRV_ERROR
+DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
+		     IMG_HANDLE *hReservation);
+
+IMG_INTERNAL void
+DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *hPMR,
+		IMG_DEVMEM_OFFSET_T *puiPMROffset);
+
+IMG_INTERNAL void
+DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
+				PVRSRV_MEMALLOCFLAGS_T *puiFlags);
+
+IMG_INTERNAL SHARED_DEV_CONNECTION
+DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc);
+
+PVRSRV_ERROR
+DevmemLocalImport(SHARED_DEV_CONNECTION hDevConnection,
+				  IMG_HANDLE hExtHandle,
+				  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+				  DEVMEM_MEMDESC **ppsMemDescPtr,
+				  IMG_DEVMEM_SIZE_T *puiSizePtr,
+				  const IMG_CHAR *pszAnnotation);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
+                         IMG_DEV_VIRTADDR sDevVAddr);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetFaultAddress(DEVMEM_CONTEXT *psContext,
+                      IMG_DEV_VIRTADDR *psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemInvalidateFBSCTable(DEVMEM_CONTEXT *psContext,
+                          IMG_UINT64 ui64FBSCEntries);
+
+/* DevmemGetHeapLog2PageSize()
+ *
+ * Get the page size used for a certain heap.
+ */
+IMG_UINT32
+DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap);
+
+/* DevmemGetMemFlags()
+ *
+ * Get the memalloc flags for a certain memdesc.
+ */
+PVRSRV_MEMALLOCFLAGS_T
+DevmemGetMemAllocFlags(DEVMEM_MEMDESC *psMemDesc);
+
+/* DevmemGetHeapReservedSize()
+ *
+ * Get the reserved size used for a certain heap.
+ */
+IMG_DEVMEM_SIZE_T
+DevmemGetHeapReservedSize(DEVMEM_HEAP *psHeap);
+
+/*************************************************************************/ /*!
+@Function       RegisterDevMemPFNotify
+@Description    Registers that the application wants to be signaled when a page
+                fault occurs.
+
+@Input          psContext      Memory context the process that would like to
+                               be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR:  PVRSRV_OK on success. Otherwise, a PVRSRV_
+                               error code
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext,
+                       IMG_BOOL       bRegister);
+
+/*************************************************************************/ /*!
+@Function       DevmemHeapSetPremapStatus
+@Description    In some special cases like virtualisation, a device memory heap
+			    must be entirely backed by physical memory and mapped into the
+				device's virtual address space. This is done at context creation.
+			    When objects are allocated from such a heap, the mapping part
+			    must be skipped. The 'bPremapped' flag dictates if allocations
+			    are to be mapped or not.
+
+@Input          psHeap            Device memory heap to be updated
+@Input          IsPremapped       The premapping status to be set
+*/ /**************************************************************************/
+IMG_INTERNAL void
+DevmemHeapSetPremapStatus(DEVMEM_HEAP *psHeap, IMG_BOOL IsPremapped);
+
+#endif /* #ifndef SRVCLIENT_DEVICEMEM_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_pdump.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_pdump.h
new file mode 100644
index 000000000000..09b28afe7e51
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_pdump.h
@@ -0,0 +1,363 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management PDump internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to PDump device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_PDUMP_H
+#define DEVICEMEM_PDUMP_H
+
+#include "devicemem.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+
+#if defined(PDUMP)
+/*
+ * DevmemPDumpLoadMem()
+ *
+ * takes a memory descriptor, offset, and size, and takes the current contents
+ * of the memory at that location and writes it to the prm pdump file, and
+ * emits a pdump LDB to load the data from that file.  The intention here is
+ * that the contents of the simulated buffer upon pdump playback will be made
+ * to be the same as they are when this command is run, enabling pdump of
+ * cases where the memory has been modified externally, i.e. by the host cpu
+ * or by a third party.
+ */
+void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpLoadZeroMem()
+ *
+ * As DevmemPDumpLoadMem() but the PDump allocation will be populated with
+ * zeros from the zero page in the parameter stream
+ */
+void
+DevmemPDumpLoadZeroMem(DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpLoadMemValue32()
+ *
+ * As above but dumps the value at a dword-aligned address in plain text to
+ * the pdump script2 file. Useful for patching a buffer at pdump playback by
+ * simply editing the script output file.
+ *
+ * (The same functionality can be achieved by the above function but the
+ *  binary PARAM file must be patched in that case.)
+ */
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpMemValue64()
+ *
+ * As above but dumps the 64bit-value at a dword-aligned address in plain text
+ * to the pdump script2 file. Useful for patching a buffer at pdump playback by
+ * simply editing the script output file.
+ *
+ * (The same functionality can be achieved by the above function but the
+ *  binary PARAM file must be patched in that case.)
+ */
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpPageCatBaseToSAddr()
+ *
+ * Returns the symbolic address of a piece of memory represented by an offset
+ * into the mem descriptor.
+ */
+PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC *psMemDesc,
+                              IMG_DEVMEM_OFFSET_T *puiMemOffset,
+                              IMG_CHAR *pszName,
+                              IMG_UINT32 ui32Size);
+
+/*
+ * DevmemPDumpSaveToFile()
+ *
+ * Emits a pdump SAB to cause the current contents of the memory to be written
+ * to the given file during playback
+ */
+void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset);
+
+/*
+ * DevmemPDumpSaveToFileVirtual()
+ *
+ * Emits a pdump SAB, just like DevmemPDumpSaveToFile(), but uses the virtual
+ * address and device MMU context to cause the pdump player to traverse the
+ * MMU page tables itself.
+ */
+void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags);
+
+/*
+ * DevmemPDumpDataDescriptor()
+ *
+ * Emits a pdump CMD:OutputData, using the virtual address and device MMU
+ * context. Provides more flexibility than a pdump SAB because metadata can
+ * be passed to an external pdump player library via the command header.
+ */
+void
+DevmemPDumpDataDescriptor(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_DEVMEM_SIZE_T uiSize,
+                          const IMG_CHAR *pszFilename,
+                          IMG_UINT32 ui32HeaderType,
+                          IMG_UINT32 ui32ElementType,
+                          IMG_UINT32 ui32ElementCount,
+                          IMG_UINT32 ui32PdumpFlags);
+
+
+/*
+ *
+ * DevmemPDumpDevmemPol32()
+ *
+ * Writes a PDump 'POL' command to wait for a masked 32-bit memory location to
+ * become the specified value.
+ */
+PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags);
+
+#if defined(__KERNEL__)
+/*
+ *
+ * DevmemPDumpDevmemCheck32()
+ *
+ * Writes a PDump 'POL' command to run a single-shot check for a masked
+ * 32-bit memory location to match the specified value.
+ */
+PVRSRV_ERROR
+DevmemPDumpDevmemCheck32(const DEVMEM_MEMDESC *psMemDesc,
+                         IMG_DEVMEM_OFFSET_T uiOffset,
+                         IMG_UINT32 ui32Value,
+                         IMG_UINT32 ui32Mask,
+                         PDUMP_POLL_OPERATOR eOperator,
+                         PDUMP_FLAGS_T ui32PDumpFlags);
+#endif
+
+/*
+ * DevmemPDumpCBP()
+ *
+ * Polls for space in circular buffer. Reads the read offset from memory and
+ * waits until there is enough space to write the packet.
+ *
+ * psMemDesc     - MemDesc which contains the read offset
+ * uiReadOffset  - Offset into MemDesc to the read offset
+ * uiWriteOffset - Current write offset
+ * uiPacketSize  - Size of packet to write
+ * uiBufferSize  - Size of circular buffer
+ */
+PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize);
+
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMem)
+#endif
+static INLINE void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMemValue32)
+#endif
+static INLINE void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMemValue64)
+#endif
+static INLINE void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui64Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpPageCatBaseToSAddr)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC *psMemDesc,
+                              IMG_DEVMEM_OFFSET_T *puiMemOffset,
+                              IMG_CHAR *pszName,
+                              IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(puiMemOffset);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpSaveToFile)
+#endif
+static INLINE void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(uiFileOffset);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpSaveToFileVirtual)
+#endif
+static INLINE void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PdumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpDevmemPol32)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpCBP)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+#endif /* DEVICEMEM_PDUMP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_utils.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_utils.h
new file mode 100644
index 000000000000..9e24c1dceca7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicemem_utils.h
@@ -0,0 +1,582 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management internal utility functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally by device memory management
+                code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_UTILS_H
+#define DEVICEMEM_UTILS_H
+
+#include "devicemem.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "osmmap.h"
+#include "pvrsrv_memallocflags_internal.h"
+
+#define DEVMEM_HEAPNAME_MAXLENGTH 160
+
+/*
+ * VA heap size should be at least OS page size. This check is validated in the DDK.
+ */
+#define DEVMEM_HEAP_MINIMUM_SIZE                     0x10000 /* 64KB is MAX anticipated OS page size */
+
+#if defined(DEVMEM_DEBUG) && defined(REFCOUNT_DEBUG)
+#define DEVMEM_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_ERROR, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define DEVMEM_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+/* If we need a "hMapping" but we don't have a server-side mapping, we poison
+ * the entry with this value so that it's easily recognised in the debugger.
+ * Note that this is potentially a valid handle, but then so is NULL, which is
+ * no better, indeed worse, as it's not obvious in the debugger. The value
+ * doesn't matter. We _never_ use it (and because it's valid, we never assert
+ * it isn't this) but it's nice to have a value in the source code that we can
+ * grep for if things go wrong.
+ */
+#define LACK_OF_MAPPING_POISON ((IMG_HANDLE)0x6116dead)
+#define LACK_OF_RESERVATION_POISON ((IMG_HANDLE)0x7117dead)
+
+#define DEVICEMEM_HISTORY_ALLOC_INDEX_NONE 0xFFFFFFFF
+
+struct DEVMEM_CONTEXT_TAG
+{
+
+	SHARED_DEV_CONNECTION hDevConnection;
+
+	/* Number of heaps that have been created in this context
+	 * (regardless of whether they have allocations)
+	 */
+	IMG_UINT32 uiNumHeaps;
+
+	/* Each "DEVMEM_CONTEXT" has a counterpart in the server, which
+	 * is responsible for handling the mapping into device MMU.
+	 * We have a handle to that here.
+	 */
+	IMG_HANDLE hDevMemServerContext;
+
+	/* Number of automagically created heaps in this context,
+	 *  i.e. those that are born at context creation time from the
+	 * chosen "heap config" or "blueprint"
+	 */
+	IMG_UINT32 uiAutoHeapCount;
+
+	/* Pointer to array of such heaps */
+	struct DEVMEM_HEAP_TAG **ppsAutoHeapArray;
+
+	/* The cache line size for use when allocating memory,
+	 * as it is not queryable on the client side
+	 */
+	IMG_UINT32 ui32CPUCacheLineSize;
+
+	/* Private data handle for device specific data */
+	IMG_HANDLE hPrivData;
+};
+
+/* Flags that record how a heaps virtual address space is managed. */
+#define DEVMEM_HEAP_MANAGER_UNKNOWN      0
+/* Heap VAs assigned by the client of Services APIs, heap's RA not used at all. */
+#define DEVMEM_HEAP_MANAGER_USER         (1U << 0)
+/* Heap VAs managed by the OSs kernel, VA from CPU mapping call used */
+#define DEVMEM_HEAP_MANAGER_KERNEL       (1U << 1)
+/* Heap VAs managed by the heap's own RA */
+#define DEVMEM_HEAP_MANAGER_RA           (1U << 2)
+/* Heap VAs managed jointly by Services and the client of Services.
+ * The reserved region of the heap is managed explicitly by the client of Services
+ * The non-reserved region of the heap is managed by the heap's own RA */
+#define DEVMEM_HEAP_MANAGER_DUAL_USER_RA (DEVMEM_HEAP_MANAGER_USER | DEVMEM_HEAP_MANAGER_RA)
+
+struct DEVMEM_HEAP_TAG
+{
+	/* Name of heap - for debug and lookup purposes. */
+	IMG_CHAR *pszName;
+
+	/* Number of live imports in the heap */
+	ATOMIC_T hImportCount;
+
+	/* Base address and size of heap, required by clients due to some
+	 * requesters not being full range
+	 */
+	IMG_DEV_VIRTADDR sBaseAddress;
+	DEVMEM_SIZE_T uiSize;
+
+	DEVMEM_SIZE_T uiReservedRegionSize; /* uiReservedRegionLength in DEVMEM_HEAP_BLUEPRINT */
+
+	/* The heap manager, describing if the space is managed by the user, an RA,
+	 * kernel or combination */
+	IMG_UINT32 ui32HeapManagerFlags;
+
+	/* This RA is for managing sub-allocations within the imports (PMRs)
+	 * within the heap's virtual space. RA only used in DevmemSubAllocate()
+	 * to track sub-allocated buffers.
+	 *
+	 * Resource Span - a PMR import added when the RA calls the
+	 *                 imp_alloc CB (SubAllocImportAlloc) which returns the
+	 *                 PMR import and size (span length).
+	 * Resource - an allocation/buffer i.e. a MemDesc. Resource size represents
+	 *            the size of the sub-allocation.
+	 */
+	RA_ARENA *psSubAllocRA;
+	IMG_CHAR *pszSubAllocRAName;
+
+	/* The psQuantizedVMRA is for the coarse allocation (PMRs) of virtual
+	 * space from the heap.
+	 *
+	 * Resource Span - the heap's VM space from base to base+length,
+	 *                 only one is added at heap creation.
+	 * Resource - a PMR import associated with the heap. Dynamic number
+	 *            as memory is allocated/freed from or mapped/unmapped to
+	 *            the heap. Resource size follows PMR logical size.
+	 */
+	RA_ARENA *psQuantizedVMRA;
+	IMG_CHAR *pszQuantizedVMRAName;
+
+	/* We also need to store a copy of the quantum size in order to feed
+	 * this down to the server.
+	 */
+	IMG_UINT32 uiLog2Quantum;
+
+	/* Store a copy of the minimum import alignment */
+	IMG_UINT32 uiLog2ImportAlignment;
+
+	/* The parent memory context for this heap */
+	struct DEVMEM_CONTEXT_TAG *psCtx;
+
+	/* Lock to protect this structure */
+	POS_LOCK hLock;
+
+	/* Each "DEVMEM_HEAP" has a counterpart in the server, which is
+	 * responsible for handling the mapping into device MMU.
+	 * We have a handle to that here.
+	 */
+	IMG_HANDLE hDevMemServerHeap;
+
+	/* This heap is fully allocated and premapped into the device address space.
+	 * Used in virtualisation for firmware heaps of Guest and optionally Host
+	 * drivers. */
+	IMG_BOOL bPremapped;
+};
+
+typedef IMG_UINT32 DEVMEM_PROPERTIES_T;                  /*!< Typedef for Devicemem properties */
+#define DEVMEM_PROPERTIES_EXPORTABLE         (1UL<<0)    /*!< Is it exportable? */
+#define DEVMEM_PROPERTIES_IMPORTED           (1UL<<1)    /*!< Is it imported from another process? */
+#define DEVMEM_PROPERTIES_SUBALLOCATABLE     (1UL<<2)    /*!< Is it suballocatable? */
+#define DEVMEM_PROPERTIES_IMPORT_IS_ZEROED   (1UL<<4)    /*!< Is the memory fully zeroed? */
+#define DEVMEM_PROPERTIES_IMPORT_IS_CLEAN    (1UL<<5)    /*!< Is the memory clean, i.e. not been used before? */
+#define DEVMEM_PROPERTIES_SECURE             (1UL<<6)    /*!< Is it a special secure buffer? No CPU maps allowed! */
+#define DEVMEM_PROPERTIES_IMPORT_IS_POISONED (1UL<<7)    /*!< Is the memory fully poisoned? */
+#define DEVMEM_PROPERTIES_NO_CPU_MAPPING     (1UL<<8)    /* No CPU Mapping is allowed, RW attributes
+                                                            are further derived from allocation memory flags */
+#define DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE	 (1UL<<9)    /* No sparse resizing allowed, once a memory
+                                                            layout is chosen, no change allowed later */
+
+
+typedef struct DEVMEM_DEVICE_IMPORT_TAG
+{
+	DEVMEM_HEAP *psHeap;            /*!< Heap this import is bound to */
+	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the import */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
+	IMG_HANDLE hReservation;        /*!< Device memory reservation handle */
+	IMG_HANDLE hMapping;            /*!< Device mapping handle */
+	IMG_BOOL bMapped;               /*!< This is import mapped? */
+	POS_LOCK hLock;                 /*!< Lock to protect the device import */
+} DEVMEM_DEVICE_IMPORT;
+
+typedef struct DEVMEM_CPU_IMPORT_TAG
+{
+	void *pvCPUVAddr;               /*!< CPU virtual address of the import */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the CPU virtual address */
+	IMG_HANDLE hOSMMapData;         /*!< CPU mapping handle */
+	POS_LOCK hLock;                 /*!< Lock to protect the CPU import */
+} DEVMEM_CPU_IMPORT;
+
+typedef struct DEVMEM_IMPORT_TAG
+{
+	SHARED_DEV_CONNECTION hDevConnection;
+	IMG_DEVMEM_ALIGN_T uiAlign;         /*!< Alignment of the PMR */
+	DEVMEM_SIZE_T uiSize;               /*!< Size of import */
+	ATOMIC_T hRefCount;                 /*!< Refcount for this import */
+	DEVMEM_PROPERTIES_T uiProperties;   /*!< Stores properties of an import like if
+	                                         it is exportable, pinned or suballocatable */
+	IMG_HANDLE hPMR;                    /*!< Handle to the PMR */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;     /*!< Flags for this import */
+	POS_LOCK hLock;                     /*!< Lock to protect the import */
+
+	DEVMEM_DEVICE_IMPORT sDeviceImport; /*!< Device specifics of the import */
+	DEVMEM_CPU_IMPORT sCPUImport;       /*!< CPU specifics of the import */
+} DEVMEM_IMPORT;
+
+typedef struct DEVMEM_DEVICE_MEMDESC_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the allocation */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
+	POS_LOCK hLock;                 /*!< Lock to protect device memdesc */
+} DEVMEM_DEVICE_MEMDESC;
+
+typedef struct DEVMEM_CPU_MEMDESC_TAG
+{
+	void *pvCPUVAddr;           /*!< CPU virtual address of the import */
+	IMG_UINT32 ui32RefCount;    /*!< Refcount of the device CPU address */
+	POS_LOCK hLock;             /*!< Lock to protect CPU memdesc */
+} DEVMEM_CPU_MEMDESC;
+
+struct DEVMEM_MEMDESC_TAG
+{
+	DEVMEM_IMPORT *psImport;                /*!< Import this memdesc is on */
+	IMG_DEVMEM_OFFSET_T uiOffset;           /*!< Offset into import where our allocation starts */
+	IMG_DEVMEM_SIZE_T uiAllocSize;          /*!< Size of the allocation */
+	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
+	POS_LOCK hLock;                         /*!< Lock to protect memdesc */
+	IMG_HANDLE hPrivData;
+
+	DEVMEM_DEVICE_MEMDESC sDeviceMemDesc;   /*!< Device specifics of the memdesc */
+	DEVMEM_CPU_MEMDESC sCPUMemDesc;         /*!< CPU specifics of the memdesc */
+
+	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this memdesc */
+
+	IMG_UINT32 ui32AllocationIndex;
+
+#if defined(DEBUG)
+	IMG_BOOL bPoisonOnFree;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
+#endif
+};
+
+/* The physical descriptor used to store handles and information of device
+ * physical allocations.
+ */
+struct DEVMEMX_PHYS_MEMDESC_TAG
+{
+	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
+	IMG_UINT32 uiLog2PageSize;              /*!< Page size */
+	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;         /*!< Flags for this import */
+	IMG_HANDLE hPMR;                        /*!< Handle to the PMR */
+	DEVMEM_CPU_IMPORT sCPUImport;           /*!< CPU specifics of the memdesc */
+	SHARED_DEV_CONNECTION hConnection;      /*!< Services connection for the server */
+	void *pvUserData;						/*!< User data */
+};
+
+/* The virtual descriptor used to store handles and information of a device
+ * virtual range and the mappings to it.
+ */
+struct DEVMEMX_VIRT_MEMDESC_TAG
+{
+	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;         /*!< Flags for this import */
+	DEVMEMX_PHYSDESC **apsPhysDescTable;    /*!< Table to store links to physical descs */
+	DEVMEM_DEVICE_IMPORT sDeviceImport;     /*!< Device specifics of the memdesc */
+
+	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this virt memdesc */
+	IMG_UINT32 ui32AllocationIndex;         /*!< To track mappings in this range */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
+#endif
+};
+
+#define DEVICEMEM_UTILS_NO_ADDRESS 0
+
+/******************************************************************************
+@Function       DevmemValidateParams
+@Description    Check if flags are conflicting and if align is a size multiple.
+
+@Input          uiSize      Size of the import.
+@Input          uiAlign     Alignment of the import.
+@Input          puiFlags    Pointer to the flags for the import.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemValidateParams(IMG_DEVMEM_SIZE_T uiSize,
+                                   IMG_DEVMEM_ALIGN_T uiAlign,
+                                   PVRSRV_MEMALLOCFLAGS_T *puiFlags);
+
+/******************************************************************************
+@Function       DevmemImportStructAlloc
+@Description    Allocates memory for an import struct. Does not allocate a PMR!
+                Create locks for CPU and Devmem mappings.
+
+@Input          hDevConnection  Connection to use for calls from the import.
+@Input          ppsImport       The import to allocate.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructAlloc(SHARED_DEV_CONNECTION hDevConnection,
+                                      DEVMEM_IMPORT **ppsImport);
+
+/******************************************************************************
+@Function       DevmemImportStructInit
+@Description    Initialises the import struct with the given parameters.
+                Set it's refcount to 1!
+
+@Input          psImport     The import to initialise.
+@Input          uiSize       Size of the import.
+@Input          uiAlign      Alignment of allocations in the import.
+@Input          uiMapFlags
+@Input          hPMR         Reference to the PMR of this import struct.
+@Input          uiProperties Properties of the import. Is it exportable,
+                              imported, suballocatable?
+******************************************************************************/
+void DevmemImportStructInit(DEVMEM_IMPORT *psImport,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             IMG_DEVMEM_ALIGN_T uiAlign,
+                             PVRSRV_MEMALLOCFLAGS_T uiMapFlags,
+                             IMG_HANDLE hPMR,
+                             DEVMEM_PROPERTIES_T uiProperties);
+
+/******************************************************************************
+@Function       DevmemImportStructDevMap
+@Description    NEVER call after the last DevmemMemDescRelease()
+                Maps the PMR referenced by the import struct to the device's
+                virtual address space.
+                Does nothing but increase the cpu mapping refcount if the
+                import struct was already mapped.
+
+@Input          psHeap    The heap to map to.
+@Input          bMap      Caller can choose if the import should be really
+                          mapped in the page tables or if just a virtual range
+                          should be reserved and the refcounts increased.
+@Input          psImport  The import we want to map.
+@Input          uiOptionalMapAddress  An optional address to map to.
+                                      Pass DEVICEMEM_UTILS_NOADDRESS if not used.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructDevMap(DEVMEM_HEAP *psHeap,
+                                       IMG_BOOL bMap,
+                                       DEVMEM_IMPORT *psImport,
+                                       IMG_UINT64 uiOptionalMapAddress);
+
+/******************************************************************************
+@Function       DevmemImportStructDevUnmap
+@Description    Unmaps the PMR referenced by the import struct from the
+                device's virtual address space.
+                If this was not the last remaining CPU mapping on the import
+                struct only the cpu mapping refcount is decreased.
+@return         A boolean to signify if the import was unmapped.
+******************************************************************************/
+IMG_BOOL DevmemImportStructDevUnmap(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructCPUMap
+@Description    NEVER call after the last DevmemMemDescRelease()
+                Maps the PMR referenced by the import struct to the CPU's
+                virtual address space.
+                Does nothing but increase the cpu mapping refcount if the
+                import struct was already mapped.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructCPUMap(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructCPUUnmap
+@Description    Unmaps the PMR referenced by the import struct from the CPU's
+                virtual address space.
+                If this was not the last remaining CPU mapping on the import
+                struct only the cpu mapping refcount is decreased.
+******************************************************************************/
+void DevmemImportStructCPUUnmap(DEVMEM_IMPORT *psImport);
+
+
+/******************************************************************************
+@Function       DevmemImportStructAcquire
+@Description    Acquire an import struct by increasing it's refcount.
+******************************************************************************/
+void DevmemImportStructAcquire(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructRelease
+@Description    Reduces the refcount of the import struct.
+                Destroys the import in the case it was the last reference.
+                Destroys underlying PMR if this import was the last reference
+                to it.
+@return         A boolean to signal if the import was destroyed. True = yes.
+******************************************************************************/
+IMG_BOOL DevmemImportStructRelease(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportDiscard
+@Description    Discard a created, but uninitialised import structure.
+                This must only be called before DevmemImportStructInit
+                after which DevmemImportStructRelease must be used to
+                "free" the import structure.
+******************************************************************************/
+void DevmemImportDiscard(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemMemDescAlloc
+@Description    Allocates a MemDesc and create it's various locks.
+                Zero the allocated memory.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemMemDescAlloc(DEVMEM_MEMDESC **ppsMemDesc);
+
+#if defined(DEBUG)
+/******************************************************************************
+@Function       DevmemMemDescSetPoF
+@Description    Sets the Poison on Free flag to true for this MemDesc if the
+                given MemAllocFlags have the Poison on Free bit set.
+                Poison on Free is a debug only feature.
+******************************************************************************/
+void DevmemMemDescSetPoF(DEVMEM_MEMDESC *psMemDesc, PVRSRV_MEMALLOCFLAGS_T uiFlags);
+#endif
+
+/******************************************************************************
+@Function       DevmemMemDescInit
+@Description    Sets the given offset and import struct fields in the MemDesc.
+                Initialises refcount to 1 and other values to 0.
+
+@Input          psMemDesc    MemDesc to initialise.
+@Input          uiOffset     Offset in the import structure.
+@Input          psImport     Import the MemDesc is on.
+@Input          uiAllocSize  Size of the allocation
+******************************************************************************/
+void DevmemMemDescInit(DEVMEM_MEMDESC *psMemDesc,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						DEVMEM_IMPORT *psImport,
+						IMG_DEVMEM_SIZE_T uiAllocSize);
+
+/******************************************************************************
+@Function       DevmemMemDescAcquire
+@Description    Acquires the MemDesc by increasing it's refcount.
+******************************************************************************/
+void DevmemMemDescAcquire(DEVMEM_MEMDESC *psMemDesc);
+
+/******************************************************************************
+@Function       DevmemMemDescRelease
+@Description    Releases the MemDesc by reducing it's refcount.
+                Destroy the MemDesc if it's recount is 0.
+                Destroy the import struct the MemDesc is on if that was the
+                last MemDesc on the import, probably following the destruction
+                of the underlying PMR.
+@return         A boolean to signal if the MemDesc was destroyed. True = yes.
+******************************************************************************/
+IMG_BOOL DevmemMemDescRelease(DEVMEM_MEMDESC *psMemDesc);
+
+/******************************************************************************
+@Function       DevmemMemDescDiscard
+@Description    Discard a created, but uninitialised MemDesc structure.
+                This must only be called before DevmemMemDescInit after
+                which DevmemMemDescRelease must be used to "free" the
+                MemDesc structure.
+******************************************************************************/
+void DevmemMemDescDiscard(DEVMEM_MEMDESC *psMemDesc);
+
+
+/******************************************************************************
+@Function       GetImportProperties
+@Description    Atomically read psImport->uiProperties
+                It's possible that another thread modifies uiProperties
+                immediately after this function returns, making its result
+                stale. So, it's recommended to use this function only to
+                check if certain non-volatile flags were set.
+******************************************************************************/
+static INLINE DEVMEM_PROPERTIES_T GetImportProperties(DEVMEM_IMPORT *psImport)
+{
+	DEVMEM_PROPERTIES_T uiProperties;
+
+	OSLockAcquire(psImport->hLock);
+	uiProperties = psImport->uiProperties;
+	OSLockRelease(psImport->hLock);
+	return uiProperties;
+}
+
+/******************************************************************************
+@Function       DevmemCPUMemSet
+@Description    Given a CPU Mapped Devmem address, set the memory at that
+                range (address, address + size) to the uiPattern provided.
+                Flags determine the OS abstracted MemSet method to use.
+******************************************************************************/
+static INLINE void DevmemCPUMemSet(void *pvMem,
+                                   IMG_UINT8 uiPattern,
+                                   IMG_DEVMEM_SIZE_T uiSize,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags))
+	{
+		OSDeviceMemSet(pvMem, uiPattern, uiSize);
+	}
+	else
+	{
+		/* it's safe to use OSCachedMemSet() for cached and wc memory */
+		OSCachedMemSet(pvMem, uiPattern, uiSize);
+	}
+}
+
+/******************************************************************************
+@Function       DevmemCPUMapCheckImportProperties
+@Description    Given a MemDesc check that the import properties are correct
+                to allow for mapping the MemDesc to the CPU.
+                Returns PVRSRV_OK on success.
+******************************************************************************/
+static INLINE PVRSRV_ERROR DevmemCPUMapCheckImportProperties(DEVMEM_MEMDESC *psMemDesc)
+{
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiProperties & DEVMEM_PROPERTIES_SECURE)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+					"%s: Allocation is a secure buffer. "
+					"It should not be possible to map to CPU, but for security "
+					"validation this will be allowed for testing purposes, "
+					"as long as the buffer is pinned.",
+					__func__));
+	}
+#endif
+
+	if (uiProperties & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU Mapping is not possible on this allocation!",
+				__func__));
+		return PVRSRV_ERROR_INVALID_MAP_REQUEST;
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* DEVICEMEM_UTILS_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx.h
new file mode 100644
index 000000000000..86049c4a705c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx.h
@@ -0,0 +1,221 @@
+/*************************************************************************/ /*!
+@File
+@Title          X Device Memory Management core internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Internal interface for extended device memory management.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEMX_H
+#define DEVICEMEMX_H
+
+#include "img_types.h"
+#include "devicemem_typedefs.h"
+#include "devicemem_utils.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "osfunc.h"
+
+/* DevmemXAllocPhysical()
+ *
+ * Allocate physical device memory and return a physical
+ * descriptor for it.
+ */
+PVRSRV_ERROR
+DevmemXAllocPhysical(DEVMEM_CONTEXT *psCtx,
+                    IMG_UINT32 uiNumPages,
+                    IMG_UINT32 uiLog2PageSize,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    const IMG_CHAR *pszText,
+                    DEVMEMX_PHYSDESC **ppsPhysDesc);
+
+/* DevmemXReleasePhysical()
+ *
+ * Removes a physical device allocation if all references
+ * to it are dropped, otherwise just decreases the refcount.
+ */
+void
+DevmemXReleasePhysical(DEVMEMX_PHYSDESC *psPhysDesc);
+
+/* DevmemAllocVirtualAddr()
+ *
+ * Reserve a requested device virtual range and return
+ * a virtual descriptor for it.
+ */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemXAllocVirtualAddr(DEVMEM_HEAP* hHeap,
+                   IMG_UINT32 uiNumPages,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   const IMG_CHAR *pszText,
+                   IMG_DEV_VIRTADDR sVirtAddr,
+                   DEVMEMX_VIRTDESC **ppsVirtDesc);
+
+/* DevmemAllocVirtual()
+ *
+ * Allocate and reserve a device virtual range and return
+ * a virtual descriptor for it.
+ */
+PVRSRV_ERROR
+DevmemXAllocVirtual(DEVMEM_HEAP* hHeap,
+                   IMG_UINT32 uiNumPages,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   const IMG_CHAR *pszText,
+                   DEVMEMX_VIRTDESC **ppsVirtDesc,
+                   IMG_DEV_VIRTADDR *psVirtAddr);
+
+/* DevmemXFreeVirtual()
+ *
+ * Removes a device virtual range if all mappings on it
+ * have been removed.
+ */
+PVRSRV_ERROR
+DevmemXFreeVirtual(DEVMEMX_VIRTDESC *psVirtDesc);
+
+/* DevmemXMapVirtualRange()
+ *
+ * Map memory from a physical descriptor into a virtual range.
+ */
+PVRSRV_ERROR
+DevmemXMapVirtualRange(IMG_UINT32 ui32PageCount,
+                      DEVMEMX_PHYSDESC *psPhysDesc,
+                      IMG_UINT32 ui32PhysOffset,
+                      DEVMEMX_VIRTDESC *psVirtDesc,
+                      IMG_UINT32 ui32VirtOffset);
+
+/* DevmemXUnmapVirtualRange()
+ *
+ * Unmap pages from a device virtual range.
+ */
+PVRSRV_ERROR
+DevmemXUnmapVirtualRange(IMG_UINT32 ui32PageCount,
+                        DEVMEMX_VIRTDESC *psVirtDesc,
+                        IMG_UINT32 ui32VirtPgOffset);
+
+/* DevmemXMapVirtualRangeToBackingPage()
+ *
+ * Map a virtual range to kernel backing page.
+ */
+PVRSRV_ERROR
+DevmemXMapVirtualRangeToBackingPage(DEVMEMX_VIRTDESC *psVirtDesc,
+                                    PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags,
+                                    IMG_UINT32 ui32VirtPgOffset,
+                                    IMG_UINT32 ui32PageCount);
+
+/* DevmemXMapPhysicalToCPU()
+ *
+ * Map a full physical descriptor to CPU space.
+ */
+PVRSRV_ERROR
+DevmemXMapPhysicalToCPU(DEVMEMX_PHYSDESC *psMemAllocPhys,
+                       IMG_CPU_VIRTADDR *psVirtAddr);
+
+/* DevmemXUnmapPhysicalToCPU()
+ *
+ * Remove the CPU mapping from the descriptor.
+ */
+PVRSRV_ERROR
+DevmemXUnmapPhysicalToCPU(DEVMEMX_PHYSDESC *psMemAllocPhys);
+
+/* DevmemXReacquireCpuVirtAddr()
+ *
+ * Reacquire the CPU mapping by incrementing the refcount.
+ */
+void
+DevmemXReacquireCpuVirtAddr(DEVMEMX_PHYSDESC *psPhysDesc,
+                            void **ppvCpuVirtAddr);
+
+/* DevmemXReleaseCpuVirtAddr()
+ *
+ * Release CPU mapping by decrementing the refcount.
+ */
+void
+DevmemXReleaseCpuVirtAddr(DEVMEMX_PHYSDESC *psPhysDesc);
+
+/* DevmemXCreateDevmemMemDesc()
+ *
+ * Create a devmem memdesc from a physical and
+ * virtual descriptor.
+ * Always destroy with DevmemXFreeDevmemMemDesc().
+ */
+
+PVRSRV_ERROR
+DevmemXCreateDevmemMemDesc(DEVMEMX_PHYSDESC *psPhysDesc,
+                           DEVMEMX_VIRTDESC *psVirtDesc,
+                           DEVMEM_MEMDESC **ppsMemDesc);
+
+/* DevmemXFreeDevmemMemDesc()
+ *
+ * Free the memdesc again. Has no impact on the underlying
+ * physical and virtual descriptors.
+ */
+PVRSRV_ERROR
+DevmemXFreeDevmemMemDesc(DEVMEM_MEMDESC *psMemDesc);
+
+PVRSRV_ERROR
+DevmemXFlagCompatibilityCheck(PVRSRV_MEMALLOCFLAGS_T uiPhysFlags,
+                              PVRSRV_MEMALLOCFLAGS_T uiVirtFlags);
+
+PVRSRV_ERROR
+DevmemXPhysDescAlloc(DEVMEMX_PHYSDESC **ppsPhysDesc);
+
+void
+DevmemXPhysDescInit(DEVMEMX_PHYSDESC *psPhysDesc,
+                    IMG_HANDLE hPMR,
+                    IMG_UINT32 uiNumPages,
+                    IMG_UINT32 uiLog2PageSize,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    SHARED_DEV_CONNECTION hConnection);
+
+void
+DevmemXPhysDescFree(DEVMEMX_PHYSDESC *psPhysDesc);
+
+void
+DevmemXPhysDescAcquire(DEVMEMX_PHYSDESC *psPhysDesc,
+                       IMG_UINT32 uiAcquireCount);
+void
+DevmemXPhysDescRelease(DEVMEMX_PHYSDESC *psPhysDesc,
+                       IMG_UINT32 uiReleaseCount);
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemXGetImportUID(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_UINT64       *pui64UID);
+#endif
+
+#endif /* DEVICEMEMX_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx_pdump.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx_pdump.h
new file mode 100644
index 000000000000..b6e99f7eca66
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/devicememx_pdump.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File
+@Title          X Device Memory Management PDump internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to PDump device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEMX_PDUMP_H
+#define DEVICEMEMX_PDUMP_H
+
+#include "devicememx.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+
+#if defined(PDUMP)
+/*
+ * DevmemXPDumpLoadMem()
+ *
+ * Same as DevmemPDumpLoadMem().
+ */
+IMG_INTERNAL void
+DevmemXPDumpLoadMem(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    PDUMP_FLAGS_T uiPDumpFlags);
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemXPDumpLoadMem)
+#endif
+
+static INLINE void
+DevmemXPDumpLoadMem(DEVMEMX_PHYSDESC *psMemDescPhys,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDescPhys);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+#endif /* PDUMP */
+#endif /* DEVICEMEMX_PDUMP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash.h
new file mode 100644
index 000000000000..feb879c1489a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash.h
@@ -0,0 +1,255 @@
+/*************************************************************************/ /*!
+@File
+@Title          Self scaling hash tables
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements simple self scaling hash tables.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HASH_H
+#define HASH_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * Keys passed to the comparison function are only guaranteed to be aligned on
+ * an uintptr_t boundary.
+ */
+typedef IMG_UINT32 HASH_FUNC(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+typedef IMG_BOOL HASH_KEY_COMP(size_t uKeySize, void *pKey1, void *pKey2);
+
+typedef struct _HASH_TABLE_ HASH_TABLE;
+
+typedef PVRSRV_ERROR (*HASH_pfnCallback) (
+	uintptr_t k,
+	uintptr_t v,
+	void* pvPriv
+);
+
+#if defined(DEBUG)
+#else
+#define HASH_CREATE(LEN)		HASH_Create(LEN)
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Func_Default
+@Description    Hash function intended for hashing keys composed of uintptr_t
+                arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey         A pointer to the key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Func_Default(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares keys composed of uintptr_t arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey1        Pointer to first hash key to compare.
+@Input          pKey2        Pointer to second hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Key_Comp_Default(size_t uKeySize, void *pKey1, void *pKey2);
+
+/*************************************************************************/ /*!
+@Function       HASH_Create_Extended
+@Description    Create a self scaling hash table, using the supplied key size,
+                and the supplied hash and key comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Input          uKeySize     The size of the key, in bytes.
+@Input          pfnHashFunc  Pointer to hash function.
+@Input          pfnKeyComp   Pointer to key comparison function.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+HASH_TABLE * HASH_Create_Extended_Int(IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp);
+#if defined(DEBUG)
+#define HASH_Create_Extended(LEN, KS, FUN, CMP)		HASH_Create_Extended_Debug(LEN, KS, FUN, CMP, __FILE__, __LINE__)
+HASH_TABLE * HASH_Create_Extended_Debug (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp,
+										 const char *file, const unsigned int line);
+#else
+#define HASH_Create_Extended	HASH_Create_Extended_Int
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Create
+@Description    Create a self scaling hash table with a key consisting of a
+                single uintptr_t, and using the default hash and key
+                comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+HASH_TABLE * HASH_Create_Int(IMG_UINT32 uInitialLen);
+#if defined(DEBUG)
+#define HASH_Create(LEN)		HASH_Create_Debug(LEN, __FILE__, __LINE__)
+HASH_TABLE * HASH_Create_Debug (IMG_UINT32 uInitialLen, const char *file, const unsigned int line);
+#else
+#define HASH_Create				HASH_Create_Int
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete_Extended
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table should have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Input          bWarn        Set false to suppress warnings in the case of
+                             deletion with active entries.
+@Return         None
+*/ /**************************************************************************/
+void HASH_Delete_Extended(HASH_TABLE *pHash, IMG_BOOL bWarn);
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table must have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Return         None
+*/ /**************************************************************************/
+void HASH_Delete(HASH_TABLE *pHash);
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert_Extended
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to the key.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Insert_Extended(HASH_TABLE *pHash, void *pKey, uintptr_t v);
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Insert(HASH_TABLE *pHash, uintptr_t k, uintptr_t v);
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove_Extended
+@Description    Remove a key from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Remove_Extended(HASH_TABLE *pHash, void *pKey);
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove
+@Description    Remove a key value pair from a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Remove(HASH_TABLE *pHash, uintptr_t k);
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve_Extended
+@Description    Retrieve a value from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Retrieve_Extended(HASH_TABLE *pHash, void *pKey);
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve
+@Description    Retrieve a value from a hash table created with HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Retrieve(HASH_TABLE *pHash, uintptr_t k);
+
+/*************************************************************************/ /*!
+@Function       HASH_Iterate
+@Description    Iterate over every entry in the hash table.
+@Input          pHash        Hash table to iterate.
+@Input          pfnCallback  Callback to call with the key and data for each
+.                            entry in the hash table
+@Return         Callback error if any, otherwise PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR HASH_Iterate(HASH_TABLE *pHash, HASH_pfnCallback pfnCallback, void* args);
+
+/*************************************************************************/ /*!
+@Function       HASH_Count
+@Description    Retrieve the number of entries in the hash table.
+@Input          pHash        The hash table.
+@Return         The number of entries.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Count(HASH_TABLE *pHash);
+
+#ifdef HASH_TRACE
+/*************************************************************************/ /*!
+@Function       HASH_Dump
+@Description    Dump out some information about a hash table.
+@Input          pHash         The hash table.
+*/ /**************************************************************************/
+void HASH_Dump(HASH_TABLE *pHash);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HASH_H */
+
+/******************************************************************************
+ End of file (hash.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash_functions.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash_functions.h
new file mode 100644
index 000000000000..154770848be4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/hash_functions.h
@@ -0,0 +1,66 @@
+/*************************************************************************/ /*!
+@File
+@Title          Reusable hash functions for hash.c.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common hash functions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+
+/*************************************************************************/ /*!
+@Function       HASH_Djb2_Hash
+@Description    Hash function intended for hashing string keys. This function
+                implements DJB2 algorithm.
+@Input          uKeySize     The size of the string hash key, in bytes.
+@Input          pKey         A pointer to the string key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Djb2_Hash(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares string keys.
+@Input          uKeySize     The size of the string key.
+@Input          pKey1        Pointer to first string hash key to compare.
+@Input          pKey2        Pointer to second string hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Djb2_Compare(size_t uKeySize, void *pKey1, void *pKey2);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer.h
new file mode 100644
index 000000000000..96daa9727c47
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@File           htbuffer.h
+@Title          Host Trace Buffer shared API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_H
+#define HTBUFFER_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "htbuffer_sf.h"
+#include "htbuffer_types.h"
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+ @Input         hSrvHandle      Server Handle
+ @Input         ui32NumFlagGroups Number of group enable flags words
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+ @Input         ui32LogLevel    Log level to record
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+ @Input         eLogPidMode     Enable logging for all or specific processes,
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogPidMode,
+	HTB_OPMODE_CTRL eOpMode);
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_H */
+/*****************************************************************************
+ End of file (htbuffer.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer_init.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer_init.h
new file mode 100644
index 000000000000..d114579964b4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/htbuffer_init.h
@@ -0,0 +1,114 @@
+/*************************************************************************/ /*!
+@File           htbuffer_init.h
+@Title          Host Trace Buffer functions needed for Services initialisation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_INIT_H
+#define HTBUFFER_INIT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+/*************************************************************************/ /*!
+ @Function      HTBConfigure
+ @Description   Configure the Host Trace Buffer.
+                Once these parameters are set they may not be changed
+
+ @Input         hSrvHandle      Server Handle
+
+ @Input         pszBufferName   Name to use for the TL buffer, this will be
+                                required to request trace data from the TL
+
+ @Input         ui32BufferSize  Requested TL buffer size in bytes
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBConfigure(
+	IMG_HANDLE hSrvHandle,
+	IMG_CHAR * pszBufferName,
+	IMG_UINT32 ui32BufferSize
+);
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         hSrvHandle      Server Handle
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogMode,
+	HTB_OPMODE_CTRL eOpMode
+);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_INIT_H */
+/*****************************************************************************
+ End of file (htbuffer_init.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/lock.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/lock.h
new file mode 100644
index 000000000000..386e3b2f88a2
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/lock.h
@@ -0,0 +1,527 @@
+/*************************************************************************/ /*!
+@File           lock.h
+@Title          Locking interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal locking interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOCK_H
+#define LOCK_H
+
+/* In Linux kernel mode we are using the kernel mutex implementation directly
+ * with macros. This allows us to use the kernel lockdep feature for lock
+ * debugging. */
+#include "lock_types.h"
+#include "pvr_debug.h"
+
+#if defined(__linux__) && defined(__KERNEL__)
+
+#include "allocmem.h"
+#include <linux/atomic.h>
+#include <linux/version.h>
+
+#define OSLockCreateNoStats(phLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(phLock) = OSAllocMemNoStats(sizeof(struct mutex)); \
+	if (*(phLock)) { mutex_init(*(phLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSLockCreate(phLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(phLock) = OSAllocMem(sizeof(struct mutex)); \
+	if (*(phLock)) { mutex_init(*(phLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSLockDestroy(hLock) ({mutex_destroy((hLock)); OSFreeMem((hLock));})
+#define OSLockDestroyNoStats(hLock) ({mutex_destroy((hLock)); OSFreeMemNoStats((hLock));})
+
+#define OSLockAcquire(hLock) ({mutex_lock((hLock));})
+#define OSLockAcquireNested(hLock, subclass) ({mutex_lock_nested((hLock), (subclass));})
+#define OSLockRelease(hLock) ({mutex_unlock((hLock));})
+
+#define OSLockIsLocked(hLock) ((mutex_is_locked((hLock)) == 1) ? IMG_TRUE : IMG_FALSE)
+#define OSTryLockAcquire(hLock) ((mutex_trylock(hLock) == 1) ? IMG_TRUE : IMG_FALSE)
+
+#if defined(DEBUG)
+#if defined(CONFIG_LOCKDEP)
+#define OSLockHeldAssert(hLock) ({ lockdep_assert_held((hLock)); })
+#else
+#define OSLockHeldAssert(hLock) ({ PVR_ASSERT(OSLockIsLocked((hLock)) == IMG_TRUE); })
+#endif
+#else
+#define OSLockHeldAssert(hLock)
+#endif
+
+#define OSSpinLockCreate(_ppsLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(_ppsLock) = OSAllocMem(sizeof(spinlock_t)); \
+	if (*(_ppsLock)) {spin_lock_init(*(_ppsLock)); e = PVRSRV_OK;} \
+	e;})
+#define OSSpinLockDestroy(_psLock) ({OSFreeMem(_psLock);})
+
+typedef unsigned long OS_SPINLOCK_FLAGS;
+#define OSSpinLockAcquire(_pLock, _flags) spin_lock_irqsave(_pLock, _flags)
+#define OSSpinLockRelease(_pLock, _flags) spin_unlock_irqrestore(_pLock, _flags)
+
+/* These _may_ be reordered or optimized away entirely by the compiler/hw */
+#define OSAtomicRead(pCounter)	atomic_read(pCounter)
+#define OSAtomicWrite(pCounter, i)	atomic_set(pCounter, i)
+
+/* The following atomic operations, in addition to being SMP-safe, also
+   imply a memory barrier around the operation */
+#define OSAtomicIncrement(pCounter) atomic_inc_return(pCounter)
+#define OSAtomicDecrement(pCounter) atomic_dec_return(pCounter)
+#define OSAtomicCompareExchange(pCounter, oldv, newv) atomic_cmpxchg(pCounter,oldv,newv)
+#define OSAtomicExchange(pCounter, iNewVal) atomic_xchg(pCounter, iNewVal)
+
+static inline IMG_INT OSAtomicOr(ATOMIC_T *pCounter, IMG_INT iVal)
+{
+	IMG_INT iOldVal, iLastVal, iNewVal;
+
+	iLastVal = OSAtomicRead(pCounter);
+	do
+	{
+		iOldVal = iLastVal;
+		iNewVal = iOldVal | iVal;
+
+		iLastVal = OSAtomicCompareExchange(pCounter, iOldVal, iNewVal);
+	}
+	while (iOldVal != iLastVal);
+
+	return iNewVal;
+}
+
+#define OSAtomicAdd(pCounter, incr) atomic_add_return(incr,pCounter)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)
+#define OSAtomicAddUnless(pCounter, incr, test) atomic_fetch_add_unless(pCounter, (incr), (test))
+#else
+#define OSAtomicAddUnless(pCounter, incr, test) __atomic_add_unless(pCounter,incr,test)
+#endif
+
+#define OSAtomicSubtract(pCounter, incr) atomic_add_return(-(incr),pCounter)
+#define OSAtomicSubtractUnless(pCounter, incr, test) OSAtomicAddUnless(pCounter, -(incr), (test))
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+/**************************************************************************/ /*!
+@Function       OSLockCreate
+@Description    Creates an operating system lock object.
+@Output         phLock           The created lock.
+@Return         PVRSRV_OK on success. PVRSRV_ERROR_OUT_OF_MEMORY if the driver
+                cannot allocate CPU memory needed for the lock.
+                PVRSRV_ERROR_INIT_FAILURE if the Operating System fails to
+                allocate the lock.
+ */ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR OSLockCreate(POS_LOCK *phLock);
+#if defined(INTEGRITY_OS) || defined(__QNXNTO__)
+#define OSLockCreateNoStats OSLockCreate
+#endif
+
+/**************************************************************************/ /*!
+@Function       OSLockDestroy
+@Description    Destroys an operating system lock object.
+@Input          hLock            The lock to be destroyed.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockDestroy(POS_LOCK hLock);
+
+#if defined(INTEGRITY_OS) || defined(__QNXNTO__)
+#define OSLockDestroyNoStats OSLockDestroy
+#endif
+/**************************************************************************/ /*!
+@Function       OSLockAcquire
+@Description    Acquires an operating system lock.
+                NB. This function must not return until the lock is acquired
+                (meaning the implementation should not timeout or return with
+                an error, as the caller will assume they have the lock).
+@Input          hLock            The lock to be acquired.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockAcquire(POS_LOCK hLock);
+
+/**************************************************************************/ /*!
+@Function       OSTryLockAcquire
+@Description    Try to acquire an operating system lock.
+                NB. If lock is acquired successfully in the first attempt,
+                then the function returns true and else it will return false.
+@Input          hLock            The lock to be acquired.
+@Return         IMG_TRUE if lock acquired successfully,
+                IMG_FALSE otherwise.
+ */ /**************************************************************************/
+IMG_INTERNAL
+IMG_BOOL OSTryLockAcquire(POS_LOCK hLock);
+
+/* Nested notation isn't used in UM or other OS's */
+/**************************************************************************/ /*!
+@Function       OSLockAcquireNested
+@Description    For operating systems other than Linux, this equates to an
+                OSLockAcquire() call. On Linux, this function wraps a call
+                to mutex_lock_nested(). This recognises the scenario where
+                there may be multiple subclasses within a particular class
+                of lock. In such cases, the order in which the locks belonging
+                these various subclasses are acquired is important and must be
+                validated.
+@Input          hLock            The lock to be acquired.
+@Input          subclass         The subclass of the lock.
+@Return         None.
+ */ /**************************************************************************/
+#define OSLockAcquireNested(hLock, subclass) OSLockAcquire((hLock))
+
+/**************************************************************************/ /*!
+@Function       OSLockRelease
+@Description    Releases an operating system lock.
+@Input          hLock            The lock to be released.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockRelease(POS_LOCK hLock);
+
+/**************************************************************************/ /*!
+@Function       OSLockIsLocked
+@Description    Tests whether or not an operating system lock is currently
+                locked.
+@Input          hLock            The lock to be tested.
+@Return         IMG_TRUE if locked, IMG_FALSE if not locked.
+ */ /**************************************************************************/
+IMG_INTERNAL
+IMG_BOOL OSLockIsLocked(POS_LOCK hLock);
+
+#define OSLockHeldAssert(hLock) PVR_ASSERT(OSLockIsLocked((hLock)) == IMG_TRUE)
+
+#if defined(__linux__)
+
+/* Use GCC intrinsics (read/write semantics consistent with kernel-side implementation) */
+#define OSAtomicRead(pCounter) (*(volatile IMG_INT32 *)&(pCounter)->counter)
+#define OSAtomicWrite(pCounter, i) ((pCounter)->counter = (IMG_INT32) i)
+#define OSAtomicIncrement(pCounter) __sync_add_and_fetch((&(pCounter)->counter), 1)
+#define OSAtomicDecrement(pCounter) __sync_sub_and_fetch((&(pCounter)->counter), 1)
+#define OSAtomicCompareExchange(pCounter, oldv, newv) \
+	__sync_val_compare_and_swap((&(pCounter)->counter), oldv, newv)
+#define OSAtomicOr(pCounter, iVal) __sync_or_and_fetch((&(pCounter)->counter), iVal)
+
+static inline IMG_UINT32 OSAtomicExchange(ATOMIC_T *pCounter, IMG_UINT32 iNewVal)
+{
+	IMG_UINT32 iOldVal;
+	IMG_UINT32 iLastVal;
+
+	iLastVal = OSAtomicRead(pCounter);
+	do
+	{
+		iOldVal = iLastVal;
+		iLastVal = OSAtomicCompareExchange(pCounter, iOldVal, iNewVal);
+	}
+	while (iOldVal != iLastVal);
+
+	return iOldVal;
+}
+
+#define OSAtomicAdd(pCounter, incr) __sync_add_and_fetch((&(pCounter)->counter), incr)
+#define OSAtomicAddUnless(pCounter, incr, test) ({ \
+	IMG_INT32 c; IMG_INT32 old; \
+	c = OSAtomicRead(pCounter); \
+	while (1) { \
+		if (c == (test)) break; \
+		old = OSAtomicCompareExchange(pCounter, c, c+(incr)); \
+		if (old == c) break; \
+		c = old; \
+	} c; })
+
+#define OSAtomicSubtract(pCounter, incr) OSAtomicAdd(pCounter, -(incr))
+#define OSAtomicSubtractUnless(pCounter, incr, test) OSAtomicAddUnless(pCounter, -(incr), test)
+
+#else
+
+/*************************************************************************/ /*!
+@Function       OSAtomicRead
+@Description    Read the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to read
+@Return         The value of the atomic variable
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicRead(const ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicWrite
+@Description    Write the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be written to
+@Input          v               The value to write
+@Return         None
+*/ /**************************************************************************/
+IMG_INTERNAL
+void OSAtomicWrite(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/* For the following atomic operations, in addition to being SMP-safe,
+   should also  have a memory barrier around each operation */
+/*************************************************************************/ /*!
+@Function       OSAtomicIncrement
+@Description    Increment the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be incremented
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicIncrement(ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicDecrement
+@Description    Decrement the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be decremented
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicDecrement(ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicAdd
+@Description    Add a specified value to a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to add the value to
+@Input          v               The value to be added
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicAdd(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicAddUnless
+@Description    Add a specified value to a variable atomically unless it
+                already equals a particular value.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to add the value to
+@Input          v               The value to be added to 'pCounter'
+@Input          t               The test value. If 'pCounter' equals this,
+                                its value will not be adjusted
+@Return         The old value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicAddUnless(ATOMIC_T *pCounter, IMG_INT32 v, IMG_INT32 t);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicSubtract
+@Description    Subtract a specified value to a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to subtract the value from
+@Input          v               The value to be subtracted
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicSubtract(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicSubtractUnless
+@Description    Subtract a specified value from a variable atomically unless
+                it already equals a particular value.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to subtract the value from
+@Input          v               The value to be subtracted from 'pCounter'
+@Input          t               The test value. If 'pCounter' equals this,
+                                its value will not be adjusted
+@Return         The old value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicSubtractUnless(ATOMIC_T *pCounter, IMG_INT32 v, IMG_INT32 t);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicCompareExchange
+@Description    Set a variable to a given value only if it is currently
+                equal to a specified value. The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be checked and
+                                possibly updated
+@Input          oldv            The value the atomic variable must have in
+                                order to be modified
+@Input          newv            The value to write to the atomic variable if
+                                it equals 'oldv'
+@Return         The old value of *pCounter
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicCompareExchange(ATOMIC_T *pCounter, IMG_INT32 oldv, IMG_INT32 newv);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicExchange
+@Description    Set a variable to a given value and retrieve previous value.
+                The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be updated
+@Input          iNewVal         The value to write to the atomic variable
+@Return         The previous value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicExchange(ATOMIC_T *pCounter, IMG_INT32 iNewVal);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicOr
+@Description    Set a variable to the bitwise or of its current value and the
+                specified value. Equivalent to *pCounter |= iVal.
+                The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be updated
+@Input          iVal            The value to bitwise or against
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicOr(ATOMIC_T *pCounter, IMG_INT32 iVal);
+
+/* For now, spin-locks are required on Linux only, so other platforms fake
+ * spinlocks with normal mutex locks */
+/*! Type definitions for OS_SPINLOCK accessor and creation / deletion */
+typedef unsigned long OS_SPINLOCK_FLAGS;
+/*! Pointer to an OS Spinlock */
+#define POS_SPINLOCK POS_LOCK
+/*! Wrapper for OSLockCreate() */
+#define OSSpinLockCreate(ppLock) OSLockCreate(ppLock)
+/*! Wrapper for OSLockDestroy() */
+#define OSSpinLockDestroy(pLock) OSLockDestroy(pLock)
+/*! Wrapper for OSLockAcquire() */
+#define OSSpinLockAcquire(pLock, flags) {flags = 0; OSLockAcquire(pLock);}
+/*! Wrapper for OSLockRelease() */
+#define OSSpinLockRelease(pLock, flags) {flags = 0; OSLockRelease(pLock);}
+
+#endif /* defined(__linux__) */
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+static inline PVRSRV_ERROR OSAtomicAddUnlessOverflow(ATOMIC_T *pCounter, IMG_INT iAdd, IMG_INT iMax)
+{
+	IMG_INT iCurrent;
+	IMG_INT iOld, iNew;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Should be using subtract function */
+	if (iAdd <= 0)
+	{
+		PVR_ASSERT(!"Attempt to add negative number, rejecting.");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	iCurrent = OSAtomicRead(pCounter);
+
+	do
+	{
+		iOld = iCurrent;
+#if defined(__GNUC__) || defined(__clang__)
+		if (iAdd > (iMax - iCurrent) || __builtin_sadd_overflow(iCurrent, iAdd, &iNew))
+		{
+			/* We would overflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+		}
+#else
+		if (iAdd > (iMax - iCurrent) || iAdd > (IMG_INT_MAX - iCurrent))
+		{
+			/* We would overflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_OVERFLOW;
+		}
+		iNew = iCurrent + iAdd;
+#endif
+		iCurrent = OSAtomicCompareExchange(pCounter, iOld, iNew);
+	}
+	while (iOld != iCurrent);
+
+	return eError;
+}
+
+static inline PVRSRV_ERROR OSAtomicSubtractUnlessUnderflow(ATOMIC_T *pCounter, IMG_INT iSub, IMG_INT iMin)
+{
+	IMG_INT iCurrent;
+	IMG_INT iOld, iNew;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Should be using add function */
+	if (iSub <= 0)
+	{
+		PVR_ASSERT(!"Attempt to subtract negative number, rejecting.");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	iCurrent = OSAtomicRead(pCounter);
+
+	do
+	{
+		iOld = iCurrent;
+#if defined(__GNUC__) || defined(__clang__)
+		if (iSub < (iMin + iCurrent) || __builtin_ssub_overflow(iCurrent, iSub, &iNew))
+		{
+			/* We would underflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_UNDERFLOW;
+		}
+#else
+		if (iCurrent < (iMin + iSub) || iCurrent < (IMG_INT_MIN + iSub))
+		{
+			/* We would underflow so return error and do nothing */
+			return PVRSRV_ERROR_ATOMIC_UNDERFLOW;
+		}
+		iNew = iCurrent - iSub;
+#endif
+		iCurrent = OSAtomicCompareExchange(pCounter, iOld, iNew);
+	}
+	while (iOld != iCurrent);
+
+	return eError;
+}
+
+#endif /* LOCK_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/osmmap.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/osmmap.h
new file mode 100644
index 000000000000..40a509d19440
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/osmmap.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS Interface for mapping PMRs into CPU space.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS abstraction for the mmap2 interface for mapping PMRs into
+                User Mode memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSMMAP_H
+#define OSMMAP_H
+
+#include <powervr/mem_types.h>
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+/*************************************************************************/ /*!
+@Function       OSMMapPMR
+@Description    Maps the specified PMR into CPU memory so that it may be
+                accessed by the user process.
+                Whether the memory is mapped read only, read/write, or not at
+                all, is dependent on the PMR itself.
+                The PMR handle is opaque to the user, and lower levels of this
+                stack ensure that the handle is private to this process, such
+                that this API cannot be abused to gain access to other people's
+                PMRs. The OS implementation of this function should return the
+                virtual address and length for the User to use. The "PrivData"
+                is to be stored opaquely by the caller (N.B. he should make no
+                assumptions, in particular, NULL is a valid handle) and given
+                back to the call to OSMUnmapPMR.
+                The OS implementation is free to use the PrivData handle for
+                any purpose it sees fit.
+@Input          hBridge              The bridge handle.
+@Input          hPMR                 The handle of the PMR to be mapped.
+@Input          uiPMRLength          The size of the PMR.
+@Input          uiFlags              Flags indicating how the mapping should
+                                     be done (read-only, etc). These may not
+                                     be honoured if the PMR does not permit
+                                     them.
+@Output         phOSMMapPrivDataOut  Returned private data.
+@Output         ppvMappingAddressOut The returned mapping.
+@Output         puiMappingLengthOut  The size of the returned mapping.
+@Return         PVRSRV_OK on success, failure code otherwise.
+ */ /*************************************************************************/
+PVRSRV_ERROR
+OSMMapPMR(IMG_HANDLE hBridge,
+          IMG_HANDLE hPMR,
+          IMG_DEVMEM_SIZE_T uiPMRLength,
+          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+          IMG_HANDLE *phOSMMapPrivDataOut,
+          void **ppvMappingAddressOut,
+          size_t *puiMappingLengthOut);
+
+/*************************************************************************/ /*!
+@Function       OSMUnmapPMR
+@Description    Unmaps the specified PMR from CPU memory.
+                This function is the counterpart to OSMMapPMR.
+                The caller is required to pass the PMR handle back in along
+                with the same 3-tuple of information that was returned by the
+                call to OSMMapPMR in phOSMMapPrivDataOut.
+                It is possible to unmap only part of the original mapping
+                with this call, by specifying only the address range to be
+                unmapped in pvMappingAddress and uiMappingLength.
+@Input          hBridge              The bridge handle.
+@Input          hPMR                 The handle of the PMR to be unmapped.
+@Input          hOSMMapPrivData      The OS private data of the mapping.
+@Input          pvMappingAddress     The address to be unmapped.
+@Input          uiMappingLength      The size to be unmapped.
+@Return         PVRSRV_OK on success, failure code otherwise.
+ */ /*************************************************************************/
+void
+OSMUnmapPMR(IMG_HANDLE hBridge,
+            IMG_HANDLE hPMR,
+            IMG_HANDLE hOSMMapPrivData,
+            void *pvMappingAddress,
+            size_t uiMappingLength);
+
+#endif /* OSMMAP_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/proc_stats.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/proc_stats.h
new file mode 100644
index 000000000000..72c14c5f8e4d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/proc_stats.h
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File
+@Title          Process and driver statistic definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PROC_STATS_H
+#define PROC_STATS_H
+
+#define DKP_HIDDEN "hidden"
+
+#define PROCESS_STAT_KMALLOC                   X(PVRSRV_PROCESS_STAT_TYPE_KMALLOC,                "MemoryUsageKMalloc",                "host_kmalloc")
+#define PROCESS_STAT_KMALLOC_MAX               X(PVRSRV_PROCESS_STAT_TYPE_KMALLOC_MAX,            "MemoryUsageKMallocMax",             DKP_HIDDEN)
+
+#define PROCESS_STAT_VMALLOC                   X(PVRSRV_PROCESS_STAT_TYPE_VMALLOC,                "MemoryUsageVMalloc",                "host_vmalloc")
+#define PROCESS_STAT_VMALLOC_MAX               X(PVRSRV_PROCESS_STAT_TYPE_VMALLOC_MAX,            "MemoryUsageVMallocMax",             DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_PAGES_PT_UMA        X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA,     "MemoryUsageAllocPTMemoryUMA",       "host_mem_dev_pt")
+#define PROCESS_STAT_ALLOC_PAGES_PT_UMA_MAX    X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA_MAX, "MemoryUsageAllocPTMemoryUMAMax",    DKP_HIDDEN)
+
+#define PROCESS_STAT_VMAP_PT_UMA               X(PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA,            "MemoryUsageVMapPTUMA",              DKP_HIDDEN)
+#define PROCESS_STAT_VMAP_PT_UMA_MAX           X(PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA_MAX,        "MemoryUsageVMapPTUMAMax",           DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_PAGES_PT_LMA        X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA,     "MemoryUsageAllocPTMemoryLMA",       "local_mem_dev_pt")
+#define PROCESS_STAT_ALLOC_PAGES_PT_LMA_MAX    X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA_MAX, "MemoryUsageAllocPTMemoryLMAMax",    DKP_HIDDEN)
+
+#define PROCESS_STAT_IOREMAP_PT_LMA            X(PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA,         "MemoryUsageIORemapPTLMA",           DKP_HIDDEN)
+#define PROCESS_STAT_IOREMAP_PT_LMA_MAX        X(PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA_MAX,     "MemoryUsageIORemapPTLMAMax",        DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_LMA_PAGES           X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES,        "MemoryUsageAllocGPUMemLMA",         "local_mem_dev_buf")
+#define PROCESS_STAT_ALLOC_LMA_PAGES_MAX       X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES_MAX,    "MemoryUsageAllocGPUMemLMAMax",      DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES          X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES,       "MemoryUsageZombieGPUMemLMA",        "local_mem_dev_buf_purgeable")
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX      X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES_MAX,   "MemoryUsageZombieGPUMemLMAMax",     DKP_HIDDEN)
+#else
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX
+#endif
+
+#define PROCESS_STAT_ALLOC_UMA_PAGES           X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES,        "MemoryUsageAllocGPUMemUMA",         "host_mem_dev_buf")
+#define PROCESS_STAT_ALLOC_UMA_PAGES_MAX       X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES_MAX,    "MemoryUsageAllocGPUMemUMAMax",      DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES          X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES,       "MemoryUsageZombieGPUMemUMA",        "host_mem_dev_buf_purgeable")
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX      X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES_MAX,   "MemoryUsageZombieGPUMemUMAMax",     DKP_HIDDEN)
+#else
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX
+#endif
+
+#define PROCESS_STAT_MAP_UMA_LMA_PAGES         X(PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES,      "MemoryUsageMappedGPUMemUMA/LMA",    DKP_HIDDEN)
+#define PROCESS_STAT_MAP_UMA_LMA_PAGES_MAX     X(PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES_MAX,  "MemoryUsageMappedGPUMemUMA/LMAMax", DKP_HIDDEN)
+
+#define PROCESS_STAT_DMA_BUF_IMPORT            X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT,         "MemoryUsageDmaBufImport",           "dma_buf_import")
+#define PROCESS_STAT_DMA_BUF_IMPORT_MAX        X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT_MAX,     "MemoryUsageDmaBufImportMax",        DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_DMA_BUF_ZOMBIE            X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE,         "MemoryUsageDmaBufZombie",           "dma_buf_purgeable")
+#define PROCESS_STAT_DMA_BUF_ZOMBIE_MAX        X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE_MAX,     "MemoryUsageDmaBufZombieMax",        DKP_HIDDEN)
+#else
+#define PROCESS_STAT_DMA_BUF_ZOMBIE
+#define PROCESS_STAT_DMA_BUF_ZOMBIE_MAX
+#endif
+
+#define PROCESS_STAT_TOTAL                     X(PVRSRV_PROCESS_STAT_TYPE_TOTAL,                  "MemoryUsageTotal",                  DKP_HIDDEN)
+#define PROCESS_STAT_TOTAL_MAX                 X(PVRSRV_PROCESS_STAT_TYPE_TOTAL_MAX,              "MemoryUsageTotalMax",               DKP_HIDDEN)
+
+/* Process stat keys */
+#define PVRSRV_PROCESS_STAT_KEY \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_KMALLOC_MAX \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_VMALLOC_MAX \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA_MAX \
+	PROCESS_STAT_VMAP_PT_UMA \
+	PROCESS_STAT_VMAP_PT_UMA_MAX \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA_MAX \
+	PROCESS_STAT_IOREMAP_PT_LMA \
+	PROCESS_STAT_IOREMAP_PT_LMA_MAX \
+	PROCESS_STAT_ALLOC_LMA_PAGES \
+	PROCESS_STAT_ALLOC_LMA_PAGES_MAX \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ALLOC_UMA_PAGES_MAX \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX \
+	PROCESS_STAT_MAP_UMA_LMA_PAGES \
+	PROCESS_STAT_MAP_UMA_LMA_PAGES_MAX \
+	PROCESS_STAT_DMA_BUF_IMPORT \
+	PROCESS_STAT_DMA_BUF_IMPORT_MAX \
+	PROCESS_STAT_DMA_BUF_ZOMBIE \
+	PROCESS_STAT_DMA_BUF_ZOMBIE_MAX \
+	PROCESS_STAT_TOTAL \
+	PROCESS_STAT_TOTAL_MAX
+
+#if defined(SUPPORT_LINUX_FDINFO)
+/* DKP process stats within the drm-memory-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_MEMORY \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES
+
+/* DKP process stats within the drm-shared-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_SHARED \
+	PROCESS_STAT_DMA_BUF_IMPORT \
+	PROCESS_STAT_DMA_BUF_ZOMBIE
+
+/* DKP process stats within the drm-total-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_TOTAL \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_LMA_PAGES
+
+/* DKP process stats within the drm-resident-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_RESIDENT \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+
+/* DKP process stats within the drm-purgeable-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_PURGEABLE \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_DMA_BUF_ZOMBIE
+
+/* DKP process stats within the drm-active-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_ACTIVE
+
+#endif
+
+/* X-Macro for Device stat keys */
+#define PVRSRV_DEVICE_STAT_KEY \
+	X(PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS, "Connections") \
+	X(PVRSRV_DEVICE_STAT_TYPE_MAX_CONNECTIONS, "ConnectionsMax") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_OOMS, "RenderContextOutOfMemoryEvents") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_PRS, "RenderContextPartialRenders") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_GROWS, "RenderContextGrows") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_PUSH_GROWS, "RenderContextPushGrows") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_TA_STORES, "RenderContextTAStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_3D_STORES, "RenderContext3DStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_CDM_STORES, "RenderContextCDMStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_TDM_STORES, "RenderContextTDMStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_RAY_STORES, "RenderContextRayStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_APP, "ZSBufferRequestsByApp") \
+	X(PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_FW, "ZSBufferRequestsByFirmware") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_APP, "FreeListGrowRequestsByApp") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_FW, "FreeListGrowRequestsByFirmware") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_PAGES_INIT, "FreeListInitialPages") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_MAX_PAGES, "FreeListMaxPages") \
+	X(PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT, "MemoryOOMCountDeviceVirtual") \
+	X(PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT, "MemoryOOMCountPhysicalHeap") \
+	X(PVRSRV_DEVICE_STAT_TYPE_INVALID_VIRTMEM, "MemoryOOMCountDeviceVirtualAtAddr")
+
+/* X-Macro for Driver stat keys */
+#define PVRSRV_DRIVER_STAT_KEY \
+	X(PVRSRV_DRIVER_STAT_TYPE_KMALLOC, "MemoryUsageKMalloc") \
+	X(PVRSRV_DRIVER_STAT_TYPE_KMALLOC_MAX, "MemoryUsageKMallocMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMALLOC, "MemoryUsageVMalloc") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMALLOC_MAX, "MemoryUsageVMallocMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, "MemoryUsageAllocPTMemoryUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA_MAX, "MemoryUsageAllocPTMemoryUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, "MemoryUsageVMapPTUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA_MAX, "MemoryUsageVMapPTUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, "MemoryUsageAllocPTMemoryLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA_MAX, "MemoryUsageAllocPTMemoryLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, "MemoryUsageIORemapPTLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA_MAX, "MemoryUsageIORemapPTLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, "MemoryUsageAllocGPUMemLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA_MAX, "MemoryUsageAllocGPUMemLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, "MemoryUsageZombieGPUMemLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA_MAX, "MemoryUsageZombieGPUMemLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, "MemoryUsageAllocGPUMemUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_MAX, "MemoryUsageAllocGPUMemUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, "MemoryUsageZombieGPUMemUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA_MAX, "MemoryUsageZombieGPUMemUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, "MemoryUsageAllocGPUMemUMAPool") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL_MAX, "MemoryUsageAllocGPUMemUMAPoolMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, "MemoryUsageMappedGPUMemUMA/LMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA_MAX, "MemoryUsageMappedGPUMemUMA/LMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, "MemoryUsageDmaBufImport") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT_MAX, "MemoryUsageDmaBufImportMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, "MemoryUsageDmaBufZombie") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE_MAX, "MemoryUsageDmaBufZombieMax")
+
+
+typedef enum {
+#define X(stat_type, stat_str, drm_str) stat_type,
+	PVRSRV_PROCESS_STAT_KEY
+#undef X
+	PVRSRV_PROCESS_STAT_TYPE_COUNT
+}PVRSRV_PROCESS_STAT_TYPE;
+
+typedef enum {
+#define X(stat_type, stat_str) stat_type,
+	PVRSRV_DEVICE_STAT_KEY
+#undef X
+	PVRSRV_DEVICE_STAT_TYPE_COUNT
+}PVRSRV_DEVICE_STAT_TYPE;
+
+typedef enum {
+#define X(stat_type, stat_str) stat_type,
+	PVRSRV_DRIVER_STAT_KEY
+#undef X
+	PVRSRV_DRIVER_STAT_TYPE_COUNT
+}PVRSRV_DRIVER_STAT_TYPE;
+
+#endif // PROC_STATS_H
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/ra.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/ra.h
new file mode 100644
index 000000000000..ee4492afa1ed
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/ra.h
@@ -0,0 +1,763 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Allocator API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RA_H
+#define RA_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define RA_MAX_NAME_LENGTH 20
+
+/** Resource arena.
+ *  struct _RA_ARENA_ deliberately opaque
+ */
+typedef struct _RA_ARENA_ RA_ARENA;			//PRQA S 3313
+
+/** Resource arena's iterator.
+ *  struct _RA_ARENA_ITERATOR_ deliberately opaque
+ */
+typedef struct _RA_ARENA_ITERATOR_ RA_ARENA_ITERATOR;
+
+typedef struct _RA_ITERATOR_DATA_ {
+	IMG_UINT64 uiAddr;
+	IMG_UINT64 uiSize;
+	/* Standard Iteration: is the segment free
+	 * Span Iteration: is the span a free import
+	 */
+	IMG_BOOL bFree;
+} RA_ITERATOR_DATA;
+
+/** Resource arena usage statistics.
+ *  struct _RA_USAGE_STATS
+ */
+typedef struct _RA_USAGE_STATS {
+	IMG_UINT64	ui64TotalArenaSize;
+	IMG_UINT64	ui64FreeArenaSize;
+} RA_USAGE_STATS, *PRA_USAGE_STATS;
+
+/*
+ * Per-Arena handle - this is private data for the caller of the RA.
+ * The RA knows nothing about this data. It is given it in RA_Create, and
+ * promises to pass it to calls to the ImportAlloc and ImportFree callbacks
+ */
+typedef IMG_HANDLE RA_PERARENA_HANDLE;
+/*
+ * Per-Import handle - this is private data for the caller of the RA.
+ * The RA knows nothing about this data. It is given it on a per-import basis,
+ * basis, either the "initial" import at RA_Create time, or further imports
+ * via the ImportAlloc callback. It sends it back via the ImportFree callback,
+ * and also provides it in answer to any RA_Alloc request to signify from
+ * which "import" the allocation came.
+ */
+typedef IMG_HANDLE RA_PERISPAN_HANDLE;
+
+typedef IMG_UINT64 RA_BASE_T;
+typedef IMG_UINT32 RA_LOG2QUANTUM_T;
+typedef IMG_UINT64 RA_LENGTH_T;
+typedef IMG_UINT32 RA_POLICY_T;
+
+typedef struct _RA_BASE_MULTI_ RA_BASE_MULTI_T;
+
+typedef IMG_UINT32 RA_BASE_ARRAY_SIZE_T;
+
+
+/*
+ * RA_BASE_ARRAY can represent a number of bases of which are packed,
+ * that is, they can be one of two types, a Real Base or a Ghost base.
+ * A Real Base is a base that has been created by the RA and is used to
+ * represent an allocated region, it has an entry in the RA Hash table and
+ * as such has a BT associated with it.
+ * A Ghost base is a fabricated base address generated at chunk boundaries
+ * given by the caller. These are used to divide a RealBase into
+ * arbitrary regions that the caller requires e.g. 4k pages. Ghost bases don't
+ * exist from the RA memory tracking point of view but they do exist and are treated
+ * as base addresses from the PoV of the caller. This allows the RA to allocate in
+ * largest possible lengths meaning fewer alloc calls whilst allowing the chunk
+ * flexibility for callers. Ghost refers to the concept that they
+ * don't exist in this RA internals context but do in the callers (LMA) context i.e.
+ * they appear Real from another perspective but we the RA know they are a ghost of the
+ * Real Base.
+ * */
+typedef RA_BASE_T RA_BASE_ARRAY_T[IMG_FLEX_ARRAY_MEMBER];
+
+/* Since 0x0 is a valid BaseAddr, we rely on max 64-bit value to be an invalid
+ * page address.
+ */
+#define INVALID_BASE_ADDR (IMG_UINT64_MAX)
+/* Used to check for duplicated alloc indices in sparse alloc path
+ * prior to attempting allocations */
+#define RA_BASE_SPARSE_PREP_ALLOC_ADDR (IMG_UINT64_MAX - 1)
+#define RA_BASE_FLAGS_MASK 0xFFF /* 12 Bits 4k alignment. */
+#define RA_BASE_FLAGS_LOG2 12
+#define RA_BASE_CHUNK_LOG2_MAX 64
+#define RA_BASE_GHOST_BIT (1ULL << 0)
+#define RA_BASE_STRIP_GHOST_BIT(uiBase) ((uiBase) & ~(RA_BASE_GHOST_BIT))
+#define RA_BASE_SET_GHOST_BIT(uiBase)   ((uiBase) |= RA_BASE_GHOST_BIT)
+#define RA_BASE_IS_GHOST(uiBase) (BITMASK_HAS((uiBase), RA_BASE_GHOST_BIT) && (uiBase) != INVALID_BASE_ADDR)
+#define RA_BASE_IS_REAL(uiBase) (!BITMASK_HAS((uiBase), RA_BASE_GHOST_BIT))
+#define RA_BASE_IS_SPARSE_PREP(uiBase) ((uiBase) == RA_BASE_SPARSE_PREP_ALLOC_ADDR)
+#define RA_BASE_IS_INVALID(uiBase) ((uiBase) == INVALID_BASE_ADDR)
+
+typedef struct _RA_MULTIBASE_ITERATOR_ RA_MULTIBASE_ITERATOR;
+
+/* Lock classes: describes the level of nesting between different arenas.
+ * A RA with lockclass x is permitted to obtain locks of a lockclass less than x.
+ * A RA with lockclass 0 cannot obtain the lock of any other RA. */
+#define RA_LOCKCLASS_0 0
+#define RA_LOCKCLASS_1 1
+#define RA_LOCKCLASS_2 2
+#define RA_LOCKCLASS_3 3
+
+#define RA_NO_IMPORT_MULTIPLIER 1
+
+/*
+ * Allocation Policies that govern the resource areas.
+ * */
+
+/* --- Resource allocation policy definitions ---
+* | 31.........5|.......4....|......3....|........2.............|1...................0|
+* | Reserved    | Non-Contig | No split  | Area bucket selection| Alloc node selection|
+*/
+
+/*
+ * Fast allocation policy allows to pick the first node
+ * that satisfies the request.
+ * It is the default policy for all arenas.
+ *  */
+#define RA_POLICY_ALLOC_FAST			(0U)
+/*
+ * Optimal allocation policy allows to pick the lowest size node
+ * that satisfies the request. This picking policy helps in reducing the fragmentation.
+ * This minimises the necessity to split the nodes more often as the optimal
+ * ones are picked.
+ * As a result any future higher size allocation requests are likely to succeed
+ */
+#define RA_POLICY_ALLOC_OPTIMAL		(1U)
+#define RA_POLICY_ALLOC_NODE_SELECT_MASK			(3U)
+
+/*
+ * Bucket selection policies
+ * */
+/* Assured bucket policy makes sure the selected bucket is guaranteed
+ * to satisfy the given request. Generally Nodes picked up from such a
+ * bucket need to be further split. However picking node that belongs to this
+ * bucket is likely to succeed and thus promises better response times */
+#define RA_POLICY_BUCKET_ASSURED_FIT		(0U)
+/*
+ * Best fit bucket policy selects a bucket with free nodes that are likely
+ * to satisfy the request and nodes that are close to the requested size.
+ * Nodes picked up from this bucket may likely to satisfy the request but not
+ * guaranteed. Failing to satisfy the request from this bucket mean further
+ * higher size buckets are selected in the later iterations till the request
+ * is satisfied.
+ *
+ * Hence response times may vary depending on availability of free nodes
+ * that satisfy the request.
+ * */
+#define RA_POLICY_BUCKET_BEST_FIT		(4U)
+#define RA_POLICY_BUCKET_MASK			(4U)
+
+/* This flag ensures the imports will not be split up and Allocations will always get
+ * their own import
+ */
+#define RA_POLICY_NO_SPLIT			(8U)
+#define RA_POLICY_NO_SPLIT_MASK		(8U)
+
+/* This flag is used in physmem_lma only. it is used to decide if we should
+ * activate the non-contiguous allocation feature of RA MultiAlloc.
+ * Requirements for activation are that the OS implements the
+ * OSMapPageArrayToKernelVA function in osfunc which allows for mapping
+ * physically sparse pages as a virtually contiguous range.
+ * */
+#define RA_POLICY_ALLOC_ALLOW_NONCONTIG      (16U)
+#define RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK (16U)
+
+/*
+ * Default Arena Policy
+ * */
+#define RA_POLICY_DEFAULT			(RA_POLICY_ALLOC_FAST | RA_POLICY_BUCKET_ASSURED_FIT)
+
+/*
+ * Flags in an "import" must match the flags for an allocation
+ */
+typedef IMG_UINT64 RA_FLAGS_T;
+
+typedef struct _RA_IMPORT_ {
+	RA_BASE_T base;           /* Allocation base */
+	RA_LENGTH_T uSize;        /* Allocation size */
+	RA_PERISPAN_HANDLE hPriv; /* Per import private data */
+} RA_IMPORT;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_ALLOC_SINGLE
+@Description    RA import allocate function create a single span when
+                requesting extra resource for the arena.
+@Input          RA_PERARENA_HANDLE RA handle
+@Input          RA_LENGTH_T        Request size
+@Input          RA_FLAGS_T         RA flags
+@Input          RA_LENGTH_T        Base Alignment
+@Input          IMG_CHAR           Annotation
+@Output         RA_IMPORT          Attributes of an import.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_RA_IMPORT_ALLOC_SINGLE)(RA_PERARENA_HANDLE,
+                                                   RA_LENGTH_T,
+                                                   RA_FLAGS_T,
+                                                   RA_LENGTH_T,
+                                                   const IMG_CHAR*,
+                                                   RA_IMPORT*);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_ALLOC_MULTI
+@Description    RA import allocate function is able to create multiple spans
+                for a single import request.
+                The pointer to the array of inputs will be provided but
+                can be set to a larger array via OSAllocMem
+                if the count is not large enough.
+                The pointer will be freed by OSFreeMem after the callback.
+                It is permissible for the callback to set:
+                count == 0 or array pointer == NULL
+                but the callback MUST return an error in these cases.
+@Input          RA_PERARENA_HANDLE RA handle
+@Input          RA_LENGTH_T        Request size
+@Input          RA_FLAGS_T         RA flags
+@Input          RA_LENGTH_T        Base Alignment
+@Input          IMG_CHAR           Annotation
+@Inout          IMG_UINT32         Num of imports = Num of elements in array below.
+                                   Will always be >= 1. Must be set to the number of
+                                   imports returned.
+@Inout          RA_IMPORT          Pointer to array of imports. Will always point
+                                   to an array of equal length as Num of Imports.
+                                   Will be freed by OSFreeMem.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_RA_IMPORT_ALLOC_MULTI)(RA_PERARENA_HANDLE,
+                                                  RA_LENGTH_T,
+                                                  RA_FLAGS_T,
+                                                  RA_LENGTH_T,
+                                                  const IMG_CHAR*,
+                                                  IMG_UINT32*,
+                                                  RA_IMPORT**);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_FREE
+@Description    RA free imported allocation
+@Input          RA_PERARENA_HANDLE   RA handle
+@Input          RA_BASE_T            Allocation base
+@Input          RA_PERISPAN_HANDLE   Per import private data
+*/ /**************************************************************************/
+typedef void (*PFN_RA_IMPORT_FREE)(RA_PERARENA_HANDLE,
+                                   RA_BASE_T,
+                                   RA_PERISPAN_HANDLE);
+
+/**
+ *  @Function   RA_Create
+ *
+ *  @Description    To create a resource arena.
+ *
+ *  @Input name - the name of the arena for diagnostic purposes.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui32LockClass - the lock class level this arena uses.
+ *  @Input imp_alloc - a resource allocation callback or 0.
+ *  @Input imp_free - a resource de-allocation callback or 0.
+ *  @Input per_arena_handle - private handle passed to alloc and free or 0.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+ */
+RA_ARENA *
+RA_Create(IMG_CHAR *name,
+          /* subsequent imports: */
+          RA_LOG2QUANTUM_T uLog2Quantum,
+          IMG_UINT32 ui32LockClass,
+          PFN_RA_IMPORT_ALLOC_SINGLE imp_alloc,
+          PFN_RA_IMPORT_FREE imp_free,
+          RA_PERARENA_HANDLE per_arena_handle,
+          RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_CreateMulti
+ *
+ *  @Description    To create a resource arena using the
+ *                  ALLOC_MULTI callback.
+ *
+ *  @Input name - the name of the arena for diagnostic purposes.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui32LockClass - the lock class level this arena uses.
+ *  @Input imp_alloc - a resource allocation callback or 0.
+ *  @Input imp_free - a resource de-allocation callback or 0.
+ *  @Input arena_handle - private handle passed to alloc and free or 0.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+ */
+RA_ARENA *
+RA_CreateMulti(IMG_CHAR *name,
+               RA_LOG2QUANTUM_T uLog2Quantum,
+               IMG_UINT32 ui32LockClass,
+               PFN_RA_IMPORT_ALLOC_MULTI imp_alloc,
+               PFN_RA_IMPORT_FREE imp_free,
+               RA_PERARENA_HANDLE arena_handle,
+               RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_Create_With_Span
+ *
+ *  @Description
+ *
+ *  Create a resource arena and initialises it, with a given resource span.
+ *
+ *  @Input name - String briefly describing the RA's purpose.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui64CpuBase - CPU Physical Base Address of the RA.
+ *  @Input ui64SpanDevBase - Device Physical Base Address of the RA.
+ *  @Input ui64SpanSize - Size of the span to add to the created RA.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+*/
+RA_ARENA *
+RA_Create_With_Span(IMG_CHAR *name,
+                    RA_LOG2QUANTUM_T uLog2Quantum,
+                    IMG_UINT64 ui64CpuBase,
+                    IMG_UINT64 ui64SpanDevBase,
+                    IMG_UINT64 ui64SpanSize,
+                    RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_Delete
+ *
+ *  @Description
+ *
+ *  To delete a resource arena. All resources allocated from the arena
+ *  must be freed before deleting the arena.
+ *
+ *  @Input  pArena - the arena to delete.
+ *  @Return None
+ */
+void
+RA_Delete(RA_ARENA *pArena);
+
+/**
+ *  @Function   RA_Add
+ *
+ *  @Description
+ *
+ *  To add a resource span to an arena. The span must not overlap with
+ *  any span previously added to the arena.
+ *
+ *  @Input pArena - the arena to add a span into.
+ *  @Input base - the base of the span.
+ *  @Input uSize - the extent of the span.
+ *  @Input hPriv - handle associated to the span (reserved for user uses)
+ *  @Return IMG_TRUE - success, IMG_FALSE - failure
+ */
+IMG_BOOL
+RA_Add(RA_ARENA *pArena,
+       RA_BASE_T base,
+       RA_LENGTH_T uSize,
+       RA_FLAGS_T uFlags,
+       RA_PERISPAN_HANDLE hPriv);
+
+/**
+ *  @Function   RA_Alloc
+ *
+ *  @Description    To allocate resource from an arena.
+ *
+ *  @Input  pArena - the arena
+ *  @Input  uRequestSize - the size of resource segment requested.
+ *  @Input  uImportMultiplier - Import x-times of the uRequestSize
+ *          for future RA_Alloc calls.
+ *          Use RA_NO_IMPORT_MULTIPLIER to import the exact size.
+ *  @Input  uImportFlags - flags influencing allocation policy.
+ *  @Input  uAlignment - the alignment constraint required for the
+ *          allocated segment, use 0 if alignment not required.
+ *  @Input  pszAnnotation - a string to describe the allocation
+ *  @Output base - allocated base resource
+ *  @Output pActualSize - the actual_size of resource segment allocated,
+ *          typically rounded up by quantum.
+ *  @Output phPriv - the user reference associated with allocated
+ *          resource span.
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_Alloc(RA_ARENA *pArena,
+         RA_LENGTH_T uRequestSize,
+         IMG_UINT8 uImportMultiplier,
+         RA_FLAGS_T uImportFlags,
+         RA_LENGTH_T uAlignment,
+         const IMG_CHAR *pszAnnotation,
+         RA_BASE_T *base,
+         RA_LENGTH_T *pActualSize,
+         RA_PERISPAN_HANDLE *phPriv);
+
+/*************************************************************************/ /*!
+@Function       RA_AllocMulti
+@Description    To allocate resource from an arena.
+                This method of allocation can be used to guarantee that if there
+                is enough space in the RA and the contiguity given is the
+                greatest common divisor of the contiguities used on this RA
+                the allocation can be made.
+                Allocations with contiguity less than the current GCD
+                (Greatest Common Divisor) abiding to pow2 are also guaranteed to
+                succeed. See scenario 4.
+                Allocations are not guaranteed but still reduce fragmentation
+                using this method when multiple contiguities are used e.g.
+                4k & 16k and the current allocation has a contiguity higher than
+                the greatest common divisor used.
+                Scenarios with Log 2 contiguity examples:
+                1. All allocations have contiguity of 4k. Allocations can be
+                guaranteed given enough RA space since the GCD is always used.
+                2. Allocations of 4k and 16k contiguity have been previously
+                made on this RA. A new allocation of 4k contiguity is guaranteed
+                to succeed given enough RA space since the contiguity is the GCD.
+                3. Allocations of 4k and 16k contiguity have been previously made
+                on this RA. A new allocation of 16k contiguity is not guaranteed
+                to succeed since it is not the GCD of all contiguities used.
+                4. Contiguity 16k and 64k already exist, a 4k contiguity
+                allocation would be guaranteed to succeed but would now be the
+                new GCD. So further allocations would be required to match this
+                GCD to guarantee success.
+                This method does not suffer the same fragmentation pitfalls
+                as RA_Alloc as it constructs the allocation size from many
+                smaller constituent allocations, these are represented and returned
+                in the given array. In addition, Ghost bases are generated in
+                array entries conforming to the chunk size, this allows for
+                representing chunks of any size that work as page addrs
+                in upper levels.
+                The aforementioned array must be at least of size
+                uRequestsize / uiChunkSize, this ensures there is at least one
+                array entry per chunk required.
+                This function must have a uiChunkSize value of
+                at least 4096, this is to ensure space for the base type encoding.
+@Input          pArena            The arena
+@Input          uRequestSize      The size of resource requested.
+@Input          uiLog2ChunkSize   The log2 contiguity multiple of the bases i.e all
+                                  Real bases must be a multiple in size of this
+                                  size, also used to generate Ghost bases.
+                                  Allocations will also be aligned to this value.
+@Input          uImportMultiplier Import x-times more for future requests if
+                                  we have to import new resource.
+@Input          uImportFlags      Flags influencing allocation policy.
+                                  required, otherwise must be a power of 2.
+@Input          pszAnnotation     String to describe the allocation
+@InOut          aBaseArray        Array of bases to populate.
+@Input          uiBaseArraySize   Size of the array to populate.
+@Output         bPhysContig       Are the allocations made in the RA physically
+                                  contiguous.
+@Return         PVRSRV_OK - success
+*/ /**************************************************************************/
+PVRSRV_ERROR
+RA_AllocMulti(RA_ARENA *pArena,
+               RA_LENGTH_T uRequestSize,
+               IMG_UINT32 uiLog2ChunkSize,
+               IMG_UINT8 uImportMultiplier,
+               RA_FLAGS_T uImportFlags,
+               const IMG_CHAR *pszAnnotation,
+               RA_BASE_ARRAY_T aBaseArray,
+               RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+               IMG_BOOL *bPhysContig);
+
+/**
+ * @Function   RA_AllocMultiSparse
+ *
+ * @Description    To Alloc resource from an RA arena at the specified indices.
+ *                 This function follows the same conditions and functionality as
+ *                 RA_AllocMulti although with the added aspect of specifying the
+ *                 indices to allocate in the Base Array. This means we can still
+ *                 attempt to maintain contiguity where possible with the aim of
+ *                 reducing fragmentation and increasing occurrence of optimal free
+ *                 scenarios.
+ * @Input          pArena            The Arena
+ * @Input          uiLog2ChunkSize   The log2 contiguity multiple of the bases i.e all
+ *                                   Real bases must be a multiple in size of this
+ *                                   size, also used to generate Ghost bases.
+ *                                   Allocations will also be aligned to this value.
+ * @Input          uImportMultiplier Import x-times more for future requests if
+ *                                   we have to import new resource.
+ * @Input          uImportFlags      Flags influencing allocation policy.
+ *                                   required, otherwise must be a power of 2.
+ * @Input          pszAnnotation     String to describe the allocation
+ * @InOut          aBaseArray        Array of bases to populate.
+ * @Input          uiBaseArraySize   Size of the array to populate.
+ * @Input          puiAllocIndices   The indices into the array to alloc, if indices are NULL
+ *                                   then we will allocate uiAllocCount chunks sequentially.
+ * @InOut          uiAllocCount      The number of bases to alloc from the array.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_AllocMultiSparse(RA_ARENA *pArena,
+                     IMG_UINT32 uiLog2ChunkSize,
+                     IMG_UINT8 uImportMultiplier,
+                     RA_FLAGS_T uImportFlags,
+                     const IMG_CHAR *pszAnnotation,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                     IMG_UINT32 *puiAllocIndices,
+                     IMG_UINT32 uiAllocCount);
+/**
+ *  @Function   RA_FreeMulti
+ *
+ *  @Description    To free a multi-base resource constructed using
+ *                  a call to RA_AllocMulti.
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aBaseArray - The array to free bases from.
+ *  @Input  uiBaseArraysize - Size of the array to free bases from.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_FreeMulti(RA_ARENA *pArena,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize);
+
+/**
+ *  @Function   RA_FreeMultiSparse
+ *
+ *  @Description    To free part of a multi-base resource constructed using
+ *                  a call to RA_AllocMulti.
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aBaseArray - The array to free bases from.
+ *  @Input  uiBaseArraysize - Size of the array to free bases from.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases.
+ *  @Input  puiFreeIndices - The indices into the array to free.
+ *  @InOut  puiFreeCount - The number of bases to free from the array, becomes the number
+ *                         of bases actually free'd. The in value may differ from the out
+ *                         value in cases of error when freeing. The out value can then be
+ *                         used in upper levels to keep any mem tracking structures consistent
+ *                         with what was actually freed before the error occurred.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_FreeMultiSparse(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                    IMG_UINT32 uiLog2ChunkSize,
+                    IMG_UINT32 *puiFreeIndices,
+                    IMG_UINT32 *puiFreeCount);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/**
+ *  @Function   RA_TransferMultiSparseIndices
+ *
+ *  @Description   Transfers a set of indices specified in puiTransferIndices from
+ *                 aSrcBaseArray to aDstBaseArray.
+ *                 Called when some pages of the base array need to be
+ *                 transfered to another base array. As a result of this call,
+ *                 some ghost addresses in aBaseArray might be converted to
+ *                 real addresses before being transferred..
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aSrcBaseArray - The array to transfer bases from.
+ *  @Input  uiSrcBaseArraySize - Size of the array to transfer bases from.
+ *  @Input  aDstBaseArray - The array to transfer bases to.
+ *  @Input  uiDstBaseArraySize - Size of the array to transfer bases to.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases.
+ *  @Input  puiTransferIndices - The indices into the array to be extracted.
+ *  @InOut  puiTransferCount - The number of bases to prepare for extraction.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_TransferMultiSparseIndices(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aSrcBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiSrcBaseArraySize,
+                              RA_BASE_ARRAY_T aDstBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiDstBaseArraySize,
+                              IMG_UINT32 uiLog2ChunkSize,
+                              IMG_UINT32 *puiTransferIndices,
+                              IMG_UINT32 *puiTransferCount);
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+/**
+ *  @Function   RA_Alloc_Range
+ *
+ *  @Description
+ *
+ *  To allocate a resource at a specified base from an arena.
+ *
+ *  @Input  pArena - the arena
+ *  @Input  uRequestSize - the size of resource segment requested.
+ *  @Input  uImportFlags - flags influencing allocation policy.
+ *  @Input  uAlignment - the alignment constraint required for the
+ *          allocated segment, use 0 if alignment not required.
+ *  @Input  base - allocated base resource
+ *  @Output pActualSize - the actual_size of resource segment allocated,
+ *          typically rounded up by quantum.
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_Alloc_Range(RA_ARENA *pArena,
+		  RA_LENGTH_T uRequestSize,
+		  RA_FLAGS_T uImportFlags,
+		  RA_LENGTH_T uAlignment,
+		  RA_BASE_T base,
+		  RA_LENGTH_T *pActualSize);
+
+/**
+ *  @Function   RA_Free
+ *
+ *  @Description    To free a resource segment.
+ *
+ *  @Input  pArena - the arena the segment was originally allocated from.
+ *  @Input  base - the base of the resource span to free.
+ *
+ *  @Return None
+ */
+void
+RA_Free(RA_ARENA *pArena, RA_BASE_T base);
+
+/**
+ *  @Function   RA_SwapSparseMem
+ *
+ *  @Description    Swaps chunk sized allocations at X<->Y indices.
+ *                  The function is most optimal when Indices are provided
+ *                  in ascending order, this allows the internals to optimally
+ *                  swap based on contiguity and reduces the amount of ghost to
+ *                  real conversion performed. Note this function can also be used
+ *                  to move pages, in this case, we effectively swap real allocations
+ *                  with invalid marked bases.
+ *  @Input  pArena     - The arena.
+ *  @InOut  aBaseArray - The array to Swap bases in.
+ *  @Input  uiBaseArraysize - Size of the array to Swap bases in.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases
+ *                            and size the Real chunks.
+ *  @Input  puiXIndices - Set of X indices to swap with parallel indices in Y.
+ *  @Input  puiYIndices - Set of Y indices to swap with parallel indices in X.
+ *  @Input  uiSwapCount - Number of indices to swap.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_SwapSparseMem(RA_ARENA *pArena,
+                  RA_BASE_ARRAY_T aBaseArray,
+                  RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                  IMG_UINT32 uiLog2ChunkSize,
+                  IMG_UINT32 *puiXIndices,
+                  IMG_UINT32 *puiYIndices,
+                  IMG_UINT32 uiSwapCount);
+
+/**
+ *  @Function   RA_Get_Usage_Stats
+ *
+ *  @Description    To collect the arena usage statistics.
+ *
+ *  @Input  pArena - the arena to acquire usage statistics from.
+ *  @Input  psRAStats - the buffer to hold the usage statistics of the arena.
+ *
+ *  @Return None
+ */
+IMG_INTERNAL void
+RA_Get_Usage_Stats(RA_ARENA *pArena, PRA_USAGE_STATS psRAStats);
+
+/**
+ *  @Function   RA_GetArenaName
+ *
+ *  @Description    To obtain the arena name.
+ *
+ *  @Input  pArena - the arena to acquire the name from.
+ *
+ *  @Return IMG_CHAR* Arena name.
+ */
+IMG_INTERNAL IMG_CHAR *
+RA_GetArenaName(RA_ARENA *pArena);
+
+IMG_INTERNAL RA_ARENA_ITERATOR *
+RA_IteratorAcquire(RA_ARENA *pArena, IMG_BOOL bIncludeFreeSegments);
+
+IMG_INTERNAL void
+RA_IteratorReset(RA_ARENA_ITERATOR *pIter);
+
+IMG_INTERNAL void
+RA_IteratorRelease(RA_ARENA_ITERATOR *pIter);
+
+/*************************************************************************/ /*!
+@Function       RA_IteratorNext
+@Description    Function used to iterate over segments in the arena. Segments
+                can either be allocated or free. Iteration can be configured
+                to include free segments when creating the iterator via
+                RA_IteratorAcquire.
+
+@Input          pIter        Iteration handle used to keep state between calls
+@InOut          pData        Data about current block iteration, base and size
+                             etc...
+@Return         IMG_BOOL     Boolean value signalling if next element exists
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNext(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData);
+
+/*************************************************************************/ /*
+@Function       RA_IteratorNextSpan
+@Description    Function used to iterate over spans in the arena. Spans
+                can either be allocated or free. Iteration can be configured
+                to include free spans when creating the iterator via
+                RA_IteratorAcquire. Spans are not allocations on the arena but
+                instead the regions of resource added that allocations can
+                be made from.
+                See RA_Add().
+
+@Input          pIter        Iteration handle used to keep state between calls
+@InOut          pData        Data about current block iteration, base and size
+                             etc...
+@Return         IMG_BOOL     Boolean value signalling if next element exists
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNextSpan(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData);
+
+/*************************************************************************/ /*!
+@Function       RA_BlockDump
+@Description    Debug dump of all memory allocations within the RA and the space
+                between. A '#' represents a block of memory (the arena's quantum
+                in size) that has been allocated whereas a '.' represents a free
+                block.
+@Input          pArena        The arena to dump.
+@Input          pfnLogDump    The dumping method.
+@Input          pPrivData     Data to be passed into the pfnLogDump method.
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RA_BlockDump(RA_ARENA *pArena,
+             __printf(2, 3) void (*pfnLogDump)(void*, IMG_CHAR*, ...),
+             void *pPrivData);
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync.h
new file mode 100644
index 000000000000..f126915060f4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync.h
@@ -0,0 +1,292 @@
+/*************************************************************************/ /*!
+@File
+@Title          Synchronisation interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_H
+#define SYNC_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "sync_prim_internal.h"
+#include "pdumpdefs.h"
+#include "dllist.h"
+#include "pvr_debug.h"
+
+#include "device_connection.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncPrimContextCreate
+
+@Description    Create a new synchronisation context
+
+@Input          hBridge                 Bridge handle
+
+@Input          hDeviceNode             Device node handle
+
+@Output         hSyncPrimContext        Handle to the created synchronisation
+                                        primitive context
+
+@Return         PVRSRV_OK if the synchronisation primitive context was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimContextCreate(SHARED_DEV_CONNECTION hDevConnection,
+                      PSYNC_PRIM_CONTEXT    *hSyncPrimContext);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimContextDestroy
+
+@Description    Destroy a synchronisation context
+
+@Input          hSyncPrimContext        Handle to the synchronisation
+                                        primitive context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimContextDestroy(PSYNC_PRIM_CONTEXT hSyncPrimContext);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimAlloc
+
+@Description    Allocate a new synchronisation primitive on the specified
+                synchronisation context
+
+@Input          hSyncPrimContext        Handle to the synchronisation
+                                        primitive context
+
+@Output         ppsSync                 Created synchronisation primitive
+
+@Input          pszClassName            Sync source annotation
+
+@Return         PVRSRV_OK if the synchronisation primitive was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimAlloc(PSYNC_PRIM_CONTEXT      hSyncPrimContext,
+              PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+              const IMG_CHAR          *pszClassName);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimFree
+
+@Description    Free a synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to free
+
+@Return         PVRSRV_OK if the synchronisation primitive was
+                successfully freed
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimFree(PVRSRV_CLIENT_SYNC_PRIM *psSync);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimSet
+
+@Description    Set the synchronisation primitive to a value
+
+@Input          psSync                  The synchronisation primitive to set
+
+@Input          ui32Value               Value to set it to
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimSet(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+
+#if defined(NO_HARDWARE)
+
+/*************************************************************************/ /*!
+@Function       SyncPrimNoHwUpdate
+
+@Description    Updates the synchronisation primitive value (in NoHardware drivers)
+
+@Input          psSync                  The synchronisation primitive to update
+
+@Input          ui32Value               Value to update it to
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimNoHwUpdate(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+#endif
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       SyncPrimPDump
+
+@Description    PDump the current value of the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpValue
+
+@Description    PDump the ui32Value as the value of the synchronisation
+                primitive (regardless of the current value).
+
+@Input          psSync          The synchronisation primitive to PDump
+@Input          ui32Value       Value to give to the sync prim on the pdump
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpPol
+
+@Description    Do a PDump poll of the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Input          ui32Value               Value to poll for
+
+@Input          ui32Mask                PDump mask operator
+
+@Input          ui32PDumpFlags          PDump flags
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT32 ui32Value,
+				 IMG_UINT32 ui32Mask,
+				 PDUMP_POLL_OPERATOR eOperator,
+				 IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpCBP
+
+@Description    Do a PDump CB poll using the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Input          uiWriteOffset           Current write offset of buffer
+
+@Input          uiPacketSize            Size of the packet to write into CB
+
+@Input          uiBufferSize            Size of the CB
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT64 uiWriteOffset,
+				 IMG_UINT64 uiPacketSize,
+				 IMG_UINT64 uiBufferSize);
+
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpValue)
+#endif
+static INLINE void
+SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDump)
+#endif
+static INLINE void
+SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpPol)
+#endif
+static INLINE void
+SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT32 ui32Value,
+				 IMG_UINT32 ui32Mask,
+				 PDUMP_POLL_OPERATOR eOperator,
+				 IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpCBP)
+#endif
+static INLINE void
+SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT64 uiWriteOffset,
+				 IMG_UINT64 uiPacketSize,
+				 IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+}
+#endif /* PDUMP */
+#endif /* SYNC_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync_internal.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync_internal.h
new file mode 100644
index 000000000000..929b66e0f37c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/sync_internal.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the internal client side interface for services
+                synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_INTERNAL
+#define SYNC_INTERNAL
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "ra.h"
+#include "dllist.h"
+#include "lock.h"
+#include "devicemem.h"
+#include "sync_prim_internal.h"
+
+#define LOCAL_SYNC_PRIM_RESET_VALUE 0
+#define LOCAL_SYNC_PRIM_POISON_VALUE 0xa5a5a5a5u
+
+/*
+	Debug feature to protect against GP DM page faults when
+	sync prims are freed by client before work is completed.
+*/
+#define LOCAL_SYNC_BLOCK_RETAIN_FIRST
+
+/*
+	Private structure's
+*/
+#define SYNC_PRIM_NAME_SIZE		50
+typedef struct SYNC_PRIM_CONTEXT_TAG
+{
+	SHARED_DEV_CONNECTION       hDevConnection;
+	IMG_CHAR					azName[SYNC_PRIM_NAME_SIZE];	/*!< Name of the RA */
+	RA_ARENA					*psSubAllocRA;					/*!< RA context */
+	IMG_CHAR					azSpanName[SYNC_PRIM_NAME_SIZE];/*!< Name of the span RA */
+	RA_ARENA					*psSpanRA;						/*!< RA used for span management of SubAllocRA */
+	ATOMIC_T				hRefCount;	/*!< Ref count for this context */
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	IMG_HANDLE					hFirstSyncPrim; /*!< Handle to the first allocated sync prim */
+#endif
+} SYNC_PRIM_CONTEXT;
+
+typedef struct SYNC_PRIM_BLOCK_TAG
+{
+	SYNC_PRIM_CONTEXT	*psContext;				/*!< Our copy of the services connection */
+	IMG_HANDLE			hServerSyncPrimBlock;	/*!< Server handle for this block */
+	IMG_UINT32			ui32SyncBlockSize;		/*!< Size of the sync prim block */
+	IMG_UINT32			ui32FirmwareAddr;		/*!< Firmware address */
+	DEVMEM_MEMDESC		*hMemDesc;				/*!< Host mapping handle */
+	IMG_UINT32 __iomem	*pui32LinAddr;			/*!< User CPU mapping */
+	IMG_UINT64			uiSpanBase;				/*!< Base of this import in the span RA */
+	DLLIST_NODE			sListNode;				/*!< List node for the sync block list */
+} SYNC_PRIM_BLOCK;
+
+typedef struct SYNC_PRIM_TAG
+{
+	PVRSRV_CLIENT_SYNC_PRIM	sCommon;		/*!< Client visible part of the sync prim */
+	ATOMIC_T				hRefCount;		/*!< Ref count for this sync */
+	SYNC_PRIM_BLOCK			*psSyncBlock;	/*!< Synchronisation block this primitive is allocated on */
+	IMG_UINT64				uiSpanAddr;		/*!< Span address of the sync */
+	IMG_HANDLE				hRecord;		/*!< Sync record handle */
+} SYNC_PRIM;
+
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimGetFirmwareAddr(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 *pui32FwAddr);
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimLocalGetHandleAndOffset(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+							IMG_HANDLE *phBlock,
+							IMG_UINT32 *pui32Offset);
+
+
+#endif	/* SYNC_INTERNAL */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/tlclient.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tlclient.h
new file mode 100644
index 000000000000..00f7aa8fc043
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tlclient.h
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File           tlclient.h
+@Title          Services Transport Layer shared API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common API used in both clients and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef TLCLIENT_H
+#define TLCLIENT_H
+
+
+#include "img_defs.h"
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_error.h"
+
+
+/* This value is used for the hSrvHandle argument in the client API when
+ * called directly from the kernel which will lead to a direct bridge access.
+ */
+#define DIRECT_BRIDGE_HANDLE	((IMG_HANDLE)0xDEADBEEFU)
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientOpenStream
+ @Description   Open a descriptor onto an existing kernel transport stream.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         pszName         Address of the stream name string, no longer
+                                than PRVSRVTL_MAX_STREAM_NAME_SIZE.
+ @Input         ui32Mode        Unused
+ @Output        phSD            Address of a pointer to an stream object
+ @Return        PVRSRV_ERROR_NOT_FOUND          when named stream not found
+ @Return        PVRSRV_ERROR_ALREADY_OPEN       stream already open by another
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_TIMEOUT            timed out, stream not found
+ @Return        PVRSRV_ERROR                    for other system codes
+*/ /**************************************************************************/
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientOpenStream(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR* pszName,
+		IMG_UINT32   ui32Mode,
+		IMG_HANDLE*  phSD);
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientCloseStream
+ @Description   Close and release the stream connection to Services kernel
+                server transport layer. Any outstanding Acquire will be
+                released.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle is not known
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCloseStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD);
+
+/*************************************************************************/ /*!
+ @Function      TLClientDiscoverStreams
+ @Description   Finds all streams that's name starts with pszNamePattern and
+                ends with a number.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         pszNamePattern  Name pattern. Must be beginning of a string.
+ @Output        aszStreams      Array of numbers from end of the discovered
+                names.
+ @inOut         pui32NumFound   When input, max number that can fit into
+                                pui32Streams. When output, number of
+                                discovered streams.
+ @Return        PVRSRV_ERROR    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientDiscoverStreams(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR *pszNamePattern,
+		IMG_CHAR aszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+		IMG_UINT32 *pui32NumFound);
+
+/*************************************************************************/ /*!
+ @Function      TLClientReserveStream
+ @Description   Reserves a region with given size in the stream. If the stream
+                is already reserved the function will return an error.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Output        ppui8Data       pointer to the buffer
+ @Input         ui32Size        size of the data
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLClientStreamReserve2
+ @Description   Reserves a region with given size in the stream. If the stream
+                is already reserved the function will return an error.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Output        ppui8Data       pointer to the buffer
+ @Input         ui32Size        size of the data
+ @Input         ui32SizeMin     minimum size of the data
+ @Input         ui32Available   available space in buffer
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream2(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size,
+		IMG_UINT32 ui32SizeMin,
+		IMG_UINT32 *pui32Available);
+
+/*************************************************************************/ /*!
+ @Function      TLClientStreamCommit
+ @Description   Commits previously reserved region in the stream and therefore
+                allows next reserves.
+                This function call has to be preceded by the call to
+                TLClientReserveStream or TLClientReserveStream2.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Input         ui32Size        Size of the data
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCommitStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLClientAcquireData
+ @Description   When there is data available in the stream buffer this call
+                returns with the address and length of the data buffer the
+                client can safely read. This buffer may contain one or more
+                packets of data.
+                If no data is available then this call blocks until it becomes
+                available. However if the stream has been destroyed while
+                waiting then a resource unavailable error will be returned to
+                the caller. Clients must pair this call with a ReleaseData
+                call.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Output        ppPacketBuf     Address of a pointer to an byte buffer. On exit
+                                pointer contains address of buffer to read from
+ @Output        puiBufLen       Pointer to an integer. On exit it is the size
+                                of the data to read from the packet buffer
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              release not called beforehand
+ @Return        PVRSRV_ERROR_TIMEOUT            block timed out, no data
+ @Return        PVRSRV_ERROR                    for other system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientAcquireData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE  hSD,
+		IMG_PBYTE*  ppPacketBuf,
+		IMG_UINT32* puiBufLen);
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientReleaseData
+ @Description   Called after client has read the stream data out of the buffer
+                The data is subsequently flushed from the stream buffer to make
+                room for more data packets from the stream source.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known to TL
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              acquire not called beforehand
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD);
+
+/*************************************************************************/ /*!
+ @Function      TLClientReleaseDataLess
+ @Description   Called after client has read only some data out of the buffer
+                and wishes to complete the read early i.e. does not want to
+                read the full data that the acquire call returned e.g read just
+                one packet from the stream.
+                The data is subsequently flushed from the stream buffer to make
+                room for more data packets from the stream source.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Input         uiActualReadLen Size of data read, in bytes. Must be on a TL
+                                packet boundary.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS     when read length too big
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known to TL
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              acquire not called beforehand
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseDataLess(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD, IMG_UINT32 uiActualReadLen);
+
+/*************************************************************************/ /*!
+ @Function      TLClientWriteData
+ @Description   Writes data to the stream.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Input         ui32Size        Size of the data
+ @Input         pui8Data        Pointer to data
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientWriteData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size,
+		IMG_BYTE *pui8Data);
+
+
+#endif /* TLCLIENT_H */
+
+/******************************************************************************
+ End of file (tlclient.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/shared/include/tutilsdefs.h b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tutilsdefs.h
new file mode 100644
index 000000000000..b89e4a4bbb1f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/shared/include/tutilsdefs.h
@@ -0,0 +1,230 @@
+/*************************************************************************/ /*!
+@File           tutilsdefs.h
+@Title          Testing utils bridge defines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared structures and constants between client and server sides
+                of tutils bridge
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TUTILSDEFS_H
+#define TUTILSDEFS_H
+
+
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_sync_km.h"
+
+/******************************************************************************
+ *
+ * TEST Related definitions and constants
+ */
+#define PVR_TL_TEST_STREAM_BRIDGE_NAME "TLBRIDGE_TEST"
+#define PVR_TL_TEST_UMBASE             0x00202000
+#define PVR_TL_TEST_OFFSET             0x0008
+#define PVR_TL_TEST_LEN                0x0010
+
+#define PVR_TL_TEST_STREAM2_NAME	"TLSTREAM2_TEST"
+#define PVR_TL_TEST_STREAM2_SIZE	2
+
+#define PVR_TL_TEST_STREAM3_NAME	"TLSTREAM3_TEST"
+#define PVR_TL_TEST_STREAM3_SIZE	256
+
+// This constant, when used as a parameter in StreamCreate, lessens the size of
+// the buffer that is created for a stream, to avoid going over a page boundary.
+#define PVR_TL_TEST_STREAM_BUFFER_REDUCTION 32
+
+#define PVR_TL_TEST_CMD_SOURCE_START			10
+typedef struct _PVR_TL_TEST_CMD_SOURCE_START_IN_
+{
+	/* Stream name must always be first in struct */
+	IMG_CHAR    pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_UINT16  uiStreamSizeInPages;  /* # of 4Kb pages */
+	IMG_UINT16  uiInterval;           /* in milliseconds */
+	IMG_UINT16  uiCallbackKicks;      /* 0 for no limit of timer call backs */
+	IMG_UINT16  uiEOSMarkerKicks;     /* Insert EOS Marker every N Kicks, 0 for none */
+	IMG_UINT16  uiPacketSizeInBytes;  /* 0 for random size between 1..255 size in bytes */
+	IMG_UINT32  uiStreamCreateFlags;  /* See TLStreamCreate() */
+	IMG_UINT16  uiStartDelay;         /* 0 for normal uiInterval delay, one off delay in ms */
+	IMG_BOOL    bDoNotDeleteStream;   /* When true the stream is not deleted on self
+	                                   * cleanup sources only the timers and other resources are */
+	IMG_BOOL    bDelayStreamCreate;   /* When true the stream used in the source is created
+	                                   * in the first kick. False for normal behaviour where
+	                                   * the stream is created in the bridge source start context */
+} PVR_TL_TEST_CMD_SOURCE_START_IN;
+
+
+#define PVR_TL_TEST_CMD_SOURCE_STOP		11
+typedef struct _PVR_TL_TEST_CMD_SOURCE_STOP_IN_
+{
+	/* Stream name must always be first in struct */
+	IMG_CHAR  pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_BOOL  bDoNotDeleteStream;
+} PVR_TL_TEST_CMD_SOURCE_STOP_IN;
+
+#define PVR_TL_TEST_CMD_SOURCE_START2	12	/* Uses two stage data submit */
+typedef PVR_TL_TEST_CMD_SOURCE_START_IN PVR_TL_TEST_CMD_SOURCE_START2_IN;
+
+#define PVR_TL_TEST_CMD_DEBUG_LEVEL	13
+/* No typedef, uses integer uiIn1 in union */
+
+#define PVR_TL_TEST_CMD_DUMP_TL_STATE	14
+/* No typedef, uses integer uiIn1 in union */
+
+#define PVR_TL_TEST_CMD_STREAM_CREATE	15
+typedef struct _PVR_TL_TEST_CMD_STREAM_CREATE_IN_
+{
+	/* Stream name must always be first in struct */
+	IMG_CHAR    pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_UINT16  uiStreamSizeInPages;
+	IMG_UINT32  uiStreamCreateFlags;
+	IMG_BOOL    bWithOpenCallback;
+} PVR_TL_TEST_CMD_STREAM_CREATE_IN;
+
+#define PVR_TL_TEST_CMD_STREAM_CLOSE	16
+typedef struct _PVR_TL_TEST_CMD_STREAM_NAME_IN_
+{
+	IMG_CHAR  pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+} PVR_TL_TEST_CMD_STREAM_NAME_IN;
+
+#define PVR_TL_TEST_CMD_STREAM_OPEN 17
+
+#define PVR_TL_TEST_CMD_DUMP_HWPERF_STATE 18
+
+#define PVR_TL_TEST_CMD_FLUSH_HWPERF_FWBUF 19
+
+#define PVR_TL_TEST_CMD_DUMP_PDUMP_STATE 21
+
+typedef union _PVR_TL_TEST_CMD_IN_
+{
+	PVR_TL_TEST_CMD_SOURCE_START_IN sStart;
+	PVR_TL_TEST_CMD_SOURCE_STOP_IN  sStop;
+/*	PVR_TL_TEST_CMD_SOURCE_START_IN sStart2;  Used by #12, use sStart instead */
+	IMG_UINT32	uiIn1;						 /* Used by #13, #14 */
+	PVR_TL_TEST_CMD_STREAM_CREATE_IN  sCreate;
+	PVR_TL_TEST_CMD_STREAM_NAME_IN sName;
+	IMG_UINT32 uiParams[6];
+} PVR_TL_TEST_CMD_IN;
+
+/* Has to be the largest test IN structure */
+#define PVR_TL_TEST_PARAM_MAX_SIZE  (sizeof(PVR_TL_TEST_CMD_IN)+4)
+
+#define PVR_TL_TEST_CMD_SET_PWR_STATE              22
+#define PVR_TL_TEST_CMD_GET_PWR_STATE              23
+#define PVR_TL_TEST_CMD_SET_DWT_PWR_CHANGE_COUNTER 24
+#define PVR_TL_TEST_CMD_GET_DWT_PWR_CHANGE_COUNTER 25
+
+#define PVR_TL_TEST_PWR_STATE_ON  1
+#define PVR_TL_TEST_PWR_STATE_OFF 0
+
+/****************************************************************************
+ * PowMonTestThread IOCTL calls and constants
+ */
+
+#define PVR_POWMON_CMD_GET_ESTIMATES	1
+#define PVR_POWMON_CMD_SET_THREAD_LATENCY	2
+#define PVR_POWMON_CMD_TEST_THREAD_UPDATE_STATE	3
+
+#define PVR_POWMON_TEST_THREAD_RESUME	1
+#define PVR_POWMON_TEST_THREAD_PAUSE	0
+
+/****************************************************************************
+ * PowerTestThread IOCTL calls and constants
+ */
+
+#define PVR_POWER_TEST_CMD_DVFS                 1
+#define PVR_POWER_TEST_CMD_FORCED_IDLE          2
+#define PVR_POWER_TEST_CMD_CANCEL_FORCED_IDLE   3
+#define PVR_POWER_TEST_CMD_POWER_ON             4
+#define PVR_POWER_TEST_CMD_POWER_OFF            5
+#define PVR_POWER_TEST_CMD_APM_LATENCY          6
+#define PVR_POWER_TEST_CMD_INVALID              7
+
+#define PVR_POWER_TEST_NON_FORCED 0
+#define PVR_POWER_TEST_FORCED     1
+
+/****************************************************************************
+ * SyncCheckpointTest IOCTL types
+ */
+
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CONTEXT_CREATE		26
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CONTEXT_DESTROY		27
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_REGISTER_FUNCS		28
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CREATE				29
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CREATE_NULL_CTXT	30
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_CREATE_NULL_RTRN	31
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_DESTROY				32
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_SIGNAL				33
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_ERROR				34
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_IS_SIGNALLED		35
+#define PVR_TL_TEST_CMD_SYNC_CHECKPOINT_IS_ERRORED			36
+
+typedef struct _PVR_TL_TEST_CMD_CHECKPOINT_CREATE_IN_
+{
+	/* Checkpoint name must always be first in struct */
+	IMG_CHAR    pszCheckpointName[PVRSRV_SYNC_NAME_LENGTH];
+	IMG_UINT16  uiStreamSizeInPages;
+	IMG_UINT32  uiStreamCreateFlags;
+} PVR_TL_TEST_CMD_CHECKPOINT_CREATE_IN;
+
+#define PVR_TL_TEST_CMD_SET_STREAM_OPEN_COUNTER 37
+#define PVR_TL_TEST_CMD_GET_STREAM_OPEN_COUNTER 38
+
+typedef struct _PVR_TL_TEST_CMD_STREAM_OPEN_COUNTER_IN_
+{
+	IMG_CHAR    pszStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+	IMG_UINT32  ui32Counter;
+} PVR_TL_TEST_CMD_STREAM_OPEN_COUNTER_IN;
+
+/****************************************************************************
+ * KmallocThreshold IOCTL types
+ */
+
+#define PVR_TL_TEST_CMD_KMALLOC 39
+
+typedef struct _PVR_TL_TEST_CMD_KMALLOC_IN_
+{
+	IMG_UINT32 uiAllocCount;
+	IMG_UINT32 uiAllocSize;
+	IMG_UINT32 uiFailedAllocThreshold;
+	IMG_UINT32 uiFailedAllocFrequency;
+} PVR_TL_TEST_CMD_KMALLOC_IN;
+
+#endif /* TUTILSDEFS_H */
+
+/******************************************************************************
+ End of file (tutilsdefs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/interrupt_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/interrupt_support.c
new file mode 100644
index 000000000000..c67d45352050
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/interrupt_support.c
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/interrupt.h>
+
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "interrupt_support.h"
+
+typedef struct LISR_DATA_TAG
+{
+	IMG_UINT32	ui32IRQ;
+	PFN_SYS_LISR	pfnLISR;
+	void		*pvData;
+} LISR_DATA;
+
+static irqreturn_t SystemISRWrapper(int irq, void *dev_id)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *)dev_id;
+
+	PVR_UNREFERENCED_PARAMETER(irq);
+
+	if (psLISRData)
+	{
+		if (psLISRData->pfnLISR(psLISRData->pvData))
+		{
+			return IRQ_HANDLED;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Missing interrupt data", __func__));
+	}
+
+	return IRQ_NONE;
+}
+
+PVRSRV_ERROR OSInstallSystemLISR(IMG_HANDLE *phLISR,
+				 IMG_UINT32 ui32IRQ,
+				 const IMG_CHAR *pszDevName,
+				 PFN_SYS_LISR pfnLISR,
+				 void *pvData,
+				 IMG_UINT32 ui32Flags)
+{
+	LISR_DATA *psLISRData;
+	unsigned long ulIRQFlags = 0;
+
+	if (pfnLISR == NULL || pvData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32Flags & ~SYS_IRQ_FLAG_MASK)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	switch (ui32Flags & SYS_IRQ_FLAG_TRIGGER_MASK)
+	{
+		case SYS_IRQ_FLAG_TRIGGER_DEFAULT:
+			break;
+		case SYS_IRQ_FLAG_TRIGGER_LOW:
+			ulIRQFlags |= IRQF_TRIGGER_LOW;
+			break;
+		case SYS_IRQ_FLAG_TRIGGER_HIGH:
+			ulIRQFlags |= IRQF_TRIGGER_HIGH;
+			break;
+		default:
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32Flags & SYS_IRQ_FLAG_SHARED)
+	{
+		ulIRQFlags |= IRQF_SHARED;
+	}
+
+	psLISRData = OSAllocMem(sizeof(*psLISRData));
+	if (psLISRData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psLISRData->ui32IRQ = ui32IRQ;
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+
+	if (request_irq(ui32IRQ, SystemISRWrapper, ulIRQFlags, pszDevName, psLISRData))
+	{
+		OSFreeMem(psLISRData);
+
+		return PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER;
+	}
+
+	*phLISR = (IMG_HANDLE)psLISRData;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSUninstallSystemLISR(IMG_HANDLE hLISR)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *)hLISR;
+
+	if (psLISRData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	free_irq(psLISRData->ui32IRQ, psLISRData);
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/pci_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/pci_support.c
new file mode 100644
index 000000000000..a8321a493e3e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/env/linux/pci_support.c
@@ -0,0 +1,643 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/pci.h>
+
+#if defined(CONFIG_MTRR)
+#include <asm/mtrr.h>
+#endif
+
+#include "pci_support.h"
+#include "allocmem.h"
+
+typedef	struct _PVR_PCI_DEV_TAG
+{
+	struct pci_dev		*psPCIDev;
+	HOST_PCI_INIT_FLAGS	ePCIFlags;
+	IMG_BOOL		abPCIResourceInUse[DEVICE_COUNT_RESOURCE];
+	int			iMTRR[DEVICE_COUNT_RESOURCE];
+} PVR_PCI_DEV;
+
+/*************************************************************************/ /*!
+@Function       OSPCISetDev
+@Description    Set a PCI device for subsequent use.
+@Input          pvPCICookie             Pointer to OS specific PCI structure
+@Input          eFlags                  Flags
+@Return		PVRSRV_PCI_DEV_HANDLE   Pointer to PCI device handle
+*/ /**************************************************************************/
+PVRSRV_PCI_DEV_HANDLE OSPCISetDev(void *pvPCICookie, HOST_PCI_INIT_FLAGS eFlags)
+{
+	int err;
+	IMG_UINT32 i;
+	PVR_PCI_DEV *psPVRPCI;
+
+	psPVRPCI = OSAllocMem(sizeof(*psPVRPCI));
+	if (psPVRPCI == NULL)
+	{
+		printk(KERN_ERR "OSPCISetDev: Couldn't allocate PVR PCI structure\n");
+		return NULL;
+	}
+
+	psPVRPCI->psPCIDev = (struct pci_dev *)pvPCICookie;
+	psPVRPCI->ePCIFlags = eFlags;
+
+	err = pci_enable_device(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCISetDev: Couldn't enable device (%d)\n", err);
+		OSFreeMem(psPVRPCI);
+		return NULL;
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_set_master(psPVRPCI->psPCIDev);
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_MSI)		/* PRQA S 3358 */ /* misuse of enums */
+	{
+#if defined(CONFIG_PCI_MSI)
+		err = pci_enable_msi(psPVRPCI->psPCIDev);
+		if (err != 0)
+		{
+			printk(KERN_ERR "OSPCISetDev: Couldn't enable MSI (%d)", err);
+			psPVRPCI->ePCIFlags &= ~HOST_PCI_INIT_FLAG_MSI;	/* PRQA S 1474,3358,4130 */ /* misuse of enums */
+		}
+#else
+		printk(KERN_ERR "OSPCISetDev: MSI support not enabled in the kernel");
+#endif
+	}
+
+	/* Initialise the PCI resource and MTRR tracking array */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		psPVRPCI->abPCIResourceInUse[i] = IMG_FALSE;
+		psPVRPCI->iMTRR[i] = -1;
+	}
+
+	return (PVRSRV_PCI_DEV_HANDLE)psPVRPCI;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAcquireDev
+@Description    Acquire a PCI device for subsequent use.
+@Input          ui16VendorID            Vendor PCI ID
+@Input          ui16DeviceID            Device PCI ID
+@Input          eFlags                  Flags
+@Return		PVRSRV_PCI_DEV_HANDLE   Pointer to PCI device handle
+*/ /**************************************************************************/
+PVRSRV_PCI_DEV_HANDLE OSPCIAcquireDev(IMG_UINT16 ui16VendorID,
+				      IMG_UINT16 ui16DeviceID,
+				      HOST_PCI_INIT_FLAGS eFlags)
+{
+	struct pci_dev *psPCIDev;
+
+	psPCIDev = pci_get_device(ui16VendorID, ui16DeviceID, NULL);
+	if (psPCIDev == NULL)
+	{
+		return NULL;
+	}
+
+	return OSPCISetDev((void *)psPCIDev, eFlags);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIIRQ
+@Description    Get the interrupt number for the device.
+@Input          hPVRPCI                 PCI device handle
+@Output         pui16DeviceID           Pointer to where the interrupt number
+                                        should be returned
+@Return		PVRSRV_ERROR            Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIIRQ(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 *pui32IRQ)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (pui32IRQ == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui32IRQ = psPVRPCI->psPCIDev->irq;
+
+	return PVRSRV_OK;
+}
+
+/* Functions supported by OSPCIAddrRangeFunc */
+enum HOST_PCI_ADDR_RANGE_FUNC
+{
+	HOST_PCI_ADDR_RANGE_FUNC_LEN,
+	HOST_PCI_ADDR_RANGE_FUNC_START,
+	HOST_PCI_ADDR_RANGE_FUNC_END,
+	HOST_PCI_ADDR_RANGE_FUNC_REQUEST,
+	HOST_PCI_ADDR_RANGE_FUNC_RELEASE
+};
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeFunc
+@Description    Internal support function for various address range related
+                functions
+@Input          eFunc                   Function to perform
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Function dependent value
+*/ /**************************************************************************/
+static IMG_UINT64 OSPCIAddrRangeFunc(enum HOST_PCI_ADDR_RANGE_FUNC eFunc,
+										 PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+										 IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (ui32Index >= DEVICE_COUNT_RESOURCE)
+	{
+		printk(KERN_ERR "OSPCIAddrRangeFunc: Index out of range");
+		return 0;
+	}
+
+	switch (eFunc)
+	{
+		case HOST_PCI_ADDR_RANGE_FUNC_LEN:
+		{
+			return pci_resource_len(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_START:
+		{
+			return pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_END:
+		{
+			return pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_REQUEST:
+		{
+			int err = pci_request_region(psPVRPCI->psPCIDev, (IMG_INT)ui32Index, PVRSRV_MODNAME);
+			if (err != 0)
+			{
+				printk(KERN_ERR "OSPCIAddrRangeFunc: pci_request_region_failed (%d)", err);
+				return 0;
+			}
+			psPVRPCI->abPCIResourceInUse[ui32Index] = IMG_TRUE;
+			return 1;
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_RELEASE:
+		{
+			if (psPVRPCI->abPCIResourceInUse[ui32Index])
+			{
+				pci_release_region(psPVRPCI->psPCIDev, (IMG_INT)ui32Index);
+				psPVRPCI->abPCIResourceInUse[ui32Index] = IMG_FALSE;
+			}
+			return 1;
+		}
+		default:
+		{
+			printk(KERN_ERR "OSPCIAddrRangeFunc: Unknown function");
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeLen
+@Description    Returns length of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Length of address range or 0 if no
+                                        such range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeLen(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_LEN, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeStart
+@Description    Returns the start of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Start of address range or 0 if no
+                                        such range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeStart(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_START, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeEnd
+@Description    Returns the end of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              End of address range or 0 if no such
+                                        range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeEnd(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_END, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIRequestAddrRange
+@Description    Request a given address range index for subsequent use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIRequestAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+								   IMG_UINT32 ui32Index)
+{
+	if (OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_REQUEST, hPVRPCI, ui32Index) == 0)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseAddrRange
+@Description    Release a given address range that is no longer being used
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	if (OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_RELEASE, hPVRPCI, ui32Index) == 0)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIRequestAddrRegion
+@Description    Request a given region from an address range for subsequent use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Input          uiOffset              Offset into the address range that forms
+                                        the start of the region
+@Input          uiLength              Length of the region
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIRequestAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+									IMG_UINT32 ui32Index,
+									IMG_UINT64 uiOffset,
+									IMG_UINT64 uiLength)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start;
+	resource_size_t end;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+
+	/* Check that the requested region is valid */
+	if ((start + uiOffset + uiLength - 1) > end)
+	{
+		return PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH;
+	}
+
+	if (pci_resource_flags(psPVRPCI->psPCIDev, ui32Index) & IORESOURCE_IO)
+	{
+		if (request_region(start + uiOffset, uiLength, PVRSRV_MODNAME) == NULL)
+		{
+			return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		}
+	}
+	else
+	{
+		if (request_mem_region(start + uiOffset, uiLength, PVRSRV_MODNAME) == NULL)
+		{
+			return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseAddrRegion
+@Description    Release a given region, from an address range, that is no
+                longer in use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Input          ui32Offset              Offset into the address range that forms
+                                        the start of the region
+@Input          ui32Length              Length of the region
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+									IMG_UINT32 ui32Index,
+									IMG_UINT64 uiOffset,
+									IMG_UINT64 uiLength)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start;
+	resource_size_t end;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+
+	/* Check that the region is valid */
+	if ((start + uiOffset + uiLength - 1) > end)
+	{
+		return PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH;
+	}
+
+	if (pci_resource_flags(psPVRPCI->psPCIDev, ui32Index) & IORESOURCE_IO)
+	{
+		release_region(start + uiOffset, uiLength);
+	}
+	else
+	{
+		release_mem_region(start + uiOffset, uiLength);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseDev
+@Description    Release a PCI device that is no longer being used
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int i;
+
+	/* Release all PCI regions that are currently in use */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			pci_release_region(psPVRPCI->psPCIDev, i);
+			psPVRPCI->abPCIResourceInUse[i] = IMG_FALSE;
+		}
+	}
+
+#if defined(CONFIG_PCI_MSI)
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_MSI)		/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_disable_msi(psPVRPCI->psPCIDev);
+	}
+#endif
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_clear_master(psPVRPCI->psPCIDev);
+	}
+
+	pci_disable_device(psPVRPCI->psPCIDev);
+
+	OSFreeMem(psPVRPCI);
+	/*not nulling pointer, copy on stack*/
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCISuspendDev
+@Description    Prepare PCI device to be turned off by power management
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCISuspendDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int i;
+	int err;
+
+	/* Release all PCI regions that are currently in use */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			pci_release_region(psPVRPCI->psPCIDev, i);
+		}
+	}
+
+	err = pci_save_state(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCISuspendDev: pci_save_state_failed (%d)", err);
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+
+	pci_disable_device(psPVRPCI->psPCIDev);
+
+	err = pci_set_power_state(psPVRPCI->psPCIDev, pci_choose_state(psPVRPCI->psPCIDev, PMSG_SUSPEND));
+	switch (err)
+	{
+		case 0:
+			break;
+		case -EIO:
+			printk(KERN_ERR "OSPCISuspendDev: device doesn't support PCI PM");
+			break;
+		case -EINVAL:
+			printk(KERN_ERR "OSPCISuspendDev: can't enter requested power state");
+			break;
+		default:
+			printk(KERN_ERR "OSPCISuspendDev: pci_set_power_state failed (%d)", err);
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIResumeDev
+@Description    Prepare a PCI device to be resumed by power management
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIResumeDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int err;
+	int i;
+
+	err = pci_set_power_state(psPVRPCI->psPCIDev, pci_choose_state(psPVRPCI->psPCIDev, PMSG_ON));
+	switch (err)
+	{
+		case 0:
+			break;
+		case -EIO:
+			printk(KERN_ERR "OSPCIResumeDev: device doesn't support PCI PM");
+			break;
+		case -EINVAL:
+			printk(KERN_ERR "OSPCIResumeDev: can't enter requested power state");
+			return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+		default:
+			printk(KERN_ERR "OSPCIResumeDev: pci_set_power_state failed (%d)", err);
+			return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	pci_restore_state(psPVRPCI->psPCIDev);
+
+	err = pci_enable_device(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCIResumeDev: Couldn't enable device (%d)", err);
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+		pci_set_master(psPVRPCI->psPCIDev);
+
+	/* Restore the PCI resource tracking array */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			err = pci_request_region(psPVRPCI->psPCIDev, i, PVRSRV_MODNAME);
+			if (err != 0)
+			{
+				printk(KERN_ERR "OSPCIResumeDev: pci_request_region_failed (region %d, error %d)", i, err);
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIGetVendorDeviceIDs
+@Description    Retrieve PCI vendor ID and device ID.
+@Input          hPVRPCI                 PCI device handle
+@Output         pui16VendorID           Vendor ID
+@Output         pui16DeviceID           Device ID
+@Return         PVRSRV_ERROR            Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIGetVendorDeviceIDs(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+                                     IMG_UINT16 *pui16VendorID,
+                                     IMG_UINT16 *pui16DeviceID)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	struct pci_dev *psPCIDev;
+
+	if (psPVRPCI == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psPCIDev = psPVRPCI->psPCIDev;
+	if (psPCIDev == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui16VendorID = psPCIDev->vendor;
+	*pui16DeviceID = psPCIDev->device;
+
+	return PVRSRV_OK;
+}
+
+#if defined(CONFIG_MTRR)
+
+/*************************************************************************/ /*!
+@Function       OSPCIClearResourceMTRRs
+@Description    Clear any BIOS-configured MTRRs for a PCI memory region
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start, end;
+	int res;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index) + 1;
+
+	res = arch_io_reserve_memtype_wc(start, end - start);
+	if (res)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	res = arch_phys_wc_add(start, end - start);
+	if (res < 0)
+	{
+		arch_io_free_memtype_wc(start, end - start);
+
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	psPVRPCI->iMTRR[ui32Index] = res;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseResourceMTRRs
+@Description    Release resources allocated by OSPCIClearResourceMTRRs
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+*/ /**************************************************************************/
+void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (psPVRPCI->iMTRR[ui32Index] >= 0)
+	{
+		arch_phys_wc_del(psPVRPCI->iMTRR[ui32Index]);
+		psPVRPCI->iMTRR[ui32Index] = -1;
+
+		{
+			resource_size_t start, end;
+
+			start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+			end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index) + 1;
+
+			arch_io_free_memtype_wc(start, end - start);
+		}
+	}
+}
+#endif /* defined(CONFIG_MTRR) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/sysconfig_cmn.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/sysconfig_cmn.c
new file mode 100644
index 000000000000..d8a09182dc04
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/sysconfig_cmn.c
@@ -0,0 +1,167 @@
+/*************************************************************************/ /*!
+@File
+@Title          Sysconfig layer common to all platforms
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements system layer functions common to all platforms
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "pvr_debug.h"
+#include "os_apphint.h"
+//#include "physmem.h"
+
+void SysRGXErrorNotify(IMG_HANDLE hSysData,
+                       PVRSRV_ROBUSTNESS_NOTIFY_DATA *psErrorData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+	{
+		IMG_UINT32 ui32DgbLvl;
+
+		switch (psErrorData->eResetReason)
+		{
+			case RGX_CONTEXT_RESET_REASON_NONE:
+			case RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP:
+			case RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP:
+			case RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING:
+			case RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING:
+			case RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH:
+			case RGX_CONTEXT_RESET_REASON_GPU_ECC_OK:
+			case RGX_CONTEXT_RESET_REASON_FW_ECC_OK:
+			{
+				ui32DgbLvl = PVR_DBG_MESSAGE;
+				break;
+			}
+			case RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR:
+			case RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR:
+			case RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR:
+			case RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR:
+			{
+				ui32DgbLvl = PVR_DBG_WARNING;
+				break;
+			}
+			case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_FW_ECC_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_WATCHDOG:
+			case RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT:
+			case RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR:
+			case RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH:
+			case RGX_CONTEXT_RESET_REASON_DCLS_ERR:
+			{
+				ui32DgbLvl = PVR_DBG_ERROR;
+				break;
+			}
+			default:
+			{
+				PVR_ASSERT(false && "Unhandled reset reason");
+				ui32DgbLvl = PVR_DBG_ERROR;
+				break;
+			}
+		}
+
+		if (psErrorData->pid > 0)
+		{
+			PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, " PID %d experienced error %d",
+					 psErrorData->pid, psErrorData->eResetReason);
+		}
+		else
+		{
+			PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, " Device experienced error %d",
+					 psErrorData->eResetReason);
+		}
+
+		switch (psErrorData->eResetReason)
+		{
+			case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+			{
+				PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, "   ExtJobRef 0x%x, DM %d",
+						 psErrorData->uErrData.sChecksumErrData.ui32ExtJobRef,
+						 psErrorData->uErrData.sChecksumErrData.eDM);
+			break;
+			}
+			default:
+			{
+				break;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psErrorData);
+#endif /* PVRSRV_NEED_PVR_DPF */
+}
+
+IMG_UINT64 SysRestrictGpuLocalPhysheap(IMG_UINT64 uiHeapSize)
+{
+	return uiHeapSize;
+}
+
+IMG_BOOL SysRestrictGpuLocalAddPrivateHeap(void)
+{
+	return IMG_FALSE;
+}
+
+IMG_BOOL SysDefaultToCpuLocalHeap(void)
+{
+//#if (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault = IMG_FALSE;
+	IMG_BOOL bSetToCPULocal = IMG_FALSE;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState,
+			PhysHeapHybridDefault2CpuLocal, &bAppHintDefault, &bSetToCPULocal);
+	OSFreeAppHintState(pvAppHintState);
+
+	return bSetToCPULocal;
+//#else
+//	return IMG_FALSE;
+//#endif
+}
+/******************************************************************************
+ End of file (sysconfig_cmn.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/common/uma_heap_fns.c b/drivers/gpu/drm/img/img-volcanic/services/system/common/uma_heap_fns.c
new file mode 100644
index 000000000000..77e481a74e42
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/common/uma_heap_fns.c
@@ -0,0 +1,88 @@
+/*****************************************************************************
+@File
+@Title          Boilerplate heap functions for UMA systems
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements generic nop conversions for physical addresses in
+                UMA systems.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#include "uma_heap_fns.h"
+
+/*
+ * CPU to Device physical address translation
+ */
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				   IMG_UINT32 ui32NumOfAddr,
+				   IMG_DEV_PHYADDR *psDevPAddr,
+				   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+	}
+}
+/*
+ * Device to CPU physical address translation
+ */
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				   IMG_UINT32 ui32NumOfAddr,
+				   IMG_CPU_PHYADDR *psCpuPAddr,
+				   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+	}
+}
+
+PHYS_HEAP_FUNCTIONS g_sUmaHeapFns = {
+	.pfnCpuPAddrToDevPAddr = UMAPhysHeapCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = UMAPhysHeapDevPAddrToCpuPAddr,
+};
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/Kbuild.mk
new file mode 100644
index 000000000000..6bd9041535b3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/Kbuild.mk
@@ -0,0 +1,58 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += \
+	services/system/$(PVR_SYSTEM)/sysconfig.o \
+	services/system/common/sysconfig_cmn.o \
+	services/server/common/vmm_pvz_client.o \
+	services/server/common/vmm_pvz_server.o \
+	services/server/common/vz_vmm_pvz.o \
+	services/server/common/vz_vmm_vm.o \
+	services/system/eswin_cpu/common/vmm_type_$(VMM_TYPE).o
+
+ccflags-y += \
+	-I$(TOP)/services/system/eswin_cpu/common/env/linux \
+	-I$(TOP)/services/system/common/env/linux \
+	-I$(TOP)/services/include/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc \
+	-I$(TOP)/include/system/$(PVR_SYSTEM)
+	
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/env/linux/dma_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/env/linux/dma_support.c
new file mode 100644
index 000000000000..977f9c85d037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/env/linux/dma_support.c
@@ -0,0 +1,335 @@
+/*************************************************************************/ /*!
+@File           dma_support.c
+@Title          System DMA support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a contiguous memory allocator (i.e. DMA allocator);
+                APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#endif
+
+#include "allocmem.h"
+#include "dma_support.h"
+
+#define DMA_MAX_IOREMAP_ENTRIES 8
+static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
+static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};
+static IMG_UINT32 gsDmaIoRemapRef[DMA_MAX_IOREMAP_ENTRIES] = {0};
+
+/*!
+******************************************************************************
+ @Function			SysDmaAllocMem
+
+ @Description		Allocates physically contiguous memory
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDmaAlloc != NULL && psDmaAlloc->pvOSDevice != NULL)
+	{
+#if defined(__linux__)
+		psDmaAlloc->pvVirtAddr =
+				dma_alloc_coherent((struct device *)psDmaAlloc->pvOSDevice,
+								   (size_t) psDmaAlloc->ui64Size,
+								   (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr,
+								   GFP_KERNEL);
+		PVR_LOG_RETURN_IF_FALSE((NULL != psDmaAlloc->pvVirtAddr), "dma_alloc_coherent() failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
+#else
+		#error "Provide OS implementation of DMA allocation";
+#endif
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaFreeMem
+
+ @Description		Free physically contiguous memory
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
+{
+	if (psDmaAlloc && psDmaAlloc->pvVirtAddr)
+	{
+#if defined(__linux__)
+		dma_free_coherent((struct device *)psDmaAlloc->pvOSDevice,
+						  (size_t) psDmaAlloc->ui64Size,
+						  psDmaAlloc->pvVirtAddr,
+						  (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
+#else
+		#error "Provide OS implementation of DMA deallocation";
+#endif
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaRegisterForIoRemapping
+
+ @Description		Registers DMA_ALLOC for manual I/O remapping
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (psDmaAlloc->pvVirtAddr == NULL)
+	{
+		/* Check if an I/O remap entry already exists for this request */
+		for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+		{
+			if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+			{
+				PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr);
+				break;
+			}
+		}
+
+		if (ui32Idx < DMA_MAX_IOREMAP_ENTRIES)
+		{
+			IMG_UINT64 ui64Offset;
+			ui64Offset = psDmaAlloc->sBusAddr.uiAddr - gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr;
+			psDmaAlloc->pvVirtAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr + (uintptr_t)ui64Offset;
+			gsDmaIoRemapRef[ui32Idx] += 1;
+			return PVRSRV_OK;
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check if there is a free I/O remap table entry for this request */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0)
+		{
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL);
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
+			break;
+		}
+	}
+
+	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
+	{
+		return eError;
+	}
+
+	gsDmaIoRemapArray[ui32Idx].ui64Size = psDmaAlloc->ui64Size;
+	gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
+	gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;
+	gsDmaIoRemapRef[ui32Idx] += 1;
+
+	PVR_LOG(("DMA: register I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+			psDmaAlloc->pvVirtAddr,
+			psDmaAlloc->sBusAddr.uiAddr,
+			psDmaAlloc->ui64Size));
+
+	gbEnableDmaIoRemapping = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDeregisterForIoRemapping
+
+ @Description		Deregisters DMA_ALLOC from manual I/O remapping
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return;
+	}
+
+	/* Remove specified entry from the list of I/O remap entries */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+		{
+			if (! --gsDmaIoRemapRef[ui32Idx])
+			{
+				PVR_LOG(("DMA: deregister I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+						gsDmaIoRemapArray[ui32Idx].pvVirtAddr,
+						gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr,
+						gsDmaIoRemapArray[ui32Idx].ui64Size));
+
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
+				gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
+				gsDmaIoRemapArray[ui32Idx].ui64Size = 0;
+			}
+
+			break;
+		}
+	}
+
+	/* Check if no other I/O remap entries exists for remapping */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
+		{
+			break;
+		}
+	}
+
+	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
+	{
+		/* No entries found so disable remapping */
+		gbEnableDmaIoRemapping = IMG_FALSE;
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDevPAddrToCpuVAddr
+
+ @Description		Maps a DMA_ALLOC physical address to CPU virtual address
+
+ @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+ ******************************************************************************/
+IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
+{
+	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return pvDMAVirtAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
+				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: PA: 0x%llx => VA: 0x%p",
+					uiAddr, pvDMAVirtAddr));
+
+				break;
+			}
+		}
+	}
+
+	return pvDMAVirtAddr;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaCpuVAddrToDevPAddr
+
+ @Description		Maps a DMA_ALLOC CPU virtual address to physical address
+
+ @Return			Non-zero value on success. Otherwise, a 0
+ ******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
+{
+	IMG_UINT64 uiAddr = 0;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return uiAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: VA: 0x%p => PA: 0x%llx",
+					pvDMAVirtAddr, uiAddr));
+
+				break;
+			}
+		}
+	}
+
+	return uiAddr;
+}
+
+/******************************************************************************
+ End of file (dma_support.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/vmm_type_stub.c b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/vmm_type_stub.c
new file mode 100644
index 000000000000..0f52c3447e88
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/common/vmm_type_stub.c
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File			vmm_type_stub.c
+@Title          Stub VM manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Sample stub (no-operation) VM manager implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+StubVMMMapDevPhysHeap(IMG_UINT64 ui64Size,
+					  IMG_UINT64 ui64Addr)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(ui64Addr);
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+StubVMMUnmapDevPhysHeap(void)
+{
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static VMM_PVZ_CONNECTION gsStubVmmPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&StubVMMMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&StubVMMUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+                                    PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+    PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsStubVmmPvz;
+	PVR_DPF((PVR_DBG_ERROR, "Using a stub VM manager type, no runtime VZ support"));
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+                             PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+    PVR_UNREFERENCED_PARAMETER(psDevConfig);
+    PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_stub.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/include/dma_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/include/dma_support.h
new file mode 100644
index 000000000000..cc1e4ebc31ab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/include/dma_support.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@File           dma_support.h
+@Title          Device contiguous memory allocator and I/O re-mapper
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides a contiguous memory allocator API; mainly
+                used for allocating / ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_SUPPORT_H
+#define DMA_SUPPORT_H
+
+#include "osfunc.h"
+#include "pvrsrv.h"
+
+typedef struct _DMA_ALLOC_
+{
+	IMG_UINT64       ui64Size;
+	IMG_CPU_VIRTADDR pvVirtAddr;
+	IMG_DEV_PHYADDR  sBusAddr;
+	void             *pvOSDevice;
+} DMA_ALLOC;
+
+/*!
+*******************************************************************************
+ @Function      SysDmaAllocMem
+ @Description   Allocates physically contiguous memory
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaFreeMem
+ @Description   Free physically contiguous memory
+ @Return        void
+******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psCmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaRegisterForIoRemapping
+ @Description   Registers DMA_ALLOC for manual I/O remapping
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDeregisterForIoRemapping
+ @Description   Deregisters DMA_ALLOC from manual I/O remapping
+ @Return        void
+******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDevPAddrToCpuVAddr
+ @Description   Maps a DMA_ALLOC physical address to CPU virtual address
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+IMG_CPU_VIRTADDR
+SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaCpuVAddrToDevPAddr
+ @Description   Maps a DMA_ALLOC CPU virtual address to physical address
+ @Return        Non-zero value on success. Otherwise, a 0
+******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr);
+
+#endif /* DMA_SUPPORT_H */
+
+/******************************************************************************
+ End of file (dma_support.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysconfig.c
new file mode 100644
index 000000000000..2271aee366be
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysconfig.c
@@ -0,0 +1,582 @@
+/*************************************************************************/ /*
+########################################################################### ###
+#@File
+#@Copyright ESWIN
+#@Auther: Limei<limei@eswin.com>
+#@Date:2020-04-03
+#@History:
+#  ChenShuo 2020-08-21 adapt for zhimo-kernel
+### ###########################################################################
+
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "sysinfo.h"
+
+#include "physheap.h"
+
+#include <linux/dma-mapping.h>
+
+#include "rgxdevice.h"
+#include "interrupt_support.h"
+#include "osfunc.h"
+
+#include <linux/version.h>
+
+#include "sysinfo.h"
+#include "apollo_regs.h"
+
+
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include <linux/platform_device.h>
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/dma-map-ops.h>
+
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include "rgxdevice.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+#include "validation_soc.h"
+#endif
+
+extern unsigned int _corefreq_div;
+
+IMG_UINT64 *cpu_cache_flush_addr = NULL;
+// static int cache_reg_map = 0;
+#if 0
+extern void eswin_l2_flush64(phys_addr_t addr, size_t size);
+#else
+void eswin_l2_flush64(phys_addr_t addr, size_t size) {
+        arch_sync_dma_for_device(addr, size, DMA_TO_DEVICE);
+};
+#endif
+void riscv_invalidate_addr(phys_addr_t addr, size_t size,IMG_BOOL virtual) {
+
+ printk(KERN_ALERT "eswin:%s not implement now start 0x%x size=0x%x\n",__func__,addr,size);	
+}
+void riscv_flush_addr(IMG_UINT64 cpuaddr,IMG_UINT64 bytes_size, IMG_BOOL virtual)
+{
+    IMG_UINT64 bStart = 0;
+    IMG_UINT64 bEnd = 0;
+    // IMG_UINT64 bBase;
+    IMG_UINT64 line_size = (IMG_UINT64)cache_line_size();
+
+    if (virtual)//if cpuaddr is virtual address, here need to convert it to physical address
+        bStart =  page_to_phys(virt_to_page(cpuaddr)) & RGX_ESWIN_CPU_ADDR_MASK;
+    else
+        bStart =  cpuaddr & RGX_ESWIN_CPU_ADDR_MASK;;
+
+    bEnd = bStart + bytes_size;
+
+    bEnd = PVR_ALIGN((IMG_UINT64)bEnd, line_size);
+
+    #if 1
+    eswin_l2_flush64((phys_addr_t)bStart, (size_t)(bEnd-bStart));
+    #else
+    if (cache_reg_map == 0){
+        cache_reg_map = 1;
+	    cpu_cache_flush_addr = (IMG_UINT64 __iomem *)ioremap(RGX_ESWIN_CPU_CACHE_FLUSH_ADDR, 0x10);
+    }
+    if (cpu_cache_flush_addr == NULL){
+        printk(KERN_ALERT "NULL pointer file %s line %d\n", __FILE__, __LINE__);
+        return;
+    }
+    mb();
+    for (bBase = bStart; bBase < bEnd; bBase += line_size)
+    {
+        *(IMG_UINT64 *)cpu_cache_flush_addr = bBase;
+    }
+    mb();
+    #endif
+}
+
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+    PVRSRV_ERROR eError;
+    PVR_UNREFERENCED_PARAMETER(hSysData);
+
+    #ifndef NO_HARDWARE
+    eError = OSInstallSystemLISR(phLISRData, ui32IRQ, pszName, pfnLISR, pvData, SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);	
+    if (eError != PVRSRV_OK)
+		PVR_DPF((PVR_DBG_ERROR, "%s: install error %d != PVRSRV_OK", __func__, eError));
+    #endif
+    return eError;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return OSUninstallSystemLISR(hLISRData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+    IMG_UINT32 ui32Idx;
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+    IMG_UINT32 ui32Idx;
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+	}
+}
+
+
+static PHYS_HEAP_FUNCTIONS gsPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	UMAPhysHeapCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	UMAPhysHeapDevPAddrToCpuPAddr,
+	/* pfnGetRegionId */
+	NULL,
+};
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut,
+                                    PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 ui32NextHeapID = 0;
+	IMG_UINT32 uiHeapCount = 1;
+    
+
+	uiHeapCount += !PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig) ? 1:0;
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+    //eswin fixme: not sure if following states are okay!
+	pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+	ui32NextHeapID++;
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+        //eswin fixme: not sure if following states are okay!
+        pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+        pasPhysHeaps[ui32NextHeapID].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+		ui32NextHeapID++;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+        int ret;
+
+        printk(KERN_WARNING "eswin zdbg1107 %s\n", __func__);
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+
+	/* eswin, assert the reset */
+	if (psDevConfig->rstc_spu)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_spu);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_spu);
+        }
+	if (psDevConfig->rstc_jones)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_jones);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_jones);
+        }
+	if (psDevConfig->rstc_gray)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_gray);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_gray);
+        }
+	if (psDevConfig->rstc_cfg)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_cfg);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_cfg);
+        }
+	if (psDevConfig->rstc_axi)
+        {
+		ret = reset_control_assert(psDevConfig->rstc_axi);
+                WARN_ON(0 != ret);
+                reset_control_put(psDevConfig->rstc_axi);
+        }
+
+	/* eswin, close the clk */
+        if (psDevConfig->aclk)
+        {
+		clk_disable_unprepare(psDevConfig->aclk);
+		clk_put(psDevConfig->aclk);
+        }
+        if (psDevConfig->cfg_clk)
+        {
+            clk_disable_unprepare(psDevConfig->cfg_clk);
+	    clk_put(psDevConfig->cfg_clk);
+        }
+        if (psDevConfig->gray_clk)
+        {
+            clk_disable_unprepare(psDevConfig->gray_clk);
+	    clk_put(psDevConfig->gray_clk);
+        }
+
+	OSFreeMem(psDevConfig);
+}
+
+void riscv_flush_cache_range(IMG_HANDLE hSysData,
+                                        PVRSRV_CACHE_OP eRequestType,
+                                        void *pvVirtStart,
+                                        void *pvVirtEnd,
+                                        IMG_CPU_PHYADDR sCPUPhysStart,
+                                        IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	IMG_UINT64 bStart = sCPUPhysStart.uiAddr;
+	IMG_UINT64 bEnd = sCPUPhysEnd.uiAddr;
+
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	// PVR_UNREFERENCED_PARAMETER(eRequestType);
+	PVR_UNREFERENCED_PARAMETER(pvVirtStart);
+	PVR_UNREFERENCED_PARAMETER(pvVirtEnd);
+	switch (eRequestType)
+	{
+	case PVRSRV_CACHE_OP_FLUSH:
+		riscv_flush_addr(bStart, (bEnd - bStart), IMG_FALSE);
+		break;
+	case PVRSRV_CACHE_OP_INVALIDATE:
+		riscv_flush_addr(bStart, (bEnd - bStart), IMG_FALSE);
+		//printk(KERN_ALERT "%s: ---eswin----warning-----PVRSRV_CACHE_OP_INVALIDATE not implement now ------- \n", __func__);
+		break;
+	case PVRSRV_CACHE_OP_CLEAN:
+		riscv_flush_addr(bStart, (bEnd - bStart), IMG_FALSE);
+		break;
+	default:
+		printk(KERN_ALERT "%s: unhandled eRequestType val=0x%x \n", __func__, eRequestType);
+	}
+}
+
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+#ifndef NO_HARDWARE
+	IMG_UINT32 rgx_freq=0;
+	struct resource res;
+	struct device_node *np=NULL;
+	int ret;
+	struct platform_device *pdev = to_platform_device((struct device *)pvOSDevice);
+	struct device *dev = (struct device *)pvOSDevice;
+#endif //CONFIG_SPARSE_IRQ 
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+    psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount, psDevConfig);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+#ifndef NO_HARDWARE
+	np = dev->of_node;
+
+	if (!np)
+	{
+		pr_err("%s:%d can not find node img,gpu\n", __func__, __LINE__);
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto ErrorFreeDevConfig;
+	}
+	// gpu clk enable
+	psDevConfig->aclk = of_clk_get_by_name(np, "aclk");
+	if (IS_ERR(psDevConfig->aclk))
+	{
+		ret = PTR_ERR(psDevConfig->aclk);
+		dev_err(&pdev->dev, "failed to get aclk: %d\n", ret);
+		return ret;
+	}
+
+	// get gpu clk, it is supposed to be 800MHz
+	rgx_freq = clk_get_rate(psDevConfig->aclk);
+	printk(KERN_ALERT "%s:%d, zdbg1107 eswin print : read back aclk  %dHZ \n", __func__, __LINE__, rgx_freq);
+
+	ret = clk_prepare_enable(psDevConfig->aclk);
+	if (ret)
+	{
+		dev_err(&pdev->dev, "failed to enable aclk: %d\n", ret);
+		return ret;
+	}
+
+	psDevConfig->cfg_clk = of_clk_get_by_name(np, "cfg_clk");
+	if (IS_ERR(psDevConfig->cfg_clk))
+	{
+		ret = PTR_ERR(psDevConfig->cfg_clk);
+		dev_err(&pdev->dev, "failed to get cfg_clk: %d\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(psDevConfig->cfg_clk);
+	if (ret)
+	{
+		dev_err(&pdev->dev, "failed to enable cfg_clk: %d\n", ret);
+		return ret;
+	}
+
+	psDevConfig->gray_clk = of_clk_get_by_name(np, "gray_clk");
+	if (IS_ERR(psDevConfig->gray_clk))
+	{
+		ret = PTR_ERR(psDevConfig->gray_clk);
+		dev_err(&pdev->dev, "failed to get gray_clk: %d\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(psDevConfig->gray_clk);
+	if (ret)
+	{
+		dev_err(&pdev->dev, "failed to enable gray_clk: %d\n", ret);
+		return ret;
+	}
+	clk_disable_unprepare(psDevConfig->aclk);
+	clk_disable_unprepare(psDevConfig->cfg_clk);
+	clk_disable_unprepare(psDevConfig->gray_clk);
+
+	if (_corefreq_div > 800000000)
+		_corefreq_div = 800000000;
+
+	rgx_freq= clk_round_rate(psDevConfig->aclk, _corefreq_div);//24M -> 800M
+	if (rgx_freq > 0) {
+		ret = clk_set_rate(psDevConfig->aclk, rgx_freq);
+		if (ret) {
+			dev_err(&pdev->dev, "failed to set aclk: %d\n", ret);
+			return ret;
+		}
+	}
+	rgx_freq = clk_get_rate(psDevConfig->aclk);
+	psRGXTimingInfo->ui32CoreClockSpeed = rgx_freq;
+
+	clk_prepare_enable(psDevConfig->aclk);
+	clk_prepare_enable(psDevConfig->cfg_clk);
+	clk_prepare_enable(psDevConfig->gray_clk);
+
+
+	// reset gpu
+	psDevConfig->rstc_axi = of_reset_control_get_optional_exclusive(np, "axi");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_axi))
+	{
+		dev_err(&pdev->dev, "Failed to get axi reset handle\n");
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_axi);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_cfg = of_reset_control_get_optional_exclusive(np, "cfg");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_cfg))
+	{
+		dev_err(&pdev->dev, "Failed to get cfg reset handle\n");
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_cfg);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_gray = of_reset_control_get_optional_exclusive(np, "gray");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_gray))
+	{
+		dev_err(&pdev->dev, "Failed to get gray reset handle\n");
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_gray);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_jones = of_reset_control_get_optional_exclusive(np, "jones");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_jones))
+	{
+		dev_err(&pdev->dev, "Failed to get jones reset handle\n");
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_jones);
+	WARN_ON(0 != ret);
+
+	psDevConfig->rstc_spu = of_reset_control_get_optional_exclusive(np, "spu");
+	if (IS_ERR_OR_NULL(psDevConfig->rstc_spu))
+	{
+		dev_err(&pdev->dev, "Failed to get spu reset handle\n");
+		return -EFAULT;
+	}
+	ret = reset_control_reset(psDevConfig->rstc_spu);
+	WARN_ON(0 != ret);
+#else
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+#endif //CONFIG_SPARSE_IRQ
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszVersion             = NULL;
+#ifndef NO_HARDWARE
+	psDevConfig->pszName = pdev->name;
+	printk(KERN_ALERT "%s: --------------->dev_name=%s\n", __func__, psDevConfig->pszName);
+
+	if (of_address_to_resource(np, 0, &res))
+	{
+		printk(KERN_ALERT "%s: failed to get resource of rgx register\n", __func__);
+		eError = PVRSRV_ERROR_INVALID_MEMINFO;
+		goto ErrorFreeDevConfig;
+	}
+	psDevConfig->sRegsCpuPBase.uiAddr = res.start;
+	psDevConfig->ui32RegsSize = resource_size(&res);
+	pr_err("%s: --------------->reg_base=%llx, reg_size=%llx\n", __func__, res.start, resource_size(&res));
+
+	psDevConfig->ui32IRQ = irq_of_parse_and_map(np, 0);
+	pr_err("%s: --------------->ui32IRQ=%d\n", __func__, psDevConfig->ui32IRQ);
+	if (!psDevConfig->ui32IRQ)
+	{
+		printk(KERN_ALERT "%s: failed to map GPU interrupt\n", __func__);
+		eError = PVRSRV_ERROR_MAPPING_NOT_FOUND;
+		goto ErrorFreeDevConfig;
+	}
+#else
+	/* Device setup information */
+	psDevConfig->pszName                = SYS_RGX_DEV_NAME;
+	psDevConfig->sRegsCpuPBase.uiAddr   = RGX_ESWIN_GPU_REG_BASE;
+	psDevConfig->ui32RegsSize           = RGX_ESWIN_GPU_REG_SIZE;
+	psDevConfig->ui32IRQ                = RGX_ESWIN_IRQ_ID;
+#endif
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+    //eswin fixme: DRIVER_MODE_NATIVE okay?
+    psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+	if (! PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		/* Virtualization support services needs to know which heap ID corresponds to FW */
+		// psDevConfig->aui32PhysHeapID[PVRSRV_DEVICE_PHYS_HEAP_FW_LOCAL] = PHYS_HEAP_IDX_VIRTFW;
+	}
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+	psDevConfig->hSysData               = NULL;
+	psDevConfig->bDevicePA0IsValid       = IMG_FALSE;
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	/* Pdump validation system registers */
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+	PVRSRVConfigureSysCtrl(NULL, PDUMP_FLAGS_CONTINUOUS);
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	PVRSRVConfigureTrustedDevice(NULL, PDUMP_FLAGS_CONTINUOUS);
+#endif
+#endif
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_ERROR eError;
+    dma_set_mask(pvOSDevice, DMA_BIT_MASK(40)); //?why is '40', maybe need ask the hw engineer
+
+    eError = DeviceConfigCreate(pvOSDevice, ppsDevConfig);
+
+    (*ppsDevConfig)->pfnHostCacheMaintenance = riscv_flush_cache_range;
+    (*ppsDevConfig)->bHasPhysicalCacheMaintenance = OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+	return eError;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysinfo.h
new file mode 100644
index 000000000000..ec76a48bb4bc
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/eswin_cpu/sysinfo.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+ ########################################################################### ###
+ #@File
+ #@Copyright ESWIN
+ #@Auther: Limei<limei@eswin.com>
+ #@Date:2020-04-03
+ #@History:
+ #  ChenShuo 2020-08-21 adapt for zhimo-kernel
+ ### ###########################################################################
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+#if 0
+#include "config_kernel.h"
+#endif
+//#define RGX_ESWIN_PROJECT_V91
+
+#define SYS_RGX_DEV_NAME    "pvrsrvkm"
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#define RGX_ESWIN_DISP_NUM_BUFS 2 //pay attention to this val
+
+#ifndef RGX_ESWIN_PROJECT_U84
+#define RGX_ESWIN_PROJECT_U84
+#endif
+
+#ifdef RGX_ESWIN_PROJECT_U84
+    #define RGX_ESWIN_CPU_ADDR_W        41/*this is based on hw, Notes:eswin support 41bits width addrees, 3D-GPU support 40bits(1TB)*/
+    #define RGX_ESWIN_CPU_ADDR_MASK     ((1UL<<RGX_ESWIN_CPU_ADDR_W)-1)/*this is based on hw*/
+    #define RGX_ESWIN_IRQ_ID           14 
+    /* display infor resolution and rate */
+    #define RGX_ESWIN_DISP_DEF_W        1280
+    #define RGX_ESWIN_DISP_DEF_H        720
+    #define RGX_ESWIN_DISP_DEF_R        60
+    /* ddr address from the view of CPU*/
+    #define RGX_ESWIN_MEM_BASE_CPU          (0x80000000UL) //0x8,000,0000, ddr mem address from CPU view
+
+    #ifdef  RGX_ESWIN_DISP_OFFSCREEN    /* fix address used by disp/vby1 for offscreen test */
+    #define RGX_ESWIN_DISP_MEM_START    (0x81B00000UL) //(0x9B000000)
+    #define RGX_ESWIN_MEM_SIZE          (RGX_ESWIN_DISP_MEM_START - RGX_ESWIN_MEM_BASE_CPU)
+    #else
+    #define RGX_ESWIN_MEM_SIZE          (0x20000000UL)
+    #endif
+    /* the cache flush reg address of CPU */
+    #define RGX_ESWIN_CPU_CACHE_FLUSH_ADDR  0x02010200
+    /* GPU reg address and size*/
+    #ifndef NO_HARDWARE
+	#define RGX_ESWIN_GPU_REG_BASE      0x51400000 //0x51400000 on haps
+    #else
+    #define RGX_ESWIN_GPU_REG_BASE      0x21400000 
+    #endif
+    #define RGX_ESWIN_GPU_REG_SIZE      0xFFFFF
+    /* GPU clk */
+    #define RGX_NOHW_CORE_CLOCK_SPEED   5000000//850000000// 850MHz GPU clk //100000000 //e.g. 100 MHz
+
+    #ifdef LMA  /* mem address offset from the view of GPU */
+        #define RGX_ESWIN_MEM_BASE_GPU   0x80000000UL //0x4,0000,0000
+    #endif
+#else
+    #error "please update the proper Project infor"
+    #define RGX_ESWIN_DISP_DEF_W        4096
+    #define RGX_ESWIN_DISP_DEF_H        2160
+    #define RGX_NOHW_CORE_CLOCK_SPEED   100000000 //e.g. 100 MHz
+#endif
+
+#define RGX_ESWIN_DISP_DEF_TOTAL_SIZE   (RGX_ESWIN_DISP_DEF_W*RGX_ESWIN_DISP_DEF_H*4*RGX_ESWIN_DISP_NUM_BUFS)
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (240000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (120000) 
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#define SYS_RGX_OF_COMPATIBLE "img,gpu"
+
+#endif	/* !defined(__SYSINFO_H__) */
+
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/interrupt_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/interrupt_support.h
new file mode 100644
index 000000000000..b87772d223cb
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/interrupt_support.h
@@ -0,0 +1,103 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(INTERRUPT_SUPPORT_H)
+#define INTERRUPT_SUPPORT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_device.h"
+
+/*! Default trigger type for the interrupt line. */
+#define SYS_IRQ_FLAG_TRIGGER_DEFAULT (0x0 << 0)
+/*! Interrupt triggered when interrupt line is low. */
+#define SYS_IRQ_FLAG_TRIGGER_LOW     (0x1 << 0)
+/*! Interrupt triggered when interrupt line is high. */
+#define SYS_IRQ_FLAG_TRIGGER_HIGH    (0x2 << 0)
+/*! Interrupt trigger mask. */
+#define SYS_IRQ_FLAG_TRIGGER_MASK    (SYS_IRQ_FLAG_TRIGGER_DEFAULT | \
+                                      SYS_IRQ_FLAG_TRIGGER_LOW | \
+                                      SYS_IRQ_FLAG_TRIGGER_HIGH)
+/*! The irq is allowed to be shared among several devices. */
+#define SYS_IRQ_FLAG_SHARED          (0x1 << 8)
+
+/*! Interrupt flags mask. */
+#define SYS_IRQ_FLAG_MASK            (SYS_IRQ_FLAG_TRIGGER_MASK | \
+                                      SYS_IRQ_FLAG_SHARED)
+
+/*************************************************************************/ /*!
+@Description    Pointer to a system Low-level Interrupt Service Routine (LISR).
+@Input  pvData  Private data provided to the LISR.
+@Return         IMG_TRUE if interrupt handled, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_SYS_LISR)(void *pvData);
+
+/*************************************************************************/ /*!
+@Function       OSInstallSystemLISR
+@Description    Installs a system low-level interrupt handler
+@Output         phLISR                  On return, contains a handle to the
+                                        installed LISR
+@Input          ui32IRQ                 The IRQ number for which the
+                                        interrupt handler should be installed
+@Input          pszDevName              Name of the device for which the handler
+                                        is being installed
+@Input          pfnLISR                 A pointer to an interrupt handler
+                                        function
+@Input          pvData                  A pointer to data that should be passed
+                                        to pfnLISR when it is called
+@Input          ui32Flags               Interrupt flags
+@Return         PVRSRV_OK on success, a failure code otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInstallSystemLISR(IMG_HANDLE *phLISR,
+				 IMG_UINT32 ui32IRQ,
+				 const IMG_CHAR *pszDevName,
+				 PFN_SYS_LISR pfnLISR,
+				 void *pvData,
+				 IMG_UINT32 ui32Flags);
+
+/*************************************************************************/ /*!
+@Function       OSUninstallSystemLISR
+@Description    Uninstalls a system low-level interrupt handler
+@Input          hLISRData              The handle to the LISR to uninstall
+@Return         PVRSRV_OK on success, a failure code otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR OSUninstallSystemLISR(IMG_HANDLE hLISRData);
+#endif /* !defined(INTERRUPT_SUPPORT_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/pci_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/pci_support.h
new file mode 100644
index 000000000000..29746c672047
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/pci_support.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PCI_SUPPORT_H
+#define PCI_SUPPORT_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(__linux__)
+#include <linux/pci.h>
+#define TO_PCI_COOKIE(dev) to_pci_dev((struct device *)(dev))
+#else
+#define TO_PCI_COOKIE(dev) (dev)
+#endif
+
+typedef enum _HOST_PCI_INIT_FLAGS_
+{
+	HOST_PCI_INIT_FLAG_BUS_MASTER	= 0x00000001,
+	HOST_PCI_INIT_FLAG_MSI		= 0x00000002,
+	HOST_PCI_INIT_FLAG_FORCE_I32	= 0x7fffffff
+} HOST_PCI_INIT_FLAGS;
+
+struct _PVRSRV_PCI_DEV_OPAQUE_STRUCT_;
+typedef struct _PVRSRV_PCI_DEV_OPAQUE_STRUCT_ *PVRSRV_PCI_DEV_HANDLE;
+
+PVRSRV_PCI_DEV_HANDLE OSPCIAcquireDev(IMG_UINT16 ui16VendorID, IMG_UINT16 ui16DeviceID, HOST_PCI_INIT_FLAGS eFlags);
+PVRSRV_PCI_DEV_HANDLE OSPCISetDev(void *pvPCICookie, HOST_PCI_INIT_FLAGS eFlags);
+PVRSRV_ERROR OSPCIReleaseDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIIRQ(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 *pui32IRQ);
+IMG_UINT64 OSPCIAddrRangeLen(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+IMG_UINT64 OSPCIAddrRangeStart(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+IMG_UINT64 OSPCIAddrRangeEnd(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIRequestAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIReleaseAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIRequestAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index, IMG_UINT64 uiOffset, IMG_UINT64 uiLength);
+PVRSRV_ERROR OSPCIReleaseAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index, IMG_UINT64 uiOffset, IMG_UINT64 uiLength);
+PVRSRV_ERROR OSPCISuspendDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIResumeDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIGetVendorDeviceIDs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT16 *pui16VendorID, IMG_UINT16 *pui16DeviceID);
+
+#if defined(CONFIG_MTRR)
+PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+#else
+static inline PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRPCI);
+	PVR_UNREFERENCED_PARAMETER(ui32Index);
+	return PVRSRV_OK;
+}
+
+static inline void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRPCI);
+	PVR_UNREFERENCED_PARAMETER(ui32Index);
+}
+#endif
+
+#endif /* PCI_SUPPORT_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/syscommon.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/syscommon.h
new file mode 100644
index 000000000000..b7babda19cab
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/syscommon.h
@@ -0,0 +1,175 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common System APIs and structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides common system-specific declarations and
+                macros that are supported by all systems
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(SYSCOMMON_H)
+#define SYSCOMMON_H
+
+#include "img_types.h"
+#include "pvr_notifier.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_error.h"
+
+/*************************************************************************/ /*!
+@Description    Pointer to a Low-level Interrupt Service Routine (LISR).
+@Input  pvData  Private data provided to the LISR.
+@Return         True if interrupt handled, false otherwise.
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_LISR)(void *pvData);
+
+/**************************************************************************/ /*!
+@Function       SysDevInit
+@Description    System specific device initialisation function.
+@Input          pvOSDevice          pointer to the OS device reference
+@Input          ppsDevConfig        returned device configuration info
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig);
+
+/**************************************************************************/ /*!
+@Function       SysDevDeInit
+@Description    System specific device deinitialisation function.
+@Input          psDevConfig        device configuration info of the device to be
+                                   deinitialised
+@Return         None.
+*/ /***************************************************************************/
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/**************************************************************************/ /*!
+@Function       SysDebugInfo
+@Description    Dump system specific device debug information.
+@Input          psDevConfig         pointer to device configuration info
+@Input          pfnDumpDebugPrintf  the 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     optional file identifier to be passed to
+                                    the 'printf' function if required
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile);
+
+/**************************************************************************/ /*!
+@Function       SysInstallDeviceLISR
+@Description    Installs the system Low-level Interrupt Service Routine (LISR)
+                which handles low-level processing of interrupts from the device
+                (GPU).
+                The LISR will be invoked when the device raises an interrupt. An
+                LISR may not be descheduled, so code which needs to do so should
+                be placed in an MISR.
+                The installed LISR will schedule any MISRs once it has completed
+                its interrupt processing, by calling OSScheduleMISR().
+@Input          hSysData      pointer to the system data of the device
+@Input          ui32IRQ       the IRQ on which the LISR is to be installed
+@Input          pszName       name of the module installing the LISR
+@Input          pfnLISR       pointer to the function to be installed as the
+                              LISR
+@Input          pvData        private data provided to the LISR
+@Output         phLISRData    handle to the installed LISR (to be used for a
+                              subsequent uninstall)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData);
+
+/**************************************************************************/ /*!
+@Function       SysUninstallDeviceLISR
+@Description    Uninstalls the system Low-level Interrupt Service Routine (LISR)
+                which handles low-level processing of interrupts from the device
+                (GPU).
+@Input          hLISRData     handle of the LISR to be uninstalled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData);
+
+/**************************************************************************/ /*!
+@Function       SysRGXErrorNotify
+@Description    Error reporting callback function, registered as the
+                pfnSysDevErrorNotify member of the PVRSRV_DEVICE_CONFIG
+                struct. System layer will be notified of device errors and
+                resets via this callback.
+                NB. implementers should ensure that the minimal amount of
+                work is done in this callback function, as it will be
+                executed in the main RGX MISR. (e.g. any blocking or lengthy
+                work should be performed by a worker queue/thread instead).
+@Input          hSysData      pointer to the system data of the device
+@Output         psErrorData   structure containing details of the reported error
+@Return         None.
+*/ /***************************************************************************/
+void SysRGXErrorNotify(IMG_HANDLE hSysData,
+                       PVRSRV_ROBUSTNESS_NOTIFY_DATA *psErrorData);
+
+/**************************************************************************/ /*!
+@Function       SysRestrictGpuLocalPhysheap
+@Description    If the Restriction apphint has been set, validate the
+                restriction value and return the new GPU_LOCAL heap size.
+
+@Input          uiHeapSize      Current syslayer detected GPU_LOCAL heap size.
+@Return         IMG_UINT64      New GPU_LOCAL heap size in bytes.
+*/ /***************************************************************************/
+IMG_UINT64 SysRestrictGpuLocalPhysheap(IMG_UINT64 uiHeapSize);
+
+/**************************************************************************/ /*!
+@Function       SysRestrictGpuLocalAddPrivateHeap
+@Description    Determine if the restriction apphint has been set.
+
+@Return         IMG_BOOL        IMG_TRUE if the restriction apphint has been
+                                set.
+*/ /***************************************************************************/
+IMG_BOOL SysRestrictGpuLocalAddPrivateHeap(void);
+
+/**************************************************************************/ /*!
+@Function       SysDefaultToCpuLocalHeap
+@Description    Determine if the Default Heap should be CPU_LOCAL
+                Can only be used on TC_MEMORY_HYBRID systems.
+
+@Return         IMG_BOOL        IMG_TRUE if the Default heap apphint has been
+                                set.
+*/ /***************************************************************************/
+IMG_BOOL SysDefaultToCpuLocalHeap(void);
+
+#endif /* !defined(SYSCOMMON_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/sysvalidation.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/sysvalidation.h
new file mode 100644
index 000000000000..e3e731e8b810
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/sysvalidation.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Validation System APIs and structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+                needed for hardware validation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSVALIDATION_H)
+#define SYSVALIDATION_H
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "img_types.h"
+#include "rgxdefs_km.h"
+#include "virt_validation_defs.h"
+
+void SysInitFirewall(IMG_HANDLE hSysData,
+                     IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                     IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+
+#if defined(EMULATOR)
+void SysSetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState);
+void SysSetTrustedDeviceAceEnabled(IMG_HANDLE hSysData);
+#endif
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+#endif /* !defined(SYSVALIDATION_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/include/uma_heap_fns.h b/drivers/gpu/drm/img/img-volcanic/services/system/include/uma_heap_fns.h
new file mode 100644
index 000000000000..bfef1e520880
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/include/uma_heap_fns.h
@@ -0,0 +1,53 @@
+/*****************************************************************************
+@File
+@Title          Boilerplate heap functions for UMA systems
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares generic nop conversions for physical addresses in UMA
+                systems.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef UMA_HEAP_FNS_H
+#define UMA_HEAP_FNS_H
+
+#include "physheap_config.h"
+
+extern PHYS_HEAP_FUNCTIONS g_sUmaHeapFns;
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/plato/Kbuild.mk
new file mode 100644
index 000000000000..84b92c971b0a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/Kbuild.mk
@@ -0,0 +1,53 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += \
+	services/system/$(PVR_SYSTEM)/sysconfig.o \
+	services/server/common/vmm_pvz_client.o \
+	services/server/common/vmm_pvz_server.o \
+	services/server/common/vz_vmm_pvz.o \
+	services/server/common/vz_vmm_vm.o \
+	services/system/rogue/common/vmm_type_$(VMM_TYPE).o
+
+ccflags-y += \
+	-I$(TOP)/services/system/rogue/common/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec/plato
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/plato.h b/drivers/gpu/drm/img/img-volcanic/services/system/plato/plato.h
new file mode 100644
index 000000000000..1aaf14d94c62
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/plato.h
@@ -0,0 +1,337 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PLATO_H)
+#define PLATO_H
+
+#define PLATO_SYSTEM_NAME			"Plato"
+
+/* Valid values for the PLATO_MEMORY_CONFIG configuration option */
+#define PLATO_MEMORY_LOCAL			(1)
+#define PLATO_MEMORY_HOST			(2)
+#define PLATO_MEMORY_HYBRID			(3)
+
+/* Interrupt defines */
+typedef enum _PLATO_IRQ_
+{
+    PLATO_IRQ_GPU = 0,
+    PLATO_IRQ_PDP,
+    PLATO_IRQ_HDMI,
+    PLATO_IRQ_MAX,
+}PLATO_IRQ;
+
+#define PLATO_GPU_INT_SHIFT				(0)
+#define PLATO_PDP_INT_SHIFT				(8)
+#define PLATO_HDMI_INT_SHIFT			(9)
+#define PLATO_HDMI_WAKEUP_INT_SHIFT		(11)
+#define PLATO_TEMP_A_INT_SHIFT			(12)
+
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#define PHYS_HEAP_ID_CPU_LOCAL 0
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_CPU_LOCAL 1
+#define PHYS_HEAP_ID_PDP_LOCAL 2
+#define PHYS_HEAP_ID_NON_MAPPABLE 3
+#define PHYS_HEAP_ID_FW_LOCAL 4
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_PDP_LOCAL 1
+#define PHYS_HEAP_ID_NON_MAPPABLE 2
+#endif
+
+#define DCPDP_PHYS_HEAP_ID PHYS_HEAP_ID_PDP_LOCAL
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+#define RGX_PLATO_RESERVE_DC_MEM_SIZE (PLATO_DC_MEM_SIZE_MB * 1024 * 1024)
+#endif
+
+#define SYS_PLATO_REG_DMA_BASENUM	(0)
+#define SYS_PLATO_REG_DMA_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_PCI_BASENUM	(1)
+#define SYS_PLATO_REG_REGION_SIZE	(4 * 1024 * 1024)
+
+#define SYS_PLATO_SRAM_BASENUM		(2)
+#define SYS_PLATO_SRAM_SIZE			(2 * 1024 * 1024)
+/*
+ * Give system region a whole span of the reg space including
+ * RGX registers. That's because there are sys register segments
+ * both before and after the RGX segment.
+ */
+#define SYS_PLATO_REG_SYS_OFFSET			(0x0)
+#define SYS_PLATO_REG_SYS_SIZE				(4 * 1024 * 1024)
+
+/* Entire Peripheral region */
+#define SYS_PLATO_REG_PERIP_OFFSET			(0x20000)
+#define SYS_PLATO_REG_PERIP_SIZE			(164 * 1024)
+
+/* Chip level registers */
+#define SYS_PLATO_REG_CHIP_LEVEL_OFFSET		(SYS_PLATO_REG_PERIP_OFFSET)
+#define SYS_PLATO_REG_CHIP_LEVEL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_TEMPA_OFFSET			(0x80000)
+#define SYS_PLATO_REG_TEMPA_SIZE			(64 * 1024)
+
+/* USB not included */
+
+#define SYS_PLATO_REG_DDR_A_CTRL_OFFSET		(0x120000)
+#define SYS_PLATO_REG_DDR_A_CTRL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_B_CTRL_OFFSET		(0x130000)
+#define SYS_PLATO_REG_DDR_B_CTRL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_A_PUBL_OFFSET		(0x140000)
+#define SYS_PLATO_REG_DDR_A_PUBL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_B_PUBL_OFFSET		(0x150000)
+#define SYS_PLATO_REG_DDR_B_PUBL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_NOC_OFFSET			(0x160000)
+#define SYS_PLATO_REG_NOC_SIZE				(64 * 1024)
+
+/* Debug NOC registers */
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_CTRL_OFFSET (0x1500)
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_DATA_OFFSET (0x1580)
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_PUBL_OFFSET (0x1600)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_CTRL_OFFSET (0x1680)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_DATA_OFFSET (0x1700)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_PUBL_OFFSET (0x1780)
+#define SYS_PLATO_REG_NOC_DBG_DISPLAY_S_OFFSET  (0x1800)
+#define SYS_PLATO_REG_NOC_DBG_GPIO_0_S_OFFSET   (0x1900)
+#define SYS_PLATO_REG_NOC_DBG_GPIO_1_S_OFFSET   (0x1980)
+#define SYS_PLATO_REG_NOC_DBG_GPU_S_OFFSET      (0x1A00)
+#define SYS_PLATO_REG_NOC_DBG_PCI_PHY_OFFSET    (0x1A80)
+#define SYS_PLATO_REG_NOC_DBG_PCI_REG_OFFSET    (0x1B00)
+#define SYS_PLATO_REG_NOC_DBG_PCI_S_OFFSET      (0x1B80)
+#define SYS_PLATO_REG_NOC_DBG_PERIPH_S_OFFSET   (0x1c00)
+#define SYS_PLATO_REG_NOC_DBG_RET_REG_OFFSET    (0x1D00)
+#define SYS_PLATO_REG_NOC_DBG_SERVICE_OFFSET    (0x1E00)
+
+#define SYS_PLATO_REG_RGX_OFFSET			(0x170000)
+#define SYS_PLATO_REG_RGX_SIZE				(64 * 1024)
+
+#define SYS_PLATO_REG_AON_OFFSET			(0x180000)
+#define SYS_PLATO_REG_AON_SIZE				(64 * 1024)
+
+#define SYS_PLATO_REG_PDP_OFFSET			(0x200000)
+#define SYS_PLATO_REG_PDP_SIZE				(128 * 1024)
+
+#define SYS_PLATO_REG_PDP_BIF_OFFSET        (SYS_PLATO_REG_PDP_OFFSET + SYS_PLATO_REG_PDP_SIZE)
+#define SYS_PLATO_REG_PDP_BIF_SIZE          (0x200)
+
+#define SYS_PLATO_REG_HDMI_OFFSET           (SYS_PLATO_REG_PDP_OFFSET + 0x20000)
+#define SYS_PLATO_REG_HDMI_SIZE             (128 * 1024)
+
+/* Device memory (including HP mapping) on base register 4 */
+#define SYS_DEV_MEM_PCI_BASENUM		(4)
+
+/* Device memory size */
+#define ONE_GB_IN_BYTES				(0x40000000ULL)
+#define SYS_DEV_MEM_REGION_SIZE		(PLATO_MEMORY_SIZE_GIGABYTES * ONE_GB_IN_BYTES)
+
+/* Plato DDR offset in device memory map at 32GB */
+#define PLATO_DDR_DEV_PHYSICAL_BASE	(0x800000000)
+
+/* DRAM is split at 48GB */
+#define PLATO_DRAM_SPLIT_ADDR (0xc00000000)
+
+/* Plato DDR region is aliased if less than 32GB memory is present.
+ * This defines memory base closest to the DRAM split point.
+ * If 32GB is present this is equal to PLATO_DDR_DEV_PHYSICAL_BASE */
+#define PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE	(PLATO_DRAM_SPLIT_ADDR - (SYS_DEV_MEM_REGION_SIZE >> 1))
+
+#define PLATO_DDR_ALIASED_DEV_PHYSICAL_END	(PLATO_DRAM_SPLIT_ADDR + (SYS_DEV_MEM_REGION_SIZE >> 1))
+
+#define PLATO_DDR_ALIASED_DEV_SEGMENT_SIZE	((32ULL / PLATO_MEMORY_SIZE_GIGABYTES) * ONE_GB_IN_BYTES)
+
+/* Plato Host memory offset in device memory map at 512GB */
+#define PLATO_HOSTRAM_DEV_PHYSICAL_BASE (0x8000000000)
+
+/* Plato PLL, DDR/GPU and PDP clocks */
+#define PLATO_PLL_REF_CLOCK_SPEED		(19200000)
+
+/* 600 MHz */
+#define PLATO_MEM_CLOCK_SPEED			(600000000)
+#define PLATO_MIN_MEM_CLOCK_SPEED		(600000000)
+#define PLATO_MAX_MEM_CLOCK_SPEED		(800000000)
+
+/* 396 MHz (~400 MHz) on HW, around 1MHz on the emulator */
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+#define	PLATO_RGX_CORE_CLOCK_SPEED		(1000000)
+#define	PLATO_RGX_MIN_CORE_CLOCK_SPEED		(1000000)
+#define	PLATO_RGX_MAX_CORE_CLOCK_SPEED		(1000000)
+#else
+
+#define	PLATO_RGX_CORE_CLOCK_SPEED		(396000000)
+#define	PLATO_RGX_MIN_CORE_CLOCK_SPEED	(396000000)
+#define	PLATO_RGX_MAX_CORE_CLOCK_SPEED	(742500000)
+#endif
+
+#define PLATO_MIN_PDP_CLOCK_SPEED				(165000000)
+#define PLATO_TARGET_HDMI_SFR_CLOCK_SPEED		(27000000)
+#define PLATO_TARGET_HDMI_CEC_CLOCK_SPEED		(32768)
+
+#define REG_TO_CELSIUS(reg)			(((reg) * 352/4096) - 109)
+#define CELSIUS_TO_REG(temp)		((((temp) + 109) * 4096) / 352)
+#define PLATO_MAX_TEMP_CELSIUS		(100)
+
+#define PLATO_LMA_HEAP_REGION_MAPPABLE			0
+#define PLATO_LMA_HEAP_REGION_NONMAPPABLE		1
+
+#if defined(ENABLE_PLATO_HDMI)
+
+#if defined(HDMI_PDUMP)
+/* Hard coded video formats for pdump type run only */
+#define VIDEO_FORMAT_1280_720p          0
+#define VIDEO_FORMAT_1920_1080p         1
+#define DC_DEFAULT_VIDEO_FORMAT     (VIDEO_FORMAT_1920_1080p)
+#endif
+
+#define DEFAULT_BORDER_WIDTH_PIXELS     (0)
+#define HDMI_DTD_MAX                    (20) // Reference driver has 37...seems like a lot
+
+typedef enum
+{
+	ENCODING_RGB = 0,
+	ENCODING_YCC444,
+	ENCODING_YCC422,
+	ENCODING_YCC420
+} ENCODING_T;
+
+typedef enum
+{
+	ITU601 = 1,
+	ITU709,
+	EXTENDED_COLORIMETRY
+} COLORIMETRY_T;
+
+typedef struct
+{
+	/** VIC code */
+	IMG_UINT32 mCode;
+	/** Identifies modes that ONLY can be displayed in YCC 4:2:0 */
+	IMG_UINT8 mLimitedToYcc420;
+	/** Identifies modes that can also be displayed in YCC 4:2:0 */
+	IMG_UINT8 mYcc420;
+	IMG_UINT16 mPixelRepetitionInput;
+	/** in units of 10KHz */
+	IMG_UINT16 mPixelClock;
+	/** 1 for interlaced, 0 progressive */
+	IMG_UINT8 mInterlaced;
+	IMG_UINT16 mHActive;
+	IMG_UINT16 mHBlanking;
+	IMG_UINT16 mHBorder;
+	IMG_UINT16 mHImageSize;
+	IMG_UINT16 mHFrontPorchWidth;
+	IMG_UINT16 mHBackPorchWidth;
+	IMG_UINT16 mHSyncPulseWidth;
+	/** 0 for Active low, 1 active high */
+	IMG_UINT8 mHSyncPolarity;
+	IMG_UINT16 mVActive;
+	IMG_UINT16 mVBlanking;
+	IMG_UINT16 mVBorder;
+	IMG_UINT16 mVImageSize;
+	IMG_UINT16 mVFrontPorchWidth;
+	IMG_UINT16 mVBackPorchWidth;
+	IMG_UINT16 mVSyncPulseWidth;
+	/** 0 for Active low, 1 active high */
+	IMG_UINT8 mVSyncPolarity;
+
+    IMG_UINT8 mValid;
+} DTD;
+
+/* Monitor range limits comes from Display Descriptor (EDID) */
+typedef struct _MONITOR_RANGE_LIMITS
+{
+	IMG_UINT8 vMinRate;
+	IMG_UINT8 vMaxRate;
+	IMG_UINT8 hMinRate;
+	IMG_UINT8 hMaxRate;
+	IMG_UINT8 maxPixelClock;
+	IMG_UINT8 valid;
+} MONITOR_RANGE_LIMITS;
+
+typedef struct
+{
+    IMG_UINT8               mHdmi;
+    ENCODING_T              mEncodingOut;
+    ENCODING_T              mEncodingIn;
+    IMG_UINT8               mColorResolution;
+    IMG_UINT8               mPixelRepetitionFactor;
+    DTD                     mDtdList[HDMI_DTD_MAX];
+    IMG_UINT8               mDtdIndex;
+    IMG_UINT8               mDtdActiveIndex;
+    IMG_UINT8               mRgbQuantizationRange;
+    IMG_UINT8               mPixelPackingDefaultPhase;
+    IMG_UINT8               mColorimetry;
+    IMG_UINT8               mScanInfo;
+    IMG_UINT8               mActiveFormatAspectRatio;
+    IMG_UINT8               mNonUniformScaling;
+    IMG_UINT8               mExtColorimetry;
+    IMG_UINT8               mItContent;
+    IMG_UINT16              mEndTopBar;
+    IMG_UINT16              mStartBottomBar;
+    IMG_UINT16              mEndLeftBar;
+    IMG_UINT16              mStartRightBar;
+    IMG_UINT16              mCscFilter;
+    IMG_UINT16              mCscA[4];
+    IMG_UINT16              mCscC[4];
+    IMG_UINT16              mCscB[4];
+    IMG_UINT16              mCscScale;
+    IMG_UINT8               mHdmiVideoFormat;
+    IMG_UINT8               m3dStructure;
+    IMG_UINT8               m3dExtData;
+    IMG_UINT8               mHdmiVic;
+    IMG_UINT8               mDataEnablePolarity;
+    MONITOR_RANGE_LIMITS    mRangeLimits;
+    IMG_UINT8               mPreferredTimingIncluded;
+} VIDEO_PARAMS;
+
+/* Callback for HDMI->PDP */
+PVRSRV_ERROR PDPInitialize(VIDEO_PARAMS * pVideoParams, IMG_UINT32 uiInstance);
+
+#endif /* ENABLE_PLATO_HDMI */
+
+#endif /* if !defined(PLATO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.c
new file mode 100644
index 000000000000..488d8caf4e27
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.c
@@ -0,0 +1,2869 @@
+/*************************************************************************/ /*!
+@File
+@Title			System Configuration
+@Copyright		Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description	System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_memallocflags.h"
+#include "syscommon.h"
+#include "power.h"
+#include "sysinfo.h"
+#include "plato.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pci_support.h"
+#include "interrupt_support.h"
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+
+#include "plato_aon_regs.h"
+#include "plato_top_regs.h"
+#include "plato_ddr_ctrl_regs.h"
+#include "plato_ddr_publ_regs.h"
+
+/* BUP flags */
+#define PLATO_DDR_TRAINING_RETRY_MAX 5
+#define PLATO_DDR_KINGSTON 1
+
+#if defined(PLATO_LOG_CHECKPOINTS)
+#define PLATO_CHECKPOINT PVR_LOG(("- %s: %d", __func__, __LINE__))
+#else
+#define PLATO_CHECKPOINT
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+#define	HOST_PCI_INIT_FLAGS	0
+#else
+#define	HOST_PCI_INIT_FLAGS	HOST_PCI_INIT_FLAG_BUS_MASTER
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoLocalCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoLocalDevPAddrToCpuPAddr,
+};
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoSystemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoSystemDevPAddrToCpuPAddr,
+};
+#endif
+
+/* Clock speed module parameters */
+static IMG_UINT32 ui32MemClockSpeed  = PLATO_MEM_CLOCK_SPEED;
+static IMG_UINT32 ui32CoreClockSpeed = PLATO_RGX_CORE_CLOCK_SPEED;
+
+/* Helpers for getting DDR/GPU/PLL clock speed */
+IMG_UINT32 SysGetPlatoMemClockSpeed(void)
+{
+	IMG_UINT32 ret = ui32MemClockSpeed;
+
+	(ui32MemClockSpeed > PLATO_MAX_MEM_CLOCK_SPEED) ? ret = PLATO_MAX_MEM_CLOCK_SPEED : 0;
+	(ui32MemClockSpeed < PLATO_MIN_MEM_CLOCK_SPEED) ? ret = PLATO_MIN_MEM_CLOCK_SPEED : 0;
+
+	return ret;
+}
+
+IMG_UINT32 SysGetPlatoCoreClockSpeed(void)
+{
+	IMG_UINT32 ret = ui32CoreClockSpeed;
+
+	(ui32CoreClockSpeed > PLATO_RGX_MAX_CORE_CLOCK_SPEED) ? ret = PLATO_RGX_MAX_CORE_CLOCK_SPEED : 0;
+	(ui32CoreClockSpeed < PLATO_RGX_MIN_CORE_CLOCK_SPEED) ? ret = PLATO_RGX_MIN_CORE_CLOCK_SPEED : 0;
+
+	return ret;
+}
+
+/*
+ * This function returns a PLL clock value which can be easily reduced
+ * (using integer dividers) to get 148.5 MHz HDMI pixel clock.
+ *
+ * NOTE: Accurate values are returned only in the cases
+ *       which are known by the function.
+ */
+IMG_UINT32 SysGetPlatoPLLClockSpeed(IMG_UINT32 ui32ClockSpeed)
+{
+    /*
+     * Force the lowest possible PLL clock in case when the requested
+     * clock speed is higher than the largest value known by the function.
+     */
+    IMG_UINT32 ui32PLLClockSpeed = ui32ClockSpeed;
+    IMG_UINT32 ui32AccClockSpeed = ui32ClockSpeed;
+
+    /* Tweak the values if the requested clock speed is a supported one */
+    (ui32ClockSpeed <= 742500000) ? (ui32PLLClockSpeed = 742500000,  ui32AccClockSpeed = 742500000) : 0;
+    (ui32ClockSpeed <= 668250000) ? (ui32PLLClockSpeed = 1336500000, ui32AccClockSpeed = 668250000) : 0;
+    (ui32ClockSpeed <= 594000000) ? (ui32PLLClockSpeed = 1188000000, ui32AccClockSpeed = 594000000) : 0;
+    (ui32ClockSpeed <= 544500000) ? (ui32PLLClockSpeed = 1633500000, ui32AccClockSpeed = 544500000) : 0;
+    (ui32ClockSpeed <= 519750000) ? (ui32PLLClockSpeed = 1039500000, ui32AccClockSpeed = 519750000) : 0;
+    (ui32ClockSpeed <= 495000000) ? (ui32PLLClockSpeed = 1485000000, ui32AccClockSpeed = 495000000) : 0;
+    (ui32ClockSpeed <= 445500000) ? (ui32PLLClockSpeed = 891000000,  ui32AccClockSpeed = 445500000) : 0;
+    (ui32ClockSpeed <= 408375000) ? (ui32PLLClockSpeed = 1633500000, ui32AccClockSpeed = 408375000) : 0;
+    (ui32ClockSpeed <= 396000000) ? (ui32PLLClockSpeed = 1188000000, ui32AccClockSpeed = 396000000) : 0;
+
+    /*
+     * If the requested core clock speed matches the expected one, there is
+     * no need to do any additional adjustments of the PLL clock speed.
+     */
+    if (ui32ClockSpeed == ui32AccClockSpeed)
+    {
+        return ui32PLLClockSpeed;
+    }
+
+    /* Do fine grained adjustment if the requested clock speed is different than expected */
+    return ((((IMG_UINT64)ui32PLLClockSpeed << 32) / ui32AccClockSpeed) >> 32) * ui32ClockSpeed;
+}
+
+#if defined(__linux__)
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+module_param_named(sys_mem_clk_speed,  ui32MemClockSpeed,  uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(sys_mem_clk_speed, "Plato memory clock speed in Hz (600000000 - 800000000)");
+module_param_named(sys_core_clk_speed, ui32CoreClockSpeed, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(sys_core_clk_speed, "Plato core clock speed in Hz (396000000 - 742500000)");
+
+EXPORT_SYMBOL(SysGetPlatoMemClockSpeed);
+EXPORT_SYMBOL(SysGetPlatoCoreClockSpeed);
+EXPORT_SYMBOL(SysGetPlatoPLLClockSpeed);
+#endif
+
+#define PLATO_HAS_NON_MAPPABLE(dev) (dev->ui64MappedMemSize < SYS_DEV_MEM_REGION_SIZE)
+
+static IMG_BOOL PollPr(IMG_UINT32* base, IMG_UINT32 reg, IMG_UINT32 val, IMG_UINT32 msk, IMG_UINT32 cnt, IMG_UINT32 intrvl)
+{
+	IMG_UINT32 polnum;
+
+	for (polnum = 0; polnum < cnt; polnum++)
+	{
+		if ((OSReadHWReg32(base, reg) & msk) == val)
+		{
+			break;
+		}
+		OSSleepms(intrvl);
+	}
+	if (polnum == cnt)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Poll failed for register: 0x%08X", (unsigned int)reg));
+		return false;
+	}
+
+	return true;
+}
+
+#define Poll(base,reg,val,msk) PollPr(base,reg,val,msk,10,10)
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	return NULL;
+}
+
+static
+IMG_BOOL SystemISRHandler(void *pvData)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)pvData;
+	IMG_UINT32 ui32InterruptStatus;
+	IMG_UINT32 i;
+	IMG_BOOL bInterruptServiced = IMG_FALSE;
+
+#if defined(VIRTUAL_PLATFORM)
+	/* On virtual platform all interrupt handlers need to be called */
+	ui32InterruptStatus = (0x1 << PLATO_GPU_INT_SHIFT) | (0x1 << PLATO_PDP_INT_SHIFT);
+#else
+	{
+		void* pvSystemDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+		ui32InterruptStatus = OSReadHWReg32(pvSystemDbgPeripRegs , PLATO_TOP_CR_INT_STATUS);
+		PVR_DPF((PVR_DBG_MESSAGE, "%s:%d Interrupt Status (%u)",
+				 __func__, __LINE__, ui32InterruptStatus));
+	}
+#endif
+
+	for (i = 0; i < ARRAY_SIZE(psDevData->sInterruptData); i++)
+	{
+		if ((ui32InterruptStatus & psDevData->sInterruptData[i].ui32InterruptFlag) != 0)
+		{
+			bInterruptServiced = psDevData->sInterruptData[i].pfnLISR(psDevData->sInterruptData[i].pvData);
+		}
+	}
+
+	return bInterruptServiced;
+}
+
+static void EnableInterrupt(SYS_DATA *psDevData, IMG_UINT32 ui32InterruptFlag)
+{
+#if !defined(VIRTUAL_PLATFORM)
+	IMG_UINT32 ui32Value;
+
+	void* pvSystemDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+
+	ui32Value = OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	ui32Value |= ui32InterruptFlag;
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK, ui32Value);
+
+	(void) OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	OSWaitus(10);
+#endif
+}
+
+static void DisableInterrupt(SYS_DATA *psDevData, IMG_UINT32 ui32InterruptFlag)
+{
+#if !defined(VIRTUAL_PLATFORM)
+	IMG_UINT32 ui32Value;
+
+	void* pvSystemDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+
+	ui32Value = OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	ui32Value &= ~ui32InterruptFlag;
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK, ui32Value);
+
+	(void) OSReadHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PCI_INT_MASK);
+	OSWaitus(10);
+#endif
+}
+
+PVRSRV_ERROR SysGetPlatoGPUTemp(SYS_DATA * psDevData, IMG_UINT32 *ui32Temp);
+
+static IMG_UINT32 HashNOCRegisterAddress(IMG_UINT32 ui32Addr)
+{
+	IMG_UINT32 ui32Hash_bits = (ui32Addr & 0x4000 ? 1 : 0);
+	if (ui32Addr & 0x2000)
+		++ui32Hash_bits;
+	if (ui32Addr & 0x1000)
+		++ui32Hash_bits;
+	if (ui32Addr & 0x0800)
+		++ui32Hash_bits;
+	if ((ui32Hash_bits == 1) || (ui32Hash_bits == 3))
+		ui32Addr ^= 0x100;
+
+	return ui32Addr;
+}
+
+
+typedef struct PLATO_NOC_DBG_REG
+{
+	IMG_UINT32 ui32Offset;
+	IMG_CHAR *pszName;
+}PLATO_NOC_DBG_REGS;
+
+static PLATO_NOC_DBG_REGS asPlatoNOCDbgRegs[] = {
+	{SYS_PLATO_REG_NOC_DBG_DDR_A_CTRL_OFFSET, "DDR A Ctrl"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_A_DATA_OFFSET, "DDR A Data"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_A_PUBL_OFFSET, "DDR A Publ"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_B_CTRL_OFFSET, "DDR B Ctrl"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_B_DATA_OFFSET, "DDR B Data"},
+	{SYS_PLATO_REG_NOC_DBG_DDR_B_PUBL_OFFSET, "DDR B Publ"},
+
+	{SYS_PLATO_REG_NOC_DBG_DISPLAY_S_OFFSET, "Display S"},
+	{SYS_PLATO_REG_NOC_DBG_GPIO_0_S_OFFSET,  "GPIO 0 S"},
+	{SYS_PLATO_REG_NOC_DBG_GPIO_1_S_OFFSET,  "GPIO 1 S"},
+	{SYS_PLATO_REG_NOC_DBG_GPU_S_OFFSET,	 "GPU S"},
+	{SYS_PLATO_REG_NOC_DBG_PCI_PHY_OFFSET,	 "PCI PHY"},
+	{SYS_PLATO_REG_NOC_DBG_PCI_REG_OFFSET,	 "PCI Reg"},
+	{SYS_PLATO_REG_NOC_DBG_PCI_S_OFFSET,	 "PCI S"},
+	{SYS_PLATO_REG_NOC_DBG_PERIPH_S_OFFSET,  "Periph S"},
+	{SYS_PLATO_REG_NOC_DBG_RET_REG_OFFSET,	 "Ret Reg"},
+	{SYS_PLATO_REG_NOC_DBG_SERVICE_OFFSET,	 "Service"}
+};
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)psDevConfig->hSysData;
+	void *pvDbgPeripRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void *pvNocRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_NOC_OFFSET;
+	void *pvAonRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	IMG_UINT32 ui32Val;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32GPUTemp;
+
+	PVR_DUMPDEBUG_LOG("------[ System Debug - Device ID:%u ]------", psDevConfig->psDevNode->sDevId.ui32InternalID);
+
+	if ((eError = SysGetPlatoGPUTemp(psDevData, &ui32GPUTemp)) != PVRSRV_OK)
+	{
+		return eError;
+	}
+	PVR_DUMPDEBUG_LOG("GPU Temp: %u °C", ui32GPUTemp);
+	PVR_DUMPDEBUG_LOG("PCI_PHY_STATUS: 0x%x",
+					   OSReadHWReg32(pvDbgPeripRegs, PLATO_TOP_CR_PCI_PHY_STATUS));
+	for (i = 0; i <= 0x1C; i += 4)
+	{
+		PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", i,
+						   OSReadHWReg32(pvNocRegs, i));
+	}
+	PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", 0x50,
+					   OSReadHWReg32(pvNocRegs, 0x50));
+	PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", 0x54,
+					   OSReadHWReg32(pvNocRegs, 0x54));
+	PVR_DUMPDEBUG_LOG("NOC Offset 0x%x: 0x%x", 0x58,
+					   OSReadHWReg32(pvNocRegs, 0x58));
+	PVR_DUMPDEBUG_LOG("AON Offset 0x%x: 0x%x", PLATO_AON_CR_BM_STATUS,
+					   OSReadHWReg32(pvAonRegs, PLATO_AON_CR_BM_STATUS));
+	PVR_DUMPDEBUG_LOG("AON Offset 0x%x: 0x%x", PLATO_AON_CR_PLL_STATUS,
+					   OSReadHWReg32(pvAonRegs, PLATO_AON_CR_PLL_STATUS));
+
+	PVR_DUMPDEBUG_LOG("NOC Error Status: ");
+	for (i = 0; i < ARRAY_SIZE(asPlatoNOCDbgRegs); i++)
+	{
+		IMG_UINT32 ui32Offset = asPlatoNOCDbgRegs[i].ui32Offset + 0xC;
+		IMG_UINT32 ui32HashedOffset = HashNOCRegisterAddress(ui32Offset);
+
+		ui32Val = OSReadHWReg32(pvNocRegs, ui32HashedOffset);
+		if (ui32Val == 0)
+		{
+			PVR_DUMPDEBUG_LOG("%s ok", asPlatoNOCDbgRegs[i].pszName);
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, ui32Val, ui32Offset);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x8), ui32Offset + 0x8);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0xC), ui32Offset + 0xC);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x14), ui32Offset + 0x14);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x18), ui32Offset + 0x18);
+			PVR_DUMPDEBUG_LOG("%s error val 0x%x (offset 0x%x)", asPlatoNOCDbgRegs[i].pszName, OSReadHWReg32(pvNocRegs, ui32HashedOffset + 0x1C), ui32Offset + 0x1C);
+		}
+
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PlatoMapRegisters(SYS_DATA *psDevData, IMG_UINT32 ui32BaseNum, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Size, void ** pvCpuVAddr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CPU_PHYADDR	sCpuPBase;
+
+	sCpuPBase.uiAddr = OSPCIAddrRangeStart(psDevData->hRGXPCI, ui32BaseNum) + ui32Offset;
+
+#if !defined(VIRTUAL_PLATFORM)
+	/* Reserve all register banks up to HDMI bank */
+	eError = OSPCIRequestAddrRegion(psDevData->hRGXPCI, ui32BaseNum, ui32Offset, ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the system address region(BAR %d)",
+				 __func__, ui32BaseNum));
+		goto ErrorReturn;
+	}
+
+	*pvCpuVAddr = OSMapPhysToLin(sCpuPBase, ui32Size, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (*pvCpuVAddr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers (BAR %d)",
+				 __func__, ui32BaseNum));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+#endif
+
+	return eError;
+
+#if !defined(VIRTUAL_PLATFORM)
+ErrorSystemRegReleaseAddrRegion:
+	OSPCIReleaseAddrRegion(psDevData->hRGXPCI, SYS_PLATO_REG_PCI_BASENUM, SYS_PLATO_REG_SYS_OFFSET, SYS_PLATO_REG_PDP_OFFSET - SYS_PLATO_REG_SYS_OFFSET);
+
+ErrorReturn:
+	return eError;
+#endif
+}
+
+void PlatoUnmapRegisters(SYS_DATA * psDevData, void * pvBase, IMG_UINT32 ui32BaseNum, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Size)
+{
+	OSUnMapPhysToLin(pvBase, ui32Size);
+	OSPCIReleaseAddrRegion(psDevData->hRGXPCI, ui32BaseNum, ui32Offset, ui32Size);
+}
+
+
+static PVRSRV_ERROR ConfigPlatoDram(void * pvPublRegs, void * pvCtrlRegs, void * pvAonRegs,
+									IMG_UINT32 pvBLDRData[PLATO_DDR_PUBL_DATX_LANE_COUNT][PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE],
+									IMG_UINT32 ui32ResetFlags)
+{
+	/*
+
+	Phase 1: Program the DWC_ddr_umctl2 registers
+
+	*/
+
+	/* Single rank only for Kingston DDRs*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_MSTR, 0x41040001);
+
+	OSWaitus(100);
+
+	/*refresh timings*/
+#if defined(PLATO_DDR_KINGSTON)
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x0081008B);
+#else
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x006100BB);
+#endif
+
+	// Train DDR sequence
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT0, 0x00020100);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT1, 0x00010000);
+
+#if defined(PLATO_DDR_KINGSTON)
+	/*write recovery */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT3, 0x01700000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00280000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT5, 0x0012000c);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0f0e2112);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x00040618);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x0506040B);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x06020307);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x090e0403);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x0d0e000e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a05);
+
+	/*impedance registers */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x30ab002b);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+
+	/*refresh control*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e01020);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x0078007e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0057000e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+	/*DFI Timings*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x02878208);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x00f000ff);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x80100010);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x088105c3);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000000);
+#else
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT3, 0x0d700000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00180000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_INIT5, 0x00090009);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0c101a0e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x000a0313);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x04050509);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x06020306);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x070c0202);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x0d0e000e);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a07);
+	/*impedance registers */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x10800020);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+
+	/*refresh control*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e06010);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x00600031);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0004002a);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+
+	/*DFI Timings*/
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x02878206);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFILPCFG0, 0x07111031);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFILPCFG1, 0x00000050);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x006a006f);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x0d0b02b6);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x00100010);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+#endif
+
+	/* Single rank only on Kingston */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP0, 0x00001F1F);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP1, 0x00070707);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP2, 0x00000000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP3, 0x0F000000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP4, 0x00000F0F);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP5, 0x06060606);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP6, 0x06060606);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP7, 0x00000F0F);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ADDRMAP8, 0x00000707);
+
+#if defined(PLATO_DDR_KINGSTON)
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x06000604);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTMAP, 0x99c5b050);
+#else
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x0d0f0740);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_ODTMAP, 0x99c5b050);
+#endif
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PERFHPR1, 0x9f008f23);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_SCHED, 0x00003f00);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PERFLPR1, 0x18000064);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PERFWR1, 0x18000096);
+
+	/* Setup the virtual channels */
+	OSWriteHWReg32(pvCtrlRegs, 0x00000410, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000414, 0x00000000);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000418, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000041C, 0x00000001);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000420, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000424, 0x00000002);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000428, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000042C, 0x00000003);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000430, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000434, 0x00000004);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000438, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000043C, 0x00000005);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000440, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000444, 0x00000006);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000448, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000044C, 0x00000007);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000450, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000454, 0x00000008);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000458, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000045C, 0x00000009);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000460, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000464, 0x0000000A);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000468, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000046C, 0x0000000B);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000470, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000474, 0x0000000C);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000478, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x0000047C, 0x0000000D);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000480, 0x0000000F);
+	OSWriteHWReg32(pvCtrlRegs, 0x00000484, 0x0000000E);
+
+	Poll(pvCtrlRegs, 0x484, 0x0000000E, 0x0000000F);
+
+	OSWaitus(1000);
+
+	PLATO_CHECKPOINT;
+
+	/*
+
+	Phase 2: Deassert soft reset signal core_ddrc_rstn
+
+	*/
+
+	/* Now getting DRAM controller out of reset */
+	OSWriteHWReg32(pvAonRegs, PLATO_AON_CR_RESET_CTRL, ui32ResetFlags | OSReadHWReg32(pvAonRegs, PLATO_AON_CR_RESET_CTRL));
+
+	OSWaitus(1000);
+
+	/* ECC disable */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DBG1, 0x00000000);
+	/* power related */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PWRCTL, 0x00000000);
+	/* Enabling AXI input port (Port control) */
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_PCTRL, 0x00000001);
+
+	PLATO_CHECKPOINT;
+	/*
+
+	Phase 7: Set DFIMISC.dfi_init_complete_en to 1
+
+	(skipped on emu?)
+	*/
+
+	/*
+
+	Phase 3: Start PHY initialization by accessing relevant PUB registers
+
+	*/
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DCR_OFFSET, 0x0000040B);
+
+#if defined(PLATO_DDR_KINGSTON)
+	/* IF DDR_RISE_RISE */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0064641F);
+
+		/*MR Registers */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0x170);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000400);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000228);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x06220308);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x281b1004);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00060120);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x01200807);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x00300c08);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR2_OFFSET, 0x00F09088);
+#else
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PTR0_OFFSET, 0x10000010);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PTR1_OFFSET, 0x271012c0);
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PTR2_OFFSET, 0x00080421);
+
+	/* IF DDR_RISE_RISE */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0064641F);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0xd70);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000000);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000018);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x061c0B06);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x28200400);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00040005);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x01180805);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x00250B06);
+	/* IF TRAIN_DDR */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR2_OFFSET, 0x00F09088);
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+#endif
+
+	/* DISABLE VT COMPENSATION */
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR6_OFFSET, 0x00013001);
+	//OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGCR4_OFFSET, 0x40000000);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXCCR_OFFSET, 0x20C01884);
+
+	/* VREF CHANGE */
+	/*
+	OSWriteHWReg32(pvPublRegs, 0x0710, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0810, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0910, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0A10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0B10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0C10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0D10, 0x0E00083C);
+	OSWriteHWReg32(pvPublRegs, 0x0E10, 0x0E00083C);
+	*/
+
+	PLATO_CHECKPOINT;
+	/*
+
+	Phase 4: Trigger PHY initialization: Impedance, PLL, and DDL; assert PHY reset
+
+	*/
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00000073);
+
+	/*
+
+	Phase 5: Monitor PHY initialization status by polling the PUB register PGSR0
+
+	*/
+	Poll(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET, 0xF, 0xF);
+
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000000);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+	OSWriteHWReg32(pvCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000001);
+
+	PLATO_CHECKPOINT;
+
+	/*
+
+	Phase 6: Indicate to the PUB that the controller will perform SDRAM
+	initialization by setting PIR.INIT and PIR.CTLDINIT, and
+	poll PGSR0.IDONE
+
+	*/
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00040001);
+	Poll(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET, 0x11, 0x11);
+
+	/*
+
+	Phase 8: Wait for DWC_ddr_umctl2 to move to "normal" operating
+	mode by monitoring STAT.operating_mode signal
+
+	*/
+	PollPr(pvCtrlRegs, PLATO_DDR_CTRL_STAT, 0x01, 0x01, 10, 100);
+
+	OSWaitus(100);
+
+	/* IF TRAIN_DDR */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTCR0_OFFSET, 0x8000B1C7);
+	/*single rank only for Kingston */
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DTCR1_OFFSET, 0x00010237);
+
+#if defined(PLATO_DDR_BDLR_TRAINING)
+	/* Bit delay line register training */
+	{
+		IMG_UINT8 ui8Lane = 0;
+		IMG_UINT8 ui8Reg = 0;
+		for (ui8Lane = 0; ui8Lane < PLATO_DDR_PUBL_DATX_LANE_COUNT; ui8Lane++)
+		{
+			for (ui8Reg = 0; ui8Reg < PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE; ui8Reg++)
+			{
+				// PVR_LOG(("- %s: BDLR Lane %d Register %d: 0x%08x", __func__, ui8Lane, ui8Reg, pvBLDRData[ui8Lane][ui8Reg]));
+				OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXnBDLR_OFFSET(ui8Lane, ui8Reg), pvBLDRData[ui8Lane][ui8Reg]);
+			}
+		}
+	}
+
+	/* Poll on general status register 2 for each lane */
+	{
+		IMG_UINT8 ui8Lane;
+		for (ui8Lane = 0; ui8Lane < PLATO_DDR_PUBL_DATX_LANE_COUNT; ui8Lane++)
+		{
+			Poll(pvPublRegs, PLATO_DDR_PUBL_DXnGSR_OFFSET(ui8Lane, 2), 0, 0x001FFFFF);
+		}
+	}
+#endif
+
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x0000ff72);
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x0000ff73);
+	Poll(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET, 0x80000fff, 0xfff80fff);
+	Poll(pvCtrlRegs, PLATO_DDR_CTRL_STAT, 0x1, 0x1);
+
+	/* Setting the Anti Glitch OFF (?), Disabling On Die pullup/pulldowns */
+#if defined(PLATO_DDR_KINGSTON)
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXCCR_OFFSET, 0x02401884);
+#else
+	OSWriteHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXCCR_OFFSET, 0x02400004);
+#endif
+
+	if (OSReadHWReg32(pvPublRegs, PLATO_DDR_PUBL_PGSR0_OFFSET) != 0x80000fff)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: DDR Training failed", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	OSWaitus(100);
+
+#if defined(DEBUG)
+	{
+		IMG_UINT8 lane = 0;
+		IMG_UINT8 reg = 0;
+		for (lane = 0; lane < PLATO_DDR_PUBL_DATX_LANE_COUNT; lane++)
+		{
+			for (reg = 0; reg < PLATO_DDR_PUBL_DXGSR_REGS_PER_LANE; reg++)
+			{
+				PVR_LOG(("DX%dGSR%d: 0x%08x", lane, reg,
+					OSReadHWReg32(pvPublRegs, PLATO_DDR_PUBL_DXnGSR_OFFSET(lane, reg))));
+			}
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+#if defined(EMULATOR)
+static PVRSRV_ERROR ConfigPlatoEmu(SYS_DATA *psDevData)
+{
+	IMG_BOOL bPollPassed;
+	/* To be obtained differently */
+	void* pvSystemDbgPeripRegs		= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void* pvSystemDbgAonRegs		= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	void* pvSystemDbgDdrACtrlRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_CTRL_OFFSET;
+	void* pvSystemDbgDdrAPublRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_PUBL_OFFSET;
+	void* pvSystemDbgDdrBCtrlRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_CTRL_OFFSET;
+	void* pvSystemDbgDdrBPublRegs	= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_PUBL_OFFSET;
+	void* pvSystemDbgNocRegs		= psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_NOC_OFFSET;
+
+#if defined(ENABLE_PLATO_HDMI)
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, 0x3370A03);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, 0x3370A03, 0x3370A03);
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1, 0x1);
+#endif
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1, 0x1);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, 0x01101037);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, 0x01101037, 0x01101037);
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, 0x00780000);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, 0x00780000, 0x00780000);
+
+	/* Waiting for DDR PLL getting locked */
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x2, 0x2, -1 , 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01, 0x01);
+
+	OSWaitus(100);
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x011);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x011, 0x011);
+#else
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x01);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, 0x01, 0x01);
+#endif
+	/* PLL Lock is done */
+
+	OSWaitus(1000);
+
+	/* Enabling PDP gated clock output - 198 MHz */
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, 0x00001210);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, 0x00001210, 0x00001210);
+
+	OSWaitus(100);
+
+	/* PDP needs HDMI clocks on for framegrabber, start them here */
+	/* Enabling HDMI gated clock output - 148.5 MHz */
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, 0x00001310);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, 0x00001310, 0x00001310);
+
+	OSWaitus(100);
+	PVR_LOG(("%s: Enabled PDP and HDMI clocks", __func__));
+
+	/* GPU PLL configuration */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, 0x0110103D);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, 0x0110103D, 0x0110103D);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, 0x00E00000);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, 0x00E00000, 0x00E00000);
+	/* Waiting for GPU PLL getting locked */
+
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x3, 0x3, -1, 10);
+	/* GPU and DDR PLL Locked */
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x1, 0x1);
+
+	OSWaitus(100);
+
+	/* Enabling gated clock output */
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, 0x011);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, 0x011, 0x011);
+
+	OSWaitus(1000);
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	/* Setting dual memory interleaved mode */
+	OSWriteHWReg32(pvSystemDbgNocRegs, 0x00000050, 0x01);
+	PollPr(pvSystemDbgNocRegs, 0x00000058, 0x01, 0x1, 1, 10);
+
+	/* Now putting DRAM controller out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0F30);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0F30, 0x0F30);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	OSWaitus(10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0A30);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0A30, 0xA30);
+
+#else
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0330);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0330, 0x0330);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	OSWaitus(10);
+
+	/* Now putting DRAM controller into reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0230);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0230, 0x0230);
+
+#endif
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/*
+
+	Phase 1: Program the DWC_ddr_umctl2 registers
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001);
+	Poll(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001, 0x8F040001);
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001);
+	Poll(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_MSTR, 0x8F040001, 0x8F040001);
+#endif
+
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x007f0056);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHTMG, 0x007f0056);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT3, 0x01140000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT3, 0x01140000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT1, 0x00010000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT1, 0x00010000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT0, 0x00020001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT0, 0x00020001);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00280000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT4, 0x00280000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_INIT5, 0x000C000C);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_INIT5, 0x000C000C);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0f132312);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG0, 0x0f132312);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x00080419);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG1, 0x00080419);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x0507050b);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG2, 0x0507050b);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG3, 0x00002008);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x07020407);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG4, 0x07020407);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x090e0403);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG5, 0x090e0403);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x020e000e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG6, 0x020e000e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG7, 0x00000c0e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a05);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DRAMTMG8, 0x01010a05);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x30ab002b);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ZQCTL0, 0x30ab002b);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ZQCTL1, 0x00000070);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ZQCTL2, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e01020);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL0, 0x00e01020);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x0078007e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL1, 0x0078007e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0057000e);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL2, 0x0057000e);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_RFSHCTL3, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x028A8208);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFITMG0, 0x028A8208);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFITMG1, 0x00020202);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD0, 0x00400003);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x00F000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD1, 0x00F000FF);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x80100010);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD2, 0x80100010);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x00100010);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIUPD3, 0x00100010);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP0, 0x00001414);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP1, 0x00070707);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP2, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP3, 0x0F000000);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP4, 0x00000F0F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP5, 0x06060606);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ADDRMAP6, 0x0F0F0606);
+
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP0, 0x00001414);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP1, 0x00070707);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP2, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP3, 0x0F000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP4, 0x00000F0F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP5, 0x06060606);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ADDRMAP6, 0x0F0F0606);
+
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x04000400);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_ODTCFG, 0x04000400);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_SCHED, 0x00003F00);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_SCHED, 0x00003F00);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PERFHPR1, 0x0F0000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PERFHPR1, 0x0F0000FF);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PERFLPR1, 0x0F0000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PERFLPR1, 0x0F0000FF);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PERFWR1, 0x0F0000FF);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PERFWR1, 0x0F0000FF);
+
+	/* Setup the virtual channels */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000410, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000414, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000418, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000041C, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000420, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000424, 0x00000002);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000428, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000042C, 0x00000003);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000430, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000434, 0x00000004);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000438, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000043C, 0x00000005);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000440, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000444, 0x00000006);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000448, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000044C, 0x00000007);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000450, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000454, 0x00000008);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000458, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000045C, 0x00000009);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000460, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000464, 0x0000000A);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000468, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000046C, 0x0000000B);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000470, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000474, 0x0000000C);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000478, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x0000047C, 0x0000000D);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000480, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, 0x00000484, 0x0000000E);
+
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000410, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000414, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000418, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000041C, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000420, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000424, 0x00000002);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000428, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000042C, 0x00000003);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000430, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000434, 0x00000004);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000438, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000043C, 0x00000005);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000440, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000444, 0x00000006);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000448, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000044C, 0x00000007);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000450, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000454, 0x00000008);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000458, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000045C, 0x00000009);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000460, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000464, 0x0000000A);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000468, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000046C, 0x0000000B);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000470, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000474, 0x0000000C);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000478, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x0000047C, 0x0000000D);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000480, 0x0000000F);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, 0x00000484, 0x0000000E);
+
+	/* DRAM controller registers configuration done */
+
+	OSWaitus(100);
+
+	/*
+
+	Phase 2: Deassert soft reset signal core_ddrc_rstn
+
+	*/
+
+	/* Now getting DRAM controller out of reset */
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F30);
+#else
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000330);
+#endif
+
+	OSWaitus(100);
+
+	/* ECC disable */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DBG1, 0x00000000);
+	/* power related */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PWRCTL, 0x00000000);
+	/* Enabling AXI input port (Port control) */
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_PCTRL, 0x00000001);
+
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DBG1, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PWRCTL, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_PCTRL, 0x00000001);
+
+	/*
+
+	Phase 7: Set DFIMISC.dfi_init_complete_en to 1
+
+	(skipped on emu?)
+	*/
+
+	/*
+
+	Phase 3: Start PHY initialization by accessing relevant PUB registers
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DCR_OFFSET, 0x0000040B);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DCR_OFFSET, 0x0000040B);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PTR0_OFFSET, 0x10000010);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PTR0_OFFSET, 0x10000010);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PTR1_OFFSET, 0x00800080);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PTR1_OFFSET, 0x00800080);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PTR2_OFFSET, 0x00080421);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PTR2_OFFSET, 0x00080421);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0020641F);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DSGCR_OFFSET, 0x0020641F);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0x00000114);
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000028);
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR0_OFFSET, 0x00000114);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR1_OFFSET, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR2_OFFSET, 0x00000028);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_MR3_OFFSET, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x040F0406);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR0_OFFSET, 0x040F0406);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x28110402);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR1_OFFSET, 0x28110402);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00030002);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR2_OFFSET, 0x00030002);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR3_OFFSET, 0x02000101);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x00190602);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR4_OFFSET, 0x00190602);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x0018040B);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_DTPR5_OFFSET, 0x0018040B);
+
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PGCR1_OFFSET, 0x020046A0);
+
+	/*
+
+	Phase 4: Trigger PHY initialization: Impedance, PLL, and DDL; assert PHY reset
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00000073);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00000073);
+
+	/*
+
+	Phase 5: Monitor PHY initialization status by polling the PUB register PGSR0
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000000);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000000);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_DFIMISC, 0x00000001);
+
+	OSWriteHWReg32(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000001);
+	OSWriteHWReg32(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_SWCTL, 0x00000001);
+
+	/*
+
+	Phase 6: Indicate to the PUB that the controller will perform SDRAM
+	initialization by setting PIR.INIT and PIR.CTLDINIT, and
+	poll PGSR0.IDONE
+
+	*/
+	OSWriteHWReg32(pvSystemDbgDdrAPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00040001);
+	OSWriteHWReg32(pvSystemDbgDdrBPublRegs, PLATO_DDR_PUBL_PIR_OFFSET, 0x00040001);
+
+	/*
+
+	Phase 8: Wait for DWC_ddr_umctl2 to move to "normal" operating
+	mode by monitoring STAT.operating_mode signal
+
+	*/
+	PollPr(pvSystemDbgDdrACtrlRegs, PLATO_DDR_CTRL_STAT, 0x01, 0x01, 10, 100);
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	PollPr(pvSystemDbgDdrBCtrlRegs, PLATO_DDR_CTRL_STAT, 0x01, 0x01, 10, 100);
+#endif
+	OSWaitus(100);
+
+	/* Getting GPU And DDR A out of reset */
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12, 0x00000F12, -1, 100);
+#else
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312, 0x00000312, -1, 100);
+#endif
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	/* setting CR_ISO_CTRL:CR_GPU_CLK_E */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_ISO_CTRL, 0x000001F);
+
+	return PVRSRV_OK;
+}
+
+#else // HW
+
+static PVRSRV_ERROR CheckPDPAccessibility(SYS_DATA *psDevData)
+{
+	#define PLATO_PDP_REGISTER_CORE_ID_OFFSET (0x04e0)
+	#define VALID_PDP_CORE_ID (0x7010003)
+
+	void* pvPDPRegs;
+	IMG_UINT32 ui32CoreId;
+
+	if (PlatoMapRegisters(psDevData, SYS_PLATO_REG_PCI_BASENUM, SYS_PLATO_REG_PDP_OFFSET,
+		SYS_PLATO_REG_PDP_SIZE, &pvPDPRegs) != PVRSRV_OK || pvPDPRegs == NULL)
+	{
+		return PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE;
+	}
+
+	ui32CoreId = OSReadHWReg32(pvPDPRegs, PLATO_PDP_REGISTER_CORE_ID_OFFSET);
+	if (ui32CoreId != VALID_PDP_CORE_ID)
+	{
+		PVR_LOG(("PDP Access failed, expected 0x%x but got 0x%x", VALID_PDP_CORE_ID, ui32CoreId));
+		PlatoUnmapRegisters(psDevData, pvPDPRegs, SYS_PLATO_REG_PCI_BASENUM,
+			SYS_PLATO_REG_PDP_OFFSET, SYS_PLATO_REG_PDP_SIZE);
+		return PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE;
+	}
+
+	PlatoUnmapRegisters(psDevData, pvPDPRegs, SYS_PLATO_REG_PCI_BASENUM,
+		SYS_PLATO_REG_PDP_OFFSET, SYS_PLATO_REG_PDP_SIZE);
+	return PVRSRV_OK;
+}
+
+/*
+ * Helpers for getting integer and fractional values which are needed when
+ * programming PLATO_AON_CR_{GPU|DDR}_PLL_CTRL_{0|1} registers for DDR/GPU PLLs.
+ *
+ * PLL_CLOCK_SPEED = (PLL_REF_CLOCK_SPEED * REFDIV) *
+ *                   (PLL_INT + (PLL_FRAC / 2^24)) / POSTDIV1 / POSTDIV2
+ *
+ * NOTE: It's assumed that REFDIV, POSTDIV1 and POSTDIV2 are '1' in all cases.
+ */
+static IMG_UINT32 GetPlatoPLLInt(IMG_UINT32 ui32PLLClock)
+{
+	return ui32PLLClock / PLATO_PLL_REF_CLOCK_SPEED;
+}
+
+static IMG_UINT32 GetPlatoPLLFrac(IMG_UINT32 ui32PLLClock)
+{
+	/* Shift to get 24 bits for fractional part after div */
+	IMG_UINT64 shift = (IMG_UINT64)ui32PLLClock << 24;
+
+	/* Div and return only fractional part of the result */
+	return (shift / PLATO_PLL_REF_CLOCK_SPEED) & ((1 << 24) - 1);
+}
+
+/*
+ * Helper for getting value of integer divider for GPU clock.
+ */
+static IMG_UINT32 GetPlatoGPUVDiv0(IMG_UINT32 ui32PLLClock, IMG_UINT32 ui32CoreClockSpeed)
+{
+	IMG_UINT32 div, ret;
+
+	div = ui32PLLClock / ui32CoreClockSpeed;
+
+	/* Bias the result by (-1) with saturation, then clip it */
+	ret = (div - (div > 0)) & (PLATO_CR_GPUV_DIV_0_MASK >> PLATO_CR_GPUV_DIV_0_SHIFT);
+
+	/* Check for lost result after clipping, saturate if so */
+	return (div > 1) && (ret != (div - (div > 0))) ?
+			(PLATO_CR_GPUV_DIV_0_MASK >> PLATO_CR_GPUV_DIV_0_SHIFT) : ret;
+}
+
+/*
+ * Helpers for getting values of integer dividers for PDP clock.
+ *
+ * NOTE: Use only if PLL clock speed > ~350 MHz.
+ */
+static IMG_UINT32 GetPlatoPDPV0Div0(IMG_UINT32 ui32PLLClock)
+{
+	IMG_UINT32 div, ret;
+
+	div = ui32PLLClock / PLATO_MIN_PDP_CLOCK_SPEED;
+
+	/* Bias the result by (-1) with saturation, then clip it */
+	ret = (div - (div > 0)) & (PLATO_CR_PDPV0_DIV_0_MASK >> PLATO_CR_PDPV0_DIV_0_SHIFT);
+
+	/* Check for lost result after clipping, saturate if so */
+	return (div > 1) && (ret != (div - (div > 0))) ?
+			(PLATO_CR_PDPV0_DIV_0_MASK >> PLATO_CR_PDPV0_DIV_0_SHIFT) : ret;
+}
+
+static IMG_UINT32 GetPlatoPDPV1Div0(IMG_UINT32 ui32PLLClock)
+{
+	IMG_UINT32 div, ret;
+
+	div = (ui32PLLClock / (GetPlatoPDPV0Div0(ui32PLLClock) + 1)) / PLATO_MIN_PDP_CLOCK_SPEED;
+
+	/* Bias the result by (-1) with saturation, then clip it */
+	ret = (div - (div > 0)) & (PLATO_CR_PDPV1_DIV_0_MASK >> PLATO_CR_PDPV1_DIV_0_SHIFT);
+
+	/* Check for lost result after clipping, saturate if so */
+	return (div > 1) && (ret != (div - (div > 0))) ?
+			(PLATO_CR_PDPV1_DIV_0_MASK >> PLATO_CR_PDPV1_DIV_0_SHIFT) : ret;
+}
+
+#if defined(ENABLE_PLATO_HDMI)
+/*
+ * Helpers for getting values of integer dividers for HDMICEC clocks.
+ *
+ * NOTE: They strive to get clock speed of HDMI-SFR as close to
+ *       27 MHz as possible.
+ */
+static IMG_UINT32 GetPlatoHDMICECV0Div0(IMG_UINT32 ui32PLLClock)
+{
+    IMG_UINT32 ui32HDMICECV0Div0, ui32HDMICECV0Div0Limit;
+    IMG_UINT32 ui32HDMICECV1Div0, ui32HDMICECV1Div0Limit;
+    IMG_UINT32 ui32HDMISFRClockSpeed;
+
+    ui32HDMICECV0Div0Limit = PLATO_CR_HDMICECV0_DIV_0_MASK >> PLATO_CR_HDMICECV0_DIV_0_SHIFT;
+    ui32HDMICECV1Div0Limit = PLATO_CR_HDMICECV1_DIV_0_MASK >> PLATO_CR_HDMICECV1_DIV_0_SHIFT;
+
+    ui32HDMICECV0Div0 = 0;
+    while (ui32HDMICECV0Div0 < ui32HDMICECV0Div0Limit)
+    {
+        ui32HDMICECV1Div0 = 0;
+        while (ui32HDMICECV1Div0 < ui32HDMICECV1Div0Limit)
+        {
+            ui32HDMISFRClockSpeed = ui32PLLClock / (ui32HDMICECV0Div0 + 1) / (ui32HDMICECV1Div0 + 1);
+
+            if (ui32HDMISFRClockSpeed <= PLATO_TARGET_HDMI_SFR_CLOCK_SPEED)
+            {
+                /* Done, value of the divider found */
+                return ui32HDMICECV0Div0;
+            }
+
+            ui32HDMICECV1Div0++;
+        }
+
+        ui32HDMICECV0Div0++;
+    }
+
+    /* Here the function returns the highest possible value of the divider */
+    return ui32HDMICECV0Div0;
+}
+
+static IMG_UINT32 GetPlatoHDMICECV1Div0(IMG_UINT32 ui32PLLClock)
+{
+    IMG_UINT64 div, ret;
+
+    /* Calculate the divider using 32.32 fixed point math */
+    div = (IMG_UINT64)ui32PLLClock << 32;
+    div /= GetPlatoHDMICECV0Div0(ui32PLLClock) + 1;
+    div /= PLATO_TARGET_HDMI_SFR_CLOCK_SPEED;
+
+    /* Round up if the fractional part is present */
+    div = (div >> 32) + ((div & 0xFFFFFFFF) > 0);
+
+    /* Bias the result by (-1) with saturation, then clip it */
+    ret = (div - (div > 0)) & (PLATO_CR_HDMICECV1_DIV_0_MASK >> PLATO_CR_HDMICECV1_DIV_0_SHIFT);
+
+    /* Check for lost result after clipping, saturate if so */
+    return (div > 1) && (ret != (div - (div > 0))) ?
+            (PLATO_CR_HDMICECV1_DIV_0_MASK >> PLATO_CR_HDMICECV1_DIV_0_SHIFT) : ret;
+}
+
+static IMG_UINT32 GetPlatoHDMICECV2Div0(IMG_UINT32 ui32PLLClock)
+{
+    IMG_UINT64 div, ret;
+
+    /* Calculate the divider using 32.32 fixed point math */
+    div = (IMG_UINT64)ui32PLLClock << 32;
+    div /= GetPlatoHDMICECV0Div0(ui32PLLClock) + 1;
+    div /= GetPlatoHDMICECV1Div0(ui32PLLClock) + 1;
+    div /= PLATO_TARGET_HDMI_CEC_CLOCK_SPEED;
+
+    /* Round up if the fractional part is present */
+    div = (div >> 32) + ((div & 0xFFFFFFFF) > 0);
+
+    /* Bias the result by (-1) with saturation, then clip it */
+    ret = (div - (div > 0)) & (PLATO_CR_HDMICECV2_DIV_0_MASK >> PLATO_CR_HDMICECV2_DIV_0_SHIFT);
+
+    /* Check for lost result after clipping, saturate if so */
+    return (div > 1) && (ret != (div - (div > 0))) ?
+            (PLATO_CR_HDMICECV2_DIV_0_MASK >> PLATO_CR_HDMICECV2_DIV_0_SHIFT) : ret;
+}
+#endif
+
+#if defined(PLATO_DUAL_CHANNEL_DDR)
+
+static PVRSRV_ERROR ConfigPlatoDualChannel(SYS_DATA *psDevData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bPollPassed;
+
+	void* pvSystemDbgPeripRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void* pvSystemDbgAonRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	void* pvSystemDbgDdrACtrlRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_CTRL_OFFSET;
+	void* pvSystemDbgDdrAPublRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_PUBL_OFFSET;
+	void* pvSystemDbgDdrBCtrlRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_CTRL_OFFSET;
+	void* pvSystemDbgDdrBPublRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_B_PUBL_OFFSET;
+	void* pvSystemDbgNocRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_NOC_OFFSET;
+
+	IMG_UINT32 ui32BDLRSetupDddrA[PLATO_DDR_PUBL_DATX_LANE_COUNT][PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE] =
+		{{0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F}};
+	IMG_UINT32 ui32BDLRSetupDddrB[PLATO_DDR_PUBL_DATX_LANE_COUNT][PLATO_DDR_PUBL_DXBDLR_REGS_PER_LANE] =
+		{{0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x14141414, 0x14141414, 0x00141414},
+		 {0x0F0F0F0F, 0x0F0F0F0F, 0x000F0F0F}};
+
+	IMG_UINT32 ui32MemClockSpeed, ui32MemPLLClockSpeed;
+	IMG_UINT32 ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl1, ui32MemClockCtrl;
+
+	IMG_UINT32 ui32CoreClockSpeed, ui32CorePLLClockSpeed;
+	IMG_UINT32 ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl1, ui32CoreClockCtrl;
+
+#if defined(ENABLE_PLATO_HDMI)
+	IMG_UINT32 ui32HDMICECClockCtrl;
+#endif
+
+	IMG_UINT32 ui32PDPClockCtrl, ui32HDMIClockCtrl;
+
+	/* Plato Soft reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x40);
+	OSSleepms(100);
+
+	/* Temporary fix for non-32GB PCI BAR Plato boards that seem to not signal getting out reset */
+	if (!PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x030, 0xF0);
+		if (!bPollPassed)
+		{
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+	}
+
+	/* On non-32GB PCI BAR Plato boards, bring display subsystem out of reset with PLL bypassed. */
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x0);
+		OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, PLATO_CR_DISPLAY_RESET_MASK);
+		OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x1);
+	}
+
+	/* Setting dual memory interleaved mode */
+	OSWriteHWReg32(pvSystemDbgNocRegs, 0x00000050, 0x01);
+	PollPr(pvSystemDbgNocRegs, 0x00000058, 0x01, 0x1, 1, 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1, 0x1);
+
+	/* Setup DDR PLL's */
+	ui32MemClockSpeed = SysGetPlatoMemClockSpeed();
+	ui32MemPLLClockSpeed = ui32MemClockSpeed;
+
+	ui32MemClockPLLCtrl0 = (GetPlatoPLLInt(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FBDIV_SHIFT);
+	ui32MemClockPLLCtrl0 &= PLATO_CR_DDR_PLL_FBDIV_MASK;
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_REFDIV_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV1_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV2_SHIFT);
+
+	ui32MemClockPLLCtrl1 = (GetPlatoPLLFrac(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FRAC_SHIFT);
+	ui32MemClockPLLCtrl1 &= PLATO_CR_DDR_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1, ui32MemClockPLLCtrl1);
+
+	PVR_LOG(("- %s: DDR clock: %u Hz (%u MHz)",
+			 __func__, ui32MemClockSpeed, ui32MemClockSpeed/1000000));
+
+	/* Setup GPU PLL's */
+	ui32CoreClockSpeed = SysGetPlatoCoreClockSpeed();
+	ui32CorePLLClockSpeed = SysGetPlatoPLLClockSpeed(ui32CoreClockSpeed);
+
+	ui32CoreClockPLLCtrl0 = (GetPlatoPLLInt(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FBDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 &= PLATO_CR_GPU_PLL_FBDIV_MASK;
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_REFDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV1_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV2_SHIFT);
+
+	ui32CoreClockPLLCtrl1 = (GetPlatoPLLFrac(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FRAC_SHIFT);
+	ui32CoreClockPLLCtrl1 &= PLATO_CR_GPU_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1, ui32CoreClockPLLCtrl1);
+
+	PVR_LOG(("- %s: GPU clock: %u Hz (%u MHz)",
+			 __func__, ui32CoreClockSpeed,
+			 ui32CoreClockSpeed / 1000000));
+	PVR_LOG(("- %s: GPU-PLL clock: %u Hz (%u MHz)",
+			 __func__, ui32CorePLLClockSpeed,
+			 ui32CorePLLClockSpeed / 1000000));
+
+#if defined(ENABLE_PLATO_HDMI)
+	{
+		IMG_UINT32 ui32HdmiCecV0 = GetPlatoHDMICECV0Div0(ui32CorePLLClockSpeed);
+		IMG_UINT32 ui32HdmiCecV1 = GetPlatoHDMICECV1Div0(ui32CorePLLClockSpeed);
+		IMG_UINT32 ui32HdmiCecV2 = GetPlatoHDMICECV2Div0(ui32CorePLLClockSpeed);
+
+		PVR_LOG(("- %s: Setting HDMI CEC dividers:", __func__));
+		PVR_LOG(("- %s:		Hdmi CEC div0: %u(+1)", __func__, ui32HdmiCecV0));
+		PVR_LOG(("- %s:		Hdmi CEC div1: %u(+1)", __func__, ui32HdmiCecV1));
+		PVR_LOG(("- %s:		Hdmi CEC div2: %u(+1)", __func__, ui32HdmiCecV2));
+
+		/* Setup HDMI CEC clock outputs */
+		ui32HDMICECClockCtrl = 0;
+		ui32HDMICECClockCtrl |= (ui32HdmiCecV0 << PLATO_CR_HDMICECV0_DIV_0_SHIFT);
+		ui32HDMICECClockCtrl |= (ui32HdmiCecV1 << PLATO_CR_HDMICECV1_DIV_0_SHIFT);
+		ui32HDMICECClockCtrl |= (ui32HdmiCecV2 << PLATO_CR_HDMICECV2_DIV_0_SHIFT);
+
+		OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl);
+		Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl, ui32HDMICECClockCtrl);
+
+		OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1);
+		Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1, 0x1);
+	}
+#endif
+
+	PLATO_CHECKPOINT;
+
+	/* Waiting for DDR and GPU PLL's to lock */
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x3, 0x3, -1 , 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01, 0x01);
+	/* PLL Lock is done */
+
+	OSWaitus(1000);
+
+	/* Enabling gated clock output for DDR A/B */
+	ui32MemClockCtrl = (1 << PLATO_CR_DDRAG_GATE_EN_SHIFT) | (1 << PLATO_CR_DDRBG_GATE_EN_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl, ui32MemClockCtrl);
+
+	/* Enabling gated clock output for GPU and dividing the clock */
+	ui32CoreClockCtrl = (1 << PLATO_CR_GPUG_GATE_EN_SHIFT);
+	ui32CoreClockCtrl |= (GetPlatoGPUVDiv0(ui32CorePLLClockSpeed, ui32CoreClockSpeed) << PLATO_CR_GPUV_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl, ui32CoreClockCtrl);
+
+	PLATO_CHECKPOINT;
+
+	OSWaitus(100);
+
+	/* Enabling PDP gated clock output >= 165 MHz for <= 1080p */
+	ui32PDPClockCtrl = (1 << PLATO_CR_PDPG_GATE_EN_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV0Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV0_DIV_0_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV1Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV1_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl, ui32PDPClockCtrl);
+
+	OSWaitus(100);
+
+	/*
+	 * Enabling HDMI gated clock output. PDP needs HDMI clocks on for framegrabber.
+	 *
+	 * NOTE: The dividers will be reconfigured in video.c,
+	 *       for now they are set to their highest values.
+	 */
+	ui32HDMIClockCtrl = (1 << PLATO_CR_HDMIG_GATE_EN_SHIFT);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV0_DIV_0_MASK);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV1_DIV_0_MASK);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl, ui32HDMIClockCtrl);
+
+	OSWaitus(100);
+
+	PVR_LOG(("%s: Enabled PDP and HDMI clocks", __func__));
+	PLATO_CHECKPOINT;
+
+	/* Now putting DRAM controller out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL,
+		PLATO_CR_DDR_A_DATA_RESET_N_MASK | PLATO_CR_DDR_A_CTRL_RESET_N_MASK |
+		PLATO_CR_DDR_B_DATA_RESET_N_MASK | PLATO_CR_DDR_B_CTRL_RESET_N_MASK);
+	OSWaitus(100);
+
+	/* Now putting DRAM controller into reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, PLATO_CR_DDR_A_CTRL_RESET_N_MASK | PLATO_CR_DDR_B_CTRL_RESET_N_MASK);
+
+	/* Always configure DDR A */
+	eError = ConfigPlatoDram(pvSystemDbgDdrAPublRegs, pvSystemDbgDdrACtrlRegs, pvSystemDbgAonRegs,
+		ui32BDLRSetupDddrA, PLATO_CR_DDR_A_DATA_RESET_N_MASK | PLATO_CR_DDR_A_CTRL_RESET_N_MASK);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("DDR Bank A setup failed. Init cannot proceed."));
+		return eError;
+	}
+
+	PVR_LOG(("- %s: Finished DDR A Setup", __func__));
+
+	/* Configure DDR B */
+	eError = ConfigPlatoDram(pvSystemDbgDdrBPublRegs, pvSystemDbgDdrBCtrlRegs, pvSystemDbgAonRegs,
+		ui32BDLRSetupDddrB, PLATO_CR_DDR_B_DATA_RESET_N_MASK | PLATO_CR_DDR_B_CTRL_RESET_N_MASK);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("DDR Bank B setup failed. Init cannot proceed."));
+		return eError;
+	}
+
+	PVR_LOG(("- %s: Finished DDR B Setup", __func__));
+
+	/* Getting GPU And DDR A/B out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000F12, 0x00000F12, -1, 100);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* setting CR_ISO_CTRL:CR_GPU_CLK_E */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_ISO_CTRL, 0x000001F);
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		eError = CheckPDPAccessibility(psDevData);
+	}
+
+	return eError;
+
+}
+#else
+
+static PVRSRV_ERROR ConfigPlatoSingleChannel(SYS_DATA *psDevData)
+{
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bPollPassed;
+
+	void* pvSystemDbgPeripRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_PERIP_OFFSET;
+	void* pvSystemDbgAonRegs	  = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_AON_OFFSET;
+	void* pvSystemDbgDdrACtrlRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_CTRL_OFFSET;
+	void* pvSystemDbgDdrAPublRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_DDR_A_PUBL_OFFSET;
+
+	IMG_UINT32 ui32MemClockSpeed, ui32MemPLLClockSpeed;
+	IMG_UINT32 ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl1, ui32MemClockCtrl;
+
+	IMG_UINT32 ui32CoreClockSpeed;
+	IMG_UINT32 ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl1, ui32CoreClockCtrl;
+
+#if defined(ENABLE_PLATO_HDMI)
+	IMG_UINT32 ui32HDMICECClockCtrl;
+#endif
+
+	IMG_UINT32 ui32PDPClockCtrl, ui32HDMIClockCtrl;
+
+	/* Plato Soft reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x40);
+	OSSleepms(1000);
+	bPollPassed = Poll(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x030, 0xF0);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	PLATO_CHECKPOINT;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_NOC_CLK_CTRL, 0x1, 0x1);
+
+	/* Setup DDR PLL's */
+	ui32MemClockSpeed = SysGetPlatoMemClockSpeed();
+	ui32MemPLLClockSpeed = ui32MemClockSpeed;
+
+	ui32MemClockPLLCtrl0 = (GetPlatoPLLInt(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FBDIV_SHIFT);
+	ui32MemClockPLLCtrl0 &= PLATO_CR_DDR_PLL_FBDIV_MASK;
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_REFDIV_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV1_SHIFT);
+	ui32MemClockPLLCtrl0 |= (1 << PLATO_CR_DDR_PLL_POSTDIV2_SHIFT);
+
+	ui32MemClockPLLCtrl1 = (GetPlatoPLLFrac(ui32MemPLLClockSpeed) << PLATO_CR_DDR_PLL_FRAC_SHIFT);
+	ui32MemClockPLLCtrl1 &= PLATO_CR_DDR_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_0, ui32MemClockPLLCtrl0, ui32MemClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_DDR_PLL_CTRL_1, ui32MemClockPLLCtrl1, ui32MemClockPLLCtrl1);
+
+	PVR_LOG(("- %s: DDR clock: %u", __func__, ui32MemClockSpeed));
+
+	/* Setup GPU PLL's */
+	ui32CoreClockSpeed = SysGetPlatoCoreClockSpeed();
+	ui32CorePLLClockSpeed = SysGetPlatoPLLClockSpeed(ui32CoreClockSpeed);
+
+	ui32CoreClockPLLCtrl0 = (GetPlatoPLLInt(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FBDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 &= PLATO_CR_GPU_PLL_FBDIV_MASK;
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_REFDIV_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV1_SHIFT);
+	ui32CoreClockPLLCtrl0 |= (1 << PLATO_CR_GPU_PLL_POSTDIV2_SHIFT);
+
+	ui32CoreClockPLLCtrl1 = (GetPlatoPLLFrac(ui32CorePLLClockSpeed) << PLATO_CR_GPU_PLL_FRAC_SHIFT);
+	ui32CoreClockPLLCtrl1 &= PLATO_CR_GPU_PLL_FRAC_MASK;
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_0, ui32CoreClockPLLCtrl0, ui32CoreClockPLLCtrl0);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_GPU_PLL_CTRL_1, ui32CoreClockPLLCtrl1, ui32CoreClockPLLCtrl1);
+
+	PVR_LOG(("- %s: GPU clock: %u", __func__, ui32CoreClockSpeed));
+
+#if defined(ENABLE_PLATO_HDMI)
+	/* Setup HDMI CEC clock outputs */
+	ui32HDMICECClockCtrl = 0;
+	ui32HDMICECClockCtrl |= (GetPlatoHDMICECV0Div0(ui32CorePLLClockSpeed) << PLATO_CR_HDMICECV0_DIV_0_SHIFT);
+	ui32HDMICECClockCtrl |= (GetPlatoHDMICECV1Div0(ui32CorePLLClockSpeed) << PLATO_CR_HDMICECV1_DIV_0_SHIFT);
+	ui32HDMICECClockCtrl |= (GetPlatoHDMICECV2Div0(ui32CorePLLClockSpeed) << PLATO_CR_HDMICECV2_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CEC_CLK_CTRL, ui32HDMICECClockCtrl, ui32HDMICECClockCtrl);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_I2C_CLK_CTRL, 0x1, 0x1);
+#endif
+
+	PLATO_CHECKPOINT;
+
+	/* Waiting for DDR and GPU PLL's to lock */
+	PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_STATUS, 0x3, 0x3, -1 , 10);
+
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01);
+	Poll(pvSystemDbgAonRegs, PLATO_AON_CR_PLL_BYPASS, 0x01, 0x01);
+	/* PLL Lock is done */
+
+	OSWaitus(100);
+
+	/* Enabling gated clock output for DDR A/B */
+	ui32MemClockCtrl = (1 << PLATO_CR_DDRAG_GATE_EN_SHIFT) | (1 << PLATO_CR_DDRBG_GATE_EN_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_DDR_CLK_CTRL, ui32MemClockCtrl, ui32MemClockCtrl);
+
+	/* Enabling gated clock output for GPU and dividing the clock */
+	ui32CoreClockCtrl = (1 << PLATO_CR_GPUG_GATE_EN_SHIFT);
+	ui32CoreClockCtrl |= (GetPlatoGPUVDiv0(ui32CorePLLClockSpeed, ui32CoreClockSpeed) << PLATO_CR_GPUV_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_GPU_CLK_CTRL, ui32CoreClockCtrl, ui32CoreClockCtrl);
+
+	PLATO_CHECKPOINT;
+
+	OSWaitus(1000);
+
+	/* Enabling PDP gated clock output >= 165 MHz for <= 1080p */
+	ui32PDPClockCtrl = (1 << PLATO_CR_PDPG_GATE_EN_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV0Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV0_DIV_0_SHIFT);
+	ui32PDPClockCtrl |= (GetPlatoPDPV1Div0(ui32CorePLLClockSpeed) << PLATO_CR_PDPV1_DIV_0_SHIFT);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_PDP_CLK_CTRL, ui32PDPClockCtrl, ui32PDPClockCtrl);
+
+	OSWaitus(100);
+
+	/*
+	 * Enabling HDMI gated clock output. PDP needs HDMI clocks on for framegrabber.
+	 *
+	 * NOTE: The dividers will be reconfigured in video.c,
+	 *       for now they are set to their highest values.
+	 */
+	ui32HDMIClockCtrl = (1 << PLATO_CR_HDMIG_GATE_EN_SHIFT);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV0_DIV_0_MASK);
+	ui32HDMIClockCtrl |= (PLATO_CR_HDMIV1_DIV_0_MASK);
+
+	OSWriteHWReg32(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl);
+	Poll(pvSystemDbgPeripRegs, PLATO_TOP_CR_HDMI_CLK_CTRL, ui32HDMIClockCtrl, ui32HDMIClockCtrl);
+
+	OSWaitus(100);
+
+	PVR_LOG(("%s: Enabled PDP and HDMI clocks", __func__));
+
+	PLATO_CHECKPOINT;
+
+	/* Now putting DRAM A controller out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0300);
+	OSWaitus(100);
+
+	/* Now putting DRAM A controller into reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x0200);
+
+	/* Configure DRAM A control and publ regs */
+	eError = ConfigPlatoDram(pvSystemDbgDdrAPublRegs, pvSystemDbgDdrACtrlRegs, pvSystemDbgAonRegs, PLATO_CR_DDR_A_DATA_RESET_N_MASK | PLATO_CR_DDR_A_CTRL_RESET_N_MASK);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("DDR Bank setup failed. Init cannot proceed."));
+		return eError;
+	}
+	PVR_LOG(("- %s: Finished DDR A Setup", __func__));
+
+	/* Getting GPU and DDR A out of reset */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312);
+	bPollPassed = PollPr(pvSystemDbgAonRegs, PLATO_AON_CR_RESET_CTRL, 0x00000312, 0x00000312, -1, 100);
+	if (!bPollPassed)
+	{
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+	/* setting CR_ISO_CTRL:CR_GPU_CLK_E */
+	OSWriteHWReg32(pvSystemDbgAonRegs, PLATO_AON_CR_ISO_CTRL, 0x000001F);
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		eError = CheckPDPAccessibility(psDevData);
+	}
+
+	return eError;
+}
+
+#endif
+
+static PVRSRV_ERROR ConfigPlatoHw(SYS_DATA *psDevData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32MaxRestartCount = PLATO_PDP_RELOADS_MAX;
+	IMG_UINT32 ui32RestartCount = 0;
+
+	PLATO_CHECKPOINT;
+
+	/* Config Plato until PDP registers become accessible */
+	do
+	{
+		#if defined(PLATO_DUAL_CHANNEL_DDR)
+		eError = ConfigPlatoDualChannel(psDevData);
+		#else
+		eError = ConfigPlatoSingleChannel(psDevData);
+		#endif
+
+		ui32RestartCount++;
+
+		if (eError == PVRSRV_OK)
+		{
+			break;
+		}
+
+		PVR_LOG(("PDP registers inaccessible after %d out of %d tries.",
+					ui32RestartCount, ui32MaxRestartCount));
+
+	} while (ui32RestartCount < ui32MaxRestartCount &&
+			 eError == PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE);
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData) && eError == PVRSRV_OK)
+	{
+		PVR_LOG(("Got correct PDP register values after %d tries!",
+					ui32RestartCount));
+	}
+
+	if (ui32RestartCount == ui32MaxRestartCount && eError == PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE)
+	{
+		PVR_LOG(("Failed to access PDP registers after %d tries!",
+					ui32RestartCount));
+	}
+
+	return eError;
+}
+
+#endif
+
+static PVRSRV_ERROR ConfigPlato(SYS_DATA *psDevData)
+{
+	#if defined(EMULATOR)
+	return ConfigPlatoEmu(psDevData);
+	#elif defined(VIRTUAL_PLATFORM)
+	return PVRSRV_OK;
+	#else
+	return ConfigPlatoHw(psDevData);
+	#endif
+}
+
+static PVRSRV_ERROR PlatoLocalMemoryTest(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_UINT64 i, j = 0;
+	IMG_UINT32 tmp = 0;
+	IMG_UINT32 chunk = sizeof(IMG_UINT32) * 10;
+
+	IMG_UINT64 ui64TestMemoryBase = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr;
+	IMG_UINT64 ui64TestMemorySize = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+
+	PVR_LOG(("- %s: Starting Local memory test from "
+			 "0x%llx to 0x%llx (in CPU space)",
+			 __func__, ui64TestMemoryBase,
+			 ui64TestMemoryBase + ui64TestMemorySize));
+
+	while (j < ui64TestMemorySize)
+	{
+		IMG_CPU_PHYADDR myPaddr;
+		IMG_UINT32* pui32Virt;
+
+		myPaddr.uiAddr = ui64TestMemoryBase + j;
+		pui32Virt = OSMapPhysToLin(myPaddr, chunk, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		for (i = 0; i < chunk/sizeof(IMG_UINT32); i++)
+		{
+			*(pui32Virt + i) = 0xdeadbeef;
+			OSWriteMemoryBarrier(pui32Virt);
+			tmp = *(pui32Virt + i);
+			if (tmp != 0xdeadbeef)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Local memory read-write test failed at address=0x%llx: written 0x%x, read 0x%x",
+						ui64TestMemoryBase + ((i * sizeof(IMG_UINT32)) + j) , (IMG_UINT32) 0xdeadbeef, tmp));
+
+				OSUnMapPhysToLin(pui32Virt, chunk);
+				return PVRSRV_ERROR_SYSTEM_LOCAL_MEMORY_INIT_FAIL;
+			}
+		}
+
+		OSUnMapPhysToLin(pui32Virt, chunk);
+
+		j+= (1024 * 1024 * 500);
+	}
+
+	PVR_LOG(("Local memory read-write test passed!"));
+
+	return PVRSRV_OK;
+}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+/* Queries PCI BAR register, reads its base address and returns the reported size */
+static PVRSRV_ERROR AcquireLocalMappableMemory(IMG_HANDLE hPCI,
+											   IMG_PUINT64 pui64MemCpuPAddr,
+											   IMG_PUINT64 pui64MemSize)
+{
+	IMG_UINT16 uiVendorID, uiDevID;
+	IMG_UINT64 uiBarSize;
+
+	PVRSRV_ERROR eError;
+
+	OSPCIGetVendorDeviceIDs(hPCI, &uiVendorID, &uiDevID);
+	if (uiDevID != SYS_RGX_DEV_DEVICE_ID)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected device ID 0x%X",
+				 __func__, uiDevID));
+
+		return PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+	}
+
+	uiBarSize = OSPCIAddrRangeLen(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+
+	/* Reserve the address region for whole BAR */
+	eError = OSPCIRequestAddrRegion(hPCI, SYS_DEV_MEM_PCI_BASENUM, 0, uiBarSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device memory region not available",
+				 __func__));
+
+		return eError;
+	}
+
+	/* Clear any BIOS-configured MTRRs */
+	eError = OSPCIClearResourceMTRRs(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Failed to clear BIOS MTRRs (Error %d)",
+				 __func__, eError));
+		/* Soft-fail, the driver can limp along. */
+	}
+
+	*pui64MemCpuPAddr = OSPCIAddrRangeStart(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+	*pui64MemSize = uiBarSize;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void ReleaseLocalMappableMemory(IMG_HANDLE hPCI,
+											  IMG_UINT64 ui64MemCpuPAddr,
+											  IMG_UINT64 ui64MemSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64MemCpuPAddr);
+
+	OSPCIReleaseAddrRegion(hPCI, SYS_DEV_MEM_PCI_BASENUM, 0, ui64MemSize);
+
+	OSPCIReleaseResourceMTRRs(hPCI, SYS_DEV_MEM_PCI_BASENUM);
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+
+static PVRSRV_ERROR InitLocalMemory(SYS_DATA *psDevData,
+									PHYS_HEAP_CONFIG *pasPhysHeaps,
+									IMG_UINT32 uiPhysHeapCount,
+									IMG_HANDLE hPhysHeapPrivData)
+{
+	IMG_UINT64 ui64MappedMemSize = psDevData->ui64MappedMemSize;
+	IMG_UINT64 ui64MappedMemCpuPAddr = psDevData->ui64MappedMemCpuPAddr;
+	IMG_UINT64 ui64MappedMemDevPAddr;
+	PHYS_HEAP_CONFIG *psPhysHeap;
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	IMG_UINT64 uiFwCarveoutSize;
+#if defined(SUPPORT_AUTOVZ)
+	/* Carveout out enough LMA memory to hold the heaps of
+	 * all supported OSIDs and the FW page tables */
+	uiFwCarveoutSize = (RGX_NUM_OS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE) +
+						RGX_FIRMWARE_MAX_PAGETABLE_SIZE;
+#elif defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* Carveout out enough LMA memory to hold the heaps of all supported OSIDs */
+	uiFwCarveoutSize = (RGX_NUM_OS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+#else
+	/* Create a memory carveout just for the Host's Firmware heap.
+	 * Guests will allocate their own physical memory. */
+	uiFwCarveoutSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+#endif
+#endif /* (RGX_NUM_OS_SUPPORTED > 1) */
+
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->pszPDumpMemspaceName = "LMA";
+	psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+
+	/* Configure mappable heap region */
+	psPhysHeap->sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+	if (!PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		/*
+		 * With a BAR size that's greater than the actual memory size,
+		 * move base CPU address of the heap region to use last
+		 * aliased part of odd region and first aliased part of even region.
+		 *
+		 * This allows to use full available memory in one contiguous heap region.
+		 */
+		ui64MappedMemCpuPAddr +=
+			PLATO_DRAM_SPLIT_ADDR - (SYS_DEV_MEM_REGION_SIZE >> 1) - PLATO_DDR_DEV_PHYSICAL_BASE;
+
+		/*
+		 * As we can't read the actual memory size from the BAR we limit it to
+		 * a value set during build time.
+		 */
+		ui64MappedMemSize = SYS_DEV_MEM_REGION_SIZE;
+
+		ui64MappedMemDevPAddr = PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE;
+	}
+	else
+	{
+		ui64MappedMemDevPAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+		/*
+		 * With a BAR size less than the actual memory size (8GB),
+		 * we need to dynamically calculate the device base address
+		 * that the PCI memory window is pointing to. The address depends on
+		 * what address in host memory space the memory BAR was assigned by the BIOS.
+		 *
+		 * Bits x to 34 in host CPU base address decide on where
+		 * within the DRAM region the BAR points to, where x is the shift calculated
+		 * below based on the mapped memory size (BAR).
+		 */
+		{
+			IMG_UINT32 ui32Shift = 0;
+			IMG_UINT64 ui64Mask;
+			ui32Shift = __builtin_ffsll(psDevData->ui64MappedMemSize)-1;
+			PVR_ASSERT(ui32Shift < 35U);
+			ui64Mask = (1ULL << (35U-ui32Shift)) - 1ULL;
+			PVR_ASSERT(ui64Mask != 0);
+
+			ui64MappedMemDevPAddr += ui64MappedMemCpuPAddr & (ui64Mask << ui32Shift);
+		}
+
+		/*
+		 * Our aliased address can point anywhere in the 32GB - 64GB range,
+		 * we now need to determine the closest aliased address to the split point (48GB).
+		 * This is done by first finding the offset from the previous segment (mod below)
+		 * and adding it to either the DRAM split point or the start of the aliased
+		 * region that's closest to the split point.
+		 */
+		if (ui64MappedMemDevPAddr >= PLATO_DRAM_SPLIT_ADDR)
+		{
+			ui64MappedMemDevPAddr = PLATO_DRAM_SPLIT_ADDR + (ui64MappedMemDevPAddr % PLATO_DDR_ALIASED_DEV_SEGMENT_SIZE);
+		}
+		else
+		{
+			ui64MappedMemDevPAddr = PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE + (ui64MappedMemDevPAddr % PLATO_DDR_ALIASED_DEV_SEGMENT_SIZE);
+		}
+	}
+
+	/* Confidence check on mapped device physical address */
+	PVR_ASSERT(ui64MappedMemDevPAddr >= PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE && ui64MappedMemDevPAddr < PLATO_DDR_ALIASED_DEV_PHYSICAL_END);
+
+	psPhysHeap->sStartAddr.uiAddr = ui64MappedMemCpuPAddr;
+	psPhysHeap->sCardBase.uiAddr = ui64MappedMemDevPAddr;
+	psPhysHeap->uiSize = ui64MappedMemSize;
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	psPhysHeap->uiSize -= uiFwCarveoutSize;
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+	psPhysHeap->uiSize -= RGX_PLATO_RESERVE_DC_MEM_SIZE;
+#endif
+
+	/* Setup non-mappable region if BAR size is less than actual memory size (8GB) */
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		IMG_UINT64 ui64PrecedingRegionBase = 0;
+		IMG_UINT64 ui64PrecedingRegionSize = 0;
+		IMG_UINT64 ui64FollowingRegionBase = 0;
+		IMG_UINT64 ui64FollowingRegionSize = 0;
+
+		psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_NON_MAPPABLE];
+		psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeap->pszPDumpMemspaceName = "LMA";
+		psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+		psPhysHeap->hPrivData = hPhysHeapPrivData;
+		psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_PRIVATE;
+		psPhysHeap->sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+		/*
+		 * If mapped region is not at the base of memory,
+		 * then it is preceded by a non-mappable region
+		 */
+		 ui64PrecedingRegionBase = PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE;
+		 ui64PrecedingRegionSize = ui64MappedMemDevPAddr - ui64PrecedingRegionBase;
+
+		/*
+		 * If mapped region is not at the end of memory,
+		 * then it is followed by a non-mappable region
+		 */
+		ui64FollowingRegionBase = ui64MappedMemDevPAddr + psDevData->ui64MappedMemSize;
+		ui64FollowingRegionSize =
+			PLATO_DDR_ALIASED_DEV_PHYSICAL_END - (ui64MappedMemDevPAddr + psDevData->ui64MappedMemSize);
+
+		/* Use only bigger region for now */
+		if (ui64FollowingRegionSize > ui64PrecedingRegionSize)
+		{
+			psPhysHeap->sCardBase.uiAddr = ui64FollowingRegionBase;
+			psPhysHeap->uiSize = ui64FollowingRegionSize;
+		}
+		else
+		{
+			psPhysHeap->sCardBase.uiAddr = ui64PrecedingRegionBase;
+			psPhysHeap->uiSize = ui64PrecedingRegionSize;
+		}
+
+		psPhysHeap->sStartAddr.uiAddr = 0;
+
+		PVR_LOG(("Added non-mappable local memory region. Base = 0x%016llx, Size=0x%016llx",
+					psPhysHeap->sCardBase.uiAddr,
+					psPhysHeap->uiSize));
+
+		PVR_ASSERT(psPhysHeap->uiSize < SYS_DEV_MEM_REGION_SIZE);
+	}
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_FW_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->pszPDumpMemspaceName = "LMA";
+	psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+
+	psPhysHeap->sCardBase.uiAddr =
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+
+	psPhysHeap->sStartAddr.uiAddr =
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+	psPhysHeap->uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->pszPDumpMemspaceName = "LMA";
+	psPhysHeap->psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_DISPLAY;
+
+	psPhysHeap->sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+
+	psPhysHeap->sStartAddr.uiAddr =
+#if (RGX_NUM_OS_SUPPORTED > 1)
+		pasPhysHeaps[PHYS_HEAP_ID_FW_LOCAL].sStartAddr.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_FW_LOCAL].uiSize;
+#else
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+		pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+#endif
+	psPhysHeap->uiSize = RGX_PLATO_RESERVE_DC_MEM_SIZE;
+#endif
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHostMemory(SYS_DATA *psDevData,
+								   PHYS_HEAP_CONFIG *pasPhysHeaps,
+								   IMG_UINT32 uiPhysHeapCount,
+								   IMG_HANDLE hPhysHeapPrivData)
+{
+	PHYS_HEAP_CONFIG *psPhysHeap;
+
+	PVR_ASSERT(uiPhysHeapCount == 1);
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_CPU_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeap->pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeap->psMemFuncs = &gsHostPhysHeapFuncs;
+	psPhysHeap->hPrivData = hPhysHeapPrivData;
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+	PVR_DPF((PVR_DBG_WARNING, "Initialising CPU_LOCAL UMA Host PhysHeaps"));
+#endif
+
+	psPhysHeap->sCardBase.uiAddr = PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHybridMemory(SYS_DATA *psDevData,
+									 PHYS_HEAP_CONFIG *pasPhysHeaps,
+									 IMG_UINT32 uiPhysHeapCount,
+									 IMG_HANDLE hPhysHeapPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiPhysHeapCount >= PHYS_HEAP_ID_PDP_LOCAL);
+
+	eError = InitHostMemory(psDevData, pasPhysHeaps,
+							1, hPhysHeapPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/*
+	 * InitLocalMemory should set up the correct heaps regardless of whether the
+	 * memory configuration is 'local' or 'hybrid'.
+	 */
+	eError = InitLocalMemory(psDevData, pasPhysHeaps,
+							 uiPhysHeapCount, hPhysHeapPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Adjust the pdump memory space names */
+	pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].pszPDumpMemspaceName = "LMA";
+	pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL].pszPDumpMemspaceName = "LMA";
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR InitMemory(SYS_DATA *psDevData,
+							   PVRSRV_DEVICE_CONFIG *psDevConfig,
+							   PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+							   IMG_UINT32 *puiPhysHeapCountOut)
+{
+	IMG_UINT32 uiHeapCount = 1;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	uiHeapCount++;
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	uiHeapCount++;
+#endif
+
+#if (RGX_NUM_OS_SUPPORTED > 1)
+#if (PLATO_MEMORY_CONFIG != PLATO_MEMORY_HYBRID)
+#error "VZ support implemented only hybrid memory configuration (PLATO_MEMORY_HYBRID)"
+#endif
+	uiHeapCount++;
+#endif
+
+	if (PLATO_HAS_NON_MAPPABLE(psDevData))
+	{
+		uiHeapCount++;
+	}
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+	eError = InitLocalMemory(psDevData, pasPhysHeaps,
+							 uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	eError = InitHostMemory(psDevData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	eError = InitHybridMemory(psDevData, pasPhysHeaps,
+							  uiHeapCount, psDevConfig);
+#endif
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void DeInitMemory(PHYS_HEAP_CONFIG *pasPhysHeaps,
+								IMG_UINT32 uiPhysHeapCount)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr;
+
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr;
+		}
+	}
+}
+
+void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr)
+{
+	IMG_BOOL bInvalidAddress=IMG_FALSE;
+	IMG_DEV_PHYADDR sMappableRegionCardBase;
+	IMG_UINT64	uiMaxSize;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	sMappableRegionCardBase = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase;
+	uiMaxSize = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uiSize;
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	uiMaxSize += RGX_PLATO_RESERVE_DC_MEM_SIZE;
+#endif
+
+	if (psDevPAddr[0].uiAddr > (sMappableRegionCardBase.uiAddr + uiMaxSize) ||
+	    psDevPAddr[0].uiAddr < sMappableRegionCardBase.uiAddr)
+	{
+		bInvalidAddress = IMG_TRUE;
+	}
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr;
+
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			if (psDevPAddr[ui32Idx].uiAddr > (sMappableRegionCardBase.uiAddr + uiMaxSize) ||
+			    psDevPAddr[ui32Idx].uiAddr < sMappableRegionCardBase.uiAddr)
+			{
+				bInvalidAddress = IMG_TRUE;
+			}
+
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sCardBase.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].sStartAddr.uiAddr;
+		}
+	}
+
+	if (bInvalidAddress)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"/*****************************WARNING**********************************************/"));
+		PVR_DPF((PVR_DBG_ERROR,"%s: Given Device Physical Address belongs to non-mappable region",__func__));
+		PVR_DPF((PVR_DBG_ERROR,"Accessing the buffer may lead to unexpected behaviour"));
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_ERROR,"Input Device Physical Address count: %d", ui32NumOfAddr));
+		{
+			IMG_UINT32 ui32Idx;
+			for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			{
+				if (psDevPAddr[ui32Idx].uiAddr > (sMappableRegionCardBase.uiAddr + uiMaxSize) ||
+				    psDevPAddr[ui32Idx].uiAddr < sMappableRegionCardBase.uiAddr)
+				{
+					PVR_DPF((PVR_DBG_ERROR,"Device Physical Address : 0x%llx", psDevPAddr[ui32Idx].uiAddr));
+				}
+			}
+		}
+		dump_stack();
+#endif
+		PVR_DPF((PVR_DBG_ERROR,"/**********************************************************************************/"));
+	}
+
+}
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+		}
+	}
+}
+
+void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+		}
+	}
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   SYS_DATA *psDevData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PVRSRV_ERROR eError;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	/* Set up the RGX timing information */
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed;
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Set up the device config */
+	eError = InitMemory(psDevData, psDevConfig, &psDevConfig->pasPhysHeaps,
+						&psDevConfig->ui32PhysHeapCount);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psDevConfig);
+
+		return eError;
+	}
+
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevConfig->pszName = PLATO_SYSTEM_NAME;
+	psDevConfig->pszVersion = GetDeviceVersionString(psDevData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr =
+		OSPCIAddrRangeStart(psDevData->hRGXPCI, SYS_PLATO_REG_PCI_BASENUM);
+	psDevConfig->sRegsCpuPBase.uiAddr += SYS_PLATO_REG_RGX_OFFSET;
+
+	psDevConfig->ui32RegsSize = SYS_PLATO_REG_RGX_SIZE;
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+	psDevConfig->bHasNonMappableLocalMemory = PLATO_HAS_NON_MAPPABLE(psDevData);
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psDevData;
+
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	/* device error notify callback function */
+	psDevConfig->pfnSysDevErrorNotify = NULL;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->pszVersion)
+	{
+		OSFreeMem(psDevConfig->pszVersion);
+	}
+
+	DeInitMemory(psDevConfig->pasPhysHeaps, psDevConfig->ui32PhysHeapCount);
+
+	/*
+	 * The device config, RGX data and RGX timing info are part of the same
+	 * allocation so do only one free.
+	 */
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psDevData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(pvOSDevice);
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevData = OSAllocZMem(sizeof(*psDevData));
+	if (psDevData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psDevData->hRGXPCI = OSPCISetDev(TO_PCI_COOKIE(pvOSDevice), HOST_PCI_INIT_FLAGS);
+	if (!psDevData->hRGXPCI)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire PCI device",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+		goto ErrorFreeDevData;
+	}
+
+	/*
+		Reserve and map system registers (does not include DMA) up to PDP
+	 */
+	psDevData->uiSystemRegSize = SYS_PLATO_REG_PDP_OFFSET - SYS_PLATO_REG_SYS_OFFSET;
+
+	eError = PlatoMapRegisters(psDevData, SYS_PLATO_REG_PCI_BASENUM, SYS_PLATO_REG_SYS_OFFSET,
+		psDevData->uiSystemRegSize, &psDevData->pvSystemRegCpuVBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers",
+				 __func__));
+		goto ErrorPCIReleaseDevice;
+	}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	eError = AcquireLocalMappableMemory(psDevData->hRGXPCI,
+										&psDevData->ui64MappedMemCpuPAddr,
+										&psDevData->ui64MappedMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorPlatoRegMap;
+	}
+
+	PVR_LOG(("Detected Plato mappable memory size: 0x%llx",
+			 psDevData->ui64MappedMemSize));
+#endif
+
+#if !defined(VIRTUAL_PLATFORM)
+	eError = ConfigPlato(psDevData);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseLocalMappableMemory;
+	}
+#endif
+
+	eError = OSPCIIRQ(psDevData->hRGXPCI, &psDevData->ui32IRQ);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't get IRQ", __func__));
+
+		goto ErrorReleaseLocalMappableMemory;
+	}
+
+	/* Register our handler */
+	eError = OSInstallSystemLISR(&psDevData->hLISR,
+								 psDevData->ui32IRQ,
+								 PLATO_SYSTEM_NAME,
+								 SystemISRHandler,
+								 psDevData,
+								 SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to install the system device interrupt handler",
+				 __func__));
+
+		goto ErrorReleaseLocalMappableMemory;
+	}
+
+	eError = DeviceConfigCreate(pvOSDevice, psDevData, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorOSUninstallSystemLISR;
+	}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	/*
+	 * When using local memory perform a read-write test
+	 * to confirm that the memory has been configured correctly
+	 */
+	eError = PlatoLocalMemoryTest(psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorDeviceConfigDestroy;
+	}
+#endif
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+ErrorDeviceConfigDestroy:
+	DeviceConfigDestroy(psDevConfig);
+#endif
+
+ErrorOSUninstallSystemLISR:
+	OSUninstallSystemLISR(psDevData->hLISR);
+
+ErrorReleaseLocalMappableMemory:
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	ReleaseLocalMappableMemory(psDevData->hRGXPCI,
+							   psDevData->ui64MappedMemCpuPAddr,
+							   psDevData->ui64MappedMemSize);
+
+ErrorPlatoRegMap:
+#endif
+#if !defined(VIRTUAL_PLATFORM)
+	PlatoUnmapRegisters(psDevData, psDevData->pvSystemRegCpuVBase, SYS_PLATO_REG_PCI_BASENUM,
+		SYS_PLATO_REG_SYS_OFFSET, psDevData->uiSystemRegSize);
+#endif
+
+ErrorPCIReleaseDevice:
+	OSPCIReleaseDev(psDevData->hRGXPCI);
+
+ErrorFreeDevData:
+	OSFreeMem(psDevData);
+
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)psDevConfig->hSysData;
+
+	DeviceConfigDestroy(psDevConfig);
+
+	OSUninstallSystemLISR(psDevData->hLISR);
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	ReleaseLocalMappableMemory(psDevData->hRGXPCI,
+							   psDevData->ui64MappedMemCpuPAddr,
+							   psDevData->ui64MappedMemSize);
+#endif
+
+#if !defined(VIRTUAL_PLATFORM)
+	PlatoUnmapRegisters(psDevData, psDevData->pvSystemRegCpuVBase, SYS_PLATO_REG_PCI_BASENUM,
+		SYS_PLATO_REG_SYS_OFFSET, psDevData->uiSystemRegSize);
+#endif
+
+	OSPCIReleaseDev(psDevData->hRGXPCI);
+	OSFreeMem(psDevData);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psDevData = (SYS_DATA *)hSysData;
+	IMG_UINT32 ui32InterruptFlag;
+
+	switch (ui32IRQ)
+	{
+		case PLATO_IRQ_GPU:
+			ui32InterruptFlag = (0x1 << PLATO_GPU_INT_SHIFT);
+			break;
+		case PLATO_IRQ_PDP:
+			ui32InterruptFlag = (0x1 << PLATO_PDP_INT_SHIFT);
+			break;
+		case PLATO_IRQ_HDMI:
+			ui32InterruptFlag = (0x1 << PLATO_HDMI_INT_SHIFT);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d",
+					 __func__, ui32IRQ));
+			return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	if (psDevData->sInterruptData[ui32IRQ].pfnLISR)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ISR for %s already installed!",
+				 __func__, pszName));
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+
+	psDevData->sInterruptData[ui32IRQ].psSysData		= psDevData;
+	psDevData->sInterruptData[ui32IRQ].pszName		= pszName;
+	psDevData->sInterruptData[ui32IRQ].pfnLISR		= pfnLISR;
+	psDevData->sInterruptData[ui32IRQ].pvData		= pvData;
+	psDevData->sInterruptData[ui32IRQ].ui32InterruptFlag	= ui32InterruptFlag;
+
+	*phLISRData = &psDevData->sInterruptData[ui32IRQ];
+
+	EnableInterrupt(psDevData, psDevData->sInterruptData[ui32IRQ].ui32InterruptFlag);
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, psDevData->ui32IRQ));
+
+	return PVRSRV_OK;
+
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+
+	SYS_INTERRUPT_DATA *psInterruptData = (SYS_INTERRUPT_DATA *)hLISRData;
+
+	PVR_ASSERT(psInterruptData);
+
+	PVR_LOG(("Uninstalling device LISR %s on IRQ %d", psInterruptData->pszName, ((SYS_DATA *)psInterruptData->psSysData)->ui32IRQ));
+
+	/* Disable interrupts for this device */
+#if !defined(VIRTUAL_PLATFORM)
+	DisableInterrupt(psInterruptData->psSysData, psInterruptData->ui32InterruptFlag);
+#endif
+
+	/* Reset the interrupt data */
+	psInterruptData->pszName = NULL;
+	psInterruptData->psSysData = NULL;
+	psInterruptData->pfnLISR = NULL;
+	psInterruptData->pvData = NULL;
+	psInterruptData->ui32InterruptFlag = 0;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysGetPlatoGPUTemp(SYS_DATA * psDevData, IMG_UINT32 *ui32Temp)
+{
+	IMG_UINT32 ui32Value = 0;
+#if defined(PLATO_TEMPA_SENSOR_IMPLEMENTED)
+	void* pvSystemTempRegs;
+
+	/* Temp sensor A starts at 0x80000 within peripheral regs */
+	pvSystemTempRegs = psDevData->pvSystemRegCpuVBase + SYS_PLATO_REG_TEMPA_OFFSET;
+
+	OSWriteHWReg32(pvSystemTempRegs, 0x04, 0x3);
+	Poll(pvSystemTempRegs, 0x10, (1 << 16), (1 << 16));
+	ui32Value = OSReadHWReg32(pvSystemTempRegs, 0x10) & 0xFFF;
+
+	ui32Value = ((ui32Value * 352) / 4096) - 109;
+#endif  // PLATO_TEMPA_SENSOR_IMPLEMENTED
+
+	*ui32Temp = ui32Value;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.h
new file mode 100644
index 000000000000..479f9f990135
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysconfig.h
@@ -0,0 +1,135 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSCONFIG_H)
+#define SYSCONFIG_H
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "interrupt_support.h"
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+#define MAX_SYSTEMS 32
+
+typedef struct _SYS_INTERRUPT_DATA_
+{
+	void			*psSysData;
+	const IMG_CHAR	*pszName;
+	PFN_SYS_LISR	pfnLISR;
+	void			*pvData;
+	IMG_UINT32		ui32InterruptFlag;
+	IMG_UINT32		ui32IRQ;
+} SYS_INTERRUPT_DATA;
+
+typedef struct _SYS_DATA_ SYS_DATA;
+
+struct _SYS_DATA_
+{
+	IMG_HANDLE					hRGXPCI;
+
+	void						*pvSystemRegCpuVBase;
+	size_t						uiSystemRegSize;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	IMG_UINT64					ui64MappedMemCpuPAddr;
+	IMG_UINT64					ui64MappedMemSize;
+#endif
+
+	IMG_HANDLE					hLISR;
+	IMG_UINT32					ui32IRQ;
+
+	/* Rogue, PDP and HDMI */
+	SYS_INTERRUPT_DATA			sInterruptData[PLATO_IRQ_MAX];
+};
+
+/* Helpers for getting DDR/GPU/PLL clock speed */
+IMG_UINT32 SysGetPlatoMemClockSpeed(void);
+IMG_UINT32 SysGetPlatoCoreClockSpeed(void);
+IMG_UINT32 SysGetPlatoPLLClockSpeed(IMG_UINT32 ui32ClockSpeed);
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr);
+
+void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr);
+#endif
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr);
+
+void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr);
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+PVRSRV_ERROR PlatoMapRegisters(SYS_DATA *psDevData,
+	IMG_UINT32 ui32BaseNum,
+	IMG_UINT32 ui32Offset,
+	IMG_UINT32 ui32Size,
+	void ** pvCpuVAddr);
+
+void PlatoUnmapRegisters(SYS_DATA *psDevData,
+	void * pvBase,
+	IMG_UINT32 ui32BaseNum,
+	IMG_UINT32 ui32Offset,
+	IMG_UINT32 ui32Size);
+
+#if (PLATO_MEMORY_CONFIG != PLATO_MEMORY_LOCAL) && \
+	(PLATO_MEMORY_CONFIG != PLATO_MEMORY_HYBRID) && \
+	(PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#error "PLATO_MEMORY_CONFIG not valid"
+#endif
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* !defined(SYSCONFIG_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysinfo.h
new file mode 100644
index 000000000000..a7d2857d3c7c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/plato/sysinfo.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+#define SYS_RGX_DEV_VENDOR_ID	(0x1AEE)
+#define SYS_RGX_DEV_DEVICE_ID	(0x0003)
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM) || defined(EMULATOR)
+/* Emulator clock ~600 times slower than HW */
+#define MAX_HW_TIME_US                           (300000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (100000)
+#else
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#endif
+
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#endif /* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/Kbuild.mk
new file mode 100644
index 000000000000..14b67465936f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/Kbuild.mk
@@ -0,0 +1,45 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+	services/system/$(PVR_SYSTEM)/sysconfig.o
+
+ccflags-y += \
+	-I$(TOP)/services/system/rogue/common/env/linux
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.c
new file mode 100644
index 000000000000..df4613df5fe7
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.c
@@ -0,0 +1,746 @@
+/*************************************************************************/ /*!
+@File
+@Title			System Configuration
+@Copyright		Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description	System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_memallocflags.h"
+#include "syscommon.h"
+#include "power.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pci_support.h"
+#include "interrupt_support.h"
+#include "plato_drv.h"
+#include <linux/dma-mapping.h>
+
+#define PLATO_HAS_NON_MAPPABLE(sys) (sys->pdata->has_nonmappable == true)
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#define PHYS_HEAP_ID_CPU_LOCAL 0
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_FW_SHARED 1
+#define PHYS_HEAP_ID_CPU_LOCAL 2
+#if defined(SUPPORT_PLATO_DISPLAY)
+#define PHYS_HEAP_ID_PDP_LOCAL 3
+#define PHYS_HEAP_ID_NON_MAPPABLE 4
+#else
+#define PHYS_HEAP_ID_NON_MAPPABLE 3
+#endif
+
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+#define PHYS_HEAP_ID_GPU_LOCAL 0
+#define PHYS_HEAP_ID_FW_SHARED 1
+#if defined(SUPPORT_PLATO_DISPLAY)
+#define PHYS_HEAP_ID_PDP_LOCAL 2
+#define PHYS_HEAP_ID_NON_MAPPABLE 3
+#else
+#define PHYS_HEAP_ID_NON_MAPPABLE 2
+#endif
+#endif
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+static_assert(PHYS_HEAP_ID_PDP_LOCAL == PVRSRV_PHYS_HEAP_CONFIG_PDP_LOCAL_ID, "PDP heap ID mismatch.");
+#endif
+
+static struct plato_debug_register plato_noc_regs[] = {
+	{"NOC Offset 0x00", 0x00, 0},
+	{"NOC Offset 0x04", 0x04, 0},
+	{"NOC Offset 0x08", 0x08, 0},
+	{"NOC Offset 0x0C", 0x0C, 0},
+	{"NOC Offset 0x10", 0x10, 0},
+	{"NOC Offset 0x14", 0x14, 0},
+	{"NOC Offset 0x18", 0x18, 0},
+	{"NOC Offset 0x1C", 0x1C, 0},
+	{"NOC Offset 0x50", 0x50, 0},
+	{"NOC Offset 0x54", 0x54, 0},
+	{"NOC Offset 0x58", 0x58, 0},
+	{"DDR A Ctrl", SYS_PLATO_REG_NOC_DBG_DDR_A_CTRL_OFFSET, 0},
+	{"DDR A Data", SYS_PLATO_REG_NOC_DBG_DDR_A_DATA_OFFSET, 0},
+	{"DDR A Publ", SYS_PLATO_REG_NOC_DBG_DDR_A_PUBL_OFFSET, 0},
+	{"DDR B Ctrl", SYS_PLATO_REG_NOC_DBG_DDR_B_CTRL_OFFSET, 0},
+	{"DDR B Data", SYS_PLATO_REG_NOC_DBG_DDR_B_DATA_OFFSET, 0},
+	{"DDR B Publ", SYS_PLATO_REG_NOC_DBG_DDR_B_PUBL_OFFSET, 0},
+	{"Display S", SYS_PLATO_REG_NOC_DBG_DISPLAY_S_OFFSET, 0},
+	{"GPIO 0 S", SYS_PLATO_REG_NOC_DBG_GPIO_0_S_OFFSET, 0},
+	{"GPIO 1 S", SYS_PLATO_REG_NOC_DBG_GPIO_1_S_OFFSET, 0},
+	{"GPU S", SYS_PLATO_REG_NOC_DBG_GPU_S_OFFSET, 0},
+	{"PCI PHY", SYS_PLATO_REG_NOC_DBG_PCI_PHY_OFFSET, 0},
+	{"PCI Reg", SYS_PLATO_REG_NOC_DBG_PCI_REG_OFFSET, 0},
+	{"PCI S", SYS_PLATO_REG_NOC_DBG_PCI_S_OFFSET, 0},
+	{"Periph S", SYS_PLATO_REG_NOC_DBG_PERIPH_S_OFFSET, 0},
+	{"Ret Reg", SYS_PLATO_REG_NOC_DBG_RET_REG_OFFSET, 0},
+	{"Service", SYS_PLATO_REG_NOC_DBG_SERVICE_OFFSET, 0},
+};
+
+static struct plato_debug_register plato_aon_regs[] = {
+	{"AON Offset 0x0000", 0x0000, 0},
+	{"AON Offset 0x0070", 0x0070, 0},
+};
+
+typedef struct _SYS_DATA_ {
+	struct platform_device *pdev;
+	struct resource *registers;
+	struct plato_rogue_platform_data *pdata;
+} SYS_DATA;
+
+typedef struct {
+	struct device *psDev;
+	int iInterruptID;
+	void *pvData;
+	PFN_LISR pfnLISR;
+} LISR_DATA;
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	return NULL;
+}
+
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 i = 0;
+
+	PVR_DUMPDEBUG_LOG("------[ Plato System Debug ]------");
+
+	if (plato_debug_info(psSysData->pdev->dev.parent, &plato_noc_regs[0], &plato_aon_regs[0]))
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	for (i = 0; i < ARRAY_SIZE(plato_noc_regs); i++)
+		PVR_DUMPDEBUG_LOG("%s: 0x%x", plato_noc_regs[i].description, plato_noc_regs[i].value);
+
+	for (i = 0; i < ARRAY_SIZE(plato_aon_regs); i++)
+		PVR_DUMPDEBUG_LOG("%s: 0x%x", plato_aon_regs[i].description, plato_aon_regs[i].value);
+
+	return PVRSRV_OK;
+}
+
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+
+static PVRSRV_ERROR InitLocalHeaps(SYS_DATA *psSysData,
+									PHYS_HEAP_CONFIG *pasPhysHeaps,
+									IMG_UINT32 uiPhysHeapCount,
+									IMG_HANDLE hPhysHeapPrivData)
+{
+	PHYS_HEAP_CONFIG *psPhysHeap;
+	IMG_UINT64 ui64FwCarveoutSize = (RGX_NUM_DRIVERS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = "lma_gpu_local";
+	psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+	/* Configure mappable heap region */
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = psSysData->pdata->rogue_heap_mappable.base;
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = psSysData->pdata->rogue_heap_dev_addr;
+	psPhysHeap->uConfig.sLMA.uiSize = psSysData->pdata->rogue_heap_mappable.size - ui64FwCarveoutSize;
+
+	PVR_LOG(("Added mappable local memory heap. Base = 0x%016llx, Size=0x%016llx",
+			psPhysHeap->uConfig.sLMA.sCardBase.uiAddr,
+			psPhysHeap->uConfig.sLMA.uiSize));
+
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_FW_SHARED];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = "lma_fw_shared";
+	psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+	/* Configure mappable heap region */
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = psSysData->pdata->rogue_heap_mappable.base +
+	                                             pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.uiSize;
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = psSysData->pdata->rogue_heap_dev_addr +
+	                                            pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.uiSize;
+	psPhysHeap->uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+	PVR_LOG(("Added mappable Firmware heap. Base = 0x%016llx, Size=0x%016llx",
+			psPhysHeap->uConfig.sLMA.sCardBase.uiAddr,
+			psPhysHeap->uConfig.sLMA.uiSize));
+
+	/* Setup non-mappable region if BAR size is less than actual memory size (8GB) */
+	if (PLATO_HAS_NON_MAPPABLE(psSysData))
+	{
+		psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_NON_MAPPABLE];
+		psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_PRIVATE;
+		psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+		psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+		psPhysHeap->uConfig.sLMA.pszHeapName = "lma_gpu_private";
+		psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+		psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = psSysData->pdata->rogue_heap_nonmappable.base;
+		psPhysHeap->uConfig.sLMA.uiSize = psSysData->pdata->rogue_heap_nonmappable.size;
+		psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = 0;
+
+		PVR_LOG(("Added non-mappable local memory heap. Base = 0x%016llx, Size=0x%016llx",
+					psPhysHeap->uConfig.sLMA.sCardBase.uiAddr,
+					psPhysHeap->uConfig.sLMA.uiSize));
+
+		PVR_ASSERT(psPhysHeap->uConfig.sLMA.uiSize < SYS_DEV_MEM_REGION_SIZE);
+	}
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+	psPhysHeap = &pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_EXTERNAL;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = &gsLocalPhysHeapFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = "lma_pdp_local";
+	psPhysHeap->uConfig.sLMA.hPrivData = hPhysHeapPrivData;
+
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = PLATO_DDR_DEV_PHYSICAL_BASE;
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = psSysData->pdata->pdp_heap.base;
+	psPhysHeap->uConfig.sLMA.uiSize = psSysData->pdata->pdp_heap.size;
+
+	PVR_LOG(("Added PDP heap. Base = 0x%016llx, Size=0x%016llx",
+			psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr,
+			psPhysHeap->uConfig.sLMA.uiSize));
+#endif
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHostHeaps(SYS_DATA *psSysData,
+								PHYS_HEAP_CONFIG *pasPhysHeaps,
+								IMG_UINT32 uiPhysHeapCount,
+								IMG_HANDLE hPhysHeapPrivData)
+{
+	PHYS_HEAP_CONFIG *psPhysHeap;
+
+	PVR_ASSERT(uiPhysHeapCount == 1);
+
+	psPhysHeap = &pasPhysHeaps[0];
+	psPhysHeap->eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeap->uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeap->uConfig.sUMA.psMemFuncs = &gsHostPhysHeapFuncs;
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeap->uConfig.sUMA.pszHeapName = "uma_gpu_local";
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	psPhysHeap->ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+	psPhysHeap->uConfig.sUMA.pszHeapName = "uma_cpu_local";
+	PVR_DPF((PVR_DBG_WARNING, "Initialising CPU_LOCAL UMA Host PhysHeaps"));
+#if !defined(SUPPORT_PLATO_DISPLAY)
+	psPhysHeap->ui32UsageFlags |= PHYS_HEAP_USAGE_EXTERNAL;
+#endif
+#endif
+	psPhysHeap->uConfig.sUMA.sCardBase.uiAddr = PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	psPhysHeap->uConfig.sUMA.hPrivData = hPhysHeapPrivData;
+
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHybridHeaps(SYS_DATA *psSysData,
+									PHYS_HEAP_CONFIG *pasPhysHeaps,
+									IMG_UINT32 uiPhysHeapCount,
+									IMG_HANDLE hPhysHeapPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiPhysHeapCount >= PHYS_HEAP_ID_NON_MAPPABLE);
+
+	eError = InitHostHeaps(psSysData,
+							&pasPhysHeaps[PHYS_HEAP_ID_CPU_LOCAL], 1,
+							hPhysHeapPrivData);
+	if (eError != PVRSRV_OK)
+		return eError;
+
+	/*
+	 * InitLocalHeaps should set up the correct heaps regardless of whether the
+	 * memory configuration is 'local' or 'hybrid'.
+	 */
+	eError = InitLocalHeaps(psSysData, pasPhysHeaps,
+							 uiPhysHeapCount, hPhysHeapPrivData);
+	if (eError != PVRSRV_OK) {
+		return eError;
+	}
+
+	/* Adjust the pdump memory space names */
+	pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.pszPDumpMemspaceName = "LMA0";
+	pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.pszHeapName = "lma0_gpu_local";
+	pasPhysHeaps[PHYS_HEAP_ID_FW_SHARED].uConfig.sLMA.pszPDumpMemspaceName = "LMA0";
+	pasPhysHeaps[PHYS_HEAP_ID_FW_SHARED].uConfig.sLMA.pszHeapName = "lma0_fw_shared";
+#if defined(SUPPORT_PLATO_DISPLAY)
+	pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL].uConfig.sLMA.pszPDumpMemspaceName = "LMA1";
+	pasPhysHeaps[PHYS_HEAP_ID_PDP_LOCAL].uConfig.sLMA.pszHeapName = "lma1_pdp_local";
+#endif
+	return PVRSRV_OK;
+}
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR PhysHeapsCreate(SYS_DATA *psSysData,
+									PVRSRV_DEVICE_CONFIG *psDevConfig,
+									PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	IMG_UINT32 uiHeapCount = 2; /* GPU_LOCAL + FW_SHARED*/
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	uiHeapCount++;
+#endif
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+	uiHeapCount++;
+#endif
+
+	if (PLATO_HAS_NON_MAPPABLE(psSysData))
+	{
+		uiHeapCount++;
+	}
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+	eError = InitLocalHeaps(psSysData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+	eError = InitHostHeaps(psSysData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+	eError = InitHybridHeaps(psSysData, pasPhysHeaps,
+							uiHeapCount, psDevConfig);
+#endif
+
+	if (eError != PVRSRV_OK) {
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr;
+
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx) {
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr;
+		}
+	}
+}
+
+static void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr,
+					IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr -
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr +
+		psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr;
+
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx) {
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr -
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sCardBase.uiAddr +
+				psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr;
+		}
+	}
+}
+
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr + PLATO_HOSTRAM_DEV_PHYSICAL_BASE;
+	}
+}
+
+static void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+	if (ui32NumOfAddr > 1) {
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr - PLATO_HOSTRAM_DEV_PHYSICAL_BASE);
+	}
+}
+
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   SYS_DATA *psSysData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PVRSRV_ERROR eError;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	/* Set up the RGX timing information */
+	psRGXTimingInfo->ui32CoreClockSpeed = plato_core_clock_speed(&psSysData->pdev->dev);
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Initialize heaps */
+	eError = PhysHeapsCreate(psSysData, psDevConfig, &psDevConfig->pasPhysHeaps,
+						&psDevConfig->ui32PhysHeapCount);
+	if (eError != PVRSRV_OK) {
+		OSFreeMem(psDevConfig);
+		return eError;
+	}
+
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevConfig->pszName = PLATO_SYSTEM_NAME;
+	psDevConfig->pszVersion = GetDeviceVersionString(psSysData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr = psSysData->registers->start;
+	psDevConfig->ui32RegsSize = SYS_PLATO_REG_RGX_SIZE;
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+	psDevConfig->bHasNonMappableLocalMemory = PLATO_HAS_NON_MAPPABLE(psSysData);
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	psDevConfig->ui32IRQ = PLATO_INTERRUPT_GPU;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psSysData;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->pszVersion)
+		OSFreeMem(psDevConfig->pszVersion);
+
+	if (psDevConfig->pasPhysHeaps)
+		OSFreeMem(psDevConfig->pasPhysHeaps);
+
+	/*
+	 * The device config, RGX data and RGX timing info are part of the same
+	 * allocation so do only one free.
+	 */
+	OSFreeMem(psDevConfig);
+}
+
+static PVRSRV_ERROR PlatoLocalMemoryTest(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_UINT64 i, j = 0;
+	IMG_UINT32 tmp = 0;
+	IMG_UINT32 chunk = sizeof(IMG_UINT32) * 10;
+
+	IMG_UINT64 ui64TestMemoryBase = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.sStartAddr.uiAddr;
+	IMG_UINT64 ui64TestMemorySize = psDevConfig->pasPhysHeaps[PHYS_HEAP_ID_GPU_LOCAL].uConfig.sLMA.uiSize;
+
+	PVR_LOG(("%s: Starting Local memory test from 0x%llx to 0x%llx (in CPU space)",
+			 __func__, ui64TestMemoryBase, ui64TestMemoryBase + ui64TestMemorySize));
+
+	while (j < ui64TestMemorySize) {
+		IMG_CPU_PHYADDR myPaddr;
+		IMG_UINT32 *pui32Virt;
+
+		myPaddr.uiAddr = ui64TestMemoryBase + j;
+		pui32Virt = OSMapPhysToLin(myPaddr, chunk, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		for (i = 0; i < chunk/sizeof(IMG_UINT32); i++) {
+			*(pui32Virt + i) = 0xdeadbeef;
+			OSWriteMemoryBarrier(pui32Virt);
+			tmp = *(pui32Virt + i);
+			if (tmp != 0xdeadbeef) {
+				PVR_DPF((PVR_DBG_ERROR,
+						"Local memory read-write test failed at address=0x%llx: written 0x%x, read 0x%x",
+						ui64TestMemoryBase + ((i * sizeof(IMG_UINT32)) + j), (IMG_UINT32) 0xdeadbeef, tmp));
+
+				OSUnMapPhysToLin(pui32Virt, chunk);
+				return PVRSRV_ERROR_SYSTEM_LOCAL_MEMORY_INIT_FAIL;
+			}
+		}
+
+		OSUnMapPhysToLin(pui32Virt, chunk);
+
+		j += (1024 * 1024 * 500);
+	}
+
+	PVR_LOG(("Local memory read-write test passed!"));
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psSysData;
+	IMG_UINT32 uiRegistersSize;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(pvOSDevice);
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (psSysData == NULL)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+
+	/* Retrieve platform device and data */
+	psSysData->pdev = to_platform_device((struct device *) pvOSDevice);
+	psSysData->pdata = psSysData->pdev->dev.platform_data;
+
+	/* Enable plato PCI */
+	if (plato_enable(psSysData->pdev->dev.parent)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to enable PCI device", __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrFreeSysData;
+	}
+
+	psSysData->registers = platform_get_resource_byname(psSysData->pdev, IORESOURCE_MEM, PLATO_ROGUE_RESOURCE_REGS);
+	if (!psSysData->registers) {
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get Rogue register information",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		goto ErrorDevDisable;
+	}
+
+	/* Check the address range is large enough. */
+	uiRegistersSize = resource_size(psSysData->registers);
+	if (uiRegistersSize < SYS_PLATO_REG_RGX_SIZE) {
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough (was %08X, required 0x%08x)",
+				 __func__, uiRegistersSize, SYS_PLATO_REG_RGX_SIZE));
+
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorDevDisable;
+	}
+
+#if !defined(VIRTUAL_PLATFORM)
+	/* Reserve the rogue registers address range */
+	if (!request_mem_region(psSysData->registers->start,
+							uiRegistersSize,
+							PVRSRV_MODNAME)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Rogue register memory region not available", __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrorDevDisable;
+	}
+#endif
+
+	eError = DeviceConfigCreate(pvOSDevice, psSysData, &psDevConfig);
+	if (eError != PVRSRV_OK) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create device config", __func__));
+		goto ErrorReleaseMemRegion;
+	}
+
+	PlatoLocalMemoryTest(psDevConfig);
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorReleaseMemRegion:
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+ErrorDevDisable:
+	plato_disable(psSysData->pdev->dev.parent);
+ErrFreeSysData:
+	OSFreeMem(psSysData);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevConfig->hSysData;
+
+	DeviceConfigDestroy(psDevConfig);
+
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+	plato_disable(psSysData->pdev->dev.parent);
+
+	OSFreeMem(psSysData);
+}
+
+static void PlatoInterruptHandler(void *pvData)
+{
+	LISR_DATA *psLISRData = pvData;
+
+	psLISRData->pfnLISR(psLISRData->pvData);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	LISR_DATA *psLISRData;
+	PVRSRV_ERROR eError;
+
+	/* Should only accept GPU interrupts through this API */
+	if (ui32IRQ != PLATO_INTERRUPT_GPU) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid %d", __func__, ui32IRQ));
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psLISRData = OSAllocZMem(sizeof(*psLISRData));
+	if (!psLISRData)
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+	psLISRData->iInterruptID = ui32IRQ;
+	psLISRData->psDev = psSysData->pdev->dev.parent;
+
+	if (plato_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, PlatoInterruptHandler, psLISRData)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_set_interrupt_handler() failed", __func__));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_free_data;
+	}
+
+	if (plato_enable_interrupt(psLISRData->psDev, psLISRData->iInterruptID)) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_enable_interrupt() failed", __func__));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_unset_interrupt_handler;
+	}
+
+	*phLISRData = psLISRData;
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, ui32IRQ));
+
+	return PVRSRV_OK;
+
+err_unset_interrupt_handler:
+	plato_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+err_free_data:
+	OSFreeMem(psLISRData);
+	return eError;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *) hLISRData;
+	int err;
+
+	err = plato_disable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_enable_interrupt() failed (%d)", __func__, err));
+	}
+
+	err = plato_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: plato_set_interrupt_handler() failed (%d)", __func__, err));
+	}
+
+	PVR_TRACE(("Uninstalled device LISR " IMG_PFN_FMTSPEC " from irq %u", psLISRData->pfnLISR, psLISRData->iInterruptID));
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.h
new file mode 100644
index 000000000000..5761da3a1bf6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysconfig.h
@@ -0,0 +1,105 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSCONFIG_H)
+#define SYSCONFIG_H
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "plato_drv.h"
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+#define MAX_SYSTEMS 32
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_DEV_PHYADDR *psDevPAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void PlatoLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr,
+					IMG_DEV_PHYADDR *psDevPAddr);
+#endif
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static void PlatoSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_DEV_PHYADDR *psDevPAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void PlatoSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					IMG_UINT32 ui32NumOfAddr,
+					IMG_CPU_PHYADDR *psCpuPAddr,
+					IMG_DEV_PHYADDR *psDevPAddr);
+#endif /* (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID) */
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs = {
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoLocalCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoLocalDevPAddrToCpuPAddr,
+};
+#endif
+
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST) || (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs = {
+	/* pfnCpuPAddrToDevPAddr */
+	PlatoSystemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	PlatoSystemDevPAddrToCpuPAddr,
+};
+#endif
+
+#if (PLATO_MEMORY_CONFIG != PLATO_MEMORY_LOCAL) && \
+	(PLATO_MEMORY_CONFIG != PLATO_MEMORY_HYBRID) && \
+	(PLATO_MEMORY_CONFIG == PLATO_MEMORY_HOST)
+#error "PLATO_MEMORY_CONFIG not valid"
+#endif
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* !defined(SYSCONFIG_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysinfo.h
new file mode 100644
index 000000000000..fd796a75797f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_linux_plato/sysinfo.h
@@ -0,0 +1,86 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+#if defined(__KERNEL__)
+#include "plato_drv.h"
+#endif
+
+#define SYS_RGX_DEV_VENDOR_ID	(0x1AEE)
+#define SYS_RGX_DEV_DEVICE_ID	(0x0003)
+
+#if defined(__KERNEL__)
+#if defined(PLATO_MULTI_DEVICE)
+#define SYS_RGX_DEV_NAME_0	PLATO_MAKE_DEVICE_NAME_ROGUE(0)
+#define SYS_RGX_DEV_NAME_1	PLATO_MAKE_DEVICE_NAME_ROGUE(1)
+#define SYS_RGX_DEV_NAME_2	PLATO_MAKE_DEVICE_NAME_ROGUE(2)
+#define SYS_RGX_DEV_NAME_3	PLATO_MAKE_DEVICE_NAME_ROGUE(3)
+#else
+#define SYS_RGX_DEV_NAME	PLATO_DEVICE_NAME_ROGUE
+#endif
+#endif
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM) || defined(EMULATOR)
+/* Emulator clock ~600 times slower than HW */
+#define MAX_HW_TIME_US                           (300000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1000000)
+
+#if defined(VIRTUAL_PLATFORM)
+#define EVENT_OBJECT_TIMEOUT_US                  (120000000)
+#elif defined(EMULATOR)
+#define EVENT_OBJECT_TIMEOUT_US                  (2000000)
+#endif
+
+#else
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(100000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+#endif
+
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#endif /* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/Kbuild.mk
new file mode 100644
index 000000000000..41b532383198
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/Kbuild.mk
@@ -0,0 +1,52 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+ services/system/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/system/common/uma_heap_fns.o
+
+ifeq ($(SUPPORT_ION),1)
+$(PVRSRV_MODNAME)-y += services/system/common/env/linux/ion_support_generic.o
+endif
+
+ifeq ($(PVR_ARCH),volcanic)
+ ccflags-y += -DVOLCANIC_FBCDCV31
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.c
new file mode 100644
index 000000000000..4543f8e7d25f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.c
@@ -0,0 +1,330 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#include "uma_heap_fns.h"
+
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+
+
+/*
+ * In systems that support trusted device address protection, there are three
+ * physical heaps from which pages should be allocated:
+ * - one heap for normal allocations
+ * - one heap for allocations holding META code memory
+ * - one heap for allocations holding secured DRM data
+ */
+
+#define PHYS_HEAP_IDX_GENERAL     0
+#define PHYS_HEAP_IDX_FW          1
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define PHYS_HEAP_IDX_TDFWMEM     2
+#define PHYS_HEAP_IDX_TDSECUREBUF 3
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+#define PHYS_HEAP_IDX_FW_MEMORY   2
+#endif
+
+/* Change to test CPU_LOCAL sys layers*/
+#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_GPU_LOCAL
+//#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_CPU_LOCAL
+
+#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_GPU_LOCAL
+//#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_CPU_LOCAL
+
+#if defined(PVRSRV_ENABLE_XD_MEM)
+static DECLARE_PHYS_HEAP_SPAS_REGION(sSpasSystemMemory);
+#endif
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	/*
+	 * This function is called during device initialisation, which on Linux,
+	 * means it won't be called concurrently. As such, there's no need to
+	 * protect it with a lock or use an atomic variable.
+	 */
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiHeapCount = 2;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	uiHeapCount += 2;
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	uiHeapCount += 1;
+#endif
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].ui32UsageFlags = UMA_HEAP_USAGE_FLAG;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.pszHeapName = "uma_local";
+#if defined(PVRSRV_ENABLE_XD_MEM)
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.psSpasRegion = &sSpasSystemMemory;
+#endif
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	/* Provide a specific set of Intermediate Physical Address modifiers
+	 * for the default heap. Also name this as "SYSMEM_IPA" to aid PDump
+	 * processing.
+	 * Modifiers are:
+	 *  IPAPolicyDefault = 3
+	 *  IPAPolicyMask = 7
+	 *  IPAPolicyShift = 52
+	 * This will result in the PhysAddr values having bits 53..55 set with '3'
+	 * by default.
+	 */
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM_IPA";
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].sIPAConfig.ui8IPAPolicyDefault = 3;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].sIPAConfig.ui8IPAPolicyMask = 7;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].sIPAConfig.ui8IPAPolicyShift = 52;
+#endif
+
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].eType = PHYS_HEAP_TYPE_UMA;
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM_FW";
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	 pasPhysHeaps[PHYS_HEAP_IDX_FW].uConfig.sUMA.pszHeapName = "uma_fw_shared";
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].uConfig.sUMA.pszPDumpMemspaceName = "TDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].uConfig.sUMA.pszHeapName = "uma_tdfwmem";
+
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_SECURE;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].uConfig.sUMA.pszPDumpMemspaceName = "TDSECBUFMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].uConfig.sUMA.pszHeapName = "uma_tdsecbufmem";
+
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].uConfig.sUMA.pszPDumpMemspaceName = "DEDICATEDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].uConfig.sUMA.pszHeapName = "uma_dedicatedfwmem";
+#endif
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+#if defined(SUPPORT_AXI_ACE_TEST)
+		if ( ui64Features & RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		}else
+#endif
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_NONE;
+		}
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszName                = "nohw";
+	psDevConfig->pszVersion             = NULL;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr   = 0x00f00000;
+	psDevConfig->ui32RegsSize           = 0x100000;
+	psDevConfig->ui32IRQ                = 0x00000bad;
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+	psDevConfig->eDefaultHeap = UMA_DEFAULT_HEAP;
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+
+	psDevConfig->bDevicePA0IsValid       = IMG_FALSE;
+	psDevConfig->pfnSysDevFeatureDepInit = SysDevFeatureDepInit;
+
+	/* Setup other system specific stuff */
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	/* Pdump validation system registers */
+
+#if !defined(HW_ERN_66622) && !defined(VOLCANIC_FBCDCV31)
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+#else
+	psDevConfig->bHasFBCDCVersion31 = IMG_TRUE;
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	PVR_UNREFERENCED_PARAMETER(ui32IRQ);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(pfnLISR);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(phLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.h
new file mode 100644
index 000000000000..acbadc22e6a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysconfig.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(SYSCCONFIG_H)
+#define SYSCCONFIG_H
+
+
+#define RGX_NOHW_CORE_CLOCK_SPEED 100000000
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* SYSCCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysinfo.h
new file mode 100644
index 000000000000..709b54c3bd0b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_nohw/sysinfo.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME    "rgxnohw"
+#endif
+
+#endif	/* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/Kbuild.mk
new file mode 100644
index 000000000000..9d2ca9476931
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/Kbuild.mk
@@ -0,0 +1,48 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += services/system/$(PVR_SYSTEM)/sysconfig.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/$(PVR_ARCH)/common/vmm_type_$(VMM_TYPE).o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.c
new file mode 100644
index 000000000000..99b8366c11a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.c
@@ -0,0 +1,310 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the system layer for QEMU vexpress virtual-platform
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#endif
+
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "interrupt_support.h"
+#include "vz_vmm_pvz.h"
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+static RGX_TIMING_INFORMATION	gsRGXTimingInfo;
+static RGX_DATA					gsRGXData;
+static PVRSRV_DEVICE_CONFIG		gsDevices[1];
+static PHYS_HEAP_FUNCTIONS		gsPhysHeapFuncs;
+static PHYS_HEAP_CONFIG			gsPhysHeapConfig[2];
+
+#if defined(SUPPORT_PDVFS)
+static const IMG_OPP asOPPTable[] =
+{
+	{ 824,  240000000},
+	{ 856,  280000000},
+	{ 935,  380000000},
+	{ 982,  440000000},
+	{ 1061, 540000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(asOPPTable[0]))
+
+static void SetFrequency(IMG_UINT32 ui32Frequency) {}
+
+static void SetVoltage(IMG_UINT32 ui32Volt) {}
+#endif
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+#if defined(SUPPORT_AXI_ACE_TEST)
+		if ( ui64Features & RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+		{
+			gsDevices[0].eCacheSnoopingMode     = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		}
+		else
+#endif
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_EMULATED;
+		}
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	IMG_UINT32 ui32NextPhysHeapID = 0;
+#if defined(__linux__)
+	int iIrq;
+	struct resource *psDevMemRes = NULL;
+	struct platform_device *psDev;
+
+	psDev = to_platform_device((struct device *)pvOSDevice);
+#endif
+
+	if (gsDevices[0].pvOSDevice)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+#endif
+
+	/*
+	 * Setup information about physical memory heap(s) we have
+	 */
+	gsPhysHeapFuncs.pfnCpuPAddrToDevPAddr = UMAPhysHeapCpuPAddrToDevPAddr;
+	gsPhysHeapFuncs.pfnDevPAddrToCpuPAddr = UMAPhysHeapDevPAddrToCpuPAddr;
+
+	gsPhysHeapConfig[ui32NextPhysHeapID].pszPDumpMemspaceName = "SYSMEM";
+	gsPhysHeapConfig[ui32NextPhysHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	gsPhysHeapConfig[ui32NextPhysHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	gsPhysHeapConfig[ui32NextPhysHeapID].hPrivData = NULL;
+	gsPhysHeapConfig[ui32NextPhysHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	ui32NextPhysHeapID += 1;
+
+	/*
+	 * Setup RGX specific timing data
+	 */
+	gsRGXTimingInfo.ui32CoreClockSpeed        = RGX_VEXPRESS_CORE_CLOCK_SPEED;
+	gsRGXTimingInfo.bEnableActivePM           = IMG_FALSE;
+	gsRGXTimingInfo.bEnableRDPowIsland        = IMG_FALSE;
+	gsRGXTimingInfo.ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/*
+	 *Setup RGX specific data
+	 */
+	gsRGXData.psRGXTimingInfo = &gsRGXTimingInfo;
+
+	/*
+	 * Setup device
+	 */
+	gsDevices[0].pvOSDevice				= pvOSDevice;
+	gsDevices[0].pszName                = "vexpress";
+	gsDevices[0].pszVersion             = NULL;
+
+	/* Device setup information */
+#if defined(__linux__)
+	psDevMemRes = platform_get_resource(psDev, IORESOURCE_MEM, 0);
+	if (psDevMemRes)
+	{
+		gsDevices[0].sRegsCpuPBase.uiAddr = psDevMemRes->start;
+		gsDevices[0].ui32RegsSize         = (unsigned int)(psDevMemRes->end - psDevMemRes->start);
+	}
+	else
+#endif
+	{
+#if defined(__linux__)
+		PVR_LOG(("%s: platform_get_resource() failed, using mmio/sz 0x%x/0x%x",
+				__func__,
+				VEXPRESS_GPU_PBASE,
+				VEXPRESS_GPU_SIZE));
+#endif
+		gsDevices[0].sRegsCpuPBase.uiAddr   = VEXPRESS_GPU_PBASE;
+		gsDevices[0].ui32RegsSize           = VEXPRESS_GPU_SIZE;
+	}
+
+#if defined(__linux__)
+	iIrq = platform_get_irq(psDev, 0);
+	if (iIrq >= 0)
+	{
+		gsDevices[0].ui32IRQ = (IMG_UINT32) iIrq;
+	}
+	else
+#endif
+	{
+#if defined(__linux__)
+		PVR_LOG(("%s: platform_get_irq() failed, using irq %d",
+				__func__,
+				VEXPRESS_IRQ_GPU));
+#endif
+		gsDevices[0].ui32IRQ = VEXPRESS_IRQ_GPU;
+	}
+
+	/* Device's physical heaps */
+	gsDevices[0].pasPhysHeaps = gsPhysHeapConfig;
+	gsDevices[0].ui32PhysHeapCount = ARRAY_SIZE(gsPhysHeapConfig);
+	gsDevices[0].eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	/* No power management on VIRTUAL_PLATFORM system */
+	gsDevices[0].pfnPrePowerState       = NULL;
+	gsDevices[0].pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	gsDevices[0].pfnClockFreqGet        = NULL;
+
+	gsDevices[0].hDevData               = &gsRGXData;
+
+	gsDevices[0].pfnSysDevFeatureDepInit = &SysDevFeatureDepInit;
+
+	gsDevices[0].bHasFBCDCVersion31	= IMG_FALSE;
+	gsDevices[0].bDevicePA0IsValid	= IMG_FALSE;
+
+	/* device error notify callback function */
+	gsDevices[0].pfnSysDevErrorNotify = NULL;
+
+	/* Virtualization support services needs to know which heap ID corresponds to FW */
+	PVR_ASSERT(ui32NextPhysHeapID < ARRAY_SIZE(gsPhysHeapConfig));
+	gsPhysHeapConfig[ui32NextPhysHeapID].pszPDumpMemspaceName = "SYSMEM";
+	gsPhysHeapConfig[ui32NextPhysHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	gsPhysHeapConfig[ui32NextPhysHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	gsPhysHeapConfig[ui32NextPhysHeapID].hPrivData = NULL;
+	gsPhysHeapConfig[ui32NextPhysHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+
+#if defined(SUPPORT_PDVFS)
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+
+	*ppsDevConfig = &gsDevices[0];
+
+	return PVRSRV_OK;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	psDevConfig->pvOSDevice = NULL;
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	return OSInstallSystemLISR(phLISRData, ui32IRQ, pszName, pfnLISR, pvData,
+								SYS_IRQ_FLAG_TRIGGER_DEFAULT);
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return OSUninstallSystemLISR(hLISRData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.h
new file mode 100644
index 000000000000..db7625c698ea
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysconfig.h
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(__SYSCCONFIG_H__)
+#define __SYSCCONFIG_H__
+
+#define VEXPRESS_IRQ_GPU 63
+#define VEXPRESS_GPU_SIZE 0x00010000
+#define VEXPRESS_GPU_PBASE 0x021B0000
+
+#define RGX_VEXPRESS_CORE_CLOCK_SPEED (320 * 1000 * 1000)
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (100)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* __SYSCCONFIG_H__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysinfo.h
new file mode 100644
index 000000000000..66acc666273b
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rgx_vexpress/sysinfo.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM)
+#define MAX_HW_TIME_US								(5000000)
+#else
+#define MAX_HW_TIME_US								(500000)
+#endif
+
+#define FATAL_ERROR_DETECTION_POLL_MS				(10000)
+#if defined(VIRTUAL_PLATFORM)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT		(10000)
+#else
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT		(1500)//(10000)
+#endif
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT	(3600000)
+#define WAIT_TRY_COUNT								(10000)
+
+
+#if defined(VIRTUAL_PLATFORM)
+#define SYS_RGX_OF_COMPATIBLE "img,gpu"
+#endif
+
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME "rgx_vexpress"
+#endif
+
+#endif	/* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/env/linux/dma_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/env/linux/dma_support.c
new file mode 100644
index 000000000000..c561d0622d5f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/env/linux/dma_support.c
@@ -0,0 +1,523 @@
+/*************************************************************************/ /*!
+@File           dma_support.c
+@Title          System DMA support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a contiguous memory allocator (i.e. DMA allocator);
+                APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/mm.h>
+#include <asm/page.h>
+#include <linux/device.h>
+#include <linux/highmem.h>
+#include <linux/vmalloc.h>
+#include <linux/dma-mapping.h>
+#include <asm-generic/getorder.h>
+
+#include "allocmem.h"
+#include "dma_support.h"
+#include "pvr_vmap.h"
+#include "kernel_compatibility.h"
+
+#define DMA_MAX_IOREMAP_ENTRIES 2
+static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
+static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};
+
+static void*
+SysDmaAcquireKernelAddress(struct page *psPage, IMG_UINT64 ui64Size, DMA_ALLOC *psDmaAlloc)
+{
+	IMG_BOOL bPageByPage = IMG_TRUE;
+	IMG_UINT32 uiIdx;
+	void *pvVirtAddr = NULL;
+	IMG_UINT32 ui32PgCount = (IMG_UINT32)(ui64Size >> OSGetPageShift());
+	PVRSRV_DEVICE_NODE *psDevNode = OSAllocZMemNoStats(sizeof(*psDevNode));
+	PVRSRV_DEVICE_CONFIG *psDevConfig = OSAllocZMemNoStats(sizeof(*psDevConfig));
+	struct page **pagearray = OSAllocZMemNoStats(ui32PgCount * sizeof(struct page *));
+	void *pvOSDevice = psDmaAlloc->pvOSDevice;
+#if defined(CONFIG_ARM64)
+	pgprot_t prot = pgprot_writecombine(PAGE_KERNEL);
+#else
+	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
+#endif
+
+	/* Validate all required dynamic tmp buffer allocations */
+	if (psDevNode == NULL || psDevConfig == NULL || pagearray == NULL)
+	{
+		if (psDevNode)
+		{
+			OSFreeMem(psDevNode);
+		}
+
+		if (psDevConfig)
+		{
+			OSFreeMem(psDevConfig);
+		}
+
+		if (pagearray)
+		{
+			OSFreeMem(pagearray);
+		}
+
+		goto e0;
+	}
+
+	/* Fake psDevNode->psDevConfig->pvOSDevice */
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevNode->psDevConfig = psDevConfig;
+
+	/* Evict any page data contents from d-cache */
+	for (uiIdx = 0; uiIdx < ui32PgCount; uiIdx++)
+	{
+		void *pvVirtStart, *pvVirtEnd;
+		IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+
+		/* Prepare array required for vmap */
+		pagearray[uiIdx] = &psPage[uiIdx];
+
+		if (bPageByPage)
+		{
+#if defined(CONFIG_64BIT)
+			bPageByPage = IMG_FALSE;
+
+			pvVirtStart = kmap(&psPage[uiIdx]);
+			pvVirtEnd = pvVirtStart + ui64Size;
+
+			sCPUPhysStart.uiAddr = page_to_phys(&psPage[uiIdx]);
+			sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + ui64Size;
+			/* all pages have a kernel linear address, flush entire range */
+#else
+			pvVirtStart = kmap(&psPage[uiIdx]);
+			pvVirtEnd = pvVirtStart + PAGE_SIZE;
+
+			sCPUPhysStart.uiAddr = page_to_phys(&psPage[uiIdx]);
+			sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+			/* pages might be from HIGHMEM, need to kmap/flush per page */
+#endif
+
+			/* Fallback to range-based d-cache flush */
+			OSCPUCacheInvalidateRangeKM(psDevNode,
+										pvVirtStart, pvVirtEnd,
+										sCPUPhysStart, sCPUPhysEnd);
+
+			kunmap(&psPage[uiIdx]);
+		}
+	}
+
+	/* Remap pages into VMALLOC space */
+	pvVirtAddr = pvr_vmap(pagearray, ui32PgCount, VM_MAP, prot);
+	psDmaAlloc->PageProps = prot;
+
+	/* Clean-up tmp buffers */
+	OSFreeMem(psDevConfig);
+	OSFreeMem(psDevNode);
+	OSFreeMem(pagearray);
+
+e0:
+	return pvVirtAddr;
+}
+
+static void SysDmaReleaseKernelAddress(void *pvVirtAddr, IMG_UINT64 ui64Size, pgprot_t pgprot)
+{
+	pvr_vunmap(pvVirtAddr, ui64Size >> OSGetPageShift(), pgprot);
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaAllocMem
+
+ @Description		Allocates physically contiguous memory
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct device *psDev;
+	struct page *psPage;
+	size_t uiSize;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->hHandle ||
+		psDmaAlloc->pvVirtAddr ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->sBusAddr.uiAddr ||
+		psDmaAlloc->pvOSDevice == NULL)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+	psDev = (struct device *)psDmaAlloc->pvOSDevice;
+
+	psDmaAlloc->hHandle = dma_alloc_coherent(psDev, uiSize, (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr, GFP_KERNEL);
+
+	if (psDmaAlloc->hHandle)
+	{
+		psDmaAlloc->pvVirtAddr = psDmaAlloc->hHandle;
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"Allocated DMA buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
+				psDmaAlloc->pvVirtAddr,
+				psDmaAlloc->sBusAddr.uiAddr,
+				uiSize));
+	}
+	else if ((psPage = alloc_pages(GFP_KERNEL, get_order(uiSize))))
+	{
+		psDmaAlloc->sBusAddr.uiAddr = dma_map_page(psDev, psPage, 0, uiSize, DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(psDev, psDmaAlloc->sBusAddr.uiAddr))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"dma_map_page() failed, page 0x%p order %d",
+					psPage,
+					get_order(uiSize)));
+			__free_pages(psPage, get_order(uiSize));
+			goto e0;
+		}
+		psDmaAlloc->psPage = psPage;
+
+		psDmaAlloc->pvVirtAddr = SysDmaAcquireKernelAddress(psPage, uiSize, psDmaAlloc);
+		if (! psDmaAlloc->pvVirtAddr)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"SysDmaAcquireKernelAddress() failed, page 0x%p order %d",
+					psPage,
+					get_order(uiSize)));
+			dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
+			__free_pages(psPage, get_order(uiSize));
+			goto e0;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"Allocated contiguous buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
+				psDmaAlloc->pvVirtAddr,
+				psDmaAlloc->sBusAddr.uiAddr,
+				uiSize));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,	"Unable to allocate contiguous buffer, size: 0x"IMG_SIZE_FMTSPECX, uiSize));
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+	}
+
+e0:
+	PVR_LOG_RETURN_IF_FALSE((psDmaAlloc->pvVirtAddr), "DMA/CMA allocation failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaFreeMem
+
+ @Description		Free physically contiguous memory
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
+{
+	size_t uiSize;
+	struct device *psDev;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+	psDev = (struct device *)psDmaAlloc->pvOSDevice;
+
+	if (psDmaAlloc->pvVirtAddr != psDmaAlloc->hHandle)
+	{
+		SysDmaReleaseKernelAddress(psDmaAlloc->pvVirtAddr, uiSize, psDmaAlloc->PageProps);
+	}
+
+	if (! psDmaAlloc->hHandle)
+	{
+		struct page *psPage;
+		dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
+		psPage = psDmaAlloc->psPage;
+		__free_pages(psPage, get_order(uiSize));
+		return;
+	}
+
+	dma_free_coherent(psDev, uiSize, psDmaAlloc->hHandle, (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaRegisterForIoRemapping
+
+ @Description		Registers DMA_ALLOC for manual I/O remapping
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	size_t uiSize;
+	IMG_UINT32 ui32Idx;
+	IMG_BOOL bTabEntryFound = IMG_TRUE;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		/* Check if an I/O remap entry exists for remapping */
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL)
+		{
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0);
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
+			break;
+		}
+	}
+
+	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
+	{
+		bTabEntryFound = IMG_FALSE;
+	}
+
+	if (bTabEntryFound)
+	{
+		IMG_BOOL bSameVAddr, bSamePAddr, bSameSize;
+
+		bSamePAddr = gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == psDmaAlloc->sBusAddr.uiAddr;
+		bSameVAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr;
+		bSameSize = gsDmaIoRemapArray[ui32Idx].ui64Size == uiSize;
+
+		if (bSameVAddr)
+		{
+			if (bSamePAddr && bSameSize)
+			{
+				eError = PVRSRV_OK;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_ALREADY_EXISTS;
+			}
+		}
+		else
+		{
+			PVR_ASSERT(bSamePAddr == IMG_FALSE);
+
+			gsDmaIoRemapArray[ui32Idx].ui64Size = uiSize;
+			gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
+			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;
+
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: register I/O remap: "
+					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
+					psDmaAlloc->pvVirtAddr,
+					psDmaAlloc->sBusAddr.uiAddr,
+					uiSize));
+
+			gbEnableDmaIoRemapping = IMG_TRUE;
+			eError = PVRSRV_OK;
+		}
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDeregisterForIoRemapping
+
+ @Description		Deregisters DMA_ALLOC from manual I/O remapping
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	__maybe_unused size_t uiSize;
+	IMG_UINT32 ui32Idx;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+
+	/* Remove specified entries from list of I/O remap entries */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr)
+		{
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
+			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
+			gsDmaIoRemapArray[ui32Idx].ui64Size = 0;
+
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: deregister I/O remap: "
+					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
+					psDmaAlloc->pvVirtAddr,
+					psDmaAlloc->sBusAddr.uiAddr,
+					uiSize));
+
+			break;
+		}
+	}
+
+	/* Check if no other I/O remap entries exists for remapping */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
+		{
+			break;
+		}
+	}
+
+	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
+	{
+		/* No entries found so disable remapping */
+		gbEnableDmaIoRemapping = IMG_FALSE;
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDevPAddrToCpuVAddr
+
+ @Description		Maps a DMA_ALLOC physical address to CPU virtual address
+
+ @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+ ******************************************************************************/
+IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
+{
+	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return pvDMAVirtAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
+				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: PA: 0x%llx => VA: 0x%p",
+					uiAddr, pvDMAVirtAddr));
+
+				break;
+			}
+		}
+	}
+
+	return pvDMAVirtAddr;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaCpuVAddrToDevPAddr
+
+ @Description		Maps a DMA_ALLOC CPU virtual address to physical address
+
+ @Return			Non-zero value on success. Otherwise, a 0
+ ******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
+{
+	IMG_UINT64 uiAddr = 0;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return uiAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: VA: 0x%p => PA: 0x%llx",
+					pvDMAVirtAddr, uiAddr));
+
+				break;
+			}
+		}
+	}
+
+	return uiAddr;
+}
+
+/******************************************************************************
+ End of file (dma_support.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_stub.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_stub.c
new file mode 100644
index 000000000000..a2237281d703
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_stub.c
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File			vmm_type_stub.c
+@Title          Stub VM manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Sample stub (no-operation) VM manager implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+StubVMMMapDevPhysHeap(IMG_UINT64 ui64Size,
+					  IMG_UINT64 ui64Addr)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(ui64Addr);
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+StubVMMUnmapDevPhysHeap(void)
+{
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static VMM_PVZ_CONNECTION gsStubVmmPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&StubVMMMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&StubVMMUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsStubVmmPvz;
+	PVR_DPF((PVR_DBG_MESSAGE, "Using a stub VM manager type, no hypercall support"));
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_stub.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_vzfpga.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_vzfpga.c
new file mode 100644
index 000000000000..d379fa061cc1
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/common/vmm_type_vzfpga.c
@@ -0,0 +1,170 @@
+/*************************************************************************/ /*!
+@File           vmm_type_vzfpga.c
+@Title          VZFPGA manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    VM manager implementation to support vzfpga platform
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+#include "rgxdevice.h"
+
+#include "vz_vm.h"
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+GetDriverIDFromHeapBase(IMG_UINT64 ui64Addr, IMG_UINT32 *pui32DriverID)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_DEV_PHYADDR sHostFwHeapBaseDevPA = {0};
+	PHYS_HEAP *psHostFwHeap = NULL;
+	PVRSRV_DEVICE_NODE *psHostDevNode = PVRSRVGetDeviceInstance(0);
+
+	PVR_LOG_RETURN_IF_FALSE((psHostDevNode != NULL),
+							"Host Device Node not initialised.",
+							PVRSRV_ERROR_NO_DEVICENODE_FOUND);
+
+	psHostFwHeap = psHostDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+	PVR_LOG_RETURN_IF_FALSE((psHostFwHeap != NULL),
+							"Host Fw heap not initialised.",
+							PVRSRV_ERROR_PHYSHEAP_ID_INVALID);
+
+	eErr = PhysHeapGetDevPAddr(psHostFwHeap, &sHostFwHeapBaseDevPA);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PhysHeapGetDevPAddr");
+
+	*pui32DriverID = (ui64Addr - sHostFwHeapBaseDevPA.uiAddr) / PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+	PVR_LOG_RETURN_IF_FALSE((*pui32DriverID >= RGXFW_GUEST_DRIVER_ID_START) &&
+							(*pui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+							"Invalid Guest DriverID",
+							PVRSRV_ERROR_INVALID_PVZ_OSID);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+VZFPGAMapDevPhysHeap(IMG_UINT64 ui64Size,
+					 IMG_UINT64 ui64Addr)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT32 ui32GuestDriverID;
+
+	eErr = GetDriverIDFromHeapBase(ui64Addr, &ui32GuestDriverID);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "GetDriverIDFromHeapBase");
+
+	eErr = PvzServerOnVmOnline(ui32GuestDriverID, 0);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerOnVmOnline");
+
+	eErr = PvzServerMapDevPhysHeap(ui32GuestDriverID, 0, ui64Size, ui64Addr);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerMapDevPhysHeap");
+
+	return eErr;
+}
+
+static PVRSRV_ERROR
+VZFPGAUnmapDevPhysHeap(void)
+{
+	IMG_UINT32 ui32ID;
+
+	/* During shutdown, the Guests will be deinitialised in reverse order */
+	for (ui32ID=(RGX_NUM_DRIVERS_SUPPORTED-1);
+		 ui32ID >= RGXFW_HOST_DRIVER_ID; ui32ID--)
+	{
+		if (IsVmOnline(ui32ID, 0))
+		{
+			PvzServerUnmapDevPhysHeap(ui32ID, 0);
+			PvzServerOnVmOffline(ui32ID, 0);
+			break;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static VMM_PVZ_CONNECTION gsVZFPGAPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&VZFPGAMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&VZFPGAUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsVZFPGAPvz;
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_vzfpga.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/include/dma_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/include/dma_support.h
new file mode 100644
index 000000000000..c1d22bddf523
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/include/dma_support.h
@@ -0,0 +1,117 @@
+/*************************************************************************/ /*!
+@File           dma_support.h
+@Title          Device contiguous memory allocator and I/O re-mapper
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides a contiguous memory allocator API; mainly
+                used for allocating / ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_SUPPORT_H
+#define DMA_SUPPORT_H
+
+#include "osfunc.h"
+#include "pvrsrv.h"
+
+typedef struct _DMA_ALLOC_
+{
+	IMG_UINT64       ui64Size;
+	IMG_CPU_VIRTADDR pvVirtAddr;
+	IMG_DEV_PHYADDR  sBusAddr;
+	IMG_HANDLE       hHandle;
+#if defined(__linux__)
+	struct page      *psPage;
+	pgprot_t         PageProps;
+#endif
+	void             *pvOSDevice;
+} DMA_ALLOC;
+
+/*!
+*******************************************************************************
+ @Function      SysDmaAllocMem
+ @Description   Allocates physically contiguous memory
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaFreeMem
+ @Description   Free physically contiguous memory
+ @Return        void
+******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psCmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaRegisterForIoRemapping
+ @Description   Registers DMA_ALLOC for manual I/O remapping
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDeregisterForIoRemapping
+ @Description   Deregisters DMA_ALLOC from manual I/O remapping
+ @Return        void
+******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDevPAddrToCpuVAddr
+ @Description   Maps a DMA_ALLOC physical address to CPU virtual address
+ @Return        IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+******************************************************************************/
+IMG_CPU_VIRTADDR
+SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaCpuVAddrToDevPAddr
+ @Description   Maps a DMA_ALLOC CPU virtual address to physical address
+ @Return        Non-zero value on success. Otherwise, a 0
+******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr);
+
+#endif /* DMA_SUPPORT_H */
+
+/******************************************************************************
+ End of file (dma_support.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/Kbuild.mk
new file mode 100644
index 000000000000..e5594c7feabe
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/Kbuild.mk
@@ -0,0 +1,55 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+ifeq ($(KERNEL_DRIVER_DIR),)
+ SYSTEM_BASEDIR := services/system/rogue/$(PVR_SYSTEM)
+else
+ SYSTEM_BASEDIR := external/$(KERNEL_DRIVER_DIR)/$(PVR_SYSTEM)
+endif
+
+$(PVRSRV_MODNAME)-y += \
+ $(SYSTEM_BASEDIR)/mt8173_mfgsys.o \
+ $(SYSTEM_BASEDIR)/mt8173_sysconfig.o \
+ services/system/common/uma_heap_fns.o
+
+ifeq ($(SUPPORT_ION),1)
+ $(PVRSRV_MODNAME)-y += \
+  services/system/common/env/linux/ion_support_generic.o
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.c
new file mode 100644
index 000000000000..8aace9dae30a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.c
@@ -0,0 +1,329 @@
+/*
+* Copyright (c) 2014 MediaTek Inc.
+* Author: Chiawen Lee <chiawen.lee@mediatek.com>
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*/
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <linux/thermal.h>
+
+#include "mt8173_mfgsys.h"
+
+static const char * const top_mfg_clk_name[] = {
+	"mfg_mem_in_sel",
+	"mfg_axi_in_sel",
+	"top_axi",
+	"top_mem",
+};
+
+#define MAX_TOP_MFG_CLK ARRAY_SIZE(top_mfg_clk_name)
+
+#define REG_MFG_AXI BIT(0)
+#define REG_MFG_MEM BIT(1)
+#define REG_MFG_G3D BIT(2)
+#define REG_MFG_26M BIT(3)
+#define REG_MFG_ALL (REG_MFG_AXI | REG_MFG_MEM | REG_MFG_G3D | REG_MFG_26M)
+
+#define REG_MFG_CG_STA 0x00
+#define REG_MFG_CG_SET 0x04
+#define REG_MFG_CG_CLR 0x08
+
+static void mtk_mfg_clr_clock_gating(void __iomem *reg)
+{
+	writel(REG_MFG_ALL, reg + REG_MFG_CG_CLR);
+}
+
+static int mtk_mfg_prepare_clock(struct mtk_mfg *mfg)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < MAX_TOP_MFG_CLK; i++) {
+		ret = clk_prepare(mfg->top_clk[i]);
+		if (ret)
+			goto unwind;
+	}
+	ret = clk_prepare(mfg->top_mfg);
+	if (ret)
+		goto unwind;
+
+	return 0;
+unwind:
+	while (i--)
+		clk_unprepare(mfg->top_clk[i]);
+
+	return ret;
+}
+
+static void mtk_mfg_unprepare_clock(struct mtk_mfg *mfg)
+{
+	int i;
+
+	clk_unprepare(mfg->top_mfg);
+	for (i = MAX_TOP_MFG_CLK - 1; i >= 0; i--)
+		clk_unprepare(mfg->top_clk[i]);
+}
+
+static int mtk_mfg_enable_clock(struct mtk_mfg *mfg)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < MAX_TOP_MFG_CLK; i++) {
+		ret = clk_enable(mfg->top_clk[i]);
+		if (ret)
+			goto unwind;
+	}
+	ret = clk_enable(mfg->top_mfg);
+	if (ret)
+		goto unwind;
+	mtk_mfg_clr_clock_gating(mfg->reg_base);
+
+	return 0;
+unwind:
+	while (i--)
+		clk_disable(mfg->top_clk[i]);
+
+	return ret;
+}
+
+static void mtk_mfg_disable_clock(struct mtk_mfg *mfg)
+{
+	int i;
+
+	clk_disable(mfg->top_mfg);
+	for (i = MAX_TOP_MFG_CLK - 1; i >= 0; i--)
+		clk_disable(mfg->top_clk[i]);
+}
+
+static void mtk_mfg_enable_hw_apm(struct mtk_mfg *mfg)
+{
+	writel(0x003c3d4d, mfg->reg_base + 0x24);
+	writel(0x4d45440b, mfg->reg_base + 0x28);
+	writel(0x7a710184, mfg->reg_base + 0xe0);
+	writel(0x835f6856, mfg->reg_base + 0xe4);
+	writel(0x002b0234, mfg->reg_base + 0xe8);
+	writel(0x80000000, mfg->reg_base + 0xec);
+	writel(0x08000000, mfg->reg_base + 0xa0);
+}
+
+int mtk_mfg_enable(struct mtk_mfg *mfg)
+{
+	int ret;
+
+	ret = pm_runtime_get_sync(mfg->dev);
+	if (ret < 0)
+		return ret;
+
+	ret = mtk_mfg_enable_clock(mfg);
+	if (ret)
+		goto err_pm_runtime_put;
+
+	mtk_mfg_enable_hw_apm(mfg);
+
+	return 0;
+
+err_pm_runtime_put:
+	pm_runtime_put_sync(mfg->dev);
+	return ret;
+}
+
+static void mtk_mfg_disable_hw_apm(struct mtk_mfg *mfg)
+{
+	writel(0x00, mfg->reg_base + 0xec);
+}
+
+void mtk_mfg_disable(struct mtk_mfg *mfg)
+{
+	mtk_mfg_disable_hw_apm(mfg);
+
+	mtk_mfg_disable_clock(mfg);
+	pm_runtime_put_sync(mfg->dev);
+}
+
+int mtk_mfg_freq_set(struct mtk_mfg *mfg, unsigned long freq)
+{
+	int ret;
+
+	ret = clk_prepare_enable(mfg->top_mfg);
+	if (ret) {
+		dev_err(mfg->dev, "enable and prepare top_mfg failed, %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_set_parent(mfg->top_mfg, mfg->clk26m);
+	if (ret) {
+		dev_err(mfg->dev, "Set clk parent to clk26m failed, %d\n", ret);
+		goto unprepare_top_mfg;
+	}
+
+	ret = clk_set_rate(mfg->mmpll, freq);
+	if (ret)
+		dev_err(mfg->dev, "Set freq to %lu Hz failed, %d\n", freq, ret);
+
+	ret = clk_set_parent(mfg->top_mfg, mfg->top_mmpll);
+	if (ret)
+		dev_err(mfg->dev, "Set clk parent to top_mmpll failed, %d\n", ret);
+
+unprepare_top_mfg:
+	clk_disable_unprepare(mfg->top_mfg);
+
+	if (!ret)
+		dev_dbg(mfg->dev, "Freq set to %lu Hz\n", freq);
+
+	return ret;
+}
+
+int mtk_mfg_volt_set(struct mtk_mfg *mfg, int volt)
+{
+	int ret;
+
+	ret = regulator_set_voltage(mfg->vgpu, volt, volt);
+	if (ret != 0) {
+		dev_err(mfg->dev, "Set voltage to %u uV failed, %d\n",
+			volt, ret);
+		return ret;
+	}
+
+	dev_dbg(mfg->dev, "Voltage set to %d uV\n", volt);
+
+	return 0;
+}
+
+static int mtk_mfg_bind_device_resource(struct mtk_mfg *mfg)
+{
+	struct device *dev = mfg->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	int i;
+	struct resource *res;
+
+	mfg->top_clk = devm_kcalloc(dev, MAX_TOP_MFG_CLK,
+				    sizeof(*mfg->top_clk), GFP_KERNEL);
+	if (!mfg->top_clk)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+	mfg->rgx_start = res->start;
+	mfg->rgx_size = resource_size(res);
+
+	mfg->rgx_irq = platform_get_irq_byname(pdev, "RGX");
+	if (mfg->rgx_irq < 0)
+		return mfg->rgx_irq;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	mfg->reg_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(mfg->reg_base))
+		return PTR_ERR(mfg->reg_base);
+
+	mfg->mmpll = devm_clk_get(dev, "mmpll_clk");
+	if (IS_ERR(mfg->mmpll)) {
+		dev_err(dev, "devm_clk_get mmpll_clk failed !!!\n");
+		return PTR_ERR(mfg->mmpll);
+	}
+
+	for (i = 0; i < MAX_TOP_MFG_CLK; i++) {
+		mfg->top_clk[i] = devm_clk_get(dev, top_mfg_clk_name[i]);
+		if (IS_ERR(mfg->top_clk[i])) {
+			dev_err(dev, "devm_clk_get %s failed !!!\n",
+				top_mfg_clk_name[i]);
+			return PTR_ERR(mfg->top_clk[i]);
+		}
+	}
+
+	mfg->top_mfg = devm_clk_get(dev, "top_mfg");
+	if (IS_ERR(mfg->top_mfg)) {
+		dev_err(dev, "devm_clk_get top_mfg failed !!!\n");
+		return PTR_ERR(mfg->top_mfg);
+	}
+
+	mfg->top_mmpll = devm_clk_get(dev, "top_mmpll");
+	if (IS_ERR(mfg->top_mmpll)) {
+		dev_err(dev, "devm_clk_get top_mmpll failed !!!\n");
+		return PTR_ERR(mfg->top_mmpll);
+	}
+
+	mfg->clk26m = devm_clk_get(dev, "clk26m");
+	if (IS_ERR(mfg->clk26m)) {
+		dev_err(dev, "devm_clk_get clk26m failed !!!\n");
+		return PTR_ERR(mfg->clk26m);
+	}
+
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	mfg->tz = thermal_zone_get_zone_by_name("cpu_thermal");
+	if (IS_ERR(mfg->tz)) {
+		dev_warn(dev, "Failed to get cpu_thermal zone\n");
+	}
+#endif
+
+	mfg->vgpu = devm_regulator_get(dev, "mfgsys-power");
+	if (IS_ERR(mfg->vgpu))
+		return PTR_ERR(mfg->vgpu);
+
+	pm_runtime_enable(dev);
+
+	return 0;
+}
+
+static void mtk_mfg_unbind_device_resource(struct mtk_mfg *mfg)
+{
+	struct device *dev = mfg->dev;
+
+	pm_runtime_disable(dev);
+}
+
+struct mtk_mfg *mtk_mfg_create(struct device *dev)
+{
+	int err;
+	struct mtk_mfg *mfg;
+
+	mtk_mfg_debug("mtk_mfg_create Begin\n");
+
+	mfg = devm_kzalloc(dev, sizeof(*mfg), GFP_KERNEL);
+	if (!mfg)
+		return ERR_PTR(-ENOMEM);
+	mfg->dev = dev;
+
+	err = mtk_mfg_bind_device_resource(mfg);
+	if (err != 0)
+		return ERR_PTR(err);
+
+	mutex_init(&mfg->set_power_state);
+
+	err = mtk_mfg_prepare_clock(mfg);
+	if (err)
+		goto err_unbind_resource;
+
+	mtk_mfg_debug("mtk_mfg_create End\n");
+
+	return mfg;
+err_unbind_resource:
+	mtk_mfg_unbind_device_resource(mfg);
+
+	return ERR_PTR(err);
+}
+
+void mtk_mfg_destroy(struct mtk_mfg *mfg)
+{
+	mtk_mfg_unprepare_clock(mfg);
+
+	mtk_mfg_unbind_device_resource(mfg);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.h
new file mode 100644
index 000000000000..c8f07e6f7a26
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_mfgsys.h
@@ -0,0 +1,63 @@
+/*
+* Copyright (c) 2014 MediaTek Inc.
+* Author: Chiawen Lee <chiawen.lee@mediatek.com>
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*/
+
+#ifndef MT8173_MFGSYS_H
+#define MT8173_MFGSYS_H
+
+#include <linux/device.h>
+
+/* unit ms, timeout interval for DVFS detection */
+#define MTK_DVFS_SWITCH_INTERVAL  300
+
+#define ENABLE_MTK_MFG_DEBUG 0
+
+#if ENABLE_MTK_MFG_DEBUG
+#define mtk_mfg_debug(fmt, args...) pr_info("[MFG]" fmt, ##args)
+#else
+#define mtk_mfg_debug(fmt, args...) do { } while (0)
+#endif
+
+struct mtk_mfg {
+	struct device *dev;
+
+	struct clk **top_clk;
+	void __iomem *reg_base;
+
+	resource_size_t rgx_start;
+	resource_size_t rgx_size;
+	int rgx_irq;
+
+	/* mutex protect for set power state */
+	struct mutex set_power_state;
+
+	/* for gpu device freq/volt update */
+	struct regulator *vgpu;
+	struct clk *mmpll;
+	struct clk *top_mfg;
+	struct clk *top_mmpll;
+	struct clk *clk26m;
+
+	struct thermal_zone_device *tz;
+};
+
+struct mtk_mfg *mtk_mfg_create(struct device *dev);
+void mtk_mfg_destroy(struct mtk_mfg *mfg);
+
+int mtk_mfg_enable(struct mtk_mfg *mfg);
+void mtk_mfg_disable(struct mtk_mfg *mfg);
+
+int mtk_mfg_freq_set(struct mtk_mfg *mfg, unsigned long freq);
+int mtk_mfg_volt_set(struct mtk_mfg *mfg, int volt);
+
+#endif /* MT8173_MFGSYS_H*/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_sysconfig.c
new file mode 100644
index 000000000000..58e4543670d9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/mt8173_sysconfig.c
@@ -0,0 +1,544 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/thermal.h>
+#if defined(CONFIG_DEVFREQ_THERMAL)
+#include <linux/devfreq_cooling.h>
+#endif
+
+#include "physheap.h"
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "syscommon.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#include "uma_heap_fns.h"
+
+#include "mt8173_mfgsys.h"
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS 10
+#define RGX_HW_CORE_CLOCK_SPEED 395000000
+#define MT8173_SYSTEM_NAME "mt8173"
+
+static IMG_HANDLE ghSysData;
+
+typedef struct
+{
+	IMG_UINT32 ui32IRQ;
+	PFN_LISR pfnLISR;
+	void *pvLISRData;
+} LISR_WRAPPER_DATA;
+
+static irqreturn_t MTKLISRWrapper(int iIrq, void *pvData)
+{
+	LISR_WRAPPER_DATA *psWrapperData = pvData;
+
+	if (psWrapperData->pfnLISR(psWrapperData->pvLISRData))
+	{
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static PVRSRV_ERROR MTKSysDevPrePowerState(
+		IMG_HANDLE hSysData,
+		PVRSRV_SYS_POWER_STATE eNewPowerState,
+		PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+		PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	struct mtk_mfg *mfg = hSysData;
+
+	mtk_mfg_debug("MTKSysDevPrePowerState (%d->%d), bPwrFlags = 0x%08x\n",
+		      eCurrentPowerState, eNewPowerState, ePwrFlags);
+
+	mutex_lock(&mfg->set_power_state);
+
+	if ((PVRSRV_SYS_POWER_STATE_OFF == eNewPowerState) &&
+	    (PVRSRV_SYS_POWER_STATE_ON == eCurrentPowerState))
+		mtk_mfg_disable(mfg);
+
+	mutex_unlock(&mfg->set_power_state);
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR MTKSysDevPostPowerState(
+		IMG_HANDLE hSysData,
+		PVRSRV_SYS_POWER_STATE eNewPowerState,
+		PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+		PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	struct mtk_mfg *mfg = hSysData;
+	PVRSRV_ERROR ret;
+
+	mtk_mfg_debug("MTKSysDevPostPowerState (%d->%d)\n",
+		      eCurrentPowerState, eNewPowerState);
+
+	mutex_lock(&mfg->set_power_state);
+
+	if ((PVRSRV_SYS_POWER_STATE_ON == eNewPowerState) &&
+	    (PVRSRV_SYS_POWER_STATE_OFF == eCurrentPowerState)) {
+		if (mtk_mfg_enable(mfg)) {
+			ret = PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE;
+			goto done;
+		}
+	}
+
+	ret = PVRSRV_OK;
+done:
+	mutex_unlock(&mfg->set_power_state);
+
+	return ret;
+}
+
+#ifdef SUPPORT_LINUX_DVFS
+#if defined(CONFIG_DEVFREQ_THERMAL)
+
+#define FALLBACK_STATIC_TEMPERATURE 65000
+
+/* Temperatures on power over-temp-and-voltage curve (C) */
+static const int vt_temperatures[] = { 25, 45, 65, 85, 105 };
+
+/* Voltages on power over-temp-and-voltage curve (mV) */
+static const int vt_voltages[] = { 900, 1000, 1130 };
+
+#define POWER_TABLE_NUM_TEMP ARRAY_SIZE(vt_temperatures)
+#define POWER_TABLE_NUM_VOLT ARRAY_SIZE(vt_voltages)
+
+static const unsigned int
+power_table[POWER_TABLE_NUM_VOLT][POWER_TABLE_NUM_TEMP] = {
+	/*   25     45      65      85     105 */
+	{ 14540, 35490,  60420, 120690, 230000 },  /*  900 mV */
+	{ 21570, 41910,  82380, 159140, 298620 },  /* 1000 mV */
+	{ 32320, 72950, 111320, 209290, 382700 },  /* 1130 mV */
+};
+
+/** Frequency and Power in Khz and mW respectively */
+static const int f_range[] = {253500, 299000, 396500, 455000, 494000, 598000};
+static const IMG_UINT32 max_dynamic_power[] = {612, 722, 957, 1100, 1194, 1445};
+
+static u32 interpolate(int value, const int *x, const unsigned int *y, int len)
+{
+	u64 tmp64;
+	u32 dx;
+	u32 dy;
+	int i, ret;
+
+	if (value <= x[0])
+		return y[0];
+	if (value >= x[len - 1])
+		return y[len - 1];
+
+	for (i = 1; i < len - 1; i++) {
+		/* If value is identical, no need to interpolate */
+		if (value == x[i])
+			return y[i];
+		if (value < x[i])
+			break;
+	}
+
+	/* Linear interpolation between the two (x,y) points */
+	dy = y[i] - y[i - 1];
+	dx = x[i] - x[i - 1];
+
+	tmp64 = value - x[i - 1];
+	tmp64 *= dy;
+	do_div(tmp64, dx);
+	ret = y[i - 1] + tmp64;
+
+	return ret;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+static unsigned long mtk_mfg_get_static_power(struct devfreq *df,
+					      unsigned long voltage)
+#else
+static unsigned long mtk_mfg_get_static_power(unsigned long voltage)
+#endif
+{
+	struct mtk_mfg *mfg = ghSysData;
+	struct thermal_zone_device *tz = mfg->tz;
+	unsigned long power;
+	int temperature = FALLBACK_STATIC_TEMPERATURE;
+	int low_idx = 0, high_idx = POWER_TABLE_NUM_VOLT - 1;
+	int i;
+
+	if (!tz)
+		return 0;
+
+	if (tz->ops->get_temp(tz, &temperature))
+		dev_warn(mfg->dev, "Failed to read temperature\n");
+	do_div(temperature, 1000);
+
+	for (i = 0; i < POWER_TABLE_NUM_VOLT; i++) {
+		if (voltage <= vt_voltages[POWER_TABLE_NUM_VOLT - 1 - i])
+			high_idx = POWER_TABLE_NUM_VOLT - 1 - i;
+
+		if (voltage >= vt_voltages[i])
+			low_idx = i;
+	}
+
+	if (low_idx == high_idx) {
+		power = interpolate(temperature,
+				    vt_temperatures,
+				    &power_table[low_idx][0],
+				    POWER_TABLE_NUM_TEMP);
+	} else {
+		unsigned long dvt =
+				vt_voltages[high_idx] - vt_voltages[low_idx];
+		unsigned long power1, power2;
+
+		power1 = interpolate(temperature,
+				     vt_temperatures,
+				     &power_table[high_idx][0],
+				     POWER_TABLE_NUM_TEMP);
+
+		power2 = interpolate(temperature,
+				     vt_temperatures,
+				     &power_table[low_idx][0],
+				     POWER_TABLE_NUM_TEMP);
+
+		power = (power1 - power2) * (voltage - vt_voltages[low_idx]);
+		do_div(power, dvt);
+		power += power2;
+	}
+
+	/* convert to mw */
+	do_div(power, 1000);
+
+	mtk_mfg_debug("mtk_mfg_get_static_power: %lu at Temperature %d\n",
+		      power, temperature);
+	return power;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+static unsigned long mtk_mfg_get_dynamic_power(struct devfreq *df,
+					       unsigned long freq,
+					       unsigned long voltage)
+#else
+static unsigned long mtk_mfg_get_dynamic_power(unsigned long freq,
+					       unsigned long voltage)
+#endif
+{
+	#define NUM_RANGE  ARRAY_SIZE(f_range)
+	/** Frequency and Power in Khz and mW respectively */
+	IMG_INT32 i, low_idx = 0, high_idx = NUM_RANGE - 1;
+	IMG_UINT32 power;
+
+	for (i = 0; i < NUM_RANGE; i++) {
+		if (freq <= f_range[NUM_RANGE - 1 - i])
+			high_idx = NUM_RANGE - 1 - i;
+
+		if (freq >= f_range[i])
+			low_idx = i;
+	}
+
+	if (low_idx == high_idx) {
+		power = max_dynamic_power[low_idx];
+	} else {
+		IMG_UINT32 f_interval = f_range[high_idx] - f_range[low_idx];
+		IMG_UINT32 p_interval = max_dynamic_power[high_idx] -
+				max_dynamic_power[low_idx];
+
+		power = p_interval * (freq - f_range[low_idx]);
+		do_div(power, f_interval);
+		power += max_dynamic_power[low_idx];
+	}
+
+	power = (IMG_UINT32)div_u64((IMG_UINT64)power * voltage * voltage,
+				    1000000UL);
+
+	return power;
+	#undef NUM_RANGE
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0))
+static int mtk_mfg_get_real_power(struct devfreq *df,
+						   u32 *power,
+					       unsigned long freq,
+					       unsigned long voltage)
+{
+	if (!df || !power)
+		return -EINVAL;
+
+	*power = mtk_mfg_get_static_power(df, voltage) +
+		     mtk_mfg_get_dynamic_power(df, freq, voltage);
+
+	return 0;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) */
+
+static struct devfreq_cooling_power sPowerOps = {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+	.get_static_power = mtk_mfg_get_static_power,
+	.get_dynamic_power = mtk_mfg_get_dynamic_power,
+#else
+	.get_real_power = mtk_mfg_get_real_power,
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+};
+#endif
+
+static void SetFrequency(IMG_HANDLE hSysData, IMG_UINT32 freq)
+{
+	struct mtk_mfg *mfg = hSysData;
+
+	/* freq is in Hz */
+	mtk_mfg_freq_set(mfg, freq);
+}
+
+static void SetVoltage(IMG_HANDLE hSysData, IMG_UINT32 volt)
+{
+	struct mtk_mfg *mfg = hSysData;
+
+	mtk_mfg_volt_set(mfg, volt);
+}
+#endif
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   struct mtk_mfg *mfg,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *psPhysHeapConfig;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo) +
+							  sizeof(*psPhysHeapConfig));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+	psPhysHeapConfig = (PHYS_HEAP_CONFIG *)((IMG_CHAR *)psRGXTimingInfo + sizeof(*psRGXTimingInfo));
+
+	/* Set up the RGX timing information */
+	psRGXTimingInfo->ui32CoreClockSpeed = RGX_HW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM = IMG_TRUE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_TRUE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	psPhysHeapConfig->eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeapConfig->ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeapConfig->uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeapConfig->uConfig.sUMA.psMemFuncs = &g_sUmaHeapFns;
+	psPhysHeapConfig->uConfig.sUMA.pszHeapName = "uma_gpu_local";
+	psPhysHeapConfig->uConfig.sUMA.hPrivData = NULL;
+
+	psDevConfig->pasPhysHeaps = psPhysHeapConfig;
+	psDevConfig->ui32PhysHeapCount = 1U;
+
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevConfig->pszName = MT8173_SYSTEM_NAME;
+	psDevConfig->pszVersion = NULL;
+
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+	psDevConfig->bDevicePA0IsValid = IMG_FALSE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = (IMG_HANDLE) mfg;
+	ghSysData = psDevConfig->hSysData;
+
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	psDevConfig->ui32IRQ = mfg->rgx_irq;
+
+	psDevConfig->sRegsCpuPBase.uiAddr = mfg->rgx_start;
+	psDevConfig->ui32RegsSize = mfg->rgx_size;
+
+#ifdef SUPPORT_LINUX_DVFS
+	psDevConfig->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32PollMs = MTK_DVFS_SWITCH_INTERVAL;
+#if defined(CONFIG_DEVFREQ_THERMAL)
+	psDevConfig->sDVFS.sDVFSDeviceCfg.psPowerOps = &sPowerOps;
+#endif
+
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	/* power management on HW system */
+	psDevConfig->pfnPrePowerState = MTKSysDevPrePowerState;
+	psDevConfig->pfnPostPowerState = MTKSysDevPostPowerState;
+
+	/* clock frequency */
+	psDevConfig->pfnClockFreqGet = NULL;
+
+	/* device error notify callback function */
+	psDevConfig->pfnSysDevErrorNotify = NULL;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	/*
+	 * The device config, RGX data and RGX timing info are part of the same
+	 * allocation so do only one free.
+	 */
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	struct device *dev = pvOSDevice;
+	struct mtk_mfg *mfg;
+	PVRSRV_ERROR eError;
+
+	mfg = mtk_mfg_create(dev);
+	if (IS_ERR(mfg)) {
+		if (PTR_ERR(mfg) == -EPROBE_DEFER)
+			return PVRSRV_ERROR_PROBE_DEFER;
+		else
+			return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	dma_set_mask(dev, DMA_BIT_MASK(33));
+
+	eError = DeviceConfigCreate(pvOSDevice, mfg, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		mtk_mfg_destroy(mfg);
+
+		return eError;
+	}
+
+	*ppsDevConfig = psDevConfig;
+
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	return PVRSRV_OK;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	struct mtk_mfg *mfg = psDevConfig->hSysData;
+
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	DeviceConfigDestroy(psDevConfig);
+
+	mtk_mfg_destroy(mfg);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	LISR_WRAPPER_DATA *psWrapperData;
+
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	psWrapperData = OSAllocMem(sizeof(*psWrapperData));
+	if (!psWrapperData)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psWrapperData->ui32IRQ = ui32IRQ;
+	psWrapperData->pfnLISR = pfnLISR;
+	psWrapperData->pvLISRData = pvData;
+
+	if (request_irq(ui32IRQ, MTKLISRWrapper, IRQF_TRIGGER_LOW, pszName,
+					psWrapperData))
+	{
+		OSFreeMem(psWrapperData);
+
+		return PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER;
+	}
+
+	*phLISRData = (IMG_HANDLE) psWrapperData;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_WRAPPER_DATA *psWrapperData = hLISRData;
+
+	free_irq(psWrapperData->ui32IRQ, psWrapperData);
+
+	OSFreeMem(psWrapperData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+			  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			  void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/sysinfo.h
new file mode 100644
index 000000000000..bd7bc38e3412
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/mt8173/sysinfo.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (1000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (20000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#define SYS_RGX_OF_COMPATIBLE "mediatek,mt8173-gpu"
+
+#endif	/* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/Kbuild.mk
new file mode 100644
index 000000000000..55d0369bcc6d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/Kbuild.mk
@@ -0,0 +1,49 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+	services/system/rogue/$(PVR_SYSTEM)/sysconfig.o \
+	services/system/common/sysconfig_cmn.o
+
+ccflags-y += \
+	-I$(TOP)/services/system/rogue/common/env/linux -I$(TOP)/services/system/rogue/common/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc \
+	-I$(TOP)/include/rogue/system/rgx_tc -I$(TOP)/include/system/rgx_tc
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysconfig.c
new file mode 100644
index 000000000000..16191d7de281
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysconfig.c
@@ -0,0 +1,1740 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "sysinfo.h"
+#include "apollo_regs.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "syscommon.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwutils.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_support.h"
+#include "ion_sys.h"
+#endif
+
+#include "vmm_pvz_server.h"
+#include "pvr_bridge_k.h"
+#include "pvr_drv.h"
+#include "tc_drv.h"
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+
+#define SECURE_FW_MEM_SIZE (0x400000)  /*  4MB */
+#define SECURE_MEM_SIZE    (0x4000000) /* 64MB */
+
+typedef struct
+{
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;
+	IMG_UINT64 uiSize;
+	IMG_BOOL bUsed;
+} CARD_PHYS_HEAP_CONFIG_SPEC;
+
+#define HEAP_SPEC_IDX_GPU_PRIVATE (0U)
+#define HEAP_SPEC_IDX_GPU_LOCAL   (1U)
+
+static const CARD_PHYS_HEAP_CONFIG_SPEC gasCardHeapTemplate[] =
+{
+	{
+	 PHYS_HEAP_USAGE_GPU_PRIVATE,
+	 0,					/* determined at runtime by apphints */
+	 false				/* determined at runtime by apphints */
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_LOCAL,
+	 0,					/* determined at runtime */
+	 true
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_SECURE,
+	 SECURE_MEM_SIZE,
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	 true
+#else
+	 false
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PRIVATE,
+	 SECURE_FW_MEM_SIZE,
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	 true
+#else
+	 false
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_SHARED,
+#if defined(SUPPORT_SECURITY_VALIDATION) && defined(RGX_PREMAP_FW_HEAPS)
+	 /* simultaneous virtualisation and security support requires premapped heaps,
+	  * i.e. FW_PRIVATE and FW_SHARED must fit contiguously into Fw's VA heap (RGX_FIRMWARE_RAW_HEAP_SIZE) */
+	 RGX_FIRMWARE_RAW_HEAP_SIZE - SECURE_FW_MEM_SIZE,
+#else
+	 RGX_FIRMWARE_RAW_HEAP_SIZE,
+#endif
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	true				/* VZ drivers need dedicated Fw heaps */
+#else
+	false				/* Native drivers can fallback on GPU_LOCAL for Fw mem */
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PREMAP_PT,
+	 RGX_FIRMWARE_MAX_PAGETABLE_SIZE,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	 true
+#else
+	 false
+#endif
+	}
+};
+
+#define ODIN_MEMORY_HYBRID_DEVICE_BASE 0x400000000
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+
+#define UI64_TOPWORD_IS_ZERO(ui64) ((ui64 >> 32) == 0)
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+
+/* Fake DVFS configuration used purely for testing purposes */
+
+static const IMG_OPP asOPPTable[] =
+{
+	{ 8,  25000000},
+	{ 16, 50000000},
+	{ 32, 75000000},
+	{ 64, 100000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+static void SetFrequency(IMG_HANDLE hSysData, IMG_UINT32 ui32Frequency)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetFrequency %u", ui32Frequency));
+}
+
+static void SetVoltage(IMG_HANDLE hSysData, IMG_UINT32 ui32Voltage)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetVoltage %u", ui32Voltage));
+}
+
+#endif
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCLocalCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCLocalDevPAddrToCpuPAddr,
+};
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHostCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHostDevPAddrToCpuPAddr,
+};
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHybridPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHybridCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHybridDevPAddrToCpuPAddr
+};
+
+typedef struct _SYS_DATA_ SYS_DATA;
+
+struct _SYS_DATA_
+{
+	struct platform_device *pdev;
+
+	struct tc_rogue_platform_data *pdata;
+
+	struct resource *registers;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_client *ion_client;
+	struct ion_handle *ion_rogue_allocation;
+#endif
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	void *pvS3Buffer;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+};
+
+#define SYSTEM_INFO_FORMAT_STRING "FPGA Revision: %s - TCF Core Revision: %s - TCF Core Target Build ID: %s - PCI Version: %s - Macro Version: %s"
+#define FPGA_REV_MAX_LEN      8 /* current longest format: "x.y.z" */
+#define TCF_CORE_REV_MAX_LEN  8 /* current longest format: "x.y.z" */
+#define TCF_CORE_CFG_MAX_LEN  4 /* current longest format: "x" */
+#define PCI_VERSION_MAX_LEN   4 /* current longest format: "x" */
+#define MACRO_VERSION_MAX_LEN 8 /* current longest format: "x.yz" */
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	int err;
+	char str_fpga_rev[FPGA_REV_MAX_LEN]={0};
+	char str_tcf_core_rev[TCF_CORE_REV_MAX_LEN]={0};
+	char str_tcf_core_target_build_id[TCF_CORE_CFG_MAX_LEN]={0};
+	char str_pci_ver[PCI_VERSION_MAX_LEN]={0};
+	char str_macro_ver[MACRO_VERSION_MAX_LEN]={0};
+
+	IMG_CHAR *pszVersion;
+	IMG_UINT32 ui32StringLength;
+
+	err = tc_sys_strings(psSysData->pdev->dev.parent,
+							 str_fpga_rev, sizeof(str_fpga_rev),
+							 str_tcf_core_rev, sizeof(str_tcf_core_rev),
+							 str_tcf_core_target_build_id, sizeof(str_tcf_core_target_build_id),
+							 str_pci_ver, sizeof(str_pci_ver),
+							 str_macro_ver, sizeof(str_macro_ver));
+	if (err)
+	{
+		return NULL;
+	}
+
+	/* Calculate how much space we need to allocate for the string */
+	ui32StringLength = OSStringLength(SYSTEM_INFO_FORMAT_STRING);
+	ui32StringLength += OSStringLength(str_fpga_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_target_build_id);
+	ui32StringLength += OSStringLength(str_pci_ver);
+	ui32StringLength += OSStringLength(str_macro_ver);
+
+	/* Create the version string */
+	pszVersion = OSAllocMem(ui32StringLength * sizeof(IMG_CHAR));
+	if (pszVersion)
+	{
+		OSSNPrintf(&pszVersion[0], ui32StringLength,
+				   SYSTEM_INFO_FORMAT_STRING,
+				   str_fpga_rev,
+				   str_tcf_core_rev,
+				   str_tcf_core_target_build_id,
+				   str_pci_ver,
+				   str_macro_ver);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to create format string", __func__));
+	}
+
+	return pszVersion;
+}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+static SYS_DATA *gpsIonPrivateData;
+
+PVRSRV_ERROR IonInit(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYS_DATA *psSysData = pvPrivateData;
+	gpsIonPrivateData = psSysData;
+
+	psSysData->ion_client = ion_client_create(psSysData->pdata->ion_device, SYS_RGX_DEV_NAME);
+	if (IS_ERR(psSysData->ion_client))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create ION client (%ld)", __func__, PTR_ERR(psSysData->ion_client)));
+		eError = PVRSRV_ERROR_ION_NO_CLIENT;
+		goto err_out;
+	}
+	/* Allocate the whole rogue ion heap and pass that to services to manage */
+	psSysData->ion_rogue_allocation = ion_alloc(psSysData->ion_client, psSysData->pdata->rogue_heap_memory_size, 4096, (1 << psSysData->pdata->ion_heap_id), 0);
+	if (IS_ERR(psSysData->ion_rogue_allocation))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate ION rogue buffer (%ld)", __func__, PTR_ERR(psSysData->ion_rogue_allocation)));
+		eError = PVRSRV_ERROR_ION_FAILED_TO_ALLOC;
+		goto err_destroy_client;
+
+	}
+
+	return PVRSRV_OK;
+err_destroy_client:
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+err_out:
+	return eError;
+}
+
+void IonDeinit(void)
+{
+	SYS_DATA *psSysData = gpsIonPrivateData;
+	ion_free(psSysData->ion_client, psSysData->ion_rogue_allocation);
+	psSysData->ion_rogue_allocation = NULL;
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+}
+
+struct ion_device *IonDevAcquire(void)
+{
+	return gpsIonPrivateData->pdata->ion_device;
+}
+
+void IonDevRelease(struct ion_device *ion_device)
+{
+	PVR_ASSERT(ion_device == gpsIonPrivateData->pdata->ion_device);
+}
+#endif /* defined(SUPPORT_ION) */
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+			psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+			psDevPAddr[ui32Idx].uiAddr + psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	if (sizeof(*psDevPAddr) == sizeof(*psCpuPAddr))
+	{
+		OSCachedMemCopy(psDevPAddr, psCpuPAddr, uiNumOfAddr * sizeof(*psDevPAddr));
+		return;
+	}
+
+	/* In this case we may have a 32bit host, so we can't do a memcpy */
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (uiNumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < uiNumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	if (sizeof(*psCpuPAddr) == sizeof(*psDevPAddr))
+	{
+		OSCachedMemCopy(psCpuPAddr, psDevPAddr, uiNumOfAddr * sizeof(*psCpuPAddr));
+		return;
+	}
+
+	/* In this case we may have a 32bit host, so we can't do a memcpy.
+	 * Check we are not dropping any data from the 64bit dev addr */
+	PVR_ASSERT(UI64_TOPWORD_IS_ZERO(psDevPAddr[0].uiAddr));
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (uiNumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < uiNumOfAddr; ++ui32Idx)
+		{
+			PVR_ASSERT(UI64_TOPWORD_IS_ZERO(psDevPAddr[ui32Idx].uiAddr));
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+
+}
+
+static inline
+IMG_CHAR* GetHeapName(PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_LOCAL))    return "lma_gpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_SECURE))   return "lma_gpu_secure";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_PRIVATE))  return "lma_gpu_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PRIVATE))   return "lma_fw_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_SHARED))    return "lma_fw_shared";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PREMAP_PT)) return "lma_fw_pagetables";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_CPU_LOCAL))    return "lma_cpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_DISPLAY))      return "lma_gpu_display";
+	else                                                     return "Unexpected Heap";
+}
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+		    (psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base) +
+		    ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+}
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+		    (psDevPAddr[ui32Idx].uiAddr - ODIN_MEMORY_HYBRID_DEVICE_BASE) +
+		    psSysData->pdata->tc_memory_base;
+	}
+}
+
+static PVRSRV_ERROR
+InitLocalHeap(PHYS_HEAP_CONFIG *psPhysHeap,
+			  IMG_UINT64 uiBaseAddr, IMG_UINT64 uiStartAddr,
+			  IMG_UINT64 uiSize, PHYS_HEAP_FUNCTIONS *psFuncs,
+			  PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = ui32Flags;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = psFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = GetHeapName(ui32Flags);
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = uiBaseAddr;
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(uiStartAddr);
+	psPhysHeap->uConfig.sLMA.uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+CreateCardGPUHeaps(const SYS_DATA *psSysData,
+				   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+				   PHYS_HEAP_CONFIG *pasPhysHeaps,
+				   PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				   IMG_UINT32 *pui32HeapIdx,
+				   IMG_UINT64 ui64CardAddr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->rogue_heap_memory_base;
+	IMG_UINT32 ui32SpecIdx;
+
+	for (ui32SpecIdx = 0; ui32SpecIdx < ARRAY_SIZE(gasCardHeapTemplate); ui32SpecIdx++)
+	{
+		if (pasCardHeapSpec[ui32SpecIdx].bUsed)
+		{
+			IMG_UINT64 ui64HeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+
+			eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+								   ui64CardAddr,
+								   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+								   ui64HeapSize,
+								   psHeapFuncs,
+								   pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			ui64CardAddr  += pasCardHeapSpec[ui32SpecIdx].uiSize;
+			ui64StartAddr += pasCardHeapSpec[ui32SpecIdx].uiSize;
+			(*pui32HeapIdx)++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+static PVRSRV_ERROR
+CreateCardEXTHeap(const SYS_DATA *psSysData,
+				  PHYS_HEAP_CONFIG *pasPhysHeaps,
+				  PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				  IMG_UINT32 *pui32HeapIdx,
+				  IMG_UINT64 ui64CardBase)
+{
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->pdp_heap_memory_base;
+	IMG_UINT64 ui64Size = psSysData->pdata->pdp_heap_memory_size;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+						   ui64CardBase + psSysData->pdata->rogue_heap_memory_size,
+						   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+						   ui64Size, psHeapFuncs,
+						   PHYS_HEAP_USAGE_EXTERNAL | PHYS_HEAP_USAGE_DISPLAY);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	(*pui32HeapIdx)++;
+
+	return PVRSRV_OK;
+}
+#endif
+
+static PVRSRV_ERROR
+InitLocalHeaps(const SYS_DATA *psSysData,
+			   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			   PHYS_HEAP_CONFIG *pasPhysHeaps,
+			   IMG_UINT32 *pui32HeapIdx)
+{
+	PHYS_HEAP_FUNCTIONS *psHeapFuncs;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64CardBase;
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHybridPhysHeapFuncs;
+		ui64CardBase = ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+	else if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHostPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->tc_memory_base;
+	}
+	else
+	{
+		psHeapFuncs = &gsLocalPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->rogue_heap_memory_base - psSysData->pdata->tc_memory_base;
+	}
+
+	eError = CreateCardGPUHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+	eError = CreateCardEXTHeap(psSysData, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+InitHostHeaps(const SYS_DATA *psSysData, PHYS_HEAP_CONFIG *pasPhysHeaps, IMG_UINT32 *pui32HeapIdx)
+{
+	if (psSysData->pdata->mem_mode != TC_MEMORY_LOCAL)
+	{
+		pasPhysHeaps[*pui32HeapIdx].eType = PHYS_HEAP_TYPE_UMA;
+		pasPhysHeaps[*pui32HeapIdx].ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.psMemFuncs = &gsHostPhysHeapFuncs;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszHeapName = "uma_cpu_local";
+
+		(*pui32HeapIdx)++;
+
+		PVR_DPF((PVR_DBG_WARNING,
+				 "Initialising CPU_LOCAL UMA Host PhysHeaps with memory mode: %d",
+				 psSysData->pdata->mem_mode));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsInit(const SYS_DATA *psSysData,
+			  CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			  PHYS_HEAP_CONFIG *pasPhysHeaps,
+			  void *pvPrivData, IMG_UINT32 ui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32HeapCounter = 0;
+
+	eError = InitLocalHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	eError = InitHostHeaps(psSysData, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((ui32HeapCounter == ui32NumHeaps),
+							"Number of PhysHeapConfigs set up doesn't match the initial requirement.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	/* Initialise fields that don't change between memory modes.
+	 * Fix up heap IDs. This is needed for multi-testchip systems to
+	 * ensure the heap IDs are unique as this is what Services expects.
+	 */
+	for (i = 0; i < ui32NumHeaps; i++)
+	{
+		switch (pasPhysHeaps[i].eType)
+		{
+		case PHYS_HEAP_TYPE_UMA:
+			pasPhysHeaps[i].uConfig.sUMA.hPrivData = pvPrivData;
+			break;
+		case PHYS_HEAP_TYPE_LMA:
+			pasPhysHeaps[i].uConfig.sLMA.hPrivData = pvPrivData;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Invalid PHYS_HEAP_TYPE: %u in %s",
+			                        pasPhysHeaps[i].eType,
+			                        __func__));
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapSetRequirements(const SYS_DATA *psSysData,
+						CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+						IMG_UINT32 *pui32CardPhysHeapCfgCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64FreeCardMemory = psSysData->pdata->rogue_heap_memory_size;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_PRIVATE) &&
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_LOCAL),
+		"PhysHeapConfigs not set correctly in the system layer.", PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < ARRAY_SIZE(gasCardHeapTemplate); i++)
+	{
+		if (pasCardHeapSpec[i].bUsed)
+		{
+			/* Determine the memory requirements of heaps with a fixed size */
+			ui64FreeCardMemory -= pasCardHeapSpec[i].uiSize;
+
+			/* Count card physheap configs used by the system */
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	if (SysRestrictGpuLocalAddPrivateHeap())
+	{
+		IMG_UINT64 ui64GpuSharedMem = SysRestrictGpuLocalPhysheap(ui64FreeCardMemory);
+
+		if (ui64GpuSharedMem == ui64FreeCardMemory)
+		{
+			/* No memory reserved for GPU private use, special heap not needed */
+		}
+		else
+		{
+			/* Set up the GPU private heap */
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed = true;
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].uiSize = ui64FreeCardMemory - ui64GpuSharedMem;
+			ui64FreeCardMemory = ui64GpuSharedMem;
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	/* all remaining memory card memory goes to GPU_LOCAL */
+	pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].uiSize = ui64FreeCardMemory;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsCreate(const SYS_DATA *psSysData, PVRSRV_DEVICE_CONFIG *psDevConfig,
+				PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+				IMG_UINT32 *puiPhysHeapCountOut)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps = 0;
+	CARD_PHYS_HEAP_CONFIG_SPEC asCardHeapSpec[ARRAY_SIZE(gasCardHeapTemplate)];
+
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->mem_mode == TC_MEMORY_LOCAL) ||
+							(psSysData->pdata->mem_mode == TC_MEMORY_HYBRID),
+							"Unsupported memory mode", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	/* Initialise the local heap specs with the build-time template */
+	memcpy(asCardHeapSpec, gasCardHeapTemplate, sizeof(gasCardHeapTemplate));
+
+	eError = PhysHeapSetRequirements(psSysData, asCardHeapSpec, &ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psDevConfig->bHasNonMappableLocalMemory = asCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed;
+
+	if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* CPU_LOCAL heap */
+		ui32NumHeaps++;
+	}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL ||
+		psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* EXTERNAL / DISPLAY heap */
+		ui32NumHeaps++;
+	}
+#endif
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * ui32NumHeaps);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = PhysHeapsInit(psSysData, asCardHeapSpec, pasPhysHeaps, psDevConfig, ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = ui32NumHeaps;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->pszVersion)
+	{
+		OSFreeMem(psDevConfig->pszVersion);
+	}
+
+	OSFreeMem(psDevConfig->pasPhysHeaps);
+
+	OSFreeMem(psDevConfig);
+}
+
+static void odinTCDevPhysAddr2DmaAddr(PVRSRV_DEVICE_CONFIG *psDevConfig,
+									  IMG_DMA_ADDR *psDmaAddr,
+									  IMG_DEV_PHYADDR *psDevPAddr,
+									  IMG_BOOL *pbValid,
+									  IMG_UINT32 ui32NumAddr,
+									  IMG_BOOL bSparseAlloc)
+{
+	IMG_CPU_PHYADDR sCpuPAddr = {0};
+	IMG_UINT32 ui32Idx;
+
+	/* Fast path */
+	if (!bSparseAlloc)
+	{
+		/* In Odin, DMA address space is the same as host CPU */
+		TCLocalDevPAddrToCpuPAddr(psDevConfig,
+								  1,
+								  &sCpuPAddr,
+								  psDevPAddr);
+		psDmaAddr->uiAddr = sCpuPAddr.uiAddr;
+	}
+	else
+	{
+		for (ui32Idx = 0; ui32Idx < ui32NumAddr; ui32Idx++)
+		{
+			if (pbValid[ui32Idx])
+			{
+				TCLocalDevPAddrToCpuPAddr(psDevConfig,
+										  1,
+										  &sCpuPAddr,
+										  &psDevPAddr[ui32Idx]);
+				psDmaAddr[ui32Idx].uiAddr = sCpuPAddr.uiAddr;
+			}
+			else
+			{
+				/* Invalid DMA address marker */
+				psDmaAddr[ui32Idx].uiAddr = ~((IMG_UINT64)0x0);
+			}
+		}
+	}
+}
+
+static void * odinTCgetCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig, char *name)
+{
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	return tc_dma_chan(psDev->parent, name);
+}
+
+static void odinTCFreeCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig,
+							   void* channel)
+{
+
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	struct dma_chan *chan = (struct dma_chan*) channel;
+
+	tc_dma_chan_free(psDev->parent, chan);
+}
+
+static void GetDriverMode(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DeviceID;
+
+	/*
+	 * Drivers with virtualization support should check if the mode in which the
+	 * driver must control a device has been explicitly specified at load time
+	 * through module parameters.
+	 * Multi-device platforms must find the internal ID of the device currently
+	 * being created when checking for its associated DriverMode parameter.
+	 */
+	if (PVRSRVAcquireInternalID(&ui32DeviceID) != PVRSRV_OK)
+	{
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		return;
+	}
+
+	if (psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID] == DRIVER_MODE_DEFAULT)
+	{
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		void __iomem *pvRegBase;
+
+		pvRegBase = (void __iomem *) OSMapPhysToLin(psDevConfig->sRegsCpuPBase, psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (pvRegBase == NULL)
+		{
+			/* failed to map register bank, default to native mode */
+			psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		}
+		else
+		{
+			IMG_UINT64 ui64ClkCtrl;
+
+			/* the CLK_CTRL register is valid only in the Os 0 (Host) register bank
+			 * if it reads 0 then we can conclude this Os is set up to run as Guest */
+#if defined(RGX_CR_CLK_CTRL)
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL);
+#else
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL1);
+#endif
+			OSUnMapPhysToLin((void __force *) pvRegBase, psDevConfig->ui32RegsSize);
+
+			psDevConfig->eDriverMode = (ui64ClkCtrl != 0) ? (DRIVER_MODE_HOST) : (DRIVER_MODE_GUEST);
+		}
+#else
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+#endif
+	}
+	else
+	{
+		psDevConfig->eDriverMode = psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID];
+	}
+}
+
+static PVRSRV_ERROR DeviceConfigCreate(SYS_DATA *psSysData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+							 psSysData->pdata->mem_mode == TC_MEMORY_LOCAL),
+							"Multidevice virtualization setup supported only on Odin device with TC_MEMORY_LOCAL",
+							PVRSRV_ERROR_INVALID_DEVICE);
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *) IMG_OFFSET_ADDR(psDevConfig, sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *) IMG_OFFSET_ADDR(psRGXData, sizeof(*psRGXData));
+
+	/* Setup RGX specific timing data */
+#if defined(TC_APOLLO_BONNIE)
+	/* For BonnieTC there seems to be an additional 5x multiplier that occurs to the clock as measured speed is 540Mhz not 108Mhz. */
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent)  * 6 * 5;
+#elif defined(TC_APOLLO_ES2)
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent)  * 6;
+#else
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent) /
+											tc_core_clock_multiplex(psSysData->pdev->dev.parent);
+#endif
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice = &psSysData->pdev->dev;
+	psDevConfig->pszName = "tc";
+	psDevConfig->pszVersion = GetDeviceVersionString(psSysData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr = psSysData->registers->start;
+	psDevConfig->ui32RegsSize = resource_size(psSysData->registers);
+
+	PVRSRVAcquireInternalID(&ui32DeviceID);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* Rogue FPGA images are correctly routing the OSID interrupts
+	 * for cores with the IRQ_PER_OS feature */
+	psDevConfig->ui32IRQ = TC_INTERRUPT_OSID0 + ui32DeviceID;
+#else
+	psDevConfig->ui32IRQ = TC_INTERRUPT_EXT;
+#endif
+
+	GetDriverMode(psDevConfig);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* If there is device running in native mode, prevent any attempts at
+	 * creating any Guest devices, as there will be no Host to support them.
+	 * Currently the VZFPGA supports only one physical GPU. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+		PVRSRV_DEVICE_NODE *psDN;
+
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		for (psDN = psPVRSRVData->psDeviceNodeList; psDN != NULL; psDN = psDN->psNext)
+		{
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDN))
+			{
+				OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+				PVR_DPF((PVR_DBG_ERROR, "%s() Device %u is already running in native mode, no other Guests supported in the system.",  __func__, psDN->sDevId.ui32InternalID));
+				eError = PVRSRV_ERROR_INVALID_DEVICE;
+				goto ErrorFreeDevConfig;
+			}
+		}
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+#endif
+
+	eError = PhysHeapsCreate(psSysData, psDevConfig, &pasPhysHeaps, &uiPhysHeapCount);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psDevConfig->eDefaultHeap = SysDefaultToCpuLocalHeap() ?
+		    PVRSRV_PHYS_HEAP_CPU_LOCAL : PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+	else
+	{
+		psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+
+	psDevConfig->pasPhysHeaps = pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount = uiPhysHeapCount;
+
+	/* Only required for LMA but having this always set shouldn't be a problem */
+	psDevConfig->bDevicePA0IsValid = IMG_TRUE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psSysData;
+
+#if defined(SUPPORT_ALT_REGBASE)
+	if (psSysData->pdata->mem_mode != TC_MEMORY_LOCAL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: alternative GPU register base is "
+					"supported only in LMA mode", __func__));
+		goto ErrorFreeDevConfig;
+	}
+	else
+	{
+		/* Using display memory base as the alternative GPU register base,
+		 * since the display memory range is not used by the firmware. */
+		IMG_CPU_PHYADDR  sDisplayMemAddr;
+
+		sDisplayMemAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psSysData->pdata->pdp_heap_memory_base);
+		TCLocalCpuPAddrToDevPAddr(psDevConfig, 1,
+		                          &psDevConfig->sAltRegsGpuPBase,
+		                          &sDisplayMemAddr);
+	}
+#endif
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/* Fake DVFS configuration used purely for testing purposes */
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32PollMs = 1000;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	/* DMA channel config */
+	psDevConfig->pfnSlaveDMAGetChan = odinTCgetCDMAChan;
+	psDevConfig->pfnSlaveDMAFreeChan = odinTCFreeCDMAChan;
+	psDevConfig->pfnDevPhysAddr2DmaAddr = odinTCDevPhysAddr2DmaAddr;
+	psDevConfig->pszDmaTxChanName = psSysData->pdata->tc_dma_tx_chan_name;
+	psDevConfig->pszDmaRxChanName = psSysData->pdata->tc_dma_rx_chan_name;
+	psDevConfig->bHasDma = IMG_TRUE;
+	/* Following two values are expressed in number of bytes */
+	psDevConfig->ui32DmaTransferUnit = 1;
+	psDevConfig->ui32DmaAlignment = 1;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+/* #define _DBG(...) PVR_LOG((__VA_ARGS__)) */
+#define _DBG(...)
+
+static PVRSRV_ERROR PrePower(IMG_HANDLE hSysData,
+                             PVRSRV_SYS_POWER_STATE eNewPowerState,
+                             PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                             PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize;
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags: 0x%08x", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags);
+
+	/* The transition might be both from ON or OFF states to OFF state so check
+	 * only for the *new* state. Also this is only valid for suspend requests. */
+	if (eNewPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = LMA_HeapIteratorCreate(psSysData->psDevConfig->psDevNode,
+	                                PVRSRV_PHYS_HEAP_GPU_LOCAL,
+	                                &psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorCreate", return_error);
+
+	eError = LMA_HeapIteratorGetHeapStats(psSysData->psHeapIter, &uiHeapTotalSize,
+	                                      &uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorGetHeapStats",
+	                      return_error);
+	uiHeapFreeSize = uiHeapTotalSize - uiHeapUsedSize;
+
+	_DBG("(%s()) heap stats: total=0x%" IMG_UINT64_FMTSPECx ", "
+	     "used=0x%" IMG_UINT64_FMTSPECx ", free=0x%" IMG_UINT64_FMTSPECx,
+	     __func__, uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize);
+
+	psSysData->pvS3Buffer = OSAllocMem(uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_NOMEM(psSysData->pvS3Buffer, eError, destroy_iterator);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		if (uiOffset + uiSize > uiHeapUsedSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "uiOffset = %" IMG_UINT64_FMTSPECx ", "
+			        "uiSize = %" IMG_UINT64_FMTSPECx, uiOffset, uiSize));
+
+			PVR_LOG_GOTO_WITH_ERROR("LMA_HeapIteratorNext", eError,
+			                        PVRSRV_ERROR_INVALID_OFFSET,
+			                        free_buffer);
+		}
+
+		TCLocalDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                          &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy((IMG_BYTE *) psSysData->pvS3Buffer + uiOffset, pvCpuVAddr,
+		       uiSize);
+		/* and now poison it */
+		memset(pvCpuVAddr, 0x9b, uiSize);
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+destroy_iterator:
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+return_error:
+	return eError;
+}
+
+static PVRSRV_ERROR PostPower(IMG_HANDLE hSysData,
+                              PVRSRV_SYS_POWER_STATE eNewPowerState,
+                              PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags=0x%08x; buffer=%px", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags,
+	     psSysData->pvS3Buffer);
+
+	/* The transition might be both to ON or OFF states from OFF state so check
+	 * only for the *current* state. Also this is only valid for resume requests. */
+	if (eCurrentPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ) ||
+	    psSysData->pvS3Buffer == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = LMA_HeapIteratorReset(psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorReset", free_buffer);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		TCLocalDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                          &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy(pvCpuVAddr, (IMG_BYTE *) psSysData->pvS3Buffer + uiOffset,
+		       uiSize);
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+
+	return eError;
+}
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psSysData;
+	resource_size_t uiRegistersSize;
+	IMG_UINT32 ui32MinRegBankSize;
+	PVRSRV_ERROR eError;
+	int err = 0;
+
+	PVR_ASSERT(pvOSDevice);
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (psSysData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->pdev = to_platform_device((struct device *)pvOSDevice);
+	psSysData->pdata = psSysData->pdev->dev.platform_data;
+
+	/*
+	 * The device cannot address system memory, so there is no DMA
+	 * limitation.
+	 */
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL)
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(64));
+	}
+	else
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+	}
+
+	err = tc_enable(psSysData->pdev->dev.parent);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to enable PCI device (%d)", __func__, err));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrFreeSysData;
+	}
+
+	psSysData->registers = platform_get_resource_byname(psSysData->pdev,
+														IORESOURCE_MEM,
+														"rogue-regs");
+	if (!psSysData->registers)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get Rogue register information",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		goto ErrorDevDisable;
+	}
+
+	/* Check the address range is large enough. */
+	uiRegistersSize = resource_size(psSysData->registers);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* each GPU instance gets the minimum 64kb register range */
+	ui32MinRegBankSize = RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE;
+#else
+	/* the GPU gets the entire 64MB IO range */
+	ui32MinRegBankSize = SYS_RGX_REG_REGION_SIZE;
+#endif
+
+	if (uiRegistersSize < ui32MinRegBankSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough (was %pa, required 0x%08x)",
+				 __func__, &uiRegistersSize, ui32MinRegBankSize));
+
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorDevDisable;
+	}
+
+	/* Reserve the address range */
+	if (!request_mem_region(psSysData->registers->start,
+							resource_size(psSysData->registers),
+							SYS_RGX_DEV_NAME))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+
+		goto ErrorDevDisable;
+	}
+
+	eError = DeviceConfigCreate(psSysData, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseMemRegion;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	eError = IonInit(psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise ION", __func__));
+		goto ErrorDeviceConfigDestroy;
+	}
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	/* power functions */
+	psDevConfig->pfnPrePowerState = PrePower;
+	psDevConfig->pfnPostPowerState = PostPower;
+
+	psSysData->psDevConfig = psDevConfig;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+ErrorDeviceConfigDestroy:
+	DeviceConfigDestroy(psDevConfig);
+#endif
+ErrorReleaseMemRegion:
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+ErrorDevDisable:
+	tc_disable(psSysData->pdev->dev.parent);
+ErrFreeSysData:
+	OSFreeMem(psSysData);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevConfig->hSysData;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	IonDeinit();
+#endif
+
+	DeviceConfigDestroy(psDevConfig);
+
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+	tc_disable(psSysData->pdev->dev.parent);
+
+	OSFreeMem(psSysData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+#if defined(TC_APOLLO_TCF5)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	return PVRSRV_OK;
+#else
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	u32 tmp = 0;
+	u32 pll;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	if (tc_sys_info(psSysData->pdev->dev.parent, &tmp, &pll))
+		goto err_out;
+
+	if (tmp > 0)
+		PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", tmp);
+	PVR_DUMPDEBUG_LOG("PLL status: %x", pll);
+
+err_out:
+	return eError;
+#endif
+}
+
+typedef struct
+{
+	struct device *psDev;
+	int iInterruptID;
+	void *pvData;
+	PFN_LISR pfnLISR;
+} LISR_DATA;
+
+static void TCInterruptHandler(void* pvData)
+{
+	LISR_DATA *psLISRData = pvData;
+	psLISRData->pfnLISR(psLISRData->pvData);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	LISR_DATA *psLISRData;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if ((ui32IRQ != TC_INTERRUPT_EXT) &&
+		(ui32IRQ < TC_INTERRUPT_OSID0) && (ui32IRQ > TC_INTERRUPT_OSID7))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d", __func__, ui32IRQ));
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psLISRData = OSAllocZMem(sizeof(*psLISRData));
+	if (!psLISRData)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+	psLISRData->iInterruptID = ui32IRQ;
+	psLISRData->psDev = psSysData->pdev->dev.parent;
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, TCInterruptHandler, psLISRData);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_free_data;
+	}
+
+	err = tc_enable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_enable_interrupt() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_unset_interrupt_handler;
+	}
+
+	*phLISRData = psLISRData;
+	eError = PVRSRV_OK;
+
+	PVR_TRACE(("Installed device LISR " IMG_PFN_FMTSPEC " with tc module to ID %u",
+			   pfnLISR, ui32IRQ));
+
+err_out:
+	return eError;
+err_unset_interrupt_handler:
+	tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+err_free_data:
+	OSFreeMem(psLISRData);
+	goto err_out;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *) hLISRData;
+	int err;
+
+	err = tc_disable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_disable_interrupt() failed (%d)", __func__, err));
+	}
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+	}
+
+	PVR_TRACE(("Uninstalled device LISR " IMG_PFN_FMTSPEC " with tc module from ID %u",
+			   psLISRData->pfnLISR, psLISRData->iInterruptID));
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
+
+/****************************************************************************************************/
+/****                                   VM migration test code                                   ****/
+/****************************************************************************************************/
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode);
+static void PreMigrationDeviceSuspend(struct drm_device *psDev);
+static void PostMigrationDeviceResume(struct drm_device *psDev);
+
+void PVRVMMigration(unsigned int src, unsigned int dest);
+EXPORT_SYMBOL(PVRVMMigration);
+
+#define SWAP_REGSBASE_PTR(a, b) do \
+	{ \
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		b = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+	} while (0)
+
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psHostNode = PVRSRVGetDeviceInstance(0);
+	PVRSRV_RGXDEV_INFO *psSrcInfo = psSrcNode->pvDevice;
+	PVRSRV_RGXDEV_INFO *psDestInfo = psDestNode->pvDevice;
+	PVRSRV_DEVICE_CONFIG *psSrcConfig = psSrcNode->psDevConfig;
+	PVRSRV_DEVICE_CONFIG *psDestConfig = psDestNode->psDevConfig;
+	LISR_DATA *psSrcLISRData = (LISR_DATA *) psSrcInfo->pvLISRData;
+	void *pfnLISR = psSrcLISRData->pfnLISR;
+	IMG_UINT32 ui32SrcHyperLane, ui32DestHyperLane;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psHostNode != NULL) &&
+								  (psHostNode->psDevConfig != NULL)),
+								 "Device 0 (expected Host) not initialised.");
+
+	/* Determine the HyperLane ID used by a Guest Device from the Register Bank Base address used */
+	ui32SrcHyperLane = (psSrcConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psSrcConfig->ui32RegsSize;
+	ui32DestHyperLane = (psDestConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psDestConfig->ui32RegsSize;
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Swapping hyperlanes between Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32SrcHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32DestHyperLane));
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resulting configuration:    Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32DestHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32SrcHyperLane));
+
+	/* swap the register bank details */
+	SWAP_REGSBASE_PTR(psSrcInfo->pvRegsBaseKM, psDestInfo->pvRegsBaseKM);
+	SWAP(psSrcConfig->sRegsCpuPBase.uiAddr, psDestConfig->sRegsCpuPBase.uiAddr);
+	/* DevConfig->ui32RegsSize remains the same */
+
+	/* Swap interrupt lines between devices */
+	eError = SysUninstallDeviceLISR(psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+	eError = SysUninstallDeviceLISR(psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	SWAP(psSrcConfig->ui32IRQ, psDestConfig->ui32IRQ);
+
+	eError = SysInstallDeviceLISR(psSrcConfig->hSysData,
+								  psSrcConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psSrcNode,
+								  &psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+
+	eError = SysInstallDeviceLISR(psDestConfig->hSysData,
+								  psDestConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psDestNode,
+								  &psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	/* Swap contents of LMA carveouts between virtual devices */
+	{
+		/* Guest Raw Fw Heap mapping is done using the Host Devices */
+		PHYS_HEAP *psSrcHeap = NULL;
+		PHYS_HEAP *psDestHeap = NULL;
+		IMG_DEV_PHYADDR sSrcHeapBase, sDestHeapBase;
+
+		psSrcHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32SrcHyperLane];
+		psDestHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DestHyperLane];
+
+		PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcHeap != NULL) &&
+									  (psDestHeap != NULL)),
+									 "Guest firmware heaps not premapped by the Host Device.");
+
+		eError = PhysHeapGetDevPAddr(psSrcHeap, &sSrcHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(src fw heap)");
+		eError = PhysHeapGetDevPAddr(psDestHeap, &sDestHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(dest fw heap)");
+
+		eError = PvzServerUnmapDevPhysHeap(ui32SrcHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(src fw heap)");
+		eError = PvzServerUnmapDevPhysHeap(ui32DestHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(dest fw heap)");
+
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32SrcHyperLane]);
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32DestHyperLane]);
+
+		/* create new heaps with new base addresses */
+		eError = PvzServerMapDevPhysHeap(ui32SrcHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sDestHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(src fw heap)");
+		eError = PvzServerMapDevPhysHeap(ui32DestHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sSrcHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(dest fw heap)");
+	}
+}
+
+static void PreMigrationDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(eError == PVRSRV_OK,
+	                           "LinuxBridgeBlockClientsAccess()");
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * be declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+	}
+}
+
+static void PostMigrationDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+									PVRSRV_SYS_POWER_STATE_ON,
+									PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ);
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+}
+
+void PVRVMMigration(unsigned int src, unsigned int dest)
+{
+	PVRSRV_DEVICE_NODE *psSrcNode = PVRSRVGetDeviceInstance(src);
+	PVRSRV_DEVICE_NODE *psDestNode = PVRSRVGetDeviceInstance(dest);
+	struct device *psSrcDev, *psDestDev;
+	struct drm_device *psSrcDrmDev, *psDestDrmDev;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode != NULL) && (psDestNode != NULL) && (psSrcNode != psDestNode)),
+								 "Invalid Device IDs requested for migration.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE) &&
+								  (psDestNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)),
+								 "Devices not fully initialised.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig != NULL) &&
+								  (psDestNode->psDevConfig != NULL)),
+								 "Device config structure is NULL.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig->pvOSDevice != NULL) &&
+								  (psDestNode->psDevConfig->pvOSDevice != NULL)),
+								 "Linux kernel device pointer is NULL.");
+
+	psSrcDev = psSrcNode->psDevConfig->pvOSDevice;
+	psDestDev = psDestNode->psDevConfig->pvOSDevice;
+	psSrcDrmDev = dev_get_drvdata(psSrcDev);
+	psDestDrmDev = dev_get_drvdata(psDestDev);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcDrmDev != NULL) &&
+								  (psDestDrmDev != NULL)),
+								 "Linux kernel drm_device pointer is NULL.");
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psSrcNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psSrcDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psDestNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psDestDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Migrating vGPU resources (regbank, irq, osid)", __func__));
+	SwapHyperlanes(psSrcNode, psDestNode);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psSrcNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psSrcDrmDev);
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psDestNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psDestDrmDev);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysinfo.h
new file mode 100644
index 000000000000..6355a98482c3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_linux_tc/sysinfo.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM) || defined(FPGA)
+#define MAX_HW_TIME_US                           (240000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (120000)
+#else
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#endif
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#if defined(VIRTUAL_PLATFORM)
+#define EVENT_OBJECT_TIMEOUT_US                  (120000000)
+#elif defined(TC_APOLLO_TCF5)
+#define EVENT_OBJECT_TIMEOUT_US                  (2000000)
+#else
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+#endif
+
+#define SYS_RGX_DEV_NAME "tc_rogue"
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/Kbuild.mk
new file mode 100644
index 000000000000..0873e072163f
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/Kbuild.mk
@@ -0,0 +1,53 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += services/system/rogue/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/rogue/common/vmm_type_$(VMM_TYPE).o
+
+ifeq ($(SUPPORT_ION),1)
+$(PVRSRVKM_NAME)-y += services/system/common/env/linux/ion_support_generic.o
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.c
new file mode 100644
index 000000000000..a3a34ed37e8a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.c
@@ -0,0 +1,344 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+#include "rgx_bvnc_defs_km.h"
+/*
+ * In systems that support trusted device address protection, there are three
+ * physical heaps from which pages should be allocated:
+ * - one heap for normal allocations
+ * - one heap for allocations holding META code memory
+ * - one heap for allocations holding secured DRM data
+ */
+
+#define PHYS_HEAP_IDX_GENERAL     0
+#define PHYS_HEAP_IDX_FW          1
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define PHYS_HEAP_IDX_TDFWMEM     2
+#define PHYS_HEAP_IDX_TDSECUREBUF 3
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+#define PHYS_HEAP_IDX_FW_MEMORY   2
+#endif
+
+/* Change to test CPU_LOCAL sys layers*/
+#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_GPU_LOCAL
+//#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_CPU_LOCAL
+
+#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_GPU_LOCAL
+//#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_CPU_LOCAL
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static PHYS_HEAP_FUNCTIONS gsPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	UMAPhysHeapCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	UMAPhysHeapDevPAddrToCpuPAddr,
+};
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	/*
+	 * This function is called during device initialisation, which on Linux,
+	 * means it won't be called concurrently. As such, there's no need to
+	 * protect it with a lock or use an atomic variable.
+	 */
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiHeapCount = 2;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	uiHeapCount += 2;
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	uiHeapCount += 1;
+#endif
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_GENERAL].ui32UsageFlags = UMA_HEAP_USAGE_FLAG;
+
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].pszPDumpMemspaceName = "SYSMEM_FW";
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW].ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].pszPDumpMemspaceName = "TDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDFWMEM].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].pszPDumpMemspaceName = "TDSECBUFMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_TDSECUREBUF].ui32UsageFlags =
+		PHYS_HEAP_USAGE_GPU_SECURE;
+
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].pszPDumpMemspaceName = "DEDICATEDFWMEM";
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[PHYS_HEAP_IDX_FW_MEMORY].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+#endif
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+#if defined(SUPPORT_AXI_ACE_TEST)
+		if ( ui64Features & RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		}else
+#endif
+		{
+			psDevConfig->eCacheSnoopingMode		= PVRSRV_DEVICE_SNOOP_NONE;
+		}
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszName                = "nohw";
+	psDevConfig->pszVersion             = NULL;
+	psDevConfig->pfnSysDevFeatureDepInit = SysDevFeatureDepInit;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr   = 0x00f00000;
+	psDevConfig->ui32RegsSize           = 0x4000;
+	psDevConfig->ui32IRQ                = 0x00000bad;
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+	psDevConfig->eDefaultHeap = UMA_DEFAULT_HEAP;
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	psDevConfig->bHasFBCDCVersion31      = IMG_FALSE;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+
+	/* Setup other system specific stuff */
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	PVR_UNREFERENCED_PARAMETER(ui32IRQ);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(pfnLISR);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(phLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.h
new file mode 100644
index 000000000000..acbadc22e6a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysconfig.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(SYSCCONFIG_H)
+#define SYSCCONFIG_H
+
+
+#define RGX_NOHW_CORE_CLOCK_SPEED 100000000
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* SYSCCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysinfo.h
new file mode 100644
index 000000000000..566fc4bb7256
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_nohw/sysinfo.h
@@ -0,0 +1,57 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME    "rgxnohw"
+#endif
+
+#endif	/* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/Kbuild.mk
new file mode 100644
index 000000000000..485b59bfa0e6
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/Kbuild.mk
@@ -0,0 +1,55 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += \
+ services/system/rogue/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/rogue/common/vmm_type_$(VMM_TYPE).o
+
+ccflags-y += \
+ -I$(TOP)/services/system/rogue/common/env/linux -I$(TOP)/services/system/rogue/common/env/linux \
+ -I$(TOP)/include/rogue/system/$(PVR_SYSTEM) -I$(TOP)/include/system/$(PVR_SYSTEM)
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.c
new file mode 100644
index 000000000000..e02bd6823920
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.c
@@ -0,0 +1,2721 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "power.h"
+#include "sysinfo.h"
+#include "apollo_regs.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pci_support.h"
+#include "interrupt_support.h"
+#include "tcf_clk_ctrl.h"
+#include "tcf_pll.h"
+#include "sysdata.h"
+#include "dc_server.h"
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+/*
+Notes on device virtualization support on PCI systems:
+	Host/Guest driver expectations in VM domain:
+		- RGX discovery available on vPCI/bus.
+		- RGX PCI/BAR available on vPCI/bus.
+	Guest driver limitations in VM domain:
+		- Does not perform device management.
+		- Does not power RGX up/down.
+ */
+#include "vz_vmm_pvz.h"
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+#define	HOST_PCI_INIT_FLAGS	0
+#else
+#define	HOST_PCI_INIT_FLAGS	HOST_PCI_INIT_FLAG_BUS_MASTER
+#endif
+
+/* The VM information page size which is allocated from LMA memory, this
+   page allows VZ drivers to communicate information without going via
+   the hypervisor. This is used only by VZ drivers to synchronise
+   cross-VM interrupt handling */
+#define VM_INFOPG_SIZE          0x1000
+
+/* This is used only by guest drivers during interrupt handling */
+#define IRQ_ACK_RETRY_CLEAR_MAX 2
+
+/* Interrupt defines */
+typedef enum _APOLLO_IRQ_
+{
+	APOLLO_IRQ_GPU = 0,
+	APOLLO_IRQ_PDP,
+	APOLLO_IRQ_MAX,
+} APOLLO_IRQ;
+
+/* Clock speed module parameters */
+static IMG_UINT32 ui32MemClockSpeed  = RGX_TC_MEM_CLOCK_SPEED;
+static IMG_UINT32 ui32CoreClockSpeed = RGX_TC_CORE_CLOCK_SPEED;
+#if !defined(TC_APOLLO_ES2)
+static IMG_UINT32 ui32SysClockSpeed = RGX_TC_SYS_CLOCK_SPEED;
+#endif
+#if defined(TC_APOLLO_TCF5)
+static IMG_UINT32 ui32MemReadLatency = 0;
+static IMG_UINT32 ui32MemWriteResponseLatency = 0;
+#endif
+
+#if defined(__linux__)
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+module_param_named(sys_mem_clk_speed,  ui32MemClockSpeed,  uint, S_IRUGO | S_IWUSR);
+module_param_named(sys_core_clk_speed, ui32CoreClockSpeed, uint, S_IRUGO | S_IWUSR);
+#if !defined(TC_APOLLO_ES2)
+module_param_named(sys_sysif_clk_speed, ui32SysClockSpeed, uint, S_IRUGO | S_IWUSR);
+#endif
+#if defined(TC_APOLLO_TCF5)
+module_param_named(sys_mem_latency, ui32MemReadLatency, uint, S_IRUGO | S_IWUSR);
+module_param_named(sys_wresp_latency, ui32MemWriteResponseLatency, uint, S_IRUGO | S_IWUSR);
+#endif
+#endif
+
+/* Specifies the maximum number of ApolloHardReset() attempts */
+#define APOLLO_RESET_MAX_RETRIES	10
+
+#define SYSTEM_INFO_FORMAT_STRING "FPGA Revision: %s - TCF Core Revision: %s - TCF Core Target Build ID: %s - PCI Version: %s - Macro Version: %s"
+#define FPGA_REV_MAX_LEN      8 /* current longest format: "x.y.z" */
+#define TCF_CORE_REV_MAX_LEN  8 /* current longest format: "x.y.z" */
+#define TCF_CORE_CFG_MAX_LEN  4 /* current longest format: "x" */
+#define PCI_VERSION_MAX_LEN   4 /* current longest format: "x" */
+#define MACRO_VERSION_MAX_LEN 8 /* current longest format: "x.yz" */
+
+#define HEX2DEC(hexIntVal)    ((((hexIntVal) >> 4) * 10) + ((hexIntVal) & 0x0F))
+
+static PVRSRV_ERROR InitMemory(PVRSRV_DEVICE_CONFIG *psSysConfig, SYS_DATA *psSysData);
+static INLINE void DeInitMemory(PVRSRV_DEVICE_CONFIG *psSysConfig, SYS_DATA *psSysData);
+
+size_t GetMemorySize(IMG_CPU_PHYADDR sMemCpuPhyAddr, IMG_UINT64 pciBarSize);
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	IMG_CHAR apszFPGARev[FPGA_REV_MAX_LEN];
+	IMG_CHAR apszCoreRev[TCF_CORE_REV_MAX_LEN];
+	IMG_CHAR apszConfigRev[TCF_CORE_CFG_MAX_LEN];
+	IMG_CHAR apszPCIVer[PCI_VERSION_MAX_LEN];
+	IMG_CHAR apszMacroVer[MACRO_VERSION_MAX_LEN];
+	IMG_CHAR *pszVersion;
+	IMG_UINT32 ui32StringLength;
+	IMG_UINT32 ui32Value;
+	IMG_CPU_PHYADDR	sHostFPGARegCpuPBase;
+	void *pvHostFPGARegCpuVBase;
+
+	/* To get some of the version information we need to read from a register that we don't normally have
+	   mapped. Map it temporarily (without trying to reserve it) to get the information we need. */
+	sHostFPGARegCpuPBase.uiAddr	= OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM) + 0x40F0;
+	pvHostFPGARegCpuVBase		= OSMapPhysToLin(sHostFPGARegCpuPBase, 0x04, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvHostFPGARegCpuVBase == NULL)
+	{
+		return NULL;
+	}
+
+	/* Create the components of the PCI and macro versions */
+	ui32Value = OSReadHWReg32(pvHostFPGARegCpuVBase, 0);
+	OSSNPrintf(apszPCIVer, sizeof(apszPCIVer), "%d",
+	           HEX2DEC((ui32Value & 0x00FF0000) >> 16));
+	OSSNPrintf(apszMacroVer, sizeof(apszMacroVer), "%d.%d",
+	           ((ui32Value & 0x00000F00) >> 8),
+	           HEX2DEC((ui32Value & 0x000000FF) >> 0));
+
+	/* Unmap the register now that we no longer need it */
+	OSUnMapPhysToLin(pvHostFPGARegCpuVBase, 0x04);
+
+	/*
+	 * Check bits 7:0 of register 0x28 (TCF_CORE_REV_REG or SW_IF_VERSION
+	 * depending on its own value) to find out how the driver should
+	 * generate the strings for FPGA and core revision.
+	 */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_SW_IF_VERSION);
+	ui32Value = (ui32Value & VERSION_MASK) >> VERSION_SHIFT;
+
+	if (ui32Value == 0)
+	{
+		/* Create the components of the TCF core revision number */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TCF_CORE_REV_REG);
+		OSSNPrintf(apszCoreRev, sizeof(apszCoreRev), "%d.%d.%d",
+		           HEX2DEC((ui32Value & TCF_CORE_REV_REG_MAJOR_MASK) >> TCF_CORE_REV_REG_MAJOR_SHIFT),
+		           HEX2DEC((ui32Value & TCF_CORE_REV_REG_MINOR_MASK) >> TCF_CORE_REV_REG_MINOR_SHIFT),
+		           HEX2DEC((ui32Value & TCF_CORE_REV_REG_MAINT_MASK) >> TCF_CORE_REV_REG_MAINT_SHIFT));
+
+		/* Create the components of the FPGA revision number */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_FPGA_REV_REG);
+		OSSNPrintf(apszFPGARev, sizeof(apszFPGARev), "%d.%d.%d",
+		           HEX2DEC((ui32Value & FPGA_REV_REG_MAJOR_MASK) >> FPGA_REV_REG_MAJOR_SHIFT),
+		           HEX2DEC((ui32Value & FPGA_REV_REG_MINOR_MASK) >> FPGA_REV_REG_MINOR_SHIFT),
+		           HEX2DEC((ui32Value & FPGA_REV_REG_MAINT_MASK) >> FPGA_REV_REG_MAINT_SHIFT));
+	}
+	else if (ui32Value == 1)
+	{
+		/* Create the components of the TCF core revision number */
+		OSSNPrintf(apszCoreRev, sizeof(apszCoreRev), "%d", ui32Value);
+
+		/* Create the components of the FPGA revision number */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_REL);
+		OSSNPrintf(apszFPGARev, sizeof(apszFPGARev), "%d.%d",
+		           HEX2DEC((ui32Value & MAJOR_MASK) >> MAJOR_SHIFT),
+		           HEX2DEC((ui32Value & MINOR_MASK) >> MINOR_SHIFT));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: unrecognised SW_IF_VERSION 0x%08x", __func__, ui32Value));
+
+		/* Create the components of the TCF core revision number */
+		OSSNPrintf(apszCoreRev, sizeof(apszCoreRev), "%d", ui32Value);
+
+		/* Create the components of the FPGA revision number */
+		OSSNPrintf(apszFPGARev, sizeof(apszFPGARev), "N/A");
+	}
+
+	/* Create the component of the TCF core target build ID */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TCF_CORE_TARGET_BUILD_CFG);
+	OSSNPrintf(apszConfigRev, sizeof(apszConfigRev), "%d",
+	           (ui32Value & TCF_CORE_TARGET_BUILD_ID_MASK) >> TCF_CORE_TARGET_BUILD_ID_SHIFT);
+
+	/* Calculate how much space we need to allocate for the string */
+	ui32StringLength = OSStringLength(SYSTEM_INFO_FORMAT_STRING);
+	ui32StringLength += OSStringLength(apszFPGARev);
+	ui32StringLength += OSStringLength(apszCoreRev);
+	ui32StringLength += OSStringLength(apszConfigRev);
+	ui32StringLength += OSStringLength(apszPCIVer);
+	ui32StringLength += OSStringLength(apszMacroVer);
+
+	/* Create the version string */
+	pszVersion = OSAllocMem(ui32StringLength * sizeof(IMG_CHAR));
+	if (pszVersion)
+	{
+		OSSNPrintf(&pszVersion[0], ui32StringLength,
+		           SYSTEM_INFO_FORMAT_STRING,
+		           apszFPGARev,
+		           apszCoreRev,
+		           apszConfigRev,
+		           apszPCIVer,
+		           apszMacroVer);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to create format string", __func__));
+	}
+
+	return pszVersion;
+}
+
+static IMG_BOOL SystemISRHandler(void *pvData)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 ui32InterruptStatus;
+	IMG_UINT32 ui32InterruptClear = 0;
+	IMG_UINT32 ui32InterruptClear2 = 0;
+	SYS_DATA *psSysData = (SYS_DATA *)pvData;
+
+	/* Read device interrupt status register on entry, for VZ multiple drivers might sample same value */
+	ui32InterruptStatus = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_STATUS);
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		/*
+		  Multi-VM PCI device sharing/virtualization
+
+		  PCI device sharing is problematic, almost all hypervisors do not support it; for
+		  those that do (i.e. Fiasco/L4), it's not possible to correctly handle clearing
+		  device interrupt register concurrently from multiple VM/drivers without the aid
+		  of an external interrupt routing logic that forwards vIRQ to VM, also this logic
+		  would have to be implemented per-hypervisor environment. An alternative approach
+		  used here is to always perform the clearing of the device interrupt register in
+		  the host driver and only do so in the guest as a last resort mostly so that the
+		  guest can still function on a VM shared interrupt line with same or other active
+		  VM device drivers. NOTE: This assumes underlying hypervisor or vm manager walks
+		  and delivers vIRQ into each VM vPCI/bus for every RGX interrupt it receives thus
+		  ensuring interrupt events are not lost, it's OK if pending interrupts are merged.
+		*/
+		if ((ui32InterruptStatus & psSysData->sInterruptData[0].ui32InterruptFlag) ||
+			(ui32InterruptStatus & psSysData->sInterruptData[1].ui32InterruptFlag))
+		{
+			/* Set RGX/PDP interrupt status, we cannot assume what's interrupting */
+			ui32InterruptClear = (0x1 << EXT_INT_SHIFT) | (0x1 << PDP1_INT_SHIFT);
+			ui32InterruptClear2 = ui32InterruptClear;
+
+			if (PVRSRV_VZ_MODE_IS(HOST))
+			{
+				/*
+				   Assigning a fixed driver (i.e. the host) the responsibility of clearing
+				   the RGX interrupt register does not always work because the hypervisor
+				   might schedule another VM and run it long enough for that VM OS/kernel
+				   to detect that RGX interrupt line has not been cleared after multiple
+				   successive back-to-back calls to the guest interrupt handler. To help
+				   mitigate against this, the host driver updates this counter for valid
+				   RGX interrupt events only and the guest driver samples it to know how
+				   to respond/acknowledge interrupt events to its OS/kernel.
+				 */
+				(void) OSAtomicIncrement((void*)psSysData->pui32IRQCount);
+			}
+		}
+
+		/*
+		  The idea here is to always assume the driver is being called due to RGX IRQ event
+		  because multiple devices may be sharing same interrupt line both at BIOS hypervisor
+		  level and/or VM level and due to possibility of race between host/guest clearing
+		  the interrupt register. Doing this means drivers never miss any IRQ event & all
+		  LISR routine is guaranteed to be called for every HW irq event, the down side is
+		  that the LISR routine is called in all driver VM for every IRQ event on line.
+		 */
+		ui32InterruptStatus = 0xFFFFFFFF;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		/* Should this array grow, ensure to unroll the loop for the VZ condition above */
+		if ((ui32InterruptStatus & psSysData->sInterruptData[i].ui32InterruptFlag) != 0)
+		{
+			psSysData->sInterruptData[i].pfnLISR(psSysData->sInterruptData[i].pvData);
+			ui32InterruptClear |= psSysData->sInterruptData[i].ui32InterruptFlag;
+		}
+	}
+
+	if (PVRSRV_VZ_MODE_IS(HOST))
+	{
+		/* Only clear interrupt register if device has actually interrupted */
+		ui32InterruptClear = ui32InterruptClear2 ? ui32InterruptClear : 0;
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST))
+	{
+		/* If neither of two conditions below are met, then RGX/PDP is not interrupting */
+		IMG_UINT32 ui32CurrentIrqCount = OSAtomicRead((void *)psSysData->pui32IRQCount);
+		static IMG_UINT32 ui32IrqAckRetryClearCount = 0;
+		static IMG_UINT32 ui32LastIrqCount = 0;
+		ui32InterruptClear = 0;
+
+		if (ui32CurrentIrqCount != ui32LastIrqCount)
+		{
+			/*
+			   One complication is knowing if the RGX device is the only device on its PCI
+			   interrupt line either within VM or down at the BIOS/hypervisor level and
+			   if RGX device is also sharing line with another VM PCI device driver. As
+			   it is impossible to determine this, guest driver requires a way of filtering
+			   RGX IRQ events such that it can positively respond back to the OS/kernel
+			   that it has handled interrupts due to it else the OS/kernel might disable
+			   the RGX interrupt line which on some hypervisor environments will impact
+			   interrupt delivery to all the VMs sharing the PCI/RGX device.
+			*/
+			ui32LastIrqCount = ui32CurrentIrqCount;
+			return IMG_TRUE;
+		}
+		else if (ui32InterruptClear2)
+		{
+			/*
+			   The guest driver has been called with the RGX/PDP interrupt flag bits still
+			   pending so we need to determine if this is a back-to-back call. If not we
+			   give priority to the host driver to clear it else guest drivers does it.
+			   This can be due to guest VM/driver running ahead of host VM/driver for this
+			   interrupt event and/or the hypervisor scheduler has parked momentarily the
+			   host VM/driver whilst running the guest VM/driver across many cycles right
+			   up to this point.
+			*/
+			if (ui32IrqAckRetryClearCount < IRQ_ACK_RETRY_CLEAR_MAX)
+			{
+				/* NOTE: This assumes OS/kernel calls-back into this ISR a couple of times
+				   before giving-up and disabling the interrupt line if said interrupt
+				   line is still asserted after each ISR invocation */
+				ui32IrqAckRetryClearCount = ui32IrqAckRetryClearCount + 1;
+				return IMG_TRUE;
+			}
+
+			/*
+			   Now the race between the host/guest(s) driver here can result in two outcomes.
+			   First of which is that the loser(s) will clear the interrupt register after it
+			   has been cleared by the winner and before the device has raised another IRQ
+			   event - the effects of this should be benign. Second is that this write will
+			   happen after the device has raised another IRQ event - the effects of this is
+			   should be OK so long as the hypervisor delivers the corresponding vIRQ.
+			 */
+			ui32InterruptClear = ui32InterruptClear2;
+			ui32IrqAckRetryClearCount = 0;
+		}
+	}
+
+	if (ui32InterruptClear)
+	{
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_CLEAR, ui32InterruptClear);
+
+		/*
+		   CPU-PCI-WRITE-BUFFER
+
+		   On PC platforms, this BIOS performance enhancing feature controls the chipset's
+		   CPU-to-PCI write buffer used to store PCI writes from the CPU before being written
+		   onto the PCI bus. By reading from the register, this CPU-to-PCI write buffer is
+		   flushed. Without this read, at times the kernel reports unhandled IRQs due to
+		   above interrupt clear not being reflected on the IRQ line after exit from the
+		   interrupt handler.
+		 */
+		(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_CLEAR);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+#if defined(TC_APOLLO_ES2) || defined(TC_APOLLO_BONNIE)
+
+static PVRSRV_ERROR SPI_Read(void *pvLinRegBaseAddr,
+							 IMG_UINT32 ui32Offset,
+							 IMG_UINT32 *pui32Value)
+{
+	IMG_UINT32 ui32Count = 0;
+	*pui32Value = 0;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	OSWriteHWReg32(pvLinRegBaseAddr,
+				   TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR,
+				   TCF_SPI_MST_RDNWR_MASK | ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(100);
+
+	while (((OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_STATUS)) != 0x08) &&
+			(ui32Count < 10000))
+	{
+		ui32Count++;
+	}
+
+	if (ui32Count < 10000)
+	{
+		*pui32Value = OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_RDATA);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SPI_Read: Time out reading SPI register (0x%x)", ui32Offset));
+		return PVRSRV_ERROR_TIMEOUT_POLLING_FOR_VALUE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void SPI_Write(void *pvLinRegBaseAddr,
+					  IMG_UINT32 ui32Offset,
+					  IMG_UINT32 ui32Value)
+{
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR, ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_WDATA, ui32Value);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(1000);
+}
+
+static IMG_BOOL IsInterfaceAligned(IMG_UINT32 ui32Eyes,
+								   IMG_UINT32 ui32ClkTaps,
+								   IMG_UINT32 ui32TrainAck)
+{
+	IMG_UINT32	ui32MaxEyeStart = ui32Eyes >> 16;
+	IMG_UINT32	ui32MinEyeEnd = ui32Eyes & 0xffff;
+
+	IMG_BOOL	bTrainingComplete = (ui32ClkTaps & 0x10000);
+
+	IMG_BOOL	bTrainAckComplete = (ui32TrainAck & 0x100);
+	IMG_UINT32	ui32FailedAcks = (ui32TrainAck & 0xf0) >> 4;
+
+
+	//If either the training or training ack failed, we haven't aligned
+	if (!bTrainingComplete || !bTrainAckComplete)
+	{
+		return IMG_FALSE;
+	}
+
+	//If the max eye >= min eye it means the readings are nonsense
+	if (ui32MaxEyeStart >= ui32MinEyeEnd)
+	{
+		return IMG_FALSE;
+	}
+
+	//If we failed the ack pattern more than 4 times
+	if (ui32FailedAcks > 4)
+	{
+		return IMG_FALSE;
+	}
+
+	//If there is less than 7 taps
+	//(240ps @40ps/tap, this number should be lower for the fpga, since its taps are bigger
+	//We should really calculate the "7" based on the interface clock speed.
+	if ((ui32MinEyeEnd - ui32MaxEyeStart) < 7)
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static IMG_UINT32 SAI_Read(void *base, IMG_UINT32 addr)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, 0);
+	OSWriteHWReg32(base, 0x300, 0x200 | addr);
+	OSWriteHWReg32(base, 0x318, 1);
+	return OSReadHWReg32(base, 0x310);
+}
+
+static PVRSRV_ERROR ApolloHardReset(SYS_DATA *psSysData)
+{
+	IMG_BOOL bAlignmentOK = IMG_FALSE;
+	IMG_INT resetAttempts = 0;
+	IMG_UINT32 ui32ClkResetN;
+#if !defined(TC_APOLLO_BONNIE)
+	IMG_UINT32 ui32DUTCtrl;
+#endif
+	//This is required for SPI reset which is not yet implemented.
+	//IMG_UINT32 ui32AuxResetsN;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+#if !defined(TC_APOLLO_BONNIE)
+	//power down
+	ui32DUTCtrl = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1);
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_0V9EN;
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_1V8EN;
+	ui32DUTCtrl |= DUT_CTRL_VCC_IO_INH;
+	ui32DUTCtrl |= DUT_CTRL_VCC_CORE_INH;
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1, ui32DUTCtrl);
+#endif
+
+	OSSleepms(500);
+
+	//set clock speed here, before reset.
+#if defined(TC_APOLLO_BONNIE)
+	//The ES2 FPGA builds do not like their core clocks being set (it takes apollo down).
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_CORE_CLK0, ui32CoreClockSpeed / 1000000);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_CORE_DRP_GO, 1);
+#endif
+
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_MEMIF_CLK0, ui32MemClockSpeed / 1000000);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x1000 | TCF_PLL_PLL_MEM_DRP_GO, 1);
+
+	PVR_LOG(("Setting clocks to %uMHz/%uMHz", ui32CoreClockSpeed / 1000000, ui32MemClockSpeed / 1000000));
+
+	OSWaitus(400);
+
+	//Put DCM, DUT, DDR, PDP1, and PDP2 into reset
+	ui32ClkResetN = (0x1 << GLB_CLKG_EN_SHIFT);
+	ui32ClkResetN |= (0x1 << SCB_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+
+	OSSleepms(100);
+
+#if !defined(TC_APOLLO_BONNIE)
+	//Enable the voltage control regulators on DUT
+	ui32DUTCtrl = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1);
+	ui32DUTCtrl |= DUT_CTRL_VCC_0V9EN;
+	ui32DUTCtrl |= DUT_CTRL_VCC_1V8EN;
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_IO_INH;
+	ui32DUTCtrl &= ~DUT_CTRL_VCC_CORE_INH;
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1, ui32DUTCtrl);
+
+	OSSleepms(300);
+#endif
+
+	//Take DCM, DDR, PDP1, and PDP2 out of reset
+	ui32ClkResetN |= (0x1 << DDR_RESETN_SHIFT);
+	ui32ClkResetN |= (0x1 << DUT_DCM_RESETN_SHIFT);
+	ui32ClkResetN |= (0x1 << PDP1_RESETN_SHIFT);
+	ui32ClkResetN |= (0x1 << PDP2_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+
+#if !defined(TC_APOLLO_BONNIE)
+	//Set ODT to a specific value that seems to provide the most stable signals.
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0x11, 0x413130);
+#endif
+
+	//Take DUT out of reset
+	ui32ClkResetN |= (0x1 << DUT_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+	OSSleepms(100);
+
+	//try to enable the core clock PLL
+	{
+		IMG_UINT32 ui32DUTResets;
+		IMG_UINT32 ui32DUTGPIO1;
+		IMG_UINT32 ui32PLLStatus = 0;
+
+		/* Un-bypass the PLL on the DUT */
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0x1, 0x0);
+		ui32DUTResets = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, 0x320);
+		ui32DUTResets |= 0x1;
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x320, ui32DUTResets);
+		ui32DUTResets &= 0xfffffffe;
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x320, ui32DUTResets);
+		OSSleepms(1000);
+
+		if (SPI_Read(psSysData->pvSystemRegCpuVBase, 0x2, &ui32PLLStatus) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Unable to read PLL status"));
+		}
+
+		if (ui32PLLStatus != 0x1)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PLL has failed to lock, status = %x", ui32PLLStatus));
+		}
+		else
+		{
+			/* Select DUT PLL as core clock */
+			ui32DUTGPIO1 = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, 0x108);
+			ui32DUTGPIO1 &= 0xfffffff7;
+			OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, 0x108, ui32DUTGPIO1);
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "PLL has been set to 6x"));
+	}
+
+	while (!bAlignmentOK && resetAttempts < 10)
+	{
+		IMG_UINT32 ui32Eyes     = 0;
+		IMG_UINT32 ui32ClkTaps  = 0;
+		IMG_UINT32 ui32TrainAck = 0;
+
+		IMG_INT bank;
+
+
+		resetAttempts += 1;
+
+		//reset the DUT to allow the SAI to retrain
+		ui32ClkResetN &= ~(0x1 << DUT_RESETN_SHIFT);
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+		OSWaitus(100);
+		ui32ClkResetN |= (0x1 << DUT_RESETN_SHIFT);
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32ClkResetN);
+		OSWaitus(100);
+
+		//Assume alignment passed, if any bank fails on either DUT or FPGA
+		//we will set this to false and try again for a max of 10 times.
+		bAlignmentOK = IMG_TRUE;
+
+		//for each of the banks
+		for (bank = 0; bank < 10; bank++)
+		{
+
+			//check alignment on the DUT
+			IMG_UINT32 ui32BankBase = 0x7000 + (0x1000 * bank);
+
+			SPI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x4, &ui32Eyes);
+			SPI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x3, &ui32ClkTaps);
+			SPI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x6, &ui32TrainAck);
+
+			if (!IsInterfaceAligned(ui32Eyes, ui32ClkTaps, ui32TrainAck))
+			{
+				bAlignmentOK = IMG_FALSE;
+				break;
+			}
+
+			//check alignment on the FPGA
+			ui32BankBase = 0xb0 + (0x10 * bank);
+
+			ui32Eyes = SAI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x4);
+			ui32ClkTaps = SAI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x3);
+			ui32TrainAck = SAI_Read(psSysData->pvSystemRegCpuVBase, ui32BankBase + 0x6);
+
+			if (!IsInterfaceAligned(ui32Eyes, ui32ClkTaps, ui32TrainAck))
+			{
+				bAlignmentOK = IMG_FALSE;
+				break;
+			}
+		}
+	}
+
+	if (!bAlignmentOK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Unable to initialise the testchip (interface alignment failure), please restart the system."));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	if (resetAttempts > 1)
+	{
+		PVR_LOG(("Note: The testchip required more than one reset to find a good interface alignment!"));
+		PVR_LOG(("      This should be harmless, but if you do suspect foul play, please reset the machine."));
+		PVR_LOG(("      If you continue to see this message you may want to report it to PowerVR Verification Platforms."));
+	}
+
+#if !defined(TC_APOLLO_BONNIE)
+	/* Enable the temperature sensor */
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xc, 0); //power up
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xc, 2); //reset
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xc, 6); //init & run
+#endif
+
+	{
+		IMG_UINT32 ui32HoodCtrl;
+		SPI_Read(psSysData->pvSystemRegCpuVBase, 0xF, &ui32HoodCtrl);
+		ui32HoodCtrl |= 0x1;
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0xF, ui32HoodCtrl);
+	}
+
+	{
+		IMG_UINT32 ui32RevID = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, 0x10);
+
+		IMG_UINT32 ui32BuildInc = (ui32RevID >> 12) & 0xff;
+
+		if (ui32BuildInc)
+		{
+			PVR_LOG(("BE WARNED: You are not running a tagged release of the FPGA image!"));
+			PVR_LOG(("Owner: 0x%01x, Inc: 0x%02x", (ui32RevID >> 20) & 0xf, ui32BuildInc));
+		}
+
+		PVR_LOG(("FPGA Release: %u.%02u", ui32RevID >> 8 & 0xf, ui32RevID & 0xff));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HardResetDevice(SYS_DATA *psSysData, PVRSRV_DEVICE_CONFIG *psDevice)
+{
+	RGX_TIMING_INFORMATION *psTimingInfo;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiValue;
+
+	for (uiValue = 0; uiValue < APOLLO_RESET_MAX_RETRIES; uiValue++)
+	{
+		eError = ApolloHardReset(psSysData);
+		if (eError == PVRSRV_OK)
+		{
+			break;
+		}
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "ApolloHardReset");
+
+	psTimingInfo = ((RGX_DATA *) psDevice->hDevData)->psRGXTimingInfo;
+
+	/* Make the core clock speed value available to the driver (matching the
+	 * one set by ApolloHardReset()) */
+#if defined(TC_APOLLO_BONNIE)
+	/* For BonnieTC there seems to be an additional 5x multiplier that occurs to
+	 * the clock as measured speed is 540Mhz not 108Mhz. */
+	psTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed * 6 * 5;
+#else /* defined(TC_APOLLO_BONNIE) */
+	psTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed * 6;
+#endif /* defined(TC_APOLLO_BONNIE) */
+
+#if defined(TC_APOLLO_BONNIE)
+	/* enable ASTC via SPI */
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, 0xf, &uiValue);
+	PVR_LOG_IF_ERROR(eError, "SPI_Read");
+
+	uiValue |= 0x1 << 4;
+
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0xf, uiValue);
+#endif /* defined(TC_APOLLO_BONNIE) */
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR PCIInitDev(PVRSRV_DEVICE_CONFIG *psDevice, void *pvOSDevice)
+{
+	SYS_DATA *psSysData;
+	IMG_CPU_PHYADDR	sApolloRegCpuPBase;
+	IMG_UINT32 uiApolloRegSize;
+	PVRSRV_ERROR eError;
+#if defined(TC_APOLLO_BONNIE)
+	IMG_UINT32 ui32Value;
+#endif
+
+	PVR_ASSERT(pvOSDevice);
+
+	if (psDevice->pvOSDevice)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+#endif
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (!psSysData)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->hRGXPCI = OSPCISetDev(TO_PCI_COOKIE(pvOSDevice), HOST_PCI_INIT_FLAGS);
+	if (!psSysData->hRGXPCI)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire PCI device", __func__));
+		eError = PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+		goto ErrorFreeSysData;
+	}
+
+	/* Get Apollo register information */
+	sApolloRegCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM);
+	uiApolloRegSize	 = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM));
+
+	/* Check the address range is large enough. */
+	if (uiApolloRegSize < SYS_APOLLO_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Apollo register region isn't big enough (was 0x%08X, required 0x%08X)",
+			 __func__, uiApolloRegSize, SYS_APOLLO_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorPCIReleaseDevice;
+	}
+
+	/* The Apollo register range contains several register regions. Request only the system control register region */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the system register region",
+				 __func__));
+		goto ErrorPCIReleaseDevice;
+	}
+	psSysData->sSystemRegCpuPBase.uiAddr = sApolloRegCpuPBase.uiAddr + SYS_APOLLO_REG_SYS_OFFSET;
+	psSysData->uiSystemRegSize = SYS_APOLLO_REG_REGION_SIZE;
+
+	/* Setup Rogue register information */
+	psDevice->sRegsCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->ui32RegsSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM));
+
+	/* Save data for this device */
+	psDevice->hSysData = (IMG_HANDLE)psSysData;
+
+	/* Check the address range is large enough. */
+	if (psDevice->ui32RegsSize < SYS_RGX_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough "
+				 "(was 0x%08x, required 0x%08x)",
+				 __func__, psDevice->ui32RegsSize, SYS_RGX_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/* Reserve the address range */
+	eError = OSPCIRequestAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/*
+	   Map in system registers so we can:
+	   - Configure the memory mode (LMA, UMA or LMA/UMA hybrid)
+	   - Hard reset Apollo
+	   - Clear interrupts
+	*/
+	psSysData->pvSystemRegCpuVBase = OSMapPhysToLin(psSysData->sSystemRegCpuPBase,
+													psSysData->uiSystemRegSize,
+													PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (psSysData->pvSystemRegCpuVBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRGXRegReleaseAddrRange;
+	}
+
+	eError = HardResetDevice(psSysData, psDevice);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnMapAddrRange);
+
+	eError = InitMemory(psDevice, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorUnMapAddrRange;
+	}
+
+	psSysData->pszVersion = GetDeviceVersionString(psSysData);
+	if (psSysData->pszVersion)
+	{
+		psDevice->pszVersion = psSysData->pszVersion;
+	}
+
+	eError = OSPCIIRQ(psSysData->hRGXPCI, &psSysData->ui32IRQ);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't get IRQ", __func__));
+		goto ErrorDeInitMemory;
+	}
+
+	/* Register our handler */
+	eError = OSInstallSystemLISR(&psSysData->hLISR,
+								psSysData->ui32IRQ,
+								psDevice->pszName,
+								SystemISRHandler,
+								psSysData,
+								SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to install the system device interrupt handler",
+				 __func__));
+
+		goto ErrorDeInitMemory;
+	}
+
+	psDevice->pvOSDevice = pvOSDevice;
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	psSysData->psDevConfig = psDevice;
+#endif
+
+	return PVRSRV_OK;
+
+ErrorDeInitMemory:
+	DeInitMemory(psDevice, psSysData);
+
+ErrorUnMapAddrRange:
+	OSUnMapPhysToLin(psSysData->pvSystemRegCpuVBase, psSysData->uiSystemRegSize);
+	psSysData->pvSystemRegCpuVBase = NULL;
+
+ErrorRGXRegReleaseAddrRange:
+	OSPCIReleaseAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->sRegsCpuPBase.uiAddr	= 0;
+	psDevice->ui32RegsSize		= 0;
+
+ErrorSystemRegReleaseAddrRegion:
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	psSysData->sSystemRegCpuPBase.uiAddr	= 0;
+	psSysData->uiSystemRegSize		= 0;
+
+ErrorPCIReleaseDevice:
+	OSPCIReleaseDev(psSysData->hRGXPCI);
+	psSysData->hRGXPCI = NULL;
+
+ErrorFreeSysData:
+	OSFreeMem(psSysData);
+	psDevice->hSysData = NULL;
+
+	return eError;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevice,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVRSRV_ERROR	eError;
+	SYS_DATA		*psSysData = psDevice->hSysData;
+	IMG_UINT32		ui32RegVal;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+#if !defined(TC_APOLLO_BONNIE)
+	/* Read the temperature */
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, TCF_TEMP_SENSOR_SPI_OFFSET, &ui32RegVal);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SysDebugInfo: SPI_Read failed for register 0x%x (0x%x)", TCF_TEMP_SENSOR_SPI_OFFSET, eError));
+		goto SysDebugInfo_exit;
+	}
+
+	PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", TCF_TEMP_SENSOR_TO_C(ui32RegVal));
+#endif
+
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, 0x2, &ui32RegVal);
+	PVR_DUMPDEBUG_LOG("PLL status: %x", ui32RegVal);
+
+#if !defined(TC_APOLLO_BONNIE)
+SysDebugInfo_exit:
+#endif
+	return eError;
+}
+
+#else /* if defined(TC_APOLLO_ES2) else TC_APOLLO_ES1 or TC_APOLLO_TCF5 */
+
+#define pol(base,reg,val,msk) \
+	do { \
+		int polnum; \
+		for (polnum = 0; polnum < 500; polnum++) \
+		{ \
+			if ((OSReadHWReg32(base, reg) & msk) == val) \
+			{ \
+				break; \
+			} \
+			OSSleepms(1); \
+		} \
+		if (polnum == 500) \
+		{ \
+			PVR_DPF((PVR_DBG_WARNING, "Poll failed for register: 0x%08X", (unsigned int)reg)); \
+		} \
+	} while (0)
+
+#define polrgx(reg,val,msk) pol(pvRegsBaseKM, reg, val, msk)
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+/* Fake DVFS configuration used purely for testing purposes */
+
+static const IMG_OPP asOPPTable[] =
+{
+	{ 8,  25000000},
+	{ 16, 50000000},
+	{ 32, 75000000},
+	{ 64, 100000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+static void SetFrequency(IMG_UINT32 ui32Frequency)
+{
+	PVR_DPF((PVR_DBG_ERROR, "SetFrequency %u", ui32Frequency));
+}
+
+static void SetVoltage(IMG_UINT32 ui32Voltage)
+{
+	PVR_DPF((PVR_DBG_ERROR, "SetVoltage %u", ui32Voltage));
+}
+#endif
+
+static void SPI_Write(void *pvLinRegBaseAddr,
+					  IMG_UINT32 ui32Offset,
+					  IMG_UINT32 ui32Value)
+{
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR, ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_WDATA, ui32Value);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(1000);
+
+}
+
+static PVRSRV_ERROR SPI_Read(void *pvLinRegBaseAddr,
+							 IMG_UINT32 ui32Offset,
+							 IMG_UINT32 *pui32Value)
+{
+	IMG_UINT32 ui32Count = 0;
+	*pui32Value = ui32Count;
+	PVRSRV_VZ_RET_IF_MODE(NATIVE, PVRSRV_OK);
+
+	OSWriteHWReg32(pvLinRegBaseAddr,
+				   TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR,
+				   TCF_SPI_MST_RDNWR_MASK | ui32Offset);
+	OSWriteHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_GO, TCF_SPI_MST_GO_MASK);
+	OSWaitus(1000);
+
+	while (((OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_STATUS)) != 0x08) &&
+			(ui32Count < 10000))
+	{
+		ui32Count++;
+	}
+
+	if (ui32Count < 10000)
+	{
+		*pui32Value = OSReadHWReg32(pvLinRegBaseAddr, TCF_CLK_CTRL_TCF_SPI_MST_RDATA);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SPI_Read: Time out reading SPI register (0x%x)", ui32Offset));
+		return PVRSRV_ERROR_TIMEOUT_POLLING_FOR_VALUE;
+	}
+
+	return PVRSRV_OK;
+}
+
+#if !defined(TC_APOLLO_TCF5)
+#if !defined(VIRTUAL_PLATFORM)
+/* This is a quick RGX bootup (built-in-self) test */
+static void RGXInitBistery(IMG_CPU_PHYADDR sRegisters, IMG_UINT32 ui32Size)
+{
+	void *pvRegsBaseKM;
+	IMG_UINT instance;
+	IMG_UINT i;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	pvRegsBaseKM = OSMapPhysToLin(sRegisters, ui32Size, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	/* Force clocks on */
+	OSWriteHWReg32(pvRegsBaseKM, 0, 0x55555555);
+	OSWriteHWReg32(pvRegsBaseKM, 4, 0x55555555);
+
+	polrgx(0xa18, 0x05000000, 0x05000000);
+	OSWriteHWReg32(pvRegsBaseKM, 0xa10, 0x048000b0);
+	OSWriteHWReg32(pvRegsBaseKM, 0xa08, 0x55111111);
+	polrgx(0xa18, 0x05000000, 0x05000000);
+
+	/* Clear PDS CSRM and USRM to prevent ERRORs at end of test */
+	OSWriteHWReg32(pvRegsBaseKM, 0x630, 0x1);
+	OSWriteHWReg32(pvRegsBaseKM, 0x648, 0x1);
+	OSWriteHWReg32(pvRegsBaseKM, 0x608, 0x1);
+
+	/* Run BIST for SLC (43) */
+	/* Reset BIST */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0x8);
+	OSWaitus(100);
+
+	/* Clear BIST controller */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0x10);
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		/* Start BIST */
+		OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0x4);
+
+		OSWaitus(100);
+
+		/* Wait for pause */
+		polrgx(0x7000, ui32Pol, ui32Pol);
+	}
+	OSWaitus(100);
+
+	/* Check results for 43 RAMs */
+	polrgx(0x7010, 0xffffffff, 0xffffffff);
+	polrgx(0x7014, 0x7, 0x7);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 8);
+	OSWriteHWReg32(pvRegsBaseKM, 0x7008, 0);
+	OSWriteHWReg32(pvRegsBaseKM, 0x7000, 6);
+	polrgx(0x7000, 0x00010000, 0x00010000);
+	OSWaitus(100);
+
+	polrgx(0x75B0, 0, ~0U);
+	polrgx(0x75B4, 0, ~0U);
+	polrgx(0x75B8, 0, ~0U);
+	polrgx(0x75BC, 0, ~0U);
+	polrgx(0x75C0, 0, ~0U);
+	polrgx(0x75C4, 0, ~0U);
+	polrgx(0x75C8, 0, ~0U);
+	polrgx(0x75CC, 0, ~0U);
+
+	/* Sidekick */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7040, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7040, 0x10);
+	//OSWriteHWReg32(pvRegsBaseKM, 0x7000, 0);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7040, 4);
+		OSWaitus(100);
+		polrgx(0x7040, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7050, 0xffffffff, 0xffffffff);
+	polrgx(0x7054, 0xffffffff, 0xffffffff);
+	polrgx(0x7058, 0x1, 0x1);
+
+	/* USC */
+	for (instance = 0; instance < 4; instance++)
+	{
+		OSWriteHWReg32(pvRegsBaseKM, 0x8010, instance);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7088, 8);
+		OSWaitus(100);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7088, 0x10);
+		OSWaitus(100);
+
+		for (i = 0; i < 3; i++)
+		{
+			IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+			OSWriteHWReg32(pvRegsBaseKM, 0x7088, 4);
+			OSWaitus(100);
+			polrgx(0x7088, ui32Pol, ui32Pol);
+		}
+
+		OSWaitus(100);
+		polrgx(0x7098, 0xffffffff, 0xffffffff);
+		polrgx(0x709c, 0xffffffff, 0xffffffff);
+		polrgx(0x70a0, 0x3f, 0x3f);
+	}
+
+	/* tpumcul0 DustA and DustB */
+	for (instance = 0; instance < 2; instance++)
+	{
+		OSWriteHWReg32(pvRegsBaseKM, 0x8018, instance);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7380, 8);
+		OSWaitus(100);
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7380, 0x10);
+		OSWaitus(100);
+
+		for (i = 0; i < 3; i++)
+		{
+			IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+			OSWriteHWReg32(pvRegsBaseKM, 0x7380, 4);
+			OSWaitus(100);
+			polrgx(0x7380, ui32Pol, ui32Pol);
+		}
+
+		OSWaitus(100);
+		polrgx(0x7390, 0x1fff, 0x1fff);
+	}
+
+	/* TA */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7500, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7500, 0x10);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7500, 4);
+		OSWaitus(100);
+		polrgx(0x7500, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7510, 0x1fffffff, 0x1fffffff);
+
+	/* Rasterisation */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7540, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7540, 0x10);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7540, 4);
+		OSWaitus(100);
+		polrgx(0x7540, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7550, 0xffffffff, 0xffffffff);
+	polrgx(0x7554, 0xffffffff, 0xffffffff);
+	polrgx(0x7558, 0xf, 0xf);
+
+	/* hub_bifpmache */
+	OSWriteHWReg32(pvRegsBaseKM, 0x7588, 8);
+	OSWaitus(100);
+
+	OSWriteHWReg32(pvRegsBaseKM, 0x7588, 0x10);
+	OSWaitus(100);
+
+	for (i = 0; i < 3; i++)
+	{
+		IMG_UINT32 ui32Pol = i == 2 ? 0x10000 : 0x20000;
+
+		OSWriteHWReg32(pvRegsBaseKM, 0x7588, 4);
+		OSWaitus(100);
+		polrgx(0x7588, ui32Pol, ui32Pol);
+	}
+
+	OSWaitus(100);
+	polrgx(0x7598, 0xffffffff, 0xffffffff);
+	polrgx(0x759c, 0xffffffff, 0xffffffff);
+	polrgx(0x75a0, 0x1111111f, 0x1111111f);
+
+	OSUnMapPhysToLin(pvRegsBaseKM, ui32Size);
+}
+#endif /* !defined(VIRTUAL_PLATFORM) */
+#endif /* !defined(TC_APOLLO_TCF5) */
+
+#if defined(TC_APOLLO_TCF5)
+static void ApolloFPGAUpdateDUTClockFreq(SYS_DATA   *psSysData,
+                                         IMG_UINT32 *pui32CoreClockSpeed,
+                                         IMG_UINT32 *pui32MemClockSpeed)
+{
+#if defined(SUPPORT_FPGA_DUT_CLK_INFO)
+	IMG_UINT32 ui32Reg;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* DUT_CLK_INFO available only if SW_IF_VERSION >= 1 */
+	ui32Reg = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_SW_IF_VERSION);
+	ui32Reg = (ui32Reg & VERSION_MASK) >> VERSION_SHIFT;
+
+	if (ui32Reg >= 1)
+	{
+		ui32Reg = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CLK_INFO);
+
+		if ((ui32Reg != 0) && (ui32Reg != 0xbaadface) && (ui32Reg != 0xffffffff))
+		{
+			PVR_LOG(("TCF_CLK_CTRL_DUT_CLK_INFO = %08x", ui32Reg));
+			PVR_LOG(("Overriding provided DUT clock values: core %u, mem %u",
+			         *pui32CoreClockSpeed, *pui32MemClockSpeed));
+
+			*pui32CoreClockSpeed = ((ui32Reg & CORE_MASK) >> CORE_SHIFT) * 1000000;
+			*pui32MemClockSpeed = ((ui32Reg & MEM_MASK) >> MEM_SHIFT) * 1000000;
+		}
+	}
+#endif
+
+	PVR_LOG(("DUT clock values: core %u, mem %u",
+	         *pui32CoreClockSpeed, *pui32MemClockSpeed));
+}
+#endif
+
+static PVRSRV_ERROR SetClocks(SYS_DATA *psSysData,
+							  IMG_UINT32 ui32CoreClock,
+							  IMG_UINT32 ui32MemClock,
+							  IMG_UINT32 ui32SysClock)
+{
+	IMG_CPU_PHYADDR	sPLLRegCpuPBase;
+	void *pvPLLRegCpuVBase;
+	IMG_UINT32 ui32Value;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* Reserve the PLL register region and map the registers in */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_PLL_OFFSET, SYS_APOLLO_REG_PLL_SIZE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the PLL register region (%d)",
+				 __func__, eError));
+		return eError;
+	}
+	sPLLRegCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM) + SYS_APOLLO_REG_PLL_OFFSET;
+
+	pvPLLRegCpuVBase = OSMapPhysToLin(sPLLRegCpuPBase, SYS_APOLLO_REG_PLL_SIZE, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvPLLRegCpuVBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map PLL registers", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Modify the core clock */
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_CORE_CLK0, (ui32CoreClock / 1000000));
+
+	ui32Value = 0x1 << PLL_CORE_DRP_GO_SHIFT;
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_CORE_DRP_GO, ui32Value);
+
+	OSSleepms(600);
+
+	/* Modify the memory clock */
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_MEMIF_CLK0, (ui32MemClock / 1000000));
+
+	ui32Value = 0x1 << PLL_MEM_DRP_GO_SHIFT;
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_MEM_DRP_GO, ui32Value);
+
+	OSSleepms(600);
+#if defined(TC_APOLLO_TCF5)
+	/* Modify the sys clock */
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_SYSIF_CLK0, (ui32SysClock / 1000000));
+
+	ui32Value = 0x1 << PLL_MEM_DRP_GO_SHIFT;
+	OSWriteHWReg32(pvPLLRegCpuVBase, TCF_PLL_PLL_SYS_DRP_GO, ui32Value);
+
+	OSSleepms(600);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32SysClock);
+#endif
+
+	/* Unmap and release the PLL registers since we no longer need access to them */
+	OSUnMapPhysToLin(pvPLLRegCpuVBase, SYS_APOLLO_REG_PLL_SIZE);
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_PLL_OFFSET, SYS_APOLLO_REG_PLL_SIZE);
+
+	return PVRSRV_OK;
+}
+
+static void ApolloHardReset(SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	IMG_UINT32 ui32PolValue;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+#if defined(TC_APOLLO_TCF5)
+#if defined(TC_APOLLO_TCF5_BVNC_NOT_SUPPORTED)
+	PVR_DPF((PVR_DBG_WARNING,"\n********************************************************************************"));
+	PVR_DPF((PVR_DBG_WARNING, "%s: You are using a core which is not supported on TCF5.", __func__));
+	PVR_DPF((PVR_DBG_WARNING, "A set of fallback clock frequencies will be used. Depending on the core, you"));
+	PVR_DPF((PVR_DBG_WARNING, "could experience some problems (driver running slower, failing after some time"));
+	PVR_DPF((PVR_DBG_WARNING, "or unable to start at all)."));
+	PVR_DPF((PVR_DBG_WARNING, "********************************************************************************\n"));
+#endif
+
+	/* Put everything under reset */
+	ui32Value = (0x1 << GLB_CLKG_EN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	OSSleepms(4);
+
+	ApolloFPGAUpdateDUTClockFreq(psSysData, &ui32CoreClockSpeed, &ui32MemClockSpeed);
+
+	if (SetClocks(psSysData, ui32CoreClockSpeed, ui32MemClockSpeed, ui32SysClockSpeed) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to set the core/system/memory clocks",
+				 __func__));
+	}
+
+	/* Remove units from reset */
+	ui32Value |= (0x1 << DUT_DCM_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << SCB_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << PDP2_RESETN_SHIFT);
+	ui32Value |= (0x1 << PDP1_RESETN_SHIFT);
+	ui32Value |= (0x1 << DDR_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DUT_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32PolValue = 0x4;
+#else
+	ui32Value = (0x1 << GLB_CLKG_EN_SHIFT);
+	ui32Value |= (0x1 << SCB_RESETN_SHIFT);
+	ui32Value |= (0x1 << PDP2_RESETN_SHIFT);
+	ui32Value |= (0x1 << PDP1_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DDR_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DUT_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32Value |= (0x1 << DUT_DCM_RESETN_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_CLK_AND_RST_CTRL, ui32Value);
+
+	ui32PolValue = 0x7;
+#endif
+
+	OSSleepms(4);
+
+	pol(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DCM_LOCK_STATUS, ui32PolValue, DCM_LOCK_STATUS_MASK);
+#undef pol
+#undef polrgx
+}
+
+#if defined(TC_APOLLO_TCF5)
+static void SetMemoryLatency(SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Latency = 0, ui32RegVal = 0;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	if (ui32MemReadLatency <= 4)
+	{
+		/* The total memory read latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a read.
+		 * Set the memory read latency to 0 cycles.
+		 */
+		ui32MemReadLatency = 0;
+	}
+	else
+	{
+		ui32MemReadLatency -= 4;
+
+		PVR_LOG(("%s: Setting memory read latency to %u cycles",
+		         __func__, ui32MemReadLatency));
+	}
+
+	if (ui32MemWriteResponseLatency <= 2)
+	{
+		/* The total memory write latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a write.
+		 * Set the memory write latency to 0 cycles.
+		 */
+		ui32MemWriteResponseLatency = 0;
+	}
+	else
+	{
+		ui32MemWriteResponseLatency -= 2;
+
+		PVR_LOG(("%s: Setting memory write response latency to %u cycles",
+		         __func__, ui32MemWriteResponseLatency));
+	}
+
+	ui32Latency = (ui32MemWriteResponseLatency << 16) | ui32MemReadLatency;
+
+	/* Configure latency register */
+	SPI_Write(psSysData->pvSystemRegCpuVBase, 0x1009, ui32Latency);
+
+	/* Read back latency register */
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, 0x1009, &ui32RegVal);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: SPI_Read failed (%u)", __func__, eError));
+	}
+
+	if (ui32RegVal != ui32Latency)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Register value mismatch (actual: 0x%08x, expected: 0x%08x)",
+		         __func__, ui32RegVal, ui32Latency));
+	}
+}
+#endif
+
+static PVRSRV_ERROR PCIInitDev(PVRSRV_DEVICE_CONFIG *psDevice, void *pvOSDevice)
+{
+	SYS_DATA *psSysData;
+	IMG_CPU_PHYADDR	sApolloRegCpuPBase;
+	IMG_UINT32 uiApolloRegSize;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevice);
+
+	if (psDevice->pvOSDevice)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+#endif
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (!psSysData)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->hRGXPCI = OSPCISetDev(TO_PCI_COOKIE(pvOSDevice), HOST_PCI_INIT_FLAGS);
+	if (!psSysData->hRGXPCI)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire PCI device", __func__));
+		eError = PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+		goto ErrorFreeSysData;
+	}
+
+	/* Get Apollo register information */
+	sApolloRegCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM);
+	uiApolloRegSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM));
+
+	/* Check the address range is large enough. */
+	if (uiApolloRegSize < SYS_APOLLO_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Apollo register region isn't big enough "
+				 "(was 0x%08X, required 0x%08X)",
+				 __func__, uiApolloRegSize, SYS_APOLLO_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorPCIReleaseDevice;
+	}
+
+	/* The Apollo register range contains several register regions. Request only the system control register region */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to request the system register region",
+				 __func__));
+		goto ErrorPCIReleaseDevice;
+	}
+	psSysData->sSystemRegCpuPBase.uiAddr = sApolloRegCpuPBase.uiAddr + SYS_APOLLO_REG_SYS_OFFSET;
+	psSysData->uiSystemRegSize = SYS_APOLLO_REG_REGION_SIZE;
+
+	/* Setup Rogue register information */
+	psDevice->sRegsCpuPBase.uiAddr = OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->ui32RegsSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM));
+
+	/* Save data for this device */
+	psDevice->hSysData = (IMG_HANDLE)psSysData;
+
+	/* Check the address range is large enough. */
+	if (psDevice->ui32RegsSize < SYS_RGX_REG_REGION_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough "
+				 "(was 0x%08x, required 0x%08x)",
+				 __func__, psDevice->ui32RegsSize, SYS_RGX_REG_REGION_SIZE));
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/* Reserve the address range */
+	eError = OSPCIRequestAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		goto ErrorSystemRegReleaseAddrRegion;
+	}
+
+	/*
+	   Map in system registers so we can:
+	   - Configure the memory mode (LMA, UMA or LMA/UMA hybrid)
+	   - Hard reset Apollo
+	   - Run BIST
+	   - Clear interrupts
+	*/
+	psSysData->pvSystemRegCpuVBase = OSMapPhysToLin(psSysData->sSystemRegCpuPBase,
+													psSysData->uiSystemRegSize,
+													PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (psSysData->pvSystemRegCpuVBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map system registers",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorRGXRegReleaseAddrRange;
+	}
+
+	ApolloHardReset(psSysData);
+
+#if defined(TC_APOLLO_TCF5)
+	SetMemoryLatency(psSysData);
+#endif
+
+#if !defined(TC_APOLLO_TCF5)
+#if !defined(VIRTUAL_PLATFORM)
+	RGXInitBistery(psDevice->sRegsCpuPBase, psDevice->ui32RegsSize);
+	ApolloHardReset(psSysData);
+#endif
+	if (SetClocks(psSysData, ui32CoreClockSpeed, ui32MemClockSpeed, ui32SysClockSpeed) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to set the core/system/memory clocks",
+				 __func__));
+	}
+#endif
+
+	eError = InitMemory(psDevice, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorUnMapAddrRange;
+	}
+
+#if defined(TC_APOLLO_TCF5)
+	((RGX_DATA *)psDevice->hDevData)->psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed;
+	OSSleepms(600);
+#else
+	if (! PVRSRV_VZ_MODE_IS(GUEST))
+	{
+		IMG_UINT32 ui32Value;
+		/* Enable the rogue PLL (defaults to 3x), giving a Rogue clock of 3 x RGX_TC_CORE_CLOCK_SPEED */
+		ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1);
+		OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_DUT_CONTROL_1, ui32Value & 0xFFFFFFFB);
+
+		((RGX_DATA *)psDevice->hDevData)->psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockSpeed * 3;
+
+		OSSleepms(600);
+
+		/* Enable the temperature sensor */
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0x3, 0x46);
+	}
+
+	{
+		IMG_UINT32 ui32HoodCtrl;
+		SPI_Read(psSysData->pvSystemRegCpuVBase, 0x9, &ui32HoodCtrl);
+		ui32HoodCtrl |= 0x1;
+		SPI_Write(psSysData->pvSystemRegCpuVBase, 0x9, ui32HoodCtrl);
+	}
+#endif
+
+	psSysData->pszVersion = GetDeviceVersionString(psSysData);
+	if (psSysData->pszVersion)
+	{
+		psDevice->pszVersion = psSysData->pszVersion;
+	}
+
+	eError = OSPCIIRQ(psSysData->hRGXPCI, &psSysData->ui32IRQ);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't get IRQ", __func__));
+		goto ErrorDeInitMemory;
+	}
+
+	/* Register our handler */
+	eError = OSInstallSystemLISR(&psSysData->hLISR,
+								psSysData->ui32IRQ,
+								psDevice->pszName,
+								SystemISRHandler,
+								psSysData,
+								SYS_IRQ_FLAG_TRIGGER_DEFAULT | SYS_IRQ_FLAG_SHARED);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to install the system device interrupt handler",
+				 __func__));
+		goto ErrorDeInitMemory;
+	}
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/* Fake DVFS configuration used purely for testing purposes */
+	psDevice->sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	psDevice->sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	psDevice->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevice->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	psDevice->sDVFS.sDVFSDeviceCfg.ui32PollMs = 1000;
+	psDevice->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevice->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevice->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	psDevice->pvOSDevice = pvOSDevice;
+
+	return PVRSRV_OK;
+
+ErrorDeInitMemory:
+	DeInitMemory(psDevice, psSysData);
+
+ErrorUnMapAddrRange:
+	OSUnMapPhysToLin(psSysData->pvSystemRegCpuVBase, psSysData->uiSystemRegSize);
+	psSysData->pvSystemRegCpuVBase = NULL;
+
+ErrorRGXRegReleaseAddrRange:
+	OSPCIReleaseAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	psDevice->sRegsCpuPBase.uiAddr	= 0;
+	psDevice->ui32RegsSize		= 0;
+
+ErrorSystemRegReleaseAddrRegion:
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	psSysData->sSystemRegCpuPBase.uiAddr	= 0;
+	psSysData->uiSystemRegSize		= 0;
+
+ErrorPCIReleaseDevice:
+	OSPCIReleaseDev(psSysData->hRGXPCI);
+	psSysData->hRGXPCI = NULL;
+
+ErrorFreeSysData:
+	OSFreeMem(psSysData);
+	psDevice->hSysData = NULL;
+
+	return eError;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevice,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+#if defined(TC_APOLLO_TCF5)
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	return PVRSRV_OK;
+#else
+	PVRSRV_ERROR	eError;
+	SYS_DATA		*psSysData = psDevice->hSysData;
+	IMG_UINT32		ui32RegOffset;
+	IMG_UINT32		ui32RegVal;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	/* Read the temperature */
+	ui32RegOffset = 0x5;
+	eError = SPI_Read(psSysData->pvSystemRegCpuVBase, ui32RegOffset, &ui32RegVal);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SysDebugInfo: SPI_Read failed for register 0x%x (0x%x)", ui32RegOffset, eError));
+		goto SysDebugInfo_exit;
+	}
+
+	PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", (ui32RegVal * 233 / 4096) - 66);
+
+SysDebugInfo_exit:
+	return eError;
+#endif
+}
+#endif /* if defined(TC_APOLLO_ES2) else TC_APOLLO_TCF5 */
+
+/*
+ * GetMemorySize:
+ * For a given PCI phys addr and size, go through the memory to check the actual size.
+ * Do this by writing to address zero, and then checking positions in the memory to see if there
+ * is any aliasing. e.g. some devices may advertise 1GB of memory but in reality only have half that.
+ */
+size_t GetMemorySize(IMG_CPU_PHYADDR sMemCpuPhyAddr, IMG_UINT64 pciBarSize)
+{
+#define MEM_PROBE_VALUE 0xa5a5a5a5
+	void *pvHostMapped;
+	volatile IMG_UINT32* pMemZero;
+	size_t mem_size = 0;
+
+	PVR_ASSERT((pciBarSize & (pciBarSize-1)) == 0);   // PCI BARs are always a power of 2
+	pvHostMapped = OSMapPhysToLin(sMemCpuPhyAddr, pciBarSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvHostMapped == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot map PCI memory at phys 0x%"\
+				IMG_UINT64_FMTSPECx" for size 0x%"IMG_UINT64_FMTSPECx,\
+				__func__, (IMG_UINT64) sMemCpuPhyAddr.uiAddr, pciBarSize));
+	}
+	else
+	{
+		pMemZero = (IMG_UINT32*)pvHostMapped;
+		mem_size = 1024*1024;    // assume at least 1MB of memory
+		while (mem_size < pciBarSize)
+		{
+			volatile IMG_UINT32* pMemProbe = pMemZero + mem_size/4;
+
+			// check that the memory at offset actually exists and is not aliased
+			*pMemProbe = MEM_PROBE_VALUE;
+			*pMemZero = 0;
+			if (*pMemProbe != MEM_PROBE_VALUE)
+				break;
+			*pMemProbe = 0;     // reset value
+			mem_size <<= 1;     // double the offset we probe each time.
+		}
+		OSUnMapPhysToLin(pvHostMapped, pciBarSize);
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Local memory size 0x" IMG_SIZE_FMTSPECX " (PCI BAR size 0x%llx)", __func__, mem_size, pciBarSize));
+	}
+
+	return mem_size;
+}
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL) || (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static PVRSRV_ERROR AcquireLocalMemory(SYS_DATA *psSysData, IMG_CPU_PHYADDR *psMemCpuPAddr, size_t *puiMemSize)
+{
+	IMG_UINT16 uiVendorID, uiDevID;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiMemSize;
+	IMG_UINT32 ui32Value;
+	IMG_UINT32 uiMemLimit;
+	IMG_UINT32 ui32PCIVersion;
+	void *pvHostFPGARegCpuVBase;
+	IMG_CPU_PHYADDR	sHostFPGARegCpuPBase;
+
+	OSPCIGetVendorDeviceIDs(psSysData->hRGXPCI, &uiVendorID, &uiDevID);
+	if (uiDevID != SYS_RGX_DEV_DEVICE_ID)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected device ID 0x%X",
+				 __func__, uiDevID));
+		return PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND;
+	}
+
+	/* To get some of the version information we need to read from a register that we don't normally have
+	   mapped. Map it temporarily (without trying to reserve it) to get the information we need. */
+	sHostFPGARegCpuPBase.uiAddr	= OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM) + 0x40F0;
+	pvHostFPGARegCpuVBase = OSMapPhysToLin(sHostFPGARegCpuPBase, 0x04, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	ui32Value = OSReadHWReg32(pvHostFPGARegCpuVBase, 0);
+
+	/* Unmap the register now that we no longer need it */
+	OSUnMapPhysToLin(pvHostFPGARegCpuVBase, 0x04);
+
+	ui32PCIVersion = HEX2DEC((ui32Value & 0x00FF0000) >> 16);
+
+	if (ui32PCIVersion < 18)
+	{
+		PVR_DPF((PVR_DBG_WARNING,"\n********************************************************************************"));
+		PVR_DPF((PVR_DBG_WARNING, "%s: You have an outdated test chip fpga image.", __func__));
+		PVR_DPF((PVR_DBG_WARNING, "Restricting available graphics memory to 512mb as a workaround."));
+		PVR_DPF((PVR_DBG_WARNING, "This restriction may cause test failures for those tests"));
+		PVR_DPF((PVR_DBG_WARNING, "that require a large amount of graphics memory."));
+		PVR_DPF((PVR_DBG_WARNING, "Please speak to your customer support representative about"));
+		PVR_DPF((PVR_DBG_WARNING, "upgrading your test chip fpga image."));
+		PVR_DPF((PVR_DBG_WARNING, "********************************************************************************\n"));
+
+		/* limit to 512mb */
+		uiMemLimit = 512 * 1024 * 1024;
+		uiMemSize = TRUNCATE_64BITS_TO_32BITS(OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+	}
+	else
+	{
+		IMG_CPU_PHYADDR sMemCpuPhyAddr;
+		IMG_UINT64 pciBarSize = OSPCIAddrRangeLen(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM);
+		sMemCpuPhyAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+
+		uiMemSize = TRUNCATE_64BITS_TO_32BITS(GetMemorySize(sMemCpuPhyAddr, pciBarSize) - SYS_DEV_MEM_BROKEN_BYTES);
+		uiMemLimit = SYS_DEV_MEM_REGION_SIZE;
+	}
+
+	if (uiMemLimit > uiMemSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device memory region smaller than requested "
+				 "(got 0x%08X, requested 0x%08X)",
+				 __func__, uiMemSize, uiMemLimit));
+	}
+	else if (uiMemLimit < uiMemSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Limiting device memory region size to 0x%08X from 0x%08X",
+				 __func__, uiMemLimit, uiMemSize));
+		uiMemSize = uiMemLimit;
+	}
+
+	/* Reserve the address region */
+	eError = OSPCIRequestAddrRegion(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM, 0, uiMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device memory region not available",
+				 __func__));
+		return eError;
+	}
+
+	/* Clear any BIOS-configured MTRRs */
+	eError = OSPCIClearResourceMTRRs(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Failed to clear BIOS MTRRs",
+				 __func__));
+		/* Soft-fail, the driver can limp along. */
+	}
+
+	psMemCpuPAddr->uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+	*puiMemSize = (size_t)uiMemSize;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void ReleaseLocalMemory(SYS_DATA *psSysData, IMG_CPU_PHYADDR *psMemCpuPAddr, size_t uiMemSize)
+{
+	IMG_CPU_PHYADDR sMemCpuPBaseAddr;
+	IMG_UINT32 uiOffset;
+
+	sMemCpuPBaseAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(OSPCIAddrRangeStart(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM));
+
+	PVR_ASSERT(psMemCpuPAddr->uiAddr >= sMemCpuPBaseAddr.uiAddr);
+
+	uiOffset = (IMG_UINT32)(uintptr_t)(psMemCpuPAddr->uiAddr - sMemCpuPBaseAddr.uiAddr);
+
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM, uiOffset, (IMG_UINT32)uiMemSize);
+
+	OSPCIReleaseResourceMTRRs(psSysData->hRGXPCI, SYS_DEV_MEM_PCI_BASENUM);
+}
+
+static INLINE PVRSRV_ERROR InitLocalMemoryHeaps(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT64 ui64CardBase = 0;
+	IMG_UINT32 ui32HeapNum = 0;
+	PVRSRV_ERROR eError;
+
+	eError = AcquireLocalMemory(psSysData, &psSysData->sLocalMemCpuPBase, &psSysData->uiLocalMemSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		/* In VZ mode, allocate LMA space for the shared cross-VM information page */
+		psSysData->pui32IRQCount = (void*) OSMapPhysToLin(psSysData->sLocalMemCpuPBase,
+		                                                  (IMG_UINT64)VM_INFOPG_SIZE,
+		                                                  PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+		if (psSysData->pui32IRQCount == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map local memory page",
+					__func__));
+			return PVRSRV_ERROR_MAPPING_NOT_FOUND;
+		}
+
+		/* Initialise the only cross-VM information page entry to zero */
+		(void) OSAtomicWrite((void*)psSysData->pui32IRQCount, 0);
+
+		/* Update LMA memory range spec. to reflect above deduction */
+		psSysData->uiLocalMemSize -= (IMG_UINT64)VM_INFOPG_SIZE;
+		ui64CardBase = (IMG_UINT64)VM_INFOPG_SIZE;
+	}
+
+	/* Setup the Rogue LMA heap */
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr =
+			psSysData->sLocalMemCpuPBase.uiAddr;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr =
+#if (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+			psSysData->sLocalMemCpuPBase.uiAddr +
+#endif
+			ui64CardBase;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize =
+			psSysData->uiLocalMemSize
+#if defined(SUPPORT_DISPLAY_CLASS)
+			- RGX_TC_RESERVE_DC_MEM_SIZE
+#endif
+			;
+	ui32HeapNum++;
+
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	/* Setup the DC heap */
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr =
+		psDevConfig->pasPhysHeaps[ui32HeapNum - 1].sStartAddr.uiAddr +
+		ui64CardBase + psDevConfig->pasPhysHeaps[ui32HeapNum - 1].uiSize;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr =
+#if (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+		psSysData->sLocalMemCpuPBase.uiAddr +
+#endif
+		ui64CardBase + psDevConfig->pasPhysHeaps[ui32HeapNum - 1].uiSize;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize = RGX_TC_RESERVE_DC_MEM_SIZE;
+	ui32HeapNum++;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static INLINE void DeInitLocalMemoryHeaps(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32HeapNum = 0;
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		OSUnMapPhysToLin(psSysData->pui32IRQCount, VM_INFOPG_SIZE);
+	}
+
+	ReleaseLocalMemory(psSysData, &psSysData->sLocalMemCpuPBase, psSysData->uiLocalMemSize);
+
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize = 0;
+	ui32HeapNum++;
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sStartAddr.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].sCardBase.uiAddr = 0;
+	psDevConfig->pasPhysHeaps[ui32HeapNum].uiSize = 0;
+	ui32HeapNum++;
+#endif
+}
+#endif /* (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL) || (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID) */
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static PVRSRV_ERROR InitLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalMemoryHeaps(psDevConfig, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* No additional setup needed by guests drivers */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* Configure Apollo for regression compatibility (i.e. local memory) mode */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	ui32Value &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK | HOST_PHY_MODE_MASK);
+	ui32Value |= (0x1 << ADDRESS_FORCE_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+
+	return PVRSRV_OK;
+}
+
+static void DeInitLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	DeInitLocalMemoryHeaps(psDevConfig, psSysData);
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set the register back to the default value */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, 0x1 << ADDRESS_FORCE_SHIFT);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+}
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static PVRSRV_ERROR InitHostMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* Configure Apollo for host only mode */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	ui32Value &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK | HOST_PHY_MODE_MASK);
+	ui32Value |= (0x1 << HOST_ONLY_MODE_SHIFT);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+
+	return PVRSRV_OK;
+}
+
+static void DeInitHostMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set the register back to the default value */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, 0x1 << ADDRESS_FORCE_SHIFT);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+}
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static PVRSRV_ERROR InitHybridMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalMemoryHeaps(psDevConfig, psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* No additional setup needed for guest driver(s) */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, PVRSRV_OK);
+
+	/* No additional setup needed for the Rogue UMA heap */
+
+	/* Configure Apollo for host physical (i.e. hybrid) mode */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	ui32Value &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK | HOST_PHY_MODE_MASK);
+	ui32Value |= ((0x1 << HOST_ONLY_MODE_SHIFT) | (0x1 << HOST_PHY_MODE_SHIFT));
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+
+	/* Setup the start address of the 1GB window used by the _GPU_ to access local memory.
+	 * The value used here is the start address of the 1GB window used by the _CPU_
+	 * to access the local memory on the TC. This means that both CPU and GPU will use
+	 * the same range of physical addresses to access local memory, and any access
+	 * out of that range will be directed to system memory instead.
+	 * In the hybrid config case the conversion cpuPaddr <-> devPaddr is an assignment.
+	 */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase,
+					TCF_CLK_CTRL_HOST_PHY_OFFSET,
+					psSysData->sLocalMemCpuPBase.uiAddr);
+	OSWaitus(10);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_HOST_PHY_OFFSET);
+	OSWaitus(10);
+
+	return PVRSRV_OK;
+}
+
+static void DeInitHybridMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	DeInitLocalMemoryHeaps(psDevConfig, psSysData);
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set the register back to the default value */
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL, 0x1 << ADDRESS_FORCE_SHIFT);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_TEST_CTRL);
+	OSWaitus(10);
+}
+#endif
+
+static PVRSRV_ERROR InitMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+	eError = InitLocalMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+	eError = InitHostMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+	eError = InitHybridMemory(psDevConfig, psSysData);
+#endif
+
+	return eError;
+}
+
+static INLINE void DeInitMemory(PVRSRV_DEVICE_CONFIG *psDevConfig, SYS_DATA *psSysData)
+{
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+	DeInitLocalMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+	DeInitHostMemory(psDevConfig, psSysData);
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+	DeInitHybridMemory(psDevConfig, psSysData);
+#endif
+}
+
+static void EnableInterrupt(SYS_DATA *psSysData, IMG_UINT32 ui32InterruptFlag)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	/* Set sense to active high */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_OP_CFG);
+	ui32Value &= ~(INT_SENSE_MASK);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_OP_CFG, ui32Value);
+	OSWaitus(1000);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_OP_CFG);
+
+	/* Enable Rogue and PDP1 interrupts */
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	ui32Value |= ui32InterruptFlag;
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	OSWaitus(10);
+}
+
+static void DisableInterrupt(SYS_DATA *psSysData, IMG_UINT32 ui32InterruptFlag)
+{
+	IMG_UINT32 ui32Value;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST);
+
+	ui32Value = OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	ui32Value &= ~(ui32InterruptFlag);
+	OSWriteHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE, ui32Value);
+
+	/* Flush register write */
+	(void)OSReadHWReg32(psSysData->pvSystemRegCpuVBase, TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	OSWaitus(10);
+}
+
+static void PCIDeInitDev(PVRSRV_DEVICE_CONFIG *psDevice)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevice->hSysData;
+
+	psDevice->hSysData = NULL;
+	psDevice->pszVersion = NULL;
+
+	OSUninstallSystemLISR(psSysData->hLISR);
+
+	if (psSysData->pszVersion)
+	{
+		OSFreeMem(psSysData->pszVersion);
+		psSysData->pszVersion = NULL;
+	}
+
+	DeInitMemory(psDevice, psSysData);
+
+	OSUnMapPhysToLin(psSysData->pvSystemRegCpuVBase, psSysData->uiSystemRegSize);
+	OSPCIReleaseAddrRange(psSysData->hRGXPCI, SYS_RGX_REG_PCI_BASENUM);
+	OSPCIReleaseAddrRegion(psSysData->hRGXPCI, SYS_APOLLO_REG_PCI_BASENUM, SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+
+	OSPCIReleaseDev(psSysData->hRGXPCI);
+
+	if (psSysData->pszVersion)
+	{
+		OSFreeMem(psSysData->pszVersion);
+	}
+
+	OSFreeMem(psSysData);
+
+	psDevice->pvOSDevice = NULL;
+}
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr - psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr - psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+		}
+	}
+}
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr + psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr + psDevConfig->pasPhysHeaps[PVRSRV_PHYS_HEAP_GPU_LOCAL].sStartAddr.uiAddr;
+		}
+	}
+}
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static void TCSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void TCSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+}
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* The CPU and GPU physical addresses are always the same in Hybrid model
+	 * However on 64 bit platforms, RAM mapped above 4G of processor physical address space
+	 * cannot be accessed by this version of the test chip due to its 32bit interface.
+	 *
+	 * In fact its possible to further optimise this function by eliminating this
+	 * unnecessary copy
+	*/
+#if defined(CONFIG_64BIT)
+	PVR_ASSERT(sizeof(IMG_CPU_PHYADDR) == sizeof(IMG_DEV_PHYADDR));
+	OSCachedMemCopy(psDevPAddr, psCpuPAddr, (ui32NumOfAddr * sizeof(*psCpuPAddr)));
+#else
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+#endif
+}
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* The CPU and GPU physical addresses are always the same in Hybrid model
+	 * However on 64 bit platforms, RAM mapped above 4G of processor physical address space
+	 * cannot be accessed by this version of the test chip due to its 32bit interface.
+	 *
+	 * In fact its possible to further optimise this function by eliminating this
+	 * unnecessary copy
+	*/
+
+#if defined(CONFIG_64BIT)
+	PVR_ASSERT(sizeof(IMG_CPU_PHYADDR) == sizeof(IMG_DEV_PHYADDR));
+	OSCachedMemCopy(psCpuPAddr, psDevPAddr, (ui32NumOfAddr * sizeof(*psCpuPAddr)));
+#else
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+#endif
+
+}
+
+#endif /* (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID) */
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+// #define _DBG(...) PVR_LOG((__VA_ARGS__))
+#define _DBG(...)
+
+static PVRSRV_ERROR PrePower(IMG_HANDLE hSysData,
+                             PVRSRV_SYS_POWER_STATE eNewPowerState,
+                             PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                             PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize;
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags: 0x%08x", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags);
+
+	/* The transition might be both from ON or OFF states to OFF state so check
+	 * only for the *new* state. Also this is only valid for suspend requests. */
+	if (eNewPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_SUSPEND_REQ))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* disable interrupts */
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		IMG_UINT32 uiIntFlag = psSysData->sInterruptData[i].ui32InterruptFlag;
+		if (uiIntFlag != 0)
+		{
+			DisableInterrupt(psSysData, uiIntFlag);
+		}
+	}
+
+	/* dump video RAM content to system RAM */
+
+	eError = LMA_HeapIteratorCreate(psSysData->psDevConfig->psDevNode,
+	                                PHYS_HEAP_USAGE_GPU_LOCAL,
+	                                &psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorCreate", enable_interrupts);
+
+	eError = LMA_HeapIteratorGetHeapStats(psSysData->psHeapIter, &uiHeapTotalSize,
+	                                      &uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorGetHeapStats",
+	                      enable_interrupts);
+	uiHeapFreeSize = uiHeapTotalSize - uiHeapUsedSize;
+
+	_DBG("(%s()) heap stats: total=0x%" IMG_UINT64_FMTSPECx ", "
+	     "used=0x%" IMG_UINT64_FMTSPECx ", free=0x%" IMG_UINT64_FMTSPECx,
+	     __func__, uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize);
+
+	psSysData->pvSuspendBuffer = OSAllocMem(uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_NOMEM(psSysData->pvSuspendBuffer, eError, destroy_iterator);
+
+	_DBG("(%s()) buffer=%px", __func__, psSysData->pvSuspendBuffer);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		if (uiOffset + uiSize > uiHeapUsedSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "uiOffset = %" IMG_UINT64_FMTSPECx ", "
+			        "uiSize = %" IMG_UINT64_FMTSPECx, uiOffset, uiSize));
+
+			PVR_LOG_GOTO_WITH_ERROR("LMA_HeapIteratorNext", eError,
+			                        PVRSRV_ERROR_INVALID_OFFSET,
+			                        free_buffer);
+		}
+
+		TCHybridDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                           &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy((IMG_BYTE *) psSysData->pvSuspendBuffer + uiOffset, pvCpuVAddr,
+		       uiSize);
+#ifdef DEBUG
+		/* and now poison it - for debugging purposes so we know that the
+		 * resume procedure works */
+		memset(pvCpuVAddr, 0x9b, uiSize);
+#endif
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	/* suspend the PCI device */
+	OSPCISuspendDev(psSysData->hRGXPCI);
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvSuspendBuffer);
+	psSysData->pvSuspendBuffer = NULL;
+destroy_iterator:
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+enable_interrupts:
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		IMG_UINT32 uiIntFlag = psSysData->sInterruptData[i].ui32InterruptFlag;
+		if (uiIntFlag != 0)
+		{
+			DisableInterrupt(psSysData, uiIntFlag);
+		}
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR PostPower(IMG_HANDLE hSysData,
+                              PVRSRV_SYS_POWER_STATE eNewPowerState,
+                              PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags=0x%08x; buffer=%px", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags,
+	     psSysData->pvSuspendBuffer);
+
+	/* restore the video RAM */
+
+	/* The transition might be both to ON or OFF states from OFF state so check
+	 * only for the *current* state. Also this is only valid for resume requests. */
+	if ((eCurrentPowerState != eNewPowerState && eCurrentPowerState != PVRSRV_SYS_POWER_STATE_OFF) ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_RESUME_REQ))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* resume the PCI device */
+	OSPCIResumeDev(psSysData->hRGXPCI);
+
+	if (psSysData->pvSuspendBuffer != NULL)
+	{
+		eError = LMA_HeapIteratorReset(psSysData->psHeapIter);
+		PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorReset", free_buffer);
+
+		while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+		{
+			void *pvCpuVAddr;
+			IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+			TCHybridDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+			                           &sDevPAddr);
+
+			pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+			                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+			_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+			     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+			     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+			     pvCpuVAddr, uiSize);
+
+			/* copy memory */
+			memcpy(pvCpuVAddr, (IMG_BYTE *) psSysData->pvSuspendBuffer + uiOffset,
+			       uiSize);
+
+			uiOffset += uiSize;
+
+			OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+		}
+
+		OSFreeMem(psSysData->pvSuspendBuffer);
+		psSysData->pvSuspendBuffer = NULL;
+	}
+
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+
+	/* reset the TestChip */
+
+	eError = HardResetDevice(psSysData, psSysData->psDevConfig);
+	PVR_GOTO_IF_ERROR(eError, free_buffer);
+
+	/* enable interrupts */
+	for (i = 0; i < ARRAY_SIZE(psSysData->sInterruptData); i++)
+	{
+		IMG_UINT32 uiIntFlag = psSysData->sInterruptData[i].ui32InterruptFlag;
+		if (uiIntFlag != 0)
+		{
+			EnableInterrupt(psSysData, uiIntFlag);
+		}
+	}
+
+	/* reset DC devices */
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = psSysData->psDevConfig->psDevNode;
+		IMG_UINT32 uiDeviceCount = 0, auiDeviceIndex[16];
+
+		eError = DCDevicesEnumerate(NULL, psDevNode, ARRAY_SIZE(auiDeviceIndex),
+		                            &uiDeviceCount, auiDeviceIndex);
+		PVR_LOG_IF_ERROR(eError, "DCDevicesEnumerate");
+
+		for (i = 0; i < uiDeviceCount; i++)
+		{
+			DC_DEVICE *psDCDevice;
+			eError = DCDeviceAcquire(NULL, psDevNode, auiDeviceIndex[i],
+			                         &psDCDevice);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "DCDeviceAcquire");
+				continue;
+			}
+
+			eError = DCResetDevice(psDCDevice);
+			PVR_LOG_IF_ERROR(eError, "DCResetDevice");
+
+			eError = DCDeviceRelease(psDCDevice);
+			PVR_LOG_IF_ERROR(eError, "DCDeviceRelease");
+		}
+	}
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvSuspendBuffer);
+	psSysData->pvSuspendBuffer = NULL;
+
+	return eError;
+}
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = &gsDevices[0];
+	PVRSRV_ERROR eError;
+
+	eError = PCIInitDev(psDevConfig, pvOSDevice);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	/* power functions */
+	psDevConfig->pfnPrePowerState = PrePower;
+	psDevConfig->pfnPostPowerState = PostPower;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PCIDeInitDev(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32InterruptFlag;
+
+	switch (ui32IRQ)
+	{
+		case APOLLO_IRQ_GPU:
+			ui32InterruptFlag = (0x1 << EXT_INT_SHIFT);
+			break;
+		case APOLLO_IRQ_PDP:
+			ui32InterruptFlag = (0x1 << PDP1_INT_SHIFT);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d",
+					 __func__, ui32IRQ));
+			return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	if (psSysData->sInterruptData[ui32IRQ].pfnLISR)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ISR for %s already installed!",
+				 __func__, pszName));
+		return PVRSRV_ERROR_ISR_ALREADY_INSTALLED;
+	}
+	else if (ui32IRQ > ARRAY_SIZE(psSysData->sInterruptData))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Invalid IRQ %d, exceeds max. IRQ " IMG_SIZE_FMTSPEC "!",
+				 __func__, ui32IRQ, ARRAY_SIZE(psSysData->sInterruptData)));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psSysData->sInterruptData[ui32IRQ].psSysData	= psSysData;
+	psSysData->sInterruptData[ui32IRQ].pszName		= pszName;
+	psSysData->sInterruptData[ui32IRQ].pfnLISR		= pfnLISR;
+	psSysData->sInterruptData[ui32IRQ].pvData		= pvData;
+	psSysData->sInterruptData[ui32IRQ].ui32InterruptFlag = ui32InterruptFlag;
+
+	*phLISRData = &psSysData->sInterruptData[ui32IRQ];
+
+	EnableInterrupt(psSysData, psSysData->sInterruptData[ui32IRQ].ui32InterruptFlag);
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, psSysData->ui32IRQ));
+
+	return eError;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	SYS_INTERRUPT_DATA *psInterruptData = (SYS_INTERRUPT_DATA *)hLISRData;
+	SYS_DATA *psSysData = psInterruptData->psSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psInterruptData);
+
+	PVR_LOG(("Uninstalling device LISR %s on IRQ %d", psInterruptData->pszName, psSysData->ui32IRQ));
+
+	/* Disable interrupts for this device */
+	DisableInterrupt(psInterruptData->psSysData, psInterruptData->ui32InterruptFlag);
+
+	/* Reset the interrupt data */
+	psInterruptData->pszName = NULL;
+	psInterruptData->psSysData = NULL;
+	psInterruptData->pfnLISR = NULL;
+	psInterruptData->pvData = NULL;
+	psInterruptData->ui32InterruptFlag = 0;
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.h
new file mode 100644
index 000000000000..7f4172ad6d9c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysconfig.h
@@ -0,0 +1,318 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSCONFIG_H__)
+#define __SYSCONFIG_H__
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+/* Valid values for TC_MEMORY_CONFIG configuration option */
+#define TC_MEMORY_LOCAL			(1)
+#define TC_MEMORY_HOST			(2)
+#define TC_MEMORY_HYBRID		(3)
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+/* Memory reserved for use by the PDP DC. */
+#define RGX_TC_RESERVE_DC_MEM_SIZE	((TC_DISPLAY_MEM_SIZE) * 1024 * 1024)
+#endif
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+
+static PVRSRV_DEVICE_CONFIG gsDevices[];
+
+static RGX_TIMING_INFORMATION gsRGXTimingInfo =
+{
+	/* ui32CoreClockSpeed */
+	0,	/* Initialize to 0, real value will be set in PCIInitDev() */
+	/* bEnableActivePM */
+	IMG_TRUE,
+	/* bEnableRDPowIsland */
+	IMG_FALSE,
+	/* ui32ActivePMLatencyms */
+	SYS_RGX_ACTIVE_POWER_LATENCY_MS
+};
+
+static RGX_DATA gsRGXData =
+{
+	/* psRGXTimingInfo */
+	&gsRGXTimingInfo
+};
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr);
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static void TCSystemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCSystemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					   IMG_UINT32 ui32NumOfAddr,
+					   IMG_CPU_PHYADDR *psCpuPAddr,
+					   IMG_DEV_PHYADDR *psDevPAddr);
+
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+					  IMG_UINT32 ui32NumOfAddr,
+					  IMG_CPU_PHYADDR *psCpuPAddr,
+					  IMG_DEV_PHYADDR *psDevPAddr);
+
+#endif /* (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID) */
+
+#if (TC_MEMORY_CONFIG == TC_MEMORY_LOCAL)
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	TCLocalCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	TCLocalDevPAddrToCpuPAddr,
+};
+
+static PHYS_HEAP_CONFIG	gsPhysHeapConfig[] =
+{
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsLocalPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_GPU_LOCAL,
+	},
+#if defined(SUPPORT_DISPLAY_CLASS)
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsLocalPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_DISPLAY,
+	},
+#endif
+};
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HOST)
+static PHYS_HEAP_FUNCTIONS gsSystemPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	TCSystemCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	TCSystemDevPAddrToCpuPAddr,
+};
+
+static PHYS_HEAP_CONFIG	gsPhysHeapConfig[] =
+{
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_UMA,
+		/* pszPDumpMemspaceName */
+		"SYSMEM",
+		/* psMemFuncs */
+		&gsSystemPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_GPU_LOCAL,
+	}
+};
+#elif (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+static PHYS_HEAP_FUNCTIONS gsHybridPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	TCHybridCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	TCHybridDevPAddrToCpuPAddr,
+};
+
+static PHYS_HEAP_CONFIG	gsPhysHeapConfig[] =
+{
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsHybridPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_GPU_LOCAL,
+	},
+#if defined(SUPPORT_DISPLAY_CLASS)
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_LMA,
+		/* pszPDumpMemspaceName */
+		"LMA",
+		/* psMemFuncs */
+		&gsHybridPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_DISPLAY,
+	},
+#endif
+	{
+		/* eType */
+		PHYS_HEAP_TYPE_UMA,
+		/* pszPDumpMemspaceName */
+		"SYSMEM",
+		/* psMemFuncs */
+		&gsHybridPhysHeapFuncs,
+		/* sStartAddr */
+		{0},
+		/* sCardBase */
+		{0},
+		/* uiSize */
+		0,
+		/* hPrivData */
+		(IMG_HANDLE)&gsDevices[0],
+		/* ui32UsageFlags */
+		PHYS_HEAP_USAGE_CPU_LOCAL,
+	}
+};
+#else
+#error "TC_MEMORY_CONFIG not valid"
+#endif
+
+static PVRSRV_DEVICE_CONFIG gsDevices[] =
+{
+	{
+		.pvOSDevice		= NULL,
+		.psDevNode		= NULL,
+		.pszName		= "apollo",
+		.pszVersion		= NULL,
+
+		/* Device setup information */
+		.sRegsCpuPBase		= { 0 },
+		.ui32RegsSize		= 0,
+		.ui32IRQ			= 0,
+
+		.eCacheSnoopingMode	= PVRSRV_DEVICE_SNOOP_NONE,
+
+		.hDevData			= &gsRGXData,
+		.hSysData			= NULL,
+
+		.bHasNonMappableLocalMemory	= IMG_FALSE,
+
+		/* Physical memory heaps */
+		.pasPhysHeaps		= &gsPhysHeapConfig[0],
+		.ui32PhysHeapCount	= ARRAY_SIZE(gsPhysHeapConfig),
+
+		.pfnPrePowerState	= NULL,
+		.pfnPostPowerState	= NULL,
+		.bHasFBCDCVersion31	= IMG_FALSE,
+
+		/* Only required for LMA but having this always set shouldn't be a problem */
+		.bDevicePA0IsValid	= IMG_TRUE,
+
+		.pfnClockFreqGet	= NULL,
+
+		.pfnCheckMemAllocSize = NULL,
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		.pfnTDSendFWImage		= NULL,
+		.pfnTDSetPowerParams	= NULL,
+		.pfnTDRGXStart			= NULL,
+		.pfnTDRGXStop			= NULL,
+#endif
+		.pfnSysDevFeatureDepInit	= NULL
+	}
+};
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* !defined(__SYSCONFIG_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysdata.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysdata.h
new file mode 100644
index 000000000000..0a0fed814ae9
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysdata.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Data Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv_device.h"
+#include "interrupt_support.h"
+
+#if !defined(__SYSDATA_H__)
+#define __SYSDATA_H__
+
+typedef struct _SYS_INTERRUPT_DATA_
+{
+	void			*psSysData;
+	const IMG_CHAR	*pszName;
+	PFN_SYS_LISR	pfnLISR;
+	void			*pvData;
+	IMG_UINT32		ui32InterruptFlag;
+	IMG_UINT32		ui32IRQ;
+} SYS_INTERRUPT_DATA;
+
+typedef struct _SYS_DATA_
+{
+	IMG_HANDLE			hRGXPCI;
+
+	IMG_CHAR			*pszVersion;
+
+	IMG_CPU_PHYADDR		sSystemRegCpuPBase;
+	void				*pvSystemRegCpuVBase;
+	size_t				uiSystemRegSize;
+
+	IMG_CPU_PHYADDR		sLocalMemCpuPBase;
+	size_t				uiLocalMemSize;
+
+	IMG_HANDLE			hLISR;
+	IMG_UINT32			ui32IRQ;
+
+	/* Used by VZ host/guest(s) drivers */
+	IMG_UINT32			*pui32IRQCount;
+
+	/* Rogue and PDP */
+	SYS_INTERRUPT_DATA	sInterruptData[2];
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	void *pvSuspendBuffer;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+} SYS_DATA;
+
+#endif /* !defined(__SYSDATA_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysinfo.h
new file mode 100644
index 000000000000..042b4e8ceb5a
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/rogue/rgx_tc/sysinfo.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+#define SYS_RGX_DEV_VENDOR_ID	(0x1010)
+#define SYS_RGX_DEV_DEVICE_ID	(0x1CF2)
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM)
+#define MAX_HW_TIME_US                           (5000000)
+#else
+#define MAX_HW_TIME_US                           (500000)
+#endif
+
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/env/linux/dma_support.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/env/linux/dma_support.c
new file mode 100644
index 000000000000..977f9c85d037
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/env/linux/dma_support.c
@@ -0,0 +1,335 @@
+/*************************************************************************/ /*!
+@File           dma_support.c
+@Title          System DMA support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a contiguous memory allocator (i.e. DMA allocator);
+                APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#endif
+
+#include "allocmem.h"
+#include "dma_support.h"
+
+#define DMA_MAX_IOREMAP_ENTRIES 8
+static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
+static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};
+static IMG_UINT32 gsDmaIoRemapRef[DMA_MAX_IOREMAP_ENTRIES] = {0};
+
+/*!
+******************************************************************************
+ @Function			SysDmaAllocMem
+
+ @Description		Allocates physically contiguous memory
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDmaAlloc != NULL && psDmaAlloc->pvOSDevice != NULL)
+	{
+#if defined(__linux__)
+		psDmaAlloc->pvVirtAddr =
+				dma_alloc_coherent((struct device *)psDmaAlloc->pvOSDevice,
+								   (size_t) psDmaAlloc->ui64Size,
+								   (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr,
+								   GFP_KERNEL);
+		PVR_LOG_RETURN_IF_FALSE((NULL != psDmaAlloc->pvVirtAddr), "dma_alloc_coherent() failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
+#else
+		#error "Provide OS implementation of DMA allocation";
+#endif
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaFreeMem
+
+ @Description		Free physically contiguous memory
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
+{
+	if (psDmaAlloc && psDmaAlloc->pvVirtAddr)
+	{
+#if defined(__linux__)
+		dma_free_coherent((struct device *)psDmaAlloc->pvOSDevice,
+						  (size_t) psDmaAlloc->ui64Size,
+						  psDmaAlloc->pvVirtAddr,
+						  (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
+#else
+		#error "Provide OS implementation of DMA deallocation";
+#endif
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaRegisterForIoRemapping
+
+ @Description		Registers DMA_ALLOC for manual I/O remapping
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (psDmaAlloc->pvVirtAddr == NULL)
+	{
+		/* Check if an I/O remap entry already exists for this request */
+		for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+		{
+			if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+			{
+				PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr);
+				break;
+			}
+		}
+
+		if (ui32Idx < DMA_MAX_IOREMAP_ENTRIES)
+		{
+			IMG_UINT64 ui64Offset;
+			ui64Offset = psDmaAlloc->sBusAddr.uiAddr - gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr;
+			psDmaAlloc->pvVirtAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr + (uintptr_t)ui64Offset;
+			gsDmaIoRemapRef[ui32Idx] += 1;
+			return PVRSRV_OK;
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check if there is a free I/O remap table entry for this request */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0)
+		{
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL);
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
+			break;
+		}
+	}
+
+	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
+	{
+		return eError;
+	}
+
+	gsDmaIoRemapArray[ui32Idx].ui64Size = psDmaAlloc->ui64Size;
+	gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
+	gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;
+	gsDmaIoRemapRef[ui32Idx] += 1;
+
+	PVR_LOG(("DMA: register I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+			psDmaAlloc->pvVirtAddr,
+			psDmaAlloc->sBusAddr.uiAddr,
+			psDmaAlloc->ui64Size));
+
+	gbEnableDmaIoRemapping = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDeregisterForIoRemapping
+
+ @Description		Deregisters DMA_ALLOC from manual I/O remapping
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	IMG_UINT32 ui32Idx;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		return;
+	}
+
+	/* Remove specified entry from the list of I/O remap entries */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr <= psDmaAlloc->sBusAddr.uiAddr &&
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr + gsDmaIoRemapArray[ui32Idx].ui64Size >= psDmaAlloc->sBusAddr.uiAddr + psDmaAlloc->ui64Size)
+		{
+			if (! --gsDmaIoRemapRef[ui32Idx])
+			{
+				PVR_LOG(("DMA: deregister I/O remap: VA: 0x%p, PA: 0x%llx, Size: 0x%llx",
+						gsDmaIoRemapArray[ui32Idx].pvVirtAddr,
+						gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr,
+						gsDmaIoRemapArray[ui32Idx].ui64Size));
+
+				gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
+				gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
+				gsDmaIoRemapArray[ui32Idx].ui64Size = 0;
+			}
+
+			break;
+		}
+	}
+
+	/* Check if no other I/O remap entries exists for remapping */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
+		{
+			break;
+		}
+	}
+
+	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
+	{
+		/* No entries found so disable remapping */
+		gbEnableDmaIoRemapping = IMG_FALSE;
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDevPAddrToCpuVAddr
+
+ @Description		Maps a DMA_ALLOC physical address to CPU virtual address
+
+ @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+ ******************************************************************************/
+IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
+{
+	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return pvDMAVirtAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
+				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: PA: 0x%llx => VA: 0x%p",
+					uiAddr, pvDMAVirtAddr));
+
+				break;
+			}
+		}
+	}
+
+	return pvDMAVirtAddr;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaCpuVAddrToDevPAddr
+
+ @Description		Maps a DMA_ALLOC CPU virtual address to physical address
+
+ @Return			Non-zero value on success. Otherwise, a 0
+ ******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
+{
+	IMG_UINT64 uiAddr = 0;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return uiAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: VA: 0x%p => PA: 0x%llx",
+					pvDMAVirtAddr, uiAddr));
+
+				break;
+			}
+		}
+	}
+
+	return uiAddr;
+}
+
+/******************************************************************************
+ End of file (dma_support.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_stub.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_stub.c
new file mode 100644
index 000000000000..2dc3fe1e92d3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_stub.c
@@ -0,0 +1,119 @@
+/*************************************************************************/ /*!
+@File			vmm_type_stub.c
+@Title          Stub VM manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Sample stub (no-operation) VM manager implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+#if 0
+//eswin: conflict with eswin version!
+static PVRSRV_ERROR
+StubVMMMapDevPhysHeap(IMG_UINT64 ui64Size,
+					  IMG_UINT64 ui64Addr)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(ui64Addr);
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+StubVMMUnmapDevPhysHeap(void)
+{
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static VMM_PVZ_CONNECTION gsStubVmmPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&StubVMMMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&StubVMMUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsStubVmmPvz;
+	PVR_DPF((PVR_DBG_MESSAGE, "Using a stub VM manager type, no hypercall support"));
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+#endif
+
+/******************************************************************************
+ End of file (vmm_type_stub.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_vzfpga.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_vzfpga.c
new file mode 100644
index 000000000000..71f85d715f17
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/common/vmm_type_vzfpga.c
@@ -0,0 +1,169 @@
+/*************************************************************************/ /*!
+@File           vmm_type_vzfpga.c
+@Title          VZFPGA manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    VM manager implementation to support vzfpga platform
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vz_vm.h"
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+GetDriverIDFromHeapBase(IMG_UINT64 ui64Addr, IMG_UINT32 *pui32DriverID)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_DEV_PHYADDR sHostFwHeapBaseDevPA = {0};
+	PHYS_HEAP *psHostFwHeap = NULL;
+	PVRSRV_DEVICE_NODE *psHostDevNode = PVRSRVGetDeviceInstance(0);
+
+	PVR_LOG_RETURN_IF_FALSE((psHostDevNode != NULL),
+							"Host Device Node not initialised.",
+							PVRSRV_ERROR_NO_DEVICENODE_FOUND);
+
+	psHostFwHeap = psHostDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+	PVR_LOG_RETURN_IF_FALSE((psHostFwHeap != NULL),
+							"Host Fw heap not initialised.",
+							PVRSRV_ERROR_PHYSHEAP_ID_INVALID);
+
+	eErr = PhysHeapGetDevPAddr(psHostFwHeap, &sHostFwHeapBaseDevPA);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PhysHeapGetDevPAddr");
+
+	*pui32DriverID = (ui64Addr - sHostFwHeapBaseDevPA.uiAddr) / PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+	PVR_LOG_RETURN_IF_FALSE((*pui32DriverID >= RGXFW_GUEST_DRIVER_ID_START) &&
+							(*pui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+							"Invalid Guest DriverID",
+							PVRSRV_ERROR_INVALID_PVZ_OSID);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+VZFPGAMapDevPhysHeap(IMG_UINT64 ui64Size,
+					 IMG_UINT64 ui64Addr)
+{
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+	IMG_UINT32 ui32GuestDriverID;
+
+	eErr = GetDriverIDFromHeapBase(ui64Addr, &ui32GuestDriverID);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "GetDriverIDFromHeapBase");
+
+	eErr = PvzServerOnVmOnline(ui32GuestDriverID, 0);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerOnVmOnline");
+
+	eErr = PvzServerMapDevPhysHeap(ui32GuestDriverID, 0, ui64Size, ui64Addr);
+	PVR_LOG_RETURN_IF_ERROR(eErr, "PvzServerMapDevPhysHeap");
+
+	return eErr;
+}
+
+static PVRSRV_ERROR
+VZFPGAUnmapDevPhysHeap(void)
+{
+	IMG_UINT32 ui32ID;
+
+	/* During shutdown, the Guests will be deinitialised in reverse order */
+	for (ui32ID=(RGX_NUM_DRIVERS_SUPPORTED-1);
+		 ui32ID >= RGXFW_HOST_DRIVER_ID; ui32ID--)
+	{
+		if (IsVmOnline(ui32ID, 0))
+		{
+			PvzServerUnmapDevPhysHeap(ui32ID, 0);
+			PvzServerOnVmOffline(ui32ID, 0);
+			break;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static VMM_PVZ_CONNECTION gsVZFPGAPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&VZFPGAMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&VZFPGAUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		&PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		&PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		&PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		&PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		&PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsVZFPGAPvz;
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_vzfpga.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/include/dma_support.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/include/dma_support.h
new file mode 100644
index 000000000000..97445e8cf9ef
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/include/dma_support.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@File           dma_support.h
+@Title          Device contiguous memory allocator and I/O re-mapper
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides a contiguous memory allocator API; mainly
+                used for allocating / ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_SUPPORT_H
+#define DMA_SUPPORT_H
+
+#include "osfunc.h"
+#include "pvrsrv.h"
+
+typedef struct _DMA_ALLOC_
+{
+	IMG_UINT64       ui64Size;
+	IMG_CPU_VIRTADDR pvVirtAddr;
+	IMG_DEV_PHYADDR  sBusAddr;
+	void             *pvOSDevice;
+} DMA_ALLOC;
+
+/*!
+*******************************************************************************
+ @Function      SysDmaAllocMem
+ @Description   Allocates physically contiguous memory
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaFreeMem
+ @Description   Free physically contiguous memory
+ @Return        void
+******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psCmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaRegisterForIoRemapping
+ @Description   Registers DMA_ALLOC for manual I/O remapping
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDeregisterForIoRemapping
+ @Description   Deregisters DMA_ALLOC from manual I/O remapping
+ @Return        void
+******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDevPAddrToCpuVAddr
+ @Description   Maps a DMA_ALLOC physical address to CPU virtual address
+ @Return        IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+******************************************************************************/
+IMG_CPU_VIRTADDR
+SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaCpuVAddrToDevPAddr
+ @Description   Maps a DMA_ALLOC CPU virtual address to physical address
+ @Return        Non-zero value on success. Otherwise, a 0
+******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr);
+
+#endif /* DMA_SUPPORT_H */
+
+/******************************************************************************
+ End of file (dma_support.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/Kbuild.mk
new file mode 100644
index 000000000000..3a0b79b13c7c
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/Kbuild.mk
@@ -0,0 +1,52 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+$(PVRSRV_MODNAME)-y += \
+	services/system/volcanic/$(PVR_SYSTEM)/sysconfig.o \
+	services/system/common/sysconfig_cmn.o
+
+$(PVRSRV_MODNAME)-y += \
+	services/system/volcanic/$(PVR_SYSTEM)/fpga.o
+
+ccflags-y += \
+	-I$(TOP)/services/system/volcanic/common/env/linux \
+	-I$(TOP)/services/include/env/linux \
+	-I$(TOP)/kernel/drivers/staging/imgtec/tc \
+	-I$(TOP)/include/volcanic/system/rgx_tc -I$(TOP)/include/system/rgx_tc
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.c
new file mode 100644
index 000000000000..2698c12bf534
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.c
@@ -0,0 +1,302 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#include <linux/module.h>
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pci_support.h"
+#include "os_apphint.h"
+#include "pdump_km.h"
+#include "rgxtbdefs_km.h"
+
+
+#include "fpga.h"
+
+/*!
+ ******************************************************************************
+ * System FPGA parameters
+ *****************************************************************************/
+static IMG_UINT32 gui32SysTBBWDropN                 = 0;
+static IMG_UINT32 gui32SysTBBWPeriod                = 0;
+static IMG_UINT32 gui32SysTBQOS0Min                 = 0;
+static IMG_UINT32 gui32SysTBQOS0Max                 = 0;
+static IMG_UINT32 gui32SysTBQOS15Min                = 0;
+static IMG_UINT32 gui32SysTBQOS15Max                = 0;
+static IMG_UINT32 gui32SysTBQOSDist                 = 0;
+static IMG_UINT32 gui32SysTBMemArbiter              = 0;
+static IMG_UINT32 gui32SysTBMaxIdOutRW              = 0;
+static IMG_UINT32 gui32SysTBMaxIdOutWr              = 0;
+static IMG_UINT32 gui32SysTBMaxIdOutRd              = 0;
+/* these allow raw writes to RGX_TB_QOS_RD_LATENCY and RGX_TB_QOS_WR_LATENCY */
+static IMG_UINT64 gui64SysTBQOSLatencyRd            = 0;
+static IMG_UINT64 gui64SysTBQOSLatencyWr            = 0;
+
+#if defined(__linux__)
+#include <linux/module.h>
+#include <linux/dma-mapping.h>
+module_param_named(sys_tb_bandwidth_drop,            gui32SysTBBWDropN,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_bandwidth_period,          gui32SysTBBWPeriod,                uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos0_min,                  gui32SysTBQOS0Min,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos0_max,                  gui32SysTBQOS0Max,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos15_min,                 gui32SysTBQOS15Min,                uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos15_max,                 gui32SysTBQOS15Max,                uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos_dist,                  gui32SysTBQOSDist,                 uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_mem_arbiter,               gui32SysTBMemArbiter,              uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_max_id_outstanding_rdwr,   gui32SysTBMaxIdOutRW,              uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_max_id_outstanding_wr,     gui32SysTBMaxIdOutWr,              uint,   S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_max_id_outstanding_rd,     gui32SysTBMaxIdOutRd,              uint,   S_IRUGO | S_IWUSR);
+
+module_param_named(sys_tb_qos_latency_rd,            gui64SysTBQOSLatencyRd,            ullong,  S_IRUGO | S_IWUSR);
+module_param_named(sys_tb_qos_latency_wr,            gui64SysTBQOSLatencyWr,            ullong,  S_IRUGO | S_IWUSR);
+#endif
+
+
+/*
+ * work out the length in bits for a register field from its ~CLRMSK.
+ * Assumes the relevant bits are all contiguous.
+ */
+static IMG_UINT32 FieldLengthBits(IMG_UINT32 mask)
+{
+	IMG_UINT32 count = 0;
+
+	while (mask != 0)
+	{
+		count += (mask & 1);
+		mask >>= 1;
+	}
+	return count;
+}
+
+
+static IMG_UINT32 TBBandwidthLimiterGet(void)
+{
+	IMG_UINT32 ui32BandwidthLimiter = 0;
+
+	/* create bandwidth limiter reg value */
+	if (gui32SysTBBWDropN != 0  ||  gui32SysTBBWPeriod != 0)
+	{
+		IMG_UINT32 ui31DropN_ext;
+		IMG_UINT32 ui31Period_ext;
+
+		/* get EXT bits from the apphint values */
+		ui31DropN_ext = gui32SysTBBWDropN >> FieldLengthBits(~RGX_TB_BW_LIMITER_DROPN_CLRMSK);
+		ui31Period_ext = gui32SysTBBWPeriod >> FieldLengthBits(~RGX_TB_BW_LIMITER_PERIOD_CLRMSK);
+
+		ui32BandwidthLimiter = (RGX_TB_BW_LIMITER_ENABLE_EN << RGX_TB_BW_LIMITER_ENABLE_SHIFT) |
+							   ((gui32SysTBBWDropN << RGX_TB_BW_LIMITER_DROPN_SHIFT) & ~RGX_TB_BW_LIMITER_DROPN_CLRMSK) |
+							   ((gui32SysTBBWPeriod << RGX_TB_BW_LIMITER_PERIOD_SHIFT) & ~RGX_TB_BW_LIMITER_PERIOD_CLRMSK) |
+							   ((ui31DropN_ext << RGX_TB_BW_LIMITER_DROPN_EXT_SHIFT) & ~RGX_TB_BW_LIMITER_DROPN_EXT_CLRMSK) |
+							   ((ui31Period_ext << RGX_TB_BW_LIMITER_PERIOD_EXT_SHIFT) & ~RGX_TB_BW_LIMITER_PERIOD_EXT_CLRMSK);
+	}
+
+	return ui32BandwidthLimiter;
+}
+
+
+/*
+ * These latencies can be specified in total using gui64SysTBQOSLatencyRd/Wr or in individual fields.
+ * If full register is specified then the individual fields are ignored.
+ * Individual fields only allow same values to be set for read and write.
+ */
+static IMG_UINT64 TBQOSReadLatencyGet(void)
+{
+	if (gui64SysTBQOSLatencyRd != 0)
+	{
+		return gui64SysTBQOSLatencyRd;
+	}
+
+	return ((((IMG_UINT64)gui32SysTBQOS15Max) << RGX_TB_QOS_RD_LATENCY_MAX_15_SHIFT) & ~RGX_TB_QOS_RD_LATENCY_MAX_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS15Min) << RGX_TB_QOS_RD_LATENCY_MIN_15_SHIFT) & ~RGX_TB_QOS_RD_LATENCY_MIN_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Max)  << RGX_TB_QOS_RD_LATENCY_MAX_0_SHIFT)  & ~RGX_TB_QOS_RD_LATENCY_MAX_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Min)  << RGX_TB_QOS_RD_LATENCY_MIN_0_SHIFT)  & ~RGX_TB_QOS_RD_LATENCY_MIN_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOSDist)  << RGX_TB_QOS_RD_LATENCY_DIST_SHIFT)  & ~RGX_TB_QOS_RD_LATENCY_DIST_CLRMSK);
+}
+
+static IMG_UINT64 TBQOSWriteLatencyGet(void)
+{
+	if (gui64SysTBQOSLatencyWr != 0)
+	{
+		return gui64SysTBQOSLatencyWr;
+	}
+
+	return ((((IMG_UINT64)gui32SysTBQOS15Max) << RGX_TB_QOS_WR_LATENCY_MAX_15_SHIFT) & ~RGX_TB_QOS_WR_LATENCY_MAX_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS15Min) << RGX_TB_QOS_WR_LATENCY_MIN_15_SHIFT) & ~RGX_TB_QOS_WR_LATENCY_MIN_15_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Max)  << RGX_TB_QOS_WR_LATENCY_MAX_0_SHIFT)  & ~RGX_TB_QOS_WR_LATENCY_MAX_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOS0Min)  << RGX_TB_QOS_WR_LATENCY_MIN_0_SHIFT)  & ~RGX_TB_QOS_WR_LATENCY_MIN_0_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBQOSDist)  << RGX_TB_QOS_WR_LATENCY_DIST_SHIFT)  & ~RGX_TB_QOS_WR_LATENCY_DIST_CLRMSK);
+}
+
+static IMG_UINT32 TBMemArbiterGet(void)
+{
+	return gui32SysTBMemArbiter;
+}
+
+static IMG_UINT64 TBQOSMaxIdOutstandingGet(void)
+{
+	return ((((IMG_UINT64)gui32SysTBMaxIdOutRW) << RGX_TB_MAX_ID_OUTSTANDING_RD_WR_SHIFT) & ~RGX_TB_MAX_ID_OUTSTANDING_RD_WR_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBMaxIdOutWr) << RGX_TB_MAX_ID_OUTSTANDING_WRITE_SHIFT) & ~RGX_TB_MAX_ID_OUTSTANDING_WRITE_CLRMSK) |
+	       ((((IMG_UINT64)gui32SysTBMaxIdOutRd) << RGX_TB_MAX_ID_OUTSTANDING_READ_SHIFT) & ~RGX_TB_MAX_ID_OUTSTANDING_READ_CLRMSK);
+}
+
+
+PVRSRV_ERROR FPGA_Reset(struct resource *registers, IMG_BOOL bFullReset)
+{
+	IMG_CPU_PHYADDR	sWrapperRegsCpuPBase;
+	void			*pvWrapperRegs;
+	IMG_UINT32		ui32BandwidthLimiter;
+	IMG_UINT64		ui64ReadQOSLatency;
+	IMG_UINT64		ui64WriteQOSLatency;
+	IMG_UINT32		ui32MemArbiter;
+	IMG_UINT64		ui64MaxIdOutstanding;
+
+	sWrapperRegsCpuPBase.uiAddr = registers->start + FPGA_RGX_TB_REG_WRAPPER_OFFSET;
+
+	/*
+		Create a temporary mapping of the FPGA wrapper registers in order to reset
+		required registers.
+	*/
+	pvWrapperRegs = OSMapPhysToLin(sWrapperRegsCpuPBase,
+								   FPGA_RGX_TB_REG_WRAPPER_SIZE,
+								   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvWrapperRegs == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"%s: Failed to create wrapper register mapping", __func__));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	/*
+		Set the bandwidth limiter if required.
+	*/
+	ui32BandwidthLimiter = TBBandwidthLimiterGet();
+	if (ui32BandwidthLimiter != 0)
+	{
+		OSWriteHWReg32(pvWrapperRegs, RGX_TB_BW_LIMITER, ui32BandwidthLimiter);
+		(void) OSReadHWReg32(pvWrapperRegs, RGX_TB_BW_LIMITER);
+		PVR_LOG(("%s: Bandwidth limiter = 0x%08X", __func__, OSReadHWReg32(pvWrapperRegs, RGX_TB_BW_LIMITER)));
+	}
+
+	/*
+		Set the read/write QoS latency values if required.
+	*/
+	ui64ReadQOSLatency = TBQOSReadLatencyGet();
+	if (ui64ReadQOSLatency != 0)
+	{
+		OSWriteHWReg64(pvWrapperRegs, RGX_TB_QOS_RD_LATENCY, ui64ReadQOSLatency);
+		(void) OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_RD_LATENCY);
+		PVR_LOG(("%s: QOS Read latency = 0x%016llX", __func__, OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_RD_LATENCY)));
+	}
+
+	ui64WriteQOSLatency = TBQOSWriteLatencyGet();
+	if (ui64WriteQOSLatency != 0)
+	{
+		OSWriteHWReg64(pvWrapperRegs, RGX_TB_QOS_WR_LATENCY, ui64WriteQOSLatency);
+		(void) OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_WR_LATENCY);
+		PVR_LOG(("%s: QOS Write latency = 0x%016llX", __func__, OSReadHWReg64(pvWrapperRegs, RGX_TB_QOS_WR_LATENCY)));
+	}
+
+	ui32MemArbiter = TBMemArbiterGet();
+	if (ui32MemArbiter != 0)
+	{
+		OSWriteHWReg32(pvWrapperRegs, RGX_TB_MEM_ARBITER, ui32MemArbiter);
+		(void) OSReadHWReg32(pvWrapperRegs, RGX_TB_MEM_ARBITER);
+		PVR_LOG(("%s: Mem arbiter = 0x%08X", __func__, OSReadHWReg32(pvWrapperRegs, RGX_TB_MEM_ARBITER)));
+	}
+
+	ui64MaxIdOutstanding = TBQOSMaxIdOutstandingGet();
+	if (ui64MaxIdOutstanding != 0)
+	{
+		OSWriteHWReg64(pvWrapperRegs, RGX_TB_MAX_ID_OUTSTANDING, ui64MaxIdOutstanding);
+		(void) OSReadHWReg64(pvWrapperRegs, RGX_TB_MAX_ID_OUTSTANDING);
+		PVR_LOG(("%s: Max Id Outstanding = 0x%016llX", __func__, OSReadHWReg64(pvWrapperRegs, RGX_TB_MAX_ID_OUTSTANDING)));
+	}
+
+	/*
+		Remove the temporary register mapping.
+	*/
+	OSUnMapPhysToLin(pvWrapperRegs, FPGA_RGX_TB_REG_WRAPPER_SIZE);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR FPGA_SysDebugInfo(struct resource *registers,
+                               DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                               void *pvDumpDebugFile)
+{
+	IMG_CPU_PHYADDR	sWrapperRegsCpuPBase;
+	void			*pvWrapperRegs;
+
+	sWrapperRegsCpuPBase.uiAddr = registers->start + FPGA_RGX_TB_REG_WRAPPER_OFFSET;
+
+	/*
+		Create a temporary mapping of the FPGA wrapper registers.
+	*/
+	pvWrapperRegs = OSMapPhysToLin(sWrapperRegsCpuPBase,
+								   FPGA_RGX_TB_REG_WRAPPER_SIZE,
+								   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (pvWrapperRegs == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"%s: Failed to create wrapper register mapping", __func__));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+#define SYS_FPGA_DBG_R32(R)	PVR_DUMPDEBUG_LOG("%-29s : 0x%08X", #R, (IMG_UINT32)OSReadHWReg32(pvWrapperRegs, RGX_TB_##R))
+#define SYS_FPGA_DBG_R64(R)	PVR_DUMPDEBUG_LOG("%-29s : 0x%016llX", #R, (IMG_UINT64)OSReadHWReg64(pvWrapperRegs, RGX_TB_##R))
+
+	SYS_FPGA_DBG_R32(BW_LIMITER);
+
+	SYS_FPGA_DBG_R64(QOS_RD_LATENCY);
+	SYS_FPGA_DBG_R64(QOS_WR_LATENCY);
+
+	SYS_FPGA_DBG_R32(MEM_ARBITER);
+	SYS_FPGA_DBG_R64(MAX_ID_OUTSTANDING);
+
+	/*
+		Remove the temporary register mapping.
+	*/
+	OSUnMapPhysToLin(pvWrapperRegs, FPGA_RGX_TB_REG_WRAPPER_SIZE);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.h
new file mode 100644
index 000000000000..d810efc45d19
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/fpga.h
@@ -0,0 +1,67 @@
+/*************************************************************************/ /*!
+@File
+@Title          FPGA PCI header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines FPGA PCI registers
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*
+ * Defines for FPGA System Bus Decode.
+ */
+
+#define FPGA_RGX_TB_REG_PCI_BASENUM    2
+#define FPGA_RGX_TB_REG_SIZE           (16 * 1024)
+#define FPGA_RGX_TB_REG_WRAPPER_OFFSET 0x1000000
+#define FPGA_RGX_TB_REG_WRAPPER_SIZE   0x8000
+
+
+/*
+ * Interface to the Reset function
+ */
+PVRSRV_ERROR FPGA_Reset(struct resource *registers, IMG_BOOL bFullReset);
+
+#if 1
+#define DevReset(psSysData, flags)  FPGA_Reset((psSysData)->registers, flags)
+#else
+#define DevReset(psSysData, flags)  PVRSRV_OK
+#endif
+
+PVRSRV_ERROR FPGA_SysDebugInfo(struct resource *registers,
+                               DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                               void *pvDumpDebugFile);
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysconfig.c
new file mode 100644
index 000000000000..0dda8636bf8e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysconfig.c
@@ -0,0 +1,1772 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "sysinfo.h"
+#include "apollo_regs.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "syscommon.h"
+#include "os_apphint.h"
+#include "rgxfwutils.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_support.h"
+#include "ion_sys.h"
+#endif
+
+#include "vmm_pvz_server.h"
+#include "pvr_bridge_k.h"
+#include "pvr_drv.h"
+#include "tc_drv.h"
+#include "fpga.h"
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/delay.h>
+
+#define SECURE_FW_MEM_SIZE   (0x80000) /* 512Kb */
+#define SECURE_MEM_SIZE    (0x4000000) /*  64MB */
+
+typedef struct
+{
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;
+	IMG_UINT64 uiSize;
+	IMG_BOOL bUsed;
+	IMG_UINT32 ui32DriverModeMask;
+} CARD_PHYS_HEAP_CONFIG_SPEC;
+
+#define PHYSHEAP_ALL_DRIVERS	(1 << DRIVER_MODE_NATIVE | \
+								 1 << DRIVER_MODE_HOST   | \
+								 1 << DRIVER_MODE_GUEST)
+
+#define PHYSHEAP_NO_GUESTS	(1 << DRIVER_MODE_NATIVE | \
+							 1 << DRIVER_MODE_HOST)
+
+#define PHYSHEAP_ONLY_VZ	(1 << DRIVER_MODE_GUEST | \
+							 1 << DRIVER_MODE_HOST)
+
+#define HEAP_SPEC_IDX_GPU_PRIVATE (0U)
+#define HEAP_SPEC_IDX_GPU_LOCAL   (1U)
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(FPGA) && !defined(SUPPORT_SECURITY_VALIDATION)
+static PVRSRV_ERROR TEE_LoadFirmwareWrapper(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams);
+extern PVRSRV_ERROR TEE_LoadFirmware(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams);
+extern PVRSRV_ERROR TEE_SetPowerParams(IMG_HANDLE hSysData, PVRSRV_TD_POWER_PARAMS *psTDPowerParams);
+extern PVRSRV_ERROR TEE_RGXStart(IMG_HANDLE hSysData);
+extern PVRSRV_ERROR TEE_RGXStop(IMG_HANDLE hSysData);
+#endif
+
+/*
+	 -----------------------------------------------------------------------
+	|                      Phys Heap definition matrix                      |
+	 -----------------------------------------------------------------------
+	|   Phys    |    GPU    |    GPU    |    Fw     |    Fw     |    Fw     |
+	|   Heap    |  Private  |   Secure  |  Private  |  Shared   |  PageTab  |
+	 -----------------------------------------------------------------------
+	|  VzMode   |   N H G   |   N H G   |   N H G   |   N H G   |   N H G   |
+	 -----------------------------------------------------------------------
+	|  TEE      |   1 1 1   |   1 1 1   |   1 1 0   |   1 1 1   |   1 1 0   |
+	|  PREMAP   |   1 1 1   |   0 0 0   |   0 0 0   |   1 1 1   |   1 1 0   |
+	|  DEFAULT  |   1 1 1   |   0 0 0   |   0 0 0   |   0 0 1   |   0 0 0   |
+	 -----------------------------------------------------------------------
+*/
+static const CARD_PHYS_HEAP_CONFIG_SPEC gasCardHeapTemplate[] =
+{
+	{
+	 PHYS_HEAP_USAGE_GPU_PRIVATE,
+	 0,					/* determined at runtime by apphints */
+	 false,				/* determined at runtime by apphints */
+	 PHYSHEAP_ALL_DRIVERS
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_LOCAL,
+	 0,					/* determined at runtime */
+	 true,
+	 PHYSHEAP_ALL_DRIVERS
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_SECURE,
+	 SECURE_MEM_SIZE,
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	 true,
+#else
+	 false,
+#endif
+	 PHYSHEAP_ALL_DRIVERS
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PREMAP_PT,
+	 RGX_FIRMWARE_MAX_PAGETABLE_SIZE,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	 true,
+#else
+	 false,
+#endif
+	 PHYSHEAP_NO_GUESTS
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PRIVATE,
+	 SECURE_FW_MEM_SIZE,
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	 true,
+#else
+	 false,
+#endif
+	 PHYSHEAP_NO_GUESTS
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_SHARED,
+	 RGX_FIRMWARE_RAW_HEAP_SIZE,
+	 true,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	 PHYSHEAP_ALL_DRIVERS
+#else
+	 PHYSHEAP_ONLY_VZ
+#endif
+	}
+};
+
+#define ODIN_MEMORY_HYBRID_DEVICE_BASE 0x400000000
+
+#define VALI_MEMORY_DEVICE_BASE (0x800000000U)
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_DEVICE_CONFIG *apsDevCfgs[PVRSRV_MAX_DEVICES] = {0};
+#endif
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+
+/* Fake DVFS configuration used purely for testing purposes */
+
+static const IMG_OPP asOPPTable[] =
+{
+	{ 8,  25000000},
+	{ 16, 50000000},
+	{ 32, 75000000},
+	{ 64, 100000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+static void SetFrequency(IMG_HANDLE hSysData, IMG_UINT32 ui32Frequency)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetFrequency %u", ui32Frequency));
+}
+
+static void SetVoltage(IMG_HANDLE hSysData, IMG_UINT32 ui32Voltage)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetVoltage %u", ui32Voltage));
+}
+
+#endif
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCLocalCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCLocalDevPAddrToCpuPAddr,
+};
+
+static void TCValiLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCValiLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsValiLocalPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCValiLocalCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCValiLocalDevPAddrToCpuPAddr,
+};
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHostCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHostDevPAddrToCpuPAddr,
+};
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHybridPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHybridCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHybridDevPAddrToCpuPAddr
+};
+
+typedef struct _SYS_DATA_ SYS_DATA;
+
+struct _SYS_DATA_
+{
+	IMG_UINT32 ui32SysDataSize;
+	IMG_UINT64 ui64GpuRegisterBase;
+	IMG_UINT64 ui64FwHeapCpuBase;
+	IMG_UINT64 ui64FwHeapGpuBase;
+	IMG_UINT64 ui64FwPrivateHeapSize;
+	IMG_UINT64 ui64FwTotalHeapSize;
+	IMG_UINT64 ui64FwHeapStride;
+	IMG_UINT64 ui64FwPageTableHeapCpuBase;
+	IMG_UINT64 ui64FwPageTableHeapGpuBase;
+	IMG_UINT64 ui64FwPageTableHeapSize;
+
+	PVRSRV_DEVICE_FEATURE_CONFIG sDevFeatureCfg;
+
+	struct platform_device *pdev;
+
+	struct tc_rogue_platform_data *pdata;
+
+	struct resource *registers;
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_client *ion_client;
+	struct ion_handle *ion_rogue_allocation;
+#endif
+};
+
+#define SYSTEM_INFO_FORMAT_STRING "FPGA Revision: %s - TCF Core Revision: %s - TCF Core Target Build ID: %s - PCI Version: %s - Macro Version: %s"
+#define FPGA_REV_MAX_LEN      8 /* current longest format: "x.y.z" */
+#define TCF_CORE_REV_MAX_LEN  8 /* current longest format: "x.y.z" */
+#define TCF_CORE_CFG_MAX_LEN  4 /* current longest format: "x" */
+#define PCI_VERSION_MAX_LEN   4 /* current longest format: "x" */
+#define MACRO_VERSION_MAX_LEN 8 /* current longest format: "x.yz" */
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	int err;
+	char str_fpga_rev[FPGA_REV_MAX_LEN]={0};
+	char str_tcf_core_rev[TCF_CORE_REV_MAX_LEN]={0};
+	char str_tcf_core_target_build_id[TCF_CORE_CFG_MAX_LEN]={0};
+	char str_pci_ver[PCI_VERSION_MAX_LEN]={0};
+	char str_macro_ver[MACRO_VERSION_MAX_LEN]={0};
+
+	IMG_CHAR *pszVersion;
+	IMG_UINT32 ui32StringLength;
+
+	err = tc_sys_strings(psSysData->pdev->dev.parent,
+							 str_fpga_rev, sizeof(str_fpga_rev),
+							 str_tcf_core_rev, sizeof(str_tcf_core_rev),
+							 str_tcf_core_target_build_id, sizeof(str_tcf_core_target_build_id),
+							 str_pci_ver, sizeof(str_pci_ver),
+							 str_macro_ver, sizeof(str_macro_ver));
+	if (err)
+	{
+		return NULL;
+	}
+
+	/* Calculate how much space we need to allocate for the string */
+	ui32StringLength = OSStringLength(SYSTEM_INFO_FORMAT_STRING);
+	ui32StringLength += OSStringLength(str_fpga_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_target_build_id);
+	ui32StringLength += OSStringLength(str_pci_ver);
+	ui32StringLength += OSStringLength(str_macro_ver);
+
+	/* Create the version string */
+	pszVersion = OSAllocMem(ui32StringLength * sizeof(IMG_CHAR));
+	if (pszVersion)
+	{
+		OSSNPrintf(&pszVersion[0], ui32StringLength,
+				   SYSTEM_INFO_FORMAT_STRING,
+				   str_fpga_rev,
+				   str_tcf_core_rev,
+				   str_tcf_core_target_build_id,
+				   str_pci_ver,
+				   str_macro_ver);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to create format string", __func__));
+	}
+
+	return pszVersion;
+}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+static SYS_DATA *gpsIonPrivateData;
+
+PVRSRV_ERROR IonInit(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYS_DATA *psSysData = pvPrivateData;
+	gpsIonPrivateData = psSysData;
+
+	psSysData->ion_client = ion_client_create(psSysData->pdata->ion_device, SYS_RGX_DEV_NAME);
+	if (IS_ERR(psSysData->ion_client))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create ION client (%ld)", __func__, PTR_ERR(psSysData->ion_client)));
+		eError = PVRSRV_ERROR_ION_NO_CLIENT;
+		goto err_out;
+	}
+	/* Allocate the whole rogue ion heap and pass that to services to manage */
+	psSysData->ion_rogue_allocation = ion_alloc(psSysData->ion_client, psSysData->pdata->rogue_heap_memory_size, 4096, (1 << psSysData->pdata->ion_heap_id), 0);
+	if (IS_ERR(psSysData->ion_rogue_allocation))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate ION rogue buffer (%ld)", __func__, PTR_ERR(psSysData->ion_rogue_allocation)));
+		eError = PVRSRV_ERROR_ION_FAILED_TO_ALLOC;
+		goto err_destroy_client;
+
+	}
+
+	return PVRSRV_OK;
+err_destroy_client:
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+err_out:
+	return eError;
+}
+
+void IonDeinit(void)
+{
+	SYS_DATA *psSysData = gpsIonPrivateData;
+	ion_free(psSysData->ion_client, psSysData->ion_rogue_allocation);
+	psSysData->ion_rogue_allocation = NULL;
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+}
+
+struct ion_device *IonDevAcquire(void)
+{
+	return gpsIonPrivateData->pdata->ion_device;
+}
+
+void IonDevRelease(struct ion_device *ion_device)
+{
+	PVR_ASSERT(ion_device == gpsIonPrivateData->pdata->ion_device);
+}
+#endif /* defined(SUPPORT_ION) */
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+			psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+			psDevPAddr[ui32Idx].uiAddr + psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCValiLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+		    psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base + VALI_MEMORY_DEVICE_BASE;
+	}
+}
+
+static void TCValiLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                          IMG_UINT32 ui32NumOfAddr,
+                                          IMG_CPU_PHYADDR *psCpuPAddr,
+                                          IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+		    psDevPAddr[ui32Idx].uiAddr - VALI_MEMORY_DEVICE_BASE + psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_ASSERT(sizeof(*psDevPAddr) == sizeof(*psCpuPAddr));
+	OSCachedMemCopy(psDevPAddr, psCpuPAddr, uiNumOfAddr * sizeof(*psDevPAddr));
+}
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_ASSERT(sizeof(*psCpuPAddr) == sizeof(*psDevPAddr));
+	OSCachedMemCopy(psCpuPAddr, psDevPAddr, uiNumOfAddr * sizeof(*psCpuPAddr));
+}
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+		    (psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base) +
+		    ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+}
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+		    (psDevPAddr[ui32Idx].uiAddr - ODIN_MEMORY_HYBRID_DEVICE_BASE) +
+		    psSysData->pdata->tc_memory_base;
+	}
+}
+
+static inline
+IMG_CHAR* GetHeapName(PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_LOCAL))    return "lma_gpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_SECURE))   return "lma_gpu_secure";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_PRIVATE))  return "lma_gpu_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PRIVATE))   return "lma_fw_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_SHARED))    return "lma_fw_shared";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PREMAP_PT)) return "lma_fw_pagetables";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_CPU_LOCAL))    return "lma_cpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_DISPLAY))      return "lma_gpu_display";
+	else                                                     return "Unexpected Heap";
+}
+
+static PVRSRV_ERROR
+InitLocalHeap(PHYS_HEAP_CONFIG *psPhysHeap,
+			  IMG_UINT64 uiBaseAddr, IMG_UINT64 uiStartAddr,
+			  IMG_UINT64 uiSize, PHYS_HEAP_FUNCTIONS *psFuncs,
+			  PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = ui32Flags;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = psFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = GetHeapName(ui32Flags);
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(uiStartAddr);
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = uiBaseAddr;
+	psPhysHeap->uConfig.sLMA.uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+CreateCardGPUHeaps(SYS_DATA *psSysData,
+				   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+				   PHYS_HEAP_CONFIG *pasPhysHeaps,
+				   PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				   IMG_UINT32 *pui32HeapIdx,
+				   IMG_UINT64 ui64CardAddr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->rogue_heap_memory_base;
+	IMG_UINT32 ui32SpecIdx;
+
+	for (ui32SpecIdx = 0; ui32SpecIdx < ARRAY_SIZE(gasCardHeapTemplate); ui32SpecIdx++)
+	{
+		if (pasCardHeapSpec[ui32SpecIdx].bUsed)
+		{
+			IMG_UINT64 ui64HeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+
+			eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+								   ui64CardAddr,
+								   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+								   ui64HeapSize,
+								   psHeapFuncs,
+								   pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+			/* save the heap details in a structure passed to the tee_ddk module */
+			if (BITMASK_HAS(pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags, PHYS_HEAP_USAGE_FW_PREMAP_PT))
+			{
+				psSysData->ui64FwPageTableHeapCpuBase = ui64StartAddr;
+				psSysData->ui64FwPageTableHeapGpuBase = ui64CardAddr;
+				psSysData->ui64FwPageTableHeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+			}
+			else if (BITMASK_HAS(pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags, PHYS_HEAP_USAGE_FW_PRIVATE))
+			{
+				psSysData->ui64FwPrivateHeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+				psSysData->ui64FwHeapCpuBase = ui64StartAddr;
+				psSysData->ui64FwHeapGpuBase = ui64CardAddr;
+			}
+			else if (BITMASK_HAS(pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags, PHYS_HEAP_USAGE_FW_SHARED))
+			{
+				psSysData->ui64FwTotalHeapSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+			}
+#endif
+
+			ui64CardAddr  += ui64HeapSize;
+			ui64StartAddr += ui64HeapSize;
+			(*pui32HeapIdx)++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+CreateCardEXTHeap(const SYS_DATA *psSysData,
+				  PHYS_HEAP_CONFIG *pasPhysHeaps,
+				  PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				  IMG_UINT32 *pui32HeapIdx,
+				  IMG_UINT64 ui64CardBase)
+{
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->pdp_heap_memory_base;
+	IMG_UINT64 ui64Size = psSysData->pdata->pdp_heap_memory_size;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+						   ui64CardBase + psSysData->pdata->rogue_heap_memory_size,
+						   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+						   ui64Size, psHeapFuncs,
+						   PHYS_HEAP_USAGE_EXTERNAL | PHYS_HEAP_USAGE_DISPLAY);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	(*pui32HeapIdx)++;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+InitLocalHeaps(SYS_DATA *psSysData,
+			   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			   PHYS_HEAP_CONFIG *pasPhysHeaps,
+			   IMG_UINT32 *pui32HeapIdx)
+{
+	PHYS_HEAP_FUNCTIONS *psHeapFuncs;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64CardBase;
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHybridPhysHeapFuncs;
+		ui64CardBase = ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+	else if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHostPhysHeapFuncs;
+		ui64CardBase = 0;
+	}
+	else if (psSysData->pdata->baseboard == TC_BASEBOARD_VALI)
+	{
+		psHeapFuncs = &gsValiLocalPhysHeapFuncs;
+		ui64CardBase = VALI_MEMORY_DEVICE_BASE;
+	}
+	else
+	{
+		psHeapFuncs = &gsLocalPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->rogue_heap_memory_base - psSysData->pdata->tc_memory_base;
+	}
+
+	eError = CreateCardGPUHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	eError = CreateCardEXTHeap(psSysData, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+InitHostHeaps(const SYS_DATA *psSysData, PHYS_HEAP_CONFIG *pasPhysHeaps, IMG_UINT32 *pui32HeapIdx)
+{
+	PVR_UNREFERENCED_PARAMETER(psSysData);
+
+	if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		pasPhysHeaps[*pui32HeapIdx].eType = PHYS_HEAP_TYPE_UMA;
+		pasPhysHeaps[*pui32HeapIdx].ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.psMemFuncs = &gsHostPhysHeapFuncs;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszHeapName = "uma_cpu_local";
+
+		(*pui32HeapIdx)++;
+
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Initialising CPU_LOCAL UMA Host PhysHeaps with memory mode: %d",
+		         psSysData->pdata->mem_mode));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsInit(SYS_DATA *psSysData,
+			  CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			  PHYS_HEAP_CONFIG *pasPhysHeaps,
+			  void *pvPrivData, IMG_UINT32 ui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32HeapCounter = 0;
+
+	eError = InitLocalHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	eError = InitHostHeaps(psSysData, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((ui32HeapCounter == ui32NumHeaps),
+							"Number of PhysHeapConfigs set up doesn't match the initial requirement.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	/* Initialise fields that don't change between memory modes.
+	 * Fix up heap IDs. This is needed for multi-testchip systems to
+	 * ensure the heap IDs are unique as this is what Services expects.
+	 */
+	for (i = 0; i < ui32NumHeaps; i++)
+	{
+		switch (pasPhysHeaps[i].eType)
+		{
+		case PHYS_HEAP_TYPE_LMA:
+			pasPhysHeaps[i].uConfig.sLMA.hPrivData = (void*) pvPrivData;
+			break;
+		case PHYS_HEAP_TYPE_UMA:
+			pasPhysHeaps[i].uConfig.sUMA.hPrivData = (void*) pvPrivData;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Invalid PHYS_HEAP_TYPE: %u in %s",
+			                        pasPhysHeaps[i].eType,
+			                        __func__));
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapSetRequirements(const SYS_DATA *psSysData,
+						PVRSRV_DEVICE_CONFIG *psDevConfig,
+						CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+						IMG_UINT32 *pui32CardPhysHeapCfgCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64FreeCardMemory = psSysData->pdata->rogue_heap_memory_size;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_PRIVATE) &&
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_LOCAL),
+		"PhysHeapConfigs not set correctly in the system layer.", PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < ARRAY_SIZE(gasCardHeapTemplate); i++)
+	{
+		if (pasCardHeapSpec[i].bUsed)
+		{
+			if (BITMASK_HAS(pasCardHeapSpec[i].ui32DriverModeMask,
+							BIT(psDevConfig->eDriverMode)))
+			{
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+				IMG_UINT32 ui32UsageFlags = pasCardHeapSpec[i].ui32UsageFlags;
+
+				if (BITMASK_HAS(ui32UsageFlags, PHYS_HEAP_USAGE_FW_SHARED) &&
+					(psDevConfig->eDriverMode != DRIVER_MODE_GUEST))
+				{
+					/* The Firmware private heap in which the TEE loads the Fw binary
+					 * carves some memory out of the Host/Native driver's Fw shared heap.
+					 * Both Fw Private and Fw Shared heaps are premapped by the TEE as
+					 * one contiguous range of RGX_FIRMWARE_RAW_HEAP_SIZE */
+					pasCardHeapSpec[i].uiSize -= SECURE_FW_MEM_SIZE;
+				}
+#endif
+
+				/* Determine the memory requirements of heaps with a fixed size */
+				ui64FreeCardMemory -= pasCardHeapSpec[i].uiSize;
+
+				/* Count card physheap configs used by the system */
+				(*pui32CardPhysHeapCfgCount)++;
+			}
+			else
+			{
+				/* Heap not used in this driver mode */
+				pasCardHeapSpec[i].bUsed = false;
+			}
+		}
+	}
+
+	if (SysRestrictGpuLocalAddPrivateHeap())
+	{
+		IMG_UINT64 ui64GpuSharedMem = SysRestrictGpuLocalPhysheap(ui64FreeCardMemory);
+
+		if (ui64GpuSharedMem == ui64FreeCardMemory)
+		{
+			/* No memory reserved for GPU private use, special heap not needed */
+		}
+		else
+		{
+			/* Set up the GPU private heap */
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed = true;
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].uiSize = ui64FreeCardMemory - ui64GpuSharedMem;
+			ui64FreeCardMemory = ui64GpuSharedMem;
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	/* all remaining memory card memory goes to GPU_LOCAL */
+	pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].uiSize = ui64FreeCardMemory;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsCreate(SYS_DATA *psSysData, PVRSRV_DEVICE_CONFIG *psDevConfig,
+				PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+				IMG_UINT32 *puiPhysHeapCountOut)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps = 0;
+	CARD_PHYS_HEAP_CONFIG_SPEC asCardHeapSpec[ARRAY_SIZE(gasCardHeapTemplate)];
+
+	/* Initialise the local heap specs with the build-time template */
+	memcpy(asCardHeapSpec, gasCardHeapTemplate, sizeof(gasCardHeapTemplate));
+
+	eError = PhysHeapSetRequirements(psSysData, psDevConfig, asCardHeapSpec, &ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psDevConfig->bHasNonMappableLocalMemory = asCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed;
+
+	if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* CPU_LOCAL heap also required */
+		ui32NumHeaps++;
+	}
+
+	/* DISPLAY heap is always present */
+	ui32NumHeaps++;
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * ui32NumHeaps);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = PhysHeapsInit(psSysData, asCardHeapSpec, pasPhysHeaps, psDevConfig, ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = ui32NumHeaps;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	IMG_UINT32 ui32DeviceID;
+
+	for (ui32DeviceID=0; ui32DeviceID < PVRSRV_MAX_DEVICES; ui32DeviceID++)
+	{
+		PVRSRV_DEVICE_CONFIG *psDC = apsDevCfgs[ui32DeviceID];
+		if (psDC == psDevConfig)
+		{
+			apsDevCfgs[ui32DeviceID] = NULL;
+		}
+	}
+#endif
+
+	if (psDevConfig->pszVersion)
+	{
+		OSFreeMem(psDevConfig->pszVersion);
+	}
+
+	OSFreeMem(psDevConfig->pasPhysHeaps);
+
+	OSFreeMem(psDevConfig);
+}
+
+static void odinTCDevPhysAddr2DmaAddr(PVRSRV_DEVICE_CONFIG *psDevConfig,
+									  IMG_DMA_ADDR *psDmaAddr,
+									  IMG_DEV_PHYADDR *psDevPAddr,
+									  IMG_BOOL *pbValid,
+									  IMG_UINT32 ui32NumAddr,
+									  IMG_BOOL bSparseAlloc)
+{
+	IMG_CPU_PHYADDR sCpuPAddr = {0};
+	IMG_UINT32 ui32Idx;
+
+	/* Fast path */
+	if (!bSparseAlloc)
+	{
+		/* In Odin, DMA address space is the same as host CPU */
+		TCLocalDevPAddrToCpuPAddr(psDevConfig,
+								  1,
+								  &sCpuPAddr,
+								  psDevPAddr);
+		psDmaAddr->uiAddr = sCpuPAddr.uiAddr;
+	}
+	else
+	{
+		for (ui32Idx = 0; ui32Idx < ui32NumAddr; ui32Idx++)
+		{
+			if (pbValid[ui32Idx])
+			{
+				TCLocalDevPAddrToCpuPAddr(psDevConfig,
+										  1,
+										  &sCpuPAddr,
+										  &psDevPAddr[ui32Idx]);
+				psDmaAddr[ui32Idx].uiAddr = sCpuPAddr.uiAddr;
+			}
+			else
+			{
+				/* Invalid DMA address marker */
+				psDmaAddr[ui32Idx].uiAddr = ~((IMG_UINT64)0x0);
+			}
+		}
+	}
+}
+
+static void * odinTCgetCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig, char *name)
+{
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	return tc_dma_chan(psDev->parent, name);
+}
+
+static void odinTCFreeCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig,
+							   void* channel)
+{
+
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	struct dma_chan *chan = (struct dma_chan*) channel;
+
+	tc_dma_chan_free(psDev->parent, chan);
+}
+
+static void GetDriverMode(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DeviceID;
+
+	/*
+	 * Drivers with virtualization support should check if the mode in which the
+	 * driver must control a device has been explicitly specified at load time
+	 * through module parameters.
+	 * Multi-device platforms must find the internal ID of the device currently
+	 * being created when checking for its associated DriverMode parameter.
+	 */
+	if (PVRSRVAcquireInternalID(&ui32DeviceID) != PVRSRV_OK)
+	{
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		return;
+	}
+
+	if (psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID] == DRIVER_MODE_DEFAULT)
+	{
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		void __iomem *pvRegBase;
+
+		pvRegBase = (void __iomem *) OSMapPhysToLin(psDevConfig->sRegsCpuPBase, psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (pvRegBase == NULL)
+		{
+			/* failed to map register bank, default to native mode */
+			psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		}
+		else
+		{
+			IMG_UINT64 ui64ClkCtrl;
+
+			/* the CLK_CTRL register is valid only in the Os 0 (Host) register bank
+			 * if it reads 0 then we can conclude this Os is set up to run as Guest */
+#if defined(RGX_CR_CLK_CTRL)
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL);
+#else
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL1);
+#endif
+			OSUnMapPhysToLin((void __force *) pvRegBase, psDevConfig->ui32RegsSize);
+
+			psDevConfig->eDriverMode = (ui64ClkCtrl != 0) ? (DRIVER_MODE_HOST) : (DRIVER_MODE_GUEST);
+		}
+#else
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+#endif
+	}
+	else
+	{
+		psDevConfig->eDriverMode = psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID];
+	}
+}
+
+static PVRSRV_ERROR PrepareFWImage(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams);
+
+static PVRSRV_ERROR PrepareFWImage(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams)
+{
+	if ((psTDFWParams != NULL) &&
+		(psTDFWParams->pvFirmware != 0) &&
+		(psTDFWParams->ui32FirmwareSize > 0))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware image size = %u;",
+						__func__, psTDFWParams->ui32FirmwareSize));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,  "%s: Invalid Firmware image.", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Firmware binary not signed on this platform */
+	psTDFWParams->pvSignature = NULL;
+	psTDFWParams->ui32SignatureSize = 0;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DeviceConfigCreate(SYS_DATA *psSysData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+							 psSysData->pdata->mem_mode == TC_MEMORY_LOCAL),
+							"Multidevice virtualization setup supported only on Odin device with TC_MEMORY_LOCAL",
+							PVRSRV_ERROR_INVALID_DEVICE);
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = IMG_OFFSET_ADDR(psDevConfig, sizeof(*psDevConfig));
+	psRGXTimingInfo = IMG_OFFSET_ADDR(psRGXData, sizeof(*psRGXData));
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent) /
+											tc_core_clock_multiplex(psSysData->pdev->dev.parent);
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice = &psSysData->pdev->dev;
+	psDevConfig->pszName = "tc";
+	psDevConfig->pszVersion = GetDeviceVersionString(psSysData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr = psSysData->registers->start;
+	psDevConfig->ui32RegsSize = resource_size(psSysData->registers);
+
+	PVRSRVAcquireInternalID(&ui32DeviceID);
+	/*
+	 * Current FPGA images route don't route the GPU's OSID IRQs to their proper card signals.
+	 * All OSID IRQs are ORed together and output on the legacy DUT_IRQ pin.
+	 * To work around this, we pass the legacy DUT IRQ to the Host driver and leave the floating
+	 * OSID IRQs to Guest Drivers. When an IRQ is received via the DUT line, the system layer's
+	 * IRQ handler can query the Firmware's state register to find out which device/OSID emitted
+	 * an interrupt and call the appropriate handler for each device.
+	 */
+	psDevConfig->ui32IRQ = (ui32DeviceID == 0) ? TC_INTERRUPT_EXT : (TC_INTERRUPT_OSID0 + ui32DeviceID);
+
+	GetDriverMode(psDevConfig);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* If there is device running in native mode, prevent any attempts at
+	 * creating any Guest devices, as there will be no Host to support them.
+	 * Currently the VZFPGA supports only one physical GPU. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		IMG_UINT32 i;
+
+		for (i=0; i < PVRSRV_MAX_DEVICES; i++)
+		{
+			PVRSRV_DEVICE_CONFIG *psDC = apsDevCfgs[i];
+			if ((psDC != NULL) && (PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDC)))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s() Device %u is already running in native mode, no other Guests supported in the system.",  __func__, psDC->psDevNode->sDevId.ui32InternalID));
+				eError = PVRSRV_ERROR_INVALID_DEVICE;
+				goto ErrorFreeDevConfig;
+			}
+		}
+	}
+
+	apsDevCfgs[ui32DeviceID] = psDevConfig;
+#endif
+
+	eError = PhysHeapsCreate(psSysData, psDevConfig, &pasPhysHeaps, &uiPhysHeapCount);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	psDevConfig->pasPhysHeaps = pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount = uiPhysHeapCount;
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psDevConfig->eDefaultHeap = SysDefaultToCpuLocalHeap() ?
+		    PVRSRV_PHYS_HEAP_CPU_LOCAL : PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+	else
+	{
+		psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+
+	/* Only required for LMA but having this always set shouldn't be a problem */
+	psDevConfig->bDevicePA0IsValid = IMG_TRUE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psSysData;
+
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/* Fake DVFS configuration used purely for testing purposes */
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32PollMs = 1000;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_FALSE;
+
+	/* DMA channel config */
+	psDevConfig->pfnSlaveDMAGetChan = odinTCgetCDMAChan;
+	psDevConfig->pfnSlaveDMAFreeChan = odinTCFreeCDMAChan;
+	psDevConfig->pfnDevPhysAddr2DmaAddr = odinTCDevPhysAddr2DmaAddr;
+	psDevConfig->pszDmaTxChanName = psSysData->pdata->tc_dma_tx_chan_name;
+	psDevConfig->pszDmaRxChanName = psSysData->pdata->tc_dma_rx_chan_name;
+	psDevConfig->bHasDma = IMG_TRUE;
+	/* Following two values are expressed in number of bytes */
+	psDevConfig->ui32DmaTransferUnit = 1;
+	psDevConfig->ui32DmaAlignment = 1;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(FPGA) && !defined(SUPPORT_SECURITY_VALIDATION)
+	psDevConfig->pfnTDSendFWImage = TEE_LoadFirmwareWrapper;
+	psDevConfig->pfnTDSetPowerParams = TEE_SetPowerParams;
+	psDevConfig->pfnTDRGXStart = TEE_RGXStart;
+	psDevConfig->pfnTDRGXStop = TEE_RGXStop;
+#endif
+	psDevConfig->pfnPrepareFWImage = PrepareFWImage;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psSysData;
+	resource_size_t uiRegistersSize;
+	IMG_UINT32 ui32MinRegBankSize;
+	PVRSRV_ERROR eError;
+	int err = 0;
+
+	PVR_ASSERT(pvOSDevice);
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (psSysData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->pdev = to_platform_device((struct device *)pvOSDevice);
+	psSysData->pdata = psSysData->pdev->dev.platform_data;
+
+	/*
+	 * The device cannot address system memory, so there is no DMA
+	 * limitation.
+	 */
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL)
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(64));
+	}
+	else
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+	}
+
+	err = tc_enable(psSysData->pdev->dev.parent);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to enable PCI device (%d)", __func__, err));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrFreeSysData;
+	}
+
+	psSysData->registers = platform_get_resource_byname(psSysData->pdev,
+														IORESOURCE_MEM,
+														"rogue-regs");
+	if (!psSysData->registers)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get Rogue register information",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		goto ErrorDevDisable;
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	psSysData->ui64FwPrivateHeapSize = SECURE_FW_MEM_SIZE;
+	psSysData->ui64GpuRegisterBase = psSysData->registers->start;
+	psSysData->ui32SysDataSize = sizeof(SYS_DATA);
+
+	{
+		void       *pvAppHintState = NULL;
+		IMG_UINT64 ui64AppHintDefault;
+
+		OSCreateAppHintState(&pvAppHintState);
+		ui64AppHintDefault = PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+		OSGetAppHintUINT64(APPHINT_NO_DEVICE, pvAppHintState, GuestFWHeapStride,
+		                     &ui64AppHintDefault, &psSysData->ui64FwHeapStride);
+		OSFreeAppHintState(pvAppHintState);
+	}
+#else
+	psSysData->ui64FwPrivateHeapSize = 0;
+#endif
+
+	/* Check the address range is large enough. */
+	uiRegistersSize = resource_size(psSysData->registers);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* each GPU instance gets the minimum 64kb register range */
+	ui32MinRegBankSize = RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE;
+#else
+	/* the GPU gets the entire 64MB IO range */
+	ui32MinRegBankSize = SYS_RGX_REG_REGION_SIZE;
+#endif
+
+	if (uiRegistersSize < ui32MinRegBankSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough (was %pa, required 0x%08x)",
+				 __func__, &uiRegistersSize, ui32MinRegBankSize));
+
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorDevDisable;
+	}
+
+	/* Reserve the address range */
+	if (!request_mem_region(psSysData->registers->start,
+							resource_size(psSysData->registers),
+							SYS_RGX_DEV_NAME))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+
+		goto ErrorDevDisable;
+	}
+
+	if (psSysData->pdata->baseboard != TC_BASEBOARD_VALI)
+	{
+		/*
+		 * Reset the device as required.
+		 */
+		eError = DevReset(psSysData, IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't reset device", __func__));
+			goto ErrorDevDisable;
+		}
+	}
+
+	eError = DeviceConfigCreate(psSysData, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseMemRegion;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	eError = IonInit(psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise ION", __func__));
+		goto ErrorDeviceConfigDestroy;
+	}
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+ErrorDeviceConfigDestroy:
+	DeviceConfigDestroy(psDevConfig);
+#endif
+ErrorReleaseMemRegion:
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+ErrorDevDisable:
+	tc_disable(psSysData->pdev->dev.parent);
+ErrFreeSysData:
+	OSFreeMem(psSysData);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevConfig->hSysData;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	IonDeinit();
+#endif
+
+	DeviceConfigDestroy(psDevConfig);
+
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+	tc_disable(psSysData->pdev->dev.parent);
+
+	OSFreeMem(psSysData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+#if defined(TC_APOLLO_TCF5)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	return PVRSRV_OK;
+#else
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	u32 tmp = 0;
+	u32 pll;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	if (tc_sys_info(psSysData->pdev->dev.parent, &tmp, &pll))
+		goto err_out;
+
+	if (tmp > 0)
+		PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", tmp);
+	PVR_DUMPDEBUG_LOG("PLL status: %x", pll);
+
+    eError = FPGA_SysDebugInfo(psSysData->registers,
+                               pfnDumpDebugPrintf,
+                               pvDumpDebugFile);
+    if (eError != PVRSRV_OK)
+    {
+        PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't dump registers from device", __func__));
+        goto err_out;
+    }
+
+err_out:
+	return eError;
+#endif
+}
+
+typedef struct
+{
+	struct device *psDev;
+	int iInterruptID;
+	void *pvData;
+	PFN_LISR pfnLISR;
+} LISR_DATA;
+
+static void TCInterruptHandler(void* pvData)
+{
+#if (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	LISR_DATA *psLISRData = pvData;
+	psLISRData->pfnLISR(psLISRData->pvData);
+#else
+	IMG_UINT32 ui32DeviceID;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	for (ui32DeviceID=0; ui32DeviceID < PVRSRV_MAX_DEVICES; ui32DeviceID++)
+	{
+		PVRSRV_DEVICE_CONFIG *psDC = apsDevCfgs[ui32DeviceID];
+
+		if (psDC != NULL)
+		{
+			PVRSRV_RGXDEV_INFO *psDI = (PVRSRV_RGXDEV_INFO *) psDC->psDevNode->pvDevice;
+
+			if ((psDI != NULL) && (psDI->pvRegsBaseKM != NULL))
+			{
+				IMG_UINT32 ui32Status = OSReadHWReg32(psDI->pvRegsBaseKM, RGX_CR_IRQ_OS0_EVENT_STATUS);
+
+				if (ui32Status)
+				{
+					LISR_DATA *psLISRData = psDI->pvLISRData;
+					psLISRData->pfnLISR(psLISRData->pvData);
+				}
+			}
+		}
+	}
+#endif
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	LISR_DATA *psLISRData;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if ((ui32IRQ != TC_INTERRUPT_EXT) &&
+		((ui32IRQ < TC_INTERRUPT_OSID0) || (ui32IRQ > TC_INTERRUPT_OSID7)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d", __func__, ui32IRQ));
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psLISRData = OSAllocZMem(sizeof(*psLISRData));
+	if (!psLISRData)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+	psLISRData->iInterruptID = ui32IRQ;
+	psLISRData->psDev = psSysData->pdev->dev.parent;
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, TCInterruptHandler, psLISRData);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_free_data;
+	}
+
+	err = tc_enable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_enable_interrupt() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_unset_interrupt_handler;
+	}
+
+	*phLISRData = psLISRData;
+	eError = PVRSRV_OK;
+
+	PVR_TRACE(("Installed device LISR " IMG_PFN_FMTSPEC " to irq %u", pfnLISR, ui32IRQ));
+
+err_out:
+	return eError;
+err_unset_interrupt_handler:
+	tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+err_free_data:
+	OSFreeMem(psLISRData);
+	goto err_out;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *) hLISRData;
+	int err;
+
+	err = tc_disable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_disable_interrupt() failed (%d)", __func__, err));
+	}
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+	}
+
+	PVR_TRACE(("Uninstalled device LISR " IMG_PFN_FMTSPEC " from irq %u", psLISRData->pfnLISR, psLISRData->iInterruptID));
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(FPGA) && !defined(SUPPORT_SECURITY_VALIDATION)
+/* The TEE needs to query the device's hardware feature capabilities and ERNs/BRNs.
+ * On first entry to the TEE, supply the config structure embedded in SYS_DATA */
+static PVRSRV_ERROR TEE_LoadFirmwareWrapper(IMG_HANDLE hSysData, PVRSRV_FW_PARAMS *psTDFWParams)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	psDevInfo = psDeviceNode->pvDevice;
+	psSysData->sDevFeatureCfg = psDevInfo->sDevFeatureCfg;
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return TEE_LoadFirmware(hSysData, psTDFWParams);
+}
+#endif
+
+/****************************************************************************************************/
+/****                                   VM migration test code                                   ****/
+/****************************************************************************************************/
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode);
+static void PreMigrationDeviceSuspend(struct drm_device *psDev);
+static void PostMigrationDeviceResume(struct drm_device *psDev);
+
+void PVRVMMigration(unsigned int src, unsigned int dest);
+EXPORT_SYMBOL(PVRVMMigration);
+
+#define SWAP_REGSBASE_PTR(a, b) do \
+	{ \
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		b = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+	} while (0)
+
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psHostNode = PVRSRVGetDeviceInstance(0);
+	PVRSRV_RGXDEV_INFO *psSrcInfo = psSrcNode->pvDevice;
+	PVRSRV_RGXDEV_INFO *psDestInfo = psDestNode->pvDevice;
+	PVRSRV_DEVICE_CONFIG *psSrcConfig = psSrcNode->psDevConfig;
+	PVRSRV_DEVICE_CONFIG *psDestConfig = psDestNode->psDevConfig;
+	LISR_DATA *psSrcLISRData = (LISR_DATA *) psSrcInfo->pvLISRData;
+	void *pfnLISR = psSrcLISRData->pfnLISR;
+	IMG_UINT32 ui32SrcHyperLane, ui32DestHyperLane;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psHostNode != NULL) &&
+								  (psHostNode->psDevConfig != NULL)),
+								 "Device 0 (expected Host) not initialised.");
+
+	/* Determine the HyperLane ID used by a Guest Device from the Register Bank Base address used */
+	ui32SrcHyperLane = (psSrcConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psSrcConfig->ui32RegsSize;
+	ui32DestHyperLane = (psDestConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psDestConfig->ui32RegsSize;
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Swapping hyperlanes between Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32SrcHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32DestHyperLane));
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resulting configuration:    Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32DestHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32SrcHyperLane));
+
+	/* swap the register bank details */
+	SWAP_REGSBASE_PTR(psSrcInfo->pvRegsBaseKM, psDestInfo->pvRegsBaseKM);
+	SWAP(psSrcConfig->sRegsCpuPBase.uiAddr, psDestConfig->sRegsCpuPBase.uiAddr);
+	/* DevConfig->ui32RegsSize remains the same */
+
+	/* Swap interrupt lines between devices */
+	eError = SysUninstallDeviceLISR(psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+	eError = SysUninstallDeviceLISR(psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	SWAP(psSrcConfig->ui32IRQ, psDestConfig->ui32IRQ);
+
+	eError = SysInstallDeviceLISR(psSrcConfig->hSysData,
+								  psSrcConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psSrcNode,
+								  &psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+
+	eError = SysInstallDeviceLISR(psDestConfig->hSysData,
+								  psDestConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psDestNode,
+								  &psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	/* Swap contents of LMA carveouts between virtual devices */
+	{
+		/* Guest Raw Fw Heap mapping is done using the Host Devices */
+		PHYS_HEAP *psSrcHeap = NULL;
+		PHYS_HEAP *psDestHeap = NULL;
+		IMG_DEV_PHYADDR sSrcHeapBase, sDestHeapBase;
+
+		psSrcHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32SrcHyperLane];
+		psDestHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DestHyperLane];
+
+		PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcHeap != NULL) &&
+									  (psDestHeap != NULL)),
+									 "Guest firmware heaps not premapped by the Host Device.");
+
+		eError = PhysHeapGetDevPAddr(psSrcHeap, &sSrcHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(src fw heap)");
+		eError = PhysHeapGetDevPAddr(psDestHeap, &sDestHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(dest fw heap)");
+
+		eError = PvzServerUnmapDevPhysHeap(ui32SrcHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(src fw heap)");
+		eError = PvzServerUnmapDevPhysHeap(ui32DestHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(dest fw heap)");
+
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32SrcHyperLane]);
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32DestHyperLane]);
+
+		/* create new heaps with new base addresses */
+		eError = PvzServerMapDevPhysHeap(ui32SrcHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sDestHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(src fw heap)");
+		eError = PvzServerMapDevPhysHeap(ui32DestHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sSrcHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(dest fw heap)");
+	}
+}
+
+static void PreMigrationDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(eError == PVRSRV_OK,
+	                           "LinuxBridgeBlockClientsAccess()");
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * be declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+	}
+}
+
+static void PostMigrationDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+									PVRSRV_SYS_POWER_STATE_ON,
+									PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ);
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+}
+
+void PVRVMMigration(unsigned int src, unsigned int dest)
+{
+	PVRSRV_DEVICE_NODE *psSrcNode = PVRSRVGetDeviceInstance(src);
+	PVRSRV_DEVICE_NODE *psDestNode = PVRSRVGetDeviceInstance(dest);
+	struct device *psSrcDev, *psDestDev;
+	struct drm_device *psSrcDrmDev, *psDestDrmDev;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode != NULL) && (psDestNode != NULL) && (psSrcNode != psDestNode)),
+								 "Invalid Device IDs requested for migration.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE) &&
+								  (psDestNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)),
+								 "Devices not fully initialised.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig != NULL) &&
+								  (psDestNode->psDevConfig != NULL)),
+								 "Device config structure is NULL.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig->pvOSDevice != NULL) &&
+								  (psDestNode->psDevConfig->pvOSDevice != NULL)),
+								 "Linux kernel device pointer is NULL.");
+
+	psSrcDev = psSrcNode->psDevConfig->pvOSDevice;
+	psDestDev = psDestNode->psDevConfig->pvOSDevice;
+	psSrcDrmDev = dev_get_drvdata(psSrcDev);
+	psDestDrmDev = dev_get_drvdata(psDestDev);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcDrmDev != NULL) &&
+								  (psDestDrmDev != NULL)),
+								 "Linux kernel drm_device pointer is NULL.");
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psSrcNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psSrcDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psDestNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psDestDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Migrating vGPU resources (regbank, irq, osid)", __func__));
+	SwapHyperlanes(psSrcNode, psDestNode);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psSrcNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psSrcDrmDev);
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psDestNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psDestDrmDev);
+}
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysinfo.h
new file mode 100644
index 000000000000..91b41bc4b66d
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_linux_tc/sysinfo.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM)
+#define MAX_HW_TIME_US                           (240000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (120000)
+#define EVENT_OBJECT_TIMEOUT_US                  (120000000)
+#else
+#define MAX_HW_TIME_US                           (20000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (10000) // (1500)
+
+#if defined(TC_APOLLO_TCF5)
+#define EVENT_OBJECT_TIMEOUT_US                  (2000000)
+#else
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+#endif
+
+#endif
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+
+
+#define SYS_RGX_DEV_NAME "tc_rogue"
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/Kbuild.mk
new file mode 100644
index 000000000000..21e77fc75786
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/Kbuild.mk
@@ -0,0 +1,53 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+PVRSRVKM_NAME = $(PVRSRV_MODNAME)
+
+$(PVRSRVKM_NAME)-y += services/system/volcanic/$(PVR_SYSTEM)/sysconfig.o \
+ services/system/common/sysconfig_cmn.o \
+ services/server/common/vmm_pvz_client.o \
+ services/server/common/vmm_pvz_server.o \
+ services/server/common/vz_vmm_pvz.o \
+ services/server/common/vz_vmm_vm.o \
+ services/system/volcanic/common/vmm_type_$(VMM_TYPE).o
+
+ifeq ($(SUPPORT_ION),1)
+$(PVRSRVKM_NAME)-y += services/system/common/env/linux/ion_support_generic.o
+endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.c
new file mode 100644
index 000000000000..89b8ff3ba0f3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.c
@@ -0,0 +1,352 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "vz_vmm_pvz.h"
+#include "allocmem.h"
+#include "sysinfo.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#include "pvr_debug.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#if defined(__linux__)
+#include <linux/dma-mapping.h>
+#endif
+/*
+ * In systems that support trusted device address protection, there are three
+ * physical heaps from which pages should be allocated:
+ * - one heap for normal allocations
+ * - one heap for allocations holding META code memory
+ * - one heap for allocations holding secured DRM data
+ */
+
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+#include "validation_soc.h"
+#endif
+
+/* Change to test CPU_LOCAL sys layers*/
+#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_GPU_LOCAL
+//#define UMA_HEAP_USAGE_FLAG PHYS_HEAP_USAGE_CPU_LOCAL
+#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_GPU_LOCAL
+//#define UMA_DEFAULT_HEAP PVRSRV_PHYS_HEAP_CPU_LOCAL
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static PHYS_HEAP_FUNCTIONS gsPhysHeapFuncs =
+{
+	/* pfnCpuPAddrToDevPAddr */
+	UMAPhysHeapCpuPAddrToDevPAddr,
+	/* pfnDevPAddrToCpuPAddr */
+	UMAPhysHeapDevPAddrToCpuPAddr,
+};
+
+static PVRSRV_ERROR PhysHeapsCreate(PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+									IMG_UINT32 *puiPhysHeapCountOut)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 ui32NextHeapID = 0;
+	IMG_UINT32 uiHeapCount = 1;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	uiHeapCount += 2;
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	uiHeapCount += 1;
+#endif
+	uiHeapCount += !PVRSRV_VZ_MODE_IS(NATIVE) ? 1:0;
+
+	pasPhysHeaps = OSAllocZMem(sizeof(*pasPhysHeaps) * uiHeapCount);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "SYSMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = UMA_HEAP_USAGE_FLAG;
+	ui32NextHeapID++;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "TDFWMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	ui32NextHeapID++;
+
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "TDSECBUFMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags =
+		PHYS_HEAP_USAGE_GPU_SECURE;
+	ui32NextHeapID++;
+
+#elif defined(SUPPORT_DEDICATED_FW_MEMORY)
+	pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "DEDICATEDFWMEM";
+	pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+	pasPhysHeaps[ui32NextHeapID].ui32UsageFlags =
+		PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+	ui32NextHeapID++;
+#endif
+
+	if (! PVRSRV_VZ_MODE_IS(NATIVE))
+	{
+		pasPhysHeaps[ui32NextHeapID].pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[ui32NextHeapID].eType = PHYS_HEAP_TYPE_UMA;
+		pasPhysHeaps[ui32NextHeapID].psMemFuncs = &gsPhysHeapFuncs;
+		pasPhysHeaps[ui32NextHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_FW_MAIN;
+		ui32NextHeapID++;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = uiHeapCount;
+
+	return PVRSRV_OK;
+}
+
+static void PhysHeapsDestroy(PHYS_HEAP_CONFIG *pasPhysHeaps)
+{
+	OSFreeMem(pasPhysHeaps);
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__)
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+
+	eError = PhysHeapsCreate(&pasPhysHeaps, &uiPhysHeapCount);
+	if (eError)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = RGX_NOHW_CORE_CLOCK_SPEED;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice				= pvOSDevice;
+	psDevConfig->pszName                = "nohw";
+	psDevConfig->pszVersion             = NULL;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr   = 0x00f00baa;
+	psDevConfig->ui32RegsSize           = 0x4000;
+	psDevConfig->ui32IRQ                = 0x00000bad;
+
+	psDevConfig->pasPhysHeaps			= pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount		= uiPhysHeapCount;
+
+	psDevConfig->eDefaultHeap = UMA_DEFAULT_HEAP;
+
+	/* No power management on no HW system */
+	psDevConfig->pfnPrePowerState       = NULL;
+	psDevConfig->pfnPostPowerState      = NULL;
+
+	/* No clock frequency either */
+	psDevConfig->pfnClockFreqGet        = NULL;
+
+	psDevConfig->hDevData               = psRGXData;
+
+	psDevConfig->bDevicePA0IsValid       = IMG_FALSE;
+	psDevConfig->pfnSysDevFeatureDepInit = NULL;
+
+	/* Setup other system specific stuff */
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	/* Pdump validation system registers */
+#if defined(SUPPORT_VALIDATION) && defined(PDUMP)
+	{
+		PVRSRV_DEVICE_NODE* psDeviceNode;
+
+		/* Pretend we are calling for device 0 for now
+		   (sDeviceNode.sDevId.ui32InternalID is 0 above)
+		   until this code can be moved elsewhere.
+		 */
+		psDeviceNode = OSAllocZMem(sizeof(*psDeviceNode));
+		PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "OSAllocZMem");
+
+		PVRSRVConfigureSysCtrl(psDeviceNode, NULL, PDUMP_FLAGS_CONTINUOUS);
+#if defined(SUPPORT_SECURITY_VALIDATION)
+		PVRSRVConfigureTrustedDevice(psDeviceNode, NULL, PDUMP_FLAGS_CONTINUOUS);
+#endif
+		OSFreeMem(psDeviceNode);
+	}
+#endif
+
+	psDevConfig->bHasFBCDCVersion31 = IMG_TRUE;
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	PhysHeapsDestroy(psDevConfig->pasPhysHeaps);
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	PVR_UNREFERENCED_PARAMETER(ui32IRQ);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(pfnLISR);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(phLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hLISRData);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.h
new file mode 100644
index 000000000000..acbadc22e6a4
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysconfig.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(SYSCCONFIG_H)
+#define SYSCCONFIG_H
+
+
+#define RGX_NOHW_CORE_CLOCK_SPEED 100000000
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (0)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* SYSCCONFIG_H */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysinfo.h
new file mode 100644
index 000000000000..566fc4bb7256
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/volcanic/rgx_nohw/sysinfo.h
@@ -0,0 +1,57 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSINFO_H)
+#define SYSINFO_H
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME    "rgxnohw"
+#endif
+
+#endif	/* !defined(SYSINFO_H) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/Kbuild.mk b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/Kbuild.mk
new file mode 100644
index 000000000000..fe58742c81a3
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/Kbuild.mk
@@ -0,0 +1,42 @@
+########################################################################### ###
+#@File
+#@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+#@License       Dual MIT/GPLv2
+#
+# The contents of this file are subject to the MIT license as set out below.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 2 ("GPL") in which case the provisions
+# of GPL are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# GPL, and not to allow others to use your version of this file under the terms
+# of the MIT license, indicate your decision by deleting the provisions above
+# and replace them with the notice and other provisions required by GPL as set
+# out in the file called "GPL-COPYING" included in this distribution. If you do
+# not delete the provisions above, a recipient may use your version of this file
+# under the terms of either the MIT license or GPL.
+#
+# This License is also included in this distribution in the file called
+# "MIT-COPYING".
+#
+# EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+# PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+### ###########################################################################
+
+$(PVRSRV_MODNAME)-y += services/system/$(PVR_SYSTEM)/sysconfig.o
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.c b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.c
new file mode 100644
index 000000000000..6b65db8c40fd
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.c
@@ -0,0 +1,664 @@
+/*************************************************************************/ /*!
+@File           sysconfig.c
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+
+#include "img_defs.h"
+#include "physheap.h"
+#include "pvrsrv.h"
+#include "rgxdevice.h"
+#include "interrupt_support.h"
+#include "sysconfig.h"
+
+#define VZ_EXAMPLE_SYSTEM_NAME "vz_example"
+
+static void SysCpuPAToDevPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_DEV_PHYADDR *psDevPA, IMG_CPU_PHYADDR *psCpuPA);
+static void SysDevPAToCpuPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_CPU_PHYADDR *psCpuPA, IMG_DEV_PHYADDR *psDevPA);
+
+typedef struct _SYS_DATA_
+{
+	IMG_HANDLE hSysLISRData;
+	PFN_LISR pfnDeviceLISR;
+	void *pvDeviceLISRData;
+	PVRSRV_DRIVER_MODE eDriverMode;
+} SYS_DATA;
+
+typedef enum _PHYS_HEAP_IDX_
+{
+	PHYS_HEAP_IDX_SYSMEM,
+	PHYS_HEAP_IDX_FIRMWARE,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP_IDX_FW_PT,
+#endif
+	PHYS_HEAP_IDX_COUNT,
+} PHYS_HEAP_IDX;
+
+static PHYS_HEAP_FUNCTIONS      gsPhysHeapFuncs = {
+	SysCpuPAToDevPA,        /* pfnCpuPAddrToDevPAddr */
+	SysDevPAToCpuPA,        /* pfnDevPAddrToCpuPAddr */
+};
+
+/*
+ * CPU Physical to Device Physical address translation:
+ * Template implementation below assumes CPU and GPU views of physical memory are identical
+ */
+static void SysCpuPAToDevPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_DEV_PHYADDR *psDevPA, IMG_CPU_PHYADDR *psCpuPA)
+{
+	/* Optimise common case */
+	psDevPA[0].uiAddr = psCpuPA[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPA[ui32Idx].uiAddr = psCpuPA[ui32Idx].uiAddr;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+}
+
+/*
+ * Device Physical to CPU Physical address translation:
+ * Template implementation below assumes CPU and GPU views of physical memory are identical
+ */
+static void SysDevPAToCpuPA(IMG_HANDLE hPrivData, IMG_UINT32 ui32NumOfAddr,
+                            IMG_CPU_PHYADDR *psCpuPA, IMG_DEV_PHYADDR *psDevPA)
+{
+	/* Optimise common case */
+	psCpuPA[0].uiAddr = psDevPA[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPA[ui32Idx].uiAddr = psDevPA[ui32Idx].uiAddr;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+}
+
+static IMG_BOOL SystemISRHandler(void *pvData)
+{
+	SYS_DATA *psSysData = pvData;
+	IMG_BOOL bHandled;
+
+	/* Any special system interrupt handling goes here */
+
+#if defined(RGX_SINGLE_IRQ_WORKAROUND)
+	if (psSysData->eDriverMode == DRIVER_MODE_HOST)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = psSysData->pvDeviceLISRData;
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		IMG_UINT32 ui32DriverID;
+
+		/* When hardware support for multiple IRQs is not implemented, the virtualized GPU
+		 * cannot notify drivers of events using dedicated interrupt lines. The single line
+		 * available is routed to Host driver, which in turn broadcasts the IRQ to all other
+		 * active Drivers currently connected to the Firmware. */
+		FOREACH_ACTIVE_DRIVER(psDevInfo, ui32DriverID)
+		{
+			/* the customer must implement a method which allows the Host driver to trigger
+			 * an interrupt signal into each operating system running a Guest GPU driver.
+			 * This method is platform specific. */
+
+			/* INJECT_IRQ_INTO_VM(ui32DriverID); */
+		}
+		END_FOREACH_ACTIVE_DRIVER
+	}
+#endif
+
+	bHandled = psSysData->pfnDeviceLISR(psSysData->pvDeviceLISRData);
+	return bHandled;
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+                                  IMG_UINT32 ui32IRQ,
+                                  const IMG_CHAR *pszName,
+                                  PFN_LISR pfnLISR,
+                                  void *pvData,
+                                  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	PVRSRV_ERROR eError;
+
+	if (psSysData->hSysLISRData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ISR for %s already installed!", __func__, pszName));
+		return PVRSRV_ERROR_CANT_REGISTER_CALLBACK;
+	}
+
+	/* Wrap the device LISR */
+	psSysData->pfnDeviceLISR = pfnLISR;
+	psSysData->pvDeviceLISRData = pvData;
+
+	eError = OSInstallSystemLISR(&psSysData->hSysLISRData, ui32IRQ, pszName,
+	                             SystemISRHandler, psSysData,
+	                             SYS_IRQ_FLAG_TRIGGER_DEFAULT);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	*phLISRData = psSysData;
+
+	PVR_LOG(("Installed device LISR %s on IRQ %d", pszName, ui32IRQ));
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hLISRData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSysData);
+
+	eError = OSUninstallSystemLISR(psSysData->hSysLISRData);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* clear interrupt data */
+	psSysData->pfnDeviceLISR    = NULL;
+	psSysData->pvDeviceLISRData = NULL;
+	psSysData->hSysLISRData     = NULL;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR SysPrePower(IMG_HANDLE hSysData,
+                                PVRSRV_SYS_POWER_STATE eNewPowerState,
+                                PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                                PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+
+	if (psSysData->eDriverMode == DRIVER_MODE_GUEST)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", PVRSRV_OK);
+
+	PVR_UNREFERENCED_PARAMETER(ePwrFlags);
+
+	/* on powering down */
+	if (eNewPowerState != PVRSRV_SYS_POWER_STATE_ON)
+	{
+		IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+
+		void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+		                                 SOC_REGBANK_SIZE,
+		                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		OSWriteHWReg32(pvSocRegs, POW_DOMAIN_DISABLE_REG, POW_DOMAIN_GPU);
+		OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR SysPostPower(IMG_HANDLE hSysData,
+                                 PVRSRV_SYS_POWER_STATE eNewPowerState,
+                                 PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                                 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+
+	if (psSysData->eDriverMode == DRIVER_MODE_GUEST)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", PVRSRV_OK);
+
+	PVR_UNREFERENCED_PARAMETER(ePwrFlags);
+
+	/* on powering up */
+	if (eCurrentPowerState != PVRSRV_SYS_POWER_STATE_ON)
+	{
+		IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+
+		void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+		                                 SOC_REGBANK_SIZE,
+		                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		OSWriteHWReg32(pvSocRegs, POW_DOMAIN_ENABLE_REG, POW_DOMAIN_GPU);
+		OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_SYS_POWER_STATE RGXGpuDomainPower(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+
+	void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+	                                 SOC_REGBANK_SIZE,
+	                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	IMG_UINT32 ui32SocDomainPower = OSReadHWReg32(pvSocRegs, POW_DOMAIN_STATUS_REG);
+
+	bool bGpuDomainIsPowered = BITMASK_HAS(ui32SocDomainPower, POW_DOMAIN_GPU);
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+
+	return (bGpuDomainIsPowered) ? PVRSRV_SYS_POWER_STATE_ON : PVRSRV_SYS_POWER_STATE_OFF;
+}
+
+static void SysDevFeatureDepInit(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT64 ui64Features)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Features);
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+}
+
+static PVRSRV_DRIVER_MODE GetDriverMode(struct platform_device *psDev)
+{
+	PVRSRV_DRIVER_MODE eDriverMode;
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (of_property_read_u32(psDev->dev.of_node, "vz-mode", (IMG_UINT32*) &eDriverMode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Cannot retrieve driver mode from Device Tree. "
+								  "Default to native mode.", __func__));
+		eDriverMode = DRIVER_MODE_NATIVE;
+	}
+#else
+	eDriverMode = DRIVER_MODE_NATIVE;
+#endif
+
+	return eDriverMode;
+}
+
+static PVRSRV_ERROR DeviceConfigCreate(void *pvOSDevice,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct platform_device *psDev;
+	struct resource *dev_res = NULL;
+	int dev_irq;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	SYS_DATA *psSysData;
+	PHYS_HEAP_CONFIG *psPhysHeapConfig;
+	IMG_UINT32 ui32DeviceID;
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo) +
+							  sizeof(*psSysData) +
+							  sizeof(*psPhysHeapConfig) * PHYS_HEAP_IDX_COUNT);
+
+	PVR_LOG_GOTO_IF_NOMEM(psDevConfig, eError, err_sys);
+
+	psRGXData = (RGX_DATA *)((IMG_CHAR *)psDevConfig + sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *)((IMG_CHAR *)psRGXData + sizeof(*psRGXData));
+	psSysData = (SYS_DATA *)((IMG_CHAR *)psRGXTimingInfo + sizeof(*psRGXTimingInfo));
+	psPhysHeapConfig = (PHYS_HEAP_CONFIG *)((IMG_CHAR *)psSysData + sizeof(*psSysData));
+
+	psDev = to_platform_device((struct device *)pvOSDevice);
+
+	dma_set_mask(pvOSDevice, DMA_BIT_MASK(40));
+
+	/*
+	 * Drivers with virtualization support should check if the mode in which the
+	 * driver must control a device has been explicitly specified at load time
+	 * through module parameters.
+	 * Multi-device platforms must find the internal ID of the device currently
+	 * being created when checking for its associated DriverMode parameter.
+	 */
+	eError = PVRSRVAcquireInternalID(&ui32DeviceID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireInternalID", err_sys);
+
+	if (psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID] == DRIVER_MODE_DEFAULT)
+	{
+		psDevConfig->eDriverMode = GetDriverMode(psDev);
+	}
+	else
+	{
+		psDevConfig->eDriverMode = psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID];
+	}
+
+	dev_irq = platform_get_irq(psDev, 0);
+	if (dev_irq < 0)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("platform_get_irq", eError, PVRSRV_ERROR_INVALID_DEVICE, err_sys);
+	}
+
+	dev_res = platform_get_resource(psDev, IORESOURCE_MEM, 0);
+	if (dev_res == NULL)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("platform_get_resource", eError, PVRSRV_ERROR_INVALID_DEVICE, err_sys);
+	}
+
+	/* Device Setup */
+	psDevConfig->pvOSDevice              = pvOSDevice;
+	psDevConfig->pszName                 = VZ_EXAMPLE_SYSTEM_NAME;
+	psDevConfig->pszVersion              = NULL;
+
+	/* Device setup information */
+	psDevConfig->sRegsCpuPBase.uiAddr    = dev_res->start;
+	psDevConfig->ui32IRQ                 = dev_irq;
+
+	psDevConfig->ui32RegsSize            = (unsigned int)(dev_res->end - dev_res->start);
+
+	PVR_ASSERT(psDevConfig->ui32RegsSize >= RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+
+	/* Power management */
+	psDevConfig->pfnPrePowerState        = SysPrePower;
+	psDevConfig->pfnPostPowerState       = SysPostPower;
+	psDevConfig->pfnGpuDomainPower       = PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) ? NULL : RGXGpuDomainPower;
+
+	/* Setup RGX specific timing data */
+	psRGXTimingInfo->ui32CoreClockSpeed        = DEFAULT_CLOCK_RATE;
+	psRGXTimingInfo->bEnableActivePM           = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland        = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms     = 0;
+
+	/* Setup RGX specific data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Minimal configuration */
+	psDevConfig->pfnClockFreqGet         = NULL;
+	psDevConfig->hDevData                = psRGXData;
+	psDevConfig->hSysData                = psSysData;
+	psDevConfig->pfnSysDevFeatureDepInit = SysDevFeatureDepInit;
+	psDevConfig->bHasFBCDCVersion31      = IMG_FALSE;
+
+	psDevConfig->pasPhysHeaps = psPhysHeapConfig;
+
+	return eError;
+
+err_sys:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+/* Obtain the IPA of the carveout range reserved for this VM */
+static IMG_UINT64 GetFwCarveoutBase(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	struct platform_device *psDev = to_platform_device((struct device *)psDevConfig->pvOSDevice);
+	IMG_UINT64 ui64BaseAddress;
+
+	if (of_property_read_u64(psDev->dev.of_node, "fw-carveout", &ui64BaseAddress))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Cannot retrieve firmware carveout address from Device Tree."
+								  " Using default Base Address: 0x%llX",
+								  __func__, FW_CARVEOUT_IPA_BASE));
+		ui64BaseAddress = FW_CARVEOUT_IPA_BASE;
+	}
+
+	return ui64BaseAddress;
+}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+static IMG_UINT64 GetFwPageTableCarveoutBase(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	struct platform_device *psDev = to_platform_device((struct device *)psDevConfig->pvOSDevice);
+	IMG_UINT64 ui64BaseAddress;
+
+	if (of_property_read_u64(psDev->dev.of_node, "fw-pt-carveout", &ui64BaseAddress))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Cannot retrieve firmware page table carveout address"
+								  " from Device Tree. Using default Base Address: 0x%llX",
+								  __func__, FW_PT_CARVEOUT_IPA_BASE));
+		ui64BaseAddress = FW_PT_CARVEOUT_IPA_BASE;
+	}
+
+	return ui64BaseAddress;
+}
+#endif
+#endif
+
+static PVRSRV_ERROR PhysHeapCfgCreate(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_CPU_PHYADDR sCpuBase;
+	IMG_DEV_PHYADDR sDeviceBase;
+	PHYS_HEAP_CONFIG *psPhysHeapConfig = psDevConfig->pasPhysHeaps;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Virtualization driver */
+	/* Heap configuration for general use */
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].eType = PHYS_HEAP_TYPE_UMA;
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.pszHeapName = "uma_gpu_local";
+	psPhysHeapConfig[PHYS_HEAP_IDX_SYSMEM].uConfig.sUMA.hPrivData = NULL;
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	{
+		/*
+		 * In a static memory VZ setup, the size and base addresses of
+		 * all Host and Guest drivers' Firmware heaps are laid out
+		 * consecutively in a physically contiguous memory range known
+		 * in advance by the Host driver.
+		 *
+		 * During the Host driver initialisation, it maps the entire range
+		 * into the Firmware's virtual address space. No other mapping
+		 * operations into the Firmware's VA space are needed after this.
+		 * Guest driver must know only the base address of the range
+		 * assign to it during system planning stage.
+		 *
+		 * The system integrator must ensure that:
+		 *  - physically contiguous RAM region used as a Firmware heap
+		 *    is not managed by any OS or Hypervisor (a carveout)
+		 *  - Host driver must come online before any Guest drivers
+		 *  - Host driver sets up the Firmware before Guests submits work
+		 */
+
+		sCpuBase.uiAddr = GetFwCarveoutBase(psDevConfig);
+		SysCpuPAToDevPA(NULL, 1, &sDeviceBase, &sCpuBase);
+
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.pszPDumpMemspaceName = "SYSMEM_FW";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.psMemFuncs = &gsPhysHeapFuncs;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.pszHeapName = "lma_fw_shared";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.sStartAddr = sCpuBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.sCardBase = sDeviceBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sLMA.hPrivData = NULL;
+	}
+#else
+	/* Dynamic Firmware heap allocation */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		/*
+		 * Guest drivers must provide a physically contiguous memory
+		 * range to the Host via a PVZ call to have it mapped into
+		 * the Firmware's address space. Guest drivers use the OS
+		 * kernel's DMA/CMA allocator to obtain a DMA buffer to be
+		 * used as a firmware heap. This memory will be managed
+		 * internally by the Guest driver after the heap is created.
+		 */
+		DMA_ALLOC *psDmaAlloc = OSAllocZMem(sizeof(DMA_ALLOC));
+
+		eError = (psDmaAlloc == NULL) ? PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_OK;
+		if (eError == PVRSRV_OK)
+		{
+			psDmaAlloc->pvOSDevice = psDevConfig->pvOSDevice;
+			psDmaAlloc->ui64Size = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+			eError = SysDmaAllocMem(psDmaAlloc);
+			if (eError == PVRSRV_OK)
+			{
+				eError = SysDmaRegisterForIoRemapping(psDmaAlloc);
+
+				if (eError == PVRSRV_OK)
+				{
+					sCpuBase.uiAddr = psDmaAlloc->sBusAddr.uiAddr;
+					SysCpuPAToDevPA(NULL, 1, &sDeviceBase, &sCpuBase);
+
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].eType = PHYS_HEAP_TYPE_DMA;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.pszPDumpMemspaceName = "SYSMEM_FW";
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.psMemFuncs = &gsPhysHeapFuncs;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.pszHeapName = "dma_fw_shared";
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.sStartAddr = sCpuBase;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.sCardBase = sDeviceBase;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+					psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.hPrivData = psDmaAlloc;
+				}
+				else
+				{
+					OSFreeMem(psDmaAlloc);
+					SysDmaFreeMem(psDmaAlloc);
+				}
+			}
+			else
+			{
+				OSFreeMem(psDmaAlloc);
+			}
+		}
+	}
+	else
+	{
+		/*
+		 * The Host or Native driver uses memory managed by
+		 * the kernel on a page granularity and creates on-demand
+		 * mappings into the Firmware's address space.
+		 */
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].eType = PHYS_HEAP_TYPE_UMA;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM_FW";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.pszHeapName = "uma_fw_shared";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FIRMWARE].uConfig.sUMA.hPrivData = NULL;
+	}
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		sCpuBase.uiAddr = GetFwPageTableCarveoutBase(psDevConfig);
+		SysCpuPAToDevPA(NULL, 1, &sDeviceBase, &sCpuBase);
+
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].eType = PHYS_HEAP_TYPE_LMA;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].ui32UsageFlags = PHYS_HEAP_USAGE_FW_PREMAP_PT;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.pszPDumpMemspaceName = "SYSMEM_FW_PT";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.psMemFuncs = &gsPhysHeapFuncs;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.pszHeapName = "lma_fw_pt";
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.sStartAddr = sCpuBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.sCardBase = sDeviceBase;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.uiSize = RGX_FIRMWARE_MAX_PAGETABLE_SIZE;
+		psPhysHeapConfig[PHYS_HEAP_IDX_FW_PT].uConfig.sLMA.hPrivData = NULL;
+	}
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+	/* Device's physical heaps */
+	psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	psDevConfig->ui32PhysHeapCount = PHYS_HEAP_IDX_COUNT;
+
+	return eError;
+}
+
+static void PhysHeapCfgDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		DMA_ALLOC *psDmaAlloc = psDevConfig->pasPhysHeaps[PHYS_HEAP_IDX_FIRMWARE].uConfig.sDMA.hPrivData;
+
+		SysDmaDeregisterForIoRemapping(psDmaAlloc);
+		SysDmaFreeMem(psDmaAlloc);
+		OSFreeMem(psDmaAlloc);
+	}
+#endif
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	PVRSRV_ERROR eError;
+
+	eError = DeviceConfigCreate(pvOSDevice, &psDevConfig);
+	if (eError == PVRSRV_OK)
+	{
+		eError = PhysHeapCfgCreate(psDevConfig);
+		if (eError != PVRSRV_OK)
+		{
+			OSFreeMem(psDevConfig);
+		}
+
+		*ppsDevConfig = psDevConfig;
+	}
+
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PhysHeapCfgDestroy(psDevConfig);
+
+	OSFreeMem(psDevConfig);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+						  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						  void *pvDumpDebugFile)
+{
+	/* print here any system information useful for debug dumps */
+
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.h b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.h
new file mode 100644
index 000000000000..0670c7dde015
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysconfig.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File           sysconfig.h
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(__SYSCCONFIG_H__)
+#define __SYSCCONFIG_H__
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS 100
+
+#define DEFAULT_CLOCK_RATE			IMG_UINT64_C(600000000)
+
+/* fixed IPA Base of the memory carveout reserved for the GPU Firmware Heaps */
+#define FW_CARVEOUT_IPA_BASE		IMG_UINT64_C(0x7E000000)
+
+/* fixed IPA Base of the memory carveout reserved for the Firmware's Page Tables */
+#define FW_PT_CARVEOUT_IPA_BASE		IMG_UINT64_C(0x8F000000)
+
+/* mock SoC registers */
+#define SOC_REGBANK_BASE			IMG_UINT64_C(0xF0000000)
+#define SOC_REGBANK_SIZE			IMG_UINT32_C(0x10000)
+#define POW_DOMAIN_ENABLE_REG		IMG_UINT32_C(0xA000)
+#define POW_DOMAIN_DISABLE_REG		IMG_UINT32_C(0xA008)
+#define POW_DOMAIN_STATUS_REG		IMG_UINT32_C(0xA010)
+
+#define POW_DOMAIN_GPU				IMG_UINT32_C(0x1)
+
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif /* __SYSCCONFIG_H__ */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysinfo.h b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysinfo.h
new file mode 100644
index 000000000000..0ab175e0692e
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/sysinfo.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (10000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME                         "powervr-vz-example"
+#endif
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.c b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.c
new file mode 100644
index 000000000000..94a7629c1f51
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.c
@@ -0,0 +1,240 @@
+/*************************************************************************/ /*!
+@File           vz_validation.c
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "sysconfig.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "vz_validation.h"
+#include "virt_validation_defs.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef struct _VALIDATION_CTRL_
+{
+	IMG_BOOL	bUnload;
+	IMG_HANDLE	hMPUWatchdogThread;
+	IMG_HANDLE	hSystemWatchdogEvObj;
+} VALIDATION_CTRL;
+
+static VALIDATION_CTRL gsValidCtrl = {false, NULL, NULL};
+
+static void SysPrintAndResetFaultStatusRegister(void)
+{
+	IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+	IMG_UINT32 ui32MPUEventStatus;
+
+	void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+	                                 SOC_REGBANK_SIZE,
+	                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	ui32MPUEventStatus = OSReadHWReg32(pvSocRegs, MPU_EVENT_STATUS_REG);
+
+	if (ui32MPUEventStatus & MPU_GPU_BUS_REQUESTER)
+	{
+		IMG_UINT32 ui32MPUEventOSID      = OSReadHWReg32(pvSocRegs, MPU_EVENT_OSID_REG);
+		IMG_UINT32 ui32MPUEventAddress   = OSReadHWReg32(pvSocRegs, MPU_EVENT_ADDRESS_REG);
+		IMG_UINT32 ui32MPUEventDirection = OSReadHWReg32(pvSocRegs, MPU_EVENT_DIRECTION_REG);
+
+		PVR_DPF((PVR_DBG_ERROR, "MPU fault event: GPU attempted an illegal %s access to address 0x%08X while emitting OSID %u",
+		                        (ui32MPUEventDirection == MPU_WRITE_ACCESS) ? "WRITE" : "READ",
+		                        ui32MPUEventAddress,
+		                        ui32MPUEventOSID));
+
+		OSWriteHWReg32(pvSocRegs, MPU_EVENT_CLEAR_REG, 1);
+	}
+
+	OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+}
+
+void SysInitValidation(IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                       IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS])
+{
+	IMG_CPU_PHYADDR sSoCRegBase = {SOC_REGBANK_BASE};
+	IMG_UINT32 ui32OSID, ui32Region;
+
+	void* pvSocRegs = OSMapPhysToLin(sSoCRegBase,
+	                                 SOC_REGBANK_SIZE,
+	                                 PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	IMG_UINT64 aui64RegMin[GPUVIRT_VALIDATION_MAX_OS][GPUVIRT_VALIDATION_NUM_REGIONS] =
+	{
+		{MPU_PROTECTED_RANGE0_START_REG, MPU_PROTECTED_RANGE8_START_REG},
+		{MPU_PROTECTED_RANGE1_START_REG, MPU_PROTECTED_RANGE9_START_REG},
+		{MPU_PROTECTED_RANGE2_START_REG, MPU_PROTECTED_RANGE10_START_REG},
+		{MPU_PROTECTED_RANGE3_START_REG, MPU_PROTECTED_RANGE11_START_REG},
+		{MPU_PROTECTED_RANGE4_START_REG, MPU_PROTECTED_RANGE12_START_REG},
+		{MPU_PROTECTED_RANGE5_START_REG, MPU_PROTECTED_RANGE13_START_REG},
+		{MPU_PROTECTED_RANGE6_START_REG, MPU_PROTECTED_RANGE14_START_REG},
+		{MPU_PROTECTED_RANGE7_START_REG, MPU_PROTECTED_RANGE15_START_REG}
+	};
+
+	IMG_UINT64 aui64RegMax[GPUVIRT_VALIDATION_MAX_OS][GPUVIRT_VALIDATION_NUM_REGIONS] =
+	{
+		{MPU_PROTECTED_RANGE0_END_REG, MPU_PROTECTED_RANGE8_END_REG},
+		{MPU_PROTECTED_RANGE1_END_REG, MPU_PROTECTED_RANGE9_END_REG},
+		{MPU_PROTECTED_RANGE2_END_REG, MPU_PROTECTED_RANGE10_END_REG},
+		{MPU_PROTECTED_RANGE3_END_REG, MPU_PROTECTED_RANGE11_END_REG},
+		{MPU_PROTECTED_RANGE4_END_REG, MPU_PROTECTED_RANGE12_END_REG},
+		{MPU_PROTECTED_RANGE5_END_REG, MPU_PROTECTED_RANGE13_END_REG},
+		{MPU_PROTECTED_RANGE6_END_REG, MPU_PROTECTED_RANGE14_END_REG},
+		{MPU_PROTECTED_RANGE7_END_REG, MPU_PROTECTED_RANGE15_END_REG}
+	};
+
+	IMG_UINT64 aui64RangeOSIDAccessReg[] =
+	{
+		MPU_PROTECTED_RANGE0_OSID_REG,
+		MPU_PROTECTED_RANGE1_OSID_REG,
+		MPU_PROTECTED_RANGE2_OSID_REG,
+		MPU_PROTECTED_RANGE3_OSID_REG,
+		MPU_PROTECTED_RANGE4_OSID_REG,
+		MPU_PROTECTED_RANGE5_OSID_REG,
+		MPU_PROTECTED_RANGE6_OSID_REG,
+		MPU_PROTECTED_RANGE7_OSID_REG,
+		MPU_PROTECTED_RANGE8_OSID_REG,
+		MPU_PROTECTED_RANGE9_OSID_REG,
+		MPU_PROTECTED_RANGE10_OSID_REG,
+		MPU_PROTECTED_RANGE11_OSID_REG,
+		MPU_PROTECTED_RANGE12_OSID_REG,
+		MPU_PROTECTED_RANGE13_OSID_REG,
+		MPU_PROTECTED_RANGE14_OSID_REG,
+		MPU_PROTECTED_RANGE15_OSID_REG
+	};
+
+	for (ui32OSID = 0; ui32OSID < GPUVIRT_VALIDATION_NUM_OS ; ui32OSID++)
+	{
+		/* every OSID gets access to 2 ranges: one secure range and one shared with all the other OSIDs
+		 * e.g. OSID 0: secure RANGE 0 & shared RANGE 8 */
+		OSWriteHWReg64(pvSocRegs, aui64RangeOSIDAccessReg[ui32OSID], ui32OSID);
+		OSWriteHWReg64(pvSocRegs, aui64RangeOSIDAccessReg[ui32OSID+GPUVIRT_VALIDATION_MAX_OS], ui32OSID);
+
+		for (ui32Region=0; ui32Region < GPUVIRT_VALIDATION_NUM_REGIONS; ui32Region++)
+		{
+			OSWriteHWReg64(pvSocRegs, aui64RegMin[ui32OSID][ui32Region], aui64OSidMin[ui32Region][ui32OSID]);
+			OSWriteHWReg64(pvSocRegs, aui64RegMax[ui32OSID][ui32Region], aui64OSidMax[ui32Region][ui32OSID]);
+		}
+	}
+
+	OSWriteHWReg32(pvSocRegs, MPU_PROTECTION_ENABLE_REG, 1);
+
+	OSUnMapPhysToLin(pvSocRegs, SOC_REGBANK_SIZE);
+}
+
+static void MPUWatchdogThread(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hOSEvent;
+	IMG_UINT32 ui32Timeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+
+	/* Open an event on the system watchdog event object so we can listen on it
+	   and abort the system watchdog thread. */
+	eError = OSEventObjectOpen(gsValidCtrl.hSystemWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+	while (!gsValidCtrl.bUnload)
+	{
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+		if (eError == PVRSRV_OK)
+		{
+			if (gsValidCtrl.bUnload)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Shutdown event received.", __func__));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Power state change event received.", __func__));
+			}
+		}
+		else if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			/* If timeout do nothing otherwise print warning message. */
+			PVR_DPF((PVR_DBG_ERROR, "%s: "
+					"Error (%d) when waiting for event!", __func__, eError));
+		}
+
+		SysPrintAndResetFaultStatusRegister();
+	}
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+void CreateMPUWatchdogThread(void)
+{
+	/* Create the SysWatchdog Event Object */
+	OSEventObjectCreate("PVRSRV_SYSWDG_EVENTOBJECT", &gsValidCtrl.hSystemWatchdogEvObj);
+
+	/* Create a thread which is used to detect fatal errors */
+	OSThreadCreatePriority(&gsValidCtrl.hMPUWatchdogThread,
+	                       "emu_check_fault_wdg",
+	                       MPUWatchdogThread,
+	                       NULL,
+	                       IMG_TRUE,
+	                       &gsValidCtrl,
+	                       0);
+}
+
+void DestroyMPUWatchdogThread(void)
+{
+	gsValidCtrl.bUnload = IMG_TRUE;
+
+	if (gsValidCtrl.hMPUWatchdogThread)
+	{
+		if (gsValidCtrl.hSystemWatchdogEvObj)
+		{
+			OSEventObjectSignal(gsValidCtrl.hSystemWatchdogEvObj);
+		}
+		LOOP_UNTIL_TIMEOUT(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			OSThreadDestroy(gsValidCtrl.hMPUWatchdogThread);
+			gsValidCtrl.hMPUWatchdogThread = NULL;
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT();
+	}
+
+	if (gsValidCtrl.hSystemWatchdogEvObj)
+	{
+		OSEventObjectDestroy(gsValidCtrl.hSystemWatchdogEvObj);
+		gsValidCtrl.hSystemWatchdogEvObj = NULL;
+	}
+}
+
+#endif
diff --git a/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.h b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.h
new file mode 100644
index 000000000000..f68facb42d04
--- /dev/null
+++ b/drivers/gpu/drm/img/img-volcanic/services/system/vz_example/vz_validation.h
@@ -0,0 +1,55 @@
+/*************************************************************************/ /*!
+@File           vz_validation.h
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__VZ_VALIDATION_H__)
+#define __VZ_VALIDATION_H__
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+void CreateMPUWatchdogThread(void);
+void DestroyMPUWatchdogThread(void);
+
+void SysInitValidation(IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                       IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+#endif
+
+#endif /* __VZ_VALIDATION_H__ */
diff --git a/drivers/gpu/drm/img/kernel_compatibility.h b/drivers/gpu/drm/img/kernel_compatibility.h
new file mode 100644
index 000000000000..048f4449ae68
--- /dev/null
+++ b/drivers/gpu/drm/img/kernel_compatibility.h
@@ -0,0 +1,404 @@
+/*************************************************************************/ /*!
+@Title          Kernel versions compatibility macros
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_COMPATIBILITY_H__
+#define __KERNEL_COMPATIBILITY_H__
+
+#include <linux/version.h>
+#include <linux/compiler.h>
+
+/* Explicitly error out if DDK is built against out-of-support Linux kernel */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0))
+#error Linux kernels older than 4.9.0 are not supported
+#endif
+
+/*
+ * Stop supporting an old kernel? Remove the top block.
+ * New incompatible kernel?       Append a new block at the bottom.
+ *
+ * Please write your version test as `VERSION < X.Y`, and use the earliest
+ * possible version :)
+ *
+ * If including this header file in other files, this should always be the
+ * last file included, as it can affect definitions/declarations in files
+ * included after it.
+ */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+#define refcount_read(r) atomic_read(r)
+#define drm_mm_insert_node(mm, node, size) drm_mm_insert_node(mm, node, size, 0, DRM_MM_SEARCH_DEFAULT)
+
+#define drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd) drm_helper_mode_fill_fb_struct(fb, mode_cmd)
+
+/*
+ * In Linux Kernels >= 4.12 for x86 another level of page tables has been
+ * added. The added level (p4d) sits between pgd and pud, so when it
+ * doesn`t exist, pud_offset function takes pgd as a parameter instead
+ * of p4d.
+ */
+#define p4d_t pgd_t
+#define p4d_offset(pgd, address) (pgd)
+#define p4d_none(p4d) (0)
+#define p4d_bad(p4d) (0)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+
+#define drm_mode_object_get(obj)          drm_mode_object_reference(obj)
+#define drm_mode_object_put(obj)          drm_mode_object_unreference(obj)
+#define drm_connector_get(obj)            drm_connector_reference(obj)
+#define drm_connector_put(obj)            drm_connector_unreference(obj)
+#define drm_framebuffer_get(obj)          drm_framebuffer_reference(obj)
+#define drm_framebuffer_put(obj)          drm_framebuffer_unreference(obj)
+#define drm_gem_object_get(obj)           drm_gem_object_reference(obj)
+#define drm_gem_object_put_locked(obj)    drm_gem_object_unreference(obj)
+#define __drm_gem_object_put(obj)         __drm_gem_object_unreference(obj)
+#define drm_property_blob_get(obj)        drm_property_reference_blob(obj)
+#define drm_property_blob_put(obj)        drm_property_unreference_blob(obj)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+
+#define drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, format_modifiers, type, name, ...) \
+	({ (void) format_modifiers; drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, type, name, ##__VA_ARGS__); })
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0))
+
+#define drm_dev_put(dev) drm_dev_unref(dev)
+
+#define drm_mode_object_find(dev, file_priv, id, type) drm_mode_object_find(dev, id, type)
+#define drm_encoder_find(dev, file_priv, id) drm_encoder_find(dev, id)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_plane_helper_check_state(plane_state, \
+												 &__clip, \
+												 min_scale, max_scale, \
+												 can_position, \
+												 can_update_disabled); \
+		__ret; \
+	})
+
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_atomic_helper_check_plane_state(plane_state, \
+														crtc_state, \
+														&__clip, \
+														min_scale, max_scale, \
+														can_position, \
+														can_update_disabled); \
+		__ret; \
+	})
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+
+#define drm_connector_attach_encoder(connector, encoder) \
+	drm_mode_connector_attach_encoder(connector, encoder)
+
+#define drm_connector_update_edid_property(connector, edid) \
+	drm_mode_connector_update_edid_property(connector, edid)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+
+/*
+ * Work around architectures, e.g. MIPS, that define copy_from_user and
+ * copy_to_user as macros that call access_ok, as this gets redefined below.
+ * As of kernel 4.12, these functions are no longer defined per-architecture
+ * so this work around isn't needed.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#if defined(copy_from_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_from_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+	return copy_from_user(to, from, n);
+}
+
+#undef copy_from_user
+#define copy_from_user(to, from, n) __copy_from_user(to, from, n)
+#endif
+
+#if defined(copy_to_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_to_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+	return copy_to_user(to, from, n);
+}
+
+#undef copy_to_user
+#define copy_to_user(to, from, n) __copy_to_user(to, from, n)
+#endif
+#endif
+
+/*
+ * Linux 5.0 dropped the type argument.
+ *
+ * This is unused in at least Linux 3.4 and above for all architectures other
+ * than 'um' (User Mode Linux), which stopped using it in 4.2.
+ */
+#if defined(access_ok)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around access_ok being defined as a macro.
+  */
+static inline int
+__pvr_access_ok_compat(int type, const void __user * addr, unsigned long size)
+{
+	return access_ok(type, addr, size);
+}
+
+#undef access_ok
+#define access_ok(addr, size) __pvr_access_ok_compat(0, addr, size)
+#else
+#define access_ok(addr, size) access_ok(0, addr, size)
+#endif
+
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+#define MODULE_IMPORT_NS(ns)
+#endif
+
+/*
+ * Before v5.8, the "struct mm" has a semaphore named "mmap_sem" which is
+ * renamed to "mmap_lock" in v5.8. Moreover, new APIs are provided to
+ * access this lock starting from v5.8.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+
+#define mmap_write_lock(mm)   down_write(&mm->mmap_sem)
+#define mmap_write_unlock(mm) up_write(&mm->mmap_sem)
+
+#define mmap_read_lock(mm)    down_read(&mm->mmap_sem)
+#define mmap_read_unlock(mm)  up_read(&mm->mmap_sem)
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#define drm_gem_object_put(obj) drm_gem_object_unreference_unlocked(obj)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+#define drm_gem_object_put(obj) drm_gem_object_put_unlocked(obj)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+
+#define drm_prime_pages_to_sg(dev, pages, nr_pages) \
+	drm_prime_pages_to_sg(pages, nr_pages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+
+struct dma_buf_map {
+	void *vaddr;
+};
+
+#define dma_buf_vmap(dmabuf, map) \
+	({ \
+		(map)->vaddr = dma_buf_vmap(dmabuf); \
+		(map)->vaddr ? 0 : ((dmabuf) && (dmabuf)->ops->vmap) ? -ENOMEM : -EINVAL; \
+	})
+
+#define dma_buf_vunmap(dmabuf, map) \
+	({ \
+		dma_buf_vunmap(dmabuf, (map)->vaddr); \
+		(map)->vaddr = NULL; \
+	})
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0))
+
+#define drm_prime_sg_to_page_array(sgt, pages, npages) \
+	drm_prime_sg_to_page_addr_arrays(sgt, pages, NULL, npages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0))
+
+#define drm_gem_plane_helper_prepare_fb drm_gem_fb_prepare_fb
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0)) */
+
+/*
+ * Linux 5.11 renames the privileged uaccess routines for arm64 and Android
+ * kernel v5.10 merges the change as well. These routines are only used for
+ * arm64 so CONFIG_ARM64 testing can be ignored.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) || \
+	((LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) && !defined(ANDROID))
+#define uaccess_enable_privileged() uaccess_enable()
+#define uaccess_disable_privileged() uaccess_disable()
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0))
+#define pde_data PDE_DATA
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+#define kthread_complete_and_exit(comp, ret) complete_and_exit(comp, ret);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0))
+#define iosys_map dma_buf_map
+#define iosys_map_set_vaddr dma_buf_map_set_vaddr
+#define iosys_map_set_vaddr_iomem dma_buf_map_set_vaddr_iomem
+#define iosys_map_clear dma_buf_map_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+
+#define register_shrinker(shrinker, name) \
+	register_shrinker(shrinker)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0))
+#define DRM_PLANE_NO_SCALING DRM_PLANE_HELPER_NO_SCALING
+#define drm_plane_helper_destroy drm_primary_helper_destroy
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0))
+#define genl_split_ops genl_ops
+#define COMPAT_FB_INFO fbdev
+#define drm_fb_helper_alloc_info drm_fb_helper_alloc_fbi
+#define drm_fb_helper_unregister_info drm_fb_helper_unregister_fbi
+#else
+#define COMPAT_FB_INFO info
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) || \
+        ((LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) && !defined(ANDROID))
+static inline void pvr_vm_flags_set(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags |= flags;
+}
+static inline void pvr_vm_flags_init(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags = flags;
+}
+static inline void pvr_vm_flags_clear(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags &= ~flags;
+}
+#else
+#define pvr_vm_flags_set  vm_flags_set
+#define pvr_vm_flags_init vm_flags_init
+#define pvr_vm_flags_clear vm_flags_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0))
+#define pvr_class_create(name) class_create(THIS_MODULE, name)
+#else
+#define pvr_class_create(name) class_create(name)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+#define thermal_tripless_zone_device_register(type, devdata, ops, tzp) \
+	thermal_zone_device_register((type), 0, 0, (devdata), (ops), (tzp), 0, 0)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0)) */
+
+#if defined(__GNUC__)
+#define GCC_VERSION_AT_LEAST(major, minor) \
+	(__GNUC__ > (major) || \
+	(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#define GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if defined(__clang__)
+#define CLANG_VERSION_AT_LEAST(major) \
+	(__clang_major__ >= (major))
+#else
+#define CLANG_VERSION_AT_LEAST(major) 0
+#endif
+
+#if !defined(__fallthrough)
+	#if GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+		#define __fallthrough __attribute__((__fallthrough__))
+	#else
+		#define __fallthrough
+	#endif
+#endif
+
+#endif /* __KERNEL_COMPATIBILITY_H__ */
diff --git a/drivers/gpu/drm/img/kernel_config_compatibility.h b/drivers/gpu/drm/img/kernel_config_compatibility.h
new file mode 100644
index 000000000000..63effd65ecf9
--- /dev/null
+++ b/drivers/gpu/drm/img/kernel_config_compatibility.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@Title          Kernel config compatibility define options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file is exclusively for Linux config kernel options.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_CONFIG_COMPATIBILITY_H__
+#define __KERNEL_CONFIG_COMPATIBILITY_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0))
+#ifdef SUPPORT_DRM_FBDEV_EMULATION
+#define CONFIG_DRM_FBDEV_EMULATION
+#endif
+#endif
+
+#endif /* __KERNEL_CONFIG_COMPATIBILITY_H__ */
diff --git a/drivers/gpu/drm/img/kernel_nospec.h b/drivers/gpu/drm/img/kernel_nospec.h
new file mode 100644
index 000000000000..ef6fee368327
--- /dev/null
+++ b/drivers/gpu/drm/img/kernel_nospec.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@Title          Macro to limit CPU speculative execution in kernel code
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_NOSPEC_H__
+#define __KERNEL_NOSPEC_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 2) ||			\
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) &&		\
+	 LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 18)) ||		\
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) &&		\
+	 LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 81)))
+#include <linux/kernel.h>
+#include <linux/bug.h>
+#include <linux/nospec.h>
+#else
+#define array_index_nospec(index, size) (index)
+#endif
+
+/*
+ * For Ubuntu kernels, the features available for a given Linux version code
+ * may not match those in upstream kernels. This is the case for the
+ * availability of the array_index_nospec macro.
+ */
+#if !defined(array_index_nospec)
+#define array_index_nospec(index, size) (index)
+#endif
+
+#endif /* __KERNEL_NOSPEC_H__ */
diff --git a/drivers/gpu/drm/img/pvr_dma_resv.h b/drivers/gpu/drm/img/pvr_dma_resv.h
new file mode 100644
index 000000000000..a51c9de84ada
--- /dev/null
+++ b/drivers/gpu/drm/img/pvr_dma_resv.h
@@ -0,0 +1,80 @@
+/*************************************************************************/ /*!
+@Title          Kernel reservation object compatibility header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __PVR_DMA_RESV_H__
+#define __PVR_DMA_RESV_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <linux/dma-resv.h>
+#else
+#include <linux/reservation.h>
+
+/* Reservation object types */
+#define dma_resv			reservation_object
+#define dma_resv_list			reservation_object_list
+
+/* Reservation object functions */
+#define dma_resv_add_excl_fence		reservation_object_add_excl_fence
+#define dma_resv_add_shared_fence	reservation_object_add_shared_fence
+#define dma_resv_fini			reservation_object_fini
+#define dma_resv_get_excl		reservation_object_get_excl
+#define dma_resv_get_list		reservation_object_get_list
+#define dma_resv_held			reservation_object_held
+#define dma_resv_init			reservation_object_init
+#define dma_resv_reserve_shared		reservation_object_reserve_shared
+#define dma_resv_test_signaled_rcu	reservation_object_test_signaled_rcu
+#define dma_resv_wait_timeout_rcu	reservation_object_wait_timeout_rcu
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0))
+
+#define dma_resv_shared_list   dma_resv_get_list
+#define dma_resv_excl_fence    dma_resv_get_excl
+#define dma_resv_wait_timeout  dma_resv_wait_timeout_rcu
+#define dma_resv_test_signaled dma_resv_test_signaled_rcu
+#define dma_resv_get_fences    dma_resv_get_fences_rcu
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)) */
+
+#endif /* __PVR_DMA_RESV_H__ */
diff --git a/drivers/gpu/drm/img/pvr_linux_fence.h b/drivers/gpu/drm/img/pvr_linux_fence.h
new file mode 100644
index 000000000000..b9c542a26a65
--- /dev/null
+++ b/drivers/gpu/drm/img/pvr_linux_fence.h
@@ -0,0 +1,103 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence compatibility header
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_LINUX_FENCE_H__)
+#define __PVR_LINUX_FENCE_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+#include <linux/fence.h>
+#else
+#include <linux/dma-fence.h>
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+/* Structures */
+#define	dma_fence fence
+#define dma_fence_array fence_array
+#define	dma_fence_cb fence_cb
+#define	dma_fence_ops fence_ops
+
+/* Defines and Enums */
+#define DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT FENCE_FLAG_ENABLE_SIGNAL_BIT
+#define DMA_FENCE_FLAG_SIGNALED_BIT FENCE_FLAG_SIGNALED_BIT
+#define DMA_FENCE_FLAG_USER_BITS FENCE_FLAG_USER_BITS
+
+#define DMA_FENCE_ERR FENCE_ERR
+#define	DMA_FENCE_TRACE FENCE_TRACE
+#define DMA_FENCE_WARN FENCE_WARN
+
+/* Functions */
+#define dma_fence_add_callback fence_add_callback
+#define dma_fence_context_alloc fence_context_alloc
+#define dma_fence_default_wait fence_default_wait
+#define dma_fence_is_signaled fence_is_signaled
+#define dma_fence_enable_sw_signaling fence_enable_sw_signaling
+#define dma_fence_free fence_free
+#define dma_fence_get fence_get
+#define dma_fence_get_rcu fence_get_rcu
+#define dma_fence_init fence_init
+#define dma_fence_is_array fence_is_array
+#define dma_fence_put fence_put
+#define dma_fence_signal fence_signal
+#define dma_fence_wait fence_wait
+#define to_dma_fence_array to_fence_array
+
+static inline signed long
+dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
+{
+	signed long lret;
+
+	lret = fence_wait_timeout(fence, intr, timeout);
+	if (lret || timeout)
+		return lret;
+
+	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1 : 0;
+}
+
+#endif
+
+#endif /* !defined(__PVR_LINUX_FENCE_H__) */
diff --git a/drivers/gpu/drm/img/pvr_vmap.h b/drivers/gpu/drm/img/pvr_vmap.h
new file mode 100644
index 000000000000..6424b20e764c
--- /dev/null
+++ b/drivers/gpu/drm/img/pvr_vmap.h
@@ -0,0 +1,83 @@
+/*
+ * @File        pvr_vmap.h
+ * @Title       Utility functions for virtual memory mapping
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PVR_VMAP_H
+#define PVR_VMAP_H
+
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+
+static inline void *pvr_vmap(struct page **pages,
+			     unsigned int count,
+			     __maybe_unused unsigned long flags,
+			     pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	return vmap(pages, count, flags, prot);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	return vm_map_ram(pages, count, -1, prot);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		return vm_map_ram(pages, count, -1);
+	else
+		return vmap(pages, count, flags, prot);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+static inline void pvr_vunmap(const void *vaddr,
+			      __maybe_unused unsigned int count,
+			      __maybe_unused pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	vunmap(vaddr);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	vm_unmap_ram(vaddr, count);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		vm_unmap_ram(vaddr, count);
+	else
+		vunmap(vaddr);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+#endif /* PVR_VMAP_H */
diff --git a/drivers/gpu/drm/img/pvrversion.h b/drivers/gpu/drm/img/pvrversion.h
new file mode 100644
index 000000000000..4d17f6e55594
--- /dev/null
+++ b/drivers/gpu/drm/img/pvrversion.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File           pvrversion.h
+@Title          PowerVR version numbers and strings.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Version numbers and strings for PowerVR components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRVERSION_H
+#define PVRVERSION_H
+
+#define PVRVERSION_MAJ               24U
+#define PVRVERSION_MIN               2U
+
+#define PVRVERSION_FAMILY           "rogueddk"
+#define PVRVERSION_BRANCHNAME       "24.2"
+#define PVRVERSION_BUILD             6643903
+#define PVRVERSION_BSCONTROL        "Rogue_DDK_Linux_WS"
+
+#define PVRVERSION_STRING           "Rogue_DDK_Linux_WS rogueddk 24.2@6643903"
+#define PVRVERSION_STRING_SHORT     "24.2@6643903"
+
+#define COPYRIGHT_TXT               "Copyright (c) Imagination Technologies Ltd. All Rights Reserved."
+
+#define PVRVERSION_BUILD_HI          664
+#define PVRVERSION_BUILD_LO          3903
+#define PVRVERSION_STRING_NUMERIC   "24.2.664.3903"
+
+#define PVRVERSION_PACK(MAJOR,MINOR) (((IMG_UINT32)((IMG_UINT32)(MAJOR) & 0xFFFFU) << 16U) | (((MINOR) & 0xFFFFU) << 0U))
+#define PVRVERSION_UNPACK_MAJ(VERSION) (((VERSION) >> 16U) & 0xFFFFU)
+#define PVRVERSION_UNPACK_MIN(VERSION) (((VERSION) >> 0U) & 0xFFFFU)
+
+#endif /* PVRVERSION_H */
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 0b3f4267130c..4cdbaa8e1f6e 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -343,8 +343,10 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 		.no_wait_gpu = false
 	};
 	struct ttm_tt *ttm = bo->ttm;
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 	struct ttm_resource_manager *man =
 			ttm_manager_type(bo->bdev, bo->resource->mem_type);
+#endif
 	pgprot_t prot;
 	int ret;
 
@@ -354,6 +356,7 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 	if (ret)
 		return ret;
 
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 	if (num_pages == 1 && ttm->caching == ttm_cached &&
 	    !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) {
 		/*
@@ -364,7 +367,9 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 		map->bo_kmap_type = ttm_bo_map_kmap;
 		map->page = ttm->pages[start_page];
 		map->virtual = kmap(map->page);
-	} else {
+	} else
+#endif
+	{
 		/*
 		 * We need to use vmap to get the desired page protection
 		 * or to make the buffer object look contiguous.
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
index b3fffe7b5062..aa137ead5cc5 100644
--- a/drivers/gpu/drm/ttm/ttm_module.c
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -74,7 +74,8 @@ pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp)
 #endif /* CONFIG_UML */
 #endif /* __i386__ || __x86_64__ */
 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
-	defined(__powerpc__) || defined(__mips__) || defined(__loongarch__)
+	defined(__powerpc__) || defined(__mips__) || defined(__loongarch__) || \
+	defined(__riscv)
 	if (caching == ttm_write_combined)
 		tmp = pgprot_writecombine(tmp);
 	else
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 46ff9c75bb12..a1aa688c389d 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -187,7 +187,11 @@ void ttm_resource_init(struct ttm_buffer_object *bo,
 	res->bus.addr = NULL;
 	res->bus.offset = 0;
 	res->bus.is_iomem = false;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	res->bus.caching = ttm_write_combined;
+#else
 	res->bus.caching = ttm_cached;
+#endif
 	res->bo = bo;
 
 	man = ttm_manager_type(bo->bdev, place->mem_type);
@@ -670,10 +674,13 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 	} else {
 		iter_io->needs_unmap = true;
 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
+
+#ifndef CONFIG_SOC_SIFIVE_EIC7700
 		if (mem->bus.caching == ttm_write_combined)
 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
 						  ioremap_wc(mem->bus.offset,
 							     mem->size));
+
 		else if (mem->bus.caching == ttm_cached)
 			iosys_map_set_vaddr(&iter_io->dmap,
 					    memremap(mem->bus.offset, mem->size,
@@ -681,6 +688,12 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 						     MEMREMAP_WT |
 						     MEMREMAP_WC));
 
+#else
+		if (mem->bus.caching == ttm_write_combined || mem->bus.caching == ttm_cached)
+			iosys_map_set_vaddr_iomem(&iter_io->dmap,
+						  ioremap_wc(mem->bus.offset,
+							     mem->size));
+#endif
 		/* If uncached requested or if mapping cached or wc failed */
 		if (iosys_map_is_null(&iter_io->dmap))
 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index bf9601351fa3..f89051265b45 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -154,7 +154,11 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm,
 	ttm->dma_address = NULL;
 	ttm->swap_storage = NULL;
 	ttm->sg = bo->sg;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	ttm->caching = ttm_write_combined;
+#else
 	ttm->caching = caching;
+#endif
 }
 
 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo,
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index a4c361b6619c..4af13e26e962 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -312,6 +312,15 @@ config SENSORS_AXI_FAN_CONTROL
 	  This driver can also be built as a module. If so, the module
 	  will be called axi-fan-control
 
+config SENSORS_ESWIN_FAN_CONTROL
+	tristate "ESWIN FAN Control Core driver"
+	help
+	  If you say yes here you get support for the Analog Devices
+	  ESWIN FAN monitoring core.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called eswin-fan-control
+
 config SENSORS_K8TEMP
 	tristate "AMD Athlon64/FX or Opteron temperature sensor"
 	depends on X86 && PCI
@@ -559,6 +568,12 @@ config SENSORS_DA9055
 	  This driver can also be built as a module. If so, the module
 	  will be called da9055-hwmon.
 
+config SENSORS_ESWIN_PVT
+	tristate "Eswin Process, Voltage, Temperature sensor driver"
+	depends on HWMON
+	help
+	  If you say yes here you get support for Eswin PVT sensor.
+
 config SENSORS_I5K_AMB
 	tristate "FB-DIMM AMB temperature sensor on Intel 5000 series chipsets"
 	depends on PCI
@@ -1488,6 +1503,13 @@ config SENSORS_LM95245
 	  This driver can also be built as a module. If so, the module
 	  will be called lm95245.
 
+config SENSORS_PAC1934
+	tristate "microchip PAC193X  Hardware Monitoring"
+	depends on I2C
+	help
+	  If you say yes here you get support for microchip PAC193X
+	  Single/Multi-Channel DC Power/Energy  Monitor with Accumulator.
+
 config SENSORS_PC87360
 	tristate "National Semiconductor PC87360 family"
 	depends on !PPC
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 4ac9452b5430..e32c22b61622 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -72,6 +72,8 @@ obj-$(CONFIG_SENSORS_EMC1403)	+= emc1403.o
 obj-$(CONFIG_SENSORS_EMC2103)	+= emc2103.o
 obj-$(CONFIG_SENSORS_EMC2305)	+= emc2305.o
 obj-$(CONFIG_SENSORS_EMC6W201)	+= emc6w201.o
+obj-$(CONFIG_SENSORS_ESWIN_FAN_CONTROL)	+= eswin-fan-control.o
+obj-$(CONFIG_SENSORS_ESWIN_PVT)	+= eswin_pvt.o
 obj-$(CONFIG_SENSORS_F71805F)	+= f71805f.o
 obj-$(CONFIG_SENSORS_F71882FG)	+= f71882fg.o
 obj-$(CONFIG_SENSORS_F75375S)	+= f75375s.o
@@ -173,6 +175,7 @@ obj-$(CONFIG_SENSORS_NTC_THERMISTOR)	+= ntc_thermistor.o
 obj-$(CONFIG_SENSORS_NZXT_KRAKEN2) += nzxt-kraken2.o
 obj-$(CONFIG_SENSORS_NZXT_SMART2) += nzxt-smart2.o
 obj-$(CONFIG_SENSORS_OXP) += oxp-sensors.o
+obj-$(CONFIG_SENSORS_PAC1934)	+= pac193x.o
 obj-$(CONFIG_SENSORS_PC87360)	+= pc87360.o
 obj-$(CONFIG_SENSORS_PC87427)	+= pc87427.o
 obj-$(CONFIG_SENSORS_PCF8591)	+= pcf8591.o
diff --git a/drivers/hwmon/eswin-fan-control.c b/drivers/hwmon/eswin-fan-control.c
new file mode 100644
index 000000000000..aa230a88166e
--- /dev/null
+++ b/drivers/hwmon/eswin-fan-control.c
@@ -0,0 +1,514 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Fan Control CORE driver
+ *
+ * Copyright 2022 ESWIN Inc.
+ */
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+#include <linux/wait.h>
+
+#define FAN_PWM_DUTY           0x0
+#define FAN_PWM_PERIOD         0x1
+#define FAN_PWM_FREE	       0x2
+
+/* register map */
+#define REG_FAN_INT            0x0
+#define REG_FAN_RPM            0x4
+
+/* wait for 50 times pwm period to trigger read interrupt */
+#define TIMEOUT(period)        nsecs_to_jiffies(50*(period))
+
+struct eswin_fan_control_data {
+	struct reset_control *fan_rst;
+	struct clk *clk;
+	void __iomem *base;
+	struct device *hdev;
+	unsigned long clk_rate;
+	int pwm_id;
+	struct pwm_device *pwm;
+	wait_queue_head_t wq;
+	bool wait_flag;
+	int irq;
+	/* pwm minimum period */
+	u32 min_period;
+	/* pulses per revolution */
+	u32 ppr;
+	/* revolutions per minute */
+	u32 rpm;
+};
+
+static inline void fan_iowrite(const u32 val, const u32 reg,
+			       const struct eswin_fan_control_data *ctl)
+{
+	iowrite32(val, ctl->base + reg);
+}
+
+static inline u32 fan_ioread(const u32 reg,
+			     const struct eswin_fan_control_data *ctl)
+{
+	return ioread32(ctl->base + reg);
+}
+
+static ssize_t eswin_fan_pwm_ctl_show(struct device *dev, struct device_attribute *da, char *buf)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	long temp = 0;
+
+	if (FAN_PWM_DUTY == attr->index) {
+		temp = pwm_get_duty_cycle(ctl->pwm);
+	}
+	else if (FAN_PWM_PERIOD == attr->index) {
+		temp = pwm_get_period(ctl->pwm);
+	}
+	else {
+		dev_err(dev, "get error attr index 0x%x\n", attr->index);
+	}
+
+	return sprintf(buf, "%lu\n", temp);
+}
+
+static ssize_t eswin_fan_pwm_ctl_store(struct device *dev, struct device_attribute *da,
+				     const char *buf, size_t count)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	struct pwm_state state;
+	int ret;
+
+	pwm_get_state(ctl->pwm, &state);
+
+	if (FAN_PWM_DUTY == attr->index) {
+		long val = 0;
+		ret = kstrtoul(buf, 10, &val);
+		if (ret)
+			return ret;
+
+		state.duty_cycle = val;
+	}
+	else if (FAN_PWM_PERIOD == attr->index) {
+		long val = 0;
+		ret = kstrtoul(buf, 10, &val);
+		if (ret)
+			return ret;
+		if (val >= ctl->min_period)
+			state.period = val;
+		else
+			dev_err(dev, "invalid pwm period!\n");
+	}
+	else {
+		dev_err(dev, "get error attr index 0x%x\n", attr->index);
+	}
+
+	pwm_apply_might_sleep(ctl->pwm, &state);
+
+	return count;
+}
+
+static ssize_t eswin_fan_pwm_free_store(struct device *dev, struct device_attribute *da,
+				     const char *buf, size_t count)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+	long val;
+	int ret;
+	ret = kstrtoul(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val) {
+		pwm_put(ctl->pwm);
+	}
+
+	return count;
+}
+
+static long eswin_fan_control_get_pwm_duty(const struct eswin_fan_control_data *ctl)
+{
+	struct pwm_state state;
+	int duty;
+
+	pwm_get_state(ctl->pwm, &state);
+	duty = pwm_get_relative_duty_cycle(&state, 256);
+
+	return duty;
+}
+
+static long eswin_fan_control_get_fan_rpm(struct eswin_fan_control_data *ctl)
+{
+	unsigned int val;
+	long period, timeout;
+	int ret;
+
+	ctl->wait_flag = false;
+	period = pwm_get_period(ctl->pwm);
+	timeout = msecs_to_jiffies(1500);
+
+	val = fan_ioread(REG_FAN_INT, ctl);
+	val = val | 0x1;
+	fan_iowrite(val, REG_FAN_INT, ctl);
+
+	/* wait read interrupt */
+	ret = wait_event_interruptible_timeout(ctl->wq,
+                                        ctl->wait_flag,
+                                        timeout);
+
+	if (!ret){
+		/* timeout, set rpm to 0 */
+		ctl->rpm = 0;
+	}
+
+	if(ctl->rpm)
+		ctl->rpm = DIV_ROUND_CLOSEST(60 * ctl->clk_rate, ctl->ppr * ctl->rpm);
+
+	return ret;
+}
+
+static int eswin_fan_control_read_fan(struct device *dev, u32 attr, long *val)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+
+	switch (attr) {
+	case hwmon_fan_input:
+		if(!eswin_fan_control_get_fan_rpm(ctl)){
+			dev_err(dev, "wait read interrupt timeout!\n");
+		}
+		*val = ctl->rpm;
+		return 0;
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_read_pwm(struct device *dev, u32 attr, long *val)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+
+	switch (attr) {
+	case hwmon_pwm_input:
+		*val = eswin_fan_control_get_pwm_duty(ctl);
+		return 0;
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_set_pwm_duty(const long val, struct eswin_fan_control_data *ctl)
+{
+	struct pwm_state state;
+
+	pwm_get_state(ctl->pwm, &state);
+	pwm_set_relative_duty_cycle(&state, val, 256);
+	pwm_apply_might_sleep(ctl->pwm, &state);
+
+	return 0;
+}
+
+static int eswin_fan_control_write_pwm(struct device *dev, u32 attr, long val)
+{
+	struct eswin_fan_control_data *ctl = dev_get_drvdata(dev);
+
+	switch (attr) {
+		case hwmon_pwm_input:
+	if((val < 25) || (val > 255))
+	{
+		dev_err(dev,"pwm range is form 25 to 255\n");
+		return -EINVAL;
+	}
+	else
+		return eswin_fan_control_set_pwm_duty(val, ctl);
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int eswin_fan_control_read_labels(struct device *dev,
+					enum hwmon_sensor_types type,
+					u32 attr, int channel, const char **str)
+{
+	switch (type) {
+	case hwmon_fan:
+		*str = "FAN";
+		return 0;
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_read(struct device *dev,
+				enum hwmon_sensor_types type,
+				u32 attr, int channel, long *val)
+{
+	switch (type) {
+	case hwmon_fan:
+		return eswin_fan_control_read_fan(dev, attr, val);
+	case hwmon_pwm:
+		return eswin_fan_control_read_pwm(dev, attr, val);
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static int eswin_fan_control_write(struct device *dev,
+				 enum hwmon_sensor_types type,
+				 u32 attr, int channel, long val)
+{
+	switch (type) {
+	case hwmon_pwm:
+		return eswin_fan_control_write_pwm(dev, attr, val);
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static umode_t eswin_fan_control_fan_is_visible(const u32 attr)
+{
+	switch (attr) {
+	case hwmon_fan_input:
+	case hwmon_fan_label:
+		return 0444;
+	default:
+		return 0;
+	}
+}
+
+static umode_t eswin_fan_control_pwm_is_visible(const u32 attr)
+{
+	switch (attr) {
+	case hwmon_pwm_input:
+		return 0644;
+	default:
+		return 0;
+	}
+}
+
+static umode_t eswin_fan_control_is_visible(const void *data,
+					enum hwmon_sensor_types type,
+					u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_fan:
+		return eswin_fan_control_fan_is_visible(attr);
+	case hwmon_pwm:
+		return eswin_fan_control_pwm_is_visible(attr);
+	default:
+		return 0;
+	}
+}
+
+static irqreturn_t eswin_fan_control_irq_handler(int irq, void *data)
+{
+	struct eswin_fan_control_data *ctl = (struct eswin_fan_control_data *)data;
+	u32 status = 0;
+
+	status = fan_ioread(REG_FAN_INT, ctl);
+	if (0x3 == (status & 0x3)){
+		ctl->rpm = fan_ioread(REG_FAN_RPM, ctl);
+
+		/* clear interrupt */
+		fan_iowrite(0x5, REG_FAN_INT, ctl);
+
+		/* wake up fan_rpm read */
+		ctl->wait_flag = true;
+		wake_up_interruptible(&ctl->wq);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int eswin_fan_control_init(struct eswin_fan_control_data *ctl,
+				const struct device_node *np)
+{
+	int ret;
+	/* get fan pulses per revolution */
+	ret = of_property_read_u32(np, "pulses-per-revolution", &ctl->ppr);
+	if (ret)
+		return ret;
+
+	/* 1, 2 and 4 are the typical and accepted values */
+	if (ctl->ppr != 1 && ctl->ppr != 2 && ctl->ppr != 4)
+		return -EINVAL;
+
+	/* get pwm minimum period */
+	ret = of_property_read_u32(np, "pwm-minimum-period", &ctl->min_period);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static void eswin_fan_control_remove(void *data)
+{
+	int ret;
+	struct eswin_fan_control_data *ctl = data;
+	pwm_put(ctl->pwm);
+	ret = reset_control_assert(ctl->fan_rst);
+	WARN_ON(0 != ret);
+	clk_disable_unprepare(ctl->clk);
+}
+
+static const struct hwmon_channel_info *eswin_fan_control_info[] = {
+	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT),
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_LABEL),
+	NULL
+};
+
+static const struct hwmon_ops eswin_fan_control_hwmon_ops = {
+	.is_visible = eswin_fan_control_is_visible,
+	.read = eswin_fan_control_read,
+	.write = eswin_fan_control_write,
+	.read_string = eswin_fan_control_read_labels,
+};
+
+static const struct hwmon_chip_info eswin_chip_info = {
+	.ops = &eswin_fan_control_hwmon_ops,
+	.info = eswin_fan_control_info,
+};
+
+static SENSOR_DEVICE_ATTR_RW(fan_pwm_duty,   eswin_fan_pwm_ctl,    FAN_PWM_DUTY);
+static SENSOR_DEVICE_ATTR_RW(fan_pwm_period, eswin_fan_pwm_ctl,    FAN_PWM_PERIOD);
+static SENSOR_DEVICE_ATTR_WO(fan_pwm_free,   eswin_fan_pwm_free,   FAN_PWM_FREE);
+
+static struct attribute *eswin_fan_control_attrs[] = {
+	&sensor_dev_attr_fan_pwm_duty.dev_attr.attr,
+	&sensor_dev_attr_fan_pwm_period.dev_attr.attr,
+	&sensor_dev_attr_fan_pwm_free.dev_attr.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(eswin_fan_control);
+
+static const struct of_device_id eswin_fan_control_of_match[] = {
+	{ .compatible = "eswin-fan-control"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, eswin_fan_control_of_match);
+
+static int eswin_fan_control_probe(struct platform_device *pdev)
+{
+	struct eswin_fan_control_data *ctl;
+	const struct of_device_id *id;
+	const char *name = "eswin_fan_control";
+	struct pwm_state state;
+	struct pwm_args pwm_args;
+	int ret;
+
+	id = of_match_node(eswin_fan_control_of_match, pdev->dev.of_node);
+	if (!id)
+		return -EINVAL;
+
+	ctl = devm_kzalloc(&pdev->dev, sizeof(*ctl), GFP_KERNEL);
+	if (!ctl)
+		return -ENOMEM;
+
+	ctl->base = devm_platform_ioremap_resource(pdev, 0);
+
+	if (IS_ERR(ctl->base))
+		return PTR_ERR(ctl->base);
+
+	ctl->clk = devm_clk_get(&pdev->dev, "pclk");
+	if (IS_ERR(ctl->clk)) {
+		dev_err(&pdev->dev, "Couldn't get the clock for fan-controller\n");
+		return -ENODEV;
+	}
+
+	ret = clk_prepare_enable(ctl->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to enable clock for fan-controller\n");
+		return ret;
+	}
+
+	ctl->clk_rate = clk_get_rate(ctl->clk);
+	if (!ctl->clk_rate)
+		return -EINVAL;
+
+	ctl->fan_rst = devm_reset_control_get_optional(&pdev->dev, "fan_rst");
+	if (IS_ERR_OR_NULL(ctl->fan_rst)) {
+		dev_err(&pdev->dev, "Failed to get fan_rst reset handle\n");
+		return -EFAULT;
+	}
+	ret = reset_control_reset(ctl->fan_rst);
+	WARN_ON(0 != ret);
+
+	init_waitqueue_head(&ctl->wq);
+
+	ctl->irq = platform_get_irq(pdev, 0);
+	if (ctl->irq < 0)
+		return ctl->irq;
+
+	ret = devm_request_threaded_irq(&pdev->dev, ctl->irq,
+					eswin_fan_control_irq_handler, NULL,
+					IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+					pdev->driver_override, ctl);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request an irq, %d", ret);
+		return ret;
+	}
+
+	ret = eswin_fan_control_init(ctl, pdev->dev.of_node);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to initialize device\n");
+		return ret;
+	}
+
+	ctl->pwm = pwm_get(&pdev->dev, NULL);
+	if (IS_ERR(ctl->pwm)) {
+		ret = PTR_ERR(ctl->pwm);
+		dev_err(&pdev->dev, "Failed to request pwm device: %d\n", ret);
+		return ret;
+	}
+
+	pwm_get_state(ctl->pwm, &state);
+
+	/* Then fill it with the reference config */
+	pwm_get_args(ctl->pwm, &pwm_args);
+
+	state.period = pwm_args.period;
+	state.duty_cycle = state.period * 255 / 256; /* default set max speed */
+
+	dev_info(&pdev->dev, "state.period: %lld state.duty_cycle: %lld\n",
+			state.period,state.duty_cycle);
+
+	ret = pwm_apply_might_sleep(ctl->pwm, &state);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to apply initial PWM state: %d\n",
+			ret);
+	}
+	pwm_enable(ctl->pwm);
+
+	ret = devm_add_action_or_reset(&pdev->dev, eswin_fan_control_remove, ctl);
+	if (ret)
+		return ret;
+
+	ctl->hdev = devm_hwmon_device_register_with_info(&pdev->dev,
+							 name,
+							 ctl,
+							 &eswin_chip_info,
+							 eswin_fan_control_groups);
+	dev_info(&pdev->dev, "eswin fan control init done\n");
+	return PTR_ERR_OR_ZERO(ctl->hdev);
+}
+
+static struct platform_driver eswin_fan_control_driver = {
+	.driver = {
+		.name = "eswin_fan_control_driver",
+		.of_match_table = eswin_fan_control_of_match,
+	},
+	.probe = eswin_fan_control_probe,
+};
+module_platform_driver(eswin_fan_control_driver);
+
+MODULE_AUTHOR("Han Min <hanmin@eswincomputing.com>");
+MODULE_DESCRIPTION("ESWIN Fan Control CORE driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/eswin_pvt.c b/drivers/hwmon/eswin_pvt.c
new file mode 100644
index 000000000000..6436330fde1c
--- /dev/null
+++ b/drivers/hwmon/eswin_pvt.c
@@ -0,0 +1,874 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN PVT Device Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Bingzheng Yu <yubingzheng@eswincomputing.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/hwmon.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/ktime.h>
+#include <linux/limits.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/seqlock.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include "eswin_pvt.h"
+
+/*
+ * For the sake of the code simplification we created the sensors info table
+ * with the sensor names, activation modes, threshold registers base address
+ * and the thresholds bit fields.
+ */
+static const struct pvt_sensor_info pvt_info[] = {
+	PVT_SENSOR_INFO(0, "CPU Core Temperature", hwmon_temp, TEMP, TTHRES),
+	PVT_SENSOR_INFO(0, "CPU Core Voltage", hwmon_in, VOLT, VTHRES),
+	PVT_SENSOR_INFO(1, "CPU Core Low-Vt", hwmon_in, LVT, LTHRES),
+	PVT_SENSOR_INFO(2, "CPU Core UltraLow-Vt", hwmon_in, ULVT, ULTHRES),
+	PVT_SENSOR_INFO(3, "CPU Core Standard-Vt", hwmon_in, SVT, STHRES),
+};
+
+/*
+ * The original translation formulae of the temperature (in degrees of Celsius)
+ * to PVT data and vice-versa are following:
+ * N = 6.0818e-8*(T^4) +1.2873e-5*(T^3) + 7.2244e-3*(T^2) + 3.6484*(T^1) +
+ *     1.6198e2,
+ * T = -1.8439e-11*(N^4) + 8.0705e-8*(N^3) + -1.8501e-4*(N^2) +
+ *     3.2843e-1*(N^1) - 4.8690e1,
+ * where T = [-40, 125]C and N = [27, 771].
+ * They must be accordingly altered to be suitable for the integer arithmetics.
+ * The technique is called 'factor redistribution', which just makes sure the
+ * multiplications and divisions are made so to have a result of the operations
+ * within the integer numbers limit. In addition we need to translate the
+ * formulae to accept millidegrees of Celsius. Here what they look like after
+ * the alterations:
+ * N = (60818e-20*(T^4) + 12873e-14*(T^3) + 72244e-9*(T^2) + 36484e-3*T +
+ *     16198e2) / 1e4,
+ * T = -18439e-12*(N^4) + 80705e-9*(N^3) - 185010e-6*(N^2) + 328430e-3*N -
+ *     48690,
+ * where T = [-40000, 125000] mC and N = [27, 771].
+ */
+static const struct pvt_poly __maybe_unused poly_temp_to_N = {
+	.total_divider = 10000,
+	.terms = {
+		{4, 60818, 10000, 10000},
+		{3, 12873, 10000, 100},
+		{2, 72244, 10000, 10},
+		{1, 36484, 1000, 1},
+		{0, 1619800, 1, 1}
+	}
+};
+
+static const struct pvt_poly poly_N_to_temp = {
+	.total_divider = 1,
+	.terms = {
+		{4, -18439, 1000, 1},
+		{3, 80705, 1000, 1},
+		{2, -185010, 1000, 1},
+		{1, 328430, 1000, 1},
+		{0, -48690, 1, 1}
+	}
+};
+
+/*
+ * Similar alterations are performed for the voltage conversion equations.
+ * The original formulae are:
+ * N = 1.3905e3*V - 5.7685e2,
+ * V = (N + 5.7685e2) / 1.3905e3,
+ * where V = [0.72, 0.88] V and N = [424, 646].
+ * After the optimization they looks as follows:
+ * N = (13905e-3*V - 5768.5) / 10,
+ * V = (N * 10^5 / 13905 + 57685 * 10^3 / 13905) / 10.
+ * where V = [720, 880] mV and N = [424, 646].
+ */
+static const struct pvt_poly __maybe_unused poly_volt_to_N = {
+	.total_divider = 10,
+	.terms = {
+		{1, 13905, 1000, 1},
+		{0, -57685, 1, 10}
+	}
+};
+
+static const struct pvt_poly poly_N_to_volt = {
+	.total_divider = 10,
+	.terms = {
+		{1, 100000, 13905, 1},
+		{0, 57685000, 1, 13905}
+	}
+};
+
+/*
+ * Here is the polynomial calculation function, which performs the
+ * redistributed terms calculations. It's pretty straightforward. We walk
+ * over each degree term up to the free one, and perform the redistributed
+ * multiplication of the term coefficient, its divider (as for the rationale
+ * fraction representation), data power and the rational fraction divider
+ * leftover. Then all of this is collected in a total sum variable, which
+ * value is normalized by the total divider before being returned.
+ */
+static long eswin_pvt_calc_poly(const struct pvt_poly *poly, long data)
+{
+	const struct pvt_poly_term *term = poly->terms;
+	long tmp, ret = 0;
+	int deg;
+	do {
+		tmp = term->coef;
+		for (deg = 0; deg < term->deg; ++deg)
+			tmp = mult_frac(tmp, data, term->divider);
+		ret += tmp / term->divider_leftover;
+	} while ((term++)->deg);
+
+	return ret / poly->total_divider;
+}
+
+static inline u32 eswin_pvt_update(void __iomem *reg, u32 mask, u32 data)
+{
+	u32 old;
+
+	old = readl_relaxed(reg);
+	writel((old & ~mask) | (data & mask), reg);
+
+	return old & mask;
+}
+
+static inline void eswin_pvt_set_mode(struct pvt_hwmon *pvt, u32 mode)
+{
+	u32 old;
+
+	mode = FIELD_PREP(PVT_MODE_MASK, mode);
+
+	old = eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_MODE, PVT_MODE_MASK, mode);
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+static inline u32 eswin_pvt_calc_trim(long temp)
+{
+	temp = clamp_val(temp, 0, PVT_TRIM_TEMP);
+
+	return DIV_ROUND_UP(temp, PVT_TRIM_STEP);
+}
+
+static inline void eswin_pvt_set_trim(struct pvt_hwmon *pvt, u32 val)
+{
+	u32 old;
+
+	old = eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	writel(val, pvt->regs + PVT_TRIM);
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+static irqreturn_t eswin_pvt_hard_isr(int irq, void *data)
+{
+	struct pvt_hwmon *pvt = data;
+	struct pvt_cache *cache;
+	u32 val;
+
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	/*
+	 * Nothing special for alarm-less driver. Just read the data, update
+	 * the cache and notify a waiter of this event.
+	 */
+
+	val = readl(pvt->regs + PVT_DATA);
+
+	cache = &pvt->cache[pvt->sensor];
+
+	WRITE_ONCE(cache->data, FIELD_GET(PVT_DATA_OUT, val));
+
+	complete(&cache->conversion);
+
+	return IRQ_HANDLED;
+}
+
+static inline umode_t eswin_pvt_limit_is_visible(enum pvt_sensor_type type)
+{
+	return 0;
+}
+
+static inline umode_t eswin_pvt_pvt_alarm_is_visible(enum pvt_sensor_type type)
+{
+	return 0;
+}
+
+static int eswin_pvt_read_data(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			 long *val)
+{
+	struct pvt_cache *cache = &pvt->cache[type];
+	unsigned long timeout;
+	u32 data;
+	int ret;
+
+	/*
+	 * Lock PVT conversion interface until data cache is updated. The
+	 * data read procedure is following: set the requested PVT sensor
+	 * mode, enable IRQ and conversion, wait until conversion is finished,
+	 * then disable conversion and IRQ, and read the cached data.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	pvt->sensor = type;
+	eswin_pvt_set_mode(pvt, pvt_info[type].mode);
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, PVT_ENA_EN);
+
+	/*
+	 * Wait with timeout since in case if the sensor is suddenly powered
+	 * down the request won't be completed and the caller will hang up on
+	 * this procedure until the power is back up again. Multiply the
+	 * timeout by the factor of two to prevent a false timeout.
+	 */
+	timeout = 2 * usecs_to_jiffies(ktime_to_us(pvt->timeout));
+	if(type==PVT_TEMP){
+		timeout = 20 * usecs_to_jiffies(ktime_to_us(pvt->timeout));
+	}
+	ret = wait_for_completion_timeout(&cache->conversion, timeout);
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	data = READ_ONCE(cache->data);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	if (!ret)
+		return -ETIMEDOUT;
+
+	if (type == PVT_TEMP)
+		*val = eswin_pvt_calc_poly(&poly_N_to_temp, data);
+	else if (type == PVT_VOLT)
+		*val = eswin_pvt_calc_poly(&poly_N_to_volt, data);
+	else
+		*val = data;
+
+	return 0;
+}
+
+static int eswin_pvt_read_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			  bool is_low, long *val)
+{
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_write_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			   bool is_low, long val)
+{
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_read_alarm(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
+			  bool is_low, long *val)
+{
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_channel_info *pvt_channel_info[] = {
+	HWMON_CHANNEL_INFO(chip,
+			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT | HWMON_T_TYPE | HWMON_T_LABEL |
+			   HWMON_T_OFFSET),
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL),
+	NULL
+};
+
+static inline bool eswin_pvt_hwmon_channel_is_valid(enum hwmon_sensor_types type,
+					      int ch)
+{
+	switch (type) {
+	case hwmon_temp:
+		if (ch < 0 || ch >= PVT_TEMP_CHS)
+			return false;
+		break;
+	case hwmon_in:
+		if (ch < 0 || ch >= PVT_VOLT_CHS)
+			return false;
+		break;
+	default:
+		break;
+	}
+
+	/* The rest of the types are independent from the channel number. */
+	return true;
+}
+
+static umode_t eswin_pvt_hwmon_is_visible(const void *data,
+				    enum hwmon_sensor_types type,
+				    u32 attr, int ch)
+{
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return 0;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return 0644;
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+		case hwmon_temp_type:
+		case hwmon_temp_label:
+			return 0444;
+		case hwmon_temp_min:
+		case hwmon_temp_max:
+			return eswin_pvt_limit_is_visible(ch);
+		case hwmon_temp_min_alarm:
+		case hwmon_temp_max_alarm:
+			return eswin_pvt_pvt_alarm_is_visible(ch);
+		case hwmon_temp_offset:
+			return 0644;
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+		case hwmon_in_label:
+			return 0444;
+		case hwmon_in_min:
+		case hwmon_in_max:
+			return eswin_pvt_limit_is_visible(PVT_VOLT + ch);
+		case hwmon_in_min_alarm:
+		case hwmon_in_max_alarm:
+			return eswin_pvt_pvt_alarm_is_visible(PVT_VOLT + ch);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_read_trim(struct pvt_hwmon *pvt, long *val)
+{
+	u32 data;
+
+	data = readl(pvt->regs + PVT_TRIM);
+	*val = data;
+
+	return 0;
+}
+
+static int eswin_pvt_write_trim(struct pvt_hwmon *pvt, long val)
+{
+	int ret;
+	/*
+	 * Serialize trim update, since a part of the register is changed and
+	 * the controller is supposed to be disabled during this operation.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	eswin_pvt_set_trim(pvt, val);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_read_timeout(struct pvt_hwmon *pvt, long *val)
+{
+	int ret;
+
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	/* Return the result in msec as hwmon sysfs interface requires. */
+	*val = ktime_to_ms(pvt->timeout);
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_write_timeout(struct pvt_hwmon *pvt, long val)
+{
+	unsigned long rate;
+	ktime_t kt, cache;
+	u32 data;
+	int ret;
+
+	rate = clk_get_rate(pvt->clk);
+	if (!rate)
+		return -ENODEV;
+
+	/*
+	 * If alarms are enabled, the requested timeout must be divided
+	 * between all available sensors to have the requested delay
+	 * applicable to each individual sensor.
+	 */
+	cache = kt = ms_to_ktime(val);
+
+	/*
+	 * Subtract a constant lag, which always persists due to the limited
+	 * PVT sampling rate. Make sure the timeout is not negative.
+	 */
+	kt = ktime_sub_ns(kt, PVT_TOUT_MIN);
+	if (ktime_to_ns(kt) < 0)
+		kt = ktime_set(0, 0);
+
+	/*
+	 * Finally recalculate the timeout in terms of the reference clock
+	 * period.
+	 */
+	data = ktime_divns(kt * rate, NSEC_PER_SEC);
+
+	/*
+	 * Update the measurements delay, but lock the interface first, since
+	 * we have to disable PVT in order to have the new delay actually
+	 * updated.
+	 */
+	ret = mutex_lock_interruptible(&pvt->iface_mtx);
+	if (ret)
+		return ret;
+
+	pvt->timeout = cache;
+
+	mutex_unlock(&pvt->iface_mtx);
+
+	return 0;
+}
+
+static int eswin_pvt_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int ch, long *val)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return eswin_pvt_read_timeout(pvt, val);
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			return eswin_pvt_read_data(pvt, ch, val);
+		case hwmon_temp_type:
+			*val = 1;
+			return 0;
+		case hwmon_temp_min:
+			return eswin_pvt_read_limit(pvt, ch, true, val);
+		case hwmon_temp_max:
+			return eswin_pvt_read_limit(pvt, ch, false, val);
+		case hwmon_temp_min_alarm:
+			return eswin_pvt_read_alarm(pvt, ch, true, val);
+		case hwmon_temp_max_alarm:
+			return eswin_pvt_read_alarm(pvt, ch, false, val);
+		case hwmon_temp_offset:
+			return eswin_pvt_read_trim(pvt, val);
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+			return eswin_pvt_read_data(pvt, PVT_VOLT + ch, val);
+		case hwmon_in_min:
+			return eswin_pvt_read_limit(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max:
+			return eswin_pvt_read_limit(pvt, PVT_VOLT + ch, false, val);
+		case hwmon_in_min_alarm:
+			return eswin_pvt_read_alarm(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max_alarm:
+			return eswin_pvt_read_alarm(pvt, PVT_VOLT + ch, false, val);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_hwmon_read_string(struct device *dev,
+				 enum hwmon_sensor_types type,
+				 u32 attr, int ch, const char **str)
+{
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_label:
+			*str = pvt_info[ch].label;
+			return 0;
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_label:
+			*str = pvt_info[PVT_VOLT + ch].label;
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int eswin_pvt_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
+			   u32 attr, int ch, long val)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	if (!eswin_pvt_hwmon_channel_is_valid(type, ch))
+		return -EINVAL;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return eswin_pvt_write_timeout(pvt, val);
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_min:
+			return eswin_pvt_write_limit(pvt, ch, true, val);
+		case hwmon_temp_max:
+			return eswin_pvt_write_limit(pvt, ch, false, val);
+		case hwmon_temp_offset:
+			return eswin_pvt_write_trim(pvt, val);
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_min:
+			return eswin_pvt_write_limit(pvt, PVT_VOLT + ch, true, val);
+		case hwmon_in_max:
+			return eswin_pvt_write_limit(pvt, PVT_VOLT + ch, false, val);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_ops pvt_hwmon_ops = {
+	.is_visible = eswin_pvt_hwmon_is_visible,
+	.read = eswin_pvt_hwmon_read,
+	.read_string = eswin_pvt_hwmon_read_string,
+	.write = eswin_pvt_hwmon_write
+};
+
+static const struct hwmon_chip_info pvt_hwmon_info = {
+	.ops = &pvt_hwmon_ops,
+	.info = pvt_channel_info
+};
+
+static void pvt_clear_data(void *data)
+{
+	struct pvt_hwmon *pvt = data;
+	int idx;
+
+	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
+		complete_all(&pvt->cache[idx].conversion);
+
+	mutex_destroy(&pvt->iface_mtx);
+}
+
+static struct pvt_hwmon *eswin_pvt_create_data(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct pvt_hwmon *pvt;
+	int ret, idx;
+
+	pvt = devm_kzalloc(dev, sizeof(*pvt), GFP_KERNEL);
+	if (!pvt)
+		return ERR_PTR(-ENOMEM);
+
+	ret = devm_add_action(dev, pvt_clear_data, pvt);
+	if (ret) {
+		dev_err(dev, "Can't add PVT data clear action\n");
+		return ERR_PTR(ret);
+	}
+
+	pvt->dev = dev;
+	pvt->sensor = PVT_SENSOR_FIRST;
+	mutex_init(&pvt->iface_mtx);
+
+	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
+		init_completion(&pvt->cache[idx].conversion);
+
+	return pvt;
+}
+
+static int eswin_pvt_request_regs(struct pvt_hwmon *pvt)
+{
+	struct platform_device *pdev = to_platform_device(pvt->dev);
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(pvt->dev, "Couldn't find PVT memresource\n");
+		return -EINVAL;
+	}
+
+	pvt->regs = devm_ioremap_resource(pvt->dev, res);
+	if (IS_ERR(pvt->regs))
+		return PTR_ERR(pvt->regs);
+
+	return 0;
+}
+
+static void eswin_pvt_remove(void *data)
+{
+	int ret;
+	struct pvt_hwmon *pvt = data;
+	ret = reset_control_assert(pvt->pvt_rst);
+	WARN_ON(0 != ret);
+	clk_disable_unprepare(pvt->clk);
+}
+
+static int eswin_pvt_request_clks(struct pvt_hwmon *pvt)
+{
+	int ret;
+
+	pvt->clk = devm_clk_get(pvt->dev, "pvt_clk");
+	if (IS_ERR(pvt->clk)) {
+		dev_err(pvt->dev, "Couldn't get PVT clock\n");
+		return -ENODEV;
+	}
+
+	ret = clk_prepare_enable(pvt->clk);
+	if (ret) {
+		dev_err(pvt->dev, "Couldn't enable the PVT clocks\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_request_rst(struct pvt_hwmon *pvt)
+{
+	int ret;
+	pvt->pvt_rst = devm_reset_control_get_optional(pvt->dev, "pvt_rst");
+	if(IS_ERR_OR_NULL(pvt->pvt_rst)){
+		dev_err(pvt->dev, "Couldn't get PVT reset\n");
+	}
+	ret = reset_control_reset(pvt->pvt_rst);
+	WARN_ON(0 != ret);
+	return 0;
+}
+
+static int eswin_pvt_check_pwr(struct pvt_hwmon *pvt)
+{
+	unsigned long tout;
+	int ret = 0;
+	u32 data;
+
+	/*
+	 * Test out the sensor conversion functionality. If it is not done on
+	 * time then the domain must have been unpowered and we won't be able
+	 * to use the device later in this driver.
+	 * Note If the power source is lost during the normal driver work the
+	 * data read procedure will either return -ETIMEDOUT (for the
+	 * alarm-less driver configuration) or just stop the repeated
+	 * conversion. In the later case alas we won't be able to detect the
+	 * problem.
+	 */
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, PVT_ENA_EN);
+	readl(pvt->regs + PVT_DATA);
+
+	tout = PVT_TOUT_MIN / NSEC_PER_USEC;
+	usleep_range(tout, 2 * tout);
+
+	data = readl(pvt->regs + PVT_DATA);
+
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	return ret;
+}
+
+static int eswin_pvt_init_iface(struct pvt_hwmon *pvt)
+{
+	unsigned long rate;
+
+	rate = clk_get_rate(pvt->clk);
+	if (!rate) {
+		dev_err(pvt->dev, "Invalid reference clock rate\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * Make sure all interrupts and controller are disabled so not to
+	 * accidentally have ISR executed before the driver data is fully
+	 * initialized. Clear the IRQ status as well.
+	 */
+	eswin_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eswin_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	readl(pvt->regs + PVT_DATA);
+
+	/* Setup default sensor mode, timeout and temperature trim. */
+	eswin_pvt_set_mode(pvt, pvt_info[pvt->sensor].mode);
+
+	/*
+	 * Preserve the current ref-clock based delay (Ttotal) between the
+	 * sensors data samples in the driver data so not to recalculate it
+	 * each time on the data requests and timeout reads. It consists of the
+	 * delay introduced by the internal ref-clock timer (N / Fclk) and the
+	 * constant timeout caused by each conversion latency (Tmin):
+	 *   Ttotal = N / Fclk + Tmin
+	 * If alarms are enabled the sensors are polled one after another and
+	 * in order to get the next measurement of a particular sensor the
+	 * caller will have to wait for at most until all the others are
+	 * polled. In that case the formulae will look a bit different:
+	 *   Ttotal = 5 * (N / Fclk + Tmin)
+	 */
+
+	pvt->timeout = ktime_set(PVT_TOUT_DEF, 0);
+	pvt->timeout = ktime_divns(pvt->timeout, rate);
+	pvt->timeout = ktime_add_ns(pvt->timeout, PVT_TOUT_MIN);
+
+	return 0;
+}
+
+static int eswin_pvt_request_irq(struct pvt_hwmon *pvt)
+{
+	struct platform_device *pdev = to_platform_device(pvt->dev);
+	int ret;
+
+	pvt->irq = platform_get_irq(pdev, 0);
+	if (pvt->irq < 0)
+		return pvt->irq;
+
+	ret = devm_request_threaded_irq(pvt->dev, pvt->irq,
+					eswin_pvt_hard_isr, NULL,
+					IRQF_SHARED | IRQF_TRIGGER_HIGH,
+					"pvt", pvt);
+	if (ret) {
+		dev_err(pvt->dev, "Couldn't request PVT IRQ\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_create_hwmon(struct pvt_hwmon *pvt)
+{
+	pvt->hwmon = devm_hwmon_device_register_with_info(pvt->dev, "pvt", pvt,
+		&pvt_hwmon_info, NULL);
+	if (IS_ERR(pvt->hwmon)) {
+		dev_err(pvt->dev, "Couldn't create hwmon device\n");
+		return PTR_ERR(pvt->hwmon);
+	}
+
+	return 0;
+}
+
+static int eswin_pvt_probe(struct platform_device *pdev)
+{
+	struct pvt_hwmon *pvt;
+	int ret;
+
+	pvt = eswin_pvt_create_data(pdev);
+	if (IS_ERR(pvt))
+		return PTR_ERR(pvt);
+
+	ret = eswin_pvt_request_regs(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_request_clks(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_request_rst(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_check_pwr(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_init_iface(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_request_irq(pvt);
+	if (ret)
+		return ret;
+
+	ret = eswin_pvt_create_hwmon(pvt);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(pvt->dev, eswin_pvt_remove, pvt);
+	if (ret) {
+		dev_err(pvt->dev, "Can't add PVT clocks disable action\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id pvt_of_match[] = {
+	{ .compatible = "eswin,eswin-pvt" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, pvt_of_match);
+
+static struct platform_driver pvt_driver = {
+	.probe = eswin_pvt_probe,
+	.driver = {
+		.name = "eswin-pvt",
+		.of_match_table = pvt_of_match
+	},
+};
+module_platform_driver(pvt_driver);
+
+MODULE_DESCRIPTION("Eswin PVT driver");
+MODULE_AUTHOR("Yu Bingzheng <yubingzheng@eswincomputing.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/hwmon/eswin_pvt.h b/drivers/hwmon/eswin_pvt.h
new file mode 100644
index 000000000000..1c0c9252317c
--- /dev/null
+++ b/drivers/hwmon/eswin_pvt.h
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) ESWIN Electronics Co.Ltd
+ *
+ * Eswin Process, Voltage, Temperature sensor driver
+ */
+#ifndef __HWMON_ESWIN_PVT_H__
+#define __HWMON_ESWIN_PVT_H__
+
+#include <linux/completion.h>
+#include <linux/hwmon.h>
+#include <linux/kernel.h>
+#include <linux/ktime.h>
+#include <linux/mutex.h>
+#include <linux/seqlock.h>
+
+/* Eswin PVT registers and their bitfields */
+#define PVT_DIV               0x00
+#define PVT_TRIM              0x04
+#define PVT_TRIM_G            GENMASK(4,0)
+#define PVT_TRIM_O            GENMASK(13,8)
+#define PVT_MODE              0x08
+#define PVT_MODE_MASK         GENMASK(2, 0)
+#define PVT_CTRL_MODE_TEMP    0x0
+#define PVT_CTRL_MODE_VOLT    0x4
+#define PVT_CTRL_MODE_LVT     0x1
+#define PVT_CTRL_MODE_ULVT    0x2
+#define PVT_CTRL_MODE_SVT     0x3
+#define PVT_MODE_PSAMPLE_0    BIT(0)
+#define PVT_MODE_PSAMPLE_1    BIT(1)
+#define PVT_MODE_VSAMPLE      BIT(2)
+#define PVT_ENA               0x0c
+#define PVT_ENA_EN            BIT(0)
+#define PVT_INT               0x10
+#define PVT_INT_CLR           BIT(1)
+#define PVT_DATA              0x14
+#define PVT_DATA_OUT          GENMASK(9,0)
+
+/* alarm related */
+#define PVT_TTHRES			0x08
+#define PVT_VTHRES			0x0C
+#define PVT_LTHRES			0x10
+#define PVT_ULTHRES			0x14
+#define PVT_STHRES			0x18
+#define PVT_INTR_DVALID			BIT(0)
+#define PVT_INTR_TTHRES_LO		BIT(1)
+#define PVT_INTR_TTHRES_HI		BIT(2)
+#define PVT_INTR_VTHRES_LO		BIT(3)
+#define PVT_INTR_VTHRES_HI		BIT(4)
+#define PVT_INTR_LTHRES_LO		BIT(5)
+#define PVT_INTR_LTHRES_HI		BIT(6)
+#define PVT_INTR_ULTHRES_LO		BIT(7)
+#define PVT_INTR_ULTHRES_HI		BIT(8)
+#define PVT_INTR_STHRES_LO		BIT(9)
+#define PVT_INTR_STHRES_HI		BIT(10)
+
+/*
+ * PVT sensors-related limits and default values
+ * @PVT_TEMP_MIN: Minimal temperature in millidegrees of Celsius.
+ * @PVT_TEMP_MAX: Maximal temperature in millidegrees of Celsius.
+ * @PVT_TEMP_CHS: Number of temperature hwmon channels.
+ * @PVT_VOLT_MIN: Minimal voltage in mV.
+ * @PVT_VOLT_MAX: Maximal voltage in mV.
+ * @PVT_VOLT_CHS: Number of voltage hwmon channels.
+ * @PVT_DATA_MIN: Minimal PVT raw data value.
+ * @PVT_DATA_MAX: Maximal PVT raw data value.
+ * @PVT_TRIM_MIN: Minimal temperature sensor trim value.
+ * @PVT_TRIM_MAX: Maximal temperature sensor trim value.
+ * @PVT_TRIM_DEF: Default temperature sensor trim value (set a proper value
+ *		  when one is determined for ESWIN SoC).
+ * @PVT_TRIM_TEMP: Maximum temperature encoded by the trim factor.
+ * @PVT_TRIM_STEP: Temperature stride corresponding to the trim value.
+ * @PVT_TOUT_MIN: Minimal timeout between samples in nanoseconds.
+ * @PVT_TOUT_DEF: Default data measurements timeout. In case if alarms are
+ *		  activated the PVT IRQ is enabled to be raised after each
+ *		  conversion in order to have the thresholds checked and the
+ *		  converted value cached. Too frequent conversions may cause
+ *		  the system CPU overload. Lets set the 50ms delay between
+ *		  them by default to prevent this.
+ */
+#define PVT_TEMP_MIN		-40000L
+#define PVT_TEMP_MAX		125000L
+#define PVT_TEMP_CHS		1
+#define PVT_VOLT_MIN		720L
+#define PVT_VOLT_MAX		880L
+#define PVT_VOLT_CHS		4
+#define PVT_DATA_MIN		0
+#define PVT_DATA_DATA_FLD       0
+#define PVT_CTRL_TRIM_FLD       4
+#define PVT_CTRL_TRIM_MASK      GENMASK(8,4)
+#define PVT_DATA_MAX		(PVT_DATA_DATA_MASK >> PVT_DATA_DATA_FLD)
+#define PVT_TRIM_MIN		0
+#define PVT_TRIM_MAX		(PVT_CTRL_TRIM_MASK >> PVT_CTRL_TRIM_FLD)
+#define PVT_TRIM_TEMP		7130
+#define PVT_TRIM_STEP		(PVT_TRIM_TEMP / PVT_TRIM_MAX)
+#define PVT_TRIM_DEF		0
+#define PVT_TOUT_MIN		(NSEC_PER_SEC / 3000)
+# define PVT_TOUT_DEF		0
+
+/*
+ * enum pvt_sensor_type - ESWIN PVT sensor types (correspond to each PVT
+ *			  sampling mode)
+ * @PVT_SENSOR*: helpers to traverse the sensors in loops.
+ * @PVT_TEMP: PVT Temperature sensor.
+ * @PVT_VOLT: PVT Voltage sensor.
+ * @PVT_LVT: PVT Low-Voltage threshold sensor.
+ * @PVT_HVT: PVT High-Voltage threshold sensor.
+ * @PVT_SVT: PVT Standard-Voltage threshold sensor.
+ */
+enum pvt_sensor_type {
+	PVT_SENSOR_FIRST,
+	PVT_TEMP = PVT_SENSOR_FIRST,
+	PVT_VOLT,
+	PVT_LVT,
+	PVT_ULVT,
+	PVT_SVT,
+	PVT_SENSOR_LAST = PVT_SVT,
+	PVT_SENSORS_NUM
+};
+
+/*
+ * struct pvt_sensor_info - ESWIN PVT sensor informational structure
+ * @channel: Sensor channel ID.
+ * @label: hwmon sensor label.
+ * @mode: PVT mode corresponding to the channel.
+ * @thres_base: upper and lower threshold values of the sensor.
+ * @thres_sts_lo: low threshold status bitfield.
+ * @thres_sts_hi: high threshold status bitfield.
+ * @type: Sensor type.
+ * @attr_min_alarm: Min alarm attribute ID.
+ * @attr_min_alarm: Max alarm attribute ID.
+ */
+struct pvt_sensor_info {
+	int channel;
+	const char *label;
+	u32 mode;
+	unsigned long thres_base;
+	u32 thres_sts_lo;
+	u32 thres_sts_hi;
+	enum hwmon_sensor_types type;
+	u32 attr_min_alarm;
+	u32 attr_max_alarm;
+};
+
+#define PVT_SENSOR_INFO(_ch, _label, _type, _mode, _thres)	\
+	{							\
+		.channel = _ch,					\
+		.label = _label,				\
+		.mode = PVT_CTRL_MODE_ ##_mode,			\
+		.thres_base = PVT_ ##_thres,			\
+		.thres_sts_lo = PVT_INTR_ ##_thres## _LO,	\
+		.thres_sts_hi = PVT_INTR_ ##_thres## _HI,	\
+		.type = _type,					\
+		.attr_min_alarm = _type## _min,			\
+		.attr_max_alarm = _type## _max,			\
+	}
+
+/*
+ * struct pvt_cache - PVT sensors data cache
+ * @data: data cache in raw format.
+ * @thres_sts_lo: low threshold status saved on the previous data conversion.
+ * @thres_sts_hi: high threshold status saved on the previous data conversion.
+ * @data_seqlock: cached data seq-lock.
+ * @conversion: data conversion completion.
+ */
+struct pvt_cache {
+	u32 data;
+	struct completion conversion;
+};
+
+/*
+ * struct pvt_hwmon - Eswin PVT private data
+ * @dev: device structure of the PVT platform device.
+ * @hwmon: hwmon device structure.
+ * @regs: pointer to the Eswin PVT registers region.
+ * @irq: PVT events IRQ number.
+ * @clk: PVT core clock (1.2MHz).
+ * @pvt_rst: pointer to the reset descriptor.
+ * @iface_mtx: Generic interface mutex (used to lock the alarm registers
+ *	       when the alarms enabled, or the data conversion interface
+ *	       if alarms are disabled).
+ * @sensor: current PVT sensor the data conversion is being performed for.
+ * @cache: data cache descriptor.
+ * @timeout: conversion timeout cache.
+ */
+struct pvt_hwmon {
+	struct device *dev;
+	struct device *hwmon;
+
+	void __iomem *regs;
+	int irq;
+
+	struct clk *clk;
+	struct reset_control *pvt_rst;
+	struct mutex iface_mtx;
+	enum pvt_sensor_type sensor;
+	struct pvt_cache cache[PVT_SENSORS_NUM];
+	ktime_t timeout;
+};
+
+/*
+ * struct pvt_poly_term - a term descriptor of the PVT data translation
+ *			  polynomial
+ * @deg: degree of the term.
+ * @coef: multiplication factor of the term.
+ * @divider: distributed divider per each degree.
+ * @divider_leftover: divider leftover, which couldn't be redistributed.
+ */
+struct pvt_poly_term {
+	unsigned int deg;
+	long coef;
+	long divider;
+	long divider_leftover;
+};
+
+/*
+ * struct pvt_poly - PVT data translation polynomial descriptor
+ * @total_divider: total data divider.
+ * @terms: polynomial terms up to a free one.
+ */
+struct pvt_poly {
+	long total_divider;
+	struct pvt_poly_term terms[];
+};
+
+#endif /* __HWMON_ESWIN_PVT_H__ */
+
diff --git a/drivers/hwmon/pac193x.c b/drivers/hwmon/pac193x.c
new file mode 100644
index 000000000000..4b40a41fd4c5
--- /dev/null
+++ b/drivers/hwmon/pac193x.c
@@ -0,0 +1,818 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Driver for microchip  pac1931,pac1932,pac1933,pac1934 power monitor chips
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yang Wei <yangwei1@eswincomputing.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/i2c.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/jiffies.h>
+#include <linux/of_device.h>
+#include <linux/of.h>
+#include <linux/delay.h>
+#include <linux/util_macros.h>
+
+#define PAC193X_MAX_CHAN_CNT 4
+
+#define PAC193X_REGISTERS 0x27
+#define PAC193X_VPOWERN_ACC_LEN 6
+#define PAC193X_VOLT_VALUE_LEN 2
+#define PAC193X_VPOWERN_VALUE_LEN 4
+#define PAC193X_ACC_COUNT_LEN 3
+
+#define PAC193X_CMD_REFRESH 0x0
+#define PAC193X_CMD_CTRL 0x1
+#define PAC193X_CMD_ACC_COUNT 0x2
+#define PAC193X_CMD_VPOWER1_ACC 0x3
+#define PAC193X_CMD_VPOWER2_ACC 0x4
+#define PAC193X_CMD_VPOWER3_ACC 0x5
+#define PAC193X_CMD_VPOWER4_ACC 0x6
+#define PAC193X_CMD_VBUS1 0x7
+#define PAC193X_CMD_VBUS2 0x8
+#define PAC193X_CMD_VBUS3 0x9
+#define PAC193X_CMD_VBUS4 0xa
+#define PAC193X_CMD_VSENSE1 0xb
+#define PAC193X_CMD_VSENSE2 0xc
+#define PAC193X_CMD_VSENSE3 0xd
+#define PAC193X_CMD_VSENSE4 0xe
+#define PAC193X_CMD_VBUS1_AVG 0xf
+#define PAC193X_CMD_VBUS2_AVG 0x10
+#define PAC193X_CMD_VBUS3_AVG 0x11
+#define PAC193X_CMD_VBUS4_AVG 0x12
+#define PAC193X_CMD_VSENSE1_AVG 0x13
+#define PAC193X_CMD_VSENSE2_AVG 0x14
+#define PAC193X_CMD_VSENSE3_AVG 0x15
+#define PAC193X_CMD_VSENSE4_AVG 0x16
+#define PAC193X_CMD_VPOWER1 0x17
+#define PAC193X_CMD_VPOWER2 0x18
+#define PAC193X_CMD_VPOWER3 0x19
+#define PAC193X_CMD_VPOWER4 0x1a
+#define PAC193X_CMD_CHANNEL_SMBUS 0x1c
+#define PAC193X_CMD_NEG_PWR 0x1D
+#define PAC193X_CMD_REFRESH_G 0x1E
+#define PAC193X_CMD_REFRESH_V 0x1F
+#define PAC193X_CMD_SLOW 0x20
+#define PAC193X_CMD_CTRL_ACT 0x21
+#define PAC193X_CMD_DIS_ACT 0x22
+#define PAC193X_CMD_NEG_PWR_ACT 0x23
+#define PAC193X_CMD_CTRL_LAT 0x24
+#define PAC193X_CMD_DIS_LAT 0x25
+#define PAC193X_CMD_NEG_PWR_LAT 0x26
+#define PAC193X_CMD_PID 0xFD
+#define PAC193X_CMD_MID 0xFE
+#define PAC193X_CMD_REVERSION_ID 0xFF
+
+#define PAC1932X_COSTANT_PWR_M 3200000000ull /* 3.2V^2*1000mO*/
+#define PAC1932X_COSTANT_CURRENT_M 100000 /* 100mv*1000mO*/
+
+struct pac193x_data {
+	struct mutex config_lock;
+	struct i2c_client *client;
+	u32 update_time_ms;
+	u32 energy_acc_count;
+	struct workqueue_struct *update_workqueue;
+	struct delayed_work update_work;
+	u32 vbus_denominator[PAC193X_MAX_CHAN_CNT];
+	u32 vsense_denominator[PAC193X_MAX_CHAN_CNT];
+	u32 vpower_denominator[PAC193X_MAX_CHAN_CNT];
+	u32 sense_resistances[PAC193X_MAX_CHAN_CNT];
+	u32 sample_rate;
+};
+
+enum PAC193X_CHAN_INDEX {
+	pac1931_index = 0,
+	pac1932_index,
+	pac1933_index,
+	pac1934_index
+};
+
+static int pac193x_get_energy(struct device *dev, u8 commmd, u8 chan, long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u64 cache = 0;
+	u8 *pcache = (u8 *)&cache;
+	u64 energy_value = 0;
+
+	commmd = commmd + chan;
+	dev_dbg(dev, "%s.%d,chan:%d,commad:%d,LEN:%ld\n", __FUNCTION__,
+		__LINE__, chan, commmd, sizeof(cache));
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_i2c_block_data(data->client, commmd,
+					    PAC193X_VPOWERN_ACC_LEN, pcache);
+	mutex_unlock(&data->config_lock);
+
+	energy_value = ((cache & 0xff) << 40) | (((cache >> 8) & 0xff) << 32) |
+		       (((cache >> 16) & 0xff) << 24) |
+		       (((cache >> 24) & 0xff) << 16) |
+		       (((cache >> 32) & 0xff) << 8) | (((cache >> 40) & 0xff));
+	energy_value = energy_value & 0xffffffffffffULL;
+	dev_dbg(dev,
+		"%s.%d,commd:0x%x,value:%lld,ret:%d,resistances:%u,denominator:%u,sample_rate:%u\n",
+		__FUNCTION__, __LINE__, commmd, energy_value, ret,
+		data->sense_resistances[chan], data->vpower_denominator[chan],
+		data->sample_rate);
+
+	/* energy=3200000*Vpower/(Rsense*denominator*sample_rate) */
+	energy_value = ((energy_value / (u64)data->vpower_denominator[chan]) *
+			(PAC1932X_COSTANT_PWR_M /
+			 (data->sense_resistances[chan] * data->sample_rate)));
+
+	*val = energy_value;
+
+	return 0;
+}
+
+static u16 pac193x_get_word(struct pac193x_data *data, u8 commmd)
+{
+	int ret;
+	u16 cache = 0;
+
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_word_data(data->client, commmd);
+	mutex_unlock(&data->config_lock);
+	cache = ((ret & 0xff) << 8) | ((ret >> 8) & 0xff);
+	dev_dbg(&data->client->dev, "%s.%d,commd:0x%x,value:0x%x,ret:0x%x\n",
+		__FUNCTION__, __LINE__, commmd, cache, ret);
+
+	return cache;
+}
+
+static int pac193x_get_volt(struct device *dev, u8 commmd, u8 chan, long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u16 cache = 0;
+
+	commmd = commmd + chan;
+	dev_dbg(dev, "%s.%d,commd:0x%x,chan:%d,vbus_denominator:%d\n",
+		__FUNCTION__, __LINE__, commmd, chan,
+		data->vbus_denominator[chan]);
+	cache = pac193x_get_word(data, commmd);
+	/*to mV*/
+	cache = cache * 1000 / data->vbus_denominator[chan];
+	*val = cache;
+	return 0;
+}
+
+static int pac193x_get_current(struct device *dev, u8 commmd, u8 chan,
+			       long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u16 cache = 0;
+
+	commmd = commmd + chan;
+	dev_dbg(dev,
+		"%s.%d,commd:0x%x,chan:%d,vbus_denominator:%d,resistances:%d\n",
+		__FUNCTION__, __LINE__, commmd, chan,
+		data->vsense_denominator[chan], data->sense_resistances[chan]);
+	cache = pac193x_get_word(data, commmd);
+	/* I=Vsense*100/(Rsense*denominator)	*/
+	cache = cache * PAC1932X_COSTANT_CURRENT_M /
+		(data->vsense_denominator[chan] *
+		 data->sense_resistances[chan]);
+	*val = cache;
+	return 0;
+}
+
+static int pac193x_get_power(struct device *dev, u8 commmd, u8 chan, long *val)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u32 cache = 0;
+	u8 *pcache = (u8 *)&cache;
+	u64 pwr_value = 0;
+
+	commmd = commmd + chan;
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_i2c_block_data(data->client, commmd,
+					    PAC193X_VPOWERN_VALUE_LEN, pcache);
+	mutex_unlock(&data->config_lock);
+	pwr_value = ((cache & 0xff) << 24) | (((cache >> 8) & 0xff) << 16) |
+		    (((cache >> 16) & 0xff) << 8) | ((cache >> 24) & 0xff);
+	pwr_value = pwr_value >> 4;
+	dev_dbg(dev,
+		"%s.%d,commd:0x%x,chan:%d,value:0x%x,pwr_value:0x%llx,%llu,ret:%d,resistances:%u,denominator:%u\n",
+		__FUNCTION__, __LINE__, commmd, chan, cache, pwr_value,
+		pwr_value, ret, data->sense_resistances[chan],
+		data->vpower_denominator[chan]);
+	/* pwr=3200000*Vpower/Rsense*denominator */
+	pwr_value = pwr_value * PAC1932X_COSTANT_PWR_M /
+		    ((u64)data->sense_resistances[chan] *
+		     (u64)data->vpower_denominator[chan]);
+	*val = pwr_value;
+	return 0;
+}
+
+static int pac193x_send_refresh_cmd(struct pac193x_data *data, u8 command)
+{
+	int ret = 0;
+
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_write_byte(data->client, command);
+	mutex_unlock(&data->config_lock);
+	dev_dbg(&data->client->dev, "%s.%d,commd:0x%x,ret:%d\n", __FUNCTION__,
+		__LINE__, command, ret);
+	return ret;
+}
+
+static ssize_t pac193x_refresh_store(struct device *dev,
+				     struct device_attribute *da,
+				     const char *buf, size_t count)
+{
+	struct sensor_device_attribute *sensor_attr = to_sensor_dev_attr(da);
+	struct pac193x_data *data = dev_get_drvdata(dev);
+
+	pac193x_send_refresh_cmd(data, (u8)sensor_attr->index);
+	return count;
+}
+
+static struct sensor_device_attribute pac1934_refreshs[] = {
+	SENSOR_ATTR_WO(refresh_clear_acc, pac193x_refresh, PAC193X_CMD_REFRESH),
+	SENSOR_ATTR_WO(refresh_all_193x, pac193x_refresh,
+		       PAC193X_CMD_REFRESH_G),
+	SENSOR_ATTR_WO(refresh_updata_value, pac193x_refresh,
+		       PAC193X_CMD_REFRESH_V),
+};
+
+static u8 pac193x_read_byte_data(struct pac193x_data *data, u8 command)
+{
+	int cache = 0;
+	int cnt = 0;
+	while (1) {
+		mutex_lock(&data->config_lock);
+		cache = i2c_smbus_read_byte_data(data->client, command);
+		mutex_unlock(&data->config_lock);
+		if (0xff != cache) {
+			break;
+		}
+		cnt++;
+		if (cnt > 100) {
+			dev_err(&data->client->dev,
+				"get command:%d value error\n", command);
+			return 0xff;
+		}
+	}
+	return (u8)cache;
+}
+
+static ssize_t pac193x_ctrl_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0, act_val = 0, lat_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL);
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL_ACT);
+	lat_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL_LAT);
+
+	return sprintf(buf,
+		       "%16s:%6s,%6s,%6s\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n",
+		       "", "set", "act", "latch",
+			   "sample_rate",(set_val >> 6) & 0x3, (act_val >> 6) & 0x3, (lat_val >> 6) & 0x3,
+			   "SLEEP", (set_val >> 5) & 0x1,(act_val >> 5) & 0x1, (lat_val >> 5) & 0x1,
+			   "SING", (set_val >> 4) & 0x1, (act_val >> 4) & 0x1, (lat_val >> 4) & 0x1,
+			   "ALERT_PIN", (set_val >> 3) & 0x1, (act_val >> 3) & 0x1, (lat_val >> 3) & 0x1,
+			   "ALERT_CC", (set_val >> 2) & 0x1, (act_val >> 2) & 0x1, (lat_val >> 2) & 0x1,
+			   "OVF ALERT", (set_val >> 1) & 0x1, (act_val >> 1) & 0x1, (lat_val >> 1) & 0x1,
+			   "OVF", (set_val >> 0) & 0x1, (act_val >> 0) & 0x1, (lat_val >> 0) & 0x1);
+}
+
+int pac193x_common_reg_set(struct device *dev, const char *buf, u8 commad)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	unsigned long val;
+	int status;
+	int ret = 0;
+
+	status = kstrtoul(buf, 10, &val);
+	if (status < 0)
+		return status;
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_write_byte_data(data->client, commad, (u8)val);
+	mutex_unlock(&data->config_lock);
+	return ret;
+}
+
+static ssize_t pac193x_ctrl_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_CTRL);
+}
+
+DEVICE_ATTR(control, S_IWUSR | S_IRUGO, pac193x_ctrl_show, pac193x_ctrl_store);
+
+static ssize_t pac193x_acc_count_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u32 cache = 0;
+	u8 *pcache = (u8 *)&cache;
+	u32 acc_cnt = 0;
+	mutex_lock(&data->config_lock);
+	ret = i2c_smbus_read_i2c_block_data(data->client, PAC193X_CMD_ACC_COUNT,
+					    PAC193X_ACC_COUNT_LEN, pcache);
+	mutex_unlock(&data->config_lock);
+
+	acc_cnt = ((cache & 0xff) << 16) | (((cache >> 8) & 0xff) << 8) |
+		  ((cache >> 16) & 0xff);
+	acc_cnt = acc_cnt & 0xffffff;
+
+	ret = sysfs_emit(buf, "%u\n", acc_cnt);
+	return ret;
+}
+
+DEVICE_ATTR(acc_count, 0400, pac193x_acc_count_show, NULL);
+
+static ssize_t pac193x_dis_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0, act_val = 0, lat_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_CHANNEL_SMBUS);
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_DIS_ACT);
+	lat_val = pac193x_read_byte_data(data, PAC193X_CMD_DIS_LAT);
+
+	return sprintf(buf,
+		       "%16s:%6s,%6s,%6s\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n",
+		       "", "set", "act", "latch",
+			   "CH1_OFF", (set_val >> 7) & 0x1, (act_val >> 7) & 0x1, (lat_val >> 7) & 0x1,
+			   "CH2_OFF", (set_val >> 6) & 0x1, (act_val >> 6) & 0x1, (lat_val >> 6) & 0x1,
+			   "CH3_OFF", (set_val >> 5) & 0x1, (act_val >> 5) & 0x1, (lat_val >> 5) & 0x1,
+			   "CH4_OFF", (set_val >> 4) & 0x1, (act_val >> 4) & 0x1, (lat_val >> 4) & 0x1,
+			   "TIMEOUT", (set_val >> 3) & 0x1, (act_val >> 3) & 0x1, (lat_val >> 3) & 0x1,
+			   "BYTE COUNT", (set_val >> 2) & 0x1, (act_val >> 2) & 0x1, (lat_val >> 2) & 0x1,
+			   "NO SKIP", (set_val >> 1) & 0x1, (act_val >> 1) & 0x1, (lat_val >> 1) & 0x1);
+}
+
+static ssize_t pac193x_dis_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_CHANNEL_SMBUS);
+}
+
+DEVICE_ATTR(disable_chan_pmbus, S_IWUSR | S_IRUGO, pac193x_dis_show,
+	    pac193x_dis_store);
+
+static ssize_t pac193x_neg_pwr_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0, act_val = 0, lat_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR);
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR_ACT);
+	lat_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR_LAT);
+
+	return sprintf(buf,
+		       "%16s:%6s,%6s,%6s\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n"
+		       "%16s:%6d,%6d,%6d\n",
+		       "", "set", "act", "latch",
+			   "CH1_BIDI", (set_val >> 7) & 0x1, (act_val >> 7) & 0x1, (lat_val >> 7) & 0x1,
+			   "CH2_BIDI", (set_val >> 6) & 0x1, (act_val >> 6) & 0x1, (lat_val >> 6) & 0x1,
+			   "CH3_BIDI", (set_val >> 5) & 0x1, (act_val >> 5) & 0x1, (lat_val >> 5) & 0x1,
+			   "CH4_BIDI", (set_val >> 4) & 0x1, (act_val >> 4) & 0x1, (lat_val >> 5) & 0x1,
+			   "CH1_BIDV", (set_val >> 3) & 0x1, (act_val >> 3) & 0x1, (lat_val >> 3) & 0x1,
+			   "CH2_BIDV", (set_val >> 2) & 0x1, (act_val >> 2) & 0x1, (lat_val >> 2) & 0x1,
+			   "CH3_BIDV", (set_val >> 1) & 0x1, (act_val >> 1) & 0x1, (lat_val >> 1) & 0x1,
+			   "CH4_BIDV", (set_val >> 0) & 0x1, (act_val >> 0) & 0x1, (lat_val >> 0) & 0x1);
+}
+
+static ssize_t pac193x_neg_pwr_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_NEG_PWR);
+}
+
+DEVICE_ATTR(neg_pwr, S_IWUSR | S_IRUGO, pac193x_neg_pwr_show,
+	    pac193x_neg_pwr_store);
+
+static ssize_t pac193x_slow_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	u8 set_val = 0;
+
+	set_val = pac193x_read_byte_data(data, PAC193X_CMD_SLOW);
+
+	return sprintf(buf,
+		       "SLOW:%d,SLOW-LH:%d,SLOW-HL:%d, R_RISE:%d,"
+		       "R_V_RISE:%d,R_FALL:%d,R_V_FALL:%d,POR:%d\n",
+		       (set_val >> 7) & 0x1, (set_val >> 6) & 0x1,
+		       (set_val >> 5) & 0x1, (set_val >> 4) & 0x1,
+		       (set_val >> 3) & 0x1, (set_val >> 2) & 0x1,
+		       (set_val >> 1) & 0x1, (set_val >> 0) & 0x1);
+}
+
+static ssize_t pac193x_slow_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	return pac193x_common_reg_set(dev, buf, PAC193X_CMD_SLOW);
+}
+
+DEVICE_ATTR(slow_ctrl, S_IWUSR | S_IRUGO, pac193x_slow_show,
+	    pac193x_slow_store);
+
+static ssize_t pac193x_version_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct pac193x_data *data = dev_get_drvdata(dev);
+	int ret;
+	u8 pid = 0, mid = 0, rid = 0;
+
+	pid = pac193x_read_byte_data(data, PAC193X_CMD_PID);
+	mid = pac193x_read_byte_data(data, PAC193X_CMD_MID);
+	rid = pac193x_read_byte_data(data, PAC193X_CMD_REVERSION_ID);
+
+	ret = sysfs_emit(buf, "PID:0x%x,MID:0x%x,RID:0x%x\n", pid, mid, rid);
+	return ret;
+}
+DEVICE_ATTR(pac193x_version, 0400, pac193x_version_show, NULL);
+
+static struct attribute *pac193x_attrs[] = {
+	&dev_attr_control.attr,
+	&dev_attr_acc_count.attr,
+	&dev_attr_neg_pwr.attr,
+	&dev_attr_disable_chan_pmbus.attr,
+	&dev_attr_slow_ctrl.attr,
+	&dev_attr_pac193x_version.attr,
+	&pac1934_refreshs[0].dev_attr.attr,
+	&pac1934_refreshs[1].dev_attr.attr,
+	&pac1934_refreshs[2].dev_attr.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(pac193x);
+
+static const struct hwmon_channel_info *pac1931_info[] = {
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_AVERAGE,
+			    HWMON_I_INPUT | HWMON_I_AVERAGE),
+	HWMON_CHANNEL_INFO(curr, HWMON_C_INPUT | HWMON_C_AVERAGE),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT), NULL
+};
+static const struct hwmon_channel_info *pac1932_info[] = {
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE),
+	HWMON_CHANNEL_INFO(curr, HWMON_C_INPUT | HWMON_C_AVERAGE,
+			   HWMON_C_INPUT | HWMON_C_AVERAGE),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT, HWMON_P_INPUT),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT, HWMON_E_INPUT), NULL
+};
+static const struct hwmon_channel_info *pac1933_info[] = {
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE),
+	HWMON_CHANNEL_INFO(curr, HWMON_C_INPUT | HWMON_C_AVERAGE,
+			   HWMON_C_INPUT | HWMON_C_AVERAGE,
+			   HWMON_C_INPUT | HWMON_C_AVERAGE),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT, HWMON_P_INPUT, HWMON_P_INPUT),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT, HWMON_E_INPUT, HWMON_E_INPUT),
+	NULL
+};
+static const struct hwmon_channel_info *pac1934_info[] = {
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE,
+			   HWMON_I_INPUT | HWMON_I_AVERAGE),
+	HWMON_CHANNEL_INFO(curr,
+			   HWMON_C_INPUT | HWMON_C_AVERAGE,
+			   HWMON_C_INPUT | HWMON_C_AVERAGE,
+			   HWMON_C_INPUT | HWMON_C_AVERAGE,
+			   HWMON_C_INPUT | HWMON_C_AVERAGE),
+	HWMON_CHANNEL_INFO(power, HWMON_P_INPUT, HWMON_P_INPUT, HWMON_P_INPUT,
+			   HWMON_P_INPUT),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT, HWMON_E_INPUT, HWMON_E_INPUT,
+			   HWMON_E_INPUT),
+	NULL
+};
+
+static umode_t pac1934x_is_visible(const void *_data,
+				   enum hwmon_sensor_types type, u32 attr,
+				   int channel)
+{
+	switch (type) {
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+		case hwmon_in_average:
+			if(channel == 0)
+			{
+				return 0;
+			}
+			else
+			{
+				return S_IRUGO;
+			}
+		}
+		break;
+	case hwmon_curr:
+		switch (attr) {
+		case hwmon_curr_input:
+		case hwmon_curr_average:
+			return S_IRUGO;
+		}
+		break;
+	case hwmon_power:
+		switch (attr) {
+		case hwmon_power_input:
+			return S_IRUGO;
+		}
+		break;
+	case hwmon_energy:
+		switch (attr) {
+		case hwmon_energy_input:
+			return S_IRUGO;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int pac193x_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
+{
+	switch (type) {
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+			return pac193x_get_volt(dev, PAC193X_CMD_VBUS1, channel - 1,
+						val);
+		case hwmon_in_average:
+			return pac193x_get_volt(dev, PAC193X_CMD_VBUS1_AVG,
+						channel - 1, val);
+		}
+		break;
+	case hwmon_curr:
+		switch (attr) {
+		case hwmon_curr_input:
+			return pac193x_get_current(dev, PAC193X_CMD_VSENSE1,
+						   channel, val);
+		case hwmon_curr_average:
+			return pac193x_get_current(dev, PAC193X_CMD_VSENSE1_AVG,
+						   channel, val);
+		}
+		break;
+	case hwmon_power:
+		switch (attr) {
+		case hwmon_power_input:
+			return pac193x_get_power(dev, PAC193X_CMD_VPOWER1,
+						 channel, val);
+		}
+		break;
+	case hwmon_energy:
+		switch (attr) {
+		case hwmon_energy_input:
+			return pac193x_get_energy(dev, PAC193X_CMD_VPOWER1_ACC,
+						  channel, val);
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static const struct hwmon_ops pac193x_hwmon_ops = {
+	.is_visible = pac1934x_is_visible,
+	.read = pac193x_read,
+};
+
+static struct hwmon_chip_info pac193x_chip_info = {
+	.ops = &pac193x_hwmon_ops,
+	.info = NULL,
+};
+
+static void update_reg_data(struct work_struct *work)
+{
+	static u32 updata_cnt = 0;
+	struct pac193x_data *data;
+	int act_val = 0, ctrl_val = 0, slow_val = 0;
+	int num = 0;
+	bool is_neg = false;
+
+	data = container_of(work, struct pac193x_data, update_work.work);
+	if ((data->energy_acc_count == 0)||(updata_cnt < data->energy_acc_count)) {
+		pac193x_send_refresh_cmd(data, PAC193X_CMD_REFRESH_V);
+		updata_cnt++;
+	} else {
+		pac193x_send_refresh_cmd(data, PAC193X_CMD_REFRESH);
+		updata_cnt = 0;
+	}
+
+	act_val = pac193x_read_byte_data(data, PAC193X_CMD_NEG_PWR_ACT);
+
+	for (num = 0; num < PAC193X_MAX_CHAN_CNT; num++) {
+		is_neg = false;
+		if (0x1 == ((act_val >> num) & 0x1)) {
+			/* Vsource=32*Vbus/2^15 = Vbus/2^10=Vbus/1024  */
+			data->vbus_denominator[3 - num] = 1024;
+			is_neg = true;
+		} else {
+			/* Vsource=32*Vbus/2^16 = Vbus/2^10=Vbus/1024  */
+			data->vbus_denominator[3 - num] = 2048;
+		}
+
+		if (0x1 == ((act_val >> (num + 4)) & 0x1)) {
+			/* 2^15  */
+			data->vsense_denominator[3 - num] = 32768;
+			is_neg = true;
+		} else {
+			/*2^16 */
+			data->vsense_denominator[3 - num] = 65536;
+		}
+		if (true == is_neg) {
+			/* 2^28  */
+			data->vpower_denominator[3 - num] = 134217728;
+		} else {
+			/* 2^29  */
+			data->vpower_denominator[3 - num] = 268435456;
+		}
+	}
+
+	slow_val = pac193x_read_byte_data(data, PAC193X_CMD_SLOW);
+	ctrl_val = pac193x_read_byte_data(data, PAC193X_CMD_CTRL_ACT);
+	if ((0x1 == ((slow_val >> 7) & 0x1)) &&
+	    (0x0 == ((ctrl_val >> 3) & 0x1))) {
+		data->sample_rate = 8;
+	} else {
+		switch ((ctrl_val >> 6) & 0x3) {
+		case 0:
+			data->sample_rate = 1024;
+			break;
+		case 1:
+			data->sample_rate = 256;
+			break;
+		case 2:
+			data->sample_rate = 64;
+			break;
+		case 3:
+			data->sample_rate = 8;
+			break;
+		default:
+			break;
+		}
+	}
+	queue_delayed_work(data->update_workqueue, &data->update_work,
+			   msecs_to_jiffies(data->update_time_ms));
+}
+
+static int pac193x_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct pac193x_data *data;
+	struct device *hwmon_dev;
+	int ret = 0;
+	int num = 0;
+	enum PAC193X_CHAN_INDEX chan_index =
+		(enum PAC193X_CHAN_INDEX)of_device_get_match_data(&client->dev);
+
+	switch (chan_index) {
+	case pac1931_index:
+		pac193x_chip_info.info = pac1931_info;
+		break;
+	case pac1932_index:
+		pac193x_chip_info.info = pac1932_info;
+		break;
+	case pac1933_index:
+		pac193x_chip_info.info = pac1933_info;
+		break;
+	case pac1934_index:
+		pac193x_chip_info.info = pac1934_info;
+		break;
+	default:
+		break;
+	}
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	for (num = 0; num < PAC193X_MAX_CHAN_CNT; num++) {
+		data->vbus_denominator[num] = 1;
+		data->vsense_denominator[num] = 1;
+		data->sense_resistances[num] = 1;
+	}
+
+	ret = of_property_read_u32(dev->of_node, "update_time_ms",
+				   &data->update_time_ms);
+	if (0 != ret) {
+		dev_err(dev, "can not get update_time_ms:%d\n", ret);
+		data->update_time_ms = 100;
+	}
+
+	ret = of_property_read_u32(dev->of_node, "energy_acc_count",
+				   &data->energy_acc_count);
+	if (0 != ret) {
+		dev_err(dev, "can not get energy_acc_count:%d\n", ret);
+		data->energy_acc_count = 0;
+	}
+	ret = of_property_read_u32_array(dev->of_node, "shunt_resistors",
+					 data->sense_resistances,
+					 PAC193X_MAX_CHAN_CNT);
+	if (0 != ret) {
+		dev_err(dev, "can not get sense_resistances:%d\n", ret);
+	}
+
+	dev_info(dev,
+		"update_time:%d,energy_acc_count:%d,resistances:%d,%d,%d,%dmOhm\n",
+		data->update_time_ms, data->energy_acc_count,
+		data->sense_resistances[0], data->sense_resistances[1],
+		data->sense_resistances[2], data->sense_resistances[3]);
+	mutex_init(&data->config_lock);
+	data->client = client;
+	mutex_lock(&data->config_lock);
+	i2c_smbus_write_byte_data(data->client, PAC193X_CMD_CTRL, 0x8);
+	mutex_unlock(&data->config_lock);
+	pac193x_send_refresh_cmd(data, PAC193X_CMD_REFRESH);
+
+	hwmon_dev = devm_hwmon_device_register_with_info(
+		dev, client->name, data, &pac193x_chip_info, pac193x_groups);
+	if (IS_ERR(hwmon_dev))
+		return PTR_ERR(hwmon_dev);
+
+	data->update_workqueue = create_workqueue("update_workqueue");
+
+	INIT_DELAYED_WORK(&data->update_work, update_reg_data);
+	queue_delayed_work(data->update_workqueue, &data->update_work,
+			   msecs_to_jiffies(0));
+
+	return 0;
+}
+
+static const struct of_device_id __maybe_unused pac193x_of_match[] = {
+	{ .compatible = "microchip,pac1931", .data = (void *)pac1931_index },
+	{ .compatible = "microchip,pac1932", .data = (void *)pac1932_index },
+	{ .compatible = "microchip,pac1933", .data = (void *)pac1933_index },
+	{ .compatible = "microchip,pac1934", .data = (void *)pac1934_index },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pac193x_of_match);
+
+static struct i2c_driver pac193x_driver = {
+	.driver = {
+		.name	= "pac193x",
+		.of_match_table = of_match_ptr(pac193x_of_match),
+	},
+	.probe = pac193x_probe,
+};
+
+module_i2c_driver(pac193x_driver);
+
+MODULE_AUTHOR("Yang Wei <yangwei1@eswincomputing.com");
+MODULE_DESCRIPTION("pac193x driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index d57c5adf932e..d41bf5699987 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -7,6 +7,12 @@ config IOMMU_IOVA
 config IOMMU_API
 	bool
 
+config IOMMU_ESWIN
+	bool "IOMMU EIC7700 Support"
+	help
+	  Say Y here if you want to compile device drivers for IO Memory
+	  management implementation for EIC7700
+
 menuconfig IOMMU_SUPPORT
 	bool "IOMMU Hardware Support"
 	depends on MMU
@@ -28,7 +34,7 @@ config IOMMU_IO_PGTABLE
 config IOMMU_IO_PGTABLE_LPAE
 	bool "ARMv7/v8 Long Descriptor Format"
 	select IOMMU_IO_PGTABLE
-	depends on ARM || ARM64 || COMPILE_TEST
+	depends on ARM || ARM64 || RISCV || COMPILE_TEST
 	depends on !GENERIC_ATOMIC64	# for cmpxchg64()
 	help
 	  Enable support for the ARM long descriptor pagetable format.
@@ -389,7 +395,7 @@ config ARM_SMMU_QCOM_DEBUG
 
 config ARM_SMMU_V3
 	tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
-	depends on ARM64
+	depends on ARM64 || RISCV
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_LPAE
 	select GENERIC_MSI_IRQ
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 769e43d780ce..91c3ed52daf9 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-y += amd/ intel/ arm/ iommufd/
+obj-$(CONFIG_IOMMU_ESWIN) += eswin/
 obj-$(CONFIG_IOMMU_API) += iommu.o
 obj-$(CONFIG_IOMMU_API) += iommu-traces.o
 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 68b81f9c2f4b..cd7f1f6bf304 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -30,8 +30,24 @@
 #include "arm-smmu-v3.h"
 #include "../../dma-iommu.h"
 #include "../../iommu-sva.h"
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+#include <dt-bindings/memory/eic7700-sid.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 
+#define ESWIN_SMMU_IRQ_CLEAR_REG	1
+
+/* smmu interrupt clear bits */
+#define TCU_U84_EVENT_Q_IRPT_NS_CLR_BIT     9
+#define TCU_U84_PRI_Q_IRPT_NS_CLR_BIT       10
+#define TCU_U84_GLOBAL_IRPT_NS_CLR_BIT      13
+#endif
+
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+static bool disable_bypass = false;
+#else
 static bool disable_bypass = true;
+#endif
 module_param(disable_bypass, bool, 0444);
 MODULE_PARM_DESC(disable_bypass,
 	"Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
@@ -1349,6 +1365,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
 			STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
 
 		BUG_ON(ste_live);
+
 		dst[1] = cpu_to_le64(
 			 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
 			 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
@@ -1560,6 +1577,33 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
 	return ret;
 }
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+static void eswin_smmu_irq_clear(struct arm_smmu_device *smmu, int clearbit)
+{
+	int bitmask;
+	bitmask = BIT(clearbit);
+
+	regmap_write(smmu->regmap, smmu->smmu_irq_clear_reg, bitmask);
+}
+
+static irqreturn_t eswin_smmu_irq_clear_handler(int irq, void *dev)
+{
+	struct arm_smmu_device *smmu = dev;
+
+	if (irq == smmu->evtq.q.irq) {
+			eswin_smmu_irq_clear(smmu, TCU_U84_EVENT_Q_IRPT_NS_CLR_BIT);
+	}
+	else if (irq == smmu->priq.q.irq) {
+			eswin_smmu_irq_clear(smmu, TCU_U84_PRI_Q_IRPT_NS_CLR_BIT);
+	}
+	else {
+		return IRQ_NONE;
+	}
+
+    return IRQ_WAKE_THREAD;
+}
+#endif
+
 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
 {
 	int i, ret;
@@ -1664,6 +1708,10 @@ static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
 	u32 gerror, gerrorn, active;
 	struct arm_smmu_device *smmu = dev;
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	eswin_smmu_irq_clear(smmu, TCU_U84_GLOBAL_IRPT_NS_CLR_BIT);
+#endif
+
 	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
 	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
 
@@ -2402,6 +2450,7 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master)
 
 	master->domain = NULL;
 	master->ats_enabled = false;
+
 	arm_smmu_install_ste_for_dev(master);
 }
 
@@ -2453,7 +2502,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 		ret = -EINVAL;
 		goto out_unlock;
 	}
-
 	master->domain = smmu_domain;
 
 	/*
@@ -2617,7 +2665,6 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
 		}
 		if (ret)
 			break;
-
 		rb_link_node(&new_stream->node, parent_node, new_node);
 		rb_insert_color(&new_stream->node, &smmu->streams);
 	}
@@ -3205,10 +3252,17 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
 	/* Request interrupt lines */
 	irq = smmu->evtq.q.irq;
 	if (irq) {
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+		ret = devm_request_threaded_irq(smmu->dev, irq, eswin_smmu_irq_clear_handler,
+						arm_smmu_evtq_thread,
+						IRQF_ONESHOT,
+						"arm-smmu-v3-evtq", smmu);
+#else
 		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 						arm_smmu_evtq_thread,
 						IRQF_ONESHOT,
 						"arm-smmu-v3-evtq", smmu);
+#endif
 		if (ret < 0)
 			dev_warn(smmu->dev, "failed to enable evtq irq\n");
 	} else {
@@ -3228,11 +3282,19 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
 		irq = smmu->priq.q.irq;
 		if (irq) {
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+			ret = devm_request_threaded_irq(smmu->dev, irq, eswin_smmu_irq_clear_handler,
+							arm_smmu_priq_thread,
+							IRQF_ONESHOT,
+							"arm-smmu-v3-priq",
+							smmu);
+#else
 			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 							arm_smmu_priq_thread,
 							IRQF_ONESHOT,
 							"arm-smmu-v3-priq",
 							smmu);
+#endif
 			if (ret < 0)
 				dev_warn(smmu->dev,
 					 "failed to enable priq irq\n");
@@ -3523,8 +3585,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
 
 	if (reg & IDR0_HYP) {
 		smmu->features |= ARM_SMMU_FEAT_HYP;
+		#ifdef CONFIG_ARM64
 		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
 			smmu->features |= ARM_SMMU_FEAT_E2H;
+		#endif
 	}
 
 	/*
@@ -3849,8 +3913,23 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 		smmu->page1 = smmu->base;
 	}
 
-	/* Interrupt lines */
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	/* eswin, syscon devie is used for clearing the smmu interrupt */
+	smmu->regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (IS_ERR(smmu->regmap)) {
+		dev_err(smmu->dev, "No syscfg phandle specified\n");
+		return PTR_ERR(smmu->regmap);
+	}
 
+	ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg", ESWIN_SMMU_IRQ_CLEAR_REG,
+					&smmu->smmu_irq_clear_reg);
+	if (ret) {
+		dev_err(dev, "can't get SMMU irq clear reg offset (%d)\n", ret);
+		return ret;
+	}
+#endif
+
+	/* Interrupt lines */
 	irq = platform_get_irq_byname_optional(pdev, "combined");
 	if (irq > 0)
 		smmu->combined_irq = irq;
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 9915850dd4db..36da241ec165 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -682,6 +682,11 @@ struct arm_smmu_device {
 
 	struct rb_root			streams;
 	struct mutex			streams_mutex;
+
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	struct regmap *regmap;
+	int     smmu_irq_clear_reg;
+#endif
 };
 
 struct arm_smmu_stream {
diff --git a/drivers/iommu/eswin/Makefile b/drivers/iommu/eswin/Makefile
new file mode 100644
index 000000000000..d63610d53d4f
--- /dev/null
+++ b/drivers/iommu/eswin/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_IOMMU_ESWIN) += eic7700-sid.o
diff --git a/drivers/iommu/eswin/eic7700-sid.c b/drivers/iommu/eswin/eic7700-sid.c
new file mode 100644
index 000000000000..33fbfa4fb53b
--- /dev/null
+++ b/drivers/iommu/eswin/eic7700-sid.c
@@ -0,0 +1,661 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 ESWIN.  All rights reserved.
+ * Author: Lin Min <linmin@eswin.com>
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/bitfield.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/regmap.h>
+#include <linux/iommu.h>
+#include <linux/mfd/syscon.h>
+#include <dt-bindings/memory/eic7700-sid.h>
+
+#define DYMN_CSR_EN_REG_OFFSET            0x0
+#define DYMN_CSR_GNT_REG_OFFSET           0x4
+
+#define MCPU_SP0_DYMN_CSR_EN_BIT    3
+#define MCPU_SP0_DYMN_CSR_GNT_BIT   3
+
+#define AWSMMUSID	GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID	GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID	GENMASK(15, 8)	// The sid of read operation
+#define ARSMMUSSID	GENMASK(7, 0)	// The ssid of read operation
+
+struct eic7700_sid_client {
+	const char *name;
+	unsigned int sid;
+	unsigned int reg_offset;
+};
+
+struct eic7700_sid_soc {
+	const struct eic7700_sid_client *clients;
+	unsigned int num_clients;
+};
+
+/* The syscon registers for tbu power up(down) must be configured so that
+    tcu can be aware of tbu up and down.
+
+ */
+struct tbu_reg_cfg_info {
+	unsigned int reg_offset;
+	unsigned int qreqn_pd_bit;
+	unsigned int qacceptn_pd_bit;
+};
+
+struct tbu_priv {
+	atomic_t refcount;
+	int nid;
+	const struct eic7700_tbu_client *tbu_client_p;
+	struct mutex tbu_priv_lock;
+};
+
+struct eic7700_tbu_client {
+	/* tbu_id: bit[3:0] is for major ID, bit[7:4] is for minor ID;
+	   For example, tbu of dsp3 is tbu7_3, the tbu_ID is 0x73. It measn tbu7_3
+	*/
+	u32 tbu_id;
+	struct tbu_reg_cfg_info tbu_reg_info;
+	int (*tbu_power_ctl_register) (int nid, struct tbu_priv *tbu_priv_p, bool is_powerUp);
+};
+
+struct eic7700_tbu_soc {
+	const struct eic7700_tbu_client *tbu_clients;
+	unsigned int num_tbuClients;
+};
+
+
+
+struct tbu_power_soc {
+	struct tbu_priv *tbu_priv_array;
+	unsigned int num_tbuClients;
+};
+
+struct eic7700_sid {
+	void __iomem *regs;
+	resource_size_t start;
+	const struct eic7700_sid_soc *soc;
+	struct mutex eswin_dynm_sid_cfg_en_lock;
+	struct tbu_power_soc *tbu_power_soc;
+	struct mutex tbu_reg_lock;
+};
+struct eic7700_sid *syscon_sid_cfg[MAX_NUMNODES] = {NULL};
+
+static int eic7700_tbu_power_ctl_register(int nid, struct tbu_priv *tbu_priv_p, bool is_powerUp);
+
+static int eic7700_tbu_powr_priv_init(struct tbu_power_soc **tbu_power_soc_pp, int nid);
+
+int eic7700_dynm_sid_enable(int nid)
+{
+	unsigned long reg_val;
+	struct eic7700_sid *mc = NULL;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		pr_err("%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return -EFAULT;
+	#else
+		pr_info("%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+
+	mc = syscon_sid_cfg[nid];
+	if (mc == NULL)
+		return -EFAULT;
+
+	mutex_lock(&mc->eswin_dynm_sid_cfg_en_lock);
+	reg_val = readl(mc->regs + DYMN_CSR_EN_REG_OFFSET);
+	set_bit(MCPU_SP0_DYMN_CSR_EN_BIT, &reg_val);
+	writel(reg_val, mc->regs + DYMN_CSR_EN_REG_OFFSET);
+
+	while(1) {
+		reg_val = readl(mc->regs + DYMN_CSR_GNT_REG_OFFSET) & (1 << MCPU_SP0_DYMN_CSR_GNT_BIT);
+		if (reg_val)
+			break;
+
+		msleep(10);
+	}
+	reg_val = readl(mc->regs + DYMN_CSR_EN_REG_OFFSET);
+	clear_bit(MCPU_SP0_DYMN_CSR_EN_BIT, &reg_val);
+	writel(reg_val, mc->regs + DYMN_CSR_EN_REG_OFFSET);
+	mutex_unlock(&mc->eswin_dynm_sid_cfg_en_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(eic7700_dynm_sid_enable);
+
+int eic7700_aon_sid_cfg(struct device *dev)
+{
+	int ret = 0;
+	struct regmap *regmap;
+	int aon_sid_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	int i,sid_count;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct device_node *np_syscon;
+	int syscon_cell_size = 0;
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_info(dev, "dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "No eswin,syscfg phandle specified\n");
+		return -1;
+	}
+
+	np_syscon = of_parse_phandle(dev->of_node, "eswin,syscfg", 0);
+	if (np_syscon) {
+		if (of_property_read_u32(np_syscon, "#syscon-cells", &syscon_cell_size)) {
+			of_node_put(np_syscon);
+			dev_err(dev, "failed to get #syscon-cells of sys_con\n");
+			return -1;
+		}
+		of_node_put(np_syscon);
+	}
+
+	sid_count = of_count_phandle_with_args(dev->of_node,
+					"eswin,syscfg", "#syscon-cells");
+
+	dev_dbg(dev, "sid_count=%d, fwspec->num_ids=%d, syscon_cell_size=%d\n",
+			sid_count, fwspec->num_ids, syscon_cell_size);
+
+	if (sid_count < 0) {
+		dev_err(dev, "failed to parse eswin,syscfg property!\n");
+		return -1;
+	}
+
+	if (fwspec->num_ids != sid_count) {
+		dev_err(dev, "num_ids(%d) is NOT equal to num of sid regs(%d)!\n",
+			fwspec->num_ids, sid_count);
+		return -1;
+	}
+
+	for (i = 0; i < sid_count; i++) {
+		sid = fwspec->ids[i];
+		ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg", (syscon_cell_size + 1)*i+1,
+					&aon_sid_reg);
+		if (ret) {
+			dev_err(dev, "can't get sid cfg reg offset in sys_con(errno:%d)\n", ret);
+			return ret;
+		}
+
+		/* make the reading sid the same as writing sid, ssid is fixed to zero */
+		rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+		rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+		rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+		rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+		regmap_write(regmap, aon_sid_reg, rdwr_sid_ssid);
+
+		ret = eic7700_dynm_sid_enable(dev_to_node(dev));
+		if (ret < 0)
+			dev_err(dev, "failed to config streamID(%d) for %s!\n", sid, of_node_full_name(dev->of_node));
+		else
+			dev_info(dev, "success to config dma streamID(%d) for %s!\n", sid, of_node_full_name(dev->of_node));
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(eic7700_aon_sid_cfg);
+
+static int of_parse_syscon_nodes(struct device_node *np, int *nid_p)
+{
+	#ifdef CONFIG_NUMA
+	int nid;
+	int r;
+
+	r = of_property_read_u32(np, "numa-node-id", &nid);
+	if (r)
+		return -EINVAL;
+
+	pr_debug("Syscon on %u\n", nid);
+	if (nid >= MAX_NUMNODES) {
+		pr_warn("Node id %u exceeds maximum value\n", nid);
+		return -EINVAL;
+	}
+	else
+		*nid_p = nid;
+	#else
+		*nid_p = 0;
+	#endif
+
+	pr_debug("%s, nid = %d\n", __func__, *nid_p);
+
+	return 0;
+}
+
+#if defined(CONFIG_RISCV)
+static const struct eic7700_sid_client eic7700_sid_clients[] = {
+	{
+		.name = "scpu",
+		.sid = EIC7700_SID_SCPU,
+		.reg_offset = SCPU_SID_REG_OFFSET,
+	},
+	{
+		.name = "dma1",
+		.sid = EIC7700_SID_DMA1,
+		.reg_offset = DMA1_SID_REG_OFFSET,
+	},
+	//{
+	//	.name = "crypt",
+	//	.sid = EIC7700_SID_CRYPT,
+	//	.reg_offset = CRYPT_SID_REG_OFFSET,
+	//}
+};
+
+static const struct eic7700_sid_soc eic7700_sid_soc = {
+	.num_clients = ARRAY_SIZE(eic7700_sid_clients),
+	.clients = eic7700_sid_clients,
+};
+#endif
+
+static const struct of_device_id eic7700_sid_of_match[] = {
+	{ .compatible = "eswin,eic7700-scu-sys-con", .data = &eic7700_sid_soc },
+	{ /* sentinel */ }
+};
+
+static int __init eic7700_init_streamID(void)
+{
+	const struct of_device_id *match;
+	struct device_node *root, *child = NULL;
+	struct resource regs;
+	struct eic7700_sid *mc = NULL;
+	int nid;
+	int ret = 0;
+
+	root = of_find_node_by_name(NULL, "soc");
+	for_each_child_of_node(root, child) {
+		match = of_match_node(eic7700_sid_of_match, child);
+		if (match && of_node_get(child)) {
+			if (of_address_to_resource(child, 0, &regs) < 0) {
+				pr_err("failed to get scu register\n");
+				of_node_put(child);
+				ret = -ENXIO;
+				break;
+			}
+			if (of_parse_syscon_nodes(child, &nid) < 0) {
+				pr_err("failed to get numa-node-id\n");
+				of_node_put(child);
+				ret = -ENXIO;
+				break;
+			}
+
+			/* program scu sreamID related registers */
+			mc = kzalloc(sizeof(*mc), GFP_KERNEL);
+			if (!mc) {
+				of_node_put(child);
+				pr_err("failed to kzalloc\n");
+				ret = -ENOMEM;
+				break;
+			}
+
+			mc->soc = match->data;
+			mc->regs = ioremap(regs.start, resource_size(&regs));
+			if (IS_ERR(mc->regs)) {
+				pr_err("failed to ioremap scu reges\n");
+				of_node_put(child);
+				ret = PTR_ERR(mc->regs);
+				kfree(mc);
+				break;
+			}
+			mc->start = regs.start;
+			mutex_init(&mc->eswin_dynm_sid_cfg_en_lock);
+
+			if (eic7700_tbu_powr_priv_init(&mc->tbu_power_soc, nid)) {
+				pr_err("failed to kzalloc for tbu_power_priv_arry\n");
+				of_node_put(child);
+				iounmap(mc->regs);
+				kfree(mc);
+				ret = -ENOMEM;
+				WARN_ON(1);
+				break;
+			}
+			mutex_init(&mc->tbu_reg_lock);
+
+			syscon_sid_cfg[nid] = mc;
+			pr_debug("%s, syscon_sid_cfg[%d] addr is 0x%px\n", __func__, nid, syscon_sid_cfg[nid]);
+
+			of_node_put(child);
+		}
+	}
+	of_node_put(root);
+
+	return ret;
+}
+
+early_initcall(eic7700_init_streamID);
+
+
+
+static const struct eic7700_tbu_client eic7700_tbu_clients[] = {
+	{
+		.tbu_id = EIC7700_TBUID_0x0, // ISP, DW200 share the tbu0
+		.tbu_reg_info = {0x3d8, 7, 6},
+		.tbu_power_ctl_register = eic7700_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x10, // tbu1_0 is only for video decoder
+		.tbu_reg_info = {0x3d4, 31, 30},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x11, // tbu1_1 is only video encoder
+		.tbu_reg_info = {0x3d4, 23, 22},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x12, // tbu1_2 is only Jpeg encoder
+		.tbu_reg_info = {0x3d4, 7, 6},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x13, // tbu1_3 is only Jpeg decoder
+		.tbu_reg_info = {0x3d4, 15, 14},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x2, // Ethernet, sata, usb, dma0, emmc, sd, sdio share the tbu2
+		.tbu_reg_info = {0x3d8, 15, 14},
+		.tbu_power_ctl_register = eic7700_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x3, // tbu3 is only for pcie
+		.tbu_reg_info = {0x3d8, 23, 22},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x4, // scpu, crypto, lpcpu, dma1 share the tbu4
+		.tbu_reg_info = {0x3d8, 31, 30},
+		.tbu_power_ctl_register = eic7700_tbu_power_ctl_register,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x5, // tbu5 is only NPU
+		.tbu_reg_info = {0x3d0, 15, 14},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x70, // tbu7_0 is only dsp0
+		.tbu_reg_info = {0x3f8, 7, 6},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x71, // tbu7_1 is only dsp1
+		.tbu_reg_info = {0x3f8, 15, 14},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x72, // tbu7_2 is only dsp2
+		.tbu_reg_info = {0x3f8, 23, 22},
+		.tbu_power_ctl_register = NULL,
+	},
+	{
+		.tbu_id = EIC7700_TBUID_0x73, // tbu7_3 is only dsp3
+		.tbu_reg_info = {0x3f8, 31, 30},
+		.tbu_power_ctl_register = NULL,
+	},
+};
+
+static const struct eic7700_tbu_soc eic7700_tbu_soc = {
+	.num_tbuClients = ARRAY_SIZE(eic7700_tbu_clients),
+	.tbu_clients = eic7700_tbu_clients,
+};
+
+static int __do_eic7700_tbu_power_ctl(int nid, bool is_powerUp, const struct tbu_reg_cfg_info *tbu_reg_info_p)
+{
+	int ret = 0;
+	unsigned long reg_val;
+	struct eic7700_sid *mc = NULL;
+	int loop_cnt = 0;
+	mc = syscon_sid_cfg[nid];
+	if (mc == NULL)
+		return -EFAULT;
+
+	mutex_lock(&mc->tbu_reg_lock);
+	if (is_powerUp) {
+		reg_val = readl(mc->regs + tbu_reg_info_p->reg_offset);
+		set_bit(tbu_reg_info_p->qreqn_pd_bit, &reg_val);
+		writel(reg_val, mc->regs + tbu_reg_info_p->reg_offset);
+		pr_debug("reg_offset=0x%03x, tbu_val=0x%x\n",
+			tbu_reg_info_p->reg_offset, readl(mc->regs + tbu_reg_info_p->reg_offset));
+		pr_debug("%s, power up!\n", __func__);
+	}
+	else {
+		reg_val = readl(mc->regs + tbu_reg_info_p->reg_offset);
+		clear_bit(tbu_reg_info_p->qreqn_pd_bit, &reg_val);
+		writel(reg_val, mc->regs + tbu_reg_info_p->reg_offset);
+		do {
+			reg_val = readl(mc->regs + tbu_reg_info_p->reg_offset);
+			pr_debug("reg_offset=0x%03x, tbu_val=0x%lx, BIT(qacceptn_pd_bit)=0x%lx\n",
+				tbu_reg_info_p->reg_offset, reg_val, BIT(tbu_reg_info_p->qacceptn_pd_bit));
+			if ((reg_val & BIT(tbu_reg_info_p->qacceptn_pd_bit)) == 0) {
+				pr_debug("%s, power down!\n", __func__);
+				break;
+			}
+			mdelay(10);
+			loop_cnt++;
+			if (loop_cnt > 10) {
+				WARN_ON(1); // it should never happen.
+				break;
+			}
+		}while (1);
+
+		if(loop_cnt > 10) {
+			ret = -1;
+		}
+	}
+	mutex_unlock(&mc->tbu_reg_lock);
+
+	return ret;
+}
+
+#define do_eic7700_tbu_power_up(nid, tbu_reg_info_p)	__do_eic7700_tbu_power_ctl(nid, true, tbu_reg_info_p)
+#define do_eic7700_tbu_power_down(nid, tbu_reg_info_p)	__do_eic7700_tbu_power_ctl(nid, false, tbu_reg_info_p)
+
+static int tbu_power_down_ref_release(atomic_t *ref)
+{
+	int ret = 0;
+	struct tbu_priv *tbu_priv_p = container_of(ref, struct tbu_priv, refcount);
+	int nid = tbu_priv_p->nid;
+	const struct tbu_reg_cfg_info *tbu_reg_info_p = &tbu_priv_p->tbu_client_p->tbu_reg_info;
+
+	WARN_ON(!tbu_priv_p);
+	if (!tbu_priv_p)
+		return -1;
+
+	ret = do_eic7700_tbu_power_down(nid, tbu_reg_info_p);
+
+	return ret;
+}
+
+static int eic7700_tbu_power_ctl_register(int nid, struct tbu_priv *tbu_priv_p, bool is_powerUp)
+{
+	int ret = 0;
+	const struct eic7700_tbu_client *tbu_client_p = tbu_priv_p->tbu_client_p;
+	const struct tbu_reg_cfg_info *tbu_reg_info_p = &tbu_priv_p->tbu_client_p->tbu_reg_info;
+	unsigned int old_refcount;
+
+	mutex_lock(&tbu_priv_p->tbu_priv_lock);
+	old_refcount = atomic_read(&tbu_priv_p->refcount);
+
+	pr_debug("%s, nid=%d, is_powerUp=%d, tbu_priv_p addr is 0x%px\n",
+		__func__, nid, is_powerUp, tbu_priv_p);
+	if (is_powerUp == false) { //power down
+		if (unlikely(0 == old_refcount)) {
+			pr_debug("%s, tbu_id 0x%02x is down already!\n", __func__, tbu_client_p->tbu_id);
+			goto tbu_finish;
+		}
+
+		if (atomic_sub_return(1, &tbu_priv_p->refcount) == 0) {
+			ret = tbu_power_down_ref_release(&tbu_priv_p->refcount);
+		}
+		else {
+			pr_debug("Can't power down tbu 0x%02x, it's used by other modules right now!\n",
+				tbu_client_p->tbu_id);
+		}
+
+	}
+	else { //power up
+		if (0 == old_refcount) {
+			ret = do_eic7700_tbu_power_up(nid, tbu_reg_info_p);
+		}
+		else {
+			pr_debug("tbu 0x%02x is already power up!", tbu_client_p->tbu_id);
+		}
+		atomic_add(1, &tbu_priv_p->refcount);
+	}
+
+tbu_finish:
+	mutex_unlock(&tbu_priv_p->tbu_priv_lock);
+
+	return ret;
+
+}
+
+static int eic7700_tbu_powr_priv_init(struct tbu_power_soc **tbu_power_soc_pp, int nid)
+{
+	int ret = 0;
+	int i;
+	unsigned int num_tbuClients = eic7700_tbu_soc.num_tbuClients;
+	struct tbu_power_soc *tbu_power_soc_p;
+	struct tbu_priv *tbu_priv_p;
+	unsigned int alloc_size;
+
+	pr_debug("%s:%d\n", __func__, __LINE__);
+
+	tbu_power_soc_p = kzalloc(sizeof(struct tbu_power_soc), GFP_KERNEL);
+	if (!tbu_power_soc_p)
+		return -ENOMEM;
+	pr_debug("%s:%d, tbu_power_soc_p(0x%px)\n", __func__, __LINE__, tbu_power_soc_p);
+
+	alloc_size = num_tbuClients * sizeof(struct tbu_priv);
+	tbu_priv_p = kzalloc(alloc_size, GFP_KERNEL);
+	if (!tbu_priv_p) {
+		ret = -ENOMEM;
+		goto err_tbu_priv_p;
+	}
+	tbu_power_soc_p->tbu_priv_array = tbu_priv_p;
+	pr_debug("%s:%d, num_tbu=%d,sizeof(struct tbu_priv)=0x%lx, alloc_size=0x%x, tbu_priv_p=0x%px\n",
+		__func__, __LINE__, num_tbuClients, sizeof(struct tbu_priv), alloc_size, tbu_priv_p);
+
+	for (i = 0; i < eic7700_tbu_soc.num_tbuClients; i++) {
+		tbu_priv_p->nid = nid;
+		atomic_set(&tbu_priv_p->refcount, 0);
+		tbu_priv_p->tbu_client_p = &eic7700_tbu_soc.tbu_clients[i];
+		mutex_init(&tbu_priv_p->tbu_priv_lock);
+		pr_debug("%s, nid %d, tbu 0x%02x, tbu_priv_p(0x%px), sizeof(struct tbu_priv)=0x%lx\n", __func__, nid, tbu_priv_p->tbu_client_p->tbu_id, tbu_priv_p, sizeof(struct tbu_priv));
+		tbu_priv_p++;
+	}
+	tbu_power_soc_p->num_tbuClients = num_tbuClients;
+
+	*tbu_power_soc_pp = tbu_power_soc_p;
+
+	return 0;
+
+err_tbu_priv_p:
+	kfree(tbu_power_soc_p);
+
+	return ret;
+
+}
+
+static int eic7700_get_tbu_priv(int nid, u32 tbu_id, struct tbu_priv **tbu_priv_pp)
+{
+	int i;
+	struct eic7700_sid *mc = syscon_sid_cfg[nid];
+	struct tbu_power_soc *tbu_power_soc_p = mc->tbu_power_soc;
+	struct tbu_priv *tbu_priv_p = tbu_power_soc_p->tbu_priv_array;
+
+	pr_debug("%s,  syscon_sid_cfg[%d] addr is 0x%px, tbu_id=0x%02x, tbu_power_soc_p is 0x%px\n",
+		__func__, nid, syscon_sid_cfg[nid], tbu_id, tbu_power_soc_p);
+
+	for (i = 0; i < tbu_power_soc_p->num_tbuClients; i++) {
+		if (tbu_id == tbu_priv_p->tbu_client_p->tbu_id) {
+			*tbu_priv_pp = tbu_priv_p;
+			pr_debug("%s, found tbu_id 0x%02x, tbu_priv_array[%d] tbu_priv_p is 0x%px\n",
+				__func__, tbu_id, i, tbu_priv_p);
+			return 0;
+		}
+		tbu_priv_p++;
+	}
+
+	return -1;
+}
+
+/***********************************************************************************************
+   eic7700_tbu_power(struct device *dev, bool is_powerUp) is for powering up or down
+   the tbus of the device module which is under smmu.
+   Drivers should call eic7700_tbu_power(dev, true) when probing afer clk of the tbu is on,
+   and call call eic7700_tbu_power(dev, false) when removing driver before clk of the tbu is off.
+
+   Input:
+	struct device *dev	The struct device of the driver that calls this API.
+	bool is_powerUp		true: power up the tbus;  false: power down the tbus.
+   Return:
+	zero:		successfully power up/down
+	none zero:	faild to power up/down
+***********************************************************************************************/
+int eic7700_tbu_power(struct device *dev, bool is_powerUp)
+{
+	int ret = 0;
+	struct device_node *node = dev->of_node;
+	int nid = dev_to_node(dev);
+	u32 tbu_id;
+	const struct eic7700_tbu_client *tbu_client_p = NULL;
+	struct tbu_priv *tbu_priv_p;
+	int tbu_num = 0;
+
+	if (nid == NUMA_NO_NODE) {
+	#ifdef CONFIG_NUMA
+		pr_err("%s:%d, NUMA_NO_NODE\n", __func__, __LINE__);
+		return -EFAULT;
+	#else
+		pr_info("%s:%d, NUMA_NO_NODE, single DIE\n", __func__, __LINE__);
+		nid = 0;
+	#endif
+	}
+
+	pr_debug("%s called!\n", __func__);
+	of_property_for_each_u32(node, "tbus", tbu_id) {
+		pr_debug("tbus = <0x%02x>\n", tbu_id);
+		if (0 == eic7700_get_tbu_priv(nid, tbu_id, &tbu_priv_p)) {
+			tbu_client_p = tbu_priv_p->tbu_client_p;
+			if (tbu_client_p->tbu_power_ctl_register) {
+				ret = tbu_client_p->tbu_power_ctl_register(nid, tbu_priv_p, is_powerUp);
+				if (ret)
+					return ret;
+			}
+			else {
+				ret = __do_eic7700_tbu_power_ctl(nid, is_powerUp, &tbu_client_p->tbu_reg_info);
+				if (ret)
+					return ret;
+			}
+			tbu_num++;
+		}
+		else if (tbu_id == EIC7700_TBUID_0xF00) {
+			tbu_num++;
+		}
+		else {
+			pr_err("tbu power ctl failed!, Couldn't find tbu 0x%x\n", tbu_id);
+			return -1;
+		}
+	}
+
+	if (tbu_num == 0) {
+		pr_err("Err,tbu NOT defined in dts!!!!\n");
+		WARN_ON(1);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(eic7700_tbu_power);
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index bc7e2ad37002..5aa588dba2d8 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -214,6 +214,34 @@ config MMC_SDHCI_OF_ESDHC
 
 	  If unsure, say N.
 
+config MMC_SDHCI_OF_ESWIN
+	tristate "SDHCI OF support for the Eswin SDHCI controllers"
+	depends on MMC_SDHCI_PLTFM
+	depends on OF
+	depends on COMMON_CLK
+	select MMC_CQHCI
+	help
+	  This selects the Eswin Secure Digital Host Controller Interface
+	  (SDHCI).
+	
+	  If you have a controller with this interface, say Y or M here.
+	
+	  If unsure, say N.
+
+config MMC_SDHCI_OF_SDIO_ESWIN
+	tristate "SDHCI OF SDIO support for the Eswin SDHCI controllers"
+	depends on MMC_SDHCI_PLTFM
+	depends on OF
+	depends on COMMON_CLK
+	select MMC_CQHCI
+	help
+	  This selects the Eswin Secure Digital Host Controller Interface
+	  (SDHCI-SDIO).
+	
+	  If you have a controller with this interface, say Y or M here.
+	
+	  If unsure, say N.
+
 config MMC_SDHCI_OF_HLWD
 	tristate "SDHCI OF support for the Nintendo Wii SDHCI controllers"
 	depends on MMC_SDHCI_PLTFM
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index a693fa3d3f1c..2c7167dfdaac 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -84,6 +84,10 @@ obj-$(CONFIG_MMC_SDHCI_OF_ARASAN)	+= sdhci-of-arasan.o
 obj-$(CONFIG_MMC_SDHCI_OF_ASPEED)	+= sdhci-of-aspeed.o
 obj-$(CONFIG_MMC_SDHCI_OF_AT91)		+= sdhci-of-at91.o
 obj-$(CONFIG_MMC_SDHCI_OF_ESDHC)	+= sdhci-of-esdhc.o
+eswin-sdhci-emmc-objs  := sdhci-of-eswin.o sdhci-eswin.o
+obj-$(CONFIG_MMC_SDHCI_OF_ESWIN)       += eswin-sdhci-emmc.o
+eswin-sdhci-sdio-objs  := sdhci-of-eswin-sdio.o sdhci-eswin.o
+obj-$(CONFIG_MMC_SDHCI_OF_SDIO_ESWIN)  += eswin-sdhci-sdio.o
 obj-$(CONFIG_MMC_SDHCI_OF_HLWD)		+= sdhci-of-hlwd.o
 obj-$(CONFIG_MMC_SDHCI_OF_DWCMSHC)	+= sdhci-of-dwcmshc.o
 obj-$(CONFIG_MMC_SDHCI_OF_SPARX5)	+= sdhci-of-sparx5.o
diff --git a/drivers/mmc/host/sdhci-eswin.c b/drivers/mmc/host/sdhci-eswin.c
new file mode 100644
index 000000000000..b941fa90f710
--- /dev/null
+++ b/drivers/mmc/host/sdhci-eswin.c
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Emmc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswin.com>
+ */
+#include <linux/delay.h>
+#include <linux/reset.h>
+#include "sdhci-eswin.h"
+
+static void eswin_mshc_coreclk_config(struct sdhci_host *host, uint16_t divisor,
+					unsigned int flag_sel)
+{
+	struct sdhci_pltfm_host *pltfm_host;
+	struct eswin_sdhci_data *eswin_sdhci;
+	u32 val = 0;
+	u32 delay = 0xfffff;
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	val = readl(eswin_sdhci->core_clk_reg);
+	val &= ~MSHC_CORE_CLK_ENABLE;
+	writel(val, eswin_sdhci->core_clk_reg);
+	while (delay--)
+		;
+	val &= ~(MSHC_CORE_CLK_FREQ_BIT_MASK << MSHC_CORE_CLK_FREQ_BIT_SHIFT);
+	val |= (divisor & MSHC_CORE_CLK_FREQ_BIT_MASK)
+	       << MSHC_CORE_CLK_FREQ_BIT_SHIFT;
+	val &= ~(MSHC_CORE_CLK_SEL_BIT);
+	val |= flag_sel;
+	writel(val, eswin_sdhci->core_clk_reg);
+
+	udelay(100);
+	val |= MSHC_CORE_CLK_ENABLE;
+	writel(val, eswin_sdhci->core_clk_reg);
+	mdelay(1);
+}
+
+static void eswin_mshc_coreclk_disable(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host;
+	struct eswin_sdhci_data *eswin_sdhci;
+	u32 val = 0;
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	val = readl(eswin_sdhci->core_clk_reg);
+	val &= ~MSHC_CORE_CLK_ENABLE;
+	writel(val, eswin_sdhci->core_clk_reg);
+}
+
+void eswin_sdhci_disable_card_clk(struct sdhci_host *host)
+{
+	unsigned int clk;
+
+	/* Reset SD Clock Enable */
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+	clk &= ~SDHCI_CLOCK_CARD_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+}
+
+void eswin_sdhci_enable_card_clk(struct sdhci_host *host)
+{
+	ktime_t timeout;
+	unsigned int clk;
+
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+
+	clk |= SDHCI_CLOCK_INT_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+
+	/* Wait max 150 ms */
+	timeout = ktime_add_ms(ktime_get(), 150);
+	while (1) {
+		bool timedout = ktime_after(ktime_get(), timeout);
+		clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+		if (clk & SDHCI_CLOCK_INT_STABLE)
+			break;
+		if (timedout) {
+			pr_err("%s: Internal clock never stabilised.\n",
+					mmc_hostname(host->mmc));
+			return;
+		}
+		udelay(10);
+	}
+
+	clk |= SDHCI_CLOCK_CARD_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+	mdelay(1);
+}
+
+void eswin_sdhci_set_core_clock(struct sdhci_host *host,
+				       unsigned int clock)
+{
+	unsigned int div, divide;
+	unsigned int flag_sel, max_clk;
+
+	if (clock == 0) {
+		eswin_mshc_coreclk_disable(host);
+		return;
+	}
+
+	if (SDHCI_CLK_208M % clock == 0) {
+		flag_sel = 1;
+		max_clk = SDHCI_CLK_208M;
+	} else {
+		flag_sel = 0;
+		max_clk = SDHCI_CLK_200M;
+	}
+
+	for (div = 1; div <= MAX_CORE_CLK_DIV; div++) {
+		if ((max_clk / div) <= clock)
+			break;
+	}
+	div--;
+
+	if (div == 0 || div == 1) {
+		divide = 2;
+	} else {
+		divide = (div + 1) * 2;
+	}
+	pr_debug("%s: clock:%d timing:%d\n", mmc_hostname(host->mmc), clock, host->timing);
+
+	eswin_sdhci_disable_card_clk(host);
+	eswin_mshc_coreclk_config(host, divide, flag_sel);
+	eswin_sdhci_enable_card_clk(host);
+	mdelay(2);
+}
+
+static void eswin_sdhci_set_clk_delays(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+
+	clk_set_phase(clk_data->sampleclk,
+			clk_data->clk_phase_in[host->timing]);
+	clk_set_phase(clk_data->sdcardclk,
+			clk_data->clk_phase_out[host->timing]);
+}
+
+static void eswin_sdhci_dt_read_clk_phase(struct device *dev,
+					struct eswin_sdhci_clk_data *clk_data,
+					unsigned int timing, const char *prop)
+{
+	struct device_node *np = dev->of_node;
+
+	int clk_phase[2] = { 0 };
+
+	/*
+	 * Read Tap Delay values from DT, if the DT does not contain the
+	 * Tap Values then use the pre-defined values.
+	 */
+	if (of_property_read_variable_u32_array(np, prop, &clk_phase[0], 2,
+						0)) {
+		dev_dbg(dev, "Using predefined clock phase for %s = %d %d\n",
+			prop, clk_data->clk_phase_in[timing],
+			clk_data->clk_phase_out[timing]);
+		return;
+	}
+
+	/* The values read are Input and Output Clock Delays in order */
+	clk_data->clk_phase_in[timing] = clk_phase[0];
+	clk_data->clk_phase_out[timing] = clk_phase[1];
+}
+
+/**
+ * eswin_dt_parse_clk_phases - Read Clock Delay values from DT
+ *
+ * @dev:        Pointer to our struct device.
+ * @clk_data:       Pointer to the Clock Data structure
+ *
+ * Called at initialization to parse the values of Clock Delays.
+ */
+void eswin_sdhci_dt_parse_clk_phases(struct device *dev,
+				struct eswin_sdhci_clk_data *clk_data)
+{
+	clk_data->set_clk_delays = eswin_sdhci_set_clk_delays;
+
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_LEGACY,
+				      "clk-phase-legacy");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_HS,
+				      "clk-phase-mmc-hs");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_SD_HS,
+				      "clk-phase-sd-hs");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR12,
+				      "clk-phase-uhs-sdr12");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR25,
+				      "clk-phase-uhs-sdr25");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR50,
+				      "clk-phase-uhs-sdr50");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_SDR104,
+				      "clk-phase-uhs-sdr104");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_UHS_DDR50,
+				      "clk-phase-uhs-ddr50");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_DDR52,
+				      "clk-phase-mmc-ddr52");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_HS200,
+				      "clk-phase-mmc-hs200");
+	eswin_sdhci_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_HS400,
+				      "clk-phase-mmc-hs400");
+}
+
+unsigned int eswin_convert_drive_impedance_ohm(struct platform_device *pdev,
+					     unsigned int dr_ohm)
+{
+	switch (dr_ohm) {
+	case 100:
+		return PHYCTRL_DR_100OHM;
+	case 66:
+		return PHYCTRL_DR_66OHM;
+	case 50:
+		return PHYCTRL_DR_50OHM;
+	case 40:
+		return PHYCTRL_DR_40OHM;
+	case 33:
+		return PHYCTRL_DR_33OHM;
+	}
+
+	dev_warn(&pdev->dev, "Invalid value %u for drive-impedance-ohm.\n",
+		 dr_ohm);
+	return PHYCTRL_DR_50OHM;
+}
+
+static void eswin_sdhci_do_reset(struct eswin_sdhci_data *eswin_sdhci)
+{
+	int ret;
+
+	ret = reset_control_assert(eswin_sdhci->txrx_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(eswin_sdhci->phy_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(eswin_sdhci->prstn);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(eswin_sdhci->arstn);
+	WARN_ON(0 != ret);
+
+	mdelay(2);
+
+	ret = reset_control_deassert(eswin_sdhci->txrx_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(eswin_sdhci->phy_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(eswin_sdhci->prstn);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(eswin_sdhci->arstn);
+	WARN_ON(0 != ret);
+}
+
+int eswin_sdhci_reset_init(struct device *dev,
+				 struct eswin_sdhci_data *eswin_sdhci)
+{
+	int ret = 0;
+	eswin_sdhci->txrx_rst = devm_reset_control_get_optional(dev, "txrx_rst");
+	if (IS_ERR_OR_NULL(eswin_sdhci->txrx_rst)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->txrx_rst),
+			      "txrx_rst reset not found.\n");
+		return -EFAULT;
+	}
+
+	eswin_sdhci->phy_rst = devm_reset_control_get_optional(dev, "phy_rst");
+	if (IS_ERR_OR_NULL(eswin_sdhci->phy_rst)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->phy_rst),
+			      "phy_rst reset not found.\n");
+		return -EFAULT;
+	}
+
+	eswin_sdhci->prstn = devm_reset_control_get_optional(dev, "prstn");
+	if (IS_ERR_OR_NULL(eswin_sdhci->prstn)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->prstn),
+			      "prstn reset not found.\n");
+		return -EFAULT;
+	}
+
+	eswin_sdhci->arstn = devm_reset_control_get_optional(dev, "arstn");
+	if (IS_ERR_OR_NULL(eswin_sdhci->arstn)) {
+		dev_err_probe(dev, PTR_ERR(eswin_sdhci->arstn),
+			      "arstn reset not found.\n");
+		return -EFAULT;
+	}
+	eswin_sdhci_do_reset(eswin_sdhci);
+
+	return ret;
+}
diff --git a/drivers/mmc/host/sdhci-eswin.h b/drivers/mmc/host/sdhci-eswin.h
new file mode 100644
index 000000000000..5274766f0501
--- /dev/null
+++ b/drivers/mmc/host/sdhci-eswin.h
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN SDHCI Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswin.com>
+ */
+#ifndef _DRIVERS_MMC_SDHCI_ESWIN_H
+#define _DRIVERS_MMC_SDHCI_ESWIN_H
+
+#include <linux/reset.h>
+#include <linux/clk-provider.h>
+#include "sdhci-pltfm.h"
+
+#define MSHC_CARD_CLK_STABLE BIT(28)
+#define MSHC_INT_BCLK_STABLE BIT(16)
+#define MSHC_INT_ACLK_STABLE BIT(8)
+#define MSHC_INT_TMCLK_STABLE BIT(0)
+#define MSHC_INT_CLK_STABLE                             \
+	(MSHC_CARD_CLK_STABLE | MSHC_INT_ACLK_STABLE | \
+	 MSHC_INT_BCLK_STABLE | MSHC_INT_TMCLK_STABLE)
+#define MSHC_HOST_VAL_STABLE BIT(0)
+#define EMMC0_CARD_DETECT BIT(9)
+#define EMMC0_CARD_WRITE_PROT BIT(8)
+
+#define MSHC_CORE_CLK_ENABLE BIT(16)
+#define MSHC_CORE_CLK_FREQ_BIT_SHIFT 4
+#define MSHC_CORE_CLK_FREQ_BIT_MASK 0xfffu
+#define MSHC_CORE_CLK_SEL_BIT BIT(0)
+
+/* Controller does not have CD wired and will not function normally without */
+#define SDHCI_ESWIN_QUIRK_FORCE_CDTEST BIT(0)
+/* Controller immediately reports SDHCI_CLOCK_INT_STABLE after enabling the
+ * internal clock even when the clock isn't stable */
+#define SDHCI_ESWIN_QUIRK_CLOCK_UNSTABLE BIT(1)
+
+#define ESWIN_SDHCI_CTRL_HS400 0x0007 // Non-standard, for eswin,these bits are 0x7
+
+#define SDHCI_CLK_208M   208000000
+#define SDHCI_CLK_200M   200000000
+
+#define AWSMMUSID GENMASK(31, 24)  // The sid of write operation
+#define AWSMMUSSID GENMASK(23, 16)  // The ssid of write operation
+#define ARSMMUSID GENMASK(15, 8)  // The sid of read operation
+#define ARSMMUSSID GENMASK(7, 0)  // The ssid of read operation
+
+/* DWC_mshc_map/DWC_mshc_phy_block register */
+#define DWC_MSHC_PTR_PHY_R 0x300
+#define PHY_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x00)
+#define PHY_CMDPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x04)
+#define PHY_DATAPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x06)
+#define PHY_CLKPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x08)
+#define PHY_STBPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x0a)
+#define PHY_RSTNPAD_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x0c)
+#define PHY_PADTEST_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x0e)
+#define PHY_PADTEST_OUT_R (DWC_MSHC_PTR_PHY_R + 0x10)
+#define PHY_PADTEST_IN_R (DWC_MSHC_PTR_PHY_R + 0x12)
+#define PHY_PRBS_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x18)
+#define PHY_PHYLBK_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x1a)
+#define PHY_COMMDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x1c)
+#define PHY_SDCLKDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x1d)
+#define PHY_SDCLKDL_DC_R (DWC_MSHC_PTR_PHY_R + 0x1e)
+#define PHY_SMPLDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x20)
+#define PHY_ATDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x21)
+#define PHY_DLL_CTRL_R (DWC_MSHC_PTR_PHY_R + 0x24)
+#define PHY_DLL_CNFG1_R (DWC_MSHC_PTR_PHY_R + 0x25)
+#define PHY_DLL_CNFG2_R (DWC_MSHC_PTR_PHY_R + 0x26)
+#define PHY_DLLDL_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x28)
+#define PHY_DLL_OFFST_R (DWC_MSHC_PTR_PHY_R + 0x29)
+#define PHY_DLLMST_TSTDC_R (DWC_MSHC_PTR_PHY_R + 0x2a)
+#define PHY_DLLBT_CNFG_R (DWC_MSHC_PTR_PHY_R + 0x2c)
+#define PHY_DLL_STATUS_R (DWC_MSHC_PTR_PHY_R + 0x2e)
+#define PHY_DLLDBG_MLKDC_R (DWC_MSHC_PTR_PHY_R + 0x30)
+#define PHY_DLLDBG_SLKDC_R (DWC_MSHC_PTR_PHY_R + 0x32)
+
+#define ENABLE 1
+#define DISABLE 0
+/* strength definition */
+#define PHYCTRL_DR_33OHM 0xee
+#define PHYCTRL_DR_40OHM 0xcc
+#define PHYCTRL_DR_50OHM 0x88
+#define PHYCTRL_DR_66OHM 0x44
+#define PHYCTRL_DR_100OHM 0x00
+
+#define PHY_PAD_MAX_DRIVE_STRENGTH 0xf
+#define PHY_CLK_MAX_DELAY_MASK 0x7f
+#define PHY_PAD_SP_DRIVE_SHIF 16
+#define PHY_PAD_SN_DRIVE_SHIF 20
+
+#define PHY_RSTN BIT(0)
+#define PHY_UPDATE_DELAY_CODE BIT(4)
+
+#define VENDOR_EMMC_CTRL_R 0x52c
+#define EMMC_CRAD_PRESENT BIT(0)
+#define EMMC_RST_N_OE BIT(3)
+#define EMMC_RST_N  BIT(2)
+
+#define PHY_SLEW_0 0x0
+#define PHY_SLEW_1 0x1
+#define PHY_SLEW_2 0x2
+#define PHY_SLEW_3 0x3
+#define PHY_TX_SLEW_CTRL_P_BIT_SHIFT 5
+#define PHY_TX_SLEW_CTRL_N_BIT_SHIFT 9
+
+#define PHY_PULL_BIT_SHIF 0x3
+#define PHY_PULL_DISABLED 0x0
+#define PHY_PULL_UP 0x1
+#define PHY_PULL_DOWN 0x2
+#define PHY_PULL_MASK 0x3
+
+#define PHY_PAD_RXSEL_0 0x0
+#define PHY_PAD_RXSEL_1 0x1
+
+#define VENDOR_AT_CTRL_R 0x540
+#define LATENCY_LT_BIT_OFFSET 19
+#define LATENCY_LT_MASK 0x3
+
+#define LATENCY_LT_1 0x0
+#define LATENCY_LT_2 0x1
+#define LATENCY_LT_3 0x2
+#define LATENCY_LT_4 0x3
+#define SW_TUNE_ENABLE BIT(4)
+
+#define VENDOR_AT_SATA_R 0x544
+#define MAX_PHASE_CODE 0xff
+
+#define DLL_ENABEL BIT(0)
+#define DLL_LOCK_STS BIT(0)
+#define DLL_ERROR_STS BIT(1)
+#define PHY_DELAY_CODE_MASK 0x7f
+#define PHY_DELAY_CODE_MAX 0x7f
+
+#define MAX_CORE_CLK_DIV 0xfff
+
+/**
+ * struct eswin_sdhci_clk_ops - Clock Operations for eswin SD controller
+ *
+ * @sdcardclk_ops:  The output clock related operations
+ * @sampleclk_ops:  The sample clock related operations
+ */
+struct eswin_sdhci_clk_ops {
+	const struct clk_ops *sdcardclk_ops;
+	const struct clk_ops *sampleclk_ops;
+};
+
+/**
+ * struct eswin_sdhci_clk_data - ESWIN Controller Clock Data.
+ *
+ * @sdcardclk_hw:   Struct for the clock we might provide to a PHY.
+ * @sdcardclk:      Pointer to normal 'struct clock' for sdcardclk_hw.
+ * @sampleclk_hw:   Struct for the clock we might provide to a PHY.
+ * @sampleclk:      Pointer to normal 'struct clock' for sampleclk_hw.
+ * @clk_phase_in:   Array of Input Clock Phase Delays for all speed modes
+ * @clk_phase_out:  Array of Output Clock Phase Delays for all speed modes
+ * @set_clk_delays: Function pointer for setting Clock Delays
+ * @clk_of_data:    Platform specific runtime clock data storage pointer
+ */
+struct eswin_sdhci_clk_data {
+	struct clk_hw sdcardclk_hw;
+	struct clk *sdcardclk;
+	struct clk_hw sampleclk_hw;
+	struct clk *sampleclk;
+	int clk_phase_in[MMC_TIMING_MMC_HS400 + 1];
+	int clk_phase_out[MMC_TIMING_MMC_HS400 + 1];
+	void (*set_clk_delays)(struct sdhci_host *host);
+	void *clk_of_data;
+};
+
+struct eswin_sdhci_phy_data {
+	unsigned int drive_impedance;
+	unsigned int enable_strobe_pulldown;
+	unsigned int enable_data_pullup;
+	unsigned int enable_cmd_pullup;
+	unsigned int delay_code;
+};
+
+/**
+ * struct eswin_sdhci_data - ESWIN Controller Data
+ *
+ * @host:       Pointer to the main SDHCI host structure.
+ * @clk_ahb:        Pointer to the AHB clock
+ * @has_cqe:        True if controller has command queuing engine.
+ * @clk_data:       Struct for the ESWIN Controller Clock Data.
+ * @clk_ops:        Struct for the ESWIN Controller Clock Operations.
+ * @soc_ctl_base:   Pointer to regmap for syscon for soc_ctl registers.
+ * @soc_ctl_map:    Map to get offsets into soc_ctl registers.
+ * @quirks:     ESWIN deviations from spec.
+ * @phy:        ESWIN sdhci phy configs.
+ * @private:    private for spec driver.
+ */
+struct eswin_sdhci_data {
+	struct sdhci_host *host;
+	struct clk *clk_ahb;
+	bool has_cqe;
+	struct eswin_sdhci_clk_data clk_data;
+	const struct eswin_sdhci_clk_ops *clk_ops;
+	unsigned int quirks;
+	void __iomem *core_clk_reg;
+	struct reset_control *txrx_rst;
+	struct reset_control *phy_rst;
+	struct reset_control *prstn;
+	struct reset_control *arstn;
+	struct eswin_sdhci_phy_data phy;
+	unsigned long private[] ____cacheline_aligned;
+};
+
+struct eswin_sdhci_of_data {
+	const struct sdhci_pltfm_data *pdata;
+	const struct eswin_sdhci_clk_ops *clk_ops;
+};
+
+void eswin_sdhci_set_core_clock(struct sdhci_host *host,
+				       unsigned int clock);
+void eswin_sdhci_disable_card_clk(struct sdhci_host *host);
+void eswin_sdhci_enable_card_clk(struct sdhci_host *host);
+void eswin_sdhci_dt_parse_clk_phases(struct device *dev,
+				struct eswin_sdhci_clk_data *clk_data);
+unsigned int eswin_convert_drive_impedance_ohm(struct platform_device *pdev,
+					     unsigned int dr_ohm);
+int eswin_sdhci_reset_init(struct device *dev,
+				 struct eswin_sdhci_data *eswin_sdhci);
+
+#endif /* _DRIVERS_MMC_SDHCI_ESWIN_H */
diff --git a/drivers/mmc/host/sdhci-of-eswin-sdio.c b/drivers/mmc/host/sdhci-of-eswin-sdio.c
new file mode 100644
index 000000000000..c69e5f10f240
--- /dev/null
+++ b/drivers/mmc/host/sdhci-of-eswin-sdio.c
@@ -0,0 +1,1079 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN sdio Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswin.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include <linux/reset.h>
+#include "cqhci.h"
+#include "sdhci-pltfm.h"
+
+#include <linux/eic7700-sid-cfg.h>
+#include <linux/bitfield.h>
+#include <linux/iommu.h>
+#include "sdhci-eswin.h"
+
+
+#define ESWIN_SDHCI_SD_CQE_BASE_ADDR 0x180
+#define ESWIN_SDHCI_SD0_INT_STATUS 0x608
+#define ESWIN_SDHCI_SD0_PWR_CTRL 0x60c
+#define ESWIN_SDHCI_SD1_INT_STATUS 0x708
+#define ESWIN_SDHCI_SD1_PWR_CTRL 0x70c
+
+#define DELAY_RANGE_THRESHOLD   20
+
+struct eswin_sdio_private {
+	int phase_code;
+	unsigned int enable_sw_tuning;
+};
+
+static inline void *sdhci_sdio_priv(struct eswin_sdhci_data *sdio)
+{
+	return sdio->private;
+}
+
+static void eswin_sdhci_sdio_set_clock(struct sdhci_host *host,
+				       unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_clk_data *clk_data =
+		&eswin_sdhci_sdio->clk_data;
+
+	/* Set the Input and Output Clock Phase Delays */
+	if (clk_data->set_clk_delays) {
+		clk_data->set_clk_delays(host);
+	}
+
+	eswin_sdhci_set_core_clock(host, clock);
+	sdhci_set_clock(host, clock);
+
+	if (eswin_sdhci_sdio->quirks & SDHCI_ESWIN_QUIRK_CLOCK_UNSTABLE)
+		/*
+                 * Some controllers immediately report SDHCI_CLOCK_INT_STABLE
+                 * after enabling the clock even though the clock is not
+                 * stable. Trying to use a clock without waiting here results
+                 * in EILSEQ while detecting some older/slower cards. The
+                 * chosen delay is the maximum delay from sdhci_set_clock.
+                 */
+		msleep(20);
+}
+
+static void eswin_sdhci_sdio_config_phy_delay(struct sdhci_host *host,
+					      int delay)
+{
+	delay &= PHY_CLK_MAX_DELAY_MASK;
+
+	/*phy clk delay line config*/
+	sdhci_writeb(host, PHY_UPDATE_DELAY_CODE, PHY_SDCLKDL_CNFG_R);
+	sdhci_writeb(host, delay, PHY_SDCLKDL_DC_R);
+	sdhci_writeb(host, 0x0, PHY_SDCLKDL_CNFG_R);
+}
+
+static void eswin_sdhci_sdio_config_phy(struct sdhci_host *host)
+{
+	unsigned int val = 0;
+	unsigned int drv = 0;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci =
+		sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_phy_data *phy = &eswin_sdhci->phy;
+
+	drv = phy->drive_impedance << PHY_PAD_SP_DRIVE_SHIF;
+	pr_debug("%s: phy drv=0x%x \n",mmc_hostname(host->mmc), drv);
+
+	eswin_sdhci_disable_card_clk(host);
+
+	/* reset phy,config phy's pad */
+	sdhci_writel(host, drv | (~PHY_RSTN), PHY_CNFG_R);
+	/*CMDPAD_CNFS*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+	      (phy->enable_cmd_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_CMDPAD_CNFG_R);
+	pr_debug("%s: phy cmd=0x%x\n",mmc_hostname(host->mmc), val);
+
+	/*DATA PAD CNFG*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+	      (phy->enable_data_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_DATAPAD_CNFG_R);
+	pr_debug("%s: phy data=0x%x\n",mmc_hostname(host->mmc), val);
+
+	/*Clock PAD Setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) | PHY_PAD_RXSEL_0;
+	sdhci_writew(host, val, PHY_CLKPAD_CNFG_R);
+	pr_debug("%s: phy clk=0x%x\n",mmc_hostname(host->mmc), val);
+	mdelay(2);
+
+	/*PHY RSTN PAD setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+	      (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+	      (PHY_PULL_UP << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_RSTNPAD_CNFG_R);
+
+	sdhci_writel(host, drv | PHY_RSTN, PHY_CNFG_R);
+
+	eswin_sdhci_sdio_config_phy_delay(host, phy->delay_code);
+
+	eswin_sdhci_enable_card_clk(host);
+}
+
+static void eswin_sdhci_sdio_reset(struct sdhci_host *host, u8 mask)
+{
+	u8 ctrl;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+
+	/* Disable signal and interrupts before resetting the phy.
+	 * Doing this avoids ISR to serve any undesired interrupts during
+	 * reset and avoid producing the fake register dump during probe.
+	 */
+	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
+	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
+	sdhci_reset(host, mask);
+	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
+	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+
+	if (eswin_sdhci_sdio->quirks & SDHCI_ESWIN_QUIRK_FORCE_CDTEST) {
+		ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
+		ctrl |= SDHCI_CTRL_CDTEST_INS | SDHCI_CTRL_CDTEST_EN;
+		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
+	}
+	if (mask == SDHCI_RESET_ALL) {	// after reset all,the phy`s config will be clear.
+		eswin_sdhci_sdio_config_phy(host);
+	}
+}
+
+static int eswin_sdhci_sdio_delay_tuning(struct sdhci_host *host, u32 opcode)
+{
+	int ret;
+	int delay = -1;
+	int i = 0;
+	int delay_min = -1;
+	int delay_max = -1;
+	int delay_range = -1;
+
+	int cmd_error = 0;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci =
+		sdhci_pltfm_priv(pltfm_host);
+
+	for (i = 0; i <= PHY_DELAY_CODE_MAX; i++) {
+		eswin_sdhci_disable_card_clk(host);
+		eswin_sdhci_sdio_config_phy_delay(host, i);
+		eswin_sdhci_enable_card_clk(host);
+		ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+		if (ret) {
+			host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+			udelay(200);
+			if (delay_min != -1 && delay_max != -1) {
+				if (delay_max - delay_min > delay_range) {
+					delay_range = delay_max - delay_min;
+					delay = (delay_min + delay_max) / 2;
+					if (delay_range > DELAY_RANGE_THRESHOLD)
+						break;
+				}
+				delay_min = -1;
+				delay_max = -1;
+			}
+		} else {
+			if (delay_min == -1) {
+				delay_min = i;
+				continue;
+			} else {
+				delay_max = i;
+				continue;
+			}
+		}
+	}
+	if (delay == -1) {
+		pr_err("%s: delay code tuning failed!\n",
+		       mmc_hostname(host->mmc));
+		eswin_sdhci_disable_card_clk(host);
+		eswin_sdhci_sdio_config_phy_delay(host,
+						  eswin_sdhci->phy.delay_code);
+		eswin_sdhci_enable_card_clk(host);
+
+		return ret;
+	}
+
+	pr_info("%s: set delay:0x%x\n", mmc_hostname(host->mmc), delay);
+	eswin_sdhci_disable_card_clk(host);
+	eswin_sdhci_sdio_config_phy_delay(host, delay);
+	eswin_sdhci_enable_card_clk(host);
+
+	return 0;
+}
+
+static int eswin_sdhci_sdio_phase_code_tuning(struct sdhci_host *host,
+					      u32 opcode)
+{
+	int cmd_error = 0;
+	int ret = 0;
+	int phase_code = 0;
+	int code_min = -1;
+	int code_max = -1;
+
+	for (phase_code = 0; phase_code <= MAX_PHASE_CODE; phase_code++) {
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, phase_code, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+
+		ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+		if (ret) {
+			host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+			udelay(200);
+			if (code_min != -1 && code_max != -1)
+				break;
+		} else {
+			if (code_min == -1) {
+				code_min = phase_code;
+				continue;
+			} else {
+				code_max = phase_code;
+				continue;
+			}
+		}
+	}
+	if (code_min == -1 && code_max == -1) {
+		pr_err("%s: phase code tuning failed!\n",
+		       mmc_hostname(host->mmc));
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+		return -EIO;
+	}
+
+	phase_code = (code_min + code_max) / 2;
+	pr_info("%s: set phase_code:0x%x\n", mmc_hostname(host->mmc), phase_code);
+
+	eswin_sdhci_disable_card_clk(host);
+	sdhci_writew(host, phase_code, VENDOR_AT_SATA_R);
+	eswin_sdhci_enable_card_clk(host);
+
+	return 0;
+}
+
+static int eswin_sdhci_sdio_executing_tuning(struct sdhci_host *host,
+					     u32 opcode)
+{
+	u32 ctrl;
+	u32 val;
+	int ret = 0;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct eswin_sdhci_data *eswin_sdhci_sdio;
+	struct eswin_sdio_private *eswin_sdio_priv;
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci_sdio = sdhci_pltfm_priv(pltfm_host);
+	eswin_sdio_priv = sdhci_sdio_priv(eswin_sdhci_sdio);
+
+	if (!eswin_sdio_priv->enable_sw_tuning) {
+		if (eswin_sdio_priv->phase_code != -1) {
+			eswin_sdhci_disable_card_clk(host);
+			sdhci_writew(host, eswin_sdio_priv->phase_code, VENDOR_AT_SATA_R);
+			eswin_sdhci_enable_card_clk(host);
+		}
+		return 0;
+	}
+
+	eswin_sdhci_disable_card_clk(host);
+
+	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	ctrl &= ~SDHCI_CTRL_TUNED_CLK;
+	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
+
+	val = sdhci_readl(host, VENDOR_AT_CTRL_R);
+	val |= SW_TUNE_ENABLE;
+	sdhci_writew(host, val, VENDOR_AT_CTRL_R);
+	sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+
+	eswin_sdhci_enable_card_clk(host);
+
+	sdhci_writew(host, 0x0, SDHCI_CMD_DATA);
+
+	ret = eswin_sdhci_sdio_delay_tuning(host, opcode);
+	if (ret < 0) {
+		return ret;
+	}
+
+	ret = eswin_sdhci_sdio_phase_code_tuning(host, opcode);
+	if (ret < 0) {
+		return ret;
+	}
+
+	return 0;
+}
+
+static u32 eswin_sdhci_sdio_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+	int cmd_error = 0;
+	int data_error = 0;
+
+	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+		return intmask;
+
+	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+	return 0;
+}
+
+static void eswin_sdhci_sdio_dumpregs(struct mmc_host *mmc)
+{
+	sdhci_dumpregs(mmc_priv(mmc));
+}
+
+static void eswin_sdhci_sdio_cqe_enable(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	u32 reg;
+
+	reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	while (reg & SDHCI_DATA_AVAILABLE) {
+		sdhci_readl(host, SDHCI_BUFFER);
+		reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	}
+
+	sdhci_cqe_enable(mmc);
+}
+
+static const struct cqhci_host_ops eswin_sdhci_sdio_cqhci_ops = {
+	.enable = eswin_sdhci_sdio_cqe_enable,
+	.disable = sdhci_cqe_disable,
+	.dumpregs = eswin_sdhci_sdio_dumpregs,
+};
+
+static const struct sdhci_ops eswin_sdhci_sdio_cqe_ops = {
+	.set_clock = eswin_sdhci_sdio_set_clock,
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = eswin_sdhci_sdio_reset,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+	.set_power = sdhci_set_power_and_bus_voltage,
+	.irq = eswin_sdhci_sdio_cqhci_irq,
+	.platform_execute_tuning = eswin_sdhci_sdio_executing_tuning,
+
+};
+
+static const struct sdhci_pltfm_data eswin_sdhci_sdio_cqe_pdata = {
+	.ops = &eswin_sdhci_sdio_cqe_ops,
+	.quirks = SDHCI_QUIRK_BROKEN_CQE | SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+		   SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
+};
+
+#ifdef CONFIG_PM_SLEEP
+/**
+ * eswin_sdhci_sdio_suspend- Suspend method for the driver
+ * @dev:        Address of the device structure
+ *
+ * Put the device in a low power state.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
+		mmc_retune_needed(host->mmc);
+
+	if (eswin_sdhci_sdio->has_cqe) {
+		ret = cqhci_suspend(host->mmc);
+		if (ret)
+			return ret;
+	}
+
+	ret = sdhci_suspend_host(host);
+	if (ret)
+		return ret;
+
+	clk_disable(pltfm_host->clk);
+	clk_disable(eswin_sdhci_sdio->clk_ahb);
+
+	return 0;
+}
+
+/**
+ * eswin_sdhci_sdio_resume- Resume method for the driver
+ * @dev:        Address of the device structure
+ *
+ * Resume operation after suspend
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_enable(eswin_sdhci_sdio->clk_ahb);
+	if (ret) {
+		dev_err(dev, "Cannot enable AHB clock.\n");
+		return ret;
+	}
+
+	ret = clk_enable(pltfm_host->clk);
+	if (ret) {
+		dev_err(dev, "Cannot enable SD clock.\n");
+		return ret;
+	}
+
+	ret = sdhci_resume_host(host);
+	if (ret) {
+		dev_err(dev, "Cannot resume host.\n");
+		return ret;
+	}
+
+	if (eswin_sdhci_sdio->has_cqe)
+		return cqhci_resume(host->mmc);
+
+	return 0;
+}
+#endif /* ! CONFIG_PM_SLEEP */
+
+static SIMPLE_DEV_PM_OPS(eswin_sdhci_sdio_dev_pm_ops, eswin_sdhci_sdio_suspend,
+			 eswin_sdhci_sdio_resume);
+
+/**
+ * eswin_sdhci_sdio_sdcardclk_recalc_rate- Return the card clock rate
+ *
+ * @hw:                 Pointer to the hardware clock structure.
+ * @parent_rate:                The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the SD card clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The card clock rate.
+ */
+static unsigned long
+eswin_sdhci_sdio_sdcardclk_recalc_rate(struct clk_hw *hw,
+				       unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data = container_of(
+		hw, struct eswin_sdhci_clk_data, sdcardclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci_sdio->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sdio_sdcardclk_ops = {
+	.recalc_rate = eswin_sdhci_sdio_sdcardclk_recalc_rate,
+};
+
+/**
+ * eswin_sdhci_sdio_sampleclk_recalc_rate- Return the sampling clock rate
+ *
+ * @hw:                 Pointer to the hardware clock structure.
+ * @parent_rate:                The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the sampling clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The sample clock rate.
+ */
+static unsigned long
+eswin_sdhci_sdio_sampleclk_recalc_rate(struct clk_hw *hw,
+				       unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data = container_of(
+		hw, struct eswin_sdhci_clk_data, sampleclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci_sdio->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sdio_sampleclk_ops = {
+	.recalc_rate = eswin_sdhci_sdio_sampleclk_recalc_rate,
+};
+
+static const struct eswin_sdhci_clk_ops eswin_sdio_clk_ops = {
+	.sdcardclk_ops = &eswin_sdio_sdcardclk_ops,
+	.sampleclk_ops = &eswin_sdio_sampleclk_ops,
+};
+
+static struct eswin_sdhci_of_data eswin_sdhci_fu800_sdio_data = {
+	.pdata = &eswin_sdhci_sdio_cqe_pdata,
+	.clk_ops = &eswin_sdio_clk_ops,
+};
+
+static const struct of_device_id eswin_sdhci_sdio_of_match[] = {
+	/* SoC-specific compatible strings*/
+	{
+		.compatible = "eswin,sdhci-sdio",
+		.data = &eswin_sdhci_fu800_sdio_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, eswin_sdhci_sdio_of_match);
+
+/**
+ * eswin_sdhci_sdio_register_sdcardclk- Register the sdcardclk for a PHY to use
+ *
+ * @sdhci_arasan:       Our private data structure.
+ * @clk_xin:            Pointer to the functional clock
+ * @dev:                Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_register_sdcardclk(
+	struct eswin_sdhci_data *eswin_sdhci_sdio, struct clk *clk_xin,
+	struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		&eswin_sdhci_sdio->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sdcardclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 0,
+					    &sdcardclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sdcardclk_init.parent_names = &parent_clk_name;
+	sdcardclk_init.num_parents = 1;
+	sdcardclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sdcardclk_init.ops = eswin_sdhci_sdio->clk_ops->sdcardclk_ops;
+
+	clk_data->sdcardclk_hw.init = &sdcardclk_init;
+	clk_data->sdcardclk = devm_clk_register(dev, &clk_data->sdcardclk_hw);
+	if (IS_ERR(clk_data->sdcardclk))
+		return PTR_ERR(clk_data->sdcardclk);
+	clk_data->sdcardclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+				  clk_data->sdcardclk);
+	if (ret)
+		dev_err(dev, "Failed to add sdcard clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_sdio_register_sampleclk - Register the sampleclk for a PHY to use
+ *
+ * @sdhci_arasan:       Our private data structure.
+ * @clk_xin:            Pointer to the functional clock
+ * @dev:                Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_sdio_register_sampleclk(
+	struct eswin_sdhci_data *eswin_sdhci_sdio, struct clk *clk_xin,
+	struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		&eswin_sdhci_sdio->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sampleclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 1,
+					    &sampleclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sampleclk_init.parent_names = &parent_clk_name;
+	sampleclk_init.num_parents = 1;
+	sampleclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sampleclk_init.ops = eswin_sdhci_sdio->clk_ops->sampleclk_ops;
+
+	clk_data->sampleclk_hw.init = &sampleclk_init;
+	clk_data->sampleclk = devm_clk_register(dev, &clk_data->sampleclk_hw);
+	if (IS_ERR(clk_data->sampleclk))
+		return PTR_ERR(clk_data->sampleclk);
+	clk_data->sampleclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+				  clk_data->sampleclk);
+	if (ret)
+		dev_err(dev, "Failed to add sample clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_sdio_unregister_sdclk- Undoes sdhci_arasan_register_sdclk()
+ *
+ * @dev:                Pointer to our struct device.
+ *
+ * Should be called any time we're exiting and sdhci_arasan_register_sdclk()
+ * returned success.
+ */
+static void eswin_sdhci_sdio_unregister_sdclk(struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+
+	if (!of_find_property(np, "#clock-cells", NULL))
+		return;
+
+	of_clk_del_provider(dev->of_node);
+}
+
+/**
+ * eswin_sdhci_sdio_register_sdclk- Register the sdcardclk for a PHY to use
+ *
+ * @eswin_sdhci_sdio:   Our private data structure.
+ * @clk_xin:            Pointer to the functional clock
+ * @dev:                Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Note: without seriously re-architecting SDHCI's clock code and testing on
+ * all platforms, there's no way to create a totally beautiful clock here
+ * with all clock ops implemented.      Instead, we'll just create a clock that can
+ * be queried and set the CLK_GET_RATE_NOCACHE attribute to tell common clock
+ * framework that we're doing things behind its back.  This should be sufficient
+ * to create nice clean device tree bindings and later (if needed) we can try
+ * re-architecting SDHCI if we see some benefit to it.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int
+eswin_sdhci_sdio_register_sdclk(struct eswin_sdhci_data *eswin_sdhci_sdio,
+				struct clk *clk_xin, struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	u32 num_clks = 0;
+	int ret;
+
+	/* Providing a clock to the PHY is optional; no error if missing */
+	if (of_property_read_u32(np, "#clock-cells", &num_clks) < 0)
+		return 0;
+
+	ret = eswin_sdhci_sdio_register_sdcardclk(eswin_sdhci_sdio, clk_xin,
+						  dev);
+	if (ret)
+		return ret;
+
+	if (num_clks) {
+		ret = eswin_sdhci_sdio_register_sampleclk(eswin_sdhci_sdio,
+							  clk_xin, dev);
+		if (ret) {
+			eswin_sdhci_sdio_unregister_sdclk(dev);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+eswin_sdhci_sdio_add_host(struct eswin_sdhci_data *eswin_sdhci_sdio)
+{
+	struct sdhci_host *host = eswin_sdhci_sdio->host;
+	struct cqhci_host *cq_host;
+	bool dma64;
+	int ret;
+
+	if (!eswin_sdhci_sdio->has_cqe)
+		return sdhci_add_host(host);
+
+	ret = sdhci_setup_host(host);
+	if (ret)
+		return ret;
+
+	cq_host = devm_kzalloc(host->mmc->parent, sizeof(*cq_host), GFP_KERNEL);
+	if (!cq_host) {
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	cq_host->mmio = host->ioaddr + ESWIN_SDHCI_SD_CQE_BASE_ADDR;
+	cq_host->ops = &eswin_sdhci_sdio_cqhci_ops;
+
+	dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
+	if (dma64)
+		cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+
+	ret = cqhci_init(cq_host, host->mmc, dma64);
+	if (ret)
+		goto cleanup;
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto cleanup;
+
+	return 0;
+
+cleanup:
+	sdhci_cleanup_host(host);
+	return ret;
+}
+
+static int eswin_sdhci_sdio_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_sdio_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev,
+			"dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+					 &hsp_mmu_sdio_reg);
+	if (ret) {
+		dev_err(dev, "can't get sdio sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_sdio_reg, rdwr_sid_ssid);
+
+	ret = eic7700_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config sdio streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config sdio streamID(%d)!\n", sid);
+
+	return ret;
+}
+
+static int eswin_sdhci_sdio_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct clk *clk_xin;
+	struct clk *clk_spll2_fout3;
+	struct clk *clk_mux;
+	struct sdhci_host *host;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct eswin_sdhci_data *eswin_sdhci_sdio;
+	struct eswin_sdio_private *eswin_sdio_priv;
+	struct regmap *regmap;
+	const struct eswin_sdhci_of_data *data;
+	unsigned int sdio_id = 0;
+	unsigned int val = 0;
+
+	data = of_device_get_match_data(dev);
+	host = sdhci_pltfm_init(pdev, data->pdata, sizeof(*eswin_sdhci_sdio) + sizeof(*eswin_sdio_priv));
+
+	if (IS_ERR(host))
+		return PTR_ERR(host);
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci_sdio = sdhci_pltfm_priv(pltfm_host);
+	eswin_sdhci_sdio->host = host;
+	eswin_sdhci_sdio->has_cqe = false;
+	eswin_sdio_priv = sdhci_sdio_priv(eswin_sdhci_sdio);
+
+	ret = of_property_read_u32(dev->of_node, "core-clk-reg", &val);
+	if (ret) {
+		dev_err(dev, "get core clk reg failed.\n");
+		goto err_pltfm_free;
+	}
+
+	eswin_sdhci_sdio->core_clk_reg = ioremap(val, 0x4);
+	if (!eswin_sdhci_sdio->core_clk_reg) {
+		dev_err(dev, "ioremap core clk reg failed.\n");
+		goto err_pltfm_free;
+	}
+
+	ret = of_property_read_u32(dev->of_node, "sdio-id", &sdio_id);
+	if (ret) {
+		dev_err(dev, "get sdio-id failed.\n");
+		goto err_pltfm_free;
+	}
+
+	sdhci_get_of_property(pdev);
+
+	eswin_sdhci_sdio->clk_ops = data->clk_ops;
+	eswin_sdhci_sdio->clk_ahb = devm_clk_get(dev, "clk_ahb");
+	if (IS_ERR(eswin_sdhci_sdio->clk_ahb)) {
+		ret = dev_err_probe(dev, PTR_ERR(eswin_sdhci_sdio->clk_ahb),
+				    "clk_ahb clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	clk_xin = devm_clk_get(dev, "clk_xin");
+	if (IS_ERR(clk_xin)) {
+		ret = dev_err_probe(dev, PTR_ERR(clk_xin),
+				    "clk_xin clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	clk_spll2_fout3 = devm_clk_get(dev, "clk_spll2_fout3");
+
+	if (IS_ERR(clk_spll2_fout3)) {
+		ret = dev_err_probe(dev, PTR_ERR(clk_spll2_fout3),
+				    "clk_spll2_fout3 clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	if (of_device_is_compatible(np, "eswin,sdhci-sdio")) {
+		clk_mux = devm_clk_get(dev, "clk_mux1_1");
+		if (IS_ERR(clk_mux)) {
+			ret = dev_err_probe(dev, PTR_ERR(clk_mux),
+					    "clk_mux1_1 clock not found.\n");
+			goto err_pltfm_free;
+		}
+		/*switch the core clk source*/
+		clk_set_parent(clk_mux, clk_spll2_fout3);
+	}
+
+	ret = clk_prepare_enable(eswin_sdhci_sdio->clk_ahb);
+	if (ret) {
+		dev_err(dev, "Unable to enable AHB clock.\n");
+		goto err_pltfm_free;
+	}
+	/* If clock-frequency property is set, use the provided value */
+	if (pltfm_host->clock && pltfm_host->clock != clk_get_rate(clk_xin)) {
+		ret = clk_set_rate(clk_xin, pltfm_host->clock);
+		if (ret) {
+			dev_err(&pdev->dev, "Failed to set SD clock rate\n");
+			goto clk_dis_ahb;
+		}
+	}
+
+	ret = clk_prepare_enable(clk_xin);
+	if (ret) {
+		dev_err(dev, "Unable to enable SD clock.\n");
+		goto clk_dis_ahb;
+	}
+
+	pltfm_host->clk = clk_xin;
+	ret = eswin_sdhci_sdio_register_sdclk(eswin_sdhci_sdio, clk_xin, dev);
+	if (ret)
+		goto clk_disable_all;
+
+	ret = eswin_sdhci_reset_init(dev, eswin_sdhci_sdio);
+	if (ret < 0) {
+		dev_err(dev, "failed to reset\n");
+		goto clk_disable_all;
+	}
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	if (sdio_id == 0) {
+		regmap_write(regmap, ESWIN_SDHCI_SD0_INT_STATUS,
+			     MSHC_INT_CLK_STABLE);
+		regmap_write(regmap, ESWIN_SDHCI_SD0_PWR_CTRL, MSHC_HOST_VAL_STABLE);
+	} else {
+		regmap_write(regmap, ESWIN_SDHCI_SD1_INT_STATUS,
+			     MSHC_INT_CLK_STABLE);
+		regmap_write(regmap, ESWIN_SDHCI_SD1_PWR_CTRL, MSHC_HOST_VAL_STABLE);
+	}
+
+	ret = eswin_sdhci_sdio_sid_cfg(dev);
+	if (ret < 0) {
+		dev_err(dev, "failed to use smmu\n");
+		goto clk_disable_all;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "delay_code", &val)) {
+		eswin_sdhci_sdio->phy.delay_code = val;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "drive-impedance-ohm", &val))
+		eswin_sdhci_sdio->phy.drive_impedance =
+			eswin_convert_drive_impedance_ohm(pdev, val);
+
+	if (of_property_read_bool(dev->of_node, "enable-cmd-pullup"))
+		eswin_sdhci_sdio->phy.enable_cmd_pullup = ENABLE;
+	else
+		eswin_sdhci_sdio->phy.enable_cmd_pullup = DISABLE;
+
+	if (of_property_read_bool(dev->of_node, "enable-data-pullup"))
+		eswin_sdhci_sdio->phy.enable_data_pullup = ENABLE;
+	else
+		eswin_sdhci_sdio->phy.enable_data_pullup = DISABLE;
+
+	if (of_property_read_bool(dev->of_node, "enable_sw_tuning"))
+		eswin_sdio_priv->enable_sw_tuning = ENABLE;
+	else
+		eswin_sdio_priv->enable_sw_tuning = DISABLE;
+
+	if (!of_property_read_u32(dev->of_node, "phase_code", &val)) {
+		eswin_sdio_priv->phase_code = val;
+	} else {
+		eswin_sdio_priv->phase_code = -1;
+	}
+
+	eswin_sdhci_dt_parse_clk_phases(dev, &eswin_sdhci_sdio->clk_data);
+	ret = mmc_of_parse(host->mmc);
+	if (ret) {
+		ret = dev_err_probe(dev, ret, "parsing dt failed.\n");
+		goto unreg_clk;
+	}
+
+	eic7700_tbu_power(&pdev->dev, true);
+
+	ret = eswin_sdhci_sdio_add_host(eswin_sdhci_sdio);
+	if (ret)
+		goto unreg_clk;
+
+	return 0;
+
+unreg_clk:
+	eswin_sdhci_sdio_unregister_sdclk(dev);
+clk_disable_all:
+	clk_disable_unprepare(clk_xin);
+clk_dis_ahb:
+	clk_disable_unprepare(eswin_sdhci_sdio->clk_ahb);
+err_pltfm_free:
+	if (eswin_sdhci_sdio->core_clk_reg)
+		iounmap(eswin_sdhci_sdio->core_clk_reg);
+
+	sdhci_pltfm_free(pdev);
+	return ret;
+}
+
+static int eswin_sdhci_sdio_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci_sdio =
+		sdhci_pltfm_priv(pltfm_host);
+	struct clk *clk_ahb = eswin_sdhci_sdio->clk_ahb;
+	void __iomem *core_clk_reg = eswin_sdhci_sdio->core_clk_reg;
+
+	sdhci_pltfm_remove(pdev);
+	eic7700_tbu_power(&pdev->dev, false);
+
+	if (eswin_sdhci_sdio->txrx_rst) {
+		ret = reset_control_assert(eswin_sdhci_sdio->txrx_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci_sdio->phy_rst) {
+		ret = reset_control_assert(eswin_sdhci_sdio->phy_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci_sdio->prstn) {
+		ret = reset_control_assert(eswin_sdhci_sdio->prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci_sdio->arstn) {
+		ret = reset_control_assert(eswin_sdhci_sdio->arstn);
+		WARN_ON(0 != ret);
+	}
+
+	eswin_sdhci_sdio_unregister_sdclk(&pdev->dev);
+	clk_disable_unprepare(clk_ahb);
+	iounmap(core_clk_reg);
+
+	return 0;
+}
+
+static struct platform_driver eswin_sdhci_sdio_driver = {
+	.driver = {
+		.name = "eswin-sdhci-sdio",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.of_match_table = eswin_sdhci_sdio_of_match,
+		.pm = &eswin_sdhci_sdio_dev_pm_ops,
+	},
+	.probe = eswin_sdhci_sdio_probe,
+	.remove = eswin_sdhci_sdio_remove,
+};
+
+static __init int eswin_sdhci_sdio_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&eswin_sdhci_sdio_driver);
+	if (ret) {
+		pr_err("%s: failed to register platform driver\n",
+			__func__);
+	}
+
+	return ret;
+}
+
+static void __exit eswin_sdhci_sdio_exit(void)
+{
+	platform_driver_unregister(&eswin_sdhci_sdio_driver);
+}
+
+/*Cause EMMC is often used as a system disk(mmc0), we need the SD driver to run later than the EMMC driver*/
+late_initcall(eswin_sdhci_sdio_init);
+module_exit(eswin_sdhci_sdio_exit);
+
+MODULE_DESCRIPTION("Driver for the Eswin SDHCI Controller");
+MODULE_AUTHOR("Eswin");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mmc/host/sdhci-of-eswin.c b/drivers/mmc/host/sdhci-of-eswin.c
new file mode 100644
index 000000000000..008b243dff32
--- /dev/null
+++ b/drivers/mmc/host/sdhci-of-eswin.c
@@ -0,0 +1,1104 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Emmc Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: liangshuang <liangshuang@eswin.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include "cqhci.h"
+#include "sdhci-pltfm.h"
+#include <linux/reset.h>
+#include <linux/iommu.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/eic7700-sid-cfg.h>
+#include "sdhci-eswin.h"
+
+#define SDHCI_EMMC0_INT_STATUS 0x508
+#define SDHCI_EMMC0_PWR_CLEAR 0x50c
+
+//EMMC_DWC_MSHC_CRYPTO_CFG_PTR 8 -- parameter
+#define eswin_sdhci_VENDOR_REGISTER_BASEADDR 0x800
+#define eswin_sdhci_VENDOR_EMMC_CTRL_REGISTER 0x2c
+#define VENDOR_ENHANCED_STROBE BIT(8)
+
+#define eswin_sdhci_CQE_BASE_ADDR eswin_sdhci_VENDOR_REGISTER_BASEADDR
+
+/* Controller does not have CD wired and will not function normally without */
+#define eswin_sdhci_QUIRK_FORCE_CDTEST BIT(0)
+/* Controller immediately reports SDHCI_CLOCK_INT_STABLE after enabling the
+ * internal clock even when the clock isn't stable */
+#define eswin_sdhci_QUIRK_CLOCK_UNSTABLE BIT(1)
+
+/*
+ * On some SoCs the syscon area has a feature where the upper 16-bits of
+ * each 32-bit register act as a write mask for the lower 16-bits.  This allows
+ * atomic updates of the register without locking.  This macro is used on SoCs
+ * that have that feature.
+ */
+#define HIWORD_UPDATE(val, mask, shift) \
+	((val) << (shift) | (mask) << ((shift) + 16))
+
+#define ESWIN_EMMC_CORE_CLK_REG 0x51828160
+
+static void eswin_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+
+	/* Set the Input and Output Clock Phase Delays */
+	if (clk_data->set_clk_delays)
+		clk_data->set_clk_delays(host);
+
+	eswin_sdhci_set_core_clock(host, clock);
+	sdhci_set_clock(host, clock);
+
+	/*
+	 * Some controllers immediately report SDHCI_CLOCK_INT_STABLE
+	 * after enabling the clock even though the clock is not
+	 * stable. Trying to use a clock without waiting here results
+	 * in EILSEQ while detecting some older/slower cards. The
+	 * chosen delay is the maximum delay from sdhci_set_clock.
+	 */
+	if (eswin_sdhci->quirks & SDHCI_ESWIN_QUIRK_CLOCK_UNSTABLE)
+		msleep(20);
+}
+
+#if !defined(__FPGA) && !defined(__ZEBU)
+static void eswin_sdhci_hs400_enhanced_strobe(struct mmc_host *mmc,
+					      struct mmc_ios *ios)
+{
+	u32 vendor;
+	struct sdhci_host *host = mmc_priv(mmc);
+
+	vendor = sdhci_readl(host, eswin_sdhci_VENDOR_EMMC_CTRL_REGISTER);
+	if (ios->enhanced_strobe)
+		vendor |= VENDOR_ENHANCED_STROBE;
+	else
+		vendor &= ~VENDOR_ENHANCED_STROBE;
+
+	sdhci_writel(host, vendor, eswin_sdhci_VENDOR_EMMC_CTRL_REGISTER);
+}
+#endif
+
+static void eswin_sdhci_config_phy_delay(struct sdhci_host *host, int delay)
+{
+	delay &= PHY_CLK_MAX_DELAY_MASK;
+
+	/*phy clk delay line config*/
+	sdhci_writeb(host, PHY_UPDATE_DELAY_CODE, PHY_SDCLKDL_CNFG_R);
+	sdhci_writeb(host, delay, PHY_SDCLKDL_DC_R);
+	sdhci_writeb(host, 0x0, PHY_SDCLKDL_CNFG_R);
+}
+
+static void eswin_sdhci_config_phy(struct sdhci_host *host)
+{
+	unsigned int val = 0;
+	unsigned int drv = 0;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct eswin_sdhci_phy_data *phy = &eswin_sdhci->phy;
+
+	drv = phy->drive_impedance << PHY_PAD_SP_DRIVE_SHIF;
+
+	pr_debug("%s: phy drv=0x%x\n", mmc_hostname(host->mmc), drv);
+
+	val = sdhci_readw(host, VENDOR_EMMC_CTRL_R);
+	val |= EMMC_CRAD_PRESENT;  // emmc card
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+
+	eswin_sdhci_disable_card_clk(host);
+
+	/* reset phy,config phy's pad */
+	sdhci_writel(host, drv | (~PHY_RSTN), PHY_CNFG_R);
+	/*CMDPAD_CNFS*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		(phy->enable_cmd_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_CMDPAD_CNFG_R);
+	pr_debug("%s: phy cmd=0x%x\n", mmc_hostname(host->mmc), val);
+
+	/*DATA PAD CNFG*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		(phy->enable_data_pullup << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_DATAPAD_CNFG_R);
+	pr_debug("%s: phy data=0x%x\n", mmc_hostname(host->mmc), val);
+
+	/*Clock PAD Setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) | PHY_PAD_RXSEL_0;
+	sdhci_writew(host, val, PHY_CLKPAD_CNFG_R);
+	pr_debug("%s: phy clk=0x%x\n", mmc_hostname(host->mmc), val);
+
+	/*PHY strobe PAD setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		((phy->enable_strobe_pulldown * PHY_PULL_DOWN) << PHY_PULL_BIT_SHIF) |
+		PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_STBPAD_CNFG_R);
+	pr_debug("%s: phy strobe=0x%x\n", mmc_hostname(host->mmc), val);
+	mdelay(2);
+
+	/*PHY RSTN PAD setting*/
+	val = (PHY_SLEW_2 << PHY_TX_SLEW_CTRL_P_BIT_SHIFT) |
+		(PHY_SLEW_2 << PHY_TX_SLEW_CTRL_N_BIT_SHIFT) |
+		(PHY_PULL_UP << PHY_PULL_BIT_SHIF) | PHY_PAD_RXSEL_1;
+	sdhci_writew(host, val, PHY_RSTNPAD_CNFG_R);
+	pr_debug("%s: phy rstn=0x%x\n", mmc_hostname(host->mmc), val);
+
+	sdhci_writel(host, drv | PHY_RSTN, PHY_CNFG_R);
+
+	eswin_sdhci_config_phy_delay(host, phy->delay_code);
+
+	eswin_sdhci_enable_card_clk(host);
+}
+
+static void eswin_sdhci_reset(struct sdhci_host *host, u8 mask)
+{
+	u8 ctrl;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	/* Disable signal and interrupts before resetting the phy.
+	 * Doing this avoids ISR to serve any undesired interrupts during
+	 * reset and avoid producing the fake register dump during probe.
+	 */
+	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
+	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
+	sdhci_reset(host, mask);
+	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
+	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+
+	if (eswin_sdhci->quirks & SDHCI_ESWIN_QUIRK_FORCE_CDTEST) {
+		ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
+		ctrl |= SDHCI_CTRL_CDTEST_INS | SDHCI_CTRL_CDTEST_EN;
+		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
+	}
+
+	if (mask == SDHCI_RESET_ALL) { // after reset all,the phy`s config will be clear.
+		eswin_sdhci_config_phy(host);
+	}
+}
+
+static u32 eswin_sdhci_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+	int cmd_error = 0;
+	int data_error = 0;
+
+	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+		return intmask;
+
+	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+	return 0;
+}
+
+static void eswin_sdhci_dumpregs(struct mmc_host *mmc)
+{
+	sdhci_dumpregs(mmc_priv(mmc));
+}
+
+static void eswin_sdhci_cqe_enable(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	u32 reg;
+
+	reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	while (reg & SDHCI_DATA_AVAILABLE) {
+		sdhci_readl(host, SDHCI_BUFFER);
+		reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	}
+
+	sdhci_cqe_enable(mmc);
+}
+
+static int eswin_sdhci_delay_tuning(struct sdhci_host *host, u32 opcode)
+{
+	int ret;
+	int delay = 0;
+	int i = 0;
+	int delay_min = -1;
+	int delay_max = -1;
+	int cmd_error = 0;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+
+	for (i = 0; i <= PHY_DELAY_CODE_MAX; i++) {
+		eswin_sdhci_disable_card_clk(host);
+		eswin_sdhci_config_phy_delay(host, i);
+		eswin_sdhci_enable_card_clk(host);
+		ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+		if (ret) {
+			host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+			udelay(200);
+			if (delay_min != -1 && delay_max != -1)
+				break;
+		} else {
+			if (delay_min == -1) {
+				delay_min = i;
+				continue;
+			} else {
+				delay_max = i;
+				continue;
+			}
+		}
+	}
+	if (delay_min == -1 && delay_max == -1) {
+		pr_err("%s: delay code tuning failed!\n",
+					mmc_hostname(host->mmc));
+		eswin_sdhci_disable_card_clk(host);
+		eswin_sdhci_config_phy_delay(host, eswin_sdhci->phy.delay_code);
+		eswin_sdhci_enable_card_clk(host);
+
+		return ret;
+	}
+
+	delay = (delay_min + delay_max) / 2;
+	pr_info("%s: set delay:0x%x\n", mmc_hostname(host->mmc), delay);
+	eswin_sdhci_disable_card_clk(host);
+	eswin_sdhci_config_phy_delay(host, delay);
+	eswin_sdhci_enable_card_clk(host);
+
+	return 0;
+}
+
+static int eswin_sdhci_phase_code_tuning(struct sdhci_host *host, u32 opcode)
+{
+	int cmd_error = 0;
+	int ret = 0;
+	int phase_code = 0;
+	int code_min = -1;
+	int code_max = -1;
+
+	for (phase_code = 0; phase_code <= MAX_PHASE_CODE; phase_code++) {
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, phase_code, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+
+		ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
+		if (ret) {
+			host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+			udelay(200);
+			if (code_min != -1 && code_max != -1)
+				break;
+		} else {
+			if (code_min == -1) {
+				code_min = phase_code;
+				continue;
+			} else {
+				code_max = phase_code;
+				continue;
+			}
+		}
+	}
+	if (code_min == -1 && code_max == -1) {
+		pr_err("%s: phase code tuning failed!\n",
+					mmc_hostname(host->mmc));
+		eswin_sdhci_disable_card_clk(host);
+		sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+		eswin_sdhci_enable_card_clk(host);
+		return -EIO;
+	}
+
+	phase_code = (code_min + code_max) / 2;
+	pr_info("%s: set phase_code:0x%x\n", mmc_hostname(host->mmc), phase_code);
+
+	eswin_sdhci_disable_card_clk(host);
+	sdhci_writew(host, phase_code, VENDOR_AT_SATA_R);
+	eswin_sdhci_enable_card_clk(host);
+
+	return 0;
+}
+
+static int eswin_sdhci_executing_tuning(struct sdhci_host *host, u32 opcode)
+{
+	u32 ctrl;
+	u32 val;
+	int ret = 0;
+
+	eswin_sdhci_disable_card_clk(host);
+
+	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	ctrl &= ~SDHCI_CTRL_TUNED_CLK;
+	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
+
+	val = sdhci_readl(host, VENDOR_AT_CTRL_R);
+	val |= SW_TUNE_ENABLE;
+	sdhci_writew(host, val, VENDOR_AT_CTRL_R);
+	sdhci_writew(host, 0, VENDOR_AT_SATA_R);
+
+	eswin_sdhci_enable_card_clk(host);
+
+	sdhci_writew(host, 0x0, SDHCI_CMD_DATA);
+
+	ret = eswin_sdhci_delay_tuning(host, opcode);
+	if (ret < 0) {
+		return ret;
+	}
+	ret = eswin_sdhci_phase_code_tuning(host, opcode);
+	if (ret < 0) {
+		return ret;
+	}
+
+	return 0;
+}
+
+void eswin_sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
+{
+	u32 val;
+	u32 status;
+	u32 timeout = 0;
+	u16 ctrl_2;
+
+	ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	/* Select Bus Speed Mode for host */
+	ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
+	if ((timing == MMC_TIMING_MMC_HS200) ||
+	    (timing == MMC_TIMING_UHS_SDR104))
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
+	else if (timing == MMC_TIMING_UHS_SDR12)
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
+	else if (timing == MMC_TIMING_UHS_SDR25)
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
+	else if (timing == MMC_TIMING_UHS_SDR50)
+		ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
+	else if ((timing == MMC_TIMING_UHS_DDR50) ||
+		 (timing == MMC_TIMING_MMC_DDR52))
+		ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
+	else if (timing == MMC_TIMING_MMC_HS400)
+		ctrl_2 |= ESWIN_SDHCI_CTRL_HS400; /* Non-standard */
+	sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
+
+
+	/*
+	 * here need make dll locked when in hs400 at 200MHz
+	 */
+	if ((timing == MMC_TIMING_MMC_HS400) && (host->clock == 200000000)) {
+		eswin_sdhci_disable_card_clk(host);
+
+		val = sdhci_readl(host, VENDOR_AT_CTRL_R);
+		val &= ~(LATENCY_LT_MASK << LATENCY_LT_BIT_OFFSET);
+		val |= (LATENCY_LT_3 << LATENCY_LT_MASK);
+		sdhci_writel(host, val, VENDOR_AT_CTRL_R);
+
+		sdhci_writeb(host, 0x23, PHY_DLL_CNFG1_R);
+		sdhci_writeb(host, 0x02, PHY_DLL_CNFG2_R);
+		sdhci_writeb(host, 0x60, PHY_DLLDL_CNFG_R);
+		sdhci_writeb(host, 0x00, PHY_DLL_OFFST_R);
+		sdhci_writew(host, 0xffff, PHY_DLLBT_CNFG_R);
+
+		eswin_sdhci_enable_card_clk(host);
+		sdhci_writeb(host, DLL_ENABEL, PHY_DLL_CTRL_R);
+		udelay(100);
+
+		while (1) {
+			status = sdhci_readb(host, PHY_DLL_STATUS_R);
+			if (status & DLL_LOCK_STS) {
+				pr_debug("%s: locked status:0x%x\n", mmc_hostname(host->mmc), status);
+				break;
+			}
+			timeout++;
+			udelay(100);
+			if (timeout > 10000) {
+				pr_err("%s: DLL lock failed!status:0x%x\n",
+					mmc_hostname(host->mmc), status);
+				return;
+			}
+		}
+
+		status = sdhci_readb(host, PHY_DLL_STATUS_R);
+		if (status & DLL_ERROR_STS) {
+			pr_err("%s: DLL lock failed!err_status:0x%x\n",
+				mmc_hostname(host->mmc), status);
+		} else {
+			pr_debug("%s: DLL lock is success\n", mmc_hostname(host->mmc));
+		}
+	}
+}
+
+static const struct cqhci_host_ops eswin_sdhci_cqhci_ops = {
+	.enable = eswin_sdhci_cqe_enable,
+	.disable = sdhci_cqe_disable,
+	.dumpregs = eswin_sdhci_dumpregs,
+};
+
+static const struct sdhci_ops eswin_sdhci_cqe_ops = {
+	.set_clock = eswin_sdhci_set_clock,
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = eswin_sdhci_reset,
+	.set_uhs_signaling = eswin_sdhci_set_uhs_signaling,
+	.set_power = sdhci_set_power_and_bus_voltage,
+	.irq = eswin_sdhci_cqhci_irq,
+	.platform_execute_tuning = eswin_sdhci_executing_tuning,
+};
+
+static const struct sdhci_pltfm_data eswin_sdhci_cqe_pdata = {
+	.ops = &eswin_sdhci_cqe_ops,
+	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+#if defined(__DISABLE_HS200)
+		SDHCI_QUIRK2_BROKEN_HS200 |
+#endif
+		SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
+};
+
+#ifdef CONFIG_PM_SLEEP
+/**
+ * eswin_sdhci_suspend - Suspend method for the driver
+ * @dev:    Address of the device structure
+ *
+ * Put the device in a low power state.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
+		mmc_retune_needed(host->mmc);
+
+	if (eswin_sdhci->has_cqe) {
+		ret = cqhci_suspend(host->mmc);
+		if (ret)
+			return ret;
+	}
+
+	ret = sdhci_suspend_host(host);
+	if (ret)
+		return ret;
+
+	clk_disable(pltfm_host->clk);
+	clk_disable(eswin_sdhci->clk_ahb);
+
+	return 0;
+}
+
+/**
+ * eswin_sdhci_resume - Resume method for the driver
+ * @dev:    Address of the device structure
+ *
+ * Resume operation after suspend
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_enable(eswin_sdhci->clk_ahb);
+	if (ret) {
+		dev_err(dev, "Cannot enable AHB clock.\n");
+		return ret;
+	}
+
+	ret = clk_enable(pltfm_host->clk);
+	if (ret) {
+		dev_err(dev, "Cannot enable SD clock.\n");
+		return ret;
+	}
+
+	ret = sdhci_resume_host(host);
+	if (ret) {
+		dev_err(dev, "Cannot resume host.\n");
+		return ret;
+	}
+
+	if (eswin_sdhci->has_cqe)
+		return cqhci_resume(host->mmc);
+
+	return 0;
+}
+#endif /* ! CONFIG_PM_SLEEP */
+
+static SIMPLE_DEV_PM_OPS(eswin_sdhci_dev_pm_ops, eswin_sdhci_suspend,
+			 eswin_sdhci_resume);
+
+/**
+ * eswin_sdhci_sdcardclk_recalc_rate - Return the card clock rate
+ *
+ * @hw:         Pointer to the hardware clock structure.
+ * @parent_rate:        The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the SD card clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The card clock rate.
+ */
+static unsigned long
+eswin_sdhci_sdcardclk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		container_of(hw, struct eswin_sdhci_clk_data, sdcardclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sdcardclk_ops = {
+	.recalc_rate = eswin_sdhci_sdcardclk_recalc_rate,
+};
+
+/**
+ * eswin_sdhci_sampleclk_recalc_rate - Return the sampling clock rate
+ *
+ * @hw:         Pointer to the hardware clock structure.
+ * @parent_rate:        The parent rate (should be rate of clk_xin).
+ *
+ * Return the current actual rate of the sampling clock.  This can be used
+ * to communicate with out PHY.
+ *
+ * Return: The sample clock rate.
+ */
+static unsigned long
+eswin_sdhci_sampleclk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct eswin_sdhci_clk_data *clk_data =
+		container_of(hw, struct eswin_sdhci_clk_data, sampleclk_hw);
+	struct eswin_sdhci_data *eswin_sdhci =
+		container_of(clk_data, struct eswin_sdhci_data, clk_data);
+	struct sdhci_host *host = eswin_sdhci->host;
+
+	return host->mmc->actual_clock;
+}
+
+static const struct clk_ops eswin_sampleclk_ops = {
+	.recalc_rate = eswin_sdhci_sampleclk_recalc_rate,
+};
+
+static const struct eswin_sdhci_clk_ops eswin_clk_ops = {
+	.sdcardclk_ops = &eswin_sdcardclk_ops,
+	.sampleclk_ops = &eswin_sampleclk_ops,
+};
+
+static struct eswin_sdhci_of_data eswin_sdhci_fu800_data = {
+	.pdata = &eswin_sdhci_cqe_pdata,
+	.clk_ops = &eswin_clk_ops,
+};
+
+static const struct of_device_id eswin_sdhci_of_match[] = {
+	{
+		.compatible = "eswin,emmc-sdhci-5.1",
+		.data = &eswin_sdhci_fu800_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, eswin_sdhci_of_match);
+
+/**
+ * eswin_sdhci_register_sdcardclk - Register the sdcardclk for a PHY to use
+ *
+ * @eswin_sdhci:    Our private data structure.
+ * @clk_xin:        Pointer to the functional clock
+ * @dev:        Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_register_sdcardclk(struct eswin_sdhci_data *eswin_sdhci,
+									struct clk *clk_xin, struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sdcardclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 0,
+						&sdcardclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sdcardclk_init.parent_names = &parent_clk_name;
+	sdcardclk_init.num_parents = 1;
+	sdcardclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sdcardclk_init.ops = eswin_sdhci->clk_ops->sdcardclk_ops;
+
+	clk_data->sdcardclk_hw.init = &sdcardclk_init;
+	clk_data->sdcardclk = devm_clk_register(dev, &clk_data->sdcardclk_hw);
+	if (IS_ERR(clk_data->sdcardclk))
+		return PTR_ERR(clk_data->sdcardclk);
+	clk_data->sdcardclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+							clk_data->sdcardclk);
+	if (ret)
+		dev_err(dev, "Failed to add sdcard clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_register_sampleclk - Register the sampleclk for a PHY to use
+ *
+ * @eswin_sdhci:    Our private data structure.
+ * @clk_xin:        Pointer to the functional clock
+ * @dev:        Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_register_sampleclk(struct eswin_sdhci_data *eswin_sdhci,
+						struct clk *clk_xin, struct device *dev)
+{
+	struct eswin_sdhci_clk_data *clk_data = &eswin_sdhci->clk_data;
+	struct device_node *np = dev->of_node;
+	struct clk_init_data sampleclk_init;
+	const char *parent_clk_name;
+	int ret;
+
+	ret = of_property_read_string_index(np, "clock-output-names", 1,
+						&sampleclk_init.name);
+	if (ret) {
+		dev_err(dev, "DT has #clock-cells but no clock-output-names\n");
+		return ret;
+	}
+
+	parent_clk_name = __clk_get_name(clk_xin);
+	sampleclk_init.parent_names = &parent_clk_name;
+	sampleclk_init.num_parents = 1;
+	sampleclk_init.flags = CLK_GET_RATE_NOCACHE;
+	sampleclk_init.ops = eswin_sdhci->clk_ops->sampleclk_ops;
+
+	clk_data->sampleclk_hw.init = &sampleclk_init;
+	clk_data->sampleclk = devm_clk_register(dev, &clk_data->sampleclk_hw);
+	if (IS_ERR(clk_data->sampleclk))
+		return PTR_ERR(clk_data->sampleclk);
+	clk_data->sampleclk_hw.init = NULL;
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get,
+				  clk_data->sampleclk);
+	if (ret)
+		dev_err(dev, "Failed to add sample clock provider\n");
+
+	return ret;
+}
+
+/**
+ * eswin_sdhci_unregister_sdclk - Undoes eswin_sdhci_register_sdclk()
+ *
+ * @dev:        Pointer to our struct device.
+ *
+ * Should be called any time we're exiting and eswin_sdhci_register_sdclk()
+ * returned success.
+ */
+static void eswin_sdhci_unregister_sdclk(struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+
+	if (!of_find_property(np, "#clock-cells", NULL))
+		return;
+
+	of_clk_del_provider(dev->of_node);
+}
+
+/**
+ * eswin_sdhci_register_sdclk - Register the sdcardclk for a PHY to use
+ *
+ * @eswin_sdhci:    Our private data structure.
+ * @clk_xin:        Pointer to the functional clock
+ * @dev:        Pointer to our struct device.
+ *
+ * Some PHY devices need to know what the actual card clock is.  In order for
+ * them to find out, we'll provide a clock through the common clock framework
+ * for them to query.
+ *
+ * Note: without seriously re-architecting SDHCI's clock code and testing on
+ * all platforms, there's no way to create a totally beautiful clock here
+ * with all clock ops implemented.  Instead, we'll just create a clock that can
+ * be queried and set the CLK_GET_RATE_NOCACHE attribute to tell common clock
+ * framework that we're doing things behind its back.  This should be sufficient
+ * to create nice clean device tree bindings and later (if needed) we can try
+ * re-architecting SDHCI if we see some benefit to it.
+ *
+ * Return: 0 on success and error value on error
+ */
+static int eswin_sdhci_register_sdclk(struct eswin_sdhci_data *eswin_sdhci,
+				      struct clk *clk_xin, struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	u32 num_clks = 0;
+	int ret;
+
+	/* Providing a clock to the PHY is optional; no error if missing */
+	if (of_property_read_u32(np, "#clock-cells", &num_clks) < 0)
+		return 0;
+
+	ret = eswin_sdhci_register_sdcardclk(eswin_sdhci, clk_xin, dev);
+	if (ret)
+		return ret;
+
+	if (num_clks) {
+		ret = eswin_sdhci_register_sampleclk(eswin_sdhci, clk_xin, dev);
+		if (ret) {
+			eswin_sdhci_unregister_sdclk(dev);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int eswin_sdhci_add_host(struct eswin_sdhci_data *eswin_sdhci)
+{
+	struct sdhci_host *host = eswin_sdhci->host;
+	struct cqhci_host *cq_host;
+	bool dma64;
+	int ret;
+
+	if (!eswin_sdhci->has_cqe)
+		return sdhci_add_host(host);
+
+	ret = sdhci_setup_host(host);
+	if (ret)
+		return ret;
+
+	cq_host = devm_kzalloc(host->mmc->parent, sizeof(*cq_host), GFP_KERNEL);
+	if (!cq_host) {
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	cq_host->mmio = host->ioaddr + eswin_sdhci_CQE_BASE_ADDR;
+	cq_host->ops = &eswin_sdhci_cqhci_ops;
+
+	dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
+	if (dma64)
+		cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+
+	ret = cqhci_init(cq_host, host->mmc, dma64);
+	if (ret)
+		goto cleanup;
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto cleanup;
+
+	return 0;
+
+cleanup:
+	sdhci_cleanup_host(host);
+	return ret;
+}
+
+static int eswin_emmc_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_emmc_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev,
+			"dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+					 &hsp_mmu_emmc_reg);
+	if (ret) {
+		dev_err(dev, "can't get emmc sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_emmc_reg, rdwr_sid_ssid);
+
+	ret = eic7700_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config emmc streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config emmc streamID(%d)!\n", sid);
+
+	return ret;
+}
+
+static int eswin_sdhci_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct clk *clk_xin;
+	struct sdhci_host *host;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct device *dev = &pdev->dev;
+	struct eswin_sdhci_data *eswin_sdhci;
+	const struct eswin_sdhci_of_data *data;
+	struct regmap *regmap;
+	unsigned int val = 0;
+
+	data = of_device_get_match_data(dev);
+	host = sdhci_pltfm_init(pdev, data->pdata, sizeof(*eswin_sdhci));
+	if (IS_ERR(host))
+		return PTR_ERR(host);
+
+	pltfm_host = sdhci_priv(host);
+	eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	eswin_sdhci->host = host;
+	eswin_sdhci->clk_ops = data->clk_ops;
+
+	eswin_sdhci->core_clk_reg = ioremap(ESWIN_EMMC_CORE_CLK_REG, 0x4);
+	if (!eswin_sdhci->core_clk_reg) {
+		dev_err(dev, "ioremap core clk reg failed.\n");
+		goto err_pltfm_free;
+	}
+
+	eswin_sdhci->clk_ahb = devm_clk_get(dev, "clk_ahb");
+	if (IS_ERR(eswin_sdhci->clk_ahb)) {
+		ret = dev_err_probe(dev, PTR_ERR(eswin_sdhci->clk_ahb),
+				    "clk_ahb clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	clk_xin = devm_clk_get(dev, "clk_xin");
+	if (IS_ERR(clk_xin)) {
+		ret = dev_err_probe(dev, PTR_ERR(clk_xin),
+				    "clk_xin clock not found.\n");
+		goto err_pltfm_free;
+	}
+
+	ret = clk_prepare_enable(eswin_sdhci->clk_ahb);
+	if (ret) {
+		dev_err(dev, "Unable to enable AHB clock.\n");
+		goto err_pltfm_free;
+	}
+
+	ret = clk_prepare_enable(clk_xin);
+	if (ret) {
+		dev_err(dev, "Unable to enable SD clock.\n");
+		goto clk_dis_ahb;
+	}
+
+	ret = eswin_sdhci_reset_init(dev, eswin_sdhci);
+	if (ret < 0) {
+		dev_err(dev, "failed to reset\n");
+		goto clk_disable_all;
+	}
+
+	eic7700_tbu_power(dev, true);
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						 "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		goto clk_disable_all;
+	}
+
+	regmap_write(regmap, SDHCI_EMMC0_INT_STATUS, MSHC_INT_CLK_STABLE);
+	regmap_write(regmap, SDHCI_EMMC0_PWR_CLEAR, MSHC_HOST_VAL_STABLE);
+
+	/* smmu */
+	eswin_emmc_sid_cfg(dev);
+
+	if (!of_property_read_u32(dev->of_node, "delay_code", &val)) {
+		eswin_sdhci->phy.delay_code = val;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "drive-impedance-ohm", &val))
+		eswin_sdhci->phy.drive_impedance =
+			eswin_convert_drive_impedance_ohm(pdev, val);
+
+	if (of_property_read_bool(dev->of_node, "enable-cmd-pullup"))
+		eswin_sdhci->phy.enable_cmd_pullup = ENABLE;
+	else
+		eswin_sdhci->phy.enable_cmd_pullup = DISABLE;
+
+	if (of_property_read_bool(dev->of_node, "enable-data-pullup"))
+		eswin_sdhci->phy.enable_data_pullup = ENABLE;
+	else
+		eswin_sdhci->phy.enable_data_pullup = DISABLE;
+
+	if (of_property_read_bool(dev->of_node, "enable-strobe-pulldown"))
+		eswin_sdhci->phy.enable_strobe_pulldown = ENABLE;
+	else
+		eswin_sdhci->phy.enable_strobe_pulldown = DISABLE;
+
+	sdhci_get_of_property(pdev);
+
+	pltfm_host->clk = clk_xin;
+
+	ret = eswin_sdhci_register_sdclk(eswin_sdhci, clk_xin, dev);
+	if (ret)
+		goto clk_disable_all;
+
+	eswin_sdhci_dt_parse_clk_phases(dev, &eswin_sdhci->clk_data);
+
+	ret = mmc_of_parse(host->mmc);
+	if (ret) {
+		ret = dev_err_probe(dev, ret, "parsing dt failed.\n");
+		goto unreg_clk;
+	}
+
+#if !defined(__FPGA) && !defined(__ZEBU)
+	if (of_device_is_compatible(dev->of_node, "eswin,sdhci-5.1")) {
+		host->mmc_host_ops.hs400_enhanced_strobe =
+			eswin_sdhci_hs400_enhanced_strobe;
+		eswin_sdhci->has_cqe = true;
+		host->mmc->caps2 |= MMC_CAP2_CQE;
+
+		if (!of_property_read_bool(dev->of_node, "disable-cqe-dcmd"))
+			host->mmc->caps2 |= MMC_CAP2_CQE_DCMD;
+	}
+#endif
+
+#if !defined(__ADMA3_DISABLE)
+	sdhci_enable_v4_mode(eswin_sdhci->host);
+#endif
+
+	ret = eswin_sdhci_add_host(eswin_sdhci);
+	if (ret)
+		goto unreg_clk;
+
+	return 0;
+
+unreg_clk:
+	eswin_sdhci_unregister_sdclk(dev);
+clk_disable_all:
+	clk_disable_unprepare(clk_xin);
+clk_dis_ahb:
+	clk_disable_unprepare(eswin_sdhci->clk_ahb);
+err_pltfm_free:
+	if (eswin_sdhci->core_clk_reg)
+		iounmap(eswin_sdhci->core_clk_reg);
+	sdhci_pltfm_free(pdev);
+	return ret;
+}
+
+static int eswin_sdhci_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct eswin_sdhci_data *eswin_sdhci = sdhci_pltfm_priv(pltfm_host);
+	struct clk *clk_ahb = eswin_sdhci->clk_ahb;
+	void __iomem *core_clk_reg = eswin_sdhci->core_clk_reg;
+
+	sdhci_pltfm_remove(pdev);
+	eic7700_tbu_power(&pdev->dev, false);
+
+	if (eswin_sdhci->txrx_rst) {
+		ret = reset_control_assert(eswin_sdhci->txrx_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci->phy_rst) {
+		ret = reset_control_assert(eswin_sdhci->phy_rst);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci->prstn) {
+		ret = reset_control_assert(eswin_sdhci->prstn);
+		WARN_ON(0 != ret);
+	}
+
+	if (eswin_sdhci->arstn) {
+		ret = reset_control_assert(eswin_sdhci->arstn);
+		WARN_ON(0 != ret);
+	}
+	eswin_sdhci_unregister_sdclk(&pdev->dev);
+	clk_disable_unprepare(clk_ahb);
+	iounmap(core_clk_reg);
+
+	return 0;
+}
+
+static void emmc_hard_reset(struct sdhci_host *host)
+{
+	unsigned int val;
+
+	val = sdhci_readw(host, VENDOR_EMMC_CTRL_R);
+	val |= EMMC_RST_N_OE;
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+	val &= ~EMMC_RST_N;
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+	mdelay(20);
+	val |= EMMC_RST_N;
+	sdhci_writew(host, val, VENDOR_EMMC_CTRL_R);
+}
+
+static void eswin_sdhci_shutdown(struct platform_device *pdev)
+{
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+
+	if (!host)
+		return;
+
+	emmc_hard_reset(host);
+	host->ops->reset(host, SDHCI_RESET_ALL);
+	platform_set_drvdata(pdev, NULL);
+}
+
+static struct platform_driver eswin_sdhci_driver =
+{
+	.driver = {
+		.name = "sdhci-eswin",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.of_match_table = eswin_sdhci_of_match,
+		.pm = &eswin_sdhci_dev_pm_ops,
+	},
+	.probe = eswin_sdhci_probe,
+	.remove = eswin_sdhci_remove,
+	.shutdown = eswin_sdhci_shutdown,
+};
+
+module_platform_driver(eswin_sdhci_driver);
+
+MODULE_DESCRIPTION("Driver for the ESWIN SDHCI Controller");
+MODULE_AUTHOR("liangshuang@eswin.com");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index 92d7d5a00b84..3084e53d6427 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -66,6 +66,14 @@ config DWMAC_ANARION
 
 	  This selects the Anarion SoC glue layer support for the stmmac driver.
 
+config DWMAC_EIC7700
+	tristate "Support for Eswin EIC77xx ethernet driver"
+	select CRC32
+	select MII
+	depends on OF && HAS_DMA
+	help
+	  Support for Eswin EIC7700 ethernet driver.
+
 config DWMAC_INGENIC
 	tristate "Ingenic MAC support"
 	default MACH_INGENIC
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index 5b57aee19267..9ea190bf6752 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -14,6 +14,7 @@ stmmac-$(CONFIG_STMMAC_SELFTESTS) += stmmac_selftests.o
 # Ordering matters. Generic driver must be last.
 obj-$(CONFIG_STMMAC_PLATFORM)	+= stmmac-platform.o
 obj-$(CONFIG_DWMAC_ANARION)	+= dwmac-anarion.o
+obj-$(CONFIG_DWMAC_EIC7700) += dwmac-eic7700.o
 obj-$(CONFIG_DWMAC_INGENIC)	+= dwmac-ingenic.o
 obj-$(CONFIG_DWMAC_IPQ806X)	+= dwmac-ipq806x.o
 obj-$(CONFIG_DWMAC_LPC18XX)	+= dwmac-lpc18xx.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
new file mode 100644
index 000000000000..9e595c2757db
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
@@ -0,0 +1,582 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Eswin DWC Ethernet linux driver
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/ethtool.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/ioport.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_net.h>
+#include <linux/mfd/syscon.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/stmmac.h>
+#include <linux/iommu.h>
+#include "stmmac_platform.h"
+#include "dwmac4.h"
+#include <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/eic7700-sid-cfg.h>
+#include <linux/gpio/consumer.h>
+
+/* eth_phy_ctrl_offset eth0:0x100; eth1:0x200 */
+#define ETH_TX_CLK_SEL          BIT(16)
+#define ETH_PHY_INTF_SELI       BIT(0)
+
+/* eth_axi_lp_ctrl_offset eth0:0x108; eth1:0x208 */
+#define ETH_CSYSREQ_VAL         BIT(0)
+
+/* hsp_aclk_ctrl_offset (0x148) */
+#define HSP_ACLK_CLKEN           BIT(31)
+#define HSP_ACLK_DIVSOR          (0x2 << 4)
+
+/* hsp_cfg_ctrl_offset (0x14c) */
+#define HSP_CFG_CLKEN            BIT(31)
+#define SCU_HSP_PCLK_EN          BIT(30)
+#define HSP_CFG_CTRL_REGSET      (HSP_CFG_CLKEN | SCU_HSP_PCLK_EN)
+
+/* RTL8211F PHY Configurations for LEDs */
+#define PHY_ADDR                 0
+#define PHY_PAGE_SWITCH_REG      31
+#define PHY_LED_CFG_REG          16
+#define PHY_LED_PAGE_CFG         0xd04
+
+#define AWSMMUSID   GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID  GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID   GENMASK(15, 8)  // The sid of read operation
+#define ARSMMUSSID  GENMASK(7, 0)   // The ssid of read operation
+
+struct dwc_qos_priv {
+	struct device *dev;
+	int dev_id;
+	struct regmap *crg_regmap;
+	struct regmap *hsp_regmap;
+	struct reset_control *rst;
+	struct clk *clk_app;
+	struct clk *clk_tx;
+	struct regmap *rgmii_sel;
+	struct gpio_desc *phy_reset;
+	struct stmmac_priv *stmpriv;
+	int phyled_cfgs[3];
+};
+
+static int dwc_eth_dwmac_config_dt(struct platform_device *pdev,
+				   struct plat_stmmacenet_data *plat_dat)
+{
+	struct device *dev = &pdev->dev;
+	u32 burst_map = 0;
+	u32 bit_index = 0;
+	u32 a_index = 0;
+
+	if (!plat_dat->axi) {
+		plat_dat->axi = kzalloc(sizeof(struct stmmac_axi), GFP_KERNEL);
+
+		if (!plat_dat->axi)
+			return -ENOMEM;
+	}
+
+	plat_dat->axi->axi_lpi_en = device_property_read_bool(dev,
+							      "snps,en-lpi");
+	if (device_property_read_u32(dev, "snps,write-requests",
+				     &plat_dat->axi->axi_wr_osr_lmt)) {
+		/**
+		 * Since the register has a reset value of 1, if property
+		 * is missing, default to 1.
+		 */
+		plat_dat->axi->axi_wr_osr_lmt = 1;
+	} else {
+		/**
+		 * If property exists, to keep the behavior from dwc_eth_qos,
+		 * subtract one after parsing.
+		 */
+		plat_dat->axi->axi_wr_osr_lmt--;
+	}
+
+	if (device_property_read_u32(dev, "snps,read-requests",
+				     &plat_dat->axi->axi_rd_osr_lmt)) {
+		/**
+		 * Since the register has a reset value of 1, if property
+		 * is missing, default to 1.
+		 */
+		plat_dat->axi->axi_rd_osr_lmt = 1;
+	} else {
+		/**
+		 * If property exists, to keep the behavior from dwc_eth_qos,
+		 * subtract one after parsing.
+		 */
+		plat_dat->axi->axi_rd_osr_lmt--;
+	}
+	device_property_read_u32(dev, "snps,burst-map", &burst_map);
+
+	/* converts burst-map bitmask to burst array */
+	for (bit_index = 0; bit_index < 7; bit_index++) {
+		if (burst_map & (1 << bit_index)) {
+			switch (bit_index) {
+			case 0:
+			plat_dat->axi->axi_blen[a_index] = 4; break;
+			case 1:
+			plat_dat->axi->axi_blen[a_index] = 8; break;
+			case 2:
+			plat_dat->axi->axi_blen[a_index] = 16; break;
+			case 3:
+			plat_dat->axi->axi_blen[a_index] = 32; break;
+			case 4:
+			plat_dat->axi->axi_blen[a_index] = 64; break;
+			case 5:
+			plat_dat->axi->axi_blen[a_index] = 128; break;
+			case 6:
+			plat_dat->axi->axi_blen[a_index] = 256; break;
+			default:
+			break;
+			}
+			a_index++;
+		}
+	}
+
+	/* dwc-qos needs GMAC4, AAL, TSO and PMT */
+	plat_dat->has_gmac4 = 1;
+	plat_dat->dma_cfg->aal = 1;
+	plat_dat->flags |= STMMAC_FLAG_TSO_EN;
+	plat_dat->pmt = 1;
+
+	return 0;
+}
+
+static int eswin_eth_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_eth_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev, "dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+				&hsp_mmu_eth_reg);
+	if (ret) {
+		dev_err(dev, "can't get eth sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid  = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_eth_reg, rdwr_sid_ssid);
+
+	ret = eic7700_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+		dev_err(dev, "failed to config eth streamID(%d)!\n", sid);
+	else
+		dev_dbg(dev, "success to config eth streamID(%d)!\n", sid);
+
+	return ret;
+}
+
+static void dwc_qos_fix_speed(void *priv, unsigned int speed, unsigned int mode)
+{
+	unsigned long rate = 125000000;
+	int err, data = 0;
+	struct dwc_qos_priv *dwc_priv = (struct dwc_qos_priv *)priv;
+
+	switch (speed) {
+	case SPEED_1000:
+		rate = 125000000;
+
+		if (dwc_priv->dev_id == 0) {
+			regmap_write(dwc_priv->hsp_regmap, 0x118, 0x800c8023);
+			regmap_write(dwc_priv->hsp_regmap, 0x11c, 0x0c0c0c0c);
+			regmap_write(dwc_priv->hsp_regmap, 0x114, 0x23232323);
+		} else {
+			regmap_write(dwc_priv->hsp_regmap, 0x218, 0x80268025);
+			regmap_write(dwc_priv->hsp_regmap, 0x21c, 0x26262626);
+			regmap_write(dwc_priv->hsp_regmap, 0x214, 0x25252525);
+		}
+
+		if (dwc_priv->stmpriv) {
+			data = mdiobus_read(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, PHY_LED_PAGE_CFG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_LED_CFG_REG, dwc_priv->phyled_cfgs[0]);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, data);
+		}
+
+		break;
+	case SPEED_100:
+		rate = 25000000;
+
+		if (dwc_priv->dev_id == 0) {
+			regmap_write(dwc_priv->hsp_regmap, 0x118, 0x803f8050);
+			regmap_write(dwc_priv->hsp_regmap, 0x11c, 0x3f3f3f3f);
+			regmap_write(dwc_priv->hsp_regmap, 0x114, 0x50505050);
+		} else {
+			regmap_write(dwc_priv->hsp_regmap, 0x218, 0x80588048);
+			regmap_write(dwc_priv->hsp_regmap, 0x21c, 0x58585858);
+			regmap_write(dwc_priv->hsp_regmap, 0x214, 0x48484848);
+		}
+
+		if (dwc_priv->stmpriv) {
+			data = mdiobus_read(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, PHY_LED_PAGE_CFG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_LED_CFG_REG, dwc_priv->phyled_cfgs[1]);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, data);
+		}
+
+		break;
+	case SPEED_10:
+		rate = 2500000;
+
+		if (dwc_priv->dev_id == 0) {
+			regmap_write(dwc_priv->hsp_regmap, 0x118, 0x0);
+			regmap_write(dwc_priv->hsp_regmap, 0x11c, 0x0);
+			regmap_write(dwc_priv->hsp_regmap, 0x114, 0x0);
+		} else {
+			regmap_write(dwc_priv->hsp_regmap, 0x218, 0x0);
+			regmap_write(dwc_priv->hsp_regmap, 0x21c, 0x0);
+			regmap_write(dwc_priv->hsp_regmap, 0x214, 0x0);
+		}
+
+		if (dwc_priv->stmpriv) {
+			data = mdiobus_read(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, PHY_LED_PAGE_CFG);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_LED_CFG_REG, dwc_priv->phyled_cfgs[2]);
+			mdiobus_write(dwc_priv->stmpriv->mii, PHY_ADDR, PHY_PAGE_SWITCH_REG, data);
+		}
+
+		break;
+	default:
+		dev_err(dwc_priv->dev, "invalid speed %u\n", speed);
+		break;
+	}
+
+	err = clk_set_rate(dwc_priv->clk_tx, rate);
+	if (err < 0)
+	{
+		dev_err(dwc_priv->dev, "failed to set TX rate: %d\n", err);
+	}
+}
+
+static int dwc_qos_probe(struct platform_device *pdev,
+			 struct plat_stmmacenet_data *plat_dat,
+			 struct stmmac_resources *stmmac_res)
+{
+	struct dwc_qos_priv *dwc_priv;
+	int ret;
+	int err;
+	u32 hsp_aclk_ctrl_offset;
+	u32 hsp_aclk_ctrl_regset;
+	u32 hsp_cfg_ctrl_offset;
+	u32 eth_axi_lp_ctrl_offset;
+	u32 eth_phy_ctrl_offset;
+	u32 eth_phy_ctrl_regset;
+	u32 rgmiisel_offset;
+	u32 rgmiisel_regset;
+
+	dwc_priv = devm_kzalloc(&pdev->dev, sizeof(*dwc_priv), GFP_KERNEL);
+	if (!dwc_priv)
+		return -ENOMEM;
+
+	if (device_property_read_u32(&pdev->dev, "id", &dwc_priv->dev_id)) {
+		dev_err(&pdev->dev, "Can not read device id!\n");
+		return -EINVAL;
+	}
+
+	dwc_priv->dev = &pdev->dev;
+	dwc_priv->phy_reset = devm_gpiod_get(&pdev->dev, "rst", GPIOD_OUT_LOW);
+	if (IS_ERR(dwc_priv->phy_reset)) {
+		dev_err(&pdev->dev, "Reset gpio not specified\n");
+		return -EINVAL;
+	}
+
+	gpiod_set_value(dwc_priv->phy_reset, 0);
+
+	dwc_priv->rgmii_sel = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,rgmiisel");
+	if (IS_ERR(dwc_priv->rgmii_sel)){
+		dev_dbg(&pdev->dev, "rgmiisel not specified\n");
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,led-cfgs", 0, &dwc_priv->phyled_cfgs[0]);
+	if (ret) {
+		dev_warn(&pdev->dev, "can't get led cfgs for 1Gbps mode (%d)\n", ret);
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,led-cfgs", 1, &dwc_priv->phyled_cfgs[1]);
+	if (ret) {
+		dev_warn(&pdev->dev, "can't get led cfgs for 100Mbps mode (%d)\n", ret);
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,led-cfgs", 2, &dwc_priv->phyled_cfgs[2]);
+	if (ret) {
+		dev_warn(&pdev->dev, "can't get led cfgs for 10Mbps mode (%d)\n", ret);
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,rgmiisel", 1, &rgmiisel_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get rgmiisel_offset (%d)\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,rgmiisel", 2, &rgmiisel_regset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get rgmiisel_regset (%d)\n", ret);
+		return ret;
+	}
+
+	regmap_write(dwc_priv->rgmii_sel, rgmiisel_offset, rgmiisel_regset);
+
+	dwc_priv->crg_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,syscrg_csr");
+	if (IS_ERR(dwc_priv->crg_regmap)){
+		dev_dbg(&pdev->dev, "No syscrg_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 1,
+                                    &hsp_aclk_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_aclk_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_read(dwc_priv->crg_regmap, hsp_aclk_ctrl_offset, &hsp_aclk_ctrl_regset);
+	hsp_aclk_ctrl_regset |= (HSP_ACLK_CLKEN | HSP_ACLK_DIVSOR);
+	regmap_write(dwc_priv->crg_regmap, hsp_aclk_ctrl_offset, hsp_aclk_ctrl_regset);
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscrg_csr", 2,
+                                    &hsp_cfg_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get hsp_cfg_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_write(dwc_priv->crg_regmap, hsp_cfg_ctrl_offset, HSP_CFG_CTRL_REGSET);
+
+	dwc_priv->hsp_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(dwc_priv->hsp_regmap)){
+		dev_dbg(&pdev->dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 2,
+                                    &eth_phy_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get eth_phy_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_read(dwc_priv->hsp_regmap, eth_phy_ctrl_offset, &eth_phy_ctrl_regset);
+	eth_phy_ctrl_regset |= (ETH_TX_CLK_SEL | ETH_PHY_INTF_SELI);
+	regmap_write(dwc_priv->hsp_regmap, eth_phy_ctrl_offset, eth_phy_ctrl_regset);
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,hsp_sp_csr", 3,
+                                    &eth_axi_lp_ctrl_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get eth_axi_lp_ctrl_offset (%d)\n", ret);
+		return ret;
+	}
+	regmap_write(dwc_priv->hsp_regmap, eth_axi_lp_ctrl_offset, ETH_CSYSREQ_VAL);
+
+	dwc_priv->clk_app = devm_clk_get(&pdev->dev, "app");
+	if (IS_ERR(dwc_priv->clk_app)) {
+		dev_err(&pdev->dev, "app clock not found.\n");
+		return PTR_ERR(dwc_priv->clk_app);
+	}
+
+	err = clk_prepare_enable(dwc_priv->clk_app);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to enable app clock: %d\n",
+			err);
+		return err;
+	}
+
+	dwc_priv->clk_tx = devm_clk_get(&pdev->dev, "tx");
+	if (IS_ERR(plat_dat->pclk)) {
+		dev_err(&pdev->dev, "tx clock not found.\n");
+		return PTR_ERR(dwc_priv->clk_tx);
+	}
+
+	err = clk_prepare_enable(dwc_priv->clk_tx);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to enable tx clock: %d\n",
+            err);
+		return err;
+	}
+	dwc_priv->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, "ethrst");
+	if (IS_ERR(dwc_priv->rst)) {
+		return PTR_ERR(dwc_priv->rst);
+	}
+
+	ret = reset_control_assert(dwc_priv->rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(dwc_priv->rst);
+	WARN_ON(0 != ret);
+
+	ret = eic7700_tbu_power(&pdev->dev, true);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to power on tbu\n");
+		return ret;
+	}
+
+	plat_dat->fix_mac_speed = dwc_qos_fix_speed;
+	plat_dat->bsp_priv = dwc_priv;
+	plat_dat->phy_addr = PHY_ADDR;
+
+	return 0;
+}
+
+static int dwc_qos_remove(struct platform_device *pdev)
+{
+	int ret;
+	struct dwc_qos_priv *dwc_priv = get_stmmac_bsp_priv(&pdev->dev);
+
+	ret = eic7700_tbu_power(&pdev->dev, false);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to power down tbu\n");
+		return ret;
+	}
+
+	reset_control_assert(dwc_priv->rst);
+	clk_disable_unprepare(dwc_priv->clk_tx);
+	clk_disable_unprepare(dwc_priv->clk_app);
+
+	devm_gpiod_put(&pdev->dev, dwc_priv->phy_reset);
+
+	return 0;
+}
+
+struct dwc_eth_dwmac_data {
+	int (*probe)(struct platform_device *pdev,
+		     struct plat_stmmacenet_data *data,
+		     struct stmmac_resources *res);
+	int (*remove)(struct platform_device *pdev);
+};
+
+static const struct dwc_eth_dwmac_data dwc_qos_data = {
+	.probe = dwc_qos_probe,
+	.remove = dwc_qos_remove,
+};
+
+static int dwc_eth_dwmac_probe(struct platform_device *pdev)
+{
+	const struct dwc_eth_dwmac_data *data;
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+	struct net_device *ndev = NULL;
+	struct stmmac_priv *stmpriv = NULL;
+	struct dwc_qos_priv *dwc_priv = NULL;
+	int ret;
+
+	data = device_get_match_data(&pdev->dev);
+
+	memset(&stmmac_res, 0, sizeof(struct stmmac_resources));
+
+	/**
+	 * Since stmmac_platform supports name IRQ only, basic platform
+	 * resource initialization is done in the glue logic.
+	 */
+	stmmac_res.irq = platform_get_irq(pdev, 0);
+	if (stmmac_res.irq < 0)
+		return stmmac_res.irq;
+	stmmac_res.wol_irq = stmmac_res.irq;
+	stmmac_res.addr = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(stmmac_res.addr))
+		return PTR_ERR(stmmac_res.addr);
+
+	plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return PTR_ERR(plat_dat);
+
+	ret = data->probe(pdev, plat_dat, &stmmac_res);
+	if (ret < 0) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to probe subdriver: %d\n",
+				ret);
+
+		goto remove_config;
+	}
+
+	ret = dwc_eth_dwmac_config_dt(pdev, plat_dat);
+	if (ret)
+		goto remove;
+
+    ret =  eswin_eth_sid_cfg(&pdev->dev);
+	if (ret)
+		goto remove;
+
+	ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res);
+	if (ret)
+		goto remove;
+
+	ndev = dev_get_drvdata(&pdev->dev);
+	stmpriv = netdev_priv(ndev);
+	dwc_priv = (struct dwc_qos_priv *)plat_dat->bsp_priv;
+	dwc_priv->stmpriv = stmpriv;
+
+	return ret;
+
+remove:
+	data->remove(pdev);
+remove_config:
+	stmmac_remove_config_dt(pdev, plat_dat);
+
+	return ret;
+}
+
+static int dwc_eth_dwmac_remove(struct platform_device *pdev)
+{
+	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	const struct dwc_eth_dwmac_data *data;
+	int err;
+
+	data = device_get_match_data(&pdev->dev);
+
+	stmmac_dvr_remove(&pdev->dev);
+
+	err = data->remove(pdev);
+	if (err < 0)
+		dev_err(&pdev->dev, "failed to remove subdriver: %d\n", err);
+
+	stmmac_remove_config_dt(pdev, priv->plat);
+
+	return err;
+}
+
+static const struct of_device_id dwc_eth_dwmac_match[] = {
+	{ .compatible = "eswin,eic7700-qos-eth", .data = &dwc_qos_data },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, dwc_eth_dwmac_match);
+
+static struct platform_driver eic7700_eth_dwmac_driver = {
+	.probe  = dwc_eth_dwmac_probe,
+	.remove = dwc_eth_dwmac_remove,
+	.driver = {
+		.name           = "eic7700-eth-dwmac",
+		.pm             = &stmmac_pltfr_pm_ops,
+		.of_match_table = dwc_eth_dwmac_match,
+	},
+};
+module_platform_driver(eic7700_eth_dwmac_driver);
+
+MODULE_AUTHOR("Eswin");
+MODULE_DESCRIPTION("Eswin eic7700 qos ethernet driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index ab96da43e0c2..09f44434f854 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -415,4 +415,12 @@ config PCIE_VISCONTI_HOST
 	  Say Y here if you want PCIe controller support on Toshiba Visconti SoC.
 	  This driver supports TMPV7708 SoC.
 
+config PCIE_ESWIN
+	tristate "ESWIN PCIe host controller"
+	depends on PCI_MSI
+	depends on SOC_SIFIVE || COMPILE_TEST
+	select PCIE_DW_HOST
+	help
+	  Say Y here if you want PCIe controller support for the ESWIN.
+
 endmenu
diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile
index bf5c311875a1..927459a28dc7 100644
--- a/drivers/pci/controller/dwc/Makefile
+++ b/drivers/pci/controller/dwc/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_PCIE_TEGRA194) += pcie-tegra194.o
 obj-$(CONFIG_PCIE_UNIPHIER) += pcie-uniphier.o
 obj-$(CONFIG_PCIE_UNIPHIER_EP) += pcie-uniphier-ep.o
 obj-$(CONFIG_PCIE_VISCONTI_HOST) += pcie-visconti.o
+obj-$(CONFIG_PCIE_ESWIN) += pcie-eswin.o
 
 # The following drivers are for devices that use the generic ACPI
 # pci_root.c driver but don't support standard ECAM config access.
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index a7170fd0e847..7456e0e57850 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -668,21 +668,31 @@ static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp)
 
 	i = 0;
 	resource_list_for_each_entry(entry, &pp->bridge->windows) {
+		u64 total_size = resource_size(entry->res);
+		u64 region_start = entry->res->start;
+
 		if (resource_type(entry->res) != IORESOURCE_MEM)
 			continue;
 
-		if (pci->num_ob_windows <= ++i)
-			break;
+		do {
+			u64 region_size = min(total_size, pci->region_limit + 1);
 
-		ret = dw_pcie_prog_outbound_atu(pci, i, PCIE_ATU_TYPE_MEM,
-						entry->res->start,
-						entry->res->start - entry->offset,
-						resource_size(entry->res));
-		if (ret) {
-			dev_err(pci->dev, "Failed to set MEM range %pr\n",
-				entry->res);
-			return ret;
-		}
+			if (pci->num_ob_windows <= ++i)
+				break;
+
+			ret = dw_pcie_prog_outbound_atu(pci, i, PCIE_ATU_TYPE_MEM,
+							region_start,
+							region_start - entry->offset,
+							region_size);
+			if (ret) {
+				dev_err(pci->dev, "Failed to set MEM range %pr\n",
+					entry->res);
+				return ret;
+			}
+
+			region_start += region_size;
+			total_size -= region_size;
+		} while (total_size);
 	}
 
 	if (pp->io_size) {
diff --git a/drivers/pci/controller/dwc/pcie-eswin.c b/drivers/pci/controller/dwc/pcie-eswin.c
new file mode 100644
index 000000000000..edba9017666e
--- /dev/null
+++ b/drivers/pci/controller/dwc/pcie-eswin.c
@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN PCIe root complex driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yu Ning <ningyu@eswincomputing.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/resource.h>
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/reset.h>
+#include <linux/eic7700-sid-cfg.h>
+#include <linux/gpio/consumer.h>
+#include <linux/property.h>
+#include "pcie-designware.h"
+
+struct eswin_pcie {
+	struct dw_pcie pci;
+	void __iomem *mgmt_base;
+	struct gpio_desc *reset;
+	struct clk *pcie_aux;
+	struct clk *pcie_cfg;
+	struct clk *pcie_cr;
+	struct clk *pcie_aclk;
+	struct reset_control *powerup_rst;
+	struct reset_control *cfg_rst;
+	struct reset_control *perst;
+};
+
+#define PCIE_PM_SEL_AUX_CLK		BIT(16)
+#define PCIEMGMT_APP_HOLD_PHY_RST	BIT(6)
+#define PCIEMGMT_APP_LTSSM_ENABLE	BIT(5)
+#define PCIEMGMT_DEVICE_TYPE_MASK	0xf
+
+#define PCIEMGMT_CTRL0_OFFSET	0x0
+#define PCIEMGMT_STATUS0_OFFSET	0x100
+
+#define PCIE_TYPE_DEV_VEND_ID	0x0
+#define PCIE_DSP_PF0_MSI_CAP	0x50
+#define PCIE_NEXT_CAP_PTR		0x70
+#define DEVICE_CONTROL_DEVICE_STATUS	0x78
+
+#define PCIE_MSI_MULTIPLE_MSG_32	(0x5<<17)
+#define PCIE_MSI_MULTIPLE_MSG_MASK	(0x7<<17)
+
+#define PCIEMGMT_LINKUP_STATE_VALIDATE  ((0x11<<2)|0x3)
+#define PCIEMGMT_LINKUP_STATE_MASK      0xff
+
+static void eswin_pcie_shutdown(struct platform_device *pdev)
+{
+	struct eswin_pcie *pcie = platform_get_drvdata(pdev);
+
+	/* Bring down link, so bootloader gets clean state in case of reboot */
+	reset_control_assert(pcie->perst);
+}
+
+static int eswin_pcie_start_link(struct dw_pcie *pci)
+{
+	struct device *dev = pci->dev;
+	struct eswin_pcie *pcie = dev_get_drvdata(dev);
+	u32 val;
+
+	/* Enable LTSSM */
+	val = readl_relaxed(pcie->mgmt_base + PCIEMGMT_CTRL0_OFFSET);
+	val |= PCIEMGMT_APP_LTSSM_ENABLE;
+	writel_relaxed(val, pcie->mgmt_base + PCIEMGMT_CTRL0_OFFSET);
+	return 0;
+}
+
+static int eswin_pcie_link_up(struct dw_pcie *pci)
+{
+	struct device *dev = pci->dev;
+	struct eswin_pcie *pcie = dev_get_drvdata(dev);
+	u32 val;
+
+	val = readl_relaxed(pcie->mgmt_base + PCIEMGMT_STATUS0_OFFSET);
+	if ((val & PCIEMGMT_LINKUP_STATE_MASK) == PCIEMGMT_LINKUP_STATE_VALIDATE)
+		return 1;
+	else
+		return 0;
+}
+
+static int eswin_pcie_clk_enable(struct eswin_pcie *pcie)
+{
+	int ret;
+
+	ret = clk_prepare_enable(pcie->pcie_cr);
+	if (ret) {
+		pr_err("PCIe: failed to enable cr clk: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(pcie->pcie_aclk);
+	if (ret) {
+		pr_err("PCIe: failed to enable aclk: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(pcie->pcie_cfg);
+	if (ret) {
+		pr_err("PCIe: failed to enable cfg_clk: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(pcie->pcie_aux);
+	if (ret) {
+		pr_err("PCIe: failed to enable aux_clk: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eswin_pcie_clk_disable(struct eswin_pcie *eswin_pcie)
+{
+	clk_disable_unprepare(eswin_pcie->pcie_aux);
+	clk_disable_unprepare(eswin_pcie->pcie_cfg);
+	clk_disable_unprepare(eswin_pcie->pcie_cr);
+	clk_disable_unprepare(eswin_pcie->pcie_aclk);
+
+	return 0;
+}
+
+static int eswin_pcie_power_on(struct eswin_pcie *pcie)
+{
+	int ret = 0;
+
+	/* pciet_cfg_rstn */
+	ret = reset_control_reset(pcie->cfg_rst);
+	WARN_ON(0 != ret);
+
+	/* pciet_powerup_rstn */
+	ret = reset_control_reset(pcie->powerup_rst);
+	WARN_ON(0 != ret);
+
+	return ret;
+}
+
+static int eswin_pcie_power_off(struct eswin_pcie *eswin_pcie)
+{
+	reset_control_assert(eswin_pcie->perst);
+
+	reset_control_assert(eswin_pcie->powerup_rst);
+
+	reset_control_assert(eswin_pcie->cfg_rst);
+
+	return 0;
+}
+
+int eswin_evb_socket_power_on(struct device *dev)
+{
+	int err_desc=0;
+	struct gpio_desc *gpio;
+	gpio = devm_gpiod_get(dev, "pci-socket", GPIOD_OUT_LOW);
+	err_desc = IS_ERR(gpio);
+
+	if (err_desc) {
+		pr_debug("No power control gpio found, maybe not needed\n");
+		return 0;
+	}
+
+	gpiod_set_value(gpio,1);
+
+	return err_desc;
+}
+
+static int eswin_pcie_host_init(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct eswin_pcie *pcie = dev_get_drvdata(pci->dev);
+	int ret;
+	u32 val;
+
+	/* pciet_aux_clken, pcie_cfg_clken */
+	ret = eswin_pcie_clk_enable(pcie);
+	if (ret)
+		return ret;
+
+	ret = eswin_pcie_power_on(pcie);
+	if (ret)
+		return ret;
+
+	ret = eic7700_tbu_power(pcie->pci.dev, true);
+	if (ret)
+		return ret;
+
+	/* set device type : rc */
+	val = readl_relaxed(pcie->mgmt_base + PCIEMGMT_CTRL0_OFFSET);
+	val &= 0xfffffff0;
+	writel_relaxed(val|0x4, pcie->mgmt_base + PCIEMGMT_CTRL0_OFFSET);
+
+	ret = reset_control_assert(pcie->perst);
+	WARN_ON(0 != ret);
+
+	eswin_evb_socket_power_on(pcie->pci.dev);
+	msleep(100);
+	ret = reset_control_deassert(pcie->perst);
+	WARN_ON(0 != ret);
+
+	/* app_hold_phy_rst */
+	val = readl_relaxed(pcie->mgmt_base + PCIEMGMT_CTRL0_OFFSET);
+	val &= ~(0x40);
+	writel_relaxed(val, pcie->mgmt_base + PCIEMGMT_CTRL0_OFFSET);
+
+	/* wait pm_sel_aux_clk to 0 */
+	while (1) {
+		val = readl_relaxed(pcie->mgmt_base + PCIEMGMT_STATUS0_OFFSET);
+		if (!(val & PCIE_PM_SEL_AUX_CLK)) {
+			break;
+		}
+		msleep(1);
+	}
+
+	/* config eswin vendor id and eic7700 device id */
+	dw_pcie_writel_dbi(pci, PCIE_TYPE_DEV_VEND_ID, 0x20301fe1);
+
+	/* lane fix config, real driver NOT need, default x4 */
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL);
+	val &= 0xffffff80;
+	val |= 0x44;
+	dw_pcie_writel_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL, val);
+
+	val = dw_pcie_readl_dbi(pci, DEVICE_CONTROL_DEVICE_STATUS);
+	val &= ~(0x7<<5);
+	val |= (0x2<<5);
+	dw_pcie_writel_dbi(pci, DEVICE_CONTROL_DEVICE_STATUS, val);
+
+	/*  config support 32 msi vectors */
+	val = dw_pcie_readl_dbi(pci, PCIE_DSP_PF0_MSI_CAP);
+	val &= ~PCIE_MSI_MULTIPLE_MSG_MASK;
+	val |= PCIE_MSI_MULTIPLE_MSG_32;
+	dw_pcie_writel_dbi(pci, PCIE_DSP_PF0_MSI_CAP, val);
+
+	/* disable msix cap */
+	val = dw_pcie_readl_dbi(pci, PCIE_NEXT_CAP_PTR);
+	val &= 0xffff00ff;
+	dw_pcie_writel_dbi(pci, PCIE_NEXT_CAP_PTR, val);
+
+	return 0;
+}
+
+static const struct dw_pcie_host_ops eswin_pcie_host_ops = {
+	.host_init = eswin_pcie_host_init,
+};
+
+static const struct dw_pcie_ops dw_pcie_ops = {
+	.start_link = eswin_pcie_start_link,
+	.link_up = eswin_pcie_link_up,
+};
+
+static int __exit eswin_pcie_remove(struct platform_device *pdev)
+{
+	struct eswin_pcie *pcie = platform_get_drvdata(pdev);
+
+	dw_pcie_host_deinit(&pcie->pci.pp);
+
+	eic7700_tbu_power(&pdev->dev, false);
+	eswin_pcie_power_off(pcie);
+	eswin_pcie_clk_disable(pcie);
+
+	return 0;
+}
+
+static int eswin_pcie_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dw_pcie *pci;
+	struct eswin_pcie *pcie;
+
+	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
+	if (!pcie)
+		return -ENOMEM;
+	pci = &pcie->pci;
+	pci->dev = dev;
+	pci->ops = &dw_pcie_ops;
+	pci->pp.ops = &eswin_pcie_host_ops;
+
+	/* SiFive specific region: mgmt */
+	pcie->mgmt_base = devm_platform_ioremap_resource_byname(pdev, "mgmt");
+	if (IS_ERR(pcie->mgmt_base))
+		return PTR_ERR(pcie->mgmt_base);
+
+	/* Fetch clocks */
+	pcie->pcie_aux = devm_clk_get(dev, "pcie_aux_clk");
+	if (IS_ERR(pcie->pcie_aux)) {
+		dev_err(dev, "pcie_aux clock source missing or invalid\n");
+		return PTR_ERR(pcie->pcie_aux);
+	}
+
+	pcie->pcie_cfg = devm_clk_get(dev, "pcie_cfg_clk");
+	if (IS_ERR(pcie->pcie_cfg)) {
+		dev_err(dev, "pcie_cfg_clk clock source missing or invalid\n");
+		return PTR_ERR(pcie->pcie_cfg);
+	}
+
+	pcie->pcie_cr = devm_clk_get(dev, "pcie_cr_clk");
+	if (IS_ERR(pcie->pcie_cr)) {
+		dev_err(dev, "pcie_cr_clk clock source missing or invalid\n");
+		return PTR_ERR(pcie->pcie_cr);
+	}
+
+	pcie->pcie_aclk = devm_clk_get(dev, "pcie_aclk");
+
+	if (IS_ERR(pcie->pcie_aclk)) {
+		dev_err(dev, "pcie_aclk clock source missing or invalid\n");
+		return PTR_ERR(pcie->pcie_aclk);
+	}
+
+	/* Fetch reset */
+	pcie->powerup_rst = devm_reset_control_get_optional(&pdev->dev, "pcie_powerup");
+	if (IS_ERR_OR_NULL(pcie->powerup_rst)) {
+		dev_err_probe(dev, PTR_ERR(pcie->powerup_rst), "unable to get powerup reset\n");
+	}
+
+	pcie->cfg_rst = devm_reset_control_get_optional(&pdev->dev, "pcie_cfg");
+	if (IS_ERR_OR_NULL(pcie->cfg_rst)) {
+		dev_err_probe(dev, PTR_ERR(pcie->cfg_rst), "unable to get cfg reset\n");
+	}
+
+	pcie->perst = devm_reset_control_get_optional(&pdev->dev, "pcie_pwren");
+	if (IS_ERR_OR_NULL(pcie->perst)) {
+		dev_err_probe(dev, PTR_ERR(pcie->perst), "unable to get perst\n");
+	}
+
+	platform_set_drvdata(pdev, pcie);
+
+	return dw_pcie_host_init(&pci->pp);
+}
+
+static const struct of_device_id eswin_pcie_of_match[] = {
+	{ .compatible = "eswin,eic7700-pcie", },
+	{},
+};
+
+static struct platform_driver eswin_pcie_driver = {
+	.driver = {
+		   .name = "eswin-pcie",
+		   .of_match_table = eswin_pcie_of_match,
+		   .suppress_bind_attrs = true,
+	},
+	.probe = eswin_pcie_probe,
+	.remove = __exit_p(eswin_pcie_remove),
+	.shutdown = eswin_pcie_shutdown,
+};
+
+module_platform_driver(eswin_pcie_driver);
+
+MODULE_DEVICE_TABLE(of, eswin_pcie_of_match);
+MODULE_DESCRIPTION("PCIe host controller driver for ESWIN EIC7700 SoCs");
+MODULE_AUTHOR("Ning Yu <ningyu@eswincomputing.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 273d67ecf6d2..5b2257589a73 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -155,6 +155,25 @@ config QCOM_L3_PMU
 	   Adds the L3 cache PMU into the perf events subsystem for
 	   monitoring L3 cache events.
 
+config SIFIVE_CCACHE_PMU
+	tristate "SiFive Composable Cache PMU"
+	depends on RISCV || COMPILE_TEST
+	help
+	  Support for the Composable Cache performance monitoring unit (PMU) on
+	  SiFive platforms. The Composable Cache PMU provides up to 64 counters
+	  for measuring whole-system L2/L3 cache performance using the perf
+	  events subsystem.
+
+config SIFIVE_PL2_PMU
+	tristate "SiFive Private L2 Cache PMU"
+	depends on RISCV || COMPILE_TEST
+	depends on OF
+	help
+	  Support for the Private L2 Cache performance monitoring unit (PMU) on
+	  SiFive platforms. The Private L2 Cache PMU provides up to 64 counters
+	  for measuring per-program or per-hart L2 cache performance using the
+	  perf events subsystem.
+
 config THUNDERX2_PMU
 	tristate "Cavium ThunderX2 SoC PMU UNCORE"
 	depends on ARCH_THUNDER2 || COMPILE_TEST
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 16b3ec4db916..85115136e84b 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -15,6 +15,8 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
 obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
 obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
 obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
+obj-$(CONFIG_SIFIVE_CCACHE_PMU) += sifive_ccache_pmu.o
+obj-$(CONFIG_SIFIVE_PL2_PMU) += sifive_pl2_pmu.o
 obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
 obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
 obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o
diff --git a/drivers/perf/sifive_ccache_pmu.c b/drivers/perf/sifive_ccache_pmu.c
new file mode 100644
index 000000000000..8c9ef0d09f48
--- /dev/null
+++ b/drivers/perf/sifive_ccache_pmu.c
@@ -0,0 +1,577 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SiFive Composable Cache PMU driver
+ *
+ * Copyright (C) 2022-2024 SiFive, Inc.
+ * Copyright (C) Eric Lin <eric.lin@sifive.com>
+ *
+ */
+
+#include <linux/cpuhotplug.h>
+#include <linux/cpumask.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+
+#define CCACHE_SELECT_OFFSET		0x2000
+#define CCACHE_CLIENT_FILTER_OFFSET	0x2800
+#define CCACHE_COUNTER_OFFSET		0x3000
+
+#define CCACHE_PMU_MAX_COUNTERS		64
+
+struct sifive_ccache_pmu {
+	struct pmu			pmu;
+	struct hlist_node		node;
+	struct notifier_block		cpu_pm_nb;
+	void __iomem			*base;
+	DECLARE_BITMAP(used_mask, CCACHE_PMU_MAX_COUNTERS);
+	unsigned int			cpu;
+	int				n_counters;
+	struct perf_event		*events[] __counted_by(n_counters);
+};
+
+#define to_ccache_pmu(p) (container_of(p, struct sifive_ccache_pmu, pmu))
+
+#ifndef readq
+static inline u64 readq(void __iomem *addr)
+{
+	return readl(addr) | (((u64)readl(addr + 4)) << 32);
+}
+#endif
+
+#ifndef writeq
+static inline void writeq(u64 v, void __iomem *addr)
+{
+	writel(lower_32_bits(v), addr);
+	writel(upper_32_bits(v), addr + 4);
+}
+#endif
+
+/*
+ * sysfs attributes
+ *
+ * We export:
+ * - cpumask, used by perf user space and other tools to know on which CPUs to create events
+ * - events, used by perf user space and other tools to create events symbolically, e.g.:
+ *     perf stat -a -e sifive_ccache_pmu/event=inner_put_partial_data_hit/ ls
+ *     perf stat -a -e sifive_ccache_pmu/event=0x101/ ls
+ * - formats, used by perf user space and other tools to configure events
+ */
+
+/* cpumask */
+static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct sifive_ccache_pmu *ccache_pmu = dev_get_drvdata(dev);
+
+	if (ccache_pmu->cpu >= nr_cpu_ids)
+		return 0;
+
+	return sysfs_emit(buf, "%d\n", ccache_pmu->cpu);
+};
+
+static DEVICE_ATTR_RO(cpumask);
+
+static struct attribute *sifive_ccache_pmu_cpumask_attrs[] = {
+	&dev_attr_cpumask.attr,
+	NULL,
+};
+
+static const struct attribute_group sifive_ccache_pmu_cpumask_group = {
+	.attrs = sifive_ccache_pmu_cpumask_attrs,
+};
+
+/* events */
+static ssize_t sifive_ccache_pmu_event_show(struct device *dev, struct device_attribute *attr,
+					    char *page)
+{
+	struct perf_pmu_events_attr *pmu_attr;
+
+	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
+	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
+}
+
+#define SET_EVENT_SELECT(_event, _set)	(BIT_ULL((_event) + 8) | (_set))
+#define CCACHE_PMU_EVENT_ATTR(_name, _event, _set) \
+	PMU_EVENT_ATTR_ID(_name, sifive_ccache_pmu_event_show, SET_EVENT_SELECT(_event, _set))
+
+enum ccache_pmu_event_set1 {
+	INNER_PUT_FULL_DATA = 0,
+	INNER_PUT_PARTIAL_DATA,
+	INNER_ATOMIC_DATA,
+	INNER_GET,
+	INNER_PREFETCH_READ,
+	INNER_PREFETCH_WRITE,
+	INNER_ACQUIRE_BLOCK_NTOB,
+	INNER_ACQUIRE_BLOCK_NTOT,
+	INNER_ACQUIRE_BLOCK_BTOT,
+	INNER_ACQUIRE_PERM_NTOT,
+	INNER_ACQUIRE_PERM_BTOT,
+	INNER_RELEASE_TTOB,
+	INNER_RELEASE_TTON,
+	INNER_RELEASE_BTON,
+	INNER_RELEASE_DATA_TTOB,
+	INNER_RELEASE_DATA_TTON,
+	INNER_RELEASE_DATA_BTON,
+	OUTER_PROBE_BLOCK_TOT,
+	OUTER_PROBE_BLOCK_TOB,
+	OUTER_PROBE_BLOCK_TON,
+	CCACHE_PMU_MAX_EVENT1_IDX
+};
+
+enum ccache_pmu_event_set2 {
+	INNER_PUT_FULL_DATA_HIT = 0,
+	INNER_PUT_PARTIAL_DATA_HIT,
+	INNER_ATOMIC_DATA_HIT,
+	INNER_GET_HIT,
+	INNER_PREFETCH_HIT,
+	INNER_ACQUIRE_BLOCK_HIT,
+	INNER_ACQUIRE_PERM_HIT,
+	INNER_RELEASE_HIT,
+	INNER_RELEASE_DATA_HIT,
+	OUTER_PROBE_HIT,
+	INNER_PUT_FULL_DATA_HIT_SHARED,
+	INNER_PUT_PARTIAL_DATA_HIT_SHARED,
+	INNER_ATOMIC_DATA_HIT_SHARED,
+	INNER_GET_HIT_SHARED,
+	INNER_PREFETCH_HIT_SHARED,
+	INNER_ACQUIRE_BLOCK_HIT_SHARED,
+	INNER_ACQUIRE_PERM_HIT_SHARED,
+	OUTER_PROBE_HIT_SHARED,
+	OUTER_PROBE_HIT_DIRTY,
+	CCACHE_PMU_MAX_EVENT2_IDX
+};
+
+enum ccache_pmu_event_set3 {
+	OUTER_ACQUIRE_BLOCK_NTOB_MISS = 0,
+	OUTER_ACQUIRE_BLOCK_NTOT_MISS,
+	OUTER_ACQUIRE_BLOCK_BTOT_MISS,
+	OUTER_ACQUIRE_PERM_NTOT_MISS,
+	OUTER_ACQUIRE_PERM_BTOT_MISS,
+	OUTER_RELEASE_TTOB_EVICTION,
+	OUTER_RELEASE_TTON_EVICTION,
+	OUTER_RELEASE_BTON_EVICTION,
+	OUTER_RELEASE_DATA_TTOB_NOT_APPLICABLE,
+	OUTER_RELEASE_DATA_TTON_DIRTY_EVICTION,
+	OUTER_RELEASE_DATA_BTON_NOT_APPLICABLE,
+	INNER_PROBE_BLOCK_TOT_CODE_MISS_HITS_OTHER_HARTS,
+	INNER_PROBE_BLOCK_TOB_LOAD_MISS_HITS_OTHER_HARTS,
+	INNER_PROBE_BLOCK_TON_STORE_MISS_HITS_OTHER_HARTS,
+	CCACHE_PMU_MAX_EVENT3_IDX
+};
+
+enum ccache_pmu_event_set4 {
+	INNER_HINT_HITS_INFLIGHT_MISS = 0,
+	CCACHE_PMU_MAX_EVENT4_IDX
+};
+
+static struct attribute *sifive_ccache_pmu_events[] = {
+	/*  pmEventSelect1 */
+	CCACHE_PMU_EVENT_ATTR(inner_put_full_data, INNER_PUT_FULL_DATA, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_put_partial_data, INNER_PUT_PARTIAL_DATA, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_atomic_data, INNER_ATOMIC_DATA, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_get, INNER_GET, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_prefetch_read, INNER_PREFETCH_READ, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_prefetch_write, INNER_PREFETCH_WRITE, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_block_ntob, INNER_ACQUIRE_BLOCK_NTOB, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_block_ntot, INNER_ACQUIRE_BLOCK_NTOT, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_block_btot, INNER_ACQUIRE_BLOCK_BTOT, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_perm_ntot, INNER_ACQUIRE_PERM_NTOT, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_perm_btot, INNER_ACQUIRE_PERM_BTOT, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_release_ttob, INNER_RELEASE_TTOB, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_release_tton, INNER_RELEASE_TTON, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_release_bton, INNER_RELEASE_BTON, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_release_data_ttob, INNER_RELEASE_DATA_TTOB, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_release_data_tton, INNER_RELEASE_DATA_TTON, 1),
+	CCACHE_PMU_EVENT_ATTR(inner_release_data_bton, INNER_RELEASE_DATA_BTON, 1),
+	CCACHE_PMU_EVENT_ATTR(outer_probe_block_tot, OUTER_PROBE_BLOCK_TOT, 1),
+	CCACHE_PMU_EVENT_ATTR(outer_probe_block_tob, OUTER_PROBE_BLOCK_TOB, 1),
+	CCACHE_PMU_EVENT_ATTR(outer_probe_block_ton, OUTER_PROBE_BLOCK_TON, 1),
+
+	/*  pmEventSelect2 */
+	CCACHE_PMU_EVENT_ATTR(inner_put_full_data_hit, INNER_PUT_FULL_DATA_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_put_partial_data_hit, INNER_PUT_PARTIAL_DATA_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_atomic_data_hit, INNER_ATOMIC_DATA_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_get_hit, INNER_GET_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_prefetch_hit, INNER_PREFETCH_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_block_hit, INNER_ACQUIRE_BLOCK_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_perm_hit, INNER_ACQUIRE_PERM_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_release_hit, INNER_RELEASE_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_release_data_hit, INNER_RELEASE_DATA_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(outer_probe_hit, OUTER_PROBE_HIT, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_put_full_data_hit_shared, INNER_PUT_FULL_DATA_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_put_partial_data_hit_shared,
+			      INNER_PUT_PARTIAL_DATA_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_atomic_data_hit_shared, INNER_ATOMIC_DATA_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_get_hit_shared, INNER_GET_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_prefetch_hit_shared, INNER_PREFETCH_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_block_hit_shared, INNER_ACQUIRE_BLOCK_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(inner_acquire_perm_hit_shared, INNER_ACQUIRE_PERM_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(outer_probe_hit_shared, OUTER_PROBE_HIT_SHARED, 2),
+	CCACHE_PMU_EVENT_ATTR(outer_probe_hit_dirty, OUTER_PROBE_HIT_DIRTY, 2),
+
+	/*  pmEventSelect3 */
+	CCACHE_PMU_EVENT_ATTR(outer_acquire_block_ntob_miss, OUTER_ACQUIRE_BLOCK_NTOB_MISS, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_acquire_block_ntot_miss, OUTER_ACQUIRE_BLOCK_NTOT_MISS, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_acquire_block_btot_miss, OUTER_ACQUIRE_BLOCK_BTOT_MISS, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_acquire_perm_ntot_miss, OUTER_ACQUIRE_PERM_NTOT_MISS, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_acquire_perm_btot_miss, OUTER_ACQUIRE_PERM_BTOT_MISS, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_release_ttob_eviction, OUTER_RELEASE_TTOB_EVICTION, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_release_tton_eviction, OUTER_RELEASE_TTON_EVICTION, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_release_bton_eviction, OUTER_RELEASE_BTON_EVICTION, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_release_data_ttob_not_applicable,
+			      OUTER_RELEASE_DATA_TTOB_NOT_APPLICABLE, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_release_data_tton_dirty_eviction,
+			      OUTER_RELEASE_DATA_TTON_DIRTY_EVICTION, 3),
+	CCACHE_PMU_EVENT_ATTR(outer_release_data_bton_not_applicable,
+			      OUTER_RELEASE_DATA_BTON_NOT_APPLICABLE, 3),
+	CCACHE_PMU_EVENT_ATTR(inner_probe_block_tot_code_miss_hits_other_harts,
+			      INNER_PROBE_BLOCK_TOT_CODE_MISS_HITS_OTHER_HARTS, 3),
+	CCACHE_PMU_EVENT_ATTR(inner_probe_block_tob_load_miss_hits_other_harts,
+			      INNER_PROBE_BLOCK_TOB_LOAD_MISS_HITS_OTHER_HARTS, 3),
+	CCACHE_PMU_EVENT_ATTR(inner_probe_block_ton_store_miss_hits_other_harts,
+			      INNER_PROBE_BLOCK_TON_STORE_MISS_HITS_OTHER_HARTS, 3),
+
+	/*  pm_event_select4 */
+	CCACHE_PMU_EVENT_ATTR(inner_hint_hits_inflight_miss, INNER_HINT_HITS_INFLIGHT_MISS, 4),
+	NULL
+};
+
+static struct attribute_group sifive_ccache_pmu_events_group = {
+	.name = "events",
+	.attrs = sifive_ccache_pmu_events,
+};
+
+/* formats */
+PMU_FORMAT_ATTR(event, "config:0-63");
+
+static struct attribute *sifive_ccache_pmu_formats[] = {
+	&format_attr_event.attr,
+	NULL,
+};
+
+static struct attribute_group sifive_ccache_pmu_format_group = {
+	.name = "format",
+	.attrs = sifive_ccache_pmu_formats,
+};
+
+/*
+ * Per PMU device attribute groups
+ */
+
+static const struct attribute_group *sifive_ccache_pmu_attr_grps[] = {
+	&sifive_ccache_pmu_cpumask_group,
+	&sifive_ccache_pmu_events_group,
+	&sifive_ccache_pmu_format_group,
+	NULL,
+};
+
+/*
+ * Event Initialization
+ */
+
+static int sifive_ccache_pmu_event_init(struct perf_event *event)
+{
+	struct sifive_ccache_pmu *ccache_pmu = to_ccache_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	u64 config = event->attr.config;
+	u64 ev_type = config >> 8;
+	u64 set = config & 0xff;
+
+	/* Check if this is a valid set and event */
+	switch (set) {
+	case 1:
+		if (ev_type >= BIT_ULL(CCACHE_PMU_MAX_EVENT1_IDX))
+			return -ENOENT;
+		break;
+	case 2:
+		if (ev_type >= BIT_ULL(CCACHE_PMU_MAX_EVENT2_IDX))
+			return -ENOENT;
+		break;
+	case 3:
+		if (ev_type >= BIT_ULL(CCACHE_PMU_MAX_EVENT3_IDX))
+			return -ENOENT;
+		break;
+	case 4:
+		if (ev_type >= BIT_ULL(CCACHE_PMU_MAX_EVENT4_IDX))
+			return -ENOENT;
+		break;
+	default:
+		return -ENOENT;
+	}
+
+	/* Do not allocate the hardware counter yet */
+	hwc->idx = -1;
+	hwc->config = config;
+
+	event->cpu = ccache_pmu->cpu;
+
+	return 0;
+}
+
+/*
+ * pmu->read: read and update the counter
+ */
+static void sifive_ccache_pmu_read(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	u64 prev_raw_count, new_raw_count;
+	u64 oldval;
+
+	do {
+		prev_raw_count = local64_read(&hwc->prev_count);
+		new_raw_count = readq((void *)hwc->event_base);
+
+		oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, new_raw_count);
+	} while (oldval != prev_raw_count);
+
+	local64_add(new_raw_count - prev_raw_count, &event->count);
+}
+
+/*
+ * State transition functions:
+ *
+ * start()/stop() & add()/del()
+ */
+
+/*
+ * pmu->start: start the event
+ */
+static void sifive_ccache_pmu_start(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
+		return;
+
+	hwc->state = 0;
+
+	/* Set initial value to 0 */
+	local64_set(&hwc->prev_count, 0);
+	writeq(0, (void *)hwc->event_base);
+
+	/* Enable this counter to count events */
+	writeq(hwc->config, (void *)hwc->config_base);
+}
+
+/*
+ * pmu->stop: stop the counter
+ */
+static void sifive_ccache_pmu_stop(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (hwc->state & PERF_HES_STOPPED)
+		return;
+
+	/* Disable this counter to count events */
+	writeq(0, (void *)hwc->config_base);
+	sifive_ccache_pmu_read(event);
+
+	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+}
+
+/*
+ * pmu->add: add the event to the PMU
+ */
+static int sifive_ccache_pmu_add(struct perf_event *event, int flags)
+{
+	struct sifive_ccache_pmu *ccache_pmu = to_ccache_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx;
+
+	/* Find an available counter idx to use for this event */
+	do {
+		idx = find_first_zero_bit(ccache_pmu->used_mask, ccache_pmu->n_counters);
+		if (idx >= ccache_pmu->n_counters)
+			return -EAGAIN;
+	} while (test_and_set_bit(idx, ccache_pmu->used_mask));
+
+	hwc->config_base = (unsigned long)ccache_pmu->base + CCACHE_SELECT_OFFSET + 8 * idx;
+	hwc->event_base = (unsigned long)ccache_pmu->base + CCACHE_COUNTER_OFFSET + 8 * idx;
+	hwc->idx = idx;
+	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+
+	ccache_pmu->events[idx] = event;
+
+	if (flags & PERF_EF_START)
+		sifive_ccache_pmu_start(event, PERF_EF_RELOAD);
+
+	perf_event_update_userpage(event);
+
+	return 0;
+}
+
+/*
+ * pmu->del: delete the event from the PMU
+ */
+static void sifive_ccache_pmu_del(struct perf_event *event, int flags)
+{
+	struct sifive_ccache_pmu *ccache_pmu = to_ccache_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	/* Stop and release this counter */
+	sifive_ccache_pmu_stop(event, PERF_EF_UPDATE);
+
+	ccache_pmu->events[idx] = NULL;
+	clear_bit(idx, ccache_pmu->used_mask);
+
+	perf_event_update_userpage(event);
+}
+
+/*
+ * Driver initialization
+ */
+
+static void sifive_ccache_pmu_hw_init(const struct sifive_ccache_pmu *ccache_pmu)
+{
+	/* Disable the client filter (not supported by this driver) */
+	writeq(0, ccache_pmu->base + CCACHE_CLIENT_FILTER_OFFSET);
+}
+
+static int sifive_ccache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
+{
+	struct sifive_ccache_pmu *ccache_pmu =
+		hlist_entry_safe(node, struct sifive_ccache_pmu, node);
+
+	if (ccache_pmu->cpu >= nr_cpu_ids)
+		ccache_pmu->cpu = cpu;
+
+	return 0;
+}
+
+static int sifive_ccache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+	struct sifive_ccache_pmu *ccache_pmu =
+		hlist_entry_safe(node, struct sifive_ccache_pmu, node);
+
+	/* Do nothing if this CPU does not own the events */
+	if (cpu != ccache_pmu->cpu)
+		return 0;
+
+	/* Pick a random online CPU */
+	ccache_pmu->cpu = cpumask_any_but(cpu_online_mask, cpu);
+	if (ccache_pmu->cpu >= nr_cpu_ids)
+		return 0;
+
+	/* Migrate PMU events from this CPU to the target CPU */
+	perf_pmu_migrate_context(&ccache_pmu->pmu, cpu, ccache_pmu->cpu);
+
+	return 0;
+}
+
+static int sifive_ccache_pmu_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sifive_ccache_pmu *ccache_pmu;
+	u32 n_counters;
+	int ret;
+
+	/* Instances without a sifive,perfmon-counters property do not contain a PMU */
+	ret = device_property_read_u32(dev, "sifive,perfmon-counters", &n_counters);
+	if (ret || !n_counters)
+		return -ENODEV;
+
+	ccache_pmu = devm_kzalloc(dev, struct_size(ccache_pmu, events, n_counters), GFP_KERNEL);
+	if (!ccache_pmu)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ccache_pmu);
+
+	ccache_pmu->pmu = (struct pmu) {
+		.parent		= dev,
+		.attr_groups	= sifive_ccache_pmu_attr_grps,
+		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
+		.task_ctx_nr	= perf_invalid_context,
+		.event_init	= sifive_ccache_pmu_event_init,
+		.add		= sifive_ccache_pmu_add,
+		.del		= sifive_ccache_pmu_del,
+		.start		= sifive_ccache_pmu_start,
+		.stop		= sifive_ccache_pmu_stop,
+		.read		= sifive_ccache_pmu_read,
+	};
+	ccache_pmu->cpu = nr_cpu_ids;
+	ccache_pmu->n_counters = n_counters;
+
+	ccache_pmu->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ccache_pmu->base))
+		return PTR_ERR(ccache_pmu->base);
+
+	sifive_ccache_pmu_hw_init(ccache_pmu);
+
+	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_RISCV_SIFIVE_CCACHE_ONLINE, &ccache_pmu->node);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to add CPU hotplug instance\n");
+
+	ret = perf_pmu_register(&ccache_pmu->pmu, "sifive_ccache_pmu", -1);
+	if (ret) {
+		dev_err_probe(dev, ret, "Failed to register PMU\n");
+		goto err_remove_instance;
+	}
+
+	return 0;
+
+err_remove_instance:
+	cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_SIFIVE_CCACHE_ONLINE, &ccache_pmu->node);
+
+	return ret;
+}
+
+static void sifive_ccache_pmu_remove(struct platform_device *pdev)
+{
+	struct sifive_ccache_pmu *ccache_pmu = platform_get_drvdata(pdev);
+
+	perf_pmu_unregister(&ccache_pmu->pmu);
+	cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_SIFIVE_CCACHE_ONLINE, &ccache_pmu->node);
+}
+
+static const struct of_device_id sifive_ccache_pmu_of_match[] = {
+	{ .compatible = "sifive,ccache0" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, sifive_ccache_pmu_of_match);
+
+static struct platform_driver sifive_ccache_pmu_driver = {
+	.probe	= sifive_ccache_pmu_probe,
+	.remove_new	= sifive_ccache_pmu_remove,
+	.driver	= {
+		.name		= "sifive_ccache_pmu",
+		.of_match_table	= sifive_ccache_pmu_of_match,
+	},
+};
+
+static void __exit sifive_ccache_pmu_exit(void)
+{
+	platform_driver_unregister(&sifive_ccache_pmu_driver);
+	cpuhp_remove_multi_state(CPUHP_AP_PERF_RISCV_SIFIVE_CCACHE_ONLINE);
+}
+module_exit(sifive_ccache_pmu_exit);
+
+static int __init sifive_ccache_pmu_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_SIFIVE_CCACHE_ONLINE,
+				      "perf/sifive/ccache:online",
+				      sifive_ccache_pmu_online_cpu,
+				      sifive_ccache_pmu_offline_cpu);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&sifive_ccache_pmu_driver);
+	if (ret)
+		goto err_remove_state;
+
+	return 0;
+
+err_remove_state:
+	cpuhp_remove_multi_state(CPUHP_AP_PERF_RISCV_SIFIVE_CCACHE_ONLINE);
+
+	return ret;
+}
+module_init(sifive_ccache_pmu_init);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/perf/sifive_pl2_pmu.c b/drivers/perf/sifive_pl2_pmu.c
new file mode 100644
index 000000000000..d0bbac0dec06
--- /dev/null
+++ b/drivers/perf/sifive_pl2_pmu.c
@@ -0,0 +1,748 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SiFive Private L2 Cache PMU driver
+ *
+ * Copyright (C) 2018-2024 SiFive, Inc.
+ */
+
+#include <linux/cpu_pm.h>
+#include <linux/cpumask.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/refcount.h>
+
+#define PL2_SELECT_OFFSET		0x2000
+#define PL2_CLIENT_FILTER_OFFSET	0x2800
+#define PL2_COUNTER_OFFSET		0x3000
+
+#define PL2_PMU_MAX_COUNTERS		64
+
+struct sifive_pl2_pmu_event {
+	void __iomem			*base;
+	DECLARE_BITMAP(used_mask, PL2_PMU_MAX_COUNTERS);
+	unsigned int			cpu;
+	int				n_counters;
+	struct perf_event		*events[] __counted_by(n_counters);
+};
+
+struct sifive_pl2_pmu {
+	struct pmu			pmu;
+	struct notifier_block		cpu_pm_nb;
+	refcount_t			refcount;
+	struct sifive_pl2_pmu_event	*__percpu *event;
+};
+
+#define to_pl2_pmu(p) (container_of(p, struct sifive_pl2_pmu, pmu))
+
+static DEFINE_MUTEX(g_mutex);
+static struct sifive_pl2_pmu *g_pl2_pmu;
+
+#ifndef readq
+static inline u64 readq(void __iomem *addr)
+{
+	return readl(addr) | (((u64)readl(addr + 4)) << 32);
+}
+#endif
+
+#ifndef writeq
+static inline void writeq(u64 v, void __iomem *addr)
+{
+	writel(lower_32_bits(v), addr);
+	writel(upper_32_bits(v), addr + 4);
+}
+#endif
+
+/*
+ * sysfs attributes
+ *
+ * We export:
+ * - events, used by perf user space and other tools to create events symbolically, e.g.:
+ *     perf stat -a -e sifive_pl2_pmu/event=inner_put_partial_data_hit/ ls
+ *     perf stat -a -e sifive_pl2_pmu/event=0x101/ ls
+ * - formats, used by perf user space and other tools to configure events
+ */
+
+/* events */
+static ssize_t sifive_pl2_pmu_event_show(struct device *dev, struct device_attribute *attr,
+					 char *page)
+{
+	struct perf_pmu_events_attr *pmu_attr;
+
+	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
+	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
+}
+
+#define SET_EVENT_SELECT(_event, _set)	(BIT_ULL((_event) + 8) | (_set))
+#define PL2_PMU_EVENT_ATTR(_name, _event, _set) \
+	PMU_EVENT_ATTR_ID(_name, sifive_pl2_pmu_event_show, SET_EVENT_SELECT(_event, _set))
+
+enum pl2_pmu_event_set1 {
+	INNER_PUT_FULL_DATA = 0,
+	INNER_PUT_PARTIAL_DATA,
+	INNER_ATOMIC_DATA,
+	INNER_GET,
+	INNER_PREFETCH_READ,
+	INNER_PREFETCH_WRITE,
+	INNER_ACQUIRE_BLOCK_NTOB,
+	INNER_ACQUIRE_BLOCK_NTOT,
+	INNER_ACQUIRE_BLOCK_BTOT,
+	INNER_ACQUIRE_PERM_NTOT,
+	INNER_ACQUIRE_PERM_BTOT,
+	INNER_RELEASE_TTOB,
+	INNER_RELEASE_TTON,
+	INNER_RELEASE_BTON,
+	INNER_RELEASE_DATA_TTOB,
+	INNER_RELEASE_DATA_TTON,
+	INNER_RELEASE_DATA_BTON,
+	INNER_RELEASE_DATA_TTOT,
+	INNER_PROBE_BLOCK_TOT,
+	INNER_PROBE_BLOCK_TOB,
+	INNER_PROBE_BLOCK_TON,
+	INNER_PROBE_PERM_TON,
+	INNER_PROBE_ACK_TTOB,
+	INNER_PROBE_ACK_TTON,
+	INNER_PROBE_ACK_BTON,
+	INNER_PROBE_ACK_TTOT,
+	INNER_PROBE_ACK_BTOB,
+	INNER_PROBE_ACK_NTON,
+	INNER_PROBE_ACK_DATA_TTOB,
+	INNER_PROBE_ACK_DATA_TTON,
+	INNER_PROBE_ACK_DATA_TTOT,
+	PL2_PMU_MAX_EVENT1_IDX
+};
+
+enum pl2_pmu_event_set2 {
+	INNER_PUT_FULL_DATA_HIT = 0,
+	INNER_PUT_PARTIAL_DATA_HIT,
+	INNER_ATOMIC_DATA_HIT,
+	INNER_GET_HIT,
+	INNER_PREFETCH_READ_HIT,
+	INNER_ACQUIRE_BLOCK_NTOB_HIT,
+	INNER_ACQUIRE_PERM_NTOT_HIT,
+	INNER_RELEASE_TTOB_HIT,
+	INNER_RELEASE_DATA_TTOB_HIT,
+	OUTER_PROBE_BLOCK_TOT_HIT,
+	INNER_PUT_FULL_DATA_HIT_SHARED,
+	INNER_PUT_PARTIAL_DATA_HIT_SHARED,
+	INNER_ATOMIC_DATA_HIT_SHARED,
+	INNER_GET_HIT_SHARED,
+	INNER_PREFETCH_READ_HIT_SHARED,
+	INNER_ACQUIRE_BLOCK_NTOB_HIT_SHARED,
+	INNER_ACQUIRE_PERM_NTOT_HIT_SHARED,
+	INNER_RELEASE_TTOB_HIT_SHARED,
+	INNER_RELEASE_DATA_TTOB_HIT_SHARED,
+	OUTER_PROBE_BLOCK_TOT_HIT_SHARED,
+	OUTER_PROBE_BLOCK_TOT_HIT_DIRTY,
+	PL2_PMU_MAX_EVENT2_IDX
+};
+
+enum pl2_pmu_event_set3 {
+	OUTER_PUT_FULL_DATA = 0,
+	OUTER_PUT_PARTIAL_DATA,
+	OUTER_ATOMIC_DATA,
+	OUTER_GET,
+	OUTER_PREFETCH_READ,
+	OUTER_PREFETCH_WRITE,
+	OUTER_ACQUIRE_BLOCK_NTOB,
+	OUTER_ACQUIRE_BLOCK_NTOT,
+	OUTER_ACQUIRE_BLOCK_BTOT,
+	OUTER_ACQUIRE_PERM_NTOT,
+	OUTER_ACQUIRE_PERM_BTOT,
+	OUTER_RELEARE_TTOB,
+	OUTER_RELEARE_TTON,
+	OUTER_RELEARE_BTON,
+	OUTER_RELEARE_DATA_TTOB,
+	OUTER_RELEARE_DATA_TTON,
+	OUTER_RELEARE_DATA_BTON,
+	OUTER_RELEARE_DATA_TTOT,
+	OUTER_PROBE_BLOCK_TOT,
+	OUTER_PROBE_BLOCK_TOB,
+	OUTER_PROBE_BLOCK_TON,
+	OUTER_PROBE_PERM_TON,
+	OUTER_PROBE_ACK_TTOB,
+	OUTER_PROBE_ACK_TTON,
+	OUTER_PROBE_ACK_BTON,
+	OUTER_PROBE_ACK_TTOT,
+	OUTER_PROBE_ACK_BTOB,
+	OUTER_PROBE_ACK_NTON,
+	OUTER_PROBE_ACK_DATA_TTOB,
+	OUTER_PROBE_ACK_DATA_TTON,
+	OUTER_PROBE_ACK_DATA_TTOT,
+	PL2_PMU_MAX_EVENT3_IDX
+};
+
+enum pl2_pmu_event_set4 {
+	INNER_HINT_HITS_MSHR = 0,
+	INNER_READ_HITS_MSHR,
+	INNER_WRITE_HITS_MSHR,
+	INNER_READ_REPLAY,
+	INNER_WRITE_REPLAY,
+	OUTER_PROBE_REPLAY,
+	REPLAY,
+	SLEEP_BY_MISS_QUEUE,
+	SLEEP_BY_EVICT_QUEUE,
+	SLEEP_FOR_BACK_PROBE,
+	SLEEP,
+	PL2_PMU_MAX_EVENT4_IDX
+};
+
+enum pl2_pmu_event_set5 {
+	READ_SLEEP_TIMER_EXPIRE = 0,
+	READ_OLDEST_TIMER_EXPIRE,
+	WRITE_SLEEP_TIMER_EXPIRE,
+	WRITE_OLDEST_TIMER_EXPIRE,
+	READ_SLEEP,
+	READ_DIR_UPDATE_WAKEUP,
+	READ_MISS_QUEUE_WAKEUP,
+	READ_EVICT_QUEUE_WAKEUP,
+	READ_SLEEP_TIMER_WAKEUP,
+	WRITE_SLEEP,
+	WRITE_DIR_UPDATE_WAKEUP,
+	WRITE_MISS_QUEUE_WAKEUP,
+	WRITE_EVICT_QUEUE_WAKEUP,
+	WRITE_SLEEP_TIMER_WAKEUP,
+	PL2_PMU_MAX_EVENT5_IDX
+};
+
+static struct attribute *sifive_pl2_pmu_events[] = {
+	PL2_PMU_EVENT_ATTR(inner_put_full_data, INNER_PUT_FULL_DATA, 1),
+	PL2_PMU_EVENT_ATTR(inner_put_partial_data, INNER_PUT_PARTIAL_DATA, 1),
+	PL2_PMU_EVENT_ATTR(inner_atomic_data, INNER_ATOMIC_DATA, 1),
+	PL2_PMU_EVENT_ATTR(inner_get, INNER_GET, 1),
+	PL2_PMU_EVENT_ATTR(inner_prefetch_read, INNER_PREFETCH_READ, 1),
+	PL2_PMU_EVENT_ATTR(inner_prefetch_write, INNER_PREFETCH_WRITE, 1),
+	PL2_PMU_EVENT_ATTR(inner_acquire_block_ntob, INNER_ACQUIRE_BLOCK_NTOB, 1),
+	PL2_PMU_EVENT_ATTR(inner_acquire_block_ntot, INNER_ACQUIRE_BLOCK_NTOT, 1),
+	PL2_PMU_EVENT_ATTR(inner_acquire_block_btot, INNER_ACQUIRE_BLOCK_BTOT, 1),
+	PL2_PMU_EVENT_ATTR(inner_acquire_perm_ntot, INNER_ACQUIRE_PERM_NTOT, 1),
+	PL2_PMU_EVENT_ATTR(inner_acquire_perm_btot, INNER_ACQUIRE_PERM_BTOT, 1),
+	PL2_PMU_EVENT_ATTR(inner_release_ttob, INNER_RELEASE_TTOB, 1),
+	PL2_PMU_EVENT_ATTR(inner_release_tton, INNER_RELEASE_TTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_release_bton, INNER_RELEASE_BTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_release_data_ttob, INNER_RELEASE_DATA_TTOB, 1),
+	PL2_PMU_EVENT_ATTR(inner_release_data_tton, INNER_RELEASE_DATA_TTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_release_data_bton, INNER_RELEASE_DATA_BTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_release_data_ttot, INNER_RELEASE_DATA_TTOT, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_block_tot, INNER_PROBE_BLOCK_TOT, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_block_tob, INNER_PROBE_BLOCK_TOB, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_block_ton, INNER_PROBE_BLOCK_TON, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_perm_ton, INNER_PROBE_PERM_TON, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_ttob, INNER_PROBE_ACK_TTOB, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_tton, INNER_PROBE_ACK_TTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_bton, INNER_PROBE_ACK_BTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_ttot, INNER_PROBE_ACK_TTOT, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_btob, INNER_PROBE_ACK_BTOB, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_nton, INNER_PROBE_ACK_NTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_data_ttob, INNER_PROBE_ACK_DATA_TTOB, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_data_tton, INNER_PROBE_ACK_DATA_TTON, 1),
+	PL2_PMU_EVENT_ATTR(inner_probe_ack_data_ttot, INNER_PROBE_ACK_DATA_TTOT, 1),
+
+	PL2_PMU_EVENT_ATTR(inner_put_full_data_hit, INNER_PUT_FULL_DATA_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_put_partial_data_hit, INNER_PUT_PARTIAL_DATA_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_atomic_data_hit, INNER_ATOMIC_DATA_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_get_hit, INNER_GET_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_prefetch_read_hit, INNER_PREFETCH_READ_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_acquire_block_ntob_hit, INNER_ACQUIRE_BLOCK_NTOB_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_acquire_perm_ntot_hit, INNER_ACQUIRE_PERM_NTOT_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_release_ttob_hit, INNER_RELEASE_TTOB_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_release_data_ttob_hit, INNER_RELEASE_DATA_TTOB_HIT, 2),
+	PL2_PMU_EVENT_ATTR(outer_probe_block_tot_hit, OUTER_PROBE_BLOCK_TOT_HIT, 2),
+	PL2_PMU_EVENT_ATTR(inner_put_full_data_hit_shared, INNER_PUT_FULL_DATA_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_put_partial_data_hit_shared, INNER_PUT_PARTIAL_DATA_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_atomic_data_hit_shared, INNER_ATOMIC_DATA_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_get_hit_shared, INNER_GET_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_prefetch_read_hit_shared, INNER_PREFETCH_READ_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_acquire_block_ntob_hit_shared,
+			   INNER_ACQUIRE_BLOCK_NTOB_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_acquire_perm_ntot_hit_shared,
+			   INNER_ACQUIRE_PERM_NTOT_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_release_ttob_hit_shared, INNER_RELEASE_TTOB_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(inner_release_data_ttob_hit_shared,
+			   INNER_RELEASE_DATA_TTOB_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(outer_probe_block_tot_hit_shared, OUTER_PROBE_BLOCK_TOT_HIT_SHARED, 2),
+	PL2_PMU_EVENT_ATTR(outer_probe_block_tot_hit_dirty, OUTER_PROBE_BLOCK_TOT_HIT_DIRTY, 2),
+
+	PL2_PMU_EVENT_ATTR(outer_put_full_data, OUTER_PUT_FULL_DATA, 3),
+	PL2_PMU_EVENT_ATTR(outer_put_partial_data, OUTER_PUT_PARTIAL_DATA, 3),
+	PL2_PMU_EVENT_ATTR(outer_atomic_data, OUTER_ATOMIC_DATA, 3),
+	PL2_PMU_EVENT_ATTR(outer_get, OUTER_GET, 3),
+	PL2_PMU_EVENT_ATTR(outer_prefetch_read, OUTER_PREFETCH_READ, 3),
+	PL2_PMU_EVENT_ATTR(outer_prefetch_write, OUTER_PREFETCH_WRITE, 3),
+	PL2_PMU_EVENT_ATTR(outer_acquire_block_ntob, OUTER_ACQUIRE_BLOCK_NTOB, 3),
+	PL2_PMU_EVENT_ATTR(outer_acquire_block_ntot, OUTER_ACQUIRE_BLOCK_NTOT, 3),
+	PL2_PMU_EVENT_ATTR(outer_acquire_block_btot, OUTER_ACQUIRE_BLOCK_BTOT, 3),
+	PL2_PMU_EVENT_ATTR(outer_acquire_perm_ntot, OUTER_ACQUIRE_PERM_NTOT, 3),
+	PL2_PMU_EVENT_ATTR(outer_acquire_perm_btot, OUTER_ACQUIRE_PERM_BTOT, 3),
+	PL2_PMU_EVENT_ATTR(outer_release_ttob, OUTER_RELEARE_TTOB, 3),
+	PL2_PMU_EVENT_ATTR(outer_release_tton, OUTER_RELEARE_TTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_release_bton, OUTER_RELEARE_BTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_release_data_ttob, OUTER_RELEARE_DATA_TTOB, 3),
+	PL2_PMU_EVENT_ATTR(outer_release_data_tton, OUTER_RELEARE_DATA_TTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_release_data_bton, OUTER_RELEARE_DATA_BTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_release_data_ttot, OUTER_RELEARE_DATA_TTOT, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_block_tot, OUTER_PROBE_BLOCK_TOT, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_block_tob, OUTER_PROBE_BLOCK_TOB, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_block_ton, OUTER_PROBE_BLOCK_TON, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_perm_ton, OUTER_PROBE_PERM_TON, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_ttob, OUTER_PROBE_ACK_TTOB, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_tton, OUTER_PROBE_ACK_TTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_bton, OUTER_PROBE_ACK_BTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_ttot, OUTER_PROBE_ACK_TTOT, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_btob, OUTER_PROBE_ACK_BTOB, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_nton, OUTER_PROBE_ACK_NTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_data_ttob, OUTER_PROBE_ACK_DATA_TTOB, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_data_tton, OUTER_PROBE_ACK_DATA_TTON, 3),
+	PL2_PMU_EVENT_ATTR(outer_probe_ack_data_ttot, OUTER_PROBE_ACK_DATA_TTOT, 3),
+
+	PL2_PMU_EVENT_ATTR(inner_hint_hits_mshr, INNER_HINT_HITS_MSHR, 4),
+	PL2_PMU_EVENT_ATTR(inner_read_hits_mshr, INNER_READ_HITS_MSHR, 4),
+	PL2_PMU_EVENT_ATTR(inner_write_hits_mshr, INNER_WRITE_HITS_MSHR, 4),
+	PL2_PMU_EVENT_ATTR(inner_read_replay, INNER_READ_REPLAY, 4),
+	PL2_PMU_EVENT_ATTR(inner_write_replay, INNER_WRITE_REPLAY, 4),
+	PL2_PMU_EVENT_ATTR(outer_probe_replay, OUTER_PROBE_REPLAY, 4),
+	PL2_PMU_EVENT_ATTR(replay, REPLAY, 4),
+	PL2_PMU_EVENT_ATTR(sleep_by_miss_queue, SLEEP_BY_MISS_QUEUE, 4),
+	PL2_PMU_EVENT_ATTR(sleep_by_evict_queue, SLEEP_BY_EVICT_QUEUE, 4),
+	PL2_PMU_EVENT_ATTR(sleep_for_back_probe, SLEEP_FOR_BACK_PROBE, 4),
+	PL2_PMU_EVENT_ATTR(sleep, SLEEP, 4),
+
+	PL2_PMU_EVENT_ATTR(read_sleep_timer_expire, READ_SLEEP_TIMER_EXPIRE, 5),
+	PL2_PMU_EVENT_ATTR(read_oldest_timer_expire, READ_OLDEST_TIMER_EXPIRE, 5),
+	PL2_PMU_EVENT_ATTR(write_sleep_timer_expire, WRITE_SLEEP_TIMER_EXPIRE, 5),
+	PL2_PMU_EVENT_ATTR(write_oldest_timer_expire, WRITE_OLDEST_TIMER_EXPIRE, 5),
+	PL2_PMU_EVENT_ATTR(read_sleep, READ_SLEEP, 5),
+	PL2_PMU_EVENT_ATTR(read_dir_update_wakeup, READ_DIR_UPDATE_WAKEUP, 5),
+	PL2_PMU_EVENT_ATTR(read_miss_queue_wakeup, READ_MISS_QUEUE_WAKEUP, 5),
+	PL2_PMU_EVENT_ATTR(read_evict_queue_wakeup, READ_EVICT_QUEUE_WAKEUP, 5),
+	PL2_PMU_EVENT_ATTR(read_sleep_timer_wakeup, READ_SLEEP_TIMER_WAKEUP, 5),
+	PL2_PMU_EVENT_ATTR(write_sleep, WRITE_SLEEP, 5),
+	PL2_PMU_EVENT_ATTR(write_dir_update_wakeup, WRITE_DIR_UPDATE_WAKEUP, 5),
+	PL2_PMU_EVENT_ATTR(write_miss_queue_wakeup, WRITE_MISS_QUEUE_WAKEUP, 5),
+	PL2_PMU_EVENT_ATTR(write_evict_queue_wakeup, WRITE_EVICT_QUEUE_WAKEUP, 5),
+	PL2_PMU_EVENT_ATTR(write_sleep_timer_wakeup, WRITE_SLEEP_TIMER_WAKEUP, 5),
+	NULL
+};
+
+static struct attribute_group sifive_pl2_pmu_events_group = {
+	.name = "events",
+	.attrs = sifive_pl2_pmu_events,
+};
+
+/* formats */
+PMU_FORMAT_ATTR(event, "config:0-63");
+
+static struct attribute *sifive_pl2_pmu_formats[] = {
+	&format_attr_event.attr,
+	NULL,
+};
+
+static struct attribute_group sifive_pl2_pmu_format_group = {
+	.name = "format",
+	.attrs = sifive_pl2_pmu_formats,
+};
+
+/*
+ * Per PMU device attribute groups
+ */
+
+static const struct attribute_group *sifive_pl2_pmu_attr_grps[] = {
+	&sifive_pl2_pmu_events_group,
+	&sifive_pl2_pmu_format_group,
+	NULL,
+};
+
+/*
+ * Event Initialization
+ */
+
+static int sifive_pl2_pmu_event_init(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	u64 config = event->attr.config;
+	u64 ev_type = config >> 8;
+	u64 set = config & 0xff;
+
+	/* Check if this is a valid set and event */
+	switch (set) {
+	case 1:
+		if (ev_type >= BIT_ULL(PL2_PMU_MAX_EVENT1_IDX))
+			return -ENOENT;
+		break;
+	case 2:
+		if (ev_type >= BIT_ULL(PL2_PMU_MAX_EVENT2_IDX))
+			return -ENOENT;
+		break;
+	case 3:
+		if (ev_type >= BIT_ULL(PL2_PMU_MAX_EVENT3_IDX))
+			return -ENOENT;
+		break;
+	case 4:
+		if (ev_type >= BIT_ULL(PL2_PMU_MAX_EVENT4_IDX))
+			return -ENOENT;
+		break;
+	case 5:
+		if (ev_type >= BIT_ULL(PL2_PMU_MAX_EVENT5_IDX))
+			return -ENOENT;
+		break;
+	default:
+		return -ENOENT;
+	}
+
+	/* Do not allocate the hardware counter yet */
+	hwc->idx = -1;
+	hwc->config = config;
+
+	return 0;
+}
+
+/*
+ * pmu->read: read and update the counter
+ */
+static void sifive_pl2_pmu_read(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	u64 prev_raw_count, new_raw_count;
+	u64 oldval;
+
+	do {
+		prev_raw_count = local64_read(&hwc->prev_count);
+		new_raw_count = readq((void *)hwc->event_base);
+
+		oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, new_raw_count);
+	} while (oldval != prev_raw_count);
+
+	local64_add(new_raw_count - prev_raw_count, &event->count);
+}
+
+/*
+ * State transition functions:
+ *
+ * start()/stop() & add()/del()
+ */
+
+/*
+ * pmu->start: start the event
+ */
+static void sifive_pl2_pmu_start(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
+		return;
+
+	hwc->state = 0;
+
+	/* Set initial value to 0 */
+	local64_set(&hwc->prev_count, 0);
+	writeq(0, (void *)hwc->event_base);
+
+	/* Enable this counter to count events */
+	writeq(hwc->config, (void *)hwc->config_base);
+}
+
+/*
+ * pmu->stop: stop the counter
+ */
+static void sifive_pl2_pmu_stop(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (hwc->state & PERF_HES_STOPPED)
+		return;
+
+	/* Disable this counter to count events */
+	writeq(0, (void *)hwc->config_base);
+	sifive_pl2_pmu_read(event);
+
+	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+}
+
+/*
+ * pmu->add: add the event to the PMU
+ */
+static int sifive_pl2_pmu_add(struct perf_event *event, int flags)
+{
+	struct sifive_pl2_pmu *pl2_pmu = to_pl2_pmu(event->pmu);
+	struct sifive_pl2_pmu_event *ptr = *this_cpu_ptr(pl2_pmu->event);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx;
+
+	/* Find an available counter idx to use for this event */
+	do {
+		idx = find_first_zero_bit(ptr->used_mask, ptr->n_counters);
+		if (idx >= ptr->n_counters)
+			return -EAGAIN;
+	} while (test_and_set_bit(idx, ptr->used_mask));
+
+	hwc->config_base = (unsigned long)ptr->base + PL2_SELECT_OFFSET + 8 * idx;
+	hwc->event_base = (unsigned long)ptr->base + PL2_COUNTER_OFFSET + 8 * idx;
+	hwc->idx = idx;
+	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+
+	ptr->events[idx] = event;
+
+	if (flags & PERF_EF_START)
+		sifive_pl2_pmu_start(event, PERF_EF_RELOAD);
+
+	perf_event_update_userpage(event);
+
+	return 0;
+}
+
+/*
+ * pmu->del: delete the event from the PMU
+ */
+static void sifive_pl2_pmu_del(struct perf_event *event, int flags)
+{
+	struct sifive_pl2_pmu *pl2_pmu = to_pl2_pmu(event->pmu);
+	struct sifive_pl2_pmu_event *ptr = *this_cpu_ptr(pl2_pmu->event);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	/* Stop and release this counter */
+	sifive_pl2_pmu_stop(event, PERF_EF_UPDATE);
+
+	ptr->events[idx] = NULL;
+	clear_bit(idx, ptr->used_mask);
+
+	perf_event_update_userpage(event);
+}
+
+/*
+ * pmu->filter: check if the PMU can be used with a CPU
+ */
+static bool sifive_pl2_pmu_filter(struct pmu *pmu, int cpu)
+{
+	struct sifive_pl2_pmu *pl2_pmu = to_pl2_pmu(pmu);
+	struct sifive_pl2_pmu_event *ptr = *this_cpu_ptr(pl2_pmu->event);
+
+	/* Filter out CPUs with no PL2 instance (no percpu data allocated) */
+	return !ptr;
+}
+
+/*
+ * Driver initialization
+ */
+
+static void sifive_pl2_pmu_hw_init(const struct sifive_pl2_pmu_event *ptr)
+{
+	/* Disable the client filter (not supported by this driver) */
+	writeq(0, ptr->base + PL2_CLIENT_FILTER_OFFSET);
+}
+
+static int sifive_pl2_pmu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v)
+{
+	struct sifive_pl2_pmu *pl2_pmu = container_of(nb, struct sifive_pl2_pmu, cpu_pm_nb);
+	struct sifive_pl2_pmu_event *ptr = *this_cpu_ptr(pl2_pmu->event);
+	struct perf_event *event;
+
+	if (!ptr || bitmap_empty(ptr->used_mask, PL2_PMU_MAX_COUNTERS))
+		return NOTIFY_OK;
+
+	for (int idx = 0; idx < ptr->n_counters; idx++) {
+		event = ptr->events[idx];
+		if (!event)
+			continue;
+
+		switch (cmd) {
+		case CPU_PM_ENTER:
+			/* Stop and update the counter */
+			sifive_pl2_pmu_stop(event, PERF_EF_UPDATE);
+			break;
+		case CPU_PM_ENTER_FAILED:
+		case CPU_PM_EXIT:
+			/* Restore and enable the counter */
+			sifive_pl2_pmu_start(event, PERF_EF_RELOAD);
+			break;
+		default:
+			break;
+		}
+	}
+
+	return NOTIFY_OK;
+}
+
+static int sifive_pl2_pmu_pm_register(struct sifive_pl2_pmu *pl2_pmu)
+{
+	if (!IS_ENABLED(CONFIG_CPU_PM))
+		return 0;
+
+	pl2_pmu->cpu_pm_nb.notifier_call = sifive_pl2_pmu_pm_notify;
+	return cpu_pm_register_notifier(&pl2_pmu->cpu_pm_nb);
+}
+
+static void sifive_pl2_pmu_pm_unregister(struct sifive_pl2_pmu *pl2_pmu)
+{
+	if (!IS_ENABLED(CONFIG_CPU_PM))
+		return;
+
+	cpu_pm_unregister_notifier(&pl2_pmu->cpu_pm_nb);
+}
+
+static struct sifive_pl2_pmu *sifive_pl2_pmu_get(void)
+{
+	struct sifive_pl2_pmu *pl2_pmu;
+	int ret;
+
+	guard(mutex)(&g_mutex);
+
+	pl2_pmu = g_pl2_pmu;
+	if (pl2_pmu) {
+		refcount_inc(&pl2_pmu->refcount);
+		return pl2_pmu;
+	}
+
+	pl2_pmu = kzalloc(sizeof(*pl2_pmu), GFP_KERNEL);
+	if (!pl2_pmu)
+		return ERR_PTR(-ENOMEM);
+
+	pl2_pmu->pmu = (struct pmu) {
+		.attr_groups	= sifive_pl2_pmu_attr_grps,
+		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
+		.task_ctx_nr	= perf_sw_context,
+		.event_init	= sifive_pl2_pmu_event_init,
+		.add		= sifive_pl2_pmu_add,
+		.del		= sifive_pl2_pmu_del,
+		.start		= sifive_pl2_pmu_start,
+		.stop		= sifive_pl2_pmu_stop,
+		.read		= sifive_pl2_pmu_read,
+		.filter		= sifive_pl2_pmu_filter,
+	};
+
+	refcount_set(&pl2_pmu->refcount, 1);
+
+	pl2_pmu->event = alloc_percpu(typeof(*pl2_pmu->event));
+	if (!pl2_pmu->event) {
+		ret = -ENOMEM;
+		goto err_free;
+	}
+
+	ret = sifive_pl2_pmu_pm_register(pl2_pmu);
+	if (ret)
+		goto err_free_percpu;
+
+	ret = perf_pmu_register(&pl2_pmu->pmu, "sifive_pl2_pmu", -1);
+	if (ret) {
+		pr_err("%s: Failed to register PMU: %d\n", __func__, ret);
+		goto err_unregister_pm;
+	}
+
+	g_pl2_pmu = pl2_pmu;
+
+	return pl2_pmu;
+
+err_unregister_pm:
+	sifive_pl2_pmu_pm_unregister(pl2_pmu);
+err_free_percpu:
+	free_percpu(pl2_pmu->event);
+err_free:
+	kfree(pl2_pmu);
+
+	return ERR_PTR(ret);
+}
+
+static void sifive_pl2_pmu_put(void)
+{
+	struct sifive_pl2_pmu *pl2_pmu;
+
+	guard(mutex)(&g_mutex);
+
+	pl2_pmu = g_pl2_pmu;
+	if (!refcount_dec_and_test(&pl2_pmu->refcount))
+		return;
+
+	g_pl2_pmu = NULL;
+	perf_pmu_unregister(&pl2_pmu->pmu);
+	sifive_pl2_pmu_pm_unregister(pl2_pmu);
+	free_percpu(pl2_pmu->event);
+	kfree(pl2_pmu);
+}
+
+static int sifive_pl2_pmu_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev_of_node(dev);
+	struct sifive_pl2_pmu_event *ptr;
+	struct sifive_pl2_pmu *pl2_pmu;
+	unsigned int cpu;
+	u32 n_counters;
+	int ret;
+
+	/* Instances without a sifive,perfmon-counters property do not contain a PMU */
+	ret = of_property_read_u32(np, "sifive,perfmon-counters", &n_counters);
+	if (ret || !n_counters)
+		return -ENODEV;
+
+	/* Determine the CPU affinity of this PL2 instance */
+	for_each_possible_cpu(cpu) {
+		struct device_node *cache_node, *cpu_node;
+
+		cpu_node = of_cpu_device_node_get(cpu);
+		if (!cpu_node)
+			continue;
+
+		cache_node = of_parse_phandle(cpu_node, "next-level-cache", 0);
+		of_node_put(cpu_node);
+		if (!cache_node)
+			continue;
+
+		of_node_put(cache_node);
+		if (cache_node == np)
+			break;
+	}
+	if (cpu >= nr_cpu_ids)
+		return -ENODEV;
+
+	ptr = devm_kzalloc(dev, struct_size(ptr, events, n_counters), GFP_KERNEL);
+	if (!ptr)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ptr);
+
+	ptr->cpu = cpu;
+	ptr->n_counters = n_counters;
+
+	ptr->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ptr->base))
+		return PTR_ERR(ptr->base);
+
+	sifive_pl2_pmu_hw_init(ptr);
+
+	pl2_pmu = sifive_pl2_pmu_get();
+	if (IS_ERR(pl2_pmu))
+		return PTR_ERR(pl2_pmu);
+
+	*per_cpu_ptr(pl2_pmu->event, cpu) = ptr;
+
+	return 0;
+}
+
+static void sifive_pl2_pmu_remove(struct platform_device *pdev)
+{
+	struct sifive_pl2_pmu_event *ptr = platform_get_drvdata(pdev);
+
+	*per_cpu_ptr(g_pl2_pmu->event, ptr->cpu) = NULL;
+	sifive_pl2_pmu_put();
+}
+
+static const struct of_device_id sifve_pl2_pmu_of_match[] = {
+	{ .compatible = "sifive,pl2cache1" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, sifve_pl2_pmu_of_match);
+
+static struct platform_driver sifive_pl2_pmu_driver = {
+	.probe	= sifive_pl2_pmu_probe,
+	.remove_new	= sifive_pl2_pmu_remove,
+	.driver	= {
+		.name		= "sifive_pl2_pmu",
+		.of_match_table	= sifve_pl2_pmu_of_match,
+	},
+};
+module_platform_driver(sifive_pl2_pmu_driver);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 7dfb7190580e..fc0f01e31bcc 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -195,6 +195,16 @@ config PINCTRL_EQUILIBRIUM
 	  desired pin functions, configure GPIO attributes for LGM SoC pins.
 	  Pin muxing and pin config settings are retrieved from device tree.
 
+config PINCTRL_EIC7700
+	tristate "EIC7700 PINCTRL driver"
+	select GENERIC_PINCONF
+	select PINMUX
+	select PINCONF
+	select GENERIC_PINCTRL_GROUPS
+	select GENERIC_PINMUX_FUNCTIONS
+	help
+	  say Y here to add pinctrl eic7700 driver
+
 config PINCTRL_GEMINI
 	bool
 	depends on ARCH_GEMINI
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index dd6cda270294..3e51b32bd161 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -22,6 +22,7 @@ obj-$(CONFIG_PINCTRL_DA850_PUPD) += pinctrl-da850-pupd.o
 obj-$(CONFIG_PINCTRL_DA9062)	+= pinctrl-da9062.o
 obj-$(CONFIG_PINCTRL_DIGICOLOR)	+= pinctrl-digicolor.o
 obj-$(CONFIG_PINCTRL_EQUILIBRIUM)   += pinctrl-equilibrium.o
+obj-$(CONFIG_PINCTRL_EIC7700)  += pinctrl-eic7700.o
 obj-$(CONFIG_PINCTRL_GEMINI)	+= pinctrl-gemini.o
 obj-$(CONFIG_PINCTRL_INGENIC)	+= pinctrl-ingenic.o
 obj-$(CONFIG_PINCTRL_K210)	+= pinctrl-k210.o
diff --git a/drivers/pinctrl/pinctrl-eic7700.c b/drivers/pinctrl/pinctrl-eic7700.c
new file mode 100644
index 000000000000..8db8b8d1877a
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-eic7700.c
@@ -0,0 +1,1360 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Pinctrl Controller Platform Device Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yulin Lu <luyulin@eswincomputing.com>
+ */
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/regmap.h>
+#include "pinctrl-utils.h"
+#include "core.h"
+
+#define ESWIN_MIO_NUM 164
+
+#define ESWIN_PINCONF_IE	   BIT(0)
+#define ESWIN_PINCONF_PULLUP   BIT(1)
+#define ESWIN_PINCONF_PULLDOWN BIT(2)
+#define ESWIN_PINCONF_DRIVER_STRENGTH_MASK  0xf  // 0111 1000
+#define ESWIN_PINCONF_DRIVER_SHIFT 3
+#define ESWIN_PINCONF_SMT BIT(7)
+
+struct eswin_function_desc {
+	const char *name;
+	const char * const *groups;
+	int ngroups;
+	u32 mux_val;
+	u32 mux_mask;
+};
+
+struct eswin_group_desc{
+	const char *name;
+	const unsigned int *pins;
+	const unsigned int npins;
+
+};
+
+struct eswin_pinctrl {
+	struct pinctrl_dev *pctrl;
+	void __iomem		*base;
+	const struct eswin_group_desc *groups;
+	unsigned int ngroups;
+	const struct eswin_function_desc *funcs;
+	unsigned int nfuncs;
+};
+
+static const struct pinctrl_pin_desc eswin_pins[] = {
+
+	PINCTRL_PIN(0, "CHIP_MODE"),
+	PINCTRL_PIN(1, "MODE_SET0"),
+	PINCTRL_PIN(2, "MODE_SET1"),
+	PINCTRL_PIN(3, "MODE_SET2"),
+	PINCTRL_PIN(4, "MODE_SET3"),
+	PINCTRL_PIN(5, "XIN"),
+	PINCTRL_PIN(6, "RTC_XIN"),
+	PINCTRL_PIN(7, "RST_OUT_N"),
+	PINCTRL_PIN(8,"KEY_RESET_N"),
+	PINCTRL_PIN(9,"RST_IN_N"),
+	PINCTRL_PIN(10,"POR_IN_N"),
+	PINCTRL_PIN(11,"POR_OUT_N"),
+	PINCTRL_PIN(12,"GPIO0"),
+	PINCTRL_PIN(13,"POR_SEL"),
+	PINCTRL_PIN(14,"JTAG0_TCK"),
+	PINCTRL_PIN(15,"JTAG0_TMS"),
+	PINCTRL_PIN(16,"JTAG0_TDI"),
+	PINCTRL_PIN(17,"JTAG0_TDO"),
+	PINCTRL_PIN(18,"JTAG0_TRST"),
+	PINCTRL_PIN(19,"SPI2_CS0_N"),
+	PINCTRL_PIN(20,"JTAG1_TCK"),
+	PINCTRL_PIN(21,"JTAG1_TMS"),
+	PINCTRL_PIN(22,"JTAG1_TDI"),
+	PINCTRL_PIN(23,"JTAG1_TDO"),
+	PINCTRL_PIN(24,"JTAG1_TRST"),
+	PINCTRL_PIN(25,"SPI2_CS1_N"),
+	PINCTRL_PIN(26,"PCIE_CLKREQ_N"),
+	PINCTRL_PIN(27,"PCIE_WAKE_N"),
+	PINCTRL_PIN(28,"PCIE_PERST_N"),
+	PINCTRL_PIN(29,"HDMI_SCL"),
+	PINCTRL_PIN(30,"HDMI_SDA"),
+	PINCTRL_PIN(31,"HDMI_CEC"),
+	PINCTRL_PIN(32,"JTAG2_TRST"),
+	PINCTRL_PIN(33,"RGMII0_CLK_125"),
+	PINCTRL_PIN(34,"RGMII0_TXEN"),
+	PINCTRL_PIN(35,"RGMII0_TXCLK"),
+	PINCTRL_PIN(36,"RGMII0_TXD0"),
+	PINCTRL_PIN(37,"RGMII0_TXD1"),
+	PINCTRL_PIN(38,"RGMII0_TXD2"),
+	PINCTRL_PIN(39,"RGMII0_TXD3"),
+	PINCTRL_PIN(40,"I2S0_BCLK"),
+	PINCTRL_PIN(41,"I2S0_WCLK"),
+	PINCTRL_PIN(42,"I2S0_SDI"),
+	PINCTRL_PIN(43,"I2S0_SDO"),
+	PINCTRL_PIN(44,"I2S_MCLK"),
+	PINCTRL_PIN(45,"RGMII0_RXCLK"),
+	PINCTRL_PIN(46,"RGMII0_RXDV"),
+	PINCTRL_PIN(47,"RGMII0_RXD0"),
+	PINCTRL_PIN(48,"RGMII0_RXD1"),
+	PINCTRL_PIN(49,"RGMII0_RXD2"),
+	PINCTRL_PIN(50,"RGMII0_RXD3"),
+	PINCTRL_PIN(51,"I2S2_BCLK"),
+	PINCTRL_PIN(52,"I2S2_WCLK"),
+	PINCTRL_PIN(53,"I2S2_SDI"),
+	PINCTRL_PIN(54,"I2S2_SDO"),
+	PINCTRL_PIN(55,"GPIO27"),
+	PINCTRL_PIN(56,"GPIO28"),
+	PINCTRL_PIN(57,"GPIO29"),
+	PINCTRL_PIN(58,"RGMII0_MDC"),
+	PINCTRL_PIN(59,"RGMII0_MDIO"),
+	PINCTRL_PIN(60,"RGMII0_INTB"),
+	PINCTRL_PIN(61,"RGMII1_CLK_125"),
+	PINCTRL_PIN(62,"RGMII1_TXEN"),
+	PINCTRL_PIN(63,"RGMII1_TXCLK"),
+	PINCTRL_PIN(64,"RGMII1_TXD0"),
+	PINCTRL_PIN(65,"RGMII1_TXD1"),
+	PINCTRL_PIN(66,"RGMII1_TXD2"),
+	PINCTRL_PIN(67,"RGMII1_TXD3"),
+	PINCTRL_PIN(68,"I2S1_BCLK"),
+	PINCTRL_PIN(69,"I2S1_WCLK"),
+	PINCTRL_PIN(70,"I2S1_SDI"),
+	PINCTRL_PIN(71,"I2S1_SDO"),
+	PINCTRL_PIN(72,"GPIO34"),
+	PINCTRL_PIN(73,"RGMII1_RXCLK"),
+	PINCTRL_PIN(74,"RGMII1_RXDV"),
+	PINCTRL_PIN(75,"RGMII1_RXD0"),
+	PINCTRL_PIN(76,"RGMII1_RXD1"),
+	PINCTRL_PIN(77,"RGMII1_RXD2"),
+	PINCTRL_PIN(78,"RGMII1_RXD3"),
+	PINCTRL_PIN(79,"SPI1_CS0_N"),
+	PINCTRL_PIN(80,"SPI1_CLK"),
+	PINCTRL_PIN(81,"SPI1_D0"),
+	PINCTRL_PIN(82,"SPI1_D1"),
+	PINCTRL_PIN(83,"SPI1_D2"),
+	PINCTRL_PIN(84,"SPI1_D3"),
+	PINCTRL_PIN(85,"SPI1_CS1_N"),
+	PINCTRL_PIN(86,"RGMII1_MDC"),
+	PINCTRL_PIN(87,"RGMII1_MDIO"),
+	PINCTRL_PIN(88,"RGMII1_INTB"),
+	PINCTRL_PIN(89,"USB0_PWREN"),
+	PINCTRL_PIN(90,"USB1_PWREN"),
+	PINCTRL_PIN(91,"I2C0_SCL"),
+	PINCTRL_PIN(92,"I2C0_SDA"),
+	PINCTRL_PIN(93,"I2C1_SCL"),
+	PINCTRL_PIN(94,"I2C1_SDA"),
+	PINCTRL_PIN(95,"I2C2_SCL"),
+	PINCTRL_PIN(96,"I2C2_SDA"),
+	PINCTRL_PIN(97,"I2C3_SCL"),
+	PINCTRL_PIN(98,"I2C3_SDA"),
+	PINCTRL_PIN(99,"I2C4_SCL"),
+	PINCTRL_PIN(100,"I2C4_SDA"),
+	PINCTRL_PIN(101,"I2C5_SCL"),
+	PINCTRL_PIN(102,"I2C5_SDA"),
+	PINCTRL_PIN(103,"UART0_TX"),
+	PINCTRL_PIN(104,"UART0_RX"),
+	PINCTRL_PIN(105,"UART1_TX"),
+	PINCTRL_PIN(106,"UART1_RX"),
+	PINCTRL_PIN(107,"UART1_CTS"),
+	PINCTRL_PIN(108,"UART1_RTS"),
+	PINCTRL_PIN(109,"UART2_TX"),
+	PINCTRL_PIN(110,"UART2_RX"),
+	PINCTRL_PIN(111,"JTAG2_TCK"),
+	PINCTRL_PIN(112,"JTAG2_TMS"),
+	PINCTRL_PIN(113,"JTAG2_TDI"),
+	PINCTRL_PIN(114,"JTAG2_TDO"),
+	PINCTRL_PIN(115,"FAN_PWM"),
+	PINCTRL_PIN(116,"FAN_TACH"),
+	PINCTRL_PIN(117,"MIPI_CSI0_XVS"),
+	PINCTRL_PIN(118,"MIPI_CSI0_XHS"),
+	PINCTRL_PIN(119,"MIPI_CSI0_MCLK"),
+	PINCTRL_PIN(120,"MIPI_CSI1_XVS"),
+	PINCTRL_PIN(121,"MIPI_CSI1_XHS"),
+	PINCTRL_PIN(122,"MIPI_CSI1_MCLK"),
+	PINCTRL_PIN(123,"MIPI_CSI2_XVS"),
+	PINCTRL_PIN(124,"MIPI_CSI2_XHS"),
+	PINCTRL_PIN(125,"MIPI_CSI2_MCLK"),
+	PINCTRL_PIN(126,"MIPI_CSI3_XVS"),
+	PINCTRL_PIN(127,"MIPI_CSI3_XHS"),
+	PINCTRL_PIN(128,"MIPI_CSI3_MCLK"),
+	PINCTRL_PIN(129,"MIPI_CSI4_XVS"),
+	PINCTRL_PIN(130,"MIPI_CSI4_XHS"),
+	PINCTRL_PIN(131,"MIPI_CSI4_MCLK"),
+	PINCTRL_PIN(132,"MIPI_CSI5_XVS"),
+	PINCTRL_PIN(133,"MIPI_CSI5_XHS"),
+	PINCTRL_PIN(134,"MIPI_CSI5_MCLK"),
+	PINCTRL_PIN(135,"SPI3_CS_N"),
+	PINCTRL_PIN(136,"SPI3_CLK"),
+	PINCTRL_PIN(137,"SPI3_DI"),
+	PINCTRL_PIN(138,"SPI3_DO"),
+	PINCTRL_PIN(139,"GPIO92"),
+	PINCTRL_PIN(140,"GPIO93"),
+	PINCTRL_PIN(141,"S_MODE"),
+	PINCTRL_PIN(142,"GPIO95"),
+	PINCTRL_PIN(143,"SPI0_CS_N"),
+	PINCTRL_PIN(144,"SPI0_CLK"),
+	PINCTRL_PIN(145,"SPI0_D0"),
+	PINCTRL_PIN(146,"SPI0_D1"),
+	PINCTRL_PIN(147,"SPI0_D2"),
+	PINCTRL_PIN(148,"SPI0_D3"),
+	PINCTRL_PIN(149,"I2C10_SCL"),
+	PINCTRL_PIN(150,"I2C10_SDA"),
+	PINCTRL_PIN(151,"I2C11_SCL"),
+	PINCTRL_PIN(152,"I2C11_SDA"),
+	PINCTRL_PIN(153,"GPIO106"),
+	PINCTRL_PIN(154,"BOOT_SEL0"),
+	PINCTRL_PIN(155,"BOOT_SEL1"),
+	PINCTRL_PIN(156,"BOOT_SEL2"),
+	PINCTRL_PIN(157,"BOOT_SEL3"),
+	PINCTRL_PIN(158,"GPIO111"),
+	PINCTRL_PIN(159,"D2D_SERDES_STATUS_IN"),
+	PINCTRL_PIN(160,"D2D_SERDES_STATUS_OUT"),
+	PINCTRL_PIN(161,"LPDDR_REF_CLK"),
+
+};
+
+#define ESWIN_PINCTRL_GRP(_name) \
+	{ \
+		.name = #_name "_group", \
+		.pins = _name ## _pins, \
+		.npins = ARRAY_SIZE(_name ## _pins), \
+	}
+
+//func0
+static const char * const sdio0_group[] = {"sdio0_group"};
+static const char * const sdio1_group[] = {"sdio1_group"};
+static const char * const por_sel_group[] = {"por_sel_group"};
+static const char * const jtag0_group[] = {"jtag0_group"};
+static const char * const jtag1_group[] = {"jtag1_group"};
+static const char * const spi2_cs_group[] = {"spi2_cs_group"};
+static const char * const pcie_group[] = {"pcie_group"};
+static const char * const hdmi_group[] = {"hdmi_group"};
+static const char * const jtag2_group[] = {"jtag2_group"};
+static const char * const rgmii0_group[] = {"rgmii0_group"};
+static const char * const i2s0_group[] = {"i2s0_group"};
+static const char * const i2s1_group[] = {"i2s1_group"};
+static const char * const i2s2_group[] = {"i2s2_group"};
+static const char * const por_time_sel0_group[] = {"por_time_sel0_group"};
+static const char * const por_time_sel1_group[] = {"por_time_sel1_group"};
+static const char * const rgmii1_group[] = {"rgmii1_group"};
+static const char * const spi1_group[] = {"spi1_group"};
+static const char * const usb0_pwren_group[] = {"usb0_pwren_group"};
+static const char * const usb1_pwren_group[] = {"usb1_pwren_group"};
+static const char * const i2c0_group[] = {"i2c0_group"};
+static const char * const i2c1_group[] = {"i2c1_group"};
+static const char * const i2c2_group[] = {"i2c2_group"};
+static const char * const i2c3_group[] = {"i2c3_group"};
+static const char * const i2c4_group[] = {"i2c4_group"};
+static const char * const i2c5_group[] = {"i2c5_group"};
+static const char * const uart0_group[] = {"uart0_group"};
+static const char * const uart1_group[] = {"uart1_group"};
+static const char * const uart2_group[] = {"uart2_group"};
+
+static const char * const pwm0_group[] = {"pwm0_group"};
+static const char * const fan_tach_group[] = {"fan_tach_group"};
+static const char * const mipi_csi0_group[] = {"mipi_csi0_group"};
+static const char * const mipi_csi1_group[] = {"mipi_csi1_group"};
+static const char * const mipi_csi2_group[] = {"mipi_csi2_group"};
+static const char * const mipi_csi3_group[] = {"mipi_csi3_group"};
+static const char * const mipi_csi4_group[] = {"mipi_csi4_group"};
+static const char * const mipi_csi5_group[] = {"mipi_csi5_group"};
+static const char * const spi3_group[] = {"spi3_group"};
+static const char * const i2c8_group[] = {"i2c8_group"};
+static const char * const s_mode_group[] = {"s_mode_group"};
+static const char * const pinmux_ddr_refclk_sel_group[] = {"pinmux_ddr_refclk_sel_group"};
+static const char * const spi0_group[] = {"spi0_group"};
+static const char * const i2c10_group[] = {"i2c10_group"};
+static const char * const i2c11_group[] = {"i2c11_group"};
+static const char * const boot_sel_group[] = {"boot_sel_group"};
+static const char * const lpddr_ref_clk_group[] = {"lpddr_ref_clk_group"};
+
+//func1
+static const char * const spi2_clk_group[] = {"spi2_clk_group"};
+static const char * const spi2_d0_group[] = {"spi2_d0_group"};
+static const char * const spi2_d1_d2_d3_group[] = {"spi2_d1_d2_d3_group"};
+
+static const char * const sata_act_led_group[] = {"sata_act_led_group"};
+static const char * const emmc_led_control_group[] = {"emmc_led_control_group"};
+static const char * const sd0_led_control_group[] = {"sd0_led_control_group"};
+static const char * const i2c9_group[] = {"i2c9_group"};
+static const char * const sd1_led_control_group[] = {"sd1_led_control_group"};
+static const char * const pwm1_group[] = {"pwm1_group"};
+static const char * const pwm2_group[] = {"pwm2_group"};
+static const char * const i2c6_group[] = {"i2c6_group"};
+static const char * const i2c7_group[] = {"i2c7_group"};
+static const char * const mipi_csi_xtrig_group[] = {"mipi_csi_xtrig_group"};
+
+//gpio
+static const char * const gpio0_group[] = {"gpio0_group"};
+static const char * const gpio1_group[] = {"gpio1_group"};
+static const char * const gpio2_group[] = {"gpio2_group"};
+static const char * const gpio3_group[] = {"gpio3_group"};
+static const char * const gpio4_group[] = {"gpio4_group"};
+static const char * const gpio5_group[] = {"gpio5_group"};
+static const char * const gpio6_group[] = {"gpio6_group"};
+static const char * const gpio7_group[] = {"gpio7_group"};
+static const char * const gpio8_group[] = {"gpio8_group"};
+static const char * const gpio9_group[] = {"gpio9_group"};
+static const char * const gpio10_group[] = {"gpio10_group"};
+static const char * const gpio11_group[] = {"gpio11_group"};
+static const char * const gpio12_group[] = {"gpio12_group"};
+static const char * const gpio13_group[] = {"gpio13_group"};
+static const char * const gpio14_group[] = {"gpio14_group"};
+static const char * const gpio15_group[] = {"gpio15_group"};
+static const char * const gpio16_group[] = {"gpio16_group"};
+static const char * const gpio17_group[] = {"gpio17_group"};
+static const char * const gpio18_group[] = {"gpio18_group"};
+static const char * const gpio19_group[] = {"gpio19_group"};
+static const char * const gpio20_group[] = {"gpio20_group"};
+static const char * const gpio21_group[] = {"gpio21_group"};
+static const char * const gpio22_group[] = {"gpio22_group"};
+static const char * const gpio23_group[] = {"gpio23_group"};
+static const char * const gpio24_group[] = {"gpio24_group"};
+static const char * const gpio25_group[] = {"gpio25_group"};
+static const char * const gpio26_group[] = {"gpio26_group"};
+static const char * const gpio27_group[] = {"gpio27_group"};
+static const char * const gpio28_group[] = {"gpio28_group"};
+static const char * const gpio29_group[] = {"gpio29_group"};
+static const char * const gpio30_group[] = {"gpio30_group"};
+static const char * const gpio31_group[] = {"gpio31_group"};
+static const char * const gpio32_group[] = {"gpio32_group"};
+static const char * const gpio33_group[] = {"gpio33_group"};
+static const char * const gpio34_group[] = {"gpio34_group"};
+static const char * const gpio35_group[] = {"gpio35_group"};
+static const char * const gpio36_group[] = {"gpio36_group"};
+static const char * const gpio37_group[] = {"gpio37_group"};
+static const char * const gpio38_group[] = {"gpio38_group"};
+static const char * const gpio39_group[] = {"gpio39_group"};
+static const char * const gpio40_group[] = {"gpio40_group"};
+static const char * const gpio41_group[] = {"gpio41_group"};
+static const char * const gpio42_group[] = {"gpio42_group"};
+static const char * const gpio43_group[] = {"gpio43_group"};
+static const char * const gpio44_group[] = {"gpio44_group"};
+static const char * const gpio45_group[] = {"gpio45_group"};
+static const char * const gpio46_group[] = {"gpio46_group"};
+static const char * const gpio47_group[] = {"gpio47_group"};
+static const char * const gpio48_group[] = {"gpio48_group"};
+static const char * const gpio49_group[] = {"gpio49_group"};
+
+static const char * const gpio50_group[] = {"gpio50_group"};
+static const char * const gpio51_group[] = {"gpio51_group"};
+static const char * const gpio52_group[] = {"gpio52_group"};
+static const char * const gpio53_group[] = {"gpio53_group"};
+static const char * const gpio54_group[] = {"gpio54_group"};
+static const char * const gpio55_group[] = {"gpio55_group"};
+static const char * const gpio56_group[] = {"gpio56_group"};
+static const char * const gpio57_group[] = {"gpio57_group"};
+static const char * const gpio58_group[] = {"gpio58_group"};
+static const char * const gpio59_group[] = {"gpio59_group"};
+
+static const char * const gpio60_group[] = {"gpio60_group"};
+static const char * const gpio61_group[] = {"gpio61_group"};
+static const char * const gpio62_group[] = {"gpio62_group"};
+static const char * const gpio63_group[] = {"gpio63_group"};
+static const char * const gpio64_group[] = {"gpio64_group"};
+static const char * const gpio65_group[] = {"gpio65_group"};
+static const char * const gpio66_group[] = {"gpio66_group"};
+static const char * const gpio67_group[] = {"gpio67_group"};
+static const char * const gpio68_group[] = {"gpio68_group"};
+static const char * const gpio69_group[] = {"gpio69_group"};
+
+static const char * const gpio70_group[] = {"gpio70_group"};
+static const char * const gpio71_group[] = {"gpio71_group"};
+static const char * const gpio72_group[] = {"gpio72_group"};
+static const char * const gpio73_group[] = {"gpio73_group"};
+static const char * const gpio74_group[] = {"gpio74_group"};
+static const char * const gpio75_group[] = {"gpio75_group"};
+static const char * const gpio76_group[] = {"gpio76_group"};
+static const char * const gpio77_group[] = {"gpio77_group"};
+static const char * const gpio78_group[] = {"gpio78_group"};
+static const char * const gpio79_group[] = {"gpio79_group"};
+
+static const char * const gpio80_group[] = {"gpio80_group"};
+static const char * const gpio81_group[] = {"gpio81_group"};
+static const char * const gpio82_group[] = {"gpio82_group"};
+static const char * const gpio83_group[] = {"gpio83_group"};
+static const char * const gpio84_group[] = {"gpio84_group"};
+static const char * const gpio85_group[] = {"gpio85_group"};
+static const char * const gpio86_group[] = {"gpio86_group"};
+static const char * const gpio87_group[] = {"gpio87_group"};
+static const char * const gpio88_group[] = {"gpio88_group"};
+static const char * const gpio89_group[] = {"gpio89_group"};
+
+static const char * const gpio90_group[] = {"gpio90_group"};
+static const char * const gpio91_group[] = {"gpio91_group"};
+static const char * const gpio92_group[] = {"gpio92_group"};
+static const char * const gpio93_group[] = {"gpio93_group"};
+static const char * const gpio94_group[] = {"gpio94_group"};
+static const char * const gpio95_group[] = {"gpio95_group"};
+static const char * const gpio96_group[] = {"gpio96_group"};
+static const char * const gpio97_group[] = {"gpio97_group"};
+static const char * const gpio98_group[] = {"gpio98_group"};
+static const char * const gpio99_group[] = {"gpio99_group"};
+
+static const char * const gpio100_group[] = {"gpio100_group"};
+static const char * const gpio101_group[] = {"gpio101_group"};
+static const char * const gpio102_group[] = {"gpio102_group"};
+static const char * const gpio103_group[] = {"gpio103_group"};
+static const char * const gpio104_group[] = {"gpio104_group"};
+static const char * const gpio105_group[] = {"gpio105_group"};
+static const char * const gpio106_group[] = {"gpio106_group"};
+static const char * const gpio107_group[] = {"gpio107_group"};
+static const char * const gpio108_group[] = {"gpio108_group"};
+static const char * const gpio109_group[] = {"gpio109_group"};
+static const char * const gpio110_group[] = {"gpio110_group"};
+static const char * const gpio111_group[] = {"gpio111_group"};
+
+//func3
+static const char * const uart4_group[] = {"uart4_group"};
+static const char * const uart3_group[] = {"uart3_group"};
+
+//func6
+static const char * const csi_mon_out_group[] = {"csi_mon_out_group"};
+static const char * const csi_ocla_clk_group[] = {"csi_ocla_clk_group"};
+static const char * const csi_mon_out_valid_group[] = {"csi_mon_out_valid_group"};
+static const char * const csi_parity_error_group[] = {"csi_parity_error_group"};
+static const char * const csi_dtb_out_group[] = {"csi_dtb_out_group"};
+static const char * const csi_phy_sel_group[] = {"csi_phy_sel_group"};
+static const char * const vc_g2d0_debug_out_group[] = {"vc_g2d0_debug_out_group"};
+static const char * const vc_g2d1_debug_out_group[] = {"vc_g2d1_debug_out_group"};
+static const char * const sata_mpll_clk_group[] = {"sata_mpll_clk_group"};
+static const char * const sata_ref_repeat_clk_m_group[] = {"sata_ref_repeat_clk_m_group"};
+static const char * const sata_ref_repeat_clk_p_group[] = {"sata_ref_repeat_clk_p_group"};
+
+//func0
+static const unsigned int sdio0_pins[] = {1,2};
+static const unsigned int sdio1_pins[] = {3,4};
+static const unsigned int por_sel_pins[] = {13};
+static const unsigned int jtag0_pins[] = {14,15,16,17};
+static const unsigned int jtag1_pins[] = {20,21,22,23};
+static const unsigned int spi2_cs_pins[] = {19,25};
+static const unsigned int pcie_pins[] = {26,27,28};
+static const unsigned int hdmi_pins[] = {29,30,31};
+static const unsigned int jtag2_pins[] = {32,111,112,113,114};
+static const unsigned int rgmii0_pins[] = {33,34,35,36,37,38,39,45,46,47,48,49,50,58,59,60};
+static const unsigned int i2s0_pins[] = {40,41,42,43,44};
+static const unsigned int i2s1_pins[] = {68,69,70,71,44};
+static const unsigned int i2s2_pins[] = {51,52,53,54,44};
+static const unsigned int por_time_sel0_pins[] = {57};
+static const unsigned int por_time_sel1_pins[] = {72};
+static const unsigned int rgmii1_pins[] = {61,62,63,64,65,66,67,73,74,75,76,77,78,86,87,88};
+static const unsigned int spi1_pins[] = {79,80,81,82,83,84,85};
+static const unsigned int usb0_pwren_pins[] = {89};
+static const unsigned int usb1_pwren_pins[] = {90};
+static const unsigned int i2c0_pins[] = {91,92};
+static const unsigned int i2c1_pins[] = {93,94};
+static const unsigned int i2c2_pins[] = {95,96};
+static const unsigned int i2c3_pins[] = {97,98};
+static const unsigned int i2c4_pins[] = {99,100};
+static const unsigned int i2c5_pins[] = {101,102};
+static const unsigned int uart0_pins[] = {103,104};
+static const unsigned int uart1_pins[] = {105,106,107,108};
+static const unsigned int uart2_pins[] = {109,110};
+static const unsigned int pwm0_pins[] = {115};
+static const unsigned int fan_tach_pins[] = {116};
+static const unsigned int mipi_csi0_pins[] = {117,118,119};
+static const unsigned int mipi_csi1_pins[] = {120,121,122};
+static const unsigned int mipi_csi2_pins[] = {123,124,125};
+static const unsigned int mipi_csi3_pins[] = {126,127,128};
+static const unsigned int mipi_csi4_pins[] = {129,130,131};
+static const unsigned int mipi_csi5_pins[] = {132,133,134};
+static const unsigned int spi3_pins[] = {135,136,137,138};
+static const unsigned int i2c8_pins[] = {139,140};
+static const unsigned int s_mode_pins[] = {141};
+static const unsigned int pinmux_ddr_refclk_sel_pins[] = {142};
+static const unsigned int spi0_pins[] = {143,144,145,146,147,148};
+static const unsigned int i2c10_pins[] = {149,150};
+static const unsigned int i2c11_pins[] = {151,152};
+static const unsigned int boot_sel_pins[] = {154,155,156,157};
+static const unsigned int lpddr_ref_clk_pins[] = {158};
+
+//func1
+static const unsigned int spi2_clk_pins[] = {14};
+static const unsigned int spi2_d0_pins[] = {15};
+static const unsigned int spi2_d1_d2_d3_pins[] = {16,17,18};
+
+static const unsigned int sata_act_led_pins[] = {55};
+static const unsigned int emmc_led_control_pins[] = {57};
+static const unsigned int sd0_led_control_pins[] = {72};
+static const unsigned int i2c9_pins[] = {81,82};
+static const unsigned int sd1_led_control_pins[] = {83};
+static const unsigned int pwm1_pins[] = {84};
+static const unsigned int pwm2_pins[] = {85};
+static const unsigned int i2c6_pins[] = {107,108};
+static const unsigned int i2c7_pins[] = {109,110};
+static const unsigned int mipi_csi_xtrig_pins[] = {139,140};
+
+//gpio
+static const unsigned int gpio0_pins[] = {12};
+static const unsigned int gpio1_pins[] = {14};
+static const unsigned int gpio2_pins[] = {15};
+static const unsigned int gpio3_pins[] = {16};
+static const unsigned int gpio4_pins[] = {17};
+static const unsigned int gpio5_pins[] = {18};
+static const unsigned int gpio6_pins[] = {19};
+static const unsigned int gpio7_pins[] = {20};
+static const unsigned int gpio8_pins[] = {21};
+static const unsigned int gpio9_pins[] = {22};
+
+static const unsigned int gpio10_pins[] = {23};
+static const unsigned int gpio11_pins[] = {24};
+static const unsigned int gpio12_pins[] = {25};
+static const unsigned int gpio13_pins[] = {1};
+static const unsigned int gpio14_pins[] = {2};
+static const unsigned int gpio15_pins[] = {3};
+static const unsigned int gpio16_pins[] = {4};
+static const unsigned int gpio17_pins[] = {32};
+static const unsigned int gpio18_pins[] = {40};
+static const unsigned int gpio19_pins[] = {41};
+
+static const unsigned int gpio20_pins[] = {42};
+static const unsigned int gpio21_pins[] = {43};
+static const unsigned int gpio22_pins[] = {44};
+static const unsigned int gpio23_pins[] = {51};
+static const unsigned int gpio24_pins[] = {52};
+static const unsigned int gpio25_pins[] = {53};
+static const unsigned int gpio26_pins[] = {54};
+static const unsigned int gpio27_pins[] = {55};
+static const unsigned int gpio28_pins[] = {56};
+static const unsigned int gpio29_pins[] = {57};
+
+static const unsigned int gpio30_pins[] = {68};
+static const unsigned int gpio31_pins[] = {69};
+static const unsigned int gpio32_pins[] = {70};
+static const unsigned int gpio33_pins[] = {71};
+static const unsigned int gpio34_pins[] = {72};
+static const unsigned int gpio35_pins[] = {79};
+static const unsigned int gpio36_pins[] = {80};
+static const unsigned int gpio37_pins[] = {81};
+static const unsigned int gpio38_pins[] = {82};
+static const unsigned int gpio39_pins[] = {83};
+
+static const unsigned int gpio40_pins[] = {84};
+static const unsigned int gpio41_pins[] = {85};
+static const unsigned int gpio42_pins[] = {89};
+static const unsigned int gpio43_pins[] = {90};
+static const unsigned int gpio44_pins[] = {91};
+static const unsigned int gpio45_pins[] = {92};
+static const unsigned int gpio46_pins[] = {93};
+static const unsigned int gpio47_pins[] = {94};
+static const unsigned int gpio48_pins[] = {95};
+static const unsigned int gpio49_pins[] = {96};
+
+static const unsigned int gpio50_pins[] = {97};
+static const unsigned int gpio51_pins[] = {98};
+static const unsigned int gpio52_pins[] = {99};
+static const unsigned int gpio53_pins[] = {100};
+static const unsigned int gpio54_pins[] = {101};
+static const unsigned int gpio55_pins[] = {102};
+static const unsigned int gpio56_pins[] = {103};
+static const unsigned int gpio57_pins[] = {104};
+static const unsigned int gpio58_pins[] = {105};
+static const unsigned int gpio59_pins[] = {106};
+
+static const unsigned int gpio60_pins[] = {107};
+static const unsigned int gpio61_pins[] = {108};
+static const unsigned int gpio62_pins[] = {109};
+static const unsigned int gpio63_pins[] = {110};
+static const unsigned int gpio64_pins[] = {111};
+static const unsigned int gpio65_pins[] = {112};
+static const unsigned int gpio66_pins[] = {113};
+static const unsigned int gpio67_pins[] = {114};
+static const unsigned int gpio68_pins[] = {115};
+static const unsigned int gpio69_pins[] = {116};
+
+static const unsigned int gpio70_pins[] = {117};
+static const unsigned int gpio71_pins[] = {118};
+static const unsigned int gpio72_pins[] = {119};
+static const unsigned int gpio73_pins[] = {120};
+static const unsigned int gpio74_pins[] = {121};
+static const unsigned int gpio75_pins[] = {122};
+static const unsigned int gpio76_pins[] = {123};
+static const unsigned int gpio77_pins[] = {124};
+static const unsigned int gpio78_pins[] = {125};
+static const unsigned int gpio79_pins[] = {126};
+
+static const unsigned int gpio80_pins[] = {127};
+static const unsigned int gpio81_pins[] = {128};
+static const unsigned int gpio82_pins[] = {129};
+static const unsigned int gpio83_pins[] = {130};
+static const unsigned int gpio84_pins[] = {131};
+static const unsigned int gpio85_pins[] = {132};
+static const unsigned int gpio86_pins[] = {133};
+static const unsigned int gpio87_pins[] = {134};
+static const unsigned int gpio88_pins[] = {135};
+static const unsigned int gpio89_pins[] = {136};
+
+static const unsigned int gpio90_pins[] = {137};
+static const unsigned int gpio91_pins[] = {138};
+static const unsigned int gpio92_pins[] = {139};
+static const unsigned int gpio93_pins[] = {140};
+static const unsigned int gpio94_pins[] = {141};
+static const unsigned int gpio95_pins[] = {142};
+static const unsigned int gpio96_pins[] = {143};
+static const unsigned int gpio97_pins[] = {144};
+static const unsigned int gpio98_pins[] = {145};
+static const unsigned int gpio99_pins[] = {146};
+
+static const unsigned int gpio100_pins[] = {147};
+static const unsigned int gpio101_pins[] = {148};
+static const unsigned int gpio102_pins[] = {149};
+static const unsigned int gpio103_pins[] = {150};
+static const unsigned int gpio104_pins[] = {151};
+static const unsigned int gpio105_pins[] = {152};
+static const unsigned int gpio106_pins[] = {153};
+static const unsigned int gpio107_pins[] = {154};
+static const unsigned int gpio108_pins[] = {155};
+static const unsigned int gpio109_pins[] = {156};
+static const unsigned int gpio110_pins[] = {157};
+static const unsigned int gpio111_pins[] = {158};
+
+//func3
+static const unsigned int uart4_pins[] = {81,82};
+static const unsigned int uart3_pins[] = {139,140};
+
+//func6
+static const unsigned int csi_mon_out_pins[] = {32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55};
+static const unsigned int csi_ocla_clk_pins[] = {96};
+static const unsigned int csi_mon_out_valid_pins[] = {97};
+static const unsigned int csi_parity_error_pins[] = {98};
+static const unsigned int csi_dtb_out_pins[] = {99,100,101,102,129,130,131,132};
+static const unsigned int csi_phy_sel_pins[] = {133,134,109};
+static const unsigned int vc_g2d0_debug_out_pins[] = {110,111,112,113,114,115,116,117};
+static const unsigned int vc_g2d1_debug_out_pins[] = {118,119,120,121,122,123,124,125};
+static const unsigned int sata_mpll_clk_pins[] = {126};
+static const unsigned int sata_ref_repeat_clk_m_pins[] = {127};
+static const unsigned int sata_ref_repeat_clk_p_pins[] = {128};
+
+static const struct eswin_group_desc eswin_pinctrl_groups[] =
+{
+	//func0
+	ESWIN_PINCTRL_GRP(sdio0),
+	ESWIN_PINCTRL_GRP(sdio1),
+	ESWIN_PINCTRL_GRP(por_sel),
+	ESWIN_PINCTRL_GRP(jtag0),
+	ESWIN_PINCTRL_GRP(jtag1),
+	ESWIN_PINCTRL_GRP(spi2_cs),
+	ESWIN_PINCTRL_GRP(pcie),
+	ESWIN_PINCTRL_GRP(hdmi),
+	ESWIN_PINCTRL_GRP(jtag2),
+	ESWIN_PINCTRL_GRP(rgmii0),
+	ESWIN_PINCTRL_GRP(i2s0),
+	ESWIN_PINCTRL_GRP(i2s1),
+	ESWIN_PINCTRL_GRP(i2s2),
+	ESWIN_PINCTRL_GRP(por_time_sel0),
+	ESWIN_PINCTRL_GRP(por_time_sel1),
+	ESWIN_PINCTRL_GRP(rgmii1),
+	ESWIN_PINCTRL_GRP(spi1),
+	ESWIN_PINCTRL_GRP(usb0_pwren),
+	ESWIN_PINCTRL_GRP(usb1_pwren),
+	ESWIN_PINCTRL_GRP(i2c0),
+	ESWIN_PINCTRL_GRP(i2c1),
+	ESWIN_PINCTRL_GRP(i2c2),
+	ESWIN_PINCTRL_GRP(i2c3),
+	ESWIN_PINCTRL_GRP(i2c4),
+	ESWIN_PINCTRL_GRP(i2c5),
+	ESWIN_PINCTRL_GRP(uart0),
+	ESWIN_PINCTRL_GRP(uart1),
+	ESWIN_PINCTRL_GRP(uart2),
+	ESWIN_PINCTRL_GRP(pwm0),
+	ESWIN_PINCTRL_GRP(fan_tach),
+	ESWIN_PINCTRL_GRP(mipi_csi0),
+	ESWIN_PINCTRL_GRP(mipi_csi1),
+	ESWIN_PINCTRL_GRP(mipi_csi2),
+	ESWIN_PINCTRL_GRP(mipi_csi3),
+	ESWIN_PINCTRL_GRP(mipi_csi4),
+	ESWIN_PINCTRL_GRP(mipi_csi5),
+	ESWIN_PINCTRL_GRP(spi3),
+	ESWIN_PINCTRL_GRP(i2c8),
+	ESWIN_PINCTRL_GRP(s_mode),
+	ESWIN_PINCTRL_GRP(pinmux_ddr_refclk_sel),
+	ESWIN_PINCTRL_GRP(spi0),
+	ESWIN_PINCTRL_GRP(i2c10),
+	ESWIN_PINCTRL_GRP(i2c11),
+	ESWIN_PINCTRL_GRP(boot_sel),
+	ESWIN_PINCTRL_GRP(lpddr_ref_clk),
+
+	//func1
+	ESWIN_PINCTRL_GRP(spi2_clk),
+	ESWIN_PINCTRL_GRP(spi2_d0),
+	ESWIN_PINCTRL_GRP(spi2_d1_d2_d3),
+
+	ESWIN_PINCTRL_GRP(sata_act_led),
+	ESWIN_PINCTRL_GRP(emmc_led_control),
+	ESWIN_PINCTRL_GRP(sd0_led_control),
+	ESWIN_PINCTRL_GRP(i2c9),
+	ESWIN_PINCTRL_GRP(sd1_led_control),
+	ESWIN_PINCTRL_GRP(pwm1),
+	ESWIN_PINCTRL_GRP(pwm2),
+	ESWIN_PINCTRL_GRP(i2c6),
+	ESWIN_PINCTRL_GRP(i2c7),
+	ESWIN_PINCTRL_GRP(mipi_csi_xtrig),
+
+	//gpio
+	ESWIN_PINCTRL_GRP(gpio0),
+	ESWIN_PINCTRL_GRP(gpio1),
+	ESWIN_PINCTRL_GRP(gpio2),
+	ESWIN_PINCTRL_GRP(gpio3),
+	ESWIN_PINCTRL_GRP(gpio4),
+	ESWIN_PINCTRL_GRP(gpio5),
+	ESWIN_PINCTRL_GRP(gpio6),
+	ESWIN_PINCTRL_GRP(gpio7),
+	ESWIN_PINCTRL_GRP(gpio8),
+	ESWIN_PINCTRL_GRP(gpio9),
+	ESWIN_PINCTRL_GRP(gpio10),
+	ESWIN_PINCTRL_GRP(gpio11),
+	ESWIN_PINCTRL_GRP(gpio12),
+	ESWIN_PINCTRL_GRP(gpio13),
+	ESWIN_PINCTRL_GRP(gpio14),
+	ESWIN_PINCTRL_GRP(gpio15),
+	ESWIN_PINCTRL_GRP(gpio16),
+	ESWIN_PINCTRL_GRP(gpio17),
+	ESWIN_PINCTRL_GRP(gpio18),
+	ESWIN_PINCTRL_GRP(gpio19),
+	ESWIN_PINCTRL_GRP(gpio20),
+	ESWIN_PINCTRL_GRP(gpio21),
+	ESWIN_PINCTRL_GRP(gpio22),
+	ESWIN_PINCTRL_GRP(gpio23),
+	ESWIN_PINCTRL_GRP(gpio24),
+	ESWIN_PINCTRL_GRP(gpio25),
+	ESWIN_PINCTRL_GRP(gpio26),
+	ESWIN_PINCTRL_GRP(gpio27),
+	ESWIN_PINCTRL_GRP(gpio28),
+	ESWIN_PINCTRL_GRP(gpio29),
+	ESWIN_PINCTRL_GRP(gpio30),
+	ESWIN_PINCTRL_GRP(gpio31),
+	ESWIN_PINCTRL_GRP(gpio32),
+	ESWIN_PINCTRL_GRP(gpio33),
+	ESWIN_PINCTRL_GRP(gpio34),
+	ESWIN_PINCTRL_GRP(gpio35),
+	ESWIN_PINCTRL_GRP(gpio36),
+	ESWIN_PINCTRL_GRP(gpio37),
+	ESWIN_PINCTRL_GRP(gpio38),
+	ESWIN_PINCTRL_GRP(gpio39),
+	ESWIN_PINCTRL_GRP(gpio40),
+	ESWIN_PINCTRL_GRP(gpio41),
+	ESWIN_PINCTRL_GRP(gpio42),
+	ESWIN_PINCTRL_GRP(gpio43),
+	ESWIN_PINCTRL_GRP(gpio44),
+	ESWIN_PINCTRL_GRP(gpio45),
+	ESWIN_PINCTRL_GRP(gpio46),
+	ESWIN_PINCTRL_GRP(gpio47),
+	ESWIN_PINCTRL_GRP(gpio48),
+	ESWIN_PINCTRL_GRP(gpio49),
+
+	ESWIN_PINCTRL_GRP(gpio50),
+	ESWIN_PINCTRL_GRP(gpio51),
+	ESWIN_PINCTRL_GRP(gpio52),
+	ESWIN_PINCTRL_GRP(gpio53),
+	ESWIN_PINCTRL_GRP(gpio54),
+	ESWIN_PINCTRL_GRP(gpio55),
+	ESWIN_PINCTRL_GRP(gpio56),
+	ESWIN_PINCTRL_GRP(gpio57),
+	ESWIN_PINCTRL_GRP(gpio58),
+	ESWIN_PINCTRL_GRP(gpio59),
+
+	ESWIN_PINCTRL_GRP(gpio60),
+	ESWIN_PINCTRL_GRP(gpio61),
+	ESWIN_PINCTRL_GRP(gpio62),
+	ESWIN_PINCTRL_GRP(gpio63),
+	ESWIN_PINCTRL_GRP(gpio64),
+	ESWIN_PINCTRL_GRP(gpio65),
+	ESWIN_PINCTRL_GRP(gpio66),
+	ESWIN_PINCTRL_GRP(gpio67),
+	ESWIN_PINCTRL_GRP(gpio68),
+	ESWIN_PINCTRL_GRP(gpio69),
+
+	ESWIN_PINCTRL_GRP(gpio70),
+	ESWIN_PINCTRL_GRP(gpio71),
+	ESWIN_PINCTRL_GRP(gpio72),
+	ESWIN_PINCTRL_GRP(gpio73),
+	ESWIN_PINCTRL_GRP(gpio74),
+	ESWIN_PINCTRL_GRP(gpio75),
+	ESWIN_PINCTRL_GRP(gpio76),
+	ESWIN_PINCTRL_GRP(gpio77),
+	ESWIN_PINCTRL_GRP(gpio78),
+	ESWIN_PINCTRL_GRP(gpio79),
+
+	ESWIN_PINCTRL_GRP(gpio80),
+	ESWIN_PINCTRL_GRP(gpio81),
+	ESWIN_PINCTRL_GRP(gpio82),
+	ESWIN_PINCTRL_GRP(gpio83),
+	ESWIN_PINCTRL_GRP(gpio84),
+	ESWIN_PINCTRL_GRP(gpio85),
+	ESWIN_PINCTRL_GRP(gpio86),
+	ESWIN_PINCTRL_GRP(gpio87),
+	ESWIN_PINCTRL_GRP(gpio88),
+	ESWIN_PINCTRL_GRP(gpio89),
+
+	ESWIN_PINCTRL_GRP(gpio90),
+	ESWIN_PINCTRL_GRP(gpio91),
+	ESWIN_PINCTRL_GRP(gpio92),
+	ESWIN_PINCTRL_GRP(gpio93),
+	ESWIN_PINCTRL_GRP(gpio94),
+	ESWIN_PINCTRL_GRP(gpio95),
+	ESWIN_PINCTRL_GRP(gpio96),
+	ESWIN_PINCTRL_GRP(gpio97),
+	ESWIN_PINCTRL_GRP(gpio98),
+	ESWIN_PINCTRL_GRP(gpio99),
+
+	ESWIN_PINCTRL_GRP(gpio100),
+	ESWIN_PINCTRL_GRP(gpio101),
+	ESWIN_PINCTRL_GRP(gpio102),
+	ESWIN_PINCTRL_GRP(gpio103),
+	ESWIN_PINCTRL_GRP(gpio104),
+	ESWIN_PINCTRL_GRP(gpio105),
+	ESWIN_PINCTRL_GRP(gpio106),
+	ESWIN_PINCTRL_GRP(gpio107),
+	ESWIN_PINCTRL_GRP(gpio108),
+	ESWIN_PINCTRL_GRP(gpio109),
+	ESWIN_PINCTRL_GRP(gpio110),
+	ESWIN_PINCTRL_GRP(gpio111),
+
+	//func3
+	ESWIN_PINCTRL_GRP(uart4),
+	ESWIN_PINCTRL_GRP(uart3),
+
+	//func6
+	ESWIN_PINCTRL_GRP(csi_mon_out),
+	ESWIN_PINCTRL_GRP(csi_ocla_clk),
+	ESWIN_PINCTRL_GRP(csi_mon_out_valid),
+	ESWIN_PINCTRL_GRP(csi_parity_error),
+	ESWIN_PINCTRL_GRP(csi_dtb_out),
+	ESWIN_PINCTRL_GRP(csi_phy_sel),
+	ESWIN_PINCTRL_GRP(vc_g2d0_debug_out),
+	ESWIN_PINCTRL_GRP(vc_g2d1_debug_out),
+	ESWIN_PINCTRL_GRP(sata_mpll_clk),
+	ESWIN_PINCTRL_GRP(sata_ref_repeat_clk_m),
+	ESWIN_PINCTRL_GRP(sata_ref_repeat_clk_p),
+};
+
+#define ESWIN_PINMUX_FUNCTION(_func_name, _mux_val, _mask)\
+	{				   \
+		.name = #_func_name"_func",			 \
+		.groups = _func_name##_group,	  \
+		.ngroups = ARRAY_SIZE(_func_name##_group),  \
+		.mux_val = _mux_val,			\
+		.mux_mask = _mask,		  \
+	}
+
+#define ESWIN_PINMUX_SHIFT  16
+#define ESWIN_PINMUX_MASK   (0x07 << ESWIN_PINMUX_SHIFT)
+
+static const struct eswin_function_desc eswin_pinmux_functions[] = {
+
+	//func0
+	ESWIN_PINMUX_FUNCTION(sdio0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(sdio1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(por_sel, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(jtag0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(jtag1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(spi2_cs, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(pcie, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(hdmi, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(jtag2, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(rgmii0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2s0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2s1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2s2, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(por_time_sel0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(por_time_sel1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(rgmii1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(spi1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(usb0_pwren, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(usb1_pwren, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c2, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c3, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c4, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c5, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(uart0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(uart1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(uart2, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(pwm0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(fan_tach, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(mipi_csi0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(mipi_csi1, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(mipi_csi2, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(mipi_csi3, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(mipi_csi4, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(mipi_csi5, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(spi3, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c8, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(s_mode, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(pinmux_ddr_refclk_sel, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(spi0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c10, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c11, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(boot_sel, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(lpddr_ref_clk, 0, ESWIN_PINMUX_MASK),
+
+	//func1
+	ESWIN_PINMUX_FUNCTION(spi2_clk, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(spi2_d0, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(spi2_d1_d2_d3, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(sata_act_led, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(emmc_led_control, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(sd0_led_control, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c9, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(sd1_led_control, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(pwm1, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(pwm2, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c6, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(i2c7, 1, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(mipi_csi_xtrig, 1, ESWIN_PINMUX_MASK),
+
+	//gpio
+	ESWIN_PINMUX_FUNCTION(gpio0, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio1, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio2, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio3, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio4, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio5, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio6, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio7, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio8, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio9, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio10, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio11, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio12, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio13, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio14, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio15, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio16, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio17, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio18, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio19, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio20, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio21, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio22, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio23, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio24, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio25, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio26, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio27, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio28, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio29, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio30, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio31, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio32, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio33, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio34, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio35, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio36, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio37, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio38, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio39, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio40, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio41, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio42, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio43, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio44, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio45, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio46, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio47, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio48, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio49, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio50, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio51, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio52, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio53, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio54, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio55, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio56, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio57, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio58, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio59, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio60, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio61, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio62, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio63, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio64, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio65, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio66, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio67, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio68, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio69, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio70, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio71, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio72, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio73, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio74, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio75, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio76, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio77, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio78, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio79, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio80, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio81, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio82, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio83, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio84, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio85, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio86, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio87, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio88, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio89, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio90, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio91, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio92, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio93, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio94, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio95, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio96, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio97, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio98, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio99, 2, ESWIN_PINMUX_MASK),
+
+	ESWIN_PINMUX_FUNCTION(gpio100, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio101, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio102, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio103, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio104, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio105, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio106, 0, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio107, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio108, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio109, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio110, 2, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(gpio111, 0, ESWIN_PINMUX_MASK),
+
+	//func3
+	ESWIN_PINMUX_FUNCTION(uart4, 3, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(uart3, 3, ESWIN_PINMUX_MASK),
+
+	//func6
+	ESWIN_PINMUX_FUNCTION(csi_mon_out, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(csi_ocla_clk, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(csi_mon_out_valid, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(csi_parity_error, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(csi_dtb_out, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(csi_phy_sel, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(vc_g2d0_debug_out, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(vc_g2d1_debug_out, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(sata_mpll_clk, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(sata_ref_repeat_clk_m, 6, ESWIN_PINMUX_MASK),
+	ESWIN_PINMUX_FUNCTION(sata_ref_repeat_clk_p, 6, ESWIN_PINMUX_MASK),
+};
+
+/* pinctrl */
+static int eswin_pctrl_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+	return pctrl->ngroups;
+}
+
+static const char *eswin_pctrl_get_group_name(struct pinctrl_dev *pctldev,
+						 unsigned int selector)
+{
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+	return pctrl->groups[selector].name;
+}
+
+static int eswin_pctrl_get_group_pins(struct pinctrl_dev *pctldev,
+					 unsigned int selector,
+					 const unsigned int **pins,
+					 unsigned int *num_pins)
+{
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+	*pins = pctrl->groups[selector].pins;
+	*num_pins = pctrl->groups[selector].npins;
+	return 0;
+}
+
+static const struct pinctrl_ops eswin_pinctrl_ops = {
+	.get_groups_count = eswin_pctrl_get_groups_count,
+	.get_group_name = eswin_pctrl_get_group_name,
+	.get_group_pins = eswin_pctrl_get_group_pins,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_all,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+/* pinmux */
+static int eswin_pmux_get_functions_count(struct pinctrl_dev *pctldev)
+{
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->nfuncs;
+}
+
+static const char *eswin_pmux_get_function_name(struct pinctrl_dev *pctldev,
+						   unsigned int selector)
+{
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->funcs[selector].name;
+}
+
+static int eswin_pmux_get_function_groups(struct pinctrl_dev *pctldev,
+					 unsigned int selector,
+					 const char * const **groups,
+					 unsigned * const num_groups)
+{
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+	*groups = pctrl->funcs[selector].groups;
+	*num_groups = pctrl->funcs[selector].ngroups;
+	return 0;
+}
+
+static int eswin_pinmux_set_mux(struct pinctrl_dev *pctldev,
+				   unsigned int function,
+				   unsigned int  group)
+{
+	int i ;
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+	const struct eswin_group_desc *pgrp = &pctrl->groups[group];
+	const struct eswin_function_desc *func = &pctrl->funcs[function];
+
+	for(i = 0 ;i< pgrp->npins;i++){
+		u32 reg ;
+		unsigned int pin = pgrp->pins[i];
+		reg = readl(pctrl->base + 4*pin);
+		reg &= ~ESWIN_PINMUX_MASK;
+		reg |= (func->mux_val << ESWIN_PINMUX_SHIFT);
+		writel(reg,pctrl->base + 4*pin);
+	}
+	return 0;
+}
+
+static const struct pinmux_ops eswin_pinmux_ops = {
+	.get_functions_count = eswin_pmux_get_functions_count,
+	.get_function_name = eswin_pmux_get_function_name,
+	.get_function_groups = eswin_pmux_get_function_groups,
+	.set_mux = eswin_pinmux_set_mux,
+};
+
+/* pinconfig */
+static int eswin_pinconf_cfg_get(struct pinctrl_dev *pctldev,
+		   unsigned pin,
+			  unsigned long *config)
+{
+	u32 reg=0;
+	int ret=0;
+	unsigned int arg = 0;
+	unsigned int param = pinconf_to_config_param(*config);
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	if (pin >= ESWIN_MIO_NUM){
+		return -ENOTSUPP;
+	}
+	reg = readl(pctrl->base + 4*pin);
+	if (ret)
+		return -EIO;
+
+	switch (param) {
+	case PIN_CONFIG_BIAS_PULL_UP:
+	if (!(arg=(reg & ESWIN_PINCONF_PULLUP))){
+		return -EINVAL;}
+		break;
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+	if (!(arg = (reg & ESWIN_PINCONF_PULLDOWN))){
+		return -EINVAL;}
+		break;
+	case PIN_CONFIG_DRIVE_STRENGTH:
+	arg = (reg & (ESWIN_PINCONF_DRIVER_STRENGTH_MASK
+						   <<ESWIN_PINCONF_DRIVER_SHIFT));
+		break;
+	case PIN_CONFIG_INPUT_ENABLE:
+	if (!(arg = (reg & ESWIN_PINCONF_IE))){
+		return -EINVAL;}
+	break;
+	case PIN_CONFIG_INPUT_SCHMITT:
+	if (!(arg = (reg & ESWIN_PINCONF_SMT))){
+		return -EINVAL;}
+	break;
+	default:
+	return -ENOTSUPP;
+	}
+	*config = pinconf_to_config_packed(param, arg);
+	return 0;
+}
+
+int eswin_pinconf_cfg_set(struct pinctrl_dev *pctldev,
+			unsigned pin,
+			unsigned long *configs,
+			unsigned num_configs)
+{
+	int i=0, ret=0;
+	u32 reg = 0;
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	if (pin >= ESWIN_MIO_NUM)
+		return -ENOTSUPP;
+	reg = readl(pctrl->base + 4*pin);
+
+	if (ret)
+		return -EIO;
+
+	for (i = 0; i < num_configs; i++) {
+		unsigned int param = pinconf_to_config_param(configs[i]);
+		unsigned int arg = pinconf_to_config_argument(configs[i]);
+
+		switch (param) {
+			case PIN_CONFIG_INPUT_ENABLE:
+				reg &=~ESWIN_PINCONF_IE;
+				reg |= (arg<<0);
+				break;
+			case PIN_CONFIG_BIAS_PULL_UP:
+				reg &=~ESWIN_PINCONF_PULLUP;
+				reg |= (arg<<1);
+				break;
+			case PIN_CONFIG_BIAS_PULL_DOWN:
+				reg &=~ESWIN_PINCONF_PULLDOWN;
+				reg |= (arg<<2);
+				break;
+			case PIN_CONFIG_DRIVE_STRENGTH:
+				reg &= ~(ESWIN_PINCONF_DRIVER_STRENGTH_MASK<<3);
+				reg |= (arg<<3);
+				break;
+			case PIN_CONFIG_INPUT_SCHMITT:
+				reg &= ~ESWIN_PINCONF_SMT;
+				reg |= (arg<<7);
+				break;
+			default:
+				dev_warn(pctldev->dev,
+					"unsupported configuration parameter '%u'\n",
+					param);
+				continue;
+		}
+
+
+	}
+		writel(reg,pctrl->base + 4*pin);
+
+	if (ret)
+		return -EIO;
+	return 0;
+}
+
+static int eswin_pinconf_group_set(struct pinctrl_dev *pctldev,
+			unsigned selector,
+			unsigned long *configs,
+			unsigned num_configs)
+
+{
+	int i=0, ret=0;
+	struct eswin_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+	const struct eswin_group_desc *pgrp = &pctrl->groups[selector];
+
+	for (i = 0; i < pgrp->npins; i++) {
+		ret = eswin_pinconf_cfg_set(pctldev, pgrp->pins[i], configs,
+					 num_configs);
+	if (ret)
+		  return ret;
+	}
+	return 0;
+}
+
+static const struct pinconf_ops eswin_pinconf_ops = {
+	.is_generic = true,
+	.pin_config_get = eswin_pinconf_cfg_get,
+	.pin_config_set = eswin_pinconf_cfg_set,
+	.pin_config_group_set = eswin_pinconf_group_set,
+};
+
+static struct pinctrl_desc eswin_desc = {
+	.name = "eswin_pinctrl",
+	.pins = eswin_pins,
+	.npins = ARRAY_SIZE(eswin_pins),
+	.pctlops = &eswin_pinctrl_ops,
+	.pmxops = &eswin_pinmux_ops,
+	.confops = &eswin_pinconf_ops,
+	.owner = THIS_MODULE,
+};
+
+static int eswin_pinctrl_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct eswin_pinctrl *pctrl;
+	pctrl = devm_kzalloc(&pdev->dev, sizeof(*pctrl), GFP_KERNEL);
+	if (!pctrl)
+		return -ENOMEM;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+	pctrl->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pctrl->base))
+		return PTR_ERR(pctrl->base);
+
+	pctrl->groups = eswin_pinctrl_groups;
+	pctrl->ngroups = ARRAY_SIZE(eswin_pinctrl_groups);
+	pctrl->funcs = eswin_pinmux_functions;
+	pctrl->nfuncs = ARRAY_SIZE(eswin_pinmux_functions);
+
+	pctrl->pctrl = devm_pinctrl_register(&pdev->dev, &eswin_desc, pctrl);
+	if (IS_ERR(pctrl->pctrl)){
+		return PTR_ERR(pctrl->pctrl);
+	}
+
+	platform_set_drvdata(pdev, pctrl);
+
+	dev_info(&pdev->dev, "eswin pinctrl initialized\n");
+
+	return 0;
+}
+
+static int eswin_pinctrl_remove(struct platform_device *platform_dev)
+{
+	struct eswin_pinctrl *eswin_pinctrl_ptr = platform_get_drvdata(platform_dev);
+	pinctrl_unregister(eswin_pinctrl_ptr->pctrl);
+	return 0;
+}
+
+static const struct of_device_id eswin_pinctrl_of_match[] = {
+	{ .compatible = "eswin,eic7700-pinctrl" },
+	{ }
+};
+
+static struct platform_driver eswin_pinctrl_driver = {
+	.driver = {
+		.name = "eswin-pinctrl",
+		.of_match_table = eswin_pinctrl_of_match,
+	},
+	.probe = eswin_pinctrl_probe,
+	.remove = eswin_pinctrl_remove,
+};
+
+static int __init eswin_pinctrl_init(void)
+{
+	return platform_driver_register(&eswin_pinctrl_driver);
+}
+postcore_initcall(eswin_pinctrl_init);
+
+static void __exit eswin_pinctrl_exit(void)
+{
+	platform_driver_unregister(&eswin_pinctrl_driver);
+}
+module_exit(eswin_pinctrl_exit);
+
+MODULE_DESCRIPTION("ESWIN Pinctrl Controller Platform Device Drivers");
+MODULE_AUTHOR("luyulin@eswincomputing.com");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index 8ebcddf91f7b..a7c9ac6c7870 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -205,6 +205,12 @@ config PWM_EP93XX
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-ep93xx.
 
+config PWM_ESWIN
+	tristate "ESWIN PWM support"
+	help
+	  Generic PWM framework driver for the PWM controller found on
+	  ESWIN SoCs.
+
 config PWM_FSL_FTM
 	tristate "Freescale FlexTimer Module (FTM) PWM support"
 	depends on HAS_IOMEM
diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile
index c822389c2a24..f3619959f508 100644
--- a/drivers/pwm/Makefile
+++ b/drivers/pwm/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_PWM_CRC)		+= pwm-crc.o
 obj-$(CONFIG_PWM_CROS_EC)	+= pwm-cros-ec.o
 obj-$(CONFIG_PWM_DWC)		+= pwm-dwc.o
 obj-$(CONFIG_PWM_EP93XX)	+= pwm-ep93xx.o
+obj-$(CONFIG_PWM_ESWIN)	+= pwm-eswin.o
 obj-$(CONFIG_PWM_FSL_FTM)	+= pwm-fsl-ftm.o
 obj-$(CONFIG_PWM_HIBVT)		+= pwm-hibvt.o
 obj-$(CONFIG_PWM_IMG)		+= pwm-img.o
diff --git a/drivers/pwm/pwm-eswin.c b/drivers/pwm/pwm-eswin.c
new file mode 100644
index 000000000000..1d15a3e06c8b
--- /dev/null
+++ b/drivers/pwm/pwm-eswin.c
@@ -0,0 +1,326 @@
+/*
+ * Copyright (c) 2022, ESWIN, Inc.
+ * Author: zhangchunyun@eswincomputing.com
+ */
+
+#include <linux/bitops.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/pm_runtime.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+#include <linux/time.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+
+#define ESWIN_TIM_LD_CNT(n)   ((n) * 0x14)
+#define ESWIN_TIM_LD_CNT2(n)  (((n) * 4) + 0xb0)
+#define ESWIN_TIM_CUR_VAL(n)  (((n) * 0x14) + 0x04)
+#define ESWIN_TIM_CTRL(n)     (((n) * 0x14) + 0x08)
+#define ESWIN_TIM_EOI(n)      (((n) * 0x14) + 0x0c)
+#define ESWIN_TIM_INT_STS(n)  (((n) * 0x14) + 0x10)
+
+#define ESWIN_TIMERS_INT_STS  0xa0
+#define ESWIN_TIMERS_EOI      0xa4
+#define ESWIN_TIMERS_RAW_INT_STS  0xa8
+#define ESWIN_TIMERS_COMP_VERSION 0xac
+
+#define ESWIN_TIMERS_TOTAL    8
+#define NSEC_TO_SEC	1000000000
+
+/* Timer Control Register */
+#define ESWIN_TIM_CTRL_EN     BIT(0)
+#define ESWIN_TIM_CTRL_MODE   BIT(1)
+#define ESWIN_TIM_CTRL_MODE_FREE  (0 << 1)
+#define ESWIN_TIM_CTRL_MODE_USER  (1 << 1)
+#define ESWIN_TIM_CTRL_INT_MASK   BIT(2)
+#define ESWIN_TIM_CTRL_PWM    BIT(3)
+
+struct eswin_pwm_ctx {
+	u32 cnt;
+	u32 cnt2;
+	u32 ctrl;
+};
+
+struct eswin_pwm {
+	struct pwm_chip chip;
+	void __iomem *base;
+	struct clk *clk;
+	struct clk *pclk;
+	struct eswin_pwm_ctx ctx[ESWIN_TIMERS_TOTAL];
+	struct reset_control * pwm_rst;
+	u32 clk_period_ns;
+};
+
+#define to_eswin_pwm(p)   (container_of((p), struct eswin_pwm, chip))
+
+static inline u32 eswin_pwm_readl(struct eswin_pwm *eswin, u32 offset)
+{
+	return readl(eswin->base + offset);
+}
+
+static inline void eswin_pwm_writel(struct eswin_pwm *eswin, u32 value, u32 offset)
+{
+	writel(value, eswin->base + offset);
+}
+
+static void __eswin_pwm_set_enable(struct eswin_pwm *eswin, int pwm, int enabled)
+{
+	u32 reg;
+
+	reg = eswin_pwm_readl(eswin, ESWIN_TIM_CTRL(pwm));
+
+	if (enabled)
+		reg |= ESWIN_TIM_CTRL_EN;
+	else
+		reg &= ~ESWIN_TIM_CTRL_EN;
+
+	eswin_pwm_writel(eswin, reg, ESWIN_TIM_CTRL(pwm));
+	reg = eswin_pwm_readl(eswin, ESWIN_TIM_CTRL(pwm));
+}
+
+static int __eswin_pwm_configure_timer(struct eswin_pwm *eswin,
+       struct pwm_device *pwm,
+       const struct pwm_state *state)
+{
+	u64 tmp;
+	u32 ctrl;
+	u32 high;
+	u32 low;
+
+	/*
+	¦* Calculate width of low and high period in terms of input clock
+	¦* periods and check are the result within HW limits between 1 and
+	¦* 2^32 periods.
+	¦*/
+
+	tmp = DIV_ROUND_CLOSEST_ULL(state->duty_cycle, eswin->clk_period_ns);
+	if (tmp < 1 || tmp > (1ULL << 32))
+              return -ERANGE;
+	high = tmp - 1;
+
+	tmp = DIV_ROUND_CLOSEST_ULL(state->period - state->duty_cycle,
+         eswin->clk_period_ns);
+	if (tmp < 1 || tmp > (1ULL << 32))
+        return -ERANGE;
+	low = tmp - 1;
+	/*
+	¦* Specification says timer usage flow is to disable timer, then
+	¦* program it followed by enable. It also says Load Count is loaded
+	¦* into timer after it is enabled - either after a disable or
+	¦* a reset. Based on measurements it happens also without disable
+	¦* whenever Load Count is updated. But follow the specification.
+	¦*/
+	__eswin_pwm_set_enable(eswin, pwm->hwpwm, false);
+
+	/*
+	¦* Write Load Count and Load Count 2 registers. Former defines the
+	¦* width of low period and latter the width of high period in terms
+	¦* multiple of input clock periods:
+	¦* Width = ((Count + 1) * input clock period).
+	¦*/
+	eswin_pwm_writel(eswin, low, ESWIN_TIM_LD_CNT(pwm->hwpwm));
+	eswin_pwm_writel(eswin, high, ESWIN_TIM_LD_CNT2(pwm->hwpwm));
+
+	/*
+	¦* Set user-defined mode, timer reloads from Load Count registers
+	¦* when it counts down to 0.
+	¦* Set PWM mode, it makes output to toggle and width of low and high
+	¦* periods are set by Load Count registers.
+	¦*/
+	ctrl = ESWIN_TIM_CTRL_MODE_USER | ESWIN_TIM_CTRL_PWM;
+	eswin_pwm_writel(eswin, ctrl, ESWIN_TIM_CTRL(pwm->hwpwm));
+
+	/*
+	¦* Enable timer. Output starts from low period.
+	¦*/
+	__eswin_pwm_set_enable(eswin, pwm->hwpwm, state->enabled);
+
+	return 0;
+}
+
+static int eswin_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
+	const struct pwm_state *state)
+{
+	struct eswin_pwm *eswin = to_eswin_pwm(chip);
+	struct pwm_state curstate;
+	int ret = 0;
+
+	ret = clk_enable(eswin->pclk);
+
+	ret = clk_enable(eswin->clk);
+
+	pwm_get_state(pwm, &curstate);
+
+	__eswin_pwm_configure_timer(eswin, pwm, state);
+
+	return 0;
+}
+
+static int eswin_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
+	struct pwm_state *state)
+{
+	struct eswin_pwm *eswin = to_eswin_pwm(chip);
+	u64 duty, period;
+
+	pm_runtime_get_sync(chip->dev);
+
+	state->enabled = !!(eswin_pwm_readl(eswin,
+		ESWIN_TIM_CTRL(pwm->hwpwm)) & ESWIN_TIM_CTRL_EN);
+
+	duty = eswin_pwm_readl(eswin, ESWIN_TIM_LD_CNT(pwm->hwpwm));
+	duty += 1;
+	duty *= eswin->clk_period_ns;
+	state->duty_cycle = duty;
+
+	period = eswin_pwm_readl(eswin, ESWIN_TIM_LD_CNT2(pwm->hwpwm));
+	period += 1;
+	period *= eswin->clk_period_ns;
+	period += duty;
+	state->period = period;
+
+	state->polarity = PWM_POLARITY_INVERSED;
+
+	pm_runtime_put_sync(chip->dev);
+
+	return 0;
+}
+
+
+static const struct pwm_ops eswin_pwm_ops = {
+	.get_state = eswin_pwm_get_state,
+	.apply = eswin_pwm_apply,
+	.owner = THIS_MODULE,
+};
+
+static const struct of_device_id eswin_pwm_dt_ids[] = {
+	{ .compatible = "eswin,pwm-eswin", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, eswin_pwm_dt_ids);
+
+static int eswin_pwm_probe(struct platform_device *pdev)
+{
+	struct eswin_pwm *pc;
+	int ret, count;
+	struct resource *res;
+	int clk_rate;
+
+	pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL);
+	if (!pc)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+
+	pc->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pc->base))
+		return PTR_ERR(pc->base);
+
+	pc->clk = devm_clk_get(&pdev->dev, "pwm");
+	if (IS_ERR(pc->clk)) {
+		pc->clk = devm_clk_get(&pdev->dev, "pclk");
+        if (IS_ERR(pc->clk))
+			return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk),
+                    "Can't get PWM clk\n");
+	}
+
+	count = of_count_phandle_with_args(pdev->dev.of_node,
+                      "clocks", "#clock-cells");
+	if (count == 2)
+		pc->pclk = devm_clk_get(&pdev->dev, "pclk");
+	else
+		pc->pclk = pc->clk;
+
+	if (IS_ERR(pc->pclk)) {
+		ret = PTR_ERR(pc->pclk);
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Can't get APB clk: %d\n", ret);
+		return ret;
+	}
+
+	clk_rate = clk_get_rate(pc->pclk);
+	pc->clk_period_ns = DIV_ROUND_CLOSEST_ULL(NSEC_TO_SEC, clk_rate);
+
+	/* pwm reset init */
+	pc->pwm_rst = devm_reset_control_get_optional(&pdev->dev, "pwmrst");
+	if(IS_ERR_OR_NULL(pc->pwm_rst)) {
+		dev_err(&pdev->dev, "Failed to get pwmrst reset handle\n");
+		return -EFAULT;
+	}
+
+	ret = clk_prepare_enable(pc->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Can't prepare enable PWM clk: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(pc->pclk);
+	if (ret) {
+		dev_err(&pdev->dev, "Can't prepare enable APB clk: %d\n", ret);
+		goto err_clk;
+	}
+
+	/* reset pwm */
+	ret = reset_control_assert(pc->pwm_rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(pc->pwm_rst);
+	WARN_ON(0 != ret);
+
+	platform_set_drvdata(pdev, pc);
+
+	pc->chip.dev = &pdev->dev;
+	pc->chip.ops = &eswin_pwm_ops;
+	pc->chip.npwm = 3;
+
+	ret = pwmchip_add(&pc->chip);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret);
+		goto err_pclk;
+	}
+
+	return 0;
+
+err_pclk:
+    clk_disable_unprepare(pc->pclk);
+err_clk:
+    clk_disable_unprepare(pc->clk);
+
+	return ret;
+}
+
+static int eswin_pwm_remove(struct platform_device *pdev)
+{
+	struct eswin_pwm *pc = platform_get_drvdata(pdev);
+
+	pwmchip_remove(&pc->chip);
+
+	clk_disable_unprepare(pc->pclk);
+	clk_disable_unprepare(pc->clk);
+
+	return 0;
+}
+
+static struct platform_driver eswin_pwm_driver = {
+	.driver = {
+		.name = "eswin-pwm",
+		.of_match_table = eswin_pwm_dt_ids,
+	},
+	.probe = eswin_pwm_probe,
+	.remove = eswin_pwm_remove,
+};
+module_platform_driver(eswin_pwm_driver);
+
+MODULE_DESCRIPTION("eswin SoC PWM driver");
+MODULE_AUTHOR("zhangchunyun@eswincomputing.com");
+MODULE_LICENSE("GPL");
+
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index ccd59ddd7610..f84d0a674eb7 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -66,6 +66,12 @@ config RESET_BRCMSTB_RESCAL
 	  This enables the RESCAL reset controller for SATA, PCIe0, or PCIe1 on
 	  BCM7216.
 
+config RESET_ESWIN_EIC7700
+	tristate "Eswin EIC7700 Reset Driver"
+	default n
+	help
+	  This enables the reset controller driver for Eswin EIC7700 SoCs.
+
 config RESET_HSDK
 	bool "Synopsys HSDK Reset Driver"
 	depends on HAS_IOMEM
diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile
index 8270da8a4baa..d4d4dfe96c65 100644
--- a/drivers/reset/Makefile
+++ b/drivers/reset/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_RESET_BCM6345) += reset-bcm6345.o
 obj-$(CONFIG_RESET_BERLIN) += reset-berlin.o
 obj-$(CONFIG_RESET_BRCMSTB) += reset-brcmstb.o
 obj-$(CONFIG_RESET_BRCMSTB_RESCAL) += reset-brcmstb-rescal.o
+obj-$(CONFIG_RESET_ESWIN_EIC7700) += reset-eswin.o
 obj-$(CONFIG_RESET_HSDK) += reset-hsdk.o
 obj-$(CONFIG_RESET_IMX7) += reset-imx7.o
 obj-$(CONFIG_RESET_INTEL_GW) += reset-intel-gw.o
diff --git a/drivers/reset/reset-eswin.c b/drivers/reset/reset-eswin.c
new file mode 100755
index 000000000000..8bd01ac89203
--- /dev/null
+++ b/drivers/reset/reset-eswin.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Reset Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+
+#define SYSCRG_CLEAR_BOOT_INFO_OFFSET    (0x30C)
+#define CLEAR_BOOT_FLAG_BIT              BIT_ULL(0)
+#define SYSCRG_RESET_OFFSET              (0x400)
+
+/**
+ * struct eswin_reset_data - reset controller information structure
+ * @rcdev: reset controller entity
+ * @dev: reset controller device pointer
+ * @idr: idr structure for mapping ids to reset control structures
+ */
+struct eswin_reset_data {
+	struct reset_controller_dev rcdev;
+	struct device *dev;
+	struct idr idr;
+	struct regmap *regmap;
+};
+
+/**
+ * struct eswin_reset_control - reset control structure
+ * @dev_id: SoC-specific device identifier
+ * @reset_bit: reset mask to use for toggling reset
+ */
+struct eswin_reset_control {
+	u32 dev_id;
+	u32 reset_bit;
+};
+
+#define to_eswin_reset_data(p)	\
+	container_of((p), struct eswin_reset_data, rcdev)
+
+/**
+ * eswin_reset_set() - program a device's reset
+ * @rcdev: reset controller entity
+ * @id: ID of the reset to toggle
+ * @assert: boolean flag to indicate assert or deassert
+ *
+ * This is a common internal function used to assert or deassert a device's
+ * reset by clear and set the reset bit. The device's reset is asserted if the
+ * @assert argument is true, or deasserted if @assert argument is false.
+ *
+ * Return: 0 for successful request, else a corresponding error value
+ */
+static int eswin_reset_set(struct reset_controller_dev *rcdev,
+			    unsigned long id, bool assert)
+{
+	struct eswin_reset_data *data = to_eswin_reset_data(rcdev);
+	struct eswin_reset_control *control;
+	int ret;
+
+	control = idr_find(&data->idr, id);
+
+	dev_dbg(rcdev->dev, "dev_id 0x%x reset_bit 0x%x assert 0x%x\r\n",
+		control->dev_id, control->reset_bit, assert);
+
+	if (!control)
+		return -EINVAL;
+
+	if (assert) {
+		ret = regmap_clear_bits(data->regmap, SYSCRG_RESET_OFFSET + control->dev_id * sizeof(u32),
+			control->reset_bit);
+	} else {
+		ret = regmap_set_bits(data->regmap, SYSCRG_RESET_OFFSET + control->dev_id * sizeof(u32),
+			control->reset_bit);
+	}
+
+	return ret;
+}
+
+static int eswin_reset_reset(struct reset_controller_dev *rcdev,
+			      unsigned long id)
+{
+	int ret;
+
+	ret = eswin_reset_set(rcdev, id, true);
+	if (0 != ret) {
+		return ret;
+	}
+	usleep_range(10, 15);
+	ret = eswin_reset_set(rcdev, id, false);
+	if (0 != ret) {
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eswin_reset_assert(struct reset_controller_dev *rcdev,
+			      unsigned long id)
+{
+	return eswin_reset_set(rcdev, id, true);
+}
+
+static int eswin_reset_deassert(struct reset_controller_dev *rcdev,
+				unsigned long id)
+{
+	return eswin_reset_set(rcdev, id, false);
+}
+
+static const struct reset_control_ops eswin_reset_ops = {
+	.reset		= eswin_reset_reset,
+	.assert		= eswin_reset_assert,
+	.deassert	= eswin_reset_deassert,
+};
+
+static int eswin_reset_of_xlate_lookup_id(int id, void *p, void *data)
+{
+	struct of_phandle_args *reset_spec = (struct of_phandle_args *)data;
+	struct eswin_reset_control *slot_control = (struct eswin_reset_control *)p;
+
+	if (reset_spec->args[0] == slot_control->dev_id
+		&& reset_spec->args[1] == slot_control->reset_bit) {
+			return id;
+	} else {
+		return 0;
+	}
+}
+
+/**
+ * eswin_reset_of_xlate() - translate a set of OF arguments to a reset ID
+ * @rcdev: reset controller entity
+ * @reset_spec: OF reset argument specifier
+ *
+ * This function performs the translation of the reset argument specifier
+ * values defined in a reset consumer device node. The function allocates a
+ * reset control structure for that device reset, and will be used by the
+ * driver for performing any reset functions on that reset. An idr structure
+ * is allocated and used to map to the reset control structure. This idr
+ * is used by the driver to do reset lookups.
+ *
+ * Return: 0 for successful request, else a corresponding error value
+ */
+static int eswin_reset_of_xlate(struct reset_controller_dev *rcdev,
+				 const struct of_phandle_args *reset_spec)
+{
+	struct eswin_reset_data *data = to_eswin_reset_data(rcdev);
+	struct eswin_reset_control *control;
+	int ret;
+
+	if (WARN_ON(reset_spec->args_count != rcdev->of_reset_n_cells))
+		return -EINVAL;
+
+	ret = idr_for_each(&data->idr, eswin_reset_of_xlate_lookup_id, (void *)reset_spec);
+	if (0 != ret) {
+		return ret;
+	}
+
+	control = devm_kzalloc(data->dev, sizeof(*control), GFP_KERNEL);
+	if (!control)
+		return -ENOMEM;
+
+	control->dev_id = reset_spec->args[0];
+	control->reset_bit = reset_spec->args[1];
+
+	return idr_alloc(&data->idr, control, 0, 0, GFP_KERNEL);
+}
+
+static const struct of_device_id eswin_reset_dt_ids[] = {
+	 { .compatible = "eswin,eic7700-reset", },
+	 { /* sentinel */ },
+};
+
+static int eswin_reset_probe(struct platform_device *pdev)
+{
+	struct eswin_reset_data *data;
+	struct device *parent;
+
+	parent = pdev->dev.parent;
+	if (!parent) {
+		dev_err(&pdev->dev, "no parent\n");
+		return -ENODEV;
+	}
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->regmap = syscon_node_to_regmap(parent->of_node);
+	if (IS_ERR(data->regmap)) {
+		dev_err(&pdev->dev, "failed to get parent regmap\n");
+		return PTR_ERR(data->regmap);
+	}
+
+	platform_set_drvdata(pdev, data);
+
+	data->rcdev.owner = THIS_MODULE;
+	data->rcdev.ops = &eswin_reset_ops;
+	data->rcdev.of_node = pdev->dev.of_node;
+	data->rcdev.of_reset_n_cells = 2;
+	data->rcdev.of_xlate = eswin_reset_of_xlate;
+	data->rcdev.dev = &pdev->dev;
+	data->dev = &pdev->dev;
+	idr_init(&data->idr);
+
+	/*clear boot flag so u84 and scpu could be reseted by software*/
+	regmap_set_bits(data->regmap, SYSCRG_CLEAR_BOOT_INFO_OFFSET, CLEAR_BOOT_FLAG_BIT);
+	msleep(50);
+
+	platform_set_drvdata(pdev, data);
+
+	return devm_reset_controller_register(&pdev->dev, &data->rcdev);
+}
+
+static int eswin_reset_remove(struct platform_device *pdev)
+{
+	struct eswin_reset_data *data = platform_get_drvdata(pdev);
+
+	idr_destroy(&data->idr);
+
+	return 0;
+}
+
+static struct platform_driver eswin_reset_driver = {
+	.probe	= eswin_reset_probe,
+	.remove = eswin_reset_remove,
+	.driver = {
+		.name		= "eswin-reset",
+		.of_match_table	= eswin_reset_dt_ids,
+	},
+};
+
+static int __init eic7700_reset_init(void)
+{
+	return platform_driver_register(&eswin_reset_driver);
+}
+arch_initcall(eic7700_reset_init);
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index 92f46a6312c2..f17ef9c9d509 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -306,6 +306,15 @@ config RTC_DRV_DS1672
 	  This driver can also be built as a module. If so, the module
 	  will be called rtc-ds1672.
 
+config RTC_DRV_ESWIN
+	tristate "eswin EIC7700 RTC"
+	help
+	  If you say yes here you get support for the eswin EIC7700 real time
+	  clock.
+
+	  This driver can also be built as a module, if so, the module
+	  will be called "rtc-eswin".
+
 config RTC_DRV_HYM8563
 	tristate "Haoyu Microelectronics HYM8563"
 	depends on OF
diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile
index fd209883ee2e..9a53a6b92990 100644
--- a/drivers/rtc/Makefile
+++ b/drivers/rtc/Makefile
@@ -65,6 +65,7 @@ obj-$(CONFIG_RTC_DRV_DS3232)	+= rtc-ds3232.o
 obj-$(CONFIG_RTC_DRV_EFI)	+= rtc-efi.o
 obj-$(CONFIG_RTC_DRV_EM3027)	+= rtc-em3027.o
 obj-$(CONFIG_RTC_DRV_EP93XX)	+= rtc-ep93xx.o
+obj-$(CONFIG_RTC_DRV_ESWIN)	+= rtc-eswin.o
 obj-$(CONFIG_RTC_DRV_FM3130)	+= rtc-fm3130.o
 obj-$(CONFIG_RTC_DRV_FSL_FTM_ALARM)	+= rtc-fsl-ftm-alarm.o
 obj-$(CONFIG_RTC_DRV_FTRTC010)	+= rtc-ftrtc010.o
diff --git a/drivers/rtc/rtc-eswin.c b/drivers/rtc/rtc-eswin.c
new file mode 100644
index 000000000000..e83f43baa60d
--- /dev/null
+++ b/drivers/rtc/rtc-eswin.c
@@ -0,0 +1,322 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN rtc driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Author: zhangpengcheng@eswincomputing.com
+ */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/rtc.h>
+#include <linux/reset.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+
+#define RTC_INT_TO_U84  0xffff9fff
+/* RTC CSR Registers */
+#define RTC_CCVR        0x00
+#define RTC_CMR         0x04
+#define RTC_CLR         0x08
+#define RTC_CCR         0x0C
+#define RTC_CCR_IE      BIT(0)
+#define RTC_CCR_MASK    BIT(1)
+#define RTC_CCR_EN      BIT(2)
+#define RTC_CCR_WEN     BIT(3)
+#define RTC_CCR_PEN     BIT(4)
+#define RTC_STAT        0x10
+#define RTC_STAT_BIT    BIT(0)
+#define RTC_RSTAT       0x14
+#define RTC_EOI         0x18
+#define RTC_VER         0x1C
+#define RTC_CPSR        0x20
+#define RTC_CPCVR       0x24
+
+struct eswin_rtc_dev {
+	struct rtc_device *rtc;
+	struct device *dev;
+	unsigned long alarm_time;
+	void __iomem *csr_base;
+	struct clk *clk;
+	unsigned int irq_wake;
+	struct reset_control *rst_rtc;
+};
+
+static int eswin_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct eswin_rtc_dev *pdata = dev_get_drvdata(dev);
+	rtc_time64_to_tm(readl(pdata->csr_base + RTC_CCVR), tm);
+	return rtc_valid_tm(tm);
+}
+
+static int eswin_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct eswin_rtc_dev *pdata = dev_get_drvdata(dev);
+	unsigned long tr;
+
+	tr = rtc_tm_to_time64(tm);
+	writel(tr, pdata->csr_base + RTC_CLR);
+	readl(pdata->csr_base + RTC_CLR); /* Force a barrier */
+
+	return 0;
+}
+
+static int eswin_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct eswin_rtc_dev *pdata = dev_get_drvdata(dev);
+	rtc_time64_to_tm(pdata->alarm_time, &alrm->time);
+	alrm->enabled = readl(pdata->csr_base + RTC_CCR) & RTC_CCR_IE;
+
+	return 0;
+}
+
+static int eswin_rtc_alarm_irq_enable(struct device *dev, u32 enabled)
+{
+	struct eswin_rtc_dev *pdata = dev_get_drvdata(dev);
+	u32 ccr;
+
+	ccr = readl(pdata->csr_base + RTC_CCR);
+	if (enabled) {
+		ccr &= ~RTC_CCR_MASK;
+		ccr |= RTC_CCR_IE;
+	} else {
+		ccr &= ~RTC_CCR_IE;
+		ccr |= RTC_CCR_MASK;
+	}
+	writel(ccr, pdata->csr_base + RTC_CCR);
+
+	return 0;
+}
+
+static int eswin_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct eswin_rtc_dev *pdata = dev_get_drvdata(dev);
+	unsigned long rtc_time;
+	unsigned long alarm_time;
+	rtc_time = readl(pdata->csr_base + RTC_CCVR);
+	alarm_time = rtc_tm_to_time64(&alrm->time);
+
+	pdata->alarm_time = alarm_time;
+	writel((u32) pdata->alarm_time, pdata->csr_base + RTC_CMR);
+
+	eswin_rtc_alarm_irq_enable(dev, alrm->enabled);
+
+	return 0;
+}
+
+static const struct rtc_class_ops eswin_rtc_ops = {
+	.read_time  = eswin_rtc_read_time,
+	.set_time   = eswin_rtc_set_time,
+	.read_alarm = eswin_rtc_read_alarm,
+	.set_alarm  = eswin_rtc_set_alarm,
+	.alarm_irq_enable = eswin_rtc_alarm_irq_enable,
+};
+
+static irqreturn_t eswin_rtc_interrupt(int irq, void *id)
+{
+	struct eswin_rtc_dev *pdata = (struct eswin_rtc_dev *) id;
+	/* Check if interrupt asserted */
+	if (!(readl(pdata->csr_base + RTC_STAT) & RTC_STAT_BIT))
+		return IRQ_NONE;
+
+	/* Clear interrupt */
+	readl(pdata->csr_base + RTC_EOI);
+
+	rtc_update_irq(pdata->rtc, 1, RTC_IRQF | RTC_AF);
+
+	return IRQ_HANDLED;
+}
+
+static int eswin_rtc_probe(struct platform_device *pdev)
+{
+	struct eswin_rtc_dev *pdata;
+	struct resource *res;
+	int ret;
+	int irq;
+	unsigned int reg_val;
+	unsigned int int_off;
+	unsigned int clk_freq;
+	struct regmap *regmap;
+	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, pdata);
+	pdata->dev = &pdev->dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pdata->csr_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pdata->csr_base))
+		return PTR_ERR(pdata->csr_base);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "No IRQ resource\n");
+		return irq;
+	}
+	ret = devm_request_irq(&pdev->dev, irq, eswin_rtc_interrupt, 0,
+                   dev_name(&pdev->dev), pdata);
+	if (ret) {
+		dev_err(&pdev->dev, "Could not request IRQ\n");
+		return ret;
+	}
+
+	/* update RTC interrupt to u84 */
+	regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "eswin,syscfg");
+	if (IS_ERR(regmap)) {
+		dev_err(&pdev->dev, "No syscfg phandle specified\n");
+		return PTR_ERR(regmap);
+	}
+
+	ret = of_property_read_u32_index(pdev->dev.of_node, "eswin,syscfg", 1, &int_off);
+	if (ret) {
+		dev_err(&pdev->dev, "No rtc interrupt offset found\n");
+		return -1;
+	}
+	regmap_read(regmap, int_off, &reg_val);
+	reg_val &= (RTC_INT_TO_U84);
+	regmap_write(regmap, int_off, reg_val);
+
+	ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &clk_freq);
+	if (ret) {
+		dev_err(&pdev->dev, "No rtc clock-frequency found\n");
+	}
+	/* rtc reset init*/
+	pdata->rst_rtc = devm_reset_control_get_optional(&pdev->dev, "rtcrst");
+	if (IS_ERR_OR_NULL(pdata->rst_rtc)) {
+		dev_err(&pdev->dev, "Failed to get rtcrst reset handle\n");
+		return -EFAULT;
+	}
+
+	/* get RTC clock */
+	pdata->clk = devm_clk_get(&pdev->dev, "rtcclk");
+	if (IS_ERR(pdata->clk)) {
+		dev_err(&pdev->dev, "Couldn't get the clock for RTC\n");
+		return -ENODEV;
+	}
+	/* Enable the clock */
+	clk_prepare_enable(pdata->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable RTC clock: %d\n", ret);
+		return -ENODEV;
+	}
+	/* reset rtc */
+	ret = reset_control_assert(pdata->rst_rtc);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(pdata->rst_rtc);
+	WARN_ON(0 != ret);
+
+	/* Turn on the clock and the crystal */
+	reg_val = readl(pdata->csr_base + RTC_CCR);
+	writel(RTC_CCR_EN | reg_val, pdata->csr_base + RTC_CCR);
+
+	/* Turn on the prescaler and set the value */
+	writel(clk_freq, pdata->csr_base + RTC_CPSR);
+	reg_val = readl(pdata->csr_base + RTC_CCR);
+	writel(RTC_CCR_PEN | reg_val, pdata->csr_base + RTC_CCR);
+
+	device_init_wakeup(&pdev->dev, 1);
+
+	pdata->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
+                     &eswin_rtc_ops, THIS_MODULE);
+	if (IS_ERR(pdata->rtc)) {
+		clk_disable_unprepare(pdata->clk);
+		return PTR_ERR(pdata->rtc);
+	}
+
+	return 0;
+}
+
+static int eswin_rtc_remove(struct platform_device *pdev)
+{
+	struct eswin_rtc_dev *pdata = platform_get_drvdata(pdev);
+
+	eswin_rtc_alarm_irq_enable(&pdev->dev, 0);
+	device_init_wakeup(&pdev->dev, 0);
+	clk_disable_unprepare(pdata->clk);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int eswin_rtc_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct eswin_rtc_dev *pdata = platform_get_drvdata(pdev);
+	int irq;
+
+	irq = platform_get_irq(pdev, 0);
+	if (device_may_wakeup(&pdev->dev)) {
+		if (!enable_irq_wake(irq))
+			pdata->irq_wake = 1;
+	} else {
+		eswin_rtc_alarm_irq_enable(dev, 0);
+		clk_disable(pdata->clk);
+	}
+
+	return 0;
+}
+
+static int eswin_rtc_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct eswin_rtc_dev *pdata = platform_get_drvdata(pdev);
+	int irq;
+
+	irq = platform_get_irq(pdev, 0);
+	if (device_may_wakeup(&pdev->dev)) {
+		if (pdata->irq_wake) {
+			disable_irq_wake(irq);
+			pdata->irq_wake = 0;
+		}
+	} else {
+		clk_enable(pdata->clk);
+		eswin_rtc_alarm_irq_enable(dev, 1);
+	}
+
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(eswin_rtc_pm_ops, eswin_rtc_suspend, eswin_rtc_resume);
+
+#ifdef CONFIG_OF
+static const struct of_device_id eswin_rtc_of_match[] = {
+	{.compatible = "eswin,eic7700-rtc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, eswin_rtc_of_match);
+#endif
+
+static struct platform_driver eswin_rtc_driver = {
+	.probe      = eswin_rtc_probe,
+	.remove     = eswin_rtc_remove,
+	.driver     = {
+		.name   = "eswin-rtc",
+		.pm = &eswin_rtc_pm_ops,
+		.of_match_table = of_match_ptr(eswin_rtc_of_match),
+	},
+};
+
+module_platform_driver(eswin_rtc_driver);
+
+MODULE_DESCRIPTION("eswin eic7700 RTC driver");
+MODULE_AUTHOR("zhangpengcheng@eswin.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/sifive/Kconfig b/drivers/soc/sifive/Kconfig
index e86870be34c9..54ef82af67af 100644
--- a/drivers/soc/sifive/Kconfig
+++ b/drivers/soc/sifive/Kconfig
@@ -4,6 +4,7 @@ if SOC_SIFIVE || SOC_STARFIVE
 
 config SIFIVE_CCACHE
 	bool "Sifive Composable Cache controller"
+	select RISCV_NONSTANDARD_CACHE_OPS
 	help
 	  Support for the composable cache controller on SiFive platforms.
 
diff --git a/drivers/soc/sifive/sifive_ccache.c b/drivers/soc/sifive/sifive_ccache.c
index 3684f5b40a80..928ab5dcd3c3 100644
--- a/drivers/soc/sifive/sifive_ccache.c
+++ b/drivers/soc/sifive/sifive_ccache.c
@@ -8,13 +8,16 @@
 
 #define pr_fmt(fmt) "CCACHE: " fmt
 
+#include <linux/align.h>
 #include <linux/debugfs.h>
 #include <linux/interrupt.h>
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
 #include <linux/device.h>
 #include <linux/bitfield.h>
+#include <asm/cacheflush.h>
 #include <asm/cacheinfo.h>
+#include <asm/dma-noncoherent.h>
 #include <soc/sifive/sifive_ccache.h>
 
 #define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100
@@ -44,6 +47,10 @@
 
 #define SIFIVE_CCACHE_MAX_ECCINTR 4
 
+#define SIFIVE_CCACHE_FLUSH64 0x200
+#define SIFIVE_CCACHE_FLUSH32 0x240
+#define SIFIVE_CCACHE_LINE_SIZE 64
+
 static void __iomem *ccache_base;
 static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];
 static struct riscv_cacheinfo_ops ccache_cache_ops;
@@ -56,6 +63,10 @@ enum {
 	DIR_UNCORR,
 };
 
+enum {
+	QUIRK_NONSTANDARD_CACHE_OPS = BIT(0),
+};
+
 #ifdef CONFIG_DEBUG_FS
 static struct dentry *sifive_test;
 
@@ -107,11 +118,41 @@ static const struct of_device_id sifive_ccache_ids[] = {
 	{ .compatible = "sifive,fu540-c000-ccache" },
 	{ .compatible = "sifive,fu740-c000-ccache" },
 	{ .compatible = "sifive,ccache0" },
+	{ .compatible = "sifive,eic7700",
+	  .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS) },
 	{ /* end of table */ }
 };
 
 static ATOMIC_NOTIFIER_HEAD(ccache_err_chain);
 
+#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
+static void ccache_flush_range(phys_addr_t start, size_t len)
+{
+	phys_addr_t end = start + len;
+	phys_addr_t line;
+
+	if (!len)
+	    return;
+
+	mb();
+	for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end;
+		line += SIFIVE_CCACHE_LINE_SIZE) {
+#ifdef CONFIG_32BIT
+		writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32);
+#else
+		writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64);
+#endif
+		mb();
+	}
+}
+
+static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = {
+	.wback = &ccache_flush_range,
+	.inv = &ccache_flush_range,
+	.wback_inv = &ccache_flush_range,
+};
+#endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */
+
 int register_sifive_ccache_error_notifier(struct notifier_block *nb)
 {
 	return atomic_notifier_chain_register(&ccache_err_chain, nb);
@@ -210,11 +251,18 @@ static int __init sifive_ccache_init(void)
 	struct device_node *np;
 	struct resource res;
 	int i, rc, intr_num;
+	const struct of_device_id *match;
+	unsigned long quirks;
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	unsigned int config, ways;
+#endif
 
-	np = of_find_matching_node(NULL, sifive_ccache_ids);
+	np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match);
 	if (!np)
 		return -ENODEV;
 
+	quirks = (uintptr_t)match->data;
+
 	if (of_address_to_resource(np, 0, &res)) {
 		rc = -ENODEV;
 		goto err_node_put;
@@ -249,6 +297,20 @@ static int __init sifive_ccache_init(void)
 	}
 	of_node_put(np);
 
+#ifdef CONFIG_SOC_SIFIVE_EIC7700
+	config = readl(ccache_base + SIFIVE_CCACHE_CONFIG);
+	ways = (config >> 8) & 0xff;
+	writel(ways-1, ccache_base + SIFIVE_CCACHE_WAYENABLE);
+#endif
+
+#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
+	if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) {
+			riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE;
+			riscv_noncoherent_supported();
+			riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops);
+	}
+#endif
+
 	ccache_config_read();
 
 	ccache_cache_ops.get_priv_group = ccache_get_priv_group;
@@ -269,4 +331,4 @@ static int __init sifive_ccache_init(void)
 	return rc;
 }
 
-device_initcall(sifive_ccache_init);
+arch_initcall(sifive_ccache_init);
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 3ce0fd5df8e9..07e874de0ebf 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -360,6 +360,12 @@ config SPI_EP93XX
 	  This enables using the Cirrus EP93xx SPI controller in master
 	  mode.
 
+config SPI_ESWIN_BOOTSPI
+	tristate "Eswin Computing Boot SPI controller"
+	help
+	  This enables using the Eswin Computing Boot SPI controller in master
+	  mode.
+
 config SPI_FALCON
 	bool "Falcon SPI controller support"
 	depends on SOC_FALCON
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index 6af54842b9fa..0dc6ae53485b 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -50,6 +50,7 @@ obj-$(CONFIG_SPI_DW_BT1)		+= spi-dw-bt1.o
 obj-$(CONFIG_SPI_DW_MMIO)		+= spi-dw-mmio.o
 obj-$(CONFIG_SPI_DW_PCI)		+= spi-dw-pci.o
 obj-$(CONFIG_SPI_EP93XX)		+= spi-ep93xx.o
+obj-$(CONFIG_SPI_ESWIN_BOOTSPI)	+= spi-eswin-bootspi.o
 obj-$(CONFIG_SPI_FALCON)		+= spi-falcon.o
 obj-$(CONFIG_SPI_FSI)			+= spi-fsi.o
 obj-$(CONFIG_SPI_FSL_CPM)		+= spi-fsl-cpm.o
diff --git a/drivers/spi/spi-eswin-bootspi.c b/drivers/spi/spi-eswin-bootspi.c
new file mode 100644
index 000000000000..c8fbc1b3fc91
--- /dev/null
+++ b/drivers/spi/spi-eswin-bootspi.c
@@ -0,0 +1,658 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN BootSpi Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/bitfield.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+#include <linux/mtd/spi-nor.h>
+
+/* Register offsets */
+#define ES_SPI_CSR_00			0x00	/*WRITE_STATUS_REG_TIME*/
+#define ES_SPI_CSR_01			0x04	/*SPI_BUS_MODE*/
+#define ES_SPI_CSR_02			0x08	/*ERASE_COUNTER_TAP*/
+#define ES_SPI_CSR_03			0x0c	/*DMA_EN_HCLK_STATUS*/
+#define ES_SPI_CSR_04			0x10	/*FAST_READ_CONTROL*/
+#define ES_SPI_CSR_05			0x14	/*SPI_FLASH_WR_NUM*/
+#define ES_SPI_CSR_06			0x18	/*SPI_FLASH_COMMAND*/
+#define ES_SPI_CSR_07			0x1c	/*INTERRUPT_CONTROL*/
+#define ES_SPI_CSR_08			0x20	/*DMA_REQUEST_TAP*/
+#define ES_SPI_CSR_09			0x24	/*SPI_FLASH_WR_ADDRESS*/
+#define ES_SPI_CSR_10			0x28	/*PAGE_PROGRAM_TIME*/
+#define ES_SPI_CSR_11			0x2c	/*SECTOR_ERASE_TIME*/
+#define ES_SPI_CSR_12			0x30	/*SMALL_BLOCK_ERASE_TIME*/
+#define ES_SPI_CSR_13			0x34	/*LARGE_BLOCK_ERASE_TIME*/
+#define ES_SPI_CSR_14			0x38	/*CHIP_ERASE_TIME*/
+#define ES_SPI_CSR_15			0x3c	/*CHIP_DESELECT_TIME*/
+#define ES_SPI_CSR_16			0x40	/*POWER_DOWN_TIME*/
+
+#define ES_SYSCSR_SPIMODECFG			0x340
+
+#define ES_CONCSR_SPI_INTSEL			0x3c0
+
+#define SPI_COMMAND_VALID				0x01
+#define SPI_COMMAND_MOVE_VALUE			0x00
+#define SPI_COMMAND_CODE_FIELD_POSITION 0X06
+#define SPI_COMMAND_MOVE_FIELD_POSITION 0X05
+#define SPI_COMMAND_TYPE_FIELD_POSITION 0X01
+
+/* Bit fields in CTRLR0 */
+/*
+ * Only present when SSI_MAX_XFER_SIZE=16. This is the default, and the only
+ * option before version 3.23a.
+ */
+#define SPI_INTSEL_MASK			GENMASK(11, 10)
+#define INT_ROUTED_U84			0x0
+#define INT_ROUTED_LPCPU		0x1
+#define INT_ROUTED_SCPU			0x3u
+
+#define RX_TIMEOUT			5000		/* timeout in ms */
+
+#define SPI_COMMAND_INIT_VALUE       0XFFFFC000
+#define FLASH_PAGE_SIZE      0x100
+
+typedef enum {
+	SPI_FLASH_WR_BYTE  = 1,
+	SPI_FLASH_WR_2BYTE = 2,
+	SPI_FLASH_WR_WORD  = 4,
+} SPI_FLASH_WR_NUM_T;
+
+typedef enum {
+	SPI_FAST_READ_DEFAULT = 0,
+	SPI_FAST_READ_ENABLE  = 3 /*WHEN SPI QUAD0 OR DUAL MODE*/
+} SPI_FAST_READ_CTL_T;
+
+typedef enum { STANDARD_SPI = 0, DUAL_SPI, QUAD_SPI } SPI_BUS_MODE_T;
+
+typedef enum {
+	SPIC_CMD_TYPE_SPI_PROGRAM = 0,
+	SPIC_CMD_TYPE_WRITE_STATUS_REGISTER,
+	SPIC_CMD_TYPE_READ_STATUS_REGISTER,
+	SPIC_CMD_TYPE_SECTOR_ERASE,
+	SPIC_CMD_TYPE_BLOCK_ERASE_TYPE1,
+	SPIC_CMD_TYPE_BLOCK_ERASE_TYPE2,
+	SPIC_CMD_TYPE_CHIP_ERASE,
+	SPIC_CMD_TYPE_POWER_DOWN,
+	SPIC_CMD_TYPE_RELEASE_POWER_DOWM,
+	SPIC_CMD_TYPE_ENTER_OR_EXIT_32BIT_MODE,
+	SPIC_CMD_TYPE_READ_SECURITY_REG,
+	SPIC_CMD_TYPE_ERASE_SECURITY_REG,
+	SPIC_CMD_TYPE_WRITE_SECURITY_REG,
+	SPIC_CMD_TYPE_READ_DATA,
+	SPIC_CMD_TYPE_READ_MANUFACTURED_ID,
+	SPIC_CMD_TYPE_READ_JEDEC_ID
+} SPI_FLASH_COMMAND_TYPE_T;
+
+#define SPIC_CMD_CODE_POWER_DOWN              0xb9
+#define SPIC_CMD_CODE_RELEASE_POWER_DOWN       0xab
+#define SPIC_CMD_CODE_ENABLE_RESET            0x66
+#define SPIC_CMD_CODE_RESET                   0x99
+
+struct es_spi_priv {
+	struct clk *cfg_clk;
+	struct clk *clk;
+	struct reset_control *rstc;
+	struct gpio_desc *cs_gpio;	/* External chip-select gpio */
+
+	void __iomem *regs;
+	void __iomem *sys_regs;
+	void __iomem *flash_base;
+	unsigned int freq;		/* Default frequency */
+	unsigned int mode;
+
+	const void *tx;
+	u32 opcode;
+	u32 cmd_type;
+	u64 addr;
+	void *rx;
+	u32 fifo_len;			/* depth of the FIFO buffer */
+	u32 max_xfer;			/* Maximum transfer size (in bits) */
+
+	int bits_per_word;
+	int len;
+	u8 cs;				/* chip select pin */
+	u8 tmode;			/* TR/TO/RO/EEPROM */
+	u8 type;			/* SPI/SSP/MicroWire */
+	struct spi_controller *master;
+	struct device *dev;
+	int irq;
+};
+
+static inline u32 eswin_bootspi_read(struct es_spi_priv *priv, u32 offset)
+{
+	return readl(priv->regs + offset);
+}
+
+static inline void eswin_bootspi_write(struct es_spi_priv *priv, u32 offset, u32 val)
+{
+	writel(val, priv->regs + offset);
+}
+
+static inline u32 eswin_bootspi_data_read(struct es_spi_priv *priv, u32 offset)
+{
+	return readl(priv->flash_base + offset);
+}
+
+static inline void eswin_bootspi_data_write(struct es_spi_priv *priv, u32 offset, u32 val)
+{
+	writel(val, priv->flash_base + offset);
+}
+
+static int eswin_bootspi_wait_over(struct es_spi_priv *priv)
+{
+	u32 val;
+	struct device *dev = priv->dev;
+
+	if (readl_poll_timeout(priv->regs + ES_SPI_CSR_06, val,
+		(!(val & 0x1)), 10, RX_TIMEOUT * 1000)) {
+			dev_err(dev, "eswin_bootspi_wait_over : timeout!!\n");
+			return -ETIMEDOUT;
+	}
+	return 0;
+}
+
+/**
+ * @brief spi read and write cfg
+ */
+static void eswin_bootspi_read_write_cfg(struct es_spi_priv *priv, u32 byte, u32 addr)
+{
+	eswin_bootspi_write(priv, ES_SPI_CSR_09, addr);
+	eswin_bootspi_write(priv, ES_SPI_CSR_05, byte);
+	eswin_bootspi_write(priv, ES_SPI_CSR_04, SPI_FAST_READ_DEFAULT);
+	eswin_bootspi_write(priv, ES_SPI_CSR_01, STANDARD_SPI);
+}
+
+/**
+ *  @brief write data from dest address to flash
+ */
+static void eswin_bootspi_send_data(struct es_spi_priv *priv,
+	const u8 *dest, u32 size)
+{
+	u32 offset = 0;
+	u8 *buff = (u8 *)dest;
+	u32 data = 0;
+	int i;
+	struct device *dev = priv->dev;
+
+	dev_dbg(dev,"wrtie spi data\n");
+	while (size >= SPI_FLASH_WR_WORD) {
+		data = (buff[0]) | (buff[1] << 8) |(buff[2] << 16) | (buff[3] << 24);
+		for (i = 0; i < 4; i++) {
+			dev_dbg(dev,"0x%x ", buff[i]);
+		}
+		dev_dbg(dev,"0x%x ", data);
+		eswin_bootspi_data_write(priv, offset, data);
+		offset = offset + SPI_FLASH_WR_WORD;
+		size = size - SPI_FLASH_WR_WORD;
+		buff = buff + SPI_FLASH_WR_WORD;
+	}
+	data = 0;
+	if (size != 0) {
+		for (i = 0; i < size; i++) {
+			data |=buff[i] << (8 * i);
+			dev_dbg(dev,"0x%x ", buff[i]);
+		}
+		dev_dbg(dev,"0x%x ", data);
+		eswin_bootspi_data_write(priv, offset, data);
+	}
+}
+
+/**
+ *  @brief Read data from flash to dest address
+ */
+static void eswin_bootspi_recv_data(struct es_spi_priv *priv, u8 *dest, u32 size)
+{
+	u32 offset = 0;
+	u8 *buff = NULL;
+	u32 data = 0xFFFFFFFF;
+	int i;
+	struct device *dev = priv->dev;
+
+	dev_dbg(dev,"read spi data\n");
+	while (size >= SPI_FLASH_WR_WORD) {
+		buff = (u8 *)dest;
+		data = eswin_bootspi_data_read(priv, offset);
+		dev_dbg(dev,"0x%x ", data);
+		for (i = 0; i < SPI_FLASH_WR_WORD; i++) {
+			*buff = (u8)(data >> (8 * i));
+			dev_dbg(dev,"0x%x ", *buff);
+			buff++;
+		}
+		dest = dest + SPI_FLASH_WR_WORD;
+		offset = offset + SPI_FLASH_WR_WORD;
+		size = size - SPI_FLASH_WR_WORD;
+	}
+	if (size != 0) {
+		buff = (u8 *)dest;
+		data = eswin_bootspi_data_read(priv, offset);
+		for (i = 0; i < size; i++) {
+			*buff = (u8)(data >> (8 * i));
+			dev_dbg(dev,"0x%x ", *buff);
+			buff++;
+		}
+	}
+	dev_dbg(dev,"\n");
+}
+
+
+/**
+ * @brief spi send command
+ */
+static void eswin_bootspi_cmd_cfg(struct es_spi_priv *priv, u32 code, u32 type)
+{
+	u32 command = eswin_bootspi_read(priv, ES_SPI_CSR_06);
+	struct device *dev = priv->dev;
+
+	command &= ~((0xFF << 6) | (0x1 << 5) | (0xF << 1) | 0x1);
+	command |= ((code << SPI_COMMAND_CODE_FIELD_POSITION) |
+		(SPI_COMMAND_MOVE_VALUE << SPI_COMMAND_MOVE_FIELD_POSITION) |
+		(type << SPI_COMMAND_TYPE_FIELD_POSITION) | SPI_COMMAND_VALID);
+
+	eswin_bootspi_write(priv, ES_SPI_CSR_06, command);
+	dev_dbg(dev, "[%s %d]: write command 0x%x, read back command 0x%x\n",
+		__func__,__LINE__, command, eswin_bootspi_read(priv, ES_SPI_CSR_06));
+}
+/**
+ * @brief  spi write flash
+ * @param [in]  offset: address of flash to be write
+ * @param [in]  wr_dest: Address of data to be sent
+ * @param [in]  size: size of flash to be write
+ */
+void eswin_bootspi_writer(struct es_spi_priv *priv)
+{
+	u32 write_size = 0, offset, cmd_code;
+	u32 cmd_type = priv->cmd_type;
+	const u8 *wr_dest = priv->tx;
+	int size = priv->len;
+
+	offset = priv->addr;
+	cmd_code = priv->opcode;
+
+	if (size == 0) {
+		// if(SPIC_CMD_TYPE_SECTOR_ERASE == cmd_type)
+		{
+			eswin_bootspi_read_write_cfg(priv, write_size, offset);
+			eswin_bootspi_cmd_cfg(priv, cmd_code, cmd_type);
+			eswin_bootspi_wait_over(priv);
+		}
+	}
+	while (size > 0) {
+		write_size = size;
+		if (write_size > FLASH_PAGE_SIZE) {
+			write_size = FLASH_PAGE_SIZE;
+		}
+		eswin_bootspi_read_write_cfg(priv, write_size, offset);
+		eswin_bootspi_send_data(priv, wr_dest, write_size);
+		eswin_bootspi_cmd_cfg(priv, cmd_code, cmd_type);
+		eswin_bootspi_wait_over(priv);
+		wr_dest += write_size;
+		offset += write_size;
+		size = size - write_size;
+	}
+}
+
+static void eswin_bootspi_reader(struct es_spi_priv *priv)
+{
+	int read_size = 0;
+	u32 offset = priv->addr;
+	u32 cmd_code = priv->opcode;
+	u32 cmd_type = priv->cmd_type;
+	u8 *mem_dest = priv->rx;
+	int size = priv->len;
+
+	while (size > 0) {
+		read_size = size;
+		if (read_size > FLASH_PAGE_SIZE) {
+			read_size = FLASH_PAGE_SIZE;
+		}
+
+		eswin_bootspi_read_write_cfg(priv, read_size, offset);
+		eswin_bootspi_cmd_cfg(priv, cmd_code, cmd_type);
+		eswin_bootspi_wait_over(priv);
+		eswin_bootspi_recv_data(priv, mem_dest, read_size);
+		mem_dest += read_size;
+		offset += read_size;
+		size = size - read_size;
+	}
+}
+
+/*
+ * We define external_cs_manage function as 'weak' as some targets
+ * (like MSCC Ocelot) don't control the external CS pin using a GPIO
+ * controller. These SoCs use specific registers to control by
+ * software the SPI pins (and especially the CS).
+ */
+
+static void external_cs_manage(struct es_spi_priv *priv, bool on)
+{
+	gpiod_set_value(priv->cs_gpio, on ? 1 : 0);
+}
+
+/* The size of ctrl1 limits data transfers to 64K */
+static int eswin_bootspi_adjust_op_size(struct spi_mem *mem,
+					struct spi_mem_op *op)
+{
+	op->data.nbytes = min(op->data.nbytes, (unsigned int)SZ_64K);
+
+	return 0;
+}
+
+/*
+ * The controller only supports Standard SPI mode, Duall mode and
+ * Quad mode. Double sanitize the ops here to avoid OOB access.
+ */
+static bool eswin_bootspi_supports_op(struct spi_mem *mem,
+				      const struct spi_mem_op *op)
+{
+	return spi_mem_default_supports_op(mem, op);
+}
+
+static int eswin_bootspi_exec_op(struct spi_mem *mem,
+				 const struct spi_mem_op *op)
+{
+	bool read = op->data.dir == SPI_MEM_DATA_IN;
+	int ret = 0;
+	struct es_spi_priv *priv = spi_master_get_devdata(mem->spi->master);
+	struct device *dev = priv->dev;
+
+	priv->addr = op->addr.val;
+	priv->opcode = op->cmd.opcode;
+
+	dev_dbg(dev, "\n[%s %d]: addr=0x%llx opcode=0x%x\n", __func__,__LINE__,
+		priv->addr, priv->opcode);
+
+	if ( priv->opcode == SPINOR_OP_WREN
+		|| priv->opcode == SPINOR_OP_WRDI)
+		return 0;
+
+	switch(priv->opcode) {
+		case SPINOR_OP_RDID:
+		case SPINOR_OP_RDSFDP:
+			priv->cmd_type = SPIC_CMD_TYPE_READ_JEDEC_ID;
+			break;
+		case SPINOR_OP_BE_4K:
+		case SPINOR_OP_BE_4K_PMC:
+			priv->opcode = SPINOR_OP_BE_4K;
+			priv->cmd_type = SPIC_CMD_TYPE_SECTOR_ERASE;
+			break;
+		case SPINOR_OP_BE_32K:
+			priv->cmd_type = SPIC_CMD_TYPE_BLOCK_ERASE_TYPE1;
+			break;
+		case SPINOR_OP_SE:
+			priv->cmd_type = SPIC_CMD_TYPE_BLOCK_ERASE_TYPE2;
+			break;
+		case SPINOR_OP_CHIP_ERASE:
+			priv->cmd_type = SPIC_CMD_TYPE_CHIP_ERASE;
+			break;
+		case SPINOR_OP_PP:
+		case SPINOR_OP_PP_1_1_4:
+		case SPINOR_OP_PP_1_4_4:
+		case SPINOR_OP_PP_1_1_8:
+		case SPINOR_OP_PP_1_8_8:
+			priv->opcode = SPINOR_OP_PP;
+			priv->cmd_type = SPIC_CMD_TYPE_SPI_PROGRAM;
+			break;
+		case SPINOR_OP_READ:
+		case SPINOR_OP_READ_FAST:
+		case SPINOR_OP_READ_1_1_2:
+		case SPINOR_OP_READ_1_2_2:
+		case SPINOR_OP_READ_1_1_4:
+		case SPINOR_OP_READ_1_4_4:
+		case SPINOR_OP_READ_1_1_8:
+		case SPINOR_OP_READ_1_8_8:
+			priv->opcode = SPINOR_OP_READ;
+			priv->cmd_type = SPIC_CMD_TYPE_READ_DATA;
+			break;
+		case SPINOR_OP_RDSR:
+		case SPINOR_OP_RDSR2:
+			priv->cmd_type = SPIC_CMD_TYPE_READ_STATUS_REGISTER;
+			break;
+		case SPINOR_OP_WRSR:
+		case SPINOR_OP_WRSR2:
+			priv->cmd_type = SPIC_CMD_TYPE_WRITE_STATUS_REGISTER;
+			break;
+		case SPIC_CMD_CODE_POWER_DOWN:
+			priv->cmd_type = SPIC_CMD_TYPE_POWER_DOWN;
+			break;
+		case SPIC_CMD_CODE_RELEASE_POWER_DOWN:
+			priv->cmd_type = SPIC_CMD_TYPE_RELEASE_POWER_DOWM;
+			break;
+		case SPIC_CMD_CODE_ENABLE_RESET:
+		case SPIC_CMD_CODE_RESET:
+			priv->cmd_type = SPIC_CMD_TYPE_SPI_PROGRAM;
+			break;
+		default:
+			dev_warn(dev, "[%s %d]: unsupport opcode = 0x%x, return sucess directly!\n",
+				__func__,__LINE__, priv->opcode);
+			return 0;
+	}
+
+	dev_dbg(dev, "[%s %d]: data direction=%d, opcode = 0x%x, cmd_type 0x%x\n",
+		__func__,__LINE__, op->data.dir, priv->opcode, priv->cmd_type);
+	external_cs_manage(priv, false);
+
+	if (read) {
+		priv->rx = op->data.buf.in;
+		priv->len = op->data.nbytes;
+		dev_dbg(dev, "[%s %d]: read len = %u\n", __func__,__LINE__, op->data.nbytes);
+		eswin_bootspi_reader(priv);
+	} else {
+		priv->tx = op->data.buf.out;
+		priv->len = op->data.nbytes;
+		/* Fill up the write fifo before starting the transfer */
+		dev_dbg(dev, "[%s %d]: write len = 0x%x  tx_addr 0x%px\n", __func__,__LINE__,
+			op->data.nbytes, priv->tx);
+		eswin_bootspi_writer(priv);
+		if (eswin_bootspi_wait_over(priv) < 0) {
+			dev_err(dev, "eswin_bootspi_wait_over ETIMEDOUT\n");
+			ret = -ETIMEDOUT;
+		}
+	}
+	external_cs_manage(priv, true);
+	dev_dbg(dev, "%u bytes xfered\n", op->data.nbytes);
+	return ret;
+}
+
+static const struct spi_controller_mem_ops eswin_bootspi_mem_ops = {
+	.adjust_op_size = eswin_bootspi_adjust_op_size,
+	.supports_op = eswin_bootspi_supports_op,
+	.exec_op = eswin_bootspi_exec_op,
+};
+
+static int eswin_bootspi_setup(struct spi_device *spi)
+{
+	struct es_spi_priv *priv = spi_master_get_devdata(spi->master);
+	struct device *dev = priv->dev;
+	int vaule = 0;
+	int ret;
+
+	ret = clk_prepare_enable(priv->cfg_clk);
+	if (ret) {
+		dev_err(dev, "could not enable cfg clock: %d\n", ret);
+		goto err_cfg_clk;
+	}
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret) {
+		dev_err(dev, "could not enable clock: %d\n", ret);
+		goto err_clk;
+	}
+	/* set rate to 50M*/
+	ret = clk_set_rate(priv->clk, 50000000);
+	if (ret) {
+		dev_err(dev, "could not enable clock: %d\n", ret);
+		goto err_clk;
+	}
+
+	reset_control_deassert(priv->rstc);
+
+	/* switch bootspi to cpu mode*/
+	vaule = readl(priv->sys_regs + ES_SYSCSR_SPIMODECFG);
+	vaule |= 0x1;
+	writel(vaule, priv->sys_regs + ES_SYSCSR_SPIMODECFG);
+
+	/* Basic HW init */
+	eswin_bootspi_write(priv, ES_SPI_CSR_08, 0x0);
+	return ret;
+
+err_clk:
+	clk_disable(priv->cfg_clk);
+err_cfg_clk:
+	return ret;
+}
+
+static int eswin_bootspi_probe(struct platform_device *pdev)
+{
+	struct es_spi_priv *priv;
+	struct spi_controller *master;
+	int ret = 0;
+	struct device *dev = &pdev->dev;
+
+	master = spi_alloc_master(&pdev->dev, sizeof(*priv));
+	if (!master)
+		return -ENOMEM;
+
+	master->mode_bits = SPI_CPOL | SPI_CPHA;
+	master->flags = SPI_MASTER_HALF_DUPLEX;
+	master->setup = eswin_bootspi_setup;
+	master->dev.of_node = pdev->dev.of_node;
+	master->bits_per_word_mask = SPI_BPW_MASK(32) | SPI_BPW_MASK(16) |
+				     SPI_BPW_MASK(8);
+	master->mem_ops = &eswin_bootspi_mem_ops;
+	master->num_chipselect = 1;
+
+	priv = spi_master_get_devdata(master);
+	priv->master = master;
+	priv->dev = &pdev->dev;
+	platform_set_drvdata(pdev, priv);
+
+	priv->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(priv->regs)) {
+		dev_err(dev, "%s %d: failed to map registers\n", __func__,__LINE__);
+		return PTR_ERR(priv->regs);
+	}
+
+	priv->sys_regs = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(priv->sys_regs)) {
+		dev_err(dev,"%s %d: failed to map sys registers\n", __func__, __LINE__);
+		return PTR_ERR(priv->sys_regs);
+	}
+
+	priv->flash_base = devm_platform_ioremap_resource(pdev, 2);
+	if (IS_ERR(priv->flash_base)) {
+		dev_err(dev,"%s %d: failed to map sys registers\n", __func__, __LINE__);
+		return PTR_ERR(priv->flash_base);
+	}
+
+	priv->cfg_clk = devm_clk_get(dev, "cfg_clk");
+	if (IS_ERR(priv->cfg_clk)) {
+		dev_err(dev, "%s %d:could not get cfg clk: %ld\n", __func__,__LINE__,
+			PTR_ERR(priv->cfg_clk));
+		return PTR_ERR(priv->cfg_clk);
+	}
+
+	priv->clk = devm_clk_get(dev, "clk");
+	if (IS_ERR(priv->clk)) {
+		dev_err(dev, "%s %d:could not get clk: %ld\n",__func__,__LINE__, PTR_ERR(priv->rstc));
+		return PTR_ERR(priv->clk);
+	}
+	priv->rstc = devm_reset_control_get_optional_exclusive(dev, "rst");
+	if (IS_ERR(priv->rstc)) {
+		dev_err(dev, "%s %d:could not get rst: %ld\n", __func__,__LINE__, PTR_ERR(priv->rstc));
+		return PTR_ERR(priv->rstc);
+	}
+
+	priv->cs_gpio = devm_gpiod_get(dev, "cs", GPIOD_OUT_LOW);
+	if (IS_ERR(priv->cs_gpio)) {
+		dev_err(dev, "%s %d: couldn't request gpio! (error %ld)\n", __func__,__LINE__,
+			PTR_ERR(priv->cs_gpio));
+		return PTR_ERR(priv->cs_gpio);
+	}
+
+	priv->max_xfer = 32;
+	dev_info(dev, "ssi_max_xfer_size=%u\n", priv->max_xfer);
+
+	/* Currently only bits_per_word == 8 supported */
+	priv->bits_per_word = 8;
+	priv->tmode = 0; /* Tx & Rx */
+
+	if (!priv->fifo_len) {
+		priv->fifo_len = 256;
+	}
+	ret = devm_spi_register_controller(dev, master);
+	if (ret)
+		goto err_put_master;
+
+	dev_info(&pdev->dev, "fifo_len %d, %s mode.\n", priv->fifo_len, priv->irq ? "irq" : "polling");
+	return 0;
+
+err_put_master:
+	spi_master_put(master);
+	return ret;
+}
+
+static int eswin_bootspi_remove(struct platform_device *pdev)
+{
+	struct es_spi_priv *priv = platform_get_drvdata(pdev);
+	struct spi_controller *master = priv->master;
+
+	spi_master_put(master);
+	return 0;
+}
+
+static const struct of_device_id eswin_bootspi_of_match[] = {
+	{ .compatible = "eswin,bootspi", .data = NULL},
+	{ /* end of table */}
+};
+MODULE_DEVICE_TABLE(of, eswin_bootspi_of_match);
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id eswin_bootspi_acpi_match[] = {
+	{"eswin,bootspi", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, eswin_bootspi_acpi_match);
+#endif
+
+static struct platform_driver eswin_bootspi_driver = {
+	.probe		= eswin_bootspi_probe,
+	.remove		= eswin_bootspi_remove,
+	.driver		= {
+		.name	= "eswin-bootspi",
+		.of_match_table = eswin_bootspi_of_match,
+#ifdef CONFIG_ACPI
+		.acpi_match_table = eswin_bootspi_acpi_match,
+#endif
+	},
+};
+module_platform_driver(eswin_bootspi_driver);
+
+MODULE_AUTHOR("Huangyifeng <huangyifeng@eswincomputing.com>");
+MODULE_DESCRIPTION("Eswin Boot SPI Controller Driver for EIC770X SoCs");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index bdc568a4ab66..cec46091a716 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -87,7 +87,7 @@ config SERIAL_EARLYCON_SEMIHOST
 
 config SERIAL_EARLYCON_RISCV_SBI
 	bool "Early console using RISC-V SBI"
-	depends on RISCV_SBI_V01
+	depends on RISCV_SBI
 	select SERIAL_CORE
 	select SERIAL_CORE_CONSOLE
 	select SERIAL_EARLYCON
diff --git a/drivers/tty/serial/earlycon-riscv-sbi.c b/drivers/tty/serial/earlycon-riscv-sbi.c
index 27afb0b74ea7..0162155f0c83 100644
--- a/drivers/tty/serial/earlycon-riscv-sbi.c
+++ b/drivers/tty/serial/earlycon-riscv-sbi.c
@@ -15,17 +15,38 @@ static void sbi_putc(struct uart_port *port, unsigned char c)
 	sbi_console_putchar(c);
 }
 
-static void sbi_console_write(struct console *con,
-			      const char *s, unsigned n)
+static void sbi_0_1_console_write(struct console *con,
+				  const char *s, unsigned int n)
 {
 	struct earlycon_device *dev = con->data;
 	uart_console_write(&dev->port, s, n, sbi_putc);
 }
 
+static void sbi_dbcn_console_write(struct console *con,
+				   const char *s, unsigned int n)
+{
+	int ret;
+
+	while (n) {
+		ret = sbi_debug_console_write(s, n);
+		if (ret < 0)
+			break;
+
+		s += ret;
+		n -= ret;
+	}
+}
+
 static int __init early_sbi_setup(struct earlycon_device *device,
 				  const char *opt)
 {
-	device->con->write = sbi_console_write;
+	if (sbi_debug_console_available)
+		device->con->write = sbi_dbcn_console_write;
+	else if (IS_ENABLED(CONFIG_RISCV_SBI_V01))
+		device->con->write = sbi_0_1_console_write;
+	else
+		return -ENODEV;
+
 	return 0;
 }
 EARLYCON_DECLARE(sbi, early_sbi_setup);
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index 0944cfae8b55..10ae308a141e 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -1407,6 +1407,7 @@ static int hub_configure(struct usb_hub *hub,
 	unsigned unit_load;
 	unsigned full_load;
 	unsigned maxchild;
+	int try_cnt = 0;
 
 	hub->buffer = kmalloc(sizeof(*hub->buffer), GFP_KERNEL);
 	if (!hub->buffer) {
@@ -1427,6 +1428,7 @@ static int hub_configure(struct usb_hub *hub,
 		goto fail;
 	}
 
+retry:
 	/* Request the entire hub descriptor.
 	 * hub->descriptor can handle USB_MAXCHILDREN ports,
 	 * but a (non-SS) hub can/will return fewer bytes here.
@@ -1446,9 +1448,15 @@ static int hub_configure(struct usb_hub *hub,
 		ret = -ENODEV;
 		goto fail;
 	} else if (hub->descriptor->bNbrPorts == 0) {
-		message = "hub doesn't have any ports!";
-		ret = -ENODEV;
-		goto fail;
+		try_cnt++;
+		if (try_cnt < 10) {
+			mdelay(10);
+			goto retry;
+		} else {
+			message = "hub doesn't have any ports!";
+			ret = -ENODEV;
+			goto fail;
+                }
 	}
 
 	/*
diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig
index 98efcbb76c88..3571543e8348 100644
--- a/drivers/usb/dwc3/Kconfig
+++ b/drivers/usb/dwc3/Kconfig
@@ -118,6 +118,15 @@ config USB_DWC3_OF_SIMPLE
 	  Currently supports Xilinx and Qualcomm DWC USB3 IP.
 	  Say 'Y' or 'M' if you have one such device.
 
+config USB_DWC3_ESWIN
+	tristate "Eswin Platforms"
+	depends on OF
+	depends on USB=y || USB=USB_DWC3
+	default USB_DWC3
+	help
+	  Support of USB2/3 functionality in Eswin platforms.
+	  say 'Y' or 'M' if you have one such device.
+
 config USB_DWC3_ST
 	tristate "STMicroelectronics Platforms"
 	depends on (ARCH_STI || COMPILE_TEST) && OF
diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile
index fe1493d4bbe5..d75a79bb453c 100644
--- a/drivers/usb/dwc3/Makefile
+++ b/drivers/usb/dwc3/Makefile
@@ -55,3 +55,4 @@ obj-$(CONFIG_USB_DWC3_QCOM)		+= dwc3-qcom.o
 obj-$(CONFIG_USB_DWC3_IMX8MP)		+= dwc3-imx8mp.o
 obj-$(CONFIG_USB_DWC3_XILINX)		+= dwc3-xilinx.o
 obj-$(CONFIG_USB_DWC3_OCTEON)		+= dwc3-octeon.o
+obj-$(CONFIG_USB_DWC3_ESWIN)		+= dwc3-eswin.o
diff --git a/drivers/usb/dwc3/dwc3-eswin.c b/drivers/usb/dwc3/dwc3-eswin.c
new file mode 100755
index 000000000000..23a8a7152ce9
--- /dev/null
+++ b/drivers/usb/dwc3/dwc3-eswin.c
@@ -0,0 +1,673 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * eswin Specific Glue layer
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Han Min <hanmin@eswincomputing.com>
+ */
+
+#include <linux/async.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <linux/extcon.h>
+#include <linux/freezer.h>
+#include <linux/iopoll.h>
+#include <linux/reset.h>
+#include <linux/usb.h>
+#include <linux/pm.h>
+#include <linux/usb/hcd.h>
+#include <linux/usb/ch9.h>
+#include <linux/extcon-provider.h>
+#include <linux/iommu.h>
+#include <linux/mfd/syscon.h>
+#include <linux/bitfield.h>
+#include <linux/eic7700-sid-cfg.h>
+#include <linux/regmap.h>
+#include <linux/gpio/consumer.h>
+#include "core.h"
+#include "io.h"
+
+#define AWSMMUSID GENMASK(31, 24) // The sid of write operation
+#define AWSMMUSSID GENMASK(23, 16) // The ssid of write operation
+#define ARSMMUSID GENMASK(15, 8) // The sid of read operation
+#define ARSMMUSSID GENMASK(7, 0) // The ssid of read operation
+
+#define HSP_USB_VBUS_FSEL 0x2a
+#define HSP_USB_MPLL_DEFAULT 0x0
+
+#define HSP_USB_BUS_FILTER_EN (0x1 << 0)
+#define HSP_USB_BUS_CLKEN_GM (0x1 << 9)
+#define HSP_USB_BUS_CLKEN_GS (0x1 << 16)
+#define HSP_USB_BUS_SW_RST (0x1 << 24)
+#define HSP_USB_BUS_CLK_EN (0x1 << 28)
+
+#define HSP_USB_AXI_LP_XM_CSYSREQ (0x1 << 0)
+#define HSP_USB_AXI_LP_XS_CSYSREQ (0x1 << 16)
+
+struct dwc3_eswin {
+	int num_clocks;
+	bool connected;
+	bool suspended;
+	bool force_mode;
+	bool is_phy_on;
+	struct device *dev;
+	struct clk **clks;
+	struct dwc3 *dwc;
+	struct extcon_dev *edev;
+	struct usb_hcd *hcd;
+	struct notifier_block device_nb;
+	struct notifier_block host_nb;
+	struct work_struct otg_work;
+	struct mutex lock;
+	struct reset_control *vaux_rst;
+	struct device *child_dev;
+	enum usb_role new_usb_role;
+};
+
+static ssize_t dwc3_mode_show(struct device *device,
+			      struct device_attribute *attr, char *buf)
+{
+	struct dwc3_eswin *eswin = dev_get_drvdata(device);
+	struct dwc3 *dwc = eswin->dwc;
+	int ret;
+
+	switch (dwc->current_dr_role) {
+	case USB_DR_MODE_HOST:
+		ret = sprintf(buf, "host\n");
+		break;
+	case USB_DR_MODE_PERIPHERAL:
+		ret = sprintf(buf, "peripheral\n");
+		break;
+	case USB_DR_MODE_OTG:
+		ret = sprintf(buf, "otg\n");
+		break;
+	default:
+		ret = sprintf(buf, "UNKNOWN\n");
+	}
+
+	return ret;
+}
+
+static ssize_t dwc3_mode_store(struct device *device,
+			       struct device_attribute *attr,
+			       const char *buf, size_t count)
+{
+	struct dwc3_eswin *eswin = dev_get_drvdata(device);
+	struct dwc3 *dwc = eswin->dwc;
+	enum usb_role new_role;
+	struct usb_role_switch *role_sw = dwc->role_sw;
+
+	if (!strncmp(buf, "1", 1) || !strncmp(buf, "host", 4)) {
+		new_role = USB_ROLE_HOST;
+	} else if (!strncmp(buf, "0", 1) || !strncmp(buf, "peripheral", 10)) {
+		new_role = USB_ROLE_DEVICE;
+	} else {
+		dev_info(eswin->dev, "illegal dr_mode\n");
+		return count;
+	}
+	eswin->force_mode = true;
+
+	mutex_lock(&eswin->lock);
+	usb_role_switch_set_role(role_sw, new_role);
+	mutex_unlock(&eswin->lock);
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(dwc3_mode);
+
+static struct attribute *dwc3_eswin_attrs[] = {
+	&dev_attr_dwc3_mode.attr,
+	NULL,
+};
+
+static struct attribute_group dwc3_eswin_attr_group = {
+	.name = NULL, /* we want them in the same directory */
+	.attrs = dwc3_eswin_attrs,
+};
+
+static int dwc3_eswin_device_notifier(struct notifier_block *nb,
+					 unsigned long event, void *ptr)
+{
+	struct dwc3_eswin *eswin =
+		container_of(nb, struct dwc3_eswin, device_nb);
+
+	mutex_lock(&eswin->lock);
+	eswin->new_usb_role = USB_ROLE_DEVICE;
+	mutex_unlock(&eswin->lock);
+	if (!eswin->suspended)
+		schedule_work(&eswin->otg_work);
+
+	return NOTIFY_DONE;
+}
+
+static int dwc3_eswin_host_notifier(struct notifier_block *nb,
+				    unsigned long event, void *ptr)
+{
+	struct dwc3_eswin *eswin = container_of(nb, struct dwc3_eswin, host_nb);
+	mutex_lock(&eswin->lock);
+	eswin->new_usb_role = USB_ROLE_HOST;
+	mutex_unlock(&eswin->lock);
+	if (!eswin->suspended)
+		schedule_work(&eswin->otg_work);
+
+	return NOTIFY_DONE;
+}
+
+static void dwc3_eswin_otg_extcon_evt_work(struct work_struct *work)
+{
+	struct dwc3_eswin *eswin =
+		container_of(work, struct dwc3_eswin, otg_work);
+	struct usb_role_switch *role_sw = eswin->dwc->role_sw;
+	if (true == eswin->force_mode) {
+		return;
+	}
+	mutex_lock(&eswin->lock);
+	usb_role_switch_set_role(role_sw, eswin->new_usb_role);
+	mutex_unlock(&eswin->lock);
+}
+
+static int dwc3_eswin_get_extcon_dev(struct dwc3_eswin *eswin)
+{
+	struct device *dev = eswin->dev;
+	struct extcon_dev *edev;
+	s32 ret = 0;
+
+	if (device_property_read_bool(dev, "extcon")) {
+		edev = extcon_get_edev_by_phandle(dev, 0);
+		if (IS_ERR(edev)) {
+			if (PTR_ERR(edev) != -EPROBE_DEFER)
+				dev_err(dev, "couldn't get extcon device\n");
+			return PTR_ERR(edev);
+		}
+		eswin->edev = edev;
+		eswin->device_nb.notifier_call = dwc3_eswin_device_notifier;
+		ret = devm_extcon_register_notifier(dev, edev, EXTCON_USB,
+						    &eswin->device_nb);
+		if (ret < 0)
+			dev_err(dev, "failed to register notifier for USB\n");
+
+		eswin->host_nb.notifier_call = dwc3_eswin_host_notifier;
+		ret = devm_extcon_register_notifier(dev, edev, EXTCON_USB_HOST,
+						    &eswin->host_nb);
+		if (ret < 0)
+			dev_err(dev, "failed to register notifier for USB-HOST\n");
+	}
+
+	return 0;
+}
+
+static int dwc3_eswin_deassert(struct dwc3_eswin *eswin)
+{
+	int rc;
+
+	if (eswin->vaux_rst) {
+		rc = reset_control_deassert(eswin->vaux_rst);
+		WARN_ON(0 != rc);
+	}
+
+	return 0;
+}
+
+static int dwc3_eswin_assert(struct dwc3_eswin *eswin)
+{
+	int rc = 0;
+
+	if (eswin->vaux_rst) {
+		rc = reset_control_assert(eswin->vaux_rst);
+		WARN_ON(0 != rc);
+	}
+
+	return 0;
+}
+
+static int dwc_usb_clk_init(struct device *dev)
+{
+	struct regmap *regmap;
+	u32 hsp_usb_bus;
+	u32 hsp_usb_axi_lp;
+	u32 hsp_usb_vbus_freq;
+	u32 hsp_usb_mpll;
+	int ret;
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return -1;
+	}
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+				&hsp_usb_bus);
+	if (ret) {
+		dev_err(dev, "can't get usb sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 2,
+				&hsp_usb_axi_lp);
+	if (ret) {
+		dev_err(dev, "can't get usb sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 3,
+				&hsp_usb_vbus_freq);
+	if (ret) {
+		dev_err(dev, "can't get usb sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 4,
+				&hsp_usb_mpll);
+	if (ret) {
+		dev_err(dev, "can't get usb sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/*
+	 * usb1 clock init
+	 * ref clock is 24M, below need to be set to satisfy usb phy requirement(125M)
+	 */
+	regmap_write(regmap, hsp_usb_vbus_freq, HSP_USB_VBUS_FSEL);
+	regmap_write(regmap, hsp_usb_mpll, HSP_USB_MPLL_DEFAULT);
+
+	/*
+	 * reset usb core and usb phy
+	 */
+	regmap_write(regmap, hsp_usb_bus,
+		     HSP_USB_BUS_FILTER_EN | HSP_USB_BUS_CLKEN_GM |
+			     HSP_USB_BUS_CLKEN_GS | HSP_USB_BUS_SW_RST |
+			     HSP_USB_BUS_CLK_EN);
+	regmap_write(regmap, hsp_usb_axi_lp,
+		     HSP_USB_AXI_LP_XM_CSYSREQ | HSP_USB_AXI_LP_XS_CSYSREQ);
+
+	return 0;
+}
+
+int dwc3_sid_cfg(struct device *dev)
+{
+	int ret;
+	struct regmap *regmap;
+	int hsp_mmu_usb_reg;
+	u32 rdwr_sid_ssid;
+	u32 sid;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+
+	/* not behind smmu, use the default reset value(0x0) of the reg as streamID*/
+	if (fwspec == NULL) {
+		dev_dbg(dev, "dev is not behind smmu, skip configuration of sid\n");
+		return 0;
+	}
+	sid = fwspec->ids[0];
+
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,hsp_sp_csr");
+	if (IS_ERR(regmap)) {
+		dev_dbg(dev, "No hsp_sp_csr phandle specified\n");
+		return 0;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, "eswin,hsp_sp_csr", 1,
+				&hsp_mmu_usb_reg);
+	if (ret) {
+		dev_err(dev, "can't get usb sid cfg reg offset (%d)\n", ret);
+		return ret;
+	}
+
+	/* make the reading sid the same as writing sid, ssid is fixed to zero */
+	rdwr_sid_ssid = FIELD_PREP(AWSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSID, sid);
+	rdwr_sid_ssid |= FIELD_PREP(AWSMMUSSID, 0);
+	rdwr_sid_ssid |= FIELD_PREP(ARSMMUSSID, 0);
+	regmap_write(regmap, hsp_mmu_usb_reg, rdwr_sid_ssid);
+
+	ret = eic7700_dynm_sid_enable(dev_to_node(dev));
+	if (ret < 0)
+	{
+		dev_err(dev, "failed to config usb streamID(%d)!\n", sid);
+	}
+	else
+	{
+		dev_dbg(dev, "success to config usb streamID(%d)!\n", sid);
+	}
+
+	return ret;
+}
+
+static int dwc3_eswin_probe(struct platform_device *pdev)
+{
+	struct dwc3_eswin *eswin;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node, *child;
+	struct platform_device *child_pdev;
+	unsigned int count;
+	int ret;
+	int i;
+	int err_desc = 0;
+	struct gpio_desc *hub_gpio;
+
+	hub_gpio = devm_gpiod_get(dev, "hub-rst", GPIOD_OUT_HIGH);
+	err_desc = IS_ERR(hub_gpio);
+
+	if (!err_desc) {
+		gpiod_set_raw_value(hub_gpio, 1);
+	}
+
+	eswin = devm_kzalloc(dev, sizeof(*eswin), GFP_KERNEL);
+	if (!eswin)
+		return -ENOMEM;
+
+	count = of_clk_get_parent_count(np);
+	if (!count)
+		return -ENOENT;
+
+	eswin->num_clocks = count;
+	eswin->force_mode = false;
+	eswin->clks = devm_kcalloc(dev, eswin->num_clocks, sizeof(struct clk *),
+				   GFP_KERNEL);
+	if (!eswin->clks)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, eswin);
+
+	mutex_init(&eswin->lock);
+
+	eswin->dev = dev;
+
+	mutex_lock(&eswin->lock);
+
+	for (i = 0; i < eswin->num_clocks; i++) {
+		struct clk *clk;
+		clk = of_clk_get(np, i);
+		if (IS_ERR(clk)) {
+			ret = PTR_ERR(clk);
+			goto err0;
+		}
+		ret = clk_prepare_enable(clk);
+		if (ret < 0) {
+			clk_put(clk);
+			goto err0;
+		}
+
+		eswin->clks[i] = clk;
+	}
+
+	eswin->vaux_rst = devm_reset_control_get(dev, "vaux");
+	if (IS_ERR_OR_NULL(eswin->vaux_rst)) {
+		dev_err(dev, "Failed to asic0_rst handle\n");
+		return -EFAULT;
+	}
+
+	dwc3_eswin_deassert(eswin);
+	dwc_usb_clk_init(dev);
+
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "get_sync failed with err %d\n", ret);
+		goto err1;
+	}
+
+	child = of_get_child_by_name(np, "dwc3");
+	if (!child) {
+		dev_err(dev, "failed to find dwc3 core node\n");
+		ret = -ENODEV;
+		goto err1;
+	}
+
+	/* Allocate and initialize the core */
+	ret = of_platform_populate(np, NULL, NULL, dev);
+	if (ret) {
+		dev_err(dev, "failed to create dwc3 core\n");
+		goto err1;
+	}
+
+	INIT_WORK(&eswin->otg_work, dwc3_eswin_otg_extcon_evt_work);
+
+	child_pdev = of_find_device_by_node(child);
+	if (!child_pdev) {
+		dev_err(dev, "failed to find dwc3 core device\n");
+		ret = -ENODEV;
+		goto err2;
+	}
+
+	eswin->dwc = platform_get_drvdata(child_pdev);
+	if (!eswin->dwc) {
+		dev_err(dev, "failed to get drvdata dwc3\n");
+		ret = -EPROBE_DEFER;
+		goto err2;
+	}
+	eswin->child_dev = &child_pdev->dev;
+
+	ret = eic7700_tbu_power(eswin->child_dev, true);
+	if (ret)
+	{
+		dev_err(dev, "tbu power on failed %d\n", ret);
+		goto err2;
+	}
+
+	ret = dwc3_sid_cfg(&child_pdev->dev);
+	if (ret)
+		goto err3;
+
+	ret = dwc3_eswin_get_extcon_dev(eswin);
+	if (ret < 0)
+		goto err3;
+
+	mutex_unlock(&eswin->lock);
+	ret = sysfs_create_group(&dev->kobj, &dwc3_eswin_attr_group);
+	if (ret)
+		dev_err(dev, "failed to create sysfs group: %d\n", ret);
+
+	return ret;
+
+err3:
+	ret = eic7700_tbu_power(eswin->child_dev, false);
+	if (ret)
+	{
+		dev_err(dev, "tbu power2 off failed %d\n", ret);
+	}
+
+err2:
+	cancel_work_sync(&eswin->otg_work);
+	of_platform_depopulate(dev);
+
+err1:
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+	dwc3_eswin_assert(eswin);
+
+err0:
+	for (i = 0; i < eswin->num_clocks && eswin->clks[i]; i++) {
+		if (!pm_runtime_status_suspended(dev))
+			clk_disable(eswin->clks[i]);
+		clk_unprepare(eswin->clks[i]);
+		clk_put(eswin->clks[i]);
+	}
+
+	mutex_unlock(&eswin->lock);
+
+	return ret;
+}
+
+static int dwc3_eswin_remove(struct platform_device *pdev)
+{
+	struct dwc3_eswin *eswin = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	int i = 0;
+	int ret = 0;
+	cancel_work_sync(&eswin->otg_work);
+
+	sysfs_remove_group(&dev->kobj, &dwc3_eswin_attr_group);
+
+	/* Restore hcd state before unregistering xhci */
+	if (eswin->edev && !eswin->connected) {
+		struct usb_hcd *hcd = dev_get_drvdata(&eswin->dwc->xhci->dev);
+
+		pm_runtime_get_sync(dev);
+
+		/*
+		 * The xhci code does not expect that HCDs have been removed.
+		 * It will unconditionally call usb_remove_hcd() when the xhci
+		 * driver is unloaded in of_platform_depopulate(). This results
+		 * in a crash if the HCDs were already removed. To avoid this
+		 * crash, add the HCDs here as dummy operation.
+		 * This code should be removed after pm runtime support
+		 * has been added to xhci.
+		 */
+		if (hcd->state == HC_STATE_HALT) {
+			usb_add_hcd(hcd, hcd->irq, IRQF_SHARED);
+			usb_add_hcd(hcd->shared_hcd, hcd->irq, IRQF_SHARED);
+		}
+	}
+
+	of_platform_depopulate(dev);
+
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+
+	ret = eic7700_tbu_power(eswin->child_dev, false);
+	if (ret)
+	{
+		dev_err(dev, "tbu power off failed %d\n", ret);
+	}
+
+	dwc3_eswin_assert(eswin);
+	for (i = 0; i < eswin->num_clocks; i++) {
+		if (!pm_runtime_status_suspended(dev))
+			clk_disable(eswin->clks[i]);
+		clk_unprepare(eswin->clks[i]);
+		clk_put(eswin->clks[i]);
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int dwc3_eswin_runtime_suspend(struct device *dev)
+{
+	struct dwc3_eswin *eswin = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < eswin->num_clocks; i++)
+		clk_disable(eswin->clks[i]);
+
+	device_init_wakeup(dev, false);
+
+	return 0;
+}
+
+static int dwc3_eswin_runtime_resume(struct device *dev)
+{
+	struct dwc3_eswin *eswin = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < eswin->num_clocks; i++)
+		clk_enable(eswin->clks[i]);
+
+	device_init_wakeup(dev, true);
+
+	return 0;
+}
+
+static int __maybe_unused dwc3_eswin_suspend(struct device *dev)
+{
+	struct dwc3_eswin *eswin = dev_get_drvdata(dev);
+	struct dwc3 *dwc = eswin->dwc;
+
+	eswin->suspended = true;
+	cancel_work_sync(&eswin->otg_work);
+
+	/*
+	 * The flag of is_phy_on is only true if
+	 * the DWC3 is in Host mode.
+	 */
+	if (eswin->is_phy_on) {
+		phy_power_off(dwc->usb2_generic_phy);
+
+		/*
+		 * If link state is Rx.Detect, it means that
+		 * no usb device is connecting with the DWC3
+		 * Host, and need to power off the USB3 PHY.
+		 */
+		dwc->link_state = dwc3_gadget_get_link_state(dwc);
+		if (dwc->link_state == DWC3_LINK_STATE_RX_DET)
+			phy_power_off(dwc->usb3_generic_phy);
+	}
+
+	return 0;
+}
+
+static int __maybe_unused dwc3_eswin_resume(struct device *dev)
+{
+	struct dwc3_eswin *eswin = dev_get_drvdata(dev);
+	struct dwc3 *dwc = eswin->dwc;
+
+	eswin->suspended = false;
+
+	if (eswin->is_phy_on) {
+		phy_power_on(dwc->usb2_generic_phy);
+
+		if (dwc->link_state == DWC3_LINK_STATE_RX_DET)
+			phy_power_on(dwc->usb3_generic_phy);
+	}
+
+	if (eswin->edev)
+		schedule_work(&eswin->otg_work);
+
+	return 0;
+}
+
+static const struct dev_pm_ops dwc3_eswin_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(dwc3_eswin_suspend, dwc3_eswin_resume)
+	SET_RUNTIME_PM_OPS(dwc3_eswin_runtime_suspend,
+			   dwc3_eswin_runtime_resume, NULL)
+};
+
+#define DEV_PM_OPS      (&dwc3_eswin_dev_pm_ops)
+#else
+#define DEV_PM_OPS	NULL
+#endif /* CONFIG_PM */
+
+static const struct of_device_id eswin_dwc3_match[] = {
+	{ .compatible = "eswin,eic7700-dwc3" },
+	{ /* Sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(of, eswin_dwc3_match);
+
+static struct platform_driver dwc3_eswin_driver = {
+	.probe		= dwc3_eswin_probe,
+	.remove		= dwc3_eswin_remove,
+	.driver		= {
+		.name	= "eswin-dwc3",
+		.pm	= DEV_PM_OPS,
+		.of_match_table = eswin_dwc3_match,
+	},
+};
+
+module_platform_driver(dwc3_eswin_driver);
+
+MODULE_ALIAS("platform:eswin-dwc3");
+MODULE_AUTHOR("Han Min <hanmin@eswin.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("DesignWare USB3 ESWIN Glue Layer");
diff --git a/drivers/usb/typec/tcpm/Kconfig b/drivers/usb/typec/tcpm/Kconfig
index 0b2993fef564..760fd03fc751 100644
--- a/drivers/usb/typec/tcpm/Kconfig
+++ b/drivers/usb/typec/tcpm/Kconfig
@@ -63,6 +63,15 @@ config TYPEC_FUSB302
 	  Type-C Port Controller Manager to provide USB PD and USB
 	  Type-C functionalities.
 
+config TYPEC_FUSB303B
+	tristate "Onsemi FUSB303B Type-C chip driver"
+	depends on I2C
+	depends on EXTCON || !EXTCON
+	help
+	  The Onsemi FUSB303B Type-C chip driver that works with
+	  Type-C Port Controller Manager to provide USB
+	  Type-C functionalities.
+
 config TYPEC_WCOVE
 	tristate "Intel WhiskeyCove PMIC USB Type-C PHY driver"
 	depends on ACPI
diff --git a/drivers/usb/typec/tcpm/Makefile b/drivers/usb/typec/tcpm/Makefile
index 7a8cad0c0bdb..d2b501e32012 100644
--- a/drivers/usb/typec/tcpm/Makefile
+++ b/drivers/usb/typec/tcpm/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_TYPEC_TCPM)		+= tcpm.o
 obj-$(CONFIG_TYPEC_FUSB302)		+= fusb302.o
+obj-$(CONFIG_TYPEC_FUSB303B)		+= fusb303b.o
 obj-$(CONFIG_TYPEC_WCOVE)		+= typec_wcove.o
 typec_wcove-y				:= wcove.o
 obj-$(CONFIG_TYPEC_TCPCI)		+= tcpci.o
diff --git a/drivers/usb/typec/tcpm/fusb303b.c b/drivers/usb/typec/tcpm/fusb303b.c
new file mode 100644
index 000000000000..b17c554c2b66
--- /dev/null
+++ b/drivers/usb/typec/tcpm/fusb303b.c
@@ -0,0 +1,643 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Onsemi FUSB303B Type-C Chip Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: Yang Wei <yangwei1@eswincomputing.com>
+ */
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/proc_fs.h>
+#include <linux/regulator/consumer.h>
+#include <linux/sched/clock.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/usb.h>
+#include <linux/usb/typec.h>
+#include <linux/usb/tcpm.h>
+#include <linux/usb/pd.h>
+#include <linux/workqueue.h>
+
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+#include <linux/usb/role.h>
+
+#define FUSB303B_REG_DEVICE_ID 0X1
+#define FUSB303B_REG_DEVICE_TYPE 0X2
+#define FUSB303B_REG_PORTROLE 0X3
+#define FUSB303B_REG_CONTROL 0X4
+#define FUSB303B_REG_CONTROL1 0X5
+#define FUSB303B_REG_RESET 0XA
+#define FUSB303B_REG_MASK 0XE
+#define FUSB303B_REG_MASK1 0XF
+#define FUSB303B_REG_STATUS 0X11
+#define FUSB303B_REG_TYPE 0X13
+#define FUSB303B_REG_INTERRUPT 0X14
+#define FUSB303B_REG_INTERRUPT1 0X15
+
+#define FUSB303B_PORTROLE_DRP BIT(2)
+#define FUSB303B_PORTROLE_SINK BIT(1)
+#define FUSB303B_PORTROLE_SOURCE BIT(0)
+
+#define FUSB303B_CONTROL_DCABLE_EN BIT(3)
+#define FUSB303B_CONTROL_INT_MASK BIT(0)
+
+#define FUSB303B_CONTROL1_REMEDY_EN BIT(7)
+#define FUSB303B_CONTROL1_ENABLE BIT(3)
+
+#define FUSB303B_STATUS_VBUSOK BIT(3)
+#define FUSB303B_STATUS_ATTACH BIT(0)
+
+#define FUSB303B_INT_I_VBUS_CHG BIT(4)
+#define FUSB303B_INT_I_DETACH BIT(1)
+#define FUSB303B_INT_I_ATTACH BIT(0)
+
+#define FUSB303B_TYPE_SINK BIT(4)
+#define FUSB303B_TYPE_SOURCE BIT(3)
+
+#define FUSB_REG_MASK_M_VBUS_CHG BIT(4)
+#define FUSB_REG_MASK_M_DETACH BIT(1)
+#define FUSB_REG_MASK_M_ATTACH BIT(0)
+
+#define LOG_BUFFER_ENTRIES 1024
+#define LOG_BUFFER_ENTRY_SIZE 128
+
+struct fusb303b_chip
+{
+	struct device *dev;
+	struct i2c_client *i2c_client;
+	struct fwnode_handle *fwnode;
+	spinlock_t irq_lock;
+	struct work_struct irq_work;
+	struct gpio_desc *gpio_int_n;
+	int gpio_int_n_irq;
+	struct usb_role_switch *role_sw;
+	/* lock for sharing chip states */
+	struct mutex lock;
+	bool vbus_ok;
+	bool attch_ok;
+#ifdef CONFIG_DEBUG_FS
+	struct dentry *dentry;
+	/* lock for log buffer access */
+	struct mutex logbuffer_lock;
+	int logbuffer_head;
+	int logbuffer_tail;
+	u8 *logbuffer[LOG_BUFFER_ENTRIES];
+#endif
+};
+
+#ifdef CONFIG_DEBUG_FS
+static bool fusb303b_log_full(struct fusb303b_chip *chip)
+{
+	return chip->logbuffer_tail ==
+	       (chip->logbuffer_head + 1) % LOG_BUFFER_ENTRIES;
+}
+
+__printf(2, 0) static void _fusb303b_log(struct fusb303b_chip *chip,
+					 const char *fmt, va_list args)
+{
+	char tmpbuffer[LOG_BUFFER_ENTRY_SIZE];
+	u64 ts_nsec = local_clock();
+	unsigned long rem_nsec;
+
+	if (!chip->logbuffer[chip->logbuffer_head]) {
+		chip->logbuffer[chip->logbuffer_head] =
+			kzalloc(LOG_BUFFER_ENTRY_SIZE, GFP_KERNEL);
+		if (!chip->logbuffer[chip->logbuffer_head])
+			return;
+	}
+
+	vsnprintf(tmpbuffer, sizeof(tmpbuffer), fmt, args);
+
+	mutex_lock(&chip->logbuffer_lock);
+
+	if (fusb303b_log_full(chip)) {
+		chip->logbuffer_head = max(chip->logbuffer_head - 1, 0);
+		strlcpy(tmpbuffer, "overflow", sizeof(tmpbuffer));
+	}
+
+	if (chip->logbuffer_head < 0 ||
+	    chip->logbuffer_head >= LOG_BUFFER_ENTRIES) {
+		dev_warn(chip->dev, "Bad log buffer index %d\n",
+			 chip->logbuffer_head);
+		goto abort;
+	}
+
+	if (!chip->logbuffer[chip->logbuffer_head]) {
+		dev_warn(chip->dev, "Log buffer index %d is NULL\n",
+			 chip->logbuffer_head);
+		goto abort;
+	}
+
+	rem_nsec = do_div(ts_nsec, 1000000000);
+	scnprintf(chip->logbuffer[chip->logbuffer_head], LOG_BUFFER_ENTRY_SIZE,
+		  "[%5lu.%06lu] %s", (unsigned long)ts_nsec, rem_nsec / 1000,
+		  tmpbuffer);
+	chip->logbuffer_head = (chip->logbuffer_head + 1) % LOG_BUFFER_ENTRIES;
+
+abort:
+	mutex_unlock(&chip->logbuffer_lock);
+}
+
+__printf(2, 3) static void fusb303b_log(struct fusb303b_chip *chip,
+					const char *fmt, ...)
+{
+	va_list args;
+
+	va_start(args, fmt);
+	_fusb303b_log(chip, fmt, args);
+	va_end(args);
+}
+
+static int fusb303b_debug_show(struct seq_file *s, void *v)
+{
+	struct fusb303b_chip *chip = (struct fusb303b_chip *)s->private;
+	int tail;
+
+	mutex_lock(&chip->logbuffer_lock);
+	tail = chip->logbuffer_tail;
+	while (tail != chip->logbuffer_head) {
+		seq_printf(s, "%s\n", chip->logbuffer[tail]);
+		tail = (tail + 1) % LOG_BUFFER_ENTRIES;
+	}
+	if (!seq_has_overflowed(s))
+		chip->logbuffer_tail = tail;
+	mutex_unlock(&chip->logbuffer_lock);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(fusb303b_debug);
+
+static void fusb303b_debugfs_init(struct fusb303b_chip *chip)
+{
+	char name[NAME_MAX];
+
+	mutex_init(&chip->logbuffer_lock);
+	snprintf(name, NAME_MAX, "fusb303b-%s", dev_name(chip->dev));
+	chip->dentry = debugfs_create_dir(name, usb_debug_root);
+	debugfs_create_file("log", S_IFREG | 0444, chip->dentry, chip,
+			    &fusb303b_debug_fops);
+}
+
+static void fusb303b_debugfs_exit(struct fusb303b_chip *chip)
+{
+	debugfs_remove(chip->dentry);
+}
+
+#else
+
+static void fusb303b_log(const struct fusb303b_chip *chip, const char *fmt, ...)
+{
+}
+static void fusb303b_debugfs_init(const struct fusb303b_chip *chip)
+{
+}
+static void fusb303b_debugfs_exit(const struct fusb303b_chip *chip)
+{
+}
+
+#endif
+
+static int fusb303b_i2c_write(struct fusb303b_chip *chip, u8 address, u8 data)
+{
+	int ret = 0;
+
+	ret = i2c_smbus_write_byte_data(chip->i2c_client, address, data);
+	if (ret < 0)
+		fusb303b_log(chip, "cannot write 0x%02x to 0x%02x, ret=%d",
+			     data, address, ret);
+
+	return ret;
+}
+
+static int fusb303b_i2c_read(struct fusb303b_chip *chip, u8 address, u8 *data)
+{
+	int ret = 0;
+
+	ret = i2c_smbus_read_byte_data(chip->i2c_client, address);
+	*data = (u8)ret;
+	if (ret < 0)
+		fusb303b_log(chip, "cannot read %02x, ret=%d", address, ret);
+
+	return ret;
+}
+
+static int fusb303b_i2c_mask_write(struct fusb303b_chip *chip, u8 address,
+				   u8 mask, u8 value)
+{
+	int ret = 0;
+	u8 data;
+
+	ret = fusb303b_i2c_read(chip, address, &data);
+	if (ret < 0)
+		return ret;
+	data &= ~mask;
+	data |= value;
+	ret = fusb303b_i2c_write(chip, address, data);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static int fusb303b_i2c_clear_bits(struct fusb303b_chip *chip, u8 address,
+				   u8 clear_bits)
+{
+	return fusb303b_i2c_mask_write(chip, address, clear_bits, 0x00);
+}
+
+static int fusb303b_sw_reset(struct fusb303b_chip *chip)
+{
+	int ret = 0;
+
+	ret = fusb303b_i2c_write(chip, FUSB303B_REG_RESET, 1);
+	if (ret < 0)
+		fusb303b_log(chip, "cannot sw reset the chip, ret=%d", ret);
+	else
+		fusb303b_log(chip, "sw reset");
+
+	return ret;
+}
+
+/*
+ * initialize interrupt on the chip
+ * - unmasked interrupt: VBUS_OK
+ */
+static int fusb303b_init_interrupt(struct fusb303b_chip *chip)
+{
+	int ret = 0;
+	u8 int_unmask = FUSB_REG_MASK_M_VBUS_CHG |
+                        FUSB_REG_MASK_M_DETACH | FUSB_REG_MASK_M_ATTACH;
+	ret = fusb303b_i2c_write(chip, FUSB303B_REG_MASK,
+                                0xFF & ~(int_unmask));
+	if (ret < 0)
+		return ret;
+	ret = fusb303b_i2c_write(chip, FUSB303B_REG_MASK1, 0xFF);
+	if (ret < 0)
+		return ret;
+
+	ret = fusb303b_i2c_clear_bits(chip, FUSB303B_REG_CONTROL,
+                                    FUSB303B_CONTROL_INT_MASK);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static int fusb303b_init(struct fusb303b_chip *chip)
+{
+	int ret = 0;
+	u8 data;
+
+	ret = fusb303b_sw_reset(chip);
+	if (ret < 0)
+		return ret;
+	fusb303b_i2c_read(chip, FUSB303B_REG_STATUS, &data);
+	fusb303b_i2c_mask_write(chip, FUSB303B_REG_CONTROL,
+                                FUSB303B_CONTROL_DCABLE_EN,
+                                FUSB303B_CONTROL_DCABLE_EN);
+	fusb303b_i2c_mask_write(chip, FUSB303B_REG_CONTROL1,
+                                FUSB303B_CONTROL1_ENABLE | FUSB303B_CONTROL1_REMEDY_EN,
+                                FUSB303B_CONTROL1_ENABLE | FUSB303B_CONTROL1_REMEDY_EN);
+	ret = fusb303b_init_interrupt(chip);
+	if (ret < 0)
+		return ret;
+
+	ret = fusb303b_i2c_read(chip, FUSB303B_REG_STATUS, &data);
+	if (ret < 0)
+		return ret;
+	chip->vbus_ok = !!(data & FUSB303B_STATUS_VBUSOK);
+	chip->attch_ok = !!(data & FUSB303B_STATUS_ATTACH);
+	ret = fusb303b_i2c_read(chip, FUSB303B_REG_DEVICE_ID, &data);
+	if (ret < 0)
+		return ret;
+	fusb303b_log(chip, "fusb303b device ID: 0x%02x", data);
+
+	ret = fusb303b_i2c_read(chip, FUSB303B_REG_DEVICE_TYPE, &data);
+	if (ret < 0)
+		return ret;
+	fusb303b_log(chip, "fusb303b type:0x%02x", data);
+
+	return ret;
+}
+
+static s32 fusb303b_set_port_check(struct fusb303b_chip *chip,
+                                    enum typec_port_type port_type)
+{
+	s32 ret = 0;
+
+	fusb303b_log(chip, "%s.%d port_type:%d",
+                    __FUNCTION__, __LINE__, port_type);
+	switch (port_type)
+	{
+            case TYPEC_PORT_DRP:
+                ret = fusb303b_i2c_write(chip, FUSB303B_REG_PORTROLE,
+                                        FUSB303B_PORTROLE_DRP);
+                break;
+            case TYPEC_PORT_SRC:
+                ret = fusb303b_i2c_write(chip, FUSB303B_REG_PORTROLE,
+                                        FUSB303B_PORTROLE_SOURCE);
+                break;
+            default:
+                ret = fusb303b_i2c_write(chip, FUSB303B_REG_PORTROLE,
+                                        FUSB303B_PORTROLE_SINK);
+                break;
+	}
+
+	return ret;
+}
+
+int fusb303b_set_usb_role(struct fusb303b_chip *chip)
+{
+	u8 type = 0;
+	int ret = 0;
+
+	if ((true == chip->attch_ok) && (true == chip->vbus_ok))
+	{
+		ret = fusb303b_i2c_read(chip, FUSB303B_REG_TYPE, &type);
+		if (ret < 0)
+		{
+			fusb303b_log(chip, "read type error:%d", ret);
+			return ret;
+		}
+		fusb303b_log(chip, "%s type: 0x%02x", __func__, type);
+		if (FUSB303B_TYPE_SOURCE == (FUSB303B_TYPE_SOURCE & type))
+		{
+			usb_role_switch_set_role(chip->role_sw, USB_ROLE_HOST);
+			fusb303b_log(chip, "set usb to host");
+		}
+		else
+		{
+			usb_role_switch_set_role(chip->role_sw, USB_ROLE_DEVICE);
+			fusb303b_log(chip, "set usb to device");
+			if (FUSB303B_TYPE_SINK != (FUSB303B_TYPE_SINK & type))
+			{
+				fusb303b_log(chip, "illegel type:0x%02x,set usb to device", type);
+			}
+		}
+	}
+
+	return 0;
+}
+
+static irqreturn_t fusb303b_irq_intn(int irq, void *dev_id)
+{
+	struct fusb303b_chip *chip = dev_id;
+	int ret = 0;
+	u8 interrupt = 0, interrupt1 = 0, status = 0;
+
+	mutex_lock(&chip->lock);
+	ret = fusb303b_i2c_read(chip, FUSB303B_REG_INTERRUPT, &interrupt);
+	if (ret < 0)
+		goto done;
+	ret = fusb303b_i2c_read(chip, FUSB303B_REG_INTERRUPT1, &interrupt1);
+	if (ret < 0)
+		goto done;
+	ret = fusb303b_i2c_read(chip, FUSB303B_REG_STATUS, &status);
+	if (ret < 0)
+		goto done;
+
+	fusb303b_log(chip, "IRQ: 0x%02x,0x%02x status: 0x%02x",
+				 interrupt, interrupt1, status);
+
+	if (interrupt & FUSB303B_INT_I_VBUS_CHG)
+	{
+		chip->vbus_ok = !!(status & FUSB303B_STATUS_VBUSOK);
+		fusb303b_log(chip, "IRQ: VBUS_OK, vbus=%s",
+                    chip->vbus_ok ? "On" : "Off");
+	}
+	if (interrupt & (FUSB303B_INT_I_ATTACH | FUSB303B_INT_I_DETACH))
+	{
+		chip->attch_ok = !!(status & FUSB303B_STATUS_ATTACH);
+		fusb303b_log(chip, "IRQ: attach OK, attach=%s",
+                        chip->attch_ok ? "On" : "Off");
+	}
+	fusb303b_set_usb_role(chip);
+	if (0 != interrupt)
+		fusb303b_i2c_write(chip, FUSB303B_REG_INTERRUPT, interrupt);
+	if (0 != interrupt1)
+		fusb303b_i2c_write(chip, FUSB303B_REG_INTERRUPT1, interrupt1);
+
+done:
+	mutex_unlock(&chip->lock);
+	return IRQ_HANDLED;
+}
+
+static int init_gpio(struct fusb303b_chip *chip)
+{
+	struct device *dev = chip->dev;
+	int ret = 0;
+
+	chip->gpio_int_n = devm_gpiod_get(dev, "int", GPIOD_IN);
+	if (IS_ERR(chip->gpio_int_n))
+	{
+		fusb303b_log(chip, "failed to request gpio_int_n\n");
+		return PTR_ERR(chip->gpio_int_n);
+	}
+	ret = gpiod_to_irq(chip->gpio_int_n);
+	if (ret < 0)
+	{
+		fusb303b_log(chip, "cannot request IRQ for GPIO Int_N, ret=%d", ret);
+		return ret;
+	}
+	chip->gpio_int_n_irq = ret;
+
+	return 0;
+}
+
+static const struct property_entry port_props[] = {
+	PROPERTY_ENTRY_STRING("data-role", "dual"),
+	PROPERTY_ENTRY_STRING("power-role", "dual"),
+	PROPERTY_ENTRY_STRING("try-power-role", "sink"),
+	{}};
+
+static struct fwnode_handle *fusb303b_fwnode_get(struct device *dev)
+{
+	struct fwnode_handle *fwnode;
+	fwnode = device_get_named_child_node(dev, "connector");
+	if (!fwnode)
+		fwnode = fwnode_create_software_node(port_props, NULL);
+
+	return fwnode;
+}
+
+static int fusb303b_probe(struct i2c_client *client)
+{
+	struct fusb303b_chip *chip;
+	struct device *dev = &client->dev;
+	int ret = 0;
+	struct regmap *regmap;
+	int irq_sel_reg;
+	int irq_sel_bit;
+	const char *cap_str;
+	int usb_data_role = 0;
+	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "eswin,syscfg");
+	if (!IS_ERR(regmap)) {
+		ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg",
+						 1, &irq_sel_reg);
+		if (ret) {
+			dev_err(dev,
+				"can't get irq cfg reg offset in sys_con(errno:%d)\n",
+				ret);
+			return ret;
+		}
+		ret = of_property_read_u32_index(dev->of_node, "eswin,syscfg",
+						 2, &irq_sel_bit);
+		if (ret) {
+			dev_err(dev,
+				"can't get irq cfg bit offset in sys_con(errno:%d)\n",
+				ret);
+			return ret;
+		}
+		regmap_clear_bits(regmap, irq_sel_reg, BIT_ULL(irq_sel_bit));
+	}
+
+	chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL);
+	if (!chip)
+		return -ENOMEM;
+	chip->i2c_client = client;
+	chip->dev = &client->dev;
+	mutex_init(&chip->lock);
+	spin_lock_init(&chip->irq_lock);
+	fusb303b_init(chip);
+	fusb303b_debugfs_init(chip);
+	if (client->irq)
+	{
+		chip->gpio_int_n_irq = client->irq;
+	}
+	else
+	{
+		ret = init_gpio(chip);
+		if (ret < 0)
+			goto destroy_workqueue;
+	}
+	chip->fwnode = fusb303b_fwnode_get(dev);
+	if (IS_ERR(chip->fwnode))
+	{
+		ret = PTR_ERR(chip->fwnode);
+		goto destroy_workqueue;
+	}
+	/*
+	 * This fwnode has a "compatible" property, but is never populated as a
+	 * struct device. Instead we simply parse it to read the properties.
+	 * This it breaks fw_devlink=on. To maintain backward compatibility
+	 * with existing DT files, we work around this by deleting any
+	 * fwnode_links to/from this fwnode.
+	 */
+	fw_devlink_purge_absent_suppliers(chip->fwnode);
+
+	/* USB data support is optional */
+	ret = fwnode_property_read_string(chip->fwnode, "data-role", &cap_str);
+	if (ret == 0)
+	{
+		ret = typec_find_port_data_role(cap_str);
+		if (ret < 0)
+		{
+			fusb303b_log(chip, "%s is not leage data-role\n", cap_str);
+			goto put_fwnode;
+		}
+		usb_data_role = ret;
+	}
+	else
+	{
+		fusb303b_log(chip, "cannot find data-role in dts\n");
+	}
+	chip->role_sw = usb_role_switch_get(chip->dev);
+	if (IS_ERR(chip->role_sw))
+	{
+		ret = PTR_ERR(chip->role_sw);
+		fusb303b_log(chip, "get role_sw error");
+		goto put_fwnode;
+	}
+
+	ret = devm_request_threaded_irq(dev, chip->gpio_int_n_irq, NULL,
+									fusb303b_irq_intn,
+									IRQF_ONESHOT | IRQF_TRIGGER_LOW,
+									"fusb303b_interrupt_int_n", chip);
+	if (ret < 0)
+	{
+		fusb303b_log(chip, "cannot request IRQ for GPIO Int_N, ret=%d", ret);
+		goto err_put_role;
+	}
+
+	enable_irq_wake(chip->gpio_int_n_irq);
+	fusb303b_set_port_check(chip, usb_data_role);
+
+	i2c_set_clientdata(client, chip);
+	fusb303b_log(chip, "Kernel thread created successfully");
+	return ret;
+err_put_role:
+	usb_role_switch_put(chip->role_sw);
+put_fwnode:
+	fwnode_handle_put(chip->fwnode);
+destroy_workqueue:
+	fusb303b_debugfs_exit(chip);
+
+	return ret;
+}
+
+static void fusb303b_remove(struct i2c_client *client)
+{
+	struct fusb303b_chip *chip = i2c_get_clientdata(client);
+
+	disable_irq_wake(chip->gpio_int_n_irq);
+	free_irq(chip->gpio_int_n_irq, chip);
+	usb_role_switch_put(chip->role_sw);
+	fwnode_handle_put(chip->fwnode);
+	fusb303b_debugfs_exit(chip);
+}
+
+static const struct of_device_id fusb303b_dt_match[] = {
+	{.compatible = "fcs,fusb303b"},
+	{},
+};
+MODULE_DEVICE_TABLE(of, fusb303b_dt_match);
+
+static const struct i2c_device_id fusb303b_i2c_device_id[] = {
+	{"typec_fusb303b", 0},
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, fusb303b_i2c_device_id);
+
+static struct i2c_driver fusb303b_driver = {
+	.driver = {
+		.name = "typec_fusb303b",
+		.of_match_table = of_match_ptr(fusb303b_dt_match),
+	},
+	.probe = fusb303b_probe,
+	.remove = fusb303b_remove,
+	.id_table = fusb303b_i2c_device_id,
+};
+module_i2c_driver(fusb303b_driver);
+
+MODULE_AUTHOR("Yang Wei <yangwei1@eswincomputing.com>");
+MODULE_DESCRIPTION("Onsemi FUSB303B Type-C Chip Driver");
+MODULE_LICENSE("GPL");
diff --git a/include/drm/bridge/dw_hdmi.h b/include/drm/bridge/dw_hdmi.h
index 6a46baa0737c..85bab1773f38 100644
--- a/include/drm/bridge/dw_hdmi.h
+++ b/include/drm/bridge/dw_hdmi.h
@@ -160,6 +160,13 @@ struct dw_hdmi_plat_data {
 	const struct dw_hdmi_phy_config *phy_config;
 	int (*configure_phy)(struct dw_hdmi *hdmi, void *data,
 			     unsigned long mpixelclock);
+	unsigned long (*get_input_bus_format)(void *data);
+	unsigned long (*get_output_bus_format)(void *data);
+	unsigned long (*get_enc_in_encoding)(void *data);
+	unsigned long (*get_enc_out_encoding)(void *data);
+
+	/* Vendor Property support */
+	const struct dw_hdmi_property_ops *property_ops;
 
 	unsigned int disable_cec : 1;
 };
diff --git a/include/dt-bindings/clock/eic7700-clock.h b/include/dt-bindings/clock/eic7700-clock.h
new file mode 100755
index 000000000000..866507498876
--- /dev/null
+++ b/include/dt-bindings/clock/eic7700-clock.h
@@ -0,0 +1,623 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Clk Provider Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef __DTS_EIC7700_CLOCK_H
+#define __DTS_EIC7700_CLOCK_H
+
+#define EIC7700_NONE_CLOCK	0
+
+/* fixed rate */
+#define EIC7700_XTAL_24M                        1
+#define EIC7700_XTAL_32K                        2
+#define EIC7700_PLL_CPU                         3	/*for cpu clk*/
+#define EIC7700_SPLL0_FOUT1                     4
+#define EIC7700_SPLL0_FOUT2                     5
+#define EIC7700_SPLL0_FOUT3                     6
+#define EIC7700_SPLL1_FOUT1                     7
+#define EIC7700_SPLL1_FOUT2                     8
+#define EIC7700_SPLL1_FOUT3                     9
+#define EIC7700_SPLL2_FOUT1                     10
+#define EIC7700_SPLL2_FOUT2                     11
+#define EIC7700_SPLL2_FOUT3                     12
+#define EIC7700_VPLL_FOUT1                      13
+#define EIC7700_VPLL_FOUT2                      14
+#define EIC7700_VPLL_FOUT3                      15
+#define EIC7700_APLL_FOUT1                      16
+#define EIC7700_APLL_FOUT2                      17
+#define EIC7700_APLL_FOUT3                      18
+#define EIC7700_EXT_MCLK                        19
+#define EIC7700_PLL_DDR                         20
+#define EIC7700_LPDDR_REF_BAK                   21
+
+
+/* mux clocks */
+#define  EIC7700_MUX_U_CPU_ROOT_3MUX1_GFREE                         30	/*for cpu clk*/
+#define  EIC7700_MUX_U_CPU_ACLK_2MUX1_GFREE                         31	/*for cpu clk*/
+#define  EIC7700_MUX_U_DSP_ACLK_ROOT_2MUX1_GFREE                    32
+#define  EIC7700_MUX_U_D2D_ACLK_ROOT_2MUX1_GFREE                    33
+#define  EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_0                         34
+#define  EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_1                         35
+#define  EIC7700_MUX_U_MSHCORE_ROOT_3MUX1_2                         36
+#define  EIC7700_MUX_U_NPU_LLCLK_3MUX1_GFREE                        37
+#define  EIC7700_MUX_U_NPU_CORE_3MUX1_GFREE                         38
+#define  EIC7700_MUX_U_VI_ACLK_ROOT_2MUX1_GFREE                     39
+#define  EIC7700_MUX_U_VI_DVP_ROOT_2MUX1_GFREE                      40
+#define  EIC7700_MUX_U_VI_DIG_ISP_ROOT_2MUX1_GFREE                  41
+#define  EIC7700_MUX_U_VO_ACLK_ROOT_2MUX1_GFREE                     42
+#define  EIC7700_MUX_U_VO_PIXEL_ROOT_2MUX1                          43
+#define  EIC7700_MUX_U_VO_CEC_2MUX1                                 44
+#define  EIC7700_MUX_U_VCDEC_ROOT_2MUX1_GFREE                       45
+#define  EIC7700_MUX_U_VCACLK_ROOT_2MUX1_GFREE                      46
+#define  EIC7700_MUX_U_RTC_2MUX1                                    47
+#define  EIC7700_MUX_U_SYSCFG_CLK_ROOT_2MUX1_GFREE                  48
+#define  EIC7700_MUX_U_NOCNSP_XTAL_2MUX1                            49
+#define  EIC7700_MUX_U_BOOTSPI_CLK_2MUX1_GFREE                      50
+#define  EIC7700_MUX_U_SCPU_CORE_CLK_2MUX1_GFREE                    51
+#define  EIC7700_MUX_U_LPCPU_CORE_CLK_2MUX1_GFREE                   52
+#define  EIC7700_MUX_GPU_ACLK_XTAL_2MUX1                            53
+#define  EIC7700_MUX_U_DSP_ACLK_XTAL_2MUX1                          54
+#define  EIC7700_MUX_U_D2D_ACLK_XTAL_2MUX1                          55
+#define  EIC7700_MUX_U_HSP_ACLK_XTAL_2MUX1                          56
+#define  EIC7700_MUX_U_PCIE_ACLK_XTAL_2MUX1                         57
+#define  EIC7700_MUX_U_NPU_ACLK_XTAL_2MUX1                          58
+#define  EIC7700_MUX_U_NPU_LLC_XTAL_2MUX1                           59
+#define  EIC7700_MUX_U_NPU_CORE_XTAL_2MUX1                          60
+#define  EIC7700_MUX_U_VI_ACLK_XTAL_2MUX1                           61
+#define  EIC7700_MUX_U_VI_DVP_XTAL_2MUX1                            62
+#define  EIC7700_MUX_U_VI_DIG_ISP_XTAL_2MUX1                        63
+#define  EIC7700_MUX_U_VI_SHUTTER_XTAL_2MUX1_0                      64
+#define  EIC7700_MUX_U_VI_SHUTTER_XTAL_2MUX1_1                      65
+#define  EIC7700_MUX_U_VI_SHUTTER_XTAL_2MUX1_2                      66
+#define  EIC7700_MUX_U_VI_SHUTTER_XTAL_2MUX1_3                      67
+#define  EIC7700_MUX_U_VI_SHUTTER_XTAL_2MUX1_4                      68
+#define  EIC7700_MUX_U_VI_SHUTTER_XTAL_2MUX1_5                      69
+#define  EIC7700_MUX_U_VO_ACLK_XTAL_2MUX1                           70
+#define  EIC7700_MUX_U_IESMCLK_XTAL_2MUX1                           71
+#define  EIC7700_MUX_U_VO_PIXEL_XTAL_2MUX1                          72
+#define  EIC7700_MUX_U_VO_MCLK_2MUX_EXT_MCLK                        73
+#define  EIC7700_MUX_U_VC_ACLK_XTAL_2MUX1                           74
+#define  EIC7700_MUX_U_JD_XTAL_2MUX1                                75
+#define  EIC7700_MUX_U_JE_XTAL_2MUX1                                76
+#define  EIC7700_MUX_U_VE_XTAL_2MUX1                                77
+#define  EIC7700_MUX_U_VD_XTAL_2MUX1                                78
+#define  EIC7700_MUX_U_SATA_PHY_2MUX1                               79
+#define  EIC7700_MUX_U_AONDMA_AXI2MUX1_GFREE                        80
+#define  EIC7700_MUX_U_CRYPTO_XTAL_2MUX1                            81
+#define  EIC7700_MUX_U_RMII_REF_2MUX                                82
+#define  EIC7700_MUX_U_ETH_CORE_2MUX1                               83
+#define  EIC7700_MUX_U_VI_DW_ROOT_2MUX1                             84
+#define  EIC7700_MUX_U_VI_DW_XTAL_2MUX1                             85
+#define  EIC7700_MUX_U_NPU_E31_3MUX1_GFREE                          86
+#define  EIC7700_MUX_U_DDR_ACLK_ROOT_2MUX1_GFREE                    87
+
+/* divider clocks */
+#define EIC7700_DIVDER_U_SYS_CFG_DIV_DYNM                           100
+#define EIC7700_DIVDER_U_NOC_NSP_DIV_DYNM                           101
+#define EIC7700_DIVDER_U_BOOTSPI_DIV_DYNM                           102
+#define EIC7700_DIVDER_U_SCPU_CORE_DIV_DYNM                         103
+#define EIC7700_DIVDER_U_LPCPU_CORE_DIV_DYNM                        104
+#define EIC7700_DIVDER_U_GPU_ACLK_DIV_DYNM                          105
+#define EIC7700_DIVDER_U_DSP_ACLK_DIV_DYNM                          106
+#define EIC7700_DIVDER_U_D2D_ACLK_DIV_DYNM                          107
+#define EIC7700_DIVDER_U_HSP_ACLK_DIV_DYNM                          108
+#define EIC7700_DIVDER_U_ETH_TXCLK_DIV_DYNM_0                       109
+#define EIC7700_DIVDER_U_ETH_TXCLK_DIV_DYNM_1                       110
+#define EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_0                       111
+#define EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_1                       112
+#define EIC7700_DIVDER_U_MSHC_CORE_DIV_DYNM_2                       113
+#define EIC7700_DIVDER_U_PCIE_ACLK_DIV_DYNM                         114
+#define EIC7700_DIVDER_U_NPU_ACLK_DIV_DYNM                          115
+#define EIC7700_DIVDER_U_NPU_LLC_SRC0_DIV_DYNM                      116
+#define EIC7700_DIVDER_U_NPU_LLC_SRC1_DIV_DYNM                      117
+#define EIC7700_DIVDER_U_NPU_CORECLK_DIV_DYNM                       118
+#define EIC7700_DIVDER_U_VI_ACLK_DIV_DYNM                           119
+#define EIC7700_DIVDER_U_VI_DVP_DIV_DYNM                            120
+#define EIC7700_DIVDER_U_VI_DIG_ISP_DIV_DYNM                        121
+#define EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_0                      122
+#define EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_1                      123
+#define EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_2                      124
+#define EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_3                      125
+#define EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_4                      126
+#define EIC7700_DIVDER_U_VI_SHUTTER_DIV_DYNM_5                      127
+#define EIC7700_DIVDER_U_VO_ACLK_DIV_DYNM                           128
+#define EIC7700_DIVDER_U_IESMCLK_DIV_DYNM                           129
+#define EIC7700_DIVDER_U_VO_PIXEL_DIV_DYNM                          130
+#define EIC7700_DIVDER_U_VO_MCLK_DIV_DYNM                           131
+#define EIC7700_DIVDER_U_VC_ACLK_DIV_DYNM                           132
+#define EIC7700_DIVDER_U_JD_DIV_DYNM                                133
+#define EIC7700_DIVDER_U_JE_DIV_DYNM                                134
+#define EIC7700_DIVDER_U_VE_DIV_DYNM                                135
+#define EIC7700_DIVDER_U_VD_DIV_DYNM                                136
+#define EIC7700_DIVDER_U_G2D_DIV_DYNM                               137
+#define EIC7700_DIVDER_U_AONDMA_AXI_DIV_DYNM                        138
+#define EIC7700_DIVDER_U_CRYPTO_DIV_DYNM                            139
+#define EIC7700_DIVDER_U_VI_DW_DIV_DYNM                             140
+#define EIC7700_DIVDER_U_NPU_E31_DIV_DYNM                           141
+#define EIC7700_DIVDER_U_SATA_PHY_REF_DIV_DYNM                      142
+#define EIC7700_DIVDER_U_DSP_0_ACLK_DIV_DYNM                        143
+#define EIC7700_DIVDER_U_DSP_1_ACLK_DIV_DYNM                        144
+#define EIC7700_DIVDER_U_DSP_2_ACLK_DIV_DYNM                        145
+#define EIC7700_DIVDER_U_DSP_3_ACLK_DIV_DYNM                        146
+#define EIC7700_DIVDER_U_DDR_ACLK_DIV_DYNM                          147
+#define EIC7700_DIVDER_U_AON_RTC_DIV_DYNM                           148
+#define EIC7700_DIVDER_U_U84_RTC_TOGGLE_DIV_DYNM                    149
+#define EIC7700_DIVDER_U_VO_CEC_DIV_DYNM                            150
+
+/* gate clocks */
+#define EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_0                 200
+#define EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_1                 201
+#define EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_2                 202
+#define EIC7700_GATE_CLK_CPU_EXT_SRC_CORE_CLK_3                 203
+#define EIC7700_GATE_CLK_CPU_TRACE_CLK_0                        204
+#define EIC7700_GATE_CLK_CPU_TRACE_CLK_1                        205
+#define EIC7700_GATE_CLK_CPU_TRACE_CLK_2                        206
+#define EIC7700_GATE_CLK_CPU_TRACE_CLK_3                        207
+#define EIC7700_GATE_CLK_CPU_DEBUG_CLK                          208
+#define EIC7700_GATE_CLK_CPU_TRACE_COM_CLK                      209
+#define EIC7700_GATE_CLK_CPU_CLK                                210
+#define EIC7700_GATE_CLK_SPLL0_FOUT2                            211
+#define EIC7700_GATE_CLK_VPLL_FOUT2                             212
+#define EIC7700_GATE_CLK_VPLL_FOUT3                             213
+#define EIC7700_GATE_CLK_APLL_FOUT1                             214
+#define EIC7700_GATE_CLK_APLL_FOUT2                             215
+#define EIC7700_GATE_CLK_APLL_FOUT3                             216
+#define EIC7700_GATE_EXT_MCLK                                   217
+#define EIC7700_GATE_CLK_1M                                     218
+#define EIC7700_GATE_CLK_SYS_CFG                                219
+#define EIC7700_GATE_CLK_MIPI_TXESC                             220
+#define EIC7700_GATE_NOC_CFG_CLK                                221
+#define EIC7700_GATE_NOC_NSP_CLK                                222
+#define EIC7700_GATE_CLK_BOOTSPI                                223
+#define EIC7700_GATE_CLK_BOOTSPI_CFG                            224
+#define EIC7700_GATE_CLK_U84_CORE_LP                            225
+#define EIC7700_GATE_CLK_SCPU_CORE                              226
+#define EIC7700_GATE_CLK_SCPU_BUS                               227
+#define EIC7700_GATE_CLK_LPCPU_CORE                             228
+#define EIC7700_GATE_CLK_LPCPU_BUS                              229
+#define EIC7700_GATE_GPU_ACLK                                   230
+#define EIC7700_GATE_GPU_GRAY_CLK                               231
+#define EIC7700_GATE_GPU_CFG_CLK                                232
+#define EIC7700_GATE_CLK_DSP_ROOT                               233
+#define EIC7700_GATE_DSPT_ACLK                                  234
+#define EIC7700_GATE_DSPT_CFG_CLK                               235
+#define EIC7700_GATE_CLK_D2DDR_ACLK                             236
+#define EIC7700_GATE_D2D_ACLK                                   237
+#define EIC7700_GATE_D2D_CFG_CLK                                238
+#define EIC7700_GATE_CLK_HSP_ACLK                               239
+#define EIC7700_GATE_CLK_HSP_CFGCLK                             240
+#define EIC7700_GATE_TCU_ACLK                                   241
+#define EIC7700_GATE_TCU_CFG_CLK                                242
+#define EIC7700_GATE_DDRT_CFG_CLK                               243
+#define EIC7700_GATE_DDRT1_CFG_CLK                              244
+#define EIC7700_GATE_DDRT0_P0_ACLK                              245
+#define EIC7700_GATE_DDRT0_P1_ACLK                              246
+#define EIC7700_GATE_DDRT0_P2_ACLK                              247
+#define EIC7700_GATE_DDRT0_P3_ACLK                              248
+#define EIC7700_GATE_DDRT0_P4_ACLK                              249
+#define EIC7700_GATE_DDRT1_P0_ACLK                              250
+#define EIC7700_GATE_DDRT1_P1_ACLK                              251
+#define EIC7700_GATE_DDRT1_P2_ACLK                              252
+#define EIC7700_GATE_DDRT1_P3_ACLK                              253
+#define EIC7700_GATE_DDRT1_P4_ACLK                              254
+#define EIC7700_GATE_HSP_ACLK                                   255
+#define EIC7700_GATE_HSP_CFG_CLK                                256
+#define EIC7700_GATE_HSP_SATA_RBC_CLK                           257
+#define EIC7700_GATE_HSP_SATA_OOB_CLK                           258
+#define EIC7700_GATE_HSP_SATA_PMALIVE_CLK                       259
+#define EIC7700_GATE_HSP_ETH_APP_CLK                            260
+#define EIC7700_GATE_HSP_ETH_CSR_CLK                            261
+#define EIC7700_GATE_HSP_ETH0_CORE_CLK                          262
+#define EIC7700_GATE_HSP_ETH1_CORE_CLK                          263
+#define EIC7700_GATE_HSP_MSHC0_CORE_CLK                         264
+#define EIC7700_GATE_HSP_MSHC1_CORE_CLK                         265
+#define EIC7700_GATE_HSP_MSHC2_CORE_CLK                         266
+#define EIC7700_GATE_HSP_MSHC0_TMR_CLK                          267
+#define EIC7700_GATE_HSP_MSHC1_TMR_CLK                          268
+#define EIC7700_GATE_HSP_MSHC2_TMR_CLK                          269
+#define EIC7700_GATE_HSP_USB0_SUSPEND_CLK                       270
+#define EIC7700_GATE_HSP_USB1_SUSPEND_CLK                       271
+#define EIC7700_GATE_PCIET_ACLK                                 272
+#define EIC7700_GATE_PCIET_CFG_CLK                              273
+#define EIC7700_GATE_PCIET_CR_CLK                               274
+#define EIC7700_GATE_PCIET_AUX_CLK                              275
+#define EIC7700_GATE_NPU_ACLK                                   276
+#define EIC7700_GATE_NPU_CFG_CLK                                277
+#define EIC7700_GATE_CLK_NPU_LLC_SRC0                           278
+#define EIC7700_GATE_CLK_NPU_LLC_SRC1                           279
+#define EIC7700_GATE_NPU_LLC_ACLK                               280
+#define EIC7700_GATE_CLK_NPU_CORE_ST1                           281
+#define EIC7700_GATE_NPU_CLK                                    282
+#define EIC7700_GATE_NPU_E31_CLK                                283
+#define EIC7700_GATE_CLK_VI_ACLK_ST1                            284
+#define EIC7700_GATE_VI_ACLK                                    285
+#define EIC7700_GATE_VI_DVP_CLK                                 286
+#define EIC7700_GATE_VI_CFG_CLK                                 287
+#define EIC7700_GATE_VI_DIG_DW_CLK                              288
+#define EIC7700_GATE_VI_DIG_ISP_CLK                             289
+#define EIC7700_GATE_VI_SHUTTER_0                               290
+#define EIC7700_GATE_VI_SHUTTER_1                               291
+#define EIC7700_GATE_VI_SHUTTER_2                               292
+#define EIC7700_GATE_VI_SHUTTER_3                               293
+#define EIC7700_GATE_VI_SHUTTER_4                               294
+#define EIC7700_GATE_VI_SHUTTER_5                               295
+#define EIC7700_GATE_VI_PHY_TXCLKESC                            296
+#define EIC7700_GATE_VI_PHY_CFG                                 297
+#define EIC7700_GATE_VO_ACLK                                    298
+#define EIC7700_GATE_VO_CFG_CLK                                 299
+#define EIC7700_GATE_VO_HDMI_IESMCLK                            300
+#define EIC7700_GATE_VO_PIXEL_CLK                               301
+#define EIC7700_GATE_VO_I2S_MCLK                                302
+#define EIC7700_GATE_VO_CR_CLK                                  303
+#define EIC7700_GATE_VO_CEC_CLK                                 304
+#define EIC7700_GATE_CLK_VC_ROOT                                305
+#define EIC7700_GATE_VC_ACLK                                    306
+#define EIC7700_GATE_VC_CFG_CLK                                 307
+#define EIC7700_GATE_VC_JE_CLK                                  308
+#define EIC7700_GATE_VC_JD_CLK                                  309
+#define EIC7700_GATE_VC_VE_CLK                                  310
+#define EIC7700_GATE_VC_VD_CLK                                  311
+#define EIC7700_GATE_G2D_CFG_CLK                                312
+#define EIC7700_GATE_CLK_G2D_ST2                                313
+#define EIC7700_GATE_G2D_CLK                                    314
+#define EIC7700_GATE_G2D_ACLK                                   315
+#define EIC7700_GATE_CLK_PVT_INNER                              316
+#define EIC7700_GATE_PVT_CLK_0                                  317
+#define EIC7700_GATE_PVT_CLK_1                                  318
+#define EIC7700_GATE_PVT_CLK_2                                  319
+#define EIC7700_GATE_PVT_CLK_3                                  320
+#define EIC7700_GATE_PVT_CLK_4                                  321
+#define EIC7700_GATE_CLK_AONDMA_CFG                             322
+#define EIC7700_GATE_CLK_AONDMA_AXI_ST3                         323
+#define EIC7700_GATE_AONDMA_ACLK                                324
+#define EIC7700_GATE_AON_ACLK                                   325
+#define EIC7700_GATE_TIMER_CLK_0                                326
+#define EIC7700_GATE_TIMER_CLK_1                                327
+#define EIC7700_GATE_TIMER_CLK_2                                328
+#define EIC7700_GATE_TIMER_CLK_3                                329
+#define EIC7700_GATE_TIMER_PCLK_0                               330
+#define EIC7700_GATE_TIMER_PCLK_1                               331
+#define EIC7700_GATE_TIMER_PCLK_2                               332
+#define EIC7700_GATE_TIMER_PCLK_3                               333
+#define EIC7700_GATE_TIMER3_CLK8                                334
+#define EIC7700_GATE_CLK_RTC_CFG                                335
+#define EIC7700_GATE_CLK_RTC                                    336
+#define EIC7700_GATE_HSP_RMII_REF_0                             337
+#define EIC7700_GATE_HSP_RMII_REF_1                             338
+#define EIC7700_GATE_CLK_PKA_CFG                                339
+#define EIC7700_GATE_CLK_SPACC_CFG                              340
+#define EIC7700_GATE_CLK_CRYPTO                                 341
+#define EIC7700_GATE_CLK_TRNG_CFG                               342
+#define EIC7700_GATE_CLK_OTP_CFG                                343
+#define EIC7700_GATE_CLMM_CFG_CLK                               344
+#define EIC7700_GATE_CLMM_DEB_CLK                               345
+#define EIC7700_GATE_CLK_MAILBOX_0                              346
+#define EIC7700_GATE_CLK_MAILBOX_1                              347
+#define EIC7700_GATE_CLK_MAILBOX_2                              348
+#define EIC7700_GATE_CLK_MAILBOX_3                              349
+#define EIC7700_GATE_CLK_MAILBOX_4                              350
+#define EIC7700_GATE_CLK_MAILBOX_5                              351
+#define EIC7700_GATE_CLK_MAILBOX_6                              352
+#define EIC7700_GATE_CLK_MAILBOX_7                              353
+#define EIC7700_GATE_CLK_MAILBOX_8                              354
+#define EIC7700_GATE_CLK_MAILBOX_9                              355
+#define EIC7700_GATE_CLK_MAILBOX_10                             356
+#define EIC7700_GATE_CLK_MAILBOX_11                             357
+#define EIC7700_GATE_CLK_MAILBOX_12                             358
+#define EIC7700_GATE_CLK_MAILBOX_13                             359
+#define EIC7700_GATE_CLK_MAILBOX_14                             360
+#define EIC7700_GATE_CLK_MAILBOX_15                             361
+#define EIC7700_GATE_CLK_APLL_TEST_OUT                          362
+#define EIC7700_GATE_CLK_CPLL_TEST_OUT                          363
+#define EIC7700_GATE_CLK_HSP_DFT150M                            364
+#define EIC7700_GATE_CLK_HSP_DFT300M                            365
+#define EIC7700_GATE_CLK_HSP_DFT600M                            366
+#define EIC7700_GATE_CLK_VI_DFT400M                             367
+#define EIC7700_GATE_CLK_VI_DFT500M                             368
+#define EIC7700_GATE_CLK_VO_DFT300M                             369
+#define EIC7700_GATE_CLK_VO_DFT600M                             370
+#define EIC7700_GATE_CLK_D2D_DFT300M                            371
+#define EIC7700_GATE_CLK_D2D_DFT600M                            372
+#define EIC7700_GATE_CLK_PCIE_DFT125M                           373
+#define EIC7700_GATE_CLK_PCIE_DFT200M                           374
+#define EIC7700_GATE_CLK_DDR_PLL_BYP_CLK                        375
+#define EIC7700_GATE_CLK_DDR_RX_TEST_CLK                        376
+#define EIC7700_GATE_LSP_I2C0_PCLK                              377
+#define EIC7700_GATE_LSP_I2C1_PCLK                              378
+#define EIC7700_GATE_LSP_I2C2_PCLK                              379
+#define EIC7700_GATE_LSP_I2C3_PCLK                              380
+#define EIC7700_GATE_LSP_I2C4_PCLK                              381
+#define EIC7700_GATE_LSP_I2C5_PCLK                              382
+#define EIC7700_GATE_LSP_I2C6_PCLK                              383
+#define EIC7700_GATE_LSP_I2C7_PCLK                              384
+#define EIC7700_GATE_LSP_I2C8_PCLK                              385
+#define EIC7700_GATE_LSP_I2C9_PCLK                              386
+#define EIC7700_GATE_LSP_WDT0_PCLK                              387
+#define EIC7700_GATE_LSP_WDT1_PCLK                              388
+#define EIC7700_GATE_LSP_WDT2_PCLK                              389
+#define EIC7700_GATE_LSP_WDT3_PCLK                              390
+#define EIC7700_GATE_LSP_SSI0_PCLK                              391
+#define EIC7700_GATE_LSP_SSI1_PCLK                              392
+#define EIC7700_GATE_LSP_PVT_PCLK                               393
+#define EIC7700_GATE_AON_I2C0_PCLK                              394
+#define EIC7700_GATE_AON_I2C1_PCLK                              395
+#define EIC7700_GATE_LSP_UART0_PCLK                             396
+#define EIC7700_GATE_LSP_UART1_PCLK                             397
+#define EIC7700_GATE_LSP_UART2_PCLK                             398
+#define EIC7700_GATE_LSP_UART3_PCLK                             399
+#define EIC7700_GATE_LSP_UART4_PCLK                             400
+#define EIC7700_GATE_LSP_TIMER_PCLK                             401
+#define EIC7700_GATE_LSP_FAN_PCLK                               402
+#define EIC7700_GATE_LSP_PVT0_CLK                               403
+#define EIC7700_GATE_LSP_PVT1_CLK                               404
+#define EIC7700_GATE_RESERVED_1                                 405
+#define EIC7700_GATE_RESERVED_2                                 406
+#define EIC7700_GATE_RESERVED_3                                 407
+#define EIC7700_GATE_VC_JE_PCLK                                 408
+#define EIC7700_GATE_VC_JD_PCLK                                 409
+#define EIC7700_GATE_VC_VE_PCLK                                 410
+#define EIC7700_GATE_VC_VD_PCLK                                 411
+#define EIC7700_GATE_VC_MON_PCLK                                412
+#define EIC7700_GATE_HSP_DMA0_CLK                               413
+
+/*fixed factor clocks*/
+#define EIC7700_FIXED_FACTOR_U_CPU_DIV2                         450
+#define EIC7700_FIXED_FACTOR_U_CLK_1M_DIV24                     451
+#define EIC7700_FIXED_FACTOR_U_MIPI_TXESC_DIV10                 452
+#define EIC7700_FIXED_FACTOR_U_U84_CORE_LP_DIV2                 453
+#define EIC7700_FIXED_FACTOR_U_SCPU_BUS_DIV2                    454
+#define EIC7700_FIXED_FACTOR_U_LPCPU_BUS_DIV2                   455
+#define EIC7700_FIXED_FACTOR_U_PCIE_CR_DIV2                     456
+#define EIC7700_FIXED_FACTOR_U_PCIE_AUX_DIV4                    457
+#define EIC7700_FIXED_FACTOR_U_PVT_DIV20                        458
+#define EIC7700_FIXED_FACTOR_U_DFT100M_DIV4                     459
+#define EIC7700_FIXED_FACTOR_U_DFT125M_DIV2                     460
+#define EIC7700_FIXED_FACTOR_U_DFT150M_DIV2                     461
+#define EIC7700_FIXED_FACTOR_U_DFT100M_DIV2                     462
+#define EIC7700_FIXED_FACTOR_U_DFT500M_DIV3                     463
+#define EIC7700_FIXED_FACTOR_U_DFT500M_DIV2                     464
+#define EIC7700_FIXED_FACTOR_SPLL0_TEST_DIV8                    465
+#define EIC7700_FIXED_FACTOR_SPLL1_TEST_DIV6                    466
+#define EIC7700_FIXED_FACTOR_SPLL2_TEST_DIV4                    467
+#define EIC7700_FIXED_FACTOR_U_HSP_RMII_REF_DIV6                468
+#define EIC7700_FIXED_FACTOR_U_DRR_DIV8                         469
+
+
+/*clocks list for consumer*/
+#define EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_0                      500
+#define EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_1                      501
+#define EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_2                      502
+#define EIC7700_CLK_CPU_EXT_SRC_CORE_CLK_3                      503
+#define EIC7700_CLK_CPU_TRACE_CLK_0                             504
+#define EIC7700_CLK_CPU_TRACE_CLK_1                             505
+#define EIC7700_CLK_CPU_TRACE_CLK_2                             506
+#define EIC7700_CLK_CPU_TRACE_CLK_3                             507
+#define EIC7700_CLK_CPU_DEBUG_CLK                               508
+#define EIC7700_CLK_CPU_TRACE_COM_CLK                           509
+#define EIC7700_CLK_CPU_CLK                                     510
+#define EIC7700_CLK_CLK_1M                                      511
+#define EIC7700_CLK_CLK_SYS_CFG                                 512
+#define EIC7700_CLK_CLK_MIPI_TXESC                              513
+#define EIC7700_CLK_NOC_CFG_CLK                                 514
+#define EIC7700_CLK_NOC_NSP_CLK                                 515
+#define EIC7700_CLK_CLK_BOOTSPI                                 516
+#define EIC7700_CLK_CLK_BOOTSPI_CFG                             517
+#define EIC7700_CLK_CLK_U84_CORE_LP                             518
+#define EIC7700_CLK_CLK_SCPU_CORE                               519
+#define EIC7700_CLK_CLK_SCPU_BUS                                520
+#define EIC7700_CLK_CLK_LPCPU_CORE                              521
+#define EIC7700_CLK_CLK_LPCPU_BUS                               522
+#define EIC7700_CLK_GPU_ACLK                                    523
+#define EIC7700_CLK_GPU_GRAY_CLK                                524
+#define EIC7700_CLK_GPU_CFG_CLK                                 525
+#define EIC7700_CLK_CLK_DSP_ROOT                                526
+#define EIC7700_CLK_DSPT_ACLK                                   527
+#define EIC7700_CLK_DSPT_CFG_CLK                                528
+#define EIC7700_CLK_CLK_D2DDR_ACLK                              529
+#define EIC7700_CLK_D2D_ACLK                                    530
+#define EIC7700_CLK_D2D_CFG_CLK                                 531
+#define EIC7700_CLK_TCU_ACLK                                    532
+#define EIC7700_CLK_TCU_CFG_CLK                                 533
+#define EIC7700_CLK_DDRT_CFG_CLK                                534
+#define EIC7700_CLK_DDRT0_P0_ACLK                               535
+#define EIC7700_CLK_DDRT0_P1_ACLK                               536
+#define EIC7700_CLK_DDRT0_P2_ACLK                               537
+#define EIC7700_CLK_DDRT0_P3_ACLK                               538
+#define EIC7700_CLK_DDRT0_P4_ACLK                               539
+#define EIC7700_CLK_DDRT1_P0_ACLK                               540
+#define EIC7700_CLK_DDRT1_P1_ACLK                               541
+#define EIC7700_CLK_DDRT1_P2_ACLK                               542
+#define EIC7700_CLK_DDRT1_P3_ACLK                               543
+#define EIC7700_CLK_DDRT1_P4_ACLK                               544
+#define EIC7700_CLK_HSP_ACLK                                    545
+#define EIC7700_CLK_HSP_CFG_CLK                                 546
+#define EIC7700_CLK_HSP_SATA_RBC_CLK                            547
+#define EIC7700_CLK_HSP_SATA_OOB_CLK                            548
+#define EIC7700_CLK_HSP_SATA_PMALIVE_CLK                        549
+#define EIC7700_CLK_HSP_ETH_APP_CLK                             550
+#define EIC7700_CLK_HSP_ETH_CSR_CLK                             551
+#define EIC7700_CLK_HSP_ETH0_CORE_CLK                           552
+#define EIC7700_CLK_HSP_ETH1_CORE_CLK                           553
+#define EIC7700_CLK_HSP_MSHC0_CORE_CLK                          554
+#define EIC7700_CLK_HSP_MSHC1_CORE_CLK                          555
+#define EIC7700_CLK_HSP_MSHC2_CORE_CLK                          556
+#define EIC7700_CLK_HSP_MSHC0_TMR_CLK                           557
+#define EIC7700_CLK_HSP_MSHC1_TMR_CLK                           558
+#define EIC7700_CLK_HSP_MSHC2_TMR_CLK                           559
+#define EIC7700_CLK_HSP_USB0_SUSPEND_CLK                        560
+#define EIC7700_CLK_HSP_USB1_SUSPEND_CLK                        561
+#define EIC7700_CLK_PCIET_ACLK                                  562
+#define EIC7700_CLK_PCIET_CFG_CLK                               563
+#define EIC7700_CLK_PCIET_CR_CLK                                564
+#define EIC7700_CLK_PCIET_AUX_CLK                               565
+#define EIC7700_CLK_NPU_ACLK                                    566
+#define EIC7700_CLK_NPU_CFG_CLK                                 567
+#define EIC7700_CLK_CLK_NPU_LLC_SRC0                            568
+#define EIC7700_CLK_CLK_NPU_LLC_SRC1                            569
+#define EIC7700_CLK_NPU_LLC_ACLK                                570
+#define EIC7700_CLK_CLK_NPU_CORE_ST1                            571
+#define EIC7700_CLK_NPU_CLK                                     572
+#define EIC7700_CLK_NPU_E31_CLK                                 573
+#define EIC7700_CLK_CLK_VI_ACLK_ST1                             574
+#define EIC7700_CLK_VI_ACLK                                     575
+#define EIC7700_CLK_VI_DVP_CLK                                  576
+#define EIC7700_CLK_VI_CFG_CLK                                  577
+#define EIC7700_CLK_VI_DIG_DW_CLK                               578
+#define EIC7700_CLK_VI_DIG_ISP_CLK                              579
+#define EIC7700_CLK_VI_SHUTTER_0                                580
+#define EIC7700_CLK_VI_SHUTTER_1                                581
+#define EIC7700_CLK_VI_SHUTTER_2                                582
+#define EIC7700_CLK_VI_SHUTTER_3                                583
+#define EIC7700_CLK_VI_SHUTTER_4                                584
+#define EIC7700_CLK_VI_SHUTTER_5                                585
+#define EIC7700_CLK_VI_PHY_TXCLKESC                             586
+#define EIC7700_CLK_VI_PHY_CFG                                  587
+#define EIC7700_CLK_VO_ACLK                                     588
+#define EIC7700_CLK_VO_CFG_CLK                                  589
+#define EIC7700_CLK_VO_HDMI_IESMCLK                             590
+#define EIC7700_CLK_VO_PIXEL_CLK                                591
+#define EIC7700_CLK_VO_I2S_MCLK                                 592
+#define EIC7700_CLK_VO_CR_CLK                                   593
+#define EIC7700_CLK_VO_CEC_CLK                                  594
+#define EIC7700_CLK_CLK_VC_ROOT                                 595
+#define EIC7700_CLK_VC_ACLK                                     596
+#define EIC7700_CLK_VC_CFG_CLK                                  597
+#define EIC7700_CLK_VC_JE_CLK                                   598
+#define EIC7700_CLK_VC_JD_CLK                                   599
+#define EIC7700_CLK_VC_VE_CLK                                   600
+#define EIC7700_CLK_VC_VD_CLK                                   601
+#define EIC7700_CLK_G2D_CFG_CLK                                 602
+#define EIC7700_CLK_CLK_G2D_ST2                                 603
+#define EIC7700_CLK_G2D_CLK                                     604
+#define EIC7700_CLK_G2D_ACLK                                    605
+#define EIC7700_CLK_CLK_RESERVED                                606
+#define EIC7700_CLK_PVT_CLK_0                                   607
+#define EIC7700_CLK_PVT_CLK_1                                   608
+#define EIC7700_CLK_HSP_RMII_REF_0                              609
+#define EIC7700_CLK_HSP_RMII_REF_1                              610
+#define EIC7700_CLK_HSP_SATA_PHY_REF                            611
+#define EIC7700_CLK_AONDMA_CFG                                  612
+#define EIC7700_CLK_CLK_AONDMA_AXI_ST3                          613
+#define EIC7700_CLK_AONDMA_ACLK                                 614
+#define EIC7700_CLK_AON_ACLK                                    615
+#define EIC7700_CLK_TIMER_CLK_0                                 616      //AON timer
+#define EIC7700_CLK_TIMER_CLK_1                                 617
+#define EIC7700_CLK_TIMER_CLK_2                                 618
+#define EIC7700_CLK_TIMER_CLK_3                                 619
+#define EIC7700_CLK_TIMER_PCLK_0                                620
+#define EIC7700_CLK_TIMER_PCLK_1                                621
+#define EIC7700_CLK_TIMER_PCLK_2                                622
+#define EIC7700_CLK_TIMER_PCLK_3                                623
+#define EIC7700_CLK_TIMER3_CLK8                                 624
+#define EIC7700_CLK_CLK_RTC_CFG                                 625  // AON rtc
+#define EIC7700_CLK_CLK_RTC                                     626  // AON rtc
+#define EIC7700_CLK_CLK_U84_RTC_TOGGLE                          627
+#define EIC7700_CLK_UNUSED_1                                    628
+#define EIC7700_CLK_CLK_PKA_CFG                                 629
+#define EIC7700_CLK_CLK_SPACC_CFG                               630
+#define EIC7700_CLK_CLK_CRYPTO                                  631
+#define EIC7700_CLK_CLK_TRNG_CFG                                632
+#define EIC7700_CLK_CLK_OTP_CFG                                 633
+#define EIC7700_CLK_CLMM_CFG_CLK                                634
+#define EIC7700_CLK_CLMM_DEB_CLK                                635
+#define EIC7700_CLK_DDR_PLL_BYP_CLK                             636
+#define EIC7700_CLK_DDR_PLL_REF_AND_DFI_CLK                     637
+#define EIC7700_CLK_DDR_RX_TEST_CLK                             638
+#define EIC7700_CLK_MAILBOX_0                                   638
+#define EIC7700_CLK_MAILBOX_1                                   639
+#define EIC7700_CLK_MAILBOX_2                                   640
+#define EIC7700_CLK_MAILBOX_3                                   641
+#define EIC7700_CLK_MAILBOX_4                                   642
+#define EIC7700_CLK_MAILBOX_5                                   643
+#define EIC7700_CLK_MAILBOX_6                                   644
+#define EIC7700_CLK_MAILBOX_7                                   645
+#define EIC7700_CLK_MAILBOX_8                                   646
+#define EIC7700_CLK_MAILBOX_9                                   647
+#define EIC7700_CLK_MAILBOX_10                                  648
+#define EIC7700_CLK_MAILBOX_11                                  649
+#define EIC7700_CLK_MAILBOX_12                                  650
+#define EIC7700_CLK_MAILBOX_13                                  651
+#define EIC7700_CLK_MAILBOX_14                                  652
+#define EIC7700_CLK_MAILBOX_15                                  653
+#define EIC7700_CLK_LSP_I2C0_PCLK                               654
+#define EIC7700_CLK_LSP_I2C1_PCLK                               655
+#define EIC7700_CLK_LSP_I2C2_PCLK                               656
+#define EIC7700_CLK_LSP_I2C3_PCLK                               657
+#define EIC7700_CLK_LSP_I2C4_PCLK                               658
+#define EIC7700_CLK_LSP_I2C5_PCLK                               659
+#define EIC7700_CLK_LSP_I2C6_PCLK                               660
+#define EIC7700_CLK_LSP_I2C7_PCLK                               661
+#define EIC7700_CLK_LSP_I2C8_PCLK                               662
+#define EIC7700_CLK_LSP_I2C9_PCLK                               663
+#define EIC7700_CLK_LSP_WDT0_PCLK                               664
+#define EIC7700_CLK_LSP_WDT1_PCLK                               665
+#define EIC7700_CLK_LSP_WDT2_PCLK                               666
+#define EIC7700_CLK_LSP_WDT3_PCLK                               667
+#define EIC7700_CLK_LSP_SSI0_PCLK                               668
+#define EIC7700_CLK_LSP_SSI1_PCLK                               669
+#define EIC7700_CLK_LSP_PVT_PCLK                                670
+#define EIC7700_CLK_AON_I2C0_PCLK                               671
+#define EIC7700_CLK_AON_I2C1_PCLK                               672
+#define EIC7700_CLK_LSP_UART0_PCLK                              673
+#define EIC7700_CLK_LSP_UART1_PCLK                              674
+#define EIC7700_CLK_LSP_UART2_PCLK                              675
+#define EIC7700_CLK_LSP_UART3_PCLK                              676
+#define EIC7700_CLK_LSP_UART4_PCLK                              677
+#define EIC7700_CLK_LSP_TIMER_PCLK                              678         //LSP timer
+#define EIC7700_CLK_LSP_FAN_PCLK                                679
+#define EIC7700_CLK_DSP_ACLK_0                                  680
+#define EIC7700_CLK_DSP_ACLK_1                                  681
+#define EIC7700_CLK_DSP_ACLK_2                                  682
+#define EIC7700_CLK_DSP_ACLK_3                                  683
+
+#define EIC7700_CLK_VC_JE_PCLK                                  685
+#define EIC7700_CLK_VC_JD_PCLK                                  686
+#define EIC7700_CLK_VC_VE_PCLK                                  687
+#define EIC7700_CLK_VC_VD_PCLK                                  688
+#define EIC7700_CLK_VC_MON_PCLK                                 689
+
+#define EIC7700_CLK_HSP_DMA0_CLK                                690
+
+#define EIC7700_NR_CLKS                                         700
+
+/* run frquency */
+#define CLK_FREQ_1800M					1800000000
+#define CLK_FREQ_1700M					1700000000
+#define CLK_FREQ_1600M					1600000000
+#define CLK_FREQ_1500M					1500000000
+#define CLK_FREQ_1400M					1400000000
+#define CLK_FREQ_1300M					1300000000
+#define CLK_FREQ_1200M					1200000000
+#define CLK_FREQ_1000M					1000000000
+#define CLK_FREQ_900M					900000000
+#define CLK_FREQ_800M					800000000
+#define CLK_FREQ_700M					700000000
+#define CLK_FREQ_600M					600000000
+#define CLK_FREQ_500M					500000000
+#define CLK_FREQ_400M					400000000
+#define CLK_FREQ_200M					200000000
+#define CLK_FREQ_100M					100000000
+#define CLK_FREQ_24M					24000000
+
+#define APLL_HIGH_FREQ					983040000
+#define APLL_LOW_FREQ 					225792000
+
+#endif /*endif __DTS_EIC7700_CLOCK_H*/
diff --git a/include/dt-bindings/interconnect/eic7700.h b/include/dt-bindings/interconnect/eic7700.h
new file mode 100644
index 000000000000..984b7cc0840f
--- /dev/null
+++ b/include/dt-bindings/interconnect/eic7700.h
@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Interconnect driver for Eswin EIC7700 SoC
+ *
+ * Copyright (C) 2022  Beiging Eswin Co. Ltd
+ * Author: Huangyifeng <huangyifeng@eswincomputing.com>
+ */
+
+#ifndef _DT_BINDINGS_INTERCONNECT_EIC7700_H_
+#define _DT_BINDINGS_INTERCONNECT_EIC7700_H_
+
+#define OFFSET0                              (0)
+#define OFFSET1                              (1)
+#define OFFSET2                              (2)
+#define OFFSET3                              (3)
+#define OFFSET4                              (4)
+#define OFFSET5                              (5)
+#define OFFSET6                              (6)
+#define OFFSET7                              (7)
+#define OFFSET8                              (8)
+#define OFFSET9                              (9)
+#define OFFSET10                             (10)
+#define OFFSET11                             (11)
+#define OFFSET12                             (12)
+#define OFFSET13                             (13)
+#define OFFSET14                             (14)
+#define OFFSET15                             (15)
+#define OFFSET16                             (16)
+#define OFFSET17                             (17)
+#define OFFSET18                             (18)
+#define OFFSET19                             (19)
+#define OFFSET20                             (20)
+#define OFFSET21                             (21)
+#define OFFSET22                             (22)
+#define OFFSET23                             (23)
+
+#define OFFSET31                             (31)
+
+/*sideband manager module id defination*/
+/*sys noc*/
+#define SBM_AON_SNOC_SP0	0
+#define SBM_DSPT_SNOC		1
+#define SBM_JTAG_SNOC		2
+#define SBM_MCPUT_SNOC_D2D	3
+#define SBM_MCPUT_SNOC_MP	4
+#define SBM_MCPUT_SNOC_SP0	5
+#define SBM_MCPUT_SNOC_SP1	6
+#define SBM_NPU_SNOC_SP0	7
+#define SBM_NPU_SNOC_SP1	8
+#define SBM_PCIET_SNOC_P	9
+#define SBM_SPISLV_PCIET_SNOC	10
+#define SBM_TBU4_SNOC		11
+#define SBM_TCU_SNOC		12
+#define SBM_SNOC_AON		13
+#define SBM_SNOC_DDR0_P1	14
+#define SBM_SNOC_DDR0_P2	15
+#define SBM_SNOC_DDR1_P1	16
+#define SBM_SNOC_DDR1_P2	17
+#define SBM_SNOC_DSPT		18
+#define SBM_SNOC_MCPUT_D2D	19
+#define SBM_SNOC_NPU		20
+#define SBM_SNOC_PCIET		21
+
+/*cfg noc*/
+#define SBM_CLMM		30
+#define SBM_CNOC_AON		31
+#define SBM_CNOC_DDRT0_CTRL	32
+#define SBM_CNOC_DDRT0_PHY	33
+#define SBM_CNOC_DDRT1_CTRL	34
+#define SBM_CNOC_DDRT1_PHY	35
+#define SBM_CNOC_DSPT		36
+#define SBM_CNOC_GPU		37
+#define SBM_CNOC_HSP		38
+#define SBM_CNOC_LSP_APB2	39
+#define SBM_CNOC_LSP_APB3	40
+#define SBM_CNOC_LSP_APB4	41
+#define SBM_CNOC_LSP_APB6	42
+#define SBM_CNOC_MCPUT_D2D	43
+#define SBM_CNOC_NPU		44
+#define SBM_CNOC_PCIET_P	45
+#define SBM_CNOC_PCIET_X	46
+#define SBM_CNOC_TCU		47
+#define SBM_CNOC_VC		48
+#define SBM_CNOC_VI		49
+#define SBM_CNOC_VO		50
+
+/*llc noc*/
+#define SBM_LNOC_NPU_LLC0		60
+#define SBM_LNOC_NPU_LLC1		61
+#define SBM_LNOC_DDRT0_P0		62
+#define SBM_LNOC_DDRT1_P0		63
+
+/*media noc*/
+#define SBM_MNOC_GPU			70
+#define SBM_MNOC_TBU2			71
+#define SBM_MNOC_VC			72
+#define SBM_MNOC_DDRT0_P3		73
+#define SBM_MNOC_DDRT1_P3		74
+
+/*realtime noc*/
+#define SBM_RNOC_TBU0			80
+#define SBM_RNOC_VO			81
+#define SBM_RNOC_DDRT0_P4		82
+#define SBM_RNOC_DDRT1_P4		83
+
+/*RouteID defination*/
+#define        aon_snoc_sp0_I_O        0x0
+#define        dspt_snoc_I_O           0x1
+#define        jtag_snoc_I_O           0x2
+#define        mcput_snoc_d2d_I_O      0x3
+#define        mcput_snoc_mp_I_O       0x4
+#define        mcput_snoc_sp0_I_O      0x5
+#define        mcput_snoc_sp1_I_O      0x6
+#define        mnoc_snoc_I_O           0x7
+#define        npu_snoc_sp0_I_O        0x8
+#define        npu_snoc_sp1_I_O        0x9
+#define        pciet_snoc_p_I_O        0xA
+#define        rnoc_snoc_I_O           0xB
+#define        spislv_tbu3_snoc_I_O    0xC
+#define        tbu4_snoc_I_O           0xD
+#define        tcu_snoc_I_O            0xE
+#define        RESERVED0               0xF
+
+#define        snoc_aon_T_O            0x0
+#define        snoc_cnoc_T_O           0x1
+#define        snoc_ddrt0_p1_T_O       0x2
+#define        snoc_ddrt0_p2_T_O       0x3
+#define        snoc_ddrt1_p1_T_O       0x4
+#define        snoc_ddrt1_p2_T_O       0x5
+#define        snoc_dspt_T_O           0x6
+#define        snoc_lnoc_T_O           0x7
+#define        snoc_mcput_d2d_T_O      0x8
+#define        snoc_mnoc_T_O           0x9
+#define        snoc_npu_T_O            0xA
+#define        snoc_pciet_T_O          0xB
+#define        snoc_rnoc_T_O           0xC
+#define        snoc_service_T_O        0xD
+#define        RESERVED1               0xE
+#define        RESERVED2               0xF
+
+#endif /* _DT_BINDINGS_INTERCONNECT_EIC7700_H_ */
diff --git a/include/dt-bindings/mailbox/eswin-mailbox.h b/include/dt-bindings/mailbox/eswin-mailbox.h
new file mode 100755
index 000000000000..1c82418dbadc
--- /dev/null
+++ b/include/dt-bindings/mailbox/eswin-mailbox.h
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN Mailbox Driver
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef _DTS_ESWIN_MAILBOX_H_
+#define _DTS_ESWIN_MAILBOX_H_
+
+#define ESWIN_MAILBOX_U84_TO_SCPU_REG_BASE   0x50a00000 /*maibox 0*/
+#define ESWIN_MAILBOX_SCPU_TO_U84_REG_BASE   0x50a10000 /*maibox 1*/
+
+#define ESWIN_MAILBOX_U84_TO_LPCPU_REG_BASE  0x50a20000 /*maibox 2*/
+#define ESWIN_MAILBOX_LPCPU_TO_U84_REG_BASE  0x50a30000 /*maibox 3*/
+
+#define ESWIN_MAILBOX_U84_TO_NPU_0_REG_BASE  0x50a40000 /*maibox 4*/
+#define ESWIN_MAILBOX_NPU_0_TO_U84_REG_BASE  0x50a50000 /*maibox 5*/
+
+#define ESWIN_MAILBOX_U84_TO_NPU_1_REG_BASE  0x50a60000 /*maibox 6*/
+#define ESWIN_MAILBOX_NP1_0_TO_U84_REG_BASE  0x50a70000 /*maibox 7*/
+
+#define ESWIN_MAILBOX_U84_TO_DSP_0_REG_BASE  0x50a80000 /*maibox 8*/
+#define ESWIN_MAILBOX_DSP_0_TO_U84_REG_BASE  0x50a90000 /*maibox 9*/
+
+#define ESWIN_MAILBOX_U84_TO_DSP_1_REG_BASE  0x50aa0000 /*maibox 10*/
+#define ESWIN_MAILBOX_DSP_1_TO_U84_REG_BASE  0x50ab0000 /*maibox 11*/
+
+#define ESWIN_MAILBOX_U84_TO_DSP_2_REG_BASE  0x50ac0000 /*maibox 12*/
+#define ESWIN_MAILBOX_DSP_2_TO_U84_REG_BASE  0x50ad0000 /*maibox 13*/
+
+#define ESWIN_MAILBOX_U84_TO_DSP_3_REG_BASE  0x50ae0000 /*maibox 14*/
+#define ESWIN_MAILBOX_DSP_3_TO_U84_REG_BASE  0x50af0000 /*maibox 15*/
+
+#define BIT0                              (1 << 0)
+#define BIT1                              (1 << 1)
+#define BIT2                              (1 << 2)
+#define BIT3                              (1 << 3)
+#define BIT4                              (1 << 4)
+#define BIT5                              (1 << 5)
+#define BIT6                              (1 << 6)
+#define BIT7                              (1 << 7)
+#define BIT8                              (1 << 8)
+#define BIT9                              (1 << 9)
+#define BIT10                             (1 << 10)
+#define BIT11                             (1 << 11)
+#define BIT12                             (1 << 12)
+#define BIT13                             (1 << 13)
+#define BIT14                             (1 << 14)
+#define BIT31                             (1 << 31)
+
+#define ESWIN_MAILBOX_WR_LOCK_BIT_U84      BIT0
+#define ESWIN_MAILBOX_WR_LOCK_BIT_SCPU     BIT1
+#define ESWIN_MAILBOX_WR_LOCK_BIT_LPCPU    BIT2
+#define ESWIN_MAILBOX_WR_LOCK_BIT_NPU_0    BIT3
+#define ESWIN_MAILBOX_WR_LOCK_BIT_NPU_1    BIT4
+#define ESWIN_MAILBOX_WR_LOCK_BIT_DSP_0    BIT5
+#define ESWIN_MAILBOX_WR_LOCK_BIT_DSP_1    BIT6
+#define ESWIN_MAILBOX_WR_LOCK_BIT_DSP_2    BIT7
+#define ESWIN_MAILBOX_WR_LOCK_BIT_DSP_3    BIT8
+
+
+#define ESWIN_MAIBOX_U84_IRQ_BIT           BIT0
+#define ESWIN_MAIBOX_SCPU_IRQ_BIT          BIT1
+#define ESWIN_MAIBOX_LPCPU_IRQ_BIT         BIT2
+#define ESWIN_MAIBOX_NPU_0_IRQ_BIT         BIT3
+#define ESWIN_MAIBOX_NPU_1_IRQ_BIT         BIT4
+#define ESWIN_MAIBOX_DSP_0_IRQ_BIT         BIT5
+#define ESWIN_MAIBOX_DSP_1_IRQ_BIT         BIT6
+#define ESWIN_MAIBOX_DSP_2_IRQ_BIT         BIT7
+#define ESWIN_MAIBOX_DSP_3_IRQ_BIT         BIT8
+
+#endif /* _DTS_ESWIN_MAILBOX_H_ */
diff --git a/include/dt-bindings/memory/eic7700-sid.h b/include/dt-bindings/memory/eic7700-sid.h
new file mode 100644
index 000000000000..0fa589ec93b1
--- /dev/null
+++ b/include/dt-bindings/memory/eic7700-sid.h
@@ -0,0 +1,138 @@
+#ifndef DT_BINDINGS_MEMORY_EIC7700_SID_H
+#define DT_BINDINGS_MEMORY_EIC7700_SID_H
+
+#define EIC7700_SID_DMA0	1
+
+#define EIC7700_SID_JDEC	2
+
+#define EIC7700_SID_JENC	3
+
+/* NPU DMA*/
+#define EIC7700_SID_NPU_DMA	4
+
+/* NPU-E31 */
+#define EIC7700_SID_NPU_E31	5
+
+/* Video In */
+#define EIC7700_SID_ISP0	6
+
+//#define EIC7700_SID_ISP1	EIC7700_SID_ISP0
+
+#define EIC7700_SID_DW		8
+
+#define EIC7700_SID_DVP		9
+
+/* High Speed */
+#define EIC7700_SID_USB0	10
+
+#define EIC7700_SID_USB1	11
+
+#define EIC7700_SID_ETH0	12
+
+#define EIC7700_SID_ETH1	13
+
+#define EIC7700_SID_SATA	14
+
+#define EIC7700_SID_EMMC0	15
+
+#define EIC7700_SID_SD0		16
+
+#define EIC7700_SID_SD1		17
+
+
+/* DSP */
+#define EIC7700_SID_DSP_0	18
+#define EIC7700_SID_DSP_1	19
+#define EIC7700_SID_DSP_2	20
+#define EIC7700_SID_DSP_3	21
+
+/* CODEC */
+//#define EIC7700_SID_VDEC	EIC7700_SID_JDEC
+
+//#define EIC7700_SID_VENC	EIC7700_SID_JENC
+
+/*** AON subsystem ***/
+/* Secure CPU */
+#define EIC7700_SID_SCPU	24
+#define SCPU_SID_REG_OFFSET	0x1004
+
+/* Low power CPU */
+#define EIC7700_SID_LCPU	25
+#define LCPU_SID_REG_OFFSET	0x2004
+
+/* Always on, DMA1 */
+#define EIC7700_SID_DMA1	26
+#define DMA1_SID_REG_OFFSET	0x3004
+
+/* crypt */
+//#define EIC7700_SID_CRYPT	EIC7700_SID_SCPU
+#define CRYPT_SID_REG_OFFSET	0x4004
+
+///*** for iova mapping test ***/
+//#define EIC7700_SID_DEV_FOO_A        28
+//#define EIC7700_SID_DEV_FOO_B        29
+//#define EIC7700_SID_DEV_FOO_FOR_DIE1	30
+
+
+/*** tbu id ***/
+/* tbu_id: bit[3:0] is for major, bit[7:4] is for minor;
+	For example, tbu of dsp3 is tbu7_3, the bu 0x73. It measn tbu7_3
+*/
+#define	EIC7700_TBUID_0x0	0x0
+
+#define	EIC7700_TBUID_0x10	0x10
+#define	EIC7700_TBUID_0x11	0x11
+#define	EIC7700_TBUID_0x12	0x12
+#define	EIC7700_TBUID_0x13	0x13
+
+#define	EIC7700_TBUID_0x2	0x2
+
+#define	EIC7700_TBUID_0x3	0x3
+
+#define	EIC7700_TBUID_0x4	0x4
+
+#define	EIC7700_TBUID_0x5	0x5
+
+#define	EIC7700_TBUID_0x70	0x70
+#define	EIC7700_TBUID_0x71	0x71
+#define	EIC7700_TBUID_0x72	0x72
+#define	EIC7700_TBUID_0x73	0x73
+
+#define	EIC7700_TBUID_0xF00	0xF00	// simulation for EIC7700_SID_DEV_FOO_A/B, No real tbu attached infact
+
+
+/* For better use by devices in dts, create tbu alias for devices*/
+#define EIC7700_TBUID_ISP	EIC7700_TBUID_0x0
+#define EIC7700_TBUID_DW	EIC7700_TBUID_ISP
+
+#define EIC7700_TBUID_VDEC	EIC7700_TBUID_0x10
+#define EIC7700_TBUID_VENC	EIC7700_TBUID_0x11
+#define EIC7700_TBUID_JENC	EIC7700_TBUID_0x12
+#define EIC7700_TBUID_JDEC	EIC7700_TBUID_0x13
+
+//high speed modules share the same tbu2
+#define EIC7700_TBUID_DMA0	EIC7700_TBUID_0x2
+#define EIC7700_TBUID_USB	EIC7700_TBUID_DMA0
+#define EIC7700_TBUID_ETH	EIC7700_TBUID_DMA0
+#define EIC7700_TBUID_SATA	EIC7700_TBUID_DMA0
+#define EIC7700_TBUID_EMMC	EIC7700_TBUID_DMA0
+#define EIC7700_TBUID_SD	EIC7700_TBUID_DMA0
+
+#define EIC7700_TBUID_PCIE	EIC7700_TBUID_0x3
+
+//scpu, crypto, lpcpu, dma1 share the same tbu4
+#define EIC7700_TBUID_SCPU	EIC7700_TBUID_0x4
+#define EIC7700_TBUID_CRYPT	EIC7700_TBUID_SCPU
+#define EIC7700_TBUID_DMA1	EIC7700_TBUID_SCPU
+#define EIC7700_TBUID_LPCPU	EIC7700_TBUID_SCPU
+
+//npu
+#define EIC7700_TBUID_NPU	EIC7700_TBUID_0x5
+
+//dsp
+#define EIC7700_TBUID_DSP0	EIC7700_TBUID_0x70
+#define EIC7700_TBUID_DSP1	EIC7700_TBUID_0x71
+#define EIC7700_TBUID_DSP2	EIC7700_TBUID_0x72
+#define EIC7700_TBUID_DSP3	EIC7700_TBUID_0x73
+
+#endif
diff --git a/include/dt-bindings/reset/eic7700-syscrg.h b/include/dt-bindings/reset/eic7700-syscrg.h
new file mode 100755
index 000000000000..ea1ff1874a85
--- /dev/null
+++ b/include/dt-bindings/reset/eic7700-syscrg.h
@@ -0,0 +1,693 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN SysCrg Definition
+ *
+ * Copyright 2024, Beijing ESWIN Computing Technology Co., Ltd.. All rights reserved.
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * Authors: HuangYiFeng<huangyifeng@eswincomputing.com>
+ */
+
+#ifndef __DT_ESWIN_EIC7700_SYSCRG_H__
+#define __DT_ESWIN_EIC7700_SYSCRG_H__
+
+/*REG OFFSET OF SYS-CRG*/
+
+#define EIC7700_REG_OFFSET_SPLL0_CFG_0                          0X0000
+#define EIC7700_REG_OFFSET_SPLL0_CFG_1                          0X0004
+#define EIC7700_REG_OFFSET_SPLL0_CFG_2                          0X0008
+#define EIC7700_REG_OFFSET_SPLL0_DSKEWCAL                       0X000C
+#define EIC7700_REG_OFFSET_SPLL0_SSC                            0X0010
+#define EIC7700_REG_OFFSET_SPLL1_CFG_0                          0X0014
+#define EIC7700_REG_OFFSET_SPLL1_CFG_1                          0X0018
+#define EIC7700_REG_OFFSET_SPLL1_CFG_2                          0X001C
+#define EIC7700_REG_OFFSET_SPLL1_DSKEWCAL                       0X0020
+#define EIC7700_REG_OFFSET_SPLL1_SSC                            0X0024
+#define EIC7700_REG_OFFSET_SPLL2_CFG_0                          0X0028
+#define EIC7700_REG_OFFSET_SPLL2_CFG_1                          0X002C
+#define EIC7700_REG_OFFSET_SPLL2_CFG_2                          0X0030
+#define EIC7700_REG_OFFSET_SPLL2_DSKEWCAL                       0X0034
+#define EIC7700_REG_OFFSET_SPLL2_SSC                            0X0038
+#define EIC7700_REG_OFFSET_VPLL_CFG_0                           0X003C
+#define EIC7700_REG_OFFSET_VPLL_CFG_1                           0X0040
+#define EIC7700_REG_OFFSET_VPLL_CFG_2                           0X0044
+#define EIC7700_REG_OFFSET_VPLL_DSKEWCAL                        0X0048
+#define EIC7700_REG_OFFSET_VPLL_SSC                             0X004C
+#define EIC7700_REG_OFFSET_APLL_CFG_0                           0X0050
+#define EIC7700_REG_OFFSET_APLL_CFG_1                           0X0054
+#define EIC7700_REG_OFFSET_APLL_CFG_2                           0X0058
+#define EIC7700_REG_OFFSET_APLL_DSKEWCAL                        0X005C
+#define EIC7700_REG_OFFSET_APLL_SSC                             0X0060
+#define EIC7700_REG_OFFSET_MCPUT_PLL_CFG_0                      0X0064
+#define EIC7700_REG_OFFSET_MCPUT_PLL_CFG_1                      0X0068
+#define EIC7700_REG_OFFSET_MCPUT_PLL_CFG_2                      0X006C
+#define EIC7700_REG_OFFSET_MCPUT_PLL_DSKEWCAL                   0X0070
+#define EIC7700_REG_OFFSET_MCPUT_PLL_SSC                        0X0074
+#define EIC7700_REG_OFFSET_DDRT_PLL_CFG_0                       0X0078
+#define EIC7700_REG_OFFSET_DDRT_PLL_CFG_1                       0X007C
+#define EIC7700_REG_OFFSET_DDRT_PLL_CFG_2                       0X0080
+#define EIC7700_REG_OFFSET_DDRT_PLL_DSKEWCAL                    0X0084
+#define EIC7700_REG_OFFSET_DDRT_PLL_SSC                         0X0088
+#define EIC7700_REG_OFFSET_PLL_STATUS                           0X00A4
+#define EIC7700_REG_OFFSET_NOC_CLK_CTRL                         0X100
+#define EIC7700_REG_OFFSET_BOOTSPI_CLK_CTRL                     0X104
+#define EIC7700_REG_OFFSET_BOOTSPI_CFGCLK_CTRL                  0X108
+#define EIC7700_REG_OFFSET_SCPU_CORECLK_CTRL                    0X10C
+#define EIC7700_REG_OFFSET_SCPU_BUSCLK_CTRL                     0X110
+#define EIC7700_REG_OFFSET_LPCPU_CORECLK_CTRL                   0X114
+#define EIC7700_REG_OFFSET_LPCPU_BUSCLK_CTRL                    0X118
+#define EIC7700_REG_OFFSET_TCU_ACLK_CTRL                        0X11C
+#define EIC7700_REG_OFFSET_TCU_CFG_CTRL                         0X120
+#define EIC7700_REG_OFFSET_DDR_CLK_CTRL                         0X124
+#define EIC7700_REG_OFFSET_DDR1_CLK_CTRL                        0X128
+#define EIC7700_REG_OFFSET_GPU_ACLK_CTRL                        0X12C
+#define EIC7700_REG_OFFSET_GPU_CFG_CTRL                         0X130
+#define EIC7700_REG_OFFSET_GPU_GRAY_CTRL                        0X134
+#define EIC7700_REG_OFFSET_DSP_ACLK_CTRL                        0X138
+#define EIC7700_REG_OFFSET_DSP_CFG_CTRL                         0X13C
+#define EIC7700_REG_OFFSET_D2D_ACLK_CTRL                        0X140
+#define EIC7700_REG_OFFSET_D2D_CFG_CTRL                         0X144
+#define EIC7700_REG_OFFSET_HSP_ACLK_CTRL                        0X148
+#define EIC7700_REG_OFFSET_HSP_CFG_CTRL                         0X14C
+#define EIC7700_REG_OFFSET_SATA_RBC_CTRL                        0X150
+#define EIC7700_REG_OFFSET_SATA_OOB_CTRL                        0X154
+#define EIC7700_REG_OFFSET_ETH0_CTRL                            0X158
+#define EIC7700_REG_OFFSET_ETH1_CTRL                            0X15C
+#define EIC7700_REG_OFFSET_MSHC0_CORECLK_CTRL                   0X160
+#define EIC7700_REG_OFFSET_MSHC1_CORECLK_CTRL                   0X164
+#define EIC7700_REG_OFFSET_MSHC2_CORECLK_CTRL                   0X168
+#define EIC7700_REG_OFFSET_MSHC_USB_SLWCLK                      0X16C
+#define EIC7700_REG_OFFSET_PCIE_ACLK_CTRL                       0X170
+#define EIC7700_REG_OFFSET_PCIE_CFG_CTRL                        0X174
+#define EIC7700_REG_OFFSET_NPU_ACLK_CTRL                        0X178
+#define EIC7700_REG_OFFSET_NPU_LLC_CTRL                         0X17C
+#define EIC7700_REG_OFFSET_NPU_CORE_CTRL                        0X180
+#define EIC7700_REG_OFFSET_VI_DWCLK_CTRL                        0X184
+#define EIC7700_REG_OFFSET_VI_ACLK_CTRL                         0X188
+#define EIC7700_REG_OFFSET_VI_DIG_ISP_CLK_CTRL                  0X18C
+#define EIC7700_REG_OFFSET_VI_DVP_CLK_CTRL                      0X190
+#define EIC7700_REG_OFFSET_VI_SHUTTER0                          0X194
+#define EIC7700_REG_OFFSET_VI_SHUTTER1                          0X198
+#define EIC7700_REG_OFFSET_VI_SHUTTER2                          0X19C
+#define EIC7700_REG_OFFSET_VI_SHUTTER3                          0X1A0
+#define EIC7700_REG_OFFSET_VI_SHUTTER4                          0X1A4
+#define EIC7700_REG_OFFSET_VI_SHUTTER5                          0X1A8
+#define EIC7700_REG_OFFSET_VI_PHY_CLKCTRL                       0X1AC
+#define EIC7700_REG_OFFSET_VO_ACLK_CTRL                         0X1B0
+#define EIC7700_REG_OFFSET_VO_IESMCLK_CTRL                      0X1B4
+#define EIC7700_REG_OFFSET_VO_PIXEL_CTRL                        0X1B8
+#define EIC7700_REG_OFFSET_VO_MCLK_CTRL                         0X1BC
+#define EIC7700_REG_OFFSET_VO_PHY_CLKCTRL                       0X1C0
+#define EIC7700_REG_OFFSET_VC_ACLK_CTRL                         0X1C4
+#define EIC7700_REG_OFFSET_VCDEC_ROOTCLK_CTRL                   0X1C8
+#define EIC7700_REG_OFFSET_G2D_CTRL                             0X1CC
+#define EIC7700_REG_OFFSET_VC_CLKEN_CTRL                        0X1D0
+#define EIC7700_REG_OFFSET_JE_CLK_CTRL                          0X1D4
+#define EIC7700_REG_OFFSET_JD_CLK_CTRL                          0X1D8
+#define EIC7700_REG_OFFSET_VD_CLK_CTRL                          0X1DC
+#define EIC7700_REG_OFFSET_VE_CLK_CTRL                          0X1E0
+#define EIC7700_REG_OFFSET_AON_DMA_CLK_CTRL                     0X1E4
+#define EIC7700_REG_OFFSET_TIMER_CLK_CTRL                       0X1E8
+#define EIC7700_REG_OFFSET_RTC_CLK_CTRL                         0X1EC
+#define EIC7700_REG_OFFSET_PKA_CLK_CTRL                         0X1F0
+#define EIC7700_REG_OFFSET_SPACC_CLK_CTRL                       0X1F4
+#define EIC7700_REG_OFFSET_TRNG_CLK_CTRL                        0X1F8
+#define EIC7700_REG_OFFSET_OTP_CLK_CTRL                         0X1FC
+#define EIC7700_REG_OFFSET_LSP_CLK_EN0                          0X200
+#define EIC7700_REG_OFFSET_LSP_CLK_EN1                          0X204
+#define EIC7700_REG_OFFSET_U84_CLK_CTRL                         0X208
+#define EIC7700_REG_OFFSET_SYSCFG_CLK_CTRL                      0X20C
+#define EIC7700_REG_OFFSET_I2C0_CLK_CTRL                        0X210
+#define EIC7700_REG_OFFSET_I2C1_CLK_CTRL                        0X214
+#define EIC7700_REG_OFFSET_DFT_CLK_CTRL                         0X280
+#define EIC7700_REG_OFFSET_SYS_SWRST_VALUE                      0X300
+#define EIC7700_REG_OFFSET_CLR_RST_STATUS                       0X304
+#define EIC7700_REG_OFFSET_DIE_STATUS                           0X308
+#define EIC7700_REG_OFFSET_CLR_BOOT_INFO                        0X30C
+#define EIC7700_REG_OFFSET_SCPU_BOOT_ADDRESS                    0X310
+#define EIC7700_REG_OFFSET_LPCPU_BOOT_ADDRESS                   0X314
+#define EIC7700_REG_OFFSET_NPUE31_BOOT_ADDRESS                  0X318
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS0_HI                 0X31C
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS0_LOW                0X320
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS1_HI                 0X324
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS1_LOW                0X328
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS2_HI                 0X32C
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS2_LOW                0X330
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS3_HI                 0X334
+#define EIC7700_REG_OFFSET_U84_BOOT_ADDRESS3_LOW                0X338
+#define EIC7700_REG_OFFSET_BOOT_SEL_STAT                        0X33C
+#define EIC7700_REG_OFFSET_BOOT_SPI_CFG                         0X340
+#define EIC7700_REG_OFFSET_SNOC_RST_CTRL                        0X400
+#define EIC7700_REG_OFFSET_GPU_RST_CTRL                         0X404
+#define EIC7700_REG_OFFSET_DSP_RST_CTRL                         0X408
+#define EIC7700_REG_OFFSET_D2D_RST_CTRL                         0X40C
+#define EIC7700_REG_OFFSET_DDR_RST_CTRL                         0X410
+#define EIC7700_REG_OFFSET_TCU_RST_CTRL                         0X414
+#define EIC7700_REG_OFFSET_NPU_RST_CTRL                         0X418
+#define EIC7700_REG_OFFSET_HSPDMA_RST_CTRL                      0X41C
+#define EIC7700_REG_OFFSET_PCIE_RST_CTRL                        0X420
+#define EIC7700_REG_OFFSET_I2C_RST_CTRL                         0X424
+#define EIC7700_REG_OFFSET_FAN_RST_CTRL                         0X428
+#define EIC7700_REG_OFFSET_PVT_RST_CTRL                         0X42C
+#define EIC7700_REG_OFFSET_MBOX_RST_CTRL                        0X430
+#define EIC7700_REG_OFFSET_UART_RST_CTRL                        0X434
+#define EIC7700_REG_OFFSET_GPIO_RST_CTRL                        0X438
+#define EIC7700_REG_OFFSET_TIMER_RST_CTRL                       0X43C
+#define EIC7700_REG_OFFSET_SSI_RST_CTRL                         0X440
+#define EIC7700_REG_OFFSET_WDT_RST_CTRL                         0X444
+#define EIC7700_REG_OFFSET_LSP_CFGRST_CTRL                      0X448
+#define EIC7700_REG_OFFSET_U84_RST_CTRL                         0X44C
+#define EIC7700_REG_OFFSET_SCPU_RST_CTRL                        0X450
+#define EIC7700_REG_OFFSET_LPCPU_RST_CTRL                       0X454
+#define EIC7700_REG_OFFSET_VC_RST_CTRL                          0X458
+#define EIC7700_REG_OFFSET_JD_RST_CTRL                          0X45C
+#define EIC7700_REG_OFFSET_JE_RST_CTRL                          0X460
+#define EIC7700_REG_OFFSET_VD_RST_CTRL                          0X464
+#define EIC7700_REG_OFFSET_VE_RST_CTRL                          0X468
+#define EIC7700_REG_OFFSET_G2D_RST_CTRL                         0X46C
+#define EIC7700_REG_OFFSET_VI_RST_CTRL                          0X470
+#define EIC7700_REG_OFFSET_DVP_RST_CTRL                         0X474
+#define EIC7700_REG_OFFSET_ISP0_RST_CTRL                        0X478
+#define EIC7700_REG_OFFSET_ISP1_RST_CTRL                        0X47C
+#define EIC7700_REG_OFFSET_SHUTTER_RST_CTRL                     0X480
+#define EIC7700_REG_OFFSET_VO_PHYRST_CTRL                       0X484
+#define EIC7700_REG_OFFSET_VO_I2SRST_CTRL                       0X488
+#define EIC7700_REG_OFFSET_VO_RST_CTRL                          0X48C
+#define EIC7700_REG_OFFSET_BOOTSPI_RST_CTRL                     0X490
+#define EIC7700_REG_OFFSET_I2C1_RST_CTRL                        0X494
+#define EIC7700_REG_OFFSET_I2C0_RST_CTRL                        0X498
+#define EIC7700_REG_OFFSET_DMA1_RST_CTRL                        0X49C
+#define EIC7700_REG_OFFSET_FPRT_RST_CTRL                        0X4A0
+#define EIC7700_REG_OFFSET_HBLOCK_RST_CTRL                      0X4A4
+#define EIC7700_REG_OFFSET_SECSR_RST_CTRL                       0X4A8
+#define EIC7700_REG_OFFSET_OTP_RST_CTRL                         0X4AC
+#define EIC7700_REG_OFFSET_PKA_RST_CTRL                         0X4B0
+#define EIC7700_REG_OFFSET_SPACC_RST_CTRL                       0X4B4
+#define EIC7700_REG_OFFSET_TRNG_RST_CTRL                        0X4B8
+#define EIC7700_REG_OFFSET_TIMER0_RST_CTRL                      0X4C0
+#define EIC7700_REG_OFFSET_TIMER1_RST_CTRL                      0X4C4
+#define EIC7700_REG_OFFSET_TIMER2_RST_CTRL                      0X4C8
+#define EIC7700_REG_OFFSET_TIMER3_RST_CTRL                      0X4CC
+#define EIC7700_REG_OFFSET_RTC_RST_CTRL                         0X4D0
+#define EIC7700_REG_OFFSET_MNOC_RST_CTRL                        0X4D4
+#define EIC7700_REG_OFFSET_RNOC_RST_CTRL                        0X4D8
+#define EIC7700_REG_OFFSET_CNOC_RST_CTRL                        0X4DC
+#define EIC7700_REG_OFFSET_LNOC_RST_CTRL                        0X4E0
+
+/*
+ * RESET DEV ID  FOR EACH RESET CONSUMER
+ *
+ */
+#define		SNOC_RST_CTRL					0X00
+#define		GPU_RST_CTRL					0X01
+#define		DSP_RST_CTRL					0X02
+#define		D2D_RST_CTRL					0X03
+#define		DDR_RST_CTRL					0X04
+#define		TCU_RST_CTRL					0X05
+#define		NPU_RST_CTRL					0X06
+#define		HSPDMA_RST_CTRL					0X07
+#define		PCIE_RST_CTRL					0X08
+#define		I2C_RST_CTRL					0X09
+#define		FAN_RST_CTRL					0X0A
+#define		PVT_RST_CTRL					0X0B
+#define		MBOX_RST_CTRL					0X0C
+#define		UART_RST_CTRL					0X0D
+#define		GPIO_RST_CTRL					0X0E
+#define		TIMER_RST_CTRL					0X0F
+#define		SSI_RST_CTRL					0X10
+#define		WDT_RST_CTRL					0X11
+#define		LSP_CFGRST_CTRL					0X12
+#define		U84_RST_CTRL					0X13
+#define		SCPU_RST_CTRL					0X14
+#define		LPCPU_RST_CTRL					0X15
+#define		VC_RST_CTRL					0X16
+#define		JD_RST_CTRL					0X17
+#define		JE_RST_CTRL					0X18
+#define		VD_RST_CTRL					0X19
+#define		VE_RST_CTRL					0X1A
+#define		G2D_RST_CTRL					0X1B
+#define		VI_RST_CTRL					0X1C
+#define		DVP_RST_CTRL					0X1D
+#define		ISP0_RST_CTRL					0X1E
+#define		ISP1_RST_CTRL					0X1F
+#define		SHUTTER_RST_CTRL				0X20
+#define		VO_PHYRST_CTRL					0X21
+#define		VO_I2SRST_CTRL					0X22
+#define		VO_RST_CTRL					0X23
+#define		BOOTSPI_RST_CTRL				0X24
+#define		I2C1_RST_CTRL					0X25
+#define		I2C0_RST_CTRL					0X26
+#define		DMA1_RST_CTRL					0X27
+#define		FPRT_RST_CTRL					0X28
+#define		HBLOCK_RST_CTRL					0X29
+#define		SECSR_RST_CTRL					0X2A
+#define		OTP_RST_CTRL					0X2B
+#define		PKA_RST_CTRL					0X2C
+#define		SPACC_RST_CTRL					0X2D
+#define		TRNG_RST_CTRL					0X2E
+#define		RESERVED					0X2F
+#define		TIMER0_RST_CTRL					0X30
+#define		TIMER1_RST_CTRL					0X31
+#define		TIMER2_RST_CTRL					0X32
+#define		TIMER3_RST_CTRL					0X33
+#define		RTC_RST_CTRL					0X34
+#define		MNOC_RST_CTRL					0X35
+#define		RNOC_RST_CTRL					0X36
+#define		CNOC_RST_CTRL					0X37
+#define		LNOC_RST_CTRL					0X38
+
+#define BIT0                              (1 << 0)
+#define BIT1                              (1 << 1)
+#define BIT2                              (1 << 2)
+#define BIT3                              (1 << 3)
+#define BIT4                              (1 << 4)
+#define BIT5                              (1 << 5)
+#define BIT6                              (1 << 6)
+#define BIT7                              (1 << 7)
+#define BIT8                              (1 << 8)
+#define BIT9                              (1 << 9)
+#define BIT10                             (1 << 10)
+#define BIT11                             (1 << 11)
+#define BIT12                             (1 << 12)
+#define BIT13                             (1 << 13)
+#define BIT14                             (1 << 14)
+#define BIT15                             (1 << 15)
+#define BIT16                             (1 << 16)
+#define BIT17                             (1 << 17)
+#define BIT18                             (1 << 18)
+#define BIT19                             (1 << 19)
+#define BIT20                             (1 << 20)
+#define BIT21                             (1 << 21)
+#define BIT22                             (1 << 22)
+#define BIT23                             (1 << 23)
+#define BIT24                             (1 << 24)
+#define BIT25                             (1 << 25)
+#define BIT26                             (1 << 26)
+#define BIT27                             (1 << 27)
+#define BIT28                             (1 << 28)
+#define BIT29                             (1 << 29)
+#define BIT30                             (1 << 30)
+#define BIT31                             (1 << 31)
+
+/*
+	CONSUMER RESET CONTROL BIT
+*/
+/*SNOC*/
+#define		SW_NOC_NSP_RSTN						BIT0
+#define		SW_NOC_CFG_RSTN						BIT1
+#define		SW_RNOC_NSP_RSTN					BIT2
+#define		SW_SNOC_TCU_ARSTN					BIT3
+#define		SW_SNOC_U84_ARSTN					BIT4
+#define		SW_SNOC_PCIET_XSRSTN					BIT5
+#define		SW_SNOC_PCIET_XMRSTN					BIT6
+#define		SW_SNOC_PCIET_PRSTN					BIT7
+#define		SW_SNOC_NPU_ARSTN					BIT8
+#define		SW_SNOC_JTAG_ARSTN					BIT9
+#define		SW_SNOC_DSPT_ARSTN					BIT10
+#define		SW_SNOC_DDRC1_P2_ARSTN					BIT11
+#define		SW_SNOC_DDRC1_P1_ARSTN					BIT12
+#define		SW_SNOC_DDRC0_P2_ARSTN					BIT13
+#define		SW_SNOC_DDRC0_P1_ARSTN					BIT14
+#define		SW_SNOC_D2D_ARSTN					BIT15
+#define		SW_SNOC_AON_ARSTN					BIT16
+
+/*GPU*/
+#define		SW_GPU_AXI_RSTN						BIT0
+#define		SW_GPU_CFG_RSTN						BIT1
+#define		SW_GPU_GRAY_RSTN					BIT2
+#define		SW_GPU_JONES_RSTN					BIT3
+#define		SW_GPU_SPU_RSTN						BIT4
+
+/*DSP*/
+#define		SW_DSP_AXI_RSTN						BIT0
+#define		SW_DSP_CFG_RSTN						BIT1
+#define		SW_DSP_DIV4_RSTN					BIT2
+#define		SW_DSP_DIV_RSTN_0					BIT4
+#define		SW_DSP_DIV_RSTN_1					BIT5
+#define		SW_DSP_DIV_RSTN_2					BIT6
+#define		SW_DSP_DIV_RSTN_3					BIT7
+
+/*D2D*/
+#define		SW_D2D_AXI_RSTN						BIT0
+#define		SW_D2D_CFG_RSTN						BIT1
+#define		SW_D2D_PRST_N						BIT2
+#define		SW_D2D_RAW_PCS_RST_N					BIT4
+#define		SW_D2D_RX_RST_N						BIT5
+#define		SW_D2D_TX_RST_N						BIT6
+#define		SW_D2D_CORE_RST_N					BIT7
+
+/*TCU*/
+#define		SW_TCU_AXI_RSTN						BIT0
+#define		SW_TCU_CFG_RSTN						BIT1
+#define		TBU_RSTN_0						BIT4
+#define		TBU_RSTN_1						BIT5
+#define		TBU_RSTN_2						BIT6
+#define		TBU_RSTN_3						BIT7
+#define		TBU_RSTN_4						BIT8
+#define		TBU_RSTN_5						BIT9
+#define		TBU_RSTN_6						BIT10
+#define		TBU_RSTN_7						BIT11
+#define		TBU_RSTN_8						BIT12
+#define		TBU_RSTN_9						BIT13
+#define		TBU_RSTN_10						BIT14
+#define		TBU_RSTN_11						BIT15
+#define		TBU_RSTN_12						BIT16
+#define		TBU_RSTN_13						BIT17
+#define		TBU_RSTN_14						BIT18
+#define		TBU_RSTN_15						BIT19
+#define		TBU_RSTN_16						BIT20
+
+/*NPU*/
+#define		SW_NPU_AXI_RSTN						BIT0
+#define		SW_NPU_CFG_RSTN						BIT1
+#define		SW_NPU_CORE_RSTN					BIT2
+#define		SW_NPU_E31CORE_RSTN					BIT3
+#define		SW_NPU_E31BUS_RSTN					BIT4
+#define		SW_NPU_E31DBG_RSTN					BIT5
+#define		SW_NPU_LLC_RSTN						BIT6
+
+/*HSP DMA*/
+#define		SW_HSP_AXI_RSTN						BIT0
+#define		SW_HSP_CFG_RSTN						BIT1
+#define		SW_HSP_POR_RSTN						BIT2
+#define		SW_MSHC0_PHY_RSTN					BIT3
+#define		SW_MSHC1_PHY_RSTN					BIT4
+#define		SW_MSHC2_PHY_RSTN					BIT5
+#define		SW_MSHC0_TXRX_RSTN					BIT6
+#define		SW_MSHC1_TXRX_RSTN					BIT7
+#define		SW_MSHC2_TXRX_RSTN					BIT8
+#define		SW_SATA_ASIC0_RSTN					BIT9
+#define		SW_SATA_OOB_RSTN					BIT10
+#define		SW_SATA_PMALIVE_RSTN					BIT11
+#define		SW_SATA_RBC_RSTN					BIT12
+#define		SW_DMA0_RST_N						BIT13
+#define		SW_HSP_DMA0_RSTN					BIT14
+#define		SW_USB0_VAUX_RSTN					BIT15
+#define		SW_USB1_VAUX_RSTN					BIT16
+#define		SW_HSP_SD1_PRSTN					BIT17
+#define		SW_HSP_SD0_PRSTN					BIT18
+#define		SW_HSP_EMMC_PRSTN					BIT19
+#define		SW_HSP_DMA_PRSTN					BIT20
+#define		SW_HSP_SD1_ARSTN					BIT21
+#define		SW_HSP_SD0_ARSTN					BIT22
+#define		SW_HSP_EMMC_ARSTN					BIT23
+#define		SW_HSP_DMA_ARSTN					BIT24
+#define		SW_HSP_ETH1_ARSTN					BIT25
+#define		SW_HSP_ETH0_ARSTN					BIT26
+#define		SW_HSP_SATA_ARSTN					BIT27
+
+/*PCIE*/
+#define		SW_PCIE_CFG_RSTN					BIT0
+#define		SW_PCIE_POWERUP_RSTN					BIT1
+#define		SW_PCIE_PERST_N						BIT2
+
+/*I2C*/
+#define		SW_I2C_RST_N_0						BIT0
+#define		SW_I2C_RST_N_1						BIT1
+#define		SW_I2C_RST_N_2						BIT2
+#define		SW_I2C_RST_N_3						BIT3
+#define		SW_I2C_RST_N_4						BIT4
+#define		SW_I2C_RST_N_5						BIT5
+#define		SW_I2C_RST_N_6						BIT6
+#define		SW_I2C_RST_N_7						BIT7
+#define		SW_I2C_RST_N_8						BIT8
+#define		SW_I2C_RST_N_9						BIT9
+
+/*FAN*/
+#define		SW_FAN_RST_N						BIT0
+
+/*PVT*/
+#define		SW_PVT_RST_N_0						BIT0
+#define		SW_PVT_RST_N_1						BIT1
+
+/*MBOX*/
+#define		SW_MBOX_RST_N_0					BIT0
+#define		SW_MBOX_RST_N_1					BIT1
+#define		SW_MBOX_RST_N_2					BIT2
+#define		SW_MBOX_RST_N_3					BIT3
+#define		SW_MBOX_RST_N_4					BIT4
+#define		SW_MBOX_RST_N_5					BIT5
+#define		SW_MBOX_RST_N_6					BIT6
+#define		SW_MBOX_RST_N_7					BIT7
+#define		SW_MBOX_RST_N_8					BIT8
+#define		SW_MBOX_RST_N_9					BIT9
+#define		SW_MBOX_RST_N_10				BIT10
+#define		SW_MBOX_RST_N_11				BIT11
+#define		SW_MBOX_RST_N_12				BIT12
+#define		SW_MBOX_RST_N_13				BIT13
+#define		SW_MBOX_RST_N_14				BIT14
+#define		SW_MBOX_RST_N_15				BIT15
+
+/*UART*/
+#define		SW_UART_RST_N_0					BIT0
+#define		SW_UART_RST_N_1					BIT1
+#define		SW_UART_RST_N_2					BIT2
+#define		SW_UART_RST_N_3					BIT3
+#define		SW_UART_RST_N_4					BIT4
+
+/*GPIO*/
+/*
+#define		SW_GPIO_RST_N_0					BIT0
+#define		SW_GPIO_RST_N_1					BIT1
+*/
+
+/*TIMER*/
+#define		SW_TIMER_RST_N					BIT0
+
+/*SSI*/
+#define		SW_SSI_RST_N_0					BIT0
+#define		SW_SSI_RST_N_1					BIT1
+
+/*WDT*/
+#define		SW_WDT_RST_N_0					BIT0
+#define		SW_WDT_RST_N_1					BIT1
+#define		SW_WDT_RST_N_2					BIT2
+#define		SW_WDT_RST_N_3					BIT3
+
+/*LSP CFG*/
+#define		SW_LSP_CFG_RSTN					BIT0
+
+/*U84 CFG*/
+#define		SW_U84_CORE_RSTN_0				BIT0
+#define		SW_U84_CORE_RSTN_1				BIT1
+#define		SW_U84_CORE_RSTN_2				BIT2
+#define		SW_U84_CORE_RSTN_3				BIT3
+#define		SW_U84_BUS_RSTN					BIT4
+#define		SW_U84_DBG_RSTN					BIT5
+#define		SW_U84_TRACECOM_RSTN				BIT6
+#define		SW_U84_TRACE_RSTN_0				BIT8
+#define		SW_U84_TRACE_RSTN_1				BIT9
+#define		SW_U84_TRACE_RSTN_2				BIT10
+#define		SW_U84_TRACE_RSTN_3				BIT11
+
+/*SCPU*/
+#define		SW_SCPU_CORE_RSTN				BIT0
+#define		SW_SCPU_BUS_RSTN				BIT1
+#define		SW_SCPU_DBG_RSTN				BIT2
+
+/*LPCPU*/
+#define		SW_LPCPU_CORE_RSTN				BIT0
+#define		SW_LPCPU_BUS_RSTN				BIT1
+#define		SW_LPCPU_DBG_RSTN				BIT2
+
+/*VC*/
+#define		SW_VC_CFG_RSTN					BIT0
+#define		SW_VC_AXI_RSTN					BIT1
+#define		SW_VC_MONCFG_RSTN				BIT2
+
+/*JD*/
+#define		SW_JD_CFG_RSTN					BIT0
+#define		SW_JD_AXI_RSTN					BIT1
+
+/*JE*/
+#define		SW_JE_CFG_RSTN					BIT0
+#define		SW_JE_AXI_RSTN					BIT1
+
+/*VD*/
+#define		SW_VD_CFG_RSTN					BIT0
+#define		SW_VD_AXI_RSTN					BIT1
+
+/*VE*/
+#define		SW_VE_AXI_RSTN					BIT0
+#define		SW_VE_CFG_RSTN					BIT1
+
+/*G2D*/
+#define		SW_G2D_CORE_RSTN				BIT0
+#define		SW_G2D_CFG_RSTN					BIT1
+#define		SW_G2D_AXI_RSTN					BIT2
+
+/*VI*/
+#define		SW_VI_AXI_RSTN					BIT0
+#define		SW_VI_CFG_RSTN					BIT1
+#define		SW_VI_DWE_RSTN					BIT2
+
+/*DVP*/
+#define		SW_VI_DVP_RSTN					BIT0
+
+/*ISP0*/
+#define		SW_VI_ISP0_RSTN					BIT0
+
+/*ISP1*/
+#define		SW_VI_ISP1_RSTN					BIT0
+
+/*SHUTTR*/
+#define		SW_VI_SHUTTER_RSTN_0				BIT0
+#define		SW_VI_SHUTTER_RSTN_1				BIT1
+#define		SW_VI_SHUTTER_RSTN_2				BIT2
+#define		SW_VI_SHUTTER_RSTN_3				BIT3
+#define		SW_VI_SHUTTER_RSTN_4				BIT4
+#define		SW_VI_SHUTTER_RSTN_5				BIT5
+
+/*VO PHY*/
+#define		SW_VO_MIPI_PRSTN				BIT0
+#define		SW_VO_PRSTN					BIT1
+#define		SW_VO_HDMI_PRSTN				BIT3
+#define		SW_HDMI_PHYCTRL_RSTN				BIT4
+#define		SW_VO_HDMI_RSTN					BIT5
+
+/*VO I2S*/
+#define		SW_VO_I2S_RSTN					BIT0
+#define		SW_VO_I2S_PRSTN					BIT1
+
+/*VO*/
+#define		SW_VO_AXI_RSTN					BIT0
+#define		SW_VO_CFG_RSTN					BIT1
+#define		SW_VO_DC_RSTN					BIT2
+#define		SW_VO_DC_PRSTN					BIT3
+
+/*BOOTSPI*/
+#define		SW_BOOTSPI_HRSTN				BIT0
+#define		SW_BOOTSPI_RSTN					BIT1
+
+/*I2C1*/
+#define		SW_I2C1_PRSTN					BIT0
+
+/*I2C0*/
+#define		SW_I2C0_PRSTN					BIT0
+
+/*DMA1*/
+#define		SW_DMA1_ARSTN					BIT0
+#define		SW_DMA1_HRSTN					BIT1
+
+/*FPRT*/
+#define		SW_FP_PRT_HRSTN					BIT0
+
+/*HBLOCK*/
+#define		SW_HBLOCK_HRSTN					BIT0
+
+/*SECSR*/
+#define		SW_SECSR_HRSTN					BIT0
+
+/*OTP*/
+#define		SW_OTP_PRSTN					BIT0
+
+/*PKA*/
+#define		SW_PKA_HRSTN					BIT0
+
+/*SPACC*/
+#define		SW_SPACC_RSTN					BIT0
+
+/*TRNG*/
+#define		SW_TRNG_HRSTN					BIT0
+
+/*TIMER0*/
+#define		SW_TIMER0_RSTN_0				BIT0
+#define		SW_TIMER0_RSTN_1				BIT1
+#define		SW_TIMER0_RSTN_2				BIT2
+#define		SW_TIMER0_RSTN_3				BIT3
+#define		SW_TIMER0_RSTN_4				BIT4
+#define		SW_TIMER0_RSTN_5				BIT5
+#define		SW_TIMER0_RSTN_6				BIT6
+#define		SW_TIMER0_RSTN_7				BIT7
+#define		SW_TIMER0_PRSTN					BIT8
+
+/*TIMER1*/
+#define		SW_TIMER1_RSTN_0				BIT0
+#define		SW_TIMER1_RSTN_1				BIT1
+#define		SW_TIMER1_RSTN_2				BIT2
+#define		SW_TIMER1_RSTN_3				BIT3
+#define		SW_TIMER1_RSTN_4				BIT4
+#define		SW_TIMER1_RSTN_5				BIT5
+#define		SW_TIMER1_RSTN_6				BIT6
+#define		SW_TIMER1_RSTN_7				BIT7
+#define		SW_TIMER1_PRSTN					BIT8
+
+/*TIMER2*/
+#define		SW_TIMER2_RSTN_0				BIT0
+#define		SW_TIMER2_RSTN_1				BIT1
+#define		SW_TIMER2_RSTN_2				BIT2
+#define		SW_TIMER2_RSTN_3				BIT3
+#define		SW_TIMER2_RSTN_4				BIT4
+#define		SW_TIMER2_RSTN_5				BIT5
+#define		SW_TIMER2_RSTN_6				BIT6
+#define		SW_TIMER2_RSTN_7				BIT7
+#define		SW_TIMER2_PRSTN					BIT8
+
+/*TIMER3*/
+#define		SW_TIMER3_RSTN_0				BIT0
+#define		SW_TIMER3_RSTN_1				BIT1
+#define		SW_TIMER3_RSTN_2				BIT2
+#define		SW_TIMER3_RSTN_3				BIT3
+#define		SW_TIMER3_RSTN_4				BIT4
+#define		SW_TIMER3_RSTN_5				BIT5
+#define		SW_TIMER3_RSTN_6				BIT6
+#define		SW_TIMER3_RSTN_7				BIT7
+#define		SW_TIMER3_PRSTN					BIT8
+
+/*RTC*/
+#define		SW_RTC_RSTN					BIT0
+
+/*MNOC*/
+#define		SW_MNOC_SNOC_NSP_RSTN				BIT0
+#define		SW_MNOC_VC_ARSTN				BIT1
+#define		SW_MNOC_CFG_RSTN				BIT2
+#define		SW_MNOC_HSP_ARSTN				BIT3
+#define		SW_MNOC_GPU_ARSTN				BIT4
+#define		SW_MNOC_DDRC1_P3_ARSTN				BIT5
+#define		SW_MNOC_DDRC0_P3_ARSTN				BIT6
+
+/*RNOC*/
+#define		SW_RNOC_VO_ARSTN				BIT0
+#define		SW_RNOC_VI_ARSTN				BIT1
+#define		SW_RNOC_SNOC_NSP_RSTN				BIT2
+#define		SW_RNOC_CFG_RSTN				BIT3
+#define		SW_MNOC_DDRC1_P4_ARSTN				BIT4
+#define		SW_MNOC_DDRC0_P4_ARSTN				BIT5
+
+/*CNOC*/
+#define		SW_CNOC_VO_CFG_RSTN				BIT0
+#define		SW_CNOC_VI_CFG_RSTN				BIT1
+#define		SW_CNOC_VC_CFG_RSTN				BIT2
+#define		SW_CNOC_TCU_CFG_RSTN				BIT3
+#define		SW_CNOC_PCIET_CFG_RSTN				BIT4
+#define		SW_CNOC_NPU_CFG_RSTN				BIT5
+#define		SW_CNOC_LSP_CFG_RSTN				BIT6
+#define		SW_CNOC_HSP_CFG_RSTN				BIT7
+#define		SW_CNOC_GPU_CFG_RSTN				BIT8
+#define		SW_CNOC_DSPT_CFG_RSTN				BIT9
+#define		SW_CNOC_DDRT1_CFG_RSTN				BIT10
+#define		SW_CNOC_DDRT0_CFG_RSTN				BIT11
+#define		SW_CNOC_D2D_CFG_RSTN				BIT12
+#define		SW_CNOC_CFG_RSTN				BIT13
+#define		SW_CNOC_CLMM_CFG_RSTN				BIT14
+#define		SW_CNOC_AON_CFG_RSTN				BIT15
+
+/*LNOC*/
+#define		SW_LNOC_CFG_RSTN				BIT0
+#define		SW_LNOC_NPU_LLC_ARSTN				BIT1
+#define		SW_LNOC_DDRC1_P0_ARSTN				BIT2
+#define		SW_LNOC_DDRC0_P0_ARSTN				BIT3
+
+#endif /*endif __DT_ESWIN_EIC7700_SYSCRG_H__*/
diff --git a/include/dt-bindings/riscv/physical-memory.h b/include/dt-bindings/riscv/physical-memory.h
new file mode 100644
index 000000000000..7cb2e58fa8c1
--- /dev/null
+++ b/include/dt-bindings/riscv/physical-memory.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+
+#ifndef _DT_BINDINGS_RISCV_PHYSICAL_MEMORY_H
+#define _DT_BINDINGS_RISCV_PHYSICAL_MEMORY_H
+
+#define PMA_READ			(1 << 0)
+#define PMA_WRITE			(1 << 1)
+#define PMA_EXECUTE			(1 << 2)
+#define PMA_AMO_MASK			(3 << 4)
+#define PMA_AMO_NONE			(0 << 4)
+#define PMA_AMO_SWAP			(1 << 4)
+#define PMA_AMO_LOGICAL			(2 << 4)
+#define PMA_AMO_ARITHMETIC		(3 << 4)
+#define PMA_RSRV_MASK			(3 << 6)
+#define PMA_RSRV_NONE			(0 << 6)
+#define PMA_RSRV_NON_EVENTUAL		(1 << 6)
+#define PMA_RSRV_EVENTUAL		(2 << 6)
+
+#define PMA_RW				(PMA_READ | PMA_WRITE)
+#define PMA_RWA				(PMA_RW | PMA_AMO_ARITHMETIC | PMA_RSRV_EVENTUAL)
+#define PMA_RWX				(PMA_RW | PMA_EXECUTE)
+#define PMA_RWXA			(PMA_RWA | PMA_EXECUTE)
+
+#define PMA_ORDER_MASK			(3 << 8)
+#define PMA_ORDER_IO_RELAXED		(0 << 8)
+#define PMA_ORDER_IO_STRONG		(1 << 8)
+#define PMA_ORDER_MEMORY		(2 << 8)
+#define PMA_READ_IDEMPOTENT		(1 << 10)
+#define PMA_WRITE_IDEMPOTENT		(1 << 11)
+#define PMA_CACHEABLE			(1 << 12)
+#define PMA_COHERENT			(1 << 13)
+
+#define PMA_UNSAFE			(1 << 15)
+
+#define PMA_IO				(PMA_ORDER_IO_RELAXED)
+#define PMA_NONCACHEABLE_MEMORY		(PMA_ORDER_MEMORY | PMA_READ_IDEMPOTENT | \
+						PMA_WRITE_IDEMPOTENT)
+#define PMA_NONCOHERENT_MEMORY		(PMA_NONCACHEABLE_MEMORY | PMA_CACHEABLE)
+#define PMA_NORMAL_MEMORY		(PMA_NONCOHERENT_MEMORY | PMA_COHERENT)
+
+#define PMR_ALIAS_MASK			(0xff << 24)
+#define PMR_ALIAS(n)			((n) << 24)
+
+#endif /* _DT_BINDINGS_RISCV_PHYSICAL_MEMORY_H */
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 624d4a38c358..6f8663711c90 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -244,6 +244,7 @@ enum cpuhp_state {
 	CPUHP_AP_PERF_POWERPC_TRACE_IMC_ONLINE,
 	CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE,
 	CPUHP_AP_PERF_POWERPC_HV_GPCI_ONLINE,
+	CPUHP_AP_PERF_RISCV_SIFIVE_CCACHE_ONLINE,
 	CPUHP_AP_PERF_CSKY_ONLINE,
 	CPUHP_AP_WATCHDOG_ONLINE,
 	CPUHP_AP_WORKQUEUE_ONLINE,
diff --git a/include/linux/eic7700-sid-cfg.h b/include/linux/eic7700-sid-cfg.h
new file mode 100644
index 000000000000..60f47630b8c8
--- /dev/null
+++ b/include/linux/eic7700-sid-cfg.h
@@ -0,0 +1,8 @@
+#ifndef EIC7700_SID_CFG_H
+#define EIC7700_SID_CFG_H
+
+int eic7700_dynm_sid_enable(int nid);
+int eic7700_aon_sid_cfg(struct device *dev);
+int eic7700_tbu_power(struct device *dev, bool is_powerUp);
+
+#endif
diff --git a/include/linux/fpu.h b/include/linux/fpu.h
new file mode 100644
index 000000000000..2fb63e22913b
--- /dev/null
+++ b/include/linux/fpu.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _LINUX_FPU_H
+#define _LINUX_FPU_H
+
+#ifdef _LINUX_FPU_COMPILATION_UNIT
+#error FP code must be compiled separately. See Documentation/core-api/floating-point.rst.
+#endif
+
+#include <asm/fpu.h>
+
+#endif
diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst
index 0afd75472679..89b57740ef09 100644
--- a/scripts/Makefile.modinst
+++ b/scripts/Makefile.modinst
@@ -76,7 +76,11 @@ quiet_cmd_install = INSTALL $@
 ifdef INSTALL_MOD_STRIP
 
 ifeq ($(INSTALL_MOD_STRIP),1)
+ifeq ($(ARCH),riscv)
+strip-option := --strip-debug --discard-locals
+else
 strip-option := --strip-debug
+endif
 else
 strip-option := $(INSTALL_MOD_STRIP)
 endif
diff --git a/sound/soc/Kconfig b/sound/soc/Kconfig
index 439fa631c342..24ee3416d6a6 100644
--- a/sound/soc/Kconfig
+++ b/sound/soc/Kconfig
@@ -85,6 +85,7 @@ source "sound/soc/au1x/Kconfig"
 source "sound/soc/bcm/Kconfig"
 source "sound/soc/cirrus/Kconfig"
 source "sound/soc/dwc/Kconfig"
+source "sound/soc/eswin/Kconfig"
 source "sound/soc/fsl/Kconfig"
 source "sound/soc/google/Kconfig"
 source "sound/soc/hisilicon/Kconfig"
diff --git a/sound/soc/Makefile b/sound/soc/Makefile
index 8376fdb217ed..d36d824db893 100644
--- a/sound/soc/Makefile
+++ b/sound/soc/Makefile
@@ -42,6 +42,7 @@ obj-$(CONFIG_SND_SOC)	+= au1x/
 obj-$(CONFIG_SND_SOC)	+= bcm/
 obj-$(CONFIG_SND_SOC)	+= cirrus/
 obj-$(CONFIG_SND_SOC)	+= dwc/
+obj-$(CONFIG_SND_SOC)	+= eswin/
 obj-$(CONFIG_SND_SOC)	+= fsl/
 obj-$(CONFIG_SND_SOC)	+= google/
 obj-$(CONFIG_SND_SOC)	+= hisilicon/
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index 6d105a23c828..363fc75ae167 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -2405,4 +2405,5 @@ config SND_SOC_LPASS_TX_MACRO
 	select SND_SOC_LPASS_MACRO_COMMON
 	tristate "Qualcomm TX Macro in LPASS(Low Power Audio SubSystem)"
 
+source "sound/soc/codecs/eswin/Kconfig"
 endmenu
diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile
index a87e56938ce5..6205558e39fe 100644
--- a/sound/soc/codecs/Makefile
+++ b/sound/soc/codecs/Makefile
@@ -772,3 +772,4 @@ obj-$(CONFIG_SND_SOC_LPASS_TX_MACRO)	+= snd-soc-lpass-tx-macro.o
 
 # Mux
 obj-$(CONFIG_SND_SOC_SIMPLE_MUX)	+= snd-soc-simple-mux.o
+obj-$(CONFIG_ESWIN_SND_SOC_CODECS)	+= eswin/
diff --git a/sound/soc/codecs/eswin/Kconfig b/sound/soc/codecs/eswin/Kconfig
new file mode 100755
index 000000000000..3b715354ef7e
--- /dev/null
+++ b/sound/soc/codecs/eswin/Kconfig
@@ -0,0 +1,20 @@
+menuconfig ESWIN_SND_SOC_CODECS
+	tristate "ESWIN CODEC drivers"
+	default n
+	help
+	  Say Y or M if you want to add support for codecs attached to
+	  the ESWIN Asoc interface. You will also need
+	  to select the audio interfaces to support below.
+
+#if ESWIN_SND_SOC_CODECS
+
+config ESWIN_SND_ES8388_CODEC
+	tristate "ESWIN Audio es8388 codec"
+	depends on ESWIN_SND_SOC_CODECS
+	default n
+	help
+	  ESWIN Audio codec,
+	  es8388 codec,
+	  this codec is internal
+
+#endif #ESWIN_SND_SOC_CODECS
diff --git a/sound/soc/codecs/eswin/Makefile b/sound/soc/codecs/eswin/Makefile
new file mode 100755
index 000000000000..ee696d62096c
--- /dev/null
+++ b/sound/soc/codecs/eswin/Makefile
@@ -0,0 +1,3 @@
+esw_es8328_codec-objs := es8328-i2c.o es8328.o
+
+obj-$(CONFIG_ESWIN_SND_ES8388_CODEC) += esw_es8328_codec.o
\ No newline at end of file
diff --git a/sound/soc/codecs/eswin/es8328-i2c.c b/sound/soc/codecs/eswin/es8328-i2c.c
new file mode 100644
index 000000000000..2a59aa9fc168
--- /dev/null
+++ b/sound/soc/codecs/eswin/es8328-i2c.c
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * es8328-i2c.c  --  ES8328 ALSA SoC I2C Audio driver
+ *
+ * Copyright 2014 Sutajio Ko-Usagi PTE LTD
+ *
+ * Author: Sean Cross <xobs@kosagi.com>
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/regmap.h>
+
+#include <sound/soc.h>
+
+#include "es8328.h"
+
+static const struct i2c_device_id es8328_id[] = {
+	{ "es8328", 0 },
+	{ "es8388", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, es8328_id);
+
+static const struct of_device_id es8328_of_match[] = {
+	{ .compatible = "eswin,es8388", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, es8328_of_match);
+
+static int es8328_i2c_probe(struct i2c_client *i2c)
+{
+	return es8328_probe(&i2c->dev,
+			devm_regmap_init_i2c(i2c, &es8328_regmap_config));
+}
+
+static struct i2c_driver es8328_i2c_driver = {
+	.driver = {
+		.name		= "es8328",
+		.of_match_table = es8328_of_match,
+	},
+	.probe    = es8328_i2c_probe,
+	.id_table = es8328_id,
+};
+
+module_i2c_driver(es8328_i2c_driver);
+
+MODULE_DESCRIPTION("ASoC ES8328 audio CODEC I2C driver");
+MODULE_AUTHOR("Sean Cross <xobs@kosagi.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/eswin/es8328.c b/sound/soc/codecs/eswin/es8328.c
new file mode 100644
index 000000000000..1fcf73f448c2
--- /dev/null
+++ b/sound/soc/codecs/eswin/es8328.c
@@ -0,0 +1,1087 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * es8328.c  --  ES8328 ALSA SoC Audio driver
+ *
+ * Copyright 2014 Sutajio Ko-Usagi PTE LTD
+ *
+ * Author: Sean Cross <xobs@kosagi.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/of_device.h>
+#include <linux/module.h>
+#include <linux/pm.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/regulator/consumer.h>
+#include <linux/gpio/consumer.h>
+
+#include <sound/core.h>
+#include <sound/initval.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/tlv.h>
+
+#include "es8328.h"
+
+#define MIN_CHANNEL_NUM		2
+#define MAX_CHANNEL_NUM		2
+
+static const unsigned int rates_12288[] = {
+	8000, 12000, 16000, 24000, 32000, 48000, 96000,
+};
+
+static const int ratios_12288[] = {
+	10, 7, 6, 4, 3, 2, 0,
+};
+
+static const struct snd_pcm_hw_constraint_list constraints_12288 = {
+	.count	= ARRAY_SIZE(rates_12288),
+	.list	= rates_12288,
+};
+
+static const unsigned int rates_11289[] = {
+	8018, 11025, 22050, 44100, 88200,
+};
+
+static const int ratios_11289[] = {
+	9, 7, 4, 2, 0,
+};
+
+static const struct snd_pcm_hw_constraint_list constraints_11289 = {
+	.count	= ARRAY_SIZE(rates_11289),
+	.list	= rates_11289,
+};
+
+/* regulator supplies for sgtl5000, VDDD is an optional external supply */
+enum sgtl5000_regulator_supplies {
+	DVDD,
+	AVDD,
+	PVDD,
+	HPVDD,
+	ES8328_SUPPLY_NUM
+};
+
+/* vddd is optional supply */
+static const char * const supply_names[ES8328_SUPPLY_NUM] = {
+	"DVDD",
+	"AVDD",
+	"PVDD",
+	"HPVDD",
+};
+
+#define ES8328_RATES (SNDRV_PCM_RATE_192000 | \
+		SNDRV_PCM_RATE_96000 | \
+		SNDRV_PCM_RATE_88200 | \
+		SNDRV_PCM_RATE_8000_48000)
+#define ES8328_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | \
+		SNDRV_PCM_FMTBIT_S18_3LE | \
+		SNDRV_PCM_FMTBIT_S20_3LE | \
+		SNDRV_PCM_FMTBIT_S24_LE | \
+		SNDRV_PCM_FMTBIT_S32_LE)
+
+struct es8328_priv {
+	struct regmap *regmap;
+	struct clk *clk;
+	int playback_fs;
+	bool deemph;
+	int mclkdiv2;
+	const struct snd_pcm_hw_constraint_list *sysclk_constraints;
+	const int *mclk_ratios;
+	bool provider;
+	struct regulator_bulk_data supplies[ES8328_SUPPLY_NUM];
+
+	u32 eswin_plat;
+	struct snd_soc_component *component;
+	struct gpio_desc *front_jack_gpio;
+	struct gpio_desc *back_jack_gpio;
+};
+
+/*
+ * ES8328 Controls
+ */
+
+static const char * const adcpol_txt[] = {"Normal", "L Invert", "R Invert",
+					  "L + R Invert"};
+static SOC_ENUM_SINGLE_DECL(adcpol,
+			    ES8328_ADCCONTROL6, 6, adcpol_txt);
+
+static const DECLARE_TLV_DB_SCALE(play_tlv, -3000, 100, 0);
+static const DECLARE_TLV_DB_SCALE(dac_adc_tlv, -9600, 50, 0);
+static const DECLARE_TLV_DB_SCALE(bypass_tlv, -1500, 300, 0);
+static const DECLARE_TLV_DB_SCALE(mic_tlv, 0, 300, 0);
+
+static const struct {
+	int rate;
+	unsigned int val;
+} deemph_settings[] = {
+	{ 0,     ES8328_DACCONTROL6_DEEMPH_OFF },
+	{ 32000, ES8328_DACCONTROL6_DEEMPH_32k },
+	{ 44100, ES8328_DACCONTROL6_DEEMPH_44_1k },
+	{ 48000, ES8328_DACCONTROL6_DEEMPH_48k },
+};
+
+static int es8328_set_deemph(struct snd_soc_component *component)
+{
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+	int val, i, best;
+
+	/*
+	 * If we're using deemphasis select the nearest available sample
+	 * rate.
+	 */
+	if (es8328->deemph) {
+		best = 0;
+		for (i = 1; i < ARRAY_SIZE(deemph_settings); i++) {
+			if (abs(deemph_settings[i].rate - es8328->playback_fs) <
+			    abs(deemph_settings[best].rate - es8328->playback_fs))
+				best = i;
+		}
+
+		val = deemph_settings[best].val;
+	} else {
+		val = ES8328_DACCONTROL6_DEEMPH_OFF;
+	}
+
+	dev_dbg(component->dev, "Set deemphasis %d\n", val);
+
+	return snd_soc_component_update_bits(component, ES8328_DACCONTROL6,
+			ES8328_DACCONTROL6_DEEMPH_MASK, val);
+}
+
+static int es8328_get_deemph(struct snd_kcontrol *kcontrol,
+			     struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol);
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = es8328->deemph;
+	return 0;
+}
+
+static int es8328_put_deemph(struct snd_kcontrol *kcontrol,
+			     struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol);
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+	unsigned int deemph = ucontrol->value.integer.value[0];
+	int ret;
+
+	if (deemph > 1)
+		return -EINVAL;
+
+	if (es8328->deemph == deemph)
+		return 0;
+
+	ret = es8328_set_deemph(component);
+	if (ret < 0)
+		return ret;
+
+	es8328->deemph = deemph;
+
+	return 1;
+}
+static int dump_flag = 0;
+static u32 g_reg = 1;
+static u32 g_get_cnt = 0;
+static u32 g_put_cnt = 0;
+
+int esw_codec_dump_get(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	ucontrol->value.integer.value[0] = dump_flag;
+	return 0;
+}
+
+int esw_codec_dump_info(struct snd_kcontrol *kcontrol,
+					    struct snd_ctl_elem_info *uinfo)
+{
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN;
+	uinfo->count = 1;
+	uinfo->value.integer.min = 0;
+	uinfo->value.integer.max = 1;
+	return 0;
+}
+
+int esw_codec_dump_put(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	int dump_onoff	= ucontrol->value.integer.value[0];
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	u32 reg, ret, i;
+
+	printk("codec dump onoff:%d\n", dump_onoff);
+	dump_flag = dump_onoff;
+	if (dump_onoff == true) {
+		printk("statr codec dump\n");
+		for (i = 0; i < 53; i++) {
+			ret = regmap_read(component->regmap, i, &reg);
+			if (ret != 0) {
+				printk("reag reg[%d] failed!\n", i);
+			}
+			printk("reg[%d]:0x%x\n", i, reg);
+		}
+	}
+	return 0;
+}
+
+int esw_codec_reg_get(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	u32 val;
+	int ret;
+
+	g_get_cnt++;
+
+	if (g_get_cnt == 1) {
+		ucontrol->value.integer.value[0] = g_reg;
+	} else {
+		ret = regmap_read(component->regmap, g_reg, &val);
+		if (ret != 0) {
+			printk("read reg[%d] failed!\n", g_reg);
+		}
+		printk("codec read reg[%d]:0x%x\n", g_reg, val);
+		ucontrol->value.integer.value[1] = val;
+		g_get_cnt = 0;
+	}
+	return 0;
+}
+
+int esw_codec_reg_info(struct snd_kcontrol *kcontrol,
+					    struct snd_ctl_elem_info *uinfo)
+{
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+	uinfo->count = 2;
+	uinfo->value.integer.min = 0;
+	uinfo->value.integer.max = 256;
+	return 0;
+}
+
+int esw_codec_reg_put(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	int ret;
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+
+	g_put_cnt++;
+
+	if (g_put_cnt == 1) {
+		g_reg = ucontrol->value.integer.value[0];
+	} else {
+		printk("codec write reg:%d, val:0x%x\n", g_reg, (u32)ucontrol->value.integer.value[1]);
+		ret = regmap_write(component->regmap, g_reg, (u32)ucontrol->value.integer.value[1]);
+		if (ret != 0) {
+			printk("write reg[%d] failed!\n", g_reg);
+		}
+		g_put_cnt = 0;
+	}
+
+	return 0;
+}
+
+static const struct snd_kcontrol_new es8328_snd_controls[] = {
+	SOC_DOUBLE_R_TLV("Capture Digital Volume",
+		ES8328_ADCCONTROL8, ES8328_ADCCONTROL9,
+		 0, 0xc0, 1, dac_adc_tlv),
+	SOC_SINGLE("Capture ZC Switch", ES8328_ADCCONTROL7, 5, 1, 0),
+
+	SOC_SINGLE_BOOL_EXT("DAC Deemphasis Switch", 0,
+		    es8328_get_deemph, es8328_put_deemph),
+
+	SOC_ENUM("Capture Polarity", adcpol),
+
+	SOC_SINGLE_TLV("Left Mixer Left Bypass Volume",
+			ES8328_DACCONTROL17, 3, 7, 1, bypass_tlv),
+	SOC_SINGLE_TLV("Left Mixer Right Bypass Volume",
+			ES8328_DACCONTROL19, 3, 7, 1, bypass_tlv),
+	SOC_SINGLE_TLV("Right Mixer Left Bypass Volume",
+			ES8328_DACCONTROL18, 3, 7, 1, bypass_tlv),
+	SOC_SINGLE_TLV("Right Mixer Right Bypass Volume",
+			ES8328_DACCONTROL20, 3, 7, 1, bypass_tlv),
+
+	SOC_DOUBLE_R_TLV("PCM Volume",
+			ES8328_LDACVOL, ES8328_RDACVOL,
+			0, ES8328_LDACVOL_MAX, 1, dac_adc_tlv),
+
+	SOC_DOUBLE_R_TLV("Output 1 Playback Volume",
+			ES8328_LOUT1VOL, ES8328_ROUT1VOL,
+			0, ES8328_OUT1VOL_MAX, 0, play_tlv),
+
+	SOC_DOUBLE_R_TLV("Output 2 Playback Volume",
+			ES8328_LOUT2VOL, ES8328_ROUT2VOL,
+			0, ES8328_OUT2VOL_MAX, 0, play_tlv),
+
+	SOC_DOUBLE_TLV("Mic PGA Volume", ES8328_ADCCONTROL1,
+			4, 0, 8, 0, mic_tlv),
+	{
+		.iface = SNDRV_CTL_ELEM_IFACE_CARD,
+		.name = "Reg Dump",
+		.index = 0,
+		.access = SNDRV_CTL_ELEM_ACCESS_READWRITE,
+		.info = esw_codec_dump_info,
+		.get = esw_codec_dump_get,
+		.put = esw_codec_dump_put,
+	},
+	{
+		.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
+		.name = "Reg Write",
+		.access = SNDRV_CTL_ELEM_ACCESS_TLV_READ | SNDRV_CTL_ELEM_ACCESS_READWRITE,
+		.info = esw_codec_reg_info,
+		.get = esw_codec_reg_get,
+		.put = esw_codec_reg_put,
+	}
+};
+
+/*
+ * DAPM Controls
+ */
+
+static const char * const es8328_line_texts[] = {
+	"Line 1", "Line 2", "PGA", "Differential"};
+
+static const struct soc_enum es8328_lline_enum =
+	SOC_ENUM_SINGLE(ES8328_DACCONTROL16, 3,
+			      ARRAY_SIZE(es8328_line_texts),
+			      es8328_line_texts);
+static const struct snd_kcontrol_new es8328_left_line_controls =
+	SOC_DAPM_ENUM("Route", es8328_lline_enum);
+
+static const struct soc_enum es8328_rline_enum =
+	SOC_ENUM_SINGLE(ES8328_DACCONTROL16, 0,
+			      ARRAY_SIZE(es8328_line_texts),
+			      es8328_line_texts);
+static const struct snd_kcontrol_new es8328_right_line_controls =
+	SOC_DAPM_ENUM("Route", es8328_rline_enum);
+
+/* Left Mixer */
+static const struct snd_kcontrol_new es8328_left_mixer_controls[] = {
+	SOC_DAPM_SINGLE("Playback Switch", ES8328_DACCONTROL17, 7, 1, 0),
+	SOC_DAPM_SINGLE("Left Bypass Switch", ES8328_DACCONTROL17, 6, 1, 0),
+	SOC_DAPM_SINGLE("Right Playback Switch", ES8328_DACCONTROL18, 7, 1, 0),
+	SOC_DAPM_SINGLE("Right Bypass Switch", ES8328_DACCONTROL18, 6, 1, 0),
+};
+
+/* Right Mixer */
+static const struct snd_kcontrol_new es8328_right_mixer_controls[] = {
+	SOC_DAPM_SINGLE("Left Playback Switch", ES8328_DACCONTROL19, 7, 1, 0),
+	SOC_DAPM_SINGLE("Left Bypass Switch", ES8328_DACCONTROL19, 6, 1, 0),
+	SOC_DAPM_SINGLE("Playback Switch", ES8328_DACCONTROL20, 7, 1, 0),
+	SOC_DAPM_SINGLE("Right Bypass Switch", ES8328_DACCONTROL20, 6, 1, 0),
+};
+
+static const char * const es8328_pga_sel[] = {
+	"Line 1", "Line 2", "Line 3", "Differential"};
+
+/* Left PGA Mux */
+static const struct soc_enum es8328_lpga_enum =
+	SOC_ENUM_SINGLE(ES8328_ADCCONTROL2, 6,
+			      ARRAY_SIZE(es8328_pga_sel),
+			      es8328_pga_sel);
+static const struct snd_kcontrol_new es8328_left_pga_controls =
+	SOC_DAPM_ENUM("Route", es8328_lpga_enum);
+
+/* Right PGA Mux */
+static const struct soc_enum es8328_rpga_enum =
+	SOC_ENUM_SINGLE(ES8328_ADCCONTROL2, 4,
+			      ARRAY_SIZE(es8328_pga_sel),
+			      es8328_pga_sel);
+static const struct snd_kcontrol_new es8328_right_pga_controls =
+	SOC_DAPM_ENUM("Route", es8328_rpga_enum);
+
+/* Differential Mux */
+static const char * const es8328_diff_sel[] = {"Line 1", "Line 2"};
+static SOC_ENUM_SINGLE_DECL(diffmux,
+			    ES8328_ADCCONTROL3, 7, es8328_diff_sel);
+static const struct snd_kcontrol_new es8328_diffmux_controls =
+	SOC_DAPM_ENUM("Route", diffmux);
+
+/* Mono ADC Mux */
+static const char * const es8328_mono_mux[] = {"Stereo", "Mono (Left)",
+	"Mono (Right)", "Digital Mono"};
+static SOC_ENUM_SINGLE_DECL(monomux,
+			    ES8328_ADCCONTROL3, 3, es8328_mono_mux);
+static const struct snd_kcontrol_new es8328_monomux_controls =
+	SOC_DAPM_ENUM("Route", monomux);
+
+static const struct snd_soc_dapm_widget es8328_dapm_widgets[] = {
+	SND_SOC_DAPM_MUX("Differential Mux", SND_SOC_NOPM, 0, 0,
+		&es8328_diffmux_controls),
+	SND_SOC_DAPM_MUX("Left ADC Mux", SND_SOC_NOPM, 0, 0,
+		&es8328_monomux_controls),
+	SND_SOC_DAPM_MUX("Right ADC Mux", SND_SOC_NOPM, 0, 0,
+		&es8328_monomux_controls),
+
+	SND_SOC_DAPM_MUX("Left PGA Mux", ES8328_ADCPOWER,
+			ES8328_ADCPOWER_AINL_OFF, 1,
+			&es8328_left_pga_controls),
+	SND_SOC_DAPM_MUX("Right PGA Mux", ES8328_ADCPOWER,
+			ES8328_ADCPOWER_AINR_OFF, 1,
+			&es8328_right_pga_controls),
+
+	SND_SOC_DAPM_MUX("Left Line Mux", SND_SOC_NOPM, 0, 0,
+		&es8328_left_line_controls),
+	SND_SOC_DAPM_MUX("Right Line Mux", SND_SOC_NOPM, 0, 0,
+		&es8328_right_line_controls),
+
+	SND_SOC_DAPM_ADC("Right ADC", "Right Capture", ES8328_ADCPOWER,
+			ES8328_ADCPOWER_ADCR_OFF, 1),
+	SND_SOC_DAPM_ADC("Left ADC", "Left Capture", ES8328_ADCPOWER,
+			ES8328_ADCPOWER_ADCL_OFF, 1),
+
+	SND_SOC_DAPM_SUPPLY("Mic Bias", ES8328_ADCPOWER,
+			ES8328_ADCPOWER_MIC_BIAS_OFF, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Mic Bias Gen", ES8328_ADCPOWER,
+			ES8328_ADCPOWER_ADC_BIAS_GEN_OFF, 1, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("DAC STM", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_DACSTM_RESET, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC STM", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_ADCSTM_RESET, 1, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("DAC DIG", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_DACDIG_OFF, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC DIG", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_ADCDIG_OFF, 1, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("DAC DLL", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_DACDLL_OFF, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC DLL", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_ADCDLL_OFF, 1, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("ADC Vref", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_ADCVREF_OFF, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("DAC Vref", ES8328_CHIPPOWER,
+			ES8328_CHIPPOWER_DACVREF_OFF, 1, NULL, 0),
+
+	SND_SOC_DAPM_DAC("Right DAC", "Right Playback", ES8328_DACPOWER,
+			ES8328_DACPOWER_RDAC_OFF, 1),
+	SND_SOC_DAPM_DAC("Left DAC", "Left Playback", ES8328_DACPOWER,
+			ES8328_DACPOWER_LDAC_OFF, 1),
+
+	SND_SOC_DAPM_MIXER("Left Mixer", SND_SOC_NOPM, 0, 0,
+		&es8328_left_mixer_controls[0],
+		ARRAY_SIZE(es8328_left_mixer_controls)),
+	SND_SOC_DAPM_MIXER("Right Mixer", SND_SOC_NOPM, 0, 0,
+		&es8328_right_mixer_controls[0],
+		ARRAY_SIZE(es8328_right_mixer_controls)),
+
+	SND_SOC_DAPM_PGA("Right Out 2", ES8328_DACPOWER,
+			ES8328_DACPOWER_ROUT2_ON, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("Left Out 2", ES8328_DACPOWER,
+			ES8328_DACPOWER_LOUT2_ON, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("Right Out 1", ES8328_DACPOWER,
+			ES8328_DACPOWER_ROUT1_ON, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("Left Out 1", ES8328_DACPOWER,
+			ES8328_DACPOWER_LOUT1_ON, 0, NULL, 0),
+
+	SND_SOC_DAPM_OUTPUT("LOUT1"),
+	SND_SOC_DAPM_OUTPUT("ROUT1"),
+	SND_SOC_DAPM_OUTPUT("LOUT2"),
+	SND_SOC_DAPM_OUTPUT("ROUT2"),
+
+	SND_SOC_DAPM_INPUT("LINPUT1"),
+	SND_SOC_DAPM_INPUT("LINPUT2"),
+	SND_SOC_DAPM_INPUT("RINPUT1"),
+	SND_SOC_DAPM_INPUT("RINPUT2"),
+};
+
+static const struct snd_soc_dapm_route es8328_dapm_routes[] = {
+
+	{ "Left Line Mux", "Line 1", "LINPUT1" },
+	{ "Left Line Mux", "Line 2", "LINPUT2" },
+	{ "Left Line Mux", "PGA", "Left PGA Mux" },
+	{ "Left Line Mux", "Differential", "Differential Mux" },
+
+	{ "Right Line Mux", "Line 1", "RINPUT1" },
+	{ "Right Line Mux", "Line 2", "RINPUT2" },
+	{ "Right Line Mux", "PGA", "Right PGA Mux" },
+	{ "Right Line Mux", "Differential", "Differential Mux" },
+
+	{ "Left PGA Mux", "Line 1", "LINPUT1" },
+	{ "Left PGA Mux", "Line 2", "LINPUT2" },
+	{ "Left PGA Mux", "Differential", "Differential Mux" },
+
+	{ "Right PGA Mux", "Line 1", "RINPUT1" },
+	{ "Right PGA Mux", "Line 2", "RINPUT2" },
+	{ "Right PGA Mux", "Differential", "Differential Mux" },
+
+	{ "Differential Mux", "Line 1", "LINPUT1" },
+	{ "Differential Mux", "Line 1", "RINPUT1" },
+	{ "Differential Mux", "Line 2", "LINPUT2" },
+	{ "Differential Mux", "Line 2", "RINPUT2" },
+
+	{ "Left ADC Mux", "Stereo", "Left PGA Mux" },
+	{ "Left ADC Mux", "Mono (Left)", "Left PGA Mux" },
+	{ "Left ADC Mux", "Digital Mono", "Left PGA Mux" },
+
+	{ "Right ADC Mux", "Stereo", "Right PGA Mux" },
+	{ "Right ADC Mux", "Mono (Right)", "Right PGA Mux" },
+	{ "Right ADC Mux", "Digital Mono", "Right PGA Mux" },
+
+	{ "Left ADC", NULL, "Left ADC Mux" },
+	{ "Right ADC", NULL, "Right ADC Mux" },
+
+	{ "ADC DIG", NULL, "ADC STM" },
+	{ "ADC DIG", NULL, "ADC Vref" },
+	{ "ADC DIG", NULL, "ADC DLL" },
+
+	{ "Left ADC", NULL, "ADC DIG" },
+	{ "Right ADC", NULL, "ADC DIG" },
+
+	{ "Mic Bias", NULL, "Mic Bias Gen" },
+
+	{ "Left ADC", NULL, "Mic Bias" },
+	{ "Right ADC", NULL, "Mic Bias" },
+
+	{ "Left Line Mux", "Line 1", "LINPUT1" },
+	{ "Left Line Mux", "Line 2", "LINPUT2" },
+	{ "Left Line Mux", "PGA", "Left PGA Mux" },
+	{ "Left Line Mux", "Differential", "Differential Mux" },
+
+	{ "Right Line Mux", "Line 1", "RINPUT1" },
+	{ "Right Line Mux", "Line 2", "RINPUT2" },
+	{ "Right Line Mux", "PGA", "Right PGA Mux" },
+	{ "Right Line Mux", "Differential", "Differential Mux" },
+
+	{ "Left Out 1", NULL, "Left DAC" },
+	{ "Right Out 1", NULL, "Right DAC" },
+	{ "Left Out 2", NULL, "Left DAC" },
+	{ "Right Out 2", NULL, "Right DAC" },
+
+	{ "Left Mixer", "Playback Switch", "Left DAC" },
+	{ "Left Mixer", "Left Bypass Switch", "Left Line Mux" },
+	{ "Left Mixer", "Right Playback Switch", "Right DAC" },
+	{ "Left Mixer", "Right Bypass Switch", "Right Line Mux" },
+
+	{ "Right Mixer", "Left Playback Switch", "Left DAC" },
+	{ "Right Mixer", "Left Bypass Switch", "Left Line Mux" },
+	{ "Right Mixer", "Playback Switch", "Right DAC" },
+	{ "Right Mixer", "Right Bypass Switch", "Right Line Mux" },
+
+	{ "DAC DIG", NULL, "DAC STM" },
+	{ "DAC DIG", NULL, "DAC Vref" },
+	{ "DAC DIG", NULL, "DAC DLL" },
+
+	{ "Left DAC", NULL, "DAC DIG" },
+	{ "Right DAC", NULL, "DAC DIG" },
+
+	{ "Left Out 1", NULL, "Left Mixer" },
+	{ "LOUT1", NULL, "Left Out 1" },
+	{ "Right Out 1", NULL, "Right Mixer" },
+	{ "ROUT1", NULL, "Right Out 1" },
+
+	{ "Left Out 2", NULL, "Left Mixer" },
+	{ "LOUT2", NULL, "Left Out 2" },
+	{ "Right Out 2", NULL, "Right Mixer" },
+	{ "ROUT2", NULL, "Right Out 2" },
+};
+
+static int es8328_mute(struct snd_soc_dai *dai, int mute, int direction)
+{
+	return snd_soc_component_update_bits(dai->component, ES8328_DACCONTROL3,
+			ES8328_DACCONTROL3_DACMUTE,
+			mute ? ES8328_DACCONTROL3_DACMUTE : 0);
+}
+
+static int es8328_startup(struct snd_pcm_substream *substream,
+			  struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+
+	if (es8328->provider && es8328->sysclk_constraints)
+		snd_pcm_hw_constraint_list(substream->runtime, 0,
+				SNDRV_PCM_HW_PARAM_RATE,
+				es8328->sysclk_constraints);
+
+	return 0;
+}
+
+static int es8328_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params,
+	struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+	int i;
+	int reg;
+	int wl;
+	int ratio;
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		reg = ES8328_DACCONTROL2;
+	else
+		reg = ES8328_ADCCONTROL5;
+
+	if (es8328->provider) {
+		if (!es8328->sysclk_constraints) {
+			dev_err(component->dev, "No MCLK configured\n");
+			return -EINVAL;
+		}
+
+		for (i = 0; i < es8328->sysclk_constraints->count; i++)
+			if (es8328->sysclk_constraints->list[i] ==
+			    params_rate(params))
+				break;
+
+		if (i == es8328->sysclk_constraints->count) {
+			dev_err(component->dev,
+				"LRCLK %d unsupported with current clock\n",
+				params_rate(params));
+			return -EINVAL;
+		}
+		ratio = es8328->mclk_ratios[i];
+	} else {
+		ratio = 0;
+		es8328->mclkdiv2 = 0;
+	}
+
+	snd_soc_component_update_bits(component, ES8328_MASTERMODE,
+			ES8328_MASTERMODE_MCLKDIV2,
+			es8328->mclkdiv2 ? ES8328_MASTERMODE_MCLKDIV2 : 0);
+
+	switch (params_width(params)) {
+	case 16:
+		wl = 3;
+		break;
+	case 18:
+		wl = 2;
+		break;
+	case 20:
+		wl = 1;
+		break;
+	case 24:
+		wl = 0;
+		break;
+	case 32:
+		wl = 4;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		snd_soc_component_update_bits(component, ES8328_DACCONTROL1,
+				ES8328_DACCONTROL1_DACWL_MASK,
+				wl << ES8328_DACCONTROL1_DACWL_SHIFT);
+
+		es8328->playback_fs = params_rate(params);
+		es8328_set_deemph(component);
+	} else
+		snd_soc_component_update_bits(component, ES8328_ADCCONTROL4,
+				ES8328_ADCCONTROL4_ADCWL_MASK,
+				wl << ES8328_ADCCONTROL4_ADCWL_SHIFT);
+
+	return snd_soc_component_update_bits(component, reg, ES8328_RATEMASK, ratio);
+}
+
+static int es8328_set_sysclk(struct snd_soc_dai *codec_dai,
+		int clk_id, unsigned int freq, int dir)
+{
+	struct snd_soc_component *component = codec_dai->component;
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+	int mclkdiv2 = 0;
+
+	switch (freq) {
+	case 0:
+		es8328->sysclk_constraints = NULL;
+		es8328->mclk_ratios = NULL;
+		break;
+	case 22579200:
+		mclkdiv2 = 1;
+		fallthrough;
+	case 11289600:
+		es8328->sysclk_constraints = &constraints_11289;
+		es8328->mclk_ratios = ratios_11289;
+		break;
+	case 24576000:
+		mclkdiv2 = 1;
+		fallthrough;
+	case 12288000:
+		es8328->sysclk_constraints = &constraints_12288;
+		es8328->mclk_ratios = ratios_12288;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	es8328->mclkdiv2 = mclkdiv2;
+	return 0;
+}
+
+static int es8328_set_dai_fmt(struct snd_soc_dai *codec_dai,
+		unsigned int fmt)
+{
+	struct snd_soc_component *component = codec_dai->component;
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+	u8 dac_mode = 0;
+	u8 adc_mode = 0;
+
+	switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) {
+	case SND_SOC_DAIFMT_CBP_CFP:
+		/* Master serial port mode, with BCLK generated automatically */
+		snd_soc_component_update_bits(component, ES8328_MASTERMODE,
+				    ES8328_MASTERMODE_MSC,
+				    ES8328_MASTERMODE_MSC);
+		es8328->provider = true;
+		break;
+	case SND_SOC_DAIFMT_CBC_CFC:
+		/* Slave serial port mode */
+		snd_soc_component_update_bits(component, ES8328_MASTERMODE,
+				    ES8328_MASTERMODE_MSC, 0);
+		es8328->provider = false;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* interface format */
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		dac_mode |= ES8328_DACCONTROL1_DACFORMAT_I2S;
+		adc_mode |= ES8328_ADCCONTROL4_ADCFORMAT_I2S;
+		break;
+	case SND_SOC_DAIFMT_RIGHT_J:
+		dac_mode |= ES8328_DACCONTROL1_DACFORMAT_RJUST;
+		adc_mode |= ES8328_ADCCONTROL4_ADCFORMAT_RJUST;
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		dac_mode |= ES8328_DACCONTROL1_DACFORMAT_LJUST;
+		adc_mode |= ES8328_ADCCONTROL4_ADCFORMAT_LJUST;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* clock inversion */
+	if ((fmt & SND_SOC_DAIFMT_INV_MASK) != SND_SOC_DAIFMT_NB_NF)
+		return -EINVAL;
+
+	snd_soc_component_update_bits(component, ES8328_DACCONTROL1,
+			ES8328_DACCONTROL1_DACFORMAT_MASK, dac_mode);
+	snd_soc_component_update_bits(component, ES8328_ADCCONTROL4,
+			ES8328_ADCCONTROL4_ADCFORMAT_MASK, adc_mode);
+	snd_soc_component_update_bits(component, ES8328_DACCONTROL21,
+			ES8328_DACCONTROL21_SLRCK, ES8328_DACCONTROL21_SLRCK);
+
+	/* Set Capture Digital Volume */
+	snd_soc_component_write(component, ES8328_ADCCONTROL8, 0);
+	snd_soc_component_write(component, ES8328_ADCCONTROL9, 0);
+
+	/* Set PCM Volume */
+	snd_soc_component_write(component, ES8328_LDACVOL, 0);
+	snd_soc_component_write(component, ES8328_RDACVOL, 0);
+
+	/* Set L/R Out Volume */
+	snd_soc_component_write(component, ES8328_LOUT1VOL, 0x1e);
+	snd_soc_component_write(component, ES8328_ROUT1VOL, 0x1e);
+	snd_soc_component_write(component, ES8328_LOUT2VOL, 0x1e);
+	snd_soc_component_write(component, ES8328_ROUT2VOL, 0x1e);
+
+	/* Set MIC PGA Volume */
+	snd_soc_component_write(component, ES8328_ADCCONTROL1, 0x88);
+
+	if (es8328->eswin_plat == 2) {
+		if (gpiod_get_value(es8328->front_jack_gpio) == 1 && gpiod_get_value(es8328->back_jack_gpio) == 0) {
+			/* Select default capture path ---> LIN1 */
+			snd_soc_component_write(component, ES8328_ADCCONTROL2, 0);
+		} else {
+			/* Select default capture path ---> LIN2 */
+			snd_soc_component_write(component, ES8328_ADCCONTROL2, 0x50);
+		}
+	} else {
+		/* Select default capture path ---> phone mic */
+		snd_soc_component_write(component, ES8328_ADCCONTROL2, 0xf0);
+	}
+
+	snd_soc_component_update_bits(component, ES8328_ADCCONTROL3,
+			ES8328_ADCCONTROL3_DS, 0);
+
+	/* Select Playback path */
+	snd_soc_component_update_bits(component, ES8328_DACCONTROL17,
+			ES8328_DACCONTROL17_LD2LO, ES8328_DACCONTROL17_LD2LO);
+	snd_soc_component_update_bits(component, ES8328_DACCONTROL20,
+			ES8328_DACCONTROL20_RD2RO, ES8328_DACCONTROL20_RD2RO);
+
+	return 0;
+}
+
+static int es8328_set_bias_level(struct snd_soc_component *component,
+				 enum snd_soc_bias_level level)
+{
+	switch (level) {
+	case SND_SOC_BIAS_ON:
+		break;
+
+	case SND_SOC_BIAS_PREPARE:
+		/* VREF, VMID=2x50k, digital enabled */
+		snd_soc_component_write(component, ES8328_CHIPPOWER, 0);
+		snd_soc_component_update_bits(component, ES8328_CONTROL1,
+				ES8328_CONTROL1_VMIDSEL_MASK |
+				ES8328_CONTROL1_ENREF,
+				ES8328_CONTROL1_VMIDSEL_50k |
+				ES8328_CONTROL1_ENREF);
+		break;
+
+	case SND_SOC_BIAS_STANDBY:
+		if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF) {
+			snd_soc_component_update_bits(component, ES8328_CONTROL1,
+					ES8328_CONTROL1_VMIDSEL_MASK |
+					ES8328_CONTROL1_ENREF,
+					ES8328_CONTROL1_VMIDSEL_5k |
+					ES8328_CONTROL1_ENREF);
+
+			/* Charge caps */
+			msleep(100);
+		}
+
+		snd_soc_component_write(component, ES8328_CONTROL2,
+				ES8328_CONTROL2_OVERCURRENT_ON |
+				ES8328_CONTROL2_THERMAL_SHUTDOWN_ON);
+
+		/* VREF, VMID=2*500k, digital stopped */
+		snd_soc_component_update_bits(component, ES8328_CONTROL1,
+				ES8328_CONTROL1_VMIDSEL_MASK |
+				ES8328_CONTROL1_ENREF,
+				ES8328_CONTROL1_VMIDSEL_500k |
+				ES8328_CONTROL1_ENREF);
+		break;
+
+	case SND_SOC_BIAS_OFF:
+		snd_soc_component_update_bits(component, ES8328_CONTROL1,
+				ES8328_CONTROL1_VMIDSEL_MASK |
+				ES8328_CONTROL1_ENREF,
+				0);
+		break;
+	}
+	return 0;
+}
+
+static const struct snd_soc_dai_ops es8328_dai_ops = {
+	.startup	= es8328_startup,
+	.hw_params	= es8328_hw_params,
+	.mute_stream	= es8328_mute,
+	.set_sysclk	= es8328_set_sysclk,
+	.set_fmt	= es8328_set_dai_fmt,
+	.no_capture_mute = 1,
+};
+
+static struct snd_soc_dai_driver es8328_dai[3] = {
+	{
+		.name = "es8328-0-hifi-analog",
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ES8328_RATES,
+			.formats = ES8328_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ES8328_RATES,
+			.formats = ES8328_FORMATS,
+		},
+		.ops = &es8328_dai_ops,
+		.symmetric_rate = 1,
+	},
+	{
+		.name = "es8328-1-hifi-analog",
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ES8328_RATES,
+			.formats = ES8328_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ES8328_RATES,
+			.formats = ES8328_FORMATS,
+		},
+		.ops = &es8328_dai_ops,
+		.symmetric_rate = 1,
+	},
+	{
+		.name = "es8328-2-hifi-analog",
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ES8328_RATES,
+			.formats = ES8328_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ES8328_RATES,
+			.formats = ES8328_FORMATS,
+		},
+		.ops = &es8328_dai_ops,
+		.symmetric_rate = 1,
+	},
+};
+
+static int es8328_suspend(struct snd_soc_component *component)
+{
+	return 0;
+}
+
+static int es8328_resume(struct snd_soc_component *component)
+{
+	struct regmap *regmap = dev_get_regmap(component->dev, NULL);
+	int ret;
+
+	regcache_mark_dirty(regmap);
+	ret = regcache_sync(regmap);
+	if (ret) {
+		dev_err(component->dev, "unable to sync regcache\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int es8328_component_probe(struct snd_soc_component *component)
+{
+	struct es8328_priv *es8328 = snd_soc_component_get_drvdata(component);
+
+	es8328->component = component;
+
+	return 0;
+}
+
+static void es8328_remove(struct snd_soc_component *component)
+{
+}
+
+const struct regmap_config es8328_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 8,
+	.max_register	= ES8328_REG_MAX,
+	.cache_type	= REGCACHE_MAPLE,
+	.use_single_read = true,
+	.use_single_write = true,
+};
+EXPORT_SYMBOL_GPL(es8328_regmap_config);
+
+static const struct snd_soc_component_driver es8328_component_driver = {
+	.probe			= es8328_component_probe,
+	.remove			= es8328_remove,
+	.suspend		= es8328_suspend,
+	.resume			= es8328_resume,
+	.set_bias_level		= es8328_set_bias_level,
+	.controls		= es8328_snd_controls,
+	.num_controls		= ARRAY_SIZE(es8328_snd_controls),
+	.dapm_widgets		= es8328_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(es8328_dapm_widgets),
+	.dapm_routes		= es8328_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(es8328_dapm_routes),
+	.suspend_bias_off	= 1,
+	.idle_bias_on		= 1,
+	.use_pmdown_time	= 1,
+	.endianness		= 1,
+};
+
+static irqreturn_t es8328_jack_irq(int irq, void *data)
+{
+	struct es8328_priv *es8328 = data;
+	struct snd_soc_component *comp = es8328->component;
+	int front_jack_value, back_jack_value;
+
+	if (!es8328->front_jack_gpio || !es8328->back_jack_gpio) {
+		dev_warn(comp->dev, "jack gpio desc is null\n");
+		return IRQ_NONE;
+	}
+
+	front_jack_value = gpiod_get_value(es8328->front_jack_gpio);
+	back_jack_value = gpiod_get_value(es8328->back_jack_gpio);
+
+	dev_dbg(comp->dev, "front jack value:%d, back jack value:%d\n", front_jack_value, back_jack_value);
+
+	if (back_jack_value == 0 && front_jack_value == 1) {
+		/* Select Capture path ---> LIN1 */
+		regmap_write(comp->regmap, ES8328_ADCCONTROL2, 0);
+	} else {
+		/* Select Capture path ---> LIN2 */
+		regmap_write(comp->regmap, ES8328_ADCCONTROL2, 0x50);
+	}
+
+	return IRQ_HANDLED;
+}
+
+int es8328_probe(struct device *dev, struct regmap *regmap)
+{
+	struct es8328_priv *es8328;
+	int ret;
+
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	es8328 = devm_kzalloc(dev, sizeof(*es8328), GFP_KERNEL);
+	if (es8328 == NULL)
+		return -ENOMEM;
+
+	es8328->regmap = regmap;
+
+	dev_set_drvdata(dev, es8328);
+
+	ret = device_property_read_u32(dev, "eswin-plat", &es8328->eswin_plat);
+	if (0 != ret) {
+		es8328->eswin_plat = 0;
+	}
+	dev_info(dev, "eswin platform:%d\n", es8328->eswin_plat);
+
+	if (es8328->eswin_plat == 2) {
+		es8328->front_jack_gpio = devm_gpiod_get(dev, "front-jack", GPIOD_IN);
+		ret = IS_ERR(es8328->front_jack_gpio);
+		if(ret) {
+			dev_err(dev, "can not get front jack gpio\n");
+		}
+
+		es8328->back_jack_gpio = devm_gpiod_get(dev, "back-jack", GPIOD_IN);
+		ret = IS_ERR(es8328->back_jack_gpio);
+		if(ret) {
+			dev_err(dev, "can not get back jack gpio\n");
+		}
+
+		ret = devm_request_threaded_irq(dev, gpiod_to_irq(es8328->front_jack_gpio), NULL, es8328_jack_irq,
+				IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+				"front jack", es8328);
+		if (ret) {
+			dev_err(dev, "Failed to request front irq[%d], ret:%d\n", gpiod_to_irq(es8328->back_jack_gpio), ret);
+		}
+
+		ret = devm_request_threaded_irq(dev, gpiod_to_irq(es8328->back_jack_gpio), NULL, es8328_jack_irq,
+				IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+				"back jack", es8328);
+		if (ret) {
+			dev_err(dev, "Failed to request back irq[%d], ret:%d\n", gpiod_to_irq(es8328->back_jack_gpio), ret);
+		}
+	}
+
+	if (of_node_name_prefix(dev->of_node, "es8388-0")) {
+		ret = devm_snd_soc_register_component(dev,
+				&es8328_component_driver, &es8328_dai[0], 1);
+	} else if (of_node_name_prefix(dev->of_node, "es8388-1")) {
+		ret = devm_snd_soc_register_component(dev,
+				&es8328_component_driver, &es8328_dai[1], 1);
+	} else {
+		ret = devm_snd_soc_register_component(dev,
+				&es8328_component_driver, &es8328_dai[2], 1);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(es8328_probe);
+
+MODULE_DESCRIPTION("ASoC ES8328 driver");
+MODULE_AUTHOR("Sean Cross <xobs@kosagi.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/eswin/es8328.h b/sound/soc/codecs/eswin/es8328.h
new file mode 100644
index 000000000000..c47988b5ea1b
--- /dev/null
+++ b/sound/soc/codecs/eswin/es8328.h
@@ -0,0 +1,284 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * es8328.h  --  ES8328 ALSA SoC Audio driver
+ */
+
+#ifndef _ES8328_H
+#define _ES8328_H
+
+#include <linux/regmap.h>
+
+struct device;
+
+extern const struct regmap_config es8328_regmap_config;
+int es8328_probe(struct device *dev, struct regmap *regmap);
+
+#define ES8328_DACLVOL 46
+#define ES8328_DACRVOL 47
+#define ES8328_DACCTL 28
+#define ES8328_RATEMASK (0x1f << 0)
+
+#define ES8328_CONTROL1		0x00
+#define ES8328_CONTROL1_VMIDSEL_OFF (0 << 0)
+#define ES8328_CONTROL1_VMIDSEL_50k (1 << 0)
+#define ES8328_CONTROL1_VMIDSEL_500k (2 << 0)
+#define ES8328_CONTROL1_VMIDSEL_5k (3 << 0)
+#define ES8328_CONTROL1_VMIDSEL_MASK (3 << 0)
+#define ES8328_CONTROL1_ENREF (1 << 2)
+#define ES8328_CONTROL1_SEQEN (1 << 3)
+#define ES8328_CONTROL1_SAMEFS (1 << 4)
+#define ES8328_CONTROL1_DACMCLK_ADC (0 << 5)
+#define ES8328_CONTROL1_DACMCLK_DAC (1 << 5)
+#define ES8328_CONTROL1_LRCM (1 << 6)
+#define ES8328_CONTROL1_SCP_RESET (1 << 7)
+
+#define ES8328_CONTROL2		0x01
+#define ES8328_CONTROL2_VREF_BUF_OFF (1 << 0)
+#define ES8328_CONTROL2_VREF_LOWPOWER (1 << 1)
+#define ES8328_CONTROL2_IBIASGEN_OFF (1 << 2)
+#define ES8328_CONTROL2_ANALOG_OFF (1 << 3)
+#define ES8328_CONTROL2_VREF_BUF_LOWPOWER (1 << 4)
+#define ES8328_CONTROL2_VCM_MOD_LOWPOWER (1 << 5)
+#define ES8328_CONTROL2_OVERCURRENT_ON (1 << 6)
+#define ES8328_CONTROL2_THERMAL_SHUTDOWN_ON (1 << 7)
+
+#define ES8328_CHIPPOWER	0x02
+#define ES8328_CHIPPOWER_DACVREF_OFF 0
+#define ES8328_CHIPPOWER_ADCVREF_OFF 1
+#define ES8328_CHIPPOWER_DACDLL_OFF 2
+#define ES8328_CHIPPOWER_ADCDLL_OFF 3
+#define ES8328_CHIPPOWER_DACSTM_RESET 4
+#define ES8328_CHIPPOWER_ADCSTM_RESET 5
+#define ES8328_CHIPPOWER_DACDIG_OFF 6
+#define ES8328_CHIPPOWER_ADCDIG_OFF 7
+
+#define ES8328_ADCPOWER		0x03
+#define ES8328_ADCPOWER_INT1_LOWPOWER 0
+#define ES8328_ADCPOWER_FLASH_ADC_LOWPOWER 1
+#define ES8328_ADCPOWER_ADC_BIAS_GEN_OFF 2
+#define ES8328_ADCPOWER_MIC_BIAS_OFF 3
+#define ES8328_ADCPOWER_ADCR_OFF 4
+#define ES8328_ADCPOWER_ADCL_OFF 5
+#define ES8328_ADCPOWER_AINR_OFF 6
+#define ES8328_ADCPOWER_AINL_OFF 7
+
+#define ES8328_DACPOWER		0x04
+#define ES8328_DACPOWER_OUT3_ON 0
+#define ES8328_DACPOWER_MONO_ON 1
+#define ES8328_DACPOWER_ROUT2_ON 2
+#define ES8328_DACPOWER_LOUT2_ON 3
+#define ES8328_DACPOWER_ROUT1_ON 4
+#define ES8328_DACPOWER_LOUT1_ON 5
+#define ES8328_DACPOWER_RDAC_OFF 6
+#define ES8328_DACPOWER_LDAC_OFF 7
+
+#define ES8328_CHIPLOPOW1	0x05
+#define ES8328_CHIPLOPOW2	0x06
+#define ES8328_ANAVOLMANAG	0x07
+
+#define ES8328_MASTERMODE	0x08
+#define ES8328_MASTERMODE_BCLKDIV (0 << 0)
+#define ES8328_MASTERMODE_BCLK_INV (1 << 5)
+#define ES8328_MASTERMODE_MCLKDIV2 (1 << 6)
+#define ES8328_MASTERMODE_MSC (1 << 7)
+
+#define ES8328_ADCCONTROL1	0x09
+#define ES8328_ADCCONTROL2	0x0a
+#define ES8328_ADCCONTROL3	0x0b
+#define ES8328_ADCCONTROL3_DS (1 << 7)
+
+#define ES8328_ADCCONTROL4	0x0c
+#define ES8328_ADCCONTROL4_ADCFORMAT_MASK (3 << 0)
+#define ES8328_ADCCONTROL4_ADCFORMAT_I2S (0 << 0)
+#define ES8328_ADCCONTROL4_ADCFORMAT_LJUST (1 << 0)
+#define ES8328_ADCCONTROL4_ADCFORMAT_RJUST (2 << 0)
+#define ES8328_ADCCONTROL4_ADCFORMAT_PCM (3 << 0)
+#define ES8328_ADCCONTROL4_ADCWL_SHIFT 2
+#define ES8328_ADCCONTROL4_ADCWL_MASK (7 << 2)
+#define ES8328_ADCCONTROL4_ADCLRP_I2S_POL_NORMAL (0 << 5)
+#define ES8328_ADCCONTROL4_ADCLRP_I2S_POL_INV (1 << 5)
+#define ES8328_ADCCONTROL4_ADCLRP_PCM_MSB_CLK2 (0 << 5)
+#define ES8328_ADCCONTROL4_ADCLRP_PCM_MSB_CLK1 (1 << 5)
+
+#define ES8328_ADCCONTROL5	0x0d
+#define ES8328_ADCCONTROL5_RATEMASK (0x1f << 0)
+
+#define ES8328_ADCCONTROL6	0x0e
+
+#define ES8328_ADCCONTROL7	0x0f
+#define ES8328_ADCCONTROL7_ADC_MUTE (1 << 2)
+#define ES8328_ADCCONTROL7_ADC_LER (1 << 3)
+#define ES8328_ADCCONTROL7_ADC_ZERO_CROSS (1 << 4)
+#define ES8328_ADCCONTROL7_ADC_SOFT_RAMP (1 << 5)
+#define ES8328_ADCCONTROL7_ADC_RAMP_RATE_4 (0 << 6)
+#define ES8328_ADCCONTROL7_ADC_RAMP_RATE_8 (1 << 6)
+#define ES8328_ADCCONTROL7_ADC_RAMP_RATE_16 (2 << 6)
+#define ES8328_ADCCONTROL7_ADC_RAMP_RATE_32 (3 << 6)
+
+#define ES8328_ADCCONTROL8	0x10
+#define ES8328_ADCCONTROL9	0x11
+#define ES8328_ADCCONTROL10	0x12
+#define ES8328_ADCCONTROL11	0x13
+#define ES8328_ADCCONTROL12	0x14
+#define ES8328_ADCCONTROL13	0x15
+#define ES8328_ADCCONTROL14	0x16
+
+#define ES8328_DACCONTROL1	0x17
+#define ES8328_DACCONTROL1_DACFORMAT_MASK (3 << 1)
+#define ES8328_DACCONTROL1_DACFORMAT_I2S (0 << 1)
+#define ES8328_DACCONTROL1_DACFORMAT_LJUST (1 << 1)
+#define ES8328_DACCONTROL1_DACFORMAT_RJUST (2 << 1)
+#define ES8328_DACCONTROL1_DACFORMAT_PCM (3 << 1)
+#define ES8328_DACCONTROL1_DACWL_SHIFT 3
+#define ES8328_DACCONTROL1_DACWL_MASK (7 << 3)
+#define ES8328_DACCONTROL1_DACLRP_I2S_POL_NORMAL (0 << 6)
+#define ES8328_DACCONTROL1_DACLRP_I2S_POL_INV (1 << 6)
+#define ES8328_DACCONTROL1_DACLRP_PCM_MSB_CLK2 (0 << 6)
+#define ES8328_DACCONTROL1_DACLRP_PCM_MSB_CLK1 (1 << 6)
+#define ES8328_DACCONTROL1_LRSWAP (1 << 7)
+
+#define ES8328_DACCONTROL2	0x18
+#define ES8328_DACCONTROL2_RATEMASK (0x1f << 0)
+#define ES8328_DACCONTROL2_DOUBLESPEED (1 << 5)
+
+#define ES8328_DACCONTROL3	0x19
+#define ES8328_DACCONTROL3_AUTOMUTE (1 << 2)
+#define ES8328_DACCONTROL3_DACMUTE (1 << 2)
+#define ES8328_DACCONTROL3_LEFTGAINVOL (1 << 3)
+#define ES8328_DACCONTROL3_DACZEROCROSS (1 << 4)
+#define ES8328_DACCONTROL3_DACSOFTRAMP (1 << 5)
+#define ES8328_DACCONTROL3_DACRAMPRATE (3 << 6)
+
+#define ES8328_LDACVOL 0x1a
+#define ES8328_LDACVOL_MASK (0 << 0)
+#define ES8328_LDACVOL_MAX (0xc0)
+
+#define ES8328_RDACVOL 0x1b
+#define ES8328_RDACVOL_MASK (0 << 0)
+#define ES8328_RDACVOL_MAX (0xc0)
+
+#define ES8328_DACCONTROL6	0x1c
+#define ES8328_DACCONTROL6_CLICKFREE (1 << 3)
+#define ES8328_DACCONTROL6_DAC_INVR (1 << 4)
+#define ES8328_DACCONTROL6_DAC_INVL (1 << 5)
+#define ES8328_DACCONTROL6_DEEMPH_MASK (3 << 6)
+#define ES8328_DACCONTROL6_DEEMPH_OFF (0 << 6)
+#define ES8328_DACCONTROL6_DEEMPH_32k (1 << 6)
+#define ES8328_DACCONTROL6_DEEMPH_44_1k (2 << 6)
+#define ES8328_DACCONTROL6_DEEMPH_48k (3 << 6)
+
+#define ES8328_DACCONTROL7	0x1d
+#define ES8328_DACCONTROL7_VPP_SCALE_3p5	(0 << 0)
+#define ES8328_DACCONTROL7_VPP_SCALE_4p0	(1 << 0)
+#define ES8328_DACCONTROL7_VPP_SCALE_3p0	(2 << 0)
+#define ES8328_DACCONTROL7_VPP_SCALE_2p5	(3 << 0)
+#define ES8328_DACCONTROL7_SHELVING_STRENGTH (1 << 2) /* In eights */
+#define ES8328_DACCONTROL7_MONO		(1 << 5)
+#define ES8328_DACCONTROL7_ZEROR	(1 << 6)
+#define ES8328_DACCONTROL7_ZEROL	(1 << 7)
+
+/* Shelving filter */
+#define ES8328_DACCONTROL8	0x1e
+#define ES8328_DACCONTROL9	0x1f
+#define ES8328_DACCONTROL10	0x20
+#define ES8328_DACCONTROL11	0x21
+#define ES8328_DACCONTROL12	0x22
+#define ES8328_DACCONTROL13	0x23
+#define ES8328_DACCONTROL14	0x24
+#define ES8328_DACCONTROL15	0x25
+
+#define ES8328_DACCONTROL16	0x26
+#define ES8328_DACCONTROL16_RMIXSEL_RIN1 (0 << 0)
+#define ES8328_DACCONTROL16_RMIXSEL_RIN2 (1 << 0)
+#define ES8328_DACCONTROL16_RMIXSEL_RIN3 (2 << 0)
+#define ES8328_DACCONTROL16_RMIXSEL_RADC (3 << 0)
+#define ES8328_DACCONTROL16_LMIXSEL_LIN1 (0 << 3)
+#define ES8328_DACCONTROL16_LMIXSEL_LIN2 (1 << 3)
+#define ES8328_DACCONTROL16_LMIXSEL_LIN3 (2 << 3)
+#define ES8328_DACCONTROL16_LMIXSEL_LADC (3 << 3)
+
+#define ES8328_DACCONTROL17	0x27
+#define ES8328_DACCONTROL17_LI2LOVOL (7 << 3)
+#define ES8328_DACCONTROL17_LI2LO (1 << 6)
+#define ES8328_DACCONTROL17_LD2LO (1 << 7)
+
+#define ES8328_DACCONTROL18	0x28
+#define ES8328_DACCONTROL18_RI2LOVOL (7 << 3)
+#define ES8328_DACCONTROL18_RI2LO (1 << 6)
+#define ES8328_DACCONTROL18_RD2LO (1 << 7)
+
+#define ES8328_DACCONTROL19	0x29
+#define ES8328_DACCONTROL19_LI2ROVOL (7 << 3)
+#define ES8328_DACCONTROL19_LI2RO (1 << 6)
+#define ES8328_DACCONTROL19_LD2RO (1 << 7)
+
+#define ES8328_DACCONTROL20	0x2a
+#define ES8328_DACCONTROL20_RI2ROVOL (7 << 3)
+#define ES8328_DACCONTROL20_RI2RO (1 << 6)
+#define ES8328_DACCONTROL20_RD2RO (1 << 7)
+
+#define ES8328_DACCONTROL21	0x2b
+#define ES8328_DACCONTROL21_SLRCK (1 << 7)
+
+#define ES8328_DACCONTROL22	0x2c
+#define ES8328_DACCONTROL22_RI2MOVOL (7 << 3)
+#define ES8328_DACCONTROL22_RI2MO (1 << 6)
+#define ES8328_DACCONTROL22_RD2MO (1 << 7)
+
+#define ES8328_DACCONTROL23	0x2d
+#define ES8328_DACCONTROL23_MOUTINV		(1 << 1)
+#define ES8328_DACCONTROL23_HPSWPOL		(1 << 2)
+#define ES8328_DACCONTROL23_HPSWEN		(1 << 3)
+#define ES8328_DACCONTROL23_VROI_1p5k		(0 << 4)
+#define ES8328_DACCONTROL23_VROI_40k		(1 << 4)
+#define ES8328_DACCONTROL23_OUT3_VREF		(0 << 5)
+#define ES8328_DACCONTROL23_OUT3_ROUT1		(1 << 5)
+#define ES8328_DACCONTROL23_OUT3_MONOOUT	(2 << 5)
+#define ES8328_DACCONTROL23_OUT3_RIGHT_MIXER	(3 << 5)
+#define ES8328_DACCONTROL23_ROUT2INV		(1 << 7)
+
+/* LOUT1 Amplifier */
+#define ES8328_LOUT1VOL 0x2e
+#define ES8328_LOUT1VOL_MASK (0 << 5)
+#define ES8328_LOUT1VOL_MAX (0x24)
+
+/* ROUT1 Amplifier */
+#define ES8328_ROUT1VOL 0x2f
+#define ES8328_ROUT1VOL_MASK (0 << 5)
+#define ES8328_ROUT1VOL_MAX (0x24)
+
+#define ES8328_OUT1VOL_MAX (0x24)
+
+/* LOUT2 Amplifier */
+#define ES8328_LOUT2VOL 0x30
+#define ES8328_LOUT2VOL_MASK (0 << 5)
+#define ES8328_LOUT2VOL_MAX (0x24)
+
+/* ROUT2 Amplifier */
+#define ES8328_ROUT2VOL 0x31
+#define ES8328_ROUT2VOL_MASK (0 << 5)
+#define ES8328_ROUT2VOL_MAX (0x24)
+
+#define ES8328_OUT2VOL_MAX (0x24)
+
+/* Mono Out Amplifier */
+#define ES8328_MONOOUTVOL 0x32
+#define ES8328_MONOOUTVOL_MASK (0 << 5)
+#define ES8328_MONOOUTVOL_MAX (0x24)
+
+#define ES8328_DACCONTROL29	0x33
+#define ES8328_DACCONTROL30	0x34
+
+#define ES8328_SYSCLK		0
+
+#define ES8328_REG_MAX		0x35
+
+#define ES8328_1536FS		1536
+#define ES8328_1024FS		1024
+#define ES8328_768FS		768
+#define ES8328_512FS		512
+#define ES8328_384FS		384
+#define ES8328_256FS		256
+#define ES8328_128FS		128
+
+#endif
diff --git a/sound/soc/eswin/Kconfig b/sound/soc/eswin/Kconfig
new file mode 100644
index 000000000000..7eaebda84b6a
--- /dev/null
+++ b/sound/soc/eswin/Kconfig
@@ -0,0 +1,10 @@
+menu "SND ESWIN SOC"
+
+config SND_ESWIN_DW_I2S
+	tristate "Eswin Dw I2S Device Driver"
+	select SND_SOC_GENERIC_DMAENGINE_PCM
+	help
+	 Say Y or M if you want to add support for I2S driver for
+	 Eswin dw I2S device. The device supports up to
+	 a maximum of 8 channels each for play and record.
+endmenu
diff --git a/sound/soc/eswin/Makefile b/sound/soc/eswin/Makefile
new file mode 100644
index 000000000000..2c133c695dbe
--- /dev/null
+++ b/sound/soc/eswin/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# ESWIN Platform Support
+snd-soc-i2s-objs := esw-i2s.o esw-audio-proc.o
+
+obj-$(CONFIG_SND_ESWIN_DW_I2S) += snd-soc-i2s.o
\ No newline at end of file
diff --git a/sound/soc/eswin/esw-audio-proc.c b/sound/soc/eswin/esw-audio-proc.c
new file mode 100644
index 000000000000..7823348b60ed
--- /dev/null
+++ b/sound/soc/eswin/esw-audio-proc.c
@@ -0,0 +1,496 @@
+/*
+ *
+ * Copyright (C) 2021 ESWIN, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
+#include <linux/kernel.h>
+#include <linux/seq_file.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/uaccess.h>
+#include <linux/eventfd.h>
+#include <linux/ioctl.h>
+#include <linux/device.h>
+#include <linux/mutex.h>
+
+// proc data definition
+typedef enum
+{
+	ADEC_CREATE_CHN,
+	ADEC_SEND_STREAM,
+	ADEC_GET_FRAME,
+	ADEC_PARSE_PACKET,
+	ADEC_DECODE_STREAM,
+	AENC_CREATE_CHN,
+	AENC_SEND_FRAME,
+	AENC_GET_STREAM,
+	AENC_ENCODE_FRAME,
+	AGC_PROCESS,
+	ANS_PROCESS,
+	AEC_PROCESS,
+	DRC_PROCESS,
+	EQ_PROCESS,
+	DCBLOCK_PROCESS,
+	VOLUME_PROCESS,
+	SRC_HOST_PROCESS,
+	SRC_DAI_PROCESS,
+	HOST_PROCESS,
+	DAI_PROCESS,
+	AO_START,
+	AO_PROCESS_FRAME,
+	AO_WRITE_FRAME,
+	AO_STOP,
+	AI_START,
+	AI_READ_FRAME,
+	AI_PROCESS_FRAME,
+	AI_STOP,
+} PERF_MARK;
+
+static struct proc_dir_entry *proc_esaudio;
+static int g_switch = 0;
+
+// Device data definition
+#define MAX_PERF_SIZE 1024
+enum DEVICES_ID{
+	INVALID_DEVICE = -1,
+	AO = 0,
+	AI,
+	AENC,
+	ADEC,
+	NUM_DEVICES,
+};
+static const char *device_names[NUM_DEVICES] = {"ao", "ai", "aenc", "adec"};
+static int audio_proc_major[NUM_DEVICES] = {0};
+static struct class *audio_proc_class = NULL;
+static struct device *audio_proc_device[NUM_DEVICES] = {NULL};
+static int32_t *g_perf_data[NUM_DEVICES] = {NULL};
+static DEFINE_MUTEX(audio_proc_class_lock);
+
+static void show_aenc_data(struct seq_file *m)
+{
+	seq_printf(m,"----------------------------------------------------AENC PERF STATISTIC BEGIN"
+				"----------------------------------------------------\n");
+	seq_printf(m, "\n");
+	seq_printf(m, "------------------------------------------------------------------------"
+				"----------------------------------------------------------\n");
+	seq_printf(m, "audio encoder performance(us):\n");
+	seq_printf(m, "%-14s%-14s%-14s%-14s\n", "create_chn", "send_frame", "get_stream", "encode_frame");
+	seq_printf(m, "----------------------------------------------------------------------------------"
+				"------------------------------------------------\n");
+	seq_printf(m, "%-14d%-14d%-14d%-14d\n", g_perf_data[AENC][AENC_CREATE_CHN], g_perf_data[AENC][AENC_SEND_FRAME],
+				g_perf_data[AENC][AENC_GET_STREAM], g_perf_data[AENC][AENC_ENCODE_FRAME]);
+	seq_printf(m, "\n");
+	seq_printf(m, "-----------------------------------------------------AENC PERF STATISTIC END"
+	"-----------------------------------------------------\n");
+}
+
+static void show_adec_data(struct seq_file *m)
+{
+	seq_printf(m, "----------------------------------------------------ADEC PERF STATISTIC BEGIN"
+					"----------------------------------------------------\n");
+	seq_printf(m, "\n");
+	seq_printf(m, "--------------------------------------------------------------------------"
+				"--------------------------------------------------------\n");
+	seq_printf(m, "audio decoder performance(us):\n");
+	seq_printf(m, "%-14s%-14s%-14s%-14s%-14s\n", "create_chn", "send_stream", "get_frame",
+				"parse_packet", "decode_stream");
+	seq_printf(m, "------------------------------------------------------------------------------------"
+				"----------------------------------------------\n");
+	seq_printf(m, "%-14d%-14d%-14d%-14d%-14d\n", g_perf_data[ADEC][ADEC_CREATE_CHN],
+				g_perf_data[ADEC][ADEC_SEND_STREAM], g_perf_data[ADEC][ADEC_GET_FRAME],
+				g_perf_data[ADEC][ADEC_PARSE_PACKET], g_perf_data[ADEC][ADEC_DECODE_STREAM]);
+	seq_printf(m, "\n");
+	seq_printf(m,"-----------------------------------------------------ADEC PERF STATISTIC END"
+				"-----------------------------------------------------\n");
+}
+
+static void show_ao_data(struct seq_file *m)
+{
+	seq_printf(m,"----------------------------------------------------AO PERF STATISTIC BEGIN"
+	"----------------------------------------------------\n");
+	seq_printf(m, "\n");
+	seq_printf(m, "---------------------------------------------------------------------------"
+				"-------------------------------------------------------\n");
+	seq_printf(m, "audio output performance(us):\n");
+	seq_printf(m, "%-14s%-24s%-24s\n", "ao_start", "ao_process_frame", "ao_write_frame");
+	seq_printf(m, "----------------------------------------------------------------------------"
+				"------------------------------------------------------\n");
+	seq_printf(m, "%-14d%-24d%-24d\n", g_perf_data[AO][AO_START], g_perf_data[AO][AO_PROCESS_FRAME],
+				g_perf_data[AO][AO_WRITE_FRAME]);
+	seq_printf(m, "\n");
+	seq_printf(m, "-----------------------------------------------------------------------------"
+				"-----------------------------------------------------\n");
+	seq_printf(m, "audio argorithm performance(ns/1ms):\n");
+	seq_printf(m, "%-14s%-14s%-14s%-14s%-14s%-14s%-14s%-14s%-14s\n", "agc", "ans", "eq", "hpf",
+				"volume", "src-host","src-dai", "host", "dai");
+	seq_printf(m, "------------------------------------------------------------------------------"
+				"----------------------------------------------------\n");
+	seq_printf(m, "%-14d%-14d%-14d%-14d%-14d%-14d%-14d%-14d%-14d\n", g_perf_data[AO][AGC_PROCESS],
+				g_perf_data[AO][ANS_PROCESS],g_perf_data[AO][EQ_PROCESS],
+				g_perf_data[AO][DCBLOCK_PROCESS],g_perf_data[AO][VOLUME_PROCESS],
+				g_perf_data[AO][SRC_HOST_PROCESS],g_perf_data[AO][SRC_DAI_PROCESS],
+				g_perf_data[AO][HOST_PROCESS], g_perf_data[AO][DAI_PROCESS]);
+	seq_printf(m, "\n");
+	seq_printf(m,"-----------------------------------------------------AO PERF STATISTIC END"
+				"-----------------------------------------------------\n");
+}
+
+static void show_ai_data(struct seq_file *m)
+{
+	seq_printf(m,"----------------------------------------------------AI PERF STATISTIC BEGIN"
+				"----------------------------------------------------\n");
+	seq_printf(m, "\n");
+	seq_printf(m, "---------------------------------------------------------------------------"
+				"-------------------------------------------------------\n");
+	seq_printf(m, "audio input performance(us):\n");
+	seq_printf(m, "%-14s%-24s%-24s\n", "ai_start", "ai_read_frame", "ai_process_frame");
+	seq_printf(m, "----------------------------------------------------------------------------"
+				"------------------------------------------------------\n");
+	seq_printf(m, "%-14d%-24d%-24d\n", g_perf_data[AI][AI_START], g_perf_data[AI][AI_READ_FRAME],
+				g_perf_data[AI][AI_PROCESS_FRAME]);
+	seq_printf(m, "\n");
+	seq_printf(m, "-----------------------------------------------------------------------------"
+				"-----------------------------------------------------\n");
+	seq_printf(m, "audio argorithm performance(ns/1ms):\n");
+	seq_printf(m, "%-14s%-14s%-14s%-14s%-14s%-14s%-14s%-14s%-14s%-14s\n", "agc", "ans", "drc", "eq",
+				"hpf", "volume","src-host", "src-dai", "host", "dai");
+	seq_printf(m, "------------------------------------------------------------------------------"
+				"----------------------------------------------------\n");
+	seq_printf(m, "%-14d%-14d%-14d%-14d%-14d%-14d%-14d%-14d%-14d%-14d\n", g_perf_data[AI][AGC_PROCESS],
+		g_perf_data[AI][ANS_PROCESS],g_perf_data[AI][DRC_PROCESS], g_perf_data[AI][EQ_PROCESS],
+		g_perf_data[AI][DCBLOCK_PROCESS], g_perf_data[AI][VOLUME_PROCESS],g_perf_data[AI][SRC_HOST_PROCESS],
+		g_perf_data[AI][SRC_DAI_PROCESS], g_perf_data[AI][HOST_PROCESS], g_perf_data[AI][DAI_PROCESS]);
+	seq_printf(m, "\n");
+	seq_printf(m,"-----------------------------------------------------AI PERF STATISTIC END"
+				"-----------------------------------------------------\n");
+}
+
+
+static int audio_info_show(struct seq_file *m, void *p)
+{
+	int i;
+	const char *fileName = m->file->f_path.dentry->d_name.name;
+	enum DEVICES_ID deviceID = INVALID_DEVICE;
+
+	pr_info("audio_info_show:%s\n", m->file->f_path.dentry->d_name.name);
+
+	for (i = 0; i < NUM_DEVICES; ++i) {
+		if (strcmp(fileName, device_names[i]) == 0) {
+			deviceID = i;
+			break;
+		}
+	}
+
+	if (deviceID == INVALID_DEVICE) {
+		pr_err("deviceID is INVALID\n");
+		return -EINVAL;
+	}
+
+ 	if (g_switch == 0) {
+		seq_printf(m, "The switch is not turned on, pls first turn on the switch.\n");
+		return 0;
+	}
+
+	switch (deviceID) {
+	case AI:
+		show_ai_data(m);
+		break;
+	case AO:
+		show_ao_data(m);
+		break;
+	case AENC:
+		show_aenc_data(m);
+		break;
+	case ADEC:
+		show_adec_data(m);
+		break;
+	default:
+		pr_err("deviceID is INVALID\n");
+		break;
+		}
+	return 0;
+}
+
+static int info_open(struct inode *inode, struct file *flip)
+{
+	return single_open(flip, audio_info_show, NULL);
+}
+
+static int switch_show(struct seq_file *m, void *p)
+{
+	seq_printf(m, "--------------------AUDIO Performance Switch--------------------\n");
+	seq_printf(m, "\n");
+	seq_printf(m, "AUDIO Performance Switch Status Value:%d\n", g_switch);
+	seq_printf(m, "\n");
+	return 0;
+}
+
+static int switch_open(struct inode *inode, struct file *flip)
+{
+	return single_open(flip, switch_show, NULL);
+}
+
+static ssize_t switch_write(struct file *flip, const char __user *buf, size_t size, loff_t *pos)
+{
+	u16 data;
+	u8 value;
+
+	if (size > 2) {
+		return -EINVAL;
+	}
+	if (copy_from_user(&data, buf, size)) {
+		return -EFAULT;
+	}
+	value = data & 0xff;
+
+	value -= '0';
+	if (!(value == 1 || value == 0)) {
+		printk("%s, %d, data=%d is not correct, pls use 1 or 0.\n", __func__, __LINE__, value);
+		return -EINVAL;
+	}
+
+	g_switch = value ? 1 : 0;
+
+	return size;
+}
+
+static struct proc_ops proc_info_fops = {
+	.proc_open = info_open,
+	.proc_read = seq_read,
+	.proc_release = single_release,
+};
+
+static struct proc_ops proc_switch_fops = {
+	.proc_open = switch_open,
+	.proc_read = seq_read,
+	.proc_release = single_release,
+	.proc_write = switch_write,
+};
+
+int audio_create_procfs(void)
+{
+	proc_esaudio = proc_mkdir("es_audio", NULL);
+	if (proc_esaudio == NULL) {
+		pr_err("create es_audio dir err.\n");
+		return -ENOMEM;
+	}
+
+	if (!proc_create("ao", 0644, proc_esaudio, &proc_info_fops)) {
+		pr_err("error create proc ao file.\n");
+		goto err_ao;
+	}
+
+	if (!proc_create("ai", 0644, proc_esaudio, &proc_info_fops)) {
+		pr_err("error create proc ai file.\n");
+		goto err_ai;
+	}
+
+	if (!proc_create("aenc", 0644, proc_esaudio, &proc_info_fops)) {
+		pr_err("error create proc aenc file.\n");
+		goto err_aenc;
+	}
+
+	if (!proc_create("adec", 0644, proc_esaudio, &proc_info_fops)) {
+		pr_err("error create proc adec file.\n");
+		goto err_adec;
+	}
+
+	if (!proc_create("switch", 0644, proc_esaudio, &proc_switch_fops)) {
+		pr_err("error create proc switch file.\n");
+		goto err_switch;
+	}
+
+	return 0;
+
+err_switch:
+	remove_proc_entry("adec", proc_esaudio);
+err_adec:
+	remove_proc_entry("aenc", proc_esaudio);
+err_aenc:
+	remove_proc_entry("ai", proc_esaudio);
+err_ai:
+	remove_proc_entry("ao", proc_esaudio);
+err_ao:
+	remove_proc_entry("es_audio", NULL);
+	return -1;
+}
+
+void audio_remove_procfs(void)
+{
+	remove_proc_entry("switch", proc_esaudio);
+
+	remove_proc_entry("adec", proc_esaudio);
+
+	remove_proc_entry("aenc", proc_esaudio);
+
+	remove_proc_entry("ai", proc_esaudio);
+
+	remove_proc_entry("ao", proc_esaudio);
+
+	remove_proc_entry("es_audio", NULL);
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+// audio dev implementation
+static int audio_dev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	int i;
+	const char *fileName = file->f_path.dentry->d_name.name;
+	enum DEVICES_ID deviceID = INVALID_DEVICE;
+	unsigned long size = vma->vm_end - vma->vm_start;
+
+	if (size > (MAX_PERF_SIZE * sizeof(int32_t))) {
+		pr_err("audio_dev_mmap: size:%ld > %ld.\n", size, MAX_PERF_SIZE * sizeof(int32_t));
+		return -EINVAL;
+	}
+
+	for (i = 0; i < NUM_DEVICES; ++i) {
+		if (strcmp(fileName, device_names[i]) == 0) {
+			deviceID = i;
+			break;
+		}
+	}
+
+	if (deviceID == INVALID_DEVICE) {
+		pr_err("deviceID is INVALID\n");
+		return -EINVAL;
+	}
+
+	// Remap the shared memory into the process's address space
+	if (remap_pfn_range(vma, vma->vm_start, virt_to_phys(g_perf_data[deviceID]) >> PAGE_SHIFT,
+						size, vma->vm_page_prot)) {
+		pr_err("Failed to remap shared memory.\n");
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
+static const struct file_operations dev_fops = {
+	.owner = THIS_MODULE,
+	.mmap = audio_dev_mmap,
+};
+
+static bool g_proc_initialized = false;
+
+int audio_proc_module_init(void)
+{
+	int i, ret;
+	struct device *dev;
+
+	pr_info("audio_proc_module_init enter.\n");
+
+	mutex_lock(&audio_proc_class_lock);
+
+	if (g_proc_initialized) {
+		mutex_unlock(&audio_proc_class_lock);
+		return 0;
+	}
+
+	audio_proc_class = class_create("audio_proc_class");
+	if (IS_ERR(audio_proc_class)) {
+		pr_err("Failed to create audio_proc_class\n");
+		mutex_unlock(&audio_proc_class_lock);
+		return PTR_ERR(audio_proc_class);
+	}
+
+	for (i = 0; i < NUM_DEVICES; ++i) {
+		g_perf_data[i] = kmalloc(MAX_PERF_SIZE * sizeof(int32_t), GFP_KERNEL);
+		if (!g_perf_data[i]) {
+			pr_err("Failed to allocate shared memory for '%s'\n", device_names[i]);
+			goto cleanup;
+		}
+
+		memset(g_perf_data[i], 0, MAX_PERF_SIZE * sizeof(int32_t));
+
+		ret = register_chrdev(0, device_names[i], &dev_fops);
+		if (ret < 0) {
+			pr_err("Failed to register character device '%s'\n", device_names[i]);
+			goto cleanup;
+		}
+
+		audio_proc_major[i] = ret;
+
+		dev = device_create(audio_proc_class, NULL, MKDEV(audio_proc_major[i], 0), NULL, device_names[i]);
+		if (IS_ERR(dev)) {
+			pr_err("Failed to create device node '%s'\n", device_names[i]);
+			goto cleanup;
+		}
+
+		audio_proc_device[i] = dev;
+	}
+
+	audio_create_procfs();
+
+	g_proc_initialized = true;
+
+	pr_info("es_audio_proc: initialized\n");
+	mutex_unlock(&audio_proc_class_lock);
+	return 0;
+
+cleanup:
+	for (i = 0; i < NUM_DEVICES; ++i) {
+		if (g_perf_data[i]) {
+			kfree(g_perf_data[i]);
+			g_perf_data[i] = NULL;
+		}
+
+		if (audio_proc_major[i]) {
+			unregister_chrdev(audio_proc_major[i], device_names[i]);
+		}
+
+		if (audio_proc_device[i]) {
+			device_destroy(audio_proc_class, MKDEV(audio_proc_major[i], 0));
+		}
+	}
+
+	class_destroy(audio_proc_class);
+	mutex_unlock(&audio_proc_class_lock);
+	return ret;
+}
+
+static bool g_proc_uninitialized = false;
+
+void audio_proc_module_exit(void)
+{
+	int i;
+
+	mutex_lock(&audio_proc_class_lock);
+
+	if (g_proc_uninitialized) {
+		mutex_unlock(&audio_proc_class_lock);
+		return;
+	}
+
+	audio_remove_procfs();
+	for (i = 0; i < NUM_DEVICES; ++i) {
+		device_destroy(audio_proc_class, MKDEV(audio_proc_major[i], 0));
+		unregister_chrdev(audio_proc_major[i], device_names[i]);
+		kfree(g_perf_data[i]);
+		g_perf_data[i] = NULL;
+	}
+
+	class_destroy(audio_proc_class);
+
+	g_proc_uninitialized = true;
+
+	mutex_unlock(&audio_proc_class_lock);
+	pr_info("es_audio_proc: uninitialized\n");
+}
diff --git a/sound/soc/eswin/esw-audio-proc.h b/sound/soc/eswin/esw-audio-proc.h
new file mode 100644
index 000000000000..1fd3f377671d
--- /dev/null
+++ b/sound/soc/eswin/esw-audio-proc.h
@@ -0,0 +1,23 @@
+/*
+ *
+ * Copyright (C) 2021 ESWIN, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#ifndef _ES_AUDIO_PROC_H_
+#define _ES_AUDIO_PROC_H_
+
+int audio_proc_module_init(void);
+void audio_proc_module_exit(void);
+
+#endif
diff --git a/sound/soc/eswin/esw-i2s.c b/sound/soc/eswin/esw-i2s.c
new file mode 100755
index 000000000000..d30bc41f5c6d
--- /dev/null
+++ b/sound/soc/eswin/esw-i2s.c
@@ -0,0 +1,905 @@
+/*
+ * ALSA SoC Synopsys I2S Audio Layer
+ *
+ * sound/soc/dwc/designware_i2s.c
+ *
+ * Copyright (C) 2010 ST Microelectronics
+ * Rajeev Kumar <rajeevkumar.linux@gmail.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ *
+ */
+
+/*
+ * Copyright (C) 2021 ESWIN, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#include <sound/pcm.h>
+#include <linux/device.h>
+#include <sound/soc.h>
+#include <linux/pm_runtime.h>
+#include <linux/platform_device.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <sound/pcm_params.h>
+#include <asm/io.h>
+#include <sound/asound.h>
+#include <sound/designware_i2s.h>
+#include <linux/irqreturn.h>
+#include <linux/clk.h>
+#include <sound/soc-dai.h>
+#include <linux/ioport.h>
+#include <linux/err.h>
+#include <linux/pm.h>
+#include <linux/mfd/syscon.h>
+#include <linux/reset.h>
+#include <linux/dma-map-ops.h>
+#include "esw-i2s.h"
+#include "esw-audio-proc.h"
+
+#define VO_MCLK_DIVSOR_MASK    0xff0
+#define VO_MCLK_DIVSOR_OFFSET  4
+
+#define MAX_SAMPLE_RATE_SUPPORT (192000UL)
+#define MAX_SAMPLE_RATE_CLK (MAX_SAMPLE_RATE_SUPPORT * 32 * 2) // 32 bits, 2channels
+
+#define VO_TOP_CSR             0x50280000UL
+#define VO_I2S0_DIV_NUM        0x2000
+#define VO_I2S1_DIV_NUM        0x2004
+#define VO_I2S2_DIV_NUM        0x2008
+#define DIV_NUM_MASK           0x1f
+
+#define ESW_I2S_RATES (SNDRV_PCM_RATE_192000 | \
+			SNDRV_PCM_RATE_96000 | \
+			SNDRV_PCM_RATE_48000 | \
+			SNDRV_PCM_RATE_32000 | \
+			SNDRV_PCM_RATE_16000 | \
+			SNDRV_PCM_RATE_8000)
+#define ESW_I2S_FORMATS (SNDRV_PCM_FMTBIT_S32_LE)
+
+#define I2S0_IO_ADDR 0x51600124
+
+#define HDMI_DAI_NAME "i2s0-hdmi"
+
+static struct clk *g_mclk;
+
+static u32 dmaen_txch[] = {
+	DMAEN_TXCH_0,
+	DMAEN_TXCH_1,
+	DMAEN_TXCH_2,
+	DMAEN_TXCH_3
+};
+
+static u32 dmaen_rxch[] = {
+	DMAEN_RXCH_0,
+	DMAEN_RXCH_1,
+	DMAEN_RXCH_2,
+	DMAEN_RXCH_3
+};
+
+/* Maximum bit resolution of a channel - not uniformly spaced */
+static const u32 fifo_width[COMP_MAX_WORDSIZE] = {
+	12, 16, 20, 24, 32, 0, 0, 0
+};
+
+/* Width of (DMA) bus */
+static const u32 bus_widths[COMP_MAX_DATA_WIDTH] = {
+	DMA_SLAVE_BUSWIDTH_1_BYTE,
+	DMA_SLAVE_BUSWIDTH_2_BYTES,
+	DMA_SLAVE_BUSWIDTH_4_BYTES,
+	DMA_SLAVE_BUSWIDTH_UNDEFINED
+};
+
+static inline u32 i2s_read_reg(void *io_base, int reg)
+{
+	return readl((char *)io_base + reg);
+}
+
+static inline void i2s_write_reg(void *io_base, int reg, u32 val)
+{
+	writel(val, (char *)io_base + reg);
+}
+
+static inline void i2s_disable_channels(struct i2s_dev *i2s_drvdata, u32 stream)
+{
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		i2s_write_reg(i2s_drvdata->i2s_base, TER(0), 0);
+	} else {
+		i2s_write_reg(i2s_drvdata->i2s_base, RER(0), 0);
+	}
+}
+
+static void i2s_config(struct i2s_dev *i2s_drvdata, int stream)
+{
+	u32 ch_reg;
+	struct i2s_clk_config_data *config = &i2s_drvdata->config;
+	i2s_disable_channels(i2s_drvdata, stream);
+	for (ch_reg = 0; ch_reg < (config->chan_nr / 2); ch_reg++) {
+		if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+			i2s_write_reg(i2s_drvdata->i2s_base, TCR(ch_reg),
+				      i2s_drvdata->xfer_resolution);
+			i2s_write_reg(i2s_drvdata->i2s_base, TFCR(ch_reg),
+				      i2s_drvdata->fifo_th - 1);
+			i2s_write_reg(i2s_drvdata->i2s_base, TER(ch_reg), 1);
+		} else {
+			i2s_write_reg(i2s_drvdata->i2s_base, RCR(ch_reg),
+				      i2s_drvdata->xfer_resolution);
+			i2s_write_reg(i2s_drvdata->i2s_base, RFCR(ch_reg),
+				      i2s_drvdata->fifo_th - 1);
+			i2s_write_reg(i2s_drvdata->i2s_base, RER(ch_reg), 1);
+		}
+    }
+}
+
+static inline void i2s_enable_irqs(struct i2s_dev *i2s_drvdata, u32 stream,
+				   int chan_nr)
+{
+	u32 i, irq;
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			irq = i2s_read_reg(i2s_drvdata->i2s_base, IMR(i));
+			i2s_write_reg(i2s_drvdata->i2s_base, IMR(i), irq & ~0x30);
+		}
+	} else {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			irq = i2s_read_reg(i2s_drvdata->i2s_base, IMR(i));
+			i2s_write_reg(i2s_drvdata->i2s_base, IMR(i), irq & ~0x03);
+		}
+    }
+}
+
+static inline void i2s_enable_dedicated_dma(struct i2s_dev *i2s_drvdata, u32 stream,
+				   int chan_nr)
+{
+	u32 i, dmacr;
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+			i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr | dmaen_txch[i]);
+		}
+	} else {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+			i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr | dmaen_rxch[i]);
+		}
+    }
+}
+
+static inline void i2s_disable_dedicated_dma(struct i2s_dev *i2s_drvdata, u32 stream,
+				   int chan_nr)
+{
+	u32 i, dmacr;
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+			i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr & ~dmaen_txch[i]);
+		}
+	} else {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+			i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr & ~dmaen_rxch[i]);
+		}
+    }
+}
+
+static inline void i2s_enable_combined_dma(struct i2s_dev *i2s_drvdata, u32 stream)
+{
+	u32 dmacr;
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+		i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr | DMAEN_TXBLOCK);
+
+	} else {
+		dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+		i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr | DMAEN_RXBLOCK);
+    }
+}
+
+static inline void i2s_disable_combined_dma(struct i2s_dev *i2s_drvdata, u32 stream)
+{
+	u32 dmacr;
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+		i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr & ~DMAEN_TXBLOCK);
+
+	} else {
+		dmacr = i2s_read_reg(i2s_drvdata->i2s_base, DMACR);
+		i2s_write_reg(i2s_drvdata->i2s_base, DMACR, dmacr & ~DMAEN_RXBLOCK);
+    }
+}
+
+static void i2s_start(struct i2s_dev *i2s_drvdata,
+		      struct snd_pcm_substream *substream)
+{
+	struct i2s_clk_config_data *config = &i2s_drvdata->config;
+	i2s_write_reg(i2s_drvdata->i2s_base, IER, 1);
+	if (i2s_drvdata->use_pio) {
+		i2s_enable_irqs(i2s_drvdata, substream->stream, config->chan_nr);
+	}
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		i2s_write_reg(i2s_drvdata->i2s_base, ITER, 1);
+	} else {
+		i2s_write_reg(i2s_drvdata->i2s_base, IRER, 1);
+	}
+	if (!i2s_drvdata->use_pio) {
+		i2s_enable_dedicated_dma(i2s_drvdata, substream->stream, config->chan_nr);
+	}
+	i2s_write_reg(i2s_drvdata->i2s_base, CER, 1);
+}
+
+static inline void i2s_clear_irqs(struct i2s_dev *i2s_drvdata, u32 stream)
+{
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		i2s_read_reg(i2s_drvdata->i2s_base, TOR(0));
+	} else {
+		i2s_read_reg(i2s_drvdata->i2s_base, ROR(0));
+	}
+}
+
+static inline void i2s_disable_irqs(struct i2s_dev *i2s_drvdata, u32 stream,
+				    int chan_nr)
+{
+	u32 i, irq;
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			irq = i2s_read_reg(i2s_drvdata->i2s_base, IMR(i));
+			i2s_write_reg(i2s_drvdata->i2s_base, IMR(i), irq | 0x30);
+		}
+	} else {
+		for (i = 0; i < (chan_nr / 2); i++) {
+			irq = i2s_read_reg(i2s_drvdata->i2s_base, IMR(i));
+			i2s_write_reg(i2s_drvdata->i2s_base, IMR(i), irq | 0x03);
+		}
+	}
+}
+
+static void i2s_stop(struct i2s_dev *i2s_drvdata,
+		struct snd_pcm_substream *substream)
+{
+	if (i2s_drvdata->use_pio) {
+		i2s_clear_irqs(i2s_drvdata, substream->stream);
+	}
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		i2s_write_reg(i2s_drvdata->i2s_base, ITER, 0);
+	} else {
+		i2s_write_reg(i2s_drvdata->i2s_base, IRER, 0);
+	}
+	if (i2s_drvdata->use_pio) {
+		i2s_disable_irqs(i2s_drvdata, substream->stream, 2);
+	} else {
+		i2s_disable_dedicated_dma(i2s_drvdata, substream->stream, 2);
+	}
+	if (!i2s_drvdata->active) {
+		i2s_write_reg(i2s_drvdata->i2s_base, CER, 0);
+		i2s_write_reg(i2s_drvdata->i2s_base, IER, 0);
+	}
+}
+
+static int i2s_configure_res_by_dt(struct i2s_dev *dev,
+								   struct resource *res)
+{
+	struct snd_soc_component *component;
+	struct dmaengine_pcm *pcm;
+	u32 comp1 = i2s_read_reg(dev->i2s_base, I2S_COMP_PARAM_1);
+	u32 comp2 = i2s_read_reg(dev->i2s_base, I2S_COMP_PARAM_2);
+	u32 fifo_depth;
+	u32 idx;
+	u32 idx2;
+
+	dev_info(dev->dev, "comp1:0x%x, comp2:0x%x\n", comp1, comp2);
+	fifo_depth = 1 << (1 + COMP1_FIFO_DEPTH_GLOBAL(comp1));
+	idx = COMP1_APB_DATA_WIDTH(comp1);
+
+	if (WARN_ON(idx >= ARRAY_SIZE(bus_widths))) {
+		dev_err(dev->dev, "idx:%d inval\n", idx);
+		return -EINVAL;
+	}
+
+	if (COMP1_MODE_EN(comp1)) {
+		dev_dbg(dev->dev, "eswin: i2s master mode supported\n");
+		dev->capability |= DW_I2S_MASTER;
+	} else {
+		dev_dbg(dev->dev, "eswin: i2s slave mode supported\n");
+		dev->capability |= DW_I2S_SLAVE;
+	}
+	dev->fifo_th = fifo_depth / 2;
+
+	component = snd_soc_lookup_component(dev->dev, SND_DMAENGINE_PCM_DRV_NAME);
+	if (!component) {
+		dev_err(dev->dev, "Can not find snd_soc_component\n");
+		return -1;
+	}
+
+	pcm = soc_component_to_pcm(component);
+	if (COMP1_TX_ENABLED(comp1)) {
+		idx2 = COMP1_TX_WORDSIZE_0(comp1);
+		dev->capability |= DWC_I2S_PLAY;
+		/* only  configure Combined DMA addr, Our scenario is not Dedicated DMA case */
+		dev->play_dma_data.addr_width = bus_widths[idx];
+		dev->play_dma_data.fifo_size = fifo_depth *
+			(fifo_width[idx2]) >> 3;
+
+		dev->play_dma_data.addr = res->start + TXDMA_CH(0);
+		dev->play_dma_data.maxburst = 16;
+	}
+	if (COMP1_RX_ENABLED(comp1)) {
+		idx2 = COMP2_RX_WORDSIZE_0(comp2);
+		dev->capability |= DWC_I2S_RECORD;
+		/* only  configure Combined DMA addr, Our scenario is not Dedicated DMA case */
+		dev->capture_dma_data.addr_width = bus_widths[idx];
+		dev->capture_dma_data.fifo_size = fifo_depth *
+			(fifo_width[idx2]) >> 3;
+
+		dev->capture_dma_data.addr = res->start + RXDMA_CH(0);
+		dev->capture_dma_data.maxburst = 16;
+	}
+	return 0;
+}
+
+static int i2s_startup(struct snd_pcm_substream *substream,
+		struct snd_soc_dai *cpu_dai)
+{
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+	struct snd_soc_dai_link *dai_link = rtd->dai_link;
+
+	dai_link->trigger_stop = SND_SOC_TRIGGER_ORDER_LDC;
+
+	if (!strcmp(cpu_dai->name, HDMI_DAI_NAME))
+	{
+		dai_link->playback_only = 1;
+	}
+
+	return 0;
+}
+
+
+static int i2s_hw_params(struct snd_pcm_substream *substream,
+		struct snd_pcm_hw_params *params, struct snd_soc_dai *dai)
+{
+	struct i2s_dev *i2s_drvdata = snd_soc_dai_get_drvdata(dai);
+	struct i2s_clk_config_data *config = &i2s_drvdata->config;
+	uint32_t div_num = 0;
+	uint32_t div_num_reg;
+
+	dev_dbg(i2s_drvdata->dev, "sample rate:%d, chan:%d, width:%d\n",
+			 params_rate(params), params_channels(params), params_width(params));
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		config->data_width = 16;
+		i2s_drvdata->ccr = CLOCK_CYCLES_32 << CCR_WSS_POS |
+					NO_CLOCK_GATING;
+		i2s_drvdata->xfer_resolution = RESOLUTION_16_BIT;
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		config->data_width = 24;
+		i2s_drvdata->ccr = CLOCK_CYCLES_32 << CCR_WSS_POS |
+					NO_CLOCK_GATING;
+		i2s_drvdata->xfer_resolution = RESOLUTION_24_BIT;
+		break;
+	case SNDRV_PCM_FORMAT_S32_LE:
+		config->data_width = 32;
+		i2s_drvdata->ccr = CLOCK_CYCLES_32 << CCR_WSS_POS |
+					NO_CLOCK_GATING;
+		i2s_drvdata->xfer_resolution = RESOLUTION_32_BIT;
+		break;
+	default:
+		dev_err(i2s_drvdata->dev, "eswin-i2s: unsupported PCM fmt");
+		return -EINVAL;
+	}
+	config->chan_nr = MAX_CHANNEL_NUM;
+	switch (config->chan_nr) {
+	case TWO_CHANNEL_SUPPORT:
+		break;
+	default:
+		dev_err(i2s_drvdata->dev, "channel not supported\n");
+		return -EINVAL;
+	}
+	i2s_config(i2s_drvdata, substream->stream);
+	i2s_write_reg(i2s_drvdata->i2s_base, CCR, i2s_drvdata->ccr);
+	config->sample_rate = params_rate(params);
+	if (i2s_drvdata->capability & DW_I2S_MASTER) {
+		if (MAX_SAMPLE_RATE_SUPPORT % config->sample_rate != 0) {
+			dev_err(i2s_drvdata->dev, "Not support sample rate: %d\n", config->sample_rate);
+			return -EINVAL;
+		}
+
+		div_num = MAX_SAMPLE_RATE_SUPPORT / config->sample_rate - 1;
+
+		if (i2s_drvdata->active) {
+			if (i2s_drvdata->i2s_div_num != div_num) {
+				dev_err(i2s_drvdata->dev, "Not support the playback and capture clocks are different\n");
+				return -EINVAL;
+			}
+		} else {
+			div_num_reg = i2s_read_reg(i2s_drvdata->i2s_div_base, 0) & ~DIV_NUM_MASK;
+			div_num_reg |= div_num;
+
+			dev_dbg(i2s_drvdata->dev, "div num:0x%x\n", div_num);
+			i2s_drvdata->i2s_div_num = div_num;
+			i2s_write_reg(i2s_drvdata->i2s_div_base, 0, div_num_reg);
+		}
+	}
+
+	return 0;
+}
+
+static int i2s_prepare(struct snd_pcm_substream *substream,
+			  struct snd_soc_dai *dai)
+{
+	struct i2s_dev *i2s_drvdata = snd_soc_dai_get_drvdata(dai);
+	 if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		i2s_write_reg(i2s_drvdata->i2s_base, TXFFR, 1);
+	else
+		i2s_write_reg(i2s_drvdata->i2s_base, RXFFR, 1);
+
+	return 0;
+}
+
+static int i2s_trigger(struct snd_pcm_substream *substream,
+		int cmd, struct snd_soc_dai *dai)
+{
+	struct i2s_dev *i2s_drvdata = snd_soc_dai_get_drvdata(dai);
+	int ret = 0;
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		i2s_drvdata->active++;
+		i2s_start(i2s_drvdata, substream);
+		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+			i2s_drvdata->playback_active = true;
+		} else {
+			i2s_drvdata->capture_active = true;
+		}
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		i2s_drvdata->active--;
+		i2s_stop(i2s_drvdata, substream);
+		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+			i2s_drvdata->playback_active = false;
+		} else {
+			i2s_drvdata->capture_active = false;
+		}
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int i2s_set_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt)
+{
+	struct i2s_dev *dev = snd_soc_dai_get_drvdata(cpu_dai);
+	int ret = 0;
+	switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) {
+	case SND_SOC_DAIFMT_BC_FC:
+		if (dev->capability & DW_I2S_SLAVE)
+			ret = 0;
+		else
+			ret = -EINVAL;
+		break;
+	case SND_SOC_DAIFMT_BP_FP:
+		if (dev->capability & DW_I2S_MASTER)
+			ret = 0;
+		else
+			ret = -EINVAL;
+		break;
+	case SND_SOC_DAIFMT_BC_FP:
+	case SND_SOC_DAIFMT_BP_FC:
+		ret = -EINVAL;
+		break;
+	default:
+		dev_dbg(dev->dev, "dwc : Invalid clock provider format\n");
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int i2s_pcm_dai_probe(struct snd_soc_dai *dai)
+{
+	struct i2s_dev *i2s_drvdata = snd_soc_dai_get_drvdata(dai);
+
+	snd_soc_dai_init_dma_data(dai, &i2s_drvdata->play_dma_data, &i2s_drvdata->capture_dma_data);
+	return 0;
+}
+
+static const struct snd_soc_dai_ops i2s_dai_ops = {
+	.probe		= i2s_pcm_dai_probe,
+	.startup	= i2s_startup,
+	.hw_params	= i2s_hw_params,
+	.prepare	= i2s_prepare,
+	.trigger	= i2s_trigger,
+	.set_fmt	= i2s_set_fmt,
+};
+
+#ifdef CONFIG_PM
+static int i2s_runtime_suspend(struct device *dev)
+{
+	struct i2s_dev *i2s_drvdata = dev_get_drvdata(dev);
+
+	dev_dbg(i2s_drvdata->dev, "%s\n", __func__);
+
+	clk_disable(g_mclk);
+
+	return 0;
+}
+
+static int i2s_runtime_resume(struct device *dev)
+{
+	struct i2s_dev *i2s_drvdata = dev_get_drvdata(dev);
+
+	dev_dbg(i2s_drvdata->dev, "%s\n", __func__);
+	clk_enable(g_mclk);
+
+	return 0;
+}
+
+static int i2s_suspend(struct snd_soc_component *component)
+{
+	struct i2s_dev *i2s_drvdata = snd_soc_component_get_drvdata(component);
+
+	dev_dbg(i2s_drvdata->dev, "%s\n", __func__);
+	if(!pm_runtime_suspended(i2s_drvdata->dev)) {
+		dev_dbg(i2s_drvdata->dev, "disable clk\n");
+		clk_disable(g_mclk);
+	}
+
+	return 0;
+}
+
+static int i2s_resume(struct snd_soc_component *component)
+{
+	struct i2s_dev *i2s_drvdata = snd_soc_component_get_drvdata(component);
+	struct snd_soc_dai *dai = NULL;
+	int stream;
+
+	dev_dbg(i2s_drvdata->dev, "%s\n", __func__);
+	if(!pm_runtime_suspended(i2s_drvdata->dev)) {
+		dev_dbg(i2s_drvdata->dev, "enable clk\n");
+		clk_enable(g_mclk);
+		for_each_component_dais(component, dai) {
+			for_each_pcm_streams(stream)
+				if (snd_soc_dai_stream_active(dai, stream))
+					i2s_config(i2s_drvdata, stream);
+		}
+	}
+
+	return 0;
+}
+#else
+#define i2s_suspend NULL
+#define i2s_resume NULL
+#endif
+
+static int i2s_reset(struct platform_device *pdev, struct i2s_dev *i2s)
+{
+	struct reset_control *rst;
+	struct reset_control *prst;
+	struct reset_control *voprst;
+	int ret;
+
+	rst = devm_reset_control_get_optional_exclusive(&pdev->dev, "i2srst");
+	if (IS_ERR(rst)) {
+		return PTR_ERR(rst);
+	}
+
+	prst = devm_reset_control_get_optional_exclusive(&pdev->dev, "i2sprst");
+	if (IS_ERR(prst)) {
+		return PTR_ERR(prst);
+	}
+
+	voprst = devm_reset_control_get_optional_exclusive(&pdev->dev, "voprst");
+	if (IS_ERR(prst)) {
+		return PTR_ERR(prst);
+	}
+
+	ret = reset_control_assert(rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_assert(prst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(rst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(prst);
+	WARN_ON(0 != ret);
+	ret = reset_control_deassert(voprst);
+	WARN_ON(0 != ret);
+
+	return 0;
+}
+
+static int i2s_open(struct snd_soc_component *component,
+			      struct snd_pcm_substream *substream)
+{
+	struct i2s_dev *i2s_drvdata = snd_soc_component_get_drvdata(component);
+
+	if ((substream->stream == SNDRV_PCM_STREAM_PLAYBACK && i2s_drvdata->playback_active == true) ||
+		(substream->stream == SNDRV_PCM_STREAM_CAPTURE && i2s_drvdata->capture_active == true)) {
+		dev_err(i2s_drvdata->dev, "i2s is busying\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_component_driver i2s_component = {
+	.name         = "i2s0",
+	.open         = i2s_open,
+	.suspend      = i2s_suspend,
+	.resume       = i2s_resume,
+};
+
+static struct snd_soc_dai_driver i2s_dai[4] = {
+	{
+		.name = HDMI_DAI_NAME,
+		.id = 0,
+		.ops = &i2s_dai_ops,
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ESW_I2S_RATES,
+			.formats = ESW_I2S_FORMATS,
+		},
+	},
+	{
+		.name = "i2s0",
+		.id = 1,
+		.ops = &i2s_dai_ops,
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ESW_I2S_RATES,
+			.formats = ESW_I2S_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ESW_I2S_RATES,
+			.formats = ESW_I2S_FORMATS,
+		},
+	},
+	{
+		.name = "i2s1",
+		.id = 0,
+		.ops = &i2s_dai_ops,
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ESW_I2S_RATES,
+			.formats = ESW_I2S_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ESW_I2S_RATES,
+			.formats = ESW_I2S_FORMATS,
+		},
+	},
+	{
+		.name = "i2s2",
+		.id = 0,
+		.ops = &i2s_dai_ops,
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ESW_I2S_RATES,
+			.formats = ESW_I2S_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture",
+			.channels_min = MIN_CHANNEL_NUM,
+			.channels_max = MAX_CHANNEL_NUM,
+			.rates = ESW_I2S_RATES,
+			.formats = ESW_I2S_FORMATS,
+		},
+	},
+};
+
+static int i2s_probe(struct platform_device *pdev)
+{
+	struct i2s_dev *i2s_drvdata;
+	struct resource *res;
+	int ret;
+	const char *clk_id;
+	struct snd_dmaengine_pcm_config *config;
+	void __iomem *i2s0_io_base;
+	int reg_val;
+
+	dev_info(&pdev->dev, "dev name:%s\n", pdev->dev.of_node->name);
+	i2s_drvdata = devm_kzalloc(&pdev->dev, sizeof(*i2s_drvdata), GFP_KERNEL);
+	if (!i2s_drvdata)
+		return -ENOMEM;
+
+	config = devm_kzalloc(&pdev->dev,
+			sizeof(struct snd_dmaengine_pcm_config), GFP_KERNEL);
+	if (!config)
+		return -ENOMEM;
+	config->chan_names[SNDRV_PCM_STREAM_PLAYBACK] = "tx";
+	config->chan_names[SNDRV_PCM_STREAM_CAPTURE] = "rx";
+	config->prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config;
+
+	res = platform_get_resource(pdev,IORESOURCE_MEM, 0);
+	i2s_drvdata->i2s_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(i2s_drvdata->i2s_base)) {
+		dev_err(&pdev->dev, "devm_ioremap_resource failed\n");
+		return PTR_ERR(i2s_drvdata->i2s_base);
+	}
+	i2s_drvdata->dev = &pdev->dev;
+
+	if (of_node_name_prefix(pdev->dev.of_node, "i2s0")) {
+		clk_id = "mclk";
+		g_mclk = devm_clk_get(&pdev->dev, clk_id);
+		if (IS_ERR(g_mclk))
+			return PTR_ERR(g_mclk);
+		ret = clk_prepare_enable(g_mclk);
+		if (ret < 0)
+			return ret;
+		i2s_drvdata->clk = g_mclk;
+		ret = clk_set_rate(g_mclk, MAX_SAMPLE_RATE_CLK);
+		if (ret) {
+			dev_err(i2s_drvdata->dev, "Can't set I2S clock rate: %d\n", ret);
+		}
+
+		ret = i2s_reset(pdev, i2s_drvdata);
+		if (ret != 0) {
+			dev_err(&pdev->dev, "i2s_reset failed\n");
+			goto err_probe;
+		}
+
+		if (!of_property_read_bool(pdev->dev.of_node, "io_reuse_enable")) {
+			i2s0_io_base = devm_ioremap(&pdev->dev, I2S0_IO_ADDR, 12);
+			if (!i2s0_io_base) {
+				dev_err(i2s_drvdata->dev, "failed to remap i2s0 io ctl\n");
+				return -ENOMEM;
+			}
+
+			/* set the i2s0 WCLK io to GPIO func */
+			reg_val = readl((char *)i2s0_io_base);
+			reg_val &= 0xfff8ffff;
+			reg_val |= 0x20000;
+			writel(reg_val, (char *)i2s0_io_base);
+
+			/* set the i2s0 SDI io to GPIO func */
+			reg_val = readl((char *)i2s0_io_base + 4);
+			reg_val &= 0xfff8ffff;
+			reg_val |= 0x20000;
+			writel(reg_val, (char *)i2s0_io_base + 4);
+
+			/* set the i2s0 SDO io to GPIO func */
+			reg_val = readl((char *)i2s0_io_base + 8);
+			reg_val &= 0xfff8ffff;
+			reg_val |= 0x20000;
+			writel(reg_val, (char *)i2s0_io_base + 8);
+		}
+	}
+
+	dev_set_drvdata(&pdev->dev, i2s_drvdata);
+
+	if (of_node_name_prefix(pdev->dev.of_node, "i2s0")) {
+		i2s_drvdata->i2s_div_base = devm_ioremap(i2s_drvdata->dev, VO_TOP_CSR + VO_I2S0_DIV_NUM, 4);
+		if (!i2s_drvdata->i2s_div_base) {
+			dev_err(&pdev->dev, "failed to remap i2s0 div config\n");
+			ret = -ENOMEM;
+			goto err_probe;
+		}
+		ret = devm_snd_soc_register_component(&pdev->dev, &i2s_component,
+					&i2s_dai[0], 2);
+	} else if (of_node_name_prefix(pdev->dev.of_node, "i2s1")) {
+		i2s_drvdata->i2s_div_base = devm_ioremap(i2s_drvdata->dev, VO_TOP_CSR + VO_I2S1_DIV_NUM, 4);
+		if (!i2s_drvdata->i2s_div_base) {
+			dev_err(&pdev->dev, "failed to remap i2s1 div config\n");
+			ret = -ENOMEM;
+			goto err_probe;
+		}
+		ret = devm_snd_soc_register_component(&pdev->dev, &i2s_component,
+					&i2s_dai[2], 1);
+	} else {
+		i2s_drvdata->i2s_div_base = devm_ioremap(i2s_drvdata->dev, VO_TOP_CSR + VO_I2S2_DIV_NUM, 4);
+		if (!i2s_drvdata->i2s_div_base) {
+			dev_err(&pdev->dev, "failed to remap i2s2 div config\n");
+			ret = -ENOMEM;
+			goto err_probe;
+		}
+		ret = devm_snd_soc_register_component(&pdev->dev, &i2s_component,
+					&i2s_dai[3], 1);
+	}
+	if (ret != 0) {
+		dev_err(&pdev->dev, "not able to register dai\n");
+		goto err_probe;
+	}
+
+	i2s_drvdata->use_pio = false;
+	ret = devm_snd_dmaengine_pcm_register(&pdev->dev, config, 0);
+	if (ret) {
+		dev_err(&pdev->dev, "could not register pcm: %d\n", ret);
+		goto err_probe;
+	}
+
+	i2s_drvdata->i2s_reg_comp1 = I2S_COMP_PARAM_1;
+	i2s_drvdata->i2s_reg_comp2 = I2S_COMP_PARAM_2;
+	ret = i2s_configure_res_by_dt(i2s_drvdata, res);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "i2s_configure_res_by_dt failed\n");
+		goto err_probe;
+	}
+
+	pm_runtime_enable(&pdev->dev);
+
+	audio_proc_module_init();
+
+#ifdef CONFIG_PM
+	clk_disable(i2s_drvdata->clk);
+#endif
+	return 0;
+err_probe:
+	clk_disable_unprepare(i2s_drvdata->clk);
+
+	return ret;
+}
+
+static int i2s_remove(struct platform_device *pdev)
+{
+	struct i2s_dev *i2s_drvdata = dev_get_drvdata(&pdev->dev);
+
+	clk_disable_unprepare(i2s_drvdata->clk);
+
+	pm_runtime_disable(&pdev->dev);
+
+	audio_proc_module_exit();
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id i2s_of_match[] = {
+	{ .compatible = "snps,i2s", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, i2s_of_match);
+#endif
+
+static const struct dev_pm_ops i2s_pm_ops = {
+	SET_RUNTIME_PM_OPS(i2s_runtime_suspend, i2s_runtime_resume, NULL)
+};
+
+static struct platform_driver i2s_driver = {
+	.probe		= i2s_probe,
+	.remove		= i2s_remove,
+	.driver		= {
+		.name	= "i2s",
+		.of_match_table = of_match_ptr(i2s_of_match),
+		.pm = &i2s_pm_ops,
+	},
+};
+
+module_platform_driver(i2s_driver);
+
+MODULE_AUTHOR("ESWIN, INC.");
+MODULE_DESCRIPTION("I2S driver");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/eswin/esw-i2s.h b/sound/soc/eswin/esw-i2s.h
new file mode 100644
index 000000000000..6c1a243223a4
--- /dev/null
+++ b/sound/soc/eswin/esw-i2s.h
@@ -0,0 +1,180 @@
+/*
+ * Copyright (ST) 2012 Rajeev Kumar (rajeevkumar.linux@gmail.com)
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+/*
+ *
+ * Copyright (C) 2021 ESWIN, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#ifndef __I2S_H
+#define __I2S_H
+
+#include <linux/device.h>
+#include <sound/soc.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <linux/clk.h>
+#include <linux/types.h>
+#include <sound/dmaengine_pcm.h>
+#include <sound/designware_i2s.h>
+
+
+/* common register for all channel */
+#define IER		0x000
+#define IRER		0x004
+#define ITER		0x008
+#define CER		0x00C
+#define CCR		0x010
+#define RXFFR		0x014
+#define TXFFR		0x018
+
+/* DMA Control Register Offset */
+#define DMACR       	0x200
+/* DMA Control Register fields */
+#define DMAEN_TXBLOCK	BIT(17)
+#define DMAEN_RXBLOCK   BIT(16)
+#define DMAEN_TXCH_3    BIT(11)
+#define DMAEN_TXCH_2    BIT(10)
+#define DMAEN_TXCH_1	BIT(9)
+#define DMAEN_TXCH_0    BIT(8)
+#define DMAEN_RXCH_3	BIT(3)
+#define DMAEN_RXCH_2    BIT(2)
+#define DMAEN_RXCH_1	BIT(1)
+#define DMAEN_RXCH_0    BIT(0)
+
+/* Interrupt status register fields */
+#define ISR_TXFO	BIT(5)
+#define ISR_TXFE	BIT(4)
+#define ISR_RXFO	BIT(1)
+#define ISR_RXDA	BIT(0)
+
+/* I2STxRxRegisters for all channels */
+#define LRBR_LTHR(x)	(0x40 * x + 0x020)
+#define RRBR_RTHR(x)	(0x40 * x + 0x024)
+#define RER(x)		(0x40 * x + 0x028)
+#define TER(x)		(0x40 * x + 0x02C)
+#define RCR(x)		(0x40 * x + 0x030)
+#define TCR(x)		(0x40 * x + 0x034)
+#define ISR(x)		(0x40 * x + 0x038)
+#define IMR(x)		(0x40 * x + 0x03C)
+#define ROR(x)		(0x40 * x + 0x040)
+#define TOR(x)		(0x40 * x + 0x044)
+#define RFCR(x)		(0x40 * x + 0x048)
+#define TFCR(x)		(0x40 * x + 0x04C)
+#define RFF(x)		(0x40 * x + 0x050)
+#define TFF(x)		(0x40 * x + 0x054)
+
+/* I2SCOMPRegisters */
+#define I2S_COMP_PARAM_2	0x01F0
+#define I2S_COMP_PARAM_1	0x01F4
+#define I2S_COMP_VERSION	0x01F8
+#define I2S_COMP_TYPE		0x01FC
+
+/* I2S DMA registers */
+#define RXDMA_CH(x)		(0x4 * x + 0x204)
+#define TXDMA_CH(x)		(0x4 * x + 0x214)
+
+/*
+ * Component parameter register fields - define the I2S block's
+ * configuration.
+ */
+#define	COMP1_TX_WORDSIZE_3(r)	(((r) & GENMASK(27, 25)) >> 25)
+#define	COMP1_TX_WORDSIZE_2(r)	(((r) & GENMASK(24, 22)) >> 22)
+#define	COMP1_TX_WORDSIZE_1(r)	(((r) & GENMASK(21, 19)) >> 19)
+#define	COMP1_TX_WORDSIZE_0(r)	(((r) & GENMASK(18, 16)) >> 16)
+#define	COMP1_TX_CHANNELS(r)	(((r) & GENMASK(10, 9)) >> 9)
+#define	COMP1_RX_CHANNELS(r)	(((r) & GENMASK(8, 7)) >> 7)
+#define	COMP1_RX_ENABLED(r)	(((r) & BIT(6)) >> 6)
+#define	COMP1_TX_ENABLED(r)	(((r) & BIT(5)) >> 5)
+#define	COMP1_MODE_EN(r)	(((r) & BIT(4)) >> 4)
+#define	COMP1_FIFO_DEPTH_GLOBAL(r)	(((r) & GENMASK(3, 2)) >> 2)
+#define	COMP1_APB_DATA_WIDTH(r)	(((r) & GENMASK(1, 0)) >> 0)
+#define	COMP2_RX_WORDSIZE_3(r)	(((r) & GENMASK(12, 10)) >> 10)
+#define	COMP2_RX_WORDSIZE_2(r)	(((r) & GENMASK(9, 7)) >> 7)
+#define	COMP2_RX_WORDSIZE_1(r)	(((r) & GENMASK(5, 3)) >> 3)
+#define	COMP2_RX_WORDSIZE_0(r)	(((r) & GENMASK(2, 0)) >> 0)
+
+/* Number of entries in WORDSIZE and DATA_WIDTH parameter registers */
+#define	COMP_MAX_WORDSIZE	(1 << 3)
+#define	COMP_MAX_DATA_WIDTH	(1 << 2)
+#define MAX_CHANNEL_NUM		2
+#define MIN_CHANNEL_NUM		2
+#define STEREO		0
+#define TDM		1
+
+#define CCR_SCLKG_POS 0
+#define CCR_WSS_POS   3
+
+enum {
+	CLOCK_CYCLES_16,
+	CLOCK_CYCLES_24,
+	CLOCK_CYCLES_32
+};
+
+enum {
+	NO_CLOCK_GATING,
+	GATE_CLOCK_CYCLES_12,
+	GATE_CLOCK_CYCLES_16,
+	GATE_CLOCK_CYCLES_20,
+	GATE_CLOCK_CYCLES_24
+};
+
+enum {
+	IGNORE_WORD_LENGTH,
+	RESOLUTION_12_BIT,
+	RESOLUTION_16_BIT,
+	RESOLUTION_20_BIT,
+	RESOLUTION_24_BIT,
+	RESOLUTION_32_BIT
+};
+
+struct i2s_dev {
+	void __iomem *i2s_base;
+	struct clk *clk;
+	struct device *dev;
+	unsigned int i2s_reg_comp1;
+	unsigned int i2s_reg_comp2;
+	unsigned int capability;
+	u32 fifo_th;
+	bool use_pio;
+	/* data related to DMA transfers b/w i2s and DMAC */
+	struct snd_dmaengine_dai_dma_data play_dma_data;
+	struct snd_dmaengine_dai_dma_data capture_dma_data;
+	struct i2s_clk_config_data config;
+	struct snd_pcm_substream __rcu *tx_substream;
+	struct snd_pcm_substream __rcu *rx_substream;
+	unsigned int (*tx_fn)(struct i2s_dev *dev,
+			struct snd_pcm_runtime *runtime, unsigned int tx_ptr,
+			bool *period_elapsed,int type);
+	unsigned int (*rx_fn)(struct i2s_dev *dev,
+			struct snd_pcm_runtime *runtime, unsigned int rx_ptr,
+			bool *period_elapsed,int type);
+	unsigned int tx_ptr;
+	unsigned int rx_ptr;
+	u32 xfer_resolution;
+	int active;
+	u32 ccr;
+	void __iomem *i2s_div_base;
+	u32 i2s_div_num;
+	bool playback_active;
+	bool capture_active;
+};
+
+#endif /* __I2S_H */
diff --git a/tools/perf/pmu-events/arch/riscv/mapfile.csv b/tools/perf/pmu-events/arch/riscv/mapfile.csv
index c61b3d6ef616..6328fd3f45ee 100644
--- a/tools/perf/pmu-events/arch/riscv/mapfile.csv
+++ b/tools/perf/pmu-events/arch/riscv/mapfile.csv
@@ -15,3 +15,4 @@
 #
 #MVENDORID-MARCHID-MIMPID,Version,Filename,EventType
 0x489-0x8000000000000007-0x[[:xdigit:]]+,v1,sifive/u74,core
+0x489-0x8000000000000008-0x[[:xdigit:]]+,v1,sifive/p550,core
diff --git a/tools/perf/pmu-events/arch/riscv/sifive/p550/cycle-and-instruction-count.json b/tools/perf/pmu-events/arch/riscv/sifive/p550/cycle-and-instruction-count.json
new file mode 100644
index 000000000000..b530032994d9
--- /dev/null
+++ b/tools/perf/pmu-events/arch/riscv/sifive/p550/cycle-and-instruction-count.json
@@ -0,0 +1,12 @@
+[
+	{
+		"EventName": "CORE_CLOCK_CYCLES",
+		"EventCode": "0x165",
+		"BriefDescription": "Counts core clock cycles"
+	},
+	{
+		"EventName": "INSTRUCTIONS_RETIRED",
+		"EventCode": "0x265",
+		"BriefDescription": "Counts instructions retired"
+	}
+]
diff --git a/tools/perf/pmu-events/arch/riscv/sifive/p550/firmware.json b/tools/perf/pmu-events/arch/riscv/sifive/p550/firmware.json
new file mode 100644
index 000000000000..9b4a032186a7
--- /dev/null
+++ b/tools/perf/pmu-events/arch/riscv/sifive/p550/firmware.json
@@ -0,0 +1,68 @@
+[
+  {
+    "ArchStdEvent": "FW_MISALIGNED_LOAD"
+  },
+  {
+    "ArchStdEvent": "FW_MISALIGNED_STORE"
+  },
+  {
+    "ArchStdEvent": "FW_ACCESS_LOAD"
+  },
+  {
+    "ArchStdEvent": "FW_ACCESS_STORE"
+  },
+  {
+    "ArchStdEvent": "FW_ILLEGAL_INSN"
+  },
+  {
+    "ArchStdEvent": "FW_SET_TIMER"
+  },
+  {
+    "ArchStdEvent": "FW_IPI_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_IPI_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_FENCE_I_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_FENCE_I_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_ASID_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_VMID_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_VMID_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_ASID_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_ASID_RECEIVED"
+  }
+]
diff --git a/tools/perf/pmu-events/arch/riscv/sifive/p550/instruction.json b/tools/perf/pmu-events/arch/riscv/sifive/p550/instruction.json
new file mode 100644
index 000000000000..284e4c1566e0
--- /dev/null
+++ b/tools/perf/pmu-events/arch/riscv/sifive/p550/instruction.json
@@ -0,0 +1,92 @@
+[
+  {
+    "EventName": "EXCEPTION_TAKEN",
+    "EventCode": "0x100",
+    "BriefDescription": "Counts exceptions taken"
+  },
+  {
+    "EventName": "INTEGER_LOAD_RETIRED",
+    "EventCode": "0x200",
+    "BriefDescription": "Counts integer load instructions retired"
+  },
+  {
+    "EventName": "INTEGER_STORE_RETIRED",
+    "EventCode": "0x400",
+    "BriefDescription": "Counts integer store instructions retired"
+  },
+  {
+    "EventName": "ATOMIC_MEMORY_RETIRED",
+    "EventCode": "0x800",
+    "BriefDescription": "Counts atomic memory instructions retired"
+  },
+  {
+    "EventName": "SYSTEM_INSTRUCTION_RETIRED",
+    "EventCode": "0x1000",
+    "BriefDescription": "Counts system instructions retired (CSR, WFI, MRET, etc.)"
+  },
+  {
+    "EventName": "INTEGER_ARITHMETIC_RETIRED",
+    "EventCode": "0x2000",
+    "BriefDescription": "Counts integer arithmetic instructions retired"
+  },
+  {
+    "EventName": "CONDITIONAL_BRANCH_RETIRED",
+    "EventCode": "0x4000",
+    "BriefDescription": "Counts conditional branch instructions retired"
+  },
+  {
+    "EventName": "JAL_INSTRUCTION_RETIRED",
+    "EventCode": "0x8000",
+    "BriefDescription": "Counts jump-and-link instructions retired"
+  },
+  {
+    "EventName": "JALR_INSTRUCTION_RETIRED",
+    "EventCode": "0x10000",
+    "BriefDescription": "Counts indirect jump instructions (JALR) retired"
+  },
+  {
+    "EventName": "INTEGER_MULTIPLICATION_RETIRED",
+    "EventCode": "0x20000",
+    "BriefDescription": "Counts integer multiplication instructions retired"
+  },
+  {
+    "EventName": "INTEGER_DIVISION_RETIRED",
+    "EventCode": "0x40000",
+    "BriefDescription": "Counts integer division instructions retired"
+  },
+  {
+    "EventName": "FP_LOAD_RETIRED",
+    "EventCode": "0x80000",
+    "BriefDescription": "Counts floating-point load instructions retired"
+  },
+  {
+    "EventName": "FP_STORE_RETIRED",
+    "EventCode": "0x100000",
+    "BriefDescription": "Counts floating-point store instructions retired"
+  },
+  {
+    "EventName": "FP_ADD_RETIRED",
+    "EventCode": "0x200000",
+    "BriefDescription": "Counts floating-point add instructions retired"
+  },
+  {
+    "EventName": "FP_MUL_RETIRED",
+    "EventCode": "0x400000",
+    "BriefDescription": "Counts floating-point multiply instructions retired"
+  },
+  {
+    "EventName": "FP_MULADD_RETIRED",
+    "EventCode": "0x800000",
+    "BriefDescription": "Counts floating-point fused multiply-add instructions retired"
+  },
+  {
+    "EventName": "FP_DIV_SQRT_RETIRED",
+    "EventCode": "0x1000000",
+    "BriefDescription": "Counts floating point divide or square root instructions retired"
+  },
+  {
+    "EventName": "OTHER_FP_RETIRED",
+    "EventCode": "0x2000000",
+    "BriefDescription": "Counts other floating-point instructions retired"
+  }
+]
diff --git a/tools/perf/pmu-events/arch/riscv/sifive/p550/memory.json b/tools/perf/pmu-events/arch/riscv/sifive/p550/memory.json
new file mode 100644
index 000000000000..9351c9074ed7
--- /dev/null
+++ b/tools/perf/pmu-events/arch/riscv/sifive/p550/memory.json
@@ -0,0 +1,47 @@
+[
+	{
+		"EventName": "ICACHE_MISS",
+		"EventCode": "0x102",
+		"BriefDescription": "Counts instruction cache misses"
+	},
+	{
+		"EventName": "DCACHE_MISS",
+		"EventCode": "0x202",
+		"BriefDescription": "Counts data cache misses"
+	},
+	{
+		"EventName": "DCACHE_RELEASE",
+		"EventCode": "0x402",
+		"BriefDescription": "Counts writeback requests from the data cache"
+	},
+	{
+		"EventName": "ITLB_MISS",
+		"EventCode": "0x802",
+		"BriefDescription": "Counts Instruction TLB misses caused by instruction address translation requests"
+	},
+	{
+		"EventName": "DTLB_MISS",
+		"EventCode": "0x1002",
+		"BriefDescription": "Counts Data TLB misses caused by data address translation requests"
+	},
+	{
+		"EventName": "UTLB_MISS",
+		"EventCode": "0x2002",
+		"BriefDescription": "Counts Unified TLB misses caused by address translation requests"
+	},
+	{
+		"EventName": "UTLB_HIT",
+		"EventCode": "0x4002",
+		"BriefDescription": "Counts Unified TLB hits for address translation requests"
+	},
+	{
+		"EventName": "PTE_CACHE_MISS",
+		"EventCode": "0x8002",
+		"BriefDescription": "Counts Page Table Entry cache misses"
+	},
+	{
+		"EventName": "PTE_CACHE_HIT",
+		"EventCode": "0x10002",
+		"BriefDescription": "Counts Page Table Entry cache hits"
+	}
+]
diff --git a/tools/perf/pmu-events/arch/riscv/sifive/p550/microarch.json b/tools/perf/pmu-events/arch/riscv/sifive/p550/microarch.json
new file mode 100644
index 000000000000..d9cdb7d747ee
--- /dev/null
+++ b/tools/perf/pmu-events/arch/riscv/sifive/p550/microarch.json
@@ -0,0 +1,57 @@
+[
+  {
+    "EventName": "ADDRESSGEN_INTERLOCK",
+    "EventCode": "0x101",
+    "BriefDescription": "Counts cycles with an address-generation interlock"
+  },
+  {
+    "EventName": "LONGLATENCY_INTERLOCK",
+    "EventCode": "0x201",
+    "BriefDescription": "Counts cycles with a long-latency interlock"
+  },
+  {
+    "EventName": "CSR_INTERLOCK",
+    "EventCode": "0x401",
+    "BriefDescription": "Counts cycles with a CSR interlock"
+  },
+  {
+    "EventName": "ICACHE_BLOCKED",
+    "EventCode": "0x801",
+    "BriefDescription": "Counts cycles in which the instruction cache was not able to provide an instruction"
+  },
+  {
+    "EventName": "DCACHE_BLOCKED",
+    "EventCode": "0x1001",
+    "BriefDescription": "Counts cycles in which the data cache blocked an instruction"
+  },
+  {
+    "EventName": "BRANCH_DIRECTION_MISPREDICTION",
+    "EventCode": "0x2001",
+    "BriefDescription": "Counts mispredictions of conditional branch direction (taken/not taken)"
+  },
+  {
+    "EventName": "BRANCH_TARGET_MISPREDICTION",
+    "EventCode": "0x4001",
+    "BriefDescription": "Counts mispredictions of the target PC of control-flow instructions"
+  },
+  {
+    "EventName": "PIPELINE_FLUSH",
+    "EventCode": "0x8001",
+    "BriefDescription": "Counts flushes of the core pipeline. Common causes include fence.i and CSR accesses"
+  },
+  {
+    "EventName": "REPLAY",
+    "EventCode": "0x10001",
+    "BriefDescription": "Counts instruction replays"
+  },
+  {
+    "EventName": "INTEGER_MUL_DIV_INTERLOCK",
+    "EventCode": "0x20001",
+    "BriefDescription": "Counts cycles with a multiply or divide interlock"
+  },
+  {
+    "EventName": "FP_INTERLOCK",
+    "EventCode": "0x40001",
+    "BriefDescription": "Counts cycles with a floating-point interlock"
+  }
+]
